whispercpp 1.3.0 → 1.3.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (787) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +6 -0
  3. data/LICENSE +1 -1
  4. data/README.md +216 -424
  5. data/Rakefile +79 -11
  6. data/ext/.gitignore +11 -0
  7. data/ext/dependencies.rb +61 -0
  8. data/ext/extconf.rb +18 -26
  9. data/ext/options.rb +221 -0
  10. data/ext/ruby_whisper.c +159 -0
  11. data/ext/ruby_whisper.h +27 -2
  12. data/ext/ruby_whisper_context.c +641 -0
  13. data/ext/ruby_whisper_error.c +52 -0
  14. data/ext/ruby_whisper_model.c +232 -0
  15. data/ext/ruby_whisper_params.c +1301 -0
  16. data/ext/ruby_whisper_segment.c +143 -0
  17. data/ext/ruby_whisper_transcribe.cpp +87 -0
  18. data/ext/ruby_whisper_vad_params.c +288 -0
  19. data/ext/sources/.dockerignore +3 -0
  20. data/ext/sources/.github/workflows/bindings-ruby.yml +21 -0
  21. data/ext/sources/CMakeGraphVizOptions.cmake +8 -0
  22. data/ext/sources/CMakeLists.txt +251 -0
  23. data/ext/sources/bindings/javascript/CMakeLists.txt +41 -0
  24. data/ext/sources/bindings/javascript/emscripten.cpp +93 -0
  25. data/ext/sources/bindings/javascript/libwhisper.worker.js +1 -0
  26. data/ext/sources/bindings/javascript/package-tmpl.json +26 -0
  27. data/ext/sources/bindings/javascript/package.json +26 -0
  28. data/ext/sources/bindings/javascript/whisper.js +19 -0
  29. data/ext/sources/build-xcframework.sh +547 -0
  30. data/ext/sources/ci/run.sh +336 -0
  31. data/ext/sources/close-issue.yml +28 -0
  32. data/ext/sources/cmake/DefaultTargetOptions.cmake +16 -0
  33. data/ext/sources/cmake/FindFFmpeg.cmake +163 -0
  34. data/ext/sources/cmake/build-info.cmake +60 -0
  35. data/ext/sources/cmake/git-vars.cmake +22 -0
  36. data/ext/sources/cmake/whisper-config.cmake.in +65 -0
  37. data/ext/sources/cmake/whisper.pc.in +10 -0
  38. data/ext/sources/examples/CMakeLists.txt +124 -0
  39. data/ext/sources/examples/addon.node/CMakeLists.txt +31 -0
  40. data/ext/sources/examples/addon.node/__test__/whisper.spec.js +37 -0
  41. data/ext/sources/examples/addon.node/addon.cpp +438 -0
  42. data/ext/sources/examples/addon.node/index.js +54 -0
  43. data/ext/sources/examples/addon.node/package.json +16 -0
  44. data/ext/sources/examples/bench/CMakeLists.txt +8 -0
  45. data/ext/sources/examples/bench/bench.cpp +175 -0
  46. data/ext/sources/examples/bench.wasm/CMakeLists.txt +49 -0
  47. data/ext/sources/examples/bench.wasm/emscripten.cpp +87 -0
  48. data/ext/sources/examples/bench.wasm/index-tmpl.html +284 -0
  49. data/ext/sources/examples/cli/CMakeLists.txt +8 -0
  50. data/ext/sources/examples/cli/cli.cpp +1294 -0
  51. data/ext/sources/examples/coi-serviceworker.js +146 -0
  52. data/ext/sources/examples/command/CMakeLists.txt +10 -0
  53. data/ext/sources/examples/command/command.cpp +776 -0
  54. data/ext/sources/examples/command/commands.txt +9 -0
  55. data/ext/sources/examples/command.wasm/CMakeLists.txt +50 -0
  56. data/ext/sources/examples/command.wasm/emscripten.cpp +327 -0
  57. data/ext/sources/examples/command.wasm/index-tmpl.html +414 -0
  58. data/ext/sources/examples/common-ggml.cpp +238 -0
  59. data/ext/sources/examples/common-ggml.h +18 -0
  60. data/ext/sources/examples/common-sdl.cpp +227 -0
  61. data/ext/sources/examples/common-sdl.h +49 -0
  62. data/ext/sources/examples/common-whisper.cpp +168 -0
  63. data/ext/sources/examples/common-whisper.h +24 -0
  64. data/ext/sources/examples/common.cpp +675 -0
  65. data/ext/sources/examples/common.h +322 -0
  66. data/ext/sources/examples/deprecation-warning/CMakeLists.txt +6 -0
  67. data/ext/sources/examples/deprecation-warning/deprecation-warning.cpp +38 -0
  68. data/ext/sources/examples/ffmpeg-transcode.cpp +368 -0
  69. data/ext/sources/examples/generate-karaoke.sh +57 -0
  70. data/ext/sources/examples/grammar-parser.cpp +423 -0
  71. data/ext/sources/examples/grammar-parser.h +29 -0
  72. data/ext/sources/examples/helpers.js +191 -0
  73. data/ext/sources/examples/json.hpp +24596 -0
  74. data/ext/sources/examples/livestream.sh +112 -0
  75. data/ext/sources/examples/lsp/CMakeLists.txt +9 -0
  76. data/ext/sources/examples/lsp/lsp.cpp +467 -0
  77. data/ext/sources/examples/lsp/whisper.vim +362 -0
  78. data/ext/sources/examples/miniaudio.h +93468 -0
  79. data/ext/sources/examples/python/test_whisper_processor.py +7 -0
  80. data/ext/sources/examples/python/whisper_processor.py +54 -0
  81. data/ext/sources/examples/quantize/CMakeLists.txt +6 -0
  82. data/ext/sources/examples/quantize/quantize.cpp +223 -0
  83. data/ext/sources/examples/server/CMakeLists.txt +12 -0
  84. data/ext/sources/examples/server/bench.js +29 -0
  85. data/ext/sources/examples/server/httplib.h +10497 -0
  86. data/ext/sources/examples/server/server.cpp +1091 -0
  87. data/ext/sources/examples/server.py +115 -0
  88. data/ext/sources/examples/stb_vorbis.c +5584 -0
  89. data/ext/sources/examples/stream/CMakeLists.txt +10 -0
  90. data/ext/sources/examples/stream/stream.cpp +429 -0
  91. data/ext/sources/examples/stream.wasm/CMakeLists.txt +49 -0
  92. data/ext/sources/examples/stream.wasm/emscripten.cpp +216 -0
  93. data/ext/sources/examples/stream.wasm/index-tmpl.html +414 -0
  94. data/ext/sources/examples/sycl/CMakeLists.txt +9 -0
  95. data/ext/sources/examples/sycl/build.sh +22 -0
  96. data/ext/sources/examples/sycl/ls-sycl-device.cpp +11 -0
  97. data/ext/sources/examples/sycl/run-whisper.sh +17 -0
  98. data/ext/sources/examples/talk-llama/CMakeLists.txt +40 -0
  99. data/ext/sources/examples/talk-llama/eleven-labs.py +80 -0
  100. data/ext/sources/examples/talk-llama/llama-adapter.cpp +388 -0
  101. data/ext/sources/examples/talk-llama/llama-adapter.h +76 -0
  102. data/ext/sources/examples/talk-llama/llama-arch.cpp +1746 -0
  103. data/ext/sources/examples/talk-llama/llama-arch.h +437 -0
  104. data/ext/sources/examples/talk-llama/llama-batch.cpp +374 -0
  105. data/ext/sources/examples/talk-llama/llama-batch.h +89 -0
  106. data/ext/sources/examples/talk-llama/llama-chat.cpp +663 -0
  107. data/ext/sources/examples/talk-llama/llama-chat.h +58 -0
  108. data/ext/sources/examples/talk-llama/llama-context.cpp +2676 -0
  109. data/ext/sources/examples/talk-llama/llama-context.h +276 -0
  110. data/ext/sources/examples/talk-llama/llama-cparams.cpp +5 -0
  111. data/ext/sources/examples/talk-llama/llama-cparams.h +41 -0
  112. data/ext/sources/examples/talk-llama/llama-grammar.cpp +1229 -0
  113. data/ext/sources/examples/talk-llama/llama-grammar.h +173 -0
  114. data/ext/sources/examples/talk-llama/llama-graph.cpp +1618 -0
  115. data/ext/sources/examples/talk-llama/llama-graph.h +640 -0
  116. data/ext/sources/examples/talk-llama/llama-hparams.cpp +95 -0
  117. data/ext/sources/examples/talk-llama/llama-hparams.h +190 -0
  118. data/ext/sources/examples/talk-llama/llama-impl.cpp +167 -0
  119. data/ext/sources/examples/talk-llama/llama-impl.h +61 -0
  120. data/ext/sources/examples/talk-llama/llama-io.cpp +15 -0
  121. data/ext/sources/examples/talk-llama/llama-io.h +35 -0
  122. data/ext/sources/examples/talk-llama/llama-kv-cache.cpp +2739 -0
  123. data/ext/sources/examples/talk-llama/llama-kv-cache.h +502 -0
  124. data/ext/sources/examples/talk-llama/llama-kv-cells.h +379 -0
  125. data/ext/sources/examples/talk-llama/llama-memory.cpp +1 -0
  126. data/ext/sources/examples/talk-llama/llama-memory.h +32 -0
  127. data/ext/sources/examples/talk-llama/llama-mmap.cpp +600 -0
  128. data/ext/sources/examples/talk-llama/llama-mmap.h +68 -0
  129. data/ext/sources/examples/talk-llama/llama-model-loader.cpp +1138 -0
  130. data/ext/sources/examples/talk-llama/llama-model-loader.h +169 -0
  131. data/ext/sources/examples/talk-llama/llama-model-saver.cpp +281 -0
  132. data/ext/sources/examples/talk-llama/llama-model-saver.h +37 -0
  133. data/ext/sources/examples/talk-llama/llama-model.cpp +13814 -0
  134. data/ext/sources/examples/talk-llama/llama-model.h +425 -0
  135. data/ext/sources/examples/talk-llama/llama-quant.cpp +966 -0
  136. data/ext/sources/examples/talk-llama/llama-quant.h +1 -0
  137. data/ext/sources/examples/talk-llama/llama-sampling.cpp +2575 -0
  138. data/ext/sources/examples/talk-llama/llama-sampling.h +32 -0
  139. data/ext/sources/examples/talk-llama/llama-vocab.cpp +3340 -0
  140. data/ext/sources/examples/talk-llama/llama-vocab.h +131 -0
  141. data/ext/sources/examples/talk-llama/llama.cpp +354 -0
  142. data/ext/sources/examples/talk-llama/llama.h +1377 -0
  143. data/ext/sources/examples/talk-llama/prompts/talk-alpaca.txt +23 -0
  144. data/ext/sources/examples/talk-llama/speak +40 -0
  145. data/ext/sources/examples/talk-llama/speak.bat +1 -0
  146. data/ext/sources/examples/talk-llama/speak.ps1 +14 -0
  147. data/ext/sources/examples/talk-llama/talk-llama.cpp +808 -0
  148. data/ext/sources/examples/talk-llama/unicode-data.cpp +7034 -0
  149. data/ext/sources/examples/talk-llama/unicode-data.h +20 -0
  150. data/ext/sources/examples/talk-llama/unicode.cpp +849 -0
  151. data/ext/sources/examples/talk-llama/unicode.h +66 -0
  152. data/ext/sources/examples/vad-speech-segments/CMakeLists.txt +8 -0
  153. data/ext/sources/examples/vad-speech-segments/speech.cpp +143 -0
  154. data/ext/sources/examples/wchess/CMakeLists.txt +10 -0
  155. data/ext/sources/examples/wchess/libwchess/CMakeLists.txt +19 -0
  156. data/ext/sources/examples/wchess/libwchess/Chessboard.cpp +803 -0
  157. data/ext/sources/examples/wchess/libwchess/Chessboard.h +33 -0
  158. data/ext/sources/examples/wchess/libwchess/WChess.cpp +193 -0
  159. data/ext/sources/examples/wchess/libwchess/WChess.h +63 -0
  160. data/ext/sources/examples/wchess/libwchess/test-chessboard.cpp +117 -0
  161. data/ext/sources/examples/wchess/wchess.cmd/CMakeLists.txt +8 -0
  162. data/ext/sources/examples/wchess/wchess.cmd/wchess.cmd.cpp +249 -0
  163. data/ext/sources/examples/whisper.wasm/CMakeLists.txt +50 -0
  164. data/ext/sources/examples/whisper.wasm/emscripten.cpp +118 -0
  165. data/ext/sources/examples/whisper.wasm/index-tmpl.html +658 -0
  166. data/ext/sources/ggml/CMakeLists.txt +390 -0
  167. data/ext/sources/ggml/cmake/BuildTypes.cmake +54 -0
  168. data/ext/sources/ggml/cmake/GitVars.cmake +22 -0
  169. data/ext/sources/ggml/cmake/common.cmake +26 -0
  170. data/ext/sources/ggml/cmake/ggml-config.cmake.in +152 -0
  171. data/ext/sources/ggml/include/ggml-alloc.h +76 -0
  172. data/ext/sources/ggml/include/ggml-backend.h +354 -0
  173. data/ext/sources/ggml/include/ggml-blas.h +25 -0
  174. data/ext/sources/ggml/include/ggml-cann.h +123 -0
  175. data/ext/sources/ggml/include/ggml-cpp.h +39 -0
  176. data/ext/sources/ggml/include/ggml-cpu.h +143 -0
  177. data/ext/sources/ggml/include/ggml-cuda.h +47 -0
  178. data/ext/sources/ggml/include/ggml-kompute.h +50 -0
  179. data/ext/sources/ggml/include/ggml-metal.h +66 -0
  180. data/ext/sources/ggml/include/ggml-opencl.h +26 -0
  181. data/ext/sources/ggml/include/ggml-opt.h +237 -0
  182. data/ext/sources/ggml/include/ggml-rpc.h +33 -0
  183. data/ext/sources/ggml/include/ggml-sycl.h +49 -0
  184. data/ext/sources/ggml/include/ggml-vulkan.h +29 -0
  185. data/ext/{ggml.h → sources/ggml/include/ggml.h} +621 -821
  186. data/ext/sources/ggml/include/gguf.h +202 -0
  187. data/ext/sources/ggml/src/CMakeLists.txt +346 -0
  188. data/ext/sources/ggml/src/ggml-alloc.c +1042 -0
  189. data/ext/sources/ggml/src/ggml-amx/CMakeLists.txt +107 -0
  190. data/ext/sources/ggml/src/ggml-amx/common.h +94 -0
  191. data/ext/sources/ggml/src/ggml-amx/ggml-amx.cpp +446 -0
  192. data/ext/sources/ggml/src/ggml-amx/mmq.cpp +2510 -0
  193. data/ext/sources/ggml/src/ggml-amx/mmq.h +17 -0
  194. data/ext/sources/ggml/src/ggml-backend-impl.h +255 -0
  195. data/ext/sources/ggml/src/ggml-backend-reg.cpp +586 -0
  196. data/ext/sources/ggml/src/ggml-backend.cpp +2011 -0
  197. data/ext/sources/ggml/src/ggml-blas/CMakeLists.txt +87 -0
  198. data/ext/sources/ggml/src/ggml-blas/ggml-blas.cpp +517 -0
  199. data/ext/sources/ggml/src/ggml-cann/CMakeLists.txt +74 -0
  200. data/ext/sources/ggml/src/ggml-cann/Doxyfile +2579 -0
  201. data/ext/sources/ggml/src/ggml-cann/acl_tensor.cpp +181 -0
  202. data/ext/sources/ggml/src/ggml-cann/acl_tensor.h +258 -0
  203. data/ext/sources/ggml/src/ggml-cann/aclnn_ops.cpp +3193 -0
  204. data/ext/sources/ggml/src/ggml-cann/aclnn_ops.h +1125 -0
  205. data/ext/sources/ggml/src/ggml-cann/common.h +420 -0
  206. data/ext/sources/ggml/src/ggml-cann/ggml-cann.cpp +2606 -0
  207. data/ext/sources/ggml/src/ggml-cann/kernels/CMakeLists.txt +30 -0
  208. data/ext/sources/ggml/src/ggml-cann/kernels/ascendc_kernels.h +19 -0
  209. data/ext/sources/ggml/src/ggml-cann/kernels/dup.cpp +234 -0
  210. data/ext/sources/ggml/src/ggml-cann/kernels/get_row_f16.cpp +197 -0
  211. data/ext/sources/ggml/src/ggml-cann/kernels/get_row_f32.cpp +190 -0
  212. data/ext/sources/ggml/src/ggml-cann/kernels/get_row_q4_0.cpp +204 -0
  213. data/ext/sources/ggml/src/ggml-cann/kernels/get_row_q8_0.cpp +191 -0
  214. data/ext/sources/ggml/src/ggml-cann/kernels/quantize_f16_q8_0.cpp +218 -0
  215. data/ext/sources/ggml/src/ggml-cann/kernels/quantize_f32_q8_0.cpp +216 -0
  216. data/ext/sources/ggml/src/ggml-cann/kernels/quantize_float_to_q4_0.cpp +295 -0
  217. data/ext/sources/ggml/src/ggml-common.h +1857 -0
  218. data/ext/sources/ggml/src/ggml-cpu/CMakeLists.txt +504 -0
  219. data/ext/sources/ggml/src/ggml-cpu/amx/amx.cpp +221 -0
  220. data/ext/sources/ggml/src/ggml-cpu/amx/amx.h +8 -0
  221. data/ext/sources/ggml/src/ggml-cpu/amx/common.h +91 -0
  222. data/ext/sources/ggml/src/ggml-cpu/amx/mmq.cpp +2511 -0
  223. data/ext/sources/ggml/src/ggml-cpu/amx/mmq.h +10 -0
  224. data/ext/sources/ggml/src/ggml-cpu/binary-ops.cpp +158 -0
  225. data/ext/sources/ggml/src/ggml-cpu/binary-ops.h +16 -0
  226. data/ext/sources/ggml/src/ggml-cpu/cmake/FindSIMD.cmake +100 -0
  227. data/ext/sources/ggml/src/ggml-cpu/common.h +72 -0
  228. data/ext/sources/ggml/src/ggml-cpu/cpu-feats-x86.cpp +327 -0
  229. data/ext/sources/ggml/src/ggml-cpu/ggml-cpu-aarch64.cpp +6431 -0
  230. data/ext/sources/ggml/src/ggml-cpu/ggml-cpu-aarch64.h +8 -0
  231. data/ext/sources/ggml/src/ggml-cpu/ggml-cpu-hbm.cpp +55 -0
  232. data/ext/sources/ggml/src/ggml-cpu/ggml-cpu-hbm.h +8 -0
  233. data/ext/sources/ggml/src/ggml-cpu/ggml-cpu-impl.h +508 -0
  234. data/ext/sources/ggml/src/ggml-cpu/ggml-cpu-quants.c +13747 -0
  235. data/ext/sources/ggml/src/ggml-cpu/ggml-cpu-quants.h +63 -0
  236. data/ext/sources/ggml/src/ggml-cpu/ggml-cpu-traits.cpp +36 -0
  237. data/ext/sources/ggml/src/ggml-cpu/ggml-cpu-traits.h +38 -0
  238. data/ext/sources/ggml/src/ggml-cpu/ggml-cpu.c +3510 -0
  239. data/ext/sources/ggml/src/ggml-cpu/ggml-cpu.cpp +671 -0
  240. data/ext/sources/ggml/src/ggml-cpu/kleidiai/kernels.cpp +337 -0
  241. data/ext/sources/ggml/src/ggml-cpu/kleidiai/kernels.h +95 -0
  242. data/ext/sources/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp +482 -0
  243. data/ext/sources/ggml/src/ggml-cpu/kleidiai/kleidiai.h +17 -0
  244. data/ext/sources/ggml/src/ggml-cpu/llamafile/sgemm.cpp +3544 -0
  245. data/ext/sources/ggml/src/ggml-cpu/llamafile/sgemm.h +14 -0
  246. data/ext/sources/ggml/src/ggml-cpu/ops.cpp +8903 -0
  247. data/ext/sources/ggml/src/ggml-cpu/ops.h +110 -0
  248. data/ext/sources/ggml/src/ggml-cpu/simd-mappings.h +892 -0
  249. data/ext/sources/ggml/src/ggml-cpu/unary-ops.cpp +186 -0
  250. data/ext/sources/ggml/src/ggml-cpu/unary-ops.h +28 -0
  251. data/ext/sources/ggml/src/ggml-cpu/vec.cpp +252 -0
  252. data/ext/sources/ggml/src/ggml-cpu/vec.h +818 -0
  253. data/ext/sources/ggml/src/ggml-cuda/CMakeLists.txt +184 -0
  254. data/ext/sources/ggml/src/ggml-cuda/acc.cu +61 -0
  255. data/ext/sources/ggml/src/ggml-cuda/acc.cuh +5 -0
  256. data/ext/sources/ggml/src/ggml-cuda/arange.cu +34 -0
  257. data/ext/sources/ggml/src/ggml-cuda/arange.cuh +5 -0
  258. data/ext/sources/ggml/src/ggml-cuda/argmax.cu +91 -0
  259. data/ext/sources/ggml/src/ggml-cuda/argmax.cuh +3 -0
  260. data/ext/sources/ggml/src/ggml-cuda/argsort.cu +104 -0
  261. data/ext/sources/ggml/src/ggml-cuda/argsort.cuh +3 -0
  262. data/ext/sources/ggml/src/ggml-cuda/binbcast.cu +363 -0
  263. data/ext/sources/ggml/src/ggml-cuda/binbcast.cuh +9 -0
  264. data/ext/sources/ggml/src/ggml-cuda/clamp.cu +45 -0
  265. data/ext/sources/ggml/src/ggml-cuda/clamp.cuh +5 -0
  266. data/ext/sources/ggml/src/ggml-cuda/common.cuh +828 -0
  267. data/ext/sources/ggml/src/ggml-cuda/concat.cu +221 -0
  268. data/ext/sources/ggml/src/ggml-cuda/concat.cuh +5 -0
  269. data/ext/sources/ggml/src/ggml-cuda/conv-transpose-1d.cu +89 -0
  270. data/ext/sources/ggml/src/ggml-cuda/conv-transpose-1d.cuh +5 -0
  271. data/ext/sources/ggml/src/ggml-cuda/convert.cu +730 -0
  272. data/ext/sources/ggml/src/ggml-cuda/convert.cuh +26 -0
  273. data/ext/sources/ggml/src/ggml-cuda/count-equal.cu +64 -0
  274. data/ext/sources/ggml/src/ggml-cuda/count-equal.cuh +5 -0
  275. data/ext/sources/ggml/src/ggml-cuda/cp-async.cuh +57 -0
  276. data/ext/sources/ggml/src/ggml-cuda/cpy.cu +705 -0
  277. data/ext/sources/ggml/src/ggml-cuda/cpy.cuh +11 -0
  278. data/ext/sources/ggml/src/ggml-cuda/cross-entropy-loss.cu +189 -0
  279. data/ext/sources/ggml/src/ggml-cuda/cross-entropy-loss.cuh +7 -0
  280. data/ext/sources/ggml/src/ggml-cuda/dequantize.cuh +103 -0
  281. data/ext/sources/ggml/src/ggml-cuda/diagmask.cu +40 -0
  282. data/ext/sources/ggml/src/ggml-cuda/diagmask.cuh +5 -0
  283. data/ext/sources/ggml/src/ggml-cuda/fattn-common.cuh +881 -0
  284. data/ext/sources/ggml/src/ggml-cuda/fattn-mma-f16.cuh +1471 -0
  285. data/ext/sources/ggml/src/ggml-cuda/fattn-tile-f16.cu +357 -0
  286. data/ext/sources/ggml/src/ggml-cuda/fattn-tile-f16.cuh +3 -0
  287. data/ext/sources/ggml/src/ggml-cuda/fattn-tile-f32.cu +365 -0
  288. data/ext/sources/ggml/src/ggml-cuda/fattn-tile-f32.cuh +3 -0
  289. data/ext/sources/ggml/src/ggml-cuda/fattn-vec-f16.cuh +482 -0
  290. data/ext/sources/ggml/src/ggml-cuda/fattn-vec-f32.cuh +472 -0
  291. data/ext/sources/ggml/src/ggml-cuda/fattn-wmma-f16.cu +634 -0
  292. data/ext/sources/ggml/src/ggml-cuda/fattn-wmma-f16.cuh +3 -0
  293. data/ext/sources/ggml/src/ggml-cuda/fattn.cu +346 -0
  294. data/ext/sources/ggml/src/ggml-cuda/fattn.cuh +3 -0
  295. data/ext/sources/ggml/src/ggml-cuda/getrows.cu +275 -0
  296. data/ext/sources/ggml/src/ggml-cuda/getrows.cuh +15 -0
  297. data/ext/sources/ggml/src/ggml-cuda/ggml-cuda.cu +3505 -0
  298. data/ext/sources/ggml/src/ggml-cuda/gla.cu +93 -0
  299. data/ext/sources/ggml/src/ggml-cuda/gla.cuh +3 -0
  300. data/ext/sources/ggml/src/ggml-cuda/im2col.cu +103 -0
  301. data/ext/sources/ggml/src/ggml-cuda/im2col.cuh +5 -0
  302. data/ext/sources/ggml/src/ggml-cuda/mma.cuh +396 -0
  303. data/ext/sources/ggml/src/ggml-cuda/mmq.cu +324 -0
  304. data/ext/sources/ggml/src/ggml-cuda/mmq.cuh +3217 -0
  305. data/ext/sources/ggml/src/ggml-cuda/mmv.cu +336 -0
  306. data/ext/sources/ggml/src/ggml-cuda/mmv.cuh +12 -0
  307. data/ext/sources/ggml/src/ggml-cuda/mmvq.cu +595 -0
  308. data/ext/sources/ggml/src/ggml-cuda/mmvq.cuh +12 -0
  309. data/ext/sources/ggml/src/ggml-cuda/norm.cu +458 -0
  310. data/ext/sources/ggml/src/ggml-cuda/norm.cuh +11 -0
  311. data/ext/sources/ggml/src/ggml-cuda/opt-step-adamw.cu +78 -0
  312. data/ext/sources/ggml/src/ggml-cuda/opt-step-adamw.cuh +5 -0
  313. data/ext/sources/ggml/src/ggml-cuda/out-prod.cu +68 -0
  314. data/ext/sources/ggml/src/ggml-cuda/out-prod.cuh +3 -0
  315. data/ext/sources/ggml/src/ggml-cuda/pad.cu +49 -0
  316. data/ext/sources/ggml/src/ggml-cuda/pad.cuh +5 -0
  317. data/ext/sources/ggml/src/ggml-cuda/pool2d.cu +94 -0
  318. data/ext/sources/ggml/src/ggml-cuda/pool2d.cuh +5 -0
  319. data/ext/sources/ggml/src/ggml-cuda/quantize.cu +190 -0
  320. data/ext/sources/ggml/src/ggml-cuda/quantize.cuh +27 -0
  321. data/ext/sources/ggml/src/ggml-cuda/rope.cu +456 -0
  322. data/ext/sources/ggml/src/ggml-cuda/rope.cuh +7 -0
  323. data/ext/sources/ggml/src/ggml-cuda/scale.cu +31 -0
  324. data/ext/sources/ggml/src/ggml-cuda/scale.cuh +5 -0
  325. data/ext/sources/ggml/src/ggml-cuda/softmax.cu +283 -0
  326. data/ext/sources/ggml/src/ggml-cuda/softmax.cuh +7 -0
  327. data/ext/sources/ggml/src/ggml-cuda/ssm-conv.cu +148 -0
  328. data/ext/sources/ggml/src/ggml-cuda/ssm-conv.cuh +3 -0
  329. data/ext/sources/ggml/src/ggml-cuda/ssm-scan.cu +153 -0
  330. data/ext/sources/ggml/src/ggml-cuda/ssm-scan.cuh +3 -0
  331. data/ext/sources/ggml/src/ggml-cuda/sum.cu +45 -0
  332. data/ext/sources/ggml/src/ggml-cuda/sum.cuh +5 -0
  333. data/ext/sources/ggml/src/ggml-cuda/sumrows.cu +39 -0
  334. data/ext/sources/ggml/src/ggml-cuda/sumrows.cuh +5 -0
  335. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_16.cu +5 -0
  336. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_8.cu +10 -0
  337. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_1.cu +10 -0
  338. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_2.cu +10 -0
  339. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_4.cu +10 -0
  340. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_16.cu +5 -0
  341. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_4.cu +10 -0
  342. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_8.cu +10 -0
  343. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_1.cu +10 -0
  344. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_2.cu +10 -0
  345. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_16.cu +5 -0
  346. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_2.cu +10 -0
  347. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_4.cu +10 -0
  348. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_8.cu +10 -0
  349. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_64-ncols2_1.cu +10 -0
  350. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_1.cu +10 -0
  351. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_2.cu +10 -0
  352. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_4.cu +10 -0
  353. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_8.cu +10 -0
  354. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-f16.cu +5 -0
  355. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q4_0.cu +5 -0
  356. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q4_1.cu +5 -0
  357. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q5_0.cu +5 -0
  358. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q5_1.cu +5 -0
  359. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q8_0.cu +5 -0
  360. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-f16.cu +5 -0
  361. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q4_0.cu +5 -0
  362. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q4_1.cu +5 -0
  363. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q5_0.cu +5 -0
  364. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q5_1.cu +5 -0
  365. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q8_0.cu +5 -0
  366. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-f16.cu +5 -0
  367. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q4_0.cu +5 -0
  368. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q4_1.cu +5 -0
  369. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q5_0.cu +5 -0
  370. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q5_1.cu +5 -0
  371. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q8_0.cu +5 -0
  372. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-f16.cu +5 -0
  373. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q4_0.cu +5 -0
  374. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q4_1.cu +5 -0
  375. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q5_0.cu +5 -0
  376. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q5_1.cu +5 -0
  377. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q8_0.cu +5 -0
  378. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-f16.cu +5 -0
  379. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q4_0.cu +5 -0
  380. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q4_1.cu +5 -0
  381. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q5_0.cu +5 -0
  382. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q5_1.cu +5 -0
  383. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q8_0.cu +5 -0
  384. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-f16.cu +5 -0
  385. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q4_0.cu +5 -0
  386. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q4_1.cu +5 -0
  387. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q5_0.cu +5 -0
  388. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q5_1.cu +5 -0
  389. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q8_0.cu +5 -0
  390. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs256-f16-f16.cu +5 -0
  391. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-f16.cu +5 -0
  392. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q4_0.cu +5 -0
  393. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q4_1.cu +5 -0
  394. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q5_0.cu +5 -0
  395. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q5_1.cu +5 -0
  396. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q8_0.cu +5 -0
  397. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-f16.cu +5 -0
  398. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q4_0.cu +5 -0
  399. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q4_1.cu +5 -0
  400. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q5_0.cu +5 -0
  401. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q5_1.cu +5 -0
  402. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q8_0.cu +5 -0
  403. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-f16.cu +5 -0
  404. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q4_0.cu +5 -0
  405. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q4_1.cu +5 -0
  406. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q5_0.cu +5 -0
  407. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q5_1.cu +5 -0
  408. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q8_0.cu +5 -0
  409. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-f16.cu +5 -0
  410. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q4_0.cu +5 -0
  411. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q4_1.cu +5 -0
  412. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q5_0.cu +5 -0
  413. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q5_1.cu +5 -0
  414. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q8_0.cu +5 -0
  415. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-f16.cu +5 -0
  416. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q4_0.cu +5 -0
  417. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q4_1.cu +5 -0
  418. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q5_0.cu +5 -0
  419. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q5_1.cu +5 -0
  420. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q8_0.cu +5 -0
  421. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-f16.cu +5 -0
  422. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q4_0.cu +5 -0
  423. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q4_1.cu +5 -0
  424. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q5_0.cu +5 -0
  425. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q5_1.cu +5 -0
  426. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q8_0.cu +5 -0
  427. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-f16.cu +5 -0
  428. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q4_0.cu +5 -0
  429. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q4_1.cu +5 -0
  430. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q5_0.cu +5 -0
  431. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q5_1.cu +5 -0
  432. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q8_0.cu +5 -0
  433. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs256-f16-f16.cu +5 -0
  434. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-f16.cu +5 -0
  435. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q4_0.cu +5 -0
  436. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q4_1.cu +5 -0
  437. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q5_0.cu +5 -0
  438. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q5_1.cu +5 -0
  439. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q8_0.cu +5 -0
  440. data/ext/sources/ggml/src/ggml-cuda/template-instances/generate_cu_files.py +78 -0
  441. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq1_s.cu +5 -0
  442. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_s.cu +5 -0
  443. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xs.cu +5 -0
  444. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xxs.cu +5 -0
  445. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_s.cu +5 -0
  446. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_xxs.cu +5 -0
  447. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_nl.cu +5 -0
  448. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_xs.cu +5 -0
  449. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_k.cu +5 -0
  450. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q3_k.cu +5 -0
  451. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_0.cu +5 -0
  452. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_1.cu +5 -0
  453. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_k.cu +5 -0
  454. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_0.cu +5 -0
  455. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_1.cu +5 -0
  456. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_k.cu +5 -0
  457. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q6_k.cu +5 -0
  458. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q8_0.cu +5 -0
  459. data/ext/sources/ggml/src/ggml-cuda/tsembd.cu +47 -0
  460. data/ext/sources/ggml/src/ggml-cuda/tsembd.cuh +5 -0
  461. data/ext/sources/ggml/src/ggml-cuda/unary.cu +289 -0
  462. data/ext/sources/ggml/src/ggml-cuda/unary.cuh +59 -0
  463. data/ext/sources/ggml/src/ggml-cuda/upscale.cu +51 -0
  464. data/ext/sources/ggml/src/ggml-cuda/upscale.cuh +5 -0
  465. data/ext/sources/ggml/src/ggml-cuda/vecdotq.cuh +1135 -0
  466. data/ext/sources/ggml/src/ggml-cuda/vendors/cuda.h +15 -0
  467. data/ext/sources/ggml/src/ggml-cuda/vendors/hip.h +243 -0
  468. data/ext/sources/ggml/src/ggml-cuda/vendors/musa.h +140 -0
  469. data/ext/sources/ggml/src/ggml-cuda/wkv.cu +199 -0
  470. data/ext/sources/ggml/src/ggml-cuda/wkv.cuh +7 -0
  471. data/ext/sources/ggml/src/ggml-hip/CMakeLists.txt +131 -0
  472. data/ext/sources/ggml/src/ggml-impl.h +601 -0
  473. data/ext/sources/ggml/src/ggml-kompute/CMakeLists.txt +166 -0
  474. data/ext/sources/ggml/src/ggml-kompute/ggml-kompute.cpp +2251 -0
  475. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/common.comp +112 -0
  476. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_add.comp +58 -0
  477. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_addrow.comp +25 -0
  478. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_cpy_f16_f16.comp +52 -0
  479. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_cpy_f16_f32.comp +52 -0
  480. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_cpy_f32_f16.comp +52 -0
  481. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_cpy_f32_f32.comp +52 -0
  482. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_diagmask.comp +30 -0
  483. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_gelu.comp +22 -0
  484. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows.comp +17 -0
  485. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows_f16.comp +31 -0
  486. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows_f32.comp +31 -0
  487. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows_q4_0.comp +38 -0
  488. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows_q4_1.comp +39 -0
  489. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows_q6_k.comp +44 -0
  490. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul.comp +52 -0
  491. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_f16.comp +69 -0
  492. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_mat_f32.comp +51 -0
  493. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q4_0.comp +33 -0
  494. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q4_1.comp +35 -0
  495. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q4_k.comp +140 -0
  496. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q6_k.comp +106 -0
  497. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q8_0.comp +73 -0
  498. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mv_q_n.comp +52 -0
  499. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mv_q_n_pre.comp +28 -0
  500. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_norm.comp +84 -0
  501. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_relu.comp +21 -0
  502. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_rmsnorm.comp +53 -0
  503. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_rope_neox_f16.comp +52 -0
  504. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_rope_neox_f32.comp +52 -0
  505. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_rope_norm_f16.comp +52 -0
  506. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_rope_norm_f32.comp +52 -0
  507. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_scale.comp +19 -0
  508. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_scale_8.comp +23 -0
  509. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_silu.comp +22 -0
  510. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_softmax.comp +72 -0
  511. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/rope_common.comp +71 -0
  512. data/ext/sources/ggml/src/ggml-metal/CMakeLists.txt +120 -0
  513. data/ext/sources/ggml/src/ggml-metal/ggml-metal-impl.h +622 -0
  514. data/ext/sources/ggml/src/ggml-metal/ggml-metal.m +5998 -0
  515. data/ext/sources/ggml/src/ggml-metal/ggml-metal.metal +7089 -0
  516. data/ext/sources/ggml/src/ggml-musa/CMakeLists.txt +113 -0
  517. data/ext/sources/ggml/src/ggml-musa/mudnn.cu +112 -0
  518. data/ext/sources/ggml/src/ggml-musa/mudnn.cuh +12 -0
  519. data/ext/sources/ggml/src/ggml-opencl/CMakeLists.txt +96 -0
  520. data/ext/sources/ggml/src/ggml-opencl/ggml-opencl.cpp +5124 -0
  521. data/ext/sources/ggml/src/ggml-opencl/kernels/add.cl +83 -0
  522. data/ext/sources/ggml/src/ggml-opencl/kernels/clamp.cl +20 -0
  523. data/ext/sources/ggml/src/ggml-opencl/kernels/cpy.cl +184 -0
  524. data/ext/sources/ggml/src/ggml-opencl/kernels/cvt.cl +118 -0
  525. data/ext/sources/ggml/src/ggml-opencl/kernels/diag_mask_inf.cl +58 -0
  526. data/ext/sources/ggml/src/ggml-opencl/kernels/embed_kernel.py +26 -0
  527. data/ext/sources/ggml/src/ggml-opencl/kernels/gelu.cl +62 -0
  528. data/ext/sources/ggml/src/ggml-opencl/kernels/gemv_noshuffle.cl +268 -0
  529. data/ext/sources/ggml/src/ggml-opencl/kernels/gemv_noshuffle_general.cl +274 -0
  530. data/ext/sources/ggml/src/ggml-opencl/kernels/get_rows.cl +163 -0
  531. data/ext/sources/ggml/src/ggml-opencl/kernels/im2col_f16.cl +57 -0
  532. data/ext/sources/ggml/src/ggml-opencl/kernels/im2col_f32.cl +57 -0
  533. data/ext/sources/ggml/src/ggml-opencl/kernels/mul.cl +79 -0
  534. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mat_Ab_Bi_8x4.cl +139 -0
  535. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_f16_f16.cl +118 -0
  536. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32.cl +118 -0
  537. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_1row.cl +94 -0
  538. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_l4.cl +84 -0
  539. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_f32_f32.cl +118 -0
  540. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32.cl +192 -0
  541. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_16x_flat.cl +307 -0
  542. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_8x_flat.cl +265 -0
  543. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_8x_flat.cl +272 -0
  544. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_v.cl +254 -0
  545. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q6_k.cl +190 -0
  546. data/ext/sources/ggml/src/ggml-opencl/kernels/norm.cl +81 -0
  547. data/ext/sources/ggml/src/ggml-opencl/kernels/relu.cl +16 -0
  548. data/ext/sources/ggml/src/ggml-opencl/kernels/rms_norm.cl +96 -0
  549. data/ext/sources/ggml/src/ggml-opencl/kernels/rope.cl +721 -0
  550. data/ext/sources/ggml/src/ggml-opencl/kernels/scale.cl +16 -0
  551. data/ext/sources/ggml/src/ggml-opencl/kernels/silu.cl +30 -0
  552. data/ext/sources/ggml/src/ggml-opencl/kernels/softmax_4_f16.cl +87 -0
  553. data/ext/sources/ggml/src/ggml-opencl/kernels/softmax_4_f32.cl +87 -0
  554. data/ext/sources/ggml/src/ggml-opencl/kernels/softmax_f16.cl +86 -0
  555. data/ext/sources/ggml/src/ggml-opencl/kernels/softmax_f32.cl +86 -0
  556. data/ext/sources/ggml/src/ggml-opencl/kernels/transpose.cl +84 -0
  557. data/ext/sources/ggml/src/ggml-opt.cpp +1037 -0
  558. data/ext/sources/ggml/src/ggml-quants.c +5232 -0
  559. data/ext/sources/ggml/src/ggml-quants.h +100 -0
  560. data/ext/sources/ggml/src/ggml-rpc/CMakeLists.txt +9 -0
  561. data/ext/sources/ggml/src/ggml-rpc/ggml-rpc.cpp +1813 -0
  562. data/ext/sources/ggml/src/ggml-sycl/CMakeLists.txt +189 -0
  563. data/ext/sources/ggml/src/ggml-sycl/backend.hpp +37 -0
  564. data/ext/sources/ggml/src/ggml-sycl/binbcast.cpp +345 -0
  565. data/ext/sources/ggml/src/ggml-sycl/binbcast.hpp +39 -0
  566. data/ext/sources/ggml/src/ggml-sycl/common.cpp +83 -0
  567. data/ext/sources/ggml/src/ggml-sycl/common.hpp +589 -0
  568. data/ext/sources/ggml/src/ggml-sycl/concat.cpp +195 -0
  569. data/ext/sources/ggml/src/ggml-sycl/concat.hpp +20 -0
  570. data/ext/sources/ggml/src/ggml-sycl/conv.cpp +101 -0
  571. data/ext/sources/ggml/src/ggml-sycl/conv.hpp +20 -0
  572. data/ext/sources/ggml/src/ggml-sycl/convert.cpp +623 -0
  573. data/ext/sources/ggml/src/ggml-sycl/convert.hpp +34 -0
  574. data/ext/sources/ggml/src/ggml-sycl/cpy.cpp +700 -0
  575. data/ext/sources/ggml/src/ggml-sycl/cpy.hpp +11 -0
  576. data/ext/sources/ggml/src/ggml-sycl/dequantize.hpp +791 -0
  577. data/ext/sources/ggml/src/ggml-sycl/dmmv.cpp +1162 -0
  578. data/ext/sources/ggml/src/ggml-sycl/dmmv.hpp +27 -0
  579. data/ext/sources/ggml/src/ggml-sycl/dpct/helper.hpp +2957 -0
  580. data/ext/sources/ggml/src/ggml-sycl/element_wise.cpp +1511 -0
  581. data/ext/sources/ggml/src/ggml-sycl/element_wise.hpp +75 -0
  582. data/ext/sources/ggml/src/ggml-sycl/gemm.hpp +99 -0
  583. data/ext/sources/ggml/src/ggml-sycl/getrows.cpp +309 -0
  584. data/ext/sources/ggml/src/ggml-sycl/getrows.hpp +20 -0
  585. data/ext/sources/ggml/src/ggml-sycl/ggml-sycl.cpp +4493 -0
  586. data/ext/sources/ggml/src/ggml-sycl/gla.cpp +106 -0
  587. data/ext/sources/ggml/src/ggml-sycl/gla.hpp +8 -0
  588. data/ext/sources/ggml/src/ggml-sycl/im2col.cpp +136 -0
  589. data/ext/sources/ggml/src/ggml-sycl/im2col.hpp +21 -0
  590. data/ext/sources/ggml/src/ggml-sycl/mmq.cpp +3030 -0
  591. data/ext/sources/ggml/src/ggml-sycl/mmq.hpp +33 -0
  592. data/ext/sources/ggml/src/ggml-sycl/mmvq.cpp +1110 -0
  593. data/ext/sources/ggml/src/ggml-sycl/mmvq.hpp +27 -0
  594. data/ext/sources/ggml/src/ggml-sycl/norm.cpp +501 -0
  595. data/ext/sources/ggml/src/ggml-sycl/norm.hpp +26 -0
  596. data/ext/sources/ggml/src/ggml-sycl/outprod.cpp +47 -0
  597. data/ext/sources/ggml/src/ggml-sycl/outprod.hpp +10 -0
  598. data/ext/sources/ggml/src/ggml-sycl/presets.hpp +74 -0
  599. data/ext/sources/ggml/src/ggml-sycl/quants.hpp +83 -0
  600. data/ext/sources/ggml/src/ggml-sycl/rope.cpp +361 -0
  601. data/ext/sources/ggml/src/ggml-sycl/rope.hpp +20 -0
  602. data/ext/sources/ggml/src/ggml-sycl/softmax.cpp +261 -0
  603. data/ext/sources/ggml/src/ggml-sycl/softmax.hpp +20 -0
  604. data/ext/sources/ggml/src/ggml-sycl/sycl_hw.cpp +13 -0
  605. data/ext/sources/ggml/src/ggml-sycl/sycl_hw.hpp +23 -0
  606. data/ext/sources/ggml/src/ggml-sycl/tsembd.cpp +72 -0
  607. data/ext/sources/ggml/src/ggml-sycl/tsembd.hpp +20 -0
  608. data/ext/sources/ggml/src/ggml-sycl/vecdotq.hpp +1215 -0
  609. data/ext/sources/ggml/src/ggml-sycl/wkv.cpp +293 -0
  610. data/ext/sources/ggml/src/ggml-sycl/wkv.hpp +10 -0
  611. data/ext/sources/ggml/src/ggml-threading.cpp +12 -0
  612. data/ext/sources/ggml/src/ggml-threading.h +14 -0
  613. data/ext/sources/ggml/src/ggml-vulkan/CMakeLists.txt +196 -0
  614. data/ext/sources/ggml/src/ggml-vulkan/cmake/host-toolchain.cmake.in +15 -0
  615. data/ext/sources/ggml/src/ggml-vulkan/ggml-vulkan.cpp +10700 -0
  616. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/CMakeLists.txt +39 -0
  617. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/acc.comp +29 -0
  618. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/add.comp +29 -0
  619. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/argmax.comp +51 -0
  620. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/argsort.comp +69 -0
  621. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/clamp.comp +17 -0
  622. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/concat.comp +41 -0
  623. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/contig_copy.comp +49 -0
  624. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/conv2d_dw.comp +105 -0
  625. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/copy.comp +23 -0
  626. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/copy_from_quant.comp +51 -0
  627. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/copy_to_quant.comp +242 -0
  628. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/cos.comp +17 -0
  629. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/count_equal.comp +31 -0
  630. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_f32.comp +20 -0
  631. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.comp +462 -0
  632. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.comp +699 -0
  633. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_head.comp +13 -0
  634. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_m.comp +42 -0
  635. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_s.comp +35 -0
  636. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_s.comp +44 -0
  637. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xs.comp +43 -0
  638. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xxs.comp +48 -0
  639. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_s.comp +39 -0
  640. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_xxs.comp +49 -0
  641. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_nl.comp +32 -0
  642. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_xs.comp +34 -0
  643. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q2_k.comp +34 -0
  644. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q3_k.comp +42 -0
  645. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_0.comp +30 -0
  646. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_1.comp +32 -0
  647. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_k.comp +68 -0
  648. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_0.comp +34 -0
  649. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_1.comp +35 -0
  650. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_k.comp +70 -0
  651. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q6_k.comp +33 -0
  652. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q8_0.comp +31 -0
  653. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/diag_mask_inf.comp +34 -0
  654. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/div.comp +27 -0
  655. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp +337 -0
  656. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.comp +162 -0
  657. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp +360 -0
  658. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp +267 -0
  659. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_split_k_reduce.comp +59 -0
  660. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/gelu.comp +25 -0
  661. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/gelu_quick.comp +23 -0
  662. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/generic_binary_head.comp +64 -0
  663. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/generic_head.comp +9 -0
  664. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/generic_unary_head.comp +76 -0
  665. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/get_rows.comp +33 -0
  666. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_quant.comp +41 -0
  667. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/group_norm.comp +66 -0
  668. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/im2col.comp +100 -0
  669. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/l2_norm.comp +41 -0
  670. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/leaky_relu.comp +22 -0
  671. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul.comp +27 -0
  672. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_split_k_reduce.comp +48 -0
  673. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec.comp +169 -0
  674. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_base.comp +118 -0
  675. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_m.comp +82 -0
  676. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_s.comp +79 -0
  677. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_s.comp +90 -0
  678. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xs.comp +87 -0
  679. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xxs.comp +87 -0
  680. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_s.comp +90 -0
  681. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_xxs.comp +88 -0
  682. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_nc.comp +118 -0
  683. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_p021.comp +154 -0
  684. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q2_k.comp +130 -0
  685. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q3_k.comp +132 -0
  686. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q4_k.comp +136 -0
  687. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q5_k.comp +167 -0
  688. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q6_k.comp +130 -0
  689. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp +868 -0
  690. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp +441 -0
  691. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp +442 -0
  692. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_funcs.comp +99 -0
  693. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/norm.comp +44 -0
  694. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_adamw.comp +42 -0
  695. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/pad.comp +28 -0
  696. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/pool2d.comp +74 -0
  697. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/quantize_q8_1.comp +77 -0
  698. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/relu.comp +21 -0
  699. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/repeat.comp +26 -0
  700. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/repeat_back.comp +37 -0
  701. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp +52 -0
  702. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_back.comp +55 -0
  703. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rope_head.comp +58 -0
  704. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rope_multi.comp +60 -0
  705. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rope_neox.comp +43 -0
  706. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rope_norm.comp +43 -0
  707. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rope_vision.comp +47 -0
  708. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/scale.comp +24 -0
  709. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/sigmoid.comp +20 -0
  710. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/silu.comp +22 -0
  711. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/silu_back.comp +26 -0
  712. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/sin.comp +17 -0
  713. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/soft_max.comp +173 -0
  714. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_back.comp +50 -0
  715. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/square.comp +17 -0
  716. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/sub.comp +29 -0
  717. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.comp +37 -0
  718. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/tanh.comp +20 -0
  719. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/test_bfloat16_support.comp +7 -0
  720. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/test_coopmat2_support.comp +7 -0
  721. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/test_coopmat_support.comp +7 -0
  722. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/test_integer_dot_support.comp +7 -0
  723. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/timestep_embedding.comp +41 -0
  724. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/types.comp +1373 -0
  725. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/upscale.comp +36 -0
  726. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +751 -0
  727. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/wkv6.comp +87 -0
  728. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/wkv7.comp +91 -0
  729. data/ext/sources/ggml/src/ggml.c +6550 -0
  730. data/ext/sources/ggml/src/gguf.cpp +1330 -0
  731. data/ext/{whisper.h → sources/include/whisper.h} +91 -24
  732. data/ext/sources/src/CMakeLists.txt +143 -0
  733. data/ext/sources/src/coreml/whisper-decoder-impl.h +158 -0
  734. data/ext/sources/src/coreml/whisper-decoder-impl.m +226 -0
  735. data/ext/sources/src/coreml/whisper-encoder-impl.h +154 -0
  736. data/ext/sources/src/coreml/whisper-encoder-impl.m +222 -0
  737. data/ext/sources/src/coreml/whisper-encoder.h +26 -0
  738. data/ext/sources/src/coreml/whisper-encoder.mm +73 -0
  739. data/ext/sources/src/openvino/whisper-openvino-encoder.cpp +108 -0
  740. data/ext/sources/src/openvino/whisper-openvino-encoder.h +31 -0
  741. data/ext/sources/src/whisper-arch.h +197 -0
  742. data/ext/{whisper.cpp → sources/src/whisper.cpp} +2535 -835
  743. data/ext/sources/tests/CMakeLists.txt +105 -0
  744. data/ext/sources/tests/earnings21/eval.mk +58 -0
  745. data/ext/sources/tests/earnings21/eval.py +68 -0
  746. data/ext/sources/tests/earnings21/normalizers/__init__.py +2 -0
  747. data/ext/sources/tests/earnings21/normalizers/basic.py +80 -0
  748. data/ext/sources/tests/earnings21/normalizers/english.json +1741 -0
  749. data/ext/sources/tests/earnings21/normalizers/english.py +550 -0
  750. data/ext/sources/tests/earnings21/requirements.txt +6 -0
  751. data/ext/sources/tests/en-0-ref.txt +1 -0
  752. data/ext/sources/tests/en-1-ref.txt +1 -0
  753. data/ext/sources/tests/en-2-ref.txt +1 -0
  754. data/ext/sources/tests/es-0-ref.txt +1 -0
  755. data/ext/sources/tests/librispeech/eval.mk +39 -0
  756. data/ext/sources/tests/librispeech/eval.py +47 -0
  757. data/ext/sources/tests/librispeech/normalizers/__init__.py +2 -0
  758. data/ext/sources/tests/librispeech/normalizers/basic.py +80 -0
  759. data/ext/sources/tests/librispeech/normalizers/english.json +1741 -0
  760. data/ext/sources/tests/librispeech/normalizers/english.py +550 -0
  761. data/ext/sources/tests/librispeech/requirements.txt +6 -0
  762. data/ext/sources/tests/run-tests.sh +130 -0
  763. data/ext/sources/tests/test-c.c +3 -0
  764. data/ext/sources/tests/test-vad-full.cpp +54 -0
  765. data/ext/sources/tests/test-vad.cpp +83 -0
  766. data/ext/sources/tests/test-whisper.js +58 -0
  767. data/extsources.rb +34 -0
  768. data/lib/whisper/model/uri.rb +178 -0
  769. data/sig/whisper.rbs +480 -0
  770. data/tests/helper.rb +35 -0
  771. data/tests/jfk_reader/.gitignore +5 -0
  772. data/tests/jfk_reader/extconf.rb +3 -0
  773. data/tests/jfk_reader/jfk_reader.c +68 -0
  774. data/tests/test_callback.rb +202 -0
  775. data/tests/test_error.rb +20 -0
  776. data/tests/test_model.rb +109 -0
  777. data/tests/test_package.rb +46 -0
  778. data/tests/test_params.rb +297 -0
  779. data/tests/test_segment.rb +74 -0
  780. data/tests/test_vad.rb +19 -0
  781. data/tests/test_vad_params.rb +103 -0
  782. data/tests/test_whisper.rb +212 -124
  783. data/whispercpp.gemspec +37 -0
  784. metadata +794 -13
  785. data/ext/dr_wav.h +0 -6434
  786. data/ext/ggml.c +0 -21755
  787. data/ext/ruby_whisper.cpp +0 -426
@@ -0,0 +1,3193 @@
1
+ /*
2
+ * Copyright (c) 2023-2024 The ggml authors
3
+ *
4
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
5
+ * of this software and associated documentation files (the "Software"), to
6
+ * deal in the Software without restriction, including without limitation the
7
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
8
+ * sell copies of the Software, and to permit persons to whom the Software is
9
+ * furnished to do so, subject to the following conditions:
10
+ *
11
+ * The above copyright notice and this permission notice shall be included in
12
+ * all copies or substantial portions of the Software.
13
+ *
14
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
20
+ * IN THE SOFTWARE.
21
+ */
22
+
23
+ #include "aclnn_ops.h"
24
+
25
+ #include <aclnnop/aclnn_addcdiv.h>
26
+ #include <aclnnop/aclnn_avgpool2d.h>
27
+ #include <aclnnop/aclnn_batch_matmul.h>
28
+ #include <aclnnop/aclnn_cast.h>
29
+ #include <aclnnop/aclnn_constant_pad_nd.h>
30
+ #include <aclnnop/aclnn_copy.h>
31
+ #include <aclnnop/aclnn_div.h>
32
+ #include <aclnnop/aclnn_embedding.h>
33
+ #include <aclnnop/aclnn_exp.h>
34
+ #include <aclnnop/aclnn_fill_scalar.h>
35
+ #include <aclnnop/aclnn_group_norm.h>
36
+ #include <aclnnop/aclnn_index_fill_tensor.h>
37
+ #include <aclnnop/aclnn_layer_norm.h>
38
+ #include <aclnnop/aclnn_matmul.h>
39
+ #include <aclnnop/aclnn_max_pool.h>
40
+ #include <aclnnop/aclnn_mm.h>
41
+ #include <aclnnop/aclnn_permute.h>
42
+ #include <aclnnop/aclnn_pow_tensor_tensor.h>
43
+ #include <aclnnop/aclnn_reduce_sum.h>
44
+ #include <aclnnop/aclnn_repeat.h>
45
+ #include <aclnnop/aclnn_repeat_interleave.h>
46
+ #include <aclnnop/aclnn_roll.h>
47
+ #include <aclnnop/aclnn_softmax.h>
48
+ #include <aclnnop/aclnn_tril.h>
49
+ #include <aclnnop/aclnn_triu.h>
50
+ #include <aclnnop/aclnn_upsample_nearest_2d.h>
51
+ #include <aclnnop/aclnn_weight_quant_batch_matmul_v2.h>
52
+ #include <aclnnop/aclnn_argmax.h>
53
+ #include <aclnnop/aclnn_sum.h>
54
+ #include <aclnnop/aclnn_rms_norm.h>
55
+ #include <aclnnop/aclnn_im2col.h>
56
+ #include <aclnnop/aclnn_add.h>
57
+ #include <aclnnop/aclnn_sub.h>
58
+ #include <aclnnop/aclnn_mul.h>
59
+ #include <aclnnop/aclnn_div.h>
60
+ #include <aclnnop/aclnn_convolution.h>
61
+ #include <aclnnop/aclnn_elu.h>
62
+ #include <aclnnop/aclnn_log.h>
63
+ #include <aclnnop/aclnn_mean.h>
64
+ #include <aclnnop/aclnn_reflection_pad1d.h>
65
+ #include <aclnnop/aclnn_eq_tensor.h>
66
+ #include <aclnnop/aclnn_gt_scalar.h>
67
+ #include <aclnnop/aclnn_pow.h>
68
+ #include <aclnnop/aclnn_grouped_matmul_v2.h>
69
+ #include <aclnnop/aclnn_fused_infer_attention_score_v2.h>
70
+ #include <float.h>
71
+
72
+ #include <cmath>
73
+ #include <cstring>
74
+ #include <exception>
75
+ #include <vector>
76
+
77
+ #include "ggml-impl.h"
78
+ #include "ggml.h"
79
+
80
+ #define GGML_COMMON_DECL_C
81
+
82
+ #include "../ggml-common.h"
83
+
84
+
85
+ void bcast_shape(ggml_tensor * src0, ggml_tensor * src1, ggml_tensor * dst, aclTensor ** acl_src0,
86
+ aclTensor ** acl_src1, aclTensor ** acl_dst) {
87
+ GGML_ASSERT(ggml_are_same_shape(src0, dst) && ggml_can_repeat(src1, src0));
88
+ // Need bcast
89
+ if (!ggml_are_same_shape(src0, src1) && ggml_cann_need_bcast(src0, src1)) {
90
+ BCAST_SHAPE(src0, src1)
91
+ *acl_src0 = ggml_cann_create_tensor(src0, BCAST_PARAM(src0));
92
+ *acl_src1 = ggml_cann_create_tensor(src1, BCAST_PARAM(src1));
93
+ *acl_dst = ggml_cann_create_tensor(dst, BCAST_PARAM(src0));
94
+ } else {
95
+ *acl_src0 = ggml_cann_create_tensor(src0);
96
+ *acl_src1 = ggml_cann_create_tensor(src1);
97
+ *acl_dst = ggml_cann_create_tensor(dst);
98
+ }
99
+ }
100
+
101
+ void ggml_cann_unary_op(
102
+ std::function<void(ggml_backend_cann_context&, aclTensor*, aclTensor*)> unary_op,
103
+ ggml_backend_cann_context& ctx, ggml_tensor* dst) {
104
+ ggml_tensor* src = dst->src[0];
105
+
106
+ aclTensor* acl_src = ggml_cann_create_tensor(src);
107
+ aclTensor* acl_dst = ggml_cann_create_tensor(dst);
108
+
109
+ unary_op(ctx, acl_src, acl_dst);
110
+ ggml_cann_release_resources(ctx, acl_src, acl_dst);
111
+ }
112
+
113
+ /**
114
+ * @brief Repeats elements of a tensor along each dimension according to the
115
+ * specified repeat array.
116
+ *
117
+ * @param ctx The context for the CANN backend operations.
118
+ * @param acl_src The source tensor to be repeated.
119
+ * @param acl_dst The destination tensor after repeating.
120
+ * @param repeat_array The array specifying the number of repetitions along each
121
+ * dimension.
122
+ */
123
+ static void aclnn_repeat(ggml_backend_cann_context& ctx, aclTensor* acl_src,
124
+ aclTensor* acl_dst, int64_t* repeat_array) {
125
+ // repeat tensor along each dim with repeat_array
126
+ aclIntArray* repeats = aclCreateIntArray(repeat_array, GGML_MAX_DIMS);
127
+
128
+ GGML_CANN_CALL_ACLNN_OP(ctx, Repeat, acl_src, repeats, acl_dst);
129
+ ggml_cann_release_resources(ctx, repeats);
130
+ }
131
+
132
+ /**
133
+ * @brief Casts the data type of a source tensor to a destination tensor.
134
+ *
135
+ * This function casts the data type of the source tensor `acl_src` to the
136
+ * specified data type `cast_data_type` and stores the result in the destination
137
+ * tensor `acl_dst`.
138
+ *
139
+ * @param ctx The context for the CANN backend operations.
140
+ * @param acl_src The source tensor whose data type will be casted.
141
+ * @param acl_dst The destination tensor where the casted result will be stored.
142
+ * @param cast_data_type The target data type to which the source tensor will be
143
+ * casted.
144
+ */
145
+ static void aclnn_cast(ggml_backend_cann_context& ctx, aclTensor* acl_src,
146
+ aclTensor* acl_dst, aclDataType cast_data_type) {
147
+ GGML_CANN_CALL_ACLNN_OP(ctx, Cast, acl_src, cast_data_type, acl_dst);
148
+ }
149
+
150
+ void ggml_cann_repeat(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
151
+ ggml_tensor* src = dst->src[0];
152
+ GGML_ASSERT(ggml_can_repeat(src, dst));
153
+
154
+ aclTensor* acl_src = ggml_cann_create_tensor(src);
155
+ aclTensor* acl_dst = ggml_cann_create_tensor(dst);
156
+
157
+ int64_t repeatsArray[] = {dst->ne[3] / src->ne[3], dst->ne[2] / src->ne[2],
158
+ dst->ne[1] / src->ne[1], dst->ne[0] / src->ne[0]};
159
+
160
+ aclnn_repeat(ctx, acl_src, acl_dst, repeatsArray);
161
+ ggml_cann_release_resources(ctx, acl_src, acl_dst);
162
+ }
163
+
164
+ void aclnn_add(ggml_backend_cann_context& ctx, aclTensor* acl_src0,
165
+ aclTensor* acl_src1, aclTensor* acl_dst) {
166
+ float alphaValue = 1.0f;
167
+ aclScalar* alpha = aclCreateScalar(&alphaValue, aclDataType::ACL_FLOAT);
168
+ if (acl_dst != nullptr)
169
+ GGML_CANN_CALL_ACLNN_OP(ctx, Add, acl_src0, acl_src1, alpha, acl_dst);
170
+ else
171
+ GGML_CANN_CALL_ACLNN_OP(ctx, InplaceAdd, acl_src0, acl_src1, alpha);
172
+ ggml_cann_release_resources(ctx, alpha);
173
+ }
174
+
175
+ void aclnn_sub(ggml_backend_cann_context& ctx, aclTensor* acl_src0,
176
+ aclTensor* acl_src1, aclTensor* acl_dst) {
177
+ float alphaValue = 1.0f;
178
+ aclScalar* alpha = aclCreateScalar(&alphaValue, aclDataType::ACL_FLOAT);
179
+ if (acl_dst != nullptr)
180
+ GGML_CANN_CALL_ACLNN_OP(ctx, Sub, acl_src0, acl_src1, alpha, acl_dst);
181
+ else
182
+ GGML_CANN_CALL_ACLNN_OP(ctx, InplaceSub, acl_src0, acl_src1, alpha);
183
+ ggml_cann_release_resources(ctx, alpha);
184
+ }
185
+
186
+ void aclnn_mul(ggml_backend_cann_context& ctx, aclTensor* acl_src,
187
+ aclTensor* acl_other, aclTensor* acl_dst) {
188
+ if (acl_dst != nullptr)
189
+ GGML_CANN_CALL_ACLNN_OP(ctx, Mul, acl_src, acl_other, acl_dst);
190
+ else
191
+ GGML_CANN_CALL_ACLNN_OP(ctx, InplaceMul, acl_src, acl_other);
192
+ }
193
+
194
+ void aclnn_div(ggml_backend_cann_context& ctx, aclTensor* acl_src,
195
+ aclTensor* acl_other, aclTensor* acl_dst) {
196
+ if (acl_dst != nullptr)
197
+ GGML_CANN_CALL_ACLNN_OP(ctx, Div, acl_src, acl_other, acl_dst);
198
+ else
199
+ GGML_CANN_CALL_ACLNN_OP(ctx, InplaceDiv, acl_src, acl_other);
200
+ }
201
+
202
+ /**
203
+ * @brief Multiplies elements of a tensor by a scalar value, optionally
204
+ * in-place.
205
+ *
206
+ * This function multiplies each element of the source tensor `acl_src` by the
207
+ * scalar `scale` and stores the result in the destination tensor `acl_dst`. If
208
+ * `inplace` is true, `acl_dst` will not be used and the operation is performed
209
+ * in-place on `acl_src`.
210
+ * The operation is defined as:
211
+ * \f[
212
+ * \text {acl_dst }_i=\text {acl_src }_i \times \text {scale}
213
+ * \f]
214
+ *
215
+ * @param ctx The context for the CANN backend operations.
216
+ * @param acl_src The source tensor whose elements will be multiplied.
217
+ * @param scale The scalar value by which each element of `acl_src` will be
218
+ * multiplied.
219
+ * @param acl_dst The destination tensor where the result will be stored if
220
+ * `inplace` is false.
221
+ * @param inplace Flag indicating whether to perform the operation in-place on
222
+ * `acl_src`.
223
+ */
224
+ static void aclnn_muls(ggml_backend_cann_context& ctx, aclTensor* acl_src,
225
+ float scale, aclTensor* acl_dst, bool inplace) {
226
+ aclScalar* acl_scale = aclCreateScalar(&scale, aclDataType::ACL_FLOAT);
227
+ if (inplace) {
228
+ GGML_CANN_CALL_ACLNN_OP(ctx, InplaceMuls, acl_src, acl_scale);
229
+ } else {
230
+ GGML_CANN_CALL_ACLNN_OP(ctx, Muls, acl_src, acl_scale, acl_dst);
231
+ }
232
+ ggml_cann_release_resources(ctx, acl_scale);
233
+ }
234
+
235
+ void ggml_cann_leaky_relu(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
236
+ ggml_tensor* src = dst->src[0];
237
+
238
+ GGML_ASSERT(src->type == GGML_TYPE_F32);
239
+ GGML_ASSERT(dst->type == GGML_TYPE_F32);
240
+
241
+ aclTensor* acl_src = ggml_cann_create_tensor(src);
242
+ aclTensor* acl_dst = ggml_cann_create_tensor(dst);
243
+
244
+ float negative_slope;
245
+ memcpy(&negative_slope, dst->op_params, sizeof(float));
246
+ aclScalar* acl_negative_slope =
247
+ aclCreateScalar(&negative_slope, aclDataType::ACL_FLOAT);
248
+
249
+ GGML_CANN_CALL_ACLNN_OP(ctx, LeakyRelu, acl_src, acl_negative_slope, acl_dst);
250
+ ggml_cann_release_resources(ctx, acl_negative_slope, acl_src, acl_dst);
251
+ }
252
+
253
+ /**
254
+ * @brief Concatenates a list of tensors along a specified dimension and stores
255
+ * the result in a destination tensor.
256
+ *
257
+ * @param ctx The context for the CANN backend operations.
258
+ * @param tensorList The list of tensors to be concatenated.
259
+ * @param acl_dst The destination tensor where the concatenated result will be
260
+ * stored.
261
+ * @param concat_dim The dimension along which the tensors will be concatenated.
262
+ */
263
+ static void aclnn_concat(ggml_backend_cann_context& ctx,
264
+ aclTensorList* tensorList, aclTensor* acl_dst,
265
+ int64_t concat_dim) {
266
+ GGML_CANN_CALL_ACLNN_OP(ctx, Cat, tensorList, concat_dim, acl_dst);
267
+ }
268
+
269
+ void ggml_cann_concat(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
270
+ ggml_tensor* src0 = dst->src[0];
271
+ ggml_tensor* src1 = dst->src[1];
272
+ aclTensor* acl_src0 = ggml_cann_create_tensor(src0);
273
+ aclTensor* acl_src1 = ggml_cann_create_tensor(src1);
274
+ aclTensor* acl_dst = ggml_cann_create_tensor(dst);
275
+
276
+ const int32_t dim = ggml_get_op_params_i32(dst, 0);
277
+
278
+ GGML_ASSERT(dim >= 0 && dim < 4);
279
+ int32_t acl_dim = 3 - dim;
280
+
281
+ aclTensor* tensors[] = {acl_src0, acl_src1};
282
+ aclTensorList* tensor_list = aclCreateTensorList(tensors, 2);
283
+ aclnn_concat(ctx, tensor_list, acl_dst, acl_dim);
284
+
285
+ ggml_cann_release_resources(ctx, tensor_list, acl_dst);
286
+ }
287
+
288
+ /**
289
+ * @brief Creates a tensor with values starting from `start`, incremented by
290
+ * `step`, and ending before `stop`.
291
+ *
292
+ * This function performs the operation:
293
+ * \f[
294
+ * \text {out }_{i+1}=\text {out }_i+\text {step}
295
+ * \f]
296
+ * the range is [start, stop).
297
+ *
298
+ * @param ctx The context for the CANN backend operations.
299
+ * @param acl_dst The destination tensor where the values will be stored.
300
+ * @param start The starting value of the range.
301
+ * @param stop The ending value of the range (exclusive).
302
+ * @param step The step size between consecutive values.
303
+ * @param n_elements The number of elements in the destination tensor.
304
+ */
305
+ static void aclnn_arange(ggml_backend_cann_context& ctx, aclTensor* acl_dst,
306
+ float start, float stop, float step,
307
+ int64_t n_elements) {
308
+ int64_t steps = (int64_t)std::ceil((stop - start) / step);
309
+ GGML_ASSERT(n_elements == steps);
310
+
311
+ aclScalar* acl_start = aclCreateScalar(&start, aclDataType::ACL_FLOAT);
312
+ aclScalar* acl_end = aclCreateScalar(&stop, aclDataType::ACL_FLOAT);
313
+ aclScalar* acl_step = aclCreateScalar(&step, aclDataType::ACL_FLOAT);
314
+
315
+ GGML_CANN_CALL_ACLNN_OP(ctx, Arange, acl_start, acl_end, acl_step, acl_dst);
316
+ ggml_cann_release_resources(ctx, acl_start, acl_end, acl_step);
317
+ }
318
+
319
+ void ggml_cann_arange(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
320
+ GGML_ASSERT(dst->type == GGML_TYPE_F32);
321
+
322
+ aclTensor* acl_dst = ggml_cann_create_tensor(dst);
323
+
324
+ int64_t n_elements = ggml_nelements(dst);
325
+ float start;
326
+ float stop;
327
+ float step;
328
+ memcpy(&start, (float*)dst->op_params + 0, sizeof(float));
329
+ memcpy(&stop, (float*)dst->op_params + 1, sizeof(float));
330
+ memcpy(&step, (float*)dst->op_params + 2, sizeof(float));
331
+
332
+ aclnn_arange(ctx, acl_dst, start, stop, step, n_elements);
333
+ ggml_cann_release_resources(ctx, acl_dst);
334
+ }
335
+
336
+ void ggml_cann_clamp(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
337
+ ggml_tensor* src = dst->src[0];
338
+
339
+ float min;
340
+ float max;
341
+ memcpy(&min, dst->op_params, sizeof(float));
342
+ memcpy(&max, (float*)dst->op_params + 1, sizeof(float));
343
+
344
+ aclTensor* acl_src = ggml_cann_create_tensor(src);
345
+ aclTensor* acl_dst = ggml_cann_create_tensor(dst);
346
+
347
+ aclScalar* acl_min = aclCreateScalar(&min, aclDataType::ACL_FLOAT);
348
+ aclScalar* acl_max = aclCreateScalar(&max, aclDataType::ACL_FLOAT);
349
+
350
+ GGML_CANN_CALL_ACLNN_OP(ctx, Clamp, acl_src, acl_min, acl_max, acl_dst);
351
+ ggml_cann_release_resources(ctx, acl_min, acl_max, acl_src, acl_dst);
352
+ }
353
+
354
+ void ggml_cann_scale(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
355
+ ggml_tensor* src = dst->src[0];
356
+
357
+ // scale factor
358
+ float v;
359
+ memcpy(&v, dst->op_params, sizeof(float));
360
+
361
+ aclScalar* scale = aclCreateScalar(&v, aclDataType::ACL_FLOAT);
362
+ aclTensor* acl_src = ggml_cann_create_tensor(src);
363
+ aclTensor* acl_dst = ggml_cann_create_tensor(dst);
364
+
365
+ GGML_CANN_CALL_ACLNN_OP(ctx, Muls, acl_src, scale, acl_dst);
366
+ ggml_cann_release_resources(ctx, scale, acl_src, acl_dst);
367
+ }
368
+
369
+ void ggml_cann_argsort(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
370
+ ggml_tensor* src = dst->src[0];
371
+ enum ggml_sort_order order = (enum ggml_sort_order)dst->op_params[0];
372
+
373
+ aclTensor* acl_src = ggml_cann_create_tensor(src);
374
+ aclTensor* acl_dst = ggml_cann_create_tensor(dst);
375
+ ggml_cann_pool_alloc temp_buffer_allocator(
376
+ ctx.pool(), ggml_nelements(dst) * sizeof(int64_t));
377
+ void* buffer = temp_buffer_allocator.get();
378
+ aclTensor* tmp_tensor =
379
+ ggml_cann_create_tensor(buffer, ACL_INT64, ggml_type_size(dst->type),
380
+ dst->ne, dst->nb, GGML_MAX_DIMS);
381
+ GGML_CANN_CALL_ACLNN_OP(ctx, Argsort, acl_src, -1, (order == GGML_SORT_ORDER_DESC ? true : false),
382
+ tmp_tensor);
383
+ GGML_CANN_CALL_ACLNN_OP(ctx, Cast, tmp_tensor, ggml_cann_type_mapping(dst->type), acl_dst);
384
+ ggml_cann_release_resources(ctx, acl_src, tmp_tensor, acl_dst);
385
+ }
386
+
387
+ void ggml_cann_norm(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
388
+ ggml_tensor* src = dst->src[0];
389
+
390
+ aclTensor* acl_src = ggml_cann_create_tensor(src);
391
+ aclTensor* acl_dst = ggml_cann_create_tensor(dst);
392
+
393
+ float eps;
394
+ memcpy(&eps, dst->op_params, sizeof(float));
395
+
396
+ std::vector<int64_t> normData = {dst->ne[0]};
397
+ aclIntArray* norm = aclCreateIntArray(normData.data(), normData.size());
398
+ GGML_CANN_CALL_ACLNN_OP(ctx, LayerNorm, acl_src, norm, nullptr, nullptr,
399
+ eps, acl_dst, nullptr, nullptr);
400
+ ggml_cann_release_resources(ctx, norm, acl_src, acl_dst);
401
+ }
402
+
403
+ void ggml_cann_group_norm(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
404
+ ggml_tensor* src = dst->src[0];
405
+
406
+ aclTensor* acl_src = ggml_cann_create_tensor(src);
407
+ aclTensor* acl_dst = ggml_cann_create_tensor(dst);
408
+
409
+ int n_groups = dst->op_params[0];
410
+
411
+ float eps;
412
+ memcpy(&eps, dst->op_params + 1, sizeof(float));
413
+
414
+ int64_t N = src->ne[3];
415
+ int64_t C = src->ne[2];
416
+ int64_t HxW = src->ne[1] * src->ne[0];
417
+
418
+ size_t type_size = ggml_type_size(src->type);
419
+ int64_t ne[] = {n_groups, N};
420
+ size_t nb[] = {type_size, type_size * n_groups};
421
+ size_t n_bytes = N * n_groups;
422
+
423
+ ggml_cann_pool_alloc temp_buffer_allocator(ctx.pool(), n_bytes * 2);
424
+ void* buffer = temp_buffer_allocator.get();
425
+ aclTensor* acl_mean_out = ggml_cann_create_tensor(
426
+ buffer, ACL_FLOAT, type_size, ne, nb, ACL_FORMAT_ND);
427
+ aclTensor* acl_rstd_out = ggml_cann_create_tensor(
428
+ (char*)buffer + n_bytes, ACL_FLOAT, type_size, ne, nb, ACL_FORMAT_ND);
429
+
430
+ GGML_CANN_CALL_ACLNN_OP(ctx, GroupNorm, acl_src, nullptr, nullptr, N, C, HxW, n_groups, eps,
431
+ acl_dst, acl_mean_out, acl_rstd_out);
432
+ ggml_cann_release_resources(ctx, acl_src, acl_dst, acl_mean_out, acl_rstd_out);
433
+ }
434
+
435
+ void ggml_cann_acc(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
436
+ ggml_tensor* src0 = dst->src[0];
437
+ ggml_tensor* src1 = dst->src[1];
438
+
439
+ size_t nb1 = ((int32_t*)dst->op_params)[0];
440
+ size_t nb2 = ((int32_t*)dst->op_params)[1];
441
+ size_t nb3 = ((int32_t*)dst->op_params)[2];
442
+ size_t offset = ((int32_t*)dst->op_params)[3];
443
+ bool inplace = (bool)((int32_t*)dst->op_params)[4];
444
+
445
+ size_t param_nb[] = {ggml_element_size(src0), nb1, nb2, nb3};
446
+
447
+ aclTensor* acl_dst = ggml_cann_create_tensor(
448
+ dst, src1->ne, param_nb, GGML_MAX_DIMS, ACL_FORMAT_ND, offset);
449
+ aclTensor* acl_src1 = ggml_cann_create_tensor(src1);
450
+
451
+ aclScalar* alpha = nullptr;
452
+ float alphaValue = 1.0f;
453
+ alpha = aclCreateScalar(&alphaValue, aclDataType::ACL_FLOAT);
454
+
455
+ if (!inplace) {
456
+ size_t cpy_size = ggml_nbytes(dst);
457
+ ggml_cann_async_memcpy(ctx, dst->data, src0->data, cpy_size,
458
+ ACL_MEMCPY_DEVICE_TO_DEVICE);
459
+ aclTensor* acl_src0 = ggml_cann_create_tensor(
460
+ src0, src1->ne, src0->nb, GGML_MAX_DIMS, ACL_FORMAT_ND, offset);
461
+
462
+ GGML_CANN_CALL_ACLNN_OP(ctx, Add, acl_src0, acl_src1, alpha, acl_dst);
463
+ ggml_cann_release_resources(ctx, acl_src0);
464
+ } else {
465
+ GGML_CANN_CALL_ACLNN_OP(ctx, InplaceAdd, acl_dst, acl_src1, alpha);
466
+ }
467
+ ggml_cann_release_resources(ctx, acl_src1, acl_dst);
468
+ }
469
+
470
+ /**
471
+ * @brief Performs sum reduction on a given tensor along specified dimensions.
472
+ *
473
+ * This function reduces the input tensor by summing along the specified dimensions.
474
+ *
475
+ * @param ctx The context for the CANN backend operations.
476
+ * @param dst The destination tensor where the reduced result will be stored.
477
+ * @param dim An array of dimension indices.
478
+ * @param dim_size The number of dimensions.
479
+ */
480
+ static void aclnn_reduce_sum(ggml_backend_cann_context& ctx, ggml_tensor* dst,
481
+ int64_t* dim, size_t dim_size) {
482
+ GGML_ASSERT(dst->ne[0] == 1);
483
+ ggml_tensor* src = dst->src[0];
484
+ aclTensor* acl_src = ggml_cann_create_tensor(src);
485
+ aclTensor* acl_dst = ggml_cann_create_tensor(dst);
486
+ aclIntArray* reduce_dims = aclCreateIntArray(dim, dim_size);
487
+
488
+ GGML_CANN_CALL_ACLNN_OP(ctx, ReduceSum, acl_src, reduce_dims, true,
489
+ ggml_cann_type_mapping(dst->type), acl_dst);
490
+ ggml_cann_release_resources(ctx, acl_src, acl_dst, reduce_dims);
491
+ }
492
+
493
+ void ggml_cann_sum_rows(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
494
+ int64_t reduce_dims[] = {3};
495
+ aclnn_reduce_sum(ctx, dst, reduce_dims, 1);
496
+ }
497
+
498
+ void ggml_cann_sum(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
499
+ int64_t reduce_dims[] = {0, 1, 2, 3};
500
+ aclnn_reduce_sum(ctx, dst, reduce_dims, 4);
501
+ }
502
+
503
+ void ggml_cann_upsample_nearest2d(ggml_backend_cann_context& ctx,
504
+ ggml_tensor* dst) {
505
+ ggml_tensor* src = dst->src[0];
506
+ aclTensor* acl_src =
507
+ ggml_cann_create_tensor(src, nullptr, nullptr, 0, ACL_FORMAT_NCHW);
508
+ aclTensor* acl_dst =
509
+ ggml_cann_create_tensor(dst, nullptr, nullptr, 0, ACL_FORMAT_NCHW);
510
+
511
+ std::vector<int64_t> output_size{dst->ne[1], dst->ne[0]};
512
+ auto output_size_array = aclCreateIntArray(output_size.data(), 2);
513
+
514
+ GGML_CANN_CALL_ACLNN_OP(ctx, UpsampleNearest2d, acl_src, output_size_array, acl_dst);
515
+ ggml_cann_release_resources(ctx, acl_src, acl_dst, output_size_array);
516
+ }
517
+
518
+ /**
519
+ * @brief Pads a tensor with a specified value along each dimension.
520
+ *
521
+ * This function performs padding of the source tensor `acl_src` and stores the
522
+ * result in the destination tensor `acl_dst`. The padding values for each
523
+ * dimension are specified in the `paddings` array.
524
+ *
525
+ * @param ctx The context for the CANN backend operations.
526
+ * @param acl_src The source tensor to be padded.
527
+ * @param acl_dst The destination tensor where the padded result will be stored.
528
+ * @param paddings An array specifying the padding values for each dimension.
529
+ * The size of the array should be twice the number of dimensions of the tensor.
530
+ * @param value The value to be used for padding. The default value is 0.0.
531
+ */
532
+ static void aclnn_pad(ggml_backend_cann_context& ctx, aclTensor* acl_src,
533
+ aclTensor* acl_dst, int64_t* paddings,
534
+ float value = 0.0f) {
535
+ aclIntArray* acl_pad = aclCreateIntArray(paddings, GGML_MAX_DIMS * 2);
536
+ aclScalar* acl_value = aclCreateScalar(&value, aclDataType::ACL_FLOAT);
537
+
538
+ GGML_CANN_CALL_ACLNN_OP(ctx, ConstantPadNd, acl_src, acl_pad, acl_value, acl_dst);
539
+ ggml_cann_release_resources(ctx, acl_pad, acl_value);
540
+ }
541
+
542
+ void ggml_cann_pad(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
543
+ ggml_tensor* src = dst->src[0];
544
+ aclTensor* acl_src = ggml_cann_create_tensor(src);
545
+ aclTensor* acl_dst = ggml_cann_create_tensor(dst);
546
+
547
+ // padding: value in the array means how much distance will be padding.
548
+ // the position of elements in the array means which dirction to padding,
549
+ // each position means: [dim0.front, dim0.behind, dim1.front, dim1.behind,
550
+ // dim2.front, dim2.behind, dim3.front, dim3.behind]
551
+ int64_t paddings[] = {
552
+ 0, dst->ne[0] - src->ne[0], 0, dst->ne[1] - src->ne[1],
553
+ 0, dst->ne[2] - src->ne[2], 0, dst->ne[3] - src->ne[3]};
554
+ aclnn_pad(ctx, acl_src, acl_dst, paddings);
555
+ ggml_cann_release_resources(ctx, acl_src, acl_dst);
556
+ }
557
+
558
+ /**
559
+ * @brief Performs 2D average pooling on the input tensor and stores the result
560
+ * in the destination tensor.
561
+ *
562
+ * This function performs average pooling on the source tensor and stores the
563
+ * result in the destination tensor. The pooling parameters (kernel size,
564
+ * strides, padding) are specified in the `op_params` of the destination tensor.
565
+ *
566
+ * @param ctx The context for the CANN backend operations.
567
+ * @param dst The destination tensor where the result will be stored. The source
568
+ * tensor is referenced by `dst->src[0]`.
569
+ */
570
+ static void ggml_cann_avg_pool2d(ggml_backend_cann_context& ctx,
571
+ ggml_tensor* dst) {
572
+ ggml_tensor* src = dst->src[0];
573
+ GGML_ASSERT(src->type == GGML_TYPE_F32);
574
+ GGML_ASSERT(dst->type == GGML_TYPE_F32);
575
+
576
+ aclTensor* acl_src =
577
+ ggml_cann_create_tensor(src, nullptr, nullptr, 0, ACL_FORMAT_NCHW);
578
+ aclTensor* acl_dst =
579
+ ggml_cann_create_tensor(dst, nullptr, nullptr, 0, ACL_FORMAT_NCHW);
580
+
581
+ const int32_t* opts = (const int32_t*)dst->op_params;
582
+ const int k0 = opts[1];
583
+ const int k1 = opts[2];
584
+ const int s0 = opts[3];
585
+ const int s1 = opts[4];
586
+ const int p0 = opts[5];
587
+ const int p1 = opts[6];
588
+
589
+ std::vector<int64_t> kernel_dims = {k1, k0};
590
+ std::vector<int64_t> stride_dims = {s1, s0};
591
+ std::vector<int64_t> padding_avg_dims = {p1, p0}; // (padH, padW)
592
+
593
+ auto* kernel_size = aclCreateIntArray(kernel_dims.data(), 2);
594
+ auto* strides = aclCreateIntArray(stride_dims.data(), 2);
595
+ auto* paddings_avg = aclCreateIntArray(padding_avg_dims.data(), 2);
596
+
597
+ bool ceil_mode = false;
598
+ bool count_include_pad = true;
599
+ int64_t divisor_override = 0;
600
+ int8_t cube_math_type = 0;
601
+ #ifdef ASCEND_310P
602
+ cube_math_type = 1;
603
+ #endif
604
+
605
+ GGML_CANN_CALL_ACLNN_OP(ctx, AvgPool2d, acl_src, kernel_size, strides, paddings_avg,
606
+ ceil_mode, count_include_pad, divisor_override,
607
+ cube_math_type, acl_dst);
608
+ ggml_cann_release_resources(ctx, acl_src, acl_dst, kernel_size, strides,
609
+ paddings_avg);
610
+ }
611
+
612
+ /**
613
+ * @brief Performs 2D max pooling on the input tensor and stores the result in
614
+ * the destination tensor.
615
+ *
616
+ * This function performs max pooling on the source tensor and stores the result
617
+ * in the destination tensor. The pooling parameters (kernel size, strides,
618
+ * padding) are specified in the `op_params` of the destination tensor.
619
+ *
620
+ * @param ctx The context for the CANN backend operations.
621
+ * @param dst The destination tensor where the result will be stored. The source
622
+ * tensor is referenced by `dst->src[0]`.
623
+ */
624
+ static void ggml_cann_max_pool2d(ggml_backend_cann_context& ctx,
625
+ ggml_tensor* dst) {
626
+ ggml_tensor* src = dst->src[0];
627
+ GGML_ASSERT(src->type == GGML_TYPE_F32);
628
+ GGML_ASSERT(dst->type == GGML_TYPE_F32);
629
+
630
+ aclTensor* acl_src =
631
+ ggml_cann_create_tensor(src, nullptr, nullptr, 0, ACL_FORMAT_NCHW);
632
+ aclTensor* acl_dst =
633
+ ggml_cann_create_tensor(dst, nullptr, nullptr, 0, ACL_FORMAT_NCHW);
634
+
635
+ const int32_t* opts = (const int32_t*)dst->op_params;
636
+ const int k0 = opts[1];
637
+ const int k1 = opts[2];
638
+ const int s0 = opts[3];
639
+ const int s1 = opts[4];
640
+ const int p0 = opts[5];
641
+ const int p1 = opts[6];
642
+
643
+ int64_t temp_ne[] = {src->ne[0] + p0 * 2, src->ne[1] + p1 * 2, src->ne[2],
644
+ src->ne[3]};
645
+ size_t temp_nb[GGML_MAX_DIMS];
646
+
647
+ temp_nb[0] = ggml_element_size(src);
648
+ for (int i = 1; i < GGML_MAX_DIMS; i++) {
649
+ temp_nb[i] = temp_nb[i - 1] * temp_ne[i - 1];
650
+ }
651
+
652
+ ggml_cann_pool_alloc temp_buffer_allocator(
653
+ ctx.pool(), ggml_nbytes(src) + p0 * 2 + p1 * 2 * src->nb[1]);
654
+ void* buffer = temp_buffer_allocator.get();
655
+ aclTensor* tmp_tensor = ggml_cann_create_tensor(
656
+ buffer, ACL_FLOAT, ggml_element_size(src), temp_ne, temp_nb,
657
+ GGML_MAX_DIMS, ACL_FORMAT_NCHW);
658
+
659
+ // pad: see padding in ggml_cann_pad()
660
+ int64_t paddings[] = {p0, p0, p1, p1, 0, 0, 0, 0};
661
+ float value = -FLT_MAX;
662
+ aclnn_pad(ctx, acl_src, tmp_tensor, paddings, value);
663
+
664
+ // max_pool
665
+ std::vector<int64_t> kernel_dims = {k1, k0};
666
+ std::vector<int64_t> stride_dims = {s1, s0};
667
+ // padding_max_dims: [dim0_start, dim0_end, dim1_start, dim1_end]
668
+ std::vector<int64_t> padding_max_dims = {0, 0, 0, 0};
669
+ std::vector<int64_t> dilation_size = {1, 1};
670
+ auto* kernel_size = aclCreateIntArray(kernel_dims.data(), 2);
671
+ auto* strides = aclCreateIntArray(stride_dims.data(), 2);
672
+ auto* paddings_max = aclCreateIntArray(padding_max_dims.data(), 4);
673
+ auto* dilations = aclCreateIntArray(dilation_size.data(), 2);
674
+
675
+ bool ceil_mode = false;
676
+ int64_t auto_pads = 0;
677
+ GGML_CANN_CALL_ACLNN_OP(ctx, MaxPool, tmp_tensor, kernel_size, strides, auto_pads,
678
+ paddings_max, dilations, ceil_mode, acl_dst);
679
+ ggml_cann_release_resources(ctx, acl_src, acl_dst, tmp_tensor, kernel_size,
680
+ strides, paddings_max, dilations);
681
+ }
682
+
683
+ void ggml_cann_pool2d(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
684
+ const int32_t* opts = (const int32_t*)dst->op_params;
685
+ enum ggml_op_pool op = static_cast<ggml_op_pool>(opts[0]);
686
+ switch (op) {
687
+ case GGML_OP_POOL_AVG:
688
+ ggml_cann_avg_pool2d(ctx, dst);
689
+ break;
690
+ case GGML_OP_POOL_MAX:
691
+ ggml_cann_max_pool2d(ctx, dst);
692
+ break;
693
+ case GGML_OP_POOL_COUNT:
694
+ GGML_ABORT("fatal error");
695
+ break;
696
+ }
697
+ }
698
+
699
+ /**
700
+ * @brief Copies data from the source tensor to the destination tensor.
701
+ *
702
+ * This function copies data from the source tensor `acl_src` to the destination
703
+ * tensor `acl_dst`.
704
+ *
705
+ * @param ctx The context for the CANN backend operations.
706
+ * @param acl_src The source tensor from which data will be copied.
707
+ * @param acl_dst The destination tensor where the data will be copied to.
708
+ */
709
+ static void cann_copy(ggml_backend_cann_context& ctx, aclTensor* acl_src,
710
+ aclTensor* acl_dst) {
711
+ GGML_CANN_CALL_ACLNN_OP(ctx, InplaceCopy, acl_dst, acl_src);
712
+ }
713
+
714
+ void ggml_cann_dup(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
715
+ ggml_tensor* src0 = dst->src[0];
716
+
717
+ aclTensor* acl_src = ggml_cann_create_tensor(src0);
718
+ aclTensor* acl_dst = ggml_cann_create_tensor(dst);
719
+ if (ggml_are_same_shape(src0, dst)) {
720
+ if (dst->type == src0->type) {
721
+ cann_copy(ctx, acl_src, acl_dst);
722
+ } else {
723
+ aclnn_cast(ctx, acl_src, acl_dst, ggml_cann_type_mapping(dst->type));
724
+ }
725
+ } else {
726
+ if (ggml_is_contiguous(src0) && ggml_is_contiguous(dst)) {
727
+ if (dst->type == src0->type) {
728
+ size_t cpy_size = ggml_nbytes(dst);
729
+ ggml_cann_async_memcpy(ctx, dst->data, src0->data, cpy_size,
730
+ ACL_MEMCPY_DEVICE_TO_DEVICE);
731
+ return;
732
+ } else {
733
+ ggml_cann_pool_alloc src_buffer_allocator(
734
+ ctx.pool(),
735
+ ggml_nelements(dst) * ggml_type_size(dst->type));
736
+ void* src_trans_buffer = src_buffer_allocator.get();
737
+ size_t src_trans_nb[GGML_MAX_DIMS];
738
+ src_trans_nb[0] = ggml_type_size(dst->type);
739
+ for (int i = 1; i < GGML_MAX_DIMS; i++) {
740
+ src_trans_nb[i] = src_trans_nb[i - 1] * src0->ne[i - 1];
741
+ }
742
+ aclTensor* src_trans_tensor = ggml_cann_create_tensor(
743
+ src_trans_buffer, ggml_cann_type_mapping(dst->type),
744
+ ggml_type_size(dst->type), src0->ne, src_trans_nb,
745
+ GGML_MAX_DIMS);
746
+
747
+ aclnn_cast(ctx, acl_src, src_trans_tensor, ggml_cann_type_mapping(dst->type));
748
+ size_t cpy_size = ggml_nbytes(dst);
749
+ ggml_cann_async_memcpy(ctx, dst->data, src_trans_buffer, cpy_size,
750
+ ACL_MEMCPY_DEVICE_TO_DEVICE);
751
+ ggml_cann_release_resources(ctx, src_trans_tensor);
752
+ return;
753
+ }
754
+ } else if (ggml_is_contiguous(dst)) {
755
+ ggml_cann_pool_alloc src_buffer_allocator(
756
+ ctx.pool(), ggml_nelements(dst) * ggml_type_size(dst->type));
757
+ void* src_trans_buffer = src_buffer_allocator.get();
758
+ size_t src_trans_nb[GGML_MAX_DIMS];
759
+ src_trans_nb[0] = ggml_type_size(dst->type);
760
+ for (int i = 1; i < GGML_MAX_DIMS; i++) {
761
+ src_trans_nb[i] = src_trans_nb[i - 1] * src0->ne[i - 1];
762
+ }
763
+ aclTensor* src_trans_tensor = ggml_cann_create_tensor(
764
+ src_trans_buffer, ggml_cann_type_mapping(dst->type),
765
+ ggml_type_size(dst->type), src0->ne, src_trans_nb,
766
+ GGML_MAX_DIMS);
767
+
768
+ aclnn_cast(ctx, acl_src, src_trans_tensor, ggml_cann_type_mapping(dst->type));
769
+
770
+ size_t cpy_size = ggml_nbytes(dst);
771
+ ggml_cann_async_memcpy(ctx, dst->data, src_trans_buffer, cpy_size,
772
+ ACL_MEMCPY_DEVICE_TO_DEVICE);
773
+ ggml_cann_release_resources(ctx, src_trans_tensor);
774
+ return;
775
+ } else {
776
+ GGML_ABORT("Unsupport dst is not tontiguous.");
777
+ }
778
+ }
779
+ ggml_cann_release_resources(ctx, acl_src, acl_dst);
780
+ }
781
+
782
+ /**
783
+ * @brief Creates an ACL tensor initialized with zeros using a provided buffer.
784
+ *
785
+ * This function initializes a tensor with zeros using the specified buffer and
786
+ * tensor parameters.
787
+ *
788
+ * @param ctx The context for the CANN backend operations.
789
+ * @param buffer The buffer to be used for the tensor data.
790
+ * @param n_bytes The size of the buffer in bytes.
791
+ * @param ne An array specifying the extents (sizes) of each dimension of the
792
+ * tensor.
793
+ * @param dims The number of dimensions of the tensor.
794
+ * @param type The data type of the tensor.
795
+ * @param type_size The size of each element in the tensor data type.
796
+ * @return An ACL tensor initialized with zeros.
797
+ */
798
+ static aclTensor* aclnn_zero(ggml_backend_cann_context& ctx, void* buffer,
799
+ size_t n_bytes, int64_t* ne, int64_t dims,
800
+ aclDataType type, size_t type_size) {
801
+ size_t nb[GGML_MAX_DIMS];
802
+ nb[0] = type_size;
803
+ for (int i = 1; i < dims; i++) {
804
+ nb[i] = nb[i - 1] * ne[i - 1];
805
+ }
806
+
807
+ ggml_cann_async_memset(ctx, buffer, n_bytes, 0);
808
+ aclTensor* zero =
809
+ ggml_cann_create_tensor(buffer, type, type_size, ne, nb, dims);
810
+ return zero;
811
+ }
812
+
813
+ /**
814
+ * @brief Creates an ACL tensor initialized with value using a provided buffer.
815
+ *
816
+ * This function initializes a tensor with value using the specified buffer and
817
+ * tensor parameters.
818
+ *
819
+ * @param ctx The context for the CANN backend operations.
820
+ * @param buffer The buffer to be used for the tensor data.
821
+ * @param n_bytes The size of the buffer in bytes.
822
+ * @param ne An array specifying the extents (sizes) of each dimension of the
823
+ * tensor.
824
+ * @param dims The number of dimensions of the tensor.
825
+ * @param type The data type of the tensor.
826
+ * @param type_size The size of each element in the tensor data type.
827
+ * @param value The value to be used for initializing the tensor (default
828
+ * is 1.0).
829
+ * @return An ACL tensor initialized with value.
830
+ */
831
+ static aclTensor* aclnn_values(ggml_backend_cann_context& ctx, void* buffer,
832
+ size_t n_bytes, int64_t* ne, int64_t dims,
833
+ aclDataType type, size_t type_size,
834
+ float value = 1.0f) {
835
+ aclTensor* acl_tensor =
836
+ aclnn_zero(ctx, buffer, n_bytes, ne, dims, type, type_size);
837
+ float alpha_host = 1.0f;
838
+ aclScalar* alpha = aclCreateScalar(&alpha_host, aclDataType::ACL_FLOAT);
839
+ aclScalar* other = aclCreateScalar(&value, aclDataType::ACL_FLOAT);
840
+ GGML_CANN_CALL_ACLNN_OP(ctx, InplaceAdds, acl_tensor, other, alpha);
841
+ return acl_tensor;
842
+ }
843
+
844
+ void ggml_cann_rms_norm(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
845
+ ggml_tensor* src = dst->src[0];
846
+
847
+ aclTensor* acl_src = ggml_cann_create_tensor(src);
848
+ aclTensor* acl_dst = ggml_cann_create_tensor(dst);
849
+
850
+ float eps;
851
+ memcpy(&eps, dst->op_params, sizeof(float));
852
+ size_t one_tensor_n_bytes = src->ne[0] * ggml_element_size(src);
853
+ ggml_cann_pool_alloc one_tensor_allocator(ctx.pool(), one_tensor_n_bytes);
854
+
855
+ aclTensor* acl_gamma = aclnn_values(
856
+ ctx, one_tensor_allocator.get(), one_tensor_n_bytes, src->ne, 1,
857
+ ggml_cann_type_mapping(src->type), ggml_element_size(src));
858
+
859
+ size_t zero_tensor_n_bytes =
860
+ src->ne[1] * src->ne[2] * src->ne[3] * ggml_element_size(src);
861
+ ggml_cann_pool_alloc zero_tensor_allocator(ctx.pool(), zero_tensor_n_bytes);
862
+ aclTensor* acl_rstd =
863
+ aclnn_zero(ctx, zero_tensor_allocator.get(), zero_tensor_n_bytes,
864
+ src->ne, GGML_MAX_DIMS, ggml_cann_type_mapping(src->type),
865
+ ggml_element_size(src));
866
+ GGML_CANN_CALL_ACLNN_OP(ctx, RmsNorm, acl_src, acl_gamma, eps, acl_dst, acl_rstd);
867
+ ggml_cann_release_resources(ctx, acl_src, acl_dst, acl_gamma, acl_rstd);
868
+ }
869
+
870
+ // TODO: performace is low.
871
+ void ggml_cann_diag_mask(ggml_backend_cann_context& ctx, ggml_tensor* dst,
872
+ float value) {
873
+ ggml_tensor* src = dst->src[0];
874
+
875
+ aclTensor* acl_src = ggml_cann_create_tensor(src);
876
+ aclTensor* acl_dst = ggml_cann_create_tensor(dst);
877
+
878
+ const int n_past = ((int32_t*)dst->op_params)[0];
879
+
880
+ size_t one_tensor_n_bytes = src->ne[0] * src->ne[1] * src->ne[2] *
881
+ src->ne[3] * ggml_element_size(src);
882
+ ggml_cann_pool_alloc one_tensor_allocator(ctx.pool(), one_tensor_n_bytes);
883
+
884
+ aclTensor* mask_tensor =
885
+ aclnn_values(ctx, one_tensor_allocator.get(), one_tensor_n_bytes,
886
+ src->ne, GGML_MAX_DIMS, ggml_cann_type_mapping(src->type),
887
+ ggml_element_size(src), value);
888
+
889
+ aclScalar* alpha = nullptr;
890
+ float alphaValue = 1.0f;
891
+ alpha = aclCreateScalar(&alphaValue, aclDataType::ACL_FLOAT);
892
+
893
+ GGML_CANN_CALL_ACLNN_OP(ctx, InplaceTriu, mask_tensor, n_past + 1);
894
+ GGML_CANN_CALL_ACLNN_OP(ctx, Tril, acl_src, n_past + 1, acl_dst);
895
+ GGML_CANN_CALL_ACLNN_OP(ctx, InplaceAdd, acl_dst, mask_tensor, alpha);
896
+ ggml_cann_release_resources(ctx, alpha, acl_src, acl_dst, mask_tensor);
897
+ }
898
+
899
+ /**
900
+ * @brief Permutes the dimensions of a tensor according to a specified order.
901
+ *
902
+ * This function permutes the dimensions of the source tensor `acl_src`
903
+ * according to the order specified in the `new_dim` array and stores the result
904
+ * in the destination tensor `acl_dst`.
905
+ *
906
+ * @param ctx The context for the CANN backend operations.
907
+ * @param acl_src The source tensor whose dimensions will be permuted.
908
+ * @param acl_dst The destination tensor where the permuted result will be
909
+ * stored.
910
+ * @param new_dim An array specifying the new order of dimensions for the
911
+ * tensor.
912
+ * @param dims The number of dimensions in the tensor.
913
+ */
914
+ static void aclnn_permute(ggml_backend_cann_context& ctx, aclTensor* acl_src,
915
+ aclTensor* acl_dst, int64_t* new_dim, uint64_t dims) {
916
+ aclIntArray* acl_dims = aclCreateIntArray(new_dim, dims);
917
+ GGML_CANN_CALL_ACLNN_OP(ctx, Permute, acl_src, acl_dims, acl_dst);
918
+ ggml_cann_release_resources(ctx, acl_dims);
919
+ }
920
+
921
+ static void ggml_cann_im2col_2d_post_process(ggml_backend_cann_context& ctx,
922
+ ggml_tensor* dst,
923
+ ggml_tensor* src1,
924
+ aclTensor* tmp_cast_tensor,
925
+ aclTensor* tmp_im2col_tensor) {
926
+ // Permute: [N, IC * KH * KW, OW * OH] -> [N, OW * OH, IC * KH * KW]
927
+ int64_t dst_ne[] = {dst->ne[0], dst->ne[1] * dst->ne[2], dst->ne[3]};
928
+ size_t dst_nb[] = {dst->nb[0], dst->nb[1], dst->nb[3]};
929
+ aclTensor* acl_dst =
930
+ ggml_cann_create_tensor(dst, dst_ne, dst_nb, GGML_MAX_DIMS - 1);
931
+
932
+ int64_t permute_dim[] = {0, 2, 1};
933
+ if (src1->type != dst->type) {
934
+ aclnn_permute(ctx, tmp_cast_tensor, acl_dst, permute_dim, 3);
935
+ } else {
936
+ aclnn_permute(ctx, tmp_im2col_tensor, acl_dst, permute_dim, 3);
937
+ }
938
+
939
+ ggml_cann_release_resources(ctx, acl_dst);
940
+ }
941
+
942
+ static void ggml_cann_im2col_1d_post_process(
943
+ ggml_backend_cann_context& ctx, ggml_tensor* dst, ggml_tensor* src1,
944
+ aclTensor* tmp_cast_tensor, aclTensor* tmp_im2col_tensor,
945
+ const std::vector<int64_t>& im2col_op_params) {
946
+ // get params
947
+ const int64_t KH = im2col_op_params[0];
948
+ const int64_t KW = im2col_op_params[1];
949
+ const int64_t IW = im2col_op_params[2];
950
+ const int64_t IC = im2col_op_params[3];
951
+ const int64_t N = im2col_op_params[4];
952
+ const int64_t OH = im2col_op_params[5];
953
+ const int64_t OW = im2col_op_params[6];
954
+ const int64_t s0 = im2col_op_params[7];
955
+ const int64_t p0 = im2col_op_params[8];
956
+ const int64_t d0 = im2col_op_params[9];
957
+ const int64_t n_bytes_factor = im2col_op_params[10];
958
+
959
+ // Permute: [N, IC * KH * KW, OW * OH] ->
960
+ // [N, OW * OH * n_bytes_factor, IC * KH * KW]
961
+ ggml_cann_pool_alloc tmp_permute_allocator(ctx.pool());
962
+ tmp_permute_allocator.alloc(ggml_nbytes(dst) * n_bytes_factor);
963
+ void* tmp_permute_buffer = tmp_permute_allocator.get();
964
+
965
+ int64_t tmp_permute_ne[] = {IC * KH * KW, OW * OH * n_bytes_factor, N};
966
+ size_t tmp_permute_nb[GGML_MAX_DIMS - 1];
967
+ tmp_permute_nb[0] = ggml_type_size(dst->type);
968
+ for (int i = 1; i < GGML_MAX_DIMS - 1; i++) {
969
+ tmp_permute_nb[i] = tmp_permute_nb[i - 1] * tmp_permute_ne[i - 1];
970
+ }
971
+
972
+ aclTensor* tmp_permute_tensor = ggml_cann_create_tensor(
973
+ tmp_permute_buffer, ggml_cann_type_mapping(dst->type),
974
+ ggml_type_size(dst->type), tmp_permute_ne, tmp_permute_nb,
975
+ GGML_MAX_DIMS - 1, ACL_FORMAT_ND);
976
+
977
+ int64_t permute_dim[] = {0, 2, 1};
978
+ if (src1->type != dst->type) {
979
+ aclnn_permute(ctx, tmp_cast_tensor, tmp_permute_tensor, permute_dim, 3);
980
+ } else {
981
+ aclnn_permute(ctx, tmp_im2col_tensor, tmp_permute_tensor, permute_dim,
982
+ 3);
983
+ }
984
+
985
+ // number of times the kernel moves in W dimension
986
+ const int n_step_w = (IW + 2 * p0 - d0 * (KW - 1) - 1) / s0 + 1;
987
+ size_t offset;
988
+ void *cur_dst_buffer = dst->data, *cur_permute_buffer = tmp_permute_buffer;
989
+
990
+ // memory copy with offset to restore 1D im2col from 2d
991
+ if (IC > 1) {
992
+ offset = IC * KH * KW * n_step_w * ggml_type_size(dst->type);
993
+ size_t size_cpy = KH * KW * ggml_type_size(dst->type);
994
+
995
+ for (int c = 0; c < IC; c++) {
996
+ cur_permute_buffer = (char*)tmp_permute_buffer + offset +
997
+ KH * KW * c * ggml_type_size(dst->type);
998
+ cur_dst_buffer = (char*)dst->data +
999
+ c * KH * KW * n_step_w * ggml_type_size(dst->type);
1000
+
1001
+ for (int i = 0; i < n_step_w; i++) {
1002
+ ggml_cann_async_memcpy(ctx, cur_dst_buffer, cur_permute_buffer, size_cpy,
1003
+ ACL_MEMCPY_DEVICE_TO_DEVICE);
1004
+ cur_dst_buffer =
1005
+ (char*)cur_dst_buffer + KH * KW * ggml_type_size(dst->type);
1006
+ cur_permute_buffer = (char*)cur_permute_buffer +
1007
+ KH * KW * IC * ggml_type_size(dst->type);
1008
+ }
1009
+ }
1010
+ } else {
1011
+ offset = KH * KW * n_step_w *
1012
+ ggml_type_size(dst->type); // equal to ggml_nbytes(dst)
1013
+ ggml_cann_async_memcpy(ctx, dst->data, (char*)tmp_permute_buffer + offset, offset,
1014
+ ACL_MEMCPY_DEVICE_TO_DEVICE);
1015
+ }
1016
+
1017
+ ggml_cann_release_resources(ctx, tmp_permute_tensor);
1018
+ }
1019
+
1020
+ void ggml_cann_im2col(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
1021
+ ggml_tensor* src0 = dst->src[0]; // kernel
1022
+ ggml_tensor* src1 = dst->src[1]; // input
1023
+
1024
+ GGML_TENSOR_BINARY_OP_LOCALS;
1025
+
1026
+ // aclnnIm2col only works on 2D. set s1, p1, d1 to 1 to perform 2D
1027
+ // im2col and do post-processing to restore it to 1D.
1028
+ const bool is_2D = ((const int32_t*)(dst->op_params))[6] == 1;
1029
+ const int32_t s0 = ((const int32_t*)(dst->op_params))[0];
1030
+ const int32_t s1 = is_2D ? ((const int32_t*)(dst->op_params))[1] : 1;
1031
+ const int32_t p0 = ((const int32_t*)(dst->op_params))[2];
1032
+ const int32_t p1 = is_2D ? ((const int32_t*)(dst->op_params))[3] : 1;
1033
+ const int32_t d0 = ((const int32_t*)(dst->op_params))[4];
1034
+ const int32_t d1 = is_2D ? ((const int32_t*)(dst->op_params))[5] : 1;
1035
+
1036
+ const int64_t N = ne13;
1037
+ const int64_t IC = ne12;
1038
+ const int64_t KH = ne01;
1039
+ const int64_t KW = ne00;
1040
+ const int64_t IW = ne10;
1041
+
1042
+ const int64_t OH = is_2D ? ne2 : 1;
1043
+ const int64_t OW = ne1;
1044
+
1045
+ // memory allocated increased to 3x when is_2D == false
1046
+ const int64_t n_bytes_factor = is_2D ? 1 : 3;
1047
+
1048
+ // im2col: [N,C,H,W] -> [N, IC * KH * KW, OW * OH * n_bytes_factor]
1049
+ aclTensor* acl_src1 = ggml_cann_create_tensor(src1);
1050
+ int64_t tmp_im2col_ne[] = {OW * OH * n_bytes_factor, IC * KH * KW, N};
1051
+ size_t tmp_im2col_nb[GGML_MAX_DIMS - 1];
1052
+
1053
+ tmp_im2col_nb[0] = ggml_type_size(src1->type);
1054
+ for (int i = 1; i < GGML_MAX_DIMS - 1; i++) {
1055
+ tmp_im2col_nb[i] = tmp_im2col_nb[i - 1] * tmp_im2col_ne[i - 1];
1056
+ }
1057
+
1058
+ // Calculate im2col.
1059
+ // If dst is f16, tmp_buffer is f32, we need alloc src.typesize *
1060
+ // dst.elemcount.
1061
+ ggml_cann_pool_alloc im2col_allocator(
1062
+ ctx.pool(),
1063
+ ggml_nelements(dst) * ggml_element_size(src1) * n_bytes_factor);
1064
+ void* tmp_im2col_buffer = im2col_allocator.get();
1065
+
1066
+ aclTensor* tmp_im2col_tensor = ggml_cann_create_tensor(
1067
+ tmp_im2col_buffer, ggml_cann_type_mapping(src1->type),
1068
+ ggml_type_size(src1->type), tmp_im2col_ne, tmp_im2col_nb,
1069
+ GGML_MAX_DIMS - 1, ACL_FORMAT_ND);
1070
+
1071
+ std::vector<int64_t> kernel_dims = {KH, KW};
1072
+ std::vector<int64_t> dilation_size = {d1, d0};
1073
+ std::vector<int64_t> padding_dims = {p1, p0};
1074
+ std::vector<int64_t> stride_dims = {s1, s0};
1075
+ auto* kernel_size = aclCreateIntArray(kernel_dims.data(), 2);
1076
+ auto* dilations = aclCreateIntArray(dilation_size.data(), 2);
1077
+ auto* paddings = aclCreateIntArray(padding_dims.data(), 2);
1078
+ auto* strides = aclCreateIntArray(stride_dims.data(), 2);
1079
+ GGML_CANN_CALL_ACLNN_OP(ctx, Im2col, acl_src1, kernel_size, dilations,
1080
+ paddings, strides, tmp_im2col_tensor);
1081
+
1082
+ // Cast if dst is f16.
1083
+ aclTensor* tmp_cast_tensor = nullptr;
1084
+ ggml_cann_pool_alloc tmp_cast_allocator(ctx.pool());
1085
+ void* tmp_cast_buffer = nullptr;
1086
+ if (src1->type != dst->type) {
1087
+ tmp_cast_allocator.alloc(ggml_nbytes(dst) * n_bytes_factor);
1088
+ tmp_cast_buffer = tmp_cast_allocator.get();
1089
+ size_t temp_cast_nb[GGML_MAX_DIMS - 1];
1090
+ temp_cast_nb[0] = ggml_type_size(dst->type);
1091
+ for (int i = 1; i < GGML_MAX_DIMS - 1; i++) {
1092
+ temp_cast_nb[i] = temp_cast_nb[i - 1] * tmp_im2col_ne[i - 1];
1093
+ }
1094
+
1095
+ tmp_cast_tensor = ggml_cann_create_tensor(
1096
+ tmp_cast_buffer, ggml_cann_type_mapping(dst->type),
1097
+ ggml_type_size(dst->type), tmp_im2col_ne, temp_cast_nb,
1098
+ GGML_MAX_DIMS - 1, ACL_FORMAT_ND);
1099
+ aclnn_cast(ctx, tmp_im2col_tensor, tmp_cast_tensor, ggml_cann_type_mapping(dst->type));
1100
+ }
1101
+
1102
+ // post-processing
1103
+ if (is_2D) {
1104
+ ggml_cann_im2col_2d_post_process(ctx, dst, src1, tmp_cast_tensor,
1105
+ tmp_im2col_tensor);
1106
+ } else {
1107
+ std::vector<int64_t> im2col_op_params = {
1108
+ KH, KW, IW, IC, N, OH, OW, s0, p0, d0, n_bytes_factor};
1109
+ ggml_cann_im2col_1d_post_process(ctx, dst, src1, tmp_cast_tensor,
1110
+ tmp_im2col_tensor, im2col_op_params);
1111
+ }
1112
+
1113
+ ggml_cann_release_resources(ctx, acl_src1, tmp_im2col_tensor, tmp_cast_tensor,
1114
+ kernel_size, dilations, paddings, strides);
1115
+ }
1116
+
1117
+ /**
1118
+ * @brief Applies element-wise exponential function to the elements of a tensor.
1119
+ *
1120
+ * This function computes the exponential of each element in the source tensor
1121
+ * `acl_src` and stores the result back into the same tensor.
1122
+ * The operation is defined as:
1123
+ * \f[
1124
+ * \text {acl_src }_i=e^{acl\_src_i}
1125
+ * \f]
1126
+ *
1127
+ * @param ctx The context for the CANN backend operations.
1128
+ * @param acl_src The tensor on which the exponential function will be applied.
1129
+ */
1130
+ static void aclnn_exp(ggml_backend_cann_context& ctx, aclTensor* acl_src) {
1131
+ GGML_CANN_CALL_ACLNN_OP(ctx, InplaceExp, acl_src);
1132
+ }
1133
+
1134
+ void aclnn_cos(ggml_backend_cann_context& ctx, aclTensor* acl_src,
1135
+ aclTensor* acl_dst) {
1136
+ GGML_CANN_CALL_ACLNN_OP(ctx, Cos, acl_src, acl_dst);
1137
+ }
1138
+
1139
+ void aclnn_sin(ggml_backend_cann_context& ctx, aclTensor* acl_src,
1140
+ aclTensor* acl_dst) {
1141
+ GGML_CANN_CALL_ACLNN_OP(ctx, Sin, acl_src, acl_dst);
1142
+ }
1143
+
1144
+ void ggml_cann_timestep_embedding(ggml_backend_cann_context& ctx,
1145
+ ggml_tensor* dst) {
1146
+ const ggml_tensor* src = dst->src[0];
1147
+
1148
+ GGML_ASSERT(src->type == GGML_TYPE_F32);
1149
+ GGML_ASSERT(dst->type == GGML_TYPE_F32);
1150
+
1151
+ const int dim = dst->op_params[0];
1152
+ const int max_period = dst->op_params[1];
1153
+ int half = dim / 2;
1154
+
1155
+ aclTensor* acl_src = ggml_cann_create_tensor(src);
1156
+
1157
+ // arange: [0, ..., half)
1158
+ float start = 0;
1159
+ float stop = half;
1160
+ float step = 1;
1161
+ int64_t n_elements_arange = half;
1162
+ int64_t tmp_arange_ne[] = {half};
1163
+ size_t tmp_arange_nb[] = {sizeof(dst->type)};
1164
+
1165
+ ggml_cann_pool_alloc arange_allocator(ctx.pool(), half * sizeof(dst->type));
1166
+ void* tmp_arange_buffer = arange_allocator.get();
1167
+ aclTensor* tmp_arange_tensor = ggml_cann_create_tensor(
1168
+ tmp_arange_buffer, ggml_cann_type_mapping(dst->type),
1169
+ ggml_type_size(dst->type), tmp_arange_ne, tmp_arange_nb,
1170
+ GGML_MAX_DIMS - 3, ACL_FORMAT_ND);
1171
+
1172
+ aclnn_arange(ctx, tmp_arange_tensor, start, stop, step, n_elements_arange);
1173
+
1174
+ // freq
1175
+ float freq_param = -logf(max_period) / half;
1176
+ bool inplace = true;
1177
+ aclnn_muls(ctx, tmp_arange_tensor, freq_param, nullptr, inplace);
1178
+ aclnn_exp(ctx, tmp_arange_tensor);
1179
+
1180
+ // permute: src [0,1,2,3]->[0,1,3,2]
1181
+ int64_t tmp_permute_ne[] = {src->ne[1], src->ne[0], src->ne[2], src->ne[3]};
1182
+ size_t tmp_permute_nb[GGML_MAX_DIMS];
1183
+ tmp_permute_nb[0] = ggml_type_size(src->type);
1184
+ for (int i = 1; i < GGML_MAX_DIMS; i++) {
1185
+ tmp_permute_nb[i] = tmp_permute_nb[i - 1] * tmp_permute_ne[i - 1];
1186
+ }
1187
+
1188
+ ggml_cann_pool_alloc permute_allocator(ctx.pool(), ggml_nbytes(src));
1189
+ void* tmp_permute_buffer = permute_allocator.get();
1190
+ aclTensor* tmp_permute_tensor = ggml_cann_create_tensor(
1191
+ tmp_permute_buffer, ggml_cann_type_mapping(src->type),
1192
+ ggml_type_size(src->type), tmp_permute_ne, tmp_permute_nb,
1193
+ GGML_MAX_DIMS, ACL_FORMAT_ND);
1194
+ int64_t permute_dim[] = {0, 1, 3, 2};
1195
+ int64_t num_dims = 4;
1196
+ aclnn_permute(ctx, acl_src, tmp_permute_tensor, permute_dim, num_dims);
1197
+
1198
+ // timestep * freq
1199
+ int64_t tmp_mul_ne[] = {src->ne[1] * half, src->ne[0], src->ne[2],
1200
+ src->ne[3]};
1201
+ size_t tmp_mul_nb[GGML_MAX_DIMS];
1202
+ tmp_mul_nb[0] = ggml_type_size(src->type);
1203
+ for (int i = 1; i < GGML_MAX_DIMS; i++) {
1204
+ tmp_mul_nb[i] = tmp_mul_nb[i - 1] * tmp_mul_ne[i - 1];
1205
+ }
1206
+
1207
+ int mul_nelements =
1208
+ src->ne[1] * half * src->ne[0] * src->ne[2] * src->ne[3];
1209
+
1210
+ ggml_cann_pool_alloc mul_allocator(
1211
+ ctx.pool(), mul_nelements * ggml_type_size(src->type));
1212
+ void* tmp_mul_buffer = mul_allocator.get();
1213
+ aclTensor* tmp_mul_tensor = ggml_cann_create_tensor(
1214
+ tmp_mul_buffer, ggml_cann_type_mapping(src->type),
1215
+ ggml_type_size(src->type), tmp_mul_ne, tmp_mul_nb, GGML_MAX_DIMS,
1216
+ ACL_FORMAT_ND);
1217
+ aclnn_mul(ctx, tmp_permute_tensor, tmp_arange_tensor, tmp_mul_tensor);
1218
+
1219
+ // cos
1220
+ ggml_cann_pool_alloc cos_allocator(
1221
+ ctx.pool(), mul_nelements * ggml_type_size(src->type));
1222
+ void* tmp_cos_buffer = cos_allocator.get();
1223
+ aclTensor* tmp_cos_tensor = ggml_cann_create_tensor(
1224
+ tmp_cos_buffer, ggml_cann_type_mapping(dst->type),
1225
+ ggml_type_size(dst->type), tmp_mul_ne, tmp_mul_nb, GGML_MAX_DIMS,
1226
+ ACL_FORMAT_ND);
1227
+
1228
+ aclnn_cos(ctx, tmp_mul_tensor, tmp_cos_tensor);
1229
+
1230
+ // sin
1231
+ ggml_cann_pool_alloc sin_allocator(
1232
+ ctx.pool(), mul_nelements * ggml_type_size(src->type));
1233
+ void* tmp_sin_buffer = sin_allocator.get();
1234
+ aclTensor* tmp_sin_tensor = ggml_cann_create_tensor(
1235
+ tmp_sin_buffer, ggml_cann_type_mapping(dst->type),
1236
+ ggml_type_size(dst->type), tmp_mul_ne, tmp_mul_nb, GGML_MAX_DIMS,
1237
+ ACL_FORMAT_ND);
1238
+
1239
+ aclnn_sin(ctx, tmp_mul_tensor, tmp_sin_tensor);
1240
+
1241
+ // concat
1242
+ int64_t concat_dim = 3;
1243
+ aclTensor* acl_dst = ggml_cann_create_tensor(dst);
1244
+ aclTensor* tensors[] = {tmp_cos_tensor, tmp_sin_tensor};
1245
+ aclTensorList* tensor_list = aclCreateTensorList(tensors, 2);
1246
+ aclnn_concat(ctx, tensor_list, acl_dst, concat_dim);
1247
+
1248
+ // release
1249
+ // segmentation fault when delete both tensorList and his elements.
1250
+ ggml_cann_release_resources(ctx, tensor_list, acl_src, tmp_arange_tensor,
1251
+ tmp_permute_tensor, tmp_mul_tensor, acl_dst);
1252
+ }
1253
+
1254
+ /**
1255
+ * @brief Fills a tensor with a scalar value.
1256
+ *
1257
+ * This function fills the destination tensor `acl_dst` with the scalar value
1258
+ * `scalar`.
1259
+ *
1260
+ * @param ctx The context for the CANN backend operations.
1261
+ * @param scalar The scalar value used to fill the tensor.
1262
+ * @param acl_dst The destination tensor to be filled with the scalar value.
1263
+ */
1264
+ static void aclnn_fill_scalar(ggml_backend_cann_context& ctx, float scalar,
1265
+ aclTensor* acl_dst) {
1266
+ auto acl_scalar = aclCreateScalar(&scalar, aclDataType::ACL_FLOAT);
1267
+ GGML_CANN_CALL_ACLNN_OP(ctx, InplaceFillScalar, acl_dst, acl_scalar);
1268
+ ggml_cann_release_resources(ctx, acl_scalar);
1269
+ }
1270
+
1271
+ /**
1272
+ * @brief Raises each element of a tensor to the power of the corresponding
1273
+ * element in another tensor.
1274
+ *
1275
+ * This function computes the element-wise power of the destination tensor
1276
+ * `acl_dst` raised to the power of the exponent tensor `acl_exp`.
1277
+ * The operation is defined as:
1278
+ * \f[
1279
+ * \text {acl_dst }_i=acl\_dst_i^{\text {acl_exp }_i}
1280
+ * \f]
1281
+ *
1282
+ * @param ctx The context for the CANN backend operations.
1283
+ * @param acl_dst The destination tensor, which also serves as the base tensor.
1284
+ * @param acl_exp The exponent tensor, each element of which is used to raise
1285
+ * the corresponding element in the destination tensor.
1286
+ */
1287
+ static void aclnn_pow_tensor_tensor(ggml_backend_cann_context& ctx,
1288
+ aclTensor* acl_dst, aclTensor* acl_exp) {
1289
+ GGML_CANN_CALL_ACLNN_OP(ctx, InplacePowTensorTensor, acl_dst, acl_exp);
1290
+ }
1291
+
1292
+ /**
1293
+ * @brief Applies the Alibi (Attention with Linear Biases) mechanism to the
1294
+ * @details This function implements the Alibi mechanism, which introduces
1295
+ * learnable biases into the attention scores to simulate relative
1296
+ * position encoding without the need for explicit positional
1297
+ * embeddings.
1298
+ *
1299
+ * @param ctx The backend CANN context for executing operations.
1300
+ * @param acl_src The source tensor representing the query or key.
1301
+ * @param acl_position The position tensor containing relative positions.
1302
+ * @param acl_dst The destination tensor where the result will be stored.
1303
+ * @param n_head The number of attention heads.
1304
+ * @param src_ne The dimensions of the source tensor.
1305
+ * @param src_nb0 The byte size of the first dimension of the source
1306
+ tensor.
1307
+ * @param max_bias The maximum bias value used in the Alibi mechanism.
1308
+ * @param dst The destination tensor object for additional metadata.
1309
+ *
1310
+ * The function performs the following steps:
1311
+ * 1. Calculates the logarithm floor of the number of heads to determine the
1312
+ base for bias calculation.
1313
+ * 2. Initializes arrays with arithmetic sequences and fills them with bias
1314
+ values.
1315
+ * 3. Computes the bias tensor based on the calculated biases and arithmetic
1316
+ sequences.
1317
+ * 4. Reshapes the bias tensor to match the dimensions of the input tensors.
1318
+ * 5. Multiplies the position tensor by the bias tensor.
1319
+ * 6. Adds the result of the multiplication to the source tensor to produce the
1320
+ final output.
1321
+ */
1322
+ static void aclnn_alibi(ggml_backend_cann_context& ctx, aclTensor* acl_src,
1323
+ aclTensor* acl_position, aclTensor* acl_dst,
1324
+ const int n_head, int64_t* src_ne, const size_t src_nb0,
1325
+ float max_bias, ggml_tensor* dst) {
1326
+ const int64_t ne2_ne3 = src_ne[2] * src_ne[3];
1327
+ GGML_ASSERT(src_nb0 == sizeof(float));
1328
+ GGML_ASSERT(n_head == src_ne[2]);
1329
+
1330
+ const int n_heads_log2_floor = 1u << (uint32_t)floor(log2(n_head));
1331
+
1332
+ float m0 = powf(2.0f, -(max_bias) / n_heads_log2_floor);
1333
+ float m1 = powf(2.0f, -(max_bias / 2.0f) / n_heads_log2_floor);
1334
+
1335
+ // init arange
1336
+ ggml_cann_pool_alloc arange_allocator(ctx.pool(),
1337
+ ne2_ne3 * ggml_type_size(dst->type));
1338
+ void* tmp_arange_buffer = arange_allocator.get();
1339
+
1340
+ // arange1: [1, ..., n_heads_log2_floor+1)
1341
+ float start = 1;
1342
+ float stop = n_heads_log2_floor + 1;
1343
+ float step = 1;
1344
+ int64_t n_elements_arange = n_heads_log2_floor;
1345
+
1346
+ int64_t tmp_arange1_ne[] = {n_heads_log2_floor};
1347
+ size_t tmp_arange1_nb[] = {sizeof(dst->type)};
1348
+ aclTensor* tmp_arange1_tensor = ggml_cann_create_tensor(
1349
+ tmp_arange_buffer, ggml_cann_type_mapping(dst->type),
1350
+ ggml_type_size(dst->type), tmp_arange1_ne, tmp_arange1_nb,
1351
+ GGML_MAX_DIMS - 3, ACL_FORMAT_ND);
1352
+
1353
+ aclnn_arange(ctx, tmp_arange1_tensor, start, stop, step, n_elements_arange);
1354
+
1355
+ aclTensor* tmp_arange2_tensor = nullptr;
1356
+ if (n_heads_log2_floor < ne2_ne3) {
1357
+ // arange2: [1, ..., 2 * (k - n_heads_log2_floor) + 1)
1358
+ start = 1;
1359
+ stop = 2 * (ne2_ne3 - n_heads_log2_floor) + 1;
1360
+ step = 2;
1361
+ n_elements_arange = ne2_ne3 - n_heads_log2_floor;
1362
+ int64_t tmp_arange2_ne[] = {ne2_ne3 - n_heads_log2_floor};
1363
+ size_t tmp_arange2_nb[] = {sizeof(dst->type)};
1364
+
1365
+ aclTensor* tmp_arange2_tensor = ggml_cann_create_tensor(
1366
+ (char*)tmp_arange_buffer +
1367
+ n_heads_log2_floor * ggml_type_size(dst->type),
1368
+ ggml_cann_type_mapping(dst->type), ggml_type_size(dst->type),
1369
+ tmp_arange2_ne, tmp_arange2_nb, GGML_MAX_DIMS - 3, ACL_FORMAT_ND);
1370
+ aclnn_arange(ctx, tmp_arange2_tensor, start, stop, step,
1371
+ n_elements_arange);
1372
+ }
1373
+
1374
+ // init mk_base
1375
+ ggml_cann_pool_alloc mk_base_allocator(ctx.pool(),
1376
+ ne2_ne3 * ggml_type_size(dst->type));
1377
+ void* tmp_mk_base_buffer = mk_base_allocator.get();
1378
+ int64_t tmp_mk_base1_ne[] = {n_heads_log2_floor};
1379
+ size_t tmp_mk_base1_nb[] = {sizeof(dst->type)};
1380
+ aclTensor* tmp_mk_base1_tensor = ggml_cann_create_tensor(
1381
+ tmp_mk_base_buffer, ggml_cann_type_mapping(dst->type),
1382
+ ggml_type_size(dst->type), tmp_mk_base1_ne, tmp_mk_base1_nb,
1383
+ GGML_MAX_DIMS - 3, ACL_FORMAT_ND);
1384
+
1385
+ aclnn_fill_scalar(ctx, m0, tmp_mk_base1_tensor);
1386
+
1387
+ aclTensor* tmp_mk_base2_tensor = nullptr;
1388
+ if (n_heads_log2_floor < ne2_ne3) {
1389
+ int64_t tmp_mk_base2_ne[] = {ne2_ne3 - n_heads_log2_floor};
1390
+ size_t tmp_mk_base2_nb[] = {sizeof(dst->type)};
1391
+ aclTensor* tmp_mk_base2_tensor = ggml_cann_create_tensor(
1392
+ (char*)tmp_mk_base_buffer +
1393
+ n_heads_log2_floor * ggml_type_size(dst->type),
1394
+ ggml_cann_type_mapping(dst->type), ggml_type_size(dst->type),
1395
+ tmp_mk_base2_ne, tmp_mk_base2_nb, GGML_MAX_DIMS - 3, ACL_FORMAT_ND);
1396
+ aclnn_fill_scalar(ctx, m1, tmp_mk_base2_tensor);
1397
+ }
1398
+
1399
+ // init mk
1400
+ int64_t tmp_mk_base_ne[] = {ne2_ne3};
1401
+ size_t tmp_mk_base_nb[] = {sizeof(dst->type)};
1402
+ aclTensor* tmp_mk_base_tensor = ggml_cann_create_tensor(
1403
+ tmp_mk_base_buffer, ggml_cann_type_mapping(dst->type),
1404
+ ggml_type_size(dst->type), tmp_mk_base_ne, tmp_mk_base_nb,
1405
+ GGML_MAX_DIMS - 3, ACL_FORMAT_ND);
1406
+ aclTensor* tmp_arange_tensor = ggml_cann_create_tensor(
1407
+ tmp_arange_buffer, ggml_cann_type_mapping(dst->type),
1408
+ ggml_type_size(dst->type), tmp_mk_base_ne, tmp_mk_base_nb,
1409
+ GGML_MAX_DIMS - 3, ACL_FORMAT_ND);
1410
+ aclnn_pow_tensor_tensor(ctx, tmp_mk_base_tensor, tmp_arange_tensor);
1411
+
1412
+ // reshape mk
1413
+ int64_t tmp_mk_ne[] = {1, 1, src_ne[2], src_ne[3]};
1414
+ size_t tmp_mk_nb[GGML_MAX_DIMS];
1415
+ tmp_mk_nb[0] = ggml_type_size(dst->type);
1416
+ for (int i = 1; i < GGML_MAX_DIMS; i++) {
1417
+ tmp_mk_nb[i] = tmp_mk_nb[i - 1] * tmp_mk_ne[i - 1];
1418
+ }
1419
+ aclTensor* tmp_mk_tensor = ggml_cann_create_tensor(
1420
+ tmp_mk_base_buffer, ggml_cann_type_mapping(dst->type),
1421
+ ggml_type_size(dst->type), tmp_mk_ne, tmp_mk_nb, GGML_MAX_DIMS,
1422
+ ACL_FORMAT_ND);
1423
+
1424
+ // acl_position * mk
1425
+ int64_t tmp_output_ne[] = {src_ne[0], src_ne[1], src_ne[2], src_ne[3]};
1426
+ size_t tmp_output_nb[GGML_MAX_DIMS];
1427
+ tmp_output_nb[0] = ggml_type_size(dst->type);
1428
+ for (int i = 1; i < GGML_MAX_DIMS; i++) {
1429
+ tmp_output_nb[i] = tmp_output_nb[i - 1] * tmp_output_ne[i - 1];
1430
+ }
1431
+ ggml_cann_pool_alloc output_allocator(ctx.pool(), ggml_nbytes(dst));
1432
+ void* tmp_output_buffer = output_allocator.get();
1433
+ aclTensor* tmp_output_tensor = ggml_cann_create_tensor(
1434
+ tmp_output_buffer, ggml_cann_type_mapping(dst->type),
1435
+ ggml_type_size(dst->type), tmp_output_ne, tmp_output_nb, GGML_MAX_DIMS,
1436
+ ACL_FORMAT_ND);
1437
+ aclnn_mul(ctx, acl_position, tmp_mk_tensor, tmp_output_tensor);
1438
+
1439
+ // add
1440
+ aclnn_add(ctx, tmp_output_tensor, acl_src, acl_dst);
1441
+ ggml_cann_release_resources(ctx, tmp_arange1_tensor, tmp_arange2_tensor,
1442
+ tmp_mk_base1_tensor, tmp_mk_base2_tensor, tmp_mk_base_tensor,
1443
+ tmp_arange_tensor, tmp_mk_tensor, tmp_output_tensor);
1444
+ }
1445
+
1446
+ void ggml_cann_cpy(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
1447
+ ggml_cann_dup(ctx, dst);
1448
+ }
1449
+
1450
+ /**
1451
+ * @brief Applies the softmax function to a tensor along a specified dimension.
1452
+ *
1453
+ * This function computes the softmax of the source tensor `acl_src` along the
1454
+ * specified dimension `dim` and stores the result in the destination tensor
1455
+ * `acl_dst`.
1456
+ *
1457
+ * @param ctx The context for the CANN backend operations.
1458
+ * @param acl_src The source tensor on which the softmax function will be
1459
+ * applied.
1460
+ * @param dim The dimension along which the softmax function will be computed.
1461
+ * @param acl_dst The destination tensor where the softmax results will be
1462
+ * stored.
1463
+ */
1464
+ static void aclnn_softmax(ggml_backend_cann_context& ctx, aclTensor* acl_src,
1465
+ int64_t dim, aclTensor* acl_dst) {
1466
+ GGML_CANN_CALL_ACLNN_OP(ctx, Softmax, acl_src, dim, acl_dst);
1467
+ }
1468
+
1469
+ void ggml_cann_softmax(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
1470
+ ggml_tensor* src0 = dst->src[0];
1471
+ ggml_tensor* src1 = dst->src[1]; // mask
1472
+
1473
+ aclTensor* acl_src0 = ggml_cann_create_tensor(src0);
1474
+ aclTensor* acl_dst = ggml_cann_create_tensor(dst);
1475
+
1476
+ float scale = 1.0f;
1477
+ float max_bias = 0.0f;
1478
+
1479
+ memcpy(&scale, (float*)dst->op_params + 0, sizeof(float));
1480
+ memcpy(&max_bias, (float*)dst->op_params + 1, sizeof(float));
1481
+
1482
+ // input mul scale
1483
+ aclScalar* acl_scale = aclCreateScalar(&scale, aclDataType::ACL_FLOAT);
1484
+
1485
+ size_t n_bytes = ggml_nbytes(src0);
1486
+ ggml_cann_pool_alloc mul_scale_allocator(ctx.pool(), n_bytes);
1487
+ void* input_mul_scale_buffer = mul_scale_allocator.get();
1488
+ aclTensor* acl_input_mul_scale_tensor = ggml_cann_create_tensor(
1489
+ input_mul_scale_buffer, ACL_FLOAT, ggml_type_size(src0->type), src0->ne,
1490
+ src0->nb, GGML_MAX_DIMS);
1491
+
1492
+ bool inplace = false;
1493
+ aclnn_muls(ctx, acl_src0, scale, acl_input_mul_scale_tensor, inplace);
1494
+
1495
+ // mask
1496
+ aclTensor* acl_src1_fp32_tensor = nullptr;
1497
+ aclTensor* tmp_mask_tensor = nullptr;
1498
+ ggml_cann_pool_alloc src1_fp32_allocator(ctx.pool());
1499
+ if (src1) {
1500
+ const bool use_f16 = src1->type == GGML_TYPE_F16;
1501
+ if (use_f16) {
1502
+ // cast to fp32
1503
+ size_t n_bytes = ggml_nelements(src1) * sizeof(float_t);
1504
+ size_t src1_fp32_nb[GGML_MAX_DIMS];
1505
+ src1_fp32_nb[0] = sizeof(float_t);
1506
+ for (int i = 1; i < GGML_MAX_DIMS; i++) {
1507
+ src1_fp32_nb[i] = src1_fp32_nb[i - 1] * src1->ne[i - 1];
1508
+ }
1509
+ src1_fp32_allocator.alloc(n_bytes);
1510
+ void* src1_fp32_buffer = src1_fp32_allocator.get();
1511
+ acl_src1_fp32_tensor = ggml_cann_create_tensor(
1512
+ src1_fp32_buffer, ACL_FLOAT, sizeof(float), src1->ne,
1513
+ src1_fp32_nb, GGML_MAX_DIMS);
1514
+ aclTensor* acl_src1 = ggml_cann_create_tensor(src1);
1515
+ aclnn_cast(ctx, acl_src1, acl_src1_fp32_tensor, ACL_FLOAT);
1516
+ ggml_cann_release_resources(ctx, acl_src1);
1517
+ } else {
1518
+ acl_src1_fp32_tensor = ggml_cann_create_tensor(src1);
1519
+ }
1520
+
1521
+ // broadcast the mask across rows, only use ne11 of ne01 in mask
1522
+ if (src1->ne[1] != src0->ne[1]) {
1523
+ // mask shape: [1,1,ne11,ne10]
1524
+ int64_t tmp_mask_ne[] = {src0->ne[0], src0->ne[1], 1, 1};
1525
+ size_t tmp_mask_nb[GGML_MAX_DIMS];
1526
+ tmp_mask_nb[0] = sizeof(float_t);
1527
+ for (int i = 1; i < GGML_MAX_DIMS; i++) {
1528
+ tmp_mask_nb[i] = tmp_mask_nb[i - 1] * tmp_mask_ne[i - 1];
1529
+ }
1530
+ tmp_mask_tensor = ggml_cann_create_tensor(
1531
+ src1->data, ACL_FLOAT, sizeof(float), tmp_mask_ne, tmp_mask_nb,
1532
+ GGML_MAX_DIMS, ACL_FORMAT_ND);
1533
+ }
1534
+
1535
+ // alibi
1536
+ const int n_head = src0->ne[2];
1537
+ const size_t src_nb0 = src0->nb[0];
1538
+
1539
+ n_bytes = ggml_nbytes(dst);
1540
+ ggml_cann_pool_alloc output_allocator(ctx.pool(), n_bytes);
1541
+ void* output_buffer = output_allocator.get();
1542
+ aclTensor* alibi_output_tensor = ggml_cann_create_tensor(
1543
+ output_buffer, ACL_FLOAT, ggml_type_size(dst->type), dst->ne,
1544
+ dst->nb, GGML_MAX_DIMS);
1545
+ if (max_bias <= 0.0f) {
1546
+ // slope = 1.0
1547
+ if (tmp_mask_tensor) {
1548
+ aclnn_add(ctx, tmp_mask_tensor, acl_input_mul_scale_tensor,
1549
+ alibi_output_tensor);
1550
+ } else {
1551
+ aclnn_add(ctx, acl_src1_fp32_tensor, acl_input_mul_scale_tensor,
1552
+ alibi_output_tensor);
1553
+ }
1554
+ } else {
1555
+ // slope != 1.0
1556
+ if (tmp_mask_tensor) {
1557
+ aclnn_alibi(ctx, acl_input_mul_scale_tensor, tmp_mask_tensor,
1558
+ alibi_output_tensor, n_head, src0->ne, src_nb0,
1559
+ max_bias, dst);
1560
+ } else {
1561
+ aclnn_alibi(ctx, acl_input_mul_scale_tensor,
1562
+ acl_src1_fp32_tensor, alibi_output_tensor, n_head,
1563
+ src0->ne, src_nb0, max_bias, dst);
1564
+ }
1565
+ }
1566
+
1567
+ // softmax
1568
+ aclnn_softmax(ctx, alibi_output_tensor, 3, acl_dst);
1569
+ ggml_cann_release_resources(ctx, alibi_output_tensor);
1570
+ } else {
1571
+ aclnn_softmax(ctx, acl_input_mul_scale_tensor, 3, acl_dst);
1572
+ }
1573
+
1574
+ ggml_cann_release_resources(ctx, acl_src0, acl_src1_fp32_tensor, acl_dst,
1575
+ acl_scale, acl_input_mul_scale_tensor, tmp_mask_tensor);
1576
+ }
1577
+
1578
+ /**
1579
+ * @brief Performs embedding operation on a 4D tensor using the CANN backend.
1580
+ *
1581
+ * This function extracts slices from the source tensor (`src_buffer`),
1582
+ * index tensor (`index`), and destination tensor (`dst`), and performs an
1583
+ * embedding operation on them. The embedding operation is applied by iterating
1584
+ * over the last two dimensions of the source tensor, creating the necessary
1585
+ * tensors for the source, index, and output, and executing the embedding operation.
1586
+ *
1587
+ * @param ctx The context for CANN backend operations.
1588
+ * @param src_buffer The source buffer holding the data for the source tensor.
1589
+ * @param src_ne The dimensions of the source tensor.
1590
+ * @param src_nb The strides (byte offsets) of the source tensor.
1591
+ * @param index The index tensor used in the embedding operation.
1592
+ * @param dst The destination tensor where the result will be stored.
1593
+ */
1594
+ static void aclnn_embedding_4d(ggml_backend_cann_context& ctx, void* src_buffer,
1595
+ int64_t* src_ne, size_t* src_nb, ggml_tensor* index,
1596
+ ggml_tensor* dst) {
1597
+ for (int64_t i = 0; i < src_ne[3]; i++) {
1598
+ for (int64_t j = 0; j < src_ne[2]; j++) {
1599
+ // src
1600
+ int64_t acl_src_ne[2] = {src_ne[0], src_ne[1]};
1601
+ size_t acl_src_nb[2] = {src_nb[0], src_nb[1]};
1602
+ aclTensor* acl_src_tensor = ggml_cann_create_tensor(
1603
+ (char*)src_buffer + i * src_nb[3] + j * src_nb[2],
1604
+ ggml_cann_type_mapping(dst->type), ggml_element_size(dst),
1605
+ acl_src_ne, acl_src_nb, 2);
1606
+
1607
+ // index
1608
+ int64_t acl_index_ne[1] = {index->ne[0]};
1609
+ size_t acl_index_nb[1] = {index->nb[0]};
1610
+ aclTensor* acl_index = ggml_cann_create_tensor(
1611
+ (char*)index->data + i * index->nb[2] + j * index->nb[1],
1612
+ ggml_cann_type_mapping(index->type), ggml_element_size(index),
1613
+ acl_index_ne, acl_index_nb, 1);
1614
+
1615
+ // out
1616
+ int64_t acl_out_ne[2] = {dst->ne[0], dst->ne[1]};
1617
+ size_t acl_out_nb[2] = {dst->nb[0], dst->nb[1]};
1618
+ aclTensor* acl_out = ggml_cann_create_tensor(
1619
+ (char*)dst->data + i * dst->nb[3] + j * dst->nb[2],
1620
+ ggml_cann_type_mapping(dst->type), ggml_element_size(dst),
1621
+ acl_out_ne, acl_out_nb, 2);
1622
+ GGML_CANN_CALL_ACLNN_OP(ctx, Embedding, acl_src_tensor, acl_index, acl_out);
1623
+ ggml_cann_release_resources(ctx, acl_src_tensor, acl_index, acl_out);
1624
+ }
1625
+ }
1626
+ }
1627
+
1628
+ void ggml_cann_get_rows(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
1629
+ ggml_tensor* src0 = dst->src[0]; // src
1630
+ ggml_tensor* src1 = dst->src[1]; // index
1631
+
1632
+ switch (src0->type) {
1633
+ case GGML_TYPE_F32: {
1634
+ aclnn_embedding_4d(ctx, src0->data, src0->ne, src0->nb, src1,
1635
+ dst);
1636
+ break;
1637
+ }
1638
+ case GGML_TYPE_F16: {
1639
+ aclTensor* acl_src0 = ggml_cann_create_tensor(src0);
1640
+ ggml_cann_pool_alloc src_buffer_allocator(
1641
+ ctx.pool(), ggml_nelements(src0) * sizeof(float_t));
1642
+ void* src_trans_buffer = src_buffer_allocator.get();
1643
+ size_t src_trans_nb[GGML_MAX_DIMS];
1644
+ src_trans_nb[0] = sizeof(float_t);
1645
+ for (int i = 1; i < GGML_MAX_DIMS; i++) {
1646
+ src_trans_nb[i] = src_trans_nb[i - 1] * src0->ne[i - 1];
1647
+ }
1648
+ aclTensor* src_trans_tensor = ggml_cann_create_tensor(
1649
+ src_trans_buffer, ACL_FLOAT, ggml_type_size(dst->type),
1650
+ src0->ne, src_trans_nb, GGML_MAX_DIMS);
1651
+ aclnn_cast(ctx, acl_src0, src_trans_tensor, ggml_cann_type_mapping(dst->type));
1652
+ aclnn_embedding_4d(ctx, src_trans_buffer, src0->ne,
1653
+ src_trans_nb, src1, dst);
1654
+ ggml_cann_release_resources(ctx, acl_src0, src_trans_tensor);
1655
+ break;
1656
+ }
1657
+ case GGML_TYPE_Q8_0: {
1658
+ // add 1 dim for bcast mul.
1659
+ size_t weight_nb[GGML_MAX_DIMS + 1], scale_nb[GGML_MAX_DIMS + 1],
1660
+ dequant_nb[GGML_MAX_DIMS + 1];
1661
+ int64_t weight_ne[GGML_MAX_DIMS + 1], scale_ne[GGML_MAX_DIMS + 1],
1662
+ *dequant_ne;
1663
+ int64_t scale_offset = 0;
1664
+
1665
+ // [3,4,5,64] -> [3,4,5,2,32]
1666
+ weight_ne[0] = QK8_0;
1667
+ weight_ne[1] = src0->ne[0] / QK8_0;
1668
+ weight_nb[0] = sizeof(int8_t);
1669
+ weight_nb[1] = weight_nb[0] * weight_ne[0];
1670
+ for (int i = 2; i < GGML_MAX_DIMS + 1; i++) {
1671
+ weight_ne[i] = src0->ne[i - 1];
1672
+ weight_nb[i] = weight_nb[i - 1] * weight_ne[i - 1];
1673
+ }
1674
+
1675
+ // [3,4,5,64] -> [3,4,5,2,1]
1676
+ scale_ne[0] = 1;
1677
+ scale_ne[1] = src0->ne[0] / QK8_0;
1678
+ scale_nb[0] = sizeof(uint16_t);
1679
+ scale_nb[1] = scale_nb[0] * scale_ne[0];
1680
+ for (int i = 2; i < GGML_MAX_DIMS + 1; i++) {
1681
+ scale_ne[i] = src0->ne[i - 1];
1682
+ scale_nb[i] = scale_nb[i - 1] * scale_ne[i - 1];
1683
+ }
1684
+
1685
+ // [3,4,5,64] -> [3,4,5,2,32]
1686
+ dequant_ne = weight_ne;
1687
+ dequant_nb[0] = sizeof(float_t);
1688
+ for (int i = 1; i < GGML_MAX_DIMS + 1; i++) {
1689
+ dequant_nb[i] = dequant_nb[i - 1] * dequant_ne[i - 1];
1690
+ }
1691
+
1692
+ scale_offset = ggml_nelements(src0) * sizeof(int8_t);
1693
+ ggml_cann_pool_alloc dequant_buffer_allocator(
1694
+ ctx.pool(), ggml_nelements(src0) * sizeof(float_t));
1695
+
1696
+ aclTensor* acl_weight_tensor = ggml_cann_create_tensor(
1697
+ src0->data, ACL_INT8, sizeof(int8_t), weight_ne, weight_nb,
1698
+ GGML_MAX_DIMS + 1);
1699
+ aclTensor* acl_scale_tensor = ggml_cann_create_tensor(
1700
+ src0->data, ACL_FLOAT16, sizeof(uint16_t), scale_ne, scale_nb,
1701
+ GGML_MAX_DIMS + 1, ACL_FORMAT_ND, scale_offset);
1702
+ aclTensor* dequant_tensor = ggml_cann_create_tensor(
1703
+ dequant_buffer_allocator.get(), ACL_FLOAT, sizeof(float_t),
1704
+ dequant_ne, dequant_nb, GGML_MAX_DIMS + 1);
1705
+
1706
+ aclnn_mul(ctx, acl_weight_tensor, acl_scale_tensor, dequant_tensor);
1707
+ dequant_nb[0] = sizeof(float_t);
1708
+ dequant_ne = src0->ne;
1709
+ for (int i = 1; i < GGML_MAX_DIMS; i++) {
1710
+ dequant_nb[i] = dequant_nb[i - 1] * src0->ne[i - 1];
1711
+ }
1712
+
1713
+ aclnn_embedding_4d(ctx, dequant_buffer_allocator.get(),
1714
+ dequant_ne, dequant_nb, src1, dst);
1715
+
1716
+ ggml_cann_release_resources(ctx, dequant_tensor);
1717
+ break;
1718
+ }
1719
+ default:
1720
+ GGML_ABORT("Unsupported tensor type for GGML_OP_GET_ROWS");
1721
+ break;
1722
+ }
1723
+ }
1724
+
1725
+ /**
1726
+ * @brief Repeats elements of a tensor along a specified dimension.
1727
+ *
1728
+ * This function repeats each element of the source tensor `acl_src` a specified
1729
+ * number of times (`repeats`) along the specified dimension `dim` and stores
1730
+ * the result in the destination tensor `acl_dst`.
1731
+ *
1732
+ * @param ctx The context for the CANN backend operations.
1733
+ * @param acl_src The source tensor whose elements will be repeated.
1734
+ * @param acl_dst The destination tensor where the repeated elements will be
1735
+ * stored.
1736
+ * @param dim The dimension along which the elements will be repeated.
1737
+ * @param repeats The number of times each element will be repeated.
1738
+ * @param output_size The size of the output tensor.
1739
+ */
1740
+ static void aclnn_repeat_interleave(ggml_backend_cann_context& ctx,
1741
+ aclTensor* acl_src, aclTensor* acl_dst,
1742
+ int64_t dim, int64_t repeats,
1743
+ int64_t output_size) {
1744
+ GGML_CANN_CALL_ACLNN_OP(ctx, RepeatInterleaveIntWithDim, acl_src, repeats, dim,
1745
+ output_size, acl_dst);
1746
+ }
1747
+
1748
+ /**
1749
+ * @brief Performs matrix multiplication with floating-point precision on
1750
+ * tensors using the CANN backend.
1751
+ *
1752
+ * This function performs matrix multiplication of the input tensor and the
1753
+ * weight tensor, handling broadcasting and transposing as needed, and stores
1754
+ * the result in the destination tensor `dst`.
1755
+ *
1756
+ * @param ctx The context for the CANN backend operations.
1757
+ * @param dst The destination tensor where the result of the matrix
1758
+ * multiplication will be stored.
1759
+ */
1760
+ static void ggml_cann_mat_mul_fp(ggml_backend_cann_context& ctx,
1761
+ ggml_tensor* dst) {
1762
+ ggml_tensor* weight = dst->src[0]; // weight
1763
+ ggml_tensor* input = dst->src[1]; // input
1764
+
1765
+ // when weight ne2 or ne3 is 1, aclnnMatmulGetWorkspaceSize will auto
1766
+ // broadcast, when weight ne2 or ne3 is not 1, weight need repeat.
1767
+ BCAST_MUL_MAT_SHAPE(input, weight, dst);
1768
+
1769
+ int64_t n_dims = bcast_dims;
1770
+ if (bcast_input_ne[3] == bcast_weight_ne[3] && bcast_input_ne[3] == 1) {
1771
+ if (bcast_input_ne[2] == 1 && bcast_weight_ne[2] == 1) {
1772
+ n_dims = 2;
1773
+ } else if (bcast_input_ne[2] == 1) {
1774
+ n_dims = 3;
1775
+ }
1776
+ }
1777
+
1778
+ aclTensor* acl_input_tensor =
1779
+ ggml_cann_create_tensor(input, bcast_input_ne, bcast_input_nb, n_dims);
1780
+ int64_t transpose_ne[] = {bcast_weight_ne[1], bcast_weight_ne[0],
1781
+ bcast_weight_ne[2], bcast_weight_ne[3],
1782
+ bcast_weight_ne[4], bcast_weight_ne[5]};
1783
+ size_t transpose_nb[] = {bcast_weight_nb[1], bcast_weight_nb[0],
1784
+ bcast_weight_nb[2], bcast_weight_nb[3],
1785
+ bcast_weight_nb[4], bcast_weight_nb[5]};
1786
+ aclTensor* acl_weight_tensor =
1787
+ ggml_cann_create_tensor(weight, transpose_ne, transpose_nb, n_dims);
1788
+ aclTensor* acl_dst =
1789
+ ggml_cann_create_tensor(dst, bcast_dst_ne, bcast_dst_nb, n_dims);
1790
+
1791
+ switch (n_dims) {
1792
+ case 2:
1793
+ GGML_CANN_CALL_ACLNN_OP(ctx, Mm, acl_input_tensor, acl_weight_tensor, acl_dst, 2);
1794
+ break;
1795
+ case 3:
1796
+ GGML_CANN_CALL_ACLNN_OP(ctx, BatchMatMul, acl_input_tensor, acl_weight_tensor, acl_dst, 2);
1797
+ break;
1798
+ default:
1799
+ // ALLOW_FP32_DOWN_PRECISION, when input is
1800
+ // fp32, atlas a2 will transpose it to HFLOAT32.
1801
+ GGML_CANN_CALL_ACLNN_OP(ctx, Matmul, acl_input_tensor, acl_weight_tensor, acl_dst, 1);
1802
+ break;
1803
+ }
1804
+
1805
+ ggml_cann_release_resources(ctx, acl_weight_tensor, acl_input_tensor, acl_dst);
1806
+ }
1807
+
1808
+ /**
1809
+ * @brief Performs matrix multiplication with quantized weights and
1810
+ * floating-point inputs using the CANN backend.
1811
+ *
1812
+ * This function performs matrix multiplication of the input tensor `src1` and
1813
+ * the weight tensor `src0`, handling broadcasting, transposing, and
1814
+ * quantization as needed, and stores the result in the destination tensor
1815
+ * `dst`.
1816
+ *
1817
+ * @param ctx The context for the CANN backend operations.
1818
+ * @param dst The destination tensor where the result of the matrix
1819
+ * multiplication will be stored.
1820
+ */
1821
+ static void ggml_cann_mul_mat_quant(ggml_backend_cann_context& ctx,
1822
+ ggml_tensor* dst,
1823
+ const enum ggml_type type) {
1824
+ ggml_tensor* src0 = dst->src[0]; // weight
1825
+ ggml_tensor* src1 = dst->src[1]; // input
1826
+
1827
+ // The shape of the weight is NCHW.
1828
+ // Matrix multiplication uses HW dims.
1829
+ // HC is regarded as batch.
1830
+ // weight need transpose.
1831
+ float weight_elem_size;
1832
+ if (type == GGML_TYPE_Q4_0) {
1833
+ weight_elem_size = float(sizeof(uint8_t)) / 2;
1834
+ } else if (type == GGML_TYPE_Q8_0) {
1835
+ weight_elem_size = float(sizeof(uint8_t));
1836
+ } else {
1837
+ GGML_ABORT("Only support Q4_0 and Q8_0 MUL_MAT");
1838
+ }
1839
+ float weight_nb[] = {src0->ne[0] * weight_elem_size, weight_elem_size};
1840
+ size_t weight_stride = src0->ne[1] * src0->ne[0] * weight_elem_size;
1841
+ size_t weight_size = weight_stride * src0->ne[2] * src0->ne[3];
1842
+
1843
+ // scale stored at the end of weight. Also need transpose.
1844
+ size_t scale_elem_size = sizeof(uint16_t);
1845
+ size_t scale_nb[] = {src0->ne[0] / QK8_0 * scale_elem_size,
1846
+ scale_elem_size};
1847
+ size_t scale_stride = src0->ne[1] * src0->ne[0] / QK8_0 * scale_elem_size;
1848
+ char* scale_offset = (char*)src0->data + weight_size;
1849
+
1850
+ // input
1851
+ size_t input_elem_size = sizeof(uint16_t);
1852
+ int64_t input_ne[] = {src1->ne[0], src1->ne[1]};
1853
+ size_t input_nb[] = {input_elem_size, input_ne[0] * input_elem_size};
1854
+ size_t input_stride = input_ne[0] * input_ne[1] * input_elem_size;
1855
+ ggml_cann_pool_alloc input_alloctor(ctx.pool());
1856
+ void* input_buffer = src1->data;
1857
+
1858
+ // case in
1859
+ if (src1->type != GGML_TYPE_F16) {
1860
+ aclTensor* acl_src1_tensor = ggml_cann_create_tensor(src1);
1861
+ input_buffer =
1862
+ input_alloctor.alloc(ggml_nelements(src1) * input_elem_size);
1863
+
1864
+ int64_t* input_cast_ne = src1->ne;
1865
+ size_t input_cast_nb[GGML_MAX_DIMS];
1866
+ input_cast_nb[0] = sizeof(uint16_t);
1867
+ for (int i = 1; i < GGML_MAX_DIMS; i++) {
1868
+ input_cast_nb[i] = input_cast_nb[i - 1] * input_cast_ne[i - 1];
1869
+ }
1870
+
1871
+ aclTensor* acl_input_tensor = ggml_cann_create_tensor(
1872
+ input_buffer, ACL_FLOAT16, input_elem_size, input_cast_ne,
1873
+ input_cast_nb, GGML_MAX_DIMS);
1874
+ aclnn_cast(ctx, acl_src1_tensor, acl_input_tensor, ACL_FLOAT16);
1875
+ ggml_cann_release_resources(ctx, acl_input_tensor, acl_src1_tensor);
1876
+ }
1877
+
1878
+ // output
1879
+ size_t output_elem_size = sizeof(uint16_t);
1880
+ size_t output_nb[] = {output_elem_size, dst->ne[0] * output_elem_size};
1881
+ ggml_cann_pool_alloc output_allocator(ctx.pool());
1882
+ void* output_buffer =
1883
+ output_allocator.alloc(ggml_nelements(dst) * output_elem_size);
1884
+ size_t output_stride = dst->ne[0] * dst->ne[1] * output_elem_size;
1885
+
1886
+ // aclnn
1887
+ int64_t max_elem_size = 65535;
1888
+ int64_t split_size = (src0->ne[1] / max_elem_size) + 1;
1889
+ ggml_cann_pool_alloc workspace_allocator(ctx.pool());
1890
+ for (int64_t n1 = 0; n1 < src1->ne[3]; n1++) {
1891
+ for (int64_t c1 = 0; c1 < src1->ne[2]; c1++) {
1892
+ int64_t n0 = n1 / (src1->ne[3] / src0->ne[3]);
1893
+ int64_t c0 = c1 / (src1->ne[2] / src0->ne[2]);
1894
+
1895
+ int64_t batch1 = (n1 * src1->ne[2]) + c1;
1896
+ int64_t batch0 = (n0 * src0->ne[2]) + c0;
1897
+
1898
+ aclTensor* acl_input_tensor = ggml_cann_create_tensor(
1899
+ (char*)input_buffer + batch1 * input_stride, ACL_FLOAT16,
1900
+ input_elem_size, input_ne, input_nb, 2);
1901
+
1902
+ // first split
1903
+ int64_t weight_ne_offset = 0;
1904
+ int64_t weight_ne[2] = {
1905
+ max_elem_size > src0->ne[1] ? src0->ne[1] : max_elem_size,
1906
+ src0->ne[0]};
1907
+ int64_t scale_ne_offset = 0;
1908
+ int64_t scale_ne[2] = {weight_ne[0], weight_ne[1] / QK8_0};
1909
+ int64_t output_ne_offset = 0;
1910
+ int64_t output_ne[2] = {weight_ne[0], dst->ne[1]};
1911
+
1912
+ aclTensor* acl_weight_tensor = ggml_cann_create_tensor(
1913
+ (char*)src0->data + batch0 * weight_stride,
1914
+ ggml_cann_type_mapping(type), weight_elem_size, weight_ne,
1915
+ weight_nb, 2, ACL_FORMAT_ND, weight_ne_offset);
1916
+ aclTensor* acl_scale_tensor = ggml_cann_create_tensor(
1917
+ scale_offset + batch0 * scale_stride, ACL_FLOAT16,
1918
+ scale_elem_size, scale_ne, scale_nb, 2, ACL_FORMAT_ND,
1919
+ scale_ne_offset);
1920
+ aclTensor* acl_output_tensor = ggml_cann_create_tensor(
1921
+ (char*)output_buffer + batch1 * output_stride, ACL_FLOAT16,
1922
+ output_elem_size, output_ne, output_nb, 2, ACL_FORMAT_ND,
1923
+ output_ne_offset);
1924
+ int64_t antiquantGroupSize = 0;
1925
+ if (src0->ne[0] > QK8_0) {
1926
+ antiquantGroupSize = QK8_0;
1927
+ }
1928
+ GGML_CANN_CALL_ACLNN_OP(ctx, WeightQuantBatchMatmulV2, acl_input_tensor,
1929
+ acl_weight_tensor, acl_scale_tensor, nullptr,
1930
+ nullptr, nullptr, nullptr, antiquantGroupSize,
1931
+ acl_output_tensor);
1932
+ ggml_cann_release_resources(ctx, acl_weight_tensor, acl_scale_tensor, acl_output_tensor);
1933
+
1934
+ // other splits
1935
+ for (int64_t split = 1; split < split_size; split++) {
1936
+ weight_ne_offset +=
1937
+ weight_elem_size * weight_ne[0] * weight_ne[1];
1938
+ weight_ne[0] = max_elem_size * (split + 1) > src0->ne[1]
1939
+ ? src0->ne[1] - (max_elem_size * split)
1940
+ : max_elem_size;
1941
+ scale_ne_offset += scale_elem_size * scale_ne[0] * scale_ne[1];
1942
+ scale_ne[0] = weight_ne[0];
1943
+ output_ne_offset +=
1944
+ output_elem_size * output_ne[0] * output_ne[1];
1945
+ output_ne[0] = weight_ne[0];
1946
+
1947
+ acl_weight_tensor = ggml_cann_create_tensor(
1948
+ (char*)src0->data + batch0 * weight_stride,
1949
+ ggml_cann_type_mapping(type), weight_elem_size, weight_ne,
1950
+ weight_nb, 2, ACL_FORMAT_ND, weight_ne_offset);
1951
+ acl_scale_tensor = ggml_cann_create_tensor(
1952
+ scale_offset + batch0 * scale_stride, ACL_FLOAT16,
1953
+ scale_elem_size, scale_ne, scale_nb, 2, ACL_FORMAT_ND,
1954
+ scale_ne_offset);
1955
+ acl_output_tensor = ggml_cann_create_tensor(
1956
+ (char*)output_buffer + batch1 * output_stride, ACL_FLOAT16,
1957
+ output_elem_size, output_ne, output_nb, 2, ACL_FORMAT_ND,
1958
+ output_ne_offset);
1959
+ GGML_CANN_CALL_ACLNN_OP(ctx, WeightQuantBatchMatmulV2, acl_input_tensor,
1960
+ acl_weight_tensor, acl_scale_tensor, nullptr,
1961
+ nullptr, nullptr, nullptr, antiquantGroupSize,
1962
+ acl_output_tensor);
1963
+ ggml_cann_release_resources(ctx, acl_weight_tensor, acl_scale_tensor, acl_output_tensor);
1964
+ }
1965
+
1966
+ ggml_cann_release_resources(ctx, acl_input_tensor);
1967
+ }
1968
+ }
1969
+
1970
+ // cast out
1971
+ if (dst->type != GGML_TYPE_F16) {
1972
+ int64_t* output_cast_ne = dst->ne;
1973
+ size_t output_cast_nb[GGML_MAX_DIMS];
1974
+ output_cast_nb[0] = sizeof(uint16_t);
1975
+ for (int i = 1; i < GGML_MAX_DIMS; i++) {
1976
+ output_cast_nb[i] = output_cast_nb[i - 1] * output_cast_ne[i - 1];
1977
+ }
1978
+
1979
+ aclTensor* acl_output_tensor = ggml_cann_create_tensor(
1980
+ output_buffer, ACL_FLOAT16, output_elem_size, output_cast_ne,
1981
+ output_cast_nb, GGML_MAX_DIMS);
1982
+ aclTensor* acl_dst_tensor = ggml_cann_create_tensor(dst);
1983
+ aclnn_cast(ctx, acl_output_tensor, acl_dst_tensor, ggml_cann_type_mapping(dst->type));
1984
+
1985
+ ggml_cann_release_resources(ctx, acl_output_tensor, acl_dst_tensor);
1986
+ }
1987
+ }
1988
+
1989
+ void ggml_cann_mul_mat(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
1990
+ const enum ggml_type type = dst->src[0]->type;
1991
+ switch (type) {
1992
+ case GGML_TYPE_F32:
1993
+ case GGML_TYPE_F16:
1994
+ ggml_cann_mat_mul_fp(ctx, dst);
1995
+ break;
1996
+ case GGML_TYPE_Q4_0:
1997
+ case GGML_TYPE_Q8_0:
1998
+ ggml_cann_mul_mat_quant(ctx, dst, type);
1999
+ break;
2000
+ default:
2001
+ GGML_ABORT("Unsupported type for mul_mat");
2002
+ break;
2003
+ }
2004
+ }
2005
+
2006
+ /**
2007
+ * @brief Rolls the elements of a tensor along a specified dimension.
2008
+ *
2009
+ * This function rolls the elements of the source tensor `acl_src` by the
2010
+ * specified shifts `shifts` along the specified dimensions `dims`, and stores
2011
+ * the result in the destination tensor `acl_dst`.
2012
+ *
2013
+ * @param ctx The context for the CANN backend operations.
2014
+ * @param acl_src The source tensor whose elements will be rolled.
2015
+ * @param acl_dst The destination tensor where the rolled elements will be
2016
+ * stored.
2017
+ * @param shifts An array specifying the number of positions by which elements
2018
+ * are shifted.
2019
+ * @param dims An array specifying the dimensions along which elements are
2020
+ * shifted.
2021
+ */
2022
+ static void aclnn_roll(ggml_backend_cann_context& ctx, aclTensor* acl_src,
2023
+ aclTensor* acl_dst, int64_t* shifts, int64_t* dims) {
2024
+ aclIntArray* acl_shifts = aclCreateIntArray(shifts, 1);
2025
+ aclIntArray* acl_dims = aclCreateIntArray(dims, 1);
2026
+ GGML_CANN_CALL_ACLNN_OP(ctx, Roll, acl_src, acl_shifts, acl_dims, acl_dst);
2027
+ ggml_cann_release_resources(ctx, acl_shifts, acl_dims);
2028
+ }
2029
+
2030
+ /**
2031
+ * @brief Fills specified positions of a tensor with a scalar value.
2032
+ *
2033
+ * This function fills the positions in the source tensor `acl_src` specified by
2034
+ * `index` along the dimension `dim` with the scalar value `value`.
2035
+ *
2036
+ * @param ctx The context for the CANN backend operations.
2037
+ * @param acl_src The source tensor where the positions will be filled.
2038
+ * @param dim The dimension along which the positions are specified.
2039
+ * @param index An array specifying the positions to be filled.
2040
+ * @param index_num The number of positions specified in the index array.
2041
+ * @param value The scalar value used to fill the specified positions.
2042
+ */
2043
+ static void aclnn_index_fill_tensor(ggml_backend_cann_context& ctx,
2044
+ aclTensor* acl_src, int64_t dim,
2045
+ int64_t* index, int64_t index_num,
2046
+ float value) {
2047
+ aclIntArray* acl_index = aclCreateIntArray(index, index_num);
2048
+ aclScalar* acl_value = aclCreateScalar(&value, aclDataType::ACL_FLOAT);
2049
+ GGML_CANN_CALL_ACLNN_OP(ctx, InplaceIndexFillTensor, acl_src, dim, acl_index, acl_value);
2050
+ ggml_cann_release_resources(ctx, acl_index, acl_value);
2051
+ }
2052
+
2053
+ static void aclnn_cache_init(ggml_backend_cann_context& ctx, ggml_tensor* dst,
2054
+ aclTensor* acl_cos_repeat_tensor,
2055
+ aclTensor* acl_sin_repeat_tensor,
2056
+ float theta_scale, float freq_scale,
2057
+ float attn_factor, bool is_neox) {
2058
+ // int sin/cos cache, cache has different repeat method depond on
2059
+ // @param.is_neox
2060
+
2061
+ ggml_tensor* src0 = dst->src[0]; // input
2062
+ ggml_tensor* src1 = dst->src[1]; // position
2063
+ ggml_tensor* src2 = dst->src[2]; // freq_factors
2064
+
2065
+ GGML_TENSOR_BINARY_OP_LOCALS
2066
+
2067
+ // theta_scale arange, [0,1,...,ne00/2 - 1]
2068
+ int64_t theta_scale_length = ne00 / 2;
2069
+ ggml_cann_pool_alloc theta_scale_allocator(ctx.pool(),
2070
+ theta_scale_length * sizeof(float_t));
2071
+ void* theta_scale_buffer = theta_scale_allocator.get();
2072
+ int64_t theta_scale_ne[] = {theta_scale_length, 1, 1, 1};
2073
+ size_t theta_scale_nb[] = {sizeof(float_t), sizeof(float_t), sizeof(float_t),
2074
+ theta_scale_length * sizeof(float_t)};
2075
+
2076
+ aclTensor* acl_theta_scale_tensor =
2077
+ ggml_cann_create_tensor(theta_scale_buffer, ACL_FLOAT, sizeof(float_t),
2078
+ theta_scale_ne, theta_scale_nb, GGML_MAX_DIMS);
2079
+ float start = 0;
2080
+ float step = 1;
2081
+ float stop = ne00 / 2;
2082
+ float n_elements = ne00 / 2;
2083
+ aclnn_arange(ctx, acl_theta_scale_tensor, start, stop, step, n_elements);
2084
+
2085
+ // power
2086
+ aclScalar* acl_theta_scale = aclCreateScalar(&theta_scale, aclDataType::ACL_FLOAT);
2087
+ GGML_CANN_CALL_ACLNN_OP(ctx, PowScalarTensor, acl_theta_scale, acl_theta_scale_tensor,
2088
+ acl_theta_scale_tensor);
2089
+
2090
+ // freq_scale
2091
+ if (freq_scale != 1) {
2092
+ aclnn_muls(ctx, acl_theta_scale_tensor, freq_scale, nullptr, true);
2093
+ }
2094
+
2095
+ // freq_factors
2096
+ if (src2) {
2097
+ aclTensor* acl_freq_factors_tensor = ggml_cann_create_tensor(
2098
+ src2->data, ggml_cann_type_mapping(src2->type),
2099
+ ggml_type_size(src2->type), theta_scale_ne, theta_scale_nb, GGML_MAX_DIMS);
2100
+ aclnn_div(ctx, acl_theta_scale_tensor, acl_freq_factors_tensor);
2101
+ ggml_cann_release_resources(ctx, acl_freq_factors_tensor);
2102
+ }
2103
+
2104
+ // position
2105
+ GGML_ASSERT(src1->type == GGML_TYPE_I32);
2106
+ int64_t position_length = src1->ne[0];
2107
+ int64_t position_ne[] = {1, 1, position_length, 1};
2108
+ size_t position_nb[] = {sizeof(int32_t), sizeof(int32_t), sizeof(int32_t),
2109
+ sizeof(int32_t) * position_length};
2110
+ aclTensor* acl_position_tensor = ggml_cann_create_tensor(
2111
+ src1->data, ggml_cann_type_mapping(src1->type),
2112
+ ggml_type_size(src1->type), position_ne, position_nb, GGML_MAX_DIMS);
2113
+
2114
+ // power * position
2115
+ int64_t theta_length = theta_scale_length * position_length;
2116
+ ggml_cann_pool_alloc theta_allocator(ctx.pool(),
2117
+ theta_length * sizeof(float_t));
2118
+ void* theta_buffer = theta_allocator.get();
2119
+ int64_t theta_ne[] = {theta_scale_length, 1, position_length, 1};
2120
+ size_t theta_nb[GGML_MAX_DIMS];
2121
+ theta_nb[0] = sizeof(float_t);
2122
+ for (int i = 1; i < GGML_MAX_DIMS; i++) {
2123
+ theta_nb[i] = theta_nb[i - 1] * theta_ne[i - 1];
2124
+ }
2125
+ aclTensor* acl_theta_tensor =
2126
+ ggml_cann_create_tensor(theta_buffer, ACL_FLOAT, sizeof(float_t),
2127
+ theta_ne, theta_nb, GGML_MAX_DIMS);
2128
+ aclnn_mul(ctx, acl_position_tensor, acl_theta_scale_tensor,
2129
+ acl_theta_tensor);
2130
+
2131
+ // sin/cos
2132
+ ggml_cann_pool_alloc sin_allocator(ctx.pool(),
2133
+ theta_length * sizeof(float_t));
2134
+ void* sin_buffer = sin_allocator.get();
2135
+ aclTensor* acl_sin_tensor = ggml_cann_create_tensor(
2136
+ sin_buffer, ACL_FLOAT, sizeof(float_t), theta_ne, theta_nb,
2137
+ GGML_MAX_DIMS, ACL_FORMAT_ND);
2138
+ aclnn_sin(ctx, acl_theta_tensor, acl_sin_tensor);
2139
+
2140
+ ggml_cann_pool_alloc cos_allocator(ctx.pool(),
2141
+ theta_length * sizeof(float_t));
2142
+ void* cos_buffer = cos_allocator.get();
2143
+ aclTensor* acl_cos_tensor = ggml_cann_create_tensor(
2144
+ cos_buffer, ACL_FLOAT, sizeof(float_t), theta_ne, theta_nb,
2145
+ GGML_MAX_DIMS, ACL_FORMAT_ND);
2146
+ aclnn_cos(ctx, acl_theta_tensor, acl_cos_tensor);
2147
+
2148
+ // attn_factor
2149
+ if (attn_factor != 1) {
2150
+ aclnn_muls(ctx, acl_sin_tensor, attn_factor, nullptr, true);
2151
+ aclnn_muls(ctx, acl_cos_tensor, attn_factor, nullptr, true);
2152
+ }
2153
+
2154
+ // repeat
2155
+ if (is_neox) {
2156
+ int64_t repeatsArray[] = {1, 1, 1, 2};
2157
+ aclnn_repeat(ctx, acl_sin_tensor, acl_sin_repeat_tensor, repeatsArray);
2158
+ aclnn_repeat(ctx, acl_cos_tensor, acl_cos_repeat_tensor, repeatsArray);
2159
+ } else {
2160
+ int64_t num_repeats = 2;
2161
+ int64_t dim = 3;
2162
+ int64_t output_size = theta_scale_length * num_repeats;
2163
+ aclnn_repeat_interleave(ctx, acl_sin_tensor, acl_sin_repeat_tensor, dim,
2164
+ num_repeats, output_size);
2165
+ aclnn_repeat_interleave(ctx, acl_cos_tensor, acl_cos_repeat_tensor, dim,
2166
+ num_repeats, output_size);
2167
+ }
2168
+
2169
+ // release
2170
+ ggml_cann_release_resources(ctx, acl_theta_scale_tensor, acl_position_tensor,
2171
+ acl_theta_tensor, acl_sin_tensor, acl_cos_tensor, acl_theta_scale);
2172
+ }
2173
+
2174
+ #ifdef __cplusplus
2175
+ extern "C" {
2176
+ #endif
2177
+ aclnnStatus aclnnRotaryPositionEmbeddingGetWorkspaceSize(
2178
+ const aclTensor* x, const aclTensor* cos, const aclTensor* sin,
2179
+ int64_t mode, const aclTensor* yOut, uint64_t* workspaceSize,
2180
+ aclOpExecutor** executor);
2181
+ aclnnStatus aclnnRotaryPositionEmbedding(void* workspace,
2182
+ uint64_t workspaceSize,
2183
+ aclOpExecutor* executor,
2184
+ aclrtStream stream);
2185
+ #ifdef __cplusplus
2186
+ }
2187
+ #endif
2188
+
2189
+ void ggml_cann_rope(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
2190
+ // TODO: use ascendc
2191
+ // Only test with LLAMA model.
2192
+ ggml_tensor* src0 = dst->src[0]; // input
2193
+
2194
+ // param
2195
+ float freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow;
2196
+ // const int n_past = ((int32_t *) dst->op_params)[0];
2197
+ const int n_dims = ((int32_t*)dst->op_params)[1];
2198
+ const int mode = ((int32_t*)dst->op_params)[2];
2199
+ // const int n_ctx = ((int32_t *) dst->op_params)[3];
2200
+ const int n_ctx_orig = ((int32_t*)dst->op_params)[4];
2201
+
2202
+ GGML_TENSOR_UNARY_OP_LOCALS
2203
+
2204
+ memcpy(&freq_base, (int32_t*)dst->op_params + 5, sizeof(float));
2205
+ memcpy(&freq_scale, (int32_t*)dst->op_params + 6, sizeof(float));
2206
+ memcpy(&ext_factor, (int32_t*)dst->op_params + 7, sizeof(float));
2207
+ memcpy(&attn_factor, (int32_t*)dst->op_params + 8, sizeof(float));
2208
+ memcpy(&beta_fast, (int32_t*)dst->op_params + 9, sizeof(float));
2209
+ memcpy(&beta_slow, (int32_t*)dst->op_params + 10, sizeof(float));
2210
+
2211
+ // TODO: n_dims <= ne0
2212
+ GGML_ASSERT(n_dims == ne0);
2213
+ GGML_ASSERT(n_dims % 2 == 0);
2214
+ // TODO: ext_factor != 0
2215
+ GGML_ASSERT(ext_factor == 0);
2216
+
2217
+ const float theta_scale = powf(freq_base, -2.0f / n_dims);
2218
+
2219
+ float corr_dims[2];
2220
+ ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast,
2221
+ beta_slow, corr_dims);
2222
+
2223
+ const bool is_neox = mode & GGML_ROPE_TYPE_NEOX;
2224
+
2225
+ // init cos/sin cache
2226
+ ggml_cann_pool_alloc sin_allocator(
2227
+ ctx.pool(), ne00 * ne02 * sizeof(float_t));
2228
+ ggml_cann_pool_alloc cos_allocator(
2229
+ ctx.pool(), ne00 * ne02 * sizeof(float_t));
2230
+ void* sin_buffer = sin_allocator.get();
2231
+ void* cos_buffer = cos_allocator.get();
2232
+
2233
+ int64_t sin_reshape_ne[4] = {ne00, 1, ne02, 1};
2234
+ size_t sin_reshape_nb[GGML_MAX_DIMS];
2235
+ sin_reshape_nb[0] = sizeof(float_t);
2236
+ for (int i = 1; i < GGML_MAX_DIMS; i++) {
2237
+ sin_reshape_nb[i] = sin_reshape_nb[i - 1] * sin_reshape_ne[i - 1];
2238
+ }
2239
+ aclTensor* acl_sin_reshape_tensor =
2240
+ ggml_cann_create_tensor(sin_buffer, ACL_FLOAT, sizeof(float_t),
2241
+ sin_reshape_ne, sin_reshape_nb, GGML_MAX_DIMS);
2242
+ aclTensor* acl_cos_reshape_tensor =
2243
+ ggml_cann_create_tensor(cos_buffer, ACL_FLOAT, sizeof(float_t),
2244
+ sin_reshape_ne, sin_reshape_nb, GGML_MAX_DIMS);
2245
+ aclnn_cache_init(ctx, dst, acl_cos_reshape_tensor, acl_sin_reshape_tensor,
2246
+ theta_scale, freq_scale, attn_factor, is_neox);
2247
+
2248
+ aclTensor* acl_src = ggml_cann_create_tensor(src0);
2249
+ aclTensor* acl_dst = ggml_cann_create_tensor(dst);
2250
+
2251
+ #ifdef ASCEND_310P
2252
+ // Special ROPE operation for 310P
2253
+
2254
+ // roll input
2255
+ void* input_roll_buffer;
2256
+ aclTensor* acl_minus_one_tensor;
2257
+ void* minus_one_scale_buffer = nullptr;
2258
+ ggml_cann_pool_alloc roll_allocator(ctx.pool(), ggml_nbytes(src0));
2259
+ ggml_cann_pool_alloc minus_one_scale_allocator(
2260
+ ctx.pool(), sizeof(float_t) * src0->ne[0]);
2261
+ if (!is_neox) {
2262
+ // roll input: [q0,q1,q2,q3,...] -> [q1,q0,q3,q2,...]
2263
+ input_roll_buffer = roll_allocator.get();
2264
+ int64_t input_roll_ne[4] = {2, src0->ne[1] * (src0->ne[0] / 2),
2265
+ src0->ne[2], src0->ne[3]};
2266
+ size_t input_roll_nb[GGML_MAX_DIMS];
2267
+ input_roll_nb[0] = ggml_type_size(src0->type);
2268
+ for (int i = 1; i < GGML_MAX_DIMS; i++) {
2269
+ input_roll_nb[i] = input_roll_nb[i - 1] * input_roll_ne[i - 1];
2270
+ }
2271
+ aclTensor* acl_input_roll_tensor = ggml_cann_create_tensor(
2272
+ input_roll_buffer, ggml_cann_type_mapping(src0->type),
2273
+ ggml_type_size(src0->type), input_roll_ne, input_roll_nb,
2274
+ GGML_MAX_DIMS);
2275
+ aclTensor* acl_input_tensor = ggml_cann_create_tensor(
2276
+ src0->data, ggml_cann_type_mapping(src0->type),
2277
+ ggml_type_size(src0->type), input_roll_ne, input_roll_nb,
2278
+ GGML_MAX_DIMS);
2279
+
2280
+ int64_t shifts[] = {1};
2281
+ int64_t dims[] = {3};
2282
+ aclnn_roll(ctx, acl_input_tensor, acl_input_roll_tensor, shifts, dims);
2283
+ ggml_cann_release_resources(ctx, acl_input_roll_tensor, acl_input_tensor);
2284
+
2285
+ // init [-1, 1, -1, 1, ...]
2286
+ minus_one_scale_buffer = minus_one_scale_allocator.get();
2287
+
2288
+ int64_t minus_one_ne[4] = {src0->ne[0], 1, 1, 1};
2289
+ size_t minus_one_nb[GGML_MAX_DIMS];
2290
+ minus_one_nb[0] = sizeof(float_t);
2291
+ for (int i = 1; i < GGML_MAX_DIMS; i++) {
2292
+ minus_one_nb[i] = minus_one_nb[i - 1] * minus_one_ne[i - 1];
2293
+ }
2294
+ acl_minus_one_tensor = aclnn_values(
2295
+ ctx, minus_one_scale_buffer, sizeof(float_t) * src0->ne[0],
2296
+ minus_one_ne, GGML_MAX_DIMS, ACL_FLOAT, sizeof(float_t), 1);
2297
+ int64_t dim = 3;
2298
+ int64_t* index = new int64_t[src0->ne[0]];
2299
+ for (int i = 0; i < src0->ne[0]; i++) {
2300
+ index[i] = i / 2 * 2;
2301
+ }
2302
+ int64_t index_num = src0->ne[0];
2303
+ float value = -1;
2304
+ aclnn_index_fill_tensor(ctx, acl_minus_one_tensor, dim, index,
2305
+ index_num, value);
2306
+ } else {
2307
+ // roll input: [q0,q1,q2,...] ->
2308
+ // [q_half,q_half+1,...,q_end,q0,q1,...q_half-1]
2309
+ input_roll_buffer = roll_allocator.get();
2310
+ aclTensor* acl_input_roll_tensor = ggml_cann_create_tensor(
2311
+ input_roll_buffer, ggml_cann_type_mapping(src0->type),
2312
+ ggml_type_size(src0->type), src0->ne, src0->nb, GGML_MAX_DIMS);
2313
+ aclTensor* acl_input_tensor = ggml_cann_create_tensor(src0);
2314
+
2315
+ int64_t shifts[] = {src0->ne[0] / 2};
2316
+ int64_t dims[] = {3};
2317
+ aclnn_roll(ctx, acl_input_tensor, acl_input_roll_tensor, shifts, dims);
2318
+
2319
+ ggml_cann_release_resources(ctx, acl_input_roll_tensor, acl_input_tensor);
2320
+ // init [-1, -1, -1, 1, 1,1,...]
2321
+ minus_one_scale_buffer = minus_one_scale_allocator.get();
2322
+ int64_t minus_one_ne[4] = {src0->ne[0], 1, 1, 1};
2323
+ size_t minus_one_nb[GGML_MAX_DIMS];
2324
+ minus_one_nb[0] = sizeof(float_t);
2325
+ for (int i = 1; i < GGML_MAX_DIMS; i++) {
2326
+ minus_one_nb[i] = minus_one_nb[i - 1] * minus_one_ne[i - 1];
2327
+ }
2328
+ acl_minus_one_tensor = aclnn_values(
2329
+ ctx, minus_one_scale_buffer, sizeof(float_t) * src0->ne[0],
2330
+ minus_one_ne, GGML_MAX_DIMS, ACL_FLOAT, sizeof(float_t), 1);
2331
+ // -1 * first half
2332
+ int64_t first_half_ne[4] = {src0->ne[0] / 2, 1, 1, 1};
2333
+ size_t first_half_nb[GGML_MAX_DIMS];
2334
+ first_half_nb[0] = sizeof(float_t);
2335
+ for (int i = 1; i < GGML_MAX_DIMS; i++) {
2336
+ first_half_nb[i] = first_half_nb[i - 1] * first_half_ne[i - 1];
2337
+ }
2338
+ aclTensor* acl_first_half_tensor = ggml_cann_create_tensor(
2339
+ minus_one_scale_buffer, ACL_FLOAT, sizeof(float_t), first_half_ne,
2340
+ first_half_nb, GGML_MAX_DIMS);
2341
+ bool inplace = true;
2342
+ float scale = -1;
2343
+ aclnn_muls(ctx, acl_first_half_tensor, scale, nullptr, inplace);
2344
+ ggml_cann_release_resources(ctx, acl_first_half_tensor);
2345
+ }
2346
+
2347
+ // TODO: n_dims < ne0
2348
+ GGML_ASSERT(n_dims == src0->ne[0]);
2349
+
2350
+ // input * scale
2351
+ ggml_cann_pool_alloc roll_mul_scale_allocator(ctx.pool(),
2352
+ ggml_nbytes(src0));
2353
+ void* input_roll_mul_scale_buffer = roll_mul_scale_allocator.get();
2354
+ size_t input_nb[GGML_MAX_DIMS];
2355
+ input_nb[0] = ggml_type_size(src0->type);
2356
+ for (int i = 1; i < GGML_MAX_DIMS; i++) {
2357
+ input_nb[i] = input_nb[i - 1] * src0->ne[i - 1];
2358
+ }
2359
+ aclTensor* acl_input_roll_mul_scale_tensor = ggml_cann_create_tensor(
2360
+ input_roll_mul_scale_buffer, ggml_cann_type_mapping(src0->type),
2361
+ ggml_type_size(src0->type), src0->ne, input_nb, GGML_MAX_DIMS);
2362
+ aclTensor* acl_input_roll_reshape_tensor = ggml_cann_create_tensor(
2363
+ input_roll_buffer, ggml_cann_type_mapping(src0->type),
2364
+ ggml_type_size(src0->type), src0->ne, input_nb, GGML_MAX_DIMS);
2365
+
2366
+ aclnn_mul(ctx, acl_input_roll_reshape_tensor, acl_minus_one_tensor,
2367
+ acl_input_roll_mul_scale_tensor);
2368
+
2369
+ // output
2370
+ void* output_fp32_buffer;
2371
+ if (src0->type == GGML_TYPE_F32) {
2372
+ aclnn_mul(ctx, acl_src, acl_cos_reshape_tensor);
2373
+ aclnn_mul(ctx, acl_input_roll_mul_scale_tensor,
2374
+ acl_sin_reshape_tensor);
2375
+ aclnn_add(ctx, acl_src, acl_input_roll_mul_scale_tensor, acl_dst);
2376
+ // TODO: ne0 != n_dims in mode2
2377
+ } else if (src0->type == GGML_TYPE_F16) {
2378
+ size_t input_fp32_nb[GGML_MAX_DIMS];
2379
+ input_fp32_nb[0] = sizeof(float_t);
2380
+ for (int i = 1; i < GGML_MAX_DIMS; i++) {
2381
+ input_fp32_nb[i] = input_fp32_nb[i - 1] * dst->ne[i - 1];
2382
+ }
2383
+ ggml_cann_pool_alloc fp32_allocator1(
2384
+ ctx.pool(), ggml_nelements(dst) * sizeof(float_t));
2385
+ void* input_fp32_buffer1 = fp32_allocator1.get();
2386
+ aclTensor* input_fp32_tensor1 = ggml_cann_create_tensor(
2387
+ input_fp32_buffer1, ACL_FLOAT, sizeof(float_t), dst->ne,
2388
+ input_fp32_nb, GGML_MAX_DIMS);
2389
+ ggml_cann_pool_alloc fp32_allocator2(
2390
+ ctx.pool(), ggml_nelements(dst) * sizeof(float_t));
2391
+ void* input_fp32_buffer2 = fp32_allocator2.get();
2392
+ aclTensor* input_fp32_tensor2 = ggml_cann_create_tensor(
2393
+ input_fp32_buffer2, ACL_FLOAT, sizeof(float_t), dst->ne,
2394
+ input_fp32_nb, GGML_MAX_DIMS);
2395
+
2396
+ ggml_cann_pool_alloc fp32_allocator(
2397
+ ctx.pool(), ggml_nelements(dst) * sizeof(float_t));
2398
+ output_fp32_buffer = fp32_allocator.get();
2399
+ aclTensor* output_fp32_tensor = ggml_cann_create_tensor(
2400
+ output_fp32_buffer, ACL_FLOAT, sizeof(float_t), dst->ne,
2401
+ input_fp32_nb, GGML_MAX_DIMS);
2402
+ aclnn_mul(ctx, acl_src, acl_cos_reshape_tensor, input_fp32_tensor1);
2403
+ aclnn_mul(ctx, acl_input_roll_mul_scale_tensor, acl_sin_reshape_tensor,
2404
+ input_fp32_tensor2);
2405
+ aclnn_add(ctx, input_fp32_tensor1, input_fp32_tensor2,
2406
+ output_fp32_tensor);
2407
+ aclnn_cast(ctx, output_fp32_tensor, acl_dst, ACL_FLOAT16);
2408
+
2409
+ ggml_cann_release_resources(ctx, input_fp32_tensor1, input_fp32_tensor2,
2410
+ output_fp32_tensor, acl_sin_reshape_tensor,
2411
+ acl_minus_one_tensor, acl_input_roll_mul_scale_tensor,
2412
+ acl_input_roll_reshape_tensor, acl_src);
2413
+ }
2414
+ return;
2415
+ #endif
2416
+
2417
+ // ggml_mode = 0 --> aclnn_model = 1
2418
+ int64_t acl_mode = mode == 0 ? 1 : mode;
2419
+
2420
+ switch (src0->type) {
2421
+ case GGML_TYPE_F32: {
2422
+ GGML_CANN_CALL_ACLNN_OP(ctx, RotaryPositionEmbedding, acl_src,
2423
+ acl_cos_reshape_tensor, acl_sin_reshape_tensor, acl_mode, acl_dst);
2424
+ break;
2425
+ }
2426
+ case GGML_TYPE_F16: {
2427
+ ggml_cann_pool_alloc src_trans_allocator(
2428
+ ctx.pool(), ggml_nelements(src0) * sizeof(float));
2429
+ void* src_trans_buffer = src_trans_allocator.get();
2430
+ ggml_cann_pool_alloc dst_trans_allocator(
2431
+ ctx.pool(), ggml_nelements(dst) * sizeof(float));
2432
+ void* dst_trans_buffer = dst_trans_allocator.get();
2433
+
2434
+ size_t src_trans_nb[GGML_MAX_DIMS];
2435
+ src_trans_nb[0] = sizeof(float);
2436
+ for (int i = 1; i < GGML_MAX_DIMS; i++) {
2437
+ src_trans_nb[i] = src_trans_nb[i - 1] * src0->ne[i - 1];
2438
+ }
2439
+
2440
+ aclTensor* acl_src_trans_tensor = ggml_cann_create_tensor(
2441
+ src_trans_buffer, ACL_FLOAT, sizeof(float), src0->ne, src_trans_nb,
2442
+ GGML_MAX_DIMS);
2443
+ aclTensor* acl_dst_trans_tensor = ggml_cann_create_tensor(
2444
+ dst_trans_buffer, ACL_FLOAT, sizeof(float), dst->ne, src_trans_nb,
2445
+ GGML_MAX_DIMS);
2446
+
2447
+ aclnn_cast(ctx, acl_src, acl_src_trans_tensor, ACL_FLOAT);
2448
+
2449
+ GGML_CANN_CALL_ACLNN_OP(ctx, RotaryPositionEmbedding, acl_src_trans_tensor,
2450
+ acl_cos_reshape_tensor, acl_sin_reshape_tensor, acl_mode,
2451
+ acl_dst_trans_tensor);
2452
+
2453
+ aclnn_cast(ctx, acl_dst_trans_tensor, acl_dst, ACL_FLOAT16);
2454
+
2455
+ ggml_cann_release_resources(ctx, acl_src_trans_tensor,
2456
+ acl_dst_trans_tensor);
2457
+ break;
2458
+ }
2459
+ default:
2460
+ GGML_ABORT("Unsupported tensor type for GGML_OP_ROPE");
2461
+ break;
2462
+ }
2463
+ ggml_cann_release_resources(ctx, acl_cos_reshape_tensor,
2464
+ acl_sin_reshape_tensor, acl_src, acl_dst);
2465
+ }
2466
+
2467
+
2468
+ void ggml_cann_argmax(ggml_backend_cann_context& ctx, ggml_tensor* dst){
2469
+ ggml_tensor * src0 = dst->src[0];
2470
+
2471
+ aclTensor* acl_src = ggml_cann_create_tensor(src0);
2472
+ aclTensor* acl_dst = ggml_cann_create_tensor(dst, dst->ne, dst->nb, 3);
2473
+
2474
+ GGML_CANN_CALL_ACLNN_OP(ctx, ArgMax, acl_src, 3, false, acl_dst);
2475
+
2476
+ ggml_cann_release_resources(ctx, acl_src, acl_dst);
2477
+ }
2478
+
2479
+ void ggml_cann_conv_transpose_1d(ggml_backend_cann_context& ctx, ggml_tensor* dst){
2480
+ ggml_tensor * src0 = dst->src[0];
2481
+ ggml_tensor * src1 = dst->src[1];
2482
+
2483
+ // stride
2484
+ int64_t s0 = ((const int32_t*)(dst->op_params))[0];
2485
+
2486
+ aclTensor* acl_input = ggml_cann_create_tensor(src1, src1->ne, src1->nb, 3, ACL_FORMAT_NCL);
2487
+ aclTensor* acl_weight = ggml_cann_create_tensor(src0, src0->ne, src0->nb, 3, ACL_FORMAT_NCL);
2488
+ aclTensor* acl_dst = ggml_cann_create_tensor(dst, dst->ne, dst->nb, 3, ACL_FORMAT_NCL);
2489
+
2490
+ int64_t strideVal[1];
2491
+ strideVal[0] = s0;
2492
+ aclIntArray *stride = aclCreateIntArray(strideVal, 1);
2493
+ int64_t paddingVal[] = {0};
2494
+ aclIntArray *padding = aclCreateIntArray(paddingVal, 1);
2495
+ int64_t dilationVal[] = {1};
2496
+ aclIntArray *dilation = aclCreateIntArray(dilationVal, 1);
2497
+ bool transposed = true;
2498
+ int64_t groups = 1;
2499
+ int8_t cubeMathType = 0;
2500
+
2501
+ #ifdef ASCEND_310P
2502
+ cubeMathType = 1;
2503
+ #endif
2504
+
2505
+ GGML_CANN_CALL_ACLNN_OP(ctx, Convolution, acl_input, acl_weight, nullptr, stride,
2506
+ padding, dilation, transposed, padding, groups, acl_dst, cubeMathType);
2507
+
2508
+ ggml_cann_release_resources(ctx, acl_weight, acl_dst, stride, padding, dilation);
2509
+ }
2510
+
2511
+ void ggml_cann_elu(ggml_backend_cann_context& ctx, ggml_tensor* dst){
2512
+ ggml_tensor * src0 = dst->src[0];
2513
+
2514
+ aclTensor* acl_input = ggml_cann_create_tensor(src0);
2515
+ aclTensor* acl_dst = ggml_cann_create_tensor(dst);
2516
+
2517
+ float alphaValue = 1.0f;
2518
+ aclScalar* alpha = nullptr;
2519
+ alpha = aclCreateScalar(&alphaValue, aclDataType::ACL_FLOAT);
2520
+
2521
+ GGML_CANN_CALL_ACLNN_OP(ctx, Elu, acl_input, alpha, alpha, alpha,
2522
+ acl_dst);
2523
+
2524
+ ggml_cann_release_resources(ctx, acl_input, acl_dst, alpha);
2525
+ }
2526
+
2527
+ void ggml_cann_mean(ggml_backend_cann_context& ctx, ggml_tensor* dst){
2528
+ ggml_tensor * src0 = dst->src[0];
2529
+
2530
+ aclTensor* acl_src = ggml_cann_create_tensor(src0);
2531
+ aclTensor* acl_dst = ggml_cann_create_tensor(dst);
2532
+
2533
+ int64_t reduceDimValue[] = {3};
2534
+ aclIntArray* reduceDim = aclCreateIntArray(reduceDimValue, 1);
2535
+ bool keepDim = true;
2536
+
2537
+ GGML_CANN_CALL_ACLNN_OP(ctx, Mean, acl_src, reduceDim, keepDim, ACL_FLOAT, acl_dst);
2538
+
2539
+ ggml_cann_release_resources(ctx, acl_src, acl_dst, reduceDim);
2540
+ }
2541
+
2542
+ void ggml_cann_pad_reflect_1d(ggml_backend_cann_context& ctx, ggml_tensor* dst){
2543
+ ggml_tensor * src0 = dst->src[0];
2544
+ int32_t *opts = (int32_t *) dst->op_params;
2545
+ int64_t paddingsArray[2] = {opts[0], opts[1]};
2546
+ aclIntArray* paddings = aclCreateIntArray(paddingsArray, 2);
2547
+
2548
+ for (int64_t i = 0; i < src0->ne[3]; i++) {
2549
+ aclTensor* acl_src = ggml_cann_create_tensor(
2550
+ (char*)src0->data + i * src0->ne[3],
2551
+ ggml_cann_type_mapping(src0->type), ggml_element_size(src0),
2552
+ src0->ne, src0->nb, 3);
2553
+
2554
+ aclTensor* acl_dst = ggml_cann_create_tensor(
2555
+ (char*)dst->data + i * src0->ne[3],
2556
+ ggml_cann_type_mapping(dst->type), ggml_element_size(dst),
2557
+ dst->ne, dst->nb, 3);
2558
+
2559
+ GGML_CANN_CALL_ACLNN_OP(ctx, ReflectionPad1d, acl_src, paddings, acl_dst);
2560
+
2561
+ ggml_cann_release_resources(ctx, acl_src, acl_dst);
2562
+ }
2563
+ ggml_cann_release_resources(ctx, paddings);
2564
+ }
2565
+
2566
+ void ggml_cann_count_equal(ggml_backend_cann_context& ctx, ggml_tensor* dst){
2567
+ ggml_tensor * src0 = dst->src[0];
2568
+ ggml_tensor * src1 = dst->src[1];
2569
+
2570
+ aclTensor* acl_self = ggml_cann_create_tensor(src0);
2571
+ aclTensor* acl_other = ggml_cann_create_tensor(src1);
2572
+
2573
+ GGML_CANN_CALL_ACLNN_OP(ctx, InplaceEqTensor, acl_self, acl_other);
2574
+
2575
+ ggml_cann_sum(ctx, dst);
2576
+
2577
+ ggml_cann_release_resources(ctx, acl_self, acl_other);
2578
+ }
2579
+
2580
+ void ggml_cann_step(ggml_backend_cann_context& ctx, ggml_tensor* dst){
2581
+ ggml_tensor * src0 = dst->src[0];
2582
+
2583
+ aclTensor* acl_src = ggml_cann_create_tensor(src0);
2584
+ aclTensor* acl_dst = ggml_cann_create_tensor(dst);
2585
+
2586
+ float alphaValue = 0.0f;
2587
+ aclScalar* alpha = nullptr;
2588
+ alpha = aclCreateScalar(&alphaValue, aclDataType::ACL_FLOAT);
2589
+
2590
+ GGML_CANN_CALL_ACLNN_OP(ctx, GtScalar, acl_src, alpha, acl_dst);
2591
+
2592
+ ggml_cann_release_resources(ctx, acl_src, acl_dst, alpha);
2593
+ }
2594
+
2595
+ /**
2596
+ * @brief Performs expert-specific matrix multiplication (MoE) with
2597
+ * floating-point precision using the CANN backend.
2598
+ *
2599
+ * This function executes a matrix multiplication operation tailored for
2600
+ * Mixture of Experts (MoE) models, where the input tensor is multiplied
2601
+ * with expert-specific weight matrices. It uses the CANN backend for
2602
+ * efficient computation and stores the result in the destination tensor `dst`.
2603
+ * The operation may leverage identity-based optimizations or routing masks
2604
+ * as part of sparse expert selection.
2605
+ *
2606
+ * @param ctx The context for executing CANN backend operations.
2607
+ * @param dst The destination tensor where the MoE multiplication result
2608
+ * will be stored.
2609
+ *
2610
+ * @note This function assumes floating-point data types and is designed for
2611
+ * MoE architectures, possibly involving sparse expert routing.
2612
+ */
2613
+ static void ggml_cann_mul_mat_id_fp(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
2614
+ //dst [M, K, N, 1]
2615
+ ggml_tensor * src0 = dst->src[0]; //src0 [D, M, A, 1]
2616
+ ggml_tensor * src1 = dst->src[1]; //src1 [D, B, N, 1], B = K or B = 1
2617
+ ggml_tensor * ids = dst->src[2]; //ids [K, N]
2618
+
2619
+ GGML_TENSOR_BINARY_OP_LOCALS
2620
+
2621
+ // copy index from npu to cpu
2622
+ int64_t n_as = ne02; // A
2623
+ int64_t n_ids = ids->ne[0]; // K
2624
+
2625
+ std::vector<char> ids_host(ggml_nbytes(ids));
2626
+ ggml_cann_async_memcpy(ctx, ids_host.data(), ids->data, ggml_nbytes(ids),
2627
+ ACL_MEMCPY_DEVICE_TO_HOST);
2628
+ ACL_CHECK(aclrtSynchronizeStream(ctx.stream()));
2629
+
2630
+ char * src0_original = (char *) src0->data;
2631
+ char * src1_original = (char *) src1->data;
2632
+ char * dst_original = (char *) dst->data;
2633
+ size_t ori_src0_nb[4] = {nb00, nb01, nb02, nb03};
2634
+
2635
+ // src0 is F16, src1 is F32, dst is F32
2636
+ ggml_cann_pool_alloc src0_cast_allocator;
2637
+ if (src0->type == GGML_TYPE_F16) {
2638
+ src0_cast_allocator.alloc(ctx.pool(), sizeof(float) * ggml_nelements(src0));
2639
+ void* src0_cast_buf = src0_cast_allocator.get();
2640
+
2641
+ size_t cast_nb[GGML_MAX_DIMS];
2642
+ cast_nb[0] = sizeof(float_t);
2643
+ for (int i = 1; i < GGML_MAX_DIMS; i++) {
2644
+ cast_nb[i] = cast_nb[i - 1] * src0->ne[i - 1];
2645
+ }
2646
+
2647
+ aclTensor* acl_src0_f16 = ggml_cann_create_tensor(src0);
2648
+ aclTensor* acl_cast = ggml_cann_create_tensor(src0_cast_buf,
2649
+ ACL_FLOAT, sizeof(float), src0->ne, cast_nb, 4);
2650
+ GGML_CANN_CALL_ACLNN_OP(ctx, Cast, acl_src0_f16, ACL_FLOAT, acl_cast);
2651
+ ggml_cann_release_resources(ctx, acl_cast, acl_src0_f16);
2652
+
2653
+ src0_original = (char *) src0_cast_buf;
2654
+ memcpy(ori_src0_nb, cast_nb, sizeof(ori_src0_nb));
2655
+ }
2656
+
2657
+ std::vector<aclTensor*> src0_tensor_vec;
2658
+ std::vector<aclTensor*> src1_tensor_vec;
2659
+ std::vector<aclTensor*> dst_tensor_vec;
2660
+ for (int64_t iid1 = 0; iid1 < ids->ne[1]; iid1++) {
2661
+ for (int64_t id = 0; id < n_ids; id++) {
2662
+ // src0_row [M, D] -> weight && permute
2663
+ int64_t src0_ne[2] = {ne01, ne00};
2664
+ size_t src0_nb[2] = {ori_src0_nb[1], ori_src0_nb[0]};
2665
+ // src1_row [D, 1] -> input
2666
+ int64_t src1_ne[2] = {ne10, 1};
2667
+ size_t src1_nb[2] = {nb10, nb11};
2668
+ // dst_row [M, 1] -> out
2669
+ int64_t dst_ne[2] = {ne0, 1};
2670
+ size_t dst_nb[2] = {nb0, nb1};
2671
+
2672
+ // expert index
2673
+ int32_t i02 = *(int32_t *) (ids_host.data() + iid1*ids->nb[1] + id*ids->nb[0]);
2674
+ GGML_ASSERT(i02 >= 0 && i02 < n_as);
2675
+
2676
+ // If B = 1 (broadcast), always use 0; otherwise, use id.
2677
+ int64_t i11 = (ne11 == 1 ? 0 : id);
2678
+ int64_t i12 = iid1;
2679
+
2680
+ int64_t i1 = id;
2681
+ int64_t i2 = i12;
2682
+
2683
+ void* src0_tmp_ptr = src0_original + i02*ori_src0_nb[2];
2684
+ void* src1_tmp_ptr = src1_original + i11*nb11 + i12*nb12;
2685
+ void* dst_tmp_ptr = dst_original + i1*nb1 + i2*nb2;
2686
+
2687
+ aclTensor* acl_src0 = ggml_cann_create_tensor(src0_tmp_ptr,
2688
+ ACL_FLOAT, sizeof(float),
2689
+ src0_ne, src0_nb, 2);
2690
+ aclTensor* acl_src1 = ggml_cann_create_tensor(src1_tmp_ptr,
2691
+ ACL_FLOAT, sizeof(float),
2692
+ src1_ne, src1_nb, 2);
2693
+ aclTensor* acl_dst = ggml_cann_create_tensor(dst_tmp_ptr,
2694
+ ACL_FLOAT, sizeof(float),
2695
+ dst_ne, dst_nb, 2);
2696
+
2697
+ src0_tensor_vec.push_back(acl_src0);
2698
+ src1_tensor_vec.push_back(acl_src1);
2699
+ dst_tensor_vec.push_back(acl_dst);
2700
+ }
2701
+ }
2702
+
2703
+ size_t GROUP_SIZE = 128;
2704
+ // GroupedMatmulV2 required tensor_list.size < 128
2705
+ for (size_t i = 0; i < src0_tensor_vec.size(); i += GROUP_SIZE) {
2706
+ // split and call GroupedMatmulV2
2707
+ size_t end = std::min(i + GROUP_SIZE, src0_tensor_vec.size());
2708
+ std::vector<aclTensor*> src0_tensor_vec_split(src0_tensor_vec.begin() + i, src0_tensor_vec.begin() + end);
2709
+ std::vector<aclTensor*> src1_tensor_vec_split(src1_tensor_vec.begin() + i, src1_tensor_vec.begin() + end);
2710
+ std::vector<aclTensor*> dst_tensor_vec_split(dst_tensor_vec.begin() + i, dst_tensor_vec.begin() + end);
2711
+
2712
+ aclTensorList* src0_tensor_list = aclCreateTensorList(src0_tensor_vec_split.data(), src0_tensor_vec_split.size());
2713
+ aclTensorList* src1_tensor_list = aclCreateTensorList(src1_tensor_vec_split.data(), src1_tensor_vec_split.size());
2714
+ aclTensorList* dst_tensor_list = aclCreateTensorList(dst_tensor_vec_split.data(), dst_tensor_vec_split.size());
2715
+
2716
+ GGML_CANN_CALL_ACLNN_OP(ctx, GroupedMatmulV2, src1_tensor_list, src0_tensor_list,
2717
+ nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, 0, -1, dst_tensor_list);
2718
+
2719
+ ggml_cann_release_resources(ctx, src0_tensor_list, src1_tensor_list, dst_tensor_list);
2720
+ }
2721
+ return;
2722
+ }
2723
+
2724
+ /**
2725
+ * @brief Performs expert-specific matrix multiplication (MoE) with
2726
+ * quantized precision using the CANN backend.
2727
+ *
2728
+ * This function executes a matrix multiplication operation tailored for
2729
+ * Mixture of Experts (MoE) models, where the input tensor is multiplied
2730
+ * with expert-specific quantized weight matrices. It leverages the CANN
2731
+ * backend to perform efficient low-precision computations and stores the
2732
+ * quantized result in the destination tensor `dst`.
2733
+ *
2734
+ * Quantization techniques reduce memory footprint and improve performance
2735
+ * by using lower-bit representations (e.g., int8) instead of floating-point.
2736
+ * This function is designed to work with such formats and may incorporate
2737
+ * optimizations like identity-based fast paths or routing masks for sparse
2738
+ * expert selection.
2739
+ *
2740
+ * @param ctx The context for executing CANN backend operations.
2741
+ * @param dst The destination tensor where the quantized MoE multiplication result
2742
+ * will be stored.
2743
+ *
2744
+ * @note This function assumes quantized data types and is designed for
2745
+ * MoE architectures with potential sparse expert routing.
2746
+ */
2747
+ static void ggml_cann_mul_mat_id_quant(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
2748
+ // TODO: Use aclnnGroupedMatMul
2749
+ //dst [M, K, N, 1]
2750
+ ggml_tensor * src0 = dst->src[0]; //src0 [D, M, A, 1]
2751
+ ggml_tensor * src1 = dst->src[1]; //src1 [D, B, N, 1], B = K or B = 1
2752
+ ggml_tensor * ids = dst->src[2]; //ids [K, N]
2753
+
2754
+ GGML_TENSOR_BINARY_OP_LOCALS
2755
+
2756
+ // copy index from npu to cpu
2757
+ int64_t n_as = ne02; // A
2758
+ int64_t n_ids = ids->ne[0]; // K
2759
+
2760
+ std::vector<char> ids_host(ggml_nbytes(ids));
2761
+ ggml_cann_async_memcpy(ctx, ids_host.data(), ids->data, ggml_nbytes(ids),
2762
+ ACL_MEMCPY_DEVICE_TO_HOST);
2763
+ ACL_CHECK(aclrtSynchronizeStream(ctx.stream()));
2764
+
2765
+ char * src0_original = (char *) src0->data;
2766
+ char * src1_original = (char *) src1->data;
2767
+ char * dst_original = (char *) dst->data;
2768
+
2769
+ ggml_tensor src0_row = *src0;
2770
+ ggml_tensor src1_row = *src1;
2771
+ ggml_tensor dst_row = *dst;
2772
+
2773
+ const enum ggml_type type = dst->src[0]->type;
2774
+ float weight_elem_size;
2775
+ if (type == GGML_TYPE_Q4_0) {
2776
+ weight_elem_size = float(sizeof(uint8_t)) / 2;
2777
+ } else if (type == GGML_TYPE_Q8_0) {
2778
+ weight_elem_size = float(sizeof(uint8_t));
2779
+ } else {
2780
+ GGML_ABORT("MUL_MAT_ID only support quant type Q4_0 and Q8_0 ");
2781
+ }
2782
+
2783
+ // src0_row [D, M, 1, 1] weight without permute
2784
+ src0_row.ne[2] = 1;
2785
+ src0_row.ne[3] = 1;
2786
+ src0_row.nb[0] = weight_elem_size;
2787
+ src0_row.nb[1] = weight_elem_size * ne00;
2788
+ src0_row.nb[2] = weight_elem_size * ne00;
2789
+ src0_row.nb[3] = weight_elem_size * ne00;
2790
+ size_t weight_stride = ne00 * ne01 * weight_elem_size;
2791
+ size_t weight_size = weight_stride * ne02 * ne03;
2792
+
2793
+ // scale [D, M, 1, 1] -> scale && permute
2794
+ size_t scale_elem_size = sizeof(uint16_t);
2795
+ size_t scale_stride = src0->ne[1] * src0->ne[0] / QK8_0 * scale_elem_size;
2796
+
2797
+ // src1_row [D, 1, 1, 1] -> input
2798
+ src1_row.ne[1] = 1;
2799
+ src1_row.ne[2] = 1;
2800
+ src1_row.ne[3] = 1;
2801
+ src1_row.nb[2] = nb11;
2802
+ src1_row.nb[3] = nb11;
2803
+
2804
+ // dst_row [M, 1, 1, 1] -> out
2805
+ dst_row.ne[1] = 1;
2806
+ dst_row.ne[2] = 1;
2807
+ dst_row.ne[3] = 1;
2808
+ dst_row.nb[2] = nb1;
2809
+ dst_row.nb[3] = nb1;
2810
+
2811
+ //create weight for one row
2812
+ ggml_cann_pool_alloc weight_allocator(ctx.pool());
2813
+ void* weight_buffer = weight_allocator.alloc(nb02);
2814
+ for (int64_t iid1 = 0; iid1 < ids->ne[1]; iid1++) {
2815
+ for (int64_t id = 0; id < n_ids; id++) {
2816
+ // expert index
2817
+ int32_t i02 = *(int32_t *) (ids_host.data() + iid1*ids->nb[1] + id*ids->nb[0]);
2818
+ GGML_ASSERT(i02 >= 0 && i02 < n_as);
2819
+
2820
+ // If B = 1 (broadcast), always use 0; otherwise, use id.
2821
+ int64_t i11 = (ne11 == 1 ? 0 : id);
2822
+ int64_t i12 = iid1;
2823
+
2824
+ int64_t i1 = id;
2825
+ int64_t i2 = i12;
2826
+
2827
+ void* src0_tmp_ptr = src0_original + i02*weight_stride;
2828
+ void* scale_tmp_ptr = src0_original + weight_size + i02*scale_stride;
2829
+ void* src1_tmp_ptr = src1_original + i11*nb11 + i12*nb12;
2830
+ void* dst_tmp_ptr = dst_original + i1*nb1 + i2*nb2;
2831
+
2832
+ // mem cpy
2833
+ ggml_cann_async_memcpy(ctx, weight_buffer, src0_tmp_ptr, weight_stride,
2834
+ ACL_MEMCPY_DEVICE_TO_DEVICE);
2835
+ void* scale_buffer = (char*)weight_buffer + weight_stride;
2836
+ ggml_cann_async_memcpy(ctx, scale_buffer, scale_tmp_ptr, scale_stride,
2837
+ ACL_MEMCPY_DEVICE_TO_DEVICE);
2838
+
2839
+ src0_row.data = weight_buffer;
2840
+ src1_row.data = src1_tmp_ptr;
2841
+ dst_row.data = dst_tmp_ptr;
2842
+ dst_row.src[0] = &src0_row;
2843
+ dst_row.src[1] = &src1_row;
2844
+
2845
+ ggml_cann_mul_mat(ctx, &dst_row);
2846
+ }
2847
+ }
2848
+ return;
2849
+ }
2850
+
2851
+ void ggml_cann_mul_mat_id(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
2852
+ const enum ggml_type type = dst->src[0]->type;
2853
+ switch (type) {
2854
+ case GGML_TYPE_F32:
2855
+ case GGML_TYPE_F16:
2856
+ ggml_cann_mul_mat_id_fp(ctx, dst);
2857
+ break;
2858
+ case GGML_TYPE_Q4_0:
2859
+ case GGML_TYPE_Q8_0:
2860
+ ggml_cann_mul_mat_id_quant(ctx, dst);
2861
+ break;
2862
+ default:
2863
+ GGML_ABORT("Unsupported type for mul_mat_id");
2864
+ break;
2865
+ }
2866
+ }
2867
+
2868
+ void ggml_cann_flash_attn_ext(ggml_backend_cann_context& ctx, ggml_tensor* dst){
2869
+
2870
+ ggml_tensor* src0 = dst->src[0]; // q, fp32
2871
+ ggml_tensor* src1 = dst->src[1]; // k, fp16
2872
+ ggml_tensor* src2 = dst->src[2]; // v, fp16
2873
+ ggml_tensor* src3 = dst->src[3]; // mask, fp16
2874
+
2875
+ float maxBias = 0.0f;
2876
+ float scaleValue = 1.0f;
2877
+ float logitSoftcap = 0.0f;
2878
+ memcpy(&scaleValue, (float*)dst->op_params + 0, sizeof(float));
2879
+ memcpy(&maxBias, (float*)dst->op_params + 1, sizeof(float));
2880
+ memcpy(&logitSoftcap, (float*)dst->op_params + 2, sizeof(float));
2881
+
2882
+ if(logitSoftcap == 0.0f){
2883
+ size_t faElemSize = sizeof(uint16_t);
2884
+ auto faDataType = ACL_FLOAT16; //ACL_BF16;
2885
+
2886
+ aclTensor* acl_src0_f16_tensor = nullptr;
2887
+ aclTensor* acl_src1_f16_tensor = nullptr;
2888
+ aclTensor* acl_src2_f16_tensor = nullptr;
2889
+ aclTensor* acl_dst_f16_tensor = nullptr;
2890
+
2891
+ // Step 1: cast the src0 (Query) to fp16 if needed
2892
+ ggml_cann_pool_alloc src0_f16_allocator(ctx.pool());
2893
+ void* src0_f16_buffer = nullptr;
2894
+
2895
+ if(ggml_cann_type_mapping(src0->type) != faDataType){
2896
+ aclTensor* acl_src0_f32_tensor = ggml_cann_create_tensor(src0);
2897
+ src0_f16_buffer = src0_f16_allocator.alloc(
2898
+ ggml_nelements(src0) * faElemSize);
2899
+
2900
+ int64_t* src0_f16_ne = src0->ne;
2901
+ size_t src0_f16_nb[GGML_MAX_DIMS];
2902
+ src0_f16_nb[0] = sizeof(uint16_t);
2903
+ for(int i = 1; i < GGML_MAX_DIMS; ++i){
2904
+ src0_f16_nb[i] = src0_f16_nb[i - 1] * src0_f16_ne[i - 1];
2905
+ }
2906
+
2907
+ acl_src0_f16_tensor = ggml_cann_create_tensor(
2908
+ src0_f16_buffer, faDataType, faElemSize,
2909
+ src0_f16_ne, src0_f16_nb, GGML_MAX_DIMS
2910
+ );
2911
+ aclnn_cast(ctx, acl_src0_f32_tensor, acl_src0_f16_tensor, faDataType);
2912
+ ggml_cann_release_resources(ctx, acl_src0_f32_tensor);
2913
+ }else{
2914
+ acl_src0_f16_tensor = ggml_cann_create_tensor(src0);
2915
+ }
2916
+
2917
+ // Step 2: create the acl tensors for src1 (Key), src2 (Value),
2918
+ // and the direct output from FusedInferAttention
2919
+
2920
+ acl_src1_f16_tensor = ggml_cann_create_tensor(src1);
2921
+ acl_src2_f16_tensor = ggml_cann_create_tensor(src2);
2922
+
2923
+ ggml_cann_pool_alloc out_f16_allocator(ctx.pool());
2924
+ void* out_f16_buffer = out_f16_allocator.alloc(
2925
+ ggml_nelements(dst) * faElemSize);
2926
+
2927
+ int64_t* out_f16_ne = src0->ne;
2928
+ size_t out_f16_nb[GGML_MAX_DIMS];
2929
+ out_f16_nb[0] = faElemSize;
2930
+ for(int i = 1; i < GGML_MAX_DIMS; ++i){
2931
+ out_f16_nb[i] = out_f16_nb[i - 1] * out_f16_ne[i - 1];
2932
+ }
2933
+
2934
+ acl_dst_f16_tensor = ggml_cann_create_tensor(
2935
+ out_f16_buffer, faDataType, faElemSize,
2936
+ out_f16_ne, out_f16_nb, GGML_MAX_DIMS
2937
+ );
2938
+
2939
+ // Step 3: create the PSEShift tensor if needed
2940
+ // this tensor is considered as mask (f16) in the llama.cpp
2941
+
2942
+ aclTensor* bcast_pse_tensor = nullptr;
2943
+ int64_t bcast_pse_ne[GGML_MAX_DIMS];
2944
+ size_t bcast_pse_nb[GGML_MAX_DIMS];
2945
+ ggml_cann_pool_alloc bcast_pse_allocator(ctx.pool());
2946
+ void* bcast_pse_buffer = nullptr;
2947
+
2948
+ if(src3 != nullptr){
2949
+ bcast_pse_buffer = bcast_pse_allocator.alloc(
2950
+ ggml_nelements(src3) * src0->ne[2] * sizeof(uint16_t));
2951
+
2952
+ if(src0->ne[1] > 1){
2953
+ // Case 1: broadcast pse for prefill stage with multiple head
2954
+ aclTensor* acl_mask_f16_tensor = ggml_cann_create_tensor(src3);
2955
+ bcast_pse_ne[0] = src3->ne[0];
2956
+ bcast_pse_ne[1] = src3->ne[1];
2957
+ bcast_pse_ne[2] = src0->ne[2];
2958
+ bcast_pse_ne[3] = src3->ne[3];
2959
+
2960
+ bcast_pse_nb[0] = sizeof(uint16_t);
2961
+ for(int i = 1; i < GGML_MAX_DIMS; ++i){
2962
+ bcast_pse_nb[i] = bcast_pse_nb[i - 1] * bcast_pse_ne[i - 1];
2963
+ }
2964
+
2965
+ bcast_pse_tensor = ggml_cann_create_tensor(
2966
+ bcast_pse_buffer, ACL_FLOAT16, sizeof(uint16_t),
2967
+ bcast_pse_ne, bcast_pse_nb, GGML_MAX_DIMS);
2968
+
2969
+ int64_t repeats[] = {1, src0->ne[2], 1, 1};
2970
+ aclnn_repeat(ctx, acl_mask_f16_tensor, bcast_pse_tensor, repeats);
2971
+
2972
+ ggml_cann_release_resources(ctx, acl_mask_f16_tensor);
2973
+ }else{
2974
+ // Case 2: trunc the first row and broadcast pse for decode stage with multiple head
2975
+ int64_t trunc_pse_ne[GGML_MAX_DIMS] = {src3->ne[0], src0->ne[1], src3->ne[2], src3->ne[3]};
2976
+ size_t* trunc_pse_nb = src3->nb;
2977
+
2978
+ aclTensor* acl_mask_f16_trunc_tensor = ggml_cann_create_tensor(
2979
+ src3->data, ACL_FLOAT16, sizeof(uint16_t),
2980
+ trunc_pse_ne, trunc_pse_nb, GGML_MAX_DIMS);
2981
+
2982
+ bcast_pse_ne[0] = src3->ne[0];
2983
+ bcast_pse_ne[1] = src0->ne[1];
2984
+ bcast_pse_ne[2] = src0->ne[2];
2985
+ bcast_pse_ne[3] = src3->ne[3];
2986
+
2987
+ bcast_pse_nb[0] = sizeof(uint16_t);
2988
+ for(int i = 1; i < GGML_MAX_DIMS; ++i){
2989
+ bcast_pse_nb[i] = bcast_pse_nb[i - 1] * bcast_pse_ne[i - 1];
2990
+ }
2991
+
2992
+ bcast_pse_tensor = ggml_cann_create_tensor(
2993
+ bcast_pse_buffer, ACL_FLOAT16, sizeof(uint16_t),
2994
+ bcast_pse_ne, bcast_pse_nb, GGML_MAX_DIMS);
2995
+
2996
+ int64_t repeats[] = {1, src0->ne[2], 1, 1};
2997
+ aclnn_repeat(ctx, acl_mask_f16_trunc_tensor, bcast_pse_tensor, repeats);
2998
+
2999
+ ggml_cann_release_resources(ctx, acl_mask_f16_trunc_tensor);
3000
+ }
3001
+
3002
+ // Compute the slope if needed. Derived from ggml_cann_softmax().
3003
+ if(maxBias != 0.0f){
3004
+ // alibi
3005
+ const int64_t ne2_ne3 = src0->ne[2] * src0->ne[3];
3006
+ const int64_t n_head = src0->ne[2];
3007
+ const int n_heads_log2_floor = 1u << (uint32_t)floor(log2(n_head));
3008
+ float m0 = powf(2.0f, -(maxBias) / n_heads_log2_floor);
3009
+ float m1 = powf(2.0f, -(maxBias / 2.0f) / n_heads_log2_floor);
3010
+ // init arange
3011
+ ggml_cann_pool_alloc arange_allocator(ctx.pool(),
3012
+ ne2_ne3 * faElemSize);
3013
+ void* tmp_arange_buffer = arange_allocator.get();
3014
+
3015
+ // arange1: [1, ..., n_heads_log2_floor+1)
3016
+ float start = 1;
3017
+ float stop = n_heads_log2_floor + 1;
3018
+ float step = 1;
3019
+ int64_t n_elements_arange = n_heads_log2_floor;
3020
+
3021
+ int64_t tmp_arange1_ne[] = {n_heads_log2_floor};
3022
+ size_t tmp_arange1_nb[] = {faElemSize};
3023
+ aclTensor* tmp_arange1_tensor = ggml_cann_create_tensor(
3024
+ tmp_arange_buffer, faDataType, faElemSize,
3025
+ tmp_arange1_ne, tmp_arange1_nb,
3026
+ GGML_MAX_DIMS - 3, ACL_FORMAT_ND);
3027
+
3028
+ aclnn_arange(ctx, tmp_arange1_tensor, start, stop, step, n_elements_arange);
3029
+
3030
+ aclTensor* tmp_arange2_tensor = nullptr;
3031
+ if (n_heads_log2_floor < ne2_ne3) {
3032
+ // arange2: [1, ..., 2 * (k - n_heads_log2_floor) + 1)
3033
+ start = 1;
3034
+ stop = 2 * (ne2_ne3 - n_heads_log2_floor) + 1;
3035
+ step = 2;
3036
+ n_elements_arange = ne2_ne3 - n_heads_log2_floor;
3037
+ int64_t tmp_arange2_ne[] = {ne2_ne3 - n_heads_log2_floor};
3038
+ size_t tmp_arange2_nb[] = {faElemSize};
3039
+
3040
+ aclTensor* tmp_arange2_tensor = ggml_cann_create_tensor(
3041
+ (char*)tmp_arange_buffer +
3042
+ n_heads_log2_floor * faElemSize,
3043
+ faDataType, faElemSize,
3044
+ tmp_arange2_ne, tmp_arange2_nb, GGML_MAX_DIMS - 3, ACL_FORMAT_ND);
3045
+ aclnn_arange(ctx, tmp_arange2_tensor, start, stop, step,
3046
+ n_elements_arange);
3047
+ }
3048
+
3049
+ // init mk_base
3050
+ ggml_cann_pool_alloc mk_base_allocator(ctx.pool(),
3051
+ ne2_ne3 * faElemSize);
3052
+ void* tmp_mk_base_buffer = mk_base_allocator.get();
3053
+ int64_t tmp_mk_base1_ne[] = {n_heads_log2_floor};
3054
+ size_t tmp_mk_base1_nb[] = {faElemSize};
3055
+ aclTensor* tmp_mk_base1_tensor = ggml_cann_create_tensor(
3056
+ tmp_mk_base_buffer, faDataType, faElemSize,
3057
+ tmp_mk_base1_ne, tmp_mk_base1_nb,
3058
+ GGML_MAX_DIMS - 3, ACL_FORMAT_ND);
3059
+
3060
+ aclnn_fill_scalar(ctx, m0, tmp_mk_base1_tensor);
3061
+
3062
+ aclTensor* tmp_mk_base2_tensor = nullptr;
3063
+ if (n_heads_log2_floor < ne2_ne3) {
3064
+ int64_t tmp_mk_base2_ne[] = {ne2_ne3 - n_heads_log2_floor};
3065
+ size_t tmp_mk_base2_nb[] = {faElemSize};
3066
+ aclTensor* tmp_mk_base2_tensor = ggml_cann_create_tensor(
3067
+ (char*)tmp_mk_base_buffer +
3068
+ n_heads_log2_floor * faElemSize,
3069
+ faDataType, faElemSize,
3070
+ tmp_mk_base2_ne, tmp_mk_base2_nb, GGML_MAX_DIMS - 3, ACL_FORMAT_ND);
3071
+ aclnn_fill_scalar(ctx, m1, tmp_mk_base2_tensor);
3072
+ }
3073
+
3074
+ // init mk
3075
+ int64_t tmp_mk_base_ne[] = {ne2_ne3};
3076
+ size_t tmp_mk_base_nb[] = {faElemSize};
3077
+ aclTensor* tmp_mk_base_tensor = ggml_cann_create_tensor(
3078
+ tmp_mk_base_buffer, faDataType, faElemSize,
3079
+ tmp_mk_base_ne, tmp_mk_base_nb,
3080
+ GGML_MAX_DIMS - 3, ACL_FORMAT_ND);
3081
+ aclTensor* tmp_arange_tensor = ggml_cann_create_tensor(
3082
+ tmp_arange_buffer, faDataType, faElemSize,
3083
+ tmp_mk_base_ne, tmp_mk_base_nb,
3084
+ GGML_MAX_DIMS - 3, ACL_FORMAT_ND);
3085
+ aclnn_pow_tensor_tensor(ctx, tmp_mk_base_tensor, tmp_arange_tensor);
3086
+
3087
+ // reshape mk
3088
+ int64_t tmp_mk_ne[] = {1, 1, src0->ne[2], src0->ne[3]};
3089
+ size_t tmp_mk_nb[GGML_MAX_DIMS];
3090
+ tmp_mk_nb[0] = faElemSize;
3091
+ for (int i = 1; i < GGML_MAX_DIMS; i++) {
3092
+ tmp_mk_nb[i] = tmp_mk_nb[i - 1] * tmp_mk_ne[i - 1];
3093
+ }
3094
+ aclTensor* tmp_mk_tensor = ggml_cann_create_tensor(
3095
+ tmp_mk_base_buffer, faDataType, faElemSize,
3096
+ tmp_mk_ne, tmp_mk_nb, GGML_MAX_DIMS,
3097
+ ACL_FORMAT_ND);
3098
+ GGML_CANN_CALL_ACLNN_OP(ctx, InplaceMul, bcast_pse_tensor, tmp_mk_tensor);
3099
+
3100
+ ggml_cann_release_resources(ctx, tmp_arange1_tensor, tmp_arange2_tensor,
3101
+ tmp_mk_base1_tensor, tmp_mk_base2_tensor, tmp_mk_base_tensor,
3102
+ tmp_arange_tensor, tmp_mk_tensor);
3103
+ }
3104
+ }
3105
+
3106
+ // Step 4: set the inputs for FusedInferAttention.
3107
+ int kvTensorNum = 1;
3108
+ aclTensor* acl_q_tensor = acl_src0_f16_tensor;
3109
+ aclTensor* acl_k_tensors[] = {acl_src1_f16_tensor};
3110
+ aclTensor* acl_v_tensors[] = {acl_src2_f16_tensor};
3111
+ auto acl_k_tensor_list = aclCreateTensorList(acl_k_tensors, kvTensorNum);
3112
+ auto acl_v_tensor_list = aclCreateTensorList(acl_v_tensors, kvTensorNum);
3113
+
3114
+ int64_t numHeads = src0->ne[2]; // N
3115
+ int64_t numKeyValueHeads = src1->ne[2];
3116
+ // double scaleValue = 1 / sqrt(src0->ne[0]); // 1/sqrt(d)
3117
+ int64_t preTokens = 65535;
3118
+ int64_t nextTokens = 65535;
3119
+ char layout[5] = {'B', 'N', 'S', 'D', 0};
3120
+ int64_t sparseMode = 0;
3121
+ int64_t innerPrecise = (src0->ne[1] == 1) ? 0 : 2;
3122
+ int64_t blockSize = 0;
3123
+ int64_t antiquantMode = 0;
3124
+ bool softmaxLseFlag = false;
3125
+ int64_t keyAntiquantMode = 0;
3126
+ int64_t valueAntiquantMode = 0;
3127
+
3128
+ // Step 5: launch the FusedInferAttentionScoreV2 kernel.
3129
+ // Refer to https://gitee.com/ascend/cann-ops-adv/blob/master/docs/FusedInferAttentionScoreV2.md
3130
+
3131
+ GGML_CANN_CALL_ACLNN_OP(ctx, FusedInferAttentionScoreV2,
3132
+ acl_q_tensor, acl_k_tensor_list, acl_v_tensor_list, // q, k, v
3133
+ bcast_pse_tensor, nullptr, // pse, mask
3134
+ nullptr, nullptr, // actSeqLen, actSeqLenkv
3135
+ nullptr, nullptr, // deqScale1, quantScale1
3136
+ nullptr, nullptr, nullptr, // deqScale2, quantScale2, quantOffset2
3137
+ nullptr, nullptr, // antiquantScale, antiquantOffset
3138
+ nullptr, // blockTable
3139
+ nullptr, nullptr, // qPadSize, kvPadSize
3140
+ nullptr, nullptr, // kAntiquantScale, kAntiQuantOffset
3141
+ nullptr, nullptr, // vAntiquantScale, vAntiQuantOffset
3142
+ nullptr, nullptr, nullptr, // kSharedPrefix, vSharedPrefix, actSharedLen
3143
+ numHeads, scaleValue, // heads, scaleValue
3144
+ preTokens, nextTokens, // preTokens, nextTokens
3145
+ layout, // inputLayout
3146
+ numKeyValueHeads, // numKVHeads
3147
+ sparseMode, innerPrecise, // sparseMode, innerPrecise
3148
+ blockSize, antiquantMode, // blockSize, antiquantMode
3149
+ softmaxLseFlag, // softmaxLseFlag
3150
+ keyAntiquantMode, valueAntiquantMode, // keyAntiqMode, valueAntiqMode
3151
+ acl_dst_f16_tensor, // attentionOut
3152
+ nullptr // softmaxLse
3153
+ );
3154
+
3155
+ // Step 6: post-processing, permute and cast to f32
3156
+
3157
+ int64_t new_dim[] = {0, 2, 1, 3};
3158
+ aclTensor* acl_dst_tensor = ggml_cann_create_tensor(dst);
3159
+
3160
+ if(ggml_cann_type_mapping(dst->type) != faDataType){
3161
+ ggml_cann_pool_alloc perm_out_f16_allocator(ctx.pool());
3162
+ perm_out_f16_allocator.alloc(ggml_nelements(dst) * faElemSize);
3163
+ void* perm_out_f16_buffer = perm_out_f16_allocator.get();
3164
+
3165
+ int64_t* perm_out_f16_ne = dst->ne;
3166
+ size_t perm_out_f16_nb[GGML_MAX_DIMS];
3167
+ perm_out_f16_nb[0] = faElemSize;
3168
+ for(int i = 1; i < GGML_MAX_DIMS; ++i){
3169
+ perm_out_f16_nb[i] = perm_out_f16_nb[i - 1] * perm_out_f16_ne[i - 1];
3170
+ }
3171
+ aclTensor* acl_perm_out_f16_tensor = ggml_cann_create_tensor(
3172
+ perm_out_f16_buffer, faDataType, faElemSize,
3173
+ perm_out_f16_ne, perm_out_f16_nb, GGML_MAX_DIMS);
3174
+ aclnn_permute(ctx, acl_dst_f16_tensor, acl_perm_out_f16_tensor, new_dim, GGML_MAX_DIMS);
3175
+ aclnn_cast(ctx,
3176
+ acl_perm_out_f16_tensor, acl_dst_tensor, ggml_cann_type_mapping(dst->type));
3177
+ ggml_cann_release_resources(ctx, acl_perm_out_f16_tensor);
3178
+ }else{
3179
+ // only need to permute
3180
+ aclnn_permute(ctx, acl_dst_f16_tensor, acl_dst_tensor, new_dim, GGML_MAX_DIMS);
3181
+ }
3182
+ ggml_cann_release_resources(ctx, acl_src0_f16_tensor,
3183
+ acl_src1_f16_tensor,
3184
+ acl_src2_f16_tensor,
3185
+ acl_dst_f16_tensor,
3186
+ acl_dst_tensor);
3187
+ if(src3 != nullptr){
3188
+ ggml_cann_release_resources(ctx, bcast_pse_tensor);
3189
+ }
3190
+ }else{
3191
+ GGML_ABORT("Function is not implemented.");
3192
+ }
3193
+ }