whispercpp 1.3.1 → 1.3.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (797) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +4 -3
  3. data/README.md +92 -31
  4. data/Rakefile +26 -7
  5. data/ext/.gitignore +5 -7
  6. data/ext/dependencies.rb +61 -0
  7. data/ext/extconf.rb +21 -198
  8. data/ext/options.rb +221 -0
  9. data/ext/ruby_whisper.c +159 -0
  10. data/ext/ruby_whisper.h +17 -2
  11. data/ext/ruby_whisper_context.c +641 -0
  12. data/ext/ruby_whisper_error.c +52 -0
  13. data/ext/ruby_whisper_model.c +232 -0
  14. data/ext/ruby_whisper_params.c +1301 -0
  15. data/ext/ruby_whisper_segment.c +143 -0
  16. data/ext/ruby_whisper_transcribe.cpp +87 -0
  17. data/ext/ruby_whisper_vad_params.c +288 -0
  18. data/ext/sources/.dockerignore +3 -0
  19. data/ext/sources/.github/workflows/bindings-ruby.yml +21 -0
  20. data/ext/sources/CMakeGraphVizOptions.cmake +8 -0
  21. data/ext/sources/CMakeLists.txt +251 -0
  22. data/ext/sources/bindings/javascript/CMakeLists.txt +41 -0
  23. data/ext/sources/bindings/javascript/emscripten.cpp +93 -0
  24. data/ext/sources/bindings/javascript/libwhisper.worker.js +1 -0
  25. data/ext/sources/bindings/javascript/package-tmpl.json +26 -0
  26. data/ext/sources/bindings/javascript/package.json +26 -0
  27. data/ext/sources/bindings/javascript/whisper.js +19 -0
  28. data/ext/sources/build-xcframework.sh +547 -0
  29. data/ext/sources/ci/run.sh +336 -0
  30. data/ext/sources/close-issue.yml +28 -0
  31. data/ext/sources/cmake/DefaultTargetOptions.cmake +16 -0
  32. data/ext/sources/cmake/FindFFmpeg.cmake +163 -0
  33. data/ext/sources/cmake/build-info.cmake +60 -0
  34. data/ext/sources/cmake/git-vars.cmake +22 -0
  35. data/ext/sources/cmake/whisper-config.cmake.in +65 -0
  36. data/ext/sources/cmake/whisper.pc.in +10 -0
  37. data/ext/sources/examples/CMakeLists.txt +124 -0
  38. data/ext/sources/examples/addon.node/CMakeLists.txt +31 -0
  39. data/ext/sources/examples/addon.node/__test__/whisper.spec.js +37 -0
  40. data/ext/sources/examples/addon.node/addon.cpp +438 -0
  41. data/ext/sources/examples/addon.node/index.js +54 -0
  42. data/ext/sources/examples/addon.node/package.json +16 -0
  43. data/ext/sources/examples/bench/CMakeLists.txt +8 -0
  44. data/ext/sources/examples/bench/bench.cpp +175 -0
  45. data/ext/sources/examples/bench.wasm/CMakeLists.txt +49 -0
  46. data/ext/sources/examples/bench.wasm/emscripten.cpp +87 -0
  47. data/ext/sources/examples/bench.wasm/index-tmpl.html +284 -0
  48. data/ext/sources/examples/cli/CMakeLists.txt +8 -0
  49. data/ext/sources/examples/cli/cli.cpp +1294 -0
  50. data/ext/sources/examples/coi-serviceworker.js +146 -0
  51. data/ext/sources/examples/command/CMakeLists.txt +10 -0
  52. data/ext/sources/examples/command/command.cpp +776 -0
  53. data/ext/sources/examples/command/commands.txt +9 -0
  54. data/ext/sources/examples/command.wasm/CMakeLists.txt +50 -0
  55. data/ext/sources/examples/command.wasm/emscripten.cpp +327 -0
  56. data/ext/sources/examples/command.wasm/index-tmpl.html +414 -0
  57. data/ext/sources/examples/common-ggml.cpp +238 -0
  58. data/ext/sources/examples/common-ggml.h +18 -0
  59. data/ext/sources/examples/common-sdl.cpp +227 -0
  60. data/ext/sources/examples/common-sdl.h +49 -0
  61. data/ext/sources/examples/common-whisper.cpp +168 -0
  62. data/ext/sources/examples/common-whisper.h +24 -0
  63. data/ext/sources/examples/common.cpp +675 -0
  64. data/ext/sources/examples/common.h +322 -0
  65. data/ext/sources/examples/deprecation-warning/CMakeLists.txt +6 -0
  66. data/ext/sources/examples/deprecation-warning/deprecation-warning.cpp +38 -0
  67. data/ext/sources/examples/ffmpeg-transcode.cpp +368 -0
  68. data/ext/sources/examples/generate-karaoke.sh +57 -0
  69. data/ext/sources/examples/grammar-parser.cpp +423 -0
  70. data/ext/sources/examples/grammar-parser.h +29 -0
  71. data/ext/sources/examples/helpers.js +191 -0
  72. data/ext/sources/examples/json.hpp +24596 -0
  73. data/ext/sources/examples/livestream.sh +112 -0
  74. data/ext/sources/examples/lsp/CMakeLists.txt +9 -0
  75. data/ext/sources/examples/lsp/lsp.cpp +467 -0
  76. data/ext/sources/examples/lsp/whisper.vim +362 -0
  77. data/ext/sources/examples/miniaudio.h +93468 -0
  78. data/ext/sources/examples/python/test_whisper_processor.py +7 -0
  79. data/ext/sources/examples/python/whisper_processor.py +54 -0
  80. data/ext/sources/examples/quantize/CMakeLists.txt +6 -0
  81. data/ext/sources/examples/quantize/quantize.cpp +223 -0
  82. data/ext/sources/examples/server/CMakeLists.txt +12 -0
  83. data/ext/sources/examples/server/bench.js +29 -0
  84. data/ext/sources/examples/server/httplib.h +10497 -0
  85. data/ext/sources/examples/server/server.cpp +1091 -0
  86. data/ext/sources/examples/server.py +115 -0
  87. data/ext/sources/examples/stb_vorbis.c +5584 -0
  88. data/ext/sources/examples/stream/CMakeLists.txt +10 -0
  89. data/ext/sources/examples/stream/stream.cpp +429 -0
  90. data/ext/sources/examples/stream.wasm/CMakeLists.txt +49 -0
  91. data/ext/sources/examples/stream.wasm/emscripten.cpp +216 -0
  92. data/ext/sources/examples/stream.wasm/index-tmpl.html +414 -0
  93. data/ext/sources/examples/sycl/CMakeLists.txt +9 -0
  94. data/ext/sources/examples/sycl/build.sh +22 -0
  95. data/ext/sources/examples/sycl/ls-sycl-device.cpp +11 -0
  96. data/ext/sources/examples/sycl/run-whisper.sh +17 -0
  97. data/ext/sources/examples/talk-llama/CMakeLists.txt +40 -0
  98. data/ext/sources/examples/talk-llama/eleven-labs.py +80 -0
  99. data/ext/sources/examples/talk-llama/llama-adapter.cpp +388 -0
  100. data/ext/sources/examples/talk-llama/llama-adapter.h +76 -0
  101. data/ext/sources/examples/talk-llama/llama-arch.cpp +1746 -0
  102. data/ext/sources/examples/talk-llama/llama-arch.h +437 -0
  103. data/ext/sources/examples/talk-llama/llama-batch.cpp +374 -0
  104. data/ext/sources/examples/talk-llama/llama-batch.h +89 -0
  105. data/ext/sources/examples/talk-llama/llama-chat.cpp +663 -0
  106. data/ext/sources/examples/talk-llama/llama-chat.h +58 -0
  107. data/ext/sources/examples/talk-llama/llama-context.cpp +2676 -0
  108. data/ext/sources/examples/talk-llama/llama-context.h +276 -0
  109. data/ext/sources/examples/talk-llama/llama-cparams.cpp +5 -0
  110. data/ext/sources/examples/talk-llama/llama-cparams.h +41 -0
  111. data/ext/sources/examples/talk-llama/llama-grammar.cpp +1229 -0
  112. data/ext/sources/examples/talk-llama/llama-grammar.h +173 -0
  113. data/ext/sources/examples/talk-llama/llama-graph.cpp +1618 -0
  114. data/ext/sources/examples/talk-llama/llama-graph.h +640 -0
  115. data/ext/sources/examples/talk-llama/llama-hparams.cpp +95 -0
  116. data/ext/sources/examples/talk-llama/llama-hparams.h +190 -0
  117. data/ext/sources/examples/talk-llama/llama-impl.cpp +167 -0
  118. data/ext/sources/examples/talk-llama/llama-impl.h +61 -0
  119. data/ext/sources/examples/talk-llama/llama-io.cpp +15 -0
  120. data/ext/sources/examples/talk-llama/llama-io.h +35 -0
  121. data/ext/sources/examples/talk-llama/llama-kv-cache.cpp +2739 -0
  122. data/ext/sources/examples/talk-llama/llama-kv-cache.h +502 -0
  123. data/ext/sources/examples/talk-llama/llama-kv-cells.h +379 -0
  124. data/ext/sources/examples/talk-llama/llama-memory.cpp +1 -0
  125. data/ext/sources/examples/talk-llama/llama-memory.h +32 -0
  126. data/ext/sources/examples/talk-llama/llama-mmap.cpp +600 -0
  127. data/ext/sources/examples/talk-llama/llama-mmap.h +68 -0
  128. data/ext/sources/examples/talk-llama/llama-model-loader.cpp +1138 -0
  129. data/ext/sources/examples/talk-llama/llama-model-loader.h +169 -0
  130. data/ext/sources/examples/talk-llama/llama-model-saver.cpp +281 -0
  131. data/ext/sources/examples/talk-llama/llama-model-saver.h +37 -0
  132. data/ext/sources/examples/talk-llama/llama-model.cpp +13814 -0
  133. data/ext/sources/examples/talk-llama/llama-model.h +425 -0
  134. data/ext/sources/examples/talk-llama/llama-quant.cpp +966 -0
  135. data/ext/sources/examples/talk-llama/llama-quant.h +1 -0
  136. data/ext/sources/examples/talk-llama/llama-sampling.cpp +2575 -0
  137. data/ext/sources/examples/talk-llama/llama-sampling.h +32 -0
  138. data/ext/sources/examples/talk-llama/llama-vocab.cpp +3340 -0
  139. data/ext/sources/examples/talk-llama/llama-vocab.h +131 -0
  140. data/ext/sources/examples/talk-llama/llama.cpp +354 -0
  141. data/ext/sources/examples/talk-llama/llama.h +1377 -0
  142. data/ext/sources/examples/talk-llama/prompts/talk-alpaca.txt +23 -0
  143. data/ext/sources/examples/talk-llama/speak +40 -0
  144. data/ext/sources/examples/talk-llama/speak.bat +1 -0
  145. data/ext/sources/examples/talk-llama/speak.ps1 +14 -0
  146. data/ext/sources/examples/talk-llama/talk-llama.cpp +808 -0
  147. data/ext/sources/examples/talk-llama/unicode-data.cpp +7034 -0
  148. data/ext/sources/examples/talk-llama/unicode-data.h +20 -0
  149. data/ext/sources/examples/talk-llama/unicode.cpp +849 -0
  150. data/ext/sources/examples/talk-llama/unicode.h +66 -0
  151. data/ext/sources/examples/vad-speech-segments/CMakeLists.txt +8 -0
  152. data/ext/sources/examples/vad-speech-segments/speech.cpp +143 -0
  153. data/ext/sources/examples/wchess/CMakeLists.txt +10 -0
  154. data/ext/sources/examples/wchess/libwchess/CMakeLists.txt +19 -0
  155. data/ext/sources/examples/wchess/libwchess/Chessboard.cpp +803 -0
  156. data/ext/sources/examples/wchess/libwchess/Chessboard.h +33 -0
  157. data/ext/sources/examples/wchess/libwchess/WChess.cpp +193 -0
  158. data/ext/sources/examples/wchess/libwchess/WChess.h +63 -0
  159. data/ext/sources/examples/wchess/libwchess/test-chessboard.cpp +117 -0
  160. data/ext/sources/examples/wchess/wchess.cmd/CMakeLists.txt +8 -0
  161. data/ext/sources/examples/wchess/wchess.cmd/wchess.cmd.cpp +249 -0
  162. data/ext/sources/examples/whisper.wasm/CMakeLists.txt +50 -0
  163. data/ext/sources/examples/whisper.wasm/emscripten.cpp +118 -0
  164. data/ext/sources/examples/whisper.wasm/index-tmpl.html +658 -0
  165. data/ext/sources/ggml/CMakeLists.txt +390 -0
  166. data/ext/sources/ggml/cmake/BuildTypes.cmake +54 -0
  167. data/ext/sources/ggml/cmake/GitVars.cmake +22 -0
  168. data/ext/sources/ggml/cmake/common.cmake +26 -0
  169. data/ext/sources/ggml/cmake/ggml-config.cmake.in +152 -0
  170. data/ext/{ggml → sources/ggml}/include/ggml-alloc.h +1 -1
  171. data/ext/{ggml → sources/ggml}/include/ggml-backend.h +9 -7
  172. data/ext/{ggml → sources/ggml}/include/ggml-cpp.h +2 -1
  173. data/ext/{ggml → sources/ggml}/include/ggml-cpu.h +9 -1
  174. data/ext/{ggml → sources/ggml}/include/ggml-metal.h +1 -1
  175. data/ext/{ggml → sources/ggml}/include/ggml-opt.h +49 -28
  176. data/ext/{ggml → sources/ggml}/include/ggml-rpc.h +6 -1
  177. data/ext/{ggml → sources/ggml}/include/ggml-vulkan.h +0 -2
  178. data/ext/{ggml → sources/ggml}/include/ggml.h +182 -265
  179. data/ext/sources/ggml/include/gguf.h +202 -0
  180. data/ext/sources/ggml/src/CMakeLists.txt +346 -0
  181. data/ext/{ggml → sources/ggml}/src/ggml-alloc.c +34 -29
  182. data/ext/sources/ggml/src/ggml-amx/CMakeLists.txt +107 -0
  183. data/ext/{ggml → sources/ggml}/src/ggml-backend-impl.h +1 -2
  184. data/ext/{ggml → sources/ggml}/src/ggml-backend-reg.cpp +87 -53
  185. data/ext/{ggml → sources/ggml}/src/ggml-backend.cpp +26 -14
  186. data/ext/sources/ggml/src/ggml-blas/CMakeLists.txt +87 -0
  187. data/ext/sources/ggml/src/ggml-cann/CMakeLists.txt +74 -0
  188. data/ext/sources/ggml/src/ggml-cann/Doxyfile +2579 -0
  189. data/ext/{ggml → sources/ggml}/src/ggml-cann/acl_tensor.cpp +10 -4
  190. data/ext/{ggml → sources/ggml}/src/ggml-cann/acl_tensor.h +5 -5
  191. data/ext/{ggml → sources/ggml}/src/ggml-cann/aclnn_ops.cpp +1272 -1506
  192. data/ext/sources/ggml/src/ggml-cann/aclnn_ops.h +1125 -0
  193. data/ext/{ggml → sources/ggml}/src/ggml-cann/common.h +135 -1
  194. data/ext/{ggml → sources/ggml}/src/ggml-cann/ggml-cann.cpp +564 -146
  195. data/ext/sources/ggml/src/ggml-cann/kernels/CMakeLists.txt +30 -0
  196. data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/dup.cpp +3 -5
  197. data/ext/{ggml → sources/ggml}/src/ggml-common.h +12 -8
  198. data/ext/sources/ggml/src/ggml-cpu/CMakeLists.txt +504 -0
  199. data/ext/{ggml → sources/ggml}/src/ggml-cpu/amx/amx.cpp +2 -1
  200. data/ext/sources/ggml/src/ggml-cpu/binary-ops.cpp +158 -0
  201. data/ext/sources/ggml/src/ggml-cpu/binary-ops.h +16 -0
  202. data/ext/sources/ggml/src/ggml-cpu/cmake/FindSIMD.cmake +100 -0
  203. data/ext/sources/ggml/src/ggml-cpu/common.h +72 -0
  204. data/ext/{ggml → sources/ggml}/src/ggml-cpu/cpu-feats-x86.cpp +5 -1
  205. data/ext/sources/ggml/src/ggml-cpu/ggml-cpu-aarch64.cpp +6431 -0
  206. data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-impl.h +163 -41
  207. data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-quants.c +4029 -1117
  208. data/ext/sources/ggml/src/ggml-cpu/ggml-cpu.c +3510 -0
  209. data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu.cpp +67 -18
  210. data/ext/sources/ggml/src/ggml-cpu/kleidiai/kernels.cpp +337 -0
  211. data/ext/sources/ggml/src/ggml-cpu/kleidiai/kernels.h +95 -0
  212. data/ext/sources/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp +482 -0
  213. data/ext/sources/ggml/src/ggml-cpu/kleidiai/kleidiai.h +17 -0
  214. data/ext/sources/ggml/src/ggml-cpu/llamafile/sgemm.cpp +3544 -0
  215. data/ext/sources/ggml/src/ggml-cpu/llamafile/sgemm.h +14 -0
  216. data/ext/sources/ggml/src/ggml-cpu/ops.cpp +8903 -0
  217. data/ext/sources/ggml/src/ggml-cpu/ops.h +110 -0
  218. data/ext/sources/ggml/src/ggml-cpu/simd-mappings.h +892 -0
  219. data/ext/sources/ggml/src/ggml-cpu/unary-ops.cpp +186 -0
  220. data/ext/sources/ggml/src/ggml-cpu/unary-ops.h +28 -0
  221. data/ext/sources/ggml/src/ggml-cpu/vec.cpp +252 -0
  222. data/ext/sources/ggml/src/ggml-cpu/vec.h +818 -0
  223. data/ext/sources/ggml/src/ggml-cuda/CMakeLists.txt +184 -0
  224. data/ext/sources/ggml/src/ggml-cuda/acc.cu +61 -0
  225. data/ext/sources/ggml/src/ggml-cuda/acc.cuh +5 -0
  226. data/ext/sources/ggml/src/ggml-cuda/arange.cu +34 -0
  227. data/ext/sources/ggml/src/ggml-cuda/arange.cuh +5 -0
  228. data/ext/sources/ggml/src/ggml-cuda/argmax.cu +91 -0
  229. data/ext/sources/ggml/src/ggml-cuda/argmax.cuh +3 -0
  230. data/ext/sources/ggml/src/ggml-cuda/argsort.cu +104 -0
  231. data/ext/sources/ggml/src/ggml-cuda/argsort.cuh +3 -0
  232. data/ext/sources/ggml/src/ggml-cuda/binbcast.cu +363 -0
  233. data/ext/sources/ggml/src/ggml-cuda/binbcast.cuh +9 -0
  234. data/ext/sources/ggml/src/ggml-cuda/clamp.cu +45 -0
  235. data/ext/sources/ggml/src/ggml-cuda/clamp.cuh +5 -0
  236. data/ext/sources/ggml/src/ggml-cuda/common.cuh +828 -0
  237. data/ext/sources/ggml/src/ggml-cuda/concat.cu +221 -0
  238. data/ext/sources/ggml/src/ggml-cuda/concat.cuh +5 -0
  239. data/ext/sources/ggml/src/ggml-cuda/conv-transpose-1d.cu +89 -0
  240. data/ext/sources/ggml/src/ggml-cuda/conv-transpose-1d.cuh +5 -0
  241. data/ext/sources/ggml/src/ggml-cuda/convert.cu +730 -0
  242. data/ext/sources/ggml/src/ggml-cuda/convert.cuh +26 -0
  243. data/ext/sources/ggml/src/ggml-cuda/count-equal.cu +64 -0
  244. data/ext/sources/ggml/src/ggml-cuda/count-equal.cuh +5 -0
  245. data/ext/sources/ggml/src/ggml-cuda/cp-async.cuh +57 -0
  246. data/ext/sources/ggml/src/ggml-cuda/cpy.cu +705 -0
  247. data/ext/sources/ggml/src/ggml-cuda/cpy.cuh +11 -0
  248. data/ext/sources/ggml/src/ggml-cuda/cross-entropy-loss.cu +189 -0
  249. data/ext/sources/ggml/src/ggml-cuda/cross-entropy-loss.cuh +7 -0
  250. data/ext/sources/ggml/src/ggml-cuda/dequantize.cuh +103 -0
  251. data/ext/sources/ggml/src/ggml-cuda/diagmask.cu +40 -0
  252. data/ext/sources/ggml/src/ggml-cuda/diagmask.cuh +5 -0
  253. data/ext/sources/ggml/src/ggml-cuda/fattn-common.cuh +881 -0
  254. data/ext/sources/ggml/src/ggml-cuda/fattn-mma-f16.cuh +1471 -0
  255. data/ext/sources/ggml/src/ggml-cuda/fattn-tile-f16.cu +357 -0
  256. data/ext/sources/ggml/src/ggml-cuda/fattn-tile-f16.cuh +3 -0
  257. data/ext/sources/ggml/src/ggml-cuda/fattn-tile-f32.cu +365 -0
  258. data/ext/sources/ggml/src/ggml-cuda/fattn-tile-f32.cuh +3 -0
  259. data/ext/sources/ggml/src/ggml-cuda/fattn-vec-f16.cuh +482 -0
  260. data/ext/sources/ggml/src/ggml-cuda/fattn-vec-f32.cuh +472 -0
  261. data/ext/sources/ggml/src/ggml-cuda/fattn-wmma-f16.cu +634 -0
  262. data/ext/sources/ggml/src/ggml-cuda/fattn-wmma-f16.cuh +3 -0
  263. data/ext/sources/ggml/src/ggml-cuda/fattn.cu +346 -0
  264. data/ext/sources/ggml/src/ggml-cuda/fattn.cuh +3 -0
  265. data/ext/sources/ggml/src/ggml-cuda/getrows.cu +275 -0
  266. data/ext/sources/ggml/src/ggml-cuda/getrows.cuh +15 -0
  267. data/ext/sources/ggml/src/ggml-cuda/ggml-cuda.cu +3505 -0
  268. data/ext/sources/ggml/src/ggml-cuda/gla.cu +93 -0
  269. data/ext/sources/ggml/src/ggml-cuda/gla.cuh +3 -0
  270. data/ext/sources/ggml/src/ggml-cuda/im2col.cu +103 -0
  271. data/ext/sources/ggml/src/ggml-cuda/im2col.cuh +5 -0
  272. data/ext/sources/ggml/src/ggml-cuda/mma.cuh +396 -0
  273. data/ext/sources/ggml/src/ggml-cuda/mmq.cu +324 -0
  274. data/ext/sources/ggml/src/ggml-cuda/mmq.cuh +3217 -0
  275. data/ext/sources/ggml/src/ggml-cuda/mmv.cu +336 -0
  276. data/ext/sources/ggml/src/ggml-cuda/mmv.cuh +12 -0
  277. data/ext/sources/ggml/src/ggml-cuda/mmvq.cu +595 -0
  278. data/ext/sources/ggml/src/ggml-cuda/mmvq.cuh +12 -0
  279. data/ext/sources/ggml/src/ggml-cuda/norm.cu +458 -0
  280. data/ext/sources/ggml/src/ggml-cuda/norm.cuh +11 -0
  281. data/ext/sources/ggml/src/ggml-cuda/opt-step-adamw.cu +78 -0
  282. data/ext/sources/ggml/src/ggml-cuda/opt-step-adamw.cuh +5 -0
  283. data/ext/sources/ggml/src/ggml-cuda/out-prod.cu +68 -0
  284. data/ext/sources/ggml/src/ggml-cuda/out-prod.cuh +3 -0
  285. data/ext/sources/ggml/src/ggml-cuda/pad.cu +49 -0
  286. data/ext/sources/ggml/src/ggml-cuda/pad.cuh +5 -0
  287. data/ext/sources/ggml/src/ggml-cuda/pool2d.cu +94 -0
  288. data/ext/sources/ggml/src/ggml-cuda/pool2d.cuh +5 -0
  289. data/ext/sources/ggml/src/ggml-cuda/quantize.cu +190 -0
  290. data/ext/sources/ggml/src/ggml-cuda/quantize.cuh +27 -0
  291. data/ext/sources/ggml/src/ggml-cuda/rope.cu +456 -0
  292. data/ext/sources/ggml/src/ggml-cuda/rope.cuh +7 -0
  293. data/ext/sources/ggml/src/ggml-cuda/scale.cu +31 -0
  294. data/ext/sources/ggml/src/ggml-cuda/scale.cuh +5 -0
  295. data/ext/sources/ggml/src/ggml-cuda/softmax.cu +283 -0
  296. data/ext/sources/ggml/src/ggml-cuda/softmax.cuh +7 -0
  297. data/ext/sources/ggml/src/ggml-cuda/ssm-conv.cu +148 -0
  298. data/ext/sources/ggml/src/ggml-cuda/ssm-conv.cuh +3 -0
  299. data/ext/sources/ggml/src/ggml-cuda/ssm-scan.cu +153 -0
  300. data/ext/sources/ggml/src/ggml-cuda/ssm-scan.cuh +3 -0
  301. data/ext/sources/ggml/src/ggml-cuda/sum.cu +45 -0
  302. data/ext/sources/ggml/src/ggml-cuda/sum.cuh +5 -0
  303. data/ext/sources/ggml/src/ggml-cuda/sumrows.cu +39 -0
  304. data/ext/sources/ggml/src/ggml-cuda/sumrows.cuh +5 -0
  305. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_16.cu +5 -0
  306. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_8.cu +10 -0
  307. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_1.cu +10 -0
  308. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_2.cu +10 -0
  309. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_4.cu +10 -0
  310. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_16.cu +5 -0
  311. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_4.cu +10 -0
  312. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_8.cu +10 -0
  313. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_1.cu +10 -0
  314. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_2.cu +10 -0
  315. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_16.cu +5 -0
  316. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_2.cu +10 -0
  317. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_4.cu +10 -0
  318. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_8.cu +10 -0
  319. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_64-ncols2_1.cu +10 -0
  320. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_1.cu +10 -0
  321. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_2.cu +10 -0
  322. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_4.cu +10 -0
  323. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_8.cu +10 -0
  324. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-f16.cu +5 -0
  325. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q4_0.cu +5 -0
  326. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q4_1.cu +5 -0
  327. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q5_0.cu +5 -0
  328. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q5_1.cu +5 -0
  329. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q8_0.cu +5 -0
  330. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-f16.cu +5 -0
  331. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q4_0.cu +5 -0
  332. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q4_1.cu +5 -0
  333. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q5_0.cu +5 -0
  334. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q5_1.cu +5 -0
  335. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q8_0.cu +5 -0
  336. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-f16.cu +5 -0
  337. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q4_0.cu +5 -0
  338. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q4_1.cu +5 -0
  339. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q5_0.cu +5 -0
  340. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q5_1.cu +5 -0
  341. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q8_0.cu +5 -0
  342. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-f16.cu +5 -0
  343. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q4_0.cu +5 -0
  344. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q4_1.cu +5 -0
  345. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q5_0.cu +5 -0
  346. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q5_1.cu +5 -0
  347. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q8_0.cu +5 -0
  348. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-f16.cu +5 -0
  349. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q4_0.cu +5 -0
  350. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q4_1.cu +5 -0
  351. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q5_0.cu +5 -0
  352. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q5_1.cu +5 -0
  353. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q8_0.cu +5 -0
  354. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-f16.cu +5 -0
  355. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q4_0.cu +5 -0
  356. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q4_1.cu +5 -0
  357. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q5_0.cu +5 -0
  358. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q5_1.cu +5 -0
  359. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q8_0.cu +5 -0
  360. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs256-f16-f16.cu +5 -0
  361. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-f16.cu +5 -0
  362. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q4_0.cu +5 -0
  363. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q4_1.cu +5 -0
  364. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q5_0.cu +5 -0
  365. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q5_1.cu +5 -0
  366. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q8_0.cu +5 -0
  367. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-f16.cu +5 -0
  368. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q4_0.cu +5 -0
  369. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q4_1.cu +5 -0
  370. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q5_0.cu +5 -0
  371. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q5_1.cu +5 -0
  372. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q8_0.cu +5 -0
  373. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-f16.cu +5 -0
  374. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q4_0.cu +5 -0
  375. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q4_1.cu +5 -0
  376. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q5_0.cu +5 -0
  377. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q5_1.cu +5 -0
  378. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q8_0.cu +5 -0
  379. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-f16.cu +5 -0
  380. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q4_0.cu +5 -0
  381. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q4_1.cu +5 -0
  382. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q5_0.cu +5 -0
  383. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q5_1.cu +5 -0
  384. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q8_0.cu +5 -0
  385. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-f16.cu +5 -0
  386. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q4_0.cu +5 -0
  387. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q4_1.cu +5 -0
  388. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q5_0.cu +5 -0
  389. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q5_1.cu +5 -0
  390. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q8_0.cu +5 -0
  391. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-f16.cu +5 -0
  392. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q4_0.cu +5 -0
  393. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q4_1.cu +5 -0
  394. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q5_0.cu +5 -0
  395. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q5_1.cu +5 -0
  396. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q8_0.cu +5 -0
  397. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-f16.cu +5 -0
  398. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q4_0.cu +5 -0
  399. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q4_1.cu +5 -0
  400. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q5_0.cu +5 -0
  401. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q5_1.cu +5 -0
  402. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q8_0.cu +5 -0
  403. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs256-f16-f16.cu +5 -0
  404. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-f16.cu +5 -0
  405. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q4_0.cu +5 -0
  406. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q4_1.cu +5 -0
  407. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q5_0.cu +5 -0
  408. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q5_1.cu +5 -0
  409. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q8_0.cu +5 -0
  410. data/ext/sources/ggml/src/ggml-cuda/template-instances/generate_cu_files.py +78 -0
  411. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq1_s.cu +5 -0
  412. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_s.cu +5 -0
  413. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xs.cu +5 -0
  414. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xxs.cu +5 -0
  415. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_s.cu +5 -0
  416. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_xxs.cu +5 -0
  417. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_nl.cu +5 -0
  418. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_xs.cu +5 -0
  419. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_k.cu +5 -0
  420. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q3_k.cu +5 -0
  421. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_0.cu +5 -0
  422. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_1.cu +5 -0
  423. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_k.cu +5 -0
  424. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_0.cu +5 -0
  425. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_1.cu +5 -0
  426. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_k.cu +5 -0
  427. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q6_k.cu +5 -0
  428. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q8_0.cu +5 -0
  429. data/ext/sources/ggml/src/ggml-cuda/tsembd.cu +47 -0
  430. data/ext/sources/ggml/src/ggml-cuda/tsembd.cuh +5 -0
  431. data/ext/sources/ggml/src/ggml-cuda/unary.cu +289 -0
  432. data/ext/sources/ggml/src/ggml-cuda/unary.cuh +59 -0
  433. data/ext/sources/ggml/src/ggml-cuda/upscale.cu +51 -0
  434. data/ext/sources/ggml/src/ggml-cuda/upscale.cuh +5 -0
  435. data/ext/sources/ggml/src/ggml-cuda/vecdotq.cuh +1135 -0
  436. data/ext/{ggml → sources/ggml}/src/ggml-cuda/vendors/cuda.h +1 -0
  437. data/ext/{ggml → sources/ggml}/src/ggml-cuda/vendors/hip.h +57 -0
  438. data/ext/{ggml → sources/ggml}/src/ggml-cuda/vendors/musa.h +7 -1
  439. data/ext/sources/ggml/src/ggml-cuda/wkv.cu +199 -0
  440. data/ext/sources/ggml/src/ggml-cuda/wkv.cuh +7 -0
  441. data/ext/sources/ggml/src/ggml-hip/CMakeLists.txt +131 -0
  442. data/ext/{ggml → sources/ggml}/src/ggml-impl.h +64 -19
  443. data/ext/sources/ggml/src/ggml-kompute/CMakeLists.txt +166 -0
  444. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/common.comp +112 -0
  445. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_add.comp +58 -0
  446. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_addrow.comp +25 -0
  447. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_cpy_f16_f16.comp +52 -0
  448. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_cpy_f16_f32.comp +52 -0
  449. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_cpy_f32_f16.comp +52 -0
  450. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_cpy_f32_f32.comp +52 -0
  451. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_diagmask.comp +30 -0
  452. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_gelu.comp +22 -0
  453. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows.comp +17 -0
  454. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows_f16.comp +31 -0
  455. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows_f32.comp +31 -0
  456. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows_q4_0.comp +38 -0
  457. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows_q4_1.comp +39 -0
  458. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows_q6_k.comp +44 -0
  459. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul.comp +52 -0
  460. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_f16.comp +69 -0
  461. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_mat_f32.comp +51 -0
  462. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q4_0.comp +33 -0
  463. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q4_1.comp +35 -0
  464. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q4_k.comp +140 -0
  465. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q6_k.comp +106 -0
  466. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q8_0.comp +73 -0
  467. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mv_q_n.comp +52 -0
  468. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mv_q_n_pre.comp +28 -0
  469. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_norm.comp +84 -0
  470. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_relu.comp +21 -0
  471. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_rmsnorm.comp +53 -0
  472. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_rope_neox_f16.comp +52 -0
  473. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_rope_neox_f32.comp +52 -0
  474. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_rope_norm_f16.comp +52 -0
  475. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_rope_norm_f32.comp +52 -0
  476. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_scale.comp +19 -0
  477. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_scale_8.comp +23 -0
  478. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_silu.comp +22 -0
  479. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_softmax.comp +72 -0
  480. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/rope_common.comp +71 -0
  481. data/ext/sources/ggml/src/ggml-metal/CMakeLists.txt +120 -0
  482. data/ext/sources/ggml/src/ggml-metal/ggml-metal-impl.h +622 -0
  483. data/ext/{ggml → sources/ggml}/src/ggml-metal/ggml-metal.m +2178 -1064
  484. data/ext/{ggml → sources/ggml}/src/ggml-metal/ggml-metal.metal +1575 -1218
  485. data/ext/sources/ggml/src/ggml-musa/CMakeLists.txt +113 -0
  486. data/ext/sources/ggml/src/ggml-musa/mudnn.cu +112 -0
  487. data/ext/sources/ggml/src/ggml-musa/mudnn.cuh +12 -0
  488. data/ext/sources/ggml/src/ggml-opencl/CMakeLists.txt +96 -0
  489. data/ext/sources/ggml/src/ggml-opencl/ggml-opencl.cpp +5124 -0
  490. data/ext/sources/ggml/src/ggml-opencl/kernels/add.cl +83 -0
  491. data/ext/sources/ggml/src/ggml-opencl/kernels/clamp.cl +20 -0
  492. data/ext/sources/ggml/src/ggml-opencl/kernels/cpy.cl +184 -0
  493. data/ext/sources/ggml/src/ggml-opencl/kernels/cvt.cl +118 -0
  494. data/ext/sources/ggml/src/ggml-opencl/kernels/diag_mask_inf.cl +58 -0
  495. data/ext/sources/ggml/src/ggml-opencl/kernels/embed_kernel.py +26 -0
  496. data/ext/sources/ggml/src/ggml-opencl/kernels/gelu.cl +62 -0
  497. data/ext/sources/ggml/src/ggml-opencl/kernels/gemv_noshuffle.cl +268 -0
  498. data/ext/sources/ggml/src/ggml-opencl/kernels/gemv_noshuffle_general.cl +274 -0
  499. data/ext/sources/ggml/src/ggml-opencl/kernels/get_rows.cl +163 -0
  500. data/ext/sources/ggml/src/ggml-opencl/kernels/im2col_f16.cl +57 -0
  501. data/ext/sources/ggml/src/ggml-opencl/kernels/im2col_f32.cl +57 -0
  502. data/ext/sources/ggml/src/ggml-opencl/kernels/mul.cl +79 -0
  503. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mat_Ab_Bi_8x4.cl +139 -0
  504. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_f16_f16.cl +118 -0
  505. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32.cl +118 -0
  506. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_1row.cl +94 -0
  507. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_l4.cl +84 -0
  508. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_f32_f32.cl +118 -0
  509. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32.cl +192 -0
  510. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_16x_flat.cl +307 -0
  511. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_8x_flat.cl +265 -0
  512. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_8x_flat.cl +272 -0
  513. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_v.cl +254 -0
  514. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q6_k.cl +190 -0
  515. data/ext/sources/ggml/src/ggml-opencl/kernels/norm.cl +81 -0
  516. data/ext/sources/ggml/src/ggml-opencl/kernels/relu.cl +16 -0
  517. data/ext/sources/ggml/src/ggml-opencl/kernels/rms_norm.cl +96 -0
  518. data/ext/sources/ggml/src/ggml-opencl/kernels/rope.cl +721 -0
  519. data/ext/sources/ggml/src/ggml-opencl/kernels/scale.cl +16 -0
  520. data/ext/sources/ggml/src/ggml-opencl/kernels/silu.cl +30 -0
  521. data/ext/sources/ggml/src/ggml-opencl/kernels/softmax_4_f16.cl +87 -0
  522. data/ext/sources/ggml/src/ggml-opencl/kernels/softmax_4_f32.cl +87 -0
  523. data/ext/sources/ggml/src/ggml-opencl/kernels/softmax_f16.cl +86 -0
  524. data/ext/sources/ggml/src/ggml-opencl/kernels/softmax_f32.cl +86 -0
  525. data/ext/sources/ggml/src/ggml-opencl/kernels/transpose.cl +84 -0
  526. data/ext/{ggml → sources/ggml}/src/ggml-opt.cpp +373 -190
  527. data/ext/{ggml → sources/ggml}/src/ggml-quants.c +114 -120
  528. data/ext/sources/ggml/src/ggml-rpc/CMakeLists.txt +9 -0
  529. data/ext/{ggml → sources/ggml}/src/ggml-rpc/ggml-rpc.cpp +480 -73
  530. data/ext/sources/ggml/src/ggml-sycl/CMakeLists.txt +189 -0
  531. data/ext/sources/ggml/src/ggml-sycl/backend.hpp +37 -0
  532. data/ext/sources/ggml/src/ggml-sycl/binbcast.cpp +345 -0
  533. data/ext/sources/ggml/src/ggml-sycl/binbcast.hpp +39 -0
  534. data/ext/{ggml → sources/ggml}/src/ggml-sycl/common.cpp +20 -32
  535. data/ext/sources/ggml/src/ggml-sycl/common.hpp +589 -0
  536. data/ext/{ggml → sources/ggml}/src/ggml-sycl/concat.cpp +32 -33
  537. data/ext/sources/ggml/src/ggml-sycl/concat.hpp +20 -0
  538. data/ext/{ggml → sources/ggml}/src/ggml-sycl/conv.cpp +4 -2
  539. data/ext/sources/ggml/src/ggml-sycl/conv.hpp +20 -0
  540. data/ext/{ggml → sources/ggml}/src/ggml-sycl/convert.cpp +104 -28
  541. data/ext/sources/ggml/src/ggml-sycl/convert.hpp +34 -0
  542. data/ext/sources/ggml/src/ggml-sycl/cpy.cpp +700 -0
  543. data/ext/sources/ggml/src/ggml-sycl/cpy.hpp +11 -0
  544. data/ext/sources/ggml/src/ggml-sycl/dequantize.hpp +791 -0
  545. data/ext/{ggml → sources/ggml}/src/ggml-sycl/dmmv.cpp +156 -17
  546. data/ext/sources/ggml/src/ggml-sycl/dmmv.hpp +27 -0
  547. data/ext/sources/ggml/src/ggml-sycl/dpct/helper.hpp +2957 -0
  548. data/ext/sources/ggml/src/ggml-sycl/element_wise.cpp +1511 -0
  549. data/ext/sources/ggml/src/ggml-sycl/element_wise.hpp +75 -0
  550. data/ext/sources/ggml/src/ggml-sycl/gemm.hpp +99 -0
  551. data/ext/sources/ggml/src/ggml-sycl/getrows.cpp +309 -0
  552. data/ext/sources/ggml/src/ggml-sycl/getrows.hpp +20 -0
  553. data/ext/{ggml → sources/ggml}/src/ggml-sycl/ggml-sycl.cpp +1004 -1240
  554. data/ext/sources/ggml/src/ggml-sycl/gla.cpp +106 -0
  555. data/ext/sources/ggml/src/ggml-sycl/gla.hpp +8 -0
  556. data/ext/sources/ggml/src/ggml-sycl/im2col.cpp +136 -0
  557. data/ext/sources/ggml/src/ggml-sycl/im2col.hpp +21 -0
  558. data/ext/{ggml → sources/ggml}/src/ggml-sycl/mmq.cpp +0 -1
  559. data/ext/sources/ggml/src/ggml-sycl/mmq.hpp +33 -0
  560. data/ext/{ggml → sources/ggml}/src/ggml-sycl/mmvq.cpp +261 -166
  561. data/ext/sources/ggml/src/ggml-sycl/mmvq.hpp +27 -0
  562. data/ext/{ggml → sources/ggml}/src/ggml-sycl/norm.cpp +204 -81
  563. data/ext/sources/ggml/src/ggml-sycl/norm.hpp +26 -0
  564. data/ext/{ggml → sources/ggml}/src/ggml-sycl/outprod.cpp +8 -17
  565. data/ext/sources/ggml/src/ggml-sycl/outprod.hpp +10 -0
  566. data/ext/sources/ggml/src/ggml-sycl/presets.hpp +74 -0
  567. data/ext/sources/ggml/src/ggml-sycl/quants.hpp +83 -0
  568. data/ext/sources/ggml/src/ggml-sycl/rope.cpp +361 -0
  569. data/ext/sources/ggml/src/ggml-sycl/rope.hpp +20 -0
  570. data/ext/{ggml → sources/ggml}/src/ggml-sycl/softmax.cpp +35 -25
  571. data/ext/sources/ggml/src/ggml-sycl/softmax.hpp +20 -0
  572. data/ext/sources/ggml/src/ggml-sycl/sycl_hw.cpp +13 -0
  573. data/ext/sources/ggml/src/ggml-sycl/sycl_hw.hpp +23 -0
  574. data/ext/{ggml → sources/ggml}/src/ggml-sycl/tsembd.cpp +3 -3
  575. data/ext/sources/ggml/src/ggml-sycl/tsembd.hpp +20 -0
  576. data/ext/sources/ggml/src/ggml-sycl/vecdotq.hpp +1215 -0
  577. data/ext/sources/ggml/src/ggml-sycl/wkv.cpp +293 -0
  578. data/ext/sources/ggml/src/ggml-sycl/wkv.hpp +10 -0
  579. data/ext/sources/ggml/src/ggml-vulkan/CMakeLists.txt +196 -0
  580. data/ext/sources/ggml/src/ggml-vulkan/cmake/host-toolchain.cmake.in +15 -0
  581. data/ext/{ggml → sources/ggml}/src/ggml-vulkan/ggml-vulkan.cpp +3130 -1087
  582. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/CMakeLists.txt +39 -0
  583. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/acc.comp +29 -0
  584. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/add.comp +29 -0
  585. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/argmax.comp +51 -0
  586. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/argsort.comp +69 -0
  587. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/clamp.comp +17 -0
  588. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/concat.comp +41 -0
  589. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/contig_copy.comp +49 -0
  590. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/conv2d_dw.comp +105 -0
  591. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/copy.comp +23 -0
  592. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/copy_from_quant.comp +51 -0
  593. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/copy_to_quant.comp +242 -0
  594. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/cos.comp +17 -0
  595. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/count_equal.comp +31 -0
  596. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_f32.comp +20 -0
  597. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.comp +462 -0
  598. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.comp +699 -0
  599. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_head.comp +13 -0
  600. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_m.comp +42 -0
  601. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_s.comp +35 -0
  602. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_s.comp +44 -0
  603. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xs.comp +43 -0
  604. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xxs.comp +48 -0
  605. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_s.comp +39 -0
  606. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_xxs.comp +49 -0
  607. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_nl.comp +32 -0
  608. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_xs.comp +34 -0
  609. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q2_k.comp +34 -0
  610. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q3_k.comp +42 -0
  611. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_0.comp +30 -0
  612. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_1.comp +32 -0
  613. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_k.comp +68 -0
  614. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_0.comp +34 -0
  615. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_1.comp +35 -0
  616. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_k.comp +70 -0
  617. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q6_k.comp +33 -0
  618. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q8_0.comp +31 -0
  619. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/diag_mask_inf.comp +34 -0
  620. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/div.comp +27 -0
  621. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp +337 -0
  622. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.comp +162 -0
  623. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp +360 -0
  624. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp +267 -0
  625. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_split_k_reduce.comp +59 -0
  626. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/gelu.comp +25 -0
  627. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/gelu_quick.comp +23 -0
  628. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/generic_binary_head.comp +64 -0
  629. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/generic_head.comp +9 -0
  630. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/generic_unary_head.comp +76 -0
  631. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/get_rows.comp +33 -0
  632. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_quant.comp +41 -0
  633. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/group_norm.comp +66 -0
  634. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/im2col.comp +100 -0
  635. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/l2_norm.comp +41 -0
  636. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/leaky_relu.comp +22 -0
  637. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul.comp +27 -0
  638. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_split_k_reduce.comp +48 -0
  639. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec.comp +169 -0
  640. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_base.comp +118 -0
  641. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_m.comp +82 -0
  642. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_s.comp +79 -0
  643. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_s.comp +90 -0
  644. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xs.comp +87 -0
  645. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xxs.comp +87 -0
  646. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_s.comp +90 -0
  647. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_xxs.comp +88 -0
  648. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_nc.comp +118 -0
  649. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_p021.comp +154 -0
  650. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q2_k.comp +130 -0
  651. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q3_k.comp +132 -0
  652. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q4_k.comp +136 -0
  653. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q5_k.comp +167 -0
  654. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q6_k.comp +130 -0
  655. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp +868 -0
  656. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp +441 -0
  657. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp +442 -0
  658. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_funcs.comp +99 -0
  659. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/norm.comp +44 -0
  660. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_adamw.comp +42 -0
  661. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/pad.comp +28 -0
  662. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/pool2d.comp +74 -0
  663. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/quantize_q8_1.comp +77 -0
  664. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/relu.comp +21 -0
  665. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/repeat.comp +26 -0
  666. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/repeat_back.comp +37 -0
  667. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp +52 -0
  668. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_back.comp +55 -0
  669. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rope_head.comp +58 -0
  670. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rope_multi.comp +60 -0
  671. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rope_neox.comp +43 -0
  672. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rope_norm.comp +43 -0
  673. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rope_vision.comp +47 -0
  674. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/scale.comp +24 -0
  675. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/sigmoid.comp +20 -0
  676. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/silu.comp +22 -0
  677. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/silu_back.comp +26 -0
  678. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/sin.comp +17 -0
  679. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/soft_max.comp +173 -0
  680. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_back.comp +50 -0
  681. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/square.comp +17 -0
  682. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/sub.comp +29 -0
  683. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.comp +37 -0
  684. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/tanh.comp +20 -0
  685. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/test_bfloat16_support.comp +7 -0
  686. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/test_coopmat2_support.comp +7 -0
  687. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/test_coopmat_support.comp +7 -0
  688. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/test_integer_dot_support.comp +7 -0
  689. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/timestep_embedding.comp +41 -0
  690. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/types.comp +1373 -0
  691. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/upscale.comp +36 -0
  692. data/ext/{ggml → sources/ggml}/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +193 -35
  693. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/wkv6.comp +87 -0
  694. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/wkv7.comp +91 -0
  695. data/ext/{ggml → sources/ggml}/src/ggml.c +676 -1820
  696. data/ext/sources/ggml/src/gguf.cpp +1330 -0
  697. data/ext/{include → sources/include}/whisper.h +68 -2
  698. data/ext/sources/src/CMakeLists.txt +143 -0
  699. data/ext/{src → sources/src}/coreml/whisper-decoder-impl.h +27 -15
  700. data/ext/{src → sources/src}/coreml/whisper-decoder-impl.m +35 -10
  701. data/ext/{src → sources/src}/coreml/whisper-encoder-impl.h +21 -9
  702. data/ext/{src → sources/src}/coreml/whisper-encoder-impl.m +28 -3
  703. data/ext/sources/src/coreml/whisper-encoder.mm +73 -0
  704. data/ext/sources/src/whisper-arch.h +197 -0
  705. data/ext/{src → sources/src}/whisper.cpp +1905 -374
  706. data/ext/sources/tests/CMakeLists.txt +105 -0
  707. data/ext/sources/tests/earnings21/eval.mk +58 -0
  708. data/ext/sources/tests/earnings21/eval.py +68 -0
  709. data/ext/sources/tests/earnings21/normalizers/__init__.py +2 -0
  710. data/ext/sources/tests/earnings21/normalizers/basic.py +80 -0
  711. data/ext/sources/tests/earnings21/normalizers/english.json +1741 -0
  712. data/ext/sources/tests/earnings21/normalizers/english.py +550 -0
  713. data/ext/sources/tests/earnings21/requirements.txt +6 -0
  714. data/ext/sources/tests/en-0-ref.txt +1 -0
  715. data/ext/sources/tests/en-1-ref.txt +1 -0
  716. data/ext/sources/tests/en-2-ref.txt +1 -0
  717. data/ext/sources/tests/es-0-ref.txt +1 -0
  718. data/ext/sources/tests/librispeech/eval.mk +39 -0
  719. data/ext/sources/tests/librispeech/eval.py +47 -0
  720. data/ext/sources/tests/librispeech/normalizers/__init__.py +2 -0
  721. data/ext/sources/tests/librispeech/normalizers/basic.py +80 -0
  722. data/ext/sources/tests/librispeech/normalizers/english.json +1741 -0
  723. data/ext/sources/tests/librispeech/normalizers/english.py +550 -0
  724. data/ext/sources/tests/librispeech/requirements.txt +6 -0
  725. data/ext/sources/tests/run-tests.sh +130 -0
  726. data/ext/sources/tests/test-c.c +3 -0
  727. data/ext/sources/tests/test-vad-full.cpp +54 -0
  728. data/ext/sources/tests/test-vad.cpp +83 -0
  729. data/ext/sources/tests/test-whisper.js +58 -0
  730. data/extsources.rb +33 -5
  731. data/lib/whisper/model/uri.rb +149 -128
  732. data/sig/whisper.rbs +480 -0
  733. data/tests/helper.rb +28 -0
  734. data/tests/test_callback.rb +45 -3
  735. data/tests/test_error.rb +2 -2
  736. data/tests/test_model.rb +38 -0
  737. data/tests/test_package.rb +18 -3
  738. data/tests/test_params.rb +145 -8
  739. data/tests/test_segment.rb +10 -19
  740. data/tests/test_vad.rb +19 -0
  741. data/tests/test_vad_params.rb +103 -0
  742. data/tests/test_whisper.rb +37 -37
  743. data/whispercpp.gemspec +5 -4
  744. metadata +766 -111
  745. data/ext/cpu.mk +0 -9
  746. data/ext/examples/dr_wav.h +0 -8815
  747. data/ext/ggml/src/ggml-cann/aclnn_ops.h +0 -592
  748. data/ext/ggml/src/ggml-cpu/ggml-cpu-aarch64.cpp +0 -4262
  749. data/ext/ggml/src/ggml-cpu/ggml-cpu.c +0 -14123
  750. data/ext/ggml/src/ggml-cpu/llamafile/sgemm.cpp +0 -1884
  751. data/ext/ggml/src/ggml-cpu/llamafile/sgemm.h +0 -14
  752. data/ext/ggml/src/ggml-metal/ggml-metal-impl.h +0 -288
  753. data/ext/ggml/src/ggml-sycl/element_wise.cpp +0 -1030
  754. data/ext/ggml/src/ggml-sycl/im2col.cpp +0 -126
  755. data/ext/ggml/src/ggml-sycl/rope.cpp +0 -276
  756. data/ext/ggml/src/ggml-sycl/wkv6.cpp +0 -141
  757. data/ext/metal-embed.mk +0 -17
  758. data/ext/metal.mk +0 -6
  759. data/ext/ruby_whisper.cpp +0 -1909
  760. data/ext/scripts/get-flags.mk +0 -38
  761. data/lib/whisper.rb +0 -2
  762. /data/ext/{ggml → sources/ggml}/include/ggml-blas.h +0 -0
  763. /data/ext/{ggml → sources/ggml}/include/ggml-cann.h +0 -0
  764. /data/ext/{ggml → sources/ggml}/include/ggml-cuda.h +0 -0
  765. /data/ext/{ggml → sources/ggml}/include/ggml-kompute.h +0 -0
  766. /data/ext/{ggml → sources/ggml}/include/ggml-opencl.h +0 -0
  767. /data/ext/{ggml → sources/ggml}/include/ggml-sycl.h +0 -0
  768. /data/ext/{ggml → sources/ggml}/src/ggml-amx/common.h +0 -0
  769. /data/ext/{ggml → sources/ggml}/src/ggml-amx/ggml-amx.cpp +0 -0
  770. /data/ext/{ggml → sources/ggml}/src/ggml-amx/mmq.cpp +0 -0
  771. /data/ext/{ggml → sources/ggml}/src/ggml-amx/mmq.h +0 -0
  772. /data/ext/{ggml → sources/ggml}/src/ggml-blas/ggml-blas.cpp +0 -0
  773. /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/ascendc_kernels.h +0 -0
  774. /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/get_row_f16.cpp +0 -0
  775. /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/get_row_f32.cpp +0 -0
  776. /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/get_row_q4_0.cpp +0 -0
  777. /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/get_row_q8_0.cpp +0 -0
  778. /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/quantize_f16_q8_0.cpp +0 -0
  779. /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/quantize_f32_q8_0.cpp +0 -0
  780. /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/quantize_float_to_q4_0.cpp +0 -0
  781. /data/ext/{ggml → sources/ggml}/src/ggml-cpu/amx/amx.h +0 -0
  782. /data/ext/{ggml → sources/ggml}/src/ggml-cpu/amx/common.h +0 -0
  783. /data/ext/{ggml → sources/ggml}/src/ggml-cpu/amx/mmq.cpp +0 -0
  784. /data/ext/{ggml → sources/ggml}/src/ggml-cpu/amx/mmq.h +0 -0
  785. /data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-aarch64.h +0 -0
  786. /data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-hbm.cpp +0 -0
  787. /data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-hbm.h +0 -0
  788. /data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-quants.h +0 -0
  789. /data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-traits.cpp +0 -0
  790. /data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-traits.h +0 -0
  791. /data/ext/{ggml → sources/ggml}/src/ggml-kompute/ggml-kompute.cpp +0 -0
  792. /data/ext/{ggml → sources/ggml}/src/ggml-quants.h +0 -0
  793. /data/ext/{ggml → sources/ggml}/src/ggml-threading.cpp +0 -0
  794. /data/ext/{ggml → sources/ggml}/src/ggml-threading.h +0 -0
  795. /data/ext/{src → sources/src}/coreml/whisper-encoder.h +0 -0
  796. /data/ext/{src → sources/src}/openvino/whisper-openvino-encoder.cpp +0 -0
  797. /data/ext/{src → sources/src}/openvino/whisper-openvino-encoder.h +0 -0
@@ -0,0 +1,3544 @@
1
+ // Copyright 2024 Mozilla Foundation
2
+ //
3
+ // Permission is hereby granted, free of charge, to any person obtaining
4
+ // a copy of this software and associated documentation files (the
5
+ // "Software"), to deal in the Software without restriction, including
6
+ // without limitation the rights to use, copy, modify, merge, publish,
7
+ // distribute, sublicense, and/or sell copies of the Software, and to
8
+ // permit persons to whom the Software is furnished to do so, subject to
9
+ // the following conditions:
10
+ //
11
+ // The above copyright notice and this permission notice shall be
12
+ // included in all copies or substantial portions of the Software.
13
+ //
14
+ // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
15
+ // EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16
+ // MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
17
+ // NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
18
+ // BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
19
+ // ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
20
+ // CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ // SOFTWARE.
22
+
23
+ //
24
+ // _ _ ___ _ _ ___
25
+ // | |_(_)_ _ _ _| _ ) | /_\ / __|
26
+ // | _| | ' \ || | _ \ |__ / _ \\__ \.
27
+ // \__|_|_||_\_, |___/____/_/ \_\___/
28
+ // |__/
29
+ //
30
+ // BASIC LINEAR ALGEBRA SUBPROGRAMS
31
+ //
32
+ //
33
+ // This file implements multithreaded CPU matrix multiplication for the
34
+ // common contiguous use case C = Aᵀ * B. These kernels are designed to
35
+ // have excellent performance[1] for matrices that fit in the CPU cache
36
+ // without imposing any overhead such as cache filling or malloc calls.
37
+ //
38
+ // This implementation does not guarantee any upper bound with rounding
39
+ // errors, which grow along with k. Our goal's to maximally exploit the
40
+ // hardware for performance, and then use whatever resources remain for
41
+ // improving numerical accuracy.
42
+ //
43
+ // [1] J. Tunney, ‘LLaMA Now Goes Faster on CPUs’, Mar. 2024. [Online].
44
+ // Available: https://justine.lol/matmul/. [Accessed: 29-Mar-2024].
45
+
46
+ #if defined(__GNUC__)
47
+ #pragma GCC diagnostic ignored "-Wpedantic"
48
+ #pragma GCC diagnostic ignored "-Wignored-attributes"
49
+ #endif
50
+
51
+ #include "sgemm.h"
52
+ #include "ggml-impl.h"
53
+ #include "ggml-cpu-impl.h"
54
+ #include "ggml-quants.h"
55
+
56
+ #include <atomic>
57
+ #include <array>
58
+ #include <type_traits>
59
+
60
+ #ifdef _MSC_VER
61
+ #define NOINLINE __declspec(noinline)
62
+ #else
63
+ #define NOINLINE __attribute__((__noinline__))
64
+ #endif
65
+
66
+ #if defined(__ARM_NEON) || defined(__AVX512F__)
67
+ #define VECTOR_REGISTERS 32
68
+ #else
69
+ #define VECTOR_REGISTERS 16
70
+ #endif
71
+
72
+ #define MM256_SET_M128I(a, b) _mm256_insertf128_si256(_mm256_castsi128_si256(b), (a), 1)
73
+
74
+ namespace {
75
+
76
+ inline float unhalf(ggml_fp16_t d) {
77
+ return GGML_FP16_TO_FP32(d);
78
+ }
79
+
80
+ ////////////////////////////////////////////////////////////////////////////////////////////////////
81
+ // VECTORIZED ARITHMETIC OPERATIONS
82
+
83
+ #if defined(__SSE__) || defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)
84
+ inline __m128 add(__m128 x, __m128 y) { return _mm_add_ps(x, y); }
85
+ inline __m128 sub(__m128 x, __m128 y) { return _mm_sub_ps(x, y); }
86
+ inline __m128 mul(__m128 x, __m128 y) { return _mm_mul_ps(x, y); }
87
+ #endif // __SSE__
88
+
89
+ #if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)
90
+ inline __m256 add(__m256 x, __m256 y) { return _mm256_add_ps(x, y); }
91
+ inline __m256 sub(__m256 x, __m256 y) { return _mm256_sub_ps(x, y); }
92
+ inline __m256 mul(__m256 x, __m256 y) { return _mm256_mul_ps(x, y); }
93
+ #endif // __AVX__
94
+
95
+ #if defined(__AVX512F__)
96
+ inline __m512 add(__m512 x, __m512 y) { return _mm512_add_ps(x, y); }
97
+ inline __m512 sub(__m512 x, __m512 y) { return _mm512_sub_ps(x, y); }
98
+ inline __m512 mul(__m512 x, __m512 y) { return _mm512_mul_ps(x, y); }
99
+ #endif // __AVX512F__
100
+
101
+ #if defined(__ARM_NEON)
102
+ inline float32x4_t add(float32x4_t x, float32x4_t y) { return vaddq_f32(x, y); }
103
+ inline float32x4_t sub(float32x4_t x, float32x4_t y) { return vsubq_f32(x, y); }
104
+ inline float32x4_t mul(float32x4_t x, float32x4_t y) { return vmulq_f32(x, y); }
105
+ #endif // __ARM_NEON
106
+
107
+ #if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
108
+ inline float16x8_t add(float16x8_t x, float16x8_t y) { return vaddq_f16(x, y); }
109
+ inline float16x8_t sub(float16x8_t x, float16x8_t y) { return vsubq_f16(x, y); }
110
+ inline float16x8_t mul(float16x8_t x, float16x8_t y) { return vmulq_f16(x, y); }
111
+ #endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
112
+
113
+ #if defined(__MMA__)
114
+ typedef vector unsigned char vec_t;
115
+ typedef __vector_quad acc_t;
116
+ #endif
117
+ ////////////////////////////////////////////////////////////////////////////////////////////////////
118
+ // VECTORIZED FUSED MULTIPLY ADD
119
+
120
+ /**
121
+ * Computes a * b + c.
122
+ */
123
+ template <typename T, typename U>
124
+ inline U madd(T a, T b, U c) {
125
+ return add(mul(a, b), c);
126
+ }
127
+
128
+ #if defined(__FMA__)
129
+ #if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)
130
+ template <>
131
+ inline __m256 madd(__m256 a, __m256 b, __m256 c) {
132
+ return _mm256_fmadd_ps(a, b, c);
133
+ }
134
+ #endif
135
+ #if defined(__AVX512F__)
136
+ template <>
137
+ inline __m512 madd(__m512 a, __m512 b, __m512 c) {
138
+ return _mm512_fmadd_ps(a, b, c);
139
+ }
140
+ #endif
141
+ #if defined(__AVX512BF16__)
142
+ template <>
143
+ inline __m512 madd(__m512bh a, __m512bh b, __m512 c) {
144
+ return _mm512_dpbf16_ps(c, a, b);
145
+ }
146
+ template <>
147
+ inline __m256 madd(__m256bh a, __m256bh b, __m256 c) {
148
+ return _mm256_dpbf16_ps(c, a, b);
149
+ }
150
+ #endif
151
+ #endif
152
+
153
+ #if defined(__ARM_FEATURE_FMA)
154
+ template <>
155
+ inline float32x4_t madd(float32x4_t a, float32x4_t b, float32x4_t c) {
156
+ return vfmaq_f32(c, b, a);
157
+ }
158
+ #if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && !defined(_MSC_VER)
159
+ template <>
160
+ inline float16x8_t madd(float16x8_t a, float16x8_t b, float16x8_t c) {
161
+ return vfmaq_f16(c, b, a);
162
+ }
163
+ #endif
164
+ #endif
165
+
166
+ ////////////////////////////////////////////////////////////////////////////////////////////////////
167
+ // VECTORIZED HORIZONTAL SUM
168
+
169
+ #if defined(__ARM_NEON)
170
+ inline float hsum(float32x4_t x) {
171
+ return vaddvq_f32(x);
172
+ }
173
+ #endif // __ARM_NEON
174
+
175
+ #if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && !defined(_MSC_VER)
176
+ inline float hsum(float16x8_t x) {
177
+ return vaddvq_f32(vaddq_f32(vcvt_f32_f16(vget_low_f16(x)),
178
+ vcvt_f32_f16(vget_high_f16(x))));
179
+ }
180
+ #endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
181
+
182
+ #if defined(__SSE__) || defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)
183
+ inline float hsum(__m128 x) {
184
+ #if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)
185
+ x = _mm_add_ps(x, _mm_movehl_ps(x, x));
186
+ x = _mm_add_ss(x, _mm_movehdup_ps(x));
187
+ #else
188
+ __m128 t;
189
+ t = _mm_shuffle_ps(x, x, _MM_SHUFFLE(2, 3, 0, 1));
190
+ x = _mm_add_ps(x, t);
191
+ t = _mm_movehl_ps(t, x);
192
+ x = _mm_add_ss(x, t);
193
+ #endif
194
+ return _mm_cvtss_f32(x);
195
+ }
196
+ #endif
197
+
198
+ #if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)
199
+ inline float hsum(__m256 x) {
200
+ return hsum(_mm_add_ps(_mm256_extractf128_ps(x, 1),
201
+ _mm256_castps256_ps128(x)));
202
+ }
203
+ #endif // __AVX__
204
+
205
+ #if defined(__AVX512F__)
206
+ inline float hsum(__m512 x) {
207
+ return _mm512_reduce_add_ps(x);
208
+ }
209
+ #endif // __AVX512F__
210
+
211
+ ////////////////////////////////////////////////////////////////////////////////////////////////////
212
+ // VECTORIZED MEMORY LOADING
213
+
214
+ template <typename T, typename U> T load(const U *);
215
+
216
+ #if defined(__ARM_NEON)
217
+ template <> inline float32x4_t load(const float *p) {
218
+ return vld1q_f32(p);
219
+ }
220
+ #if !defined(_MSC_VER)
221
+ // FIXME: this should check for __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
222
+ template <> inline float16x8_t load(const ggml_fp16_t *p) {
223
+ return vld1q_f16((const float16_t *)p);
224
+ }
225
+ template <> inline float32x4_t load(const ggml_fp16_t *p) {
226
+ return vcvt_f32_f16(vld1_f16((const float16_t *)p));
227
+ }
228
+ #endif // _MSC_VER
229
+ #endif // __ARM_NEON
230
+
231
+ #if defined(__SSE__) || defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)
232
+ template <> inline __m128 load(const float *p) {
233
+ return _mm_loadu_ps(p);
234
+ }
235
+ #endif // __SSE__
236
+
237
+ #if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)
238
+ template <> inline __m256 load(const float *p) {
239
+ return _mm256_loadu_ps(p);
240
+ }
241
+ #endif // __AVX__
242
+
243
+ #if defined(__AVX2__) || defined(__AVX512F__)
244
+ template <> inline __m256 load(const ggml_bf16_t *p) {
245
+ return _mm256_castsi256_ps(
246
+ _mm256_slli_epi32(_mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i *)p)), 16));
247
+ }
248
+ #endif // __AVX2__
249
+
250
+ #if defined(__F16C__)
251
+ template <> inline __m256 load(const ggml_fp16_t *p) {
252
+ return _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)p));
253
+ }
254
+ #endif // __F16C__
255
+
256
+ #if defined(__AVX512F__)
257
+ template <> inline __m512 load(const float *p) {
258
+ return _mm512_loadu_ps(p);
259
+ }
260
+ template <> inline __m512 load(const ggml_fp16_t *p) {
261
+ return _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)p));
262
+ }
263
+ template <> inline __m512 load(const ggml_bf16_t *p) {
264
+ return _mm512_castsi512_ps(
265
+ _mm512_slli_epi32(_mm512_cvtepu16_epi32(_mm256_loadu_si256((const __m256i *)p)), 16));
266
+ }
267
+ #endif // __AVX512F__
268
+
269
+ #if defined(__AVX512BF16__)
270
+ template <> inline __m512bh load(const ggml_bf16_t *p) {
271
+ return (__m512bh)_mm512_loadu_ps((const float *)p);
272
+ }
273
+ template <> inline __m256bh load(const ggml_bf16_t *p) {
274
+ return (__m256bh)_mm256_loadu_ps((const float *)p);
275
+ }
276
+ template <> inline __m512bh load(const float *p) {
277
+ return _mm512_cvtne2ps_pbh(_mm512_loadu_ps(p + 16), _mm512_loadu_ps(p));
278
+ }
279
+ template <> inline __m256bh load(const float *p) {
280
+ return _mm512_cvtneps_pbh(_mm512_loadu_ps(p));
281
+ }
282
+ #endif
283
+
284
+ ////////////////////////////////////////////////////////////////////////////////////////////////////
285
+ // FLOATING POINT MATRIX MULTIPLICATION
286
+
287
+ template <int M>
288
+ static inline int64_t BLOCK_SIZE(size_t m) {
289
+ const int64_t NB_BLOC_M = (m + M - 1) / M;
290
+ return (m % NB_BLOC_M == 0) ? m / NB_BLOC_M : (m / NB_BLOC_M) + 1;
291
+ }
292
+
293
+ static constexpr inline int64_t BLOC_POS(int64_t ib, int64_t ibN, int64_t bloc_size) {
294
+ return ib < ibN ? ib * bloc_size : ibN * bloc_size + (ib - ibN) * (bloc_size - 1);
295
+ }
296
+
297
+ template <int KN, typename D, typename V, typename TA, typename TB, typename TC>
298
+ class tinyBLAS {
299
+ public:
300
+ tinyBLAS(const ggml_compute_params * params, int64_t k,
301
+ const TA *A, int64_t lda,
302
+ const TB *B, int64_t ldb,
303
+ TC *C, int64_t ldc)
304
+ : params(params), A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc) {
305
+ }
306
+
307
+ bool matmul(int64_t m, int64_t n) {
308
+ if (k % KN != 0)
309
+ return false;
310
+ // compute RM for only need tile with size RM&RM-1
311
+ #if VECTOR_REGISTERS == 32
312
+ if (m % 16 == 0 && (m/16 >= params->nth)) {
313
+ const int64_t SIZE_N = BLOCK_SIZE<6>(n);
314
+ mnpack<4, 6, 4>(m, n, SIZE_N, 12);
315
+ return true;
316
+ }
317
+ if (m % 8 == 0 ) {
318
+ const int64_t SIZE_N = BLOCK_SIZE<6>(n);
319
+ mnpack<4, 6, 2>(m, n, SIZE_N, 12);
320
+ return true;
321
+ }
322
+ if (m % 4 == 0) {
323
+ const int64_t SIZE_N = BLOCK_SIZE<6>(n);
324
+ mnpack<4, 6, 1>(m, n, SIZE_N, 12);
325
+ return true;
326
+ }
327
+ #else // VECTOR_REGISTERS == 16
328
+ if (m % 16 == 0 && (m/16 >= params->nth)) {
329
+ const int64_t SIZE_N = BLOCK_SIZE<3>(n);
330
+ mnpack<4, 3, 4>(m, n, SIZE_N, 24);
331
+ return true;
332
+ }
333
+ if (m % 8 == 0 ) {
334
+ const int64_t SIZE_N = BLOCK_SIZE<3>(n);
335
+ mnpack<4, 3, 2>(m, n, SIZE_N, 24);
336
+ return true;
337
+ }
338
+ if (m % 4 == 0) {
339
+ const int64_t SIZE_N = BLOCK_SIZE<3>(n);
340
+ mnpack<4, 3, 1>(m, n, SIZE_N, 24);
341
+ return true;
342
+ }
343
+ #endif
344
+ return false;
345
+ }
346
+
347
+ private:
348
+ template <int RM, int RN, int BM>
349
+ inline void mnpack(int64_t m, int64_t n, int64_t SIZE_N, int64_t BN) {
350
+ if (SIZE_N == RN) {
351
+ return gemm<RM, RN, BM>(m, n, BN);
352
+ }
353
+ if constexpr (RN > 1) {
354
+ return mnpack<RM, RN-1, BM>(m, n, SIZE_N, BN);
355
+ } else {
356
+ GGML_LOG_ERROR("mnpack<%d, %d> bloc size not supported\n", RM, (int)SIZE_N);
357
+ GGML_ASSERT(false); // we have miss something.
358
+ }
359
+ }
360
+
361
+ template <int RM, int RN>
362
+ inline void gemm_bloc(int64_t ii, int64_t jj) {
363
+ D Cv[RN][RM] = {};
364
+ for (int64_t l = 0; l < k; l += KN) {
365
+ // help compiler for op order.
366
+ if constexpr (RM <= RN) {
367
+ V Av[RM];
368
+ for (int64_t i = 0; i < RM; ++i) {
369
+ Av[i] = load<V>(A + lda * (ii + i) + l);
370
+ }
371
+ for (int64_t j = 0; j < RN; ++j) {
372
+ V Bv = load<V>(B + ldb * (jj + j) + l);
373
+ for (int64_t i = 0; i < RM; ++i) {
374
+ Cv[j][i] = madd(Av[i], Bv, Cv[j][i]);
375
+ }
376
+ }
377
+ } else {
378
+ V Bv[RN];
379
+ for (int64_t j = 0; j < RN; ++j) {
380
+ Bv[j] = load<V>(B + ldb * (jj + j) + l);
381
+ }
382
+ for (int64_t i = 0; i < RM; ++i) {
383
+ V Av = load<V>(A + lda * (ii + i) + l);
384
+ for (int64_t j = 0; j < RN; ++j) {
385
+ Cv[j][i] = madd(Av, Bv[j], Cv[j][i]);
386
+ }
387
+ }
388
+ }
389
+ }
390
+ for (int64_t j = 0; j < RN; ++j)
391
+ for (int64_t i = 0; i < RM; ++i)
392
+ C[ldc * (jj + j) + (ii + i)] = hsum(Cv[j][i]);
393
+ }
394
+
395
+ template <int RM, int RN, int BM>
396
+ NOINLINE void gemm(int64_t m, int64_t n, int64_t BN) {
397
+ static std::atomic<int64_t> current_chunk;
398
+
399
+ GGML_ASSERT(m % (RM * BM) == 0);
400
+ const int64_t ytiles = m / (RM * BM);
401
+ const int64_t xtiles = (n + RN -1) / RN;
402
+ const int64_t jj_RN = (xtiles - (xtiles * RN - n));
403
+
404
+ // "round" bloc_size to "nearest" BN
405
+ const int64_t NB_BN = xtiles < BN ? 1 : (xtiles + BN / 2) / BN;
406
+ const int64_t SIZE_BN = xtiles % NB_BN == 0 ? xtiles / NB_BN : xtiles / NB_BN + 1;
407
+ const int64_t jj_BN = (NB_BN - (NB_BN * SIZE_BN - xtiles));
408
+ const int64_t nb_job = ytiles * NB_BN;
409
+
410
+ if (params->ith == 0) {
411
+ GGML_ASSERT( jj_BN * SIZE_BN + (NB_BN - jj_BN) * (SIZE_BN - 1) == xtiles);
412
+ // Every thread starts at ith, so the first unprocessed chunk is nth. This save a bit of coordination right at the start.
413
+ std::atomic_store_explicit(&current_chunk, (int64_t)params->nth, std::memory_order_relaxed);
414
+ }
415
+
416
+ ggml_barrier(params->threadpool);
417
+
418
+ int64_t job = params->ith;
419
+ while (job < nb_job) {
420
+ const int64_t ii = (job % ytiles) * RM * BM;
421
+ const int64_t jb = job / ytiles;
422
+ const int64_t jr0 = BLOC_POS(jb , jj_BN, SIZE_BN);
423
+ const int64_t jrN = BLOC_POS(jb+1, jj_BN, SIZE_BN);
424
+
425
+ const int64_t jj0 = BLOC_POS(jr0, jj_RN, RN);
426
+ const int64_t jj2 = BLOC_POS(jrN, jj_RN, RN);
427
+ const int64_t jj1 = jj2 < jj_RN * RN ? jj2 : jj_RN * RN;
428
+
429
+ for (int64_t bi = 0; bi < BM * RM; bi += RM) {
430
+ int64_t jj = jj0;
431
+ for (; jj < jj1; jj += RN) {
432
+ gemm_bloc<RM, RN>(ii + bi, jj);
433
+ }
434
+ if constexpr (RN > 1) {
435
+ for (; jj < jj2; jj += RN - 1) {
436
+ gemm_bloc<RM, RN-1>(ii + bi, jj);
437
+ }
438
+ }
439
+ GGML_ASSERT(jj == jj2);
440
+ }
441
+
442
+ // next step.
443
+ job = std::atomic_fetch_add_explicit(&current_chunk, (int64_t)1, std::memory_order_relaxed);
444
+ }
445
+
446
+ ggml_barrier(params->threadpool);
447
+ return;
448
+ }
449
+
450
+ const ggml_compute_params * params;
451
+ const TA *const A;
452
+ const TB *const B;
453
+ TC *const C;
454
+ const int64_t k;
455
+ const int64_t lda;
456
+ const int64_t ldb;
457
+ const int64_t ldc;
458
+ };
459
+
460
+ //////////////////////////////////////////////////////////////////////////////////////////
461
+ // QUANT ZERO MATRIX MULTIPLICATION
462
+
463
+ #if defined(__ARM_FEATURE_DOTPROD)
464
+ template <typename TA>
465
+ class tinyBLAS_Q0_ARM {
466
+ public:
467
+ tinyBLAS_Q0_ARM(int64_t k,
468
+ const TA *A, int64_t lda,
469
+ const block_q8_0 *B, int64_t ldb,
470
+ float *C, int64_t ldc,
471
+ int ith, int nth)
472
+ : A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc), ith(ith), nth(nth) {
473
+ }
474
+
475
+ void matmul(int64_t m, int64_t n) {
476
+ mnpack(0, m, 0, n);
477
+ }
478
+
479
+ private:
480
+ NOINLINE void mnpack(int64_t m0, int64_t m, int64_t n0, int64_t n) {
481
+ int64_t mc, nc, mp, np;
482
+ switch ((MIN(m - m0, 3) << 4) | MIN(n - n0, 3ll)) {
483
+ case 0x33:
484
+ mc = 3;
485
+ nc = 3;
486
+ gemm<3, 3>(m0, m, n0, n);
487
+ break;
488
+ case 0x32:
489
+ mc = 3;
490
+ nc = 2;
491
+ gemm<3, 2>(m0, m, n0, n);
492
+ break;
493
+ case 0x23:
494
+ mc = 2;
495
+ nc = 3;
496
+ gemm<2, 3>(m0, m, n0, n);
497
+ break;
498
+ case 0x22:
499
+ mc = 2;
500
+ nc = 2;
501
+ gemm<2, 2>(m0, m, n0, n);
502
+ break;
503
+ case 0x31:
504
+ mc = 3;
505
+ nc = 1;
506
+ gemm<3, 1>(m0, m, n0, n);
507
+ break;
508
+ case 0x13:
509
+ mc = 1;
510
+ nc = 3;
511
+ gemm<1, 3>(m0, m, n0, n);
512
+ break;
513
+ case 0x21:
514
+ mc = 2;
515
+ nc = 1;
516
+ gemm<2, 1>(m0, m, n0, n);
517
+ break;
518
+ case 0x12:
519
+ mc = 1;
520
+ nc = 2;
521
+ gemm<1, 2>(m0, m, n0, n);
522
+ break;
523
+ case 0x11:
524
+ mc = 1;
525
+ nc = 1;
526
+ gemm<1, 1>(m0, m, n0, n);
527
+ break;
528
+ default:
529
+ return;
530
+ }
531
+ mp = m0 + (m - m0) / mc * mc;
532
+ np = n0 + (n - n0) / nc * nc;
533
+ mnpack(mp, m, n0, np);
534
+ mnpack(m0, m, np, n);
535
+ }
536
+
537
+ template <int RM, int RN>
538
+ NOINLINE void gemm(int64_t m0, int64_t m, int64_t n0, int64_t n) {
539
+ int64_t ytiles = (m - m0) / RM;
540
+ int64_t xtiles = (n - n0) / RN;
541
+ int64_t tiles = xtiles * ytiles;
542
+ int64_t duty = (tiles + nth - 1) / nth;
543
+ int64_t start = duty * ith;
544
+ int64_t end = start + duty;
545
+ if (end > tiles)
546
+ end = tiles;
547
+ for (int64_t job = start; job < end; ++job) {
548
+ int64_t ii = m0 + job / xtiles * RM;
549
+ int64_t jj = n0 + job % xtiles * RN;
550
+ float32x4_t Cv[RN][RM] = {};
551
+ for (int64_t l = 0; l < k; ++l)
552
+ for (int64_t j = 0; j < RN; ++j)
553
+ for (int64_t i = 0; i < RM; ++i)
554
+ Cv[j][i] = vmlaq_n_f32(Cv[j][i],
555
+ vcvtq_f32_s32(vdotq_s32(
556
+ vdotq_s32(vdupq_n_s32(0),
557
+ load_lo(A + lda * (ii + i) + l),
558
+ load_lo(B + ldb * (jj + j) + l)),
559
+ load_hi(A + lda * (ii + i) + l),
560
+ load_hi(B + ldb * (jj + j) + l))),
561
+ unhalf(A[lda * (ii + i) + l].d) *
562
+ unhalf(B[ldb * (jj + j) + l].d));
563
+ for (int64_t j = 0; j < RN; ++j)
564
+ for (int64_t i = 0; i < RM; ++i)
565
+ C[ldc * (jj + j) + (ii + i)] = hsum(Cv[j][i]);
566
+ }
567
+ }
568
+
569
+ inline int8x16_t load_lo(const block_q8_0 *b) {
570
+ return vld1q_s8(b->qs);
571
+ }
572
+
573
+ inline int8x16_t load_hi(const block_q8_0 *b) {
574
+ return vld1q_s8(b->qs + 16);
575
+ }
576
+
577
+ inline int8x16_t load_lo(const block_q4_0 *b) {
578
+ return vsubq_s8(vreinterpretq_s8_u8(vandq_u8(vld1q_u8(b->qs),
579
+ vdupq_n_u8(0x0f))),
580
+ vdupq_n_s8(0x8));
581
+ }
582
+
583
+ inline int8x16_t load_hi(const block_q4_0 *b) {
584
+ return vsubq_s8(vreinterpretq_s8_u8(vshrq_n_u8(vld1q_u8(b->qs), 4)),
585
+ vdupq_n_s8(0x8));
586
+ }
587
+
588
+ const TA *const A;
589
+ const block_q8_0 *const B;
590
+ float *const C;
591
+ const int64_t k;
592
+ const int64_t lda;
593
+ const int64_t ldb;
594
+ const int64_t ldc;
595
+ const int ith;
596
+ const int nth;
597
+ };
598
+ #endif // __ARM_FEATURE_DOTPROD
599
+
600
+ #if defined(__AVX2__) || defined(__AVX512F__) || defined(__AVX__)
601
+ template <typename TA, typename TB, typename TC>
602
+ class tinyBLAS_Q0_AVX {
603
+ public:
604
+ tinyBLAS_Q0_AVX(int64_t k,
605
+ const TA *A, int64_t lda,
606
+ const TB *B, int64_t ldb,
607
+ TC *C, int64_t ldc,
608
+ int ith, int nth)
609
+ : A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc), ith(ith), nth(nth) {
610
+ const int8_t kvalues_iq4nl[16] = {
611
+ -127, -104, -83, -65,
612
+ -49, -35, -22, -10,
613
+ 1, 13, 25, 38,
614
+ 53, 69, 89, 113
615
+ };
616
+
617
+ iq4nlt = _mm_loadu_si128((const __m128i *)kvalues_iq4nl);
618
+ }
619
+
620
+ void matmul(int64_t m, int64_t n) {
621
+ mnpack(0, m, 0, n);
622
+ }
623
+
624
+ private:
625
+ void mnpack(int64_t m0, int64_t m, int64_t n0, int64_t n) {
626
+ int64_t mc, nc, mp, np;
627
+ switch ((MIN(m - m0, 4) << 4) | MIN(n - n0, 4)) {
628
+ #if VECTOR_REGISTERS == 32
629
+ case 0x44:
630
+ mc = 4;
631
+ nc = 4;
632
+ #if defined(__AVX2__) && defined(__F16C__)
633
+ gemm4xN<4>(m0, m, n0, n);
634
+ #else
635
+ gemm<4, 4>(m0, m, n0, n);
636
+ #endif
637
+ break;
638
+ case 0x43:
639
+ mc = 4;
640
+ nc = 3;
641
+ #if defined(__AVX2__) && defined(__F16C__)
642
+ gemm4xN<3>(m0, m, n0, n);
643
+ #else
644
+ gemm<4, 3>(m0, m, n0, n);
645
+ #endif
646
+ break;
647
+ case 0x34:
648
+ mc = 3;
649
+ nc = 4;
650
+ #if defined(__AVX2__) && defined(__F16C__)
651
+ gemmMx4<3>(m0, m, n0, n);
652
+ #else
653
+ gemm<3, 4>(m0, m, n0, n);
654
+ #endif
655
+ break;
656
+ case 0x33:
657
+ mc = 3;
658
+ nc = 3;
659
+ gemm<3, 3>(m0, m, n0, n);
660
+ break;
661
+ case 0x42:
662
+ mc = 4;
663
+ nc = 2;
664
+ #if defined(__AVX2__) && defined(__F16C__)
665
+ gemm4xN<2>(m0, m, n0, n);
666
+ #else
667
+ gemm<4, 2>(m0, m, n0, n);
668
+ #endif
669
+ break;
670
+ case 0x24:
671
+ mc = 2;
672
+ nc = 4;
673
+ #if defined(__AVX2__) && defined(__F16C__)
674
+ gemmMx4<2>(m0, m, n0, n);
675
+ #else
676
+ gemm<2, 4>(m0, m, n0, n);
677
+ #endif
678
+ break;
679
+ #else
680
+ case 0x44:
681
+ case 0x43:
682
+ case 0x42:
683
+ mc = 4;
684
+ nc = 2;
685
+ #if defined(__AVX2__) && defined(__F16C__)
686
+ gemm4xN<2>(m0, m, n0, n);
687
+ #else
688
+ gemm<4, 2>(m0, m, n0, n);
689
+ #endif
690
+ break;
691
+ case 0x34:
692
+ case 0x24:
693
+ mc = 2;
694
+ nc = 4;
695
+ #if defined(__AVX2__) && defined(__F16C__)
696
+ gemmMx4<2>(m0, m, n0, n);
697
+ #else
698
+ gemm<2, 4>(m0, m, n0, n);
699
+ #endif
700
+ break;
701
+ case 0x33:
702
+ #endif
703
+ case 0x32:
704
+ mc = 3;
705
+ nc = 2;
706
+ gemm<3, 2>(m0, m, n0, n);
707
+ break;
708
+ case 0x23:
709
+ mc = 2;
710
+ nc = 3;
711
+ gemm<2, 3>(m0, m, n0, n);
712
+ break;
713
+ case 0x41:
714
+ mc = 4;
715
+ nc = 1;
716
+ #if defined(__AVX2__) && defined(__F16C__)
717
+ gemm4xN<1>(m0, m, n0, n);
718
+ #else
719
+ gemm<4, 1>(m0, m, n0, n);
720
+ #endif
721
+ break;
722
+ case 0x22:
723
+ mc = 2;
724
+ nc = 2;
725
+ gemm<2, 2>(m0, m, n0, n);
726
+ break;
727
+ case 0x14:
728
+ mc = 1;
729
+ nc = 4;
730
+ #if defined(__AVX2__) && defined(__F16C__)
731
+ gemmMx4<1>(m0, m, n0, n);
732
+ #else
733
+ gemm<1, 4>(m0, m, n0, n);
734
+ #endif
735
+ break;
736
+ case 0x31:
737
+ mc = 3;
738
+ nc = 1;
739
+ gemm<3, 1>(m0, m, n0, n);
740
+ break;
741
+ case 0x13:
742
+ mc = 1;
743
+ nc = 3;
744
+ gemm<1, 3>(m0, m, n0, n);
745
+ break;
746
+ case 0x21:
747
+ mc = 2;
748
+ nc = 1;
749
+ gemm<2, 1>(m0, m, n0, n);
750
+ break;
751
+ case 0x12:
752
+ mc = 1;
753
+ nc = 2;
754
+ gemm<1, 2>(m0, m, n0, n);
755
+ break;
756
+ case 0x11:
757
+ mc = 1;
758
+ nc = 1;
759
+ gemm<1, 1>(m0, m, n0, n);
760
+ break;
761
+ default:
762
+ return;
763
+ }
764
+ mp = m0 + (m - m0) / mc * mc;
765
+ np = n0 + (n - n0) / nc * nc;
766
+ mnpack(mp, m, n0, np);
767
+ mnpack(m0, m, np, n);
768
+ }
769
+
770
+ #if defined(__AVX2__) && defined(__F16C__)
771
+ // Templated functions for gemm of dimensions 4xN
772
+ template <int RN>
773
+ NOINLINE void gemm4xN(int64_t m0, int64_t m, int64_t n0, int64_t n) {
774
+ int64_t ytiles = (m - m0) / 4;
775
+ int64_t xtiles = (n - n0) / RN;
776
+ int64_t tiles = xtiles * ytiles;
777
+ int64_t duty = (tiles + nth - 1) / nth;
778
+ int64_t start = duty * ith;
779
+ int64_t end = start + duty;
780
+ if (end > tiles)
781
+ end = tiles;
782
+ for (int64_t job = start; job < end; ++job) {
783
+ int64_t ii = m0 + job / xtiles * 4;
784
+ int64_t jj = n0 + job % xtiles * RN;
785
+ __m256 Cv[RN][4] = {};
786
+ for (int64_t l = 0; l < k; ++l) {
787
+ uint64_t a_delta = ((uint64_t)A[lda * (ii + 3) + l].d << 48) | ((uint64_t)A[lda * (ii + 2) + l].d << 32) | ((uint64_t)A[lda * (ii + 1) + l].d << 16) | (A[lda * (ii + 0) + l].d);
788
+ // Convert delta values for four blocks to float values
789
+ __m128 da = _mm_cvtph_ps(_mm_set_epi64x(0, a_delta));
790
+ __m256i avec0 = load(A + lda * (ii + 0) + l);
791
+ __m256i avec1 = load(A + lda * (ii + 1) + l);
792
+ __m256i avec2 = load(A + lda * (ii + 2) + l);
793
+ __m256i avec3 = load(A + lda * (ii + 3) + l);
794
+ for (int64_t j = 0; j < RN; ++j) {
795
+ __m128 db = _mm_set1_ps(unhalf(B[ldb * (jj + j) + l].d));
796
+ // Computation of product of delta values for four blocks and replicate it across 256 bit lane
797
+ __m256 dvec = _mm256_castps128_ps256(_mm_mul_ps(da, db));
798
+ dvec = _mm256_permute2f128_ps(dvec ,dvec, 0);
799
+ // Computation of dot product and multiplication with appropriate delta value products
800
+ Cv[j][0] = madd(_mm256_shuffle_ps(dvec, dvec, 0),
801
+ updot(_mm256_sign_epi8(avec0, avec0),
802
+ _mm256_sign_epi8(load(B + ldb * (jj + j) + l), avec0)),
803
+ Cv[j][0]);
804
+ Cv[j][1] = madd(_mm256_shuffle_ps(dvec, dvec, 85),
805
+ updot(_mm256_sign_epi8(avec1, avec1),
806
+ _mm256_sign_epi8(load(B + ldb * (jj + j) + l), avec1)),
807
+ Cv[j][1]);
808
+ Cv[j][2] = madd(_mm256_shuffle_ps(dvec, dvec, 170),
809
+ updot(_mm256_sign_epi8(avec2, avec2),
810
+ _mm256_sign_epi8(load(B + ldb * (jj + j) + l), avec2)),
811
+ Cv[j][2]);
812
+ Cv[j][3] = madd(_mm256_shuffle_ps(dvec, dvec, 255),
813
+ updot(_mm256_sign_epi8(avec3, avec3),
814
+ _mm256_sign_epi8(load(B + ldb * (jj + j) + l), avec3)),
815
+ Cv[j][3]);
816
+ }
817
+ }
818
+
819
+ for (int64_t j = 0; j < RN; ++j)
820
+ for (int64_t i = 0; i < 4; ++i)
821
+ C[ldc * (jj + j) + (ii + i)] = hsum(Cv[j][i]);
822
+ }
823
+ }
824
+
825
+ // Templated functions for gemm of dimensions Mx4
826
+ template <int RM>
827
+ NOINLINE void gemmMx4(int64_t m0, int64_t m, int64_t n0, int64_t n) {
828
+ int64_t ytiles = (m - m0) / RM;
829
+ int64_t xtiles = (n - n0) / 4;
830
+ int64_t tiles = xtiles * ytiles;
831
+ int64_t duty = (tiles + nth - 1) / nth;
832
+ int64_t start = duty * ith;
833
+ int64_t end = start + duty;
834
+ if (end > tiles)
835
+ end = tiles;
836
+ for (int64_t job = start; job < end; ++job) {
837
+ int64_t ii = m0 + job / xtiles * RM;
838
+ int64_t jj = n0 + job % xtiles * 4;
839
+ __m256 Cv[4][RM] = {};
840
+ for (int64_t l = 0; l < k; ++l) {
841
+ uint64_t b_delta = ((uint64_t)B[ldb * (jj + 3) + l].d << 48) | ((uint64_t)B[ldb * (jj + 2) + l].d << 32) | ((uint64_t)B[ldb * (jj + 1) + l].d << 16) | (B[ldb * (jj + 0) + l].d);
842
+ // Convert delta values for four blocks to float values
843
+ __m128 db = _mm_cvtph_ps(_mm_set_epi64x(0, b_delta));
844
+ __m256i bvec0 = load(B + ldb * (jj + 0) + l);
845
+ __m256i bvec1 = load(B + ldb * (jj + 1) + l);
846
+ __m256i bvec2 = load(B + ldb * (jj + 2) + l);
847
+ __m256i bvec3 = load(B + ldb * (jj + 3) + l);
848
+ for (int64_t i = 0; i < RM; ++i) {
849
+ __m128 da = _mm_set1_ps(unhalf((A[lda * (ii + i) + l].d)));
850
+ // Computation of product of delta values for four blocks and replicate it across 256 bit lane
851
+ __m256 dvec = _mm256_castps128_ps256(_mm_mul_ps(da, db));
852
+ dvec = _mm256_permute2f128_ps(dvec ,dvec, 0);
853
+ // Computation of dot product and multiplication with appropriate delta value products
854
+ Cv[0][i] = madd(_mm256_shuffle_ps(dvec, dvec, 0),
855
+ updot(_mm256_sign_epi8(load(A + lda * (ii + i) + l),
856
+ load(A + lda * (ii + i) + l)),
857
+ _mm256_sign_epi8(bvec0, load(A + lda * (ii + i) + l))),
858
+ Cv[0][i]);
859
+ Cv[1][i] = madd(_mm256_shuffle_ps(dvec, dvec, 85),
860
+ updot(_mm256_sign_epi8(load(A + lda * (ii + i) + l),
861
+ load(A + lda * (ii + i) + l)),
862
+ _mm256_sign_epi8(bvec1, load(A + lda * (ii + i) + l))),
863
+ Cv[1][i]);
864
+ Cv[2][i] = madd(_mm256_shuffle_ps(dvec, dvec, 170),
865
+ updot(_mm256_sign_epi8(load(A + lda * (ii + i) + l),
866
+ load(A + lda * (ii + i) + l)),
867
+ _mm256_sign_epi8(bvec2, load(A + lda * (ii + i) + l))),
868
+ Cv[2][i]);
869
+ Cv[3][i] = madd(_mm256_shuffle_ps(dvec, dvec, 255),
870
+ updot(_mm256_sign_epi8(load(A + lda * (ii + i) + l),
871
+ load(A + lda * (ii + i) + l)),
872
+ _mm256_sign_epi8(bvec3, load(A + lda * (ii + i) + l))),
873
+ Cv[3][i]);
874
+ }
875
+ }
876
+ for (int64_t j = 0; j < 4; ++j)
877
+ for (int64_t i = 0; i < RM; ++i)
878
+ C[ldc * (jj + j) + (ii + i)] = hsum(Cv[j][i]);
879
+ }
880
+ }
881
+ #endif
882
+
883
+ template <int RM, int RN>
884
+ NOINLINE void gemm(int64_t m0, int64_t m, int64_t n0, int64_t n) {
885
+ int64_t ytiles = (m - m0) / RM;
886
+ int64_t xtiles = (n - n0) / RN;
887
+ int64_t tiles = xtiles * ytiles;
888
+ int64_t duty = (tiles + nth - 1) / nth;
889
+ int64_t start = duty * ith;
890
+ int64_t end = start + duty;
891
+ if (end > tiles)
892
+ end = tiles;
893
+ for (int64_t job = start; job < end; ++job) {
894
+ int64_t ii = m0 + job / xtiles * RM;
895
+ int64_t jj = n0 + job % xtiles * RN;
896
+ __m256 Cv[RN][RM] = {};
897
+ for (int64_t l = 0; l < k; ++l)
898
+ for (int64_t j = 0; j < RN; ++j)
899
+ for (int64_t i = 0; i < RM; ++i) {
900
+ #if defined(__AVX2__)
901
+ __m256 udTmp = updot(_mm256_sign_epi8(load(A + lda * (ii + i) + l),
902
+ load(A + lda * (ii + i) + l)),
903
+ _mm256_sign_epi8(load(B + ldb * (jj + j) + l),
904
+ load(A + lda * (ii + i) + l)));
905
+ #else
906
+ __m128i ali0 = load0(A + lda * (ii + i) + l);
907
+ __m128i ali1 = load1(A + lda * (ii + i) + l);
908
+ __m128i blj0 = load0(B + ldb * (jj + j) + l);
909
+ __m128i blj1 = load1(B + ldb * (jj + j) + l);
910
+
911
+ __m128i sepAA0 = _mm_sign_epi8(ali0, ali0);
912
+ __m128i sepAA1 = _mm_sign_epi8(ali1, ali1);
913
+ __m128i sepBA0 = _mm_sign_epi8(blj0, ali0);
914
+ __m128i sepBA1 = _mm_sign_epi8(blj1, ali1);
915
+
916
+ // updot
917
+ const __m128i oneFill = _mm_set1_epi16(1);
918
+ __m128i mad0 = _mm_maddubs_epi16(sepAA0, sepBA0);
919
+ __m128i mad1 = _mm_maddubs_epi16(sepAA1, sepBA1);
920
+ __m256 udTmp = _mm256_cvtepi32_ps(MM256_SET_M128I(_mm_madd_epi16(oneFill, mad1), _mm_madd_epi16(oneFill, mad0)));
921
+ #endif
922
+ Cv[j][i] = madd(_mm256_set1_ps(unhalf(A[lda * (ii + i) + l].d) *
923
+ unhalf(B[ldb * (jj + j) + l].d)),
924
+ udTmp,
925
+ Cv[j][i]);
926
+ }
927
+ for (int64_t j = 0; j < RN; ++j)
928
+ for (int64_t i = 0; i < RM; ++i)
929
+ C[ldc * (jj + j) + (ii + i)] = hsum(Cv[j][i]);
930
+ }
931
+ }
932
+
933
+ inline __m256i load(const block_q8_0 *b) {
934
+ return _mm256_loadu_si256((const __m256i *)b->qs);
935
+ }
936
+
937
+ inline __m128i load0(const block_q8_0 *b) {
938
+ return _mm_loadu_si128((const __m128i *)b->qs);
939
+ }
940
+
941
+ inline __m128i load1(const block_q8_0 *b) {
942
+ return _mm_loadu_si128(((const __m128i *)b->qs) + 1);
943
+ }
944
+
945
+ inline __m256i load(const block_q4_0 *b) {
946
+ return _mm256_sub_epi8(denibble(b->qs), _mm256_set1_epi8(8));
947
+ }
948
+
949
+ inline __m128i load0(const block_q4_0 *b) {
950
+ const __m128i x = _mm_loadu_si128((const __m128i *)(b->qs));
951
+ return _mm_sub_epi8(_mm_and_si128(_mm_set1_epi8(15), x), _mm_set1_epi8(8));
952
+ }
953
+
954
+ inline __m128i load1(const block_q4_0 *b) {
955
+ const __m128i x = _mm_loadu_si128((const __m128i *)(b->qs));
956
+ return _mm_sub_epi8(_mm_and_si128(_mm_set1_epi8(15), _mm_srli_epi16(x, 4)), _mm_set1_epi8(8));
957
+ }
958
+
959
+ inline __m256i load(const block_q5_0 *b) {
960
+ return _mm256_or_si256(denibble(b->qs), bittobyte(b->qh));
961
+ }
962
+
963
+ inline __m128i load0(const block_q5_0* b) {
964
+ const __m128i x = _mm_loadu_si128((const __m128i *)(b->qs));
965
+ uint32_t x32;
966
+ memcpy(&x32, b->qh, sizeof(uint32_t));
967
+ __m128i qxl = _mm_and_si128(_mm_set1_epi8(15), x);
968
+ __m128i bytesl = _mm_cmpeq_epi8(_mm_set1_epi64x(-1),
969
+ _mm_or_si128(_mm_set1_epi64x(0x7fbfdfeff7fbfdfe),
970
+ _mm_shuffle_epi8(_mm_set1_epi32(x32),
971
+ _mm_set_epi64x(0x0101010101010101, 0x0000000000000000))));
972
+ bytesl = _mm_andnot_si128(bytesl, _mm_set1_epi8((char)0xF0));
973
+ return _mm_or_si128(qxl, bytesl);
974
+ }
975
+
976
+ inline __m128i load1(const block_q5_0* b) {
977
+ const __m128i x = _mm_loadu_si128((const __m128i *)(b->qs));
978
+ uint32_t x32;
979
+ memcpy(&x32, b->qh, sizeof(uint32_t));
980
+ __m128i qxh = _mm_and_si128(_mm_set1_epi8(15), _mm_srli_epi16(x, 4));
981
+ __m128i bytesh = _mm_cmpeq_epi8(_mm_set1_epi64x(-1),
982
+ _mm_or_si128(_mm_set1_epi64x(0x7fbfdfeff7fbfdfe),
983
+ _mm_shuffle_epi8(_mm_set1_epi32(x32),
984
+ _mm_set_epi64x(0x0303030303030303, 0x0202020202020202))));
985
+ bytesh = _mm_andnot_si128(bytesh, _mm_set1_epi8((char)0xF0));
986
+ return _mm_or_si128(qxh, bytesh);
987
+ }
988
+
989
+ inline __m256i load(const block_iq4_nl *b) {
990
+ return MM256_SET_M128I(load1(b), load0(b));
991
+ }
992
+
993
+ inline __m128i load0(const block_iq4_nl *b) {
994
+ const __m128i x = _mm_loadu_si128((const __m128i *)(b->qs));
995
+ return _mm_shuffle_epi8(iq4nlt, _mm_and_si128(_mm_set1_epi8(15), x));
996
+ }
997
+
998
+ inline __m128i load1(const block_iq4_nl *b) {
999
+ const __m128i x = _mm_loadu_si128((const __m128i *)(b->qs));
1000
+ return _mm_shuffle_epi8(iq4nlt, _mm_and_si128(_mm_set1_epi8(15), _mm_srli_epi16(x, 4)));
1001
+ }
1002
+
1003
+ inline __m256 updot(__m256i u, __m256i s) {
1004
+ __m256i res;
1005
+ #if defined(__AVX512VNNI__) && defined(__AVX512VL__)
1006
+ res = _mm256_dpbusd_epi32(_mm256_setzero_si256(), u, s);
1007
+ #elif defined(__AVXVNNI__)
1008
+ res = _mm256_dpbusd_avx_epi32(_mm256_setzero_si256(), u, s);
1009
+ #else
1010
+ res = _mm256_madd_epi16(_mm256_set1_epi16(1), _mm256_maddubs_epi16(u, s));
1011
+ #endif
1012
+ return _mm256_cvtepi32_ps(res);
1013
+ }
1014
+
1015
+ static inline __m256i denibble(const uint8_t *p) {
1016
+ __m128i x = _mm_loadu_si128((const __m128i *)p);
1017
+ return _mm256_and_si256(_mm256_set1_epi8(15),
1018
+ _mm256_insertf128_si256(_mm256_castsi128_si256(x),
1019
+ _mm_srli_epi16(x, 4), 1));
1020
+ }
1021
+
1022
+ static inline __m256i bittobyte(const uint8_t *p) {
1023
+ uint32_t x32;
1024
+ memcpy(&x32, p, sizeof(uint32_t));
1025
+ __m256i bytes = _mm256_cmpeq_epi8(_mm256_set1_epi64x(-1),
1026
+ _mm256_or_si256(_mm256_set1_epi64x(0x7fbfdfeff7fbfdfe),
1027
+ _mm256_shuffle_epi8(_mm256_set1_epi32(x32),
1028
+ _mm256_set_epi64x(0x0303030303030303, 0x0202020202020202,
1029
+ 0x0101010101010101, 0x0000000000000000))));
1030
+ return _mm256_andnot_si256(bytes, _mm256_set1_epi8((char)0xF0));
1031
+ }
1032
+
1033
+ const TA *const A;
1034
+ const TB *const B;
1035
+ TC *const C;
1036
+ const int64_t k;
1037
+ const int64_t lda;
1038
+ const int64_t ldb;
1039
+ const int64_t ldc;
1040
+ const int ith;
1041
+ const int nth;
1042
+ __m128i iq4nlt;
1043
+ };
1044
+ #endif // __AVX__
1045
+
1046
+ //PPC Implementation
1047
+ #if defined(__MMA__)
1048
+
1049
+ #define SAVE_ACC(ACC, ii, jj) \
1050
+ __builtin_mma_disassemble_acc(vec_C, ACC); \
1051
+ for (int I = 0; I < 4; I++) { \
1052
+ for (int J = 0; J < 4; J++) { \
1053
+ *((float*)(C+ii+((jj+J)*ldc)+I)) = *((float*)&vec_C[I]+J); \
1054
+ } \
1055
+ } \
1056
+
1057
+ template <typename TA, typename TB, typename TC>
1058
+ class tinyBLAS_BF16_PPC {
1059
+ public:
1060
+ tinyBLAS_BF16_PPC(int64_t k,
1061
+ const TA *A, int64_t lda,
1062
+ const TB *B, int64_t ldb,
1063
+ TC *C, int64_t ldc,
1064
+ int ith, int nth)
1065
+ : A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc), ith(ith), nth(nth) {
1066
+ }
1067
+
1068
+ void matmul(int64_t m, int64_t n) {
1069
+ mnpack(0, m, 0, n);
1070
+ }
1071
+
1072
+ private:
1073
+ void vector_permute_store(vec_t *c, int numVec, unsigned char *vecOffset) {
1074
+ vec_t t[8], s[8];
1075
+ vec_t swiz1 = {0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23};
1076
+ vec_t swiz2 = {8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31};
1077
+ vec_t swiz3 = {0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23};
1078
+ vec_t swiz4 = {8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31};
1079
+
1080
+ if (numVec == 2) {
1081
+ t[0] = vec_perm(c[0], c[1], swiz1);
1082
+ t[1] = vec_perm(c[2], c[3], swiz1);
1083
+ s[0] = vec_perm(t[0], t[1], swiz3);
1084
+ s[1] = vec_perm(t[0], t[1], swiz4);
1085
+ vec_xst(s[0], 0, (vec_t*)vecOffset);
1086
+ vec_xst(s[1], 0, (vec_t*)(vecOffset + 16));
1087
+ } else if (numVec == 4) {
1088
+ t[0] = vec_perm(c[0], c[1], swiz1);
1089
+ t[1] = vec_perm(c[0], c[1], swiz2);
1090
+ t[2] = vec_perm(c[2], c[3], swiz1);
1091
+ t[3] = vec_perm(c[2], c[3], swiz2);
1092
+ s[0] = vec_perm(t[0], t[2], swiz3);
1093
+ s[1] = vec_perm(t[0], t[2], swiz4);
1094
+ s[2] = vec_perm(t[1], t[3], swiz3);
1095
+ s[3] = vec_perm(t[1], t[3], swiz4);
1096
+ for (int i = 0; i < 4; ++i)
1097
+ vec_xst(s[i], 0, (vec_t*)(vecOffset + i * 16));
1098
+ } else if (numVec == 8) {
1099
+ for (int i = 0; i < 4; i += 2) {
1100
+ t[i+0] = vec_perm(c[i+0], c[i+1], swiz1);
1101
+ t[i+1] = vec_perm(c[i+0], c[i+1], swiz2);
1102
+ }
1103
+ for (int i = 4; i < 8; i += 2) {
1104
+ t[i+0] = vec_perm(c[i+0], c[i+1], swiz1);
1105
+ t[i+1] = vec_perm(c[i+0], c[i+1], swiz2);
1106
+ }
1107
+ s[0] = vec_perm(t[0], t[2], swiz3);
1108
+ s[1] = vec_perm(t[0], t[2], swiz4);
1109
+ s[2] = vec_perm(t[1], t[3], swiz3);
1110
+ s[3] = vec_perm(t[1], t[3], swiz4);
1111
+ s[4] = vec_perm(t[4], t[6], swiz3);
1112
+ s[5] = vec_perm(t[4], t[6], swiz4);
1113
+ s[6] = vec_perm(t[5], t[7], swiz3);
1114
+ s[7] = vec_perm(t[5], t[7], swiz4);
1115
+ for (int i = 0; i < 8; ++i)
1116
+ vec_xst(s[i], 0, (vec_t*)(vecOffset + i * 16));
1117
+ }
1118
+ }
1119
+
1120
+ void packNormal(const TA* a, int64_t lda, int rows, int cols, unsigned char* vec) {
1121
+ int64_t i, j;
1122
+ TA *aoffset = NULL;
1123
+ unsigned char *vecOffset = NULL;
1124
+ TA * aoffsets[8];
1125
+ vector unsigned char c_arr[8];
1126
+ aoffset = const_cast<TA*>(a);
1127
+ vecOffset = vec;
1128
+ j = (rows >> 3);
1129
+ if (j > 0) {
1130
+ do {
1131
+ if (cols == 4) {
1132
+ aoffsets[0] = aoffset;
1133
+ for (int it = 1; it < 4; ++it)
1134
+ aoffsets[it] = aoffsets[it-1] + lda;
1135
+ aoffset += 4 * lda;
1136
+ for (int i = 0; i < 4; ++i)
1137
+ c_arr[i] = vec_xl(0, (vector unsigned char*)aoffsets[i]);
1138
+ vector_permute_store(c_arr, 4, vecOffset);
1139
+ for (int i = 0; i<4; i++)
1140
+ aoffsets[i] = aoffsets[i]+lda;
1141
+ vecOffset +=64;
1142
+ }
1143
+ i = (cols >> 3);
1144
+ if (i > 0) {
1145
+ aoffsets[0] = aoffset;
1146
+ for (int it = 1; it < 8; ++it) {
1147
+ aoffsets[it] = aoffsets[it-1] + lda;
1148
+ }
1149
+ aoffset += 8 * lda;
1150
+ do {
1151
+ for (int it = 0; it < 8; ++it)
1152
+ c_arr[it] = vec_xl(0, (vector unsigned char*)aoffsets[it]);
1153
+ vector_permute_store(c_arr, 8, vecOffset);
1154
+ for (int it = 0; it < 8; ++it)
1155
+ aoffsets[it] = aoffsets[it] + 8*lda;
1156
+ vecOffset += 128;
1157
+ i--;
1158
+ } while(i > 0);
1159
+ }
1160
+ j--;
1161
+ } while(j > 0);
1162
+ }
1163
+ if (rows & 4) {
1164
+ aoffsets[0] = aoffset;
1165
+ for (int it = 1; it < 4; ++it)
1166
+ aoffsets[it] = aoffsets[it-1] + lda;
1167
+ aoffset += 4 * lda;
1168
+ if (cols == 4) {
1169
+ for (int it = 0; it < 4; ++it)
1170
+ c_arr[it] = vec_xl(0, (vector unsigned char*)aoffsets[it]);
1171
+ vector_permute_store(c_arr, 2, vecOffset);
1172
+ for (int it = 0; it< 4; it++)
1173
+ aoffsets[it] = aoffsets[it] + lda;
1174
+ vecOffset += 32;
1175
+ }
1176
+ i = (cols >> 3);
1177
+ if (i > 0) {
1178
+ do {
1179
+ for (int it = 0; it < 4; ++it)
1180
+ c_arr[it] = vec_xl(0, (vector unsigned char*)aoffsets[it]);
1181
+ vector_permute_store(c_arr, 4, vecOffset);
1182
+ for (int it = 0; it< 4; it++)
1183
+ aoffsets[it] = aoffsets[it] + 8*lda;
1184
+ vecOffset += 64;
1185
+ i--;
1186
+ } while(i > 0);
1187
+ }
1188
+ }
1189
+ if (rows & 3) {
1190
+ aoffsets[0] = aoffset;
1191
+ for (int it = 1; it < 4; ++it)
1192
+ aoffsets[it] = aoffsets[it-1] + lda;
1193
+ if (cols == 4) {
1194
+ switch(rows) {
1195
+ case 3: c_arr[2] = vec_xl(0, (vector unsigned char*)aoffsets[2]);
1196
+ case 2: c_arr[1] = vec_xl(0, (vector unsigned char*)aoffsets[1]);
1197
+ case 1: c_arr[0] = vec_xl(0, (vector unsigned char*)aoffsets[0]);
1198
+ break;
1199
+ }
1200
+ vector_permute_store(c_arr, 2, vecOffset);
1201
+ for (int it = 0; it< 4; it++)
1202
+ aoffsets[it] = aoffsets[it] + lda;
1203
+ vecOffset += 32;
1204
+ }
1205
+ i = (cols >> 3);
1206
+ if (i > 0) {
1207
+ do {
1208
+ switch(rows) {
1209
+ case 3: c_arr[2] = vec_xl(0, (vector unsigned char*)aoffsets[2]);
1210
+ case 2: c_arr[1] = vec_xl(0, (vector unsigned char*)aoffsets[1]);
1211
+ case 1: c_arr[0] = vec_xl(0, (vector unsigned char*)aoffsets[0]);
1212
+ break;
1213
+ }
1214
+ vector_permute_store(c_arr, 4, vecOffset);
1215
+ for (int it = 0; it <4; it++)
1216
+ aoffsets[it] = aoffsets[it] + 8* lda;
1217
+ vecOffset += 64;
1218
+ i--;
1219
+ } while(i > 0);
1220
+ }
1221
+ }
1222
+ }
1223
+
1224
+ void mnpack(int64_t m0, int64_t m, int64_t n0, int64_t n) {
1225
+ int64_t mc, nc, mp, np;
1226
+ int m_rem = MIN(m - m0, 8);
1227
+ int n_rem = MIN(n - n0, 8);
1228
+
1229
+ if (m_rem >= 8 && n_rem >= 8) {
1230
+ mc = 8;
1231
+ nc = 8;
1232
+ gemm<8,8>(m0, m, n0, n);
1233
+ } else if (m_rem >= 4 && n_rem >= 8) {
1234
+ mc = 4;
1235
+ nc = 8;
1236
+ gemm<4,8>(m0, m, n0, n);
1237
+ } else if (m_rem >=8 && n_rem >=4){
1238
+ mc = 8;
1239
+ nc = 4;
1240
+ gemm<8,4>(m0, m, n0, n);
1241
+ } else if ((m_rem < 4) && (n_rem >= 8)) {
1242
+ nc = 8;
1243
+ switch(m_rem) {
1244
+ case 1:
1245
+ mc = 1;
1246
+ gemm_Mx8<1>(m0, m, n0, n);
1247
+ break;
1248
+ case 2:
1249
+ mc = 2;
1250
+ gemm_Mx8<2>(m0, m, n0, n);
1251
+ break;
1252
+ case 3:
1253
+ mc = 3;
1254
+ gemm_Mx8<3>(m0, m, n0, n);
1255
+ break;
1256
+ default:
1257
+ return;
1258
+ }
1259
+ } else if (m_rem >= 4 && n_rem >= 4) {
1260
+ mc = 4;
1261
+ nc = 4;
1262
+ gemm_small<4, 4>(m0, m, n0, n);
1263
+ } else if ((m_rem > 4) && (n_rem < 4)) {
1264
+ mc = 4;
1265
+ switch(n_rem) {
1266
+ case 1:
1267
+ nc = 1;
1268
+ gemm_small<4, 1>(m0, m, n0, n);
1269
+ break;
1270
+ case 2:
1271
+ nc = 2;
1272
+ gemm_small<4, 2>(m0, m, n0, n);
1273
+ break;
1274
+ case 3:
1275
+ nc = 3;
1276
+ gemm_small<4, 3>(m0, m, n0, n);
1277
+ break;
1278
+
1279
+ default:
1280
+ return;
1281
+ }
1282
+ } else {
1283
+ switch((m_rem << 4) | n_rem) {
1284
+ case 0x43:
1285
+ mc = 4;
1286
+ nc = 3;
1287
+ gemm_small<4, 3>(m0, m, n0, n);
1288
+ break;
1289
+ case 0x42:
1290
+ mc = 4;
1291
+ nc = 2;
1292
+ gemm_small<4, 2>(m0, m, n0, n);
1293
+ break;
1294
+ case 0x41:
1295
+ mc = 4;
1296
+ nc = 1;
1297
+ gemm_small<4, 1>(m0, m, n0, n);
1298
+ break;
1299
+ case 0x34:
1300
+ mc = 3;
1301
+ nc = 4;
1302
+ gemm_small<3, 4>(m0, m, n0, n);
1303
+ break;
1304
+ case 0x33:
1305
+ mc = 3;
1306
+ nc = 3;
1307
+ gemm_small<3, 3>(m0, m, n0, n);
1308
+ break;
1309
+ case 0x32:
1310
+ mc = 3;
1311
+ nc = 2;
1312
+ gemm_small<3, 2>(m0, m, n0, n);
1313
+ break;
1314
+ case 0x31:
1315
+ mc = 3;
1316
+ nc = 1;
1317
+ gemm_small<3, 1>(m0, m, n0, n);
1318
+ break;
1319
+ case 0x24:
1320
+ mc = 2;
1321
+ nc = 4;
1322
+ gemm_small<2,4>(m0, m, n0, n);
1323
+ break;
1324
+ case 0x23:
1325
+ mc = 2;
1326
+ nc = 3;
1327
+ gemm_small<2, 3>(m0, m, n0, n);
1328
+ break;
1329
+ case 0x22:
1330
+ mc = 2;
1331
+ nc = 2;
1332
+ gemm_small<2, 2>(m0, m, n0, n);
1333
+ break;
1334
+ case 0x21:
1335
+ mc = 2;
1336
+ nc = 1;
1337
+ gemm_small<2, 1>(m0, m, n0, n);
1338
+ break;
1339
+ case 0x14:
1340
+ mc = 1;
1341
+ nc = 4;
1342
+ gemm_small<1, 4>(m0, m, n0, n);
1343
+ break;
1344
+ case 0x13:
1345
+ mc = 1;
1346
+ nc = 3;
1347
+ gemm_small<1, 3>(m0, m, n0, n);
1348
+ break;
1349
+ case 0x12:
1350
+ mc = 1;
1351
+ nc = 2;
1352
+ gemm_small<1, 2>(m0, m, n0, n);
1353
+ break;
1354
+ case 0x11:
1355
+ mc = 1;
1356
+ nc = 1;
1357
+ gemm_small<1, 1>(m0, m, n0, n);
1358
+ break;
1359
+ default:
1360
+ return;
1361
+ }
1362
+ }
1363
+ mp = m0 + (m - m0) / mc * mc;
1364
+ np = n0 + (n - n0) / nc * nc;
1365
+ mnpack(mp, m, n0, np);
1366
+ mnpack(m0, m, np, n);
1367
+ }
1368
+
1369
+ void KERNEL_4x8(int64_t ii, int64_t jj) {
1370
+ vec_t vec_A[4], vec_B[8] , vec_C[4];
1371
+ acc_t acc_0, acc_1;
1372
+ __builtin_mma_xxsetaccz(&acc_0);
1373
+ __builtin_mma_xxsetaccz(&acc_1);
1374
+ for (int l = 0; l < k; l+=8) {
1375
+ packNormal((A+(ii*lda)+l), lda, 4, 8, (uint8_t*)vec_A);
1376
+ packNormal((B+(jj*ldb)+l), ldb, 8, 8, (uint8_t*)vec_B);
1377
+ for (int x = 0; x < 4; x++) {
1378
+ __builtin_mma_xvbf16ger2pp(&acc_0, vec_A[x], vec_B[x]);
1379
+ __builtin_mma_xvbf16ger2pp(&acc_1, vec_A[x], vec_B[x+4]);
1380
+ }
1381
+ }
1382
+ SAVE_ACC(&acc_0, ii, jj);
1383
+ SAVE_ACC(&acc_1, ii, jj+4);
1384
+ }
1385
+
1386
+ void KERNEL_8x4(int64_t ii, int64_t jj) {
1387
+ vec_t vec_A[8], vec_B[4] , vec_C[4];
1388
+ acc_t acc_0, acc_1;
1389
+ __builtin_mma_xxsetaccz(&acc_0);
1390
+ __builtin_mma_xxsetaccz(&acc_1);
1391
+ for (int l = 0; l < k; l+=8) {
1392
+ packNormal((A+(ii*lda)+l), lda, 8, 8, (uint8_t*)vec_A);
1393
+ packNormal((B+(jj*ldb)+l), ldb, 8, 4, (uint8_t*)vec_B);
1394
+ for (int x = 0; x < 4; x++) {
1395
+ __builtin_mma_xvbf16ger2pp(&acc_0, vec_A[x], vec_B[x]);
1396
+ __builtin_mma_xvbf16ger2pp(&acc_1, vec_A[x+4], vec_B[x]);
1397
+ }
1398
+ }
1399
+ SAVE_ACC(&acc_0, ii, jj);
1400
+ SAVE_ACC(&acc_1, ii+4, jj);
1401
+ }
1402
+
1403
+
1404
+ void KERNEL_8x8(int64_t ii, int64_t jj) {
1405
+ vec_t vec_A[8], vec_B[8], vec_C[4];
1406
+ acc_t acc_0, acc_1, acc_2, acc_3;
1407
+ __builtin_mma_xxsetaccz(&acc_0);
1408
+ __builtin_mma_xxsetaccz(&acc_1);
1409
+ __builtin_mma_xxsetaccz(&acc_2);
1410
+ __builtin_mma_xxsetaccz(&acc_3);
1411
+ for (int l = 0; l < k; l+=8) {
1412
+ packNormal(A+(ii*lda)+l, lda, 8, 8, (uint8_t*)vec_A);
1413
+ packNormal(B+(jj*ldb)+l, ldb, 8, 8, (uint8_t*)vec_B);
1414
+ for (int x = 0; x < 4; x++) {
1415
+ __builtin_mma_xvbf16ger2pp(&acc_0, vec_A[x], vec_B[x]);
1416
+ __builtin_mma_xvbf16ger2pp(&acc_1, (vec_t)vec_A[x], (vec_t)vec_B[x+4]);
1417
+ __builtin_mma_xvbf16ger2pp(&acc_2, (vec_t)vec_A[x+4], (vec_t)vec_B[x]);
1418
+ __builtin_mma_xvbf16ger2pp(&acc_3, (vec_t)vec_A[x+4], (vec_t)vec_B[x+4]);
1419
+ }
1420
+ }
1421
+
1422
+ SAVE_ACC(&acc_0, ii, jj);
1423
+ SAVE_ACC(&acc_1, ii, jj+4);
1424
+ SAVE_ACC(&acc_2, ii+4, jj);
1425
+ SAVE_ACC(&acc_3, ii+4, jj+4);
1426
+ }
1427
+
1428
+ template<int RM, int RN>
1429
+ void gemm_small(int64_t m0, int64_t m, int64_t n0, int64_t n) {
1430
+ int64_t ytiles = (m - m0) / RM;
1431
+ int64_t xtiles = (n - n0) / RN;
1432
+ int64_t tiles = xtiles * ytiles;
1433
+ int64_t duty = (tiles + nth - 1) / nth;
1434
+ int64_t start = duty * ith;
1435
+ int64_t end = start + duty;
1436
+ if (end > tiles)
1437
+ end = tiles;
1438
+ for (int64_t job = start; job < end; ++job) {
1439
+ int64_t ii = m0 + job / xtiles * RM;
1440
+ int64_t jj = n0 + job % xtiles * RN;
1441
+ vec_t vec_C[4];
1442
+ acc_t acc_0;
1443
+ __builtin_mma_xxsetaccz(&acc_0);
1444
+ vec_t vec_A[2], vec_B[2];
1445
+ for (int l=0; l<k; l+=4) {
1446
+ packNormal(A+(ii*lda)+l, lda, RM, 4, (uint8_t*)vec_A);
1447
+ packNormal(B+(jj*ldb)+l, ldb, RN, 4, (uint8_t*)vec_B);
1448
+ for (int x = 0; x<2; x++) {
1449
+ __builtin_mma_xvbf16ger2pp(&acc_0, vec_A[x], vec_B[x]);
1450
+ }
1451
+ }
1452
+ __builtin_mma_disassemble_acc(vec_C, &acc_0);
1453
+ for (int I = 0; I < RM; I++) {
1454
+ for (int J = 0; J < RN; J++) {
1455
+ *((TC*)(C+ii+((jj+J)*ldc)+I)) = *((TC*)&vec_C[I]+J);
1456
+ }
1457
+ }
1458
+ }
1459
+ }
1460
+
1461
+ template<int RM>
1462
+ void gemm_Mx8(int64_t m0, int64_t m, int64_t n0, int64_t n) {
1463
+ int RN = 8;
1464
+ int64_t ytiles = (m - m0) / RM;
1465
+ int64_t xtiles = (n - n0) / RN;
1466
+ int64_t tiles = xtiles * ytiles;
1467
+ int64_t duty = (tiles + nth - 1) / nth;
1468
+ int64_t start = duty * ith;
1469
+ int64_t end = start + duty;
1470
+ if (end > tiles)
1471
+ end = tiles;
1472
+ for (int64_t job = start; job < end; ++job) {
1473
+ int64_t ii = m0 + job / xtiles * RM;
1474
+ int64_t jj = n0 + job % xtiles * RN;
1475
+ vec_t vec_C[4];
1476
+ acc_t acc_0, acc_1;
1477
+ __builtin_mma_xxsetaccz(&acc_0);
1478
+ __builtin_mma_xxsetaccz(&acc_1);
1479
+ vec_t vec_A[4], vec_B[8];
1480
+ for (int l=0; l<k; l+=8) {
1481
+ packNormal(A+(ii*lda)+l, lda, RM, 8, (uint8_t*)vec_A);
1482
+ packNormal(B+(jj*ldb)+l, ldb, RN, 8, (uint8_t*)vec_B);
1483
+ for (int x = 0; x<4; x++) {
1484
+ __builtin_mma_xvbf16ger2pp(&acc_0, vec_A[x], vec_B[x]);
1485
+ __builtin_mma_xvbf16ger2pp(&acc_1, vec_A[x], vec_B[x+4]);
1486
+ }
1487
+ }
1488
+ __builtin_mma_disassemble_acc(vec_C, &acc_0);
1489
+ for (int I = 0; I < RM; I++) {
1490
+ for (int J = 0; J < 4; J++) {
1491
+ *((TC*)(C+ii+((jj+J)*ldc)+I)) = *((TC*)&vec_C[I]+J);
1492
+ }
1493
+ }
1494
+ __builtin_mma_disassemble_acc(vec_C, &acc_1);
1495
+ for (int I = 0; I < RM; I++) {
1496
+ for (int J = 0; J < 4; J++) {
1497
+ *((TC*)(C+ii+((jj+4+J)*ldc)+I)) = *((TC*)&vec_C[I]+J);
1498
+ }
1499
+ }
1500
+ }
1501
+ }
1502
+
1503
+ template<int RM, int RN>
1504
+ inline void kernel(int64_t ii, int64_t jj) {
1505
+ if constexpr(RM == 4 && RN == 8) {
1506
+ KERNEL_4x8(ii,jj);
1507
+ } else if constexpr(RM == 8 && RN == 8) {
1508
+ KERNEL_8x8(ii,jj);
1509
+ } else if constexpr(RM == 8 && RN == 4) {
1510
+ KERNEL_8x4(ii,jj);
1511
+ } else {
1512
+ static_assert(false, "RN/RM values not supported");
1513
+ }
1514
+ }
1515
+
1516
+ template <int RM, int RN>
1517
+ NOINLINE void gemm(int64_t m0, int64_t m, int64_t n0, int64_t n) {
1518
+ int64_t ytiles = (m - m0) / RM;
1519
+ int64_t xtiles = (n - n0) / RN;
1520
+ int64_t tiles = xtiles * ytiles;
1521
+ int64_t duty = (tiles + nth - 1) / nth;
1522
+ int64_t start = duty * ith;
1523
+ int64_t end = start + duty;
1524
+ if (end > tiles)
1525
+ end = tiles;
1526
+ for (int64_t job = start; job < end; ++job) {
1527
+ int64_t ii = m0 + job / xtiles * RM;
1528
+ int64_t jj = n0 + job % xtiles * RN;
1529
+ kernel<RM, RN>(ii, jj);
1530
+ }
1531
+ }
1532
+
1533
+ const TA *const A;
1534
+ const TB *const B;
1535
+ TC *C;
1536
+ const int64_t k;
1537
+ const int64_t lda;
1538
+ const int64_t ldb;
1539
+ const int64_t ldc;
1540
+ const int ith;
1541
+ const int nth;
1542
+ };
1543
+
1544
+ template <typename TA, typename TB, typename TC>
1545
+ class tinyBLAS_Q0_PPC {
1546
+ public:
1547
+ tinyBLAS_Q0_PPC(int64_t k,
1548
+ const TA *A, int64_t lda,
1549
+ const TB *B, int64_t ldb,
1550
+ TC *C, int64_t ldc,
1551
+ int ith, int nth)
1552
+ : A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc), ith(ith), nth(nth) {
1553
+ }
1554
+
1555
+ void matmul(int64_t m, int64_t n) {
1556
+ mnpack(0, m, 0, n);
1557
+ }
1558
+
1559
+ private:
1560
+
1561
+ template<int RM, int RN>
1562
+ inline void save_res(int ii, int jj, int idx, vector float* fin_res) {
1563
+ for (int I = 0; I < RM; I++) {
1564
+ for (int J = 0; J < RN; J++) {
1565
+ *((float*)(C+ii+((jj+J)*ldc)+I)) = *((float*)&fin_res[idx+I]+J);
1566
+ }
1567
+ }
1568
+ }
1569
+
1570
+ template<int size>
1571
+ inline void compute(acc_t* ACC, int c_idx, int s_idx, std::array<int, size>& comparray, vector float* vs, vector float* fin_res) {
1572
+ vector signed int vec_C[4];
1573
+ vector float CA[4] = {0};
1574
+ vector float res[4] = {0};
1575
+ __builtin_mma_disassemble_acc(vec_C, ACC);
1576
+ for (int i = 0; i < 4; i++) {
1577
+ CA[i] = vec_splats((float)(((double)comparray[c_idx+i]) * -128.0));
1578
+ res[i] = vec_add(vec_ctf(vec_C[i], 0), CA[i]);
1579
+ fin_res[s_idx+i] = vec_madd(res[i], vs[s_idx+i], fin_res[s_idx+i]);
1580
+ }
1581
+ }
1582
+
1583
+ template<typename VA, typename VB, int size>
1584
+ void packNormalInt4(const TA* a, int64_t lda, int rows, int cols, VA* vec, std::array<int, size>& comparray) {
1585
+ int64_t i, j;
1586
+ TA *aoffset = NULL;
1587
+ VA *vecOffset = NULL;
1588
+ TA *aoffset1 = NULL, *aoffset2 = NULL, *aoffset3 = NULL, *aoffset4 = NULL;
1589
+ TA *aoffset5 = NULL, *aoffset6 = NULL, *aoffset7 = NULL, *aoffset8 = NULL;
1590
+ VB c1[2] = {0}, c2[2] = {0}, c3[2] = {0}, c4[2] = {0};
1591
+ VB c5[2] = {0}, c6[2] = {0}, c7[2] = {0}, c8[2] = {0};
1592
+ VB t1, t2, t3, t4, t5, t6, t7, t8;
1593
+ const vector signed char lowMask = vec_splats((signed char)0xF);
1594
+ const vector unsigned char v4 = vec_splats((unsigned char)0x4);
1595
+ const vector signed char v8 = vec_splats((signed char)0x8);
1596
+ aoffset = const_cast<TA*>(a);
1597
+ vecOffset = vec;
1598
+ vector unsigned char swiz1 = {0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23};
1599
+ vector unsigned char swiz2 = {8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31};
1600
+ vector unsigned char swiz3 = {0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27};
1601
+ vector unsigned char swiz4 = {4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31};
1602
+ vector signed int vsum = {0};
1603
+ vector signed int vsum2 = {0};
1604
+
1605
+ j = (rows >> 3);
1606
+ if (j > 0) {
1607
+ do {
1608
+ aoffset1 = aoffset;
1609
+ aoffset2 = aoffset1 + lda;
1610
+ aoffset3 = aoffset2 + lda;
1611
+ aoffset4 = aoffset3 + lda;
1612
+ aoffset5 = aoffset4 + lda;
1613
+ aoffset6 = aoffset5 + lda;
1614
+ aoffset7 = aoffset6 + lda;
1615
+ aoffset8 = aoffset7 + lda;
1616
+ aoffset += 8 * lda;
1617
+
1618
+ i = (cols >> 2);
1619
+ if (i > 0) {
1620
+ do {
1621
+ c1[1] = reinterpret_cast<VB>(vec_xl(0, aoffset1->qs));
1622
+ c2[1] = reinterpret_cast<VB>(vec_xl(0, aoffset2->qs));
1623
+ c3[1] = reinterpret_cast<VB>(vec_xl(0, aoffset3->qs));
1624
+ c4[1] = reinterpret_cast<VB>(vec_xl(0, aoffset4->qs));
1625
+ c5[1] = reinterpret_cast<VB>(vec_xl(0, aoffset5->qs));
1626
+ c6[1] = reinterpret_cast<VB>(vec_xl(0, aoffset6->qs));
1627
+ c7[1] = reinterpret_cast<VB>(vec_xl(0, aoffset7->qs));
1628
+ c8[1] = reinterpret_cast<VB>(vec_xl(0, aoffset8->qs));
1629
+
1630
+ c1[0] = vec_and(c1[1], lowMask);
1631
+ c1[1] = vec_sr(c1[1], v4);
1632
+ c1[0] = vec_sub(c1[0], v8);
1633
+ c1[1] = vec_sub(c1[1], v8);
1634
+ vsum = vec_sum4s(c1[0], vsum);
1635
+ vsum2 = vec_sum4s(c1[1], vsum2);
1636
+ vsum = vec_add(vsum, vsum2);
1637
+ comparray[0] = vsum[0] + vsum[1] + vsum[2] + vsum[3];
1638
+ vsum = vec_splats(0);
1639
+ vsum2 = vec_splats(0);
1640
+
1641
+ c2[0] = vec_and(c2[1], lowMask);
1642
+ c2[1] = vec_sr(c2[1], v4);
1643
+ c2[0] = vec_sub(c2[0], v8);
1644
+ c2[1] = vec_sub(c2[1], v8);
1645
+ vsum = vec_sum4s(c2[0], vsum);
1646
+ vsum2 = vec_sum4s(c2[1], vsum2);
1647
+ vsum = vec_add(vsum, vsum2);
1648
+ comparray[1] = vsum[0] + vsum[1] + vsum[2] + vsum[3];
1649
+ vsum = vec_splats(0);
1650
+ vsum2 = vec_splats(0);
1651
+
1652
+ c3[0] = vec_and(c3[1], lowMask);
1653
+ c3[1] = vec_sr(c3[1], v4);
1654
+ c3[0] = vec_sub(c3[0], v8);
1655
+ c3[1] = vec_sub(c3[1], v8);
1656
+ vsum = vec_sum4s(c3[0], vsum);
1657
+ vsum2 = vec_sum4s(c3[1], vsum2);
1658
+ vsum = vec_add(vsum, vsum2);
1659
+ comparray[2] = vsum[0] + vsum[1] + vsum[2] + vsum[3];
1660
+ vsum = vec_splats(0);
1661
+ vsum2 = vec_splats(0);
1662
+
1663
+ c4[0] = vec_and(c4[1], lowMask);
1664
+ c4[1] = vec_sr(c4[1], v4);
1665
+ c4[0] = vec_sub(c4[0], v8);
1666
+ c4[1] = vec_sub(c4[1], v8);
1667
+ vsum = vec_sum4s(c4[0], vsum);
1668
+ vsum2 = vec_sum4s(c4[1], vsum2);
1669
+ vsum = vec_add(vsum, vsum2);
1670
+ comparray[3] = vsum[0] + vsum[1] + vsum[2] + vsum[3];
1671
+ vsum = vec_splats(0);
1672
+ vsum2 = vec_splats(0);
1673
+
1674
+ c5[0] = vec_and(c5[1], lowMask);
1675
+ c5[1] = vec_sr(c5[1], v4);
1676
+ c5[0] = vec_sub(c5[0], v8);
1677
+ c5[1] = vec_sub(c5[1], v8);
1678
+ vsum = vec_sum4s(c5[0], vsum);
1679
+ vsum2 = vec_sum4s(c5[1], vsum2);
1680
+ vsum = vec_add(vsum, vsum2);
1681
+ comparray[4] = vsum[0] + vsum[1] + vsum[2] + vsum[3];
1682
+ vsum = vec_splats(0);
1683
+ vsum2 = vec_splats(0);
1684
+
1685
+ c6[0] = vec_and(c6[1], lowMask);
1686
+ c6[1] = vec_sr(c6[1], v4);
1687
+ c6[0] = vec_sub(c6[0], v8);
1688
+ c6[1] = vec_sub(c6[1], v8);
1689
+ vsum = vec_sum4s(c6[0], vsum);
1690
+ vsum2 = vec_sum4s(c6[1], vsum2);
1691
+ vsum = vec_add(vsum, vsum2);
1692
+ comparray[5] = vsum[0] + vsum[1] + vsum[2] + vsum[3];
1693
+ vsum = vec_splats(0);
1694
+ vsum2 = vec_splats(0);
1695
+
1696
+ c7[0] = vec_and(c7[1], lowMask);
1697
+ c7[1] = vec_sr(c7[1], v4);
1698
+ c7[0] = vec_sub(c7[0], v8);
1699
+ c7[1] = vec_sub(c7[1], v8);
1700
+ vsum = vec_sum4s(c7[0], vsum);
1701
+ vsum2 = vec_sum4s(c7[1], vsum2);
1702
+ vsum = vec_add(vsum, vsum2);
1703
+ comparray[6] = vsum[0] + vsum[1] + vsum[2] + vsum[3];
1704
+ vsum = vec_splats(0);
1705
+ vsum2 = vec_splats(0);
1706
+
1707
+ c8[0] = vec_and(c8[1], lowMask);
1708
+ c8[1] = vec_sr(c8[1], v4);
1709
+ c8[0] = vec_sub(c8[0], v8);
1710
+ c8[1] = vec_sub(c8[1], v8);
1711
+ vsum = vec_sum4s(c8[0], vsum);
1712
+ vsum2 = vec_sum4s(c8[1], vsum2);
1713
+ vsum = vec_add(vsum, vsum2);
1714
+ comparray[7] = vsum[0] + vsum[1] + vsum[2] + vsum[3];
1715
+ vsum = vec_splats(0);
1716
+ vsum2 = vec_splats(0);
1717
+
1718
+ t1 = vec_perm(c1[0], c2[0], swiz1);
1719
+ t2 = vec_perm(c1[0], c2[0], swiz2);
1720
+ t3 = vec_perm(c3[0], c4[0], swiz1);
1721
+ t4 = vec_perm(c3[0], c4[0], swiz2);
1722
+ t5 = vec_perm(t1, t3, swiz3);
1723
+ t6 = vec_perm(t1, t3, swiz4);
1724
+ t7 = vec_perm(t2, t4, swiz3);
1725
+ t8 = vec_perm(t2, t4, swiz4);
1726
+ vec_xst(t5, 0, vecOffset);
1727
+ vec_xst(t6, 0, vecOffset+16);
1728
+ vec_xst(t7, 0, vecOffset+32);
1729
+ vec_xst(t8, 0, vecOffset+48);
1730
+
1731
+ t1 = vec_perm(c1[1], c2[1], swiz1);
1732
+ t2 = vec_perm(c1[1], c2[1], swiz2);
1733
+ t3 = vec_perm(c3[1], c4[1], swiz1);
1734
+ t4 = vec_perm(c3[1], c4[1], swiz2);
1735
+ t5 = vec_perm(t1, t3, swiz3);
1736
+ t6 = vec_perm(t1, t3, swiz4);
1737
+ t7 = vec_perm(t2, t4, swiz3);
1738
+ t8 = vec_perm(t2, t4, swiz4);
1739
+ vec_xst(t5, 0, vecOffset+64);
1740
+ vec_xst(t6, 0, vecOffset+80);
1741
+ vec_xst(t7, 0, vecOffset+96);
1742
+ vec_xst(t8, 0, vecOffset+112);
1743
+
1744
+ t1 = vec_perm(c5[0], c6[0], swiz1);
1745
+ t2 = vec_perm(c5[0], c6[0], swiz2);
1746
+ t3 = vec_perm(c7[0], c8[0], swiz1);
1747
+ t4 = vec_perm(c7[0], c8[0], swiz2);
1748
+ t5 = vec_perm(t1, t3, swiz3);
1749
+ t6 = vec_perm(t1, t3, swiz4);
1750
+ t7 = vec_perm(t2, t4, swiz3);
1751
+ t8 = vec_perm(t2, t4, swiz4);
1752
+ vec_xst(t5, 0, vecOffset+128);
1753
+ vec_xst(t6, 0, vecOffset+144);
1754
+ vec_xst(t7, 0, vecOffset+160);
1755
+ vec_xst(t8, 0, vecOffset+176);
1756
+
1757
+ t1 = vec_perm(c5[1], c6[1], swiz1);
1758
+ t2 = vec_perm(c5[1], c6[1], swiz2);
1759
+ t3 = vec_perm(c7[1], c8[1], swiz1);
1760
+ t4 = vec_perm(c7[1], c8[1], swiz2);
1761
+ t5 = vec_perm(t1, t3, swiz3);
1762
+ t6 = vec_perm(t1, t3, swiz4);
1763
+ t7 = vec_perm(t2, t4, swiz3);
1764
+ t8 = vec_perm(t2, t4, swiz4);
1765
+ vec_xst(t5, 0, vecOffset+192);
1766
+ vec_xst(t6, 0, vecOffset+208);
1767
+ vec_xst(t7, 0, vecOffset+224);
1768
+ vec_xst(t8, 0, vecOffset+240);
1769
+
1770
+ aoffset1 += lda;
1771
+ aoffset2 += lda;
1772
+ aoffset3 += lda;
1773
+ aoffset4 += lda;
1774
+ aoffset5 += lda;
1775
+ aoffset6 += lda;
1776
+ aoffset7 += lda;
1777
+ aoffset8 += lda;
1778
+ vecOffset += 256;
1779
+ i--;
1780
+ } while (i > 0);
1781
+ }
1782
+ j--;
1783
+ } while (j > 0);
1784
+ }
1785
+
1786
+ if (rows & 4) {
1787
+ aoffset1 = aoffset;
1788
+ aoffset2 = aoffset1 + lda;
1789
+ aoffset3 = aoffset2 + lda;
1790
+ aoffset4 = aoffset3 + lda;
1791
+ aoffset += 4 * lda;
1792
+
1793
+ i = (cols >> 2);
1794
+ if (i > 0) {
1795
+ do {
1796
+ c1[1] = reinterpret_cast<VB>(vec_xl(0, aoffset1->qs));
1797
+ c2[1] = reinterpret_cast<VB>(vec_xl(0, aoffset2->qs));
1798
+ c3[1] = reinterpret_cast<VB>(vec_xl(0, aoffset3->qs));
1799
+ c4[1] = reinterpret_cast<VB>(vec_xl(0, aoffset4->qs));
1800
+
1801
+ c1[0] = vec_and(c1[1], lowMask);
1802
+ c1[1] = vec_sr(c1[1], v4);
1803
+ c1[0] = vec_sub(c1[0], v8);
1804
+ c1[1] = vec_sub(c1[1], v8);
1805
+ vsum = vec_sum4s(c1[0], vsum);
1806
+ vsum2 = vec_sum4s(c1[1], vsum2);
1807
+ vsum = vec_add(vsum, vsum2);
1808
+ comparray[0] = vsum[0] + vsum[1] + vsum[2] + vsum[3];
1809
+ vsum = vec_splats(0);
1810
+ vsum2 = vec_splats(0);
1811
+
1812
+ c2[0] = vec_and(c2[1], lowMask);
1813
+ c2[1] = vec_sr(c2[1], v4);
1814
+ c2[0] = vec_sub(c2[0], v8);
1815
+ c2[1] = vec_sub(c2[1], v8);
1816
+ vsum = vec_sum4s(c2[0], vsum);
1817
+ vsum2 = vec_sum4s(c2[1], vsum2);
1818
+ vsum = vec_add(vsum, vsum2);
1819
+ comparray[1] = vsum[0] + vsum[1] + vsum[2] + vsum[3];
1820
+ vsum = vec_splats(0);
1821
+ vsum2 = vec_splats(0);
1822
+
1823
+ c3[0] = vec_and(c3[1], lowMask);
1824
+ c3[1] = vec_sr(c3[1], v4);
1825
+ c3[0] = vec_sub(c3[0], v8);
1826
+ c3[1] = vec_sub(c3[1], v8);
1827
+ vsum = vec_sum4s(c3[0], vsum);
1828
+ vsum2 = vec_sum4s(c3[1], vsum2);
1829
+ vsum = vec_add(vsum, vsum2);
1830
+ comparray[2] = vsum[0] + vsum[1] + vsum[2] + vsum[3];
1831
+ vsum = vec_splats(0);
1832
+ vsum2 = vec_splats(0);
1833
+
1834
+ c4[0] = vec_and(c4[1], lowMask);
1835
+ c4[1] = vec_sr(c4[1], v4);
1836
+ c4[0] = vec_sub(c4[0], v8);
1837
+ c4[1] = vec_sub(c4[1], v8);
1838
+ vsum = vec_sum4s(c4[0], vsum);
1839
+ vsum2 = vec_sum4s(c4[1], vsum2);
1840
+ vsum = vec_add(vsum, vsum2);
1841
+ comparray[3] = vsum[0] + vsum[1] + vsum[2] + vsum[3];
1842
+ vsum = vec_splats(0);
1843
+ vsum2 = vec_splats( 0);
1844
+
1845
+ t1 = vec_perm(c1[0], c2[0], swiz1);
1846
+ t2 = vec_perm(c1[0], c2[0], swiz2);
1847
+ t3 = vec_perm(c3[0], c4[0], swiz1);
1848
+ t4 = vec_perm(c3[0], c4[0], swiz2);
1849
+ t5 = vec_perm(t1, t3, swiz3);
1850
+ t6 = vec_perm(t1, t3, swiz4);
1851
+ t7 = vec_perm(t2, t4, swiz3);
1852
+ t8 = vec_perm(t2, t4, swiz4);
1853
+ vec_xst(t5, 0, vecOffset);
1854
+ vec_xst(t6, 0, vecOffset+16);
1855
+ vec_xst(t7, 0, vecOffset+32);
1856
+ vec_xst(t8, 0, vecOffset+48);
1857
+
1858
+ t1 = vec_perm(c1[1], c2[1], swiz1);
1859
+ t2 = vec_perm(c1[1], c2[1], swiz2);
1860
+ t3 = vec_perm(c3[1], c4[1], swiz1);
1861
+ t4 = vec_perm(c3[1], c4[1], swiz2);
1862
+ t5 = vec_perm(t1, t3, swiz3);
1863
+ t6 = vec_perm(t1, t3, swiz4);
1864
+ t7 = vec_perm(t2, t4, swiz3);
1865
+ t8 = vec_perm(t2, t4, swiz4);
1866
+ vec_xst(t5, 0, vecOffset+64);
1867
+ vec_xst(t6, 0, vecOffset+80);
1868
+ vec_xst(t7, 0, vecOffset+96);
1869
+ vec_xst(t8, 0, vecOffset+112);
1870
+
1871
+ aoffset1 += lda;
1872
+ aoffset2 += lda;
1873
+ aoffset3 += lda;
1874
+ aoffset4 += lda;
1875
+ vecOffset += 128;
1876
+ i--;
1877
+ } while (i > 0);
1878
+ }
1879
+ }
1880
+
1881
+ if (rows & 3) {
1882
+ aoffset1 = aoffset;
1883
+ aoffset2 = aoffset1 + lda;
1884
+ aoffset3 = aoffset2 + lda;
1885
+ i = (cols >> 2);
1886
+ if (i > 0) {
1887
+ do {
1888
+ switch(rows) {
1889
+ case 3: c3[1] = reinterpret_cast<VB>(vec_xl(0, aoffset3->qs));
1890
+ case 2: c2[1] = reinterpret_cast<VB>(vec_xl(0, aoffset2->qs));
1891
+ case 1: c1[1] = reinterpret_cast<VB>(vec_xl(0, aoffset1->qs));
1892
+ break;
1893
+ }
1894
+ c1[0] = vec_and(c1[1], lowMask);
1895
+ c1[1] = vec_sr(c1[1], v4);
1896
+ c1[0] = vec_sub(c1[0], v8);
1897
+ c1[1] = vec_sub(c1[1], v8);
1898
+ vsum = vec_sum4s(c1[0], vsum);
1899
+ vsum2 = vec_sum4s(c1[1], vsum2);
1900
+ vsum = vec_add(vsum, vsum2);
1901
+ comparray[0] = vsum[0] + vsum[1] + vsum[2] + vsum[3];
1902
+ vsum = vec_splats(0);
1903
+ vsum2 = vec_splats(0);
1904
+
1905
+ c2[0] = vec_and(c2[1], lowMask);
1906
+ c2[1] = vec_sr(c2[1], v4);
1907
+ c2[0] = vec_sub(c2[0], v8);
1908
+ c2[1] = vec_sub(c2[1], v8);
1909
+ vsum = vec_sum4s(c2[0], vsum);
1910
+ vsum2 = vec_sum4s(c2[1], vsum2);
1911
+ vsum = vec_add(vsum, vsum2);
1912
+ comparray[1] = vsum[0] + vsum[1] + vsum[2] + vsum[3];
1913
+ vsum = vec_splats(0);
1914
+ vsum2 = vec_splats(0);
1915
+
1916
+ c3[0] = vec_and(c3[1], lowMask);
1917
+ c3[1] = vec_sr(c3[1], v4);
1918
+ c3[0] = vec_sub(c3[0], v8);
1919
+ c3[1] = vec_sub(c3[1], v8);
1920
+ vsum = vec_sum4s(c3[0], vsum);
1921
+ vsum2 = vec_sum4s(c3[1], vsum2);
1922
+ vsum = vec_add(vsum, vsum2);
1923
+ comparray[2] = vsum[0] + vsum[1] + vsum[2] + vsum[3];
1924
+ vsum = vec_splats(0);
1925
+ vsum2 = vec_splats(0);
1926
+
1927
+ c4[0] = vec_and(c4[1], lowMask);
1928
+ c4[1] = vec_sr(c4[1], v4);
1929
+ c4[0] = vec_sub(c4[0], v8);
1930
+ c4[1] = vec_sub(c4[1], v8);
1931
+ vsum = vec_sum4s(c4[0], vsum);
1932
+ vsum2 = vec_sum4s(c4[1], vsum2);
1933
+ vsum = vec_add(vsum, vsum2);
1934
+ comparray[3] = vsum[0] + vsum[1] + vsum[2] + vsum[3];
1935
+ vsum = vec_splats(0);
1936
+ vsum2 = vec_splats(0);
1937
+
1938
+ t1 = vec_perm(c1[0], c2[0], swiz1);
1939
+ t2 = vec_perm(c1[0], c2[0], swiz2);
1940
+ t3 = vec_perm(c3[0], c4[0], swiz1);
1941
+ t4 = vec_perm(c3[0], c4[0], swiz2);
1942
+ t5 = vec_perm(t1, t3, swiz3);
1943
+ t6 = vec_perm(t1, t3, swiz4);
1944
+ t7 = vec_perm(t2, t4, swiz3);
1945
+ t8 = vec_perm(t2, t4, swiz4);
1946
+ vec_xst(t5, 0, vecOffset);
1947
+ vec_xst(t6, 0, vecOffset+16);
1948
+ vec_xst(t7, 0, vecOffset+32);
1949
+ vec_xst(t8, 0, vecOffset+48);
1950
+
1951
+ t1 = vec_perm(c1[1], c2[1], swiz1);
1952
+ t2 = vec_perm(c1[1], c2[1], swiz2);
1953
+ t3 = vec_perm(c3[1], c4[1], swiz1);
1954
+ t4 = vec_perm(c3[1], c4[1], swiz2);
1955
+ t5 = vec_perm(t1, t3, swiz3);
1956
+ t6 = vec_perm(t1, t3, swiz4);
1957
+ t7 = vec_perm(t2, t4, swiz3);
1958
+ t8 = vec_perm(t2, t4, swiz4);
1959
+ vec_xst(t5, 0, vecOffset+64);
1960
+ vec_xst(t6, 0, vecOffset+80);
1961
+ vec_xst(t7, 0, vecOffset+96);
1962
+ vec_xst(t8, 0, vecOffset+112);
1963
+ aoffset1 += lda;
1964
+ aoffset2 += lda;
1965
+ aoffset3 += lda;
1966
+ vecOffset += 128;
1967
+ i--;
1968
+ } while(i > 0);
1969
+ }
1970
+ }
1971
+ }
1972
+
1973
+ template<typename VA, typename VB>
1974
+ void packNormal(const TB* a, int64_t lda, int rows, int cols, VA* vec, bool flip) {
1975
+ int64_t i, j;
1976
+ TB *aoffset = NULL;
1977
+ VA *vecOffset = NULL;
1978
+ TB *aoffset1 = NULL, *aoffset2 = NULL, *aoffset3 = NULL, *aoffset4 = NULL;
1979
+ TB *aoffset5 = NULL, *aoffset6 = NULL, *aoffset7 = NULL, *aoffset8 = NULL;
1980
+ __vector_pair C1, C2, C3, C4, C5, C6, C7, C8;
1981
+ VB c1[2] = {0}, c2[2] = {0}, c3[2] = {0}, c4[2]={0};
1982
+ VB c5[2] = {0}, c6[2] = {0}, c7[2] = {0}, c8[2]={0};
1983
+ VB t1, t2, t3, t4, t5, t6, t7, t8;
1984
+ vector unsigned char xor_vector;
1985
+ uint8_t flip_vec = 0x80;
1986
+ xor_vector = vec_splats(flip_vec);
1987
+ vector unsigned char swiz1 = {0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23};
1988
+ vector unsigned char swiz2 = {8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31};
1989
+ vector unsigned char swiz3 = {0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27};
1990
+ vector unsigned char swiz4 = {4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31};
1991
+
1992
+ aoffset = const_cast<TB*>(a);
1993
+ vecOffset = vec;
1994
+ j = (rows >> 3);
1995
+ if (j > 0) {
1996
+ do {
1997
+ aoffset1 = aoffset;
1998
+ aoffset2 = aoffset1 + lda;
1999
+ aoffset3 = aoffset2 + lda;
2000
+ aoffset4 = aoffset3 + lda;
2001
+ aoffset5 = aoffset4 + lda;
2002
+ aoffset6 = aoffset5 + lda;
2003
+ aoffset7 = aoffset6 + lda;
2004
+ aoffset8 = aoffset7 + lda;
2005
+ aoffset += 8 * lda;
2006
+
2007
+ i = (cols >> 3);
2008
+ if (i > 0) {
2009
+ do {
2010
+ C1 = __builtin_vsx_lxvp(0, (__vector_pair*)aoffset1->qs);
2011
+ C2 = __builtin_vsx_lxvp(0, (__vector_pair*)aoffset2->qs);
2012
+ C3 = __builtin_vsx_lxvp(0, (__vector_pair*)aoffset3->qs);
2013
+ C4 = __builtin_vsx_lxvp(0, (__vector_pair*)aoffset4->qs);
2014
+ C5 = __builtin_vsx_lxvp(0, (__vector_pair*)aoffset5->qs);
2015
+ C6 = __builtin_vsx_lxvp(0, (__vector_pair*)aoffset6->qs);
2016
+ C7 = __builtin_vsx_lxvp(0, (__vector_pair*)aoffset7->qs);
2017
+ C8 = __builtin_vsx_lxvp(0, (__vector_pair*)aoffset8->qs);
2018
+
2019
+ __builtin_vsx_disassemble_pair(c1, &C1);
2020
+ __builtin_vsx_disassemble_pair(c2, &C2);
2021
+ __builtin_vsx_disassemble_pair(c3, &C3);
2022
+ __builtin_vsx_disassemble_pair(c4, &C4);
2023
+ __builtin_vsx_disassemble_pair(c5, &C5);
2024
+ __builtin_vsx_disassemble_pair(c6, &C6);
2025
+ __builtin_vsx_disassemble_pair(c7, &C7);
2026
+ __builtin_vsx_disassemble_pair(c8, &C8);
2027
+
2028
+ t1 = vec_perm(c1[0], c2[0], swiz1);
2029
+ t2 = vec_perm(c1[0], c2[0], swiz2);
2030
+ t3 = vec_perm(c3[0], c4[0], swiz1);
2031
+ t4 = vec_perm(c3[0], c4[0], swiz2);
2032
+ t5 = vec_perm(t1, t3, swiz3);
2033
+ t6 = vec_perm(t1, t3, swiz4);
2034
+ t7 = vec_perm(t2, t4, swiz3);
2035
+ t8 = vec_perm(t2, t4, swiz4);
2036
+ if (flip == true) {
2037
+ t5 = vec_xor(t5, xor_vector);
2038
+ t6 = vec_xor(t6, xor_vector);
2039
+ t7 = vec_xor(t7, xor_vector);
2040
+ t8 = vec_xor(t8, xor_vector);
2041
+ }
2042
+ vec_xst(t5, 0, vecOffset);
2043
+ vec_xst(t6, 0, vecOffset+16);
2044
+ vec_xst(t7, 0, vecOffset+32);
2045
+ vec_xst(t8, 0, vecOffset+48);
2046
+
2047
+ t1 = vec_perm(c1[1], c2[1], swiz1);
2048
+ t2 = vec_perm(c1[1], c2[1], swiz2);
2049
+ t3 = vec_perm(c3[1], c4[1], swiz1);
2050
+ t4 = vec_perm(c3[1], c4[1], swiz2);
2051
+ t5 = vec_perm(t1, t3, swiz3);
2052
+ t6 = vec_perm(t1, t3, swiz4);
2053
+ t7 = vec_perm(t2, t4, swiz3);
2054
+ t8 = vec_perm(t2, t4, swiz4);
2055
+ if (flip == true) {
2056
+ t5 = vec_xor(t5, xor_vector);
2057
+ t6 = vec_xor(t6, xor_vector);
2058
+ t7 = vec_xor(t7, xor_vector);
2059
+ t8 = vec_xor(t8, xor_vector);
2060
+ }
2061
+ vec_xst(t5, 0, vecOffset+64);
2062
+ vec_xst(t6, 0, vecOffset+80);
2063
+ vec_xst(t7, 0, vecOffset+96);
2064
+ vec_xst(t8, 0, vecOffset+112);
2065
+
2066
+ t1 = vec_perm(c5[0], c6[0], swiz1);
2067
+ t2 = vec_perm(c5[0], c6[0], swiz2);
2068
+ t3 = vec_perm(c7[0], c8[0], swiz1);
2069
+ t4 = vec_perm(c7[0], c8[0], swiz2);
2070
+ t5 = vec_perm(t1, t3, swiz3);
2071
+ t6 = vec_perm(t1, t3, swiz4);
2072
+ t7 = vec_perm(t2, t4, swiz3);
2073
+ t8 = vec_perm(t2, t4, swiz4);
2074
+ if (flip == true) {
2075
+ t5 = vec_xor(t5, xor_vector);
2076
+ t6 = vec_xor(t6, xor_vector);
2077
+ t7 = vec_xor(t7, xor_vector);
2078
+ t8 = vec_xor(t8, xor_vector);
2079
+ }
2080
+ vec_xst(t5, 0, vecOffset+128);
2081
+ vec_xst(t6, 0, vecOffset+144);
2082
+ vec_xst(t7, 0, vecOffset+160);
2083
+ vec_xst(t8, 0, vecOffset+176);
2084
+
2085
+ t1 = vec_perm(c5[1], c6[1], swiz1);
2086
+ t2 = vec_perm(c5[1], c6[1], swiz2);
2087
+ t3 = vec_perm(c7[1], c8[1], swiz1);
2088
+ t4 = vec_perm(c7[1], c8[1], swiz2);
2089
+ t5 = vec_perm(t1, t3, swiz3);
2090
+ t6 = vec_perm(t1, t3, swiz4);
2091
+ t7 = vec_perm(t2, t4, swiz3);
2092
+ t8 = vec_perm(t2, t4, swiz4);
2093
+ if (flip == true) {
2094
+ t5 = vec_xor(t5, xor_vector);
2095
+ t6 = vec_xor(t6, xor_vector);
2096
+ t7 = vec_xor(t7, xor_vector);
2097
+ t8 = vec_xor(t8, xor_vector);
2098
+ }
2099
+ vec_xst(t5, 0, vecOffset+192);
2100
+ vec_xst(t6, 0, vecOffset+208);
2101
+ vec_xst(t7, 0, vecOffset+224);
2102
+ vec_xst(t8, 0, vecOffset+240);
2103
+
2104
+ aoffset1 += lda;
2105
+ aoffset2 += lda;
2106
+ aoffset3 += lda;
2107
+ aoffset4 += lda;
2108
+ aoffset5 += lda;
2109
+ aoffset6 += lda;
2110
+ aoffset7 += lda;
2111
+ aoffset8 += lda;
2112
+ vecOffset += 256;
2113
+ i--;
2114
+ } while(i > 0);
2115
+ }
2116
+ j--;
2117
+ } while(j > 0);
2118
+ }
2119
+
2120
+ if (rows & 4) {
2121
+ aoffset1 = aoffset;
2122
+ aoffset2 = aoffset1 + lda;
2123
+ aoffset3 = aoffset2 + lda;
2124
+ aoffset4 = aoffset3 + lda;
2125
+ aoffset += 4 * lda;
2126
+
2127
+ i = (cols >> 3);
2128
+ if (i > 0) {
2129
+ do {
2130
+ C1 = __builtin_vsx_lxvp(0, (__vector_pair*)aoffset1->qs);
2131
+ C2 = __builtin_vsx_lxvp(0, (__vector_pair*)aoffset2->qs);
2132
+ C3 = __builtin_vsx_lxvp(0, (__vector_pair*)aoffset3->qs);
2133
+ C4 = __builtin_vsx_lxvp(0, (__vector_pair*)aoffset4->qs);
2134
+
2135
+ __builtin_vsx_disassemble_pair(c1, &C1);
2136
+ __builtin_vsx_disassemble_pair(c2, &C2);
2137
+ __builtin_vsx_disassemble_pair(c3, &C3);
2138
+ __builtin_vsx_disassemble_pair(c4, &C4);
2139
+
2140
+ t1 = vec_perm(c1[0], c2[0], swiz1);
2141
+ t2 = vec_perm(c1[0], c2[0], swiz2);
2142
+ t3 = vec_perm(c3[0], c4[0], swiz1);
2143
+ t4 = vec_perm(c3[0], c4[0], swiz2);
2144
+ t5 = vec_perm(t1, t3, swiz3);
2145
+ t6 = vec_perm(t1, t3, swiz4);
2146
+ t7 = vec_perm(t2, t4, swiz3);
2147
+ t8 = vec_perm(t2, t4, swiz4);
2148
+ if (flip == true) {
2149
+ t5 = vec_xor(t5, xor_vector);
2150
+ t6 = vec_xor(t6, xor_vector);
2151
+ t7 = vec_xor(t7, xor_vector);
2152
+ t8 = vec_xor(t8, xor_vector);
2153
+ }
2154
+ vec_xst(t5, 0, vecOffset);
2155
+ vec_xst(t6, 0, vecOffset+16);
2156
+ vec_xst(t7, 0, vecOffset+32);
2157
+ vec_xst(t8, 0, vecOffset+48);
2158
+
2159
+ t1 = vec_perm(c1[1], c2[1], swiz1);
2160
+ t2 = vec_perm(c1[1], c2[1], swiz2);
2161
+ t3 = vec_perm(c3[1], c4[1], swiz1);
2162
+ t4 = vec_perm(c3[1], c4[1], swiz2);
2163
+ t5 = vec_perm(t1, t3, swiz3);
2164
+ t6 = vec_perm(t1, t3, swiz4);
2165
+ t7 = vec_perm(t2, t4, swiz3);
2166
+ t8 = vec_perm(t2, t4, swiz4);
2167
+ if (flip == true) {
2168
+ t5 = vec_xor(t5, xor_vector);
2169
+ t6 = vec_xor(t6, xor_vector);
2170
+ t7 = vec_xor(t7, xor_vector);
2171
+ t8 = vec_xor(t8, xor_vector);
2172
+ }
2173
+ vec_xst(t5, 0, vecOffset+64);
2174
+ vec_xst(t6, 0, vecOffset+80);
2175
+ vec_xst(t7, 0, vecOffset+96);
2176
+ vec_xst(t8, 0, vecOffset+112);
2177
+
2178
+ aoffset1 += lda;
2179
+ aoffset2 += lda;
2180
+ aoffset3 += lda;
2181
+ aoffset4 += lda;
2182
+ vecOffset += 128;
2183
+ i--;
2184
+ } while(i > 0);
2185
+ }
2186
+ }
2187
+ if (rows & 3) {
2188
+ aoffset1 = aoffset;
2189
+ aoffset2 = aoffset1 + lda;
2190
+ aoffset3 = aoffset2 + lda;
2191
+ i = (cols >> 3);
2192
+ if (i > 0) {
2193
+ do {
2194
+ switch(rows) {
2195
+ case 3: C3 = __builtin_vsx_lxvp(0, (__vector_pair*)aoffset3->qs);
2196
+ __builtin_vsx_disassemble_pair(c3, &C3);
2197
+ case 2: C2 = __builtin_vsx_lxvp(0, (__vector_pair*)aoffset2->qs);
2198
+ __builtin_vsx_disassemble_pair(c2, &C2);
2199
+ case 1: C1 = __builtin_vsx_lxvp(0, (__vector_pair*)aoffset1->qs);
2200
+ __builtin_vsx_disassemble_pair(c1, &C1);
2201
+ break;
2202
+ }
2203
+ t1 = vec_perm(c1[0], c2[0], swiz1);
2204
+ t2 = vec_perm(c1[0], c2[0], swiz2);
2205
+ t3 = vec_perm(c3[0], c4[0], swiz1);
2206
+ t4 = vec_perm(c3[0], c4[0], swiz2);
2207
+ t5 = vec_perm(t1, t3, swiz3);
2208
+ t6 = vec_perm(t1, t3, swiz4);
2209
+ t7 = vec_perm(t2, t4, swiz3);
2210
+ t8 = vec_perm(t2, t4, swiz4);
2211
+ if (flip == true) {
2212
+ t5 = vec_xor(t5, xor_vector);
2213
+ t6 = vec_xor(t6, xor_vector);
2214
+ t7 = vec_xor(t7, xor_vector);
2215
+ t8 = vec_xor(t8, xor_vector);
2216
+ }
2217
+ vec_xst(t5, 0, vecOffset);
2218
+ vec_xst(t6, 0, vecOffset+16);
2219
+ vec_xst(t7, 0, vecOffset+32);
2220
+ vec_xst(t8, 0, vecOffset+48);
2221
+
2222
+ t1 = vec_perm(c1[1], c2[1], swiz1);
2223
+ t2 = vec_perm(c1[1], c2[1], swiz2);
2224
+ t3 = vec_perm(c3[1], c4[1], swiz1);
2225
+ t4 = vec_perm(c3[1], c4[1], swiz2);
2226
+ t5 = vec_perm(t1, t3, swiz3);
2227
+ t6 = vec_perm(t1, t3, swiz4);
2228
+ t7 = vec_perm(t2, t4, swiz3);
2229
+ t8 = vec_perm(t2, t4, swiz4);
2230
+ if (flip == true) {
2231
+ t5 = vec_xor(t5, xor_vector);
2232
+ t6 = vec_xor(t6, xor_vector);
2233
+ t7 = vec_xor(t7, xor_vector);
2234
+ t8 = vec_xor(t8, xor_vector);
2235
+ }
2236
+ vec_xst(t5, 0, vecOffset+64);
2237
+ vec_xst(t6, 0, vecOffset+80);
2238
+ vec_xst(t7, 0, vecOffset+96);
2239
+ vec_xst(t8, 0, vecOffset+112);
2240
+
2241
+ aoffset1 += lda;
2242
+ aoffset2 += lda;
2243
+ aoffset3 += lda;
2244
+ vecOffset += 128;
2245
+ i--;
2246
+ } while(i > 0);
2247
+ }
2248
+ }
2249
+ }
2250
+
2251
+ void mnpack(int64_t m0, int64_t m, int64_t n0, int64_t n) {
2252
+ int64_t mc, nc, mp, np;
2253
+ int m_rem = MIN(m - m0, 8);
2254
+ int n_rem = MIN(n - n0, 8);
2255
+ // TO-DO: KERNEL_16x8 and KERNEL_8x16 are having some performance
2256
+ // issues. After resolving them, below code will be enabled.
2257
+ /*if (m_rem >= 16 && n_rem >= 8) {
2258
+ mc = 16;
2259
+ nc = 8;
2260
+ gemm<16,8>(m0, m, n0, n);
2261
+ } else if(m_rem >= 8 && n_rem >= 16) {
2262
+ mc = 8;
2263
+ nc = 16;
2264
+ gemm<8,16>(m0, m, n0, n);
2265
+ }*/
2266
+ if (m_rem >= 8 && n_rem >= 8) {
2267
+ mc = 8;
2268
+ nc = 8;
2269
+ gemm<8,8>(m0, m, n0, n);
2270
+ } else if (m_rem >= 4 && n_rem >= 8) {
2271
+ mc = 4;
2272
+ nc = 8;
2273
+ gemm<4,8>(m0, m, n0, n);
2274
+ } else if (m_rem >= 8 && n_rem >= 4) {
2275
+ mc = 8;
2276
+ nc = 4;
2277
+ gemm<8,4>(m0, m, n0, n);
2278
+ } else if (m_rem >= 4 && n_rem >= 4) {
2279
+ mc = 4;
2280
+ nc = 4;
2281
+ gemm_small<4, 4>(m0, m, n0, n);
2282
+ } else if ((m_rem < 4) && (n_rem > 4)) {
2283
+ nc = 4;
2284
+ switch(m_rem) {
2285
+ case 1:
2286
+ mc = 1;
2287
+ gemm_small<1, 4>(m0, m, n0, n);
2288
+ break;
2289
+ case 2:
2290
+ mc = 2;
2291
+ gemm_small<2, 4>(m0, m, n0, n);
2292
+ break;
2293
+ case 3:
2294
+ mc = 3;
2295
+ gemm_small<3, 4>(m0, m, n0, n);
2296
+ break;
2297
+ default:
2298
+ return;
2299
+ }
2300
+ } else if ((m_rem > 4) && (n_rem < 4)) {
2301
+ mc = 4;
2302
+ switch(n_rem) {
2303
+ case 1:
2304
+ nc = 1;
2305
+ gemm_small<4, 1>(m0, m, n0, n);
2306
+ break;
2307
+ case 2:
2308
+ nc = 2;
2309
+ gemm_small<4, 2>(m0, m, n0, n);
2310
+ break;
2311
+ case 3:
2312
+ nc = 3;
2313
+ gemm_small<4, 3>(m0, m, n0, n);
2314
+ break;
2315
+ default:
2316
+ return;
2317
+ }
2318
+ } else {
2319
+ switch((m_rem << 4) | n_rem) {
2320
+ case 0x43:
2321
+ mc = 4;
2322
+ nc = 3;
2323
+ gemm_small<4, 3>(m0, m, n0, n);
2324
+ break;
2325
+ case 0x42:
2326
+ mc = 4;
2327
+ nc = 2;
2328
+ gemm_small<4, 2>(m0, m, n0, n);
2329
+ break;
2330
+ case 0x41:
2331
+ mc = 4;
2332
+ nc = 1;
2333
+ gemm_small<4, 1>(m0, m, n0, n);
2334
+ break;
2335
+ case 0x34:
2336
+ mc = 3;
2337
+ nc = 4;
2338
+ gemm_small<3, 4>(m0, m, n0, n);
2339
+ break;
2340
+ case 0x33:
2341
+ mc = 3;
2342
+ nc = 3;
2343
+ gemm_small<3, 3>(m0, m, n0, n);
2344
+ break;
2345
+ case 0x32:
2346
+ mc = 3;
2347
+ nc = 2;
2348
+ gemm_small<3, 2>(m0, m, n0, n);
2349
+ break;
2350
+ case 0x31:
2351
+ mc = 3;
2352
+ nc = 1;
2353
+ gemm_small<3, 1>(m0, m, n0, n);
2354
+ break;
2355
+ case 0x24:
2356
+ mc = 2;
2357
+ nc = 4;
2358
+ gemm_small<2, 4>(m0, m, n0, n);
2359
+ break;
2360
+ case 0x23:
2361
+ mc = 2;
2362
+ nc = 3;
2363
+ gemm_small<2, 3>(m0, m, n0, n);
2364
+ break;
2365
+ case 0x22:
2366
+ mc = 2;
2367
+ nc = 2;
2368
+ gemm_small<2, 2>(m0, m, n0, n);
2369
+ break;
2370
+ case 0x21:
2371
+ mc = 2;
2372
+ nc = 1;
2373
+ gemm_small<2, 1>(m0, m, n0, n);
2374
+ break;
2375
+ case 0x14:
2376
+ mc = 1;
2377
+ nc = 4;
2378
+ gemm_small<1, 4>(m0, m, n0, n);
2379
+ break;
2380
+ case 0x13:
2381
+ mc = 1;
2382
+ nc = 3;
2383
+ gemm_small<1, 3>(m0, m, n0, n);
2384
+ break;
2385
+ case 0x12:
2386
+ mc = 1;
2387
+ nc = 2;
2388
+ gemm_small<1, 2>(m0, m, n0, n);
2389
+ break;
2390
+ case 0x11:
2391
+ mc = 1;
2392
+ nc = 1;
2393
+ gemm_small<1, 1>(m0, m, n0, n);
2394
+ break;
2395
+ default:
2396
+ return;
2397
+ }
2398
+ }
2399
+ mp = m0 + (m - m0) / mc * mc;
2400
+ np = n0 + (n - n0) / nc * nc;
2401
+ mnpack(mp, m, n0, np);
2402
+ mnpack(m0, m, np, n);
2403
+ }
2404
+
2405
+ void KERNEL_4x8(int64_t ii, int64_t jj) {
2406
+ vec_t vec_A[8], vec_B[16] = {0};
2407
+ acc_t acc_0, acc_1;
2408
+ std::array<int, 4> comparray {};
2409
+ vector float fin_res[8] = {0};
2410
+ vector float vs[8] = {0};
2411
+ bool isAblock_q4 = std::is_same_v<TA, block_q4_0>;
2412
+ for (int l = 0; l < k; l++) {
2413
+ __builtin_mma_xxsetaccz(&acc_0);
2414
+ __builtin_mma_xxsetaccz(&acc_1);
2415
+ if (std::is_same_v<TA, block_q4_0>) {
2416
+ packNormalInt4<int8_t, vector signed char, 4>((A+(ii*lda)+l), lda, 4, 4, (int8_t*)vec_A, comparray);
2417
+ } else {
2418
+ packNormal<int8_t, vector signed char>((const TB*)(A+(ii*lda)+l), lda, 4, 8, (int8_t*)vec_A, false);
2419
+ }
2420
+ packNormal<uint8_t, vector unsigned char>((B+(jj*ldb)+l), ldb, 8, 8, (uint8_t*)vec_B, true);
2421
+ for(int x = 0; x < 8; x++) {
2422
+ __builtin_mma_xvi8ger4pp(&acc_0, vec_A[x], vec_B[x]);
2423
+ __builtin_mma_xvi8ger4pp(&acc_1, vec_A[x], vec_B[x+8]);
2424
+ }
2425
+ for (int I = 0; I<4; I++) {
2426
+ for (int J = 0; J<4; J++) {
2427
+ *((float*)&vs[I]+J) = (unhalf((A+((ii+I)*lda)+l)->d) * unhalf((B+((jj+J)*ldb)+l)->d));
2428
+ *((float*)&vs[I+4]+J) = (unhalf((A+((ii+I)*lda)+l)->d) * unhalf((B+((jj+J+4)*ldb)+l)->d));
2429
+ }
2430
+ }
2431
+ if (!isAblock_q4) {
2432
+ auto aoffset = A+(ii*lda)+l;
2433
+ for (int i = 0; i < 4; i++) {
2434
+ comparray[i] = 0;
2435
+ int ca = 0;
2436
+ auto *at = aoffset->qs;
2437
+ for (int j = 0; j < 32; j++)
2438
+ ca += (int)*at++;
2439
+ comparray[i] = ca;
2440
+ aoffset += lda;
2441
+ }
2442
+ }
2443
+ compute<4>(&acc_0, 0, 0, comparray, vs, fin_res);
2444
+ compute<4>(&acc_1, 0, 4, comparray, vs, fin_res);
2445
+ }
2446
+ save_res<4, 4>(ii, jj, 0, fin_res);
2447
+ save_res<4, 4>(ii, jj+4, 4, fin_res);
2448
+ }
2449
+
2450
+ void KERNEL_8x4(int64_t ii, int64_t jj) {
2451
+ vec_t vec_A[16], vec_B[8] = {0};
2452
+ acc_t acc_0, acc_1;
2453
+ std::array<int, 8> comparray {};
2454
+ vector float fin_res[8] = {0};
2455
+ vector float vs[8] = {0};
2456
+ bool isAblock_q4 = std::is_same_v<TA, block_q4_0>;
2457
+ for (int l = 0; l < k; l++) {
2458
+ __builtin_mma_xxsetaccz(&acc_0);
2459
+ __builtin_mma_xxsetaccz(&acc_1);
2460
+ if (std::is_same_v<TA, block_q4_0>) {
2461
+ packNormalInt4<int8_t, vector signed char, 8>((A+(ii*lda)+l), lda, 8, 4, (int8_t*)vec_A, comparray);
2462
+ } else {
2463
+ packNormal<int8_t, vector signed char>((const TB*)(A+(ii*lda)+l), lda, 8, 8, (int8_t*)vec_A, false);
2464
+ }
2465
+ packNormal<uint8_t, vector unsigned char>((B+(jj*ldb)+l), ldb, 4, 8, (uint8_t*)vec_B, true);
2466
+ for(int x = 0; x < 8; x++) {
2467
+ __builtin_mma_xvi8ger4pp(&acc_0, vec_A[x], vec_B[x]);
2468
+ __builtin_mma_xvi8ger4pp(&acc_1, vec_A[x+8], vec_B[x]);
2469
+ }
2470
+ for (int I = 0; I<8; I++) {
2471
+ for (int J = 0; J<4; J++) {
2472
+ *((float*)&vs[I]+J) = (unhalf((A+((ii+I)*lda)+l)->d) * unhalf((B+((jj+J)*ldb)+l)->d));
2473
+ }
2474
+ }
2475
+ if (!isAblock_q4) {
2476
+ auto aoffset = A+(ii*lda)+l;
2477
+ for (int i = 0; i < 8; i++) {
2478
+ comparray[i] = 0;
2479
+ int ca = 0;
2480
+ auto *at = aoffset->qs;
2481
+ for (int j = 0; j < 32; j++)
2482
+ ca += (int)*at++;
2483
+ comparray[i] = ca;
2484
+ aoffset += lda;
2485
+ }
2486
+ }
2487
+ compute<8>(&acc_0, 0, 0, comparray, vs, fin_res);
2488
+ compute<8>(&acc_1, 4, 4, comparray, vs, fin_res);
2489
+ }
2490
+ save_res<4, 4>(ii, jj, 0, fin_res);
2491
+ save_res<4, 4>(ii+4, jj, 4, fin_res);
2492
+ }
2493
+
2494
+ void KERNEL_8x8(int64_t ii, int64_t jj) {
2495
+ vec_t vec_A[16], vec_B[16] = {0};
2496
+ acc_t acc_0, acc_1, acc_2, acc_3;
2497
+ std::array<int, 8> comparray {};
2498
+ vector float fin_res[16] = {0};
2499
+ vector float vs[16] = {0};
2500
+ bool isAblock_q4 = std::is_same_v<TA, block_q4_0>;
2501
+ for (int l = 0; l < k; l++) {
2502
+ __builtin_mma_xxsetaccz(&acc_0);
2503
+ __builtin_mma_xxsetaccz(&acc_1);
2504
+ __builtin_mma_xxsetaccz(&acc_2);
2505
+ __builtin_mma_xxsetaccz(&acc_3);
2506
+ if (std::is_same_v<TA, block_q4_0>) {
2507
+ packNormalInt4<int8_t, vector signed char, 8>((A+(ii*lda)+l), lda, 8, 4, (int8_t*)vec_A, comparray);
2508
+ } else {
2509
+ packNormal<int8_t, vector signed char>((const TB*)(A+(ii*lda)+l), lda, 8, 8, (int8_t*)vec_A, false);
2510
+ }
2511
+ packNormal<uint8_t, vector unsigned char>((B+(jj*ldb)+l), ldb, 8, 8, (uint8_t*)vec_B, true);
2512
+ for(int x = 0; x < 8; x++) {
2513
+ __builtin_mma_xvi8ger4pp(&acc_0, vec_A[x], vec_B[x]);
2514
+ __builtin_mma_xvi8ger4pp(&acc_1, vec_A[x+8], vec_B[x]);
2515
+ __builtin_mma_xvi8ger4pp(&acc_2, vec_A[x], vec_B[x+8]);
2516
+ __builtin_mma_xvi8ger4pp(&acc_3, vec_A[x+8], vec_B[x+8]);
2517
+ }
2518
+ for (int I = 0; I<8; I++) {
2519
+ for (int J = 0; J<4; J++) {
2520
+ *((float*)&vs[I]+J) = (unhalf((A+((ii+I)*lda)+l)->d) * unhalf((B+((jj+J)*ldb)+l)->d));
2521
+ *((float*)&vs[I+8]+J) = (unhalf((A+((ii+I)*lda)+l)->d) * unhalf((B+((jj+J+4)*ldb)+l)->d));
2522
+ }
2523
+ }
2524
+ if (!isAblock_q4) {
2525
+ auto aoffset = A+(ii*lda)+l;
2526
+ for (int i = 0; i < 8; i++) {
2527
+ comparray[i] = 0;
2528
+ int ca = 0;
2529
+ auto *at = aoffset->qs;
2530
+ for (int j = 0; j < 32; j++)
2531
+ ca += (int)*at++;
2532
+ comparray[i] = ca;
2533
+ aoffset += lda;
2534
+ }
2535
+ }
2536
+ compute<8>(&acc_0, 0, 0, comparray, vs, fin_res);
2537
+ compute<8>(&acc_1, 4, 4, comparray, vs, fin_res);
2538
+ compute<8>(&acc_2, 0, 8, comparray, vs, fin_res);
2539
+ compute<8>(&acc_3, 4, 12, comparray, vs, fin_res);
2540
+ }
2541
+ save_res<4, 4>(ii, jj, 0, fin_res);
2542
+ save_res<4, 4>(ii+4, jj, 4, fin_res);
2543
+ save_res<4, 4>(ii, jj+4, 8, fin_res);
2544
+ save_res<4, 4>(ii+4, jj+4, 12, fin_res);
2545
+ }
2546
+
2547
+ template<int RM, int RN>
2548
+ void gemm_small(int64_t m0, int64_t m, int64_t n0, int64_t n) {
2549
+ int64_t ytiles = (m - m0) / RM;
2550
+ int64_t xtiles = (n - n0) / RN;
2551
+ int64_t tiles = xtiles * ytiles;
2552
+ int64_t duty = (tiles + nth - 1) / nth;
2553
+ int64_t start = duty * ith;
2554
+ int64_t end = start + duty;
2555
+ vec_t vec_A[8] = {0}, vec_B[8] = {0};
2556
+ vector signed int vec_C[4];
2557
+ acc_t acc_0;
2558
+ bool isAblock_q4 = std::is_same_v<TA, block_q4_0>;
2559
+
2560
+ if (end > tiles)
2561
+ end = tiles;
2562
+ for (int64_t job = start; job < end; ++job) {
2563
+ int64_t ii = m0 + job / xtiles * RM;
2564
+ int64_t jj = n0 + job % xtiles * RN;
2565
+ std::array<int, 4> comparray{};
2566
+ vector float res[4] = {0};
2567
+ vector float fin_res[4] = {0};
2568
+ vector float vs[4] = {0};
2569
+ vector float CA[4] = {0};
2570
+ __builtin_prefetch((A+(ii*lda)+0)->qs, 0, 1); // prefetch first value
2571
+ __builtin_prefetch((B+(jj*ldb)+0)->qs, 0, 1); // prefetch first value
2572
+ for (int l = 0; l < k; l++) {
2573
+ __builtin_prefetch((A+(ii*lda)+(l+1))->qs, 0, 1); // prefetch one loop ahead
2574
+ __builtin_prefetch((B+(jj*ldb)+(l+1))->qs, 0, 1); // prefetch one loop ahead
2575
+ __builtin_mma_xxsetaccz(&acc_0);
2576
+ if (isAblock_q4) {
2577
+ packNormalInt4<int8_t, vector signed char, 4>((A+(ii*lda)+l), lda, RM, 4, (int8_t*)vec_A, comparray);
2578
+ } else {
2579
+ packNormal<int8_t, vector signed char>((const TB*)(A+(ii*lda)+l), lda, RM, 8, (int8_t*)vec_A, false);
2580
+ }
2581
+ packNormal<uint8_t, vector unsigned char>((B+(jj*ldb)+l), ldb, RN, 8, (uint8_t*)vec_B, true);
2582
+ for(int x = 0; x < 8; x+=4) {
2583
+ __builtin_mma_xvi8ger4pp(&acc_0, vec_A[x], vec_B[x]);
2584
+ __builtin_mma_xvi8ger4pp(&acc_0, vec_A[x+1], vec_B[x+1]);
2585
+ __builtin_mma_xvi8ger4pp(&acc_0, vec_A[x+2], vec_B[x+2]);
2586
+ __builtin_mma_xvi8ger4pp(&acc_0, vec_A[x+3], vec_B[x+3]);
2587
+ }
2588
+ for (int I = 0; I<RM; I++) {
2589
+ for (int J = 0; J<RN; J++) {
2590
+ *((float*)&vs[I]+J) = (unhalf((A+((ii+I)*lda)+l)->d) * unhalf((B+((jj+J)*ldb)+l)->d));
2591
+ }
2592
+ }
2593
+ __builtin_mma_disassemble_acc(vec_C, &acc_0);
2594
+ if (!isAblock_q4) {
2595
+ auto aoffset = A+(ii*lda)+l;
2596
+ for (int i = 0; i < RM; i++) {
2597
+ comparray[i] = 0;
2598
+ int ca = 0;
2599
+ auto *at = aoffset->qs;
2600
+ for (int j = 0; j < 32; j++)
2601
+ ca += (int)*at++;
2602
+ comparray[i] = ca;
2603
+ aoffset += lda;
2604
+ }
2605
+ }
2606
+ for (int i = 0; i < RM; i++) {
2607
+ CA[i] = vec_splats((float)(((double)comparray[i]) * -128.0));
2608
+ res[i] = vec_add(vec_ctf(vec_C[i], 0), CA[i]);
2609
+ fin_res[i] = vec_madd(res[i], vs[i], fin_res[i]);
2610
+ }
2611
+ }
2612
+ save_res<RM, RN>(ii, jj, 0, fin_res);
2613
+ }
2614
+ }
2615
+
2616
+ template<int RM, int RN>
2617
+ inline void kernel(int64_t ii, int64_t jj) {
2618
+ if constexpr(RM == 4 && RN == 8) {
2619
+ KERNEL_4x8(ii,jj);
2620
+ } else if constexpr(RM == 8 && RN == 4) {
2621
+ KERNEL_8x4(ii,jj);
2622
+ } else if constexpr(RM == 8 && RN == 8) {
2623
+ KERNEL_8x8(ii,jj);
2624
+ } else {
2625
+ static_assert(false, "RN/RM values not supported");
2626
+ }
2627
+ }
2628
+
2629
+ template <int RM, int RN>
2630
+ NOINLINE void gemm(int64_t m0, int64_t m, int64_t n0, int64_t n) {
2631
+ int64_t ytiles = (m - m0) / RM;
2632
+ int64_t xtiles = (n - n0) / RN;
2633
+ int64_t tiles = xtiles * ytiles;
2634
+ int64_t duty = (tiles + nth - 1) / nth;
2635
+ int64_t start = duty * ith;
2636
+ int64_t end = start + duty;
2637
+ if (end > tiles)
2638
+ end = tiles;
2639
+ for (int64_t job = start; job < end; ++job) {
2640
+ int64_t ii = m0 + job / xtiles * RM;
2641
+ int64_t jj = n0 + job % xtiles * RN;
2642
+ kernel<RM, RN>(ii, jj);
2643
+ }
2644
+ }
2645
+
2646
+ const TA *const A;
2647
+ const TB *const B;
2648
+ TC *C;
2649
+ TA *At;
2650
+ TB *Bt;
2651
+ const int64_t k;
2652
+ const int64_t lda;
2653
+ const int64_t ldb;
2654
+ const int64_t ldc;
2655
+ const int ith;
2656
+ const int nth;
2657
+ };
2658
+
2659
+ template <typename TA, typename TB, typename TC>
2660
+ class tinyBLAS_PPC {
2661
+ public:
2662
+ tinyBLAS_PPC(int64_t k,
2663
+ const TA *A, int64_t lda,
2664
+ const TB *B, int64_t ldb,
2665
+ TC *C, int64_t ldc,
2666
+ int ith, int nth)
2667
+ : A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc), ith(ith), nth(nth) {
2668
+ }
2669
+
2670
+ void matmul(int64_t m, int64_t n) {
2671
+ mnpack(0, m, 0, n);
2672
+ }
2673
+
2674
+ private:
2675
+
2676
+ void (tinyBLAS_PPC::*kernel)(int64_t, int64_t);
2677
+
2678
+ template<typename VA>
2679
+ void packTranspose(const TA* a, int64_t lda, int rows, int cols, TA* vec) {
2680
+ int64_t i, j;
2681
+ TA *aoffset = NULL, *boffset = NULL;
2682
+ TA *aoffset1 = NULL, *aoffset2 = NULL, *aoffset3 = NULL, *aoffset4 = NULL;
2683
+ TA *aoffset5 = NULL, *aoffset6 = NULL, *aoffset7 = NULL, *aoffset8 = NULL;
2684
+ __vector_pair C1, C2, C3, C4, C5, C6, C7, C8;
2685
+ VA c1[2] = {0}, c2[2] = {0}, c3[2] = {0}, c4[2] = {0};
2686
+ VA c5[2] = {0}, c6[2] = {0}, c7[2] = {0}, c8[2] = {0};
2687
+ VA t1, t2, t3, t4, t5, t6, t7, t8;
2688
+ aoffset = const_cast<TA*>(a);
2689
+ boffset = vec;
2690
+ j = (rows >> 3);
2691
+ if (j > 0) {
2692
+
2693
+ do {
2694
+ aoffset1 = aoffset;
2695
+ aoffset2 = aoffset1 + lda;
2696
+ aoffset3 = aoffset2 + lda;
2697
+ aoffset4 = aoffset3 + lda;
2698
+ aoffset5 = aoffset4 + lda;
2699
+ aoffset6 = aoffset5 + lda;
2700
+ aoffset7 = aoffset6 + lda;
2701
+ aoffset8 = aoffset7 + lda;
2702
+ aoffset += 8 * lda;
2703
+ i = (cols >> 3);
2704
+ if (i > 0) {
2705
+ do {
2706
+ C1 = __builtin_vsx_lxvp(0, (__vector_pair*)aoffset1);
2707
+ C2 = __builtin_vsx_lxvp(0, (__vector_pair*)aoffset2);
2708
+ C3 = __builtin_vsx_lxvp(0, (__vector_pair*)aoffset3);
2709
+ C4 = __builtin_vsx_lxvp(0, (__vector_pair*)aoffset4);
2710
+ C5 = __builtin_vsx_lxvp(0, (__vector_pair*)aoffset5);
2711
+ C6 = __builtin_vsx_lxvp(0, (__vector_pair*)aoffset6);
2712
+ C7 = __builtin_vsx_lxvp(0, (__vector_pair*)aoffset7);
2713
+ C8 = __builtin_vsx_lxvp(0, (__vector_pair*)aoffset8);
2714
+ __builtin_vsx_disassemble_pair(c1, &C1);
2715
+ __builtin_vsx_disassemble_pair(c2, &C2);
2716
+ __builtin_vsx_disassemble_pair(c3, &C3);
2717
+ __builtin_vsx_disassemble_pair(c4, &C4);
2718
+ __builtin_vsx_disassemble_pair(c5, &C5);
2719
+ __builtin_vsx_disassemble_pair(c6, &C6);
2720
+ __builtin_vsx_disassemble_pair(c7, &C7);
2721
+ __builtin_vsx_disassemble_pair(c8, &C8);
2722
+
2723
+ t1 = vec_mergeh(c1[0], c2[0]);
2724
+ t2 = vec_mergeh(c3[0], c4[0]);
2725
+ t3 = vec_mergeh(c5[0], c6[0]);
2726
+ t4 = vec_mergeh(c7[0], c8[0]);
2727
+ t5 = vec_xxpermdi(t1, t2, 0);
2728
+ t6 = vec_xxpermdi(t3, t4, 0);
2729
+ t7 = vec_xxpermdi(t1, t2, 3);
2730
+ t8 = vec_xxpermdi(t3, t4, 3);
2731
+ vec_xst(t5, 0, boffset);
2732
+ vec_xst(t6, 0, boffset+4);
2733
+ vec_xst(t7, 0, boffset+8);
2734
+ vec_xst(t8, 0, boffset+12);
2735
+
2736
+ t1 = vec_mergel(c1[0], c2[0]);
2737
+ t2 = vec_mergel(c3[0], c4[0]);
2738
+ t3 = vec_mergel(c5[0], c6[0]);
2739
+ t4 = vec_mergel(c7[0], c8[0]);
2740
+ t5 = vec_xxpermdi(t1, t2, 0);
2741
+ t6 = vec_xxpermdi(t3, t4, 0);
2742
+ t7 = vec_xxpermdi(t1, t2, 3);
2743
+ t8 = vec_xxpermdi(t3, t4, 3);
2744
+ vec_xst(t5, 0, boffset+16);
2745
+ vec_xst(t6, 0, boffset+20);
2746
+ vec_xst(t7, 0, boffset+24);
2747
+ vec_xst(t8, 0, boffset+28);
2748
+
2749
+ t1 = vec_mergeh(c1[1], c2[1]);
2750
+ t2 = vec_mergeh(c3[1], c4[1]);
2751
+ t3 = vec_mergeh(c5[1], c6[1]);
2752
+ t4 = vec_mergeh(c7[1], c8[1]);
2753
+ t5 = vec_xxpermdi(t1, t2, 0);
2754
+ t6 = vec_xxpermdi(t3, t4, 0);
2755
+ t7 = vec_xxpermdi(t1, t2, 3);
2756
+ t8 = vec_xxpermdi(t3, t4, 3);
2757
+ vec_xst(t5, 0, boffset+32);
2758
+ vec_xst(t6, 0, boffset+36);
2759
+ vec_xst(t7, 0, boffset+40);
2760
+ vec_xst(t8, 0, boffset+44);
2761
+
2762
+ t1 = vec_mergel(c1[1], c2[1]);
2763
+ t2 = vec_mergel(c3[1], c4[1]);
2764
+ t3 = vec_mergel(c5[1], c6[1]);
2765
+ t4 = vec_mergel(c7[1], c8[1]);
2766
+ t5 = vec_xxpermdi(t1, t2, 0);
2767
+ t6 = vec_xxpermdi(t3, t4, 0);
2768
+ t7 = vec_xxpermdi(t1, t2, 3);
2769
+ t8 = vec_xxpermdi(t3, t4, 3);
2770
+ vec_xst(t5, 0, boffset+48);
2771
+ vec_xst(t6, 0, boffset+52);
2772
+ vec_xst(t7, 0, boffset+56);
2773
+ vec_xst(t8, 0, boffset+60);
2774
+
2775
+ aoffset1 += 8*lda;
2776
+ aoffset2 += 8*lda;
2777
+ aoffset3 += 8*lda;
2778
+ aoffset4 += 8*lda;
2779
+ boffset += 64;
2780
+ i--;
2781
+ } while(i > 0);
2782
+ }
2783
+ if (cols & 4) {
2784
+ c1[0] = vec_xl(0, aoffset1);
2785
+ c2[0] = vec_xl(0, aoffset2);
2786
+ c3[0] = vec_xl(0, aoffset3);
2787
+ c4[0] = vec_xl(0, aoffset4);
2788
+ c5[0] = vec_xl(0, aoffset5);
2789
+ c6[0] = vec_xl(0, aoffset6);
2790
+ c7[0] = vec_xl(0, aoffset7);
2791
+ c8[0] = vec_xl(0, aoffset8);
2792
+
2793
+ t1 = vec_mergeh(c1[0], c2[0]);
2794
+ t2 = vec_mergeh(c3[0], c4[0]);
2795
+ t3 = vec_mergeh(c5[0], c6[0]);
2796
+ t4 = vec_mergeh(c7[0], c8[0]);
2797
+ t5 = vec_xxpermdi(t1, t2, 0);
2798
+ t6 = vec_xxpermdi(t3, t4, 0);
2799
+ t7 = vec_xxpermdi(t1, t2, 3);
2800
+ t8 = vec_xxpermdi(t3, t4, 3);
2801
+ vec_xst(t5, 0, boffset);
2802
+ vec_xst(t6, 0, boffset+4);
2803
+ vec_xst(t7, 0, boffset+8);
2804
+ vec_xst(t8, 0, boffset+12);
2805
+
2806
+ t1 = vec_mergel(c1[0], c2[0]);
2807
+ t2 = vec_mergel(c3[0], c4[0]);
2808
+ t3 = vec_mergel(c5[0], c6[0]);
2809
+ t4 = vec_mergel(c7[0], c8[0]);
2810
+ t5 = vec_xxpermdi(t1, t2, 0);
2811
+ t6 = vec_xxpermdi(t3, t4, 0);
2812
+ t7 = vec_xxpermdi(t1, t2, 3);
2813
+ t8 = vec_xxpermdi(t3, t4, 3);
2814
+ vec_xst(t5, 0, boffset+16);
2815
+ vec_xst(t6, 0, boffset+20);
2816
+ vec_xst(t7, 0, boffset+24);
2817
+ vec_xst(t8, 0, boffset+28);
2818
+ }
2819
+ j--;
2820
+ } while(j > 0);
2821
+ }
2822
+
2823
+ if (rows & 4) {
2824
+ aoffset1 = aoffset;
2825
+ aoffset2 = aoffset1 + lda;
2826
+ aoffset3 = aoffset2 + lda;
2827
+ aoffset4 = aoffset3 + lda;
2828
+ aoffset += 4 * lda;
2829
+ i = (cols >> 3);
2830
+ if (i > 0) {
2831
+ do {
2832
+ C1 = __builtin_vsx_lxvp(0, (__vector_pair*)aoffset1);
2833
+ C2 = __builtin_vsx_lxvp(0, (__vector_pair*)aoffset2);
2834
+ C3 = __builtin_vsx_lxvp(0, (__vector_pair*)aoffset3);
2835
+ C4 = __builtin_vsx_lxvp(0, (__vector_pair*)aoffset4);
2836
+ __builtin_vsx_disassemble_pair(c1, &C1);
2837
+ __builtin_vsx_disassemble_pair(c2, &C2);
2838
+ __builtin_vsx_disassemble_pair(c3, &C3);
2839
+ __builtin_vsx_disassemble_pair(c4, &C4);
2840
+
2841
+ t1 = vec_mergeh(c1[0], c2[0]);
2842
+ t2 = vec_mergeh(c3[0], c4[0]);
2843
+ t3 = vec_mergel(c1[0], c2[0]);
2844
+ t4 = vec_mergel(c3[0], c4[0]);
2845
+ t5 = vec_xxpermdi(t1, t2, 0);
2846
+ t6 = vec_xxpermdi(t1, t2, 3);
2847
+ t7 = vec_xxpermdi(t3, t4, 0);
2848
+ t8 = vec_xxpermdi(t3, t4, 3);
2849
+ vec_xst(t5, 0, boffset);
2850
+ vec_xst(t6, 0, boffset+4);
2851
+ vec_xst(t7, 0, boffset+8);
2852
+ vec_xst(t8, 0, boffset+12);
2853
+
2854
+ t1 = vec_mergeh(c1[1], c2[1]);
2855
+ t2 = vec_mergeh(c3[1], c4[1]);
2856
+ t3 = vec_mergel(c1[1], c2[1]);
2857
+ t4 = vec_mergel(c3[1], c4[1]);
2858
+ t5 = vec_xxpermdi(t1, t2, 0);
2859
+ t6 = vec_xxpermdi(t1, t2, 3);
2860
+ t7 = vec_xxpermdi(t3, t4, 0);
2861
+ t8 = vec_xxpermdi(t3, t4, 3);
2862
+ vec_xst(t5, 0, boffset+16);
2863
+ vec_xst(t6, 0, boffset+20);
2864
+ vec_xst(t7, 0, boffset+24);
2865
+ vec_xst(t8, 0, boffset+28);
2866
+
2867
+ aoffset1 += 8*lda;
2868
+ aoffset2 += 8*lda;
2869
+ aoffset3 += 8*lda;
2870
+ aoffset4 += 8*lda;
2871
+ boffset += 32;
2872
+ i--;
2873
+ } while(i > 0);
2874
+ }
2875
+
2876
+ if (cols & 4) {
2877
+ c1[0] = vec_xl(0, aoffset1);
2878
+ c2[0] = vec_xl(0, aoffset2);
2879
+ c3[0] = vec_xl(0, aoffset3);
2880
+ c4[0] = vec_xl(0, aoffset4);
2881
+
2882
+ t1 = vec_mergeh(c1[0], c2[0]);
2883
+ t2 = vec_mergeh(c3[0], c4[0]);
2884
+ t3 = vec_xxpermdi(t1, t2, 0);
2885
+ t4 = vec_xxpermdi(t1, t2, 3);
2886
+ vec_xst(t3, 0, boffset);
2887
+ vec_xst(t4, 0, boffset+4);
2888
+
2889
+ t1 = vec_mergel(c1[0], c2[0]);
2890
+ t2 = vec_mergel(c3[0], c4[0]);
2891
+ t3 = vec_xxpermdi(t1, t2, 0);
2892
+ t4 = vec_xxpermdi(t1, t2, 3);
2893
+ vec_xst(t3, 0, boffset+8);
2894
+ vec_xst(t4, 0, boffset+12);
2895
+ }
2896
+ }
2897
+ if (rows & 3) {
2898
+ aoffset1 = aoffset;
2899
+ aoffset2 = aoffset1 + lda;
2900
+ aoffset3 = aoffset2 + lda;
2901
+ if (cols & 4) {
2902
+ c1[0] = vec_xl(0, aoffset1);
2903
+ c2[0] = vec_xl(0, aoffset2);
2904
+ c3[0] = vec_xl(0, aoffset3);
2905
+
2906
+ t1 = vec_mergeh(c1[0], c2[0]);
2907
+ t2 = vec_mergeh(c3[0], c4[0]);
2908
+ t3 = vec_xxpermdi(t1, t2, 0);
2909
+ t4 = vec_xxpermdi(t1, t2, 3);
2910
+ vec_xst(t3, 0, boffset);
2911
+ vec_xst(t4, 0, boffset+4);
2912
+
2913
+ t1 = vec_mergel(c1[0], c2[0]);
2914
+ t2 = vec_mergel(c3[0], c4[0]);
2915
+ t3 = vec_xxpermdi(t1, t2, 0);
2916
+ t4 = vec_xxpermdi(t1, t2, 3);
2917
+ vec_xst(t3, 0, boffset+8);
2918
+ vec_xst(t4, 0, boffset+12);
2919
+ }
2920
+ }
2921
+ }
2922
+
2923
+ void KERNEL_4x4(int64_t ii, int64_t jj) {
2924
+ vec_t vec_A[4], vec_B[4], vec_C[4];
2925
+ acc_t acc_0;
2926
+ __builtin_mma_xxsetaccz(&acc_0);
2927
+ for (int l = 0; l < k; l+=4) {
2928
+ packTranspose<vector float>(A+(ii*lda)+l, lda, 4, 4, (TA*)vec_A);
2929
+ packTranspose<vector float>(B+(jj*ldb)+l, ldb, 4, 4, (TA*)vec_B);
2930
+ __builtin_mma_xvf32gerpp(&acc_0, vec_A[0], vec_B[0]);
2931
+ __builtin_mma_xvf32gerpp(&acc_0, vec_A[1], vec_B[1]);
2932
+ __builtin_mma_xvf32gerpp(&acc_0, vec_A[2], vec_B[2]);
2933
+ __builtin_mma_xvf32gerpp(&acc_0, vec_A[3], vec_B[3]);
2934
+ }
2935
+ SAVE_ACC(&acc_0, ii, jj);
2936
+ }
2937
+
2938
+ void KERNEL_4x8(int64_t ii, int64_t jj) {
2939
+ vec_t vec_A[4], vec_B[8], vec_C[4];
2940
+ acc_t acc_0, acc_1;
2941
+ __builtin_mma_xxsetaccz(&acc_0);
2942
+ __builtin_mma_xxsetaccz(&acc_1);
2943
+ for (int64_t l = 0; l < k; l+=4) {
2944
+ packTranspose<vector float>(A+(ii*lda)+l, lda, 4, 4, (TA*)vec_A);
2945
+ packTranspose<vector float>(B+(jj*ldb)+l, ldb, 8, 4, (TA*)vec_B);
2946
+ __builtin_mma_xvf32gerpp(&acc_0, vec_A[0], (vec_t)vec_B[0]);
2947
+ __builtin_mma_xvf32gerpp(&acc_1, vec_A[0], (vec_t)vec_B[1]);
2948
+ __builtin_mma_xvf32gerpp(&acc_0, vec_A[1], (vec_t)vec_B[2]);
2949
+ __builtin_mma_xvf32gerpp(&acc_1, vec_A[1], (vec_t)vec_B[3]);
2950
+ __builtin_mma_xvf32gerpp(&acc_0, vec_A[2], (vec_t)vec_B[4]);
2951
+ __builtin_mma_xvf32gerpp(&acc_1, vec_A[2], (vec_t)vec_B[5]);
2952
+ __builtin_mma_xvf32gerpp(&acc_0, vec_A[3], (vec_t)vec_B[6]);
2953
+ __builtin_mma_xvf32gerpp(&acc_1, vec_A[3], (vec_t)vec_B[7]);
2954
+ }
2955
+ SAVE_ACC(&acc_0, ii, jj);
2956
+ SAVE_ACC(&acc_1, ii, jj+4);
2957
+ }
2958
+
2959
+ void KERNEL_8x4(int64_t ii, int64_t jj) {
2960
+ vec_t vec_A[8], vec_B[4], vec_C[4];
2961
+ acc_t acc_0, acc_1;
2962
+ __builtin_mma_xxsetaccz(&acc_0);
2963
+ __builtin_mma_xxsetaccz(&acc_1);
2964
+ for (int64_t l = 0; l < k; l+=4) {
2965
+ packTranspose<vector float>(A+(ii*lda)+l, lda, 8, 4, (TA*)vec_A);
2966
+ packTranspose<vector float>(B+(jj*ldb)+l, ldb, 4, 4, (TA*)vec_B);
2967
+ __builtin_mma_xvf32gerpp(&acc_0, (vec_t)vec_A[0], vec_B[0]);
2968
+ __builtin_mma_xvf32gerpp(&acc_1, (vec_t)vec_A[1], vec_B[0]);
2969
+ __builtin_mma_xvf32gerpp(&acc_0, (vec_t)vec_A[2], vec_B[1]);
2970
+ __builtin_mma_xvf32gerpp(&acc_1, (vec_t)vec_A[3], vec_B[1]);
2971
+ __builtin_mma_xvf32gerpp(&acc_0, (vec_t)vec_A[4], vec_B[2]);
2972
+ __builtin_mma_xvf32gerpp(&acc_1, (vec_t)vec_A[5], vec_B[2]);
2973
+ __builtin_mma_xvf32gerpp(&acc_0, (vec_t)vec_A[6], vec_B[3]);
2974
+ __builtin_mma_xvf32gerpp(&acc_1, (vec_t)vec_A[7], vec_B[3]);
2975
+ }
2976
+ SAVE_ACC(&acc_0, ii, jj);
2977
+ SAVE_ACC(&acc_1, ii+4, jj);
2978
+ }
2979
+
2980
+ void KERNEL_8x8(int64_t ii, int64_t jj) {
2981
+ vec_t vec_A[16], vec_B[16], vec_C[4];
2982
+ acc_t acc_0, acc_1, acc_2, acc_3;
2983
+ __builtin_mma_xxsetaccz(&acc_0);
2984
+ __builtin_mma_xxsetaccz(&acc_1);
2985
+ __builtin_mma_xxsetaccz(&acc_2);
2986
+ __builtin_mma_xxsetaccz(&acc_3);
2987
+ for (int l = 0; l < k; l+=8) {
2988
+ packTranspose<vector float>(A+(ii*lda)+l, lda, 8, 8, (TA*)vec_A);
2989
+ packTranspose<vector float>(B+(jj*ldb)+l, ldb, 8, 8, (TA*)vec_B);
2990
+ for(int x = 0; x < 16; x+=2) {
2991
+ __builtin_mma_xvf32gerpp(&acc_0, (vec_t)vec_A[x], vec_B[x]);
2992
+ __builtin_mma_xvf32gerpp(&acc_1, (vec_t)vec_A[x], vec_B[x+1]);
2993
+ __builtin_mma_xvf32gerpp(&acc_2, (vec_t)vec_A[x+1], vec_B[x]);
2994
+ __builtin_mma_xvf32gerpp(&acc_3, (vec_t)vec_A[x+1], vec_B[x+1]);
2995
+ }
2996
+ }
2997
+ SAVE_ACC(&acc_0, ii, jj);
2998
+ SAVE_ACC(&acc_1, ii, jj+4);
2999
+ SAVE_ACC(&acc_2, ii+4, jj);
3000
+ SAVE_ACC(&acc_3, ii+4, jj+4);
3001
+ }
3002
+
3003
+ void mnpack(int64_t m0, int64_t m, int64_t n0, int64_t n) {
3004
+ int64_t mc, nc, mp, np;
3005
+ int m_rem = MIN(m - m0, 16);
3006
+ int n_rem = MIN(n - n0, 16);
3007
+ if (m_rem >= 16 && n_rem >= 8) {
3008
+ mc = 8;
3009
+ nc = 8;
3010
+ gemm<8,8>(m0, m, n0, n);
3011
+ } else if(m_rem >= 8 && n_rem >= 16) {
3012
+ mc = 8;
3013
+ nc = 8;
3014
+ gemm<8,8>(m0, m, n0, n);
3015
+ } else if (m_rem >= 8 && n_rem >= 8) {
3016
+ mc = 8;
3017
+ nc = 8;
3018
+ gemm<8,8>(m0, m, n0, n);
3019
+ } else if (m_rem >= 4 && n_rem >= 8) {
3020
+ mc = 4;
3021
+ nc = 8;
3022
+ gemm<4,8>(m0, m, n0, n);
3023
+ } else if (m_rem >= 8 && n_rem >= 4) {
3024
+ mc = 8;
3025
+ nc = 4;
3026
+ gemm<8,4>(m0, m, n0, n);
3027
+ } else if (m_rem >= 4 && n_rem >= 4) {
3028
+ mc = 4;
3029
+ nc = 4;
3030
+ gemm<4,4>(m0, m, n0, n);
3031
+ } else if ((m_rem < 4) && (n_rem > 4)) {
3032
+ nc = 4;
3033
+ switch(m_rem) {
3034
+ case 1:
3035
+ mc = 1;
3036
+ gemm_small(m0, m, n0, n, mc, nc);
3037
+ break;
3038
+ case 2:
3039
+ mc = 2;
3040
+ gemm_small(m0, m, n0, n, mc, nc);
3041
+ break;
3042
+ case 3:
3043
+ mc = 3;
3044
+ gemm_small(m0, m, n0, n, mc, nc);
3045
+ break;
3046
+ default:
3047
+ return;
3048
+ }
3049
+ } else if ((m_rem > 4) && (n_rem < 4)) {
3050
+ mc = 4;
3051
+ switch(n_rem) {
3052
+ case 1:
3053
+ nc = 1;
3054
+ gemm_small(m0, m, n0, n, mc, nc);
3055
+ break;
3056
+ case 2:
3057
+ nc = 2;
3058
+ gemm_small(m0, m, n0, n, mc, nc);
3059
+ break;
3060
+ case 3:
3061
+ nc = 3;
3062
+ gemm_small(m0, m, n0, n, mc, nc);
3063
+ break;
3064
+ default:
3065
+ return;
3066
+ }
3067
+ } else {
3068
+ switch((m_rem << 4) | n_rem) {
3069
+ case 0x43:
3070
+ mc = 4;
3071
+ nc = 3;
3072
+ gemm_small(m0, m, n0, n, mc, nc);
3073
+ break;
3074
+ case 0x42:
3075
+ mc = 4;
3076
+ nc = 2;
3077
+ gemm_small(m0, m, n0, n, mc, nc);
3078
+ break;
3079
+ case 0x41:
3080
+ mc = 4;
3081
+ nc = 1;
3082
+ gemm_small(m0, m, n0, n, mc, nc);
3083
+ break;
3084
+ case 0x34:
3085
+ mc = 3;
3086
+ nc = 4;
3087
+ gemm_small(m0, m, n0, n, mc, nc);
3088
+ break;
3089
+ case 0x33:
3090
+ mc = 3;
3091
+ nc = 3;
3092
+ gemm_small(m0, m, n0, n, mc, nc);
3093
+ break;
3094
+ case 0x32:
3095
+ mc = 3;
3096
+ nc = 2;
3097
+ gemm_small(m0, m, n0, n, mc, nc);
3098
+ break;
3099
+ case 0x31:
3100
+ mc = 3;
3101
+ nc = 1;
3102
+ gemm_small(m0, m, n0, n, mc, nc);
3103
+ break;
3104
+ case 0x24:
3105
+ mc = 2;
3106
+ nc = 4;
3107
+ gemm_small(m0, m, n0, n, mc, nc);
3108
+ break;
3109
+ case 0x23:
3110
+ mc = 2;
3111
+ nc = 3;
3112
+ gemm_small(m0, m, n0, n, mc, nc);
3113
+ break;
3114
+ case 0x22:
3115
+ mc = 2;
3116
+ nc = 2;
3117
+ gemm_small(m0, m, n0, n, mc, nc);
3118
+ break;
3119
+ case 0x21:
3120
+ mc = 2;
3121
+ nc = 1;
3122
+ gemm_small(m0, m, n0, n, mc, nc);
3123
+ break;
3124
+ case 0x14:
3125
+ mc = 1;
3126
+ nc = 4;
3127
+ gemm_small(m0, m, n0, n, mc, nc);
3128
+ break;
3129
+ case 0x13:
3130
+ mc = 1;
3131
+ nc = 3;
3132
+ gemm_small(m0, m, n0, n, mc, nc);
3133
+ break;
3134
+ case 0x12:
3135
+ mc = 1;
3136
+ nc = 2;
3137
+ gemm_small(m0, m, n0, n, mc, nc);
3138
+ break;
3139
+ case 0x11:
3140
+ mc = 1;
3141
+ nc = 1;
3142
+ gemm_small(m0, m, n0, n, mc, nc);
3143
+ break;
3144
+ default:
3145
+ return;
3146
+ }
3147
+ }
3148
+ mp = m0 + (m - m0) / mc * mc;
3149
+ np = n0 + (n - n0) / nc * nc;
3150
+ mnpack(mp, m, n0, np);
3151
+ mnpack(m0, m, np, n);
3152
+ }
3153
+
3154
+ void gemm_small(int64_t m0, int64_t m, int64_t n0, int64_t n, int RM, int RN) {
3155
+ int64_t ytiles = (m - m0) / RM;
3156
+ int64_t xtiles = (n - n0) / RN;
3157
+ int64_t tiles = xtiles * ytiles;
3158
+ int64_t duty = (tiles + nth - 1) / nth;
3159
+ int64_t start = duty * ith;
3160
+ int64_t end = start + duty;
3161
+ if (end > tiles)
3162
+ end = tiles;
3163
+ for (int64_t job = start; job < end; ++job) {
3164
+ int64_t ii = m0 + job / xtiles * RM;
3165
+ int64_t jj = n0 + job % xtiles * RN;
3166
+ vec_t vec_C[4];
3167
+ acc_t acc_0;
3168
+ __builtin_mma_xxsetaccz(&acc_0);
3169
+ vec_t vec_A[4] {0}, vec_B[4] = {0};
3170
+ for (int l=0; l<k; l+=4) {
3171
+ /* 'GEMV Forwarding' concept is used in first two conditional loops.
3172
+ * when one of the matrix has a single row/column, the elements are
3173
+ * broadcasted, instead of using packing routine to prepack the
3174
+ * matrix elements.
3175
+ */
3176
+ if (RM == 1) {
3177
+ TA* a = const_cast<TA*>(A+(ii)*lda+l);
3178
+ packTranspose<vector float>(B+(jj*ldb)+l, ldb, RN, 4, (TA*)vec_B);
3179
+ vec_A[0] = (vec_t)vec_xl(0,a);
3180
+ vec_A[1] = (vec_t)vec_splats(*((TA*)&vec_A+1));
3181
+ vec_A[2] = (vec_t)vec_splats(*((TA*)&vec_A+2));
3182
+ vec_A[3] = (vec_t)vec_splats(*((TA*)&vec_A+3));
3183
+ } else if (RN == 1) {
3184
+ packTranspose<vector float>(A+(ii*lda)+l, lda, RM, 4, (TA*)vec_A);
3185
+ TB* b = const_cast<TB*>(B+(jj)*ldb+l);
3186
+ vec_B[0] = (vec_t)vec_xl(0,b);
3187
+ vec_B[1] = (vec_t)vec_splats(*((TB*)&vec_B+1));
3188
+ vec_B[2] = (vec_t)vec_splats(*((TB*)&vec_B+2));
3189
+ vec_B[3] = (vec_t)vec_splats(*((TB*)&vec_B+3));
3190
+ } else {
3191
+ packTranspose<vector float>(A+(ii*lda)+l, lda, RM, 4, (TA*)vec_A);
3192
+ packTranspose<vector float>(B+(jj*ldb)+l, ldb, RN, 4, (TA*)vec_B);
3193
+ }
3194
+ __builtin_mma_xvf32gerpp(&acc_0, vec_A[0], vec_B[0]);
3195
+ __builtin_mma_xvf32gerpp(&acc_0, vec_A[1], vec_B[1]);
3196
+ __builtin_mma_xvf32gerpp(&acc_0, vec_A[2], vec_B[2]);
3197
+ __builtin_mma_xvf32gerpp(&acc_0, vec_A[3], vec_B[3]);
3198
+ }
3199
+ __builtin_mma_disassemble_acc(vec_C, &acc_0);
3200
+ for (int I = 0; I < RM; I++) {
3201
+ for (int J = 0; J < RN; J++) {
3202
+ *((TC*)(C+ii+((jj+J)*ldc)+I)) = *((TC*)&vec_C[I]+J);
3203
+ }
3204
+ }
3205
+ }
3206
+ }
3207
+
3208
+ template <int RM, int RN>
3209
+ NOINLINE void gemm(int64_t m0, int64_t m, int64_t n0, int64_t n) {
3210
+ int64_t ytiles = (m - m0) / RM;
3211
+ int64_t xtiles = (n - n0) / RN;
3212
+ int64_t tiles = xtiles * ytiles;
3213
+ int64_t duty = (tiles + nth - 1) / nth;
3214
+ int64_t start = duty * ith;
3215
+ int64_t end = start + duty;
3216
+ if (RM == 4 && RN == 4) {
3217
+ kernel = &tinyBLAS_PPC::KERNEL_4x4;
3218
+ } else if (RM == 4 && RN == 8) {
3219
+ kernel = &tinyBLAS_PPC::KERNEL_4x8;
3220
+ } else if (RM == 8 && RN == 4) {
3221
+ kernel = &tinyBLAS_PPC::KERNEL_8x4;
3222
+ } else if (RM == 8 && RN == 8) {
3223
+ kernel = &tinyBLAS_PPC::KERNEL_8x8;
3224
+ }
3225
+ if (end > tiles)
3226
+ end = tiles;
3227
+ for (int64_t job = start; job < end; ++job) {
3228
+ int64_t ii = m0 + job / xtiles * RM;
3229
+ int64_t jj = n0 + job % xtiles * RN;
3230
+ (this->*kernel)(ii, jj);
3231
+ }
3232
+ }
3233
+
3234
+ const TA *const A;
3235
+ const TB *const B;
3236
+ TC *C;
3237
+ TA *At;
3238
+ TB *Bt;
3239
+ const int64_t k;
3240
+ const int64_t lda;
3241
+ const int64_t ldb;
3242
+ const int64_t ldc;
3243
+ const int ith;
3244
+ const int nth;
3245
+ };
3246
+ #endif
3247
+ } // namespace
3248
+
3249
+ /**
3250
+ * Performs optimized matrix multiplication on CPU.
3251
+ *
3252
+ * This subroutine may compute C = Aᵀ * B with column major ordering.
3253
+ * Despite its name, this isn't a generalized implementation. Work is
3254
+ * only performed when a handwritten kernel is written and available.
3255
+ * Otherwise the caller should fall back to a general matmul routine.
3256
+ *
3257
+ * For example, for single-threaded single-precision GEMM you can say
3258
+ *
3259
+ * llamafile_sgemm(m, n, k, A, lda, B, ldb, C, ldc,
3260
+ * 0, 1,
3261
+ * GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32);
3262
+ *
3263
+ * @param m is rows in `A` and `C`
3264
+ * @param n is cols in `B` and `C`
3265
+ * @param k is cols in `A` and rows in `B`
3266
+ * @param A is first input matrix (always transposed)
3267
+ * @param lda is row stride of `A`
3268
+ * @param B is second input matrix (never transposed)
3269
+ * @param ldb is row stride of `B`
3270
+ * @param C is input/output array of output matrices
3271
+ * @param ldc is row stride of `C`
3272
+ * @param ith is thread id (must be less than `nth`)
3273
+ * @param nth is number of threads (must be greater than zero)
3274
+ * @param Atype is GGML data type of `A`
3275
+ * @param Btype is GGML data type of `B`
3276
+ * @param Ctype is GGML data type of `C`
3277
+ * @return true if this function was able to service the matmul request
3278
+ */
3279
+ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64_t n, int64_t k,
3280
+ const void *A, int64_t lda, const void *B, int64_t ldb, void *C,
3281
+ int64_t ldc, int Atype, int Btype, int Ctype) {
3282
+
3283
+ assert(m >= 0);
3284
+ assert(n >= 0);
3285
+ assert(k >= 0);
3286
+ assert(lda >= k);
3287
+ assert(ldb >= k);
3288
+ assert(ldc >= m);
3289
+ assert(params->nth > 0);
3290
+ assert(params->ith < params->nth);
3291
+
3292
+ // only enable sgemm for prompt processing
3293
+ #if !defined(__MMA__)
3294
+ if (n < 2)
3295
+ return false;
3296
+ #endif
3297
+
3298
+ if (Ctype != GGML_TYPE_F32)
3299
+ return false;
3300
+
3301
+ switch (Atype) {
3302
+
3303
+ case GGML_TYPE_F32: {
3304
+ if (Btype != GGML_TYPE_F32)
3305
+ return false;
3306
+ #if defined(__AVX512F__)
3307
+ tinyBLAS<16, __m512, __m512, float, float, float> tb{ params,
3308
+ k, (const float *)A, lda,
3309
+ (const float *)B, ldb,
3310
+ (float *)C, ldc};
3311
+ return tb.matmul(m, n);
3312
+ #elif defined(__AVX__) || defined(__AVX2__)
3313
+ tinyBLAS<8, __m256, __m256, float, float, float> tb{ params,
3314
+ k, (const float *)A, lda,
3315
+ (const float *)B, ldb,
3316
+ (float *)C, ldc};
3317
+ return tb.matmul(m, n);
3318
+ #elif defined(__ARM_NEON)
3319
+ if (n < 4)
3320
+ return false;
3321
+ tinyBLAS<4, float32x4_t, float32x4_t, float, float, float> tb{ params,
3322
+ k, (const float *)A, lda,
3323
+ (const float *)B, ldb,
3324
+ (float *)C, ldc};
3325
+ return tb.matmul(m, n);
3326
+ #elif defined(__MMA__)
3327
+ if (k % 8)
3328
+ return false;
3329
+ tinyBLAS_PPC<float, float, float> tb{
3330
+ k, (const float *)A, lda,
3331
+ (const float *)B, ldb,
3332
+ (float *)C, ldc,
3333
+ params->ith, params->nth};
3334
+ tb.matmul(m, n);
3335
+ return true;
3336
+ #else
3337
+ return false;
3338
+ #endif
3339
+ }
3340
+
3341
+ case GGML_TYPE_BF16: {
3342
+ #if defined(__AVX512BF16__)
3343
+ if (Btype == GGML_TYPE_BF16) {
3344
+ tinyBLAS<32, __m512, __m512bh, ggml_bf16_t, ggml_bf16_t, float> tb{ params, k,
3345
+ (const ggml_bf16_t *)A, lda,
3346
+ (const ggml_bf16_t *)B, ldb,
3347
+ (float *)C, ldc};
3348
+ return tb.matmul(m, n);
3349
+ }
3350
+ #elif defined(__AVX512F__)
3351
+ if (Btype == GGML_TYPE_BF16) {
3352
+ tinyBLAS<16, __m512, __m512, ggml_bf16_t, ggml_bf16_t, float> tb{ params, k,
3353
+ (const ggml_bf16_t *)A, lda,
3354
+ (const ggml_bf16_t *)B, ldb,
3355
+ (float *)C, ldc};
3356
+ return tb.matmul(m, n);
3357
+ }
3358
+ #elif defined(__AVX2__)
3359
+ if (Btype == GGML_TYPE_BF16) {
3360
+ tinyBLAS<8, __m256, __m256, ggml_bf16_t, ggml_bf16_t, float> tb{ params, k,
3361
+ (const ggml_bf16_t *)A, lda,
3362
+ (const ggml_bf16_t *)B, ldb,
3363
+ (float *)C, ldc};
3364
+ return tb.matmul(m, n);
3365
+ }
3366
+ #elif defined(__MMA__)
3367
+ if ((k % 8))
3368
+ return false;
3369
+ if(Btype == GGML_TYPE_BF16) {
3370
+ tinyBLAS_BF16_PPC<ggml_bf16_t, ggml_bf16_t, float> tb{ k,
3371
+ (const ggml_bf16_t *)A, lda,
3372
+ (const ggml_bf16_t *)B, ldb,
3373
+ (float *)C, ldc,
3374
+ params->ith, params->nth};
3375
+ tb.matmul(m, n);
3376
+ return true;
3377
+ }
3378
+ #endif
3379
+ return false;
3380
+ }
3381
+
3382
+ case GGML_TYPE_F16: {
3383
+ #if defined(__AVX512F__)
3384
+ if (Btype == GGML_TYPE_F16) {
3385
+ tinyBLAS<16, __m512, __m512, ggml_fp16_t, ggml_fp16_t, float> tb{ params, k,
3386
+ (const ggml_fp16_t *)A, lda,
3387
+ (const ggml_fp16_t *)B, ldb,
3388
+ (float *)C, ldc};
3389
+ return tb.matmul(m, n);
3390
+ }
3391
+ #elif (defined(__AVX__) || defined(__AVX2__)) && defined(__F16C__)
3392
+ if (Btype == GGML_TYPE_F16) {
3393
+ tinyBLAS<8, __m256, __m256, ggml_fp16_t, ggml_fp16_t, float> tb{ params, k,
3394
+ (const ggml_fp16_t *)A, lda,
3395
+ (const ggml_fp16_t *)B, ldb,
3396
+ (float *)C, ldc};
3397
+ return tb.matmul(m, n);
3398
+ }
3399
+ #elif defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && !defined(_MSC_VER)
3400
+ if (n < 8)
3401
+ return false;
3402
+ if (Btype == GGML_TYPE_F16) {
3403
+ tinyBLAS<8, float16x8_t, float16x8_t, ggml_fp16_t, ggml_fp16_t, float> tb{ params,
3404
+ k, (const ggml_fp16_t *)A, lda,
3405
+ (const ggml_fp16_t *)B, ldb,
3406
+ (float *)C, ldc};
3407
+ return tb.matmul(m, n);
3408
+ }
3409
+ #elif defined(__ARM_NEON) && !defined(_MSC_VER)
3410
+ if (Btype == GGML_TYPE_F32) {
3411
+ tinyBLAS<4, float32x4_t, float32x4_t, ggml_fp16_t, float, float> tb{ params,
3412
+ k, (const ggml_fp16_t *)A, lda,
3413
+ (const float *)B, ldb,
3414
+ (float *)C, ldc};
3415
+ return tb.matmul(m, n);
3416
+ }
3417
+ #endif
3418
+ return false;
3419
+ }
3420
+
3421
+ case GGML_TYPE_Q8_0: {
3422
+ if (Btype != GGML_TYPE_Q8_0)
3423
+ return false;
3424
+ #if defined(__AVX2__) || defined(__AVX512F__) || defined(__AVX__)
3425
+ tinyBLAS_Q0_AVX<block_q8_0, block_q8_0, float> tb{
3426
+ k, (const block_q8_0 *)A, lda,
3427
+ (const block_q8_0 *)B, ldb,
3428
+ (float *)C, ldc,
3429
+ params->ith, params->nth};
3430
+ tb.matmul(m, n);
3431
+ return true;
3432
+ #elif defined(__ARM_FEATURE_DOTPROD)
3433
+ tinyBLAS_Q0_ARM<block_q8_0> tb{
3434
+ k, (const block_q8_0 *)A, lda,
3435
+ (const block_q8_0 *)B, ldb,
3436
+ (float *)C, ldc,
3437
+ params->ith, params->nth};
3438
+ tb.matmul(m, n);
3439
+ return true;
3440
+ #elif defined(__MMA__)
3441
+ //TO-DO: Remove this condition once gemv forwarding is enabled.
3442
+ if (n < 8 && n != 4)
3443
+ return false;
3444
+ if (m < 8 && m != 4)
3445
+ return false;
3446
+ tinyBLAS_Q0_PPC<block_q8_0, block_q8_0, float> tb{
3447
+ k, (const block_q8_0 *)A, lda,
3448
+ (const block_q8_0 *)B, ldb,
3449
+ (float *)C, ldc,
3450
+ params->ith, params->nth};
3451
+ tb.matmul(m, n);
3452
+ return true;
3453
+ #else
3454
+ return false;
3455
+ #endif
3456
+ }
3457
+
3458
+ case GGML_TYPE_Q4_0: {
3459
+ if (Btype != GGML_TYPE_Q8_0)
3460
+ return false;
3461
+ #if defined(__AVX2__) || defined(__AVX512F__) || defined(__AVX__)
3462
+ tinyBLAS_Q0_AVX<block_q4_0, block_q8_0, float> tb{
3463
+ k, (const block_q4_0 *)A, lda,
3464
+ (const block_q8_0 *)B, ldb,
3465
+ (float *)C, ldc,
3466
+ params->ith, params->nth};
3467
+ tb.matmul(m, n);
3468
+ return true;
3469
+ #elif defined(__ARM_FEATURE_DOTPROD)
3470
+ tinyBLAS_Q0_ARM<block_q4_0> tb{
3471
+ k, (const block_q4_0 *)A, lda,
3472
+ (const block_q8_0 *)B, ldb,
3473
+ (float *)C, ldc,
3474
+ params->ith, params->nth};
3475
+ tb.matmul(m, n);
3476
+ return true;
3477
+ #elif defined(__MMA__)
3478
+ //TO-DO: Remove this condition once gemv forwarding is enabled.
3479
+ if (n < 8 && n != 4)
3480
+ return false;
3481
+ if (m < 8 && m != 4)
3482
+ return false;
3483
+ tinyBLAS_Q0_PPC<block_q4_0, block_q8_0, float> tb{
3484
+ k, (const block_q4_0 *)A, lda,
3485
+ (const block_q8_0 *)B, ldb,
3486
+ (float *)C, ldc,
3487
+ params->ith, params->nth};
3488
+ tb.matmul(m, n);
3489
+ return true;
3490
+ #else
3491
+ return false;
3492
+ #endif
3493
+ }
3494
+
3495
+ case GGML_TYPE_Q5_0: {
3496
+ if (Btype != GGML_TYPE_Q8_0)
3497
+ return false;
3498
+ #if defined(__AVX2__) || defined(__AVX512F__) || defined(__AVX__)
3499
+ tinyBLAS_Q0_AVX<block_q5_0, block_q8_0, float> tb{
3500
+ k, (const block_q5_0 *)A, lda,
3501
+ (const block_q8_0 *)B, ldb,
3502
+ (float *)C, ldc,
3503
+ params->ith, params->nth};
3504
+ tb.matmul(m, n);
3505
+ return true;
3506
+ #else
3507
+ return false;
3508
+ #endif
3509
+ }
3510
+
3511
+ case GGML_TYPE_IQ4_NL: {
3512
+ if (Btype != GGML_TYPE_Q8_0)
3513
+ return false;
3514
+ #if defined(__AVX2__) || defined(__AVX512F__) || defined(__AVX__)
3515
+ tinyBLAS_Q0_AVX<block_iq4_nl, block_q8_0, float> tb{
3516
+ k, (const block_iq4_nl *)A, lda,
3517
+ (const block_q8_0 *)B, ldb,
3518
+ (float *)C, ldc,
3519
+ params->ith, params->nth};
3520
+ tb.matmul(m, n);
3521
+ return true;
3522
+ #else
3523
+ return false;
3524
+ #endif
3525
+ }
3526
+
3527
+ default:
3528
+ return false;
3529
+ }
3530
+
3531
+ (void)params;
3532
+ (void)m;
3533
+ (void)n;
3534
+ (void)k;
3535
+ (void)A;
3536
+ (void)lda;
3537
+ (void)B;
3538
+ (void)ldb;
3539
+ (void)C;
3540
+ (void)ldc;
3541
+ (void)Atype;
3542
+ (void)Btype;
3543
+ (void)Ctype;
3544
+ }