whispercpp 1.3.1 → 1.3.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (797) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +4 -3
  3. data/README.md +92 -31
  4. data/Rakefile +26 -7
  5. data/ext/.gitignore +5 -7
  6. data/ext/dependencies.rb +61 -0
  7. data/ext/extconf.rb +21 -198
  8. data/ext/options.rb +221 -0
  9. data/ext/ruby_whisper.c +159 -0
  10. data/ext/ruby_whisper.h +17 -2
  11. data/ext/ruby_whisper_context.c +641 -0
  12. data/ext/ruby_whisper_error.c +52 -0
  13. data/ext/ruby_whisper_model.c +232 -0
  14. data/ext/ruby_whisper_params.c +1301 -0
  15. data/ext/ruby_whisper_segment.c +143 -0
  16. data/ext/ruby_whisper_transcribe.cpp +87 -0
  17. data/ext/ruby_whisper_vad_params.c +288 -0
  18. data/ext/sources/.dockerignore +3 -0
  19. data/ext/sources/.github/workflows/bindings-ruby.yml +21 -0
  20. data/ext/sources/CMakeGraphVizOptions.cmake +8 -0
  21. data/ext/sources/CMakeLists.txt +251 -0
  22. data/ext/sources/bindings/javascript/CMakeLists.txt +41 -0
  23. data/ext/sources/bindings/javascript/emscripten.cpp +93 -0
  24. data/ext/sources/bindings/javascript/libwhisper.worker.js +1 -0
  25. data/ext/sources/bindings/javascript/package-tmpl.json +26 -0
  26. data/ext/sources/bindings/javascript/package.json +26 -0
  27. data/ext/sources/bindings/javascript/whisper.js +19 -0
  28. data/ext/sources/build-xcframework.sh +547 -0
  29. data/ext/sources/ci/run.sh +336 -0
  30. data/ext/sources/close-issue.yml +28 -0
  31. data/ext/sources/cmake/DefaultTargetOptions.cmake +16 -0
  32. data/ext/sources/cmake/FindFFmpeg.cmake +163 -0
  33. data/ext/sources/cmake/build-info.cmake +60 -0
  34. data/ext/sources/cmake/git-vars.cmake +22 -0
  35. data/ext/sources/cmake/whisper-config.cmake.in +65 -0
  36. data/ext/sources/cmake/whisper.pc.in +10 -0
  37. data/ext/sources/examples/CMakeLists.txt +124 -0
  38. data/ext/sources/examples/addon.node/CMakeLists.txt +31 -0
  39. data/ext/sources/examples/addon.node/__test__/whisper.spec.js +37 -0
  40. data/ext/sources/examples/addon.node/addon.cpp +438 -0
  41. data/ext/sources/examples/addon.node/index.js +54 -0
  42. data/ext/sources/examples/addon.node/package.json +16 -0
  43. data/ext/sources/examples/bench/CMakeLists.txt +8 -0
  44. data/ext/sources/examples/bench/bench.cpp +175 -0
  45. data/ext/sources/examples/bench.wasm/CMakeLists.txt +49 -0
  46. data/ext/sources/examples/bench.wasm/emscripten.cpp +87 -0
  47. data/ext/sources/examples/bench.wasm/index-tmpl.html +284 -0
  48. data/ext/sources/examples/cli/CMakeLists.txt +8 -0
  49. data/ext/sources/examples/cli/cli.cpp +1294 -0
  50. data/ext/sources/examples/coi-serviceworker.js +146 -0
  51. data/ext/sources/examples/command/CMakeLists.txt +10 -0
  52. data/ext/sources/examples/command/command.cpp +776 -0
  53. data/ext/sources/examples/command/commands.txt +9 -0
  54. data/ext/sources/examples/command.wasm/CMakeLists.txt +50 -0
  55. data/ext/sources/examples/command.wasm/emscripten.cpp +327 -0
  56. data/ext/sources/examples/command.wasm/index-tmpl.html +414 -0
  57. data/ext/sources/examples/common-ggml.cpp +238 -0
  58. data/ext/sources/examples/common-ggml.h +18 -0
  59. data/ext/sources/examples/common-sdl.cpp +227 -0
  60. data/ext/sources/examples/common-sdl.h +49 -0
  61. data/ext/sources/examples/common-whisper.cpp +168 -0
  62. data/ext/sources/examples/common-whisper.h +24 -0
  63. data/ext/sources/examples/common.cpp +675 -0
  64. data/ext/sources/examples/common.h +322 -0
  65. data/ext/sources/examples/deprecation-warning/CMakeLists.txt +6 -0
  66. data/ext/sources/examples/deprecation-warning/deprecation-warning.cpp +38 -0
  67. data/ext/sources/examples/ffmpeg-transcode.cpp +368 -0
  68. data/ext/sources/examples/generate-karaoke.sh +57 -0
  69. data/ext/sources/examples/grammar-parser.cpp +423 -0
  70. data/ext/sources/examples/grammar-parser.h +29 -0
  71. data/ext/sources/examples/helpers.js +191 -0
  72. data/ext/sources/examples/json.hpp +24596 -0
  73. data/ext/sources/examples/livestream.sh +112 -0
  74. data/ext/sources/examples/lsp/CMakeLists.txt +9 -0
  75. data/ext/sources/examples/lsp/lsp.cpp +467 -0
  76. data/ext/sources/examples/lsp/whisper.vim +362 -0
  77. data/ext/sources/examples/miniaudio.h +93468 -0
  78. data/ext/sources/examples/python/test_whisper_processor.py +7 -0
  79. data/ext/sources/examples/python/whisper_processor.py +54 -0
  80. data/ext/sources/examples/quantize/CMakeLists.txt +6 -0
  81. data/ext/sources/examples/quantize/quantize.cpp +223 -0
  82. data/ext/sources/examples/server/CMakeLists.txt +12 -0
  83. data/ext/sources/examples/server/bench.js +29 -0
  84. data/ext/sources/examples/server/httplib.h +10497 -0
  85. data/ext/sources/examples/server/server.cpp +1091 -0
  86. data/ext/sources/examples/server.py +115 -0
  87. data/ext/sources/examples/stb_vorbis.c +5584 -0
  88. data/ext/sources/examples/stream/CMakeLists.txt +10 -0
  89. data/ext/sources/examples/stream/stream.cpp +429 -0
  90. data/ext/sources/examples/stream.wasm/CMakeLists.txt +49 -0
  91. data/ext/sources/examples/stream.wasm/emscripten.cpp +216 -0
  92. data/ext/sources/examples/stream.wasm/index-tmpl.html +414 -0
  93. data/ext/sources/examples/sycl/CMakeLists.txt +9 -0
  94. data/ext/sources/examples/sycl/build.sh +22 -0
  95. data/ext/sources/examples/sycl/ls-sycl-device.cpp +11 -0
  96. data/ext/sources/examples/sycl/run-whisper.sh +17 -0
  97. data/ext/sources/examples/talk-llama/CMakeLists.txt +40 -0
  98. data/ext/sources/examples/talk-llama/eleven-labs.py +80 -0
  99. data/ext/sources/examples/talk-llama/llama-adapter.cpp +388 -0
  100. data/ext/sources/examples/talk-llama/llama-adapter.h +76 -0
  101. data/ext/sources/examples/talk-llama/llama-arch.cpp +1746 -0
  102. data/ext/sources/examples/talk-llama/llama-arch.h +437 -0
  103. data/ext/sources/examples/talk-llama/llama-batch.cpp +374 -0
  104. data/ext/sources/examples/talk-llama/llama-batch.h +89 -0
  105. data/ext/sources/examples/talk-llama/llama-chat.cpp +663 -0
  106. data/ext/sources/examples/talk-llama/llama-chat.h +58 -0
  107. data/ext/sources/examples/talk-llama/llama-context.cpp +2676 -0
  108. data/ext/sources/examples/talk-llama/llama-context.h +276 -0
  109. data/ext/sources/examples/talk-llama/llama-cparams.cpp +5 -0
  110. data/ext/sources/examples/talk-llama/llama-cparams.h +41 -0
  111. data/ext/sources/examples/talk-llama/llama-grammar.cpp +1229 -0
  112. data/ext/sources/examples/talk-llama/llama-grammar.h +173 -0
  113. data/ext/sources/examples/talk-llama/llama-graph.cpp +1618 -0
  114. data/ext/sources/examples/talk-llama/llama-graph.h +640 -0
  115. data/ext/sources/examples/talk-llama/llama-hparams.cpp +95 -0
  116. data/ext/sources/examples/talk-llama/llama-hparams.h +190 -0
  117. data/ext/sources/examples/talk-llama/llama-impl.cpp +167 -0
  118. data/ext/sources/examples/talk-llama/llama-impl.h +61 -0
  119. data/ext/sources/examples/talk-llama/llama-io.cpp +15 -0
  120. data/ext/sources/examples/talk-llama/llama-io.h +35 -0
  121. data/ext/sources/examples/talk-llama/llama-kv-cache.cpp +2739 -0
  122. data/ext/sources/examples/talk-llama/llama-kv-cache.h +502 -0
  123. data/ext/sources/examples/talk-llama/llama-kv-cells.h +379 -0
  124. data/ext/sources/examples/talk-llama/llama-memory.cpp +1 -0
  125. data/ext/sources/examples/talk-llama/llama-memory.h +32 -0
  126. data/ext/sources/examples/talk-llama/llama-mmap.cpp +600 -0
  127. data/ext/sources/examples/talk-llama/llama-mmap.h +68 -0
  128. data/ext/sources/examples/talk-llama/llama-model-loader.cpp +1138 -0
  129. data/ext/sources/examples/talk-llama/llama-model-loader.h +169 -0
  130. data/ext/sources/examples/talk-llama/llama-model-saver.cpp +281 -0
  131. data/ext/sources/examples/talk-llama/llama-model-saver.h +37 -0
  132. data/ext/sources/examples/talk-llama/llama-model.cpp +13814 -0
  133. data/ext/sources/examples/talk-llama/llama-model.h +425 -0
  134. data/ext/sources/examples/talk-llama/llama-quant.cpp +966 -0
  135. data/ext/sources/examples/talk-llama/llama-quant.h +1 -0
  136. data/ext/sources/examples/talk-llama/llama-sampling.cpp +2575 -0
  137. data/ext/sources/examples/talk-llama/llama-sampling.h +32 -0
  138. data/ext/sources/examples/talk-llama/llama-vocab.cpp +3340 -0
  139. data/ext/sources/examples/talk-llama/llama-vocab.h +131 -0
  140. data/ext/sources/examples/talk-llama/llama.cpp +354 -0
  141. data/ext/sources/examples/talk-llama/llama.h +1377 -0
  142. data/ext/sources/examples/talk-llama/prompts/talk-alpaca.txt +23 -0
  143. data/ext/sources/examples/talk-llama/speak +40 -0
  144. data/ext/sources/examples/talk-llama/speak.bat +1 -0
  145. data/ext/sources/examples/talk-llama/speak.ps1 +14 -0
  146. data/ext/sources/examples/talk-llama/talk-llama.cpp +808 -0
  147. data/ext/sources/examples/talk-llama/unicode-data.cpp +7034 -0
  148. data/ext/sources/examples/talk-llama/unicode-data.h +20 -0
  149. data/ext/sources/examples/talk-llama/unicode.cpp +849 -0
  150. data/ext/sources/examples/talk-llama/unicode.h +66 -0
  151. data/ext/sources/examples/vad-speech-segments/CMakeLists.txt +8 -0
  152. data/ext/sources/examples/vad-speech-segments/speech.cpp +143 -0
  153. data/ext/sources/examples/wchess/CMakeLists.txt +10 -0
  154. data/ext/sources/examples/wchess/libwchess/CMakeLists.txt +19 -0
  155. data/ext/sources/examples/wchess/libwchess/Chessboard.cpp +803 -0
  156. data/ext/sources/examples/wchess/libwchess/Chessboard.h +33 -0
  157. data/ext/sources/examples/wchess/libwchess/WChess.cpp +193 -0
  158. data/ext/sources/examples/wchess/libwchess/WChess.h +63 -0
  159. data/ext/sources/examples/wchess/libwchess/test-chessboard.cpp +117 -0
  160. data/ext/sources/examples/wchess/wchess.cmd/CMakeLists.txt +8 -0
  161. data/ext/sources/examples/wchess/wchess.cmd/wchess.cmd.cpp +249 -0
  162. data/ext/sources/examples/whisper.wasm/CMakeLists.txt +50 -0
  163. data/ext/sources/examples/whisper.wasm/emscripten.cpp +118 -0
  164. data/ext/sources/examples/whisper.wasm/index-tmpl.html +658 -0
  165. data/ext/sources/ggml/CMakeLists.txt +390 -0
  166. data/ext/sources/ggml/cmake/BuildTypes.cmake +54 -0
  167. data/ext/sources/ggml/cmake/GitVars.cmake +22 -0
  168. data/ext/sources/ggml/cmake/common.cmake +26 -0
  169. data/ext/sources/ggml/cmake/ggml-config.cmake.in +152 -0
  170. data/ext/{ggml → sources/ggml}/include/ggml-alloc.h +1 -1
  171. data/ext/{ggml → sources/ggml}/include/ggml-backend.h +9 -7
  172. data/ext/{ggml → sources/ggml}/include/ggml-cpp.h +2 -1
  173. data/ext/{ggml → sources/ggml}/include/ggml-cpu.h +9 -1
  174. data/ext/{ggml → sources/ggml}/include/ggml-metal.h +1 -1
  175. data/ext/{ggml → sources/ggml}/include/ggml-opt.h +49 -28
  176. data/ext/{ggml → sources/ggml}/include/ggml-rpc.h +6 -1
  177. data/ext/{ggml → sources/ggml}/include/ggml-vulkan.h +0 -2
  178. data/ext/{ggml → sources/ggml}/include/ggml.h +182 -265
  179. data/ext/sources/ggml/include/gguf.h +202 -0
  180. data/ext/sources/ggml/src/CMakeLists.txt +346 -0
  181. data/ext/{ggml → sources/ggml}/src/ggml-alloc.c +34 -29
  182. data/ext/sources/ggml/src/ggml-amx/CMakeLists.txt +107 -0
  183. data/ext/{ggml → sources/ggml}/src/ggml-backend-impl.h +1 -2
  184. data/ext/{ggml → sources/ggml}/src/ggml-backend-reg.cpp +87 -53
  185. data/ext/{ggml → sources/ggml}/src/ggml-backend.cpp +26 -14
  186. data/ext/sources/ggml/src/ggml-blas/CMakeLists.txt +87 -0
  187. data/ext/sources/ggml/src/ggml-cann/CMakeLists.txt +74 -0
  188. data/ext/sources/ggml/src/ggml-cann/Doxyfile +2579 -0
  189. data/ext/{ggml → sources/ggml}/src/ggml-cann/acl_tensor.cpp +10 -4
  190. data/ext/{ggml → sources/ggml}/src/ggml-cann/acl_tensor.h +5 -5
  191. data/ext/{ggml → sources/ggml}/src/ggml-cann/aclnn_ops.cpp +1272 -1506
  192. data/ext/sources/ggml/src/ggml-cann/aclnn_ops.h +1125 -0
  193. data/ext/{ggml → sources/ggml}/src/ggml-cann/common.h +135 -1
  194. data/ext/{ggml → sources/ggml}/src/ggml-cann/ggml-cann.cpp +564 -146
  195. data/ext/sources/ggml/src/ggml-cann/kernels/CMakeLists.txt +30 -0
  196. data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/dup.cpp +3 -5
  197. data/ext/{ggml → sources/ggml}/src/ggml-common.h +12 -8
  198. data/ext/sources/ggml/src/ggml-cpu/CMakeLists.txt +504 -0
  199. data/ext/{ggml → sources/ggml}/src/ggml-cpu/amx/amx.cpp +2 -1
  200. data/ext/sources/ggml/src/ggml-cpu/binary-ops.cpp +158 -0
  201. data/ext/sources/ggml/src/ggml-cpu/binary-ops.h +16 -0
  202. data/ext/sources/ggml/src/ggml-cpu/cmake/FindSIMD.cmake +100 -0
  203. data/ext/sources/ggml/src/ggml-cpu/common.h +72 -0
  204. data/ext/{ggml → sources/ggml}/src/ggml-cpu/cpu-feats-x86.cpp +5 -1
  205. data/ext/sources/ggml/src/ggml-cpu/ggml-cpu-aarch64.cpp +6431 -0
  206. data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-impl.h +163 -41
  207. data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-quants.c +4029 -1117
  208. data/ext/sources/ggml/src/ggml-cpu/ggml-cpu.c +3510 -0
  209. data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu.cpp +67 -18
  210. data/ext/sources/ggml/src/ggml-cpu/kleidiai/kernels.cpp +337 -0
  211. data/ext/sources/ggml/src/ggml-cpu/kleidiai/kernels.h +95 -0
  212. data/ext/sources/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp +482 -0
  213. data/ext/sources/ggml/src/ggml-cpu/kleidiai/kleidiai.h +17 -0
  214. data/ext/sources/ggml/src/ggml-cpu/llamafile/sgemm.cpp +3544 -0
  215. data/ext/sources/ggml/src/ggml-cpu/llamafile/sgemm.h +14 -0
  216. data/ext/sources/ggml/src/ggml-cpu/ops.cpp +8903 -0
  217. data/ext/sources/ggml/src/ggml-cpu/ops.h +110 -0
  218. data/ext/sources/ggml/src/ggml-cpu/simd-mappings.h +892 -0
  219. data/ext/sources/ggml/src/ggml-cpu/unary-ops.cpp +186 -0
  220. data/ext/sources/ggml/src/ggml-cpu/unary-ops.h +28 -0
  221. data/ext/sources/ggml/src/ggml-cpu/vec.cpp +252 -0
  222. data/ext/sources/ggml/src/ggml-cpu/vec.h +818 -0
  223. data/ext/sources/ggml/src/ggml-cuda/CMakeLists.txt +184 -0
  224. data/ext/sources/ggml/src/ggml-cuda/acc.cu +61 -0
  225. data/ext/sources/ggml/src/ggml-cuda/acc.cuh +5 -0
  226. data/ext/sources/ggml/src/ggml-cuda/arange.cu +34 -0
  227. data/ext/sources/ggml/src/ggml-cuda/arange.cuh +5 -0
  228. data/ext/sources/ggml/src/ggml-cuda/argmax.cu +91 -0
  229. data/ext/sources/ggml/src/ggml-cuda/argmax.cuh +3 -0
  230. data/ext/sources/ggml/src/ggml-cuda/argsort.cu +104 -0
  231. data/ext/sources/ggml/src/ggml-cuda/argsort.cuh +3 -0
  232. data/ext/sources/ggml/src/ggml-cuda/binbcast.cu +363 -0
  233. data/ext/sources/ggml/src/ggml-cuda/binbcast.cuh +9 -0
  234. data/ext/sources/ggml/src/ggml-cuda/clamp.cu +45 -0
  235. data/ext/sources/ggml/src/ggml-cuda/clamp.cuh +5 -0
  236. data/ext/sources/ggml/src/ggml-cuda/common.cuh +828 -0
  237. data/ext/sources/ggml/src/ggml-cuda/concat.cu +221 -0
  238. data/ext/sources/ggml/src/ggml-cuda/concat.cuh +5 -0
  239. data/ext/sources/ggml/src/ggml-cuda/conv-transpose-1d.cu +89 -0
  240. data/ext/sources/ggml/src/ggml-cuda/conv-transpose-1d.cuh +5 -0
  241. data/ext/sources/ggml/src/ggml-cuda/convert.cu +730 -0
  242. data/ext/sources/ggml/src/ggml-cuda/convert.cuh +26 -0
  243. data/ext/sources/ggml/src/ggml-cuda/count-equal.cu +64 -0
  244. data/ext/sources/ggml/src/ggml-cuda/count-equal.cuh +5 -0
  245. data/ext/sources/ggml/src/ggml-cuda/cp-async.cuh +57 -0
  246. data/ext/sources/ggml/src/ggml-cuda/cpy.cu +705 -0
  247. data/ext/sources/ggml/src/ggml-cuda/cpy.cuh +11 -0
  248. data/ext/sources/ggml/src/ggml-cuda/cross-entropy-loss.cu +189 -0
  249. data/ext/sources/ggml/src/ggml-cuda/cross-entropy-loss.cuh +7 -0
  250. data/ext/sources/ggml/src/ggml-cuda/dequantize.cuh +103 -0
  251. data/ext/sources/ggml/src/ggml-cuda/diagmask.cu +40 -0
  252. data/ext/sources/ggml/src/ggml-cuda/diagmask.cuh +5 -0
  253. data/ext/sources/ggml/src/ggml-cuda/fattn-common.cuh +881 -0
  254. data/ext/sources/ggml/src/ggml-cuda/fattn-mma-f16.cuh +1471 -0
  255. data/ext/sources/ggml/src/ggml-cuda/fattn-tile-f16.cu +357 -0
  256. data/ext/sources/ggml/src/ggml-cuda/fattn-tile-f16.cuh +3 -0
  257. data/ext/sources/ggml/src/ggml-cuda/fattn-tile-f32.cu +365 -0
  258. data/ext/sources/ggml/src/ggml-cuda/fattn-tile-f32.cuh +3 -0
  259. data/ext/sources/ggml/src/ggml-cuda/fattn-vec-f16.cuh +482 -0
  260. data/ext/sources/ggml/src/ggml-cuda/fattn-vec-f32.cuh +472 -0
  261. data/ext/sources/ggml/src/ggml-cuda/fattn-wmma-f16.cu +634 -0
  262. data/ext/sources/ggml/src/ggml-cuda/fattn-wmma-f16.cuh +3 -0
  263. data/ext/sources/ggml/src/ggml-cuda/fattn.cu +346 -0
  264. data/ext/sources/ggml/src/ggml-cuda/fattn.cuh +3 -0
  265. data/ext/sources/ggml/src/ggml-cuda/getrows.cu +275 -0
  266. data/ext/sources/ggml/src/ggml-cuda/getrows.cuh +15 -0
  267. data/ext/sources/ggml/src/ggml-cuda/ggml-cuda.cu +3505 -0
  268. data/ext/sources/ggml/src/ggml-cuda/gla.cu +93 -0
  269. data/ext/sources/ggml/src/ggml-cuda/gla.cuh +3 -0
  270. data/ext/sources/ggml/src/ggml-cuda/im2col.cu +103 -0
  271. data/ext/sources/ggml/src/ggml-cuda/im2col.cuh +5 -0
  272. data/ext/sources/ggml/src/ggml-cuda/mma.cuh +396 -0
  273. data/ext/sources/ggml/src/ggml-cuda/mmq.cu +324 -0
  274. data/ext/sources/ggml/src/ggml-cuda/mmq.cuh +3217 -0
  275. data/ext/sources/ggml/src/ggml-cuda/mmv.cu +336 -0
  276. data/ext/sources/ggml/src/ggml-cuda/mmv.cuh +12 -0
  277. data/ext/sources/ggml/src/ggml-cuda/mmvq.cu +595 -0
  278. data/ext/sources/ggml/src/ggml-cuda/mmvq.cuh +12 -0
  279. data/ext/sources/ggml/src/ggml-cuda/norm.cu +458 -0
  280. data/ext/sources/ggml/src/ggml-cuda/norm.cuh +11 -0
  281. data/ext/sources/ggml/src/ggml-cuda/opt-step-adamw.cu +78 -0
  282. data/ext/sources/ggml/src/ggml-cuda/opt-step-adamw.cuh +5 -0
  283. data/ext/sources/ggml/src/ggml-cuda/out-prod.cu +68 -0
  284. data/ext/sources/ggml/src/ggml-cuda/out-prod.cuh +3 -0
  285. data/ext/sources/ggml/src/ggml-cuda/pad.cu +49 -0
  286. data/ext/sources/ggml/src/ggml-cuda/pad.cuh +5 -0
  287. data/ext/sources/ggml/src/ggml-cuda/pool2d.cu +94 -0
  288. data/ext/sources/ggml/src/ggml-cuda/pool2d.cuh +5 -0
  289. data/ext/sources/ggml/src/ggml-cuda/quantize.cu +190 -0
  290. data/ext/sources/ggml/src/ggml-cuda/quantize.cuh +27 -0
  291. data/ext/sources/ggml/src/ggml-cuda/rope.cu +456 -0
  292. data/ext/sources/ggml/src/ggml-cuda/rope.cuh +7 -0
  293. data/ext/sources/ggml/src/ggml-cuda/scale.cu +31 -0
  294. data/ext/sources/ggml/src/ggml-cuda/scale.cuh +5 -0
  295. data/ext/sources/ggml/src/ggml-cuda/softmax.cu +283 -0
  296. data/ext/sources/ggml/src/ggml-cuda/softmax.cuh +7 -0
  297. data/ext/sources/ggml/src/ggml-cuda/ssm-conv.cu +148 -0
  298. data/ext/sources/ggml/src/ggml-cuda/ssm-conv.cuh +3 -0
  299. data/ext/sources/ggml/src/ggml-cuda/ssm-scan.cu +153 -0
  300. data/ext/sources/ggml/src/ggml-cuda/ssm-scan.cuh +3 -0
  301. data/ext/sources/ggml/src/ggml-cuda/sum.cu +45 -0
  302. data/ext/sources/ggml/src/ggml-cuda/sum.cuh +5 -0
  303. data/ext/sources/ggml/src/ggml-cuda/sumrows.cu +39 -0
  304. data/ext/sources/ggml/src/ggml-cuda/sumrows.cuh +5 -0
  305. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_16.cu +5 -0
  306. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_8.cu +10 -0
  307. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_1.cu +10 -0
  308. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_2.cu +10 -0
  309. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_4.cu +10 -0
  310. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_16.cu +5 -0
  311. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_4.cu +10 -0
  312. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_8.cu +10 -0
  313. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_1.cu +10 -0
  314. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_2.cu +10 -0
  315. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_16.cu +5 -0
  316. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_2.cu +10 -0
  317. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_4.cu +10 -0
  318. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_8.cu +10 -0
  319. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_64-ncols2_1.cu +10 -0
  320. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_1.cu +10 -0
  321. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_2.cu +10 -0
  322. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_4.cu +10 -0
  323. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_8.cu +10 -0
  324. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-f16.cu +5 -0
  325. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q4_0.cu +5 -0
  326. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q4_1.cu +5 -0
  327. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q5_0.cu +5 -0
  328. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q5_1.cu +5 -0
  329. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q8_0.cu +5 -0
  330. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-f16.cu +5 -0
  331. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q4_0.cu +5 -0
  332. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q4_1.cu +5 -0
  333. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q5_0.cu +5 -0
  334. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q5_1.cu +5 -0
  335. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q8_0.cu +5 -0
  336. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-f16.cu +5 -0
  337. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q4_0.cu +5 -0
  338. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q4_1.cu +5 -0
  339. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q5_0.cu +5 -0
  340. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q5_1.cu +5 -0
  341. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q8_0.cu +5 -0
  342. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-f16.cu +5 -0
  343. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q4_0.cu +5 -0
  344. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q4_1.cu +5 -0
  345. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q5_0.cu +5 -0
  346. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q5_1.cu +5 -0
  347. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q8_0.cu +5 -0
  348. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-f16.cu +5 -0
  349. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q4_0.cu +5 -0
  350. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q4_1.cu +5 -0
  351. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q5_0.cu +5 -0
  352. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q5_1.cu +5 -0
  353. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q8_0.cu +5 -0
  354. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-f16.cu +5 -0
  355. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q4_0.cu +5 -0
  356. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q4_1.cu +5 -0
  357. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q5_0.cu +5 -0
  358. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q5_1.cu +5 -0
  359. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q8_0.cu +5 -0
  360. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs256-f16-f16.cu +5 -0
  361. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-f16.cu +5 -0
  362. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q4_0.cu +5 -0
  363. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q4_1.cu +5 -0
  364. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q5_0.cu +5 -0
  365. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q5_1.cu +5 -0
  366. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q8_0.cu +5 -0
  367. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-f16.cu +5 -0
  368. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q4_0.cu +5 -0
  369. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q4_1.cu +5 -0
  370. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q5_0.cu +5 -0
  371. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q5_1.cu +5 -0
  372. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q8_0.cu +5 -0
  373. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-f16.cu +5 -0
  374. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q4_0.cu +5 -0
  375. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q4_1.cu +5 -0
  376. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q5_0.cu +5 -0
  377. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q5_1.cu +5 -0
  378. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q8_0.cu +5 -0
  379. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-f16.cu +5 -0
  380. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q4_0.cu +5 -0
  381. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q4_1.cu +5 -0
  382. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q5_0.cu +5 -0
  383. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q5_1.cu +5 -0
  384. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q8_0.cu +5 -0
  385. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-f16.cu +5 -0
  386. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q4_0.cu +5 -0
  387. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q4_1.cu +5 -0
  388. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q5_0.cu +5 -0
  389. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q5_1.cu +5 -0
  390. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q8_0.cu +5 -0
  391. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-f16.cu +5 -0
  392. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q4_0.cu +5 -0
  393. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q4_1.cu +5 -0
  394. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q5_0.cu +5 -0
  395. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q5_1.cu +5 -0
  396. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q8_0.cu +5 -0
  397. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-f16.cu +5 -0
  398. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q4_0.cu +5 -0
  399. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q4_1.cu +5 -0
  400. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q5_0.cu +5 -0
  401. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q5_1.cu +5 -0
  402. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q8_0.cu +5 -0
  403. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs256-f16-f16.cu +5 -0
  404. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-f16.cu +5 -0
  405. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q4_0.cu +5 -0
  406. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q4_1.cu +5 -0
  407. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q5_0.cu +5 -0
  408. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q5_1.cu +5 -0
  409. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q8_0.cu +5 -0
  410. data/ext/sources/ggml/src/ggml-cuda/template-instances/generate_cu_files.py +78 -0
  411. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq1_s.cu +5 -0
  412. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_s.cu +5 -0
  413. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xs.cu +5 -0
  414. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xxs.cu +5 -0
  415. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_s.cu +5 -0
  416. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_xxs.cu +5 -0
  417. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_nl.cu +5 -0
  418. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_xs.cu +5 -0
  419. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_k.cu +5 -0
  420. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q3_k.cu +5 -0
  421. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_0.cu +5 -0
  422. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_1.cu +5 -0
  423. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_k.cu +5 -0
  424. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_0.cu +5 -0
  425. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_1.cu +5 -0
  426. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_k.cu +5 -0
  427. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q6_k.cu +5 -0
  428. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q8_0.cu +5 -0
  429. data/ext/sources/ggml/src/ggml-cuda/tsembd.cu +47 -0
  430. data/ext/sources/ggml/src/ggml-cuda/tsembd.cuh +5 -0
  431. data/ext/sources/ggml/src/ggml-cuda/unary.cu +289 -0
  432. data/ext/sources/ggml/src/ggml-cuda/unary.cuh +59 -0
  433. data/ext/sources/ggml/src/ggml-cuda/upscale.cu +51 -0
  434. data/ext/sources/ggml/src/ggml-cuda/upscale.cuh +5 -0
  435. data/ext/sources/ggml/src/ggml-cuda/vecdotq.cuh +1135 -0
  436. data/ext/{ggml → sources/ggml}/src/ggml-cuda/vendors/cuda.h +1 -0
  437. data/ext/{ggml → sources/ggml}/src/ggml-cuda/vendors/hip.h +57 -0
  438. data/ext/{ggml → sources/ggml}/src/ggml-cuda/vendors/musa.h +7 -1
  439. data/ext/sources/ggml/src/ggml-cuda/wkv.cu +199 -0
  440. data/ext/sources/ggml/src/ggml-cuda/wkv.cuh +7 -0
  441. data/ext/sources/ggml/src/ggml-hip/CMakeLists.txt +131 -0
  442. data/ext/{ggml → sources/ggml}/src/ggml-impl.h +64 -19
  443. data/ext/sources/ggml/src/ggml-kompute/CMakeLists.txt +166 -0
  444. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/common.comp +112 -0
  445. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_add.comp +58 -0
  446. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_addrow.comp +25 -0
  447. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_cpy_f16_f16.comp +52 -0
  448. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_cpy_f16_f32.comp +52 -0
  449. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_cpy_f32_f16.comp +52 -0
  450. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_cpy_f32_f32.comp +52 -0
  451. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_diagmask.comp +30 -0
  452. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_gelu.comp +22 -0
  453. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows.comp +17 -0
  454. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows_f16.comp +31 -0
  455. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows_f32.comp +31 -0
  456. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows_q4_0.comp +38 -0
  457. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows_q4_1.comp +39 -0
  458. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows_q6_k.comp +44 -0
  459. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul.comp +52 -0
  460. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_f16.comp +69 -0
  461. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_mat_f32.comp +51 -0
  462. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q4_0.comp +33 -0
  463. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q4_1.comp +35 -0
  464. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q4_k.comp +140 -0
  465. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q6_k.comp +106 -0
  466. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q8_0.comp +73 -0
  467. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mv_q_n.comp +52 -0
  468. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mv_q_n_pre.comp +28 -0
  469. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_norm.comp +84 -0
  470. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_relu.comp +21 -0
  471. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_rmsnorm.comp +53 -0
  472. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_rope_neox_f16.comp +52 -0
  473. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_rope_neox_f32.comp +52 -0
  474. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_rope_norm_f16.comp +52 -0
  475. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_rope_norm_f32.comp +52 -0
  476. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_scale.comp +19 -0
  477. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_scale_8.comp +23 -0
  478. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_silu.comp +22 -0
  479. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_softmax.comp +72 -0
  480. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/rope_common.comp +71 -0
  481. data/ext/sources/ggml/src/ggml-metal/CMakeLists.txt +120 -0
  482. data/ext/sources/ggml/src/ggml-metal/ggml-metal-impl.h +622 -0
  483. data/ext/{ggml → sources/ggml}/src/ggml-metal/ggml-metal.m +2178 -1064
  484. data/ext/{ggml → sources/ggml}/src/ggml-metal/ggml-metal.metal +1575 -1218
  485. data/ext/sources/ggml/src/ggml-musa/CMakeLists.txt +113 -0
  486. data/ext/sources/ggml/src/ggml-musa/mudnn.cu +112 -0
  487. data/ext/sources/ggml/src/ggml-musa/mudnn.cuh +12 -0
  488. data/ext/sources/ggml/src/ggml-opencl/CMakeLists.txt +96 -0
  489. data/ext/sources/ggml/src/ggml-opencl/ggml-opencl.cpp +5124 -0
  490. data/ext/sources/ggml/src/ggml-opencl/kernels/add.cl +83 -0
  491. data/ext/sources/ggml/src/ggml-opencl/kernels/clamp.cl +20 -0
  492. data/ext/sources/ggml/src/ggml-opencl/kernels/cpy.cl +184 -0
  493. data/ext/sources/ggml/src/ggml-opencl/kernels/cvt.cl +118 -0
  494. data/ext/sources/ggml/src/ggml-opencl/kernels/diag_mask_inf.cl +58 -0
  495. data/ext/sources/ggml/src/ggml-opencl/kernels/embed_kernel.py +26 -0
  496. data/ext/sources/ggml/src/ggml-opencl/kernels/gelu.cl +62 -0
  497. data/ext/sources/ggml/src/ggml-opencl/kernels/gemv_noshuffle.cl +268 -0
  498. data/ext/sources/ggml/src/ggml-opencl/kernels/gemv_noshuffle_general.cl +274 -0
  499. data/ext/sources/ggml/src/ggml-opencl/kernels/get_rows.cl +163 -0
  500. data/ext/sources/ggml/src/ggml-opencl/kernels/im2col_f16.cl +57 -0
  501. data/ext/sources/ggml/src/ggml-opencl/kernels/im2col_f32.cl +57 -0
  502. data/ext/sources/ggml/src/ggml-opencl/kernels/mul.cl +79 -0
  503. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mat_Ab_Bi_8x4.cl +139 -0
  504. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_f16_f16.cl +118 -0
  505. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32.cl +118 -0
  506. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_1row.cl +94 -0
  507. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_l4.cl +84 -0
  508. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_f32_f32.cl +118 -0
  509. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32.cl +192 -0
  510. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_16x_flat.cl +307 -0
  511. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_8x_flat.cl +265 -0
  512. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_8x_flat.cl +272 -0
  513. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_v.cl +254 -0
  514. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q6_k.cl +190 -0
  515. data/ext/sources/ggml/src/ggml-opencl/kernels/norm.cl +81 -0
  516. data/ext/sources/ggml/src/ggml-opencl/kernels/relu.cl +16 -0
  517. data/ext/sources/ggml/src/ggml-opencl/kernels/rms_norm.cl +96 -0
  518. data/ext/sources/ggml/src/ggml-opencl/kernels/rope.cl +721 -0
  519. data/ext/sources/ggml/src/ggml-opencl/kernels/scale.cl +16 -0
  520. data/ext/sources/ggml/src/ggml-opencl/kernels/silu.cl +30 -0
  521. data/ext/sources/ggml/src/ggml-opencl/kernels/softmax_4_f16.cl +87 -0
  522. data/ext/sources/ggml/src/ggml-opencl/kernels/softmax_4_f32.cl +87 -0
  523. data/ext/sources/ggml/src/ggml-opencl/kernels/softmax_f16.cl +86 -0
  524. data/ext/sources/ggml/src/ggml-opencl/kernels/softmax_f32.cl +86 -0
  525. data/ext/sources/ggml/src/ggml-opencl/kernels/transpose.cl +84 -0
  526. data/ext/{ggml → sources/ggml}/src/ggml-opt.cpp +373 -190
  527. data/ext/{ggml → sources/ggml}/src/ggml-quants.c +114 -120
  528. data/ext/sources/ggml/src/ggml-rpc/CMakeLists.txt +9 -0
  529. data/ext/{ggml → sources/ggml}/src/ggml-rpc/ggml-rpc.cpp +480 -73
  530. data/ext/sources/ggml/src/ggml-sycl/CMakeLists.txt +189 -0
  531. data/ext/sources/ggml/src/ggml-sycl/backend.hpp +37 -0
  532. data/ext/sources/ggml/src/ggml-sycl/binbcast.cpp +345 -0
  533. data/ext/sources/ggml/src/ggml-sycl/binbcast.hpp +39 -0
  534. data/ext/{ggml → sources/ggml}/src/ggml-sycl/common.cpp +20 -32
  535. data/ext/sources/ggml/src/ggml-sycl/common.hpp +589 -0
  536. data/ext/{ggml → sources/ggml}/src/ggml-sycl/concat.cpp +32 -33
  537. data/ext/sources/ggml/src/ggml-sycl/concat.hpp +20 -0
  538. data/ext/{ggml → sources/ggml}/src/ggml-sycl/conv.cpp +4 -2
  539. data/ext/sources/ggml/src/ggml-sycl/conv.hpp +20 -0
  540. data/ext/{ggml → sources/ggml}/src/ggml-sycl/convert.cpp +104 -28
  541. data/ext/sources/ggml/src/ggml-sycl/convert.hpp +34 -0
  542. data/ext/sources/ggml/src/ggml-sycl/cpy.cpp +700 -0
  543. data/ext/sources/ggml/src/ggml-sycl/cpy.hpp +11 -0
  544. data/ext/sources/ggml/src/ggml-sycl/dequantize.hpp +791 -0
  545. data/ext/{ggml → sources/ggml}/src/ggml-sycl/dmmv.cpp +156 -17
  546. data/ext/sources/ggml/src/ggml-sycl/dmmv.hpp +27 -0
  547. data/ext/sources/ggml/src/ggml-sycl/dpct/helper.hpp +2957 -0
  548. data/ext/sources/ggml/src/ggml-sycl/element_wise.cpp +1511 -0
  549. data/ext/sources/ggml/src/ggml-sycl/element_wise.hpp +75 -0
  550. data/ext/sources/ggml/src/ggml-sycl/gemm.hpp +99 -0
  551. data/ext/sources/ggml/src/ggml-sycl/getrows.cpp +309 -0
  552. data/ext/sources/ggml/src/ggml-sycl/getrows.hpp +20 -0
  553. data/ext/{ggml → sources/ggml}/src/ggml-sycl/ggml-sycl.cpp +1004 -1240
  554. data/ext/sources/ggml/src/ggml-sycl/gla.cpp +106 -0
  555. data/ext/sources/ggml/src/ggml-sycl/gla.hpp +8 -0
  556. data/ext/sources/ggml/src/ggml-sycl/im2col.cpp +136 -0
  557. data/ext/sources/ggml/src/ggml-sycl/im2col.hpp +21 -0
  558. data/ext/{ggml → sources/ggml}/src/ggml-sycl/mmq.cpp +0 -1
  559. data/ext/sources/ggml/src/ggml-sycl/mmq.hpp +33 -0
  560. data/ext/{ggml → sources/ggml}/src/ggml-sycl/mmvq.cpp +261 -166
  561. data/ext/sources/ggml/src/ggml-sycl/mmvq.hpp +27 -0
  562. data/ext/{ggml → sources/ggml}/src/ggml-sycl/norm.cpp +204 -81
  563. data/ext/sources/ggml/src/ggml-sycl/norm.hpp +26 -0
  564. data/ext/{ggml → sources/ggml}/src/ggml-sycl/outprod.cpp +8 -17
  565. data/ext/sources/ggml/src/ggml-sycl/outprod.hpp +10 -0
  566. data/ext/sources/ggml/src/ggml-sycl/presets.hpp +74 -0
  567. data/ext/sources/ggml/src/ggml-sycl/quants.hpp +83 -0
  568. data/ext/sources/ggml/src/ggml-sycl/rope.cpp +361 -0
  569. data/ext/sources/ggml/src/ggml-sycl/rope.hpp +20 -0
  570. data/ext/{ggml → sources/ggml}/src/ggml-sycl/softmax.cpp +35 -25
  571. data/ext/sources/ggml/src/ggml-sycl/softmax.hpp +20 -0
  572. data/ext/sources/ggml/src/ggml-sycl/sycl_hw.cpp +13 -0
  573. data/ext/sources/ggml/src/ggml-sycl/sycl_hw.hpp +23 -0
  574. data/ext/{ggml → sources/ggml}/src/ggml-sycl/tsembd.cpp +3 -3
  575. data/ext/sources/ggml/src/ggml-sycl/tsembd.hpp +20 -0
  576. data/ext/sources/ggml/src/ggml-sycl/vecdotq.hpp +1215 -0
  577. data/ext/sources/ggml/src/ggml-sycl/wkv.cpp +293 -0
  578. data/ext/sources/ggml/src/ggml-sycl/wkv.hpp +10 -0
  579. data/ext/sources/ggml/src/ggml-vulkan/CMakeLists.txt +196 -0
  580. data/ext/sources/ggml/src/ggml-vulkan/cmake/host-toolchain.cmake.in +15 -0
  581. data/ext/{ggml → sources/ggml}/src/ggml-vulkan/ggml-vulkan.cpp +3130 -1087
  582. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/CMakeLists.txt +39 -0
  583. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/acc.comp +29 -0
  584. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/add.comp +29 -0
  585. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/argmax.comp +51 -0
  586. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/argsort.comp +69 -0
  587. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/clamp.comp +17 -0
  588. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/concat.comp +41 -0
  589. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/contig_copy.comp +49 -0
  590. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/conv2d_dw.comp +105 -0
  591. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/copy.comp +23 -0
  592. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/copy_from_quant.comp +51 -0
  593. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/copy_to_quant.comp +242 -0
  594. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/cos.comp +17 -0
  595. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/count_equal.comp +31 -0
  596. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_f32.comp +20 -0
  597. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.comp +462 -0
  598. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.comp +699 -0
  599. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_head.comp +13 -0
  600. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_m.comp +42 -0
  601. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_s.comp +35 -0
  602. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_s.comp +44 -0
  603. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xs.comp +43 -0
  604. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xxs.comp +48 -0
  605. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_s.comp +39 -0
  606. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_xxs.comp +49 -0
  607. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_nl.comp +32 -0
  608. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_xs.comp +34 -0
  609. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q2_k.comp +34 -0
  610. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q3_k.comp +42 -0
  611. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_0.comp +30 -0
  612. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_1.comp +32 -0
  613. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_k.comp +68 -0
  614. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_0.comp +34 -0
  615. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_1.comp +35 -0
  616. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_k.comp +70 -0
  617. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q6_k.comp +33 -0
  618. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q8_0.comp +31 -0
  619. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/diag_mask_inf.comp +34 -0
  620. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/div.comp +27 -0
  621. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp +337 -0
  622. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.comp +162 -0
  623. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp +360 -0
  624. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp +267 -0
  625. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_split_k_reduce.comp +59 -0
  626. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/gelu.comp +25 -0
  627. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/gelu_quick.comp +23 -0
  628. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/generic_binary_head.comp +64 -0
  629. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/generic_head.comp +9 -0
  630. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/generic_unary_head.comp +76 -0
  631. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/get_rows.comp +33 -0
  632. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_quant.comp +41 -0
  633. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/group_norm.comp +66 -0
  634. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/im2col.comp +100 -0
  635. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/l2_norm.comp +41 -0
  636. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/leaky_relu.comp +22 -0
  637. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul.comp +27 -0
  638. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_split_k_reduce.comp +48 -0
  639. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec.comp +169 -0
  640. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_base.comp +118 -0
  641. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_m.comp +82 -0
  642. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_s.comp +79 -0
  643. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_s.comp +90 -0
  644. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xs.comp +87 -0
  645. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xxs.comp +87 -0
  646. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_s.comp +90 -0
  647. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_xxs.comp +88 -0
  648. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_nc.comp +118 -0
  649. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_p021.comp +154 -0
  650. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q2_k.comp +130 -0
  651. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q3_k.comp +132 -0
  652. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q4_k.comp +136 -0
  653. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q5_k.comp +167 -0
  654. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q6_k.comp +130 -0
  655. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp +868 -0
  656. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp +441 -0
  657. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp +442 -0
  658. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_funcs.comp +99 -0
  659. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/norm.comp +44 -0
  660. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_adamw.comp +42 -0
  661. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/pad.comp +28 -0
  662. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/pool2d.comp +74 -0
  663. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/quantize_q8_1.comp +77 -0
  664. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/relu.comp +21 -0
  665. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/repeat.comp +26 -0
  666. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/repeat_back.comp +37 -0
  667. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp +52 -0
  668. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_back.comp +55 -0
  669. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rope_head.comp +58 -0
  670. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rope_multi.comp +60 -0
  671. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rope_neox.comp +43 -0
  672. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rope_norm.comp +43 -0
  673. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rope_vision.comp +47 -0
  674. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/scale.comp +24 -0
  675. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/sigmoid.comp +20 -0
  676. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/silu.comp +22 -0
  677. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/silu_back.comp +26 -0
  678. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/sin.comp +17 -0
  679. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/soft_max.comp +173 -0
  680. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_back.comp +50 -0
  681. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/square.comp +17 -0
  682. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/sub.comp +29 -0
  683. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.comp +37 -0
  684. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/tanh.comp +20 -0
  685. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/test_bfloat16_support.comp +7 -0
  686. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/test_coopmat2_support.comp +7 -0
  687. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/test_coopmat_support.comp +7 -0
  688. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/test_integer_dot_support.comp +7 -0
  689. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/timestep_embedding.comp +41 -0
  690. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/types.comp +1373 -0
  691. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/upscale.comp +36 -0
  692. data/ext/{ggml → sources/ggml}/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +193 -35
  693. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/wkv6.comp +87 -0
  694. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/wkv7.comp +91 -0
  695. data/ext/{ggml → sources/ggml}/src/ggml.c +676 -1820
  696. data/ext/sources/ggml/src/gguf.cpp +1330 -0
  697. data/ext/{include → sources/include}/whisper.h +68 -2
  698. data/ext/sources/src/CMakeLists.txt +143 -0
  699. data/ext/{src → sources/src}/coreml/whisper-decoder-impl.h +27 -15
  700. data/ext/{src → sources/src}/coreml/whisper-decoder-impl.m +35 -10
  701. data/ext/{src → sources/src}/coreml/whisper-encoder-impl.h +21 -9
  702. data/ext/{src → sources/src}/coreml/whisper-encoder-impl.m +28 -3
  703. data/ext/sources/src/coreml/whisper-encoder.mm +73 -0
  704. data/ext/sources/src/whisper-arch.h +197 -0
  705. data/ext/{src → sources/src}/whisper.cpp +1905 -374
  706. data/ext/sources/tests/CMakeLists.txt +105 -0
  707. data/ext/sources/tests/earnings21/eval.mk +58 -0
  708. data/ext/sources/tests/earnings21/eval.py +68 -0
  709. data/ext/sources/tests/earnings21/normalizers/__init__.py +2 -0
  710. data/ext/sources/tests/earnings21/normalizers/basic.py +80 -0
  711. data/ext/sources/tests/earnings21/normalizers/english.json +1741 -0
  712. data/ext/sources/tests/earnings21/normalizers/english.py +550 -0
  713. data/ext/sources/tests/earnings21/requirements.txt +6 -0
  714. data/ext/sources/tests/en-0-ref.txt +1 -0
  715. data/ext/sources/tests/en-1-ref.txt +1 -0
  716. data/ext/sources/tests/en-2-ref.txt +1 -0
  717. data/ext/sources/tests/es-0-ref.txt +1 -0
  718. data/ext/sources/tests/librispeech/eval.mk +39 -0
  719. data/ext/sources/tests/librispeech/eval.py +47 -0
  720. data/ext/sources/tests/librispeech/normalizers/__init__.py +2 -0
  721. data/ext/sources/tests/librispeech/normalizers/basic.py +80 -0
  722. data/ext/sources/tests/librispeech/normalizers/english.json +1741 -0
  723. data/ext/sources/tests/librispeech/normalizers/english.py +550 -0
  724. data/ext/sources/tests/librispeech/requirements.txt +6 -0
  725. data/ext/sources/tests/run-tests.sh +130 -0
  726. data/ext/sources/tests/test-c.c +3 -0
  727. data/ext/sources/tests/test-vad-full.cpp +54 -0
  728. data/ext/sources/tests/test-vad.cpp +83 -0
  729. data/ext/sources/tests/test-whisper.js +58 -0
  730. data/extsources.rb +33 -5
  731. data/lib/whisper/model/uri.rb +149 -128
  732. data/sig/whisper.rbs +480 -0
  733. data/tests/helper.rb +28 -0
  734. data/tests/test_callback.rb +45 -3
  735. data/tests/test_error.rb +2 -2
  736. data/tests/test_model.rb +38 -0
  737. data/tests/test_package.rb +18 -3
  738. data/tests/test_params.rb +145 -8
  739. data/tests/test_segment.rb +10 -19
  740. data/tests/test_vad.rb +19 -0
  741. data/tests/test_vad_params.rb +103 -0
  742. data/tests/test_whisper.rb +37 -37
  743. data/whispercpp.gemspec +5 -4
  744. metadata +766 -111
  745. data/ext/cpu.mk +0 -9
  746. data/ext/examples/dr_wav.h +0 -8815
  747. data/ext/ggml/src/ggml-cann/aclnn_ops.h +0 -592
  748. data/ext/ggml/src/ggml-cpu/ggml-cpu-aarch64.cpp +0 -4262
  749. data/ext/ggml/src/ggml-cpu/ggml-cpu.c +0 -14123
  750. data/ext/ggml/src/ggml-cpu/llamafile/sgemm.cpp +0 -1884
  751. data/ext/ggml/src/ggml-cpu/llamafile/sgemm.h +0 -14
  752. data/ext/ggml/src/ggml-metal/ggml-metal-impl.h +0 -288
  753. data/ext/ggml/src/ggml-sycl/element_wise.cpp +0 -1030
  754. data/ext/ggml/src/ggml-sycl/im2col.cpp +0 -126
  755. data/ext/ggml/src/ggml-sycl/rope.cpp +0 -276
  756. data/ext/ggml/src/ggml-sycl/wkv6.cpp +0 -141
  757. data/ext/metal-embed.mk +0 -17
  758. data/ext/metal.mk +0 -6
  759. data/ext/ruby_whisper.cpp +0 -1909
  760. data/ext/scripts/get-flags.mk +0 -38
  761. data/lib/whisper.rb +0 -2
  762. /data/ext/{ggml → sources/ggml}/include/ggml-blas.h +0 -0
  763. /data/ext/{ggml → sources/ggml}/include/ggml-cann.h +0 -0
  764. /data/ext/{ggml → sources/ggml}/include/ggml-cuda.h +0 -0
  765. /data/ext/{ggml → sources/ggml}/include/ggml-kompute.h +0 -0
  766. /data/ext/{ggml → sources/ggml}/include/ggml-opencl.h +0 -0
  767. /data/ext/{ggml → sources/ggml}/include/ggml-sycl.h +0 -0
  768. /data/ext/{ggml → sources/ggml}/src/ggml-amx/common.h +0 -0
  769. /data/ext/{ggml → sources/ggml}/src/ggml-amx/ggml-amx.cpp +0 -0
  770. /data/ext/{ggml → sources/ggml}/src/ggml-amx/mmq.cpp +0 -0
  771. /data/ext/{ggml → sources/ggml}/src/ggml-amx/mmq.h +0 -0
  772. /data/ext/{ggml → sources/ggml}/src/ggml-blas/ggml-blas.cpp +0 -0
  773. /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/ascendc_kernels.h +0 -0
  774. /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/get_row_f16.cpp +0 -0
  775. /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/get_row_f32.cpp +0 -0
  776. /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/get_row_q4_0.cpp +0 -0
  777. /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/get_row_q8_0.cpp +0 -0
  778. /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/quantize_f16_q8_0.cpp +0 -0
  779. /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/quantize_f32_q8_0.cpp +0 -0
  780. /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/quantize_float_to_q4_0.cpp +0 -0
  781. /data/ext/{ggml → sources/ggml}/src/ggml-cpu/amx/amx.h +0 -0
  782. /data/ext/{ggml → sources/ggml}/src/ggml-cpu/amx/common.h +0 -0
  783. /data/ext/{ggml → sources/ggml}/src/ggml-cpu/amx/mmq.cpp +0 -0
  784. /data/ext/{ggml → sources/ggml}/src/ggml-cpu/amx/mmq.h +0 -0
  785. /data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-aarch64.h +0 -0
  786. /data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-hbm.cpp +0 -0
  787. /data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-hbm.h +0 -0
  788. /data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-quants.h +0 -0
  789. /data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-traits.cpp +0 -0
  790. /data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-traits.h +0 -0
  791. /data/ext/{ggml → sources/ggml}/src/ggml-kompute/ggml-kompute.cpp +0 -0
  792. /data/ext/{ggml → sources/ggml}/src/ggml-quants.h +0 -0
  793. /data/ext/{ggml → sources/ggml}/src/ggml-threading.cpp +0 -0
  794. /data/ext/{ggml → sources/ggml}/src/ggml-threading.h +0 -0
  795. /data/ext/{src → sources/src}/coreml/whisper-encoder.h +0 -0
  796. /data/ext/{src → sources/src}/openvino/whisper-openvino-encoder.cpp +0 -0
  797. /data/ext/{src → sources/src}/openvino/whisper-openvino-encoder.h +0 -0
@@ -0,0 +1,1330 @@
1
+ #include "ggml.h"
2
+ #include "ggml-backend.h"
3
+ #include "ggml-impl.h"
4
+ #include "gguf.h"
5
+
6
+ #include <cinttypes>
7
+ #include <cstddef>
8
+ #include <cstdint>
9
+ #include <cstdio>
10
+ #include <cstdlib>
11
+ #include <cstring>
12
+ #include <map>
13
+ #include <new>
14
+ #include <stdexcept>
15
+ #include <string>
16
+ #include <vector>
17
+
18
+ template <typename T>
19
+ struct type_to_gguf_type;
20
+
21
+ template <>
22
+ struct type_to_gguf_type<uint8_t> {
23
+ static constexpr enum gguf_type value = GGUF_TYPE_UINT8;
24
+ };
25
+
26
+ template <>
27
+ struct type_to_gguf_type<int8_t> {
28
+ static constexpr enum gguf_type value = GGUF_TYPE_INT8;
29
+ };
30
+
31
+ template <>
32
+ struct type_to_gguf_type<uint16_t> {
33
+ static constexpr enum gguf_type value = GGUF_TYPE_UINT16;
34
+ };
35
+
36
+ template <>
37
+ struct type_to_gguf_type<int16_t> {
38
+ static constexpr enum gguf_type value = GGUF_TYPE_INT16;
39
+ };
40
+
41
+ template <>
42
+ struct type_to_gguf_type<uint32_t> {
43
+ static constexpr enum gguf_type value = GGUF_TYPE_UINT32;
44
+ };
45
+
46
+ template <>
47
+ struct type_to_gguf_type<int32_t> {
48
+ static constexpr enum gguf_type value = GGUF_TYPE_INT32;
49
+ };
50
+
51
+ template <>
52
+ struct type_to_gguf_type<float> {
53
+ static constexpr enum gguf_type value = GGUF_TYPE_FLOAT32;
54
+ };
55
+
56
+ template <>
57
+ struct type_to_gguf_type<bool> {
58
+ static constexpr enum gguf_type value = GGUF_TYPE_BOOL;
59
+ };
60
+
61
+ template <>
62
+ struct type_to_gguf_type<std::string> {
63
+ static constexpr enum gguf_type value = GGUF_TYPE_STRING;
64
+ };
65
+
66
+ template <>
67
+ struct type_to_gguf_type<uint64_t> {
68
+ static constexpr enum gguf_type value = GGUF_TYPE_UINT64;
69
+ };
70
+
71
+ template <>
72
+ struct type_to_gguf_type<int64_t> {
73
+ static constexpr enum gguf_type value = GGUF_TYPE_INT64;
74
+ };
75
+
76
+ template <>
77
+ struct type_to_gguf_type<double> {
78
+ static constexpr enum gguf_type value = GGUF_TYPE_FLOAT64;
79
+ };
80
+
81
+ static const std::map<gguf_type, size_t> GGUF_TYPE_SIZE = {
82
+ {GGUF_TYPE_UINT8, sizeof(uint8_t)},
83
+ {GGUF_TYPE_INT8, sizeof(int8_t)},
84
+ {GGUF_TYPE_UINT16, sizeof(uint16_t)},
85
+ {GGUF_TYPE_INT16, sizeof(int16_t)},
86
+ {GGUF_TYPE_UINT32, sizeof(uint32_t)},
87
+ {GGUF_TYPE_INT32, sizeof(int32_t)},
88
+ {GGUF_TYPE_FLOAT32, sizeof(float)},
89
+ {GGUF_TYPE_BOOL, sizeof(int8_t)},
90
+ {GGUF_TYPE_STRING, 0}, // undefined
91
+ {GGUF_TYPE_ARRAY, 0}, // undefined
92
+ {GGUF_TYPE_UINT64, sizeof(uint64_t)},
93
+ {GGUF_TYPE_INT64, sizeof(int64_t)},
94
+ {GGUF_TYPE_FLOAT64, sizeof(double)},
95
+ };
96
+ static_assert(GGUF_TYPE_COUNT == 13, "GGUF_TYPE_COUNT != 13");
97
+
98
+ static const std::map<gguf_type, const char *> GGUF_TYPE_NAME = {
99
+ {GGUF_TYPE_UINT8, "u8"},
100
+ {GGUF_TYPE_INT8, "i8"},
101
+ {GGUF_TYPE_UINT16, "u16"},
102
+ {GGUF_TYPE_INT16, "i16"},
103
+ {GGUF_TYPE_UINT32, "u32"},
104
+ {GGUF_TYPE_INT32, "i32"},
105
+ {GGUF_TYPE_FLOAT32, "f32"},
106
+ {GGUF_TYPE_BOOL, "bool"},
107
+ {GGUF_TYPE_STRING, "str"},
108
+ {GGUF_TYPE_ARRAY, "arr"},
109
+ {GGUF_TYPE_UINT64, "u64"},
110
+ {GGUF_TYPE_INT64, "i64"},
111
+ {GGUF_TYPE_FLOAT64, "f64"},
112
+ };
113
+ static_assert(GGUF_TYPE_COUNT == 13, "GGUF_TYPE_COUNT != 13");
114
+
115
+ size_t gguf_type_size(enum gguf_type type) {
116
+ auto it = GGUF_TYPE_SIZE.find(type);
117
+ return it == GGUF_TYPE_SIZE.end() ? 0 : it->second;
118
+ }
119
+
120
+ struct gguf_kv {
121
+ std::string key;
122
+
123
+ bool is_array;
124
+ enum gguf_type type;
125
+
126
+ std::vector<int8_t> data;
127
+ std::vector<std::string> data_string;
128
+
129
+ template <typename T>
130
+ gguf_kv(const std::string & key, const T value)
131
+ : key(key), is_array(false), type(type_to_gguf_type<T>::value) {
132
+ GGML_ASSERT(!key.empty());
133
+ data.resize(sizeof(T));
134
+ memcpy(data.data(), &value, sizeof(T));
135
+ }
136
+
137
+ template <typename T>
138
+ gguf_kv(const std::string & key, const std::vector<T> & value)
139
+ : key(key), is_array(true), type(type_to_gguf_type<T>::value) {
140
+ GGML_ASSERT(!key.empty());
141
+ data.resize(value.size()*sizeof(T));
142
+ for (size_t i = 0; i < value.size(); ++i) {
143
+ const T tmp = value[i];
144
+ memcpy(data.data() + i*sizeof(T), &tmp, sizeof(T));
145
+ }
146
+ }
147
+
148
+ gguf_kv(const std::string & key, const std::string & value)
149
+ : key(key), is_array(false), type(GGUF_TYPE_STRING) {
150
+ GGML_ASSERT(!key.empty());
151
+ data_string.push_back(value);
152
+ }
153
+
154
+ gguf_kv(const std::string & key, const std::vector<std::string> & value)
155
+ : key(key), is_array(true), type(GGUF_TYPE_STRING) {
156
+ GGML_ASSERT(!key.empty());
157
+ data_string = value;
158
+ }
159
+
160
+ const std::string & get_key() const {
161
+ return key;
162
+ }
163
+
164
+ const enum gguf_type & get_type() const {
165
+ return type;
166
+ }
167
+
168
+ size_t get_ne() const {
169
+ if (type == GGUF_TYPE_STRING) {
170
+ const size_t ne = data_string.size();
171
+ GGML_ASSERT(is_array || ne == 1);
172
+ return ne;
173
+ }
174
+ const size_t type_size = gguf_type_size(type);
175
+ GGML_ASSERT(data.size() % type_size == 0);
176
+ const size_t ne = data.size() / type_size;
177
+ GGML_ASSERT(is_array || ne == 1);
178
+ return ne;
179
+ }
180
+
181
+ template <typename T>
182
+ const T & get_val(const size_t i = 0) const {
183
+ GGML_ASSERT(type_to_gguf_type<T>::value == type);
184
+ if constexpr (std::is_same<T, std::string>::value) {
185
+ GGML_ASSERT(data_string.size() >= i+1);
186
+ return data_string[i];
187
+ }
188
+ const size_t type_size = gguf_type_size(type);
189
+ GGML_ASSERT(data.size() % type_size == 0);
190
+ GGML_ASSERT(data.size() >= (i+1)*type_size);
191
+ return reinterpret_cast<const T *>(data.data())[i];
192
+ }
193
+
194
+ void cast(const enum gguf_type new_type) {
195
+ const size_t new_type_size = gguf_type_size(new_type);
196
+ GGML_ASSERT(data.size() % new_type_size == 0);
197
+ type = new_type;
198
+ }
199
+ };
200
+
201
+ struct gguf_tensor_info {
202
+ struct ggml_tensor t; // for holding the equivalent info
203
+ uint64_t offset; // offset from start of `data`, must be a multiple of `ALIGNMENT`
204
+ };
205
+
206
+ struct gguf_context {
207
+ uint32_t version = GGUF_VERSION;
208
+
209
+ std::vector<struct gguf_kv> kv;
210
+ std::vector<struct gguf_tensor_info> info;
211
+
212
+ size_t alignment = GGUF_DEFAULT_ALIGNMENT;
213
+ size_t offset = 0; // offset of `data` from beginning of file
214
+ size_t size = 0; // size of `data` in bytes
215
+
216
+ void * data = nullptr;
217
+ };
218
+
219
+ struct gguf_reader {
220
+ FILE * file;
221
+
222
+ gguf_reader(FILE * file) : file(file) {}
223
+
224
+ template <typename T>
225
+ bool read(T & dst) const {
226
+ return fread(&dst, 1, sizeof(dst), file) == sizeof(dst);
227
+ }
228
+
229
+ template <typename T>
230
+ bool read(std::vector<T> & dst, const size_t n) const {
231
+ dst.resize(n);
232
+ for (size_t i = 0; i < dst.size(); ++i) {
233
+ if constexpr (std::is_same<T, bool>::value) {
234
+ bool tmp;
235
+ if (!read(tmp)) {
236
+ return false;
237
+ }
238
+ dst[i] = tmp;
239
+ } else {
240
+ if (!read(dst[i])) {
241
+ return false;
242
+ }
243
+ }
244
+ }
245
+ return true;
246
+ }
247
+
248
+ bool read(bool & dst) const {
249
+ int8_t tmp = -1;
250
+ if (!read(tmp)) {
251
+ return false;
252
+ }
253
+ dst = tmp != 0;
254
+ return true;
255
+ }
256
+
257
+ bool read(enum ggml_type & dst) const {
258
+ int32_t tmp = -1;
259
+ if (!read(tmp)) {
260
+ return false;
261
+ }
262
+ dst = ggml_type(tmp);
263
+ return true;
264
+ }
265
+
266
+ bool read(enum gguf_type & dst) const {
267
+ int32_t tmp = -1;
268
+ if (!read(tmp)) {
269
+ return false;
270
+ }
271
+ dst = gguf_type(tmp);
272
+ return true;
273
+ }
274
+
275
+ bool read(std::string & dst) const {
276
+ uint64_t size = -1;
277
+ if (!read(size)) {
278
+ return false;
279
+ }
280
+ dst.resize(size);
281
+ return fread(dst.data(), 1, dst.length(), file) == dst.length();
282
+ }
283
+
284
+ bool read(void * dst, const size_t size) const {
285
+ return fread(dst, 1, size, file) == size;
286
+ }
287
+ };
288
+
289
+ struct gguf_context * gguf_init_empty(void) {
290
+ return new gguf_context;
291
+ }
292
+
293
+ template<typename T>
294
+ bool gguf_read_emplace_helper(const struct gguf_reader & gr, std::vector<struct gguf_kv> & kv, const std::string & key, const bool is_array, const size_t n) {
295
+ if (is_array) {
296
+ std::vector<T> value;
297
+ try {
298
+ if (!gr.read(value, n)) {
299
+ return false;
300
+ }
301
+ } catch (std::length_error &) {
302
+ GGML_LOG_ERROR("%s: encountered length_error while reading value for key '%s'\n", __func__, key.c_str());
303
+ return false;
304
+ } catch (std::bad_alloc &) {
305
+ GGML_LOG_ERROR("%s: encountered bad_alloc error while reading value for key '%s'\n", __func__, key.c_str());
306
+ return false;
307
+ }
308
+ kv.emplace_back(key, value);
309
+ } else {
310
+ T value;
311
+ if (!gr.read(value)) {
312
+ return false;
313
+ }
314
+ kv.emplace_back(key, value);
315
+ }
316
+ return true;
317
+ }
318
+
319
+ struct gguf_context * gguf_init_from_file_impl(FILE * file, struct gguf_init_params params) {
320
+ const struct gguf_reader gr(file);
321
+ struct gguf_context * ctx = new gguf_context;
322
+
323
+ bool ok = true;
324
+
325
+ // file magic
326
+ {
327
+ std::vector<char> magic;
328
+ ok = ok && gr.read(magic, 4);
329
+
330
+ if (!ok) {
331
+ GGML_LOG_ERROR("%s: failed to read magic\n", __func__);
332
+ gguf_free(ctx);
333
+ return nullptr;
334
+ }
335
+
336
+ for (uint32_t i = 0; i < magic.size(); i++) {
337
+ if (magic[i] != GGUF_MAGIC[i]) {
338
+ GGML_LOG_ERROR("%s: invalid magic characters: '%c%c%c%c', expected 'GGUF'\n", __func__, magic[0], magic[1], magic[2], magic[3]);
339
+ gguf_free(ctx);
340
+ return nullptr;
341
+ }
342
+ }
343
+ }
344
+
345
+ // header
346
+ int64_t n_kv = 0;
347
+ int64_t n_tensors = 0;
348
+
349
+ if (ok && gr.read(ctx->version)) {
350
+ if (ctx->version == 1) {
351
+ GGML_LOG_ERROR("%s: GGUFv1 is no longer supported, please use a more up-to-date version\n", __func__);
352
+ ok = false;
353
+ }
354
+ if (ctx->version > GGUF_VERSION) {
355
+ GGML_LOG_ERROR("%s: this GGUF file is version %" PRIu32 " but this software only supports up to version %d\n",
356
+ __func__, ctx->version, GGUF_VERSION);
357
+ ok = false;
358
+ }
359
+ } else {
360
+ ok = false;
361
+ }
362
+
363
+ if (ok && gr.read(n_tensors)) {
364
+ static_assert(sizeof(size_t) <= 8 && sizeof(gguf_tensor_info) >= 2, "int64_t insufficient for indexing");
365
+ if (n_tensors < 0 || n_tensors > int64_t(SIZE_MAX/sizeof(gguf_tensor_info))) {
366
+ GGML_LOG_ERROR("%s: number of tensors is %" PRIi64 " but must be in [0, %zu]\n",
367
+ __func__, n_tensors, SIZE_MAX/sizeof(gguf_tensor_info));
368
+ ok = false;
369
+ }
370
+ } else {
371
+ ok = false;
372
+ }
373
+
374
+ if (ok && gr.read(n_kv)) {
375
+ static_assert(sizeof(size_t) <= 8 && sizeof(gguf_tensor_info) >= 2, "int64_t insufficient for indexing");
376
+ if (n_kv < 0 || n_kv > int64_t(SIZE_MAX/sizeof(gguf_kv))) {
377
+ GGML_LOG_ERROR("%s: number of key value pairs is %" PRIi64 " but must be in [0, %zu]\n",
378
+ __func__, n_kv, SIZE_MAX/sizeof(gguf_kv));
379
+ ok = false;
380
+ }
381
+ } else {
382
+ ok = false;
383
+ }
384
+
385
+ if (!ok) {
386
+ GGML_LOG_ERROR("%s: failed to read header\n", __func__);
387
+ gguf_free(ctx);
388
+ return nullptr;
389
+ }
390
+
391
+ // KV pairs
392
+ {
393
+ for (int64_t i = 0; ok && i < n_kv; ++i) {
394
+ std::string key;
395
+ gguf_type type = gguf_type(-1);
396
+ bool is_array = false;
397
+ uint64_t n = 1;
398
+
399
+ try {
400
+ ok = ok && gr.read(key);
401
+ } catch (std::length_error &) {
402
+ GGML_LOG_ERROR("%s: encountered length_error while reading key %" PRIi64 "\n", __func__, i);
403
+ ok = false;
404
+ } catch (std::bad_alloc &) {
405
+ GGML_LOG_ERROR("%s: encountered bad_alloc error while reading key %" PRIi64 "\n", __func__, i);
406
+ ok = false;
407
+ }
408
+ for (size_t j = 0; ok && j < ctx->kv.size(); ++j) {
409
+ if (key == ctx->kv[j].key) {
410
+ GGML_LOG_ERROR("%s: duplicate key '%s' for tensors %zu and %" PRIi64 " \n", __func__, key.c_str(), j, i);
411
+ ok = false;
412
+ }
413
+ }
414
+ if (!ok) {
415
+ break;
416
+ }
417
+
418
+ ok = ok && gr.read(type);
419
+ if (type == GGUF_TYPE_ARRAY) {
420
+ is_array = true;
421
+ ok = ok && gr.read(type);
422
+ ok = ok && gr.read(n);
423
+ }
424
+ if (!ok) {
425
+ break;
426
+ }
427
+
428
+ switch (type) {
429
+ case GGUF_TYPE_UINT8: ok = ok && gguf_read_emplace_helper<uint8_t> (gr, ctx->kv, key, is_array, n); break;
430
+ case GGUF_TYPE_INT8: ok = ok && gguf_read_emplace_helper<int8_t> (gr, ctx->kv, key, is_array, n); break;
431
+ case GGUF_TYPE_UINT16: ok = ok && gguf_read_emplace_helper<uint16_t> (gr, ctx->kv, key, is_array, n); break;
432
+ case GGUF_TYPE_INT16: ok = ok && gguf_read_emplace_helper<int16_t> (gr, ctx->kv, key, is_array, n); break;
433
+ case GGUF_TYPE_UINT32: ok = ok && gguf_read_emplace_helper<uint32_t> (gr, ctx->kv, key, is_array, n); break;
434
+ case GGUF_TYPE_INT32: ok = ok && gguf_read_emplace_helper<int32_t> (gr, ctx->kv, key, is_array, n); break;
435
+ case GGUF_TYPE_FLOAT32: ok = ok && gguf_read_emplace_helper<float> (gr, ctx->kv, key, is_array, n); break;
436
+ case GGUF_TYPE_BOOL: ok = ok && gguf_read_emplace_helper<bool> (gr, ctx->kv, key, is_array, n); break;
437
+ case GGUF_TYPE_STRING: ok = ok && gguf_read_emplace_helper<std::string>(gr, ctx->kv, key, is_array, n); break;
438
+ case GGUF_TYPE_UINT64: ok = ok && gguf_read_emplace_helper<uint64_t> (gr, ctx->kv, key, is_array, n); break;
439
+ case GGUF_TYPE_INT64: ok = ok && gguf_read_emplace_helper<int64_t> (gr, ctx->kv, key, is_array, n); break;
440
+ case GGUF_TYPE_FLOAT64: ok = ok && gguf_read_emplace_helper<double> (gr, ctx->kv, key, is_array, n); break;
441
+ case GGUF_TYPE_ARRAY:
442
+ default:
443
+ {
444
+ GGML_LOG_ERROR("%s: key '%s' has invalid GGUF type %d\n", __func__, key.c_str(), type);
445
+ ok = false;
446
+ } break;
447
+ }
448
+ }
449
+
450
+ if (!ok) {
451
+ GGML_LOG_ERROR("%s: failed to read key-value pairs\n", __func__);
452
+ gguf_free(ctx);
453
+ return nullptr;
454
+ }
455
+ GGML_ASSERT(int64_t(ctx->kv.size()) == n_kv);
456
+
457
+ const int alignment_idx = gguf_find_key(ctx, GGUF_KEY_GENERAL_ALIGNMENT);
458
+ ctx->alignment = alignment_idx == -1 ? GGUF_DEFAULT_ALIGNMENT : gguf_get_val_u32(ctx, alignment_idx);
459
+
460
+ if (ctx->alignment == 0 || (ctx->alignment & (ctx->alignment - 1)) != 0) {
461
+ GGML_LOG_ERROR("%s: alignment %zu is not a power of 2\n", __func__, ctx->alignment);
462
+ gguf_free(ctx);
463
+ return nullptr;
464
+ }
465
+ }
466
+
467
+ // read the tensor info
468
+ for (int64_t i = 0; ok && i < n_tensors; ++i) {
469
+ struct gguf_tensor_info info;
470
+
471
+ // tensor name
472
+ {
473
+ std::string name;
474
+ try {
475
+ ok = ok && gr.read(name);
476
+ } catch (std::length_error &) {
477
+ GGML_LOG_ERROR("%s: encountered length_error while reading tensor name %" PRIi64 "\n", __func__, i);
478
+ ok = false;
479
+ } catch (std::bad_alloc &) {
480
+ GGML_LOG_ERROR("%s: encountered bad_alloc error while reading tensor name %" PRIi64 "\n", __func__, i);
481
+ ok = false;
482
+ }
483
+ if (name.length() >= GGML_MAX_NAME) {
484
+ GGML_LOG_ERROR("%s: tensor name %" PRIi64 " is too long: %zu >= %d\n", __func__, i, name.length(), GGML_MAX_NAME);
485
+ ok = false;
486
+ break;
487
+ }
488
+ ggml_set_name(&info.t, name.c_str());
489
+
490
+ // make sure there are no duplicate tensor names
491
+ for (int64_t j = 0; ok && j < i; ++j) {
492
+ if (strcmp(info.t.name, ctx->info[j].t.name) == 0) {
493
+ GGML_LOG_ERROR("%s: duplicate tensor name '%s' for tensors %" PRIi64 " and %" PRIi64 "\n", __func__, info.t.name, j, i);
494
+ ok = false;
495
+ break;
496
+ }
497
+ }
498
+ }
499
+ if (!ok) {
500
+ break;
501
+ }
502
+
503
+ // tensor shape
504
+ {
505
+ uint32_t n_dims = -1;
506
+ ok = ok && gr.read(n_dims);
507
+ if (n_dims > GGML_MAX_DIMS) {
508
+ GGML_LOG_ERROR("%s: tensor '%s' has invalid number of dimensions: %" PRIu32 " > %" PRIu32 "\n",
509
+ __func__, info.t.name, n_dims, GGML_MAX_DIMS);
510
+ ok = false;
511
+ break;
512
+ }
513
+ for (uint32_t j = 0; ok && j < GGML_MAX_DIMS; ++j) {
514
+ info.t.ne[j] = 1;
515
+ if (j < n_dims) {
516
+ ok = ok && gr.read(info.t.ne[j]);
517
+ }
518
+
519
+ // check that all ne are non-negative
520
+ if (info.t.ne[j] < 0) {
521
+ GGML_LOG_ERROR("%s: tensor '%s' dimension %" PRIu32 " has invalid number of elements: %" PRIi64 " < 0\n",
522
+ __func__, info.t.name, j, info.t.ne[j]);
523
+ ok = false;
524
+ break;
525
+ }
526
+ }
527
+
528
+ // check that the total number of elements is representable
529
+ if (ok && ((INT64_MAX/info.t.ne[1] <= info.t.ne[0]) ||
530
+ (INT64_MAX/info.t.ne[2] <= info.t.ne[0]*info.t.ne[1]) ||
531
+ (INT64_MAX/info.t.ne[3] <= info.t.ne[0]*info.t.ne[1]*info.t.ne[2]))) {
532
+
533
+ GGML_LOG_ERROR("%s: total number of elements in tensor '%s' with shape "
534
+ "(%" PRIi64 ", %" PRIi64 ", %" PRIi64 ", %" PRIi64 ") is >= %" PRIi64 "\n",
535
+ __func__, info.t.name, info.t.ne[0], info.t.ne[1], info.t.ne[2], info.t.ne[3], INT64_MAX);
536
+ ok = false;
537
+ break;
538
+ }
539
+ }
540
+ if (!ok) {
541
+ break;
542
+ }
543
+
544
+ // tensor type
545
+ {
546
+ ok = ok && gr.read(info.t.type);
547
+
548
+ // check that tensor type is within defined range
549
+ if (info.t.type < 0 || info.t.type >= GGML_TYPE_COUNT) {
550
+ GGML_LOG_ERROR("%s: tensor '%s' has invalid ggml type %d (%s)\n",
551
+ __func__, info.t.name, info.t.type, ggml_type_name(info.t.type));
552
+ ok = false;
553
+ break;
554
+ }
555
+ const size_t type_size = ggml_type_size(info.t.type);
556
+ const int64_t blck_size = ggml_blck_size(info.t.type);
557
+
558
+ // check that row size is divisible by block size
559
+ if (blck_size == 0 || info.t.ne[0] % blck_size != 0) {
560
+ GGML_LOG_ERROR("%s: tensor '%s' of type %d (%s) has %" PRId64 " elements per row, "
561
+ "not a multiple of block size (%" PRId64 ")\n",
562
+ __func__, info.t.name, (int) info.t.type, ggml_type_name(info.t.type), info.t.ne[0], blck_size);
563
+ ok = false;
564
+ break;
565
+ }
566
+
567
+ // calculate byte offsets given the tensor shape and type
568
+ info.t.nb[0] = type_size;
569
+ info.t.nb[1] = info.t.nb[0]*(info.t.ne[0]/blck_size);
570
+ for (int j = 2; j < GGML_MAX_DIMS; ++j) {
571
+ info.t.nb[j] = info.t.nb[j - 1]*info.t.ne[j - 1];
572
+ }
573
+ }
574
+ if (!ok) {
575
+ break;
576
+ }
577
+
578
+ // tensor data offset within buffer
579
+ ok = ok && gr.read(info.offset);
580
+
581
+ ctx->info.push_back(info);
582
+ }
583
+
584
+ if (!ok) {
585
+ GGML_LOG_ERROR("%s: failed to read tensor info\n", __func__);
586
+ gguf_free(ctx);
587
+ return nullptr;
588
+ }
589
+ GGML_ASSERT(int64_t(ctx->info.size()) == n_tensors);
590
+
591
+ // we require the data section to be aligned, so take into account any padding
592
+ if (fseek(file, GGML_PAD(ftell(file), ctx->alignment), SEEK_SET) != 0) {
593
+ GGML_LOG_ERROR("%s: failed to seek to beginning of data section\n", __func__);
594
+ gguf_free(ctx);
595
+ return nullptr;
596
+ }
597
+
598
+ // store the current file offset - this is where the data section starts
599
+ ctx->offset = ftell(file);
600
+
601
+ // compute the total size of the data section, taking into account the alignment
602
+ {
603
+ ctx->size = 0;
604
+ for (size_t i = 0; i < ctx->info.size(); ++i) {
605
+ const gguf_tensor_info & ti = ctx->info[i];
606
+ if (ti.offset != ctx->size) {
607
+ GGML_LOG_ERROR("%s: tensor '%s' has offset %" PRIu64 ", expected %zu\n",
608
+ __func__, ti.t.name, ti.offset, ctx->size);
609
+ GGML_LOG_ERROR("%s: failed to read tensor data\n", __func__);
610
+ gguf_free(ctx);
611
+ return nullptr;
612
+ }
613
+ ctx->size += GGML_PAD(ggml_nbytes(&ti.t), ctx->alignment);
614
+ }
615
+ }
616
+
617
+ // load the tensor data only if requested
618
+ if (params.ctx != nullptr) {
619
+ // if the provided gguf_context is no_alloc, then we create "empty" tensors and do not read the binary blob
620
+ // otherwise, we load the binary blob into the created ggml_context as well, and point the "data" members of
621
+ // the ggml_tensor structs to the appropriate locations in the binary blob
622
+
623
+ // compute the exact size needed for the new ggml_context
624
+ const size_t mem_size =
625
+ params.no_alloc ?
626
+ (n_tensors )*ggml_tensor_overhead() :
627
+ (n_tensors + 1)*ggml_tensor_overhead() + ctx->size;
628
+
629
+ struct ggml_init_params pdata = {
630
+ /*mem_size =*/ mem_size,
631
+ /*mem_buffer =*/ nullptr,
632
+ /*no_alloc =*/ params.no_alloc,
633
+ };
634
+
635
+ *params.ctx = ggml_init(pdata);
636
+ if (*params.ctx == nullptr) {
637
+ GGML_LOG_ERROR("%s: failed to initialize ggml context for storing tensors\n", __func__);
638
+ gguf_free(ctx);
639
+ return nullptr;
640
+ }
641
+
642
+ struct ggml_context * ctx_data = *params.ctx;
643
+
644
+ struct ggml_tensor * data = nullptr;
645
+
646
+ if (!params.no_alloc) {
647
+ data = ggml_new_tensor_1d(ctx_data, GGML_TYPE_I8, ctx->size);
648
+
649
+ ok = ok && data != nullptr;
650
+
651
+ if (ok) {
652
+ ggml_set_name(data, "GGUF tensor data binary blob");
653
+ }
654
+
655
+ // read the binary blob with the tensor data
656
+ ok = ok && gr.read(data->data, ctx->size);
657
+
658
+ if (!ok) {
659
+ GGML_LOG_ERROR("%s: failed to read tensor data binary blob\n", __func__);
660
+ ggml_free(ctx_data);
661
+ *params.ctx = nullptr;
662
+ gguf_free(ctx);
663
+ return nullptr;
664
+ }
665
+
666
+ ctx->data = data->data;
667
+ }
668
+
669
+ ggml_set_no_alloc(ctx_data, true);
670
+
671
+ // create the tensors
672
+ for (size_t i = 0; i < ctx->info.size(); ++i) {
673
+ const struct gguf_tensor_info & info = ctx->info[i];
674
+
675
+ struct ggml_tensor * cur = ggml_new_tensor(ctx_data, info.t.type, GGML_MAX_DIMS, info.t.ne);
676
+
677
+ ok = ok && cur != nullptr;
678
+
679
+ if (!ok) {
680
+ break;
681
+ }
682
+
683
+ ggml_set_name(cur, info.t.name);
684
+
685
+ // point the data member to the appropriate location in the binary blob using the tensor info
686
+ if (!params.no_alloc) {
687
+ cur->data = (char *) data->data + info.offset;
688
+ }
689
+ }
690
+
691
+ if (!ok) {
692
+ GGML_LOG_ERROR("%s: failed to create tensors\n", __func__);
693
+ ggml_free(ctx_data);
694
+ *params.ctx = nullptr;
695
+ gguf_free(ctx);
696
+ return nullptr;
697
+ }
698
+
699
+ ggml_set_no_alloc(ctx_data, params.no_alloc);
700
+ }
701
+
702
+ return ctx;
703
+ }
704
+
705
+ struct gguf_context * gguf_init_from_file(const char * fname, struct gguf_init_params params) {
706
+ FILE * file = ggml_fopen(fname, "rb");
707
+
708
+ if (!file) {
709
+ GGML_LOG_ERROR("%s: failed to open GGUF file '%s'\n", __func__, fname);
710
+ return nullptr;
711
+ }
712
+
713
+ struct gguf_context * result = gguf_init_from_file_impl(file, params);
714
+ fclose(file);
715
+ return result;
716
+ }
717
+
718
+ void gguf_free(struct gguf_context * ctx) {
719
+ if (ctx == nullptr) {
720
+ return;
721
+ }
722
+ delete ctx;
723
+ }
724
+
725
+ const char * gguf_type_name(enum gguf_type type) {
726
+ auto it = GGUF_TYPE_NAME.find(type);
727
+ return it == GGUF_TYPE_NAME.end() ? nullptr : it->second;
728
+ }
729
+
730
+ uint32_t gguf_get_version(const struct gguf_context * ctx) {
731
+ return ctx->version;
732
+ }
733
+
734
+ size_t gguf_get_alignment(const struct gguf_context * ctx) {
735
+ return ctx->alignment;
736
+ }
737
+
738
+ size_t gguf_get_data_offset(const struct gguf_context * ctx) {
739
+ return ctx->offset;
740
+ }
741
+
742
+ int64_t gguf_get_n_kv(const struct gguf_context * ctx) {
743
+ return ctx->kv.size();
744
+ }
745
+
746
+ int64_t gguf_find_key(const struct gguf_context * ctx, const char * key) {
747
+ // return -1 if key not found
748
+ int64_t keyfound = -1;
749
+
750
+ const int64_t n_kv = gguf_get_n_kv(ctx);
751
+
752
+ for (int64_t i = 0; i < n_kv; ++i) {
753
+ if (strcmp(key, gguf_get_key(ctx, i)) == 0) {
754
+ keyfound = i;
755
+ break;
756
+ }
757
+ }
758
+
759
+ return keyfound;
760
+ }
761
+
762
+ const char * gguf_get_key(const struct gguf_context * ctx, int64_t key_id) {
763
+ GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
764
+ return ctx->kv[key_id].get_key().c_str();
765
+ }
766
+
767
+ enum gguf_type gguf_get_kv_type(const struct gguf_context * ctx, int64_t key_id) {
768
+ GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
769
+ return ctx->kv[key_id].is_array ? GGUF_TYPE_ARRAY : ctx->kv[key_id].get_type();
770
+ }
771
+
772
+ enum gguf_type gguf_get_arr_type(const struct gguf_context * ctx, int64_t key_id) {
773
+ GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
774
+ GGML_ASSERT(ctx->kv[key_id].is_array);
775
+ return ctx->kv[key_id].get_type();
776
+ }
777
+
778
+ const void * gguf_get_arr_data(const struct gguf_context * ctx, int64_t key_id) {
779
+ GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
780
+ GGML_ASSERT(ctx->kv[key_id].get_type() != GGUF_TYPE_STRING);
781
+ return ctx->kv[key_id].data.data();
782
+ }
783
+
784
+ const char * gguf_get_arr_str(const struct gguf_context * ctx, int64_t key_id, size_t i) {
785
+ GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
786
+ GGML_ASSERT(ctx->kv[key_id].get_type() == GGUF_TYPE_STRING);
787
+ return ctx->kv[key_id].data_string[i].c_str();
788
+ }
789
+
790
+ size_t gguf_get_arr_n(const struct gguf_context * ctx, int64_t key_id) {
791
+ GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
792
+
793
+ if (ctx->kv[key_id].type == GGUF_TYPE_STRING) {
794
+ return ctx->kv[key_id].data_string.size();
795
+ }
796
+
797
+ const size_t type_size = gguf_type_size(ctx->kv[key_id].type);
798
+ GGML_ASSERT(ctx->kv[key_id].data.size() % type_size == 0);
799
+ return ctx->kv[key_id].data.size() / type_size;
800
+ }
801
+
802
+ uint8_t gguf_get_val_u8(const struct gguf_context * ctx, int64_t key_id) {
803
+ GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
804
+ GGML_ASSERT(ctx->kv[key_id].get_ne() == 1);
805
+ return ctx->kv[key_id].get_val<uint8_t>();
806
+ }
807
+
808
+ int8_t gguf_get_val_i8(const struct gguf_context * ctx, int64_t key_id) {
809
+ GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
810
+ GGML_ASSERT(ctx->kv[key_id].get_ne() == 1);
811
+ return ctx->kv[key_id].get_val<int8_t>();
812
+ }
813
+
814
+ uint16_t gguf_get_val_u16(const struct gguf_context * ctx, int64_t key_id) {
815
+ GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
816
+ GGML_ASSERT(ctx->kv[key_id].get_ne() == 1);
817
+ return ctx->kv[key_id].get_val<uint16_t>();
818
+ }
819
+
820
+ int16_t gguf_get_val_i16(const struct gguf_context * ctx, int64_t key_id) {
821
+ GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
822
+ GGML_ASSERT(ctx->kv[key_id].get_ne() == 1);
823
+ return ctx->kv[key_id].get_val<int16_t>();
824
+ }
825
+
826
+ uint32_t gguf_get_val_u32(const struct gguf_context * ctx, int64_t key_id) {
827
+ GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
828
+ GGML_ASSERT(ctx->kv[key_id].get_ne() == 1);
829
+ return ctx->kv[key_id].get_val<uint32_t>();
830
+ }
831
+
832
+ int32_t gguf_get_val_i32(const struct gguf_context * ctx, int64_t key_id) {
833
+ GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
834
+ GGML_ASSERT(ctx->kv[key_id].get_ne() == 1);
835
+ return ctx->kv[key_id].get_val<int32_t>();
836
+ }
837
+
838
+ float gguf_get_val_f32(const struct gguf_context * ctx, int64_t key_id) {
839
+ GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
840
+ GGML_ASSERT(ctx->kv[key_id].get_ne() == 1);
841
+ return ctx->kv[key_id].get_val<float>();
842
+ }
843
+
844
+ uint64_t gguf_get_val_u64(const struct gguf_context * ctx, int64_t key_id) {
845
+ GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
846
+ GGML_ASSERT(ctx->kv[key_id].get_ne() == 1);
847
+ return ctx->kv[key_id].get_val<uint64_t>();
848
+ }
849
+
850
+ int64_t gguf_get_val_i64(const struct gguf_context * ctx, int64_t key_id) {
851
+ GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
852
+ GGML_ASSERT(ctx->kv[key_id].get_ne() == 1);
853
+ return ctx->kv[key_id].get_val<int64_t>();
854
+ }
855
+
856
+ double gguf_get_val_f64(const struct gguf_context * ctx, int64_t key_id) {
857
+ GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
858
+ GGML_ASSERT(ctx->kv[key_id].get_ne() == 1);
859
+ return ctx->kv[key_id].get_val<double>();
860
+ }
861
+
862
+ bool gguf_get_val_bool(const struct gguf_context * ctx, int64_t key_id) {
863
+ GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
864
+ GGML_ASSERT(ctx->kv[key_id].get_ne() == 1);
865
+ return ctx->kv[key_id].get_val<bool>();
866
+ }
867
+
868
+ const char * gguf_get_val_str(const struct gguf_context * ctx, int64_t key_id) {
869
+ GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
870
+ GGML_ASSERT(ctx->kv[key_id].get_ne() == 1);
871
+ return ctx->kv[key_id].get_val<std::string>().c_str();
872
+ }
873
+
874
+ const void * gguf_get_val_data(const struct gguf_context * ctx, int64_t key_id) {
875
+ GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
876
+ GGML_ASSERT(ctx->kv[key_id].get_ne() == 1);
877
+ GGML_ASSERT(ctx->kv[key_id].get_type() != GGUF_TYPE_STRING);
878
+ return ctx->kv[key_id].data.data();
879
+ }
880
+
881
+ int64_t gguf_get_n_tensors(const struct gguf_context * ctx) {
882
+ return ctx->info.size();
883
+ }
884
+
885
+ int64_t gguf_find_tensor(const struct gguf_context * ctx, const char * name) {
886
+ // return -1 if tensor not found
887
+ int64_t tensor_id = -1;
888
+
889
+ const int64_t n_tensors = gguf_get_n_tensors(ctx);
890
+
891
+ for (int64_t i = 0; i < n_tensors; ++i) {
892
+ if (strcmp(name, gguf_get_tensor_name(ctx, i)) == 0) {
893
+ tensor_id = i;
894
+ break;
895
+ }
896
+ }
897
+
898
+ return tensor_id;
899
+ }
900
+
901
+ size_t gguf_get_tensor_offset(const struct gguf_context * ctx, int64_t tensor_id) {
902
+ GGML_ASSERT(tensor_id >= 0 && tensor_id < gguf_get_n_tensors(ctx));
903
+ return ctx->info[tensor_id].offset;
904
+ }
905
+
906
+ const char * gguf_get_tensor_name(const struct gguf_context * ctx, int64_t tensor_id) {
907
+ GGML_ASSERT(tensor_id >= 0 && tensor_id < gguf_get_n_tensors(ctx));
908
+ return ctx->info[tensor_id].t.name;
909
+ }
910
+
911
+ enum ggml_type gguf_get_tensor_type(const struct gguf_context * ctx, int64_t tensor_id) {
912
+ GGML_ASSERT(tensor_id >= 0 && tensor_id < gguf_get_n_tensors(ctx));
913
+ return ctx->info[tensor_id].t.type;
914
+ }
915
+
916
+ size_t gguf_get_tensor_size(const struct gguf_context * ctx, int64_t tensor_id) {
917
+ GGML_ASSERT(tensor_id >= 0 && tensor_id < gguf_get_n_tensors(ctx));
918
+ return ggml_nbytes(&ctx->info[tensor_id].t);
919
+ }
920
+
921
+ int64_t gguf_remove_key(struct gguf_context * ctx, const char * key) {
922
+ const int64_t key_id = gguf_find_key(ctx, key);
923
+ if (key_id >= 0) {
924
+ ctx->kv.erase(ctx->kv.begin() + key_id);
925
+ }
926
+ return key_id;
927
+ }
928
+
929
+ template<typename T>
930
+ static void gguf_check_reserved_keys(const std::string & key, const T val) {
931
+ if (key == GGUF_KEY_GENERAL_ALIGNMENT) {
932
+ if constexpr (std::is_same<T, uint32_t>::value) {
933
+ GGML_ASSERT(val > 0 && (val & (val - 1)) == 0 && GGUF_KEY_GENERAL_ALIGNMENT " must be power of 2");
934
+ } else {
935
+ GGML_UNUSED(val);
936
+ GGML_ABORT(GGUF_KEY_GENERAL_ALIGNMENT " must be type u32");
937
+ }
938
+ }
939
+ }
940
+
941
+ void gguf_set_val_u8(struct gguf_context * ctx, const char * key, uint8_t val) {
942
+ gguf_check_reserved_keys(key, val);
943
+ gguf_remove_key(ctx, key);
944
+ ctx->kv.emplace_back(key, val);
945
+ }
946
+
947
+ void gguf_set_val_i8(struct gguf_context * ctx, const char * key, int8_t val) {
948
+ gguf_check_reserved_keys(key, val);
949
+ gguf_remove_key(ctx, key);
950
+ ctx->kv.emplace_back(key, val);
951
+ }
952
+
953
+ void gguf_set_val_u16(struct gguf_context * ctx, const char * key, uint16_t val) {
954
+ gguf_check_reserved_keys(key, val);
955
+ gguf_remove_key(ctx, key);
956
+ ctx->kv.emplace_back(key, val);
957
+ }
958
+
959
+ void gguf_set_val_i16(struct gguf_context * ctx, const char * key, int16_t val) {
960
+ gguf_check_reserved_keys(key, val);
961
+ gguf_remove_key(ctx, key);
962
+ ctx->kv.emplace_back(key, val);
963
+ }
964
+
965
+ void gguf_set_val_u32(struct gguf_context * ctx, const char * key, uint32_t val) {
966
+ gguf_check_reserved_keys(key, val);
967
+ gguf_remove_key(ctx, key);
968
+ ctx->kv.emplace_back(key, val);
969
+ }
970
+
971
+ void gguf_set_val_i32(struct gguf_context * ctx, const char * key, int32_t val) {
972
+ gguf_check_reserved_keys(key, val);
973
+ gguf_remove_key(ctx, key);
974
+ ctx->kv.emplace_back(key, val);
975
+ }
976
+
977
+ void gguf_set_val_f32(struct gguf_context * ctx, const char * key, float val) {
978
+ gguf_check_reserved_keys(key, val);
979
+ gguf_remove_key(ctx, key);
980
+ ctx->kv.emplace_back(key, val);
981
+ }
982
+
983
+ void gguf_set_val_u64(struct gguf_context * ctx, const char * key, uint64_t val) {
984
+ gguf_check_reserved_keys(key, val);
985
+ gguf_remove_key(ctx, key);
986
+ ctx->kv.emplace_back(key, val);
987
+ }
988
+
989
+ void gguf_set_val_i64(struct gguf_context * ctx, const char * key, int64_t val) {
990
+ gguf_check_reserved_keys(key, val);
991
+ gguf_remove_key(ctx, key);
992
+ ctx->kv.emplace_back(key, val);
993
+ }
994
+
995
+ void gguf_set_val_f64(struct gguf_context * ctx, const char * key, double val) {
996
+ gguf_check_reserved_keys(key, val);
997
+ gguf_remove_key(ctx, key);
998
+ ctx->kv.emplace_back(key, val);
999
+ }
1000
+
1001
+ void gguf_set_val_bool(struct gguf_context * ctx, const char * key, bool val) {
1002
+ gguf_check_reserved_keys(key, val);
1003
+ gguf_remove_key(ctx, key);
1004
+ ctx->kv.emplace_back(key, val);
1005
+ }
1006
+
1007
+ void gguf_set_val_str(struct gguf_context * ctx, const char * key, const char * val) {
1008
+ gguf_check_reserved_keys(key, val);
1009
+ gguf_remove_key(ctx, key);
1010
+ ctx->kv.emplace_back(key, std::string(val));
1011
+ }
1012
+
1013
+ void gguf_set_arr_data(struct gguf_context * ctx, const char * key, enum gguf_type type, const void * data, size_t n) {
1014
+ gguf_check_reserved_keys(key, data);
1015
+ gguf_remove_key(ctx, key);
1016
+
1017
+ const size_t nbytes = n*gguf_type_size(type);
1018
+ std::vector<int8_t> tmp(nbytes);
1019
+ if (!tmp.empty()) {
1020
+ memcpy(tmp.data(), data, nbytes);
1021
+ }
1022
+ ctx->kv.emplace_back(key, tmp);
1023
+ ctx->kv.back().cast(type);
1024
+ }
1025
+
1026
+ void gguf_set_arr_str(struct gguf_context * ctx, const char * key, const char ** data, size_t n) {
1027
+ gguf_check_reserved_keys(key, data);
1028
+ gguf_remove_key(ctx, key);
1029
+
1030
+ std::vector<std::string> tmp(n);
1031
+ for (size_t i = 0; i < n; ++i) {
1032
+ tmp[i] = data[i];
1033
+ }
1034
+ ctx->kv.emplace_back(key, tmp);
1035
+ }
1036
+
1037
+ // set or add KV pairs from another context
1038
+ void gguf_set_kv(struct gguf_context * ctx, const struct gguf_context * src) {
1039
+ const int64_t n_kv = gguf_get_n_kv(src);
1040
+ for (int64_t i = 0; i < n_kv; ++i) {
1041
+ const struct gguf_kv & kv = src->kv[i];
1042
+
1043
+ if (!kv.is_array) {
1044
+ switch (kv.get_type()) {
1045
+ case GGUF_TYPE_UINT8: gguf_set_val_u8 (ctx, kv.get_key().c_str(), kv.get_val<uint8_t>()); break;
1046
+ case GGUF_TYPE_INT8: gguf_set_val_i8 (ctx, kv.get_key().c_str(), kv.get_val<int8_t>()); break;
1047
+ case GGUF_TYPE_UINT16: gguf_set_val_u16 (ctx, kv.get_key().c_str(), kv.get_val<uint16_t>()); break;
1048
+ case GGUF_TYPE_INT16: gguf_set_val_i16 (ctx, kv.get_key().c_str(), kv.get_val<int16_t>()); break;
1049
+ case GGUF_TYPE_UINT32: gguf_set_val_u32 (ctx, kv.get_key().c_str(), kv.get_val<uint32_t>()); break;
1050
+ case GGUF_TYPE_INT32: gguf_set_val_i32 (ctx, kv.get_key().c_str(), kv.get_val<int32_t>()); break;
1051
+ case GGUF_TYPE_FLOAT32: gguf_set_val_f32 (ctx, kv.get_key().c_str(), kv.get_val<float>()); break;
1052
+ case GGUF_TYPE_UINT64: gguf_set_val_u64 (ctx, kv.get_key().c_str(), kv.get_val<uint64_t>()); break;
1053
+ case GGUF_TYPE_INT64: gguf_set_val_i64 (ctx, kv.get_key().c_str(), kv.get_val<int64_t>()); break;
1054
+ case GGUF_TYPE_FLOAT64: gguf_set_val_f64 (ctx, kv.get_key().c_str(), kv.get_val<double>()); break;
1055
+ case GGUF_TYPE_BOOL: gguf_set_val_bool(ctx, kv.get_key().c_str(), kv.get_val<bool>()); break;
1056
+ case GGUF_TYPE_STRING: gguf_set_val_str (ctx, kv.get_key().c_str(), kv.get_val<std::string>().c_str()); break;
1057
+ case GGUF_TYPE_ARRAY:
1058
+ default: GGML_ABORT("invalid type");
1059
+ }
1060
+ continue;
1061
+ }
1062
+
1063
+ const size_t ne = kv.get_ne();
1064
+
1065
+ switch (kv.get_type()) {
1066
+ case GGUF_TYPE_UINT8:
1067
+ case GGUF_TYPE_INT8:
1068
+ case GGUF_TYPE_UINT16:
1069
+ case GGUF_TYPE_INT16:
1070
+ case GGUF_TYPE_UINT32:
1071
+ case GGUF_TYPE_INT32:
1072
+ case GGUF_TYPE_FLOAT32:
1073
+ case GGUF_TYPE_UINT64:
1074
+ case GGUF_TYPE_INT64:
1075
+ case GGUF_TYPE_FLOAT64:
1076
+ case GGUF_TYPE_BOOL: {
1077
+ gguf_set_arr_data(ctx, kv.get_key().c_str(), kv.get_type(), kv.data.data(), ne);
1078
+ } break;
1079
+ case GGUF_TYPE_STRING: {
1080
+ std::vector<const char *> tmp(ne);
1081
+ for (size_t j = 0; j < ne; ++j) {
1082
+ tmp[j] = kv.data_string[j].c_str();
1083
+ }
1084
+ gguf_set_arr_str(ctx, kv.get_key().c_str(), tmp.data(), ne);
1085
+ } break;
1086
+ case GGUF_TYPE_ARRAY:
1087
+ default: GGML_ABORT("invalid type");
1088
+ }
1089
+ }
1090
+ }
1091
+
1092
+ void gguf_add_tensor(
1093
+ struct gguf_context * ctx,
1094
+ const struct ggml_tensor * tensor) {
1095
+ GGML_ASSERT(tensor);
1096
+ if (gguf_find_tensor(ctx, tensor->name) != -1) {
1097
+ GGML_ABORT("duplicate tensor name: %s", tensor->name);
1098
+ }
1099
+
1100
+ struct gguf_tensor_info ti;
1101
+ ti.t = *tensor;
1102
+ ti.offset = ctx->info.empty() ? 0 :
1103
+ ctx->info.back().offset + GGML_PAD(ggml_nbytes(&ctx->info.back().t), ctx->alignment);
1104
+ ctx->info.push_back(ti);
1105
+ }
1106
+
1107
+ void gguf_set_tensor_type(struct gguf_context * ctx, const char * name, enum ggml_type type) {
1108
+ const int64_t tensor_id = gguf_find_tensor(ctx, name);
1109
+ if (tensor_id < 0) {
1110
+ GGML_ABORT("tensor not found: %s", name);
1111
+ }
1112
+ struct ggml_tensor * tensor = &ctx->info[tensor_id].t;
1113
+ const size_t type_size = ggml_type_size(type);
1114
+ const int64_t blck_size = ggml_blck_size(type);
1115
+
1116
+ tensor->type = type;
1117
+ GGML_ASSERT(tensor->ne[0] % blck_size == 0 && "tensor row size not divisible by block size of new type");
1118
+
1119
+ tensor->nb[0] = type_size;
1120
+ tensor->nb[1] = tensor->nb[0]*(tensor->ne[0]/blck_size);
1121
+ for (int i = 2; i < GGML_MAX_DIMS; i++) {
1122
+ tensor->nb[i] = tensor->nb[i - 1]*tensor->ne[i - 1];
1123
+ }
1124
+
1125
+ // update offsets
1126
+ const int64_t n_tensors = gguf_get_n_tensors(ctx);
1127
+ for (int64_t i = tensor_id + 1; i < n_tensors; ++i) {
1128
+ ctx->info[i].offset = ctx->info[i - 1].offset + GGML_PAD(ggml_nbytes(&ctx->info[i - 1].t), ctx->alignment);
1129
+ }
1130
+ }
1131
+
1132
+ void gguf_set_tensor_data(struct gguf_context * ctx, const char * name, const void * data) {
1133
+ const int64_t tensor_id = gguf_find_tensor(ctx, name);
1134
+ if (tensor_id < 0) {
1135
+ GGML_ABORT("tensor not found: %s", name);
1136
+ }
1137
+
1138
+ ctx->info[tensor_id].t.data = (void *)(uintptr_t)data; // double cast suppresses warning about casting away const
1139
+ }
1140
+
1141
+ struct gguf_writer {
1142
+ std::vector<int8_t> & buf;
1143
+
1144
+ gguf_writer(std::vector<int8_t> & buf) : buf(buf) {}
1145
+
1146
+ template <typename T>
1147
+ void write(const T & val) const {
1148
+ for (size_t i = 0; i < sizeof(val); ++i) {
1149
+ buf.push_back(reinterpret_cast<const int8_t *>(&val)[i]);
1150
+ }
1151
+ }
1152
+
1153
+ void write(const std::vector<int8_t> & val) const {
1154
+ buf.insert(buf.end(), val.begin(), val.end());
1155
+ }
1156
+
1157
+ void write(const bool & val) const {
1158
+ const int8_t val8 = val ? 1 : 0;
1159
+ write(val8);
1160
+ }
1161
+
1162
+ void write(const std::string & val) const {
1163
+ {
1164
+ const uint64_t n = val.length();
1165
+ write(n);
1166
+ }
1167
+ for (size_t i = 0; i < val.length(); ++i) {
1168
+ buf.push_back(reinterpret_cast<const int8_t *>(val.data())[i]);
1169
+ }
1170
+ }
1171
+
1172
+ void write(const char * val) const {
1173
+ write(std::string(val));
1174
+ }
1175
+
1176
+ void write(const enum ggml_type & val) const {
1177
+ write(int32_t(val));
1178
+ }
1179
+
1180
+ void write(const enum gguf_type & val) const {
1181
+ write(int32_t(val));
1182
+ }
1183
+
1184
+ void write(const struct gguf_kv & kv) const {
1185
+ const uint64_t ne = kv.get_ne();
1186
+
1187
+ write(kv.get_key());
1188
+
1189
+ if (kv.is_array) {
1190
+ write(GGUF_TYPE_ARRAY);
1191
+ write(kv.get_type());
1192
+ write(ne);
1193
+ } else {
1194
+ write(kv.get_type());
1195
+ }
1196
+
1197
+ switch (kv.get_type()) {
1198
+ case GGUF_TYPE_UINT8:
1199
+ case GGUF_TYPE_INT8:
1200
+ case GGUF_TYPE_UINT16:
1201
+ case GGUF_TYPE_INT16:
1202
+ case GGUF_TYPE_UINT32:
1203
+ case GGUF_TYPE_INT32:
1204
+ case GGUF_TYPE_FLOAT32:
1205
+ case GGUF_TYPE_UINT64:
1206
+ case GGUF_TYPE_INT64:
1207
+ case GGUF_TYPE_FLOAT64: {
1208
+ write(kv.data);
1209
+ } break;
1210
+ case GGUF_TYPE_BOOL: {
1211
+ for (size_t i = 0; i < ne; ++i) {
1212
+ write(kv.get_val<bool>(i));
1213
+ }
1214
+ } break;
1215
+ case GGUF_TYPE_STRING: {
1216
+ for (size_t i = 0; i < ne; ++i) {
1217
+ write(kv.get_val<std::string>(i));
1218
+ }
1219
+ } break;
1220
+ case GGUF_TYPE_ARRAY:
1221
+ default: GGML_ABORT("invalid type");
1222
+ }
1223
+ }
1224
+
1225
+ void write_tensor_meta(const struct gguf_tensor_info & info) const {
1226
+ write(info.t.name);
1227
+
1228
+ const uint32_t n_dims = ggml_n_dims(&info.t);
1229
+ write(n_dims);
1230
+
1231
+ for (uint32_t j = 0; j < n_dims; ++j) {
1232
+ write(info.t.ne[j]);
1233
+ }
1234
+ write(info.t.type);
1235
+ write(info.offset);
1236
+ }
1237
+
1238
+ void pad(const size_t alignment) const {
1239
+ while (buf.size() % alignment != 0) {
1240
+ const int8_t zero = 0;
1241
+ write(zero);
1242
+ }
1243
+ }
1244
+
1245
+ void write_tensor_data(const struct gguf_tensor_info & info, const size_t offset_data, const size_t alignment) const {
1246
+ GGML_ASSERT(buf.size() - offset_data == info.offset);
1247
+
1248
+ GGML_ASSERT(ggml_is_contiguous(&info.t));
1249
+ const size_t offset = buf.size();
1250
+ const size_t nbytes = ggml_nbytes(&info.t);
1251
+
1252
+ buf.resize(offset + nbytes);
1253
+ if (info.t.buffer) {
1254
+ ggml_backend_tensor_get(&info.t, buf.data() + offset, 0, nbytes);
1255
+ } else {
1256
+ GGML_ASSERT(info.t.data);
1257
+ memcpy(buf.data() + offset, info.t.data, nbytes);
1258
+ }
1259
+
1260
+ pad(alignment);
1261
+ }
1262
+ };
1263
+
1264
+ void gguf_write_to_buf(const struct gguf_context * ctx, std::vector<int8_t> & buf, bool only_meta) {
1265
+ const struct gguf_writer gw(buf);
1266
+
1267
+ const int64_t n_kv = gguf_get_n_kv(ctx);
1268
+ const int64_t n_tensors = gguf_get_n_tensors(ctx);
1269
+
1270
+ // write header
1271
+ gw.write(GGUF_MAGIC[0]);
1272
+ gw.write(GGUF_MAGIC[1]);
1273
+ gw.write(GGUF_MAGIC[2]);
1274
+ gw.write(GGUF_MAGIC[3]);
1275
+ gw.write(ctx->version);
1276
+ gw.write(n_tensors);
1277
+ gw.write(n_kv);
1278
+
1279
+ // write key-value pairs
1280
+ for (int64_t i = 0; i < n_kv; ++i) {
1281
+ gw.write(ctx->kv[i]);
1282
+ }
1283
+
1284
+ // write tensor info
1285
+ for (int64_t i = 0; i < n_tensors; ++i) {
1286
+ gw.write_tensor_meta(ctx->info[i]);
1287
+ }
1288
+
1289
+ // we require the data section to be aligned
1290
+ gw.pad(ctx->alignment);
1291
+
1292
+ if (only_meta) {
1293
+ return;
1294
+ }
1295
+
1296
+ const size_t offset_data = gw.buf.size();
1297
+
1298
+ // write tensor data
1299
+ for (int64_t i = 0; i < n_tensors; ++i) {
1300
+ gw.write_tensor_data(ctx->info[i], offset_data, ctx->alignment);
1301
+ }
1302
+ }
1303
+
1304
+ bool gguf_write_to_file(const struct gguf_context * ctx, const char * fname, bool only_meta) {
1305
+ FILE * file = ggml_fopen(fname, "wb");
1306
+
1307
+ if (!file) {
1308
+ GGML_LOG_ERROR("%s: failed to open file '%s' for writing GGUF data\n", __func__, fname);
1309
+ return false;
1310
+ }
1311
+
1312
+ std::vector<int8_t> buf;
1313
+ gguf_write_to_buf(ctx, buf, only_meta);
1314
+ const bool ok = fwrite(buf.data(), 1, buf.size(), file) == buf.size();
1315
+ fclose(file);
1316
+ return ok;
1317
+ }
1318
+
1319
+ size_t gguf_get_meta_size(const struct gguf_context * ctx) {
1320
+ // only return size
1321
+ std::vector<int8_t> buf;
1322
+ gguf_write_to_buf(ctx, buf, /*only_meta =*/ true);
1323
+ return buf.size();
1324
+ }
1325
+
1326
+ void gguf_get_meta_data(const struct gguf_context * ctx, void * data) {
1327
+ std::vector<int8_t> buf;
1328
+ gguf_write_to_buf(ctx, buf, /*only_meta =*/ true);
1329
+ memcpy(data, buf.data(), buf.size());
1330
+ }