whispercpp 1.3.1 → 1.3.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (797) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +4 -3
  3. data/README.md +92 -31
  4. data/Rakefile +26 -7
  5. data/ext/.gitignore +5 -7
  6. data/ext/dependencies.rb +61 -0
  7. data/ext/extconf.rb +21 -198
  8. data/ext/options.rb +221 -0
  9. data/ext/ruby_whisper.c +159 -0
  10. data/ext/ruby_whisper.h +17 -2
  11. data/ext/ruby_whisper_context.c +641 -0
  12. data/ext/ruby_whisper_error.c +52 -0
  13. data/ext/ruby_whisper_model.c +232 -0
  14. data/ext/ruby_whisper_params.c +1301 -0
  15. data/ext/ruby_whisper_segment.c +143 -0
  16. data/ext/ruby_whisper_transcribe.cpp +87 -0
  17. data/ext/ruby_whisper_vad_params.c +288 -0
  18. data/ext/sources/.dockerignore +3 -0
  19. data/ext/sources/.github/workflows/bindings-ruby.yml +21 -0
  20. data/ext/sources/CMakeGraphVizOptions.cmake +8 -0
  21. data/ext/sources/CMakeLists.txt +251 -0
  22. data/ext/sources/bindings/javascript/CMakeLists.txt +41 -0
  23. data/ext/sources/bindings/javascript/emscripten.cpp +93 -0
  24. data/ext/sources/bindings/javascript/libwhisper.worker.js +1 -0
  25. data/ext/sources/bindings/javascript/package-tmpl.json +26 -0
  26. data/ext/sources/bindings/javascript/package.json +26 -0
  27. data/ext/sources/bindings/javascript/whisper.js +19 -0
  28. data/ext/sources/build-xcframework.sh +547 -0
  29. data/ext/sources/ci/run.sh +336 -0
  30. data/ext/sources/close-issue.yml +28 -0
  31. data/ext/sources/cmake/DefaultTargetOptions.cmake +16 -0
  32. data/ext/sources/cmake/FindFFmpeg.cmake +163 -0
  33. data/ext/sources/cmake/build-info.cmake +60 -0
  34. data/ext/sources/cmake/git-vars.cmake +22 -0
  35. data/ext/sources/cmake/whisper-config.cmake.in +65 -0
  36. data/ext/sources/cmake/whisper.pc.in +10 -0
  37. data/ext/sources/examples/CMakeLists.txt +124 -0
  38. data/ext/sources/examples/addon.node/CMakeLists.txt +31 -0
  39. data/ext/sources/examples/addon.node/__test__/whisper.spec.js +37 -0
  40. data/ext/sources/examples/addon.node/addon.cpp +438 -0
  41. data/ext/sources/examples/addon.node/index.js +54 -0
  42. data/ext/sources/examples/addon.node/package.json +16 -0
  43. data/ext/sources/examples/bench/CMakeLists.txt +8 -0
  44. data/ext/sources/examples/bench/bench.cpp +175 -0
  45. data/ext/sources/examples/bench.wasm/CMakeLists.txt +49 -0
  46. data/ext/sources/examples/bench.wasm/emscripten.cpp +87 -0
  47. data/ext/sources/examples/bench.wasm/index-tmpl.html +284 -0
  48. data/ext/sources/examples/cli/CMakeLists.txt +8 -0
  49. data/ext/sources/examples/cli/cli.cpp +1294 -0
  50. data/ext/sources/examples/coi-serviceworker.js +146 -0
  51. data/ext/sources/examples/command/CMakeLists.txt +10 -0
  52. data/ext/sources/examples/command/command.cpp +776 -0
  53. data/ext/sources/examples/command/commands.txt +9 -0
  54. data/ext/sources/examples/command.wasm/CMakeLists.txt +50 -0
  55. data/ext/sources/examples/command.wasm/emscripten.cpp +327 -0
  56. data/ext/sources/examples/command.wasm/index-tmpl.html +414 -0
  57. data/ext/sources/examples/common-ggml.cpp +238 -0
  58. data/ext/sources/examples/common-ggml.h +18 -0
  59. data/ext/sources/examples/common-sdl.cpp +227 -0
  60. data/ext/sources/examples/common-sdl.h +49 -0
  61. data/ext/sources/examples/common-whisper.cpp +168 -0
  62. data/ext/sources/examples/common-whisper.h +24 -0
  63. data/ext/sources/examples/common.cpp +675 -0
  64. data/ext/sources/examples/common.h +322 -0
  65. data/ext/sources/examples/deprecation-warning/CMakeLists.txt +6 -0
  66. data/ext/sources/examples/deprecation-warning/deprecation-warning.cpp +38 -0
  67. data/ext/sources/examples/ffmpeg-transcode.cpp +368 -0
  68. data/ext/sources/examples/generate-karaoke.sh +57 -0
  69. data/ext/sources/examples/grammar-parser.cpp +423 -0
  70. data/ext/sources/examples/grammar-parser.h +29 -0
  71. data/ext/sources/examples/helpers.js +191 -0
  72. data/ext/sources/examples/json.hpp +24596 -0
  73. data/ext/sources/examples/livestream.sh +112 -0
  74. data/ext/sources/examples/lsp/CMakeLists.txt +9 -0
  75. data/ext/sources/examples/lsp/lsp.cpp +467 -0
  76. data/ext/sources/examples/lsp/whisper.vim +362 -0
  77. data/ext/sources/examples/miniaudio.h +93468 -0
  78. data/ext/sources/examples/python/test_whisper_processor.py +7 -0
  79. data/ext/sources/examples/python/whisper_processor.py +54 -0
  80. data/ext/sources/examples/quantize/CMakeLists.txt +6 -0
  81. data/ext/sources/examples/quantize/quantize.cpp +223 -0
  82. data/ext/sources/examples/server/CMakeLists.txt +12 -0
  83. data/ext/sources/examples/server/bench.js +29 -0
  84. data/ext/sources/examples/server/httplib.h +10497 -0
  85. data/ext/sources/examples/server/server.cpp +1091 -0
  86. data/ext/sources/examples/server.py +115 -0
  87. data/ext/sources/examples/stb_vorbis.c +5584 -0
  88. data/ext/sources/examples/stream/CMakeLists.txt +10 -0
  89. data/ext/sources/examples/stream/stream.cpp +429 -0
  90. data/ext/sources/examples/stream.wasm/CMakeLists.txt +49 -0
  91. data/ext/sources/examples/stream.wasm/emscripten.cpp +216 -0
  92. data/ext/sources/examples/stream.wasm/index-tmpl.html +414 -0
  93. data/ext/sources/examples/sycl/CMakeLists.txt +9 -0
  94. data/ext/sources/examples/sycl/build.sh +22 -0
  95. data/ext/sources/examples/sycl/ls-sycl-device.cpp +11 -0
  96. data/ext/sources/examples/sycl/run-whisper.sh +17 -0
  97. data/ext/sources/examples/talk-llama/CMakeLists.txt +40 -0
  98. data/ext/sources/examples/talk-llama/eleven-labs.py +80 -0
  99. data/ext/sources/examples/talk-llama/llama-adapter.cpp +388 -0
  100. data/ext/sources/examples/talk-llama/llama-adapter.h +76 -0
  101. data/ext/sources/examples/talk-llama/llama-arch.cpp +1746 -0
  102. data/ext/sources/examples/talk-llama/llama-arch.h +437 -0
  103. data/ext/sources/examples/talk-llama/llama-batch.cpp +374 -0
  104. data/ext/sources/examples/talk-llama/llama-batch.h +89 -0
  105. data/ext/sources/examples/talk-llama/llama-chat.cpp +663 -0
  106. data/ext/sources/examples/talk-llama/llama-chat.h +58 -0
  107. data/ext/sources/examples/talk-llama/llama-context.cpp +2676 -0
  108. data/ext/sources/examples/talk-llama/llama-context.h +276 -0
  109. data/ext/sources/examples/talk-llama/llama-cparams.cpp +5 -0
  110. data/ext/sources/examples/talk-llama/llama-cparams.h +41 -0
  111. data/ext/sources/examples/talk-llama/llama-grammar.cpp +1229 -0
  112. data/ext/sources/examples/talk-llama/llama-grammar.h +173 -0
  113. data/ext/sources/examples/talk-llama/llama-graph.cpp +1618 -0
  114. data/ext/sources/examples/talk-llama/llama-graph.h +640 -0
  115. data/ext/sources/examples/talk-llama/llama-hparams.cpp +95 -0
  116. data/ext/sources/examples/talk-llama/llama-hparams.h +190 -0
  117. data/ext/sources/examples/talk-llama/llama-impl.cpp +167 -0
  118. data/ext/sources/examples/talk-llama/llama-impl.h +61 -0
  119. data/ext/sources/examples/talk-llama/llama-io.cpp +15 -0
  120. data/ext/sources/examples/talk-llama/llama-io.h +35 -0
  121. data/ext/sources/examples/talk-llama/llama-kv-cache.cpp +2739 -0
  122. data/ext/sources/examples/talk-llama/llama-kv-cache.h +502 -0
  123. data/ext/sources/examples/talk-llama/llama-kv-cells.h +379 -0
  124. data/ext/sources/examples/talk-llama/llama-memory.cpp +1 -0
  125. data/ext/sources/examples/talk-llama/llama-memory.h +32 -0
  126. data/ext/sources/examples/talk-llama/llama-mmap.cpp +600 -0
  127. data/ext/sources/examples/talk-llama/llama-mmap.h +68 -0
  128. data/ext/sources/examples/talk-llama/llama-model-loader.cpp +1138 -0
  129. data/ext/sources/examples/talk-llama/llama-model-loader.h +169 -0
  130. data/ext/sources/examples/talk-llama/llama-model-saver.cpp +281 -0
  131. data/ext/sources/examples/talk-llama/llama-model-saver.h +37 -0
  132. data/ext/sources/examples/talk-llama/llama-model.cpp +13814 -0
  133. data/ext/sources/examples/talk-llama/llama-model.h +425 -0
  134. data/ext/sources/examples/talk-llama/llama-quant.cpp +966 -0
  135. data/ext/sources/examples/talk-llama/llama-quant.h +1 -0
  136. data/ext/sources/examples/talk-llama/llama-sampling.cpp +2575 -0
  137. data/ext/sources/examples/talk-llama/llama-sampling.h +32 -0
  138. data/ext/sources/examples/talk-llama/llama-vocab.cpp +3340 -0
  139. data/ext/sources/examples/talk-llama/llama-vocab.h +131 -0
  140. data/ext/sources/examples/talk-llama/llama.cpp +354 -0
  141. data/ext/sources/examples/talk-llama/llama.h +1377 -0
  142. data/ext/sources/examples/talk-llama/prompts/talk-alpaca.txt +23 -0
  143. data/ext/sources/examples/talk-llama/speak +40 -0
  144. data/ext/sources/examples/talk-llama/speak.bat +1 -0
  145. data/ext/sources/examples/talk-llama/speak.ps1 +14 -0
  146. data/ext/sources/examples/talk-llama/talk-llama.cpp +808 -0
  147. data/ext/sources/examples/talk-llama/unicode-data.cpp +7034 -0
  148. data/ext/sources/examples/talk-llama/unicode-data.h +20 -0
  149. data/ext/sources/examples/talk-llama/unicode.cpp +849 -0
  150. data/ext/sources/examples/talk-llama/unicode.h +66 -0
  151. data/ext/sources/examples/vad-speech-segments/CMakeLists.txt +8 -0
  152. data/ext/sources/examples/vad-speech-segments/speech.cpp +143 -0
  153. data/ext/sources/examples/wchess/CMakeLists.txt +10 -0
  154. data/ext/sources/examples/wchess/libwchess/CMakeLists.txt +19 -0
  155. data/ext/sources/examples/wchess/libwchess/Chessboard.cpp +803 -0
  156. data/ext/sources/examples/wchess/libwchess/Chessboard.h +33 -0
  157. data/ext/sources/examples/wchess/libwchess/WChess.cpp +193 -0
  158. data/ext/sources/examples/wchess/libwchess/WChess.h +63 -0
  159. data/ext/sources/examples/wchess/libwchess/test-chessboard.cpp +117 -0
  160. data/ext/sources/examples/wchess/wchess.cmd/CMakeLists.txt +8 -0
  161. data/ext/sources/examples/wchess/wchess.cmd/wchess.cmd.cpp +249 -0
  162. data/ext/sources/examples/whisper.wasm/CMakeLists.txt +50 -0
  163. data/ext/sources/examples/whisper.wasm/emscripten.cpp +118 -0
  164. data/ext/sources/examples/whisper.wasm/index-tmpl.html +658 -0
  165. data/ext/sources/ggml/CMakeLists.txt +390 -0
  166. data/ext/sources/ggml/cmake/BuildTypes.cmake +54 -0
  167. data/ext/sources/ggml/cmake/GitVars.cmake +22 -0
  168. data/ext/sources/ggml/cmake/common.cmake +26 -0
  169. data/ext/sources/ggml/cmake/ggml-config.cmake.in +152 -0
  170. data/ext/{ggml → sources/ggml}/include/ggml-alloc.h +1 -1
  171. data/ext/{ggml → sources/ggml}/include/ggml-backend.h +9 -7
  172. data/ext/{ggml → sources/ggml}/include/ggml-cpp.h +2 -1
  173. data/ext/{ggml → sources/ggml}/include/ggml-cpu.h +9 -1
  174. data/ext/{ggml → sources/ggml}/include/ggml-metal.h +1 -1
  175. data/ext/{ggml → sources/ggml}/include/ggml-opt.h +49 -28
  176. data/ext/{ggml → sources/ggml}/include/ggml-rpc.h +6 -1
  177. data/ext/{ggml → sources/ggml}/include/ggml-vulkan.h +0 -2
  178. data/ext/{ggml → sources/ggml}/include/ggml.h +182 -265
  179. data/ext/sources/ggml/include/gguf.h +202 -0
  180. data/ext/sources/ggml/src/CMakeLists.txt +346 -0
  181. data/ext/{ggml → sources/ggml}/src/ggml-alloc.c +34 -29
  182. data/ext/sources/ggml/src/ggml-amx/CMakeLists.txt +107 -0
  183. data/ext/{ggml → sources/ggml}/src/ggml-backend-impl.h +1 -2
  184. data/ext/{ggml → sources/ggml}/src/ggml-backend-reg.cpp +87 -53
  185. data/ext/{ggml → sources/ggml}/src/ggml-backend.cpp +26 -14
  186. data/ext/sources/ggml/src/ggml-blas/CMakeLists.txt +87 -0
  187. data/ext/sources/ggml/src/ggml-cann/CMakeLists.txt +74 -0
  188. data/ext/sources/ggml/src/ggml-cann/Doxyfile +2579 -0
  189. data/ext/{ggml → sources/ggml}/src/ggml-cann/acl_tensor.cpp +10 -4
  190. data/ext/{ggml → sources/ggml}/src/ggml-cann/acl_tensor.h +5 -5
  191. data/ext/{ggml → sources/ggml}/src/ggml-cann/aclnn_ops.cpp +1272 -1506
  192. data/ext/sources/ggml/src/ggml-cann/aclnn_ops.h +1125 -0
  193. data/ext/{ggml → sources/ggml}/src/ggml-cann/common.h +135 -1
  194. data/ext/{ggml → sources/ggml}/src/ggml-cann/ggml-cann.cpp +564 -146
  195. data/ext/sources/ggml/src/ggml-cann/kernels/CMakeLists.txt +30 -0
  196. data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/dup.cpp +3 -5
  197. data/ext/{ggml → sources/ggml}/src/ggml-common.h +12 -8
  198. data/ext/sources/ggml/src/ggml-cpu/CMakeLists.txt +504 -0
  199. data/ext/{ggml → sources/ggml}/src/ggml-cpu/amx/amx.cpp +2 -1
  200. data/ext/sources/ggml/src/ggml-cpu/binary-ops.cpp +158 -0
  201. data/ext/sources/ggml/src/ggml-cpu/binary-ops.h +16 -0
  202. data/ext/sources/ggml/src/ggml-cpu/cmake/FindSIMD.cmake +100 -0
  203. data/ext/sources/ggml/src/ggml-cpu/common.h +72 -0
  204. data/ext/{ggml → sources/ggml}/src/ggml-cpu/cpu-feats-x86.cpp +5 -1
  205. data/ext/sources/ggml/src/ggml-cpu/ggml-cpu-aarch64.cpp +6431 -0
  206. data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-impl.h +163 -41
  207. data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-quants.c +4029 -1117
  208. data/ext/sources/ggml/src/ggml-cpu/ggml-cpu.c +3510 -0
  209. data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu.cpp +67 -18
  210. data/ext/sources/ggml/src/ggml-cpu/kleidiai/kernels.cpp +337 -0
  211. data/ext/sources/ggml/src/ggml-cpu/kleidiai/kernels.h +95 -0
  212. data/ext/sources/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp +482 -0
  213. data/ext/sources/ggml/src/ggml-cpu/kleidiai/kleidiai.h +17 -0
  214. data/ext/sources/ggml/src/ggml-cpu/llamafile/sgemm.cpp +3544 -0
  215. data/ext/sources/ggml/src/ggml-cpu/llamafile/sgemm.h +14 -0
  216. data/ext/sources/ggml/src/ggml-cpu/ops.cpp +8903 -0
  217. data/ext/sources/ggml/src/ggml-cpu/ops.h +110 -0
  218. data/ext/sources/ggml/src/ggml-cpu/simd-mappings.h +892 -0
  219. data/ext/sources/ggml/src/ggml-cpu/unary-ops.cpp +186 -0
  220. data/ext/sources/ggml/src/ggml-cpu/unary-ops.h +28 -0
  221. data/ext/sources/ggml/src/ggml-cpu/vec.cpp +252 -0
  222. data/ext/sources/ggml/src/ggml-cpu/vec.h +818 -0
  223. data/ext/sources/ggml/src/ggml-cuda/CMakeLists.txt +184 -0
  224. data/ext/sources/ggml/src/ggml-cuda/acc.cu +61 -0
  225. data/ext/sources/ggml/src/ggml-cuda/acc.cuh +5 -0
  226. data/ext/sources/ggml/src/ggml-cuda/arange.cu +34 -0
  227. data/ext/sources/ggml/src/ggml-cuda/arange.cuh +5 -0
  228. data/ext/sources/ggml/src/ggml-cuda/argmax.cu +91 -0
  229. data/ext/sources/ggml/src/ggml-cuda/argmax.cuh +3 -0
  230. data/ext/sources/ggml/src/ggml-cuda/argsort.cu +104 -0
  231. data/ext/sources/ggml/src/ggml-cuda/argsort.cuh +3 -0
  232. data/ext/sources/ggml/src/ggml-cuda/binbcast.cu +363 -0
  233. data/ext/sources/ggml/src/ggml-cuda/binbcast.cuh +9 -0
  234. data/ext/sources/ggml/src/ggml-cuda/clamp.cu +45 -0
  235. data/ext/sources/ggml/src/ggml-cuda/clamp.cuh +5 -0
  236. data/ext/sources/ggml/src/ggml-cuda/common.cuh +828 -0
  237. data/ext/sources/ggml/src/ggml-cuda/concat.cu +221 -0
  238. data/ext/sources/ggml/src/ggml-cuda/concat.cuh +5 -0
  239. data/ext/sources/ggml/src/ggml-cuda/conv-transpose-1d.cu +89 -0
  240. data/ext/sources/ggml/src/ggml-cuda/conv-transpose-1d.cuh +5 -0
  241. data/ext/sources/ggml/src/ggml-cuda/convert.cu +730 -0
  242. data/ext/sources/ggml/src/ggml-cuda/convert.cuh +26 -0
  243. data/ext/sources/ggml/src/ggml-cuda/count-equal.cu +64 -0
  244. data/ext/sources/ggml/src/ggml-cuda/count-equal.cuh +5 -0
  245. data/ext/sources/ggml/src/ggml-cuda/cp-async.cuh +57 -0
  246. data/ext/sources/ggml/src/ggml-cuda/cpy.cu +705 -0
  247. data/ext/sources/ggml/src/ggml-cuda/cpy.cuh +11 -0
  248. data/ext/sources/ggml/src/ggml-cuda/cross-entropy-loss.cu +189 -0
  249. data/ext/sources/ggml/src/ggml-cuda/cross-entropy-loss.cuh +7 -0
  250. data/ext/sources/ggml/src/ggml-cuda/dequantize.cuh +103 -0
  251. data/ext/sources/ggml/src/ggml-cuda/diagmask.cu +40 -0
  252. data/ext/sources/ggml/src/ggml-cuda/diagmask.cuh +5 -0
  253. data/ext/sources/ggml/src/ggml-cuda/fattn-common.cuh +881 -0
  254. data/ext/sources/ggml/src/ggml-cuda/fattn-mma-f16.cuh +1471 -0
  255. data/ext/sources/ggml/src/ggml-cuda/fattn-tile-f16.cu +357 -0
  256. data/ext/sources/ggml/src/ggml-cuda/fattn-tile-f16.cuh +3 -0
  257. data/ext/sources/ggml/src/ggml-cuda/fattn-tile-f32.cu +365 -0
  258. data/ext/sources/ggml/src/ggml-cuda/fattn-tile-f32.cuh +3 -0
  259. data/ext/sources/ggml/src/ggml-cuda/fattn-vec-f16.cuh +482 -0
  260. data/ext/sources/ggml/src/ggml-cuda/fattn-vec-f32.cuh +472 -0
  261. data/ext/sources/ggml/src/ggml-cuda/fattn-wmma-f16.cu +634 -0
  262. data/ext/sources/ggml/src/ggml-cuda/fattn-wmma-f16.cuh +3 -0
  263. data/ext/sources/ggml/src/ggml-cuda/fattn.cu +346 -0
  264. data/ext/sources/ggml/src/ggml-cuda/fattn.cuh +3 -0
  265. data/ext/sources/ggml/src/ggml-cuda/getrows.cu +275 -0
  266. data/ext/sources/ggml/src/ggml-cuda/getrows.cuh +15 -0
  267. data/ext/sources/ggml/src/ggml-cuda/ggml-cuda.cu +3505 -0
  268. data/ext/sources/ggml/src/ggml-cuda/gla.cu +93 -0
  269. data/ext/sources/ggml/src/ggml-cuda/gla.cuh +3 -0
  270. data/ext/sources/ggml/src/ggml-cuda/im2col.cu +103 -0
  271. data/ext/sources/ggml/src/ggml-cuda/im2col.cuh +5 -0
  272. data/ext/sources/ggml/src/ggml-cuda/mma.cuh +396 -0
  273. data/ext/sources/ggml/src/ggml-cuda/mmq.cu +324 -0
  274. data/ext/sources/ggml/src/ggml-cuda/mmq.cuh +3217 -0
  275. data/ext/sources/ggml/src/ggml-cuda/mmv.cu +336 -0
  276. data/ext/sources/ggml/src/ggml-cuda/mmv.cuh +12 -0
  277. data/ext/sources/ggml/src/ggml-cuda/mmvq.cu +595 -0
  278. data/ext/sources/ggml/src/ggml-cuda/mmvq.cuh +12 -0
  279. data/ext/sources/ggml/src/ggml-cuda/norm.cu +458 -0
  280. data/ext/sources/ggml/src/ggml-cuda/norm.cuh +11 -0
  281. data/ext/sources/ggml/src/ggml-cuda/opt-step-adamw.cu +78 -0
  282. data/ext/sources/ggml/src/ggml-cuda/opt-step-adamw.cuh +5 -0
  283. data/ext/sources/ggml/src/ggml-cuda/out-prod.cu +68 -0
  284. data/ext/sources/ggml/src/ggml-cuda/out-prod.cuh +3 -0
  285. data/ext/sources/ggml/src/ggml-cuda/pad.cu +49 -0
  286. data/ext/sources/ggml/src/ggml-cuda/pad.cuh +5 -0
  287. data/ext/sources/ggml/src/ggml-cuda/pool2d.cu +94 -0
  288. data/ext/sources/ggml/src/ggml-cuda/pool2d.cuh +5 -0
  289. data/ext/sources/ggml/src/ggml-cuda/quantize.cu +190 -0
  290. data/ext/sources/ggml/src/ggml-cuda/quantize.cuh +27 -0
  291. data/ext/sources/ggml/src/ggml-cuda/rope.cu +456 -0
  292. data/ext/sources/ggml/src/ggml-cuda/rope.cuh +7 -0
  293. data/ext/sources/ggml/src/ggml-cuda/scale.cu +31 -0
  294. data/ext/sources/ggml/src/ggml-cuda/scale.cuh +5 -0
  295. data/ext/sources/ggml/src/ggml-cuda/softmax.cu +283 -0
  296. data/ext/sources/ggml/src/ggml-cuda/softmax.cuh +7 -0
  297. data/ext/sources/ggml/src/ggml-cuda/ssm-conv.cu +148 -0
  298. data/ext/sources/ggml/src/ggml-cuda/ssm-conv.cuh +3 -0
  299. data/ext/sources/ggml/src/ggml-cuda/ssm-scan.cu +153 -0
  300. data/ext/sources/ggml/src/ggml-cuda/ssm-scan.cuh +3 -0
  301. data/ext/sources/ggml/src/ggml-cuda/sum.cu +45 -0
  302. data/ext/sources/ggml/src/ggml-cuda/sum.cuh +5 -0
  303. data/ext/sources/ggml/src/ggml-cuda/sumrows.cu +39 -0
  304. data/ext/sources/ggml/src/ggml-cuda/sumrows.cuh +5 -0
  305. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_16.cu +5 -0
  306. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_8.cu +10 -0
  307. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_1.cu +10 -0
  308. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_2.cu +10 -0
  309. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_4.cu +10 -0
  310. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_16.cu +5 -0
  311. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_4.cu +10 -0
  312. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_8.cu +10 -0
  313. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_1.cu +10 -0
  314. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_2.cu +10 -0
  315. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_16.cu +5 -0
  316. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_2.cu +10 -0
  317. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_4.cu +10 -0
  318. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_8.cu +10 -0
  319. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_64-ncols2_1.cu +10 -0
  320. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_1.cu +10 -0
  321. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_2.cu +10 -0
  322. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_4.cu +10 -0
  323. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_8.cu +10 -0
  324. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-f16.cu +5 -0
  325. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q4_0.cu +5 -0
  326. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q4_1.cu +5 -0
  327. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q5_0.cu +5 -0
  328. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q5_1.cu +5 -0
  329. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q8_0.cu +5 -0
  330. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-f16.cu +5 -0
  331. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q4_0.cu +5 -0
  332. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q4_1.cu +5 -0
  333. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q5_0.cu +5 -0
  334. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q5_1.cu +5 -0
  335. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q8_0.cu +5 -0
  336. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-f16.cu +5 -0
  337. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q4_0.cu +5 -0
  338. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q4_1.cu +5 -0
  339. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q5_0.cu +5 -0
  340. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q5_1.cu +5 -0
  341. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q8_0.cu +5 -0
  342. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-f16.cu +5 -0
  343. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q4_0.cu +5 -0
  344. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q4_1.cu +5 -0
  345. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q5_0.cu +5 -0
  346. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q5_1.cu +5 -0
  347. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q8_0.cu +5 -0
  348. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-f16.cu +5 -0
  349. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q4_0.cu +5 -0
  350. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q4_1.cu +5 -0
  351. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q5_0.cu +5 -0
  352. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q5_1.cu +5 -0
  353. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q8_0.cu +5 -0
  354. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-f16.cu +5 -0
  355. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q4_0.cu +5 -0
  356. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q4_1.cu +5 -0
  357. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q5_0.cu +5 -0
  358. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q5_1.cu +5 -0
  359. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q8_0.cu +5 -0
  360. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs256-f16-f16.cu +5 -0
  361. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-f16.cu +5 -0
  362. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q4_0.cu +5 -0
  363. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q4_1.cu +5 -0
  364. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q5_0.cu +5 -0
  365. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q5_1.cu +5 -0
  366. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q8_0.cu +5 -0
  367. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-f16.cu +5 -0
  368. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q4_0.cu +5 -0
  369. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q4_1.cu +5 -0
  370. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q5_0.cu +5 -0
  371. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q5_1.cu +5 -0
  372. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q8_0.cu +5 -0
  373. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-f16.cu +5 -0
  374. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q4_0.cu +5 -0
  375. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q4_1.cu +5 -0
  376. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q5_0.cu +5 -0
  377. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q5_1.cu +5 -0
  378. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q8_0.cu +5 -0
  379. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-f16.cu +5 -0
  380. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q4_0.cu +5 -0
  381. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q4_1.cu +5 -0
  382. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q5_0.cu +5 -0
  383. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q5_1.cu +5 -0
  384. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q8_0.cu +5 -0
  385. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-f16.cu +5 -0
  386. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q4_0.cu +5 -0
  387. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q4_1.cu +5 -0
  388. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q5_0.cu +5 -0
  389. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q5_1.cu +5 -0
  390. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q8_0.cu +5 -0
  391. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-f16.cu +5 -0
  392. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q4_0.cu +5 -0
  393. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q4_1.cu +5 -0
  394. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q5_0.cu +5 -0
  395. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q5_1.cu +5 -0
  396. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q8_0.cu +5 -0
  397. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-f16.cu +5 -0
  398. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q4_0.cu +5 -0
  399. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q4_1.cu +5 -0
  400. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q5_0.cu +5 -0
  401. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q5_1.cu +5 -0
  402. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q8_0.cu +5 -0
  403. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs256-f16-f16.cu +5 -0
  404. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-f16.cu +5 -0
  405. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q4_0.cu +5 -0
  406. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q4_1.cu +5 -0
  407. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q5_0.cu +5 -0
  408. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q5_1.cu +5 -0
  409. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q8_0.cu +5 -0
  410. data/ext/sources/ggml/src/ggml-cuda/template-instances/generate_cu_files.py +78 -0
  411. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq1_s.cu +5 -0
  412. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_s.cu +5 -0
  413. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xs.cu +5 -0
  414. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xxs.cu +5 -0
  415. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_s.cu +5 -0
  416. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_xxs.cu +5 -0
  417. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_nl.cu +5 -0
  418. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_xs.cu +5 -0
  419. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_k.cu +5 -0
  420. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q3_k.cu +5 -0
  421. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_0.cu +5 -0
  422. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_1.cu +5 -0
  423. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_k.cu +5 -0
  424. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_0.cu +5 -0
  425. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_1.cu +5 -0
  426. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_k.cu +5 -0
  427. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q6_k.cu +5 -0
  428. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q8_0.cu +5 -0
  429. data/ext/sources/ggml/src/ggml-cuda/tsembd.cu +47 -0
  430. data/ext/sources/ggml/src/ggml-cuda/tsembd.cuh +5 -0
  431. data/ext/sources/ggml/src/ggml-cuda/unary.cu +289 -0
  432. data/ext/sources/ggml/src/ggml-cuda/unary.cuh +59 -0
  433. data/ext/sources/ggml/src/ggml-cuda/upscale.cu +51 -0
  434. data/ext/sources/ggml/src/ggml-cuda/upscale.cuh +5 -0
  435. data/ext/sources/ggml/src/ggml-cuda/vecdotq.cuh +1135 -0
  436. data/ext/{ggml → sources/ggml}/src/ggml-cuda/vendors/cuda.h +1 -0
  437. data/ext/{ggml → sources/ggml}/src/ggml-cuda/vendors/hip.h +57 -0
  438. data/ext/{ggml → sources/ggml}/src/ggml-cuda/vendors/musa.h +7 -1
  439. data/ext/sources/ggml/src/ggml-cuda/wkv.cu +199 -0
  440. data/ext/sources/ggml/src/ggml-cuda/wkv.cuh +7 -0
  441. data/ext/sources/ggml/src/ggml-hip/CMakeLists.txt +131 -0
  442. data/ext/{ggml → sources/ggml}/src/ggml-impl.h +64 -19
  443. data/ext/sources/ggml/src/ggml-kompute/CMakeLists.txt +166 -0
  444. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/common.comp +112 -0
  445. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_add.comp +58 -0
  446. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_addrow.comp +25 -0
  447. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_cpy_f16_f16.comp +52 -0
  448. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_cpy_f16_f32.comp +52 -0
  449. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_cpy_f32_f16.comp +52 -0
  450. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_cpy_f32_f32.comp +52 -0
  451. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_diagmask.comp +30 -0
  452. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_gelu.comp +22 -0
  453. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows.comp +17 -0
  454. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows_f16.comp +31 -0
  455. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows_f32.comp +31 -0
  456. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows_q4_0.comp +38 -0
  457. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows_q4_1.comp +39 -0
  458. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows_q6_k.comp +44 -0
  459. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul.comp +52 -0
  460. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_f16.comp +69 -0
  461. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_mat_f32.comp +51 -0
  462. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q4_0.comp +33 -0
  463. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q4_1.comp +35 -0
  464. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q4_k.comp +140 -0
  465. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q6_k.comp +106 -0
  466. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q8_0.comp +73 -0
  467. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mv_q_n.comp +52 -0
  468. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mv_q_n_pre.comp +28 -0
  469. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_norm.comp +84 -0
  470. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_relu.comp +21 -0
  471. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_rmsnorm.comp +53 -0
  472. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_rope_neox_f16.comp +52 -0
  473. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_rope_neox_f32.comp +52 -0
  474. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_rope_norm_f16.comp +52 -0
  475. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_rope_norm_f32.comp +52 -0
  476. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_scale.comp +19 -0
  477. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_scale_8.comp +23 -0
  478. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_silu.comp +22 -0
  479. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_softmax.comp +72 -0
  480. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/rope_common.comp +71 -0
  481. data/ext/sources/ggml/src/ggml-metal/CMakeLists.txt +120 -0
  482. data/ext/sources/ggml/src/ggml-metal/ggml-metal-impl.h +622 -0
  483. data/ext/{ggml → sources/ggml}/src/ggml-metal/ggml-metal.m +2178 -1064
  484. data/ext/{ggml → sources/ggml}/src/ggml-metal/ggml-metal.metal +1575 -1218
  485. data/ext/sources/ggml/src/ggml-musa/CMakeLists.txt +113 -0
  486. data/ext/sources/ggml/src/ggml-musa/mudnn.cu +112 -0
  487. data/ext/sources/ggml/src/ggml-musa/mudnn.cuh +12 -0
  488. data/ext/sources/ggml/src/ggml-opencl/CMakeLists.txt +96 -0
  489. data/ext/sources/ggml/src/ggml-opencl/ggml-opencl.cpp +5124 -0
  490. data/ext/sources/ggml/src/ggml-opencl/kernels/add.cl +83 -0
  491. data/ext/sources/ggml/src/ggml-opencl/kernels/clamp.cl +20 -0
  492. data/ext/sources/ggml/src/ggml-opencl/kernels/cpy.cl +184 -0
  493. data/ext/sources/ggml/src/ggml-opencl/kernels/cvt.cl +118 -0
  494. data/ext/sources/ggml/src/ggml-opencl/kernels/diag_mask_inf.cl +58 -0
  495. data/ext/sources/ggml/src/ggml-opencl/kernels/embed_kernel.py +26 -0
  496. data/ext/sources/ggml/src/ggml-opencl/kernels/gelu.cl +62 -0
  497. data/ext/sources/ggml/src/ggml-opencl/kernels/gemv_noshuffle.cl +268 -0
  498. data/ext/sources/ggml/src/ggml-opencl/kernels/gemv_noshuffle_general.cl +274 -0
  499. data/ext/sources/ggml/src/ggml-opencl/kernels/get_rows.cl +163 -0
  500. data/ext/sources/ggml/src/ggml-opencl/kernels/im2col_f16.cl +57 -0
  501. data/ext/sources/ggml/src/ggml-opencl/kernels/im2col_f32.cl +57 -0
  502. data/ext/sources/ggml/src/ggml-opencl/kernels/mul.cl +79 -0
  503. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mat_Ab_Bi_8x4.cl +139 -0
  504. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_f16_f16.cl +118 -0
  505. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32.cl +118 -0
  506. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_1row.cl +94 -0
  507. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_l4.cl +84 -0
  508. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_f32_f32.cl +118 -0
  509. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32.cl +192 -0
  510. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_16x_flat.cl +307 -0
  511. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_8x_flat.cl +265 -0
  512. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_8x_flat.cl +272 -0
  513. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_v.cl +254 -0
  514. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q6_k.cl +190 -0
  515. data/ext/sources/ggml/src/ggml-opencl/kernels/norm.cl +81 -0
  516. data/ext/sources/ggml/src/ggml-opencl/kernels/relu.cl +16 -0
  517. data/ext/sources/ggml/src/ggml-opencl/kernels/rms_norm.cl +96 -0
  518. data/ext/sources/ggml/src/ggml-opencl/kernels/rope.cl +721 -0
  519. data/ext/sources/ggml/src/ggml-opencl/kernels/scale.cl +16 -0
  520. data/ext/sources/ggml/src/ggml-opencl/kernels/silu.cl +30 -0
  521. data/ext/sources/ggml/src/ggml-opencl/kernels/softmax_4_f16.cl +87 -0
  522. data/ext/sources/ggml/src/ggml-opencl/kernels/softmax_4_f32.cl +87 -0
  523. data/ext/sources/ggml/src/ggml-opencl/kernels/softmax_f16.cl +86 -0
  524. data/ext/sources/ggml/src/ggml-opencl/kernels/softmax_f32.cl +86 -0
  525. data/ext/sources/ggml/src/ggml-opencl/kernels/transpose.cl +84 -0
  526. data/ext/{ggml → sources/ggml}/src/ggml-opt.cpp +373 -190
  527. data/ext/{ggml → sources/ggml}/src/ggml-quants.c +114 -120
  528. data/ext/sources/ggml/src/ggml-rpc/CMakeLists.txt +9 -0
  529. data/ext/{ggml → sources/ggml}/src/ggml-rpc/ggml-rpc.cpp +480 -73
  530. data/ext/sources/ggml/src/ggml-sycl/CMakeLists.txt +189 -0
  531. data/ext/sources/ggml/src/ggml-sycl/backend.hpp +37 -0
  532. data/ext/sources/ggml/src/ggml-sycl/binbcast.cpp +345 -0
  533. data/ext/sources/ggml/src/ggml-sycl/binbcast.hpp +39 -0
  534. data/ext/{ggml → sources/ggml}/src/ggml-sycl/common.cpp +20 -32
  535. data/ext/sources/ggml/src/ggml-sycl/common.hpp +589 -0
  536. data/ext/{ggml → sources/ggml}/src/ggml-sycl/concat.cpp +32 -33
  537. data/ext/sources/ggml/src/ggml-sycl/concat.hpp +20 -0
  538. data/ext/{ggml → sources/ggml}/src/ggml-sycl/conv.cpp +4 -2
  539. data/ext/sources/ggml/src/ggml-sycl/conv.hpp +20 -0
  540. data/ext/{ggml → sources/ggml}/src/ggml-sycl/convert.cpp +104 -28
  541. data/ext/sources/ggml/src/ggml-sycl/convert.hpp +34 -0
  542. data/ext/sources/ggml/src/ggml-sycl/cpy.cpp +700 -0
  543. data/ext/sources/ggml/src/ggml-sycl/cpy.hpp +11 -0
  544. data/ext/sources/ggml/src/ggml-sycl/dequantize.hpp +791 -0
  545. data/ext/{ggml → sources/ggml}/src/ggml-sycl/dmmv.cpp +156 -17
  546. data/ext/sources/ggml/src/ggml-sycl/dmmv.hpp +27 -0
  547. data/ext/sources/ggml/src/ggml-sycl/dpct/helper.hpp +2957 -0
  548. data/ext/sources/ggml/src/ggml-sycl/element_wise.cpp +1511 -0
  549. data/ext/sources/ggml/src/ggml-sycl/element_wise.hpp +75 -0
  550. data/ext/sources/ggml/src/ggml-sycl/gemm.hpp +99 -0
  551. data/ext/sources/ggml/src/ggml-sycl/getrows.cpp +309 -0
  552. data/ext/sources/ggml/src/ggml-sycl/getrows.hpp +20 -0
  553. data/ext/{ggml → sources/ggml}/src/ggml-sycl/ggml-sycl.cpp +1004 -1240
  554. data/ext/sources/ggml/src/ggml-sycl/gla.cpp +106 -0
  555. data/ext/sources/ggml/src/ggml-sycl/gla.hpp +8 -0
  556. data/ext/sources/ggml/src/ggml-sycl/im2col.cpp +136 -0
  557. data/ext/sources/ggml/src/ggml-sycl/im2col.hpp +21 -0
  558. data/ext/{ggml → sources/ggml}/src/ggml-sycl/mmq.cpp +0 -1
  559. data/ext/sources/ggml/src/ggml-sycl/mmq.hpp +33 -0
  560. data/ext/{ggml → sources/ggml}/src/ggml-sycl/mmvq.cpp +261 -166
  561. data/ext/sources/ggml/src/ggml-sycl/mmvq.hpp +27 -0
  562. data/ext/{ggml → sources/ggml}/src/ggml-sycl/norm.cpp +204 -81
  563. data/ext/sources/ggml/src/ggml-sycl/norm.hpp +26 -0
  564. data/ext/{ggml → sources/ggml}/src/ggml-sycl/outprod.cpp +8 -17
  565. data/ext/sources/ggml/src/ggml-sycl/outprod.hpp +10 -0
  566. data/ext/sources/ggml/src/ggml-sycl/presets.hpp +74 -0
  567. data/ext/sources/ggml/src/ggml-sycl/quants.hpp +83 -0
  568. data/ext/sources/ggml/src/ggml-sycl/rope.cpp +361 -0
  569. data/ext/sources/ggml/src/ggml-sycl/rope.hpp +20 -0
  570. data/ext/{ggml → sources/ggml}/src/ggml-sycl/softmax.cpp +35 -25
  571. data/ext/sources/ggml/src/ggml-sycl/softmax.hpp +20 -0
  572. data/ext/sources/ggml/src/ggml-sycl/sycl_hw.cpp +13 -0
  573. data/ext/sources/ggml/src/ggml-sycl/sycl_hw.hpp +23 -0
  574. data/ext/{ggml → sources/ggml}/src/ggml-sycl/tsembd.cpp +3 -3
  575. data/ext/sources/ggml/src/ggml-sycl/tsembd.hpp +20 -0
  576. data/ext/sources/ggml/src/ggml-sycl/vecdotq.hpp +1215 -0
  577. data/ext/sources/ggml/src/ggml-sycl/wkv.cpp +293 -0
  578. data/ext/sources/ggml/src/ggml-sycl/wkv.hpp +10 -0
  579. data/ext/sources/ggml/src/ggml-vulkan/CMakeLists.txt +196 -0
  580. data/ext/sources/ggml/src/ggml-vulkan/cmake/host-toolchain.cmake.in +15 -0
  581. data/ext/{ggml → sources/ggml}/src/ggml-vulkan/ggml-vulkan.cpp +3130 -1087
  582. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/CMakeLists.txt +39 -0
  583. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/acc.comp +29 -0
  584. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/add.comp +29 -0
  585. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/argmax.comp +51 -0
  586. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/argsort.comp +69 -0
  587. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/clamp.comp +17 -0
  588. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/concat.comp +41 -0
  589. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/contig_copy.comp +49 -0
  590. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/conv2d_dw.comp +105 -0
  591. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/copy.comp +23 -0
  592. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/copy_from_quant.comp +51 -0
  593. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/copy_to_quant.comp +242 -0
  594. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/cos.comp +17 -0
  595. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/count_equal.comp +31 -0
  596. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_f32.comp +20 -0
  597. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.comp +462 -0
  598. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.comp +699 -0
  599. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_head.comp +13 -0
  600. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_m.comp +42 -0
  601. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_s.comp +35 -0
  602. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_s.comp +44 -0
  603. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xs.comp +43 -0
  604. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xxs.comp +48 -0
  605. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_s.comp +39 -0
  606. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_xxs.comp +49 -0
  607. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_nl.comp +32 -0
  608. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_xs.comp +34 -0
  609. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q2_k.comp +34 -0
  610. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q3_k.comp +42 -0
  611. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_0.comp +30 -0
  612. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_1.comp +32 -0
  613. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_k.comp +68 -0
  614. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_0.comp +34 -0
  615. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_1.comp +35 -0
  616. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_k.comp +70 -0
  617. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q6_k.comp +33 -0
  618. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q8_0.comp +31 -0
  619. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/diag_mask_inf.comp +34 -0
  620. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/div.comp +27 -0
  621. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp +337 -0
  622. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.comp +162 -0
  623. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp +360 -0
  624. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp +267 -0
  625. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_split_k_reduce.comp +59 -0
  626. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/gelu.comp +25 -0
  627. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/gelu_quick.comp +23 -0
  628. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/generic_binary_head.comp +64 -0
  629. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/generic_head.comp +9 -0
  630. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/generic_unary_head.comp +76 -0
  631. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/get_rows.comp +33 -0
  632. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_quant.comp +41 -0
  633. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/group_norm.comp +66 -0
  634. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/im2col.comp +100 -0
  635. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/l2_norm.comp +41 -0
  636. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/leaky_relu.comp +22 -0
  637. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul.comp +27 -0
  638. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_split_k_reduce.comp +48 -0
  639. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec.comp +169 -0
  640. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_base.comp +118 -0
  641. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_m.comp +82 -0
  642. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_s.comp +79 -0
  643. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_s.comp +90 -0
  644. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xs.comp +87 -0
  645. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xxs.comp +87 -0
  646. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_s.comp +90 -0
  647. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_xxs.comp +88 -0
  648. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_nc.comp +118 -0
  649. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_p021.comp +154 -0
  650. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q2_k.comp +130 -0
  651. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q3_k.comp +132 -0
  652. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q4_k.comp +136 -0
  653. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q5_k.comp +167 -0
  654. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q6_k.comp +130 -0
  655. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp +868 -0
  656. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp +441 -0
  657. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp +442 -0
  658. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_funcs.comp +99 -0
  659. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/norm.comp +44 -0
  660. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_adamw.comp +42 -0
  661. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/pad.comp +28 -0
  662. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/pool2d.comp +74 -0
  663. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/quantize_q8_1.comp +77 -0
  664. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/relu.comp +21 -0
  665. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/repeat.comp +26 -0
  666. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/repeat_back.comp +37 -0
  667. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp +52 -0
  668. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_back.comp +55 -0
  669. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rope_head.comp +58 -0
  670. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rope_multi.comp +60 -0
  671. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rope_neox.comp +43 -0
  672. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rope_norm.comp +43 -0
  673. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rope_vision.comp +47 -0
  674. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/scale.comp +24 -0
  675. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/sigmoid.comp +20 -0
  676. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/silu.comp +22 -0
  677. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/silu_back.comp +26 -0
  678. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/sin.comp +17 -0
  679. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/soft_max.comp +173 -0
  680. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_back.comp +50 -0
  681. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/square.comp +17 -0
  682. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/sub.comp +29 -0
  683. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.comp +37 -0
  684. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/tanh.comp +20 -0
  685. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/test_bfloat16_support.comp +7 -0
  686. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/test_coopmat2_support.comp +7 -0
  687. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/test_coopmat_support.comp +7 -0
  688. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/test_integer_dot_support.comp +7 -0
  689. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/timestep_embedding.comp +41 -0
  690. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/types.comp +1373 -0
  691. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/upscale.comp +36 -0
  692. data/ext/{ggml → sources/ggml}/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +193 -35
  693. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/wkv6.comp +87 -0
  694. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/wkv7.comp +91 -0
  695. data/ext/{ggml → sources/ggml}/src/ggml.c +676 -1820
  696. data/ext/sources/ggml/src/gguf.cpp +1330 -0
  697. data/ext/{include → sources/include}/whisper.h +68 -2
  698. data/ext/sources/src/CMakeLists.txt +143 -0
  699. data/ext/{src → sources/src}/coreml/whisper-decoder-impl.h +27 -15
  700. data/ext/{src → sources/src}/coreml/whisper-decoder-impl.m +35 -10
  701. data/ext/{src → sources/src}/coreml/whisper-encoder-impl.h +21 -9
  702. data/ext/{src → sources/src}/coreml/whisper-encoder-impl.m +28 -3
  703. data/ext/sources/src/coreml/whisper-encoder.mm +73 -0
  704. data/ext/sources/src/whisper-arch.h +197 -0
  705. data/ext/{src → sources/src}/whisper.cpp +1905 -374
  706. data/ext/sources/tests/CMakeLists.txt +105 -0
  707. data/ext/sources/tests/earnings21/eval.mk +58 -0
  708. data/ext/sources/tests/earnings21/eval.py +68 -0
  709. data/ext/sources/tests/earnings21/normalizers/__init__.py +2 -0
  710. data/ext/sources/tests/earnings21/normalizers/basic.py +80 -0
  711. data/ext/sources/tests/earnings21/normalizers/english.json +1741 -0
  712. data/ext/sources/tests/earnings21/normalizers/english.py +550 -0
  713. data/ext/sources/tests/earnings21/requirements.txt +6 -0
  714. data/ext/sources/tests/en-0-ref.txt +1 -0
  715. data/ext/sources/tests/en-1-ref.txt +1 -0
  716. data/ext/sources/tests/en-2-ref.txt +1 -0
  717. data/ext/sources/tests/es-0-ref.txt +1 -0
  718. data/ext/sources/tests/librispeech/eval.mk +39 -0
  719. data/ext/sources/tests/librispeech/eval.py +47 -0
  720. data/ext/sources/tests/librispeech/normalizers/__init__.py +2 -0
  721. data/ext/sources/tests/librispeech/normalizers/basic.py +80 -0
  722. data/ext/sources/tests/librispeech/normalizers/english.json +1741 -0
  723. data/ext/sources/tests/librispeech/normalizers/english.py +550 -0
  724. data/ext/sources/tests/librispeech/requirements.txt +6 -0
  725. data/ext/sources/tests/run-tests.sh +130 -0
  726. data/ext/sources/tests/test-c.c +3 -0
  727. data/ext/sources/tests/test-vad-full.cpp +54 -0
  728. data/ext/sources/tests/test-vad.cpp +83 -0
  729. data/ext/sources/tests/test-whisper.js +58 -0
  730. data/extsources.rb +33 -5
  731. data/lib/whisper/model/uri.rb +149 -128
  732. data/sig/whisper.rbs +480 -0
  733. data/tests/helper.rb +28 -0
  734. data/tests/test_callback.rb +45 -3
  735. data/tests/test_error.rb +2 -2
  736. data/tests/test_model.rb +38 -0
  737. data/tests/test_package.rb +18 -3
  738. data/tests/test_params.rb +145 -8
  739. data/tests/test_segment.rb +10 -19
  740. data/tests/test_vad.rb +19 -0
  741. data/tests/test_vad_params.rb +103 -0
  742. data/tests/test_whisper.rb +37 -37
  743. data/whispercpp.gemspec +5 -4
  744. metadata +766 -111
  745. data/ext/cpu.mk +0 -9
  746. data/ext/examples/dr_wav.h +0 -8815
  747. data/ext/ggml/src/ggml-cann/aclnn_ops.h +0 -592
  748. data/ext/ggml/src/ggml-cpu/ggml-cpu-aarch64.cpp +0 -4262
  749. data/ext/ggml/src/ggml-cpu/ggml-cpu.c +0 -14123
  750. data/ext/ggml/src/ggml-cpu/llamafile/sgemm.cpp +0 -1884
  751. data/ext/ggml/src/ggml-cpu/llamafile/sgemm.h +0 -14
  752. data/ext/ggml/src/ggml-metal/ggml-metal-impl.h +0 -288
  753. data/ext/ggml/src/ggml-sycl/element_wise.cpp +0 -1030
  754. data/ext/ggml/src/ggml-sycl/im2col.cpp +0 -126
  755. data/ext/ggml/src/ggml-sycl/rope.cpp +0 -276
  756. data/ext/ggml/src/ggml-sycl/wkv6.cpp +0 -141
  757. data/ext/metal-embed.mk +0 -17
  758. data/ext/metal.mk +0 -6
  759. data/ext/ruby_whisper.cpp +0 -1909
  760. data/ext/scripts/get-flags.mk +0 -38
  761. data/lib/whisper.rb +0 -2
  762. /data/ext/{ggml → sources/ggml}/include/ggml-blas.h +0 -0
  763. /data/ext/{ggml → sources/ggml}/include/ggml-cann.h +0 -0
  764. /data/ext/{ggml → sources/ggml}/include/ggml-cuda.h +0 -0
  765. /data/ext/{ggml → sources/ggml}/include/ggml-kompute.h +0 -0
  766. /data/ext/{ggml → sources/ggml}/include/ggml-opencl.h +0 -0
  767. /data/ext/{ggml → sources/ggml}/include/ggml-sycl.h +0 -0
  768. /data/ext/{ggml → sources/ggml}/src/ggml-amx/common.h +0 -0
  769. /data/ext/{ggml → sources/ggml}/src/ggml-amx/ggml-amx.cpp +0 -0
  770. /data/ext/{ggml → sources/ggml}/src/ggml-amx/mmq.cpp +0 -0
  771. /data/ext/{ggml → sources/ggml}/src/ggml-amx/mmq.h +0 -0
  772. /data/ext/{ggml → sources/ggml}/src/ggml-blas/ggml-blas.cpp +0 -0
  773. /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/ascendc_kernels.h +0 -0
  774. /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/get_row_f16.cpp +0 -0
  775. /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/get_row_f32.cpp +0 -0
  776. /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/get_row_q4_0.cpp +0 -0
  777. /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/get_row_q8_0.cpp +0 -0
  778. /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/quantize_f16_q8_0.cpp +0 -0
  779. /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/quantize_f32_q8_0.cpp +0 -0
  780. /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/quantize_float_to_q4_0.cpp +0 -0
  781. /data/ext/{ggml → sources/ggml}/src/ggml-cpu/amx/amx.h +0 -0
  782. /data/ext/{ggml → sources/ggml}/src/ggml-cpu/amx/common.h +0 -0
  783. /data/ext/{ggml → sources/ggml}/src/ggml-cpu/amx/mmq.cpp +0 -0
  784. /data/ext/{ggml → sources/ggml}/src/ggml-cpu/amx/mmq.h +0 -0
  785. /data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-aarch64.h +0 -0
  786. /data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-hbm.cpp +0 -0
  787. /data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-hbm.h +0 -0
  788. /data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-quants.h +0 -0
  789. /data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-traits.cpp +0 -0
  790. /data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-traits.h +0 -0
  791. /data/ext/{ggml → sources/ggml}/src/ggml-kompute/ggml-kompute.cpp +0 -0
  792. /data/ext/{ggml → sources/ggml}/src/ggml-quants.h +0 -0
  793. /data/ext/{ggml → sources/ggml}/src/ggml-threading.cpp +0 -0
  794. /data/ext/{ggml → sources/ggml}/src/ggml-threading.h +0 -0
  795. /data/ext/{src → sources/src}/coreml/whisper-encoder.h +0 -0
  796. /data/ext/{src → sources/src}/openvino/whisper-openvino-encoder.cpp +0 -0
  797. /data/ext/{src → sources/src}/openvino/whisper-openvino-encoder.h +0 -0
@@ -0,0 +1,2676 @@
1
+ #include "llama-context.h"
2
+
3
+ #include "llama-impl.h"
4
+ #include "llama-io.h"
5
+ #include "llama-mmap.h"
6
+ #include "llama-model.h"
7
+ #include "llama-kv-cache.h"
8
+
9
+ #include <cstring>
10
+ #include <stdexcept>
11
+ #include <cinttypes>
12
+
13
+ //
14
+ // llama_context
15
+ //
16
+
17
+ llama_context::llama_context(
18
+ const llama_model & model,
19
+ llama_context_params params) :
20
+ model(model) {
21
+ LLAMA_LOG_INFO("%s: constructing llama_context\n", __func__);
22
+
23
+ t_start_us = model.t_start_us;
24
+ t_load_us = model.t_load_us;
25
+
26
+ const auto & hparams = model.hparams;
27
+
28
+ cparams.n_seq_max = std::max(1u, params.n_seq_max);
29
+ if (cparams.n_seq_max > LLAMA_MAX_PARALLEL_SEQUENCES) {
30
+ throw std::runtime_error("n_seq_max must be <= " + std::to_string(LLAMA_MAX_PARALLEL_SEQUENCES));
31
+ }
32
+
33
+ cparams.n_threads = params.n_threads;
34
+ cparams.n_threads_batch = params.n_threads_batch;
35
+ cparams.yarn_ext_factor = params.yarn_ext_factor;
36
+ cparams.yarn_attn_factor = params.yarn_attn_factor;
37
+ cparams.yarn_beta_fast = params.yarn_beta_fast;
38
+ cparams.yarn_beta_slow = params.yarn_beta_slow;
39
+ cparams.defrag_thold = params.defrag_thold;
40
+ cparams.embeddings = params.embeddings;
41
+ cparams.offload_kqv = params.offload_kqv;
42
+ cparams.flash_attn = params.flash_attn;
43
+ cparams.no_perf = params.no_perf;
44
+ cparams.pooling_type = params.pooling_type;
45
+ cparams.warmup = false;
46
+
47
+ cparams.n_ctx = params.n_ctx == 0 ? hparams.n_ctx_train : params.n_ctx;
48
+ cparams.rope_freq_base = params.rope_freq_base == 0.0f ? hparams.rope_freq_base_train : params.rope_freq_base;
49
+ cparams.rope_freq_scale = params.rope_freq_scale == 0.0f ? hparams.rope_freq_scale_train : params.rope_freq_scale;
50
+
51
+ cparams.n_ctx_orig_yarn = params.yarn_orig_ctx != 0 ? params.yarn_orig_ctx :
52
+ hparams.n_ctx_orig_yarn != 0 ? hparams.n_ctx_orig_yarn :
53
+ hparams.n_ctx_train;
54
+
55
+ cparams.cb_eval = params.cb_eval;
56
+ cparams.cb_eval_user_data = params.cb_eval_user_data;
57
+
58
+ auto rope_scaling_type = params.rope_scaling_type;
59
+ if (rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED) {
60
+ rope_scaling_type = hparams.rope_scaling_type_train;
61
+ }
62
+
63
+ if (rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_NONE) {
64
+ cparams.rope_freq_scale = 1.0f; // never scale if scaling type is none
65
+ }
66
+
67
+ if (cparams.yarn_ext_factor < 0.0f) { // negative indicates 'not set'
68
+ cparams.yarn_ext_factor = rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_YARN ? 1.0f : 0.0f;
69
+ }
70
+
71
+ cparams.yarn_attn_factor *= hparams.rope_attn_factor;
72
+
73
+ if (cparams.pooling_type == LLAMA_POOLING_TYPE_UNSPECIFIED) {
74
+ if (hparams.pooling_type == LLAMA_POOLING_TYPE_UNSPECIFIED) {
75
+ cparams.pooling_type = LLAMA_POOLING_TYPE_NONE;
76
+ } else {
77
+ cparams.pooling_type = hparams.pooling_type;
78
+ }
79
+ }
80
+
81
+ if (params.attention_type == LLAMA_ATTENTION_TYPE_UNSPECIFIED) {
82
+ cparams.causal_attn = hparams.causal_attn;
83
+ } else {
84
+ cparams.causal_attn = params.attention_type == LLAMA_ATTENTION_TYPE_CAUSAL;
85
+ }
86
+
87
+ // with causal attention, the batch size is limited by the context size
88
+ cparams.n_batch = cparams.causal_attn ? std::min(cparams.n_ctx, params.n_batch) : params.n_batch;
89
+
90
+ // the batch has to be at least GGML_KQ_MASK_PAD because we will be padding the KQ_mask
91
+ // this is required by GPU kernels in order to avoid out-of-bounds accesses (e.g. ggml_flash_attn_ext)
92
+ // ref: https://github.com/ggerganov/llama.cpp/pull/5021
93
+ // TODO: this padding is not needed for the cache-less context so we should probably move it to llama_context_kv_self
94
+ if (cparams.n_batch < GGML_KQ_MASK_PAD) {
95
+ LLAMA_LOG_WARN("%s: n_batch is less than GGML_KQ_MASK_PAD - increasing to %d\n", __func__, GGML_KQ_MASK_PAD);
96
+ cparams.n_batch = GGML_KQ_MASK_PAD;
97
+ }
98
+
99
+ cparams.n_ubatch = std::min(cparams.n_batch, params.n_ubatch == 0 ? params.n_batch : params.n_ubatch);
100
+
101
+ cparams.op_offload = params.op_offload;
102
+
103
+ const uint32_t n_ctx_per_seq = cparams.n_ctx / cparams.n_seq_max;
104
+
105
+ LLAMA_LOG_INFO("%s: n_seq_max = %u\n", __func__, cparams.n_seq_max);
106
+ LLAMA_LOG_INFO("%s: n_ctx = %u\n", __func__, cparams.n_ctx);
107
+ LLAMA_LOG_INFO("%s: n_ctx_per_seq = %u\n", __func__, n_ctx_per_seq);
108
+ LLAMA_LOG_INFO("%s: n_batch = %u\n", __func__, cparams.n_batch);
109
+ LLAMA_LOG_INFO("%s: n_ubatch = %u\n", __func__, cparams.n_ubatch);
110
+ LLAMA_LOG_INFO("%s: causal_attn = %d\n", __func__, cparams.causal_attn);
111
+ LLAMA_LOG_INFO("%s: flash_attn = %d\n", __func__, cparams.flash_attn);
112
+ LLAMA_LOG_INFO("%s: freq_base = %.1f\n", __func__, cparams.rope_freq_base);
113
+ LLAMA_LOG_INFO("%s: freq_scale = %g\n", __func__, cparams.rope_freq_scale);
114
+
115
+ if (n_ctx_per_seq < hparams.n_ctx_train) {
116
+ LLAMA_LOG_WARN("%s: n_ctx_per_seq (%u) < n_ctx_train (%u) -- the full capacity of the model will not be utilized\n",
117
+ __func__, n_ctx_per_seq, hparams.n_ctx_train);
118
+ }
119
+
120
+ if (n_ctx_per_seq > hparams.n_ctx_train) {
121
+ LLAMA_LOG_WARN("%s: n_ctx_per_seq (%u) > n_ctx_train (%u) -- possible training context overflow\n",
122
+ __func__, n_ctx_per_seq, hparams.n_ctx_train);
123
+ }
124
+
125
+ if (!hparams.vocab_only) {
126
+ // GPU backends
127
+ for (auto * dev : model.devices) {
128
+ ggml_backend_t backend = ggml_backend_dev_init(dev, nullptr);
129
+ if (backend == nullptr) {
130
+ throw std::runtime_error(format("failed to initialize %s backend", ggml_backend_dev_name(dev)));
131
+ }
132
+ backends.emplace_back(backend);
133
+ }
134
+
135
+ // add ACCEL backends (such as BLAS)
136
+ for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {
137
+ ggml_backend_dev_t dev = ggml_backend_dev_get(i);
138
+ if (ggml_backend_dev_type(dev) == GGML_BACKEND_DEVICE_TYPE_ACCEL) {
139
+ ggml_backend_t backend = ggml_backend_dev_init(dev, nullptr);
140
+ if (backend == nullptr) {
141
+ throw std::runtime_error(format("failed to initialize %s backend", ggml_backend_dev_name(dev)));
142
+ }
143
+ backends.emplace_back(backend);
144
+ }
145
+ }
146
+
147
+ // add CPU backend
148
+ backend_cpu = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr);
149
+ if (backend_cpu == nullptr) {
150
+ throw std::runtime_error("failed to initialize CPU backend");
151
+ }
152
+ backends.emplace_back(backend_cpu);
153
+
154
+ // create a list of the set_n_threads functions in the backends
155
+ for (auto & backend : backends) {
156
+ ggml_backend_dev_t dev = ggml_backend_get_device(backend.get());
157
+ ggml_backend_reg_t reg = dev ? ggml_backend_dev_backend_reg(dev) : nullptr;
158
+ if (reg) {
159
+ auto ggml_backend_set_n_threads_fn = (ggml_backend_set_n_threads_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_set_n_threads");
160
+ if (ggml_backend_set_n_threads_fn) {
161
+ set_n_threads_fns.emplace_back(backend.get(), ggml_backend_set_n_threads_fn);
162
+ }
163
+ }
164
+ }
165
+
166
+ llama_set_abort_callback(this, params.abort_callback, params.abort_callback_data);
167
+
168
+ // graph outputs buffer
169
+ {
170
+ // resized during inference when a batch uses more outputs
171
+ if ((uint32_t) output_reserve(params.n_seq_max) < params.n_seq_max) {
172
+ throw std::runtime_error("failed to reserve initial output buffer");
173
+ }
174
+
175
+ LLAMA_LOG_INFO("%s: %10s output buffer size = %8.2f MiB\n", __func__,
176
+ ggml_backend_buffer_name (buf_output.get()),
177
+ ggml_backend_buffer_get_size(buf_output.get()) / 1024.0 / 1024.0);
178
+ }
179
+ }
180
+
181
+ // init the memory module
182
+ if (!hparams.vocab_only) {
183
+ llama_memory_params params_mem = {
184
+ /*.type_k =*/ params.type_k,
185
+ /*.type_v =*/ params.type_v,
186
+ /*.swa_full =*/ params.swa_full,
187
+ };
188
+
189
+ memory.reset(model.create_memory(params_mem, cparams));
190
+ }
191
+
192
+ // init backends
193
+ if (!hparams.vocab_only) {
194
+ LLAMA_LOG_DEBUG("%s: enumerating backends\n", __func__);
195
+
196
+ backend_buft.clear();
197
+ backend_ptrs.clear();
198
+
199
+ for (auto & backend : backends) {
200
+ auto * buft = ggml_backend_get_default_buffer_type(backend.get());
201
+ auto backend_type = ggml_backend_dev_type(ggml_backend_get_device(backend.get()));
202
+
203
+ if (backend_type == GGML_BACKEND_DEVICE_TYPE_CPU && !model.devices.empty()) {
204
+ // use the host buffer of the first device CPU for faster transfer of the intermediate state
205
+ auto * dev = model.devices[0];
206
+ auto * host_buft = ggml_backend_dev_host_buffer_type(dev);
207
+ if (host_buft) {
208
+ buft = host_buft;
209
+ }
210
+ }
211
+
212
+ backend_buft.push_back(buft);
213
+ backend_ptrs.push_back(backend.get());
214
+ }
215
+
216
+ LLAMA_LOG_DEBUG("%s: backend_ptrs.size() = %zu\n", __func__, backend_ptrs.size());
217
+
218
+ const size_t max_nodes = this->graph_max_nodes();
219
+
220
+ LLAMA_LOG_DEBUG("%s: max_nodes = %zu\n", __func__, max_nodes);
221
+
222
+ // buffer used to store the computation graph and the tensor meta data
223
+ buf_compute_meta.resize(ggml_tensor_overhead()*max_nodes + ggml_graph_overhead_custom(max_nodes, false));
224
+
225
+ // TODO: move these checks to ggml_backend_sched
226
+ // enabling pipeline parallelism in the scheduler increases memory usage, so it is only done when necessary
227
+ bool pipeline_parallel =
228
+ model.n_devices() > 1 &&
229
+ model.params.n_gpu_layers > (int) model.hparams.n_layer &&
230
+ model.params.split_mode == LLAMA_SPLIT_MODE_LAYER &&
231
+ cparams.offload_kqv &&
232
+ !model.has_tensor_overrides();
233
+
234
+ // pipeline parallelism requires support for async compute and events in all devices
235
+ if (pipeline_parallel) {
236
+ for (auto & backend : backends) {
237
+ auto dev_type = ggml_backend_dev_type(ggml_backend_get_device(backend.get()));
238
+ if (dev_type == GGML_BACKEND_DEVICE_TYPE_CPU) {
239
+ // ignore CPU backend
240
+ continue;
241
+ }
242
+ auto * dev = ggml_backend_get_device(backend.get());
243
+ ggml_backend_dev_props props;
244
+ ggml_backend_dev_get_props(dev, &props);
245
+ if (!props.caps.async || !props.caps.events) {
246
+ // device does not support async compute or events
247
+ pipeline_parallel = false;
248
+ break;
249
+ }
250
+ }
251
+ }
252
+
253
+ sched.reset(ggml_backend_sched_new(backend_ptrs.data(), backend_buft.data(), backend_ptrs.size(), max_nodes, pipeline_parallel, cparams.op_offload));
254
+
255
+ if (pipeline_parallel) {
256
+ LLAMA_LOG_INFO("%s: pipeline parallelism enabled (n_copies=%d)\n", __func__, ggml_backend_sched_get_n_copies(sched.get()));
257
+ }
258
+ }
259
+
260
+ // reserve worst-case graph
261
+ if (!hparams.vocab_only && memory) {
262
+ const uint32_t n_seqs = 1; // TODO: worst-case number of sequences
263
+ const uint32_t n_tokens = std::min(cparams.n_ctx, cparams.n_ubatch);
264
+
265
+ llama_token token = model.vocab.token_bos(); // not actually used by llama_build_graph, but required to choose between token and embedding inputs graph
266
+
267
+ // restore later
268
+ // TODO: something cleaner
269
+ const auto n_outputs_save = n_outputs;
270
+
271
+ LLAMA_LOG_DEBUG("%s: worst-case: n_tokens = %d, n_seqs = %d, n_outputs = %d\n", __func__, n_tokens, n_seqs, n_outputs);
272
+
273
+ int n_splits_pp = -1;
274
+ int n_nodes_pp = -1;
275
+
276
+ int n_splits_tg = -1;
277
+ int n_nodes_tg = -1;
278
+
279
+ // simulate full KV cache
280
+ llama_kv_cache * kv_self = static_cast<llama_kv_cache *>(memory.get());
281
+
282
+ kv_self->set_full();
283
+
284
+ cross.v_embd.clear();
285
+
286
+ // reserve pp graph first so that buffers are only allocated once
287
+ {
288
+ llama_ubatch ubatch_pp = { true, n_tokens, n_tokens / n_seqs, n_seqs, &token, nullptr, nullptr, nullptr, nullptr, nullptr};
289
+
290
+ // max number of outputs
291
+ n_outputs = ubatch_pp.n_tokens;
292
+
293
+ LLAMA_LOG_DEBUG("%s: reserving graph for n_tokens = %d, n_seqs = %d\n", __func__, ubatch_pp.n_tokens, ubatch_pp.n_seqs);
294
+
295
+ auto * gf = graph_init();
296
+ graph_build(ctx_compute.get(), gf, ubatch_pp, LLM_GRAPH_TYPE_DEFAULT);
297
+
298
+ if (!ggml_backend_sched_reserve(sched.get(), gf)) {
299
+ throw std::runtime_error("failed to allocate compute pp buffers");
300
+ }
301
+
302
+ n_splits_pp = ggml_backend_sched_get_n_splits(sched.get());
303
+ n_nodes_pp = ggml_graph_n_nodes(gf);
304
+ }
305
+
306
+ // reserve with tg graph to get the number of splits and nodes
307
+ {
308
+ llama_ubatch ubatch_tg = { true, 1, 1, n_seqs, &token, nullptr, nullptr, nullptr, nullptr, nullptr};
309
+
310
+ n_outputs = ubatch_tg.n_tokens;
311
+
312
+ LLAMA_LOG_DEBUG("%s: reserving graph for n_tokens = %d, n_seqs = %d\n", __func__, ubatch_tg.n_tokens, ubatch_tg.n_seqs);
313
+
314
+ auto * gf = graph_init();
315
+ graph_build(ctx_compute.get(), gf, ubatch_tg, LLM_GRAPH_TYPE_DEFAULT);
316
+
317
+ if (!ggml_backend_sched_reserve(sched.get(), gf)) {
318
+ throw std::runtime_error("failed to allocate compute tg buffers");
319
+ }
320
+
321
+ n_splits_tg = ggml_backend_sched_get_n_splits(sched.get());
322
+ n_nodes_tg = ggml_graph_n_nodes(gf);
323
+ }
324
+
325
+ // reserve again with pp graph to avoid ggml-alloc reallocations during inference
326
+ {
327
+ llama_ubatch ubatch_pp = { true, n_tokens, n_tokens / n_seqs, n_seqs, &token, nullptr, nullptr, nullptr, nullptr, nullptr};
328
+
329
+ n_outputs = ubatch_pp.n_tokens;
330
+
331
+ LLAMA_LOG_DEBUG("%s: reserving graph for n_tokens = %d, n_seqs = %d\n", __func__, ubatch_pp.n_tokens, ubatch_pp.n_seqs);
332
+
333
+ auto * gf = graph_init();
334
+ graph_build(ctx_compute.get(), gf, ubatch_pp, LLM_GRAPH_TYPE_DEFAULT);
335
+
336
+ if (!ggml_backend_sched_reserve(sched.get(), gf)) {
337
+ throw std::runtime_error("failed to allocate compute pp buffers");
338
+ }
339
+ }
340
+
341
+ n_outputs = n_outputs_save;
342
+
343
+ for (size_t i = 0; i < backend_ptrs.size(); ++i) {
344
+ ggml_backend_t backend = backend_ptrs[i];
345
+ ggml_backend_buffer_type_t buft = backend_buft[i];
346
+ size_t size = ggml_backend_sched_get_buffer_size(sched.get(), backend);
347
+ if (size > 1) {
348
+ LLAMA_LOG_INFO("%s: %10s compute buffer size = %8.2f MiB\n", __func__,
349
+ ggml_backend_buft_name(buft),
350
+ size / 1024.0 / 1024.0);
351
+ }
352
+ }
353
+
354
+ if (n_nodes_pp == n_nodes_tg) {
355
+ LLAMA_LOG_INFO("%s: graph nodes = %d\n", __func__, n_nodes_pp);
356
+ } else {
357
+ LLAMA_LOG_INFO("%s: graph nodes = %d (with bs=%d), %d (with bs=1)\n", __func__, n_nodes_pp, n_tokens, n_nodes_tg);
358
+ }
359
+
360
+ if (n_splits_pp == n_splits_tg) {
361
+ LLAMA_LOG_INFO("%s: graph splits = %d\n", __func__, n_splits_pp);
362
+ } else {
363
+ LLAMA_LOG_INFO("%s: graph splits = %d (with bs=%d), %d (with bs=1)\n", __func__, n_splits_pp, n_tokens, n_splits_tg);
364
+ }
365
+ }
366
+ }
367
+
368
+ llama_context::~llama_context() {
369
+ ggml_opt_free(opt_ctx);
370
+ }
371
+
372
+ void llama_context::synchronize() {
373
+ ggml_backend_sched_synchronize(sched.get());
374
+
375
+ // FIXME: if multiple single tokens are evaluated without a synchronization,
376
+ // the stats will be added to the prompt evaluation stats
377
+ // this should only happen when using batch size 1 to evaluate a batch
378
+
379
+ // add the evaluation to the stats
380
+ if (n_queued_tokens == 1) {
381
+ if (!cparams.no_perf) {
382
+ t_eval_us += ggml_time_us() - t_compute_start_us;
383
+ }
384
+ n_eval++;
385
+ } else if (n_queued_tokens > 1) {
386
+ if (!cparams.no_perf) {
387
+ t_p_eval_us += ggml_time_us() - t_compute_start_us;
388
+ }
389
+ n_p_eval += n_queued_tokens;
390
+ }
391
+
392
+ // get a more accurate load time, upon first eval
393
+ if (n_queued_tokens > 0 && !has_evaluated_once) {
394
+ t_load_us = ggml_time_us() - t_start_us;
395
+ has_evaluated_once = true;
396
+ }
397
+
398
+ n_queued_tokens = 0;
399
+ t_compute_start_us = 0;
400
+ }
401
+
402
+ const llama_model & llama_context::get_model() const {
403
+ return model;
404
+ }
405
+
406
+ const llama_cparams & llama_context::get_cparams() const {
407
+ return cparams;
408
+ }
409
+
410
+ ggml_backend_sched_t llama_context::get_sched() const {
411
+ return sched.get();
412
+ }
413
+
414
+ ggml_context * llama_context::get_ctx_compute() const {
415
+ return ctx_compute.get();
416
+ }
417
+
418
+ uint32_t llama_context::n_ctx() const {
419
+ return cparams.n_ctx;
420
+ }
421
+
422
+ uint32_t llama_context::n_ctx_per_seq() const {
423
+ return cparams.n_ctx / cparams.n_seq_max;
424
+ }
425
+
426
+ uint32_t llama_context::n_batch() const {
427
+ return cparams.n_batch;
428
+ }
429
+
430
+ uint32_t llama_context::n_ubatch() const {
431
+ return cparams.n_ubatch;
432
+ }
433
+
434
+ uint32_t llama_context::n_seq_max() const {
435
+ return cparams.n_seq_max;
436
+ }
437
+
438
+ uint32_t llama_context::n_threads() const {
439
+ return cparams.n_threads;
440
+ }
441
+
442
+ uint32_t llama_context::n_threads_batch() const {
443
+ return cparams.n_threads_batch;
444
+ }
445
+
446
+ llama_kv_cache * llama_context::get_kv_self() {
447
+ llama_kv_cache * kv_self = static_cast<llama_kv_cache *>(memory.get());
448
+ return kv_self;
449
+ }
450
+
451
+ const llama_kv_cache * llama_context::get_kv_self() const {
452
+ llama_kv_cache * kv_self = static_cast<llama_kv_cache *>(memory.get());
453
+ return kv_self;
454
+ }
455
+
456
+ void llama_context::kv_self_update() {
457
+ bool need_reserve = false;
458
+
459
+ llama_kv_cache * kv_self = static_cast<llama_kv_cache *>(memory.get());
460
+
461
+ need_reserve = kv_self->update(*this);
462
+
463
+ // reserve a worst case graph if needed
464
+ if (need_reserve) {
465
+ LLAMA_LOG_DEBUG("%s: reserving a worst case graph\n", __func__);
466
+
467
+ // build worst-case graph
468
+ uint32_t n_seqs = 1; // TODO: worst-case number of sequences
469
+ uint32_t n_tokens = std::min(cparams.n_ctx, cparams.n_ubatch);
470
+
471
+ // simulate full KV cache
472
+ kv_self->set_full();
473
+
474
+ llama_token token = model.vocab.token_bos(); // not actually used by llama_build_graph, but required to choose between token and embedding inputs graph
475
+ llama_ubatch ubatch = { true, n_tokens, n_tokens / n_seqs, n_seqs, &token, nullptr, nullptr, nullptr, nullptr, nullptr};
476
+
477
+ auto * gf = graph_init();
478
+ graph_build(ctx_compute.get(), gf, ubatch, LLM_GRAPH_TYPE_DEFAULT);
479
+
480
+ // initialize scheduler with the worst-case graph
481
+ ggml_backend_sched_reset(sched.get());
482
+ if (!ggml_backend_sched_reserve(sched.get(), gf)) {
483
+ LLAMA_LOG_ERROR("%s: failed to allocate compute buffers\n", __func__);
484
+ }
485
+ }
486
+ }
487
+
488
+ enum llama_pooling_type llama_context::pooling_type() const {
489
+ return cparams.pooling_type;
490
+ }
491
+
492
+ float * llama_context::get_logits() {
493
+ return logits;
494
+ }
495
+
496
+ float * llama_context::get_logits_ith(int32_t i) {
497
+ int32_t j = -1;
498
+
499
+ try {
500
+ if (logits == nullptr) {
501
+ throw std::runtime_error("no logits");
502
+ }
503
+
504
+ if (i < 0) {
505
+ j = n_outputs + i;
506
+ if (j < 0) {
507
+ throw std::runtime_error(format("negative index out of range [0, %d)", n_outputs));
508
+ }
509
+ } else if ((size_t) i >= output_ids.size()) {
510
+ throw std::runtime_error(format("out of range [0, %zu)", output_ids.size()));
511
+ } else {
512
+ j = output_ids[i];
513
+ }
514
+
515
+ if (j < 0) {
516
+ throw std::runtime_error(format("batch.logits[%d] != true", i));
517
+ }
518
+ if (j >= n_outputs) {
519
+ // This should not happen
520
+ throw std::runtime_error(format("corrupt output buffer (j=%d, n_outputs=%d)", j, n_outputs));
521
+ }
522
+
523
+ return logits + j*model.vocab.n_tokens();
524
+ } catch (const std::exception & err) {
525
+ LLAMA_LOG_ERROR("%s: invalid logits id %d, reason: %s\n", __func__, i, err.what());
526
+ #ifndef NDEBUG
527
+ GGML_ABORT("fatal error");
528
+ #else
529
+ return nullptr;
530
+ #endif
531
+ }
532
+ }
533
+
534
+ float * llama_context::get_embeddings() {
535
+ return embd;
536
+ }
537
+
538
+ float * llama_context::get_embeddings_ith(int32_t i) {
539
+ int32_t j = -1;
540
+
541
+ try {
542
+ if (embd == nullptr) {
543
+ throw std::runtime_error("no embeddings");
544
+ }
545
+
546
+ if (i < 0) {
547
+ j = n_outputs + i;
548
+ if (j < 0) {
549
+ throw std::runtime_error(format("negative index out of range [0, %d)", n_outputs));
550
+ }
551
+ } else if ((size_t) i >= output_ids.size()) {
552
+ throw std::runtime_error(format("out of range [0, %zu)", output_ids.size()));
553
+ } else {
554
+ j = output_ids[i];
555
+ }
556
+
557
+ if (j < 0) {
558
+ throw std::runtime_error(format("batch.logits[%d] != true", i));
559
+ }
560
+ if (j >= n_outputs) {
561
+ // This should not happen
562
+ throw std::runtime_error(format("corrupt output buffer (j=%d, n_outputs=%d)", j, n_outputs));
563
+ }
564
+
565
+ return embd + j*model.hparams.n_embd;
566
+ } catch (const std::exception & err) {
567
+ LLAMA_LOG_ERROR("%s: invalid embeddings id %d, reason: %s\n", __func__, i, err.what());
568
+ #ifndef NDEBUG
569
+ GGML_ABORT("fatal error");
570
+ #else
571
+ return nullptr;
572
+ #endif
573
+ }
574
+ }
575
+
576
+ float * llama_context::get_embeddings_seq(llama_seq_id seq_id) {
577
+ auto it = embd_seq.find(seq_id);
578
+ if (it == embd_seq.end()) {
579
+ return nullptr;
580
+ }
581
+
582
+ return it->second.data();
583
+ }
584
+
585
+ void llama_context::attach_threadpool(
586
+ ggml_threadpool_t threadpool,
587
+ ggml_threadpool_t threadpool_batch) {
588
+ LLAMA_LOG_DEBUG("%s: call\n", __func__);
589
+
590
+ this->threadpool = threadpool;
591
+ this->threadpool_batch = threadpool_batch ? threadpool_batch : threadpool;
592
+ }
593
+
594
+ void llama_context::detach_threadpool() {
595
+ LLAMA_LOG_DEBUG("%s: call\n", __func__);
596
+
597
+ this->threadpool = nullptr;
598
+ this->threadpool_batch = nullptr;
599
+ }
600
+
601
+ void llama_context::set_n_threads(int32_t n_threads, int32_t n_threads_batch) {
602
+ LLAMA_LOG_DEBUG("%s: n_threads = %d, n_threads_batch = %d\n", __func__, n_threads, n_threads_batch);
603
+
604
+ cparams.n_threads = n_threads;
605
+ cparams.n_threads_batch = n_threads_batch;
606
+ }
607
+
608
+ void llama_context::set_abort_callback(bool (*abort_callback)(void * data), void * abort_callback_data) {
609
+ LLAMA_LOG_DEBUG("%s: call\n", __func__);
610
+
611
+ this->abort_callback = abort_callback;
612
+ this->abort_callback_data = abort_callback_data;
613
+
614
+ for (auto & backend : backends) {
615
+ auto * reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend.get()));
616
+ auto * set_abort_callback_fn = (ggml_backend_set_abort_callback_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_set_abort_callback");
617
+ if (set_abort_callback_fn) {
618
+ set_abort_callback_fn(backend.get(), this->abort_callback, this->abort_callback_data);
619
+ }
620
+ }
621
+ }
622
+
623
+ void llama_context::set_embeddings(bool value) {
624
+ LLAMA_LOG_DEBUG("%s: value = %d\n", __func__, value);
625
+
626
+ cparams.embeddings = value;
627
+ }
628
+
629
+ void llama_context::set_causal_attn(bool value) {
630
+ LLAMA_LOG_DEBUG("%s: value = %d\n", __func__, value);
631
+
632
+ cparams.causal_attn = value;
633
+ }
634
+
635
+ void llama_context::set_warmup(bool value) {
636
+ LLAMA_LOG_DEBUG("%s: value = %d\n", __func__, value);
637
+
638
+ cparams.warmup = value;
639
+ }
640
+
641
+ void llama_context::set_adapter_lora(
642
+ llama_adapter_lora * adapter,
643
+ float scale) {
644
+ LLAMA_LOG_DEBUG("%s: adapter = %p, scale = %f\n", __func__, (void *) adapter, scale);
645
+
646
+ loras[adapter] = scale;
647
+ }
648
+
649
+ bool llama_context::rm_adapter_lora(
650
+ llama_adapter_lora * adapter) {
651
+ LLAMA_LOG_DEBUG("%s: adapter = %p\n", __func__, (void *) adapter);
652
+
653
+ auto pos = loras.find(adapter);
654
+ if (pos != loras.end()) {
655
+ loras.erase(pos);
656
+ return true;
657
+ }
658
+
659
+ return false;
660
+ }
661
+
662
+ void llama_context::clear_adapter_lora() {
663
+ LLAMA_LOG_DEBUG("%s: call\n", __func__);
664
+
665
+ loras.clear();
666
+ }
667
+
668
+ bool llama_context::apply_adapter_cvec(
669
+ const float * data,
670
+ size_t len,
671
+ int32_t n_embd,
672
+ int32_t il_start,
673
+ int32_t il_end) {
674
+ LLAMA_LOG_DEBUG("%s: il_start = %d, il_end = %d\n", __func__, il_start, il_end);
675
+
676
+ return cvec.apply(model, data, len, n_embd, il_start, il_end);
677
+ }
678
+
679
+ int llama_context::encode(llama_batch & inp_batch) {
680
+ if (inp_batch.n_tokens == 0) {
681
+ LLAMA_LOG_ERROR("%s: n_tokens == 0\n", __func__);
682
+ return -1;
683
+ }
684
+
685
+ // temporary allocate memory for the input batch if needed
686
+ // note: during encode, we always pass the full sequence starting from pos = 0
687
+ llama_batch_allocr batch_allocr(inp_batch, inp_batch.pos ? -1 : 0);
688
+
689
+ const llama_batch & batch = batch_allocr.batch;
690
+ const int32_t n_tokens = batch.n_tokens;
691
+
692
+ const auto & hparams = model.hparams;
693
+
694
+ GGML_ASSERT((!batch.token && batch.embd) || (batch.token && !batch.embd)); // NOLINT
695
+
696
+ // TODO: move the validation to the llama_batch_allocr
697
+ if (batch.token) {
698
+ for (int32_t i = 0; i < n_tokens; ++i) {
699
+ if (batch.token[i] < 0 || (uint32_t) batch.token[i] >= model.vocab.n_tokens()) {
700
+ LLAMA_LOG_ERROR("%s: invalid token[%d] = %d\n", __func__, i, batch.token[i]);
701
+ return -1;
702
+ }
703
+
704
+ if (batch.seq_id && (batch.seq_id[i][0] < 0 || batch.seq_id[i][0] >= LLAMA_MAX_PARALLEL_SEQUENCES)) {
705
+ LLAMA_LOG_ERROR("%s: invalid seq_id[%d] = %d > %d\n", __func__, i, batch.seq_id[i][0], LLAMA_MAX_PARALLEL_SEQUENCES);
706
+ throw -1;
707
+ }
708
+ }
709
+ }
710
+
711
+ // micro-batching is not possible for non-causal encoding, so we process the batch in a single shot
712
+ GGML_ASSERT(cparams.n_ubatch >= (uint32_t) n_tokens && "encoder requires n_ubatch >= n_tokens");
713
+
714
+ if (t_compute_start_us == 0) {
715
+ t_compute_start_us = ggml_time_us();
716
+ }
717
+
718
+ embd_seq.clear();
719
+
720
+ n_queued_tokens += n_tokens;
721
+
722
+ const int64_t n_embd = hparams.n_embd;
723
+
724
+ llama_sbatch sbatch = llama_sbatch(batch, n_embd, /* simple_split */ true, /* logits_all */ true);
725
+
726
+ const llama_ubatch ubatch = sbatch.split_simple(n_tokens);
727
+
728
+ // reserve output buffer
729
+ if (output_reserve(n_tokens) < n_tokens) {
730
+ LLAMA_LOG_ERROR("%s: could not reserve space for batch with %u outputs\n", __func__, n_tokens);
731
+ return -2;
732
+ };
733
+
734
+ for (int32_t i = 0; i < n_tokens; ++i) {
735
+ output_ids[i] = i;
736
+ }
737
+
738
+ n_outputs = n_tokens;
739
+
740
+ //batch_manager->prepare(ubatch);
741
+
742
+ ggml_backend_sched_reset(sched.get());
743
+ ggml_backend_sched_set_eval_callback(sched.get(), cparams.cb_eval, cparams.cb_eval_user_data);
744
+
745
+ const auto causal_attn_org = cparams.causal_attn;
746
+
747
+ // always use non-causal attention for encoder graphs
748
+ // TODO: this is a tmp solution until we have a proper way to support enc-dec models
749
+ // ref: https://github.com/ggml-org/llama.cpp/pull/12181#issuecomment-2730451223
750
+ cparams.causal_attn = false;
751
+
752
+ auto * gf = graph_init();
753
+ auto res = graph_build(ctx_compute.get(), gf, ubatch, LLM_GRAPH_TYPE_ENCODER);
754
+
755
+ ggml_backend_sched_alloc_graph(sched.get(), gf);
756
+
757
+ res->set_inputs(&ubatch);
758
+
759
+ cparams.causal_attn = causal_attn_org;
760
+
761
+ const auto compute_status = graph_compute(gf, n_tokens > 1);
762
+ switch (compute_status) {
763
+ case GGML_STATUS_SUCCESS:
764
+ break;
765
+ case GGML_STATUS_ABORTED:
766
+ return 2;
767
+ case GGML_STATUS_ALLOC_FAILED:
768
+ return -2;
769
+ case GGML_STATUS_FAILED:
770
+ default:
771
+ return -3;
772
+ }
773
+
774
+ auto * t_embd = res->get_embd_pooled() ? res->get_embd_pooled() : res->get_embd();
775
+
776
+ // extract embeddings
777
+ if (t_embd) {
778
+ ggml_backend_t backend_embd = ggml_backend_sched_get_tensor_backend(sched.get(), t_embd);
779
+ GGML_ASSERT(backend_embd != nullptr);
780
+
781
+ switch (cparams.pooling_type) {
782
+ case LLAMA_POOLING_TYPE_NONE:
783
+ {
784
+ // extract token embeddings
785
+ GGML_ASSERT(embd != nullptr);
786
+
787
+ GGML_ASSERT(n_tokens*n_embd <= (int64_t) embd_size);
788
+ ggml_backend_tensor_get_async(backend_embd, t_embd, embd, 0, n_tokens*n_embd*sizeof(float));
789
+ } break;
790
+ case LLAMA_POOLING_TYPE_MEAN:
791
+ case LLAMA_POOLING_TYPE_CLS:
792
+ case LLAMA_POOLING_TYPE_LAST:
793
+ {
794
+ // extract sequence embeddings
795
+ auto & embd_seq_out = embd_seq;
796
+ embd_seq_out.clear();
797
+
798
+ GGML_ASSERT(!ubatch.equal_seqs); // TODO: handle equal splits
799
+
800
+ for (int32_t i = 0; i < n_tokens; i++) {
801
+ const llama_seq_id seq_id = ubatch.seq_id[i][0];
802
+ if (embd_seq_out.find(seq_id) != embd_seq_out.end()) {
803
+ continue;
804
+ }
805
+ embd_seq_out[seq_id].resize(n_embd);
806
+ ggml_backend_tensor_get_async(backend_embd, t_embd, embd_seq_out[seq_id].data(), (n_embd*seq_id)*sizeof(float), n_embd*sizeof(float));
807
+ }
808
+ } break;
809
+ case LLAMA_POOLING_TYPE_RANK:
810
+ {
811
+ // extract the rerank score - a single float per sequence
812
+ auto & embd_seq_out = embd_seq;
813
+
814
+ for (uint32_t s = 0; s < ubatch.n_seqs; ++s) {
815
+ const llama_seq_id seq_id = ubatch.seq_id[s][0];
816
+ if (embd_seq_out.find(seq_id) != embd_seq_out.end()) {
817
+ continue;
818
+ }
819
+ embd_seq_out[seq_id].resize(1);
820
+ ggml_backend_tensor_get_async(backend_embd, t_embd, embd_seq_out[seq_id].data(), (seq_id)*sizeof(float), sizeof(float));
821
+ }
822
+ } break;
823
+ case LLAMA_POOLING_TYPE_UNSPECIFIED:
824
+ {
825
+ GGML_ABORT("unknown pooling type");
826
+ }
827
+ }
828
+ }
829
+
830
+ // Reset state for the next token before backend sync, to allow the CPU activities in the reset to
831
+ // overlap with device computation.
832
+ ggml_backend_sched_reset(sched.get());
833
+
834
+ // TODO: hacky solution
835
+ if (model.arch == LLM_ARCH_T5 && t_embd) {
836
+ //cross.t_embd = t_embd;
837
+
838
+ synchronize();
839
+
840
+ cross.n_embd = t_embd->ne[0];
841
+ cross.n_enc = t_embd->ne[1];
842
+ cross.v_embd.resize(cross.n_embd*cross.n_enc);
843
+ memcpy(cross.v_embd.data(), embd, ggml_nbytes(t_embd));
844
+
845
+ // remember the sequence ids used during the encoding - needed for cross attention later
846
+ cross.seq_ids_enc.resize(n_tokens);
847
+ for (int32_t i = 0; i < n_tokens; i++) {
848
+ cross.seq_ids_enc[i].clear();
849
+ for (int s = 0; s < ubatch.n_seq_id[i]; s++) {
850
+ llama_seq_id seq_id = ubatch.seq_id[i][s];
851
+ cross.seq_ids_enc[i].insert(seq_id);
852
+ }
853
+ }
854
+ }
855
+
856
+ return 0;
857
+ }
858
+
859
+ int llama_context::decode(llama_batch & inp_batch) {
860
+ if (!memory) {
861
+ LLAMA_LOG_DEBUG("%s: cannot decode batches with this context (calling encode() instead)\n", __func__);
862
+ return encode(inp_batch);
863
+ }
864
+
865
+ if (inp_batch.n_tokens == 0) {
866
+ LLAMA_LOG_ERROR("%s: n_tokens == 0\n", __func__);
867
+ return -1;
868
+ }
869
+
870
+ if (!inp_batch.pos) {
871
+ if (inp_batch.seq_id) {
872
+ LLAMA_LOG_ERROR("%s: pos == NULL, but seq_id != NULL\n", __func__);
873
+ return -1;
874
+ }
875
+ }
876
+
877
+ llama_kv_cache * kv_self = static_cast<llama_kv_cache *>(memory.get());
878
+
879
+ // temporary allocate memory for the input batch if needed
880
+ llama_batch_allocr batch_allocr(inp_batch, inp_batch.pos ? -1 : kv_self->seq_pos_max(0) + 1);
881
+
882
+ const llama_batch & batch = batch_allocr.batch;
883
+
884
+ const auto & vocab = model.vocab;
885
+ const auto & hparams = model.hparams;
886
+
887
+ const int32_t n_vocab = vocab.n_tokens();
888
+
889
+ const int64_t n_tokens_all = batch.n_tokens;
890
+ const int64_t n_embd = hparams.n_embd;
891
+
892
+ llama_kv_cache_guard kv_guard(kv_self);
893
+
894
+ GGML_ASSERT((!batch.token && batch.embd) || (batch.token && !batch.embd)); // NOLINT
895
+
896
+ // TODO: move the validation to the llama_batch_allocr
897
+ if (batch.token) {
898
+ for (int64_t i = 0; i < n_tokens_all; ++i) {
899
+ if (batch.token[i] < 0 || (uint32_t) batch.token[i] >= model.vocab.n_tokens()) {
900
+ LLAMA_LOG_ERROR("%s: invalid token[%" PRId64 "] = %d\n", __func__, i, batch.token[i]);
901
+ return -1;
902
+ }
903
+
904
+ if (batch.seq_id && (batch.seq_id[i][0] < 0 || batch.seq_id[i][0] >= LLAMA_MAX_PARALLEL_SEQUENCES)) {
905
+ LLAMA_LOG_ERROR("%s: invalid seq_id[%" PRId64 "] = %d >= %d\n", __func__, i, batch.seq_id[i][0], LLAMA_MAX_PARALLEL_SEQUENCES);
906
+ return -1;
907
+ }
908
+ }
909
+ }
910
+
911
+ GGML_ASSERT(n_tokens_all <= cparams.n_batch);
912
+
913
+ GGML_ASSERT((cparams.causal_attn || cparams.n_ubatch >= n_tokens_all) && "non-causal attention requires n_ubatch >= n_tokens");
914
+
915
+ if (t_compute_start_us == 0) {
916
+ t_compute_start_us = ggml_time_us();
917
+ }
918
+ n_queued_tokens += n_tokens_all;
919
+
920
+ // this indicates we are doing pooled embedding, so we ignore batch.logits and output all tokens
921
+ const bool embd_pooled = cparams.embeddings && cparams.pooling_type != LLAMA_POOLING_TYPE_NONE;
922
+
923
+ embd_seq.clear();
924
+
925
+ int64_t n_outputs_all = 0;
926
+
927
+ // count outputs
928
+ if (batch.logits && !embd_pooled) {
929
+ for (uint32_t i = 0; i < n_tokens_all; ++i) {
930
+ n_outputs_all += batch.logits[i] != 0;
931
+ }
932
+ } else if (embd_pooled) {
933
+ n_outputs_all = n_tokens_all;
934
+ } else {
935
+ // keep last output only
936
+ n_outputs_all = 1;
937
+ }
938
+
939
+ llama_sbatch sbatch = kv_self->sbatch_init(batch, /* logits_all */ n_outputs_all == n_tokens_all);
940
+
941
+ // reserve output buffer
942
+ if (output_reserve(n_outputs_all) < n_outputs_all) {
943
+ LLAMA_LOG_ERROR("%s: could not reserve space for batch with %" PRId64 " outputs\n", __func__, n_outputs_all);
944
+ return -2;
945
+ };
946
+
947
+ // handle any pending defrags/shifts
948
+ kv_self_update();
949
+
950
+ int64_t n_outputs_prev = 0;
951
+
952
+ while (sbatch.n_tokens > 0) {
953
+ llama_ubatch ubatch = kv_self->ubatch_next(sbatch, cparams.n_ubatch, embd_pooled);
954
+
955
+ // count the outputs in this u_batch
956
+ {
957
+ int32_t n_outputs_new = 0;
958
+
959
+ if (n_outputs_all == n_tokens_all) {
960
+ n_outputs_new = ubatch.n_tokens;
961
+ } else {
962
+ GGML_ASSERT(ubatch.output);
963
+ for (uint32_t i = 0; i < ubatch.n_tokens; i++) {
964
+ n_outputs_new += (int32_t) (ubatch.output[i] != 0);
965
+ }
966
+ }
967
+
968
+ // needs to happen before the graph is built
969
+ n_outputs = n_outputs_new;
970
+ }
971
+
972
+ // find KV slot
973
+ if (!kv_self->find_slot(ubatch)) {
974
+ return 1;
975
+ }
976
+
977
+ ggml_backend_sched_reset(sched.get());
978
+ ggml_backend_sched_set_eval_callback(sched.get(), cparams.cb_eval, cparams.cb_eval_user_data);
979
+
980
+ auto * gf = graph_init();
981
+ auto res = graph_build(ctx_compute.get(), gf, ubatch, LLM_GRAPH_TYPE_DECODER);
982
+
983
+ // LLAMA_LOG_INFO("graph build time: %.3f ms (%d nodes, %d leafs)\n", (ggml_time_us() - t_start_us)/1000.0, gf->n_nodes, gf->n_leafs);
984
+
985
+ ggml_backend_sched_alloc_graph(sched.get(), gf);
986
+
987
+ res->set_inputs(&ubatch);
988
+
989
+ const auto compute_status = graph_compute(gf, ubatch.n_tokens > 1);
990
+ if (compute_status != GGML_STATUS_SUCCESS) {
991
+ switch (compute_status) {
992
+ case GGML_STATUS_ABORTED:
993
+ return 2;
994
+ case GGML_STATUS_ALLOC_FAILED:
995
+ return -2;
996
+ case GGML_STATUS_FAILED:
997
+ default:
998
+ return -3;
999
+ }
1000
+ }
1001
+
1002
+ // plot the computation graph in dot format (for debugging purposes)
1003
+ //if (n_past%100 == 0) {
1004
+ // ggml_graph_dump_dot(gf, NULL, "llama.dot");
1005
+ //}
1006
+
1007
+ auto * t_logits = cparams.embeddings ? nullptr : res->get_logits();
1008
+ auto * t_embd = cparams.embeddings ? res->get_embd() : nullptr;
1009
+
1010
+ if (t_embd && res->get_embd_pooled()) {
1011
+ t_embd = res->get_embd_pooled();
1012
+ }
1013
+
1014
+ // extract logits
1015
+ if (t_logits && n_outputs > 0) {
1016
+ ggml_backend_t backend_res = ggml_backend_sched_get_tensor_backend(sched.get(), t_logits);
1017
+ GGML_ASSERT(backend_res != nullptr);
1018
+ GGML_ASSERT(logits != nullptr);
1019
+
1020
+ float * logits_out = logits + n_outputs_prev*n_vocab;
1021
+
1022
+ if (n_outputs) {
1023
+ GGML_ASSERT( n_outputs_prev + n_outputs <= n_outputs_all);
1024
+ GGML_ASSERT((n_outputs_prev + n_outputs)*n_vocab <= (int64_t) logits_size);
1025
+ ggml_backend_tensor_get_async(backend_res, t_logits, logits_out, 0, n_outputs*n_vocab*sizeof(float));
1026
+ }
1027
+ }
1028
+
1029
+ // extract embeddings
1030
+ if (t_embd && n_outputs > 0) {
1031
+ ggml_backend_t backend_embd = ggml_backend_sched_get_tensor_backend(sched.get(), t_embd);
1032
+ GGML_ASSERT(backend_embd != nullptr);
1033
+
1034
+ switch (cparams.pooling_type) {
1035
+ case LLAMA_POOLING_TYPE_NONE:
1036
+ {
1037
+ // extract token embeddings
1038
+ GGML_ASSERT(embd != nullptr);
1039
+ float * embd_out = embd + n_outputs_prev*n_embd;
1040
+
1041
+ if (n_outputs) {
1042
+ GGML_ASSERT( n_outputs_prev + n_outputs <= n_outputs_all);
1043
+ GGML_ASSERT((n_outputs_prev + n_outputs)*n_embd <= (int64_t) embd_size);
1044
+ ggml_backend_tensor_get_async(backend_embd, t_embd, embd_out, 0, n_outputs*n_embd*sizeof(float));
1045
+ }
1046
+ } break;
1047
+ case LLAMA_POOLING_TYPE_MEAN:
1048
+ case LLAMA_POOLING_TYPE_CLS:
1049
+ case LLAMA_POOLING_TYPE_LAST:
1050
+ {
1051
+ // extract sequence embeddings (cleared before processing each batch)
1052
+ auto & embd_seq_out = embd_seq;
1053
+
1054
+ for (uint32_t s = 0; s < ubatch.n_seqs; ++s) {
1055
+ const llama_seq_id seq_id = ubatch.seq_id[s][0];
1056
+ if (embd_seq_out.find(seq_id) != embd_seq_out.end()) {
1057
+ continue;
1058
+ }
1059
+ embd_seq_out[seq_id].resize(n_embd);
1060
+ ggml_backend_tensor_get_async(backend_embd, t_embd, embd_seq_out[seq_id].data(), (n_embd*seq_id)*sizeof(float), n_embd*sizeof(float));
1061
+ }
1062
+ } break;
1063
+ case LLAMA_POOLING_TYPE_RANK:
1064
+ {
1065
+ // extract the rerank score - a single float per sequence
1066
+ auto & embd_seq_out = embd_seq;
1067
+
1068
+ for (uint32_t s = 0; s < ubatch.n_seqs; ++s) {
1069
+ const llama_seq_id seq_id = ubatch.seq_id[s][0];
1070
+ if (embd_seq_out.find(seq_id) != embd_seq_out.end()) {
1071
+ continue;
1072
+ }
1073
+ embd_seq_out[seq_id].resize(1);
1074
+ ggml_backend_tensor_get_async(backend_embd, t_embd, embd_seq_out[seq_id].data(), (seq_id)*sizeof(float), sizeof(float));
1075
+ }
1076
+ } break;
1077
+ case LLAMA_POOLING_TYPE_UNSPECIFIED:
1078
+ {
1079
+ GGML_ABORT("unknown pooling type");
1080
+ }
1081
+ }
1082
+ }
1083
+
1084
+ n_outputs_prev += n_outputs;
1085
+ }
1086
+
1087
+ // finalize the batch processing
1088
+ kv_guard.commit();
1089
+
1090
+ // set to total number of outputs in the batch, for use in llama_get_logits_ith
1091
+ n_outputs = n_outputs_all;
1092
+
1093
+ // set output mappings
1094
+ {
1095
+ bool sorted_output = true;
1096
+
1097
+ auto & out_ids = sbatch.out_ids;
1098
+
1099
+ GGML_ASSERT(out_ids.size() == (size_t) n_outputs_all);
1100
+
1101
+ for (int64_t i = 0; i < n_outputs_all; ++i) {
1102
+ int64_t out_id = out_ids[i];
1103
+ output_ids[out_id] = i;
1104
+ if (out_id != i) {
1105
+ sorted_output = false;
1106
+ }
1107
+ }
1108
+
1109
+ // make the outputs have the same order they had in the user-provided batch
1110
+ // note: this is mostly relevant for recurrent models atm
1111
+ if (!sorted_output) {
1112
+ const uint32_t n_vocab = model.vocab.n_tokens();
1113
+ const uint32_t n_embd = model.hparams.n_embd;
1114
+
1115
+ GGML_ASSERT((size_t) n_outputs == out_ids.size());
1116
+
1117
+ // TODO: is there something more efficient which also minimizes swaps?
1118
+ // selection sort, to minimize swaps (from https://en.wikipedia.org/wiki/Selection_sort)
1119
+ for (int32_t i = 0; i < n_outputs - 1; ++i) {
1120
+ int32_t j_min = i;
1121
+ for (int32_t j = i + 1; j < n_outputs; ++j) {
1122
+ if (out_ids[j] < out_ids[j_min]) {
1123
+ j_min = j;
1124
+ }
1125
+ }
1126
+ if (j_min == i) { continue; }
1127
+ std::swap(out_ids[i], out_ids[j_min]);
1128
+ if (logits_size > 0) {
1129
+ for (uint32_t k = 0; k < n_vocab; k++) {
1130
+ std::swap(logits[i*n_vocab + k], logits[j_min*n_vocab + k]);
1131
+ }
1132
+ }
1133
+ if (embd_size > 0) {
1134
+ for (uint32_t k = 0; k < n_embd; k++) {
1135
+ std::swap(embd[i*n_embd + k], embd[j_min*n_embd + k]);
1136
+ }
1137
+ }
1138
+ }
1139
+ std::fill(output_ids.begin(), output_ids.end(), -1);
1140
+ for (int32_t i = 0; i < n_outputs; ++i) {
1141
+ output_ids[out_ids[i]] = i;
1142
+ }
1143
+ }
1144
+ }
1145
+
1146
+ // wait for the computation to finish (automatically done when obtaining the model output)
1147
+ //synchronize();
1148
+
1149
+ // decide if we need to defrag the kv cache
1150
+ if (cparams.defrag_thold > 0.0f) {
1151
+ kv_self->defrag_sched(cparams.defrag_thold);
1152
+ }
1153
+
1154
+ // Reset state for the next token before backend sync, to allow the CPU activities in the reset to
1155
+ // overlap with device computation.
1156
+ ggml_backend_sched_reset(sched.get());
1157
+
1158
+ return 0;
1159
+ }
1160
+
1161
+ //
1162
+ // output
1163
+ //
1164
+
1165
+ int32_t llama_context::output_reserve(int32_t n_outputs) {
1166
+ const auto & hparams = model.hparams;
1167
+ const auto & vocab = model.vocab;
1168
+
1169
+ const int64_t n_outputs_max = std::max<int64_t>(n_outputs, n_seq_max());
1170
+
1171
+ const auto n_batch = cparams.n_batch;
1172
+ const auto n_vocab = vocab.n_tokens();
1173
+ const auto n_embd = hparams.n_embd;
1174
+
1175
+ // TODO: use a per-batch flag for logits presence instead
1176
+ bool has_logits = !cparams.embeddings;
1177
+ bool has_embd = cparams.embeddings && (cparams.pooling_type == LLAMA_POOLING_TYPE_NONE);
1178
+
1179
+ // TODO: hacky enc-dec support
1180
+ if (model.arch == LLM_ARCH_T5) {
1181
+ has_logits = true;
1182
+ has_embd = true;
1183
+ }
1184
+
1185
+ logits_size = has_logits ? n_vocab*n_outputs_max : 0;
1186
+ embd_size = has_embd ? n_embd*n_outputs_max : 0;
1187
+
1188
+ if (output_ids.empty()) {
1189
+ // init, never resized afterwards
1190
+ output_ids.resize(n_batch);
1191
+ }
1192
+
1193
+ const size_t prev_size = buf_output ? ggml_backend_buffer_get_size(buf_output.get()) : 0;
1194
+ const size_t new_size = (logits_size + embd_size) * sizeof(float);
1195
+
1196
+ // alloc only when more than the current capacity is required
1197
+ // TODO: also consider shrinking the buffer
1198
+ if (!buf_output || prev_size < new_size) {
1199
+ if (buf_output) {
1200
+ #ifndef NDEBUG
1201
+ // This doesn't happen often, but may be annoying in some cases (like the HellaSwag benchmark)
1202
+ LLAMA_LOG_INFO("%s: reallocating output buffer from size %.02f MiB to %.02f MiB\n", __func__, prev_size / 1024.0 / 1024.0, new_size / 1024.0 / 1024.0);
1203
+ #endif
1204
+ buf_output = nullptr;
1205
+ logits = nullptr;
1206
+ embd = nullptr;
1207
+ }
1208
+
1209
+ auto * buft = ggml_backend_cpu_buffer_type();
1210
+ // try to use the host buffer of the device where the output tensor is allocated for faster transfer to system memory
1211
+ auto * output_dev = model.dev_output();
1212
+ auto * output_dev_host_buft = output_dev ? ggml_backend_dev_host_buffer_type(output_dev) : nullptr;
1213
+ if (output_dev_host_buft) {
1214
+ buft = output_dev_host_buft;
1215
+ }
1216
+ buf_output.reset(ggml_backend_buft_alloc_buffer(buft, new_size));
1217
+ if (buf_output == nullptr) {
1218
+ LLAMA_LOG_ERROR("%s: failed to allocate output buffer of size %.2f MiB\n", __func__, new_size / (1024.0 * 1024.0));
1219
+ return 0;
1220
+ }
1221
+ }
1222
+
1223
+ float * output_base = (float *) ggml_backend_buffer_get_base(buf_output.get());
1224
+
1225
+ logits = has_logits ? output_base : nullptr;
1226
+ embd = has_embd ? output_base + logits_size : nullptr;
1227
+
1228
+ // set all ids as invalid (negative)
1229
+ std::fill(output_ids.begin(), output_ids.end(), -1);
1230
+
1231
+ this->n_outputs = 0;
1232
+ this->n_outputs_max = n_outputs_max;
1233
+
1234
+ return n_outputs_max;
1235
+ }
1236
+
1237
+ //
1238
+ // graph
1239
+ //
1240
+
1241
+ int32_t llama_context::graph_max_nodes() const {
1242
+ return std::max<int32_t>(65536, 5*model.n_tensors());
1243
+ }
1244
+
1245
+ ggml_cgraph * llama_context::graph_init() {
1246
+ ggml_init_params params = {
1247
+ /*.mem_size =*/ buf_compute_meta.size(),
1248
+ /*.mem_buffer =*/ buf_compute_meta.data(),
1249
+ /*.no_alloc =*/ true,
1250
+ };
1251
+
1252
+ ctx_compute.reset(ggml_init(params));
1253
+
1254
+ return ggml_new_graph_custom(ctx_compute.get(), graph_max_nodes(), false);
1255
+ }
1256
+
1257
+ llm_graph_result_ptr llama_context::graph_build(
1258
+ ggml_context * ctx,
1259
+ ggml_cgraph * gf,
1260
+ const llama_ubatch & ubatch,
1261
+ llm_graph_type gtype) {
1262
+ return model.build_graph(
1263
+ {
1264
+ /*.ctx =*/ ctx,
1265
+ /*.arch =*/ model.arch,
1266
+ /*.hparams =*/ model.hparams,
1267
+ /*.cparams =*/ cparams,
1268
+ /*.ubatch =*/ ubatch,
1269
+ /*.sched =*/ sched.get(),
1270
+ /*.backend_cpu =*/ backend_cpu,
1271
+ /*.cvec =*/ &cvec,
1272
+ /*.loras =*/ &loras,
1273
+ /*.memory =*/ memory.get(),
1274
+ /*.cross =*/ &cross,
1275
+ /*.n_outputs =*/ n_outputs,
1276
+ /*.cb =*/ graph_get_cb(),
1277
+ }, gf, gtype);
1278
+ }
1279
+
1280
+ ggml_status llama_context::graph_compute(
1281
+ ggml_cgraph * gf,
1282
+ bool batched) {
1283
+ int n_threads = batched ? cparams.n_threads_batch : cparams.n_threads;
1284
+ ggml_threadpool_t tp = batched ? threadpool_batch : threadpool;
1285
+
1286
+ if (backend_cpu != nullptr) {
1287
+ auto * reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend_cpu));
1288
+ auto * set_threadpool_fn = (decltype(ggml_backend_cpu_set_threadpool) *) ggml_backend_reg_get_proc_address(reg, "ggml_backend_cpu_set_threadpool");
1289
+ set_threadpool_fn(backend_cpu, tp);
1290
+ }
1291
+
1292
+ // set the number of threads for all the backends
1293
+ for (const auto & set_n_threads_fn : set_n_threads_fns) {
1294
+ set_n_threads_fn.second(set_n_threads_fn.first, n_threads);
1295
+ }
1296
+
1297
+ auto status = ggml_backend_sched_graph_compute_async(sched.get(), gf);
1298
+ if (status != GGML_STATUS_SUCCESS) {
1299
+ LLAMA_LOG_ERROR("%s: ggml_backend_sched_graph_compute_async failed with error %d\n", __func__, status);
1300
+ }
1301
+
1302
+ // fprintf(stderr, "splits: %d\n", ggml_backend_sched_get_n_splits(sched));
1303
+
1304
+ return status;
1305
+ }
1306
+
1307
+ llm_graph_cb llama_context::graph_get_cb() const {
1308
+ return [&](const llama_ubatch & ubatch, ggml_tensor * cur, const char * name, int il) {
1309
+ if (il >= 0) {
1310
+ ggml_format_name(cur, "%s-%d", name, il);
1311
+ } else {
1312
+ ggml_set_name(cur, name);
1313
+ }
1314
+
1315
+ if (!cparams.offload_kqv) {
1316
+ if (strcmp(name, "kqv_merged_cont") == 0) {
1317
+ // all nodes between the KV store and the attention output are run on the CPU
1318
+ ggml_backend_sched_set_tensor_backend(sched.get(), cur, backend_cpu);
1319
+ }
1320
+ }
1321
+
1322
+ // norm may be automatically assigned to the backend of the previous layer, increasing data transfer between backends
1323
+ // FIXME: fix in ggml_backend_sched
1324
+ const bool full_offload = model.params.n_gpu_layers > (int) model.hparams.n_layer;
1325
+ if (ubatch.n_tokens < 32 || full_offload) {
1326
+ if (il != -1 && strcmp(name, "norm") == 0) {
1327
+ const auto & dev_layer = model.dev_layer(il);
1328
+ for (const auto & backend : backends) {
1329
+ if (ggml_backend_get_device(backend.get()) == dev_layer) {
1330
+ if (ggml_backend_supports_op(backend.get(), cur)) {
1331
+ ggml_backend_sched_set_tensor_backend(sched.get(), cur, backend.get());
1332
+ }
1333
+ }
1334
+ }
1335
+ }
1336
+ }
1337
+ };
1338
+ }
1339
+
1340
+ //
1341
+ // state save/load
1342
+ //
1343
+
1344
+ class llama_io_write_dummy : public llama_io_write_i {
1345
+ public:
1346
+ llama_io_write_dummy() = default;
1347
+
1348
+ void write(const void * /* src */, size_t size) override {
1349
+ size_written += size;
1350
+ }
1351
+
1352
+ void write_tensor(const ggml_tensor * /* tensor */, size_t /* offset */, size_t size) override {
1353
+ size_written += size;
1354
+ }
1355
+
1356
+ size_t n_bytes() override {
1357
+ return size_written;
1358
+ }
1359
+
1360
+ private:
1361
+ size_t size_written = 0;
1362
+ };
1363
+
1364
+ class llama_io_write_buffer : public llama_io_write_i {
1365
+ public:
1366
+ llama_io_write_buffer(
1367
+ uint8_t * p, size_t len) : ptr(p), buf_size(len) {}
1368
+
1369
+ void write(const void * src, size_t size) override {
1370
+ if (size > buf_size) {
1371
+ throw std::runtime_error("unexpectedly reached end of buffer");
1372
+ }
1373
+ memcpy(ptr, src, size);
1374
+ ptr += size;
1375
+ size_written += size;
1376
+ buf_size -= size;
1377
+ }
1378
+
1379
+ void write_tensor(const ggml_tensor * tensor, size_t offset, size_t size) override {
1380
+ if (size > buf_size) {
1381
+ throw std::runtime_error("unexpectedly reached end of buffer");
1382
+ }
1383
+ ggml_backend_tensor_get(tensor, ptr, offset, size);
1384
+ ptr += size;
1385
+ size_written += size;
1386
+ buf_size -= size;
1387
+ }
1388
+
1389
+ size_t n_bytes() override {
1390
+ return size_written;
1391
+ }
1392
+
1393
+ private:
1394
+ uint8_t * ptr;
1395
+ size_t buf_size = 0;
1396
+ size_t size_written = 0;
1397
+ };
1398
+
1399
+ class llama_io_read_buffer : public llama_io_read_i {
1400
+ public:
1401
+ llama_io_read_buffer(const uint8_t * p, size_t len) : ptr(p), buf_size(len) {}
1402
+
1403
+ const uint8_t * read(size_t size) override {
1404
+ const uint8_t * base_ptr = ptr;
1405
+ if (size > buf_size) {
1406
+ throw std::runtime_error("unexpectedly reached end of buffer");
1407
+ }
1408
+ ptr += size;
1409
+ size_read += size;
1410
+ buf_size -= size;
1411
+ return base_ptr;
1412
+ }
1413
+
1414
+ void read_to(void * dst, size_t size) override {
1415
+ memcpy(dst, read(size), size);
1416
+ }
1417
+
1418
+ size_t n_bytes() override {
1419
+ return size_read;
1420
+ }
1421
+
1422
+ private:
1423
+ const uint8_t * ptr;
1424
+ size_t buf_size = 0;
1425
+ size_t size_read = 0;
1426
+ };
1427
+
1428
+ class llama_io_write_file : public llama_io_write_i {
1429
+ public:
1430
+ llama_io_write_file(llama_file * f) : file(f) {}
1431
+
1432
+ void write(const void * src, size_t size) override {
1433
+ file->write_raw(src, size);
1434
+ size_written += size;
1435
+ }
1436
+
1437
+ void write_tensor(const ggml_tensor * tensor, size_t offset, size_t size) override {
1438
+ temp_buffer.resize(size);
1439
+ ggml_backend_tensor_get(tensor, temp_buffer.data(), offset, size);
1440
+ write(temp_buffer.data(), temp_buffer.size());
1441
+ }
1442
+
1443
+ size_t n_bytes() override {
1444
+ return size_written;
1445
+ }
1446
+
1447
+ private:
1448
+ llama_file * file;
1449
+ size_t size_written = 0;
1450
+ std::vector<uint8_t> temp_buffer;
1451
+ };
1452
+
1453
+ class llama_io_read_file : public llama_io_read_i {
1454
+ public:
1455
+ llama_io_read_file(llama_file * f) : file(f) {}
1456
+
1457
+ void read_to(void * dst, size_t size) override {
1458
+ file->read_raw(dst, size);
1459
+ size_read += size;
1460
+ }
1461
+
1462
+ const uint8_t * read(size_t size) override {
1463
+ temp_buffer.resize(size);
1464
+ read_to(temp_buffer.data(), size);
1465
+ return temp_buffer.data();
1466
+ }
1467
+
1468
+ size_t n_bytes() override {
1469
+ return size_read;
1470
+ }
1471
+
1472
+ private:
1473
+ llama_file * file;
1474
+ size_t size_read = 0;
1475
+ std::vector<uint8_t> temp_buffer;
1476
+ };
1477
+
1478
+ size_t llama_context::state_get_size() {
1479
+ llama_io_write_dummy io;
1480
+ try {
1481
+ return state_write_data(io);
1482
+ } catch (const std::exception & err) {
1483
+ LLAMA_LOG_ERROR("%s: error getting state size: %s\n", __func__, err.what());
1484
+ return 0;
1485
+ }
1486
+ }
1487
+
1488
+ size_t llama_context::state_get_data(uint8_t * dst, size_t size) {
1489
+ llama_io_write_buffer io(dst, size);
1490
+ try {
1491
+ return state_write_data(io);
1492
+ } catch (const std::exception & err) {
1493
+ LLAMA_LOG_ERROR("%s: error saving state: %s\n", __func__, err.what());
1494
+ return 0;
1495
+ }
1496
+ }
1497
+
1498
+ size_t llama_context::state_set_data(const uint8_t * src, size_t size) {
1499
+ llama_io_read_buffer io(src, size);
1500
+ try {
1501
+ return state_read_data(io);
1502
+ } catch (const std::exception & err) {
1503
+ LLAMA_LOG_ERROR("%s: error loading state: %s\n", __func__, err.what());
1504
+ return 0;
1505
+ }
1506
+ }
1507
+
1508
+ size_t llama_context::state_seq_get_size(llama_seq_id seq_id) {
1509
+ llama_io_write_dummy io;
1510
+ try {
1511
+ return state_seq_write_data(io, seq_id);
1512
+ } catch (const std::exception & err) {
1513
+ LLAMA_LOG_ERROR("%s: error getting state size: %s\n", __func__, err.what());
1514
+ return 0;
1515
+ }
1516
+ }
1517
+
1518
+ size_t llama_context::state_seq_get_data(llama_seq_id seq_id, uint8_t * dst, size_t size) {
1519
+ llama_io_write_buffer io(dst, size);
1520
+ try {
1521
+ return state_seq_write_data(io, seq_id);
1522
+ } catch (const std::exception & err) {
1523
+ LLAMA_LOG_ERROR("%s: error saving state: %s\n", __func__, err.what());
1524
+ return 0;
1525
+ }
1526
+ }
1527
+
1528
+ size_t llama_context::state_seq_set_data(llama_seq_id seq_id, const uint8_t * src, size_t size) {
1529
+ llama_io_read_buffer io(src, size);
1530
+ try {
1531
+ return state_seq_read_data(io, seq_id);
1532
+ } catch (const std::exception & err) {
1533
+ LLAMA_LOG_ERROR("%s: error loading state: %s\n", __func__, err.what());
1534
+ return 0;
1535
+ }
1536
+ }
1537
+
1538
+ bool llama_context::state_load_file(const char * filepath, llama_token * tokens_out, size_t n_token_capacity, size_t * n_token_count_out) {
1539
+ llama_file file(filepath, "rb");
1540
+
1541
+ // sanity checks
1542
+ {
1543
+ const uint32_t magic = file.read_u32();
1544
+ const uint32_t version = file.read_u32();
1545
+
1546
+ if (magic != LLAMA_SESSION_MAGIC || version != LLAMA_SESSION_VERSION) {
1547
+ LLAMA_LOG_ERROR("%s: unknown (magic, version) for session file: %08x, %08x\n", __func__, magic, version);
1548
+ return false;
1549
+ }
1550
+ }
1551
+
1552
+ // load the prompt
1553
+ {
1554
+ const uint32_t n_token_count = file.read_u32();
1555
+
1556
+ if (n_token_count > n_token_capacity) {
1557
+ LLAMA_LOG_ERROR("%s: token count in session file exceeded capacity! %u > %zu\n", __func__, n_token_count, n_token_capacity);
1558
+ return false;
1559
+ }
1560
+
1561
+ file.read_raw(tokens_out, sizeof(llama_token) * n_token_count);
1562
+ *n_token_count_out = n_token_count;
1563
+ }
1564
+
1565
+ // restore the context state
1566
+ {
1567
+ const size_t n_state_size_cur = file.size() - file.tell();
1568
+
1569
+ llama_io_read_file io( &file);
1570
+ const size_t n_read = state_read_data(io);
1571
+
1572
+ if (n_read != n_state_size_cur) {
1573
+ LLAMA_LOG_ERROR("%s: did not read all of the session file data! size %zu, got %zu\n", __func__, n_state_size_cur, n_read);
1574
+ return false;
1575
+ }
1576
+ }
1577
+
1578
+ return true;
1579
+ }
1580
+
1581
+ bool llama_context::state_save_file(const char * filepath, const llama_token * tokens, size_t n_token_count) {
1582
+ llama_file file(filepath, "wb");
1583
+
1584
+ file.write_u32(LLAMA_SESSION_MAGIC);
1585
+ file.write_u32(LLAMA_SESSION_VERSION);
1586
+
1587
+ // save the prompt
1588
+ file.write_u32((uint32_t) n_token_count);
1589
+ file.write_raw(tokens, sizeof(llama_token) * n_token_count);
1590
+
1591
+ // save the context state using stream saving
1592
+ llama_io_write_file io(&file);
1593
+ state_write_data(io);
1594
+
1595
+ return true;
1596
+ }
1597
+
1598
+ size_t llama_context::state_seq_load_file(llama_seq_id seq_id, const char * filepath, llama_token * tokens_out, size_t n_token_capacity, size_t * n_token_count_out) {
1599
+ llama_file file(filepath, "rb");
1600
+
1601
+ // version checks
1602
+ {
1603
+ const uint32_t magic = file.read_u32();
1604
+ const uint32_t version = file.read_u32();
1605
+
1606
+ if (magic != LLAMA_STATE_SEQ_MAGIC || version != LLAMA_STATE_SEQ_VERSION) {
1607
+ LLAMA_LOG_ERROR("%s: unknown (magic, version) for sequence state file: %08x, %08x\n", __func__, magic, version);
1608
+ return 0;
1609
+ }
1610
+ }
1611
+
1612
+ // load the prompt
1613
+ {
1614
+ const uint32_t n_token_count = file.read_u32();
1615
+
1616
+ if (n_token_count > n_token_capacity) {
1617
+ LLAMA_LOG_ERROR("%s: token count in sequence state file exceeded capacity! %u > %zu\n", __func__, n_token_count, n_token_capacity);
1618
+ return 0;
1619
+ }
1620
+
1621
+ file.read_raw(tokens_out, sizeof(llama_token) * n_token_count);
1622
+ *n_token_count_out = n_token_count;
1623
+ }
1624
+
1625
+ // restore the context state
1626
+ {
1627
+ const size_t state_size = file.size() - file.tell();
1628
+ llama_io_read_file io(&file);
1629
+ const size_t nread = state_seq_read_data(io, seq_id);
1630
+ if (!nread) {
1631
+ LLAMA_LOG_ERROR("%s: failed to restore sequence state\n", __func__);
1632
+ return 0;
1633
+ }
1634
+ GGML_ASSERT(nread <= state_size);
1635
+ GGML_ASSERT(nread + sizeof(uint32_t) * 3 + sizeof(llama_token) * *n_token_count_out == file.tell());
1636
+ }
1637
+
1638
+ return file.tell();
1639
+ }
1640
+
1641
+ size_t llama_context::state_seq_save_file(llama_seq_id seq_id, const char * filepath, const llama_token * tokens, size_t n_token_count) {
1642
+ llama_file file(filepath, "wb");
1643
+
1644
+ file.write_u32(LLAMA_STATE_SEQ_MAGIC);
1645
+ file.write_u32(LLAMA_STATE_SEQ_VERSION);
1646
+
1647
+ // save the prompt
1648
+ file.write_u32((uint32_t) n_token_count);
1649
+ file.write_raw(tokens, sizeof(llama_token) * n_token_count);
1650
+
1651
+ // save the context state using stream saving
1652
+ llama_io_write_file io(&file);
1653
+ state_seq_write_data(io, seq_id);
1654
+
1655
+ const size_t res = file.tell();
1656
+ GGML_ASSERT(res == sizeof(uint32_t) * 3 + sizeof(llama_token) * n_token_count + io.n_bytes());
1657
+
1658
+ return res;
1659
+ }
1660
+
1661
+ size_t llama_context::state_write_data(llama_io_write_i & io) {
1662
+ LLAMA_LOG_DEBUG("%s: writing state\n", __func__);
1663
+
1664
+ // write model info
1665
+ {
1666
+ LLAMA_LOG_DEBUG("%s: - writing model info\n", __func__);
1667
+
1668
+ const std::string arch_str = llm_arch_name(model.arch);
1669
+ io.write_string(arch_str);
1670
+ // TODO: add more model-specific info which should prevent loading the session file if not identical
1671
+ }
1672
+
1673
+ // write output ids
1674
+ {
1675
+ LLAMA_LOG_DEBUG("%s: - writing output ids\n", __func__);
1676
+
1677
+ const auto n_outputs = this->n_outputs;
1678
+ const auto & output_ids = this->output_ids;
1679
+
1680
+ std::vector<int32_t> w_output_pos;
1681
+
1682
+ GGML_ASSERT(n_outputs <= n_outputs_max);
1683
+
1684
+ w_output_pos.resize(n_outputs);
1685
+
1686
+ // build a more compact representation of the output ids
1687
+ for (size_t i = 0; i < n_batch(); ++i) {
1688
+ // map an output id to a position in the batch
1689
+ int32_t pos = output_ids[i];
1690
+ if (pos >= 0) {
1691
+ GGML_ASSERT(pos < n_outputs);
1692
+ w_output_pos[pos] = i;
1693
+ }
1694
+ }
1695
+
1696
+ io.write(&n_outputs, sizeof(n_outputs));
1697
+
1698
+ if (n_outputs) {
1699
+ io.write(w_output_pos.data(), n_outputs * sizeof(int32_t));
1700
+ }
1701
+ }
1702
+
1703
+ // write logits
1704
+ {
1705
+ LLAMA_LOG_DEBUG("%s: - writing logits\n", __func__);
1706
+
1707
+ const uint64_t logits_size = std::min((uint64_t) this->logits_size, (uint64_t) n_outputs * model.vocab.n_tokens());
1708
+
1709
+ io.write(&logits_size, sizeof(logits_size));
1710
+
1711
+ if (logits_size) {
1712
+ io.write(logits, logits_size * sizeof(float));
1713
+ }
1714
+ }
1715
+
1716
+ // write embeddings
1717
+ {
1718
+ LLAMA_LOG_DEBUG("%s: - writing embeddings\n", __func__);
1719
+
1720
+ const uint64_t embd_size = std::min((uint64_t) this->embd_size, (uint64_t) n_outputs * model.hparams.n_embd);
1721
+
1722
+ io.write(&embd_size, sizeof(embd_size));
1723
+
1724
+ if (embd_size) {
1725
+ io.write(embd, embd_size * sizeof(float));
1726
+ }
1727
+ }
1728
+
1729
+ llama_kv_cache * kv_self = static_cast<llama_kv_cache *>(memory.get());
1730
+
1731
+ if (kv_self != nullptr) {
1732
+ LLAMA_LOG_DEBUG("%s: - writing KV self\n", __func__);
1733
+ kv_self->state_write(io);
1734
+ }
1735
+
1736
+ return io.n_bytes();
1737
+ }
1738
+
1739
+ size_t llama_context::state_read_data(llama_io_read_i & io) {
1740
+ LLAMA_LOG_DEBUG("%s: reading state\n", __func__);
1741
+
1742
+ // read model info
1743
+ {
1744
+ LLAMA_LOG_DEBUG("%s: - reading model info\n", __func__);
1745
+
1746
+ const std::string cur_arch_str = llm_arch_name(model.arch);
1747
+
1748
+ std::string arch_str;
1749
+ io.read_string(arch_str);
1750
+ if (cur_arch_str != arch_str) {
1751
+ throw std::runtime_error(format("wrong model arch: '%s' instead of '%s'", arch_str.c_str(), cur_arch_str.c_str()));
1752
+ }
1753
+ // TODO: add more info which needs to be identical but which is not verified otherwise
1754
+ }
1755
+
1756
+ // read output ids
1757
+ {
1758
+ LLAMA_LOG_DEBUG("%s: - reading output ids\n", __func__);
1759
+
1760
+ auto n_outputs = this->n_outputs;
1761
+ io.read_to(&n_outputs, sizeof(n_outputs));
1762
+
1763
+ if (n_outputs > output_reserve(n_outputs)) {
1764
+ throw std::runtime_error("could not reserve outputs");
1765
+ }
1766
+
1767
+ std::vector<int32_t> output_pos;
1768
+
1769
+ if (n_outputs) {
1770
+ output_pos.resize(n_outputs);
1771
+ io.read_to(output_pos.data(), n_outputs * sizeof(int32_t));
1772
+
1773
+ for (int32_t i = 0; i < (int32_t) output_pos.size(); ++i) {
1774
+ int32_t id = output_pos[i];
1775
+ if ((uint32_t) id >= n_batch()) {
1776
+ throw std::runtime_error(format("invalid output id, %d does not fit in batch size of %u", id, n_batch()));
1777
+ }
1778
+ this->output_ids[id] = i;
1779
+ }
1780
+
1781
+ this->n_outputs = n_outputs;
1782
+ }
1783
+ }
1784
+
1785
+ // read logits
1786
+ {
1787
+ LLAMA_LOG_DEBUG("%s: - reading logits\n", __func__);
1788
+
1789
+ uint64_t logits_size;
1790
+ io.read_to(&logits_size, sizeof(logits_size));
1791
+
1792
+ if (this->logits_size < logits_size) {
1793
+ throw std::runtime_error("logits buffer too small");
1794
+ }
1795
+
1796
+ if (logits_size) {
1797
+ io.read_to(this->logits, logits_size * sizeof(float));
1798
+ }
1799
+ }
1800
+
1801
+ // read embeddings
1802
+ {
1803
+ LLAMA_LOG_DEBUG("%s: - reading embeddings\n", __func__);
1804
+
1805
+ uint64_t embd_size;
1806
+ io.read_to(&embd_size, sizeof(embd_size));
1807
+
1808
+ if (this->embd_size < embd_size) {
1809
+ throw std::runtime_error("embeddings buffer too small");
1810
+ }
1811
+
1812
+ if (embd_size) {
1813
+ io.read_to(this->embd, embd_size * sizeof(float));
1814
+ }
1815
+ }
1816
+
1817
+ if (memory) {
1818
+ LLAMA_LOG_DEBUG("%s: - reading KV self\n", __func__);
1819
+
1820
+ llama_kv_cache * kv_self = static_cast<llama_kv_cache *>(memory.get());
1821
+
1822
+ kv_self->state_read(io);
1823
+ }
1824
+
1825
+ return io.n_bytes();
1826
+ }
1827
+
1828
+ size_t llama_context::state_seq_write_data(llama_io_write_i & io, llama_seq_id seq_id) {
1829
+ GGML_UNUSED(seq_id);
1830
+
1831
+ if (memory) {
1832
+ llama_kv_cache * kv_self = static_cast<llama_kv_cache *>(memory.get());
1833
+
1834
+ kv_self->state_write(io, seq_id);
1835
+ }
1836
+
1837
+ return io.n_bytes();
1838
+ }
1839
+
1840
+ size_t llama_context::state_seq_read_data(llama_io_read_i & io, llama_seq_id seq_id) {
1841
+ GGML_UNUSED(seq_id);
1842
+
1843
+ if (memory) {
1844
+ llama_kv_cache * kv_self = static_cast<llama_kv_cache *>(memory.get());
1845
+
1846
+ kv_self->state_read(io, seq_id);
1847
+ }
1848
+
1849
+ return io.n_bytes();
1850
+ }
1851
+
1852
+ //
1853
+ // perf
1854
+ //
1855
+
1856
+ llama_perf_context_data llama_context::perf_get_data() const {
1857
+ llama_perf_context_data data = {};
1858
+
1859
+ data.t_start_ms = 1e-3 * t_start_us;
1860
+ data.t_load_ms = 1e-3 * t_load_us;
1861
+ data.t_p_eval_ms = 1e-3 * t_p_eval_us;
1862
+ data.t_eval_ms = 1e-3 * t_eval_us;
1863
+ data.n_p_eval = std::max(1, n_p_eval);
1864
+ data.n_eval = std::max(1, n_eval);
1865
+
1866
+ return data;
1867
+ }
1868
+
1869
+ void llama_context::perf_reset() {
1870
+ t_start_us = ggml_time_us();
1871
+ t_eval_us = n_eval = 0;
1872
+ t_p_eval_us = n_p_eval = 0;
1873
+ }
1874
+
1875
+ //
1876
+ // training
1877
+ //
1878
+
1879
+ static void llama_set_param(struct ggml_tensor * tensor, llama_opt_param_filter param_filter, void * userdata) {
1880
+ if (!tensor || tensor->type != GGML_TYPE_F32) {
1881
+ return;
1882
+ }
1883
+ if (!param_filter(tensor, userdata)) {
1884
+ return;
1885
+ }
1886
+ if (strcmp(tensor->name, "token_embd.weight") == 0) {
1887
+ return; // FIXME
1888
+ }
1889
+ if (strcmp(tensor->name, "rope_freqs.weight") == 0) {
1890
+ return; // FIXME
1891
+ }
1892
+ ggml_set_param(tensor);
1893
+ }
1894
+
1895
+ void llama_context::opt_init(struct llama_model * model, struct llama_opt_params lopt_params) {
1896
+ GGML_ASSERT(!opt_ctx);
1897
+ model->hparams.n_ctx_train = lopt_params.n_ctx_train > 0 ? lopt_params.n_ctx_train : n_ctx();
1898
+ const uint32_t n_batch = std::min(this->n_batch(), model->hparams.n_ctx_train);
1899
+ const uint32_t n_ubatch = std::min(this->n_ubatch(), n_batch);
1900
+ GGML_ASSERT(model->hparams.n_ctx_train % n_batch == 0);
1901
+ GGML_ASSERT(n_batch % n_ubatch == 0);
1902
+
1903
+ ggml_opt_params opt_params = ggml_opt_default_params(sched.get(), GGML_OPT_LOSS_TYPE_CROSS_ENTROPY);
1904
+ opt_params.opt_period = n_batch / n_ubatch;
1905
+ opt_params.get_opt_pars = lopt_params.get_opt_pars;
1906
+ opt_params.get_opt_pars_ud = lopt_params.get_opt_pars_ud;
1907
+
1908
+ opt_ctx = ggml_opt_init(opt_params);
1909
+
1910
+ llama_opt_param_filter param_filter = lopt_params.param_filter;
1911
+ void * param_filter_ud = lopt_params.param_filter_ud;
1912
+
1913
+ //llama_set_param(model->tok_embd, param_filter, param_filter_ud); // FIXME
1914
+ llama_set_param(model->type_embd, param_filter, param_filter_ud);
1915
+ llama_set_param(model->pos_embd, param_filter, param_filter_ud);
1916
+ llama_set_param(model->tok_norm, param_filter, param_filter_ud);
1917
+ llama_set_param(model->tok_norm_b, param_filter, param_filter_ud);
1918
+ llama_set_param(model->output_norm, param_filter, param_filter_ud);
1919
+ llama_set_param(model->output_norm_b, param_filter, param_filter_ud);
1920
+ llama_set_param(model->output, param_filter, param_filter_ud);
1921
+ llama_set_param(model->output_b, param_filter, param_filter_ud);
1922
+ llama_set_param(model->output_norm_enc, param_filter, param_filter_ud);
1923
+ llama_set_param(model->cls, param_filter, param_filter_ud);
1924
+ llama_set_param(model->cls_b, param_filter, param_filter_ud);
1925
+ llama_set_param(model->cls_out, param_filter, param_filter_ud);
1926
+ llama_set_param(model->cls_out_b, param_filter, param_filter_ud);
1927
+
1928
+ for (struct llama_layer & layer : model->layers) {
1929
+ for (size_t i = 0; i < sizeof(layer)/sizeof(struct ggml_tensor *); ++i) {
1930
+ llama_set_param(reinterpret_cast<struct ggml_tensor **>(&layer)[i], param_filter, param_filter_ud);
1931
+ }
1932
+ }
1933
+ }
1934
+
1935
+ void llama_context::opt_epoch_iter(
1936
+ ggml_opt_dataset_t dataset,
1937
+ ggml_opt_result_t result,
1938
+ const std::vector<llama_token> & tokens,
1939
+ const std::vector<llama_token> & labels_sparse,
1940
+ llama_batch & batch,
1941
+ ggml_opt_epoch_callback callback,
1942
+ bool train,
1943
+ int64_t idata_in_loop,
1944
+ int64_t ndata_in_loop,
1945
+ int64_t t_loop_start) {
1946
+ GGML_ASSERT(opt_ctx);
1947
+ const uint32_t n_ctx = llama_model_n_ctx_train(&model);
1948
+ const uint32_t n_batch = std::min(this->n_batch(), n_ctx);
1949
+ const uint32_t n_ubatch = std::min(this->n_ubatch(), n_batch);
1950
+
1951
+ llama_kv_cache * kv_self = static_cast<llama_kv_cache *>(memory.get());
1952
+
1953
+ kv_self->clear();
1954
+ llama_kv_cache_guard kv_guard(kv_self);
1955
+
1956
+ for (uint32_t pos_ctx = 0; pos_ctx < n_ctx; pos_ctx += n_batch) {
1957
+ batch.n_tokens = n_batch;
1958
+ for (uint32_t pos_batch = 0; pos_batch < n_batch; ++pos_batch) {
1959
+ batch.token [pos_batch] = tokens[pos_ctx + pos_batch];
1960
+ batch.pos [pos_batch] = pos_ctx + pos_batch;
1961
+ batch.n_seq_id[pos_batch] = 1;
1962
+ batch.seq_id [pos_batch][0] = 0;
1963
+ batch.logits [pos_batch] = true;
1964
+ }
1965
+
1966
+ const auto n_tokens_all = batch.n_tokens;
1967
+
1968
+ n_queued_tokens += n_tokens_all;
1969
+
1970
+ // this indicates we are doing pooled embedding, so we ignore batch.logits and output all tokens
1971
+ const bool embd_pooled = cparams.embeddings && cparams.pooling_type != LLAMA_POOLING_TYPE_NONE;
1972
+
1973
+ embd_seq.clear();
1974
+
1975
+ int64_t n_outputs_all = n_tokens_all;
1976
+
1977
+ llama_sbatch sbatch = kv_self->sbatch_init(batch, /*logits_all =*/ true);
1978
+
1979
+ // reserve output buffer
1980
+ if (output_reserve(n_outputs_all) < n_outputs_all) {
1981
+ LLAMA_LOG_ERROR("%s: could not reserve space for batch with %" PRId64 " outputs\n", __func__, n_outputs_all);
1982
+ GGML_ABORT("TODO: handle this error");
1983
+ };
1984
+
1985
+ for (uint32_t pos_batch = 0; pos_batch < n_batch; pos_batch += n_ubatch) {
1986
+ llama_ubatch ubatch = kv_self->ubatch_next(sbatch, cparams.n_ubatch, embd_pooled);
1987
+
1988
+ n_outputs = ubatch.n_tokens;
1989
+
1990
+ // TODO: not sure if this is needed
1991
+ if (!kv_self->find_slot(ubatch)) {
1992
+ LLAMA_LOG_WARN("%s: failed to find KV cache slot for ubatch of size %d\n", __func__, ubatch.n_tokens);
1993
+
1994
+ GGML_ABORT("TODO: handle this error");
1995
+ }
1996
+
1997
+ auto * gf = graph_init();
1998
+ auto res = graph_build(ctx_compute.get(), gf, ubatch, LLM_GRAPH_TYPE_DEFAULT);
1999
+
2000
+ struct ggml_context * ctx_compute_opt;
2001
+ {
2002
+ const size_t size_gf = ggml_graph_size(gf);
2003
+ const size_t size_meta = 4*size_gf*ggml_tensor_overhead() + 2*ggml_graph_overhead_custom(size_gf, /*grads = */ true);
2004
+ struct ggml_init_params params = {
2005
+ /*.mem_size =*/ size_meta,
2006
+ /*.mem_buffer =*/ nullptr,
2007
+ /*.no_alloc =*/ true,
2008
+ };
2009
+ ctx_compute_opt = ggml_init(params);
2010
+ }
2011
+ ggml_opt_prepare_alloc(opt_ctx, ctx_compute_opt, gf, res->get_tokens(), res->get_logits());
2012
+ ggml_opt_alloc(opt_ctx, train);
2013
+ res->set_inputs(&ubatch);
2014
+ {
2015
+ struct ggml_tensor * labels = ggml_opt_labels(opt_ctx);
2016
+ GGML_ASSERT(labels->ne[1] == n_ubatch);
2017
+ ggml_set_zero(labels);
2018
+ const float onef = 1.0f;
2019
+ for (uint32_t pos_ubatch = 0; pos_ubatch < n_ubatch; ++pos_ubatch) {
2020
+ const uint32_t ilabel = pos_ctx + pos_batch + pos_ubatch;
2021
+ GGML_ASSERT(labels_sparse[ilabel] < labels->ne[0]);
2022
+ ggml_backend_tensor_set(labels, &onef, (pos_ubatch*labels->ne[0] + labels_sparse[ilabel])*sizeof(float), sizeof(float));
2023
+ }
2024
+ }
2025
+ ggml_opt_eval(opt_ctx, result);
2026
+ if (callback) {
2027
+ callback(train, opt_ctx, dataset, result, idata_in_loop + (pos_ctx + pos_batch)/n_ubatch + 1, ndata_in_loop, t_loop_start);
2028
+ }
2029
+ ggml_free(ctx_compute_opt);
2030
+ }
2031
+ }
2032
+
2033
+ kv_guard.commit();
2034
+ }
2035
+
2036
+ void llama_context::opt_epoch(
2037
+ ggml_opt_dataset_t dataset,
2038
+ ggml_opt_result_t result_train,
2039
+ ggml_opt_result_t result_eval,
2040
+ int64_t idata_split,
2041
+ ggml_opt_epoch_callback callback_train,
2042
+ ggml_opt_epoch_callback callback_eval) {
2043
+ const uint32_t n_ctx = this->n_ctx();
2044
+ const uint32_t n_batch = std::min(cparams.n_batch, n_ctx);
2045
+ const uint32_t n_ubatch = std::min(cparams.n_ubatch, n_batch);
2046
+ const int64_t ndata = ggml_opt_dataset_ndata(dataset);
2047
+
2048
+ GGML_ASSERT(idata_split >= 0);
2049
+ GGML_ASSERT(idata_split <= ndata);
2050
+
2051
+ const uint32_t ubatch_per_ctx = n_ctx / n_ubatch;
2052
+
2053
+ struct llama_batch batch = llama_batch_init(n_batch, 0, 1);
2054
+ std::vector<llama_token> tokens(n_ctx);
2055
+ std::vector<llama_token> labels_sparse(n_ctx);
2056
+
2057
+ int64_t idata = 0;
2058
+
2059
+ int64_t t_loop_start = ggml_time_us();
2060
+ int64_t ndata_in_loop = idata_split*ubatch_per_ctx;
2061
+ for (; idata < idata_split; ++idata) {
2062
+ constexpr bool train = true;
2063
+ const int64_t idata_in_loop = idata*ubatch_per_ctx;
2064
+
2065
+ ggml_opt_dataset_get_batch_host(dataset, tokens.data(), n_ctx*sizeof(llama_token), labels_sparse.data(), idata);
2066
+ opt_epoch_iter(dataset, result_train, tokens, labels_sparse, batch,
2067
+ callback_train, train, idata_in_loop, ndata_in_loop, t_loop_start);
2068
+ }
2069
+
2070
+ t_loop_start = ggml_time_us();
2071
+ ndata_in_loop = (ndata - idata_split)*ubatch_per_ctx;
2072
+ for (; idata < ndata; ++idata) {
2073
+ constexpr bool train = false;
2074
+ const int64_t idata_in_loop = (idata - idata_split)*ubatch_per_ctx;
2075
+
2076
+ ggml_opt_dataset_get_batch_host(dataset, tokens.data(), n_ctx*sizeof(llama_token), labels_sparse.data(), idata);
2077
+ opt_epoch_iter(dataset, result_eval, tokens, labels_sparse, batch,
2078
+ callback_eval, train, idata_in_loop, ndata_in_loop, t_loop_start);
2079
+ }
2080
+
2081
+ llama_batch_free(batch);
2082
+ }
2083
+
2084
+ //
2085
+ // interface implementation
2086
+ //
2087
+
2088
+ llama_context_params llama_context_default_params() {
2089
+ llama_context_params result = {
2090
+ /*.n_ctx =*/ 512,
2091
+ /*.n_batch =*/ 2048,
2092
+ /*.n_ubatch =*/ 512,
2093
+ /*.n_seq_max =*/ 1,
2094
+ /*.n_threads =*/ GGML_DEFAULT_N_THREADS, // TODO: better default
2095
+ /*.n_threads_batch =*/ GGML_DEFAULT_N_THREADS,
2096
+ /*.rope_scaling_type =*/ LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED,
2097
+ /*.pooling_type =*/ LLAMA_POOLING_TYPE_UNSPECIFIED,
2098
+ /*.attention_type =*/ LLAMA_ATTENTION_TYPE_UNSPECIFIED,
2099
+ /*.rope_freq_base =*/ 0.0f,
2100
+ /*.rope_freq_scale =*/ 0.0f,
2101
+ /*.yarn_ext_factor =*/ -1.0f,
2102
+ /*.yarn_attn_factor =*/ 1.0f,
2103
+ /*.yarn_beta_fast =*/ 32.0f,
2104
+ /*.yarn_beta_slow =*/ 1.0f,
2105
+ /*.yarn_orig_ctx =*/ 0,
2106
+ /*.defrag_thold =*/ -1.0f,
2107
+ /*.cb_eval =*/ nullptr,
2108
+ /*.cb_eval_user_data =*/ nullptr,
2109
+ /*.type_k =*/ GGML_TYPE_F16,
2110
+ /*.type_v =*/ GGML_TYPE_F16,
2111
+ /*.abort_callback =*/ nullptr,
2112
+ /*.abort_callback_data =*/ nullptr,
2113
+ /*.embeddings =*/ false,
2114
+ /*.offload_kqv =*/ true,
2115
+ /*.flash_attn =*/ false,
2116
+ /*.no_perf =*/ true,
2117
+ /*.op_offload =*/ true,
2118
+ /*.swa_full =*/ true,
2119
+ };
2120
+
2121
+ return result;
2122
+ }
2123
+
2124
+ llama_context * llama_init_from_model(
2125
+ llama_model * model,
2126
+ llama_context_params params) {
2127
+ if (!model) {
2128
+ LLAMA_LOG_ERROR("%s: model cannot be NULL\n", __func__);
2129
+ return nullptr;
2130
+ }
2131
+
2132
+ if (params.n_batch == 0 && params.n_ubatch == 0) {
2133
+ LLAMA_LOG_ERROR("%s: n_batch and n_ubatch cannot both be zero\n", __func__);
2134
+ return nullptr;
2135
+ }
2136
+
2137
+ if (params.n_ctx == 0 && model->hparams.n_ctx_train == 0) {
2138
+ LLAMA_LOG_ERROR("%s: n_ctx and model->hparams.n_ctx_train cannot both be zero\n", __func__);
2139
+ return nullptr;
2140
+ }
2141
+
2142
+ if (params.flash_attn && model->arch == LLM_ARCH_GROK) {
2143
+ LLAMA_LOG_WARN("%s: flash_attn is not compatible with Grok - forcing off\n", __func__);
2144
+ params.flash_attn = false;
2145
+ }
2146
+
2147
+ if (ggml_is_quantized(params.type_v) && !params.flash_attn) {
2148
+ LLAMA_LOG_ERROR("%s: V cache quantization requires flash_attn\n", __func__);
2149
+ return nullptr;
2150
+ }
2151
+
2152
+ try {
2153
+ auto * ctx = new llama_context(*model, params);
2154
+ return ctx;
2155
+ } catch (const std::exception & err) {
2156
+ LLAMA_LOG_ERROR("%s: failed to initialize the context: %s\n", __func__, err.what());
2157
+ }
2158
+
2159
+ return nullptr;
2160
+ }
2161
+
2162
+ // deprecated
2163
+ llama_context * llama_new_context_with_model(
2164
+ llama_model * model,
2165
+ llama_context_params params) {
2166
+ return llama_init_from_model(model, params);
2167
+ }
2168
+
2169
+ void llama_free(llama_context * ctx) {
2170
+ delete ctx;
2171
+ }
2172
+
2173
+ uint32_t llama_n_ctx(const llama_context * ctx) {
2174
+ return ctx->n_ctx();
2175
+ }
2176
+
2177
+ uint32_t llama_n_batch(const llama_context * ctx) {
2178
+ return ctx->n_batch();
2179
+ }
2180
+
2181
+ uint32_t llama_n_ubatch(const llama_context * ctx) {
2182
+ return ctx->n_ubatch();
2183
+ }
2184
+
2185
+ uint32_t llama_n_seq_max(const llama_context * ctx) {
2186
+ return ctx->n_seq_max();
2187
+ }
2188
+
2189
+ const llama_model * llama_get_model(const llama_context * ctx) {
2190
+ return &ctx->get_model();
2191
+ }
2192
+
2193
+ llama_kv_cache * llama_get_kv_self(llama_context * ctx) {
2194
+ return ctx->get_kv_self();
2195
+ }
2196
+
2197
+ void llama_kv_self_update(llama_context * ctx) {
2198
+ ctx->kv_self_update();
2199
+ }
2200
+
2201
+ enum llama_pooling_type llama_pooling_type(const llama_context * ctx) {
2202
+ return ctx->pooling_type();
2203
+ }
2204
+
2205
+ void llama_attach_threadpool(
2206
+ llama_context * ctx,
2207
+ ggml_threadpool_t threadpool,
2208
+ ggml_threadpool_t threadpool_batch) {
2209
+ ctx->attach_threadpool(threadpool, threadpool_batch);
2210
+ }
2211
+
2212
+ void llama_detach_threadpool(llama_context * ctx) {
2213
+ ctx->detach_threadpool();
2214
+ }
2215
+
2216
+ void llama_set_n_threads(llama_context * ctx, int32_t n_threads, int32_t n_threads_batch) {
2217
+ ctx->set_n_threads(n_threads, n_threads_batch);
2218
+ }
2219
+
2220
+ int32_t llama_n_threads(llama_context * ctx) {
2221
+ return ctx->n_threads();
2222
+ }
2223
+
2224
+ int32_t llama_n_threads_batch(llama_context * ctx) {
2225
+ return ctx->n_threads_batch();
2226
+ }
2227
+
2228
+ void llama_set_abort_callback(llama_context * ctx, bool (*abort_callback)(void * data), void * abort_callback_data) {
2229
+ ctx->set_abort_callback(abort_callback, abort_callback_data);
2230
+ }
2231
+
2232
+ void llama_set_embeddings(llama_context * ctx, bool embeddings) {
2233
+ ctx->set_embeddings(embeddings);
2234
+ }
2235
+
2236
+ void llama_set_causal_attn(llama_context * ctx, bool causal_attn) {
2237
+ ctx->set_causal_attn(causal_attn);
2238
+ }
2239
+
2240
+ void llama_set_warmup(llama_context * ctx, bool warmup) {
2241
+ ctx->set_warmup(warmup);
2242
+ }
2243
+
2244
+ void llama_synchronize(llama_context * ctx) {
2245
+ ctx->synchronize();
2246
+ }
2247
+
2248
+ float * llama_get_logits(llama_context * ctx) {
2249
+ ctx->synchronize();
2250
+
2251
+ return ctx->get_logits();
2252
+ }
2253
+
2254
+ float * llama_get_logits_ith(llama_context * ctx, int32_t i) {
2255
+ ctx->synchronize();
2256
+
2257
+ return ctx->get_logits_ith(i);
2258
+ }
2259
+
2260
+ float * llama_get_embeddings(llama_context * ctx) {
2261
+ ctx->synchronize();
2262
+
2263
+ return ctx->get_embeddings();
2264
+ }
2265
+
2266
+ float * llama_get_embeddings_ith(llama_context * ctx, int32_t i) {
2267
+ ctx->synchronize();
2268
+
2269
+ return ctx->get_embeddings_ith(i);
2270
+ }
2271
+
2272
+ float * llama_get_embeddings_seq(llama_context * ctx, llama_seq_id seq_id) {
2273
+ ctx->synchronize();
2274
+
2275
+ return ctx->get_embeddings_seq(seq_id);
2276
+ }
2277
+
2278
+ // llama adapter API
2279
+
2280
+ int32_t llama_set_adapter_lora(
2281
+ llama_context * ctx,
2282
+ llama_adapter_lora * adapter,
2283
+ float scale) {
2284
+ ctx->set_adapter_lora(adapter, scale);
2285
+
2286
+ return 0;
2287
+ }
2288
+
2289
+ int32_t llama_rm_adapter_lora(
2290
+ llama_context * ctx,
2291
+ llama_adapter_lora * adapter) {
2292
+ bool res = ctx->rm_adapter_lora(adapter);
2293
+
2294
+ return res ? 0 : -1;
2295
+ }
2296
+
2297
+ void llama_clear_adapter_lora(llama_context * ctx) {
2298
+ ctx->clear_adapter_lora();
2299
+ }
2300
+
2301
+ int32_t llama_apply_adapter_cvec(
2302
+ llama_context * ctx,
2303
+ const float * data,
2304
+ size_t len,
2305
+ int32_t n_embd,
2306
+ int32_t il_start,
2307
+ int32_t il_end) {
2308
+ bool res = ctx->apply_adapter_cvec(data, len, n_embd, il_start, il_end);
2309
+
2310
+ return res ? 0 : -1;
2311
+ }
2312
+
2313
+ //
2314
+ // kv cache
2315
+ //
2316
+
2317
+ // deprecated
2318
+ int32_t llama_kv_self_n_tokens(const llama_context * ctx) {
2319
+ const auto * kv = ctx->get_kv_self();
2320
+ if (!kv) {
2321
+ return 0;
2322
+ }
2323
+
2324
+ int32_t res = 0;
2325
+
2326
+ for (uint32_t s = 0; s < ctx->get_cparams().n_seq_max; s++) {
2327
+ const llama_pos p0 = kv->seq_pos_min(s);
2328
+ const llama_pos p1 = kv->seq_pos_max(s);
2329
+
2330
+ if (p0 >= 0) {
2331
+ res += (p1 - p0) + 1;
2332
+ }
2333
+ }
2334
+
2335
+ return res;
2336
+ }
2337
+
2338
+ // deprecated
2339
+ // note: this is the same as above - will be removed anyway, so it's ok
2340
+ int32_t llama_kv_self_used_cells(const llama_context * ctx) {
2341
+ const auto * kv = ctx->get_kv_self();
2342
+ if (!kv) {
2343
+ return 0;
2344
+ }
2345
+
2346
+ int32_t res = 0;
2347
+
2348
+ for (uint32_t s = 0; s < ctx->get_cparams().n_seq_max; s++) {
2349
+ const llama_pos p0 = kv->seq_pos_min(s);
2350
+ const llama_pos p1 = kv->seq_pos_max(s);
2351
+
2352
+ if (p0 >= 0) {
2353
+ res += (p1 - p0) + 1;
2354
+ }
2355
+ }
2356
+
2357
+ return res;
2358
+ }
2359
+
2360
+ void llama_kv_self_clear(llama_context * ctx) {
2361
+ auto * kv = ctx->get_kv_self();
2362
+ if (!kv) {
2363
+ return;
2364
+ }
2365
+
2366
+ kv->clear();
2367
+ }
2368
+
2369
+ bool llama_kv_self_seq_rm(
2370
+ llama_context * ctx,
2371
+ llama_seq_id seq_id,
2372
+ llama_pos p0,
2373
+ llama_pos p1) {
2374
+ auto * kv = ctx->get_kv_self();
2375
+ if (!kv) {
2376
+ return true;
2377
+ }
2378
+
2379
+ return kv->seq_rm(seq_id, p0, p1);
2380
+ }
2381
+
2382
+ void llama_kv_self_seq_cp(
2383
+ llama_context * ctx,
2384
+ llama_seq_id seq_id_src,
2385
+ llama_seq_id seq_id_dst,
2386
+ llama_pos p0,
2387
+ llama_pos p1) {
2388
+ auto * kv = ctx->get_kv_self();
2389
+ if (!kv) {
2390
+ return;
2391
+ }
2392
+
2393
+ kv->seq_cp(seq_id_src, seq_id_dst, p0, p1);
2394
+ }
2395
+
2396
+ void llama_kv_self_seq_keep(llama_context * ctx, llama_seq_id seq_id) {
2397
+ auto * kv = ctx->get_kv_self();
2398
+ if (!kv) {
2399
+ return;
2400
+ }
2401
+
2402
+ kv->seq_keep(seq_id);
2403
+ }
2404
+
2405
+ void llama_kv_self_seq_add(
2406
+ llama_context * ctx,
2407
+ llama_seq_id seq_id,
2408
+ llama_pos p0,
2409
+ llama_pos p1,
2410
+ llama_pos delta) {
2411
+ auto * kv = ctx->get_kv_self();
2412
+ if (!kv) {
2413
+ return;
2414
+ }
2415
+
2416
+ kv->seq_add(seq_id, p0, p1, delta);
2417
+ }
2418
+
2419
+ void llama_kv_self_seq_div(
2420
+ llama_context * ctx,
2421
+ llama_seq_id seq_id,
2422
+ llama_pos p0,
2423
+ llama_pos p1,
2424
+ int d) {
2425
+ auto * kv = ctx->get_kv_self();
2426
+ if (!kv) {
2427
+ return;
2428
+ }
2429
+
2430
+ kv->seq_div(seq_id, p0, p1, d);
2431
+ }
2432
+
2433
+ llama_pos llama_kv_self_seq_pos_min(llama_context * ctx, llama_seq_id seq_id) {
2434
+ const auto * kv = ctx->get_kv_self();
2435
+ if (!kv) {
2436
+ return -1;
2437
+ }
2438
+
2439
+ return kv->seq_pos_min(seq_id);
2440
+ }
2441
+
2442
+ llama_pos llama_kv_self_seq_pos_max(llama_context * ctx, llama_seq_id seq_id) {
2443
+ const auto * kv = ctx->get_kv_self();
2444
+ if (!kv) {
2445
+ return -1;
2446
+ }
2447
+
2448
+ return kv->seq_pos_max(seq_id);
2449
+ }
2450
+
2451
+ void llama_kv_self_defrag(llama_context * ctx) {
2452
+ auto * kv = ctx->get_kv_self();
2453
+ if (!kv) {
2454
+ return;
2455
+ }
2456
+
2457
+ // force defrag
2458
+ kv->defrag_sched(-1.0f);
2459
+ }
2460
+
2461
+ bool llama_kv_self_can_shift(const llama_context * ctx) {
2462
+ const auto * kv = ctx->get_kv_self();
2463
+ if (!kv) {
2464
+ return false;
2465
+ }
2466
+
2467
+ return kv->get_can_shift();
2468
+ }
2469
+
2470
+ // llama state API
2471
+
2472
+ // deprecated
2473
+ size_t llama_get_state_size(llama_context * ctx) {
2474
+ return llama_state_get_size(ctx);
2475
+ }
2476
+
2477
+ // deprecated
2478
+ size_t llama_copy_state_data(llama_context * ctx, uint8_t * dst) {
2479
+ return llama_state_get_data(ctx, dst, -1);
2480
+ }
2481
+
2482
+ // deprecated
2483
+ size_t llama_set_state_data(llama_context * ctx, const uint8_t * src) {
2484
+ return llama_state_set_data(ctx, src, -1);
2485
+ }
2486
+
2487
+ // deprecated
2488
+ bool llama_load_session_file(llama_context * ctx, const char * path_session, llama_token * tokens_out, size_t n_token_capacity, size_t * n_token_count_out) {
2489
+ return llama_state_load_file(ctx, path_session, tokens_out, n_token_capacity, n_token_count_out);
2490
+ }
2491
+
2492
+ // deprecated
2493
+ bool llama_save_session_file(llama_context * ctx, const char * path_session, const llama_token * tokens, size_t n_token_count) {
2494
+ return llama_state_save_file(ctx, path_session, tokens, n_token_count);
2495
+ }
2496
+
2497
+ // Returns the *actual* size of the state.
2498
+ // Intended to be used when saving to state to a buffer.
2499
+ size_t llama_state_get_size(llama_context * ctx) {
2500
+ return ctx->state_get_size();
2501
+ }
2502
+
2503
+ size_t llama_state_get_data(llama_context * ctx, uint8_t * dst, size_t size) {
2504
+ ctx->synchronize();
2505
+
2506
+ return ctx->state_get_data(dst, size);
2507
+ }
2508
+
2509
+ // Sets the state reading from the specified source address
2510
+ size_t llama_state_set_data(llama_context * ctx, const uint8_t * src, size_t size) {
2511
+ ctx->synchronize();
2512
+
2513
+ return ctx->state_set_data(src, size);
2514
+ }
2515
+
2516
+ bool llama_state_load_file(llama_context * ctx, const char * path_session, llama_token * tokens_out, size_t n_token_capacity, size_t * n_token_count_out) {
2517
+ ctx->synchronize();
2518
+
2519
+ try {
2520
+ return ctx->state_load_file(path_session, tokens_out, n_token_capacity, n_token_count_out);
2521
+ } catch (const std::exception & err) {
2522
+ LLAMA_LOG_ERROR("%s: error loading session file: %s\n", __func__, err.what());
2523
+ return false;
2524
+ }
2525
+ }
2526
+
2527
+ bool llama_state_save_file(llama_context * ctx, const char * path_session, const llama_token * tokens, size_t n_token_count) {
2528
+ ctx->synchronize();
2529
+
2530
+ try {
2531
+ return ctx->state_save_file(path_session, tokens, n_token_count);
2532
+ } catch (const std::exception & err) {
2533
+ LLAMA_LOG_ERROR("%s: error saving session file: %s\n", __func__, err.what());
2534
+ return false;
2535
+ }
2536
+ }
2537
+
2538
+ size_t llama_state_seq_get_size(llama_context * ctx, llama_seq_id seq_id) {
2539
+ return ctx->state_seq_get_size(seq_id);
2540
+ }
2541
+
2542
+ size_t llama_state_seq_get_data(llama_context * ctx, uint8_t * dst, size_t size, llama_seq_id seq_id) {
2543
+ ctx->synchronize();
2544
+
2545
+ return ctx->state_seq_get_data(seq_id, dst, size);
2546
+ }
2547
+
2548
+ size_t llama_state_seq_set_data(llama_context * ctx, const uint8_t * src, size_t size, llama_seq_id seq_id) {
2549
+ ctx->synchronize();
2550
+
2551
+ return ctx->state_seq_set_data(seq_id, src, size);
2552
+ }
2553
+
2554
+ size_t llama_state_seq_save_file(llama_context * ctx, const char * filepath, llama_seq_id seq_id, const llama_token * tokens, size_t n_token_count) {
2555
+ ctx->synchronize();
2556
+
2557
+ try {
2558
+ return ctx->state_seq_save_file(seq_id, filepath, tokens, n_token_count);
2559
+ } catch (const std::exception & err) {
2560
+ LLAMA_LOG_ERROR("%s: error saving sequence state file: %s\n", __func__, err.what());
2561
+ return 0;
2562
+ }
2563
+ }
2564
+
2565
+ size_t llama_state_seq_load_file(llama_context * ctx, const char * filepath, llama_seq_id dest_seq_id, llama_token * tokens_out, size_t n_token_capacity, size_t * n_token_count_out) {
2566
+ ctx->synchronize();
2567
+
2568
+ try {
2569
+ return ctx->state_seq_load_file(dest_seq_id, filepath, tokens_out, n_token_capacity, n_token_count_out);
2570
+ } catch (const std::exception & err) {
2571
+ LLAMA_LOG_ERROR("%s: error loading sequence state file: %s\n", __func__, err.what());
2572
+ return 0;
2573
+ }
2574
+ }
2575
+
2576
+ ///
2577
+
2578
+ int32_t llama_encode(
2579
+ llama_context * ctx,
2580
+ llama_batch batch) {
2581
+ const int ret = ctx->encode(batch);
2582
+ if (ret != 0) {
2583
+ LLAMA_LOG_ERROR("%s: failed to encode, ret = %d\n", __func__, ret);
2584
+ }
2585
+
2586
+ return ret;
2587
+ }
2588
+
2589
+ int32_t llama_decode(
2590
+ llama_context * ctx,
2591
+ llama_batch batch) {
2592
+ int ret = ctx->decode(batch);
2593
+
2594
+ // defrag and try again
2595
+ // TODO: distinguish return code when we are sure that even after defrag there is no space available
2596
+ if (ret == 1) {
2597
+ llama_kv_self_defrag(ctx);
2598
+ ret = ctx->decode(batch);
2599
+
2600
+ if (ret == 1) {
2601
+ LLAMA_LOG_WARN("%s: failed to find KV cache slot for batch of size %d\n", __func__, batch.n_tokens);
2602
+
2603
+ return ret;
2604
+ }
2605
+ }
2606
+
2607
+ if (ret != 0) {
2608
+ LLAMA_LOG_ERROR("%s: failed to decode, ret = %d\n", __func__, ret);
2609
+ }
2610
+
2611
+ return ret;
2612
+ }
2613
+
2614
+ //
2615
+ // perf
2616
+ //
2617
+
2618
+ llama_perf_context_data llama_perf_context(const llama_context * ctx) {
2619
+ llama_perf_context_data data = {};
2620
+
2621
+ if (ctx == nullptr) {
2622
+ return data;
2623
+ }
2624
+
2625
+ data = ctx->perf_get_data();
2626
+
2627
+ return data;
2628
+ }
2629
+
2630
+ void llama_perf_context_print(const llama_context * ctx) {
2631
+ const auto data = llama_perf_context(ctx);
2632
+
2633
+ const double t_end_ms = 1e-3 * ggml_time_us();
2634
+
2635
+ LLAMA_LOG_INFO("%s: load time = %10.2f ms\n", __func__, data.t_load_ms);
2636
+ LLAMA_LOG_INFO("%s: prompt eval time = %10.2f ms / %5d tokens (%8.2f ms per token, %8.2f tokens per second)\n",
2637
+ __func__, data.t_p_eval_ms, data.n_p_eval, data.t_p_eval_ms / data.n_p_eval, 1e3 / data.t_p_eval_ms * data.n_p_eval);
2638
+ LLAMA_LOG_INFO("%s: eval time = %10.2f ms / %5d runs (%8.2f ms per token, %8.2f tokens per second)\n",
2639
+ __func__, data.t_eval_ms, data.n_eval, data.t_eval_ms / data.n_eval, 1e3 / data.t_eval_ms * data.n_eval);
2640
+ LLAMA_LOG_INFO("%s: total time = %10.2f ms / %5d tokens\n", __func__, (t_end_ms - data.t_start_ms), (data.n_p_eval + data.n_eval));
2641
+ }
2642
+
2643
+ void llama_perf_context_reset(llama_context * ctx) {
2644
+ ctx->perf_reset();
2645
+ }
2646
+
2647
+ //
2648
+ // training
2649
+ //
2650
+
2651
+ bool llama_opt_param_filter_all(const struct ggml_tensor * tensor, void * userdata) {
2652
+ GGML_UNUSED(tensor);
2653
+ GGML_UNUSED(userdata);
2654
+ return true;
2655
+ }
2656
+
2657
+ void llama_opt_init(struct llama_context * ctx, struct llama_model * model, struct llama_opt_params lopt_params) {
2658
+ ctx->opt_init(model, lopt_params);
2659
+ }
2660
+
2661
+ void llama_opt_epoch(
2662
+ struct llama_context * ctx,
2663
+ ggml_opt_dataset_t dataset,
2664
+ ggml_opt_result_t result_train,
2665
+ ggml_opt_result_t result_eval,
2666
+ int64_t idata_split,
2667
+ ggml_opt_epoch_callback callback_train,
2668
+ ggml_opt_epoch_callback callback_eval) {
2669
+ ctx->opt_epoch(
2670
+ dataset,
2671
+ result_train,
2672
+ result_eval,
2673
+ idata_split,
2674
+ callback_train,
2675
+ callback_eval);
2676
+ }