elelem-ggml 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (1367) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE.txt +21 -0
  3. data/exe/elelem-ggml +8 -0
  4. data/ext/elelem/ggml/elelem.c +466 -0
  5. data/ext/elelem/ggml/extconf.rb +52 -0
  6. data/lib/elelem/ggml/chat_ml.rb +23 -0
  7. data/lib/elelem/ggml/gguf.rb +82 -0
  8. data/lib/elelem/ggml/provider.rb +63 -0
  9. data/lib/elelem/ggml/tokenizer.rb +95 -0
  10. data/lib/elelem/ggml/version.rb +7 -0
  11. data/lib/elelem/ggml.rb +37 -0
  12. data/vendor/ggml/AUTHORS +335 -0
  13. data/vendor/ggml/CMakeLists.txt +499 -0
  14. data/vendor/ggml/CONTRIBUTING.md +3 -0
  15. data/vendor/ggml/LICENSE +21 -0
  16. data/vendor/ggml/README.md +49 -0
  17. data/vendor/ggml/cmake/FindNCCL.cmake +36 -0
  18. data/vendor/ggml/cmake/GitVars.cmake +22 -0
  19. data/vendor/ggml/cmake/common.cmake +50 -0
  20. data/vendor/ggml/cmake/ggml-config.cmake.in +216 -0
  21. data/vendor/ggml/ggml.pc.in +10 -0
  22. data/vendor/ggml/include/ggml-alloc.h +86 -0
  23. data/vendor/ggml/include/ggml-backend.h +437 -0
  24. data/vendor/ggml/include/ggml-blas.h +25 -0
  25. data/vendor/ggml/include/ggml-cann.h +123 -0
  26. data/vendor/ggml/include/ggml-cpp.h +39 -0
  27. data/vendor/ggml/include/ggml-cpu.h +152 -0
  28. data/vendor/ggml/include/ggml-cuda.h +47 -0
  29. data/vendor/ggml/include/ggml-et.h +28 -0
  30. data/vendor/ggml/include/ggml-hexagon.h +19 -0
  31. data/vendor/ggml/include/ggml-metal.h +61 -0
  32. data/vendor/ggml/include/ggml-opencl.h +26 -0
  33. data/vendor/ggml/include/ggml-openvino.h +37 -0
  34. data/vendor/ggml/include/ggml-opt.h +256 -0
  35. data/vendor/ggml/include/ggml-rpc.h +35 -0
  36. data/vendor/ggml/include/ggml-sycl.h +57 -0
  37. data/vendor/ggml/include/ggml-virtgpu.h +14 -0
  38. data/vendor/ggml/include/ggml-vulkan.h +29 -0
  39. data/vendor/ggml/include/ggml-webgpu.h +19 -0
  40. data/vendor/ggml/include/ggml-zdnn.h +17 -0
  41. data/vendor/ggml/include/ggml-zendnn.h +22 -0
  42. data/vendor/ggml/include/ggml.h +2963 -0
  43. data/vendor/ggml/include/gguf.h +211 -0
  44. data/vendor/ggml/requirements.txt +12 -0
  45. data/vendor/ggml/scripts/gen-authors.sh +9 -0
  46. data/vendor/ggml/scripts/make-release-checks.sh +96 -0
  47. data/vendor/ggml/scripts/make-release-desc.sh +69 -0
  48. data/vendor/ggml/scripts/make-release-summary.txt +33 -0
  49. data/vendor/ggml/scripts/release.sh +195 -0
  50. data/vendor/ggml/scripts/sync-llama-am.sh +167 -0
  51. data/vendor/ggml/scripts/sync-llama.last +1 -0
  52. data/vendor/ggml/scripts/sync-llama.sh +21 -0
  53. data/vendor/ggml/scripts/sync-whisper-am.sh +138 -0
  54. data/vendor/ggml/scripts/sync-whisper.last +1 -0
  55. data/vendor/ggml/scripts/sync-whisper.sh +17 -0
  56. data/vendor/ggml/src/CMakeLists.txt +634 -0
  57. data/vendor/ggml/src/ggml-alloc.c +1249 -0
  58. data/vendor/ggml/src/ggml-backend-dl.cpp +48 -0
  59. data/vendor/ggml/src/ggml-backend-dl.h +45 -0
  60. data/vendor/ggml/src/ggml-backend-impl.h +291 -0
  61. data/vendor/ggml/src/ggml-backend-meta.cpp +2511 -0
  62. data/vendor/ggml/src/ggml-backend-reg.cpp +605 -0
  63. data/vendor/ggml/src/ggml-backend.cpp +2517 -0
  64. data/vendor/ggml/src/ggml-blas/CMakeLists.txt +101 -0
  65. data/vendor/ggml/src/ggml-blas/ggml-blas.cpp +530 -0
  66. data/vendor/ggml/src/ggml-cann/CMakeLists.txt +89 -0
  67. data/vendor/ggml/src/ggml-cann/acl_tensor.cpp +195 -0
  68. data/vendor/ggml/src/ggml-cann/acl_tensor.h +349 -0
  69. data/vendor/ggml/src/ggml-cann/aclnn_ops.cpp +4479 -0
  70. data/vendor/ggml/src/ggml-cann/aclnn_ops.h +1191 -0
  71. data/vendor/ggml/src/ggml-cann/common.h +651 -0
  72. data/vendor/ggml/src/ggml-cann/ggml-cann.cpp +3070 -0
  73. data/vendor/ggml/src/ggml-common.h +1911 -0
  74. data/vendor/ggml/src/ggml-cpu/CMakeLists.txt +685 -0
  75. data/vendor/ggml/src/ggml-cpu/amx/amx.cpp +249 -0
  76. data/vendor/ggml/src/ggml-cpu/amx/amx.h +8 -0
  77. data/vendor/ggml/src/ggml-cpu/amx/common.h +115 -0
  78. data/vendor/ggml/src/ggml-cpu/amx/mmq.cpp +2511 -0
  79. data/vendor/ggml/src/ggml-cpu/amx/mmq.h +10 -0
  80. data/vendor/ggml/src/ggml-cpu/arch/arm/cpu-feats.cpp +41 -0
  81. data/vendor/ggml/src/ggml-cpu/arch/arm/quants.c +4319 -0
  82. data/vendor/ggml/src/ggml-cpu/arch/arm/repack.cpp +5156 -0
  83. data/vendor/ggml/src/ggml-cpu/arch/loongarch/quants.c +2309 -0
  84. data/vendor/ggml/src/ggml-cpu/arch/powerpc/cpu-feats.cpp +82 -0
  85. data/vendor/ggml/src/ggml-cpu/arch/powerpc/quants.c +2304 -0
  86. data/vendor/ggml/src/ggml-cpu/arch/riscv/cpu-feats.cpp +38 -0
  87. data/vendor/ggml/src/ggml-cpu/arch/riscv/quants.c +6596 -0
  88. data/vendor/ggml/src/ggml-cpu/arch/riscv/repack.cpp +1703 -0
  89. data/vendor/ggml/src/ggml-cpu/arch/s390/cpu-feats.cpp +50 -0
  90. data/vendor/ggml/src/ggml-cpu/arch/s390/quants.c +1465 -0
  91. data/vendor/ggml/src/ggml-cpu/arch/wasm/quants.c +1292 -0
  92. data/vendor/ggml/src/ggml-cpu/arch/x86/cpu-feats.cpp +327 -0
  93. data/vendor/ggml/src/ggml-cpu/arch/x86/quants.c +4108 -0
  94. data/vendor/ggml/src/ggml-cpu/arch/x86/repack.cpp +6407 -0
  95. data/vendor/ggml/src/ggml-cpu/arch-fallback.h +353 -0
  96. data/vendor/ggml/src/ggml-cpu/binary-ops.cpp +154 -0
  97. data/vendor/ggml/src/ggml-cpu/binary-ops.h +16 -0
  98. data/vendor/ggml/src/ggml-cpu/cmake/FindSIMD.cmake +100 -0
  99. data/vendor/ggml/src/ggml-cpu/cmake/FindSMTIME.cmake +32 -0
  100. data/vendor/ggml/src/ggml-cpu/common.h +95 -0
  101. data/vendor/ggml/src/ggml-cpu/ggml-cpu-impl.h +539 -0
  102. data/vendor/ggml/src/ggml-cpu/ggml-cpu.c +3936 -0
  103. data/vendor/ggml/src/ggml-cpu/ggml-cpu.cpp +712 -0
  104. data/vendor/ggml/src/ggml-cpu/hbm.cpp +55 -0
  105. data/vendor/ggml/src/ggml-cpu/hbm.h +8 -0
  106. data/vendor/ggml/src/ggml-cpu/iqp.cpp +1253 -0
  107. data/vendor/ggml/src/ggml-cpu/iqp.h +39 -0
  108. data/vendor/ggml/src/ggml-cpu/kleidiai/CMakeLists.txt +14 -0
  109. data/vendor/ggml/src/ggml-cpu/kleidiai/kernels.cpp +1101 -0
  110. data/vendor/ggml/src/ggml-cpu/kleidiai/kernels.h +101 -0
  111. data/vendor/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp +1919 -0
  112. data/vendor/ggml/src/ggml-cpu/kleidiai/kleidiai.h +17 -0
  113. data/vendor/ggml/src/ggml-cpu/llamafile/sgemm.cpp +4164 -0
  114. data/vendor/ggml/src/ggml-cpu/llamafile/sgemm.h +25 -0
  115. data/vendor/ggml/src/ggml-cpu/ops.cpp +12164 -0
  116. data/vendor/ggml/src/ggml-cpu/ops.h +124 -0
  117. data/vendor/ggml/src/ggml-cpu/quants.c +1339 -0
  118. data/vendor/ggml/src/ggml-cpu/quants.h +106 -0
  119. data/vendor/ggml/src/ggml-cpu/repack.cpp +4836 -0
  120. data/vendor/ggml/src/ggml-cpu/repack.h +245 -0
  121. data/vendor/ggml/src/ggml-cpu/simd-gemm.h +226 -0
  122. data/vendor/ggml/src/ggml-cpu/simd-mappings.h +1319 -0
  123. data/vendor/ggml/src/ggml-cpu/spacemit/ime.cpp +1742 -0
  124. data/vendor/ggml/src/ggml-cpu/spacemit/ime.h +21 -0
  125. data/vendor/ggml/src/ggml-cpu/spacemit/ime1_kernels.cpp +1027 -0
  126. data/vendor/ggml/src/ggml-cpu/spacemit/ime2_kernels.cpp +5768 -0
  127. data/vendor/ggml/src/ggml-cpu/spacemit/ime_env.cpp +320 -0
  128. data/vendor/ggml/src/ggml-cpu/spacemit/ime_env.h +55 -0
  129. data/vendor/ggml/src/ggml-cpu/spacemit/ime_kernels.h +189 -0
  130. data/vendor/ggml/src/ggml-cpu/spacemit/repack.cpp +1795 -0
  131. data/vendor/ggml/src/ggml-cpu/spacemit/repack.h +14 -0
  132. data/vendor/ggml/src/ggml-cpu/spacemit/rvv_kernels.cpp +3178 -0
  133. data/vendor/ggml/src/ggml-cpu/spacemit/rvv_kernels.h +95 -0
  134. data/vendor/ggml/src/ggml-cpu/spacemit/spine_barrier.h +34 -0
  135. data/vendor/ggml/src/ggml-cpu/spacemit/spine_mem_pool.cpp +760 -0
  136. data/vendor/ggml/src/ggml-cpu/spacemit/spine_mem_pool.h +32 -0
  137. data/vendor/ggml/src/ggml-cpu/spacemit/spine_tcm.h +409 -0
  138. data/vendor/ggml/src/ggml-cpu/traits.cpp +36 -0
  139. data/vendor/ggml/src/ggml-cpu/traits.h +38 -0
  140. data/vendor/ggml/src/ggml-cpu/unary-ops.cpp +337 -0
  141. data/vendor/ggml/src/ggml-cpu/unary-ops.h +35 -0
  142. data/vendor/ggml/src/ggml-cpu/vec.cpp +613 -0
  143. data/vendor/ggml/src/ggml-cpu/vec.h +1570 -0
  144. data/vendor/ggml/src/ggml-cuda/CMakeLists.txt +267 -0
  145. data/vendor/ggml/src/ggml-cuda/acc.cu +61 -0
  146. data/vendor/ggml/src/ggml-cuda/acc.cuh +5 -0
  147. data/vendor/ggml/src/ggml-cuda/add-id.cu +58 -0
  148. data/vendor/ggml/src/ggml-cuda/add-id.cuh +3 -0
  149. data/vendor/ggml/src/ggml-cuda/allreduce.cu +971 -0
  150. data/vendor/ggml/src/ggml-cuda/allreduce.cuh +29 -0
  151. data/vendor/ggml/src/ggml-cuda/arange.cu +34 -0
  152. data/vendor/ggml/src/ggml-cuda/arange.cuh +5 -0
  153. data/vendor/ggml/src/ggml-cuda/argmax.cu +91 -0
  154. data/vendor/ggml/src/ggml-cuda/argmax.cuh +3 -0
  155. data/vendor/ggml/src/ggml-cuda/argsort.cu +292 -0
  156. data/vendor/ggml/src/ggml-cuda/argsort.cuh +20 -0
  157. data/vendor/ggml/src/ggml-cuda/binbcast.cu +574 -0
  158. data/vendor/ggml/src/ggml-cuda/binbcast.cuh +12 -0
  159. data/vendor/ggml/src/ggml-cuda/clamp.cu +45 -0
  160. data/vendor/ggml/src/ggml-cuda/clamp.cuh +5 -0
  161. data/vendor/ggml/src/ggml-cuda/col2im-1d.cu +81 -0
  162. data/vendor/ggml/src/ggml-cuda/col2im-1d.cuh +3 -0
  163. data/vendor/ggml/src/ggml-cuda/common.cuh +1678 -0
  164. data/vendor/ggml/src/ggml-cuda/concat.cu +242 -0
  165. data/vendor/ggml/src/ggml-cuda/concat.cuh +5 -0
  166. data/vendor/ggml/src/ggml-cuda/conv-transpose-1d.cu +88 -0
  167. data/vendor/ggml/src/ggml-cuda/conv-transpose-1d.cuh +5 -0
  168. data/vendor/ggml/src/ggml-cuda/conv2d-dw.cu +161 -0
  169. data/vendor/ggml/src/ggml-cuda/conv2d-dw.cuh +5 -0
  170. data/vendor/ggml/src/ggml-cuda/conv2d-transpose.cu +115 -0
  171. data/vendor/ggml/src/ggml-cuda/conv2d-transpose.cuh +5 -0
  172. data/vendor/ggml/src/ggml-cuda/conv2d.cu +167 -0
  173. data/vendor/ggml/src/ggml-cuda/conv2d.cuh +5 -0
  174. data/vendor/ggml/src/ggml-cuda/convert.cu +705 -0
  175. data/vendor/ggml/src/ggml-cuda/convert.cuh +66 -0
  176. data/vendor/ggml/src/ggml-cuda/count-equal.cu +64 -0
  177. data/vendor/ggml/src/ggml-cuda/count-equal.cuh +5 -0
  178. data/vendor/ggml/src/ggml-cuda/cp-async.cuh +57 -0
  179. data/vendor/ggml/src/ggml-cuda/cpy-utils.cuh +217 -0
  180. data/vendor/ggml/src/ggml-cuda/cpy.cu +617 -0
  181. data/vendor/ggml/src/ggml-cuda/cpy.cuh +7 -0
  182. data/vendor/ggml/src/ggml-cuda/cross-entropy-loss.cu +177 -0
  183. data/vendor/ggml/src/ggml-cuda/cross-entropy-loss.cuh +7 -0
  184. data/vendor/ggml/src/ggml-cuda/cumsum.cu +307 -0
  185. data/vendor/ggml/src/ggml-cuda/cumsum.cuh +5 -0
  186. data/vendor/ggml/src/ggml-cuda/dequantize.cuh +452 -0
  187. data/vendor/ggml/src/ggml-cuda/diag.cu +77 -0
  188. data/vendor/ggml/src/ggml-cuda/diag.cuh +5 -0
  189. data/vendor/ggml/src/ggml-cuda/diagmask.cu +40 -0
  190. data/vendor/ggml/src/ggml-cuda/diagmask.cuh +5 -0
  191. data/vendor/ggml/src/ggml-cuda/dsv4-hc.cu +294 -0
  192. data/vendor/ggml/src/ggml-cuda/dsv4-hc.cuh +6 -0
  193. data/vendor/ggml/src/ggml-cuda/fattn-common.cuh +1289 -0
  194. data/vendor/ggml/src/ggml-cuda/fattn-mma-f16.cuh +2136 -0
  195. data/vendor/ggml/src/ggml-cuda/fattn-swizzle.cuh +126 -0
  196. data/vendor/ggml/src/ggml-cuda/fattn-tile.cu +60 -0
  197. data/vendor/ggml/src/ggml-cuda/fattn-tile.cuh +1355 -0
  198. data/vendor/ggml/src/ggml-cuda/fattn-vec.cuh +611 -0
  199. data/vendor/ggml/src/ggml-cuda/fattn.cu +722 -0
  200. data/vendor/ggml/src/ggml-cuda/fattn.cuh +7 -0
  201. data/vendor/ggml/src/ggml-cuda/fill.cu +37 -0
  202. data/vendor/ggml/src/ggml-cuda/fill.cuh +3 -0
  203. data/vendor/ggml/src/ggml-cuda/fwht.cu +101 -0
  204. data/vendor/ggml/src/ggml-cuda/fwht.cuh +4 -0
  205. data/vendor/ggml/src/ggml-cuda/gated_delta_net.cu +327 -0
  206. data/vendor/ggml/src/ggml-cuda/gated_delta_net.cuh +14 -0
  207. data/vendor/ggml/src/ggml-cuda/getrows.cu +490 -0
  208. data/vendor/ggml/src/ggml-cuda/getrows.cuh +15 -0
  209. data/vendor/ggml/src/ggml-cuda/ggml-cuda.cu +5781 -0
  210. data/vendor/ggml/src/ggml-cuda/gla.cu +93 -0
  211. data/vendor/ggml/src/ggml-cuda/gla.cuh +3 -0
  212. data/vendor/ggml/src/ggml-cuda/im2col.cu +267 -0
  213. data/vendor/ggml/src/ggml-cuda/im2col.cuh +6 -0
  214. data/vendor/ggml/src/ggml-cuda/lightning-indexer.cu +588 -0
  215. data/vendor/ggml/src/ggml-cuda/lightning-indexer.cuh +4 -0
  216. data/vendor/ggml/src/ggml-cuda/mean.cu +77 -0
  217. data/vendor/ggml/src/ggml-cuda/mean.cuh +3 -0
  218. data/vendor/ggml/src/ggml-cuda/mma.cuh +1456 -0
  219. data/vendor/ggml/src/ggml-cuda/mmf.cu +191 -0
  220. data/vendor/ggml/src/ggml-cuda/mmf.cuh +908 -0
  221. data/vendor/ggml/src/ggml-cuda/mmid.cu +180 -0
  222. data/vendor/ggml/src/ggml-cuda/mmid.cuh +5 -0
  223. data/vendor/ggml/src/ggml-cuda/mmq-config-ampere.cuh +383 -0
  224. data/vendor/ggml/src/ggml-cuda/mmq-config-blackwell.cuh +37 -0
  225. data/vendor/ggml/src/ggml-cuda/mmq-config-cdna.cuh +185 -0
  226. data/vendor/ggml/src/ggml-cuda/mmq-config-pascal-dp4a.cuh +273 -0
  227. data/vendor/ggml/src/ggml-cuda/mmq-config-pascal-older.cuh +273 -0
  228. data/vendor/ggml/src/ggml-cuda/mmq-config-rdna2.cuh +273 -0
  229. data/vendor/ggml/src/ggml-cuda/mmq-config-rdna3-5.cuh +290 -0
  230. data/vendor/ggml/src/ggml-cuda/mmq-config-rdna3.cuh +274 -0
  231. data/vendor/ggml/src/ggml-cuda/mmq-config-rdna4.cuh +290 -0
  232. data/vendor/ggml/src/ggml-cuda/mmq-load-tiles.cuh +1768 -0
  233. data/vendor/ggml/src/ggml-cuda/mmq-vec-dot.cuh +1240 -0
  234. data/vendor/ggml/src/ggml-cuda/mmq.cu +386 -0
  235. data/vendor/ggml/src/ggml-cuda/mmq.cuh +1598 -0
  236. data/vendor/ggml/src/ggml-cuda/mmvf.cu +875 -0
  237. data/vendor/ggml/src/ggml-cuda/mmvf.cuh +14 -0
  238. data/vendor/ggml/src/ggml-cuda/mmvq.cu +1504 -0
  239. data/vendor/ggml/src/ggml-cuda/mmvq.cuh +18 -0
  240. data/vendor/ggml/src/ggml-cuda/moe-weighted-reduction.cu +65 -0
  241. data/vendor/ggml/src/ggml-cuda/moe-weighted-reduction.cuh +7 -0
  242. data/vendor/ggml/src/ggml-cuda/norm.cu +698 -0
  243. data/vendor/ggml/src/ggml-cuda/norm.cuh +18 -0
  244. data/vendor/ggml/src/ggml-cuda/opt-step-adamw.cu +78 -0
  245. data/vendor/ggml/src/ggml-cuda/opt-step-adamw.cuh +5 -0
  246. data/vendor/ggml/src/ggml-cuda/opt-step-sgd.cu +49 -0
  247. data/vendor/ggml/src/ggml-cuda/opt-step-sgd.cuh +5 -0
  248. data/vendor/ggml/src/ggml-cuda/out-prod.cu +125 -0
  249. data/vendor/ggml/src/ggml-cuda/out-prod.cuh +3 -0
  250. data/vendor/ggml/src/ggml-cuda/pad.cu +106 -0
  251. data/vendor/ggml/src/ggml-cuda/pad.cuh +5 -0
  252. data/vendor/ggml/src/ggml-cuda/pad_reflect_1d.cu +91 -0
  253. data/vendor/ggml/src/ggml-cuda/pad_reflect_1d.cuh +5 -0
  254. data/vendor/ggml/src/ggml-cuda/pool1d.cu +85 -0
  255. data/vendor/ggml/src/ggml-cuda/pool1d.cuh +5 -0
  256. data/vendor/ggml/src/ggml-cuda/pool2d.cu +94 -0
  257. data/vendor/ggml/src/ggml-cuda/pool2d.cuh +5 -0
  258. data/vendor/ggml/src/ggml-cuda/quantize.cu +697 -0
  259. data/vendor/ggml/src/ggml-cuda/quantize.cuh +70 -0
  260. data/vendor/ggml/src/ggml-cuda/reduce_rows.cuh +43 -0
  261. data/vendor/ggml/src/ggml-cuda/roll.cu +67 -0
  262. data/vendor/ggml/src/ggml-cuda/roll.cuh +5 -0
  263. data/vendor/ggml/src/ggml-cuda/rope.cu +941 -0
  264. data/vendor/ggml/src/ggml-cuda/rope.cuh +11 -0
  265. data/vendor/ggml/src/ggml-cuda/scale.cu +37 -0
  266. data/vendor/ggml/src/ggml-cuda/scale.cuh +5 -0
  267. data/vendor/ggml/src/ggml-cuda/set-rows.cu +398 -0
  268. data/vendor/ggml/src/ggml-cuda/set-rows.cuh +7 -0
  269. data/vendor/ggml/src/ggml-cuda/set.cu +39 -0
  270. data/vendor/ggml/src/ggml-cuda/set.cuh +7 -0
  271. data/vendor/ggml/src/ggml-cuda/snake.cu +72 -0
  272. data/vendor/ggml/src/ggml-cuda/snake.cuh +8 -0
  273. data/vendor/ggml/src/ggml-cuda/softcap.cu +37 -0
  274. data/vendor/ggml/src/ggml-cuda/softcap.cuh +5 -0
  275. data/vendor/ggml/src/ggml-cuda/softmax.cu +480 -0
  276. data/vendor/ggml/src/ggml-cuda/softmax.cuh +7 -0
  277. data/vendor/ggml/src/ggml-cuda/solve_tri.cu +273 -0
  278. data/vendor/ggml/src/ggml-cuda/solve_tri.cuh +3 -0
  279. data/vendor/ggml/src/ggml-cuda/ssm-conv.cu +206 -0
  280. data/vendor/ggml/src/ggml-cuda/ssm-conv.cuh +3 -0
  281. data/vendor/ggml/src/ggml-cuda/ssm-scan.cu +859 -0
  282. data/vendor/ggml/src/ggml-cuda/ssm-scan.cuh +3 -0
  283. data/vendor/ggml/src/ggml-cuda/sum.cu +41 -0
  284. data/vendor/ggml/src/ggml-cuda/sum.cuh +5 -0
  285. data/vendor/ggml/src/ggml-cuda/sumrows.cu +47 -0
  286. data/vendor/ggml/src/ggml-cuda/sumrows.cuh +4 -0
  287. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_16.cu +6 -0
  288. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_32.cu +6 -0
  289. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_8.cu +12 -0
  290. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_1.cu +10 -0
  291. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_2.cu +11 -0
  292. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_4.cu +12 -0
  293. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_16.cu +6 -0
  294. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_32.cu +6 -0
  295. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_4.cu +12 -0
  296. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_8.cu +12 -0
  297. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_1.cu +10 -0
  298. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_2.cu +11 -0
  299. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_16.cu +6 -0
  300. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_2.cu +11 -0
  301. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_4.cu +12 -0
  302. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_8.cu +12 -0
  303. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_64-ncols2_1.cu +10 -0
  304. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_1.cu +10 -0
  305. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_2.cu +11 -0
  306. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_4.cu +12 -0
  307. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_8.cu +12 -0
  308. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq112-dv112.cu +5 -0
  309. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq128-dv128.cu +5 -0
  310. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq192-dv128.cu +5 -0
  311. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq256-dv256.cu +5 -0
  312. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq320-dv256.cu +5 -0
  313. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq40-dv40.cu +5 -0
  314. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq512-dv512.cu +5 -0
  315. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq576-dv512.cu +5 -0
  316. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq64-dv64.cu +5 -0
  317. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq72-dv72.cu +5 -0
  318. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq80-dv80.cu +5 -0
  319. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq96-dv96.cu +5 -0
  320. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-bf16.cu +7 -0
  321. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-f16.cu +7 -0
  322. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q4_0.cu +7 -0
  323. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q4_1.cu +7 -0
  324. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q5_0.cu +7 -0
  325. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q5_1.cu +7 -0
  326. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q8_0.cu +7 -0
  327. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-bf16.cu +7 -0
  328. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-f16.cu +7 -0
  329. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q4_0.cu +7 -0
  330. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q4_1.cu +7 -0
  331. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q5_0.cu +7 -0
  332. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q5_1.cu +7 -0
  333. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q8_0.cu +7 -0
  334. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-bf16.cu +7 -0
  335. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-f16.cu +7 -0
  336. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q4_0.cu +7 -0
  337. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q4_1.cu +7 -0
  338. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q5_0.cu +7 -0
  339. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q5_1.cu +7 -0
  340. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q8_0.cu +7 -0
  341. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-bf16.cu +7 -0
  342. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-f16.cu +7 -0
  343. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q4_0.cu +7 -0
  344. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q4_1.cu +7 -0
  345. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q5_0.cu +7 -0
  346. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q5_1.cu +7 -0
  347. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q8_0.cu +7 -0
  348. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-bf16.cu +7 -0
  349. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-f16.cu +7 -0
  350. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q4_0.cu +7 -0
  351. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q4_1.cu +7 -0
  352. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q5_0.cu +7 -0
  353. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q5_1.cu +7 -0
  354. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q8_0.cu +7 -0
  355. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-bf16.cu +7 -0
  356. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-f16.cu +7 -0
  357. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q4_0.cu +7 -0
  358. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q4_1.cu +7 -0
  359. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q5_0.cu +7 -0
  360. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q5_1.cu +7 -0
  361. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q8_0.cu +7 -0
  362. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-bf16.cu +7 -0
  363. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-f16.cu +7 -0
  364. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q4_0.cu +7 -0
  365. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q4_1.cu +7 -0
  366. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q5_0.cu +7 -0
  367. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q5_1.cu +7 -0
  368. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q8_0.cu +7 -0
  369. data/vendor/ggml/src/ggml-cuda/template-instances/generate_cu_files.py +111 -0
  370. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_1.cu +5 -0
  371. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_10.cu +5 -0
  372. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_11.cu +5 -0
  373. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_12.cu +5 -0
  374. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_13.cu +5 -0
  375. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_14.cu +5 -0
  376. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_15.cu +5 -0
  377. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_16.cu +5 -0
  378. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_2.cu +5 -0
  379. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_3.cu +5 -0
  380. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_4.cu +5 -0
  381. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_5.cu +5 -0
  382. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_6.cu +5 -0
  383. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_7.cu +5 -0
  384. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_8.cu +5 -0
  385. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_9.cu +5 -0
  386. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq1_s.cu +5 -0
  387. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_s.cu +5 -0
  388. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xs.cu +5 -0
  389. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xxs.cu +5 -0
  390. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_s.cu +5 -0
  391. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_xxs.cu +5 -0
  392. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_nl.cu +5 -0
  393. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_xs.cu +5 -0
  394. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-mxfp4.cu +5 -0
  395. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-nvfp4.cu +5 -0
  396. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q1_0.cu +5 -0
  397. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_0.cu +5 -0
  398. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_k.cu +5 -0
  399. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q3_k.cu +5 -0
  400. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_0.cu +5 -0
  401. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_1.cu +5 -0
  402. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_k.cu +5 -0
  403. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_0.cu +5 -0
  404. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_1.cu +5 -0
  405. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_k.cu +5 -0
  406. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q6_k.cu +5 -0
  407. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q8_0.cu +5 -0
  408. data/vendor/ggml/src/ggml-cuda/top-k.cu +275 -0
  409. data/vendor/ggml/src/ggml-cuda/top-k.cuh +3 -0
  410. data/vendor/ggml/src/ggml-cuda/topk-moe.cu +442 -0
  411. data/vendor/ggml/src/ggml-cuda/topk-moe.cuh +31 -0
  412. data/vendor/ggml/src/ggml-cuda/tri.cu +136 -0
  413. data/vendor/ggml/src/ggml-cuda/tri.cuh +5 -0
  414. data/vendor/ggml/src/ggml-cuda/tsembd.cu +47 -0
  415. data/vendor/ggml/src/ggml-cuda/tsembd.cuh +5 -0
  416. data/vendor/ggml/src/ggml-cuda/unary.cu +721 -0
  417. data/vendor/ggml/src/ggml-cuda/unary.cuh +123 -0
  418. data/vendor/ggml/src/ggml-cuda/upscale.cu +293 -0
  419. data/vendor/ggml/src/ggml-cuda/upscale.cuh +5 -0
  420. data/vendor/ggml/src/ggml-cuda/vecdotq.cuh +1371 -0
  421. data/vendor/ggml/src/ggml-cuda/vendors/cuda.h +28 -0
  422. data/vendor/ggml/src/ggml-cuda/vendors/hip.h +301 -0
  423. data/vendor/ggml/src/ggml-cuda/vendors/musa.h +151 -0
  424. data/vendor/ggml/src/ggml-cuda/wkv.cu +253 -0
  425. data/vendor/ggml/src/ggml-cuda/wkv.cuh +7 -0
  426. data/vendor/ggml/src/ggml-et/CMakeLists.txt +246 -0
  427. data/vendor/ggml/src/ggml-et/cmake/embed_one_kernel.cmake +15 -0
  428. data/vendor/ggml/src/ggml-et/cmake/ggml-et-kernels-embed.cpp.in +6 -0
  429. data/vendor/ggml/src/ggml-et/cmake/ggml-et-kernels-embed.hpp.in +12 -0
  430. data/vendor/ggml/src/ggml-et/cmake/ggml-et-uberkernel-kernel-map.cpp.in +18 -0
  431. data/vendor/ggml/src/ggml-et/cmake/ggml-et-uberkernel-kernel-map.h.in +13 -0
  432. data/vendor/ggml/src/ggml-et/et-kernels/CMakeLists.txt +137 -0
  433. data/vendor/ggml/src/ggml-et/et-kernels/scripts/check_unimplemented_instructions.sh +36 -0
  434. data/vendor/ggml/src/ggml-et/et-kernels/src/RunBackend.sh +23 -0
  435. data/vendor/ggml/src/ggml-et/et-kernels/src/block_ops.h +997 -0
  436. data/vendor/ggml/src/ggml-et/et-kernels/src/clamp_f32.c +120 -0
  437. data/vendor/ggml/src/ggml-et/et-kernels/src/concat_f32.c +175 -0
  438. data/vendor/ggml/src/ggml-et/et-kernels/src/cont_f16.c +107 -0
  439. data/vendor/ggml/src/ggml-et/et-kernels/src/cont_f32.c +248 -0
  440. data/vendor/ggml/src/ggml-et/et-kernels/src/conv_2d_f32_me.c +807 -0
  441. data/vendor/ggml/src/ggml-et/et-kernels/src/cpy_f32_f16.c +110 -0
  442. data/vendor/ggml/src/ggml-et/et-kernels/src/crt.S +15 -0
  443. data/vendor/ggml/src/ggml-et/et-kernels/src/cumsum_f32.c +96 -0
  444. data/vendor/ggml/src/ggml-et/et-kernels/src/diag_f32.c +90 -0
  445. data/vendor/ggml/src/ggml-et/et-kernels/src/el_map_f32.c +377 -0
  446. data/vendor/ggml/src/ggml-et/et-kernels/src/fill_f32.c +87 -0
  447. data/vendor/ggml/src/ggml-et/et-kernels/src/flash_attn_ext_f16_me.c +1000 -0
  448. data/vendor/ggml/src/ggml-et/et-kernels/src/flash_attn_ext_f32.c +217 -0
  449. data/vendor/ggml/src/ggml-et/et-kernels/src/gated_delta_net_f32.c +346 -0
  450. data/vendor/ggml/src/ggml-et/et-kernels/src/get_rows_f32.c +612 -0
  451. data/vendor/ggml/src/ggml-et/et-kernels/src/ggml_tensor.h +44 -0
  452. data/vendor/ggml/src/ggml-et/et-kernels/src/glu_f32.c +606 -0
  453. data/vendor/ggml/src/ggml-et/et-kernels/src/group_norm_f32.c +171 -0
  454. data/vendor/ggml/src/ggml-et/et-kernels/src/im2col.c +130 -0
  455. data/vendor/ggml/src/ggml-et/et-kernels/src/l2_norm_f32.c +237 -0
  456. data/vendor/ggml/src/ggml-et/et-kernels/src/linker.ld +85 -0
  457. data/vendor/ggml/src/ggml-et/et-kernels/src/math_fp.h +299 -0
  458. data/vendor/ggml/src/ggml-et/et-kernels/src/mean_f32.c +220 -0
  459. data/vendor/ggml/src/ggml-et/et-kernels/src/memops.c +181 -0
  460. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0.c +358 -0
  461. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0_matrix_engine.c +368 -0
  462. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_Q8_0.c +413 -0
  463. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f16.c +142 -0
  464. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f16_matrix_engine.c +329 -0
  465. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f32.c +137 -0
  466. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f32_matrix_engine.c +155 -0
  467. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_id_Q4_0.c +169 -0
  468. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_id_Q8_0.c +160 -0
  469. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_id_f32.c +288 -0
  470. data/vendor/ggml/src/ggml-et/et-kernels/src/norm_f32.c +328 -0
  471. data/vendor/ggml/src/ggml-et/et-kernels/src/pad_f32.c +165 -0
  472. data/vendor/ggml/src/ggml-et/et-kernels/src/platform.h +545 -0
  473. data/vendor/ggml/src/ggml-et/et-kernels/src/quants.h +72 -0
  474. data/vendor/ggml/src/ggml-et/et-kernels/src/repeat_f32.c +118 -0
  475. data/vendor/ggml/src/ggml-et/et-kernels/src/rms_norm_f32.c +270 -0
  476. data/vendor/ggml/src/ggml-et/et-kernels/src/rms_norm_mul_f32.c +290 -0
  477. data/vendor/ggml/src/ggml-et/et-kernels/src/rope_f32.c +656 -0
  478. data/vendor/ggml/src/ggml-et/et-kernels/src/rwkv_wkv6_f32.c +184 -0
  479. data/vendor/ggml/src/ggml-et/et-kernels/src/rwkv_wkv7_f32.c +272 -0
  480. data/vendor/ggml/src/ggml-et/et-kernels/src/scale_f32.c +94 -0
  481. data/vendor/ggml/src/ggml-et/et-kernels/src/set_f32.c +101 -0
  482. data/vendor/ggml/src/ggml-et/et-kernels/src/set_rows_f32.c +394 -0
  483. data/vendor/ggml/src/ggml-et/et-kernels/src/softmax_f32.c +698 -0
  484. data/vendor/ggml/src/ggml-et/et-kernels/src/solve_tri_f32.c +109 -0
  485. data/vendor/ggml/src/ggml-et/et-kernels/src/sqr_f32.c +88 -0
  486. data/vendor/ggml/src/ggml-et/et-kernels/src/ssm_conv_f32.c +129 -0
  487. data/vendor/ggml/src/ggml-et/et-kernels/src/ssm_scan_f32.c +282 -0
  488. data/vendor/ggml/src/ggml-et/et-kernels/src/sum_rows_f32.c +103 -0
  489. data/vendor/ggml/src/ggml-et/et-kernels/src/tensor.h +897 -0
  490. data/vendor/ggml/src/ggml-et/et-kernels/src/tri_f32.c +244 -0
  491. data/vendor/ggml/src/ggml-et/et-kernels/src/uberkernel.c +497 -0
  492. data/vendor/ggml/src/ggml-et/et-kernels/src/unary_f32.c +705 -0
  493. data/vendor/ggml/src/ggml-et/ggml-et-common.h +86 -0
  494. data/vendor/ggml/src/ggml-et/ggml-et-cpu-compare.cpp +502 -0
  495. data/vendor/ggml/src/ggml-et/ggml-et-cpu-compare.h +54 -0
  496. data/vendor/ggml/src/ggml-et/ggml-et-kernels.cpp +508 -0
  497. data/vendor/ggml/src/ggml-et/ggml-et-kernels.h +48 -0
  498. data/vendor/ggml/src/ggml-et/ggml-et-memops.cpp +36 -0
  499. data/vendor/ggml/src/ggml-et/ggml-et-memops.h +18 -0
  500. data/vendor/ggml/src/ggml-et/ggml-et-ops.cpp +2584 -0
  501. data/vendor/ggml/src/ggml-et/ggml-et-ops.h +393 -0
  502. data/vendor/ggml/src/ggml-et/ggml-et-uberkernel-common.h +17 -0
  503. data/vendor/ggml/src/ggml-et/ggml-et.cpp +1880 -0
  504. data/vendor/ggml/src/ggml-feats.h +166 -0
  505. data/vendor/ggml/src/ggml-hexagon/CMakeLists.txt +113 -0
  506. data/vendor/ggml/src/ggml-hexagon/ggml-hexagon.cpp +6523 -0
  507. data/vendor/ggml/src/ggml-hexagon/htp/CMakeLists.txt +58 -0
  508. data/vendor/ggml/src/ggml-hexagon/htp/act-ops.c +585 -0
  509. data/vendor/ggml/src/ggml-hexagon/htp/allreduce-ops.c +398 -0
  510. data/vendor/ggml/src/ggml-hexagon/htp/allreduce-ops.h +40 -0
  511. data/vendor/ggml/src/ggml-hexagon/htp/argsort-ops.c +514 -0
  512. data/vendor/ggml/src/ggml-hexagon/htp/binary-ops.c +873 -0
  513. data/vendor/ggml/src/ggml-hexagon/htp/cmake-toolchain.cmake +155 -0
  514. data/vendor/ggml/src/ggml-hexagon/htp/concat-ops.c +277 -0
  515. data/vendor/ggml/src/ggml-hexagon/htp/cpy-ops.c +354 -0
  516. data/vendor/ggml/src/ggml-hexagon/htp/cumsum-ops.c +265 -0
  517. data/vendor/ggml/src/ggml-hexagon/htp/diag-ops.c +216 -0
  518. data/vendor/ggml/src/ggml-hexagon/htp/dma-queue.c +104 -0
  519. data/vendor/ggml/src/ggml-hexagon/htp/dma-queue.h +408 -0
  520. data/vendor/ggml/src/ggml-hexagon/htp/fill-ops.c +123 -0
  521. data/vendor/ggml/src/ggml-hexagon/htp/flash-attn-ops.c +2479 -0
  522. data/vendor/ggml/src/ggml-hexagon/htp/flash-attn-ops.h +344 -0
  523. data/vendor/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c +1157 -0
  524. data/vendor/ggml/src/ggml-hexagon/htp/get-rows-ops.c +260 -0
  525. data/vendor/ggml/src/ggml-hexagon/htp/get-rows-ops.h +77 -0
  526. data/vendor/ggml/src/ggml-hexagon/htp/hex-bitmap.h +24 -0
  527. data/vendor/ggml/src/ggml-hexagon/htp/hex-common.h +80 -0
  528. data/vendor/ggml/src/ggml-hexagon/htp/hex-dma.h +2 -0
  529. data/vendor/ggml/src/ggml-hexagon/htp/hex-dump.h +86 -0
  530. data/vendor/ggml/src/ggml-hexagon/htp/hex-fastdiv.h +37 -0
  531. data/vendor/ggml/src/ggml-hexagon/htp/hex-profile.h +64 -0
  532. data/vendor/ggml/src/ggml-hexagon/htp/hex-utils.h +65 -0
  533. data/vendor/ggml/src/ggml-hexagon/htp/hmx-fa-kernels.h +555 -0
  534. data/vendor/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h +1378 -0
  535. data/vendor/ggml/src/ggml-hexagon/htp/hmx-queue.c +170 -0
  536. data/vendor/ggml/src/ggml-hexagon/htp/hmx-queue.h +160 -0
  537. data/vendor/ggml/src/ggml-hexagon/htp/hmx-utils.h +222 -0
  538. data/vendor/ggml/src/ggml-hexagon/htp/htp-ctx.h +146 -0
  539. data/vendor/ggml/src/ggml-hexagon/htp/htp-ops.h +242 -0
  540. data/vendor/ggml/src/ggml-hexagon/htp/htp-tensor.c +300 -0
  541. data/vendor/ggml/src/ggml-hexagon/htp/htp-tensor.h +29 -0
  542. data/vendor/ggml/src/ggml-hexagon/htp/htp-vtcm.h +19 -0
  543. data/vendor/ggml/src/ggml-hexagon/htp/htp_iface.idl +23 -0
  544. data/vendor/ggml/src/ggml-hexagon/htp/hvx-arith.h +636 -0
  545. data/vendor/ggml/src/ggml-hexagon/htp/hvx-base.h +292 -0
  546. data/vendor/ggml/src/ggml-hexagon/htp/hvx-copy.h +262 -0
  547. data/vendor/ggml/src/ggml-hexagon/htp/hvx-div.h +291 -0
  548. data/vendor/ggml/src/ggml-hexagon/htp/hvx-dump.h +129 -0
  549. data/vendor/ggml/src/ggml-hexagon/htp/hvx-exp.h +255 -0
  550. data/vendor/ggml/src/ggml-hexagon/htp/hvx-fa-kernels.h +303 -0
  551. data/vendor/ggml/src/ggml-hexagon/htp/hvx-flash-attn.h +47 -0
  552. data/vendor/ggml/src/ggml-hexagon/htp/hvx-floor.h +100 -0
  553. data/vendor/ggml/src/ggml-hexagon/htp/hvx-inverse.h +210 -0
  554. data/vendor/ggml/src/ggml-hexagon/htp/hvx-log.h +118 -0
  555. data/vendor/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-flat.h +1511 -0
  556. data/vendor/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h +1200 -0
  557. data/vendor/ggml/src/ggml-hexagon/htp/hvx-norm.h +454 -0
  558. data/vendor/ggml/src/ggml-hexagon/htp/hvx-pow.h +42 -0
  559. data/vendor/ggml/src/ggml-hexagon/htp/hvx-quant.h +165 -0
  560. data/vendor/ggml/src/ggml-hexagon/htp/hvx-reduce.h +336 -0
  561. data/vendor/ggml/src/ggml-hexagon/htp/hvx-repl.h +74 -0
  562. data/vendor/ggml/src/ggml-hexagon/htp/hvx-scale.h +199 -0
  563. data/vendor/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h +181 -0
  564. data/vendor/ggml/src/ggml-hexagon/htp/hvx-sin-cos.h +90 -0
  565. data/vendor/ggml/src/ggml-hexagon/htp/hvx-sqrt.h +189 -0
  566. data/vendor/ggml/src/ggml-hexagon/htp/hvx-types.h +36 -0
  567. data/vendor/ggml/src/ggml-hexagon/htp/hvx-utils.h +24 -0
  568. data/vendor/ggml/src/ggml-hexagon/htp/im2col-ops.c +306 -0
  569. data/vendor/ggml/src/ggml-hexagon/htp/main.c +1245 -0
  570. data/vendor/ggml/src/ggml-hexagon/htp/matmul-ops.c +4078 -0
  571. data/vendor/ggml/src/ggml-hexagon/htp/matmul-ops.h +778 -0
  572. data/vendor/ggml/src/ggml-hexagon/htp/pad-ops.c +547 -0
  573. data/vendor/ggml/src/ggml-hexagon/htp/repeat-ops.c +148 -0
  574. data/vendor/ggml/src/ggml-hexagon/htp/rope-ops.c +733 -0
  575. data/vendor/ggml/src/ggml-hexagon/htp/set-rows-ops.c +230 -0
  576. data/vendor/ggml/src/ggml-hexagon/htp/set-rows-ops.h +74 -0
  577. data/vendor/ggml/src/ggml-hexagon/htp/softmax-ops.c +407 -0
  578. data/vendor/ggml/src/ggml-hexagon/htp/solve-tri-ops.c +267 -0
  579. data/vendor/ggml/src/ggml-hexagon/htp/ssm-conv.c +433 -0
  580. data/vendor/ggml/src/ggml-hexagon/htp/sum-rows-ops.c +128 -0
  581. data/vendor/ggml/src/ggml-hexagon/htp/unary-ops.c +1264 -0
  582. data/vendor/ggml/src/ggml-hexagon/htp/unary-ops.h +176 -0
  583. data/vendor/ggml/src/ggml-hexagon/htp/work-queue.c +244 -0
  584. data/vendor/ggml/src/ggml-hexagon/htp/work-queue.h +38 -0
  585. data/vendor/ggml/src/ggml-hexagon/htp-drv.cpp +425 -0
  586. data/vendor/ggml/src/ggml-hexagon/htp-drv.h +123 -0
  587. data/vendor/ggml/src/ggml-hexagon/htp-opnode.h +372 -0
  588. data/vendor/ggml/src/ggml-hexagon/libdl.h +79 -0
  589. data/vendor/ggml/src/ggml-hexagon/libggml-htp.inf +36 -0
  590. data/vendor/ggml/src/ggml-hip/CMakeLists.txt +153 -0
  591. data/vendor/ggml/src/ggml-impl.h +783 -0
  592. data/vendor/ggml/src/ggml-metal/CMakeLists.txt +240 -0
  593. data/vendor/ggml/src/ggml-metal/ggml-metal-common.cpp +474 -0
  594. data/vendor/ggml/src/ggml-metal/ggml-metal-common.h +56 -0
  595. data/vendor/ggml/src/ggml-metal/ggml-metal-context.h +41 -0
  596. data/vendor/ggml/src/ggml-metal/ggml-metal-context.m +759 -0
  597. data/vendor/ggml/src/ggml-metal/ggml-metal-device.cpp +2330 -0
  598. data/vendor/ggml/src/ggml-metal/ggml-metal-device.h +355 -0
  599. data/vendor/ggml/src/ggml-metal/ggml-metal-device.m +2473 -0
  600. data/vendor/ggml/src/ggml-metal/ggml-metal-impl.h +1345 -0
  601. data/vendor/ggml/src/ggml-metal/ggml-metal-ops.cpp +5610 -0
  602. data/vendor/ggml/src/ggml-metal/ggml-metal-ops.h +106 -0
  603. data/vendor/ggml/src/ggml-metal/ggml-metal-tuning.cpp +3732 -0
  604. data/vendor/ggml/src/ggml-metal/ggml-metal-tuning.h +77 -0
  605. data/vendor/ggml/src/ggml-metal/ggml-metal.cpp +1006 -0
  606. data/vendor/ggml/src/ggml-metal/kernels/argsort.metal +337 -0
  607. data/vendor/ggml/src/ggml-metal/kernels/binbcast.metal +228 -0
  608. data/vendor/ggml/src/ggml-metal/kernels/common.h +126 -0
  609. data/vendor/ggml/src/ggml-metal/kernels/conv.metal +724 -0
  610. data/vendor/ggml/src/ggml-metal/kernels/dequantize.h +735 -0
  611. data/vendor/ggml/src/ggml-metal/kernels/fa.metal +2445 -0
  612. data/vendor/ggml/src/ggml-metal/kernels/gated_delta_net.metal +250 -0
  613. data/vendor/ggml/src/ggml-metal/kernels/misc.metal +595 -0
  614. data/vendor/ggml/src/ggml-metal/kernels/mul_mm.metal +853 -0
  615. data/vendor/ggml/src/ggml-metal/kernels/mul_mv.metal +3225 -0
  616. data/vendor/ggml/src/ggml-metal/kernels/norm.metal +308 -0
  617. data/vendor/ggml/src/ggml-metal/kernels/pool.metal +148 -0
  618. data/vendor/ggml/src/ggml-metal/kernels/quantize.h +262 -0
  619. data/vendor/ggml/src/ggml-metal/kernels/quantize.metal +480 -0
  620. data/vendor/ggml/src/ggml-metal/kernels/reduce.metal +228 -0
  621. data/vendor/ggml/src/ggml-metal/kernels/rope.metal +333 -0
  622. data/vendor/ggml/src/ggml-metal/kernels/softmax.metal +223 -0
  623. data/vendor/ggml/src/ggml-metal/kernels/solve_tri.metal +75 -0
  624. data/vendor/ggml/src/ggml-metal/kernels/ssm.metal +473 -0
  625. data/vendor/ggml/src/ggml-metal/kernels/tri.metal +69 -0
  626. data/vendor/ggml/src/ggml-metal/kernels/unary.metal +400 -0
  627. data/vendor/ggml/src/ggml-metal/kernels/upscale.metal +179 -0
  628. data/vendor/ggml/src/ggml-metal/kernels/wkv.metal +179 -0
  629. data/vendor/ggml/src/ggml-musa/CMakeLists.txt +123 -0
  630. data/vendor/ggml/src/ggml-musa/mudnn.cu +112 -0
  631. data/vendor/ggml/src/ggml-musa/mudnn.cuh +12 -0
  632. data/vendor/ggml/src/ggml-opencl/CMakeLists.txt +246 -0
  633. data/vendor/ggml/src/ggml-opencl/cl-program-cache.cpp +453 -0
  634. data/vendor/ggml/src/ggml-opencl/cl-program-cache.h +75 -0
  635. data/vendor/ggml/src/ggml-opencl/fa_tune.h +92 -0
  636. data/vendor/ggml/src/ggml-opencl/ggml-opencl.cpp +26982 -0
  637. data/vendor/ggml/src/ggml-opencl/kernels/abs.cl +113 -0
  638. data/vendor/ggml/src/ggml-opencl/kernels/add.cl +190 -0
  639. data/vendor/ggml/src/ggml-opencl/kernels/add_id.cl +42 -0
  640. data/vendor/ggml/src/ggml-opencl/kernels/argsort.cl +86 -0
  641. data/vendor/ggml/src/ggml-opencl/kernels/clamp.cl +20 -0
  642. data/vendor/ggml/src/ggml-opencl/kernels/concat.cl +118 -0
  643. data/vendor/ggml/src/ggml-opencl/kernels/conv2d.cl +185 -0
  644. data/vendor/ggml/src/ggml-opencl/kernels/conv2d_f16_f32.cl +176 -0
  645. data/vendor/ggml/src/ggml-opencl/kernels/cpy.cl +288 -0
  646. data/vendor/ggml/src/ggml-opencl/kernels/cumsum.cl +139 -0
  647. data/vendor/ggml/src/ggml-opencl/kernels/cvt.cl +2663 -0
  648. data/vendor/ggml/src/ggml-opencl/kernels/diag.cl +27 -0
  649. data/vendor/ggml/src/ggml-opencl/kernels/diag_mask_inf.cl +58 -0
  650. data/vendor/ggml/src/ggml-opencl/kernels/div.cl +138 -0
  651. data/vendor/ggml/src/ggml-opencl/kernels/embed_kernel.py +26 -0
  652. data/vendor/ggml/src/ggml-opencl/kernels/exp.cl +125 -0
  653. data/vendor/ggml/src/ggml-opencl/kernels/expm1.cl +113 -0
  654. data/vendor/ggml/src/ggml-opencl/kernels/fill.cl +17 -0
  655. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f16.cl +421 -0
  656. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32.cl +422 -0
  657. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32_f16.cl +2656 -0
  658. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32_q4_0.cl +2011 -0
  659. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32_q8_0.cl +1840 -0
  660. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_pre_f16.cl +156 -0
  661. data/vendor/ggml/src/ggml-opencl/kernels/gated_delta_net.cl +249 -0
  662. data/vendor/ggml/src/ggml-opencl/kernels/gelu.cl +89 -0
  663. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_f32.cl +162 -0
  664. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_f32_ns.cl +376 -0
  665. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_q8_1_dp4a.cl +190 -0
  666. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_0_f32_ns.cl +324 -0
  667. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_0_q8_1_dp4a.cl +169 -0
  668. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_1_f32_ns.cl +326 -0
  669. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_k_f32_ns.cl +348 -0
  670. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_k_q8_1_dp4a.cl +209 -0
  671. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q5_0_f32_ns.cl +328 -0
  672. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q5_1_f32_ns.cl +330 -0
  673. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q5_k_f32_ns.cl +356 -0
  674. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q6_k_f32_ns.cl +335 -0
  675. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q6_k_q8_1_dp4a.cl +200 -0
  676. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q8_0_f32_ns.cl +221 -0
  677. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q8_1_dp4a.cl +226 -0
  678. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_nl_f32.cl +150 -0
  679. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_nl_q8_1_dp4a.cl +143 -0
  680. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q1_0_f32.cl +94 -0
  681. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_f32.cl +139 -0
  682. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_q8_1_dp4a.cl +127 -0
  683. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_1_f32.cl +132 -0
  684. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_f32.cl +490 -0
  685. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_q8_1_dp4a.cl +281 -0
  686. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_0_f32.cl +131 -0
  687. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_0_q8_1_dp4a.cl +235 -0
  688. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_1_f32.cl +134 -0
  689. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_f32.cl +176 -0
  690. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_q8_1_dp4a.cl +164 -0
  691. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32.cl +245 -0
  692. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32_tiled.cl +136 -0
  693. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_q8_1_dp4a.cl +144 -0
  694. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_f32.cl +129 -0
  695. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_q8_1_dp4a.cl +212 -0
  696. data/vendor/ggml/src/ggml-opencl/kernels/gemm_xmem_f16_f32_os8.cl +233 -0
  697. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32.cl +156 -0
  698. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32_ns.cl +257 -0
  699. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q4_0_f32_ns.cl +120 -0
  700. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q4_1_f32_ns.cl +123 -0
  701. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q4_k_f32_ns.cl +266 -0
  702. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q5_0_f32_ns.cl +123 -0
  703. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q5_1_f32_ns.cl +125 -0
  704. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q5_k_f32_ns.cl +160 -0
  705. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q6_k_f32_ns.cl +141 -0
  706. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq4_nl_f32.cl +307 -0
  707. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q1_0_f32.cl +125 -0
  708. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32.cl +383 -0
  709. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32_spec.cl +272 -0
  710. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_1_f32.cl +384 -0
  711. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32.cl +823 -0
  712. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_o4.cl +349 -0
  713. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_tiled.cl +118 -0
  714. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_0_f32.cl +296 -0
  715. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_1_f32.cl +299 -0
  716. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_k_f32.cl +453 -0
  717. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32.cl +409 -0
  718. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_o4.cl +372 -0
  719. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_tiled.cl +196 -0
  720. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q8_0_f32.cl +280 -0
  721. data/vendor/ggml/src/ggml-opencl/kernels/get_rows.cl +193 -0
  722. data/vendor/ggml/src/ggml-opencl/kernels/glu.cl +443 -0
  723. data/vendor/ggml/src/ggml-opencl/kernels/group_norm.cl +121 -0
  724. data/vendor/ggml/src/ggml-opencl/kernels/im2col_f16.cl +57 -0
  725. data/vendor/ggml/src/ggml-opencl/kernels/im2col_f32.cl +57 -0
  726. data/vendor/ggml/src/ggml-opencl/kernels/l2_norm.cl +71 -0
  727. data/vendor/ggml/src/ggml-opencl/kernels/mean.cl +140 -0
  728. data/vendor/ggml/src/ggml-opencl/kernels/moe_add_id_glu.cl +76 -0
  729. data/vendor/ggml/src/ggml-opencl/kernels/moe_combine.cl +79 -0
  730. data/vendor/ggml/src/ggml-opencl/kernels/moe_reorder_b.cl +30 -0
  731. data/vendor/ggml/src/ggml-opencl/kernels/moe_reorder_quant_a_q8_1.cl +64 -0
  732. data/vendor/ggml/src/ggml-opencl/kernels/moe_sort_by_expert.cl +155 -0
  733. data/vendor/ggml/src/ggml-opencl/kernels/mul.cl +152 -0
  734. data/vendor/ggml/src/ggml-opencl/kernels/mul_mat_f16_f32.cl +130 -0
  735. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_kq_kqv.cl +273 -0
  736. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_l4_lm.cl +146 -0
  737. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_f32_f32_l4_lm.cl +196 -0
  738. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_iq4_nl_f32_l4_lm.cl +171 -0
  739. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q1_0_f32_l4_lm.cl +156 -0
  740. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q4_0_f32_l4_lm.cl +163 -0
  741. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q4_1_f32_l4_lm.cl +165 -0
  742. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q4_k_f32_l4_lm.cl +189 -0
  743. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q5_0_f32_l4_lm.cl +173 -0
  744. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q5_1_f32_l4_lm.cl +175 -0
  745. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q5_k_f32_l4_lm.cl +202 -0
  746. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q6_k_f32_l4_lm.cl +158 -0
  747. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q8_0_f32_l4_lm.cl +154 -0
  748. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f16.cl +125 -0
  749. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32.cl +125 -0
  750. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_1row.cl +101 -0
  751. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_l4.cl +1237 -0
  752. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_mrow.cl +306 -0
  753. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f32_f32.cl +118 -0
  754. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_mxfp4_f32.cl +189 -0
  755. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_mxfp4_f32_flat.cl +176 -0
  756. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_q4_0_f32_8x_flat.cl +283 -0
  757. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_q8_0_f32.cl +140 -0
  758. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_q8_0_f32_flat.cl +222 -0
  759. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_iq4_nl_f32.cl +164 -0
  760. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_iq4_nl_f32_flat.cl +202 -0
  761. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_mxfp4_f32.cl +144 -0
  762. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_mxfp4_f32_flat.cl +167 -0
  763. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q1_0_f32.cl +141 -0
  764. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q1_0_f32_flat.cl +190 -0
  765. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32.cl +192 -0
  766. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_16x_flat.cl +307 -0
  767. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_8x_flat.cl +265 -0
  768. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_8x_flat.cl +272 -0
  769. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_v.cl +254 -0
  770. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_1_f32.cl +219 -0
  771. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_1_f32_flat.cl +229 -0
  772. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32.cl +182 -0
  773. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32_flat.cl +205 -0
  774. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_0_f32.cl +241 -0
  775. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_0_f32_flat.cl +243 -0
  776. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_1_f32.cl +243 -0
  777. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_1_f32_flat.cl +247 -0
  778. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_k_f32.cl +187 -0
  779. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_k_f32_flat.cl +244 -0
  780. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32.cl +194 -0
  781. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32_flat.cl +272 -0
  782. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q8_0_f32.cl +125 -0
  783. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q8_0_f32_flat.cl +202 -0
  784. data/vendor/ggml/src/ggml-opencl/kernels/neg.cl +125 -0
  785. data/vendor/ggml/src/ggml-opencl/kernels/norm.cl +164 -0
  786. data/vendor/ggml/src/ggml-opencl/kernels/pad.cl +39 -0
  787. data/vendor/ggml/src/ggml-opencl/kernels/quant_a_q8_1.cl +42 -0
  788. data/vendor/ggml/src/ggml-opencl/kernels/relu.cl +16 -0
  789. data/vendor/ggml/src/ggml-opencl/kernels/repeat.cl +38 -0
  790. data/vendor/ggml/src/ggml-opencl/kernels/rms_norm.cl +369 -0
  791. data/vendor/ggml/src/ggml-opencl/kernels/rope.cl +759 -0
  792. data/vendor/ggml/src/ggml-opencl/kernels/scale.cl +27 -0
  793. data/vendor/ggml/src/ggml-opencl/kernels/set_rows.cl +708 -0
  794. data/vendor/ggml/src/ggml-opencl/kernels/sigmoid.cl +29 -0
  795. data/vendor/ggml/src/ggml-opencl/kernels/silu.cl +30 -0
  796. data/vendor/ggml/src/ggml-opencl/kernels/softmax_4_f16.cl +108 -0
  797. data/vendor/ggml/src/ggml-opencl/kernels/softmax_4_f32.cl +108 -0
  798. data/vendor/ggml/src/ggml-opencl/kernels/softmax_f16.cl +107 -0
  799. data/vendor/ggml/src/ggml-opencl/kernels/softmax_f32.cl +107 -0
  800. data/vendor/ggml/src/ggml-opencl/kernels/softplus.cl +116 -0
  801. data/vendor/ggml/src/ggml-opencl/kernels/solve_tri.cl +51 -0
  802. data/vendor/ggml/src/ggml-opencl/kernels/sqr.cl +53 -0
  803. data/vendor/ggml/src/ggml-opencl/kernels/sqrt.cl +53 -0
  804. data/vendor/ggml/src/ggml-opencl/kernels/ssm_conv.cl +77 -0
  805. data/vendor/ggml/src/ggml-opencl/kernels/ssm_scan.cl +216 -0
  806. data/vendor/ggml/src/ggml-opencl/kernels/sub.cl +138 -0
  807. data/vendor/ggml/src/ggml-opencl/kernels/sum_rows.cl +140 -0
  808. data/vendor/ggml/src/ggml-opencl/kernels/tanh.cl +109 -0
  809. data/vendor/ggml/src/ggml-opencl/kernels/transpose.cl +143 -0
  810. data/vendor/ggml/src/ggml-opencl/kernels/tri.cl +32 -0
  811. data/vendor/ggml/src/ggml-opencl/kernels/tsembd.cl +48 -0
  812. data/vendor/ggml/src/ggml-opencl/kernels/upscale.cl +120 -0
  813. data/vendor/ggml/src/ggml-opencl/libdl.h +79 -0
  814. data/vendor/ggml/src/ggml-openvino/CMakeLists.txt +22 -0
  815. data/vendor/ggml/src/ggml-openvino/ggml-decoder.cpp +2057 -0
  816. data/vendor/ggml/src/ggml-openvino/ggml-decoder.h +417 -0
  817. data/vendor/ggml/src/ggml-openvino/ggml-openvino-extra.cpp +493 -0
  818. data/vendor/ggml/src/ggml-openvino/ggml-openvino-extra.h +216 -0
  819. data/vendor/ggml/src/ggml-openvino/ggml-openvino.cpp +1566 -0
  820. data/vendor/ggml/src/ggml-openvino/ggml-quants.cpp +1273 -0
  821. data/vendor/ggml/src/ggml-openvino/ggml-quants.h +186 -0
  822. data/vendor/ggml/src/ggml-openvino/model-cache.cpp +272 -0
  823. data/vendor/ggml/src/ggml-openvino/model-cache.h +56 -0
  824. data/vendor/ggml/src/ggml-openvino/openvino/decoder.h +133 -0
  825. data/vendor/ggml/src/ggml-openvino/openvino/frontend.cpp +27 -0
  826. data/vendor/ggml/src/ggml-openvino/openvino/frontend.h +23 -0
  827. data/vendor/ggml/src/ggml-openvino/openvino/input_model.cpp +17 -0
  828. data/vendor/ggml/src/ggml-openvino/openvino/input_model.h +29 -0
  829. data/vendor/ggml/src/ggml-openvino/openvino/node_context.h +171 -0
  830. data/vendor/ggml/src/ggml-openvino/openvino/op/add.cpp +45 -0
  831. data/vendor/ggml/src/ggml-openvino/openvino/op/add_id.cpp +76 -0
  832. data/vendor/ggml/src/ggml-openvino/openvino/op/argsort.cpp +47 -0
  833. data/vendor/ggml/src/ggml-openvino/openvino/op/clamp.cpp +33 -0
  834. data/vendor/ggml/src/ggml-openvino/openvino/op/concat.cpp +48 -0
  835. data/vendor/ggml/src/ggml-openvino/openvino/op/cont.cpp +40 -0
  836. data/vendor/ggml/src/ggml-openvino/openvino/op/cpy.cpp +295 -0
  837. data/vendor/ggml/src/ggml-openvino/openvino/op/cumsum.cpp +29 -0
  838. data/vendor/ggml/src/ggml-openvino/openvino/op/diag.cpp +58 -0
  839. data/vendor/ggml/src/ggml-openvino/openvino/op/div.cpp +146 -0
  840. data/vendor/ggml/src/ggml-openvino/openvino/op/fill.cpp +34 -0
  841. data/vendor/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp +241 -0
  842. data/vendor/ggml/src/ggml-openvino/openvino/op/gated_delta_net.cpp +327 -0
  843. data/vendor/ggml/src/ggml-openvino/openvino/op/gated_delta_net.hpp +65 -0
  844. data/vendor/ggml/src/ggml-openvino/openvino/op/gather_matmul.hpp +43 -0
  845. data/vendor/ggml/src/ggml-openvino/openvino/op/get_rows.cpp +142 -0
  846. data/vendor/ggml/src/ggml-openvino/openvino/op/glu_geglu.cpp +75 -0
  847. data/vendor/ggml/src/ggml-openvino/openvino/op/glu_geglu_quick.cpp +64 -0
  848. data/vendor/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp +110 -0
  849. data/vendor/ggml/src/ggml-openvino/openvino/op/im2col.cpp +120 -0
  850. data/vendor/ggml/src/ggml-openvino/openvino/op/l2_norm.cpp +61 -0
  851. data/vendor/ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp +283 -0
  852. data/vendor/ggml/src/ggml-openvino/openvino/op/mulmat.cpp +100 -0
  853. data/vendor/ggml/src/ggml-openvino/openvino/op/norm.cpp +58 -0
  854. data/vendor/ggml/src/ggml-openvino/openvino/op/pad.cpp +95 -0
  855. data/vendor/ggml/src/ggml-openvino/openvino/op/permute.cpp +147 -0
  856. data/vendor/ggml/src/ggml-openvino/openvino/op/pool_2d.cpp +53 -0
  857. data/vendor/ggml/src/ggml-openvino/openvino/op/repeat.cpp +48 -0
  858. data/vendor/ggml/src/ggml-openvino/openvino/op/reshape.cpp +113 -0
  859. data/vendor/ggml/src/ggml-openvino/openvino/op/rms_norm.cpp +81 -0
  860. data/vendor/ggml/src/ggml-openvino/openvino/op/roll.cpp +36 -0
  861. data/vendor/ggml/src/ggml-openvino/openvino/op/rope.cpp +303 -0
  862. data/vendor/ggml/src/ggml-openvino/openvino/op/scale.cpp +86 -0
  863. data/vendor/ggml/src/ggml-openvino/openvino/op/set.cpp +76 -0
  864. data/vendor/ggml/src/ggml-openvino/openvino/op/set_rows.cpp +95 -0
  865. data/vendor/ggml/src/ggml-openvino/openvino/op/softmax.cpp +166 -0
  866. data/vendor/ggml/src/ggml-openvino/openvino/op/solve_tri.cpp +108 -0
  867. data/vendor/ggml/src/ggml-openvino/openvino/op/sqr.cpp +35 -0
  868. data/vendor/ggml/src/ggml-openvino/openvino/op/ssm_conv.cpp +61 -0
  869. data/vendor/ggml/src/ggml-openvino/openvino/op/sum_rows.cpp +27 -0
  870. data/vendor/ggml/src/ggml-openvino/openvino/op/transpose.cpp +54 -0
  871. data/vendor/ggml/src/ggml-openvino/openvino/op/tri.cpp +82 -0
  872. data/vendor/ggml/src/ggml-openvino/openvino/op/unary_silu.cpp +27 -0
  873. data/vendor/ggml/src/ggml-openvino/openvino/op/unary_softplus.cpp +38 -0
  874. data/vendor/ggml/src/ggml-openvino/openvino/op/view.cpp +245 -0
  875. data/vendor/ggml/src/ggml-openvino/openvino/op_table.cpp +88 -0
  876. data/vendor/ggml/src/ggml-openvino/openvino/op_table.h +67 -0
  877. data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_conv.cpp +212 -0
  878. data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_conv.h +17 -0
  879. data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_sdpa.cpp +60 -0
  880. data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_sdpa.h +17 -0
  881. data/vendor/ggml/src/ggml-openvino/openvino/pass/mark_decompression_convert_constant_folding.h +29 -0
  882. data/vendor/ggml/src/ggml-openvino/openvino/pass/mark_dequantization_subgraph.h +44 -0
  883. data/vendor/ggml/src/ggml-openvino/openvino/pass/squeeze_matmul.cpp +58 -0
  884. data/vendor/ggml/src/ggml-openvino/openvino/pass/squeeze_matmul.h +17 -0
  885. data/vendor/ggml/src/ggml-openvino/openvino/rt_info/weightless_caching_attributes.hpp +41 -0
  886. data/vendor/ggml/src/ggml-openvino/openvino/translate_session.cpp +432 -0
  887. data/vendor/ggml/src/ggml-openvino/openvino/translate_session.h +29 -0
  888. data/vendor/ggml/src/ggml-openvino/openvino/utils.cpp +886 -0
  889. data/vendor/ggml/src/ggml-openvino/openvino/utils.h +88 -0
  890. data/vendor/ggml/src/ggml-openvino/utils.cpp +1496 -0
  891. data/vendor/ggml/src/ggml-openvino/utils.h +195 -0
  892. data/vendor/ggml/src/ggml-opt.cpp +1094 -0
  893. data/vendor/ggml/src/ggml-quants.c +5667 -0
  894. data/vendor/ggml/src/ggml-quants.h +115 -0
  895. data/vendor/ggml/src/ggml-rpc/CMakeLists.txt +49 -0
  896. data/vendor/ggml/src/ggml-rpc/ggml-rpc.cpp +2351 -0
  897. data/vendor/ggml/src/ggml-rpc/transport-apple.cpp +481 -0
  898. data/vendor/ggml/src/ggml-rpc/transport-apple.h +27 -0
  899. data/vendor/ggml/src/ggml-rpc/transport.cpp +739 -0
  900. data/vendor/ggml/src/ggml-rpc/transport.h +38 -0
  901. data/vendor/ggml/src/ggml-sycl/CMakeLists.txt +218 -0
  902. data/vendor/ggml/src/ggml-sycl/add-id.cpp +81 -0
  903. data/vendor/ggml/src/ggml-sycl/add-id.hpp +8 -0
  904. data/vendor/ggml/src/ggml-sycl/backend.hpp +52 -0
  905. data/vendor/ggml/src/ggml-sycl/base.hpp +36 -0
  906. data/vendor/ggml/src/ggml-sycl/binbcast.cpp +650 -0
  907. data/vendor/ggml/src/ggml-sycl/binbcast.hpp +69 -0
  908. data/vendor/ggml/src/ggml-sycl/col2im-1d.cpp +102 -0
  909. data/vendor/ggml/src/ggml-sycl/col2im-1d.hpp +8 -0
  910. data/vendor/ggml/src/ggml-sycl/common.cpp +153 -0
  911. data/vendor/ggml/src/ggml-sycl/common.hpp +1029 -0
  912. data/vendor/ggml/src/ggml-sycl/concat.cpp +509 -0
  913. data/vendor/ggml/src/ggml-sycl/concat.hpp +20 -0
  914. data/vendor/ggml/src/ggml-sycl/conv.cpp +101 -0
  915. data/vendor/ggml/src/ggml-sycl/conv.hpp +20 -0
  916. data/vendor/ggml/src/ggml-sycl/conv2d-dw.cpp +170 -0
  917. data/vendor/ggml/src/ggml-sycl/conv2d-dw.hpp +10 -0
  918. data/vendor/ggml/src/ggml-sycl/conv2d-transpose.cpp +125 -0
  919. data/vendor/ggml/src/ggml-sycl/conv2d-transpose.hpp +10 -0
  920. data/vendor/ggml/src/ggml-sycl/conv2d.cpp +150 -0
  921. data/vendor/ggml/src/ggml-sycl/conv2d.hpp +10 -0
  922. data/vendor/ggml/src/ggml-sycl/conv3d.cpp +224 -0
  923. data/vendor/ggml/src/ggml-sycl/conv3d.hpp +8 -0
  924. data/vendor/ggml/src/ggml-sycl/convert.cpp +877 -0
  925. data/vendor/ggml/src/ggml-sycl/convert.hpp +64 -0
  926. data/vendor/ggml/src/ggml-sycl/count-equal.cpp +79 -0
  927. data/vendor/ggml/src/ggml-sycl/count-equal.hpp +9 -0
  928. data/vendor/ggml/src/ggml-sycl/cpy.cpp +1433 -0
  929. data/vendor/ggml/src/ggml-sycl/cpy.hpp +537 -0
  930. data/vendor/ggml/src/ggml-sycl/cross_entropy_loss.cpp +255 -0
  931. data/vendor/ggml/src/ggml-sycl/cross_entropy_loss.hpp +7 -0
  932. data/vendor/ggml/src/ggml-sycl/cumsum.cpp +148 -0
  933. data/vendor/ggml/src/ggml-sycl/cumsum.hpp +5 -0
  934. data/vendor/ggml/src/ggml-sycl/dequantize.hpp +1681 -0
  935. data/vendor/ggml/src/ggml-sycl/diag.cpp +67 -0
  936. data/vendor/ggml/src/ggml-sycl/diag.hpp +5 -0
  937. data/vendor/ggml/src/ggml-sycl/dmmv.cpp +2227 -0
  938. data/vendor/ggml/src/ggml-sycl/dmmv.hpp +27 -0
  939. data/vendor/ggml/src/ggml-sycl/dpct/helper.hpp +3782 -0
  940. data/vendor/ggml/src/ggml-sycl/dsv4-hc.cpp +280 -0
  941. data/vendor/ggml/src/ggml-sycl/dsv4-hc.hpp +10 -0
  942. data/vendor/ggml/src/ggml-sycl/element_wise.cpp +1432 -0
  943. data/vendor/ggml/src/ggml-sycl/element_wise.hpp +135 -0
  944. data/vendor/ggml/src/ggml-sycl/esimd.hpp +589 -0
  945. data/vendor/ggml/src/ggml-sycl/fattn-buffers.cpp +56 -0
  946. data/vendor/ggml/src/ggml-sycl/fattn-buffers.hpp +63 -0
  947. data/vendor/ggml/src/ggml-sycl/fattn-common.hpp +1185 -0
  948. data/vendor/ggml/src/ggml-sycl/fattn-mkl.cpp +689 -0
  949. data/vendor/ggml/src/ggml-sycl/fattn-onednn.cpp +450 -0
  950. data/vendor/ggml/src/ggml-sycl/fattn-onednn.hpp +18 -0
  951. data/vendor/ggml/src/ggml-sycl/fattn-tile.cpp +59 -0
  952. data/vendor/ggml/src/ggml-sycl/fattn-tile.hpp +1246 -0
  953. data/vendor/ggml/src/ggml-sycl/fattn-vec.hpp +684 -0
  954. data/vendor/ggml/src/ggml-sycl/fattn.cpp +452 -0
  955. data/vendor/ggml/src/ggml-sycl/fattn.hpp +42 -0
  956. data/vendor/ggml/src/ggml-sycl/fill.cpp +55 -0
  957. data/vendor/ggml/src/ggml-sycl/fill.hpp +5 -0
  958. data/vendor/ggml/src/ggml-sycl/fusion.cpp +212 -0
  959. data/vendor/ggml/src/ggml-sycl/fusion.hpp +17 -0
  960. data/vendor/ggml/src/ggml-sycl/fwht.cpp +119 -0
  961. data/vendor/ggml/src/ggml-sycl/fwht.hpp +12 -0
  962. data/vendor/ggml/src/ggml-sycl/gated_delta_net.cpp +366 -0
  963. data/vendor/ggml/src/ggml-sycl/gated_delta_net.hpp +19 -0
  964. data/vendor/ggml/src/ggml-sycl/gemm.hpp +93 -0
  965. data/vendor/ggml/src/ggml-sycl/getrows.cpp +368 -0
  966. data/vendor/ggml/src/ggml-sycl/getrows.hpp +20 -0
  967. data/vendor/ggml/src/ggml-sycl/ggml-sycl.cpp +7028 -0
  968. data/vendor/ggml/src/ggml-sycl/gla.cpp +106 -0
  969. data/vendor/ggml/src/ggml-sycl/gla.hpp +8 -0
  970. data/vendor/ggml/src/ggml-sycl/im2col.cpp +400 -0
  971. data/vendor/ggml/src/ggml-sycl/im2col.hpp +23 -0
  972. data/vendor/ggml/src/ggml-sycl/lightning-indexer.cpp +197 -0
  973. data/vendor/ggml/src/ggml-sycl/lightning-indexer.hpp +8 -0
  974. data/vendor/ggml/src/ggml-sycl/mem.cpp +162 -0
  975. data/vendor/ggml/src/ggml-sycl/mem.hpp +16 -0
  976. data/vendor/ggml/src/ggml-sycl/mmq.cpp +3030 -0
  977. data/vendor/ggml/src/ggml-sycl/mmq.hpp +33 -0
  978. data/vendor/ggml/src/ggml-sycl/mmvq.cpp +3033 -0
  979. data/vendor/ggml/src/ggml-sycl/mmvq.hpp +76 -0
  980. data/vendor/ggml/src/ggml-sycl/norm.cpp +963 -0
  981. data/vendor/ggml/src/ggml-sycl/norm.hpp +32 -0
  982. data/vendor/ggml/src/ggml-sycl/opt-step.cpp +131 -0
  983. data/vendor/ggml/src/ggml-sycl/opt-step.hpp +6 -0
  984. data/vendor/ggml/src/ggml-sycl/outprod.cpp +83 -0
  985. data/vendor/ggml/src/ggml-sycl/outprod.hpp +10 -0
  986. data/vendor/ggml/src/ggml-sycl/pad.cpp +97 -0
  987. data/vendor/ggml/src/ggml-sycl/pad.hpp +24 -0
  988. data/vendor/ggml/src/ggml-sycl/pad_reflect_1d.cpp +100 -0
  989. data/vendor/ggml/src/ggml-sycl/pad_reflect_1d.hpp +10 -0
  990. data/vendor/ggml/src/ggml-sycl/pool.cpp +185 -0
  991. data/vendor/ggml/src/ggml-sycl/pool.hpp +22 -0
  992. data/vendor/ggml/src/ggml-sycl/presets.hpp +79 -0
  993. data/vendor/ggml/src/ggml-sycl/quantize.hpp +133 -0
  994. data/vendor/ggml/src/ggml-sycl/quants.hpp +204 -0
  995. data/vendor/ggml/src/ggml-sycl/repeat_back.cpp +76 -0
  996. data/vendor/ggml/src/ggml-sycl/repeat_back.hpp +8 -0
  997. data/vendor/ggml/src/ggml-sycl/roll.cpp +122 -0
  998. data/vendor/ggml/src/ggml-sycl/roll.hpp +20 -0
  999. data/vendor/ggml/src/ggml-sycl/rope.cpp +651 -0
  1000. data/vendor/ggml/src/ggml-sycl/rope.hpp +26 -0
  1001. data/vendor/ggml/src/ggml-sycl/set.cpp +73 -0
  1002. data/vendor/ggml/src/ggml-sycl/set.hpp +5 -0
  1003. data/vendor/ggml/src/ggml-sycl/set_rows.cpp +577 -0
  1004. data/vendor/ggml/src/ggml-sycl/set_rows.hpp +8 -0
  1005. data/vendor/ggml/src/ggml-sycl/softmax.cpp +425 -0
  1006. data/vendor/ggml/src/ggml-sycl/softmax.hpp +24 -0
  1007. data/vendor/ggml/src/ggml-sycl/solve_tri.cpp +172 -0
  1008. data/vendor/ggml/src/ggml-sycl/solve_tri.hpp +8 -0
  1009. data/vendor/ggml/src/ggml-sycl/ssm_conv.cpp +136 -0
  1010. data/vendor/ggml/src/ggml-sycl/ssm_conv.hpp +5 -0
  1011. data/vendor/ggml/src/ggml-sycl/ssm_scan.cpp +170 -0
  1012. data/vendor/ggml/src/ggml-sycl/ssm_scan.hpp +5 -0
  1013. data/vendor/ggml/src/ggml-sycl/sycl_hw.cpp +67 -0
  1014. data/vendor/ggml/src/ggml-sycl/sycl_hw.hpp +38 -0
  1015. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq112-dv112.cpp +5 -0
  1016. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq128-dv128.cpp +5 -0
  1017. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq256-dv256.cpp +5 -0
  1018. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq40-dv40.cpp +5 -0
  1019. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq512-dv512.cpp +6 -0
  1020. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq576-dv512.cpp +5 -0
  1021. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq64-dv64.cpp +5 -0
  1022. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq72-dv72.cpp +5 -0
  1023. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq80-dv80.cpp +5 -0
  1024. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq96-dv96.cpp +5 -0
  1025. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-f16.cpp +8 -0
  1026. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q4_0.cpp +8 -0
  1027. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q4_1.cpp +8 -0
  1028. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q5_0.cpp +8 -0
  1029. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q5_1.cpp +8 -0
  1030. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q8_0.cpp +8 -0
  1031. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-f16.cpp +8 -0
  1032. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q4_0.cpp +8 -0
  1033. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q4_1.cpp +8 -0
  1034. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q5_0.cpp +8 -0
  1035. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q5_1.cpp +8 -0
  1036. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q8_0.cpp +8 -0
  1037. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-f16.cpp +8 -0
  1038. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q4_0.cpp +8 -0
  1039. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q4_1.cpp +8 -0
  1040. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q5_0.cpp +8 -0
  1041. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q5_1.cpp +8 -0
  1042. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q8_0.cpp +8 -0
  1043. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-f16.cpp +8 -0
  1044. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q4_0.cpp +8 -0
  1045. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q4_1.cpp +8 -0
  1046. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q5_0.cpp +8 -0
  1047. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q5_1.cpp +8 -0
  1048. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q8_0.cpp +8 -0
  1049. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-f16.cpp +8 -0
  1050. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q4_0.cpp +8 -0
  1051. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q4_1.cpp +8 -0
  1052. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q5_0.cpp +8 -0
  1053. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q5_1.cpp +8 -0
  1054. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q8_0.cpp +8 -0
  1055. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-f16.cpp +8 -0
  1056. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q4_0.cpp +8 -0
  1057. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q4_1.cpp +8 -0
  1058. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q5_0.cpp +8 -0
  1059. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q5_1.cpp +8 -0
  1060. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q8_0.cpp +8 -0
  1061. data/vendor/ggml/src/ggml-sycl/topk-moe.cpp +620 -0
  1062. data/vendor/ggml/src/ggml-sycl/topk-moe.hpp +12 -0
  1063. data/vendor/ggml/src/ggml-sycl/tsembd.cpp +73 -0
  1064. data/vendor/ggml/src/ggml-sycl/tsembd.hpp +20 -0
  1065. data/vendor/ggml/src/ggml-sycl/type.hpp +112 -0
  1066. data/vendor/ggml/src/ggml-sycl/upscale.cpp +410 -0
  1067. data/vendor/ggml/src/ggml-sycl/upscale.hpp +9 -0
  1068. data/vendor/ggml/src/ggml-sycl/vecdotq.hpp +1726 -0
  1069. data/vendor/ggml/src/ggml-sycl/wkv.cpp +293 -0
  1070. data/vendor/ggml/src/ggml-sycl/wkv.hpp +10 -0
  1071. data/vendor/ggml/src/ggml-threading.cpp +12 -0
  1072. data/vendor/ggml/src/ggml-threading.h +14 -0
  1073. data/vendor/ggml/src/ggml-version.h.in +4 -0
  1074. data/vendor/ggml/src/ggml-virtgpu/CMakeLists.txt +70 -0
  1075. data/vendor/ggml/src/ggml-virtgpu/apir_cs_ggml-rpc-front.cpp +87 -0
  1076. data/vendor/ggml/src/ggml-virtgpu/backend/CMakeLists.txt +21 -0
  1077. data/vendor/ggml/src/ggml-virtgpu/backend/apir_cs_ggml-rpc-back.cpp +115 -0
  1078. data/vendor/ggml/src/ggml-virtgpu/backend/backend-convert.h +13 -0
  1079. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-backend.cpp +102 -0
  1080. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-buffer-type.cpp +105 -0
  1081. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-buffer.cpp +179 -0
  1082. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-device.cpp +149 -0
  1083. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched.cpp +51 -0
  1084. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched.gen.h +73 -0
  1085. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched.h +27 -0
  1086. data/vendor/ggml/src/ggml-virtgpu/backend/backend-virgl-apir.h +32 -0
  1087. data/vendor/ggml/src/ggml-virtgpu/backend/backend.cpp +144 -0
  1088. data/vendor/ggml/src/ggml-virtgpu/backend/shared/api_remoting.h +95 -0
  1089. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_backend.gen.h +94 -0
  1090. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_backend.h +50 -0
  1091. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_cs.h +378 -0
  1092. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_cs_ggml.h +232 -0
  1093. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_cs_rpc.h +58 -0
  1094. data/vendor/ggml/src/ggml-virtgpu/ggml-backend-buffer-type.cpp +81 -0
  1095. data/vendor/ggml/src/ggml-virtgpu/ggml-backend-buffer.cpp +123 -0
  1096. data/vendor/ggml/src/ggml-virtgpu/ggml-backend-device.cpp +160 -0
  1097. data/vendor/ggml/src/ggml-virtgpu/ggml-backend-reg.cpp +213 -0
  1098. data/vendor/ggml/src/ggml-virtgpu/ggml-backend.cpp +72 -0
  1099. data/vendor/ggml/src/ggml-virtgpu/ggml-remoting.h +71 -0
  1100. data/vendor/ggml/src/ggml-virtgpu/ggmlremoting_functions.yaml +166 -0
  1101. data/vendor/ggml/src/ggml-virtgpu/include/apir_hw.h +9 -0
  1102. data/vendor/ggml/src/ggml-virtgpu/regenerate_remoting.py +333 -0
  1103. data/vendor/ggml/src/ggml-virtgpu/virtgpu-apir.h +15 -0
  1104. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-backend.cpp +58 -0
  1105. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-buffer-type.cpp +110 -0
  1106. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-buffer.cpp +173 -0
  1107. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-device.cpp +194 -0
  1108. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-impl.h +36 -0
  1109. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward.gen.h +54 -0
  1110. data/vendor/ggml/src/ggml-virtgpu/virtgpu-shm.cpp +99 -0
  1111. data/vendor/ggml/src/ggml-virtgpu/virtgpu-shm.h +23 -0
  1112. data/vendor/ggml/src/ggml-virtgpu/virtgpu-utils.cpp +179 -0
  1113. data/vendor/ggml/src/ggml-virtgpu/virtgpu-utils.h +86 -0
  1114. data/vendor/ggml/src/ggml-virtgpu/virtgpu.cpp +545 -0
  1115. data/vendor/ggml/src/ggml-virtgpu/virtgpu.h +115 -0
  1116. data/vendor/ggml/src/ggml-vulkan/CMakeLists.txt +254 -0
  1117. data/vendor/ggml/src/ggml-vulkan/cmake/host-toolchain.cmake.in +15 -0
  1118. data/vendor/ggml/src/ggml-vulkan/ggml-vulkan.cpp +20349 -0
  1119. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/CMakeLists.txt +43 -0
  1120. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/acc.comp +37 -0
  1121. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/add.comp +69 -0
  1122. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/add1.comp +28 -0
  1123. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/add_id.comp +42 -0
  1124. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/arange.comp +20 -0
  1125. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/argmax.comp +60 -0
  1126. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/argsort.comp +86 -0
  1127. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/argsort_large.comp +114 -0
  1128. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/col2im_1d.comp +61 -0
  1129. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/concat.comp +41 -0
  1130. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/contig_copy.comp +53 -0
  1131. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv2d_dw.comp +105 -0
  1132. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv2d_mm.comp +473 -0
  1133. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv3d_mm.comp +424 -0
  1134. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv_transpose_1d.comp +98 -0
  1135. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy.comp +25 -0
  1136. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_from_quant.comp +51 -0
  1137. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_to_quant.comp +350 -0
  1138. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_transpose.comp +67 -0
  1139. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_transpose_02.comp +61 -0
  1140. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/count_equal.comp +31 -0
  1141. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/count_experts.comp +135 -0
  1142. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cross_entropy_loss.comp +78 -0
  1143. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cross_entropy_loss_back.comp +75 -0
  1144. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cumsum.comp +83 -0
  1145. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cumsum_multipass1.comp +60 -0
  1146. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cumsum_multipass2.comp +66 -0
  1147. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_f32.comp +20 -0
  1148. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.glsl +741 -0
  1149. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl +1466 -0
  1150. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_head.glsl +13 -0
  1151. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_m.comp +42 -0
  1152. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_s.comp +35 -0
  1153. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_s.comp +44 -0
  1154. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xs.comp +43 -0
  1155. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xxs.comp +49 -0
  1156. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_s.comp +40 -0
  1157. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_xxs.comp +51 -0
  1158. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_nl.comp +32 -0
  1159. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_xs.comp +34 -0
  1160. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_mxfp4.comp +32 -0
  1161. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_nvfp4.comp +32 -0
  1162. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q1_0.comp +29 -0
  1163. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q2_0.comp +29 -0
  1164. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q2_k.comp +34 -0
  1165. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q3_k.comp +42 -0
  1166. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_0.comp +30 -0
  1167. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_1.comp +32 -0
  1168. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_k.comp +68 -0
  1169. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_0.comp +34 -0
  1170. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_1.comp +35 -0
  1171. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_k.comp +70 -0
  1172. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q6_k.comp +33 -0
  1173. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q8_0.comp +42 -0
  1174. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_tq2_0.comp +31 -0
  1175. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/diag.comp +28 -0
  1176. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/diag_mask_inf.comp +34 -0
  1177. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/div.comp +27 -0
  1178. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dot_product_funcs.glsl +27 -0
  1179. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/fa_types.glsl +55 -0
  1180. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/bfloat16.comp +7 -0
  1181. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat.comp +7 -0
  1182. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat2.comp +7 -0
  1183. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat2_decode_vector.comp +7 -0
  1184. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/float_e2m1.comp +7 -0
  1185. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/float_e4m3.comp +7 -0
  1186. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/integer_dot.comp +7 -0
  1187. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/fill.comp +19 -0
  1188. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp +760 -0
  1189. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.glsl +223 -0
  1190. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp +647 -0
  1191. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp +487 -0
  1192. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_dequant.glsl +146 -0
  1193. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_mask_opt.comp +162 -0
  1194. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_mmq_funcs.glsl +203 -0
  1195. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_split_k_reduce.comp +121 -0
  1196. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/fwht.comp +115 -0
  1197. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/gated_delta_net.comp +189 -0
  1198. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/geglu.comp +13 -0
  1199. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/geglu_erf.comp +27 -0
  1200. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/geglu_quick.comp +11 -0
  1201. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/generic_binary_head.glsl +65 -0
  1202. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/generic_head.glsl +11 -0
  1203. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/generic_unary_head.glsl +75 -0
  1204. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/get_rows.comp +42 -0
  1205. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_back.comp +25 -0
  1206. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_quant.comp +51 -0
  1207. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/gla.comp +82 -0
  1208. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/glu_head.glsl +43 -0
  1209. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/glu_main.glsl +35 -0
  1210. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/group_norm.comp +66 -0
  1211. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/im2col.comp +138 -0
  1212. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/im2col_3d.comp +124 -0
  1213. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/l2_norm.comp +41 -0
  1214. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/lightning_indexer.comp +151 -0
  1215. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/log.comp +17 -0
  1216. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul.comp +27 -0
  1217. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_split_k_reduce.comp +48 -0
  1218. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec.comp +264 -0
  1219. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_base.glsl +230 -0
  1220. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iface.glsl +35 -0
  1221. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_m.comp +132 -0
  1222. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_s.comp +95 -0
  1223. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_s.comp +90 -0
  1224. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xs.comp +105 -0
  1225. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xxs.comp +87 -0
  1226. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_s.comp +93 -0
  1227. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_xxs.comp +88 -0
  1228. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_nc.comp +124 -0
  1229. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_p021.comp +156 -0
  1230. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q2_k.comp +128 -0
  1231. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q3_k.comp +132 -0
  1232. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q4_k.comp +134 -0
  1233. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q5_k.comp +165 -0
  1234. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q6_k.comp +130 -0
  1235. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_tq2_0.comp +102 -0
  1236. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq.comp +141 -0
  1237. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq_funcs.glsl +564 -0
  1238. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp +475 -0
  1239. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp +690 -0
  1240. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl +670 -0
  1241. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_id_funcs.glsl +89 -0
  1242. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp +317 -0
  1243. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_funcs.glsl +488 -0
  1244. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_shmem_types.glsl +99 -0
  1245. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/multi_add.comp +194 -0
  1246. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/norm.comp +44 -0
  1247. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_adamw.comp +42 -0
  1248. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_sgd.comp +22 -0
  1249. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/out_prod.comp +59 -0
  1250. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pad.comp +64 -0
  1251. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pad_reflect_1d.comp +43 -0
  1252. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pool1d.comp +65 -0
  1253. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pool2d.comp +74 -0
  1254. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/quantize_q8_1.comp +127 -0
  1255. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/reglu.comp +9 -0
  1256. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/repeat.comp +26 -0
  1257. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/repeat_back.comp +37 -0
  1258. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp +150 -0
  1259. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_back.comp +55 -0
  1260. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_partials.comp +65 -0
  1261. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/roll.comp +46 -0
  1262. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_funcs.glsl +218 -0
  1263. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_head.glsl +19 -0
  1264. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_multi.comp +17 -0
  1265. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_neox.comp +17 -0
  1266. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_norm.comp +17 -0
  1267. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_params.glsl +35 -0
  1268. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_vision.comp +17 -0
  1269. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/scale.comp +24 -0
  1270. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/silu_back.comp +26 -0
  1271. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/snake.comp +49 -0
  1272. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max.comp +195 -0
  1273. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_back.comp +54 -0
  1274. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large1.comp +62 -0
  1275. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large2.comp +79 -0
  1276. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large3.comp +65 -0
  1277. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large_common.glsl +53 -0
  1278. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/solve_tri.comp +81 -0
  1279. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/ssm_conv.comp +60 -0
  1280. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/ssm_scan.comp +134 -0
  1281. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/sub.comp +29 -0
  1282. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.comp +47 -0
  1283. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.glsl +19 -0
  1284. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/swiglu.comp +9 -0
  1285. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/swiglu_clamp.comp +12 -0
  1286. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/swiglu_oai.comp +14 -0
  1287. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/timestep_embedding.comp +42 -0
  1288. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_argsort.comp +118 -0
  1289. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_moe.comp +221 -0
  1290. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_nary_search.comp +246 -0
  1291. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_radix_select.comp +144 -0
  1292. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/tri.comp +42 -0
  1293. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl +1938 -0
  1294. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/unary.comp +168 -0
  1295. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/upscale.comp +178 -0
  1296. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/utils.glsl +37 -0
  1297. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +1367 -0
  1298. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/wkv6.comp +87 -0
  1299. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/wkv7.comp +91 -0
  1300. data/vendor/ggml/src/ggml-webgpu/CMakeLists.txt +83 -0
  1301. data/vendor/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp +3457 -0
  1302. data/vendor/ggml/src/ggml-webgpu/ggml-webgpu.cpp +4790 -0
  1303. data/vendor/ggml/src/ggml-webgpu/pre_wgsl.hpp +808 -0
  1304. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/add_id.wgsl +64 -0
  1305. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/argmax.wgsl +72 -0
  1306. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/argsort.wgsl +102 -0
  1307. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/argsort_merge.wgsl +134 -0
  1308. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/binary.wgsl +142 -0
  1309. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/common_decls.tmpl +967 -0
  1310. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/concat.wgsl +92 -0
  1311. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/conv2d.wgsl +127 -0
  1312. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/conv2d_dw.wgsl +100 -0
  1313. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/cpy.wgsl +84 -0
  1314. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/cumsum.wgsl +66 -0
  1315. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/embed_wgsl.py +89 -0
  1316. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn.wgsl +408 -0
  1317. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_decls.tmpl +134 -0
  1318. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_staging.tmpl +136 -0
  1319. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_tile.wgsl +240 -0
  1320. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_blk.wgsl +101 -0
  1321. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_reduce.wgsl +84 -0
  1322. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_split.wgsl +503 -0
  1323. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/gated_delta_net.wgsl +149 -0
  1324. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/get_rows.wgsl +794 -0
  1325. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/glu.wgsl +178 -0
  1326. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/im2col.wgsl +77 -0
  1327. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/memset.wgsl +40 -0
  1328. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_decls.tmpl +1052 -0
  1329. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id.wgsl +195 -0
  1330. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id_gather.wgsl +52 -0
  1331. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id_vec.wgsl +154 -0
  1332. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_reg_tile.wgsl +159 -0
  1333. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_subgroup_matrix.wgsl +210 -0
  1334. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec.wgsl +167 -0
  1335. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec_acc.tmpl +1560 -0
  1336. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec_q_acc.tmpl +297 -0
  1337. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/pad.wgsl +86 -0
  1338. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/quant_inner_loops.tmpl +21 -0
  1339. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/quantize_q8.wgsl +178 -0
  1340. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/repeat.wgsl +70 -0
  1341. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/rms_norm_mul.wgsl +151 -0
  1342. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/rope.wgsl +228 -0
  1343. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/row_norm.wgsl +152 -0
  1344. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/scale.wgsl +65 -0
  1345. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/set.wgsl +109 -0
  1346. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/set_rows.wgsl +108 -0
  1347. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/set_rows_quant.wgsl +224 -0
  1348. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/soft_max.wgsl +210 -0
  1349. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/solve_tri.wgsl +120 -0
  1350. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/ssm_conv.wgsl +65 -0
  1351. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/ssm_scan.wgsl +247 -0
  1352. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/sum_rows.wgsl +55 -0
  1353. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/unary.wgsl +213 -0
  1354. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/upscale.wgsl +240 -0
  1355. data/vendor/ggml/src/ggml-zdnn/CMakeLists.txt +36 -0
  1356. data/vendor/ggml/src/ggml-zdnn/common.hpp +59 -0
  1357. data/vendor/ggml/src/ggml-zdnn/ggml-zdnn.cpp +638 -0
  1358. data/vendor/ggml/src/ggml-zdnn/mmf.cpp +80 -0
  1359. data/vendor/ggml/src/ggml-zdnn/mmf.hpp +12 -0
  1360. data/vendor/ggml/src/ggml-zdnn/utils.cpp +79 -0
  1361. data/vendor/ggml/src/ggml-zdnn/utils.hpp +19 -0
  1362. data/vendor/ggml/src/ggml-zendnn/CMakeLists.txt +91 -0
  1363. data/vendor/ggml/src/ggml-zendnn/ggml-zendnn.cpp +838 -0
  1364. data/vendor/ggml/src/ggml.c +8088 -0
  1365. data/vendor/ggml/src/ggml.cpp +26 -0
  1366. data/vendor/ggml/src/gguf.cpp +1706 -0
  1367. metadata +1459 -0
@@ -0,0 +1,2517 @@
1
+ // Note: porting this file to C++ is a work in progress
2
+
3
+ #ifdef _WIN32
4
+ #define WIN32_LEAN_AND_MEAN
5
+ #ifndef NOMINMAX
6
+ # define NOMINMAX
7
+ #endif
8
+ #include <windows.h>
9
+ #endif
10
+
11
+ #include "ggml-backend.h"
12
+ #include "ggml-backend-impl.h"
13
+ #include "ggml-alloc.h"
14
+ #include "ggml-impl.h"
15
+
16
+ #include <assert.h>
17
+ #include <limits.h>
18
+ #include <stdarg.h>
19
+ #include <stdio.h>
20
+ #include <stdlib.h>
21
+ #include <string.h>
22
+ #include <algorithm>
23
+ #include <unordered_map>
24
+ #include <vector>
25
+
26
+ #ifdef __APPLE__
27
+ #include <sys/types.h>
28
+ #include <sys/sysctl.h>
29
+ #endif
30
+
31
+
32
+ // backend buffer type
33
+
34
+ const char * ggml_backend_buft_name(ggml_backend_buffer_type_t buft) {
35
+ GGML_ASSERT(buft);
36
+ return buft->iface.get_name(buft);
37
+ }
38
+
39
+ ggml_backend_buffer_t ggml_backend_buft_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {
40
+ GGML_ASSERT(buft);
41
+ if (size == 0) {
42
+ // return a dummy buffer for zero-sized allocations
43
+ return ggml_backend_buffer_init(buft, {}, NULL, 0);
44
+ }
45
+ return buft->iface.alloc_buffer(buft, size);
46
+ }
47
+
48
+ size_t ggml_backend_buft_get_alignment(ggml_backend_buffer_type_t buft) {
49
+ GGML_ASSERT(buft);
50
+ return buft->iface.get_alignment(buft);
51
+ }
52
+
53
+ size_t ggml_backend_buft_get_max_size(ggml_backend_buffer_type_t buft) {
54
+ GGML_ASSERT(buft);
55
+ // get_max_size is optional, defaults to SIZE_MAX
56
+ if (buft->iface.get_max_size) {
57
+ return buft->iface.get_max_size(buft);
58
+ }
59
+ return SIZE_MAX;
60
+ }
61
+
62
+ size_t ggml_backend_buft_get_alloc_size(ggml_backend_buffer_type_t buft, const struct ggml_tensor * tensor) {
63
+ GGML_ASSERT(buft);
64
+ // get_alloc_size is optional, defaults to ggml_nbytes
65
+ if (buft->iface.get_alloc_size) {
66
+ size_t size = buft->iface.get_alloc_size(buft, tensor);
67
+ assert(size >= ggml_nbytes(tensor));
68
+
69
+ // [TAG_ALLOC_SIZE_EXPAND]
70
+ // if you hit this assert, update ggml_backend_op_alloc_size_may_expand() accordingly
71
+ GGML_ASSERT(size <= ggml_nbytes(tensor) ||
72
+ ggml_op_is_empty(tensor->op) ||
73
+ ggml_is_quantized(tensor->type) || // [TAG_ALLOC_SIZE_EXPAND]
74
+ ggml_op_alloc_size_may_expand(tensor->op));
75
+
76
+ return size;
77
+ }
78
+ return ggml_nbytes(tensor);
79
+ }
80
+
81
+ bool ggml_backend_buft_is_host(ggml_backend_buffer_type_t buft) {
82
+ GGML_ASSERT(buft);
83
+ if (buft->iface.is_host) {
84
+ return buft->iface.is_host(buft);
85
+ }
86
+ return false;
87
+ }
88
+
89
+ ggml_backend_dev_t ggml_backend_buft_get_device(ggml_backend_buffer_type_t buft) {
90
+ GGML_ASSERT(buft);
91
+ return buft->device;
92
+ }
93
+
94
+ // backend buffer
95
+
96
+ ggml_backend_buffer_t ggml_backend_buffer_init(
97
+ ggml_backend_buffer_type_t buft,
98
+ struct ggml_backend_buffer_i iface,
99
+ void * context,
100
+ size_t size) {
101
+ ggml_backend_buffer_t buffer = new ggml_backend_buffer {
102
+ /* .interface = */ iface,
103
+ /* .buft = */ buft,
104
+ /* .context = */ context,
105
+ /* .size = */ size,
106
+ /* .usage = */ GGML_BACKEND_BUFFER_USAGE_ANY
107
+ };
108
+
109
+ return buffer;
110
+ }
111
+
112
+ const char * ggml_backend_buffer_name(ggml_backend_buffer_t buffer) {
113
+ return ggml_backend_buft_name(ggml_backend_buffer_get_type(buffer));
114
+ }
115
+
116
+ void ggml_backend_buffer_free(ggml_backend_buffer_t buffer) {
117
+ if (buffer == NULL) {
118
+ return;
119
+ }
120
+
121
+ if (buffer->iface.free_buffer != NULL) {
122
+ buffer->iface.free_buffer(buffer);
123
+ }
124
+ delete buffer;
125
+ }
126
+
127
+ size_t ggml_backend_buffer_get_size(ggml_backend_buffer_t buffer) {
128
+ GGML_ASSERT(buffer);
129
+ return buffer->size;
130
+ }
131
+
132
+ void * ggml_backend_buffer_get_base(ggml_backend_buffer_t buffer) {
133
+ GGML_ASSERT(buffer);
134
+ // get_base is optional if the buffer is zero-sized
135
+ if (!ggml_backend_buffer_is_meta(buffer) && buffer->size == 0) {
136
+ return NULL;
137
+ }
138
+
139
+ // FIXME JG: a multi_buffer has a non-zero size, according to the above comment get_base is not optional,
140
+ // I don't know whether the above comment is correct
141
+ if (!buffer->iface.get_base) {
142
+ return NULL;
143
+ }
144
+
145
+ void * base = buffer->iface.get_base(buffer);
146
+
147
+ GGML_ASSERT(base != NULL && "backend buffer base cannot be NULL");
148
+
149
+ return base;
150
+ }
151
+
152
+ enum ggml_status ggml_backend_buffer_init_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor) {
153
+ GGML_ASSERT(buffer);
154
+ // init_tensor is optional
155
+ if (buffer->iface.init_tensor) {
156
+ return buffer->iface.init_tensor(buffer, tensor);
157
+ }
158
+ return GGML_STATUS_SUCCESS;
159
+ }
160
+
161
+ void ggml_backend_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {
162
+ GGML_ASSERT(buffer);
163
+ // clear is optional if the buffer is zero-sized
164
+ if (buffer->size == 0) {
165
+ return;
166
+ }
167
+
168
+ buffer->iface.clear(buffer, value);
169
+ }
170
+
171
+ size_t ggml_backend_buffer_get_alignment(ggml_backend_buffer_t buffer) {
172
+ return ggml_backend_buft_get_alignment(ggml_backend_buffer_get_type(buffer));
173
+ }
174
+
175
+ size_t ggml_backend_buffer_get_max_size(ggml_backend_buffer_t buffer) {
176
+ return ggml_backend_buft_get_max_size(ggml_backend_buffer_get_type(buffer));
177
+ }
178
+
179
+ size_t ggml_backend_buffer_get_alloc_size(ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor) {
180
+ return ggml_backend_buft_get_alloc_size(ggml_backend_buffer_get_type(buffer), tensor);
181
+ }
182
+
183
+ bool ggml_backend_buffer_is_host(ggml_backend_buffer_t buffer) {
184
+ return ggml_backend_buft_is_host(ggml_backend_buffer_get_type(buffer));
185
+ }
186
+
187
+ void ggml_backend_buffer_set_usage(ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage) {
188
+ GGML_ASSERT(buffer);
189
+ buffer->usage = usage;
190
+
191
+ // FIXME: add a generic callback to the buffer interface
192
+ if (ggml_backend_buffer_is_multi_buffer(buffer)) {
193
+ ggml_backend_multi_buffer_set_usage(buffer, usage);
194
+ } else if (ggml_backend_buffer_is_meta(buffer)) {
195
+ ggml_backend_meta_buffer_set_usage(buffer, usage);
196
+ }
197
+ }
198
+
199
+ enum ggml_backend_buffer_usage ggml_backend_buffer_get_usage(ggml_backend_buffer_t buffer) {
200
+ GGML_ASSERT(buffer);
201
+ return buffer->usage;
202
+ }
203
+
204
+ ggml_backend_buffer_type_t ggml_backend_buffer_get_type(ggml_backend_buffer_t buffer) {
205
+ GGML_ASSERT(buffer);
206
+ return buffer->buft;
207
+ }
208
+
209
+ void ggml_backend_buffer_reset(ggml_backend_buffer_t buffer) {
210
+ GGML_ASSERT(buffer);
211
+ if (buffer->iface.reset) {
212
+ buffer->iface.reset(buffer);
213
+ }
214
+ }
215
+
216
+ bool ggml_backend_buffer_copy_tensor(const struct ggml_tensor * src, struct ggml_tensor * dst) {
217
+ ggml_backend_buffer_t dst_buf = dst->view_src ? dst->view_src->buffer : dst->buffer;
218
+ if (dst_buf->iface.cpy_tensor) {
219
+ return dst_buf->iface.cpy_tensor(dst_buf, src, dst);
220
+ }
221
+ return false;
222
+ }
223
+
224
+ // backend
225
+
226
+ ggml_guid_t ggml_backend_guid(ggml_backend_t backend) {
227
+ if (backend == NULL) {
228
+ return NULL;
229
+ }
230
+ return backend->guid;
231
+ }
232
+
233
+ const char * ggml_backend_name(ggml_backend_t backend) {
234
+ if (backend == NULL) {
235
+ return "NULL";
236
+ }
237
+ return backend->iface.get_name(backend);
238
+ }
239
+
240
+ void ggml_backend_free(ggml_backend_t backend) {
241
+ if (backend == NULL) {
242
+ return;
243
+ }
244
+
245
+ backend->iface.free(backend);
246
+ }
247
+
248
+ ggml_backend_buffer_type_t ggml_backend_get_default_buffer_type(ggml_backend_t backend) {
249
+ GGML_ASSERT(backend);
250
+ return ggml_backend_dev_buffer_type(backend->device);
251
+ }
252
+
253
+ ggml_backend_buffer_t ggml_backend_alloc_buffer(ggml_backend_t backend, size_t size) {
254
+ return ggml_backend_buft_alloc_buffer(ggml_backend_get_default_buffer_type(backend), size);
255
+ }
256
+
257
+ size_t ggml_backend_get_alignment(ggml_backend_t backend) {
258
+ return ggml_backend_buft_get_alignment(ggml_backend_get_default_buffer_type(backend));
259
+ }
260
+
261
+ size_t ggml_backend_get_max_size(ggml_backend_t backend) {
262
+ return ggml_backend_buft_get_max_size(ggml_backend_get_default_buffer_type(backend));
263
+ }
264
+
265
+ void ggml_backend_tensor_set_async(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size) {
266
+ GGML_ASSERT(backend);
267
+ GGML_ASSERT(tensor);
268
+ GGML_ASSERT(tensor->data != NULL && "tensor not allocated");
269
+ GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && "tensor write out of bounds");
270
+
271
+ if (backend->iface.set_tensor_async == NULL) {
272
+ ggml_backend_synchronize(backend);
273
+ ggml_backend_tensor_set(tensor, data, offset, size);
274
+ } else {
275
+ backend->iface.set_tensor_async(backend, tensor, data, offset, size);
276
+ }
277
+ }
278
+
279
+ void ggml_backend_tensor_get_async(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size) {
280
+ GGML_ASSERT(backend);
281
+ GGML_ASSERT(tensor);
282
+ GGML_ASSERT(tensor->data != NULL && "tensor not allocated");
283
+ GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && "tensor read out of bounds");
284
+
285
+ if (backend->iface.get_tensor_async == NULL) {
286
+ ggml_backend_synchronize(backend);
287
+ ggml_backend_tensor_get(tensor, data, offset, size);
288
+ } else {
289
+ backend->iface.get_tensor_async(backend, tensor, data, offset, size);
290
+ }
291
+ }
292
+
293
+ void ggml_backend_tensor_set_2d_async(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size,
294
+ size_t n_copies, size_t stride_tensor, size_t stride_data) {
295
+ GGML_ASSERT(backend);
296
+ GGML_ASSERT(tensor);
297
+ GGML_ASSERT(tensor->data != NULL && "tensor not allocated");
298
+
299
+ if (n_copies <= 1 || backend->iface.set_tensor_2d_async == NULL) {
300
+ for (size_t i = 0; i < n_copies; i++) {
301
+ ggml_backend_tensor_set_async(backend, tensor, (const char *) data + i*stride_data, offset + i*stride_tensor, size);
302
+ }
303
+ return;
304
+ }
305
+ if (size == 0) {
306
+ return;
307
+ }
308
+
309
+ GGML_ASSERT(tensor->data != NULL && "tensor not allocated");
310
+ GGML_ASSERT(offset + (n_copies-1)*stride_tensor + size <= ggml_nbytes(tensor) && "tensor write out of bounds");
311
+ backend->iface.set_tensor_2d_async(backend, tensor, data, offset, size, n_copies, stride_tensor, stride_data);
312
+ }
313
+
314
+ void ggml_backend_tensor_get_2d_async(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size,
315
+ size_t n_copies, size_t stride_tensor, size_t stride_data) {
316
+ GGML_ASSERT(backend);
317
+ GGML_ASSERT(tensor);
318
+ GGML_ASSERT(tensor->data != NULL && "tensor not allocated");
319
+
320
+ if (n_copies <= 1 || backend->iface.get_tensor_2d_async == NULL) {
321
+ for (size_t i = 0; i < n_copies; i++) {
322
+ ggml_backend_tensor_get_async(backend, tensor, (char *) data + i*stride_data, offset + i*stride_tensor, size);
323
+ }
324
+ return;
325
+ }
326
+ if (size == 0) {
327
+ return;
328
+ }
329
+
330
+ GGML_ASSERT(tensor->data != NULL && "tensor not allocated");
331
+ GGML_ASSERT(offset + (n_copies-1)*stride_tensor + size <= ggml_nbytes(tensor) && "tensor read out of bounds");
332
+ backend->iface.get_tensor_2d_async(backend, tensor, data, offset, size, n_copies, stride_tensor, stride_data);
333
+ }
334
+
335
+ void ggml_backend_tensor_set(struct ggml_tensor * tensor, const void * data, size_t offset, size_t size) {
336
+ GGML_ASSERT(tensor);
337
+ ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;
338
+ GGML_ASSERT(buf != NULL && "tensor buffer not set");
339
+
340
+ if (size == 0) {
341
+ return;
342
+ }
343
+
344
+ GGML_ASSERT(tensor->data != NULL && "tensor not allocated");
345
+ GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && "tensor write out of bounds");
346
+
347
+ buf->iface.set_tensor(buf, tensor, data, offset, size);
348
+ }
349
+
350
+ void ggml_backend_tensor_get(const struct ggml_tensor * tensor, void * data, size_t offset, size_t size) {
351
+ GGML_ASSERT(tensor);
352
+ ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;
353
+ GGML_ASSERT(buf != NULL && "tensor buffer not set");
354
+
355
+ if (size == 0) {
356
+ return;
357
+ }
358
+
359
+ GGML_ASSERT(tensor->data != NULL && "tensor not allocated");
360
+ GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && "tensor read out of bounds");
361
+
362
+ buf->iface.get_tensor(buf, tensor, data, offset, size);
363
+ }
364
+
365
+ void ggml_backend_tensor_set_2d(struct ggml_tensor * tensor, const void * data, size_t offset, size_t size,
366
+ size_t n_copies, size_t stride_tensor, size_t stride_data) {
367
+ GGML_ASSERT(tensor);
368
+ ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;
369
+ GGML_ASSERT(buf != NULL && "tensor buffer not set");
370
+
371
+ if (n_copies <= 1 || buf->iface.set_tensor_2d == NULL) {
372
+ for (size_t i = 0; i < n_copies; i++) {
373
+ ggml_backend_tensor_set(tensor, (const char *) data + i*stride_data, offset + i*stride_tensor, size);
374
+ }
375
+ return;
376
+ }
377
+ if (size == 0) {
378
+ return;
379
+ }
380
+
381
+ GGML_ASSERT(tensor->data != NULL && "tensor not allocated");
382
+ GGML_ASSERT(offset + (n_copies-1)*stride_tensor + size <= ggml_nbytes(tensor) && "tensor write out of bounds");
383
+
384
+ buf->iface.set_tensor_2d(buf, tensor, data, offset, size, n_copies, stride_tensor, stride_data);
385
+ }
386
+
387
+ void ggml_backend_tensor_get_2d(const struct ggml_tensor * tensor, void * data, size_t offset, size_t size,
388
+ size_t n_copies, size_t stride_tensor, size_t stride_data) {
389
+ GGML_ASSERT(tensor);
390
+ ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;
391
+ GGML_ASSERT(buf != NULL && "tensor buffer not set");
392
+
393
+ if (n_copies <= 1 || buf->iface.get_tensor_2d == NULL) {
394
+ for (size_t i = 0; i < n_copies; i++) {
395
+ ggml_backend_tensor_get(tensor, (char *) data + i*stride_data, offset + i*stride_tensor, size);
396
+ }
397
+ return;
398
+ }
399
+ if (size == 0) {
400
+ return;
401
+ }
402
+
403
+ GGML_ASSERT(tensor->data != NULL && "tensor not allocated");
404
+ GGML_ASSERT(offset + (n_copies-1)*stride_tensor + size <= ggml_nbytes(tensor) && "tensor read out of bounds");
405
+
406
+ buf->iface.get_tensor_2d(buf, tensor, data, offset, size, n_copies, stride_tensor, stride_data);
407
+ }
408
+
409
+ void ggml_backend_tensor_memset(struct ggml_tensor * tensor, uint8_t value, size_t offset, size_t size) {
410
+ GGML_ASSERT(tensor);
411
+ ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;
412
+
413
+ if (size == 0) {
414
+ return;
415
+ }
416
+
417
+ GGML_ASSERT(buf != NULL && "tensor buffer not set");
418
+ GGML_ASSERT(tensor->data != NULL && "tensor not allocated");
419
+ GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && "tensor write out of bounds");
420
+ GGML_ASSERT(buf->iface.memset_tensor != NULL && "memset not implemented by backend buffer");
421
+
422
+ buf->iface.memset_tensor(buf, tensor, value, offset, size);
423
+ }
424
+
425
+ void ggml_backend_synchronize(ggml_backend_t backend) {
426
+ GGML_ASSERT(backend);
427
+ if (backend->iface.synchronize == NULL) {
428
+ return;
429
+ }
430
+
431
+ backend->iface.synchronize(backend);
432
+ }
433
+
434
+ ggml_backend_graph_plan_t ggml_backend_graph_plan_create(ggml_backend_t backend, struct ggml_cgraph * cgraph) {
435
+ GGML_ASSERT(backend);
436
+ GGML_ASSERT(backend->iface.graph_plan_create != NULL);
437
+
438
+ return backend->iface.graph_plan_create(backend, cgraph);
439
+ }
440
+
441
+ void ggml_backend_graph_plan_free(ggml_backend_t backend, ggml_backend_graph_plan_t plan) {
442
+ GGML_ASSERT(backend);
443
+ GGML_ASSERT(backend->iface.graph_plan_free != NULL);
444
+
445
+ backend->iface.graph_plan_free(backend, plan);
446
+ }
447
+
448
+ enum ggml_status ggml_backend_graph_plan_compute(ggml_backend_t backend, ggml_backend_graph_plan_t plan) {
449
+ GGML_ASSERT(backend);
450
+ GGML_ASSERT(backend->iface.graph_plan_compute != NULL);
451
+
452
+ return backend->iface.graph_plan_compute(backend, plan);
453
+ }
454
+
455
+ enum ggml_status ggml_backend_graph_compute(ggml_backend_t backend, struct ggml_cgraph * cgraph) {
456
+ enum ggml_status err = ggml_backend_graph_compute_async(backend, cgraph);
457
+ ggml_backend_synchronize(backend);
458
+ return err;
459
+ }
460
+
461
+ enum ggml_status ggml_backend_graph_compute_async(ggml_backend_t backend, struct ggml_cgraph * cgraph) {
462
+ GGML_ASSERT(backend);
463
+ return backend->iface.graph_compute(backend, cgraph);
464
+ }
465
+
466
+ bool ggml_backend_supports_op(ggml_backend_t backend, const struct ggml_tensor * op) {
467
+ GGML_ASSERT(backend);
468
+ return ggml_backend_dev_supports_op(backend->device, op);
469
+ }
470
+
471
+ bool ggml_backend_supports_buft(ggml_backend_t backend, ggml_backend_buffer_type_t buft) {
472
+ GGML_ASSERT(backend);
473
+ return ggml_backend_dev_supports_buft(backend->device, buft);
474
+ }
475
+
476
+ bool ggml_backend_offload_op(ggml_backend_t backend, const struct ggml_tensor * op) {
477
+ GGML_ASSERT(backend);
478
+ return ggml_backend_dev_offload_op(backend->device, op);
479
+ }
480
+
481
+ ggml_backend_dev_t ggml_backend_get_device(ggml_backend_t backend) {
482
+ GGML_ASSERT(backend);
483
+ return backend->device;
484
+ }
485
+
486
+ // backend copy
487
+
488
+ void ggml_backend_tensor_copy(const struct ggml_tensor * src, struct ggml_tensor * dst) {
489
+ GGML_ASSERT(ggml_are_same_layout(src, dst) && "cannot copy tensors with different layouts");
490
+
491
+ if (src == dst) {
492
+ return;
493
+ }
494
+
495
+ if (ggml_backend_buffer_is_host(src->buffer)) {
496
+ ggml_backend_tensor_set(dst, src->data, 0, ggml_nbytes(src));
497
+ } else if (ggml_backend_buffer_is_host(dst->buffer)) {
498
+ ggml_backend_tensor_get(src, dst->data, 0, ggml_nbytes(src));
499
+ } else if (!ggml_backend_buffer_copy_tensor(src, dst)) {
500
+ #ifndef NDEBUG
501
+ GGML_LOG_DEBUG("%s: warning: slow copy from %s to %s\n", __func__, ggml_backend_buffer_name(src->buffer), ggml_backend_buffer_name(dst->buffer));
502
+ #endif // NDEBUG
503
+ size_t nbytes = ggml_nbytes(src);
504
+ void * data = malloc(nbytes);
505
+ ggml_backend_tensor_get(src, data, 0, nbytes);
506
+ ggml_backend_tensor_set(dst, data, 0, nbytes);
507
+ free(data);
508
+ }
509
+ }
510
+
511
+ void ggml_backend_tensor_copy_async(ggml_backend_t backend_src, ggml_backend_t backend_dst, const struct ggml_tensor * src, struct ggml_tensor * dst) {
512
+ GGML_ASSERT(ggml_are_same_layout(src, dst) && "cannot copy tensors with different layouts");
513
+
514
+ if (src == dst) {
515
+ return;
516
+ }
517
+
518
+ GGML_ASSERT(backend_dst);
519
+ if (backend_dst->iface.cpy_tensor_async != NULL) {
520
+ if (backend_dst->iface.cpy_tensor_async(backend_src, backend_dst, src, dst)) {
521
+ return;
522
+ }
523
+ }
524
+
525
+ // an async copy would normally happen after all the queued operations on both backends are completed
526
+ // to simulate the same behavior, we need to synchronize both backends first, and do a blocking copy
527
+ ggml_backend_synchronize(backend_src);
528
+ ggml_backend_synchronize(backend_dst);
529
+ ggml_backend_tensor_copy(src, dst);
530
+ }
531
+
532
+ // events
533
+
534
+ ggml_backend_event_t ggml_backend_event_new(ggml_backend_dev_t device) {
535
+ // null device is allowed for the transition period to the device interface
536
+ if (device == NULL || device->iface.event_new == NULL) {
537
+ return NULL;
538
+ }
539
+ return device->iface.event_new(device);
540
+ }
541
+
542
+ void ggml_backend_event_free(ggml_backend_event_t event) {
543
+ if (event == NULL) {
544
+ return;
545
+ }
546
+ event->device->iface.event_free(event->device, event);
547
+ }
548
+
549
+ void ggml_backend_event_record(ggml_backend_event_t event, ggml_backend_t backend) {
550
+ GGML_ASSERT(backend);
551
+ GGML_ASSERT(backend->iface.event_record != NULL);
552
+
553
+ backend->iface.event_record(backend, event);
554
+ }
555
+
556
+ void ggml_backend_event_synchronize(ggml_backend_event_t event) {
557
+ GGML_ASSERT(event);
558
+ GGML_ASSERT(event->device->iface.event_synchronize);
559
+
560
+ event->device->iface.event_synchronize(event->device, event);
561
+ }
562
+
563
+ void ggml_backend_event_wait(ggml_backend_t backend, ggml_backend_event_t event) {
564
+ GGML_ASSERT(backend);
565
+ GGML_ASSERT(backend->iface.event_wait != NULL);
566
+
567
+ backend->iface.event_wait(backend, event);
568
+ }
569
+
570
+ static void ggml_backend_graph_optimize(ggml_backend_t backend, struct ggml_cgraph * cgraph, struct ggml_backend_graph_optimize_params * params) {
571
+ GGML_ASSERT(backend);
572
+ if (backend->iface.graph_optimize != NULL) {
573
+ backend->iface.graph_optimize(backend, cgraph, params);
574
+ }
575
+ }
576
+
577
+ // Backend device
578
+
579
+ const char * ggml_backend_dev_name(ggml_backend_dev_t device) {
580
+ GGML_ASSERT(device);
581
+ return device->iface.get_name(device);
582
+ }
583
+
584
+ const char * ggml_backend_dev_description(ggml_backend_dev_t device) {
585
+ GGML_ASSERT(device);
586
+ return device->iface.get_description(device);
587
+ }
588
+
589
+ void ggml_backend_dev_memory(ggml_backend_dev_t device, size_t * free, size_t * total) {
590
+ GGML_ASSERT(device);
591
+ device->iface.get_memory(device, free, total);
592
+ }
593
+
594
+ enum ggml_backend_dev_type ggml_backend_dev_type(ggml_backend_dev_t device) {
595
+ GGML_ASSERT(device);
596
+ return device->iface.get_type(device);
597
+ }
598
+
599
+ void ggml_backend_dev_get_props(ggml_backend_dev_t device, struct ggml_backend_dev_props * props) {
600
+ GGML_ASSERT(device);
601
+ memset(props, 0, sizeof(*props));
602
+ device->iface.get_props(device, props);
603
+ }
604
+
605
+ ggml_backend_reg_t ggml_backend_dev_backend_reg(ggml_backend_dev_t device) {
606
+ GGML_ASSERT(device);
607
+ return device->reg;
608
+ }
609
+
610
+ ggml_backend_t ggml_backend_dev_init(ggml_backend_dev_t device, const char * params) {
611
+ GGML_ASSERT(device);
612
+ return device->iface.init_backend(device, params);
613
+ }
614
+
615
+ ggml_backend_buffer_type_t ggml_backend_dev_buffer_type(ggml_backend_dev_t device) {
616
+ GGML_ASSERT(device);
617
+ return device->iface.get_buffer_type(device);
618
+ }
619
+
620
+ ggml_backend_buffer_type_t ggml_backend_dev_host_buffer_type(ggml_backend_dev_t device) {
621
+ GGML_ASSERT(device);
622
+ if (device->iface.get_host_buffer_type == NULL) {
623
+ return NULL;
624
+ }
625
+
626
+ return device->iface.get_host_buffer_type(device);
627
+ }
628
+
629
+ ggml_backend_buffer_t ggml_backend_dev_buffer_from_host_ptr(ggml_backend_dev_t device, void * ptr, size_t size, size_t max_tensor_size) {
630
+ GGML_ASSERT(device);
631
+ return device->iface.buffer_from_host_ptr(device, ptr, size, max_tensor_size);
632
+ }
633
+
634
+ bool ggml_backend_dev_supports_op(ggml_backend_dev_t device, const struct ggml_tensor * op) {
635
+ GGML_ASSERT(device);
636
+ return device->iface.supports_op(device, op);
637
+ }
638
+
639
+ bool ggml_backend_dev_supports_buft(ggml_backend_dev_t device, ggml_backend_buffer_type_t buft) {
640
+ GGML_ASSERT(device);
641
+ return device->iface.supports_buft(device, buft);
642
+ }
643
+
644
+ bool ggml_backend_dev_offload_op(ggml_backend_dev_t device, const struct ggml_tensor * op) {
645
+ GGML_ASSERT(device);
646
+ if (device->iface.offload_op != NULL) {
647
+ return device->iface.offload_op(device, op);
648
+ }
649
+
650
+ return false;
651
+ }
652
+
653
+ // Backend (reg)
654
+
655
+ const char * ggml_backend_reg_name(ggml_backend_reg_t reg) {
656
+ GGML_ASSERT(reg);
657
+ return reg->iface.get_name(reg);
658
+ }
659
+
660
+ size_t ggml_backend_reg_dev_count(ggml_backend_reg_t reg) {
661
+ GGML_ASSERT(reg);
662
+ return reg->iface.get_device_count(reg);
663
+ }
664
+
665
+ ggml_backend_dev_t ggml_backend_reg_dev_get(ggml_backend_reg_t reg, size_t index) {
666
+ GGML_ASSERT(reg);
667
+ return reg->iface.get_device(reg, index);
668
+ }
669
+
670
+ void * ggml_backend_reg_get_proc_address(ggml_backend_reg_t reg, const char * name) {
671
+ GGML_ASSERT(reg);
672
+ if (!reg->iface.get_proc_address) {
673
+ return NULL;
674
+ }
675
+ return reg->iface.get_proc_address(reg, name);
676
+ }
677
+
678
+ // multi-buffer buffer
679
+
680
+ struct ggml_backend_multi_buffer_context {
681
+ ggml_backend_buffer_t * buffers;
682
+ size_t n_buffers;
683
+ };
684
+
685
+ static void ggml_backend_multi_buffer_free_buffer(ggml_backend_buffer_t buffer) {
686
+ GGML_ASSERT(buffer);
687
+ ggml_backend_multi_buffer_context * ctx = (ggml_backend_multi_buffer_context *) buffer->context;
688
+ for (size_t i = 0; i < ctx->n_buffers; i++) {
689
+ ggml_backend_buffer_free(ctx->buffers[i]);
690
+ }
691
+
692
+ free(ctx->buffers);
693
+ free(ctx);
694
+ }
695
+
696
+ static void ggml_backend_multi_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {
697
+ GGML_ASSERT(buffer);
698
+ ggml_backend_multi_buffer_context * ctx = (ggml_backend_multi_buffer_context *) buffer->context;
699
+ for (size_t i = 0; i < ctx->n_buffers; i++) {
700
+ ggml_backend_buffer_clear(ctx->buffers[i], value);
701
+ }
702
+ }
703
+
704
+ static const struct ggml_backend_buffer_i ggml_backend_multi_buffer_i = {
705
+ /* .free_buffer = */ ggml_backend_multi_buffer_free_buffer,
706
+ /* .get_base = */ NULL,
707
+ /* .init_tensor = */ NULL,
708
+ /* .memset_tensor = */ NULL,
709
+ /* .set_tensor = */ NULL,
710
+ /* .get_tensor = */ NULL,
711
+ /* .set_tensor_2d = */ NULL,
712
+ /* .get_tensor_2d = */ NULL,
713
+ /* .cpy_tensor = */ NULL,
714
+ /* .clear = */ ggml_backend_multi_buffer_clear,
715
+ /* .reset = */ NULL,
716
+ };
717
+
718
+ ggml_backend_buffer_t ggml_backend_multi_buffer_alloc_buffer(ggml_backend_buffer_t * buffers, size_t n_buffers) {
719
+ ggml_backend_multi_buffer_context * ctx = (ggml_backend_multi_buffer_context *) malloc(sizeof(struct ggml_backend_multi_buffer_context));
720
+ ctx->n_buffers = n_buffers;
721
+ ctx->buffers = (ggml_backend_buffer_t *) malloc(n_buffers * sizeof(ggml_backend_buffer_t));
722
+
723
+ GGML_ASSERT(ctx->buffers != NULL);
724
+
725
+ size_t total_size = 0;
726
+ for (size_t i = 0; i < n_buffers; i++) {
727
+ ctx->buffers[i] = buffers[i];
728
+ total_size += ggml_backend_buffer_get_size(buffers[i]);
729
+ }
730
+
731
+ return ggml_backend_buffer_init(buffers[0]->buft, ggml_backend_multi_buffer_i, ctx, total_size);
732
+ }
733
+
734
+ bool ggml_backend_buffer_is_multi_buffer(ggml_backend_buffer_t buffer) {
735
+ GGML_ASSERT(buffer);
736
+ return buffer->iface.free_buffer == ggml_backend_multi_buffer_free_buffer;
737
+ }
738
+
739
+ void ggml_backend_multi_buffer_set_usage(ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage) {
740
+ GGML_ASSERT(buffer);
741
+ GGML_ASSERT(ggml_backend_buffer_is_multi_buffer(buffer));
742
+ ggml_backend_multi_buffer_context * ctx = (ggml_backend_multi_buffer_context *) buffer->context;
743
+ for (size_t i = 0; i < ctx->n_buffers; i++) {
744
+ ggml_backend_buffer_set_usage(ctx->buffers[i], usage);
745
+ }
746
+ }
747
+
748
+ // creates a copy of the tensor with the same memory layout
749
+ static struct ggml_tensor * ggml_dup_tensor_layout(struct ggml_context * ctx, const struct ggml_tensor * tensor) {
750
+ struct ggml_tensor * dup = ggml_dup_tensor(ctx, tensor);
751
+ for (int i = 0; i < GGML_MAX_DIMS; i++) {
752
+ dup->nb[i] = tensor->nb[i];
753
+ }
754
+ return dup;
755
+ }
756
+
757
+ static bool ggml_is_view_op(enum ggml_op op) {
758
+ return op == GGML_OP_VIEW || op == GGML_OP_RESHAPE || op == GGML_OP_PERMUTE || op == GGML_OP_TRANSPOSE;
759
+ }
760
+
761
+ // scheduler
762
+
763
+ #ifndef GGML_SCHED_MAX_BACKENDS
764
+ #define GGML_SCHED_MAX_BACKENDS 16
765
+ #endif
766
+
767
+ #ifndef GGML_SCHED_MAX_SPLIT_INPUTS
768
+ #define GGML_SCHED_MAX_SPLIT_INPUTS 30
769
+ #endif
770
+
771
+ #ifndef GGML_SCHED_MAX_COPIES
772
+ #define GGML_SCHED_MAX_COPIES 4
773
+ #endif
774
+
775
+ struct ggml_backend_sched_split {
776
+ int backend_id;
777
+ int i_start;
778
+ int i_end;
779
+ struct ggml_tensor ** inputs;
780
+ int n_inputs;
781
+ int inputs_capacity;
782
+ // graph view of this split
783
+ struct ggml_cgraph graph;
784
+ };
785
+
786
+ struct ggml_backend_sched {
787
+ bool is_reset; // true if the scheduler has been reset since the last graph split
788
+ bool is_alloc;
789
+
790
+ int n_backends;
791
+
792
+ ggml_backend_t backends[GGML_SCHED_MAX_BACKENDS];
793
+ ggml_backend_buffer_type_t bufts[GGML_SCHED_MAX_BACKENDS];
794
+ ggml_gallocr_t galloc;
795
+
796
+ // hash map of the nodes in the graph
797
+ struct ggml_hash_set hash_set;
798
+ int * hv_tensor_backend_ids; // [hash_set.size]
799
+ struct ggml_tensor ** hv_tensor_copies; // [hash_set.size][n_backends][n_copies]
800
+
801
+ int * node_backend_ids; // [graph_size]
802
+ int * leaf_backend_ids; // [graph_size]
803
+
804
+ int * prev_node_backend_ids; // [graph_size]
805
+ int * prev_leaf_backend_ids; // [graph_size]
806
+
807
+ // copy of the graph with modified inputs
808
+ struct ggml_cgraph graph;
809
+
810
+ // graph splits
811
+ struct ggml_backend_sched_split * splits;
812
+ int n_splits;
813
+ int splits_capacity;
814
+
815
+ // pipeline parallelism support
816
+ int n_copies;
817
+ int cur_copy;
818
+ int next_copy;
819
+ ggml_backend_event_t events[GGML_SCHED_MAX_BACKENDS][GGML_SCHED_MAX_COPIES];
820
+ struct ggml_tensor ** graph_inputs;
821
+ int n_graph_inputs;
822
+ int graph_inputs_capacity;
823
+
824
+ struct ggml_context * ctx;
825
+
826
+ ggml_backend_sched_eval_callback callback_eval;
827
+ void * callback_eval_user_data;
828
+
829
+ char * context_buffer;
830
+ size_t context_buffer_size;
831
+
832
+ bool op_offload;
833
+
834
+ int debug;
835
+
836
+ // used for debugging graph reallocations [GGML_SCHED_DEBUG_REALLOC]
837
+ // ref: https://github.com/ggml-org/llama.cpp/pull/17617
838
+ int debug_realloc;
839
+ int debug_graph_size;
840
+ int debug_prev_graph_size;
841
+ };
842
+
843
+ #define hash_id(tensor) ggml_hash_find_or_insert(&sched->hash_set, tensor)
844
+ #define tensor_backend_id(tensor) sched->hv_tensor_backend_ids[hash_id(tensor)]
845
+ #define tensor_id_copy(id, backend_id, copy_id) sched->hv_tensor_copies[(id) * sched->n_backends * sched->n_copies + (backend_id) * sched->n_copies + (copy_id)]
846
+ #define tensor_copy(tensor, backend_id, copy_id) tensor_id_copy(hash_id(tensor), backend_id, copy_id)
847
+
848
+ static void ggml_backend_sched_split_inputs_grow(struct ggml_backend_sched_split * split) {
849
+ int new_cap = GGML_SCHED_MAX_SPLIT_INPUTS;
850
+ if (split->inputs_capacity > 0) {
851
+ new_cap = 2*split->inputs_capacity;
852
+ GGML_LOG_WARN("%s: increasing split inputs capacity from %d to %d\n", __func__, split->inputs_capacity, new_cap);
853
+ }
854
+ auto * pnew = (struct ggml_tensor **) realloc((void *) split->inputs, new_cap * sizeof(struct ggml_tensor *));
855
+ if (pnew == NULL) {
856
+ GGML_LOG_ERROR("%s: failed to allocate %zu bytes\n", __func__, new_cap * sizeof(struct ggml_tensor *));
857
+ GGML_ABORT("failed to grow split inputs container");
858
+ }
859
+ split->inputs = pnew;
860
+ split->inputs_capacity = new_cap;
861
+ }
862
+
863
+ static void ggml_backend_sched_graph_inputs_grow(ggml_backend_sched_t sched) {
864
+ int new_cap = GGML_SCHED_MAX_SPLIT_INPUTS;
865
+ if (sched->graph_inputs_capacity > 0) {
866
+ new_cap = 2*sched->graph_inputs_capacity;
867
+ GGML_LOG_WARN("%s: increasing graph inputs capacity from %d to %d\n", __func__, sched->graph_inputs_capacity, new_cap);
868
+ }
869
+ auto * pnew = (struct ggml_tensor **) realloc((void *) sched->graph_inputs, new_cap * sizeof(struct ggml_tensor *));
870
+ if (pnew == NULL) {
871
+ GGML_LOG_ERROR("%s: failed to allocate %zu bytes\n", __func__, new_cap * sizeof(struct ggml_tensor *));
872
+ GGML_ABORT("failed to grow graph inputs container");
873
+ }
874
+ sched->graph_inputs = pnew;
875
+ sched->graph_inputs_capacity = new_cap;
876
+ }
877
+
878
+ // returns the priority of the backend, lower id is higher priority
879
+ static int ggml_backend_sched_backend_id(ggml_backend_sched_t sched, ggml_backend_t backend) {
880
+ for (int i = 0; i < sched->n_backends; i++) {
881
+ if (sched->backends[i] == backend) {
882
+ return i;
883
+ }
884
+ }
885
+ return -1;
886
+ }
887
+
888
+ static int ggml_backend_sched_backend_from_buffer(ggml_backend_sched_t sched, const struct ggml_tensor * tensor, const struct ggml_tensor * op) {
889
+ ggml_backend_buffer_t buffer = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;
890
+ if (buffer == NULL) {
891
+ return -1;
892
+ }
893
+
894
+ // find highest prio backend that supports the buffer type and the op
895
+ for (int i = 0; i < sched->n_backends; i++) {
896
+ if (ggml_backend_supports_buft(sched->backends[i], buffer->buft) &&
897
+ ggml_backend_supports_op(sched->backends[i], op)) {
898
+ return i;
899
+ }
900
+ }
901
+
902
+ #ifndef NDEBUG
903
+ GGML_LOG_DEBUG("%s: warning: no backend supports op %s with a weight with buffer type %s used in tensor %s, the weight will need to be copied\n",
904
+ __func__, ggml_op_desc(tensor), ggml_backend_buffer_name(buffer), tensor->name);
905
+ #endif
906
+
907
+ return -1;
908
+ }
909
+
910
+ #if 0
911
+ #define GGML_SCHED_MAX_SPLITS_DEBUG 4096
912
+ static char causes[GGML_DEFAULT_GRAPH_SIZE*16 + GGML_SCHED_MAX_SPLITS_DEBUG*GGML_SCHED_MAX_SPLIT_INPUTS][128]; // debug only
913
+ #define SET_CAUSE(node, ...) sprintf(causes[hash_id(node)], __VA_ARGS__)
914
+ #define GET_CAUSE(node) causes[hash_id(node)]
915
+ #else
916
+ #define SET_CAUSE(node, ...)
917
+ #define GET_CAUSE(node) ""
918
+ #endif
919
+
920
+ // returns the backend that should be used for the node based on the current locations
921
+ static int ggml_backend_sched_backend_id_from_cur(ggml_backend_sched_t sched, struct ggml_tensor * tensor) {
922
+ // assign pre-allocated nodes to their backend
923
+ int cur_backend_id = ggml_backend_sched_backend_from_buffer(sched, tensor, tensor);
924
+ if (cur_backend_id != -1) {
925
+ SET_CAUSE(tensor, "1.dst");
926
+ return cur_backend_id;
927
+ }
928
+
929
+ // view_src
930
+ if (tensor->view_src != NULL) {
931
+ cur_backend_id = ggml_backend_sched_backend_from_buffer(sched, tensor->view_src, tensor);
932
+ if (cur_backend_id != -1) {
933
+ SET_CAUSE(tensor, "1.vsrc");
934
+ return cur_backend_id;
935
+ }
936
+ }
937
+
938
+ if (tensor->buffer || (tensor->view_src && tensor->view_src->buffer)) {
939
+ // since the tensor is pre-allocated, it cannot be moved to another backend
940
+ ggml_backend_buffer_t buffer = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;
941
+ GGML_ABORT("pre-allocated tensor (%s) in a buffer (%s) that cannot run the operation (%s)", tensor->name, ggml_backend_buffer_name(buffer), ggml_op_name(tensor->op));
942
+ }
943
+
944
+ // graph input
945
+ if (tensor->flags & GGML_TENSOR_FLAG_INPUT) {
946
+ cur_backend_id = sched->n_backends - 1; // last backend (assumed CPU)
947
+ SET_CAUSE(tensor, "1.inp");
948
+ return cur_backend_id;
949
+ }
950
+
951
+ // operations with weights are preferably run on the same backend as the weights
952
+ // TODO: there are exceptions (see below) - not an ideal solution
953
+ bool allow = true;
954
+
955
+ // skip ROPE since the rope freqs tensor is too small to choose a backend based on it
956
+ allow = allow && tensor->op != GGML_OP_ROPE;
957
+
958
+ // skip FLASH_ATTN_EXT since the sinks tensor is too small to choose a based based on it
959
+ allow = allow && tensor->op != GGML_OP_FLASH_ATTN_EXT;
960
+
961
+ if (allow) {
962
+ for (int i = 0; i < GGML_MAX_SRC; i++) {
963
+ const struct ggml_tensor * src = tensor->src[i];
964
+ if (src == NULL) {
965
+ continue;
966
+ }
967
+ if (src->buffer != NULL && src->buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS) {
968
+ int src_backend_id = ggml_backend_sched_backend_from_buffer(sched, src, tensor);
969
+ // check if a backend with higher prio wants to offload the op
970
+ if (sched->op_offload && src_backend_id == sched->n_backends - 1 && ggml_backend_buffer_is_host(src->buffer)) {
971
+ for (int b = 0; b < src_backend_id; b++) {
972
+ if (ggml_backend_supports_op(sched->backends[b], tensor) && ggml_backend_offload_op(sched->backends[b], tensor)) {
973
+ SET_CAUSE(tensor, "1.off");
974
+ return b;
975
+ }
976
+ }
977
+ }
978
+ SET_CAUSE(tensor, "1.wgt%d", i);
979
+ return src_backend_id;
980
+ }
981
+ }
982
+ }
983
+
984
+ return -1;
985
+ }
986
+
987
+ static char * fmt_size(size_t size) {
988
+ static char buffer[128];
989
+ if (size >= 1024*1024) {
990
+ snprintf(buffer, sizeof(buffer), "%zuM", size/1024/1024);
991
+ } else {
992
+ snprintf(buffer, sizeof(buffer), "%zuK", size/1024);
993
+ }
994
+ return buffer;
995
+ }
996
+
997
+ static void ggml_backend_sched_print_assignments(ggml_backend_sched_t sched, struct ggml_cgraph * graph) {
998
+ int cur_split = 0;
999
+ for (int i = 0; i < graph->n_nodes; i++) {
1000
+ if (cur_split < sched->n_splits && i == sched->splits[cur_split].i_start) {
1001
+ ggml_backend_t split_backend = sched->backends[sched->splits[cur_split].backend_id];
1002
+ GGML_LOG_DEBUG("\n## SPLIT #%d: %s # %d inputs", cur_split, ggml_backend_name(split_backend),
1003
+ sched->splits[cur_split].n_inputs);
1004
+ for (int j = 0; j < sched->splits[cur_split].n_inputs; j++) {
1005
+ if (j == 0) {
1006
+ GGML_LOG_DEBUG(": ");
1007
+ }
1008
+ GGML_LOG_DEBUG("[%s (%5.5s)] ", sched->splits[cur_split].inputs[j]->name,
1009
+ fmt_size(ggml_nbytes(sched->splits[cur_split].inputs[j])));
1010
+ }
1011
+ GGML_LOG_DEBUG("\n");
1012
+ cur_split++;
1013
+ }
1014
+ struct ggml_tensor * node = graph->nodes[i];
1015
+ if (ggml_is_view_op(node->op)) {
1016
+ continue;
1017
+ }
1018
+ if (sched->debug > 1) {
1019
+ ggml_backend_t tensor_backend = ggml_backend_sched_get_tensor_backend(sched, node);
1020
+ GGML_LOG_DEBUG("node #%3d (%10.10s): %20.20s (%5.5s) [%5.5s %8.8s] use=%d,c=%d:", i, ggml_op_desc(node), node->name,
1021
+ fmt_size(ggml_nbytes(node)), tensor_backend ? ggml_backend_name(tensor_backend) : "NULL", GET_CAUSE(node),
1022
+ graph->use_counts[ggml_hash_find(&graph->visited_hash_set, node)], node->flags & GGML_TENSOR_FLAG_COMPUTE ? 1 : 0);
1023
+ for (int j = 0; j < GGML_MAX_SRC; j++) {
1024
+ struct ggml_tensor * src = node->src[j];
1025
+ if (src == NULL) {
1026
+ continue;
1027
+ }
1028
+ ggml_backend_t src_backend = ggml_backend_sched_get_tensor_backend(sched, src);
1029
+ GGML_LOG_DEBUG(" %20.20s (%5.5s) [%5.5s %8.8s]", src->name,
1030
+ fmt_size(ggml_nbytes(src)), src_backend ? ggml_backend_name(src_backend) : "NULL", GET_CAUSE(src));
1031
+ }
1032
+ GGML_LOG_DEBUG("\n");
1033
+ }
1034
+ }
1035
+ }
1036
+
1037
+ static bool ggml_backend_sched_buffer_supported(ggml_backend_sched_t sched, struct ggml_tensor * t, int backend_id) {
1038
+ ggml_backend_buffer_t buf = t->view_src ? t->view_src->buffer : t->buffer;
1039
+ ggml_backend_buffer_type_t buft = NULL;
1040
+
1041
+ if (buf) {
1042
+ // the tensor is already allocated
1043
+ buft = buf->buft;
1044
+ } else {
1045
+ // see if the tensor already has a backend assigned, and use the buffer type of that backend
1046
+ int tensor_backend_id = tensor_backend_id(t);
1047
+ if (tensor_backend_id == -1 && t->view_src) {
1048
+ tensor_backend_id = tensor_backend_id(t->view_src);
1049
+ }
1050
+ if (tensor_backend_id != -1) {
1051
+ buft = sched->bufts[tensor_backend_id];
1052
+ }
1053
+ }
1054
+
1055
+ return buft != NULL && ggml_backend_supports_buft(sched->backends[backend_id], buft);
1056
+ }
1057
+
1058
+ static void ggml_backend_sched_set_if_supported(ggml_backend_sched_t sched, struct ggml_tensor * node, int cur_backend_id, int * node_backend_id) {
1059
+ if (ggml_backend_supports_op(sched->backends[cur_backend_id], node)) {
1060
+ *node_backend_id = cur_backend_id;
1061
+ SET_CAUSE(node, "2.sup");
1062
+ }
1063
+ }
1064
+
1065
+ // assigns backends to ops and splits the graph into subgraphs that can be computed on the same backend
1066
+ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgraph * graph) {
1067
+ // reset splits
1068
+ sched->n_splits = 0;
1069
+ sched->n_graph_inputs = 0;
1070
+ sched->is_reset = false;
1071
+
1072
+ struct ggml_init_params params = {
1073
+ /* .mem_size = */ sched->context_buffer_size,
1074
+ /* .mem_buffer = */ sched->context_buffer,
1075
+ /* .no_alloc = */ true
1076
+ };
1077
+
1078
+ ggml_free(sched->ctx);
1079
+
1080
+ sched->ctx = ggml_init(params);
1081
+ if (sched->ctx == NULL) {
1082
+ GGML_ABORT("%s: failed to initialize context\n", __func__);
1083
+ }
1084
+
1085
+ graph->uid = ggml_graph_next_uid();
1086
+
1087
+ // pass 1: assign backends to ops with pre-allocated inputs
1088
+ for (int i = 0; i < graph->n_leafs; i++) {
1089
+ struct ggml_tensor * leaf = graph->leafs[i];
1090
+ int * leaf_backend_id = &tensor_backend_id(leaf);
1091
+ // do not overwrite user assignments
1092
+ if (*leaf_backend_id == -1) {
1093
+ *leaf_backend_id = ggml_backend_sched_backend_id_from_cur(sched, leaf);
1094
+ }
1095
+ }
1096
+
1097
+ for (int i = 0; i < graph->n_nodes; i++) {
1098
+ struct ggml_tensor * node = graph->nodes[i];
1099
+ int * node_backend_id = &tensor_backend_id(node);
1100
+ // do not overwrite user assignments
1101
+ if (*node_backend_id == -1) {
1102
+ *node_backend_id = ggml_backend_sched_backend_id_from_cur(sched, node);
1103
+
1104
+ #if 0
1105
+ // src
1106
+ if (node->op == GGML_OP_NONE) {
1107
+ continue;
1108
+ }
1109
+
1110
+ for (int j = 0; j < GGML_MAX_SRC; j++) {
1111
+ struct ggml_tensor * src = node->src[j];
1112
+ if (src == NULL) {
1113
+ continue;
1114
+ }
1115
+ int * src_backend_id = &tensor_backend_id(src);
1116
+ if (*src_backend_id == -1) {
1117
+ *src_backend_id = ggml_backend_sched_backend_id_from_cur(sched, src);
1118
+ }
1119
+ }
1120
+ #endif
1121
+ }
1122
+ }
1123
+
1124
+ // pass 2: expand current backend assignments
1125
+ // assign the same backend to adjacent nodes
1126
+ // expand gpu backends (i.e. non last prio) up and down, ignoring cpu (the lowest priority backend)
1127
+ // thus, cpu will never be used unless weights are on cpu, or there are no gpu ops between cpu ops
1128
+ // ops unsupported by the backend being expanded will be left unassigned so that they can be assigned later when the locations of its inputs are known
1129
+ // expand gpu down
1130
+ {
1131
+ int cur_backend_id = -1;
1132
+ for (int i = 0; i < graph->n_nodes; i++) {
1133
+ struct ggml_tensor * node = graph->nodes[i];
1134
+ if (ggml_is_view_op(node->op)) {
1135
+ continue;
1136
+ }
1137
+ int * node_backend_id = &tensor_backend_id(node);
1138
+ if (*node_backend_id != -1) {
1139
+ if (*node_backend_id == sched->n_backends - 1) {
1140
+ // skip cpu (lowest prio backend)
1141
+ cur_backend_id = -1;
1142
+ } else {
1143
+ cur_backend_id = *node_backend_id;
1144
+ }
1145
+ } else if (cur_backend_id != -1) {
1146
+ ggml_backend_sched_set_if_supported(sched, node, cur_backend_id, node_backend_id);
1147
+ }
1148
+ }
1149
+ }
1150
+ // expand gpu up
1151
+ {
1152
+ int cur_backend_id = -1;
1153
+ for (int i = graph->n_nodes - 1; i >= 0; i--) {
1154
+ struct ggml_tensor * node = graph->nodes[i];
1155
+ if (ggml_is_view_op(node->op)) {
1156
+ continue;
1157
+ }
1158
+ int * node_backend_id = &tensor_backend_id(node);
1159
+ if (*node_backend_id != -1) {
1160
+ if (*node_backend_id == sched->n_backends - 1) {
1161
+ // skip cpu (lowest prio backend)
1162
+ cur_backend_id = -1;
1163
+ } else {
1164
+ cur_backend_id = *node_backend_id;
1165
+ }
1166
+ } else if (cur_backend_id != -1) {
1167
+ ggml_backend_sched_set_if_supported(sched, node, cur_backend_id, node_backend_id);
1168
+ }
1169
+ }
1170
+ }
1171
+ // expand rest down
1172
+ {
1173
+ int cur_backend_id = -1;
1174
+ for (int i = 0; i < graph->n_nodes; i++) {
1175
+ struct ggml_tensor * node = graph->nodes[i];
1176
+ if (ggml_is_view_op(node->op)) {
1177
+ continue;
1178
+ }
1179
+ int * node_backend_id = &tensor_backend_id(node);
1180
+ if (*node_backend_id != -1) {
1181
+ cur_backend_id = *node_backend_id;
1182
+ } else if (cur_backend_id != -1) {
1183
+ ggml_backend_sched_set_if_supported(sched, node, cur_backend_id, node_backend_id);
1184
+ }
1185
+ }
1186
+ }
1187
+ // expand rest up
1188
+ {
1189
+ int cur_backend_id = -1;
1190
+ for (int i = graph->n_nodes - 1; i >= 0; i--) {
1191
+ struct ggml_tensor * node = graph->nodes[i];
1192
+ if (ggml_is_view_op(node->op)) {
1193
+ continue;
1194
+ }
1195
+ int * node_backend_id = &tensor_backend_id(node);
1196
+ if (*node_backend_id != -1) {
1197
+ cur_backend_id = *node_backend_id;
1198
+ } else if (cur_backend_id != -1) {
1199
+ ggml_backend_sched_set_if_supported(sched, node, cur_backend_id, node_backend_id);
1200
+ }
1201
+ }
1202
+ }
1203
+
1204
+ // pass 3: upgrade nodes to higher prio backends with compatible buffer types
1205
+ // if the tensor is already in the same buffer type (*) as another higher priority backend, we should move it there
1206
+ // however, we also need to verify that the sources are in compatible buffer types
1207
+ // (*) the actual requirement is more relaxed, the buffer type of the backend should be supported by all the users of this tensor further down the graph
1208
+ // however, this is slow to verify, so we have a more strict requirement that the buffer type is the same
1209
+ // this is not uncommon since multiple backends can use host memory, with the same buffer type (eg. BLAS and CPU)
1210
+ // additionally, set remaining unassigned nodes to the backend with the most supported inputs
1211
+ // only nodes that could not be assigned during expansion due to the backend not supporting the op should be unassigned at this point
1212
+ for (int i = 0; i < graph->n_nodes; i++) {
1213
+ struct ggml_tensor * node = graph->nodes[i];
1214
+ if (ggml_is_view_op(node->op)) {
1215
+ continue;
1216
+ }
1217
+ int * node_backend_id = &tensor_backend_id(node);
1218
+ if (*node_backend_id == -1) {
1219
+ // unassigned node: find the backend with the most supported inputs
1220
+ int n_supported_best = -1;
1221
+ for (int b = 0; b < sched->n_backends; b++) {
1222
+ if (ggml_backend_supports_op(sched->backends[b], node)) {
1223
+ int n_supported = 0;
1224
+ for (int j = 0; j < GGML_MAX_SRC; j++) {
1225
+ struct ggml_tensor * src = node->src[j];
1226
+ if (src == NULL) {
1227
+ continue;
1228
+ }
1229
+ if ((tensor_backend_id(src) != -1 || tensor_backend_id(src->view_src) != -1) && ggml_backend_sched_buffer_supported(sched, src, b)) {
1230
+ n_supported++;
1231
+ }
1232
+ }
1233
+ if (n_supported > n_supported_best) {
1234
+ n_supported_best = n_supported;
1235
+ *node_backend_id = b;
1236
+ SET_CAUSE(node, "3.best");
1237
+ }
1238
+ }
1239
+ }
1240
+ } else {
1241
+ // assigned node: upgrade to higher prio backend if possible
1242
+ for (int b = 0; b < *node_backend_id; b++) {
1243
+ if (sched->bufts[b] == sched->bufts[*node_backend_id] && ggml_backend_supports_op(sched->backends[b], node)) {
1244
+ bool supported = true;
1245
+ for (int j = 0; j < GGML_MAX_SRC; j++) {
1246
+ struct ggml_tensor * src = node->src[j];
1247
+ if (src == NULL) {
1248
+ continue;
1249
+ }
1250
+ if (!ggml_backend_sched_buffer_supported(sched, src, b)) {
1251
+ supported = false;
1252
+ break;
1253
+ }
1254
+ }
1255
+ if (supported) {
1256
+ *node_backend_id = b;
1257
+ SET_CAUSE(node, "3.upg");
1258
+ break;
1259
+ }
1260
+ }
1261
+ }
1262
+ }
1263
+ }
1264
+
1265
+ // pass 4: assign backends to remaining src from dst and view_src
1266
+ for (int i = 0; i < graph->n_nodes; i++) {
1267
+ struct ggml_tensor * node = graph->nodes[i];
1268
+ int * cur_backend_id = &tensor_backend_id(node);
1269
+ if (node->view_src != NULL && *cur_backend_id == -1) {
1270
+ *cur_backend_id = tensor_backend_id(node->view_src);
1271
+ SET_CAUSE(node, "4.vsrc");
1272
+ }
1273
+ for (int j = 0; j < GGML_MAX_SRC; j++) {
1274
+ struct ggml_tensor * src = node->src[j];
1275
+ if (src == NULL) {
1276
+ continue;
1277
+ }
1278
+ int * src_backend_id = &tensor_backend_id(src);
1279
+ if (*src_backend_id == -1) {
1280
+ if (src->view_src != NULL) {
1281
+ // views are always on the same backend as the source
1282
+ *src_backend_id = tensor_backend_id(src->view_src);
1283
+ SET_CAUSE(src, "4.vsrc");
1284
+ } else {
1285
+ *src_backend_id = *cur_backend_id;
1286
+ SET_CAUSE(src, "4.cur");
1287
+ }
1288
+ }
1289
+ }
1290
+ // if the node is still unassigned, assign it to the first backend that supports it
1291
+ for (int b = 0; b < sched->n_backends && *cur_backend_id == -1; b++) {
1292
+ ggml_backend_sched_set_if_supported(sched, node, b, cur_backend_id);
1293
+ }
1294
+ GGML_ASSERT(*cur_backend_id != -1);
1295
+ }
1296
+
1297
+ // pass 5: split graph, find tensors that need to be copied
1298
+ {
1299
+ int i_split = 0;
1300
+ struct ggml_backend_sched_split * split = &sched->splits[0];
1301
+ // find the backend of the first split, skipping view ops
1302
+ int i = 0;
1303
+ for (; i < graph->n_nodes; i++) {
1304
+ struct ggml_tensor * node = graph->nodes[i];
1305
+ if (!ggml_is_view_op(node->op)) {
1306
+ split->backend_id = tensor_backend_id(node);
1307
+ break;
1308
+ }
1309
+ }
1310
+ split->i_start = 0;
1311
+ split->n_inputs = 0;
1312
+ int cur_backend_id = split->backend_id;
1313
+ for (; i < graph->n_nodes; i++) {
1314
+ struct ggml_tensor * node = graph->nodes[i];
1315
+
1316
+ if (ggml_is_view_op(node->op)) {
1317
+ continue;
1318
+ }
1319
+
1320
+ const int node_backend_id = tensor_backend_id(node);
1321
+
1322
+ GGML_ASSERT(node_backend_id != -1); // all nodes should be assigned by now, this can happen if there is no CPU fallback
1323
+
1324
+ // check if we should start a new split based on the sources of the current node
1325
+ bool need_new_split = false;
1326
+ if (node_backend_id == cur_backend_id && split->n_inputs > 0) {
1327
+ for (int j = 0; j < GGML_MAX_SRC; j++) {
1328
+ struct ggml_tensor * src = node->src[j];
1329
+ if (src == NULL) {
1330
+ continue;
1331
+ }
1332
+ // check if a weight is on a different and incompatible backend
1333
+ // by starting a new split, the memory of the previously offloaded weights can be reused
1334
+ if (src->buffer != NULL && src->buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS) {
1335
+ int src_backend_id = tensor_backend_id(src);
1336
+ if (src_backend_id != cur_backend_id && !ggml_backend_sched_buffer_supported(sched, src, cur_backend_id)) {
1337
+ need_new_split = true;
1338
+ break;
1339
+ }
1340
+ }
1341
+ // check if the split has too many inputs
1342
+ // FIXME: count the number of inputs instead of only checking when full
1343
+ if (split->n_inputs >= split->inputs_capacity) {
1344
+ const size_t id = hash_id(src);
1345
+ int src_backend_id = sched->hv_tensor_backend_ids[id];
1346
+ bool supported = ggml_backend_sched_buffer_supported(sched, src, cur_backend_id);
1347
+ if (src_backend_id != cur_backend_id && tensor_id_copy(id, cur_backend_id, 0) == NULL && !supported) {
1348
+ need_new_split = true;
1349
+ break;
1350
+ }
1351
+ }
1352
+ }
1353
+ }
1354
+
1355
+ if (node_backend_id != cur_backend_id || need_new_split) {
1356
+ split->i_end = i;
1357
+ i_split++;
1358
+ if (i_split >= sched->splits_capacity) {
1359
+ int old_cap = sched->splits_capacity;
1360
+ sched->splits_capacity *= 2;
1361
+ sched->splits = (ggml_backend_sched_split *)
1362
+ realloc(sched->splits, sched->splits_capacity * sizeof(struct ggml_backend_sched_split));
1363
+ GGML_ASSERT(sched->splits != NULL);
1364
+ for (int k = old_cap; k < sched->splits_capacity; k++) {
1365
+ memset(&sched->splits[k], 0, sizeof(struct ggml_backend_sched_split));
1366
+ }
1367
+ }
1368
+ split = &sched->splits[i_split];
1369
+ split->backend_id = node_backend_id;
1370
+ split->i_start = i;
1371
+ split->n_inputs = 0;
1372
+ cur_backend_id = node_backend_id;
1373
+ }
1374
+
1375
+ // find inputs that are not on the same backend
1376
+ for (int j = 0; j < GGML_MAX_SRC; j++) {
1377
+ struct ggml_tensor * src = node->src[j];
1378
+ if (src == NULL) {
1379
+ continue;
1380
+ }
1381
+
1382
+ size_t src_id = hash_id(src);
1383
+ const int src_backend_id = sched->hv_tensor_backend_ids[src_id];
1384
+ GGML_ASSERT(src_backend_id != -1); // all inputs should be assigned by now
1385
+
1386
+ if (src->flags & GGML_TENSOR_FLAG_INPUT && sched->n_copies > 1) {
1387
+ if (tensor_id_copy(src_id, src_backend_id, 0) == NULL) {
1388
+ ggml_backend_t backend = sched->backends[src_backend_id];
1389
+ for (int c = 0; c < sched->n_copies; c++) {
1390
+ struct ggml_tensor * tensor_copy;
1391
+ if (c == sched->cur_copy) {
1392
+ tensor_copy = src; // use the original tensor as the current copy
1393
+ } else {
1394
+ tensor_copy = ggml_dup_tensor_layout(sched->ctx, src);
1395
+ ggml_format_name(tensor_copy, "%s#%s#%d", ggml_backend_name(backend), src->name, c);
1396
+ }
1397
+ ggml_set_input(tensor_copy);
1398
+ ggml_set_output(tensor_copy); // prevent ggml-alloc from overwriting the tensor
1399
+ tensor_id_copy(src_id, src_backend_id, c) = tensor_copy;
1400
+ SET_CAUSE(tensor_copy, "4.cpy");
1401
+ }
1402
+ int n_graph_inputs = sched->n_graph_inputs++;
1403
+ if (n_graph_inputs >= sched->graph_inputs_capacity) {
1404
+ ggml_backend_sched_graph_inputs_grow(sched);
1405
+ }
1406
+ sched->graph_inputs[n_graph_inputs] = src;
1407
+ }
1408
+ }
1409
+
1410
+ if (src_backend_id != cur_backend_id && !ggml_backend_sched_buffer_supported(sched, src, cur_backend_id)) {
1411
+ // create a copy of the input in the split's backend
1412
+ if (tensor_id_copy(src_id, cur_backend_id, 0) == NULL) {
1413
+ ggml_backend_t backend = sched->backends[cur_backend_id];
1414
+ for (int c = 0; c < sched->n_copies; c++) {
1415
+ struct ggml_tensor * tensor_copy = ggml_dup_tensor_layout(sched->ctx, src);
1416
+ ggml_format_name(tensor_copy, "%s#%s#%d", ggml_backend_name(backend), src->name, c);
1417
+ if (sched->n_copies > 1) {
1418
+ ggml_set_input(tensor_copy);
1419
+ ggml_set_output(tensor_copy); // prevent ggml-alloc from overwriting the tensor
1420
+ }
1421
+ tensor_id_copy(src_id, cur_backend_id, c) = tensor_copy;
1422
+ SET_CAUSE(tensor_copy, "4.cpy");
1423
+ }
1424
+ int n_inputs = split->n_inputs++;
1425
+ if (n_inputs >= split->inputs_capacity) {
1426
+ ggml_backend_sched_split_inputs_grow(split);
1427
+ }
1428
+ split->inputs[n_inputs] = src;
1429
+ }
1430
+ node->src[j] = tensor_id_copy(src_id, cur_backend_id, sched->cur_copy);
1431
+ }
1432
+ }
1433
+ }
1434
+ split->i_end = graph->n_nodes;
1435
+ sched->n_splits = i_split + 1;
1436
+ }
1437
+
1438
+ if (sched->debug) {
1439
+ ggml_backend_sched_print_assignments(sched, graph);
1440
+ }
1441
+
1442
+ // swap node_backend_ids and leaf _backend_ids with prevs
1443
+ {
1444
+ int * tmp = sched->node_backend_ids;
1445
+ sched->node_backend_ids = sched->prev_node_backend_ids;
1446
+ sched->prev_node_backend_ids = tmp;
1447
+
1448
+ tmp = sched->leaf_backend_ids;
1449
+ sched->leaf_backend_ids = sched->prev_leaf_backend_ids;
1450
+ sched->prev_leaf_backend_ids = tmp;
1451
+ }
1452
+
1453
+ // optimize the split graphs and collect the allocation dependencies added by the backends
1454
+ // this needs to happen before we make graph_copy, so they are in sync
1455
+ // TODO: this may create many small allocations in the scheduler, restructure to use a flat array
1456
+ std::unordered_map<ggml_tensor *, std::vector<ggml_tensor *>> alloc_deps;
1457
+
1458
+ struct ggml_backend_graph_optimize_params opt_params = {
1459
+ /* .add_alloc_dep = */ [](void * user_data, ggml_tensor * tensor, ggml_tensor * until) {
1460
+ auto & deps = *(std::unordered_map<ggml_tensor *, std::vector<ggml_tensor *>> *) user_data;
1461
+ std::vector<ggml_tensor *> & keep = deps[until];
1462
+ if (std::find(keep.begin(), keep.end(), tensor) == keep.end()) {
1463
+ keep.push_back(tensor);
1464
+ }
1465
+ },
1466
+ /* .user_data = */ &alloc_deps,
1467
+ };
1468
+
1469
+ for (int i = 0; i < sched->n_splits; i++) {
1470
+ struct ggml_backend_sched_split * split = &sched->splits[i];
1471
+ split->graph = ggml_graph_view(graph, split->i_start, split->i_end);
1472
+
1473
+ ggml_backend_graph_optimize(sched->backends[split->backend_id], &split->graph, &opt_params);
1474
+ }
1475
+
1476
+ // each dep is added to graph_copy as a GGML_OP_NONE node with the kept tensors as srcs
1477
+ int n_dep_nodes = 0;
1478
+ for (const auto & it : alloc_deps) {
1479
+ n_dep_nodes += (it.second.size() + GGML_MAX_SRC - 1) / GGML_MAX_SRC;
1480
+ }
1481
+
1482
+ int total_inputs = sched->n_graph_inputs;
1483
+ for (int i = 0; i < sched->n_splits; i++) {
1484
+ total_inputs += sched->splits[i].n_inputs;
1485
+ }
1486
+ int graph_size = std::max(graph->n_nodes, graph->n_leafs) + total_inputs * 2 * sched->n_copies + n_dep_nodes;
1487
+
1488
+ // remember the actual graph_size for performing reallocation checks later [GGML_SCHED_DEBUG_REALLOC]
1489
+ sched->debug_prev_graph_size = sched->debug_graph_size;
1490
+ sched->debug_graph_size = graph_size;
1491
+
1492
+ if (sched->graph.size < graph_size) {
1493
+ sched->graph.size = graph_size;
1494
+ sched->graph.nodes = (ggml_tensor **) realloc(sched->graph.nodes, graph_size * sizeof(struct ggml_tensor *));
1495
+ sched->graph.leafs = (ggml_tensor **) realloc(sched->graph.leafs, graph_size * sizeof(struct ggml_tensor *));
1496
+ GGML_ASSERT(sched->graph.nodes != NULL);
1497
+ GGML_ASSERT(sched->graph.leafs != NULL);
1498
+ }
1499
+ sched->graph.n_nodes = 0;
1500
+ sched->graph.n_leafs = 0;
1501
+
1502
+ struct ggml_cgraph * graph_copy = &sched->graph;
1503
+
1504
+ int n_dep_nodes_added = 0;
1505
+
1506
+ for (int i = 0; i < sched->n_splits; i++) {
1507
+ struct ggml_backend_sched_split * split = &sched->splits[i];
1508
+
1509
+ // add inputs to the graph copy so that they are allocated by ggml-alloc at the start of the split
1510
+ for (int j = 0; j < split->n_inputs; j++) {
1511
+ assert(graph_copy->size > (graph_copy->n_nodes + 1));
1512
+
1513
+ struct ggml_tensor * input = split->inputs[j];
1514
+ const size_t input_id = hash_id(input);
1515
+ struct ggml_tensor * input_cpy = tensor_id_copy(input_id, split->backend_id, sched->cur_copy);
1516
+
1517
+ // add a dependency to the input source so that it is not freed before the copy is done
1518
+ struct ggml_tensor * input_dep = ggml_view_tensor(sched->ctx, input);
1519
+ input_dep->src[0] = input;
1520
+ sched->node_backend_ids[graph_copy->n_nodes] = sched->hv_tensor_backend_ids[input_id];
1521
+ graph_copy->nodes[graph_copy->n_nodes++] = input_dep;
1522
+
1523
+ // add a dependency to the input copy so that it is allocated at the start of the split
1524
+ sched->node_backend_ids[graph_copy->n_nodes] = split->backend_id;
1525
+ graph_copy->nodes[graph_copy->n_nodes++] = input_cpy;
1526
+ }
1527
+
1528
+ for (int j = split->i_start; j < split->i_end; j++) {
1529
+ assert(graph_copy->size > graph_copy->n_nodes);
1530
+ sched->node_backend_ids[graph_copy->n_nodes] = tensor_backend_id(graph->nodes[j]);
1531
+ graph_copy->nodes[graph_copy->n_nodes++] = graph->nodes[j];
1532
+
1533
+ if (alloc_deps.empty()) {
1534
+ continue;
1535
+ }
1536
+
1537
+ // add a dependency node so that the kept tensors are not freed before this node is computed
1538
+ auto it = alloc_deps.find(graph->nodes[j]);
1539
+ if (it != alloc_deps.end()) {
1540
+ const std::vector<ggml_tensor *> & keep = it->second;
1541
+ for (size_t k = 0; k < keep.size(); k += GGML_MAX_SRC) {
1542
+ struct ggml_tensor * dep = ggml_view_tensor(sched->ctx, keep[k]);
1543
+ for (size_t s = 0; s < GGML_MAX_SRC && k + s < keep.size(); s++) {
1544
+ dep->src[s] = keep[k + s];
1545
+ }
1546
+ assert(graph_copy->size > graph_copy->n_nodes);
1547
+ sched->node_backend_ids[graph_copy->n_nodes] = split->backend_id;
1548
+ graph_copy->nodes[graph_copy->n_nodes++] = dep;
1549
+ n_dep_nodes_added++;
1550
+ }
1551
+ }
1552
+ }
1553
+ }
1554
+
1555
+ // a mismatch means a backend added a dep with an `until` tensor that is not a node of the optimized graph
1556
+ GGML_ASSERT(n_dep_nodes_added == n_dep_nodes);
1557
+
1558
+ if (sched->n_copies > 1) {
1559
+ // add input copies as leafs so that they are allocated first
1560
+ for (int i = 0; i < sched->n_graph_inputs; i++) {
1561
+ struct ggml_tensor * input = sched->graph_inputs[i];
1562
+ size_t id = hash_id(input);
1563
+ int backend_id = tensor_backend_id(input);
1564
+ for (int c = 0; c < sched->n_copies; c++) {
1565
+ struct ggml_tensor * input_cpy = tensor_id_copy(id, backend_id, c);
1566
+ sched->leaf_backend_ids[graph_copy->n_leafs] = backend_id;
1567
+ assert(graph_copy->size > graph_copy->n_leafs);
1568
+ graph_copy->leafs[graph_copy->n_leafs++] = input_cpy;
1569
+ }
1570
+ }
1571
+
1572
+ for (int i = 0; i < sched->n_splits; i++) {
1573
+ struct ggml_backend_sched_split * split = &sched->splits[i];
1574
+ int backend_id = split->backend_id;
1575
+ for (int j = 0; j < split->n_inputs; j++) {
1576
+ struct ggml_tensor * input = split->inputs[j];
1577
+ size_t id = hash_id(input);
1578
+ for (int c = 0; c < sched->n_copies; c++) {
1579
+ struct ggml_tensor * input_cpy = tensor_id_copy(id, backend_id, c);
1580
+ sched->leaf_backend_ids[graph_copy->n_leafs] = backend_id;
1581
+ assert(graph_copy->size > graph_copy->n_leafs);
1582
+ graph_copy->leafs[graph_copy->n_leafs++] = input_cpy;
1583
+ }
1584
+ }
1585
+ }
1586
+ }
1587
+
1588
+ // add leafs from the original graph
1589
+ for (int i = 0; i < graph->n_leafs; i++) {
1590
+ struct ggml_tensor * leaf = graph->leafs[i];
1591
+ sched->leaf_backend_ids[graph_copy->n_leafs] = tensor_backend_id(leaf);
1592
+ assert(graph_copy->size > graph_copy->n_leafs);
1593
+ graph_copy->leafs[graph_copy->n_leafs++] = leaf;
1594
+ }
1595
+
1596
+ // set ids for all splits
1597
+ for (int i = 0; i < sched->n_splits; ++i) {
1598
+ sched->splits[i].graph.uid = ggml_graph_next_uid();
1599
+ }
1600
+ }
1601
+
1602
+ static bool ggml_backend_sched_alloc_splits(ggml_backend_sched_t sched) {
1603
+ bool backend_ids_changed = false;
1604
+ for (int i = 0; i < sched->graph.n_nodes; i++) {
1605
+ if (sched->node_backend_ids[i] != sched->prev_node_backend_ids[i] &&
1606
+ sched->bufts[sched->node_backend_ids[i]] != sched->bufts[sched->prev_node_backend_ids[i]]) {
1607
+ backend_ids_changed = true;
1608
+ break;
1609
+ }
1610
+ }
1611
+ if (!backend_ids_changed) {
1612
+ for (int i = 0; i < sched->graph.n_leafs; i++) {
1613
+ if (sched->leaf_backend_ids[i] != sched->prev_leaf_backend_ids[i] &&
1614
+ sched->bufts[sched->leaf_backend_ids[i]] != sched->bufts[sched->prev_leaf_backend_ids[i]]) {
1615
+ backend_ids_changed = true;
1616
+ break;
1617
+ }
1618
+ }
1619
+ }
1620
+
1621
+ // allocate graph
1622
+ if (backend_ids_changed || !ggml_gallocr_alloc_graph(sched->galloc, &sched->graph)) {
1623
+ #ifndef NDEBUG
1624
+ GGML_LOG_DEBUG("%s: failed to allocate graph, reserving (backend_ids_changed = %d)\n", __func__, backend_ids_changed);
1625
+ #endif
1626
+
1627
+ if (sched->debug_realloc > 0) {
1628
+ // we are interested only in situations where the graph was reallocated even though its size remained the same [GGML_SCHED_DEBUG_REALLOC]
1629
+ // example: https://github.com/ggml-org/llama.cpp/pull/17143
1630
+ const bool unexpected = !backend_ids_changed && sched->debug_prev_graph_size == sched->debug_graph_size;
1631
+
1632
+ if (unexpected || sched->debug_realloc > 1) {
1633
+ GGML_ABORT("%s: unexpected graph reallocation (graph size = %d, nodes = %d, leafs = %d), debug_realloc = %d\n", __func__,
1634
+ sched->debug_graph_size, sched->graph.n_nodes, sched->graph.n_leafs, sched->debug_realloc);
1635
+ }
1636
+ }
1637
+
1638
+ // the re-allocation may cause the split inputs to be moved to a different address
1639
+ // synchronize without ggml_backend_sched_synchronize to avoid changing cur_copy
1640
+ for (int i = 0; i < sched->n_backends; i++) {
1641
+ ggml_backend_synchronize(sched->backends[i]);
1642
+ }
1643
+
1644
+ ggml_gallocr_reserve_n(sched->galloc, &sched->graph, sched->node_backend_ids, sched->leaf_backend_ids);
1645
+ if (!ggml_gallocr_alloc_graph(sched->galloc, &sched->graph)) {
1646
+ GGML_LOG_ERROR("%s: failed to allocate graph\n", __func__);
1647
+ return false;
1648
+ }
1649
+ }
1650
+
1651
+ return true;
1652
+ }
1653
+
1654
+ static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t sched) {
1655
+ GGML_ASSERT(sched);
1656
+ struct ggml_backend_sched_split * splits = sched->splits;
1657
+
1658
+ ggml_tensor * prev_ids_tensor = nullptr;
1659
+ std::vector<int32_t> ids;
1660
+ std::vector<ggml_bitset_t> used_ids;
1661
+
1662
+ int prev_backend_id = -1;
1663
+
1664
+ for (int split_id = 0; split_id < sched->n_splits; split_id++) {
1665
+ struct ggml_backend_sched_split * split = &splits[split_id];
1666
+ int split_backend_id = split->backend_id;
1667
+ ggml_backend_t split_backend = sched->backends[split_backend_id];
1668
+
1669
+ // ensure the previous split's async work has completed before we start
1670
+ // this split, the allocator may have reused buffer regions across splits
1671
+ if (split->n_inputs == 0 && prev_backend_id >= 0 && prev_backend_id != split_backend_id) {
1672
+ if (sched->events[prev_backend_id][sched->cur_copy] != NULL) {
1673
+ ggml_backend_event_synchronize(sched->events[prev_backend_id][sched->cur_copy]);
1674
+ } else {
1675
+ ggml_backend_synchronize(sched->backends[prev_backend_id]);
1676
+ }
1677
+ }
1678
+
1679
+ // copy the input tensors to the split backend
1680
+ for (int input_id = 0; input_id < split->n_inputs; input_id++) {
1681
+ ggml_backend_t input_backend = ggml_backend_sched_get_tensor_backend(sched, split->inputs[input_id]);
1682
+ struct ggml_tensor * input = split->inputs[input_id];
1683
+ struct ggml_tensor * input_cpy = tensor_copy(input, split_backend_id, sched->cur_copy);
1684
+
1685
+ if (input->flags & GGML_TENSOR_FLAG_INPUT) {
1686
+ // inputs from the user must be copied immediately to prevent the user overwriting the data before the copy is done
1687
+ if (sched->events[split_backend_id][sched->cur_copy] != NULL) {
1688
+ ggml_backend_event_synchronize(sched->events[split_backend_id][sched->cur_copy]);
1689
+ } else {
1690
+ ggml_backend_synchronize(split_backend);
1691
+ }
1692
+ ggml_backend_tensor_copy(input, input_cpy);
1693
+ } else {
1694
+ // wait for the split backend to finish using the input before overwriting it
1695
+ if (sched->events[split_backend_id][sched->cur_copy] != NULL) {
1696
+ ggml_backend_event_wait(split_backend, sched->events[split_backend_id][sched->cur_copy]);
1697
+ } else {
1698
+ ggml_backend_synchronize(split_backend);
1699
+ }
1700
+
1701
+ // when offloading MoE weights, we can reduce the amount of data copied by copying only the experts that are used
1702
+ ggml_tensor * node = split->graph.nodes[0];
1703
+ if (split->graph.n_nodes > 0 &&
1704
+ ggml_backend_buffer_get_usage(input->buffer) == GGML_BACKEND_BUFFER_USAGE_WEIGHTS &&
1705
+ ggml_backend_buffer_is_host(input->buffer) && (
1706
+ (node->src[0] == input_cpy && node->op == GGML_OP_MUL_MAT_ID)
1707
+ //|| (node->src[1] == input_cpy && node->op == GGML_OP_ADD_ID) /* GGML_OP_ADD_ID weights are small and not worth splitting */
1708
+ )) {
1709
+
1710
+ const int64_t n_expert = node->op == GGML_OP_MUL_MAT_ID ? input->ne[2] : input->ne[1];
1711
+ const size_t expert_size = node->op == GGML_OP_MUL_MAT_ID ? input->nb[2] : input->nb[1];
1712
+
1713
+ ggml_backend_synchronize(input_backend);
1714
+
1715
+ // get the ids
1716
+ ggml_tensor * ids_tensor = node->src[2];
1717
+ ggml_backend_t ids_backend = split_backend;
1718
+
1719
+ // if the ids tensor is also an input of the split, it may not have been copied yet to the split backend
1720
+ // in that case, we use the original ids tensor
1721
+ for (int i = input_id + 1; i < split->n_inputs; i++) {
1722
+ if (ids_tensor == tensor_copy(split->inputs[i], split_backend_id, sched->cur_copy)) {
1723
+ ids_tensor = split->inputs[i];
1724
+ ids_backend = ggml_backend_sched_get_tensor_backend(sched, split->inputs[i]);
1725
+ break;
1726
+ }
1727
+ }
1728
+
1729
+ if (ids_tensor != prev_ids_tensor) {
1730
+ ids.resize(ggml_nbytes(ids_tensor) / sizeof(int32_t));
1731
+ ggml_backend_tensor_get_async(ids_backend, ids_tensor, ids.data(), 0, ggml_nbytes(ids_tensor));
1732
+ ggml_backend_synchronize(ids_backend);
1733
+
1734
+ // find the used experts
1735
+ used_ids.clear();
1736
+ used_ids.resize(ggml_bitset_size(n_expert));
1737
+ for (int64_t i1 = 0; i1 < ids_tensor->ne[1]; i1++) {
1738
+ for (int64_t i0 = 0; i0 < ids_tensor->ne[0]; i0++) {
1739
+ int32_t id = ids[i1 * ids_tensor->nb[1]/sizeof(int32_t) + i0 * ids_tensor->nb[0]/sizeof(int32_t)];
1740
+ GGML_ASSERT(id >= 0 && id < n_expert);
1741
+ ggml_bitset_set(used_ids.data(), id);
1742
+ }
1743
+ }
1744
+
1745
+ prev_ids_tensor = ids_tensor;
1746
+ }
1747
+
1748
+ // group consecutive experts and copy them together
1749
+ auto copy_experts = [&](int32_t first_id, int32_t last_id) {
1750
+ const size_t expert_offset = first_id * expert_size;
1751
+ const size_t expert_size_copy = (last_id - first_id + 1) * expert_size;
1752
+ const size_t padding = std::min<size_t>(expert_size, 512);
1753
+ const size_t padding_end = last_id < n_expert - 1 ? padding : 0;
1754
+
1755
+ ggml_backend_tensor_set_async(split_backend,
1756
+ input_cpy,
1757
+ (const uint8_t *)input->data + expert_offset, expert_offset,
1758
+ // copy a bit extra at the to ensure there are no NaNs in the padding of the last expert
1759
+ // this is necessary for MMQ in the CUDA backend
1760
+ expert_size_copy + padding_end);
1761
+ };
1762
+
1763
+ int id = 0;
1764
+ while (!ggml_bitset_get(used_ids.data(), id)) {
1765
+ id++;
1766
+ }
1767
+ int32_t first_id = id;
1768
+ int32_t last_id = first_id;
1769
+
1770
+ for (++id; id < n_expert; ++id) {
1771
+ if (!ggml_bitset_get(used_ids.data(), id)) {
1772
+ continue;
1773
+ }
1774
+
1775
+ if (id == last_id + 1) {
1776
+ last_id = id;
1777
+ continue;
1778
+ }
1779
+
1780
+ copy_experts(first_id, last_id);
1781
+
1782
+ first_id = id;
1783
+ last_id = id;
1784
+ }
1785
+ copy_experts(first_id, last_id);
1786
+ } else {
1787
+ // try async copy, but if not possible, we can still use a sync copy without synchronizing the dst backend, since we handle the synchronization here with multiple copies and events
1788
+ // TODO: add public function to facilitate this, since applications do not have direct access to the backend interface
1789
+ if (!split_backend->iface.cpy_tensor_async || !split_backend->iface.cpy_tensor_async(input_backend, split_backend, input, input_cpy)) {
1790
+ ggml_backend_synchronize(input_backend);
1791
+ if (sched->events[split_backend_id][sched->cur_copy] != NULL) {
1792
+ ggml_backend_event_synchronize(sched->events[split_backend_id][sched->cur_copy]);
1793
+ } else {
1794
+ ggml_backend_synchronize(split_backend);
1795
+ }
1796
+ ggml_backend_tensor_copy(input, input_cpy);
1797
+ }
1798
+ }
1799
+ }
1800
+ }
1801
+
1802
+ if (!sched->callback_eval) {
1803
+ enum ggml_status ec = ggml_backend_graph_compute_async(split_backend, &split->graph);
1804
+ if (ec != GGML_STATUS_SUCCESS) {
1805
+ return ec;
1806
+ }
1807
+ } else {
1808
+ // similar to ggml_backend_compare_graph_backend
1809
+ for (int j0 = 0; j0 < split->graph.n_nodes; j0++) {
1810
+ struct ggml_tensor * t = split->graph.nodes[j0];
1811
+
1812
+ // check if the user needs data from this node
1813
+ bool need = sched->callback_eval(t, true, sched->callback_eval_user_data);
1814
+
1815
+ int j1 = j0;
1816
+
1817
+ // determine the range [j0, j1] of nodes that can be computed together
1818
+ while (!need && j1 < split->graph.n_nodes - 1) {
1819
+ t = split->graph.nodes[++j1];
1820
+ need = sched->callback_eval(t, true, sched->callback_eval_user_data);
1821
+ }
1822
+
1823
+ struct ggml_cgraph gv = ggml_graph_view(&split->graph, j0, j1 + 1);
1824
+
1825
+ enum ggml_status ec = ggml_backend_graph_compute_async(split_backend, &gv);
1826
+ if (ec != GGML_STATUS_SUCCESS) {
1827
+ return ec;
1828
+ }
1829
+
1830
+ // TODO: pass backend to the callback, then the user can decide if they want to synchronize
1831
+ ggml_backend_synchronize(split_backend);
1832
+
1833
+ if (need && !sched->callback_eval(t, false, sched->callback_eval_user_data)) {
1834
+ break;
1835
+ }
1836
+
1837
+ j0 = j1;
1838
+ }
1839
+ }
1840
+
1841
+ // record the event of this split
1842
+ if (sched->events[split_backend_id][sched->cur_copy] != NULL) {
1843
+ ggml_backend_event_record(sched->events[split_backend_id][sched->cur_copy], split_backend);
1844
+ }
1845
+
1846
+ prev_backend_id = split_backend_id;
1847
+ }
1848
+
1849
+ return GGML_STATUS_SUCCESS;
1850
+ }
1851
+
1852
+ ggml_backend_sched_t ggml_backend_sched_new(
1853
+ ggml_backend_t * backends,
1854
+ ggml_backend_buffer_type_t * bufts,
1855
+ int n_backends,
1856
+ size_t graph_size,
1857
+ bool parallel,
1858
+ bool op_offload) {
1859
+ GGML_ASSERT(n_backends > 0);
1860
+ GGML_ASSERT(n_backends <= GGML_SCHED_MAX_BACKENDS);
1861
+ GGML_ASSERT(ggml_backend_dev_type(ggml_backend_get_device(backends[n_backends - 1])) == GGML_BACKEND_DEVICE_TYPE_CPU);
1862
+
1863
+ struct ggml_backend_sched * sched = (ggml_backend_sched *) calloc(1, sizeof(struct ggml_backend_sched));
1864
+
1865
+ const char * GGML_SCHED_DEBUG = getenv("GGML_SCHED_DEBUG");
1866
+ sched->debug = GGML_SCHED_DEBUG ? atoi(GGML_SCHED_DEBUG) : 0;
1867
+
1868
+ sched->debug_realloc = 0;
1869
+ #ifdef GGML_SCHED_NO_REALLOC
1870
+ sched->debug_realloc = 1;
1871
+ #endif
1872
+ const char * GGML_SCHED_DEBUG_REALLOC = getenv("GGML_SCHED_DEBUG_REALLOC");
1873
+ sched->debug_realloc = GGML_SCHED_DEBUG_REALLOC ? atoi(GGML_SCHED_DEBUG_REALLOC) : sched->debug_realloc;
1874
+
1875
+ sched->n_backends = n_backends;
1876
+ sched->n_copies = parallel ? GGML_SCHED_MAX_COPIES : 1;
1877
+
1878
+ // initialize hash table
1879
+ // FIXME: needs to be size*2 to account for leafs (do it in graph_split instead)
1880
+ sched->hash_set = ggml_hash_set_new(graph_size);
1881
+ sched->hv_tensor_backend_ids = (int *) malloc(sched->hash_set.size * sizeof(sched->hv_tensor_backend_ids[0]));
1882
+ sched->hv_tensor_copies = (ggml_tensor **) malloc(sched->hash_set.size * sched->n_backends * sched->n_copies * sizeof(struct ggml_tensor *));
1883
+
1884
+ const size_t ggml_sched_max_splits = graph_size; // at most there is one split for each node in the graph
1885
+ const size_t nodes_size = graph_size + ggml_sched_max_splits*GGML_SCHED_MAX_SPLIT_INPUTS*2;
1886
+ sched->node_backend_ids = (int *) calloc(nodes_size, sizeof(sched->node_backend_ids[0]));
1887
+ sched->leaf_backend_ids = (int *) calloc(nodes_size, sizeof(sched->leaf_backend_ids[0]));
1888
+ sched->prev_node_backend_ids = (int *) calloc(nodes_size, sizeof(sched->prev_node_backend_ids[0]));
1889
+ sched->prev_leaf_backend_ids = (int *) calloc(nodes_size, sizeof(sched->prev_leaf_backend_ids[0]));
1890
+
1891
+ sched->debug_graph_size = 0;
1892
+ sched->debug_prev_graph_size = 0;
1893
+
1894
+ sched->context_buffer_size = ggml_sched_max_splits*GGML_SCHED_MAX_SPLIT_INPUTS*2*sizeof(struct ggml_tensor) + ggml_graph_overhead_custom(graph_size, false);
1895
+ sched->context_buffer = (char *) malloc(sched->context_buffer_size);
1896
+
1897
+ const int initial_splits_capacity = 16;
1898
+ sched->splits = (ggml_backend_sched_split *) calloc(initial_splits_capacity, sizeof(sched->splits[0]));
1899
+ sched->splits_capacity = initial_splits_capacity;
1900
+
1901
+ sched->graph_inputs_capacity = GGML_SCHED_MAX_SPLIT_INPUTS;
1902
+ sched->graph_inputs = (struct ggml_tensor **) calloc(sched->graph_inputs_capacity, sizeof(struct ggml_tensor *));
1903
+
1904
+ for (int b = 0; b < n_backends; b++) {
1905
+ sched->backends[b] = backends[b];
1906
+ sched->bufts[b] = bufts ? bufts[b] : ggml_backend_get_default_buffer_type(backends[b]);
1907
+ GGML_ASSERT(ggml_backend_supports_buft(backends[b], sched->bufts[b]));
1908
+
1909
+ if (sched->n_copies > 1) {
1910
+ for (int c = 0; c < sched->n_copies; c++) {
1911
+ sched->events[b][c] = ggml_backend_event_new(backends[b]->device);
1912
+ }
1913
+ }
1914
+ }
1915
+
1916
+ sched->galloc = ggml_gallocr_new_n(sched->bufts, n_backends);
1917
+ sched->op_offload = op_offload;
1918
+
1919
+ ggml_backend_sched_reset(sched);
1920
+
1921
+ return sched;
1922
+ }
1923
+
1924
+ void ggml_backend_sched_free(ggml_backend_sched_t sched) {
1925
+ if (sched == NULL) {
1926
+ return;
1927
+ }
1928
+ for (int b = 0; b < sched->n_backends; b++) {
1929
+ for (int c = 0; c < sched->n_copies; c++) {
1930
+ ggml_backend_event_free(sched->events[b][c]);
1931
+ }
1932
+ }
1933
+ ggml_gallocr_free(sched->galloc);
1934
+ ggml_free(sched->ctx);
1935
+ ggml_hash_set_free(&sched->hash_set);
1936
+ for (int i = 0; i < sched->splits_capacity; i++) {
1937
+ free(sched->splits[i].inputs);
1938
+ }
1939
+ free(sched->splits);
1940
+ free(sched->graph_inputs);
1941
+ free(sched->hv_tensor_backend_ids);
1942
+ free(sched->hv_tensor_copies);
1943
+ free(sched->node_backend_ids);
1944
+ free(sched->leaf_backend_ids);
1945
+ free(sched->prev_node_backend_ids);
1946
+ free(sched->prev_leaf_backend_ids);
1947
+ free(sched->context_buffer);
1948
+ free(sched->graph.nodes);
1949
+ free(sched->graph.leafs);
1950
+ free(sched);
1951
+ }
1952
+
1953
+ void ggml_backend_sched_reset(ggml_backend_sched_t sched) {
1954
+ GGML_ASSERT(sched);
1955
+ // reset state for the next run
1956
+ if (!sched->is_reset) {
1957
+ ggml_hash_set_reset(&sched->hash_set);
1958
+ memset(sched->hv_tensor_backend_ids, -1, sched->hash_set.size * sizeof(sched->hv_tensor_backend_ids[0]));
1959
+ memset(sched->hv_tensor_copies, 0, sched->hash_set.size * sched->n_backends * sched->n_copies * sizeof(struct ggml_tensor *));
1960
+ sched->is_reset = true;
1961
+ }
1962
+ sched->is_alloc = false;
1963
+ }
1964
+
1965
+ void ggml_backend_sched_reserve_size(ggml_backend_sched_t sched, struct ggml_cgraph * measure_graph, size_t * sizes) {
1966
+ GGML_ASSERT(sched);
1967
+ GGML_ASSERT((int)sched->hash_set.size >= measure_graph->n_nodes + measure_graph->n_leafs);
1968
+ GGML_ASSERT(sizes);
1969
+
1970
+ ggml_backend_sched_reset(sched);
1971
+
1972
+ ggml_backend_sched_synchronize(sched);
1973
+
1974
+ ggml_backend_sched_split_graph(sched, measure_graph);
1975
+
1976
+ ggml_gallocr_reserve_n_size(sched->galloc, &sched->graph, sched->node_backend_ids, sched->leaf_backend_ids, sizes);
1977
+ }
1978
+
1979
+ bool ggml_backend_sched_reserve(ggml_backend_sched_t sched, struct ggml_cgraph * measure_graph) {
1980
+ GGML_ASSERT(sched);
1981
+ GGML_ASSERT((int)sched->hash_set.size >= measure_graph->n_nodes + measure_graph->n_leafs);
1982
+
1983
+ ggml_backend_sched_synchronize(sched);
1984
+
1985
+ ggml_backend_sched_split_graph(sched, measure_graph);
1986
+
1987
+ if (!ggml_gallocr_reserve_n(sched->galloc, &sched->graph, sched->node_backend_ids, sched->leaf_backend_ids)) {
1988
+ return false;
1989
+ }
1990
+
1991
+ ggml_backend_sched_reset(sched);
1992
+
1993
+ return true;
1994
+ }
1995
+
1996
+ bool ggml_backend_sched_alloc_graph(ggml_backend_sched_t sched, struct ggml_cgraph * graph) {
1997
+ GGML_ASSERT(sched);
1998
+ GGML_ASSERT((int)sched->hash_set.size >= graph->n_nodes + graph->n_leafs);
1999
+ GGML_ASSERT(!sched->is_alloc);
2000
+
2001
+ sched->cur_copy = sched->next_copy;
2002
+ sched->next_copy = (sched->next_copy + 1) % sched->n_copies;
2003
+
2004
+ ggml_backend_sched_split_graph(sched, graph);
2005
+
2006
+ if (!ggml_backend_sched_alloc_splits(sched)) {
2007
+ return false;
2008
+ }
2009
+
2010
+ sched->is_alloc = true;
2011
+
2012
+ return true;
2013
+ }
2014
+
2015
+ enum ggml_status ggml_backend_sched_graph_compute(ggml_backend_sched_t sched, struct ggml_cgraph * graph) {
2016
+ enum ggml_status err = ggml_backend_sched_graph_compute_async(sched, graph);
2017
+ ggml_backend_sched_synchronize(sched);
2018
+ return err;
2019
+ }
2020
+
2021
+ enum ggml_status ggml_backend_sched_graph_compute_async(ggml_backend_sched_t sched, struct ggml_cgraph * graph) {
2022
+ GGML_ASSERT(sched);
2023
+ if (!sched->is_reset && !sched->is_alloc) {
2024
+ ggml_backend_sched_reset(sched);
2025
+ }
2026
+
2027
+ if (!sched->is_alloc) {
2028
+ if (!ggml_backend_sched_alloc_graph(sched, graph)) {
2029
+ return GGML_STATUS_ALLOC_FAILED;
2030
+ }
2031
+ }
2032
+
2033
+ return ggml_backend_sched_compute_splits(sched);
2034
+ }
2035
+
2036
+ void ggml_backend_sched_synchronize(ggml_backend_sched_t sched) {
2037
+ GGML_ASSERT(sched);
2038
+ for (int i = 0; i < sched->n_backends; i++) {
2039
+ ggml_backend_synchronize(sched->backends[i]);
2040
+ }
2041
+ if (!sched->is_alloc) {
2042
+ // if the graph is not already allocated, always use copy 0 after a synchronization
2043
+ // this ensures that during generation the same copy is used every time,
2044
+ // which avoids changes in the graph that could cause CUDA or other graphs to be disabled
2045
+ sched->next_copy = 0;
2046
+ }
2047
+ }
2048
+
2049
+ void ggml_backend_sched_set_eval_callback(ggml_backend_sched_t sched, ggml_backend_sched_eval_callback callback, void * user_data) {
2050
+ GGML_ASSERT(sched);
2051
+ sched->callback_eval = callback;
2052
+ sched->callback_eval_user_data = user_data;
2053
+ }
2054
+
2055
+ int ggml_backend_sched_get_n_splits(ggml_backend_sched_t sched) {
2056
+ GGML_ASSERT(sched);
2057
+ return sched->n_splits;
2058
+ }
2059
+
2060
+ int ggml_backend_sched_get_n_copies(ggml_backend_sched_t sched) {
2061
+ GGML_ASSERT(sched);
2062
+ return sched->n_copies;
2063
+ }
2064
+
2065
+ int ggml_backend_sched_get_n_backends(ggml_backend_sched_t sched) {
2066
+ GGML_ASSERT(sched);
2067
+ return sched->n_backends;
2068
+ }
2069
+
2070
+ ggml_backend_t ggml_backend_sched_get_backend(ggml_backend_sched_t sched, int i) {
2071
+ GGML_ASSERT(sched);
2072
+ GGML_ASSERT(i >= 0 && i < sched->n_backends);
2073
+ return sched->backends[i];
2074
+ }
2075
+
2076
+ ggml_backend_buffer_type_t ggml_backend_sched_get_buffer_type(ggml_backend_sched_t sched, ggml_backend_t backend) {
2077
+ GGML_ASSERT(sched);
2078
+ int backend_index = ggml_backend_sched_backend_id(sched, backend);
2079
+ GGML_ASSERT(backend_index >= 0 && backend_index < sched->n_backends);
2080
+
2081
+ return sched->bufts[backend_index];
2082
+ }
2083
+
2084
+ size_t ggml_backend_sched_get_buffer_size(ggml_backend_sched_t sched, ggml_backend_t backend) {
2085
+ GGML_ASSERT(sched);
2086
+ int backend_index = ggml_backend_sched_backend_id(sched, backend);
2087
+ GGML_ASSERT(backend_index >= 0 && backend_index < sched->n_backends);
2088
+
2089
+ return ggml_gallocr_get_buffer_size(sched->galloc, backend_index);
2090
+ }
2091
+
2092
+ void ggml_backend_sched_set_tensor_backend(ggml_backend_sched_t sched, struct ggml_tensor * node, ggml_backend_t backend) {
2093
+ GGML_ASSERT(sched);
2094
+ int backend_index = ggml_backend_sched_backend_id(sched, backend);
2095
+ GGML_ASSERT(backend_index >= 0 && backend_index < sched->n_backends);
2096
+ tensor_backend_id(node) = backend_index;
2097
+ SET_CAUSE(node, "usr");
2098
+ sched->is_reset = false;
2099
+ }
2100
+
2101
+ ggml_backend_t ggml_backend_sched_get_tensor_backend(ggml_backend_sched_t sched, struct ggml_tensor * node) {
2102
+ GGML_ASSERT(sched);
2103
+ int backend_index = tensor_backend_id(node);
2104
+ if (backend_index == -1) {
2105
+ return NULL;
2106
+ }
2107
+ return sched->backends[backend_index];
2108
+ }
2109
+
2110
+ // utils
2111
+
2112
+ bool ggml_op_alloc_size_may_expand(enum ggml_op op) {
2113
+ switch (op) {
2114
+ case GGML_OP_FLASH_ATTN_EXT:
2115
+ case GGML_OP_MUL_MAT:
2116
+ case GGML_OP_MUL_MAT_ID:
2117
+ case GGML_OP_CUMSUM:
2118
+ case GGML_OP_ARGSORT:
2119
+ case GGML_OP_TOP_K:
2120
+ return true;
2121
+ default:
2122
+ return false;
2123
+ }
2124
+ }
2125
+
2126
+ enum ggml_status ggml_backend_view_init(struct ggml_tensor * tensor) {
2127
+ GGML_ASSERT(tensor);
2128
+ GGML_ASSERT(tensor->buffer == NULL);
2129
+ GGML_ASSERT(tensor->view_src != NULL);
2130
+ GGML_ASSERT(tensor->view_src->buffer != NULL);
2131
+ GGML_ASSERT(tensor->view_src->data != NULL);
2132
+
2133
+ tensor->buffer = tensor->view_src->buffer;
2134
+ tensor->data = (char *)tensor->view_src->data + tensor->view_offs;
2135
+ return ggml_backend_buffer_init_tensor(tensor->buffer, tensor);
2136
+ }
2137
+
2138
+ enum ggml_status ggml_backend_tensor_alloc(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, void * addr) {
2139
+ GGML_ASSERT(tensor);
2140
+ GGML_ASSERT(tensor->buffer == NULL);
2141
+ GGML_ASSERT(tensor->data == NULL);
2142
+ GGML_ASSERT(tensor->view_src == NULL);
2143
+ GGML_ASSERT(addr >= ggml_backend_buffer_get_base(buffer));
2144
+ GGML_ASSERT(ggml_backend_buffer_is_meta(buffer) ||
2145
+ (char *) addr + ggml_backend_buffer_get_alloc_size(buffer, tensor) <=
2146
+ (char *) ggml_backend_buffer_get_base(buffer) + ggml_backend_buffer_get_size(buffer));
2147
+
2148
+ tensor->buffer = buffer;
2149
+ tensor->data = addr;
2150
+ return ggml_backend_buffer_init_tensor(buffer, tensor);
2151
+ }
2152
+
2153
+ static struct ggml_tensor * graph_copy_dup_tensor(struct ggml_hash_set hash_set, struct ggml_tensor ** node_copies,
2154
+ struct ggml_context * ctx_allocated, struct ggml_context * ctx_unallocated, struct ggml_tensor * src) {
2155
+
2156
+ GGML_ASSERT(src != NULL);
2157
+ GGML_ASSERT(src->data && "graph must be allocated");
2158
+
2159
+ size_t id = ggml_hash_insert(&hash_set, src);
2160
+ if (id == GGML_HASHSET_ALREADY_EXISTS) {
2161
+ return node_copies[ggml_hash_find(&hash_set, src)];
2162
+ }
2163
+
2164
+ struct ggml_tensor * dst = ggml_dup_tensor_layout(src->data && !src->view_src ? ctx_allocated : ctx_unallocated, src);
2165
+ if (src->view_src != NULL) {
2166
+ dst->view_src = graph_copy_dup_tensor(hash_set, node_copies, ctx_allocated, ctx_unallocated, src->view_src);
2167
+ dst->view_offs = src->view_offs;
2168
+ }
2169
+ dst->op = src->op;
2170
+ dst->flags = src->flags;
2171
+ memcpy(dst->op_params, src->op_params, sizeof(dst->op_params));
2172
+ ggml_set_name(dst, src->name);
2173
+
2174
+ // copy src
2175
+ for (int i = 0; i < GGML_MAX_SRC; i++) {
2176
+ struct ggml_tensor * s = src->src[i];
2177
+ if (s == NULL) {
2178
+ continue;
2179
+ }
2180
+ dst->src[i] = graph_copy_dup_tensor(hash_set, node_copies, ctx_allocated, ctx_unallocated, s);
2181
+ }
2182
+
2183
+ node_copies[id] = dst;
2184
+ return dst;
2185
+ }
2186
+
2187
+ static void graph_copy_init_tensor(struct ggml_hash_set * hash_set, struct ggml_tensor ** node_copies, bool * node_init, struct ggml_tensor * src) {
2188
+ size_t id = ggml_hash_find(hash_set, src);
2189
+ if (node_init[id]) {
2190
+ return;
2191
+ }
2192
+ node_init[id] = true;
2193
+
2194
+ struct ggml_tensor * dst = node_copies[id];
2195
+ if (dst->view_src != NULL) {
2196
+ graph_copy_init_tensor(hash_set, node_copies, node_init, src->view_src);
2197
+ enum ggml_status status = ggml_backend_view_init(dst);
2198
+ GGML_ASSERT(status == GGML_STATUS_SUCCESS);
2199
+ }
2200
+ else {
2201
+ ggml_backend_tensor_copy(src, dst);
2202
+ }
2203
+
2204
+ // init src
2205
+ for (int i = 0; i < GGML_MAX_SRC; i++) {
2206
+ struct ggml_tensor * s = src->src[i];
2207
+ if (s == NULL) {
2208
+ continue;
2209
+ }
2210
+ graph_copy_init_tensor(hash_set, node_copies, node_init, s);
2211
+ }
2212
+ }
2213
+
2214
+ struct ggml_backend_graph_copy ggml_backend_graph_copy(ggml_backend_t backend, struct ggml_cgraph * graph) {
2215
+ GGML_ASSERT(graph);
2216
+ struct ggml_hash_set hash_set = ggml_hash_set_new(graph->visited_hash_set.size);
2217
+ struct ggml_tensor ** node_copies = (ggml_tensor **) calloc(hash_set.size, sizeof(node_copies[0])); // NOLINT
2218
+ bool * node_init = (bool *) calloc(hash_set.size, sizeof(node_init[0]));
2219
+
2220
+ struct ggml_init_params params = {
2221
+ /* .mem_size = */ ggml_tensor_overhead()*hash_set.size + ggml_graph_overhead_custom(graph->size, false),
2222
+ /* .mem_buffer = */ NULL,
2223
+ /* .no_alloc = */ true
2224
+ };
2225
+
2226
+ struct ggml_context * ctx_allocated = ggml_init(params);
2227
+ struct ggml_context * ctx_unallocated = ggml_init(params);
2228
+
2229
+ if (ctx_allocated == NULL || ctx_unallocated == NULL) {
2230
+ GGML_LOG_ERROR("%s: failed to allocate context for graph copy\n", __func__);
2231
+ ggml_hash_set_free(&hash_set);
2232
+ free(node_copies);
2233
+ free(node_init);
2234
+ ggml_free(ctx_allocated);
2235
+ ggml_free(ctx_unallocated);
2236
+ return {
2237
+ /* .buffer = */ NULL,
2238
+ /* .ctx_allocated = */ NULL,
2239
+ /* .ctx_unallocated = */ NULL,
2240
+ /* .graph = */ NULL,
2241
+ };
2242
+ }
2243
+
2244
+ // dup nodes
2245
+ for (int i = 0; i < graph->n_nodes; i++) {
2246
+ struct ggml_tensor * node = graph->nodes[i];
2247
+ graph_copy_dup_tensor(hash_set, node_copies, ctx_allocated, ctx_unallocated, node);
2248
+ }
2249
+
2250
+ // allocate nodes
2251
+ ggml_backend_buffer_t buffer = ggml_backend_alloc_ctx_tensors(ctx_allocated, backend);
2252
+ if (buffer == NULL) {
2253
+ GGML_LOG_ERROR("%s: failed to allocate buffer for graph copy\n", __func__);
2254
+ ggml_hash_set_free(&hash_set);
2255
+ free(node_copies);
2256
+ free(node_init);
2257
+ ggml_free(ctx_allocated);
2258
+ ggml_free(ctx_unallocated);
2259
+ return {
2260
+ /* .buffer = */ NULL,
2261
+ /* .ctx_allocated = */ NULL,
2262
+ /* .ctx_unallocated = */ NULL,
2263
+ /* .graph = */ NULL,
2264
+ };
2265
+ }
2266
+
2267
+ //printf("copy buffer size: %zu MB\n", ggml_backend_buffer_get_size(buffer) / 1024 / 1024);
2268
+
2269
+ // copy data and init views
2270
+ for (int i = 0; i < graph->n_nodes; i++) {
2271
+ struct ggml_tensor * node = graph->nodes[i];
2272
+ graph_copy_init_tensor(&hash_set, node_copies, node_init, node);
2273
+ }
2274
+
2275
+ // build graph copy
2276
+ struct ggml_cgraph * graph_copy = ggml_new_graph_custom(ctx_allocated, graph->size, false);
2277
+ for (int i = 0; i < graph->n_nodes; i++) {
2278
+ struct ggml_tensor * node = graph->nodes[i];
2279
+ struct ggml_tensor * node_copy = node_copies[ggml_hash_find(&hash_set, node)];
2280
+ graph_copy->nodes[i] = node_copy;
2281
+ }
2282
+ graph_copy->n_nodes = graph->n_nodes;
2283
+
2284
+ ggml_hash_set_free(&hash_set);
2285
+ free(node_copies);
2286
+ free(node_init);
2287
+
2288
+ return {
2289
+ /* .buffer = */ buffer,
2290
+ /* .ctx_allocated = */ ctx_allocated,
2291
+ /* .ctx_unallocated = */ ctx_unallocated,
2292
+ /* .graph = */ graph_copy,
2293
+ };
2294
+ }
2295
+
2296
+ void ggml_backend_graph_copy_free(struct ggml_backend_graph_copy copy) {
2297
+ ggml_backend_buffer_free(copy.buffer);
2298
+ ggml_free(copy.ctx_allocated);
2299
+ ggml_free(copy.ctx_unallocated);
2300
+ }
2301
+
2302
+ bool ggml_backend_compare_graph_backend(ggml_backend_t backend1, ggml_backend_t backend2, struct ggml_cgraph * graph, ggml_backend_eval_callback callback, void * user_data, struct ggml_tensor const * const * test_nodes, size_t num_test_nodes) {
2303
+ struct ggml_backend_graph_copy copy = ggml_backend_graph_copy(backend2, graph);
2304
+ if (copy.buffer == NULL) {
2305
+ return false;
2306
+ }
2307
+
2308
+ struct ggml_cgraph * g1 = graph;
2309
+ struct ggml_cgraph * g2 = copy.graph;
2310
+
2311
+ assert(g1->n_nodes == g2->n_nodes);
2312
+
2313
+ if (num_test_nodes != 0) {
2314
+ GGML_ASSERT(test_nodes);
2315
+ // Compute the whole graph and only test the output for specific tensors
2316
+ ggml_backend_graph_compute(backend1, g1);
2317
+ ggml_backend_graph_compute(backend2, g2);
2318
+
2319
+ bool verified = false;
2320
+ for (int i = 0; i < g1->n_nodes; i++) {
2321
+ for (size_t j = 0; j < num_test_nodes; ++j) {
2322
+ if (g1->nodes[i] == test_nodes[j]) {
2323
+ callback(i, g1->nodes[i], g2->nodes[i], user_data);
2324
+ verified = true;
2325
+ }
2326
+ }
2327
+ }
2328
+ GGML_ASSERT(verified);
2329
+ } else {
2330
+ for (int i = 0; i < g1->n_nodes; i++) {
2331
+ struct ggml_tensor * t1 = g1->nodes[i];
2332
+ struct ggml_tensor * t2 = g2->nodes[i];
2333
+
2334
+ assert(t1->op == t2->op && ggml_are_same_layout(t1, t2));
2335
+
2336
+ struct ggml_cgraph g1v = ggml_graph_view(g1, i, i + 1);
2337
+ struct ggml_cgraph g2v = ggml_graph_view(g2, i, i + 1);
2338
+
2339
+ ggml_backend_graph_compute(backend1, &g1v);
2340
+ ggml_backend_graph_compute(backend2, &g2v);
2341
+
2342
+ if (ggml_is_view_op(t1->op)) {
2343
+ continue;
2344
+ }
2345
+
2346
+ // compare results, calculate rms etc
2347
+ if (!callback(i, t1, t2, user_data)) {
2348
+ break;
2349
+ }
2350
+ }
2351
+ }
2352
+ ggml_backend_graph_copy_free(copy);
2353
+
2354
+ return true;
2355
+ }
2356
+
2357
+ // CPU backend - buffer
2358
+
2359
+ static void * ggml_backend_cpu_buffer_get_base(ggml_backend_buffer_t buffer) {
2360
+ GGML_ASSERT(buffer);
2361
+ uintptr_t data = (uintptr_t)buffer->context;
2362
+
2363
+ // align the buffer
2364
+ if (data % TENSOR_ALIGNMENT != 0) {
2365
+ data = GGML_PAD(data, TENSOR_ALIGNMENT);
2366
+ }
2367
+
2368
+ return (void *)data;
2369
+ }
2370
+
2371
+ static void ggml_backend_cpu_buffer_free_buffer(ggml_backend_buffer_t buffer) {
2372
+ GGML_ASSERT(buffer);
2373
+ ggml_aligned_free(buffer->context, buffer->size);
2374
+ }
2375
+
2376
+ static void ggml_backend_cpu_buffer_memset_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, uint8_t value, size_t offset, size_t size) {
2377
+ GGML_ASSERT(tensor);
2378
+ memset((char *)tensor->data + offset, value, size);
2379
+
2380
+ GGML_UNUSED(buffer);
2381
+ }
2382
+
2383
+ static void ggml_backend_cpu_buffer_set_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size) {
2384
+ GGML_ASSERT(tensor);
2385
+ memcpy((char *)tensor->data + offset, data, size);
2386
+
2387
+ GGML_UNUSED(buffer);
2388
+ }
2389
+
2390
+ static void ggml_backend_cpu_buffer_get_tensor(ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size) {
2391
+ GGML_ASSERT(tensor);
2392
+ memcpy(data, (const char *)tensor->data + offset, size);
2393
+
2394
+ GGML_UNUSED(buffer);
2395
+ }
2396
+
2397
+ static bool ggml_backend_cpu_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const struct ggml_tensor * src, struct ggml_tensor * dst) {
2398
+ GGML_ASSERT(src);
2399
+ if (ggml_backend_buffer_is_host(src->buffer)) {
2400
+ memcpy(dst->data, src->data, ggml_nbytes(src));
2401
+ return true;
2402
+ }
2403
+ return false;
2404
+
2405
+ GGML_UNUSED(buffer);
2406
+ }
2407
+
2408
+ static void ggml_backend_cpu_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {
2409
+ GGML_ASSERT(buffer);
2410
+ memset(buffer->context, value, buffer->size);
2411
+ }
2412
+
2413
+ static const struct ggml_backend_buffer_i ggml_backend_cpu_buffer_i = {
2414
+ /* .free_buffer = */ ggml_backend_cpu_buffer_free_buffer,
2415
+ /* .get_base = */ ggml_backend_cpu_buffer_get_base,
2416
+ /* .init_tensor = */ NULL, // no initialization required
2417
+ /* .memset_tensor = */ ggml_backend_cpu_buffer_memset_tensor,
2418
+ /* .set_tensor = */ ggml_backend_cpu_buffer_set_tensor,
2419
+ /* .get_tensor = */ ggml_backend_cpu_buffer_get_tensor,
2420
+ /* .set_tensor_2d = */ NULL,
2421
+ /* .get_tensor_2d = */ NULL,
2422
+ /* .cpy_tensor = */ ggml_backend_cpu_buffer_cpy_tensor,
2423
+ /* .clear = */ ggml_backend_cpu_buffer_clear,
2424
+ /* .reset = */ NULL,
2425
+ };
2426
+
2427
+ static const struct ggml_backend_buffer_i ggml_backend_cpu_buffer_from_ptr_i = {
2428
+ /* .free_buffer = */ NULL, // ptr is not owned by the buffer, so it does not need to be freed
2429
+ /* .get_base = */ ggml_backend_cpu_buffer_get_base,
2430
+ /* .init_tensor = */ NULL, // no initialization required
2431
+ /* .memset_tensor = */ ggml_backend_cpu_buffer_memset_tensor,
2432
+ /* .set_tensor = */ ggml_backend_cpu_buffer_set_tensor,
2433
+ /* .get_tensor = */ ggml_backend_cpu_buffer_get_tensor,
2434
+ /* .set_tensor_2d = */ NULL,
2435
+ /* .get_tensor_2d = */ NULL,
2436
+ /* .cpy_tensor = */ ggml_backend_cpu_buffer_cpy_tensor,
2437
+ /* .clear = */ ggml_backend_cpu_buffer_clear,
2438
+ /* .reset = */ NULL,
2439
+ };
2440
+
2441
+ // CPU backend buffer type
2442
+
2443
+ // this buffer type is defined here to make it available to all backends
2444
+
2445
+ static const char * ggml_backend_cpu_buffer_type_get_name(ggml_backend_buffer_type_t buft) {
2446
+ return "CPU";
2447
+
2448
+ GGML_UNUSED(buft);
2449
+ }
2450
+
2451
+ static ggml_backend_buffer_t ggml_backend_cpu_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {
2452
+ void * data = ggml_aligned_malloc(size);
2453
+
2454
+ if (data == NULL) {
2455
+ GGML_LOG_ERROR("%s: failed to allocate buffer of size %zu\n", __func__, size);
2456
+ return NULL;
2457
+ }
2458
+
2459
+ return ggml_backend_buffer_init(buft, ggml_backend_cpu_buffer_i, data, size);
2460
+ }
2461
+
2462
+ static size_t ggml_backend_cpu_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) {
2463
+ return TENSOR_ALIGNMENT;
2464
+
2465
+ GGML_UNUSED(buft);
2466
+ }
2467
+
2468
+ static bool ggml_backend_cpu_buffer_type_is_host(ggml_backend_buffer_type_t buft) {
2469
+ return true;
2470
+
2471
+ GGML_UNUSED(buft);
2472
+ }
2473
+
2474
+ ggml_backend_buffer_type_t ggml_backend_cpu_buffer_type(void) {
2475
+ static struct ggml_backend_buffer_type ggml_backend_cpu_buffer_type = {
2476
+ /* .iface = */ {
2477
+ /* .get_name = */ ggml_backend_cpu_buffer_type_get_name,
2478
+ /* .alloc_buffer = */ ggml_backend_cpu_buffer_type_alloc_buffer,
2479
+ /* .get_alignment = */ ggml_backend_cpu_buffer_type_get_alignment,
2480
+ /* .get_max_size = */ NULL, // defaults to SIZE_MAX
2481
+ /* .get_alloc_size = */ NULL, // defaults to ggml_nbytes
2482
+ /* .is_host = */ ggml_backend_cpu_buffer_type_is_host,
2483
+ },
2484
+ /* .device = */ NULL, // FIXME ggml_backend_reg_dev_get(ggml_backend_cpu_reg(), 0),
2485
+ /* .context = */ NULL,
2486
+ };
2487
+
2488
+ return &ggml_backend_cpu_buffer_type;
2489
+ }
2490
+
2491
+ static const char * ggml_backend_cpu_buffer_from_ptr_type_get_name(ggml_backend_buffer_type_t buft) {
2492
+ return "CPU_Mapped";
2493
+
2494
+ GGML_UNUSED(buft);
2495
+ }
2496
+
2497
+ static ggml_backend_buffer_type_t ggml_backend_cpu_buffer_from_ptr_type(void) {
2498
+ static struct ggml_backend_buffer_type ggml_backend_cpu_buffer_type = {
2499
+ /* .iface = */ {
2500
+ /* .get_name = */ ggml_backend_cpu_buffer_from_ptr_type_get_name,
2501
+ /* .alloc_buffer = */ ggml_backend_cpu_buffer_type_alloc_buffer,
2502
+ /* .get_alignment = */ ggml_backend_cpu_buffer_type_get_alignment,
2503
+ /* .get_max_size = */ NULL, // defaults to SIZE_MAX
2504
+ /* .get_alloc_size = */ NULL, // defaults to ggml_nbytes
2505
+ /* .is_host = */ ggml_backend_cpu_buffer_type_is_host,
2506
+ },
2507
+ /* .device = */ NULL, // FIXME ggml_backend_reg_dev_get(ggml_backend_cpu_reg(), 0),
2508
+ /* .context = */ NULL,
2509
+ };
2510
+
2511
+ return &ggml_backend_cpu_buffer_type;
2512
+ }
2513
+
2514
+ ggml_backend_buffer_t ggml_backend_cpu_buffer_from_ptr(void * ptr, size_t size) {
2515
+ GGML_ASSERT((uintptr_t)ptr % TENSOR_ALIGNMENT == 0 && "buffer pointer must be aligned");
2516
+ return ggml_backend_buffer_init(ggml_backend_cpu_buffer_from_ptr_type(), ggml_backend_cpu_buffer_from_ptr_i, ptr, size);
2517
+ }