elelem-ggml 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (1367) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE.txt +21 -0
  3. data/exe/elelem-ggml +8 -0
  4. data/ext/elelem/ggml/elelem.c +466 -0
  5. data/ext/elelem/ggml/extconf.rb +52 -0
  6. data/lib/elelem/ggml/chat_ml.rb +23 -0
  7. data/lib/elelem/ggml/gguf.rb +82 -0
  8. data/lib/elelem/ggml/provider.rb +63 -0
  9. data/lib/elelem/ggml/tokenizer.rb +95 -0
  10. data/lib/elelem/ggml/version.rb +7 -0
  11. data/lib/elelem/ggml.rb +37 -0
  12. data/vendor/ggml/AUTHORS +335 -0
  13. data/vendor/ggml/CMakeLists.txt +499 -0
  14. data/vendor/ggml/CONTRIBUTING.md +3 -0
  15. data/vendor/ggml/LICENSE +21 -0
  16. data/vendor/ggml/README.md +49 -0
  17. data/vendor/ggml/cmake/FindNCCL.cmake +36 -0
  18. data/vendor/ggml/cmake/GitVars.cmake +22 -0
  19. data/vendor/ggml/cmake/common.cmake +50 -0
  20. data/vendor/ggml/cmake/ggml-config.cmake.in +216 -0
  21. data/vendor/ggml/ggml.pc.in +10 -0
  22. data/vendor/ggml/include/ggml-alloc.h +86 -0
  23. data/vendor/ggml/include/ggml-backend.h +437 -0
  24. data/vendor/ggml/include/ggml-blas.h +25 -0
  25. data/vendor/ggml/include/ggml-cann.h +123 -0
  26. data/vendor/ggml/include/ggml-cpp.h +39 -0
  27. data/vendor/ggml/include/ggml-cpu.h +152 -0
  28. data/vendor/ggml/include/ggml-cuda.h +47 -0
  29. data/vendor/ggml/include/ggml-et.h +28 -0
  30. data/vendor/ggml/include/ggml-hexagon.h +19 -0
  31. data/vendor/ggml/include/ggml-metal.h +61 -0
  32. data/vendor/ggml/include/ggml-opencl.h +26 -0
  33. data/vendor/ggml/include/ggml-openvino.h +37 -0
  34. data/vendor/ggml/include/ggml-opt.h +256 -0
  35. data/vendor/ggml/include/ggml-rpc.h +35 -0
  36. data/vendor/ggml/include/ggml-sycl.h +57 -0
  37. data/vendor/ggml/include/ggml-virtgpu.h +14 -0
  38. data/vendor/ggml/include/ggml-vulkan.h +29 -0
  39. data/vendor/ggml/include/ggml-webgpu.h +19 -0
  40. data/vendor/ggml/include/ggml-zdnn.h +17 -0
  41. data/vendor/ggml/include/ggml-zendnn.h +22 -0
  42. data/vendor/ggml/include/ggml.h +2963 -0
  43. data/vendor/ggml/include/gguf.h +211 -0
  44. data/vendor/ggml/requirements.txt +12 -0
  45. data/vendor/ggml/scripts/gen-authors.sh +9 -0
  46. data/vendor/ggml/scripts/make-release-checks.sh +96 -0
  47. data/vendor/ggml/scripts/make-release-desc.sh +69 -0
  48. data/vendor/ggml/scripts/make-release-summary.txt +33 -0
  49. data/vendor/ggml/scripts/release.sh +195 -0
  50. data/vendor/ggml/scripts/sync-llama-am.sh +167 -0
  51. data/vendor/ggml/scripts/sync-llama.last +1 -0
  52. data/vendor/ggml/scripts/sync-llama.sh +21 -0
  53. data/vendor/ggml/scripts/sync-whisper-am.sh +138 -0
  54. data/vendor/ggml/scripts/sync-whisper.last +1 -0
  55. data/vendor/ggml/scripts/sync-whisper.sh +17 -0
  56. data/vendor/ggml/src/CMakeLists.txt +634 -0
  57. data/vendor/ggml/src/ggml-alloc.c +1249 -0
  58. data/vendor/ggml/src/ggml-backend-dl.cpp +48 -0
  59. data/vendor/ggml/src/ggml-backend-dl.h +45 -0
  60. data/vendor/ggml/src/ggml-backend-impl.h +291 -0
  61. data/vendor/ggml/src/ggml-backend-meta.cpp +2511 -0
  62. data/vendor/ggml/src/ggml-backend-reg.cpp +605 -0
  63. data/vendor/ggml/src/ggml-backend.cpp +2517 -0
  64. data/vendor/ggml/src/ggml-blas/CMakeLists.txt +101 -0
  65. data/vendor/ggml/src/ggml-blas/ggml-blas.cpp +530 -0
  66. data/vendor/ggml/src/ggml-cann/CMakeLists.txt +89 -0
  67. data/vendor/ggml/src/ggml-cann/acl_tensor.cpp +195 -0
  68. data/vendor/ggml/src/ggml-cann/acl_tensor.h +349 -0
  69. data/vendor/ggml/src/ggml-cann/aclnn_ops.cpp +4479 -0
  70. data/vendor/ggml/src/ggml-cann/aclnn_ops.h +1191 -0
  71. data/vendor/ggml/src/ggml-cann/common.h +651 -0
  72. data/vendor/ggml/src/ggml-cann/ggml-cann.cpp +3070 -0
  73. data/vendor/ggml/src/ggml-common.h +1911 -0
  74. data/vendor/ggml/src/ggml-cpu/CMakeLists.txt +685 -0
  75. data/vendor/ggml/src/ggml-cpu/amx/amx.cpp +249 -0
  76. data/vendor/ggml/src/ggml-cpu/amx/amx.h +8 -0
  77. data/vendor/ggml/src/ggml-cpu/amx/common.h +115 -0
  78. data/vendor/ggml/src/ggml-cpu/amx/mmq.cpp +2511 -0
  79. data/vendor/ggml/src/ggml-cpu/amx/mmq.h +10 -0
  80. data/vendor/ggml/src/ggml-cpu/arch/arm/cpu-feats.cpp +41 -0
  81. data/vendor/ggml/src/ggml-cpu/arch/arm/quants.c +4319 -0
  82. data/vendor/ggml/src/ggml-cpu/arch/arm/repack.cpp +5156 -0
  83. data/vendor/ggml/src/ggml-cpu/arch/loongarch/quants.c +2309 -0
  84. data/vendor/ggml/src/ggml-cpu/arch/powerpc/cpu-feats.cpp +82 -0
  85. data/vendor/ggml/src/ggml-cpu/arch/powerpc/quants.c +2304 -0
  86. data/vendor/ggml/src/ggml-cpu/arch/riscv/cpu-feats.cpp +38 -0
  87. data/vendor/ggml/src/ggml-cpu/arch/riscv/quants.c +6596 -0
  88. data/vendor/ggml/src/ggml-cpu/arch/riscv/repack.cpp +1703 -0
  89. data/vendor/ggml/src/ggml-cpu/arch/s390/cpu-feats.cpp +50 -0
  90. data/vendor/ggml/src/ggml-cpu/arch/s390/quants.c +1465 -0
  91. data/vendor/ggml/src/ggml-cpu/arch/wasm/quants.c +1292 -0
  92. data/vendor/ggml/src/ggml-cpu/arch/x86/cpu-feats.cpp +327 -0
  93. data/vendor/ggml/src/ggml-cpu/arch/x86/quants.c +4108 -0
  94. data/vendor/ggml/src/ggml-cpu/arch/x86/repack.cpp +6407 -0
  95. data/vendor/ggml/src/ggml-cpu/arch-fallback.h +353 -0
  96. data/vendor/ggml/src/ggml-cpu/binary-ops.cpp +154 -0
  97. data/vendor/ggml/src/ggml-cpu/binary-ops.h +16 -0
  98. data/vendor/ggml/src/ggml-cpu/cmake/FindSIMD.cmake +100 -0
  99. data/vendor/ggml/src/ggml-cpu/cmake/FindSMTIME.cmake +32 -0
  100. data/vendor/ggml/src/ggml-cpu/common.h +95 -0
  101. data/vendor/ggml/src/ggml-cpu/ggml-cpu-impl.h +539 -0
  102. data/vendor/ggml/src/ggml-cpu/ggml-cpu.c +3936 -0
  103. data/vendor/ggml/src/ggml-cpu/ggml-cpu.cpp +712 -0
  104. data/vendor/ggml/src/ggml-cpu/hbm.cpp +55 -0
  105. data/vendor/ggml/src/ggml-cpu/hbm.h +8 -0
  106. data/vendor/ggml/src/ggml-cpu/iqp.cpp +1253 -0
  107. data/vendor/ggml/src/ggml-cpu/iqp.h +39 -0
  108. data/vendor/ggml/src/ggml-cpu/kleidiai/CMakeLists.txt +14 -0
  109. data/vendor/ggml/src/ggml-cpu/kleidiai/kernels.cpp +1101 -0
  110. data/vendor/ggml/src/ggml-cpu/kleidiai/kernels.h +101 -0
  111. data/vendor/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp +1919 -0
  112. data/vendor/ggml/src/ggml-cpu/kleidiai/kleidiai.h +17 -0
  113. data/vendor/ggml/src/ggml-cpu/llamafile/sgemm.cpp +4164 -0
  114. data/vendor/ggml/src/ggml-cpu/llamafile/sgemm.h +25 -0
  115. data/vendor/ggml/src/ggml-cpu/ops.cpp +12164 -0
  116. data/vendor/ggml/src/ggml-cpu/ops.h +124 -0
  117. data/vendor/ggml/src/ggml-cpu/quants.c +1339 -0
  118. data/vendor/ggml/src/ggml-cpu/quants.h +106 -0
  119. data/vendor/ggml/src/ggml-cpu/repack.cpp +4836 -0
  120. data/vendor/ggml/src/ggml-cpu/repack.h +245 -0
  121. data/vendor/ggml/src/ggml-cpu/simd-gemm.h +226 -0
  122. data/vendor/ggml/src/ggml-cpu/simd-mappings.h +1319 -0
  123. data/vendor/ggml/src/ggml-cpu/spacemit/ime.cpp +1742 -0
  124. data/vendor/ggml/src/ggml-cpu/spacemit/ime.h +21 -0
  125. data/vendor/ggml/src/ggml-cpu/spacemit/ime1_kernels.cpp +1027 -0
  126. data/vendor/ggml/src/ggml-cpu/spacemit/ime2_kernels.cpp +5768 -0
  127. data/vendor/ggml/src/ggml-cpu/spacemit/ime_env.cpp +320 -0
  128. data/vendor/ggml/src/ggml-cpu/spacemit/ime_env.h +55 -0
  129. data/vendor/ggml/src/ggml-cpu/spacemit/ime_kernels.h +189 -0
  130. data/vendor/ggml/src/ggml-cpu/spacemit/repack.cpp +1795 -0
  131. data/vendor/ggml/src/ggml-cpu/spacemit/repack.h +14 -0
  132. data/vendor/ggml/src/ggml-cpu/spacemit/rvv_kernels.cpp +3178 -0
  133. data/vendor/ggml/src/ggml-cpu/spacemit/rvv_kernels.h +95 -0
  134. data/vendor/ggml/src/ggml-cpu/spacemit/spine_barrier.h +34 -0
  135. data/vendor/ggml/src/ggml-cpu/spacemit/spine_mem_pool.cpp +760 -0
  136. data/vendor/ggml/src/ggml-cpu/spacemit/spine_mem_pool.h +32 -0
  137. data/vendor/ggml/src/ggml-cpu/spacemit/spine_tcm.h +409 -0
  138. data/vendor/ggml/src/ggml-cpu/traits.cpp +36 -0
  139. data/vendor/ggml/src/ggml-cpu/traits.h +38 -0
  140. data/vendor/ggml/src/ggml-cpu/unary-ops.cpp +337 -0
  141. data/vendor/ggml/src/ggml-cpu/unary-ops.h +35 -0
  142. data/vendor/ggml/src/ggml-cpu/vec.cpp +613 -0
  143. data/vendor/ggml/src/ggml-cpu/vec.h +1570 -0
  144. data/vendor/ggml/src/ggml-cuda/CMakeLists.txt +267 -0
  145. data/vendor/ggml/src/ggml-cuda/acc.cu +61 -0
  146. data/vendor/ggml/src/ggml-cuda/acc.cuh +5 -0
  147. data/vendor/ggml/src/ggml-cuda/add-id.cu +58 -0
  148. data/vendor/ggml/src/ggml-cuda/add-id.cuh +3 -0
  149. data/vendor/ggml/src/ggml-cuda/allreduce.cu +971 -0
  150. data/vendor/ggml/src/ggml-cuda/allreduce.cuh +29 -0
  151. data/vendor/ggml/src/ggml-cuda/arange.cu +34 -0
  152. data/vendor/ggml/src/ggml-cuda/arange.cuh +5 -0
  153. data/vendor/ggml/src/ggml-cuda/argmax.cu +91 -0
  154. data/vendor/ggml/src/ggml-cuda/argmax.cuh +3 -0
  155. data/vendor/ggml/src/ggml-cuda/argsort.cu +292 -0
  156. data/vendor/ggml/src/ggml-cuda/argsort.cuh +20 -0
  157. data/vendor/ggml/src/ggml-cuda/binbcast.cu +574 -0
  158. data/vendor/ggml/src/ggml-cuda/binbcast.cuh +12 -0
  159. data/vendor/ggml/src/ggml-cuda/clamp.cu +45 -0
  160. data/vendor/ggml/src/ggml-cuda/clamp.cuh +5 -0
  161. data/vendor/ggml/src/ggml-cuda/col2im-1d.cu +81 -0
  162. data/vendor/ggml/src/ggml-cuda/col2im-1d.cuh +3 -0
  163. data/vendor/ggml/src/ggml-cuda/common.cuh +1678 -0
  164. data/vendor/ggml/src/ggml-cuda/concat.cu +242 -0
  165. data/vendor/ggml/src/ggml-cuda/concat.cuh +5 -0
  166. data/vendor/ggml/src/ggml-cuda/conv-transpose-1d.cu +88 -0
  167. data/vendor/ggml/src/ggml-cuda/conv-transpose-1d.cuh +5 -0
  168. data/vendor/ggml/src/ggml-cuda/conv2d-dw.cu +161 -0
  169. data/vendor/ggml/src/ggml-cuda/conv2d-dw.cuh +5 -0
  170. data/vendor/ggml/src/ggml-cuda/conv2d-transpose.cu +115 -0
  171. data/vendor/ggml/src/ggml-cuda/conv2d-transpose.cuh +5 -0
  172. data/vendor/ggml/src/ggml-cuda/conv2d.cu +167 -0
  173. data/vendor/ggml/src/ggml-cuda/conv2d.cuh +5 -0
  174. data/vendor/ggml/src/ggml-cuda/convert.cu +705 -0
  175. data/vendor/ggml/src/ggml-cuda/convert.cuh +66 -0
  176. data/vendor/ggml/src/ggml-cuda/count-equal.cu +64 -0
  177. data/vendor/ggml/src/ggml-cuda/count-equal.cuh +5 -0
  178. data/vendor/ggml/src/ggml-cuda/cp-async.cuh +57 -0
  179. data/vendor/ggml/src/ggml-cuda/cpy-utils.cuh +217 -0
  180. data/vendor/ggml/src/ggml-cuda/cpy.cu +617 -0
  181. data/vendor/ggml/src/ggml-cuda/cpy.cuh +7 -0
  182. data/vendor/ggml/src/ggml-cuda/cross-entropy-loss.cu +177 -0
  183. data/vendor/ggml/src/ggml-cuda/cross-entropy-loss.cuh +7 -0
  184. data/vendor/ggml/src/ggml-cuda/cumsum.cu +307 -0
  185. data/vendor/ggml/src/ggml-cuda/cumsum.cuh +5 -0
  186. data/vendor/ggml/src/ggml-cuda/dequantize.cuh +452 -0
  187. data/vendor/ggml/src/ggml-cuda/diag.cu +77 -0
  188. data/vendor/ggml/src/ggml-cuda/diag.cuh +5 -0
  189. data/vendor/ggml/src/ggml-cuda/diagmask.cu +40 -0
  190. data/vendor/ggml/src/ggml-cuda/diagmask.cuh +5 -0
  191. data/vendor/ggml/src/ggml-cuda/dsv4-hc.cu +294 -0
  192. data/vendor/ggml/src/ggml-cuda/dsv4-hc.cuh +6 -0
  193. data/vendor/ggml/src/ggml-cuda/fattn-common.cuh +1289 -0
  194. data/vendor/ggml/src/ggml-cuda/fattn-mma-f16.cuh +2136 -0
  195. data/vendor/ggml/src/ggml-cuda/fattn-swizzle.cuh +126 -0
  196. data/vendor/ggml/src/ggml-cuda/fattn-tile.cu +60 -0
  197. data/vendor/ggml/src/ggml-cuda/fattn-tile.cuh +1355 -0
  198. data/vendor/ggml/src/ggml-cuda/fattn-vec.cuh +611 -0
  199. data/vendor/ggml/src/ggml-cuda/fattn.cu +722 -0
  200. data/vendor/ggml/src/ggml-cuda/fattn.cuh +7 -0
  201. data/vendor/ggml/src/ggml-cuda/fill.cu +37 -0
  202. data/vendor/ggml/src/ggml-cuda/fill.cuh +3 -0
  203. data/vendor/ggml/src/ggml-cuda/fwht.cu +101 -0
  204. data/vendor/ggml/src/ggml-cuda/fwht.cuh +4 -0
  205. data/vendor/ggml/src/ggml-cuda/gated_delta_net.cu +327 -0
  206. data/vendor/ggml/src/ggml-cuda/gated_delta_net.cuh +14 -0
  207. data/vendor/ggml/src/ggml-cuda/getrows.cu +490 -0
  208. data/vendor/ggml/src/ggml-cuda/getrows.cuh +15 -0
  209. data/vendor/ggml/src/ggml-cuda/ggml-cuda.cu +5781 -0
  210. data/vendor/ggml/src/ggml-cuda/gla.cu +93 -0
  211. data/vendor/ggml/src/ggml-cuda/gla.cuh +3 -0
  212. data/vendor/ggml/src/ggml-cuda/im2col.cu +267 -0
  213. data/vendor/ggml/src/ggml-cuda/im2col.cuh +6 -0
  214. data/vendor/ggml/src/ggml-cuda/lightning-indexer.cu +588 -0
  215. data/vendor/ggml/src/ggml-cuda/lightning-indexer.cuh +4 -0
  216. data/vendor/ggml/src/ggml-cuda/mean.cu +77 -0
  217. data/vendor/ggml/src/ggml-cuda/mean.cuh +3 -0
  218. data/vendor/ggml/src/ggml-cuda/mma.cuh +1456 -0
  219. data/vendor/ggml/src/ggml-cuda/mmf.cu +191 -0
  220. data/vendor/ggml/src/ggml-cuda/mmf.cuh +908 -0
  221. data/vendor/ggml/src/ggml-cuda/mmid.cu +180 -0
  222. data/vendor/ggml/src/ggml-cuda/mmid.cuh +5 -0
  223. data/vendor/ggml/src/ggml-cuda/mmq-config-ampere.cuh +383 -0
  224. data/vendor/ggml/src/ggml-cuda/mmq-config-blackwell.cuh +37 -0
  225. data/vendor/ggml/src/ggml-cuda/mmq-config-cdna.cuh +185 -0
  226. data/vendor/ggml/src/ggml-cuda/mmq-config-pascal-dp4a.cuh +273 -0
  227. data/vendor/ggml/src/ggml-cuda/mmq-config-pascal-older.cuh +273 -0
  228. data/vendor/ggml/src/ggml-cuda/mmq-config-rdna2.cuh +273 -0
  229. data/vendor/ggml/src/ggml-cuda/mmq-config-rdna3-5.cuh +290 -0
  230. data/vendor/ggml/src/ggml-cuda/mmq-config-rdna3.cuh +274 -0
  231. data/vendor/ggml/src/ggml-cuda/mmq-config-rdna4.cuh +290 -0
  232. data/vendor/ggml/src/ggml-cuda/mmq-load-tiles.cuh +1768 -0
  233. data/vendor/ggml/src/ggml-cuda/mmq-vec-dot.cuh +1240 -0
  234. data/vendor/ggml/src/ggml-cuda/mmq.cu +386 -0
  235. data/vendor/ggml/src/ggml-cuda/mmq.cuh +1598 -0
  236. data/vendor/ggml/src/ggml-cuda/mmvf.cu +875 -0
  237. data/vendor/ggml/src/ggml-cuda/mmvf.cuh +14 -0
  238. data/vendor/ggml/src/ggml-cuda/mmvq.cu +1504 -0
  239. data/vendor/ggml/src/ggml-cuda/mmvq.cuh +18 -0
  240. data/vendor/ggml/src/ggml-cuda/moe-weighted-reduction.cu +65 -0
  241. data/vendor/ggml/src/ggml-cuda/moe-weighted-reduction.cuh +7 -0
  242. data/vendor/ggml/src/ggml-cuda/norm.cu +698 -0
  243. data/vendor/ggml/src/ggml-cuda/norm.cuh +18 -0
  244. data/vendor/ggml/src/ggml-cuda/opt-step-adamw.cu +78 -0
  245. data/vendor/ggml/src/ggml-cuda/opt-step-adamw.cuh +5 -0
  246. data/vendor/ggml/src/ggml-cuda/opt-step-sgd.cu +49 -0
  247. data/vendor/ggml/src/ggml-cuda/opt-step-sgd.cuh +5 -0
  248. data/vendor/ggml/src/ggml-cuda/out-prod.cu +125 -0
  249. data/vendor/ggml/src/ggml-cuda/out-prod.cuh +3 -0
  250. data/vendor/ggml/src/ggml-cuda/pad.cu +106 -0
  251. data/vendor/ggml/src/ggml-cuda/pad.cuh +5 -0
  252. data/vendor/ggml/src/ggml-cuda/pad_reflect_1d.cu +91 -0
  253. data/vendor/ggml/src/ggml-cuda/pad_reflect_1d.cuh +5 -0
  254. data/vendor/ggml/src/ggml-cuda/pool1d.cu +85 -0
  255. data/vendor/ggml/src/ggml-cuda/pool1d.cuh +5 -0
  256. data/vendor/ggml/src/ggml-cuda/pool2d.cu +94 -0
  257. data/vendor/ggml/src/ggml-cuda/pool2d.cuh +5 -0
  258. data/vendor/ggml/src/ggml-cuda/quantize.cu +697 -0
  259. data/vendor/ggml/src/ggml-cuda/quantize.cuh +70 -0
  260. data/vendor/ggml/src/ggml-cuda/reduce_rows.cuh +43 -0
  261. data/vendor/ggml/src/ggml-cuda/roll.cu +67 -0
  262. data/vendor/ggml/src/ggml-cuda/roll.cuh +5 -0
  263. data/vendor/ggml/src/ggml-cuda/rope.cu +941 -0
  264. data/vendor/ggml/src/ggml-cuda/rope.cuh +11 -0
  265. data/vendor/ggml/src/ggml-cuda/scale.cu +37 -0
  266. data/vendor/ggml/src/ggml-cuda/scale.cuh +5 -0
  267. data/vendor/ggml/src/ggml-cuda/set-rows.cu +398 -0
  268. data/vendor/ggml/src/ggml-cuda/set-rows.cuh +7 -0
  269. data/vendor/ggml/src/ggml-cuda/set.cu +39 -0
  270. data/vendor/ggml/src/ggml-cuda/set.cuh +7 -0
  271. data/vendor/ggml/src/ggml-cuda/snake.cu +72 -0
  272. data/vendor/ggml/src/ggml-cuda/snake.cuh +8 -0
  273. data/vendor/ggml/src/ggml-cuda/softcap.cu +37 -0
  274. data/vendor/ggml/src/ggml-cuda/softcap.cuh +5 -0
  275. data/vendor/ggml/src/ggml-cuda/softmax.cu +480 -0
  276. data/vendor/ggml/src/ggml-cuda/softmax.cuh +7 -0
  277. data/vendor/ggml/src/ggml-cuda/solve_tri.cu +273 -0
  278. data/vendor/ggml/src/ggml-cuda/solve_tri.cuh +3 -0
  279. data/vendor/ggml/src/ggml-cuda/ssm-conv.cu +206 -0
  280. data/vendor/ggml/src/ggml-cuda/ssm-conv.cuh +3 -0
  281. data/vendor/ggml/src/ggml-cuda/ssm-scan.cu +859 -0
  282. data/vendor/ggml/src/ggml-cuda/ssm-scan.cuh +3 -0
  283. data/vendor/ggml/src/ggml-cuda/sum.cu +41 -0
  284. data/vendor/ggml/src/ggml-cuda/sum.cuh +5 -0
  285. data/vendor/ggml/src/ggml-cuda/sumrows.cu +47 -0
  286. data/vendor/ggml/src/ggml-cuda/sumrows.cuh +4 -0
  287. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_16.cu +6 -0
  288. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_32.cu +6 -0
  289. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_8.cu +12 -0
  290. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_1.cu +10 -0
  291. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_2.cu +11 -0
  292. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_4.cu +12 -0
  293. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_16.cu +6 -0
  294. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_32.cu +6 -0
  295. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_4.cu +12 -0
  296. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_8.cu +12 -0
  297. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_1.cu +10 -0
  298. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_2.cu +11 -0
  299. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_16.cu +6 -0
  300. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_2.cu +11 -0
  301. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_4.cu +12 -0
  302. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_8.cu +12 -0
  303. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_64-ncols2_1.cu +10 -0
  304. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_1.cu +10 -0
  305. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_2.cu +11 -0
  306. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_4.cu +12 -0
  307. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_8.cu +12 -0
  308. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq112-dv112.cu +5 -0
  309. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq128-dv128.cu +5 -0
  310. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq192-dv128.cu +5 -0
  311. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq256-dv256.cu +5 -0
  312. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq320-dv256.cu +5 -0
  313. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq40-dv40.cu +5 -0
  314. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq512-dv512.cu +5 -0
  315. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq576-dv512.cu +5 -0
  316. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq64-dv64.cu +5 -0
  317. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq72-dv72.cu +5 -0
  318. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq80-dv80.cu +5 -0
  319. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq96-dv96.cu +5 -0
  320. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-bf16.cu +7 -0
  321. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-f16.cu +7 -0
  322. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q4_0.cu +7 -0
  323. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q4_1.cu +7 -0
  324. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q5_0.cu +7 -0
  325. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q5_1.cu +7 -0
  326. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q8_0.cu +7 -0
  327. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-bf16.cu +7 -0
  328. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-f16.cu +7 -0
  329. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q4_0.cu +7 -0
  330. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q4_1.cu +7 -0
  331. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q5_0.cu +7 -0
  332. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q5_1.cu +7 -0
  333. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q8_0.cu +7 -0
  334. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-bf16.cu +7 -0
  335. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-f16.cu +7 -0
  336. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q4_0.cu +7 -0
  337. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q4_1.cu +7 -0
  338. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q5_0.cu +7 -0
  339. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q5_1.cu +7 -0
  340. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q8_0.cu +7 -0
  341. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-bf16.cu +7 -0
  342. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-f16.cu +7 -0
  343. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q4_0.cu +7 -0
  344. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q4_1.cu +7 -0
  345. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q5_0.cu +7 -0
  346. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q5_1.cu +7 -0
  347. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q8_0.cu +7 -0
  348. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-bf16.cu +7 -0
  349. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-f16.cu +7 -0
  350. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q4_0.cu +7 -0
  351. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q4_1.cu +7 -0
  352. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q5_0.cu +7 -0
  353. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q5_1.cu +7 -0
  354. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q8_0.cu +7 -0
  355. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-bf16.cu +7 -0
  356. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-f16.cu +7 -0
  357. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q4_0.cu +7 -0
  358. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q4_1.cu +7 -0
  359. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q5_0.cu +7 -0
  360. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q5_1.cu +7 -0
  361. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q8_0.cu +7 -0
  362. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-bf16.cu +7 -0
  363. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-f16.cu +7 -0
  364. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q4_0.cu +7 -0
  365. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q4_1.cu +7 -0
  366. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q5_0.cu +7 -0
  367. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q5_1.cu +7 -0
  368. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q8_0.cu +7 -0
  369. data/vendor/ggml/src/ggml-cuda/template-instances/generate_cu_files.py +111 -0
  370. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_1.cu +5 -0
  371. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_10.cu +5 -0
  372. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_11.cu +5 -0
  373. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_12.cu +5 -0
  374. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_13.cu +5 -0
  375. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_14.cu +5 -0
  376. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_15.cu +5 -0
  377. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_16.cu +5 -0
  378. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_2.cu +5 -0
  379. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_3.cu +5 -0
  380. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_4.cu +5 -0
  381. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_5.cu +5 -0
  382. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_6.cu +5 -0
  383. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_7.cu +5 -0
  384. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_8.cu +5 -0
  385. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_9.cu +5 -0
  386. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq1_s.cu +5 -0
  387. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_s.cu +5 -0
  388. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xs.cu +5 -0
  389. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xxs.cu +5 -0
  390. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_s.cu +5 -0
  391. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_xxs.cu +5 -0
  392. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_nl.cu +5 -0
  393. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_xs.cu +5 -0
  394. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-mxfp4.cu +5 -0
  395. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-nvfp4.cu +5 -0
  396. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q1_0.cu +5 -0
  397. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_0.cu +5 -0
  398. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_k.cu +5 -0
  399. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q3_k.cu +5 -0
  400. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_0.cu +5 -0
  401. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_1.cu +5 -0
  402. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_k.cu +5 -0
  403. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_0.cu +5 -0
  404. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_1.cu +5 -0
  405. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_k.cu +5 -0
  406. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q6_k.cu +5 -0
  407. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q8_0.cu +5 -0
  408. data/vendor/ggml/src/ggml-cuda/top-k.cu +275 -0
  409. data/vendor/ggml/src/ggml-cuda/top-k.cuh +3 -0
  410. data/vendor/ggml/src/ggml-cuda/topk-moe.cu +442 -0
  411. data/vendor/ggml/src/ggml-cuda/topk-moe.cuh +31 -0
  412. data/vendor/ggml/src/ggml-cuda/tri.cu +136 -0
  413. data/vendor/ggml/src/ggml-cuda/tri.cuh +5 -0
  414. data/vendor/ggml/src/ggml-cuda/tsembd.cu +47 -0
  415. data/vendor/ggml/src/ggml-cuda/tsembd.cuh +5 -0
  416. data/vendor/ggml/src/ggml-cuda/unary.cu +721 -0
  417. data/vendor/ggml/src/ggml-cuda/unary.cuh +123 -0
  418. data/vendor/ggml/src/ggml-cuda/upscale.cu +293 -0
  419. data/vendor/ggml/src/ggml-cuda/upscale.cuh +5 -0
  420. data/vendor/ggml/src/ggml-cuda/vecdotq.cuh +1371 -0
  421. data/vendor/ggml/src/ggml-cuda/vendors/cuda.h +28 -0
  422. data/vendor/ggml/src/ggml-cuda/vendors/hip.h +301 -0
  423. data/vendor/ggml/src/ggml-cuda/vendors/musa.h +151 -0
  424. data/vendor/ggml/src/ggml-cuda/wkv.cu +253 -0
  425. data/vendor/ggml/src/ggml-cuda/wkv.cuh +7 -0
  426. data/vendor/ggml/src/ggml-et/CMakeLists.txt +246 -0
  427. data/vendor/ggml/src/ggml-et/cmake/embed_one_kernel.cmake +15 -0
  428. data/vendor/ggml/src/ggml-et/cmake/ggml-et-kernels-embed.cpp.in +6 -0
  429. data/vendor/ggml/src/ggml-et/cmake/ggml-et-kernels-embed.hpp.in +12 -0
  430. data/vendor/ggml/src/ggml-et/cmake/ggml-et-uberkernel-kernel-map.cpp.in +18 -0
  431. data/vendor/ggml/src/ggml-et/cmake/ggml-et-uberkernel-kernel-map.h.in +13 -0
  432. data/vendor/ggml/src/ggml-et/et-kernels/CMakeLists.txt +137 -0
  433. data/vendor/ggml/src/ggml-et/et-kernels/scripts/check_unimplemented_instructions.sh +36 -0
  434. data/vendor/ggml/src/ggml-et/et-kernels/src/RunBackend.sh +23 -0
  435. data/vendor/ggml/src/ggml-et/et-kernels/src/block_ops.h +997 -0
  436. data/vendor/ggml/src/ggml-et/et-kernels/src/clamp_f32.c +120 -0
  437. data/vendor/ggml/src/ggml-et/et-kernels/src/concat_f32.c +175 -0
  438. data/vendor/ggml/src/ggml-et/et-kernels/src/cont_f16.c +107 -0
  439. data/vendor/ggml/src/ggml-et/et-kernels/src/cont_f32.c +248 -0
  440. data/vendor/ggml/src/ggml-et/et-kernels/src/conv_2d_f32_me.c +807 -0
  441. data/vendor/ggml/src/ggml-et/et-kernels/src/cpy_f32_f16.c +110 -0
  442. data/vendor/ggml/src/ggml-et/et-kernels/src/crt.S +15 -0
  443. data/vendor/ggml/src/ggml-et/et-kernels/src/cumsum_f32.c +96 -0
  444. data/vendor/ggml/src/ggml-et/et-kernels/src/diag_f32.c +90 -0
  445. data/vendor/ggml/src/ggml-et/et-kernels/src/el_map_f32.c +377 -0
  446. data/vendor/ggml/src/ggml-et/et-kernels/src/fill_f32.c +87 -0
  447. data/vendor/ggml/src/ggml-et/et-kernels/src/flash_attn_ext_f16_me.c +1000 -0
  448. data/vendor/ggml/src/ggml-et/et-kernels/src/flash_attn_ext_f32.c +217 -0
  449. data/vendor/ggml/src/ggml-et/et-kernels/src/gated_delta_net_f32.c +346 -0
  450. data/vendor/ggml/src/ggml-et/et-kernels/src/get_rows_f32.c +612 -0
  451. data/vendor/ggml/src/ggml-et/et-kernels/src/ggml_tensor.h +44 -0
  452. data/vendor/ggml/src/ggml-et/et-kernels/src/glu_f32.c +606 -0
  453. data/vendor/ggml/src/ggml-et/et-kernels/src/group_norm_f32.c +171 -0
  454. data/vendor/ggml/src/ggml-et/et-kernels/src/im2col.c +130 -0
  455. data/vendor/ggml/src/ggml-et/et-kernels/src/l2_norm_f32.c +237 -0
  456. data/vendor/ggml/src/ggml-et/et-kernels/src/linker.ld +85 -0
  457. data/vendor/ggml/src/ggml-et/et-kernels/src/math_fp.h +299 -0
  458. data/vendor/ggml/src/ggml-et/et-kernels/src/mean_f32.c +220 -0
  459. data/vendor/ggml/src/ggml-et/et-kernels/src/memops.c +181 -0
  460. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0.c +358 -0
  461. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0_matrix_engine.c +368 -0
  462. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_Q8_0.c +413 -0
  463. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f16.c +142 -0
  464. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f16_matrix_engine.c +329 -0
  465. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f32.c +137 -0
  466. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f32_matrix_engine.c +155 -0
  467. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_id_Q4_0.c +169 -0
  468. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_id_Q8_0.c +160 -0
  469. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_id_f32.c +288 -0
  470. data/vendor/ggml/src/ggml-et/et-kernels/src/norm_f32.c +328 -0
  471. data/vendor/ggml/src/ggml-et/et-kernels/src/pad_f32.c +165 -0
  472. data/vendor/ggml/src/ggml-et/et-kernels/src/platform.h +545 -0
  473. data/vendor/ggml/src/ggml-et/et-kernels/src/quants.h +72 -0
  474. data/vendor/ggml/src/ggml-et/et-kernels/src/repeat_f32.c +118 -0
  475. data/vendor/ggml/src/ggml-et/et-kernels/src/rms_norm_f32.c +270 -0
  476. data/vendor/ggml/src/ggml-et/et-kernels/src/rms_norm_mul_f32.c +290 -0
  477. data/vendor/ggml/src/ggml-et/et-kernels/src/rope_f32.c +656 -0
  478. data/vendor/ggml/src/ggml-et/et-kernels/src/rwkv_wkv6_f32.c +184 -0
  479. data/vendor/ggml/src/ggml-et/et-kernels/src/rwkv_wkv7_f32.c +272 -0
  480. data/vendor/ggml/src/ggml-et/et-kernels/src/scale_f32.c +94 -0
  481. data/vendor/ggml/src/ggml-et/et-kernels/src/set_f32.c +101 -0
  482. data/vendor/ggml/src/ggml-et/et-kernels/src/set_rows_f32.c +394 -0
  483. data/vendor/ggml/src/ggml-et/et-kernels/src/softmax_f32.c +698 -0
  484. data/vendor/ggml/src/ggml-et/et-kernels/src/solve_tri_f32.c +109 -0
  485. data/vendor/ggml/src/ggml-et/et-kernels/src/sqr_f32.c +88 -0
  486. data/vendor/ggml/src/ggml-et/et-kernels/src/ssm_conv_f32.c +129 -0
  487. data/vendor/ggml/src/ggml-et/et-kernels/src/ssm_scan_f32.c +282 -0
  488. data/vendor/ggml/src/ggml-et/et-kernels/src/sum_rows_f32.c +103 -0
  489. data/vendor/ggml/src/ggml-et/et-kernels/src/tensor.h +897 -0
  490. data/vendor/ggml/src/ggml-et/et-kernels/src/tri_f32.c +244 -0
  491. data/vendor/ggml/src/ggml-et/et-kernels/src/uberkernel.c +497 -0
  492. data/vendor/ggml/src/ggml-et/et-kernels/src/unary_f32.c +705 -0
  493. data/vendor/ggml/src/ggml-et/ggml-et-common.h +86 -0
  494. data/vendor/ggml/src/ggml-et/ggml-et-cpu-compare.cpp +502 -0
  495. data/vendor/ggml/src/ggml-et/ggml-et-cpu-compare.h +54 -0
  496. data/vendor/ggml/src/ggml-et/ggml-et-kernels.cpp +508 -0
  497. data/vendor/ggml/src/ggml-et/ggml-et-kernels.h +48 -0
  498. data/vendor/ggml/src/ggml-et/ggml-et-memops.cpp +36 -0
  499. data/vendor/ggml/src/ggml-et/ggml-et-memops.h +18 -0
  500. data/vendor/ggml/src/ggml-et/ggml-et-ops.cpp +2584 -0
  501. data/vendor/ggml/src/ggml-et/ggml-et-ops.h +393 -0
  502. data/vendor/ggml/src/ggml-et/ggml-et-uberkernel-common.h +17 -0
  503. data/vendor/ggml/src/ggml-et/ggml-et.cpp +1880 -0
  504. data/vendor/ggml/src/ggml-feats.h +166 -0
  505. data/vendor/ggml/src/ggml-hexagon/CMakeLists.txt +113 -0
  506. data/vendor/ggml/src/ggml-hexagon/ggml-hexagon.cpp +6523 -0
  507. data/vendor/ggml/src/ggml-hexagon/htp/CMakeLists.txt +58 -0
  508. data/vendor/ggml/src/ggml-hexagon/htp/act-ops.c +585 -0
  509. data/vendor/ggml/src/ggml-hexagon/htp/allreduce-ops.c +398 -0
  510. data/vendor/ggml/src/ggml-hexagon/htp/allreduce-ops.h +40 -0
  511. data/vendor/ggml/src/ggml-hexagon/htp/argsort-ops.c +514 -0
  512. data/vendor/ggml/src/ggml-hexagon/htp/binary-ops.c +873 -0
  513. data/vendor/ggml/src/ggml-hexagon/htp/cmake-toolchain.cmake +155 -0
  514. data/vendor/ggml/src/ggml-hexagon/htp/concat-ops.c +277 -0
  515. data/vendor/ggml/src/ggml-hexagon/htp/cpy-ops.c +354 -0
  516. data/vendor/ggml/src/ggml-hexagon/htp/cumsum-ops.c +265 -0
  517. data/vendor/ggml/src/ggml-hexagon/htp/diag-ops.c +216 -0
  518. data/vendor/ggml/src/ggml-hexagon/htp/dma-queue.c +104 -0
  519. data/vendor/ggml/src/ggml-hexagon/htp/dma-queue.h +408 -0
  520. data/vendor/ggml/src/ggml-hexagon/htp/fill-ops.c +123 -0
  521. data/vendor/ggml/src/ggml-hexagon/htp/flash-attn-ops.c +2479 -0
  522. data/vendor/ggml/src/ggml-hexagon/htp/flash-attn-ops.h +344 -0
  523. data/vendor/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c +1157 -0
  524. data/vendor/ggml/src/ggml-hexagon/htp/get-rows-ops.c +260 -0
  525. data/vendor/ggml/src/ggml-hexagon/htp/get-rows-ops.h +77 -0
  526. data/vendor/ggml/src/ggml-hexagon/htp/hex-bitmap.h +24 -0
  527. data/vendor/ggml/src/ggml-hexagon/htp/hex-common.h +80 -0
  528. data/vendor/ggml/src/ggml-hexagon/htp/hex-dma.h +2 -0
  529. data/vendor/ggml/src/ggml-hexagon/htp/hex-dump.h +86 -0
  530. data/vendor/ggml/src/ggml-hexagon/htp/hex-fastdiv.h +37 -0
  531. data/vendor/ggml/src/ggml-hexagon/htp/hex-profile.h +64 -0
  532. data/vendor/ggml/src/ggml-hexagon/htp/hex-utils.h +65 -0
  533. data/vendor/ggml/src/ggml-hexagon/htp/hmx-fa-kernels.h +555 -0
  534. data/vendor/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h +1378 -0
  535. data/vendor/ggml/src/ggml-hexagon/htp/hmx-queue.c +170 -0
  536. data/vendor/ggml/src/ggml-hexagon/htp/hmx-queue.h +160 -0
  537. data/vendor/ggml/src/ggml-hexagon/htp/hmx-utils.h +222 -0
  538. data/vendor/ggml/src/ggml-hexagon/htp/htp-ctx.h +146 -0
  539. data/vendor/ggml/src/ggml-hexagon/htp/htp-ops.h +242 -0
  540. data/vendor/ggml/src/ggml-hexagon/htp/htp-tensor.c +300 -0
  541. data/vendor/ggml/src/ggml-hexagon/htp/htp-tensor.h +29 -0
  542. data/vendor/ggml/src/ggml-hexagon/htp/htp-vtcm.h +19 -0
  543. data/vendor/ggml/src/ggml-hexagon/htp/htp_iface.idl +23 -0
  544. data/vendor/ggml/src/ggml-hexagon/htp/hvx-arith.h +636 -0
  545. data/vendor/ggml/src/ggml-hexagon/htp/hvx-base.h +292 -0
  546. data/vendor/ggml/src/ggml-hexagon/htp/hvx-copy.h +262 -0
  547. data/vendor/ggml/src/ggml-hexagon/htp/hvx-div.h +291 -0
  548. data/vendor/ggml/src/ggml-hexagon/htp/hvx-dump.h +129 -0
  549. data/vendor/ggml/src/ggml-hexagon/htp/hvx-exp.h +255 -0
  550. data/vendor/ggml/src/ggml-hexagon/htp/hvx-fa-kernels.h +303 -0
  551. data/vendor/ggml/src/ggml-hexagon/htp/hvx-flash-attn.h +47 -0
  552. data/vendor/ggml/src/ggml-hexagon/htp/hvx-floor.h +100 -0
  553. data/vendor/ggml/src/ggml-hexagon/htp/hvx-inverse.h +210 -0
  554. data/vendor/ggml/src/ggml-hexagon/htp/hvx-log.h +118 -0
  555. data/vendor/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-flat.h +1511 -0
  556. data/vendor/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h +1200 -0
  557. data/vendor/ggml/src/ggml-hexagon/htp/hvx-norm.h +454 -0
  558. data/vendor/ggml/src/ggml-hexagon/htp/hvx-pow.h +42 -0
  559. data/vendor/ggml/src/ggml-hexagon/htp/hvx-quant.h +165 -0
  560. data/vendor/ggml/src/ggml-hexagon/htp/hvx-reduce.h +336 -0
  561. data/vendor/ggml/src/ggml-hexagon/htp/hvx-repl.h +74 -0
  562. data/vendor/ggml/src/ggml-hexagon/htp/hvx-scale.h +199 -0
  563. data/vendor/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h +181 -0
  564. data/vendor/ggml/src/ggml-hexagon/htp/hvx-sin-cos.h +90 -0
  565. data/vendor/ggml/src/ggml-hexagon/htp/hvx-sqrt.h +189 -0
  566. data/vendor/ggml/src/ggml-hexagon/htp/hvx-types.h +36 -0
  567. data/vendor/ggml/src/ggml-hexagon/htp/hvx-utils.h +24 -0
  568. data/vendor/ggml/src/ggml-hexagon/htp/im2col-ops.c +306 -0
  569. data/vendor/ggml/src/ggml-hexagon/htp/main.c +1245 -0
  570. data/vendor/ggml/src/ggml-hexagon/htp/matmul-ops.c +4078 -0
  571. data/vendor/ggml/src/ggml-hexagon/htp/matmul-ops.h +778 -0
  572. data/vendor/ggml/src/ggml-hexagon/htp/pad-ops.c +547 -0
  573. data/vendor/ggml/src/ggml-hexagon/htp/repeat-ops.c +148 -0
  574. data/vendor/ggml/src/ggml-hexagon/htp/rope-ops.c +733 -0
  575. data/vendor/ggml/src/ggml-hexagon/htp/set-rows-ops.c +230 -0
  576. data/vendor/ggml/src/ggml-hexagon/htp/set-rows-ops.h +74 -0
  577. data/vendor/ggml/src/ggml-hexagon/htp/softmax-ops.c +407 -0
  578. data/vendor/ggml/src/ggml-hexagon/htp/solve-tri-ops.c +267 -0
  579. data/vendor/ggml/src/ggml-hexagon/htp/ssm-conv.c +433 -0
  580. data/vendor/ggml/src/ggml-hexagon/htp/sum-rows-ops.c +128 -0
  581. data/vendor/ggml/src/ggml-hexagon/htp/unary-ops.c +1264 -0
  582. data/vendor/ggml/src/ggml-hexagon/htp/unary-ops.h +176 -0
  583. data/vendor/ggml/src/ggml-hexagon/htp/work-queue.c +244 -0
  584. data/vendor/ggml/src/ggml-hexagon/htp/work-queue.h +38 -0
  585. data/vendor/ggml/src/ggml-hexagon/htp-drv.cpp +425 -0
  586. data/vendor/ggml/src/ggml-hexagon/htp-drv.h +123 -0
  587. data/vendor/ggml/src/ggml-hexagon/htp-opnode.h +372 -0
  588. data/vendor/ggml/src/ggml-hexagon/libdl.h +79 -0
  589. data/vendor/ggml/src/ggml-hexagon/libggml-htp.inf +36 -0
  590. data/vendor/ggml/src/ggml-hip/CMakeLists.txt +153 -0
  591. data/vendor/ggml/src/ggml-impl.h +783 -0
  592. data/vendor/ggml/src/ggml-metal/CMakeLists.txt +240 -0
  593. data/vendor/ggml/src/ggml-metal/ggml-metal-common.cpp +474 -0
  594. data/vendor/ggml/src/ggml-metal/ggml-metal-common.h +56 -0
  595. data/vendor/ggml/src/ggml-metal/ggml-metal-context.h +41 -0
  596. data/vendor/ggml/src/ggml-metal/ggml-metal-context.m +759 -0
  597. data/vendor/ggml/src/ggml-metal/ggml-metal-device.cpp +2330 -0
  598. data/vendor/ggml/src/ggml-metal/ggml-metal-device.h +355 -0
  599. data/vendor/ggml/src/ggml-metal/ggml-metal-device.m +2473 -0
  600. data/vendor/ggml/src/ggml-metal/ggml-metal-impl.h +1345 -0
  601. data/vendor/ggml/src/ggml-metal/ggml-metal-ops.cpp +5610 -0
  602. data/vendor/ggml/src/ggml-metal/ggml-metal-ops.h +106 -0
  603. data/vendor/ggml/src/ggml-metal/ggml-metal-tuning.cpp +3732 -0
  604. data/vendor/ggml/src/ggml-metal/ggml-metal-tuning.h +77 -0
  605. data/vendor/ggml/src/ggml-metal/ggml-metal.cpp +1006 -0
  606. data/vendor/ggml/src/ggml-metal/kernels/argsort.metal +337 -0
  607. data/vendor/ggml/src/ggml-metal/kernels/binbcast.metal +228 -0
  608. data/vendor/ggml/src/ggml-metal/kernels/common.h +126 -0
  609. data/vendor/ggml/src/ggml-metal/kernels/conv.metal +724 -0
  610. data/vendor/ggml/src/ggml-metal/kernels/dequantize.h +735 -0
  611. data/vendor/ggml/src/ggml-metal/kernels/fa.metal +2445 -0
  612. data/vendor/ggml/src/ggml-metal/kernels/gated_delta_net.metal +250 -0
  613. data/vendor/ggml/src/ggml-metal/kernels/misc.metal +595 -0
  614. data/vendor/ggml/src/ggml-metal/kernels/mul_mm.metal +853 -0
  615. data/vendor/ggml/src/ggml-metal/kernels/mul_mv.metal +3225 -0
  616. data/vendor/ggml/src/ggml-metal/kernels/norm.metal +308 -0
  617. data/vendor/ggml/src/ggml-metal/kernels/pool.metal +148 -0
  618. data/vendor/ggml/src/ggml-metal/kernels/quantize.h +262 -0
  619. data/vendor/ggml/src/ggml-metal/kernels/quantize.metal +480 -0
  620. data/vendor/ggml/src/ggml-metal/kernels/reduce.metal +228 -0
  621. data/vendor/ggml/src/ggml-metal/kernels/rope.metal +333 -0
  622. data/vendor/ggml/src/ggml-metal/kernels/softmax.metal +223 -0
  623. data/vendor/ggml/src/ggml-metal/kernels/solve_tri.metal +75 -0
  624. data/vendor/ggml/src/ggml-metal/kernels/ssm.metal +473 -0
  625. data/vendor/ggml/src/ggml-metal/kernels/tri.metal +69 -0
  626. data/vendor/ggml/src/ggml-metal/kernels/unary.metal +400 -0
  627. data/vendor/ggml/src/ggml-metal/kernels/upscale.metal +179 -0
  628. data/vendor/ggml/src/ggml-metal/kernels/wkv.metal +179 -0
  629. data/vendor/ggml/src/ggml-musa/CMakeLists.txt +123 -0
  630. data/vendor/ggml/src/ggml-musa/mudnn.cu +112 -0
  631. data/vendor/ggml/src/ggml-musa/mudnn.cuh +12 -0
  632. data/vendor/ggml/src/ggml-opencl/CMakeLists.txt +246 -0
  633. data/vendor/ggml/src/ggml-opencl/cl-program-cache.cpp +453 -0
  634. data/vendor/ggml/src/ggml-opencl/cl-program-cache.h +75 -0
  635. data/vendor/ggml/src/ggml-opencl/fa_tune.h +92 -0
  636. data/vendor/ggml/src/ggml-opencl/ggml-opencl.cpp +26982 -0
  637. data/vendor/ggml/src/ggml-opencl/kernels/abs.cl +113 -0
  638. data/vendor/ggml/src/ggml-opencl/kernels/add.cl +190 -0
  639. data/vendor/ggml/src/ggml-opencl/kernels/add_id.cl +42 -0
  640. data/vendor/ggml/src/ggml-opencl/kernels/argsort.cl +86 -0
  641. data/vendor/ggml/src/ggml-opencl/kernels/clamp.cl +20 -0
  642. data/vendor/ggml/src/ggml-opencl/kernels/concat.cl +118 -0
  643. data/vendor/ggml/src/ggml-opencl/kernels/conv2d.cl +185 -0
  644. data/vendor/ggml/src/ggml-opencl/kernels/conv2d_f16_f32.cl +176 -0
  645. data/vendor/ggml/src/ggml-opencl/kernels/cpy.cl +288 -0
  646. data/vendor/ggml/src/ggml-opencl/kernels/cumsum.cl +139 -0
  647. data/vendor/ggml/src/ggml-opencl/kernels/cvt.cl +2663 -0
  648. data/vendor/ggml/src/ggml-opencl/kernels/diag.cl +27 -0
  649. data/vendor/ggml/src/ggml-opencl/kernels/diag_mask_inf.cl +58 -0
  650. data/vendor/ggml/src/ggml-opencl/kernels/div.cl +138 -0
  651. data/vendor/ggml/src/ggml-opencl/kernels/embed_kernel.py +26 -0
  652. data/vendor/ggml/src/ggml-opencl/kernels/exp.cl +125 -0
  653. data/vendor/ggml/src/ggml-opencl/kernels/expm1.cl +113 -0
  654. data/vendor/ggml/src/ggml-opencl/kernels/fill.cl +17 -0
  655. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f16.cl +421 -0
  656. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32.cl +422 -0
  657. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32_f16.cl +2656 -0
  658. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32_q4_0.cl +2011 -0
  659. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32_q8_0.cl +1840 -0
  660. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_pre_f16.cl +156 -0
  661. data/vendor/ggml/src/ggml-opencl/kernels/gated_delta_net.cl +249 -0
  662. data/vendor/ggml/src/ggml-opencl/kernels/gelu.cl +89 -0
  663. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_f32.cl +162 -0
  664. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_f32_ns.cl +376 -0
  665. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_q8_1_dp4a.cl +190 -0
  666. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_0_f32_ns.cl +324 -0
  667. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_0_q8_1_dp4a.cl +169 -0
  668. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_1_f32_ns.cl +326 -0
  669. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_k_f32_ns.cl +348 -0
  670. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_k_q8_1_dp4a.cl +209 -0
  671. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q5_0_f32_ns.cl +328 -0
  672. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q5_1_f32_ns.cl +330 -0
  673. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q5_k_f32_ns.cl +356 -0
  674. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q6_k_f32_ns.cl +335 -0
  675. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q6_k_q8_1_dp4a.cl +200 -0
  676. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q8_0_f32_ns.cl +221 -0
  677. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q8_1_dp4a.cl +226 -0
  678. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_nl_f32.cl +150 -0
  679. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_nl_q8_1_dp4a.cl +143 -0
  680. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q1_0_f32.cl +94 -0
  681. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_f32.cl +139 -0
  682. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_q8_1_dp4a.cl +127 -0
  683. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_1_f32.cl +132 -0
  684. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_f32.cl +490 -0
  685. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_q8_1_dp4a.cl +281 -0
  686. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_0_f32.cl +131 -0
  687. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_0_q8_1_dp4a.cl +235 -0
  688. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_1_f32.cl +134 -0
  689. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_f32.cl +176 -0
  690. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_q8_1_dp4a.cl +164 -0
  691. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32.cl +245 -0
  692. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32_tiled.cl +136 -0
  693. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_q8_1_dp4a.cl +144 -0
  694. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_f32.cl +129 -0
  695. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_q8_1_dp4a.cl +212 -0
  696. data/vendor/ggml/src/ggml-opencl/kernels/gemm_xmem_f16_f32_os8.cl +233 -0
  697. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32.cl +156 -0
  698. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32_ns.cl +257 -0
  699. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q4_0_f32_ns.cl +120 -0
  700. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q4_1_f32_ns.cl +123 -0
  701. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q4_k_f32_ns.cl +266 -0
  702. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q5_0_f32_ns.cl +123 -0
  703. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q5_1_f32_ns.cl +125 -0
  704. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q5_k_f32_ns.cl +160 -0
  705. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q6_k_f32_ns.cl +141 -0
  706. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq4_nl_f32.cl +307 -0
  707. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q1_0_f32.cl +125 -0
  708. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32.cl +383 -0
  709. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32_spec.cl +272 -0
  710. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_1_f32.cl +384 -0
  711. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32.cl +823 -0
  712. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_o4.cl +349 -0
  713. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_tiled.cl +118 -0
  714. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_0_f32.cl +296 -0
  715. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_1_f32.cl +299 -0
  716. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_k_f32.cl +453 -0
  717. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32.cl +409 -0
  718. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_o4.cl +372 -0
  719. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_tiled.cl +196 -0
  720. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q8_0_f32.cl +280 -0
  721. data/vendor/ggml/src/ggml-opencl/kernels/get_rows.cl +193 -0
  722. data/vendor/ggml/src/ggml-opencl/kernels/glu.cl +443 -0
  723. data/vendor/ggml/src/ggml-opencl/kernels/group_norm.cl +121 -0
  724. data/vendor/ggml/src/ggml-opencl/kernels/im2col_f16.cl +57 -0
  725. data/vendor/ggml/src/ggml-opencl/kernels/im2col_f32.cl +57 -0
  726. data/vendor/ggml/src/ggml-opencl/kernels/l2_norm.cl +71 -0
  727. data/vendor/ggml/src/ggml-opencl/kernels/mean.cl +140 -0
  728. data/vendor/ggml/src/ggml-opencl/kernels/moe_add_id_glu.cl +76 -0
  729. data/vendor/ggml/src/ggml-opencl/kernels/moe_combine.cl +79 -0
  730. data/vendor/ggml/src/ggml-opencl/kernels/moe_reorder_b.cl +30 -0
  731. data/vendor/ggml/src/ggml-opencl/kernels/moe_reorder_quant_a_q8_1.cl +64 -0
  732. data/vendor/ggml/src/ggml-opencl/kernels/moe_sort_by_expert.cl +155 -0
  733. data/vendor/ggml/src/ggml-opencl/kernels/mul.cl +152 -0
  734. data/vendor/ggml/src/ggml-opencl/kernels/mul_mat_f16_f32.cl +130 -0
  735. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_kq_kqv.cl +273 -0
  736. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_l4_lm.cl +146 -0
  737. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_f32_f32_l4_lm.cl +196 -0
  738. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_iq4_nl_f32_l4_lm.cl +171 -0
  739. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q1_0_f32_l4_lm.cl +156 -0
  740. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q4_0_f32_l4_lm.cl +163 -0
  741. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q4_1_f32_l4_lm.cl +165 -0
  742. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q4_k_f32_l4_lm.cl +189 -0
  743. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q5_0_f32_l4_lm.cl +173 -0
  744. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q5_1_f32_l4_lm.cl +175 -0
  745. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q5_k_f32_l4_lm.cl +202 -0
  746. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q6_k_f32_l4_lm.cl +158 -0
  747. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q8_0_f32_l4_lm.cl +154 -0
  748. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f16.cl +125 -0
  749. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32.cl +125 -0
  750. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_1row.cl +101 -0
  751. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_l4.cl +1237 -0
  752. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_mrow.cl +306 -0
  753. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f32_f32.cl +118 -0
  754. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_mxfp4_f32.cl +189 -0
  755. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_mxfp4_f32_flat.cl +176 -0
  756. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_q4_0_f32_8x_flat.cl +283 -0
  757. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_q8_0_f32.cl +140 -0
  758. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_q8_0_f32_flat.cl +222 -0
  759. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_iq4_nl_f32.cl +164 -0
  760. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_iq4_nl_f32_flat.cl +202 -0
  761. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_mxfp4_f32.cl +144 -0
  762. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_mxfp4_f32_flat.cl +167 -0
  763. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q1_0_f32.cl +141 -0
  764. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q1_0_f32_flat.cl +190 -0
  765. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32.cl +192 -0
  766. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_16x_flat.cl +307 -0
  767. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_8x_flat.cl +265 -0
  768. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_8x_flat.cl +272 -0
  769. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_v.cl +254 -0
  770. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_1_f32.cl +219 -0
  771. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_1_f32_flat.cl +229 -0
  772. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32.cl +182 -0
  773. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32_flat.cl +205 -0
  774. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_0_f32.cl +241 -0
  775. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_0_f32_flat.cl +243 -0
  776. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_1_f32.cl +243 -0
  777. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_1_f32_flat.cl +247 -0
  778. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_k_f32.cl +187 -0
  779. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_k_f32_flat.cl +244 -0
  780. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32.cl +194 -0
  781. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32_flat.cl +272 -0
  782. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q8_0_f32.cl +125 -0
  783. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q8_0_f32_flat.cl +202 -0
  784. data/vendor/ggml/src/ggml-opencl/kernels/neg.cl +125 -0
  785. data/vendor/ggml/src/ggml-opencl/kernels/norm.cl +164 -0
  786. data/vendor/ggml/src/ggml-opencl/kernels/pad.cl +39 -0
  787. data/vendor/ggml/src/ggml-opencl/kernels/quant_a_q8_1.cl +42 -0
  788. data/vendor/ggml/src/ggml-opencl/kernels/relu.cl +16 -0
  789. data/vendor/ggml/src/ggml-opencl/kernels/repeat.cl +38 -0
  790. data/vendor/ggml/src/ggml-opencl/kernels/rms_norm.cl +369 -0
  791. data/vendor/ggml/src/ggml-opencl/kernels/rope.cl +759 -0
  792. data/vendor/ggml/src/ggml-opencl/kernels/scale.cl +27 -0
  793. data/vendor/ggml/src/ggml-opencl/kernels/set_rows.cl +708 -0
  794. data/vendor/ggml/src/ggml-opencl/kernels/sigmoid.cl +29 -0
  795. data/vendor/ggml/src/ggml-opencl/kernels/silu.cl +30 -0
  796. data/vendor/ggml/src/ggml-opencl/kernels/softmax_4_f16.cl +108 -0
  797. data/vendor/ggml/src/ggml-opencl/kernels/softmax_4_f32.cl +108 -0
  798. data/vendor/ggml/src/ggml-opencl/kernels/softmax_f16.cl +107 -0
  799. data/vendor/ggml/src/ggml-opencl/kernels/softmax_f32.cl +107 -0
  800. data/vendor/ggml/src/ggml-opencl/kernels/softplus.cl +116 -0
  801. data/vendor/ggml/src/ggml-opencl/kernels/solve_tri.cl +51 -0
  802. data/vendor/ggml/src/ggml-opencl/kernels/sqr.cl +53 -0
  803. data/vendor/ggml/src/ggml-opencl/kernels/sqrt.cl +53 -0
  804. data/vendor/ggml/src/ggml-opencl/kernels/ssm_conv.cl +77 -0
  805. data/vendor/ggml/src/ggml-opencl/kernels/ssm_scan.cl +216 -0
  806. data/vendor/ggml/src/ggml-opencl/kernels/sub.cl +138 -0
  807. data/vendor/ggml/src/ggml-opencl/kernels/sum_rows.cl +140 -0
  808. data/vendor/ggml/src/ggml-opencl/kernels/tanh.cl +109 -0
  809. data/vendor/ggml/src/ggml-opencl/kernels/transpose.cl +143 -0
  810. data/vendor/ggml/src/ggml-opencl/kernels/tri.cl +32 -0
  811. data/vendor/ggml/src/ggml-opencl/kernels/tsembd.cl +48 -0
  812. data/vendor/ggml/src/ggml-opencl/kernels/upscale.cl +120 -0
  813. data/vendor/ggml/src/ggml-opencl/libdl.h +79 -0
  814. data/vendor/ggml/src/ggml-openvino/CMakeLists.txt +22 -0
  815. data/vendor/ggml/src/ggml-openvino/ggml-decoder.cpp +2057 -0
  816. data/vendor/ggml/src/ggml-openvino/ggml-decoder.h +417 -0
  817. data/vendor/ggml/src/ggml-openvino/ggml-openvino-extra.cpp +493 -0
  818. data/vendor/ggml/src/ggml-openvino/ggml-openvino-extra.h +216 -0
  819. data/vendor/ggml/src/ggml-openvino/ggml-openvino.cpp +1566 -0
  820. data/vendor/ggml/src/ggml-openvino/ggml-quants.cpp +1273 -0
  821. data/vendor/ggml/src/ggml-openvino/ggml-quants.h +186 -0
  822. data/vendor/ggml/src/ggml-openvino/model-cache.cpp +272 -0
  823. data/vendor/ggml/src/ggml-openvino/model-cache.h +56 -0
  824. data/vendor/ggml/src/ggml-openvino/openvino/decoder.h +133 -0
  825. data/vendor/ggml/src/ggml-openvino/openvino/frontend.cpp +27 -0
  826. data/vendor/ggml/src/ggml-openvino/openvino/frontend.h +23 -0
  827. data/vendor/ggml/src/ggml-openvino/openvino/input_model.cpp +17 -0
  828. data/vendor/ggml/src/ggml-openvino/openvino/input_model.h +29 -0
  829. data/vendor/ggml/src/ggml-openvino/openvino/node_context.h +171 -0
  830. data/vendor/ggml/src/ggml-openvino/openvino/op/add.cpp +45 -0
  831. data/vendor/ggml/src/ggml-openvino/openvino/op/add_id.cpp +76 -0
  832. data/vendor/ggml/src/ggml-openvino/openvino/op/argsort.cpp +47 -0
  833. data/vendor/ggml/src/ggml-openvino/openvino/op/clamp.cpp +33 -0
  834. data/vendor/ggml/src/ggml-openvino/openvino/op/concat.cpp +48 -0
  835. data/vendor/ggml/src/ggml-openvino/openvino/op/cont.cpp +40 -0
  836. data/vendor/ggml/src/ggml-openvino/openvino/op/cpy.cpp +295 -0
  837. data/vendor/ggml/src/ggml-openvino/openvino/op/cumsum.cpp +29 -0
  838. data/vendor/ggml/src/ggml-openvino/openvino/op/diag.cpp +58 -0
  839. data/vendor/ggml/src/ggml-openvino/openvino/op/div.cpp +146 -0
  840. data/vendor/ggml/src/ggml-openvino/openvino/op/fill.cpp +34 -0
  841. data/vendor/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp +241 -0
  842. data/vendor/ggml/src/ggml-openvino/openvino/op/gated_delta_net.cpp +327 -0
  843. data/vendor/ggml/src/ggml-openvino/openvino/op/gated_delta_net.hpp +65 -0
  844. data/vendor/ggml/src/ggml-openvino/openvino/op/gather_matmul.hpp +43 -0
  845. data/vendor/ggml/src/ggml-openvino/openvino/op/get_rows.cpp +142 -0
  846. data/vendor/ggml/src/ggml-openvino/openvino/op/glu_geglu.cpp +75 -0
  847. data/vendor/ggml/src/ggml-openvino/openvino/op/glu_geglu_quick.cpp +64 -0
  848. data/vendor/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp +110 -0
  849. data/vendor/ggml/src/ggml-openvino/openvino/op/im2col.cpp +120 -0
  850. data/vendor/ggml/src/ggml-openvino/openvino/op/l2_norm.cpp +61 -0
  851. data/vendor/ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp +283 -0
  852. data/vendor/ggml/src/ggml-openvino/openvino/op/mulmat.cpp +100 -0
  853. data/vendor/ggml/src/ggml-openvino/openvino/op/norm.cpp +58 -0
  854. data/vendor/ggml/src/ggml-openvino/openvino/op/pad.cpp +95 -0
  855. data/vendor/ggml/src/ggml-openvino/openvino/op/permute.cpp +147 -0
  856. data/vendor/ggml/src/ggml-openvino/openvino/op/pool_2d.cpp +53 -0
  857. data/vendor/ggml/src/ggml-openvino/openvino/op/repeat.cpp +48 -0
  858. data/vendor/ggml/src/ggml-openvino/openvino/op/reshape.cpp +113 -0
  859. data/vendor/ggml/src/ggml-openvino/openvino/op/rms_norm.cpp +81 -0
  860. data/vendor/ggml/src/ggml-openvino/openvino/op/roll.cpp +36 -0
  861. data/vendor/ggml/src/ggml-openvino/openvino/op/rope.cpp +303 -0
  862. data/vendor/ggml/src/ggml-openvino/openvino/op/scale.cpp +86 -0
  863. data/vendor/ggml/src/ggml-openvino/openvino/op/set.cpp +76 -0
  864. data/vendor/ggml/src/ggml-openvino/openvino/op/set_rows.cpp +95 -0
  865. data/vendor/ggml/src/ggml-openvino/openvino/op/softmax.cpp +166 -0
  866. data/vendor/ggml/src/ggml-openvino/openvino/op/solve_tri.cpp +108 -0
  867. data/vendor/ggml/src/ggml-openvino/openvino/op/sqr.cpp +35 -0
  868. data/vendor/ggml/src/ggml-openvino/openvino/op/ssm_conv.cpp +61 -0
  869. data/vendor/ggml/src/ggml-openvino/openvino/op/sum_rows.cpp +27 -0
  870. data/vendor/ggml/src/ggml-openvino/openvino/op/transpose.cpp +54 -0
  871. data/vendor/ggml/src/ggml-openvino/openvino/op/tri.cpp +82 -0
  872. data/vendor/ggml/src/ggml-openvino/openvino/op/unary_silu.cpp +27 -0
  873. data/vendor/ggml/src/ggml-openvino/openvino/op/unary_softplus.cpp +38 -0
  874. data/vendor/ggml/src/ggml-openvino/openvino/op/view.cpp +245 -0
  875. data/vendor/ggml/src/ggml-openvino/openvino/op_table.cpp +88 -0
  876. data/vendor/ggml/src/ggml-openvino/openvino/op_table.h +67 -0
  877. data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_conv.cpp +212 -0
  878. data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_conv.h +17 -0
  879. data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_sdpa.cpp +60 -0
  880. data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_sdpa.h +17 -0
  881. data/vendor/ggml/src/ggml-openvino/openvino/pass/mark_decompression_convert_constant_folding.h +29 -0
  882. data/vendor/ggml/src/ggml-openvino/openvino/pass/mark_dequantization_subgraph.h +44 -0
  883. data/vendor/ggml/src/ggml-openvino/openvino/pass/squeeze_matmul.cpp +58 -0
  884. data/vendor/ggml/src/ggml-openvino/openvino/pass/squeeze_matmul.h +17 -0
  885. data/vendor/ggml/src/ggml-openvino/openvino/rt_info/weightless_caching_attributes.hpp +41 -0
  886. data/vendor/ggml/src/ggml-openvino/openvino/translate_session.cpp +432 -0
  887. data/vendor/ggml/src/ggml-openvino/openvino/translate_session.h +29 -0
  888. data/vendor/ggml/src/ggml-openvino/openvino/utils.cpp +886 -0
  889. data/vendor/ggml/src/ggml-openvino/openvino/utils.h +88 -0
  890. data/vendor/ggml/src/ggml-openvino/utils.cpp +1496 -0
  891. data/vendor/ggml/src/ggml-openvino/utils.h +195 -0
  892. data/vendor/ggml/src/ggml-opt.cpp +1094 -0
  893. data/vendor/ggml/src/ggml-quants.c +5667 -0
  894. data/vendor/ggml/src/ggml-quants.h +115 -0
  895. data/vendor/ggml/src/ggml-rpc/CMakeLists.txt +49 -0
  896. data/vendor/ggml/src/ggml-rpc/ggml-rpc.cpp +2351 -0
  897. data/vendor/ggml/src/ggml-rpc/transport-apple.cpp +481 -0
  898. data/vendor/ggml/src/ggml-rpc/transport-apple.h +27 -0
  899. data/vendor/ggml/src/ggml-rpc/transport.cpp +739 -0
  900. data/vendor/ggml/src/ggml-rpc/transport.h +38 -0
  901. data/vendor/ggml/src/ggml-sycl/CMakeLists.txt +218 -0
  902. data/vendor/ggml/src/ggml-sycl/add-id.cpp +81 -0
  903. data/vendor/ggml/src/ggml-sycl/add-id.hpp +8 -0
  904. data/vendor/ggml/src/ggml-sycl/backend.hpp +52 -0
  905. data/vendor/ggml/src/ggml-sycl/base.hpp +36 -0
  906. data/vendor/ggml/src/ggml-sycl/binbcast.cpp +650 -0
  907. data/vendor/ggml/src/ggml-sycl/binbcast.hpp +69 -0
  908. data/vendor/ggml/src/ggml-sycl/col2im-1d.cpp +102 -0
  909. data/vendor/ggml/src/ggml-sycl/col2im-1d.hpp +8 -0
  910. data/vendor/ggml/src/ggml-sycl/common.cpp +153 -0
  911. data/vendor/ggml/src/ggml-sycl/common.hpp +1029 -0
  912. data/vendor/ggml/src/ggml-sycl/concat.cpp +509 -0
  913. data/vendor/ggml/src/ggml-sycl/concat.hpp +20 -0
  914. data/vendor/ggml/src/ggml-sycl/conv.cpp +101 -0
  915. data/vendor/ggml/src/ggml-sycl/conv.hpp +20 -0
  916. data/vendor/ggml/src/ggml-sycl/conv2d-dw.cpp +170 -0
  917. data/vendor/ggml/src/ggml-sycl/conv2d-dw.hpp +10 -0
  918. data/vendor/ggml/src/ggml-sycl/conv2d-transpose.cpp +125 -0
  919. data/vendor/ggml/src/ggml-sycl/conv2d-transpose.hpp +10 -0
  920. data/vendor/ggml/src/ggml-sycl/conv2d.cpp +150 -0
  921. data/vendor/ggml/src/ggml-sycl/conv2d.hpp +10 -0
  922. data/vendor/ggml/src/ggml-sycl/conv3d.cpp +224 -0
  923. data/vendor/ggml/src/ggml-sycl/conv3d.hpp +8 -0
  924. data/vendor/ggml/src/ggml-sycl/convert.cpp +877 -0
  925. data/vendor/ggml/src/ggml-sycl/convert.hpp +64 -0
  926. data/vendor/ggml/src/ggml-sycl/count-equal.cpp +79 -0
  927. data/vendor/ggml/src/ggml-sycl/count-equal.hpp +9 -0
  928. data/vendor/ggml/src/ggml-sycl/cpy.cpp +1433 -0
  929. data/vendor/ggml/src/ggml-sycl/cpy.hpp +537 -0
  930. data/vendor/ggml/src/ggml-sycl/cross_entropy_loss.cpp +255 -0
  931. data/vendor/ggml/src/ggml-sycl/cross_entropy_loss.hpp +7 -0
  932. data/vendor/ggml/src/ggml-sycl/cumsum.cpp +148 -0
  933. data/vendor/ggml/src/ggml-sycl/cumsum.hpp +5 -0
  934. data/vendor/ggml/src/ggml-sycl/dequantize.hpp +1681 -0
  935. data/vendor/ggml/src/ggml-sycl/diag.cpp +67 -0
  936. data/vendor/ggml/src/ggml-sycl/diag.hpp +5 -0
  937. data/vendor/ggml/src/ggml-sycl/dmmv.cpp +2227 -0
  938. data/vendor/ggml/src/ggml-sycl/dmmv.hpp +27 -0
  939. data/vendor/ggml/src/ggml-sycl/dpct/helper.hpp +3782 -0
  940. data/vendor/ggml/src/ggml-sycl/dsv4-hc.cpp +280 -0
  941. data/vendor/ggml/src/ggml-sycl/dsv4-hc.hpp +10 -0
  942. data/vendor/ggml/src/ggml-sycl/element_wise.cpp +1432 -0
  943. data/vendor/ggml/src/ggml-sycl/element_wise.hpp +135 -0
  944. data/vendor/ggml/src/ggml-sycl/esimd.hpp +589 -0
  945. data/vendor/ggml/src/ggml-sycl/fattn-buffers.cpp +56 -0
  946. data/vendor/ggml/src/ggml-sycl/fattn-buffers.hpp +63 -0
  947. data/vendor/ggml/src/ggml-sycl/fattn-common.hpp +1185 -0
  948. data/vendor/ggml/src/ggml-sycl/fattn-mkl.cpp +689 -0
  949. data/vendor/ggml/src/ggml-sycl/fattn-onednn.cpp +450 -0
  950. data/vendor/ggml/src/ggml-sycl/fattn-onednn.hpp +18 -0
  951. data/vendor/ggml/src/ggml-sycl/fattn-tile.cpp +59 -0
  952. data/vendor/ggml/src/ggml-sycl/fattn-tile.hpp +1246 -0
  953. data/vendor/ggml/src/ggml-sycl/fattn-vec.hpp +684 -0
  954. data/vendor/ggml/src/ggml-sycl/fattn.cpp +452 -0
  955. data/vendor/ggml/src/ggml-sycl/fattn.hpp +42 -0
  956. data/vendor/ggml/src/ggml-sycl/fill.cpp +55 -0
  957. data/vendor/ggml/src/ggml-sycl/fill.hpp +5 -0
  958. data/vendor/ggml/src/ggml-sycl/fusion.cpp +212 -0
  959. data/vendor/ggml/src/ggml-sycl/fusion.hpp +17 -0
  960. data/vendor/ggml/src/ggml-sycl/fwht.cpp +119 -0
  961. data/vendor/ggml/src/ggml-sycl/fwht.hpp +12 -0
  962. data/vendor/ggml/src/ggml-sycl/gated_delta_net.cpp +366 -0
  963. data/vendor/ggml/src/ggml-sycl/gated_delta_net.hpp +19 -0
  964. data/vendor/ggml/src/ggml-sycl/gemm.hpp +93 -0
  965. data/vendor/ggml/src/ggml-sycl/getrows.cpp +368 -0
  966. data/vendor/ggml/src/ggml-sycl/getrows.hpp +20 -0
  967. data/vendor/ggml/src/ggml-sycl/ggml-sycl.cpp +7028 -0
  968. data/vendor/ggml/src/ggml-sycl/gla.cpp +106 -0
  969. data/vendor/ggml/src/ggml-sycl/gla.hpp +8 -0
  970. data/vendor/ggml/src/ggml-sycl/im2col.cpp +400 -0
  971. data/vendor/ggml/src/ggml-sycl/im2col.hpp +23 -0
  972. data/vendor/ggml/src/ggml-sycl/lightning-indexer.cpp +197 -0
  973. data/vendor/ggml/src/ggml-sycl/lightning-indexer.hpp +8 -0
  974. data/vendor/ggml/src/ggml-sycl/mem.cpp +162 -0
  975. data/vendor/ggml/src/ggml-sycl/mem.hpp +16 -0
  976. data/vendor/ggml/src/ggml-sycl/mmq.cpp +3030 -0
  977. data/vendor/ggml/src/ggml-sycl/mmq.hpp +33 -0
  978. data/vendor/ggml/src/ggml-sycl/mmvq.cpp +3033 -0
  979. data/vendor/ggml/src/ggml-sycl/mmvq.hpp +76 -0
  980. data/vendor/ggml/src/ggml-sycl/norm.cpp +963 -0
  981. data/vendor/ggml/src/ggml-sycl/norm.hpp +32 -0
  982. data/vendor/ggml/src/ggml-sycl/opt-step.cpp +131 -0
  983. data/vendor/ggml/src/ggml-sycl/opt-step.hpp +6 -0
  984. data/vendor/ggml/src/ggml-sycl/outprod.cpp +83 -0
  985. data/vendor/ggml/src/ggml-sycl/outprod.hpp +10 -0
  986. data/vendor/ggml/src/ggml-sycl/pad.cpp +97 -0
  987. data/vendor/ggml/src/ggml-sycl/pad.hpp +24 -0
  988. data/vendor/ggml/src/ggml-sycl/pad_reflect_1d.cpp +100 -0
  989. data/vendor/ggml/src/ggml-sycl/pad_reflect_1d.hpp +10 -0
  990. data/vendor/ggml/src/ggml-sycl/pool.cpp +185 -0
  991. data/vendor/ggml/src/ggml-sycl/pool.hpp +22 -0
  992. data/vendor/ggml/src/ggml-sycl/presets.hpp +79 -0
  993. data/vendor/ggml/src/ggml-sycl/quantize.hpp +133 -0
  994. data/vendor/ggml/src/ggml-sycl/quants.hpp +204 -0
  995. data/vendor/ggml/src/ggml-sycl/repeat_back.cpp +76 -0
  996. data/vendor/ggml/src/ggml-sycl/repeat_back.hpp +8 -0
  997. data/vendor/ggml/src/ggml-sycl/roll.cpp +122 -0
  998. data/vendor/ggml/src/ggml-sycl/roll.hpp +20 -0
  999. data/vendor/ggml/src/ggml-sycl/rope.cpp +651 -0
  1000. data/vendor/ggml/src/ggml-sycl/rope.hpp +26 -0
  1001. data/vendor/ggml/src/ggml-sycl/set.cpp +73 -0
  1002. data/vendor/ggml/src/ggml-sycl/set.hpp +5 -0
  1003. data/vendor/ggml/src/ggml-sycl/set_rows.cpp +577 -0
  1004. data/vendor/ggml/src/ggml-sycl/set_rows.hpp +8 -0
  1005. data/vendor/ggml/src/ggml-sycl/softmax.cpp +425 -0
  1006. data/vendor/ggml/src/ggml-sycl/softmax.hpp +24 -0
  1007. data/vendor/ggml/src/ggml-sycl/solve_tri.cpp +172 -0
  1008. data/vendor/ggml/src/ggml-sycl/solve_tri.hpp +8 -0
  1009. data/vendor/ggml/src/ggml-sycl/ssm_conv.cpp +136 -0
  1010. data/vendor/ggml/src/ggml-sycl/ssm_conv.hpp +5 -0
  1011. data/vendor/ggml/src/ggml-sycl/ssm_scan.cpp +170 -0
  1012. data/vendor/ggml/src/ggml-sycl/ssm_scan.hpp +5 -0
  1013. data/vendor/ggml/src/ggml-sycl/sycl_hw.cpp +67 -0
  1014. data/vendor/ggml/src/ggml-sycl/sycl_hw.hpp +38 -0
  1015. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq112-dv112.cpp +5 -0
  1016. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq128-dv128.cpp +5 -0
  1017. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq256-dv256.cpp +5 -0
  1018. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq40-dv40.cpp +5 -0
  1019. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq512-dv512.cpp +6 -0
  1020. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq576-dv512.cpp +5 -0
  1021. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq64-dv64.cpp +5 -0
  1022. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq72-dv72.cpp +5 -0
  1023. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq80-dv80.cpp +5 -0
  1024. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq96-dv96.cpp +5 -0
  1025. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-f16.cpp +8 -0
  1026. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q4_0.cpp +8 -0
  1027. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q4_1.cpp +8 -0
  1028. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q5_0.cpp +8 -0
  1029. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q5_1.cpp +8 -0
  1030. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q8_0.cpp +8 -0
  1031. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-f16.cpp +8 -0
  1032. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q4_0.cpp +8 -0
  1033. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q4_1.cpp +8 -0
  1034. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q5_0.cpp +8 -0
  1035. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q5_1.cpp +8 -0
  1036. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q8_0.cpp +8 -0
  1037. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-f16.cpp +8 -0
  1038. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q4_0.cpp +8 -0
  1039. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q4_1.cpp +8 -0
  1040. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q5_0.cpp +8 -0
  1041. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q5_1.cpp +8 -0
  1042. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q8_0.cpp +8 -0
  1043. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-f16.cpp +8 -0
  1044. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q4_0.cpp +8 -0
  1045. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q4_1.cpp +8 -0
  1046. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q5_0.cpp +8 -0
  1047. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q5_1.cpp +8 -0
  1048. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q8_0.cpp +8 -0
  1049. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-f16.cpp +8 -0
  1050. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q4_0.cpp +8 -0
  1051. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q4_1.cpp +8 -0
  1052. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q5_0.cpp +8 -0
  1053. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q5_1.cpp +8 -0
  1054. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q8_0.cpp +8 -0
  1055. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-f16.cpp +8 -0
  1056. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q4_0.cpp +8 -0
  1057. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q4_1.cpp +8 -0
  1058. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q5_0.cpp +8 -0
  1059. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q5_1.cpp +8 -0
  1060. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q8_0.cpp +8 -0
  1061. data/vendor/ggml/src/ggml-sycl/topk-moe.cpp +620 -0
  1062. data/vendor/ggml/src/ggml-sycl/topk-moe.hpp +12 -0
  1063. data/vendor/ggml/src/ggml-sycl/tsembd.cpp +73 -0
  1064. data/vendor/ggml/src/ggml-sycl/tsembd.hpp +20 -0
  1065. data/vendor/ggml/src/ggml-sycl/type.hpp +112 -0
  1066. data/vendor/ggml/src/ggml-sycl/upscale.cpp +410 -0
  1067. data/vendor/ggml/src/ggml-sycl/upscale.hpp +9 -0
  1068. data/vendor/ggml/src/ggml-sycl/vecdotq.hpp +1726 -0
  1069. data/vendor/ggml/src/ggml-sycl/wkv.cpp +293 -0
  1070. data/vendor/ggml/src/ggml-sycl/wkv.hpp +10 -0
  1071. data/vendor/ggml/src/ggml-threading.cpp +12 -0
  1072. data/vendor/ggml/src/ggml-threading.h +14 -0
  1073. data/vendor/ggml/src/ggml-version.h.in +4 -0
  1074. data/vendor/ggml/src/ggml-virtgpu/CMakeLists.txt +70 -0
  1075. data/vendor/ggml/src/ggml-virtgpu/apir_cs_ggml-rpc-front.cpp +87 -0
  1076. data/vendor/ggml/src/ggml-virtgpu/backend/CMakeLists.txt +21 -0
  1077. data/vendor/ggml/src/ggml-virtgpu/backend/apir_cs_ggml-rpc-back.cpp +115 -0
  1078. data/vendor/ggml/src/ggml-virtgpu/backend/backend-convert.h +13 -0
  1079. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-backend.cpp +102 -0
  1080. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-buffer-type.cpp +105 -0
  1081. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-buffer.cpp +179 -0
  1082. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-device.cpp +149 -0
  1083. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched.cpp +51 -0
  1084. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched.gen.h +73 -0
  1085. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched.h +27 -0
  1086. data/vendor/ggml/src/ggml-virtgpu/backend/backend-virgl-apir.h +32 -0
  1087. data/vendor/ggml/src/ggml-virtgpu/backend/backend.cpp +144 -0
  1088. data/vendor/ggml/src/ggml-virtgpu/backend/shared/api_remoting.h +95 -0
  1089. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_backend.gen.h +94 -0
  1090. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_backend.h +50 -0
  1091. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_cs.h +378 -0
  1092. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_cs_ggml.h +232 -0
  1093. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_cs_rpc.h +58 -0
  1094. data/vendor/ggml/src/ggml-virtgpu/ggml-backend-buffer-type.cpp +81 -0
  1095. data/vendor/ggml/src/ggml-virtgpu/ggml-backend-buffer.cpp +123 -0
  1096. data/vendor/ggml/src/ggml-virtgpu/ggml-backend-device.cpp +160 -0
  1097. data/vendor/ggml/src/ggml-virtgpu/ggml-backend-reg.cpp +213 -0
  1098. data/vendor/ggml/src/ggml-virtgpu/ggml-backend.cpp +72 -0
  1099. data/vendor/ggml/src/ggml-virtgpu/ggml-remoting.h +71 -0
  1100. data/vendor/ggml/src/ggml-virtgpu/ggmlremoting_functions.yaml +166 -0
  1101. data/vendor/ggml/src/ggml-virtgpu/include/apir_hw.h +9 -0
  1102. data/vendor/ggml/src/ggml-virtgpu/regenerate_remoting.py +333 -0
  1103. data/vendor/ggml/src/ggml-virtgpu/virtgpu-apir.h +15 -0
  1104. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-backend.cpp +58 -0
  1105. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-buffer-type.cpp +110 -0
  1106. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-buffer.cpp +173 -0
  1107. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-device.cpp +194 -0
  1108. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-impl.h +36 -0
  1109. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward.gen.h +54 -0
  1110. data/vendor/ggml/src/ggml-virtgpu/virtgpu-shm.cpp +99 -0
  1111. data/vendor/ggml/src/ggml-virtgpu/virtgpu-shm.h +23 -0
  1112. data/vendor/ggml/src/ggml-virtgpu/virtgpu-utils.cpp +179 -0
  1113. data/vendor/ggml/src/ggml-virtgpu/virtgpu-utils.h +86 -0
  1114. data/vendor/ggml/src/ggml-virtgpu/virtgpu.cpp +545 -0
  1115. data/vendor/ggml/src/ggml-virtgpu/virtgpu.h +115 -0
  1116. data/vendor/ggml/src/ggml-vulkan/CMakeLists.txt +254 -0
  1117. data/vendor/ggml/src/ggml-vulkan/cmake/host-toolchain.cmake.in +15 -0
  1118. data/vendor/ggml/src/ggml-vulkan/ggml-vulkan.cpp +20349 -0
  1119. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/CMakeLists.txt +43 -0
  1120. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/acc.comp +37 -0
  1121. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/add.comp +69 -0
  1122. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/add1.comp +28 -0
  1123. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/add_id.comp +42 -0
  1124. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/arange.comp +20 -0
  1125. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/argmax.comp +60 -0
  1126. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/argsort.comp +86 -0
  1127. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/argsort_large.comp +114 -0
  1128. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/col2im_1d.comp +61 -0
  1129. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/concat.comp +41 -0
  1130. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/contig_copy.comp +53 -0
  1131. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv2d_dw.comp +105 -0
  1132. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv2d_mm.comp +473 -0
  1133. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv3d_mm.comp +424 -0
  1134. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv_transpose_1d.comp +98 -0
  1135. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy.comp +25 -0
  1136. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_from_quant.comp +51 -0
  1137. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_to_quant.comp +350 -0
  1138. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_transpose.comp +67 -0
  1139. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_transpose_02.comp +61 -0
  1140. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/count_equal.comp +31 -0
  1141. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/count_experts.comp +135 -0
  1142. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cross_entropy_loss.comp +78 -0
  1143. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cross_entropy_loss_back.comp +75 -0
  1144. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cumsum.comp +83 -0
  1145. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cumsum_multipass1.comp +60 -0
  1146. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cumsum_multipass2.comp +66 -0
  1147. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_f32.comp +20 -0
  1148. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.glsl +741 -0
  1149. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl +1466 -0
  1150. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_head.glsl +13 -0
  1151. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_m.comp +42 -0
  1152. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_s.comp +35 -0
  1153. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_s.comp +44 -0
  1154. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xs.comp +43 -0
  1155. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xxs.comp +49 -0
  1156. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_s.comp +40 -0
  1157. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_xxs.comp +51 -0
  1158. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_nl.comp +32 -0
  1159. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_xs.comp +34 -0
  1160. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_mxfp4.comp +32 -0
  1161. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_nvfp4.comp +32 -0
  1162. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q1_0.comp +29 -0
  1163. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q2_0.comp +29 -0
  1164. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q2_k.comp +34 -0
  1165. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q3_k.comp +42 -0
  1166. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_0.comp +30 -0
  1167. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_1.comp +32 -0
  1168. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_k.comp +68 -0
  1169. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_0.comp +34 -0
  1170. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_1.comp +35 -0
  1171. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_k.comp +70 -0
  1172. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q6_k.comp +33 -0
  1173. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q8_0.comp +42 -0
  1174. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_tq2_0.comp +31 -0
  1175. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/diag.comp +28 -0
  1176. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/diag_mask_inf.comp +34 -0
  1177. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/div.comp +27 -0
  1178. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dot_product_funcs.glsl +27 -0
  1179. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/fa_types.glsl +55 -0
  1180. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/bfloat16.comp +7 -0
  1181. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat.comp +7 -0
  1182. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat2.comp +7 -0
  1183. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat2_decode_vector.comp +7 -0
  1184. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/float_e2m1.comp +7 -0
  1185. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/float_e4m3.comp +7 -0
  1186. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/integer_dot.comp +7 -0
  1187. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/fill.comp +19 -0
  1188. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp +760 -0
  1189. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.glsl +223 -0
  1190. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp +647 -0
  1191. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp +487 -0
  1192. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_dequant.glsl +146 -0
  1193. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_mask_opt.comp +162 -0
  1194. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_mmq_funcs.glsl +203 -0
  1195. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_split_k_reduce.comp +121 -0
  1196. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/fwht.comp +115 -0
  1197. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/gated_delta_net.comp +189 -0
  1198. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/geglu.comp +13 -0
  1199. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/geglu_erf.comp +27 -0
  1200. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/geglu_quick.comp +11 -0
  1201. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/generic_binary_head.glsl +65 -0
  1202. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/generic_head.glsl +11 -0
  1203. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/generic_unary_head.glsl +75 -0
  1204. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/get_rows.comp +42 -0
  1205. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_back.comp +25 -0
  1206. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_quant.comp +51 -0
  1207. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/gla.comp +82 -0
  1208. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/glu_head.glsl +43 -0
  1209. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/glu_main.glsl +35 -0
  1210. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/group_norm.comp +66 -0
  1211. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/im2col.comp +138 -0
  1212. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/im2col_3d.comp +124 -0
  1213. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/l2_norm.comp +41 -0
  1214. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/lightning_indexer.comp +151 -0
  1215. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/log.comp +17 -0
  1216. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul.comp +27 -0
  1217. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_split_k_reduce.comp +48 -0
  1218. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec.comp +264 -0
  1219. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_base.glsl +230 -0
  1220. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iface.glsl +35 -0
  1221. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_m.comp +132 -0
  1222. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_s.comp +95 -0
  1223. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_s.comp +90 -0
  1224. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xs.comp +105 -0
  1225. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xxs.comp +87 -0
  1226. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_s.comp +93 -0
  1227. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_xxs.comp +88 -0
  1228. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_nc.comp +124 -0
  1229. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_p021.comp +156 -0
  1230. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q2_k.comp +128 -0
  1231. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q3_k.comp +132 -0
  1232. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q4_k.comp +134 -0
  1233. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q5_k.comp +165 -0
  1234. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q6_k.comp +130 -0
  1235. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_tq2_0.comp +102 -0
  1236. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq.comp +141 -0
  1237. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq_funcs.glsl +564 -0
  1238. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp +475 -0
  1239. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp +690 -0
  1240. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl +670 -0
  1241. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_id_funcs.glsl +89 -0
  1242. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp +317 -0
  1243. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_funcs.glsl +488 -0
  1244. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_shmem_types.glsl +99 -0
  1245. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/multi_add.comp +194 -0
  1246. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/norm.comp +44 -0
  1247. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_adamw.comp +42 -0
  1248. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_sgd.comp +22 -0
  1249. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/out_prod.comp +59 -0
  1250. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pad.comp +64 -0
  1251. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pad_reflect_1d.comp +43 -0
  1252. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pool1d.comp +65 -0
  1253. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pool2d.comp +74 -0
  1254. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/quantize_q8_1.comp +127 -0
  1255. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/reglu.comp +9 -0
  1256. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/repeat.comp +26 -0
  1257. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/repeat_back.comp +37 -0
  1258. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp +150 -0
  1259. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_back.comp +55 -0
  1260. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_partials.comp +65 -0
  1261. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/roll.comp +46 -0
  1262. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_funcs.glsl +218 -0
  1263. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_head.glsl +19 -0
  1264. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_multi.comp +17 -0
  1265. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_neox.comp +17 -0
  1266. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_norm.comp +17 -0
  1267. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_params.glsl +35 -0
  1268. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_vision.comp +17 -0
  1269. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/scale.comp +24 -0
  1270. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/silu_back.comp +26 -0
  1271. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/snake.comp +49 -0
  1272. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max.comp +195 -0
  1273. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_back.comp +54 -0
  1274. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large1.comp +62 -0
  1275. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large2.comp +79 -0
  1276. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large3.comp +65 -0
  1277. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large_common.glsl +53 -0
  1278. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/solve_tri.comp +81 -0
  1279. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/ssm_conv.comp +60 -0
  1280. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/ssm_scan.comp +134 -0
  1281. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/sub.comp +29 -0
  1282. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.comp +47 -0
  1283. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.glsl +19 -0
  1284. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/swiglu.comp +9 -0
  1285. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/swiglu_clamp.comp +12 -0
  1286. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/swiglu_oai.comp +14 -0
  1287. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/timestep_embedding.comp +42 -0
  1288. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_argsort.comp +118 -0
  1289. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_moe.comp +221 -0
  1290. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_nary_search.comp +246 -0
  1291. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_radix_select.comp +144 -0
  1292. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/tri.comp +42 -0
  1293. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl +1938 -0
  1294. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/unary.comp +168 -0
  1295. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/upscale.comp +178 -0
  1296. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/utils.glsl +37 -0
  1297. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +1367 -0
  1298. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/wkv6.comp +87 -0
  1299. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/wkv7.comp +91 -0
  1300. data/vendor/ggml/src/ggml-webgpu/CMakeLists.txt +83 -0
  1301. data/vendor/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp +3457 -0
  1302. data/vendor/ggml/src/ggml-webgpu/ggml-webgpu.cpp +4790 -0
  1303. data/vendor/ggml/src/ggml-webgpu/pre_wgsl.hpp +808 -0
  1304. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/add_id.wgsl +64 -0
  1305. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/argmax.wgsl +72 -0
  1306. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/argsort.wgsl +102 -0
  1307. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/argsort_merge.wgsl +134 -0
  1308. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/binary.wgsl +142 -0
  1309. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/common_decls.tmpl +967 -0
  1310. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/concat.wgsl +92 -0
  1311. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/conv2d.wgsl +127 -0
  1312. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/conv2d_dw.wgsl +100 -0
  1313. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/cpy.wgsl +84 -0
  1314. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/cumsum.wgsl +66 -0
  1315. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/embed_wgsl.py +89 -0
  1316. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn.wgsl +408 -0
  1317. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_decls.tmpl +134 -0
  1318. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_staging.tmpl +136 -0
  1319. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_tile.wgsl +240 -0
  1320. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_blk.wgsl +101 -0
  1321. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_reduce.wgsl +84 -0
  1322. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_split.wgsl +503 -0
  1323. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/gated_delta_net.wgsl +149 -0
  1324. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/get_rows.wgsl +794 -0
  1325. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/glu.wgsl +178 -0
  1326. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/im2col.wgsl +77 -0
  1327. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/memset.wgsl +40 -0
  1328. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_decls.tmpl +1052 -0
  1329. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id.wgsl +195 -0
  1330. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id_gather.wgsl +52 -0
  1331. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id_vec.wgsl +154 -0
  1332. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_reg_tile.wgsl +159 -0
  1333. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_subgroup_matrix.wgsl +210 -0
  1334. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec.wgsl +167 -0
  1335. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec_acc.tmpl +1560 -0
  1336. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec_q_acc.tmpl +297 -0
  1337. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/pad.wgsl +86 -0
  1338. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/quant_inner_loops.tmpl +21 -0
  1339. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/quantize_q8.wgsl +178 -0
  1340. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/repeat.wgsl +70 -0
  1341. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/rms_norm_mul.wgsl +151 -0
  1342. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/rope.wgsl +228 -0
  1343. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/row_norm.wgsl +152 -0
  1344. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/scale.wgsl +65 -0
  1345. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/set.wgsl +109 -0
  1346. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/set_rows.wgsl +108 -0
  1347. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/set_rows_quant.wgsl +224 -0
  1348. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/soft_max.wgsl +210 -0
  1349. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/solve_tri.wgsl +120 -0
  1350. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/ssm_conv.wgsl +65 -0
  1351. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/ssm_scan.wgsl +247 -0
  1352. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/sum_rows.wgsl +55 -0
  1353. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/unary.wgsl +213 -0
  1354. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/upscale.wgsl +240 -0
  1355. data/vendor/ggml/src/ggml-zdnn/CMakeLists.txt +36 -0
  1356. data/vendor/ggml/src/ggml-zdnn/common.hpp +59 -0
  1357. data/vendor/ggml/src/ggml-zdnn/ggml-zdnn.cpp +638 -0
  1358. data/vendor/ggml/src/ggml-zdnn/mmf.cpp +80 -0
  1359. data/vendor/ggml/src/ggml-zdnn/mmf.hpp +12 -0
  1360. data/vendor/ggml/src/ggml-zdnn/utils.cpp +79 -0
  1361. data/vendor/ggml/src/ggml-zdnn/utils.hpp +19 -0
  1362. data/vendor/ggml/src/ggml-zendnn/CMakeLists.txt +91 -0
  1363. data/vendor/ggml/src/ggml-zendnn/ggml-zendnn.cpp +838 -0
  1364. data/vendor/ggml/src/ggml.c +8088 -0
  1365. data/vendor/ggml/src/ggml.cpp +26 -0
  1366. data/vendor/ggml/src/gguf.cpp +1706 -0
  1367. metadata +1459 -0
@@ -0,0 +1,2057 @@
1
+ #include "ggml-decoder.h"
2
+
3
+ #include "ggml-impl.h"
4
+ #include "ggml-openvino-extra.h"
5
+ #include "ggml-openvino.h"
6
+ #include "ggml-quants.h"
7
+ #include "ggml.h"
8
+ #include "utils.h"
9
+
10
+ #include <algorithm>
11
+ #include <cassert>
12
+ #include <cstddef>
13
+ #include <cstdint>
14
+ #include <cstdlib>
15
+ #include <fstream>
16
+ #include <iomanip>
17
+ #include <map>
18
+ #include <memory>
19
+ #include <mutex>
20
+ #include <openvino/core/dimension.hpp>
21
+ #include <openvino/core/except.hpp>
22
+ #include <openvino/core/node.hpp>
23
+ #include <openvino/core/partial_shape.hpp>
24
+ #include <openvino/core/type/bfloat16.hpp>
25
+ #include <openvino/core/type/element_type.hpp>
26
+ #include <openvino/core/type/float16.hpp>
27
+ #include <openvino/op/constant.hpp>
28
+ #include <openvino/op/convert.hpp>
29
+ #include <openvino/runtime/tensor.hpp>
30
+ #include <ostream>
31
+ #include <set>
32
+ #include <stdexcept>
33
+ #include <string>
34
+ #include <cstring>
35
+ #include <unordered_map>
36
+ #include <vector>
37
+
38
+ GgmlOvDecoder::GgmlOvDecoder(ggml_cgraph * cgraph,
39
+ ModelParams & model_params,
40
+ ComputeParams & compute_params,
41
+ std::map<std::string, std::shared_ptr<ov::Node>> & model_weights,
42
+ bool is_static,
43
+ bool is_stateful,
44
+ bool model_is_splitted,
45
+ bool is_prefill,
46
+ int prefill_chunk_size) :
47
+ m_is_static(is_static),
48
+ m_is_stateful(is_stateful),
49
+ m_is_prefill(is_prefill),
50
+ m_naive(false),
51
+ m_prefill_chunk_size(prefill_chunk_size),
52
+ m_model_is_splitted(model_is_splitted),
53
+ m_cgraph(cgraph),
54
+ m_model_weights(model_weights),
55
+ m_model_params(model_params),
56
+ m_compute_params(compute_params) {
57
+ static bool printed_address_map = false;
58
+ if (!printed_address_map) {
59
+ if (ggml_openvino_getenv_int("GGML_OPENVINO_PRINT_CGRAPH_TENSOR_ADDRESS")) {
60
+ printed_address_map = true;
61
+ print_tensor_address_map(cgraph);
62
+ }
63
+ }
64
+
65
+ validate_cgraph();
66
+
67
+ set_input_output();
68
+ compute_node_dynamic_dims();
69
+ compute_model_inputs();
70
+ compute_model_outputs();
71
+
72
+ for (int node_n = 0; node_n < cgraph->n_nodes; node_n++) {
73
+ m_node_info_list[node_n].node_op_case = compute_op_case(m_node_info_list[node_n].node);
74
+ m_node_info_list[node_n].node_op_type = compute_op_type(m_node_info_list[node_n].node);
75
+ }
76
+
77
+ add_extra_inputs();
78
+ }
79
+
80
+ void GgmlOvDecoder::update_io(ggml_cgraph * cgraph) {
81
+ m_cgraph = cgraph;
82
+ m_model_inputs.clear();
83
+ m_model_outputs.clear();
84
+ m_node_info_list.clear();
85
+ set_input_output();
86
+ compute_model_inputs();
87
+ compute_model_outputs();
88
+ }
89
+
90
+ GgmlOvDecoder::GgmlOvDecoder(ggml_cgraph * cgraph, std::map<std::string, std::shared_ptr<ov::Node>> & model_weights) {
91
+ m_cgraph = cgraph;
92
+ m_model_weights = model_weights;
93
+ m_naive = true;
94
+ set_input_output();
95
+ compute_model_inputs();
96
+ compute_model_outputs();
97
+ for (int node_n = 0; node_n < cgraph->n_nodes; node_n++) {
98
+ m_node_info_list[node_n].node_op_case = compute_op_case(m_node_info_list[node_n].node);
99
+ m_node_info_list[node_n].node_op_type = compute_op_type(m_node_info_list[node_n].node);
100
+ }
101
+ }
102
+
103
+ namespace {
104
+ bool is_inplace_op(const ggml_tensor * node) {
105
+ return node->op == GGML_OP_SET_ROWS || node->op == GGML_OP_CPY || (node->op == GGML_OP_SCALE && node->view_src);
106
+ }
107
+
108
+ bool is_same_shape(const ggml_tensor * a, const ggml_tensor * b) {
109
+ for (int i = 0; i < GGML_MAX_DIMS; i++) {
110
+ if (a->ne[i] != b->ne[i]) {
111
+ return false;
112
+ }
113
+ }
114
+ return true;
115
+ }
116
+
117
+ bool is_conv_states_all_tensor(const ggml_tensor * tensor) {
118
+ return tensor != nullptr && strncmp(tensor->name, "conv_states_all", strlen("conv_states_all")) == 0;
119
+ }
120
+
121
+ // CPY writing the tail of conv_input (the concat of the previous conv state and the new tokens)
122
+ // back into a slot block of the recurrent state cache. Detected structurally because the rollback
123
+ // variant (cparams.n_rs_seq > 0) emits one such CPY per snapshot slot without naming them.
124
+ bool is_conv_state_writeback(const ggml_tensor * node) {
125
+ return node->op == GGML_OP_CPY && node->view_src != nullptr && GgmlOvDecoder::is_kvcache(node->view_src, nullptr) &&
126
+ node->src[0] != nullptr && node->src[0]->op == GGML_OP_VIEW && node->src[0]->src[0] != nullptr &&
127
+ node->src[0]->src[0]->op == GGML_OP_CONCAT && node->src[1] != nullptr && node->src[1]->op == GGML_OP_VIEW &&
128
+ node->src[1]->view_src == node->view_src;
129
+ }
130
+
131
+ // MoE expert aggregation (build_moe_ffn in llama-graph.cpp): each expert plane is
132
+ // `ggml_view_2d(experts, n_embd, n_tokens, experts->nb[2], i*experts->nb[1])` and the planes
133
+ // are summed with a chain of ADDs: moe_out = ((view_0 + view_1) + view_2) + ... + view_{n-1}.
134
+ // Detected structurally by walking the ADD chain and checking every leaf is a same-shape,
135
+ // same-stride VIEW of one common base tensor, indexed by a distinct expert-plane offset, and
136
+ // that the chain covers every plane of that base (leaf count == base->ne[1]). Only the
137
+ // outermost ADD of the chain satisfies this (inner ADDs see fewer leaves than base->ne[1]).
138
+ bool is_moe_expert_sum_add(const ggml_tensor * node) {
139
+ std::vector<const ggml_tensor *> leaves;
140
+ const ggml_tensor * cur = node;
141
+ while (cur->op == GGML_OP_ADD) {
142
+ if (cur->src[0] == nullptr || cur->src[1] == nullptr) {
143
+ return false;
144
+ }
145
+ leaves.push_back(cur->src[1]);
146
+ cur = cur->src[0];
147
+ }
148
+ leaves.push_back(cur);
149
+
150
+ const ggml_tensor * base = nullptr;
151
+ std::set<int64_t> plane_indices;
152
+ for (const ggml_tensor * leaf : leaves) {
153
+ if (leaf->op != GGML_OP_VIEW || leaf->src[0] == nullptr) {
154
+ return false;
155
+ }
156
+ const ggml_tensor * leaf_base = leaf->src[0];
157
+ if (base == nullptr) {
158
+ base = leaf_base;
159
+ } else if (leaf_base != base) {
160
+ return false;
161
+ }
162
+ if (leaf->ne[0] != base->ne[0] || leaf->ne[1] != base->ne[2] || leaf->ne[2] != 1 || leaf->ne[3] != 1 ||
163
+ leaf->nb[1] != base->nb[2]) {
164
+ return false;
165
+ }
166
+ if (base->nb[1] == 0 || leaf->view_offs % base->nb[1] != 0) {
167
+ return false;
168
+ }
169
+ int64_t plane = static_cast<int64_t>(leaf->view_offs / base->nb[1]);
170
+ if (plane < 0 || plane >= base->ne[1] || !plane_indices.insert(plane).second) {
171
+ return false;
172
+ }
173
+ }
174
+
175
+ return base != nullptr && base->ne[1] > 1 && plane_indices.size() == static_cast<size_t>(base->ne[1]);
176
+ }
177
+ } // namespace
178
+
179
+ static std::string get_tensor_ov_name(const ggml_cgraph * cgraph, const ggml_tensor * tensor) {
180
+ if (tensor == nullptr) {
181
+ return "";
182
+ }
183
+ const size_t hash_pos = ggml_hash_find(&cgraph->visited_hash_set, tensor);
184
+ if (((tensor->flags & GGML_TENSOR_FLAG_COMPUTE) || GgmlOvDecoder::is_kvcache(tensor, nullptr)) &&
185
+ hash_pos != GGML_HASHSET_FULL && ggml_bitset_get(cgraph->visited_hash_set.used, hash_pos)) {
186
+ return std::string(tensor->name) + "#" + std::to_string(hash_pos);
187
+ }
188
+ return tensor->name;
189
+ }
190
+
191
+ static std::string get_tensor_graph_input_ov_name(const GgmlOvDecoder * decoder,
192
+ const ggml_cgraph * cgraph,
193
+ const ggml_tensor * tensor,
194
+ const ggml_tensor * op) {
195
+ if (GgmlOvDecoder::is_inp_pos(tensor, op)) {
196
+ return "inp_pos";
197
+ }
198
+ if (GgmlOvDecoder::is_inp_emb(tensor, op)) {
199
+ return "embd";
200
+ }
201
+ if (decoder->is_stateful() && GgmlOvDecoder::is_inp_mask(tensor, op)) {
202
+ return std::string(tensor->name).find("swa") == std::string::npos ? "self_kq_mask" : "self_kq_mask_swa";
203
+ }
204
+ return get_tensor_ov_name(cgraph, tensor);
205
+ }
206
+
207
+ void GgmlOvDecoder::set_input_output() {
208
+ for (int node_n = 0; node_n < m_cgraph->n_nodes; node_n++) {
209
+ auto * node = m_cgraph->nodes[node_n];
210
+
211
+ NodeInfo current_node_info;
212
+ auto node_name = get_tensor_ov_name(m_cgraph, node);
213
+
214
+ current_node_info.node = node;
215
+ current_node_info.node_name = node_name;
216
+ current_node_info.node_op_case = 0;
217
+ current_node_info.data_addr = node->data;
218
+
219
+ for (int i = 0; i < GGML_MAX_SRC; i++) {
220
+ auto * src = node->src[i];
221
+ if (src == nullptr) {
222
+ continue;
223
+ }
224
+ auto src_name = get_tensor_ov_name(m_cgraph, src);
225
+ if (src->flags & GGML_TENSOR_FLAG_INPUT) {
226
+ src_name = get_tensor_graph_input_ov_name(this, m_cgraph, src, node);
227
+ }
228
+ current_node_info.node_inputs[src_name] = src;
229
+ current_node_info.node_inputs_names.push_back(src_name);
230
+
231
+ if (src->op == GGML_OP_VIEW) {
232
+ // Traverse upward through nested VIEW operations
233
+ std::remove_reference_t<decltype(current_node_info.node_inputs_views[src_name])> view_chain;
234
+ auto current = src;
235
+
236
+ while (current != nullptr) {
237
+ auto current_name = get_tensor_ov_name(m_cgraph, current);
238
+ if (current->flags & GGML_TENSOR_FLAG_INPUT) {
239
+ current_name = get_tensor_graph_input_ov_name(this, m_cgraph, current, node);
240
+ }
241
+ view_chain.emplace_back(current_name, current);
242
+ // If current src is also a VIEW, continue traversing
243
+ if (current->src[0] != nullptr && current->src[0]->op == GGML_OP_VIEW) {
244
+ current = current->src[0];
245
+ } else {
246
+ break;
247
+ }
248
+ }
249
+
250
+ // Assign all collected view inputs to node_inputs_views
251
+ current_node_info.node_inputs_views[src_name] = view_chain;
252
+ }
253
+ }
254
+
255
+ m_node_info_list.push_back(current_node_info);
256
+ }
257
+ }
258
+
259
+ int GgmlOvDecoder::compute_op_case(const ggml_tensor * node) const {
260
+ int op_case = 0;
261
+ switch (node->op) {
262
+ case GGML_OP_RESHAPE: {
263
+ auto name = std::string(node->name);
264
+ auto * src = node->src[0];
265
+ if (src->op == GGML_OP_RESHAPE && src->src[0]->ne[0] == node->ne[0] && src->src[0]->ne[1] == node->ne[1]) {
266
+ op_case = 4;
267
+ } else if (node->ne[0] * node->ne[1] == src->ne[0]) {
268
+ op_case = 1;
269
+ } else if (src->ne[0] * src->ne[1] == node->ne[0]) {
270
+ op_case = 2;
271
+ if (src->ne[2] * src->ne[3] == node->ne[1]) {
272
+ op_case = 5;
273
+ }
274
+ } else if (src->ne[0] * src->ne[1] * src->ne[2] == node->ne[1]) {
275
+ op_case = 3;
276
+ } else if (name.find("linear_attn_qkv_mixed") == 0 || name.find("alpha") == 0) {
277
+ op_case = 6;
278
+ } else if (name.find("linear_attn_out") == 0) {
279
+ op_case = 7;
280
+ } else if (name.find("state_predelta") == 0) {
281
+ op_case = 8;
282
+ }
283
+ break;
284
+ }
285
+ case GGML_OP_PERMUTE: {
286
+ if (node->src[0]->op != GGML_OP_VIEW) {
287
+ op_case = 1;
288
+ } else if (node->src[0]->src[0]->op == GGML_OP_NONE) {
289
+ // kv cache tensor
290
+ std::string src_name(node->view_src->name);
291
+ int layer = extract_layer_from_name(src_name).value();
292
+ if (ggml_is_contiguous(node->src[0])) {
293
+ // - 19: [ 64, 8, 256, 1] VIEW cache_k_l0 (view) [ 2, 128, 1024, 1048576]
294
+ // [ 512, 1024, 1, 1] 0: NONE cache_k_l0 [ 2, 1024, 1048576, 1048576]
295
+ // - 20: [ 64, 256, 8, 1] PERMUTE cache_k_l0 (view) (permuted) [ 2, 1024, 128, 1048576]
296
+ // [ 64, 8, 256, 1] 0: VIEW cache_k_l0 (view) [ 2, 128, 1024, 1048576]
297
+ if (!is_swa_layer(layer)) {
298
+ op_case = 3;
299
+ } else {
300
+ op_case = 4;
301
+ }
302
+ } else {
303
+ // special case of cache v when `-fa off`
304
+ // - 17: [ 256, 8, 64, 1] VIEW cache_v_l0 (view) [ 2, 131072, 2048, 1048576]
305
+ // [ 512, 1024, 1, 1] 0: NONE cache_v_l0 [ 2, 1024, 1048576, 1048576]
306
+ // - 18: [ 256, 64, 8, 1] PERMUTE cache_v_l0 (view) (permuted) [ 2, 2048, 131072, 1048576]
307
+ // [ 256, 8, 64, 1] 0: VIEW cache_v_l0 (view) [ 2, 131072, 2048, 1048576]
308
+ if (!is_swa_layer(layer)) {
309
+ op_case = 5;
310
+ } else {
311
+ op_case = 6;
312
+ }
313
+ }
314
+ } else {
315
+ // rope'ed query tensor
316
+ op_case = 2;
317
+ }
318
+ break;
319
+ }
320
+ case GGML_OP_MUL_MAT: {
321
+ if (node->src[0]->op == GGML_OP_VIEW && node->src[1]->op == GGML_OP_VIEW) {
322
+ op_case = 3;
323
+ } else if (node->src[1]->op == GGML_OP_SOFT_MAX) {
324
+ // In the case of `-fa off`, softmax is used, v_trans=true, the dynamic dim is ne[0] for cache_v
325
+ op_case = 2;
326
+ }
327
+ break;
328
+ }
329
+ case GGML_OP_GET_ROWS: {
330
+ if (node->src[1]->op == GGML_OP_VIEW) {
331
+ // GET_ROWS gathering recurrent state cache rows via the inp->s_copy index list:
332
+ // src[0] is a reshape of cache_r/cache_s, src[1] is a view of the s_copy leaf.
333
+ // op_case 3: main view (active sequences, view offset 0)
334
+ // op_case 4: extra view (defrag remainder, nonzero view offset)
335
+ if (node->src[0]->op == GGML_OP_RESHAPE && node->src[0]->src[0] != nullptr &&
336
+ is_kvcache(node->src[0]->src[0], nullptr)) {
337
+ op_case = node->src[1]->view_offs == 0 ? 1 : 2;
338
+ }
339
+ }
340
+ break;
341
+ }
342
+ case GGML_OP_ROPE: {
343
+ const int mode = node->op_params[2];
344
+ switch (mode) {
345
+ case GGML_ROPE_TYPE_NEOX: {
346
+ op_case = 1;
347
+ break;
348
+ }
349
+ case GGML_ROPE_TYPE_IMROPE: {
350
+ op_case = 2;
351
+ break;
352
+ }
353
+ default:
354
+ op_case = 0;
355
+ break;
356
+ }
357
+ break;
358
+ }
359
+ case GGML_OP_VIEW: {
360
+ if (m_is_static && node->src[0] != nullptr &&
361
+ (node->src[0]->op == GGML_OP_GATED_DELTA_NET || node->src[0]->op == GGML_OP_CONCAT)) {
362
+ // VIEW slicing a GATED_DELTA_NET combined [attn|state] output, or the conv_input
363
+ // CONCAT. The consuming CPY/RMS_NORM op recovers the true window at runtime via
364
+ // ssm_state_size / the fixed conv kernel width, so this VIEW must stay an identity
365
+ // pass-through of the full source here too (it already is on the dynamic path);
366
+ // otherwise the generic static-mode Slice below would bake in the *captured*
367
+ // cgraph's token count, which is wrong once the compiled static model runs with a
368
+ // different token count (prefill chunk size or 1).
369
+ op_case = 1;
370
+ break;
371
+ }
372
+ if (node->src[0]->op == GGML_OP_VIEW) {
373
+ auto * src = node->src[0];
374
+ if (ggml_nelements(node) != ggml_nelements(src)) {
375
+ // throw std::runtime_error("Unsupported VIEW case");
376
+ }
377
+ op_case = 0;
378
+ if (m_model_is_splitted && m_model_inputs.find(get_tensor_ov_name(m_cgraph, src)) != m_model_inputs.end()) {
379
+ op_case = 0;
380
+ }
381
+ }
382
+ {
383
+ auto * src = node->src[0];
384
+ if (ggml_nelements(node) != ggml_nelements(src)) {
385
+ // Case 4: select one slice on src dim1 (via view offset), keep src dim2 as output dim1.
386
+ // Typical pattern:
387
+ // src: ne=[N, M, K, 1], nb=[b0, b1, b2, b3]
388
+ // dst: ne=[N, K, 1, 1], nb=[b0, b2, b3, b3]
389
+ if (node->ne[0] == src->ne[0] && node->ne[1] == src->ne[2] && node->ne[2] == 1 &&
390
+ node->nb[0] == src->nb[0] && node->nb[1] == src->nb[2] && src->ne[1] > 1) {
391
+ op_case = 0;
392
+ break;
393
+ }
394
+
395
+ // General case 3: shape differs from source (one or more dims) and is handled as VIEW slicing.
396
+ int diff_count = 0;
397
+ for (int i = 0; i < GGML_MAX_DIMS; i++) {
398
+ if (node->ne[i] != src->ne[i]) {
399
+ diff_count++;
400
+ }
401
+ // if node ne[i] > src ne[i], case = 0
402
+ if (node->ne[i] > src->ne[i]) {
403
+ return 0;
404
+ }
405
+ }
406
+ if (diff_count >= 1) {
407
+ op_case = 0;
408
+ }
409
+ }
410
+ }
411
+ break;
412
+ }
413
+ case GGML_OP_RMS_NORM: {
414
+ if (node->src[0]->op == GGML_OP_VIEW) {
415
+ if (is_same_shape(node->src[0]->src[0], node->src[0])) {
416
+ op_case = 1;
417
+ } else if (node->src[0]->src[0]->op == GGML_OP_GATED_DELTA_NET) {
418
+ op_case = 2;
419
+ }
420
+ }
421
+ break;
422
+ }
423
+ case GGML_OP_POOL_2D: {
424
+ const ggml_op_pool pool_mode = static_cast<ggml_op_pool>(node->op_params[0]);
425
+ switch (pool_mode) {
426
+ case GGML_OP_POOL_MAX: {
427
+ op_case = 1;
428
+ break;
429
+ }
430
+ case GGML_OP_POOL_AVG: {
431
+ op_case = 2;
432
+ break;
433
+ }
434
+ default:
435
+ op_case = 0;
436
+ break;
437
+ }
438
+ break;
439
+ }
440
+ case GGML_OP_CPY: {
441
+ if (node->src[0]->op == GGML_OP_VIEW) {
442
+ if (node->src[0]->src[0]->op == GGML_OP_GATED_DELTA_NET) {
443
+ op_case = 1;
444
+ } else if (is_conv_state_writeback(node)) {
445
+ op_case = 2;
446
+ break;
447
+ } else if (is_conv_states_all_tensor(node->view_src) && node->src[1] != nullptr &&
448
+ node->src[1]->op == GGML_OP_VIEW && node->src[1]->view_src == node->view_src) {
449
+ op_case = 4;
450
+ break;
451
+ }
452
+ } else if (node->src[0]->op == GGML_OP_GET_ROWS && node->src[1] != nullptr &&
453
+ node->src[1]->op == GGML_OP_VIEW && node->src[1]->view_src != nullptr &&
454
+ is_kvcache(node->src[1]->view_src, nullptr)) {
455
+ // s_copy defrag remainder writeback: gathered extra state rows copied back into the cache
456
+ op_case = 3;
457
+ } else if (node->src[1] != nullptr && node->src[1]->op == GGML_OP_VIEW && node->src[1]->view_src != nullptr) {
458
+ // op_case 5: KV write for decoder self-attention (dynamic write offset)
459
+ // op_case 6: KV write for encoder self-attn or cross-attn (static offset)
460
+ const ggml_tensor * kv_buf = node->src[1]->view_src;
461
+ if (kv_buf->ne[1] == 1 && kv_buf->ne[2] == 1 && kv_buf->ne[3] == 1) {
462
+ op_case = 6;
463
+ // Forward-scan the graph for a FLASH_ATTN_EXT that reads from
464
+ // the same buffer. Having a mask (src[3] != nullptr) implies
465
+ // decoder self-attention and the write offset is dynamic.
466
+ for (int i = 0; i < m_cgraph->n_nodes; i++) {
467
+ const ggml_tensor * n = m_cgraph->nodes[i];
468
+ if (n->op != GGML_OP_FLASH_ATTN_EXT) {
469
+ continue;
470
+ }
471
+ // K (src[1]) and V (src[2]) are 3-D views whose view_src is
472
+ // the flat KV buffer we are writing to.
473
+ if ((n->src[1] != nullptr && n->src[1]->view_src == kv_buf) ||
474
+ (n->src[2] != nullptr && n->src[2]->view_src == kv_buf)) {
475
+ if (n->src[3] != nullptr) {
476
+ op_case = 5; // decoder self-attention: mask present
477
+ }
478
+ break;
479
+ }
480
+ }
481
+ }
482
+ }
483
+ break;
484
+ }
485
+ case GGML_OP_ADD: {
486
+ if (is_moe_expert_sum_add(node)) {
487
+ // Outermost ADD of a MoE expert-plane sum chain: translated as a single
488
+ // ReduceSum over the base tensor instead of N-1 chained Adds over N Slices.
489
+ op_case = 1;
490
+ }
491
+ break;
492
+ }
493
+ case GGML_OP_SCALE: {
494
+ if (node->view_src && node->buffer->usage == GGML_BACKEND_BUFFER_USAGE_ANY) {
495
+ op_case = 1;
496
+ }
497
+ break;
498
+ }
499
+ case GGML_OP_L2_NORM: {
500
+ if (std::string(node->name).find("predelta") != std::string::npos) {
501
+ op_case = 1;
502
+ }
503
+ break;
504
+ }
505
+ case GGML_OP_FLASH_ATTN_EXT: {
506
+ if (node->src[1] != nullptr && node->src[1]->op == GGML_OP_VIEW && node->src[1]->view_src != nullptr) {
507
+ const ggml_tensor * kv_buf = node->src[1]->view_src;
508
+ if (kv_buf->ne[1] == 1 && kv_buf->ne[2] == 1 && kv_buf->ne[3] == 1) {
509
+ op_case = (node->src[3] != nullptr) ? 1 : 2;
510
+ }
511
+ }
512
+ break;
513
+ }
514
+ default:
515
+ break;
516
+ }
517
+ return op_case;
518
+ }
519
+
520
+ std::optional<int> extract_layer_from_name(const std::string & name) {
521
+ size_t pos1 = name.find("_l");
522
+ if (pos1 == std::string::npos) {
523
+ return std::nullopt;
524
+ }
525
+ pos1 += 2;
526
+ size_t pos2 = name.find(' ', pos1);
527
+ if (pos2 == std::string::npos) {
528
+ pos2 = name.length();
529
+ }
530
+ std::string layer_str = name.substr(pos1, pos2 - pos1);
531
+ int layer = std::stoi(layer_str);
532
+ return layer;
533
+ }
534
+
535
+ std::pair<ModelParams, ComputeParams> GgmlOvDecoder::compute_llm_params(ggml_cgraph * cgraph, bool is_static) {
536
+ ModelParams model_params;
537
+ ComputeParams compute_params;
538
+ auto get_attention_pattern_case = [](const ggml_tensor * node) -> int {
539
+ if (node == nullptr) {
540
+ return -1;
541
+ }
542
+
543
+ switch (node->op) {
544
+ case GGML_OP_FLASH_ATTN_EXT:
545
+ if (node->src[0] == nullptr || node->src[1] == nullptr) {
546
+ return -1;
547
+ }
548
+ switch (node->src[1]->op) {
549
+ case GGML_OP_PERMUTE:
550
+ // case 0: src[1] is PERMUTE of a cache VIEW, mask required
551
+ if (node->src[3] != nullptr && node->src[1]->src[0] != nullptr &&
552
+ node->src[1]->src[0]->op == GGML_OP_VIEW) {
553
+ return 0;
554
+ }
555
+ break;
556
+ case GGML_OP_CPY:
557
+ // case 1: src[1] is CPY of a PERMUTE(VIEW), mask required
558
+ if (node->src[3] != nullptr && node->src[1]->src[0] != nullptr &&
559
+ node->src[1]->src[0]->op == GGML_OP_PERMUTE && node->src[1]->src[0]->src[0] != nullptr &&
560
+ node->src[1]->src[0]->src[0]->op == GGML_OP_VIEW) {
561
+ return 1;
562
+ }
563
+ break;
564
+ case GGML_OP_VIEW:
565
+ // cases 4/5/6: whisper - K is a direct non-contiguous VIEW_3D of a KV cache
566
+ if (node->src[1]->view_src != nullptr) {
567
+ if (node->src[3] != nullptr) {
568
+ return 4; // decoder self-attention
569
+ } else {
570
+ return 5; // cross-attention or encoder self-attention
571
+ };
572
+ }
573
+ break;
574
+ default:
575
+ break;
576
+ }
577
+ break;
578
+ case GGML_OP_SOFT_MAX:
579
+ // case 2: node op is SOFT_MAX, src 0 not null & op is MUL_MAT & the src 0 of MUL_MAT is PERMUTE & the permuted tensor src is the view of cache k
580
+ if (node->src[0] != nullptr && node->src[1] != nullptr && node->src[0]->op == GGML_OP_MUL_MAT &&
581
+ node->src[0]->src[0] != nullptr && node->src[0]->src[1] != nullptr &&
582
+ node->src[0]->src[0]->op == GGML_OP_PERMUTE && node->src[0]->src[0]->src[0] != nullptr &&
583
+ node->src[0]->src[0]->src[0]->op == GGML_OP_VIEW) {
584
+ return 2;
585
+ }
586
+ // case 3: node op is SOFT_MAX, src 0 not null & op is ADD & the src 0 of ADD is MUL_MAT & the src 0 of MUL_MAT is PERMUTE
587
+ if (node->src[0]->op == GGML_OP_ADD && node->src[0]->src[0] != nullptr &&
588
+ node->src[0]->src[0]->op == GGML_OP_MUL_MAT && node->src[0]->src[0]->src[0] != nullptr &&
589
+ node->src[0]->src[0]->src[0]->op == GGML_OP_PERMUTE) {
590
+ return 3;
591
+ }
592
+ break;
593
+ default:
594
+ break;
595
+ }
596
+
597
+ return -1;
598
+ };
599
+
600
+ bool rope_seen = false;
601
+ for (int i = 0; i < cgraph->n_nodes; i++) {
602
+ auto * node = cgraph->nodes[i];
603
+ std::string name = std::string(node->name);
604
+ const int attention_pattern_case = get_attention_pattern_case(node);
605
+ if (attention_pattern_case != -1) {
606
+ ggml_tensor * cache_k_permute = nullptr;
607
+ ggml_tensor * mask = nullptr;
608
+
609
+ switch (attention_pattern_case) {
610
+ case 0:
611
+ cache_k_permute = node->src[1];
612
+ mask = node->src[3];
613
+ break;
614
+ case 1:
615
+ cache_k_permute = node->src[1]->src[0];
616
+ mask = node->src[3];
617
+ break;
618
+ case 2:
619
+ cache_k_permute = node->src[0]->src[0];
620
+ mask = node->src[1];
621
+ break;
622
+ case 3:
623
+ cache_k_permute = node->src[0]->src[0]->src[0];
624
+ mask = node->src[1];
625
+ break;
626
+ case 4:
627
+ case 5: {
628
+ // whisper: K is a direct VIEW_3D of the KV buffer, no PERMUTE node
629
+ auto * cache_k_view = node->src[1]; // VIEW_3D of kv_self.k or kv_cross.k`
630
+ compute_params.token_len_per_seq = node->src[0]->ne[1];
631
+ if (attention_pattern_case == 4) {
632
+ compute_params.attention_size = cache_k_view->ne[1];
633
+ } else {
634
+ compute_params.attention_size_static = cache_k_view->ne[1];
635
+ }
636
+ continue;
637
+ }
638
+ default:
639
+ break;
640
+ }
641
+
642
+ assert(cache_k_permute != nullptr);
643
+
644
+ model_params.head_size = cache_k_permute->ne[0];
645
+ model_params.n_heads_kv = cache_k_permute->ne[2];
646
+ compute_params.input_len = node->src[0]->ne[1];
647
+ compute_params.token_len_per_seq = node->src[0]->ne[1];
648
+
649
+ auto * cache_k_view = cache_k_permute->src[0];
650
+ if (cache_k_view->op != GGML_OP_VIEW || mask == nullptr) {
651
+ continue;
652
+ }
653
+
654
+ ggml_tensor * cache_k = cache_k_view->src[0];
655
+ int layer = extract_layer_from_name(cache_k->name).value();
656
+
657
+ std::string mask_name(mask->name);
658
+
659
+ model_params.kv_buffer_ctx_id = ggml_backend_openvino_buffer_get_ctx_id(cache_k->buffer);
660
+ if (mask_name.find("swa") != std::string::npos) {
661
+ model_params.swa_layers.push_back(layer);
662
+ model_params.ctx_per_seq_swa = cache_k->ne[1];
663
+ } else {
664
+ model_params.ctx_per_seq = cache_k->ne[1];
665
+ model_params.n_seq = cache_k->ne[2];
666
+ }
667
+
668
+ compute_params.n_seq_active = mask->ne[3];
669
+ auto seq_size = cache_k->ne[0] * cache_k->ne[1] * ggml_type_size(cache_k->type);
670
+ size_t offset;
671
+ memcpy(&offset, cache_k_view->op_params, sizeof(size_t));
672
+ compute_params.seq_active_start = offset / seq_size;
673
+
674
+ if (mask_name.find("swa") != std::string::npos) {
675
+ compute_params.attention_size_swa = mask->ne[0];
676
+ } else {
677
+ compute_params.attention_size = mask->ne[0];
678
+ }
679
+ if (is_static) {
680
+ compute_params.attention_size = model_params.ctx_per_seq;
681
+ compute_params.attention_size_swa = model_params.ctx_per_seq_swa;
682
+ compute_params.token_len_per_seq = 1;
683
+ }
684
+ }
685
+
686
+ if (node->op == GGML_OP_MUL_MAT && node->src[0]->op == GGML_OP_PERMUTE &&
687
+ node->src[0]->src[0]->op == GGML_OP_VIEW && is_kvcache(node->src[0]->view_src, node->view_src)) {
688
+ if (node->src[1]->op == GGML_OP_PERMUTE && node->src[1]->src[0]->op == GGML_OP_VIEW &&
689
+ node->src[1]->src[0]->src[0]->op == GGML_OP_ROPE) {
690
+ compute_params.attention_size = node->ne[0];
691
+ }
692
+ }
693
+
694
+ // if the node op is TRANSPOSE and its input is PERMUTE and the source of the PERMUTE is VIEW, then get the attention size with the TRANSPOSE node ne[0] (in case no GGML_OP_FLASH_ATTN_EXT)
695
+ if (node->op == GGML_OP_TRANSPOSE && node->src[0]->op == GGML_OP_PERMUTE &&
696
+ node->src[0]->src[0]->op == GGML_OP_VIEW) {
697
+ compute_params.attention_size = node->ne[0];
698
+ if (is_static) {
699
+ compute_params.attention_size = model_params.ctx_per_seq;
700
+ }
701
+ }
702
+ if (node->op == GGML_OP_ROPE) {
703
+ if (compute_params.token_len_per_seq == -1 && node->src[1] != nullptr) {
704
+ compute_params.token_len_per_seq = ggml_nelements(node->src[1]);
705
+ }
706
+
707
+ // When multiple ROPE ops in the graph disagree on op_params (e.g. gemma4's
708
+ // mixed SWA/non-SWA layers with different n_dims or freq_base), we cannot
709
+ // share a single precomputed rope_sin/rope_cos. Track divergence so the
710
+ // translator falls back to per-op make_sin_cos in that case.
711
+ static_assert(sizeof(model_params.rope_params) == sizeof(int32_t) * 15, "rope_params size");
712
+ if (!rope_seen) {
713
+ memcpy(model_params.rope_params, node->op_params, sizeof(int32_t) * 15);
714
+ rope_seen = true;
715
+ } else if (memcmp(model_params.rope_params, node->op_params, sizeof(int32_t) * 15) != 0) {
716
+ model_params.mixed_rope_params = true;
717
+ }
718
+ }
719
+ if (node->op == GGML_OP_GATED_DELTA_NET) {
720
+ model_params.state_size = node->src[0]->ne[0];
721
+ }
722
+ if (node->op == GGML_OP_SCALE && node->view_src != nullptr && is_kvcache(node->view_src, nullptr)) {
723
+ compute_params.cache_rs_reset_len = ggml_nelements(node) / node->view_src->ne[0];
724
+ compute_params.cache_rs_reset_idx = node->src[0]->view_offs / node->view_src->ne[0];
725
+ }
726
+ // Capture the destination slot block of every recurrent state cache writeback, plus the
727
+ // conv_input window the conv state writeback copies. The active sequences occupy a
728
+ // contiguous slot block [begin, begin + n_seqs) of the cache; the block and the window move
729
+ // with the batch, so they are fed to the cached model as runtime inputs.
730
+ if (node->op == GGML_OP_CPY && node->view_src != nullptr && is_kvcache(node->view_src, nullptr) &&
731
+ node->src[1] != nullptr && node->src[1]->op == GGML_OP_VIEW && node->src[1]->view_src == node->view_src) {
732
+ const bool is_conv = is_conv_state_writeback(node);
733
+ const bool is_gdn = node->src[0]->op == GGML_OP_VIEW && node->src[0]->src[0] != nullptr &&
734
+ node->src[0]->src[0]->op == GGML_OP_GATED_DELTA_NET;
735
+ const bool is_extra = node->src[0]->op == GGML_OP_GET_ROWS;
736
+
737
+ const ggml_tensor * dest_view = node->src[1];
738
+ const ggml_tensor * cache = node->view_src;
739
+ const size_t row_bytes = cache->ne[0] * ggml_type_size(cache->type);
740
+ if (row_bytes > 0 && (is_conv || is_gdn || is_extra)) {
741
+ ComputeParams::RsWriteback writeback;
742
+ writeback.slot_begin = (int) (dest_view->view_offs / row_bytes);
743
+ if (is_conv) {
744
+ writeback.src_begin = (int) (node->src[0]->view_offs / node->src[0]->view_src->nb[0]);
745
+ } else if (is_gdn) {
746
+ writeback.src_begin = (int) (node->src[0]->view_offs / node->src[0]->view_src->nb[1]);
747
+ }
748
+ compute_params.rs_writebacks[get_tensor_ov_name(cgraph, node)] = writeback;
749
+ }
750
+ if (is_conv || is_gdn) {
751
+ compute_params.s_copy_active_slot_len = (int) dest_view->ne[1];
752
+ }
753
+ }
754
+ }
755
+ auto * output_tensor = cgraph->nodes[cgraph->n_nodes - 1];
756
+ compute_params.output_len = output_tensor->ne[1];
757
+ // for NPU, output_len is always 1 except for llama-perplexity
758
+ if (is_static && compute_params.output_len == 0) {
759
+ compute_params.output_len = 1;
760
+ }
761
+ model_params.ctx = model_params.ctx_per_seq * model_params.n_seq;
762
+ return {model_params, compute_params};
763
+ }
764
+
765
+ void GgmlOvDecoder::validate_cgraph() const {
766
+ if (m_model_params.n_seq > 1 && m_is_static == true) {
767
+ throw std::runtime_error("n_seq > 1 is not supported on NPU. Try setting -np 1.");
768
+ }
769
+ }
770
+
771
+ ov::PartialShape GgmlOvDecoder::get_graph_input_shape(const ggml_tensor * op,
772
+ const ggml_tensor * input,
773
+ int dynamic_dim_index) const {
774
+ if (m_naive) {
775
+ return input != nullptr ? ov::PartialShape{get_shape(input)} : ov::PartialShape{get_shape(op)};
776
+ }
777
+ auto name = std::string(input->name);
778
+ ov::PartialShape input_shape;
779
+
780
+ if (is_inp_tok(input, op) || is_inp_pos(input, op)) {
781
+ // tokens or positions
782
+ int len = m_is_static ? (m_is_prefill ? m_prefill_chunk_size : 1) : -1;
783
+ if (m_is_static && is_inp_pos(input, op)) {
784
+ // IMROPE stacks n_planes (t/h/w/e) position planes back to back
785
+ len *= get_inp_pos_n_planes(op);
786
+ }
787
+ input_shape = ov::PartialShape{1, 1, 1, len};
788
+
789
+ } else if (is_output_idx(input, op)) {
790
+ // output index
791
+ input_shape = ov::PartialShape{1, 1, 1, m_is_static ? m_compute_params.output_len : -1};
792
+
793
+ } else if (is_inp_mask(input, op)) {
794
+ // mask
795
+ if (m_is_static) {
796
+ input_shape = ov::PartialShape{1, 1, m_is_prefill ? m_prefill_chunk_size : 1, m_model_params.ctx};
797
+ } else if (m_is_stateful) {
798
+ input_shape = ov::PartialShape{1, 1, -1, -1};
799
+ } else {
800
+ input_shape = ov::PartialShape{-1, 1, -1, -1};
801
+ }
802
+
803
+ } else if (is_kvcache(input, op)) {
804
+ // kvcache
805
+ input_shape = ov::PartialShape{get_shape(input)};
806
+ // Whisper.cpp uses a fixed size 1D KV buffer [N, 1, 1, 1] (GGML) or [1, 1, 1, N] (OV).
807
+ // the token fill level is handled by token_len_per_seq + dynamic mask input.
808
+ // skip dynamic dim and stateful reshape for this layout.
809
+ const bool is_flat_kv = (input->ne[1] == 1 && input->ne[2] == 1 && input->ne[3] == 1);
810
+ if (!m_is_static && !is_flat_kv) {
811
+ // do not fix ctx size to make llama-bench work across test params
812
+ input_shape[2] = -1;
813
+ }
814
+ if (is_stateful() && !is_flat_kv) {
815
+ // Convert stateless KV cache layout [1, 1, seq, n_heads_kv * head_size]
816
+ // to stateful layout [1, seq, n_heads_kv, head_size].
817
+ assert(input_shape.size() == 4 && input_shape[0] == 1 && input_shape[1] == 1 &&
818
+ input_shape[2].is_dynamic() &&
819
+ input_shape[3] == (m_model_params.n_heads_kv * m_model_params.head_size));
820
+ input_shape = {input_shape[0], ov::Dimension::dynamic(), m_model_params.n_heads_kv,
821
+ m_model_params.head_size};
822
+ }
823
+
824
+ } else if (is_kv_idx(input, op)) {
825
+ // kv update index
826
+ int len = m_is_static ? (m_is_prefill ? m_prefill_chunk_size : 1) : -1;
827
+ input_shape = ov::PartialShape{1, 1, 1, len};
828
+
829
+ } else if (is_inp_s_copy(input, op) || is_s_copy_leaf(input)) {
830
+ // On NPU the total slot count (n_seq_max) is fixed at translation time, so the s_copy
831
+ // index list has a static length; on CPU/GPU it may change across compiles (defrag).
832
+ input_shape = m_is_static ? ov::PartialShape{get_shape(input)} : ov::PartialShape{1, 1, 1, -1};
833
+
834
+ } else {
835
+ input_shape = ov::PartialShape{get_shape(input)};
836
+ }
837
+ if (dynamic_dim_index != -1 && m_model_is_splitted) {
838
+ input_shape[3 - dynamic_dim_index] = -1;
839
+ }
840
+ if (op->op == GGML_OP_SOFT_MAX && op->src[1] != nullptr && op->src[1]->op == GGML_OP_NONE &&
841
+ op->src[1]->flags & GGML_TENSOR_FLAG_INPUT && op->src[1] == input) {
842
+ // for softmax input mask, the shape is [1, 1, seq_active, seq_active], where seq_active is determined by the input active sequence length instead of the kv cache sequence length
843
+ input_shape[2] = -1;
844
+ input_shape[3] = -1;
845
+ }
846
+ return input_shape;
847
+ }
848
+
849
+ bool GgmlOvDecoder::is_s_copy_leaf(const ggml_tensor * tensor) const {
850
+ if (tensor == nullptr || tensor->op != GGML_OP_NONE || m_cgraph == nullptr) {
851
+ return false;
852
+ }
853
+ for (int i = 0; i < m_cgraph->n_nodes; i++) {
854
+ const ggml_tensor * node = m_cgraph->nodes[i];
855
+ if (node->op != GGML_OP_GET_ROWS || node->src[0] == nullptr || node->src[1] == nullptr) {
856
+ continue;
857
+ }
858
+ // The index list may reach the s_copy leaf through one or more VIEWs.
859
+ const ggml_tensor * idx = node->src[1];
860
+ while (idx != nullptr && idx->op == GGML_OP_VIEW) {
861
+ idx = idx->src[0];
862
+ }
863
+ if (idx != tensor) {
864
+ continue;
865
+ }
866
+ // The gathered data must be a recurrent state cache (cache_r/cache_s).
867
+ const ggml_tensor * data = node->src[0];
868
+ while (data != nullptr && (data->op == GGML_OP_VIEW || data->op == GGML_OP_RESHAPE)) {
869
+ data = data->src[0];
870
+ }
871
+ if (data != nullptr && is_kvcache(data, nullptr)) {
872
+ return true;
873
+ }
874
+ }
875
+ return false;
876
+ }
877
+
878
+ void GgmlOvDecoder::add_extra_inputs() {
879
+ // Extra inputs:
880
+ // 1. `attention_size`, used in FLASH_ATTN where the shape of the matmul's are 256 aligned,
881
+ // see llama_kv_cache_unified::get_n_kv and llama_kv_cache_unified::get_padding.
882
+ // 2. `n_seq_active` and `seq_active_start`, used in FLASH_ATTN_EXT to indicate the active sequences in the batch
883
+
884
+ auto create_1d_input = [this](const std::string & name, int64_t value, bool force_parameter = false) {
885
+ m_model_extra_inputs[name] = {ov::element::i64, ov::Shape{1}, value, force_parameter || !m_is_static};
886
+ };
887
+
888
+ if (m_compute_params.attention_size != -1) {
889
+ create_1d_input("attention_size", m_compute_params.attention_size);
890
+ }
891
+ if (m_compute_params.attention_size_static != -1) {
892
+ create_1d_input("attention_size_static", m_compute_params.attention_size_static);
893
+ }
894
+ if (m_compute_params.attention_size_swa != -1) {
895
+ create_1d_input("attention_size_swa", m_compute_params.attention_size_swa);
896
+ }
897
+ create_1d_input("n_seq_active", m_compute_params.n_seq_active);
898
+ create_1d_input("seq_active_start", m_compute_params.seq_active_start);
899
+ create_1d_input("seq_active_end", m_compute_params.seq_active_start + m_compute_params.n_seq_active);
900
+ if (m_compute_params.token_len_per_seq != -1) {
901
+ create_1d_input("token_len_per_seq", m_compute_params.token_len_per_seq);
902
+ }
903
+ // create_1d_input("token_len", m_compute_params.token_len_per_seq * m_compute_params.n_seq_active);
904
+
905
+ if (m_compute_params.cache_rs_reset_idx != -1) {
906
+ // Whether/which cache slot to reset varies per compute call (e.g. a new sequence starting
907
+ // vs. continued decoding). can_reuse_statically() does not invalidate the cached static
908
+ // model on ComputeParams changes, so these must stay runtime Parameters even when static
909
+ // (scale.cpp op_case 1 only uses them in value comparisons, never as Slice bounds, so this
910
+ // does not reintroduce dynamic shapes).
911
+ create_1d_input("cache_rs_reset_idx", m_compute_params.cache_rs_reset_idx, /*force_parameter=*/true);
912
+ create_1d_input("cache_rs_reset_len", m_compute_params.cache_rs_reset_len, /*force_parameter=*/true);
913
+ }
914
+
915
+ if (m_compute_params.s_copy_active_slot_len != -1) {
916
+ create_1d_input("s_copy_active_slot_len", m_compute_params.s_copy_active_slot_len);
917
+ if (m_is_static) {
918
+ // Number of real tokens in the current prefill chunk. The last chunk is padded with
919
+ // fabricated token ids; attention masks them out, but the recurrent (GDN/conv) path
920
+ // would otherwise fold them into cache_r/cache_s permanently. Varies per chunk, so it
921
+ // must stay a runtime Parameter; it is only compared against a Range or used as Gather
922
+ // indices, so it does not make any shape dynamic.
923
+ create_1d_input("chunk_valid_len", get_static_n_tokens(), /*force_parameter=*/true);
924
+ }
925
+ }
926
+
927
+ for (const auto & [node_name, writeback] : m_compute_params.rs_writebacks) {
928
+ create_1d_input("rs_slot_begin_" + node_name, writeback.slot_begin);
929
+ if (!m_is_static) {
930
+ create_1d_input("rs_src_begin_" + node_name, writeback.src_begin);
931
+ }
932
+ }
933
+ }
934
+
935
+ bool GgmlOvDecoder::node_is_used_as_src(const int node_idx) {
936
+ ggml_tensor * node = m_cgraph->nodes[node_idx];
937
+ for (int i = node_idx; i < m_cgraph->n_nodes; i++) {
938
+ ggml_tensor * other_node = m_cgraph->nodes[i];
939
+ for (int j = 0; j < GGML_MAX_SRC; j++) {
940
+ if (other_node->src[j] == node) {
941
+ return true;
942
+ }
943
+ }
944
+ }
945
+ return false;
946
+ }
947
+
948
+ void GgmlOvDecoder::compute_model_inputs() {
949
+ m_model_inputs.clear();
950
+ m_inputs.clear();
951
+ for (int i = 0; i < m_cgraph->n_nodes; i++) {
952
+ ggml_tensor * node = m_cgraph->nodes[i];
953
+ // the node op is NONE means this node maybe as input of later nodes, we should add it to model inputs for this node.
954
+ if (node->op == GGML_OP_NONE && node_is_used_as_src(i)) {
955
+ std::string node_name = get_tensor_ov_name(m_cgraph, node);
956
+ if (m_model_weights.find(node_name) == m_model_weights.end()) {
957
+ m_inputs[node_name] = node;
958
+ m_model_inputs[node_name] = {get_ov_type(node),
959
+ get_graph_input_shape(node, nullptr, m_node_dynamic_dims[node])};
960
+ }
961
+ continue;
962
+ }
963
+ for (int i = 0; i < GGML_MAX_SRC; i++) {
964
+ auto * src = node->src[i];
965
+ if (src == nullptr) {
966
+ continue;
967
+ }
968
+ std::string src_name = get_tensor_ov_name(m_cgraph, src);
969
+ if (src->flags & GGML_TENSOR_FLAG_INPUT) {
970
+ src_name = get_tensor_graph_input_ov_name(this, m_cgraph, src, node);
971
+ }
972
+ if (m_model_weights.find(src_name) != m_model_weights.end()) {
973
+ continue;
974
+ }
975
+
976
+ bool is_intermediate_node = false;
977
+ for (const auto & node_info : m_node_info_list) {
978
+ if (node_info.node == src) {
979
+ is_intermediate_node = true;
980
+ break;
981
+ }
982
+ }
983
+ if (is_intermediate_node) {
984
+ continue;
985
+ }
986
+ if (m_model_inputs.find(src_name) != m_model_inputs.end()) {
987
+ continue;
988
+ }
989
+
990
+ m_inputs[src_name] = src;
991
+
992
+ ggml_backend_buffer * buffer = src->buffer;
993
+ // GGML_BACKEND_BUFFER_USAGE_ANY are kv caches
994
+ if (buffer->usage == GGML_BACKEND_BUFFER_USAGE_ANY) {
995
+ if (auto it = std::find(m_model_params.kv_names.begin(), m_model_params.kv_names.end(), src_name);
996
+ it == m_model_params.kv_names.end()) {
997
+ m_model_params.kv_names.push_back(src_name);
998
+ }
999
+ }
1000
+ // Resolve nested VIEW nodes by following src[0] until the first non-VIEW tensor.
1001
+ while (src->op == GGML_OP_VIEW && src->src[0] != nullptr) {
1002
+ src = src->src[0];
1003
+ src_name = get_tensor_ov_name(m_cgraph, src);
1004
+ }
1005
+ m_inputs[src_name] = src;
1006
+ m_model_inputs[src_name] = {get_ov_type(src),
1007
+ get_graph_input_shape(node, src, m_node_dynamic_dims[src])};
1008
+ }
1009
+ }
1010
+ }
1011
+
1012
+ void GgmlOvDecoder::compute_model_outputs() {
1013
+ m_model_outputs.clear();
1014
+ m_model_output_names.clear();
1015
+ for (int node_n = 0; node_n < m_cgraph->n_nodes; node_n++) {
1016
+ auto * cur_node = m_cgraph->nodes[node_n];
1017
+ // if the node op is NONE means this node is not used at all, we can skip it directly without adding to model outputs.
1018
+ if (cur_node->op == GGML_OP_NONE || cur_node->op == GGML_OP_VIEW || cur_node->op == GGML_OP_RESHAPE) {
1019
+ continue;
1020
+ }
1021
+ auto cur_node_use_count = m_cgraph->use_counts[ggml_hash_find(&m_cgraph->visited_hash_set, cur_node)];
1022
+ if (cur_node_use_count == 0) {
1023
+ // The output of in-place ops is the view_src tensor, which is updated in place. We should use the view_src name as the output name to make sure it can be correctly matched with the later ops that use the view_src.
1024
+ if (cur_node != nullptr && ::is_inplace_op(cur_node) && ggml_nbytes(cur_node) > 0) {
1025
+ cur_node = cur_node->view_src;
1026
+ }
1027
+ } else {
1028
+ int input_use_count = 0;
1029
+ for (int i = 0; i < m_cgraph->n_nodes; i++) {
1030
+ ggml_tensor * node = m_cgraph->nodes[i];
1031
+ for (int j = 0; j < GGML_MAX_SRC; j++) {
1032
+ if (node->src[j] != NULL && node->src[j] == cur_node) {
1033
+ input_use_count++;
1034
+ }
1035
+ }
1036
+ }
1037
+ if (input_use_count == cur_node_use_count) {
1038
+ cur_node = nullptr;
1039
+ }
1040
+ }
1041
+ if (cur_node != nullptr) {
1042
+ std::string cur_node_name = get_tensor_ov_name(m_cgraph, cur_node);
1043
+ m_model_outputs[cur_node_name] = cur_node;
1044
+ m_model_output_names.insert(cur_node_name);
1045
+ }
1046
+ }
1047
+ }
1048
+
1049
+ const ggml_tensor * GgmlOvDecoder::get_tensor_used_op(const ggml_tensor * tensor) const {
1050
+ if (tensor == nullptr) {
1051
+ return nullptr;
1052
+ }
1053
+ for (int i = 0; i < m_cgraph->n_nodes; i++) {
1054
+ const auto * node = m_cgraph->nodes[i];
1055
+ for (int j = 0; j < GGML_MAX_SRC; j++) {
1056
+ if (node->src[j] == tensor) {
1057
+ return node;
1058
+ }
1059
+ }
1060
+ }
1061
+ return nullptr;
1062
+ }
1063
+
1064
+ const ggml_tensor * GgmlOvDecoder::get_tensor_from_name(const std::string & name) const {
1065
+ for (int i = 0; i < m_cgraph->n_nodes; i++) {
1066
+ const auto * node = m_cgraph->nodes[i];
1067
+ for (int j = 0; j < GGML_MAX_SRC; j++) {
1068
+ const auto * src = node->src[j];
1069
+ if (src == nullptr) {
1070
+ break;
1071
+ }
1072
+ if (get_tensor_ov_name(m_cgraph, src) == name) {
1073
+ return src;
1074
+ }
1075
+ }
1076
+ }
1077
+ return nullptr;
1078
+ }
1079
+
1080
+ std::map<std::string, std::string> GgmlOvDecoder::get_kv_param_res_names() const {
1081
+ std::map<std::string, std::string> kv_param_res_names;
1082
+ for (const auto & name : m_model_params.kv_names) {
1083
+ kv_param_res_names[name] = name;
1084
+ }
1085
+ return kv_param_res_names;
1086
+ }
1087
+
1088
+ // MUL_MAT_ID's src[0] is the [k, m, n_expert] expert-weight tensor. It is always a constant per-expert
1089
+ // weight table -- never a computed activation -- regardless of whether the backend happened to mark its
1090
+ // buffer as GGML_BACKEND_BUFFER_USAGE_WEIGHTS (test-backend-ops, for example, never sets that usage
1091
+ // flag, unlike real inference). Without this, non-quantized (F16/F32/BF16) expert weights would fall
1092
+ // through the check below as "not a weight", get decoded as a Parameter/activation instead of a
1093
+ // Constant, and crash GatherMatmul's "only constant weights are supported" check.
1094
+ static bool is_mul_mat_id_expert_weight(const ggml_tensor * node, int src_index) {
1095
+ return node->op == GGML_OP_MUL_MAT_ID && src_index == 0;
1096
+ }
1097
+
1098
+ std::map<std::string, std::shared_ptr<ov::Node>> GgmlOvDecoder::create_weight_nodes(ggml_cgraph * cgraph, bool naive) {
1099
+ std::map<std::string, std::shared_ptr<ov::Node>> model_weights;
1100
+ auto * nodes = cgraph->nodes;
1101
+ auto n_nodes = cgraph->n_nodes;
1102
+ for (int node_i = 0; node_i < n_nodes; node_i++) {
1103
+ auto * node = nodes[node_i];
1104
+ for (int i = 0; i < GGML_MAX_SRC; i++) {
1105
+ auto * src = node->src[i];
1106
+ if (src == nullptr) {
1107
+ continue;
1108
+ }
1109
+
1110
+ std::string src_name = get_tensor_ov_name(cgraph, src);
1111
+ if (is_rope_freqs_weight(src, node)) {
1112
+ src_name = "rope_freqs.weight";
1113
+ }
1114
+ if (!src->view_src) {
1115
+ ggml_backend_buffer * buffer = src->buffer;
1116
+ if (buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS || ggml_is_quantized(src->type) ||
1117
+ is_mul_mat_id_expert_weight(node, i)) {
1118
+ if (model_weights.find(src_name) == model_weights.end()) {
1119
+ auto weight_node = create_weight_node(src, naive);
1120
+ weight_node->set_friendly_name(src_name);
1121
+ model_weights[src_name] = weight_node;
1122
+ }
1123
+ }
1124
+ }
1125
+ }
1126
+ }
1127
+ return model_weights;
1128
+ }
1129
+
1130
+ // Process-lifetime cache for weight nodes built from NON-OpenVINO buffers (e.g. the
1131
+ // token_embd.weight copy that lives in a CPU/mmap buffer and feeds GET_ROWS). Such
1132
+ // tensors have no OV buffer context to own a cached extra, so without this they are
1133
+ // re-extracted/re-requantized on every (re)compile — for token_embd that is a ~1-2 GB
1134
+ // F32 dequant each time. Keyed by tensor->data, which is stable for the process and
1135
+ // uniquely identifies the immutable weight bytes. OV-buffer weights keep using the
1136
+ // per-tensor extra cache and never reach here.
1137
+ static std::mutex g_nonov_weight_cache_mutex;
1138
+ static std::unordered_map<const void *, std::shared_ptr<ov::Node>> g_nonov_weight_cache;
1139
+
1140
+ std::set<std::string> GgmlOvDecoder::collect_weight_names(ggml_cgraph * cgraph) {
1141
+ // Mirrors the name-selection logic of create_weight_nodes() but builds no nodes,
1142
+ // so topology checks don't trigger weight extraction/requantization.
1143
+ std::set<std::string> names;
1144
+ for (int node_i = 0; node_i < cgraph->n_nodes; node_i++) {
1145
+ auto * node = cgraph->nodes[node_i];
1146
+ for (int i = 0; i < GGML_MAX_SRC; i++) {
1147
+ auto * src = node->src[i];
1148
+ if (src == nullptr) {
1149
+ continue;
1150
+ }
1151
+ std::string src_name(src->name);
1152
+ if (is_rope_freqs_weight(src, node)) {
1153
+ src_name = "rope_freqs.weight";
1154
+ }
1155
+ if (!src->view_src) {
1156
+ ggml_backend_buffer * buffer = src->buffer;
1157
+ if (buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS || ggml_is_quantized(src->type)) {
1158
+ names.insert(src_name);
1159
+ }
1160
+ }
1161
+ }
1162
+ }
1163
+ return names;
1164
+ }
1165
+
1166
+ std::shared_ptr<ov::Node> GgmlOvDecoder::create_weight_node(ggml_tensor * tensor, bool naive) {
1167
+ const bool is_ov_buffer = ggml_backend_buffer_is_openvino(tensor->buffer);
1168
+
1169
+ // Check if we have a pre-built constant from the OpenVINO backend buffer
1170
+ // This is set during ggml_backend_openvino_buffer_set_tensor
1171
+ if (tensor->extra) {
1172
+ OPENVINO_ASSERT(is_ov_buffer, "Unsupported weight tensor: " + std::string(tensor->name) +
1173
+ " Possibly this is a cpu backend repacked quantized weights");
1174
+ // Cast to our extra base type and check the type
1175
+ auto * extra_base = static_cast<ggml_openvino_extra_base *>(tensor->extra);
1176
+
1177
+ if (extra_base->type == ggml_openvino_extra_base::Type::WEIGHT) {
1178
+ // F16/F32/BF16 weight with shared-memory constant
1179
+ auto * weight_extra = static_cast<ggml_openvino_weight_extra *>(tensor->extra);
1180
+ if (weight_extra->weight_node) {
1181
+ // GGML_LOG_DEBUG("%s: using pre-built weight node for %s\n", __func__, tensor->name);
1182
+ return weight_extra->weight_node;
1183
+ }
1184
+ } else if (extra_base->type == ggml_openvino_extra_base::Type::QUANTIZED_WEIGHT) {
1185
+ // Quantized weight with pre-extracted data
1186
+ auto * quant_extra = static_cast<ggml_openvino_quantized_weight_extra *>(tensor->extra);
1187
+ if (quant_extra->weight_node) {
1188
+ // GGML_LOG_DEBUG("%s: using pre-extracted quantized weight node for %s\n", __func__, tensor->name);
1189
+ return quant_extra->weight_node;
1190
+ }
1191
+ }
1192
+ }
1193
+
1194
+ // MUL_MAT_ID expert weights are 3D GGML tensors [k, m, n_expert].
1195
+ // Keep the full reversed 4D shape when materializing non-quantized constants,
1196
+ // otherwise the expert dimension is collapsed and later Gather/MatMul logic
1197
+ // only sees a single expert slice.
1198
+ if (!ggml_is_quantized(tensor->type) && (tensor->ne[2] > 1 || tensor->ne[3] > 1)) {
1199
+ auto weight_tensor = ov::Tensor(get_ov_type(tensor), get_shape(tensor), tensor->data);
1200
+ auto weight_node = std::make_shared<ov::op::v0::Constant>(weight_tensor);
1201
+ weight_node->set_friendly_name(tensor->name);
1202
+ return weight_node;
1203
+ }
1204
+
1205
+ // Non-OV-buffer weights (CPU/mmap, e.g. the GET_ROWS token_embd copy) have no buffer
1206
+ // context to cache an extra in, so memoize them here keyed by their (stable) data
1207
+ // pointer to avoid re-extracting on every recompile. Opt-in via
1208
+ // GGML_OPENVINO_REDUCE_COMPILE_MEM or GGML_OPENVINO_MEMORY_OPTIMIZE. Skip
1209
+ // for `naive` (test/naive path) since use_bias changes the produced node.
1210
+ const bool cacheable_nonov = ggml_openvino_reduce_compile_mem_enabled() && !is_ov_buffer &&
1211
+ !naive && tensor->data != nullptr;
1212
+ if (cacheable_nonov) {
1213
+ std::lock_guard<std::mutex> lock(g_nonov_weight_cache_mutex);
1214
+ auto it = g_nonov_weight_cache.find(tensor->data);
1215
+ if (it != g_nonov_weight_cache.end()) {
1216
+ return it->second;
1217
+ }
1218
+ }
1219
+
1220
+ // There are three cases where we need to create a new weight node:
1221
+ // 1. weights are in openvino_host_buffer. Weight loading to host buffer will not trigger backend_buffer_set_tensor
1222
+ // 2. weights are in cpu/cpu_mapped buffer. On token_embd.weight goes to case 1 or 2, depending on whether mmap or direct_io is used
1223
+ // 3. test-backend-ops. buffers in test-backend-ops does not set USAGE_WEIGHT so backend_buffer_set_tensor will not create weight node
1224
+
1225
+ // GGML_LOG_DEBUG("%s: creating new weight node for %s\n", __func__, tensor->name);
1226
+ static const std::set<ggml_type> weight_types = {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16, GGML_TYPE_Q8_0,
1227
+ GGML_TYPE_Q4_0, GGML_TYPE_Q4_1, GGML_TYPE_Q5_1, GGML_TYPE_Q4_K,
1228
+ GGML_TYPE_Q5_K, GGML_TYPE_Q6_K, GGML_TYPE_MXFP4};
1229
+ if (weight_types.find(tensor->type) == weight_types.end()) {
1230
+ throw std::runtime_error("Unexpected weight tensor type: " + std::string(tensor->name) + " with type " +
1231
+ ggml_type_name(tensor->type));
1232
+ }
1233
+
1234
+ OvWeight ov_weight;
1235
+ if (ggml_is_quantized(tensor->type)) {
1236
+ auto use_bias = naive;
1237
+ if (is_ov_buffer) {
1238
+ // For quantized weights, copy raw data to a temp buffer first because
1239
+ // process_weight_tensor reads from data and writes extracted results
1240
+ // (weights/scales/zp) to output_base_ptr — they would overlap if both
1241
+ // point to tensor->data.
1242
+ size_t raw_size = ggml_nbytes(tensor);
1243
+ std::vector<uint8_t> tmp(raw_size);
1244
+ memcpy(tmp.data(), tensor->data, raw_size);
1245
+ ov_weight = process_weight_tensor(tensor, tmp.data(), tensor->data, use_bias);
1246
+ } else {
1247
+ ov_weight = process_weight_tensor(tensor, tensor->data, nullptr, use_bias);
1248
+ }
1249
+ } else {
1250
+ // For non-quantized weights (F16/F32/BF16), data is already in tensor->data.
1251
+ // process_weight_tensor will create an ov::Tensor wrapping tensor->data directly.
1252
+ ov_weight = process_weight_tensor(tensor, tensor->data, tensor->data);
1253
+ }
1254
+
1255
+ ov_weight.weight_node->set_friendly_name(tensor->name);
1256
+ if (!is_ov_buffer) {
1257
+ if (cacheable_nonov) {
1258
+ std::lock_guard<std::mutex> lock(g_nonov_weight_cache_mutex);
1259
+ // Another thread may have inserted concurrently; keep the first.
1260
+ auto [it, inserted] = g_nonov_weight_cache.emplace(tensor->data, ov_weight.weight_node);
1261
+ return it->second;
1262
+ }
1263
+ return ov_weight.weight_node;
1264
+ }
1265
+
1266
+ ggml_openvino_extra_base * extra;
1267
+ if (ov_weight.is_quantized()) {
1268
+ extra = new ggml_openvino_quantized_weight_extra(std::move(ov_weight.weights), std::move(ov_weight.scales),
1269
+ std::move(ov_weight.zp), ov_weight.weight_node);
1270
+ } else {
1271
+ extra = new ggml_openvino_weight_extra(std::move(ov_weight.weights), ov_weight.weight_node);
1272
+ }
1273
+ ggml_openvino_buffer_register_extra(tensor, extra);
1274
+
1275
+ return ov_weight.weight_node;
1276
+ }
1277
+
1278
+ void GgmlOvDecoder::dump_cgraph(const ggml_cgraph * cgraph, std::string & filename) {
1279
+ std::ofstream file(filename);
1280
+ if (!file.is_open()) {
1281
+ std::cerr << "Failed to open file" << std::endl;
1282
+ return;
1283
+ }
1284
+
1285
+ file << "=== GRAPH ===\n";
1286
+
1287
+ // clang-format off
1288
+ file << "n_nodes = " << cgraph->n_nodes << "\n";
1289
+ file << " " << std::setw(3) << "nodes"
1290
+ << std::setw(15) << "shape"
1291
+ << std::setw(20) << "op"
1292
+ << std::setw(20) << "name"
1293
+ << std::setw(3) << " "
1294
+ << std::setw(62) << "stride"
1295
+ << std::setw(20) << "buffer_type"
1296
+ << "\n";
1297
+ for (int i = 0; i < cgraph->n_nodes; i++) {
1298
+ ggml_tensor * node = cgraph->nodes[i];
1299
+
1300
+ // Get buffer type name
1301
+ const char * buf_name = "none";
1302
+ ggml_backend_buffer_t buf = node->view_src ? node->view_src->buffer : node->buffer;
1303
+ if (buf) {
1304
+ buf_name = ggml_backend_buffer_name(buf);
1305
+ }
1306
+
1307
+ file << " - " << std::setw(3) << i << ": [ "
1308
+ << std::setw(5) << node->ne[0] << ", "
1309
+ << std::setw(5) << node->ne[1] << ", "
1310
+ << std::setw(5) << node->ne[2] << ", "
1311
+ << std::setw(5) << node->ne[3] << "] "
1312
+ << std::left << std::setw(20) << ggml_op_name(node->op) << std::right << " "
1313
+ << std::left << std::setw(45) << node->name << std::right
1314
+ << std::setw(2) << "[ "
1315
+ << std::setw(0) << node->nb[0] << ", "
1316
+ << std::setw(5) << node->nb[1] << ", "
1317
+ << std::setw(5) << node->nb[2] << ", "
1318
+ << std::setw(5) << node->nb[3] << "] "
1319
+ << std::right << std::setw(15) << buf_name << std::right
1320
+ << "\n";
1321
+
1322
+ for (int i = 0; i < GGML_MAX_SRC; i++) {
1323
+ if (auto* src = node->src[i]) {
1324
+ // Get buffer type name for source
1325
+ const char * src_buf_name = "none";
1326
+ ggml_backend_buffer_t src_buf = src->view_src ? src->view_src->buffer : src->buffer;
1327
+ if (src_buf) {
1328
+ src_buf_name = ggml_backend_buffer_name(src_buf);
1329
+ }
1330
+
1331
+ file << std::setw(10) << " [ "
1332
+ << std::setw(5) << src->ne[0] << ", "
1333
+ << std::setw(5) << src->ne[1] << ", "
1334
+ << std::setw(5) << src->ne[2] << ", "
1335
+ << std::setw(5) << src->ne[3] << "] "
1336
+ << std::setw(12)
1337
+ << i << ": " << std::left << std::setw(12) << ggml_op_name(src->op) << std::right;
1338
+ file << std::left << std::setw(30) << src->name << std::right
1339
+ << std::setw(16) << "[ "
1340
+ << std::setw(0) << src->nb[0] << ", "
1341
+ << std::setw(5) << src->nb[1] << ", "
1342
+ << std::setw(5) << src->nb[2] << ", "
1343
+ << std::setw(5) << src->nb[3] << "] "
1344
+ << std::right << std::setw(15) << src_buf_name << std::right
1345
+ << "\n";
1346
+ }
1347
+ }
1348
+ }
1349
+
1350
+ file << "n_leafs = " << cgraph->n_leafs << "\n";
1351
+ for (int i = 0; i < cgraph->n_leafs; i++) {
1352
+ ggml_tensor * node = cgraph->leafs[i];
1353
+
1354
+ // Get buffer type name for leaf
1355
+ const char * leaf_buf_name = "none";
1356
+ ggml_backend_buffer_t leaf_buf = node->view_src ? node->view_src->buffer : node->buffer;
1357
+ if (leaf_buf) {
1358
+ leaf_buf_name = ggml_backend_buffer_name(leaf_buf);
1359
+ }
1360
+
1361
+ file << " - " << std::setw(3) << i << ": [ "
1362
+ << std::setw(5) << node->ne[0] << ", "
1363
+ << std::setw(5) << node->ne[1] << "] "
1364
+ << std::setw(8) << ggml_op_name(node->op) << " "
1365
+ << std::setw(16) << ggml_get_name(node)
1366
+ << std::setw(20) << leaf_buf_name << "\n";
1367
+ }
1368
+ // clang-format on
1369
+ file << "========================================\n";
1370
+
1371
+ file.close();
1372
+ }
1373
+
1374
+ void print_tensor_address_map(const ggml_cgraph * cgraph) {
1375
+ std::map<void *, std::vector<std::string>> address_map;
1376
+ for (int node_n = 0; node_n < cgraph->n_nodes; node_n++) {
1377
+ auto * node = cgraph->nodes[node_n];
1378
+ if (node->data) {
1379
+ auto it = address_map.find(node->data);
1380
+ if (it == address_map.end()) {
1381
+ address_map[node->data] = std::vector<std::string>();
1382
+ }
1383
+ address_map[node->data].push_back(node->name);
1384
+ }
1385
+ }
1386
+ for (const auto & pair : address_map) {
1387
+ std::cout << "Address: " << pair.first << std::endl;
1388
+ for (const auto & name : pair.second) {
1389
+ std::cout << name << " ; ";
1390
+ }
1391
+ std::cout << std::endl << std::endl;
1392
+ }
1393
+ }
1394
+
1395
+ ov::Shape GgmlOvDecoder::get_shape(const ggml_tensor * tensor) {
1396
+ std::vector<size_t> shape;
1397
+ for (int i = GGML_MAX_DIMS - 1; i >= 0; --i) {
1398
+ shape.push_back(static_cast<size_t>(tensor->ne[i]));
1399
+ }
1400
+ return shape;
1401
+ }
1402
+
1403
+ std::vector<size_t> GgmlOvDecoder::get_stride(const ggml_tensor * tensor) {
1404
+ std::vector<size_t> stride;
1405
+ for (int i = GGML_MAX_DIMS - 1; i >= 0; --i) {
1406
+ stride.push_back(static_cast<size_t>(tensor->nb[i]));
1407
+ }
1408
+ return stride;
1409
+ }
1410
+
1411
+ ov::element::Type GgmlOvDecoder::get_ov_type(const ggml_tensor * tensor) {
1412
+ switch (tensor->type) {
1413
+ case GGML_TYPE_F64:
1414
+ return ov::element::f64;
1415
+ case GGML_TYPE_F32:
1416
+ return ov::element::f32;
1417
+ case GGML_TYPE_F16:
1418
+ return ov::element::f16;
1419
+ case GGML_TYPE_BF16:
1420
+ return ov::element::bf16;
1421
+ case GGML_TYPE_I8:
1422
+ return ov::element::i8;
1423
+ case GGML_TYPE_I16:
1424
+ return ov::element::i16;
1425
+ case GGML_TYPE_I32:
1426
+ return ov::element::i32;
1427
+ case GGML_TYPE_I64:
1428
+ return ov::element::i64;
1429
+ default:
1430
+ return ov::element::dynamic;
1431
+ }
1432
+ }
1433
+
1434
+ ov::PartialShape GgmlOvDecoder::get_input_shape(int node_idx, const std::string & name) const {
1435
+ return ov::PartialShape(get_shape(m_node_info_list[node_idx].node_inputs.at(name)));
1436
+ }
1437
+
1438
+ std::vector<size_t> GgmlOvDecoder::get_input_stride(int node_idx, const std::string & name) const {
1439
+ return get_stride(m_node_info_list[node_idx].node_inputs.at(name));
1440
+ }
1441
+
1442
+ size_t GgmlOvDecoder::get_view_input_size(int node_idx, const std::string & name) const {
1443
+ auto it = m_node_info_list[node_idx].node_inputs_views.find(name);
1444
+ if (it != m_node_info_list[node_idx].node_inputs_views.end()) {
1445
+ return it->second.size();
1446
+ }
1447
+ return 0;
1448
+ }
1449
+
1450
+ size_t GgmlOvDecoder::get_view_input_offset(int node_idx, const std::string & name, size_t view_index) const {
1451
+ auto it = m_node_info_list[node_idx].node_inputs_views.find(name);
1452
+ if (it != m_node_info_list[node_idx].node_inputs_views.end()) {
1453
+ if (view_index < it->second.size()) {
1454
+ return it->second[view_index].second->view_offs;
1455
+ }
1456
+ }
1457
+ return 0;
1458
+ }
1459
+
1460
+ size_t GgmlOvDecoder::get_view_input_src_offset(int node_idx, const std::string & name, size_t view_index) const {
1461
+ auto it = m_node_info_list[node_idx].node_inputs_views.find(name);
1462
+ if (it != m_node_info_list[node_idx].node_inputs_views.end()) {
1463
+ if (view_index < it->second.size()) {
1464
+ auto * view_tensor = it->second[view_index].second;
1465
+ if (view_tensor && view_tensor->src[0]) {
1466
+ return view_tensor->src[0]->view_offs;
1467
+ }
1468
+ }
1469
+ }
1470
+ return 0;
1471
+ }
1472
+
1473
+ std::vector<size_t> GgmlOvDecoder::get_view_input_stride(int node_idx,
1474
+ const std::string & name,
1475
+ size_t view_index) const {
1476
+ auto it = m_node_info_list[node_idx].node_inputs_views.find(name);
1477
+ if (it != m_node_info_list[node_idx].node_inputs_views.end()) {
1478
+ if (view_index < it->second.size()) {
1479
+ return get_stride(it->second[view_index].second);
1480
+ }
1481
+ }
1482
+ return {};
1483
+ }
1484
+
1485
+ std::vector<size_t> GgmlOvDecoder::get_view_input_src_stride(int node_idx,
1486
+ const std::string & name,
1487
+ size_t view_index) const {
1488
+ auto it = m_node_info_list[node_idx].node_inputs_views.find(name);
1489
+ if (it != m_node_info_list[node_idx].node_inputs_views.end()) {
1490
+ if (view_index < it->second.size()) {
1491
+ auto * view_tensor = it->second[view_index].second;
1492
+ if (view_tensor && view_tensor->src[0]) {
1493
+ return get_stride(view_tensor->src[0]);
1494
+ }
1495
+ }
1496
+ }
1497
+ return {};
1498
+ }
1499
+
1500
+ ov::Shape GgmlOvDecoder::get_view_input_ggml_shape(int node_idx, const std::string & name, size_t view_index) const {
1501
+ auto it = m_node_info_list[node_idx].node_inputs_views.find(name);
1502
+ if (it != m_node_info_list[node_idx].node_inputs_views.end()) {
1503
+ if (view_index < it->second.size()) {
1504
+ return get_shape(it->second[view_index].second);
1505
+ }
1506
+ }
1507
+ return {};
1508
+ }
1509
+
1510
+ ov::Shape GgmlOvDecoder::get_view_input_src_ggml_shape(int node_idx,
1511
+ const std::string & name,
1512
+ size_t view_index) const {
1513
+ auto it = m_node_info_list[node_idx].node_inputs_views.find(name);
1514
+ if (it != m_node_info_list[node_idx].node_inputs_views.end()) {
1515
+ if (view_index < it->second.size()) {
1516
+ auto * view_tensor = it->second[view_index].second;
1517
+ if (view_tensor && view_tensor->src[0]) {
1518
+ return get_shape(view_tensor->src[0]);
1519
+ }
1520
+ }
1521
+ }
1522
+ return {};
1523
+ }
1524
+
1525
+ ov::PartialShape GgmlOvDecoder::get_view_input_ov_shape(int node_idx,
1526
+ const std::string & name,
1527
+ size_t view_index) const {
1528
+ auto it = m_node_info_list[node_idx].node_inputs_views.find(name);
1529
+ if (it != m_node_info_list[node_idx].node_inputs_views.end()) {
1530
+ if (view_index < it->second.size()) {
1531
+ auto * tensor = it->second[view_index].second;
1532
+ ov::PartialShape shape = ov::PartialShape{get_shape(tensor)};
1533
+
1534
+ // Check if this tensor has a dynamic dimension
1535
+ auto dynamic_it = m_node_dynamic_dims.find(tensor);
1536
+ if (dynamic_it != m_node_dynamic_dims.end() && dynamic_it->second != -1) {
1537
+ int dynamic_dim_index = dynamic_it->second;
1538
+ // GGML uses reverse indexing, so convert to OpenVINO indexing
1539
+ shape[3 - dynamic_dim_index] = m_is_static ? get_static_n_tokens() : -1;
1540
+ }
1541
+
1542
+ return shape;
1543
+ }
1544
+ }
1545
+ return {};
1546
+ }
1547
+
1548
+ ov::PartialShape GgmlOvDecoder::get_view_input_src_ov_shape(int node_idx,
1549
+ const std::string & name,
1550
+ size_t view_index) const {
1551
+ auto it = m_node_info_list[node_idx].node_inputs_views.find(name);
1552
+ if (it != m_node_info_list[node_idx].node_inputs_views.end()) {
1553
+ if (view_index < it->second.size()) {
1554
+ auto * view_tensor = it->second[view_index].second;
1555
+ if (view_tensor && view_tensor->src[0]) {
1556
+ auto * src_tensor = view_tensor->src[0];
1557
+ ov::PartialShape shape = ov::PartialShape{get_shape(src_tensor)};
1558
+
1559
+ // Check if this tensor has a dynamic dimension
1560
+ auto dynamic_it = m_node_dynamic_dims.find(src_tensor);
1561
+ if (dynamic_it != m_node_dynamic_dims.end() && dynamic_it->second != -1) {
1562
+ int dynamic_dim_index = dynamic_it->second;
1563
+ // GGML uses reverse indexing, so convert to OpenVINO indexing
1564
+ shape[3 - dynamic_dim_index] = m_is_static ? get_static_n_tokens() : -1;
1565
+ }
1566
+
1567
+ return shape;
1568
+ }
1569
+ }
1570
+ }
1571
+ return {};
1572
+ }
1573
+
1574
+ std::string GgmlOvDecoder::get_view_input_name(int node_idx, const std::string & name, size_t view_index) const {
1575
+ auto it = m_node_info_list[node_idx].node_inputs_views.find(name);
1576
+ if (it != m_node_info_list[node_idx].node_inputs_views.end()) {
1577
+ if (view_index < it->second.size()) {
1578
+ return it->second[view_index].first;
1579
+ }
1580
+ }
1581
+ return "";
1582
+ }
1583
+
1584
+ std::string GgmlOvDecoder::get_view_input_src_name(int node_idx, const std::string & name, size_t view_index) const {
1585
+ auto it = m_node_info_list[node_idx].node_inputs_views.find(name);
1586
+ if (it != m_node_info_list[node_idx].node_inputs_views.end()) {
1587
+ if (view_index < it->second.size()) {
1588
+ auto * view_tensor = it->second[view_index].second;
1589
+ if (view_tensor && view_tensor->src[0]) {
1590
+ return get_tensor_ov_name(m_cgraph, view_tensor->src[0]);
1591
+ }
1592
+ }
1593
+ }
1594
+ return "";
1595
+ }
1596
+
1597
+ ov::element::Type GgmlOvDecoder::get_input_type(int node_idx, const std::string & name) const {
1598
+ return get_ov_type(m_node_info_list[node_idx].node_inputs.at(name));
1599
+ }
1600
+
1601
+ size_t GgmlOvDecoder::get_input_size() const {
1602
+ return m_model_inputs.size();
1603
+ }
1604
+
1605
+ size_t GgmlOvDecoder::get_input_size(int node_idx) const {
1606
+ return m_node_info_list[node_idx].node_inputs_names.size();
1607
+ }
1608
+
1609
+ std::vector<std::string> GgmlOvDecoder::get_input_names(int node_idx) const {
1610
+ return m_node_info_list[node_idx].node_inputs_names;
1611
+ }
1612
+
1613
+ ov::PartialShape GgmlOvDecoder::get_output_shape(int node_idx) const {
1614
+ auto * ggml_tensor = m_node_info_list[node_idx].node;
1615
+ return ov::PartialShape(get_shape(ggml_tensor));
1616
+ }
1617
+
1618
+ ov::element::Type GgmlOvDecoder::get_output_type(const int node_idx) const {
1619
+ return get_ov_type(m_node_info_list[node_idx].node);
1620
+ }
1621
+
1622
+ std::vector<size_t> GgmlOvDecoder::get_output_stride(int node_idx) const {
1623
+ auto * ggml_tensor = m_node_info_list[node_idx].node;
1624
+ return get_stride(ggml_tensor);
1625
+ }
1626
+
1627
+ std::vector<std::string> GgmlOvDecoder::get_output_names(int node_idx) const {
1628
+ return {m_node_info_list[node_idx].node_name};
1629
+ }
1630
+
1631
+ std::string GgmlOvDecoder::get_inplace_op_src(int node_idx) const {
1632
+ auto * node = m_node_info_list[node_idx].node;
1633
+ if (!::is_inplace_op(node) || node->view_src == nullptr || ggml_nbytes(node) == 0) {
1634
+ return "";
1635
+ }
1636
+ const int op_case = m_node_info_list[node_idx].node_op_case;
1637
+ if (node->op == GGML_OP_CPY && (op_case == 1 || op_case == 2 || op_case == 3) &&
1638
+ m_compute_params.s_copy_active_slot_len == -1) {
1639
+ return "";
1640
+ }
1641
+ return get_tensor_ov_name(m_cgraph, node->view_src);
1642
+ }
1643
+
1644
+ bool GgmlOvDecoder::is_view_like_alias_of(int node_idx, const std::string & view_src_name) const {
1645
+ auto * node = m_node_info_list[node_idx].node;
1646
+ if (node->view_src == nullptr || get_tensor_ov_name(m_cgraph, node->view_src) != view_src_name) {
1647
+ return false;
1648
+ }
1649
+ return node->op == GGML_OP_RESHAPE || node->op == GGML_OP_VIEW;
1650
+ }
1651
+
1652
+ const std::string & GgmlOvDecoder::get_op_name() const {
1653
+ static const std::string unknown_name = "UNKNOWN_OP_NAME";
1654
+ return unknown_name;
1655
+ }
1656
+
1657
+ int32_t GgmlOvDecoder::get_op_dynamic_dim(int node_idx) const {
1658
+ auto it = m_node_dynamic_dims.find(m_node_info_list[node_idx].node);
1659
+ if (it == m_node_dynamic_dims.end()) {
1660
+ return -1;
1661
+ }
1662
+ return it->second;
1663
+ }
1664
+
1665
+ const std::string & GgmlOvDecoder::get_op_name(int node_idx) const {
1666
+ return m_node_info_list[node_idx].node_name;
1667
+ }
1668
+
1669
+ int32_t * GgmlOvDecoder::get_input_op_params(int node_idx, const std::string & name) const {
1670
+ return m_node_info_list[node_idx].node_inputs.at(name)->op_params;
1671
+ }
1672
+
1673
+ int32_t * GgmlOvDecoder::get_output_op_params(int node_idx) const {
1674
+ return m_node_info_list[node_idx].node->op_params;
1675
+ }
1676
+
1677
+ size_t GgmlOvDecoder::get_output_op_offset(int node_idx) const {
1678
+ return m_node_info_list[node_idx].node->view_offs;
1679
+ }
1680
+
1681
+ void GgmlOvDecoder::visit_subgraph(std::function<void(std::shared_ptr<GgmlDecoder>, int node_idx)> node_visitor) const {
1682
+ for (int node_idx = 0; node_idx < m_cgraph->n_nodes; node_idx++) {
1683
+ if (m_cgraph->nodes[node_idx]->op == GGML_OP_NONE) {
1684
+ continue;
1685
+ }
1686
+ node_visitor(std::make_shared<GgmlOvDecoder>(*this), node_idx);
1687
+ }
1688
+ }
1689
+
1690
+ std::string GgmlOvDecoder::compute_op_type(const ggml_tensor * node) {
1691
+ switch (node->op) {
1692
+ case GGML_OP_UNARY:
1693
+ return std::string("GGML_UNARY_OP_") + ggml_unary_op_name(ggml_get_unary_op(node));
1694
+ case GGML_OP_GLU:
1695
+ return std::string("GGML_GLU_OP_") + ggml_glu_op_name(ggml_get_glu_op(node));
1696
+ default:
1697
+ return std::string("GGML_OP_") + ggml_op_name(node->op);
1698
+ }
1699
+ }
1700
+
1701
+ const std::string & GgmlOvDecoder::get_op_type(int node_idx) const {
1702
+ return m_node_info_list[node_idx].node_op_type;
1703
+ }
1704
+
1705
+ const std::string & GgmlOvDecoder::get_op_type() const {
1706
+ static const std::string unknown_op = "UNKNOWN_GGML_OP";
1707
+ return unknown_op;
1708
+ }
1709
+
1710
+ void GgmlOvDecoder::compute_node_dynamic_dims() {
1711
+ auto visit_node = [&](auto && self, ggml_tensor * node) -> void {
1712
+ if (!node) {
1713
+ return;
1714
+ }
1715
+
1716
+ if (node->op == GGML_OP_CPY) {
1717
+ m_node_dynamic_dims[node] = -1;
1718
+ }
1719
+
1720
+ if (m_node_dynamic_dims.count(node)) {
1721
+ return;
1722
+ }
1723
+ for (int i = 0; i < GGML_MAX_SRC; i++) {
1724
+ ggml_tensor * src = node->src[i];
1725
+ if (src == nullptr) {
1726
+ continue;
1727
+ }
1728
+ struct ggml_tensor * root_src = nullptr;
1729
+ // if (src->org_src) {
1730
+ // root_src = src->org_src;
1731
+ // }
1732
+ if (root_src) {
1733
+ if (is_inp_tok(root_src, node) || is_inp_pos(root_src, node) || is_output_idx(root_src, node)) {
1734
+ m_node_dynamic_dims[root_src] = 0;
1735
+ m_node_dynamic_dims[src] = m_node_dynamic_dims[root_src];
1736
+ continue;
1737
+ }
1738
+ self(self, root_src);
1739
+ m_node_dynamic_dims[src] = m_node_dynamic_dims[root_src];
1740
+ } else {
1741
+ if (is_inp_tok(src, node) || is_inp_pos(src, node) || is_output_idx(src, node)) {
1742
+ m_node_dynamic_dims[src] = 0;
1743
+ continue;
1744
+ }
1745
+ if (node->op == GGML_OP_VIEW && src->op == GGML_OP_NONE && !is_stateful() && !m_model_is_splitted) {
1746
+ m_node_dynamic_dims[src] = 1;
1747
+ continue;
1748
+ }
1749
+ self(self, src);
1750
+ }
1751
+ }
1752
+ switch (node->op) {
1753
+ case GGML_OP_NONE:
1754
+ m_node_dynamic_dims[node] = -1;
1755
+ break;
1756
+ case GGML_OP_GET_ROWS:
1757
+ m_node_dynamic_dims[node] = -1;
1758
+ if (m_node_dynamic_dims[node->src[1]] != -1) {
1759
+ auto dynamic_dim_idx = m_node_dynamic_dims[node->src[1]];
1760
+ if (dynamic_dim_idx == 0) {
1761
+ m_node_dynamic_dims[node] = 1;
1762
+ } else {
1763
+ auto dynamic_dim_stride = node->src[1]->nb[dynamic_dim_idx] / ggml_type_size(node->src[1]->type) *
1764
+ ggml_type_size(node->src[0]->type);
1765
+ for (int i = 0; i < GGML_MAX_DIMS; i++) {
1766
+ if (dynamic_dim_stride == node->src[0]->nb[i]) {
1767
+ m_node_dynamic_dims[node] = i;
1768
+ break;
1769
+ }
1770
+ }
1771
+ }
1772
+ // OPENVINO_ASSERT(dynamic_dim_value == node->ne[m_node_dynamic_dims[node]],
1773
+ // "Dynamic dim value mismatch for node: " + std::string(node->name) +
1774
+ // " and its src[1]: " + std::string(node->src[1]->name));
1775
+ }
1776
+ break;
1777
+ case GGML_OP_MUL:
1778
+ case GGML_OP_MUL_MAT:
1779
+ m_node_dynamic_dims[node] = -1;
1780
+ if (m_node_dynamic_dims[node->src[0]] != -1) {
1781
+ m_node_dynamic_dims[node] = m_node_dynamic_dims[node->src[0]];
1782
+ }
1783
+ if (m_node_dynamic_dims[node->src[1]] != -1) {
1784
+ m_node_dynamic_dims[node] = m_node_dynamic_dims[node->src[1]];
1785
+ }
1786
+ break;
1787
+ case GGML_OP_PERMUTE:
1788
+ m_node_dynamic_dims[node] = -1;
1789
+ if (m_node_dynamic_dims[node->src[0]] != -1) {
1790
+ auto dynamic_dim_idx = m_node_dynamic_dims[node->src[0]];
1791
+ // auto dynamic_dim_value = node->src[0]->ne[dynamic_dim_idx];
1792
+ for (int i = 0; i < GGML_MAX_DIMS; i++) {
1793
+ if (node->op_params[i] == dynamic_dim_idx) {
1794
+ m_node_dynamic_dims[node] = i;
1795
+ break;
1796
+ }
1797
+ }
1798
+ // OPENVINO_ASSERT(dynamic_dim_value == node->ne[m_node_dynamic_dims[node]],
1799
+ // "Dynamic dim value mismatch for node: " + std::string(node->name) +
1800
+ // " and its src[0]: " + std::string(node->src[0]->name));
1801
+ }
1802
+ break;
1803
+ case GGML_OP_VIEW: {
1804
+ // Use stride-based matching: the stride of a VIEW dimension directly
1805
+ // encodes which source dimension it indexes into, so it uniquely
1806
+ // identifies the dynamic dim even when two dims share the same size.
1807
+ m_node_dynamic_dims[node] = -1;
1808
+ if (m_node_dynamic_dims[node->src[0]] != -1) {
1809
+ if (node->src[0]->op == GGML_OP_NONE) {
1810
+ m_node_dynamic_dims[node] = m_node_dynamic_dims[node->src[0]];
1811
+ break;
1812
+ }
1813
+ auto dynamic_dim_idx = m_node_dynamic_dims[node->src[0]];
1814
+ auto dynamic_dim_value = node->src[0]->ne[dynamic_dim_idx];
1815
+ auto dynamic_dim_stride =
1816
+ node->src[0]->nb[dynamic_dim_idx] / ggml_type_size(node->src[0]->type) * ggml_type_size(node->type);
1817
+ for (int i = 0; i < GGML_MAX_DIMS; i++) {
1818
+ if (node->nb[i] == dynamic_dim_stride) {
1819
+ m_node_dynamic_dims[node] = i;
1820
+ break;
1821
+ }
1822
+ }
1823
+ if (m_node_dynamic_dims[node] != -1 && dynamic_dim_value != node->ne[m_node_dynamic_dims[node]]) {
1824
+ m_node_dynamic_dims[node] = -1;
1825
+ GGML_LOG_WARN("ggml-openvino: dynamic dim value mismatch for VIEW node '%s', src[0]: '%s'\n",
1826
+ node->name, node->src[0]->name);
1827
+ }
1828
+ }
1829
+ break;
1830
+ }
1831
+ case GGML_OP_TRANSPOSE:
1832
+ case GGML_OP_RESHAPE: {
1833
+ if (is_same_shape(node->src[0], node)) {
1834
+ m_node_dynamic_dims[node] = m_node_dynamic_dims[node->src[0]];
1835
+ break;
1836
+ }
1837
+ // RESHAPE requires src[0] to be contiguous, so both src and result
1838
+ // have standard compact strides: nb[i] = type_size * prod(ne[0..i-1]).
1839
+ // Match src->nb[dynamic_dim] against result->nb[i] to find the output
1840
+ // dimension whose flat-memory boundary aligns with the source dynamic
1841
+ // boundary. This is unambiguous (result strides are strictly monotone)
1842
+ // and handles merged-lower-dim cases that ne-value matching misses.
1843
+ m_node_dynamic_dims[node] = -1;
1844
+ if (m_node_dynamic_dims[node->src[0]] != -1) {
1845
+ auto dynamic_dim_idx = m_node_dynamic_dims[node->src[0]];
1846
+ auto dynamic_dim_stride = node->src[0]->nb[dynamic_dim_idx];
1847
+ for (int i = 0; i < GGML_MAX_DIMS; i++) {
1848
+ if (node->nb[i] == dynamic_dim_stride && node->ne[i] == node->src[0]->ne[dynamic_dim_idx]) {
1849
+ m_node_dynamic_dims[node] = i;
1850
+ break;
1851
+ }
1852
+ }
1853
+ if (m_node_dynamic_dims[node] == -1) {
1854
+ GGML_LOG_WARN("ggml-openvino: cannot determine dynamic dim for RESHAPE node '%s'\n", node->name);
1855
+ }
1856
+ }
1857
+ break;
1858
+ }
1859
+ case GGML_OP_FLASH_ATTN_EXT: {
1860
+ // Output shape is hard-coded in ggml_flash_attn_ext as:
1861
+ // ne = { v->ne[0], q->ne[2], q->ne[1], q->ne[3] }
1862
+ // i.e. output dim 0 <- v dim 0 (head_size, static)
1863
+ // output dim 1 <- q dim 2 (n_heads, static)
1864
+ // output dim 2 <- q dim 1 (n_tokens, potentially dynamic)
1865
+ // output dim 3 <- q dim 3 (batch, static)
1866
+ // Using the fixed q-dim -> output-dim mapping table.
1867
+ // q is src[0]; the mapping from q's dynamic dim to the output dim is:
1868
+ // q dim 1 -> output dim 2
1869
+ // q dim 2 -> output dim 1
1870
+ // q dim 3 -> output dim 3
1871
+ // q dim 0 -> output dim 0 (head_size axis, unlikely to be dynamic)
1872
+ constexpr int q_to_out[GGML_MAX_DIMS] = {0, 2, 1, 3};
1873
+ m_node_dynamic_dims[node] = -1;
1874
+ if (m_node_dynamic_dims[node->src[0]] != -1) {
1875
+ auto q_dynamic_dim = m_node_dynamic_dims[node->src[0]];
1876
+ m_node_dynamic_dims[node] = q_to_out[q_dynamic_dim];
1877
+ }
1878
+ break;
1879
+ }
1880
+ case GGML_OP_CONT:
1881
+ m_node_dynamic_dims[node] = -1;
1882
+ if (m_node_dynamic_dims[node->src[0]] != -1) {
1883
+ auto dynamic_dim_idx = m_node_dynamic_dims[node->src[0]];
1884
+ if (ggml_are_same_shape(node, node->src[0])) {
1885
+ m_node_dynamic_dims[node] = dynamic_dim_idx;
1886
+ } else {
1887
+ size_t src_logical_nb[GGML_MAX_DIMS];
1888
+ src_logical_nb[0] = ggml_type_size(node->src[0]->type);
1889
+ src_logical_nb[1] = src_logical_nb[0] * (node->src[0]->ne[0] / ggml_blck_size(node->src[0]->type));
1890
+ for (int i = 2; i < GGML_MAX_DIMS; i++) {
1891
+ src_logical_nb[i] = src_logical_nb[i - 1] * node->src[0]->ne[i - 1];
1892
+ }
1893
+
1894
+ auto dynamic_dim_stride = src_logical_nb[dynamic_dim_idx] / ggml_type_size(node->src[0]->type) *
1895
+ ggml_type_size(node->type);
1896
+ int matched_dim_count = 0;
1897
+ int first_matched_dim = -1;
1898
+ for (int i = 0; i < GGML_MAX_DIMS; i++) {
1899
+ if (node->nb[i] == dynamic_dim_stride && node->ne[i] == node->src[0]->ne[dynamic_dim_idx]) {
1900
+ if (first_matched_dim == -1) {
1901
+ first_matched_dim = i;
1902
+ }
1903
+ m_node_dynamic_dims[node] = i;
1904
+ matched_dim_count++;
1905
+ }
1906
+ }
1907
+ if (matched_dim_count > 1 && node->src[0]->ne[dynamic_dim_idx] == 1) {
1908
+ // Single-token capture: every trailing dim is size 1 with the same stride, so
1909
+ // the match is ambiguous. The lowest index is the real axis; the rest are
1910
+ // ggml's size-1 padding. Bailing out here would bake the captured token count
1911
+ // into the static prefill model, which then runs with a different one.
1912
+ m_node_dynamic_dims[node] = first_matched_dim;
1913
+ } else if (matched_dim_count != 1) {
1914
+ m_node_dynamic_dims[node] = -1;
1915
+ GGML_LOG_WARN("ggml-openvino: cannot determine dynamic dim for CONT node '%s', src[0]: '%s'\n",
1916
+ node->name, node->src[0]->name);
1917
+ }
1918
+ }
1919
+ }
1920
+ break;
1921
+ case GGML_OP_CONCAT:
1922
+ for (int i = 0; i < GGML_MAX_DIMS; i++) {
1923
+ if (node->src[0]->ne[i] != node->ne[i]) {
1924
+ m_node_dynamic_dims[node] = i;
1925
+ break;
1926
+ }
1927
+ }
1928
+ break;
1929
+ case GGML_OP_SSM_CONV:
1930
+ case GGML_OP_GATED_DELTA_NET:
1931
+ m_node_dynamic_dims[node] = 1;
1932
+ break;
1933
+ case GGML_OP_RMS_NORM:
1934
+ case GGML_OP_L2_NORM:
1935
+ case GGML_OP_NORM:
1936
+ case GGML_OP_ADD:
1937
+ case GGML_OP_SUB:
1938
+ case GGML_OP_GLU:
1939
+ case GGML_OP_ROPE:
1940
+ case GGML_OP_SCALE:
1941
+ case GGML_OP_SOFT_MAX:
1942
+ case GGML_OP_ARGSORT:
1943
+ case GGML_OP_ADD_ID:
1944
+ case GGML_OP_UNARY:
1945
+ case GGML_OP_CUMSUM:
1946
+ case GGML_OP_FILL:
1947
+ case GGML_OP_SET:
1948
+ case GGML_OP_DIAG:
1949
+ case GGML_OP_TRI:
1950
+ case GGML_OP_REPEAT:
1951
+ // Shape-preserving elementwise ops: the dynamic dim is unchanged from src[0].
1952
+ // DIV/CLAMP are used in the MoE routing-weight normalization
1953
+ // (sum_rows -> clamp -> div). If they are left untracked here the dynamic
1954
+ // (token) dim is lost there, the captured prefill token count gets baked into
1955
+ // the downstream reshapes, and every decoder layer after layer 0 turns static
1956
+ // (which then triggers the GPU in-place-concat KV-cache corruption).
1957
+ case GGML_OP_DIV:
1958
+ case GGML_OP_CLAMP:
1959
+ case GGML_OP_PAD:
1960
+ m_node_dynamic_dims[node] = m_node_dynamic_dims[node->src[0]];
1961
+ break;
1962
+ case GGML_OP_SUM_ROWS:
1963
+ // SUM_ROWS reduces ggml axis 0 to size 1 and preserves all other axes, so the
1964
+ // dynamic dim is preserved unless it was axis 0 (then it is summed away).
1965
+ m_node_dynamic_dims[node] =
1966
+ (m_node_dynamic_dims[node->src[0]] == 0) ? -1 : m_node_dynamic_dims[node->src[0]];
1967
+ break;
1968
+ case GGML_OP_MUL_MAT_ID:
1969
+ case GGML_OP_SOLVE_TRI:
1970
+ m_node_dynamic_dims[node] = m_node_dynamic_dims[node->src[1]];
1971
+ break;
1972
+ case GGML_OP_CPY:
1973
+ case GGML_OP_SET_ROWS:
1974
+ m_node_dynamic_dims[node] = -1;
1975
+ break;
1976
+ case GGML_OP_IM2COL: {
1977
+ m_node_dynamic_dims[node] = -1;
1978
+ if (m_node_dynamic_dims[node->src[1]] != -1) {
1979
+ const bool is_2D = node->op_params[6] == 1;
1980
+ const int src_dyn = m_node_dynamic_dims[node->src[1]];
1981
+ if (is_2D) {
1982
+ if (src_dyn == 0) {
1983
+ m_node_dynamic_dims[node] = 1; // IW -> OW
1984
+ } else if (src_dyn == 1) {
1985
+ m_node_dynamic_dims[node] = 2; // IH -> OH
1986
+ } else if (src_dyn == 3) {
1987
+ m_node_dynamic_dims[node] = 3; // N -> N
1988
+ }
1989
+ } else {
1990
+ if (src_dyn == 0) {
1991
+ m_node_dynamic_dims[node] = 1; // IW -> OW
1992
+ } else if (src_dyn == 2) {
1993
+ m_node_dynamic_dims[node] = 2; // N -> N (1D: b->ne[2] is the batch/channel dim)
1994
+ }
1995
+ }
1996
+ if (m_node_dynamic_dims[node] != -1) {
1997
+ OPENVINO_ASSERT(node->src[1]->ne[src_dyn] == node->ne[m_node_dynamic_dims[node]],
1998
+ "Dynamic dim value mismatch for IM2COL node: " + std::string(node->name) +
1999
+ " and its src[1]: " + std::string(node->src[1]->name));
2000
+ }
2001
+ }
2002
+ break;
2003
+ }
2004
+ default:
2005
+ GGML_LOG_DEBUG("ggml-openvino: compute_node_dynamic_dims: unhandled op %s for node '%s'\n",
2006
+ ggml_op_name(node->op), node->name);
2007
+ break;
2008
+ }
2009
+ };
2010
+
2011
+ for (int i = 0; i < m_cgraph->n_nodes; i++) {
2012
+ ggml_tensor * node = m_cgraph->nodes[i];
2013
+ visit_node(visit_node, node);
2014
+ }
2015
+
2016
+ // print the nodes in m_cgraph name & shape with the dynamic dim (the dynamic dim is the dimension with -1 in m_node_dynamic_dims) for debugging
2017
+ if (0) {
2018
+ for (int i = 0; i < m_cgraph->n_nodes; i++) {
2019
+ ggml_tensor * node = m_cgraph->nodes[i];
2020
+ int dynamic_dim = m_node_dynamic_dims[node];
2021
+ std::cout << "[" << i << "] " << "node_name: " << node->name << " op: " << ggml_op_name(node->op)
2022
+ << " shape: [";
2023
+ for (int j = 0; j < 4; j++) {
2024
+ if (j == dynamic_dim) {
2025
+ std::cout << "*";
2026
+ } else {
2027
+ std::cout << node->ne[j];
2028
+ }
2029
+ if (j < 3) {
2030
+ std::cout << ", ";
2031
+ }
2032
+ }
2033
+ std::cout << "]" << std::endl;
2034
+ // print the src name & shape with the dynamic dim for debugging
2035
+ for (int j = 0; j < GGML_MAX_SRC; j++) {
2036
+ ggml_tensor * src = node->src[j];
2037
+ if (src == nullptr) {
2038
+ continue;
2039
+ }
2040
+ int src_dynamic_dim = m_node_dynamic_dims[src];
2041
+ std::cout << " [" << j << "] src_name: " << src->name << " [";
2042
+ for (int k = 0; k < 4; k++) {
2043
+ if (k == src_dynamic_dim) {
2044
+ std::cout << "*";
2045
+ } else {
2046
+ std::cout << src->ne[k];
2047
+ }
2048
+ if (k < 3) {
2049
+ std::cout << ", ";
2050
+ }
2051
+ }
2052
+ std::cout << "]" << std::endl;
2053
+ }
2054
+ std::cout << std::endl;
2055
+ }
2056
+ }
2057
+ }