elelem-ggml 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (1367) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE.txt +21 -0
  3. data/exe/elelem-ggml +8 -0
  4. data/ext/elelem/ggml/elelem.c +466 -0
  5. data/ext/elelem/ggml/extconf.rb +52 -0
  6. data/lib/elelem/ggml/chat_ml.rb +23 -0
  7. data/lib/elelem/ggml/gguf.rb +82 -0
  8. data/lib/elelem/ggml/provider.rb +63 -0
  9. data/lib/elelem/ggml/tokenizer.rb +95 -0
  10. data/lib/elelem/ggml/version.rb +7 -0
  11. data/lib/elelem/ggml.rb +37 -0
  12. data/vendor/ggml/AUTHORS +335 -0
  13. data/vendor/ggml/CMakeLists.txt +499 -0
  14. data/vendor/ggml/CONTRIBUTING.md +3 -0
  15. data/vendor/ggml/LICENSE +21 -0
  16. data/vendor/ggml/README.md +49 -0
  17. data/vendor/ggml/cmake/FindNCCL.cmake +36 -0
  18. data/vendor/ggml/cmake/GitVars.cmake +22 -0
  19. data/vendor/ggml/cmake/common.cmake +50 -0
  20. data/vendor/ggml/cmake/ggml-config.cmake.in +216 -0
  21. data/vendor/ggml/ggml.pc.in +10 -0
  22. data/vendor/ggml/include/ggml-alloc.h +86 -0
  23. data/vendor/ggml/include/ggml-backend.h +437 -0
  24. data/vendor/ggml/include/ggml-blas.h +25 -0
  25. data/vendor/ggml/include/ggml-cann.h +123 -0
  26. data/vendor/ggml/include/ggml-cpp.h +39 -0
  27. data/vendor/ggml/include/ggml-cpu.h +152 -0
  28. data/vendor/ggml/include/ggml-cuda.h +47 -0
  29. data/vendor/ggml/include/ggml-et.h +28 -0
  30. data/vendor/ggml/include/ggml-hexagon.h +19 -0
  31. data/vendor/ggml/include/ggml-metal.h +61 -0
  32. data/vendor/ggml/include/ggml-opencl.h +26 -0
  33. data/vendor/ggml/include/ggml-openvino.h +37 -0
  34. data/vendor/ggml/include/ggml-opt.h +256 -0
  35. data/vendor/ggml/include/ggml-rpc.h +35 -0
  36. data/vendor/ggml/include/ggml-sycl.h +57 -0
  37. data/vendor/ggml/include/ggml-virtgpu.h +14 -0
  38. data/vendor/ggml/include/ggml-vulkan.h +29 -0
  39. data/vendor/ggml/include/ggml-webgpu.h +19 -0
  40. data/vendor/ggml/include/ggml-zdnn.h +17 -0
  41. data/vendor/ggml/include/ggml-zendnn.h +22 -0
  42. data/vendor/ggml/include/ggml.h +2963 -0
  43. data/vendor/ggml/include/gguf.h +211 -0
  44. data/vendor/ggml/requirements.txt +12 -0
  45. data/vendor/ggml/scripts/gen-authors.sh +9 -0
  46. data/vendor/ggml/scripts/make-release-checks.sh +96 -0
  47. data/vendor/ggml/scripts/make-release-desc.sh +69 -0
  48. data/vendor/ggml/scripts/make-release-summary.txt +33 -0
  49. data/vendor/ggml/scripts/release.sh +195 -0
  50. data/vendor/ggml/scripts/sync-llama-am.sh +167 -0
  51. data/vendor/ggml/scripts/sync-llama.last +1 -0
  52. data/vendor/ggml/scripts/sync-llama.sh +21 -0
  53. data/vendor/ggml/scripts/sync-whisper-am.sh +138 -0
  54. data/vendor/ggml/scripts/sync-whisper.last +1 -0
  55. data/vendor/ggml/scripts/sync-whisper.sh +17 -0
  56. data/vendor/ggml/src/CMakeLists.txt +634 -0
  57. data/vendor/ggml/src/ggml-alloc.c +1249 -0
  58. data/vendor/ggml/src/ggml-backend-dl.cpp +48 -0
  59. data/vendor/ggml/src/ggml-backend-dl.h +45 -0
  60. data/vendor/ggml/src/ggml-backend-impl.h +291 -0
  61. data/vendor/ggml/src/ggml-backend-meta.cpp +2511 -0
  62. data/vendor/ggml/src/ggml-backend-reg.cpp +605 -0
  63. data/vendor/ggml/src/ggml-backend.cpp +2517 -0
  64. data/vendor/ggml/src/ggml-blas/CMakeLists.txt +101 -0
  65. data/vendor/ggml/src/ggml-blas/ggml-blas.cpp +530 -0
  66. data/vendor/ggml/src/ggml-cann/CMakeLists.txt +89 -0
  67. data/vendor/ggml/src/ggml-cann/acl_tensor.cpp +195 -0
  68. data/vendor/ggml/src/ggml-cann/acl_tensor.h +349 -0
  69. data/vendor/ggml/src/ggml-cann/aclnn_ops.cpp +4479 -0
  70. data/vendor/ggml/src/ggml-cann/aclnn_ops.h +1191 -0
  71. data/vendor/ggml/src/ggml-cann/common.h +651 -0
  72. data/vendor/ggml/src/ggml-cann/ggml-cann.cpp +3070 -0
  73. data/vendor/ggml/src/ggml-common.h +1911 -0
  74. data/vendor/ggml/src/ggml-cpu/CMakeLists.txt +685 -0
  75. data/vendor/ggml/src/ggml-cpu/amx/amx.cpp +249 -0
  76. data/vendor/ggml/src/ggml-cpu/amx/amx.h +8 -0
  77. data/vendor/ggml/src/ggml-cpu/amx/common.h +115 -0
  78. data/vendor/ggml/src/ggml-cpu/amx/mmq.cpp +2511 -0
  79. data/vendor/ggml/src/ggml-cpu/amx/mmq.h +10 -0
  80. data/vendor/ggml/src/ggml-cpu/arch/arm/cpu-feats.cpp +41 -0
  81. data/vendor/ggml/src/ggml-cpu/arch/arm/quants.c +4319 -0
  82. data/vendor/ggml/src/ggml-cpu/arch/arm/repack.cpp +5156 -0
  83. data/vendor/ggml/src/ggml-cpu/arch/loongarch/quants.c +2309 -0
  84. data/vendor/ggml/src/ggml-cpu/arch/powerpc/cpu-feats.cpp +82 -0
  85. data/vendor/ggml/src/ggml-cpu/arch/powerpc/quants.c +2304 -0
  86. data/vendor/ggml/src/ggml-cpu/arch/riscv/cpu-feats.cpp +38 -0
  87. data/vendor/ggml/src/ggml-cpu/arch/riscv/quants.c +6596 -0
  88. data/vendor/ggml/src/ggml-cpu/arch/riscv/repack.cpp +1703 -0
  89. data/vendor/ggml/src/ggml-cpu/arch/s390/cpu-feats.cpp +50 -0
  90. data/vendor/ggml/src/ggml-cpu/arch/s390/quants.c +1465 -0
  91. data/vendor/ggml/src/ggml-cpu/arch/wasm/quants.c +1292 -0
  92. data/vendor/ggml/src/ggml-cpu/arch/x86/cpu-feats.cpp +327 -0
  93. data/vendor/ggml/src/ggml-cpu/arch/x86/quants.c +4108 -0
  94. data/vendor/ggml/src/ggml-cpu/arch/x86/repack.cpp +6407 -0
  95. data/vendor/ggml/src/ggml-cpu/arch-fallback.h +353 -0
  96. data/vendor/ggml/src/ggml-cpu/binary-ops.cpp +154 -0
  97. data/vendor/ggml/src/ggml-cpu/binary-ops.h +16 -0
  98. data/vendor/ggml/src/ggml-cpu/cmake/FindSIMD.cmake +100 -0
  99. data/vendor/ggml/src/ggml-cpu/cmake/FindSMTIME.cmake +32 -0
  100. data/vendor/ggml/src/ggml-cpu/common.h +95 -0
  101. data/vendor/ggml/src/ggml-cpu/ggml-cpu-impl.h +539 -0
  102. data/vendor/ggml/src/ggml-cpu/ggml-cpu.c +3936 -0
  103. data/vendor/ggml/src/ggml-cpu/ggml-cpu.cpp +712 -0
  104. data/vendor/ggml/src/ggml-cpu/hbm.cpp +55 -0
  105. data/vendor/ggml/src/ggml-cpu/hbm.h +8 -0
  106. data/vendor/ggml/src/ggml-cpu/iqp.cpp +1253 -0
  107. data/vendor/ggml/src/ggml-cpu/iqp.h +39 -0
  108. data/vendor/ggml/src/ggml-cpu/kleidiai/CMakeLists.txt +14 -0
  109. data/vendor/ggml/src/ggml-cpu/kleidiai/kernels.cpp +1101 -0
  110. data/vendor/ggml/src/ggml-cpu/kleidiai/kernels.h +101 -0
  111. data/vendor/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp +1919 -0
  112. data/vendor/ggml/src/ggml-cpu/kleidiai/kleidiai.h +17 -0
  113. data/vendor/ggml/src/ggml-cpu/llamafile/sgemm.cpp +4164 -0
  114. data/vendor/ggml/src/ggml-cpu/llamafile/sgemm.h +25 -0
  115. data/vendor/ggml/src/ggml-cpu/ops.cpp +12164 -0
  116. data/vendor/ggml/src/ggml-cpu/ops.h +124 -0
  117. data/vendor/ggml/src/ggml-cpu/quants.c +1339 -0
  118. data/vendor/ggml/src/ggml-cpu/quants.h +106 -0
  119. data/vendor/ggml/src/ggml-cpu/repack.cpp +4836 -0
  120. data/vendor/ggml/src/ggml-cpu/repack.h +245 -0
  121. data/vendor/ggml/src/ggml-cpu/simd-gemm.h +226 -0
  122. data/vendor/ggml/src/ggml-cpu/simd-mappings.h +1319 -0
  123. data/vendor/ggml/src/ggml-cpu/spacemit/ime.cpp +1742 -0
  124. data/vendor/ggml/src/ggml-cpu/spacemit/ime.h +21 -0
  125. data/vendor/ggml/src/ggml-cpu/spacemit/ime1_kernels.cpp +1027 -0
  126. data/vendor/ggml/src/ggml-cpu/spacemit/ime2_kernels.cpp +5768 -0
  127. data/vendor/ggml/src/ggml-cpu/spacemit/ime_env.cpp +320 -0
  128. data/vendor/ggml/src/ggml-cpu/spacemit/ime_env.h +55 -0
  129. data/vendor/ggml/src/ggml-cpu/spacemit/ime_kernels.h +189 -0
  130. data/vendor/ggml/src/ggml-cpu/spacemit/repack.cpp +1795 -0
  131. data/vendor/ggml/src/ggml-cpu/spacemit/repack.h +14 -0
  132. data/vendor/ggml/src/ggml-cpu/spacemit/rvv_kernels.cpp +3178 -0
  133. data/vendor/ggml/src/ggml-cpu/spacemit/rvv_kernels.h +95 -0
  134. data/vendor/ggml/src/ggml-cpu/spacemit/spine_barrier.h +34 -0
  135. data/vendor/ggml/src/ggml-cpu/spacemit/spine_mem_pool.cpp +760 -0
  136. data/vendor/ggml/src/ggml-cpu/spacemit/spine_mem_pool.h +32 -0
  137. data/vendor/ggml/src/ggml-cpu/spacemit/spine_tcm.h +409 -0
  138. data/vendor/ggml/src/ggml-cpu/traits.cpp +36 -0
  139. data/vendor/ggml/src/ggml-cpu/traits.h +38 -0
  140. data/vendor/ggml/src/ggml-cpu/unary-ops.cpp +337 -0
  141. data/vendor/ggml/src/ggml-cpu/unary-ops.h +35 -0
  142. data/vendor/ggml/src/ggml-cpu/vec.cpp +613 -0
  143. data/vendor/ggml/src/ggml-cpu/vec.h +1570 -0
  144. data/vendor/ggml/src/ggml-cuda/CMakeLists.txt +267 -0
  145. data/vendor/ggml/src/ggml-cuda/acc.cu +61 -0
  146. data/vendor/ggml/src/ggml-cuda/acc.cuh +5 -0
  147. data/vendor/ggml/src/ggml-cuda/add-id.cu +58 -0
  148. data/vendor/ggml/src/ggml-cuda/add-id.cuh +3 -0
  149. data/vendor/ggml/src/ggml-cuda/allreduce.cu +971 -0
  150. data/vendor/ggml/src/ggml-cuda/allreduce.cuh +29 -0
  151. data/vendor/ggml/src/ggml-cuda/arange.cu +34 -0
  152. data/vendor/ggml/src/ggml-cuda/arange.cuh +5 -0
  153. data/vendor/ggml/src/ggml-cuda/argmax.cu +91 -0
  154. data/vendor/ggml/src/ggml-cuda/argmax.cuh +3 -0
  155. data/vendor/ggml/src/ggml-cuda/argsort.cu +292 -0
  156. data/vendor/ggml/src/ggml-cuda/argsort.cuh +20 -0
  157. data/vendor/ggml/src/ggml-cuda/binbcast.cu +574 -0
  158. data/vendor/ggml/src/ggml-cuda/binbcast.cuh +12 -0
  159. data/vendor/ggml/src/ggml-cuda/clamp.cu +45 -0
  160. data/vendor/ggml/src/ggml-cuda/clamp.cuh +5 -0
  161. data/vendor/ggml/src/ggml-cuda/col2im-1d.cu +81 -0
  162. data/vendor/ggml/src/ggml-cuda/col2im-1d.cuh +3 -0
  163. data/vendor/ggml/src/ggml-cuda/common.cuh +1678 -0
  164. data/vendor/ggml/src/ggml-cuda/concat.cu +242 -0
  165. data/vendor/ggml/src/ggml-cuda/concat.cuh +5 -0
  166. data/vendor/ggml/src/ggml-cuda/conv-transpose-1d.cu +88 -0
  167. data/vendor/ggml/src/ggml-cuda/conv-transpose-1d.cuh +5 -0
  168. data/vendor/ggml/src/ggml-cuda/conv2d-dw.cu +161 -0
  169. data/vendor/ggml/src/ggml-cuda/conv2d-dw.cuh +5 -0
  170. data/vendor/ggml/src/ggml-cuda/conv2d-transpose.cu +115 -0
  171. data/vendor/ggml/src/ggml-cuda/conv2d-transpose.cuh +5 -0
  172. data/vendor/ggml/src/ggml-cuda/conv2d.cu +167 -0
  173. data/vendor/ggml/src/ggml-cuda/conv2d.cuh +5 -0
  174. data/vendor/ggml/src/ggml-cuda/convert.cu +705 -0
  175. data/vendor/ggml/src/ggml-cuda/convert.cuh +66 -0
  176. data/vendor/ggml/src/ggml-cuda/count-equal.cu +64 -0
  177. data/vendor/ggml/src/ggml-cuda/count-equal.cuh +5 -0
  178. data/vendor/ggml/src/ggml-cuda/cp-async.cuh +57 -0
  179. data/vendor/ggml/src/ggml-cuda/cpy-utils.cuh +217 -0
  180. data/vendor/ggml/src/ggml-cuda/cpy.cu +617 -0
  181. data/vendor/ggml/src/ggml-cuda/cpy.cuh +7 -0
  182. data/vendor/ggml/src/ggml-cuda/cross-entropy-loss.cu +177 -0
  183. data/vendor/ggml/src/ggml-cuda/cross-entropy-loss.cuh +7 -0
  184. data/vendor/ggml/src/ggml-cuda/cumsum.cu +307 -0
  185. data/vendor/ggml/src/ggml-cuda/cumsum.cuh +5 -0
  186. data/vendor/ggml/src/ggml-cuda/dequantize.cuh +452 -0
  187. data/vendor/ggml/src/ggml-cuda/diag.cu +77 -0
  188. data/vendor/ggml/src/ggml-cuda/diag.cuh +5 -0
  189. data/vendor/ggml/src/ggml-cuda/diagmask.cu +40 -0
  190. data/vendor/ggml/src/ggml-cuda/diagmask.cuh +5 -0
  191. data/vendor/ggml/src/ggml-cuda/dsv4-hc.cu +294 -0
  192. data/vendor/ggml/src/ggml-cuda/dsv4-hc.cuh +6 -0
  193. data/vendor/ggml/src/ggml-cuda/fattn-common.cuh +1289 -0
  194. data/vendor/ggml/src/ggml-cuda/fattn-mma-f16.cuh +2136 -0
  195. data/vendor/ggml/src/ggml-cuda/fattn-swizzle.cuh +126 -0
  196. data/vendor/ggml/src/ggml-cuda/fattn-tile.cu +60 -0
  197. data/vendor/ggml/src/ggml-cuda/fattn-tile.cuh +1355 -0
  198. data/vendor/ggml/src/ggml-cuda/fattn-vec.cuh +611 -0
  199. data/vendor/ggml/src/ggml-cuda/fattn.cu +722 -0
  200. data/vendor/ggml/src/ggml-cuda/fattn.cuh +7 -0
  201. data/vendor/ggml/src/ggml-cuda/fill.cu +37 -0
  202. data/vendor/ggml/src/ggml-cuda/fill.cuh +3 -0
  203. data/vendor/ggml/src/ggml-cuda/fwht.cu +101 -0
  204. data/vendor/ggml/src/ggml-cuda/fwht.cuh +4 -0
  205. data/vendor/ggml/src/ggml-cuda/gated_delta_net.cu +327 -0
  206. data/vendor/ggml/src/ggml-cuda/gated_delta_net.cuh +14 -0
  207. data/vendor/ggml/src/ggml-cuda/getrows.cu +490 -0
  208. data/vendor/ggml/src/ggml-cuda/getrows.cuh +15 -0
  209. data/vendor/ggml/src/ggml-cuda/ggml-cuda.cu +5781 -0
  210. data/vendor/ggml/src/ggml-cuda/gla.cu +93 -0
  211. data/vendor/ggml/src/ggml-cuda/gla.cuh +3 -0
  212. data/vendor/ggml/src/ggml-cuda/im2col.cu +267 -0
  213. data/vendor/ggml/src/ggml-cuda/im2col.cuh +6 -0
  214. data/vendor/ggml/src/ggml-cuda/lightning-indexer.cu +588 -0
  215. data/vendor/ggml/src/ggml-cuda/lightning-indexer.cuh +4 -0
  216. data/vendor/ggml/src/ggml-cuda/mean.cu +77 -0
  217. data/vendor/ggml/src/ggml-cuda/mean.cuh +3 -0
  218. data/vendor/ggml/src/ggml-cuda/mma.cuh +1456 -0
  219. data/vendor/ggml/src/ggml-cuda/mmf.cu +191 -0
  220. data/vendor/ggml/src/ggml-cuda/mmf.cuh +908 -0
  221. data/vendor/ggml/src/ggml-cuda/mmid.cu +180 -0
  222. data/vendor/ggml/src/ggml-cuda/mmid.cuh +5 -0
  223. data/vendor/ggml/src/ggml-cuda/mmq-config-ampere.cuh +383 -0
  224. data/vendor/ggml/src/ggml-cuda/mmq-config-blackwell.cuh +37 -0
  225. data/vendor/ggml/src/ggml-cuda/mmq-config-cdna.cuh +185 -0
  226. data/vendor/ggml/src/ggml-cuda/mmq-config-pascal-dp4a.cuh +273 -0
  227. data/vendor/ggml/src/ggml-cuda/mmq-config-pascal-older.cuh +273 -0
  228. data/vendor/ggml/src/ggml-cuda/mmq-config-rdna2.cuh +273 -0
  229. data/vendor/ggml/src/ggml-cuda/mmq-config-rdna3-5.cuh +290 -0
  230. data/vendor/ggml/src/ggml-cuda/mmq-config-rdna3.cuh +274 -0
  231. data/vendor/ggml/src/ggml-cuda/mmq-config-rdna4.cuh +290 -0
  232. data/vendor/ggml/src/ggml-cuda/mmq-load-tiles.cuh +1768 -0
  233. data/vendor/ggml/src/ggml-cuda/mmq-vec-dot.cuh +1240 -0
  234. data/vendor/ggml/src/ggml-cuda/mmq.cu +386 -0
  235. data/vendor/ggml/src/ggml-cuda/mmq.cuh +1598 -0
  236. data/vendor/ggml/src/ggml-cuda/mmvf.cu +875 -0
  237. data/vendor/ggml/src/ggml-cuda/mmvf.cuh +14 -0
  238. data/vendor/ggml/src/ggml-cuda/mmvq.cu +1504 -0
  239. data/vendor/ggml/src/ggml-cuda/mmvq.cuh +18 -0
  240. data/vendor/ggml/src/ggml-cuda/moe-weighted-reduction.cu +65 -0
  241. data/vendor/ggml/src/ggml-cuda/moe-weighted-reduction.cuh +7 -0
  242. data/vendor/ggml/src/ggml-cuda/norm.cu +698 -0
  243. data/vendor/ggml/src/ggml-cuda/norm.cuh +18 -0
  244. data/vendor/ggml/src/ggml-cuda/opt-step-adamw.cu +78 -0
  245. data/vendor/ggml/src/ggml-cuda/opt-step-adamw.cuh +5 -0
  246. data/vendor/ggml/src/ggml-cuda/opt-step-sgd.cu +49 -0
  247. data/vendor/ggml/src/ggml-cuda/opt-step-sgd.cuh +5 -0
  248. data/vendor/ggml/src/ggml-cuda/out-prod.cu +125 -0
  249. data/vendor/ggml/src/ggml-cuda/out-prod.cuh +3 -0
  250. data/vendor/ggml/src/ggml-cuda/pad.cu +106 -0
  251. data/vendor/ggml/src/ggml-cuda/pad.cuh +5 -0
  252. data/vendor/ggml/src/ggml-cuda/pad_reflect_1d.cu +91 -0
  253. data/vendor/ggml/src/ggml-cuda/pad_reflect_1d.cuh +5 -0
  254. data/vendor/ggml/src/ggml-cuda/pool1d.cu +85 -0
  255. data/vendor/ggml/src/ggml-cuda/pool1d.cuh +5 -0
  256. data/vendor/ggml/src/ggml-cuda/pool2d.cu +94 -0
  257. data/vendor/ggml/src/ggml-cuda/pool2d.cuh +5 -0
  258. data/vendor/ggml/src/ggml-cuda/quantize.cu +697 -0
  259. data/vendor/ggml/src/ggml-cuda/quantize.cuh +70 -0
  260. data/vendor/ggml/src/ggml-cuda/reduce_rows.cuh +43 -0
  261. data/vendor/ggml/src/ggml-cuda/roll.cu +67 -0
  262. data/vendor/ggml/src/ggml-cuda/roll.cuh +5 -0
  263. data/vendor/ggml/src/ggml-cuda/rope.cu +941 -0
  264. data/vendor/ggml/src/ggml-cuda/rope.cuh +11 -0
  265. data/vendor/ggml/src/ggml-cuda/scale.cu +37 -0
  266. data/vendor/ggml/src/ggml-cuda/scale.cuh +5 -0
  267. data/vendor/ggml/src/ggml-cuda/set-rows.cu +398 -0
  268. data/vendor/ggml/src/ggml-cuda/set-rows.cuh +7 -0
  269. data/vendor/ggml/src/ggml-cuda/set.cu +39 -0
  270. data/vendor/ggml/src/ggml-cuda/set.cuh +7 -0
  271. data/vendor/ggml/src/ggml-cuda/snake.cu +72 -0
  272. data/vendor/ggml/src/ggml-cuda/snake.cuh +8 -0
  273. data/vendor/ggml/src/ggml-cuda/softcap.cu +37 -0
  274. data/vendor/ggml/src/ggml-cuda/softcap.cuh +5 -0
  275. data/vendor/ggml/src/ggml-cuda/softmax.cu +480 -0
  276. data/vendor/ggml/src/ggml-cuda/softmax.cuh +7 -0
  277. data/vendor/ggml/src/ggml-cuda/solve_tri.cu +273 -0
  278. data/vendor/ggml/src/ggml-cuda/solve_tri.cuh +3 -0
  279. data/vendor/ggml/src/ggml-cuda/ssm-conv.cu +206 -0
  280. data/vendor/ggml/src/ggml-cuda/ssm-conv.cuh +3 -0
  281. data/vendor/ggml/src/ggml-cuda/ssm-scan.cu +859 -0
  282. data/vendor/ggml/src/ggml-cuda/ssm-scan.cuh +3 -0
  283. data/vendor/ggml/src/ggml-cuda/sum.cu +41 -0
  284. data/vendor/ggml/src/ggml-cuda/sum.cuh +5 -0
  285. data/vendor/ggml/src/ggml-cuda/sumrows.cu +47 -0
  286. data/vendor/ggml/src/ggml-cuda/sumrows.cuh +4 -0
  287. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_16.cu +6 -0
  288. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_32.cu +6 -0
  289. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_8.cu +12 -0
  290. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_1.cu +10 -0
  291. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_2.cu +11 -0
  292. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_4.cu +12 -0
  293. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_16.cu +6 -0
  294. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_32.cu +6 -0
  295. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_4.cu +12 -0
  296. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_8.cu +12 -0
  297. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_1.cu +10 -0
  298. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_2.cu +11 -0
  299. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_16.cu +6 -0
  300. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_2.cu +11 -0
  301. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_4.cu +12 -0
  302. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_8.cu +12 -0
  303. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_64-ncols2_1.cu +10 -0
  304. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_1.cu +10 -0
  305. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_2.cu +11 -0
  306. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_4.cu +12 -0
  307. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_8.cu +12 -0
  308. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq112-dv112.cu +5 -0
  309. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq128-dv128.cu +5 -0
  310. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq192-dv128.cu +5 -0
  311. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq256-dv256.cu +5 -0
  312. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq320-dv256.cu +5 -0
  313. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq40-dv40.cu +5 -0
  314. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq512-dv512.cu +5 -0
  315. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq576-dv512.cu +5 -0
  316. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq64-dv64.cu +5 -0
  317. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq72-dv72.cu +5 -0
  318. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq80-dv80.cu +5 -0
  319. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq96-dv96.cu +5 -0
  320. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-bf16.cu +7 -0
  321. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-f16.cu +7 -0
  322. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q4_0.cu +7 -0
  323. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q4_1.cu +7 -0
  324. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q5_0.cu +7 -0
  325. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q5_1.cu +7 -0
  326. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q8_0.cu +7 -0
  327. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-bf16.cu +7 -0
  328. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-f16.cu +7 -0
  329. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q4_0.cu +7 -0
  330. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q4_1.cu +7 -0
  331. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q5_0.cu +7 -0
  332. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q5_1.cu +7 -0
  333. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q8_0.cu +7 -0
  334. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-bf16.cu +7 -0
  335. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-f16.cu +7 -0
  336. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q4_0.cu +7 -0
  337. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q4_1.cu +7 -0
  338. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q5_0.cu +7 -0
  339. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q5_1.cu +7 -0
  340. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q8_0.cu +7 -0
  341. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-bf16.cu +7 -0
  342. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-f16.cu +7 -0
  343. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q4_0.cu +7 -0
  344. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q4_1.cu +7 -0
  345. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q5_0.cu +7 -0
  346. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q5_1.cu +7 -0
  347. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q8_0.cu +7 -0
  348. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-bf16.cu +7 -0
  349. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-f16.cu +7 -0
  350. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q4_0.cu +7 -0
  351. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q4_1.cu +7 -0
  352. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q5_0.cu +7 -0
  353. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q5_1.cu +7 -0
  354. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q8_0.cu +7 -0
  355. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-bf16.cu +7 -0
  356. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-f16.cu +7 -0
  357. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q4_0.cu +7 -0
  358. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q4_1.cu +7 -0
  359. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q5_0.cu +7 -0
  360. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q5_1.cu +7 -0
  361. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q8_0.cu +7 -0
  362. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-bf16.cu +7 -0
  363. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-f16.cu +7 -0
  364. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q4_0.cu +7 -0
  365. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q4_1.cu +7 -0
  366. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q5_0.cu +7 -0
  367. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q5_1.cu +7 -0
  368. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q8_0.cu +7 -0
  369. data/vendor/ggml/src/ggml-cuda/template-instances/generate_cu_files.py +111 -0
  370. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_1.cu +5 -0
  371. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_10.cu +5 -0
  372. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_11.cu +5 -0
  373. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_12.cu +5 -0
  374. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_13.cu +5 -0
  375. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_14.cu +5 -0
  376. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_15.cu +5 -0
  377. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_16.cu +5 -0
  378. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_2.cu +5 -0
  379. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_3.cu +5 -0
  380. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_4.cu +5 -0
  381. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_5.cu +5 -0
  382. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_6.cu +5 -0
  383. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_7.cu +5 -0
  384. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_8.cu +5 -0
  385. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_9.cu +5 -0
  386. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq1_s.cu +5 -0
  387. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_s.cu +5 -0
  388. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xs.cu +5 -0
  389. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xxs.cu +5 -0
  390. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_s.cu +5 -0
  391. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_xxs.cu +5 -0
  392. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_nl.cu +5 -0
  393. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_xs.cu +5 -0
  394. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-mxfp4.cu +5 -0
  395. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-nvfp4.cu +5 -0
  396. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q1_0.cu +5 -0
  397. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_0.cu +5 -0
  398. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_k.cu +5 -0
  399. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q3_k.cu +5 -0
  400. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_0.cu +5 -0
  401. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_1.cu +5 -0
  402. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_k.cu +5 -0
  403. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_0.cu +5 -0
  404. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_1.cu +5 -0
  405. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_k.cu +5 -0
  406. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q6_k.cu +5 -0
  407. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q8_0.cu +5 -0
  408. data/vendor/ggml/src/ggml-cuda/top-k.cu +275 -0
  409. data/vendor/ggml/src/ggml-cuda/top-k.cuh +3 -0
  410. data/vendor/ggml/src/ggml-cuda/topk-moe.cu +442 -0
  411. data/vendor/ggml/src/ggml-cuda/topk-moe.cuh +31 -0
  412. data/vendor/ggml/src/ggml-cuda/tri.cu +136 -0
  413. data/vendor/ggml/src/ggml-cuda/tri.cuh +5 -0
  414. data/vendor/ggml/src/ggml-cuda/tsembd.cu +47 -0
  415. data/vendor/ggml/src/ggml-cuda/tsembd.cuh +5 -0
  416. data/vendor/ggml/src/ggml-cuda/unary.cu +721 -0
  417. data/vendor/ggml/src/ggml-cuda/unary.cuh +123 -0
  418. data/vendor/ggml/src/ggml-cuda/upscale.cu +293 -0
  419. data/vendor/ggml/src/ggml-cuda/upscale.cuh +5 -0
  420. data/vendor/ggml/src/ggml-cuda/vecdotq.cuh +1371 -0
  421. data/vendor/ggml/src/ggml-cuda/vendors/cuda.h +28 -0
  422. data/vendor/ggml/src/ggml-cuda/vendors/hip.h +301 -0
  423. data/vendor/ggml/src/ggml-cuda/vendors/musa.h +151 -0
  424. data/vendor/ggml/src/ggml-cuda/wkv.cu +253 -0
  425. data/vendor/ggml/src/ggml-cuda/wkv.cuh +7 -0
  426. data/vendor/ggml/src/ggml-et/CMakeLists.txt +246 -0
  427. data/vendor/ggml/src/ggml-et/cmake/embed_one_kernel.cmake +15 -0
  428. data/vendor/ggml/src/ggml-et/cmake/ggml-et-kernels-embed.cpp.in +6 -0
  429. data/vendor/ggml/src/ggml-et/cmake/ggml-et-kernels-embed.hpp.in +12 -0
  430. data/vendor/ggml/src/ggml-et/cmake/ggml-et-uberkernel-kernel-map.cpp.in +18 -0
  431. data/vendor/ggml/src/ggml-et/cmake/ggml-et-uberkernel-kernel-map.h.in +13 -0
  432. data/vendor/ggml/src/ggml-et/et-kernels/CMakeLists.txt +137 -0
  433. data/vendor/ggml/src/ggml-et/et-kernels/scripts/check_unimplemented_instructions.sh +36 -0
  434. data/vendor/ggml/src/ggml-et/et-kernels/src/RunBackend.sh +23 -0
  435. data/vendor/ggml/src/ggml-et/et-kernels/src/block_ops.h +997 -0
  436. data/vendor/ggml/src/ggml-et/et-kernels/src/clamp_f32.c +120 -0
  437. data/vendor/ggml/src/ggml-et/et-kernels/src/concat_f32.c +175 -0
  438. data/vendor/ggml/src/ggml-et/et-kernels/src/cont_f16.c +107 -0
  439. data/vendor/ggml/src/ggml-et/et-kernels/src/cont_f32.c +248 -0
  440. data/vendor/ggml/src/ggml-et/et-kernels/src/conv_2d_f32_me.c +807 -0
  441. data/vendor/ggml/src/ggml-et/et-kernels/src/cpy_f32_f16.c +110 -0
  442. data/vendor/ggml/src/ggml-et/et-kernels/src/crt.S +15 -0
  443. data/vendor/ggml/src/ggml-et/et-kernels/src/cumsum_f32.c +96 -0
  444. data/vendor/ggml/src/ggml-et/et-kernels/src/diag_f32.c +90 -0
  445. data/vendor/ggml/src/ggml-et/et-kernels/src/el_map_f32.c +377 -0
  446. data/vendor/ggml/src/ggml-et/et-kernels/src/fill_f32.c +87 -0
  447. data/vendor/ggml/src/ggml-et/et-kernels/src/flash_attn_ext_f16_me.c +1000 -0
  448. data/vendor/ggml/src/ggml-et/et-kernels/src/flash_attn_ext_f32.c +217 -0
  449. data/vendor/ggml/src/ggml-et/et-kernels/src/gated_delta_net_f32.c +346 -0
  450. data/vendor/ggml/src/ggml-et/et-kernels/src/get_rows_f32.c +612 -0
  451. data/vendor/ggml/src/ggml-et/et-kernels/src/ggml_tensor.h +44 -0
  452. data/vendor/ggml/src/ggml-et/et-kernels/src/glu_f32.c +606 -0
  453. data/vendor/ggml/src/ggml-et/et-kernels/src/group_norm_f32.c +171 -0
  454. data/vendor/ggml/src/ggml-et/et-kernels/src/im2col.c +130 -0
  455. data/vendor/ggml/src/ggml-et/et-kernels/src/l2_norm_f32.c +237 -0
  456. data/vendor/ggml/src/ggml-et/et-kernels/src/linker.ld +85 -0
  457. data/vendor/ggml/src/ggml-et/et-kernels/src/math_fp.h +299 -0
  458. data/vendor/ggml/src/ggml-et/et-kernels/src/mean_f32.c +220 -0
  459. data/vendor/ggml/src/ggml-et/et-kernels/src/memops.c +181 -0
  460. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0.c +358 -0
  461. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0_matrix_engine.c +368 -0
  462. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_Q8_0.c +413 -0
  463. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f16.c +142 -0
  464. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f16_matrix_engine.c +329 -0
  465. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f32.c +137 -0
  466. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f32_matrix_engine.c +155 -0
  467. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_id_Q4_0.c +169 -0
  468. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_id_Q8_0.c +160 -0
  469. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_id_f32.c +288 -0
  470. data/vendor/ggml/src/ggml-et/et-kernels/src/norm_f32.c +328 -0
  471. data/vendor/ggml/src/ggml-et/et-kernels/src/pad_f32.c +165 -0
  472. data/vendor/ggml/src/ggml-et/et-kernels/src/platform.h +545 -0
  473. data/vendor/ggml/src/ggml-et/et-kernels/src/quants.h +72 -0
  474. data/vendor/ggml/src/ggml-et/et-kernels/src/repeat_f32.c +118 -0
  475. data/vendor/ggml/src/ggml-et/et-kernels/src/rms_norm_f32.c +270 -0
  476. data/vendor/ggml/src/ggml-et/et-kernels/src/rms_norm_mul_f32.c +290 -0
  477. data/vendor/ggml/src/ggml-et/et-kernels/src/rope_f32.c +656 -0
  478. data/vendor/ggml/src/ggml-et/et-kernels/src/rwkv_wkv6_f32.c +184 -0
  479. data/vendor/ggml/src/ggml-et/et-kernels/src/rwkv_wkv7_f32.c +272 -0
  480. data/vendor/ggml/src/ggml-et/et-kernels/src/scale_f32.c +94 -0
  481. data/vendor/ggml/src/ggml-et/et-kernels/src/set_f32.c +101 -0
  482. data/vendor/ggml/src/ggml-et/et-kernels/src/set_rows_f32.c +394 -0
  483. data/vendor/ggml/src/ggml-et/et-kernels/src/softmax_f32.c +698 -0
  484. data/vendor/ggml/src/ggml-et/et-kernels/src/solve_tri_f32.c +109 -0
  485. data/vendor/ggml/src/ggml-et/et-kernels/src/sqr_f32.c +88 -0
  486. data/vendor/ggml/src/ggml-et/et-kernels/src/ssm_conv_f32.c +129 -0
  487. data/vendor/ggml/src/ggml-et/et-kernels/src/ssm_scan_f32.c +282 -0
  488. data/vendor/ggml/src/ggml-et/et-kernels/src/sum_rows_f32.c +103 -0
  489. data/vendor/ggml/src/ggml-et/et-kernels/src/tensor.h +897 -0
  490. data/vendor/ggml/src/ggml-et/et-kernels/src/tri_f32.c +244 -0
  491. data/vendor/ggml/src/ggml-et/et-kernels/src/uberkernel.c +497 -0
  492. data/vendor/ggml/src/ggml-et/et-kernels/src/unary_f32.c +705 -0
  493. data/vendor/ggml/src/ggml-et/ggml-et-common.h +86 -0
  494. data/vendor/ggml/src/ggml-et/ggml-et-cpu-compare.cpp +502 -0
  495. data/vendor/ggml/src/ggml-et/ggml-et-cpu-compare.h +54 -0
  496. data/vendor/ggml/src/ggml-et/ggml-et-kernels.cpp +508 -0
  497. data/vendor/ggml/src/ggml-et/ggml-et-kernels.h +48 -0
  498. data/vendor/ggml/src/ggml-et/ggml-et-memops.cpp +36 -0
  499. data/vendor/ggml/src/ggml-et/ggml-et-memops.h +18 -0
  500. data/vendor/ggml/src/ggml-et/ggml-et-ops.cpp +2584 -0
  501. data/vendor/ggml/src/ggml-et/ggml-et-ops.h +393 -0
  502. data/vendor/ggml/src/ggml-et/ggml-et-uberkernel-common.h +17 -0
  503. data/vendor/ggml/src/ggml-et/ggml-et.cpp +1880 -0
  504. data/vendor/ggml/src/ggml-feats.h +166 -0
  505. data/vendor/ggml/src/ggml-hexagon/CMakeLists.txt +113 -0
  506. data/vendor/ggml/src/ggml-hexagon/ggml-hexagon.cpp +6523 -0
  507. data/vendor/ggml/src/ggml-hexagon/htp/CMakeLists.txt +58 -0
  508. data/vendor/ggml/src/ggml-hexagon/htp/act-ops.c +585 -0
  509. data/vendor/ggml/src/ggml-hexagon/htp/allreduce-ops.c +398 -0
  510. data/vendor/ggml/src/ggml-hexagon/htp/allreduce-ops.h +40 -0
  511. data/vendor/ggml/src/ggml-hexagon/htp/argsort-ops.c +514 -0
  512. data/vendor/ggml/src/ggml-hexagon/htp/binary-ops.c +873 -0
  513. data/vendor/ggml/src/ggml-hexagon/htp/cmake-toolchain.cmake +155 -0
  514. data/vendor/ggml/src/ggml-hexagon/htp/concat-ops.c +277 -0
  515. data/vendor/ggml/src/ggml-hexagon/htp/cpy-ops.c +354 -0
  516. data/vendor/ggml/src/ggml-hexagon/htp/cumsum-ops.c +265 -0
  517. data/vendor/ggml/src/ggml-hexagon/htp/diag-ops.c +216 -0
  518. data/vendor/ggml/src/ggml-hexagon/htp/dma-queue.c +104 -0
  519. data/vendor/ggml/src/ggml-hexagon/htp/dma-queue.h +408 -0
  520. data/vendor/ggml/src/ggml-hexagon/htp/fill-ops.c +123 -0
  521. data/vendor/ggml/src/ggml-hexagon/htp/flash-attn-ops.c +2479 -0
  522. data/vendor/ggml/src/ggml-hexagon/htp/flash-attn-ops.h +344 -0
  523. data/vendor/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c +1157 -0
  524. data/vendor/ggml/src/ggml-hexagon/htp/get-rows-ops.c +260 -0
  525. data/vendor/ggml/src/ggml-hexagon/htp/get-rows-ops.h +77 -0
  526. data/vendor/ggml/src/ggml-hexagon/htp/hex-bitmap.h +24 -0
  527. data/vendor/ggml/src/ggml-hexagon/htp/hex-common.h +80 -0
  528. data/vendor/ggml/src/ggml-hexagon/htp/hex-dma.h +2 -0
  529. data/vendor/ggml/src/ggml-hexagon/htp/hex-dump.h +86 -0
  530. data/vendor/ggml/src/ggml-hexagon/htp/hex-fastdiv.h +37 -0
  531. data/vendor/ggml/src/ggml-hexagon/htp/hex-profile.h +64 -0
  532. data/vendor/ggml/src/ggml-hexagon/htp/hex-utils.h +65 -0
  533. data/vendor/ggml/src/ggml-hexagon/htp/hmx-fa-kernels.h +555 -0
  534. data/vendor/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h +1378 -0
  535. data/vendor/ggml/src/ggml-hexagon/htp/hmx-queue.c +170 -0
  536. data/vendor/ggml/src/ggml-hexagon/htp/hmx-queue.h +160 -0
  537. data/vendor/ggml/src/ggml-hexagon/htp/hmx-utils.h +222 -0
  538. data/vendor/ggml/src/ggml-hexagon/htp/htp-ctx.h +146 -0
  539. data/vendor/ggml/src/ggml-hexagon/htp/htp-ops.h +242 -0
  540. data/vendor/ggml/src/ggml-hexagon/htp/htp-tensor.c +300 -0
  541. data/vendor/ggml/src/ggml-hexagon/htp/htp-tensor.h +29 -0
  542. data/vendor/ggml/src/ggml-hexagon/htp/htp-vtcm.h +19 -0
  543. data/vendor/ggml/src/ggml-hexagon/htp/htp_iface.idl +23 -0
  544. data/vendor/ggml/src/ggml-hexagon/htp/hvx-arith.h +636 -0
  545. data/vendor/ggml/src/ggml-hexagon/htp/hvx-base.h +292 -0
  546. data/vendor/ggml/src/ggml-hexagon/htp/hvx-copy.h +262 -0
  547. data/vendor/ggml/src/ggml-hexagon/htp/hvx-div.h +291 -0
  548. data/vendor/ggml/src/ggml-hexagon/htp/hvx-dump.h +129 -0
  549. data/vendor/ggml/src/ggml-hexagon/htp/hvx-exp.h +255 -0
  550. data/vendor/ggml/src/ggml-hexagon/htp/hvx-fa-kernels.h +303 -0
  551. data/vendor/ggml/src/ggml-hexagon/htp/hvx-flash-attn.h +47 -0
  552. data/vendor/ggml/src/ggml-hexagon/htp/hvx-floor.h +100 -0
  553. data/vendor/ggml/src/ggml-hexagon/htp/hvx-inverse.h +210 -0
  554. data/vendor/ggml/src/ggml-hexagon/htp/hvx-log.h +118 -0
  555. data/vendor/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-flat.h +1511 -0
  556. data/vendor/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h +1200 -0
  557. data/vendor/ggml/src/ggml-hexagon/htp/hvx-norm.h +454 -0
  558. data/vendor/ggml/src/ggml-hexagon/htp/hvx-pow.h +42 -0
  559. data/vendor/ggml/src/ggml-hexagon/htp/hvx-quant.h +165 -0
  560. data/vendor/ggml/src/ggml-hexagon/htp/hvx-reduce.h +336 -0
  561. data/vendor/ggml/src/ggml-hexagon/htp/hvx-repl.h +74 -0
  562. data/vendor/ggml/src/ggml-hexagon/htp/hvx-scale.h +199 -0
  563. data/vendor/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h +181 -0
  564. data/vendor/ggml/src/ggml-hexagon/htp/hvx-sin-cos.h +90 -0
  565. data/vendor/ggml/src/ggml-hexagon/htp/hvx-sqrt.h +189 -0
  566. data/vendor/ggml/src/ggml-hexagon/htp/hvx-types.h +36 -0
  567. data/vendor/ggml/src/ggml-hexagon/htp/hvx-utils.h +24 -0
  568. data/vendor/ggml/src/ggml-hexagon/htp/im2col-ops.c +306 -0
  569. data/vendor/ggml/src/ggml-hexagon/htp/main.c +1245 -0
  570. data/vendor/ggml/src/ggml-hexagon/htp/matmul-ops.c +4078 -0
  571. data/vendor/ggml/src/ggml-hexagon/htp/matmul-ops.h +778 -0
  572. data/vendor/ggml/src/ggml-hexagon/htp/pad-ops.c +547 -0
  573. data/vendor/ggml/src/ggml-hexagon/htp/repeat-ops.c +148 -0
  574. data/vendor/ggml/src/ggml-hexagon/htp/rope-ops.c +733 -0
  575. data/vendor/ggml/src/ggml-hexagon/htp/set-rows-ops.c +230 -0
  576. data/vendor/ggml/src/ggml-hexagon/htp/set-rows-ops.h +74 -0
  577. data/vendor/ggml/src/ggml-hexagon/htp/softmax-ops.c +407 -0
  578. data/vendor/ggml/src/ggml-hexagon/htp/solve-tri-ops.c +267 -0
  579. data/vendor/ggml/src/ggml-hexagon/htp/ssm-conv.c +433 -0
  580. data/vendor/ggml/src/ggml-hexagon/htp/sum-rows-ops.c +128 -0
  581. data/vendor/ggml/src/ggml-hexagon/htp/unary-ops.c +1264 -0
  582. data/vendor/ggml/src/ggml-hexagon/htp/unary-ops.h +176 -0
  583. data/vendor/ggml/src/ggml-hexagon/htp/work-queue.c +244 -0
  584. data/vendor/ggml/src/ggml-hexagon/htp/work-queue.h +38 -0
  585. data/vendor/ggml/src/ggml-hexagon/htp-drv.cpp +425 -0
  586. data/vendor/ggml/src/ggml-hexagon/htp-drv.h +123 -0
  587. data/vendor/ggml/src/ggml-hexagon/htp-opnode.h +372 -0
  588. data/vendor/ggml/src/ggml-hexagon/libdl.h +79 -0
  589. data/vendor/ggml/src/ggml-hexagon/libggml-htp.inf +36 -0
  590. data/vendor/ggml/src/ggml-hip/CMakeLists.txt +153 -0
  591. data/vendor/ggml/src/ggml-impl.h +783 -0
  592. data/vendor/ggml/src/ggml-metal/CMakeLists.txt +240 -0
  593. data/vendor/ggml/src/ggml-metal/ggml-metal-common.cpp +474 -0
  594. data/vendor/ggml/src/ggml-metal/ggml-metal-common.h +56 -0
  595. data/vendor/ggml/src/ggml-metal/ggml-metal-context.h +41 -0
  596. data/vendor/ggml/src/ggml-metal/ggml-metal-context.m +759 -0
  597. data/vendor/ggml/src/ggml-metal/ggml-metal-device.cpp +2330 -0
  598. data/vendor/ggml/src/ggml-metal/ggml-metal-device.h +355 -0
  599. data/vendor/ggml/src/ggml-metal/ggml-metal-device.m +2473 -0
  600. data/vendor/ggml/src/ggml-metal/ggml-metal-impl.h +1345 -0
  601. data/vendor/ggml/src/ggml-metal/ggml-metal-ops.cpp +5610 -0
  602. data/vendor/ggml/src/ggml-metal/ggml-metal-ops.h +106 -0
  603. data/vendor/ggml/src/ggml-metal/ggml-metal-tuning.cpp +3732 -0
  604. data/vendor/ggml/src/ggml-metal/ggml-metal-tuning.h +77 -0
  605. data/vendor/ggml/src/ggml-metal/ggml-metal.cpp +1006 -0
  606. data/vendor/ggml/src/ggml-metal/kernels/argsort.metal +337 -0
  607. data/vendor/ggml/src/ggml-metal/kernels/binbcast.metal +228 -0
  608. data/vendor/ggml/src/ggml-metal/kernels/common.h +126 -0
  609. data/vendor/ggml/src/ggml-metal/kernels/conv.metal +724 -0
  610. data/vendor/ggml/src/ggml-metal/kernels/dequantize.h +735 -0
  611. data/vendor/ggml/src/ggml-metal/kernels/fa.metal +2445 -0
  612. data/vendor/ggml/src/ggml-metal/kernels/gated_delta_net.metal +250 -0
  613. data/vendor/ggml/src/ggml-metal/kernels/misc.metal +595 -0
  614. data/vendor/ggml/src/ggml-metal/kernels/mul_mm.metal +853 -0
  615. data/vendor/ggml/src/ggml-metal/kernels/mul_mv.metal +3225 -0
  616. data/vendor/ggml/src/ggml-metal/kernels/norm.metal +308 -0
  617. data/vendor/ggml/src/ggml-metal/kernels/pool.metal +148 -0
  618. data/vendor/ggml/src/ggml-metal/kernels/quantize.h +262 -0
  619. data/vendor/ggml/src/ggml-metal/kernels/quantize.metal +480 -0
  620. data/vendor/ggml/src/ggml-metal/kernels/reduce.metal +228 -0
  621. data/vendor/ggml/src/ggml-metal/kernels/rope.metal +333 -0
  622. data/vendor/ggml/src/ggml-metal/kernels/softmax.metal +223 -0
  623. data/vendor/ggml/src/ggml-metal/kernels/solve_tri.metal +75 -0
  624. data/vendor/ggml/src/ggml-metal/kernels/ssm.metal +473 -0
  625. data/vendor/ggml/src/ggml-metal/kernels/tri.metal +69 -0
  626. data/vendor/ggml/src/ggml-metal/kernels/unary.metal +400 -0
  627. data/vendor/ggml/src/ggml-metal/kernels/upscale.metal +179 -0
  628. data/vendor/ggml/src/ggml-metal/kernels/wkv.metal +179 -0
  629. data/vendor/ggml/src/ggml-musa/CMakeLists.txt +123 -0
  630. data/vendor/ggml/src/ggml-musa/mudnn.cu +112 -0
  631. data/vendor/ggml/src/ggml-musa/mudnn.cuh +12 -0
  632. data/vendor/ggml/src/ggml-opencl/CMakeLists.txt +246 -0
  633. data/vendor/ggml/src/ggml-opencl/cl-program-cache.cpp +453 -0
  634. data/vendor/ggml/src/ggml-opencl/cl-program-cache.h +75 -0
  635. data/vendor/ggml/src/ggml-opencl/fa_tune.h +92 -0
  636. data/vendor/ggml/src/ggml-opencl/ggml-opencl.cpp +26982 -0
  637. data/vendor/ggml/src/ggml-opencl/kernels/abs.cl +113 -0
  638. data/vendor/ggml/src/ggml-opencl/kernels/add.cl +190 -0
  639. data/vendor/ggml/src/ggml-opencl/kernels/add_id.cl +42 -0
  640. data/vendor/ggml/src/ggml-opencl/kernels/argsort.cl +86 -0
  641. data/vendor/ggml/src/ggml-opencl/kernels/clamp.cl +20 -0
  642. data/vendor/ggml/src/ggml-opencl/kernels/concat.cl +118 -0
  643. data/vendor/ggml/src/ggml-opencl/kernels/conv2d.cl +185 -0
  644. data/vendor/ggml/src/ggml-opencl/kernels/conv2d_f16_f32.cl +176 -0
  645. data/vendor/ggml/src/ggml-opencl/kernels/cpy.cl +288 -0
  646. data/vendor/ggml/src/ggml-opencl/kernels/cumsum.cl +139 -0
  647. data/vendor/ggml/src/ggml-opencl/kernels/cvt.cl +2663 -0
  648. data/vendor/ggml/src/ggml-opencl/kernels/diag.cl +27 -0
  649. data/vendor/ggml/src/ggml-opencl/kernels/diag_mask_inf.cl +58 -0
  650. data/vendor/ggml/src/ggml-opencl/kernels/div.cl +138 -0
  651. data/vendor/ggml/src/ggml-opencl/kernels/embed_kernel.py +26 -0
  652. data/vendor/ggml/src/ggml-opencl/kernels/exp.cl +125 -0
  653. data/vendor/ggml/src/ggml-opencl/kernels/expm1.cl +113 -0
  654. data/vendor/ggml/src/ggml-opencl/kernels/fill.cl +17 -0
  655. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f16.cl +421 -0
  656. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32.cl +422 -0
  657. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32_f16.cl +2656 -0
  658. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32_q4_0.cl +2011 -0
  659. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32_q8_0.cl +1840 -0
  660. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_pre_f16.cl +156 -0
  661. data/vendor/ggml/src/ggml-opencl/kernels/gated_delta_net.cl +249 -0
  662. data/vendor/ggml/src/ggml-opencl/kernels/gelu.cl +89 -0
  663. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_f32.cl +162 -0
  664. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_f32_ns.cl +376 -0
  665. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_q8_1_dp4a.cl +190 -0
  666. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_0_f32_ns.cl +324 -0
  667. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_0_q8_1_dp4a.cl +169 -0
  668. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_1_f32_ns.cl +326 -0
  669. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_k_f32_ns.cl +348 -0
  670. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_k_q8_1_dp4a.cl +209 -0
  671. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q5_0_f32_ns.cl +328 -0
  672. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q5_1_f32_ns.cl +330 -0
  673. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q5_k_f32_ns.cl +356 -0
  674. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q6_k_f32_ns.cl +335 -0
  675. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q6_k_q8_1_dp4a.cl +200 -0
  676. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q8_0_f32_ns.cl +221 -0
  677. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q8_1_dp4a.cl +226 -0
  678. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_nl_f32.cl +150 -0
  679. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_nl_q8_1_dp4a.cl +143 -0
  680. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q1_0_f32.cl +94 -0
  681. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_f32.cl +139 -0
  682. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_q8_1_dp4a.cl +127 -0
  683. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_1_f32.cl +132 -0
  684. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_f32.cl +490 -0
  685. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_q8_1_dp4a.cl +281 -0
  686. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_0_f32.cl +131 -0
  687. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_0_q8_1_dp4a.cl +235 -0
  688. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_1_f32.cl +134 -0
  689. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_f32.cl +176 -0
  690. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_q8_1_dp4a.cl +164 -0
  691. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32.cl +245 -0
  692. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32_tiled.cl +136 -0
  693. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_q8_1_dp4a.cl +144 -0
  694. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_f32.cl +129 -0
  695. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_q8_1_dp4a.cl +212 -0
  696. data/vendor/ggml/src/ggml-opencl/kernels/gemm_xmem_f16_f32_os8.cl +233 -0
  697. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32.cl +156 -0
  698. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32_ns.cl +257 -0
  699. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q4_0_f32_ns.cl +120 -0
  700. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q4_1_f32_ns.cl +123 -0
  701. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q4_k_f32_ns.cl +266 -0
  702. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q5_0_f32_ns.cl +123 -0
  703. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q5_1_f32_ns.cl +125 -0
  704. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q5_k_f32_ns.cl +160 -0
  705. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q6_k_f32_ns.cl +141 -0
  706. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq4_nl_f32.cl +307 -0
  707. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q1_0_f32.cl +125 -0
  708. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32.cl +383 -0
  709. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32_spec.cl +272 -0
  710. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_1_f32.cl +384 -0
  711. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32.cl +823 -0
  712. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_o4.cl +349 -0
  713. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_tiled.cl +118 -0
  714. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_0_f32.cl +296 -0
  715. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_1_f32.cl +299 -0
  716. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_k_f32.cl +453 -0
  717. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32.cl +409 -0
  718. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_o4.cl +372 -0
  719. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_tiled.cl +196 -0
  720. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q8_0_f32.cl +280 -0
  721. data/vendor/ggml/src/ggml-opencl/kernels/get_rows.cl +193 -0
  722. data/vendor/ggml/src/ggml-opencl/kernels/glu.cl +443 -0
  723. data/vendor/ggml/src/ggml-opencl/kernels/group_norm.cl +121 -0
  724. data/vendor/ggml/src/ggml-opencl/kernels/im2col_f16.cl +57 -0
  725. data/vendor/ggml/src/ggml-opencl/kernels/im2col_f32.cl +57 -0
  726. data/vendor/ggml/src/ggml-opencl/kernels/l2_norm.cl +71 -0
  727. data/vendor/ggml/src/ggml-opencl/kernels/mean.cl +140 -0
  728. data/vendor/ggml/src/ggml-opencl/kernels/moe_add_id_glu.cl +76 -0
  729. data/vendor/ggml/src/ggml-opencl/kernels/moe_combine.cl +79 -0
  730. data/vendor/ggml/src/ggml-opencl/kernels/moe_reorder_b.cl +30 -0
  731. data/vendor/ggml/src/ggml-opencl/kernels/moe_reorder_quant_a_q8_1.cl +64 -0
  732. data/vendor/ggml/src/ggml-opencl/kernels/moe_sort_by_expert.cl +155 -0
  733. data/vendor/ggml/src/ggml-opencl/kernels/mul.cl +152 -0
  734. data/vendor/ggml/src/ggml-opencl/kernels/mul_mat_f16_f32.cl +130 -0
  735. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_kq_kqv.cl +273 -0
  736. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_l4_lm.cl +146 -0
  737. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_f32_f32_l4_lm.cl +196 -0
  738. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_iq4_nl_f32_l4_lm.cl +171 -0
  739. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q1_0_f32_l4_lm.cl +156 -0
  740. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q4_0_f32_l4_lm.cl +163 -0
  741. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q4_1_f32_l4_lm.cl +165 -0
  742. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q4_k_f32_l4_lm.cl +189 -0
  743. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q5_0_f32_l4_lm.cl +173 -0
  744. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q5_1_f32_l4_lm.cl +175 -0
  745. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q5_k_f32_l4_lm.cl +202 -0
  746. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q6_k_f32_l4_lm.cl +158 -0
  747. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q8_0_f32_l4_lm.cl +154 -0
  748. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f16.cl +125 -0
  749. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32.cl +125 -0
  750. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_1row.cl +101 -0
  751. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_l4.cl +1237 -0
  752. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_mrow.cl +306 -0
  753. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f32_f32.cl +118 -0
  754. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_mxfp4_f32.cl +189 -0
  755. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_mxfp4_f32_flat.cl +176 -0
  756. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_q4_0_f32_8x_flat.cl +283 -0
  757. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_q8_0_f32.cl +140 -0
  758. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_q8_0_f32_flat.cl +222 -0
  759. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_iq4_nl_f32.cl +164 -0
  760. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_iq4_nl_f32_flat.cl +202 -0
  761. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_mxfp4_f32.cl +144 -0
  762. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_mxfp4_f32_flat.cl +167 -0
  763. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q1_0_f32.cl +141 -0
  764. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q1_0_f32_flat.cl +190 -0
  765. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32.cl +192 -0
  766. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_16x_flat.cl +307 -0
  767. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_8x_flat.cl +265 -0
  768. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_8x_flat.cl +272 -0
  769. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_v.cl +254 -0
  770. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_1_f32.cl +219 -0
  771. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_1_f32_flat.cl +229 -0
  772. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32.cl +182 -0
  773. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32_flat.cl +205 -0
  774. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_0_f32.cl +241 -0
  775. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_0_f32_flat.cl +243 -0
  776. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_1_f32.cl +243 -0
  777. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_1_f32_flat.cl +247 -0
  778. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_k_f32.cl +187 -0
  779. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_k_f32_flat.cl +244 -0
  780. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32.cl +194 -0
  781. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32_flat.cl +272 -0
  782. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q8_0_f32.cl +125 -0
  783. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q8_0_f32_flat.cl +202 -0
  784. data/vendor/ggml/src/ggml-opencl/kernels/neg.cl +125 -0
  785. data/vendor/ggml/src/ggml-opencl/kernels/norm.cl +164 -0
  786. data/vendor/ggml/src/ggml-opencl/kernels/pad.cl +39 -0
  787. data/vendor/ggml/src/ggml-opencl/kernels/quant_a_q8_1.cl +42 -0
  788. data/vendor/ggml/src/ggml-opencl/kernels/relu.cl +16 -0
  789. data/vendor/ggml/src/ggml-opencl/kernels/repeat.cl +38 -0
  790. data/vendor/ggml/src/ggml-opencl/kernels/rms_norm.cl +369 -0
  791. data/vendor/ggml/src/ggml-opencl/kernels/rope.cl +759 -0
  792. data/vendor/ggml/src/ggml-opencl/kernels/scale.cl +27 -0
  793. data/vendor/ggml/src/ggml-opencl/kernels/set_rows.cl +708 -0
  794. data/vendor/ggml/src/ggml-opencl/kernels/sigmoid.cl +29 -0
  795. data/vendor/ggml/src/ggml-opencl/kernels/silu.cl +30 -0
  796. data/vendor/ggml/src/ggml-opencl/kernels/softmax_4_f16.cl +108 -0
  797. data/vendor/ggml/src/ggml-opencl/kernels/softmax_4_f32.cl +108 -0
  798. data/vendor/ggml/src/ggml-opencl/kernels/softmax_f16.cl +107 -0
  799. data/vendor/ggml/src/ggml-opencl/kernels/softmax_f32.cl +107 -0
  800. data/vendor/ggml/src/ggml-opencl/kernels/softplus.cl +116 -0
  801. data/vendor/ggml/src/ggml-opencl/kernels/solve_tri.cl +51 -0
  802. data/vendor/ggml/src/ggml-opencl/kernels/sqr.cl +53 -0
  803. data/vendor/ggml/src/ggml-opencl/kernels/sqrt.cl +53 -0
  804. data/vendor/ggml/src/ggml-opencl/kernels/ssm_conv.cl +77 -0
  805. data/vendor/ggml/src/ggml-opencl/kernels/ssm_scan.cl +216 -0
  806. data/vendor/ggml/src/ggml-opencl/kernels/sub.cl +138 -0
  807. data/vendor/ggml/src/ggml-opencl/kernels/sum_rows.cl +140 -0
  808. data/vendor/ggml/src/ggml-opencl/kernels/tanh.cl +109 -0
  809. data/vendor/ggml/src/ggml-opencl/kernels/transpose.cl +143 -0
  810. data/vendor/ggml/src/ggml-opencl/kernels/tri.cl +32 -0
  811. data/vendor/ggml/src/ggml-opencl/kernels/tsembd.cl +48 -0
  812. data/vendor/ggml/src/ggml-opencl/kernels/upscale.cl +120 -0
  813. data/vendor/ggml/src/ggml-opencl/libdl.h +79 -0
  814. data/vendor/ggml/src/ggml-openvino/CMakeLists.txt +22 -0
  815. data/vendor/ggml/src/ggml-openvino/ggml-decoder.cpp +2057 -0
  816. data/vendor/ggml/src/ggml-openvino/ggml-decoder.h +417 -0
  817. data/vendor/ggml/src/ggml-openvino/ggml-openvino-extra.cpp +493 -0
  818. data/vendor/ggml/src/ggml-openvino/ggml-openvino-extra.h +216 -0
  819. data/vendor/ggml/src/ggml-openvino/ggml-openvino.cpp +1566 -0
  820. data/vendor/ggml/src/ggml-openvino/ggml-quants.cpp +1273 -0
  821. data/vendor/ggml/src/ggml-openvino/ggml-quants.h +186 -0
  822. data/vendor/ggml/src/ggml-openvino/model-cache.cpp +272 -0
  823. data/vendor/ggml/src/ggml-openvino/model-cache.h +56 -0
  824. data/vendor/ggml/src/ggml-openvino/openvino/decoder.h +133 -0
  825. data/vendor/ggml/src/ggml-openvino/openvino/frontend.cpp +27 -0
  826. data/vendor/ggml/src/ggml-openvino/openvino/frontend.h +23 -0
  827. data/vendor/ggml/src/ggml-openvino/openvino/input_model.cpp +17 -0
  828. data/vendor/ggml/src/ggml-openvino/openvino/input_model.h +29 -0
  829. data/vendor/ggml/src/ggml-openvino/openvino/node_context.h +171 -0
  830. data/vendor/ggml/src/ggml-openvino/openvino/op/add.cpp +45 -0
  831. data/vendor/ggml/src/ggml-openvino/openvino/op/add_id.cpp +76 -0
  832. data/vendor/ggml/src/ggml-openvino/openvino/op/argsort.cpp +47 -0
  833. data/vendor/ggml/src/ggml-openvino/openvino/op/clamp.cpp +33 -0
  834. data/vendor/ggml/src/ggml-openvino/openvino/op/concat.cpp +48 -0
  835. data/vendor/ggml/src/ggml-openvino/openvino/op/cont.cpp +40 -0
  836. data/vendor/ggml/src/ggml-openvino/openvino/op/cpy.cpp +295 -0
  837. data/vendor/ggml/src/ggml-openvino/openvino/op/cumsum.cpp +29 -0
  838. data/vendor/ggml/src/ggml-openvino/openvino/op/diag.cpp +58 -0
  839. data/vendor/ggml/src/ggml-openvino/openvino/op/div.cpp +146 -0
  840. data/vendor/ggml/src/ggml-openvino/openvino/op/fill.cpp +34 -0
  841. data/vendor/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp +241 -0
  842. data/vendor/ggml/src/ggml-openvino/openvino/op/gated_delta_net.cpp +327 -0
  843. data/vendor/ggml/src/ggml-openvino/openvino/op/gated_delta_net.hpp +65 -0
  844. data/vendor/ggml/src/ggml-openvino/openvino/op/gather_matmul.hpp +43 -0
  845. data/vendor/ggml/src/ggml-openvino/openvino/op/get_rows.cpp +142 -0
  846. data/vendor/ggml/src/ggml-openvino/openvino/op/glu_geglu.cpp +75 -0
  847. data/vendor/ggml/src/ggml-openvino/openvino/op/glu_geglu_quick.cpp +64 -0
  848. data/vendor/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp +110 -0
  849. data/vendor/ggml/src/ggml-openvino/openvino/op/im2col.cpp +120 -0
  850. data/vendor/ggml/src/ggml-openvino/openvino/op/l2_norm.cpp +61 -0
  851. data/vendor/ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp +283 -0
  852. data/vendor/ggml/src/ggml-openvino/openvino/op/mulmat.cpp +100 -0
  853. data/vendor/ggml/src/ggml-openvino/openvino/op/norm.cpp +58 -0
  854. data/vendor/ggml/src/ggml-openvino/openvino/op/pad.cpp +95 -0
  855. data/vendor/ggml/src/ggml-openvino/openvino/op/permute.cpp +147 -0
  856. data/vendor/ggml/src/ggml-openvino/openvino/op/pool_2d.cpp +53 -0
  857. data/vendor/ggml/src/ggml-openvino/openvino/op/repeat.cpp +48 -0
  858. data/vendor/ggml/src/ggml-openvino/openvino/op/reshape.cpp +113 -0
  859. data/vendor/ggml/src/ggml-openvino/openvino/op/rms_norm.cpp +81 -0
  860. data/vendor/ggml/src/ggml-openvino/openvino/op/roll.cpp +36 -0
  861. data/vendor/ggml/src/ggml-openvino/openvino/op/rope.cpp +303 -0
  862. data/vendor/ggml/src/ggml-openvino/openvino/op/scale.cpp +86 -0
  863. data/vendor/ggml/src/ggml-openvino/openvino/op/set.cpp +76 -0
  864. data/vendor/ggml/src/ggml-openvino/openvino/op/set_rows.cpp +95 -0
  865. data/vendor/ggml/src/ggml-openvino/openvino/op/softmax.cpp +166 -0
  866. data/vendor/ggml/src/ggml-openvino/openvino/op/solve_tri.cpp +108 -0
  867. data/vendor/ggml/src/ggml-openvino/openvino/op/sqr.cpp +35 -0
  868. data/vendor/ggml/src/ggml-openvino/openvino/op/ssm_conv.cpp +61 -0
  869. data/vendor/ggml/src/ggml-openvino/openvino/op/sum_rows.cpp +27 -0
  870. data/vendor/ggml/src/ggml-openvino/openvino/op/transpose.cpp +54 -0
  871. data/vendor/ggml/src/ggml-openvino/openvino/op/tri.cpp +82 -0
  872. data/vendor/ggml/src/ggml-openvino/openvino/op/unary_silu.cpp +27 -0
  873. data/vendor/ggml/src/ggml-openvino/openvino/op/unary_softplus.cpp +38 -0
  874. data/vendor/ggml/src/ggml-openvino/openvino/op/view.cpp +245 -0
  875. data/vendor/ggml/src/ggml-openvino/openvino/op_table.cpp +88 -0
  876. data/vendor/ggml/src/ggml-openvino/openvino/op_table.h +67 -0
  877. data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_conv.cpp +212 -0
  878. data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_conv.h +17 -0
  879. data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_sdpa.cpp +60 -0
  880. data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_sdpa.h +17 -0
  881. data/vendor/ggml/src/ggml-openvino/openvino/pass/mark_decompression_convert_constant_folding.h +29 -0
  882. data/vendor/ggml/src/ggml-openvino/openvino/pass/mark_dequantization_subgraph.h +44 -0
  883. data/vendor/ggml/src/ggml-openvino/openvino/pass/squeeze_matmul.cpp +58 -0
  884. data/vendor/ggml/src/ggml-openvino/openvino/pass/squeeze_matmul.h +17 -0
  885. data/vendor/ggml/src/ggml-openvino/openvino/rt_info/weightless_caching_attributes.hpp +41 -0
  886. data/vendor/ggml/src/ggml-openvino/openvino/translate_session.cpp +432 -0
  887. data/vendor/ggml/src/ggml-openvino/openvino/translate_session.h +29 -0
  888. data/vendor/ggml/src/ggml-openvino/openvino/utils.cpp +886 -0
  889. data/vendor/ggml/src/ggml-openvino/openvino/utils.h +88 -0
  890. data/vendor/ggml/src/ggml-openvino/utils.cpp +1496 -0
  891. data/vendor/ggml/src/ggml-openvino/utils.h +195 -0
  892. data/vendor/ggml/src/ggml-opt.cpp +1094 -0
  893. data/vendor/ggml/src/ggml-quants.c +5667 -0
  894. data/vendor/ggml/src/ggml-quants.h +115 -0
  895. data/vendor/ggml/src/ggml-rpc/CMakeLists.txt +49 -0
  896. data/vendor/ggml/src/ggml-rpc/ggml-rpc.cpp +2351 -0
  897. data/vendor/ggml/src/ggml-rpc/transport-apple.cpp +481 -0
  898. data/vendor/ggml/src/ggml-rpc/transport-apple.h +27 -0
  899. data/vendor/ggml/src/ggml-rpc/transport.cpp +739 -0
  900. data/vendor/ggml/src/ggml-rpc/transport.h +38 -0
  901. data/vendor/ggml/src/ggml-sycl/CMakeLists.txt +218 -0
  902. data/vendor/ggml/src/ggml-sycl/add-id.cpp +81 -0
  903. data/vendor/ggml/src/ggml-sycl/add-id.hpp +8 -0
  904. data/vendor/ggml/src/ggml-sycl/backend.hpp +52 -0
  905. data/vendor/ggml/src/ggml-sycl/base.hpp +36 -0
  906. data/vendor/ggml/src/ggml-sycl/binbcast.cpp +650 -0
  907. data/vendor/ggml/src/ggml-sycl/binbcast.hpp +69 -0
  908. data/vendor/ggml/src/ggml-sycl/col2im-1d.cpp +102 -0
  909. data/vendor/ggml/src/ggml-sycl/col2im-1d.hpp +8 -0
  910. data/vendor/ggml/src/ggml-sycl/common.cpp +153 -0
  911. data/vendor/ggml/src/ggml-sycl/common.hpp +1029 -0
  912. data/vendor/ggml/src/ggml-sycl/concat.cpp +509 -0
  913. data/vendor/ggml/src/ggml-sycl/concat.hpp +20 -0
  914. data/vendor/ggml/src/ggml-sycl/conv.cpp +101 -0
  915. data/vendor/ggml/src/ggml-sycl/conv.hpp +20 -0
  916. data/vendor/ggml/src/ggml-sycl/conv2d-dw.cpp +170 -0
  917. data/vendor/ggml/src/ggml-sycl/conv2d-dw.hpp +10 -0
  918. data/vendor/ggml/src/ggml-sycl/conv2d-transpose.cpp +125 -0
  919. data/vendor/ggml/src/ggml-sycl/conv2d-transpose.hpp +10 -0
  920. data/vendor/ggml/src/ggml-sycl/conv2d.cpp +150 -0
  921. data/vendor/ggml/src/ggml-sycl/conv2d.hpp +10 -0
  922. data/vendor/ggml/src/ggml-sycl/conv3d.cpp +224 -0
  923. data/vendor/ggml/src/ggml-sycl/conv3d.hpp +8 -0
  924. data/vendor/ggml/src/ggml-sycl/convert.cpp +877 -0
  925. data/vendor/ggml/src/ggml-sycl/convert.hpp +64 -0
  926. data/vendor/ggml/src/ggml-sycl/count-equal.cpp +79 -0
  927. data/vendor/ggml/src/ggml-sycl/count-equal.hpp +9 -0
  928. data/vendor/ggml/src/ggml-sycl/cpy.cpp +1433 -0
  929. data/vendor/ggml/src/ggml-sycl/cpy.hpp +537 -0
  930. data/vendor/ggml/src/ggml-sycl/cross_entropy_loss.cpp +255 -0
  931. data/vendor/ggml/src/ggml-sycl/cross_entropy_loss.hpp +7 -0
  932. data/vendor/ggml/src/ggml-sycl/cumsum.cpp +148 -0
  933. data/vendor/ggml/src/ggml-sycl/cumsum.hpp +5 -0
  934. data/vendor/ggml/src/ggml-sycl/dequantize.hpp +1681 -0
  935. data/vendor/ggml/src/ggml-sycl/diag.cpp +67 -0
  936. data/vendor/ggml/src/ggml-sycl/diag.hpp +5 -0
  937. data/vendor/ggml/src/ggml-sycl/dmmv.cpp +2227 -0
  938. data/vendor/ggml/src/ggml-sycl/dmmv.hpp +27 -0
  939. data/vendor/ggml/src/ggml-sycl/dpct/helper.hpp +3782 -0
  940. data/vendor/ggml/src/ggml-sycl/dsv4-hc.cpp +280 -0
  941. data/vendor/ggml/src/ggml-sycl/dsv4-hc.hpp +10 -0
  942. data/vendor/ggml/src/ggml-sycl/element_wise.cpp +1432 -0
  943. data/vendor/ggml/src/ggml-sycl/element_wise.hpp +135 -0
  944. data/vendor/ggml/src/ggml-sycl/esimd.hpp +589 -0
  945. data/vendor/ggml/src/ggml-sycl/fattn-buffers.cpp +56 -0
  946. data/vendor/ggml/src/ggml-sycl/fattn-buffers.hpp +63 -0
  947. data/vendor/ggml/src/ggml-sycl/fattn-common.hpp +1185 -0
  948. data/vendor/ggml/src/ggml-sycl/fattn-mkl.cpp +689 -0
  949. data/vendor/ggml/src/ggml-sycl/fattn-onednn.cpp +450 -0
  950. data/vendor/ggml/src/ggml-sycl/fattn-onednn.hpp +18 -0
  951. data/vendor/ggml/src/ggml-sycl/fattn-tile.cpp +59 -0
  952. data/vendor/ggml/src/ggml-sycl/fattn-tile.hpp +1246 -0
  953. data/vendor/ggml/src/ggml-sycl/fattn-vec.hpp +684 -0
  954. data/vendor/ggml/src/ggml-sycl/fattn.cpp +452 -0
  955. data/vendor/ggml/src/ggml-sycl/fattn.hpp +42 -0
  956. data/vendor/ggml/src/ggml-sycl/fill.cpp +55 -0
  957. data/vendor/ggml/src/ggml-sycl/fill.hpp +5 -0
  958. data/vendor/ggml/src/ggml-sycl/fusion.cpp +212 -0
  959. data/vendor/ggml/src/ggml-sycl/fusion.hpp +17 -0
  960. data/vendor/ggml/src/ggml-sycl/fwht.cpp +119 -0
  961. data/vendor/ggml/src/ggml-sycl/fwht.hpp +12 -0
  962. data/vendor/ggml/src/ggml-sycl/gated_delta_net.cpp +366 -0
  963. data/vendor/ggml/src/ggml-sycl/gated_delta_net.hpp +19 -0
  964. data/vendor/ggml/src/ggml-sycl/gemm.hpp +93 -0
  965. data/vendor/ggml/src/ggml-sycl/getrows.cpp +368 -0
  966. data/vendor/ggml/src/ggml-sycl/getrows.hpp +20 -0
  967. data/vendor/ggml/src/ggml-sycl/ggml-sycl.cpp +7028 -0
  968. data/vendor/ggml/src/ggml-sycl/gla.cpp +106 -0
  969. data/vendor/ggml/src/ggml-sycl/gla.hpp +8 -0
  970. data/vendor/ggml/src/ggml-sycl/im2col.cpp +400 -0
  971. data/vendor/ggml/src/ggml-sycl/im2col.hpp +23 -0
  972. data/vendor/ggml/src/ggml-sycl/lightning-indexer.cpp +197 -0
  973. data/vendor/ggml/src/ggml-sycl/lightning-indexer.hpp +8 -0
  974. data/vendor/ggml/src/ggml-sycl/mem.cpp +162 -0
  975. data/vendor/ggml/src/ggml-sycl/mem.hpp +16 -0
  976. data/vendor/ggml/src/ggml-sycl/mmq.cpp +3030 -0
  977. data/vendor/ggml/src/ggml-sycl/mmq.hpp +33 -0
  978. data/vendor/ggml/src/ggml-sycl/mmvq.cpp +3033 -0
  979. data/vendor/ggml/src/ggml-sycl/mmvq.hpp +76 -0
  980. data/vendor/ggml/src/ggml-sycl/norm.cpp +963 -0
  981. data/vendor/ggml/src/ggml-sycl/norm.hpp +32 -0
  982. data/vendor/ggml/src/ggml-sycl/opt-step.cpp +131 -0
  983. data/vendor/ggml/src/ggml-sycl/opt-step.hpp +6 -0
  984. data/vendor/ggml/src/ggml-sycl/outprod.cpp +83 -0
  985. data/vendor/ggml/src/ggml-sycl/outprod.hpp +10 -0
  986. data/vendor/ggml/src/ggml-sycl/pad.cpp +97 -0
  987. data/vendor/ggml/src/ggml-sycl/pad.hpp +24 -0
  988. data/vendor/ggml/src/ggml-sycl/pad_reflect_1d.cpp +100 -0
  989. data/vendor/ggml/src/ggml-sycl/pad_reflect_1d.hpp +10 -0
  990. data/vendor/ggml/src/ggml-sycl/pool.cpp +185 -0
  991. data/vendor/ggml/src/ggml-sycl/pool.hpp +22 -0
  992. data/vendor/ggml/src/ggml-sycl/presets.hpp +79 -0
  993. data/vendor/ggml/src/ggml-sycl/quantize.hpp +133 -0
  994. data/vendor/ggml/src/ggml-sycl/quants.hpp +204 -0
  995. data/vendor/ggml/src/ggml-sycl/repeat_back.cpp +76 -0
  996. data/vendor/ggml/src/ggml-sycl/repeat_back.hpp +8 -0
  997. data/vendor/ggml/src/ggml-sycl/roll.cpp +122 -0
  998. data/vendor/ggml/src/ggml-sycl/roll.hpp +20 -0
  999. data/vendor/ggml/src/ggml-sycl/rope.cpp +651 -0
  1000. data/vendor/ggml/src/ggml-sycl/rope.hpp +26 -0
  1001. data/vendor/ggml/src/ggml-sycl/set.cpp +73 -0
  1002. data/vendor/ggml/src/ggml-sycl/set.hpp +5 -0
  1003. data/vendor/ggml/src/ggml-sycl/set_rows.cpp +577 -0
  1004. data/vendor/ggml/src/ggml-sycl/set_rows.hpp +8 -0
  1005. data/vendor/ggml/src/ggml-sycl/softmax.cpp +425 -0
  1006. data/vendor/ggml/src/ggml-sycl/softmax.hpp +24 -0
  1007. data/vendor/ggml/src/ggml-sycl/solve_tri.cpp +172 -0
  1008. data/vendor/ggml/src/ggml-sycl/solve_tri.hpp +8 -0
  1009. data/vendor/ggml/src/ggml-sycl/ssm_conv.cpp +136 -0
  1010. data/vendor/ggml/src/ggml-sycl/ssm_conv.hpp +5 -0
  1011. data/vendor/ggml/src/ggml-sycl/ssm_scan.cpp +170 -0
  1012. data/vendor/ggml/src/ggml-sycl/ssm_scan.hpp +5 -0
  1013. data/vendor/ggml/src/ggml-sycl/sycl_hw.cpp +67 -0
  1014. data/vendor/ggml/src/ggml-sycl/sycl_hw.hpp +38 -0
  1015. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq112-dv112.cpp +5 -0
  1016. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq128-dv128.cpp +5 -0
  1017. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq256-dv256.cpp +5 -0
  1018. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq40-dv40.cpp +5 -0
  1019. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq512-dv512.cpp +6 -0
  1020. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq576-dv512.cpp +5 -0
  1021. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq64-dv64.cpp +5 -0
  1022. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq72-dv72.cpp +5 -0
  1023. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq80-dv80.cpp +5 -0
  1024. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq96-dv96.cpp +5 -0
  1025. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-f16.cpp +8 -0
  1026. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q4_0.cpp +8 -0
  1027. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q4_1.cpp +8 -0
  1028. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q5_0.cpp +8 -0
  1029. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q5_1.cpp +8 -0
  1030. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q8_0.cpp +8 -0
  1031. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-f16.cpp +8 -0
  1032. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q4_0.cpp +8 -0
  1033. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q4_1.cpp +8 -0
  1034. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q5_0.cpp +8 -0
  1035. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q5_1.cpp +8 -0
  1036. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q8_0.cpp +8 -0
  1037. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-f16.cpp +8 -0
  1038. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q4_0.cpp +8 -0
  1039. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q4_1.cpp +8 -0
  1040. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q5_0.cpp +8 -0
  1041. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q5_1.cpp +8 -0
  1042. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q8_0.cpp +8 -0
  1043. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-f16.cpp +8 -0
  1044. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q4_0.cpp +8 -0
  1045. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q4_1.cpp +8 -0
  1046. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q5_0.cpp +8 -0
  1047. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q5_1.cpp +8 -0
  1048. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q8_0.cpp +8 -0
  1049. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-f16.cpp +8 -0
  1050. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q4_0.cpp +8 -0
  1051. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q4_1.cpp +8 -0
  1052. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q5_0.cpp +8 -0
  1053. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q5_1.cpp +8 -0
  1054. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q8_0.cpp +8 -0
  1055. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-f16.cpp +8 -0
  1056. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q4_0.cpp +8 -0
  1057. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q4_1.cpp +8 -0
  1058. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q5_0.cpp +8 -0
  1059. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q5_1.cpp +8 -0
  1060. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q8_0.cpp +8 -0
  1061. data/vendor/ggml/src/ggml-sycl/topk-moe.cpp +620 -0
  1062. data/vendor/ggml/src/ggml-sycl/topk-moe.hpp +12 -0
  1063. data/vendor/ggml/src/ggml-sycl/tsembd.cpp +73 -0
  1064. data/vendor/ggml/src/ggml-sycl/tsembd.hpp +20 -0
  1065. data/vendor/ggml/src/ggml-sycl/type.hpp +112 -0
  1066. data/vendor/ggml/src/ggml-sycl/upscale.cpp +410 -0
  1067. data/vendor/ggml/src/ggml-sycl/upscale.hpp +9 -0
  1068. data/vendor/ggml/src/ggml-sycl/vecdotq.hpp +1726 -0
  1069. data/vendor/ggml/src/ggml-sycl/wkv.cpp +293 -0
  1070. data/vendor/ggml/src/ggml-sycl/wkv.hpp +10 -0
  1071. data/vendor/ggml/src/ggml-threading.cpp +12 -0
  1072. data/vendor/ggml/src/ggml-threading.h +14 -0
  1073. data/vendor/ggml/src/ggml-version.h.in +4 -0
  1074. data/vendor/ggml/src/ggml-virtgpu/CMakeLists.txt +70 -0
  1075. data/vendor/ggml/src/ggml-virtgpu/apir_cs_ggml-rpc-front.cpp +87 -0
  1076. data/vendor/ggml/src/ggml-virtgpu/backend/CMakeLists.txt +21 -0
  1077. data/vendor/ggml/src/ggml-virtgpu/backend/apir_cs_ggml-rpc-back.cpp +115 -0
  1078. data/vendor/ggml/src/ggml-virtgpu/backend/backend-convert.h +13 -0
  1079. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-backend.cpp +102 -0
  1080. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-buffer-type.cpp +105 -0
  1081. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-buffer.cpp +179 -0
  1082. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-device.cpp +149 -0
  1083. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched.cpp +51 -0
  1084. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched.gen.h +73 -0
  1085. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched.h +27 -0
  1086. data/vendor/ggml/src/ggml-virtgpu/backend/backend-virgl-apir.h +32 -0
  1087. data/vendor/ggml/src/ggml-virtgpu/backend/backend.cpp +144 -0
  1088. data/vendor/ggml/src/ggml-virtgpu/backend/shared/api_remoting.h +95 -0
  1089. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_backend.gen.h +94 -0
  1090. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_backend.h +50 -0
  1091. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_cs.h +378 -0
  1092. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_cs_ggml.h +232 -0
  1093. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_cs_rpc.h +58 -0
  1094. data/vendor/ggml/src/ggml-virtgpu/ggml-backend-buffer-type.cpp +81 -0
  1095. data/vendor/ggml/src/ggml-virtgpu/ggml-backend-buffer.cpp +123 -0
  1096. data/vendor/ggml/src/ggml-virtgpu/ggml-backend-device.cpp +160 -0
  1097. data/vendor/ggml/src/ggml-virtgpu/ggml-backend-reg.cpp +213 -0
  1098. data/vendor/ggml/src/ggml-virtgpu/ggml-backend.cpp +72 -0
  1099. data/vendor/ggml/src/ggml-virtgpu/ggml-remoting.h +71 -0
  1100. data/vendor/ggml/src/ggml-virtgpu/ggmlremoting_functions.yaml +166 -0
  1101. data/vendor/ggml/src/ggml-virtgpu/include/apir_hw.h +9 -0
  1102. data/vendor/ggml/src/ggml-virtgpu/regenerate_remoting.py +333 -0
  1103. data/vendor/ggml/src/ggml-virtgpu/virtgpu-apir.h +15 -0
  1104. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-backend.cpp +58 -0
  1105. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-buffer-type.cpp +110 -0
  1106. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-buffer.cpp +173 -0
  1107. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-device.cpp +194 -0
  1108. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-impl.h +36 -0
  1109. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward.gen.h +54 -0
  1110. data/vendor/ggml/src/ggml-virtgpu/virtgpu-shm.cpp +99 -0
  1111. data/vendor/ggml/src/ggml-virtgpu/virtgpu-shm.h +23 -0
  1112. data/vendor/ggml/src/ggml-virtgpu/virtgpu-utils.cpp +179 -0
  1113. data/vendor/ggml/src/ggml-virtgpu/virtgpu-utils.h +86 -0
  1114. data/vendor/ggml/src/ggml-virtgpu/virtgpu.cpp +545 -0
  1115. data/vendor/ggml/src/ggml-virtgpu/virtgpu.h +115 -0
  1116. data/vendor/ggml/src/ggml-vulkan/CMakeLists.txt +254 -0
  1117. data/vendor/ggml/src/ggml-vulkan/cmake/host-toolchain.cmake.in +15 -0
  1118. data/vendor/ggml/src/ggml-vulkan/ggml-vulkan.cpp +20349 -0
  1119. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/CMakeLists.txt +43 -0
  1120. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/acc.comp +37 -0
  1121. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/add.comp +69 -0
  1122. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/add1.comp +28 -0
  1123. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/add_id.comp +42 -0
  1124. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/arange.comp +20 -0
  1125. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/argmax.comp +60 -0
  1126. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/argsort.comp +86 -0
  1127. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/argsort_large.comp +114 -0
  1128. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/col2im_1d.comp +61 -0
  1129. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/concat.comp +41 -0
  1130. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/contig_copy.comp +53 -0
  1131. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv2d_dw.comp +105 -0
  1132. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv2d_mm.comp +473 -0
  1133. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv3d_mm.comp +424 -0
  1134. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv_transpose_1d.comp +98 -0
  1135. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy.comp +25 -0
  1136. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_from_quant.comp +51 -0
  1137. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_to_quant.comp +350 -0
  1138. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_transpose.comp +67 -0
  1139. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_transpose_02.comp +61 -0
  1140. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/count_equal.comp +31 -0
  1141. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/count_experts.comp +135 -0
  1142. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cross_entropy_loss.comp +78 -0
  1143. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cross_entropy_loss_back.comp +75 -0
  1144. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cumsum.comp +83 -0
  1145. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cumsum_multipass1.comp +60 -0
  1146. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cumsum_multipass2.comp +66 -0
  1147. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_f32.comp +20 -0
  1148. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.glsl +741 -0
  1149. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl +1466 -0
  1150. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_head.glsl +13 -0
  1151. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_m.comp +42 -0
  1152. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_s.comp +35 -0
  1153. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_s.comp +44 -0
  1154. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xs.comp +43 -0
  1155. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xxs.comp +49 -0
  1156. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_s.comp +40 -0
  1157. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_xxs.comp +51 -0
  1158. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_nl.comp +32 -0
  1159. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_xs.comp +34 -0
  1160. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_mxfp4.comp +32 -0
  1161. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_nvfp4.comp +32 -0
  1162. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q1_0.comp +29 -0
  1163. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q2_0.comp +29 -0
  1164. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q2_k.comp +34 -0
  1165. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q3_k.comp +42 -0
  1166. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_0.comp +30 -0
  1167. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_1.comp +32 -0
  1168. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_k.comp +68 -0
  1169. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_0.comp +34 -0
  1170. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_1.comp +35 -0
  1171. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_k.comp +70 -0
  1172. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q6_k.comp +33 -0
  1173. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q8_0.comp +42 -0
  1174. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_tq2_0.comp +31 -0
  1175. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/diag.comp +28 -0
  1176. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/diag_mask_inf.comp +34 -0
  1177. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/div.comp +27 -0
  1178. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dot_product_funcs.glsl +27 -0
  1179. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/fa_types.glsl +55 -0
  1180. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/bfloat16.comp +7 -0
  1181. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat.comp +7 -0
  1182. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat2.comp +7 -0
  1183. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat2_decode_vector.comp +7 -0
  1184. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/float_e2m1.comp +7 -0
  1185. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/float_e4m3.comp +7 -0
  1186. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/integer_dot.comp +7 -0
  1187. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/fill.comp +19 -0
  1188. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp +760 -0
  1189. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.glsl +223 -0
  1190. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp +647 -0
  1191. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp +487 -0
  1192. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_dequant.glsl +146 -0
  1193. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_mask_opt.comp +162 -0
  1194. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_mmq_funcs.glsl +203 -0
  1195. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_split_k_reduce.comp +121 -0
  1196. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/fwht.comp +115 -0
  1197. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/gated_delta_net.comp +189 -0
  1198. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/geglu.comp +13 -0
  1199. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/geglu_erf.comp +27 -0
  1200. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/geglu_quick.comp +11 -0
  1201. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/generic_binary_head.glsl +65 -0
  1202. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/generic_head.glsl +11 -0
  1203. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/generic_unary_head.glsl +75 -0
  1204. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/get_rows.comp +42 -0
  1205. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_back.comp +25 -0
  1206. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_quant.comp +51 -0
  1207. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/gla.comp +82 -0
  1208. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/glu_head.glsl +43 -0
  1209. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/glu_main.glsl +35 -0
  1210. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/group_norm.comp +66 -0
  1211. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/im2col.comp +138 -0
  1212. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/im2col_3d.comp +124 -0
  1213. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/l2_norm.comp +41 -0
  1214. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/lightning_indexer.comp +151 -0
  1215. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/log.comp +17 -0
  1216. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul.comp +27 -0
  1217. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_split_k_reduce.comp +48 -0
  1218. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec.comp +264 -0
  1219. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_base.glsl +230 -0
  1220. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iface.glsl +35 -0
  1221. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_m.comp +132 -0
  1222. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_s.comp +95 -0
  1223. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_s.comp +90 -0
  1224. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xs.comp +105 -0
  1225. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xxs.comp +87 -0
  1226. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_s.comp +93 -0
  1227. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_xxs.comp +88 -0
  1228. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_nc.comp +124 -0
  1229. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_p021.comp +156 -0
  1230. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q2_k.comp +128 -0
  1231. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q3_k.comp +132 -0
  1232. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q4_k.comp +134 -0
  1233. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q5_k.comp +165 -0
  1234. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q6_k.comp +130 -0
  1235. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_tq2_0.comp +102 -0
  1236. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq.comp +141 -0
  1237. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq_funcs.glsl +564 -0
  1238. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp +475 -0
  1239. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp +690 -0
  1240. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl +670 -0
  1241. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_id_funcs.glsl +89 -0
  1242. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp +317 -0
  1243. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_funcs.glsl +488 -0
  1244. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_shmem_types.glsl +99 -0
  1245. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/multi_add.comp +194 -0
  1246. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/norm.comp +44 -0
  1247. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_adamw.comp +42 -0
  1248. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_sgd.comp +22 -0
  1249. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/out_prod.comp +59 -0
  1250. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pad.comp +64 -0
  1251. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pad_reflect_1d.comp +43 -0
  1252. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pool1d.comp +65 -0
  1253. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pool2d.comp +74 -0
  1254. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/quantize_q8_1.comp +127 -0
  1255. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/reglu.comp +9 -0
  1256. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/repeat.comp +26 -0
  1257. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/repeat_back.comp +37 -0
  1258. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp +150 -0
  1259. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_back.comp +55 -0
  1260. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_partials.comp +65 -0
  1261. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/roll.comp +46 -0
  1262. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_funcs.glsl +218 -0
  1263. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_head.glsl +19 -0
  1264. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_multi.comp +17 -0
  1265. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_neox.comp +17 -0
  1266. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_norm.comp +17 -0
  1267. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_params.glsl +35 -0
  1268. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_vision.comp +17 -0
  1269. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/scale.comp +24 -0
  1270. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/silu_back.comp +26 -0
  1271. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/snake.comp +49 -0
  1272. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max.comp +195 -0
  1273. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_back.comp +54 -0
  1274. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large1.comp +62 -0
  1275. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large2.comp +79 -0
  1276. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large3.comp +65 -0
  1277. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large_common.glsl +53 -0
  1278. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/solve_tri.comp +81 -0
  1279. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/ssm_conv.comp +60 -0
  1280. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/ssm_scan.comp +134 -0
  1281. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/sub.comp +29 -0
  1282. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.comp +47 -0
  1283. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.glsl +19 -0
  1284. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/swiglu.comp +9 -0
  1285. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/swiglu_clamp.comp +12 -0
  1286. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/swiglu_oai.comp +14 -0
  1287. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/timestep_embedding.comp +42 -0
  1288. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_argsort.comp +118 -0
  1289. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_moe.comp +221 -0
  1290. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_nary_search.comp +246 -0
  1291. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_radix_select.comp +144 -0
  1292. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/tri.comp +42 -0
  1293. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl +1938 -0
  1294. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/unary.comp +168 -0
  1295. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/upscale.comp +178 -0
  1296. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/utils.glsl +37 -0
  1297. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +1367 -0
  1298. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/wkv6.comp +87 -0
  1299. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/wkv7.comp +91 -0
  1300. data/vendor/ggml/src/ggml-webgpu/CMakeLists.txt +83 -0
  1301. data/vendor/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp +3457 -0
  1302. data/vendor/ggml/src/ggml-webgpu/ggml-webgpu.cpp +4790 -0
  1303. data/vendor/ggml/src/ggml-webgpu/pre_wgsl.hpp +808 -0
  1304. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/add_id.wgsl +64 -0
  1305. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/argmax.wgsl +72 -0
  1306. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/argsort.wgsl +102 -0
  1307. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/argsort_merge.wgsl +134 -0
  1308. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/binary.wgsl +142 -0
  1309. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/common_decls.tmpl +967 -0
  1310. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/concat.wgsl +92 -0
  1311. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/conv2d.wgsl +127 -0
  1312. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/conv2d_dw.wgsl +100 -0
  1313. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/cpy.wgsl +84 -0
  1314. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/cumsum.wgsl +66 -0
  1315. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/embed_wgsl.py +89 -0
  1316. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn.wgsl +408 -0
  1317. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_decls.tmpl +134 -0
  1318. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_staging.tmpl +136 -0
  1319. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_tile.wgsl +240 -0
  1320. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_blk.wgsl +101 -0
  1321. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_reduce.wgsl +84 -0
  1322. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_split.wgsl +503 -0
  1323. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/gated_delta_net.wgsl +149 -0
  1324. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/get_rows.wgsl +794 -0
  1325. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/glu.wgsl +178 -0
  1326. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/im2col.wgsl +77 -0
  1327. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/memset.wgsl +40 -0
  1328. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_decls.tmpl +1052 -0
  1329. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id.wgsl +195 -0
  1330. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id_gather.wgsl +52 -0
  1331. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id_vec.wgsl +154 -0
  1332. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_reg_tile.wgsl +159 -0
  1333. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_subgroup_matrix.wgsl +210 -0
  1334. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec.wgsl +167 -0
  1335. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec_acc.tmpl +1560 -0
  1336. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec_q_acc.tmpl +297 -0
  1337. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/pad.wgsl +86 -0
  1338. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/quant_inner_loops.tmpl +21 -0
  1339. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/quantize_q8.wgsl +178 -0
  1340. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/repeat.wgsl +70 -0
  1341. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/rms_norm_mul.wgsl +151 -0
  1342. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/rope.wgsl +228 -0
  1343. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/row_norm.wgsl +152 -0
  1344. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/scale.wgsl +65 -0
  1345. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/set.wgsl +109 -0
  1346. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/set_rows.wgsl +108 -0
  1347. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/set_rows_quant.wgsl +224 -0
  1348. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/soft_max.wgsl +210 -0
  1349. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/solve_tri.wgsl +120 -0
  1350. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/ssm_conv.wgsl +65 -0
  1351. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/ssm_scan.wgsl +247 -0
  1352. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/sum_rows.wgsl +55 -0
  1353. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/unary.wgsl +213 -0
  1354. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/upscale.wgsl +240 -0
  1355. data/vendor/ggml/src/ggml-zdnn/CMakeLists.txt +36 -0
  1356. data/vendor/ggml/src/ggml-zdnn/common.hpp +59 -0
  1357. data/vendor/ggml/src/ggml-zdnn/ggml-zdnn.cpp +638 -0
  1358. data/vendor/ggml/src/ggml-zdnn/mmf.cpp +80 -0
  1359. data/vendor/ggml/src/ggml-zdnn/mmf.hpp +12 -0
  1360. data/vendor/ggml/src/ggml-zdnn/utils.cpp +79 -0
  1361. data/vendor/ggml/src/ggml-zdnn/utils.hpp +19 -0
  1362. data/vendor/ggml/src/ggml-zendnn/CMakeLists.txt +91 -0
  1363. data/vendor/ggml/src/ggml-zendnn/ggml-zendnn.cpp +838 -0
  1364. data/vendor/ggml/src/ggml.c +8088 -0
  1365. data/vendor/ggml/src/ggml.cpp +26 -0
  1366. data/vendor/ggml/src/gguf.cpp +1706 -0
  1367. metadata +1459 -0
@@ -0,0 +1,2584 @@
1
+ #include "ggml-et-ops.h"
2
+
3
+ #include "ggml-et-cpu-compare.h"
4
+ #include "ggml-et-kernels.h"
5
+ #include "ggml-impl.h"
6
+
7
+ #include <stdio.h>
8
+
9
+ #include <cstdint>
10
+
11
+ // CPU comparison configuration - can be enabled for debugging
12
+ static ggml_et_cpu_compare_config rope_cpu_compare_config = {
13
+ /* .enabled = */ false,
14
+ /* .use_cpu_result = */ false, // Replace ET result with CPU result
15
+ /* .log_differences = */ true,
16
+ /* .tolerance = */ 1e-5f,
17
+ /* .max_log_elements = */ 4096
18
+ };
19
+
20
+ static ggml_et_cpu_compare_config rms_norm_cpu_compare_config = {
21
+ /* .enabled = */ false,
22
+ /* .use_cpu_result = */ false,
23
+ /* .log_differences = */ true,
24
+ /* .tolerance = */ 1e-5f,
25
+ /* .max_log_elements = */ 4096
26
+ };
27
+
28
+ static ggml_et_cpu_compare_config norm_cpu_compare_config = {
29
+ /* .enabled = */ false,
30
+ /* .use_cpu_result = */ false,
31
+ /* .log_differences = */ true,
32
+ /* .tolerance = */ 1e-5f,
33
+ /* .max_log_elements = */ 4096
34
+ };
35
+
36
+ static ggml_et_cpu_compare_config l2_norm_cpu_compare_config = {
37
+ /* .enabled = */ false,
38
+ /* .use_cpu_result = */ false,
39
+ /* .log_differences = */ true,
40
+ /* .tolerance = */ 1e-5f,
41
+ /* .max_log_elements = */ 4096
42
+ };
43
+
44
+ static ggml_et_cpu_compare_config group_norm_cpu_compare_config = {
45
+ /* .enabled = */ false,
46
+ /* .use_cpu_result = */ false,
47
+ /* .log_differences = */ true,
48
+ /* .tolerance = */ 1e-5f,
49
+ /* .max_log_elements = */ 4096
50
+ };
51
+
52
+ static ggml_et_cpu_compare_config im2col_cpu_compare_config = {
53
+ /* .enabled = */ false,
54
+ /* .use_cpu_result = */ false,
55
+ /* .log_differences = */ true,
56
+ /* .tolerance = */ 1e-5f,
57
+ /* .max_log_elements = */ 4096
58
+ };
59
+
60
+ static ggml_et_cpu_compare_config unary_cpu_compare_config = {
61
+ /* .enabled = */ false,
62
+ /* .use_cpu_result = */ false,
63
+ /* .log_differences = */ true,
64
+ /* .tolerance = */ 1e-4f,
65
+ /* .max_log_elements = */ 4096
66
+ };
67
+
68
+ static ggml_et_cpu_compare_config sum_rows_cpu_compare_config = {
69
+ /* .enabled = */ false,
70
+ /* .use_cpu_result = */ false,
71
+ /* .log_differences = */ true,
72
+ /* .tolerance = */ 1e-5f,
73
+ /* .max_log_elements = */ 4096
74
+ };
75
+
76
+ static ggml_et_cpu_compare_config clamp_cpu_compare_config = {
77
+ /* .enabled = */ false,
78
+ /* .use_cpu_result = */ false,
79
+ /* .log_differences = */ true,
80
+ /* .tolerance = */ 1e-6f,
81
+ /* .max_log_elements = */ 4096
82
+ };
83
+
84
+ static ggml_et_cpu_compare_config mean_cpu_compare_config = {
85
+ /* .enabled = */ false,
86
+ /* .use_cpu_result = */ false,
87
+ /* .log_differences = */ true,
88
+ /* .tolerance = */ 1e-5f,
89
+ /* .max_log_elements = */ 4096
90
+ };
91
+
92
+ static ggml_et_cpu_compare_config sqr_cpu_compare_config = {
93
+ /* .enabled = */ false,
94
+ /* .use_cpu_result = */ false,
95
+ /* .log_differences = */ true,
96
+ /* .tolerance = */ 1e-6f,
97
+ /* .max_log_elements = */ 4096
98
+ };
99
+
100
+ static ggml_et_cpu_compare_config elmap_cpu_compare_config = {
101
+ /* .enabled = */ false,
102
+ /* .use_cpu_result = */ false,
103
+ /* .log_differences = */ true,
104
+ /* .tolerance = */ 1e-6f,
105
+ /* .max_log_elements = */ 4096
106
+ };
107
+
108
+ static ggml_et_cpu_compare_config glu_cpu_compare_config = {
109
+ /* .enabled = */ false,
110
+ /* .use_cpu_result = */ false,
111
+ /* .log_differences = */ true,
112
+ /* .tolerance = */ 1e-5f,
113
+ /* .max_log_elements = */ 4096
114
+ };
115
+
116
+ static ggml_et_cpu_compare_config mul_mat_cpu_compare_config = {
117
+ /* .enabled = */ false,
118
+ /* .use_cpu_result = */ false,
119
+ /* .log_differences = */ true,
120
+ /* .tolerance = */ 0.01,
121
+ /* .max_log_elements = */ 4096
122
+ };
123
+
124
+ static ggml_et_cpu_compare_config mul_mat_id_cpu_compare_config = {
125
+ /* .enabled = */ false,
126
+ /* .use_cpu_result = */ false,
127
+ /* .log_differences = */ true,
128
+ /* .tolerance = */ 0.01,
129
+ /* .max_log_elements = */ 4096
130
+ };
131
+
132
+ static ggml_et_cpu_compare_config softmax_cpu_compare_config = {
133
+ /* .enabled = */ false,
134
+ /* .use_cpu_result = */ false,
135
+ /* .log_differences = */ true,
136
+ /* .tolerance = */ 1e-5f,
137
+ /* .max_log_elements = */ 1024
138
+ };
139
+
140
+ static ggml_et_cpu_compare_config get_rows_cpu_compare_config = {
141
+ /* .enabled = */ false,
142
+ /* .use_cpu_result = */ false,
143
+ /* .log_differences = */ true,
144
+ /* .tolerance = */ 1e-6f,
145
+ /* .max_log_elements = */ 2048
146
+ };
147
+
148
+ static ggml_et_cpu_compare_config pad_cpu_compare_config = {
149
+ /* .enabled = */ false,
150
+ /* .use_cpu_result = */ false,
151
+ /* .log_differences = */ true,
152
+ /* .tolerance = */ 1e-6f,
153
+ /* .max_log_elements = */ 4096
154
+ };
155
+
156
+ static ggml_et_cpu_compare_config cont_cpu_compare_config = {
157
+ /* .enabled = */ false,
158
+ /* .use_cpu_result = */ false,
159
+ /* .log_differences = */ true,
160
+ /* .tolerance = */ 1e-6f,
161
+ /* .max_log_elements = */ 4096
162
+ };
163
+
164
+ static ggml_et_cpu_compare_config concat_cpu_compare_config = {
165
+ /* .enabled = */ false,
166
+ /* .use_cpu_result = */ false,
167
+ /* .log_differences = */ true,
168
+ /* .tolerance = */ 1e-6f,
169
+ /* .max_log_elements = */ 4096
170
+ };
171
+
172
+ static ggml_et_cpu_compare_config cumsum_cpu_compare_config = {
173
+ /* .enabled = */ false,
174
+ /* .use_cpu_result = */ false,
175
+ /* .log_differences = */ true,
176
+ /* .tolerance = */ 1e-6f,
177
+ /* .max_log_elements = */ 4096
178
+ };
179
+
180
+ static ggml_et_cpu_compare_config repeat_cpu_compare_config = {
181
+ /* .enabled = */ false,
182
+ /* .use_cpu_result = */ false,
183
+ /* .log_differences = */ true,
184
+ /* .tolerance = */ 1e-6f,
185
+ /* .max_log_elements = */ 4096
186
+ };
187
+
188
+ static ggml_et_cpu_compare_config ssm_conv_cpu_compare_config = {
189
+ /* .enabled = */ false,
190
+ /* .use_cpu_result = */ false,
191
+ /* .log_differences = */ true,
192
+ /* .tolerance = */ 1e-6f,
193
+ /* .max_log_elements = */ 4096
194
+ };
195
+
196
+ static ggml_et_cpu_compare_config rwkv_wkv6_cpu_compare_config = {
197
+ /* .enabled = */ false,
198
+ /* .use_cpu_result = */ false,
199
+ /* .log_differences = */ true,
200
+ /* .tolerance = */ 1e-4f,
201
+ /* .max_log_elements = */ 4096
202
+ };
203
+
204
+ static ggml_et_cpu_compare_config rwkv_wkv7_cpu_compare_config = {
205
+ /* .enabled = */ false,
206
+ /* .use_cpu_result = */ false,
207
+ /* .log_differences = */ true,
208
+ /* .tolerance = */ 1e-4f,
209
+ /* .max_log_elements = */ 4096
210
+ };
211
+
212
+ static ggml_et_cpu_compare_config set_rows_cpu_compare_config = {
213
+ /* .enabled = */ false,
214
+ /* .use_cpu_result = */ false,
215
+ /* .log_differences = */ true,
216
+ /* .tolerance = */ 1e-6f,
217
+ /* .max_log_elements = */ 2048
218
+ };
219
+
220
+ bool ggml_et_op_rms_norm_mul(ggml_backend_et_device_context * dev_ctx,
221
+ const ggml_tensor * rms_norm_node,
222
+ const ggml_tensor * mul_node) {
223
+ ET_PERF_START();
224
+
225
+ if (!dev_ctx || !rms_norm_node || !mul_node) {
226
+ GGML_LOG_ERROR("ET: Invalid parameters for fused RMS_NORM_MUL operation\n");
227
+ return false;
228
+ }
229
+
230
+ if (!rms_norm_node->src[0]) {
231
+ GGML_LOG_ERROR("ET: Fused RMS_NORM_MUL missing required input\n");
232
+ return false;
233
+ }
234
+
235
+ // Extract weights: the MUL operand that isn't the rms_norm output
236
+ const ggml_tensor * weights = (mul_node->src[0] == rms_norm_node) ? mul_node->src[1] : mul_node->src[0];
237
+
238
+ if (!weights) {
239
+ GGML_LOG_ERROR("ET: Fused RMS_NORM_MUL missing weights tensor\n");
240
+ return false;
241
+ }
242
+
243
+ float eps;
244
+ memcpy(&eps, rms_norm_node->op_params, sizeof(float));
245
+
246
+ ggml_et_rms_norm_mul_params params;
247
+ params.src0 = *rms_norm_node->src[0]; // input to normalize
248
+ params.src1 = *weights; // normalization weights
249
+ params.dst = *mul_node; // final output
250
+ params.eps = eps;
251
+
252
+ bool kernel_result = ggml_et_launch_kernel(dev_ctx, "rms_norm_mul_f32", &params, sizeof(params), 0xFFFFFFFF);
253
+
254
+ ET_PERF_END_EXT("RMS_NORM_MUL", "rms_norm_mul_f32", mul_node, "eps=%.6f", (double) eps);
255
+ return kernel_result;
256
+ }
257
+
258
+ bool ggml_et_op_scale(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
259
+ ET_PERF_START();
260
+
261
+ if (!dev_ctx || !node) {
262
+ GGML_LOG_ERROR("ET: Invalid parameters for SCALE operation\n");
263
+ return false;
264
+ }
265
+
266
+ if (!node->src[0]) {
267
+ GGML_LOG_ERROR("ET: SCALE operation missing required input\n");
268
+ return false;
269
+ }
270
+
271
+ if (node->type != GGML_TYPE_F32 || node->src[0]->type != GGML_TYPE_F32) {
272
+ GGML_LOG_ERROR("ET: SCALE operation with unsupported types: dst=%s src0=%s\n", ggml_type_name(node->type),
273
+ ggml_type_name(node->src[0]->type));
274
+ return false;
275
+ }
276
+
277
+ float scale, bias;
278
+ memcpy(&scale, (const float *) node->op_params + 0, sizeof(float));
279
+ memcpy(&bias, (const float *) node->op_params + 1, sizeof(float));
280
+
281
+ ggml_et_scale_params params;
282
+ params.src0 = *node->src[0];
283
+ params.dst = *node;
284
+ params.scale = scale;
285
+ params.bias = bias;
286
+
287
+ bool kernel_result = ggml_et_launch_kernel(dev_ctx, "scale_f32", &params, sizeof(params), 0xFFFFFFFF);
288
+
289
+ ET_PERF_END_EXT("SCALE", "scale_f32", node, "scale=%.6f|bias=%.6f", (double) scale, (double) bias);
290
+ return kernel_result;
291
+ }
292
+
293
+ bool ggml_et_op_sqr(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
294
+ ET_PERF_START();
295
+
296
+ if (!dev_ctx || !node) {
297
+ GGML_LOG_ERROR("ET: Invalid parameters for SQR operation\n");
298
+ return false;
299
+ }
300
+
301
+ if (!node->src[0]) {
302
+ GGML_LOG_ERROR("ET: SQR operation missing required input\n");
303
+ return false;
304
+ }
305
+
306
+ if (node->type != GGML_TYPE_F32 || node->src[0]->type != GGML_TYPE_F32) {
307
+ GGML_LOG_ERROR("ET: SQR operation with unsupported types: dst=%s src0=%s\n", ggml_type_name(node->type),
308
+ ggml_type_name(node->src[0]->type));
309
+ return false;
310
+ }
311
+
312
+ ggml_et_sqr_params params;
313
+ params.src0 = *node->src[0]; // F32 input tensor
314
+ params.dst = *node; // F32 output tensor
315
+
316
+ // Phase 1: Initialize CPU comparison context and copy source buffers (before ET kernel)
317
+ ggml_et_cpu_compare_ctx cpu_cmp_ctx;
318
+ bool cpu_comparison_active = false;
319
+ if (sqr_cpu_compare_config.enabled) {
320
+ if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_SQR)) {
321
+ cpu_comparison_active = true;
322
+ } else {
323
+ GGML_LOG_WARN("ET: Failed to initialize CPU comparison for SQR operation\n");
324
+ }
325
+ }
326
+
327
+ bool kernel_result = ggml_et_launch_kernel(dev_ctx, "sqr_f32", &params, sizeof(params), 0xFFFFFFFF);
328
+
329
+ // Phase 2: Execute CPU computation and compare with ET result (after ET kernel)
330
+ if (cpu_comparison_active) {
331
+ if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &sqr_cpu_compare_config)) {
332
+ GGML_LOG_WARN("ET: CPU comparison failed for SQR operation\n");
333
+ }
334
+ ggml_et_cpu_compare_free(&cpu_cmp_ctx);
335
+ }
336
+
337
+ ET_PERF_END("SQR", "sqr_f32", node);
338
+ return kernel_result;
339
+ }
340
+
341
+ bool ggml_et_op_sum_rows(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
342
+ ET_PERF_START();
343
+
344
+ if (!dev_ctx || !node) {
345
+ GGML_LOG_ERROR("ET: Invalid parameters for SUM_ROWS operation\n");
346
+ return false;
347
+ }
348
+
349
+ if (!node->src[0]) {
350
+ GGML_LOG_ERROR("ET: SUM_ROWS operation missing required input\n");
351
+ return false;
352
+ }
353
+
354
+ if (node->type != GGML_TYPE_F32 || node->src[0]->type != GGML_TYPE_F32) {
355
+ GGML_LOG_ERROR("ET: SUM_ROWS operation with unsupported types: dst=%s src0=%s\n", ggml_type_name(node->type),
356
+ ggml_type_name(node->src[0]->type));
357
+ return false;
358
+ }
359
+
360
+ ggml_et_sum_rows_params params;
361
+ params.src0 = *node->src[0];
362
+ params.dst = *node;
363
+
364
+ // Phase 1: Initialize CPU comparison context
365
+ ggml_et_cpu_compare_ctx cpu_cmp_ctx;
366
+ bool cpu_comparison_active = false;
367
+ if (sum_rows_cpu_compare_config.enabled) {
368
+ if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_SUM_ROWS)) {
369
+ cpu_comparison_active = true;
370
+ } else {
371
+ GGML_LOG_WARN("ET: Failed to initialize CPU comparison for SUM_ROWS operation\n");
372
+ }
373
+ }
374
+
375
+ bool kernel_result = ggml_et_launch_kernel(dev_ctx, "sum_rows_f32", &params, sizeof(params), 0xFFFFFFFF);
376
+
377
+ // Phase 2: Execute CPU computation and compare
378
+ if (cpu_comparison_active) {
379
+ if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &sum_rows_cpu_compare_config)) {
380
+ GGML_LOG_WARN("ET: CPU comparison failed for SUM_ROWS operation\n");
381
+ }
382
+ ggml_et_cpu_compare_free(&cpu_cmp_ctx);
383
+ }
384
+
385
+ ET_PERF_END("SUM_ROWS", "sum_rows_f32", node);
386
+ return kernel_result;
387
+ }
388
+
389
+ bool ggml_et_op_mean(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
390
+ ET_PERF_START();
391
+
392
+ if (!dev_ctx || !node) {
393
+ GGML_LOG_ERROR("ET: Invalid parameters for MEAN operation\n");
394
+ return false;
395
+ }
396
+
397
+ if (!node->src[0]) {
398
+ GGML_LOG_ERROR("ET: MEAN operation missing required input\n");
399
+ return false;
400
+ }
401
+
402
+ if (node->type != GGML_TYPE_F32 || node->src[0]->type != GGML_TYPE_F32) {
403
+ GGML_LOG_ERROR("ET: MEAN operation with unsupported types: dst=%s src0=%s\n", ggml_type_name(node->type),
404
+ ggml_type_name(node->src[0]->type));
405
+ return false;
406
+ }
407
+
408
+ ggml_et_mean_params params;
409
+ params.src0 = *node->src[0];
410
+ params.dst = *node;
411
+
412
+ ggml_et_cpu_compare_ctx cpu_cmp_ctx;
413
+ bool cpu_comparison_active = false;
414
+ if (mean_cpu_compare_config.enabled) {
415
+ if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_MEAN)) {
416
+ cpu_comparison_active = true;
417
+ } else {
418
+ GGML_LOG_WARN("ET: Failed to initialize CPU comparison for MEAN operation\n");
419
+ }
420
+ }
421
+
422
+ bool kernel_result = ggml_et_launch_kernel(dev_ctx, "mean_f32", &params, sizeof(params), 0xFFFFFFFF);
423
+
424
+ if (cpu_comparison_active) {
425
+ if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &mean_cpu_compare_config)) {
426
+ GGML_LOG_WARN("ET: CPU comparison failed for MEAN operation\n");
427
+ }
428
+ ggml_et_cpu_compare_free(&cpu_cmp_ctx);
429
+ }
430
+
431
+ ET_PERF_END("MEAN", "mean_f32", node);
432
+ return kernel_result;
433
+ }
434
+
435
+ bool ggml_et_op_clamp(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
436
+ ET_PERF_START();
437
+
438
+ if (!dev_ctx || !node) {
439
+ GGML_LOG_ERROR("ET: Invalid parameters for CLAMP operation\n");
440
+ return false;
441
+ }
442
+
443
+ if (!node->src[0]) {
444
+ GGML_LOG_ERROR("ET: CLAMP operation missing required input\n");
445
+ return false;
446
+ }
447
+
448
+ if (node->type != GGML_TYPE_F32 || node->src[0]->type != GGML_TYPE_F32) {
449
+ GGML_LOG_ERROR("ET: CLAMP operation with unsupported types: dst=%s src0=%s\n", ggml_type_name(node->type),
450
+ ggml_type_name(node->src[0]->type));
451
+ return false;
452
+ }
453
+
454
+ ggml_et_clamp_params params;
455
+ params.src0 = *node->src[0];
456
+ params.dst = *node;
457
+ // op_params layout per ggml.c::ggml_clamp: { min, max } as floats
458
+ memcpy(&params.min_val, (const float *) node->op_params + 0, sizeof(float));
459
+ memcpy(&params.max_val, (const float *) node->op_params + 1, sizeof(float));
460
+
461
+ ggml_et_cpu_compare_ctx cpu_cmp_ctx;
462
+ bool cpu_comparison_active = false;
463
+ if (clamp_cpu_compare_config.enabled) {
464
+ if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_CLAMP)) {
465
+ cpu_comparison_active = true;
466
+ } else {
467
+ GGML_LOG_WARN("ET: Failed to initialize CPU comparison for CLAMP operation\n");
468
+ }
469
+ }
470
+
471
+ bool kernel_result = ggml_et_launch_kernel(dev_ctx, "clamp_f32", &params, sizeof(params), 0xFFFFFFFF);
472
+
473
+ if (cpu_comparison_active) {
474
+ if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &clamp_cpu_compare_config)) {
475
+ GGML_LOG_WARN("ET: CPU comparison failed for CLAMP operation\n");
476
+ }
477
+ ggml_et_cpu_compare_free(&cpu_cmp_ctx);
478
+ }
479
+
480
+ ET_PERF_END("CLAMP", "clamp_f32", node);
481
+ return kernel_result;
482
+ }
483
+
484
+ bool ggml_et_op_unary(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
485
+ ET_PERF_START();
486
+
487
+ if (!dev_ctx || !node) {
488
+ GGML_LOG_ERROR("ET: Invalid parameters for UNARY operation\n");
489
+ return false;
490
+ }
491
+
492
+ if (!node->src[0]) {
493
+ GGML_LOG_ERROR("ET: UNARY operation missing required input\n");
494
+ return false;
495
+ }
496
+
497
+ if (node->type != GGML_TYPE_F32 || node->src[0]->type != GGML_TYPE_F32) {
498
+ GGML_LOG_ERROR("ET: UNARY operation with unsupported types: dst=%s src0=%s\n", ggml_type_name(node->type),
499
+ ggml_type_name(node->src[0]->type));
500
+ return false;
501
+ }
502
+
503
+ const ggml_unary_op uop = ggml_get_unary_op(node);
504
+ const char * op_name = ggml_unary_op_name(uop);
505
+
506
+ ggml_et_unary_params params;
507
+ params.src0 = *node->src[0]; // F32 input tensor
508
+ params.dst = *node; // F32 output tensor
509
+ params.unary_op = (int32_t) uop;
510
+
511
+ // Phase 1: Initialize CPU comparison context and copy source buffers (before ET kernel)
512
+ ggml_et_cpu_compare_ctx cpu_cmp_ctx;
513
+ bool cpu_comparison_active = false;
514
+ if (unary_cpu_compare_config.enabled) {
515
+ if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_UNARY)) {
516
+ cpu_comparison_active = true;
517
+ } else {
518
+ GGML_LOG_WARN("ET: Failed to initialize CPU comparison for UNARY/%s operation\n", op_name);
519
+ }
520
+ }
521
+
522
+ bool kernel_result = ggml_et_launch_kernel(dev_ctx, "unary_f32", &params, sizeof(params), 0xFFFFFFFF);
523
+
524
+ // Phase 2: Execute CPU computation and compare with ET result (after ET kernel)
525
+ if (cpu_comparison_active) {
526
+ if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &unary_cpu_compare_config)) {
527
+ GGML_LOG_WARN("ET: CPU comparison failed for UNARY/%s operation\n", op_name);
528
+ }
529
+ ggml_et_cpu_compare_free(&cpu_cmp_ctx);
530
+ }
531
+
532
+ ET_PERF_END_EXT("UNARY", "unary_f32", node, "op=%s", op_name);
533
+ return kernel_result;
534
+ }
535
+
536
+ bool ggml_et_op_mul(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
537
+ // Delegate to generic element map operation
538
+ return ggml_et_op_elmap(dev_ctx, node);
539
+ }
540
+
541
+ bool ggml_et_op_add(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
542
+ // Delegate to generic element map operation
543
+ return ggml_et_op_elmap(dev_ctx, node);
544
+ }
545
+
546
+ bool ggml_et_op_sub(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
547
+ // Delegate to generic element map operation
548
+ return ggml_et_op_elmap(dev_ctx, node);
549
+ }
550
+
551
+ bool ggml_et_op_elmap(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
552
+ ET_PERF_START();
553
+
554
+ if (!dev_ctx || !node) {
555
+ GGML_LOG_ERROR("ET: Invalid parameters for element map operation\n");
556
+ return false;
557
+ }
558
+
559
+ if (!node->src[0] || !node->src[1]) {
560
+ GGML_LOG_ERROR("ET: Element map operation missing required inputs\n");
561
+ return false;
562
+ }
563
+
564
+ if (node->type != GGML_TYPE_F32 || node->src[0]->type != GGML_TYPE_F32 || node->src[1]->type != GGML_TYPE_F32) {
565
+ GGML_LOG_ERROR("ET: Element map operation with unsupported types: dst=%s src0=%s src1=%s\n",
566
+ ggml_type_name(node->type), ggml_type_name(node->src[0]->type),
567
+ ggml_type_name(node->src[1]->type));
568
+ return false;
569
+ }
570
+
571
+ const char * op_name = ggml_op_name(node->op);
572
+
573
+ ggml_et_elmap_params params;
574
+ params.src0 = *node->src[0];
575
+ params.src1 = *node->src[1];
576
+ params.dst = *node; // F32 output tensor (op type stored in dst.op)
577
+
578
+ // Phase 1: Initialize CPU comparison context and copy source buffers (before ET kernel)
579
+ ggml_et_cpu_compare_ctx cpu_cmp_ctx;
580
+ bool cpu_comparison_active = false;
581
+ if (elmap_cpu_compare_config.enabled) {
582
+ if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, node->op)) {
583
+ cpu_comparison_active = true;
584
+ } else {
585
+ GGML_LOG_WARN("ET: Failed to initialize CPU comparison for %s operation\n", op_name);
586
+ }
587
+ }
588
+
589
+ // fprintf(stderr, "ET: el_map s0 [%ld, %ld, %ld, %ld] s1 [%ld, %ld, %ld, %ld]\n",
590
+ // node->src[0]->ne[0], node->src[0]->ne[1], node->src[0]->ne[2], node->src[0]->ne[3],
591
+ // node->src[1]->ne[0], node->src[1]->ne[1], node->src[1]->ne[2], node->src[1]->ne[3]);
592
+
593
+ bool kernel_result = ggml_et_launch_kernel(dev_ctx, "el_map_f32", &params, sizeof(params), 0xFFFFFFFF);
594
+
595
+ // Phase 2: Execute CPU computation and compare with ET result (after ET kernel)
596
+ if (cpu_comparison_active) {
597
+ if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &elmap_cpu_compare_config)) {
598
+ GGML_LOG_WARN("ET: CPU comparison failed for %s operation\n", op_name);
599
+ }
600
+ ggml_et_cpu_compare_free(&cpu_cmp_ctx);
601
+ }
602
+
603
+ ET_PERF_END(op_name, "el_map_f32", node);
604
+ return kernel_result;
605
+ }
606
+
607
+ bool ggml_et_op_glu(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
608
+ ET_PERF_START();
609
+
610
+ // Validate inputs
611
+ if (!dev_ctx || !node) {
612
+ GGML_LOG_ERROR("ET: Invalid parameters for GLU operation\n");
613
+ return false;
614
+ }
615
+
616
+ if (!node->src[0]) {
617
+ GGML_LOG_ERROR("ET: GLU operation missing required input\n");
618
+ return false;
619
+ }
620
+
621
+ const bool is_split_mode = node->src[1] != nullptr;
622
+
623
+ // Only support F32 (as validated by supports_op)
624
+ if (node->type != GGML_TYPE_F32 || node->src[0]->type != GGML_TYPE_F32 ||
625
+ (is_split_mode && node->src[1]->type != GGML_TYPE_F32)) {
626
+ return false;
627
+ }
628
+
629
+ // Extract GLU operation parameters from op_params
630
+ int32_t glu_op_type = ggml_get_op_params_i32(node, 0); // GLU variant (REGLU, GEGLU, SWIGLU, etc.)
631
+ int32_t swapped = ggml_get_op_params_i32(node, 1); // Whether gate/value are swapped
632
+
633
+ // Supported variants
634
+ switch (glu_op_type) {
635
+ case GGML_GLU_OP_REGLU:
636
+ case GGML_GLU_OP_GEGLU:
637
+ case GGML_GLU_OP_SWIGLU:
638
+ case GGML_GLU_OP_SWIGLU_OAI:
639
+ case GGML_GLU_OP_SWIGLU_CLAMP:
640
+ case GGML_GLU_OP_GEGLU_ERF:
641
+ case GGML_GLU_OP_GEGLU_QUICK:
642
+ break;
643
+ default:
644
+ GGML_LOG_ERROR("ET: GLU operation with unsupported variant: %s\n",
645
+ ggml_glu_op_name((ggml_glu_op) glu_op_type));
646
+ return false;
647
+ }
648
+
649
+ // Get GLU operation name for logging
650
+ const char * glu_op_name = ggml_glu_op_name((ggml_glu_op) glu_op_type);
651
+
652
+ // Pack parameters. Single-tensor mode is encoded by zeroing src1.
653
+ ggml_et_glu_params params = {};
654
+ params.src0 = *node->src[0];
655
+ if (is_split_mode) {
656
+ params.src1 = *node->src[1];
657
+ }
658
+ params.dst = *node;
659
+ params.glu_op_type = glu_op_type;
660
+ params.swapped = swapped;
661
+ params.alpha = 0.0f;
662
+ params.limit = 0.0f;
663
+ if (glu_op_type == GGML_GLU_OP_SWIGLU_OAI) {
664
+ params.alpha = ggml_get_op_params_f32(node, 2);
665
+ }
666
+ if (glu_op_type == GGML_GLU_OP_SWIGLU_OAI || glu_op_type == GGML_GLU_OP_SWIGLU_CLAMP) {
667
+ params.limit = ggml_get_op_params_f32(node, 3);
668
+ }
669
+ // Phase 1: Initialize CPU comparison context and copy source buffers (before ET kernel)
670
+ ggml_et_cpu_compare_ctx cpu_cmp_ctx;
671
+ bool cpu_comparison_active = false;
672
+ if (glu_cpu_compare_config.enabled) {
673
+ if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_GLU)) {
674
+ cpu_comparison_active = true;
675
+ } else {
676
+ GGML_LOG_WARN("ET: Failed to initialize CPU comparison for %s operation\n", glu_op_name);
677
+ }
678
+ }
679
+
680
+ // Launch ET kernel
681
+ bool kernel_result = ggml_et_launch_kernel(dev_ctx, "glu_f32", &params, sizeof(params), 0xFFFFFFFF);
682
+
683
+ // Phase 2: Execute CPU computation and compare with ET result (after ET kernel)
684
+ if (cpu_comparison_active) {
685
+ if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &glu_cpu_compare_config)) {
686
+ GGML_LOG_WARN("ET: CPU comparison failed for %s operation\n", glu_op_name);
687
+ }
688
+ ggml_et_cpu_compare_free(&cpu_cmp_ctx);
689
+ }
690
+
691
+ ET_PERF_END("GLU", "glu_f32", node);
692
+ return kernel_result;
693
+ }
694
+
695
+ bool ggml_et_op_mul_mat(ggml_backend_et_device_context * dev_ctx,
696
+ const ggml_tensor * node,
697
+ const ggml_tensor * add_node) {
698
+ ET_PERF_START();
699
+
700
+ if (!dev_ctx || !node) {
701
+ GGML_LOG_ERROR("ET: Invalid parameters for MUL_MAT operation\n");
702
+ return false;
703
+ }
704
+
705
+ if (!node->src[0] || !node->src[1]) {
706
+ GGML_LOG_ERROR("ET: MUL_MAT operation missing required inputs\n");
707
+ return false;
708
+ }
709
+
710
+ // Fused MM+ADD: when add_node is non-NULL the caller has already validated
711
+ // (Q8_0 weights, F32 acts, exact-shape ADD with stride parity to dst) via
712
+ // ggml_et_can_fuse({MUL_MAT, ADD}). The kernel writes dst = mm + bias and
713
+ // the ADD's output replaces MM's as the actual dst.
714
+ const ggml_tensor * fused_dst = add_node ? add_node : node;
715
+ const ggml_tensor * bias_tensor = nullptr;
716
+ if (add_node) {
717
+ bias_tensor = (add_node->src[0] == node) ? add_node->src[1] : add_node->src[0];
718
+ }
719
+
720
+ const char * kernel_name;
721
+ const char * src0_type_name;
722
+
723
+ if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_Q4_0 && node->src[1]->type == GGML_TYPE_F32 &&
724
+ node->src[1]->ne[1] >= 53 && // N >= 53
725
+ node->src[0]->ne[1] % 16 == 0 && // M % TILE_M
726
+ node->src[0]->ne[0] % 32 == 0) { // K % BLOCK_K (Q4_0 block)
727
+
728
+ // Matrix engine for N >= 53; partial N (via n_cur-1) and errata padding are handled in-kernel.
729
+ kernel_name = "mul_mat_Q4_0_matrix_engine";
730
+ src0_type_name = "Q4_0";
731
+
732
+ } else if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_Q4_0 &&
733
+ node->src[1]->type == GGML_TYPE_F32) {
734
+ kernel_name = "mul_mat_Q4_0"; // N < 53, or M % 16 != 0 or K % 32 != 0
735
+ src0_type_name = "Q4_0";
736
+
737
+ } else if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_Q8_0 &&
738
+ node->src[1]->type == GGML_TYPE_F32) {
739
+ kernel_name = "mul_mat_Q8_0";
740
+ src0_type_name = "Q8_0";
741
+
742
+ } else if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_F16 &&
743
+ node->src[1]->type == GGML_TYPE_F16 && node->ne[0] % 16 == 0 && node->src[0]->ne[0] % 16 == 0 &&
744
+ node->src[0]->ne[1] % 16 == 0 && node->src[1]->ne[0] != 1) {
745
+ kernel_name = "mul_mat_f16_matrix_engine";
746
+ src0_type_name = "F16";
747
+
748
+ } else if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_F16 &&
749
+ (node->src[1]->type == GGML_TYPE_F16 || node->src[1]->type == GGML_TYPE_F32)) {
750
+ kernel_name = "mul_mat_f16";
751
+ src0_type_name = "F16";
752
+
753
+ } else if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_F32 &&
754
+ node->src[1]->type == GGML_TYPE_F32 && node->ne[0] % 16 == 0 && node->src[0]->ne[0] % 16 == 0 &&
755
+ node->src[0]->ne[1] % 16 == 0 && node->src[1]->ne[0] != 1) { // GEMV is faster with the generic path
756
+
757
+ kernel_name = "mul_mat_f32_matrix_engine";
758
+ src0_type_name = "F32";
759
+ } else if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_F32 &&
760
+ (node->src[1]->type == GGML_TYPE_F16 || node->src[1]->type == GGML_TYPE_F32)) {
761
+ kernel_name = "mul_mat_f32";
762
+ src0_type_name = "F32";
763
+ } else {
764
+ GGML_LOG_ERROR("ET: MUL_MAT operation with unsupported types: dst=%s src0=%s src1=%s\n",
765
+ ggml_type_name(node->type), ggml_type_name(node->src[0]->type),
766
+ ggml_type_name(node->src[1]->type));
767
+ return false;
768
+ }
769
+
770
+ ggml_et_binary_params params;
771
+ params.src0 = *node->src[0]; // weight matrix
772
+ params.src1 = *node->src[1]; // activation matrix
773
+ params.dst = *fused_dst; // output (= add_node when fused, else node)
774
+
775
+ ggml_et_cpu_compare_ctx cpu_cmp_ctx;
776
+ bool cpu_comparison_active = false;
777
+ if (mul_mat_cpu_compare_config.enabled) {
778
+ if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, fused_dst, GGML_OP_MUL_MAT)) {
779
+ cpu_comparison_active = true;
780
+ } else {
781
+ GGML_LOG_WARN("ET: Failed to initialize CPU comparison for MUL_MAT operation\n");
782
+ }
783
+ }
784
+
785
+ bool kernel_result;
786
+ if (node->src[0]->type == GGML_TYPE_Q8_0) {
787
+ // Q8_0 kernel always takes the extended struct. bias.data is non-NULL
788
+ // only on the fused path; otherwise the kernel skips the add entirely.
789
+ ggml_et_mm_q8_params q8_params = {};
790
+ q8_params.src0 = params.src0;
791
+ q8_params.src1 = params.src1;
792
+ q8_params.dst = params.dst;
793
+ if (bias_tensor) {
794
+ q8_params.bias = *bias_tensor;
795
+ }
796
+ kernel_result = ggml_et_launch_kernel(dev_ctx, kernel_name, &q8_params, sizeof(q8_params), 0xFFFFFFFF);
797
+ } else {
798
+ // Non-Q8 MM kernels don't yet support fused-add; the graph fuse check
799
+ // already rejects non-Q8 pairs, so add_node is always nullptr here.
800
+ kernel_result = ggml_et_launch_kernel(dev_ctx, kernel_name, &params, sizeof(params), 0xFFFFFFFF);
801
+ }
802
+
803
+ // printf("Tensor error:");
804
+ // if (params.src0.data != NULL)
805
+ // {
806
+ // printf("Ptr OK\n");
807
+ // printf("node->data ptr = %p\n", node->data);
808
+ // // if (once < 100){
809
+ // // // uint64_t * host_data = (uint64_t *) node->data;
810
+ // // // printf("Tensor error: %lu\n", host_data[0]);
811
+
812
+ // // // printf("Tensor error:");
813
+ // // once++;
814
+ // // }
815
+ // }
816
+
817
+ // Phase 2: Execute CPU computation and compare with ET result (after ET kernel)
818
+ if (cpu_comparison_active) {
819
+ if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, fused_dst, &mul_mat_cpu_compare_config)) {
820
+ GGML_LOG_WARN("ET: CPU comparison failed for MUL_MAT operation\n");
821
+ }
822
+ ggml_et_cpu_compare_free(&cpu_cmp_ctx);
823
+ }
824
+
825
+ {
826
+ // Calculate actual FLOPs including batch/sequence dimensions
827
+ // dst shape: [M, N, ne2, ne3] where M=ne[1], N=ne[0]
828
+ int64_t m = node->ne[1];
829
+ int64_t n = node->ne[0];
830
+ int64_t k = node->src[0]->ne[0];
831
+ int64_t ne2 = node->ne[2];
832
+ int64_t ne3 = node->ne[3];
833
+
834
+ // Total FLOPs = (batch_size) * M * N * (2*K - 1)
835
+ // Each MxN matrix-matrix multiply does M*N*(2*K-1) FLOPs
836
+ // Broadcasting is handled by repeating computation, so count actual operations
837
+ int64_t batch_size = ne2 * ne3;
838
+ int64_t total_flops = batch_size * m * n * (2 * k - 1);
839
+
840
+ char kernel_variant[64];
841
+ snprintf(kernel_variant, sizeof(kernel_variant), "%s_%sx%s", kernel_name, src0_type_name,
842
+ ggml_type_name(node->src[1]->type));
843
+ ET_PERF_END_EXT("MUL_MAT", kernel_variant, node, "flops=%" PRId64, total_flops);
844
+ }
845
+ return kernel_result;
846
+ }
847
+
848
+ bool ggml_et_op_mul_mat_id(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
849
+ ET_PERF_START();
850
+ if (!dev_ctx || !node) {
851
+ GGML_LOG_ERROR("ET: Invalid parameters for MUL_MAT_ID operation\n");
852
+ return false;
853
+ }
854
+
855
+ if (!node->src[0] || !node->src[1] || !node->src[2]) {
856
+ GGML_LOG_ERROR("ET: MUL_MAT_ID operation missing required inputs\n");
857
+ return false;
858
+ }
859
+
860
+ const char * kernel_name;
861
+ const char * src0_type_name;
862
+
863
+ // Support Q8_0/Q4_0/F16/F32 x F32 -> F32 matrix multiplication with expert selection
864
+ if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_Q8_0 && node->src[1]->type == GGML_TYPE_F32 &&
865
+ node->src[2]->type == GGML_TYPE_I32) {
866
+ kernel_name = "mul_mat_id_Q8_0";
867
+ src0_type_name = "Q8_0";
868
+
869
+ } else if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_Q4_0 &&
870
+ node->src[1]->type == GGML_TYPE_F32 && node->src[2]->type == GGML_TYPE_I32) {
871
+ kernel_name = "mul_mat_id_Q4_0";
872
+ src0_type_name = "Q4_0";
873
+
874
+ } else if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_F16 &&
875
+ node->src[1]->type == GGML_TYPE_F32 && node->src[2]->type == GGML_TYPE_I32) {
876
+ kernel_name = "mul_mat_id_f32";
877
+ src0_type_name = "F16";
878
+
879
+ } else if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_F32 &&
880
+ node->src[1]->type == GGML_TYPE_F32 && node->src[2]->type == GGML_TYPE_I32) {
881
+ kernel_name = "mul_mat_id_f32";
882
+ src0_type_name = "F32";
883
+
884
+ } else {
885
+ GGML_LOG_ERROR("ET: MUL_MAT_ID operation with unsupported types: dst=%s src0=%s src1=%s src2=%s\n",
886
+ ggml_type_name(node->type), ggml_type_name(node->src[0]->type),
887
+ ggml_type_name(node->src[1]->type), ggml_type_name(node->src[2]->type));
888
+ return false;
889
+ }
890
+
891
+ // Pack parameters - copy full tensor structures
892
+ ggml_et_mul_mat_id_params params;
893
+ params.src0 = *node->src[0]; // Expert weight matrices (Q8_0/F16/F32)
894
+ params.src1 = *node->src[1]; // Activation matrix (F32)
895
+ params.src2 = *node->src[2]; // Expert indices (I32)
896
+ params.dst = *node; // Output matrix (F32)
897
+
898
+ // Phase 1: Initialize CPU comparison context and copy source buffers (before ET kernel)
899
+ ggml_et_cpu_compare_ctx cpu_cmp_ctx;
900
+ bool cpu_comparison_active = false;
901
+ if (mul_mat_id_cpu_compare_config.enabled) {
902
+ if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_MUL_MAT_ID)) {
903
+ cpu_comparison_active = true;
904
+ } else {
905
+ GGML_LOG_WARN("ET: Failed to initialize CPU comparison for MUL_MAT_ID operation\n");
906
+ }
907
+ }
908
+
909
+ // Launch ET kernel
910
+ bool kernel_result = ggml_et_launch_kernel(dev_ctx, kernel_name, &params, sizeof(params), 0xFFFFFFFF);
911
+
912
+ // Phase 2: Execute CPU computation and compare with ET result (after ET kernel)
913
+ if (cpu_comparison_active) {
914
+ if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &mul_mat_id_cpu_compare_config)) {
915
+ GGML_LOG_WARN("ET: CPU comparison failed for MUL_MAT_ID operation\n");
916
+ }
917
+ ggml_et_cpu_compare_free(&cpu_cmp_ctx);
918
+ }
919
+
920
+ // Calculate FLOPs (approximate - similar to MUL_MAT but with expert routing overhead)
921
+ // Each expert computation is similar to a MUL_MAT, but we only compute for selected experts
922
+ int64_t K = node->src[0]->ne[0];
923
+ int64_t M = node->src[0]->ne[1];
924
+ int64_t n_expert_used = node->src[2]->ne[0];
925
+ int64_t batch = node->src[2]->ne[1];
926
+
927
+ int64_t total_flops = batch * n_expert_used * M * (2 * K - 1);
928
+
929
+ char kernel_variant[64];
930
+ snprintf(kernel_variant, sizeof(kernel_variant), "%s_%sx%s", kernel_name, src0_type_name,
931
+ ggml_type_name(node->src[1]->type));
932
+ ET_PERF_END_EXT("MUL_MAT_ID", kernel_variant, node, "flops=%" PRId64 "|n_expert=%lld|n_expert_used=%lld",
933
+ total_flops, (long long) node->src[0]->ne[2], (long long) n_expert_used);
934
+
935
+ return kernel_result;
936
+ }
937
+
938
+ bool ggml_et_op_rope(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
939
+ ET_PERF_START();
940
+
941
+ if (!dev_ctx || !node) {
942
+ GGML_LOG_ERROR("ET: Invalid parameters for ROPE operation\n");
943
+ return false;
944
+ }
945
+
946
+ if (!node->src[0] || !node->src[1]) {
947
+ GGML_LOG_ERROR("ET: ROPE operation missing required inputs\n");
948
+ return false;
949
+ }
950
+
951
+ const char * kernel_name;
952
+
953
+ if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_F32 && node->src[1]->type == GGML_TYPE_I32) {
954
+ kernel_name = "rope_f32";
955
+ } else {
956
+ return false;
957
+ }
958
+
959
+ // Pack parameters - copy full tensor structures and op_params
960
+ ggml_et_rope_params params;
961
+ params.src0 = *node->src[0]; // F32 input tensor
962
+ params.src1 = *node->src[1]; // I32 position tensor
963
+ if (node->src[2]) {
964
+ params.src2 = *node->src[2]; // F32 frequency factors (optional)
965
+ } else {
966
+ memset(&params.src2, 0, sizeof(params.src2)); // Zero if not provided
967
+ }
968
+ params.dst = *node; // F32 output tensor
969
+
970
+ params.rope_params.n_past = ((const int32_t *) node->op_params)[0];
971
+ params.rope_params.n_dims = ((const int32_t *) node->op_params)[1];
972
+ params.rope_params.mode = ((const int32_t *) node->op_params)[2];
973
+ params.rope_params.n_ctx = ((const int32_t *) node->op_params)[3];
974
+ params.rope_params.n_ctx_orig = ((const int32_t *) node->op_params)[4];
975
+ memcpy(&params.rope_params.freq_base, (const int32_t *) node->op_params + 5, sizeof(float));
976
+ memcpy(&params.rope_params.freq_scale, (const int32_t *) node->op_params + 6, sizeof(float));
977
+ memcpy(&params.rope_params.ext_factor, (const int32_t *) node->op_params + 7, sizeof(float));
978
+ memcpy(&params.rope_params.attn_factor, (const int32_t *) node->op_params + 8, sizeof(float));
979
+ memcpy(&params.rope_params.beta_fast, (const int32_t *) node->op_params + 9, sizeof(float));
980
+ memcpy(&params.rope_params.beta_slow, (const int32_t *) node->op_params + 10, sizeof(float));
981
+ if (params.rope_params.mode & GGML_ROPE_TYPE_MROPE) {
982
+ memcpy(params.rope_params.sections, (const int32_t *) node->op_params + 11, sizeof(int32_t) * 4);
983
+ } else {
984
+ memset(params.rope_params.sections, 0, sizeof(params.rope_params.sections));
985
+ }
986
+
987
+ // Phase 1: Initialize CPU comparison context and copy source buffers (before ET kernel)
988
+ ggml_et_cpu_compare_ctx cpu_cmp_ctx;
989
+ bool cpu_comparison_active = false;
990
+ if (rope_cpu_compare_config.enabled) {
991
+ GGML_LOG_DEBUG("ET: Initializing CPU comparison for ROPE operation\n");
992
+ if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_ROPE)) {
993
+ cpu_comparison_active = true;
994
+ } else {
995
+ GGML_LOG_WARN("ET: Failed to initialize CPU comparison for ROPE operation\n");
996
+ }
997
+ }
998
+
999
+ bool kernel_result = ggml_et_launch_kernel(dev_ctx, kernel_name, &params, sizeof(params), 0xFFFFFFFF);
1000
+
1001
+ // Phase 2: Execute CPU computation and compare with ET result (after ET kernel)
1002
+ if (cpu_comparison_active) {
1003
+ if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &rope_cpu_compare_config)) {
1004
+ GGML_LOG_WARN("ET: CPU comparison failed for ROPE operation\n");
1005
+ }
1006
+ ggml_et_cpu_compare_free(&cpu_cmp_ctx);
1007
+ }
1008
+
1009
+ ET_PERF_END_EXT("ROPE", kernel_name, node, "mode=0x%x|n_dims=%d|freq_base=%.2f|freq_scale=%.2f",
1010
+ params.rope_params.mode, params.rope_params.n_dims, (double) params.rope_params.freq_base,
1011
+ (double) params.rope_params.freq_scale);
1012
+ return kernel_result;
1013
+ }
1014
+
1015
+ bool ggml_et_op_rms_norm(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
1016
+ ET_PERF_START();
1017
+
1018
+ if (!dev_ctx || !node) {
1019
+ GGML_LOG_ERROR("ET: Invalid parameters for RMS_NORM operation\n");
1020
+ return false;
1021
+ }
1022
+
1023
+ if (!node->src[0]) {
1024
+ GGML_LOG_ERROR("ET: RMS_NORM operation missing required input\n");
1025
+ return false;
1026
+ }
1027
+
1028
+ const char * kernel_name;
1029
+
1030
+ if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_F32) {
1031
+ kernel_name = "rms_norm_f32";
1032
+
1033
+ } else {
1034
+ GGML_LOG_ERROR("ET: RMS_NORM operation with unsupported types: dst=%s src0=%s\n", ggml_type_name(node->type),
1035
+ ggml_type_name(node->src[0]->type));
1036
+ return false;
1037
+ }
1038
+
1039
+ float eps;
1040
+ memcpy(&eps, node->op_params, sizeof(float));
1041
+
1042
+ ggml_et_rms_norm_params params;
1043
+ params.src0 = *node->src[0]; // F32 input tensor
1044
+ params.dst = *node; // F32 output tensor
1045
+ params.eps = eps; // Epsilon parameter for numerical stability
1046
+
1047
+ // Phase 1: Initialize CPU comparison context and copy source buffers (before ET kernel)
1048
+ ggml_et_cpu_compare_ctx cpu_cmp_ctx;
1049
+ bool cpu_comparison_active = false;
1050
+ if (rms_norm_cpu_compare_config.enabled) {
1051
+ if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_RMS_NORM)) {
1052
+ cpu_comparison_active = true;
1053
+ } else {
1054
+ GGML_LOG_WARN("ET: Failed to initialize CPU comparison for RMS_NORM operation\n");
1055
+ }
1056
+ }
1057
+
1058
+ bool kernel_result = ggml_et_launch_kernel(dev_ctx, kernel_name, &params, sizeof(params), 0xFFFFFFFF);
1059
+
1060
+ // Phase 2: Execute CPU computation and compare with ET result (after ET kernel)
1061
+ if (cpu_comparison_active) {
1062
+ if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &rms_norm_cpu_compare_config)) {
1063
+ GGML_LOG_WARN("ET: CPU comparison failed for RMS_NORM operation\n");
1064
+ }
1065
+ ggml_et_cpu_compare_free(&cpu_cmp_ctx);
1066
+ }
1067
+
1068
+ ET_PERF_END_EXT("RMS_NORM", kernel_name, node, "eps=%.6f", (double) eps);
1069
+ return kernel_result;
1070
+ }
1071
+
1072
+ bool ggml_et_op_norm(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
1073
+ ET_PERF_START();
1074
+
1075
+ if (!dev_ctx || !node) {
1076
+ GGML_LOG_ERROR("ET: Invalid parameters for NORM operation\n");
1077
+ return false;
1078
+ }
1079
+
1080
+ if (!node->src[0]) {
1081
+ GGML_LOG_ERROR("ET: NORM operation missing required input\n");
1082
+ return false;
1083
+ }
1084
+
1085
+ const char * kernel_name;
1086
+
1087
+ if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_F32) {
1088
+ kernel_name = "norm_f32";
1089
+
1090
+ } else {
1091
+ GGML_LOG_ERROR("ET: NORM operation with unsupported types: dst=%s src0=%s\n", ggml_type_name(node->type),
1092
+ ggml_type_name(node->src[0]->type));
1093
+ return false;
1094
+ }
1095
+
1096
+ float eps;
1097
+ memcpy(&eps, node->op_params, sizeof(float));
1098
+
1099
+ ggml_et_norm_params params;
1100
+ params.src0 = *node->src[0]; // F32 input tensor
1101
+ params.dst = *node; // F32 output tensor
1102
+ params.eps = eps; // Epsilon parameter for numerical stability
1103
+
1104
+ // Phase 1: Initialize CPU comparison context and copy source buffers (before ET kernel)
1105
+ ggml_et_cpu_compare_ctx cpu_cmp_ctx;
1106
+ bool cpu_comparison_active = false;
1107
+ if (norm_cpu_compare_config.enabled) {
1108
+ if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_NORM)) {
1109
+ cpu_comparison_active = true;
1110
+ } else {
1111
+ GGML_LOG_WARN("ET: Failed to initialize CPU comparison for NORM operation\n");
1112
+ }
1113
+ }
1114
+
1115
+ bool kernel_result = ggml_et_launch_kernel(dev_ctx, kernel_name, &params, sizeof(params), 0xFFFFFFFF);
1116
+
1117
+ // Phase 2: Execute CPU computation and compare with ET result (after ET kernel)
1118
+ if (cpu_comparison_active) {
1119
+ if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &norm_cpu_compare_config)) {
1120
+ GGML_LOG_WARN("ET: CPU comparison failed for NORM operation\n");
1121
+ }
1122
+ ggml_et_cpu_compare_free(&cpu_cmp_ctx);
1123
+ }
1124
+
1125
+ ET_PERF_END_EXT("NORM", kernel_name, node, "eps=%.6f", (double) eps);
1126
+ return kernel_result;
1127
+ }
1128
+
1129
+ bool ggml_et_op_l2_norm(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
1130
+ ET_PERF_START();
1131
+
1132
+ if (!dev_ctx || !node) {
1133
+ GGML_LOG_ERROR("ET: Invalid parameters for L2_NORM operation\n");
1134
+ return false;
1135
+ }
1136
+
1137
+ if (!node->src[0]) {
1138
+ GGML_LOG_ERROR("ET: L2_NORM operation missing required input\n");
1139
+ return false;
1140
+ }
1141
+
1142
+ const char * kernel_name;
1143
+
1144
+ if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_F32) {
1145
+ kernel_name = "l2_norm_f32";
1146
+
1147
+ } else {
1148
+ GGML_LOG_ERROR("ET: L2_NORM operation with unsupported types: dst=%s src0=%s\n", ggml_type_name(node->type),
1149
+ ggml_type_name(node->src[0]->type));
1150
+ return false;
1151
+ }
1152
+
1153
+ float eps;
1154
+ memcpy(&eps, node->op_params, sizeof(float));
1155
+
1156
+ ggml_et_l2_norm_params params;
1157
+ params.src0 = *node->src[0]; // F32 input tensor
1158
+ params.dst = *node; // F32 output tensor
1159
+ params.eps = eps; // Epsilon parameter for numerical stability
1160
+
1161
+ // Phase 1: Initialize CPU comparison context and copy source buffers (before ET kernel)
1162
+ ggml_et_cpu_compare_ctx cpu_cmp_ctx;
1163
+ bool cpu_comparison_active = false;
1164
+ if (l2_norm_cpu_compare_config.enabled) {
1165
+ if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_L2_NORM)) {
1166
+ cpu_comparison_active = true;
1167
+ } else {
1168
+ GGML_LOG_WARN("ET: Failed to initialize CPU comparison for L2_NORM operation\n");
1169
+ }
1170
+ }
1171
+
1172
+ bool kernel_result = ggml_et_launch_kernel(dev_ctx, kernel_name, &params, sizeof(params), 0xFFFFFFFF);
1173
+
1174
+ // Phase 2: Execute CPU computation and compare with ET result (after ET kernel)
1175
+ if (cpu_comparison_active) {
1176
+ if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &l2_norm_cpu_compare_config)) {
1177
+ GGML_LOG_WARN("ET: CPU comparison failed for L2_NORM operation\n");
1178
+ }
1179
+ ggml_et_cpu_compare_free(&cpu_cmp_ctx);
1180
+ }
1181
+
1182
+ ET_PERF_END_EXT("L2_NORM", kernel_name, node, "eps=%.6f", (double) eps);
1183
+ return kernel_result;
1184
+ }
1185
+
1186
+ bool ggml_et_op_group_norm(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
1187
+ ET_PERF_START();
1188
+
1189
+ if (!dev_ctx || !node) {
1190
+ GGML_LOG_ERROR("ET: Invalid parameters for GROUP_NORM operation\n");
1191
+ return false;
1192
+ }
1193
+
1194
+ if (!node->src[0]) {
1195
+ GGML_LOG_ERROR("ET: GROUP_NORM operation missing required input\n");
1196
+ return false;
1197
+ }
1198
+
1199
+ if (node->type != GGML_TYPE_F32 || node->src[0]->type != GGML_TYPE_F32) {
1200
+ GGML_LOG_ERROR("ET: GROUP_NORM operation with unsupported types: dst=%s src0=%s\n", ggml_type_name(node->type),
1201
+ ggml_type_name(node->src[0]->type));
1202
+ return false;
1203
+ }
1204
+
1205
+ const int32_t n_groups = ggml_get_op_params_i32(node, 0);
1206
+ float eps;
1207
+ memcpy(&eps, (const float *) node->op_params + 1, sizeof(float));
1208
+
1209
+ ggml_et_group_norm_params params;
1210
+ params.src0 = *node->src[0];
1211
+ params.dst = *node;
1212
+ params.n_groups = n_groups;
1213
+ params.eps = eps;
1214
+
1215
+ ggml_et_cpu_compare_ctx cpu_cmp_ctx;
1216
+ bool cpu_comparison_active = false;
1217
+ if (group_norm_cpu_compare_config.enabled) {
1218
+ if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_GROUP_NORM)) {
1219
+ cpu_comparison_active = true;
1220
+ } else {
1221
+ GGML_LOG_WARN("ET: Failed to initialize CPU comparison for GROUP_NORM operation\n");
1222
+ }
1223
+ }
1224
+
1225
+ bool kernel_result = ggml_et_launch_kernel(dev_ctx, "group_norm_f32", &params, sizeof(params), 0xFFFFFFFF);
1226
+
1227
+ if (cpu_comparison_active) {
1228
+ if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &group_norm_cpu_compare_config)) {
1229
+ GGML_LOG_WARN("ET: CPU comparison failed for GROUP_NORM operation\n");
1230
+ }
1231
+ ggml_et_cpu_compare_free(&cpu_cmp_ctx);
1232
+ }
1233
+
1234
+ ET_PERF_END_EXT("GROUP_NORM", "group_norm_f32", node, "eps=%.6f|n_groups=%d", (double) eps, n_groups);
1235
+ return kernel_result;
1236
+ }
1237
+
1238
+ bool ggml_et_op_im2col(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
1239
+ ET_PERF_START();
1240
+
1241
+ if (!dev_ctx || !node) {
1242
+ GGML_LOG_ERROR("ET: Invalid parameters for IM2COL operation\n");
1243
+ return false;
1244
+ }
1245
+
1246
+ if (!node->src[0] || !node->src[1]) {
1247
+ GGML_LOG_ERROR("ET: IM2COL operation missing required inputs\n");
1248
+ return false;
1249
+ }
1250
+
1251
+ const bool supported_types =
1252
+ (node->type == GGML_TYPE_F32 && node->src[1]->type == GGML_TYPE_F32) ||
1253
+ (node->type == GGML_TYPE_F16 && (node->src[1]->type == GGML_TYPE_F16 || node->src[1]->type == GGML_TYPE_F32));
1254
+
1255
+ if (!supported_types) {
1256
+ GGML_LOG_ERROR("ET: IM2COL operation with unsupported types: dst=%s src1=%s\n", ggml_type_name(node->type),
1257
+ ggml_type_name(node->src[1]->type));
1258
+ return false;
1259
+ }
1260
+
1261
+ ggml_et_im2col_params params;
1262
+ params.src0 = *node->src[0];
1263
+ params.src1 = *node->src[1];
1264
+ params.dst = *node;
1265
+
1266
+ ggml_et_cpu_compare_ctx cpu_cmp_ctx;
1267
+ bool cpu_comparison_active = false;
1268
+ if (im2col_cpu_compare_config.enabled) {
1269
+ if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_IM2COL)) {
1270
+ cpu_comparison_active = true;
1271
+ } else {
1272
+ GGML_LOG_WARN("ET: Failed to initialize CPU comparison for IM2COL operation\n");
1273
+ }
1274
+ }
1275
+
1276
+ bool kernel_result = ggml_et_launch_kernel(dev_ctx, "im2col", &params, sizeof(params), 0xFFFFFFFF);
1277
+
1278
+ if (cpu_comparison_active) {
1279
+ if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &im2col_cpu_compare_config)) {
1280
+ GGML_LOG_WARN("ET: CPU comparison failed for IM2COL operation\n");
1281
+ }
1282
+ ggml_et_cpu_compare_free(&cpu_cmp_ctx);
1283
+ }
1284
+
1285
+ ET_PERF_END("IM2COL", "im2col", node);
1286
+ return kernel_result;
1287
+ }
1288
+
1289
+ bool ggml_et_op_conv_2d(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
1290
+ ET_PERF_START();
1291
+
1292
+ if (!dev_ctx || !node) {
1293
+ return false;
1294
+ }
1295
+ if (!node->src[0] || !node->src[1]) {
1296
+ return false;
1297
+ }
1298
+ if (!node->data || !node->src[0]->data || !node->src[1]->data) {
1299
+ return false;
1300
+ }
1301
+
1302
+ // Kernel constraints (mirror supports_op; recheck here as a guard).
1303
+ const ggml_tensor * flt = node->src[0]; // [Kw, Kh, Cin, Cout]
1304
+ const ggml_tensor * in = node->src[1]; // [W, H, Cin, N]
1305
+ if (node->type != GGML_TYPE_F32 || flt->type != GGML_TYPE_F32 || in->type != GGML_TYPE_F32) {
1306
+ return false;
1307
+ }
1308
+
1309
+ const int32_t s0 = ggml_get_op_params_i32(node, 0);
1310
+ const int32_t s1 = ggml_get_op_params_i32(node, 1);
1311
+ const int32_t p0 = ggml_get_op_params_i32(node, 2);
1312
+ const int32_t p1 = ggml_get_op_params_i32(node, 3);
1313
+ const int32_t d0 = ggml_get_op_params_i32(node, 4);
1314
+ const int32_t d1 = ggml_get_op_params_i32(node, 5);
1315
+
1316
+ if (s0 < 1 || s1 < 1) {
1317
+ return false;
1318
+ }
1319
+ if (d0 != 1 || d1 != 1) {
1320
+ return false;
1321
+ }
1322
+ if (flt->ne[2] % 16 != 0 || flt->ne[3] % 16 != 0) {
1323
+ return false;
1324
+ }
1325
+ if (in->ne[3] != 1) {
1326
+ return false;
1327
+ }
1328
+ if (node->ne[0] <= 0) {
1329
+ return false; // OW > 0 (any width OK; staging path handles non-16)
1330
+ }
1331
+ (void) p0;
1332
+ (void) p1;
1333
+
1334
+ ggml_et_binary_params params;
1335
+ params.src0 = *node->src[0];
1336
+ params.src1 = *node->src[1];
1337
+ params.dst = *node;
1338
+
1339
+ bool kernel_result = ggml_et_launch_kernel(dev_ctx, "conv_2d_f32_me", &params, sizeof(params), 0xFFFFFFFFu);
1340
+
1341
+ ET_PERF_END("CONV_2D", "conv_2d_f32_me", node);
1342
+ return kernel_result;
1343
+ }
1344
+
1345
+ bool ggml_et_op_softmax(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
1346
+ ET_PERF_START();
1347
+
1348
+ if (!dev_ctx || !node) {
1349
+ GGML_LOG_ERROR("ET: Invalid parameters for SOFTMAX operation\n");
1350
+ return false;
1351
+ }
1352
+
1353
+ if (!node->src[0]) {
1354
+ GGML_LOG_ERROR("ET: SOFTMAX operation missing required input\n");
1355
+ return false;
1356
+ }
1357
+
1358
+ const char * kernel_name;
1359
+
1360
+ if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_F32) {
1361
+ kernel_name = "softmax_f32";
1362
+
1363
+ } else {
1364
+ GGML_LOG_ERROR("ET: SOFTMAX operation with unsupported types: dst=%s src0=%s\n", ggml_type_name(node->type),
1365
+ ggml_type_name(node->src[0]->type));
1366
+ return false;
1367
+ }
1368
+
1369
+ // Validate contiguity requirements
1370
+ if (!ggml_is_contiguous(node)) {
1371
+ GGML_LOG_ERROR("ET: SOFTMAX operation requires contiguous destination tensor\n");
1372
+ return false;
1373
+ }
1374
+
1375
+ if (!ggml_is_contiguous(node->src[0])) {
1376
+ GGML_LOG_ERROR("ET: SOFTMAX operation requires contiguous source tensor\n");
1377
+ return false;
1378
+ }
1379
+
1380
+ // Check optional mask tensor
1381
+ if (node->src[1]) {
1382
+ if (node->src[1]->type != GGML_TYPE_F32) {
1383
+ GGML_LOG_ERROR("ET: SOFTMAX operation with unsupported mask type: %s (F32 required)\n",
1384
+ ggml_type_name(node->src[1]->type));
1385
+ return false;
1386
+ }
1387
+ if (!ggml_is_contiguous(node->src[1])) {
1388
+ GGML_LOG_ERROR("ET: SOFTMAX operation requires contiguous mask tensor\n");
1389
+ return false;
1390
+ }
1391
+ }
1392
+
1393
+ // Check optional sinks tensor
1394
+ if (node->src[2]) {
1395
+ if (node->src[2]->type != GGML_TYPE_F32) {
1396
+ GGML_LOG_ERROR("ET: SOFTMAX operation with unsupported sinks type: %s (F32 required)\n",
1397
+ ggml_type_name(node->src[2]->type));
1398
+ return false;
1399
+ }
1400
+ if (!ggml_is_contiguous(node->src[2])) {
1401
+ GGML_LOG_ERROR("ET: SOFTMAX operation requires contiguous sinks tensor\n");
1402
+ return false;
1403
+ }
1404
+ }
1405
+
1406
+ // Extract scale and max_bias from op_params
1407
+ float scale = 1.0f;
1408
+ float max_bias = 0.0f;
1409
+ if (node->op_params) {
1410
+ memcpy(&scale, (const float *) node->op_params + 0, sizeof(float));
1411
+ memcpy(&max_bias, (const float *) node->op_params + 1, sizeof(float));
1412
+ }
1413
+
1414
+ ggml_et_softmax_params params;
1415
+ params.src0 = *node->src[0]; // F32 input tensor
1416
+ if (node->src[1]) {
1417
+ params.src1 = *node->src[1]; // F32 mask tensor
1418
+ } else {
1419
+ memset(&params.src1, 0, sizeof(params.src1)); // Zero if no mask
1420
+ }
1421
+ if (node->src[2]) {
1422
+ params.src2 = *node->src[2]; // F32 sinks tensor
1423
+ } else {
1424
+ memset(&params.src2, 0, sizeof(params.src2)); // Zero if no sinks
1425
+ }
1426
+ params.dst = *node; // F32 output tensor
1427
+ params.scale = scale; // Scale factor
1428
+ params.max_bias = max_bias; // ALiBi bias
1429
+
1430
+ // Phase 1: Initialize CPU comparison context and copy source buffers (before ET kernel)
1431
+ ggml_et_cpu_compare_ctx cpu_cmp_ctx;
1432
+ bool cpu_comparison_active = false;
1433
+ if (softmax_cpu_compare_config.enabled) {
1434
+ if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_SOFT_MAX)) {
1435
+ cpu_comparison_active = true;
1436
+ } else {
1437
+ GGML_LOG_WARN("ET: Failed to initialize CPU comparison for SOFTMAX operation\n");
1438
+ }
1439
+ }
1440
+
1441
+ bool kernel_result = ggml_et_launch_kernel(dev_ctx, kernel_name, &params, sizeof(params), 0xFFFFFFFF);
1442
+
1443
+ // Phase 2: Execute CPU computation and compare with ET result (after ET kernel)
1444
+ if (cpu_comparison_active) {
1445
+ if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &softmax_cpu_compare_config)) {
1446
+ GGML_LOG_WARN("ET: CPU comparison failed for SOFTMAX operation\n");
1447
+ }
1448
+ ggml_et_cpu_compare_free(&cpu_cmp_ctx);
1449
+ }
1450
+
1451
+ ET_PERF_END_EXT("SOFTMAX", kernel_name, node, "scale=%.6f|max_bias=%.6f|has_mask=%s", (double) scale,
1452
+ (double) max_bias, node->src[1] ? "yes" : "no");
1453
+ return kernel_result;
1454
+ }
1455
+
1456
+ bool ggml_et_op_flash_attn_ext(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
1457
+ ET_PERF_START();
1458
+
1459
+ if (!dev_ctx || !node) {
1460
+ GGML_LOG_ERROR("ET: Invalid parameters for FLASH_ATTN_EXT operation\n");
1461
+ return false;
1462
+ }
1463
+
1464
+ if (!node->src[0] || !node->src[1] || !node->src[2]) {
1465
+ GGML_LOG_ERROR("ET: FLASH_ATTN_EXT operation missing required inputs\n");
1466
+ return false;
1467
+ }
1468
+
1469
+ if (node->type != GGML_TYPE_F32 || node->src[0]->type != GGML_TYPE_F32) {
1470
+ GGML_LOG_ERROR("ET: FLASH_ATTN_EXT requires F32 Q and dst, got dst=%s q=%s\n", ggml_type_name(node->type),
1471
+ ggml_type_name(node->src[0]->type));
1472
+ return false;
1473
+ }
1474
+
1475
+ // K and V can be F16 or F32
1476
+ if ((node->src[1]->type != GGML_TYPE_F32 && node->src[1]->type != GGML_TYPE_F16) ||
1477
+ (node->src[2]->type != GGML_TYPE_F32 && node->src[2]->type != GGML_TYPE_F16)) {
1478
+ GGML_LOG_ERROR("ET: FLASH_ATTN_EXT K/V must be F16 or F32, got k=%s v=%s\n", ggml_type_name(node->src[1]->type),
1479
+ ggml_type_name(node->src[2]->type));
1480
+ return false;
1481
+ }
1482
+
1483
+ if (node->src[4] != nullptr) {
1484
+ GGML_LOG_ERROR("ET: FLASH_ATTN_EXT baseline kernel does not support sinks\n");
1485
+ return false;
1486
+ }
1487
+
1488
+ // Mask is optional; if present must be F16 or F32
1489
+ if (node->src[3] != nullptr && node->src[3]->type != GGML_TYPE_F32 && node->src[3]->type != GGML_TYPE_F16) {
1490
+ GGML_LOG_ERROR("ET: FLASH_ATTN_EXT mask must be F16 or F32, got %s\n", ggml_type_name(node->src[3]->type));
1491
+ return false;
1492
+ }
1493
+
1494
+ // Q and dst must be row-contiguous F32
1495
+ if (!ggml_is_contiguous_rows(node) || !ggml_is_contiguous_rows(node->src[0])) {
1496
+ GGML_LOG_ERROR("ET: FLASH_ATTN_EXT requires row-contiguous Q and dst\n");
1497
+ return false;
1498
+ }
1499
+
1500
+ if (node->nb[0] != sizeof(float) || node->src[0]->nb[0] != sizeof(float)) {
1501
+ GGML_LOG_ERROR("ET: FLASH_ATTN_EXT requires contiguous F32 rows for Q and dst\n");
1502
+ return false;
1503
+ }
1504
+
1505
+ // K/V must have element-sized stride in dim 0
1506
+ const size_t k_elem = node->src[1]->type == GGML_TYPE_F16 ? 2 : 4;
1507
+ const size_t v_elem = node->src[2]->type == GGML_TYPE_F16 ? 2 : 4;
1508
+ if (node->src[1]->nb[0] != k_elem || node->src[2]->nb[0] != v_elem) {
1509
+ GGML_LOG_ERROR("ET: FLASH_ATTN_EXT K/V must have element-sized stride in dim 0\n");
1510
+ return false;
1511
+ }
1512
+
1513
+ float scale = 1.0f;
1514
+ float max_bias = 0.0f;
1515
+ float logit_softcap = 0.0f;
1516
+ memcpy(&scale, (const float *) node->op_params + 0, sizeof(scale));
1517
+ memcpy(&max_bias, (const float *) node->op_params + 1, sizeof(max_bias));
1518
+ memcpy(&logit_softcap, (const float *) node->op_params + 2, sizeof(logit_softcap));
1519
+
1520
+ if (max_bias != 0.0f || logit_softcap != 0.0f) {
1521
+ GGML_LOG_ERROR("ET: FLASH_ATTN_EXT baseline kernel does not support max_bias or logit_softcap\n");
1522
+ return false;
1523
+ }
1524
+
1525
+ const ggml_prec prec = ggml_flash_attn_ext_get_prec(node);
1526
+ if (prec != GGML_PREC_F32 && prec != GGML_PREC_DEFAULT) {
1527
+ GGML_LOG_ERROR("ET: FLASH_ATTN_EXT baseline kernel only supports F32 precision\n");
1528
+ return false;
1529
+ }
1530
+
1531
+ // dk must match between Q and K; dv must match between V and dst
1532
+ if (node->src[0]->ne[0] != node->src[1]->ne[0]) {
1533
+ GGML_LOG_ERROR("ET: FLASH_ATTN_EXT dk mismatch: Q=%lld K=%lld\n", (long long) node->src[0]->ne[0],
1534
+ (long long) node->src[1]->ne[0]);
1535
+ return false;
1536
+ }
1537
+
1538
+ if (node->src[2]->ne[0] != node->ne[0]) {
1539
+ GGML_LOG_ERROR("ET: FLASH_ATTN_EXT dv mismatch: V=%lld dst=%lld\n", (long long) node->src[2]->ne[0],
1540
+ (long long) node->ne[0]);
1541
+ return false;
1542
+ }
1543
+
1544
+ if (node->src[2]->ne[0] > 512) {
1545
+ GGML_LOG_ERROR("ET: FLASH_ATTN_EXT dv=%lld exceeds maximum 512\n", (long long) node->src[2]->ne[0]);
1546
+ return false;
1547
+ }
1548
+
1549
+ if (node->src[0]->ne[0] > 512) {
1550
+ GGML_LOG_ERROR("ET: FLASH_ATTN_EXT dk=%lld exceeds maximum 512\n", (long long) node->src[0]->ne[0]);
1551
+ return false;
1552
+ }
1553
+
1554
+ // GQA: n_head_q must be a multiple of n_head_kv
1555
+ const int64_t nhq = node->src[0]->ne[2];
1556
+ const int64_t nhk = node->src[1]->ne[2];
1557
+ if (nhq % nhk != 0) {
1558
+ GGML_LOG_ERROR("ET: FLASH_ATTN_EXT n_head_q (%lld) not divisible by n_head_kv (%lld)\n", (long long) nhq,
1559
+ (long long) nhk);
1560
+ return false;
1561
+ }
1562
+
1563
+ // K and V must have matching sequence length, heads, and batch dims
1564
+ if (node->src[1]->ne[1] != node->src[2]->ne[1] || node->src[1]->ne[2] != node->src[2]->ne[2] ||
1565
+ node->src[1]->ne[3] != node->src[2]->ne[3]) {
1566
+ GGML_LOG_ERROR("ET: FLASH_ATTN_EXT K/V shape mismatch\n");
1567
+ return false;
1568
+ }
1569
+
1570
+ // dst layout checks: [dv, nhq, nq, no]
1571
+ if (node->src[0]->ne[1] != node->ne[2] || node->src[0]->ne[2] != node->ne[1] ||
1572
+ node->src[0]->ne[3] != node->ne[3]) {
1573
+ GGML_LOG_ERROR("ET: FLASH_ATTN_EXT dst shape mismatch\n");
1574
+ return false;
1575
+ }
1576
+
1577
+ // Batch dims: Q batch must match K batch
1578
+ if (node->src[0]->ne[3] != node->src[1]->ne[3]) {
1579
+ GGML_LOG_ERROR("ET: FLASH_ATTN_EXT batch dimension mismatch\n");
1580
+ return false;
1581
+ }
1582
+
1583
+ ggml_et_flash_attn_ext_params params;
1584
+ memset(&params, 0, sizeof(params));
1585
+ params.src0 = *node->src[0];
1586
+ params.src1 = *node->src[1];
1587
+ params.src2 = *node->src[2];
1588
+ if (node->src[3] != nullptr) {
1589
+ params.mask = *node->src[3];
1590
+ params.has_mask = 1;
1591
+ }
1592
+ params.dst = *node;
1593
+ params.scale = scale;
1594
+
1595
+ // Use matrix engine kernel when K/V are F16 and dk is a multiple of 32
1596
+ const char * kernel_name;
1597
+ if (node->src[1]->type == GGML_TYPE_F16 && node->src[2]->type == GGML_TYPE_F16 && (node->src[0]->ne[0] % 32) == 0) {
1598
+ kernel_name = "flash_attn_ext_f16_me";
1599
+ } else {
1600
+ kernel_name = "flash_attn_ext_f32";
1601
+ }
1602
+
1603
+ const bool kernel_result = ggml_et_launch_kernel(dev_ctx, kernel_name, &params, sizeof(params), 0xFFFFFFFF);
1604
+
1605
+ ET_PERF_END_EXT("FLASH_ATTN_EXT", kernel_name, node, "scale=%.6f", (double) scale);
1606
+ return kernel_result;
1607
+ }
1608
+
1609
+ bool ggml_et_op_get_rows(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
1610
+ ET_PERF_START();
1611
+
1612
+ if (!dev_ctx || !node) {
1613
+ GGML_LOG_ERROR("ET: Invalid parameters for GET_ROWS operation\n");
1614
+ return false;
1615
+ }
1616
+
1617
+ if (!node->src[0] || !node->src[1]) {
1618
+ GGML_LOG_ERROR("ET: GET_ROWS operation missing required inputs\n");
1619
+ return false;
1620
+ }
1621
+
1622
+ const char * kernel_name;
1623
+
1624
+ if (node->type == GGML_TYPE_F32 && node->src[1]->type == GGML_TYPE_I32 &&
1625
+ (node->src[0]->type == GGML_TYPE_F32 || node->src[0]->type == GGML_TYPE_F16 ||
1626
+ node->src[0]->type == GGML_TYPE_Q4_0 || node->src[0]->type == GGML_TYPE_Q8_0 ||
1627
+ node->src[0]->type == GGML_TYPE_Q4_K)) {
1628
+ kernel_name = "get_rows_f32";
1629
+
1630
+ } else {
1631
+ GGML_LOG_ERROR("ET: GET_ROWS operation with unsupported types: dst=%s src0=%s src1=%s\n",
1632
+ ggml_type_name(node->type), ggml_type_name(node->src[0]->type),
1633
+ ggml_type_name(node->src[1]->type));
1634
+ return false;
1635
+ }
1636
+
1637
+ // Validate contiguity requirements
1638
+ if (!ggml_is_contiguous(node)) {
1639
+ GGML_LOG_ERROR("ET: GET_ROWS operation requires contiguous destination tensor\n");
1640
+ return false;
1641
+ }
1642
+
1643
+ if (!ggml_is_contiguous(node->src[0])) {
1644
+ GGML_LOG_ERROR("ET: GET_ROWS operation requires contiguous data tensor\n");
1645
+ return false;
1646
+ }
1647
+
1648
+ if (!ggml_is_contiguous(node->src[1])) {
1649
+ GGML_LOG_ERROR("ET: GET_ROWS operation requires contiguous indices tensor\n");
1650
+ return false;
1651
+ }
1652
+
1653
+ // Validate dimension constraints from ggml implementation
1654
+ if (node->src[0]->ne[2] != node->src[1]->ne[1] || node->src[1]->ne[3] != 1) {
1655
+ GGML_LOG_ERROR(
1656
+ "ET: GET_ROWS operation dimension constraint failed: src0.ne[2]=%lld != src1.ne[1]=%lld or src1.ne[3]=%lld "
1657
+ "!= 1\n",
1658
+ (long long) node->src[0]->ne[2], (long long) node->src[1]->ne[1], (long long) node->src[1]->ne[3]);
1659
+ return false;
1660
+ }
1661
+
1662
+ ggml_et_get_rows_params params;
1663
+ params.src0 = *node->src[0]; // Data tensor (F32 or Q8_0)
1664
+ params.src1 = *node->src[1]; // Indices tensor (I32)
1665
+ params.dst = *node; // Output tensor (F32)
1666
+
1667
+ // Phase 1: Initialize CPU comparison context and copy source buffers (before ET kernel)
1668
+ ggml_et_cpu_compare_ctx cpu_cmp_ctx;
1669
+ bool cpu_comparison_active = false;
1670
+ if (get_rows_cpu_compare_config.enabled) {
1671
+ if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_GET_ROWS)) {
1672
+ cpu_comparison_active = true;
1673
+ } else {
1674
+ GGML_LOG_WARN("ET: Failed to initialize CPU comparison for GET_ROWS operation\n");
1675
+ }
1676
+ }
1677
+
1678
+ bool kernel_result = ggml_et_launch_kernel(dev_ctx, kernel_name, &params, sizeof(params), 0xFFFFFFFF);
1679
+
1680
+ // Phase 2: Execute CPU computation and compare with ET result (after ET kernel)
1681
+ if (cpu_comparison_active) {
1682
+ if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &get_rows_cpu_compare_config)) {
1683
+ GGML_LOG_WARN("ET: CPU comparison failed for GET_ROWS operation\n");
1684
+ }
1685
+ ggml_et_cpu_compare_free(&cpu_cmp_ctx);
1686
+ }
1687
+
1688
+ ET_PERF_END("GET_ROWS", kernel_name, node);
1689
+ return kernel_result;
1690
+ }
1691
+
1692
+ bool ggml_et_op_cont(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
1693
+ ET_PERF_START();
1694
+
1695
+ // Validate source tensor exists
1696
+ if (!node->src[0]) {
1697
+ GGML_LOG_ERROR("ET: CONT operation missing source tensor\n");
1698
+ return false;
1699
+ }
1700
+
1701
+ // Validate types match (input and output must be same type)
1702
+ if (node->type != node->src[0]->type) {
1703
+ GGML_LOG_ERROR("ET: CONT operation type mismatch: src=%s dst=%s\n", ggml_type_name(node->src[0]->type),
1704
+ ggml_type_name(node->type));
1705
+ return false;
1706
+ }
1707
+
1708
+ // Validate supported types
1709
+ if (node->type != GGML_TYPE_F32 && node->type != GGML_TYPE_F16) {
1710
+ GGML_LOG_ERROR("ET: CONT operation unsupported type: %s (only F32 and F16 supported)\n",
1711
+ ggml_type_name(node->type));
1712
+ return false;
1713
+ }
1714
+
1715
+ // Validate contiguity - output must be contiguous, input can be non-contiguous
1716
+ if (!ggml_is_contiguous(node)) {
1717
+ GGML_LOG_ERROR("ET: CONT operation requires contiguous output tensor\n");
1718
+ return false;
1719
+ }
1720
+
1721
+ // Select kernel based on type
1722
+ const char * kernel_name;
1723
+ if (node->type == GGML_TYPE_F32) {
1724
+ kernel_name = "cont_f32";
1725
+ } else if (node->type == GGML_TYPE_F16) {
1726
+ kernel_name = "cont_f16";
1727
+ } else {
1728
+ GGML_LOG_ERROR("ET: CONT operation with unsupported type: %s\n", ggml_type_name(node->type));
1729
+ return false;
1730
+ }
1731
+
1732
+ ggml_et_cont_params params;
1733
+ params.src0 = *node->src[0]; // Input tensor (potentially non-contiguous)
1734
+ params.dst = *node; // Output tensor (contiguous)
1735
+
1736
+ // Phase 1: Initialize CPU comparison context and copy source buffers (before ET kernel)
1737
+ ggml_et_cpu_compare_ctx cpu_cmp_ctx;
1738
+ bool cpu_comparison_active = false;
1739
+ if (cont_cpu_compare_config.enabled) {
1740
+ if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_CONT)) {
1741
+ cpu_comparison_active = true;
1742
+ } else {
1743
+ GGML_LOG_WARN("ET: Failed to initialize CPU comparison for CONT operation\n");
1744
+ }
1745
+ }
1746
+
1747
+ bool kernel_result = ggml_et_launch_kernel(dev_ctx, kernel_name, &params, sizeof(params), 0xFFFFFFFF);
1748
+
1749
+ // Phase 2: Execute CPU computation and compare with ET result (after ET kernel)
1750
+ if (cpu_comparison_active) {
1751
+ if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &cont_cpu_compare_config)) {
1752
+ GGML_LOG_WARN("ET: CPU comparison failed for CONT operation\n");
1753
+ }
1754
+ ggml_et_cpu_compare_free(&cpu_cmp_ctx);
1755
+ }
1756
+
1757
+ ET_PERF_END("CONT", kernel_name, node);
1758
+ return kernel_result;
1759
+ }
1760
+
1761
+ bool ggml_et_op_cumsum(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
1762
+ ET_PERF_START();
1763
+
1764
+ if (!dev_ctx || !node || !node->src[0]) {
1765
+ GGML_LOG_ERROR("ET: Invalid parameters for CUMSUM operation\n");
1766
+ return false;
1767
+ }
1768
+
1769
+ if (node->type != GGML_TYPE_F32 || node->src[0]->type != GGML_TYPE_F32) {
1770
+ GGML_LOG_ERROR("ET: CUMSUM operation with unsupported types: dst=%s src0=%s\n", ggml_type_name(node->type),
1771
+ ggml_type_name(node->src[0]->type));
1772
+ return false;
1773
+ }
1774
+
1775
+ const char * kernel_name = "cumsum_f32";
1776
+
1777
+ ggml_et_cumsum_params params;
1778
+ params.src0 = *node->src[0];
1779
+ params.dst = *node;
1780
+
1781
+ ggml_et_cpu_compare_ctx cpu_cmp_ctx;
1782
+ bool cpu_comparison_active = false;
1783
+ if (cumsum_cpu_compare_config.enabled) {
1784
+ if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_CUMSUM)) {
1785
+ cpu_comparison_active = true;
1786
+ } else {
1787
+ GGML_LOG_WARN("ET: Failed to initialize CPU comparison for CUMSUM operation\n");
1788
+ }
1789
+ }
1790
+
1791
+ bool kernel_result = ggml_et_launch_kernel(dev_ctx, kernel_name, &params, sizeof(params), 0xFFFFFFFF);
1792
+
1793
+ if (cpu_comparison_active) {
1794
+ if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &cumsum_cpu_compare_config)) {
1795
+ GGML_LOG_WARN("ET: CPU comparison failed for CUMSUM operation\n");
1796
+ }
1797
+ ggml_et_cpu_compare_free(&cpu_cmp_ctx);
1798
+ }
1799
+
1800
+ ET_PERF_END("CUMSUM", kernel_name, node);
1801
+ return kernel_result;
1802
+ }
1803
+
1804
+ bool ggml_et_op_cpy(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
1805
+ ET_PERF_START();
1806
+
1807
+ // CPY copies data from src[0] into the layout of dst (which matches src[1])
1808
+ // For same-type with contiguous dst, this is identical to CONT
1809
+ if (!node->src[0]) {
1810
+ GGML_LOG_ERROR("ET: CPY operation missing source tensor\n");
1811
+ return false;
1812
+ }
1813
+
1814
+ // Scalar / zero-element special path: if any dimension is 0, nothing to copy
1815
+ const int64_t nelements = node->ne[0] * node->ne[1] * node->ne[2] * node->ne[3];
1816
+ if (nelements == 0) {
1817
+ GGML_LOG_DEBUG("ET: CPY no-op (zero elements): ne=[%" PRId64 ",%" PRId64 ",%" PRId64 ",%" PRId64 "]\n",
1818
+ node->ne[0], node->ne[1], node->ne[2], node->ne[3]);
1819
+ ET_PERF_END("CPY", "noop", node);
1820
+ return true;
1821
+ }
1822
+
1823
+ // Only F32 and F16 supported for dst
1824
+ if (node->type != GGML_TYPE_F32 && node->type != GGML_TYPE_F16) {
1825
+ GGML_LOG_ERROR("ET: CPY unsupported dst type: %s\n", ggml_type_name(node->type));
1826
+ return false;
1827
+ }
1828
+
1829
+ // Select kernel based on src/dst type combination
1830
+ const char * kernel_name;
1831
+ if (node->src[0]->type == GGML_TYPE_F32 && node->type == GGML_TYPE_F32) {
1832
+ kernel_name = "cont_f32";
1833
+ } else if (node->src[0]->type == GGML_TYPE_F16 && node->type == GGML_TYPE_F16) {
1834
+ kernel_name = "cont_f16";
1835
+ } else if (node->src[0]->type == GGML_TYPE_F32 && node->type == GGML_TYPE_F16) {
1836
+ kernel_name = "cpy_f32_f16";
1837
+ } else {
1838
+ GGML_LOG_ERROR("ET: CPY unsupported type combination: src=%s dst=%s\n", ggml_type_name(node->src[0]->type),
1839
+ ggml_type_name(node->type));
1840
+ return false;
1841
+ }
1842
+
1843
+ ggml_et_cont_params params;
1844
+ params.src0 = *node->src[0];
1845
+ params.dst = *node;
1846
+
1847
+ // CPU comparison for debugging
1848
+ ggml_et_cpu_compare_ctx cpu_cmp_ctx;
1849
+ bool cpu_comparison_active = false;
1850
+ if (cont_cpu_compare_config.enabled) {
1851
+ if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_CPY)) {
1852
+ cpu_comparison_active = true;
1853
+ } else {
1854
+ GGML_LOG_WARN("ET: Failed to initialize CPU comparison for CPY operation\n");
1855
+ }
1856
+ }
1857
+
1858
+ bool kernel_result = ggml_et_launch_kernel(dev_ctx, kernel_name, &params, sizeof(params), 0xFFFFFFFF);
1859
+
1860
+ if (cpu_comparison_active) {
1861
+ if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &cont_cpu_compare_config)) {
1862
+ GGML_LOG_WARN("ET: CPU comparison failed for CPY operation\n");
1863
+ }
1864
+ ggml_et_cpu_compare_free(&cpu_cmp_ctx);
1865
+ }
1866
+
1867
+ ET_PERF_END("CPY", kernel_name, node);
1868
+ return kernel_result;
1869
+ }
1870
+
1871
+ bool ggml_et_op_concat(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
1872
+ ET_PERF_START();
1873
+
1874
+ if (!dev_ctx || !node) {
1875
+ GGML_LOG_ERROR("ET: Invalid parameters for CONCAT operation\n");
1876
+ return false;
1877
+ }
1878
+
1879
+ if (!node->src[0] || !node->src[1]) {
1880
+ GGML_LOG_ERROR("ET: CONCAT operation missing required inputs\n");
1881
+ return false;
1882
+ }
1883
+
1884
+ const char * kernel_name;
1885
+
1886
+ if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_F32 && node->src[1]->type == GGML_TYPE_F32) {
1887
+ kernel_name = "concat_f32";
1888
+
1889
+ } else {
1890
+ GGML_LOG_ERROR("ET: CONCAT operation with unsupported types: dst=%s src0=%s src1=%s\n",
1891
+ ggml_type_name(node->type), ggml_type_name(node->src[0]->type),
1892
+ ggml_type_name(node->src[1]->type));
1893
+ return false;
1894
+ }
1895
+
1896
+ int32_t dim;
1897
+ memcpy(&dim, node->op_params, sizeof(int32_t));
1898
+
1899
+ ggml_et_concat_params params;
1900
+ params.src0 = *node->src[0];
1901
+ params.src1 = *node->src[1];
1902
+ params.dst = *node;
1903
+ params.dim = dim;
1904
+
1905
+ // Phase 1: Initialize CPU comparison context and copy source buffers (before ET kernel)
1906
+ ggml_et_cpu_compare_ctx cpu_cmp_ctx;
1907
+ bool cpu_comparison_active = false;
1908
+ if (concat_cpu_compare_config.enabled) {
1909
+ if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_CONCAT)) {
1910
+ cpu_comparison_active = true;
1911
+ } else {
1912
+ GGML_LOG_WARN("ET: Failed to initialize CPU comparison for CONCAT operation\n");
1913
+ }
1914
+ }
1915
+
1916
+ bool kernel_result = ggml_et_launch_kernel(dev_ctx, kernel_name, &params, sizeof(params), 0xFFFFFFFF);
1917
+
1918
+ // Phase 2: Execute CPU computation and compare with ET result (after ET kernel)
1919
+ if (cpu_comparison_active) {
1920
+ if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &concat_cpu_compare_config)) {
1921
+ GGML_LOG_WARN("ET: CPU comparison failed for CONCAT operation\n");
1922
+ }
1923
+ ggml_et_cpu_compare_free(&cpu_cmp_ctx);
1924
+ }
1925
+
1926
+ ET_PERF_END_EXT("CONCAT", kernel_name, node, "dim=%d", dim);
1927
+ return kernel_result;
1928
+ }
1929
+
1930
+ bool ggml_et_op_repeat(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
1931
+ ET_PERF_START();
1932
+
1933
+ if (!dev_ctx || !node) {
1934
+ GGML_LOG_ERROR("ET: Invalid parameters for REPEAT operation\n");
1935
+ return false;
1936
+ }
1937
+
1938
+ if (!node->src[0]) {
1939
+ GGML_LOG_ERROR("ET: REPEAT operation missing required input\n");
1940
+ return false;
1941
+ }
1942
+
1943
+ const char * kernel_name;
1944
+
1945
+ if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_F32) {
1946
+ // No-op REPEAT (every repeat factor is 1): the output is just a copy
1947
+ // of the input. Route to cont_f32, whose contiguous fast path handles
1948
+ // arbitrary sizes (including those rejected by repeat_f32's gate,
1949
+ // e.g. ne[0]=1).
1950
+ if (ggml_are_same_shape(node->src[0], node)) {
1951
+ kernel_name = "cont_f32";
1952
+ } else {
1953
+ kernel_name = "repeat_f32";
1954
+ }
1955
+
1956
+ } else {
1957
+ GGML_LOG_ERROR("ET: REPEAT operation with unsupported types: dst=%s src0=%s\n", ggml_type_name(node->type),
1958
+ ggml_type_name(node->src[0]->type));
1959
+ return false;
1960
+ }
1961
+
1962
+ // ggml_et_cont_params and ggml_et_repeat_params have identical layouts
1963
+ // (just src0 + dst), so the same payload works for either kernel.
1964
+ ggml_et_repeat_params params;
1965
+ params.src0 = *node->src[0];
1966
+ params.dst = *node;
1967
+
1968
+ // Phase 1: Initialize CPU comparison context and copy source buffers (before ET kernel)
1969
+ ggml_et_cpu_compare_ctx cpu_cmp_ctx;
1970
+ bool cpu_comparison_active = false;
1971
+ if (repeat_cpu_compare_config.enabled) {
1972
+ if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_REPEAT)) {
1973
+ cpu_comparison_active = true;
1974
+ } else {
1975
+ GGML_LOG_WARN("ET: Failed to initialize CPU comparison for REPEAT operation\n");
1976
+ }
1977
+ }
1978
+
1979
+ bool kernel_result = ggml_et_launch_kernel(dev_ctx, kernel_name, &params, sizeof(params), 0xFFFFFFFF);
1980
+
1981
+ // Phase 2: Execute CPU computation and compare with ET result (after ET kernel)
1982
+ if (cpu_comparison_active) {
1983
+ if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &repeat_cpu_compare_config)) {
1984
+ GGML_LOG_WARN("ET: CPU comparison failed for REPEAT operation\n");
1985
+ }
1986
+ ggml_et_cpu_compare_free(&cpu_cmp_ctx);
1987
+ }
1988
+
1989
+ ET_PERF_END("REPEAT", kernel_name, node);
1990
+ return kernel_result;
1991
+ }
1992
+
1993
+ bool ggml_et_op_ssm_conv(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
1994
+ ET_PERF_START();
1995
+
1996
+ if (!dev_ctx || !node || !node->src[0] || !node->src[1]) {
1997
+ GGML_LOG_ERROR("ET: Invalid parameters for SSM_CONV operation\n");
1998
+ return false;
1999
+ }
2000
+
2001
+ if (node->type != GGML_TYPE_F32 || node->src[0]->type != GGML_TYPE_F32 || node->src[1]->type != GGML_TYPE_F32) {
2002
+ GGML_LOG_ERROR("ET: SSM_CONV operation with unsupported types: dst=%s src0=%s src1=%s\n",
2003
+ ggml_type_name(node->type), ggml_type_name(node->src[0]->type),
2004
+ ggml_type_name(node->src[1]->type));
2005
+ return false;
2006
+ }
2007
+
2008
+ const char * kernel_name = "ssm_conv_f32";
2009
+
2010
+ ggml_et_ssm_conv_params params;
2011
+ params.src0 = *node->src[0];
2012
+ params.src1 = *node->src[1];
2013
+ params.dst = *node;
2014
+
2015
+ ggml_et_cpu_compare_ctx cpu_cmp_ctx;
2016
+ bool cpu_comparison_active = false;
2017
+ if (ssm_conv_cpu_compare_config.enabled) {
2018
+ if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_SSM_CONV)) {
2019
+ cpu_comparison_active = true;
2020
+ } else {
2021
+ GGML_LOG_WARN("ET: Failed to initialize CPU comparison for SSM_CONV operation\n");
2022
+ }
2023
+ }
2024
+
2025
+ bool kernel_result = ggml_et_launch_kernel(dev_ctx, kernel_name, &params, sizeof(params), 0xFFFFFFFF);
2026
+
2027
+ if (cpu_comparison_active) {
2028
+ if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &ssm_conv_cpu_compare_config)) {
2029
+ GGML_LOG_WARN("ET: CPU comparison failed for SSM_CONV operation\n");
2030
+ }
2031
+ ggml_et_cpu_compare_free(&cpu_cmp_ctx);
2032
+ }
2033
+
2034
+ ET_PERF_END("SSM_CONV", kernel_name, node);
2035
+ return kernel_result;
2036
+ }
2037
+
2038
+ bool ggml_et_op_ssm_scan(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
2039
+ ET_PERF_START();
2040
+
2041
+ if (!dev_ctx || !node) {
2042
+ GGML_LOG_ERROR("ET: Invalid parameters for SSM_SCAN operation\n");
2043
+ return false;
2044
+ }
2045
+
2046
+ for (int i = 0; i < 7; ++i) {
2047
+ if (!node->src[i]) {
2048
+ GGML_LOG_ERROR("ET: SSM_SCAN missing required input %d\n", i);
2049
+ return false;
2050
+ }
2051
+ }
2052
+
2053
+ if (node->type != GGML_TYPE_F32 || node->src[0]->type != GGML_TYPE_F32 || node->src[1]->type != GGML_TYPE_F32 ||
2054
+ node->src[2]->type != GGML_TYPE_F32 || node->src[3]->type != GGML_TYPE_F32 ||
2055
+ node->src[4]->type != GGML_TYPE_F32 || node->src[5]->type != GGML_TYPE_F32 ||
2056
+ node->src[6]->type != GGML_TYPE_I32) {
2057
+ GGML_LOG_ERROR("ET: SSM_SCAN operation with unsupported types\n");
2058
+ return false;
2059
+ }
2060
+
2061
+ ggml_et_ssm_scan_params params;
2062
+ params.src0 = *node->src[0];
2063
+ params.src1 = *node->src[1];
2064
+ params.src2 = *node->src[2];
2065
+ params.src3 = *node->src[3];
2066
+ params.src4 = *node->src[4];
2067
+ params.src5 = *node->src[5];
2068
+ params.src6 = *node->src[6];
2069
+ params.dst = *node;
2070
+ params.K = ggml_get_op_params_i32(node, 0);
2071
+
2072
+ bool kernel_result = ggml_et_launch_kernel(dev_ctx, "ssm_scan_f32", &params, sizeof(params), 0xFFFFFFFF);
2073
+
2074
+ ET_PERF_END("SSM_SCAN", "ssm_scan_f32", node);
2075
+ return kernel_result;
2076
+ }
2077
+
2078
+ bool ggml_et_op_rwkv_wkv6(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
2079
+ ET_PERF_START();
2080
+
2081
+ if (!dev_ctx || !node) {
2082
+ GGML_LOG_ERROR("ET: Invalid parameters for RWKV_WKV6 operation\n");
2083
+ return false;
2084
+ }
2085
+
2086
+ // Validate all 6 source tensors exist
2087
+ for (int i = 0; i <= 5; i++) {
2088
+ if (!node->src[i]) {
2089
+ GGML_LOG_ERROR("ET: RWKV_WKV6 operation missing src[%d]\n", i);
2090
+ return false;
2091
+ }
2092
+ }
2093
+
2094
+ if (node->type != GGML_TYPE_F32) {
2095
+ GGML_LOG_ERROR("ET: RWKV_WKV6 only supports F32, got %s\n", ggml_type_name(node->type));
2096
+ return false;
2097
+ }
2098
+
2099
+ const char * kernel_name = "rwkv_wkv6_f32";
2100
+
2101
+ const int64_t S = node->src[0]->ne[0]; // head_size
2102
+ const int64_t H = node->src[0]->ne[1]; // num heads
2103
+ const int64_t T = node->src[1]->ne[2]; // num tokens
2104
+ const int64_t n_seqs = node->src[5]->ne[1]; // num sequences
2105
+ const int64_t C = S * H;
2106
+
2107
+ ggml_et_rwkv_wkv6_params params;
2108
+ params.k = (float *) node->src[0]->data;
2109
+ params.v = (float *) node->src[1]->data;
2110
+ params.r = (float *) node->src[2]->data;
2111
+ params.tf = (float *) node->src[3]->data;
2112
+ params.td = (float *) node->src[4]->data;
2113
+ params.state_in = (float *) node->src[5]->data;
2114
+ params.dst = (float *) node->data;
2115
+ params.C = (int32_t) C;
2116
+ params.H = (int32_t) H;
2117
+ params.S = (int32_t) S;
2118
+ params.T = (int32_t) T;
2119
+ params.n_seqs = (int32_t) n_seqs;
2120
+
2121
+ // Phase 1: Initialize CPU comparison context
2122
+ ggml_et_cpu_compare_ctx cpu_cmp_ctx;
2123
+ bool cpu_comparison_active = false;
2124
+ if (rwkv_wkv6_cpu_compare_config.enabled) {
2125
+ if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_RWKV_WKV6)) {
2126
+ cpu_comparison_active = true;
2127
+ } else {
2128
+ GGML_LOG_WARN("ET: Failed to initialize CPU comparison for RWKV_WKV6 operation\n");
2129
+ }
2130
+ }
2131
+
2132
+ bool kernel_result = ggml_et_launch_kernel(dev_ctx, kernel_name, &params, sizeof(params), 0xFFFFFFFF);
2133
+
2134
+ // Phase 2: Execute CPU computation and compare
2135
+ if (cpu_comparison_active) {
2136
+ if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &rwkv_wkv6_cpu_compare_config)) {
2137
+ GGML_LOG_WARN("ET: CPU comparison failed for RWKV_WKV6 operation\n");
2138
+ }
2139
+ ggml_et_cpu_compare_free(&cpu_cmp_ctx);
2140
+ }
2141
+
2142
+ ET_PERF_END_EXT("RWKV_WKV6", kernel_name, node, "S=%d H=%d T=%d n_seqs=%d", (int) S, (int) H, (int) T,
2143
+ (int) n_seqs);
2144
+ return kernel_result;
2145
+ }
2146
+
2147
+ bool ggml_et_op_rwkv_wkv7(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
2148
+ ET_PERF_START();
2149
+
2150
+ if (!dev_ctx || !node) {
2151
+ GGML_LOG_ERROR("ET: Invalid parameters for RWKV_WKV7 operation\n");
2152
+ return false;
2153
+ }
2154
+
2155
+ // Validate all 7 source tensors exist
2156
+ for (int i = 0; i <= 6; i++) {
2157
+ if (!node->src[i]) {
2158
+ GGML_LOG_ERROR("ET: RWKV_WKV7 operation missing src[%d]\n", i);
2159
+ return false;
2160
+ }
2161
+ }
2162
+
2163
+ if (node->type != GGML_TYPE_F32) {
2164
+ GGML_LOG_ERROR("ET: RWKV_WKV7 only supports F32, got %s\n", ggml_type_name(node->type));
2165
+ return false;
2166
+ }
2167
+
2168
+ const char * kernel_name = "rwkv_wkv7_f32";
2169
+
2170
+ const int64_t S = node->src[2]->ne[0]; // head_size
2171
+ const int64_t H = node->src[2]->ne[1]; // num heads
2172
+ const int64_t T = node->src[1]->ne[2]; // num tokens
2173
+ const int64_t n_seqs = node->src[6]->ne[1]; // num sequences
2174
+ const int64_t C = S * H;
2175
+
2176
+ ggml_et_rwkv_wkv7_params params;
2177
+ params.r = (float *) node->src[0]->data;
2178
+ params.w = (float *) node->src[1]->data;
2179
+ params.k = (float *) node->src[2]->data;
2180
+ params.v = (float *) node->src[3]->data;
2181
+ params.a = (float *) node->src[4]->data;
2182
+ params.b = (float *) node->src[5]->data;
2183
+ params.state_in = (float *) node->src[6]->data;
2184
+ params.dst = (float *) node->data;
2185
+ params.C = (int32_t) C;
2186
+ params.H = (int32_t) H;
2187
+ params.S = (int32_t) S;
2188
+ params.T = (int32_t) T;
2189
+ params.n_seqs = (int32_t) n_seqs;
2190
+
2191
+ // Phase 1: Initialize CPU comparison context
2192
+ ggml_et_cpu_compare_ctx cpu_cmp_ctx;
2193
+ bool cpu_comparison_active = false;
2194
+ if (rwkv_wkv7_cpu_compare_config.enabled) {
2195
+ if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_RWKV_WKV7)) {
2196
+ cpu_comparison_active = true;
2197
+ } else {
2198
+ GGML_LOG_WARN("ET: Failed to initialize CPU comparison for RWKV_WKV7 operation\n");
2199
+ }
2200
+ }
2201
+
2202
+ bool kernel_result = ggml_et_launch_kernel(dev_ctx, kernel_name, &params, sizeof(params), 0xFFFFFFFF);
2203
+
2204
+ // Phase 2: Execute CPU computation and compare
2205
+ if (cpu_comparison_active) {
2206
+ if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &rwkv_wkv7_cpu_compare_config)) {
2207
+ GGML_LOG_WARN("ET: CPU comparison failed for RWKV_WKV7 operation\n");
2208
+ }
2209
+ ggml_et_cpu_compare_free(&cpu_cmp_ctx);
2210
+ }
2211
+
2212
+ ET_PERF_END_EXT("RWKV_WKV7", kernel_name, node, "S=%d H=%d T=%d n_seqs=%d", (int) S, (int) H, (int) T,
2213
+ (int) n_seqs);
2214
+ return kernel_result;
2215
+ }
2216
+
2217
+ static ggml_et_cpu_compare_config gated_delta_net_cpu_compare_config = {
2218
+ /* .enabled = */ false,
2219
+ /* .use_cpu_result = */ false,
2220
+ /* .log_differences = */ true,
2221
+ /* .tolerance = */ 1e-4f,
2222
+ /* .max_log_elements = */ 4096
2223
+ };
2224
+
2225
+ bool ggml_et_op_gated_delta_net(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
2226
+ ET_PERF_START();
2227
+
2228
+ if (!dev_ctx || !node) {
2229
+ GGML_LOG_ERROR("ET: Invalid parameters for GATED_DELTA_NET operation\n");
2230
+ return false;
2231
+ }
2232
+
2233
+ // Validate all 6 source tensors exist
2234
+ for (int i = 0; i <= 5; i++) {
2235
+ if (!node->src[i]) {
2236
+ GGML_LOG_ERROR("ET: GATED_DELTA_NET operation missing src[%d]\n", i);
2237
+ return false;
2238
+ }
2239
+ }
2240
+
2241
+ if (node->type != GGML_TYPE_F32) {
2242
+ GGML_LOG_ERROR("ET: GATED_DELTA_NET only supports F32, got %s\n", ggml_type_name(node->type));
2243
+ return false;
2244
+ }
2245
+
2246
+ const char * kernel_name = "gated_delta_net_f32";
2247
+
2248
+ const ggml_tensor * src_q = node->src[0];
2249
+ const ggml_tensor * src_k = node->src[1];
2250
+ const ggml_tensor * src_v = node->src[2];
2251
+ const ggml_tensor * src_g = node->src[3];
2252
+ const ggml_tensor * src_beta = node->src[4];
2253
+ const ggml_tensor * src_state = node->src[5];
2254
+
2255
+ const int64_t S_v = src_v->ne[0];
2256
+ const int64_t H = src_v->ne[1];
2257
+ const int64_t n_tokens = src_v->ne[2];
2258
+ const int64_t n_seqs = src_v->ne[3];
2259
+ const int64_t H_q = src_q->ne[1];
2260
+ const int64_t H_k = src_k->ne[1];
2261
+ const int64_t n_seqs_q = src_q->ne[3];
2262
+ const int64_t n_seqs_k = src_k->ne[3];
2263
+
2264
+ ggml_et_gated_delta_net_params params;
2265
+ params.q = *src_q;
2266
+ params.k = *src_k;
2267
+ params.v = *src_v;
2268
+ params.g = *src_g;
2269
+ params.beta = *src_beta;
2270
+ params.state_in = *src_state;
2271
+ params.dst = *node;
2272
+ params.S_v = (int32_t) S_v;
2273
+ params.H = (int32_t) H;
2274
+ params.H_q = (int32_t) H_q;
2275
+ params.H_k = (int32_t) H_k;
2276
+ params.n_tokens = (int32_t) n_tokens;
2277
+ params.n_seqs = (int32_t) n_seqs;
2278
+ params.n_seqs_q = (int32_t) n_seqs_q;
2279
+ params.n_seqs_k = (int32_t) n_seqs_k;
2280
+ params.kda = (src_g->ne[0] == S_v) ? 1 : 0;
2281
+ params.K = ggml_get_op_params_i32(node, 0);
2282
+ params.scale = 1.0f / sqrtf((float) S_v);
2283
+
2284
+ // CPU comparison for debugging
2285
+ ggml_et_cpu_compare_ctx cpu_cmp_ctx;
2286
+ bool cpu_comparison_active = false;
2287
+ if (gated_delta_net_cpu_compare_config.enabled) {
2288
+ if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_GATED_DELTA_NET)) {
2289
+ cpu_comparison_active = true;
2290
+ } else {
2291
+ GGML_LOG_WARN("ET: Failed to initialize CPU comparison for GATED_DELTA_NET operation\n");
2292
+ }
2293
+ }
2294
+
2295
+ bool kernel_result = ggml_et_launch_kernel(dev_ctx, kernel_name, &params, sizeof(params), 0xFFFFFFFF);
2296
+
2297
+ if (cpu_comparison_active) {
2298
+ if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &gated_delta_net_cpu_compare_config)) {
2299
+ GGML_LOG_WARN("ET: CPU comparison failed for GATED_DELTA_NET operation\n");
2300
+ }
2301
+ ggml_et_cpu_compare_free(&cpu_cmp_ctx);
2302
+ }
2303
+
2304
+ ET_PERF_END_EXT("GATED_DELTA_NET", kernel_name, node, "S_v=%d H=%d n_tokens=%d n_seqs=%d kda=%d", (int) S_v,
2305
+ (int) H, (int) n_tokens, (int) n_seqs, params.kda);
2306
+ return kernel_result;
2307
+ }
2308
+
2309
+ bool ggml_et_op_set_rows(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
2310
+ ET_PERF_START();
2311
+
2312
+ if (!dev_ctx || !node) {
2313
+ GGML_LOG_ERROR("ET: Invalid parameters for SET_ROWS operation\n");
2314
+ return false;
2315
+ }
2316
+
2317
+ if (!node->src[0] || !node->src[1] || !node->src[2]) {
2318
+ GGML_LOG_ERROR(
2319
+ "ET: SET_ROWS operation missing required inputs (needs src[0]=base, src[1]=indices, src[2]=data)\n");
2320
+ return false;
2321
+ }
2322
+
2323
+ const char * kernel_name;
2324
+
2325
+ // Support F32 data with I64 indices -> F32/F16 output (scatter operation)
2326
+ if (node->src[0]->type == GGML_TYPE_F32 && node->src[1]->type == GGML_TYPE_I64 &&
2327
+ (node->type == GGML_TYPE_F32 || node->type == GGML_TYPE_F16)) {
2328
+ if (node->type == GGML_TYPE_F32 || node->type == GGML_TYPE_F16) {
2329
+ kernel_name = "set_rows_f32";
2330
+ } else {
2331
+ GGML_LOG_ERROR("ET: SET_ROWS unsupported output type: %s\n", ggml_type_name(node->type));
2332
+ return false;
2333
+ }
2334
+
2335
+ } else {
2336
+ GGML_LOG_ERROR("ET: SET_ROWS operation with unsupported types: dst=%s src0=%s src1=%s\n",
2337
+ ggml_type_name(node->type), ggml_type_name(node->src[0]->type),
2338
+ ggml_type_name(node->src[1]->type));
2339
+ return false;
2340
+ }
2341
+
2342
+ // Validate contiguity requirements
2343
+ if (!ggml_is_contiguous_rows(node)) {
2344
+ GGML_LOG_ERROR("ET: SET_ROWS operation requires contiguous-rows destination tensor\n");
2345
+ return false;
2346
+ }
2347
+
2348
+ if (!ggml_is_contiguous_rows(node->src[0])) {
2349
+ GGML_LOG_ERROR("ET: SET_ROWS operation requires contiguous-rows source tensor\n");
2350
+ return false;
2351
+ }
2352
+
2353
+ if (!ggml_is_contiguous(node->src[1])) {
2354
+ GGML_LOG_ERROR("ET: SET_ROWS operation requires contiguous indices tensor\n");
2355
+ return false;
2356
+ }
2357
+
2358
+ // Validate dimension constraints from ggml implementation
2359
+ if (!(node->ne[0] == node->src[0]->ne[0] && // same number of columns
2360
+ node->ne[2] == node->src[0]->ne[2] && // same batch size
2361
+ node->ne[3] == node->src[0]->ne[3] && // same outer dimension
2362
+ node->src[0]->ne[1] == node->src[1]->ne[0] && // src rows = index count
2363
+ node->src[0]->ne[2] % node->src[1]->ne[1] == 0 && // batch constraint
2364
+ node->src[0]->ne[3] % node->src[1]->ne[2] == 0 && // outer constraint
2365
+ node->src[1]->ne[3] == 1)) { // indices constraint
2366
+ GGML_LOG_ERROR("ET: SET_ROWS operation dimension constraint failed\n");
2367
+ return false;
2368
+ }
2369
+
2370
+ ggml_et_set_rows_params params;
2371
+ params.src0 = *node->src[0]; // F32 source data tensor
2372
+ params.src1 = *node->src[1]; // I64 indices tensor
2373
+ params.dst = *node; // F32/F16 destination tensor
2374
+
2375
+ // Phase 1: Initialize CPU comparison context and copy source buffers (before ET kernel)
2376
+ ggml_et_cpu_compare_ctx cpu_cmp_ctx;
2377
+ bool cpu_comparison_active = false;
2378
+ if (set_rows_cpu_compare_config.enabled) {
2379
+ if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_SET_ROWS)) {
2380
+ cpu_comparison_active = true;
2381
+ } else {
2382
+ GGML_LOG_WARN("ET: Failed to initialize CPU comparison for SET_ROWS operation\n");
2383
+ }
2384
+ }
2385
+
2386
+ bool kernel_result = ggml_et_launch_kernel(dev_ctx, kernel_name, &params, sizeof(params), 0xFFFFFFFF);
2387
+
2388
+ // Phase 2: Execute CPU computation and compare with ET result (after ET kernel)
2389
+ if (cpu_comparison_active) {
2390
+ if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &set_rows_cpu_compare_config)) {
2391
+ GGML_LOG_WARN("ET: CPU comparison failed for SET_ROWS operation\n");
2392
+ }
2393
+ ggml_et_cpu_compare_free(&cpu_cmp_ctx);
2394
+ }
2395
+
2396
+ ET_PERF_END("SET_ROWS", kernel_name, node);
2397
+ return kernel_result;
2398
+ }
2399
+
2400
+ bool ggml_et_op_fill(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
2401
+ ET_PERF_START();
2402
+
2403
+ ggml_et_fill_params params;
2404
+ params.dst = *node;
2405
+ memcpy(&params.c, node->op_params, sizeof(float));
2406
+
2407
+ bool kernel_result = ggml_et_launch_kernel(dev_ctx, "fill_f32", &params, sizeof(params), 0xFFFFFFFF);
2408
+
2409
+ ET_PERF_END("FILL", "fill_f32", node);
2410
+ return kernel_result;
2411
+ }
2412
+
2413
+ bool ggml_et_op_diag(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
2414
+ ET_PERF_START();
2415
+
2416
+ if (!node->src[0]) {
2417
+ GGML_LOG_ERROR("ET: DIAG operation missing source tensor\n");
2418
+ return false;
2419
+ }
2420
+
2421
+ ggml_et_diag_params params;
2422
+ params.src0 = *node->src[0];
2423
+ params.dst = *node;
2424
+
2425
+ bool kernel_result = ggml_et_launch_kernel(dev_ctx, "diag_f32", &params, sizeof(params), 0xFFFFFFFF);
2426
+
2427
+ ET_PERF_END("DIAG", "diag_f32", node);
2428
+ return kernel_result;
2429
+ }
2430
+
2431
+ bool ggml_et_op_tri(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
2432
+ ET_PERF_START();
2433
+
2434
+ if (!node->src[0]) {
2435
+ GGML_LOG_ERROR("ET: TRI operation missing source tensor\n");
2436
+ return false;
2437
+ }
2438
+
2439
+ ggml_et_tri_params params;
2440
+ params.src0 = *node->src[0];
2441
+ params.dst = *node;
2442
+ memcpy(&params.tri_type, node->op_params, sizeof(int32_t));
2443
+
2444
+ bool kernel_result = ggml_et_launch_kernel(dev_ctx, "tri_f32", &params, sizeof(params), 0xFFFFFFFF);
2445
+
2446
+ ET_PERF_END("TRI", "tri_f32", node);
2447
+ return kernel_result;
2448
+ }
2449
+
2450
+ bool ggml_et_op_solve_tri(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
2451
+ ET_PERF_START();
2452
+
2453
+ if (!node->src[0] || !node->src[1]) {
2454
+ GGML_LOG_ERROR("ET: SOLVE_TRI operation missing source tensor(s)\n");
2455
+ return false;
2456
+ }
2457
+
2458
+ ggml_et_solve_tri_params params;
2459
+ params.src0 = *node->src[0]; // A (lower-triangular)
2460
+ params.src1 = *node->src[1]; // B (RHS)
2461
+ params.dst = *node; // X (solution)
2462
+
2463
+ bool kernel_result = ggml_et_launch_kernel(dev_ctx, "solve_tri_f32", &params, sizeof(params), 0xFFFFFFFF);
2464
+
2465
+ ET_PERF_END("SOLVE_TRI", "solve_tri_f32", node);
2466
+ return kernel_result;
2467
+ }
2468
+
2469
+ bool ggml_et_op_set(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
2470
+ ET_PERF_START();
2471
+
2472
+ if (!node->src[0] || !node->src[1]) {
2473
+ GGML_LOG_ERROR("ET: SET operation missing source tensor(s)\n");
2474
+ return false;
2475
+ }
2476
+
2477
+ const bool inplace = (bool) ((const int32_t *) node->op_params)[4];
2478
+ const size_t offset = ((const int32_t *) node->op_params)[3];
2479
+ const size_t nb1 = ((const int32_t *) node->op_params)[0];
2480
+ const size_t nb2 = ((const int32_t *) node->op_params)[1];
2481
+ const size_t nb3 = ((const int32_t *) node->op_params)[2];
2482
+
2483
+ if (!inplace) {
2484
+ GGML_LOG_ERROR("ET: SET only supports inplace (inplace=%d)\n", inplace);
2485
+ return false;
2486
+ }
2487
+
2488
+ if (node->type != GGML_TYPE_F32 || node->src[0]->type != GGML_TYPE_F32 || node->src[1]->type != GGML_TYPE_F32) {
2489
+ GGML_LOG_ERROR("ET: SET only supports F32 (dst=%s src0=%s src1=%s)\n", ggml_type_name(node->type),
2490
+ ggml_type_name(node->src[0]->type), ggml_type_name(node->src[1]->type));
2491
+ return false;
2492
+ }
2493
+
2494
+ if (!ggml_are_same_shape(node, node->src[0])) {
2495
+ GGML_LOG_ERROR("ET: SET requires same-shape src0 and dst\n");
2496
+ return false;
2497
+ }
2498
+
2499
+ if (!ggml_is_contiguous(node) || !ggml_is_contiguous(node->src[0]) || !ggml_is_contiguous(node->src[1])) {
2500
+ GGML_LOG_ERROR("ET: SET requires contiguous dst, src0, and src1\n");
2501
+ return false;
2502
+ }
2503
+
2504
+ ggml_et_set_params params;
2505
+ params.src1 = *node->src[1];
2506
+ params.dst = *node;
2507
+ params.nb1 = (int32_t) nb1;
2508
+ params.nb2 = (int32_t) nb2;
2509
+ params.nb3 = (int32_t) nb3;
2510
+ params.offset = (int32_t) offset;
2511
+
2512
+ bool kernel_result = ggml_et_launch_kernel(dev_ctx, "set_f32", &params, sizeof(params), 0xFFFFFFFF);
2513
+
2514
+ ET_PERF_END("SET", "set_f32", node);
2515
+ return kernel_result;
2516
+ }
2517
+
2518
+ bool ggml_et_op_pad(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
2519
+ ET_PERF_START();
2520
+
2521
+ if (!node->src[0]) {
2522
+ GGML_LOG_ERROR("ET: PAD operation missing source tensor\n");
2523
+ return false;
2524
+ }
2525
+
2526
+ if (node->type != GGML_TYPE_F32 || node->src[0]->type != GGML_TYPE_F32) {
2527
+ GGML_LOG_ERROR("ET: PAD only supports F32 (src=%s dst=%s)\n", ggml_type_name(node->src[0]->type),
2528
+ ggml_type_name(node->type));
2529
+ return false;
2530
+ }
2531
+
2532
+ if (!ggml_is_contiguous(node)) {
2533
+ GGML_LOG_ERROR("ET: PAD requires contiguous output tensor\n");
2534
+ return false;
2535
+ }
2536
+
2537
+ if (node->src[0]->nb[0] != sizeof(float)) {
2538
+ GGML_LOG_ERROR("ET: PAD requires element-contiguous src dim0 (nb[0]=%zu)\n", (size_t) node->src[0]->nb[0]);
2539
+ return false;
2540
+ }
2541
+
2542
+ // Extract padding parameters from op_params
2543
+ const int32_t * op_params = (const int32_t *) node->op_params;
2544
+
2545
+ ggml_et_pad_params params;
2546
+ params.src0 = *node->src[0];
2547
+ params.dst = *node;
2548
+ params.lp[0] = op_params[0];
2549
+ params.rp[0] = op_params[1];
2550
+ params.lp[1] = op_params[2];
2551
+ params.rp[1] = op_params[3];
2552
+ params.lp[2] = op_params[4];
2553
+ params.rp[2] = op_params[5];
2554
+ params.lp[3] = op_params[6];
2555
+ params.rp[3] = op_params[7];
2556
+
2557
+ // v1: no dim0 padding
2558
+ if (params.lp[0] != 0 || params.rp[0] != 0) {
2559
+ GGML_LOG_ERROR("ET: PAD dim0 padding not supported (lp0=%d rp0=%d)\n", params.lp[0], params.rp[0]);
2560
+ return false;
2561
+ }
2562
+
2563
+ ggml_et_cpu_compare_ctx cpu_cmp_ctx;
2564
+ bool cpu_comparison_active = false;
2565
+ if (pad_cpu_compare_config.enabled) {
2566
+ if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_PAD)) {
2567
+ cpu_comparison_active = true;
2568
+ } else {
2569
+ GGML_LOG_WARN("ET: Failed to initialize CPU comparison for PAD operation\n");
2570
+ }
2571
+ }
2572
+
2573
+ bool kernel_result = ggml_et_launch_kernel(dev_ctx, "pad_f32", &params, sizeof(params), 0xFFFFFFFF);
2574
+
2575
+ if (cpu_comparison_active) {
2576
+ if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &pad_cpu_compare_config)) {
2577
+ GGML_LOG_WARN("ET: CPU comparison failed for PAD operation\n");
2578
+ }
2579
+ ggml_et_cpu_compare_free(&cpu_cmp_ctx);
2580
+ }
2581
+
2582
+ ET_PERF_END("PAD", "pad_f32", node);
2583
+ return kernel_result;
2584
+ }