elelem-ggml 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (1367) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE.txt +21 -0
  3. data/exe/elelem-ggml +8 -0
  4. data/ext/elelem/ggml/elelem.c +466 -0
  5. data/ext/elelem/ggml/extconf.rb +52 -0
  6. data/lib/elelem/ggml/chat_ml.rb +23 -0
  7. data/lib/elelem/ggml/gguf.rb +82 -0
  8. data/lib/elelem/ggml/provider.rb +63 -0
  9. data/lib/elelem/ggml/tokenizer.rb +95 -0
  10. data/lib/elelem/ggml/version.rb +7 -0
  11. data/lib/elelem/ggml.rb +37 -0
  12. data/vendor/ggml/AUTHORS +335 -0
  13. data/vendor/ggml/CMakeLists.txt +499 -0
  14. data/vendor/ggml/CONTRIBUTING.md +3 -0
  15. data/vendor/ggml/LICENSE +21 -0
  16. data/vendor/ggml/README.md +49 -0
  17. data/vendor/ggml/cmake/FindNCCL.cmake +36 -0
  18. data/vendor/ggml/cmake/GitVars.cmake +22 -0
  19. data/vendor/ggml/cmake/common.cmake +50 -0
  20. data/vendor/ggml/cmake/ggml-config.cmake.in +216 -0
  21. data/vendor/ggml/ggml.pc.in +10 -0
  22. data/vendor/ggml/include/ggml-alloc.h +86 -0
  23. data/vendor/ggml/include/ggml-backend.h +437 -0
  24. data/vendor/ggml/include/ggml-blas.h +25 -0
  25. data/vendor/ggml/include/ggml-cann.h +123 -0
  26. data/vendor/ggml/include/ggml-cpp.h +39 -0
  27. data/vendor/ggml/include/ggml-cpu.h +152 -0
  28. data/vendor/ggml/include/ggml-cuda.h +47 -0
  29. data/vendor/ggml/include/ggml-et.h +28 -0
  30. data/vendor/ggml/include/ggml-hexagon.h +19 -0
  31. data/vendor/ggml/include/ggml-metal.h +61 -0
  32. data/vendor/ggml/include/ggml-opencl.h +26 -0
  33. data/vendor/ggml/include/ggml-openvino.h +37 -0
  34. data/vendor/ggml/include/ggml-opt.h +256 -0
  35. data/vendor/ggml/include/ggml-rpc.h +35 -0
  36. data/vendor/ggml/include/ggml-sycl.h +57 -0
  37. data/vendor/ggml/include/ggml-virtgpu.h +14 -0
  38. data/vendor/ggml/include/ggml-vulkan.h +29 -0
  39. data/vendor/ggml/include/ggml-webgpu.h +19 -0
  40. data/vendor/ggml/include/ggml-zdnn.h +17 -0
  41. data/vendor/ggml/include/ggml-zendnn.h +22 -0
  42. data/vendor/ggml/include/ggml.h +2963 -0
  43. data/vendor/ggml/include/gguf.h +211 -0
  44. data/vendor/ggml/requirements.txt +12 -0
  45. data/vendor/ggml/scripts/gen-authors.sh +9 -0
  46. data/vendor/ggml/scripts/make-release-checks.sh +96 -0
  47. data/vendor/ggml/scripts/make-release-desc.sh +69 -0
  48. data/vendor/ggml/scripts/make-release-summary.txt +33 -0
  49. data/vendor/ggml/scripts/release.sh +195 -0
  50. data/vendor/ggml/scripts/sync-llama-am.sh +167 -0
  51. data/vendor/ggml/scripts/sync-llama.last +1 -0
  52. data/vendor/ggml/scripts/sync-llama.sh +21 -0
  53. data/vendor/ggml/scripts/sync-whisper-am.sh +138 -0
  54. data/vendor/ggml/scripts/sync-whisper.last +1 -0
  55. data/vendor/ggml/scripts/sync-whisper.sh +17 -0
  56. data/vendor/ggml/src/CMakeLists.txt +634 -0
  57. data/vendor/ggml/src/ggml-alloc.c +1249 -0
  58. data/vendor/ggml/src/ggml-backend-dl.cpp +48 -0
  59. data/vendor/ggml/src/ggml-backend-dl.h +45 -0
  60. data/vendor/ggml/src/ggml-backend-impl.h +291 -0
  61. data/vendor/ggml/src/ggml-backend-meta.cpp +2511 -0
  62. data/vendor/ggml/src/ggml-backend-reg.cpp +605 -0
  63. data/vendor/ggml/src/ggml-backend.cpp +2517 -0
  64. data/vendor/ggml/src/ggml-blas/CMakeLists.txt +101 -0
  65. data/vendor/ggml/src/ggml-blas/ggml-blas.cpp +530 -0
  66. data/vendor/ggml/src/ggml-cann/CMakeLists.txt +89 -0
  67. data/vendor/ggml/src/ggml-cann/acl_tensor.cpp +195 -0
  68. data/vendor/ggml/src/ggml-cann/acl_tensor.h +349 -0
  69. data/vendor/ggml/src/ggml-cann/aclnn_ops.cpp +4479 -0
  70. data/vendor/ggml/src/ggml-cann/aclnn_ops.h +1191 -0
  71. data/vendor/ggml/src/ggml-cann/common.h +651 -0
  72. data/vendor/ggml/src/ggml-cann/ggml-cann.cpp +3070 -0
  73. data/vendor/ggml/src/ggml-common.h +1911 -0
  74. data/vendor/ggml/src/ggml-cpu/CMakeLists.txt +685 -0
  75. data/vendor/ggml/src/ggml-cpu/amx/amx.cpp +249 -0
  76. data/vendor/ggml/src/ggml-cpu/amx/amx.h +8 -0
  77. data/vendor/ggml/src/ggml-cpu/amx/common.h +115 -0
  78. data/vendor/ggml/src/ggml-cpu/amx/mmq.cpp +2511 -0
  79. data/vendor/ggml/src/ggml-cpu/amx/mmq.h +10 -0
  80. data/vendor/ggml/src/ggml-cpu/arch/arm/cpu-feats.cpp +41 -0
  81. data/vendor/ggml/src/ggml-cpu/arch/arm/quants.c +4319 -0
  82. data/vendor/ggml/src/ggml-cpu/arch/arm/repack.cpp +5156 -0
  83. data/vendor/ggml/src/ggml-cpu/arch/loongarch/quants.c +2309 -0
  84. data/vendor/ggml/src/ggml-cpu/arch/powerpc/cpu-feats.cpp +82 -0
  85. data/vendor/ggml/src/ggml-cpu/arch/powerpc/quants.c +2304 -0
  86. data/vendor/ggml/src/ggml-cpu/arch/riscv/cpu-feats.cpp +38 -0
  87. data/vendor/ggml/src/ggml-cpu/arch/riscv/quants.c +6596 -0
  88. data/vendor/ggml/src/ggml-cpu/arch/riscv/repack.cpp +1703 -0
  89. data/vendor/ggml/src/ggml-cpu/arch/s390/cpu-feats.cpp +50 -0
  90. data/vendor/ggml/src/ggml-cpu/arch/s390/quants.c +1465 -0
  91. data/vendor/ggml/src/ggml-cpu/arch/wasm/quants.c +1292 -0
  92. data/vendor/ggml/src/ggml-cpu/arch/x86/cpu-feats.cpp +327 -0
  93. data/vendor/ggml/src/ggml-cpu/arch/x86/quants.c +4108 -0
  94. data/vendor/ggml/src/ggml-cpu/arch/x86/repack.cpp +6407 -0
  95. data/vendor/ggml/src/ggml-cpu/arch-fallback.h +353 -0
  96. data/vendor/ggml/src/ggml-cpu/binary-ops.cpp +154 -0
  97. data/vendor/ggml/src/ggml-cpu/binary-ops.h +16 -0
  98. data/vendor/ggml/src/ggml-cpu/cmake/FindSIMD.cmake +100 -0
  99. data/vendor/ggml/src/ggml-cpu/cmake/FindSMTIME.cmake +32 -0
  100. data/vendor/ggml/src/ggml-cpu/common.h +95 -0
  101. data/vendor/ggml/src/ggml-cpu/ggml-cpu-impl.h +539 -0
  102. data/vendor/ggml/src/ggml-cpu/ggml-cpu.c +3936 -0
  103. data/vendor/ggml/src/ggml-cpu/ggml-cpu.cpp +712 -0
  104. data/vendor/ggml/src/ggml-cpu/hbm.cpp +55 -0
  105. data/vendor/ggml/src/ggml-cpu/hbm.h +8 -0
  106. data/vendor/ggml/src/ggml-cpu/iqp.cpp +1253 -0
  107. data/vendor/ggml/src/ggml-cpu/iqp.h +39 -0
  108. data/vendor/ggml/src/ggml-cpu/kleidiai/CMakeLists.txt +14 -0
  109. data/vendor/ggml/src/ggml-cpu/kleidiai/kernels.cpp +1101 -0
  110. data/vendor/ggml/src/ggml-cpu/kleidiai/kernels.h +101 -0
  111. data/vendor/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp +1919 -0
  112. data/vendor/ggml/src/ggml-cpu/kleidiai/kleidiai.h +17 -0
  113. data/vendor/ggml/src/ggml-cpu/llamafile/sgemm.cpp +4164 -0
  114. data/vendor/ggml/src/ggml-cpu/llamafile/sgemm.h +25 -0
  115. data/vendor/ggml/src/ggml-cpu/ops.cpp +12164 -0
  116. data/vendor/ggml/src/ggml-cpu/ops.h +124 -0
  117. data/vendor/ggml/src/ggml-cpu/quants.c +1339 -0
  118. data/vendor/ggml/src/ggml-cpu/quants.h +106 -0
  119. data/vendor/ggml/src/ggml-cpu/repack.cpp +4836 -0
  120. data/vendor/ggml/src/ggml-cpu/repack.h +245 -0
  121. data/vendor/ggml/src/ggml-cpu/simd-gemm.h +226 -0
  122. data/vendor/ggml/src/ggml-cpu/simd-mappings.h +1319 -0
  123. data/vendor/ggml/src/ggml-cpu/spacemit/ime.cpp +1742 -0
  124. data/vendor/ggml/src/ggml-cpu/spacemit/ime.h +21 -0
  125. data/vendor/ggml/src/ggml-cpu/spacemit/ime1_kernels.cpp +1027 -0
  126. data/vendor/ggml/src/ggml-cpu/spacemit/ime2_kernels.cpp +5768 -0
  127. data/vendor/ggml/src/ggml-cpu/spacemit/ime_env.cpp +320 -0
  128. data/vendor/ggml/src/ggml-cpu/spacemit/ime_env.h +55 -0
  129. data/vendor/ggml/src/ggml-cpu/spacemit/ime_kernels.h +189 -0
  130. data/vendor/ggml/src/ggml-cpu/spacemit/repack.cpp +1795 -0
  131. data/vendor/ggml/src/ggml-cpu/spacemit/repack.h +14 -0
  132. data/vendor/ggml/src/ggml-cpu/spacemit/rvv_kernels.cpp +3178 -0
  133. data/vendor/ggml/src/ggml-cpu/spacemit/rvv_kernels.h +95 -0
  134. data/vendor/ggml/src/ggml-cpu/spacemit/spine_barrier.h +34 -0
  135. data/vendor/ggml/src/ggml-cpu/spacemit/spine_mem_pool.cpp +760 -0
  136. data/vendor/ggml/src/ggml-cpu/spacemit/spine_mem_pool.h +32 -0
  137. data/vendor/ggml/src/ggml-cpu/spacemit/spine_tcm.h +409 -0
  138. data/vendor/ggml/src/ggml-cpu/traits.cpp +36 -0
  139. data/vendor/ggml/src/ggml-cpu/traits.h +38 -0
  140. data/vendor/ggml/src/ggml-cpu/unary-ops.cpp +337 -0
  141. data/vendor/ggml/src/ggml-cpu/unary-ops.h +35 -0
  142. data/vendor/ggml/src/ggml-cpu/vec.cpp +613 -0
  143. data/vendor/ggml/src/ggml-cpu/vec.h +1570 -0
  144. data/vendor/ggml/src/ggml-cuda/CMakeLists.txt +267 -0
  145. data/vendor/ggml/src/ggml-cuda/acc.cu +61 -0
  146. data/vendor/ggml/src/ggml-cuda/acc.cuh +5 -0
  147. data/vendor/ggml/src/ggml-cuda/add-id.cu +58 -0
  148. data/vendor/ggml/src/ggml-cuda/add-id.cuh +3 -0
  149. data/vendor/ggml/src/ggml-cuda/allreduce.cu +971 -0
  150. data/vendor/ggml/src/ggml-cuda/allreduce.cuh +29 -0
  151. data/vendor/ggml/src/ggml-cuda/arange.cu +34 -0
  152. data/vendor/ggml/src/ggml-cuda/arange.cuh +5 -0
  153. data/vendor/ggml/src/ggml-cuda/argmax.cu +91 -0
  154. data/vendor/ggml/src/ggml-cuda/argmax.cuh +3 -0
  155. data/vendor/ggml/src/ggml-cuda/argsort.cu +292 -0
  156. data/vendor/ggml/src/ggml-cuda/argsort.cuh +20 -0
  157. data/vendor/ggml/src/ggml-cuda/binbcast.cu +574 -0
  158. data/vendor/ggml/src/ggml-cuda/binbcast.cuh +12 -0
  159. data/vendor/ggml/src/ggml-cuda/clamp.cu +45 -0
  160. data/vendor/ggml/src/ggml-cuda/clamp.cuh +5 -0
  161. data/vendor/ggml/src/ggml-cuda/col2im-1d.cu +81 -0
  162. data/vendor/ggml/src/ggml-cuda/col2im-1d.cuh +3 -0
  163. data/vendor/ggml/src/ggml-cuda/common.cuh +1678 -0
  164. data/vendor/ggml/src/ggml-cuda/concat.cu +242 -0
  165. data/vendor/ggml/src/ggml-cuda/concat.cuh +5 -0
  166. data/vendor/ggml/src/ggml-cuda/conv-transpose-1d.cu +88 -0
  167. data/vendor/ggml/src/ggml-cuda/conv-transpose-1d.cuh +5 -0
  168. data/vendor/ggml/src/ggml-cuda/conv2d-dw.cu +161 -0
  169. data/vendor/ggml/src/ggml-cuda/conv2d-dw.cuh +5 -0
  170. data/vendor/ggml/src/ggml-cuda/conv2d-transpose.cu +115 -0
  171. data/vendor/ggml/src/ggml-cuda/conv2d-transpose.cuh +5 -0
  172. data/vendor/ggml/src/ggml-cuda/conv2d.cu +167 -0
  173. data/vendor/ggml/src/ggml-cuda/conv2d.cuh +5 -0
  174. data/vendor/ggml/src/ggml-cuda/convert.cu +705 -0
  175. data/vendor/ggml/src/ggml-cuda/convert.cuh +66 -0
  176. data/vendor/ggml/src/ggml-cuda/count-equal.cu +64 -0
  177. data/vendor/ggml/src/ggml-cuda/count-equal.cuh +5 -0
  178. data/vendor/ggml/src/ggml-cuda/cp-async.cuh +57 -0
  179. data/vendor/ggml/src/ggml-cuda/cpy-utils.cuh +217 -0
  180. data/vendor/ggml/src/ggml-cuda/cpy.cu +617 -0
  181. data/vendor/ggml/src/ggml-cuda/cpy.cuh +7 -0
  182. data/vendor/ggml/src/ggml-cuda/cross-entropy-loss.cu +177 -0
  183. data/vendor/ggml/src/ggml-cuda/cross-entropy-loss.cuh +7 -0
  184. data/vendor/ggml/src/ggml-cuda/cumsum.cu +307 -0
  185. data/vendor/ggml/src/ggml-cuda/cumsum.cuh +5 -0
  186. data/vendor/ggml/src/ggml-cuda/dequantize.cuh +452 -0
  187. data/vendor/ggml/src/ggml-cuda/diag.cu +77 -0
  188. data/vendor/ggml/src/ggml-cuda/diag.cuh +5 -0
  189. data/vendor/ggml/src/ggml-cuda/diagmask.cu +40 -0
  190. data/vendor/ggml/src/ggml-cuda/diagmask.cuh +5 -0
  191. data/vendor/ggml/src/ggml-cuda/dsv4-hc.cu +294 -0
  192. data/vendor/ggml/src/ggml-cuda/dsv4-hc.cuh +6 -0
  193. data/vendor/ggml/src/ggml-cuda/fattn-common.cuh +1289 -0
  194. data/vendor/ggml/src/ggml-cuda/fattn-mma-f16.cuh +2136 -0
  195. data/vendor/ggml/src/ggml-cuda/fattn-swizzle.cuh +126 -0
  196. data/vendor/ggml/src/ggml-cuda/fattn-tile.cu +60 -0
  197. data/vendor/ggml/src/ggml-cuda/fattn-tile.cuh +1355 -0
  198. data/vendor/ggml/src/ggml-cuda/fattn-vec.cuh +611 -0
  199. data/vendor/ggml/src/ggml-cuda/fattn.cu +722 -0
  200. data/vendor/ggml/src/ggml-cuda/fattn.cuh +7 -0
  201. data/vendor/ggml/src/ggml-cuda/fill.cu +37 -0
  202. data/vendor/ggml/src/ggml-cuda/fill.cuh +3 -0
  203. data/vendor/ggml/src/ggml-cuda/fwht.cu +101 -0
  204. data/vendor/ggml/src/ggml-cuda/fwht.cuh +4 -0
  205. data/vendor/ggml/src/ggml-cuda/gated_delta_net.cu +327 -0
  206. data/vendor/ggml/src/ggml-cuda/gated_delta_net.cuh +14 -0
  207. data/vendor/ggml/src/ggml-cuda/getrows.cu +490 -0
  208. data/vendor/ggml/src/ggml-cuda/getrows.cuh +15 -0
  209. data/vendor/ggml/src/ggml-cuda/ggml-cuda.cu +5781 -0
  210. data/vendor/ggml/src/ggml-cuda/gla.cu +93 -0
  211. data/vendor/ggml/src/ggml-cuda/gla.cuh +3 -0
  212. data/vendor/ggml/src/ggml-cuda/im2col.cu +267 -0
  213. data/vendor/ggml/src/ggml-cuda/im2col.cuh +6 -0
  214. data/vendor/ggml/src/ggml-cuda/lightning-indexer.cu +588 -0
  215. data/vendor/ggml/src/ggml-cuda/lightning-indexer.cuh +4 -0
  216. data/vendor/ggml/src/ggml-cuda/mean.cu +77 -0
  217. data/vendor/ggml/src/ggml-cuda/mean.cuh +3 -0
  218. data/vendor/ggml/src/ggml-cuda/mma.cuh +1456 -0
  219. data/vendor/ggml/src/ggml-cuda/mmf.cu +191 -0
  220. data/vendor/ggml/src/ggml-cuda/mmf.cuh +908 -0
  221. data/vendor/ggml/src/ggml-cuda/mmid.cu +180 -0
  222. data/vendor/ggml/src/ggml-cuda/mmid.cuh +5 -0
  223. data/vendor/ggml/src/ggml-cuda/mmq-config-ampere.cuh +383 -0
  224. data/vendor/ggml/src/ggml-cuda/mmq-config-blackwell.cuh +37 -0
  225. data/vendor/ggml/src/ggml-cuda/mmq-config-cdna.cuh +185 -0
  226. data/vendor/ggml/src/ggml-cuda/mmq-config-pascal-dp4a.cuh +273 -0
  227. data/vendor/ggml/src/ggml-cuda/mmq-config-pascal-older.cuh +273 -0
  228. data/vendor/ggml/src/ggml-cuda/mmq-config-rdna2.cuh +273 -0
  229. data/vendor/ggml/src/ggml-cuda/mmq-config-rdna3-5.cuh +290 -0
  230. data/vendor/ggml/src/ggml-cuda/mmq-config-rdna3.cuh +274 -0
  231. data/vendor/ggml/src/ggml-cuda/mmq-config-rdna4.cuh +290 -0
  232. data/vendor/ggml/src/ggml-cuda/mmq-load-tiles.cuh +1768 -0
  233. data/vendor/ggml/src/ggml-cuda/mmq-vec-dot.cuh +1240 -0
  234. data/vendor/ggml/src/ggml-cuda/mmq.cu +386 -0
  235. data/vendor/ggml/src/ggml-cuda/mmq.cuh +1598 -0
  236. data/vendor/ggml/src/ggml-cuda/mmvf.cu +875 -0
  237. data/vendor/ggml/src/ggml-cuda/mmvf.cuh +14 -0
  238. data/vendor/ggml/src/ggml-cuda/mmvq.cu +1504 -0
  239. data/vendor/ggml/src/ggml-cuda/mmvq.cuh +18 -0
  240. data/vendor/ggml/src/ggml-cuda/moe-weighted-reduction.cu +65 -0
  241. data/vendor/ggml/src/ggml-cuda/moe-weighted-reduction.cuh +7 -0
  242. data/vendor/ggml/src/ggml-cuda/norm.cu +698 -0
  243. data/vendor/ggml/src/ggml-cuda/norm.cuh +18 -0
  244. data/vendor/ggml/src/ggml-cuda/opt-step-adamw.cu +78 -0
  245. data/vendor/ggml/src/ggml-cuda/opt-step-adamw.cuh +5 -0
  246. data/vendor/ggml/src/ggml-cuda/opt-step-sgd.cu +49 -0
  247. data/vendor/ggml/src/ggml-cuda/opt-step-sgd.cuh +5 -0
  248. data/vendor/ggml/src/ggml-cuda/out-prod.cu +125 -0
  249. data/vendor/ggml/src/ggml-cuda/out-prod.cuh +3 -0
  250. data/vendor/ggml/src/ggml-cuda/pad.cu +106 -0
  251. data/vendor/ggml/src/ggml-cuda/pad.cuh +5 -0
  252. data/vendor/ggml/src/ggml-cuda/pad_reflect_1d.cu +91 -0
  253. data/vendor/ggml/src/ggml-cuda/pad_reflect_1d.cuh +5 -0
  254. data/vendor/ggml/src/ggml-cuda/pool1d.cu +85 -0
  255. data/vendor/ggml/src/ggml-cuda/pool1d.cuh +5 -0
  256. data/vendor/ggml/src/ggml-cuda/pool2d.cu +94 -0
  257. data/vendor/ggml/src/ggml-cuda/pool2d.cuh +5 -0
  258. data/vendor/ggml/src/ggml-cuda/quantize.cu +697 -0
  259. data/vendor/ggml/src/ggml-cuda/quantize.cuh +70 -0
  260. data/vendor/ggml/src/ggml-cuda/reduce_rows.cuh +43 -0
  261. data/vendor/ggml/src/ggml-cuda/roll.cu +67 -0
  262. data/vendor/ggml/src/ggml-cuda/roll.cuh +5 -0
  263. data/vendor/ggml/src/ggml-cuda/rope.cu +941 -0
  264. data/vendor/ggml/src/ggml-cuda/rope.cuh +11 -0
  265. data/vendor/ggml/src/ggml-cuda/scale.cu +37 -0
  266. data/vendor/ggml/src/ggml-cuda/scale.cuh +5 -0
  267. data/vendor/ggml/src/ggml-cuda/set-rows.cu +398 -0
  268. data/vendor/ggml/src/ggml-cuda/set-rows.cuh +7 -0
  269. data/vendor/ggml/src/ggml-cuda/set.cu +39 -0
  270. data/vendor/ggml/src/ggml-cuda/set.cuh +7 -0
  271. data/vendor/ggml/src/ggml-cuda/snake.cu +72 -0
  272. data/vendor/ggml/src/ggml-cuda/snake.cuh +8 -0
  273. data/vendor/ggml/src/ggml-cuda/softcap.cu +37 -0
  274. data/vendor/ggml/src/ggml-cuda/softcap.cuh +5 -0
  275. data/vendor/ggml/src/ggml-cuda/softmax.cu +480 -0
  276. data/vendor/ggml/src/ggml-cuda/softmax.cuh +7 -0
  277. data/vendor/ggml/src/ggml-cuda/solve_tri.cu +273 -0
  278. data/vendor/ggml/src/ggml-cuda/solve_tri.cuh +3 -0
  279. data/vendor/ggml/src/ggml-cuda/ssm-conv.cu +206 -0
  280. data/vendor/ggml/src/ggml-cuda/ssm-conv.cuh +3 -0
  281. data/vendor/ggml/src/ggml-cuda/ssm-scan.cu +859 -0
  282. data/vendor/ggml/src/ggml-cuda/ssm-scan.cuh +3 -0
  283. data/vendor/ggml/src/ggml-cuda/sum.cu +41 -0
  284. data/vendor/ggml/src/ggml-cuda/sum.cuh +5 -0
  285. data/vendor/ggml/src/ggml-cuda/sumrows.cu +47 -0
  286. data/vendor/ggml/src/ggml-cuda/sumrows.cuh +4 -0
  287. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_16.cu +6 -0
  288. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_32.cu +6 -0
  289. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_8.cu +12 -0
  290. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_1.cu +10 -0
  291. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_2.cu +11 -0
  292. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_4.cu +12 -0
  293. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_16.cu +6 -0
  294. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_32.cu +6 -0
  295. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_4.cu +12 -0
  296. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_8.cu +12 -0
  297. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_1.cu +10 -0
  298. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_2.cu +11 -0
  299. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_16.cu +6 -0
  300. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_2.cu +11 -0
  301. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_4.cu +12 -0
  302. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_8.cu +12 -0
  303. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_64-ncols2_1.cu +10 -0
  304. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_1.cu +10 -0
  305. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_2.cu +11 -0
  306. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_4.cu +12 -0
  307. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_8.cu +12 -0
  308. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq112-dv112.cu +5 -0
  309. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq128-dv128.cu +5 -0
  310. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq192-dv128.cu +5 -0
  311. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq256-dv256.cu +5 -0
  312. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq320-dv256.cu +5 -0
  313. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq40-dv40.cu +5 -0
  314. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq512-dv512.cu +5 -0
  315. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq576-dv512.cu +5 -0
  316. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq64-dv64.cu +5 -0
  317. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq72-dv72.cu +5 -0
  318. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq80-dv80.cu +5 -0
  319. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq96-dv96.cu +5 -0
  320. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-bf16.cu +7 -0
  321. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-f16.cu +7 -0
  322. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q4_0.cu +7 -0
  323. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q4_1.cu +7 -0
  324. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q5_0.cu +7 -0
  325. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q5_1.cu +7 -0
  326. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q8_0.cu +7 -0
  327. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-bf16.cu +7 -0
  328. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-f16.cu +7 -0
  329. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q4_0.cu +7 -0
  330. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q4_1.cu +7 -0
  331. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q5_0.cu +7 -0
  332. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q5_1.cu +7 -0
  333. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q8_0.cu +7 -0
  334. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-bf16.cu +7 -0
  335. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-f16.cu +7 -0
  336. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q4_0.cu +7 -0
  337. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q4_1.cu +7 -0
  338. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q5_0.cu +7 -0
  339. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q5_1.cu +7 -0
  340. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q8_0.cu +7 -0
  341. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-bf16.cu +7 -0
  342. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-f16.cu +7 -0
  343. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q4_0.cu +7 -0
  344. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q4_1.cu +7 -0
  345. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q5_0.cu +7 -0
  346. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q5_1.cu +7 -0
  347. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q8_0.cu +7 -0
  348. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-bf16.cu +7 -0
  349. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-f16.cu +7 -0
  350. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q4_0.cu +7 -0
  351. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q4_1.cu +7 -0
  352. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q5_0.cu +7 -0
  353. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q5_1.cu +7 -0
  354. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q8_0.cu +7 -0
  355. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-bf16.cu +7 -0
  356. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-f16.cu +7 -0
  357. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q4_0.cu +7 -0
  358. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q4_1.cu +7 -0
  359. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q5_0.cu +7 -0
  360. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q5_1.cu +7 -0
  361. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q8_0.cu +7 -0
  362. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-bf16.cu +7 -0
  363. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-f16.cu +7 -0
  364. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q4_0.cu +7 -0
  365. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q4_1.cu +7 -0
  366. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q5_0.cu +7 -0
  367. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q5_1.cu +7 -0
  368. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q8_0.cu +7 -0
  369. data/vendor/ggml/src/ggml-cuda/template-instances/generate_cu_files.py +111 -0
  370. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_1.cu +5 -0
  371. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_10.cu +5 -0
  372. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_11.cu +5 -0
  373. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_12.cu +5 -0
  374. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_13.cu +5 -0
  375. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_14.cu +5 -0
  376. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_15.cu +5 -0
  377. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_16.cu +5 -0
  378. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_2.cu +5 -0
  379. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_3.cu +5 -0
  380. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_4.cu +5 -0
  381. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_5.cu +5 -0
  382. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_6.cu +5 -0
  383. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_7.cu +5 -0
  384. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_8.cu +5 -0
  385. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_9.cu +5 -0
  386. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq1_s.cu +5 -0
  387. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_s.cu +5 -0
  388. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xs.cu +5 -0
  389. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xxs.cu +5 -0
  390. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_s.cu +5 -0
  391. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_xxs.cu +5 -0
  392. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_nl.cu +5 -0
  393. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_xs.cu +5 -0
  394. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-mxfp4.cu +5 -0
  395. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-nvfp4.cu +5 -0
  396. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q1_0.cu +5 -0
  397. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_0.cu +5 -0
  398. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_k.cu +5 -0
  399. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q3_k.cu +5 -0
  400. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_0.cu +5 -0
  401. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_1.cu +5 -0
  402. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_k.cu +5 -0
  403. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_0.cu +5 -0
  404. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_1.cu +5 -0
  405. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_k.cu +5 -0
  406. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q6_k.cu +5 -0
  407. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q8_0.cu +5 -0
  408. data/vendor/ggml/src/ggml-cuda/top-k.cu +275 -0
  409. data/vendor/ggml/src/ggml-cuda/top-k.cuh +3 -0
  410. data/vendor/ggml/src/ggml-cuda/topk-moe.cu +442 -0
  411. data/vendor/ggml/src/ggml-cuda/topk-moe.cuh +31 -0
  412. data/vendor/ggml/src/ggml-cuda/tri.cu +136 -0
  413. data/vendor/ggml/src/ggml-cuda/tri.cuh +5 -0
  414. data/vendor/ggml/src/ggml-cuda/tsembd.cu +47 -0
  415. data/vendor/ggml/src/ggml-cuda/tsembd.cuh +5 -0
  416. data/vendor/ggml/src/ggml-cuda/unary.cu +721 -0
  417. data/vendor/ggml/src/ggml-cuda/unary.cuh +123 -0
  418. data/vendor/ggml/src/ggml-cuda/upscale.cu +293 -0
  419. data/vendor/ggml/src/ggml-cuda/upscale.cuh +5 -0
  420. data/vendor/ggml/src/ggml-cuda/vecdotq.cuh +1371 -0
  421. data/vendor/ggml/src/ggml-cuda/vendors/cuda.h +28 -0
  422. data/vendor/ggml/src/ggml-cuda/vendors/hip.h +301 -0
  423. data/vendor/ggml/src/ggml-cuda/vendors/musa.h +151 -0
  424. data/vendor/ggml/src/ggml-cuda/wkv.cu +253 -0
  425. data/vendor/ggml/src/ggml-cuda/wkv.cuh +7 -0
  426. data/vendor/ggml/src/ggml-et/CMakeLists.txt +246 -0
  427. data/vendor/ggml/src/ggml-et/cmake/embed_one_kernel.cmake +15 -0
  428. data/vendor/ggml/src/ggml-et/cmake/ggml-et-kernels-embed.cpp.in +6 -0
  429. data/vendor/ggml/src/ggml-et/cmake/ggml-et-kernels-embed.hpp.in +12 -0
  430. data/vendor/ggml/src/ggml-et/cmake/ggml-et-uberkernel-kernel-map.cpp.in +18 -0
  431. data/vendor/ggml/src/ggml-et/cmake/ggml-et-uberkernel-kernel-map.h.in +13 -0
  432. data/vendor/ggml/src/ggml-et/et-kernels/CMakeLists.txt +137 -0
  433. data/vendor/ggml/src/ggml-et/et-kernels/scripts/check_unimplemented_instructions.sh +36 -0
  434. data/vendor/ggml/src/ggml-et/et-kernels/src/RunBackend.sh +23 -0
  435. data/vendor/ggml/src/ggml-et/et-kernels/src/block_ops.h +997 -0
  436. data/vendor/ggml/src/ggml-et/et-kernels/src/clamp_f32.c +120 -0
  437. data/vendor/ggml/src/ggml-et/et-kernels/src/concat_f32.c +175 -0
  438. data/vendor/ggml/src/ggml-et/et-kernels/src/cont_f16.c +107 -0
  439. data/vendor/ggml/src/ggml-et/et-kernels/src/cont_f32.c +248 -0
  440. data/vendor/ggml/src/ggml-et/et-kernels/src/conv_2d_f32_me.c +807 -0
  441. data/vendor/ggml/src/ggml-et/et-kernels/src/cpy_f32_f16.c +110 -0
  442. data/vendor/ggml/src/ggml-et/et-kernels/src/crt.S +15 -0
  443. data/vendor/ggml/src/ggml-et/et-kernels/src/cumsum_f32.c +96 -0
  444. data/vendor/ggml/src/ggml-et/et-kernels/src/diag_f32.c +90 -0
  445. data/vendor/ggml/src/ggml-et/et-kernels/src/el_map_f32.c +377 -0
  446. data/vendor/ggml/src/ggml-et/et-kernels/src/fill_f32.c +87 -0
  447. data/vendor/ggml/src/ggml-et/et-kernels/src/flash_attn_ext_f16_me.c +1000 -0
  448. data/vendor/ggml/src/ggml-et/et-kernels/src/flash_attn_ext_f32.c +217 -0
  449. data/vendor/ggml/src/ggml-et/et-kernels/src/gated_delta_net_f32.c +346 -0
  450. data/vendor/ggml/src/ggml-et/et-kernels/src/get_rows_f32.c +612 -0
  451. data/vendor/ggml/src/ggml-et/et-kernels/src/ggml_tensor.h +44 -0
  452. data/vendor/ggml/src/ggml-et/et-kernels/src/glu_f32.c +606 -0
  453. data/vendor/ggml/src/ggml-et/et-kernels/src/group_norm_f32.c +171 -0
  454. data/vendor/ggml/src/ggml-et/et-kernels/src/im2col.c +130 -0
  455. data/vendor/ggml/src/ggml-et/et-kernels/src/l2_norm_f32.c +237 -0
  456. data/vendor/ggml/src/ggml-et/et-kernels/src/linker.ld +85 -0
  457. data/vendor/ggml/src/ggml-et/et-kernels/src/math_fp.h +299 -0
  458. data/vendor/ggml/src/ggml-et/et-kernels/src/mean_f32.c +220 -0
  459. data/vendor/ggml/src/ggml-et/et-kernels/src/memops.c +181 -0
  460. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0.c +358 -0
  461. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0_matrix_engine.c +368 -0
  462. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_Q8_0.c +413 -0
  463. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f16.c +142 -0
  464. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f16_matrix_engine.c +329 -0
  465. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f32.c +137 -0
  466. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f32_matrix_engine.c +155 -0
  467. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_id_Q4_0.c +169 -0
  468. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_id_Q8_0.c +160 -0
  469. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_id_f32.c +288 -0
  470. data/vendor/ggml/src/ggml-et/et-kernels/src/norm_f32.c +328 -0
  471. data/vendor/ggml/src/ggml-et/et-kernels/src/pad_f32.c +165 -0
  472. data/vendor/ggml/src/ggml-et/et-kernels/src/platform.h +545 -0
  473. data/vendor/ggml/src/ggml-et/et-kernels/src/quants.h +72 -0
  474. data/vendor/ggml/src/ggml-et/et-kernels/src/repeat_f32.c +118 -0
  475. data/vendor/ggml/src/ggml-et/et-kernels/src/rms_norm_f32.c +270 -0
  476. data/vendor/ggml/src/ggml-et/et-kernels/src/rms_norm_mul_f32.c +290 -0
  477. data/vendor/ggml/src/ggml-et/et-kernels/src/rope_f32.c +656 -0
  478. data/vendor/ggml/src/ggml-et/et-kernels/src/rwkv_wkv6_f32.c +184 -0
  479. data/vendor/ggml/src/ggml-et/et-kernels/src/rwkv_wkv7_f32.c +272 -0
  480. data/vendor/ggml/src/ggml-et/et-kernels/src/scale_f32.c +94 -0
  481. data/vendor/ggml/src/ggml-et/et-kernels/src/set_f32.c +101 -0
  482. data/vendor/ggml/src/ggml-et/et-kernels/src/set_rows_f32.c +394 -0
  483. data/vendor/ggml/src/ggml-et/et-kernels/src/softmax_f32.c +698 -0
  484. data/vendor/ggml/src/ggml-et/et-kernels/src/solve_tri_f32.c +109 -0
  485. data/vendor/ggml/src/ggml-et/et-kernels/src/sqr_f32.c +88 -0
  486. data/vendor/ggml/src/ggml-et/et-kernels/src/ssm_conv_f32.c +129 -0
  487. data/vendor/ggml/src/ggml-et/et-kernels/src/ssm_scan_f32.c +282 -0
  488. data/vendor/ggml/src/ggml-et/et-kernels/src/sum_rows_f32.c +103 -0
  489. data/vendor/ggml/src/ggml-et/et-kernels/src/tensor.h +897 -0
  490. data/vendor/ggml/src/ggml-et/et-kernels/src/tri_f32.c +244 -0
  491. data/vendor/ggml/src/ggml-et/et-kernels/src/uberkernel.c +497 -0
  492. data/vendor/ggml/src/ggml-et/et-kernels/src/unary_f32.c +705 -0
  493. data/vendor/ggml/src/ggml-et/ggml-et-common.h +86 -0
  494. data/vendor/ggml/src/ggml-et/ggml-et-cpu-compare.cpp +502 -0
  495. data/vendor/ggml/src/ggml-et/ggml-et-cpu-compare.h +54 -0
  496. data/vendor/ggml/src/ggml-et/ggml-et-kernels.cpp +508 -0
  497. data/vendor/ggml/src/ggml-et/ggml-et-kernels.h +48 -0
  498. data/vendor/ggml/src/ggml-et/ggml-et-memops.cpp +36 -0
  499. data/vendor/ggml/src/ggml-et/ggml-et-memops.h +18 -0
  500. data/vendor/ggml/src/ggml-et/ggml-et-ops.cpp +2584 -0
  501. data/vendor/ggml/src/ggml-et/ggml-et-ops.h +393 -0
  502. data/vendor/ggml/src/ggml-et/ggml-et-uberkernel-common.h +17 -0
  503. data/vendor/ggml/src/ggml-et/ggml-et.cpp +1880 -0
  504. data/vendor/ggml/src/ggml-feats.h +166 -0
  505. data/vendor/ggml/src/ggml-hexagon/CMakeLists.txt +113 -0
  506. data/vendor/ggml/src/ggml-hexagon/ggml-hexagon.cpp +6523 -0
  507. data/vendor/ggml/src/ggml-hexagon/htp/CMakeLists.txt +58 -0
  508. data/vendor/ggml/src/ggml-hexagon/htp/act-ops.c +585 -0
  509. data/vendor/ggml/src/ggml-hexagon/htp/allreduce-ops.c +398 -0
  510. data/vendor/ggml/src/ggml-hexagon/htp/allreduce-ops.h +40 -0
  511. data/vendor/ggml/src/ggml-hexagon/htp/argsort-ops.c +514 -0
  512. data/vendor/ggml/src/ggml-hexagon/htp/binary-ops.c +873 -0
  513. data/vendor/ggml/src/ggml-hexagon/htp/cmake-toolchain.cmake +155 -0
  514. data/vendor/ggml/src/ggml-hexagon/htp/concat-ops.c +277 -0
  515. data/vendor/ggml/src/ggml-hexagon/htp/cpy-ops.c +354 -0
  516. data/vendor/ggml/src/ggml-hexagon/htp/cumsum-ops.c +265 -0
  517. data/vendor/ggml/src/ggml-hexagon/htp/diag-ops.c +216 -0
  518. data/vendor/ggml/src/ggml-hexagon/htp/dma-queue.c +104 -0
  519. data/vendor/ggml/src/ggml-hexagon/htp/dma-queue.h +408 -0
  520. data/vendor/ggml/src/ggml-hexagon/htp/fill-ops.c +123 -0
  521. data/vendor/ggml/src/ggml-hexagon/htp/flash-attn-ops.c +2479 -0
  522. data/vendor/ggml/src/ggml-hexagon/htp/flash-attn-ops.h +344 -0
  523. data/vendor/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c +1157 -0
  524. data/vendor/ggml/src/ggml-hexagon/htp/get-rows-ops.c +260 -0
  525. data/vendor/ggml/src/ggml-hexagon/htp/get-rows-ops.h +77 -0
  526. data/vendor/ggml/src/ggml-hexagon/htp/hex-bitmap.h +24 -0
  527. data/vendor/ggml/src/ggml-hexagon/htp/hex-common.h +80 -0
  528. data/vendor/ggml/src/ggml-hexagon/htp/hex-dma.h +2 -0
  529. data/vendor/ggml/src/ggml-hexagon/htp/hex-dump.h +86 -0
  530. data/vendor/ggml/src/ggml-hexagon/htp/hex-fastdiv.h +37 -0
  531. data/vendor/ggml/src/ggml-hexagon/htp/hex-profile.h +64 -0
  532. data/vendor/ggml/src/ggml-hexagon/htp/hex-utils.h +65 -0
  533. data/vendor/ggml/src/ggml-hexagon/htp/hmx-fa-kernels.h +555 -0
  534. data/vendor/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h +1378 -0
  535. data/vendor/ggml/src/ggml-hexagon/htp/hmx-queue.c +170 -0
  536. data/vendor/ggml/src/ggml-hexagon/htp/hmx-queue.h +160 -0
  537. data/vendor/ggml/src/ggml-hexagon/htp/hmx-utils.h +222 -0
  538. data/vendor/ggml/src/ggml-hexagon/htp/htp-ctx.h +146 -0
  539. data/vendor/ggml/src/ggml-hexagon/htp/htp-ops.h +242 -0
  540. data/vendor/ggml/src/ggml-hexagon/htp/htp-tensor.c +300 -0
  541. data/vendor/ggml/src/ggml-hexagon/htp/htp-tensor.h +29 -0
  542. data/vendor/ggml/src/ggml-hexagon/htp/htp-vtcm.h +19 -0
  543. data/vendor/ggml/src/ggml-hexagon/htp/htp_iface.idl +23 -0
  544. data/vendor/ggml/src/ggml-hexagon/htp/hvx-arith.h +636 -0
  545. data/vendor/ggml/src/ggml-hexagon/htp/hvx-base.h +292 -0
  546. data/vendor/ggml/src/ggml-hexagon/htp/hvx-copy.h +262 -0
  547. data/vendor/ggml/src/ggml-hexagon/htp/hvx-div.h +291 -0
  548. data/vendor/ggml/src/ggml-hexagon/htp/hvx-dump.h +129 -0
  549. data/vendor/ggml/src/ggml-hexagon/htp/hvx-exp.h +255 -0
  550. data/vendor/ggml/src/ggml-hexagon/htp/hvx-fa-kernels.h +303 -0
  551. data/vendor/ggml/src/ggml-hexagon/htp/hvx-flash-attn.h +47 -0
  552. data/vendor/ggml/src/ggml-hexagon/htp/hvx-floor.h +100 -0
  553. data/vendor/ggml/src/ggml-hexagon/htp/hvx-inverse.h +210 -0
  554. data/vendor/ggml/src/ggml-hexagon/htp/hvx-log.h +118 -0
  555. data/vendor/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-flat.h +1511 -0
  556. data/vendor/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h +1200 -0
  557. data/vendor/ggml/src/ggml-hexagon/htp/hvx-norm.h +454 -0
  558. data/vendor/ggml/src/ggml-hexagon/htp/hvx-pow.h +42 -0
  559. data/vendor/ggml/src/ggml-hexagon/htp/hvx-quant.h +165 -0
  560. data/vendor/ggml/src/ggml-hexagon/htp/hvx-reduce.h +336 -0
  561. data/vendor/ggml/src/ggml-hexagon/htp/hvx-repl.h +74 -0
  562. data/vendor/ggml/src/ggml-hexagon/htp/hvx-scale.h +199 -0
  563. data/vendor/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h +181 -0
  564. data/vendor/ggml/src/ggml-hexagon/htp/hvx-sin-cos.h +90 -0
  565. data/vendor/ggml/src/ggml-hexagon/htp/hvx-sqrt.h +189 -0
  566. data/vendor/ggml/src/ggml-hexagon/htp/hvx-types.h +36 -0
  567. data/vendor/ggml/src/ggml-hexagon/htp/hvx-utils.h +24 -0
  568. data/vendor/ggml/src/ggml-hexagon/htp/im2col-ops.c +306 -0
  569. data/vendor/ggml/src/ggml-hexagon/htp/main.c +1245 -0
  570. data/vendor/ggml/src/ggml-hexagon/htp/matmul-ops.c +4078 -0
  571. data/vendor/ggml/src/ggml-hexagon/htp/matmul-ops.h +778 -0
  572. data/vendor/ggml/src/ggml-hexagon/htp/pad-ops.c +547 -0
  573. data/vendor/ggml/src/ggml-hexagon/htp/repeat-ops.c +148 -0
  574. data/vendor/ggml/src/ggml-hexagon/htp/rope-ops.c +733 -0
  575. data/vendor/ggml/src/ggml-hexagon/htp/set-rows-ops.c +230 -0
  576. data/vendor/ggml/src/ggml-hexagon/htp/set-rows-ops.h +74 -0
  577. data/vendor/ggml/src/ggml-hexagon/htp/softmax-ops.c +407 -0
  578. data/vendor/ggml/src/ggml-hexagon/htp/solve-tri-ops.c +267 -0
  579. data/vendor/ggml/src/ggml-hexagon/htp/ssm-conv.c +433 -0
  580. data/vendor/ggml/src/ggml-hexagon/htp/sum-rows-ops.c +128 -0
  581. data/vendor/ggml/src/ggml-hexagon/htp/unary-ops.c +1264 -0
  582. data/vendor/ggml/src/ggml-hexagon/htp/unary-ops.h +176 -0
  583. data/vendor/ggml/src/ggml-hexagon/htp/work-queue.c +244 -0
  584. data/vendor/ggml/src/ggml-hexagon/htp/work-queue.h +38 -0
  585. data/vendor/ggml/src/ggml-hexagon/htp-drv.cpp +425 -0
  586. data/vendor/ggml/src/ggml-hexagon/htp-drv.h +123 -0
  587. data/vendor/ggml/src/ggml-hexagon/htp-opnode.h +372 -0
  588. data/vendor/ggml/src/ggml-hexagon/libdl.h +79 -0
  589. data/vendor/ggml/src/ggml-hexagon/libggml-htp.inf +36 -0
  590. data/vendor/ggml/src/ggml-hip/CMakeLists.txt +153 -0
  591. data/vendor/ggml/src/ggml-impl.h +783 -0
  592. data/vendor/ggml/src/ggml-metal/CMakeLists.txt +240 -0
  593. data/vendor/ggml/src/ggml-metal/ggml-metal-common.cpp +474 -0
  594. data/vendor/ggml/src/ggml-metal/ggml-metal-common.h +56 -0
  595. data/vendor/ggml/src/ggml-metal/ggml-metal-context.h +41 -0
  596. data/vendor/ggml/src/ggml-metal/ggml-metal-context.m +759 -0
  597. data/vendor/ggml/src/ggml-metal/ggml-metal-device.cpp +2330 -0
  598. data/vendor/ggml/src/ggml-metal/ggml-metal-device.h +355 -0
  599. data/vendor/ggml/src/ggml-metal/ggml-metal-device.m +2473 -0
  600. data/vendor/ggml/src/ggml-metal/ggml-metal-impl.h +1345 -0
  601. data/vendor/ggml/src/ggml-metal/ggml-metal-ops.cpp +5610 -0
  602. data/vendor/ggml/src/ggml-metal/ggml-metal-ops.h +106 -0
  603. data/vendor/ggml/src/ggml-metal/ggml-metal-tuning.cpp +3732 -0
  604. data/vendor/ggml/src/ggml-metal/ggml-metal-tuning.h +77 -0
  605. data/vendor/ggml/src/ggml-metal/ggml-metal.cpp +1006 -0
  606. data/vendor/ggml/src/ggml-metal/kernels/argsort.metal +337 -0
  607. data/vendor/ggml/src/ggml-metal/kernels/binbcast.metal +228 -0
  608. data/vendor/ggml/src/ggml-metal/kernels/common.h +126 -0
  609. data/vendor/ggml/src/ggml-metal/kernels/conv.metal +724 -0
  610. data/vendor/ggml/src/ggml-metal/kernels/dequantize.h +735 -0
  611. data/vendor/ggml/src/ggml-metal/kernels/fa.metal +2445 -0
  612. data/vendor/ggml/src/ggml-metal/kernels/gated_delta_net.metal +250 -0
  613. data/vendor/ggml/src/ggml-metal/kernels/misc.metal +595 -0
  614. data/vendor/ggml/src/ggml-metal/kernels/mul_mm.metal +853 -0
  615. data/vendor/ggml/src/ggml-metal/kernels/mul_mv.metal +3225 -0
  616. data/vendor/ggml/src/ggml-metal/kernels/norm.metal +308 -0
  617. data/vendor/ggml/src/ggml-metal/kernels/pool.metal +148 -0
  618. data/vendor/ggml/src/ggml-metal/kernels/quantize.h +262 -0
  619. data/vendor/ggml/src/ggml-metal/kernels/quantize.metal +480 -0
  620. data/vendor/ggml/src/ggml-metal/kernels/reduce.metal +228 -0
  621. data/vendor/ggml/src/ggml-metal/kernels/rope.metal +333 -0
  622. data/vendor/ggml/src/ggml-metal/kernels/softmax.metal +223 -0
  623. data/vendor/ggml/src/ggml-metal/kernels/solve_tri.metal +75 -0
  624. data/vendor/ggml/src/ggml-metal/kernels/ssm.metal +473 -0
  625. data/vendor/ggml/src/ggml-metal/kernels/tri.metal +69 -0
  626. data/vendor/ggml/src/ggml-metal/kernels/unary.metal +400 -0
  627. data/vendor/ggml/src/ggml-metal/kernels/upscale.metal +179 -0
  628. data/vendor/ggml/src/ggml-metal/kernels/wkv.metal +179 -0
  629. data/vendor/ggml/src/ggml-musa/CMakeLists.txt +123 -0
  630. data/vendor/ggml/src/ggml-musa/mudnn.cu +112 -0
  631. data/vendor/ggml/src/ggml-musa/mudnn.cuh +12 -0
  632. data/vendor/ggml/src/ggml-opencl/CMakeLists.txt +246 -0
  633. data/vendor/ggml/src/ggml-opencl/cl-program-cache.cpp +453 -0
  634. data/vendor/ggml/src/ggml-opencl/cl-program-cache.h +75 -0
  635. data/vendor/ggml/src/ggml-opencl/fa_tune.h +92 -0
  636. data/vendor/ggml/src/ggml-opencl/ggml-opencl.cpp +26982 -0
  637. data/vendor/ggml/src/ggml-opencl/kernels/abs.cl +113 -0
  638. data/vendor/ggml/src/ggml-opencl/kernels/add.cl +190 -0
  639. data/vendor/ggml/src/ggml-opencl/kernels/add_id.cl +42 -0
  640. data/vendor/ggml/src/ggml-opencl/kernels/argsort.cl +86 -0
  641. data/vendor/ggml/src/ggml-opencl/kernels/clamp.cl +20 -0
  642. data/vendor/ggml/src/ggml-opencl/kernels/concat.cl +118 -0
  643. data/vendor/ggml/src/ggml-opencl/kernels/conv2d.cl +185 -0
  644. data/vendor/ggml/src/ggml-opencl/kernels/conv2d_f16_f32.cl +176 -0
  645. data/vendor/ggml/src/ggml-opencl/kernels/cpy.cl +288 -0
  646. data/vendor/ggml/src/ggml-opencl/kernels/cumsum.cl +139 -0
  647. data/vendor/ggml/src/ggml-opencl/kernels/cvt.cl +2663 -0
  648. data/vendor/ggml/src/ggml-opencl/kernels/diag.cl +27 -0
  649. data/vendor/ggml/src/ggml-opencl/kernels/diag_mask_inf.cl +58 -0
  650. data/vendor/ggml/src/ggml-opencl/kernels/div.cl +138 -0
  651. data/vendor/ggml/src/ggml-opencl/kernels/embed_kernel.py +26 -0
  652. data/vendor/ggml/src/ggml-opencl/kernels/exp.cl +125 -0
  653. data/vendor/ggml/src/ggml-opencl/kernels/expm1.cl +113 -0
  654. data/vendor/ggml/src/ggml-opencl/kernels/fill.cl +17 -0
  655. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f16.cl +421 -0
  656. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32.cl +422 -0
  657. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32_f16.cl +2656 -0
  658. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32_q4_0.cl +2011 -0
  659. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32_q8_0.cl +1840 -0
  660. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_pre_f16.cl +156 -0
  661. data/vendor/ggml/src/ggml-opencl/kernels/gated_delta_net.cl +249 -0
  662. data/vendor/ggml/src/ggml-opencl/kernels/gelu.cl +89 -0
  663. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_f32.cl +162 -0
  664. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_f32_ns.cl +376 -0
  665. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_q8_1_dp4a.cl +190 -0
  666. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_0_f32_ns.cl +324 -0
  667. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_0_q8_1_dp4a.cl +169 -0
  668. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_1_f32_ns.cl +326 -0
  669. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_k_f32_ns.cl +348 -0
  670. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_k_q8_1_dp4a.cl +209 -0
  671. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q5_0_f32_ns.cl +328 -0
  672. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q5_1_f32_ns.cl +330 -0
  673. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q5_k_f32_ns.cl +356 -0
  674. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q6_k_f32_ns.cl +335 -0
  675. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q6_k_q8_1_dp4a.cl +200 -0
  676. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q8_0_f32_ns.cl +221 -0
  677. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q8_1_dp4a.cl +226 -0
  678. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_nl_f32.cl +150 -0
  679. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_nl_q8_1_dp4a.cl +143 -0
  680. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q1_0_f32.cl +94 -0
  681. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_f32.cl +139 -0
  682. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_q8_1_dp4a.cl +127 -0
  683. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_1_f32.cl +132 -0
  684. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_f32.cl +490 -0
  685. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_q8_1_dp4a.cl +281 -0
  686. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_0_f32.cl +131 -0
  687. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_0_q8_1_dp4a.cl +235 -0
  688. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_1_f32.cl +134 -0
  689. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_f32.cl +176 -0
  690. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_q8_1_dp4a.cl +164 -0
  691. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32.cl +245 -0
  692. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32_tiled.cl +136 -0
  693. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_q8_1_dp4a.cl +144 -0
  694. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_f32.cl +129 -0
  695. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_q8_1_dp4a.cl +212 -0
  696. data/vendor/ggml/src/ggml-opencl/kernels/gemm_xmem_f16_f32_os8.cl +233 -0
  697. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32.cl +156 -0
  698. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32_ns.cl +257 -0
  699. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q4_0_f32_ns.cl +120 -0
  700. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q4_1_f32_ns.cl +123 -0
  701. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q4_k_f32_ns.cl +266 -0
  702. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q5_0_f32_ns.cl +123 -0
  703. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q5_1_f32_ns.cl +125 -0
  704. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q5_k_f32_ns.cl +160 -0
  705. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q6_k_f32_ns.cl +141 -0
  706. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq4_nl_f32.cl +307 -0
  707. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q1_0_f32.cl +125 -0
  708. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32.cl +383 -0
  709. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32_spec.cl +272 -0
  710. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_1_f32.cl +384 -0
  711. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32.cl +823 -0
  712. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_o4.cl +349 -0
  713. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_tiled.cl +118 -0
  714. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_0_f32.cl +296 -0
  715. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_1_f32.cl +299 -0
  716. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_k_f32.cl +453 -0
  717. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32.cl +409 -0
  718. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_o4.cl +372 -0
  719. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_tiled.cl +196 -0
  720. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q8_0_f32.cl +280 -0
  721. data/vendor/ggml/src/ggml-opencl/kernels/get_rows.cl +193 -0
  722. data/vendor/ggml/src/ggml-opencl/kernels/glu.cl +443 -0
  723. data/vendor/ggml/src/ggml-opencl/kernels/group_norm.cl +121 -0
  724. data/vendor/ggml/src/ggml-opencl/kernels/im2col_f16.cl +57 -0
  725. data/vendor/ggml/src/ggml-opencl/kernels/im2col_f32.cl +57 -0
  726. data/vendor/ggml/src/ggml-opencl/kernels/l2_norm.cl +71 -0
  727. data/vendor/ggml/src/ggml-opencl/kernels/mean.cl +140 -0
  728. data/vendor/ggml/src/ggml-opencl/kernels/moe_add_id_glu.cl +76 -0
  729. data/vendor/ggml/src/ggml-opencl/kernels/moe_combine.cl +79 -0
  730. data/vendor/ggml/src/ggml-opencl/kernels/moe_reorder_b.cl +30 -0
  731. data/vendor/ggml/src/ggml-opencl/kernels/moe_reorder_quant_a_q8_1.cl +64 -0
  732. data/vendor/ggml/src/ggml-opencl/kernels/moe_sort_by_expert.cl +155 -0
  733. data/vendor/ggml/src/ggml-opencl/kernels/mul.cl +152 -0
  734. data/vendor/ggml/src/ggml-opencl/kernels/mul_mat_f16_f32.cl +130 -0
  735. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_kq_kqv.cl +273 -0
  736. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_l4_lm.cl +146 -0
  737. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_f32_f32_l4_lm.cl +196 -0
  738. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_iq4_nl_f32_l4_lm.cl +171 -0
  739. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q1_0_f32_l4_lm.cl +156 -0
  740. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q4_0_f32_l4_lm.cl +163 -0
  741. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q4_1_f32_l4_lm.cl +165 -0
  742. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q4_k_f32_l4_lm.cl +189 -0
  743. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q5_0_f32_l4_lm.cl +173 -0
  744. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q5_1_f32_l4_lm.cl +175 -0
  745. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q5_k_f32_l4_lm.cl +202 -0
  746. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q6_k_f32_l4_lm.cl +158 -0
  747. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q8_0_f32_l4_lm.cl +154 -0
  748. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f16.cl +125 -0
  749. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32.cl +125 -0
  750. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_1row.cl +101 -0
  751. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_l4.cl +1237 -0
  752. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_mrow.cl +306 -0
  753. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f32_f32.cl +118 -0
  754. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_mxfp4_f32.cl +189 -0
  755. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_mxfp4_f32_flat.cl +176 -0
  756. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_q4_0_f32_8x_flat.cl +283 -0
  757. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_q8_0_f32.cl +140 -0
  758. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_q8_0_f32_flat.cl +222 -0
  759. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_iq4_nl_f32.cl +164 -0
  760. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_iq4_nl_f32_flat.cl +202 -0
  761. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_mxfp4_f32.cl +144 -0
  762. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_mxfp4_f32_flat.cl +167 -0
  763. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q1_0_f32.cl +141 -0
  764. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q1_0_f32_flat.cl +190 -0
  765. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32.cl +192 -0
  766. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_16x_flat.cl +307 -0
  767. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_8x_flat.cl +265 -0
  768. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_8x_flat.cl +272 -0
  769. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_v.cl +254 -0
  770. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_1_f32.cl +219 -0
  771. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_1_f32_flat.cl +229 -0
  772. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32.cl +182 -0
  773. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32_flat.cl +205 -0
  774. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_0_f32.cl +241 -0
  775. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_0_f32_flat.cl +243 -0
  776. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_1_f32.cl +243 -0
  777. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_1_f32_flat.cl +247 -0
  778. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_k_f32.cl +187 -0
  779. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_k_f32_flat.cl +244 -0
  780. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32.cl +194 -0
  781. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32_flat.cl +272 -0
  782. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q8_0_f32.cl +125 -0
  783. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q8_0_f32_flat.cl +202 -0
  784. data/vendor/ggml/src/ggml-opencl/kernels/neg.cl +125 -0
  785. data/vendor/ggml/src/ggml-opencl/kernels/norm.cl +164 -0
  786. data/vendor/ggml/src/ggml-opencl/kernels/pad.cl +39 -0
  787. data/vendor/ggml/src/ggml-opencl/kernels/quant_a_q8_1.cl +42 -0
  788. data/vendor/ggml/src/ggml-opencl/kernels/relu.cl +16 -0
  789. data/vendor/ggml/src/ggml-opencl/kernels/repeat.cl +38 -0
  790. data/vendor/ggml/src/ggml-opencl/kernels/rms_norm.cl +369 -0
  791. data/vendor/ggml/src/ggml-opencl/kernels/rope.cl +759 -0
  792. data/vendor/ggml/src/ggml-opencl/kernels/scale.cl +27 -0
  793. data/vendor/ggml/src/ggml-opencl/kernels/set_rows.cl +708 -0
  794. data/vendor/ggml/src/ggml-opencl/kernels/sigmoid.cl +29 -0
  795. data/vendor/ggml/src/ggml-opencl/kernels/silu.cl +30 -0
  796. data/vendor/ggml/src/ggml-opencl/kernels/softmax_4_f16.cl +108 -0
  797. data/vendor/ggml/src/ggml-opencl/kernels/softmax_4_f32.cl +108 -0
  798. data/vendor/ggml/src/ggml-opencl/kernels/softmax_f16.cl +107 -0
  799. data/vendor/ggml/src/ggml-opencl/kernels/softmax_f32.cl +107 -0
  800. data/vendor/ggml/src/ggml-opencl/kernels/softplus.cl +116 -0
  801. data/vendor/ggml/src/ggml-opencl/kernels/solve_tri.cl +51 -0
  802. data/vendor/ggml/src/ggml-opencl/kernels/sqr.cl +53 -0
  803. data/vendor/ggml/src/ggml-opencl/kernels/sqrt.cl +53 -0
  804. data/vendor/ggml/src/ggml-opencl/kernels/ssm_conv.cl +77 -0
  805. data/vendor/ggml/src/ggml-opencl/kernels/ssm_scan.cl +216 -0
  806. data/vendor/ggml/src/ggml-opencl/kernels/sub.cl +138 -0
  807. data/vendor/ggml/src/ggml-opencl/kernels/sum_rows.cl +140 -0
  808. data/vendor/ggml/src/ggml-opencl/kernels/tanh.cl +109 -0
  809. data/vendor/ggml/src/ggml-opencl/kernels/transpose.cl +143 -0
  810. data/vendor/ggml/src/ggml-opencl/kernels/tri.cl +32 -0
  811. data/vendor/ggml/src/ggml-opencl/kernels/tsembd.cl +48 -0
  812. data/vendor/ggml/src/ggml-opencl/kernels/upscale.cl +120 -0
  813. data/vendor/ggml/src/ggml-opencl/libdl.h +79 -0
  814. data/vendor/ggml/src/ggml-openvino/CMakeLists.txt +22 -0
  815. data/vendor/ggml/src/ggml-openvino/ggml-decoder.cpp +2057 -0
  816. data/vendor/ggml/src/ggml-openvino/ggml-decoder.h +417 -0
  817. data/vendor/ggml/src/ggml-openvino/ggml-openvino-extra.cpp +493 -0
  818. data/vendor/ggml/src/ggml-openvino/ggml-openvino-extra.h +216 -0
  819. data/vendor/ggml/src/ggml-openvino/ggml-openvino.cpp +1566 -0
  820. data/vendor/ggml/src/ggml-openvino/ggml-quants.cpp +1273 -0
  821. data/vendor/ggml/src/ggml-openvino/ggml-quants.h +186 -0
  822. data/vendor/ggml/src/ggml-openvino/model-cache.cpp +272 -0
  823. data/vendor/ggml/src/ggml-openvino/model-cache.h +56 -0
  824. data/vendor/ggml/src/ggml-openvino/openvino/decoder.h +133 -0
  825. data/vendor/ggml/src/ggml-openvino/openvino/frontend.cpp +27 -0
  826. data/vendor/ggml/src/ggml-openvino/openvino/frontend.h +23 -0
  827. data/vendor/ggml/src/ggml-openvino/openvino/input_model.cpp +17 -0
  828. data/vendor/ggml/src/ggml-openvino/openvino/input_model.h +29 -0
  829. data/vendor/ggml/src/ggml-openvino/openvino/node_context.h +171 -0
  830. data/vendor/ggml/src/ggml-openvino/openvino/op/add.cpp +45 -0
  831. data/vendor/ggml/src/ggml-openvino/openvino/op/add_id.cpp +76 -0
  832. data/vendor/ggml/src/ggml-openvino/openvino/op/argsort.cpp +47 -0
  833. data/vendor/ggml/src/ggml-openvino/openvino/op/clamp.cpp +33 -0
  834. data/vendor/ggml/src/ggml-openvino/openvino/op/concat.cpp +48 -0
  835. data/vendor/ggml/src/ggml-openvino/openvino/op/cont.cpp +40 -0
  836. data/vendor/ggml/src/ggml-openvino/openvino/op/cpy.cpp +295 -0
  837. data/vendor/ggml/src/ggml-openvino/openvino/op/cumsum.cpp +29 -0
  838. data/vendor/ggml/src/ggml-openvino/openvino/op/diag.cpp +58 -0
  839. data/vendor/ggml/src/ggml-openvino/openvino/op/div.cpp +146 -0
  840. data/vendor/ggml/src/ggml-openvino/openvino/op/fill.cpp +34 -0
  841. data/vendor/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp +241 -0
  842. data/vendor/ggml/src/ggml-openvino/openvino/op/gated_delta_net.cpp +327 -0
  843. data/vendor/ggml/src/ggml-openvino/openvino/op/gated_delta_net.hpp +65 -0
  844. data/vendor/ggml/src/ggml-openvino/openvino/op/gather_matmul.hpp +43 -0
  845. data/vendor/ggml/src/ggml-openvino/openvino/op/get_rows.cpp +142 -0
  846. data/vendor/ggml/src/ggml-openvino/openvino/op/glu_geglu.cpp +75 -0
  847. data/vendor/ggml/src/ggml-openvino/openvino/op/glu_geglu_quick.cpp +64 -0
  848. data/vendor/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp +110 -0
  849. data/vendor/ggml/src/ggml-openvino/openvino/op/im2col.cpp +120 -0
  850. data/vendor/ggml/src/ggml-openvino/openvino/op/l2_norm.cpp +61 -0
  851. data/vendor/ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp +283 -0
  852. data/vendor/ggml/src/ggml-openvino/openvino/op/mulmat.cpp +100 -0
  853. data/vendor/ggml/src/ggml-openvino/openvino/op/norm.cpp +58 -0
  854. data/vendor/ggml/src/ggml-openvino/openvino/op/pad.cpp +95 -0
  855. data/vendor/ggml/src/ggml-openvino/openvino/op/permute.cpp +147 -0
  856. data/vendor/ggml/src/ggml-openvino/openvino/op/pool_2d.cpp +53 -0
  857. data/vendor/ggml/src/ggml-openvino/openvino/op/repeat.cpp +48 -0
  858. data/vendor/ggml/src/ggml-openvino/openvino/op/reshape.cpp +113 -0
  859. data/vendor/ggml/src/ggml-openvino/openvino/op/rms_norm.cpp +81 -0
  860. data/vendor/ggml/src/ggml-openvino/openvino/op/roll.cpp +36 -0
  861. data/vendor/ggml/src/ggml-openvino/openvino/op/rope.cpp +303 -0
  862. data/vendor/ggml/src/ggml-openvino/openvino/op/scale.cpp +86 -0
  863. data/vendor/ggml/src/ggml-openvino/openvino/op/set.cpp +76 -0
  864. data/vendor/ggml/src/ggml-openvino/openvino/op/set_rows.cpp +95 -0
  865. data/vendor/ggml/src/ggml-openvino/openvino/op/softmax.cpp +166 -0
  866. data/vendor/ggml/src/ggml-openvino/openvino/op/solve_tri.cpp +108 -0
  867. data/vendor/ggml/src/ggml-openvino/openvino/op/sqr.cpp +35 -0
  868. data/vendor/ggml/src/ggml-openvino/openvino/op/ssm_conv.cpp +61 -0
  869. data/vendor/ggml/src/ggml-openvino/openvino/op/sum_rows.cpp +27 -0
  870. data/vendor/ggml/src/ggml-openvino/openvino/op/transpose.cpp +54 -0
  871. data/vendor/ggml/src/ggml-openvino/openvino/op/tri.cpp +82 -0
  872. data/vendor/ggml/src/ggml-openvino/openvino/op/unary_silu.cpp +27 -0
  873. data/vendor/ggml/src/ggml-openvino/openvino/op/unary_softplus.cpp +38 -0
  874. data/vendor/ggml/src/ggml-openvino/openvino/op/view.cpp +245 -0
  875. data/vendor/ggml/src/ggml-openvino/openvino/op_table.cpp +88 -0
  876. data/vendor/ggml/src/ggml-openvino/openvino/op_table.h +67 -0
  877. data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_conv.cpp +212 -0
  878. data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_conv.h +17 -0
  879. data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_sdpa.cpp +60 -0
  880. data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_sdpa.h +17 -0
  881. data/vendor/ggml/src/ggml-openvino/openvino/pass/mark_decompression_convert_constant_folding.h +29 -0
  882. data/vendor/ggml/src/ggml-openvino/openvino/pass/mark_dequantization_subgraph.h +44 -0
  883. data/vendor/ggml/src/ggml-openvino/openvino/pass/squeeze_matmul.cpp +58 -0
  884. data/vendor/ggml/src/ggml-openvino/openvino/pass/squeeze_matmul.h +17 -0
  885. data/vendor/ggml/src/ggml-openvino/openvino/rt_info/weightless_caching_attributes.hpp +41 -0
  886. data/vendor/ggml/src/ggml-openvino/openvino/translate_session.cpp +432 -0
  887. data/vendor/ggml/src/ggml-openvino/openvino/translate_session.h +29 -0
  888. data/vendor/ggml/src/ggml-openvino/openvino/utils.cpp +886 -0
  889. data/vendor/ggml/src/ggml-openvino/openvino/utils.h +88 -0
  890. data/vendor/ggml/src/ggml-openvino/utils.cpp +1496 -0
  891. data/vendor/ggml/src/ggml-openvino/utils.h +195 -0
  892. data/vendor/ggml/src/ggml-opt.cpp +1094 -0
  893. data/vendor/ggml/src/ggml-quants.c +5667 -0
  894. data/vendor/ggml/src/ggml-quants.h +115 -0
  895. data/vendor/ggml/src/ggml-rpc/CMakeLists.txt +49 -0
  896. data/vendor/ggml/src/ggml-rpc/ggml-rpc.cpp +2351 -0
  897. data/vendor/ggml/src/ggml-rpc/transport-apple.cpp +481 -0
  898. data/vendor/ggml/src/ggml-rpc/transport-apple.h +27 -0
  899. data/vendor/ggml/src/ggml-rpc/transport.cpp +739 -0
  900. data/vendor/ggml/src/ggml-rpc/transport.h +38 -0
  901. data/vendor/ggml/src/ggml-sycl/CMakeLists.txt +218 -0
  902. data/vendor/ggml/src/ggml-sycl/add-id.cpp +81 -0
  903. data/vendor/ggml/src/ggml-sycl/add-id.hpp +8 -0
  904. data/vendor/ggml/src/ggml-sycl/backend.hpp +52 -0
  905. data/vendor/ggml/src/ggml-sycl/base.hpp +36 -0
  906. data/vendor/ggml/src/ggml-sycl/binbcast.cpp +650 -0
  907. data/vendor/ggml/src/ggml-sycl/binbcast.hpp +69 -0
  908. data/vendor/ggml/src/ggml-sycl/col2im-1d.cpp +102 -0
  909. data/vendor/ggml/src/ggml-sycl/col2im-1d.hpp +8 -0
  910. data/vendor/ggml/src/ggml-sycl/common.cpp +153 -0
  911. data/vendor/ggml/src/ggml-sycl/common.hpp +1029 -0
  912. data/vendor/ggml/src/ggml-sycl/concat.cpp +509 -0
  913. data/vendor/ggml/src/ggml-sycl/concat.hpp +20 -0
  914. data/vendor/ggml/src/ggml-sycl/conv.cpp +101 -0
  915. data/vendor/ggml/src/ggml-sycl/conv.hpp +20 -0
  916. data/vendor/ggml/src/ggml-sycl/conv2d-dw.cpp +170 -0
  917. data/vendor/ggml/src/ggml-sycl/conv2d-dw.hpp +10 -0
  918. data/vendor/ggml/src/ggml-sycl/conv2d-transpose.cpp +125 -0
  919. data/vendor/ggml/src/ggml-sycl/conv2d-transpose.hpp +10 -0
  920. data/vendor/ggml/src/ggml-sycl/conv2d.cpp +150 -0
  921. data/vendor/ggml/src/ggml-sycl/conv2d.hpp +10 -0
  922. data/vendor/ggml/src/ggml-sycl/conv3d.cpp +224 -0
  923. data/vendor/ggml/src/ggml-sycl/conv3d.hpp +8 -0
  924. data/vendor/ggml/src/ggml-sycl/convert.cpp +877 -0
  925. data/vendor/ggml/src/ggml-sycl/convert.hpp +64 -0
  926. data/vendor/ggml/src/ggml-sycl/count-equal.cpp +79 -0
  927. data/vendor/ggml/src/ggml-sycl/count-equal.hpp +9 -0
  928. data/vendor/ggml/src/ggml-sycl/cpy.cpp +1433 -0
  929. data/vendor/ggml/src/ggml-sycl/cpy.hpp +537 -0
  930. data/vendor/ggml/src/ggml-sycl/cross_entropy_loss.cpp +255 -0
  931. data/vendor/ggml/src/ggml-sycl/cross_entropy_loss.hpp +7 -0
  932. data/vendor/ggml/src/ggml-sycl/cumsum.cpp +148 -0
  933. data/vendor/ggml/src/ggml-sycl/cumsum.hpp +5 -0
  934. data/vendor/ggml/src/ggml-sycl/dequantize.hpp +1681 -0
  935. data/vendor/ggml/src/ggml-sycl/diag.cpp +67 -0
  936. data/vendor/ggml/src/ggml-sycl/diag.hpp +5 -0
  937. data/vendor/ggml/src/ggml-sycl/dmmv.cpp +2227 -0
  938. data/vendor/ggml/src/ggml-sycl/dmmv.hpp +27 -0
  939. data/vendor/ggml/src/ggml-sycl/dpct/helper.hpp +3782 -0
  940. data/vendor/ggml/src/ggml-sycl/dsv4-hc.cpp +280 -0
  941. data/vendor/ggml/src/ggml-sycl/dsv4-hc.hpp +10 -0
  942. data/vendor/ggml/src/ggml-sycl/element_wise.cpp +1432 -0
  943. data/vendor/ggml/src/ggml-sycl/element_wise.hpp +135 -0
  944. data/vendor/ggml/src/ggml-sycl/esimd.hpp +589 -0
  945. data/vendor/ggml/src/ggml-sycl/fattn-buffers.cpp +56 -0
  946. data/vendor/ggml/src/ggml-sycl/fattn-buffers.hpp +63 -0
  947. data/vendor/ggml/src/ggml-sycl/fattn-common.hpp +1185 -0
  948. data/vendor/ggml/src/ggml-sycl/fattn-mkl.cpp +689 -0
  949. data/vendor/ggml/src/ggml-sycl/fattn-onednn.cpp +450 -0
  950. data/vendor/ggml/src/ggml-sycl/fattn-onednn.hpp +18 -0
  951. data/vendor/ggml/src/ggml-sycl/fattn-tile.cpp +59 -0
  952. data/vendor/ggml/src/ggml-sycl/fattn-tile.hpp +1246 -0
  953. data/vendor/ggml/src/ggml-sycl/fattn-vec.hpp +684 -0
  954. data/vendor/ggml/src/ggml-sycl/fattn.cpp +452 -0
  955. data/vendor/ggml/src/ggml-sycl/fattn.hpp +42 -0
  956. data/vendor/ggml/src/ggml-sycl/fill.cpp +55 -0
  957. data/vendor/ggml/src/ggml-sycl/fill.hpp +5 -0
  958. data/vendor/ggml/src/ggml-sycl/fusion.cpp +212 -0
  959. data/vendor/ggml/src/ggml-sycl/fusion.hpp +17 -0
  960. data/vendor/ggml/src/ggml-sycl/fwht.cpp +119 -0
  961. data/vendor/ggml/src/ggml-sycl/fwht.hpp +12 -0
  962. data/vendor/ggml/src/ggml-sycl/gated_delta_net.cpp +366 -0
  963. data/vendor/ggml/src/ggml-sycl/gated_delta_net.hpp +19 -0
  964. data/vendor/ggml/src/ggml-sycl/gemm.hpp +93 -0
  965. data/vendor/ggml/src/ggml-sycl/getrows.cpp +368 -0
  966. data/vendor/ggml/src/ggml-sycl/getrows.hpp +20 -0
  967. data/vendor/ggml/src/ggml-sycl/ggml-sycl.cpp +7028 -0
  968. data/vendor/ggml/src/ggml-sycl/gla.cpp +106 -0
  969. data/vendor/ggml/src/ggml-sycl/gla.hpp +8 -0
  970. data/vendor/ggml/src/ggml-sycl/im2col.cpp +400 -0
  971. data/vendor/ggml/src/ggml-sycl/im2col.hpp +23 -0
  972. data/vendor/ggml/src/ggml-sycl/lightning-indexer.cpp +197 -0
  973. data/vendor/ggml/src/ggml-sycl/lightning-indexer.hpp +8 -0
  974. data/vendor/ggml/src/ggml-sycl/mem.cpp +162 -0
  975. data/vendor/ggml/src/ggml-sycl/mem.hpp +16 -0
  976. data/vendor/ggml/src/ggml-sycl/mmq.cpp +3030 -0
  977. data/vendor/ggml/src/ggml-sycl/mmq.hpp +33 -0
  978. data/vendor/ggml/src/ggml-sycl/mmvq.cpp +3033 -0
  979. data/vendor/ggml/src/ggml-sycl/mmvq.hpp +76 -0
  980. data/vendor/ggml/src/ggml-sycl/norm.cpp +963 -0
  981. data/vendor/ggml/src/ggml-sycl/norm.hpp +32 -0
  982. data/vendor/ggml/src/ggml-sycl/opt-step.cpp +131 -0
  983. data/vendor/ggml/src/ggml-sycl/opt-step.hpp +6 -0
  984. data/vendor/ggml/src/ggml-sycl/outprod.cpp +83 -0
  985. data/vendor/ggml/src/ggml-sycl/outprod.hpp +10 -0
  986. data/vendor/ggml/src/ggml-sycl/pad.cpp +97 -0
  987. data/vendor/ggml/src/ggml-sycl/pad.hpp +24 -0
  988. data/vendor/ggml/src/ggml-sycl/pad_reflect_1d.cpp +100 -0
  989. data/vendor/ggml/src/ggml-sycl/pad_reflect_1d.hpp +10 -0
  990. data/vendor/ggml/src/ggml-sycl/pool.cpp +185 -0
  991. data/vendor/ggml/src/ggml-sycl/pool.hpp +22 -0
  992. data/vendor/ggml/src/ggml-sycl/presets.hpp +79 -0
  993. data/vendor/ggml/src/ggml-sycl/quantize.hpp +133 -0
  994. data/vendor/ggml/src/ggml-sycl/quants.hpp +204 -0
  995. data/vendor/ggml/src/ggml-sycl/repeat_back.cpp +76 -0
  996. data/vendor/ggml/src/ggml-sycl/repeat_back.hpp +8 -0
  997. data/vendor/ggml/src/ggml-sycl/roll.cpp +122 -0
  998. data/vendor/ggml/src/ggml-sycl/roll.hpp +20 -0
  999. data/vendor/ggml/src/ggml-sycl/rope.cpp +651 -0
  1000. data/vendor/ggml/src/ggml-sycl/rope.hpp +26 -0
  1001. data/vendor/ggml/src/ggml-sycl/set.cpp +73 -0
  1002. data/vendor/ggml/src/ggml-sycl/set.hpp +5 -0
  1003. data/vendor/ggml/src/ggml-sycl/set_rows.cpp +577 -0
  1004. data/vendor/ggml/src/ggml-sycl/set_rows.hpp +8 -0
  1005. data/vendor/ggml/src/ggml-sycl/softmax.cpp +425 -0
  1006. data/vendor/ggml/src/ggml-sycl/softmax.hpp +24 -0
  1007. data/vendor/ggml/src/ggml-sycl/solve_tri.cpp +172 -0
  1008. data/vendor/ggml/src/ggml-sycl/solve_tri.hpp +8 -0
  1009. data/vendor/ggml/src/ggml-sycl/ssm_conv.cpp +136 -0
  1010. data/vendor/ggml/src/ggml-sycl/ssm_conv.hpp +5 -0
  1011. data/vendor/ggml/src/ggml-sycl/ssm_scan.cpp +170 -0
  1012. data/vendor/ggml/src/ggml-sycl/ssm_scan.hpp +5 -0
  1013. data/vendor/ggml/src/ggml-sycl/sycl_hw.cpp +67 -0
  1014. data/vendor/ggml/src/ggml-sycl/sycl_hw.hpp +38 -0
  1015. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq112-dv112.cpp +5 -0
  1016. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq128-dv128.cpp +5 -0
  1017. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq256-dv256.cpp +5 -0
  1018. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq40-dv40.cpp +5 -0
  1019. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq512-dv512.cpp +6 -0
  1020. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq576-dv512.cpp +5 -0
  1021. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq64-dv64.cpp +5 -0
  1022. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq72-dv72.cpp +5 -0
  1023. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq80-dv80.cpp +5 -0
  1024. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq96-dv96.cpp +5 -0
  1025. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-f16.cpp +8 -0
  1026. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q4_0.cpp +8 -0
  1027. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q4_1.cpp +8 -0
  1028. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q5_0.cpp +8 -0
  1029. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q5_1.cpp +8 -0
  1030. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q8_0.cpp +8 -0
  1031. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-f16.cpp +8 -0
  1032. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q4_0.cpp +8 -0
  1033. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q4_1.cpp +8 -0
  1034. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q5_0.cpp +8 -0
  1035. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q5_1.cpp +8 -0
  1036. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q8_0.cpp +8 -0
  1037. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-f16.cpp +8 -0
  1038. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q4_0.cpp +8 -0
  1039. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q4_1.cpp +8 -0
  1040. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q5_0.cpp +8 -0
  1041. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q5_1.cpp +8 -0
  1042. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q8_0.cpp +8 -0
  1043. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-f16.cpp +8 -0
  1044. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q4_0.cpp +8 -0
  1045. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q4_1.cpp +8 -0
  1046. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q5_0.cpp +8 -0
  1047. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q5_1.cpp +8 -0
  1048. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q8_0.cpp +8 -0
  1049. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-f16.cpp +8 -0
  1050. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q4_0.cpp +8 -0
  1051. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q4_1.cpp +8 -0
  1052. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q5_0.cpp +8 -0
  1053. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q5_1.cpp +8 -0
  1054. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q8_0.cpp +8 -0
  1055. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-f16.cpp +8 -0
  1056. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q4_0.cpp +8 -0
  1057. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q4_1.cpp +8 -0
  1058. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q5_0.cpp +8 -0
  1059. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q5_1.cpp +8 -0
  1060. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q8_0.cpp +8 -0
  1061. data/vendor/ggml/src/ggml-sycl/topk-moe.cpp +620 -0
  1062. data/vendor/ggml/src/ggml-sycl/topk-moe.hpp +12 -0
  1063. data/vendor/ggml/src/ggml-sycl/tsembd.cpp +73 -0
  1064. data/vendor/ggml/src/ggml-sycl/tsembd.hpp +20 -0
  1065. data/vendor/ggml/src/ggml-sycl/type.hpp +112 -0
  1066. data/vendor/ggml/src/ggml-sycl/upscale.cpp +410 -0
  1067. data/vendor/ggml/src/ggml-sycl/upscale.hpp +9 -0
  1068. data/vendor/ggml/src/ggml-sycl/vecdotq.hpp +1726 -0
  1069. data/vendor/ggml/src/ggml-sycl/wkv.cpp +293 -0
  1070. data/vendor/ggml/src/ggml-sycl/wkv.hpp +10 -0
  1071. data/vendor/ggml/src/ggml-threading.cpp +12 -0
  1072. data/vendor/ggml/src/ggml-threading.h +14 -0
  1073. data/vendor/ggml/src/ggml-version.h.in +4 -0
  1074. data/vendor/ggml/src/ggml-virtgpu/CMakeLists.txt +70 -0
  1075. data/vendor/ggml/src/ggml-virtgpu/apir_cs_ggml-rpc-front.cpp +87 -0
  1076. data/vendor/ggml/src/ggml-virtgpu/backend/CMakeLists.txt +21 -0
  1077. data/vendor/ggml/src/ggml-virtgpu/backend/apir_cs_ggml-rpc-back.cpp +115 -0
  1078. data/vendor/ggml/src/ggml-virtgpu/backend/backend-convert.h +13 -0
  1079. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-backend.cpp +102 -0
  1080. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-buffer-type.cpp +105 -0
  1081. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-buffer.cpp +179 -0
  1082. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-device.cpp +149 -0
  1083. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched.cpp +51 -0
  1084. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched.gen.h +73 -0
  1085. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched.h +27 -0
  1086. data/vendor/ggml/src/ggml-virtgpu/backend/backend-virgl-apir.h +32 -0
  1087. data/vendor/ggml/src/ggml-virtgpu/backend/backend.cpp +144 -0
  1088. data/vendor/ggml/src/ggml-virtgpu/backend/shared/api_remoting.h +95 -0
  1089. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_backend.gen.h +94 -0
  1090. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_backend.h +50 -0
  1091. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_cs.h +378 -0
  1092. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_cs_ggml.h +232 -0
  1093. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_cs_rpc.h +58 -0
  1094. data/vendor/ggml/src/ggml-virtgpu/ggml-backend-buffer-type.cpp +81 -0
  1095. data/vendor/ggml/src/ggml-virtgpu/ggml-backend-buffer.cpp +123 -0
  1096. data/vendor/ggml/src/ggml-virtgpu/ggml-backend-device.cpp +160 -0
  1097. data/vendor/ggml/src/ggml-virtgpu/ggml-backend-reg.cpp +213 -0
  1098. data/vendor/ggml/src/ggml-virtgpu/ggml-backend.cpp +72 -0
  1099. data/vendor/ggml/src/ggml-virtgpu/ggml-remoting.h +71 -0
  1100. data/vendor/ggml/src/ggml-virtgpu/ggmlremoting_functions.yaml +166 -0
  1101. data/vendor/ggml/src/ggml-virtgpu/include/apir_hw.h +9 -0
  1102. data/vendor/ggml/src/ggml-virtgpu/regenerate_remoting.py +333 -0
  1103. data/vendor/ggml/src/ggml-virtgpu/virtgpu-apir.h +15 -0
  1104. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-backend.cpp +58 -0
  1105. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-buffer-type.cpp +110 -0
  1106. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-buffer.cpp +173 -0
  1107. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-device.cpp +194 -0
  1108. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-impl.h +36 -0
  1109. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward.gen.h +54 -0
  1110. data/vendor/ggml/src/ggml-virtgpu/virtgpu-shm.cpp +99 -0
  1111. data/vendor/ggml/src/ggml-virtgpu/virtgpu-shm.h +23 -0
  1112. data/vendor/ggml/src/ggml-virtgpu/virtgpu-utils.cpp +179 -0
  1113. data/vendor/ggml/src/ggml-virtgpu/virtgpu-utils.h +86 -0
  1114. data/vendor/ggml/src/ggml-virtgpu/virtgpu.cpp +545 -0
  1115. data/vendor/ggml/src/ggml-virtgpu/virtgpu.h +115 -0
  1116. data/vendor/ggml/src/ggml-vulkan/CMakeLists.txt +254 -0
  1117. data/vendor/ggml/src/ggml-vulkan/cmake/host-toolchain.cmake.in +15 -0
  1118. data/vendor/ggml/src/ggml-vulkan/ggml-vulkan.cpp +20349 -0
  1119. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/CMakeLists.txt +43 -0
  1120. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/acc.comp +37 -0
  1121. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/add.comp +69 -0
  1122. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/add1.comp +28 -0
  1123. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/add_id.comp +42 -0
  1124. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/arange.comp +20 -0
  1125. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/argmax.comp +60 -0
  1126. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/argsort.comp +86 -0
  1127. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/argsort_large.comp +114 -0
  1128. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/col2im_1d.comp +61 -0
  1129. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/concat.comp +41 -0
  1130. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/contig_copy.comp +53 -0
  1131. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv2d_dw.comp +105 -0
  1132. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv2d_mm.comp +473 -0
  1133. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv3d_mm.comp +424 -0
  1134. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv_transpose_1d.comp +98 -0
  1135. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy.comp +25 -0
  1136. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_from_quant.comp +51 -0
  1137. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_to_quant.comp +350 -0
  1138. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_transpose.comp +67 -0
  1139. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_transpose_02.comp +61 -0
  1140. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/count_equal.comp +31 -0
  1141. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/count_experts.comp +135 -0
  1142. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cross_entropy_loss.comp +78 -0
  1143. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cross_entropy_loss_back.comp +75 -0
  1144. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cumsum.comp +83 -0
  1145. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cumsum_multipass1.comp +60 -0
  1146. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cumsum_multipass2.comp +66 -0
  1147. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_f32.comp +20 -0
  1148. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.glsl +741 -0
  1149. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl +1466 -0
  1150. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_head.glsl +13 -0
  1151. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_m.comp +42 -0
  1152. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_s.comp +35 -0
  1153. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_s.comp +44 -0
  1154. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xs.comp +43 -0
  1155. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xxs.comp +49 -0
  1156. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_s.comp +40 -0
  1157. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_xxs.comp +51 -0
  1158. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_nl.comp +32 -0
  1159. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_xs.comp +34 -0
  1160. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_mxfp4.comp +32 -0
  1161. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_nvfp4.comp +32 -0
  1162. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q1_0.comp +29 -0
  1163. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q2_0.comp +29 -0
  1164. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q2_k.comp +34 -0
  1165. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q3_k.comp +42 -0
  1166. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_0.comp +30 -0
  1167. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_1.comp +32 -0
  1168. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_k.comp +68 -0
  1169. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_0.comp +34 -0
  1170. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_1.comp +35 -0
  1171. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_k.comp +70 -0
  1172. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q6_k.comp +33 -0
  1173. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q8_0.comp +42 -0
  1174. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_tq2_0.comp +31 -0
  1175. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/diag.comp +28 -0
  1176. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/diag_mask_inf.comp +34 -0
  1177. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/div.comp +27 -0
  1178. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dot_product_funcs.glsl +27 -0
  1179. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/fa_types.glsl +55 -0
  1180. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/bfloat16.comp +7 -0
  1181. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat.comp +7 -0
  1182. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat2.comp +7 -0
  1183. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat2_decode_vector.comp +7 -0
  1184. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/float_e2m1.comp +7 -0
  1185. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/float_e4m3.comp +7 -0
  1186. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/integer_dot.comp +7 -0
  1187. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/fill.comp +19 -0
  1188. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp +760 -0
  1189. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.glsl +223 -0
  1190. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp +647 -0
  1191. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp +487 -0
  1192. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_dequant.glsl +146 -0
  1193. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_mask_opt.comp +162 -0
  1194. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_mmq_funcs.glsl +203 -0
  1195. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_split_k_reduce.comp +121 -0
  1196. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/fwht.comp +115 -0
  1197. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/gated_delta_net.comp +189 -0
  1198. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/geglu.comp +13 -0
  1199. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/geglu_erf.comp +27 -0
  1200. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/geglu_quick.comp +11 -0
  1201. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/generic_binary_head.glsl +65 -0
  1202. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/generic_head.glsl +11 -0
  1203. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/generic_unary_head.glsl +75 -0
  1204. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/get_rows.comp +42 -0
  1205. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_back.comp +25 -0
  1206. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_quant.comp +51 -0
  1207. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/gla.comp +82 -0
  1208. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/glu_head.glsl +43 -0
  1209. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/glu_main.glsl +35 -0
  1210. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/group_norm.comp +66 -0
  1211. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/im2col.comp +138 -0
  1212. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/im2col_3d.comp +124 -0
  1213. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/l2_norm.comp +41 -0
  1214. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/lightning_indexer.comp +151 -0
  1215. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/log.comp +17 -0
  1216. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul.comp +27 -0
  1217. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_split_k_reduce.comp +48 -0
  1218. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec.comp +264 -0
  1219. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_base.glsl +230 -0
  1220. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iface.glsl +35 -0
  1221. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_m.comp +132 -0
  1222. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_s.comp +95 -0
  1223. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_s.comp +90 -0
  1224. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xs.comp +105 -0
  1225. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xxs.comp +87 -0
  1226. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_s.comp +93 -0
  1227. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_xxs.comp +88 -0
  1228. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_nc.comp +124 -0
  1229. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_p021.comp +156 -0
  1230. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q2_k.comp +128 -0
  1231. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q3_k.comp +132 -0
  1232. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q4_k.comp +134 -0
  1233. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q5_k.comp +165 -0
  1234. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q6_k.comp +130 -0
  1235. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_tq2_0.comp +102 -0
  1236. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq.comp +141 -0
  1237. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq_funcs.glsl +564 -0
  1238. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp +475 -0
  1239. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp +690 -0
  1240. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl +670 -0
  1241. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_id_funcs.glsl +89 -0
  1242. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp +317 -0
  1243. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_funcs.glsl +488 -0
  1244. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_shmem_types.glsl +99 -0
  1245. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/multi_add.comp +194 -0
  1246. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/norm.comp +44 -0
  1247. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_adamw.comp +42 -0
  1248. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_sgd.comp +22 -0
  1249. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/out_prod.comp +59 -0
  1250. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pad.comp +64 -0
  1251. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pad_reflect_1d.comp +43 -0
  1252. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pool1d.comp +65 -0
  1253. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pool2d.comp +74 -0
  1254. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/quantize_q8_1.comp +127 -0
  1255. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/reglu.comp +9 -0
  1256. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/repeat.comp +26 -0
  1257. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/repeat_back.comp +37 -0
  1258. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp +150 -0
  1259. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_back.comp +55 -0
  1260. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_partials.comp +65 -0
  1261. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/roll.comp +46 -0
  1262. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_funcs.glsl +218 -0
  1263. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_head.glsl +19 -0
  1264. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_multi.comp +17 -0
  1265. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_neox.comp +17 -0
  1266. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_norm.comp +17 -0
  1267. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_params.glsl +35 -0
  1268. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_vision.comp +17 -0
  1269. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/scale.comp +24 -0
  1270. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/silu_back.comp +26 -0
  1271. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/snake.comp +49 -0
  1272. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max.comp +195 -0
  1273. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_back.comp +54 -0
  1274. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large1.comp +62 -0
  1275. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large2.comp +79 -0
  1276. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large3.comp +65 -0
  1277. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large_common.glsl +53 -0
  1278. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/solve_tri.comp +81 -0
  1279. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/ssm_conv.comp +60 -0
  1280. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/ssm_scan.comp +134 -0
  1281. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/sub.comp +29 -0
  1282. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.comp +47 -0
  1283. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.glsl +19 -0
  1284. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/swiglu.comp +9 -0
  1285. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/swiglu_clamp.comp +12 -0
  1286. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/swiglu_oai.comp +14 -0
  1287. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/timestep_embedding.comp +42 -0
  1288. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_argsort.comp +118 -0
  1289. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_moe.comp +221 -0
  1290. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_nary_search.comp +246 -0
  1291. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_radix_select.comp +144 -0
  1292. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/tri.comp +42 -0
  1293. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl +1938 -0
  1294. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/unary.comp +168 -0
  1295. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/upscale.comp +178 -0
  1296. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/utils.glsl +37 -0
  1297. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +1367 -0
  1298. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/wkv6.comp +87 -0
  1299. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/wkv7.comp +91 -0
  1300. data/vendor/ggml/src/ggml-webgpu/CMakeLists.txt +83 -0
  1301. data/vendor/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp +3457 -0
  1302. data/vendor/ggml/src/ggml-webgpu/ggml-webgpu.cpp +4790 -0
  1303. data/vendor/ggml/src/ggml-webgpu/pre_wgsl.hpp +808 -0
  1304. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/add_id.wgsl +64 -0
  1305. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/argmax.wgsl +72 -0
  1306. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/argsort.wgsl +102 -0
  1307. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/argsort_merge.wgsl +134 -0
  1308. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/binary.wgsl +142 -0
  1309. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/common_decls.tmpl +967 -0
  1310. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/concat.wgsl +92 -0
  1311. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/conv2d.wgsl +127 -0
  1312. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/conv2d_dw.wgsl +100 -0
  1313. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/cpy.wgsl +84 -0
  1314. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/cumsum.wgsl +66 -0
  1315. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/embed_wgsl.py +89 -0
  1316. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn.wgsl +408 -0
  1317. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_decls.tmpl +134 -0
  1318. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_staging.tmpl +136 -0
  1319. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_tile.wgsl +240 -0
  1320. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_blk.wgsl +101 -0
  1321. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_reduce.wgsl +84 -0
  1322. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_split.wgsl +503 -0
  1323. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/gated_delta_net.wgsl +149 -0
  1324. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/get_rows.wgsl +794 -0
  1325. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/glu.wgsl +178 -0
  1326. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/im2col.wgsl +77 -0
  1327. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/memset.wgsl +40 -0
  1328. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_decls.tmpl +1052 -0
  1329. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id.wgsl +195 -0
  1330. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id_gather.wgsl +52 -0
  1331. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id_vec.wgsl +154 -0
  1332. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_reg_tile.wgsl +159 -0
  1333. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_subgroup_matrix.wgsl +210 -0
  1334. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec.wgsl +167 -0
  1335. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec_acc.tmpl +1560 -0
  1336. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec_q_acc.tmpl +297 -0
  1337. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/pad.wgsl +86 -0
  1338. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/quant_inner_loops.tmpl +21 -0
  1339. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/quantize_q8.wgsl +178 -0
  1340. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/repeat.wgsl +70 -0
  1341. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/rms_norm_mul.wgsl +151 -0
  1342. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/rope.wgsl +228 -0
  1343. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/row_norm.wgsl +152 -0
  1344. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/scale.wgsl +65 -0
  1345. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/set.wgsl +109 -0
  1346. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/set_rows.wgsl +108 -0
  1347. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/set_rows_quant.wgsl +224 -0
  1348. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/soft_max.wgsl +210 -0
  1349. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/solve_tri.wgsl +120 -0
  1350. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/ssm_conv.wgsl +65 -0
  1351. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/ssm_scan.wgsl +247 -0
  1352. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/sum_rows.wgsl +55 -0
  1353. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/unary.wgsl +213 -0
  1354. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/upscale.wgsl +240 -0
  1355. data/vendor/ggml/src/ggml-zdnn/CMakeLists.txt +36 -0
  1356. data/vendor/ggml/src/ggml-zdnn/common.hpp +59 -0
  1357. data/vendor/ggml/src/ggml-zdnn/ggml-zdnn.cpp +638 -0
  1358. data/vendor/ggml/src/ggml-zdnn/mmf.cpp +80 -0
  1359. data/vendor/ggml/src/ggml-zdnn/mmf.hpp +12 -0
  1360. data/vendor/ggml/src/ggml-zdnn/utils.cpp +79 -0
  1361. data/vendor/ggml/src/ggml-zdnn/utils.hpp +19 -0
  1362. data/vendor/ggml/src/ggml-zendnn/CMakeLists.txt +91 -0
  1363. data/vendor/ggml/src/ggml-zendnn/ggml-zendnn.cpp +838 -0
  1364. data/vendor/ggml/src/ggml.c +8088 -0
  1365. data/vendor/ggml/src/ggml.cpp +26 -0
  1366. data/vendor/ggml/src/gguf.cpp +1706 -0
  1367. metadata +1459 -0
@@ -0,0 +1,2663 @@
1
+ //------------------------------------------------------------------------------
2
+ // This file is contains kernels for data conversion.
3
+ // These kernels are used when loading the model, so its performance is less
4
+ // important.
5
+ //------------------------------------------------------------------------------
6
+ #pragma OPENCL EXTENSION cl_khr_fp16 : enable
7
+
8
+ #ifdef cl_intel_required_subgroup_size
9
+ #pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
10
+ #define INTEL_GPU 1
11
+ #define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
12
+ #define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
13
+ #elif defined(cl_qcom_reqd_sub_group_size)
14
+ #pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
15
+ #define ADRENO_GPU 1
16
+ #define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half")))
17
+ #define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
18
+ #endif
19
+
20
+ #define QK4_0 32
21
+ #define QR4_0 2
22
+ #define QK4_1 32
23
+ #define QR4_1 2
24
+ #define QK5_0 32
25
+ #define QR5_0 2
26
+ #define QK5_1 32
27
+ #define QR5_1 2
28
+ #define QK8_0 32
29
+ #define QR8_0 1
30
+ #define QK1_0 128
31
+ #define QR1_0 1
32
+ #define QK_K 256
33
+ #define K_SCALE_SIZE (3 * QK_K / 64)
34
+ #define K_QUANTS_PER_ITERATION 2
35
+
36
+ typedef char int8_t;
37
+ typedef uchar uint8_t;
38
+ typedef short int16_t;
39
+ typedef ushort uint16_t;
40
+ typedef int int32_t;
41
+ typedef uint uint32_t;
42
+
43
+ //------------------------------------------------------------------------------
44
+ // block_q1_0
45
+ //------------------------------------------------------------------------------
46
+ typedef struct {
47
+ half d; // delta
48
+ uchar qs[QK1_0/8]; // 1-bit signs (16 bytes)
49
+ } block_q1_0;
50
+
51
+ //------------------------------------------------------------------------------
52
+ // block_q4_0
53
+ //------------------------------------------------------------------------------
54
+ struct block_q4_0
55
+ {
56
+ half d;
57
+ uint8_t qs[QK4_0 / 2];
58
+ };
59
+
60
+ //------------------------------------------------------------------------------
61
+ // block_q4_1
62
+ //------------------------------------------------------------------------------
63
+ struct block_q4_1 {
64
+ half d; // delta
65
+ half m; // min
66
+ uchar qs[QK4_1 / 2]; // nibbles / quants
67
+ };
68
+
69
+ //------------------------------------------------------------------------------
70
+ // block_q5_0
71
+ //------------------------------------------------------------------------------
72
+ struct block_q5_0 {
73
+ half d; // delta
74
+ uchar qh[4]; // 5-th bit of quants
75
+ uchar qs[QK5_0 / 2]; // nibbles / quants
76
+ };
77
+
78
+ //------------------------------------------------------------------------------
79
+ // block_q5_1
80
+ //------------------------------------------------------------------------------
81
+ struct block_q5_1 {
82
+ half d; // delta
83
+ half m; // min
84
+ uchar qh[4]; // 5-th bit of quants
85
+ uchar qs[QK5_1 / 2]; // nibbles / quants
86
+ };
87
+
88
+ //------------------------------------------------------------------------------
89
+ // block_q4_k
90
+ //------------------------------------------------------------------------------
91
+ struct block_q4_K {
92
+ half d; // delta
93
+ half dm; // min
94
+ uchar s[K_SCALE_SIZE];
95
+ uchar q[QK_K / 2]; // nibbles / quants
96
+ };
97
+
98
+ //------------------------------------------------------------------------------
99
+ // block_q5_k
100
+ //------------------------------------------------------------------------------
101
+ struct block_q5_K {
102
+ half d; // delta
103
+ half dm; // min
104
+ uchar s[K_SCALE_SIZE];
105
+ uchar qh[QK_K / 8];
106
+ uchar qs[QK_K / 2]; // nibbles / quants
107
+ };
108
+
109
+ //------------------------------------------------------------------------------
110
+ // block_q6_K
111
+ //------------------------------------------------------------------------------
112
+ struct block_q6_K {
113
+ uint8_t ql[QK_K/2]; // quants, lower 4 bits
114
+ uint8_t qh[QK_K/4]; // quants, upper 2 bits
115
+ int8_t scales[QK_K/16]; // scales, quantized with 8 bits
116
+ half d; // super-block scale
117
+ };
118
+
119
+ //------------------------------------------------------------------------------
120
+ // block_iq4_nl
121
+ //------------------------------------------------------------------------------
122
+ #define QK4_NL 32
123
+
124
+ struct block_iq4_nl
125
+ {
126
+ half d;
127
+ uint8_t qs[QK4_NL / 2];
128
+ };
129
+
130
+ //------------------------------------------------------------------------------
131
+ // bf16 to f16
132
+ //------------------------------------------------------------------------------
133
+ kernel void kernel_convert_bf16_to_f16(
134
+ global const ushort * src,
135
+ global half * dst,
136
+ ulong off_dst,
137
+ ulong n
138
+ ) {
139
+ uint i = get_global_id(0);
140
+ if (i >= n) {
141
+ return;
142
+ }
143
+
144
+ dst[i + off_dst] = (half) as_float((uint) src[i] << 16);
145
+ }
146
+
147
+ //------------------------------------------------------------------------------
148
+ // f16 to bf16
149
+ //------------------------------------------------------------------------------
150
+ kernel void kernel_convert_f16_to_bf16(
151
+ global const half * src,
152
+ ulong off_src,
153
+ global ushort * dst,
154
+ ulong n
155
+ ) {
156
+ uint i = get_global_id(0);
157
+ if (i >= n) {
158
+ return;
159
+ }
160
+
161
+ float f = (float) src[i + off_src];
162
+ uint bits = as_uint(f);
163
+ if ((bits & 0x7fffffffu) > 0x7f800000u) {
164
+ // nan to quiet nan
165
+ dst[i] = (ushort)((bits >> 16) | 0x40u);
166
+ } else {
167
+ uint rounded = bits + 0x7fffu + ((bits >> 16) & 1u);
168
+ dst[i] = (ushort)(rounded >> 16);
169
+ }
170
+ }
171
+
172
+ //------------------------------------------------------------------------------
173
+ // kernel_convert_block_q1_0
174
+ // Convert block_q1_0 (AOS) to 2 separate arrays (SOA): quant bytes + scales.
175
+ // q1_0 bits are stored in natural order (bit j of byte i -> weight 8*i + j)
176
+ //------------------------------------------------------------------------------
177
+ kernel void kernel_convert_block_q1_0(
178
+ global block_q1_0 * src0,
179
+ global uchar * dst_q,
180
+ global half * dst_d
181
+ ) {
182
+ global block_q1_0 * b = (global block_q1_0 *) src0 + get_global_id(0);
183
+ global uchar * q = (global uchar *) dst_q + (QK1_0/8)*get_global_id(0);
184
+ global half * d = (global half *) dst_d + get_global_id(0);
185
+
186
+ *d = b->d;
187
+
188
+ for (int i = 0; i < QK1_0/8; ++i) {
189
+ q[i] = b->qs[i];
190
+ }
191
+ }
192
+
193
+ kernel void kernel_restore_block_q1_0(
194
+ global uchar * src_q,
195
+ global half * src_d,
196
+ global block_q1_0 * dst
197
+ ) {
198
+ global block_q1_0 * b = (global block_q1_0 *) dst + get_global_id(0);
199
+ global uchar * q = (global uchar *) src_q + (QK1_0/8)*get_global_id(0);
200
+ global half * d = (global half *) src_d + get_global_id(0);
201
+
202
+ b->d = *d;
203
+ for (int i = 0; i < QK1_0/8; ++i) {
204
+ b->qs[i] = q[i];
205
+ }
206
+ }
207
+
208
+ //------------------------------------------------------------------------------
209
+ // kernel_convert_block_q4_0
210
+ // Convert the block_q4_0 format to 2 separate arrays (AOS -> SOA).
211
+ // This kernel does not deshuffle the bits.
212
+ //------------------------------------------------------------------------------
213
+ kernel void kernel_convert_block_q4_0(
214
+ global struct block_q4_0 * src0,
215
+ global uchar * dst_q,
216
+ global half * dst_d
217
+ ) {
218
+ global struct block_q4_0 * b = (global struct block_q4_0 *) src0 + get_global_id(0);
219
+ global uchar * q = (global uchar *) dst_q + QK4_0/2*get_global_id(0);
220
+ global half * d = (global half *) dst_d + get_global_id(0);
221
+
222
+ *d = b->d;
223
+
224
+ for (int i = 0; i < QK4_0/2; ++i) {
225
+ q[i] = b->qs[i];
226
+ }
227
+ }
228
+
229
+ kernel void kernel_restore_block_q4_0(
230
+ global uchar * src_q,
231
+ global half * src_d,
232
+ global struct block_q4_0 * dst
233
+ ) {
234
+ global struct block_q4_0 * b = (global struct block_q4_0 *) dst + get_global_id(0);
235
+ global uchar * q = (global uchar *) src_q + QK4_0/2*get_global_id(0);
236
+ global half * d = (global half *) src_d + get_global_id(0);
237
+
238
+ b->d = *d;
239
+ for (int i = 0; i < QK4_0/2; ++i) {
240
+ b->qs[i] = q[i];
241
+ }
242
+ }
243
+
244
+ //------------------------------------------------------------------------------
245
+ // kernel_convert_block_q4_0_noshuffle
246
+ // Flatten q4_0 weights and unshuffle the bits
247
+ //------------------------------------------------------------------------------
248
+
249
+ kernel void kernel_convert_block_q4_0_noshuffle(
250
+ global struct block_q4_0 * src0,
251
+ global uchar * dst_q,
252
+ global half * dst_d
253
+ ) {
254
+ global struct block_q4_0 * b = (global struct block_q4_0 *) src0 + get_global_id(0);
255
+ global uchar * q = (global uchar *) dst_q + QK4_0/2*get_global_id(0);
256
+ global half * d = (global half *) dst_d + get_global_id(0);
257
+
258
+ *d = b->d;
259
+ for (int i = 0; i < QK4_0/4; ++i) {
260
+ uchar x0 = b->qs[2*i + 0];
261
+ uchar x1 = b->qs[2*i + 1];
262
+
263
+ q[i + 0 ] = convert_uchar(x0 & 0x0F) | convert_uchar((x1 & 0x0F) << 4);
264
+ q[i + QK4_0/4] = convert_uchar((x0 & 0xF0) >> 4) | convert_uchar(x1 & 0xF0);
265
+
266
+ #ifdef ADRENO_GPU
267
+ // Workaround for adreno - must have the following printf statement for
268
+ // the kernel to work properly. Otherwise it produces incorrect result.
269
+ // convert_uchar above also seems necessary.
270
+ // Compare against a large number so that it does not print anything.
271
+ // get_sub_group_local_id() also works.
272
+ if (get_global_id(0) == 65536*4096) {
273
+ printf("%04x - %02x\n", *(global ushort*)d, ((x0 & 0xF0) >> 4) | (x1 & 0xF0));
274
+ }
275
+ #endif
276
+ }
277
+ }
278
+
279
+ kernel void kernel_restore_block_q4_0_noshuffle(
280
+ global uchar * src_q,
281
+ global half * src_d,
282
+ global struct block_q4_0 * dst,
283
+ uchar mask_0F,
284
+ uchar mask_F0
285
+ ) {
286
+ global struct block_q4_0 * b = (global struct block_q4_0 *) dst + get_global_id(0);
287
+ global uchar * q = (global uchar *) src_q + QK4_0/2*get_global_id(0);
288
+ global half * d = (global half *) src_d + get_global_id(0);
289
+
290
+ b->d = *d;
291
+ for (int i = 0; i < QK4_0/4; ++i) {
292
+ uchar x0 = q[i + 0 ] ;
293
+ uchar x1 = q[i + QK4_0/4];
294
+
295
+ b->qs[2*i + 0] = convert_uchar((x0 & mask_0F) | ((x1 & mask_0F) << 4));
296
+ b->qs[2*i + 1] = convert_uchar(((x0 & mask_F0) >> 4) | (x1 & mask_F0));
297
+ }
298
+ }
299
+
300
+ kernel void kernel_convert_block_q4_0_trans4_ns(
301
+ global struct block_q4_0 * src0,
302
+ __global uint * dst_q,
303
+ __global half * dst_d,
304
+ uint ne00,
305
+ uint ne01
306
+ ) {
307
+ uint i00 = get_global_id(1);
308
+ uint i01 = get_global_id(0);
309
+ uint i02 = get_global_id(2);
310
+
311
+ if (i01 >= ne01) {
312
+ return;
313
+ }
314
+
315
+ uint ne00_blk = ne00 / QK4_0;
316
+ uint src_blk_offset = i00 + i01 * ne00_blk + i02 * ne00_blk * ne01;
317
+ uint dst_blk_offset = i01 + i00 * ne01 + i02 * ne00_blk * ne01;
318
+
319
+ global struct block_q4_0 * b = src0 + src_blk_offset;
320
+ dst_d[dst_blk_offset] = b->d;
321
+
322
+ // extract quantization and unshuffle
323
+ ushort8 pre_block = ((global ushort8 *)(&(b->qs[0])))[0];
324
+
325
+ ushort8 post_block = (ushort8)(0);
326
+
327
+ uchar * pre_block_ptr = (uchar *)(&pre_block);
328
+ uchar * post_block_ptr = (uchar *)(&post_block);
329
+
330
+ for (int i = 0; i < QK4_0 / 4; ++i) {
331
+ uchar x0 = pre_block_ptr[2*i + 0];
332
+ uchar x1 = pre_block_ptr[2*i + 1];
333
+
334
+ post_block_ptr[i + 0 ] = convert_uchar(x0 & 0x0F) | convert_uchar((x1 & 0x0F) << 4);
335
+ post_block_ptr[i + QK4_0 / 4] = convert_uchar((x0 & 0xF0) >> 4) | convert_uchar(x1 & 0xF0);
336
+ }
337
+
338
+ uint4 q_block = as_uint4(post_block);
339
+
340
+ uint offset = i02 * ne00_blk * ne01 * 4 + i00 * ne01 * 4 + i01;
341
+ dst_q[offset] = q_block.x;
342
+ dst_q[offset + ne01] = q_block.y;
343
+ dst_q[offset + ne01 * 2] = q_block.z;
344
+ dst_q[offset + ne01 * 3] = q_block.w;
345
+ }
346
+
347
+ kernel void kernel_restore_block_q4_0_trans4_ns(
348
+ __global uint * src_q,
349
+ __global half * src_d,
350
+ __global struct block_q4_0 * dst0,
351
+ uint ne00,
352
+ uint ne01
353
+ ) {
354
+ uint i00 = get_global_id(1);
355
+ uint i01 = get_global_id(0);
356
+ uint i02 = get_global_id(2);
357
+
358
+ if (i01 >= ne01) {
359
+ return;
360
+ }
361
+
362
+ uint ne00_blk = ne00 / QK4_0;
363
+ uint dst_blk_offset = i00 + i01 * ne00_blk + i02 * ne00_blk * ne01;
364
+ uint src_d_offset = i01 + i00 * ne01 + i02 * ne00_blk * ne01;
365
+
366
+ __global struct block_q4_0 * b = dst0 + dst_blk_offset;
367
+ b->d = src_d[src_d_offset];
368
+
369
+ // collect transposed quantization parts for a block
370
+ uint src_q_offset = i02 * ne00_blk * ne01 * 4 + i00 * ne01 * 4 + i01;
371
+ uint4 q_block;
372
+ q_block.x = src_q[src_q_offset];
373
+ q_block.y = src_q[src_q_offset + ne01];
374
+ q_block.z = src_q[src_q_offset + ne01 * 2];
375
+ q_block.w = src_q[src_q_offset + ne01 * 3];
376
+
377
+ ushort8 post_block = as_ushort8(q_block);
378
+ ushort8 pre_block = (ushort8)(0);
379
+
380
+ uchar * pre_block_ptr = (uchar *)(&pre_block);
381
+ uchar * post_block_ptr = (uchar *)(&post_block);
382
+
383
+ for (int i = 0; i < QK4_0 / 4; ++i) {
384
+ uchar x0 = post_block_ptr[i + 0];
385
+ uchar x1 = post_block_ptr[i + QK4_0 / 4];
386
+
387
+ pre_block_ptr[2 * i + 0] = convert_uchar(x0 & 0x0F) | convert_uchar((x1 & 0x0F) << 4);
388
+ pre_block_ptr[2 * i + 1] = convert_uchar((x0 & 0xF0) >> 4) | convert_uchar(x1 & 0xF0);
389
+ }
390
+
391
+ ((__global ushort8 *)(&(b->qs[0])))[0] = pre_block;
392
+ }
393
+
394
+ //------------------------------------------------------------------------------
395
+ // kernel_convert_block_q4_1
396
+ // Convert the block_q4_1 format to 2 separate arrays (AOS -> SOA).
397
+ // This kernel does not deshuffle the bits.
398
+ //------------------------------------------------------------------------------
399
+ kernel void kernel_convert_block_q4_1(
400
+ global struct block_q4_1 * src0,
401
+ global uchar * dst_q,
402
+ global half * dst_d,
403
+ global half * dst_m
404
+ ) {
405
+ global struct block_q4_1 * b = (global struct block_q4_1 *) src0 + get_global_id(0);
406
+ global uchar * q = (global uchar *) dst_q + QK4_1/2*get_global_id(0);
407
+ global half * d = (global half *) dst_d + get_global_id(0);
408
+ global half * m = (global half *) dst_m + get_global_id(0);
409
+
410
+ *d = b->d;
411
+ *m = b->m;
412
+
413
+ for (int i = 0; i < QK4_1/2; ++i) {
414
+ q[i] = b->qs[i];
415
+ }
416
+ }
417
+
418
+ kernel void kernel_restore_block_q4_1(
419
+ global uchar * src_q,
420
+ global half * src_d,
421
+ global half * src_m,
422
+ global struct block_q4_1 * dst
423
+ ) {
424
+ global struct block_q4_1 * b = (global struct block_q4_1 *) dst + get_global_id(0);
425
+ global uchar * q = (global uchar *) src_q + QK4_1/2*get_global_id(0);
426
+ global half * d = (global half *) src_d + get_global_id(0);
427
+ global half * m = (global half *) src_m + get_global_id(0);
428
+
429
+ b->d = *d;
430
+ b->m = *m;
431
+ for (int i = 0; i < QK4_1/2; ++i) {
432
+ b->qs[i] = q[i];
433
+ }
434
+ }
435
+
436
+ kernel void kernel_convert_block_q4_1_noshuffle(
437
+ global struct block_q4_1 * src0,
438
+ global uchar * dst_q,
439
+ global half * dst_d,
440
+ global half * dst_m
441
+ ) {
442
+ global struct block_q4_1 * b = (global struct block_q4_1 *) src0 + get_global_id(0);
443
+ global uchar * q = (global uchar *) dst_q + QK4_1/2*get_global_id(0);
444
+ global half * d = (global half *) dst_d + get_global_id(0);
445
+ global half * m = (global half *) dst_m + get_global_id(0);
446
+
447
+ *d = b->d;
448
+ *m = b->m;
449
+ for (int i = 0; i < QK4_1/4; ++i) {
450
+ uchar x0 = b->qs[2*i + 0];
451
+ uchar x1 = b->qs[2*i + 1];
452
+
453
+ q[i + 0 ] = convert_uchar(x0 & 0x0F) | convert_uchar((x1 & 0x0F) << 4);
454
+ q[i + QK4_1/4] = convert_uchar((x0 & 0xF0) >> 4) | convert_uchar(x1 & 0xF0);
455
+
456
+ #ifdef ADRENO_GPU
457
+ if (get_global_id(0) == 65536*4096) {
458
+ printf("%04x - %02x\n", *(global ushort*)d, ((x0 & 0xF0) >> 4) | (x1 & 0xF0));
459
+ }
460
+ #endif
461
+ }
462
+ }
463
+
464
+ kernel void kernel_restore_block_q4_1_noshuffle(
465
+ global uchar * src_q,
466
+ global half * src_d,
467
+ global half * src_m,
468
+ global struct block_q4_1 * dst,
469
+ uchar mask_0F,
470
+ uchar mask_F0
471
+ ) {
472
+ global struct block_q4_1 * b = (global struct block_q4_1 *) dst + get_global_id(0);
473
+ global uchar * q = (global uchar *) src_q + QK4_1/2*get_global_id(0);
474
+ global half * d = (global half *) src_d + get_global_id(0);
475
+ global half * m = (global half *) src_m + get_global_id(0);
476
+
477
+ b->d = *d;
478
+ b->m = *m;
479
+ for (int i = 0; i < QK4_1/4; ++i) {
480
+ uchar x0 = q[i + 0 ] ;
481
+ uchar x1 = q[i + QK4_1/4];
482
+
483
+ b->qs[2*i + 0] = convert_uchar((x0 & mask_0F) | ((x1 & mask_0F) << 4));
484
+ b->qs[2*i + 1] = convert_uchar(((x0 & mask_F0) >> 4) | (x1 & mask_F0));
485
+ }
486
+ }
487
+
488
+ kernel void kernel_convert_block_q4_1_trans4_ns(
489
+ __global struct block_q4_1 * src0,
490
+ __global uint * dst_q,
491
+ __global half * dst_d,
492
+ __global half * dst_m,
493
+ uint ne00,
494
+ uint ne01
495
+ ) {
496
+ uint i00 = get_global_id(1);
497
+ uint i01 = get_global_id(0);
498
+ uint i02 = get_global_id(2);
499
+
500
+ if (i01 >= ne01) {
501
+ return;
502
+ }
503
+
504
+ uint ne00_blk = ne00 / QK4_1;
505
+ uint src_blk_offset = i00 + i01 * ne00_blk + i02 * ne00_blk * ne01;
506
+ uint dst_blk_offset = i01 + i00 * ne01 + i02 * ne00_blk * ne01;
507
+
508
+ global struct block_q4_1 * b = src0 + src_blk_offset;
509
+ dst_d[dst_blk_offset] = b->d;
510
+ dst_m[dst_blk_offset] = b->m;
511
+
512
+ // extract quantization and unshuffle
513
+ ushort8 pre_block = ((global ushort8 *)(&(b->qs[0])))[0];
514
+
515
+ ushort8 post_block = (ushort8)(0);
516
+
517
+ uchar * pre_block_ptr = (uchar *)(&pre_block);
518
+ uchar * post_block_ptr = (uchar *)(&post_block);
519
+
520
+ for (int i = 0; i < QK4_1 / 4; ++i) {
521
+ uchar x0 = pre_block_ptr[2*i + 0];
522
+ uchar x1 = pre_block_ptr[2*i + 1];
523
+
524
+ post_block_ptr[i + 0 ] = convert_uchar(x0 & 0x0F) | convert_uchar((x1 & 0x0F) << 4);
525
+ post_block_ptr[i + QK4_1 / 4] = convert_uchar((x0 & 0xF0) >> 4) | convert_uchar(x1 & 0xF0);
526
+ }
527
+
528
+ uint4 q_block = as_uint4(post_block);
529
+
530
+ uint offset = i02 * ne00_blk * ne01 * 4 + i00 * ne01 * 4 + i01;
531
+ dst_q[offset] = q_block.x;
532
+ dst_q[offset + ne01] = q_block.y;
533
+ dst_q[offset + ne01 * 2] = q_block.z;
534
+ dst_q[offset + ne01 * 3] = q_block.w;
535
+ }
536
+
537
+ kernel void kernel_restore_block_q4_1_trans4_ns(
538
+ __global uint * src_q,
539
+ __global half * src_d,
540
+ __global half * src_m,
541
+ __global struct block_q4_1 * dst0,
542
+ uint ne00,
543
+ uint ne01
544
+ ) {
545
+ int i00 = get_global_id(1);
546
+ uint i01 = get_global_id(0);
547
+ uint i02 = get_global_id(2);
548
+
549
+ if (i01 >= ne01) {
550
+ return;
551
+ }
552
+
553
+ uint ne00_blk = ne00 / QK4_1;
554
+ uint dst_blk_offset = i00 + i01 * ne00_blk + i02 * ne00_blk * ne01;
555
+ uint src_dm_offset = i01 + i00 * ne01 + i02 * ne00_blk * ne01;
556
+
557
+ __global struct block_q4_1 * b = dst0 + dst_blk_offset;
558
+ b->d = src_d[src_dm_offset];
559
+ b->m = src_m[src_dm_offset];
560
+
561
+ // collect transposed quantization parts for a block
562
+ uint src_q_offset = i02 * ne00_blk * ne01 * 4 + i00 * ne01 * 4 + i01;
563
+ uint4 q_block;
564
+ q_block.x = src_q[src_q_offset];
565
+ q_block.y = src_q[src_q_offset + ne01];
566
+ q_block.z = src_q[src_q_offset + ne01 * 2];
567
+ q_block.w = src_q[src_q_offset + ne01 * 3];
568
+
569
+ ushort8 post_block = as_ushort8(q_block);
570
+ ushort8 pre_block = (ushort8)(0);
571
+
572
+ uchar * pre_block_ptr = (uchar *)(&pre_block);
573
+ uchar * post_block_ptr = (uchar *)(&post_block);
574
+
575
+ for (int i = 0; i < QK4_0 / 4; ++i) {
576
+ uchar x0 = post_block_ptr[i + 0];
577
+ uchar x1 = post_block_ptr[i + QK4_0 / 4];
578
+
579
+ pre_block_ptr[2 * i + 0] = convert_uchar(x0 & 0x0F) | convert_uchar((x1 & 0x0F) << 4);
580
+ pre_block_ptr[2 * i + 1] = convert_uchar((x0 & 0xF0) >> 4) | convert_uchar(x1 & 0xF0);
581
+ }
582
+
583
+ ((__global ushort8 *)(&(b->qs[0])))[0] = pre_block;
584
+ }
585
+
586
+ //------------------------------------------------------------------------------
587
+ // kernel_convert_block_q5_0
588
+ // Convert the block_q5_0 format to 3 separate arrays (AOS -> SOA).
589
+ // This kernel does not deshuffle the bits.
590
+ //------------------------------------------------------------------------------
591
+ kernel void kernel_convert_block_q5_0(
592
+ global struct block_q5_0 * src0,
593
+ global uchar * dst_qs,
594
+ global uint * dst_qh,
595
+ global half * dst_d,
596
+ ulong n_blk
597
+ ) {
598
+ if (get_global_id(0) >= n_blk) {
599
+ return;
600
+ }
601
+
602
+ global struct block_q5_0 * b = (global struct block_q5_0 *) src0 + get_global_id(0);
603
+ global uchar * qs = (global uchar *) dst_qs + (QK5_0/2)*get_global_id(0);
604
+ global uint * qh = (global uint *) dst_qh + get_global_id(0);
605
+ global half * d = (global half *) dst_d + get_global_id(0);
606
+
607
+ *d = b->d;
608
+ *qh = *((global uint *)(b->qh));
609
+
610
+ for (int i = 0; i < QK5_0/2; ++i) {
611
+ qs[i] = b->qs[i];
612
+ }
613
+ }
614
+
615
+ kernel void kernel_restore_block_q5_0(
616
+ global uchar * src_qs,
617
+ global uint * src_qh,
618
+ global half * src_d,
619
+ global struct block_q5_0 * dst
620
+ ) {
621
+ global struct block_q5_0 * b = (global struct block_q5_0 *) dst + get_global_id(0);
622
+ global uchar * qs = (global uchar *) src_qs + (QK5_0/2)*get_global_id(0);
623
+ global uint * qh = (global uint *) src_qh + get_global_id(0);
624
+ global half * d = (global half *) src_d + get_global_id(0);
625
+
626
+ b->d = *d;
627
+ *((global uint *)(b->qh)) = *qh;
628
+ for (int i = 0; i < QK5_0/2; ++i) {
629
+ b->qs[i] = qs[i];
630
+ }
631
+ }
632
+
633
+ kernel void kernel_convert_block_q5_0_noshuffle(
634
+ global struct block_q5_0 * src0,
635
+ global uchar * dst_q,
636
+ global uint * dst_qh,
637
+ global half * dst_d
638
+ ) {
639
+ global struct block_q5_0 * b = (global struct block_q5_0 *) src0 + get_global_id(0);
640
+ global uchar * q = (global uchar *) dst_q + QK5_0/2*get_global_id(0);
641
+ global uint * qh = (global uint *) dst_qh + get_global_id(0);
642
+ global half * d = (global half *) dst_d + get_global_id(0);
643
+
644
+ *d = b->d;
645
+ *qh = *((global uint *)(b->qh));
646
+
647
+ for (int i = 0; i < QK5_0/4; ++i) {
648
+ uchar x0 = b->qs[2*i + 0];
649
+ uchar x1 = b->qs[2*i + 1];
650
+
651
+ q[i + 0 ] = convert_uchar(x0 & 0x0F) | convert_uchar((x1 & 0x0F) << 4);
652
+ q[i + QK5_0/4] = convert_uchar((x0 & 0xF0) >> 4) | convert_uchar(x1 & 0xF0);
653
+
654
+ #ifdef ADRENO_GPU
655
+ if (get_global_id(0) == 65536*4096) {
656
+ printf("%04x - %02x\n", *(global ushort*)d, ((x0 & 0xF0) >> 4) | (x1 & 0xF0));
657
+ }
658
+ #endif
659
+ }
660
+ }
661
+
662
+ kernel void kernel_restore_block_q5_0_noshuffle(
663
+ global uchar * src_q,
664
+ global uint * src_qh,
665
+ global half * src_d,
666
+ global struct block_q5_0 * dst,
667
+ uchar mask_0F,
668
+ uchar mask_F0
669
+ ) {
670
+ global struct block_q5_0 * b = (global struct block_q5_0 *) dst + get_global_id(0);
671
+ global uchar * q = (global uchar *) src_q + QK5_0/2*get_global_id(0);
672
+ global uint * qh = (global uint *) src_qh + get_global_id(0);
673
+ global half * d = (global half *) src_d + get_global_id(0);
674
+
675
+ b->d = *d;
676
+ *((global uint *)(b->qh)) = *qh;
677
+
678
+ for (int i = 0; i < QK5_0/4; ++i) {
679
+ uchar x0 = q[i + 0 ];
680
+ uchar x1 = q[i + QK5_0/4];
681
+
682
+ b->qs[2*i + 0] = convert_uchar((x0 & mask_0F) | ((x1 & mask_0F) << 4));
683
+ b->qs[2*i + 1] = convert_uchar(((x0 & mask_F0) >> 4) | (x1 & mask_F0));
684
+ }
685
+ }
686
+
687
+ kernel void kernel_convert_block_q5_0_trans4_ns(
688
+ __global struct block_q5_0 * src0,
689
+ __global uint * dst_qs,
690
+ __global uint * dst_qh,
691
+ __global half * dst_d,
692
+ uint ne00,
693
+ uint ne01
694
+ ) {
695
+ uint i00 = get_global_id(1);
696
+ uint i01 = get_global_id(0);
697
+ uint i02 = get_global_id(2);
698
+
699
+ if (i01 >= ne01) {
700
+ return;
701
+ }
702
+
703
+ uint ne00_blk = ne00 / QK5_0;
704
+ uint src_blk_offset = i00 + i01 * ne00_blk + i02 * ne00_blk * ne01;
705
+ uint dst_blk_offset = i01 + i00 * ne01 + i02 * ne00_blk * ne01;
706
+
707
+ global struct block_q5_0 * b = src0 + src_blk_offset;
708
+ dst_d[dst_blk_offset] = b->d;
709
+
710
+ dst_qh[dst_blk_offset] = ((global uint *)(&(b->qh[0])))[0];
711
+
712
+ // extract quantization and unshuffle
713
+ ushort8 pre_block = ((global ushort8 *)(&(b->qs[0])))[0];
714
+ ushort8 post_block = (ushort8)(0);
715
+
716
+ uchar * pre_block_ptr = (uchar *)(&pre_block);
717
+ uchar * post_block_ptr = (uchar *)(&post_block);
718
+
719
+ for (int i = 0; i < QK5_0 / 4; ++i) {
720
+ uchar x0 = pre_block_ptr[2*i + 0];
721
+ uchar x1 = pre_block_ptr[2*i + 1];
722
+
723
+ post_block_ptr[i + 0 ] = convert_uchar(x0 & 0x0F) | convert_uchar((x1 & 0x0F) << 4);
724
+ post_block_ptr[i + QK5_0 / 4] = convert_uchar((x0 & 0xF0) >> 4) | convert_uchar(x1 & 0xF0);
725
+ }
726
+
727
+ uint4 q_block = as_uint4(post_block);
728
+
729
+ uint offset = i02 * ne00_blk * ne01 * 4 + i00 * ne01 * 4 + i01;
730
+ dst_qs[offset] = q_block.x;
731
+ dst_qs[offset + ne01] = q_block.y;
732
+ dst_qs[offset + ne01 * 2] = q_block.z;
733
+ dst_qs[offset + ne01 * 3] = q_block.w;
734
+ }
735
+
736
+ kernel void kernel_restore_block_q5_0_trans4_ns(
737
+ __global uint * src_qs,
738
+ __global uint * src_qh,
739
+ __global half * src_d,
740
+ __global struct block_q5_0 * dst0,
741
+ uint ne00,
742
+ uint ne01
743
+ ) {
744
+ int i00 = get_global_id(1);
745
+ uint i01 = get_global_id(0);
746
+ uint i02 = get_global_id(2);
747
+
748
+ if (i01 >= ne01) {
749
+ return;
750
+ }
751
+
752
+ uint ne00_blk = ne00 / QK5_0;
753
+ uint dst_blk_offset = i00 + i01 * ne00_blk + i02 * ne00_blk * ne01;
754
+ uint src_blk_offset = i01 + i00 * ne01 + i02 * ne00_blk * ne01;
755
+
756
+ __global struct block_q5_0 * b = dst0 + dst_blk_offset;
757
+ b->d = src_d[src_blk_offset];
758
+
759
+ ((__global uint *)(&(b->qh[0])))[0] = src_qh[src_blk_offset];
760
+
761
+ // collect transposed quantization parts for a block
762
+ uint src_q_offset = i02 * ne00_blk * ne01 * 4 + i00 * ne01 * 4 + i01;
763
+ uint4 q_block;
764
+ q_block.x = src_qs[src_q_offset];
765
+ q_block.y = src_qs[src_q_offset + ne01];
766
+ q_block.z = src_qs[src_q_offset + ne01 * 2];
767
+ q_block.w = src_qs[src_q_offset + ne01 * 3];
768
+
769
+ ushort8 post_block = as_ushort8(q_block);
770
+ ushort8 pre_block = (ushort8)(0);
771
+
772
+ uchar * pre_block_ptr = (uchar *)(&pre_block);
773
+ uchar * post_block_ptr = (uchar *)(&post_block);
774
+
775
+ for (int i = 0; i < QK5_0 / 4; ++i) {
776
+ uchar x0 = post_block_ptr[i + 0];
777
+ uchar x1 = post_block_ptr[i + QK5_0 / 4];
778
+
779
+ pre_block_ptr[2 * i + 0] = convert_uchar(x0 & 0x0F) | convert_uchar((x1 & 0x0F) << 4);
780
+ pre_block_ptr[2 * i + 1] = convert_uchar((x0 & 0xF0) >> 4) | convert_uchar(x1 & 0xF0);
781
+ }
782
+
783
+ ((__global ushort8 *)(&(b->qs[0])))[0] = pre_block;
784
+ }
785
+
786
+ //------------------------------------------------------------------------------
787
+ // kernel_convert_block_q5_1
788
+ // Convert the block_q5_1 format to 4 separate arrays (AOS -> SOA).
789
+ // This kernel does not deshuffle the bits.
790
+ //------------------------------------------------------------------------------
791
+ kernel void kernel_convert_block_q5_1(
792
+ global struct block_q5_1 * src0,
793
+ global uchar * dst_qs,
794
+ global uint * dst_qh,
795
+ global half * dst_d,
796
+ global half * dst_m,
797
+ ulong n_blk
798
+ ) {
799
+ if (get_global_id(0) >= n_blk) {
800
+ return;
801
+ }
802
+
803
+ global struct block_q5_1 * b = (global struct block_q5_1 *) src0 + get_global_id(0);
804
+ global uchar * qs = (global uchar *) dst_qs + (QK5_1/2)*get_global_id(0);
805
+ global uint * qh = (global uint *) dst_qh + get_global_id(0);
806
+ global half * d = (global half *) dst_d + get_global_id(0);
807
+ global half * m = (global half *) dst_m + get_global_id(0);
808
+
809
+ *d = b->d;
810
+ *m = b->m;
811
+ *qh = *((global uint *)(b->qh));
812
+
813
+ for (int i = 0; i < QK5_1/2; ++i) {
814
+ qs[i] = b->qs[i];
815
+ }
816
+ }
817
+
818
+ kernel void kernel_restore_block_q5_1(
819
+ global uchar * src_qs,
820
+ global uint * src_qh,
821
+ global half * src_d,
822
+ global half * src_m,
823
+ global struct block_q5_1 * dst
824
+ ) {
825
+ global struct block_q5_1 * b = (global struct block_q5_1 *) dst + get_global_id(0);
826
+ global uchar * qs = (global uchar *) src_qs + (QK5_1/2)*get_global_id(0);
827
+ global uint * qh = (global uint *) src_qh + get_global_id(0);
828
+ global half * d = (global half *) src_d + get_global_id(0);
829
+ global half * m = (global half *) src_m + get_global_id(0);
830
+
831
+ b->d = *d;
832
+ b->m = *m;
833
+ *((global uint *)(b->qh)) = *qh;
834
+ for (int i = 0; i < QK5_1/2; ++i) {
835
+ b->qs[i] = qs[i];
836
+ }
837
+ }
838
+
839
+ kernel void kernel_convert_block_q5_1_noshuffle(
840
+ global struct block_q5_1 * src0,
841
+ global uchar * dst_q,
842
+ global uint * dst_qh,
843
+ global half * dst_d,
844
+ global half * dst_m
845
+ ) {
846
+ global struct block_q5_1 * b = (global struct block_q5_1 *) src0 + get_global_id(0);
847
+ global uchar * q = (global uchar *) dst_q + QK5_1/2*get_global_id(0);
848
+ global uint * qh = (global uint *) dst_qh + get_global_id(0);
849
+ global half * d = (global half *) dst_d + get_global_id(0);
850
+ global half * m = (global half *) dst_m + get_global_id(0);
851
+
852
+ *d = b->d;
853
+ *m = b->m;
854
+ *qh = *((global uint *)(b->qh));
855
+
856
+ for (int i = 0; i < QK5_1/4; ++i) {
857
+ uchar x0 = b->qs[2*i + 0];
858
+ uchar x1 = b->qs[2*i + 1];
859
+
860
+ q[i + 0 ] = convert_uchar(x0 & 0x0F) | convert_uchar((x1 & 0x0F) << 4);
861
+ q[i + QK5_1/4] = convert_uchar((x0 & 0xF0) >> 4) | convert_uchar(x1 & 0xF0);
862
+
863
+ #ifdef ADRENO_GPU
864
+ if (get_global_id(0) == 65536*4096) {
865
+ printf("%04x - %02x\n", *(global ushort*)d, ((x0 & 0xF0) >> 4) | (x1 & 0xF0));
866
+ }
867
+ #endif
868
+ }
869
+ }
870
+
871
+ kernel void kernel_restore_block_q5_1_noshuffle(
872
+ global uchar * src_q,
873
+ global uint * src_qh,
874
+ global half * src_d,
875
+ global half * src_m,
876
+ global struct block_q5_1 * dst,
877
+ uchar mask_0F,
878
+ uchar mask_F0
879
+ ) {
880
+ global struct block_q5_1 * b = (global struct block_q5_1 *) dst + get_global_id(0);
881
+ global uchar * q = (global uchar *) src_q + QK5_1/2*get_global_id(0);
882
+ global uint * qh = (global uint *) src_qh + get_global_id(0);
883
+ global half * d = (global half *) src_d + get_global_id(0);
884
+ global half * m = (global half *) src_m + get_global_id(0);
885
+
886
+ b->d = *d;
887
+ b->m = *m;
888
+ *((global uint *)(b->qh)) = *qh;
889
+
890
+ for (int i = 0; i < QK5_1/4; ++i) {
891
+ uchar x0 = q[i + 0 ];
892
+ uchar x1 = q[i + QK5_1/4];
893
+
894
+ b->qs[2*i + 0] = convert_uchar((x0 & mask_0F) | ((x1 & mask_0F) << 4));
895
+ b->qs[2*i + 1] = convert_uchar(((x0 & mask_F0) >> 4) | (x1 & mask_F0));
896
+ }
897
+ }
898
+
899
+ kernel void kernel_convert_block_q5_1_trans4_ns(
900
+ __global struct block_q5_1 * src0,
901
+ __global uint * dst_qs,
902
+ __global uint * dst_qh,
903
+ __global half * dst_d,
904
+ __global half * dst_m,
905
+ uint ne00,
906
+ uint ne01
907
+ ) {
908
+ uint i00 = get_global_id(1);
909
+ uint i01 = get_global_id(0);
910
+ uint i02 = get_global_id(2);
911
+
912
+ if (i01 >= ne01) {
913
+ return;
914
+ }
915
+
916
+ uint ne00_blk = ne00 / QK5_1;
917
+ uint src_blk_offset = i00 + i01 * ne00_blk + i02 * ne00_blk * ne01;
918
+ uint dst_blk_offset = i01 + i00 * ne01 + i02 * ne00_blk * ne01;
919
+
920
+ global struct block_q5_1 * b = src0 + src_blk_offset;
921
+ dst_d[dst_blk_offset] = b->d;
922
+ dst_m[dst_blk_offset] = b->m;
923
+
924
+ dst_qh[dst_blk_offset] = ((global uint *)(&(b->qh[0])))[0];
925
+
926
+ // extract quantization and unshuffle
927
+ ushort8 pre_block = ((global ushort8 *)(&(b->qs[0])))[0];
928
+ ushort8 post_block = (ushort8)(0);
929
+
930
+ uchar * pre_block_ptr = (uchar *)(&pre_block);
931
+ uchar * post_block_ptr = (uchar *)(&post_block);
932
+
933
+ for (int i = 0; i < QK5_1 / 4; ++i) {
934
+ uchar x0 = pre_block_ptr[2*i + 0];
935
+ uchar x1 = pre_block_ptr[2*i + 1];
936
+
937
+ post_block_ptr[i + 0 ] = convert_uchar(x0 & 0x0F) | convert_uchar((x1 & 0x0F) << 4);
938
+ post_block_ptr[i + QK5_1 / 4] = convert_uchar((x0 & 0xF0) >> 4) | convert_uchar(x1 & 0xF0);
939
+ }
940
+
941
+ uint4 q_block = as_uint4(post_block);
942
+
943
+ uint offset = i02 * ne00_blk * ne01 * 4 + i00 * ne01 * 4 + i01;
944
+ dst_qs[offset] = q_block.x;
945
+ dst_qs[offset + ne01] = q_block.y;
946
+ dst_qs[offset + ne01 * 2] = q_block.z;
947
+ dst_qs[offset + ne01 * 3] = q_block.w;
948
+ }
949
+
950
+ kernel void kernel_restore_block_q5_1_trans4_ns(
951
+ __global uint * src_qs,
952
+ __global uint * src_qh,
953
+ __global half * src_d,
954
+ __global half * src_m,
955
+ __global struct block_q5_1 * dst0,
956
+ uint ne00,
957
+ uint ne01
958
+ ) {
959
+ int i00 = get_global_id(1);
960
+ uint i01 = get_global_id(0);
961
+ uint i02 = get_global_id(2);
962
+
963
+ if (i01 >= ne01) {
964
+ return;
965
+ }
966
+
967
+ uint ne00_blk = ne00 / QK5_1;
968
+ uint dst_blk_offset = i00 + i01 * ne00_blk + i02 * ne00_blk * ne01;
969
+ uint src_blk_offset = i01 + i00 * ne01 + i02 * ne00_blk * ne01;
970
+
971
+ __global struct block_q5_1 * b = dst0 + dst_blk_offset;
972
+ b->d = src_d[src_blk_offset];
973
+ b->m = src_m[src_blk_offset];
974
+
975
+ ((__global uint *)(&(b->qh[0])))[0] = src_qh[src_blk_offset];
976
+
977
+ // collect transposed quantization parts for a block
978
+ uint src_q_offset = i02 * ne00_blk * ne01 * 4 + i00 * ne01 * 4 + i01;
979
+ uint4 q_block;
980
+ q_block.x = src_qs[src_q_offset];
981
+ q_block.y = src_qs[src_q_offset + ne01];
982
+ q_block.z = src_qs[src_q_offset + ne01 * 2];
983
+ q_block.w = src_qs[src_q_offset + ne01 * 3];
984
+
985
+ ushort8 post_block = as_ushort8(q_block);
986
+ ushort8 pre_block = (ushort8)(0);
987
+
988
+ uchar * pre_block_ptr = (uchar *)(&pre_block);
989
+ uchar * post_block_ptr = (uchar *)(&post_block);
990
+
991
+ for (int i = 0; i < QK5_1 / 4; ++i) {
992
+ uchar x0 = post_block_ptr[i + 0];
993
+ uchar x1 = post_block_ptr[i + QK5_1 / 4];
994
+
995
+ pre_block_ptr[2 * i + 0] = convert_uchar(x0 & 0x0F) | convert_uchar((x1 & 0x0F) << 4);
996
+ pre_block_ptr[2 * i + 1] = convert_uchar((x0 & 0xF0) >> 4) | convert_uchar(x1 & 0xF0);
997
+ }
998
+ ((__global ushort8 *)(&(b->qs[0])))[0] = pre_block;
999
+ }
1000
+
1001
+ kernel void kernel_convert_block_q4_k_trans4_ns(
1002
+ __global struct block_q4_K * src0,
1003
+ __global uint * dst_q,
1004
+ __global half * dst_d,
1005
+ __global half * dst_dm,
1006
+ __global uchar * dst_s,
1007
+ uint ne00,
1008
+ uint ne01,
1009
+ uchar mask_0F,
1010
+ uchar mask_F0
1011
+ ) {
1012
+ uint i00 = get_global_id(1);
1013
+ uint i01 = get_global_id(0);
1014
+ uint i02 = get_global_id(2);
1015
+
1016
+ if (i01 >= ne01) {
1017
+ return;
1018
+ }
1019
+
1020
+ uint ne00_blk = ne00 / QK_K;
1021
+ uint src_blk_offset = i00 + i01 * ne00_blk + i02 * ne00_blk * ne01;
1022
+ uint dst_blk_offset = i01 + i00 * ne01 + i02 * ne00_blk * ne01;
1023
+
1024
+ __global struct block_q4_K * b = src0 + src_blk_offset;
1025
+
1026
+ dst_d [dst_blk_offset] = b->d;
1027
+ dst_dm[dst_blk_offset] = b->dm;
1028
+
1029
+ uint4 qv[8];
1030
+ uchar * qv_bytes = (uchar *)qv;
1031
+ for (int i = 0; i < QK_K / 64; ++i) {
1032
+ for (int j = 0; j < 16; ++j) {
1033
+ uchar x0 = b->q[i*32 + 2*j];
1034
+ uchar x1 = b->q[i*32 + 2*j + 1];
1035
+
1036
+ qv_bytes[i*32 + j ] = convert_uchar(x0 & mask_0F) | convert_uchar((x1 & mask_0F) << 4);
1037
+ qv_bytes[i*32 + j + 16] = convert_uchar((x0 & mask_F0) >> 4) | convert_uchar(x1 & mask_F0);
1038
+ }
1039
+ }
1040
+
1041
+ uint base = i02 * ne00_blk * ne01 * 32 + i00 * ne01 * 32 + i01;
1042
+ #pragma unroll
1043
+ for (int p = 0; p < 8; ++p) {
1044
+ uint4 v = qv[p];
1045
+ dst_q[base + (p * 4 + 0) * ne01] = v.x;
1046
+ dst_q[base + (p * 4 + 1) * ne01] = v.y;
1047
+ dst_q[base + (p * 4 + 2) * ne01] = v.z;
1048
+ dst_q[base + (p * 4 + 3) * ne01] = v.w;
1049
+ }
1050
+
1051
+ __global uchar * s_dst = dst_s + (i02 * ne01 + i01) * ne00_blk * K_SCALE_SIZE + i00 * K_SCALE_SIZE;
1052
+ #pragma unroll
1053
+ for (int i = 0; i < K_SCALE_SIZE; ++i) {
1054
+ s_dst[i] = b->s[i];
1055
+ }
1056
+ }
1057
+
1058
+ kernel void kernel_restore_block_q4_k_trans4_ns(
1059
+ __global uint * src_q,
1060
+ __global half * src_d,
1061
+ __global half * src_dm,
1062
+ __global uchar * src_s,
1063
+ __global struct block_q4_K * dst0,
1064
+ uint ne00,
1065
+ uint ne01,
1066
+ uchar mask_0F,
1067
+ uchar mask_F0
1068
+ ) {
1069
+ uint i00 = get_global_id(1); // block index along K
1070
+ uint i01 = get_global_id(0); // row index
1071
+ uint i02 = get_global_id(2); // batch index
1072
+
1073
+ if (i01 >= ne01) {
1074
+ return;
1075
+ }
1076
+
1077
+ uint ne00_blk = ne00 / QK_K;
1078
+
1079
+ uint src_blk_offset = i01 + i00 * ne01 + i02 * ne00_blk * ne01;
1080
+ uint dst_blk_offset = i00 + i01 * ne00_blk + i02 * ne00_blk * ne01;
1081
+
1082
+ __global struct block_q4_K * b = dst0 + dst_blk_offset;
1083
+
1084
+ b->d = src_d[src_blk_offset];
1085
+ b->dm = src_dm[src_blk_offset];
1086
+
1087
+ __global uchar * s_src = src_s + (i02 * ne01 + i01) * ne00_blk * K_SCALE_SIZE + i00 * K_SCALE_SIZE;
1088
+ for (int i = 0; i < K_SCALE_SIZE; ++i) {
1089
+ b->s[i] = s_src[i];
1090
+ }
1091
+
1092
+ uint base = i02 * ne00_blk * ne01 * 32 + i00 * ne01 * 32 + i01;
1093
+
1094
+ uint4 qv[8];
1095
+ for (int p = 0; p < 8; ++p) {
1096
+ qv[p].x = src_q[base + (p * 4 + 0) * ne01];
1097
+ qv[p].y = src_q[base + (p * 4 + 1) * ne01];
1098
+ qv[p].z = src_q[base + (p * 4 + 2) * ne01];
1099
+ qv[p].w = src_q[base + (p * 4 + 3) * ne01];
1100
+ }
1101
+
1102
+ uchar * qv_bytes = (uchar *)qv;
1103
+ for (int i = 0; i < QK_K / 64; ++i) {
1104
+ for (int j = 0; j < 16; ++j) {
1105
+ uchar lo = qv_bytes[i*32 + j];
1106
+ uchar hi = qv_bytes[i*32 + j + 16];
1107
+ b->q[i*32 + 2*j] = convert_uchar((lo & mask_0F) | ((hi & mask_0F) << 4));
1108
+ b->q[i*32 + 2*j + 1] = convert_uchar(((lo & mask_F0) >> 4) | (hi & mask_F0));
1109
+ }
1110
+ }
1111
+ }
1112
+
1113
+ //------------------------------------------------------------------------------
1114
+ // kernel_convert_block_q4_k_tiled_ns
1115
+ //
1116
+ // Tiled-wide layout for the long-vocab q4_K lm_head/embed GEMV (decode path).
1117
+ // Mirror of kernel_convert_block_q6_k_tiled_ns: recovers each weight's 4-bit
1118
+ // code in CANONICAL ggml element order (e in [0,256)) and re-packs into 32 uints
1119
+ // (8 codes/uint), stored TILED by 64 output rows so the matching GEMV
1120
+ // (gemv_noshuffle_q4_k_f32_tiled) coalesces every weight load. The 12-byte
1121
+ // packed scale block `s` and d/dm are stored per (row, K-block) tiled; the GEMV
1122
+ // re-derives the 8 (scale,min) pairs via get_scale_min_k4, exactly like the o4
1123
+ // kernel. Both ends owned here -> correct by construction vs the reference q4_K
1124
+ // dequant. Requires ne01 % 64 == 0 (gated host-side). Buffer sizes identical to
1125
+ // the trans4_ns layout.
1126
+ //
1127
+ // q uint4 granule g of (row r, K-block sb): idx = ((rt*ne00_blk+sb)*8 + g)*64 + rit
1128
+ // s (12 bytes) of (r, sb): idx = (rt*ne00_blk+sb)*64 + rit, *12
1129
+ // d/dm (half) of (r, sb): idx = (rt*ne00_blk+sb)*64 + rit
1130
+ // where rt = r/64, rit = r%64.
1131
+ //------------------------------------------------------------------------------
1132
+ kernel void kernel_convert_block_q4_k_tiled_ns(
1133
+ __global struct block_q4_K * src0,
1134
+ __global uint * dst_q, // 32 uints / superblock (4-bit codes, 8 codes/uint)
1135
+ __global half * dst_d, // 1 half / superblock
1136
+ __global half * dst_dm, // 1 half / superblock
1137
+ __global uchar * dst_s, // K_SCALE_SIZE (12) bytes / superblock
1138
+ uint ne00,
1139
+ uint ne01
1140
+ ) {
1141
+ uint i00 = get_global_id(1); // K-block index (superblock along ne00)
1142
+ uint i01 = get_global_id(0); // output row index (along ne01)
1143
+ uint i02 = get_global_id(2); // batch
1144
+
1145
+ uint ne00_blk = ne00 / QK_K;
1146
+
1147
+ uint src_blk_offset = i00 + i01 * ne00_blk + i02 * ne00_blk * ne01;
1148
+ __global struct block_q4_K * b = src0 + src_blk_offset;
1149
+
1150
+ uint rt = i01 / 64;
1151
+ uint rit = i01 % 64;
1152
+ uint tile_blk = (i02 * (ne01 / 64) + rt) * ne00_blk + i00;
1153
+
1154
+ // --- recover canonical 4-bit codes in e-order, pack 8 codes/uint ---
1155
+ uint qw[32] = {0};
1156
+ for (uint e = 0; e < 256; ++e) {
1157
+ uint g = e >> 6; // group 0..3 (q advances 32 bytes/group)
1158
+ uint within = e & 63u;
1159
+ uint hlf = within >> 5; // 0 = low nibble, 1 = high nibble
1160
+ uint l = within & 31u; // 0..31
1161
+ uchar byte = b->q[g * 32u + l];
1162
+ uint code = (hlf == 0u) ? (uint)(byte & 0x0F) : (uint)(byte >> 4);
1163
+ qw[e >> 3] |= code << ((e & 7u) * 4u);
1164
+ }
1165
+
1166
+ for (uint gr = 0; gr < 8; ++gr) {
1167
+ uint base = (tile_blk * 8u + gr) * 64u + rit; // uint4 index
1168
+ dst_q[base * 4u + 0u] = qw[gr * 4u + 0u];
1169
+ dst_q[base * 4u + 1u] = qw[gr * 4u + 1u];
1170
+ dst_q[base * 4u + 2u] = qw[gr * 4u + 2u];
1171
+ dst_q[base * 4u + 3u] = qw[gr * 4u + 3u];
1172
+ }
1173
+
1174
+ // packed scales (12 bytes), tiled per (row, block)
1175
+ __global uchar * s_dst = dst_s + (tile_blk * 64u + rit) * K_SCALE_SIZE;
1176
+ #pragma unroll
1177
+ for (int i = 0; i < K_SCALE_SIZE; ++i) {
1178
+ s_dst[i] = b->s[i];
1179
+ }
1180
+
1181
+ dst_d [tile_blk * 64u + rit] = b->d;
1182
+ dst_dm[tile_blk * 64u + rit] = b->dm;
1183
+ }
1184
+
1185
+ kernel void kernel_convert_block_q5_k_trans4_ns(
1186
+ __global struct block_q5_K * src0,
1187
+ __global uint * dst_qs,
1188
+ __global uint * dst_qh,
1189
+ __global half * dst_d,
1190
+ __global half * dst_dm,
1191
+ __global uchar * dst_s,
1192
+ uint ne00,
1193
+ uint ne01,
1194
+ uchar mask_0F,
1195
+ uchar mask_F0
1196
+ ) {
1197
+ uint i00 = get_global_id(1);
1198
+ uint i01 = get_global_id(0);
1199
+ uint i02 = get_global_id(2);
1200
+
1201
+ if (i01 >= ne01) {
1202
+ return;
1203
+ }
1204
+
1205
+ uint ne00_blk = ne00 / QK_K;
1206
+ uint src_blk_offset = i00 + i01 * ne00_blk + i02 * ne00_blk * ne01;
1207
+ uint dst_blk_offset = i01 + i00 * ne01 + i02 * ne00_blk * ne01;
1208
+
1209
+ __global struct block_q5_K * b = src0 + src_blk_offset;
1210
+
1211
+ dst_d [dst_blk_offset] = b->d;
1212
+ dst_dm[dst_blk_offset] = b->dm;
1213
+
1214
+ for (int k = 0; k < 8; k++) {
1215
+ uchar b0 = 0, b1 = 0, b2 = 0, b3 = 0;
1216
+ for (int bit = 0; bit < 8; bit++) {
1217
+ b0 |= (uchar)(((b->qh[bit] >> k) & 1) << bit);
1218
+ b1 |= (uchar)(((b->qh[8 + bit] >> k) & 1) << bit);
1219
+ b2 |= (uchar)(((b->qh[16 + bit] >> k) & 1) << bit);
1220
+ b3 |= (uchar)(((b->qh[24 + bit] >> k) & 1) << bit);
1221
+ }
1222
+ uint packed = (uint)b0 | ((uint)b1 << 8) | ((uint)b2 << 16) | ((uint)b3 << 24);
1223
+ dst_qh[i01 + (i00 * 8 + k) * ne01 + i02 * ne00_blk * 8 * ne01] = packed;
1224
+ }
1225
+
1226
+ uint4 qv[8];
1227
+ uchar * qv_bytes = (uchar *)qv;
1228
+ for (int i = 0; i < QK_K / 64; ++i) {
1229
+ for (int j = 0; j < 16; ++j) {
1230
+ uchar x0 = b->qs[i*32 + 2*j];
1231
+ uchar x1 = b->qs[i*32 + 2*j + 1];
1232
+
1233
+ qv_bytes[i*32 + j ] = convert_uchar(x0 & mask_0F) | convert_uchar((x1 & mask_0F) << 4);
1234
+ qv_bytes[i*32 + j + 16] = convert_uchar((x0 & mask_F0) >> 4) | convert_uchar(x1 & mask_F0);
1235
+ }
1236
+ }
1237
+
1238
+ uint base = i02 * ne00_blk * ne01 * 32 + i00 * ne01 * 32 + i01;
1239
+ #pragma unroll
1240
+ for (int p = 0; p < 8; ++p) {
1241
+ uint4 v = qv[p];
1242
+ dst_qs[base + (p * 4 + 0) * ne01] = v.x;
1243
+ dst_qs[base + (p * 4 + 1) * ne01] = v.y;
1244
+ dst_qs[base + (p * 4 + 2) * ne01] = v.z;
1245
+ dst_qs[base + (p * 4 + 3) * ne01] = v.w;
1246
+ }
1247
+
1248
+ __global uchar * s_dst = dst_s + (i02 * ne01 + i01) * ne00_blk * K_SCALE_SIZE + i00 * K_SCALE_SIZE;
1249
+ #pragma unroll
1250
+ for (int i = 0; i < K_SCALE_SIZE; ++i) {
1251
+ s_dst[i] = b->s[i];
1252
+ }
1253
+ }
1254
+
1255
+ kernel void kernel_restore_block_q5_k_trans4_ns(
1256
+ __global uint * src_qs,
1257
+ __global uint * src_qh,
1258
+ __global half * src_d,
1259
+ __global half * src_dm,
1260
+ __global uchar * src_s,
1261
+ __global struct block_q5_K * dst0,
1262
+ uint ne00,
1263
+ uint ne01,
1264
+ uchar mask_0F,
1265
+ uchar mask_F0
1266
+ ) {
1267
+ uint i00 = get_global_id(1); // block index along K
1268
+ uint i01 = get_global_id(0); // row index
1269
+ uint i02 = get_global_id(2); // batch index
1270
+
1271
+ if (i01 >= ne01) {
1272
+ return;
1273
+ }
1274
+
1275
+ uint ne00_blk = ne00 / QK_K;
1276
+
1277
+ uint src_blk_offset = i01 + i00 * ne01 + i02 * ne00_blk * ne01;
1278
+ uint dst_blk_offset = i00 + i01 * ne00_blk + i02 * ne00_blk * ne01;
1279
+
1280
+ __global struct block_q5_K * b = dst0 + dst_blk_offset;
1281
+
1282
+ b->d = src_d[src_blk_offset];
1283
+ b->dm = src_dm[src_blk_offset];
1284
+
1285
+ for (int j = 0; j < 32; j++) b->qh[j] = 0;
1286
+ for (int k = 0; k < 8; k++) {
1287
+ uint packed = src_qh[i01 + (i00 * 8 + k) * ne01 + i02 * ne00_blk * 8 * ne01];
1288
+ uchar b0 = (uchar)(packed & 0xFF);
1289
+ uchar b1 = (uchar)((packed >> 8) & 0xFF);
1290
+ uchar b2 = (uchar)((packed >> 16) & 0xFF);
1291
+ uchar b3 = (uchar)((packed >> 24) & 0xFF);
1292
+ for (int bit = 0; bit < 8; bit++) {
1293
+ b->qh[bit] |= (uchar)(((b0 >> bit) & 1) << k);
1294
+ b->qh[8 + bit] |= (uchar)(((b1 >> bit) & 1) << k);
1295
+ b->qh[16 + bit] |= (uchar)(((b2 >> bit) & 1) << k);
1296
+ b->qh[24 + bit] |= (uchar)(((b3 >> bit) & 1) << k);
1297
+ }
1298
+ }
1299
+
1300
+ __global uchar * s_src = src_s + (i02 * ne01 + i01) * ne00_blk * K_SCALE_SIZE + i00 * K_SCALE_SIZE;
1301
+ for (int i = 0; i < K_SCALE_SIZE; ++i) {
1302
+ b->s[i] = s_src[i];
1303
+ }
1304
+
1305
+ uint base = i02 * ne00_blk * ne01 * 32 + i00 * ne01 * 32 + i01;
1306
+
1307
+ uint4 qv[8];
1308
+ for (int p = 0; p < 8; ++p) {
1309
+ qv[p].x = src_qs[base + (p * 4 + 0) * ne01];
1310
+ qv[p].y = src_qs[base + (p * 4 + 1) * ne01];
1311
+ qv[p].z = src_qs[base + (p * 4 + 2) * ne01];
1312
+ qv[p].w = src_qs[base + (p * 4 + 3) * ne01];
1313
+ }
1314
+
1315
+ uchar * qv_bytes = (uchar *)qv;
1316
+ for (int i = 0; i < QK_K / 64; ++i) {
1317
+ for (int j = 0; j < 16; ++j) {
1318
+ uchar lo = qv_bytes[i*32 + j];
1319
+ uchar hi = qv_bytes[i*32 + j + 16];
1320
+ b->qs[i*32 + 2*j] = convert_uchar((lo & mask_0F) | ((hi & mask_0F) << 4));
1321
+ b->qs[i*32 + 2*j + 1] = convert_uchar(((lo & mask_F0) >> 4) | (hi & mask_F0));
1322
+ }
1323
+ }
1324
+ }
1325
+
1326
+ kernel void kernel_convert_block_q6_k_trans4_ns(
1327
+ __global struct block_q6_K * src0,
1328
+ __global uint * dst_ql,
1329
+ __global uint * dst_qh,
1330
+ __global half * dst_d,
1331
+ __global char * dst_s,
1332
+ uint ne00,
1333
+ uint ne01,
1334
+ uchar mask_0F,
1335
+ uchar mask_F0
1336
+ ) {
1337
+ uint i00 = get_global_id(1);
1338
+ uint i01 = get_global_id(0);
1339
+ uint i02 = get_global_id(2);
1340
+
1341
+ if (i01 >= ne01) {
1342
+ return;
1343
+ }
1344
+
1345
+ uint ne00_blk = ne00 / QK_K;
1346
+
1347
+ uint src_blk_offset = i00 + i01 * ne00_blk + i02 * ne00_blk * ne01;
1348
+ uint dst_blk_offset = i01 + i00 * ne01 + i02 * ne00_blk * ne01;
1349
+
1350
+ __global struct block_q6_K * b = src0 + src_blk_offset;
1351
+
1352
+ dst_d[dst_blk_offset] = b->d;
1353
+
1354
+ uint4 qlv[8];
1355
+ uchar * qlv_bytes = (uchar *)qlv;
1356
+ for (int i = 0; i < 2; ++i) {
1357
+ for (int j = 0; j < 16; ++j) {
1358
+ uchar x0 = b->ql[i*64 + 2*j];
1359
+ uchar x1 = b->ql[i*64 + 2*j + 1];
1360
+ uchar x2 = b->ql[i*64 + 32 + 2*j];
1361
+ uchar x3 = b->ql[i*64 + 32 + 2*j + 1];
1362
+ qlv_bytes[i*64 + j ] = convert_uchar(x0 & mask_0F) | convert_uchar((x1 & mask_0F) << 4);
1363
+ qlv_bytes[i*64 + j + 16] = convert_uchar(x2 & mask_0F) | convert_uchar((x3 & mask_0F) << 4);
1364
+ qlv_bytes[i*64 + j + 32] = convert_uchar((x0 & mask_F0) >> 4) | convert_uchar(x1 & mask_F0);
1365
+ qlv_bytes[i*64 + j + 48] = convert_uchar((x2 & mask_F0) >> 4) | convert_uchar(x3 & mask_F0);
1366
+ }
1367
+ }
1368
+
1369
+ uint ql_base = i02 * ne00_blk * ne01 * 32 + i00 * ne01 * 32 + i01;
1370
+
1371
+ #pragma unroll
1372
+ for (int p = 0; p < 8; ++p) {
1373
+ uint4 v = qlv[p];
1374
+ dst_ql[ql_base + (p * 4 + 0) * ne01] = v.x;
1375
+ dst_ql[ql_base + (p * 4 + 1) * ne01] = v.y;
1376
+ dst_ql[ql_base + (p * 4 + 2) * ne01] = v.z;
1377
+ dst_ql[ql_base + (p * 4 + 3) * ne01] = v.w;
1378
+ }
1379
+
1380
+ uint qhv[16] = {0};
1381
+
1382
+ for (int n = 0; n < 2; ++n) {
1383
+ for (int l = 0; l < 32; ++l) {
1384
+ uchar h = b->qh[n*32 + l];
1385
+ int u = l / 16;
1386
+ int bit_pos = (l % 16) * 2;
1387
+ qhv[(n*4 + 0)*2 + u] |= ((uint)((h >> 0) & 0x03)) << bit_pos;
1388
+ qhv[(n*4 + 1)*2 + u] |= ((uint)((h >> 2) & 0x03)) << bit_pos;
1389
+ qhv[(n*4 + 2)*2 + u] |= ((uint)((h >> 4) & 0x03)) << bit_pos;
1390
+ qhv[(n*4 + 3)*2 + u] |= ((uint)((h >> 6) & 0x03)) << bit_pos;
1391
+ }
1392
+ }
1393
+
1394
+ uint qh_base = i02 * ne00_blk * ne01 * 16 + i00 * ne01 * 16 + i01;
1395
+
1396
+ for (int p = 0; p < 16; ++p) {
1397
+ dst_qh[qh_base + p * ne01] = qhv[p];
1398
+ }
1399
+
1400
+ __global char * s_dst = dst_s + (i02 * ne01 + i01) * ne00_blk * 16 + i00 * 16;
1401
+ #pragma unroll
1402
+ for (int i = 0; i < 16; ++i) {
1403
+ s_dst[i] = b->scales[i];
1404
+ }
1405
+ }
1406
+
1407
+ kernel void kernel_restore_block_q6_k_trans4_ns(
1408
+ __global uint * src_ql,
1409
+ __global uint * src_qh,
1410
+ __global half * src_d,
1411
+ __global char * src_s,
1412
+ __global struct block_q6_K * dst0,
1413
+ uint ne00,
1414
+ uint ne01,
1415
+ uchar mask_0F,
1416
+ uchar mask_F0
1417
+ ) {
1418
+ uint i00 = get_global_id(1); // block index along K
1419
+ uint i01 = get_global_id(0); // row index
1420
+ uint i02 = get_global_id(2); // batch index
1421
+
1422
+ if (i01 >= ne01) {
1423
+ return;
1424
+ }
1425
+
1426
+ uint ne00_blk = ne00 / QK_K;
1427
+
1428
+ uint src_blk_offset = i01 + i00 * ne01 + i02 * ne00_blk * ne01;
1429
+ uint dst_blk_offset = i00 + i01 * ne00_blk + i02 * ne00_blk * ne01;
1430
+
1431
+ __global struct block_q6_K * b = dst0 + dst_blk_offset;
1432
+
1433
+ b->d = src_d[src_blk_offset];
1434
+
1435
+ uint ql_base = i02 * ne00_blk * ne01 * 32 + i00 * ne01 * 32 + i01;
1436
+ uint4 qlv[8];
1437
+ for (int p = 0; p < 8; ++p) {
1438
+ qlv[p].x = src_ql[ql_base + (p * 4 + 0) * ne01];
1439
+ qlv[p].y = src_ql[ql_base + (p * 4 + 1) * ne01];
1440
+ qlv[p].z = src_ql[ql_base + (p * 4 + 2) * ne01];
1441
+ qlv[p].w = src_ql[ql_base + (p * 4 + 3) * ne01];
1442
+ }
1443
+
1444
+ uchar * qlv_bytes = (uchar *)qlv;
1445
+ for (int i = 0; i < 2; ++i) {
1446
+ for (int j = 0; j < 16; ++j) {
1447
+ uchar lo_02 = qlv_bytes[i*64 + j];
1448
+ uchar lo_13 = qlv_bytes[i*64 + j + 16];
1449
+ uchar hi_02 = qlv_bytes[i*64 + j + 32];
1450
+ uchar hi_13 = qlv_bytes[i*64 + j + 48];
1451
+ b->ql[i*64 + 2*j] = convert_uchar((lo_02 & mask_0F) | ((hi_02 & mask_0F) << 4));
1452
+ b->ql[i*64 + 2*j + 1] = convert_uchar(((lo_02 & mask_F0) >> 4) | (hi_02 & mask_F0));
1453
+ b->ql[i*64 + 32 + 2*j] = convert_uchar((lo_13 & mask_0F) | ((hi_13 & mask_0F) << 4));
1454
+ b->ql[i*64 + 32 + 2*j + 1] = convert_uchar(((lo_13 & mask_F0) >> 4) | (hi_13 & mask_F0));
1455
+ }
1456
+ }
1457
+
1458
+ uint qh_base = i02 * ne00_blk * ne01 * 16 + i00 * ne01 * 16 + i01;
1459
+ uint qhv[16];
1460
+ for (int p = 0; p < 16; ++p) {
1461
+ qhv[p] = src_qh[qh_base + p * ne01];
1462
+ }
1463
+
1464
+ for (int n = 0; n < 2; ++n) {
1465
+ for (int l = 0; l < 32; ++l) {
1466
+ int u = l / 16;
1467
+ int bit_pos = (l % 16) * 2;
1468
+ uchar v0 = (uchar)((qhv[(n*4 + 0)*2 + u] >> bit_pos) & 0x03);
1469
+ uchar v1 = (uchar)((qhv[(n*4 + 1)*2 + u] >> bit_pos) & 0x03);
1470
+ uchar v2 = (uchar)((qhv[(n*4 + 2)*2 + u] >> bit_pos) & 0x03);
1471
+ uchar v3 = (uchar)((qhv[(n*4 + 3)*2 + u] >> bit_pos) & 0x03);
1472
+ b->qh[n*32 + l] = v0 | (v1 << 2) | (v2 << 4) | (v3 << 6);
1473
+ }
1474
+ }
1475
+
1476
+ __global char * s_src = src_s + (i02 * ne01 + i01) * ne00_blk * 16 + i00 * 16;
1477
+ for (int i = 0; i < 16; ++i) {
1478
+ b->scales[i] = s_src[i];
1479
+ }
1480
+ }
1481
+
1482
+ //------------------------------------------------------------------------------
1483
+ // block_mxfp4
1484
+ //------------------------------------------------------------------------------
1485
+ #define QK_MXFP4 32
1486
+ struct block_mxfp4 {
1487
+ uchar e; // E8M0
1488
+ uchar qs[QK_MXFP4 / 2];
1489
+ };
1490
+
1491
+ //------------------------------------------------------------------------------
1492
+ // kernel_convert_block_mxfp4
1493
+ // Convert the block_mxfp4 format to 2 separate arrays (AOS -> SOA).
1494
+ // This kernel does not deshuffle the bits.
1495
+ //------------------------------------------------------------------------------
1496
+ kernel void kernel_convert_block_mxfp4(
1497
+ global struct block_mxfp4 * src0,
1498
+ global uchar * dst_q,
1499
+ global uchar * dst_e
1500
+ ) {
1501
+ global struct block_mxfp4 * b = (global struct block_mxfp4 *) src0 + get_global_id(0);
1502
+ global uchar * q = (global uchar *) dst_q + QK_MXFP4 / 2 * get_global_id(0);
1503
+ global uchar * e = (global uchar *) dst_e + get_global_id(0);
1504
+
1505
+ *e = b->e;
1506
+
1507
+ for (int i = 0; i < QK_MXFP4 / 2; ++i) {
1508
+ q[i] = b->qs[i];
1509
+ }
1510
+ }
1511
+
1512
+ kernel void kernel_convert_block_mxfp4_trans(
1513
+ global struct block_mxfp4 * src0,
1514
+ __global uint4 * dst_q,
1515
+ __global uchar * dst_e,
1516
+ uint ne00,
1517
+ uint ne01
1518
+ ) {
1519
+ int i00 = get_global_id(1);
1520
+ uint i01 = get_global_id(0);
1521
+ uint i02 = get_global_id(2);
1522
+
1523
+ uint ne00_blk = ne00 / QK_MXFP4;
1524
+ uint src_blk_offset = i00 + i01 * ne00_blk + i02 * ne00_blk * ne01;
1525
+ uint dst_blk_offset = i01 + i00 * ne01 + i02 * ne00_blk * ne01;
1526
+
1527
+ global struct block_mxfp4 * b = src0 + src_blk_offset;
1528
+
1529
+ dst_q[dst_blk_offset] = ((global uint4 *)(&(b->qs[0])))[0];
1530
+ dst_e[dst_blk_offset] = b->e;
1531
+ }
1532
+
1533
+ kernel void kernel_restore_block_mxfp4(
1534
+ global uchar * src_q,
1535
+ global half * src_e,
1536
+ global struct block_mxfp4 * dst
1537
+ ) {
1538
+ global struct block_mxfp4 * b = (global struct block_mxfp4 *) dst + get_global_id(0);
1539
+ global uchar * q = (global uchar *) src_q + QK_MXFP4 / 2 * get_global_id(0);
1540
+ global uchar * e = (global uchar *) src_e + get_global_id(0);
1541
+
1542
+ b->e = *e;
1543
+ for (int i = 0; i < QK_MXFP4 / 2; ++i) {
1544
+ b->qs[i] = q[i];
1545
+ }
1546
+ }
1547
+
1548
+ kernel void kernel_restore_block_mxfp4_trans(
1549
+ __global uint4 * src_q,
1550
+ __global uchar * src_e,
1551
+ global struct block_mxfp4 * dst,
1552
+ uint ne00,
1553
+ uint ne01
1554
+ ) {
1555
+ int i00 = get_global_id(1);
1556
+ uint i01 = get_global_id(0);
1557
+ uint i02 = get_global_id(2);
1558
+
1559
+ uint ne00_blk = ne00 / QK_MXFP4;
1560
+ uint src_blk_offset = i01 + i00 * ne01 + i02 * ne00_blk * ne01;
1561
+ uint dst_blk_offset = i00 + i01 * ne00_blk + i02 * ne00_blk * ne01;
1562
+
1563
+ global struct block_mxfp4 * b = dst + dst_blk_offset;
1564
+
1565
+ ((global uint4 *)(&(b->qs[0])))[0] = src_q[src_blk_offset];
1566
+ b->e = src_e[src_blk_offset];
1567
+ }
1568
+
1569
+ //------------------------------------------------------------------------------
1570
+ // kernel_convert_block_q6_k_tiled_ns
1571
+ //
1572
+ // Tiled-wide layout for the long-vocab q6_K lm_head/embed GEMV (decode path).
1573
+ // Unlike *_trans4_ns (which mirrors the bit-interleave the legacy 2-output GEMV
1574
+ // consumes), this kernel is correct-by-construction against the CANONICAL ggml
1575
+ // q6_K dequant: it recovers each weight's 6-bit code in element order e in
1576
+ // [0,256), then re-packs low-4-bits into 32 uints (8 codes/uint) and high-2-bits
1577
+ // into 16 uints (16 codes/uint). The matching GEMV (gemv_noshuffle_q6_k_f32_tiled)
1578
+ // unpacks the same order, so both ends are owned here.
1579
+ //
1580
+ // Storage is TILED by 64 output rows so the GEMV's 64-thread tile coalesces:
1581
+ // ql uint4 granule g of (row r, K-block sb): idx = ((rt*ne00_blk + sb)*8 + g)*64 + rit
1582
+ // qh uint4 granule g: idx = ((rt*ne00_blk + sb)*4 + g)*64 + rit
1583
+ // scales (char16) of (r, sb): idx = (rt*ne00_blk + sb)*64 + rit
1584
+ // d (half) of (r, sb): idx = (rt*ne00_blk + sb)*64 + rit
1585
+ // where rt = r/64, rit = r%64. Requires ne01 % 64 == 0 (gated host-side).
1586
+ // Buffer sizes are byte-identical to the trans4_ns layout.
1587
+ //------------------------------------------------------------------------------
1588
+ kernel void kernel_convert_block_q6_k_tiled_ns(
1589
+ __global struct block_q6_K * src0,
1590
+ __global uint * dst_ql, // 32 uints / superblock (low 4 bits, 8 codes/uint)
1591
+ __global uint * dst_qh, // 16 uints / superblock (high 2 bits, 16 codes/uint)
1592
+ __global half * dst_d, // 1 half / superblock
1593
+ __global char * dst_s, // 16 chars/ superblock
1594
+ uint ne00,
1595
+ uint ne01
1596
+ ) {
1597
+ uint i00 = get_global_id(1); // K-block index (superblock along ne00)
1598
+ uint i01 = get_global_id(0); // output row index (along ne01)
1599
+ uint i02 = get_global_id(2); // batch
1600
+
1601
+ uint ne00_blk = ne00 / QK_K;
1602
+
1603
+ // Source block: row-major over (i02, i01, i00).
1604
+ uint src_blk_offset = i00 + i01 * ne00_blk + i02 * ne00_blk * ne01;
1605
+ __global struct block_q6_K * b = src0 + src_blk_offset;
1606
+
1607
+ uint rt = i01 / 64;
1608
+ uint rit = i01 % 64;
1609
+ uint tile_blk = (i02 * (ne01 / 64) + rt) * ne00_blk + i00; // tile-major (row-tile, K-block)
1610
+
1611
+ // --- recover canonical 6-bit codes, pack into ql (4b) + qh (2b) in e-order ---
1612
+ // 32 ql-uints (8 low-nibbles each) + 16 qh-uints (16 2-bit slots each).
1613
+ uint qlw[32] = {0};
1614
+ uint qhw[16] = {0};
1615
+
1616
+ for (uint e = 0; e < 256; ++e) {
1617
+ uint n = (e >= 128) ? 1u : 0u; // which 128-half
1618
+ uint within = e - n * 128u;
1619
+ uint q = within / 32u; // quadrant 0..3
1620
+ uint l = within % 32u; // 0..31
1621
+
1622
+ uint off_ql = n * 64u; // raw ql byte base for this half
1623
+ uint off_qh = n * 32u; // raw qh byte base for this half
1624
+
1625
+ uchar low4;
1626
+ uchar qlb0 = b->ql[off_ql + l];
1627
+ uchar qlb1 = b->ql[off_ql + l + 32];
1628
+ if (q == 0) low4 = qlb0 & 0x0F;
1629
+ else if (q == 1) low4 = qlb1 & 0x0F;
1630
+ else if (q == 2) low4 = (qlb0 >> 4) & 0x0F;
1631
+ else low4 = (qlb1 >> 4) & 0x0F;
1632
+
1633
+ uchar hi2 = (b->qh[off_qh + l] >> (q * 2u)) & 0x03;
1634
+
1635
+ // pack low4 (e-order): uint e/8, nibble (e%8)
1636
+ qlw[e >> 3] |= ((uint)low4) << ((e & 7u) * 4u);
1637
+ // pack hi2 (e-order): uint e/16, 2-bit slot (e%16)
1638
+ qhw[e >> 4] |= ((uint)hi2) << ((e & 15u) * 2u);
1639
+ }
1640
+
1641
+ // --- write tiled ---
1642
+ for (uint g = 0; g < 8; ++g) {
1643
+ uint base = (tile_blk * 8u + g) * 64u + rit; // uint4 index
1644
+ dst_ql[base * 4u + 0u] = qlw[g * 4u + 0u];
1645
+ dst_ql[base * 4u + 1u] = qlw[g * 4u + 1u];
1646
+ dst_ql[base * 4u + 2u] = qlw[g * 4u + 2u];
1647
+ dst_ql[base * 4u + 3u] = qlw[g * 4u + 3u];
1648
+ }
1649
+ for (uint g = 0; g < 4; ++g) {
1650
+ uint base = (tile_blk * 4u + g) * 64u + rit; // uint4 index
1651
+ dst_qh[base * 4u + 0u] = qhw[g * 4u + 0u];
1652
+ dst_qh[base * 4u + 1u] = qhw[g * 4u + 1u];
1653
+ dst_qh[base * 4u + 2u] = qhw[g * 4u + 2u];
1654
+ dst_qh[base * 4u + 3u] = qhw[g * 4u + 3u];
1655
+ }
1656
+
1657
+ // scales: 16 chars contiguous per (row, block), tiled
1658
+ __global char * s_dst = dst_s + (tile_blk * 64u + rit) * 16u;
1659
+ #pragma unroll
1660
+ for (int i = 0; i < 16; ++i) {
1661
+ s_dst[i] = b->scales[i];
1662
+ }
1663
+
1664
+ // super-block scale
1665
+ dst_d[tile_blk * 64u + rit] = b->d;
1666
+ }
1667
+
1668
+ kernel void kernel_convert_block_mxfp4_trans4_ns(
1669
+ global struct block_mxfp4 * src0,
1670
+ __global uint * dst_q,
1671
+ __global uchar * dst_e,
1672
+ uint ne00,
1673
+ uint ne01
1674
+ ) {
1675
+ uint i00 = get_global_id(1);
1676
+ uint i01 = get_global_id(0);
1677
+ uint i02 = get_global_id(2);
1678
+
1679
+ if (i01 >= ne01) {
1680
+ return;
1681
+ }
1682
+
1683
+ uint ne00_blk = ne00 / QK_MXFP4;
1684
+ uint src_blk_offset = i00 + i01 * ne00_blk + i02 * ne00_blk * ne01;
1685
+ uint dst_blk_offset = i01 + i00 * ne01 + i02 * ne00_blk * ne01;
1686
+
1687
+ global struct block_mxfp4 * b = src0 + src_blk_offset;
1688
+ dst_e[dst_blk_offset] = b->e;
1689
+
1690
+ // extract quantization and unshuffle
1691
+ ushort8 pre_block = ((global ushort8 *)(&(b->qs[0])))[0];
1692
+
1693
+ ushort8 post_block = (ushort8)(0);
1694
+
1695
+ uchar * pre_block_ptr = (uchar *)(&pre_block);
1696
+ uchar * post_block_ptr = (uchar *)(&post_block);
1697
+
1698
+ for (int i = 0; i < QK_MXFP4 / 4; ++i) {
1699
+ uchar x0 = pre_block_ptr[2*i + 0];
1700
+ uchar x1 = pre_block_ptr[2*i + 1];
1701
+
1702
+ post_block_ptr[i + 0 ] = convert_uchar(x0 & 0x0F) | convert_uchar((x1 & 0x0F) << 4);
1703
+ post_block_ptr[i + QK_MXFP4 / 4] = convert_uchar((x0 & 0xF0) >> 4) | convert_uchar(x1 & 0xF0);
1704
+ }
1705
+
1706
+ uint4 q_block = as_uint4(post_block);
1707
+
1708
+ uint offset = i02 * ne00_blk * ne01 * 4 + i00 * ne01 * 4 + i01;
1709
+ dst_q[offset] = q_block.x;
1710
+ dst_q[offset + ne01] = q_block.y;
1711
+ dst_q[offset + ne01 * 2] = q_block.z;
1712
+ dst_q[offset + ne01 * 3] = q_block.w;
1713
+ }
1714
+
1715
+ kernel void kernel_restore_block_mxfp4_trans4_ns(
1716
+ __global uint * src_q,
1717
+ __global uchar * src_e,
1718
+ __global struct block_mxfp4 * dst0,
1719
+ uint ne00,
1720
+ uint ne01
1721
+ ) {
1722
+ uint i00 = get_global_id(1);
1723
+ uint i01 = get_global_id(0);
1724
+ uint i02 = get_global_id(2);
1725
+
1726
+ if (i01 >= ne01) {
1727
+ return;
1728
+ }
1729
+
1730
+ uint ne00_blk = ne00 / QK_MXFP4;
1731
+ uint dst_blk_offset = i00 + i01 * ne00_blk + i02 * ne00_blk * ne01;
1732
+ uint src_d_offset = i01 + i00 * ne01 + i02 * ne00_blk * ne01;
1733
+
1734
+ __global struct block_mxfp4 * b = dst0 + dst_blk_offset;
1735
+ b->e = src_e[src_d_offset];
1736
+
1737
+ // collect transposed quantization parts for a block
1738
+ uint src_q_offset = i02 * ne00_blk * ne01 * 4 + i00 * ne01 * 4 + i01;
1739
+ uint4 q_block;
1740
+ q_block.x = src_q[src_q_offset];
1741
+ q_block.y = src_q[src_q_offset + ne01];
1742
+ q_block.z = src_q[src_q_offset + ne01 * 2];
1743
+ q_block.w = src_q[src_q_offset + ne01 * 3];
1744
+
1745
+ ushort8 post_block = as_ushort8(q_block);
1746
+ ushort8 pre_block = (ushort8)(0);
1747
+
1748
+ uchar * pre_block_ptr = (uchar *)(&pre_block);
1749
+ uchar * post_block_ptr = (uchar *)(&post_block);
1750
+
1751
+ for (int i = 0; i < QK_MXFP4 / 4; ++i) {
1752
+ uchar x0 = post_block_ptr[i + 0];
1753
+ uchar x1 = post_block_ptr[i + QK_MXFP4 / 4];
1754
+
1755
+ pre_block_ptr[2 * i + 0] = convert_uchar(x0 & 0x0F) | convert_uchar((x1 & 0x0F) << 4);
1756
+ pre_block_ptr[2 * i + 1] = convert_uchar((x0 & 0xF0) >> 4) | convert_uchar(x1 & 0xF0);
1757
+ }
1758
+
1759
+ ((__global ushort8 *)(&(b->qs[0])))[0] = pre_block;
1760
+ }
1761
+
1762
+
1763
+ //------------------------------------------------------------------------------
1764
+ // block_q8_0
1765
+ //------------------------------------------------------------------------------
1766
+ typedef struct {
1767
+ half d; // delta
1768
+ char qs[QK8_0]; // quants
1769
+ } block_q8_0;
1770
+
1771
+ kernel void kernel_convert_block_q8_0(
1772
+ global block_q8_0 * src0,
1773
+ global uchar * dst_q,
1774
+ global half * dst_d
1775
+ ) {
1776
+ global block_q8_0 * b = (global block_q8_0 *) src0 + get_global_id(0);
1777
+ global uchar * q = (global uchar *) dst_q + QK8_0*get_global_id(0);
1778
+ global half * d = (global half *) dst_d + get_global_id(0);
1779
+
1780
+ *d = b->d;
1781
+
1782
+ for (int i = 0; i < QK8_0; ++i) {
1783
+ q[i] = b->qs[i];
1784
+ }
1785
+ }
1786
+
1787
+ kernel void kernel_restore_block_q8_0(
1788
+ global uchar * src_q,
1789
+ global half * src_d,
1790
+ global block_q8_0 * dst
1791
+ ) {
1792
+ global block_q8_0 * b = (global block_q8_0 *) dst + get_global_id(0);
1793
+ global uchar * q = (global uchar *) src_q + QK8_0*get_global_id(0);
1794
+ global half * d = (global half *) src_d + get_global_id(0);
1795
+
1796
+ b->d = *d;
1797
+ for (int i = 0; i < QK8_0; ++i) {
1798
+ b->qs[i] = q[i];
1799
+ }
1800
+ }
1801
+
1802
+ // View-aware AoS q8_0 -> f32 dequant (f32/f32 FA path).
1803
+ kernel void kernel_dequant_q8_0_f32_view_aos(
1804
+ global char * src,
1805
+ ulong src_offset,
1806
+ ulong src_nb1,
1807
+ ulong src_nb2,
1808
+ ulong src_nb3,
1809
+ int nblk0,
1810
+ int ne1,
1811
+ int ne2,
1812
+ int ne3,
1813
+ global float * dst
1814
+ ) {
1815
+ int blk_i0 = get_global_id(0);
1816
+ int i1 = get_global_id(1);
1817
+ int batch = get_global_id(2);
1818
+
1819
+ if (blk_i0 >= nblk0) return;
1820
+ if (i1 >= ne1) return;
1821
+
1822
+ int i2 = batch % ne2;
1823
+ int i3 = batch / ne2;
1824
+ if (i3 >= ne3) return;
1825
+
1826
+ global char * block = src + src_offset + (ulong)i3*src_nb3 + (ulong)i2*src_nb2 + (ulong)i1*src_nb1 + (ulong)blk_i0 * (2 + QK8_0);
1827
+ float d = vload_half(0, (global half *)block);
1828
+ global char * qs = block + 2;
1829
+
1830
+ ulong dst_row_base = ((ulong)i3 * ne2 * ne1 + (ulong)i2 * ne1 + (ulong)i1) * nblk0;
1831
+ global float * out = dst + (dst_row_base + blk_i0) * QK8_0;
1832
+
1833
+ for (int i = 0; i < QK8_0; ++i) {
1834
+ out[i] = d * (float)qs[i];
1835
+ }
1836
+ }
1837
+
1838
+ // View-aware AoS q8_0 -> f16 dequant. Rows tight, batch strides may be gapped.
1839
+ kernel void kernel_dequant_q8_0_f16_view_aos(
1840
+ global char * src,
1841
+ ulong src_offset,
1842
+ ulong src_nb1,
1843
+ ulong src_nb2,
1844
+ ulong src_nb3,
1845
+ int nblk0,
1846
+ int ne1,
1847
+ int ne2,
1848
+ int ne3,
1849
+ global half * dst
1850
+ ) {
1851
+ int blk_i0 = get_global_id(0);
1852
+ int i1 = get_global_id(1);
1853
+ int batch = get_global_id(2);
1854
+
1855
+ if (blk_i0 >= nblk0) return;
1856
+ if (i1 >= ne1) return;
1857
+
1858
+ int i2 = batch % ne2;
1859
+ int i3 = batch / ne2;
1860
+ if (i3 >= ne3) return;
1861
+
1862
+ global char * block = src + src_offset + (ulong)i3*src_nb3 + (ulong)i2*src_nb2 + (ulong)i1*src_nb1 + (ulong)blk_i0 * (2 + QK8_0);
1863
+ float d = vload_half(0, (global half *)block);
1864
+ global char * qs = block + 2;
1865
+
1866
+ ulong dst_row_base = ((ulong)i3 * ne2 * ne1 + (ulong)i2 * ne1 + (ulong)i1) * nblk0;
1867
+ global half * out = dst + (dst_row_base + blk_i0) * QK8_0;
1868
+
1869
+ for (int i = 0; i < QK8_0; ++i) {
1870
+ out[i] = (half)(d * (float)qs[i]);
1871
+ }
1872
+ }
1873
+
1874
+ // View-aware AoS q4_0 -> f32 dequant (mirrors the q8_0 view variant).
1875
+ kernel void kernel_dequant_q4_0_f32_view_aos(
1876
+ global char * src,
1877
+ ulong src_offset,
1878
+ ulong src_nb1,
1879
+ ulong src_nb2,
1880
+ ulong src_nb3,
1881
+ int nblk0,
1882
+ int ne1,
1883
+ int ne2,
1884
+ int ne3,
1885
+ global float * dst
1886
+ ) {
1887
+ int blk_i0 = get_global_id(0);
1888
+ int i1 = get_global_id(1);
1889
+ int batch = get_global_id(2);
1890
+
1891
+ if (blk_i0 >= nblk0) return;
1892
+ if (i1 >= ne1) return;
1893
+
1894
+ int i2 = batch % ne2;
1895
+ int i3 = batch / ne2;
1896
+ if (i3 >= ne3) return;
1897
+
1898
+ global char * block = src + src_offset + (ulong)i3*src_nb3 + (ulong)i2*src_nb2 + (ulong)i1*src_nb1 + (ulong)blk_i0 * (2 + QK4_0/2);
1899
+ float d = vload_half(0, (global half *)block);
1900
+ global uchar * qs = (global uchar *)(block + 2);
1901
+
1902
+ ulong dst_row_base = ((ulong)i3 * ne2 * ne1 + (ulong)i2 * ne1 + (ulong)i1) * nblk0;
1903
+ global float * out = dst + (dst_row_base + blk_i0) * QK4_0;
1904
+
1905
+ for (int i = 0; i < QK4_0/2; ++i) {
1906
+ uchar byte = qs[i];
1907
+ int q0 = (int)(byte & 0x0F) - 8;
1908
+ int q1 = (int)(byte >> 4) - 8;
1909
+ out[i] = d * (float)q0;
1910
+ out[i + QK4_0/2] = d * (float)q1;
1911
+ }
1912
+ }
1913
+
1914
+ // View-aware AoS q4_0 -> f16 dequant (mirrors the q8_0 view variant).
1915
+ kernel void kernel_dequant_q4_0_f16_view_aos(
1916
+ global char * src,
1917
+ ulong src_offset,
1918
+ ulong src_nb1,
1919
+ ulong src_nb2,
1920
+ ulong src_nb3,
1921
+ int nblk0,
1922
+ int ne1,
1923
+ int ne2,
1924
+ int ne3,
1925
+ global half * dst
1926
+ ) {
1927
+ int blk_i0 = get_global_id(0);
1928
+ int i1 = get_global_id(1);
1929
+ int batch = get_global_id(2);
1930
+
1931
+ if (blk_i0 >= nblk0) return;
1932
+ if (i1 >= ne1) return;
1933
+
1934
+ int i2 = batch % ne2;
1935
+ int i3 = batch / ne2;
1936
+ if (i3 >= ne3) return;
1937
+
1938
+ global char * block = src + src_offset + (ulong)i3*src_nb3 + (ulong)i2*src_nb2 + (ulong)i1*src_nb1 + (ulong)blk_i0 * (2 + QK4_0/2);
1939
+ float d = vload_half(0, (global half *)block);
1940
+ global uchar * qs = (global uchar *)(block + 2);
1941
+
1942
+ ulong dst_row_base = ((ulong)i3 * ne2 * ne1 + (ulong)i2 * ne1 + (ulong)i1) * nblk0;
1943
+ global half * out = dst + (dst_row_base + blk_i0) * QK4_0;
1944
+
1945
+ for (int i = 0; i < QK4_0/2; ++i) {
1946
+ uchar byte = qs[i];
1947
+ int q0 = (int)(byte & 0x0F) - 8;
1948
+ int q1 = (int)(byte >> 4) - 8;
1949
+ out[i] = (half)(d * (float)q0);
1950
+ out[i + QK4_0/2] = (half)(d * (float)q1);
1951
+ }
1952
+ }
1953
+
1954
+ kernel void kernel_restore_block_q8_0_trans(
1955
+ global uchar * src_q,
1956
+ global half * src_d,
1957
+ global block_q8_0 * dst,
1958
+ uint ne00,
1959
+ uint ne01
1960
+ ){
1961
+ uint num_blk_per_row = ne00 / QK8_0;
1962
+
1963
+ global block_q8_0 * b = (global block_q8_0 *) dst + get_global_id(0) * num_blk_per_row;
1964
+ global uchar * q = (global uchar *) src_q + get_global_id(0) * 4; // 4 8-bit packed
1965
+ global half * d = (global half *) src_d + get_global_id(0);
1966
+
1967
+ for (uint blk = 0; blk < num_blk_per_row; blk++) {
1968
+ b->d = *d;
1969
+
1970
+ for (uint i = 0; i < QK8_0; i+=4) {
1971
+ b->qs[i] = q[0];
1972
+ b->qs[i+1] = q[1];
1973
+ b->qs[i+2] = q[2];
1974
+ b->qs[i+3] = q[3];
1975
+
1976
+ q += 4 * ne01; // M stride
1977
+ }
1978
+
1979
+ d += ne01;
1980
+
1981
+ b++;
1982
+ }
1983
+ }
1984
+
1985
+ //------------------------------------------------------------------------------
1986
+ // kernel_convert_block_q4_K
1987
+ // Convert the block_q4_K format to 4 separate arrays (AOS -> SOA).
1988
+ // This kernel does not deshuffle the bits.
1989
+ // Each thread processes a super block.
1990
+ // Mask args are just to keep the signature consistent with the no-shuffle
1991
+ // version and they are not used in this kernel.
1992
+ //------------------------------------------------------------------------------
1993
+ kernel void kernel_convert_block_q4_K(
1994
+ global struct block_q4_K * src0,
1995
+ global uchar * dst_q,
1996
+ global uchar * dst_s,
1997
+ global half * dst_d,
1998
+ global half * dst_dm,
1999
+ uchar mask_0F,
2000
+ uchar mask_F0
2001
+ ) {
2002
+ global struct block_q4_K * b = (global struct block_q4_K *) src0 + get_global_id(0);
2003
+ global uchar * q = (global uchar *) dst_q + QK_K/2*get_global_id(0);
2004
+ global uchar * s = (global uchar *) dst_s + K_SCALE_SIZE*get_global_id(0);
2005
+ global half * d = (global half *) dst_d + get_global_id(0);
2006
+ global half * dm = (global half *) dst_dm + get_global_id(0);
2007
+
2008
+ *d = b->d;
2009
+ *dm = b->dm;
2010
+
2011
+ for (int i = 0; i < QK_K/2; ++i) {
2012
+ q[i] = b->q[i];
2013
+ }
2014
+ for (int i = 0; i < K_SCALE_SIZE; ++i) {
2015
+ s[i] = b->s[i];
2016
+ }
2017
+ }
2018
+
2019
+ // Restore block_q4_K from flattened arrays.
2020
+ // Each thread processes a super block.
2021
+ // Mask args are just to keep the signature consistent with the no-shuffle ones.
2022
+ kernel void kernel_restore_block_q4_K(
2023
+ global uchar * src_q,
2024
+ global uchar * src_s,
2025
+ global half * src_d,
2026
+ global half * src_dm,
2027
+ global struct block_q4_K * dst,
2028
+ uchar mask_0F,
2029
+ uchar mask_F0
2030
+ ) {
2031
+ global struct block_q4_K * b = (global struct block_q4_K *) dst + get_global_id(0);
2032
+ global uchar * q = (global uchar *) src_q + QK_K/2*get_global_id(0);
2033
+ global uchar * s = (global uchar *) src_s + K_SCALE_SIZE*get_global_id(0);
2034
+ global half * d = (global half *) src_d + get_global_id(0);
2035
+ global half * dm = (global half *) src_dm + get_global_id(0);
2036
+
2037
+ b->d = *d;
2038
+ b->dm = *dm;
2039
+
2040
+ for (int i = 0; i < QK_K/2; ++i) {
2041
+ b->q[i] = q[i];
2042
+ }
2043
+ for (int i = 0; i < K_SCALE_SIZE; ++i) {
2044
+ b->s[i] = s[i];
2045
+ }
2046
+ }
2047
+
2048
+ kernel void kernel_convert_block_q4_K_noshuffle(
2049
+ global struct block_q4_K * src0,
2050
+ global uchar * dst_q,
2051
+ global uchar * dst_s,
2052
+ global half * dst_d,
2053
+ global half * dst_dm,
2054
+ uchar mask_0F,
2055
+ uchar mask_F0
2056
+ ) {
2057
+ global struct block_q4_K * b = (global struct block_q4_K *) src0 + get_global_id(0);
2058
+ global uchar * q = (global uchar *) dst_q + QK_K/2 * get_global_id(0);
2059
+ global uchar * s = (global uchar *) dst_s + K_SCALE_SIZE * get_global_id(0);
2060
+ global half * d = (global half *) dst_d + get_global_id(0);
2061
+ global half * dm = (global half *) dst_dm + get_global_id(0);
2062
+
2063
+ *d = b->d;
2064
+ *dm = b->dm;
2065
+
2066
+ for (int i = 0; i < QK_K / 64; ++i) {
2067
+ for (int j = 0; j < 16; ++j) {
2068
+ uchar x0 = b->q[i*32 + 2*j];
2069
+ uchar x1 = b->q[i*32 + 2*j + 1];
2070
+ q[i*32 + j] = convert_uchar(x0 & mask_0F) | convert_uchar((x1 & mask_0F) << 4);
2071
+ q[i*32 + j + 16] = convert_uchar((x0 & mask_F0) >> 4) | convert_uchar(x1 & mask_F0);
2072
+ }
2073
+ }
2074
+
2075
+ for (int i = 0; i < K_SCALE_SIZE; ++i) {
2076
+ s[i] = b->s[i];
2077
+ }
2078
+ }
2079
+
2080
+ kernel void kernel_restore_block_q4_K_noshuffle(
2081
+ global uchar * src_q,
2082
+ global uchar * src_s,
2083
+ global half * src_d,
2084
+ global half * src_dm,
2085
+ global struct block_q4_K * dst,
2086
+ uchar mask_0F,
2087
+ uchar mask_F0
2088
+ ) {
2089
+ global struct block_q4_K * b = (global struct block_q4_K *) dst + get_global_id(0);
2090
+ global uchar * q = (global uchar *) src_q + QK_K/2 * get_global_id(0);
2091
+ global uchar * s = (global uchar *) src_s + K_SCALE_SIZE * get_global_id(0);
2092
+ global half * d = (global half *) src_d + get_global_id(0);
2093
+ global half * dm = (global half *) src_dm + get_global_id(0);
2094
+
2095
+ b->d = *d;
2096
+ b->dm = *dm;
2097
+
2098
+ for (int i = 0; i < QK_K / 64; ++i) {
2099
+ for (int j = 0; j < 16; ++j) {
2100
+ uchar lo = q[i*32 + j];
2101
+ uchar hi = q[i*32 + j + 16];
2102
+ b->q[i*32 + 2*j] = convert_uchar((lo & mask_0F) | ((hi & mask_0F) << 4));
2103
+ b->q[i*32 + 2*j + 1] = convert_uchar(((lo & mask_F0) >> 4) | (hi & mask_F0));
2104
+ }
2105
+ }
2106
+
2107
+ for (int i = 0; i < K_SCALE_SIZE; ++i) {
2108
+ b->s[i] = s[i];
2109
+ }
2110
+ }
2111
+
2112
+ //------------------------------------------------------------------------------
2113
+ // kernel_convert_block_q5_K
2114
+ // Convert the block_q5_K format to 5 separate arrays (AOS -> SOA).
2115
+ // Each thread processes a super block.
2116
+ //------------------------------------------------------------------------------
2117
+ kernel void kernel_convert_block_q5_K(
2118
+ global struct block_q5_K * src0,
2119
+ global uchar * dst_q,
2120
+ global uchar * dst_qh,
2121
+ global uchar * dst_s,
2122
+ global half * dst_d,
2123
+ global half * dst_dm,
2124
+ uchar mask_0F,
2125
+ uchar mask_F0
2126
+ ) {
2127
+ global struct block_q5_K * b = (global struct block_q5_K *) src0 + get_global_id(0);
2128
+ global uchar * q = (global uchar *) dst_q + QK_K/2*get_global_id(0);
2129
+ global uchar * qh = (global uchar *) dst_qh + QK_K/8*get_global_id(0);
2130
+ global uchar * s = (global uchar *) dst_s + K_SCALE_SIZE*get_global_id(0);
2131
+ global half * d = (global half *) dst_d + get_global_id(0);
2132
+ global half * dm = (global half *) dst_dm + get_global_id(0);
2133
+
2134
+ *d = b->d;
2135
+ *dm = b->dm;
2136
+
2137
+ for (int i = 0; i < QK_K/2; ++i) {
2138
+ q[i] = b->qs[i];
2139
+ }
2140
+ for (int i = 0; i < QK_K/8; ++i) {
2141
+ qh[i] = b->qh[i];
2142
+ }
2143
+ for (int i = 0; i < K_SCALE_SIZE; ++i) {
2144
+ s[i] = b->s[i];
2145
+ }
2146
+ }
2147
+
2148
+ // Restore block_q5_K from flattened arrays.
2149
+ // Each thread processes a super block.
2150
+ kernel void kernel_restore_block_q5_K(
2151
+ global uchar * src_q,
2152
+ global uchar * src_qh,
2153
+ global uchar * src_s,
2154
+ global half * src_d,
2155
+ global half * src_dm,
2156
+ global struct block_q5_K * dst,
2157
+ uchar mask_0F,
2158
+ uchar mask_F0
2159
+ ) {
2160
+ global struct block_q5_K * b = (global struct block_q5_K *) dst + get_global_id(0);
2161
+ global uchar * q = (global uchar *) src_q + QK_K/2*get_global_id(0);
2162
+ global uchar * qh = (global uchar *) src_qh + QK_K/8*get_global_id(0);
2163
+ global uchar * s = (global uchar *) src_s + K_SCALE_SIZE*get_global_id(0);
2164
+ global half * d = (global half *) src_d + get_global_id(0);
2165
+ global half * dm = (global half *) src_dm + get_global_id(0);
2166
+
2167
+ b->d = *d;
2168
+ b->dm = *dm;
2169
+
2170
+ for (int i = 0; i < QK_K/2; ++i) {
2171
+ b->qs[i] = q[i];
2172
+ }
2173
+ for (int i = 0; i < QK_K/8; ++i) {
2174
+ b->qh[i] = qh[i];
2175
+ }
2176
+ for (int i = 0; i < K_SCALE_SIZE; ++i) {
2177
+ b->s[i] = s[i];
2178
+ }
2179
+ }
2180
+
2181
+ kernel void kernel_convert_block_q5_K_noshuffle(
2182
+ global struct block_q5_K * src0,
2183
+ global uchar * dst_q,
2184
+ global uchar * dst_qh,
2185
+ global uchar * dst_s,
2186
+ global half * dst_d,
2187
+ global half * dst_dm,
2188
+ uchar mask_0F,
2189
+ uchar mask_F0
2190
+ ) {
2191
+ global struct block_q5_K * b = (global struct block_q5_K *) src0 + get_global_id(0);
2192
+ global uchar * q = (global uchar *) dst_q + QK_K/2 * get_global_id(0);
2193
+ global uchar * qh = (global uchar *) dst_qh + QK_K/8 * get_global_id(0);
2194
+ global uchar * s = (global uchar *) dst_s + K_SCALE_SIZE * get_global_id(0);
2195
+ global half * d = (global half *) dst_d + get_global_id(0);
2196
+ global half * dm = (global half *) dst_dm + get_global_id(0);
2197
+
2198
+ *d = b->d;
2199
+ *dm = b->dm;
2200
+
2201
+ for (int i = 0; i < QK_K / 64; ++i) {
2202
+ for (int j = 0; j < 16; ++j) {
2203
+ uchar x0 = b->qs[i*32 + 2*j];
2204
+ uchar x1 = b->qs[i*32 + 2*j + 1];
2205
+ q[i*32 + j] = convert_uchar(x0 & mask_0F) | convert_uchar((x1 & mask_0F) << 4);
2206
+ q[i*32 + j + 16] = convert_uchar((x0 & mask_F0) >> 4) | convert_uchar(x1 & mask_F0);
2207
+ }
2208
+ }
2209
+
2210
+ for (int l = 0; l < QK_K/8; ++l) {
2211
+ uchar x0 = 0;
2212
+ for (int i = 0; i < 8; ++i) {
2213
+ x0 |= ((b->qh[(l%4)*8+i] >> (l/4)) & 0x01) << i;
2214
+ }
2215
+ qh[l] = x0;
2216
+ }
2217
+
2218
+ for (int i = 0; i < K_SCALE_SIZE; ++i) {
2219
+ s[i] = b->s[i];
2220
+ }
2221
+ }
2222
+
2223
+ kernel void kernel_restore_block_q5_K_noshuffle(
2224
+ global uchar * src_q,
2225
+ global uchar * src_qh,
2226
+ global uchar * src_s,
2227
+ global half * src_d,
2228
+ global half * src_dm,
2229
+ global struct block_q5_K * dst,
2230
+ uchar mask_0F,
2231
+ uchar mask_F0
2232
+ ) {
2233
+ global struct block_q5_K * b = (global struct block_q5_K *) dst + get_global_id(0);
2234
+ global uchar * q = (global uchar *) src_q + QK_K/2 * get_global_id(0);
2235
+ global uchar * qh = (global uchar *) src_qh + QK_K/8 * get_global_id(0);
2236
+ global uchar * s = (global uchar *) src_s + K_SCALE_SIZE * get_global_id(0);
2237
+ global half * d = (global half *) src_d + get_global_id(0);
2238
+ global half * dm = (global half *) src_dm + get_global_id(0);
2239
+
2240
+ b->d = *d;
2241
+ b->dm = *dm;
2242
+
2243
+ for (int i = 0; i < QK_K / 64; ++i) {
2244
+ for (int j = 0; j < 16; ++j) {
2245
+ uchar lo = q[i*32 + j];
2246
+ uchar hi = q[i*32 + j + 16];
2247
+ b->qs[i*32 + 2*j] = convert_uchar((lo & mask_0F) | ((hi & mask_0F) << 4));
2248
+ b->qs[i*32 + 2*j + 1] = convert_uchar(((lo & mask_F0) >> 4) | (hi & mask_F0));
2249
+ }
2250
+ }
2251
+
2252
+ for (int g = 0; g < 4; ++g) {
2253
+ for (int i = 0; i < 8; ++i) {
2254
+ uchar x0 = 0;
2255
+ for (int k = 0; k < 8; ++k) {
2256
+ x0 |= ((qh[4*k+g] >> i) & 0x01) << k;
2257
+ }
2258
+ b->qh[g*8+i] = x0;
2259
+ }
2260
+ }
2261
+
2262
+ for (int i = 0; i < K_SCALE_SIZE; ++i) {
2263
+ b->s[i] = s[i];
2264
+ }
2265
+ }
2266
+
2267
+ //------------------------------------------------------------------------------
2268
+ // kernel_convert_block_q6_K
2269
+ // Convert the block_q6_K format to 3 separate arrays (AOS -> SOA).
2270
+ // This kernel does not deshuffle the bits.
2271
+ // Each thread processes a super block.
2272
+ //------------------------------------------------------------------------------
2273
+ kernel void kernel_convert_block_q6_K(
2274
+ global struct block_q6_K * src0,
2275
+ global uchar * dst_ql,
2276
+ global uchar * dst_qh,
2277
+ global char * dst_s,
2278
+ global half * dst_d,
2279
+ uchar mask_lsb_8,
2280
+ ulong n_blk
2281
+ ) {
2282
+ if (get_global_id(0) >= n_blk) {
2283
+ return;
2284
+ }
2285
+ global struct block_q6_K * b = (global struct block_q6_K *) src0 + get_global_id(0);
2286
+ global uchar * ql = (global uchar *) dst_ql + QK_K/2*get_global_id(0);
2287
+ global uchar * qh = (global uchar *) dst_qh + QK_K/4*get_global_id(0);
2288
+ global char * s = (global char *) dst_s + QK_K/16*get_global_id(0);
2289
+ global half * d = (global half *) dst_d + get_global_id(0);
2290
+
2291
+ *d = b->d;
2292
+
2293
+ for (int i = 0; i < QK_K/2; ++i) {
2294
+ ql[i] = b->ql[i];
2295
+ }
2296
+ for (int i = 0; i < QK_K/4; ++i) {
2297
+ qh[i] = b->qh[i];
2298
+ }
2299
+ for (int i = 0; i < QK_K/16; ++i) {
2300
+ s[i] = b->scales[i];
2301
+ }
2302
+ }
2303
+
2304
+ // Restore block_q6_K from flattened arrays.
2305
+ // Each thread processes a super block.
2306
+ kernel void kernel_restore_block_q6_K(
2307
+ global uchar * dst_ql,
2308
+ global uchar * dst_qh,
2309
+ global char * dst_s,
2310
+ global half * dst_d,
2311
+ global struct block_q6_K * dst,
2312
+ uchar mask_lsb_8,
2313
+ ulong n_blk
2314
+ ) {
2315
+ if (get_global_id(0) >= n_blk) {
2316
+ return;
2317
+ }
2318
+ global struct block_q6_K * b = (global struct block_q6_K *) dst + get_global_id(0);
2319
+ global uchar * ql = (global uchar *) dst_ql + QK_K/2*get_global_id(0);
2320
+ global uchar * qh = (global uchar *) dst_qh + QK_K/4*get_global_id(0);
2321
+ global char * s = (global char *) dst_s + QK_K/16*get_global_id(0);
2322
+ global half * d = (global half *) dst_d + get_global_id(0);
2323
+
2324
+ b->d = *d;
2325
+
2326
+ for (int i = 0; i < QK_K/2; ++i) {
2327
+ b->ql[i] = ql[i];
2328
+ }
2329
+ for (int i = 0; i < QK_K/4; ++i) {
2330
+ b->qh[i] = qh[i];
2331
+ }
2332
+ for (int i = 0; i < QK_K/16; ++i) {
2333
+ b->scales[i] = s[i];
2334
+ }
2335
+ }
2336
+
2337
+ kernel void kernel_convert_block_q6_K_noshuffle(
2338
+ global struct block_q6_K * src0,
2339
+ global uchar * dst_ql,
2340
+ global uchar * dst_qh,
2341
+ global char * dst_s,
2342
+ global half * dst_d,
2343
+ uchar mask_lsb_8,
2344
+ ulong n_blk
2345
+ ) {
2346
+ if (get_global_id(0) >= n_blk) {
2347
+ return;
2348
+ }
2349
+ global struct block_q6_K * b = (global struct block_q6_K *) src0 + get_global_id(0);
2350
+ global uchar * ql = (global uchar *) dst_ql + QK_K/2*get_global_id(0);
2351
+ global uchar * qh = (global uchar *) dst_qh + QK_K/4*get_global_id(0);
2352
+ global char * s = (global char *) dst_s + QK_K/16*get_global_id(0);
2353
+ global half * d = (global half *) dst_d + get_global_id(0);
2354
+
2355
+ *d = b->d;
2356
+
2357
+ for (int i = 0; i < QK_K/2/4; ++i) {
2358
+ uchar x0 = b->ql[i*2 + 0] & mask_lsb_8;
2359
+ uchar x1 = b->ql[i*2 + 1] & mask_lsb_8;
2360
+ ql[i + 0] = (x0 & 0x0F) | ((x1 & 0x0F) << 4);
2361
+ ql[i + 32] = ((x0 & 0xF0) >> 4) | (x1 & 0xF0);
2362
+
2363
+ uchar x2 = b->ql[i*2 + 0 + 64] & mask_lsb_8;
2364
+ uchar x3 = b->ql[i*2 + 1 + 64] & mask_lsb_8;
2365
+ ql[i + 64] = (x2 & 0x0F) | ((x3 & 0x0F) << 4);
2366
+ ql[i + 96] = ((x2 & 0xF0) >> 4) | (x3 & 0xF0);
2367
+ }
2368
+
2369
+ for (int i = 0; i < QK_K/4/8; ++i) {
2370
+ uchar x0 = b->qh[i*4 + 0] & mask_lsb_8;
2371
+ uchar x1 = b->qh[i*4 + 1] & mask_lsb_8;
2372
+ uchar x2 = b->qh[i*4 + 2] & mask_lsb_8;
2373
+ uchar x3 = b->qh[i*4 + 3] & mask_lsb_8;
2374
+ qh[i + 0] = (x0 & 0x03) | ((x1 & 0x03) << 2) | ((x2 & 0x03) << 4) | ((x3 & 0x03) << 6);
2375
+ qh[i + 8] = ((x0 & 0x0C) >> 2) | (x1 & 0x0C) | ((x2 & 0x0C) << 2) | ((x3 & 0x0C) << 4);
2376
+ qh[i + 16] = ((x0 & 0x30) >> 4) | ((x1 & 0x30) >> 2) | (x2 & 0x30) | ((x3 & 0x30) << 2);
2377
+ qh[i + 24] = ((x0 & 0xC0) >> 6) | ((x1 & 0xC0) >> 4) | ((x2 & 0xC0) >> 2) | (x3 & 0xC0);
2378
+
2379
+ uchar x4 = b->qh[i*4 + 0 + 32] & mask_lsb_8;
2380
+ uchar x5 = b->qh[i*4 + 1 + 32] & mask_lsb_8;
2381
+ uchar x6 = b->qh[i*4 + 2 + 32] & mask_lsb_8;
2382
+ uchar x7 = b->qh[i*4 + 3 + 32] & mask_lsb_8;
2383
+ qh[i + 32] = (x4 & 0x03) | ((x5 & 0x03) << 2) | ((x6 & 0x03) << 4) | ((x7 & 0x03) << 6);
2384
+ qh[i + 40] = ((x4 & 0x0C) >> 2) | (x5 & 0x0C) | ((x6 & 0x0C) << 2) | ((x7 & 0x0C) << 4);
2385
+ qh[i + 48] = ((x4 & 0x30) >> 4) | ((x5 & 0x30) >> 2) | (x6 & 0x30) | ((x7 & 0x30) << 2);
2386
+ qh[i + 56] = ((x4 & 0xC0) >> 6) | ((x5 & 0xC0) >> 4) | ((x6 & 0xC0) >> 2) | (x7 & 0xC0);
2387
+ }
2388
+
2389
+ for (int i = 0; i < QK_K/16; ++i) {
2390
+ s[i] = b->scales[i];
2391
+ }
2392
+ }
2393
+
2394
+ kernel void kernel_restore_block_q6_K_noshuffle(
2395
+ global uchar * src_ql,
2396
+ global uchar * src_qh,
2397
+ global char * src_s,
2398
+ global half * src_d,
2399
+ global struct block_q6_K * dst,
2400
+ uchar mask_lsb_8,
2401
+ ulong n_blk
2402
+ ) {
2403
+ if (get_global_id(0) >= n_blk) {
2404
+ return;
2405
+ }
2406
+ global struct block_q6_K * b = (global struct block_q6_K *) dst + get_global_id(0);
2407
+ global uchar * ql = (global uchar *) src_ql + QK_K/2*get_global_id(0);
2408
+ global uchar * qh = (global uchar *) src_qh + QK_K/4*get_global_id(0);
2409
+ global char * s = (global char *) src_s + QK_K/16*get_global_id(0);
2410
+ global half * d = (global half *) src_d + get_global_id(0);
2411
+
2412
+ b->d = *d;
2413
+
2414
+ for (int i = 0; i < QK_K/2/4; ++i) {
2415
+ uchar x0 = ql[i + 0] & mask_lsb_8;
2416
+ uchar x1 = ql[i + 32] & mask_lsb_8;
2417
+ b->ql[i*2 + 0] = (x0 & 0x0F) | ((x1 & 0x0F) << 4);
2418
+ b->ql[i*2 + 1] = ((x0 & 0xF0) >> 4) | (x1 & 0xF0);
2419
+
2420
+ uchar x2 = ql[i + 64] & mask_lsb_8;
2421
+ uchar x3 = ql[i + 96] & mask_lsb_8;
2422
+ b->ql[i*2 + 0 + 64] = (x2 & 0x0F) | ((x3 & 0x0F) << 4);
2423
+ b->ql[i*2 + 1 + 64] = ((x2 & 0xF0) >> 4) | (x3 & 0xF0);
2424
+ }
2425
+
2426
+ for (int i = 0; i < QK_K/4/8; ++i) {
2427
+ uchar x0 = qh[i + 0] & mask_lsb_8;
2428
+ uchar x1 = qh[i + 8] & mask_lsb_8;
2429
+ uchar x2 = qh[i + 16] & mask_lsb_8;
2430
+ uchar x3 = qh[i + 24] & mask_lsb_8;
2431
+ b->qh[i*4 + 0] = (x0 & 0x03) | ((x1 & 0x03) << 2) | ((x2 & 0x03) << 4) | ((x3 & 0x03) << 6);
2432
+ b->qh[i*4 + 1] = ((x0 & 0x0C) >> 2) | (x1 & 0x0C) | ((x2 & 0x0C) << 2) | ((x3 & 0x0C) << 4);
2433
+ b->qh[i*4 + 2] = ((x0 & 0x30) >> 4) | ((x1 & 0x30) >> 2) | (x2 & 0x30) | ((x3 & 0x30) << 2);
2434
+ b->qh[i*4 + 3] = ((x0 & 0xC0) >> 6) | ((x1 & 0xC0) >> 4) | ((x2 & 0xC0) >> 2) | (x3 & 0xC0);
2435
+
2436
+ uchar x4 = qh[i + 0 + 32] & mask_lsb_8;
2437
+ uchar x5 = qh[i + 8 + 32] & mask_lsb_8;
2438
+ uchar x6 = qh[i + 16 + 32] & mask_lsb_8;
2439
+ uchar x7 = qh[i + 24 + 32] & mask_lsb_8;
2440
+ b->qh[i*4 + 0 + 32] = (x4 & 0x03) | ((x5 & 0x03) << 2) | ((x6 & 0x03) << 4) | ((x7 & 0x03) << 6);
2441
+ b->qh[i*4 + 1 + 32] = ((x4 & 0x0C) >> 2) | (x5 & 0x0C) | ((x6 & 0x0C) << 2) | ((x7 & 0x0C) << 4);
2442
+ b->qh[i*4 + 2 + 32] = ((x4 & 0x30) >> 4) | ((x5 & 0x30) >> 2) | (x6 & 0x30) | ((x7 & 0x30) << 2);
2443
+ b->qh[i*4 + 3 + 32] = ((x4 & 0xC0) >> 6) | ((x5 & 0xC0) >> 4) | ((x6 & 0xC0) >> 2) | (x7 & 0xC0);
2444
+ }
2445
+
2446
+ for (int i = 0; i < QK_K/16; ++i) {
2447
+ b->scales[i] = s[i];
2448
+ }
2449
+ }
2450
+
2451
+ //------------------------------------------------------------------------------
2452
+ // kernel_convert_block_iq4_nl
2453
+ // Convert the block_iq4_nl format to 2 separate arrays (AOS -> SOA).
2454
+ //------------------------------------------------------------------------------
2455
+ kernel void kernel_convert_block_iq4_nl(
2456
+ global struct block_iq4_nl * src0,
2457
+ global uchar * dst_q,
2458
+ global half * dst_d,
2459
+ uchar mask_0F,
2460
+ uchar mask_F0,
2461
+ ulong n_blk
2462
+ ) {
2463
+ if (get_global_id(0) >= n_blk) {
2464
+ return;
2465
+ }
2466
+ global struct block_iq4_nl * b = (global struct block_iq4_nl *) src0 + get_global_id(0);
2467
+ global uchar * q = (global uchar *) dst_q + QK4_NL/2*get_global_id(0);
2468
+ global half * d = (global half *) dst_d + get_global_id(0);
2469
+
2470
+ *d = b->d;
2471
+
2472
+ for (int i = 0; i < QK4_NL/2; ++i) {
2473
+ q[i] = b->qs[i];
2474
+ }
2475
+ }
2476
+
2477
+ kernel void kernel_restore_block_iq4_nl(
2478
+ global uchar * src_q,
2479
+ global half * src_d,
2480
+ global struct block_iq4_nl * dst,
2481
+ ulong n_blk
2482
+ ) {
2483
+ if (get_global_id(0) >= n_blk) {
2484
+ return;
2485
+ }
2486
+ global struct block_iq4_nl * b = (global struct block_iq4_nl *) dst + get_global_id(0);
2487
+ global uchar * q = (global uchar *) src_q + QK4_NL/2*get_global_id(0);
2488
+ global half * d = (global half *) src_d + get_global_id(0);
2489
+
2490
+ b->d = *d;
2491
+
2492
+ for (int i = 0; i < QK4_NL/2; ++i) {
2493
+ b->qs[i] = q[i];
2494
+ }
2495
+ }
2496
+
2497
+ kernel void kernel_convert_block_iq4_nl_noshuffle(
2498
+ global struct block_iq4_nl * src0,
2499
+ global uchar * dst_q,
2500
+ global half * dst_d,
2501
+ uchar mask_0F,
2502
+ uchar mask_F0,
2503
+ ulong n_blk
2504
+ ) {
2505
+ if (get_global_id(0) >= n_blk) {
2506
+ return;
2507
+ }
2508
+ global struct block_iq4_nl * b = (global struct block_iq4_nl *) src0 + get_global_id(0);
2509
+ global uchar * q = (global uchar *) dst_q + QK4_NL/2*get_global_id(0);
2510
+ global half * d = (global half *) dst_d + get_global_id(0);
2511
+
2512
+ *d = b->d;
2513
+ for (int i = 0; i < QK4_NL/4; ++i) {
2514
+ uchar x0 = b->qs[2*i + 0];
2515
+ uchar x1 = b->qs[2*i + 1];
2516
+
2517
+ q[i + 0 ] = convert_uchar(x0 & mask_0F) | convert_uchar((x1 & mask_0F) << 4);
2518
+ q[i + QK4_NL/4] = convert_uchar((x0 & mask_F0) >> 4) | convert_uchar(x1 & mask_F0);
2519
+ }
2520
+ }
2521
+
2522
+ kernel void kernel_restore_block_iq4_nl_noshuffle(
2523
+ global uchar * src_q,
2524
+ global half * src_d,
2525
+ global struct block_iq4_nl * dst,
2526
+ uchar mask_0F,
2527
+ uchar mask_F0,
2528
+ ulong n_blk
2529
+ ) {
2530
+ if (get_global_id(0) >= n_blk) {
2531
+ return;
2532
+ }
2533
+ global struct block_iq4_nl * b = (global struct block_iq4_nl *) dst + get_global_id(0);
2534
+ global uchar * q = (global uchar *) src_q + QK4_NL/2*get_global_id(0);
2535
+ global half * d = (global half *) src_d + get_global_id(0);
2536
+
2537
+ b->d = *d;
2538
+ for (int i = 0; i < QK4_NL/4; ++i) {
2539
+ uchar x0 = q[i + 0 ];
2540
+ uchar x1 = q[i + QK4_NL/4];
2541
+
2542
+ b->qs[2*i + 0] = convert_uchar((x0 & mask_0F) | ((x1 & mask_0F) << 4));
2543
+ b->qs[2*i + 1] = convert_uchar(((x0 & mask_F0) >> 4) | (x1 & mask_F0));
2544
+ }
2545
+ }
2546
+
2547
+ // ---------------------------------------------------------------------------
2548
+ // kernel_moe_expand_scale_q8_0
2549
+ //
2550
+ // Expand the q8_0 per-32-block scale d (one half/block, [expert][row][block]) into
2551
+ // the UNIFORM scale[16] format the generic dp4a MoE GEMM (kernel_gemm_moe_q8_1_dp4a,
2552
+ // MOE_QT=80) consumes: 16 f16 per 256-superblock (per-16-element segment), where the
2553
+ // two segments of each 32-block share the block's d. q8_0 is symmetric -> no min
2554
+ // buffer (the GEMM runs with has_min=0). The int8 weight codes are reused verbatim
2555
+ // from the existing flat q8_0 weight buffer (extra0_q8_0->q), so only the scale is
2556
+ // rebuilt here. One work-item per (row, superblock, expert).
2557
+ // ---------------------------------------------------------------------------
2558
+ kernel void kernel_moe_expand_scale_q8_0(
2559
+ global const half * src_d, // [expert][row][block], one scale per 32-block
2560
+ global half * dst_scale, // [expert][row][block][2] (FLAT per-32-block)
2561
+ int ne00,
2562
+ int ne01
2563
+ ) {
2564
+ int row = get_global_id(0);
2565
+ int blk = get_global_id(1); // 32-block index along K
2566
+ int e = get_global_id(2);
2567
+ if (row >= ne01) { return; }
2568
+
2569
+ long nb = ne00 / 32; // 32-blocks per row (K only needs % 32 == 0)
2570
+ half d = src_d[((long)e*ne01 + row)*nb + blk];
2571
+ long b = (((long)e*ne01 + row)*nb + blk) * 2;
2572
+ dst_scale[b + 0] = d;
2573
+ dst_scale[b + 1] = d;
2574
+ }
2575
+
2576
+ // ---------------------------------------------------------------------------
2577
+ // kernel_moe_expand_scale_q5_0
2578
+ //
2579
+ // q5_0 = symmetric, value = d*(code-16), code = nibble | (hi<<4) in 0..31. The
2580
+ // generic dp4a MoE GEMM keeps the unsigned code and centers via the min term:
2581
+ // scale*dp4a(code,a) - min*sum(a), scale = d, min = d*16.
2582
+ // Reads the existing q5_0 d ([expert][block][row], one half/32-block, from the
2583
+ // trans4 convert) and writes the FLAT per-32-block uniform scale[2]/min[1] in
2584
+ // [expert][row][block] order (a transpose). One work-item per (row, block, expert).
2585
+ // ---------------------------------------------------------------------------
2586
+ kernel void kernel_moe_expand_scale_q5_0(
2587
+ global const half * src_d, // [expert][block][row]
2588
+ global half * dst_scale, // [expert][row][block][2]
2589
+ global half * dst_min, // [expert][row][block]
2590
+ int ne00,
2591
+ int ne01
2592
+ ) {
2593
+ int row = get_global_id(0);
2594
+ int blk = get_global_id(1);
2595
+ int e = get_global_id(2);
2596
+ if (row >= ne01) { return; }
2597
+
2598
+ long nb = ne00 / 32;
2599
+ half d = src_d[(long)e*nb*ne01 + (long)blk*ne01 + row]; // [expert][block][row]
2600
+ long sb = (((long)e*ne01 + row)*nb + blk) * 2;
2601
+ long mb = ((long)e*ne01 + row)*nb + blk;
2602
+ dst_scale[sb + 0] = d;
2603
+ dst_scale[sb + 1] = d;
2604
+ dst_min[mb] = (half)((float)d * 16.0f);
2605
+ }
2606
+
2607
+ // ---------------------------------------------------------------------------
2608
+ // kernel_moe_expand_scale_q5_K
2609
+ //
2610
+ // q5_K value = d*sv*code + (-dm*mn), with the 6-bit packed per-sub-block scale sv
2611
+ // and min mn (8 sub-blocks of 32 per 256-superblock, decoded by get_scale_min_k4
2612
+ // from the 12-byte s[]). The generic dp4a MoE GEMM (kernel_gemm_moe_q8_1_dp4a,
2613
+ // MOE_QT=5) keeps the unsigned 5-bit code and applies scale/min via the uniform
2614
+ // per-32-block buffers:
2615
+ // acc += sc0*a_d*raw1 + sc1*a_d*raw2 - mn_u*a_s,
2616
+ // sc0 = sc1 = d*sv (both per-16 segments of a 32-block share the sub-block scale),
2617
+ // mn_u = dm*mn (positive; the GEMM subtracts it -> the -dm*mn min term).
2618
+ // q5_K's q_img (low nibbles) + qh (hi-bit plane) are already in the layout the GEMM
2619
+ // reads (same trans4_ns convert that feeds gemm_moe_q5_k_f32_ns), so only the scale
2620
+ // is rebuilt here.
2621
+ //
2622
+ // One work-item per (row, superblock, expert); each emits 8 sub-blocks.
2623
+ // ---------------------------------------------------------------------------
2624
+ kernel void kernel_moe_expand_scale_q5_K(
2625
+ global const uchar * src_s, // [expert][row][superblock][12]
2626
+ global const half * src_d, // [expert][superblock][row]
2627
+ global const half * src_dm, // [expert][superblock][row]
2628
+ global half * dst_scale, // [expert][row][32block][2]
2629
+ global half * dst_min, // [expert][row][32block]
2630
+ int ne00,
2631
+ int ne01
2632
+ ) {
2633
+ int row = get_global_id(0);
2634
+ int sb = get_global_id(1); // superblock index along K
2635
+ int e = get_global_id(2);
2636
+ if (row >= ne01) { return; }
2637
+
2638
+ long nsb = ne00 / 256; // superblocks per row
2639
+ long nblk32 = ne00 / 32; // 32-blocks per row
2640
+
2641
+ float d = (float)src_d [((long)e*nsb + sb)*ne01 + row];
2642
+ float dm = (float)src_dm[((long)e*nsb + sb)*ne01 + row];
2643
+
2644
+ __global const uchar * sc = src_s + ((long)e*ne01 + row)*nsb*12 + (long)sb*12;
2645
+
2646
+ for (int j = 0; j < 8; ++j) {
2647
+ uchar sv, mn;
2648
+ // get_scale_min_k4 (6-bit packed scale/min for sub-block j of 8)
2649
+ if (j < 4) {
2650
+ sv = sc[j] & 63;
2651
+ mn = sc[j+4] & 63;
2652
+ } else {
2653
+ sv = (sc[j+4] & 0x0F) | ((sc[j-4] & 0xC0) >> 2);
2654
+ mn = ((sc[j+4] >> 4) & 0x0F) | ((sc[j] & 0xC0) >> 2);
2655
+ }
2656
+ long sub = (long)sb*8 + j;
2657
+ long sbase = (((long)e*ne01 + row)*nblk32 + sub) * 2;
2658
+ half s_val = (half)(d * (float)sv);
2659
+ dst_scale[sbase + 0] = s_val;
2660
+ dst_scale[sbase + 1] = s_val;
2661
+ dst_min[((long)e*ne01 + row)*nblk32 + sub] = (half)(dm * (float)mn);
2662
+ }
2663
+ }