@novastera-oss/llamarn 0.4.1 → 0.4.3-beta4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (976) hide show
  1. package/RNLlamaCpp.podspec +3 -0
  2. package/android/CMakeLists.txt +2 -0
  3. package/android/src/main/cpp/include/llama.h +44 -21
  4. package/android/src/main/jniLibs/arm64-v8a/libggml-base.so +0 -0
  5. package/android/src/main/jniLibs/arm64-v8a/libggml-cpu.so +0 -0
  6. package/android/src/main/jniLibs/arm64-v8a/libggml.so +0 -0
  7. package/android/src/main/jniLibs/arm64-v8a/libllama.so +0 -0
  8. package/android/src/main/jniLibs/armeabi-v7a/libggml-base.so +0 -0
  9. package/android/src/main/jniLibs/armeabi-v7a/libggml-cpu.so +0 -0
  10. package/android/src/main/jniLibs/armeabi-v7a/libggml.so +0 -0
  11. package/android/src/main/jniLibs/armeabi-v7a/libllama.so +0 -0
  12. package/android/src/main/jniLibs/x86/libggml-base.so +0 -0
  13. package/android/src/main/jniLibs/x86/libggml-cpu.so +0 -0
  14. package/android/src/main/jniLibs/x86/libggml.so +0 -0
  15. package/android/src/main/jniLibs/x86/libllama.so +0 -0
  16. package/android/src/main/jniLibs/x86_64/libggml-base.so +0 -0
  17. package/android/src/main/jniLibs/x86_64/libggml-cpu.so +0 -0
  18. package/android/src/main/jniLibs/x86_64/libggml.so +0 -0
  19. package/android/src/main/jniLibs/x86_64/libllama.so +0 -0
  20. package/cpp/build-info.cpp +2 -2
  21. package/cpp/llama.cpp/CMakeLists.txt +12 -0
  22. package/cpp/llama.cpp/CODEOWNERS +116 -10
  23. package/cpp/llama.cpp/CONTRIBUTING.md +30 -3
  24. package/cpp/llama.cpp/README.md +13 -5
  25. package/cpp/llama.cpp/build-xcframework.sh +5 -0
  26. package/cpp/llama.cpp/cmake/riscv64-spacemit-linux-gnu-gcc.cmake +29 -0
  27. package/cpp/llama.cpp/common/CMakeLists.txt +12 -2
  28. package/cpp/llama.cpp/common/arg.cpp +303 -795
  29. package/cpp/llama.cpp/common/arg.h +2 -3
  30. package/cpp/llama.cpp/common/chat-parser-xml-toolcall.cpp +861 -0
  31. package/cpp/llama.cpp/common/chat-parser-xml-toolcall.h +45 -0
  32. package/cpp/llama.cpp/common/chat-parser.cpp +156 -15
  33. package/cpp/llama.cpp/common/chat-parser.h +13 -0
  34. package/cpp/llama.cpp/common/chat.cpp +1147 -88
  35. package/cpp/llama.cpp/common/chat.h +16 -3
  36. package/cpp/llama.cpp/common/common.cpp +70 -15
  37. package/cpp/llama.cpp/common/common.h +57 -19
  38. package/cpp/llama.cpp/common/download.cpp +1072 -0
  39. package/cpp/llama.cpp/common/download.h +55 -0
  40. package/cpp/llama.cpp/common/http.h +73 -0
  41. package/cpp/llama.cpp/common/json-partial.cpp +70 -2
  42. package/cpp/llama.cpp/common/json-schema-to-grammar.cpp +61 -22
  43. package/cpp/llama.cpp/common/json-schema-to-grammar.h +2 -0
  44. package/cpp/llama.cpp/common/log.cpp +59 -2
  45. package/cpp/llama.cpp/common/log.h +12 -4
  46. package/cpp/llama.cpp/common/sampling.cpp +84 -8
  47. package/cpp/llama.cpp/common/sampling.h +3 -1
  48. package/cpp/llama.cpp/common/speculative.cpp +1 -1
  49. package/cpp/llama.cpp/convert_hf_to_gguf.py +1608 -233
  50. package/cpp/llama.cpp/convert_hf_to_gguf_update.py +6 -1
  51. package/cpp/llama.cpp/convert_lora_to_gguf.py +37 -5
  52. package/cpp/llama.cpp/ggml/CMakeLists.txt +47 -28
  53. package/cpp/llama.cpp/ggml/include/ggml-backend.h +19 -1
  54. package/cpp/llama.cpp/ggml/include/ggml-cpu.h +1 -1
  55. package/cpp/llama.cpp/ggml/include/ggml-hexagon.h +19 -0
  56. package/cpp/llama.cpp/ggml/include/ggml-metal.h +1 -6
  57. package/cpp/llama.cpp/ggml/include/ggml-rpc.h +7 -9
  58. package/cpp/llama.cpp/ggml/include/ggml-zdnn.h +2 -1
  59. package/cpp/llama.cpp/ggml/include/ggml.h +199 -6
  60. package/cpp/llama.cpp/ggml/src/CMakeLists.txt +38 -0
  61. package/cpp/llama.cpp/ggml/src/ggml-alloc.c +299 -130
  62. package/cpp/llama.cpp/ggml/src/ggml-backend-impl.h +4 -4
  63. package/cpp/llama.cpp/ggml/src/ggml-backend-reg.cpp +21 -5
  64. package/cpp/llama.cpp/ggml/src/ggml-backend.cpp +99 -2
  65. package/cpp/llama.cpp/ggml/src/ggml-blas/CMakeLists.txt +1 -1
  66. package/cpp/llama.cpp/ggml/src/ggml-blas/ggml-blas.cpp +1 -0
  67. package/cpp/llama.cpp/ggml/src/ggml-cann/acl_tensor.cpp +57 -45
  68. package/cpp/llama.cpp/ggml/src/ggml-cann/acl_tensor.h +138 -47
  69. package/cpp/llama.cpp/ggml/src/ggml-cann/aclnn_ops.cpp +1584 -1773
  70. package/cpp/llama.cpp/ggml/src/ggml-cann/aclnn_ops.h +201 -317
  71. package/cpp/llama.cpp/ggml/src/ggml-cann/common.h +146 -187
  72. package/cpp/llama.cpp/ggml/src/ggml-cann/ggml-cann.cpp +771 -713
  73. package/cpp/llama.cpp/ggml/src/ggml-cpu/CMakeLists.txt +135 -77
  74. package/cpp/llama.cpp/ggml/src/ggml-cpu/amx/amx.cpp +5 -2
  75. package/cpp/llama.cpp/ggml/src/ggml-cpu/arch/arm/quants.c +428 -26
  76. package/cpp/llama.cpp/ggml/src/ggml-cpu/arch/loongarch/quants.c +16 -17
  77. package/cpp/llama.cpp/ggml/src/ggml-cpu/arch/riscv/quants.c +318 -145
  78. package/cpp/llama.cpp/ggml/src/ggml-cpu/arch/s390/cpu-feats.cpp +50 -0
  79. package/cpp/llama.cpp/ggml/src/ggml-cpu/arch/s390/quants.c +155 -60
  80. package/cpp/llama.cpp/ggml/src/ggml-cpu/arch/x86/repack.cpp +8 -8
  81. package/cpp/llama.cpp/ggml/src/ggml-cpu/arch-fallback.h +0 -1
  82. package/cpp/llama.cpp/ggml/src/ggml-cpu/common.h +14 -0
  83. package/cpp/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-impl.h +10 -9
  84. package/cpp/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.c +108 -64
  85. package/cpp/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.cpp +14 -4
  86. package/cpp/llama.cpp/ggml/src/ggml-cpu/kleidiai/kernels.cpp +530 -87
  87. package/cpp/llama.cpp/ggml/src/ggml-cpu/kleidiai/kernels.h +37 -45
  88. package/cpp/llama.cpp/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp +349 -127
  89. package/cpp/llama.cpp/ggml/src/ggml-cpu/ops.cpp +947 -1218
  90. package/cpp/llama.cpp/ggml/src/ggml-cpu/ops.h +5 -4
  91. package/cpp/llama.cpp/ggml/src/ggml-cpu/repack.cpp +143 -29
  92. package/cpp/llama.cpp/ggml/src/ggml-cpu/simd-mappings.h +82 -76
  93. package/cpp/llama.cpp/ggml/src/ggml-cpu/spacemit/ime.cpp +1025 -0
  94. package/cpp/llama.cpp/ggml/src/ggml-cpu/spacemit/ime.h +13 -0
  95. package/cpp/llama.cpp/ggml/src/ggml-cpu/spacemit/ime1_kernels.cpp +3196 -0
  96. package/cpp/llama.cpp/ggml/src/ggml-cpu/spacemit/ime_kernels.h +26 -0
  97. package/cpp/llama.cpp/ggml/src/ggml-cpu/unary-ops.cpp +151 -0
  98. package/cpp/llama.cpp/ggml/src/ggml-cpu/unary-ops.h +7 -0
  99. package/cpp/llama.cpp/ggml/src/ggml-cpu/vec.cpp +233 -28
  100. package/cpp/llama.cpp/ggml/src/ggml-cpu/vec.h +326 -66
  101. package/cpp/llama.cpp/ggml/src/ggml-cuda/CMakeLists.txt +12 -3
  102. package/cpp/llama.cpp/ggml/src/ggml-cuda/argsort.cu +102 -6
  103. package/cpp/llama.cpp/ggml/src/ggml-cuda/binbcast.cu +110 -76
  104. package/cpp/llama.cpp/ggml/src/ggml-cuda/common.cuh +167 -38
  105. package/cpp/llama.cpp/ggml/src/ggml-cuda/conv2d.cu +6 -11
  106. package/cpp/llama.cpp/ggml/src/ggml-cuda/convert.cuh +12 -0
  107. package/cpp/llama.cpp/ggml/src/ggml-cuda/cpy-utils.cuh +1 -1
  108. package/cpp/llama.cpp/ggml/src/ggml-cuda/cpy.cu +245 -151
  109. package/cpp/llama.cpp/ggml/src/ggml-cuda/cpy.cuh +1 -5
  110. package/cpp/llama.cpp/ggml/src/ggml-cuda/fattn-common.cuh +341 -289
  111. package/cpp/llama.cpp/ggml/src/ggml-cuda/fattn-tile.cu +49 -0
  112. package/cpp/llama.cpp/ggml/src/ggml-cuda/fattn-tile.cuh +1233 -0
  113. package/cpp/llama.cpp/ggml/src/ggml-cuda/fattn-vec.cuh +586 -0
  114. package/cpp/llama.cpp/ggml/src/ggml-cuda/fattn-wmma-f16.cu +6 -6
  115. package/cpp/llama.cpp/ggml/src/ggml-cuda/fattn-wmma-f16.cuh +48 -0
  116. package/cpp/llama.cpp/ggml/src/ggml-cuda/fattn.cu +123 -220
  117. package/cpp/llama.cpp/ggml/src/ggml-cuda/getrows.cu +41 -39
  118. package/cpp/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu +715 -45
  119. package/cpp/llama.cpp/ggml/src/ggml-cuda/im2col.cu +150 -0
  120. package/cpp/llama.cpp/ggml/src/ggml-cuda/im2col.cuh +1 -0
  121. package/cpp/llama.cpp/ggml/src/ggml-cuda/mma.cuh +321 -24
  122. package/cpp/llama.cpp/ggml/src/ggml-cuda/mmf.cu +93 -351
  123. package/cpp/llama.cpp/ggml/src/ggml-cuda/mmf.cuh +828 -1
  124. package/cpp/llama.cpp/ggml/src/ggml-cuda/mmid.cu +164 -0
  125. package/cpp/llama.cpp/ggml/src/ggml-cuda/mmid.cuh +5 -0
  126. package/cpp/llama.cpp/ggml/src/ggml-cuda/mmq.cu +3 -166
  127. package/cpp/llama.cpp/ggml/src/ggml-cuda/mmq.cuh +1 -1
  128. package/cpp/llama.cpp/ggml/src/ggml-cuda/mmvf.cu +371 -78
  129. package/cpp/llama.cpp/ggml/src/ggml-cuda/mmvf.cuh +3 -2
  130. package/cpp/llama.cpp/ggml/src/ggml-cuda/mmvq.cu +279 -147
  131. package/cpp/llama.cpp/ggml/src/ggml-cuda/mmvq.cuh +1 -1
  132. package/cpp/llama.cpp/ggml/src/ggml-cuda/norm.cu +97 -85
  133. package/cpp/llama.cpp/ggml/src/ggml-cuda/pad.cu +46 -23
  134. package/cpp/llama.cpp/ggml/src/ggml-cuda/pad_reflect_1d.cu +63 -54
  135. package/cpp/llama.cpp/ggml/src/ggml-cuda/quantize.cu +12 -10
  136. package/cpp/llama.cpp/ggml/src/ggml-cuda/rope.cu +192 -77
  137. package/cpp/llama.cpp/ggml/src/ggml-cuda/rope.cuh +2 -0
  138. package/cpp/llama.cpp/ggml/src/ggml-cuda/scale.cu +10 -9
  139. package/cpp/llama.cpp/ggml/src/ggml-cuda/set-rows.cu +137 -75
  140. package/cpp/llama.cpp/ggml/src/ggml-cuda/set.cu +39 -0
  141. package/cpp/llama.cpp/ggml/src/ggml-cuda/set.cuh +7 -0
  142. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq112-dv112.cu +5 -0
  143. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq128-dv128.cu +5 -0
  144. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq256-dv256.cu +5 -0
  145. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq40-dv40.cu +5 -0
  146. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq576-dv512.cu +5 -0
  147. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq64-dv64.cu +5 -0
  148. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq72-dv72.cu +5 -0
  149. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq80-dv80.cu +5 -0
  150. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq96-dv96.cu +5 -0
  151. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-f16.cu +7 -0
  152. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q4_0.cu +7 -0
  153. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q4_1.cu +7 -0
  154. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q5_0.cu +7 -0
  155. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q5_1.cu +7 -0
  156. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q8_0.cu +7 -0
  157. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-f16.cu +7 -0
  158. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q4_0.cu +7 -0
  159. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q4_1.cu +7 -0
  160. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q5_0.cu +7 -0
  161. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q5_1.cu +7 -0
  162. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q8_0.cu +7 -0
  163. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-f16.cu +7 -0
  164. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q4_0.cu +7 -0
  165. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q4_1.cu +7 -0
  166. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q5_0.cu +7 -0
  167. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q5_1.cu +7 -0
  168. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q8_0.cu +7 -0
  169. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-f16.cu +7 -0
  170. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q4_0.cu +7 -0
  171. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q4_1.cu +7 -0
  172. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q5_0.cu +7 -0
  173. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q5_1.cu +7 -0
  174. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q8_0.cu +7 -0
  175. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-f16.cu +7 -0
  176. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q4_0.cu +7 -0
  177. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q4_1.cu +7 -0
  178. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q5_0.cu +7 -0
  179. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q5_1.cu +7 -0
  180. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q8_0.cu +7 -0
  181. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-f16.cu +7 -0
  182. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q4_0.cu +7 -0
  183. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q4_1.cu +7 -0
  184. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q5_0.cu +7 -0
  185. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q5_1.cu +7 -0
  186. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q8_0.cu +7 -0
  187. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/generate_cu_files.py +40 -19
  188. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_1.cu +5 -0
  189. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_10.cu +5 -0
  190. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_11.cu +5 -0
  191. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_12.cu +5 -0
  192. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_13.cu +5 -0
  193. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_14.cu +5 -0
  194. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_15.cu +5 -0
  195. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_16.cu +5 -0
  196. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_2.cu +5 -0
  197. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_3.cu +5 -0
  198. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_4.cu +5 -0
  199. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_5.cu +5 -0
  200. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_6.cu +5 -0
  201. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_7.cu +5 -0
  202. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_8.cu +5 -0
  203. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_9.cu +5 -0
  204. package/cpp/llama.cpp/ggml/src/ggml-cuda/topk-moe.cu +336 -0
  205. package/cpp/llama.cpp/ggml/src/ggml-cuda/topk-moe.cuh +16 -0
  206. package/cpp/llama.cpp/ggml/src/ggml-cuda/tsembd.cu +3 -3
  207. package/cpp/llama.cpp/ggml/src/ggml-cuda/unary.cu +105 -11
  208. package/cpp/llama.cpp/ggml/src/ggml-cuda/unary.cuh +36 -0
  209. package/cpp/llama.cpp/ggml/src/ggml-cuda/upscale.cu +87 -6
  210. package/cpp/llama.cpp/ggml/src/ggml-cuda/vendors/hip.h +28 -12
  211. package/cpp/llama.cpp/ggml/src/ggml-hexagon/CMakeLists.txt +68 -0
  212. package/cpp/llama.cpp/ggml/src/ggml-hexagon/ggml-hexagon.cpp +3807 -0
  213. package/cpp/llama.cpp/ggml/src/ggml-hexagon/htp/CMakeLists.txt +40 -0
  214. package/cpp/llama.cpp/ggml/src/ggml-hexagon/htp/act-ops.c +442 -0
  215. package/cpp/llama.cpp/ggml/src/ggml-hexagon/htp/binary-ops.c +360 -0
  216. package/cpp/llama.cpp/ggml/src/ggml-hexagon/htp/cmake-toolchain.cmake +157 -0
  217. package/cpp/llama.cpp/ggml/src/ggml-hexagon/htp/htp-ctx.h +40 -0
  218. package/cpp/llama.cpp/ggml/src/ggml-hexagon/htp/htp-dma.c +69 -0
  219. package/cpp/llama.cpp/ggml/src/ggml-hexagon/htp/htp-dma.h +119 -0
  220. package/cpp/llama.cpp/ggml/src/ggml-hexagon/htp/htp-msg.h +156 -0
  221. package/cpp/llama.cpp/ggml/src/ggml-hexagon/htp/htp-ops.h +64 -0
  222. package/cpp/llama.cpp/ggml/src/ggml-hexagon/htp/htp_iface.idl +16 -0
  223. package/cpp/llama.cpp/ggml/src/ggml-hexagon/htp/hvx-exp.c +93 -0
  224. package/cpp/llama.cpp/ggml/src/ggml-hexagon/htp/hvx-inverse.c +60 -0
  225. package/cpp/llama.cpp/ggml/src/ggml-hexagon/htp/hvx-sigmoid.c +49 -0
  226. package/cpp/llama.cpp/ggml/src/ggml-hexagon/htp/hvx-utils.c +960 -0
  227. package/cpp/llama.cpp/ggml/src/ggml-hexagon/htp/hvx-utils.h +1032 -0
  228. package/cpp/llama.cpp/ggml/src/ggml-hexagon/htp/main.c +829 -0
  229. package/cpp/llama.cpp/ggml/src/ggml-hexagon/htp/matmul-ops.c +2223 -0
  230. package/cpp/llama.cpp/ggml/src/ggml-hexagon/htp/ops-utils.h +149 -0
  231. package/cpp/llama.cpp/ggml/src/ggml-hexagon/htp/rope-ops.c +418 -0
  232. package/cpp/llama.cpp/ggml/src/ggml-hexagon/htp/softmax-ops.c +402 -0
  233. package/cpp/llama.cpp/ggml/src/ggml-hexagon/htp/unary-ops.c +255 -0
  234. package/cpp/llama.cpp/ggml/src/ggml-hexagon/htp/worker-pool.c +297 -0
  235. package/cpp/llama.cpp/ggml/src/ggml-hexagon/htp/worker-pool.h +57 -0
  236. package/cpp/llama.cpp/ggml/src/ggml-hexagon/htp-utils.c +448 -0
  237. package/cpp/llama.cpp/ggml/src/ggml-hexagon/htp-utils.h +220 -0
  238. package/cpp/llama.cpp/ggml/src/ggml-hip/CMakeLists.txt +8 -13
  239. package/cpp/llama.cpp/ggml/src/ggml-impl.h +110 -12
  240. package/cpp/llama.cpp/ggml/src/ggml-metal/CMakeLists.txt +6 -5
  241. package/cpp/llama.cpp/ggml/src/ggml-metal/ggml-metal-common.cpp +446 -0
  242. package/cpp/llama.cpp/ggml/src/ggml-metal/ggml-metal-common.h +52 -0
  243. package/cpp/llama.cpp/ggml/src/ggml-metal/ggml-metal-context.h +33 -0
  244. package/cpp/llama.cpp/ggml/src/ggml-metal/ggml-metal-context.m +599 -0
  245. package/cpp/llama.cpp/ggml/src/ggml-metal/ggml-metal-device.cpp +1662 -0
  246. package/cpp/llama.cpp/ggml/src/ggml-metal/ggml-metal-device.h +251 -0
  247. package/cpp/llama.cpp/ggml/src/ggml-metal/ggml-metal-device.m +1527 -0
  248. package/cpp/llama.cpp/ggml/src/ggml-metal/ggml-metal-impl.h +244 -39
  249. package/cpp/llama.cpp/ggml/src/ggml-metal/ggml-metal-ops.cpp +3844 -0
  250. package/cpp/llama.cpp/ggml/src/ggml-metal/ggml-metal-ops.h +90 -0
  251. package/cpp/llama.cpp/ggml/src/ggml-metal/ggml-metal.cpp +723 -0
  252. package/cpp/llama.cpp/ggml/src/ggml-metal/ggml-metal.metal +3453 -1907
  253. package/cpp/llama.cpp/ggml/src/ggml-musa/CMakeLists.txt +3 -1
  254. package/cpp/llama.cpp/ggml/src/ggml-opencl/CMakeLists.txt +10 -0
  255. package/cpp/llama.cpp/ggml/src/ggml-opencl/ggml-opencl.cpp +1331 -109
  256. package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/cvt.cl +126 -0
  257. package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/flash_attn_f16.cl +31 -4
  258. package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/flash_attn_f32.cl +35 -7
  259. package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/flash_attn_f32_f16.cl +31 -4
  260. package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_f32.cl +162 -0
  261. package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32.cl +156 -0
  262. package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/get_rows.cl +36 -12
  263. package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_kq_kqv.cl +273 -0
  264. package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_l4_lm.cl +24 -10
  265. package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/mul_mm_f32_f32_l4_lm.cl +24 -10
  266. package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/mul_mm_q8_0_f32_l4_lm.cl +154 -0
  267. package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/mul_mv_id_mxfp4_f32_flat.cl +176 -0
  268. package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/mul_mv_id_q8_0_f32.cl +140 -0
  269. package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/mul_mv_id_q8_0_f32_flat.cl +222 -0
  270. package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/mul_mv_mxfp4_f32_flat.cl +167 -0
  271. package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/mul_mv_q8_0_f32.cl +125 -0
  272. package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/mul_mv_q8_0_f32_flat.cl +202 -0
  273. package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/pad.cl +29 -20
  274. package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/rms_norm.cl +25 -10
  275. package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/rope.cl +50 -24
  276. package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/set_rows.cl +123 -10
  277. package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/tsembd.cl +2 -2
  278. package/cpp/llama.cpp/ggml/src/ggml-quants.c +1 -0
  279. package/cpp/llama.cpp/ggml/src/ggml-rpc/ggml-rpc.cpp +341 -161
  280. package/cpp/llama.cpp/ggml/src/ggml-sycl/backend.hpp +6 -0
  281. package/cpp/llama.cpp/ggml/src/ggml-sycl/binbcast.cpp +6 -5
  282. package/cpp/llama.cpp/ggml/src/ggml-sycl/common.hpp +74 -15
  283. package/cpp/llama.cpp/ggml/src/ggml-sycl/concat.cpp +50 -30
  284. package/cpp/llama.cpp/ggml/src/ggml-sycl/conv.cpp +10 -4
  285. package/cpp/llama.cpp/ggml/src/ggml-sycl/convert.cpp +166 -99
  286. package/cpp/llama.cpp/ggml/src/ggml-sycl/count-equal.cpp +79 -0
  287. package/cpp/llama.cpp/ggml/src/ggml-sycl/count-equal.hpp +9 -0
  288. package/cpp/llama.cpp/ggml/src/ggml-sycl/cpy.cpp +72 -94
  289. package/cpp/llama.cpp/ggml/src/ggml-sycl/dmmv.cpp +67 -49
  290. package/cpp/llama.cpp/ggml/src/ggml-sycl/dpct/helper.hpp +21 -31
  291. package/cpp/llama.cpp/ggml/src/ggml-sycl/element_wise.cpp +252 -316
  292. package/cpp/llama.cpp/ggml/src/ggml-sycl/element_wise.hpp +6 -2
  293. package/cpp/llama.cpp/ggml/src/ggml-sycl/getrows.cpp +9 -6
  294. package/cpp/llama.cpp/ggml/src/ggml-sycl/ggml-sycl.cpp +359 -142
  295. package/cpp/llama.cpp/ggml/src/ggml-sycl/gla.cpp +2 -2
  296. package/cpp/llama.cpp/ggml/src/ggml-sycl/im2col.cpp +1 -1
  297. package/cpp/llama.cpp/ggml/src/ggml-sycl/mmq.cpp +80 -60
  298. package/cpp/llama.cpp/ggml/src/ggml-sycl/mmvq.cpp +201 -132
  299. package/cpp/llama.cpp/ggml/src/ggml-sycl/norm.cpp +230 -55
  300. package/cpp/llama.cpp/ggml/src/ggml-sycl/norm.hpp +2 -0
  301. package/cpp/llama.cpp/ggml/src/ggml-sycl/pad.cpp +97 -0
  302. package/cpp/llama.cpp/ggml/src/ggml-sycl/pad.hpp +24 -0
  303. package/cpp/llama.cpp/ggml/src/ggml-sycl/pad_reflect_1d.cpp +72 -0
  304. package/cpp/llama.cpp/ggml/src/ggml-sycl/pad_reflect_1d.hpp +8 -0
  305. package/cpp/llama.cpp/ggml/src/ggml-sycl/presets.hpp +2 -0
  306. package/cpp/llama.cpp/ggml/src/ggml-sycl/repeat_back.cpp +76 -0
  307. package/cpp/llama.cpp/ggml/src/ggml-sycl/repeat_back.hpp +8 -0
  308. package/cpp/llama.cpp/ggml/src/ggml-sycl/roll.cpp +122 -0
  309. package/cpp/llama.cpp/ggml/src/ggml-sycl/roll.hpp +20 -0
  310. package/cpp/llama.cpp/ggml/src/ggml-sycl/rope.cpp +50 -41
  311. package/cpp/llama.cpp/ggml/src/ggml-sycl/set.cpp +73 -0
  312. package/cpp/llama.cpp/ggml/src/ggml-sycl/set.hpp +5 -0
  313. package/cpp/llama.cpp/ggml/src/ggml-sycl/set_rows.cpp +45 -36
  314. package/cpp/llama.cpp/ggml/src/ggml-sycl/softmax.cpp +330 -165
  315. package/cpp/llama.cpp/ggml/src/ggml-sycl/softmax.hpp +4 -0
  316. package/cpp/llama.cpp/ggml/src/ggml-sycl/ssm_conv.cpp +127 -0
  317. package/cpp/llama.cpp/ggml/src/ggml-sycl/ssm_conv.hpp +5 -0
  318. package/cpp/llama.cpp/ggml/src/ggml-sycl/tsembd.cpp +12 -6
  319. package/cpp/llama.cpp/ggml/src/ggml-sycl/wkv.cpp +16 -12
  320. package/cpp/llama.cpp/ggml/src/ggml-vulkan/CMakeLists.txt +38 -18
  321. package/cpp/llama.cpp/ggml/src/ggml-vulkan/ggml-vulkan.cpp +4184 -2159
  322. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/abs.comp +21 -0
  323. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/acc.comp +2 -2
  324. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/add.comp +2 -2
  325. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/add1.comp +28 -0
  326. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/add_id.comp +1 -1
  327. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/arange.comp +20 -0
  328. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/argmax.comp +2 -2
  329. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/argsort.comp +33 -26
  330. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/argsort_large.comp +114 -0
  331. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/ceil.comp +22 -0
  332. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/clamp.comp +2 -2
  333. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/concat.comp +2 -2
  334. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/contig_copy.comp +2 -2
  335. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/conv2d_dw.comp +1 -1
  336. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/conv2d_mm.comp +53 -30
  337. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/conv_transpose_1d.comp +1 -1
  338. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/copy.comp +2 -2
  339. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/copy_from_quant.comp +3 -3
  340. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/copy_to_quant.comp +13 -6
  341. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/copy_transpose.comp +67 -0
  342. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/cos.comp +2 -2
  343. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/count_equal.comp +2 -2
  344. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/dequant_f32.comp +1 -1
  345. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/{dequant_funcs.comp → dequant_funcs.glsl} +138 -2
  346. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/{dequant_funcs_cm2.comp → dequant_funcs_cm2.glsl} +18 -4
  347. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/{dequant_head.comp → dequant_head.glsl} +1 -1
  348. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_m.comp +1 -1
  349. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_s.comp +1 -1
  350. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_s.comp +2 -2
  351. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xs.comp +1 -1
  352. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xxs.comp +3 -2
  353. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_s.comp +7 -6
  354. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_xxs.comp +5 -3
  355. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_nl.comp +1 -1
  356. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_xs.comp +1 -1
  357. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/dequant_mxfp4.comp +3 -3
  358. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q2_k.comp +3 -3
  359. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q3_k.comp +1 -1
  360. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_0.comp +1 -1
  361. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_1.comp +1 -1
  362. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_k.comp +3 -3
  363. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_0.comp +1 -1
  364. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_1.comp +1 -1
  365. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_k.comp +3 -3
  366. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q6_k.comp +1 -1
  367. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q8_0.comp +1 -1
  368. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/diag_mask_inf.comp +1 -1
  369. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/div.comp +2 -2
  370. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/exp.comp +3 -2
  371. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/fill.comp +19 -0
  372. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp +52 -14
  373. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/{flash_attn_base.comp → flash_attn_base.glsl} +50 -12
  374. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp +61 -12
  375. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp +54 -12
  376. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_split_k_reduce.comp +5 -1
  377. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/floor.comp +22 -0
  378. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/geglu.comp +2 -2
  379. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/geglu_erf.comp +2 -2
  380. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/geglu_quick.comp +2 -2
  381. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/gelu.comp +2 -2
  382. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/gelu_erf.comp +2 -2
  383. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/gelu_quick.comp +2 -2
  384. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/{generic_binary_head.comp → generic_binary_head.glsl} +10 -2
  385. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/get_rows.comp +21 -12
  386. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_quant.comp +28 -18
  387. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/{glu_head.comp → glu_head.glsl} +1 -1
  388. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/group_norm.comp +2 -2
  389. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/hardsigmoid.comp +22 -0
  390. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/hardswish.comp +22 -0
  391. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/im2col.comp +15 -7
  392. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/im2col_3d.comp +125 -0
  393. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/l2_norm.comp +2 -2
  394. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/leaky_relu.comp +2 -2
  395. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/log.comp +18 -0
  396. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul.comp +2 -2
  397. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec.comp +1 -1
  398. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_base.glsl +229 -0
  399. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iface.glsl +33 -0
  400. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_m.comp +1 -1
  401. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_s.comp +1 -1
  402. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_s.comp +1 -1
  403. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xs.comp +1 -1
  404. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xxs.comp +1 -1
  405. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_s.comp +1 -1
  406. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_xxs.comp +1 -1
  407. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_nc.comp +9 -7
  408. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_p021.comp +9 -7
  409. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q2_k.comp +3 -5
  410. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q3_k.comp +1 -1
  411. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q4_k.comp +3 -5
  412. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q5_k.comp +3 -5
  413. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q6_k.comp +1 -1
  414. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq.comp +140 -0
  415. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp +106 -634
  416. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp +118 -9
  417. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl +556 -0
  418. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_id_funcs.glsl +70 -0
  419. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp +77 -214
  420. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_funcs.glsl +589 -0
  421. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_shmem_types.glsl +78 -0
  422. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/multi_add.comp +97 -13
  423. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/neg.comp +20 -0
  424. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/norm.comp +2 -2
  425. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_adamw.comp +2 -2
  426. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_sgd.comp +1 -1
  427. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/pad.comp +25 -4
  428. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/pool2d.comp +1 -1
  429. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/quantize_q8_1.comp +55 -5
  430. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/reglu.comp +2 -2
  431. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/relu.comp +2 -2
  432. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/repeat.comp +2 -2
  433. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/repeat_back.comp +2 -2
  434. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp +45 -3
  435. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_back.comp +2 -2
  436. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_partials.comp +2 -2
  437. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/roll.comp +2 -2
  438. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/rope_funcs.glsl +227 -0
  439. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/rope_head.glsl +20 -0
  440. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/rope_multi.comp +5 -52
  441. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/rope_neox.comp +5 -35
  442. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/rope_norm.comp +5 -35
  443. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/rope_params.glsl +27 -0
  444. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/rope_vision.comp +5 -41
  445. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/round.comp +29 -0
  446. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/scale.comp +2 -2
  447. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/sigmoid.comp +2 -2
  448. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/silu.comp +2 -2
  449. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/silu_back.comp +2 -2
  450. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/sin.comp +2 -2
  451. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/soft_max.comp +1 -1
  452. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_back.comp +6 -2
  453. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/softplus.comp +23 -0
  454. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/sqrt.comp +2 -2
  455. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/square.comp +2 -2
  456. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/ssm_conv.comp +44 -0
  457. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/ssm_scan.comp +140 -0
  458. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/step.comp +22 -0
  459. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/sub.comp +2 -2
  460. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.comp +1 -1
  461. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/swiglu.comp +2 -2
  462. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/swiglu_oai.comp +2 -2
  463. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/tanh.comp +2 -2
  464. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/timestep_embedding.comp +5 -4
  465. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/topk_moe.comp +171 -0
  466. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/trunc.comp +22 -0
  467. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/{types.comp → types.glsl} +79 -29
  468. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/upscale.comp +36 -12
  469. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +471 -196
  470. package/cpp/llama.cpp/ggml/src/ggml-webgpu/CMakeLists.txt +8 -0
  471. package/cpp/llama.cpp/ggml/src/ggml-webgpu/ggml-webgpu.cpp +1690 -383
  472. package/cpp/llama.cpp/ggml/src/ggml-webgpu/wgsl-shaders/bin_op.tmpl.wgsl +188 -0
  473. package/cpp/llama.cpp/ggml/src/ggml-webgpu/wgsl-shaders/binary_head.tmpl +45 -0
  474. package/cpp/llama.cpp/ggml/src/ggml-webgpu/wgsl-shaders/common_decls.tmpl +930 -0
  475. package/cpp/llama.cpp/ggml/src/ggml-webgpu/wgsl-shaders/cpy.tmpl.wgsl +101 -0
  476. package/cpp/llama.cpp/ggml/src/ggml-webgpu/wgsl-shaders/embed_wgsl.py +57 -10
  477. package/cpp/llama.cpp/ggml/src/ggml-webgpu/wgsl-shaders/get_rows.tmpl.wgsl +874 -0
  478. package/cpp/llama.cpp/ggml/src/ggml-webgpu/wgsl-shaders/glu.tmpl.wgsl +323 -0
  479. package/cpp/llama.cpp/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat.tmpl.wgsl +25 -912
  480. package/cpp/llama.cpp/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_decls.tmpl +97 -0
  481. package/cpp/llama.cpp/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_reg_tile.tmpl.wgsl +247 -0
  482. package/cpp/llama.cpp/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_subgroup_matrix.tmpl.wgsl +302 -0
  483. package/cpp/llama.cpp/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec.tmpl.wgsl +267 -0
  484. package/cpp/llama.cpp/ggml/src/ggml-webgpu/wgsl-shaders/rms_norm.wgsl +123 -0
  485. package/cpp/llama.cpp/ggml/src/ggml-webgpu/wgsl-shaders/rope.tmpl.wgsl +295 -0
  486. package/cpp/llama.cpp/ggml/src/ggml-webgpu/wgsl-shaders/scale.tmpl.wgsl +90 -0
  487. package/cpp/llama.cpp/ggml/src/ggml-webgpu/wgsl-shaders/{set_rows.wgsl → set_rows.tmpl.wgsl} +38 -8
  488. package/cpp/llama.cpp/ggml/src/ggml-webgpu/wgsl-shaders/soft_max.tmpl.wgsl +345 -0
  489. package/cpp/llama.cpp/ggml/src/ggml-zdnn/common.hpp +59 -0
  490. package/cpp/llama.cpp/ggml/src/ggml-zdnn/ggml-zdnn.cpp +96 -314
  491. package/cpp/llama.cpp/ggml/src/ggml-zdnn/mmf.cpp +80 -0
  492. package/cpp/llama.cpp/ggml/src/ggml-zdnn/mmf.hpp +12 -0
  493. package/cpp/llama.cpp/ggml/src/ggml-zdnn/utils.cpp +79 -0
  494. package/cpp/llama.cpp/ggml/src/ggml-zdnn/utils.hpp +19 -0
  495. package/cpp/llama.cpp/ggml/src/ggml.c +440 -17
  496. package/cpp/llama.cpp/ggml/src/gguf.cpp +104 -29
  497. package/cpp/llama.cpp/gguf-py/gguf/constants.py +363 -13
  498. package/cpp/llama.cpp/gguf-py/gguf/gguf_writer.py +64 -0
  499. package/cpp/llama.cpp/gguf-py/gguf/lazy.py +8 -3
  500. package/cpp/llama.cpp/gguf-py/gguf/scripts/gguf_convert_endian.py +6 -0
  501. package/cpp/llama.cpp/gguf-py/gguf/tensor_mapping.py +156 -18
  502. package/cpp/llama.cpp/gguf-py/gguf/utility.py +80 -0
  503. package/cpp/llama.cpp/gguf-py/gguf/vocab.py +4 -4
  504. package/cpp/llama.cpp/include/llama.h +44 -21
  505. package/cpp/llama.cpp/media/llama1-icon-transparent.png +0 -0
  506. package/cpp/llama.cpp/media/llama1-icon-transparent.svg +77 -0
  507. package/cpp/llama.cpp/media/llama1-icon.png +0 -0
  508. package/cpp/llama.cpp/media/llama1-icon.svg +87 -0
  509. package/cpp/llama.cpp/requirements/requirements-all.txt +2 -0
  510. package/cpp/llama.cpp/requirements/requirements-convert_hf_to_gguf.txt +3 -3
  511. package/cpp/llama.cpp/requirements/requirements-convert_legacy_llama.txt +3 -1
  512. package/cpp/llama.cpp/requirements/requirements-tool_bench.txt +1 -1
  513. package/cpp/llama.cpp/src/CMakeLists.txt +101 -0
  514. package/cpp/llama.cpp/src/llama-adapter.cpp +33 -0
  515. package/cpp/llama.cpp/src/llama-adapter.h +3 -0
  516. package/cpp/llama.cpp/src/llama-arch.cpp +344 -14
  517. package/cpp/llama.cpp/src/llama-arch.h +50 -0
  518. package/cpp/llama.cpp/src/llama-batch.cpp +63 -31
  519. package/cpp/llama.cpp/src/llama-batch.h +13 -2
  520. package/cpp/llama.cpp/src/llama-chat.cpp +85 -3
  521. package/cpp/llama.cpp/src/llama-chat.h +4 -0
  522. package/cpp/llama.cpp/src/llama-context.cpp +300 -45
  523. package/cpp/llama.cpp/src/llama-context.h +16 -6
  524. package/cpp/llama.cpp/src/llama-cparams.h +2 -1
  525. package/cpp/llama.cpp/src/llama-grammar.cpp +17 -9
  526. package/cpp/llama.cpp/src/llama-graph.cpp +226 -64
  527. package/cpp/llama.cpp/src/llama-graph.h +27 -5
  528. package/cpp/llama.cpp/src/llama-hparams.cpp +53 -2
  529. package/cpp/llama.cpp/src/llama-hparams.h +48 -8
  530. package/cpp/llama.cpp/src/llama-impl.cpp +3 -3
  531. package/cpp/llama.cpp/src/llama-impl.h +2 -0
  532. package/cpp/llama.cpp/src/llama-kv-cache-iswa.cpp +13 -3
  533. package/cpp/llama.cpp/src/llama-kv-cache-iswa.h +2 -0
  534. package/cpp/llama.cpp/src/llama-kv-cache.cpp +120 -62
  535. package/cpp/llama.cpp/src/llama-kv-cache.h +13 -4
  536. package/cpp/llama.cpp/src/llama-kv-cells.h +44 -2
  537. package/cpp/llama.cpp/src/llama-memory-hybrid.cpp +19 -9
  538. package/cpp/llama.cpp/src/llama-memory-hybrid.h +2 -0
  539. package/cpp/llama.cpp/src/llama-memory-recurrent.cpp +38 -17
  540. package/cpp/llama.cpp/src/llama-memory-recurrent.h +5 -2
  541. package/cpp/llama.cpp/src/llama-memory.h +3 -0
  542. package/cpp/llama.cpp/src/llama-model-loader.cpp +2 -0
  543. package/cpp/llama.cpp/src/llama-model.cpp +1070 -12614
  544. package/cpp/llama.cpp/src/llama-model.h +40 -4
  545. package/cpp/llama.cpp/src/llama-quant.cpp +14 -6
  546. package/cpp/llama.cpp/src/llama-sampling.cpp +243 -136
  547. package/cpp/llama.cpp/src/llama-vocab.cpp +43 -3
  548. package/cpp/llama.cpp/src/llama-vocab.h +43 -39
  549. package/cpp/llama.cpp/src/llama.cpp +69 -10
  550. package/cpp/llama.cpp/src/models/afmoe.cpp +187 -0
  551. package/cpp/llama.cpp/src/models/apertus.cpp +125 -0
  552. package/cpp/llama.cpp/src/models/arcee.cpp +135 -0
  553. package/cpp/llama.cpp/src/models/arctic.cpp +138 -0
  554. package/cpp/llama.cpp/src/models/arwkv7.cpp +86 -0
  555. package/cpp/llama.cpp/src/models/baichuan.cpp +122 -0
  556. package/cpp/llama.cpp/src/models/bailingmoe.cpp +144 -0
  557. package/cpp/llama.cpp/src/models/bailingmoe2.cpp +135 -0
  558. package/cpp/llama.cpp/src/models/bert.cpp +176 -0
  559. package/cpp/llama.cpp/src/models/bitnet.cpp +160 -0
  560. package/cpp/llama.cpp/src/models/bloom.cpp +101 -0
  561. package/cpp/llama.cpp/src/models/chameleon.cpp +178 -0
  562. package/cpp/llama.cpp/src/models/chatglm.cpp +132 -0
  563. package/cpp/llama.cpp/src/models/codeshell.cpp +111 -0
  564. package/cpp/llama.cpp/src/models/cogvlm.cpp +100 -0
  565. package/cpp/llama.cpp/src/models/cohere2-iswa.cpp +131 -0
  566. package/cpp/llama.cpp/src/models/command-r.cpp +122 -0
  567. package/cpp/llama.cpp/src/models/dbrx.cpp +123 -0
  568. package/cpp/llama.cpp/src/models/deci.cpp +135 -0
  569. package/cpp/llama.cpp/src/models/deepseek.cpp +144 -0
  570. package/cpp/llama.cpp/src/models/deepseek2.cpp +237 -0
  571. package/cpp/llama.cpp/src/models/dots1.cpp +134 -0
  572. package/cpp/llama.cpp/src/models/dream.cpp +105 -0
  573. package/cpp/llama.cpp/src/models/ernie4-5-moe.cpp +150 -0
  574. package/cpp/llama.cpp/src/models/ernie4-5.cpp +110 -0
  575. package/cpp/llama.cpp/src/models/exaone.cpp +114 -0
  576. package/cpp/llama.cpp/src/models/exaone4.cpp +123 -0
  577. package/cpp/llama.cpp/src/models/falcon-h1.cpp +113 -0
  578. package/cpp/llama.cpp/src/models/falcon.cpp +120 -0
  579. package/cpp/llama.cpp/src/models/gemma-embedding.cpp +120 -0
  580. package/cpp/llama.cpp/src/models/gemma.cpp +112 -0
  581. package/cpp/llama.cpp/src/models/gemma2-iswa.cpp +125 -0
  582. package/cpp/llama.cpp/src/models/gemma3-iswa.cpp +131 -0
  583. package/cpp/llama.cpp/src/models/gemma3n-iswa.cpp +377 -0
  584. package/cpp/llama.cpp/src/models/glm4-moe.cpp +153 -0
  585. package/cpp/llama.cpp/src/models/glm4.cpp +127 -0
  586. package/cpp/llama.cpp/src/models/gpt2.cpp +105 -0
  587. package/cpp/llama.cpp/src/models/gptneox.cpp +144 -0
  588. package/cpp/llama.cpp/src/models/granite-hybrid.cpp +196 -0
  589. package/cpp/llama.cpp/src/models/granite.cpp +211 -0
  590. package/cpp/llama.cpp/src/models/graph-context-mamba.cpp +283 -0
  591. package/cpp/llama.cpp/src/models/grok.cpp +159 -0
  592. package/cpp/llama.cpp/src/models/grovemoe.cpp +141 -0
  593. package/cpp/llama.cpp/src/models/hunyuan-dense.cpp +132 -0
  594. package/cpp/llama.cpp/src/models/hunyuan-moe.cpp +154 -0
  595. package/cpp/llama.cpp/src/models/internlm2.cpp +120 -0
  596. package/cpp/llama.cpp/src/models/jais.cpp +86 -0
  597. package/cpp/llama.cpp/src/models/jamba.cpp +106 -0
  598. package/cpp/llama.cpp/src/models/lfm2.cpp +173 -0
  599. package/cpp/llama.cpp/src/models/llada-moe.cpp +122 -0
  600. package/cpp/llama.cpp/src/models/llada.cpp +99 -0
  601. package/cpp/llama.cpp/src/models/llama-iswa.cpp +174 -0
  602. package/cpp/llama.cpp/src/models/llama.cpp +155 -0
  603. package/cpp/llama.cpp/src/models/mamba.cpp +55 -0
  604. package/cpp/llama.cpp/src/models/minicpm3.cpp +199 -0
  605. package/cpp/llama.cpp/src/models/minimax-m2.cpp +124 -0
  606. package/cpp/llama.cpp/src/models/models.h +485 -0
  607. package/cpp/llama.cpp/src/models/mpt.cpp +126 -0
  608. package/cpp/llama.cpp/src/models/nemotron-h.cpp +121 -0
  609. package/cpp/llama.cpp/src/models/nemotron.cpp +122 -0
  610. package/cpp/llama.cpp/src/models/neo-bert.cpp +104 -0
  611. package/cpp/llama.cpp/src/models/olmo.cpp +121 -0
  612. package/cpp/llama.cpp/src/models/olmo2.cpp +150 -0
  613. package/cpp/llama.cpp/src/models/olmoe.cpp +124 -0
  614. package/cpp/llama.cpp/src/models/openai-moe-iswa.cpp +124 -0
  615. package/cpp/llama.cpp/src/models/openelm.cpp +124 -0
  616. package/cpp/llama.cpp/src/models/orion.cpp +123 -0
  617. package/cpp/llama.cpp/src/models/pangu-embedded.cpp +121 -0
  618. package/cpp/llama.cpp/src/models/phi2.cpp +121 -0
  619. package/cpp/llama.cpp/src/models/phi3.cpp +152 -0
  620. package/cpp/llama.cpp/src/models/plamo.cpp +110 -0
  621. package/cpp/llama.cpp/src/models/plamo2.cpp +316 -0
  622. package/cpp/llama.cpp/src/models/plm.cpp +168 -0
  623. package/cpp/llama.cpp/src/models/qwen.cpp +108 -0
  624. package/cpp/llama.cpp/src/models/qwen2.cpp +117 -0
  625. package/cpp/llama.cpp/src/models/qwen2moe.cpp +151 -0
  626. package/cpp/llama.cpp/src/models/qwen2vl.cpp +117 -0
  627. package/cpp/llama.cpp/src/models/qwen3.cpp +117 -0
  628. package/cpp/llama.cpp/src/models/qwen3moe.cpp +124 -0
  629. package/cpp/llama.cpp/src/models/qwen3vl-moe.cpp +149 -0
  630. package/cpp/llama.cpp/src/models/qwen3vl.cpp +141 -0
  631. package/cpp/llama.cpp/src/models/refact.cpp +94 -0
  632. package/cpp/llama.cpp/src/models/rwkv6-base.cpp +162 -0
  633. package/cpp/llama.cpp/src/models/rwkv6.cpp +94 -0
  634. package/cpp/llama.cpp/src/models/rwkv6qwen2.cpp +86 -0
  635. package/cpp/llama.cpp/src/models/rwkv7-base.cpp +135 -0
  636. package/cpp/llama.cpp/src/models/rwkv7.cpp +90 -0
  637. package/cpp/llama.cpp/src/models/seed-oss.cpp +124 -0
  638. package/cpp/llama.cpp/src/models/smallthinker.cpp +120 -0
  639. package/cpp/llama.cpp/src/models/smollm3.cpp +128 -0
  640. package/cpp/llama.cpp/src/models/stablelm.cpp +146 -0
  641. package/cpp/llama.cpp/src/models/starcoder.cpp +100 -0
  642. package/cpp/llama.cpp/src/models/starcoder2.cpp +121 -0
  643. package/cpp/llama.cpp/src/models/t5-dec.cpp +166 -0
  644. package/cpp/llama.cpp/src/models/t5-enc.cpp +96 -0
  645. package/cpp/llama.cpp/src/models/wavtokenizer-dec.cpp +149 -0
  646. package/cpp/llama.cpp/src/models/xverse.cpp +108 -0
  647. package/cpp/llama.cpp/src/unicode.cpp +77 -0
  648. package/cpp/llama.cpp/src/unicode.h +43 -0
  649. package/cpp/llama.cpp/vendor/cpp-httplib/CMakeLists.txt +94 -0
  650. package/cpp/llama.cpp/vendor/cpp-httplib/httplib.cpp +9339 -0
  651. package/cpp/llama.cpp/vendor/cpp-httplib/httplib.h +433 -8222
  652. package/cpp/llama.cpp/vendor/cpp-httplib/patch-boringssl.cmake +6 -0
  653. package/cpp/llama.cpp/vendor/miniaudio/miniaudio.h +4179 -1900
  654. package/cpp/llama.cpp/vendor/minja/chat-template.hpp +9 -2
  655. package/cpp/llama.cpp/vendor/minja/minja.hpp +101 -22
  656. package/ios/include/chat.h +16 -3
  657. package/ios/include/common/minja/chat-template.hpp +9 -2
  658. package/ios/include/common/minja/minja.hpp +101 -22
  659. package/ios/include/common.h +57 -19
  660. package/ios/include/json-schema-to-grammar.h +2 -0
  661. package/ios/include/llama.h +44 -21
  662. package/ios/include/log.h +12 -4
  663. package/ios/include/sampling.h +3 -1
  664. package/ios/libs/llama.xcframework/Info.plist +20 -20
  665. package/ios/libs/llama.xcframework/ios-arm64/dSYMs/llama.dSYM/Contents/Resources/DWARF/llama +0 -0
  666. package/ios/libs/llama.xcframework/ios-arm64/dSYMs/llama.dSYM/Contents/Resources/Relocations/aarch64/llama.yml +6399 -5557
  667. package/ios/libs/llama.xcframework/ios-arm64/llama.framework/Headers/ggml-backend.h +19 -1
  668. package/ios/libs/llama.xcframework/ios-arm64/llama.framework/Headers/ggml-cpu.h +1 -1
  669. package/ios/libs/llama.xcframework/ios-arm64/llama.framework/Headers/ggml-metal.h +1 -6
  670. package/ios/libs/llama.xcframework/ios-arm64/llama.framework/Headers/ggml.h +199 -6
  671. package/ios/libs/llama.xcframework/ios-arm64/llama.framework/Headers/llama.h +44 -21
  672. package/ios/libs/llama.xcframework/ios-arm64/llama.framework/llama +0 -0
  673. package/ios/libs/llama.xcframework/ios-arm64_x86_64-simulator/dSYMs/llama.dSYM/Contents/Resources/DWARF/llama +0 -0
  674. package/ios/libs/llama.xcframework/ios-arm64_x86_64-simulator/dSYMs/llama.dSYM/Contents/Resources/Relocations/aarch64/llama.yml +6362 -5520
  675. package/ios/libs/llama.xcframework/ios-arm64_x86_64-simulator/dSYMs/llama.dSYM/Contents/Resources/Relocations/x86_64/llama.yml +4813 -4241
  676. package/ios/libs/llama.xcframework/ios-arm64_x86_64-simulator/llama.framework/Headers/ggml-backend.h +19 -1
  677. package/ios/libs/llama.xcframework/ios-arm64_x86_64-simulator/llama.framework/Headers/ggml-cpu.h +1 -1
  678. package/ios/libs/llama.xcframework/ios-arm64_x86_64-simulator/llama.framework/Headers/ggml-metal.h +1 -6
  679. package/ios/libs/llama.xcframework/ios-arm64_x86_64-simulator/llama.framework/Headers/ggml.h +199 -6
  680. package/ios/libs/llama.xcframework/ios-arm64_x86_64-simulator/llama.framework/Headers/llama.h +44 -21
  681. package/ios/libs/llama.xcframework/ios-arm64_x86_64-simulator/llama.framework/llama +0 -0
  682. package/package.json +10 -4
  683. package/cpp/llama.cpp/ggml/src/ggml-cann/Doxyfile +0 -2579
  684. package/cpp/llama.cpp/ggml/src/ggml-cuda/fattn-tile-f16.cu +0 -371
  685. package/cpp/llama.cpp/ggml/src/ggml-cuda/fattn-tile-f16.cuh +0 -3
  686. package/cpp/llama.cpp/ggml/src/ggml-cuda/fattn-tile-f32.cu +0 -379
  687. package/cpp/llama.cpp/ggml/src/ggml-cuda/fattn-tile-f32.cuh +0 -3
  688. package/cpp/llama.cpp/ggml/src/ggml-cuda/fattn-vec-f16.cuh +0 -495
  689. package/cpp/llama.cpp/ggml/src/ggml-cuda/fattn-vec-f32.cuh +0 -486
  690. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-f16.cu +0 -5
  691. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q4_0.cu +0 -5
  692. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q4_1.cu +0 -5
  693. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q5_0.cu +0 -5
  694. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q5_1.cu +0 -5
  695. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q8_0.cu +0 -5
  696. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-f16.cu +0 -5
  697. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q4_0.cu +0 -5
  698. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q4_1.cu +0 -5
  699. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q5_0.cu +0 -5
  700. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q5_1.cu +0 -5
  701. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q8_0.cu +0 -5
  702. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-f16.cu +0 -5
  703. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q4_0.cu +0 -5
  704. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q4_1.cu +0 -5
  705. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q5_0.cu +0 -5
  706. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q5_1.cu +0 -5
  707. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q8_0.cu +0 -5
  708. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-f16.cu +0 -5
  709. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q4_0.cu +0 -5
  710. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q4_1.cu +0 -5
  711. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q5_0.cu +0 -5
  712. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q5_1.cu +0 -5
  713. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q8_0.cu +0 -5
  714. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-f16.cu +0 -5
  715. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q4_0.cu +0 -5
  716. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q4_1.cu +0 -5
  717. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q5_0.cu +0 -5
  718. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q5_1.cu +0 -5
  719. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q8_0.cu +0 -5
  720. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-f16.cu +0 -5
  721. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q4_0.cu +0 -5
  722. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q4_1.cu +0 -5
  723. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q5_0.cu +0 -5
  724. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q5_1.cu +0 -5
  725. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q8_0.cu +0 -5
  726. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs256-f16-f16.cu +0 -5
  727. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-f16.cu +0 -5
  728. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q4_0.cu +0 -5
  729. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q4_1.cu +0 -5
  730. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q5_0.cu +0 -5
  731. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q5_1.cu +0 -5
  732. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q8_0.cu +0 -5
  733. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-f16.cu +0 -5
  734. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q4_0.cu +0 -5
  735. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q4_1.cu +0 -5
  736. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q5_0.cu +0 -5
  737. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q5_1.cu +0 -5
  738. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q8_0.cu +0 -5
  739. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-f16.cu +0 -5
  740. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q4_0.cu +0 -5
  741. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q4_1.cu +0 -5
  742. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q5_0.cu +0 -5
  743. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q5_1.cu +0 -5
  744. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q8_0.cu +0 -5
  745. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-f16.cu +0 -5
  746. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q4_0.cu +0 -5
  747. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q4_1.cu +0 -5
  748. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q5_0.cu +0 -5
  749. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q5_1.cu +0 -5
  750. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q8_0.cu +0 -5
  751. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-f16.cu +0 -5
  752. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q4_0.cu +0 -5
  753. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q4_1.cu +0 -5
  754. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q5_0.cu +0 -5
  755. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q5_1.cu +0 -5
  756. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q8_0.cu +0 -5
  757. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-f16.cu +0 -5
  758. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q4_0.cu +0 -5
  759. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q4_1.cu +0 -5
  760. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q5_0.cu +0 -5
  761. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q5_1.cu +0 -5
  762. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q8_0.cu +0 -5
  763. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-f16.cu +0 -5
  764. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q4_0.cu +0 -5
  765. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q4_1.cu +0 -5
  766. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q5_0.cu +0 -5
  767. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q5_1.cu +0 -5
  768. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q8_0.cu +0 -5
  769. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs256-f16-f16.cu +0 -5
  770. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-f16.cu +0 -5
  771. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q4_0.cu +0 -5
  772. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q4_1.cu +0 -5
  773. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q5_0.cu +0 -5
  774. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q5_1.cu +0 -5
  775. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q8_0.cu +0 -5
  776. package/cpp/llama.cpp/ggml/src/ggml-metal/ggml-metal.m +0 -6886
  777. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_base.comp +0 -154
  778. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_funcs.comp +0 -105
  779. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/rope_head.comp +0 -55
  780. package/cpp/llama.cpp/ggml/src/ggml-webgpu/wgsl-shaders/cpy.wgsl +0 -60
  781. package/cpp/llama.cpp/ggml/src/ggml-zdnn/ggml-zdnn-impl.h +0 -97
  782. package/cpp/llama.cpp/models/ggml-vocab-aquila.gguf +0 -0
  783. package/cpp/llama.cpp/models/ggml-vocab-baichuan.gguf +0 -0
  784. package/cpp/llama.cpp/models/ggml-vocab-bert-bge.gguf +0 -0
  785. package/cpp/llama.cpp/models/ggml-vocab-bert-bge.gguf.inp +0 -112
  786. package/cpp/llama.cpp/models/ggml-vocab-bert-bge.gguf.out +0 -46
  787. package/cpp/llama.cpp/models/ggml-vocab-command-r.gguf +0 -0
  788. package/cpp/llama.cpp/models/ggml-vocab-command-r.gguf.inp +0 -112
  789. package/cpp/llama.cpp/models/ggml-vocab-command-r.gguf.out +0 -46
  790. package/cpp/llama.cpp/models/ggml-vocab-deepseek-coder.gguf +0 -0
  791. package/cpp/llama.cpp/models/ggml-vocab-deepseek-coder.gguf.inp +0 -112
  792. package/cpp/llama.cpp/models/ggml-vocab-deepseek-coder.gguf.out +0 -46
  793. package/cpp/llama.cpp/models/ggml-vocab-deepseek-llm.gguf +0 -0
  794. package/cpp/llama.cpp/models/ggml-vocab-deepseek-llm.gguf.inp +0 -112
  795. package/cpp/llama.cpp/models/ggml-vocab-deepseek-llm.gguf.out +0 -46
  796. package/cpp/llama.cpp/models/ggml-vocab-falcon.gguf +0 -0
  797. package/cpp/llama.cpp/models/ggml-vocab-falcon.gguf.inp +0 -112
  798. package/cpp/llama.cpp/models/ggml-vocab-falcon.gguf.out +0 -46
  799. package/cpp/llama.cpp/models/ggml-vocab-gpt-2.gguf +0 -0
  800. package/cpp/llama.cpp/models/ggml-vocab-gpt-2.gguf.inp +0 -112
  801. package/cpp/llama.cpp/models/ggml-vocab-gpt-2.gguf.out +0 -46
  802. package/cpp/llama.cpp/models/ggml-vocab-gpt-neox.gguf +0 -0
  803. package/cpp/llama.cpp/models/ggml-vocab-llama-bpe.gguf +0 -0
  804. package/cpp/llama.cpp/models/ggml-vocab-llama-bpe.gguf.inp +0 -112
  805. package/cpp/llama.cpp/models/ggml-vocab-llama-bpe.gguf.out +0 -46
  806. package/cpp/llama.cpp/models/ggml-vocab-llama-spm.gguf +0 -0
  807. package/cpp/llama.cpp/models/ggml-vocab-llama-spm.gguf.inp +0 -112
  808. package/cpp/llama.cpp/models/ggml-vocab-llama-spm.gguf.out +0 -46
  809. package/cpp/llama.cpp/models/ggml-vocab-mpt.gguf +0 -0
  810. package/cpp/llama.cpp/models/ggml-vocab-mpt.gguf.inp +0 -112
  811. package/cpp/llama.cpp/models/ggml-vocab-mpt.gguf.out +0 -46
  812. package/cpp/llama.cpp/models/ggml-vocab-nomic-bert-moe.gguf +0 -0
  813. package/cpp/llama.cpp/models/ggml-vocab-phi-3.gguf +0 -0
  814. package/cpp/llama.cpp/models/ggml-vocab-phi-3.gguf.inp +0 -112
  815. package/cpp/llama.cpp/models/ggml-vocab-phi-3.gguf.out +0 -46
  816. package/cpp/llama.cpp/models/ggml-vocab-qwen2.gguf +0 -0
  817. package/cpp/llama.cpp/models/ggml-vocab-qwen2.gguf.inp +0 -112
  818. package/cpp/llama.cpp/models/ggml-vocab-qwen2.gguf.out +0 -46
  819. package/cpp/llama.cpp/models/ggml-vocab-refact.gguf +0 -0
  820. package/cpp/llama.cpp/models/ggml-vocab-refact.gguf.inp +0 -112
  821. package/cpp/llama.cpp/models/ggml-vocab-refact.gguf.out +0 -46
  822. package/cpp/llama.cpp/models/ggml-vocab-starcoder.gguf +0 -0
  823. package/cpp/llama.cpp/models/ggml-vocab-starcoder.gguf.inp +0 -112
  824. package/cpp/llama.cpp/models/ggml-vocab-starcoder.gguf.out +0 -46
  825. package/cpp/llama.cpp/models/templates/ByteDance-Seed-OSS.jinja +0 -171
  826. package/cpp/llama.cpp/models/templates/CohereForAI-c4ai-command-r-plus-tool_use.jinja +0 -202
  827. package/cpp/llama.cpp/models/templates/CohereForAI-c4ai-command-r7b-12-2024-tool_use.jinja +0 -156
  828. package/cpp/llama.cpp/models/templates/Mistral-Small-3.2-24B-Instruct-2506.jinja +0 -124
  829. package/cpp/llama.cpp/models/templates/NousResearch-Hermes-2-Pro-Llama-3-8B-tool_use.jinja +0 -152
  830. package/cpp/llama.cpp/models/templates/NousResearch-Hermes-3-Llama-3.1-8B-tool_use.jinja +0 -152
  831. package/cpp/llama.cpp/models/templates/Qwen-QwQ-32B.jinja +0 -62
  832. package/cpp/llama.cpp/models/templates/Qwen-Qwen2.5-7B-Instruct.jinja +0 -54
  833. package/cpp/llama.cpp/models/templates/Qwen-Qwen3-0.6B.jinja +0 -85
  834. package/cpp/llama.cpp/models/templates/README.md +0 -25
  835. package/cpp/llama.cpp/models/templates/deepseek-ai-DeepSeek-R1-Distill-Llama-8B.jinja +0 -1
  836. package/cpp/llama.cpp/models/templates/deepseek-ai-DeepSeek-R1-Distill-Qwen-32B.jinja +0 -1
  837. package/cpp/llama.cpp/models/templates/fireworks-ai-llama-3-firefunction-v2.jinja +0 -57
  838. package/cpp/llama.cpp/models/templates/google-gemma-2-2b-it.jinja +0 -4
  839. package/cpp/llama.cpp/models/templates/ibm-granite-granite-3.3-2B-Instruct.jinja +0 -59
  840. package/cpp/llama.cpp/models/templates/llama-cpp-deepseek-r1.jinja +0 -76
  841. package/cpp/llama.cpp/models/templates/llama-cpp-rwkv-world.jinja +0 -34
  842. package/cpp/llama.cpp/models/templates/meetkai-functionary-medium-v3.1.jinja +0 -58
  843. package/cpp/llama.cpp/models/templates/meetkai-functionary-medium-v3.2.jinja +0 -287
  844. package/cpp/llama.cpp/models/templates/meta-llama-Llama-3.1-8B-Instruct.jinja +0 -109
  845. package/cpp/llama.cpp/models/templates/meta-llama-Llama-3.2-3B-Instruct.jinja +0 -93
  846. package/cpp/llama.cpp/models/templates/meta-llama-Llama-3.3-70B-Instruct.jinja +0 -109
  847. package/cpp/llama.cpp/models/templates/microsoft-Phi-3.5-mini-instruct.jinja +0 -8
  848. package/cpp/llama.cpp/models/templates/mistralai-Mistral-Nemo-Instruct-2407.jinja +0 -87
  849. package/cpp/llama.cpp/models/templates/moonshotai-Kimi-K2.jinja +0 -43
  850. package/cpp/llama.cpp/models/templates/openai-gpt-oss-120b.jinja +0 -331
  851. package/cpp/llama.cpp/models/templates/unsloth-mistral-Devstral-Small-2507.jinja +0 -105
  852. package/cpp/llama.cpp/prompts/LLM-questions.txt +0 -49
  853. package/cpp/llama.cpp/prompts/alpaca.txt +0 -1
  854. package/cpp/llama.cpp/prompts/assistant.txt +0 -31
  855. package/cpp/llama.cpp/prompts/chat-with-baichuan.txt +0 -4
  856. package/cpp/llama.cpp/prompts/chat-with-bob.txt +0 -7
  857. package/cpp/llama.cpp/prompts/chat-with-qwen.txt +0 -1
  858. package/cpp/llama.cpp/prompts/chat-with-vicuna-v0.txt +0 -7
  859. package/cpp/llama.cpp/prompts/chat-with-vicuna-v1.txt +0 -7
  860. package/cpp/llama.cpp/prompts/chat.txt +0 -28
  861. package/cpp/llama.cpp/prompts/dan-modified.txt +0 -1
  862. package/cpp/llama.cpp/prompts/dan.txt +0 -1
  863. package/cpp/llama.cpp/prompts/mnemonics.txt +0 -93
  864. package/cpp/llama.cpp/prompts/parallel-questions.txt +0 -43
  865. package/cpp/llama.cpp/prompts/reason-act.txt +0 -18
  866. package/ios/libs/llama.xcframework/macos-arm64_x86_64/dSYMs/llama.dSYM/Contents/Info.plist +0 -20
  867. package/ios/libs/llama.xcframework/macos-arm64_x86_64/dSYMs/llama.dSYM/Contents/Resources/DWARF/llama +0 -0
  868. package/ios/libs/llama.xcframework/macos-arm64_x86_64/dSYMs/llama.dSYM/Contents/Resources/Relocations/aarch64/llama.yml +0 -5524
  869. package/ios/libs/llama.xcframework/macos-arm64_x86_64/dSYMs/llama.dSYM/Contents/Resources/Relocations/x86_64/llama.yml +0 -4247
  870. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Headers/ggml-alloc.h +0 -76
  871. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Headers/ggml-backend.h +0 -354
  872. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Headers/ggml-blas.h +0 -25
  873. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Headers/ggml-cpu.h +0 -145
  874. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Headers/ggml-metal.h +0 -66
  875. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Headers/ggml-opt.h +0 -256
  876. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Headers/ggml.h +0 -2492
  877. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Headers/gguf.h +0 -202
  878. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Headers/llama.h +0 -1391
  879. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Modules/module.modulemap +0 -17
  880. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Resources/Info.plist +0 -32
  881. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/A/Headers/ggml-alloc.h +0 -76
  882. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/A/Headers/ggml-backend.h +0 -354
  883. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/A/Headers/ggml-blas.h +0 -25
  884. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/A/Headers/ggml-cpu.h +0 -145
  885. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/A/Headers/ggml-metal.h +0 -66
  886. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/A/Headers/ggml-opt.h +0 -256
  887. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/A/Headers/ggml.h +0 -2492
  888. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/A/Headers/gguf.h +0 -202
  889. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/A/Headers/llama.h +0 -1391
  890. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/A/Modules/module.modulemap +0 -17
  891. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/A/Resources/Info.plist +0 -32
  892. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/A/llama +0 -0
  893. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/Current/Headers/ggml-alloc.h +0 -76
  894. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/Current/Headers/ggml-backend.h +0 -354
  895. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/Current/Headers/ggml-blas.h +0 -25
  896. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/Current/Headers/ggml-cpu.h +0 -145
  897. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/Current/Headers/ggml-metal.h +0 -66
  898. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/Current/Headers/ggml-opt.h +0 -256
  899. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/Current/Headers/ggml.h +0 -2492
  900. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/Current/Headers/gguf.h +0 -202
  901. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/Current/Headers/llama.h +0 -1391
  902. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/Current/Modules/module.modulemap +0 -17
  903. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/Current/Resources/Info.plist +0 -32
  904. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/Current/llama +0 -0
  905. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/llama +0 -0
  906. package/ios/libs/llama.xcframework/tvos-arm64/dSYMs/llama.dSYM/Contents/Info.plist +0 -20
  907. package/ios/libs/llama.xcframework/tvos-arm64/dSYMs/llama.dSYM/Contents/Resources/DWARF/llama +0 -0
  908. package/ios/libs/llama.xcframework/tvos-arm64/dSYMs/llama.dSYM/Contents/Resources/Relocations/aarch64/llama.yml +0 -5561
  909. package/ios/libs/llama.xcframework/tvos-arm64/llama.framework/Headers/ggml-alloc.h +0 -76
  910. package/ios/libs/llama.xcframework/tvos-arm64/llama.framework/Headers/ggml-backend.h +0 -354
  911. package/ios/libs/llama.xcframework/tvos-arm64/llama.framework/Headers/ggml-blas.h +0 -25
  912. package/ios/libs/llama.xcframework/tvos-arm64/llama.framework/Headers/ggml-cpu.h +0 -145
  913. package/ios/libs/llama.xcframework/tvos-arm64/llama.framework/Headers/ggml-metal.h +0 -66
  914. package/ios/libs/llama.xcframework/tvos-arm64/llama.framework/Headers/ggml-opt.h +0 -256
  915. package/ios/libs/llama.xcframework/tvos-arm64/llama.framework/Headers/ggml.h +0 -2492
  916. package/ios/libs/llama.xcframework/tvos-arm64/llama.framework/Headers/gguf.h +0 -202
  917. package/ios/libs/llama.xcframework/tvos-arm64/llama.framework/Headers/llama.h +0 -1391
  918. package/ios/libs/llama.xcframework/tvos-arm64/llama.framework/Info.plist +0 -35
  919. package/ios/libs/llama.xcframework/tvos-arm64/llama.framework/Modules/module.modulemap +0 -17
  920. package/ios/libs/llama.xcframework/tvos-arm64/llama.framework/llama +0 -0
  921. package/ios/libs/llama.xcframework/tvos-arm64_x86_64-simulator/dSYMs/llama.dSYM/Contents/Info.plist +0 -20
  922. package/ios/libs/llama.xcframework/tvos-arm64_x86_64-simulator/dSYMs/llama.dSYM/Contents/Resources/DWARF/llama +0 -0
  923. package/ios/libs/llama.xcframework/tvos-arm64_x86_64-simulator/dSYMs/llama.dSYM/Contents/Resources/Relocations/aarch64/llama.yml +0 -5524
  924. package/ios/libs/llama.xcframework/tvos-arm64_x86_64-simulator/dSYMs/llama.dSYM/Contents/Resources/Relocations/x86_64/llama.yml +0 -4246
  925. package/ios/libs/llama.xcframework/tvos-arm64_x86_64-simulator/llama.framework/Headers/ggml-alloc.h +0 -76
  926. package/ios/libs/llama.xcframework/tvos-arm64_x86_64-simulator/llama.framework/Headers/ggml-backend.h +0 -354
  927. package/ios/libs/llama.xcframework/tvos-arm64_x86_64-simulator/llama.framework/Headers/ggml-blas.h +0 -25
  928. package/ios/libs/llama.xcframework/tvos-arm64_x86_64-simulator/llama.framework/Headers/ggml-cpu.h +0 -145
  929. package/ios/libs/llama.xcframework/tvos-arm64_x86_64-simulator/llama.framework/Headers/ggml-metal.h +0 -66
  930. package/ios/libs/llama.xcframework/tvos-arm64_x86_64-simulator/llama.framework/Headers/ggml-opt.h +0 -256
  931. package/ios/libs/llama.xcframework/tvos-arm64_x86_64-simulator/llama.framework/Headers/ggml.h +0 -2492
  932. package/ios/libs/llama.xcframework/tvos-arm64_x86_64-simulator/llama.framework/Headers/gguf.h +0 -202
  933. package/ios/libs/llama.xcframework/tvos-arm64_x86_64-simulator/llama.framework/Headers/llama.h +0 -1391
  934. package/ios/libs/llama.xcframework/tvos-arm64_x86_64-simulator/llama.framework/Info.plist +0 -35
  935. package/ios/libs/llama.xcframework/tvos-arm64_x86_64-simulator/llama.framework/Modules/module.modulemap +0 -17
  936. package/ios/libs/llama.xcframework/tvos-arm64_x86_64-simulator/llama.framework/llama +0 -0
  937. package/ios/libs/llama.xcframework/xros-arm64/dSYMs/llama.dSYM/Contents/Info.plist +0 -20
  938. package/ios/libs/llama.xcframework/xros-arm64/dSYMs/llama.dSYM/Contents/Resources/DWARF/llama +0 -0
  939. package/ios/libs/llama.xcframework/xros-arm64/dSYMs/llama.dSYM/Contents/Resources/Relocations/aarch64/llama.yml +0 -5558
  940. package/ios/libs/llama.xcframework/xros-arm64/llama.framework/Headers/ggml-alloc.h +0 -76
  941. package/ios/libs/llama.xcframework/xros-arm64/llama.framework/Headers/ggml-backend.h +0 -354
  942. package/ios/libs/llama.xcframework/xros-arm64/llama.framework/Headers/ggml-blas.h +0 -25
  943. package/ios/libs/llama.xcframework/xros-arm64/llama.framework/Headers/ggml-cpu.h +0 -145
  944. package/ios/libs/llama.xcframework/xros-arm64/llama.framework/Headers/ggml-metal.h +0 -66
  945. package/ios/libs/llama.xcframework/xros-arm64/llama.framework/Headers/ggml-opt.h +0 -256
  946. package/ios/libs/llama.xcframework/xros-arm64/llama.framework/Headers/ggml.h +0 -2492
  947. package/ios/libs/llama.xcframework/xros-arm64/llama.framework/Headers/gguf.h +0 -202
  948. package/ios/libs/llama.xcframework/xros-arm64/llama.framework/Headers/llama.h +0 -1391
  949. package/ios/libs/llama.xcframework/xros-arm64/llama.framework/Info.plist +0 -32
  950. package/ios/libs/llama.xcframework/xros-arm64/llama.framework/Modules/module.modulemap +0 -17
  951. package/ios/libs/llama.xcframework/xros-arm64/llama.framework/llama +0 -0
  952. package/ios/libs/llama.xcframework/xros-arm64_x86_64-simulator/dSYMs/llama.dSYM/Contents/Info.plist +0 -20
  953. package/ios/libs/llama.xcframework/xros-arm64_x86_64-simulator/dSYMs/llama.dSYM/Contents/Resources/DWARF/llama +0 -0
  954. package/ios/libs/llama.xcframework/xros-arm64_x86_64-simulator/dSYMs/llama.dSYM/Contents/Resources/Relocations/aarch64/llama.yml +0 -5520
  955. package/ios/libs/llama.xcframework/xros-arm64_x86_64-simulator/dSYMs/llama.dSYM/Contents/Resources/Relocations/x86_64/llama.yml +0 -4243
  956. package/ios/libs/llama.xcframework/xros-arm64_x86_64-simulator/llama.framework/Headers/ggml-alloc.h +0 -76
  957. package/ios/libs/llama.xcframework/xros-arm64_x86_64-simulator/llama.framework/Headers/ggml-backend.h +0 -354
  958. package/ios/libs/llama.xcframework/xros-arm64_x86_64-simulator/llama.framework/Headers/ggml-blas.h +0 -25
  959. package/ios/libs/llama.xcframework/xros-arm64_x86_64-simulator/llama.framework/Headers/ggml-cpu.h +0 -145
  960. package/ios/libs/llama.xcframework/xros-arm64_x86_64-simulator/llama.framework/Headers/ggml-metal.h +0 -66
  961. package/ios/libs/llama.xcframework/xros-arm64_x86_64-simulator/llama.framework/Headers/ggml-opt.h +0 -256
  962. package/ios/libs/llama.xcframework/xros-arm64_x86_64-simulator/llama.framework/Headers/ggml.h +0 -2492
  963. package/ios/libs/llama.xcframework/xros-arm64_x86_64-simulator/llama.framework/Headers/gguf.h +0 -202
  964. package/ios/libs/llama.xcframework/xros-arm64_x86_64-simulator/llama.framework/Headers/llama.h +0 -1391
  965. package/ios/libs/llama.xcframework/xros-arm64_x86_64-simulator/llama.framework/Info.plist +0 -32
  966. package/ios/libs/llama.xcframework/xros-arm64_x86_64-simulator/llama.framework/Modules/module.modulemap +0 -17
  967. package/ios/libs/llama.xcframework/xros-arm64_x86_64-simulator/llama.framework/llama +0 -0
  968. /package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/{test_bfloat16_support.comp → feature-tests/bfloat16.comp} +0 -0
  969. /package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/{test_coopmat_support.comp → feature-tests/coopmat.comp} +0 -0
  970. /package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/{test_coopmat2_support.comp → feature-tests/coopmat2.comp} +0 -0
  971. /package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/{test_integer_dot_support.comp → feature-tests/integer_dot.comp} +0 -0
  972. /package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/{generic_head.comp → generic_head.glsl} +0 -0
  973. /package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/{generic_unary_head.comp → generic_unary_head.glsl} +0 -0
  974. /package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/{glu_main.comp → glu_main.glsl} +0 -0
  975. /package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/{rte.comp → rte.glsl} +0 -0
  976. /package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/{utils.comp → utils.glsl} +0 -0
package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Headers/ggml.h DELETED
@@ -1,2492 +0,0 @@
1
- #pragma once
2
-
3
- //
4
- // GGML Tensor Library
5
- //
6
- // This documentation is still a work in progress.
7
- // If you wish some specific topics to be covered, feel free to drop a comment:
8
- //
9
- // https://github.com/ggerganov/whisper.cpp/issues/40
10
- //
11
- // ## Overview
12
- //
13
- // This library implements:
14
- //
15
- // - a set of tensor operations
16
- // - automatic differentiation
17
- // - basic optimization algorithms
18
- //
19
- // The aim of this library is to provide a minimalistic approach for various machine learning tasks. This includes,
20
- // but is not limited to, the following:
21
- //
22
- // - linear regression
23
- // - support vector machines
24
- // - neural networks
25
- //
26
- // The library allows the user to define a certain function using the available tensor operations. This function
27
- // definition is represented internally via a computation graph. Each tensor operation in the function definition
28
- // corresponds to a node in the graph. Having the computation graph defined, the user can choose to compute the
29
- // function's value and/or its gradient with respect to the input variables. Optionally, the function can be optimized
30
- // using one of the available optimization algorithms.
31
- //
32
- // For example, here we define the function: f(x) = a*x^2 + b
33
- //
34
- // {
35
- // struct ggml_init_params params = {
36
- // .mem_size = 16*1024*1024,
37
- // .mem_buffer = NULL,
38
- // };
39
- //
40
- // // memory allocation happens here
41
- // struct ggml_context * ctx = ggml_init(params);
42
- //
43
- // struct ggml_tensor * x = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1);
44
- //
45
- // ggml_set_param(ctx, x); // x is an input variable
46
- //
47
- // struct ggml_tensor * a = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1);
48
- // struct ggml_tensor * b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1);
49
- // struct ggml_tensor * x2 = ggml_mul(ctx, x, x);
50
- // struct ggml_tensor * f = ggml_add(ctx, ggml_mul(ctx, a, x2), b);
51
- //
52
- // ...
53
- // }
54
- //
55
- // Notice that the function definition above does not involve any actual computation. The computation is performed only
56
- // when the user explicitly requests it. For example, to compute the function's value at x = 2.0:
57
- //
58
- // {
59
- // ...
60
- //
61
- // struct ggml_cgraph * gf = ggml_new_graph(ctx);
62
- // ggml_build_forward_expand(gf, f);
63
- //
64
- // // set the input variable and parameter values
65
- // ggml_set_f32(x, 2.0f);
66
- // ggml_set_f32(a, 3.0f);
67
- // ggml_set_f32(b, 4.0f);
68
- //
69
- // ggml_graph_compute_with_ctx(ctx, &gf, n_threads);
70
- //
71
- // printf("f = %f\n", ggml_get_f32_1d(f, 0));
72
- //
73
- // ...
74
- // }
75
- //
76
- // The actual computation is performed in the ggml_graph_compute() function.
77
- //
78
- // The ggml_new_tensor_...() functions create new tensors. They are allocated in the memory buffer provided to the
79
- // ggml_init() function. You have to be careful not to exceed the memory buffer size. Therefore, you have to know
80
- // in advance how much memory you need for your computation. Alternatively, you can allocate a large enough memory
81
- // and after defining the computation graph, call the ggml_used_mem() function to find out how much memory was
82
- // actually needed.
83
- //
84
- // The ggml_set_param() function marks a tensor as an input variable. This is used by the automatic
85
- // differentiation and optimization algorithms.
86
- //
87
- // The described approach allows to define the function graph once and then compute its forward or backward graphs
88
- // multiple times. All computations will use the same memory buffer allocated in the ggml_init() function. This way
89
- // the user can avoid the memory allocation overhead at runtime.
90
- //
91
- // The library supports multi-dimensional tensors - up to 4 dimensions. The FP16 and FP32 data types are first class
92
- // citizens, but in theory the library can be extended to support FP8 and integer data types.
93
- //
94
- // Each tensor operation produces a new tensor. Initially the library was envisioned to support only the use of unary
95
- // and binary operations. Most of the available operations fall into one of these two categories. With time, it became
96
- // clear that the library needs to support more complex operations. The way to support these operations is not clear
97
- // yet, but a few examples are demonstrated in the following operations:
98
- //
99
- // - ggml_permute()
100
- // - ggml_conv_1d_1s()
101
- // - ggml_conv_1d_2s()
102
- //
103
- // For each tensor operator, the library implements a forward and backward computation function. The forward function
104
- // computes the output tensor value given the input tensor values. The backward function computes the adjoint of the
105
- // input tensors given the adjoint of the output tensor. For a detailed explanation of what this means, take a
106
- // calculus class, or watch the following video:
107
- //
108
- // What is Automatic Differentiation?
109
- // https://www.youtube.com/watch?v=wG_nF1awSSY
110
- //
111
- //
112
- // ## Tensor data (struct ggml_tensor)
113
- //
114
- // The tensors are stored in memory via the ggml_tensor struct. The structure provides information about the size of
115
- // the tensor, the data type, and the memory buffer where the tensor data is stored. Additionally, it contains
116
- // pointers to the "source" tensors - i.e. the tensors that were used to compute the current tensor. For example:
117
- //
118
- // {
119
- // struct ggml_tensor * c = ggml_add(ctx, a, b);
120
- //
121
- // assert(c->src[0] == a);
122
- // assert(c->src[1] == b);
123
- // }
124
- //
125
- // The multi-dimensional tensors are stored in row-major order. The ggml_tensor struct contains fields for the
126
- // number of elements in each dimension ("ne") as well as the number of bytes ("nb", a.k.a. stride). This allows
127
- // to store tensors that are not contiguous in memory, which is useful for operations such as transposition and
128
- // permutation. All tensor operations have to take the stride into account and not assume that the tensor is
129
- // contiguous in memory.
130
- //
131
- // The data of the tensor is accessed via the "data" pointer. For example:
132
- //
133
- // {
134
- // const int nx = 2;
135
- // const int ny = 3;
136
- //
137
- // struct ggml_tensor * a = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, nx, ny);
138
- //
139
- // for (int y = 0; y < ny; y++) {
140
- // for (int x = 0; x < nx; x++) {
141
- // *(float *) ((char *) a->data + y*a->nb[1] + x*a->nb[0]) = x + y;
142
- // }
143
- // }
144
- //
145
- // ...
146
- // }
147
- //
148
- // Alternatively, there are helper functions, such as ggml_get_f32_1d() and ggml_set_f32_1d() that can be used.
149
- //
150
- // ## The matrix multiplication operator (ggml_mul_mat)
151
- //
152
- // TODO
153
- //
154
- //
155
- // ## Multi-threading
156
- //
157
- // TODO
158
- //
159
- //
160
- // ## Overview of ggml.c
161
- //
162
- // TODO
163
- //
164
- //
165
- // ## SIMD optimizations
166
- //
167
- // TODO
168
- //
169
- //
170
- // ## Debugging ggml
171
- //
172
- // TODO
173
- //
174
- //
175
-
176
- #ifdef GGML_SHARED
177
- # if defined(_WIN32) && !defined(__MINGW32__)
178
- # ifdef GGML_BUILD
179
- # define GGML_API __declspec(dllexport) extern
180
- # else
181
- # define GGML_API __declspec(dllimport) extern
182
- # endif
183
- # else
184
- # define GGML_API __attribute__ ((visibility ("default"))) extern
185
- # endif
186
- #else
187
- # define GGML_API extern
188
- #endif
189
-
190
- // TODO: support for clang
191
- #ifdef __GNUC__
192
- # define GGML_DEPRECATED(func, hint) func __attribute__((deprecated(hint)))
193
- #elif defined(_MSC_VER)
194
- # define GGML_DEPRECATED(func, hint) __declspec(deprecated(hint)) func
195
- #else
196
- # define GGML_DEPRECATED(func, hint) func
197
- #endif
198
-
199
- #ifndef __GNUC__
200
- # define GGML_ATTRIBUTE_FORMAT(...)
201
- #elif defined(__MINGW32__) && !defined(__clang__)
202
- # define GGML_ATTRIBUTE_FORMAT(...) __attribute__((format(gnu_printf, __VA_ARGS__)))
203
- #else
204
- # define GGML_ATTRIBUTE_FORMAT(...) __attribute__((format(printf, __VA_ARGS__)))
205
- #endif
206
-
207
- #include <stdbool.h>
208
- #include <stddef.h>
209
- #include <stdint.h>
210
- #include <stdio.h>
211
-
212
- #define GGML_FILE_MAGIC 0x67676d6c // "ggml"
213
- #define GGML_FILE_VERSION 2
214
-
215
- #define GGML_QNT_VERSION 2 // bump this on quantization format changes
216
- #define GGML_QNT_VERSION_FACTOR 1000 // do not change this
217
-
218
- #define GGML_MAX_DIMS 4
219
- #define GGML_MAX_PARAMS 2048
220
- #define GGML_MAX_SRC 10
221
- #define GGML_MAX_N_THREADS 512
222
- #define GGML_MAX_OP_PARAMS 64
223
-
224
- #ifndef GGML_MAX_NAME
225
- # define GGML_MAX_NAME 64
226
- #endif
227
-
228
- #define GGML_DEFAULT_N_THREADS 4
229
- #define GGML_DEFAULT_GRAPH_SIZE 2048
230
-
231
- #if UINTPTR_MAX == 0xFFFFFFFF
232
- #define GGML_MEM_ALIGN 4
233
- #else
234
- #define GGML_MEM_ALIGN 16
235
- #endif
236
-
237
- #define GGML_EXIT_SUCCESS 0
238
- #define GGML_EXIT_ABORTED 1
239
-
240
- #define GGML_ROPE_TYPE_NEOX 2
241
- #define GGML_ROPE_TYPE_MROPE 8
242
- #define GGML_ROPE_TYPE_VISION 24
243
-
244
- #define GGML_MROPE_SECTIONS 4
245
-
246
- #define GGML_UNUSED(x) (void)(x)
247
- #ifdef __CUDACC__
248
- template<typename... Args>
249
- __host__ __device__ constexpr inline void ggml_unused_vars_impl(Args&&...) noexcept {}
250
- #define GGML_UNUSED_VARS(...) ggml_unused_vars_impl(__VA_ARGS__)
251
- #else
252
- #define GGML_UNUSED_VARS(...) do { (void)sizeof((__VA_ARGS__, 0)); } while(0)
253
- #endif // __CUDACC__
254
-
255
- #define GGML_PAD(x, n) (((x) + (n) - 1) & ~((n) - 1))
256
-
257
- #ifndef NDEBUG
258
- # define GGML_UNREACHABLE() do { fprintf(stderr, "statement should be unreachable\n"); abort(); } while(0)
259
- #elif defined(__GNUC__)
260
- # define GGML_UNREACHABLE() __builtin_unreachable()
261
- #elif defined(_MSC_VER)
262
- # define GGML_UNREACHABLE() __assume(0)
263
- #else
264
- # define GGML_UNREACHABLE() ((void) 0)
265
- #endif
266
-
267
- #ifdef __cplusplus
268
- # define GGML_NORETURN [[noreturn]]
269
- #elif defined(_MSC_VER)
270
- # define GGML_NORETURN __declspec(noreturn)
271
- #else
272
- # define GGML_NORETURN _Noreturn
273
- #endif
274
-
275
- #define GGML_ABORT(...) ggml_abort(__FILE__, __LINE__, __VA_ARGS__)
276
- #define GGML_ASSERT(x) if (!(x)) GGML_ABORT("GGML_ASSERT(%s) failed", #x)
277
-
278
- // used to copy the number of elements and stride in bytes of tensors into local variables.
279
- // main purpose is to reduce code duplication and improve readability.
280
- //
281
- // example:
282
- //
283
- // GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne);
284
- // GGML_TENSOR_LOCALS(size_t, nb1, src1, nb);
285
- //
286
- #define GGML_TENSOR_LOCALS_1(type, prefix, pointer, array) \
287
- const type prefix##0 = (pointer)->array[0]; \
288
- GGML_UNUSED(prefix##0);
289
- #define GGML_TENSOR_LOCALS_2(type, prefix, pointer, array) \
290
- GGML_TENSOR_LOCALS_1 (type, prefix, pointer, array) \
291
- const type prefix##1 = (pointer)->array[1]; \
292
- GGML_UNUSED(prefix##1);
293
- #define GGML_TENSOR_LOCALS_3(type, prefix, pointer, array) \
294
- GGML_TENSOR_LOCALS_2 (type, prefix, pointer, array) \
295
- const type prefix##2 = (pointer)->array[2]; \
296
- GGML_UNUSED(prefix##2);
297
- #define GGML_TENSOR_LOCALS(type, prefix, pointer, array) \
298
- GGML_TENSOR_LOCALS_3 (type, prefix, pointer, array) \
299
- const type prefix##3 = (pointer)->array[3]; \
300
- GGML_UNUSED(prefix##3);
301
-
302
- #define GGML_TENSOR_UNARY_OP_LOCALS \
303
- GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) \
304
- GGML_TENSOR_LOCALS(size_t, nb0, src0, nb) \
305
- GGML_TENSOR_LOCALS(int64_t, ne, dst, ne) \
306
- GGML_TENSOR_LOCALS(size_t, nb, dst, nb)
307
-
308
- #define GGML_TENSOR_BINARY_OP_LOCALS \
309
- GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) \
310
- GGML_TENSOR_LOCALS(size_t, nb0, src0, nb) \
311
- GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne) \
312
- GGML_TENSOR_LOCALS(size_t, nb1, src1, nb) \
313
- GGML_TENSOR_LOCALS(int64_t, ne, dst, ne) \
314
- GGML_TENSOR_LOCALS(size_t, nb, dst, nb)
315
-
316
- #define GGML_TENSOR_TERNARY_OP_LOCALS \
317
- GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) \
318
- GGML_TENSOR_LOCALS(size_t, nb0, src0, nb) \
319
- GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne) \
320
- GGML_TENSOR_LOCALS(size_t, nb1, src1, nb) \
321
- GGML_TENSOR_LOCALS(int64_t, ne2, src2, ne) \
322
- GGML_TENSOR_LOCALS(size_t, nb2, src2, nb) \
323
- GGML_TENSOR_LOCALS(int64_t, ne, dst, ne) \
324
- GGML_TENSOR_LOCALS(size_t, nb, dst, nb)
325
-
326
- #define GGML_TENSOR_BINARY_OP_LOCALS01 \
327
- GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) \
328
- GGML_TENSOR_LOCALS(size_t, nb0, src0, nb) \
329
- GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne) \
330
- GGML_TENSOR_LOCALS(size_t, nb1, src1, nb)
331
-
332
- #ifdef __cplusplus
333
- extern "C" {
334
- #endif
335
-
336
- // Function type used in fatal error callbacks
337
- typedef void (*ggml_abort_callback_t)(const char * error_message);
338
-
339
- // Set the abort callback (passing null will restore original abort functionality: printing a message to stdout)
340
- // Returns the old callback for chaining
341
- GGML_API ggml_abort_callback_t ggml_set_abort_callback(ggml_abort_callback_t callback);
342
-
343
- GGML_NORETURN GGML_ATTRIBUTE_FORMAT(3, 4)
344
- GGML_API void ggml_abort(const char * file, int line, const char * fmt, ...);
345
-
346
- enum ggml_status {
347
- GGML_STATUS_ALLOC_FAILED = -2,
348
- GGML_STATUS_FAILED = -1,
349
- GGML_STATUS_SUCCESS = 0,
350
- GGML_STATUS_ABORTED = 1,
351
- };
352
-
353
- // get ggml_status name string
354
- GGML_API const char * ggml_status_to_string(enum ggml_status status);
355
-
356
- // ieee 754-2008 half-precision float16
357
- // todo: make this not an integral type
358
- typedef uint16_t ggml_fp16_t;
359
- GGML_API float ggml_fp16_to_fp32(ggml_fp16_t);
360
- GGML_API ggml_fp16_t ggml_fp32_to_fp16(float);
361
- GGML_API void ggml_fp16_to_fp32_row(const ggml_fp16_t *, float *, int64_t);
362
- GGML_API void ggml_fp32_to_fp16_row(const float *, ggml_fp16_t *, int64_t);
363
-
364
- // google brain half-precision bfloat16
365
- typedef struct { uint16_t bits; } ggml_bf16_t;
366
- GGML_API ggml_bf16_t ggml_fp32_to_bf16(float);
367
- GGML_API float ggml_bf16_to_fp32(ggml_bf16_t); // consider just doing << 16
368
- GGML_API void ggml_bf16_to_fp32_row(const ggml_bf16_t *, float *, int64_t);
369
- GGML_API void ggml_fp32_to_bf16_row_ref(const float *, ggml_bf16_t *, int64_t);
370
- GGML_API void ggml_fp32_to_bf16_row(const float *, ggml_bf16_t *, int64_t);
371
-
372
- struct ggml_object;
373
- struct ggml_context;
374
- struct ggml_cgraph;
375
-
376
- // NOTE: always add types at the end of the enum to keep backward compatibility
377
- enum ggml_type {
378
- GGML_TYPE_F32 = 0,
379
- GGML_TYPE_F16 = 1,
380
- GGML_TYPE_Q4_0 = 2,
381
- GGML_TYPE_Q4_1 = 3,
382
- // GGML_TYPE_Q4_2 = 4, support has been removed
383
- // GGML_TYPE_Q4_3 = 5, support has been removed
384
- GGML_TYPE_Q5_0 = 6,
385
- GGML_TYPE_Q5_1 = 7,
386
- GGML_TYPE_Q8_0 = 8,
387
- GGML_TYPE_Q8_1 = 9,
388
- GGML_TYPE_Q2_K = 10,
389
- GGML_TYPE_Q3_K = 11,
390
- GGML_TYPE_Q4_K = 12,
391
- GGML_TYPE_Q5_K = 13,
392
- GGML_TYPE_Q6_K = 14,
393
- GGML_TYPE_Q8_K = 15,
394
- GGML_TYPE_IQ2_XXS = 16,
395
- GGML_TYPE_IQ2_XS = 17,
396
- GGML_TYPE_IQ3_XXS = 18,
397
- GGML_TYPE_IQ1_S = 19,
398
- GGML_TYPE_IQ4_NL = 20,
399
- GGML_TYPE_IQ3_S = 21,
400
- GGML_TYPE_IQ2_S = 22,
401
- GGML_TYPE_IQ4_XS = 23,
402
- GGML_TYPE_I8 = 24,
403
- GGML_TYPE_I16 = 25,
404
- GGML_TYPE_I32 = 26,
405
- GGML_TYPE_I64 = 27,
406
- GGML_TYPE_F64 = 28,
407
- GGML_TYPE_IQ1_M = 29,
408
- GGML_TYPE_BF16 = 30,
409
- // GGML_TYPE_Q4_0_4_4 = 31, support has been removed from gguf files
410
- // GGML_TYPE_Q4_0_4_8 = 32,
411
- // GGML_TYPE_Q4_0_8_8 = 33,
412
- GGML_TYPE_TQ1_0 = 34,
413
- GGML_TYPE_TQ2_0 = 35,
414
- // GGML_TYPE_IQ4_NL_4_4 = 36,
415
- // GGML_TYPE_IQ4_NL_4_8 = 37,
416
- // GGML_TYPE_IQ4_NL_8_8 = 38,
417
- GGML_TYPE_MXFP4 = 39, // MXFP4 (1 block)
418
- GGML_TYPE_COUNT = 40,
419
- };
420
-
421
- // precision
422
- enum ggml_prec {
423
- GGML_PREC_DEFAULT = 0, // stored as ggml_tensor.op_params, 0 by default
424
- GGML_PREC_F32 = 10,
425
- };
426
-
427
- // model file types
428
- enum ggml_ftype {
429
- GGML_FTYPE_UNKNOWN = -1,
430
- GGML_FTYPE_ALL_F32 = 0,
431
- GGML_FTYPE_MOSTLY_F16 = 1, // except 1d tensors
432
- GGML_FTYPE_MOSTLY_Q4_0 = 2, // except 1d tensors
433
- GGML_FTYPE_MOSTLY_Q4_1 = 3, // except 1d tensors
434
- GGML_FTYPE_MOSTLY_Q4_1_SOME_F16 = 4, // tok_embeddings.weight and output.weight are F16
435
- GGML_FTYPE_MOSTLY_Q8_0 = 7, // except 1d tensors
436
- GGML_FTYPE_MOSTLY_Q5_0 = 8, // except 1d tensors
437
- GGML_FTYPE_MOSTLY_Q5_1 = 9, // except 1d tensors
438
- GGML_FTYPE_MOSTLY_Q2_K = 10, // except 1d tensors
439
- GGML_FTYPE_MOSTLY_Q3_K = 11, // except 1d tensors
440
- GGML_FTYPE_MOSTLY_Q4_K = 12, // except 1d tensors
441
- GGML_FTYPE_MOSTLY_Q5_K = 13, // except 1d tensors
442
- GGML_FTYPE_MOSTLY_Q6_K = 14, // except 1d tensors
443
- GGML_FTYPE_MOSTLY_IQ2_XXS = 15, // except 1d tensors
444
- GGML_FTYPE_MOSTLY_IQ2_XS = 16, // except 1d tensors
445
- GGML_FTYPE_MOSTLY_IQ3_XXS = 17, // except 1d tensors
446
- GGML_FTYPE_MOSTLY_IQ1_S = 18, // except 1d tensors
447
- GGML_FTYPE_MOSTLY_IQ4_NL = 19, // except 1d tensors
448
- GGML_FTYPE_MOSTLY_IQ3_S = 20, // except 1d tensors
449
- GGML_FTYPE_MOSTLY_IQ2_S = 21, // except 1d tensors
450
- GGML_FTYPE_MOSTLY_IQ4_XS = 22, // except 1d tensors
451
- GGML_FTYPE_MOSTLY_IQ1_M = 23, // except 1d tensors
452
- GGML_FTYPE_MOSTLY_BF16 = 24, // except 1d tensors
453
- GGML_FTYPE_MOSTLY_MXFP4 = 25, // except 1d tensors
454
- };
455
-
456
- // available tensor operations:
457
- enum ggml_op {
458
- GGML_OP_NONE = 0,
459
-
460
- GGML_OP_DUP,
461
- GGML_OP_ADD,
462
- GGML_OP_ADD_ID,
463
- GGML_OP_ADD1,
464
- GGML_OP_ACC,
465
- GGML_OP_SUB,
466
- GGML_OP_MUL,
467
- GGML_OP_DIV,
468
- GGML_OP_SQR,
469
- GGML_OP_SQRT,
470
- GGML_OP_LOG,
471
- GGML_OP_SIN,
472
- GGML_OP_COS,
473
- GGML_OP_SUM,
474
- GGML_OP_SUM_ROWS,
475
- GGML_OP_MEAN,
476
- GGML_OP_ARGMAX,
477
- GGML_OP_COUNT_EQUAL,
478
- GGML_OP_REPEAT,
479
- GGML_OP_REPEAT_BACK,
480
- GGML_OP_CONCAT,
481
- GGML_OP_SILU_BACK,
482
- GGML_OP_NORM, // normalize
483
- GGML_OP_RMS_NORM,
484
- GGML_OP_RMS_NORM_BACK,
485
- GGML_OP_GROUP_NORM,
486
- GGML_OP_L2_NORM,
487
-
488
- GGML_OP_MUL_MAT,
489
- GGML_OP_MUL_MAT_ID,
490
- GGML_OP_OUT_PROD,
491
-
492
- GGML_OP_SCALE,
493
- GGML_OP_SET,
494
- GGML_OP_CPY,
495
- GGML_OP_CONT,
496
- GGML_OP_RESHAPE,
497
- GGML_OP_VIEW,
498
- GGML_OP_PERMUTE,
499
- GGML_OP_TRANSPOSE,
500
- GGML_OP_GET_ROWS,
501
- GGML_OP_GET_ROWS_BACK,
502
- GGML_OP_SET_ROWS,
503
- GGML_OP_DIAG,
504
- GGML_OP_DIAG_MASK_INF,
505
- GGML_OP_DIAG_MASK_ZERO,
506
- GGML_OP_SOFT_MAX,
507
- GGML_OP_SOFT_MAX_BACK,
508
- GGML_OP_ROPE,
509
- GGML_OP_ROPE_BACK,
510
- GGML_OP_CLAMP,
511
- GGML_OP_CONV_TRANSPOSE_1D,
512
- GGML_OP_IM2COL,
513
- GGML_OP_IM2COL_BACK,
514
- GGML_OP_CONV_2D,
515
- GGML_OP_CONV_3D,
516
- GGML_OP_CONV_2D_DW,
517
- GGML_OP_CONV_TRANSPOSE_2D,
518
- GGML_OP_POOL_1D,
519
- GGML_OP_POOL_2D,
520
- GGML_OP_POOL_2D_BACK,
521
- GGML_OP_UPSCALE,
522
- GGML_OP_PAD,
523
- GGML_OP_PAD_REFLECT_1D,
524
- GGML_OP_ROLL,
525
- GGML_OP_ARANGE,
526
- GGML_OP_TIMESTEP_EMBEDDING,
527
- GGML_OP_ARGSORT,
528
- GGML_OP_LEAKY_RELU,
529
-
530
- GGML_OP_FLASH_ATTN_EXT,
531
- GGML_OP_FLASH_ATTN_BACK,
532
- GGML_OP_SSM_CONV,
533
- GGML_OP_SSM_SCAN,
534
- GGML_OP_WIN_PART,
535
- GGML_OP_WIN_UNPART,
536
- GGML_OP_GET_REL_POS,
537
- GGML_OP_ADD_REL_POS,
538
- GGML_OP_RWKV_WKV6,
539
- GGML_OP_GATED_LINEAR_ATTN,
540
- GGML_OP_RWKV_WKV7,
541
-
542
- GGML_OP_UNARY,
543
-
544
- GGML_OP_MAP_CUSTOM1,
545
- GGML_OP_MAP_CUSTOM2,
546
- GGML_OP_MAP_CUSTOM3,
547
-
548
- GGML_OP_CUSTOM,
549
-
550
- GGML_OP_CROSS_ENTROPY_LOSS,
551
- GGML_OP_CROSS_ENTROPY_LOSS_BACK,
552
- GGML_OP_OPT_STEP_ADAMW,
553
- GGML_OP_OPT_STEP_SGD,
554
-
555
- GGML_OP_GLU,
556
-
557
- GGML_OP_COUNT,
558
- };
559
-
560
- enum ggml_unary_op {
561
- GGML_UNARY_OP_ABS,
562
- GGML_UNARY_OP_SGN,
563
- GGML_UNARY_OP_NEG,
564
- GGML_UNARY_OP_STEP,
565
- GGML_UNARY_OP_TANH,
566
- GGML_UNARY_OP_ELU,
567
- GGML_UNARY_OP_RELU,
568
- GGML_UNARY_OP_SIGMOID,
569
- GGML_UNARY_OP_GELU,
570
- GGML_UNARY_OP_GELU_QUICK,
571
- GGML_UNARY_OP_SILU,
572
- GGML_UNARY_OP_HARDSWISH,
573
- GGML_UNARY_OP_HARDSIGMOID,
574
- GGML_UNARY_OP_EXP,
575
- GGML_UNARY_OP_GELU_ERF,
576
-
577
- GGML_UNARY_OP_COUNT,
578
- };
579
-
580
- enum ggml_glu_op {
581
- GGML_GLU_OP_REGLU,
582
- GGML_GLU_OP_GEGLU,
583
- GGML_GLU_OP_SWIGLU,
584
- GGML_GLU_OP_SWIGLU_OAI,
585
- GGML_GLU_OP_GEGLU_ERF,
586
- GGML_GLU_OP_GEGLU_QUICK,
587
-
588
- GGML_GLU_OP_COUNT,
589
- };
590
-
591
- enum ggml_object_type {
592
- GGML_OBJECT_TYPE_TENSOR,
593
- GGML_OBJECT_TYPE_GRAPH,
594
- GGML_OBJECT_TYPE_WORK_BUFFER
595
- };
596
-
597
- enum ggml_log_level {
598
- GGML_LOG_LEVEL_NONE = 0,
599
- GGML_LOG_LEVEL_DEBUG = 1,
600
- GGML_LOG_LEVEL_INFO = 2,
601
- GGML_LOG_LEVEL_WARN = 3,
602
- GGML_LOG_LEVEL_ERROR = 4,
603
- GGML_LOG_LEVEL_CONT = 5, // continue previous log
604
- };
605
-
606
- // this tensor...
607
- enum ggml_tensor_flag {
608
- GGML_TENSOR_FLAG_INPUT = 1, // ...is an input for the GGML compute graph
609
- GGML_TENSOR_FLAG_OUTPUT = 2, // ...is an output for the GGML compute graph
610
- GGML_TENSOR_FLAG_PARAM = 4, // ...contains trainable parameters
611
- GGML_TENSOR_FLAG_LOSS = 8, // ...defines loss for numerical optimization (multiple loss tensors add up)
612
- };
613
-
614
- struct ggml_init_params {
615
- // memory pool
616
- size_t mem_size; // bytes
617
- void * mem_buffer; // if NULL, memory will be allocated internally
618
- bool no_alloc; // don't allocate memory for the tensor data
619
- };
620
-
621
- // n-dimensional tensor
622
- struct ggml_tensor {
623
- enum ggml_type type;
624
-
625
- struct ggml_backend_buffer * buffer;
626
-
627
- int64_t ne[GGML_MAX_DIMS]; // number of elements
628
- size_t nb[GGML_MAX_DIMS]; // stride in bytes:
629
- // nb[0] = ggml_type_size(type)
630
- // nb[1] = nb[0] * (ne[0] / ggml_blck_size(type)) + padding
631
- // nb[i] = nb[i-1] * ne[i-1]
632
-
633
- // compute data
634
- enum ggml_op op;
635
-
636
- // op params - allocated as int32_t for alignment
637
- int32_t op_params[GGML_MAX_OP_PARAMS / sizeof(int32_t)];
638
-
639
- int32_t flags;
640
-
641
- struct ggml_tensor * src[GGML_MAX_SRC];
642
-
643
- // source tensor and offset for views
644
- struct ggml_tensor * view_src;
645
- size_t view_offs;
646
-
647
- void * data;
648
-
649
- char name[GGML_MAX_NAME];
650
-
651
- void * extra; // extra things e.g. for ggml-cuda.cu
652
-
653
- char padding[8];
654
- };
655
-
656
- static const size_t GGML_TENSOR_SIZE = sizeof(struct ggml_tensor);
657
-
658
- // Abort callback
659
- // If not NULL, called before ggml computation
660
- // If it returns true, the computation is aborted
661
- typedef bool (*ggml_abort_callback)(void * data);
662
-
663
-
664
- //
665
- // GUID
666
- //
667
-
668
- // GUID types
669
- typedef uint8_t ggml_guid[16];
670
- typedef ggml_guid * ggml_guid_t;
671
-
672
- GGML_API bool ggml_guid_matches(ggml_guid_t guid_a, ggml_guid_t guid_b);
673
-
674
- // misc
675
-
676
- GGML_API const char * ggml_version(void);
677
- GGML_API const char * ggml_commit(void);
678
-
679
- GGML_API void ggml_time_init(void); // call this once at the beginning of the program
680
- GGML_API int64_t ggml_time_ms(void);
681
- GGML_API int64_t ggml_time_us(void);
682
- GGML_API int64_t ggml_cycles(void);
683
- GGML_API int64_t ggml_cycles_per_ms(void);
684
-
685
- // accepts a UTF-8 path, even on Windows
686
- GGML_API FILE * ggml_fopen(const char * fname, const char * mode);
687
-
688
- GGML_API void ggml_print_object (const struct ggml_object * obj);
689
- GGML_API void ggml_print_objects(const struct ggml_context * ctx);
690
-
691
- GGML_API int64_t ggml_nelements (const struct ggml_tensor * tensor);
692
- GGML_API int64_t ggml_nrows (const struct ggml_tensor * tensor);
693
- GGML_API size_t ggml_nbytes (const struct ggml_tensor * tensor);
694
- GGML_API size_t ggml_nbytes_pad(const struct ggml_tensor * tensor); // same as ggml_nbytes() but padded to GGML_MEM_ALIGN
695
-
696
- GGML_API int64_t ggml_blck_size(enum ggml_type type);
697
- GGML_API size_t ggml_type_size(enum ggml_type type); // size in bytes for all elements in a block
698
- GGML_API size_t ggml_row_size (enum ggml_type type, int64_t ne); // size in bytes for all elements in a row
699
-
700
- GGML_DEPRECATED(
701
- GGML_API double ggml_type_sizef(enum ggml_type type), // ggml_type_size()/ggml_blck_size() as float
702
- "use ggml_row_size() instead");
703
-
704
- GGML_API const char * ggml_type_name(enum ggml_type type);
705
- GGML_API const char * ggml_op_name (enum ggml_op op);
706
- GGML_API const char * ggml_op_symbol(enum ggml_op op);
707
-
708
- GGML_API const char * ggml_unary_op_name(enum ggml_unary_op op);
709
- GGML_API const char * ggml_glu_op_name(enum ggml_glu_op op);
710
- GGML_API const char * ggml_op_desc(const struct ggml_tensor * t); // unary or op name
711
-
712
- GGML_API size_t ggml_element_size(const struct ggml_tensor * tensor);
713
-
714
- GGML_API bool ggml_is_quantized(enum ggml_type type);
715
-
716
- // TODO: temporary until model loading of ggml examples is refactored
717
- GGML_API enum ggml_type ggml_ftype_to_ggml_type(enum ggml_ftype ftype);
718
-
719
- GGML_API bool ggml_is_transposed(const struct ggml_tensor * tensor);
720
- GGML_API bool ggml_is_permuted (const struct ggml_tensor * tensor);
721
- GGML_API bool ggml_is_empty (const struct ggml_tensor * tensor);
722
- GGML_API bool ggml_is_scalar (const struct ggml_tensor * tensor);
723
- GGML_API bool ggml_is_vector (const struct ggml_tensor * tensor);
724
- GGML_API bool ggml_is_matrix (const struct ggml_tensor * tensor);
725
- GGML_API bool ggml_is_3d (const struct ggml_tensor * tensor);
726
- GGML_API int ggml_n_dims (const struct ggml_tensor * tensor); // returns 1 for scalars
727
-
728
- // returns whether the tensor elements can be iterated over with a flattened index (no gaps, no permutation)
729
- GGML_API bool ggml_is_contiguous (const struct ggml_tensor * tensor);
730
- GGML_API bool ggml_is_contiguous_0(const struct ggml_tensor * tensor); // same as ggml_is_contiguous()
731
- GGML_API bool ggml_is_contiguous_1(const struct ggml_tensor * tensor); // contiguous for dims >= 1
732
- GGML_API bool ggml_is_contiguous_2(const struct ggml_tensor * tensor); // contiguous for dims >= 2
733
-
734
- // returns whether the tensor elements are allocated as one contiguous block of memory (no gaps, but permutation ok)
735
- GGML_API bool ggml_is_contiguously_allocated(const struct ggml_tensor * tensor);
736
-
737
- // true for tensor that is stored in memory as CxWxHxN and has been permuted to WxHxCxN
738
- GGML_API bool ggml_is_contiguous_channels(const struct ggml_tensor * tensor);
739
-
740
- // true if the elements in dimension 0 are contiguous, or there is just 1 block of elements
741
- GGML_API bool ggml_is_contiguous_rows(const struct ggml_tensor * tensor);
742
-
743
- GGML_API bool ggml_are_same_shape (const struct ggml_tensor * t0, const struct ggml_tensor * t1);
744
- GGML_API bool ggml_are_same_stride(const struct ggml_tensor * t0, const struct ggml_tensor * t1);
745
-
746
- GGML_API bool ggml_can_repeat(const struct ggml_tensor * t0, const struct ggml_tensor * t1);
747
-
748
- // use this to compute the memory overhead of a tensor
749
- GGML_API size_t ggml_tensor_overhead(void);
750
-
751
- GGML_API bool ggml_validate_row_data(enum ggml_type type, const void * data, size_t nbytes);
752
-
753
- // main
754
-
755
- GGML_API struct ggml_context * ggml_init (struct ggml_init_params params);
756
- GGML_API void ggml_reset(struct ggml_context * ctx);
757
- GGML_API void ggml_free (struct ggml_context * ctx);
758
-
759
- GGML_API size_t ggml_used_mem(const struct ggml_context * ctx);
760
-
761
- GGML_API bool ggml_get_no_alloc(struct ggml_context * ctx);
762
- GGML_API void ggml_set_no_alloc(struct ggml_context * ctx, bool no_alloc);
763
-
764
- GGML_API void * ggml_get_mem_buffer (const struct ggml_context * ctx);
765
- GGML_API size_t ggml_get_mem_size (const struct ggml_context * ctx);
766
- GGML_API size_t ggml_get_max_tensor_size(const struct ggml_context * ctx);
767
-
768
- GGML_API struct ggml_tensor * ggml_new_tensor(
769
- struct ggml_context * ctx,
770
- enum ggml_type type,
771
- int n_dims,
772
- const int64_t *ne);
773
-
774
- GGML_API struct ggml_tensor * ggml_new_tensor_1d(
775
- struct ggml_context * ctx,
776
- enum ggml_type type,
777
- int64_t ne0);
778
-
779
- GGML_API struct ggml_tensor * ggml_new_tensor_2d(
780
- struct ggml_context * ctx,
781
- enum ggml_type type,
782
- int64_t ne0,
783
- int64_t ne1);
784
-
785
- GGML_API struct ggml_tensor * ggml_new_tensor_3d(
786
- struct ggml_context * ctx,
787
- enum ggml_type type,
788
- int64_t ne0,
789
- int64_t ne1,
790
- int64_t ne2);
791
-
792
- GGML_API struct ggml_tensor * ggml_new_tensor_4d(
793
- struct ggml_context * ctx,
794
- enum ggml_type type,
795
- int64_t ne0,
796
- int64_t ne1,
797
- int64_t ne2,
798
- int64_t ne3);
799
-
800
- GGML_API void * ggml_new_buffer(struct ggml_context * ctx, size_t nbytes);
801
-
802
- GGML_API struct ggml_tensor * ggml_dup_tensor (struct ggml_context * ctx, const struct ggml_tensor * src);
803
- GGML_API struct ggml_tensor * ggml_view_tensor(struct ggml_context * ctx, struct ggml_tensor * src);
804
-
805
- // Context tensor enumeration and lookup
806
- GGML_API struct ggml_tensor * ggml_get_first_tensor(const struct ggml_context * ctx);
807
- GGML_API struct ggml_tensor * ggml_get_next_tensor (const struct ggml_context * ctx, struct ggml_tensor * tensor);
808
- GGML_API struct ggml_tensor * ggml_get_tensor(struct ggml_context * ctx, const char * name);
809
-
810
- // Converts a flat index into coordinates
811
- GGML_API void ggml_unravel_index(const struct ggml_tensor * tensor, int64_t i, int64_t * i0, int64_t * i1, int64_t * i2, int64_t * i3);
812
-
813
- GGML_API enum ggml_unary_op ggml_get_unary_op(const struct ggml_tensor * tensor);
814
- GGML_API enum ggml_glu_op ggml_get_glu_op(const struct ggml_tensor * tensor);
815
-
816
- GGML_API void * ggml_get_data (const struct ggml_tensor * tensor);
817
- GGML_API float * ggml_get_data_f32(const struct ggml_tensor * tensor);
818
-
819
- GGML_API const char * ggml_get_name (const struct ggml_tensor * tensor);
820
- GGML_API struct ggml_tensor * ggml_set_name ( struct ggml_tensor * tensor, const char * name);
821
- GGML_ATTRIBUTE_FORMAT(2, 3)
822
- GGML_API struct ggml_tensor * ggml_format_name( struct ggml_tensor * tensor, const char * fmt, ...);
823
-
824
- // Tensor flags
825
- GGML_API void ggml_set_input(struct ggml_tensor * tensor);
826
- GGML_API void ggml_set_output(struct ggml_tensor * tensor);
827
- GGML_API void ggml_set_param(struct ggml_tensor * tensor);
828
- GGML_API void ggml_set_loss(struct ggml_tensor * tensor);
829
-
830
- //
831
- // operations on tensors with backpropagation
832
- //
833
-
834
- GGML_API struct ggml_tensor * ggml_dup(
835
- struct ggml_context * ctx,
836
- struct ggml_tensor * a);
837
-
838
- // in-place, returns view(a)
839
- GGML_API struct ggml_tensor * ggml_dup_inplace(
840
- struct ggml_context * ctx,
841
- struct ggml_tensor * a);
842
-
843
- GGML_API struct ggml_tensor * ggml_add(
844
- struct ggml_context * ctx,
845
- struct ggml_tensor * a,
846
- struct ggml_tensor * b);
847
-
848
- GGML_API struct ggml_tensor * ggml_add_inplace(
849
- struct ggml_context * ctx,
850
- struct ggml_tensor * a,
851
- struct ggml_tensor * b);
852
-
853
- GGML_API struct ggml_tensor * ggml_add_cast(
854
- struct ggml_context * ctx,
855
- struct ggml_tensor * a,
856
- struct ggml_tensor * b,
857
- enum ggml_type type);
858
-
859
- // dst[i0, i1, i2] = a[i0, i1, i2] + b[i0, ids[i1, i2]]
860
- GGML_API struct ggml_tensor * ggml_add_id(
861
- struct ggml_context * ctx,
862
- struct ggml_tensor * a,
863
- struct ggml_tensor * b,
864
- struct ggml_tensor * ids);
865
-
866
- GGML_API struct ggml_tensor * ggml_add1(
867
- struct ggml_context * ctx,
868
- struct ggml_tensor * a,
869
- struct ggml_tensor * b);
870
-
871
- GGML_API struct ggml_tensor * ggml_add1_inplace(
872
- struct ggml_context * ctx,
873
- struct ggml_tensor * a,
874
- struct ggml_tensor * b);
875
-
876
- // dst = a
877
- // view(dst, nb1, nb2, nb3, offset) += b
878
- // return dst
879
- GGML_API struct ggml_tensor * ggml_acc(
880
- struct ggml_context * ctx,
881
- struct ggml_tensor * a,
882
- struct ggml_tensor * b,
883
- size_t nb1,
884
- size_t nb2,
885
- size_t nb3,
886
- size_t offset);
887
-
888
- GGML_API struct ggml_tensor * ggml_acc_inplace(
889
- struct ggml_context * ctx,
890
- struct ggml_tensor * a,
891
- struct ggml_tensor * b,
892
- size_t nb1,
893
- size_t nb2,
894
- size_t nb3,
895
- size_t offset);
896
-
897
- GGML_API struct ggml_tensor * ggml_sub(
898
- struct ggml_context * ctx,
899
- struct ggml_tensor * a,
900
- struct ggml_tensor * b);
901
-
902
- GGML_API struct ggml_tensor * ggml_sub_inplace(
903
- struct ggml_context * ctx,
904
- struct ggml_tensor * a,
905
- struct ggml_tensor * b);
906
-
907
- GGML_API struct ggml_tensor * ggml_mul(
908
- struct ggml_context * ctx,
909
- struct ggml_tensor * a,
910
- struct ggml_tensor * b);
911
-
912
- GGML_API struct ggml_tensor * ggml_mul_inplace(
913
- struct ggml_context * ctx,
914
- struct ggml_tensor * a,
915
- struct ggml_tensor * b);
916
-
917
- GGML_API struct ggml_tensor * ggml_div(
918
- struct ggml_context * ctx,
919
- struct ggml_tensor * a,
920
- struct ggml_tensor * b);
921
-
922
- GGML_API struct ggml_tensor * ggml_div_inplace(
923
- struct ggml_context * ctx,
924
- struct ggml_tensor * a,
925
- struct ggml_tensor * b);
926
-
927
- GGML_API struct ggml_tensor * ggml_sqr(
928
- struct ggml_context * ctx,
929
- struct ggml_tensor * a);
930
-
931
- GGML_API struct ggml_tensor * ggml_sqr_inplace(
932
- struct ggml_context * ctx,
933
- struct ggml_tensor * a);
934
-
935
- GGML_API struct ggml_tensor * ggml_sqrt(
936
- struct ggml_context * ctx,
937
- struct ggml_tensor * a);
938
-
939
- GGML_API struct ggml_tensor * ggml_sqrt_inplace(
940
- struct ggml_context * ctx,
941
- struct ggml_tensor * a);
942
-
943
- GGML_API struct ggml_tensor * ggml_log(
944
- struct ggml_context * ctx,
945
- struct ggml_tensor * a);
946
-
947
- GGML_API struct ggml_tensor * ggml_log_inplace(
948
- struct ggml_context * ctx,
949
- struct ggml_tensor * a);
950
-
951
- GGML_API struct ggml_tensor * ggml_sin(
952
- struct ggml_context * ctx,
953
- struct ggml_tensor * a);
954
-
955
- GGML_API struct ggml_tensor * ggml_sin_inplace(
956
- struct ggml_context * ctx,
957
- struct ggml_tensor * a);
958
-
959
- GGML_API struct ggml_tensor * ggml_cos(
960
- struct ggml_context * ctx,
961
- struct ggml_tensor * a);
962
-
963
- GGML_API struct ggml_tensor * ggml_cos_inplace(
964
- struct ggml_context * ctx,
965
- struct ggml_tensor * a);
966
-
967
- // return scalar
968
- GGML_API struct ggml_tensor * ggml_sum(
969
- struct ggml_context * ctx,
970
- struct ggml_tensor * a);
971
-
972
- // sums along rows, with input shape [a,b,c,d] return shape [1,b,c,d]
973
- GGML_API struct ggml_tensor * ggml_sum_rows(
974
- struct ggml_context * ctx,
975
- struct ggml_tensor * a);
976
-
977
- // mean along rows
978
- GGML_API struct ggml_tensor * ggml_mean(
979
- struct ggml_context * ctx,
980
- struct ggml_tensor * a);
981
-
982
- // argmax along rows
983
- GGML_API struct ggml_tensor * ggml_argmax(
984
- struct ggml_context * ctx,
985
- struct ggml_tensor * a);
986
-
987
- // count number of equal elements in a and b
988
- GGML_API struct ggml_tensor * ggml_count_equal(
989
- struct ggml_context * ctx,
990
- struct ggml_tensor * a,
991
- struct ggml_tensor * b);
992
-
993
- // if a is the same shape as b, and a is not parameter, return a
994
- // otherwise, return a new tensor: repeat(a) to fit in b
995
- GGML_API struct ggml_tensor * ggml_repeat(
996
- struct ggml_context * ctx,
997
- struct ggml_tensor * a,
998
- struct ggml_tensor * b);
999
-
1000
- // repeat a to the specified shape
1001
- GGML_API struct ggml_tensor * ggml_repeat_4d(
1002
- struct ggml_context * ctx,
1003
- struct ggml_tensor * a,
1004
- int64_t ne0,
1005
- int64_t ne1,
1006
- int64_t ne2,
1007
- int64_t ne3);
1008
-
1009
- // sums repetitions in a into shape of b
1010
- GGML_API struct ggml_tensor * ggml_repeat_back(
1011
- struct ggml_context * ctx,
1012
- struct ggml_tensor * a,
1013
- struct ggml_tensor * b); // sum up values that are adjacent in dims > 0 instead of repeated with same stride
1014
-
1015
- // concat a and b along dim
1016
- // used in stable-diffusion
1017
- GGML_API struct ggml_tensor * ggml_concat(
1018
- struct ggml_context * ctx,
1019
- struct ggml_tensor * a,
1020
- struct ggml_tensor * b,
1021
- int dim);
1022
-
1023
- GGML_API struct ggml_tensor * ggml_abs(
1024
- struct ggml_context * ctx,
1025
- struct ggml_tensor * a);
1026
-
1027
- GGML_API struct ggml_tensor * ggml_abs_inplace(
1028
- struct ggml_context * ctx,
1029
- struct ggml_tensor * a);
1030
-
1031
- GGML_API struct ggml_tensor * ggml_sgn(
1032
- struct ggml_context * ctx,
1033
- struct ggml_tensor * a);
1034
-
1035
- GGML_API struct ggml_tensor * ggml_sgn_inplace(
1036
- struct ggml_context * ctx,
1037
- struct ggml_tensor * a);
1038
-
1039
- GGML_API struct ggml_tensor * ggml_neg(
1040
- struct ggml_context * ctx,
1041
- struct ggml_tensor * a);
1042
-
1043
- GGML_API struct ggml_tensor * ggml_neg_inplace(
1044
- struct ggml_context * ctx,
1045
- struct ggml_tensor * a);
1046
-
1047
- GGML_API struct ggml_tensor * ggml_step(
1048
- struct ggml_context * ctx,
1049
- struct ggml_tensor * a);
1050
-
1051
- GGML_API struct ggml_tensor * ggml_step_inplace(
1052
- struct ggml_context * ctx,
1053
- struct ggml_tensor * a);
1054
-
1055
- GGML_API struct ggml_tensor * ggml_tanh(
1056
- struct ggml_context * ctx,
1057
- struct ggml_tensor * a);
1058
-
1059
- GGML_API struct ggml_tensor * ggml_tanh_inplace(
1060
- struct ggml_context * ctx,
1061
- struct ggml_tensor * a);
1062
-
1063
- GGML_API struct ggml_tensor * ggml_elu(
1064
- struct ggml_context * ctx,
1065
- struct ggml_tensor * a);
1066
-
1067
- GGML_API struct ggml_tensor * ggml_elu_inplace(
1068
- struct ggml_context * ctx,
1069
- struct ggml_tensor * a);
1070
-
1071
- GGML_API struct ggml_tensor * ggml_relu(
1072
- struct ggml_context * ctx,
1073
- struct ggml_tensor * a);
1074
-
1075
- GGML_API struct ggml_tensor * ggml_leaky_relu(
1076
- struct ggml_context * ctx,
1077
- struct ggml_tensor * a, float negative_slope, bool inplace);
1078
-
1079
- GGML_API struct ggml_tensor * ggml_relu_inplace(
1080
- struct ggml_context * ctx,
1081
- struct ggml_tensor * a);
1082
-
1083
- GGML_API struct ggml_tensor * ggml_sigmoid(
1084
- struct ggml_context * ctx,
1085
- struct ggml_tensor * a);
1086
-
1087
- GGML_API struct ggml_tensor * ggml_sigmoid_inplace(
1088
- struct ggml_context * ctx,
1089
- struct ggml_tensor * a);
1090
-
1091
- GGML_API struct ggml_tensor * ggml_gelu(
1092
- struct ggml_context * ctx,
1093
- struct ggml_tensor * a);
1094
-
1095
- GGML_API struct ggml_tensor * ggml_gelu_inplace(
1096
- struct ggml_context * ctx,
1097
- struct ggml_tensor * a);
1098
-
1099
- // GELU using erf (error function) when possible
1100
- // some backends may fallback to approximation based on Abramowitz and Stegun formula
1101
- GGML_API struct ggml_tensor * ggml_gelu_erf(
1102
- struct ggml_context * ctx,
1103
- struct ggml_tensor * a);
1104
-
1105
- GGML_API struct ggml_tensor * ggml_gelu_erf_inplace(
1106
- struct ggml_context * ctx,
1107
- struct ggml_tensor * a);
1108
-
1109
- GGML_API struct ggml_tensor * ggml_gelu_quick(
1110
- struct ggml_context * ctx,
1111
- struct ggml_tensor * a);
1112
-
1113
- GGML_API struct ggml_tensor * ggml_gelu_quick_inplace(
1114
- struct ggml_context * ctx,
1115
- struct ggml_tensor * a);
1116
-
1117
- GGML_API struct ggml_tensor * ggml_silu(
1118
- struct ggml_context * ctx,
1119
- struct ggml_tensor * a);
1120
-
1121
- GGML_API struct ggml_tensor * ggml_silu_inplace(
1122
- struct ggml_context * ctx,
1123
- struct ggml_tensor * a);
1124
-
1125
- // a - x
1126
- // b - dy
1127
- GGML_API struct ggml_tensor * ggml_silu_back(
1128
- struct ggml_context * ctx,
1129
- struct ggml_tensor * a,
1130
- struct ggml_tensor * b);
1131
-
1132
- // hardswish(x) = x * relu6(x + 3) / 6
1133
- GGML_API struct ggml_tensor * ggml_hardswish(
1134
- struct ggml_context * ctx,
1135
- struct ggml_tensor * a);
1136
-
1137
- // hardsigmoid(x) = relu6(x + 3) / 6
1138
- GGML_API struct ggml_tensor * ggml_hardsigmoid(
1139
- struct ggml_context * ctx,
1140
- struct ggml_tensor * a);
1141
-
1142
- GGML_API struct ggml_tensor * ggml_exp(
1143
- struct ggml_context * ctx,
1144
- struct ggml_tensor * a);
1145
-
1146
- GGML_API struct ggml_tensor * ggml_exp_inplace(
1147
- struct ggml_context * ctx,
1148
- struct ggml_tensor * a);
1149
-
1150
- // gated linear unit ops
1151
- // A: n columns, r rows,
1152
- // result is n / 2 columns, r rows,
1153
- // expects gate in second half of row, unless swapped is true
1154
- GGML_API struct ggml_tensor * ggml_glu(
1155
- struct ggml_context * ctx,
1156
- struct ggml_tensor * a,
1157
- enum ggml_glu_op op,
1158
- bool swapped);
1159
-
1160
- GGML_API struct ggml_tensor * ggml_reglu(
1161
- struct ggml_context * ctx,
1162
- struct ggml_tensor * a);
1163
-
1164
- GGML_API struct ggml_tensor * ggml_reglu_swapped(
1165
- struct ggml_context * ctx,
1166
- struct ggml_tensor * a);
1167
-
1168
- GGML_API struct ggml_tensor * ggml_geglu(
1169
- struct ggml_context * ctx,
1170
- struct ggml_tensor * a);
1171
-
1172
- GGML_API struct ggml_tensor * ggml_geglu_swapped(
1173
- struct ggml_context * ctx,
1174
- struct ggml_tensor * a);
1175
-
1176
- GGML_API struct ggml_tensor * ggml_swiglu(
1177
- struct ggml_context * ctx,
1178
- struct ggml_tensor * a);
1179
-
1180
- GGML_API struct ggml_tensor * ggml_swiglu_swapped(
1181
- struct ggml_context * ctx,
1182
- struct ggml_tensor * a);
1183
-
1184
- GGML_API struct ggml_tensor * ggml_geglu_erf(
1185
- struct ggml_context * ctx,
1186
- struct ggml_tensor * a);
1187
-
1188
- GGML_API struct ggml_tensor * ggml_geglu_erf_swapped(
1189
- struct ggml_context * ctx,
1190
- struct ggml_tensor * a);
1191
-
1192
- GGML_API struct ggml_tensor * ggml_geglu_quick(
1193
- struct ggml_context * ctx,
1194
- struct ggml_tensor * a);
1195
-
1196
- GGML_API struct ggml_tensor * ggml_geglu_quick_swapped(
1197
- struct ggml_context * ctx,
1198
- struct ggml_tensor * a);
1199
-
1200
- // A: n columns, r rows,
1201
- // B: n columns, r rows,
1202
- GGML_API struct ggml_tensor * ggml_glu_split(
1203
- struct ggml_context * ctx,
1204
- struct ggml_tensor * a,
1205
- struct ggml_tensor * b,
1206
- enum ggml_glu_op op);
1207
-
1208
- GGML_API struct ggml_tensor * ggml_reglu_split(
1209
- struct ggml_context * ctx,
1210
- struct ggml_tensor * a,
1211
- struct ggml_tensor * b);
1212
-
1213
- GGML_API struct ggml_tensor * ggml_geglu_split(
1214
- struct ggml_context * ctx,
1215
- struct ggml_tensor * a,
1216
- struct ggml_tensor * b);
1217
-
1218
- GGML_API struct ggml_tensor * ggml_swiglu_split(
1219
- struct ggml_context * ctx,
1220
- struct ggml_tensor * a,
1221
- struct ggml_tensor * b);
1222
-
1223
- GGML_API struct ggml_tensor * ggml_geglu_erf_split(
1224
- struct ggml_context * ctx,
1225
- struct ggml_tensor * a,
1226
- struct ggml_tensor * b);
1227
-
1228
- GGML_API struct ggml_tensor * ggml_geglu_quick_split(
1229
- struct ggml_context * ctx,
1230
- struct ggml_tensor * a,
1231
- struct ggml_tensor * b);
1232
-
1233
- GGML_API struct ggml_tensor * ggml_swiglu_oai(
1234
- struct ggml_context * ctx,
1235
- struct ggml_tensor * a,
1236
- struct ggml_tensor * b,
1237
- float alpha,
1238
- float limit);
1239
-
1240
- // normalize along rows
1241
- GGML_API struct ggml_tensor * ggml_norm(
1242
- struct ggml_context * ctx,
1243
- struct ggml_tensor * a,
1244
- float eps);
1245
-
1246
- GGML_API struct ggml_tensor * ggml_norm_inplace(
1247
- struct ggml_context * ctx,
1248
- struct ggml_tensor * a,
1249
- float eps);
1250
-
1251
- GGML_API struct ggml_tensor * ggml_rms_norm(
1252
- struct ggml_context * ctx,
1253
- struct ggml_tensor * a,
1254
- float eps);
1255
-
1256
- GGML_API struct ggml_tensor * ggml_rms_norm_inplace(
1257
- struct ggml_context * ctx,
1258
- struct ggml_tensor * a,
1259
- float eps);
1260
-
1261
- // group normalize along ne0*ne1*n_groups
1262
- // used in stable-diffusion
1263
- GGML_API struct ggml_tensor * ggml_group_norm(
1264
- struct ggml_context * ctx,
1265
- struct ggml_tensor * a,
1266
- int n_groups,
1267
- float eps);
1268
-
1269
- GGML_API struct ggml_tensor * ggml_group_norm_inplace(
1270
- struct ggml_context * ctx,
1271
- struct ggml_tensor * a,
1272
- int n_groups,
1273
- float eps);
1274
-
1275
- // l2 normalize along rows
1276
- // used in rwkv v7
1277
- GGML_API struct ggml_tensor * ggml_l2_norm(
1278
- struct ggml_context * ctx,
1279
- struct ggml_tensor * a,
1280
- float eps);
1281
-
1282
- GGML_API struct ggml_tensor * ggml_l2_norm_inplace(
1283
- struct ggml_context * ctx,
1284
- struct ggml_tensor * a,
1285
- float eps);
1286
-
1287
- // a - x
1288
- // b - dy
1289
- GGML_API struct ggml_tensor * ggml_rms_norm_back(
1290
- struct ggml_context * ctx,
1291
- struct ggml_tensor * a,
1292
- struct ggml_tensor * b,
1293
- float eps);
1294
-
1295
- // A: k columns, n rows => [ne03, ne02, n, k]
1296
- // B: k columns, m rows (i.e. we transpose it internally) => [ne03 * x, ne02 * y, m, k]
1297
- // result is n columns, m rows => [ne03 * x, ne02 * y, m, n]
1298
- GGML_API struct ggml_tensor * ggml_mul_mat(
1299
- struct ggml_context * ctx,
1300
- struct ggml_tensor * a,
1301
- struct ggml_tensor * b);
1302
-
1303
- // change the precision of a matrix multiplication
1304
- // set to GGML_PREC_F32 for higher precision (useful for phi-2)
1305
- GGML_API void ggml_mul_mat_set_prec(
1306
- struct ggml_tensor * a,
1307
- enum ggml_prec prec);
1308
-
1309
- // indirect matrix multiplication
1310
- GGML_API struct ggml_tensor * ggml_mul_mat_id(
1311
- struct ggml_context * ctx,
1312
- struct ggml_tensor * as,
1313
- struct ggml_tensor * b,
1314
- struct ggml_tensor * ids);
1315
-
1316
- // A: m columns, n rows,
1317
- // B: p columns, n rows,
1318
- // result is m columns, p rows
1319
- GGML_API struct ggml_tensor * ggml_out_prod(
1320
- struct ggml_context * ctx,
1321
- struct ggml_tensor * a,
1322
- struct ggml_tensor * b);
1323
-
1324
- //
1325
- // operations on tensors without backpropagation
1326
- //
1327
-
1328
- GGML_API struct ggml_tensor * ggml_scale(
1329
- struct ggml_context * ctx,
1330
- struct ggml_tensor * a,
1331
- float s);
1332
-
1333
- // in-place, returns view(a)
1334
- GGML_API struct ggml_tensor * ggml_scale_inplace(
1335
- struct ggml_context * ctx,
1336
- struct ggml_tensor * a,
1337
- float s);
1338
-
1339
- // x = s * a + b
1340
- GGML_API struct ggml_tensor * ggml_scale_bias(
1341
- struct ggml_context * ctx,
1342
- struct ggml_tensor * a,
1343
- float s,
1344
- float b);
1345
-
1346
- GGML_API struct ggml_tensor * ggml_scale_bias_inplace(
1347
- struct ggml_context * ctx,
1348
- struct ggml_tensor * a,
1349
- float s,
1350
- float b);
1351
-
1352
- // b -> view(a,offset,nb1,nb2,3), return modified a
1353
- GGML_API struct ggml_tensor * ggml_set(
1354
- struct ggml_context * ctx,
1355
- struct ggml_tensor * a,
1356
- struct ggml_tensor * b,
1357
- size_t nb1,
1358
- size_t nb2,
1359
- size_t nb3,
1360
- size_t offset); // in bytes
1361
-
1362
- // b -> view(a,offset,nb1,nb2,3), return view(a)
1363
- GGML_API struct ggml_tensor * ggml_set_inplace(
1364
- struct ggml_context * ctx,
1365
- struct ggml_tensor * a,
1366
- struct ggml_tensor * b,
1367
- size_t nb1,
1368
- size_t nb2,
1369
- size_t nb3,
1370
- size_t offset); // in bytes
1371
-
1372
- GGML_API struct ggml_tensor * ggml_set_1d(
1373
- struct ggml_context * ctx,
1374
- struct ggml_tensor * a,
1375
- struct ggml_tensor * b,
1376
- size_t offset); // in bytes
1377
-
1378
- GGML_API struct ggml_tensor * ggml_set_1d_inplace(
1379
- struct ggml_context * ctx,
1380
- struct ggml_tensor * a,
1381
- struct ggml_tensor * b,
1382
- size_t offset); // in bytes
1383
-
1384
- // b -> view(a,offset,nb1,nb2,3), return modified a
1385
- GGML_API struct ggml_tensor * ggml_set_2d(
1386
- struct ggml_context * ctx,
1387
- struct ggml_tensor * a,
1388
- struct ggml_tensor * b,
1389
- size_t nb1,
1390
- size_t offset); // in bytes
1391
-
1392
- // b -> view(a,offset,nb1,nb2,3), return view(a)
1393
- GGML_API struct ggml_tensor * ggml_set_2d_inplace(
1394
- struct ggml_context * ctx,
1395
- struct ggml_tensor * a,
1396
- struct ggml_tensor * b,
1397
- size_t nb1,
1398
- size_t offset); // in bytes
1399
-
1400
- // a -> b, return view(b)
1401
- GGML_API struct ggml_tensor * ggml_cpy(
1402
- struct ggml_context * ctx,
1403
- struct ggml_tensor * a,
1404
- struct ggml_tensor * b);
1405
-
1406
- GGML_API struct ggml_tensor * ggml_cast(
1407
- struct ggml_context * ctx,
1408
- struct ggml_tensor * a,
1409
- enum ggml_type type);
1410
-
1411
- // make contiguous
1412
- GGML_API struct ggml_tensor * ggml_cont(
1413
- struct ggml_context * ctx,
1414
- struct ggml_tensor * a);
1415
-
1416
- // make contiguous, with new shape
1417
- GGML_API struct ggml_tensor * ggml_cont_1d(
1418
- struct ggml_context * ctx,
1419
- struct ggml_tensor * a,
1420
- int64_t ne0);
1421
-
1422
- GGML_API struct ggml_tensor * ggml_cont_2d(
1423
- struct ggml_context * ctx,
1424
- struct ggml_tensor * a,
1425
- int64_t ne0,
1426
- int64_t ne1);
1427
-
1428
- GGML_API struct ggml_tensor * ggml_cont_3d(
1429
- struct ggml_context * ctx,
1430
- struct ggml_tensor * a,
1431
- int64_t ne0,
1432
- int64_t ne1,
1433
- int64_t ne2);
1434
-
1435
- GGML_API struct ggml_tensor * ggml_cont_4d(
1436
- struct ggml_context * ctx,
1437
- struct ggml_tensor * a,
1438
- int64_t ne0,
1439
- int64_t ne1,
1440
- int64_t ne2,
1441
- int64_t ne3);
1442
-
1443
- // return view(a), b specifies the new shape
1444
- // TODO: when we start computing gradient, make a copy instead of view
1445
- GGML_API struct ggml_tensor * ggml_reshape(
1446
- struct ggml_context * ctx,
1447
- struct ggml_tensor * a,
1448
- struct ggml_tensor * b);
1449
-
1450
- // return view(a)
1451
- // TODO: when we start computing gradient, make a copy instead of view
1452
- GGML_API struct ggml_tensor * ggml_reshape_1d(
1453
- struct ggml_context * ctx,
1454
- struct ggml_tensor * a,
1455
- int64_t ne0);
1456
-
1457
- GGML_API struct ggml_tensor * ggml_reshape_2d(
1458
- struct ggml_context * ctx,
1459
- struct ggml_tensor * a,
1460
- int64_t ne0,
1461
- int64_t ne1);
1462
-
1463
- // return view(a)
1464
- // TODO: when we start computing gradient, make a copy instead of view
1465
- GGML_API struct ggml_tensor * ggml_reshape_3d(
1466
- struct ggml_context * ctx,
1467
- struct ggml_tensor * a,
1468
- int64_t ne0,
1469
- int64_t ne1,
1470
- int64_t ne2);
1471
-
1472
- GGML_API struct ggml_tensor * ggml_reshape_4d(
1473
- struct ggml_context * ctx,
1474
- struct ggml_tensor * a,
1475
- int64_t ne0,
1476
- int64_t ne1,
1477
- int64_t ne2,
1478
- int64_t ne3);
1479
-
1480
- // offset in bytes
1481
- GGML_API struct ggml_tensor * ggml_view_1d(
1482
- struct ggml_context * ctx,
1483
- struct ggml_tensor * a,
1484
- int64_t ne0,
1485
- size_t offset);
1486
-
1487
- GGML_API struct ggml_tensor * ggml_view_2d(
1488
- struct ggml_context * ctx,
1489
- struct ggml_tensor * a,
1490
- int64_t ne0,
1491
- int64_t ne1,
1492
- size_t nb1, // row stride in bytes
1493
- size_t offset);
1494
-
1495
- GGML_API struct ggml_tensor * ggml_view_3d(
1496
- struct ggml_context * ctx,
1497
- struct ggml_tensor * a,
1498
- int64_t ne0,
1499
- int64_t ne1,
1500
- int64_t ne2,
1501
- size_t nb1, // row stride in bytes
1502
- size_t nb2, // slice stride in bytes
1503
- size_t offset);
1504
-
1505
- GGML_API struct ggml_tensor * ggml_view_4d(
1506
- struct ggml_context * ctx,
1507
- struct ggml_tensor * a,
1508
- int64_t ne0,
1509
- int64_t ne1,
1510
- int64_t ne2,
1511
- int64_t ne3,
1512
- size_t nb1, // row stride in bytes
1513
- size_t nb2, // slice stride in bytes
1514
- size_t nb3,
1515
- size_t offset);
1516
-
1517
- GGML_API struct ggml_tensor * ggml_permute(
1518
- struct ggml_context * ctx,
1519
- struct ggml_tensor * a,
1520
- int axis0,
1521
- int axis1,
1522
- int axis2,
1523
- int axis3);
1524
-
1525
- // alias for ggml_permute(ctx, a, 1, 0, 2, 3)
1526
- GGML_API struct ggml_tensor * ggml_transpose(
1527
- struct ggml_context * ctx,
1528
- struct ggml_tensor * a);
1529
-
1530
- // supports 3D: a->ne[2] == b->ne[1]
1531
- GGML_API struct ggml_tensor * ggml_get_rows(
1532
- struct ggml_context * ctx,
1533
- struct ggml_tensor * a, // data
1534
- struct ggml_tensor * b); // row indices
1535
-
1536
- GGML_API struct ggml_tensor * ggml_get_rows_back(
1537
- struct ggml_context * ctx,
1538
- struct ggml_tensor * a, // gradients of ggml_get_rows result
1539
- struct ggml_tensor * b, // row indices
1540
- struct ggml_tensor * c); // data for ggml_get_rows, only used for its shape
1541
-
1542
- // a TD [n_embd, ne1, ne2, ne3]
1543
- // b TS [n_embd, n_rows, ne02, ne03] | ne02 == ne2, ne03 == ne3
1544
- // c I64 [n_rows, ne11, ne12, 1] | c[i] in [0, ne1)
1545
- //
1546
- // undefined behavior if destination rows overlap
1547
- //
1548
- // broadcast:
1549
- // ne2 % ne11 == 0
1550
- // ne3 % ne12 == 0
1551
- //
1552
- // return view(a)
1553
- GGML_API struct ggml_tensor * ggml_set_rows(
1554
- struct ggml_context * ctx,
1555
- struct ggml_tensor * a, // destination
1556
- struct ggml_tensor * b, // source
1557
- struct ggml_tensor * c); // row indices
1558
-
1559
- GGML_API struct ggml_tensor * ggml_diag(
1560
- struct ggml_context * ctx,
1561
- struct ggml_tensor * a);
1562
-
1563
- // set elements above the diagonal to -INF
1564
- GGML_API struct ggml_tensor * ggml_diag_mask_inf(
1565
- struct ggml_context * ctx,
1566
- struct ggml_tensor * a,
1567
- int n_past);
1568
-
1569
- // in-place, returns view(a)
1570
- GGML_API struct ggml_tensor * ggml_diag_mask_inf_inplace(
1571
- struct ggml_context * ctx,
1572
- struct ggml_tensor * a,
1573
- int n_past);
1574
-
1575
- // set elements above the diagonal to 0
1576
- GGML_API struct ggml_tensor * ggml_diag_mask_zero(
1577
- struct ggml_context * ctx,
1578
- struct ggml_tensor * a,
1579
- int n_past);
1580
-
1581
- // in-place, returns view(a)
1582
- GGML_API struct ggml_tensor * ggml_diag_mask_zero_inplace(
1583
- struct ggml_context * ctx,
1584
- struct ggml_tensor * a,
1585
- int n_past);
1586
-
1587
- GGML_API struct ggml_tensor * ggml_soft_max(
1588
- struct ggml_context * ctx,
1589
- struct ggml_tensor * a);
1590
-
1591
- // in-place, returns view(a)
1592
- GGML_API struct ggml_tensor * ggml_soft_max_inplace(
1593
- struct ggml_context * ctx,
1594
- struct ggml_tensor * a);
1595
-
1596
- // a [ne0, ne01, ne02, ne03]
1597
- // mask [ne0, ne11, ne12, ne13] | ne11 >= ne01, F16 or F32, optional
1598
- //
1599
- // broadcast:
1600
- // ne02 % ne12 == 0
1601
- // ne03 % ne13 == 0
1602
- //
1603
- // fused soft_max(a*scale + mask*(ALiBi slope))
1604
- // max_bias = 0.0f for no ALiBi
1605
- GGML_API struct ggml_tensor * ggml_soft_max_ext(
1606
- struct ggml_context * ctx,
1607
- struct ggml_tensor * a,
1608
- struct ggml_tensor * mask,
1609
- float scale,
1610
- float max_bias);
1611
-
1612
- GGML_API void ggml_soft_max_add_sinks(
1613
- struct ggml_tensor * a,
1614
- struct ggml_tensor * sinks);
1615
-
1616
- GGML_API struct ggml_tensor * ggml_soft_max_ext_back(
1617
- struct ggml_context * ctx,
1618
- struct ggml_tensor * a,
1619
- struct ggml_tensor * b,
1620
- float scale,
1621
- float max_bias);
1622
-
1623
- // in-place, returns view(a)
1624
- GGML_API struct ggml_tensor * ggml_soft_max_ext_back_inplace(
1625
- struct ggml_context * ctx,
1626
- struct ggml_tensor * a,
1627
- struct ggml_tensor * b,
1628
- float scale,
1629
- float max_bias);
1630
-
1631
- // rotary position embedding
1632
- // if (mode & 1) - skip n_past elements (NOT SUPPORTED)
1633
- // if (mode & GGML_ROPE_TYPE_NEOX) - GPT-NeoX style
1634
- //
1635
- // b is an int32 vector with size a->ne[2], it contains the positions
1636
- GGML_API struct ggml_tensor * ggml_rope(
1637
- struct ggml_context * ctx,
1638
- struct ggml_tensor * a,
1639
- struct ggml_tensor * b,
1640
- int n_dims,
1641
- int mode);
1642
-
1643
- // in-place, returns view(a)
1644
- GGML_API struct ggml_tensor * ggml_rope_inplace(
1645
- struct ggml_context * ctx,
1646
- struct ggml_tensor * a,
1647
- struct ggml_tensor * b,
1648
- int n_dims,
1649
- int mode);
1650
-
1651
- // custom RoPE
1652
- // c is freq factors (e.g. phi3-128k), (optional)
1653
- GGML_API struct ggml_tensor * ggml_rope_ext(
1654
- struct ggml_context * ctx,
1655
- struct ggml_tensor * a,
1656
- struct ggml_tensor * b,
1657
- struct ggml_tensor * c,
1658
- int n_dims,
1659
- int mode,
1660
- int n_ctx_orig,
1661
- float freq_base,
1662
- float freq_scale,
1663
- float ext_factor,
1664
- float attn_factor,
1665
- float beta_fast,
1666
- float beta_slow);
1667
-
1668
- GGML_API struct ggml_tensor * ggml_rope_multi(
1669
- struct ggml_context * ctx,
1670
- struct ggml_tensor * a,
1671
- struct ggml_tensor * b,
1672
- struct ggml_tensor * c,
1673
- int n_dims,
1674
- int sections[GGML_MROPE_SECTIONS],
1675
- int mode,
1676
- int n_ctx_orig,
1677
- float freq_base,
1678
- float freq_scale,
1679
- float ext_factor,
1680
- float attn_factor,
1681
- float beta_fast,
1682
- float beta_slow);
1683
-
1684
- // in-place, returns view(a)
1685
- GGML_API struct ggml_tensor * ggml_rope_ext_inplace(
1686
- struct ggml_context * ctx,
1687
- struct ggml_tensor * a,
1688
- struct ggml_tensor * b,
1689
- struct ggml_tensor * c,
1690
- int n_dims,
1691
- int mode,
1692
- int n_ctx_orig,
1693
- float freq_base,
1694
- float freq_scale,
1695
- float ext_factor,
1696
- float attn_factor,
1697
- float beta_fast,
1698
- float beta_slow);
1699
-
1700
- GGML_API struct ggml_tensor * ggml_rope_multi_inplace(
1701
- struct ggml_context * ctx,
1702
- struct ggml_tensor * a,
1703
- struct ggml_tensor * b,
1704
- struct ggml_tensor * c,
1705
- int n_dims,
1706
- int sections[GGML_MROPE_SECTIONS],
1707
- int mode,
1708
- int n_ctx_orig,
1709
- float freq_base,
1710
- float freq_scale,
1711
- float ext_factor,
1712
- float attn_factor,
1713
- float beta_fast,
1714
- float beta_slow);
1715
-
1716
- GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_rope_custom(
1717
- struct ggml_context * ctx,
1718
- struct ggml_tensor * a,
1719
- struct ggml_tensor * b,
1720
- int n_dims,
1721
- int mode,
1722
- int n_ctx_orig,
1723
- float freq_base,
1724
- float freq_scale,
1725
- float ext_factor,
1726
- float attn_factor,
1727
- float beta_fast,
1728
- float beta_slow),
1729
- "use ggml_rope_ext instead");
1730
-
1731
- GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_rope_custom_inplace(
1732
- struct ggml_context * ctx,
1733
- struct ggml_tensor * a,
1734
- struct ggml_tensor * b,
1735
- int n_dims,
1736
- int mode,
1737
- int n_ctx_orig,
1738
- float freq_base,
1739
- float freq_scale,
1740
- float ext_factor,
1741
- float attn_factor,
1742
- float beta_fast,
1743
- float beta_slow),
1744
- "use ggml_rope_ext_inplace instead");
1745
-
1746
- // compute correction dims for YaRN RoPE scaling
1747
- GGML_API void ggml_rope_yarn_corr_dims(
1748
- int n_dims, int n_ctx_orig, float freq_base, float beta_fast, float beta_slow, float dims[2]);
1749
-
1750
- // rotary position embedding backward, i.e compute dx from dy
1751
- // a - dy
1752
- GGML_API struct ggml_tensor * ggml_rope_ext_back(
1753
- struct ggml_context * ctx,
1754
- struct ggml_tensor * a, // gradients of ggml_rope result
1755
- struct ggml_tensor * b, // positions
1756
- struct ggml_tensor * c, // freq factors
1757
- int n_dims,
1758
- int mode,
1759
- int n_ctx_orig,
1760
- float freq_base,
1761
- float freq_scale,
1762
- float ext_factor,
1763
- float attn_factor,
1764
- float beta_fast,
1765
- float beta_slow);
1766
-
1767
- GGML_API struct ggml_tensor * ggml_rope_multi_back(
1768
- struct ggml_context * ctx,
1769
- struct ggml_tensor * a,
1770
- struct ggml_tensor * b,
1771
- struct ggml_tensor * c,
1772
- int n_dims,
1773
- int sections[4],
1774
- int mode,
1775
- int n_ctx_orig,
1776
- float freq_base,
1777
- float freq_scale,
1778
- float ext_factor,
1779
- float attn_factor,
1780
- float beta_fast,
1781
- float beta_slow);
1782
-
1783
-
1784
- // clamp
1785
- // in-place, returns view(a)
1786
- GGML_API struct ggml_tensor * ggml_clamp(
1787
- struct ggml_context * ctx,
1788
- struct ggml_tensor * a,
1789
- float min,
1790
- float max);
1791
-
1792
- // im2col
1793
- // converts data into a format that effectively results in a convolution when combined with matrix multiplication
1794
- GGML_API struct ggml_tensor * ggml_im2col(
1795
- struct ggml_context * ctx,
1796
- struct ggml_tensor * a, // convolution kernel
1797
- struct ggml_tensor * b, // data
1798
- int s0, // stride dimension 0
1799
- int s1, // stride dimension 1
1800
- int p0, // padding dimension 0
1801
- int p1, // padding dimension 1
1802
- int d0, // dilation dimension 0
1803
- int d1, // dilation dimension 1
1804
- bool is_2D,
1805
- enum ggml_type dst_type);
1806
-
1807
- GGML_API struct ggml_tensor * ggml_im2col_back(
1808
- struct ggml_context * ctx,
1809
- struct ggml_tensor * a, // convolution kernel
1810
- struct ggml_tensor * b, // gradient of im2col output
1811
- int64_t * ne, // shape of im2col input
1812
- int s0, // stride dimension 0
1813
- int s1, // stride dimension 1
1814
- int p0, // padding dimension 0
1815
- int p1, // padding dimension 1
1816
- int d0, // dilation dimension 0
1817
- int d1, // dilation dimension 1
1818
- bool is_2D);
1819
-
1820
- GGML_API struct ggml_tensor * ggml_conv_1d(
1821
- struct ggml_context * ctx,
1822
- struct ggml_tensor * a, // convolution kernel
1823
- struct ggml_tensor * b, // data
1824
- int s0, // stride
1825
- int p0, // padding
1826
- int d0); // dilation
1827
-
1828
- // conv_1d with padding = half
1829
- // alias for ggml_conv_1d(a, b, s, a->ne[0]/2, d)
1830
- GGML_API struct ggml_tensor* ggml_conv_1d_ph(
1831
- struct ggml_context * ctx,
1832
- struct ggml_tensor * a, // convolution kernel
1833
- struct ggml_tensor * b, // data
1834
- int s, // stride
1835
- int d); // dilation
1836
-
1837
- // depthwise
1838
- // TODO: this is very likely wrong for some cases! - needs more testing
1839
- GGML_API struct ggml_tensor * ggml_conv_1d_dw(
1840
- struct ggml_context * ctx,
1841
- struct ggml_tensor * a, // convolution kernel
1842
- struct ggml_tensor * b, // data
1843
- int s0, // stride
1844
- int p0, // padding
1845
- int d0); // dilation
1846
-
1847
- GGML_API struct ggml_tensor * ggml_conv_1d_dw_ph(
1848
- struct ggml_context * ctx,
1849
- struct ggml_tensor * a, // convolution kernel
1850
- struct ggml_tensor * b, // data
1851
- int s0, // stride
1852
- int d0); // dilation
1853
-
1854
- GGML_API struct ggml_tensor * ggml_conv_transpose_1d(
1855
- struct ggml_context * ctx,
1856
- struct ggml_tensor * a, // convolution kernel
1857
- struct ggml_tensor * b, // data
1858
- int s0, // stride
1859
- int p0, // padding
1860
- int d0); // dilation
1861
-
1862
- GGML_API struct ggml_tensor * ggml_conv_2d(
1863
- struct ggml_context * ctx,
1864
- struct ggml_tensor * a, // convolution kernel
1865
- struct ggml_tensor * b, // data
1866
- int s0, // stride dimension 0
1867
- int s1, // stride dimension 1
1868
- int p0, // padding dimension 0
1869
- int p1, // padding dimension 1
1870
- int d0, // dilation dimension 0
1871
- int d1); // dilation dimension 1
1872
-
1873
- // kernel size is a->ne[0] x a->ne[1]
1874
- // stride is equal to kernel size
1875
- // padding is zero
1876
- // example:
1877
- // a: 16 16 3 768
1878
- // b: 1024 1024 3 1
1879
- // res: 64 64 768 1
1880
- // used in sam
1881
- GGML_API struct ggml_tensor * ggml_conv_2d_sk_p0(
1882
- struct ggml_context * ctx,
1883
- struct ggml_tensor * a,
1884
- struct ggml_tensor * b);
1885
-
1886
- // kernel size is a->ne[0] x a->ne[1]
1887
- // stride is 1
1888
- // padding is half
1889
- // example:
1890
- // a: 3 3 256 256
1891
- // b: 64 64 256 1
1892
- // res: 64 64 256 1
1893
- // used in sam
1894
- GGML_API struct ggml_tensor * ggml_conv_2d_s1_ph(
1895
- struct ggml_context * ctx,
1896
- struct ggml_tensor * a,
1897
- struct ggml_tensor * b);
1898
-
1899
- // depthwise (via im2col and mul_mat)
1900
- GGML_API struct ggml_tensor * ggml_conv_2d_dw(
1901
- struct ggml_context * ctx,
1902
- struct ggml_tensor * a, // convolution kernel
1903
- struct ggml_tensor * b, // data
1904
- int s0, // stride dimension 0
1905
- int s1, // stride dimension 1
1906
- int p0, // padding dimension 0
1907
- int p1, // padding dimension 1
1908
- int d0, // dilation dimension 0
1909
- int d1); // dilation dimension 1
1910
-
1911
- // Depthwise 2D convolution
1912
- // may be faster than ggml_conv_2d_dw, but not available in all backends
1913
- // a: KW KH 1 C convolution kernel
1914
- // b: W H C N input data
1915
- // res: W_out H_out C N
1916
- GGML_API struct ggml_tensor * ggml_conv_2d_dw_direct(
1917
- struct ggml_context * ctx,
1918
- struct ggml_tensor * a,
1919
- struct ggml_tensor * b,
1920
- int stride0,
1921
- int stride1,
1922
- int pad0,
1923
- int pad1,
1924
- int dilation0,
1925
- int dilation1);
1926
-
1927
- GGML_API struct ggml_tensor * ggml_conv_transpose_2d_p0(
1928
- struct ggml_context * ctx,
1929
- struct ggml_tensor * a,
1930
- struct ggml_tensor * b,
1931
- int stride);
1932
-
1933
- GGML_API struct ggml_tensor * ggml_conv_2d_direct(
1934
- struct ggml_context * ctx,
1935
- struct ggml_tensor * a, // convolution kernel [KW, KH, IC, OC]
1936
- struct ggml_tensor * b, // input data [W, H, C, N]
1937
- int s0, // stride dimension 0
1938
- int s1, // stride dimension 1
1939
- int p0, // padding dimension 0
1940
- int p1, // padding dimension 1
1941
- int d0, // dilation dimension 0
1942
- int d1); // dilation dimension 1
1943
-
1944
- GGML_API struct ggml_tensor * ggml_conv_3d(
1945
- struct ggml_context * ctx,
1946
- struct ggml_tensor * a, // kernel [KW, KH, KD, IC * OC]
1947
- struct ggml_tensor * b, // input [W, H, D, C * N]
1948
- int s0, // stride
1949
- int s1,
1950
- int s2,
1951
- int p0, // padding
1952
- int p1,
1953
- int p2,
1954
- int d0, // dilation
1955
- int d1,
1956
- int d2,
1957
- int n_channels,
1958
- int n_batch,
1959
- int n_channels_out);
1960
-
1961
- enum ggml_op_pool {
1962
- GGML_OP_POOL_MAX,
1963
- GGML_OP_POOL_AVG,
1964
- GGML_OP_POOL_COUNT,
1965
- };
1966
-
1967
- GGML_API struct ggml_tensor * ggml_pool_1d(
1968
- struct ggml_context * ctx,
1969
- struct ggml_tensor * a,
1970
- enum ggml_op_pool op,
1971
- int k0, // kernel size
1972
- int s0, // stride
1973
- int p0); // padding
1974
-
1975
- // the result will have 2*p0 padding for the first dimension
1976
- // and 2*p1 padding for the second dimension
1977
- GGML_API struct ggml_tensor * ggml_pool_2d(
1978
- struct ggml_context * ctx,
1979
- struct ggml_tensor * a,
1980
- enum ggml_op_pool op,
1981
- int k0,
1982
- int k1,
1983
- int s0,
1984
- int s1,
1985
- float p0,
1986
- float p1);
1987
-
1988
- GGML_API struct ggml_tensor * ggml_pool_2d_back(
1989
- struct ggml_context * ctx,
1990
- struct ggml_tensor * a,
1991
- struct ggml_tensor * af, // "a"/input used in forward pass
1992
- enum ggml_op_pool op,
1993
- int k0,
1994
- int k1,
1995
- int s0,
1996
- int s1,
1997
- float p0,
1998
- float p1);
1999
-
2000
- enum ggml_scale_mode {
2001
- GGML_SCALE_MODE_NEAREST = 0,
2002
- GGML_SCALE_MODE_BILINEAR = 1,
2003
-
2004
- GGML_SCALE_MODE_COUNT
2005
- };
2006
-
2007
- enum ggml_scale_flag {
2008
- GGML_SCALE_FLAG_ALIGN_CORNERS = (1 << 8)
2009
- };
2010
-
2011
- // interpolate
2012
- // multiplies ne0 and ne1 by scale factor
2013
- GGML_API struct ggml_tensor * ggml_upscale(
2014
- struct ggml_context * ctx,
2015
- struct ggml_tensor * a,
2016
- int scale_factor,
2017
- enum ggml_scale_mode mode);
2018
-
2019
- // interpolate
2020
- // interpolate scale to specified dimensions
2021
- GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_upscale_ext(
2022
- struct ggml_context * ctx,
2023
- struct ggml_tensor * a,
2024
- int ne0,
2025
- int ne1,
2026
- int ne2,
2027
- int ne3,
2028
- enum ggml_scale_mode mode),
2029
- "use ggml_interpolate instead");
2030
-
2031
- // Up- or downsamples the input to the specified size.
2032
- // 2D scale modes (eg. bilinear) are applied to the first two dimensions.
2033
- GGML_API struct ggml_tensor * ggml_interpolate(
2034
- struct ggml_context * ctx,
2035
- struct ggml_tensor * a,
2036
- int64_t ne0,
2037
- int64_t ne1,
2038
- int64_t ne2,
2039
- int64_t ne3,
2040
- uint32_t mode); // ggml_scale_mode [ | ggml_scale_flag...]
2041
-
2042
- // pad each dimension with zeros: [x, ..., x] -> [x, ..., x, 0, ..., 0]
2043
- GGML_API struct ggml_tensor * ggml_pad(
2044
- struct ggml_context * ctx,
2045
- struct ggml_tensor * a,
2046
- int p0,
2047
- int p1,
2048
- int p2,
2049
- int p3);
2050
-
2051
- // pad each dimension with reflection: [a, b, c, d] -> [b, a, b, c, d, c]
2052
- GGML_API struct ggml_tensor * ggml_pad_reflect_1d(
2053
- struct ggml_context * ctx,
2054
- struct ggml_tensor * a,
2055
- int p0,
2056
- int p1);
2057
-
2058
- // Move tensor elements by an offset given for each dimension. Elements that
2059
- // are shifted beyond the last position are wrapped around to the beginning.
2060
- GGML_API struct ggml_tensor * ggml_roll(
2061
- struct ggml_context * ctx,
2062
- struct ggml_tensor * a,
2063
- int shift0,
2064
- int shift1,
2065
- int shift2,
2066
- int shift3);
2067
-
2068
-
2069
- // Ref: https://github.com/CompVis/stable-diffusion/blob/main/ldm/modules/diffusionmodules/util.py#L151
2070
- // timesteps: [N,]
2071
- // return: [N, dim]
2072
- GGML_API struct ggml_tensor * ggml_timestep_embedding(
2073
- struct ggml_context * ctx,
2074
- struct ggml_tensor * timesteps,
2075
- int dim,
2076
- int max_period);
2077
-
2078
- // sort rows
2079
- enum ggml_sort_order {
2080
- GGML_SORT_ORDER_ASC,
2081
- GGML_SORT_ORDER_DESC,
2082
- };
2083
-
2084
- GGML_API struct ggml_tensor * ggml_argsort(
2085
- struct ggml_context * ctx,
2086
- struct ggml_tensor * a,
2087
- enum ggml_sort_order order);
2088
-
2089
- GGML_API struct ggml_tensor * ggml_arange(
2090
- struct ggml_context * ctx,
2091
- float start,
2092
- float stop,
2093
- float step);
2094
-
2095
- // top k elements per row
2096
- GGML_API struct ggml_tensor * ggml_top_k(
2097
- struct ggml_context * ctx,
2098
- struct ggml_tensor * a,
2099
- int k);
2100
-
2101
- #define GGML_KQ_MASK_PAD 64
2102
-
2103
- // q: [n_embd_k, n_batch, n_head, ne3 ]
2104
- // k: [n_embd_k, n_kv, n_head_kv, ne3 ]
2105
- // v: [n_embd_v, n_kv, n_head_kv, ne3 ] !! not transposed !!
2106
- // mask: [n_kv, n_batch_pad, ne32, ne33] !! n_batch_pad = GGML_PAD(n_batch, GGML_KQ_MASK_PAD) !!
2107
- // res: [n_embd_v, n_head, n_batch, ne3 ] !! permuted !!
2108
- //
2109
- // broadcast:
2110
- // n_head % n_head_kv == 0
2111
- // n_head % ne32 == 0
2112
- // ne3 % ne33 == 0
2113
- //
2114
- GGML_API struct ggml_tensor * ggml_flash_attn_ext(
2115
- struct ggml_context * ctx,
2116
- struct ggml_tensor * q,
2117
- struct ggml_tensor * k,
2118
- struct ggml_tensor * v,
2119
- struct ggml_tensor * mask,
2120
- float scale,
2121
- float max_bias,
2122
- float logit_softcap);
2123
-
2124
- GGML_API void ggml_flash_attn_ext_set_prec(
2125
- struct ggml_tensor * a,
2126
- enum ggml_prec prec);
2127
-
2128
- GGML_API enum ggml_prec ggml_flash_attn_ext_get_prec(
2129
- const struct ggml_tensor * a);
2130
-
2131
- GGML_API void ggml_flash_attn_ext_add_sinks(
2132
- struct ggml_tensor * a,
2133
- struct ggml_tensor * sinks);
2134
-
2135
- // TODO: needs to be adapted to ggml_flash_attn_ext
2136
- GGML_API struct ggml_tensor * ggml_flash_attn_back(
2137
- struct ggml_context * ctx,
2138
- struct ggml_tensor * q,
2139
- struct ggml_tensor * k,
2140
- struct ggml_tensor * v,
2141
- struct ggml_tensor * d,
2142
- bool masked);
2143
-
2144
- GGML_API struct ggml_tensor * ggml_ssm_conv(
2145
- struct ggml_context * ctx,
2146
- struct ggml_tensor * sx,
2147
- struct ggml_tensor * c);
2148
-
2149
- GGML_API struct ggml_tensor * ggml_ssm_scan(
2150
- struct ggml_context * ctx,
2151
- struct ggml_tensor * s,
2152
- struct ggml_tensor * x,
2153
- struct ggml_tensor * dt,
2154
- struct ggml_tensor * A,
2155
- struct ggml_tensor * B,
2156
- struct ggml_tensor * C,
2157
- struct ggml_tensor * ids);
2158
-
2159
- // partition into non-overlapping windows with padding if needed
2160
- // example:
2161
- // a: 768 64 64 1
2162
- // w: 14
2163
- // res: 768 14 14 25
2164
- // used in sam
2165
- GGML_API struct ggml_tensor * ggml_win_part(
2166
- struct ggml_context * ctx,
2167
- struct ggml_tensor * a,
2168
- int w);
2169
-
2170
- // reverse of ggml_win_part
2171
- // used in sam
2172
- GGML_API struct ggml_tensor * ggml_win_unpart(
2173
- struct ggml_context * ctx,
2174
- struct ggml_tensor * a,
2175
- int w0,
2176
- int h0,
2177
- int w);
2178
-
2179
- GGML_API struct ggml_tensor * ggml_unary(
2180
- struct ggml_context * ctx,
2181
- struct ggml_tensor * a,
2182
- enum ggml_unary_op op);
2183
-
2184
- GGML_API struct ggml_tensor * ggml_unary_inplace(
2185
- struct ggml_context * ctx,
2186
- struct ggml_tensor * a,
2187
- enum ggml_unary_op op);
2188
-
2189
- // used in sam
2190
- GGML_API struct ggml_tensor * ggml_get_rel_pos(
2191
- struct ggml_context * ctx,
2192
- struct ggml_tensor * a,
2193
- int qh,
2194
- int kh);
2195
-
2196
- // used in sam
2197
- GGML_API struct ggml_tensor * ggml_add_rel_pos(
2198
- struct ggml_context * ctx,
2199
- struct ggml_tensor * a,
2200
- struct ggml_tensor * pw,
2201
- struct ggml_tensor * ph);
2202
-
2203
- GGML_API struct ggml_tensor * ggml_add_rel_pos_inplace(
2204
- struct ggml_context * ctx,
2205
- struct ggml_tensor * a,
2206
- struct ggml_tensor * pw,
2207
- struct ggml_tensor * ph);
2208
-
2209
- GGML_API struct ggml_tensor * ggml_rwkv_wkv6(
2210
- struct ggml_context * ctx,
2211
- struct ggml_tensor * k,
2212
- struct ggml_tensor * v,
2213
- struct ggml_tensor * r,
2214
- struct ggml_tensor * tf,
2215
- struct ggml_tensor * td,
2216
- struct ggml_tensor * state);
2217
-
2218
- GGML_API struct ggml_tensor * ggml_gated_linear_attn(
2219
- struct ggml_context * ctx,
2220
- struct ggml_tensor * k,
2221
- struct ggml_tensor * v,
2222
- struct ggml_tensor * q,
2223
- struct ggml_tensor * g,
2224
- struct ggml_tensor * state,
2225
- float scale);
2226
-
2227
- GGML_API struct ggml_tensor * ggml_rwkv_wkv7(
2228
- struct ggml_context * ctx,
2229
- struct ggml_tensor * r,
2230
- struct ggml_tensor * w,
2231
- struct ggml_tensor * k,
2232
- struct ggml_tensor * v,
2233
- struct ggml_tensor * a,
2234
- struct ggml_tensor * b,
2235
- struct ggml_tensor * state);
2236
-
2237
- // custom operators
2238
-
2239
- typedef void (*ggml_custom1_op_t)(struct ggml_tensor * dst , const struct ggml_tensor * a, int ith, int nth, void * userdata);
2240
- typedef void (*ggml_custom2_op_t)(struct ggml_tensor * dst , const struct ggml_tensor * a, const struct ggml_tensor * b, int ith, int nth, void * userdata);
2241
- typedef void (*ggml_custom3_op_t)(struct ggml_tensor * dst , const struct ggml_tensor * a, const struct ggml_tensor * b, const struct ggml_tensor * c, int ith, int nth, void * userdata);
2242
-
2243
- #define GGML_N_TASKS_MAX (-1)
2244
- // n_tasks == GGML_N_TASKS_MAX means to use max number of tasks
2245
-
2246
- GGML_API struct ggml_tensor * ggml_map_custom1(
2247
- struct ggml_context * ctx,
2248
- struct ggml_tensor * a,
2249
- ggml_custom1_op_t fun,
2250
- int n_tasks,
2251
- void * userdata);
2252
-
2253
- GGML_API struct ggml_tensor * ggml_map_custom1_inplace(
2254
- struct ggml_context * ctx,
2255
- struct ggml_tensor * a,
2256
- ggml_custom1_op_t fun,
2257
- int n_tasks,
2258
- void * userdata);
2259
-
2260
- GGML_API struct ggml_tensor * ggml_map_custom2(
2261
- struct ggml_context * ctx,
2262
- struct ggml_tensor * a,
2263
- struct ggml_tensor * b,
2264
- ggml_custom2_op_t fun,
2265
- int n_tasks,
2266
- void * userdata);
2267
-
2268
- GGML_API struct ggml_tensor * ggml_map_custom2_inplace(
2269
- struct ggml_context * ctx,
2270
- struct ggml_tensor * a,
2271
- struct ggml_tensor * b,
2272
- ggml_custom2_op_t fun,
2273
- int n_tasks,
2274
- void * userdata);
2275
-
2276
- GGML_API struct ggml_tensor * ggml_map_custom3(
2277
- struct ggml_context * ctx,
2278
- struct ggml_tensor * a,
2279
- struct ggml_tensor * b,
2280
- struct ggml_tensor * c,
2281
- ggml_custom3_op_t fun,
2282
- int n_tasks,
2283
- void * userdata);
2284
-
2285
- GGML_API struct ggml_tensor * ggml_map_custom3_inplace(
2286
- struct ggml_context * ctx,
2287
- struct ggml_tensor * a,
2288
- struct ggml_tensor * b,
2289
- struct ggml_tensor * c,
2290
- ggml_custom3_op_t fun,
2291
- int n_tasks,
2292
- void * userdata);
2293
-
2294
- typedef void (*ggml_custom_op_t)(struct ggml_tensor * dst , int ith, int nth, void * userdata);
2295
-
2296
- GGML_API struct ggml_tensor * ggml_custom_4d(
2297
- struct ggml_context * ctx,
2298
- enum ggml_type type,
2299
- int64_t ne0,
2300
- int64_t ne1,
2301
- int64_t ne2,
2302
- int64_t ne3,
2303
- struct ggml_tensor ** args,
2304
- int n_args,
2305
- ggml_custom_op_t fun,
2306
- int n_tasks,
2307
- void * userdata);
2308
-
2309
- GGML_API struct ggml_tensor * ggml_custom_inplace(
2310
- struct ggml_context * ctx,
2311
- struct ggml_tensor * a,
2312
- struct ggml_tensor ** args,
2313
- int n_args,
2314
- ggml_custom_op_t fun,
2315
- int n_tasks,
2316
- void * userdata);
2317
-
2318
- // loss function
2319
-
2320
- GGML_API struct ggml_tensor * ggml_cross_entropy_loss(
2321
- struct ggml_context * ctx,
2322
- struct ggml_tensor * a, // logits
2323
- struct ggml_tensor * b); // labels
2324
-
2325
- GGML_API struct ggml_tensor * ggml_cross_entropy_loss_back(
2326
- struct ggml_context * ctx,
2327
- struct ggml_tensor * a, // logits
2328
- struct ggml_tensor * b, // labels
2329
- struct ggml_tensor * c); // gradients of cross_entropy_loss result
2330
-
2331
- // AdamW optimizer step
2332
- // Paper: https://arxiv.org/pdf/1711.05101v3.pdf
2333
- // PyTorch: https://pytorch.org/docs/stable/generated/torch.optim.AdamW.html
2334
- GGML_API struct ggml_tensor * ggml_opt_step_adamw(
2335
- struct ggml_context * ctx,
2336
- struct ggml_tensor * a,
2337
- struct ggml_tensor * grad,
2338
- struct ggml_tensor * m,
2339
- struct ggml_tensor * v,
2340
- struct ggml_tensor * adamw_params); // parameters such as the learning rate
2341
-
2342
- // stochastic gradient descent step (with weight decay)
2343
- GGML_API struct ggml_tensor * ggml_opt_step_sgd(
2344
- struct ggml_context * ctx,
2345
- struct ggml_tensor * a,
2346
- struct ggml_tensor * grad,
2347
- struct ggml_tensor * sgd_params); // alpha, weight decay
2348
-
2349
- //
2350
- // automatic differentiation
2351
- //
2352
-
2353
- GGML_API void ggml_build_forward_expand(struct ggml_cgraph * cgraph, struct ggml_tensor * tensor);
2354
- GGML_API void ggml_build_backward_expand(
2355
- struct ggml_context * ctx, // context for gradient computation
2356
- struct ggml_cgraph * cgraph,
2357
- struct ggml_tensor ** grad_accs);
2358
-
2359
- // graph allocation in a context
2360
- GGML_API struct ggml_cgraph * ggml_new_graph (struct ggml_context * ctx); // size = GGML_DEFAULT_GRAPH_SIZE, grads = false
2361
- GGML_API struct ggml_cgraph * ggml_new_graph_custom(struct ggml_context * ctx, size_t size, bool grads);
2362
- GGML_API struct ggml_cgraph * ggml_graph_dup (struct ggml_context * ctx, struct ggml_cgraph * cgraph, bool force_grads);
2363
- GGML_API void ggml_graph_cpy (struct ggml_cgraph * src, struct ggml_cgraph * dst);
2364
- GGML_API void ggml_graph_reset (struct ggml_cgraph * cgraph); // set regular grads + optimizer momenta to 0, set loss grad to 1
2365
- GGML_API void ggml_graph_clear (struct ggml_cgraph * cgraph);
2366
-
2367
- GGML_API int ggml_graph_size (struct ggml_cgraph * cgraph);
2368
- GGML_API struct ggml_tensor * ggml_graph_node (struct ggml_cgraph * cgraph, int i); // if i < 0, returns nodes[n_nodes + i]
2369
- GGML_API struct ggml_tensor ** ggml_graph_nodes (struct ggml_cgraph * cgraph);
2370
- GGML_API int ggml_graph_n_nodes(struct ggml_cgraph * cgraph);
2371
-
2372
- GGML_API void ggml_graph_add_node(struct ggml_cgraph * cgraph, struct ggml_tensor * tensor);
2373
-
2374
- GGML_API size_t ggml_graph_overhead(void);
2375
- GGML_API size_t ggml_graph_overhead_custom(size_t size, bool grads);
2376
-
2377
- GGML_API struct ggml_tensor * ggml_graph_get_tensor (const struct ggml_cgraph * cgraph, const char * name);
2378
- GGML_API struct ggml_tensor * ggml_graph_get_grad (const struct ggml_cgraph * cgraph, const struct ggml_tensor * node);
2379
- GGML_API struct ggml_tensor * ggml_graph_get_grad_acc(const struct ggml_cgraph * cgraph, const struct ggml_tensor * node);
2380
-
2381
- // print info and performance information for the graph
2382
- GGML_API void ggml_graph_print(const struct ggml_cgraph * cgraph);
2383
-
2384
- // dump the graph into a file using the dot format
2385
- GGML_API void ggml_graph_dump_dot(const struct ggml_cgraph * gb, const struct ggml_cgraph * gf, const char * filename);
2386
-
2387
- // TODO these functions were sandwiched in the old optimization interface, is there a better place for them?
2388
- typedef void (*ggml_log_callback)(enum ggml_log_level level, const char * text, void * user_data);
2389
-
2390
- // Set callback for all future logging events.
2391
- // If this is not called, or NULL is supplied, everything is output on stderr.
2392
- GGML_API void ggml_log_set(ggml_log_callback log_callback, void * user_data);
2393
-
2394
- GGML_API struct ggml_tensor * ggml_set_zero(struct ggml_tensor * tensor);
2395
-
2396
- //
2397
- // quantization
2398
- //
2399
-
2400
- // - ggml_quantize_init can be called multiple times with the same type
2401
- // it will only initialize the quantization tables for the first call or after ggml_quantize_free
2402
- // automatically called by ggml_quantize_chunk for convenience
2403
- //
2404
- // - ggml_quantize_free will free any memory allocated by ggml_quantize_init
2405
- // call this at the end of the program to avoid memory leaks
2406
- //
2407
- // note: these are thread-safe
2408
- //
2409
- GGML_API void ggml_quantize_init(enum ggml_type type);
2410
- GGML_API void ggml_quantize_free(void);
2411
-
2412
- // some quantization type cannot be used without an importance matrix
2413
- GGML_API bool ggml_quantize_requires_imatrix(enum ggml_type type);
2414
-
2415
- // calls ggml_quantize_init internally (i.e. can allocate memory)
2416
- GGML_API size_t ggml_quantize_chunk(
2417
- enum ggml_type type,
2418
- const float * src,
2419
- void * dst,
2420
- int64_t start,
2421
- int64_t nrows,
2422
- int64_t n_per_row,
2423
- const float * imatrix);
2424
-
2425
- #ifdef __cplusplus
2426
- // restrict not standard in C++
2427
- # if defined(__GNUC__)
2428
- # define GGML_RESTRICT __restrict__
2429
- # elif defined(__clang__)
2430
- # define GGML_RESTRICT __restrict
2431
- # elif defined(_MSC_VER)
2432
- # define GGML_RESTRICT __restrict
2433
- # else
2434
- # define GGML_RESTRICT
2435
- # endif
2436
- #else
2437
- # if defined (_MSC_VER) && (__STDC_VERSION__ < 201112L)
2438
- # define GGML_RESTRICT __restrict
2439
- # else
2440
- # define GGML_RESTRICT restrict
2441
- # endif
2442
- #endif
2443
- typedef void (*ggml_to_float_t) (const void * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
2444
- typedef void (*ggml_from_float_t)(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
2445
-
2446
- struct ggml_type_traits {
2447
- const char * type_name;
2448
- int64_t blck_size;
2449
- int64_t blck_size_interleave; // interleave elements in blocks
2450
- size_t type_size;
2451
- bool is_quantized;
2452
- ggml_to_float_t to_float;
2453
- ggml_from_float_t from_float_ref;
2454
- };
2455
-
2456
- GGML_API const struct ggml_type_traits * ggml_get_type_traits(enum ggml_type type);
2457
-
2458
- // ggml threadpool
2459
- // TODO: currently, only a few functions are in the base ggml API, while the rest are in the CPU backend
2460
- // the goal should be to create an API that other backends can use move everything to the ggml base
2461
-
2462
- // scheduling priorities
2463
- enum ggml_sched_priority {
2464
- GGML_SCHED_PRIO_LOW = -1,
2465
- GGML_SCHED_PRIO_NORMAL,
2466
- GGML_SCHED_PRIO_MEDIUM,
2467
- GGML_SCHED_PRIO_HIGH,
2468
- GGML_SCHED_PRIO_REALTIME
2469
- };
2470
-
2471
- // threadpool params
2472
- // Use ggml_threadpool_params_default() or ggml_threadpool_params_init() to populate the defaults
2473
- struct ggml_threadpool_params {
2474
- bool cpumask[GGML_MAX_N_THREADS]; // mask of cpu cores (all-zeros means use default affinity settings)
2475
- int n_threads; // number of threads
2476
- enum ggml_sched_priority prio; // thread priority
2477
- uint32_t poll; // polling level (0 - no polling, 100 - aggressive polling)
2478
- bool strict_cpu; // strict cpu placement
2479
- bool paused; // start in paused state
2480
- };
2481
-
2482
- struct ggml_threadpool; // forward declaration, see ggml.c
2483
-
2484
- typedef struct ggml_threadpool * ggml_threadpool_t;
2485
-
2486
- GGML_API struct ggml_threadpool_params ggml_threadpool_params_default(int n_threads);
2487
- GGML_API void ggml_threadpool_params_init (struct ggml_threadpool_params * p, int n_threads);
2488
- GGML_API bool ggml_threadpool_params_match (const struct ggml_threadpool_params * p0, const struct ggml_threadpool_params * p1);
2489
-
2490
- #ifdef __cplusplus
2491
- }
2492
- #endif