llama_cpp 0.16.2 → 0.17.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (177) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +18 -0
  3. data/README.md +7 -12
  4. data/ext/llama_cpp/extconf.rb +2 -43
  5. data/ext/llama_cpp/llama_cpp.cpp +8 -0
  6. data/lib/llama_cpp/version.rb +3 -3
  7. data/sig/llama_cpp.rbs +3 -0
  8. metadata +2 -171
  9. data/vendor/include/.gitkeep +0 -0
  10. data/vendor/lib/.gitkeep +0 -0
  11. data/vendor/tmp/llama.cpp/LICENSE +0 -21
  12. data/vendor/tmp/llama.cpp/Makefile +0 -1124
  13. data/vendor/tmp/llama.cpp/ggml-alloc.c +0 -1041
  14. data/vendor/tmp/llama.cpp/ggml-alloc.h +0 -76
  15. data/vendor/tmp/llama.cpp/ggml-backend-impl.h +0 -153
  16. data/vendor/tmp/llama.cpp/ggml-backend.c +0 -2225
  17. data/vendor/tmp/llama.cpp/ggml-backend.h +0 -236
  18. data/vendor/tmp/llama.cpp/ggml-blas.cpp +0 -363
  19. data/vendor/tmp/llama.cpp/ggml-blas.h +0 -23
  20. data/vendor/tmp/llama.cpp/ggml-common.h +0 -1805
  21. data/vendor/tmp/llama.cpp/ggml-cuda/acc.cu +0 -47
  22. data/vendor/tmp/llama.cpp/ggml-cuda/arange.cu +0 -34
  23. data/vendor/tmp/llama.cpp/ggml-cuda/argsort.cu +0 -104
  24. data/vendor/tmp/llama.cpp/ggml-cuda/binbcast.cu +0 -280
  25. data/vendor/tmp/llama.cpp/ggml-cuda/clamp.cu +0 -34
  26. data/vendor/tmp/llama.cpp/ggml-cuda/concat.cu +0 -196
  27. data/vendor/tmp/llama.cpp/ggml-cuda/convert.cu +0 -686
  28. data/vendor/tmp/llama.cpp/ggml-cuda/cpy.cu +0 -490
  29. data/vendor/tmp/llama.cpp/ggml-cuda/diagmask.cu +0 -40
  30. data/vendor/tmp/llama.cpp/ggml-cuda/dmmv.cu +0 -674
  31. data/vendor/tmp/llama.cpp/ggml-cuda/fattn-tile-f16.cu +0 -319
  32. data/vendor/tmp/llama.cpp/ggml-cuda/fattn-tile-f32.cu +0 -312
  33. data/vendor/tmp/llama.cpp/ggml-cuda/fattn.cu +0 -345
  34. data/vendor/tmp/llama.cpp/ggml-cuda/getrows.cu +0 -178
  35. data/vendor/tmp/llama.cpp/ggml-cuda/im2col.cu +0 -104
  36. data/vendor/tmp/llama.cpp/ggml-cuda/mmq.cu +0 -88
  37. data/vendor/tmp/llama.cpp/ggml-cuda/mmvq.cu +0 -419
  38. data/vendor/tmp/llama.cpp/ggml-cuda/norm.cu +0 -221
  39. data/vendor/tmp/llama.cpp/ggml-cuda/pad.cu +0 -49
  40. data/vendor/tmp/llama.cpp/ggml-cuda/pool2d.cu +0 -94
  41. data/vendor/tmp/llama.cpp/ggml-cuda/quantize.cu +0 -112
  42. data/vendor/tmp/llama.cpp/ggml-cuda/rope.cu +0 -271
  43. data/vendor/tmp/llama.cpp/ggml-cuda/scale.cu +0 -31
  44. data/vendor/tmp/llama.cpp/ggml-cuda/softmax.cu +0 -206
  45. data/vendor/tmp/llama.cpp/ggml-cuda/sumrows.cu +0 -40
  46. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-f16.cu +0 -5
  47. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q4_0.cu +0 -5
  48. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q4_1.cu +0 -5
  49. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q5_0.cu +0 -5
  50. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q5_1.cu +0 -5
  51. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q8_0.cu +0 -5
  52. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-f16.cu +0 -5
  53. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q4_0.cu +0 -5
  54. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q4_1.cu +0 -5
  55. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q5_0.cu +0 -5
  56. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q5_1.cu +0 -5
  57. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q8_0.cu +0 -5
  58. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-f16.cu +0 -5
  59. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q4_0.cu +0 -5
  60. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q4_1.cu +0 -5
  61. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q5_0.cu +0 -5
  62. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q5_1.cu +0 -5
  63. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q8_0.cu +0 -5
  64. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-f16.cu +0 -5
  65. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q4_0.cu +0 -5
  66. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q4_1.cu +0 -5
  67. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q5_0.cu +0 -5
  68. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q5_1.cu +0 -5
  69. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q8_0.cu +0 -5
  70. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-f16.cu +0 -5
  71. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q4_0.cu +0 -5
  72. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q4_1.cu +0 -5
  73. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q5_0.cu +0 -5
  74. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q5_1.cu +0 -5
  75. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q8_0.cu +0 -5
  76. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-f16.cu +0 -5
  77. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q4_0.cu +0 -5
  78. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q4_1.cu +0 -5
  79. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q5_0.cu +0 -5
  80. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q5_1.cu +0 -5
  81. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q8_0.cu +0 -5
  82. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs256-f16-f16.cu +0 -5
  83. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-f16.cu +0 -5
  84. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q4_0.cu +0 -5
  85. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q4_1.cu +0 -5
  86. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q5_0.cu +0 -5
  87. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q5_1.cu +0 -5
  88. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q8_0.cu +0 -5
  89. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-f16.cu +0 -5
  90. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q4_0.cu +0 -5
  91. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q4_1.cu +0 -5
  92. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q5_0.cu +0 -5
  93. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q5_1.cu +0 -5
  94. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q8_0.cu +0 -5
  95. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-f16.cu +0 -5
  96. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q4_0.cu +0 -5
  97. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q4_1.cu +0 -5
  98. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q5_0.cu +0 -5
  99. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q5_1.cu +0 -5
  100. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q8_0.cu +0 -5
  101. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-f16.cu +0 -5
  102. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q4_0.cu +0 -5
  103. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q4_1.cu +0 -5
  104. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q5_0.cu +0 -5
  105. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q5_1.cu +0 -5
  106. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q8_0.cu +0 -5
  107. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-f16.cu +0 -5
  108. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q4_0.cu +0 -5
  109. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q4_1.cu +0 -5
  110. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q5_0.cu +0 -5
  111. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q5_1.cu +0 -5
  112. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q8_0.cu +0 -5
  113. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-f16.cu +0 -5
  114. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q4_0.cu +0 -5
  115. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q4_1.cu +0 -5
  116. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q5_0.cu +0 -5
  117. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q5_1.cu +0 -5
  118. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q8_0.cu +0 -5
  119. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-f16.cu +0 -5
  120. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q4_0.cu +0 -5
  121. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q4_1.cu +0 -5
  122. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q5_0.cu +0 -5
  123. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q5_1.cu +0 -5
  124. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q8_0.cu +0 -5
  125. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs256-f16-f16.cu +0 -5
  126. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-f16.cu +0 -5
  127. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q4_0.cu +0 -5
  128. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q4_1.cu +0 -5
  129. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q5_0.cu +0 -5
  130. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q5_1.cu +0 -5
  131. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q8_0.cu +0 -5
  132. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqfloat-cpb16.cu +0 -10
  133. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqfloat-cpb32.cu +0 -9
  134. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqhalf-cpb16.cu +0 -10
  135. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqhalf-cpb32.cu +0 -10
  136. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqhalf-cpb8.cu +0 -8
  137. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q2_k.cu +0 -5
  138. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q3_k.cu +0 -5
  139. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q4_0.cu +0 -5
  140. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q4_1.cu +0 -5
  141. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q4_k.cu +0 -5
  142. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q5_0.cu +0 -5
  143. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q5_1.cu +0 -5
  144. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q5_k.cu +0 -5
  145. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q6_k.cu +0 -5
  146. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q8_0.cu +0 -5
  147. data/vendor/tmp/llama.cpp/ggml-cuda/tsembd.cu +0 -47
  148. data/vendor/tmp/llama.cpp/ggml-cuda/unary.cu +0 -314
  149. data/vendor/tmp/llama.cpp/ggml-cuda/upscale.cu +0 -51
  150. data/vendor/tmp/llama.cpp/ggml-cuda.cu +0 -3069
  151. data/vendor/tmp/llama.cpp/ggml-cuda.h +0 -44
  152. data/vendor/tmp/llama.cpp/ggml-impl.h +0 -651
  153. data/vendor/tmp/llama.cpp/ggml-kompute.cpp +0 -2038
  154. data/vendor/tmp/llama.cpp/ggml-kompute.h +0 -46
  155. data/vendor/tmp/llama.cpp/ggml-metal.h +0 -66
  156. data/vendor/tmp/llama.cpp/ggml-metal.m +0 -3273
  157. data/vendor/tmp/llama.cpp/ggml-metal.metal +0 -6540
  158. data/vendor/tmp/llama.cpp/ggml-quants.c +0 -14994
  159. data/vendor/tmp/llama.cpp/ggml-quants.h +0 -133
  160. data/vendor/tmp/llama.cpp/ggml-rpc.cpp +0 -1178
  161. data/vendor/tmp/llama.cpp/ggml-rpc.h +0 -24
  162. data/vendor/tmp/llama.cpp/ggml-sycl.cpp +0 -6351
  163. data/vendor/tmp/llama.cpp/ggml-sycl.h +0 -40
  164. data/vendor/tmp/llama.cpp/ggml-vulkan-shaders.hpp +0 -144508
  165. data/vendor/tmp/llama.cpp/ggml-vulkan.cpp +0 -7183
  166. data/vendor/tmp/llama.cpp/ggml-vulkan.h +0 -29
  167. data/vendor/tmp/llama.cpp/ggml.c +0 -22506
  168. data/vendor/tmp/llama.cpp/ggml.h +0 -2458
  169. data/vendor/tmp/llama.cpp/llama.cpp +0 -18985
  170. data/vendor/tmp/llama.cpp/llama.h +0 -1147
  171. data/vendor/tmp/llama.cpp/scripts/get-flags.mk +0 -38
  172. data/vendor/tmp/llama.cpp/sgemm.cpp +0 -1032
  173. data/vendor/tmp/llama.cpp/sgemm.h +0 -14
  174. data/vendor/tmp/llama.cpp/unicode-data.cpp +0 -7033
  175. data/vendor/tmp/llama.cpp/unicode-data.h +0 -20
  176. data/vendor/tmp/llama.cpp/unicode.cpp +0 -810
  177. data/vendor/tmp/llama.cpp/unicode.h +0 -63
data/vendor/tmp/llama.cpp/ggml.h DELETED
@@ -1,2458 +0,0 @@
1
- #pragma once
2
-
3
- //
4
- // GGML Tensor Library
5
- //
6
- // This documentation is still a work in progress.
7
- // If you wish some specific topics to be covered, feel free to drop a comment:
8
- //
9
- // https://github.com/ggerganov/whisper.cpp/issues/40
10
- //
11
- // ## Overview
12
- //
13
- // This library implements:
14
- //
15
- // - a set of tensor operations
16
- // - automatic differentiation
17
- // - basic optimization algorithms
18
- //
19
- // The aim of this library is to provide a minimalistic approach for various machine learning tasks. This includes,
20
- // but is not limited to, the following:
21
- //
22
- // - linear regression
23
- // - support vector machines
24
- // - neural networks
25
- //
26
- // The library allows the user to define a certain function using the available tensor operations. This function
27
- // definition is represented internally via a computation graph. Each tensor operation in the function definition
28
- // corresponds to a node in the graph. Having the computation graph defined, the user can choose to compute the
29
- // function's value and/or its gradient with respect to the input variables. Optionally, the function can be optimized
30
- // using one of the available optimization algorithms.
31
- //
32
- // For example, here we define the function: f(x) = a*x^2 + b
33
- //
34
- // {
35
- // struct ggml_init_params params = {
36
- // .mem_size = 16*1024*1024,
37
- // .mem_buffer = NULL,
38
- // };
39
- //
40
- // // memory allocation happens here
41
- // struct ggml_context * ctx = ggml_init(params);
42
- //
43
- // struct ggml_tensor * x = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1);
44
- //
45
- // ggml_set_param(ctx, x); // x is an input variable
46
- //
47
- // struct ggml_tensor * a = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1);
48
- // struct ggml_tensor * b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1);
49
- // struct ggml_tensor * x2 = ggml_mul(ctx, x, x);
50
- // struct ggml_tensor * f = ggml_add(ctx, ggml_mul(ctx, a, x2), b);
51
- //
52
- // ...
53
- // }
54
- //
55
- // Notice that the function definition above does not involve any actual computation. The computation is performed only
56
- // when the user explicitly requests it. For example, to compute the function's value at x = 2.0:
57
- //
58
- // {
59
- // ...
60
- //
61
- // struct ggml_cgraph * gf = ggml_new_graph(ctx);
62
- // ggml_build_forward_expand(gf, f);
63
- //
64
- // // set the input variable and parameter values
65
- // ggml_set_f32(x, 2.0f);
66
- // ggml_set_f32(a, 3.0f);
67
- // ggml_set_f32(b, 4.0f);
68
- //
69
- // ggml_graph_compute_with_ctx(ctx, &gf, n_threads);
70
- //
71
- // printf("f = %f\n", ggml_get_f32_1d(f, 0));
72
- //
73
- // ...
74
- // }
75
- //
76
- // The actual computation is performed in the ggml_graph_compute() function.
77
- //
78
- // The ggml_new_tensor_...() functions create new tensors. They are allocated in the memory buffer provided to the
79
- // ggml_init() function. You have to be careful not to exceed the memory buffer size. Therefore, you have to know
80
- // in advance how much memory you need for your computation. Alternatively, you can allocate a large enough memory
81
- // and after defining the computation graph, call the ggml_used_mem() function to find out how much memory was
82
- // actually needed.
83
- //
84
- // The ggml_set_param() function marks a tensor as an input variable. This is used by the automatic
85
- // differentiation and optimization algorithms.
86
- //
87
- // The described approach allows to define the function graph once and then compute its forward or backward graphs
88
- // multiple times. All computations will use the same memory buffer allocated in the ggml_init() function. This way
89
- // the user can avoid the memory allocation overhead at runtime.
90
- //
91
- // The library supports multi-dimensional tensors - up to 4 dimensions. The FP16 and FP32 data types are first class
92
- // citizens, but in theory the library can be extended to support FP8 and integer data types.
93
- //
94
- // Each tensor operation produces a new tensor. Initially the library was envisioned to support only the use of unary
95
- // and binary operations. Most of the available operations fall into one of these two categories. With time, it became
96
- // clear that the library needs to support more complex operations. The way to support these operations is not clear
97
- // yet, but a few examples are demonstrated in the following operations:
98
- //
99
- // - ggml_permute()
100
- // - ggml_conv_1d_1s()
101
- // - ggml_conv_1d_2s()
102
- //
103
- // For each tensor operator, the library implements a forward and backward computation function. The forward function
104
- // computes the output tensor value given the input tensor values. The backward function computes the adjoint of the
105
- // input tensors given the adjoint of the output tensor. For a detailed explanation of what this means, take a
106
- // calculus class, or watch the following video:
107
- //
108
- // What is Automatic Differentiation?
109
- // https://www.youtube.com/watch?v=wG_nF1awSSY
110
- //
111
- //
112
- // ## Tensor data (struct ggml_tensor)
113
- //
114
- // The tensors are stored in memory via the ggml_tensor struct. The structure provides information about the size of
115
- // the tensor, the data type, and the memory buffer where the tensor data is stored. Additionally, it contains
116
- // pointers to the "source" tensors - i.e. the tensors that were used to compute the current tensor. For example:
117
- //
118
- // {
119
- // struct ggml_tensor * c = ggml_add(ctx, a, b);
120
- //
121
- // assert(c->src[0] == a);
122
- // assert(c->src[1] == b);
123
- // }
124
- //
125
- // The multi-dimensional tensors are stored in row-major order. The ggml_tensor struct contains fields for the
126
- // number of elements in each dimension ("ne") as well as the number of bytes ("nb", a.k.a. stride). This allows
127
- // to store tensors that are not contiguous in memory, which is useful for operations such as transposition and
128
- // permutation. All tensor operations have to take the stride into account and not assume that the tensor is
129
- // contiguous in memory.
130
- //
131
- // The data of the tensor is accessed via the "data" pointer. For example:
132
- //
133
- // {
134
- // const int nx = 2;
135
- // const int ny = 3;
136
- //
137
- // struct ggml_tensor * a = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, nx, ny);
138
- //
139
- // for (int y = 0; y < ny; y++) {
140
- // for (int x = 0; x < nx; x++) {
141
- // *(float *) ((char *) a->data + y*a->nb[1] + x*a->nb[0]) = x + y;
142
- // }
143
- // }
144
- //
145
- // ...
146
- // }
147
- //
148
- // Alternatively, there are helper functions, such as ggml_get_f32_1d() and ggml_set_f32_1d() that can be used.
149
- //
150
- // ## The matrix multiplication operator (ggml_mul_mat)
151
- //
152
- // TODO
153
- //
154
- //
155
- // ## Multi-threading
156
- //
157
- // TODO
158
- //
159
- //
160
- // ## Overview of ggml.c
161
- //
162
- // TODO
163
- //
164
- //
165
- // ## SIMD optimizations
166
- //
167
- // TODO
168
- //
169
- //
170
- // ## Debugging ggml
171
- //
172
- // TODO
173
- //
174
- //
175
-
176
- #ifdef GGML_SHARED
177
- # if defined(_WIN32) && !defined(__MINGW32__)
178
- # ifdef GGML_BUILD
179
- # define GGML_API __declspec(dllexport)
180
- # else
181
- # define GGML_API __declspec(dllimport)
182
- # endif
183
- # else
184
- # define GGML_API __attribute__ ((visibility ("default")))
185
- # endif
186
- #else
187
- # define GGML_API
188
- #endif
189
-
190
- #ifdef GGML_MULTIPLATFORM
191
- # if defined(_WIN32)
192
- # define GGML_CALL
193
- # else
194
- # define GGML_CALL __attribute__((__ms_abi__))
195
- # endif
196
- #else
197
- # define GGML_CALL
198
- #endif
199
-
200
- // TODO: support for clang
201
- #ifdef __GNUC__
202
- # define GGML_DEPRECATED(func, hint) func __attribute__((deprecated(hint)))
203
- #elif defined(_MSC_VER)
204
- # define GGML_DEPRECATED(func, hint) __declspec(deprecated(hint)) func
205
- #else
206
- # define GGML_DEPRECATED(func, hint) func
207
- #endif
208
-
209
- #ifndef __GNUC__
210
- # define GGML_ATTRIBUTE_FORMAT(...)
211
- #elif defined(__MINGW32__)
212
- # define GGML_ATTRIBUTE_FORMAT(...) __attribute__((format(gnu_printf, __VA_ARGS__)))
213
- #else
214
- # define GGML_ATTRIBUTE_FORMAT(...) __attribute__((format(printf, __VA_ARGS__)))
215
- #endif
216
-
217
- #include <stdbool.h>
218
- #include <stddef.h>
219
- #include <stdint.h>
220
- #include <stdio.h>
221
-
222
- #define GGML_FILE_MAGIC 0x67676d6c // "ggml"
223
- #define GGML_FILE_VERSION 1
224
-
225
- #define GGML_QNT_VERSION 2 // bump this on quantization format changes
226
- #define GGML_QNT_VERSION_FACTOR 1000 // do not change this
227
-
228
- #define GGML_MAX_DIMS 4
229
- #define GGML_MAX_PARAMS 2048
230
- #define GGML_MAX_CONTEXTS 64
231
- #define GGML_MAX_SRC 10
232
- #ifndef GGML_MAX_NAME
233
- #define GGML_MAX_NAME 64
234
- #endif
235
- #define GGML_MAX_OP_PARAMS 64
236
- #define GGML_DEFAULT_N_THREADS 4
237
- #define GGML_DEFAULT_GRAPH_SIZE 2048
238
- #if UINTPTR_MAX == 0xFFFFFFFF
239
- #define GGML_MEM_ALIGN 4
240
- #else
241
- #define GGML_MEM_ALIGN 16
242
- #endif
243
-
244
- #define GGML_EXIT_SUCCESS 0
245
- #define GGML_EXIT_ABORTED 1
246
-
247
- #define GGUF_MAGIC "GGUF"
248
-
249
- #define GGUF_VERSION 3
250
-
251
- #define GGUF_DEFAULT_ALIGNMENT 32
252
-
253
- #define GGML_UNUSED(x) (void)(x)
254
-
255
- #define GGML_PAD(x, n) (((x) + (n) - 1) & ~((n) - 1))
256
-
257
- #define GGML_ASSERT(x) \
258
- do { \
259
- if (!(x)) { \
260
- fflush(stdout); \
261
- fprintf(stderr, "GGML_ASSERT: %s:%d: %s\n", __FILE__, __LINE__, #x); \
262
- ggml_print_backtrace(); \
263
- abort(); \
264
- } \
265
- } while (0)
266
-
267
- #ifndef NDEBUG
268
- #define GGML_UNREACHABLE() GGML_ASSERT(!"statement should not be reached")
269
- #elif defined(__GNUC__)
270
- #define GGML_UNREACHABLE() __builtin_unreachable()
271
- #elif defined(_MSC_VER)
272
- #define GGML_UNREACHABLE() __assume(0)
273
- #else
274
- #define GGML_UNREACHABLE() ((void) 0)
275
- #endif
276
-
277
- // used to copy the number of elements and stride in bytes of tensors into local variables.
278
- // main purpose is to reduce code duplication and improve readability.
279
- //
280
- // example:
281
- //
282
- // GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne);
283
- // GGML_TENSOR_LOCALS(size_t, nb1, src1, nb);
284
- //
285
- #define GGML_TENSOR_LOCALS_1(type, prefix, pointer, array) \
286
- const type prefix##0 = (pointer)->array[0]; \
287
- GGML_UNUSED(prefix##0);
288
- #define GGML_TENSOR_LOCALS_2(type, prefix, pointer, array) \
289
- GGML_TENSOR_LOCALS_1 (type, prefix, pointer, array) \
290
- const type prefix##1 = (pointer)->array[1]; \
291
- GGML_UNUSED(prefix##1);
292
- #define GGML_TENSOR_LOCALS_3(type, prefix, pointer, array) \
293
- GGML_TENSOR_LOCALS_2 (type, prefix, pointer, array) \
294
- const type prefix##2 = (pointer)->array[2]; \
295
- GGML_UNUSED(prefix##2);
296
- #define GGML_TENSOR_LOCALS(type, prefix, pointer, array) \
297
- GGML_TENSOR_LOCALS_3 (type, prefix, pointer, array) \
298
- const type prefix##3 = (pointer)->array[3]; \
299
- GGML_UNUSED(prefix##3);
300
-
301
- #define GGML_TENSOR_UNARY_OP_LOCALS \
302
- GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) \
303
- GGML_TENSOR_LOCALS(size_t, nb0, src0, nb) \
304
- GGML_TENSOR_LOCALS(int64_t, ne, dst, ne) \
305
- GGML_TENSOR_LOCALS(size_t, nb, dst, nb)
306
-
307
- #define GGML_TENSOR_BINARY_OP_LOCALS \
308
- GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) \
309
- GGML_TENSOR_LOCALS(size_t, nb0, src0, nb) \
310
- GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne) \
311
- GGML_TENSOR_LOCALS(size_t, nb1, src1, nb) \
312
- GGML_TENSOR_LOCALS(int64_t, ne, dst, ne) \
313
- GGML_TENSOR_LOCALS(size_t, nb, dst, nb)
314
-
315
- #define GGML_TENSOR_BINARY_OP_LOCALS01 \
316
- GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) \
317
- GGML_TENSOR_LOCALS(size_t, nb0, src0, nb) \
318
- GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne) \
319
- GGML_TENSOR_LOCALS(size_t, nb1, src1, nb)
320
-
321
- #ifdef __cplusplus
322
- extern "C" {
323
- #endif
324
-
325
- enum ggml_status {
326
- GGML_STATUS_ALLOC_FAILED = -2,
327
- GGML_STATUS_FAILED = -1,
328
- GGML_STATUS_SUCCESS = 0,
329
- GGML_STATUS_ABORTED = 1,
330
- };
331
-
332
- // get ggml_status name string
333
- GGML_API GGML_CALL const char * ggml_status_to_string(enum ggml_status status);
334
-
335
- // ieee 754-2008 half-precision float16
336
- // todo: make this not an integral type
337
- typedef uint16_t ggml_fp16_t;
338
- GGML_API float ggml_fp16_to_fp32(ggml_fp16_t);
339
- GGML_API ggml_fp16_t ggml_fp32_to_fp16(float);
340
- GGML_API void ggml_fp16_to_fp32_row(const ggml_fp16_t *, float *, int64_t);
341
- GGML_API void ggml_fp32_to_fp16_row(const float *, ggml_fp16_t *, int64_t);
342
-
343
- // google brain half-precision bfloat16
344
- typedef struct { uint16_t bits; } ggml_bf16_t;
345
- GGML_API ggml_bf16_t ggml_fp32_to_bf16(float);
346
- GGML_API float ggml_bf16_to_fp32(ggml_bf16_t); // consider just doing << 16
347
- GGML_API void ggml_bf16_to_fp32_row(const ggml_bf16_t *, float *, int64_t);
348
- GGML_API void ggml_fp32_to_bf16_row(const float *, ggml_bf16_t *, int64_t);
349
-
350
- struct ggml_object;
351
- struct ggml_context;
352
-
353
- // NOTE: always add types at the end of the enum to keep backward compatibility
354
- enum ggml_type {
355
- GGML_TYPE_F32 = 0,
356
- GGML_TYPE_F16 = 1,
357
- GGML_TYPE_Q4_0 = 2,
358
- GGML_TYPE_Q4_1 = 3,
359
- // GGML_TYPE_Q4_2 = 4, support has been removed
360
- // GGML_TYPE_Q4_3 = 5, support has been removed
361
- GGML_TYPE_Q5_0 = 6,
362
- GGML_TYPE_Q5_1 = 7,
363
- GGML_TYPE_Q8_0 = 8,
364
- GGML_TYPE_Q8_1 = 9,
365
- GGML_TYPE_Q2_K = 10,
366
- GGML_TYPE_Q3_K = 11,
367
- GGML_TYPE_Q4_K = 12,
368
- GGML_TYPE_Q5_K = 13,
369
- GGML_TYPE_Q6_K = 14,
370
- GGML_TYPE_Q8_K = 15,
371
- GGML_TYPE_IQ2_XXS = 16,
372
- GGML_TYPE_IQ2_XS = 17,
373
- GGML_TYPE_IQ3_XXS = 18,
374
- GGML_TYPE_IQ1_S = 19,
375
- GGML_TYPE_IQ4_NL = 20,
376
- GGML_TYPE_IQ3_S = 21,
377
- GGML_TYPE_IQ2_S = 22,
378
- GGML_TYPE_IQ4_XS = 23,
379
- GGML_TYPE_I8 = 24,
380
- GGML_TYPE_I16 = 25,
381
- GGML_TYPE_I32 = 26,
382
- GGML_TYPE_I64 = 27,
383
- GGML_TYPE_F64 = 28,
384
- GGML_TYPE_IQ1_M = 29,
385
- GGML_TYPE_BF16 = 30,
386
- GGML_TYPE_COUNT,
387
- };
388
-
389
- // precision
390
- enum ggml_prec {
391
- GGML_PREC_DEFAULT,
392
- GGML_PREC_F32,
393
- };
394
-
395
- enum ggml_backend_type {
396
- GGML_BACKEND_TYPE_CPU = 0,
397
- GGML_BACKEND_TYPE_GPU = 10,
398
- GGML_BACKEND_TYPE_GPU_SPLIT = 20,
399
- };
400
-
401
- // model file types
402
- enum ggml_ftype {
403
- GGML_FTYPE_UNKNOWN = -1,
404
- GGML_FTYPE_ALL_F32 = 0,
405
- GGML_FTYPE_MOSTLY_F16 = 1, // except 1d tensors
406
- GGML_FTYPE_MOSTLY_Q4_0 = 2, // except 1d tensors
407
- GGML_FTYPE_MOSTLY_Q4_1 = 3, // except 1d tensors
408
- GGML_FTYPE_MOSTLY_Q4_1_SOME_F16 = 4, // tok_embeddings.weight and output.weight are F16
409
- GGML_FTYPE_MOSTLY_Q8_0 = 7, // except 1d tensors
410
- GGML_FTYPE_MOSTLY_Q5_0 = 8, // except 1d tensors
411
- GGML_FTYPE_MOSTLY_Q5_1 = 9, // except 1d tensors
412
- GGML_FTYPE_MOSTLY_Q2_K = 10, // except 1d tensors
413
- GGML_FTYPE_MOSTLY_Q3_K = 11, // except 1d tensors
414
- GGML_FTYPE_MOSTLY_Q4_K = 12, // except 1d tensors
415
- GGML_FTYPE_MOSTLY_Q5_K = 13, // except 1d tensors
416
- GGML_FTYPE_MOSTLY_Q6_K = 14, // except 1d tensors
417
- GGML_FTYPE_MOSTLY_IQ2_XXS = 15, // except 1d tensors
418
- GGML_FTYPE_MOSTLY_IQ2_XS = 16, // except 1d tensors
419
- GGML_FTYPE_MOSTLY_IQ3_XXS = 17, // except 1d tensors
420
- GGML_FTYPE_MOSTLY_IQ1_S = 18, // except 1d tensors
421
- GGML_FTYPE_MOSTLY_IQ4_NL = 19, // except 1d tensors
422
- GGML_FTYPE_MOSTLY_IQ3_S = 20, // except 1d tensors
423
- GGML_FTYPE_MOSTLY_IQ2_S = 21, // except 1d tensors
424
- GGML_FTYPE_MOSTLY_IQ4_XS = 22, // except 1d tensors
425
- GGML_FTYPE_MOSTLY_IQ1_M = 23, // except 1d tensors
426
- GGML_FTYPE_MOSTLY_BF16 = 24, // except 1d tensors
427
- };
428
-
429
- // available tensor operations:
430
- enum ggml_op {
431
- GGML_OP_NONE = 0,
432
-
433
- GGML_OP_DUP,
434
- GGML_OP_ADD,
435
- GGML_OP_ADD1,
436
- GGML_OP_ACC,
437
- GGML_OP_SUB,
438
- GGML_OP_MUL,
439
- GGML_OP_DIV,
440
- GGML_OP_SQR,
441
- GGML_OP_SQRT,
442
- GGML_OP_LOG,
443
- GGML_OP_SUM,
444
- GGML_OP_SUM_ROWS,
445
- GGML_OP_MEAN,
446
- GGML_OP_ARGMAX,
447
- GGML_OP_REPEAT,
448
- GGML_OP_REPEAT_BACK,
449
- GGML_OP_CONCAT,
450
- GGML_OP_SILU_BACK,
451
- GGML_OP_NORM, // normalize
452
- GGML_OP_RMS_NORM,
453
- GGML_OP_RMS_NORM_BACK,
454
- GGML_OP_GROUP_NORM,
455
-
456
- GGML_OP_MUL_MAT,
457
- GGML_OP_MUL_MAT_ID,
458
- GGML_OP_OUT_PROD,
459
-
460
- GGML_OP_SCALE,
461
- GGML_OP_SET,
462
- GGML_OP_CPY,
463
- GGML_OP_CONT,
464
- GGML_OP_RESHAPE,
465
- GGML_OP_VIEW,
466
- GGML_OP_PERMUTE,
467
- GGML_OP_TRANSPOSE,
468
- GGML_OP_GET_ROWS,
469
- GGML_OP_GET_ROWS_BACK,
470
- GGML_OP_DIAG,
471
- GGML_OP_DIAG_MASK_INF,
472
- GGML_OP_DIAG_MASK_ZERO,
473
- GGML_OP_SOFT_MAX,
474
- GGML_OP_SOFT_MAX_BACK,
475
- GGML_OP_ROPE,
476
- GGML_OP_ROPE_BACK,
477
- GGML_OP_CLAMP,
478
- GGML_OP_CONV_TRANSPOSE_1D,
479
- GGML_OP_IM2COL,
480
- GGML_OP_CONV_TRANSPOSE_2D,
481
- GGML_OP_POOL_1D,
482
- GGML_OP_POOL_2D,
483
- GGML_OP_UPSCALE, // nearest interpolate
484
- GGML_OP_PAD,
485
- GGML_OP_ARANGE,
486
- GGML_OP_TIMESTEP_EMBEDDING,
487
- GGML_OP_ARGSORT,
488
- GGML_OP_LEAKY_RELU,
489
-
490
- GGML_OP_FLASH_ATTN_EXT,
491
- GGML_OP_FLASH_ATTN_BACK,
492
- GGML_OP_SSM_CONV,
493
- GGML_OP_SSM_SCAN,
494
- GGML_OP_WIN_PART,
495
- GGML_OP_WIN_UNPART,
496
- GGML_OP_GET_REL_POS,
497
- GGML_OP_ADD_REL_POS,
498
-
499
- GGML_OP_UNARY,
500
-
501
- GGML_OP_MAP_UNARY,
502
- GGML_OP_MAP_BINARY,
503
-
504
- GGML_OP_MAP_CUSTOM1_F32,
505
- GGML_OP_MAP_CUSTOM2_F32,
506
- GGML_OP_MAP_CUSTOM3_F32,
507
-
508
- GGML_OP_MAP_CUSTOM1,
509
- GGML_OP_MAP_CUSTOM2,
510
- GGML_OP_MAP_CUSTOM3,
511
-
512
- GGML_OP_CROSS_ENTROPY_LOSS,
513
- GGML_OP_CROSS_ENTROPY_LOSS_BACK,
514
-
515
- GGML_OP_COUNT,
516
- };
517
-
518
- enum ggml_unary_op {
519
- GGML_UNARY_OP_ABS,
520
- GGML_UNARY_OP_SGN,
521
- GGML_UNARY_OP_NEG,
522
- GGML_UNARY_OP_STEP,
523
- GGML_UNARY_OP_TANH,
524
- GGML_UNARY_OP_ELU,
525
- GGML_UNARY_OP_RELU,
526
- GGML_UNARY_OP_SIGMOID,
527
- GGML_UNARY_OP_GELU,
528
- GGML_UNARY_OP_GELU_QUICK,
529
- GGML_UNARY_OP_SILU,
530
- GGML_UNARY_OP_HARDSWISH,
531
- GGML_UNARY_OP_HARDSIGMOID,
532
-
533
- GGML_UNARY_OP_COUNT,
534
- };
535
-
536
- enum ggml_object_type {
537
- GGML_OBJECT_TYPE_TENSOR,
538
- GGML_OBJECT_TYPE_GRAPH,
539
- GGML_OBJECT_TYPE_WORK_BUFFER
540
- };
541
-
542
- enum ggml_log_level {
543
- GGML_LOG_LEVEL_ERROR = 2,
544
- GGML_LOG_LEVEL_WARN = 3,
545
- GGML_LOG_LEVEL_INFO = 4,
546
- GGML_LOG_LEVEL_DEBUG = 5
547
- };
548
-
549
- enum ggml_tensor_flag {
550
- GGML_TENSOR_FLAG_INPUT = 1,
551
- GGML_TENSOR_FLAG_OUTPUT = 2,
552
- GGML_TENSOR_FLAG_PARAM = 4,
553
- };
554
-
555
- // ggml object
556
- struct ggml_object {
557
- size_t offs;
558
- size_t size;
559
-
560
- struct ggml_object * next;
561
-
562
- enum ggml_object_type type;
563
-
564
- char padding[4];
565
- };
566
-
567
- static const size_t GGML_OBJECT_SIZE = sizeof(struct ggml_object);
568
-
569
- // n-dimensional tensor
570
- struct ggml_tensor {
571
- enum ggml_type type;
572
-
573
- GGML_DEPRECATED(enum ggml_backend_type backend, "use the buffer type to find the storage location of the tensor");
574
-
575
- struct ggml_backend_buffer * buffer;
576
-
577
- int64_t ne[GGML_MAX_DIMS]; // number of elements
578
- size_t nb[GGML_MAX_DIMS]; // stride in bytes:
579
- // nb[0] = ggml_type_size(type)
580
- // nb[1] = nb[0] * (ne[0] / ggml_blck_size(type)) + padding
581
- // nb[i] = nb[i-1] * ne[i-1]
582
-
583
- // compute data
584
- enum ggml_op op;
585
-
586
- // op params - allocated as int32_t for alignment
587
- int32_t op_params[GGML_MAX_OP_PARAMS / sizeof(int32_t)];
588
-
589
- int32_t flags;
590
-
591
- struct ggml_tensor * grad;
592
- struct ggml_tensor * src[GGML_MAX_SRC];
593
-
594
- // performance
595
- int perf_runs;
596
- int64_t perf_cycles;
597
- int64_t perf_time_us;
598
-
599
- struct ggml_tensor * view_src;
600
- size_t view_offs;
601
-
602
- void * data;
603
-
604
- char name[GGML_MAX_NAME];
605
-
606
- void * extra; // extra things e.g. for ggml-cuda.cu
607
-
608
- char padding[8];
609
- };
610
-
611
- static const size_t GGML_TENSOR_SIZE = sizeof(struct ggml_tensor);
612
-
613
- // Abort callback
614
- // If not NULL, called before ggml computation
615
- // If it returns true, the computation is aborted
616
- typedef bool (*ggml_abort_callback)(void * data);
617
-
618
- // the compute plan that needs to be prepared for ggml_graph_compute()
619
- // since https://github.com/ggerganov/ggml/issues/287
620
- struct ggml_cplan {
621
- size_t work_size; // size of work buffer, calculated by `ggml_graph_plan()`
622
- uint8_t * work_data; // work buffer, to be allocated by caller before calling to `ggml_graph_compute()`
623
-
624
- int n_threads;
625
-
626
- // abort ggml_graph_compute when true
627
- ggml_abort_callback abort_callback;
628
- void * abort_callback_data;
629
- };
630
-
631
- enum ggml_cgraph_eval_order {
632
- GGML_CGRAPH_EVAL_ORDER_LEFT_TO_RIGHT = 0,
633
- GGML_CGRAPH_EVAL_ORDER_RIGHT_TO_LEFT,
634
- GGML_CGRAPH_EVAL_ORDER_COUNT
635
- };
636
-
637
- struct ggml_hash_set {
638
- size_t size;
639
- struct ggml_tensor ** keys;
640
- };
641
-
642
- // computation graph
643
- struct ggml_cgraph {
644
- int size;
645
- int n_nodes;
646
- int n_leafs;
647
-
648
- struct ggml_tensor ** nodes;
649
- struct ggml_tensor ** grads;
650
- struct ggml_tensor ** leafs;
651
-
652
- struct ggml_hash_set visited_hash_table;
653
-
654
- enum ggml_cgraph_eval_order order;
655
-
656
- // performance
657
- int perf_runs;
658
- int64_t perf_cycles;
659
- int64_t perf_time_us;
660
- };
661
-
662
- // scratch buffer
663
- struct ggml_scratch {
664
- size_t offs;
665
- size_t size;
666
- void * data;
667
- };
668
-
669
- struct ggml_init_params {
670
- // memory pool
671
- size_t mem_size; // bytes
672
- void * mem_buffer; // if NULL, memory will be allocated internally
673
- bool no_alloc; // don't allocate memory for the tensor data
674
- };
675
-
676
-
677
- // compute types
678
-
679
- // NOTE: the INIT or FINALIZE pass is not scheduled unless explicitly enabled.
680
- // This behavior was changed since https://github.com/ggerganov/llama.cpp/pull/1995.
681
- enum ggml_task_type {
682
- GGML_TASK_TYPE_INIT = 0,
683
- GGML_TASK_TYPE_COMPUTE,
684
- GGML_TASK_TYPE_FINALIZE,
685
- };
686
-
687
- struct ggml_compute_params {
688
- enum ggml_task_type type;
689
-
690
- // ith = thread index, nth = number of threads
691
- int ith, nth;
692
-
693
- // work buffer for all threads
694
- size_t wsize;
695
- void * wdata;
696
- };
697
-
698
- // numa strategies
699
- enum ggml_numa_strategy {
700
- GGML_NUMA_STRATEGY_DISABLED = 0,
701
- GGML_NUMA_STRATEGY_DISTRIBUTE = 1,
702
- GGML_NUMA_STRATEGY_ISOLATE = 2,
703
- GGML_NUMA_STRATEGY_NUMACTL = 3,
704
- GGML_NUMA_STRATEGY_MIRROR = 4,
705
- GGML_NUMA_STRATEGY_COUNT
706
- };
707
-
708
- //
709
- // GUID
710
- //
711
-
712
- // GUID types
713
- typedef uint8_t ggml_guid[16];
714
- typedef ggml_guid * ggml_guid_t;
715
-
716
- GGML_API bool ggml_guid_matches(ggml_guid_t guid_a, ggml_guid_t guid_b);
717
-
718
- // misc
719
-
720
- GGML_API void ggml_time_init(void); // call this once at the beginning of the program
721
- GGML_API int64_t ggml_time_ms(void);
722
- GGML_API int64_t ggml_time_us(void);
723
- GGML_API int64_t ggml_cycles(void);
724
- GGML_API int64_t ggml_cycles_per_ms(void);
725
-
726
- GGML_API void ggml_print_backtrace(void);
727
-
728
- // accepts a UTF-8 path, even on Windows
729
- GGML_API FILE * ggml_fopen(const char * fname, const char * mode);
730
-
731
- GGML_API void ggml_numa_init(enum ggml_numa_strategy numa); // call once for better performance on NUMA systems
732
- GGML_API bool ggml_is_numa(void); // true if init detected that system has >1 NUMA node
733
-
734
- GGML_API void ggml_print_object (const struct ggml_object * obj);
735
- GGML_API void ggml_print_objects(const struct ggml_context * ctx);
736
-
737
- GGML_API GGML_CALL int64_t ggml_nelements (const struct ggml_tensor * tensor);
738
- GGML_API GGML_CALL int64_t ggml_nrows (const struct ggml_tensor * tensor);
739
- GGML_API GGML_CALL size_t ggml_nbytes (const struct ggml_tensor * tensor);
740
- GGML_API size_t ggml_nbytes_pad (const struct ggml_tensor * tensor); // same as ggml_nbytes() but padded to GGML_MEM_ALIGN
741
-
742
- GGML_API GGML_CALL int ggml_blck_size(enum ggml_type type);
743
- GGML_API GGML_CALL size_t ggml_type_size(enum ggml_type type); // size in bytes for all elements in a block
744
- GGML_API GGML_CALL size_t ggml_row_size (enum ggml_type type, int64_t ne); // size in bytes for all elements in a row
745
-
746
- GGML_DEPRECATED(
747
- GGML_API double ggml_type_sizef(enum ggml_type type), // ggml_type_size()/ggml_blck_size() as float
748
- "use ggml_row_size() instead");
749
-
750
- GGML_API GGML_CALL const char * ggml_type_name(enum ggml_type type);
751
- GGML_API GGML_CALL const char * ggml_op_name (enum ggml_op op);
752
- GGML_API const char * ggml_op_symbol(enum ggml_op op);
753
-
754
- GGML_API const char * ggml_unary_op_name(enum ggml_unary_op op);
755
- GGML_API GGML_CALL const char * ggml_op_desc(const struct ggml_tensor * t); // unary or op name
756
-
757
- GGML_API GGML_CALL size_t ggml_element_size(const struct ggml_tensor * tensor);
758
-
759
- GGML_API GGML_CALL bool ggml_is_quantized(enum ggml_type type);
760
-
761
- // TODO: temporary until model loading of ggml examples is refactored
762
- GGML_API enum ggml_type ggml_ftype_to_ggml_type(enum ggml_ftype ftype);
763
-
764
- GGML_API GGML_CALL bool ggml_is_transposed(const struct ggml_tensor * tensor);
765
- GGML_API GGML_CALL bool ggml_is_permuted (const struct ggml_tensor * tensor);
766
- GGML_API GGML_CALL bool ggml_is_empty (const struct ggml_tensor * tensor);
767
- GGML_API bool ggml_is_scalar (const struct ggml_tensor * tensor);
768
- GGML_API bool ggml_is_vector (const struct ggml_tensor * tensor);
769
- GGML_API bool ggml_is_matrix (const struct ggml_tensor * tensor);
770
- GGML_API bool ggml_is_3d (const struct ggml_tensor * tensor);
771
- GGML_API int ggml_n_dims (const struct ggml_tensor * tensor); // returns 1 for scalars
772
-
773
- GGML_API GGML_CALL bool ggml_is_contiguous (const struct ggml_tensor * tensor);
774
- GGML_API GGML_CALL bool ggml_is_contiguous_0(const struct ggml_tensor * tensor); // same as ggml_is_contiguous()
775
- GGML_API GGML_CALL bool ggml_is_contiguous_1(const struct ggml_tensor * tensor); // contiguous for dims >= 1
776
- GGML_API GGML_CALL bool ggml_is_contiguous_2(const struct ggml_tensor * tensor); // contiguous for dims >= 2
777
-
778
- GGML_API bool ggml_are_same_shape (const struct ggml_tensor * t0, const struct ggml_tensor * t1);
779
- GGML_API bool ggml_are_same_stride(const struct ggml_tensor * t0, const struct ggml_tensor * t1);
780
-
781
- // use this to compute the memory overhead of a tensor
782
- GGML_API size_t ggml_tensor_overhead(void);
783
-
784
- GGML_API bool ggml_validate_row_data(enum ggml_type type, const void * data, size_t nbytes);
785
-
786
- // main
787
-
788
- GGML_API struct ggml_context * ggml_init(struct ggml_init_params params);
789
- GGML_API void ggml_free(struct ggml_context * ctx);
790
-
791
- GGML_API size_t ggml_used_mem(const struct ggml_context * ctx);
792
-
793
- GGML_API size_t ggml_set_scratch (struct ggml_context * ctx, struct ggml_scratch scratch);
794
- GGML_API bool ggml_get_no_alloc(struct ggml_context * ctx);
795
- GGML_API void ggml_set_no_alloc(struct ggml_context * ctx, bool no_alloc);
796
-
797
- GGML_API void * ggml_get_mem_buffer (const struct ggml_context * ctx);
798
- GGML_API size_t ggml_get_mem_size (const struct ggml_context * ctx);
799
- GGML_API size_t ggml_get_max_tensor_size(const struct ggml_context * ctx);
800
-
801
- GGML_API struct ggml_tensor * ggml_new_tensor(
802
- struct ggml_context * ctx,
803
- enum ggml_type type,
804
- int n_dims,
805
- const int64_t *ne);
806
-
807
- GGML_API struct ggml_tensor * ggml_new_tensor_1d(
808
- struct ggml_context * ctx,
809
- enum ggml_type type,
810
- int64_t ne0);
811
-
812
- GGML_API struct ggml_tensor * ggml_new_tensor_2d(
813
- struct ggml_context * ctx,
814
- enum ggml_type type,
815
- int64_t ne0,
816
- int64_t ne1);
817
-
818
- GGML_API struct ggml_tensor * ggml_new_tensor_3d(
819
- struct ggml_context * ctx,
820
- enum ggml_type type,
821
- int64_t ne0,
822
- int64_t ne1,
823
- int64_t ne2);
824
-
825
- GGML_API struct ggml_tensor * ggml_new_tensor_4d(
826
- struct ggml_context * ctx,
827
- enum ggml_type type,
828
- int64_t ne0,
829
- int64_t ne1,
830
- int64_t ne2,
831
- int64_t ne3);
832
-
833
- GGML_API struct ggml_tensor * ggml_new_i32(struct ggml_context * ctx, int32_t value);
834
- GGML_API struct ggml_tensor * ggml_new_f32(struct ggml_context * ctx, float value);
835
-
836
- GGML_API struct ggml_tensor * ggml_dup_tensor (struct ggml_context * ctx, const struct ggml_tensor * src);
837
- GGML_API struct ggml_tensor * ggml_view_tensor(struct ggml_context * ctx, struct ggml_tensor * src);
838
-
839
- // Context tensor enumeration and lookup
840
- GGML_API struct ggml_tensor * ggml_get_first_tensor(const struct ggml_context * ctx);
841
- GGML_API struct ggml_tensor * ggml_get_next_tensor (const struct ggml_context * ctx, struct ggml_tensor * tensor);
842
- GGML_API struct ggml_tensor * ggml_get_tensor(struct ggml_context * ctx, const char * name);
843
-
844
- GGML_API struct ggml_tensor * ggml_set_zero(struct ggml_tensor * tensor);
845
- GGML_API struct ggml_tensor * ggml_set_i32 (struct ggml_tensor * tensor, int32_t value);
846
- GGML_API struct ggml_tensor * ggml_set_f32 (struct ggml_tensor * tensor, float value);
847
-
848
- // Converts a flat index into coordinates
849
- GGML_API void ggml_unravel_index(const struct ggml_tensor * tensor, int64_t i, int64_t * i0, int64_t * i1, int64_t * i2, int64_t * i3);
850
-
851
- GGML_API int32_t ggml_get_i32_1d(const struct ggml_tensor * tensor, int i);
852
- GGML_API void ggml_set_i32_1d(const struct ggml_tensor * tensor, int i, int32_t value);
853
-
854
- GGML_API int32_t ggml_get_i32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3);
855
- GGML_API void ggml_set_i32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3, int32_t value);
856
-
857
- GGML_API float ggml_get_f32_1d(const struct ggml_tensor * tensor, int i);
858
- GGML_API void ggml_set_f32_1d(const struct ggml_tensor * tensor, int i, float value);
859
-
860
- GGML_API float ggml_get_f32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3);
861
- GGML_API void ggml_set_f32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3, float value);
862
-
863
- GGML_API void * ggml_get_data (const struct ggml_tensor * tensor);
864
- GGML_API float * ggml_get_data_f32(const struct ggml_tensor * tensor);
865
-
866
- GGML_API GGML_CALL enum ggml_unary_op ggml_get_unary_op(const struct ggml_tensor * tensor);
867
-
868
- GGML_API const char * ggml_get_name (const struct ggml_tensor * tensor);
869
- GGML_API struct ggml_tensor * ggml_set_name ( struct ggml_tensor * tensor, const char * name);
870
- GGML_ATTRIBUTE_FORMAT(2, 3)
871
- GGML_API struct ggml_tensor * ggml_format_name( struct ggml_tensor * tensor, const char * fmt, ...);
872
-
873
- //
874
- // operations on tensors with backpropagation
875
- //
876
-
877
- GGML_API struct ggml_tensor * ggml_dup(
878
- struct ggml_context * ctx,
879
- struct ggml_tensor * a);
880
-
881
- // in-place, returns view(a)
882
- GGML_API struct ggml_tensor * ggml_dup_inplace(
883
- struct ggml_context * ctx,
884
- struct ggml_tensor * a);
885
-
886
- GGML_API struct ggml_tensor * ggml_add(
887
- struct ggml_context * ctx,
888
- struct ggml_tensor * a,
889
- struct ggml_tensor * b);
890
-
891
- GGML_API struct ggml_tensor * ggml_add_inplace(
892
- struct ggml_context * ctx,
893
- struct ggml_tensor * a,
894
- struct ggml_tensor * b);
895
-
896
- GGML_API struct ggml_tensor * ggml_add_cast(
897
- struct ggml_context * ctx,
898
- struct ggml_tensor * a,
899
- struct ggml_tensor * b,
900
- enum ggml_type type);
901
-
902
- GGML_API struct ggml_tensor * ggml_add1(
903
- struct ggml_context * ctx,
904
- struct ggml_tensor * a,
905
- struct ggml_tensor * b);
906
-
907
- GGML_API struct ggml_tensor * ggml_add1_inplace(
908
- struct ggml_context * ctx,
909
- struct ggml_tensor * a,
910
- struct ggml_tensor * b);
911
-
912
- // dst = a
913
- // view(dst, nb1, nb2, nb3, offset) += b
914
- // return dst
915
- GGML_API struct ggml_tensor * ggml_acc(
916
- struct ggml_context * ctx,
917
- struct ggml_tensor * a,
918
- struct ggml_tensor * b,
919
- size_t nb1,
920
- size_t nb2,
921
- size_t nb3,
922
- size_t offset);
923
-
924
- GGML_API struct ggml_tensor * ggml_acc_inplace(
925
- struct ggml_context * ctx,
926
- struct ggml_tensor * a,
927
- struct ggml_tensor * b,
928
- size_t nb1,
929
- size_t nb2,
930
- size_t nb3,
931
- size_t offset);
932
-
933
- GGML_API struct ggml_tensor * ggml_sub(
934
- struct ggml_context * ctx,
935
- struct ggml_tensor * a,
936
- struct ggml_tensor * b);
937
-
938
- GGML_API struct ggml_tensor * ggml_sub_inplace(
939
- struct ggml_context * ctx,
940
- struct ggml_tensor * a,
941
- struct ggml_tensor * b);
942
-
943
- GGML_API struct ggml_tensor * ggml_mul(
944
- struct ggml_context * ctx,
945
- struct ggml_tensor * a,
946
- struct ggml_tensor * b);
947
-
948
- GGML_API struct ggml_tensor * ggml_mul_inplace(
949
- struct ggml_context * ctx,
950
- struct ggml_tensor * a,
951
- struct ggml_tensor * b);
952
-
953
- GGML_API struct ggml_tensor * ggml_div(
954
- struct ggml_context * ctx,
955
- struct ggml_tensor * a,
956
- struct ggml_tensor * b);
957
-
958
- GGML_API struct ggml_tensor * ggml_div_inplace(
959
- struct ggml_context * ctx,
960
- struct ggml_tensor * a,
961
- struct ggml_tensor * b);
962
-
963
- GGML_API struct ggml_tensor * ggml_sqr(
964
- struct ggml_context * ctx,
965
- struct ggml_tensor * a);
966
-
967
- GGML_API struct ggml_tensor * ggml_sqr_inplace(
968
- struct ggml_context * ctx,
969
- struct ggml_tensor * a);
970
-
971
- GGML_API struct ggml_tensor * ggml_sqrt(
972
- struct ggml_context * ctx,
973
- struct ggml_tensor * a);
974
-
975
- GGML_API struct ggml_tensor * ggml_sqrt_inplace(
976
- struct ggml_context * ctx,
977
- struct ggml_tensor * a);
978
-
979
- GGML_API struct ggml_tensor * ggml_log(
980
- struct ggml_context * ctx,
981
- struct ggml_tensor * a);
982
-
983
- GGML_API struct ggml_tensor * ggml_log_inplace(
984
- struct ggml_context * ctx,
985
- struct ggml_tensor * a);
986
-
987
- // return scalar
988
- GGML_API struct ggml_tensor * ggml_sum(
989
- struct ggml_context * ctx,
990
- struct ggml_tensor * a);
991
-
992
- // sums along rows, with input shape [a,b,c,d] return shape [1,b,c,d]
993
- GGML_API struct ggml_tensor * ggml_sum_rows(
994
- struct ggml_context * ctx,
995
- struct ggml_tensor * a);
996
-
997
- // mean along rows
998
- GGML_API struct ggml_tensor * ggml_mean(
999
- struct ggml_context * ctx,
1000
- struct ggml_tensor * a);
1001
-
1002
- // argmax along rows
1003
- GGML_API struct ggml_tensor * ggml_argmax(
1004
- struct ggml_context * ctx,
1005
- struct ggml_tensor * a);
1006
-
1007
- // if a is the same shape as b, and a is not parameter, return a
1008
- // otherwise, return a new tensor: repeat(a) to fit in b
1009
- GGML_API struct ggml_tensor * ggml_repeat(
1010
- struct ggml_context * ctx,
1011
- struct ggml_tensor * a,
1012
- struct ggml_tensor * b);
1013
-
1014
- // sums repetitions in a into shape of b
1015
- GGML_API struct ggml_tensor * ggml_repeat_back(
1016
- struct ggml_context * ctx,
1017
- struct ggml_tensor * a,
1018
- struct ggml_tensor * b);
1019
-
1020
- // concat a and b along dim
1021
- // used in stable-diffusion
1022
- GGML_API struct ggml_tensor * ggml_concat(
1023
- struct ggml_context * ctx,
1024
- struct ggml_tensor * a,
1025
- struct ggml_tensor * b,
1026
- int dim);
1027
-
1028
- GGML_API struct ggml_tensor * ggml_abs(
1029
- struct ggml_context * ctx,
1030
- struct ggml_tensor * a);
1031
-
1032
- GGML_API struct ggml_tensor * ggml_abs_inplace(
1033
- struct ggml_context * ctx,
1034
- struct ggml_tensor * a);
1035
-
1036
- GGML_API struct ggml_tensor * ggml_sgn(
1037
- struct ggml_context * ctx,
1038
- struct ggml_tensor * a);
1039
-
1040
- GGML_API struct ggml_tensor * ggml_sgn_inplace(
1041
- struct ggml_context * ctx,
1042
- struct ggml_tensor * a);
1043
-
1044
- GGML_API struct ggml_tensor * ggml_neg(
1045
- struct ggml_context * ctx,
1046
- struct ggml_tensor * a);
1047
-
1048
- GGML_API struct ggml_tensor * ggml_neg_inplace(
1049
- struct ggml_context * ctx,
1050
- struct ggml_tensor * a);
1051
-
1052
- GGML_API struct ggml_tensor * ggml_step(
1053
- struct ggml_context * ctx,
1054
- struct ggml_tensor * a);
1055
-
1056
- GGML_API struct ggml_tensor * ggml_step_inplace(
1057
- struct ggml_context * ctx,
1058
- struct ggml_tensor * a);
1059
-
1060
- GGML_API struct ggml_tensor * ggml_tanh(
1061
- struct ggml_context * ctx,
1062
- struct ggml_tensor * a);
1063
-
1064
- GGML_API struct ggml_tensor * ggml_tanh_inplace(
1065
- struct ggml_context * ctx,
1066
- struct ggml_tensor * a);
1067
-
1068
- GGML_API struct ggml_tensor * ggml_elu(
1069
- struct ggml_context * ctx,
1070
- struct ggml_tensor * a);
1071
-
1072
- GGML_API struct ggml_tensor * ggml_elu_inplace(
1073
- struct ggml_context * ctx,
1074
- struct ggml_tensor * a);
1075
-
1076
- GGML_API struct ggml_tensor * ggml_relu(
1077
- struct ggml_context * ctx,
1078
- struct ggml_tensor * a);
1079
-
1080
- GGML_API struct ggml_tensor * ggml_leaky_relu(
1081
- struct ggml_context * ctx,
1082
- struct ggml_tensor * a, float negative_slope, bool inplace);
1083
-
1084
- GGML_API struct ggml_tensor * ggml_relu_inplace(
1085
- struct ggml_context * ctx,
1086
- struct ggml_tensor * a);
1087
-
1088
- GGML_API struct ggml_tensor * ggml_sigmoid(
1089
- struct ggml_context * ctx,
1090
- struct ggml_tensor * a);
1091
-
1092
- GGML_API struct ggml_tensor * ggml_sigmoid_inplace(
1093
- struct ggml_context * ctx,
1094
- struct ggml_tensor * a);
1095
-
1096
- GGML_API struct ggml_tensor * ggml_gelu(
1097
- struct ggml_context * ctx,
1098
- struct ggml_tensor * a);
1099
-
1100
- GGML_API struct ggml_tensor * ggml_gelu_inplace(
1101
- struct ggml_context * ctx,
1102
- struct ggml_tensor * a);
1103
-
1104
- GGML_API struct ggml_tensor * ggml_gelu_quick(
1105
- struct ggml_context * ctx,
1106
- struct ggml_tensor * a);
1107
-
1108
- GGML_API struct ggml_tensor * ggml_gelu_quick_inplace(
1109
- struct ggml_context * ctx,
1110
- struct ggml_tensor * a);
1111
-
1112
- GGML_API struct ggml_tensor * ggml_silu(
1113
- struct ggml_context * ctx,
1114
- struct ggml_tensor * a);
1115
-
1116
- GGML_API struct ggml_tensor * ggml_silu_inplace(
1117
- struct ggml_context * ctx,
1118
- struct ggml_tensor * a);
1119
-
1120
- // a - x
1121
- // b - dy
1122
- GGML_API struct ggml_tensor * ggml_silu_back(
1123
- struct ggml_context * ctx,
1124
- struct ggml_tensor * a,
1125
- struct ggml_tensor * b);
1126
-
1127
- // hardswish(x) = x * relu6(x + 3) / 6
1128
- GGML_API struct ggml_tensor * ggml_hardswish(
1129
- struct ggml_context * ctx,
1130
- struct ggml_tensor * a);
1131
-
1132
- // hardsigmoid(x) = relu6(x + 3) / 6
1133
- GGML_API struct ggml_tensor * ggml_hardsigmoid(
1134
- struct ggml_context * ctx,
1135
- struct ggml_tensor * a);
1136
-
1137
- // normalize along rows
1138
- GGML_API struct ggml_tensor * ggml_norm(
1139
- struct ggml_context * ctx,
1140
- struct ggml_tensor * a,
1141
- float eps);
1142
-
1143
- GGML_API struct ggml_tensor * ggml_norm_inplace(
1144
- struct ggml_context * ctx,
1145
- struct ggml_tensor * a,
1146
- float eps);
1147
-
1148
- GGML_API struct ggml_tensor * ggml_rms_norm(
1149
- struct ggml_context * ctx,
1150
- struct ggml_tensor * a,
1151
- float eps);
1152
-
1153
- GGML_API struct ggml_tensor * ggml_rms_norm_inplace(
1154
- struct ggml_context * ctx,
1155
- struct ggml_tensor * a,
1156
- float eps);
1157
-
1158
- // group normalize along ne0*ne1*n_groups
1159
- // used in stable-diffusion
1160
- // TODO: eps is hardcoded to 1e-6 for now
1161
- GGML_API struct ggml_tensor * ggml_group_norm(
1162
- struct ggml_context * ctx,
1163
- struct ggml_tensor * a,
1164
- int n_groups);
1165
-
1166
- GGML_API struct ggml_tensor * ggml_group_norm_inplace(
1167
- struct ggml_context * ctx,
1168
- struct ggml_tensor * a,
1169
- int n_groups);
1170
-
1171
- // a - x
1172
- // b - dy
1173
- GGML_API struct ggml_tensor * ggml_rms_norm_back(
1174
- struct ggml_context * ctx,
1175
- struct ggml_tensor * a,
1176
- struct ggml_tensor * b,
1177
- float eps);
1178
-
1179
- // A: k columns, n rows => [ne03, ne02, n, k]
1180
- // B: k columns, m rows (i.e. we transpose it internally) => [ne03 * x, ne02 * y, m, k]
1181
- // result is n columns, m rows => [ne03 * x, ne02 * y, m, n]
1182
- GGML_API struct ggml_tensor * ggml_mul_mat(
1183
- struct ggml_context * ctx,
1184
- struct ggml_tensor * a,
1185
- struct ggml_tensor * b);
1186
-
1187
- // change the precision of a matrix multiplication
1188
- // set to GGML_PREC_F32 for higher precision (useful for phi-2)
1189
- GGML_API void ggml_mul_mat_set_prec(
1190
- struct ggml_tensor * a,
1191
- enum ggml_prec prec);
1192
-
1193
- // indirect matrix multiplication
1194
- GGML_API struct ggml_tensor * ggml_mul_mat_id(
1195
- struct ggml_context * ctx,
1196
- struct ggml_tensor * as,
1197
- struct ggml_tensor * b,
1198
- struct ggml_tensor * ids);
1199
-
1200
- // A: m columns, n rows,
1201
- // B: p columns, n rows,
1202
- // result is m columns, p rows
1203
- GGML_API struct ggml_tensor * ggml_out_prod(
1204
- struct ggml_context * ctx,
1205
- struct ggml_tensor * a,
1206
- struct ggml_tensor * b);
1207
-
1208
- //
1209
- // operations on tensors without backpropagation
1210
- //
1211
-
1212
- GGML_API struct ggml_tensor * ggml_scale(
1213
- struct ggml_context * ctx,
1214
- struct ggml_tensor * a,
1215
- float s);
1216
-
1217
- // in-place, returns view(a)
1218
- GGML_API struct ggml_tensor * ggml_scale_inplace(
1219
- struct ggml_context * ctx,
1220
- struct ggml_tensor * a,
1221
- float s);
1222
-
1223
- // b -> view(a,offset,nb1,nb2,3), return modified a
1224
- GGML_API struct ggml_tensor * ggml_set(
1225
- struct ggml_context * ctx,
1226
- struct ggml_tensor * a,
1227
- struct ggml_tensor * b,
1228
- size_t nb1,
1229
- size_t nb2,
1230
- size_t nb3,
1231
- size_t offset);
1232
-
1233
- // b -> view(a,offset,nb1,nb2,3), return view(a)
1234
- GGML_API struct ggml_tensor * ggml_set_inplace(
1235
- struct ggml_context * ctx,
1236
- struct ggml_tensor * a,
1237
- struct ggml_tensor * b,
1238
- size_t nb1,
1239
- size_t nb2,
1240
- size_t nb3,
1241
- size_t offset);
1242
-
1243
- GGML_API struct ggml_tensor * ggml_set_1d(
1244
- struct ggml_context * ctx,
1245
- struct ggml_tensor * a,
1246
- struct ggml_tensor * b,
1247
- size_t offset);
1248
-
1249
- GGML_API struct ggml_tensor * ggml_set_1d_inplace(
1250
- struct ggml_context * ctx,
1251
- struct ggml_tensor * a,
1252
- struct ggml_tensor * b,
1253
- size_t offset);
1254
-
1255
- // b -> view(a,offset,nb1,nb2,3), return modified a
1256
- GGML_API struct ggml_tensor * ggml_set_2d(
1257
- struct ggml_context * ctx,
1258
- struct ggml_tensor * a,
1259
- struct ggml_tensor * b,
1260
- size_t nb1,
1261
- size_t offset);
1262
-
1263
- // b -> view(a,offset,nb1,nb2,3), return view(a)
1264
- GGML_API struct ggml_tensor * ggml_set_2d_inplace(
1265
- struct ggml_context * ctx,
1266
- struct ggml_tensor * a,
1267
- struct ggml_tensor * b,
1268
- size_t nb1,
1269
- size_t offset);
1270
-
1271
- // a -> b, return view(b)
1272
- GGML_API struct ggml_tensor * ggml_cpy(
1273
- struct ggml_context * ctx,
1274
- struct ggml_tensor * a,
1275
- struct ggml_tensor * b);
1276
-
1277
- GGML_API struct ggml_tensor * ggml_cast(
1278
- struct ggml_context * ctx,
1279
- struct ggml_tensor * a,
1280
- enum ggml_type type);
1281
-
1282
- // make contiguous
1283
- GGML_API struct ggml_tensor * ggml_cont(
1284
- struct ggml_context * ctx,
1285
- struct ggml_tensor * a);
1286
-
1287
- // make contiguous, with new shape
1288
- GGML_API struct ggml_tensor * ggml_cont_1d(
1289
- struct ggml_context * ctx,
1290
- struct ggml_tensor * a,
1291
- int64_t ne0);
1292
-
1293
- GGML_API struct ggml_tensor * ggml_cont_2d(
1294
- struct ggml_context * ctx,
1295
- struct ggml_tensor * a,
1296
- int64_t ne0,
1297
- int64_t ne1);
1298
-
1299
- GGML_API struct ggml_tensor * ggml_cont_3d(
1300
- struct ggml_context * ctx,
1301
- struct ggml_tensor * a,
1302
- int64_t ne0,
1303
- int64_t ne1,
1304
- int64_t ne2);
1305
-
1306
- GGML_API struct ggml_tensor * ggml_cont_4d(
1307
- struct ggml_context * ctx,
1308
- struct ggml_tensor * a,
1309
- int64_t ne0,
1310
- int64_t ne1,
1311
- int64_t ne2,
1312
- int64_t ne3);
1313
-
1314
- // return view(a), b specifies the new shape
1315
- // TODO: when we start computing gradient, make a copy instead of view
1316
- GGML_API struct ggml_tensor * ggml_reshape(
1317
- struct ggml_context * ctx,
1318
- struct ggml_tensor * a,
1319
- struct ggml_tensor * b);
1320
-
1321
- // return view(a)
1322
- // TODO: when we start computing gradient, make a copy instead of view
1323
- GGML_API struct ggml_tensor * ggml_reshape_1d(
1324
- struct ggml_context * ctx,
1325
- struct ggml_tensor * a,
1326
- int64_t ne0);
1327
-
1328
- GGML_API struct ggml_tensor * ggml_reshape_2d(
1329
- struct ggml_context * ctx,
1330
- struct ggml_tensor * a,
1331
- int64_t ne0,
1332
- int64_t ne1);
1333
-
1334
- // return view(a)
1335
- // TODO: when we start computing gradient, make a copy instead of view
1336
- GGML_API struct ggml_tensor * ggml_reshape_3d(
1337
- struct ggml_context * ctx,
1338
- struct ggml_tensor * a,
1339
- int64_t ne0,
1340
- int64_t ne1,
1341
- int64_t ne2);
1342
-
1343
- GGML_API struct ggml_tensor * ggml_reshape_4d(
1344
- struct ggml_context * ctx,
1345
- struct ggml_tensor * a,
1346
- int64_t ne0,
1347
- int64_t ne1,
1348
- int64_t ne2,
1349
- int64_t ne3);
1350
-
1351
- // offset in bytes
1352
- GGML_API struct ggml_tensor * ggml_view_1d(
1353
- struct ggml_context * ctx,
1354
- struct ggml_tensor * a,
1355
- int64_t ne0,
1356
- size_t offset);
1357
-
1358
- GGML_API struct ggml_tensor * ggml_view_2d(
1359
- struct ggml_context * ctx,
1360
- struct ggml_tensor * a,
1361
- int64_t ne0,
1362
- int64_t ne1,
1363
- size_t nb1, // row stride in bytes
1364
- size_t offset);
1365
-
1366
- GGML_API struct ggml_tensor * ggml_view_3d(
1367
- struct ggml_context * ctx,
1368
- struct ggml_tensor * a,
1369
- int64_t ne0,
1370
- int64_t ne1,
1371
- int64_t ne2,
1372
- size_t nb1, // row stride in bytes
1373
- size_t nb2, // slice stride in bytes
1374
- size_t offset);
1375
-
1376
- GGML_API struct ggml_tensor * ggml_view_4d(
1377
- struct ggml_context * ctx,
1378
- struct ggml_tensor * a,
1379
- int64_t ne0,
1380
- int64_t ne1,
1381
- int64_t ne2,
1382
- int64_t ne3,
1383
- size_t nb1, // row stride in bytes
1384
- size_t nb2, // slice stride in bytes
1385
- size_t nb3,
1386
- size_t offset);
1387
-
1388
- GGML_API struct ggml_tensor * ggml_permute(
1389
- struct ggml_context * ctx,
1390
- struct ggml_tensor * a,
1391
- int axis0,
1392
- int axis1,
1393
- int axis2,
1394
- int axis3);
1395
-
1396
- // alias for ggml_permute(ctx, a, 1, 0, 2, 3)
1397
- GGML_API struct ggml_tensor * ggml_transpose(
1398
- struct ggml_context * ctx,
1399
- struct ggml_tensor * a);
1400
-
1401
- // supports 3D: a->ne[2] == b->ne[1]
1402
- GGML_API struct ggml_tensor * ggml_get_rows(
1403
- struct ggml_context * ctx,
1404
- struct ggml_tensor * a,
1405
- struct ggml_tensor * b);
1406
-
1407
- GGML_API struct ggml_tensor * ggml_get_rows_back(
1408
- struct ggml_context * ctx,
1409
- struct ggml_tensor * a,
1410
- struct ggml_tensor * b,
1411
- struct ggml_tensor * c);
1412
-
1413
- GGML_API struct ggml_tensor * ggml_diag(
1414
- struct ggml_context * ctx,
1415
- struct ggml_tensor * a);
1416
-
1417
- // set elements above the diagonal to -INF
1418
- GGML_API struct ggml_tensor * ggml_diag_mask_inf(
1419
- struct ggml_context * ctx,
1420
- struct ggml_tensor * a,
1421
- int n_past);
1422
-
1423
- // in-place, returns view(a)
1424
- GGML_API struct ggml_tensor * ggml_diag_mask_inf_inplace(
1425
- struct ggml_context * ctx,
1426
- struct ggml_tensor * a,
1427
- int n_past);
1428
-
1429
- // set elements above the diagonal to 0
1430
- GGML_API struct ggml_tensor * ggml_diag_mask_zero(
1431
- struct ggml_context * ctx,
1432
- struct ggml_tensor * a,
1433
- int n_past);
1434
-
1435
- // in-place, returns view(a)
1436
- GGML_API struct ggml_tensor * ggml_diag_mask_zero_inplace(
1437
- struct ggml_context * ctx,
1438
- struct ggml_tensor * a,
1439
- int n_past);
1440
-
1441
- GGML_API struct ggml_tensor * ggml_soft_max(
1442
- struct ggml_context * ctx,
1443
- struct ggml_tensor * a);
1444
-
1445
- // in-place, returns view(a)
1446
- GGML_API struct ggml_tensor * ggml_soft_max_inplace(
1447
- struct ggml_context * ctx,
1448
- struct ggml_tensor * a);
1449
-
1450
- // fused soft_max(a*scale + mask*(ALiBi slope))
1451
- // mask is optional
1452
- // max_bias = 0.0f for no ALiBi
1453
- GGML_API struct ggml_tensor * ggml_soft_max_ext(
1454
- struct ggml_context * ctx,
1455
- struct ggml_tensor * a,
1456
- struct ggml_tensor * mask,
1457
- float scale,
1458
- float max_bias);
1459
-
1460
- GGML_API struct ggml_tensor * ggml_soft_max_back(
1461
- struct ggml_context * ctx,
1462
- struct ggml_tensor * a,
1463
- struct ggml_tensor * b);
1464
-
1465
- // in-place, returns view(a)
1466
- GGML_API struct ggml_tensor * ggml_soft_max_back_inplace(
1467
- struct ggml_context * ctx,
1468
- struct ggml_tensor * a,
1469
- struct ggml_tensor * b);
1470
-
1471
- // rotary position embedding
1472
- // if mode & 1 == 1, skip n_past elements (NOT SUPPORTED)
1473
- // if mode & 2 == 1, GPT-NeoX style
1474
- //
1475
- // b is an int32 vector with size a->ne[2], it contains the positions
1476
- // c is freq factors (e.g. phi3-128k), (optional)
1477
- GGML_API struct ggml_tensor * ggml_rope(
1478
- struct ggml_context * ctx,
1479
- struct ggml_tensor * a,
1480
- struct ggml_tensor * b,
1481
- int n_dims,
1482
- int mode);
1483
-
1484
- // in-place, returns view(a)
1485
- GGML_API struct ggml_tensor * ggml_rope_inplace(
1486
- struct ggml_context * ctx,
1487
- struct ggml_tensor * a,
1488
- struct ggml_tensor * b,
1489
- int n_dims,
1490
- int mode);
1491
-
1492
- // custom RoPE
1493
- GGML_API struct ggml_tensor * ggml_rope_ext(
1494
- struct ggml_context * ctx,
1495
- struct ggml_tensor * a,
1496
- struct ggml_tensor * b,
1497
- struct ggml_tensor * c,
1498
- int n_dims,
1499
- int mode,
1500
- int n_ctx_orig,
1501
- float freq_base,
1502
- float freq_scale,
1503
- float ext_factor,
1504
- float attn_factor,
1505
- float beta_fast,
1506
- float beta_slow);
1507
-
1508
- // in-place, returns view(a)
1509
- GGML_API struct ggml_tensor * ggml_rope_ext_inplace(
1510
- struct ggml_context * ctx,
1511
- struct ggml_tensor * a,
1512
- struct ggml_tensor * b,
1513
- struct ggml_tensor * c,
1514
- int n_dims,
1515
- int mode,
1516
- int n_ctx_orig,
1517
- float freq_base,
1518
- float freq_scale,
1519
- float ext_factor,
1520
- float attn_factor,
1521
- float beta_fast,
1522
- float beta_slow);
1523
-
1524
- GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_rope_custom(
1525
- struct ggml_context * ctx,
1526
- struct ggml_tensor * a,
1527
- struct ggml_tensor * b,
1528
- int n_dims,
1529
- int mode,
1530
- int n_ctx_orig,
1531
- float freq_base,
1532
- float freq_scale,
1533
- float ext_factor,
1534
- float attn_factor,
1535
- float beta_fast,
1536
- float beta_slow),
1537
- "use ggml_rope_ext instead");
1538
-
1539
- GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_rope_custom_inplace(
1540
- struct ggml_context * ctx,
1541
- struct ggml_tensor * a,
1542
- struct ggml_tensor * b,
1543
- int n_dims,
1544
- int mode,
1545
- int n_ctx_orig,
1546
- float freq_base,
1547
- float freq_scale,
1548
- float ext_factor,
1549
- float attn_factor,
1550
- float beta_fast,
1551
- float beta_slow),
1552
- "use ggml_rope_ext_inplace instead");
1553
-
1554
- // compute correction dims for YaRN RoPE scaling
1555
- GGML_CALL void ggml_rope_yarn_corr_dims(
1556
- int n_dims, int n_ctx_orig, float freq_base, float beta_fast, float beta_slow, float dims[2]);
1557
-
1558
- // rotary position embedding backward, i.e compute dx from dy
1559
- // a - dy
1560
- GGML_API struct ggml_tensor * ggml_rope_back(
1561
- struct ggml_context * ctx,
1562
- struct ggml_tensor * a,
1563
- struct ggml_tensor * b,
1564
- struct ggml_tensor * c,
1565
- int n_dims,
1566
- int mode,
1567
- int n_ctx_orig,
1568
- float freq_base,
1569
- float freq_scale,
1570
- float ext_factor,
1571
- float attn_factor,
1572
- float beta_fast,
1573
- float beta_slow);
1574
-
1575
- // clamp
1576
- // in-place, returns view(a)
1577
- GGML_API struct ggml_tensor * ggml_clamp(
1578
- struct ggml_context * ctx,
1579
- struct ggml_tensor * a,
1580
- float min,
1581
- float max);
1582
-
1583
- GGML_API struct ggml_tensor * ggml_im2col(
1584
- struct ggml_context * ctx,
1585
- struct ggml_tensor * a,
1586
- struct ggml_tensor * b,
1587
- int s0,
1588
- int s1,
1589
- int p0,
1590
- int p1,
1591
- int d0,
1592
- int d1,
1593
- bool is_2D,
1594
- enum ggml_type dst_type);
1595
-
1596
- GGML_API struct ggml_tensor * ggml_conv_depthwise_2d(
1597
- struct ggml_context * ctx,
1598
- struct ggml_tensor * a,
1599
- struct ggml_tensor * b,
1600
- int s0,
1601
- int s1,
1602
- int p0,
1603
- int p1,
1604
- int d0,
1605
- int d1);
1606
-
1607
- GGML_API struct ggml_tensor * ggml_conv_1d(
1608
- struct ggml_context * ctx,
1609
- struct ggml_tensor * a,
1610
- struct ggml_tensor * b,
1611
- int s0, // stride
1612
- int p0, // padding
1613
- int d0); // dilation
1614
-
1615
- // conv_1d with padding = half
1616
- // alias for ggml_conv_1d(a, b, s, a->ne[0]/2, d)
1617
- GGML_API struct ggml_tensor* ggml_conv_1d_ph(
1618
- struct ggml_context * ctx,
1619
- struct ggml_tensor * a,
1620
- struct ggml_tensor * b,
1621
- int s,
1622
- int d);
1623
-
1624
- GGML_API struct ggml_tensor * ggml_conv_transpose_1d(
1625
- struct ggml_context * ctx,
1626
- struct ggml_tensor * a,
1627
- struct ggml_tensor * b,
1628
- int s0,
1629
- int p0,
1630
- int d0);
1631
-
1632
- GGML_API struct ggml_tensor * ggml_conv_2d(
1633
- struct ggml_context * ctx,
1634
- struct ggml_tensor * a,
1635
- struct ggml_tensor * b,
1636
- int s0,
1637
- int s1,
1638
- int p0,
1639
- int p1,
1640
- int d0,
1641
- int d1);
1642
-
1643
-
1644
- // kernel size is a->ne[0] x a->ne[1]
1645
- // stride is equal to kernel size
1646
- // padding is zero
1647
- // example:
1648
- // a: 16 16 3 768
1649
- // b: 1024 1024 3 1
1650
- // res: 64 64 768 1
1651
- // used in sam
1652
- GGML_API struct ggml_tensor * ggml_conv_2d_sk_p0(
1653
- struct ggml_context * ctx,
1654
- struct ggml_tensor * a,
1655
- struct ggml_tensor * b);
1656
-
1657
- // kernel size is a->ne[0] x a->ne[1]
1658
- // stride is 1
1659
- // padding is half
1660
- // example:
1661
- // a: 3 3 256 256
1662
- // b: 64 64 256 1
1663
- // res: 64 64 256 1
1664
- // used in sam
1665
- GGML_API struct ggml_tensor * ggml_conv_2d_s1_ph(
1666
- struct ggml_context * ctx,
1667
- struct ggml_tensor * a,
1668
- struct ggml_tensor * b);
1669
-
1670
- GGML_API struct ggml_tensor * ggml_conv_transpose_2d_p0(
1671
- struct ggml_context * ctx,
1672
- struct ggml_tensor * a,
1673
- struct ggml_tensor * b,
1674
- int stride);
1675
-
1676
- enum ggml_op_pool {
1677
- GGML_OP_POOL_MAX,
1678
- GGML_OP_POOL_AVG,
1679
- GGML_OP_POOL_COUNT,
1680
- };
1681
-
1682
- GGML_API struct ggml_tensor * ggml_pool_1d(
1683
- struct ggml_context * ctx,
1684
- struct ggml_tensor * a,
1685
- enum ggml_op_pool op,
1686
- int k0, // kernel size
1687
- int s0, // stride
1688
- int p0); // padding
1689
-
1690
- // the result will have 2*p0 padding for the first dimension
1691
- // and 2*p1 padding for the second dimension
1692
- GGML_API struct ggml_tensor * ggml_pool_2d(
1693
- struct ggml_context * ctx,
1694
- struct ggml_tensor * a,
1695
- enum ggml_op_pool op,
1696
- int k0,
1697
- int k1,
1698
- int s0,
1699
- int s1,
1700
- float p0,
1701
- float p1);
1702
-
1703
- // nearest interpolate
1704
- // multiplies ne0 and ne1 by scale factor
1705
- // used in stable-diffusion
1706
- GGML_API struct ggml_tensor * ggml_upscale(
1707
- struct ggml_context * ctx,
1708
- struct ggml_tensor * a,
1709
- int scale_factor);
1710
-
1711
- // nearest interpolate
1712
- // nearest interpolate to specified dimensions
1713
- // used in tortoise.cpp
1714
- GGML_API struct ggml_tensor * ggml_upscale_ext(
1715
- struct ggml_context * ctx,
1716
- struct ggml_tensor * a,
1717
- int ne0,
1718
- int ne1,
1719
- int ne2,
1720
- int ne3);
1721
-
1722
- // pad each dimension with zeros: [x, ..., x] -> [x, ..., x, 0, ..., 0]
1723
- GGML_API struct ggml_tensor * ggml_pad(
1724
- struct ggml_context * ctx,
1725
- struct ggml_tensor * a,
1726
- int p0,
1727
- int p1,
1728
- int p2,
1729
- int p3);
1730
-
1731
- // Ref: https://github.com/CompVis/stable-diffusion/blob/main/ldm/modules/diffusionmodules/util.py#L151
1732
- // timesteps: [N,]
1733
- // return: [N, dim]
1734
- GGML_API struct ggml_tensor * ggml_timestep_embedding(
1735
- struct ggml_context * ctx,
1736
- struct ggml_tensor * timesteps,
1737
- int dim,
1738
- int max_period);
1739
-
1740
- // sort rows
1741
- enum ggml_sort_order {
1742
- GGML_SORT_ORDER_ASC,
1743
- GGML_SORT_ORDER_DESC,
1744
- };
1745
-
1746
- GGML_API struct ggml_tensor * ggml_argsort(
1747
- struct ggml_context * ctx,
1748
- struct ggml_tensor * a,
1749
- enum ggml_sort_order order);
1750
-
1751
- GGML_API struct ggml_tensor * ggml_arange(
1752
- struct ggml_context * ctx,
1753
- float start,
1754
- float stop,
1755
- float step);
1756
-
1757
- // top k elements per row
1758
- GGML_API struct ggml_tensor * ggml_top_k(
1759
- struct ggml_context * ctx,
1760
- struct ggml_tensor * a,
1761
- int k);
1762
-
1763
- #define GGML_KQ_MASK_PAD 32
1764
-
1765
- // q: [n_embd, n_batch, n_head, 1]
1766
- // k: [n_embd, n_kv, n_head_kv, 1]
1767
- // v: [n_embd, n_kv, n_head_kv, 1] !! not transposed !!
1768
- // mask: [n_kv, n_batch_pad, 1, 1] !! n_batch_pad = GGML_PAD(n_batch, GGML_KQ_MASK_PAD) !!
1769
- // res: [n_embd, n_head, n_batch, 1] !! permuted !!
1770
- GGML_API struct ggml_tensor * ggml_flash_attn_ext(
1771
- struct ggml_context * ctx,
1772
- struct ggml_tensor * q,
1773
- struct ggml_tensor * k,
1774
- struct ggml_tensor * v,
1775
- struct ggml_tensor * mask,
1776
- float scale,
1777
- float max_bias);
1778
-
1779
- GGML_API void ggml_flash_attn_ext_set_prec(
1780
- struct ggml_tensor * a,
1781
- enum ggml_prec prec);
1782
-
1783
- // TODO: needs to be adapted to ggml_flash_attn_ext
1784
- GGML_API struct ggml_tensor * ggml_flash_attn_back(
1785
- struct ggml_context * ctx,
1786
- struct ggml_tensor * q,
1787
- struct ggml_tensor * k,
1788
- struct ggml_tensor * v,
1789
- struct ggml_tensor * d,
1790
- bool masked);
1791
-
1792
- GGML_API struct ggml_tensor * ggml_ssm_conv(
1793
- struct ggml_context * ctx,
1794
- struct ggml_tensor * s,
1795
- struct ggml_tensor * x,
1796
- struct ggml_tensor * c,
1797
- struct ggml_tensor * sq);
1798
-
1799
- GGML_API struct ggml_tensor * ggml_ssm_scan(
1800
- struct ggml_context * ctx,
1801
- struct ggml_tensor * s,
1802
- struct ggml_tensor * x,
1803
- struct ggml_tensor * dt,
1804
- struct ggml_tensor * A,
1805
- struct ggml_tensor * B,
1806
- struct ggml_tensor * C,
1807
- struct ggml_tensor * sq);
1808
-
1809
- // partition into non-overlapping windows with padding if needed
1810
- // example:
1811
- // a: 768 64 64 1
1812
- // w: 14
1813
- // res: 768 14 14 25
1814
- // used in sam
1815
- GGML_API struct ggml_tensor * ggml_win_part(
1816
- struct ggml_context * ctx,
1817
- struct ggml_tensor * a,
1818
- int w);
1819
-
1820
- // reverse of ggml_win_part
1821
- // used in sam
1822
- GGML_API struct ggml_tensor * ggml_win_unpart(
1823
- struct ggml_context * ctx,
1824
- struct ggml_tensor * a,
1825
- int w0,
1826
- int h0,
1827
- int w);
1828
-
1829
- GGML_API struct ggml_tensor * ggml_unary(
1830
- struct ggml_context * ctx,
1831
- struct ggml_tensor * a,
1832
- enum ggml_unary_op op);
1833
-
1834
- GGML_API struct ggml_tensor * ggml_unary_inplace(
1835
- struct ggml_context * ctx,
1836
- struct ggml_tensor * a,
1837
- enum ggml_unary_op op);
1838
-
1839
- // used in sam
1840
- GGML_API struct ggml_tensor * ggml_get_rel_pos(
1841
- struct ggml_context * ctx,
1842
- struct ggml_tensor * a,
1843
- int qh,
1844
- int kh);
1845
-
1846
- // used in sam
1847
- GGML_API struct ggml_tensor * ggml_add_rel_pos(
1848
- struct ggml_context * ctx,
1849
- struct ggml_tensor * a,
1850
- struct ggml_tensor * pw,
1851
- struct ggml_tensor * ph);
1852
-
1853
- GGML_API struct ggml_tensor * ggml_add_rel_pos_inplace(
1854
- struct ggml_context * ctx,
1855
- struct ggml_tensor * a,
1856
- struct ggml_tensor * pw,
1857
- struct ggml_tensor * ph);
1858
-
1859
- // custom operators
1860
-
1861
- typedef void (*ggml_unary_op_f32_t) (const int, float *, const float *);
1862
- typedef void (*ggml_binary_op_f32_t)(const int, float *, const float *, const float *);
1863
-
1864
- typedef void (*ggml_custom1_op_f32_t)(struct ggml_tensor *, const struct ggml_tensor *);
1865
- typedef void (*ggml_custom2_op_f32_t)(struct ggml_tensor *, const struct ggml_tensor *, const struct ggml_tensor *);
1866
- typedef void (*ggml_custom3_op_f32_t)(struct ggml_tensor *, const struct ggml_tensor *, const struct ggml_tensor *, const struct ggml_tensor *);
1867
-
1868
- GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_map_unary_f32(
1869
- struct ggml_context * ctx,
1870
- struct ggml_tensor * a,
1871
- ggml_unary_op_f32_t fun),
1872
- "use ggml_map_custom1 instead");
1873
-
1874
- GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_map_unary_inplace_f32(
1875
- struct ggml_context * ctx,
1876
- struct ggml_tensor * a,
1877
- ggml_unary_op_f32_t fun),
1878
- "use ggml_map_custom1_inplace instead");
1879
-
1880
- GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_map_binary_f32(
1881
- struct ggml_context * ctx,
1882
- struct ggml_tensor * a,
1883
- struct ggml_tensor * b,
1884
- ggml_binary_op_f32_t fun),
1885
- "use ggml_map_custom2 instead");
1886
-
1887
- GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_map_binary_inplace_f32(
1888
- struct ggml_context * ctx,
1889
- struct ggml_tensor * a,
1890
- struct ggml_tensor * b,
1891
- ggml_binary_op_f32_t fun),
1892
- "use ggml_map_custom2_inplace instead");
1893
-
1894
- GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_map_custom1_f32(
1895
- struct ggml_context * ctx,
1896
- struct ggml_tensor * a,
1897
- ggml_custom1_op_f32_t fun),
1898
- "use ggml_map_custom1 instead");
1899
-
1900
- GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_map_custom1_inplace_f32(
1901
- struct ggml_context * ctx,
1902
- struct ggml_tensor * a,
1903
- ggml_custom1_op_f32_t fun),
1904
- "use ggml_map_custom1_inplace instead");
1905
-
1906
- GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_map_custom2_f32(
1907
- struct ggml_context * ctx,
1908
- struct ggml_tensor * a,
1909
- struct ggml_tensor * b,
1910
- ggml_custom2_op_f32_t fun),
1911
- "use ggml_map_custom2 instead");
1912
-
1913
- GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_map_custom2_inplace_f32(
1914
- struct ggml_context * ctx,
1915
- struct ggml_tensor * a,
1916
- struct ggml_tensor * b,
1917
- ggml_custom2_op_f32_t fun),
1918
- "use ggml_map_custom2_inplace instead");
1919
-
1920
- GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_map_custom3_f32(
1921
- struct ggml_context * ctx,
1922
- struct ggml_tensor * a,
1923
- struct ggml_tensor * b,
1924
- struct ggml_tensor * c,
1925
- ggml_custom3_op_f32_t fun),
1926
- "use ggml_map_custom3 instead");
1927
-
1928
- GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_map_custom3_inplace_f32(
1929
- struct ggml_context * ctx,
1930
- struct ggml_tensor * a,
1931
- struct ggml_tensor * b,
1932
- struct ggml_tensor * c,
1933
- ggml_custom3_op_f32_t fun),
1934
- "use ggml_map_custom3_inplace instead");
1935
-
1936
- // custom operators v2
1937
-
1938
- typedef void (*ggml_custom1_op_t)(struct ggml_tensor * dst , const struct ggml_tensor * a, int ith, int nth, void * userdata);
1939
- typedef void (*ggml_custom2_op_t)(struct ggml_tensor * dst , const struct ggml_tensor * a, const struct ggml_tensor * b, int ith, int nth, void * userdata);
1940
- typedef void (*ggml_custom3_op_t)(struct ggml_tensor * dst , const struct ggml_tensor * a, const struct ggml_tensor * b, const struct ggml_tensor * c, int ith, int nth, void * userdata);
1941
-
1942
- #define GGML_N_TASKS_MAX -1
1943
-
1944
- GGML_API struct ggml_tensor * ggml_map_custom1(
1945
- struct ggml_context * ctx,
1946
- struct ggml_tensor * a,
1947
- ggml_custom1_op_t fun,
1948
- int n_tasks,
1949
- void * userdata);
1950
-
1951
- GGML_API struct ggml_tensor * ggml_map_custom1_inplace(
1952
- struct ggml_context * ctx,
1953
- struct ggml_tensor * a,
1954
- ggml_custom1_op_t fun,
1955
- int n_tasks,
1956
- void * userdata);
1957
-
1958
- GGML_API struct ggml_tensor * ggml_map_custom2(
1959
- struct ggml_context * ctx,
1960
- struct ggml_tensor * a,
1961
- struct ggml_tensor * b,
1962
- ggml_custom2_op_t fun,
1963
- int n_tasks,
1964
- void * userdata);
1965
-
1966
- GGML_API struct ggml_tensor * ggml_map_custom2_inplace(
1967
- struct ggml_context * ctx,
1968
- struct ggml_tensor * a,
1969
- struct ggml_tensor * b,
1970
- ggml_custom2_op_t fun,
1971
- int n_tasks,
1972
- void * userdata);
1973
-
1974
- GGML_API struct ggml_tensor * ggml_map_custom3(
1975
- struct ggml_context * ctx,
1976
- struct ggml_tensor * a,
1977
- struct ggml_tensor * b,
1978
- struct ggml_tensor * c,
1979
- ggml_custom3_op_t fun,
1980
- int n_tasks,
1981
- void * userdata);
1982
-
1983
- GGML_API struct ggml_tensor * ggml_map_custom3_inplace(
1984
- struct ggml_context * ctx,
1985
- struct ggml_tensor * a,
1986
- struct ggml_tensor * b,
1987
- struct ggml_tensor * c,
1988
- ggml_custom3_op_t fun,
1989
- int n_tasks,
1990
- void * userdata);
1991
-
1992
- // loss function
1993
-
1994
- GGML_API struct ggml_tensor * ggml_cross_entropy_loss(
1995
- struct ggml_context * ctx,
1996
- struct ggml_tensor * a,
1997
- struct ggml_tensor * b);
1998
-
1999
- GGML_API struct ggml_tensor * ggml_cross_entropy_loss_back(
2000
- struct ggml_context * ctx,
2001
- struct ggml_tensor * a,
2002
- struct ggml_tensor * b,
2003
- struct ggml_tensor * c);
2004
-
2005
- //
2006
- // automatic differentiation
2007
- //
2008
-
2009
- GGML_API void ggml_set_param(
2010
- struct ggml_context * ctx,
2011
- struct ggml_tensor * tensor);
2012
-
2013
-
2014
- GGML_API void ggml_build_forward_expand (struct ggml_cgraph * cgraph, struct ggml_tensor * tensor);
2015
- GGML_API void ggml_build_backward_expand(struct ggml_context * ctx, struct ggml_cgraph * gf, struct ggml_cgraph * gb, bool keep);
2016
-
2017
- // graph allocation in a context
2018
- GGML_API struct ggml_cgraph * ggml_new_graph (struct ggml_context * ctx); // size = GGML_DEFAULT_GRAPH_SIZE, grads = false
2019
- GGML_API struct ggml_cgraph * ggml_new_graph_custom (struct ggml_context * ctx, size_t size, bool grads);
2020
- GGML_API struct ggml_cgraph * ggml_graph_dup (struct ggml_context * ctx, struct ggml_cgraph * cgraph);
2021
- GGML_API struct ggml_cgraph ggml_graph_view (struct ggml_cgraph * cgraph, int i0, int i1);
2022
- GGML_API void ggml_graph_cpy (struct ggml_cgraph * src, struct ggml_cgraph * dst);
2023
- GGML_API void ggml_graph_reset (struct ggml_cgraph * cgraph); // zero grads
2024
- GGML_API void ggml_graph_clear (struct ggml_cgraph * cgraph);
2025
-
2026
- GGML_API size_t ggml_graph_overhead(void);
2027
- GGML_API size_t ggml_graph_overhead_custom(size_t size, bool grads);
2028
-
2029
- // ggml_graph_plan() has to be called before ggml_graph_compute()
2030
- // when plan.work_size > 0, caller must allocate memory for plan.work_data
2031
- GGML_API struct ggml_cplan ggml_graph_plan (const struct ggml_cgraph * cgraph, int n_threads /*= GGML_DEFAULT_N_THREADS*/);
2032
- GGML_API enum ggml_status ggml_graph_compute ( struct ggml_cgraph * cgraph, struct ggml_cplan * cplan);
2033
- // same as ggml_graph_compute() but the work data is allocated as a part of the context
2034
- // note: the drawback of this API is that you must have ensured that the context has enough memory for the work data
2035
- GGML_API enum ggml_status ggml_graph_compute_with_ctx(struct ggml_context * ctx, struct ggml_cgraph * cgraph, int n_threads);
2036
-
2037
- GGML_API struct ggml_tensor * ggml_graph_get_tensor(struct ggml_cgraph * cgraph, const char * name);
2038
-
2039
- GGML_API void ggml_graph_export(const struct ggml_cgraph * cgraph, const char * fname);
2040
- GGML_API struct ggml_cgraph * ggml_graph_import(const char * fname, struct ggml_context ** ctx_data, struct ggml_context ** ctx_eval);
2041
-
2042
- // print info and performance information for the graph
2043
- GGML_API void ggml_graph_print(const struct ggml_cgraph * cgraph);
2044
-
2045
- // dump the graph into a file using the dot format
2046
- GGML_API void ggml_graph_dump_dot(const struct ggml_cgraph * gb, const struct ggml_cgraph * gf, const char * filename);
2047
-
2048
- // build gradient checkpointing backward graph gb for gf using provided checkpoints
2049
- // gb_tmp will contain original backward graph with rewritten backward process nodes,
2050
- // but without the second forward pass nodes.
2051
- GGML_API void ggml_build_backward_gradient_checkpointing(
2052
- struct ggml_context * ctx,
2053
- struct ggml_cgraph * gf,
2054
- struct ggml_cgraph * gb,
2055
- struct ggml_cgraph * gb_tmp,
2056
- struct ggml_tensor * * checkpoints,
2057
- int n_checkpoints);
2058
- //
2059
- // optimization
2060
- //
2061
-
2062
- // optimization methods
2063
- enum ggml_opt_type {
2064
- GGML_OPT_TYPE_ADAM,
2065
- GGML_OPT_TYPE_LBFGS,
2066
- };
2067
-
2068
- // linesearch methods
2069
- enum ggml_linesearch {
2070
- GGML_LINESEARCH_DEFAULT = 1,
2071
-
2072
- GGML_LINESEARCH_BACKTRACKING_ARMIJO = 0,
2073
- GGML_LINESEARCH_BACKTRACKING_WOLFE = 1,
2074
- GGML_LINESEARCH_BACKTRACKING_STRONG_WOLFE = 2,
2075
- };
2076
-
2077
- // optimization return values
2078
- enum ggml_opt_result {
2079
- GGML_OPT_RESULT_OK = 0,
2080
- GGML_OPT_RESULT_DID_NOT_CONVERGE,
2081
- GGML_OPT_RESULT_NO_CONTEXT,
2082
- GGML_OPT_RESULT_INVALID_WOLFE,
2083
- GGML_OPT_RESULT_FAIL,
2084
- GGML_OPT_RESULT_CANCEL,
2085
-
2086
- GGML_LINESEARCH_FAIL = -128,
2087
- GGML_LINESEARCH_MINIMUM_STEP,
2088
- GGML_LINESEARCH_MAXIMUM_STEP,
2089
- GGML_LINESEARCH_MAXIMUM_ITERATIONS,
2090
- GGML_LINESEARCH_INVALID_PARAMETERS,
2091
- };
2092
-
2093
- typedef void (*ggml_opt_callback)(void * data, int accum_step, float * sched, bool * cancel);
2094
- typedef void (*ggml_log_callback)(enum ggml_log_level level, const char * text, void * user_data);
2095
-
2096
- // optimization parameters
2097
- //
2098
- // see ggml.c (ggml_opt_default_params) for default values
2099
- //
2100
- struct ggml_opt_params {
2101
- enum ggml_opt_type type;
2102
-
2103
- size_t graph_size;
2104
-
2105
- int n_threads;
2106
-
2107
- // delta-based convergence test
2108
- //
2109
- // if past == 0 - disabled
2110
- // if past > 0:
2111
- // stop if |f(x) - f(x_past)| < delta * max(1, |f(x)|)
2112
- //
2113
- int past;
2114
- float delta;
2115
-
2116
- // maximum number of iterations without improvement
2117
- //
2118
- // if 0 - disabled
2119
- // if > 0:
2120
- // assume convergence if no cost improvement in this number of iterations
2121
- //
2122
- int max_no_improvement;
2123
-
2124
- bool print_forward_graph;
2125
- bool print_backward_graph;
2126
-
2127
- int n_gradient_accumulation;
2128
-
2129
- // ADAM parameters
2130
- struct {
2131
- int n_iter;
2132
-
2133
- float sched; // schedule multiplier (fixed, decay or warmup)
2134
- float decay; // weight decay for AdamW, use 0.0f to disable
2135
- int decay_min_ndim; // minimum number of tensor dimension to apply weight decay
2136
- float alpha; // learning rate
2137
- float beta1;
2138
- float beta2;
2139
- float eps; // epsilon for numerical stability
2140
- float eps_f; // epsilon for convergence test
2141
- float eps_g; // epsilon for convergence test
2142
- float gclip; // gradient clipping
2143
- } adam;
2144
-
2145
- // LBFGS parameters
2146
- struct {
2147
- int m; // number of corrections to approximate the inv. Hessian
2148
- int n_iter;
2149
- int max_linesearch;
2150
-
2151
- float eps; // convergence tolerance
2152
- float ftol; // line search tolerance
2153
- float wolfe;
2154
- float min_step;
2155
- float max_step;
2156
-
2157
- enum ggml_linesearch linesearch;
2158
- } lbfgs;
2159
- };
2160
-
2161
- struct ggml_opt_context {
2162
- struct ggml_context * ctx;
2163
- struct ggml_opt_params params;
2164
-
2165
- int iter;
2166
- int64_t nx; // number of parameter elements
2167
-
2168
- bool just_initialized;
2169
-
2170
- float loss_before;
2171
- float loss_after;
2172
-
2173
- struct {
2174
- struct ggml_tensor * g; // current gradient
2175
- struct ggml_tensor * m; // first moment
2176
- struct ggml_tensor * v; // second moment
2177
- struct ggml_tensor * pf; // past function values
2178
- float fx_best;
2179
- float fx_prev;
2180
- int n_no_improvement;
2181
- } adam;
2182
-
2183
- struct {
2184
- struct ggml_tensor * x; // current parameters
2185
- struct ggml_tensor * xp; // previous parameters
2186
- struct ggml_tensor * g; // current gradient
2187
- struct ggml_tensor * gp; // previous gradient
2188
- struct ggml_tensor * d; // search direction
2189
- struct ggml_tensor * pf; // past function values
2190
- struct ggml_tensor * lmal; // the L-BFGS memory alpha
2191
- struct ggml_tensor * lmys; // the L-BFGS memory ys
2192
- struct ggml_tensor * lms; // the L-BFGS memory s
2193
- struct ggml_tensor * lmy; // the L-BFGS memory y
2194
- float fx_best;
2195
- float step;
2196
- int j;
2197
- int k;
2198
- int end;
2199
- int n_no_improvement;
2200
- } lbfgs;
2201
- };
2202
-
2203
- GGML_API struct ggml_opt_params ggml_opt_default_params(enum ggml_opt_type type);
2204
-
2205
- // optimize the function defined by the tensor f
2206
- GGML_API enum ggml_opt_result ggml_opt(
2207
- struct ggml_context * ctx,
2208
- struct ggml_opt_params params,
2209
- struct ggml_tensor * f);
2210
-
2211
- // initialize optimizer context
2212
- GGML_API void ggml_opt_init(
2213
- struct ggml_context * ctx,
2214
- struct ggml_opt_context * opt,
2215
- struct ggml_opt_params params,
2216
- int64_t nx);
2217
-
2218
- // continue optimizing the function defined by the tensor f
2219
- GGML_API enum ggml_opt_result ggml_opt_resume(
2220
- struct ggml_context * ctx,
2221
- struct ggml_opt_context * opt,
2222
- struct ggml_tensor * f);
2223
-
2224
- // continue optimizing the function defined by the tensor f
2225
- GGML_API enum ggml_opt_result ggml_opt_resume_g(
2226
- struct ggml_context * ctx,
2227
- struct ggml_opt_context * opt,
2228
- struct ggml_tensor * f,
2229
- struct ggml_cgraph * gf,
2230
- struct ggml_cgraph * gb,
2231
- ggml_opt_callback callback,
2232
- void * callback_data);
2233
-
2234
- //
2235
- // tensor flags
2236
- //
2237
- GGML_API void ggml_set_input(struct ggml_tensor * tensor);
2238
- GGML_API void ggml_set_output(struct ggml_tensor * tensor);
2239
-
2240
- //
2241
- // quantization
2242
- //
2243
-
2244
- // - ggml_quantize_init can be called multiple times with the same type
2245
- // it will only initialize the quantization tables for the first call or after ggml_quantize_free
2246
- // automatically called by ggml_quantize_chunk for convenience
2247
- //
2248
- // - ggml_quantize_free will free any memory allocated by ggml_quantize_init
2249
- // call this at the end of the program to avoid memory leaks
2250
- //
2251
- // note: these are thread-safe
2252
- //
2253
- GGML_API void ggml_quantize_init(enum ggml_type type);
2254
- GGML_API void ggml_quantize_free(void);
2255
-
2256
- // some quantization type cannot be used without an importance matrix
2257
- GGML_API bool ggml_quantize_requires_imatrix(enum ggml_type type);
2258
-
2259
- // calls ggml_quantize_init internally (i.e. can allocate memory)
2260
- GGML_API size_t ggml_quantize_chunk(
2261
- enum ggml_type type,
2262
- const float * src,
2263
- void * dst,
2264
- int64_t start,
2265
- int64_t nrows,
2266
- int64_t n_per_row,
2267
- const float * imatrix);
2268
-
2269
- //
2270
- // gguf
2271
- //
2272
-
2273
- enum gguf_type {
2274
- GGUF_TYPE_UINT8 = 0,
2275
- GGUF_TYPE_INT8 = 1,
2276
- GGUF_TYPE_UINT16 = 2,
2277
- GGUF_TYPE_INT16 = 3,
2278
- GGUF_TYPE_UINT32 = 4,
2279
- GGUF_TYPE_INT32 = 5,
2280
- GGUF_TYPE_FLOAT32 = 6,
2281
- GGUF_TYPE_BOOL = 7,
2282
- GGUF_TYPE_STRING = 8,
2283
- GGUF_TYPE_ARRAY = 9,
2284
- GGUF_TYPE_UINT64 = 10,
2285
- GGUF_TYPE_INT64 = 11,
2286
- GGUF_TYPE_FLOAT64 = 12,
2287
- GGUF_TYPE_COUNT, // marks the end of the enum
2288
- };
2289
-
2290
- struct gguf_context;
2291
-
2292
- struct gguf_init_params {
2293
- bool no_alloc;
2294
-
2295
- // if not NULL, create a ggml_context and allocate the tensor data in it
2296
- struct ggml_context ** ctx;
2297
- };
2298
-
2299
- GGML_API struct gguf_context * gguf_init_empty(void);
2300
- GGML_API struct gguf_context * gguf_init_from_file(const char * fname, struct gguf_init_params params);
2301
- //GGML_API struct gguf_context * gguf_init_from_buffer(..);
2302
-
2303
- GGML_API void gguf_free(struct gguf_context * ctx);
2304
-
2305
- GGML_API const char * gguf_type_name(enum gguf_type type);
2306
-
2307
- GGML_API int gguf_get_version (const struct gguf_context * ctx);
2308
- GGML_API size_t gguf_get_alignment (const struct gguf_context * ctx);
2309
- GGML_API size_t gguf_get_data_offset(const struct gguf_context * ctx);
2310
- GGML_API void * gguf_get_data (const struct gguf_context * ctx);
2311
-
2312
- GGML_API int gguf_get_n_kv(const struct gguf_context * ctx);
2313
- GGML_API int gguf_find_key(const struct gguf_context * ctx, const char * key);
2314
- GGML_API const char * gguf_get_key (const struct gguf_context * ctx, int key_id);
2315
-
2316
- GGML_API enum gguf_type gguf_get_kv_type (const struct gguf_context * ctx, int key_id);
2317
- GGML_API enum gguf_type gguf_get_arr_type(const struct gguf_context * ctx, int key_id);
2318
-
2319
- // will abort if the wrong type is used for the key
2320
- GGML_API uint8_t gguf_get_val_u8 (const struct gguf_context * ctx, int key_id);
2321
- GGML_API int8_t gguf_get_val_i8 (const struct gguf_context * ctx, int key_id);
2322
- GGML_API uint16_t gguf_get_val_u16 (const struct gguf_context * ctx, int key_id);
2323
- GGML_API int16_t gguf_get_val_i16 (const struct gguf_context * ctx, int key_id);
2324
- GGML_API uint32_t gguf_get_val_u32 (const struct gguf_context * ctx, int key_id);
2325
- GGML_API int32_t gguf_get_val_i32 (const struct gguf_context * ctx, int key_id);
2326
- GGML_API float gguf_get_val_f32 (const struct gguf_context * ctx, int key_id);
2327
- GGML_API uint64_t gguf_get_val_u64 (const struct gguf_context * ctx, int key_id);
2328
- GGML_API int64_t gguf_get_val_i64 (const struct gguf_context * ctx, int key_id);
2329
- GGML_API double gguf_get_val_f64 (const struct gguf_context * ctx, int key_id);
2330
- GGML_API bool gguf_get_val_bool(const struct gguf_context * ctx, int key_id);
2331
- GGML_API const char * gguf_get_val_str (const struct gguf_context * ctx, int key_id);
2332
- GGML_API const void * gguf_get_val_data(const struct gguf_context * ctx, int key_id);
2333
- GGML_API int gguf_get_arr_n (const struct gguf_context * ctx, int key_id);
2334
- GGML_API const void * gguf_get_arr_data(const struct gguf_context * ctx, int key_id);
2335
- GGML_API const char * gguf_get_arr_str (const struct gguf_context * ctx, int key_id, int i);
2336
-
2337
- GGML_API int gguf_get_n_tensors (const struct gguf_context * ctx);
2338
- GGML_API int gguf_find_tensor (const struct gguf_context * ctx, const char * name);
2339
- GGML_API size_t gguf_get_tensor_offset(const struct gguf_context * ctx, int i);
2340
- GGML_API char * gguf_get_tensor_name (const struct gguf_context * ctx, int i);
2341
- GGML_API enum ggml_type gguf_get_tensor_type (const struct gguf_context * ctx, int i);
2342
-
2343
- // removes key if it exists
2344
- GGML_API void gguf_remove_key(struct gguf_context * ctx, const char * key);
2345
-
2346
- // overrides existing values or adds a new one
2347
- GGML_API void gguf_set_val_u8 (struct gguf_context * ctx, const char * key, uint8_t val);
2348
- GGML_API void gguf_set_val_i8 (struct gguf_context * ctx, const char * key, int8_t val);
2349
- GGML_API void gguf_set_val_u16 (struct gguf_context * ctx, const char * key, uint16_t val);
2350
- GGML_API void gguf_set_val_i16 (struct gguf_context * ctx, const char * key, int16_t val);
2351
- GGML_API void gguf_set_val_u32 (struct gguf_context * ctx, const char * key, uint32_t val);
2352
- GGML_API void gguf_set_val_i32 (struct gguf_context * ctx, const char * key, int32_t val);
2353
- GGML_API void gguf_set_val_f32 (struct gguf_context * ctx, const char * key, float val);
2354
- GGML_API void gguf_set_val_u64 (struct gguf_context * ctx, const char * key, uint64_t val);
2355
- GGML_API void gguf_set_val_i64 (struct gguf_context * ctx, const char * key, int64_t val);
2356
- GGML_API void gguf_set_val_f64 (struct gguf_context * ctx, const char * key, double val);
2357
- GGML_API void gguf_set_val_bool(struct gguf_context * ctx, const char * key, bool val);
2358
- GGML_API void gguf_set_val_str (struct gguf_context * ctx, const char * key, const char * val);
2359
- GGML_API void gguf_set_arr_data(struct gguf_context * ctx, const char * key, enum gguf_type type, const void * data, int n);
2360
- GGML_API void gguf_set_arr_str (struct gguf_context * ctx, const char * key, const char ** data, int n);
2361
-
2362
- // set or add KV pairs from another context
2363
- GGML_API void gguf_set_kv(struct gguf_context * ctx, struct gguf_context * src);
2364
-
2365
- // manage tensor info
2366
- GGML_API void gguf_add_tensor(struct gguf_context * ctx, const struct ggml_tensor * tensor);
2367
- GGML_API void gguf_set_tensor_type(struct gguf_context * ctx, const char * name, enum ggml_type type);
2368
- GGML_API void gguf_set_tensor_data(struct gguf_context * ctx, const char * name, const void * data, size_t size);
2369
-
2370
- // writing gguf files can be done in 2 ways:
2371
- //
2372
- // - write the entire gguf_context to a binary file in a single pass:
2373
- //
2374
- // gguf_write_to_file(ctx, fname);
2375
- //
2376
- // - first prepare a file with a placeholder for the meta data, write the tensor data, then write the meta data:
2377
- //
2378
- // FILE * f = fopen(fname, "wb");
2379
- // fseek(f, gguf_get_meta_size(ctx), SEEK_SET);
2380
- // fwrite(f, ...);
2381
- // void * data = gguf_meta_get_meta_data(ctx);
2382
- // fseek(f, 0, SEEK_SET);
2383
- // fwrite(f, data, gguf_get_meta_size(ctx));
2384
- // free(data);
2385
- // fclose(f);
2386
- //
2387
-
2388
- // write the entire context to a binary file
2389
- GGML_API void gguf_write_to_file(const struct gguf_context * ctx, const char * fname, bool only_meta);
2390
-
2391
- // get the size in bytes of the meta data (header, kv pairs, tensor info) including padding
2392
- GGML_API size_t gguf_get_meta_size(const struct gguf_context * ctx);
2393
- GGML_API void gguf_get_meta_data(const struct gguf_context * ctx, void * data);
2394
-
2395
- //
2396
- // system info
2397
- //
2398
-
2399
- GGML_API int ggml_cpu_has_avx (void);
2400
- GGML_API int ggml_cpu_has_avx_vnni (void);
2401
- GGML_API int ggml_cpu_has_avx2 (void);
2402
- GGML_API int ggml_cpu_has_avx512 (void);
2403
- GGML_API int ggml_cpu_has_avx512_vbmi(void);
2404
- GGML_API int ggml_cpu_has_avx512_vnni(void);
2405
- GGML_API int ggml_cpu_has_avx512_bf16(void);
2406
- GGML_API int ggml_cpu_has_fma (void);
2407
- GGML_API int ggml_cpu_has_neon (void);
2408
- GGML_API int ggml_cpu_has_sve (void);
2409
- GGML_API int ggml_cpu_has_arm_fma (void);
2410
- GGML_API int ggml_cpu_has_metal (void);
2411
- GGML_API int ggml_cpu_has_f16c (void);
2412
- GGML_API int ggml_cpu_has_fp16_va (void);
2413
- GGML_API int ggml_cpu_has_wasm_simd (void);
2414
- GGML_API int ggml_cpu_has_blas (void);
2415
- GGML_API int ggml_cpu_has_cuda (void);
2416
- GGML_API int ggml_cpu_has_vulkan (void);
2417
- GGML_API int ggml_cpu_has_kompute (void);
2418
- GGML_API int ggml_cpu_has_gpublas (void);
2419
- GGML_API int ggml_cpu_has_sse3 (void);
2420
- GGML_API int ggml_cpu_has_ssse3 (void);
2421
- GGML_API int ggml_cpu_has_sycl (void);
2422
- GGML_API int ggml_cpu_has_rpc (void);
2423
- GGML_API int ggml_cpu_has_vsx (void);
2424
- GGML_API int ggml_cpu_has_matmul_int8(void);
2425
-
2426
- //
2427
- // Internal types and functions exposed for tests and benchmarks
2428
- //
2429
-
2430
- #ifdef __cplusplus
2431
- // restrict not standard in C++
2432
- #define GGML_RESTRICT
2433
- #else
2434
- #define GGML_RESTRICT restrict
2435
- #endif
2436
- typedef void (*ggml_to_float_t) (const void * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
2437
- typedef void (*ggml_from_float_t)(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
2438
- typedef void (*ggml_vec_dot_t) (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT x, size_t bx,
2439
- const void * GGML_RESTRICT y, size_t by, int nrc);
2440
-
2441
- typedef struct {
2442
- const char * type_name;
2443
- int blck_size;
2444
- size_t type_size;
2445
- bool is_quantized;
2446
- ggml_to_float_t to_float;
2447
- ggml_from_float_t from_float;
2448
- ggml_from_float_t from_float_reference;
2449
- ggml_vec_dot_t vec_dot;
2450
- enum ggml_type vec_dot_type;
2451
- int64_t nrows; // number of rows to process simultaneously;
2452
- } ggml_type_traits_t;
2453
-
2454
- GGML_API ggml_type_traits_t ggml_internal_get_type_traits(enum ggml_type type);
2455
-
2456
- #ifdef __cplusplus
2457
- }
2458
- #endif