llama_cpp 0.16.1 → 0.17.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (177) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +25 -0
  3. data/README.md +7 -12
  4. data/ext/llama_cpp/extconf.rb +2 -42
  5. data/ext/llama_cpp/llama_cpp.cpp +20 -0
  6. data/lib/llama_cpp/version.rb +3 -3
  7. data/sig/llama_cpp.rbs +5 -0
  8. metadata +2 -171
  9. data/vendor/include/.gitkeep +0 -0
  10. data/vendor/lib/.gitkeep +0 -0
  11. data/vendor/tmp/llama.cpp/LICENSE +0 -21
  12. data/vendor/tmp/llama.cpp/Makefile +0 -1116
  13. data/vendor/tmp/llama.cpp/ggml-alloc.c +0 -1041
  14. data/vendor/tmp/llama.cpp/ggml-alloc.h +0 -76
  15. data/vendor/tmp/llama.cpp/ggml-backend-impl.h +0 -153
  16. data/vendor/tmp/llama.cpp/ggml-backend.c +0 -2214
  17. data/vendor/tmp/llama.cpp/ggml-backend.h +0 -233
  18. data/vendor/tmp/llama.cpp/ggml-blas.cpp +0 -363
  19. data/vendor/tmp/llama.cpp/ggml-blas.h +0 -23
  20. data/vendor/tmp/llama.cpp/ggml-common.h +0 -1805
  21. data/vendor/tmp/llama.cpp/ggml-cuda/acc.cu +0 -47
  22. data/vendor/tmp/llama.cpp/ggml-cuda/arange.cu +0 -34
  23. data/vendor/tmp/llama.cpp/ggml-cuda/argsort.cu +0 -104
  24. data/vendor/tmp/llama.cpp/ggml-cuda/binbcast.cu +0 -280
  25. data/vendor/tmp/llama.cpp/ggml-cuda/clamp.cu +0 -34
  26. data/vendor/tmp/llama.cpp/ggml-cuda/concat.cu +0 -196
  27. data/vendor/tmp/llama.cpp/ggml-cuda/convert.cu +0 -686
  28. data/vendor/tmp/llama.cpp/ggml-cuda/cpy.cu +0 -490
  29. data/vendor/tmp/llama.cpp/ggml-cuda/diagmask.cu +0 -40
  30. data/vendor/tmp/llama.cpp/ggml-cuda/dmmv.cu +0 -674
  31. data/vendor/tmp/llama.cpp/ggml-cuda/fattn-tile-f16.cu +0 -319
  32. data/vendor/tmp/llama.cpp/ggml-cuda/fattn-tile-f32.cu +0 -312
  33. data/vendor/tmp/llama.cpp/ggml-cuda/fattn.cu +0 -345
  34. data/vendor/tmp/llama.cpp/ggml-cuda/getrows.cu +0 -178
  35. data/vendor/tmp/llama.cpp/ggml-cuda/im2col.cu +0 -104
  36. data/vendor/tmp/llama.cpp/ggml-cuda/mmq.cu +0 -88
  37. data/vendor/tmp/llama.cpp/ggml-cuda/mmvq.cu +0 -419
  38. data/vendor/tmp/llama.cpp/ggml-cuda/norm.cu +0 -221
  39. data/vendor/tmp/llama.cpp/ggml-cuda/pad.cu +0 -49
  40. data/vendor/tmp/llama.cpp/ggml-cuda/pool2d.cu +0 -94
  41. data/vendor/tmp/llama.cpp/ggml-cuda/quantize.cu +0 -112
  42. data/vendor/tmp/llama.cpp/ggml-cuda/rope.cu +0 -271
  43. data/vendor/tmp/llama.cpp/ggml-cuda/scale.cu +0 -31
  44. data/vendor/tmp/llama.cpp/ggml-cuda/softmax.cu +0 -206
  45. data/vendor/tmp/llama.cpp/ggml-cuda/sumrows.cu +0 -40
  46. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-f16.cu +0 -5
  47. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q4_0.cu +0 -5
  48. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q4_1.cu +0 -5
  49. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q5_0.cu +0 -5
  50. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q5_1.cu +0 -5
  51. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q8_0.cu +0 -5
  52. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-f16.cu +0 -5
  53. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q4_0.cu +0 -5
  54. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q4_1.cu +0 -5
  55. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q5_0.cu +0 -5
  56. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q5_1.cu +0 -5
  57. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q8_0.cu +0 -5
  58. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-f16.cu +0 -5
  59. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q4_0.cu +0 -5
  60. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q4_1.cu +0 -5
  61. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q5_0.cu +0 -5
  62. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q5_1.cu +0 -5
  63. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q8_0.cu +0 -5
  64. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-f16.cu +0 -5
  65. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q4_0.cu +0 -5
  66. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q4_1.cu +0 -5
  67. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q5_0.cu +0 -5
  68. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q5_1.cu +0 -5
  69. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q8_0.cu +0 -5
  70. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-f16.cu +0 -5
  71. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q4_0.cu +0 -5
  72. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q4_1.cu +0 -5
  73. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q5_0.cu +0 -5
  74. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q5_1.cu +0 -5
  75. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q8_0.cu +0 -5
  76. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-f16.cu +0 -5
  77. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q4_0.cu +0 -5
  78. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q4_1.cu +0 -5
  79. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q5_0.cu +0 -5
  80. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q5_1.cu +0 -5
  81. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q8_0.cu +0 -5
  82. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs256-f16-f16.cu +0 -5
  83. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-f16.cu +0 -5
  84. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q4_0.cu +0 -5
  85. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q4_1.cu +0 -5
  86. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q5_0.cu +0 -5
  87. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q5_1.cu +0 -5
  88. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q8_0.cu +0 -5
  89. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-f16.cu +0 -5
  90. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q4_0.cu +0 -5
  91. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q4_1.cu +0 -5
  92. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q5_0.cu +0 -5
  93. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q5_1.cu +0 -5
  94. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q8_0.cu +0 -5
  95. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-f16.cu +0 -5
  96. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q4_0.cu +0 -5
  97. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q4_1.cu +0 -5
  98. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q5_0.cu +0 -5
  99. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q5_1.cu +0 -5
  100. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q8_0.cu +0 -5
  101. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-f16.cu +0 -5
  102. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q4_0.cu +0 -5
  103. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q4_1.cu +0 -5
  104. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q5_0.cu +0 -5
  105. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q5_1.cu +0 -5
  106. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q8_0.cu +0 -5
  107. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-f16.cu +0 -5
  108. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q4_0.cu +0 -5
  109. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q4_1.cu +0 -5
  110. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q5_0.cu +0 -5
  111. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q5_1.cu +0 -5
  112. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q8_0.cu +0 -5
  113. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-f16.cu +0 -5
  114. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q4_0.cu +0 -5
  115. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q4_1.cu +0 -5
  116. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q5_0.cu +0 -5
  117. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q5_1.cu +0 -5
  118. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q8_0.cu +0 -5
  119. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-f16.cu +0 -5
  120. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q4_0.cu +0 -5
  121. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q4_1.cu +0 -5
  122. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q5_0.cu +0 -5
  123. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q5_1.cu +0 -5
  124. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q8_0.cu +0 -5
  125. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs256-f16-f16.cu +0 -5
  126. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-f16.cu +0 -5
  127. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q4_0.cu +0 -5
  128. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q4_1.cu +0 -5
  129. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q5_0.cu +0 -5
  130. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q5_1.cu +0 -5
  131. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q8_0.cu +0 -5
  132. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqfloat-cpb16.cu +0 -10
  133. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqfloat-cpb32.cu +0 -9
  134. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqhalf-cpb16.cu +0 -10
  135. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqhalf-cpb32.cu +0 -10
  136. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqhalf-cpb8.cu +0 -8
  137. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q2_k.cu +0 -5
  138. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q3_k.cu +0 -5
  139. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q4_0.cu +0 -5
  140. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q4_1.cu +0 -5
  141. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q4_k.cu +0 -5
  142. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q5_0.cu +0 -5
  143. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q5_1.cu +0 -5
  144. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q5_k.cu +0 -5
  145. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q6_k.cu +0 -5
  146. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q8_0.cu +0 -5
  147. data/vendor/tmp/llama.cpp/ggml-cuda/tsembd.cu +0 -47
  148. data/vendor/tmp/llama.cpp/ggml-cuda/unary.cu +0 -286
  149. data/vendor/tmp/llama.cpp/ggml-cuda/upscale.cu +0 -51
  150. data/vendor/tmp/llama.cpp/ggml-cuda.cu +0 -3069
  151. data/vendor/tmp/llama.cpp/ggml-cuda.h +0 -44
  152. data/vendor/tmp/llama.cpp/ggml-impl.h +0 -651
  153. data/vendor/tmp/llama.cpp/ggml-kompute.cpp +0 -2038
  154. data/vendor/tmp/llama.cpp/ggml-kompute.h +0 -46
  155. data/vendor/tmp/llama.cpp/ggml-metal.h +0 -66
  156. data/vendor/tmp/llama.cpp/ggml-metal.m +0 -3267
  157. data/vendor/tmp/llama.cpp/ggml-metal.metal +0 -6540
  158. data/vendor/tmp/llama.cpp/ggml-quants.c +0 -14380
  159. data/vendor/tmp/llama.cpp/ggml-quants.h +0 -133
  160. data/vendor/tmp/llama.cpp/ggml-rpc.cpp +0 -1173
  161. data/vendor/tmp/llama.cpp/ggml-rpc.h +0 -24
  162. data/vendor/tmp/llama.cpp/ggml-sycl.cpp +0 -17429
  163. data/vendor/tmp/llama.cpp/ggml-sycl.h +0 -49
  164. data/vendor/tmp/llama.cpp/ggml-vulkan-shaders.hpp +0 -140820
  165. data/vendor/tmp/llama.cpp/ggml-vulkan.cpp +0 -7271
  166. data/vendor/tmp/llama.cpp/ggml-vulkan.h +0 -29
  167. data/vendor/tmp/llama.cpp/ggml.c +0 -22589
  168. data/vendor/tmp/llama.cpp/ggml.h +0 -2452
  169. data/vendor/tmp/llama.cpp/llama.cpp +0 -18692
  170. data/vendor/tmp/llama.cpp/llama.h +0 -1143
  171. data/vendor/tmp/llama.cpp/scripts/get-flags.mk +0 -38
  172. data/vendor/tmp/llama.cpp/sgemm.cpp +0 -1030
  173. data/vendor/tmp/llama.cpp/sgemm.h +0 -14
  174. data/vendor/tmp/llama.cpp/unicode-data.cpp +0 -6983
  175. data/vendor/tmp/llama.cpp/unicode-data.h +0 -20
  176. data/vendor/tmp/llama.cpp/unicode.cpp +0 -796
  177. data/vendor/tmp/llama.cpp/unicode.h +0 -63
data/vendor/tmp/llama.cpp/ggml.h DELETED
@@ -1,2452 +0,0 @@
1
- #pragma once
2
-
3
- //
4
- // GGML Tensor Library
5
- //
6
- // This documentation is still a work in progress.
7
- // If you wish some specific topics to be covered, feel free to drop a comment:
8
- //
9
- // https://github.com/ggerganov/whisper.cpp/issues/40
10
- //
11
- // ## Overview
12
- //
13
- // This library implements:
14
- //
15
- // - a set of tensor operations
16
- // - automatic differentiation
17
- // - basic optimization algorithms
18
- //
19
- // The aim of this library is to provide a minimalistic approach for various machine learning tasks. This includes,
20
- // but is not limited to, the following:
21
- //
22
- // - linear regression
23
- // - support vector machines
24
- // - neural networks
25
- //
26
- // The library allows the user to define a certain function using the available tensor operations. This function
27
- // definition is represented internally via a computation graph. Each tensor operation in the function definition
28
- // corresponds to a node in the graph. Having the computation graph defined, the user can choose to compute the
29
- // function's value and/or its gradient with respect to the input variables. Optionally, the function can be optimized
30
- // using one of the available optimization algorithms.
31
- //
32
- // For example, here we define the function: f(x) = a*x^2 + b
33
- //
34
- // {
35
- // struct ggml_init_params params = {
36
- // .mem_size = 16*1024*1024,
37
- // .mem_buffer = NULL,
38
- // };
39
- //
40
- // // memory allocation happens here
41
- // struct ggml_context * ctx = ggml_init(params);
42
- //
43
- // struct ggml_tensor * x = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1);
44
- //
45
- // ggml_set_param(ctx, x); // x is an input variable
46
- //
47
- // struct ggml_tensor * a = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1);
48
- // struct ggml_tensor * b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1);
49
- // struct ggml_tensor * x2 = ggml_mul(ctx, x, x);
50
- // struct ggml_tensor * f = ggml_add(ctx, ggml_mul(ctx, a, x2), b);
51
- //
52
- // ...
53
- // }
54
- //
55
- // Notice that the function definition above does not involve any actual computation. The computation is performed only
56
- // when the user explicitly requests it. For example, to compute the function's value at x = 2.0:
57
- //
58
- // {
59
- // ...
60
- //
61
- // struct ggml_cgraph * gf = ggml_new_graph(ctx);
62
- // ggml_build_forward_expand(gf, f);
63
- //
64
- // // set the input variable and parameter values
65
- // ggml_set_f32(x, 2.0f);
66
- // ggml_set_f32(a, 3.0f);
67
- // ggml_set_f32(b, 4.0f);
68
- //
69
- // ggml_graph_compute_with_ctx(ctx, &gf, n_threads);
70
- //
71
- // printf("f = %f\n", ggml_get_f32_1d(f, 0));
72
- //
73
- // ...
74
- // }
75
- //
76
- // The actual computation is performed in the ggml_graph_compute() function.
77
- //
78
- // The ggml_new_tensor_...() functions create new tensors. They are allocated in the memory buffer provided to the
79
- // ggml_init() function. You have to be careful not to exceed the memory buffer size. Therefore, you have to know
80
- // in advance how much memory you need for your computation. Alternatively, you can allocate a large enough memory
81
- // and after defining the computation graph, call the ggml_used_mem() function to find out how much memory was
82
- // actually needed.
83
- //
84
- // The ggml_set_param() function marks a tensor as an input variable. This is used by the automatic
85
- // differentiation and optimization algorithms.
86
- //
87
- // The described approach allows to define the function graph once and then compute its forward or backward graphs
88
- // multiple times. All computations will use the same memory buffer allocated in the ggml_init() function. This way
89
- // the user can avoid the memory allocation overhead at runtime.
90
- //
91
- // The library supports multi-dimensional tensors - up to 4 dimensions. The FP16 and FP32 data types are first class
92
- // citizens, but in theory the library can be extended to support FP8 and integer data types.
93
- //
94
- // Each tensor operation produces a new tensor. Initially the library was envisioned to support only the use of unary
95
- // and binary operations. Most of the available operations fall into one of these two categories. With time, it became
96
- // clear that the library needs to support more complex operations. The way to support these operations is not clear
97
- // yet, but a few examples are demonstrated in the following operations:
98
- //
99
- // - ggml_permute()
100
- // - ggml_conv_1d_1s()
101
- // - ggml_conv_1d_2s()
102
- //
103
- // For each tensor operator, the library implements a forward and backward computation function. The forward function
104
- // computes the output tensor value given the input tensor values. The backward function computes the adjoint of the
105
- // input tensors given the adjoint of the output tensor. For a detailed explanation of what this means, take a
106
- // calculus class, or watch the following video:
107
- //
108
- // What is Automatic Differentiation?
109
- // https://www.youtube.com/watch?v=wG_nF1awSSY
110
- //
111
- //
112
- // ## Tensor data (struct ggml_tensor)
113
- //
114
- // The tensors are stored in memory via the ggml_tensor struct. The structure provides information about the size of
115
- // the tensor, the data type, and the memory buffer where the tensor data is stored. Additionally, it contains
116
- // pointers to the "source" tensors - i.e. the tensors that were used to compute the current tensor. For example:
117
- //
118
- // {
119
- // struct ggml_tensor * c = ggml_add(ctx, a, b);
120
- //
121
- // assert(c->src[0] == a);
122
- // assert(c->src[1] == b);
123
- // }
124
- //
125
- // The multi-dimensional tensors are stored in row-major order. The ggml_tensor struct contains fields for the
126
- // number of elements in each dimension ("ne") as well as the number of bytes ("nb", a.k.a. stride). This allows
127
- // to store tensors that are not contiguous in memory, which is useful for operations such as transposition and
128
- // permutation. All tensor operations have to take the stride into account and not assume that the tensor is
129
- // contiguous in memory.
130
- //
131
- // The data of the tensor is accessed via the "data" pointer. For example:
132
- //
133
- // {
134
- // const int nx = 2;
135
- // const int ny = 3;
136
- //
137
- // struct ggml_tensor * a = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, nx, ny);
138
- //
139
- // for (int y = 0; y < ny; y++) {
140
- // for (int x = 0; x < nx; x++) {
141
- // *(float *) ((char *) a->data + y*a->nb[1] + x*a->nb[0]) = x + y;
142
- // }
143
- // }
144
- //
145
- // ...
146
- // }
147
- //
148
- // Alternatively, there are helper functions, such as ggml_get_f32_1d() and ggml_set_f32_1d() that can be used.
149
- //
150
- // ## The matrix multiplication operator (ggml_mul_mat)
151
- //
152
- // TODO
153
- //
154
- //
155
- // ## Multi-threading
156
- //
157
- // TODO
158
- //
159
- //
160
- // ## Overview of ggml.c
161
- //
162
- // TODO
163
- //
164
- //
165
- // ## SIMD optimizations
166
- //
167
- // TODO
168
- //
169
- //
170
- // ## Debugging ggml
171
- //
172
- // TODO
173
- //
174
- //
175
-
176
- #ifdef GGML_SHARED
177
- # if defined(_WIN32) && !defined(__MINGW32__)
178
- # ifdef GGML_BUILD
179
- # define GGML_API __declspec(dllexport)
180
- # else
181
- # define GGML_API __declspec(dllimport)
182
- # endif
183
- # else
184
- # define GGML_API __attribute__ ((visibility ("default")))
185
- # endif
186
- #else
187
- # define GGML_API
188
- #endif
189
-
190
- #ifdef GGML_MULTIPLATFORM
191
- # if defined(_WIN32)
192
- # define GGML_CALL
193
- # else
194
- # define GGML_CALL __attribute__((__ms_abi__))
195
- # endif
196
- #else
197
- # define GGML_CALL
198
- #endif
199
-
200
- // TODO: support for clang
201
- #ifdef __GNUC__
202
- # define GGML_DEPRECATED(func, hint) func __attribute__((deprecated(hint)))
203
- #elif defined(_MSC_VER)
204
- # define GGML_DEPRECATED(func, hint) __declspec(deprecated(hint)) func
205
- #else
206
- # define GGML_DEPRECATED(func, hint) func
207
- #endif
208
-
209
- #ifndef __GNUC__
210
- # define GGML_ATTRIBUTE_FORMAT(...)
211
- #elif defined(__MINGW32__)
212
- # define GGML_ATTRIBUTE_FORMAT(...) __attribute__((format(gnu_printf, __VA_ARGS__)))
213
- #else
214
- # define GGML_ATTRIBUTE_FORMAT(...) __attribute__((format(printf, __VA_ARGS__)))
215
- #endif
216
-
217
- #include <stdbool.h>
218
- #include <stddef.h>
219
- #include <stdint.h>
220
- #include <stdio.h>
221
-
222
- #define GGML_FILE_MAGIC 0x67676d6c // "ggml"
223
- #define GGML_FILE_VERSION 1
224
-
225
- #define GGML_QNT_VERSION 2 // bump this on quantization format changes
226
- #define GGML_QNT_VERSION_FACTOR 1000 // do not change this
227
-
228
- #define GGML_MAX_DIMS 4
229
- #define GGML_MAX_PARAMS 2048
230
- #define GGML_MAX_CONTEXTS 64
231
- #define GGML_MAX_SRC 10
232
- #ifndef GGML_MAX_NAME
233
- #define GGML_MAX_NAME 64
234
- #endif
235
- #define GGML_MAX_OP_PARAMS 64
236
- #define GGML_DEFAULT_N_THREADS 4
237
- #define GGML_DEFAULT_GRAPH_SIZE 2048
238
- #if UINTPTR_MAX == 0xFFFFFFFF
239
- #define GGML_MEM_ALIGN 4
240
- #else
241
- #define GGML_MEM_ALIGN 16
242
- #endif
243
-
244
- #define GGML_EXIT_SUCCESS 0
245
- #define GGML_EXIT_ABORTED 1
246
-
247
- #define GGUF_MAGIC "GGUF"
248
-
249
- #define GGUF_VERSION 3
250
-
251
- #define GGUF_DEFAULT_ALIGNMENT 32
252
-
253
- #define GGML_UNUSED(x) (void)(x)
254
-
255
- #define GGML_PAD(x, n) (((x) + (n) - 1) & ~((n) - 1))
256
-
257
- #define GGML_ASSERT(x) \
258
- do { \
259
- if (!(x)) { \
260
- fflush(stdout); \
261
- fprintf(stderr, "GGML_ASSERT: %s:%d: %s\n", __FILE__, __LINE__, #x); \
262
- ggml_print_backtrace(); \
263
- abort(); \
264
- } \
265
- } while (0)
266
-
267
- #ifndef NDEBUG
268
- #define GGML_UNREACHABLE() GGML_ASSERT(!"statement should not be reached")
269
- #elif defined(__GNUC__)
270
- #define GGML_UNREACHABLE() __builtin_unreachable()
271
- #elif defined(_MSC_VER)
272
- #define GGML_UNREACHABLE() __assume(0)
273
- #else
274
- #define GGML_UNREACHABLE() ((void) 0)
275
- #endif
276
-
277
- // used to copy the number of elements and stride in bytes of tensors into local variables.
278
- // main purpose is to reduce code duplication and improve readability.
279
- //
280
- // example:
281
- //
282
- // GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne);
283
- // GGML_TENSOR_LOCALS(size_t, nb1, src1, nb);
284
- //
285
- #define GGML_TENSOR_LOCALS_1(type, prefix, pointer, array) \
286
- const type prefix##0 = (pointer)->array[0]; \
287
- GGML_UNUSED(prefix##0);
288
- #define GGML_TENSOR_LOCALS_2(type, prefix, pointer, array) \
289
- GGML_TENSOR_LOCALS_1 (type, prefix, pointer, array) \
290
- const type prefix##1 = (pointer)->array[1]; \
291
- GGML_UNUSED(prefix##1);
292
- #define GGML_TENSOR_LOCALS_3(type, prefix, pointer, array) \
293
- GGML_TENSOR_LOCALS_2 (type, prefix, pointer, array) \
294
- const type prefix##2 = (pointer)->array[2]; \
295
- GGML_UNUSED(prefix##2);
296
- #define GGML_TENSOR_LOCALS(type, prefix, pointer, array) \
297
- GGML_TENSOR_LOCALS_3 (type, prefix, pointer, array) \
298
- const type prefix##3 = (pointer)->array[3]; \
299
- GGML_UNUSED(prefix##3);
300
-
301
- #define GGML_TENSOR_UNARY_OP_LOCALS \
302
- GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) \
303
- GGML_TENSOR_LOCALS(size_t, nb0, src0, nb) \
304
- GGML_TENSOR_LOCALS(int64_t, ne, dst, ne) \
305
- GGML_TENSOR_LOCALS(size_t, nb, dst, nb)
306
-
307
- #define GGML_TENSOR_BINARY_OP_LOCALS \
308
- GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) \
309
- GGML_TENSOR_LOCALS(size_t, nb0, src0, nb) \
310
- GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne) \
311
- GGML_TENSOR_LOCALS(size_t, nb1, src1, nb) \
312
- GGML_TENSOR_LOCALS(int64_t, ne, dst, ne) \
313
- GGML_TENSOR_LOCALS(size_t, nb, dst, nb)
314
-
315
- #ifdef __cplusplus
316
- extern "C" {
317
- #endif
318
-
319
- enum ggml_status {
320
- GGML_STATUS_ALLOC_FAILED = -2,
321
- GGML_STATUS_FAILED = -1,
322
- GGML_STATUS_SUCCESS = 0,
323
- GGML_STATUS_ABORTED = 1,
324
- };
325
-
326
- // get ggml_status name string
327
- GGML_API GGML_CALL const char * ggml_status_to_string(enum ggml_status status);
328
-
329
- // ieee 754-2008 half-precision float16
330
- // todo: make this not an integral type
331
- typedef uint16_t ggml_fp16_t;
332
- GGML_API float ggml_fp16_to_fp32(ggml_fp16_t);
333
- GGML_API ggml_fp16_t ggml_fp32_to_fp16(float);
334
- GGML_API void ggml_fp16_to_fp32_row(const ggml_fp16_t *, float *, int64_t);
335
- GGML_API void ggml_fp32_to_fp16_row(const float *, ggml_fp16_t *, int64_t);
336
-
337
- // google brain half-precision bfloat16
338
- typedef struct { uint16_t bits; } ggml_bf16_t;
339
- GGML_API ggml_bf16_t ggml_fp32_to_bf16(float);
340
- GGML_API float ggml_bf16_to_fp32(ggml_bf16_t); // consider just doing << 16
341
- GGML_API void ggml_bf16_to_fp32_row(const ggml_bf16_t *, float *, int64_t);
342
- GGML_API void ggml_fp32_to_bf16_row(const float *, ggml_bf16_t *, int64_t);
343
-
344
- struct ggml_object;
345
- struct ggml_context;
346
-
347
- // NOTE: always add types at the end of the enum to keep backward compatibility
348
- enum ggml_type {
349
- GGML_TYPE_F32 = 0,
350
- GGML_TYPE_F16 = 1,
351
- GGML_TYPE_Q4_0 = 2,
352
- GGML_TYPE_Q4_1 = 3,
353
- // GGML_TYPE_Q4_2 = 4, support has been removed
354
- // GGML_TYPE_Q4_3 = 5, support has been removed
355
- GGML_TYPE_Q5_0 = 6,
356
- GGML_TYPE_Q5_1 = 7,
357
- GGML_TYPE_Q8_0 = 8,
358
- GGML_TYPE_Q8_1 = 9,
359
- GGML_TYPE_Q2_K = 10,
360
- GGML_TYPE_Q3_K = 11,
361
- GGML_TYPE_Q4_K = 12,
362
- GGML_TYPE_Q5_K = 13,
363
- GGML_TYPE_Q6_K = 14,
364
- GGML_TYPE_Q8_K = 15,
365
- GGML_TYPE_IQ2_XXS = 16,
366
- GGML_TYPE_IQ2_XS = 17,
367
- GGML_TYPE_IQ3_XXS = 18,
368
- GGML_TYPE_IQ1_S = 19,
369
- GGML_TYPE_IQ4_NL = 20,
370
- GGML_TYPE_IQ3_S = 21,
371
- GGML_TYPE_IQ2_S = 22,
372
- GGML_TYPE_IQ4_XS = 23,
373
- GGML_TYPE_I8 = 24,
374
- GGML_TYPE_I16 = 25,
375
- GGML_TYPE_I32 = 26,
376
- GGML_TYPE_I64 = 27,
377
- GGML_TYPE_F64 = 28,
378
- GGML_TYPE_IQ1_M = 29,
379
- GGML_TYPE_BF16 = 30,
380
- GGML_TYPE_COUNT,
381
- };
382
-
383
- // precision
384
- enum ggml_prec {
385
- GGML_PREC_DEFAULT,
386
- GGML_PREC_F32,
387
- };
388
-
389
- enum ggml_backend_type {
390
- GGML_BACKEND_TYPE_CPU = 0,
391
- GGML_BACKEND_TYPE_GPU = 10,
392
- GGML_BACKEND_TYPE_GPU_SPLIT = 20,
393
- };
394
-
395
- // model file types
396
- enum ggml_ftype {
397
- GGML_FTYPE_UNKNOWN = -1,
398
- GGML_FTYPE_ALL_F32 = 0,
399
- GGML_FTYPE_MOSTLY_F16 = 1, // except 1d tensors
400
- GGML_FTYPE_MOSTLY_Q4_0 = 2, // except 1d tensors
401
- GGML_FTYPE_MOSTLY_Q4_1 = 3, // except 1d tensors
402
- GGML_FTYPE_MOSTLY_Q4_1_SOME_F16 = 4, // tok_embeddings.weight and output.weight are F16
403
- GGML_FTYPE_MOSTLY_Q8_0 = 7, // except 1d tensors
404
- GGML_FTYPE_MOSTLY_Q5_0 = 8, // except 1d tensors
405
- GGML_FTYPE_MOSTLY_Q5_1 = 9, // except 1d tensors
406
- GGML_FTYPE_MOSTLY_Q2_K = 10, // except 1d tensors
407
- GGML_FTYPE_MOSTLY_Q3_K = 11, // except 1d tensors
408
- GGML_FTYPE_MOSTLY_Q4_K = 12, // except 1d tensors
409
- GGML_FTYPE_MOSTLY_Q5_K = 13, // except 1d tensors
410
- GGML_FTYPE_MOSTLY_Q6_K = 14, // except 1d tensors
411
- GGML_FTYPE_MOSTLY_IQ2_XXS = 15, // except 1d tensors
412
- GGML_FTYPE_MOSTLY_IQ2_XS = 16, // except 1d tensors
413
- GGML_FTYPE_MOSTLY_IQ3_XXS = 17, // except 1d tensors
414
- GGML_FTYPE_MOSTLY_IQ1_S = 18, // except 1d tensors
415
- GGML_FTYPE_MOSTLY_IQ4_NL = 19, // except 1d tensors
416
- GGML_FTYPE_MOSTLY_IQ3_S = 20, // except 1d tensors
417
- GGML_FTYPE_MOSTLY_IQ2_S = 21, // except 1d tensors
418
- GGML_FTYPE_MOSTLY_IQ4_XS = 22, // except 1d tensors
419
- GGML_FTYPE_MOSTLY_IQ1_M = 23, // except 1d tensors
420
- GGML_FTYPE_MOSTLY_BF16 = 24, // except 1d tensors
421
- };
422
-
423
- // available tensor operations:
424
- enum ggml_op {
425
- GGML_OP_NONE = 0,
426
-
427
- GGML_OP_DUP,
428
- GGML_OP_ADD,
429
- GGML_OP_ADD1,
430
- GGML_OP_ACC,
431
- GGML_OP_SUB,
432
- GGML_OP_MUL,
433
- GGML_OP_DIV,
434
- GGML_OP_SQR,
435
- GGML_OP_SQRT,
436
- GGML_OP_LOG,
437
- GGML_OP_SUM,
438
- GGML_OP_SUM_ROWS,
439
- GGML_OP_MEAN,
440
- GGML_OP_ARGMAX,
441
- GGML_OP_REPEAT,
442
- GGML_OP_REPEAT_BACK,
443
- GGML_OP_CONCAT,
444
- GGML_OP_SILU_BACK,
445
- GGML_OP_NORM, // normalize
446
- GGML_OP_RMS_NORM,
447
- GGML_OP_RMS_NORM_BACK,
448
- GGML_OP_GROUP_NORM,
449
-
450
- GGML_OP_MUL_MAT,
451
- GGML_OP_MUL_MAT_ID,
452
- GGML_OP_OUT_PROD,
453
-
454
- GGML_OP_SCALE,
455
- GGML_OP_SET,
456
- GGML_OP_CPY,
457
- GGML_OP_CONT,
458
- GGML_OP_RESHAPE,
459
- GGML_OP_VIEW,
460
- GGML_OP_PERMUTE,
461
- GGML_OP_TRANSPOSE,
462
- GGML_OP_GET_ROWS,
463
- GGML_OP_GET_ROWS_BACK,
464
- GGML_OP_DIAG,
465
- GGML_OP_DIAG_MASK_INF,
466
- GGML_OP_DIAG_MASK_ZERO,
467
- GGML_OP_SOFT_MAX,
468
- GGML_OP_SOFT_MAX_BACK,
469
- GGML_OP_ROPE,
470
- GGML_OP_ROPE_BACK,
471
- GGML_OP_CLAMP,
472
- GGML_OP_CONV_TRANSPOSE_1D,
473
- GGML_OP_IM2COL,
474
- GGML_OP_CONV_TRANSPOSE_2D,
475
- GGML_OP_POOL_1D,
476
- GGML_OP_POOL_2D,
477
- GGML_OP_UPSCALE, // nearest interpolate
478
- GGML_OP_PAD,
479
- GGML_OP_ARANGE,
480
- GGML_OP_TIMESTEP_EMBEDDING,
481
- GGML_OP_ARGSORT,
482
- GGML_OP_LEAKY_RELU,
483
-
484
- GGML_OP_FLASH_ATTN_EXT,
485
- GGML_OP_FLASH_ATTN_BACK,
486
- GGML_OP_SSM_CONV,
487
- GGML_OP_SSM_SCAN,
488
- GGML_OP_WIN_PART,
489
- GGML_OP_WIN_UNPART,
490
- GGML_OP_GET_REL_POS,
491
- GGML_OP_ADD_REL_POS,
492
-
493
- GGML_OP_UNARY,
494
-
495
- GGML_OP_MAP_UNARY,
496
- GGML_OP_MAP_BINARY,
497
-
498
- GGML_OP_MAP_CUSTOM1_F32,
499
- GGML_OP_MAP_CUSTOM2_F32,
500
- GGML_OP_MAP_CUSTOM3_F32,
501
-
502
- GGML_OP_MAP_CUSTOM1,
503
- GGML_OP_MAP_CUSTOM2,
504
- GGML_OP_MAP_CUSTOM3,
505
-
506
- GGML_OP_CROSS_ENTROPY_LOSS,
507
- GGML_OP_CROSS_ENTROPY_LOSS_BACK,
508
-
509
- GGML_OP_COUNT,
510
- };
511
-
512
- enum ggml_unary_op {
513
- GGML_UNARY_OP_ABS,
514
- GGML_UNARY_OP_SGN,
515
- GGML_UNARY_OP_NEG,
516
- GGML_UNARY_OP_STEP,
517
- GGML_UNARY_OP_TANH,
518
- GGML_UNARY_OP_ELU,
519
- GGML_UNARY_OP_RELU,
520
- GGML_UNARY_OP_SIGMOID,
521
- GGML_UNARY_OP_GELU,
522
- GGML_UNARY_OP_GELU_QUICK,
523
- GGML_UNARY_OP_SILU,
524
- GGML_UNARY_OP_HARDSWISH,
525
- GGML_UNARY_OP_HARDSIGMOID,
526
-
527
- GGML_UNARY_OP_COUNT,
528
- };
529
-
530
- enum ggml_object_type {
531
- GGML_OBJECT_TYPE_TENSOR,
532
- GGML_OBJECT_TYPE_GRAPH,
533
- GGML_OBJECT_TYPE_WORK_BUFFER
534
- };
535
-
536
- enum ggml_log_level {
537
- GGML_LOG_LEVEL_ERROR = 2,
538
- GGML_LOG_LEVEL_WARN = 3,
539
- GGML_LOG_LEVEL_INFO = 4,
540
- GGML_LOG_LEVEL_DEBUG = 5
541
- };
542
-
543
- enum ggml_tensor_flag {
544
- GGML_TENSOR_FLAG_INPUT = 1,
545
- GGML_TENSOR_FLAG_OUTPUT = 2,
546
- GGML_TENSOR_FLAG_PARAM = 4,
547
- };
548
-
549
- // ggml object
550
- struct ggml_object {
551
- size_t offs;
552
- size_t size;
553
-
554
- struct ggml_object * next;
555
-
556
- enum ggml_object_type type;
557
-
558
- char padding[4];
559
- };
560
-
561
- static const size_t GGML_OBJECT_SIZE = sizeof(struct ggml_object);
562
-
563
- // n-dimensional tensor
564
- struct ggml_tensor {
565
- enum ggml_type type;
566
-
567
- GGML_DEPRECATED(enum ggml_backend_type backend, "use the buffer type to find the storage location of the tensor");
568
-
569
- struct ggml_backend_buffer * buffer;
570
-
571
- int64_t ne[GGML_MAX_DIMS]; // number of elements
572
- size_t nb[GGML_MAX_DIMS]; // stride in bytes:
573
- // nb[0] = ggml_type_size(type)
574
- // nb[1] = nb[0] * (ne[0] / ggml_blck_size(type)) + padding
575
- // nb[i] = nb[i-1] * ne[i-1]
576
-
577
- // compute data
578
- enum ggml_op op;
579
-
580
- // op params - allocated as int32_t for alignment
581
- int32_t op_params[GGML_MAX_OP_PARAMS / sizeof(int32_t)];
582
-
583
- int32_t flags;
584
-
585
- struct ggml_tensor * grad;
586
- struct ggml_tensor * src[GGML_MAX_SRC];
587
-
588
- // performance
589
- int perf_runs;
590
- int64_t perf_cycles;
591
- int64_t perf_time_us;
592
-
593
- struct ggml_tensor * view_src;
594
- size_t view_offs;
595
-
596
- void * data;
597
-
598
- char name[GGML_MAX_NAME];
599
-
600
- void * extra; // extra things e.g. for ggml-cuda.cu
601
-
602
- char padding[8];
603
- };
604
-
605
- static const size_t GGML_TENSOR_SIZE = sizeof(struct ggml_tensor);
606
-
607
- // Abort callback
608
- // If not NULL, called before ggml computation
609
- // If it returns true, the computation is aborted
610
- typedef bool (*ggml_abort_callback)(void * data);
611
-
612
- // the compute plan that needs to be prepared for ggml_graph_compute()
613
- // since https://github.com/ggerganov/ggml/issues/287
614
- struct ggml_cplan {
615
- size_t work_size; // size of work buffer, calculated by `ggml_graph_plan()`
616
- uint8_t * work_data; // work buffer, to be allocated by caller before calling to `ggml_graph_compute()`
617
-
618
- int n_threads;
619
-
620
- // abort ggml_graph_compute when true
621
- ggml_abort_callback abort_callback;
622
- void * abort_callback_data;
623
- };
624
-
625
- enum ggml_cgraph_eval_order {
626
- GGML_CGRAPH_EVAL_ORDER_LEFT_TO_RIGHT = 0,
627
- GGML_CGRAPH_EVAL_ORDER_RIGHT_TO_LEFT,
628
- GGML_CGRAPH_EVAL_ORDER_COUNT
629
- };
630
-
631
- struct ggml_hash_set {
632
- size_t size;
633
- struct ggml_tensor ** keys;
634
- };
635
-
636
- // computation graph
637
- struct ggml_cgraph {
638
- int size;
639
- int n_nodes;
640
- int n_leafs;
641
-
642
- struct ggml_tensor ** nodes;
643
- struct ggml_tensor ** grads;
644
- struct ggml_tensor ** leafs;
645
-
646
- struct ggml_hash_set visited_hash_table;
647
-
648
- enum ggml_cgraph_eval_order order;
649
-
650
- // performance
651
- int perf_runs;
652
- int64_t perf_cycles;
653
- int64_t perf_time_us;
654
- };
655
-
656
- // scratch buffer
657
- struct ggml_scratch {
658
- size_t offs;
659
- size_t size;
660
- void * data;
661
- };
662
-
663
- struct ggml_init_params {
664
- // memory pool
665
- size_t mem_size; // bytes
666
- void * mem_buffer; // if NULL, memory will be allocated internally
667
- bool no_alloc; // don't allocate memory for the tensor data
668
- };
669
-
670
-
671
- // compute types
672
-
673
- // NOTE: the INIT or FINALIZE pass is not scheduled unless explicitly enabled.
674
- // This behavior was changed since https://github.com/ggerganov/llama.cpp/pull/1995.
675
- enum ggml_task_type {
676
- GGML_TASK_TYPE_INIT = 0,
677
- GGML_TASK_TYPE_COMPUTE,
678
- GGML_TASK_TYPE_FINALIZE,
679
- };
680
-
681
- struct ggml_compute_params {
682
- enum ggml_task_type type;
683
-
684
- // ith = thread index, nth = number of threads
685
- int ith, nth;
686
-
687
- // work buffer for all threads
688
- size_t wsize;
689
- void * wdata;
690
- };
691
-
692
- // numa strategies
693
- enum ggml_numa_strategy {
694
- GGML_NUMA_STRATEGY_DISABLED = 0,
695
- GGML_NUMA_STRATEGY_DISTRIBUTE = 1,
696
- GGML_NUMA_STRATEGY_ISOLATE = 2,
697
- GGML_NUMA_STRATEGY_NUMACTL = 3,
698
- GGML_NUMA_STRATEGY_MIRROR = 4,
699
- GGML_NUMA_STRATEGY_COUNT
700
- };
701
-
702
- //
703
- // GUID
704
- //
705
-
706
- // GUID types
707
- typedef uint8_t ggml_guid[16];
708
- typedef ggml_guid * ggml_guid_t;
709
-
710
- GGML_API bool ggml_guid_matches(ggml_guid_t guid_a, ggml_guid_t guid_b);
711
-
712
- // misc
713
-
714
- GGML_API void ggml_time_init(void); // call this once at the beginning of the program
715
- GGML_API int64_t ggml_time_ms(void);
716
- GGML_API int64_t ggml_time_us(void);
717
- GGML_API int64_t ggml_cycles(void);
718
- GGML_API int64_t ggml_cycles_per_ms(void);
719
-
720
- GGML_API void ggml_print_backtrace(void);
721
-
722
- // accepts a UTF-8 path, even on Windows
723
- GGML_API FILE * ggml_fopen(const char * fname, const char * mode);
724
-
725
- GGML_API void ggml_numa_init(enum ggml_numa_strategy numa); // call once for better performance on NUMA systems
726
- GGML_API bool ggml_is_numa(void); // true if init detected that system has >1 NUMA node
727
-
728
- GGML_API void ggml_print_object (const struct ggml_object * obj);
729
- GGML_API void ggml_print_objects(const struct ggml_context * ctx);
730
-
731
- GGML_API GGML_CALL int64_t ggml_nelements (const struct ggml_tensor * tensor);
732
- GGML_API GGML_CALL int64_t ggml_nrows (const struct ggml_tensor * tensor);
733
- GGML_API GGML_CALL size_t ggml_nbytes (const struct ggml_tensor * tensor);
734
- GGML_API size_t ggml_nbytes_pad (const struct ggml_tensor * tensor); // same as ggml_nbytes() but padded to GGML_MEM_ALIGN
735
-
736
- GGML_API GGML_CALL int ggml_blck_size(enum ggml_type type);
737
- GGML_API GGML_CALL size_t ggml_type_size(enum ggml_type type); // size in bytes for all elements in a block
738
- GGML_API GGML_CALL size_t ggml_row_size (enum ggml_type type, int64_t ne); // size in bytes for all elements in a row
739
-
740
- GGML_DEPRECATED(
741
- GGML_API double ggml_type_sizef(enum ggml_type type), // ggml_type_size()/ggml_blck_size() as float
742
- "use ggml_row_size() instead");
743
-
744
- GGML_API GGML_CALL const char * ggml_type_name(enum ggml_type type);
745
- GGML_API GGML_CALL const char * ggml_op_name (enum ggml_op op);
746
- GGML_API const char * ggml_op_symbol(enum ggml_op op);
747
-
748
- GGML_API const char * ggml_unary_op_name(enum ggml_unary_op op);
749
- GGML_API GGML_CALL const char * ggml_op_desc(const struct ggml_tensor * t); // unary or op name
750
-
751
- GGML_API GGML_CALL size_t ggml_element_size(const struct ggml_tensor * tensor);
752
-
753
- GGML_API GGML_CALL bool ggml_is_quantized(enum ggml_type type);
754
-
755
- // TODO: temporary until model loading of ggml examples is refactored
756
- GGML_API enum ggml_type ggml_ftype_to_ggml_type(enum ggml_ftype ftype);
757
-
758
- GGML_API GGML_CALL bool ggml_is_transposed(const struct ggml_tensor * tensor);
759
- GGML_API GGML_CALL bool ggml_is_permuted (const struct ggml_tensor * tensor);
760
- GGML_API GGML_CALL bool ggml_is_empty (const struct ggml_tensor * tensor);
761
- GGML_API bool ggml_is_scalar (const struct ggml_tensor * tensor);
762
- GGML_API bool ggml_is_vector (const struct ggml_tensor * tensor);
763
- GGML_API bool ggml_is_matrix (const struct ggml_tensor * tensor);
764
- GGML_API bool ggml_is_3d (const struct ggml_tensor * tensor);
765
- GGML_API int ggml_n_dims (const struct ggml_tensor * tensor); // returns 1 for scalars
766
-
767
- GGML_API GGML_CALL bool ggml_is_contiguous (const struct ggml_tensor * tensor);
768
- GGML_API GGML_CALL bool ggml_is_contiguous_0(const struct ggml_tensor * tensor); // same as ggml_is_contiguous()
769
- GGML_API GGML_CALL bool ggml_is_contiguous_1(const struct ggml_tensor * tensor); // contiguous for dims >= 1
770
- GGML_API GGML_CALL bool ggml_is_contiguous_2(const struct ggml_tensor * tensor); // contiguous for dims >= 2
771
-
772
- GGML_API bool ggml_are_same_shape (const struct ggml_tensor * t0, const struct ggml_tensor * t1);
773
- GGML_API bool ggml_are_same_stride(const struct ggml_tensor * t0, const struct ggml_tensor * t1);
774
-
775
- // use this to compute the memory overhead of a tensor
776
- GGML_API size_t ggml_tensor_overhead(void);
777
-
778
- GGML_API bool ggml_validate_row_data(enum ggml_type type, const void * data, size_t nbytes);
779
-
780
- // main
781
-
782
- GGML_API struct ggml_context * ggml_init(struct ggml_init_params params);
783
- GGML_API void ggml_free(struct ggml_context * ctx);
784
-
785
- GGML_API size_t ggml_used_mem(const struct ggml_context * ctx);
786
-
787
- GGML_API size_t ggml_set_scratch (struct ggml_context * ctx, struct ggml_scratch scratch);
788
- GGML_API bool ggml_get_no_alloc(struct ggml_context * ctx);
789
- GGML_API void ggml_set_no_alloc(struct ggml_context * ctx, bool no_alloc);
790
-
791
- GGML_API void * ggml_get_mem_buffer (const struct ggml_context * ctx);
792
- GGML_API size_t ggml_get_mem_size (const struct ggml_context * ctx);
793
- GGML_API size_t ggml_get_max_tensor_size(const struct ggml_context * ctx);
794
-
795
- GGML_API struct ggml_tensor * ggml_new_tensor(
796
- struct ggml_context * ctx,
797
- enum ggml_type type,
798
- int n_dims,
799
- const int64_t *ne);
800
-
801
- GGML_API struct ggml_tensor * ggml_new_tensor_1d(
802
- struct ggml_context * ctx,
803
- enum ggml_type type,
804
- int64_t ne0);
805
-
806
- GGML_API struct ggml_tensor * ggml_new_tensor_2d(
807
- struct ggml_context * ctx,
808
- enum ggml_type type,
809
- int64_t ne0,
810
- int64_t ne1);
811
-
812
- GGML_API struct ggml_tensor * ggml_new_tensor_3d(
813
- struct ggml_context * ctx,
814
- enum ggml_type type,
815
- int64_t ne0,
816
- int64_t ne1,
817
- int64_t ne2);
818
-
819
- GGML_API struct ggml_tensor * ggml_new_tensor_4d(
820
- struct ggml_context * ctx,
821
- enum ggml_type type,
822
- int64_t ne0,
823
- int64_t ne1,
824
- int64_t ne2,
825
- int64_t ne3);
826
-
827
- GGML_API struct ggml_tensor * ggml_new_i32(struct ggml_context * ctx, int32_t value);
828
- GGML_API struct ggml_tensor * ggml_new_f32(struct ggml_context * ctx, float value);
829
-
830
- GGML_API struct ggml_tensor * ggml_dup_tensor (struct ggml_context * ctx, const struct ggml_tensor * src);
831
- GGML_API struct ggml_tensor * ggml_view_tensor(struct ggml_context * ctx, struct ggml_tensor * src);
832
-
833
- // Context tensor enumeration and lookup
834
- GGML_API struct ggml_tensor * ggml_get_first_tensor(const struct ggml_context * ctx);
835
- GGML_API struct ggml_tensor * ggml_get_next_tensor (const struct ggml_context * ctx, struct ggml_tensor * tensor);
836
- GGML_API struct ggml_tensor * ggml_get_tensor(struct ggml_context * ctx, const char * name);
837
-
838
- GGML_API struct ggml_tensor * ggml_set_zero(struct ggml_tensor * tensor);
839
- GGML_API struct ggml_tensor * ggml_set_i32 (struct ggml_tensor * tensor, int32_t value);
840
- GGML_API struct ggml_tensor * ggml_set_f32 (struct ggml_tensor * tensor, float value);
841
-
842
- // Converts a flat index into coordinates
843
- GGML_API void ggml_unravel_index(const struct ggml_tensor * tensor, int64_t i, int64_t * i0, int64_t * i1, int64_t * i2, int64_t * i3);
844
-
845
- GGML_API int32_t ggml_get_i32_1d(const struct ggml_tensor * tensor, int i);
846
- GGML_API void ggml_set_i32_1d(const struct ggml_tensor * tensor, int i, int32_t value);
847
-
848
- GGML_API int32_t ggml_get_i32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3);
849
- GGML_API void ggml_set_i32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3, int32_t value);
850
-
851
- GGML_API float ggml_get_f32_1d(const struct ggml_tensor * tensor, int i);
852
- GGML_API void ggml_set_f32_1d(const struct ggml_tensor * tensor, int i, float value);
853
-
854
- GGML_API float ggml_get_f32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3);
855
- GGML_API void ggml_set_f32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3, float value);
856
-
857
- GGML_API void * ggml_get_data (const struct ggml_tensor * tensor);
858
- GGML_API float * ggml_get_data_f32(const struct ggml_tensor * tensor);
859
-
860
- GGML_API GGML_CALL enum ggml_unary_op ggml_get_unary_op(const struct ggml_tensor * tensor);
861
-
862
- GGML_API const char * ggml_get_name (const struct ggml_tensor * tensor);
863
- GGML_API struct ggml_tensor * ggml_set_name ( struct ggml_tensor * tensor, const char * name);
864
- GGML_ATTRIBUTE_FORMAT(2, 3)
865
- GGML_API struct ggml_tensor * ggml_format_name( struct ggml_tensor * tensor, const char * fmt, ...);
866
-
867
- //
868
- // operations on tensors with backpropagation
869
- //
870
-
871
- GGML_API struct ggml_tensor * ggml_dup(
872
- struct ggml_context * ctx,
873
- struct ggml_tensor * a);
874
-
875
- // in-place, returns view(a)
876
- GGML_API struct ggml_tensor * ggml_dup_inplace(
877
- struct ggml_context * ctx,
878
- struct ggml_tensor * a);
879
-
880
- GGML_API struct ggml_tensor * ggml_add(
881
- struct ggml_context * ctx,
882
- struct ggml_tensor * a,
883
- struct ggml_tensor * b);
884
-
885
- GGML_API struct ggml_tensor * ggml_add_inplace(
886
- struct ggml_context * ctx,
887
- struct ggml_tensor * a,
888
- struct ggml_tensor * b);
889
-
890
- GGML_API struct ggml_tensor * ggml_add_cast(
891
- struct ggml_context * ctx,
892
- struct ggml_tensor * a,
893
- struct ggml_tensor * b,
894
- enum ggml_type type);
895
-
896
- GGML_API struct ggml_tensor * ggml_add1(
897
- struct ggml_context * ctx,
898
- struct ggml_tensor * a,
899
- struct ggml_tensor * b);
900
-
901
- GGML_API struct ggml_tensor * ggml_add1_inplace(
902
- struct ggml_context * ctx,
903
- struct ggml_tensor * a,
904
- struct ggml_tensor * b);
905
-
906
- // dst = a
907
- // view(dst, nb1, nb2, nb3, offset) += b
908
- // return dst
909
- GGML_API struct ggml_tensor * ggml_acc(
910
- struct ggml_context * ctx,
911
- struct ggml_tensor * a,
912
- struct ggml_tensor * b,
913
- size_t nb1,
914
- size_t nb2,
915
- size_t nb3,
916
- size_t offset);
917
-
918
- GGML_API struct ggml_tensor * ggml_acc_inplace(
919
- struct ggml_context * ctx,
920
- struct ggml_tensor * a,
921
- struct ggml_tensor * b,
922
- size_t nb1,
923
- size_t nb2,
924
- size_t nb3,
925
- size_t offset);
926
-
927
- GGML_API struct ggml_tensor * ggml_sub(
928
- struct ggml_context * ctx,
929
- struct ggml_tensor * a,
930
- struct ggml_tensor * b);
931
-
932
- GGML_API struct ggml_tensor * ggml_sub_inplace(
933
- struct ggml_context * ctx,
934
- struct ggml_tensor * a,
935
- struct ggml_tensor * b);
936
-
937
- GGML_API struct ggml_tensor * ggml_mul(
938
- struct ggml_context * ctx,
939
- struct ggml_tensor * a,
940
- struct ggml_tensor * b);
941
-
942
- GGML_API struct ggml_tensor * ggml_mul_inplace(
943
- struct ggml_context * ctx,
944
- struct ggml_tensor * a,
945
- struct ggml_tensor * b);
946
-
947
- GGML_API struct ggml_tensor * ggml_div(
948
- struct ggml_context * ctx,
949
- struct ggml_tensor * a,
950
- struct ggml_tensor * b);
951
-
952
- GGML_API struct ggml_tensor * ggml_div_inplace(
953
- struct ggml_context * ctx,
954
- struct ggml_tensor * a,
955
- struct ggml_tensor * b);
956
-
957
- GGML_API struct ggml_tensor * ggml_sqr(
958
- struct ggml_context * ctx,
959
- struct ggml_tensor * a);
960
-
961
- GGML_API struct ggml_tensor * ggml_sqr_inplace(
962
- struct ggml_context * ctx,
963
- struct ggml_tensor * a);
964
-
965
- GGML_API struct ggml_tensor * ggml_sqrt(
966
- struct ggml_context * ctx,
967
- struct ggml_tensor * a);
968
-
969
- GGML_API struct ggml_tensor * ggml_sqrt_inplace(
970
- struct ggml_context * ctx,
971
- struct ggml_tensor * a);
972
-
973
- GGML_API struct ggml_tensor * ggml_log(
974
- struct ggml_context * ctx,
975
- struct ggml_tensor * a);
976
-
977
- GGML_API struct ggml_tensor * ggml_log_inplace(
978
- struct ggml_context * ctx,
979
- struct ggml_tensor * a);
980
-
981
- // return scalar
982
- GGML_API struct ggml_tensor * ggml_sum(
983
- struct ggml_context * ctx,
984
- struct ggml_tensor * a);
985
-
986
- // sums along rows, with input shape [a,b,c,d] return shape [1,b,c,d]
987
- GGML_API struct ggml_tensor * ggml_sum_rows(
988
- struct ggml_context * ctx,
989
- struct ggml_tensor * a);
990
-
991
- // mean along rows
992
- GGML_API struct ggml_tensor * ggml_mean(
993
- struct ggml_context * ctx,
994
- struct ggml_tensor * a);
995
-
996
- // argmax along rows
997
- GGML_API struct ggml_tensor * ggml_argmax(
998
- struct ggml_context * ctx,
999
- struct ggml_tensor * a);
1000
-
1001
- // if a is the same shape as b, and a is not parameter, return a
1002
- // otherwise, return a new tensor: repeat(a) to fit in b
1003
- GGML_API struct ggml_tensor * ggml_repeat(
1004
- struct ggml_context * ctx,
1005
- struct ggml_tensor * a,
1006
- struct ggml_tensor * b);
1007
-
1008
- // sums repetitions in a into shape of b
1009
- GGML_API struct ggml_tensor * ggml_repeat_back(
1010
- struct ggml_context * ctx,
1011
- struct ggml_tensor * a,
1012
- struct ggml_tensor * b);
1013
-
1014
- // concat a and b along dim
1015
- // used in stable-diffusion
1016
- GGML_API struct ggml_tensor * ggml_concat(
1017
- struct ggml_context * ctx,
1018
- struct ggml_tensor * a,
1019
- struct ggml_tensor * b,
1020
- int dim);
1021
-
1022
- GGML_API struct ggml_tensor * ggml_abs(
1023
- struct ggml_context * ctx,
1024
- struct ggml_tensor * a);
1025
-
1026
- GGML_API struct ggml_tensor * ggml_abs_inplace(
1027
- struct ggml_context * ctx,
1028
- struct ggml_tensor * a);
1029
-
1030
- GGML_API struct ggml_tensor * ggml_sgn(
1031
- struct ggml_context * ctx,
1032
- struct ggml_tensor * a);
1033
-
1034
- GGML_API struct ggml_tensor * ggml_sgn_inplace(
1035
- struct ggml_context * ctx,
1036
- struct ggml_tensor * a);
1037
-
1038
- GGML_API struct ggml_tensor * ggml_neg(
1039
- struct ggml_context * ctx,
1040
- struct ggml_tensor * a);
1041
-
1042
- GGML_API struct ggml_tensor * ggml_neg_inplace(
1043
- struct ggml_context * ctx,
1044
- struct ggml_tensor * a);
1045
-
1046
- GGML_API struct ggml_tensor * ggml_step(
1047
- struct ggml_context * ctx,
1048
- struct ggml_tensor * a);
1049
-
1050
- GGML_API struct ggml_tensor * ggml_step_inplace(
1051
- struct ggml_context * ctx,
1052
- struct ggml_tensor * a);
1053
-
1054
- GGML_API struct ggml_tensor * ggml_tanh(
1055
- struct ggml_context * ctx,
1056
- struct ggml_tensor * a);
1057
-
1058
- GGML_API struct ggml_tensor * ggml_tanh_inplace(
1059
- struct ggml_context * ctx,
1060
- struct ggml_tensor * a);
1061
-
1062
- GGML_API struct ggml_tensor * ggml_elu(
1063
- struct ggml_context * ctx,
1064
- struct ggml_tensor * a);
1065
-
1066
- GGML_API struct ggml_tensor * ggml_elu_inplace(
1067
- struct ggml_context * ctx,
1068
- struct ggml_tensor * a);
1069
-
1070
- GGML_API struct ggml_tensor * ggml_relu(
1071
- struct ggml_context * ctx,
1072
- struct ggml_tensor * a);
1073
-
1074
- GGML_API struct ggml_tensor * ggml_leaky_relu(
1075
- struct ggml_context * ctx,
1076
- struct ggml_tensor * a, float negative_slope, bool inplace);
1077
-
1078
- GGML_API struct ggml_tensor * ggml_relu_inplace(
1079
- struct ggml_context * ctx,
1080
- struct ggml_tensor * a);
1081
-
1082
- GGML_API struct ggml_tensor * ggml_sigmoid(
1083
- struct ggml_context * ctx,
1084
- struct ggml_tensor * a);
1085
-
1086
- GGML_API struct ggml_tensor * ggml_sigmoid_inplace(
1087
- struct ggml_context * ctx,
1088
- struct ggml_tensor * a);
1089
-
1090
- GGML_API struct ggml_tensor * ggml_gelu(
1091
- struct ggml_context * ctx,
1092
- struct ggml_tensor * a);
1093
-
1094
- GGML_API struct ggml_tensor * ggml_gelu_inplace(
1095
- struct ggml_context * ctx,
1096
- struct ggml_tensor * a);
1097
-
1098
- GGML_API struct ggml_tensor * ggml_gelu_quick(
1099
- struct ggml_context * ctx,
1100
- struct ggml_tensor * a);
1101
-
1102
- GGML_API struct ggml_tensor * ggml_gelu_quick_inplace(
1103
- struct ggml_context * ctx,
1104
- struct ggml_tensor * a);
1105
-
1106
- GGML_API struct ggml_tensor * ggml_silu(
1107
- struct ggml_context * ctx,
1108
- struct ggml_tensor * a);
1109
-
1110
- GGML_API struct ggml_tensor * ggml_silu_inplace(
1111
- struct ggml_context * ctx,
1112
- struct ggml_tensor * a);
1113
-
1114
- // a - x
1115
- // b - dy
1116
- GGML_API struct ggml_tensor * ggml_silu_back(
1117
- struct ggml_context * ctx,
1118
- struct ggml_tensor * a,
1119
- struct ggml_tensor * b);
1120
-
1121
- // hardswish(x) = x * relu6(x + 3) / 6
1122
- GGML_API struct ggml_tensor * ggml_hardswish(
1123
- struct ggml_context * ctx,
1124
- struct ggml_tensor * a);
1125
-
1126
- // hardsigmoid(x) = relu6(x + 3) / 6
1127
- GGML_API struct ggml_tensor * ggml_hardsigmoid(
1128
- struct ggml_context * ctx,
1129
- struct ggml_tensor * a);
1130
-
1131
- // normalize along rows
1132
- GGML_API struct ggml_tensor * ggml_norm(
1133
- struct ggml_context * ctx,
1134
- struct ggml_tensor * a,
1135
- float eps);
1136
-
1137
- GGML_API struct ggml_tensor * ggml_norm_inplace(
1138
- struct ggml_context * ctx,
1139
- struct ggml_tensor * a,
1140
- float eps);
1141
-
1142
- GGML_API struct ggml_tensor * ggml_rms_norm(
1143
- struct ggml_context * ctx,
1144
- struct ggml_tensor * a,
1145
- float eps);
1146
-
1147
- GGML_API struct ggml_tensor * ggml_rms_norm_inplace(
1148
- struct ggml_context * ctx,
1149
- struct ggml_tensor * a,
1150
- float eps);
1151
-
1152
- // group normalize along ne0*ne1*n_groups
1153
- // used in stable-diffusion
1154
- // TODO: eps is hardcoded to 1e-6 for now
1155
- GGML_API struct ggml_tensor * ggml_group_norm(
1156
- struct ggml_context * ctx,
1157
- struct ggml_tensor * a,
1158
- int n_groups);
1159
-
1160
- GGML_API struct ggml_tensor * ggml_group_norm_inplace(
1161
- struct ggml_context * ctx,
1162
- struct ggml_tensor * a,
1163
- int n_groups);
1164
-
1165
- // a - x
1166
- // b - dy
1167
- GGML_API struct ggml_tensor * ggml_rms_norm_back(
1168
- struct ggml_context * ctx,
1169
- struct ggml_tensor * a,
1170
- struct ggml_tensor * b,
1171
- float eps);
1172
-
1173
- // A: k columns, n rows => [ne03, ne02, n, k]
1174
- // B: k columns, m rows (i.e. we transpose it internally) => [ne03 * x, ne02 * y, m, k]
1175
- // result is n columns, m rows => [ne03 * x, ne02 * y, m, n]
1176
- GGML_API struct ggml_tensor * ggml_mul_mat(
1177
- struct ggml_context * ctx,
1178
- struct ggml_tensor * a,
1179
- struct ggml_tensor * b);
1180
-
1181
- // change the precision of a matrix multiplication
1182
- // set to GGML_PREC_F32 for higher precision (useful for phi-2)
1183
- GGML_API void ggml_mul_mat_set_prec(
1184
- struct ggml_tensor * a,
1185
- enum ggml_prec prec);
1186
-
1187
- // indirect matrix multiplication
1188
- GGML_API struct ggml_tensor * ggml_mul_mat_id(
1189
- struct ggml_context * ctx,
1190
- struct ggml_tensor * as,
1191
- struct ggml_tensor * b,
1192
- struct ggml_tensor * ids);
1193
-
1194
- // A: m columns, n rows,
1195
- // B: p columns, n rows,
1196
- // result is m columns, p rows
1197
- GGML_API struct ggml_tensor * ggml_out_prod(
1198
- struct ggml_context * ctx,
1199
- struct ggml_tensor * a,
1200
- struct ggml_tensor * b);
1201
-
1202
- //
1203
- // operations on tensors without backpropagation
1204
- //
1205
-
1206
- GGML_API struct ggml_tensor * ggml_scale(
1207
- struct ggml_context * ctx,
1208
- struct ggml_tensor * a,
1209
- float s);
1210
-
1211
- // in-place, returns view(a)
1212
- GGML_API struct ggml_tensor * ggml_scale_inplace(
1213
- struct ggml_context * ctx,
1214
- struct ggml_tensor * a,
1215
- float s);
1216
-
1217
- // b -> view(a,offset,nb1,nb2,3), return modified a
1218
- GGML_API struct ggml_tensor * ggml_set(
1219
- struct ggml_context * ctx,
1220
- struct ggml_tensor * a,
1221
- struct ggml_tensor * b,
1222
- size_t nb1,
1223
- size_t nb2,
1224
- size_t nb3,
1225
- size_t offset);
1226
-
1227
- // b -> view(a,offset,nb1,nb2,3), return view(a)
1228
- GGML_API struct ggml_tensor * ggml_set_inplace(
1229
- struct ggml_context * ctx,
1230
- struct ggml_tensor * a,
1231
- struct ggml_tensor * b,
1232
- size_t nb1,
1233
- size_t nb2,
1234
- size_t nb3,
1235
- size_t offset);
1236
-
1237
- GGML_API struct ggml_tensor * ggml_set_1d(
1238
- struct ggml_context * ctx,
1239
- struct ggml_tensor * a,
1240
- struct ggml_tensor * b,
1241
- size_t offset);
1242
-
1243
- GGML_API struct ggml_tensor * ggml_set_1d_inplace(
1244
- struct ggml_context * ctx,
1245
- struct ggml_tensor * a,
1246
- struct ggml_tensor * b,
1247
- size_t offset);
1248
-
1249
- // b -> view(a,offset,nb1,nb2,3), return modified a
1250
- GGML_API struct ggml_tensor * ggml_set_2d(
1251
- struct ggml_context * ctx,
1252
- struct ggml_tensor * a,
1253
- struct ggml_tensor * b,
1254
- size_t nb1,
1255
- size_t offset);
1256
-
1257
- // b -> view(a,offset,nb1,nb2,3), return view(a)
1258
- GGML_API struct ggml_tensor * ggml_set_2d_inplace(
1259
- struct ggml_context * ctx,
1260
- struct ggml_tensor * a,
1261
- struct ggml_tensor * b,
1262
- size_t nb1,
1263
- size_t offset);
1264
-
1265
- // a -> b, return view(b)
1266
- GGML_API struct ggml_tensor * ggml_cpy(
1267
- struct ggml_context * ctx,
1268
- struct ggml_tensor * a,
1269
- struct ggml_tensor * b);
1270
-
1271
- GGML_API struct ggml_tensor * ggml_cast(
1272
- struct ggml_context * ctx,
1273
- struct ggml_tensor * a,
1274
- enum ggml_type type);
1275
-
1276
- // make contiguous
1277
- GGML_API struct ggml_tensor * ggml_cont(
1278
- struct ggml_context * ctx,
1279
- struct ggml_tensor * a);
1280
-
1281
- // make contiguous, with new shape
1282
- GGML_API struct ggml_tensor * ggml_cont_1d(
1283
- struct ggml_context * ctx,
1284
- struct ggml_tensor * a,
1285
- int64_t ne0);
1286
-
1287
- GGML_API struct ggml_tensor * ggml_cont_2d(
1288
- struct ggml_context * ctx,
1289
- struct ggml_tensor * a,
1290
- int64_t ne0,
1291
- int64_t ne1);
1292
-
1293
- GGML_API struct ggml_tensor * ggml_cont_3d(
1294
- struct ggml_context * ctx,
1295
- struct ggml_tensor * a,
1296
- int64_t ne0,
1297
- int64_t ne1,
1298
- int64_t ne2);
1299
-
1300
- GGML_API struct ggml_tensor * ggml_cont_4d(
1301
- struct ggml_context * ctx,
1302
- struct ggml_tensor * a,
1303
- int64_t ne0,
1304
- int64_t ne1,
1305
- int64_t ne2,
1306
- int64_t ne3);
1307
-
1308
- // return view(a), b specifies the new shape
1309
- // TODO: when we start computing gradient, make a copy instead of view
1310
- GGML_API struct ggml_tensor * ggml_reshape(
1311
- struct ggml_context * ctx,
1312
- struct ggml_tensor * a,
1313
- struct ggml_tensor * b);
1314
-
1315
- // return view(a)
1316
- // TODO: when we start computing gradient, make a copy instead of view
1317
- GGML_API struct ggml_tensor * ggml_reshape_1d(
1318
- struct ggml_context * ctx,
1319
- struct ggml_tensor * a,
1320
- int64_t ne0);
1321
-
1322
- GGML_API struct ggml_tensor * ggml_reshape_2d(
1323
- struct ggml_context * ctx,
1324
- struct ggml_tensor * a,
1325
- int64_t ne0,
1326
- int64_t ne1);
1327
-
1328
- // return view(a)
1329
- // TODO: when we start computing gradient, make a copy instead of view
1330
- GGML_API struct ggml_tensor * ggml_reshape_3d(
1331
- struct ggml_context * ctx,
1332
- struct ggml_tensor * a,
1333
- int64_t ne0,
1334
- int64_t ne1,
1335
- int64_t ne2);
1336
-
1337
- GGML_API struct ggml_tensor * ggml_reshape_4d(
1338
- struct ggml_context * ctx,
1339
- struct ggml_tensor * a,
1340
- int64_t ne0,
1341
- int64_t ne1,
1342
- int64_t ne2,
1343
- int64_t ne3);
1344
-
1345
- // offset in bytes
1346
- GGML_API struct ggml_tensor * ggml_view_1d(
1347
- struct ggml_context * ctx,
1348
- struct ggml_tensor * a,
1349
- int64_t ne0,
1350
- size_t offset);
1351
-
1352
- GGML_API struct ggml_tensor * ggml_view_2d(
1353
- struct ggml_context * ctx,
1354
- struct ggml_tensor * a,
1355
- int64_t ne0,
1356
- int64_t ne1,
1357
- size_t nb1, // row stride in bytes
1358
- size_t offset);
1359
-
1360
- GGML_API struct ggml_tensor * ggml_view_3d(
1361
- struct ggml_context * ctx,
1362
- struct ggml_tensor * a,
1363
- int64_t ne0,
1364
- int64_t ne1,
1365
- int64_t ne2,
1366
- size_t nb1, // row stride in bytes
1367
- size_t nb2, // slice stride in bytes
1368
- size_t offset);
1369
-
1370
- GGML_API struct ggml_tensor * ggml_view_4d(
1371
- struct ggml_context * ctx,
1372
- struct ggml_tensor * a,
1373
- int64_t ne0,
1374
- int64_t ne1,
1375
- int64_t ne2,
1376
- int64_t ne3,
1377
- size_t nb1, // row stride in bytes
1378
- size_t nb2, // slice stride in bytes
1379
- size_t nb3,
1380
- size_t offset);
1381
-
1382
- GGML_API struct ggml_tensor * ggml_permute(
1383
- struct ggml_context * ctx,
1384
- struct ggml_tensor * a,
1385
- int axis0,
1386
- int axis1,
1387
- int axis2,
1388
- int axis3);
1389
-
1390
- // alias for ggml_permute(ctx, a, 1, 0, 2, 3)
1391
- GGML_API struct ggml_tensor * ggml_transpose(
1392
- struct ggml_context * ctx,
1393
- struct ggml_tensor * a);
1394
-
1395
- // supports 3D: a->ne[2] == b->ne[1]
1396
- GGML_API struct ggml_tensor * ggml_get_rows(
1397
- struct ggml_context * ctx,
1398
- struct ggml_tensor * a,
1399
- struct ggml_tensor * b);
1400
-
1401
- GGML_API struct ggml_tensor * ggml_get_rows_back(
1402
- struct ggml_context * ctx,
1403
- struct ggml_tensor * a,
1404
- struct ggml_tensor * b,
1405
- struct ggml_tensor * c);
1406
-
1407
- GGML_API struct ggml_tensor * ggml_diag(
1408
- struct ggml_context * ctx,
1409
- struct ggml_tensor * a);
1410
-
1411
- // set elements above the diagonal to -INF
1412
- GGML_API struct ggml_tensor * ggml_diag_mask_inf(
1413
- struct ggml_context * ctx,
1414
- struct ggml_tensor * a,
1415
- int n_past);
1416
-
1417
- // in-place, returns view(a)
1418
- GGML_API struct ggml_tensor * ggml_diag_mask_inf_inplace(
1419
- struct ggml_context * ctx,
1420
- struct ggml_tensor * a,
1421
- int n_past);
1422
-
1423
- // set elements above the diagonal to 0
1424
- GGML_API struct ggml_tensor * ggml_diag_mask_zero(
1425
- struct ggml_context * ctx,
1426
- struct ggml_tensor * a,
1427
- int n_past);
1428
-
1429
- // in-place, returns view(a)
1430
- GGML_API struct ggml_tensor * ggml_diag_mask_zero_inplace(
1431
- struct ggml_context * ctx,
1432
- struct ggml_tensor * a,
1433
- int n_past);
1434
-
1435
- GGML_API struct ggml_tensor * ggml_soft_max(
1436
- struct ggml_context * ctx,
1437
- struct ggml_tensor * a);
1438
-
1439
- // in-place, returns view(a)
1440
- GGML_API struct ggml_tensor * ggml_soft_max_inplace(
1441
- struct ggml_context * ctx,
1442
- struct ggml_tensor * a);
1443
-
1444
- // fused soft_max(a*scale + mask*(ALiBi slope))
1445
- // mask is optional
1446
- // max_bias = 0.0f for no ALiBi
1447
- GGML_API struct ggml_tensor * ggml_soft_max_ext(
1448
- struct ggml_context * ctx,
1449
- struct ggml_tensor * a,
1450
- struct ggml_tensor * mask,
1451
- float scale,
1452
- float max_bias);
1453
-
1454
- GGML_API struct ggml_tensor * ggml_soft_max_back(
1455
- struct ggml_context * ctx,
1456
- struct ggml_tensor * a,
1457
- struct ggml_tensor * b);
1458
-
1459
- // in-place, returns view(a)
1460
- GGML_API struct ggml_tensor * ggml_soft_max_back_inplace(
1461
- struct ggml_context * ctx,
1462
- struct ggml_tensor * a,
1463
- struct ggml_tensor * b);
1464
-
1465
- // rotary position embedding
1466
- // if mode & 1 == 1, skip n_past elements (NOT SUPPORTED)
1467
- // if mode & 2 == 1, GPT-NeoX style
1468
- //
1469
- // b is an int32 vector with size a->ne[2], it contains the positions
1470
- // c is freq factors (e.g. phi3-128k), (optional)
1471
- GGML_API struct ggml_tensor * ggml_rope(
1472
- struct ggml_context * ctx,
1473
- struct ggml_tensor * a,
1474
- struct ggml_tensor * b,
1475
- int n_dims,
1476
- int mode);
1477
-
1478
- // in-place, returns view(a)
1479
- GGML_API struct ggml_tensor * ggml_rope_inplace(
1480
- struct ggml_context * ctx,
1481
- struct ggml_tensor * a,
1482
- struct ggml_tensor * b,
1483
- int n_dims,
1484
- int mode);
1485
-
1486
- // custom RoPE
1487
- GGML_API struct ggml_tensor * ggml_rope_ext(
1488
- struct ggml_context * ctx,
1489
- struct ggml_tensor * a,
1490
- struct ggml_tensor * b,
1491
- struct ggml_tensor * c,
1492
- int n_dims,
1493
- int mode,
1494
- int n_ctx_orig,
1495
- float freq_base,
1496
- float freq_scale,
1497
- float ext_factor,
1498
- float attn_factor,
1499
- float beta_fast,
1500
- float beta_slow);
1501
-
1502
- // in-place, returns view(a)
1503
- GGML_API struct ggml_tensor * ggml_rope_ext_inplace(
1504
- struct ggml_context * ctx,
1505
- struct ggml_tensor * a,
1506
- struct ggml_tensor * b,
1507
- struct ggml_tensor * c,
1508
- int n_dims,
1509
- int mode,
1510
- int n_ctx_orig,
1511
- float freq_base,
1512
- float freq_scale,
1513
- float ext_factor,
1514
- float attn_factor,
1515
- float beta_fast,
1516
- float beta_slow);
1517
-
1518
- GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_rope_custom(
1519
- struct ggml_context * ctx,
1520
- struct ggml_tensor * a,
1521
- struct ggml_tensor * b,
1522
- int n_dims,
1523
- int mode,
1524
- int n_ctx_orig,
1525
- float freq_base,
1526
- float freq_scale,
1527
- float ext_factor,
1528
- float attn_factor,
1529
- float beta_fast,
1530
- float beta_slow),
1531
- "use ggml_rope_ext instead");
1532
-
1533
- GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_rope_custom_inplace(
1534
- struct ggml_context * ctx,
1535
- struct ggml_tensor * a,
1536
- struct ggml_tensor * b,
1537
- int n_dims,
1538
- int mode,
1539
- int n_ctx_orig,
1540
- float freq_base,
1541
- float freq_scale,
1542
- float ext_factor,
1543
- float attn_factor,
1544
- float beta_fast,
1545
- float beta_slow),
1546
- "use ggml_rope_ext_inplace instead");
1547
-
1548
- // compute correction dims for YaRN RoPE scaling
1549
- GGML_CALL void ggml_rope_yarn_corr_dims(
1550
- int n_dims, int n_ctx_orig, float freq_base, float beta_fast, float beta_slow, float dims[2]);
1551
-
1552
- // rotary position embedding backward, i.e compute dx from dy
1553
- // a - dy
1554
- GGML_API struct ggml_tensor * ggml_rope_back(
1555
- struct ggml_context * ctx,
1556
- struct ggml_tensor * a,
1557
- struct ggml_tensor * b,
1558
- struct ggml_tensor * c,
1559
- int n_dims,
1560
- int mode,
1561
- int n_ctx_orig,
1562
- float freq_base,
1563
- float freq_scale,
1564
- float ext_factor,
1565
- float attn_factor,
1566
- float beta_fast,
1567
- float beta_slow);
1568
-
1569
- // clamp
1570
- // in-place, returns view(a)
1571
- GGML_API struct ggml_tensor * ggml_clamp(
1572
- struct ggml_context * ctx,
1573
- struct ggml_tensor * a,
1574
- float min,
1575
- float max);
1576
-
1577
- GGML_API struct ggml_tensor * ggml_im2col(
1578
- struct ggml_context * ctx,
1579
- struct ggml_tensor * a,
1580
- struct ggml_tensor * b,
1581
- int s0,
1582
- int s1,
1583
- int p0,
1584
- int p1,
1585
- int d0,
1586
- int d1,
1587
- bool is_2D,
1588
- enum ggml_type dst_type);
1589
-
1590
- GGML_API struct ggml_tensor * ggml_conv_depthwise_2d(
1591
- struct ggml_context * ctx,
1592
- struct ggml_tensor * a,
1593
- struct ggml_tensor * b,
1594
- int s0,
1595
- int s1,
1596
- int p0,
1597
- int p1,
1598
- int d0,
1599
- int d1);
1600
-
1601
- GGML_API struct ggml_tensor * ggml_conv_1d(
1602
- struct ggml_context * ctx,
1603
- struct ggml_tensor * a,
1604
- struct ggml_tensor * b,
1605
- int s0, // stride
1606
- int p0, // padding
1607
- int d0); // dilation
1608
-
1609
- // conv_1d with padding = half
1610
- // alias for ggml_conv_1d(a, b, s, a->ne[0]/2, d)
1611
- GGML_API struct ggml_tensor* ggml_conv_1d_ph(
1612
- struct ggml_context * ctx,
1613
- struct ggml_tensor * a,
1614
- struct ggml_tensor * b,
1615
- int s,
1616
- int d);
1617
-
1618
- GGML_API struct ggml_tensor * ggml_conv_transpose_1d(
1619
- struct ggml_context * ctx,
1620
- struct ggml_tensor * a,
1621
- struct ggml_tensor * b,
1622
- int s0,
1623
- int p0,
1624
- int d0);
1625
-
1626
- GGML_API struct ggml_tensor * ggml_conv_2d(
1627
- struct ggml_context * ctx,
1628
- struct ggml_tensor * a,
1629
- struct ggml_tensor * b,
1630
- int s0,
1631
- int s1,
1632
- int p0,
1633
- int p1,
1634
- int d0,
1635
- int d1);
1636
-
1637
-
1638
- // kernel size is a->ne[0] x a->ne[1]
1639
- // stride is equal to kernel size
1640
- // padding is zero
1641
- // example:
1642
- // a: 16 16 3 768
1643
- // b: 1024 1024 3 1
1644
- // res: 64 64 768 1
1645
- // used in sam
1646
- GGML_API struct ggml_tensor * ggml_conv_2d_sk_p0(
1647
- struct ggml_context * ctx,
1648
- struct ggml_tensor * a,
1649
- struct ggml_tensor * b);
1650
-
1651
- // kernel size is a->ne[0] x a->ne[1]
1652
- // stride is 1
1653
- // padding is half
1654
- // example:
1655
- // a: 3 3 256 256
1656
- // b: 64 64 256 1
1657
- // res: 64 64 256 1
1658
- // used in sam
1659
- GGML_API struct ggml_tensor * ggml_conv_2d_s1_ph(
1660
- struct ggml_context * ctx,
1661
- struct ggml_tensor * a,
1662
- struct ggml_tensor * b);
1663
-
1664
- GGML_API struct ggml_tensor * ggml_conv_transpose_2d_p0(
1665
- struct ggml_context * ctx,
1666
- struct ggml_tensor * a,
1667
- struct ggml_tensor * b,
1668
- int stride);
1669
-
1670
- enum ggml_op_pool {
1671
- GGML_OP_POOL_MAX,
1672
- GGML_OP_POOL_AVG,
1673
- GGML_OP_POOL_COUNT,
1674
- };
1675
-
1676
- GGML_API struct ggml_tensor * ggml_pool_1d(
1677
- struct ggml_context * ctx,
1678
- struct ggml_tensor * a,
1679
- enum ggml_op_pool op,
1680
- int k0, // kernel size
1681
- int s0, // stride
1682
- int p0); // padding
1683
-
1684
- // the result will have 2*p0 padding for the first dimension
1685
- // and 2*p1 padding for the second dimension
1686
- GGML_API struct ggml_tensor * ggml_pool_2d(
1687
- struct ggml_context * ctx,
1688
- struct ggml_tensor * a,
1689
- enum ggml_op_pool op,
1690
- int k0,
1691
- int k1,
1692
- int s0,
1693
- int s1,
1694
- float p0,
1695
- float p1);
1696
-
1697
- // nearest interpolate
1698
- // multiplies ne0 and ne1 by scale factor
1699
- // used in stable-diffusion
1700
- GGML_API struct ggml_tensor * ggml_upscale(
1701
- struct ggml_context * ctx,
1702
- struct ggml_tensor * a,
1703
- int scale_factor);
1704
-
1705
- // nearest interpolate
1706
- // nearest interpolate to specified dimensions
1707
- // used in tortoise.cpp
1708
- GGML_API struct ggml_tensor * ggml_upscale_ext(
1709
- struct ggml_context * ctx,
1710
- struct ggml_tensor * a,
1711
- int ne0,
1712
- int ne1,
1713
- int ne2,
1714
- int ne3);
1715
-
1716
- // pad each dimension with zeros: [x, ..., x] -> [x, ..., x, 0, ..., 0]
1717
- GGML_API struct ggml_tensor * ggml_pad(
1718
- struct ggml_context * ctx,
1719
- struct ggml_tensor * a,
1720
- int p0,
1721
- int p1,
1722
- int p2,
1723
- int p3);
1724
-
1725
- // Ref: https://github.com/CompVis/stable-diffusion/blob/main/ldm/modules/diffusionmodules/util.py#L151
1726
- // timesteps: [N,]
1727
- // return: [N, dim]
1728
- GGML_API struct ggml_tensor * ggml_timestep_embedding(
1729
- struct ggml_context * ctx,
1730
- struct ggml_tensor * timesteps,
1731
- int dim,
1732
- int max_period);
1733
-
1734
- // sort rows
1735
- enum ggml_sort_order {
1736
- GGML_SORT_ORDER_ASC,
1737
- GGML_SORT_ORDER_DESC,
1738
- };
1739
-
1740
- GGML_API struct ggml_tensor * ggml_argsort(
1741
- struct ggml_context * ctx,
1742
- struct ggml_tensor * a,
1743
- enum ggml_sort_order order);
1744
-
1745
- GGML_API struct ggml_tensor * ggml_arange(
1746
- struct ggml_context * ctx,
1747
- float start,
1748
- float stop,
1749
- float step);
1750
-
1751
- // top k elements per row
1752
- GGML_API struct ggml_tensor * ggml_top_k(
1753
- struct ggml_context * ctx,
1754
- struct ggml_tensor * a,
1755
- int k);
1756
-
1757
- #define GGML_KQ_MASK_PAD 32
1758
-
1759
- // q: [n_embd, n_batch, n_head, 1]
1760
- // k: [n_embd, n_kv, n_head_kv, 1]
1761
- // v: [n_embd, n_kv, n_head_kv, 1] !! not transposed !!
1762
- // mask: [n_kv, n_batch_pad, 1, 1] !! n_batch_pad = GGML_PAD(n_batch, GGML_KQ_MASK_PAD) !!
1763
- // res: [n_embd, n_head, n_batch, 1] !! permuted !!
1764
- GGML_API struct ggml_tensor * ggml_flash_attn_ext(
1765
- struct ggml_context * ctx,
1766
- struct ggml_tensor * q,
1767
- struct ggml_tensor * k,
1768
- struct ggml_tensor * v,
1769
- struct ggml_tensor * mask,
1770
- float scale,
1771
- float max_bias);
1772
-
1773
- GGML_API void ggml_flash_attn_ext_set_prec(
1774
- struct ggml_tensor * a,
1775
- enum ggml_prec prec);
1776
-
1777
- // TODO: needs to be adapted to ggml_flash_attn_ext
1778
- GGML_API struct ggml_tensor * ggml_flash_attn_back(
1779
- struct ggml_context * ctx,
1780
- struct ggml_tensor * q,
1781
- struct ggml_tensor * k,
1782
- struct ggml_tensor * v,
1783
- struct ggml_tensor * d,
1784
- bool masked);
1785
-
1786
- GGML_API struct ggml_tensor * ggml_ssm_conv(
1787
- struct ggml_context * ctx,
1788
- struct ggml_tensor * s,
1789
- struct ggml_tensor * x,
1790
- struct ggml_tensor * c,
1791
- struct ggml_tensor * sq);
1792
-
1793
- GGML_API struct ggml_tensor * ggml_ssm_scan(
1794
- struct ggml_context * ctx,
1795
- struct ggml_tensor * s,
1796
- struct ggml_tensor * x,
1797
- struct ggml_tensor * dt,
1798
- struct ggml_tensor * A,
1799
- struct ggml_tensor * B,
1800
- struct ggml_tensor * C,
1801
- struct ggml_tensor * sq);
1802
-
1803
- // partition into non-overlapping windows with padding if needed
1804
- // example:
1805
- // a: 768 64 64 1
1806
- // w: 14
1807
- // res: 768 14 14 25
1808
- // used in sam
1809
- GGML_API struct ggml_tensor * ggml_win_part(
1810
- struct ggml_context * ctx,
1811
- struct ggml_tensor * a,
1812
- int w);
1813
-
1814
- // reverse of ggml_win_part
1815
- // used in sam
1816
- GGML_API struct ggml_tensor * ggml_win_unpart(
1817
- struct ggml_context * ctx,
1818
- struct ggml_tensor * a,
1819
- int w0,
1820
- int h0,
1821
- int w);
1822
-
1823
- GGML_API struct ggml_tensor * ggml_unary(
1824
- struct ggml_context * ctx,
1825
- struct ggml_tensor * a,
1826
- enum ggml_unary_op op);
1827
-
1828
- GGML_API struct ggml_tensor * ggml_unary_inplace(
1829
- struct ggml_context * ctx,
1830
- struct ggml_tensor * a,
1831
- enum ggml_unary_op op);
1832
-
1833
- // used in sam
1834
- GGML_API struct ggml_tensor * ggml_get_rel_pos(
1835
- struct ggml_context * ctx,
1836
- struct ggml_tensor * a,
1837
- int qh,
1838
- int kh);
1839
-
1840
- // used in sam
1841
- GGML_API struct ggml_tensor * ggml_add_rel_pos(
1842
- struct ggml_context * ctx,
1843
- struct ggml_tensor * a,
1844
- struct ggml_tensor * pw,
1845
- struct ggml_tensor * ph);
1846
-
1847
- GGML_API struct ggml_tensor * ggml_add_rel_pos_inplace(
1848
- struct ggml_context * ctx,
1849
- struct ggml_tensor * a,
1850
- struct ggml_tensor * pw,
1851
- struct ggml_tensor * ph);
1852
-
1853
- // custom operators
1854
-
1855
- typedef void (*ggml_unary_op_f32_t) (const int, float *, const float *);
1856
- typedef void (*ggml_binary_op_f32_t)(const int, float *, const float *, const float *);
1857
-
1858
- typedef void (*ggml_custom1_op_f32_t)(struct ggml_tensor *, const struct ggml_tensor *);
1859
- typedef void (*ggml_custom2_op_f32_t)(struct ggml_tensor *, const struct ggml_tensor *, const struct ggml_tensor *);
1860
- typedef void (*ggml_custom3_op_f32_t)(struct ggml_tensor *, const struct ggml_tensor *, const struct ggml_tensor *, const struct ggml_tensor *);
1861
-
1862
- GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_map_unary_f32(
1863
- struct ggml_context * ctx,
1864
- struct ggml_tensor * a,
1865
- ggml_unary_op_f32_t fun),
1866
- "use ggml_map_custom1 instead");
1867
-
1868
- GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_map_unary_inplace_f32(
1869
- struct ggml_context * ctx,
1870
- struct ggml_tensor * a,
1871
- ggml_unary_op_f32_t fun),
1872
- "use ggml_map_custom1_inplace instead");
1873
-
1874
- GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_map_binary_f32(
1875
- struct ggml_context * ctx,
1876
- struct ggml_tensor * a,
1877
- struct ggml_tensor * b,
1878
- ggml_binary_op_f32_t fun),
1879
- "use ggml_map_custom2 instead");
1880
-
1881
- GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_map_binary_inplace_f32(
1882
- struct ggml_context * ctx,
1883
- struct ggml_tensor * a,
1884
- struct ggml_tensor * b,
1885
- ggml_binary_op_f32_t fun),
1886
- "use ggml_map_custom2_inplace instead");
1887
-
1888
- GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_map_custom1_f32(
1889
- struct ggml_context * ctx,
1890
- struct ggml_tensor * a,
1891
- ggml_custom1_op_f32_t fun),
1892
- "use ggml_map_custom1 instead");
1893
-
1894
- GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_map_custom1_inplace_f32(
1895
- struct ggml_context * ctx,
1896
- struct ggml_tensor * a,
1897
- ggml_custom1_op_f32_t fun),
1898
- "use ggml_map_custom1_inplace instead");
1899
-
1900
- GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_map_custom2_f32(
1901
- struct ggml_context * ctx,
1902
- struct ggml_tensor * a,
1903
- struct ggml_tensor * b,
1904
- ggml_custom2_op_f32_t fun),
1905
- "use ggml_map_custom2 instead");
1906
-
1907
- GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_map_custom2_inplace_f32(
1908
- struct ggml_context * ctx,
1909
- struct ggml_tensor * a,
1910
- struct ggml_tensor * b,
1911
- ggml_custom2_op_f32_t fun),
1912
- "use ggml_map_custom2_inplace instead");
1913
-
1914
- GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_map_custom3_f32(
1915
- struct ggml_context * ctx,
1916
- struct ggml_tensor * a,
1917
- struct ggml_tensor * b,
1918
- struct ggml_tensor * c,
1919
- ggml_custom3_op_f32_t fun),
1920
- "use ggml_map_custom3 instead");
1921
-
1922
- GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_map_custom3_inplace_f32(
1923
- struct ggml_context * ctx,
1924
- struct ggml_tensor * a,
1925
- struct ggml_tensor * b,
1926
- struct ggml_tensor * c,
1927
- ggml_custom3_op_f32_t fun),
1928
- "use ggml_map_custom3_inplace instead");
1929
-
1930
- // custom operators v2
1931
-
1932
- typedef void (*ggml_custom1_op_t)(struct ggml_tensor * dst , const struct ggml_tensor * a, int ith, int nth, void * userdata);
1933
- typedef void (*ggml_custom2_op_t)(struct ggml_tensor * dst , const struct ggml_tensor * a, const struct ggml_tensor * b, int ith, int nth, void * userdata);
1934
- typedef void (*ggml_custom3_op_t)(struct ggml_tensor * dst , const struct ggml_tensor * a, const struct ggml_tensor * b, const struct ggml_tensor * c, int ith, int nth, void * userdata);
1935
-
1936
- #define GGML_N_TASKS_MAX -1
1937
-
1938
- GGML_API struct ggml_tensor * ggml_map_custom1(
1939
- struct ggml_context * ctx,
1940
- struct ggml_tensor * a,
1941
- ggml_custom1_op_t fun,
1942
- int n_tasks,
1943
- void * userdata);
1944
-
1945
- GGML_API struct ggml_tensor * ggml_map_custom1_inplace(
1946
- struct ggml_context * ctx,
1947
- struct ggml_tensor * a,
1948
- ggml_custom1_op_t fun,
1949
- int n_tasks,
1950
- void * userdata);
1951
-
1952
- GGML_API struct ggml_tensor * ggml_map_custom2(
1953
- struct ggml_context * ctx,
1954
- struct ggml_tensor * a,
1955
- struct ggml_tensor * b,
1956
- ggml_custom2_op_t fun,
1957
- int n_tasks,
1958
- void * userdata);
1959
-
1960
- GGML_API struct ggml_tensor * ggml_map_custom2_inplace(
1961
- struct ggml_context * ctx,
1962
- struct ggml_tensor * a,
1963
- struct ggml_tensor * b,
1964
- ggml_custom2_op_t fun,
1965
- int n_tasks,
1966
- void * userdata);
1967
-
1968
- GGML_API struct ggml_tensor * ggml_map_custom3(
1969
- struct ggml_context * ctx,
1970
- struct ggml_tensor * a,
1971
- struct ggml_tensor * b,
1972
- struct ggml_tensor * c,
1973
- ggml_custom3_op_t fun,
1974
- int n_tasks,
1975
- void * userdata);
1976
-
1977
- GGML_API struct ggml_tensor * ggml_map_custom3_inplace(
1978
- struct ggml_context * ctx,
1979
- struct ggml_tensor * a,
1980
- struct ggml_tensor * b,
1981
- struct ggml_tensor * c,
1982
- ggml_custom3_op_t fun,
1983
- int n_tasks,
1984
- void * userdata);
1985
-
1986
- // loss function
1987
-
1988
- GGML_API struct ggml_tensor * ggml_cross_entropy_loss(
1989
- struct ggml_context * ctx,
1990
- struct ggml_tensor * a,
1991
- struct ggml_tensor * b);
1992
-
1993
- GGML_API struct ggml_tensor * ggml_cross_entropy_loss_back(
1994
- struct ggml_context * ctx,
1995
- struct ggml_tensor * a,
1996
- struct ggml_tensor * b,
1997
- struct ggml_tensor * c);
1998
-
1999
- //
2000
- // automatic differentiation
2001
- //
2002
-
2003
- GGML_API void ggml_set_param(
2004
- struct ggml_context * ctx,
2005
- struct ggml_tensor * tensor);
2006
-
2007
-
2008
- GGML_API void ggml_build_forward_expand (struct ggml_cgraph * cgraph, struct ggml_tensor * tensor);
2009
- GGML_API void ggml_build_backward_expand(struct ggml_context * ctx, struct ggml_cgraph * gf, struct ggml_cgraph * gb, bool keep);
2010
-
2011
- // graph allocation in a context
2012
- GGML_API struct ggml_cgraph * ggml_new_graph (struct ggml_context * ctx); // size = GGML_DEFAULT_GRAPH_SIZE, grads = false
2013
- GGML_API struct ggml_cgraph * ggml_new_graph_custom (struct ggml_context * ctx, size_t size, bool grads);
2014
- GGML_API struct ggml_cgraph * ggml_graph_dup (struct ggml_context * ctx, struct ggml_cgraph * cgraph);
2015
- GGML_API struct ggml_cgraph ggml_graph_view (struct ggml_cgraph * cgraph, int i0, int i1);
2016
- GGML_API void ggml_graph_cpy (struct ggml_cgraph * src, struct ggml_cgraph * dst);
2017
- GGML_API void ggml_graph_reset (struct ggml_cgraph * cgraph); // zero grads
2018
- GGML_API void ggml_graph_clear (struct ggml_cgraph * cgraph);
2019
-
2020
- GGML_API size_t ggml_graph_overhead(void);
2021
- GGML_API size_t ggml_graph_overhead_custom(size_t size, bool grads);
2022
-
2023
- // ggml_graph_plan() has to be called before ggml_graph_compute()
2024
- // when plan.work_size > 0, caller must allocate memory for plan.work_data
2025
- GGML_API struct ggml_cplan ggml_graph_plan (const struct ggml_cgraph * cgraph, int n_threads /*= GGML_DEFAULT_N_THREADS*/);
2026
- GGML_API enum ggml_status ggml_graph_compute ( struct ggml_cgraph * cgraph, struct ggml_cplan * cplan);
2027
- // same as ggml_graph_compute() but the work data is allocated as a part of the context
2028
- // note: the drawback of this API is that you must have ensured that the context has enough memory for the work data
2029
- GGML_API enum ggml_status ggml_graph_compute_with_ctx(struct ggml_context * ctx, struct ggml_cgraph * cgraph, int n_threads);
2030
-
2031
- GGML_API struct ggml_tensor * ggml_graph_get_tensor(struct ggml_cgraph * cgraph, const char * name);
2032
-
2033
- GGML_API void ggml_graph_export(const struct ggml_cgraph * cgraph, const char * fname);
2034
- GGML_API struct ggml_cgraph * ggml_graph_import(const char * fname, struct ggml_context ** ctx_data, struct ggml_context ** ctx_eval);
2035
-
2036
- // print info and performance information for the graph
2037
- GGML_API void ggml_graph_print(const struct ggml_cgraph * cgraph);
2038
-
2039
- // dump the graph into a file using the dot format
2040
- GGML_API void ggml_graph_dump_dot(const struct ggml_cgraph * gb, const struct ggml_cgraph * gf, const char * filename);
2041
-
2042
- // build gradient checkpointing backward graph gb for gf using provided checkpoints
2043
- // gb_tmp will contain original backward graph with rewritten backward process nodes,
2044
- // but without the second forward pass nodes.
2045
- GGML_API void ggml_build_backward_gradient_checkpointing(
2046
- struct ggml_context * ctx,
2047
- struct ggml_cgraph * gf,
2048
- struct ggml_cgraph * gb,
2049
- struct ggml_cgraph * gb_tmp,
2050
- struct ggml_tensor * * checkpoints,
2051
- int n_checkpoints);
2052
- //
2053
- // optimization
2054
- //
2055
-
2056
- // optimization methods
2057
- enum ggml_opt_type {
2058
- GGML_OPT_TYPE_ADAM,
2059
- GGML_OPT_TYPE_LBFGS,
2060
- };
2061
-
2062
- // linesearch methods
2063
- enum ggml_linesearch {
2064
- GGML_LINESEARCH_DEFAULT = 1,
2065
-
2066
- GGML_LINESEARCH_BACKTRACKING_ARMIJO = 0,
2067
- GGML_LINESEARCH_BACKTRACKING_WOLFE = 1,
2068
- GGML_LINESEARCH_BACKTRACKING_STRONG_WOLFE = 2,
2069
- };
2070
-
2071
- // optimization return values
2072
- enum ggml_opt_result {
2073
- GGML_OPT_RESULT_OK = 0,
2074
- GGML_OPT_RESULT_DID_NOT_CONVERGE,
2075
- GGML_OPT_RESULT_NO_CONTEXT,
2076
- GGML_OPT_RESULT_INVALID_WOLFE,
2077
- GGML_OPT_RESULT_FAIL,
2078
- GGML_OPT_RESULT_CANCEL,
2079
-
2080
- GGML_LINESEARCH_FAIL = -128,
2081
- GGML_LINESEARCH_MINIMUM_STEP,
2082
- GGML_LINESEARCH_MAXIMUM_STEP,
2083
- GGML_LINESEARCH_MAXIMUM_ITERATIONS,
2084
- GGML_LINESEARCH_INVALID_PARAMETERS,
2085
- };
2086
-
2087
- typedef void (*ggml_opt_callback)(void * data, int accum_step, float * sched, bool * cancel);
2088
- typedef void (*ggml_log_callback)(enum ggml_log_level level, const char * text, void * user_data);
2089
-
2090
- // optimization parameters
2091
- //
2092
- // see ggml.c (ggml_opt_default_params) for default values
2093
- //
2094
- struct ggml_opt_params {
2095
- enum ggml_opt_type type;
2096
-
2097
- size_t graph_size;
2098
-
2099
- int n_threads;
2100
-
2101
- // delta-based convergence test
2102
- //
2103
- // if past == 0 - disabled
2104
- // if past > 0:
2105
- // stop if |f(x) - f(x_past)| < delta * max(1, |f(x)|)
2106
- //
2107
- int past;
2108
- float delta;
2109
-
2110
- // maximum number of iterations without improvement
2111
- //
2112
- // if 0 - disabled
2113
- // if > 0:
2114
- // assume convergence if no cost improvement in this number of iterations
2115
- //
2116
- int max_no_improvement;
2117
-
2118
- bool print_forward_graph;
2119
- bool print_backward_graph;
2120
-
2121
- int n_gradient_accumulation;
2122
-
2123
- // ADAM parameters
2124
- struct {
2125
- int n_iter;
2126
-
2127
- float sched; // schedule multiplier (fixed, decay or warmup)
2128
- float decay; // weight decay for AdamW, use 0.0f to disable
2129
- int decay_min_ndim; // minimum number of tensor dimension to apply weight decay
2130
- float alpha; // learning rate
2131
- float beta1;
2132
- float beta2;
2133
- float eps; // epsilon for numerical stability
2134
- float eps_f; // epsilon for convergence test
2135
- float eps_g; // epsilon for convergence test
2136
- float gclip; // gradient clipping
2137
- } adam;
2138
-
2139
- // LBFGS parameters
2140
- struct {
2141
- int m; // number of corrections to approximate the inv. Hessian
2142
- int n_iter;
2143
- int max_linesearch;
2144
-
2145
- float eps; // convergence tolerance
2146
- float ftol; // line search tolerance
2147
- float wolfe;
2148
- float min_step;
2149
- float max_step;
2150
-
2151
- enum ggml_linesearch linesearch;
2152
- } lbfgs;
2153
- };
2154
-
2155
- struct ggml_opt_context {
2156
- struct ggml_context * ctx;
2157
- struct ggml_opt_params params;
2158
-
2159
- int iter;
2160
- int64_t nx; // number of parameter elements
2161
-
2162
- bool just_initialized;
2163
-
2164
- float loss_before;
2165
- float loss_after;
2166
-
2167
- struct {
2168
- struct ggml_tensor * g; // current gradient
2169
- struct ggml_tensor * m; // first moment
2170
- struct ggml_tensor * v; // second moment
2171
- struct ggml_tensor * pf; // past function values
2172
- float fx_best;
2173
- float fx_prev;
2174
- int n_no_improvement;
2175
- } adam;
2176
-
2177
- struct {
2178
- struct ggml_tensor * x; // current parameters
2179
- struct ggml_tensor * xp; // previous parameters
2180
- struct ggml_tensor * g; // current gradient
2181
- struct ggml_tensor * gp; // previous gradient
2182
- struct ggml_tensor * d; // search direction
2183
- struct ggml_tensor * pf; // past function values
2184
- struct ggml_tensor * lmal; // the L-BFGS memory alpha
2185
- struct ggml_tensor * lmys; // the L-BFGS memory ys
2186
- struct ggml_tensor * lms; // the L-BFGS memory s
2187
- struct ggml_tensor * lmy; // the L-BFGS memory y
2188
- float fx_best;
2189
- float step;
2190
- int j;
2191
- int k;
2192
- int end;
2193
- int n_no_improvement;
2194
- } lbfgs;
2195
- };
2196
-
2197
- GGML_API struct ggml_opt_params ggml_opt_default_params(enum ggml_opt_type type);
2198
-
2199
- // optimize the function defined by the tensor f
2200
- GGML_API enum ggml_opt_result ggml_opt(
2201
- struct ggml_context * ctx,
2202
- struct ggml_opt_params params,
2203
- struct ggml_tensor * f);
2204
-
2205
- // initialize optimizer context
2206
- GGML_API void ggml_opt_init(
2207
- struct ggml_context * ctx,
2208
- struct ggml_opt_context * opt,
2209
- struct ggml_opt_params params,
2210
- int64_t nx);
2211
-
2212
- // continue optimizing the function defined by the tensor f
2213
- GGML_API enum ggml_opt_result ggml_opt_resume(
2214
- struct ggml_context * ctx,
2215
- struct ggml_opt_context * opt,
2216
- struct ggml_tensor * f);
2217
-
2218
- // continue optimizing the function defined by the tensor f
2219
- GGML_API enum ggml_opt_result ggml_opt_resume_g(
2220
- struct ggml_context * ctx,
2221
- struct ggml_opt_context * opt,
2222
- struct ggml_tensor * f,
2223
- struct ggml_cgraph * gf,
2224
- struct ggml_cgraph * gb,
2225
- ggml_opt_callback callback,
2226
- void * callback_data);
2227
-
2228
- //
2229
- // tensor flags
2230
- //
2231
- GGML_API void ggml_set_input(struct ggml_tensor * tensor);
2232
- GGML_API void ggml_set_output(struct ggml_tensor * tensor);
2233
-
2234
- //
2235
- // quantization
2236
- //
2237
-
2238
- // - ggml_quantize_init can be called multiple times with the same type
2239
- // it will only initialize the quantization tables for the first call or after ggml_quantize_free
2240
- // automatically called by ggml_quantize_chunk for convenience
2241
- //
2242
- // - ggml_quantize_free will free any memory allocated by ggml_quantize_init
2243
- // call this at the end of the program to avoid memory leaks
2244
- //
2245
- // note: these are thread-safe
2246
- //
2247
- GGML_API void ggml_quantize_init(enum ggml_type type);
2248
- GGML_API void ggml_quantize_free(void);
2249
-
2250
- // some quantization type cannot be used without an importance matrix
2251
- GGML_API bool ggml_quantize_requires_imatrix(enum ggml_type type);
2252
-
2253
- // calls ggml_quantize_init internally (i.e. can allocate memory)
2254
- GGML_API size_t ggml_quantize_chunk(
2255
- enum ggml_type type,
2256
- const float * src,
2257
- void * dst,
2258
- int64_t start,
2259
- int64_t nrows,
2260
- int64_t n_per_row,
2261
- const float * imatrix);
2262
-
2263
- //
2264
- // gguf
2265
- //
2266
-
2267
- enum gguf_type {
2268
- GGUF_TYPE_UINT8 = 0,
2269
- GGUF_TYPE_INT8 = 1,
2270
- GGUF_TYPE_UINT16 = 2,
2271
- GGUF_TYPE_INT16 = 3,
2272
- GGUF_TYPE_UINT32 = 4,
2273
- GGUF_TYPE_INT32 = 5,
2274
- GGUF_TYPE_FLOAT32 = 6,
2275
- GGUF_TYPE_BOOL = 7,
2276
- GGUF_TYPE_STRING = 8,
2277
- GGUF_TYPE_ARRAY = 9,
2278
- GGUF_TYPE_UINT64 = 10,
2279
- GGUF_TYPE_INT64 = 11,
2280
- GGUF_TYPE_FLOAT64 = 12,
2281
- GGUF_TYPE_COUNT, // marks the end of the enum
2282
- };
2283
-
2284
- struct gguf_context;
2285
-
2286
- struct gguf_init_params {
2287
- bool no_alloc;
2288
-
2289
- // if not NULL, create a ggml_context and allocate the tensor data in it
2290
- struct ggml_context ** ctx;
2291
- };
2292
-
2293
- GGML_API struct gguf_context * gguf_init_empty(void);
2294
- GGML_API struct gguf_context * gguf_init_from_file(const char * fname, struct gguf_init_params params);
2295
- //GGML_API struct gguf_context * gguf_init_from_buffer(..);
2296
-
2297
- GGML_API void gguf_free(struct gguf_context * ctx);
2298
-
2299
- GGML_API const char * gguf_type_name(enum gguf_type type);
2300
-
2301
- GGML_API int gguf_get_version (const struct gguf_context * ctx);
2302
- GGML_API size_t gguf_get_alignment (const struct gguf_context * ctx);
2303
- GGML_API size_t gguf_get_data_offset(const struct gguf_context * ctx);
2304
- GGML_API void * gguf_get_data (const struct gguf_context * ctx);
2305
-
2306
- GGML_API int gguf_get_n_kv(const struct gguf_context * ctx);
2307
- GGML_API int gguf_find_key(const struct gguf_context * ctx, const char * key);
2308
- GGML_API const char * gguf_get_key (const struct gguf_context * ctx, int key_id);
2309
-
2310
- GGML_API enum gguf_type gguf_get_kv_type (const struct gguf_context * ctx, int key_id);
2311
- GGML_API enum gguf_type gguf_get_arr_type(const struct gguf_context * ctx, int key_id);
2312
-
2313
- // will abort if the wrong type is used for the key
2314
- GGML_API uint8_t gguf_get_val_u8 (const struct gguf_context * ctx, int key_id);
2315
- GGML_API int8_t gguf_get_val_i8 (const struct gguf_context * ctx, int key_id);
2316
- GGML_API uint16_t gguf_get_val_u16 (const struct gguf_context * ctx, int key_id);
2317
- GGML_API int16_t gguf_get_val_i16 (const struct gguf_context * ctx, int key_id);
2318
- GGML_API uint32_t gguf_get_val_u32 (const struct gguf_context * ctx, int key_id);
2319
- GGML_API int32_t gguf_get_val_i32 (const struct gguf_context * ctx, int key_id);
2320
- GGML_API float gguf_get_val_f32 (const struct gguf_context * ctx, int key_id);
2321
- GGML_API uint64_t gguf_get_val_u64 (const struct gguf_context * ctx, int key_id);
2322
- GGML_API int64_t gguf_get_val_i64 (const struct gguf_context * ctx, int key_id);
2323
- GGML_API double gguf_get_val_f64 (const struct gguf_context * ctx, int key_id);
2324
- GGML_API bool gguf_get_val_bool(const struct gguf_context * ctx, int key_id);
2325
- GGML_API const char * gguf_get_val_str (const struct gguf_context * ctx, int key_id);
2326
- GGML_API const void * gguf_get_val_data(const struct gguf_context * ctx, int key_id);
2327
- GGML_API int gguf_get_arr_n (const struct gguf_context * ctx, int key_id);
2328
- GGML_API const void * gguf_get_arr_data(const struct gguf_context * ctx, int key_id);
2329
- GGML_API const char * gguf_get_arr_str (const struct gguf_context * ctx, int key_id, int i);
2330
-
2331
- GGML_API int gguf_get_n_tensors (const struct gguf_context * ctx);
2332
- GGML_API int gguf_find_tensor (const struct gguf_context * ctx, const char * name);
2333
- GGML_API size_t gguf_get_tensor_offset(const struct gguf_context * ctx, int i);
2334
- GGML_API char * gguf_get_tensor_name (const struct gguf_context * ctx, int i);
2335
- GGML_API enum ggml_type gguf_get_tensor_type (const struct gguf_context * ctx, int i);
2336
-
2337
- // removes key if it exists
2338
- GGML_API void gguf_remove_key(struct gguf_context * ctx, const char * key);
2339
-
2340
- // overrides existing values or adds a new one
2341
- GGML_API void gguf_set_val_u8 (struct gguf_context * ctx, const char * key, uint8_t val);
2342
- GGML_API void gguf_set_val_i8 (struct gguf_context * ctx, const char * key, int8_t val);
2343
- GGML_API void gguf_set_val_u16 (struct gguf_context * ctx, const char * key, uint16_t val);
2344
- GGML_API void gguf_set_val_i16 (struct gguf_context * ctx, const char * key, int16_t val);
2345
- GGML_API void gguf_set_val_u32 (struct gguf_context * ctx, const char * key, uint32_t val);
2346
- GGML_API void gguf_set_val_i32 (struct gguf_context * ctx, const char * key, int32_t val);
2347
- GGML_API void gguf_set_val_f32 (struct gguf_context * ctx, const char * key, float val);
2348
- GGML_API void gguf_set_val_u64 (struct gguf_context * ctx, const char * key, uint64_t val);
2349
- GGML_API void gguf_set_val_i64 (struct gguf_context * ctx, const char * key, int64_t val);
2350
- GGML_API void gguf_set_val_f64 (struct gguf_context * ctx, const char * key, double val);
2351
- GGML_API void gguf_set_val_bool(struct gguf_context * ctx, const char * key, bool val);
2352
- GGML_API void gguf_set_val_str (struct gguf_context * ctx, const char * key, const char * val);
2353
- GGML_API void gguf_set_arr_data(struct gguf_context * ctx, const char * key, enum gguf_type type, const void * data, int n);
2354
- GGML_API void gguf_set_arr_str (struct gguf_context * ctx, const char * key, const char ** data, int n);
2355
-
2356
- // set or add KV pairs from another context
2357
- GGML_API void gguf_set_kv(struct gguf_context * ctx, struct gguf_context * src);
2358
-
2359
- // manage tensor info
2360
- GGML_API void gguf_add_tensor(struct gguf_context * ctx, const struct ggml_tensor * tensor);
2361
- GGML_API void gguf_set_tensor_type(struct gguf_context * ctx, const char * name, enum ggml_type type);
2362
- GGML_API void gguf_set_tensor_data(struct gguf_context * ctx, const char * name, const void * data, size_t size);
2363
-
2364
- // writing gguf files can be done in 2 ways:
2365
- //
2366
- // - write the entire gguf_context to a binary file in a single pass:
2367
- //
2368
- // gguf_write_to_file(ctx, fname);
2369
- //
2370
- // - first prepare a file with a placeholder for the meta data, write the tensor data, then write the meta data:
2371
- //
2372
- // FILE * f = fopen(fname, "wb");
2373
- // fseek(f, gguf_get_meta_size(ctx), SEEK_SET);
2374
- // fwrite(f, ...);
2375
- // void * data = gguf_meta_get_meta_data(ctx);
2376
- // fseek(f, 0, SEEK_SET);
2377
- // fwrite(f, data, gguf_get_meta_size(ctx));
2378
- // free(data);
2379
- // fclose(f);
2380
- //
2381
-
2382
- // write the entire context to a binary file
2383
- GGML_API void gguf_write_to_file(const struct gguf_context * ctx, const char * fname, bool only_meta);
2384
-
2385
- // get the size in bytes of the meta data (header, kv pairs, tensor info) including padding
2386
- GGML_API size_t gguf_get_meta_size(const struct gguf_context * ctx);
2387
- GGML_API void gguf_get_meta_data(const struct gguf_context * ctx, void * data);
2388
-
2389
- //
2390
- // system info
2391
- //
2392
-
2393
- GGML_API int ggml_cpu_has_avx (void);
2394
- GGML_API int ggml_cpu_has_avx_vnni (void);
2395
- GGML_API int ggml_cpu_has_avx2 (void);
2396
- GGML_API int ggml_cpu_has_avx512 (void);
2397
- GGML_API int ggml_cpu_has_avx512_vbmi(void);
2398
- GGML_API int ggml_cpu_has_avx512_vnni(void);
2399
- GGML_API int ggml_cpu_has_avx512_bf16(void);
2400
- GGML_API int ggml_cpu_has_fma (void);
2401
- GGML_API int ggml_cpu_has_neon (void);
2402
- GGML_API int ggml_cpu_has_sve (void);
2403
- GGML_API int ggml_cpu_has_arm_fma (void);
2404
- GGML_API int ggml_cpu_has_metal (void);
2405
- GGML_API int ggml_cpu_has_f16c (void);
2406
- GGML_API int ggml_cpu_has_fp16_va (void);
2407
- GGML_API int ggml_cpu_has_wasm_simd (void);
2408
- GGML_API int ggml_cpu_has_blas (void);
2409
- GGML_API int ggml_cpu_has_cuda (void);
2410
- GGML_API int ggml_cpu_has_vulkan (void);
2411
- GGML_API int ggml_cpu_has_kompute (void);
2412
- GGML_API int ggml_cpu_has_gpublas (void);
2413
- GGML_API int ggml_cpu_has_sse3 (void);
2414
- GGML_API int ggml_cpu_has_ssse3 (void);
2415
- GGML_API int ggml_cpu_has_sycl (void);
2416
- GGML_API int ggml_cpu_has_rpc (void);
2417
- GGML_API int ggml_cpu_has_vsx (void);
2418
- GGML_API int ggml_cpu_has_matmul_int8(void);
2419
-
2420
- //
2421
- // Internal types and functions exposed for tests and benchmarks
2422
- //
2423
-
2424
- #ifdef __cplusplus
2425
- // restrict not standard in C++
2426
- #define GGML_RESTRICT
2427
- #else
2428
- #define GGML_RESTRICT restrict
2429
- #endif
2430
- typedef void (*ggml_to_float_t) (const void * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
2431
- typedef void (*ggml_from_float_t)(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
2432
- typedef void (*ggml_vec_dot_t) (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT x, size_t bx,
2433
- const void * GGML_RESTRICT y, size_t by, int nrc);
2434
-
2435
- typedef struct {
2436
- const char * type_name;
2437
- int blck_size;
2438
- size_t type_size;
2439
- bool is_quantized;
2440
- ggml_to_float_t to_float;
2441
- ggml_from_float_t from_float;
2442
- ggml_from_float_t from_float_reference;
2443
- ggml_vec_dot_t vec_dot;
2444
- enum ggml_type vec_dot_type;
2445
- int64_t nrows; // number of rows to process simultaneously;
2446
- } ggml_type_traits_t;
2447
-
2448
- GGML_API ggml_type_traits_t ggml_internal_get_type_traits(enum ggml_type type);
2449
-
2450
- #ifdef __cplusplus
2451
- }
2452
- #endif