cui-llama.rn 1.0.1 → 1.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (27) hide show
  1. package/README.md +10 -1
  2. package/android/src/main/CMakeLists.txt +22 -19
  3. package/android/src/main/java/com/rnllama/LlamaContext.java +62 -20
  4. package/cpp/common.cpp +4 -11
  5. package/cpp/common.h +1 -1
  6. package/cpp/ggml-aarch64.c +2193 -2193
  7. package/cpp/ggml-aarch64.h +39 -39
  8. package/cpp/ggml-alloc.c +1042 -1041
  9. package/cpp/ggml-backend-impl.h +153 -153
  10. package/cpp/ggml-backend.c +2234 -2225
  11. package/cpp/ggml-backend.h +238 -236
  12. package/cpp/ggml-common.h +1829 -1829
  13. package/cpp/ggml-impl.h +655 -655
  14. package/cpp/ggml-metal.h +65 -65
  15. package/cpp/ggml-metal.m +3269 -3273
  16. package/cpp/ggml-quants.c +14860 -15022
  17. package/cpp/ggml-quants.h +132 -132
  18. package/cpp/ggml.c +16 -6
  19. package/cpp/ggml.h +2447 -2444
  20. package/cpp/llama.cpp +634 -531
  21. package/cpp/llama.h +30 -14
  22. package/cpp/log.h +737 -737
  23. package/cpp/rn-llama.hpp +9 -1
  24. package/cpp/sampling.cpp +460 -460
  25. package/cpp/sgemm.cpp +1027 -1027
  26. package/cpp/sgemm.h +14 -14
  27. package/package.json +1 -1
package/cpp/ggml.h CHANGED
@@ -1,2444 +1,2447 @@
1
- #pragma once
2
-
3
- //
4
- // GGML Tensor Library
5
- //
6
- // This documentation is still a work in progress.
7
- // If you wish some specific topics to be covered, feel free to drop a comment:
8
- //
9
- // https://github.com/ggerganov/whisper.cpp/issues/40
10
- //
11
- // ## Overview
12
- //
13
- // This library implements:
14
- //
15
- // - a set of tensor operations
16
- // - automatic differentiation
17
- // - basic optimization algorithms
18
- //
19
- // The aim of this library is to provide a minimalistic approach for various machine learning tasks. This includes,
20
- // but is not limited to, the following:
21
- //
22
- // - linear regression
23
- // - support vector machines
24
- // - neural networks
25
- //
26
- // The library allows the user to define a certain function using the available tensor operations. This function
27
- // definition is represented internally via a computation graph. Each tensor operation in the function definition
28
- // corresponds to a node in the graph. Having the computation graph defined, the user can choose to compute the
29
- // function's value and/or its gradient with respect to the input variables. Optionally, the function can be optimized
30
- // using one of the available optimization algorithms.
31
- //
32
- // For example, here we define the function: f(x) = a*x^2 + b
33
- //
34
- // {
35
- // struct lm_ggml_init_params params = {
36
- // .mem_size = 16*1024*1024,
37
- // .mem_buffer = NULL,
38
- // };
39
- //
40
- // // memory allocation happens here
41
- // struct lm_ggml_context * ctx = lm_ggml_init(params);
42
- //
43
- // struct lm_ggml_tensor * x = lm_ggml_new_tensor_1d(ctx, LM_GGML_TYPE_F32, 1);
44
- //
45
- // lm_ggml_set_param(ctx, x); // x is an input variable
46
- //
47
- // struct lm_ggml_tensor * a = lm_ggml_new_tensor_1d(ctx, LM_GGML_TYPE_F32, 1);
48
- // struct lm_ggml_tensor * b = lm_ggml_new_tensor_1d(ctx, LM_GGML_TYPE_F32, 1);
49
- // struct lm_ggml_tensor * x2 = lm_ggml_mul(ctx, x, x);
50
- // struct lm_ggml_tensor * f = lm_ggml_add(ctx, lm_ggml_mul(ctx, a, x2), b);
51
- //
52
- // ...
53
- // }
54
- //
55
- // Notice that the function definition above does not involve any actual computation. The computation is performed only
56
- // when the user explicitly requests it. For example, to compute the function's value at x = 2.0:
57
- //
58
- // {
59
- // ...
60
- //
61
- // struct lm_ggml_cgraph * gf = lm_ggml_new_graph(ctx);
62
- // lm_ggml_build_forward_expand(gf, f);
63
- //
64
- // // set the input variable and parameter values
65
- // lm_ggml_set_f32(x, 2.0f);
66
- // lm_ggml_set_f32(a, 3.0f);
67
- // lm_ggml_set_f32(b, 4.0f);
68
- //
69
- // lm_ggml_graph_compute_with_ctx(ctx, &gf, n_threads);
70
- //
71
- // printf("f = %f\n", lm_ggml_get_f32_1d(f, 0));
72
- //
73
- // ...
74
- // }
75
- //
76
- // The actual computation is performed in the lm_ggml_graph_compute() function.
77
- //
78
- // The lm_ggml_new_tensor_...() functions create new tensors. They are allocated in the memory buffer provided to the
79
- // lm_ggml_init() function. You have to be careful not to exceed the memory buffer size. Therefore, you have to know
80
- // in advance how much memory you need for your computation. Alternatively, you can allocate a large enough memory
81
- // and after defining the computation graph, call the lm_ggml_used_mem() function to find out how much memory was
82
- // actually needed.
83
- //
84
- // The lm_ggml_set_param() function marks a tensor as an input variable. This is used by the automatic
85
- // differentiation and optimization algorithms.
86
- //
87
- // The described approach allows to define the function graph once and then compute its forward or backward graphs
88
- // multiple times. All computations will use the same memory buffer allocated in the lm_ggml_init() function. This way
89
- // the user can avoid the memory allocation overhead at runtime.
90
- //
91
- // The library supports multi-dimensional tensors - up to 4 dimensions. The FP16 and FP32 data types are first class
92
- // citizens, but in theory the library can be extended to support FP8 and integer data types.
93
- //
94
- // Each tensor operation produces a new tensor. Initially the library was envisioned to support only the use of unary
95
- // and binary operations. Most of the available operations fall into one of these two categories. With time, it became
96
- // clear that the library needs to support more complex operations. The way to support these operations is not clear
97
- // yet, but a few examples are demonstrated in the following operations:
98
- //
99
- // - lm_ggml_permute()
100
- // - lm_ggml_conv_1d_1s()
101
- // - lm_ggml_conv_1d_2s()
102
- //
103
- // For each tensor operator, the library implements a forward and backward computation function. The forward function
104
- // computes the output tensor value given the input tensor values. The backward function computes the adjoint of the
105
- // input tensors given the adjoint of the output tensor. For a detailed explanation of what this means, take a
106
- // calculus class, or watch the following video:
107
- //
108
- // What is Automatic Differentiation?
109
- // https://www.youtube.com/watch?v=wG_nF1awSSY
110
- //
111
- //
112
- // ## Tensor data (struct lm_ggml_tensor)
113
- //
114
- // The tensors are stored in memory via the lm_ggml_tensor struct. The structure provides information about the size of
115
- // the tensor, the data type, and the memory buffer where the tensor data is stored. Additionally, it contains
116
- // pointers to the "source" tensors - i.e. the tensors that were used to compute the current tensor. For example:
117
- //
118
- // {
119
- // struct lm_ggml_tensor * c = lm_ggml_add(ctx, a, b);
120
- //
121
- // assert(c->src[0] == a);
122
- // assert(c->src[1] == b);
123
- // }
124
- //
125
- // The multi-dimensional tensors are stored in row-major order. The lm_ggml_tensor struct contains fields for the
126
- // number of elements in each dimension ("ne") as well as the number of bytes ("nb", a.k.a. stride). This allows
127
- // to store tensors that are not contiguous in memory, which is useful for operations such as transposition and
128
- // permutation. All tensor operations have to take the stride into account and not assume that the tensor is
129
- // contiguous in memory.
130
- //
131
- // The data of the tensor is accessed via the "data" pointer. For example:
132
- //
133
- // {
134
- // const int nx = 2;
135
- // const int ny = 3;
136
- //
137
- // struct lm_ggml_tensor * a = lm_ggml_new_tensor_2d(ctx, LM_GGML_TYPE_F32, nx, ny);
138
- //
139
- // for (int y = 0; y < ny; y++) {
140
- // for (int x = 0; x < nx; x++) {
141
- // *(float *) ((char *) a->data + y*a->nb[1] + x*a->nb[0]) = x + y;
142
- // }
143
- // }
144
- //
145
- // ...
146
- // }
147
- //
148
- // Alternatively, there are helper functions, such as lm_ggml_get_f32_1d() and lm_ggml_set_f32_1d() that can be used.
149
- //
150
- // ## The matrix multiplication operator (lm_ggml_mul_mat)
151
- //
152
- // TODO
153
- //
154
- //
155
- // ## Multi-threading
156
- //
157
- // TODO
158
- //
159
- //
160
- // ## Overview of ggml.c
161
- //
162
- // TODO
163
- //
164
- //
165
- // ## SIMD optimizations
166
- //
167
- // TODO
168
- //
169
- //
170
- // ## Debugging ggml
171
- //
172
- // TODO
173
- //
174
- //
175
-
176
- #ifdef LM_GGML_SHARED
177
- # if defined(_WIN32) && !defined(__MINGW32__)
178
- # ifdef LM_GGML_BUILD
179
- # define LM_GGML_API __declspec(dllexport)
180
- # else
181
- # define LM_GGML_API __declspec(dllimport)
182
- # endif
183
- # else
184
- # define LM_GGML_API __attribute__ ((visibility ("default")))
185
- # endif
186
- #else
187
- # define LM_GGML_API
188
- #endif
189
-
190
- #ifdef LM_GGML_MULTIPLATFORM
191
- # if defined(_WIN32)
192
- # define LM_GGML_CALL
193
- # else
194
- # define LM_GGML_CALL __attribute__((__ms_abi__))
195
- # endif
196
- #else
197
- # define LM_GGML_CALL
198
- #endif
199
-
200
- // TODO: support for clang
201
- #ifdef __GNUC__
202
- # define LM_GGML_DEPRECATED(func, hint) func __attribute__((deprecated(hint)))
203
- #elif defined(_MSC_VER)
204
- # define LM_GGML_DEPRECATED(func, hint) __declspec(deprecated(hint)) func
205
- #else
206
- # define LM_GGML_DEPRECATED(func, hint) func
207
- #endif
208
-
209
- #ifndef __GNUC__
210
- # define LM_GGML_ATTRIBUTE_FORMAT(...)
211
- #elif defined(__MINGW32__)
212
- # define LM_GGML_ATTRIBUTE_FORMAT(...) __attribute__((format(gnu_printf, __VA_ARGS__)))
213
- #else
214
- # define LM_GGML_ATTRIBUTE_FORMAT(...) __attribute__((format(printf, __VA_ARGS__)))
215
- #endif
216
-
217
- #include <stdbool.h>
218
- #include <stddef.h>
219
- #include <stdint.h>
220
- #include <stdio.h>
221
-
222
- #define LM_GGML_FILE_MAGIC 0x67676d6c // "ggml"
223
- #define LM_GGML_FILE_VERSION 1
224
-
225
- #define LM_GGML_QNT_VERSION 2 // bump this on quantization format changes
226
- #define LM_GGML_QNT_VERSION_FACTOR 1000 // do not change this
227
-
228
- #define LM_GGML_MAX_DIMS 4
229
- #define LM_GGML_MAX_PARAMS 2048
230
- #define LM_GGML_MAX_CONTEXTS 64
231
- #define LM_GGML_MAX_SRC 10
232
- #ifndef LM_GGML_MAX_NAME
233
- #define LM_GGML_MAX_NAME 64
234
- #endif
235
- #define LM_GGML_MAX_OP_PARAMS 64
236
- #define LM_GGML_DEFAULT_N_THREADS 4
237
- #define LM_GGML_DEFAULT_GRAPH_SIZE 2048
238
- #if UINTPTR_MAX == 0xFFFFFFFF
239
- #define LM_GGML_MEM_ALIGN 4
240
- #else
241
- #define LM_GGML_MEM_ALIGN 16
242
- #endif
243
-
244
- #define LM_GGML_EXIT_SUCCESS 0
245
- #define LM_GGML_EXIT_ABORTED 1
246
-
247
- #define LM_GGUF_MAGIC "GGUF"
248
-
249
- #define LM_GGUF_VERSION 3
250
-
251
- #define LM_GGUF_DEFAULT_ALIGNMENT 32
252
-
253
- #define LM_GGML_UNUSED(x) (void)(x)
254
-
255
- #define LM_GGML_PAD(x, n) (((x) + (n) - 1) & ~((n) - 1))
256
-
257
- #define LM_GGML_ASSERT(x) \
258
- do { \
259
- if (!(x)) { \
260
- fflush(stdout); \
261
- fprintf(stderr, "LM_GGML_ASSERT: %s:%d: %s\n", __FILE__, __LINE__, #x); \
262
- lm_ggml_print_backtrace(); \
263
- abort(); \
264
- } \
265
- } while (0)
266
-
267
- #ifndef NDEBUG
268
- #define LM_GGML_UNREACHABLE() LM_GGML_ASSERT(!"statement should not be reached")
269
- #elif defined(__GNUC__)
270
- #define LM_GGML_UNREACHABLE() __builtin_unreachable()
271
- #elif defined(_MSC_VER)
272
- #define LM_GGML_UNREACHABLE() __assume(0)
273
- #else
274
- #define LM_GGML_UNREACHABLE() ((void) 0)
275
- #endif
276
-
277
- // used to copy the number of elements and stride in bytes of tensors into local variables.
278
- // main purpose is to reduce code duplication and improve readability.
279
- //
280
- // example:
281
- //
282
- // LM_GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne);
283
- // LM_GGML_TENSOR_LOCALS(size_t, nb1, src1, nb);
284
- //
285
- #define LM_GGML_TENSOR_LOCALS_1(type, prefix, pointer, array) \
286
- const type prefix##0 = (pointer)->array[0]; \
287
- LM_GGML_UNUSED(prefix##0);
288
- #define LM_GGML_TENSOR_LOCALS_2(type, prefix, pointer, array) \
289
- LM_GGML_TENSOR_LOCALS_1 (type, prefix, pointer, array) \
290
- const type prefix##1 = (pointer)->array[1]; \
291
- LM_GGML_UNUSED(prefix##1);
292
- #define LM_GGML_TENSOR_LOCALS_3(type, prefix, pointer, array) \
293
- LM_GGML_TENSOR_LOCALS_2 (type, prefix, pointer, array) \
294
- const type prefix##2 = (pointer)->array[2]; \
295
- LM_GGML_UNUSED(prefix##2);
296
- #define LM_GGML_TENSOR_LOCALS(type, prefix, pointer, array) \
297
- LM_GGML_TENSOR_LOCALS_3 (type, prefix, pointer, array) \
298
- const type prefix##3 = (pointer)->array[3]; \
299
- LM_GGML_UNUSED(prefix##3);
300
-
301
- #define LM_GGML_TENSOR_UNARY_OP_LOCALS \
302
- LM_GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) \
303
- LM_GGML_TENSOR_LOCALS(size_t, nb0, src0, nb) \
304
- LM_GGML_TENSOR_LOCALS(int64_t, ne, dst, ne) \
305
- LM_GGML_TENSOR_LOCALS(size_t, nb, dst, nb)
306
-
307
- #define LM_GGML_TENSOR_BINARY_OP_LOCALS \
308
- LM_GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) \
309
- LM_GGML_TENSOR_LOCALS(size_t, nb0, src0, nb) \
310
- LM_GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne) \
311
- LM_GGML_TENSOR_LOCALS(size_t, nb1, src1, nb) \
312
- LM_GGML_TENSOR_LOCALS(int64_t, ne, dst, ne) \
313
- LM_GGML_TENSOR_LOCALS(size_t, nb, dst, nb)
314
-
315
- #define LM_GGML_TENSOR_BINARY_OP_LOCALS01 \
316
- LM_GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) \
317
- LM_GGML_TENSOR_LOCALS(size_t, nb0, src0, nb) \
318
- LM_GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne) \
319
- LM_GGML_TENSOR_LOCALS(size_t, nb1, src1, nb)
320
-
321
- #ifdef __cplusplus
322
- extern "C" {
323
- #endif
324
-
325
- enum lm_ggml_status {
326
- LM_GGML_STATUS_ALLOC_FAILED = -2,
327
- LM_GGML_STATUS_FAILED = -1,
328
- LM_GGML_STATUS_SUCCESS = 0,
329
- LM_GGML_STATUS_ABORTED = 1,
330
- };
331
-
332
- // get lm_ggml_status name string
333
- LM_GGML_API LM_GGML_CALL const char * lm_ggml_status_to_string(enum lm_ggml_status status);
334
-
335
- // ieee 754-2008 half-precision float16
336
- // todo: make this not an integral type
337
- typedef uint16_t lm_ggml_fp16_t;
338
- LM_GGML_API float lm_ggml_fp16_to_fp32(lm_ggml_fp16_t);
339
- LM_GGML_API lm_ggml_fp16_t lm_ggml_fp32_to_fp16(float);
340
- LM_GGML_API void lm_ggml_fp16_to_fp32_row(const lm_ggml_fp16_t *, float *, int64_t);
341
- LM_GGML_API void lm_ggml_fp32_to_fp16_row(const float *, lm_ggml_fp16_t *, int64_t);
342
-
343
- // google brain half-precision bfloat16
344
- typedef struct { uint16_t bits; } lm_ggml_bf16_t;
345
- LM_GGML_API lm_ggml_bf16_t lm_ggml_fp32_to_bf16(float);
346
- LM_GGML_API float lm_ggml_bf16_to_fp32(lm_ggml_bf16_t); // consider just doing << 16
347
- LM_GGML_API void lm_ggml_bf16_to_fp32_row(const lm_ggml_bf16_t *, float *, int64_t);
348
- LM_GGML_API void lm_ggml_fp32_to_bf16_row(const float *, lm_ggml_bf16_t *, int64_t);
349
-
350
- struct lm_ggml_object;
351
- struct lm_ggml_context;
352
-
353
- // NOTE: always add types at the end of the enum to keep backward compatibility
354
- enum lm_ggml_type {
355
- LM_GGML_TYPE_F32 = 0,
356
- LM_GGML_TYPE_F16 = 1,
357
- LM_GGML_TYPE_Q4_0 = 2,
358
- LM_GGML_TYPE_Q4_1 = 3,
359
- // LM_GGML_TYPE_Q4_2 = 4, support has been removed
360
- // LM_GGML_TYPE_Q4_3 = 5, support has been removed
361
- LM_GGML_TYPE_Q5_0 = 6,
362
- LM_GGML_TYPE_Q5_1 = 7,
363
- LM_GGML_TYPE_Q8_0 = 8,
364
- LM_GGML_TYPE_Q8_1 = 9,
365
- LM_GGML_TYPE_Q2_K = 10,
366
- LM_GGML_TYPE_Q3_K = 11,
367
- LM_GGML_TYPE_Q4_K = 12,
368
- LM_GGML_TYPE_Q5_K = 13,
369
- LM_GGML_TYPE_Q6_K = 14,
370
- LM_GGML_TYPE_Q8_K = 15,
371
- LM_GGML_TYPE_IQ2_XXS = 16,
372
- LM_GGML_TYPE_IQ2_XS = 17,
373
- LM_GGML_TYPE_IQ3_XXS = 18,
374
- LM_GGML_TYPE_IQ1_S = 19,
375
- LM_GGML_TYPE_IQ4_NL = 20,
376
- LM_GGML_TYPE_IQ3_S = 21,
377
- LM_GGML_TYPE_IQ2_S = 22,
378
- LM_GGML_TYPE_IQ4_XS = 23,
379
- LM_GGML_TYPE_I8 = 24,
380
- LM_GGML_TYPE_I16 = 25,
381
- LM_GGML_TYPE_I32 = 26,
382
- LM_GGML_TYPE_I64 = 27,
383
- LM_GGML_TYPE_F64 = 28,
384
- LM_GGML_TYPE_IQ1_M = 29,
385
- LM_GGML_TYPE_BF16 = 30,
386
- LM_GGML_TYPE_Q4_0_4_4 = 31,
387
- LM_GGML_TYPE_Q4_0_4_8 = 32,
388
- LM_GGML_TYPE_Q4_0_8_8 = 33,
389
- LM_GGML_TYPE_COUNT,
390
- };
391
-
392
- // precision
393
- enum lm_ggml_prec {
394
- LM_GGML_PREC_DEFAULT,
395
- LM_GGML_PREC_F32,
396
- };
397
-
398
- enum lm_ggml_backend_type {
399
- LM_GGML_BACKEND_TYPE_CPU = 0,
400
- LM_GGML_BACKEND_TYPE_GPU = 10,
401
- LM_GGML_BACKEND_TYPE_GPU_SPLIT = 20,
402
- };
403
-
404
- // model file types
405
- enum lm_ggml_ftype {
406
- LM_GGML_FTYPE_UNKNOWN = -1,
407
- LM_GGML_FTYPE_ALL_F32 = 0,
408
- LM_GGML_FTYPE_MOSTLY_F16 = 1, // except 1d tensors
409
- LM_GGML_FTYPE_MOSTLY_Q4_0 = 2, // except 1d tensors
410
- LM_GGML_FTYPE_MOSTLY_Q4_1 = 3, // except 1d tensors
411
- LM_GGML_FTYPE_MOSTLY_Q4_1_SOME_F16 = 4, // tok_embeddings.weight and output.weight are F16
412
- LM_GGML_FTYPE_MOSTLY_Q8_0 = 7, // except 1d tensors
413
- LM_GGML_FTYPE_MOSTLY_Q5_0 = 8, // except 1d tensors
414
- LM_GGML_FTYPE_MOSTLY_Q5_1 = 9, // except 1d tensors
415
- LM_GGML_FTYPE_MOSTLY_Q2_K = 10, // except 1d tensors
416
- LM_GGML_FTYPE_MOSTLY_Q3_K = 11, // except 1d tensors
417
- LM_GGML_FTYPE_MOSTLY_Q4_K = 12, // except 1d tensors
418
- LM_GGML_FTYPE_MOSTLY_Q5_K = 13, // except 1d tensors
419
- LM_GGML_FTYPE_MOSTLY_Q6_K = 14, // except 1d tensors
420
- LM_GGML_FTYPE_MOSTLY_IQ2_XXS = 15, // except 1d tensors
421
- LM_GGML_FTYPE_MOSTLY_IQ2_XS = 16, // except 1d tensors
422
- LM_GGML_FTYPE_MOSTLY_IQ3_XXS = 17, // except 1d tensors
423
- LM_GGML_FTYPE_MOSTLY_IQ1_S = 18, // except 1d tensors
424
- LM_GGML_FTYPE_MOSTLY_IQ4_NL = 19, // except 1d tensors
425
- LM_GGML_FTYPE_MOSTLY_IQ3_S = 20, // except 1d tensors
426
- LM_GGML_FTYPE_MOSTLY_IQ2_S = 21, // except 1d tensors
427
- LM_GGML_FTYPE_MOSTLY_IQ4_XS = 22, // except 1d tensors
428
- LM_GGML_FTYPE_MOSTLY_IQ1_M = 23, // except 1d tensors
429
- LM_GGML_FTYPE_MOSTLY_BF16 = 24, // except 1d tensors
430
- LM_GGML_FTYPE_MOSTLY_Q4_0_4_4 = 25, // except 1d tensors
431
- LM_GGML_FTYPE_MOSTLY_Q4_0_4_8 = 26, // except 1d tensors
432
- LM_GGML_FTYPE_MOSTLY_Q4_0_8_8 = 27, // except 1d tensors
433
- };
434
-
435
- // available tensor operations:
436
- enum lm_ggml_op {
437
- LM_GGML_OP_NONE = 0,
438
-
439
- LM_GGML_OP_DUP,
440
- LM_GGML_OP_ADD,
441
- LM_GGML_OP_ADD1,
442
- LM_GGML_OP_ACC,
443
- LM_GGML_OP_SUB,
444
- LM_GGML_OP_MUL,
445
- LM_GGML_OP_DIV,
446
- LM_GGML_OP_SQR,
447
- LM_GGML_OP_SQRT,
448
- LM_GGML_OP_LOG,
449
- LM_GGML_OP_SUM,
450
- LM_GGML_OP_SUM_ROWS,
451
- LM_GGML_OP_MEAN,
452
- LM_GGML_OP_ARGMAX,
453
- LM_GGML_OP_REPEAT,
454
- LM_GGML_OP_REPEAT_BACK,
455
- LM_GGML_OP_CONCAT,
456
- LM_GGML_OP_SILU_BACK,
457
- LM_GGML_OP_NORM, // normalize
458
- LM_GGML_OP_RMS_NORM,
459
- LM_GGML_OP_RMS_NORM_BACK,
460
- LM_GGML_OP_GROUP_NORM,
461
-
462
- LM_GGML_OP_MUL_MAT,
463
- LM_GGML_OP_MUL_MAT_ID,
464
- LM_GGML_OP_OUT_PROD,
465
-
466
- LM_GGML_OP_SCALE,
467
- LM_GGML_OP_SET,
468
- LM_GGML_OP_CPY,
469
- LM_GGML_OP_CONT,
470
- LM_GGML_OP_RESHAPE,
471
- LM_GGML_OP_VIEW,
472
- LM_GGML_OP_PERMUTE,
473
- LM_GGML_OP_TRANSPOSE,
474
- LM_GGML_OP_GET_ROWS,
475
- LM_GGML_OP_GET_ROWS_BACK,
476
- LM_GGML_OP_DIAG,
477
- LM_GGML_OP_DIAG_MASK_INF,
478
- LM_GGML_OP_DIAG_MASK_ZERO,
479
- LM_GGML_OP_SOFT_MAX,
480
- LM_GGML_OP_SOFT_MAX_BACK,
481
- LM_GGML_OP_ROPE,
482
- LM_GGML_OP_ROPE_BACK,
483
- LM_GGML_OP_CLAMP,
484
- LM_GGML_OP_CONV_TRANSPOSE_1D,
485
- LM_GGML_OP_IM2COL,
486
- LM_GGML_OP_CONV_TRANSPOSE_2D,
487
- LM_GGML_OP_POOL_1D,
488
- LM_GGML_OP_POOL_2D,
489
- LM_GGML_OP_UPSCALE, // nearest interpolate
490
- LM_GGML_OP_PAD,
491
- LM_GGML_OP_ARANGE,
492
- LM_GGML_OP_TIMESTEP_EMBEDDING,
493
- LM_GGML_OP_ARGSORT,
494
- LM_GGML_OP_LEAKY_RELU,
495
-
496
- LM_GGML_OP_FLASH_ATTN_EXT,
497
- LM_GGML_OP_FLASH_ATTN_BACK,
498
- LM_GGML_OP_SSM_CONV,
499
- LM_GGML_OP_SSM_SCAN,
500
- LM_GGML_OP_WIN_PART,
501
- LM_GGML_OP_WIN_UNPART,
502
- LM_GGML_OP_GET_REL_POS,
503
- LM_GGML_OP_ADD_REL_POS,
504
-
505
- LM_GGML_OP_UNARY,
506
-
507
- LM_GGML_OP_MAP_UNARY,
508
- LM_GGML_OP_MAP_BINARY,
509
-
510
- LM_GGML_OP_MAP_CUSTOM1_F32,
511
- LM_GGML_OP_MAP_CUSTOM2_F32,
512
- LM_GGML_OP_MAP_CUSTOM3_F32,
513
-
514
- LM_GGML_OP_MAP_CUSTOM1,
515
- LM_GGML_OP_MAP_CUSTOM2,
516
- LM_GGML_OP_MAP_CUSTOM3,
517
-
518
- LM_GGML_OP_CROSS_ENTROPY_LOSS,
519
- LM_GGML_OP_CROSS_ENTROPY_LOSS_BACK,
520
-
521
- LM_GGML_OP_COUNT,
522
- };
523
-
524
- enum lm_ggml_unary_op {
525
- LM_GGML_UNARY_OP_ABS,
526
- LM_GGML_UNARY_OP_SGN,
527
- LM_GGML_UNARY_OP_NEG,
528
- LM_GGML_UNARY_OP_STEP,
529
- LM_GGML_UNARY_OP_TANH,
530
- LM_GGML_UNARY_OP_ELU,
531
- LM_GGML_UNARY_OP_RELU,
532
- LM_GGML_UNARY_OP_SIGMOID,
533
- LM_GGML_UNARY_OP_GELU,
534
- LM_GGML_UNARY_OP_GELU_QUICK,
535
- LM_GGML_UNARY_OP_SILU,
536
- LM_GGML_UNARY_OP_HARDSWISH,
537
- LM_GGML_UNARY_OP_HARDSIGMOID,
538
-
539
- LM_GGML_UNARY_OP_COUNT,
540
- };
541
-
542
- enum lm_ggml_object_type {
543
- LM_GGML_OBJECT_TYPE_TENSOR,
544
- LM_GGML_OBJECT_TYPE_GRAPH,
545
- LM_GGML_OBJECT_TYPE_WORK_BUFFER
546
- };
547
-
548
- enum lm_ggml_log_level {
549
- LM_GGML_LOG_LEVEL_ERROR = 2,
550
- LM_GGML_LOG_LEVEL_WARN = 3,
551
- LM_GGML_LOG_LEVEL_INFO = 4,
552
- LM_GGML_LOG_LEVEL_DEBUG = 5
553
- };
554
-
555
- enum lm_ggml_tensor_flag {
556
- LM_GGML_TENSOR_FLAG_INPUT = 1,
557
- LM_GGML_TENSOR_FLAG_OUTPUT = 2,
558
- LM_GGML_TENSOR_FLAG_PARAM = 4,
559
- };
560
-
561
- // ggml object
562
- struct lm_ggml_object {
563
- size_t offs;
564
- size_t size;
565
-
566
- struct lm_ggml_object * next;
567
-
568
- enum lm_ggml_object_type type;
569
-
570
- char padding[4];
571
- };
572
-
573
- static const size_t LM_GGML_OBJECT_SIZE = sizeof(struct lm_ggml_object);
574
-
575
- // n-dimensional tensor
576
- struct lm_ggml_tensor {
577
- enum lm_ggml_type type;
578
-
579
- LM_GGML_DEPRECATED(enum lm_ggml_backend_type backend, "use the buffer type to find the storage location of the tensor");
580
-
581
- struct lm_ggml_backend_buffer * buffer;
582
-
583
- int64_t ne[LM_GGML_MAX_DIMS]; // number of elements
584
- size_t nb[LM_GGML_MAX_DIMS]; // stride in bytes:
585
- // nb[0] = lm_ggml_type_size(type)
586
- // nb[1] = nb[0] * (ne[0] / lm_ggml_blck_size(type)) + padding
587
- // nb[i] = nb[i-1] * ne[i-1]
588
-
589
- // compute data
590
- enum lm_ggml_op op;
591
-
592
- // op params - allocated as int32_t for alignment
593
- int32_t op_params[LM_GGML_MAX_OP_PARAMS / sizeof(int32_t)];
594
-
595
- int32_t flags;
596
-
597
- struct lm_ggml_tensor * grad;
598
- struct lm_ggml_tensor * src[LM_GGML_MAX_SRC];
599
-
600
- // source tensor and offset for views
601
- struct lm_ggml_tensor * view_src;
602
- size_t view_offs;
603
-
604
- void * data;
605
-
606
- char name[LM_GGML_MAX_NAME];
607
-
608
- void * extra; // extra things e.g. for ggml-cuda.cu
609
-
610
- // char padding[4];
611
- };
612
-
613
- static const size_t LM_GGML_TENSOR_SIZE = sizeof(struct lm_ggml_tensor);
614
-
615
- // Abort callback
616
- // If not NULL, called before ggml computation
617
- // If it returns true, the computation is aborted
618
- typedef bool (*lm_ggml_abort_callback)(void * data);
619
-
620
- // the compute plan that needs to be prepared for lm_ggml_graph_compute()
621
- // since https://github.com/ggerganov/ggml/issues/287
622
- struct lm_ggml_cplan {
623
- size_t work_size; // size of work buffer, calculated by `lm_ggml_graph_plan()`
624
- uint8_t * work_data; // work buffer, to be allocated by caller before calling to `lm_ggml_graph_compute()`
625
-
626
- int n_threads;
627
-
628
- // abort lm_ggml_graph_compute when true
629
- lm_ggml_abort_callback abort_callback;
630
- void * abort_callback_data;
631
- };
632
-
633
- enum lm_ggml_cgraph_eval_order {
634
- LM_GGML_CGRAPH_EVAL_ORDER_LEFT_TO_RIGHT = 0,
635
- LM_GGML_CGRAPH_EVAL_ORDER_RIGHT_TO_LEFT,
636
- LM_GGML_CGRAPH_EVAL_ORDER_COUNT
637
- };
638
-
639
- struct lm_ggml_hash_set {
640
- size_t size;
641
- struct lm_ggml_tensor ** keys;
642
- };
643
-
644
- // computation graph
645
- struct lm_ggml_cgraph {
646
- int size;
647
- int n_nodes;
648
- int n_leafs;
649
-
650
- struct lm_ggml_tensor ** nodes;
651
- struct lm_ggml_tensor ** grads;
652
- struct lm_ggml_tensor ** leafs;
653
-
654
- struct lm_ggml_hash_set visited_hash_table;
655
-
656
- enum lm_ggml_cgraph_eval_order order;
657
- };
658
-
659
- // scratch buffer
660
- struct lm_ggml_scratch {
661
- size_t offs;
662
- size_t size;
663
- void * data;
664
- };
665
-
666
- struct lm_ggml_init_params {
667
- // memory pool
668
- size_t mem_size; // bytes
669
- void * mem_buffer; // if NULL, memory will be allocated internally
670
- bool no_alloc; // don't allocate memory for the tensor data
671
- };
672
-
673
- // numa strategies
674
- enum lm_ggml_numa_strategy {
675
- LM_GGML_NUMA_STRATEGY_DISABLED = 0,
676
- LM_GGML_NUMA_STRATEGY_DISTRIBUTE = 1,
677
- LM_GGML_NUMA_STRATEGY_ISOLATE = 2,
678
- LM_GGML_NUMA_STRATEGY_NUMACTL = 3,
679
- LM_GGML_NUMA_STRATEGY_MIRROR = 4,
680
- LM_GGML_NUMA_STRATEGY_COUNT
681
- };
682
-
683
- //
684
- // GUID
685
- //
686
-
687
- // GUID types
688
- typedef uint8_t lm_ggml_guid[16];
689
- typedef lm_ggml_guid * lm_ggml_guid_t;
690
-
691
- LM_GGML_API bool lm_ggml_guid_matches(lm_ggml_guid_t guid_a, lm_ggml_guid_t guid_b);
692
-
693
- // misc
694
-
695
- LM_GGML_API void lm_ggml_time_init(void); // call this once at the beginning of the program
696
- LM_GGML_API int64_t lm_ggml_time_ms(void);
697
- LM_GGML_API int64_t lm_ggml_time_us(void);
698
- LM_GGML_API int64_t lm_ggml_cycles(void);
699
- LM_GGML_API int64_t lm_ggml_cycles_per_ms(void);
700
-
701
- LM_GGML_API void lm_ggml_print_backtrace(void);
702
-
703
- // accepts a UTF-8 path, even on Windows
704
- LM_GGML_API FILE * lm_ggml_fopen(const char * fname, const char * mode);
705
-
706
- LM_GGML_API void lm_ggml_numa_init(enum lm_ggml_numa_strategy numa); // call once for better performance on NUMA systems
707
- LM_GGML_API bool lm_ggml_is_numa(void); // true if init detected that system has >1 NUMA node
708
-
709
- LM_GGML_API void lm_ggml_print_object (const struct lm_ggml_object * obj);
710
- LM_GGML_API void lm_ggml_print_objects(const struct lm_ggml_context * ctx);
711
-
712
- LM_GGML_API LM_GGML_CALL int64_t lm_ggml_nelements (const struct lm_ggml_tensor * tensor);
713
- LM_GGML_API LM_GGML_CALL int64_t lm_ggml_nrows (const struct lm_ggml_tensor * tensor);
714
- LM_GGML_API LM_GGML_CALL size_t lm_ggml_nbytes (const struct lm_ggml_tensor * tensor);
715
- LM_GGML_API size_t lm_ggml_nbytes_pad (const struct lm_ggml_tensor * tensor); // same as lm_ggml_nbytes() but padded to LM_GGML_MEM_ALIGN
716
-
717
- LM_GGML_API LM_GGML_CALL int64_t lm_ggml_blck_size(enum lm_ggml_type type);
718
- LM_GGML_API LM_GGML_CALL size_t lm_ggml_type_size(enum lm_ggml_type type); // size in bytes for all elements in a block
719
- LM_GGML_API LM_GGML_CALL size_t lm_ggml_row_size (enum lm_ggml_type type, int64_t ne); // size in bytes for all elements in a row
720
-
721
- LM_GGML_DEPRECATED(
722
- LM_GGML_API double lm_ggml_type_sizef(enum lm_ggml_type type), // lm_ggml_type_size()/lm_ggml_blck_size() as float
723
- "use lm_ggml_row_size() instead");
724
-
725
- LM_GGML_API LM_GGML_CALL const char * lm_ggml_type_name(enum lm_ggml_type type);
726
- LM_GGML_API LM_GGML_CALL const char * lm_ggml_op_name (enum lm_ggml_op op);
727
- LM_GGML_API const char * lm_ggml_op_symbol(enum lm_ggml_op op);
728
-
729
- LM_GGML_API const char * lm_ggml_unary_op_name(enum lm_ggml_unary_op op);
730
- LM_GGML_API LM_GGML_CALL const char * lm_ggml_op_desc(const struct lm_ggml_tensor * t); // unary or op name
731
-
732
- LM_GGML_API LM_GGML_CALL size_t lm_ggml_element_size(const struct lm_ggml_tensor * tensor);
733
-
734
- LM_GGML_API LM_GGML_CALL bool lm_ggml_is_quantized(enum lm_ggml_type type);
735
-
736
- // TODO: temporary until model loading of ggml examples is refactored
737
- LM_GGML_API enum lm_ggml_type lm_ggml_ftype_to_lm_ggml_type(enum lm_ggml_ftype ftype);
738
-
739
- LM_GGML_API LM_GGML_CALL bool lm_ggml_is_transposed(const struct lm_ggml_tensor * tensor);
740
- LM_GGML_API LM_GGML_CALL bool lm_ggml_is_permuted (const struct lm_ggml_tensor * tensor);
741
- LM_GGML_API LM_GGML_CALL bool lm_ggml_is_empty (const struct lm_ggml_tensor * tensor);
742
- LM_GGML_API bool lm_ggml_is_scalar (const struct lm_ggml_tensor * tensor);
743
- LM_GGML_API bool lm_ggml_is_vector (const struct lm_ggml_tensor * tensor);
744
- LM_GGML_API bool lm_ggml_is_matrix (const struct lm_ggml_tensor * tensor);
745
- LM_GGML_API bool lm_ggml_is_3d (const struct lm_ggml_tensor * tensor);
746
- LM_GGML_API int lm_ggml_n_dims (const struct lm_ggml_tensor * tensor); // returns 1 for scalars
747
-
748
- LM_GGML_API LM_GGML_CALL bool lm_ggml_is_contiguous (const struct lm_ggml_tensor * tensor);
749
- LM_GGML_API LM_GGML_CALL bool lm_ggml_is_contiguous_0(const struct lm_ggml_tensor * tensor); // same as lm_ggml_is_contiguous()
750
- LM_GGML_API LM_GGML_CALL bool lm_ggml_is_contiguous_1(const struct lm_ggml_tensor * tensor); // contiguous for dims >= 1
751
- LM_GGML_API LM_GGML_CALL bool lm_ggml_is_contiguous_2(const struct lm_ggml_tensor * tensor); // contiguous for dims >= 2
752
-
753
- LM_GGML_API bool lm_ggml_are_same_shape (const struct lm_ggml_tensor * t0, const struct lm_ggml_tensor * t1);
754
- LM_GGML_API bool lm_ggml_are_same_stride(const struct lm_ggml_tensor * t0, const struct lm_ggml_tensor * t1);
755
-
756
- // use this to compute the memory overhead of a tensor
757
- LM_GGML_API size_t lm_ggml_tensor_overhead(void);
758
-
759
- LM_GGML_API bool lm_ggml_validate_row_data(enum lm_ggml_type type, const void * data, size_t nbytes);
760
-
761
- // main
762
-
763
- LM_GGML_API struct lm_ggml_context * lm_ggml_init(struct lm_ggml_init_params params);
764
- LM_GGML_API void lm_ggml_free(struct lm_ggml_context * ctx);
765
-
766
- LM_GGML_API size_t lm_ggml_used_mem(const struct lm_ggml_context * ctx);
767
-
768
- LM_GGML_API size_t lm_ggml_set_scratch (struct lm_ggml_context * ctx, struct lm_ggml_scratch scratch);
769
- LM_GGML_API bool lm_ggml_get_no_alloc(struct lm_ggml_context * ctx);
770
- LM_GGML_API void lm_ggml_set_no_alloc(struct lm_ggml_context * ctx, bool no_alloc);
771
-
772
- LM_GGML_API void * lm_ggml_get_mem_buffer (const struct lm_ggml_context * ctx);
773
- LM_GGML_API size_t lm_ggml_get_mem_size (const struct lm_ggml_context * ctx);
774
- LM_GGML_API size_t lm_ggml_get_max_tensor_size(const struct lm_ggml_context * ctx);
775
-
776
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_new_tensor(
777
- struct lm_ggml_context * ctx,
778
- enum lm_ggml_type type,
779
- int n_dims,
780
- const int64_t *ne);
781
-
782
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_new_tensor_1d(
783
- struct lm_ggml_context * ctx,
784
- enum lm_ggml_type type,
785
- int64_t ne0);
786
-
787
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_new_tensor_2d(
788
- struct lm_ggml_context * ctx,
789
- enum lm_ggml_type type,
790
- int64_t ne0,
791
- int64_t ne1);
792
-
793
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_new_tensor_3d(
794
- struct lm_ggml_context * ctx,
795
- enum lm_ggml_type type,
796
- int64_t ne0,
797
- int64_t ne1,
798
- int64_t ne2);
799
-
800
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_new_tensor_4d(
801
- struct lm_ggml_context * ctx,
802
- enum lm_ggml_type type,
803
- int64_t ne0,
804
- int64_t ne1,
805
- int64_t ne2,
806
- int64_t ne3);
807
-
808
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_new_i32(struct lm_ggml_context * ctx, int32_t value);
809
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_new_f32(struct lm_ggml_context * ctx, float value);
810
-
811
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_dup_tensor (struct lm_ggml_context * ctx, const struct lm_ggml_tensor * src);
812
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_view_tensor(struct lm_ggml_context * ctx, struct lm_ggml_tensor * src);
813
-
814
- // Context tensor enumeration and lookup
815
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_get_first_tensor(const struct lm_ggml_context * ctx);
816
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_get_next_tensor (const struct lm_ggml_context * ctx, struct lm_ggml_tensor * tensor);
817
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_get_tensor(struct lm_ggml_context * ctx, const char * name);
818
-
819
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_set_zero(struct lm_ggml_tensor * tensor);
820
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_set_i32 (struct lm_ggml_tensor * tensor, int32_t value);
821
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_set_f32 (struct lm_ggml_tensor * tensor, float value);
822
-
823
- // Converts a flat index into coordinates
824
- LM_GGML_API void lm_ggml_unravel_index(const struct lm_ggml_tensor * tensor, int64_t i, int64_t * i0, int64_t * i1, int64_t * i2, int64_t * i3);
825
-
826
- LM_GGML_API int32_t lm_ggml_get_i32_1d(const struct lm_ggml_tensor * tensor, int i);
827
- LM_GGML_API void lm_ggml_set_i32_1d(const struct lm_ggml_tensor * tensor, int i, int32_t value);
828
-
829
- LM_GGML_API int32_t lm_ggml_get_i32_nd(const struct lm_ggml_tensor * tensor, int i0, int i1, int i2, int i3);
830
- LM_GGML_API void lm_ggml_set_i32_nd(const struct lm_ggml_tensor * tensor, int i0, int i1, int i2, int i3, int32_t value);
831
-
832
- LM_GGML_API float lm_ggml_get_f32_1d(const struct lm_ggml_tensor * tensor, int i);
833
- LM_GGML_API void lm_ggml_set_f32_1d(const struct lm_ggml_tensor * tensor, int i, float value);
834
-
835
- LM_GGML_API float lm_ggml_get_f32_nd(const struct lm_ggml_tensor * tensor, int i0, int i1, int i2, int i3);
836
- LM_GGML_API void lm_ggml_set_f32_nd(const struct lm_ggml_tensor * tensor, int i0, int i1, int i2, int i3, float value);
837
-
838
- LM_GGML_API void * lm_ggml_get_data (const struct lm_ggml_tensor * tensor);
839
- LM_GGML_API float * lm_ggml_get_data_f32(const struct lm_ggml_tensor * tensor);
840
-
841
- LM_GGML_API LM_GGML_CALL enum lm_ggml_unary_op lm_ggml_get_unary_op(const struct lm_ggml_tensor * tensor);
842
-
843
- LM_GGML_API const char * lm_ggml_get_name (const struct lm_ggml_tensor * tensor);
844
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_set_name ( struct lm_ggml_tensor * tensor, const char * name);
845
- LM_GGML_ATTRIBUTE_FORMAT(2, 3)
846
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_format_name( struct lm_ggml_tensor * tensor, const char * fmt, ...);
847
-
848
- //
849
- // operations on tensors with backpropagation
850
- //
851
-
852
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_dup(
853
- struct lm_ggml_context * ctx,
854
- struct lm_ggml_tensor * a);
855
-
856
- // in-place, returns view(a)
857
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_dup_inplace(
858
- struct lm_ggml_context * ctx,
859
- struct lm_ggml_tensor * a);
860
-
861
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_add(
862
- struct lm_ggml_context * ctx,
863
- struct lm_ggml_tensor * a,
864
- struct lm_ggml_tensor * b);
865
-
866
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_add_inplace(
867
- struct lm_ggml_context * ctx,
868
- struct lm_ggml_tensor * a,
869
- struct lm_ggml_tensor * b);
870
-
871
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_add_cast(
872
- struct lm_ggml_context * ctx,
873
- struct lm_ggml_tensor * a,
874
- struct lm_ggml_tensor * b,
875
- enum lm_ggml_type type);
876
-
877
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_add1(
878
- struct lm_ggml_context * ctx,
879
- struct lm_ggml_tensor * a,
880
- struct lm_ggml_tensor * b);
881
-
882
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_add1_inplace(
883
- struct lm_ggml_context * ctx,
884
- struct lm_ggml_tensor * a,
885
- struct lm_ggml_tensor * b);
886
-
887
- // dst = a
888
- // view(dst, nb1, nb2, nb3, offset) += b
889
- // return dst
890
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_acc(
891
- struct lm_ggml_context * ctx,
892
- struct lm_ggml_tensor * a,
893
- struct lm_ggml_tensor * b,
894
- size_t nb1,
895
- size_t nb2,
896
- size_t nb3,
897
- size_t offset);
898
-
899
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_acc_inplace(
900
- struct lm_ggml_context * ctx,
901
- struct lm_ggml_tensor * a,
902
- struct lm_ggml_tensor * b,
903
- size_t nb1,
904
- size_t nb2,
905
- size_t nb3,
906
- size_t offset);
907
-
908
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_sub(
909
- struct lm_ggml_context * ctx,
910
- struct lm_ggml_tensor * a,
911
- struct lm_ggml_tensor * b);
912
-
913
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_sub_inplace(
914
- struct lm_ggml_context * ctx,
915
- struct lm_ggml_tensor * a,
916
- struct lm_ggml_tensor * b);
917
-
918
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_mul(
919
- struct lm_ggml_context * ctx,
920
- struct lm_ggml_tensor * a,
921
- struct lm_ggml_tensor * b);
922
-
923
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_mul_inplace(
924
- struct lm_ggml_context * ctx,
925
- struct lm_ggml_tensor * a,
926
- struct lm_ggml_tensor * b);
927
-
928
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_div(
929
- struct lm_ggml_context * ctx,
930
- struct lm_ggml_tensor * a,
931
- struct lm_ggml_tensor * b);
932
-
933
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_div_inplace(
934
- struct lm_ggml_context * ctx,
935
- struct lm_ggml_tensor * a,
936
- struct lm_ggml_tensor * b);
937
-
938
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_sqr(
939
- struct lm_ggml_context * ctx,
940
- struct lm_ggml_tensor * a);
941
-
942
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_sqr_inplace(
943
- struct lm_ggml_context * ctx,
944
- struct lm_ggml_tensor * a);
945
-
946
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_sqrt(
947
- struct lm_ggml_context * ctx,
948
- struct lm_ggml_tensor * a);
949
-
950
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_sqrt_inplace(
951
- struct lm_ggml_context * ctx,
952
- struct lm_ggml_tensor * a);
953
-
954
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_log(
955
- struct lm_ggml_context * ctx,
956
- struct lm_ggml_tensor * a);
957
-
958
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_log_inplace(
959
- struct lm_ggml_context * ctx,
960
- struct lm_ggml_tensor * a);
961
-
962
- // return scalar
963
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_sum(
964
- struct lm_ggml_context * ctx,
965
- struct lm_ggml_tensor * a);
966
-
967
- // sums along rows, with input shape [a,b,c,d] return shape [1,b,c,d]
968
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_sum_rows(
969
- struct lm_ggml_context * ctx,
970
- struct lm_ggml_tensor * a);
971
-
972
- // mean along rows
973
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_mean(
974
- struct lm_ggml_context * ctx,
975
- struct lm_ggml_tensor * a);
976
-
977
- // argmax along rows
978
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_argmax(
979
- struct lm_ggml_context * ctx,
980
- struct lm_ggml_tensor * a);
981
-
982
- // if a is the same shape as b, and a is not parameter, return a
983
- // otherwise, return a new tensor: repeat(a) to fit in b
984
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_repeat(
985
- struct lm_ggml_context * ctx,
986
- struct lm_ggml_tensor * a,
987
- struct lm_ggml_tensor * b);
988
-
989
- // sums repetitions in a into shape of b
990
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_repeat_back(
991
- struct lm_ggml_context * ctx,
992
- struct lm_ggml_tensor * a,
993
- struct lm_ggml_tensor * b);
994
-
995
- // concat a and b along dim
996
- // used in stable-diffusion
997
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_concat(
998
- struct lm_ggml_context * ctx,
999
- struct lm_ggml_tensor * a,
1000
- struct lm_ggml_tensor * b,
1001
- int dim);
1002
-
1003
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_abs(
1004
- struct lm_ggml_context * ctx,
1005
- struct lm_ggml_tensor * a);
1006
-
1007
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_abs_inplace(
1008
- struct lm_ggml_context * ctx,
1009
- struct lm_ggml_tensor * a);
1010
-
1011
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_sgn(
1012
- struct lm_ggml_context * ctx,
1013
- struct lm_ggml_tensor * a);
1014
-
1015
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_sgn_inplace(
1016
- struct lm_ggml_context * ctx,
1017
- struct lm_ggml_tensor * a);
1018
-
1019
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_neg(
1020
- struct lm_ggml_context * ctx,
1021
- struct lm_ggml_tensor * a);
1022
-
1023
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_neg_inplace(
1024
- struct lm_ggml_context * ctx,
1025
- struct lm_ggml_tensor * a);
1026
-
1027
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_step(
1028
- struct lm_ggml_context * ctx,
1029
- struct lm_ggml_tensor * a);
1030
-
1031
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_step_inplace(
1032
- struct lm_ggml_context * ctx,
1033
- struct lm_ggml_tensor * a);
1034
-
1035
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_tanh(
1036
- struct lm_ggml_context * ctx,
1037
- struct lm_ggml_tensor * a);
1038
-
1039
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_tanh_inplace(
1040
- struct lm_ggml_context * ctx,
1041
- struct lm_ggml_tensor * a);
1042
-
1043
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_elu(
1044
- struct lm_ggml_context * ctx,
1045
- struct lm_ggml_tensor * a);
1046
-
1047
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_elu_inplace(
1048
- struct lm_ggml_context * ctx,
1049
- struct lm_ggml_tensor * a);
1050
-
1051
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_relu(
1052
- struct lm_ggml_context * ctx,
1053
- struct lm_ggml_tensor * a);
1054
-
1055
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_leaky_relu(
1056
- struct lm_ggml_context * ctx,
1057
- struct lm_ggml_tensor * a, float negative_slope, bool inplace);
1058
-
1059
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_relu_inplace(
1060
- struct lm_ggml_context * ctx,
1061
- struct lm_ggml_tensor * a);
1062
-
1063
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_sigmoid(
1064
- struct lm_ggml_context * ctx,
1065
- struct lm_ggml_tensor * a);
1066
-
1067
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_sigmoid_inplace(
1068
- struct lm_ggml_context * ctx,
1069
- struct lm_ggml_tensor * a);
1070
-
1071
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_gelu(
1072
- struct lm_ggml_context * ctx,
1073
- struct lm_ggml_tensor * a);
1074
-
1075
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_gelu_inplace(
1076
- struct lm_ggml_context * ctx,
1077
- struct lm_ggml_tensor * a);
1078
-
1079
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_gelu_quick(
1080
- struct lm_ggml_context * ctx,
1081
- struct lm_ggml_tensor * a);
1082
-
1083
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_gelu_quick_inplace(
1084
- struct lm_ggml_context * ctx,
1085
- struct lm_ggml_tensor * a);
1086
-
1087
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_silu(
1088
- struct lm_ggml_context * ctx,
1089
- struct lm_ggml_tensor * a);
1090
-
1091
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_silu_inplace(
1092
- struct lm_ggml_context * ctx,
1093
- struct lm_ggml_tensor * a);
1094
-
1095
- // a - x
1096
- // b - dy
1097
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_silu_back(
1098
- struct lm_ggml_context * ctx,
1099
- struct lm_ggml_tensor * a,
1100
- struct lm_ggml_tensor * b);
1101
-
1102
- // hardswish(x) = x * relu6(x + 3) / 6
1103
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_hardswish(
1104
- struct lm_ggml_context * ctx,
1105
- struct lm_ggml_tensor * a);
1106
-
1107
- // hardsigmoid(x) = relu6(x + 3) / 6
1108
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_hardsigmoid(
1109
- struct lm_ggml_context * ctx,
1110
- struct lm_ggml_tensor * a);
1111
-
1112
- // normalize along rows
1113
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_norm(
1114
- struct lm_ggml_context * ctx,
1115
- struct lm_ggml_tensor * a,
1116
- float eps);
1117
-
1118
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_norm_inplace(
1119
- struct lm_ggml_context * ctx,
1120
- struct lm_ggml_tensor * a,
1121
- float eps);
1122
-
1123
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_rms_norm(
1124
- struct lm_ggml_context * ctx,
1125
- struct lm_ggml_tensor * a,
1126
- float eps);
1127
-
1128
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_rms_norm_inplace(
1129
- struct lm_ggml_context * ctx,
1130
- struct lm_ggml_tensor * a,
1131
- float eps);
1132
-
1133
- // group normalize along ne0*ne1*n_groups
1134
- // used in stable-diffusion
1135
- // TODO: eps is hardcoded to 1e-6 for now
1136
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_group_norm(
1137
- struct lm_ggml_context * ctx,
1138
- struct lm_ggml_tensor * a,
1139
- int n_groups);
1140
-
1141
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_group_norm_inplace(
1142
- struct lm_ggml_context * ctx,
1143
- struct lm_ggml_tensor * a,
1144
- int n_groups);
1145
-
1146
- // a - x
1147
- // b - dy
1148
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_rms_norm_back(
1149
- struct lm_ggml_context * ctx,
1150
- struct lm_ggml_tensor * a,
1151
- struct lm_ggml_tensor * b,
1152
- float eps);
1153
-
1154
- // A: k columns, n rows => [ne03, ne02, n, k]
1155
- // B: k columns, m rows (i.e. we transpose it internally) => [ne03 * x, ne02 * y, m, k]
1156
- // result is n columns, m rows => [ne03 * x, ne02 * y, m, n]
1157
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_mul_mat(
1158
- struct lm_ggml_context * ctx,
1159
- struct lm_ggml_tensor * a,
1160
- struct lm_ggml_tensor * b);
1161
-
1162
- // change the precision of a matrix multiplication
1163
- // set to LM_GGML_PREC_F32 for higher precision (useful for phi-2)
1164
- LM_GGML_API void lm_ggml_mul_mat_set_prec(
1165
- struct lm_ggml_tensor * a,
1166
- enum lm_ggml_prec prec);
1167
-
1168
- // indirect matrix multiplication
1169
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_mul_mat_id(
1170
- struct lm_ggml_context * ctx,
1171
- struct lm_ggml_tensor * as,
1172
- struct lm_ggml_tensor * b,
1173
- struct lm_ggml_tensor * ids);
1174
-
1175
- // A: m columns, n rows,
1176
- // B: p columns, n rows,
1177
- // result is m columns, p rows
1178
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_out_prod(
1179
- struct lm_ggml_context * ctx,
1180
- struct lm_ggml_tensor * a,
1181
- struct lm_ggml_tensor * b);
1182
-
1183
- //
1184
- // operations on tensors without backpropagation
1185
- //
1186
-
1187
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_scale(
1188
- struct lm_ggml_context * ctx,
1189
- struct lm_ggml_tensor * a,
1190
- float s);
1191
-
1192
- // in-place, returns view(a)
1193
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_scale_inplace(
1194
- struct lm_ggml_context * ctx,
1195
- struct lm_ggml_tensor * a,
1196
- float s);
1197
-
1198
- // b -> view(a,offset,nb1,nb2,3), return modified a
1199
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_set(
1200
- struct lm_ggml_context * ctx,
1201
- struct lm_ggml_tensor * a,
1202
- struct lm_ggml_tensor * b,
1203
- size_t nb1,
1204
- size_t nb2,
1205
- size_t nb3,
1206
- size_t offset);
1207
-
1208
- // b -> view(a,offset,nb1,nb2,3), return view(a)
1209
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_set_inplace(
1210
- struct lm_ggml_context * ctx,
1211
- struct lm_ggml_tensor * a,
1212
- struct lm_ggml_tensor * b,
1213
- size_t nb1,
1214
- size_t nb2,
1215
- size_t nb3,
1216
- size_t offset);
1217
-
1218
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_set_1d(
1219
- struct lm_ggml_context * ctx,
1220
- struct lm_ggml_tensor * a,
1221
- struct lm_ggml_tensor * b,
1222
- size_t offset);
1223
-
1224
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_set_1d_inplace(
1225
- struct lm_ggml_context * ctx,
1226
- struct lm_ggml_tensor * a,
1227
- struct lm_ggml_tensor * b,
1228
- size_t offset);
1229
-
1230
- // b -> view(a,offset,nb1,nb2,3), return modified a
1231
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_set_2d(
1232
- struct lm_ggml_context * ctx,
1233
- struct lm_ggml_tensor * a,
1234
- struct lm_ggml_tensor * b,
1235
- size_t nb1,
1236
- size_t offset);
1237
-
1238
- // b -> view(a,offset,nb1,nb2,3), return view(a)
1239
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_set_2d_inplace(
1240
- struct lm_ggml_context * ctx,
1241
- struct lm_ggml_tensor * a,
1242
- struct lm_ggml_tensor * b,
1243
- size_t nb1,
1244
- size_t offset);
1245
-
1246
- // a -> b, return view(b)
1247
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_cpy(
1248
- struct lm_ggml_context * ctx,
1249
- struct lm_ggml_tensor * a,
1250
- struct lm_ggml_tensor * b);
1251
-
1252
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_cast(
1253
- struct lm_ggml_context * ctx,
1254
- struct lm_ggml_tensor * a,
1255
- enum lm_ggml_type type);
1256
-
1257
- // make contiguous
1258
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_cont(
1259
- struct lm_ggml_context * ctx,
1260
- struct lm_ggml_tensor * a);
1261
-
1262
- // make contiguous, with new shape
1263
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_cont_1d(
1264
- struct lm_ggml_context * ctx,
1265
- struct lm_ggml_tensor * a,
1266
- int64_t ne0);
1267
-
1268
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_cont_2d(
1269
- struct lm_ggml_context * ctx,
1270
- struct lm_ggml_tensor * a,
1271
- int64_t ne0,
1272
- int64_t ne1);
1273
-
1274
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_cont_3d(
1275
- struct lm_ggml_context * ctx,
1276
- struct lm_ggml_tensor * a,
1277
- int64_t ne0,
1278
- int64_t ne1,
1279
- int64_t ne2);
1280
-
1281
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_cont_4d(
1282
- struct lm_ggml_context * ctx,
1283
- struct lm_ggml_tensor * a,
1284
- int64_t ne0,
1285
- int64_t ne1,
1286
- int64_t ne2,
1287
- int64_t ne3);
1288
-
1289
- // return view(a), b specifies the new shape
1290
- // TODO: when we start computing gradient, make a copy instead of view
1291
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_reshape(
1292
- struct lm_ggml_context * ctx,
1293
- struct lm_ggml_tensor * a,
1294
- struct lm_ggml_tensor * b);
1295
-
1296
- // return view(a)
1297
- // TODO: when we start computing gradient, make a copy instead of view
1298
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_reshape_1d(
1299
- struct lm_ggml_context * ctx,
1300
- struct lm_ggml_tensor * a,
1301
- int64_t ne0);
1302
-
1303
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_reshape_2d(
1304
- struct lm_ggml_context * ctx,
1305
- struct lm_ggml_tensor * a,
1306
- int64_t ne0,
1307
- int64_t ne1);
1308
-
1309
- // return view(a)
1310
- // TODO: when we start computing gradient, make a copy instead of view
1311
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_reshape_3d(
1312
- struct lm_ggml_context * ctx,
1313
- struct lm_ggml_tensor * a,
1314
- int64_t ne0,
1315
- int64_t ne1,
1316
- int64_t ne2);
1317
-
1318
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_reshape_4d(
1319
- struct lm_ggml_context * ctx,
1320
- struct lm_ggml_tensor * a,
1321
- int64_t ne0,
1322
- int64_t ne1,
1323
- int64_t ne2,
1324
- int64_t ne3);
1325
-
1326
- // offset in bytes
1327
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_view_1d(
1328
- struct lm_ggml_context * ctx,
1329
- struct lm_ggml_tensor * a,
1330
- int64_t ne0,
1331
- size_t offset);
1332
-
1333
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_view_2d(
1334
- struct lm_ggml_context * ctx,
1335
- struct lm_ggml_tensor * a,
1336
- int64_t ne0,
1337
- int64_t ne1,
1338
- size_t nb1, // row stride in bytes
1339
- size_t offset);
1340
-
1341
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_view_3d(
1342
- struct lm_ggml_context * ctx,
1343
- struct lm_ggml_tensor * a,
1344
- int64_t ne0,
1345
- int64_t ne1,
1346
- int64_t ne2,
1347
- size_t nb1, // row stride in bytes
1348
- size_t nb2, // slice stride in bytes
1349
- size_t offset);
1350
-
1351
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_view_4d(
1352
- struct lm_ggml_context * ctx,
1353
- struct lm_ggml_tensor * a,
1354
- int64_t ne0,
1355
- int64_t ne1,
1356
- int64_t ne2,
1357
- int64_t ne3,
1358
- size_t nb1, // row stride in bytes
1359
- size_t nb2, // slice stride in bytes
1360
- size_t nb3,
1361
- size_t offset);
1362
-
1363
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_permute(
1364
- struct lm_ggml_context * ctx,
1365
- struct lm_ggml_tensor * a,
1366
- int axis0,
1367
- int axis1,
1368
- int axis2,
1369
- int axis3);
1370
-
1371
- // alias for lm_ggml_permute(ctx, a, 1, 0, 2, 3)
1372
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_transpose(
1373
- struct lm_ggml_context * ctx,
1374
- struct lm_ggml_tensor * a);
1375
-
1376
- // supports 3D: a->ne[2] == b->ne[1]
1377
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_get_rows(
1378
- struct lm_ggml_context * ctx,
1379
- struct lm_ggml_tensor * a,
1380
- struct lm_ggml_tensor * b);
1381
-
1382
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_get_rows_back(
1383
- struct lm_ggml_context * ctx,
1384
- struct lm_ggml_tensor * a,
1385
- struct lm_ggml_tensor * b,
1386
- struct lm_ggml_tensor * c);
1387
-
1388
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_diag(
1389
- struct lm_ggml_context * ctx,
1390
- struct lm_ggml_tensor * a);
1391
-
1392
- // set elements above the diagonal to -INF
1393
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_diag_mask_inf(
1394
- struct lm_ggml_context * ctx,
1395
- struct lm_ggml_tensor * a,
1396
- int n_past);
1397
-
1398
- // in-place, returns view(a)
1399
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_diag_mask_inf_inplace(
1400
- struct lm_ggml_context * ctx,
1401
- struct lm_ggml_tensor * a,
1402
- int n_past);
1403
-
1404
- // set elements above the diagonal to 0
1405
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_diag_mask_zero(
1406
- struct lm_ggml_context * ctx,
1407
- struct lm_ggml_tensor * a,
1408
- int n_past);
1409
-
1410
- // in-place, returns view(a)
1411
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_diag_mask_zero_inplace(
1412
- struct lm_ggml_context * ctx,
1413
- struct lm_ggml_tensor * a,
1414
- int n_past);
1415
-
1416
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_soft_max(
1417
- struct lm_ggml_context * ctx,
1418
- struct lm_ggml_tensor * a);
1419
-
1420
- // in-place, returns view(a)
1421
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_soft_max_inplace(
1422
- struct lm_ggml_context * ctx,
1423
- struct lm_ggml_tensor * a);
1424
-
1425
- // fused soft_max(a*scale + mask*(ALiBi slope))
1426
- // mask is optional
1427
- // max_bias = 0.0f for no ALiBi
1428
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_soft_max_ext(
1429
- struct lm_ggml_context * ctx,
1430
- struct lm_ggml_tensor * a,
1431
- struct lm_ggml_tensor * mask,
1432
- float scale,
1433
- float max_bias);
1434
-
1435
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_soft_max_back(
1436
- struct lm_ggml_context * ctx,
1437
- struct lm_ggml_tensor * a,
1438
- struct lm_ggml_tensor * b);
1439
-
1440
- // in-place, returns view(a)
1441
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_soft_max_back_inplace(
1442
- struct lm_ggml_context * ctx,
1443
- struct lm_ggml_tensor * a,
1444
- struct lm_ggml_tensor * b);
1445
-
1446
- // rotary position embedding
1447
- // if mode & 1 == 1, skip n_past elements (NOT SUPPORTED)
1448
- // if mode & 2 == 1, GPT-NeoX style
1449
- //
1450
- // b is an int32 vector with size a->ne[2], it contains the positions
1451
- // c is freq factors (e.g. phi3-128k), (optional)
1452
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_rope(
1453
- struct lm_ggml_context * ctx,
1454
- struct lm_ggml_tensor * a,
1455
- struct lm_ggml_tensor * b,
1456
- int n_dims,
1457
- int mode);
1458
-
1459
- // in-place, returns view(a)
1460
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_rope_inplace(
1461
- struct lm_ggml_context * ctx,
1462
- struct lm_ggml_tensor * a,
1463
- struct lm_ggml_tensor * b,
1464
- int n_dims,
1465
- int mode);
1466
-
1467
- // custom RoPE
1468
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_rope_ext(
1469
- struct lm_ggml_context * ctx,
1470
- struct lm_ggml_tensor * a,
1471
- struct lm_ggml_tensor * b,
1472
- struct lm_ggml_tensor * c,
1473
- int n_dims,
1474
- int mode,
1475
- int n_ctx_orig,
1476
- float freq_base,
1477
- float freq_scale,
1478
- float ext_factor,
1479
- float attn_factor,
1480
- float beta_fast,
1481
- float beta_slow);
1482
-
1483
- // in-place, returns view(a)
1484
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_rope_ext_inplace(
1485
- struct lm_ggml_context * ctx,
1486
- struct lm_ggml_tensor * a,
1487
- struct lm_ggml_tensor * b,
1488
- struct lm_ggml_tensor * c,
1489
- int n_dims,
1490
- int mode,
1491
- int n_ctx_orig,
1492
- float freq_base,
1493
- float freq_scale,
1494
- float ext_factor,
1495
- float attn_factor,
1496
- float beta_fast,
1497
- float beta_slow);
1498
-
1499
- LM_GGML_DEPRECATED(LM_GGML_API struct lm_ggml_tensor * lm_ggml_rope_custom(
1500
- struct lm_ggml_context * ctx,
1501
- struct lm_ggml_tensor * a,
1502
- struct lm_ggml_tensor * b,
1503
- int n_dims,
1504
- int mode,
1505
- int n_ctx_orig,
1506
- float freq_base,
1507
- float freq_scale,
1508
- float ext_factor,
1509
- float attn_factor,
1510
- float beta_fast,
1511
- float beta_slow),
1512
- "use lm_ggml_rope_ext instead");
1513
-
1514
- LM_GGML_DEPRECATED(LM_GGML_API struct lm_ggml_tensor * lm_ggml_rope_custom_inplace(
1515
- struct lm_ggml_context * ctx,
1516
- struct lm_ggml_tensor * a,
1517
- struct lm_ggml_tensor * b,
1518
- int n_dims,
1519
- int mode,
1520
- int n_ctx_orig,
1521
- float freq_base,
1522
- float freq_scale,
1523
- float ext_factor,
1524
- float attn_factor,
1525
- float beta_fast,
1526
- float beta_slow),
1527
- "use lm_ggml_rope_ext_inplace instead");
1528
-
1529
- // compute correction dims for YaRN RoPE scaling
1530
- LM_GGML_CALL void lm_ggml_rope_yarn_corr_dims(
1531
- int n_dims, int n_ctx_orig, float freq_base, float beta_fast, float beta_slow, float dims[2]);
1532
-
1533
- // rotary position embedding backward, i.e compute dx from dy
1534
- // a - dy
1535
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_rope_back(
1536
- struct lm_ggml_context * ctx,
1537
- struct lm_ggml_tensor * a,
1538
- struct lm_ggml_tensor * b,
1539
- struct lm_ggml_tensor * c,
1540
- int n_dims,
1541
- int mode,
1542
- int n_ctx_orig,
1543
- float freq_base,
1544
- float freq_scale,
1545
- float ext_factor,
1546
- float attn_factor,
1547
- float beta_fast,
1548
- float beta_slow);
1549
-
1550
- // clamp
1551
- // in-place, returns view(a)
1552
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_clamp(
1553
- struct lm_ggml_context * ctx,
1554
- struct lm_ggml_tensor * a,
1555
- float min,
1556
- float max);
1557
-
1558
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_im2col(
1559
- struct lm_ggml_context * ctx,
1560
- struct lm_ggml_tensor * a,
1561
- struct lm_ggml_tensor * b,
1562
- int s0,
1563
- int s1,
1564
- int p0,
1565
- int p1,
1566
- int d0,
1567
- int d1,
1568
- bool is_2D,
1569
- enum lm_ggml_type dst_type);
1570
-
1571
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_conv_depthwise_2d(
1572
- struct lm_ggml_context * ctx,
1573
- struct lm_ggml_tensor * a,
1574
- struct lm_ggml_tensor * b,
1575
- int s0,
1576
- int s1,
1577
- int p0,
1578
- int p1,
1579
- int d0,
1580
- int d1);
1581
-
1582
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_conv_1d(
1583
- struct lm_ggml_context * ctx,
1584
- struct lm_ggml_tensor * a,
1585
- struct lm_ggml_tensor * b,
1586
- int s0, // stride
1587
- int p0, // padding
1588
- int d0); // dilation
1589
-
1590
- // conv_1d with padding = half
1591
- // alias for lm_ggml_conv_1d(a, b, s, a->ne[0]/2, d)
1592
- LM_GGML_API struct lm_ggml_tensor* lm_ggml_conv_1d_ph(
1593
- struct lm_ggml_context * ctx,
1594
- struct lm_ggml_tensor * a,
1595
- struct lm_ggml_tensor * b,
1596
- int s,
1597
- int d);
1598
-
1599
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_conv_transpose_1d(
1600
- struct lm_ggml_context * ctx,
1601
- struct lm_ggml_tensor * a,
1602
- struct lm_ggml_tensor * b,
1603
- int s0,
1604
- int p0,
1605
- int d0);
1606
-
1607
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_conv_2d(
1608
- struct lm_ggml_context * ctx,
1609
- struct lm_ggml_tensor * a,
1610
- struct lm_ggml_tensor * b,
1611
- int s0,
1612
- int s1,
1613
- int p0,
1614
- int p1,
1615
- int d0,
1616
- int d1);
1617
-
1618
-
1619
- // kernel size is a->ne[0] x a->ne[1]
1620
- // stride is equal to kernel size
1621
- // padding is zero
1622
- // example:
1623
- // a: 16 16 3 768
1624
- // b: 1024 1024 3 1
1625
- // res: 64 64 768 1
1626
- // used in sam
1627
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_conv_2d_sk_p0(
1628
- struct lm_ggml_context * ctx,
1629
- struct lm_ggml_tensor * a,
1630
- struct lm_ggml_tensor * b);
1631
-
1632
- // kernel size is a->ne[0] x a->ne[1]
1633
- // stride is 1
1634
- // padding is half
1635
- // example:
1636
- // a: 3 3 256 256
1637
- // b: 64 64 256 1
1638
- // res: 64 64 256 1
1639
- // used in sam
1640
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_conv_2d_s1_ph(
1641
- struct lm_ggml_context * ctx,
1642
- struct lm_ggml_tensor * a,
1643
- struct lm_ggml_tensor * b);
1644
-
1645
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_conv_transpose_2d_p0(
1646
- struct lm_ggml_context * ctx,
1647
- struct lm_ggml_tensor * a,
1648
- struct lm_ggml_tensor * b,
1649
- int stride);
1650
-
1651
- enum lm_ggml_op_pool {
1652
- LM_GGML_OP_POOL_MAX,
1653
- LM_GGML_OP_POOL_AVG,
1654
- LM_GGML_OP_POOL_COUNT,
1655
- };
1656
-
1657
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_pool_1d(
1658
- struct lm_ggml_context * ctx,
1659
- struct lm_ggml_tensor * a,
1660
- enum lm_ggml_op_pool op,
1661
- int k0, // kernel size
1662
- int s0, // stride
1663
- int p0); // padding
1664
-
1665
- // the result will have 2*p0 padding for the first dimension
1666
- // and 2*p1 padding for the second dimension
1667
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_pool_2d(
1668
- struct lm_ggml_context * ctx,
1669
- struct lm_ggml_tensor * a,
1670
- enum lm_ggml_op_pool op,
1671
- int k0,
1672
- int k1,
1673
- int s0,
1674
- int s1,
1675
- float p0,
1676
- float p1);
1677
-
1678
- // nearest interpolate
1679
- // multiplies ne0 and ne1 by scale factor
1680
- // used in stable-diffusion
1681
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_upscale(
1682
- struct lm_ggml_context * ctx,
1683
- struct lm_ggml_tensor * a,
1684
- int scale_factor);
1685
-
1686
- // nearest interpolate
1687
- // nearest interpolate to specified dimensions
1688
- // used in tortoise.cpp
1689
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_upscale_ext(
1690
- struct lm_ggml_context * ctx,
1691
- struct lm_ggml_tensor * a,
1692
- int ne0,
1693
- int ne1,
1694
- int ne2,
1695
- int ne3);
1696
-
1697
- // pad each dimension with zeros: [x, ..., x] -> [x, ..., x, 0, ..., 0]
1698
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_pad(
1699
- struct lm_ggml_context * ctx,
1700
- struct lm_ggml_tensor * a,
1701
- int p0,
1702
- int p1,
1703
- int p2,
1704
- int p3);
1705
-
1706
- // Ref: https://github.com/CompVis/stable-diffusion/blob/main/ldm/modules/diffusionmodules/util.py#L151
1707
- // timesteps: [N,]
1708
- // return: [N, dim]
1709
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_timestep_embedding(
1710
- struct lm_ggml_context * ctx,
1711
- struct lm_ggml_tensor * timesteps,
1712
- int dim,
1713
- int max_period);
1714
-
1715
- // sort rows
1716
- enum lm_ggml_sort_order {
1717
- LM_GGML_SORT_ORDER_ASC,
1718
- LM_GGML_SORT_ORDER_DESC,
1719
- };
1720
-
1721
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_argsort(
1722
- struct lm_ggml_context * ctx,
1723
- struct lm_ggml_tensor * a,
1724
- enum lm_ggml_sort_order order);
1725
-
1726
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_arange(
1727
- struct lm_ggml_context * ctx,
1728
- float start,
1729
- float stop,
1730
- float step);
1731
-
1732
- // top k elements per row
1733
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_top_k(
1734
- struct lm_ggml_context * ctx,
1735
- struct lm_ggml_tensor * a,
1736
- int k);
1737
-
1738
- #define LM_GGML_KQ_MASK_PAD 32
1739
-
1740
- // q: [n_embd, n_batch, n_head, 1]
1741
- // k: [n_embd, n_kv, n_head_kv, 1]
1742
- // v: [n_embd, n_kv, n_head_kv, 1] !! not transposed !!
1743
- // mask: [n_kv, n_batch_pad, 1, 1] !! n_batch_pad = LM_GGML_PAD(n_batch, LM_GGML_KQ_MASK_PAD) !!
1744
- // res: [n_embd, n_head, n_batch, 1] !! permuted !!
1745
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_flash_attn_ext(
1746
- struct lm_ggml_context * ctx,
1747
- struct lm_ggml_tensor * q,
1748
- struct lm_ggml_tensor * k,
1749
- struct lm_ggml_tensor * v,
1750
- struct lm_ggml_tensor * mask,
1751
- float scale,
1752
- float max_bias);
1753
-
1754
- LM_GGML_API void lm_ggml_flash_attn_ext_set_prec(
1755
- struct lm_ggml_tensor * a,
1756
- enum lm_ggml_prec prec);
1757
-
1758
- // TODO: needs to be adapted to lm_ggml_flash_attn_ext
1759
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_flash_attn_back(
1760
- struct lm_ggml_context * ctx,
1761
- struct lm_ggml_tensor * q,
1762
- struct lm_ggml_tensor * k,
1763
- struct lm_ggml_tensor * v,
1764
- struct lm_ggml_tensor * d,
1765
- bool masked);
1766
-
1767
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_ssm_conv(
1768
- struct lm_ggml_context * ctx,
1769
- struct lm_ggml_tensor * s,
1770
- struct lm_ggml_tensor * x,
1771
- struct lm_ggml_tensor * c,
1772
- struct lm_ggml_tensor * sq);
1773
-
1774
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_ssm_scan(
1775
- struct lm_ggml_context * ctx,
1776
- struct lm_ggml_tensor * s,
1777
- struct lm_ggml_tensor * x,
1778
- struct lm_ggml_tensor * dt,
1779
- struct lm_ggml_tensor * A,
1780
- struct lm_ggml_tensor * B,
1781
- struct lm_ggml_tensor * C,
1782
- struct lm_ggml_tensor * sq);
1783
-
1784
- // partition into non-overlapping windows with padding if needed
1785
- // example:
1786
- // a: 768 64 64 1
1787
- // w: 14
1788
- // res: 768 14 14 25
1789
- // used in sam
1790
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_win_part(
1791
- struct lm_ggml_context * ctx,
1792
- struct lm_ggml_tensor * a,
1793
- int w);
1794
-
1795
- // reverse of lm_ggml_win_part
1796
- // used in sam
1797
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_win_unpart(
1798
- struct lm_ggml_context * ctx,
1799
- struct lm_ggml_tensor * a,
1800
- int w0,
1801
- int h0,
1802
- int w);
1803
-
1804
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_unary(
1805
- struct lm_ggml_context * ctx,
1806
- struct lm_ggml_tensor * a,
1807
- enum lm_ggml_unary_op op);
1808
-
1809
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_unary_inplace(
1810
- struct lm_ggml_context * ctx,
1811
- struct lm_ggml_tensor * a,
1812
- enum lm_ggml_unary_op op);
1813
-
1814
- // used in sam
1815
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_get_rel_pos(
1816
- struct lm_ggml_context * ctx,
1817
- struct lm_ggml_tensor * a,
1818
- int qh,
1819
- int kh);
1820
-
1821
- // used in sam
1822
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_add_rel_pos(
1823
- struct lm_ggml_context * ctx,
1824
- struct lm_ggml_tensor * a,
1825
- struct lm_ggml_tensor * pw,
1826
- struct lm_ggml_tensor * ph);
1827
-
1828
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_add_rel_pos_inplace(
1829
- struct lm_ggml_context * ctx,
1830
- struct lm_ggml_tensor * a,
1831
- struct lm_ggml_tensor * pw,
1832
- struct lm_ggml_tensor * ph);
1833
-
1834
- // custom operators
1835
-
1836
- typedef void (*lm_ggml_unary_op_f32_t) (const int, float *, const float *);
1837
- typedef void (*lm_ggml_binary_op_f32_t)(const int, float *, const float *, const float *);
1838
-
1839
- typedef void (*lm_ggml_custom1_op_f32_t)(struct lm_ggml_tensor *, const struct lm_ggml_tensor *);
1840
- typedef void (*lm_ggml_custom2_op_f32_t)(struct lm_ggml_tensor *, const struct lm_ggml_tensor *, const struct lm_ggml_tensor *);
1841
- typedef void (*lm_ggml_custom3_op_f32_t)(struct lm_ggml_tensor *, const struct lm_ggml_tensor *, const struct lm_ggml_tensor *, const struct lm_ggml_tensor *);
1842
-
1843
- LM_GGML_DEPRECATED(LM_GGML_API struct lm_ggml_tensor * lm_ggml_map_unary_f32(
1844
- struct lm_ggml_context * ctx,
1845
- struct lm_ggml_tensor * a,
1846
- lm_ggml_unary_op_f32_t fun),
1847
- "use lm_ggml_map_custom1 instead");
1848
-
1849
- LM_GGML_DEPRECATED(LM_GGML_API struct lm_ggml_tensor * lm_ggml_map_unary_inplace_f32(
1850
- struct lm_ggml_context * ctx,
1851
- struct lm_ggml_tensor * a,
1852
- lm_ggml_unary_op_f32_t fun),
1853
- "use lm_ggml_map_custom1_inplace instead");
1854
-
1855
- LM_GGML_DEPRECATED(LM_GGML_API struct lm_ggml_tensor * lm_ggml_map_binary_f32(
1856
- struct lm_ggml_context * ctx,
1857
- struct lm_ggml_tensor * a,
1858
- struct lm_ggml_tensor * b,
1859
- lm_ggml_binary_op_f32_t fun),
1860
- "use lm_ggml_map_custom2 instead");
1861
-
1862
- LM_GGML_DEPRECATED(LM_GGML_API struct lm_ggml_tensor * lm_ggml_map_binary_inplace_f32(
1863
- struct lm_ggml_context * ctx,
1864
- struct lm_ggml_tensor * a,
1865
- struct lm_ggml_tensor * b,
1866
- lm_ggml_binary_op_f32_t fun),
1867
- "use lm_ggml_map_custom2_inplace instead");
1868
-
1869
- LM_GGML_DEPRECATED(LM_GGML_API struct lm_ggml_tensor * lm_ggml_map_custom1_f32(
1870
- struct lm_ggml_context * ctx,
1871
- struct lm_ggml_tensor * a,
1872
- lm_ggml_custom1_op_f32_t fun),
1873
- "use lm_ggml_map_custom1 instead");
1874
-
1875
- LM_GGML_DEPRECATED(LM_GGML_API struct lm_ggml_tensor * lm_ggml_map_custom1_inplace_f32(
1876
- struct lm_ggml_context * ctx,
1877
- struct lm_ggml_tensor * a,
1878
- lm_ggml_custom1_op_f32_t fun),
1879
- "use lm_ggml_map_custom1_inplace instead");
1880
-
1881
- LM_GGML_DEPRECATED(LM_GGML_API struct lm_ggml_tensor * lm_ggml_map_custom2_f32(
1882
- struct lm_ggml_context * ctx,
1883
- struct lm_ggml_tensor * a,
1884
- struct lm_ggml_tensor * b,
1885
- lm_ggml_custom2_op_f32_t fun),
1886
- "use lm_ggml_map_custom2 instead");
1887
-
1888
- LM_GGML_DEPRECATED(LM_GGML_API struct lm_ggml_tensor * lm_ggml_map_custom2_inplace_f32(
1889
- struct lm_ggml_context * ctx,
1890
- struct lm_ggml_tensor * a,
1891
- struct lm_ggml_tensor * b,
1892
- lm_ggml_custom2_op_f32_t fun),
1893
- "use lm_ggml_map_custom2_inplace instead");
1894
-
1895
- LM_GGML_DEPRECATED(LM_GGML_API struct lm_ggml_tensor * lm_ggml_map_custom3_f32(
1896
- struct lm_ggml_context * ctx,
1897
- struct lm_ggml_tensor * a,
1898
- struct lm_ggml_tensor * b,
1899
- struct lm_ggml_tensor * c,
1900
- lm_ggml_custom3_op_f32_t fun),
1901
- "use lm_ggml_map_custom3 instead");
1902
-
1903
- LM_GGML_DEPRECATED(LM_GGML_API struct lm_ggml_tensor * lm_ggml_map_custom3_inplace_f32(
1904
- struct lm_ggml_context * ctx,
1905
- struct lm_ggml_tensor * a,
1906
- struct lm_ggml_tensor * b,
1907
- struct lm_ggml_tensor * c,
1908
- lm_ggml_custom3_op_f32_t fun),
1909
- "use lm_ggml_map_custom3_inplace instead");
1910
-
1911
- // custom operators v2
1912
-
1913
- typedef void (*lm_ggml_custom1_op_t)(struct lm_ggml_tensor * dst , const struct lm_ggml_tensor * a, int ith, int nth, void * userdata);
1914
- typedef void (*lm_ggml_custom2_op_t)(struct lm_ggml_tensor * dst , const struct lm_ggml_tensor * a, const struct lm_ggml_tensor * b, int ith, int nth, void * userdata);
1915
- typedef void (*lm_ggml_custom3_op_t)(struct lm_ggml_tensor * dst , const struct lm_ggml_tensor * a, const struct lm_ggml_tensor * b, const struct lm_ggml_tensor * c, int ith, int nth, void * userdata);
1916
-
1917
- #define LM_GGML_N_TASKS_MAX -1
1918
-
1919
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_map_custom1(
1920
- struct lm_ggml_context * ctx,
1921
- struct lm_ggml_tensor * a,
1922
- lm_ggml_custom1_op_t fun,
1923
- int n_tasks,
1924
- void * userdata);
1925
-
1926
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_map_custom1_inplace(
1927
- struct lm_ggml_context * ctx,
1928
- struct lm_ggml_tensor * a,
1929
- lm_ggml_custom1_op_t fun,
1930
- int n_tasks,
1931
- void * userdata);
1932
-
1933
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_map_custom2(
1934
- struct lm_ggml_context * ctx,
1935
- struct lm_ggml_tensor * a,
1936
- struct lm_ggml_tensor * b,
1937
- lm_ggml_custom2_op_t fun,
1938
- int n_tasks,
1939
- void * userdata);
1940
-
1941
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_map_custom2_inplace(
1942
- struct lm_ggml_context * ctx,
1943
- struct lm_ggml_tensor * a,
1944
- struct lm_ggml_tensor * b,
1945
- lm_ggml_custom2_op_t fun,
1946
- int n_tasks,
1947
- void * userdata);
1948
-
1949
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_map_custom3(
1950
- struct lm_ggml_context * ctx,
1951
- struct lm_ggml_tensor * a,
1952
- struct lm_ggml_tensor * b,
1953
- struct lm_ggml_tensor * c,
1954
- lm_ggml_custom3_op_t fun,
1955
- int n_tasks,
1956
- void * userdata);
1957
-
1958
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_map_custom3_inplace(
1959
- struct lm_ggml_context * ctx,
1960
- struct lm_ggml_tensor * a,
1961
- struct lm_ggml_tensor * b,
1962
- struct lm_ggml_tensor * c,
1963
- lm_ggml_custom3_op_t fun,
1964
- int n_tasks,
1965
- void * userdata);
1966
-
1967
- // loss function
1968
-
1969
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_cross_entropy_loss(
1970
- struct lm_ggml_context * ctx,
1971
- struct lm_ggml_tensor * a,
1972
- struct lm_ggml_tensor * b);
1973
-
1974
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_cross_entropy_loss_back(
1975
- struct lm_ggml_context * ctx,
1976
- struct lm_ggml_tensor * a,
1977
- struct lm_ggml_tensor * b,
1978
- struct lm_ggml_tensor * c);
1979
-
1980
- //
1981
- // automatic differentiation
1982
- //
1983
-
1984
- LM_GGML_API void lm_ggml_set_param(
1985
- struct lm_ggml_context * ctx,
1986
- struct lm_ggml_tensor * tensor);
1987
-
1988
-
1989
- LM_GGML_API void lm_ggml_build_forward_expand (struct lm_ggml_cgraph * cgraph, struct lm_ggml_tensor * tensor);
1990
- LM_GGML_API void lm_ggml_build_backward_expand(struct lm_ggml_context * ctx, struct lm_ggml_cgraph * gf, struct lm_ggml_cgraph * gb, bool keep);
1991
-
1992
- // graph allocation in a context
1993
- LM_GGML_API struct lm_ggml_cgraph * lm_ggml_new_graph (struct lm_ggml_context * ctx); // size = LM_GGML_DEFAULT_GRAPH_SIZE, grads = false
1994
- LM_GGML_API struct lm_ggml_cgraph * lm_ggml_new_graph_custom (struct lm_ggml_context * ctx, size_t size, bool grads);
1995
- LM_GGML_API struct lm_ggml_cgraph * lm_ggml_graph_dup (struct lm_ggml_context * ctx, struct lm_ggml_cgraph * cgraph);
1996
- LM_GGML_API struct lm_ggml_cgraph lm_ggml_graph_view (struct lm_ggml_cgraph * cgraph, int i0, int i1);
1997
- LM_GGML_API void lm_ggml_graph_cpy (struct lm_ggml_cgraph * src, struct lm_ggml_cgraph * dst);
1998
- LM_GGML_API void lm_ggml_graph_reset (struct lm_ggml_cgraph * cgraph); // zero grads
1999
- LM_GGML_API void lm_ggml_graph_clear (struct lm_ggml_cgraph * cgraph);
2000
-
2001
- LM_GGML_API size_t lm_ggml_graph_overhead(void);
2002
- LM_GGML_API size_t lm_ggml_graph_overhead_custom(size_t size, bool grads);
2003
-
2004
- // lm_ggml_graph_plan() has to be called before lm_ggml_graph_compute()
2005
- // when plan.work_size > 0, caller must allocate memory for plan.work_data
2006
- LM_GGML_API struct lm_ggml_cplan lm_ggml_graph_plan (const struct lm_ggml_cgraph * cgraph, int n_threads /*= LM_GGML_DEFAULT_N_THREADS*/);
2007
- LM_GGML_API enum lm_ggml_status lm_ggml_graph_compute ( struct lm_ggml_cgraph * cgraph, struct lm_ggml_cplan * cplan);
2008
- // same as lm_ggml_graph_compute() but the work data is allocated as a part of the context
2009
- // note: the drawback of this API is that you must have ensured that the context has enough memory for the work data
2010
- LM_GGML_API enum lm_ggml_status lm_ggml_graph_compute_with_ctx(struct lm_ggml_context * ctx, struct lm_ggml_cgraph * cgraph, int n_threads);
2011
-
2012
- LM_GGML_API struct lm_ggml_tensor * lm_ggml_graph_get_tensor(struct lm_ggml_cgraph * cgraph, const char * name);
2013
-
2014
- LM_GGML_API void lm_ggml_graph_export(const struct lm_ggml_cgraph * cgraph, const char * fname);
2015
- LM_GGML_API struct lm_ggml_cgraph * lm_ggml_graph_import(const char * fname, struct lm_ggml_context ** ctx_data, struct lm_ggml_context ** ctx_eval);
2016
-
2017
- // print info and performance information for the graph
2018
- LM_GGML_API void lm_ggml_graph_print(const struct lm_ggml_cgraph * cgraph);
2019
-
2020
- // dump the graph into a file using the dot format
2021
- LM_GGML_API void lm_ggml_graph_dump_dot(const struct lm_ggml_cgraph * gb, const struct lm_ggml_cgraph * gf, const char * filename);
2022
-
2023
- // build gradient checkpointing backward graph gb for gf using provided checkpoints
2024
- // gb_tmp will contain original backward graph with rewritten backward process nodes,
2025
- // but without the second forward pass nodes.
2026
- LM_GGML_API void lm_ggml_build_backward_gradient_checkpointing(
2027
- struct lm_ggml_context * ctx,
2028
- struct lm_ggml_cgraph * gf,
2029
- struct lm_ggml_cgraph * gb,
2030
- struct lm_ggml_cgraph * gb_tmp,
2031
- struct lm_ggml_tensor * * checkpoints,
2032
- int n_checkpoints);
2033
- //
2034
- // optimization
2035
- //
2036
-
2037
- // optimization methods
2038
- enum lm_ggml_opt_type {
2039
- LM_GGML_OPT_TYPE_ADAM,
2040
- LM_GGML_OPT_TYPE_LBFGS,
2041
- };
2042
-
2043
- // linesearch methods
2044
- enum lm_ggml_linesearch {
2045
- LM_GGML_LINESEARCH_DEFAULT = 1,
2046
-
2047
- LM_GGML_LINESEARCH_BACKTRACKING_ARMIJO = 0,
2048
- LM_GGML_LINESEARCH_BACKTRACKING_WOLFE = 1,
2049
- LM_GGML_LINESEARCH_BACKTRACKING_STRONG_WOLFE = 2,
2050
- };
2051
-
2052
- // optimization return values
2053
- enum lm_ggml_opt_result {
2054
- LM_GGML_OPT_RESULT_OK = 0,
2055
- LM_GGML_OPT_RESULT_DID_NOT_CONVERGE,
2056
- LM_GGML_OPT_RESULT_NO_CONTEXT,
2057
- LM_GGML_OPT_RESULT_INVALID_WOLFE,
2058
- LM_GGML_OPT_RESULT_FAIL,
2059
- LM_GGML_OPT_RESULT_CANCEL,
2060
-
2061
- LM_GGML_LINESEARCH_FAIL = -128,
2062
- LM_GGML_LINESEARCH_MINIMUM_STEP,
2063
- LM_GGML_LINESEARCH_MAXIMUM_STEP,
2064
- LM_GGML_LINESEARCH_MAXIMUM_ITERATIONS,
2065
- LM_GGML_LINESEARCH_INVALID_PARAMETERS,
2066
- };
2067
-
2068
- typedef void (*lm_ggml_opt_callback)(void * data, int accum_step, float * sched, bool * cancel);
2069
- typedef void (*lm_ggml_log_callback)(enum lm_ggml_log_level level, const char * text, void * user_data);
2070
-
2071
- // optimization parameters
2072
- //
2073
- // see ggml.c (lm_ggml_opt_default_params) for default values
2074
- //
2075
- struct lm_ggml_opt_params {
2076
- enum lm_ggml_opt_type type;
2077
-
2078
- size_t graph_size;
2079
-
2080
- int n_threads;
2081
-
2082
- // delta-based convergence test
2083
- //
2084
- // if past == 0 - disabled
2085
- // if past > 0:
2086
- // stop if |f(x) - f(x_past)| < delta * max(1, |f(x)|)
2087
- //
2088
- int past;
2089
- float delta;
2090
-
2091
- // maximum number of iterations without improvement
2092
- //
2093
- // if 0 - disabled
2094
- // if > 0:
2095
- // assume convergence if no cost improvement in this number of iterations
2096
- //
2097
- int max_no_improvement;
2098
-
2099
- bool print_forward_graph;
2100
- bool print_backward_graph;
2101
-
2102
- int n_gradient_accumulation;
2103
-
2104
- // ADAM parameters
2105
- struct {
2106
- int n_iter;
2107
-
2108
- float sched; // schedule multiplier (fixed, decay or warmup)
2109
- float decay; // weight decay for AdamW, use 0.0f to disable
2110
- int decay_min_ndim; // minimum number of tensor dimension to apply weight decay
2111
- float alpha; // learning rate
2112
- float beta1;
2113
- float beta2;
2114
- float eps; // epsilon for numerical stability
2115
- float eps_f; // epsilon for convergence test
2116
- float eps_g; // epsilon for convergence test
2117
- float gclip; // gradient clipping
2118
- } adam;
2119
-
2120
- // LBFGS parameters
2121
- struct {
2122
- int m; // number of corrections to approximate the inv. Hessian
2123
- int n_iter;
2124
- int max_linesearch;
2125
-
2126
- float eps; // convergence tolerance
2127
- float ftol; // line search tolerance
2128
- float wolfe;
2129
- float min_step;
2130
- float max_step;
2131
-
2132
- enum lm_ggml_linesearch linesearch;
2133
- } lbfgs;
2134
- };
2135
-
2136
- struct lm_ggml_opt_context {
2137
- struct lm_ggml_context * ctx;
2138
- struct lm_ggml_opt_params params;
2139
-
2140
- int iter;
2141
- int64_t nx; // number of parameter elements
2142
-
2143
- bool just_initialized;
2144
-
2145
- float loss_before;
2146
- float loss_after;
2147
-
2148
- struct {
2149
- struct lm_ggml_tensor * g; // current gradient
2150
- struct lm_ggml_tensor * m; // first moment
2151
- struct lm_ggml_tensor * v; // second moment
2152
- struct lm_ggml_tensor * pf; // past function values
2153
- float fx_best;
2154
- float fx_prev;
2155
- int n_no_improvement;
2156
- } adam;
2157
-
2158
- struct {
2159
- struct lm_ggml_tensor * x; // current parameters
2160
- struct lm_ggml_tensor * xp; // previous parameters
2161
- struct lm_ggml_tensor * g; // current gradient
2162
- struct lm_ggml_tensor * gp; // previous gradient
2163
- struct lm_ggml_tensor * d; // search direction
2164
- struct lm_ggml_tensor * pf; // past function values
2165
- struct lm_ggml_tensor * lmal; // the L-BFGS memory alpha
2166
- struct lm_ggml_tensor * lmys; // the L-BFGS memory ys
2167
- struct lm_ggml_tensor * lms; // the L-BFGS memory s
2168
- struct lm_ggml_tensor * lmy; // the L-BFGS memory y
2169
- float fx_best;
2170
- float step;
2171
- int j;
2172
- int k;
2173
- int end;
2174
- int n_no_improvement;
2175
- } lbfgs;
2176
- };
2177
-
2178
- LM_GGML_API struct lm_ggml_opt_params lm_ggml_opt_default_params(enum lm_ggml_opt_type type);
2179
-
2180
- // optimize the function defined by the tensor f
2181
- LM_GGML_API enum lm_ggml_opt_result lm_ggml_opt(
2182
- struct lm_ggml_context * ctx,
2183
- struct lm_ggml_opt_params params,
2184
- struct lm_ggml_tensor * f);
2185
-
2186
- // initialize optimizer context
2187
- LM_GGML_API void lm_ggml_opt_init(
2188
- struct lm_ggml_context * ctx,
2189
- struct lm_ggml_opt_context * opt,
2190
- struct lm_ggml_opt_params params,
2191
- int64_t nx);
2192
-
2193
- // continue optimizing the function defined by the tensor f
2194
- LM_GGML_API enum lm_ggml_opt_result lm_ggml_opt_resume(
2195
- struct lm_ggml_context * ctx,
2196
- struct lm_ggml_opt_context * opt,
2197
- struct lm_ggml_tensor * f);
2198
-
2199
- // continue optimizing the function defined by the tensor f
2200
- LM_GGML_API enum lm_ggml_opt_result lm_ggml_opt_resume_g(
2201
- struct lm_ggml_context * ctx,
2202
- struct lm_ggml_opt_context * opt,
2203
- struct lm_ggml_tensor * f,
2204
- struct lm_ggml_cgraph * gf,
2205
- struct lm_ggml_cgraph * gb,
2206
- lm_ggml_opt_callback callback,
2207
- void * callback_data);
2208
-
2209
- //
2210
- // tensor flags
2211
- //
2212
- LM_GGML_API void lm_ggml_set_input(struct lm_ggml_tensor * tensor);
2213
- LM_GGML_API void lm_ggml_set_output(struct lm_ggml_tensor * tensor);
2214
-
2215
- //
2216
- // quantization
2217
- //
2218
-
2219
- // - lm_ggml_quantize_init can be called multiple times with the same type
2220
- // it will only initialize the quantization tables for the first call or after lm_ggml_quantize_free
2221
- // automatically called by lm_ggml_quantize_chunk for convenience
2222
- //
2223
- // - lm_ggml_quantize_free will free any memory allocated by lm_ggml_quantize_init
2224
- // call this at the end of the program to avoid memory leaks
2225
- //
2226
- // note: these are thread-safe
2227
- //
2228
- LM_GGML_API void lm_ggml_quantize_init(enum lm_ggml_type type);
2229
- LM_GGML_API void lm_ggml_quantize_free(void);
2230
-
2231
- // some quantization type cannot be used without an importance matrix
2232
- LM_GGML_API bool lm_ggml_quantize_requires_imatrix(enum lm_ggml_type type);
2233
-
2234
- // calls lm_ggml_quantize_init internally (i.e. can allocate memory)
2235
- LM_GGML_API size_t lm_ggml_quantize_chunk(
2236
- enum lm_ggml_type type,
2237
- const float * src,
2238
- void * dst,
2239
- int64_t start,
2240
- int64_t nrows,
2241
- int64_t n_per_row,
2242
- const float * imatrix);
2243
-
2244
- //
2245
- // gguf
2246
- //
2247
-
2248
- enum lm_gguf_type {
2249
- LM_GGUF_TYPE_UINT8 = 0,
2250
- LM_GGUF_TYPE_INT8 = 1,
2251
- LM_GGUF_TYPE_UINT16 = 2,
2252
- LM_GGUF_TYPE_INT16 = 3,
2253
- LM_GGUF_TYPE_UINT32 = 4,
2254
- LM_GGUF_TYPE_INT32 = 5,
2255
- LM_GGUF_TYPE_FLOAT32 = 6,
2256
- LM_GGUF_TYPE_BOOL = 7,
2257
- LM_GGUF_TYPE_STRING = 8,
2258
- LM_GGUF_TYPE_ARRAY = 9,
2259
- LM_GGUF_TYPE_UINT64 = 10,
2260
- LM_GGUF_TYPE_INT64 = 11,
2261
- LM_GGUF_TYPE_FLOAT64 = 12,
2262
- LM_GGUF_TYPE_COUNT, // marks the end of the enum
2263
- };
2264
-
2265
- struct lm_gguf_context;
2266
-
2267
- struct lm_gguf_init_params {
2268
- bool no_alloc;
2269
-
2270
- // if not NULL, create a lm_ggml_context and allocate the tensor data in it
2271
- struct lm_ggml_context ** ctx;
2272
- };
2273
-
2274
- LM_GGML_API struct lm_gguf_context * lm_gguf_init_empty(void);
2275
- LM_GGML_API struct lm_gguf_context * lm_gguf_init_from_file(const char * fname, struct lm_gguf_init_params params);
2276
- //LM_GGML_API struct lm_gguf_context * lm_gguf_init_from_buffer(..);
2277
-
2278
- LM_GGML_API void lm_gguf_free(struct lm_gguf_context * ctx);
2279
-
2280
- LM_GGML_API const char * lm_gguf_type_name(enum lm_gguf_type type);
2281
-
2282
- LM_GGML_API int lm_gguf_get_version (const struct lm_gguf_context * ctx);
2283
- LM_GGML_API size_t lm_gguf_get_alignment (const struct lm_gguf_context * ctx);
2284
- LM_GGML_API size_t lm_gguf_get_data_offset(const struct lm_gguf_context * ctx);
2285
- LM_GGML_API void * lm_gguf_get_data (const struct lm_gguf_context * ctx);
2286
-
2287
- LM_GGML_API int lm_gguf_get_n_kv(const struct lm_gguf_context * ctx);
2288
- LM_GGML_API int lm_gguf_find_key(const struct lm_gguf_context * ctx, const char * key);
2289
- LM_GGML_API const char * lm_gguf_get_key (const struct lm_gguf_context * ctx, int key_id);
2290
-
2291
- LM_GGML_API enum lm_gguf_type lm_gguf_get_kv_type (const struct lm_gguf_context * ctx, int key_id);
2292
- LM_GGML_API enum lm_gguf_type lm_gguf_get_arr_type(const struct lm_gguf_context * ctx, int key_id);
2293
-
2294
- // will abort if the wrong type is used for the key
2295
- LM_GGML_API uint8_t lm_gguf_get_val_u8 (const struct lm_gguf_context * ctx, int key_id);
2296
- LM_GGML_API int8_t lm_gguf_get_val_i8 (const struct lm_gguf_context * ctx, int key_id);
2297
- LM_GGML_API uint16_t lm_gguf_get_val_u16 (const struct lm_gguf_context * ctx, int key_id);
2298
- LM_GGML_API int16_t lm_gguf_get_val_i16 (const struct lm_gguf_context * ctx, int key_id);
2299
- LM_GGML_API uint32_t lm_gguf_get_val_u32 (const struct lm_gguf_context * ctx, int key_id);
2300
- LM_GGML_API int32_t lm_gguf_get_val_i32 (const struct lm_gguf_context * ctx, int key_id);
2301
- LM_GGML_API float lm_gguf_get_val_f32 (const struct lm_gguf_context * ctx, int key_id);
2302
- LM_GGML_API uint64_t lm_gguf_get_val_u64 (const struct lm_gguf_context * ctx, int key_id);
2303
- LM_GGML_API int64_t lm_gguf_get_val_i64 (const struct lm_gguf_context * ctx, int key_id);
2304
- LM_GGML_API double lm_gguf_get_val_f64 (const struct lm_gguf_context * ctx, int key_id);
2305
- LM_GGML_API bool lm_gguf_get_val_bool(const struct lm_gguf_context * ctx, int key_id);
2306
- LM_GGML_API const char * lm_gguf_get_val_str (const struct lm_gguf_context * ctx, int key_id);
2307
- LM_GGML_API const void * lm_gguf_get_val_data(const struct lm_gguf_context * ctx, int key_id);
2308
- LM_GGML_API int lm_gguf_get_arr_n (const struct lm_gguf_context * ctx, int key_id);
2309
- LM_GGML_API const void * lm_gguf_get_arr_data(const struct lm_gguf_context * ctx, int key_id);
2310
- LM_GGML_API const char * lm_gguf_get_arr_str (const struct lm_gguf_context * ctx, int key_id, int i);
2311
-
2312
- LM_GGML_API int lm_gguf_get_n_tensors (const struct lm_gguf_context * ctx);
2313
- LM_GGML_API int lm_gguf_find_tensor (const struct lm_gguf_context * ctx, const char * name);
2314
- LM_GGML_API size_t lm_gguf_get_tensor_offset(const struct lm_gguf_context * ctx, int i);
2315
- LM_GGML_API char * lm_gguf_get_tensor_name (const struct lm_gguf_context * ctx, int i);
2316
- LM_GGML_API enum lm_ggml_type lm_gguf_get_tensor_type (const struct lm_gguf_context * ctx, int i);
2317
-
2318
- // removes key if it exists
2319
- LM_GGML_API void lm_gguf_remove_key(struct lm_gguf_context * ctx, const char * key);
2320
-
2321
- // overrides existing values or adds a new one
2322
- LM_GGML_API void lm_gguf_set_val_u8 (struct lm_gguf_context * ctx, const char * key, uint8_t val);
2323
- LM_GGML_API void lm_gguf_set_val_i8 (struct lm_gguf_context * ctx, const char * key, int8_t val);
2324
- LM_GGML_API void lm_gguf_set_val_u16 (struct lm_gguf_context * ctx, const char * key, uint16_t val);
2325
- LM_GGML_API void lm_gguf_set_val_i16 (struct lm_gguf_context * ctx, const char * key, int16_t val);
2326
- LM_GGML_API void lm_gguf_set_val_u32 (struct lm_gguf_context * ctx, const char * key, uint32_t val);
2327
- LM_GGML_API void lm_gguf_set_val_i32 (struct lm_gguf_context * ctx, const char * key, int32_t val);
2328
- LM_GGML_API void lm_gguf_set_val_f32 (struct lm_gguf_context * ctx, const char * key, float val);
2329
- LM_GGML_API void lm_gguf_set_val_u64 (struct lm_gguf_context * ctx, const char * key, uint64_t val);
2330
- LM_GGML_API void lm_gguf_set_val_i64 (struct lm_gguf_context * ctx, const char * key, int64_t val);
2331
- LM_GGML_API void lm_gguf_set_val_f64 (struct lm_gguf_context * ctx, const char * key, double val);
2332
- LM_GGML_API void lm_gguf_set_val_bool(struct lm_gguf_context * ctx, const char * key, bool val);
2333
- LM_GGML_API void lm_gguf_set_val_str (struct lm_gguf_context * ctx, const char * key, const char * val);
2334
- LM_GGML_API void lm_gguf_set_arr_data(struct lm_gguf_context * ctx, const char * key, enum lm_gguf_type type, const void * data, int n);
2335
- LM_GGML_API void lm_gguf_set_arr_str (struct lm_gguf_context * ctx, const char * key, const char ** data, int n);
2336
-
2337
- // set or add KV pairs from another context
2338
- LM_GGML_API void lm_gguf_set_kv(struct lm_gguf_context * ctx, struct lm_gguf_context * src);
2339
-
2340
- // manage tensor info
2341
- LM_GGML_API void lm_gguf_add_tensor(struct lm_gguf_context * ctx, const struct lm_ggml_tensor * tensor);
2342
- LM_GGML_API void lm_gguf_set_tensor_type(struct lm_gguf_context * ctx, const char * name, enum lm_ggml_type type);
2343
- LM_GGML_API void lm_gguf_set_tensor_data(struct lm_gguf_context * ctx, const char * name, const void * data, size_t size);
2344
-
2345
- // writing gguf files can be done in 2 ways:
2346
- //
2347
- // - write the entire lm_gguf_context to a binary file in a single pass:
2348
- //
2349
- // lm_gguf_write_to_file(ctx, fname);
2350
- //
2351
- // - first prepare a file with a placeholder for the meta data, write the tensor data, then write the meta data:
2352
- //
2353
- // FILE * f = fopen(fname, "wb");
2354
- // fseek(f, lm_gguf_get_meta_size(ctx), SEEK_SET);
2355
- // fwrite(f, ...);
2356
- // void * data = lm_gguf_meta_get_meta_data(ctx);
2357
- // fseek(f, 0, SEEK_SET);
2358
- // fwrite(f, data, lm_gguf_get_meta_size(ctx));
2359
- // free(data);
2360
- // fclose(f);
2361
- //
2362
-
2363
- // write the entire context to a binary file
2364
- LM_GGML_API void lm_gguf_write_to_file(const struct lm_gguf_context * ctx, const char * fname, bool only_meta);
2365
-
2366
- // get the size in bytes of the meta data (header, kv pairs, tensor info) including padding
2367
- LM_GGML_API size_t lm_gguf_get_meta_size(const struct lm_gguf_context * ctx);
2368
- LM_GGML_API void lm_gguf_get_meta_data(const struct lm_gguf_context * ctx, void * data);
2369
-
2370
- //
2371
- // system info
2372
- //
2373
-
2374
- LM_GGML_API int lm_ggml_cpu_has_avx (void);
2375
- LM_GGML_API int lm_ggml_cpu_has_avx_vnni (void);
2376
- LM_GGML_API int lm_ggml_cpu_has_avx2 (void);
2377
- LM_GGML_API int lm_ggml_cpu_has_avx512 (void);
2378
- LM_GGML_API int lm_ggml_cpu_has_avx512_vbmi(void);
2379
- LM_GGML_API int lm_ggml_cpu_has_avx512_vnni(void);
2380
- LM_GGML_API int lm_ggml_cpu_has_avx512_bf16(void);
2381
- LM_GGML_API int lm_ggml_cpu_has_fma (void);
2382
- LM_GGML_API int lm_ggml_cpu_has_neon (void);
2383
- LM_GGML_API int lm_ggml_cpu_has_sve (void);
2384
- LM_GGML_API int lm_ggml_cpu_has_arm_fma (void);
2385
- LM_GGML_API int lm_ggml_cpu_has_metal (void);
2386
- LM_GGML_API int lm_ggml_cpu_has_f16c (void);
2387
- LM_GGML_API int lm_ggml_cpu_has_fp16_va (void);
2388
- LM_GGML_API int lm_ggml_cpu_has_wasm_simd (void);
2389
- LM_GGML_API int lm_ggml_cpu_has_blas (void);
2390
- LM_GGML_API int lm_ggml_cpu_has_cuda (void);
2391
- LM_GGML_API int lm_ggml_cpu_has_vulkan (void);
2392
- LM_GGML_API int lm_ggml_cpu_has_kompute (void);
2393
- LM_GGML_API int lm_ggml_cpu_has_gpublas (void);
2394
- LM_GGML_API int lm_ggml_cpu_has_sse3 (void);
2395
- LM_GGML_API int lm_ggml_cpu_has_ssse3 (void);
2396
- LM_GGML_API int lm_ggml_cpu_has_sycl (void);
2397
- LM_GGML_API int lm_ggml_cpu_has_rpc (void);
2398
- LM_GGML_API int lm_ggml_cpu_has_vsx (void);
2399
- LM_GGML_API int lm_ggml_cpu_has_matmul_int8(void);
2400
-
2401
- //
2402
- // Internal types and functions exposed for tests and benchmarks
2403
- //
2404
-
2405
- #ifdef __cplusplus
2406
- // restrict not standard in C++
2407
- #define LM_GGML_RESTRICT
2408
- #else
2409
- #define LM_GGML_RESTRICT restrict
2410
- #endif
2411
- typedef void (*lm_ggml_to_float_t) (const void * LM_GGML_RESTRICT x, float * LM_GGML_RESTRICT y, int64_t k);
2412
- typedef void (*lm_ggml_from_float_t)(const float * LM_GGML_RESTRICT x, void * LM_GGML_RESTRICT y, int64_t k);
2413
- typedef void (*lm_ggml_from_float_to_mat_t)
2414
- (const float * LM_GGML_RESTRICT x, void * LM_GGML_RESTRICT y, int64_t nr, int64_t k, int64_t bs);
2415
- typedef void (*lm_ggml_vec_dot_t) (int n, float * LM_GGML_RESTRICT s, size_t bs, const void * LM_GGML_RESTRICT x, size_t bx,
2416
- const void * LM_GGML_RESTRICT y, size_t by, int nrc);
2417
- typedef void (*lm_ggml_gemv_t) (int n, float * LM_GGML_RESTRICT s, size_t bs, const void * LM_GGML_RESTRICT x,
2418
- const void * LM_GGML_RESTRICT y, int nr, int nc);
2419
- typedef void (*lm_ggml_gemm_t) (int n, float * LM_GGML_RESTRICT s, size_t bs, const void * LM_GGML_RESTRICT x,
2420
- const void * LM_GGML_RESTRICT y, int nr, int nc);
2421
-
2422
- typedef struct {
2423
- const char * type_name;
2424
- int64_t blck_size;
2425
- int64_t blck_size_interleave; // interleave elements in blocks
2426
- size_t type_size;
2427
- bool is_quantized;
2428
- lm_ggml_to_float_t to_float;
2429
- lm_ggml_from_float_t from_float;
2430
- lm_ggml_from_float_t from_float_ref;
2431
- lm_ggml_from_float_to_mat_t from_float_to_mat;
2432
- lm_ggml_vec_dot_t vec_dot;
2433
- enum lm_ggml_type vec_dot_type;
2434
- int64_t nrows; // number of rows to process simultaneously
2435
- int64_t ncols; // number of columns to process simultaneously
2436
- lm_ggml_gemv_t gemv;
2437
- lm_ggml_gemm_t gemm;
2438
- } lm_ggml_type_traits_t;
2439
-
2440
- LM_GGML_API lm_ggml_type_traits_t lm_ggml_internal_get_type_traits(enum lm_ggml_type type);
2441
-
2442
- #ifdef __cplusplus
2443
- }
2444
- #endif
1
+ #pragma once
2
+
3
+ //
4
+ // GGML Tensor Library
5
+ //
6
+ // This documentation is still a work in progress.
7
+ // If you wish some specific topics to be covered, feel free to drop a comment:
8
+ //
9
+ // https://github.com/ggerganov/whisper.cpp/issues/40
10
+ //
11
+ // ## Overview
12
+ //
13
+ // This library implements:
14
+ //
15
+ // - a set of tensor operations
16
+ // - automatic differentiation
17
+ // - basic optimization algorithms
18
+ //
19
+ // The aim of this library is to provide a minimalistic approach for various machine learning tasks. This includes,
20
+ // but is not limited to, the following:
21
+ //
22
+ // - linear regression
23
+ // - support vector machines
24
+ // - neural networks
25
+ //
26
+ // The library allows the user to define a certain function using the available tensor operations. This function
27
+ // definition is represented internally via a computation graph. Each tensor operation in the function definition
28
+ // corresponds to a node in the graph. Having the computation graph defined, the user can choose to compute the
29
+ // function's value and/or its gradient with respect to the input variables. Optionally, the function can be optimized
30
+ // using one of the available optimization algorithms.
31
+ //
32
+ // For example, here we define the function: f(x) = a*x^2 + b
33
+ //
34
+ // {
35
+ // struct lm_ggml_init_params params = {
36
+ // .mem_size = 16*1024*1024,
37
+ // .mem_buffer = NULL,
38
+ // };
39
+ //
40
+ // // memory allocation happens here
41
+ // struct lm_ggml_context * ctx = lm_ggml_init(params);
42
+ //
43
+ // struct lm_ggml_tensor * x = lm_ggml_new_tensor_1d(ctx, LM_GGML_TYPE_F32, 1);
44
+ //
45
+ // lm_ggml_set_param(ctx, x); // x is an input variable
46
+ //
47
+ // struct lm_ggml_tensor * a = lm_ggml_new_tensor_1d(ctx, LM_GGML_TYPE_F32, 1);
48
+ // struct lm_ggml_tensor * b = lm_ggml_new_tensor_1d(ctx, LM_GGML_TYPE_F32, 1);
49
+ // struct lm_ggml_tensor * x2 = lm_ggml_mul(ctx, x, x);
50
+ // struct lm_ggml_tensor * f = lm_ggml_add(ctx, lm_ggml_mul(ctx, a, x2), b);
51
+ //
52
+ // ...
53
+ // }
54
+ //
55
+ // Notice that the function definition above does not involve any actual computation. The computation is performed only
56
+ // when the user explicitly requests it. For example, to compute the function's value at x = 2.0:
57
+ //
58
+ // {
59
+ // ...
60
+ //
61
+ // struct lm_ggml_cgraph * gf = lm_ggml_new_graph(ctx);
62
+ // lm_ggml_build_forward_expand(gf, f);
63
+ //
64
+ // // set the input variable and parameter values
65
+ // lm_ggml_set_f32(x, 2.0f);
66
+ // lm_ggml_set_f32(a, 3.0f);
67
+ // lm_ggml_set_f32(b, 4.0f);
68
+ //
69
+ // lm_ggml_graph_compute_with_ctx(ctx, &gf, n_threads);
70
+ //
71
+ // printf("f = %f\n", lm_ggml_get_f32_1d(f, 0));
72
+ //
73
+ // ...
74
+ // }
75
+ //
76
+ // The actual computation is performed in the lm_ggml_graph_compute() function.
77
+ //
78
+ // The lm_ggml_new_tensor_...() functions create new tensors. They are allocated in the memory buffer provided to the
79
+ // lm_ggml_init() function. You have to be careful not to exceed the memory buffer size. Therefore, you have to know
80
+ // in advance how much memory you need for your computation. Alternatively, you can allocate a large enough memory
81
+ // and after defining the computation graph, call the lm_ggml_used_mem() function to find out how much memory was
82
+ // actually needed.
83
+ //
84
+ // The lm_ggml_set_param() function marks a tensor as an input variable. This is used by the automatic
85
+ // differentiation and optimization algorithms.
86
+ //
87
+ // The described approach allows to define the function graph once and then compute its forward or backward graphs
88
+ // multiple times. All computations will use the same memory buffer allocated in the lm_ggml_init() function. This way
89
+ // the user can avoid the memory allocation overhead at runtime.
90
+ //
91
+ // The library supports multi-dimensional tensors - up to 4 dimensions. The FP16 and FP32 data types are first class
92
+ // citizens, but in theory the library can be extended to support FP8 and integer data types.
93
+ //
94
+ // Each tensor operation produces a new tensor. Initially the library was envisioned to support only the use of unary
95
+ // and binary operations. Most of the available operations fall into one of these two categories. With time, it became
96
+ // clear that the library needs to support more complex operations. The way to support these operations is not clear
97
+ // yet, but a few examples are demonstrated in the following operations:
98
+ //
99
+ // - lm_ggml_permute()
100
+ // - lm_ggml_conv_1d_1s()
101
+ // - lm_ggml_conv_1d_2s()
102
+ //
103
+ // For each tensor operator, the library implements a forward and backward computation function. The forward function
104
+ // computes the output tensor value given the input tensor values. The backward function computes the adjoint of the
105
+ // input tensors given the adjoint of the output tensor. For a detailed explanation of what this means, take a
106
+ // calculus class, or watch the following video:
107
+ //
108
+ // What is Automatic Differentiation?
109
+ // https://www.youtube.com/watch?v=wG_nF1awSSY
110
+ //
111
+ //
112
+ // ## Tensor data (struct lm_ggml_tensor)
113
+ //
114
+ // The tensors are stored in memory via the lm_ggml_tensor struct. The structure provides information about the size of
115
+ // the tensor, the data type, and the memory buffer where the tensor data is stored. Additionally, it contains
116
+ // pointers to the "source" tensors - i.e. the tensors that were used to compute the current tensor. For example:
117
+ //
118
+ // {
119
+ // struct lm_ggml_tensor * c = lm_ggml_add(ctx, a, b);
120
+ //
121
+ // assert(c->src[0] == a);
122
+ // assert(c->src[1] == b);
123
+ // }
124
+ //
125
+ // The multi-dimensional tensors are stored in row-major order. The lm_ggml_tensor struct contains fields for the
126
+ // number of elements in each dimension ("ne") as well as the number of bytes ("nb", a.k.a. stride). This allows
127
+ // to store tensors that are not contiguous in memory, which is useful for operations such as transposition and
128
+ // permutation. All tensor operations have to take the stride into account and not assume that the tensor is
129
+ // contiguous in memory.
130
+ //
131
+ // The data of the tensor is accessed via the "data" pointer. For example:
132
+ //
133
+ // {
134
+ // const int nx = 2;
135
+ // const int ny = 3;
136
+ //
137
+ // struct lm_ggml_tensor * a = lm_ggml_new_tensor_2d(ctx, LM_GGML_TYPE_F32, nx, ny);
138
+ //
139
+ // for (int y = 0; y < ny; y++) {
140
+ // for (int x = 0; x < nx; x++) {
141
+ // *(float *) ((char *) a->data + y*a->nb[1] + x*a->nb[0]) = x + y;
142
+ // }
143
+ // }
144
+ //
145
+ // ...
146
+ // }
147
+ //
148
+ // Alternatively, there are helper functions, such as lm_ggml_get_f32_1d() and lm_ggml_set_f32_1d() that can be used.
149
+ //
150
+ // ## The matrix multiplication operator (lm_ggml_mul_mat)
151
+ //
152
+ // TODO
153
+ //
154
+ //
155
+ // ## Multi-threading
156
+ //
157
+ // TODO
158
+ //
159
+ //
160
+ // ## Overview of ggml.c
161
+ //
162
+ // TODO
163
+ //
164
+ //
165
+ // ## SIMD optimizations
166
+ //
167
+ // TODO
168
+ //
169
+ //
170
+ // ## Debugging ggml
171
+ //
172
+ // TODO
173
+ //
174
+ //
175
+
176
+ #ifdef LM_GGML_SHARED
177
+ # if defined(_WIN32) && !defined(__MINGW32__)
178
+ # ifdef LM_GGML_BUILD
179
+ # define LM_GGML_API __declspec(dllexport)
180
+ # else
181
+ # define LM_GGML_API __declspec(dllimport)
182
+ # endif
183
+ # else
184
+ # define LM_GGML_API __attribute__ ((visibility ("default")))
185
+ # endif
186
+ #else
187
+ # define LM_GGML_API
188
+ #endif
189
+
190
+ #ifdef LM_GGML_MULTIPLATFORM
191
+ # if defined(_WIN32)
192
+ # define LM_GGML_CALL
193
+ # else
194
+ # define LM_GGML_CALL __attribute__((__ms_abi__))
195
+ # endif
196
+ #else
197
+ # define LM_GGML_CALL
198
+ #endif
199
+
200
+ // TODO: support for clang
201
+ #ifdef __GNUC__
202
+ # define LM_GGML_DEPRECATED(func, hint) func __attribute__((deprecated(hint)))
203
+ #elif defined(_MSC_VER)
204
+ # define LM_GGML_DEPRECATED(func, hint) __declspec(deprecated(hint)) func
205
+ #else
206
+ # define LM_GGML_DEPRECATED(func, hint) func
207
+ #endif
208
+
209
+ #ifndef __GNUC__
210
+ # define LM_GGML_ATTRIBUTE_FORMAT(...)
211
+ #elif defined(__MINGW32__)
212
+ # define LM_GGML_ATTRIBUTE_FORMAT(...) __attribute__((format(gnu_printf, __VA_ARGS__)))
213
+ #else
214
+ # define LM_GGML_ATTRIBUTE_FORMAT(...) __attribute__((format(printf, __VA_ARGS__)))
215
+ #endif
216
+
217
+ #include <stdbool.h>
218
+ #include <stddef.h>
219
+ #include <stdint.h>
220
+ #include <stdio.h>
221
+
222
+ #define LM_GGML_FILE_MAGIC 0x67676d6c // "ggml"
223
+ #define LM_GGML_FILE_VERSION 1
224
+
225
+ #define LM_GGML_QNT_VERSION 2 // bump this on quantization format changes
226
+ #define LM_GGML_QNT_VERSION_FACTOR 1000 // do not change this
227
+
228
+ #define LM_GGML_MAX_DIMS 4
229
+ #define LM_GGML_MAX_PARAMS 2048
230
+ #define LM_GGML_MAX_CONTEXTS 64
231
+ #define LM_GGML_MAX_SRC 10
232
+ #ifndef LM_GGML_MAX_NAME
233
+ #define LM_GGML_MAX_NAME 64
234
+ #endif
235
+ #define LM_GGML_MAX_OP_PARAMS 64
236
+ #define LM_GGML_DEFAULT_N_THREADS 4
237
+ #define LM_GGML_DEFAULT_GRAPH_SIZE 2048
238
+ #if UINTPTR_MAX == 0xFFFFFFFF
239
+ #define LM_GGML_MEM_ALIGN 4
240
+ #else
241
+ #define LM_GGML_MEM_ALIGN 16
242
+ #endif
243
+
244
+ #define LM_GGML_EXIT_SUCCESS 0
245
+ #define LM_GGML_EXIT_ABORTED 1
246
+
247
+ #define LM_GGUF_MAGIC "GGUF"
248
+
249
+ #define LM_GGUF_VERSION 3
250
+
251
+ #define LM_GGUF_DEFAULT_ALIGNMENT 32
252
+
253
+ #define LM_GGML_UNUSED(x) (void)(x)
254
+
255
+ #define LM_GGML_PAD(x, n) (((x) + (n) - 1) & ~((n) - 1))
256
+
257
+ #define LM_GGML_ASSERT(x) \
258
+ do { \
259
+ if (!(x)) { \
260
+ fflush(stdout); \
261
+ fprintf(stderr, "LM_GGML_ASSERT: %s:%d: %s\n", __FILE__, __LINE__, #x); \
262
+ lm_ggml_print_backtrace(); \
263
+ abort(); \
264
+ } \
265
+ } while (0)
266
+
267
+ #ifndef NDEBUG
268
+ #define LM_GGML_UNREACHABLE() LM_GGML_ASSERT(!"statement should not be reached")
269
+ #elif defined(__GNUC__)
270
+ #define LM_GGML_UNREACHABLE() __builtin_unreachable()
271
+ #elif defined(_MSC_VER)
272
+ #define LM_GGML_UNREACHABLE() __assume(0)
273
+ #else
274
+ #define LM_GGML_UNREACHABLE() ((void) 0)
275
+ #endif
276
+
277
+ // used to copy the number of elements and stride in bytes of tensors into local variables.
278
+ // main purpose is to reduce code duplication and improve readability.
279
+ //
280
+ // example:
281
+ //
282
+ // LM_GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne);
283
+ // LM_GGML_TENSOR_LOCALS(size_t, nb1, src1, nb);
284
+ //
285
+ #define LM_GGML_TENSOR_LOCALS_1(type, prefix, pointer, array) \
286
+ const type prefix##0 = (pointer)->array[0]; \
287
+ LM_GGML_UNUSED(prefix##0);
288
+ #define LM_GGML_TENSOR_LOCALS_2(type, prefix, pointer, array) \
289
+ LM_GGML_TENSOR_LOCALS_1 (type, prefix, pointer, array) \
290
+ const type prefix##1 = (pointer)->array[1]; \
291
+ LM_GGML_UNUSED(prefix##1);
292
+ #define LM_GGML_TENSOR_LOCALS_3(type, prefix, pointer, array) \
293
+ LM_GGML_TENSOR_LOCALS_2 (type, prefix, pointer, array) \
294
+ const type prefix##2 = (pointer)->array[2]; \
295
+ LM_GGML_UNUSED(prefix##2);
296
+ #define LM_GGML_TENSOR_LOCALS(type, prefix, pointer, array) \
297
+ LM_GGML_TENSOR_LOCALS_3 (type, prefix, pointer, array) \
298
+ const type prefix##3 = (pointer)->array[3]; \
299
+ LM_GGML_UNUSED(prefix##3);
300
+
301
+ #define LM_GGML_TENSOR_UNARY_OP_LOCALS \
302
+ LM_GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) \
303
+ LM_GGML_TENSOR_LOCALS(size_t, nb0, src0, nb) \
304
+ LM_GGML_TENSOR_LOCALS(int64_t, ne, dst, ne) \
305
+ LM_GGML_TENSOR_LOCALS(size_t, nb, dst, nb)
306
+
307
+ #define LM_GGML_TENSOR_BINARY_OP_LOCALS \
308
+ LM_GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) \
309
+ LM_GGML_TENSOR_LOCALS(size_t, nb0, src0, nb) \
310
+ LM_GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne) \
311
+ LM_GGML_TENSOR_LOCALS(size_t, nb1, src1, nb) \
312
+ LM_GGML_TENSOR_LOCALS(int64_t, ne, dst, ne) \
313
+ LM_GGML_TENSOR_LOCALS(size_t, nb, dst, nb)
314
+
315
+ #define LM_GGML_TENSOR_BINARY_OP_LOCALS01 \
316
+ LM_GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) \
317
+ LM_GGML_TENSOR_LOCALS(size_t, nb0, src0, nb) \
318
+ LM_GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne) \
319
+ LM_GGML_TENSOR_LOCALS(size_t, nb1, src1, nb)
320
+
321
+ #ifdef __cplusplus
322
+ extern "C" {
323
+ #endif
324
+
325
+ enum lm_ggml_status {
326
+ LM_GGML_STATUS_ALLOC_FAILED = -2,
327
+ LM_GGML_STATUS_FAILED = -1,
328
+ LM_GGML_STATUS_SUCCESS = 0,
329
+ LM_GGML_STATUS_ABORTED = 1,
330
+ };
331
+
332
+ // get lm_ggml_status name string
333
+ LM_GGML_API LM_GGML_CALL const char * lm_ggml_status_to_string(enum lm_ggml_status status);
334
+
335
+ // ieee 754-2008 half-precision float16
336
+ // todo: make this not an integral type
337
+ typedef uint16_t lm_ggml_fp16_t;
338
+ LM_GGML_API float lm_ggml_fp16_to_fp32(lm_ggml_fp16_t);
339
+ LM_GGML_API lm_ggml_fp16_t lm_ggml_fp32_to_fp16(float);
340
+ LM_GGML_API void lm_ggml_fp16_to_fp32_row(const lm_ggml_fp16_t *, float *, int64_t);
341
+ LM_GGML_API void lm_ggml_fp32_to_fp16_row(const float *, lm_ggml_fp16_t *, int64_t);
342
+
343
+ // google brain half-precision bfloat16
344
+ typedef struct { uint16_t bits; } lm_ggml_bf16_t;
345
+ LM_GGML_API lm_ggml_bf16_t lm_ggml_fp32_to_bf16(float);
346
+ LM_GGML_API float lm_ggml_bf16_to_fp32(lm_ggml_bf16_t); // consider just doing << 16
347
+ LM_GGML_API void lm_ggml_bf16_to_fp32_row(const lm_ggml_bf16_t *, float *, int64_t);
348
+ LM_GGML_API void lm_ggml_fp32_to_bf16_row(const float *, lm_ggml_bf16_t *, int64_t);
349
+
350
+ struct lm_ggml_object;
351
+ struct lm_ggml_context;
352
+
353
+ // NOTE: always add types at the end of the enum to keep backward compatibility
354
+ enum lm_ggml_type {
355
+ LM_GGML_TYPE_F32 = 0,
356
+ LM_GGML_TYPE_F16 = 1,
357
+ LM_GGML_TYPE_Q4_0 = 2,
358
+ LM_GGML_TYPE_Q4_1 = 3,
359
+ // LM_GGML_TYPE_Q4_2 = 4, support has been removed
360
+ // LM_GGML_TYPE_Q4_3 = 5, support has been removed
361
+ LM_GGML_TYPE_Q5_0 = 6,
362
+ LM_GGML_TYPE_Q5_1 = 7,
363
+ LM_GGML_TYPE_Q8_0 = 8,
364
+ LM_GGML_TYPE_Q8_1 = 9,
365
+ LM_GGML_TYPE_Q2_K = 10,
366
+ LM_GGML_TYPE_Q3_K = 11,
367
+ LM_GGML_TYPE_Q4_K = 12,
368
+ LM_GGML_TYPE_Q5_K = 13,
369
+ LM_GGML_TYPE_Q6_K = 14,
370
+ LM_GGML_TYPE_Q8_K = 15,
371
+ LM_GGML_TYPE_IQ2_XXS = 16,
372
+ LM_GGML_TYPE_IQ2_XS = 17,
373
+ LM_GGML_TYPE_IQ3_XXS = 18,
374
+ LM_GGML_TYPE_IQ1_S = 19,
375
+ LM_GGML_TYPE_IQ4_NL = 20,
376
+ LM_GGML_TYPE_IQ3_S = 21,
377
+ LM_GGML_TYPE_IQ2_S = 22,
378
+ LM_GGML_TYPE_IQ4_XS = 23,
379
+ LM_GGML_TYPE_I8 = 24,
380
+ LM_GGML_TYPE_I16 = 25,
381
+ LM_GGML_TYPE_I32 = 26,
382
+ LM_GGML_TYPE_I64 = 27,
383
+ LM_GGML_TYPE_F64 = 28,
384
+ LM_GGML_TYPE_IQ1_M = 29,
385
+ LM_GGML_TYPE_BF16 = 30,
386
+ LM_GGML_TYPE_Q4_0_4_4 = 31,
387
+ LM_GGML_TYPE_Q4_0_4_8 = 32,
388
+ LM_GGML_TYPE_Q4_0_8_8 = 33,
389
+ LM_GGML_TYPE_COUNT,
390
+ };
391
+
392
+ // precision
393
+ enum lm_ggml_prec {
394
+ LM_GGML_PREC_DEFAULT,
395
+ LM_GGML_PREC_F32,
396
+ };
397
+
398
+ enum lm_ggml_backend_type {
399
+ LM_GGML_BACKEND_TYPE_CPU = 0,
400
+ LM_GGML_BACKEND_TYPE_GPU = 10,
401
+ LM_GGML_BACKEND_TYPE_GPU_SPLIT = 20,
402
+ };
403
+
404
+ // model file types
405
+ enum lm_ggml_ftype {
406
+ LM_GGML_FTYPE_UNKNOWN = -1,
407
+ LM_GGML_FTYPE_ALL_F32 = 0,
408
+ LM_GGML_FTYPE_MOSTLY_F16 = 1, // except 1d tensors
409
+ LM_GGML_FTYPE_MOSTLY_Q4_0 = 2, // except 1d tensors
410
+ LM_GGML_FTYPE_MOSTLY_Q4_1 = 3, // except 1d tensors
411
+ LM_GGML_FTYPE_MOSTLY_Q4_1_SOME_F16 = 4, // tok_embeddings.weight and output.weight are F16
412
+ LM_GGML_FTYPE_MOSTLY_Q8_0 = 7, // except 1d tensors
413
+ LM_GGML_FTYPE_MOSTLY_Q5_0 = 8, // except 1d tensors
414
+ LM_GGML_FTYPE_MOSTLY_Q5_1 = 9, // except 1d tensors
415
+ LM_GGML_FTYPE_MOSTLY_Q2_K = 10, // except 1d tensors
416
+ LM_GGML_FTYPE_MOSTLY_Q3_K = 11, // except 1d tensors
417
+ LM_GGML_FTYPE_MOSTLY_Q4_K = 12, // except 1d tensors
418
+ LM_GGML_FTYPE_MOSTLY_Q5_K = 13, // except 1d tensors
419
+ LM_GGML_FTYPE_MOSTLY_Q6_K = 14, // except 1d tensors
420
+ LM_GGML_FTYPE_MOSTLY_IQ2_XXS = 15, // except 1d tensors
421
+ LM_GGML_FTYPE_MOSTLY_IQ2_XS = 16, // except 1d tensors
422
+ LM_GGML_FTYPE_MOSTLY_IQ3_XXS = 17, // except 1d tensors
423
+ LM_GGML_FTYPE_MOSTLY_IQ1_S = 18, // except 1d tensors
424
+ LM_GGML_FTYPE_MOSTLY_IQ4_NL = 19, // except 1d tensors
425
+ LM_GGML_FTYPE_MOSTLY_IQ3_S = 20, // except 1d tensors
426
+ LM_GGML_FTYPE_MOSTLY_IQ2_S = 21, // except 1d tensors
427
+ LM_GGML_FTYPE_MOSTLY_IQ4_XS = 22, // except 1d tensors
428
+ LM_GGML_FTYPE_MOSTLY_IQ1_M = 23, // except 1d tensors
429
+ LM_GGML_FTYPE_MOSTLY_BF16 = 24, // except 1d tensors
430
+ LM_GGML_FTYPE_MOSTLY_Q4_0_4_4 = 25, // except 1d tensors
431
+ LM_GGML_FTYPE_MOSTLY_Q4_0_4_8 = 26, // except 1d tensors
432
+ LM_GGML_FTYPE_MOSTLY_Q4_0_8_8 = 27, // except 1d tensors
433
+ };
434
+
435
+ // available tensor operations:
436
+ enum lm_ggml_op {
437
+ LM_GGML_OP_NONE = 0,
438
+
439
+ LM_GGML_OP_DUP,
440
+ LM_GGML_OP_ADD,
441
+ LM_GGML_OP_ADD1,
442
+ LM_GGML_OP_ACC,
443
+ LM_GGML_OP_SUB,
444
+ LM_GGML_OP_MUL,
445
+ LM_GGML_OP_DIV,
446
+ LM_GGML_OP_SQR,
447
+ LM_GGML_OP_SQRT,
448
+ LM_GGML_OP_LOG,
449
+ LM_GGML_OP_SUM,
450
+ LM_GGML_OP_SUM_ROWS,
451
+ LM_GGML_OP_MEAN,
452
+ LM_GGML_OP_ARGMAX,
453
+ LM_GGML_OP_REPEAT,
454
+ LM_GGML_OP_REPEAT_BACK,
455
+ LM_GGML_OP_CONCAT,
456
+ LM_GGML_OP_SILU_BACK,
457
+ LM_GGML_OP_NORM, // normalize
458
+ LM_GGML_OP_RMS_NORM,
459
+ LM_GGML_OP_RMS_NORM_BACK,
460
+ LM_GGML_OP_GROUP_NORM,
461
+
462
+ LM_GGML_OP_MUL_MAT,
463
+ LM_GGML_OP_MUL_MAT_ID,
464
+ LM_GGML_OP_OUT_PROD,
465
+
466
+ LM_GGML_OP_SCALE,
467
+ LM_GGML_OP_SET,
468
+ LM_GGML_OP_CPY,
469
+ LM_GGML_OP_CONT,
470
+ LM_GGML_OP_RESHAPE,
471
+ LM_GGML_OP_VIEW,
472
+ LM_GGML_OP_PERMUTE,
473
+ LM_GGML_OP_TRANSPOSE,
474
+ LM_GGML_OP_GET_ROWS,
475
+ LM_GGML_OP_GET_ROWS_BACK,
476
+ LM_GGML_OP_DIAG,
477
+ LM_GGML_OP_DIAG_MASK_INF,
478
+ LM_GGML_OP_DIAG_MASK_ZERO,
479
+ LM_GGML_OP_SOFT_MAX,
480
+ LM_GGML_OP_SOFT_MAX_BACK,
481
+ LM_GGML_OP_ROPE,
482
+ LM_GGML_OP_ROPE_BACK,
483
+ LM_GGML_OP_CLAMP,
484
+ LM_GGML_OP_CONV_TRANSPOSE_1D,
485
+ LM_GGML_OP_IM2COL,
486
+ LM_GGML_OP_CONV_TRANSPOSE_2D,
487
+ LM_GGML_OP_POOL_1D,
488
+ LM_GGML_OP_POOL_2D,
489
+ LM_GGML_OP_UPSCALE, // nearest interpolate
490
+ LM_GGML_OP_PAD,
491
+ LM_GGML_OP_ARANGE,
492
+ LM_GGML_OP_TIMESTEP_EMBEDDING,
493
+ LM_GGML_OP_ARGSORT,
494
+ LM_GGML_OP_LEAKY_RELU,
495
+
496
+ LM_GGML_OP_FLASH_ATTN_EXT,
497
+ LM_GGML_OP_FLASH_ATTN_BACK,
498
+ LM_GGML_OP_SSM_CONV,
499
+ LM_GGML_OP_SSM_SCAN,
500
+ LM_GGML_OP_WIN_PART,
501
+ LM_GGML_OP_WIN_UNPART,
502
+ LM_GGML_OP_GET_REL_POS,
503
+ LM_GGML_OP_ADD_REL_POS,
504
+
505
+ LM_GGML_OP_UNARY,
506
+
507
+ LM_GGML_OP_MAP_UNARY,
508
+ LM_GGML_OP_MAP_BINARY,
509
+
510
+ LM_GGML_OP_MAP_CUSTOM1_F32,
511
+ LM_GGML_OP_MAP_CUSTOM2_F32,
512
+ LM_GGML_OP_MAP_CUSTOM3_F32,
513
+
514
+ LM_GGML_OP_MAP_CUSTOM1,
515
+ LM_GGML_OP_MAP_CUSTOM2,
516
+ LM_GGML_OP_MAP_CUSTOM3,
517
+
518
+ LM_GGML_OP_CROSS_ENTROPY_LOSS,
519
+ LM_GGML_OP_CROSS_ENTROPY_LOSS_BACK,
520
+
521
+ LM_GGML_OP_COUNT,
522
+ };
523
+
524
+ enum lm_ggml_unary_op {
525
+ LM_GGML_UNARY_OP_ABS,
526
+ LM_GGML_UNARY_OP_SGN,
527
+ LM_GGML_UNARY_OP_NEG,
528
+ LM_GGML_UNARY_OP_STEP,
529
+ LM_GGML_UNARY_OP_TANH,
530
+ LM_GGML_UNARY_OP_ELU,
531
+ LM_GGML_UNARY_OP_RELU,
532
+ LM_GGML_UNARY_OP_SIGMOID,
533
+ LM_GGML_UNARY_OP_GELU,
534
+ LM_GGML_UNARY_OP_GELU_QUICK,
535
+ LM_GGML_UNARY_OP_SILU,
536
+ LM_GGML_UNARY_OP_HARDSWISH,
537
+ LM_GGML_UNARY_OP_HARDSIGMOID,
538
+
539
+ LM_GGML_UNARY_OP_COUNT,
540
+ };
541
+
542
+ enum lm_ggml_object_type {
543
+ LM_GGML_OBJECT_TYPE_TENSOR,
544
+ LM_GGML_OBJECT_TYPE_GRAPH,
545
+ LM_GGML_OBJECT_TYPE_WORK_BUFFER
546
+ };
547
+
548
+ enum lm_ggml_log_level {
549
+ LM_GGML_LOG_LEVEL_ERROR = 2,
550
+ LM_GGML_LOG_LEVEL_WARN = 3,
551
+ LM_GGML_LOG_LEVEL_INFO = 4,
552
+ LM_GGML_LOG_LEVEL_DEBUG = 5
553
+ };
554
+
555
+ enum lm_ggml_tensor_flag {
556
+ LM_GGML_TENSOR_FLAG_INPUT = 1,
557
+ LM_GGML_TENSOR_FLAG_OUTPUT = 2,
558
+ LM_GGML_TENSOR_FLAG_PARAM = 4,
559
+ };
560
+
561
+ // ggml object
562
+ struct lm_ggml_object {
563
+ size_t offs;
564
+ size_t size;
565
+
566
+ struct lm_ggml_object * next;
567
+
568
+ enum lm_ggml_object_type type;
569
+
570
+ char padding[4];
571
+ };
572
+
573
+ static const size_t LM_GGML_OBJECT_SIZE = sizeof(struct lm_ggml_object);
574
+
575
+ // n-dimensional tensor
576
+ struct lm_ggml_tensor {
577
+ enum lm_ggml_type type;
578
+
579
+ LM_GGML_DEPRECATED(enum lm_ggml_backend_type backend, "use the buffer type to find the storage location of the tensor");
580
+
581
+ struct lm_ggml_backend_buffer * buffer;
582
+
583
+ int64_t ne[LM_GGML_MAX_DIMS]; // number of elements
584
+ size_t nb[LM_GGML_MAX_DIMS]; // stride in bytes:
585
+ // nb[0] = lm_ggml_type_size(type)
586
+ // nb[1] = nb[0] * (ne[0] / lm_ggml_blck_size(type)) + padding
587
+ // nb[i] = nb[i-1] * ne[i-1]
588
+
589
+ // compute data
590
+ enum lm_ggml_op op;
591
+
592
+ // op params - allocated as int32_t for alignment
593
+ int32_t op_params[LM_GGML_MAX_OP_PARAMS / sizeof(int32_t)];
594
+
595
+ int32_t flags;
596
+
597
+ struct lm_ggml_tensor * grad;
598
+ struct lm_ggml_tensor * src[LM_GGML_MAX_SRC];
599
+
600
+ // source tensor and offset for views
601
+ struct lm_ggml_tensor * view_src;
602
+ size_t view_offs;
603
+
604
+ void * data;
605
+
606
+ char name[LM_GGML_MAX_NAME];
607
+
608
+ void * extra; // extra things e.g. for ggml-cuda.cu
609
+
610
+ // char padding[4];
611
+ };
612
+
613
+ static const size_t LM_GGML_TENSOR_SIZE = sizeof(struct lm_ggml_tensor);
614
+
615
+ // Abort callback
616
+ // If not NULL, called before ggml computation
617
+ // If it returns true, the computation is aborted
618
+ typedef bool (*lm_ggml_abort_callback)(void * data);
619
+
620
+ // the compute plan that needs to be prepared for lm_ggml_graph_compute()
621
+ // since https://github.com/ggerganov/ggml/issues/287
622
+ struct lm_ggml_cplan {
623
+ size_t work_size; // size of work buffer, calculated by `lm_ggml_graph_plan()`
624
+ uint8_t * work_data; // work buffer, to be allocated by caller before calling to `lm_ggml_graph_compute()`
625
+
626
+ int n_threads;
627
+
628
+ // abort lm_ggml_graph_compute when true
629
+ lm_ggml_abort_callback abort_callback;
630
+ void * abort_callback_data;
631
+ };
632
+
633
+ enum lm_ggml_cgraph_eval_order {
634
+ LM_GGML_CGRAPH_EVAL_ORDER_LEFT_TO_RIGHT = 0,
635
+ LM_GGML_CGRAPH_EVAL_ORDER_RIGHT_TO_LEFT,
636
+ LM_GGML_CGRAPH_EVAL_ORDER_COUNT
637
+ };
638
+
639
+ struct lm_ggml_hash_set {
640
+ size_t size;
641
+ struct lm_ggml_tensor ** keys;
642
+ };
643
+
644
+ // computation graph
645
+ struct lm_ggml_cgraph {
646
+ int size;
647
+ int n_nodes;
648
+ int n_leafs;
649
+
650
+ struct lm_ggml_tensor ** nodes;
651
+ struct lm_ggml_tensor ** grads;
652
+ struct lm_ggml_tensor ** leafs;
653
+
654
+ struct lm_ggml_hash_set visited_hash_table;
655
+
656
+ enum lm_ggml_cgraph_eval_order order;
657
+ };
658
+
659
+ // scratch buffer
660
+ struct lm_ggml_scratch {
661
+ size_t offs;
662
+ size_t size;
663
+ void * data;
664
+ };
665
+
666
+ struct lm_ggml_init_params {
667
+ // memory pool
668
+ size_t mem_size; // bytes
669
+ void * mem_buffer; // if NULL, memory will be allocated internally
670
+ bool no_alloc; // don't allocate memory for the tensor data
671
+ };
672
+
673
+ // numa strategies
674
+ enum lm_ggml_numa_strategy {
675
+ LM_GGML_NUMA_STRATEGY_DISABLED = 0,
676
+ LM_GGML_NUMA_STRATEGY_DISTRIBUTE = 1,
677
+ LM_GGML_NUMA_STRATEGY_ISOLATE = 2,
678
+ LM_GGML_NUMA_STRATEGY_NUMACTL = 3,
679
+ LM_GGML_NUMA_STRATEGY_MIRROR = 4,
680
+ LM_GGML_NUMA_STRATEGY_COUNT
681
+ };
682
+
683
+ //
684
+ // GUID
685
+ //
686
+
687
+ // GUID types
688
+ typedef uint8_t lm_ggml_guid[16];
689
+ typedef lm_ggml_guid * lm_ggml_guid_t;
690
+
691
+ LM_GGML_API bool lm_ggml_guid_matches(lm_ggml_guid_t guid_a, lm_ggml_guid_t guid_b);
692
+
693
+ // misc
694
+
695
+ LM_GGML_API void lm_ggml_time_init(void); // call this once at the beginning of the program
696
+ LM_GGML_API int64_t lm_ggml_time_ms(void);
697
+ LM_GGML_API int64_t lm_ggml_time_us(void);
698
+ LM_GGML_API int64_t lm_ggml_cycles(void);
699
+ LM_GGML_API int64_t lm_ggml_cycles_per_ms(void);
700
+
701
+ LM_GGML_API void lm_ggml_print_backtrace(void);
702
+
703
+ // accepts a UTF-8 path, even on Windows
704
+ LM_GGML_API FILE * lm_ggml_fopen(const char * fname, const char * mode);
705
+
706
+ LM_GGML_API void lm_ggml_numa_init(enum lm_ggml_numa_strategy numa); // call once for better performance on NUMA systems
707
+ LM_GGML_API bool lm_ggml_is_numa(void); // true if init detected that system has >1 NUMA node
708
+
709
+ LM_GGML_API void lm_ggml_print_object (const struct lm_ggml_object * obj);
710
+ LM_GGML_API void lm_ggml_print_objects(const struct lm_ggml_context * ctx);
711
+
712
+ LM_GGML_API LM_GGML_CALL int64_t lm_ggml_nelements (const struct lm_ggml_tensor * tensor);
713
+ LM_GGML_API LM_GGML_CALL int64_t lm_ggml_nrows (const struct lm_ggml_tensor * tensor);
714
+ LM_GGML_API LM_GGML_CALL size_t lm_ggml_nbytes (const struct lm_ggml_tensor * tensor);
715
+ LM_GGML_API size_t lm_ggml_nbytes_pad (const struct lm_ggml_tensor * tensor); // same as lm_ggml_nbytes() but padded to LM_GGML_MEM_ALIGN
716
+
717
+ LM_GGML_API LM_GGML_CALL int64_t lm_ggml_blck_size(enum lm_ggml_type type);
718
+ LM_GGML_API LM_GGML_CALL size_t lm_ggml_type_size(enum lm_ggml_type type); // size in bytes for all elements in a block
719
+ LM_GGML_API LM_GGML_CALL size_t lm_ggml_row_size (enum lm_ggml_type type, int64_t ne); // size in bytes for all elements in a row
720
+
721
+ LM_GGML_DEPRECATED(
722
+ LM_GGML_API double lm_ggml_type_sizef(enum lm_ggml_type type), // lm_ggml_type_size()/lm_ggml_blck_size() as float
723
+ "use lm_ggml_row_size() instead");
724
+
725
+ LM_GGML_API LM_GGML_CALL const char * lm_ggml_type_name(enum lm_ggml_type type);
726
+ LM_GGML_API LM_GGML_CALL const char * lm_ggml_op_name (enum lm_ggml_op op);
727
+ LM_GGML_API const char * lm_ggml_op_symbol(enum lm_ggml_op op);
728
+
729
+ LM_GGML_API const char * lm_ggml_unary_op_name(enum lm_ggml_unary_op op);
730
+ LM_GGML_API LM_GGML_CALL const char * lm_ggml_op_desc(const struct lm_ggml_tensor * t); // unary or op name
731
+
732
+ LM_GGML_API LM_GGML_CALL size_t lm_ggml_element_size(const struct lm_ggml_tensor * tensor);
733
+
734
+ LM_GGML_API LM_GGML_CALL bool lm_ggml_is_quantized(enum lm_ggml_type type);
735
+
736
+ // TODO: temporary until model loading of ggml examples is refactored
737
+ LM_GGML_API enum lm_ggml_type lm_ggml_ftype_to_lm_ggml_type(enum lm_ggml_ftype ftype);
738
+
739
+ LM_GGML_API LM_GGML_CALL bool lm_ggml_is_transposed(const struct lm_ggml_tensor * tensor);
740
+ LM_GGML_API LM_GGML_CALL bool lm_ggml_is_permuted (const struct lm_ggml_tensor * tensor);
741
+ LM_GGML_API LM_GGML_CALL bool lm_ggml_is_empty (const struct lm_ggml_tensor * tensor);
742
+ LM_GGML_API bool lm_ggml_is_scalar (const struct lm_ggml_tensor * tensor);
743
+ LM_GGML_API bool lm_ggml_is_vector (const struct lm_ggml_tensor * tensor);
744
+ LM_GGML_API bool lm_ggml_is_matrix (const struct lm_ggml_tensor * tensor);
745
+ LM_GGML_API bool lm_ggml_is_3d (const struct lm_ggml_tensor * tensor);
746
+ LM_GGML_API int lm_ggml_n_dims (const struct lm_ggml_tensor * tensor); // returns 1 for scalars
747
+
748
+ LM_GGML_API LM_GGML_CALL bool lm_ggml_is_contiguous (const struct lm_ggml_tensor * tensor);
749
+ LM_GGML_API LM_GGML_CALL bool lm_ggml_is_contiguous_0(const struct lm_ggml_tensor * tensor); // same as lm_ggml_is_contiguous()
750
+ LM_GGML_API LM_GGML_CALL bool lm_ggml_is_contiguous_1(const struct lm_ggml_tensor * tensor); // contiguous for dims >= 1
751
+ LM_GGML_API LM_GGML_CALL bool lm_ggml_is_contiguous_2(const struct lm_ggml_tensor * tensor); // contiguous for dims >= 2
752
+
753
+ LM_GGML_API bool lm_ggml_are_same_shape (const struct lm_ggml_tensor * t0, const struct lm_ggml_tensor * t1);
754
+ LM_GGML_API bool lm_ggml_are_same_stride(const struct lm_ggml_tensor * t0, const struct lm_ggml_tensor * t1);
755
+
756
+ LM_GGML_API bool lm_ggml_can_repeat(const struct lm_ggml_tensor * t0, const struct lm_ggml_tensor * t1);
757
+
758
+ // use this to compute the memory overhead of a tensor
759
+ LM_GGML_API size_t lm_ggml_tensor_overhead(void);
760
+
761
+ LM_GGML_API bool lm_ggml_validate_row_data(enum lm_ggml_type type, const void * data, size_t nbytes);
762
+
763
+ // main
764
+
765
+ LM_GGML_API struct lm_ggml_context * lm_ggml_init(struct lm_ggml_init_params params);
766
+ LM_GGML_API void lm_ggml_free(struct lm_ggml_context * ctx);
767
+
768
+ LM_GGML_API size_t lm_ggml_used_mem(const struct lm_ggml_context * ctx);
769
+
770
+ LM_GGML_API size_t lm_ggml_set_scratch (struct lm_ggml_context * ctx, struct lm_ggml_scratch scratch);
771
+ LM_GGML_API bool lm_ggml_get_no_alloc(struct lm_ggml_context * ctx);
772
+ LM_GGML_API void lm_ggml_set_no_alloc(struct lm_ggml_context * ctx, bool no_alloc);
773
+
774
+ LM_GGML_API void * lm_ggml_get_mem_buffer (const struct lm_ggml_context * ctx);
775
+ LM_GGML_API size_t lm_ggml_get_mem_size (const struct lm_ggml_context * ctx);
776
+ LM_GGML_API size_t lm_ggml_get_max_tensor_size(const struct lm_ggml_context * ctx);
777
+
778
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_new_tensor(
779
+ struct lm_ggml_context * ctx,
780
+ enum lm_ggml_type type,
781
+ int n_dims,
782
+ const int64_t *ne);
783
+
784
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_new_tensor_1d(
785
+ struct lm_ggml_context * ctx,
786
+ enum lm_ggml_type type,
787
+ int64_t ne0);
788
+
789
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_new_tensor_2d(
790
+ struct lm_ggml_context * ctx,
791
+ enum lm_ggml_type type,
792
+ int64_t ne0,
793
+ int64_t ne1);
794
+
795
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_new_tensor_3d(
796
+ struct lm_ggml_context * ctx,
797
+ enum lm_ggml_type type,
798
+ int64_t ne0,
799
+ int64_t ne1,
800
+ int64_t ne2);
801
+
802
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_new_tensor_4d(
803
+ struct lm_ggml_context * ctx,
804
+ enum lm_ggml_type type,
805
+ int64_t ne0,
806
+ int64_t ne1,
807
+ int64_t ne2,
808
+ int64_t ne3);
809
+
810
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_new_i32(struct lm_ggml_context * ctx, int32_t value);
811
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_new_f32(struct lm_ggml_context * ctx, float value);
812
+
813
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_dup_tensor (struct lm_ggml_context * ctx, const struct lm_ggml_tensor * src);
814
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_view_tensor(struct lm_ggml_context * ctx, struct lm_ggml_tensor * src);
815
+
816
+ // Context tensor enumeration and lookup
817
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_get_first_tensor(const struct lm_ggml_context * ctx);
818
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_get_next_tensor (const struct lm_ggml_context * ctx, struct lm_ggml_tensor * tensor);
819
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_get_tensor(struct lm_ggml_context * ctx, const char * name);
820
+
821
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_set_zero(struct lm_ggml_tensor * tensor);
822
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_set_i32 (struct lm_ggml_tensor * tensor, int32_t value);
823
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_set_f32 (struct lm_ggml_tensor * tensor, float value);
824
+
825
+ // Converts a flat index into coordinates
826
+ LM_GGML_API void lm_ggml_unravel_index(const struct lm_ggml_tensor * tensor, int64_t i, int64_t * i0, int64_t * i1, int64_t * i2, int64_t * i3);
827
+
828
+ LM_GGML_API int32_t lm_ggml_get_i32_1d(const struct lm_ggml_tensor * tensor, int i);
829
+ LM_GGML_API void lm_ggml_set_i32_1d(const struct lm_ggml_tensor * tensor, int i, int32_t value);
830
+
831
+ LM_GGML_API int32_t lm_ggml_get_i32_nd(const struct lm_ggml_tensor * tensor, int i0, int i1, int i2, int i3);
832
+ LM_GGML_API void lm_ggml_set_i32_nd(const struct lm_ggml_tensor * tensor, int i0, int i1, int i2, int i3, int32_t value);
833
+
834
+ LM_GGML_API float lm_ggml_get_f32_1d(const struct lm_ggml_tensor * tensor, int i);
835
+ LM_GGML_API void lm_ggml_set_f32_1d(const struct lm_ggml_tensor * tensor, int i, float value);
836
+
837
+ LM_GGML_API float lm_ggml_get_f32_nd(const struct lm_ggml_tensor * tensor, int i0, int i1, int i2, int i3);
838
+ LM_GGML_API void lm_ggml_set_f32_nd(const struct lm_ggml_tensor * tensor, int i0, int i1, int i2, int i3, float value);
839
+
840
+ LM_GGML_API void * lm_ggml_get_data (const struct lm_ggml_tensor * tensor);
841
+ LM_GGML_API float * lm_ggml_get_data_f32(const struct lm_ggml_tensor * tensor);
842
+
843
+ LM_GGML_API LM_GGML_CALL enum lm_ggml_unary_op lm_ggml_get_unary_op(const struct lm_ggml_tensor * tensor);
844
+
845
+ LM_GGML_API const char * lm_ggml_get_name (const struct lm_ggml_tensor * tensor);
846
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_set_name ( struct lm_ggml_tensor * tensor, const char * name);
847
+ LM_GGML_ATTRIBUTE_FORMAT(2, 3)
848
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_format_name( struct lm_ggml_tensor * tensor, const char * fmt, ...);
849
+
850
+ //
851
+ // operations on tensors with backpropagation
852
+ //
853
+
854
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_dup(
855
+ struct lm_ggml_context * ctx,
856
+ struct lm_ggml_tensor * a);
857
+
858
+ // in-place, returns view(a)
859
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_dup_inplace(
860
+ struct lm_ggml_context * ctx,
861
+ struct lm_ggml_tensor * a);
862
+
863
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_add(
864
+ struct lm_ggml_context * ctx,
865
+ struct lm_ggml_tensor * a,
866
+ struct lm_ggml_tensor * b);
867
+
868
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_add_inplace(
869
+ struct lm_ggml_context * ctx,
870
+ struct lm_ggml_tensor * a,
871
+ struct lm_ggml_tensor * b);
872
+
873
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_add_cast(
874
+ struct lm_ggml_context * ctx,
875
+ struct lm_ggml_tensor * a,
876
+ struct lm_ggml_tensor * b,
877
+ enum lm_ggml_type type);
878
+
879
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_add1(
880
+ struct lm_ggml_context * ctx,
881
+ struct lm_ggml_tensor * a,
882
+ struct lm_ggml_tensor * b);
883
+
884
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_add1_inplace(
885
+ struct lm_ggml_context * ctx,
886
+ struct lm_ggml_tensor * a,
887
+ struct lm_ggml_tensor * b);
888
+
889
+ // dst = a
890
+ // view(dst, nb1, nb2, nb3, offset) += b
891
+ // return dst
892
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_acc(
893
+ struct lm_ggml_context * ctx,
894
+ struct lm_ggml_tensor * a,
895
+ struct lm_ggml_tensor * b,
896
+ size_t nb1,
897
+ size_t nb2,
898
+ size_t nb3,
899
+ size_t offset);
900
+
901
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_acc_inplace(
902
+ struct lm_ggml_context * ctx,
903
+ struct lm_ggml_tensor * a,
904
+ struct lm_ggml_tensor * b,
905
+ size_t nb1,
906
+ size_t nb2,
907
+ size_t nb3,
908
+ size_t offset);
909
+
910
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_sub(
911
+ struct lm_ggml_context * ctx,
912
+ struct lm_ggml_tensor * a,
913
+ struct lm_ggml_tensor * b);
914
+
915
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_sub_inplace(
916
+ struct lm_ggml_context * ctx,
917
+ struct lm_ggml_tensor * a,
918
+ struct lm_ggml_tensor * b);
919
+
920
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_mul(
921
+ struct lm_ggml_context * ctx,
922
+ struct lm_ggml_tensor * a,
923
+ struct lm_ggml_tensor * b);
924
+
925
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_mul_inplace(
926
+ struct lm_ggml_context * ctx,
927
+ struct lm_ggml_tensor * a,
928
+ struct lm_ggml_tensor * b);
929
+
930
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_div(
931
+ struct lm_ggml_context * ctx,
932
+ struct lm_ggml_tensor * a,
933
+ struct lm_ggml_tensor * b);
934
+
935
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_div_inplace(
936
+ struct lm_ggml_context * ctx,
937
+ struct lm_ggml_tensor * a,
938
+ struct lm_ggml_tensor * b);
939
+
940
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_sqr(
941
+ struct lm_ggml_context * ctx,
942
+ struct lm_ggml_tensor * a);
943
+
944
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_sqr_inplace(
945
+ struct lm_ggml_context * ctx,
946
+ struct lm_ggml_tensor * a);
947
+
948
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_sqrt(
949
+ struct lm_ggml_context * ctx,
950
+ struct lm_ggml_tensor * a);
951
+
952
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_sqrt_inplace(
953
+ struct lm_ggml_context * ctx,
954
+ struct lm_ggml_tensor * a);
955
+
956
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_log(
957
+ struct lm_ggml_context * ctx,
958
+ struct lm_ggml_tensor * a);
959
+
960
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_log_inplace(
961
+ struct lm_ggml_context * ctx,
962
+ struct lm_ggml_tensor * a);
963
+
964
+ // return scalar
965
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_sum(
966
+ struct lm_ggml_context * ctx,
967
+ struct lm_ggml_tensor * a);
968
+
969
+ // sums along rows, with input shape [a,b,c,d] return shape [1,b,c,d]
970
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_sum_rows(
971
+ struct lm_ggml_context * ctx,
972
+ struct lm_ggml_tensor * a);
973
+
974
+ // mean along rows
975
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_mean(
976
+ struct lm_ggml_context * ctx,
977
+ struct lm_ggml_tensor * a);
978
+
979
+ // argmax along rows
980
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_argmax(
981
+ struct lm_ggml_context * ctx,
982
+ struct lm_ggml_tensor * a);
983
+
984
+ // if a is the same shape as b, and a is not parameter, return a
985
+ // otherwise, return a new tensor: repeat(a) to fit in b
986
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_repeat(
987
+ struct lm_ggml_context * ctx,
988
+ struct lm_ggml_tensor * a,
989
+ struct lm_ggml_tensor * b);
990
+
991
+ // sums repetitions in a into shape of b
992
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_repeat_back(
993
+ struct lm_ggml_context * ctx,
994
+ struct lm_ggml_tensor * a,
995
+ struct lm_ggml_tensor * b);
996
+
997
+ // concat a and b along dim
998
+ // used in stable-diffusion
999
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_concat(
1000
+ struct lm_ggml_context * ctx,
1001
+ struct lm_ggml_tensor * a,
1002
+ struct lm_ggml_tensor * b,
1003
+ int dim);
1004
+
1005
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_abs(
1006
+ struct lm_ggml_context * ctx,
1007
+ struct lm_ggml_tensor * a);
1008
+
1009
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_abs_inplace(
1010
+ struct lm_ggml_context * ctx,
1011
+ struct lm_ggml_tensor * a);
1012
+
1013
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_sgn(
1014
+ struct lm_ggml_context * ctx,
1015
+ struct lm_ggml_tensor * a);
1016
+
1017
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_sgn_inplace(
1018
+ struct lm_ggml_context * ctx,
1019
+ struct lm_ggml_tensor * a);
1020
+
1021
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_neg(
1022
+ struct lm_ggml_context * ctx,
1023
+ struct lm_ggml_tensor * a);
1024
+
1025
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_neg_inplace(
1026
+ struct lm_ggml_context * ctx,
1027
+ struct lm_ggml_tensor * a);
1028
+
1029
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_step(
1030
+ struct lm_ggml_context * ctx,
1031
+ struct lm_ggml_tensor * a);
1032
+
1033
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_step_inplace(
1034
+ struct lm_ggml_context * ctx,
1035
+ struct lm_ggml_tensor * a);
1036
+
1037
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_tanh(
1038
+ struct lm_ggml_context * ctx,
1039
+ struct lm_ggml_tensor * a);
1040
+
1041
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_tanh_inplace(
1042
+ struct lm_ggml_context * ctx,
1043
+ struct lm_ggml_tensor * a);
1044
+
1045
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_elu(
1046
+ struct lm_ggml_context * ctx,
1047
+ struct lm_ggml_tensor * a);
1048
+
1049
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_elu_inplace(
1050
+ struct lm_ggml_context * ctx,
1051
+ struct lm_ggml_tensor * a);
1052
+
1053
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_relu(
1054
+ struct lm_ggml_context * ctx,
1055
+ struct lm_ggml_tensor * a);
1056
+
1057
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_leaky_relu(
1058
+ struct lm_ggml_context * ctx,
1059
+ struct lm_ggml_tensor * a, float negative_slope, bool inplace);
1060
+
1061
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_relu_inplace(
1062
+ struct lm_ggml_context * ctx,
1063
+ struct lm_ggml_tensor * a);
1064
+
1065
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_sigmoid(
1066
+ struct lm_ggml_context * ctx,
1067
+ struct lm_ggml_tensor * a);
1068
+
1069
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_sigmoid_inplace(
1070
+ struct lm_ggml_context * ctx,
1071
+ struct lm_ggml_tensor * a);
1072
+
1073
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_gelu(
1074
+ struct lm_ggml_context * ctx,
1075
+ struct lm_ggml_tensor * a);
1076
+
1077
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_gelu_inplace(
1078
+ struct lm_ggml_context * ctx,
1079
+ struct lm_ggml_tensor * a);
1080
+
1081
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_gelu_quick(
1082
+ struct lm_ggml_context * ctx,
1083
+ struct lm_ggml_tensor * a);
1084
+
1085
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_gelu_quick_inplace(
1086
+ struct lm_ggml_context * ctx,
1087
+ struct lm_ggml_tensor * a);
1088
+
1089
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_silu(
1090
+ struct lm_ggml_context * ctx,
1091
+ struct lm_ggml_tensor * a);
1092
+
1093
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_silu_inplace(
1094
+ struct lm_ggml_context * ctx,
1095
+ struct lm_ggml_tensor * a);
1096
+
1097
+ // a - x
1098
+ // b - dy
1099
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_silu_back(
1100
+ struct lm_ggml_context * ctx,
1101
+ struct lm_ggml_tensor * a,
1102
+ struct lm_ggml_tensor * b);
1103
+
1104
+ // hardswish(x) = x * relu6(x + 3) / 6
1105
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_hardswish(
1106
+ struct lm_ggml_context * ctx,
1107
+ struct lm_ggml_tensor * a);
1108
+
1109
+ // hardsigmoid(x) = relu6(x + 3) / 6
1110
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_hardsigmoid(
1111
+ struct lm_ggml_context * ctx,
1112
+ struct lm_ggml_tensor * a);
1113
+
1114
+ // normalize along rows
1115
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_norm(
1116
+ struct lm_ggml_context * ctx,
1117
+ struct lm_ggml_tensor * a,
1118
+ float eps);
1119
+
1120
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_norm_inplace(
1121
+ struct lm_ggml_context * ctx,
1122
+ struct lm_ggml_tensor * a,
1123
+ float eps);
1124
+
1125
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_rms_norm(
1126
+ struct lm_ggml_context * ctx,
1127
+ struct lm_ggml_tensor * a,
1128
+ float eps);
1129
+
1130
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_rms_norm_inplace(
1131
+ struct lm_ggml_context * ctx,
1132
+ struct lm_ggml_tensor * a,
1133
+ float eps);
1134
+
1135
+ // group normalize along ne0*ne1*n_groups
1136
+ // used in stable-diffusion
1137
+ // TODO: eps is hardcoded to 1e-6 for now
1138
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_group_norm(
1139
+ struct lm_ggml_context * ctx,
1140
+ struct lm_ggml_tensor * a,
1141
+ int n_groups);
1142
+
1143
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_group_norm_inplace(
1144
+ struct lm_ggml_context * ctx,
1145
+ struct lm_ggml_tensor * a,
1146
+ int n_groups);
1147
+
1148
+ // a - x
1149
+ // b - dy
1150
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_rms_norm_back(
1151
+ struct lm_ggml_context * ctx,
1152
+ struct lm_ggml_tensor * a,
1153
+ struct lm_ggml_tensor * b,
1154
+ float eps);
1155
+
1156
+ // A: k columns, n rows => [ne03, ne02, n, k]
1157
+ // B: k columns, m rows (i.e. we transpose it internally) => [ne03 * x, ne02 * y, m, k]
1158
+ // result is n columns, m rows => [ne03 * x, ne02 * y, m, n]
1159
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_mul_mat(
1160
+ struct lm_ggml_context * ctx,
1161
+ struct lm_ggml_tensor * a,
1162
+ struct lm_ggml_tensor * b);
1163
+
1164
+ // change the precision of a matrix multiplication
1165
+ // set to LM_GGML_PREC_F32 for higher precision (useful for phi-2)
1166
+ LM_GGML_API void lm_ggml_mul_mat_set_prec(
1167
+ struct lm_ggml_tensor * a,
1168
+ enum lm_ggml_prec prec);
1169
+
1170
+ // indirect matrix multiplication
1171
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_mul_mat_id(
1172
+ struct lm_ggml_context * ctx,
1173
+ struct lm_ggml_tensor * as,
1174
+ struct lm_ggml_tensor * b,
1175
+ struct lm_ggml_tensor * ids);
1176
+
1177
+ // A: m columns, n rows,
1178
+ // B: p columns, n rows,
1179
+ // result is m columns, p rows
1180
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_out_prod(
1181
+ struct lm_ggml_context * ctx,
1182
+ struct lm_ggml_tensor * a,
1183
+ struct lm_ggml_tensor * b);
1184
+
1185
+ //
1186
+ // operations on tensors without backpropagation
1187
+ //
1188
+
1189
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_scale(
1190
+ struct lm_ggml_context * ctx,
1191
+ struct lm_ggml_tensor * a,
1192
+ float s);
1193
+
1194
+ // in-place, returns view(a)
1195
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_scale_inplace(
1196
+ struct lm_ggml_context * ctx,
1197
+ struct lm_ggml_tensor * a,
1198
+ float s);
1199
+
1200
+ // b -> view(a,offset,nb1,nb2,3), return modified a
1201
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_set(
1202
+ struct lm_ggml_context * ctx,
1203
+ struct lm_ggml_tensor * a,
1204
+ struct lm_ggml_tensor * b,
1205
+ size_t nb1,
1206
+ size_t nb2,
1207
+ size_t nb3,
1208
+ size_t offset);
1209
+
1210
+ // b -> view(a,offset,nb1,nb2,3), return view(a)
1211
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_set_inplace(
1212
+ struct lm_ggml_context * ctx,
1213
+ struct lm_ggml_tensor * a,
1214
+ struct lm_ggml_tensor * b,
1215
+ size_t nb1,
1216
+ size_t nb2,
1217
+ size_t nb3,
1218
+ size_t offset);
1219
+
1220
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_set_1d(
1221
+ struct lm_ggml_context * ctx,
1222
+ struct lm_ggml_tensor * a,
1223
+ struct lm_ggml_tensor * b,
1224
+ size_t offset);
1225
+
1226
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_set_1d_inplace(
1227
+ struct lm_ggml_context * ctx,
1228
+ struct lm_ggml_tensor * a,
1229
+ struct lm_ggml_tensor * b,
1230
+ size_t offset);
1231
+
1232
+ // b -> view(a,offset,nb1,nb2,3), return modified a
1233
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_set_2d(
1234
+ struct lm_ggml_context * ctx,
1235
+ struct lm_ggml_tensor * a,
1236
+ struct lm_ggml_tensor * b,
1237
+ size_t nb1,
1238
+ size_t offset);
1239
+
1240
+ // b -> view(a,offset,nb1,nb2,3), return view(a)
1241
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_set_2d_inplace(
1242
+ struct lm_ggml_context * ctx,
1243
+ struct lm_ggml_tensor * a,
1244
+ struct lm_ggml_tensor * b,
1245
+ size_t nb1,
1246
+ size_t offset);
1247
+
1248
+ // a -> b, return view(b)
1249
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_cpy(
1250
+ struct lm_ggml_context * ctx,
1251
+ struct lm_ggml_tensor * a,
1252
+ struct lm_ggml_tensor * b);
1253
+
1254
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_cast(
1255
+ struct lm_ggml_context * ctx,
1256
+ struct lm_ggml_tensor * a,
1257
+ enum lm_ggml_type type);
1258
+
1259
+ // make contiguous
1260
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_cont(
1261
+ struct lm_ggml_context * ctx,
1262
+ struct lm_ggml_tensor * a);
1263
+
1264
+ // make contiguous, with new shape
1265
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_cont_1d(
1266
+ struct lm_ggml_context * ctx,
1267
+ struct lm_ggml_tensor * a,
1268
+ int64_t ne0);
1269
+
1270
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_cont_2d(
1271
+ struct lm_ggml_context * ctx,
1272
+ struct lm_ggml_tensor * a,
1273
+ int64_t ne0,
1274
+ int64_t ne1);
1275
+
1276
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_cont_3d(
1277
+ struct lm_ggml_context * ctx,
1278
+ struct lm_ggml_tensor * a,
1279
+ int64_t ne0,
1280
+ int64_t ne1,
1281
+ int64_t ne2);
1282
+
1283
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_cont_4d(
1284
+ struct lm_ggml_context * ctx,
1285
+ struct lm_ggml_tensor * a,
1286
+ int64_t ne0,
1287
+ int64_t ne1,
1288
+ int64_t ne2,
1289
+ int64_t ne3);
1290
+
1291
+ // return view(a), b specifies the new shape
1292
+ // TODO: when we start computing gradient, make a copy instead of view
1293
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_reshape(
1294
+ struct lm_ggml_context * ctx,
1295
+ struct lm_ggml_tensor * a,
1296
+ struct lm_ggml_tensor * b);
1297
+
1298
+ // return view(a)
1299
+ // TODO: when we start computing gradient, make a copy instead of view
1300
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_reshape_1d(
1301
+ struct lm_ggml_context * ctx,
1302
+ struct lm_ggml_tensor * a,
1303
+ int64_t ne0);
1304
+
1305
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_reshape_2d(
1306
+ struct lm_ggml_context * ctx,
1307
+ struct lm_ggml_tensor * a,
1308
+ int64_t ne0,
1309
+ int64_t ne1);
1310
+
1311
+ // return view(a)
1312
+ // TODO: when we start computing gradient, make a copy instead of view
1313
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_reshape_3d(
1314
+ struct lm_ggml_context * ctx,
1315
+ struct lm_ggml_tensor * a,
1316
+ int64_t ne0,
1317
+ int64_t ne1,
1318
+ int64_t ne2);
1319
+
1320
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_reshape_4d(
1321
+ struct lm_ggml_context * ctx,
1322
+ struct lm_ggml_tensor * a,
1323
+ int64_t ne0,
1324
+ int64_t ne1,
1325
+ int64_t ne2,
1326
+ int64_t ne3);
1327
+
1328
+ // offset in bytes
1329
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_view_1d(
1330
+ struct lm_ggml_context * ctx,
1331
+ struct lm_ggml_tensor * a,
1332
+ int64_t ne0,
1333
+ size_t offset);
1334
+
1335
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_view_2d(
1336
+ struct lm_ggml_context * ctx,
1337
+ struct lm_ggml_tensor * a,
1338
+ int64_t ne0,
1339
+ int64_t ne1,
1340
+ size_t nb1, // row stride in bytes
1341
+ size_t offset);
1342
+
1343
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_view_3d(
1344
+ struct lm_ggml_context * ctx,
1345
+ struct lm_ggml_tensor * a,
1346
+ int64_t ne0,
1347
+ int64_t ne1,
1348
+ int64_t ne2,
1349
+ size_t nb1, // row stride in bytes
1350
+ size_t nb2, // slice stride in bytes
1351
+ size_t offset);
1352
+
1353
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_view_4d(
1354
+ struct lm_ggml_context * ctx,
1355
+ struct lm_ggml_tensor * a,
1356
+ int64_t ne0,
1357
+ int64_t ne1,
1358
+ int64_t ne2,
1359
+ int64_t ne3,
1360
+ size_t nb1, // row stride in bytes
1361
+ size_t nb2, // slice stride in bytes
1362
+ size_t nb3,
1363
+ size_t offset);
1364
+
1365
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_permute(
1366
+ struct lm_ggml_context * ctx,
1367
+ struct lm_ggml_tensor * a,
1368
+ int axis0,
1369
+ int axis1,
1370
+ int axis2,
1371
+ int axis3);
1372
+
1373
+ // alias for lm_ggml_permute(ctx, a, 1, 0, 2, 3)
1374
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_transpose(
1375
+ struct lm_ggml_context * ctx,
1376
+ struct lm_ggml_tensor * a);
1377
+
1378
+ // supports 3D: a->ne[2] == b->ne[1]
1379
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_get_rows(
1380
+ struct lm_ggml_context * ctx,
1381
+ struct lm_ggml_tensor * a,
1382
+ struct lm_ggml_tensor * b);
1383
+
1384
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_get_rows_back(
1385
+ struct lm_ggml_context * ctx,
1386
+ struct lm_ggml_tensor * a,
1387
+ struct lm_ggml_tensor * b,
1388
+ struct lm_ggml_tensor * c);
1389
+
1390
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_diag(
1391
+ struct lm_ggml_context * ctx,
1392
+ struct lm_ggml_tensor * a);
1393
+
1394
+ // set elements above the diagonal to -INF
1395
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_diag_mask_inf(
1396
+ struct lm_ggml_context * ctx,
1397
+ struct lm_ggml_tensor * a,
1398
+ int n_past);
1399
+
1400
+ // in-place, returns view(a)
1401
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_diag_mask_inf_inplace(
1402
+ struct lm_ggml_context * ctx,
1403
+ struct lm_ggml_tensor * a,
1404
+ int n_past);
1405
+
1406
+ // set elements above the diagonal to 0
1407
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_diag_mask_zero(
1408
+ struct lm_ggml_context * ctx,
1409
+ struct lm_ggml_tensor * a,
1410
+ int n_past);
1411
+
1412
+ // in-place, returns view(a)
1413
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_diag_mask_zero_inplace(
1414
+ struct lm_ggml_context * ctx,
1415
+ struct lm_ggml_tensor * a,
1416
+ int n_past);
1417
+
1418
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_soft_max(
1419
+ struct lm_ggml_context * ctx,
1420
+ struct lm_ggml_tensor * a);
1421
+
1422
+ // in-place, returns view(a)
1423
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_soft_max_inplace(
1424
+ struct lm_ggml_context * ctx,
1425
+ struct lm_ggml_tensor * a);
1426
+
1427
+ // fused soft_max(a*scale + mask*(ALiBi slope))
1428
+ // mask is optional
1429
+ // max_bias = 0.0f for no ALiBi
1430
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_soft_max_ext(
1431
+ struct lm_ggml_context * ctx,
1432
+ struct lm_ggml_tensor * a,
1433
+ struct lm_ggml_tensor * mask,
1434
+ float scale,
1435
+ float max_bias);
1436
+
1437
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_soft_max_back(
1438
+ struct lm_ggml_context * ctx,
1439
+ struct lm_ggml_tensor * a,
1440
+ struct lm_ggml_tensor * b);
1441
+
1442
+ // in-place, returns view(a)
1443
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_soft_max_back_inplace(
1444
+ struct lm_ggml_context * ctx,
1445
+ struct lm_ggml_tensor * a,
1446
+ struct lm_ggml_tensor * b);
1447
+
1448
+ // rotary position embedding
1449
+ // if mode & 1 == 1, skip n_past elements (NOT SUPPORTED)
1450
+ // if mode & 2 == 1, GPT-NeoX style
1451
+ //
1452
+ // b is an int32 vector with size a->ne[2], it contains the positions
1453
+ // c is freq factors (e.g. phi3-128k), (optional)
1454
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_rope(
1455
+ struct lm_ggml_context * ctx,
1456
+ struct lm_ggml_tensor * a,
1457
+ struct lm_ggml_tensor * b,
1458
+ int n_dims,
1459
+ int mode);
1460
+
1461
+ // in-place, returns view(a)
1462
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_rope_inplace(
1463
+ struct lm_ggml_context * ctx,
1464
+ struct lm_ggml_tensor * a,
1465
+ struct lm_ggml_tensor * b,
1466
+ int n_dims,
1467
+ int mode);
1468
+
1469
+ // custom RoPE
1470
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_rope_ext(
1471
+ struct lm_ggml_context * ctx,
1472
+ struct lm_ggml_tensor * a,
1473
+ struct lm_ggml_tensor * b,
1474
+ struct lm_ggml_tensor * c,
1475
+ int n_dims,
1476
+ int mode,
1477
+ int n_ctx_orig,
1478
+ float freq_base,
1479
+ float freq_scale,
1480
+ float ext_factor,
1481
+ float attn_factor,
1482
+ float beta_fast,
1483
+ float beta_slow);
1484
+
1485
+ // in-place, returns view(a)
1486
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_rope_ext_inplace(
1487
+ struct lm_ggml_context * ctx,
1488
+ struct lm_ggml_tensor * a,
1489
+ struct lm_ggml_tensor * b,
1490
+ struct lm_ggml_tensor * c,
1491
+ int n_dims,
1492
+ int mode,
1493
+ int n_ctx_orig,
1494
+ float freq_base,
1495
+ float freq_scale,
1496
+ float ext_factor,
1497
+ float attn_factor,
1498
+ float beta_fast,
1499
+ float beta_slow);
1500
+
1501
+ LM_GGML_DEPRECATED(LM_GGML_API struct lm_ggml_tensor * lm_ggml_rope_custom(
1502
+ struct lm_ggml_context * ctx,
1503
+ struct lm_ggml_tensor * a,
1504
+ struct lm_ggml_tensor * b,
1505
+ int n_dims,
1506
+ int mode,
1507
+ int n_ctx_orig,
1508
+ float freq_base,
1509
+ float freq_scale,
1510
+ float ext_factor,
1511
+ float attn_factor,
1512
+ float beta_fast,
1513
+ float beta_slow),
1514
+ "use lm_ggml_rope_ext instead");
1515
+
1516
+ LM_GGML_DEPRECATED(LM_GGML_API struct lm_ggml_tensor * lm_ggml_rope_custom_inplace(
1517
+ struct lm_ggml_context * ctx,
1518
+ struct lm_ggml_tensor * a,
1519
+ struct lm_ggml_tensor * b,
1520
+ int n_dims,
1521
+ int mode,
1522
+ int n_ctx_orig,
1523
+ float freq_base,
1524
+ float freq_scale,
1525
+ float ext_factor,
1526
+ float attn_factor,
1527
+ float beta_fast,
1528
+ float beta_slow),
1529
+ "use lm_ggml_rope_ext_inplace instead");
1530
+
1531
+ // compute correction dims for YaRN RoPE scaling
1532
+ LM_GGML_CALL void lm_ggml_rope_yarn_corr_dims(
1533
+ int n_dims, int n_ctx_orig, float freq_base, float beta_fast, float beta_slow, float dims[2]);
1534
+
1535
+ // rotary position embedding backward, i.e compute dx from dy
1536
+ // a - dy
1537
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_rope_back(
1538
+ struct lm_ggml_context * ctx,
1539
+ struct lm_ggml_tensor * a,
1540
+ struct lm_ggml_tensor * b,
1541
+ struct lm_ggml_tensor * c,
1542
+ int n_dims,
1543
+ int mode,
1544
+ int n_ctx_orig,
1545
+ float freq_base,
1546
+ float freq_scale,
1547
+ float ext_factor,
1548
+ float attn_factor,
1549
+ float beta_fast,
1550
+ float beta_slow);
1551
+
1552
+ // clamp
1553
+ // in-place, returns view(a)
1554
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_clamp(
1555
+ struct lm_ggml_context * ctx,
1556
+ struct lm_ggml_tensor * a,
1557
+ float min,
1558
+ float max);
1559
+
1560
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_im2col(
1561
+ struct lm_ggml_context * ctx,
1562
+ struct lm_ggml_tensor * a,
1563
+ struct lm_ggml_tensor * b,
1564
+ int s0,
1565
+ int s1,
1566
+ int p0,
1567
+ int p1,
1568
+ int d0,
1569
+ int d1,
1570
+ bool is_2D,
1571
+ enum lm_ggml_type dst_type);
1572
+
1573
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_conv_depthwise_2d(
1574
+ struct lm_ggml_context * ctx,
1575
+ struct lm_ggml_tensor * a,
1576
+ struct lm_ggml_tensor * b,
1577
+ int s0,
1578
+ int s1,
1579
+ int p0,
1580
+ int p1,
1581
+ int d0,
1582
+ int d1);
1583
+
1584
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_conv_1d(
1585
+ struct lm_ggml_context * ctx,
1586
+ struct lm_ggml_tensor * a,
1587
+ struct lm_ggml_tensor * b,
1588
+ int s0, // stride
1589
+ int p0, // padding
1590
+ int d0); // dilation
1591
+
1592
+ // conv_1d with padding = half
1593
+ // alias for lm_ggml_conv_1d(a, b, s, a->ne[0]/2, d)
1594
+ LM_GGML_API struct lm_ggml_tensor* lm_ggml_conv_1d_ph(
1595
+ struct lm_ggml_context * ctx,
1596
+ struct lm_ggml_tensor * a,
1597
+ struct lm_ggml_tensor * b,
1598
+ int s,
1599
+ int d);
1600
+
1601
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_conv_transpose_1d(
1602
+ struct lm_ggml_context * ctx,
1603
+ struct lm_ggml_tensor * a,
1604
+ struct lm_ggml_tensor * b,
1605
+ int s0,
1606
+ int p0,
1607
+ int d0);
1608
+
1609
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_conv_2d(
1610
+ struct lm_ggml_context * ctx,
1611
+ struct lm_ggml_tensor * a,
1612
+ struct lm_ggml_tensor * b,
1613
+ int s0,
1614
+ int s1,
1615
+ int p0,
1616
+ int p1,
1617
+ int d0,
1618
+ int d1);
1619
+
1620
+
1621
+ // kernel size is a->ne[0] x a->ne[1]
1622
+ // stride is equal to kernel size
1623
+ // padding is zero
1624
+ // example:
1625
+ // a: 16 16 3 768
1626
+ // b: 1024 1024 3 1
1627
+ // res: 64 64 768 1
1628
+ // used in sam
1629
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_conv_2d_sk_p0(
1630
+ struct lm_ggml_context * ctx,
1631
+ struct lm_ggml_tensor * a,
1632
+ struct lm_ggml_tensor * b);
1633
+
1634
+ // kernel size is a->ne[0] x a->ne[1]
1635
+ // stride is 1
1636
+ // padding is half
1637
+ // example:
1638
+ // a: 3 3 256 256
1639
+ // b: 64 64 256 1
1640
+ // res: 64 64 256 1
1641
+ // used in sam
1642
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_conv_2d_s1_ph(
1643
+ struct lm_ggml_context * ctx,
1644
+ struct lm_ggml_tensor * a,
1645
+ struct lm_ggml_tensor * b);
1646
+
1647
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_conv_transpose_2d_p0(
1648
+ struct lm_ggml_context * ctx,
1649
+ struct lm_ggml_tensor * a,
1650
+ struct lm_ggml_tensor * b,
1651
+ int stride);
1652
+
1653
+ enum lm_ggml_op_pool {
1654
+ LM_GGML_OP_POOL_MAX,
1655
+ LM_GGML_OP_POOL_AVG,
1656
+ LM_GGML_OP_POOL_COUNT,
1657
+ };
1658
+
1659
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_pool_1d(
1660
+ struct lm_ggml_context * ctx,
1661
+ struct lm_ggml_tensor * a,
1662
+ enum lm_ggml_op_pool op,
1663
+ int k0, // kernel size
1664
+ int s0, // stride
1665
+ int p0); // padding
1666
+
1667
+ // the result will have 2*p0 padding for the first dimension
1668
+ // and 2*p1 padding for the second dimension
1669
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_pool_2d(
1670
+ struct lm_ggml_context * ctx,
1671
+ struct lm_ggml_tensor * a,
1672
+ enum lm_ggml_op_pool op,
1673
+ int k0,
1674
+ int k1,
1675
+ int s0,
1676
+ int s1,
1677
+ float p0,
1678
+ float p1);
1679
+
1680
+ // nearest interpolate
1681
+ // multiplies ne0 and ne1 by scale factor
1682
+ // used in stable-diffusion
1683
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_upscale(
1684
+ struct lm_ggml_context * ctx,
1685
+ struct lm_ggml_tensor * a,
1686
+ int scale_factor);
1687
+
1688
+ // nearest interpolate
1689
+ // nearest interpolate to specified dimensions
1690
+ // used in tortoise.cpp
1691
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_upscale_ext(
1692
+ struct lm_ggml_context * ctx,
1693
+ struct lm_ggml_tensor * a,
1694
+ int ne0,
1695
+ int ne1,
1696
+ int ne2,
1697
+ int ne3);
1698
+
1699
+ // pad each dimension with zeros: [x, ..., x] -> [x, ..., x, 0, ..., 0]
1700
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_pad(
1701
+ struct lm_ggml_context * ctx,
1702
+ struct lm_ggml_tensor * a,
1703
+ int p0,
1704
+ int p1,
1705
+ int p2,
1706
+ int p3);
1707
+
1708
+ // Ref: https://github.com/CompVis/stable-diffusion/blob/main/ldm/modules/diffusionmodules/util.py#L151
1709
+ // timesteps: [N,]
1710
+ // return: [N, dim]
1711
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_timestep_embedding(
1712
+ struct lm_ggml_context * ctx,
1713
+ struct lm_ggml_tensor * timesteps,
1714
+ int dim,
1715
+ int max_period);
1716
+
1717
+ // sort rows
1718
+ enum lm_ggml_sort_order {
1719
+ LM_GGML_SORT_ORDER_ASC,
1720
+ LM_GGML_SORT_ORDER_DESC,
1721
+ };
1722
+
1723
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_argsort(
1724
+ struct lm_ggml_context * ctx,
1725
+ struct lm_ggml_tensor * a,
1726
+ enum lm_ggml_sort_order order);
1727
+
1728
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_arange(
1729
+ struct lm_ggml_context * ctx,
1730
+ float start,
1731
+ float stop,
1732
+ float step);
1733
+
1734
+ // top k elements per row
1735
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_top_k(
1736
+ struct lm_ggml_context * ctx,
1737
+ struct lm_ggml_tensor * a,
1738
+ int k);
1739
+
1740
+ #define LM_GGML_KQ_MASK_PAD 32
1741
+
1742
+ // q: [n_embd, n_batch, n_head, 1]
1743
+ // k: [n_embd, n_kv, n_head_kv, 1]
1744
+ // v: [n_embd, n_kv, n_head_kv, 1] !! not transposed !!
1745
+ // mask: [n_kv, n_batch_pad, 1, 1] !! n_batch_pad = LM_GGML_PAD(n_batch, LM_GGML_KQ_MASK_PAD) !!
1746
+ // res: [n_embd, n_head, n_batch, 1] !! permuted !!
1747
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_flash_attn_ext(
1748
+ struct lm_ggml_context * ctx,
1749
+ struct lm_ggml_tensor * q,
1750
+ struct lm_ggml_tensor * k,
1751
+ struct lm_ggml_tensor * v,
1752
+ struct lm_ggml_tensor * mask,
1753
+ float scale,
1754
+ float max_bias);
1755
+
1756
+ LM_GGML_API void lm_ggml_flash_attn_ext_set_prec(
1757
+ struct lm_ggml_tensor * a,
1758
+ enum lm_ggml_prec prec);
1759
+
1760
+ // TODO: needs to be adapted to lm_ggml_flash_attn_ext
1761
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_flash_attn_back(
1762
+ struct lm_ggml_context * ctx,
1763
+ struct lm_ggml_tensor * q,
1764
+ struct lm_ggml_tensor * k,
1765
+ struct lm_ggml_tensor * v,
1766
+ struct lm_ggml_tensor * d,
1767
+ bool masked);
1768
+
1769
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_ssm_conv(
1770
+ struct lm_ggml_context * ctx,
1771
+ struct lm_ggml_tensor * s,
1772
+ struct lm_ggml_tensor * x,
1773
+ struct lm_ggml_tensor * c,
1774
+ struct lm_ggml_tensor * sq);
1775
+
1776
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_ssm_scan(
1777
+ struct lm_ggml_context * ctx,
1778
+ struct lm_ggml_tensor * s,
1779
+ struct lm_ggml_tensor * x,
1780
+ struct lm_ggml_tensor * dt,
1781
+ struct lm_ggml_tensor * A,
1782
+ struct lm_ggml_tensor * B,
1783
+ struct lm_ggml_tensor * C,
1784
+ struct lm_ggml_tensor * sq);
1785
+
1786
+ // partition into non-overlapping windows with padding if needed
1787
+ // example:
1788
+ // a: 768 64 64 1
1789
+ // w: 14
1790
+ // res: 768 14 14 25
1791
+ // used in sam
1792
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_win_part(
1793
+ struct lm_ggml_context * ctx,
1794
+ struct lm_ggml_tensor * a,
1795
+ int w);
1796
+
1797
+ // reverse of lm_ggml_win_part
1798
+ // used in sam
1799
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_win_unpart(
1800
+ struct lm_ggml_context * ctx,
1801
+ struct lm_ggml_tensor * a,
1802
+ int w0,
1803
+ int h0,
1804
+ int w);
1805
+
1806
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_unary(
1807
+ struct lm_ggml_context * ctx,
1808
+ struct lm_ggml_tensor * a,
1809
+ enum lm_ggml_unary_op op);
1810
+
1811
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_unary_inplace(
1812
+ struct lm_ggml_context * ctx,
1813
+ struct lm_ggml_tensor * a,
1814
+ enum lm_ggml_unary_op op);
1815
+
1816
+ // used in sam
1817
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_get_rel_pos(
1818
+ struct lm_ggml_context * ctx,
1819
+ struct lm_ggml_tensor * a,
1820
+ int qh,
1821
+ int kh);
1822
+
1823
+ // used in sam
1824
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_add_rel_pos(
1825
+ struct lm_ggml_context * ctx,
1826
+ struct lm_ggml_tensor * a,
1827
+ struct lm_ggml_tensor * pw,
1828
+ struct lm_ggml_tensor * ph);
1829
+
1830
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_add_rel_pos_inplace(
1831
+ struct lm_ggml_context * ctx,
1832
+ struct lm_ggml_tensor * a,
1833
+ struct lm_ggml_tensor * pw,
1834
+ struct lm_ggml_tensor * ph);
1835
+
1836
+ // custom operators
1837
+
1838
+ typedef void (*lm_ggml_unary_op_f32_t) (const int, float *, const float *);
1839
+ typedef void (*lm_ggml_binary_op_f32_t)(const int, float *, const float *, const float *);
1840
+
1841
+ typedef void (*lm_ggml_custom1_op_f32_t)(struct lm_ggml_tensor *, const struct lm_ggml_tensor *);
1842
+ typedef void (*lm_ggml_custom2_op_f32_t)(struct lm_ggml_tensor *, const struct lm_ggml_tensor *, const struct lm_ggml_tensor *);
1843
+ typedef void (*lm_ggml_custom3_op_f32_t)(struct lm_ggml_tensor *, const struct lm_ggml_tensor *, const struct lm_ggml_tensor *, const struct lm_ggml_tensor *);
1844
+
1845
+ LM_GGML_DEPRECATED(LM_GGML_API struct lm_ggml_tensor * lm_ggml_map_unary_f32(
1846
+ struct lm_ggml_context * ctx,
1847
+ struct lm_ggml_tensor * a,
1848
+ lm_ggml_unary_op_f32_t fun),
1849
+ "use lm_ggml_map_custom1 instead");
1850
+
1851
+ LM_GGML_DEPRECATED(LM_GGML_API struct lm_ggml_tensor * lm_ggml_map_unary_inplace_f32(
1852
+ struct lm_ggml_context * ctx,
1853
+ struct lm_ggml_tensor * a,
1854
+ lm_ggml_unary_op_f32_t fun),
1855
+ "use lm_ggml_map_custom1_inplace instead");
1856
+
1857
+ LM_GGML_DEPRECATED(LM_GGML_API struct lm_ggml_tensor * lm_ggml_map_binary_f32(
1858
+ struct lm_ggml_context * ctx,
1859
+ struct lm_ggml_tensor * a,
1860
+ struct lm_ggml_tensor * b,
1861
+ lm_ggml_binary_op_f32_t fun),
1862
+ "use lm_ggml_map_custom2 instead");
1863
+
1864
+ LM_GGML_DEPRECATED(LM_GGML_API struct lm_ggml_tensor * lm_ggml_map_binary_inplace_f32(
1865
+ struct lm_ggml_context * ctx,
1866
+ struct lm_ggml_tensor * a,
1867
+ struct lm_ggml_tensor * b,
1868
+ lm_ggml_binary_op_f32_t fun),
1869
+ "use lm_ggml_map_custom2_inplace instead");
1870
+
1871
+ LM_GGML_DEPRECATED(LM_GGML_API struct lm_ggml_tensor * lm_ggml_map_custom1_f32(
1872
+ struct lm_ggml_context * ctx,
1873
+ struct lm_ggml_tensor * a,
1874
+ lm_ggml_custom1_op_f32_t fun),
1875
+ "use lm_ggml_map_custom1 instead");
1876
+
1877
+ LM_GGML_DEPRECATED(LM_GGML_API struct lm_ggml_tensor * lm_ggml_map_custom1_inplace_f32(
1878
+ struct lm_ggml_context * ctx,
1879
+ struct lm_ggml_tensor * a,
1880
+ lm_ggml_custom1_op_f32_t fun),
1881
+ "use lm_ggml_map_custom1_inplace instead");
1882
+
1883
+ LM_GGML_DEPRECATED(LM_GGML_API struct lm_ggml_tensor * lm_ggml_map_custom2_f32(
1884
+ struct lm_ggml_context * ctx,
1885
+ struct lm_ggml_tensor * a,
1886
+ struct lm_ggml_tensor * b,
1887
+ lm_ggml_custom2_op_f32_t fun),
1888
+ "use lm_ggml_map_custom2 instead");
1889
+
1890
+ LM_GGML_DEPRECATED(LM_GGML_API struct lm_ggml_tensor * lm_ggml_map_custom2_inplace_f32(
1891
+ struct lm_ggml_context * ctx,
1892
+ struct lm_ggml_tensor * a,
1893
+ struct lm_ggml_tensor * b,
1894
+ lm_ggml_custom2_op_f32_t fun),
1895
+ "use lm_ggml_map_custom2_inplace instead");
1896
+
1897
+ LM_GGML_DEPRECATED(LM_GGML_API struct lm_ggml_tensor * lm_ggml_map_custom3_f32(
1898
+ struct lm_ggml_context * ctx,
1899
+ struct lm_ggml_tensor * a,
1900
+ struct lm_ggml_tensor * b,
1901
+ struct lm_ggml_tensor * c,
1902
+ lm_ggml_custom3_op_f32_t fun),
1903
+ "use lm_ggml_map_custom3 instead");
1904
+
1905
+ LM_GGML_DEPRECATED(LM_GGML_API struct lm_ggml_tensor * lm_ggml_map_custom3_inplace_f32(
1906
+ struct lm_ggml_context * ctx,
1907
+ struct lm_ggml_tensor * a,
1908
+ struct lm_ggml_tensor * b,
1909
+ struct lm_ggml_tensor * c,
1910
+ lm_ggml_custom3_op_f32_t fun),
1911
+ "use lm_ggml_map_custom3_inplace instead");
1912
+
1913
+ // custom operators v2
1914
+
1915
+ typedef void (*lm_ggml_custom1_op_t)(struct lm_ggml_tensor * dst , const struct lm_ggml_tensor * a, int ith, int nth, void * userdata);
1916
+ typedef void (*lm_ggml_custom2_op_t)(struct lm_ggml_tensor * dst , const struct lm_ggml_tensor * a, const struct lm_ggml_tensor * b, int ith, int nth, void * userdata);
1917
+ typedef void (*lm_ggml_custom3_op_t)(struct lm_ggml_tensor * dst , const struct lm_ggml_tensor * a, const struct lm_ggml_tensor * b, const struct lm_ggml_tensor * c, int ith, int nth, void * userdata);
1918
+
1919
+ #define LM_GGML_N_TASKS_MAX -1
1920
+
1921
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_map_custom1(
1922
+ struct lm_ggml_context * ctx,
1923
+ struct lm_ggml_tensor * a,
1924
+ lm_ggml_custom1_op_t fun,
1925
+ int n_tasks,
1926
+ void * userdata);
1927
+
1928
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_map_custom1_inplace(
1929
+ struct lm_ggml_context * ctx,
1930
+ struct lm_ggml_tensor * a,
1931
+ lm_ggml_custom1_op_t fun,
1932
+ int n_tasks,
1933
+ void * userdata);
1934
+
1935
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_map_custom2(
1936
+ struct lm_ggml_context * ctx,
1937
+ struct lm_ggml_tensor * a,
1938
+ struct lm_ggml_tensor * b,
1939
+ lm_ggml_custom2_op_t fun,
1940
+ int n_tasks,
1941
+ void * userdata);
1942
+
1943
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_map_custom2_inplace(
1944
+ struct lm_ggml_context * ctx,
1945
+ struct lm_ggml_tensor * a,
1946
+ struct lm_ggml_tensor * b,
1947
+ lm_ggml_custom2_op_t fun,
1948
+ int n_tasks,
1949
+ void * userdata);
1950
+
1951
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_map_custom3(
1952
+ struct lm_ggml_context * ctx,
1953
+ struct lm_ggml_tensor * a,
1954
+ struct lm_ggml_tensor * b,
1955
+ struct lm_ggml_tensor * c,
1956
+ lm_ggml_custom3_op_t fun,
1957
+ int n_tasks,
1958
+ void * userdata);
1959
+
1960
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_map_custom3_inplace(
1961
+ struct lm_ggml_context * ctx,
1962
+ struct lm_ggml_tensor * a,
1963
+ struct lm_ggml_tensor * b,
1964
+ struct lm_ggml_tensor * c,
1965
+ lm_ggml_custom3_op_t fun,
1966
+ int n_tasks,
1967
+ void * userdata);
1968
+
1969
+ // loss function
1970
+
1971
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_cross_entropy_loss(
1972
+ struct lm_ggml_context * ctx,
1973
+ struct lm_ggml_tensor * a,
1974
+ struct lm_ggml_tensor * b);
1975
+
1976
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_cross_entropy_loss_back(
1977
+ struct lm_ggml_context * ctx,
1978
+ struct lm_ggml_tensor * a,
1979
+ struct lm_ggml_tensor * b,
1980
+ struct lm_ggml_tensor * c);
1981
+
1982
+ //
1983
+ // automatic differentiation
1984
+ //
1985
+
1986
+ LM_GGML_API void lm_ggml_set_param(
1987
+ struct lm_ggml_context * ctx,
1988
+ struct lm_ggml_tensor * tensor);
1989
+
1990
+
1991
+ LM_GGML_API void lm_ggml_build_forward_expand (struct lm_ggml_cgraph * cgraph, struct lm_ggml_tensor * tensor);
1992
+ LM_GGML_API void lm_ggml_build_backward_expand(struct lm_ggml_context * ctx, struct lm_ggml_cgraph * gf, struct lm_ggml_cgraph * gb, bool keep);
1993
+
1994
+ // graph allocation in a context
1995
+ LM_GGML_API struct lm_ggml_cgraph * lm_ggml_new_graph (struct lm_ggml_context * ctx); // size = LM_GGML_DEFAULT_GRAPH_SIZE, grads = false
1996
+ LM_GGML_API struct lm_ggml_cgraph * lm_ggml_new_graph_custom (struct lm_ggml_context * ctx, size_t size, bool grads);
1997
+ LM_GGML_API struct lm_ggml_cgraph * lm_ggml_graph_dup (struct lm_ggml_context * ctx, struct lm_ggml_cgraph * cgraph);
1998
+ LM_GGML_API struct lm_ggml_cgraph lm_ggml_graph_view (struct lm_ggml_cgraph * cgraph, int i0, int i1);
1999
+ LM_GGML_API void lm_ggml_graph_cpy (struct lm_ggml_cgraph * src, struct lm_ggml_cgraph * dst);
2000
+ LM_GGML_API void lm_ggml_graph_reset (struct lm_ggml_cgraph * cgraph); // zero grads
2001
+ LM_GGML_API void lm_ggml_graph_clear (struct lm_ggml_cgraph * cgraph);
2002
+
2003
+ LM_GGML_API size_t lm_ggml_graph_overhead(void);
2004
+ LM_GGML_API size_t lm_ggml_graph_overhead_custom(size_t size, bool grads);
2005
+
2006
+ // lm_ggml_graph_plan() has to be called before lm_ggml_graph_compute()
2007
+ // when plan.work_size > 0, caller must allocate memory for plan.work_data
2008
+ LM_GGML_API struct lm_ggml_cplan lm_ggml_graph_plan (const struct lm_ggml_cgraph * cgraph, int n_threads /*= LM_GGML_DEFAULT_N_THREADS*/);
2009
+ LM_GGML_API enum lm_ggml_status lm_ggml_graph_compute ( struct lm_ggml_cgraph * cgraph, struct lm_ggml_cplan * cplan);
2010
+ // same as lm_ggml_graph_compute() but the work data is allocated as a part of the context
2011
+ // note: the drawback of this API is that you must have ensured that the context has enough memory for the work data
2012
+ LM_GGML_API enum lm_ggml_status lm_ggml_graph_compute_with_ctx(struct lm_ggml_context * ctx, struct lm_ggml_cgraph * cgraph, int n_threads);
2013
+
2014
+ LM_GGML_API struct lm_ggml_tensor * lm_ggml_graph_get_tensor(struct lm_ggml_cgraph * cgraph, const char * name);
2015
+
2016
+ LM_GGML_API void lm_ggml_graph_export(const struct lm_ggml_cgraph * cgraph, const char * fname);
2017
+ LM_GGML_API struct lm_ggml_cgraph * lm_ggml_graph_import(const char * fname, struct lm_ggml_context ** ctx_data, struct lm_ggml_context ** ctx_eval);
2018
+
2019
+ // print info and performance information for the graph
2020
+ LM_GGML_API void lm_ggml_graph_print(const struct lm_ggml_cgraph * cgraph);
2021
+
2022
+ // dump the graph into a file using the dot format
2023
+ LM_GGML_API void lm_ggml_graph_dump_dot(const struct lm_ggml_cgraph * gb, const struct lm_ggml_cgraph * gf, const char * filename);
2024
+
2025
+ // build gradient checkpointing backward graph gb for gf using provided checkpoints
2026
+ // gb_tmp will contain original backward graph with rewritten backward process nodes,
2027
+ // but without the second forward pass nodes.
2028
+ LM_GGML_API void lm_ggml_build_backward_gradient_checkpointing(
2029
+ struct lm_ggml_context * ctx,
2030
+ struct lm_ggml_cgraph * gf,
2031
+ struct lm_ggml_cgraph * gb,
2032
+ struct lm_ggml_cgraph * gb_tmp,
2033
+ struct lm_ggml_tensor * * checkpoints,
2034
+ int n_checkpoints);
2035
+ //
2036
+ // optimization
2037
+ //
2038
+
2039
+ // optimization methods
2040
+ enum lm_ggml_opt_type {
2041
+ LM_GGML_OPT_TYPE_ADAM,
2042
+ LM_GGML_OPT_TYPE_LBFGS,
2043
+ };
2044
+
2045
+ // linesearch methods
2046
+ enum lm_ggml_linesearch {
2047
+ LM_GGML_LINESEARCH_DEFAULT = 1,
2048
+
2049
+ LM_GGML_LINESEARCH_BACKTRACKING_ARMIJO = 0,
2050
+ LM_GGML_LINESEARCH_BACKTRACKING_WOLFE = 1,
2051
+ LM_GGML_LINESEARCH_BACKTRACKING_STRONG_WOLFE = 2,
2052
+ };
2053
+
2054
+ // optimization return values
2055
+ enum lm_ggml_opt_result {
2056
+ LM_GGML_OPT_RESULT_OK = 0,
2057
+ LM_GGML_OPT_RESULT_DID_NOT_CONVERGE,
2058
+ LM_GGML_OPT_RESULT_NO_CONTEXT,
2059
+ LM_GGML_OPT_RESULT_INVALID_WOLFE,
2060
+ LM_GGML_OPT_RESULT_FAIL,
2061
+ LM_GGML_OPT_RESULT_CANCEL,
2062
+
2063
+ LM_GGML_LINESEARCH_FAIL = -128,
2064
+ LM_GGML_LINESEARCH_MINIMUM_STEP,
2065
+ LM_GGML_LINESEARCH_MAXIMUM_STEP,
2066
+ LM_GGML_LINESEARCH_MAXIMUM_ITERATIONS,
2067
+ LM_GGML_LINESEARCH_INVALID_PARAMETERS,
2068
+ };
2069
+
2070
+ typedef void (*lm_ggml_opt_callback)(void * data, int accum_step, float * sched, bool * cancel);
2071
+ typedef void (*lm_ggml_log_callback)(enum lm_ggml_log_level level, const char * text, void * user_data);
2072
+
2073
+ // optimization parameters
2074
+ //
2075
+ // see ggml.c (lm_ggml_opt_default_params) for default values
2076
+ //
2077
+ struct lm_ggml_opt_params {
2078
+ enum lm_ggml_opt_type type;
2079
+
2080
+ size_t graph_size;
2081
+
2082
+ int n_threads;
2083
+
2084
+ // delta-based convergence test
2085
+ //
2086
+ // if past == 0 - disabled
2087
+ // if past > 0:
2088
+ // stop if |f(x) - f(x_past)| < delta * max(1, |f(x)|)
2089
+ //
2090
+ int past;
2091
+ float delta;
2092
+
2093
+ // maximum number of iterations without improvement
2094
+ //
2095
+ // if 0 - disabled
2096
+ // if > 0:
2097
+ // assume convergence if no cost improvement in this number of iterations
2098
+ //
2099
+ int max_no_improvement;
2100
+
2101
+ bool print_forward_graph;
2102
+ bool print_backward_graph;
2103
+
2104
+ int n_gradient_accumulation;
2105
+
2106
+ // ADAM parameters
2107
+ struct {
2108
+ int n_iter;
2109
+
2110
+ float sched; // schedule multiplier (fixed, decay or warmup)
2111
+ float decay; // weight decay for AdamW, use 0.0f to disable
2112
+ int decay_min_ndim; // minimum number of tensor dimension to apply weight decay
2113
+ float alpha; // learning rate
2114
+ float beta1;
2115
+ float beta2;
2116
+ float eps; // epsilon for numerical stability
2117
+ float eps_f; // epsilon for convergence test
2118
+ float eps_g; // epsilon for convergence test
2119
+ float gclip; // gradient clipping
2120
+ } adam;
2121
+
2122
+ // LBFGS parameters
2123
+ struct {
2124
+ int m; // number of corrections to approximate the inv. Hessian
2125
+ int n_iter;
2126
+ int max_linesearch;
2127
+
2128
+ float eps; // convergence tolerance
2129
+ float ftol; // line search tolerance
2130
+ float wolfe;
2131
+ float min_step;
2132
+ float max_step;
2133
+
2134
+ enum lm_ggml_linesearch linesearch;
2135
+ } lbfgs;
2136
+ };
2137
+
2138
+ struct lm_ggml_opt_context {
2139
+ struct lm_ggml_context * ctx;
2140
+ struct lm_ggml_opt_params params;
2141
+
2142
+ int iter;
2143
+ int64_t nx; // number of parameter elements
2144
+
2145
+ bool just_initialized;
2146
+
2147
+ float loss_before;
2148
+ float loss_after;
2149
+
2150
+ struct {
2151
+ struct lm_ggml_tensor * g; // current gradient
2152
+ struct lm_ggml_tensor * m; // first moment
2153
+ struct lm_ggml_tensor * v; // second moment
2154
+ struct lm_ggml_tensor * pf; // past function values
2155
+ float fx_best;
2156
+ float fx_prev;
2157
+ int n_no_improvement;
2158
+ } adam;
2159
+
2160
+ struct {
2161
+ struct lm_ggml_tensor * x; // current parameters
2162
+ struct lm_ggml_tensor * xp; // previous parameters
2163
+ struct lm_ggml_tensor * g; // current gradient
2164
+ struct lm_ggml_tensor * gp; // previous gradient
2165
+ struct lm_ggml_tensor * d; // search direction
2166
+ struct lm_ggml_tensor * pf; // past function values
2167
+ struct lm_ggml_tensor * lmal; // the L-BFGS memory alpha
2168
+ struct lm_ggml_tensor * lmys; // the L-BFGS memory ys
2169
+ struct lm_ggml_tensor * lms; // the L-BFGS memory s
2170
+ struct lm_ggml_tensor * lmy; // the L-BFGS memory y
2171
+ float fx_best;
2172
+ float step;
2173
+ int j;
2174
+ int k;
2175
+ int end;
2176
+ int n_no_improvement;
2177
+ } lbfgs;
2178
+ };
2179
+
2180
+ LM_GGML_API struct lm_ggml_opt_params lm_ggml_opt_default_params(enum lm_ggml_opt_type type);
2181
+
2182
+ // optimize the function defined by the tensor f
2183
+ LM_GGML_API enum lm_ggml_opt_result lm_ggml_opt(
2184
+ struct lm_ggml_context * ctx,
2185
+ struct lm_ggml_opt_params params,
2186
+ struct lm_ggml_tensor * f);
2187
+
2188
+ // initialize optimizer context
2189
+ LM_GGML_API void lm_ggml_opt_init(
2190
+ struct lm_ggml_context * ctx,
2191
+ struct lm_ggml_opt_context * opt,
2192
+ struct lm_ggml_opt_params params,
2193
+ int64_t nx);
2194
+
2195
+ // continue optimizing the function defined by the tensor f
2196
+ LM_GGML_API enum lm_ggml_opt_result lm_ggml_opt_resume(
2197
+ struct lm_ggml_context * ctx,
2198
+ struct lm_ggml_opt_context * opt,
2199
+ struct lm_ggml_tensor * f);
2200
+
2201
+ // continue optimizing the function defined by the tensor f
2202
+ LM_GGML_API enum lm_ggml_opt_result lm_ggml_opt_resume_g(
2203
+ struct lm_ggml_context * ctx,
2204
+ struct lm_ggml_opt_context * opt,
2205
+ struct lm_ggml_tensor * f,
2206
+ struct lm_ggml_cgraph * gf,
2207
+ struct lm_ggml_cgraph * gb,
2208
+ lm_ggml_opt_callback callback,
2209
+ void * callback_data);
2210
+
2211
+ //
2212
+ // tensor flags
2213
+ //
2214
+ LM_GGML_API void lm_ggml_set_input(struct lm_ggml_tensor * tensor);
2215
+ LM_GGML_API void lm_ggml_set_output(struct lm_ggml_tensor * tensor);
2216
+
2217
+ //
2218
+ // quantization
2219
+ //
2220
+
2221
+ // - lm_ggml_quantize_init can be called multiple times with the same type
2222
+ // it will only initialize the quantization tables for the first call or after lm_ggml_quantize_free
2223
+ // automatically called by lm_ggml_quantize_chunk for convenience
2224
+ //
2225
+ // - lm_ggml_quantize_free will free any memory allocated by lm_ggml_quantize_init
2226
+ // call this at the end of the program to avoid memory leaks
2227
+ //
2228
+ // note: these are thread-safe
2229
+ //
2230
+ LM_GGML_API void lm_ggml_quantize_init(enum lm_ggml_type type);
2231
+ LM_GGML_API void lm_ggml_quantize_free(void);
2232
+
2233
+ // some quantization type cannot be used without an importance matrix
2234
+ LM_GGML_API bool lm_ggml_quantize_requires_imatrix(enum lm_ggml_type type);
2235
+
2236
+ // calls lm_ggml_quantize_init internally (i.e. can allocate memory)
2237
+ LM_GGML_API size_t lm_ggml_quantize_chunk(
2238
+ enum lm_ggml_type type,
2239
+ const float * src,
2240
+ void * dst,
2241
+ int64_t start,
2242
+ int64_t nrows,
2243
+ int64_t n_per_row,
2244
+ const float * imatrix);
2245
+
2246
+ //
2247
+ // gguf
2248
+ //
2249
+
2250
+ enum lm_gguf_type {
2251
+ LM_GGUF_TYPE_UINT8 = 0,
2252
+ LM_GGUF_TYPE_INT8 = 1,
2253
+ LM_GGUF_TYPE_UINT16 = 2,
2254
+ LM_GGUF_TYPE_INT16 = 3,
2255
+ LM_GGUF_TYPE_UINT32 = 4,
2256
+ LM_GGUF_TYPE_INT32 = 5,
2257
+ LM_GGUF_TYPE_FLOAT32 = 6,
2258
+ LM_GGUF_TYPE_BOOL = 7,
2259
+ LM_GGUF_TYPE_STRING = 8,
2260
+ LM_GGUF_TYPE_ARRAY = 9,
2261
+ LM_GGUF_TYPE_UINT64 = 10,
2262
+ LM_GGUF_TYPE_INT64 = 11,
2263
+ LM_GGUF_TYPE_FLOAT64 = 12,
2264
+ LM_GGUF_TYPE_COUNT, // marks the end of the enum
2265
+ };
2266
+
2267
+ struct lm_gguf_context;
2268
+
2269
+ struct lm_gguf_init_params {
2270
+ bool no_alloc;
2271
+
2272
+ // if not NULL, create a lm_ggml_context and allocate the tensor data in it
2273
+ struct lm_ggml_context ** ctx;
2274
+ };
2275
+
2276
+ LM_GGML_API struct lm_gguf_context * lm_gguf_init_empty(void);
2277
+ LM_GGML_API struct lm_gguf_context * lm_gguf_init_from_file(const char * fname, struct lm_gguf_init_params params);
2278
+ //LM_GGML_API struct lm_gguf_context * lm_gguf_init_from_buffer(..);
2279
+
2280
+ LM_GGML_API void lm_gguf_free(struct lm_gguf_context * ctx);
2281
+
2282
+ LM_GGML_API const char * lm_gguf_type_name(enum lm_gguf_type type);
2283
+
2284
+ LM_GGML_API int lm_gguf_get_version (const struct lm_gguf_context * ctx);
2285
+ LM_GGML_API size_t lm_gguf_get_alignment (const struct lm_gguf_context * ctx);
2286
+ LM_GGML_API size_t lm_gguf_get_data_offset(const struct lm_gguf_context * ctx);
2287
+ LM_GGML_API void * lm_gguf_get_data (const struct lm_gguf_context * ctx);
2288
+
2289
+ LM_GGML_API int lm_gguf_get_n_kv(const struct lm_gguf_context * ctx);
2290
+ LM_GGML_API int lm_gguf_find_key(const struct lm_gguf_context * ctx, const char * key);
2291
+ LM_GGML_API const char * lm_gguf_get_key (const struct lm_gguf_context * ctx, int key_id);
2292
+
2293
+ LM_GGML_API enum lm_gguf_type lm_gguf_get_kv_type (const struct lm_gguf_context * ctx, int key_id);
2294
+ LM_GGML_API enum lm_gguf_type lm_gguf_get_arr_type(const struct lm_gguf_context * ctx, int key_id);
2295
+
2296
+ // will abort if the wrong type is used for the key
2297
+ LM_GGML_API uint8_t lm_gguf_get_val_u8 (const struct lm_gguf_context * ctx, int key_id);
2298
+ LM_GGML_API int8_t lm_gguf_get_val_i8 (const struct lm_gguf_context * ctx, int key_id);
2299
+ LM_GGML_API uint16_t lm_gguf_get_val_u16 (const struct lm_gguf_context * ctx, int key_id);
2300
+ LM_GGML_API int16_t lm_gguf_get_val_i16 (const struct lm_gguf_context * ctx, int key_id);
2301
+ LM_GGML_API uint32_t lm_gguf_get_val_u32 (const struct lm_gguf_context * ctx, int key_id);
2302
+ LM_GGML_API int32_t lm_gguf_get_val_i32 (const struct lm_gguf_context * ctx, int key_id);
2303
+ LM_GGML_API float lm_gguf_get_val_f32 (const struct lm_gguf_context * ctx, int key_id);
2304
+ LM_GGML_API uint64_t lm_gguf_get_val_u64 (const struct lm_gguf_context * ctx, int key_id);
2305
+ LM_GGML_API int64_t lm_gguf_get_val_i64 (const struct lm_gguf_context * ctx, int key_id);
2306
+ LM_GGML_API double lm_gguf_get_val_f64 (const struct lm_gguf_context * ctx, int key_id);
2307
+ LM_GGML_API bool lm_gguf_get_val_bool(const struct lm_gguf_context * ctx, int key_id);
2308
+ LM_GGML_API const char * lm_gguf_get_val_str (const struct lm_gguf_context * ctx, int key_id);
2309
+ LM_GGML_API const void * lm_gguf_get_val_data(const struct lm_gguf_context * ctx, int key_id);
2310
+ LM_GGML_API int lm_gguf_get_arr_n (const struct lm_gguf_context * ctx, int key_id);
2311
+ LM_GGML_API const void * lm_gguf_get_arr_data(const struct lm_gguf_context * ctx, int key_id);
2312
+ LM_GGML_API const char * lm_gguf_get_arr_str (const struct lm_gguf_context * ctx, int key_id, int i);
2313
+
2314
+ LM_GGML_API int lm_gguf_get_n_tensors (const struct lm_gguf_context * ctx);
2315
+ LM_GGML_API int lm_gguf_find_tensor (const struct lm_gguf_context * ctx, const char * name);
2316
+ LM_GGML_API size_t lm_gguf_get_tensor_offset(const struct lm_gguf_context * ctx, int i);
2317
+ LM_GGML_API char * lm_gguf_get_tensor_name (const struct lm_gguf_context * ctx, int i);
2318
+ LM_GGML_API enum lm_ggml_type lm_gguf_get_tensor_type (const struct lm_gguf_context * ctx, int i);
2319
+
2320
+ // removes key if it exists
2321
+ LM_GGML_API void lm_gguf_remove_key(struct lm_gguf_context * ctx, const char * key);
2322
+
2323
+ // overrides existing values or adds a new one
2324
+ LM_GGML_API void lm_gguf_set_val_u8 (struct lm_gguf_context * ctx, const char * key, uint8_t val);
2325
+ LM_GGML_API void lm_gguf_set_val_i8 (struct lm_gguf_context * ctx, const char * key, int8_t val);
2326
+ LM_GGML_API void lm_gguf_set_val_u16 (struct lm_gguf_context * ctx, const char * key, uint16_t val);
2327
+ LM_GGML_API void lm_gguf_set_val_i16 (struct lm_gguf_context * ctx, const char * key, int16_t val);
2328
+ LM_GGML_API void lm_gguf_set_val_u32 (struct lm_gguf_context * ctx, const char * key, uint32_t val);
2329
+ LM_GGML_API void lm_gguf_set_val_i32 (struct lm_gguf_context * ctx, const char * key, int32_t val);
2330
+ LM_GGML_API void lm_gguf_set_val_f32 (struct lm_gguf_context * ctx, const char * key, float val);
2331
+ LM_GGML_API void lm_gguf_set_val_u64 (struct lm_gguf_context * ctx, const char * key, uint64_t val);
2332
+ LM_GGML_API void lm_gguf_set_val_i64 (struct lm_gguf_context * ctx, const char * key, int64_t val);
2333
+ LM_GGML_API void lm_gguf_set_val_f64 (struct lm_gguf_context * ctx, const char * key, double val);
2334
+ LM_GGML_API void lm_gguf_set_val_bool(struct lm_gguf_context * ctx, const char * key, bool val);
2335
+ LM_GGML_API void lm_gguf_set_val_str (struct lm_gguf_context * ctx, const char * key, const char * val);
2336
+ LM_GGML_API void lm_gguf_set_arr_data(struct lm_gguf_context * ctx, const char * key, enum lm_gguf_type type, const void * data, int n);
2337
+ LM_GGML_API void lm_gguf_set_arr_str (struct lm_gguf_context * ctx, const char * key, const char ** data, int n);
2338
+
2339
+ // set or add KV pairs from another context
2340
+ LM_GGML_API void lm_gguf_set_kv(struct lm_gguf_context * ctx, struct lm_gguf_context * src);
2341
+
2342
+ // manage tensor info
2343
+ LM_GGML_API void lm_gguf_add_tensor(struct lm_gguf_context * ctx, const struct lm_ggml_tensor * tensor);
2344
+ LM_GGML_API void lm_gguf_set_tensor_type(struct lm_gguf_context * ctx, const char * name, enum lm_ggml_type type);
2345
+ LM_GGML_API void lm_gguf_set_tensor_data(struct lm_gguf_context * ctx, const char * name, const void * data, size_t size);
2346
+
2347
+ // writing gguf files can be done in 2 ways:
2348
+ //
2349
+ // - write the entire lm_gguf_context to a binary file in a single pass:
2350
+ //
2351
+ // lm_gguf_write_to_file(ctx, fname);
2352
+ //
2353
+ // - first prepare a file with a placeholder for the meta data, write the tensor data, then write the meta data:
2354
+ //
2355
+ // FILE * f = fopen(fname, "wb");
2356
+ // fseek(f, lm_gguf_get_meta_size(ctx), SEEK_SET);
2357
+ // fwrite(f, ...);
2358
+ // void * data = lm_gguf_meta_get_meta_data(ctx);
2359
+ // fseek(f, 0, SEEK_SET);
2360
+ // fwrite(f, data, lm_gguf_get_meta_size(ctx));
2361
+ // free(data);
2362
+ // fclose(f);
2363
+ //
2364
+
2365
+ // write the entire context to a binary file
2366
+ LM_GGML_API void lm_gguf_write_to_file(const struct lm_gguf_context * ctx, const char * fname, bool only_meta);
2367
+
2368
+ // get the size in bytes of the meta data (header, kv pairs, tensor info) including padding
2369
+ LM_GGML_API size_t lm_gguf_get_meta_size(const struct lm_gguf_context * ctx);
2370
+ LM_GGML_API void lm_gguf_get_meta_data(const struct lm_gguf_context * ctx, void * data);
2371
+
2372
+ //
2373
+ // system info
2374
+ //
2375
+
2376
+ LM_GGML_API int lm_ggml_cpu_has_avx (void);
2377
+ LM_GGML_API int lm_ggml_cpu_has_avx_vnni (void);
2378
+ LM_GGML_API int lm_ggml_cpu_has_avx2 (void);
2379
+ LM_GGML_API int lm_ggml_cpu_has_avx512 (void);
2380
+ LM_GGML_API int lm_ggml_cpu_has_avx512_vbmi(void);
2381
+ LM_GGML_API int lm_ggml_cpu_has_avx512_vnni(void);
2382
+ LM_GGML_API int lm_ggml_cpu_has_avx512_bf16(void);
2383
+ LM_GGML_API int lm_ggml_cpu_has_fma (void);
2384
+ LM_GGML_API int lm_ggml_cpu_has_neon (void);
2385
+ LM_GGML_API int lm_ggml_cpu_has_sve (void);
2386
+ LM_GGML_API int lm_ggml_cpu_has_arm_fma (void);
2387
+ LM_GGML_API int lm_ggml_cpu_has_metal (void);
2388
+ LM_GGML_API int lm_ggml_cpu_has_f16c (void);
2389
+ LM_GGML_API int lm_ggml_cpu_has_fp16_va (void);
2390
+ LM_GGML_API int lm_ggml_cpu_has_wasm_simd (void);
2391
+ LM_GGML_API int lm_ggml_cpu_has_blas (void);
2392
+ LM_GGML_API int lm_ggml_cpu_has_cuda (void);
2393
+ LM_GGML_API int lm_ggml_cpu_has_vulkan (void);
2394
+ LM_GGML_API int lm_ggml_cpu_has_kompute (void);
2395
+ LM_GGML_API int lm_ggml_cpu_has_gpublas (void);
2396
+ LM_GGML_API int lm_ggml_cpu_has_sse3 (void);
2397
+ LM_GGML_API int lm_ggml_cpu_has_ssse3 (void);
2398
+ LM_GGML_API int lm_ggml_cpu_has_sycl (void);
2399
+ LM_GGML_API int lm_ggml_cpu_has_rpc (void);
2400
+ LM_GGML_API int lm_ggml_cpu_has_vsx (void);
2401
+ LM_GGML_API int lm_ggml_cpu_has_matmul_int8(void);
2402
+ LM_GGML_API int lm_ggml_cpu_has_cann (void);
2403
+
2404
+ //
2405
+ // Internal types and functions exposed for tests and benchmarks
2406
+ //
2407
+
2408
+ #ifdef __cplusplus
2409
+ // restrict not standard in C++
2410
+ #define LM_GGML_RESTRICT
2411
+ #else
2412
+ #define LM_GGML_RESTRICT restrict
2413
+ #endif
2414
+ typedef void (*lm_ggml_to_float_t) (const void * LM_GGML_RESTRICT x, float * LM_GGML_RESTRICT y, int64_t k);
2415
+ typedef void (*lm_ggml_from_float_t)(const float * LM_GGML_RESTRICT x, void * LM_GGML_RESTRICT y, int64_t k);
2416
+ typedef void (*lm_ggml_from_float_to_mat_t)
2417
+ (const float * LM_GGML_RESTRICT x, void * LM_GGML_RESTRICT y, int64_t nr, int64_t k, int64_t bs);
2418
+ typedef void (*lm_ggml_vec_dot_t) (int n, float * LM_GGML_RESTRICT s, size_t bs, const void * LM_GGML_RESTRICT x, size_t bx,
2419
+ const void * LM_GGML_RESTRICT y, size_t by, int nrc);
2420
+ typedef void (*lm_ggml_gemv_t) (int n, float * LM_GGML_RESTRICT s, size_t bs, const void * LM_GGML_RESTRICT x,
2421
+ const void * LM_GGML_RESTRICT y, int nr, int nc);
2422
+ typedef void (*lm_ggml_gemm_t) (int n, float * LM_GGML_RESTRICT s, size_t bs, const void * LM_GGML_RESTRICT x,
2423
+ const void * LM_GGML_RESTRICT y, int nr, int nc);
2424
+
2425
+ typedef struct {
2426
+ const char * type_name;
2427
+ int64_t blck_size;
2428
+ int64_t blck_size_interleave; // interleave elements in blocks
2429
+ size_t type_size;
2430
+ bool is_quantized;
2431
+ lm_ggml_to_float_t to_float;
2432
+ lm_ggml_from_float_t from_float;
2433
+ lm_ggml_from_float_t from_float_ref;
2434
+ lm_ggml_from_float_to_mat_t from_float_to_mat;
2435
+ lm_ggml_vec_dot_t vec_dot;
2436
+ enum lm_ggml_type vec_dot_type;
2437
+ int64_t nrows; // number of rows to process simultaneously
2438
+ int64_t ncols; // number of columns to process simultaneously
2439
+ lm_ggml_gemv_t gemv;
2440
+ lm_ggml_gemm_t gemm;
2441
+ } lm_ggml_type_traits_t;
2442
+
2443
+ LM_GGML_API lm_ggml_type_traits_t lm_ggml_internal_get_type_traits(enum lm_ggml_type type);
2444
+
2445
+ #ifdef __cplusplus
2446
+ }
2447
+ #endif