cui-llama.rn 1.0.2 → 1.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (26) hide show
  1. package/android/src/main/CMakeLists.txt +22 -19
  2. package/android/src/main/java/com/rnllama/LlamaContext.java +28 -21
  3. package/cpp/common.cpp +4 -11
  4. package/cpp/common.h +1 -1
  5. package/cpp/ggml-aarch64.c +2193 -2193
  6. package/cpp/ggml-aarch64.h +39 -39
  7. package/cpp/ggml-alloc.c +1042 -1041
  8. package/cpp/ggml-backend-impl.h +153 -153
  9. package/cpp/ggml-backend.c +2234 -2225
  10. package/cpp/ggml-backend.h +238 -236
  11. package/cpp/ggml-common.h +1829 -1829
  12. package/cpp/ggml-impl.h +655 -655
  13. package/cpp/ggml-metal.h +65 -65
  14. package/cpp/ggml-metal.m +3269 -3273
  15. package/cpp/ggml-quants.c +14860 -15022
  16. package/cpp/ggml-quants.h +132 -132
  17. package/cpp/ggml.c +16 -6
  18. package/cpp/ggml.h +2447 -2444
  19. package/cpp/llama.cpp +634 -531
  20. package/cpp/llama.h +30 -14
  21. package/cpp/log.h +737 -737
  22. package/cpp/rn-llama.hpp +9 -1
  23. package/cpp/sampling.cpp +460 -460
  24. package/cpp/sgemm.cpp +1027 -1027
  25. package/cpp/sgemm.h +14 -14
  26. package/package.json +1 -1
package/cpp/ggml-backend.h CHANGED
@@ -1,236 +1,238 @@
1
- #pragma once
2
-
3
- #include "ggml.h"
4
- #include "ggml-alloc.h"
5
-
6
- #ifdef __cplusplus
7
- extern "C" {
8
- #endif
9
-
10
- typedef struct lm_ggml_backend_buffer_type * lm_ggml_backend_buffer_type_t;
11
- typedef struct lm_ggml_backend_buffer * lm_ggml_backend_buffer_t;
12
- typedef struct lm_ggml_backend_event * lm_ggml_backend_event_t;
13
- typedef struct lm_ggml_backend * lm_ggml_backend_t;
14
- typedef void * lm_ggml_backend_graph_plan_t;
15
-
16
- //
17
- // Backend buffer
18
- //
19
-
20
- // buffer type
21
- LM_GGML_API const char * lm_ggml_backend_buft_name (lm_ggml_backend_buffer_type_t buft);
22
- LM_GGML_API LM_GGML_CALL lm_ggml_backend_buffer_t lm_ggml_backend_buft_alloc_buffer (lm_ggml_backend_buffer_type_t buft, size_t size);
23
- LM_GGML_API size_t lm_ggml_backend_buft_get_alignment (lm_ggml_backend_buffer_type_t buft);
24
- LM_GGML_API size_t lm_ggml_backend_buft_get_max_size (lm_ggml_backend_buffer_type_t buft);
25
- LM_GGML_API LM_GGML_CALL size_t lm_ggml_backend_buft_get_alloc_size (lm_ggml_backend_buffer_type_t buft, struct lm_ggml_tensor * tensor);
26
- LM_GGML_API bool lm_ggml_backend_buft_is_host (lm_ggml_backend_buffer_type_t buft);
27
-
28
- // buffer
29
- enum lm_ggml_backend_buffer_usage {
30
- LM_GGML_BACKEND_BUFFER_USAGE_ANY = 0,
31
- LM_GGML_BACKEND_BUFFER_USAGE_WEIGHTS = 1,
32
- };
33
-
34
- LM_GGML_API const char * lm_ggml_backend_buffer_name (lm_ggml_backend_buffer_t buffer);
35
- LM_GGML_API void lm_ggml_backend_buffer_free (lm_ggml_backend_buffer_t buffer);
36
- LM_GGML_API void * lm_ggml_backend_buffer_get_base (lm_ggml_backend_buffer_t buffer);
37
- LM_GGML_API size_t lm_ggml_backend_buffer_get_size (lm_ggml_backend_buffer_t buffer);
38
- LM_GGML_API LM_GGML_CALL void lm_ggml_backend_buffer_init_tensor (lm_ggml_backend_buffer_t buffer, struct lm_ggml_tensor * tensor);
39
- LM_GGML_API size_t lm_ggml_backend_buffer_get_alignment (lm_ggml_backend_buffer_t buffer);
40
- LM_GGML_API size_t lm_ggml_backend_buffer_get_max_size (lm_ggml_backend_buffer_t buffer);
41
- LM_GGML_API size_t lm_ggml_backend_buffer_get_alloc_size(lm_ggml_backend_buffer_t buffer, struct lm_ggml_tensor * tensor);
42
- LM_GGML_API void lm_ggml_backend_buffer_clear (lm_ggml_backend_buffer_t buffer, uint8_t value);
43
- LM_GGML_API bool lm_ggml_backend_buffer_is_host (lm_ggml_backend_buffer_t buffer);
44
- LM_GGML_API void lm_ggml_backend_buffer_set_usage (lm_ggml_backend_buffer_t buffer, enum lm_ggml_backend_buffer_usage usage);
45
- LM_GGML_API lm_ggml_backend_buffer_type_t lm_ggml_backend_buffer_get_type (lm_ggml_backend_buffer_t buffer);
46
- LM_GGML_API void lm_ggml_backend_buffer_reset (lm_ggml_backend_buffer_t buffer);
47
-
48
- //
49
- // Backend
50
- //
51
-
52
- LM_GGML_API lm_ggml_guid_t lm_ggml_backend_guid(lm_ggml_backend_t backend);
53
- LM_GGML_API const char * lm_ggml_backend_name(lm_ggml_backend_t backend);
54
- LM_GGML_API void lm_ggml_backend_free(lm_ggml_backend_t backend);
55
-
56
- LM_GGML_API lm_ggml_backend_buffer_type_t lm_ggml_backend_get_default_buffer_type(lm_ggml_backend_t backend);
57
- LM_GGML_API lm_ggml_backend_buffer_t lm_ggml_backend_alloc_buffer(lm_ggml_backend_t backend, size_t size);
58
- LM_GGML_API size_t lm_ggml_backend_get_alignment(lm_ggml_backend_t backend);
59
- LM_GGML_API size_t lm_ggml_backend_get_max_size(lm_ggml_backend_t backend);
60
-
61
- LM_GGML_API void lm_ggml_backend_tensor_set_async(lm_ggml_backend_t backend, struct lm_ggml_tensor * tensor, const void * data, size_t offset, size_t size);
62
- LM_GGML_API void lm_ggml_backend_tensor_get_async(lm_ggml_backend_t backend, const struct lm_ggml_tensor * tensor, void * data, size_t offset, size_t size);
63
-
64
- LM_GGML_API LM_GGML_CALL void lm_ggml_backend_tensor_set( struct lm_ggml_tensor * tensor, const void * data, size_t offset, size_t size);
65
- LM_GGML_API LM_GGML_CALL void lm_ggml_backend_tensor_get(const struct lm_ggml_tensor * tensor, void * data, size_t offset, size_t size);
66
-
67
- LM_GGML_API void lm_ggml_backend_synchronize(lm_ggml_backend_t backend);
68
-
69
- LM_GGML_API lm_ggml_backend_graph_plan_t lm_ggml_backend_graph_plan_create(lm_ggml_backend_t backend, struct lm_ggml_cgraph * cgraph);
70
- LM_GGML_API void lm_ggml_backend_graph_plan_free (lm_ggml_backend_t backend, lm_ggml_backend_graph_plan_t plan);
71
-
72
- LM_GGML_API enum lm_ggml_status lm_ggml_backend_graph_plan_compute (lm_ggml_backend_t backend, lm_ggml_backend_graph_plan_t plan);
73
- LM_GGML_API enum lm_ggml_status lm_ggml_backend_graph_compute (lm_ggml_backend_t backend, struct lm_ggml_cgraph * cgraph);
74
- LM_GGML_API enum lm_ggml_status lm_ggml_backend_graph_compute_async(lm_ggml_backend_t backend, struct lm_ggml_cgraph * cgraph);
75
- LM_GGML_API bool lm_ggml_backend_supports_op(lm_ggml_backend_t backend, const struct lm_ggml_tensor * op);
76
- LM_GGML_API bool lm_ggml_backend_supports_buft(lm_ggml_backend_t backend, lm_ggml_backend_buffer_type_t buft);
77
- LM_GGML_API bool lm_ggml_backend_offload_op(lm_ggml_backend_t backend, const struct lm_ggml_tensor * op);
78
-
79
- // tensor copy between different backends
80
- LM_GGML_API void lm_ggml_backend_tensor_copy(struct lm_ggml_tensor * src, struct lm_ggml_tensor * dst);
81
-
82
- // asynchronous copy
83
- // the copy is performed after all the currently queued operations in backend_src
84
- // backend_dst will wait for the copy to complete before performing other operations
85
- // automatic fallback to sync copy if async is not supported
86
- LM_GGML_API void lm_ggml_backend_tensor_copy_async(lm_ggml_backend_t backend_src, lm_ggml_backend_t backend_dst, struct lm_ggml_tensor * src, struct lm_ggml_tensor * dst);
87
-
88
- // events
89
- LM_GGML_API lm_ggml_backend_event_t lm_ggml_backend_event_new (lm_ggml_backend_t backend);
90
- LM_GGML_API void lm_ggml_backend_event_free (lm_ggml_backend_event_t event);
91
- LM_GGML_API void lm_ggml_backend_event_record (lm_ggml_backend_event_t event);
92
- LM_GGML_API void lm_ggml_backend_event_synchronize(lm_ggml_backend_event_t event);
93
- LM_GGML_API void lm_ggml_backend_event_wait (lm_ggml_backend_t backend, lm_ggml_backend_event_t event);
94
-
95
- //
96
- // CPU backend
97
- //
98
-
99
- LM_GGML_API lm_ggml_backend_t lm_ggml_backend_cpu_init(void);
100
-
101
- LM_GGML_API LM_GGML_CALL bool lm_ggml_backend_is_cpu (lm_ggml_backend_t backend);
102
- LM_GGML_API void lm_ggml_backend_cpu_set_n_threads (lm_ggml_backend_t backend_cpu, int n_threads);
103
- LM_GGML_API void lm_ggml_backend_cpu_set_abort_callback(lm_ggml_backend_t backend_cpu, lm_ggml_abort_callback abort_callback, void * abort_callback_data);
104
-
105
- // Create a backend buffer from an existing pointer
106
- LM_GGML_API LM_GGML_CALL lm_ggml_backend_buffer_t lm_ggml_backend_cpu_buffer_from_ptr(void * ptr, size_t size);
107
-
108
- LM_GGML_API LM_GGML_CALL lm_ggml_backend_buffer_type_t lm_ggml_backend_cpu_buffer_type(void);
109
-
110
- #ifdef LM_GGML_USE_CPU_HBM
111
- LM_GGML_API lm_ggml_backend_buffer_type_t lm_ggml_backend_cpu_hbm_buffer_type(void);
112
- #endif
113
-
114
- //
115
- // Backend registry
116
- //
117
-
118
- // The backend registry is a registry of all the available backends, and allows initializing backends in a generic way
119
-
120
- LM_GGML_API size_t lm_ggml_backend_reg_get_count(void);
121
- LM_GGML_API size_t lm_ggml_backend_reg_find_by_name(const char * name);
122
- LM_GGML_API lm_ggml_backend_t lm_ggml_backend_reg_init_backend_from_str(const char * backend_str); // str is backend_name:params (params is optional)
123
- LM_GGML_API const char * lm_ggml_backend_reg_get_name(size_t i);
124
- LM_GGML_API lm_ggml_backend_t lm_ggml_backend_reg_init_backend(size_t i, const char * params); // params is backend-specific
125
- LM_GGML_API lm_ggml_backend_buffer_type_t lm_ggml_backend_reg_get_default_buffer_type(size_t i);
126
- LM_GGML_API lm_ggml_backend_buffer_t lm_ggml_backend_reg_alloc_buffer(size_t i, size_t size);
127
-
128
- //
129
- // Backend scheduler
130
- //
131
-
132
- // The backend scheduler allows for multiple backends to be used together
133
- // Handles compute buffer allocation, assignment of tensors to backends, and copying of tensors between backends
134
- // The backends are selected based on:
135
- // - the backend that supports the operation
136
- // - the location of the pre-allocated tensors (e.g. the weights)
137
- /*
138
- Example usage:
139
-
140
- // operations that use tensors allocated in a buffer with USAGE_WEIGHTS will be assigned
141
- // preferrably to run on the same backend as the buffer
142
- lm_ggml_backend_buffer_set_usage(buf_weights, LM_GGML_BACKEND_BUFFER_USAGE_WEIGHTS);
143
-
144
- sched = lm_ggml_backend_sched_new({backend_gpu, backend_gpu2, backend_cpu}, NULL, num_backends, LM_GGML_DEFAULT_GRAPH_SIZE, false);
145
-
146
- // initialize buffers from a max size graph (optional)
147
- reserve_graph = build_graph(sched, max_batch_size);
148
-
149
- // manually assign nodes to a backend (optional, should not be needed in most cases)
150
- struct lm_ggml_tensor * node = lm_ggml_mul_mat(ctx, ...);
151
- lm_ggml_backend_sched_set_tensor_backend(sched, node, backend_gpu);
152
-
153
- lm_ggml_backend_sched_reserve(sched, reserve_graph);
154
-
155
- // compute
156
- graph = build_graph(sched);
157
- lm_ggml_backend_sched_graph_compute(sched, graph);
158
-
159
- // if there are graph inputs:
160
- lm_ggml_backend_sched_reset(sched);
161
- lm_ggml_backend_sched_alloc_graph(sched, graph);
162
- lm_ggml_backend_tensor_set(input_tensor, ...);
163
- lm_ggml_backend_sched_graph_compute(sched, graph);
164
- }
165
- */
166
-
167
- struct lm_ggml_backend_sched;
168
- typedef struct lm_ggml_backend_sched * lm_ggml_backend_sched_t;
169
-
170
- // when ask == true, the scheduler wants to know if the user wants to observe this node
171
- // this allows the scheduler to batch nodes together in order to evaluate them in a single call
172
- //
173
- // when ask == false, the scheduler is passing the node tensor to the user for observation
174
- // if the user returns false, the scheduler will cancel the graph compute
175
- //
176
- typedef bool (*lm_ggml_backend_sched_eval_callback)(struct lm_ggml_tensor * t, bool ask, void * user_data);
177
-
178
- // Initialize a backend scheduler
179
- LM_GGML_API lm_ggml_backend_sched_t lm_ggml_backend_sched_new(lm_ggml_backend_t * backends, lm_ggml_backend_buffer_type_t * bufts, int n_backends, size_t graph_size, bool parallel);
180
- LM_GGML_API void lm_ggml_backend_sched_free(lm_ggml_backend_sched_t sched);
181
-
182
- // Initialize backend buffers from a measure graph
183
- LM_GGML_API bool lm_ggml_backend_sched_reserve(lm_ggml_backend_sched_t sched, struct lm_ggml_cgraph * measure_graph);
184
-
185
- LM_GGML_API int lm_ggml_backend_sched_get_n_backends(lm_ggml_backend_sched_t sched);
186
- LM_GGML_API lm_ggml_backend_t lm_ggml_backend_sched_get_backend(lm_ggml_backend_sched_t sched, int i);
187
-
188
- // Get the number of splits of the last graph
189
- LM_GGML_API int lm_ggml_backend_sched_get_n_splits(lm_ggml_backend_sched_t sched);
190
- LM_GGML_API int lm_ggml_backend_sched_get_n_copies(lm_ggml_backend_sched_t sched);
191
-
192
- LM_GGML_API size_t lm_ggml_backend_sched_get_buffer_size(lm_ggml_backend_sched_t sched, lm_ggml_backend_t backend);
193
-
194
- LM_GGML_API void lm_ggml_backend_sched_set_tensor_backend(lm_ggml_backend_sched_t sched, struct lm_ggml_tensor * node, lm_ggml_backend_t backend);
195
- LM_GGML_API lm_ggml_backend_t lm_ggml_backend_sched_get_tensor_backend(lm_ggml_backend_sched_t sched, struct lm_ggml_tensor * node);
196
-
197
- // Allocate and compute graph on the backend scheduler
198
- LM_GGML_API bool lm_ggml_backend_sched_alloc_graph(lm_ggml_backend_sched_t sched, struct lm_ggml_cgraph * graph);
199
- LM_GGML_API enum lm_ggml_status lm_ggml_backend_sched_graph_compute(lm_ggml_backend_sched_t sched, struct lm_ggml_cgraph * graph);
200
- LM_GGML_API enum lm_ggml_status lm_ggml_backend_sched_graph_compute_async(lm_ggml_backend_sched_t sched, struct lm_ggml_cgraph * graph);
201
- LM_GGML_API void lm_ggml_backend_sched_synchronize(lm_ggml_backend_sched_t sched);
202
-
203
- // Reset all assignments and allocators - must be called before changing the node backends
204
- LM_GGML_API void lm_ggml_backend_sched_reset(lm_ggml_backend_sched_t sched);
205
-
206
- // Set a callback to be called for each resulting node during graph compute
207
- LM_GGML_API void lm_ggml_backend_sched_set_eval_callback(lm_ggml_backend_sched_t sched, lm_ggml_backend_sched_eval_callback callback, void * user_data);
208
-
209
- //
210
- // Utils
211
- //
212
-
213
- struct lm_ggml_backend_graph_copy {
214
- lm_ggml_backend_buffer_t buffer;
215
- struct lm_ggml_context * ctx_allocated;
216
- struct lm_ggml_context * ctx_unallocated;
217
- struct lm_ggml_cgraph * graph;
218
- };
219
-
220
- // Copy a graph to a different backend
221
- LM_GGML_API struct lm_ggml_backend_graph_copy lm_ggml_backend_graph_copy(lm_ggml_backend_t backend, struct lm_ggml_cgraph * graph);
222
- LM_GGML_API void lm_ggml_backend_graph_copy_free(struct lm_ggml_backend_graph_copy copy);
223
-
224
- typedef bool (*LM_GGML_CALL lm_ggml_backend_eval_callback)(int node_index, struct lm_ggml_tensor * t1, struct lm_ggml_tensor * t2, void * user_data);
225
-
226
- // Compare the output of two backends
227
- LM_GGML_API bool lm_ggml_backend_compare_graph_backend(lm_ggml_backend_t backend1, lm_ggml_backend_t backend2, struct lm_ggml_cgraph * graph, lm_ggml_backend_eval_callback callback, void * user_data);
228
-
229
- // Tensor initialization
230
- LM_GGML_API void lm_ggml_backend_tensor_alloc(lm_ggml_backend_buffer_t buffer, struct lm_ggml_tensor * tensor, void * addr);
231
- LM_GGML_API void lm_ggml_backend_view_init(struct lm_ggml_tensor * tensor);
232
-
233
-
234
- #ifdef __cplusplus
235
- }
236
- #endif
1
+ #pragma once
2
+
3
+ #include "ggml.h"
4
+ #include "ggml-alloc.h"
5
+
6
+ #ifdef __cplusplus
7
+ extern "C" {
8
+ #endif
9
+
10
+ typedef struct lm_ggml_backend_buffer_type * lm_ggml_backend_buffer_type_t;
11
+ typedef struct lm_ggml_backend_buffer * lm_ggml_backend_buffer_t;
12
+ typedef struct lm_ggml_backend_event * lm_ggml_backend_event_t;
13
+ typedef struct lm_ggml_backend * lm_ggml_backend_t;
14
+ typedef void * lm_ggml_backend_graph_plan_t;
15
+
16
+ //
17
+ // Backend buffer
18
+ //
19
+
20
+ // buffer type
21
+ LM_GGML_API const char * lm_ggml_backend_buft_name (lm_ggml_backend_buffer_type_t buft);
22
+ LM_GGML_API LM_GGML_CALL lm_ggml_backend_buffer_t lm_ggml_backend_buft_alloc_buffer (lm_ggml_backend_buffer_type_t buft, size_t size);
23
+ LM_GGML_API size_t lm_ggml_backend_buft_get_alignment (lm_ggml_backend_buffer_type_t buft);
24
+ LM_GGML_API size_t lm_ggml_backend_buft_get_max_size (lm_ggml_backend_buffer_type_t buft);
25
+ LM_GGML_API LM_GGML_CALL size_t lm_ggml_backend_buft_get_alloc_size (lm_ggml_backend_buffer_type_t buft, struct lm_ggml_tensor * tensor);
26
+ LM_GGML_API bool lm_ggml_backend_buft_is_host (lm_ggml_backend_buffer_type_t buft);
27
+
28
+ // buffer
29
+ enum lm_ggml_backend_buffer_usage {
30
+ LM_GGML_BACKEND_BUFFER_USAGE_ANY = 0,
31
+ LM_GGML_BACKEND_BUFFER_USAGE_WEIGHTS = 1,
32
+ LM_GGML_BACKEND_BUFFER_USAGE_COMPUTE = 2,
33
+ };
34
+
35
+ LM_GGML_API const char * lm_ggml_backend_buffer_name (lm_ggml_backend_buffer_t buffer);
36
+ LM_GGML_API void lm_ggml_backend_buffer_free (lm_ggml_backend_buffer_t buffer);
37
+ LM_GGML_API void * lm_ggml_backend_buffer_get_base (lm_ggml_backend_buffer_t buffer);
38
+ LM_GGML_API size_t lm_ggml_backend_buffer_get_size (lm_ggml_backend_buffer_t buffer);
39
+ LM_GGML_API LM_GGML_CALL void lm_ggml_backend_buffer_init_tensor (lm_ggml_backend_buffer_t buffer, struct lm_ggml_tensor * tensor);
40
+ LM_GGML_API size_t lm_ggml_backend_buffer_get_alignment (lm_ggml_backend_buffer_t buffer);
41
+ LM_GGML_API size_t lm_ggml_backend_buffer_get_max_size (lm_ggml_backend_buffer_t buffer);
42
+ LM_GGML_API size_t lm_ggml_backend_buffer_get_alloc_size(lm_ggml_backend_buffer_t buffer, struct lm_ggml_tensor * tensor);
43
+ LM_GGML_API void lm_ggml_backend_buffer_clear (lm_ggml_backend_buffer_t buffer, uint8_t value);
44
+ LM_GGML_API bool lm_ggml_backend_buffer_is_host (lm_ggml_backend_buffer_t buffer);
45
+ LM_GGML_API void lm_ggml_backend_buffer_set_usage (lm_ggml_backend_buffer_t buffer, enum lm_ggml_backend_buffer_usage usage);
46
+ LM_GGML_API enum lm_ggml_backend_buffer_usage lm_ggml_backend_buffer_get_usage (lm_ggml_backend_buffer_t buffer);
47
+ LM_GGML_API lm_ggml_backend_buffer_type_t lm_ggml_backend_buffer_get_type (lm_ggml_backend_buffer_t buffer);
48
+ LM_GGML_API void lm_ggml_backend_buffer_reset (lm_ggml_backend_buffer_t buffer);
49
+
50
+ //
51
+ // Backend
52
+ //
53
+
54
+ LM_GGML_API lm_ggml_guid_t lm_ggml_backend_guid(lm_ggml_backend_t backend);
55
+ LM_GGML_API const char * lm_ggml_backend_name(lm_ggml_backend_t backend);
56
+ LM_GGML_API void lm_ggml_backend_free(lm_ggml_backend_t backend);
57
+
58
+ LM_GGML_API lm_ggml_backend_buffer_type_t lm_ggml_backend_get_default_buffer_type(lm_ggml_backend_t backend);
59
+ LM_GGML_API lm_ggml_backend_buffer_t lm_ggml_backend_alloc_buffer(lm_ggml_backend_t backend, size_t size);
60
+ LM_GGML_API size_t lm_ggml_backend_get_alignment(lm_ggml_backend_t backend);
61
+ LM_GGML_API size_t lm_ggml_backend_get_max_size(lm_ggml_backend_t backend);
62
+
63
+ LM_GGML_API void lm_ggml_backend_tensor_set_async(lm_ggml_backend_t backend, struct lm_ggml_tensor * tensor, const void * data, size_t offset, size_t size);
64
+ LM_GGML_API void lm_ggml_backend_tensor_get_async(lm_ggml_backend_t backend, const struct lm_ggml_tensor * tensor, void * data, size_t offset, size_t size);
65
+
66
+ LM_GGML_API LM_GGML_CALL void lm_ggml_backend_tensor_set( struct lm_ggml_tensor * tensor, const void * data, size_t offset, size_t size);
67
+ LM_GGML_API LM_GGML_CALL void lm_ggml_backend_tensor_get(const struct lm_ggml_tensor * tensor, void * data, size_t offset, size_t size);
68
+
69
+ LM_GGML_API void lm_ggml_backend_synchronize(lm_ggml_backend_t backend);
70
+
71
+ LM_GGML_API lm_ggml_backend_graph_plan_t lm_ggml_backend_graph_plan_create(lm_ggml_backend_t backend, struct lm_ggml_cgraph * cgraph);
72
+ LM_GGML_API void lm_ggml_backend_graph_plan_free (lm_ggml_backend_t backend, lm_ggml_backend_graph_plan_t plan);
73
+
74
+ LM_GGML_API enum lm_ggml_status lm_ggml_backend_graph_plan_compute (lm_ggml_backend_t backend, lm_ggml_backend_graph_plan_t plan);
75
+ LM_GGML_API enum lm_ggml_status lm_ggml_backend_graph_compute (lm_ggml_backend_t backend, struct lm_ggml_cgraph * cgraph);
76
+ LM_GGML_API enum lm_ggml_status lm_ggml_backend_graph_compute_async(lm_ggml_backend_t backend, struct lm_ggml_cgraph * cgraph);
77
+ LM_GGML_API bool lm_ggml_backend_supports_op(lm_ggml_backend_t backend, const struct lm_ggml_tensor * op);
78
+ LM_GGML_API bool lm_ggml_backend_supports_buft(lm_ggml_backend_t backend, lm_ggml_backend_buffer_type_t buft);
79
+ LM_GGML_API bool lm_ggml_backend_offload_op(lm_ggml_backend_t backend, const struct lm_ggml_tensor * op);
80
+
81
+ // tensor copy between different backends
82
+ LM_GGML_API void lm_ggml_backend_tensor_copy(struct lm_ggml_tensor * src, struct lm_ggml_tensor * dst);
83
+
84
+ // asynchronous copy
85
+ // the copy is performed after all the currently queued operations in backend_src
86
+ // backend_dst will wait for the copy to complete before performing other operations
87
+ // automatic fallback to sync copy if async is not supported
88
+ LM_GGML_API void lm_ggml_backend_tensor_copy_async(lm_ggml_backend_t backend_src, lm_ggml_backend_t backend_dst, struct lm_ggml_tensor * src, struct lm_ggml_tensor * dst);
89
+
90
+ // events
91
+ LM_GGML_API lm_ggml_backend_event_t lm_ggml_backend_event_new (lm_ggml_backend_t backend);
92
+ LM_GGML_API void lm_ggml_backend_event_free (lm_ggml_backend_event_t event);
93
+ LM_GGML_API void lm_ggml_backend_event_record (lm_ggml_backend_event_t event);
94
+ LM_GGML_API void lm_ggml_backend_event_synchronize(lm_ggml_backend_event_t event);
95
+ LM_GGML_API void lm_ggml_backend_event_wait (lm_ggml_backend_t backend, lm_ggml_backend_event_t event);
96
+
97
+ //
98
+ // CPU backend
99
+ //
100
+
101
+ LM_GGML_API lm_ggml_backend_t lm_ggml_backend_cpu_init(void);
102
+
103
+ LM_GGML_API LM_GGML_CALL bool lm_ggml_backend_is_cpu (lm_ggml_backend_t backend);
104
+ LM_GGML_API void lm_ggml_backend_cpu_set_n_threads (lm_ggml_backend_t backend_cpu, int n_threads);
105
+ LM_GGML_API void lm_ggml_backend_cpu_set_abort_callback(lm_ggml_backend_t backend_cpu, lm_ggml_abort_callback abort_callback, void * abort_callback_data);
106
+
107
+ // Create a backend buffer from an existing pointer
108
+ LM_GGML_API LM_GGML_CALL lm_ggml_backend_buffer_t lm_ggml_backend_cpu_buffer_from_ptr(void * ptr, size_t size);
109
+
110
+ LM_GGML_API LM_GGML_CALL lm_ggml_backend_buffer_type_t lm_ggml_backend_cpu_buffer_type(void);
111
+
112
+ #ifdef LM_GGML_USE_CPU_HBM
113
+ LM_GGML_API lm_ggml_backend_buffer_type_t lm_ggml_backend_cpu_hbm_buffer_type(void);
114
+ #endif
115
+
116
+ //
117
+ // Backend registry
118
+ //
119
+
120
+ // The backend registry is a registry of all the available backends, and allows initializing backends in a generic way
121
+
122
+ LM_GGML_API size_t lm_ggml_backend_reg_get_count(void);
123
+ LM_GGML_API size_t lm_ggml_backend_reg_find_by_name(const char * name);
124
+ LM_GGML_API lm_ggml_backend_t lm_ggml_backend_reg_init_backend_from_str(const char * backend_str); // str is backend_name:params (params is optional)
125
+ LM_GGML_API const char * lm_ggml_backend_reg_get_name(size_t i);
126
+ LM_GGML_API lm_ggml_backend_t lm_ggml_backend_reg_init_backend(size_t i, const char * params); // params is backend-specific
127
+ LM_GGML_API lm_ggml_backend_buffer_type_t lm_ggml_backend_reg_get_default_buffer_type(size_t i);
128
+ LM_GGML_API lm_ggml_backend_buffer_t lm_ggml_backend_reg_alloc_buffer(size_t i, size_t size);
129
+
130
+ //
131
+ // Backend scheduler
132
+ //
133
+
134
+ // The backend scheduler allows for multiple backends to be used together
135
+ // Handles compute buffer allocation, assignment of tensors to backends, and copying of tensors between backends
136
+ // The backends are selected based on:
137
+ // - the backend that supports the operation
138
+ // - the location of the pre-allocated tensors (e.g. the weights)
139
+ /*
140
+ Example usage:
141
+
142
+ // operations that use tensors allocated in a buffer with USAGE_WEIGHTS will be assigned
143
+ // preferrably to run on the same backend as the buffer
144
+ lm_ggml_backend_buffer_set_usage(buf_weights, LM_GGML_BACKEND_BUFFER_USAGE_WEIGHTS);
145
+
146
+ sched = lm_ggml_backend_sched_new({backend_gpu, backend_gpu2, backend_cpu}, NULL, num_backends, LM_GGML_DEFAULT_GRAPH_SIZE, false);
147
+
148
+ // initialize buffers from a max size graph (optional)
149
+ reserve_graph = build_graph(sched, max_batch_size);
150
+
151
+ // manually assign nodes to a backend (optional, should not be needed in most cases)
152
+ struct lm_ggml_tensor * node = lm_ggml_mul_mat(ctx, ...);
153
+ lm_ggml_backend_sched_set_tensor_backend(sched, node, backend_gpu);
154
+
155
+ lm_ggml_backend_sched_reserve(sched, reserve_graph);
156
+
157
+ // compute
158
+ graph = build_graph(sched);
159
+ lm_ggml_backend_sched_graph_compute(sched, graph);
160
+
161
+ // if there are graph inputs:
162
+ lm_ggml_backend_sched_reset(sched);
163
+ lm_ggml_backend_sched_alloc_graph(sched, graph);
164
+ lm_ggml_backend_tensor_set(input_tensor, ...);
165
+ lm_ggml_backend_sched_graph_compute(sched, graph);
166
+ }
167
+ */
168
+
169
+ struct lm_ggml_backend_sched;
170
+ typedef struct lm_ggml_backend_sched * lm_ggml_backend_sched_t;
171
+
172
+ // when ask == true, the scheduler wants to know if the user wants to observe this node
173
+ // this allows the scheduler to batch nodes together in order to evaluate them in a single call
174
+ //
175
+ // when ask == false, the scheduler is passing the node tensor to the user for observation
176
+ // if the user returns false, the scheduler will cancel the graph compute
177
+ //
178
+ typedef bool (*lm_ggml_backend_sched_eval_callback)(struct lm_ggml_tensor * t, bool ask, void * user_data);
179
+
180
+ // Initialize a backend scheduler
181
+ LM_GGML_API lm_ggml_backend_sched_t lm_ggml_backend_sched_new(lm_ggml_backend_t * backends, lm_ggml_backend_buffer_type_t * bufts, int n_backends, size_t graph_size, bool parallel);
182
+ LM_GGML_API void lm_ggml_backend_sched_free(lm_ggml_backend_sched_t sched);
183
+
184
+ // Initialize backend buffers from a measure graph
185
+ LM_GGML_API bool lm_ggml_backend_sched_reserve(lm_ggml_backend_sched_t sched, struct lm_ggml_cgraph * measure_graph);
186
+
187
+ LM_GGML_API int lm_ggml_backend_sched_get_n_backends(lm_ggml_backend_sched_t sched);
188
+ LM_GGML_API lm_ggml_backend_t lm_ggml_backend_sched_get_backend(lm_ggml_backend_sched_t sched, int i);
189
+
190
+ // Get the number of splits of the last graph
191
+ LM_GGML_API int lm_ggml_backend_sched_get_n_splits(lm_ggml_backend_sched_t sched);
192
+ LM_GGML_API int lm_ggml_backend_sched_get_n_copies(lm_ggml_backend_sched_t sched);
193
+
194
+ LM_GGML_API size_t lm_ggml_backend_sched_get_buffer_size(lm_ggml_backend_sched_t sched, lm_ggml_backend_t backend);
195
+
196
+ LM_GGML_API void lm_ggml_backend_sched_set_tensor_backend(lm_ggml_backend_sched_t sched, struct lm_ggml_tensor * node, lm_ggml_backend_t backend);
197
+ LM_GGML_API lm_ggml_backend_t lm_ggml_backend_sched_get_tensor_backend(lm_ggml_backend_sched_t sched, struct lm_ggml_tensor * node);
198
+
199
+ // Allocate and compute graph on the backend scheduler
200
+ LM_GGML_API bool lm_ggml_backend_sched_alloc_graph(lm_ggml_backend_sched_t sched, struct lm_ggml_cgraph * graph);
201
+ LM_GGML_API enum lm_ggml_status lm_ggml_backend_sched_graph_compute(lm_ggml_backend_sched_t sched, struct lm_ggml_cgraph * graph);
202
+ LM_GGML_API enum lm_ggml_status lm_ggml_backend_sched_graph_compute_async(lm_ggml_backend_sched_t sched, struct lm_ggml_cgraph * graph);
203
+ LM_GGML_API void lm_ggml_backend_sched_synchronize(lm_ggml_backend_sched_t sched);
204
+
205
+ // Reset all assignments and allocators - must be called before changing the node backends
206
+ LM_GGML_API void lm_ggml_backend_sched_reset(lm_ggml_backend_sched_t sched);
207
+
208
+ // Set a callback to be called for each resulting node during graph compute
209
+ LM_GGML_API void lm_ggml_backend_sched_set_eval_callback(lm_ggml_backend_sched_t sched, lm_ggml_backend_sched_eval_callback callback, void * user_data);
210
+
211
+ //
212
+ // Utils
213
+ //
214
+
215
+ struct lm_ggml_backend_graph_copy {
216
+ lm_ggml_backend_buffer_t buffer;
217
+ struct lm_ggml_context * ctx_allocated;
218
+ struct lm_ggml_context * ctx_unallocated;
219
+ struct lm_ggml_cgraph * graph;
220
+ };
221
+
222
+ // Copy a graph to a different backend
223
+ LM_GGML_API struct lm_ggml_backend_graph_copy lm_ggml_backend_graph_copy(lm_ggml_backend_t backend, struct lm_ggml_cgraph * graph);
224
+ LM_GGML_API void lm_ggml_backend_graph_copy_free(struct lm_ggml_backend_graph_copy copy);
225
+
226
+ typedef bool (*LM_GGML_CALL lm_ggml_backend_eval_callback)(int node_index, struct lm_ggml_tensor * t1, struct lm_ggml_tensor * t2, void * user_data);
227
+
228
+ // Compare the output of two backends
229
+ LM_GGML_API bool lm_ggml_backend_compare_graph_backend(lm_ggml_backend_t backend1, lm_ggml_backend_t backend2, struct lm_ggml_cgraph * graph, lm_ggml_backend_eval_callback callback, void * user_data);
230
+
231
+ // Tensor initialization
232
+ LM_GGML_API void lm_ggml_backend_tensor_alloc(lm_ggml_backend_buffer_t buffer, struct lm_ggml_tensor * tensor, void * addr);
233
+ LM_GGML_API void lm_ggml_backend_view_init(struct lm_ggml_tensor * tensor);
234
+
235
+
236
+ #ifdef __cplusplus
237
+ }
238
+ #endif