llama_cpp 0.7.0 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (23) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +12 -0
  3. data/ext/llama_cpp/extconf.rb +1 -1
  4. data/ext/llama_cpp/llama_cpp.cpp +41 -21
  5. data/ext/llama_cpp/src/ggml-alloc.c +62 -107
  6. data/ext/llama_cpp/src/ggml-alloc.h +11 -5
  7. data/ext/llama_cpp/src/ggml-backend.c +385 -0
  8. data/ext/llama_cpp/src/ggml-backend.h +143 -0
  9. data/ext/llama_cpp/src/ggml-cuda.cu +500 -78
  10. data/ext/llama_cpp/src/ggml-cuda.h +4 -0
  11. data/ext/llama_cpp/src/ggml-metal.h +18 -1
  12. data/ext/llama_cpp/src/ggml-metal.m +396 -127
  13. data/ext/llama_cpp/src/ggml-metal.metal +290 -46
  14. data/ext/llama_cpp/src/ggml-opencl.cpp +47 -71
  15. data/ext/llama_cpp/src/ggml.c +71 -55
  16. data/ext/llama_cpp/src/ggml.h +15 -9
  17. data/ext/llama_cpp/src/k_quants.c +12 -20
  18. data/ext/llama_cpp/src/k_quants.h +5 -5
  19. data/ext/llama_cpp/src/llama.cpp +1851 -250
  20. data/ext/llama_cpp/src/llama.h +18 -12
  21. data/lib/llama_cpp/version.rb +2 -2
  22. data/sig/llama_cpp.rbs +4 -4
  23. metadata +5 -3
data/ext/llama_cpp/src/ggml-metal.m CHANGED
@@ -73,6 +73,8 @@ struct ggml_metal_context {
73
73
  GGML_METAL_DECL_KERNEL(get_rows_f16);
74
74
  GGML_METAL_DECL_KERNEL(get_rows_q4_0);
75
75
  GGML_METAL_DECL_KERNEL(get_rows_q4_1);
76
+ GGML_METAL_DECL_KERNEL(get_rows_q5_0);
77
+ GGML_METAL_DECL_KERNEL(get_rows_q5_1);
76
78
  GGML_METAL_DECL_KERNEL(get_rows_q8_0);
77
79
  GGML_METAL_DECL_KERNEL(get_rows_q2_K);
78
80
  GGML_METAL_DECL_KERNEL(get_rows_q3_K);
@@ -81,22 +83,26 @@ struct ggml_metal_context {
81
83
  GGML_METAL_DECL_KERNEL(get_rows_q6_K);
82
84
  GGML_METAL_DECL_KERNEL(rms_norm);
83
85
  GGML_METAL_DECL_KERNEL(norm);
84
- GGML_METAL_DECL_KERNEL(mul_mat_f32_f32);
85
- GGML_METAL_DECL_KERNEL(mul_mat_f16_f32);
86
- GGML_METAL_DECL_KERNEL(mul_mat_f16_f32_1row);
87
- GGML_METAL_DECL_KERNEL(mul_mat_f16_f32_l4);
88
- GGML_METAL_DECL_KERNEL(mul_mat_q4_0_f32);
89
- GGML_METAL_DECL_KERNEL(mul_mat_q4_1_f32);
90
- GGML_METAL_DECL_KERNEL(mul_mat_q8_0_f32);
91
- GGML_METAL_DECL_KERNEL(mul_mat_q2_K_f32);
92
- GGML_METAL_DECL_KERNEL(mul_mat_q3_K_f32);
93
- GGML_METAL_DECL_KERNEL(mul_mat_q4_K_f32);
94
- GGML_METAL_DECL_KERNEL(mul_mat_q5_K_f32);
95
- GGML_METAL_DECL_KERNEL(mul_mat_q6_K_f32);
86
+ GGML_METAL_DECL_KERNEL(mul_mv_f32_f32);
87
+ GGML_METAL_DECL_KERNEL(mul_mv_f16_f32);
88
+ GGML_METAL_DECL_KERNEL(mul_mv_f16_f32_1row);
89
+ GGML_METAL_DECL_KERNEL(mul_mv_f16_f32_l4);
90
+ GGML_METAL_DECL_KERNEL(mul_mv_q4_0_f32);
91
+ GGML_METAL_DECL_KERNEL(mul_mv_q4_1_f32);
92
+ GGML_METAL_DECL_KERNEL(mul_mv_q5_0_f32);
93
+ GGML_METAL_DECL_KERNEL(mul_mv_q5_1_f32);
94
+ GGML_METAL_DECL_KERNEL(mul_mv_q8_0_f32);
95
+ GGML_METAL_DECL_KERNEL(mul_mv_q2_K_f32);
96
+ GGML_METAL_DECL_KERNEL(mul_mv_q3_K_f32);
97
+ GGML_METAL_DECL_KERNEL(mul_mv_q4_K_f32);
98
+ GGML_METAL_DECL_KERNEL(mul_mv_q5_K_f32);
99
+ GGML_METAL_DECL_KERNEL(mul_mv_q6_K_f32);
96
100
  GGML_METAL_DECL_KERNEL(mul_mm_f32_f32);
97
101
  GGML_METAL_DECL_KERNEL(mul_mm_f16_f32);
98
102
  GGML_METAL_DECL_KERNEL(mul_mm_q4_0_f32);
99
103
  GGML_METAL_DECL_KERNEL(mul_mm_q4_1_f32);
104
+ GGML_METAL_DECL_KERNEL(mul_mm_q5_0_f32);
105
+ GGML_METAL_DECL_KERNEL(mul_mm_q5_1_f32);
100
106
  GGML_METAL_DECL_KERNEL(mul_mm_q8_0_f32);
101
107
  GGML_METAL_DECL_KERNEL(mul_mm_q2_K_f32);
102
108
  GGML_METAL_DECL_KERNEL(mul_mm_q3_K_f32);
@@ -109,6 +115,8 @@ struct ggml_metal_context {
109
115
  GGML_METAL_DECL_KERNEL(cpy_f32_f16);
110
116
  GGML_METAL_DECL_KERNEL(cpy_f32_f32);
111
117
  GGML_METAL_DECL_KERNEL(cpy_f16_f16);
118
+ GGML_METAL_DECL_KERNEL(concat);
119
+ GGML_METAL_DECL_KERNEL(sqr);
112
120
 
113
121
  #undef GGML_METAL_DECL_KERNEL
114
122
  };
@@ -183,56 +191,44 @@ struct ggml_metal_context * ggml_metal_init(int n_cb) {
183
191
 
184
192
  ctx->d_queue = dispatch_queue_create("ggml-metal", DISPATCH_QUEUE_CONCURRENT);
185
193
 
186
- #ifdef GGML_SWIFT
187
- // load the default.metallib file
194
+ // load library
188
195
  {
189
- NSError * error = nil;
190
-
191
- NSBundle * bundle = [NSBundle bundleForClass:[GGMLMetalClass class]];
192
- NSString * llamaBundlePath = [bundle pathForResource:@"llama_llama" ofType:@"bundle"];
193
- NSBundle * llamaBundle = [NSBundle bundleWithPath:llamaBundlePath];
194
- NSString * libPath = [llamaBundle pathForResource:@"default" ofType:@"metallib"];
195
- NSURL * libURL = [NSURL fileURLWithPath:libPath];
196
-
197
- // Load the metallib file into a Metal library
198
- ctx->library = [ctx->device newLibraryWithURL:libURL error:&error];
199
-
200
- if (error) {
201
- GGML_METAL_LOG_ERROR("%s: error: %s\n", __func__, [[error description] UTF8String]);
202
- return NULL;
203
- }
204
- }
196
+ NSBundle * bundle = nil;
197
+ #ifdef SWIFT_PACKAGE
198
+ bundle = SWIFTPM_MODULE_BUNDLE;
205
199
  #else
206
- UNUSED(msl_library_source);
207
-
208
- // read the source from "ggml-metal.metal" into a string and use newLibraryWithSource
209
- {
200
+ bundle = [NSBundle bundleForClass:[GGMLMetalClass class]];
201
+ #endif
210
202
  NSError * error = nil;
203
+ NSString * libPath = [bundle pathForResource:@"default" ofType:@"metallib"];
204
+ if (libPath != nil) {
205
+ NSURL * libURL = [NSURL fileURLWithPath:libPath];
206
+ GGML_METAL_LOG_INFO("%s: loading '%s'\n", __func__, [libPath UTF8String]);
207
+ ctx->library = [ctx->device newLibraryWithURL:libURL error:&error];
208
+ } else {
209
+ GGML_METAL_LOG_INFO("%s: default.metallib not found, loading from source\n", __func__);
210
+
211
+ NSString * sourcePath = [bundle pathForResource:@"ggml-metal" ofType:@"metal"];
212
+ GGML_METAL_LOG_INFO("%s: loading '%s'\n", __func__, [sourcePath UTF8String]);
213
+ NSString * src = [NSString stringWithContentsOfFile:sourcePath encoding:NSUTF8StringEncoding error:&error];
214
+ if (error) {
215
+ GGML_METAL_LOG_ERROR("%s: error: %s\n", __func__, [[error description] UTF8String]);
216
+ return NULL;
217
+ }
211
218
 
212
- //NSString * path = [[NSBundle mainBundle] pathForResource:@"../../examples/metal/metal" ofType:@"metal"];
213
- NSBundle * bundle = [NSBundle bundleForClass:[GGMLMetalClass class]];
214
- NSString * path = [bundle pathForResource:@"ggml-metal" ofType:@"metal"];
215
- GGML_METAL_LOG_INFO("%s: loading '%s'\n", __func__, [path UTF8String]);
216
-
217
- NSString * src = [NSString stringWithContentsOfFile:path encoding:NSUTF8StringEncoding error:&error];
218
- if (error) {
219
- GGML_METAL_LOG_ERROR("%s: error: %s\n", __func__, [[error description] UTF8String]);
220
- return NULL;
221
- }
222
-
219
+ MTLCompileOptions* options = nil;
223
220
  #ifdef GGML_QKK_64
224
- MTLCompileOptions* options = [MTLCompileOptions new];
225
- options.preprocessorMacros = @{ @"QK_K" : @(64) };
226
- ctx->library = [ctx->device newLibraryWithSource:src options:options error:&error];
227
- #else
228
- ctx->library = [ctx->device newLibraryWithSource:src options:nil error:&error];
221
+ options = [MTLCompileOptions new];
222
+ options.preprocessorMacros = @{ @"QK_K" : @(64) };
229
223
  #endif
224
+ ctx->library = [ctx->device newLibraryWithSource:src options:options error:&error];
225
+ }
226
+
230
227
  if (error) {
231
228
  GGML_METAL_LOG_ERROR("%s: error: %s\n", __func__, [[error description] UTF8String]);
232
229
  return NULL;
233
230
  }
234
231
  }
235
- #endif
236
232
 
237
233
  // load kernels
238
234
  {
@@ -264,6 +260,8 @@ struct ggml_metal_context * ggml_metal_init(int n_cb) {
264
260
  GGML_METAL_ADD_KERNEL(get_rows_f16);
265
261
  GGML_METAL_ADD_KERNEL(get_rows_q4_0);
266
262
  GGML_METAL_ADD_KERNEL(get_rows_q4_1);
263
+ GGML_METAL_ADD_KERNEL(get_rows_q5_0);
264
+ GGML_METAL_ADD_KERNEL(get_rows_q5_1);
267
265
  GGML_METAL_ADD_KERNEL(get_rows_q8_0);
268
266
  GGML_METAL_ADD_KERNEL(get_rows_q2_K);
269
267
  GGML_METAL_ADD_KERNEL(get_rows_q3_K);
@@ -272,40 +270,61 @@ struct ggml_metal_context * ggml_metal_init(int n_cb) {
272
270
  GGML_METAL_ADD_KERNEL(get_rows_q6_K);
273
271
  GGML_METAL_ADD_KERNEL(rms_norm);
274
272
  GGML_METAL_ADD_KERNEL(norm);
275
- GGML_METAL_ADD_KERNEL(mul_mat_f32_f32);
276
- GGML_METAL_ADD_KERNEL(mul_mat_f16_f32);
277
- GGML_METAL_ADD_KERNEL(mul_mat_f16_f32_1row);
278
- GGML_METAL_ADD_KERNEL(mul_mat_f16_f32_l4);
279
- GGML_METAL_ADD_KERNEL(mul_mat_q4_0_f32);
280
- GGML_METAL_ADD_KERNEL(mul_mat_q4_1_f32);
281
- GGML_METAL_ADD_KERNEL(mul_mat_q8_0_f32);
282
- GGML_METAL_ADD_KERNEL(mul_mat_q2_K_f32);
283
- GGML_METAL_ADD_KERNEL(mul_mat_q3_K_f32);
284
- GGML_METAL_ADD_KERNEL(mul_mat_q4_K_f32);
285
- GGML_METAL_ADD_KERNEL(mul_mat_q5_K_f32);
286
- GGML_METAL_ADD_KERNEL(mul_mat_q6_K_f32);
287
- GGML_METAL_ADD_KERNEL(mul_mm_f32_f32);
288
- GGML_METAL_ADD_KERNEL(mul_mm_f16_f32);
289
- GGML_METAL_ADD_KERNEL(mul_mm_q4_0_f32);
290
- GGML_METAL_ADD_KERNEL(mul_mm_q8_0_f32);
291
- GGML_METAL_ADD_KERNEL(mul_mm_q4_1_f32);
292
- GGML_METAL_ADD_KERNEL(mul_mm_q2_K_f32);
293
- GGML_METAL_ADD_KERNEL(mul_mm_q3_K_f32);
294
- GGML_METAL_ADD_KERNEL(mul_mm_q4_K_f32);
295
- GGML_METAL_ADD_KERNEL(mul_mm_q5_K_f32);
296
- GGML_METAL_ADD_KERNEL(mul_mm_q6_K_f32);
273
+ GGML_METAL_ADD_KERNEL(mul_mv_f32_f32);
274
+ GGML_METAL_ADD_KERNEL(mul_mv_f16_f32);
275
+ GGML_METAL_ADD_KERNEL(mul_mv_f16_f32_1row);
276
+ GGML_METAL_ADD_KERNEL(mul_mv_f16_f32_l4);
277
+ GGML_METAL_ADD_KERNEL(mul_mv_q4_0_f32);
278
+ GGML_METAL_ADD_KERNEL(mul_mv_q4_1_f32);
279
+ GGML_METAL_ADD_KERNEL(mul_mv_q5_0_f32);
280
+ GGML_METAL_ADD_KERNEL(mul_mv_q5_1_f32);
281
+ GGML_METAL_ADD_KERNEL(mul_mv_q8_0_f32);
282
+ GGML_METAL_ADD_KERNEL(mul_mv_q2_K_f32);
283
+ GGML_METAL_ADD_KERNEL(mul_mv_q3_K_f32);
284
+ GGML_METAL_ADD_KERNEL(mul_mv_q4_K_f32);
285
+ GGML_METAL_ADD_KERNEL(mul_mv_q5_K_f32);
286
+ GGML_METAL_ADD_KERNEL(mul_mv_q6_K_f32);
287
+ if ([ctx->device supportsFamily:MTLGPUFamilyApple7]) {
288
+ GGML_METAL_ADD_KERNEL(mul_mm_f32_f32);
289
+ GGML_METAL_ADD_KERNEL(mul_mm_f16_f32);
290
+ GGML_METAL_ADD_KERNEL(mul_mm_q4_0_f32);
291
+ GGML_METAL_ADD_KERNEL(mul_mm_q4_1_f32);
292
+ GGML_METAL_ADD_KERNEL(mul_mm_q5_0_f32);
293
+ GGML_METAL_ADD_KERNEL(mul_mm_q5_1_f32);
294
+ GGML_METAL_ADD_KERNEL(mul_mm_q8_0_f32);
295
+ GGML_METAL_ADD_KERNEL(mul_mm_q2_K_f32);
296
+ GGML_METAL_ADD_KERNEL(mul_mm_q3_K_f32);
297
+ GGML_METAL_ADD_KERNEL(mul_mm_q4_K_f32);
298
+ GGML_METAL_ADD_KERNEL(mul_mm_q5_K_f32);
299
+ GGML_METAL_ADD_KERNEL(mul_mm_q6_K_f32);
300
+ }
297
301
  GGML_METAL_ADD_KERNEL(rope_f32);
298
302
  GGML_METAL_ADD_KERNEL(rope_f16);
299
303
  GGML_METAL_ADD_KERNEL(alibi_f32);
300
304
  GGML_METAL_ADD_KERNEL(cpy_f32_f16);
301
305
  GGML_METAL_ADD_KERNEL(cpy_f32_f32);
302
306
  GGML_METAL_ADD_KERNEL(cpy_f16_f16);
307
+ GGML_METAL_ADD_KERNEL(concat);
308
+ GGML_METAL_ADD_KERNEL(sqr);
303
309
 
304
310
  #undef GGML_METAL_ADD_KERNEL
305
311
  }
306
312
 
307
- GGML_METAL_LOG_INFO("%s: hasUnifiedMemory = %s\n", __func__, ctx->device.hasUnifiedMemory ? "true" : "false");
308
313
  #if TARGET_OS_OSX
314
+ // print MTL GPU family:
315
+ GGML_METAL_LOG_INFO("%s: GPU name: %s\n", __func__, [[ctx->device name] UTF8String]);
316
+
317
+ // determine max supported GPU family
318
+ // https://developer.apple.com/metal/Metal-Shading-Language-Specification.pdf
319
+ // https://developer.apple.com/metal/Metal-Feature-Set-Tables.pdf
320
+ for (int i = MTLGPUFamilyApple1 + 20; i >= MTLGPUFamilyApple1; --i) {
321
+ if ([ctx->device supportsFamily:i]) {
322
+ GGML_METAL_LOG_INFO("%s: GPU family: MTLGPUFamilyApple%d (%d)\n", __func__, i - MTLGPUFamilyApple1 + 1, i);
323
+ break;
324
+ }
325
+ }
326
+
327
+ GGML_METAL_LOG_INFO("%s: hasUnifiedMemory = %s\n", __func__, ctx->device.hasUnifiedMemory ? "true" : "false");
309
328
  GGML_METAL_LOG_INFO("%s: recommendedMaxWorkingSetSize = %8.2f MB\n", __func__, ctx->device.recommendedMaxWorkingSetSize / 1024.0 / 1024.0);
310
329
  if (ctx->device.maxTransferRate != 0) {
311
330
  GGML_METAL_LOG_INFO("%s: maxTransferRate = %8.2f MB/s\n", __func__, ctx->device.maxTransferRate / 1024.0 / 1024.0);
@@ -339,6 +358,8 @@ void ggml_metal_free(struct ggml_metal_context * ctx) {
339
358
  GGML_METAL_DEL_KERNEL(get_rows_f16);
340
359
  GGML_METAL_DEL_KERNEL(get_rows_q4_0);
341
360
  GGML_METAL_DEL_KERNEL(get_rows_q4_1);
361
+ GGML_METAL_DEL_KERNEL(get_rows_q5_0);
362
+ GGML_METAL_DEL_KERNEL(get_rows_q5_1);
342
363
  GGML_METAL_DEL_KERNEL(get_rows_q8_0);
343
364
  GGML_METAL_DEL_KERNEL(get_rows_q2_K);
344
365
  GGML_METAL_DEL_KERNEL(get_rows_q3_K);
@@ -347,34 +368,42 @@ void ggml_metal_free(struct ggml_metal_context * ctx) {
347
368
  GGML_METAL_DEL_KERNEL(get_rows_q6_K);
348
369
  GGML_METAL_DEL_KERNEL(rms_norm);
349
370
  GGML_METAL_DEL_KERNEL(norm);
350
- GGML_METAL_DEL_KERNEL(mul_mat_f32_f32);
351
- GGML_METAL_DEL_KERNEL(mul_mat_f16_f32);
352
- GGML_METAL_DEL_KERNEL(mul_mat_f16_f32_1row);
353
- GGML_METAL_DEL_KERNEL(mul_mat_f16_f32_l4);
354
- GGML_METAL_DEL_KERNEL(mul_mat_q4_0_f32);
355
- GGML_METAL_DEL_KERNEL(mul_mat_q4_1_f32);
356
- GGML_METAL_DEL_KERNEL(mul_mat_q8_0_f32);
357
- GGML_METAL_DEL_KERNEL(mul_mat_q2_K_f32);
358
- GGML_METAL_DEL_KERNEL(mul_mat_q3_K_f32);
359
- GGML_METAL_DEL_KERNEL(mul_mat_q4_K_f32);
360
- GGML_METAL_DEL_KERNEL(mul_mat_q5_K_f32);
361
- GGML_METAL_DEL_KERNEL(mul_mat_q6_K_f32);
362
- GGML_METAL_DEL_KERNEL(mul_mm_f32_f32);
363
- GGML_METAL_DEL_KERNEL(mul_mm_f16_f32);
364
- GGML_METAL_DEL_KERNEL(mul_mm_q4_0_f32);
365
- GGML_METAL_DEL_KERNEL(mul_mm_q8_0_f32);
366
- GGML_METAL_DEL_KERNEL(mul_mm_q4_1_f32);
367
- GGML_METAL_DEL_KERNEL(mul_mm_q2_K_f32);
368
- GGML_METAL_DEL_KERNEL(mul_mm_q3_K_f32);
369
- GGML_METAL_DEL_KERNEL(mul_mm_q4_K_f32);
370
- GGML_METAL_DEL_KERNEL(mul_mm_q5_K_f32);
371
- GGML_METAL_DEL_KERNEL(mul_mm_q6_K_f32);
371
+ GGML_METAL_DEL_KERNEL(mul_mv_f32_f32);
372
+ GGML_METAL_DEL_KERNEL(mul_mv_f16_f32);
373
+ GGML_METAL_DEL_KERNEL(mul_mv_f16_f32_1row);
374
+ GGML_METAL_DEL_KERNEL(mul_mv_f16_f32_l4);
375
+ GGML_METAL_DEL_KERNEL(mul_mv_q4_0_f32);
376
+ GGML_METAL_DEL_KERNEL(mul_mv_q4_1_f32);
377
+ GGML_METAL_DEL_KERNEL(mul_mv_q5_0_f32);
378
+ GGML_METAL_DEL_KERNEL(mul_mv_q5_1_f32);
379
+ GGML_METAL_DEL_KERNEL(mul_mv_q8_0_f32);
380
+ GGML_METAL_DEL_KERNEL(mul_mv_q2_K_f32);
381
+ GGML_METAL_DEL_KERNEL(mul_mv_q3_K_f32);
382
+ GGML_METAL_DEL_KERNEL(mul_mv_q4_K_f32);
383
+ GGML_METAL_DEL_KERNEL(mul_mv_q5_K_f32);
384
+ GGML_METAL_DEL_KERNEL(mul_mv_q6_K_f32);
385
+ if ([ctx->device supportsFamily:MTLGPUFamilyApple7]) {
386
+ GGML_METAL_DEL_KERNEL(mul_mm_f32_f32);
387
+ GGML_METAL_DEL_KERNEL(mul_mm_f16_f32);
388
+ GGML_METAL_DEL_KERNEL(mul_mm_q4_0_f32);
389
+ GGML_METAL_DEL_KERNEL(mul_mm_q4_1_f32);
390
+ GGML_METAL_DEL_KERNEL(mul_mm_q5_0_f32);
391
+ GGML_METAL_DEL_KERNEL(mul_mm_q5_1_f32);
392
+ GGML_METAL_DEL_KERNEL(mul_mm_q8_0_f32);
393
+ GGML_METAL_DEL_KERNEL(mul_mm_q2_K_f32);
394
+ GGML_METAL_DEL_KERNEL(mul_mm_q3_K_f32);
395
+ GGML_METAL_DEL_KERNEL(mul_mm_q4_K_f32);
396
+ GGML_METAL_DEL_KERNEL(mul_mm_q5_K_f32);
397
+ GGML_METAL_DEL_KERNEL(mul_mm_q6_K_f32);
398
+ }
372
399
  GGML_METAL_DEL_KERNEL(rope_f32);
373
400
  GGML_METAL_DEL_KERNEL(rope_f16);
374
401
  GGML_METAL_DEL_KERNEL(alibi_f32);
375
402
  GGML_METAL_DEL_KERNEL(cpy_f32_f16);
376
403
  GGML_METAL_DEL_KERNEL(cpy_f32_f32);
377
404
  GGML_METAL_DEL_KERNEL(cpy_f16_f16);
405
+ GGML_METAL_DEL_KERNEL(concat);
406
+ GGML_METAL_DEL_KERNEL(sqr);
378
407
 
379
408
  #undef GGML_METAL_DEL_KERNEL
380
409
 
@@ -431,7 +460,7 @@ static id<MTLBuffer> ggml_metal_get_buffer(struct ggml_metal_context * ctx, stru
431
460
  for (int i = 0; i < ctx->n_buffers; ++i) {
432
461
  const int64_t ioffs = (int64_t) t->data - (int64_t) ctx->buffers[i].data;
433
462
 
434
- //metal_printf("ioffs = %10ld, tsize = %10ld, sum = %10ld, ctx->buffers[%d].size = %10ld, name = %s\n", ioffs, tsize, ioffs + tsize, i, ctx->buffers[i].size, ctx->buffers[i].name);
463
+ //GGML_METAL_LOG_INFO("ioffs = %10ld, tsize = %10ld, sum = %10ld, ctx->buffers[%d].size = %10ld, name = %s\n", ioffs, tsize, ioffs + tsize, i, ctx->buffers[i].size, ctx->buffers[i].name);
435
464
  if (ioffs >= 0 && ioffs + tsize <= (int64_t) ctx->buffers[i].size) {
436
465
  *offs = (size_t) ioffs;
437
466
 
@@ -766,6 +795,44 @@ void ggml_metal_graph_compute(
766
795
  {
767
796
  // noop
768
797
  } break;
798
+ case GGML_OP_CONCAT:
799
+ {
800
+ const int64_t nb = ne00;
801
+
802
+ [encoder setComputePipelineState:ctx->pipeline_concat];
803
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
804
+ [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];
805
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:2];
806
+ [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:3];
807
+ [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:4];
808
+ [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:5];
809
+ [encoder setBytes:&ne03 length:sizeof(ne03) atIndex:6];
810
+ [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:7];
811
+ [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:8];
812
+ [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:9];
813
+ [encoder setBytes:&nb03 length:sizeof(nb03) atIndex:10];
814
+ [encoder setBytes:&ne10 length:sizeof(ne10) atIndex:11];
815
+ [encoder setBytes:&ne11 length:sizeof(ne11) atIndex:12];
816
+ [encoder setBytes:&ne12 length:sizeof(ne12) atIndex:13];
817
+ [encoder setBytes:&ne13 length:sizeof(ne13) atIndex:14];
818
+ [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:15];
819
+ [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:16];
820
+ [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:17];
821
+ [encoder setBytes:&nb13 length:sizeof(nb13) atIndex:18];
822
+ [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:19];
823
+ [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:20];
824
+ [encoder setBytes:&ne2 length:sizeof(ne2) atIndex:21];
825
+ [encoder setBytes:&ne3 length:sizeof(ne3) atIndex:22];
826
+ [encoder setBytes:&nb0 length:sizeof(nb0) atIndex:23];
827
+ [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:24];
828
+ [encoder setBytes:&nb2 length:sizeof(nb2) atIndex:25];
829
+ [encoder setBytes:&nb3 length:sizeof(nb3) atIndex:26];
830
+ [encoder setBytes:&nb length:sizeof(nb) atIndex:27];
831
+
832
+ const int nth = MIN(1024, ne0);
833
+
834
+ [encoder dispatchThreadgroups:MTLSizeMake(ne1, ne2, ne3) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];
835
+ } break;
769
836
  case GGML_OP_ADD:
770
837
  {
771
838
  GGML_ASSERT(ggml_is_contiguous(src0));
@@ -861,9 +928,10 @@ void ggml_metal_graph_compute(
861
928
  [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
862
929
  [encoder setBytes:&scale length:sizeof(scale) atIndex:2];
863
930
 
864
- const int64_t n = ggml_nelements(dst)/4;
931
+ const int64_t n = ggml_nelements(dst);
932
+ GGML_ASSERT(n % 4 == 0);
865
933
 
866
- [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
934
+ [encoder dispatchThreadgroups:MTLSizeMake(n/4, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
867
935
  } break;
868
936
  case GGML_OP_UNARY:
869
937
  switch (ggml_get_unary_op(gf->nodes[i])) {
@@ -873,9 +941,10 @@ void ggml_metal_graph_compute(
873
941
  [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
874
942
  [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
875
943
 
876
- const int64_t n = ggml_nelements(dst)/4;
944
+ const int64_t n = ggml_nelements(dst);
945
+ GGML_ASSERT(n % 4 == 0);
877
946
 
878
- [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
947
+ [encoder dispatchThreadgroups:MTLSizeMake(n/4, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
879
948
  } break;
880
949
  case GGML_UNARY_OP_RELU:
881
950
  {
@@ -893,9 +962,10 @@ void ggml_metal_graph_compute(
893
962
  [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
894
963
  [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
895
964
 
896
- const int64_t n = ggml_nelements(dst)/4;
965
+ const int64_t n = ggml_nelements(dst);
966
+ GGML_ASSERT(n % 4 == 0);
897
967
 
898
- [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
968
+ [encoder dispatchThreadgroups:MTLSizeMake(n/4, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
899
969
  } break;
900
970
  default:
901
971
  {
@@ -903,6 +973,17 @@ void ggml_metal_graph_compute(
903
973
  GGML_ASSERT(false);
904
974
  }
905
975
  } break;
976
+ case GGML_OP_SQR:
977
+ {
978
+ GGML_ASSERT(ggml_is_contiguous(src0));
979
+
980
+ [encoder setComputePipelineState:ctx->pipeline_sqr];
981
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
982
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
983
+
984
+ const int64_t n = ggml_nelements(dst);
985
+ [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
986
+ } break;
906
987
  case GGML_OP_SOFT_MAX:
907
988
  {
908
989
  const int nth = MIN(32, ne00);
@@ -944,26 +1025,53 @@ void ggml_metal_graph_compute(
944
1025
  } break;
945
1026
  case GGML_OP_MUL_MAT:
946
1027
  {
947
- // TODO: needs to be updated after PR: https://github.com/ggerganov/ggml/pull/224
948
-
949
1028
  GGML_ASSERT(ne00 == ne10);
950
- // GGML_ASSERT(ne02 == ne12); // Should be checked on individual data types until broadcast is implemented everywhere
951
- uint gqa = ne12/ne02;
952
1029
  GGML_ASSERT(ne03 == ne13);
953
1030
 
1031
+ const uint gqa = ne12/ne02;
1032
+
1033
+ // find the break-even point where the matrix-matrix kernel becomes more efficient compared
1034
+ // to the matrix-vector kernel
1035
+ int ne11_mm_min = 1;
1036
+
1037
+ #if 0
1038
+ // the numbers below are measured on M2 Ultra for 7B and 13B models
1039
+ // these numbers do not translate to other devices or model sizes
1040
+ // TODO: need to find a better approach
1041
+ if ([ctx->device.name isEqualToString:@"Apple M2 Ultra"]) {
1042
+ switch (src0t) {
1043
+ case GGML_TYPE_F16: ne11_mm_min = 2; break;
1044
+ case GGML_TYPE_Q8_0: ne11_mm_min = 7; break;
1045
+ case GGML_TYPE_Q2_K: ne11_mm_min = 15; break;
1046
+ case GGML_TYPE_Q3_K: ne11_mm_min = 7; break;
1047
+ case GGML_TYPE_Q4_0:
1048
+ case GGML_TYPE_Q4_1: ne11_mm_min = 15; break;
1049
+ case GGML_TYPE_Q4_K: ne11_mm_min = 11; break;
1050
+ case GGML_TYPE_Q5_0: // not tested yet
1051
+ case GGML_TYPE_Q5_1: ne11_mm_min = 13; break; // not tested yet
1052
+ case GGML_TYPE_Q5_K: ne11_mm_min = 7; break;
1053
+ case GGML_TYPE_Q6_K: ne11_mm_min = 7; break;
1054
+ default: ne11_mm_min = 1; break;
1055
+ }
1056
+ }
1057
+ #endif
1058
+
954
1059
  // for now the matrix-matrix multiplication kernel only works on A14+/M1+ SoCs
955
1060
  // AMD GPU and older A-chips will reuse matrix-vector multiplication kernel
956
- if (!ggml_is_transposed(src0) &&
1061
+ if ([ctx->device supportsFamily:MTLGPUFamilyApple7] &&
1062
+ !ggml_is_transposed(src0) &&
957
1063
  !ggml_is_transposed(src1) &&
958
1064
  src1t == GGML_TYPE_F32 &&
959
- [ctx->device supportsFamily:MTLGPUFamilyApple7] &&
960
- ne00%32 == 0 &&
961
- ne11 > 2) {
1065
+ ne00 % 32 == 0 && ne00 >= 64 &&
1066
+ ne11 > ne11_mm_min) {
1067
+ //printf("matrix: ne00 = %6d, ne01 = %6d, ne02 = %6d, ne11 = %6d, ne12 = %6d\n", ne00, ne01, ne02, ne11, ne12);
962
1068
  switch (src0->type) {
963
1069
  case GGML_TYPE_F32: [encoder setComputePipelineState:ctx->pipeline_mul_mm_f32_f32]; break;
964
1070
  case GGML_TYPE_F16: [encoder setComputePipelineState:ctx->pipeline_mul_mm_f16_f32]; break;
965
1071
  case GGML_TYPE_Q4_0: [encoder setComputePipelineState:ctx->pipeline_mul_mm_q4_0_f32]; break;
966
1072
  case GGML_TYPE_Q4_1: [encoder setComputePipelineState:ctx->pipeline_mul_mm_q4_1_f32]; break;
1073
+ case GGML_TYPE_Q5_0: [encoder setComputePipelineState:ctx->pipeline_mul_mm_q5_0_f32]; break;
1074
+ case GGML_TYPE_Q5_1: [encoder setComputePipelineState:ctx->pipeline_mul_mm_q5_1_f32]; break;
967
1075
  case GGML_TYPE_Q8_0: [encoder setComputePipelineState:ctx->pipeline_mul_mm_q8_0_f32]; break;
968
1076
  case GGML_TYPE_Q2_K: [encoder setComputePipelineState:ctx->pipeline_mul_mm_q2_K_f32]; break;
969
1077
  case GGML_TYPE_Q3_K: [encoder setComputePipelineState:ctx->pipeline_mul_mm_q3_K_f32]; break;
@@ -987,17 +1095,18 @@ void ggml_metal_graph_compute(
987
1095
  [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:12];
988
1096
  [encoder setBytes:&gqa length:sizeof(gqa) atIndex:13];
989
1097
  [encoder setThreadgroupMemoryLength:8192 atIndex:0];
990
- [encoder dispatchThreadgroups:MTLSizeMake( (ne11+31)/32, (ne01+63) / 64, ne12) threadsPerThreadgroup:MTLSizeMake(128, 1, 1)];
1098
+ [encoder dispatchThreadgroups:MTLSizeMake( (ne11 + 31)/32, (ne01 + 63)/64, ne12) threadsPerThreadgroup:MTLSizeMake(128, 1, 1)];
991
1099
  } else {
992
1100
  int nth0 = 32;
993
1101
  int nth1 = 1;
994
1102
  int nrows = 1;
1103
+ //printf("vector: ne00 = %6d, ne01 = %6d, ne02 = %6d, ne11 = %6d, ne12 = %6d\n", ne00, ne01, ne02, ne11, ne12);
995
1104
 
996
1105
  // use custom matrix x vector kernel
997
1106
  switch (src0t) {
998
1107
  case GGML_TYPE_F32:
999
1108
  {
1000
- [encoder setComputePipelineState:ctx->pipeline_mul_mat_f32_f32];
1109
+ [encoder setComputePipelineState:ctx->pipeline_mul_mv_f32_f32];
1001
1110
  nrows = 4;
1002
1111
  } break;
1003
1112
  case GGML_TYPE_F16:
@@ -1005,12 +1114,12 @@ void ggml_metal_graph_compute(
1005
1114
  nth0 = 32;
1006
1115
  nth1 = 1;
1007
1116
  if (ne11 * ne12 < 4) {
1008
- [encoder setComputePipelineState:ctx->pipeline_mul_mat_f16_f32_1row];
1117
+ [encoder setComputePipelineState:ctx->pipeline_mul_mv_f16_f32_1row];
1009
1118
  } else if (ne00 >= 128 && ne01 >= 8 && ne00%4 == 0) {
1010
- [encoder setComputePipelineState:ctx->pipeline_mul_mat_f16_f32_l4];
1119
+ [encoder setComputePipelineState:ctx->pipeline_mul_mv_f16_f32_l4];
1011
1120
  nrows = ne11;
1012
1121
  } else {
1013
- [encoder setComputePipelineState:ctx->pipeline_mul_mat_f16_f32];
1122
+ [encoder setComputePipelineState:ctx->pipeline_mul_mv_f16_f32];
1014
1123
  nrows = 4;
1015
1124
  }
1016
1125
  } break;
@@ -1021,7 +1130,7 @@ void ggml_metal_graph_compute(
1021
1130
 
1022
1131
  nth0 = 8;
1023
1132
  nth1 = 8;
1024
- [encoder setComputePipelineState:ctx->pipeline_mul_mat_q4_0_f32];
1133
+ [encoder setComputePipelineState:ctx->pipeline_mul_mv_q4_0_f32];
1025
1134
  } break;
1026
1135
  case GGML_TYPE_Q4_1:
1027
1136
  {
@@ -1030,7 +1139,25 @@ void ggml_metal_graph_compute(
1030
1139
 
1031
1140
  nth0 = 8;
1032
1141
  nth1 = 8;
1033
- [encoder setComputePipelineState:ctx->pipeline_mul_mat_q4_1_f32];
1142
+ [encoder setComputePipelineState:ctx->pipeline_mul_mv_q4_1_f32];
1143
+ } break;
1144
+ case GGML_TYPE_Q5_0:
1145
+ {
1146
+ GGML_ASSERT(ne02 == 1);
1147
+ GGML_ASSERT(ne12 == 1);
1148
+
1149
+ nth0 = 8;
1150
+ nth1 = 8;
1151
+ [encoder setComputePipelineState:ctx->pipeline_mul_mv_q5_0_f32];
1152
+ } break;
1153
+ case GGML_TYPE_Q5_1:
1154
+ {
1155
+ GGML_ASSERT(ne02 == 1);
1156
+ GGML_ASSERT(ne12 == 1);
1157
+
1158
+ nth0 = 8;
1159
+ nth1 = 8;
1160
+ [encoder setComputePipelineState:ctx->pipeline_mul_mv_q5_1_f32];
1034
1161
  } break;
1035
1162
  case GGML_TYPE_Q8_0:
1036
1163
  {
@@ -1039,7 +1166,7 @@ void ggml_metal_graph_compute(
1039
1166
 
1040
1167
  nth0 = 8;
1041
1168
  nth1 = 8;
1042
- [encoder setComputePipelineState:ctx->pipeline_mul_mat_q8_0_f32];
1169
+ [encoder setComputePipelineState:ctx->pipeline_mul_mv_q8_0_f32];
1043
1170
  } break;
1044
1171
  case GGML_TYPE_Q2_K:
1045
1172
  {
@@ -1048,7 +1175,7 @@ void ggml_metal_graph_compute(
1048
1175
 
1049
1176
  nth0 = 2;
1050
1177
  nth1 = 32;
1051
- [encoder setComputePipelineState:ctx->pipeline_mul_mat_q2_K_f32];
1178
+ [encoder setComputePipelineState:ctx->pipeline_mul_mv_q2_K_f32];
1052
1179
  } break;
1053
1180
  case GGML_TYPE_Q3_K:
1054
1181
  {
@@ -1057,7 +1184,7 @@ void ggml_metal_graph_compute(
1057
1184
 
1058
1185
  nth0 = 2;
1059
1186
  nth1 = 32;
1060
- [encoder setComputePipelineState:ctx->pipeline_mul_mat_q3_K_f32];
1187
+ [encoder setComputePipelineState:ctx->pipeline_mul_mv_q3_K_f32];
1061
1188
  } break;
1062
1189
  case GGML_TYPE_Q4_K:
1063
1190
  {
@@ -1066,7 +1193,7 @@ void ggml_metal_graph_compute(
1066
1193
 
1067
1194
  nth0 = 4; //1;
1068
1195
  nth1 = 8; //32;
1069
- [encoder setComputePipelineState:ctx->pipeline_mul_mat_q4_K_f32];
1196
+ [encoder setComputePipelineState:ctx->pipeline_mul_mv_q4_K_f32];
1070
1197
  } break;
1071
1198
  case GGML_TYPE_Q5_K:
1072
1199
  {
@@ -1075,7 +1202,7 @@ void ggml_metal_graph_compute(
1075
1202
 
1076
1203
  nth0 = 2;
1077
1204
  nth1 = 32;
1078
- [encoder setComputePipelineState:ctx->pipeline_mul_mat_q5_K_f32];
1205
+ [encoder setComputePipelineState:ctx->pipeline_mul_mv_q5_K_f32];
1079
1206
  } break;
1080
1207
  case GGML_TYPE_Q6_K:
1081
1208
  {
@@ -1084,7 +1211,7 @@ void ggml_metal_graph_compute(
1084
1211
 
1085
1212
  nth0 = 2;
1086
1213
  nth1 = 32;
1087
- [encoder setComputePipelineState:ctx->pipeline_mul_mat_q6_K_f32];
1214
+ [encoder setComputePipelineState:ctx->pipeline_mul_mv_q6_K_f32];
1088
1215
  } break;
1089
1216
  default:
1090
1217
  {
@@ -1112,8 +1239,9 @@ void ggml_metal_graph_compute(
1112
1239
  [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:16];
1113
1240
  [encoder setBytes:&gqa length:sizeof(gqa) atIndex:17];
1114
1241
 
1115
- if (src0t == GGML_TYPE_Q4_0 || src0t == GGML_TYPE_Q4_1 || src0t == GGML_TYPE_Q8_0 ||
1116
- src0t == GGML_TYPE_Q2_K) {// || src0t == GGML_TYPE_Q4_K) {
1242
+ if (src0t == GGML_TYPE_Q4_0 || src0t == GGML_TYPE_Q4_1 ||
1243
+ src0t == GGML_TYPE_Q5_0 || src0t == GGML_TYPE_Q5_1 || src0t == GGML_TYPE_Q8_0 ||
1244
+ src0t == GGML_TYPE_Q2_K) { // || src0t == GGML_TYPE_Q4_K) {
1117
1245
  [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 7)/8, ne11, ne12) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];
1118
1246
  }
1119
1247
  else if (src0t == GGML_TYPE_Q4_K) {
@@ -1144,6 +1272,8 @@ void ggml_metal_graph_compute(
1144
1272
  case GGML_TYPE_F16: [encoder setComputePipelineState:ctx->pipeline_get_rows_f16]; break;
1145
1273
  case GGML_TYPE_Q4_0: [encoder setComputePipelineState:ctx->pipeline_get_rows_q4_0]; break;
1146
1274
  case GGML_TYPE_Q4_1: [encoder setComputePipelineState:ctx->pipeline_get_rows_q4_1]; break;
1275
+ case GGML_TYPE_Q5_0: [encoder setComputePipelineState:ctx->pipeline_get_rows_q5_0]; break;
1276
+ case GGML_TYPE_Q5_1: [encoder setComputePipelineState:ctx->pipeline_get_rows_q5_1]; break;
1147
1277
  case GGML_TYPE_Q8_0: [encoder setComputePipelineState:ctx->pipeline_get_rows_q8_0]; break;
1148
1278
  case GGML_TYPE_Q2_K: [encoder setComputePipelineState:ctx->pipeline_get_rows_q2_K]; break;
1149
1279
  case GGML_TYPE_Q3_K: [encoder setComputePipelineState:ctx->pipeline_get_rows_q3_K]; break;
@@ -1166,6 +1296,8 @@ void ggml_metal_graph_compute(
1166
1296
  } break;
1167
1297
  case GGML_OP_RMS_NORM:
1168
1298
  {
1299
+ GGML_ASSERT(ne00 % 4 == 0);
1300
+
1169
1301
  float eps;
1170
1302
  memcpy(&eps, dst->op_params, sizeof(float));
1171
1303
 
@@ -1208,7 +1340,7 @@ void ggml_metal_graph_compute(
1208
1340
 
1209
1341
  const int nth = MIN(1024, ne00);
1210
1342
 
1211
- const int n_past = ((int32_t *) dst->op_params)[0]; UNUSED(n_past);
1343
+ //const int n_past = ((int32_t *) dst->op_params)[0];
1212
1344
  const int n_head = ((int32_t *) dst->op_params)[1];
1213
1345
  float max_bias;
1214
1346
  memcpy(&max_bias, (int32_t *) dst->op_params + 2, sizeof(float));
@@ -1371,3 +1503,140 @@ void ggml_metal_graph_compute(
1371
1503
 
1372
1504
  }
1373
1505
  }
1506
+
1507
+ ////////////////////////////////////////////////////////////////////////////////
1508
+
1509
+ // backend interface
1510
+
1511
+ static const char * ggml_backend_metal_name(ggml_backend_t backend) {
1512
+ return "Metal";
1513
+
1514
+ UNUSED(backend);
1515
+ }
1516
+
1517
+ static void ggml_backend_metal_free(ggml_backend_t backend) {
1518
+ struct ggml_metal_context * ctx = (struct ggml_metal_context *)backend->context;
1519
+ ggml_metal_free(ctx);
1520
+ free(backend);
1521
+ }
1522
+
1523
+ static void * ggml_backend_metal_buffer_get_base(ggml_backend_buffer_t buffer) {
1524
+ return (void *)buffer->context;
1525
+ }
1526
+
1527
+ static void ggml_backend_metal_buffer_free_buffer(ggml_backend_buffer_t buffer) {
1528
+ free(buffer->context);
1529
+ UNUSED(buffer);
1530
+ }
1531
+
1532
+ static struct ggml_backend_buffer_i metal_backend_buffer_i = {
1533
+ /* .free_buffer = */ ggml_backend_metal_buffer_free_buffer,
1534
+ /* .get_base = */ ggml_backend_metal_buffer_get_base,
1535
+ /* .get_alloc_size = */ NULL, // defaults to ggml_nbytes
1536
+ /* .init_tensor = */ NULL, // no initialization required
1537
+ /* .free_tensor = */ NULL, // no cleanup required
1538
+ };
1539
+
1540
+ static ggml_backend_buffer_t ggml_backend_metal_alloc_buffer(ggml_backend_t backend, size_t size) {
1541
+ struct ggml_metal_context * ctx = (struct ggml_metal_context *)backend->context;
1542
+
1543
+ void * data = ggml_metal_host_malloc(size);
1544
+
1545
+ // TODO: set proper name of the buffers
1546
+ ggml_metal_add_buffer(ctx, "backend", data, size, 0);
1547
+
1548
+ return ggml_backend_buffer_init(backend, metal_backend_buffer_i, data, size);
1549
+ }
1550
+
1551
+ static size_t ggml_backend_metal_get_alignment(ggml_backend_t backend) {
1552
+ return 32;
1553
+ UNUSED(backend);
1554
+ }
1555
+
1556
+ static void ggml_backend_metal_set_tensor_async(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size) {
1557
+ GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && "tensor write out of bounds");
1558
+ GGML_ASSERT(tensor->data != NULL && "tensor not allocated");
1559
+
1560
+ memcpy((char *)tensor->data + offset, data, size);
1561
+
1562
+ UNUSED(backend);
1563
+ }
1564
+
1565
+ static void ggml_backend_metal_get_tensor_async(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size) {
1566
+ GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && "tensor read out of bounds");
1567
+ GGML_ASSERT(tensor->data != NULL && "tensor not allocated");
1568
+
1569
+ memcpy(data, (const char *)tensor->data + offset, size);
1570
+
1571
+ UNUSED(backend);
1572
+ }
1573
+
1574
+ static void ggml_backend_metal_synchronize(ggml_backend_t backend) {
1575
+ UNUSED(backend);
1576
+ }
1577
+
1578
+ static void ggml_backend_metal_cpy_tensor_from(ggml_backend_t backend, struct ggml_tensor * src, struct ggml_tensor * dst) {
1579
+ ggml_backend_tensor_get(src, dst->data, 0, ggml_nbytes(src));
1580
+
1581
+ UNUSED(backend);
1582
+ }
1583
+
1584
+ static void ggml_backend_metal_cpy_tensor_to(ggml_backend_t backend, struct ggml_tensor * src, struct ggml_tensor * dst) {
1585
+ ggml_backend_tensor_set_async(dst, src->data, 0, ggml_nbytes(src));
1586
+
1587
+ UNUSED(backend);
1588
+ }
1589
+
1590
+ static void ggml_backend_metal_graph_compute(ggml_backend_t backend, struct ggml_cgraph * cgraph) {
1591
+ struct ggml_metal_context * metal_ctx = (struct ggml_metal_context *)backend->context;
1592
+
1593
+ ggml_metal_graph_compute(metal_ctx, cgraph);
1594
+ }
1595
+
1596
+ static bool ggml_backend_metal_supports_op(ggml_backend_t backend, const struct ggml_tensor * op) {
1597
+ return true;
1598
+ UNUSED(backend);
1599
+ UNUSED(op);
1600
+ }
1601
+
1602
+ static struct ggml_backend_i metal_backend_i = {
1603
+ /* .get_name = */ ggml_backend_metal_name,
1604
+ /* .free = */ ggml_backend_metal_free,
1605
+ /* .alloc_buffer = */ ggml_backend_metal_alloc_buffer,
1606
+ /* .get_alignment = */ ggml_backend_metal_get_alignment,
1607
+ /* .set_tensor_async = */ ggml_backend_metal_set_tensor_async,
1608
+ /* .get_tensor_async = */ ggml_backend_metal_get_tensor_async,
1609
+ /* .synchronize = */ ggml_backend_metal_synchronize,
1610
+ /* .cpy_tensor_from = */ ggml_backend_metal_cpy_tensor_from,
1611
+ /* .cpy_tensor_to = */ ggml_backend_metal_cpy_tensor_to,
1612
+ /* .graph_plan_create = */ NULL, // the metal implementation does not require creating graph plans atm
1613
+ /* .graph_plan_free = */ NULL,
1614
+ /* .graph_plan_compute = */ NULL,
1615
+ /* .graph_compute = */ ggml_backend_metal_graph_compute,
1616
+ /* .supports_op = */ ggml_backend_metal_supports_op,
1617
+ };
1618
+
1619
+ ggml_backend_t ggml_backend_metal_init(void) {
1620
+ struct ggml_metal_context * ctx = malloc(sizeof(struct ggml_metal_context));
1621
+
1622
+ ctx = ggml_metal_init(GGML_DEFAULT_N_THREADS);
1623
+
1624
+ ggml_backend_t metal_backend = malloc(sizeof(struct ggml_backend));
1625
+
1626
+ *metal_backend = (struct ggml_backend) {
1627
+ /* .interface = */ metal_backend_i,
1628
+ /* .context = */ ctx,
1629
+ };
1630
+
1631
+ return metal_backend;
1632
+ }
1633
+
1634
+ bool ggml_backend_is_metal(ggml_backend_t backend) {
1635
+ return backend->iface.get_name == ggml_backend_metal_name;
1636
+ }
1637
+
1638
+ void ggml_backend_metal_set_n_cb(ggml_backend_t backend, int n_cb) {
1639
+ struct ggml_metal_context * ctx = (struct ggml_metal_context *)backend->context;
1640
+
1641
+ ggml_metal_set_n_cb(ctx, n_cb);
1642
+ }