llama_cpp 0.5.2 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +13 -0
- data/README.md +6 -5
- data/examples/chat.rb +13 -13
- data/examples/embedding.rb +9 -9
- data/ext/llama_cpp/llama_cpp.cpp +547 -272
- data/ext/llama_cpp/src/ggml-alloc.c +14 -8
- data/ext/llama_cpp/src/ggml-alloc.h +1 -0
- data/ext/llama_cpp/src/ggml-cuda.cu +307 -127
- data/ext/llama_cpp/src/ggml-cuda.h +1 -0
- data/ext/llama_cpp/src/ggml-metal.h +4 -0
- data/ext/llama_cpp/src/ggml-metal.m +200 -94
- data/ext/llama_cpp/src/ggml-metal.metal +264 -82
- data/ext/llama_cpp/src/ggml-opencl.cpp +3 -3
- data/ext/llama_cpp/src/ggml.c +1647 -865
- data/ext/llama_cpp/src/ggml.h +143 -52
- data/ext/llama_cpp/src/llama.cpp +1427 -635
- data/ext/llama_cpp/src/llama.h +308 -119
- data/lib/llama_cpp/version.rb +2 -2
- data/lib/llama_cpp.rb +5 -9
- data/sig/llama_cpp.rbs +65 -34
- metadata +3 -3
|
@@ -11,11 +11,14 @@
|
|
|
11
11
|
#define MIN(a, b) ((a) < (b) ? (a) : (b))
|
|
12
12
|
#define MAX(a, b) ((a) > (b) ? (a) : (b))
|
|
13
13
|
|
|
14
|
-
// TODO: temporary - reuse llama.cpp logging
|
|
15
14
|
#ifdef GGML_METAL_NDEBUG
|
|
16
|
-
#define
|
|
15
|
+
#define GGML_METAL_LOG_INFO(...)
|
|
16
|
+
#define GGML_METAL_LOG_WARN(...)
|
|
17
|
+
#define GGML_METAL_LOG_ERROR(...)
|
|
17
18
|
#else
|
|
18
|
-
#define
|
|
19
|
+
#define GGML_METAL_LOG_INFO(...) ggml_metal_log(GGML_LOG_LEVEL_INFO, __VA_ARGS__)
|
|
20
|
+
#define GGML_METAL_LOG_WARN(...) ggml_metal_log(GGML_LOG_LEVEL_WARN, __VA_ARGS__)
|
|
21
|
+
#define GGML_METAL_LOG_ERROR(...) ggml_metal_log(GGML_LOG_LEVEL_ERROR, __VA_ARGS__)
|
|
19
22
|
#endif
|
|
20
23
|
|
|
21
24
|
#define UNUSED(x) (void)(x)
|
|
@@ -66,6 +69,7 @@ struct ggml_metal_context {
|
|
|
66
69
|
GGML_METAL_DECL_KERNEL(soft_max_4);
|
|
67
70
|
GGML_METAL_DECL_KERNEL(diag_mask_inf);
|
|
68
71
|
GGML_METAL_DECL_KERNEL(diag_mask_inf_8);
|
|
72
|
+
GGML_METAL_DECL_KERNEL(get_rows_f32);
|
|
69
73
|
GGML_METAL_DECL_KERNEL(get_rows_f16);
|
|
70
74
|
GGML_METAL_DECL_KERNEL(get_rows_q4_0);
|
|
71
75
|
GGML_METAL_DECL_KERNEL(get_rows_q4_1);
|
|
@@ -77,6 +81,7 @@ struct ggml_metal_context {
|
|
|
77
81
|
GGML_METAL_DECL_KERNEL(get_rows_q6_K);
|
|
78
82
|
GGML_METAL_DECL_KERNEL(rms_norm);
|
|
79
83
|
GGML_METAL_DECL_KERNEL(norm);
|
|
84
|
+
GGML_METAL_DECL_KERNEL(mul_mat_f32_f32);
|
|
80
85
|
GGML_METAL_DECL_KERNEL(mul_mat_f16_f32);
|
|
81
86
|
GGML_METAL_DECL_KERNEL(mul_mat_f16_f32_1row);
|
|
82
87
|
GGML_METAL_DECL_KERNEL(mul_mat_f16_f32_l4);
|
|
@@ -88,6 +93,7 @@ struct ggml_metal_context {
|
|
|
88
93
|
GGML_METAL_DECL_KERNEL(mul_mat_q4_K_f32);
|
|
89
94
|
GGML_METAL_DECL_KERNEL(mul_mat_q5_K_f32);
|
|
90
95
|
GGML_METAL_DECL_KERNEL(mul_mat_q6_K_f32);
|
|
96
|
+
GGML_METAL_DECL_KERNEL(mul_mm_f32_f32);
|
|
91
97
|
GGML_METAL_DECL_KERNEL(mul_mm_f16_f32);
|
|
92
98
|
GGML_METAL_DECL_KERNEL(mul_mm_q4_0_f32);
|
|
93
99
|
GGML_METAL_DECL_KERNEL(mul_mm_q4_1_f32);
|
|
@@ -97,7 +103,8 @@ struct ggml_metal_context {
|
|
|
97
103
|
GGML_METAL_DECL_KERNEL(mul_mm_q4_K_f32);
|
|
98
104
|
GGML_METAL_DECL_KERNEL(mul_mm_q5_K_f32);
|
|
99
105
|
GGML_METAL_DECL_KERNEL(mul_mm_q6_K_f32);
|
|
100
|
-
GGML_METAL_DECL_KERNEL(
|
|
106
|
+
GGML_METAL_DECL_KERNEL(rope_f32);
|
|
107
|
+
GGML_METAL_DECL_KERNEL(rope_f16);
|
|
101
108
|
GGML_METAL_DECL_KERNEL(alibi_f32);
|
|
102
109
|
GGML_METAL_DECL_KERNEL(cpy_f32_f16);
|
|
103
110
|
GGML_METAL_DECL_KERNEL(cpy_f32_f32);
|
|
@@ -117,8 +124,37 @@ static NSString * const msl_library_source = @"see metal.metal";
|
|
|
117
124
|
@implementation GGMLMetalClass
|
|
118
125
|
@end
|
|
119
126
|
|
|
127
|
+
ggml_log_callback ggml_metal_log_callback = NULL;
|
|
128
|
+
void * ggml_metal_log_user_data = NULL;
|
|
129
|
+
|
|
130
|
+
void ggml_metal_log_set_callback(ggml_log_callback log_callback, void * user_data) {
|
|
131
|
+
ggml_metal_log_callback = log_callback;
|
|
132
|
+
ggml_metal_log_user_data = user_data;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
static void ggml_metal_log(enum ggml_log_level level, const char* format, ...){
|
|
136
|
+
if (ggml_metal_log_callback != NULL) {
|
|
137
|
+
va_list args;
|
|
138
|
+
va_start(args, format);
|
|
139
|
+
char buffer[128];
|
|
140
|
+
int len = vsnprintf(buffer, 128, format, args);
|
|
141
|
+
if (len < 128) {
|
|
142
|
+
ggml_metal_log_callback(level, buffer, ggml_metal_log_user_data);
|
|
143
|
+
} else {
|
|
144
|
+
char* buffer2 = malloc(len+1);
|
|
145
|
+
vsnprintf(buffer2, len+1, format, args);
|
|
146
|
+
buffer2[len] = 0;
|
|
147
|
+
ggml_metal_log_callback(level, buffer2, ggml_metal_log_user_data);
|
|
148
|
+
free(buffer2);
|
|
149
|
+
}
|
|
150
|
+
va_end(args);
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
|
|
120
156
|
struct ggml_metal_context * ggml_metal_init(int n_cb) {
|
|
121
|
-
|
|
157
|
+
GGML_METAL_LOG_INFO("%s: allocating\n", __func__);
|
|
122
158
|
|
|
123
159
|
id <MTLDevice> device;
|
|
124
160
|
NSString * s;
|
|
@@ -128,14 +164,14 @@ struct ggml_metal_context * ggml_metal_init(int n_cb) {
|
|
|
128
164
|
NSArray * devices = MTLCopyAllDevices();
|
|
129
165
|
for (device in devices) {
|
|
130
166
|
s = [device name];
|
|
131
|
-
|
|
167
|
+
GGML_METAL_LOG_INFO("%s: found device: %s\n", __func__, [s UTF8String]);
|
|
132
168
|
}
|
|
133
169
|
#endif
|
|
134
170
|
|
|
135
171
|
// Pick and show default Metal device
|
|
136
172
|
device = MTLCreateSystemDefaultDevice();
|
|
137
173
|
s = [device name];
|
|
138
|
-
|
|
174
|
+
GGML_METAL_LOG_INFO("%s: picking default device: %s\n", __func__, [s UTF8String]);
|
|
139
175
|
|
|
140
176
|
// Configure context
|
|
141
177
|
struct ggml_metal_context * ctx = malloc(sizeof(struct ggml_metal_context));
|
|
@@ -145,7 +181,7 @@ struct ggml_metal_context * ggml_metal_init(int n_cb) {
|
|
|
145
181
|
ctx->n_buffers = 0;
|
|
146
182
|
ctx->concur_list_len = 0;
|
|
147
183
|
|
|
148
|
-
ctx->d_queue = dispatch_queue_create("
|
|
184
|
+
ctx->d_queue = dispatch_queue_create("ggml-metal", DISPATCH_QUEUE_CONCURRENT);
|
|
149
185
|
|
|
150
186
|
#ifdef GGML_SWIFT
|
|
151
187
|
// load the default.metallib file
|
|
@@ -162,7 +198,7 @@ struct ggml_metal_context * ggml_metal_init(int n_cb) {
|
|
|
162
198
|
ctx->library = [ctx->device newLibraryWithURL:libURL error:&error];
|
|
163
199
|
|
|
164
200
|
if (error) {
|
|
165
|
-
|
|
201
|
+
GGML_METAL_LOG_ERROR("%s: error: %s\n", __func__, [[error description] UTF8String]);
|
|
166
202
|
return NULL;
|
|
167
203
|
}
|
|
168
204
|
}
|
|
@@ -175,12 +211,12 @@ struct ggml_metal_context * ggml_metal_init(int n_cb) {
|
|
|
175
211
|
|
|
176
212
|
//NSString * path = [[NSBundle mainBundle] pathForResource:@"../../examples/metal/metal" ofType:@"metal"];
|
|
177
213
|
NSBundle * bundle = [NSBundle bundleForClass:[GGMLMetalClass class]];
|
|
178
|
-
NSString * path
|
|
179
|
-
|
|
214
|
+
NSString * path = [bundle pathForResource:@"ggml-metal" ofType:@"metal"];
|
|
215
|
+
GGML_METAL_LOG_INFO("%s: loading '%s'\n", __func__, [path UTF8String]);
|
|
180
216
|
|
|
181
217
|
NSString * src = [NSString stringWithContentsOfFile:path encoding:NSUTF8StringEncoding error:&error];
|
|
182
218
|
if (error) {
|
|
183
|
-
|
|
219
|
+
GGML_METAL_LOG_ERROR("%s: error: %s\n", __func__, [[error description] UTF8String]);
|
|
184
220
|
return NULL;
|
|
185
221
|
}
|
|
186
222
|
|
|
@@ -192,7 +228,7 @@ struct ggml_metal_context * ggml_metal_init(int n_cb) {
|
|
|
192
228
|
ctx->library = [ctx->device newLibraryWithSource:src options:nil error:&error];
|
|
193
229
|
#endif
|
|
194
230
|
if (error) {
|
|
195
|
-
|
|
231
|
+
GGML_METAL_LOG_ERROR("%s: error: %s\n", __func__, [[error description] UTF8String]);
|
|
196
232
|
return NULL;
|
|
197
233
|
}
|
|
198
234
|
}
|
|
@@ -204,11 +240,11 @@ struct ggml_metal_context * ggml_metal_init(int n_cb) {
|
|
|
204
240
|
#define GGML_METAL_ADD_KERNEL(name) \
|
|
205
241
|
ctx->function_##name = [ctx->library newFunctionWithName:@"kernel_"#name]; \
|
|
206
242
|
ctx->pipeline_##name = [ctx->device newComputePipelineStateWithFunction:ctx->function_##name error:&error]; \
|
|
207
|
-
|
|
243
|
+
GGML_METAL_LOG_INFO("%s: loaded %-32s %16p | th_max = %4d | th_width = %4d\n", __func__, "kernel_"#name, (void *) ctx->pipeline_##name, \
|
|
208
244
|
(int) ctx->pipeline_##name.maxTotalThreadsPerThreadgroup, \
|
|
209
245
|
(int) ctx->pipeline_##name.threadExecutionWidth); \
|
|
210
246
|
if (error) { \
|
|
211
|
-
|
|
247
|
+
GGML_METAL_LOG_ERROR("%s: error: load pipeline error: %s\n", __func__, [[error description] UTF8String]); \
|
|
212
248
|
return NULL; \
|
|
213
249
|
}
|
|
214
250
|
|
|
@@ -224,6 +260,7 @@ struct ggml_metal_context * ggml_metal_init(int n_cb) {
|
|
|
224
260
|
GGML_METAL_ADD_KERNEL(soft_max_4);
|
|
225
261
|
GGML_METAL_ADD_KERNEL(diag_mask_inf);
|
|
226
262
|
GGML_METAL_ADD_KERNEL(diag_mask_inf_8);
|
|
263
|
+
GGML_METAL_ADD_KERNEL(get_rows_f32);
|
|
227
264
|
GGML_METAL_ADD_KERNEL(get_rows_f16);
|
|
228
265
|
GGML_METAL_ADD_KERNEL(get_rows_q4_0);
|
|
229
266
|
GGML_METAL_ADD_KERNEL(get_rows_q4_1);
|
|
@@ -235,6 +272,7 @@ struct ggml_metal_context * ggml_metal_init(int n_cb) {
|
|
|
235
272
|
GGML_METAL_ADD_KERNEL(get_rows_q6_K);
|
|
236
273
|
GGML_METAL_ADD_KERNEL(rms_norm);
|
|
237
274
|
GGML_METAL_ADD_KERNEL(norm);
|
|
275
|
+
GGML_METAL_ADD_KERNEL(mul_mat_f32_f32);
|
|
238
276
|
GGML_METAL_ADD_KERNEL(mul_mat_f16_f32);
|
|
239
277
|
GGML_METAL_ADD_KERNEL(mul_mat_f16_f32_1row);
|
|
240
278
|
GGML_METAL_ADD_KERNEL(mul_mat_f16_f32_l4);
|
|
@@ -246,6 +284,7 @@ struct ggml_metal_context * ggml_metal_init(int n_cb) {
|
|
|
246
284
|
GGML_METAL_ADD_KERNEL(mul_mat_q4_K_f32);
|
|
247
285
|
GGML_METAL_ADD_KERNEL(mul_mat_q5_K_f32);
|
|
248
286
|
GGML_METAL_ADD_KERNEL(mul_mat_q6_K_f32);
|
|
287
|
+
GGML_METAL_ADD_KERNEL(mul_mm_f32_f32);
|
|
249
288
|
GGML_METAL_ADD_KERNEL(mul_mm_f16_f32);
|
|
250
289
|
GGML_METAL_ADD_KERNEL(mul_mm_q4_0_f32);
|
|
251
290
|
GGML_METAL_ADD_KERNEL(mul_mm_q8_0_f32);
|
|
@@ -255,7 +294,8 @@ struct ggml_metal_context * ggml_metal_init(int n_cb) {
|
|
|
255
294
|
GGML_METAL_ADD_KERNEL(mul_mm_q4_K_f32);
|
|
256
295
|
GGML_METAL_ADD_KERNEL(mul_mm_q5_K_f32);
|
|
257
296
|
GGML_METAL_ADD_KERNEL(mul_mm_q6_K_f32);
|
|
258
|
-
GGML_METAL_ADD_KERNEL(
|
|
297
|
+
GGML_METAL_ADD_KERNEL(rope_f32);
|
|
298
|
+
GGML_METAL_ADD_KERNEL(rope_f16);
|
|
259
299
|
GGML_METAL_ADD_KERNEL(alibi_f32);
|
|
260
300
|
GGML_METAL_ADD_KERNEL(cpy_f32_f16);
|
|
261
301
|
GGML_METAL_ADD_KERNEL(cpy_f32_f32);
|
|
@@ -264,13 +304,13 @@ struct ggml_metal_context * ggml_metal_init(int n_cb) {
|
|
|
264
304
|
#undef GGML_METAL_ADD_KERNEL
|
|
265
305
|
}
|
|
266
306
|
|
|
267
|
-
|
|
307
|
+
GGML_METAL_LOG_INFO("%s: hasUnifiedMemory = %s\n", __func__, ctx->device.hasUnifiedMemory ? "true" : "false");
|
|
268
308
|
#if TARGET_OS_OSX
|
|
269
|
-
|
|
309
|
+
GGML_METAL_LOG_INFO("%s: recommendedMaxWorkingSetSize = %8.2f MB\n", __func__, ctx->device.recommendedMaxWorkingSetSize / 1024.0 / 1024.0);
|
|
270
310
|
if (ctx->device.maxTransferRate != 0) {
|
|
271
|
-
|
|
311
|
+
GGML_METAL_LOG_INFO("%s: maxTransferRate = %8.2f MB/s\n", __func__, ctx->device.maxTransferRate / 1024.0 / 1024.0);
|
|
272
312
|
} else {
|
|
273
|
-
|
|
313
|
+
GGML_METAL_LOG_INFO("%s: maxTransferRate = built-in GPU\n", __func__);
|
|
274
314
|
}
|
|
275
315
|
#endif
|
|
276
316
|
|
|
@@ -278,7 +318,7 @@ struct ggml_metal_context * ggml_metal_init(int n_cb) {
|
|
|
278
318
|
}
|
|
279
319
|
|
|
280
320
|
void ggml_metal_free(struct ggml_metal_context * ctx) {
|
|
281
|
-
|
|
321
|
+
GGML_METAL_LOG_INFO("%s: deallocating\n", __func__);
|
|
282
322
|
#define GGML_METAL_DEL_KERNEL(name) \
|
|
283
323
|
[ctx->function_##name release]; \
|
|
284
324
|
[ctx->pipeline_##name release];
|
|
@@ -293,7 +333,9 @@ void ggml_metal_free(struct ggml_metal_context * ctx) {
|
|
|
293
333
|
GGML_METAL_DEL_KERNEL(gelu);
|
|
294
334
|
GGML_METAL_DEL_KERNEL(soft_max);
|
|
295
335
|
GGML_METAL_DEL_KERNEL(soft_max_4);
|
|
336
|
+
GGML_METAL_DEL_KERNEL(diag_mask_inf);
|
|
296
337
|
GGML_METAL_DEL_KERNEL(diag_mask_inf_8);
|
|
338
|
+
GGML_METAL_DEL_KERNEL(get_rows_f32);
|
|
297
339
|
GGML_METAL_DEL_KERNEL(get_rows_f16);
|
|
298
340
|
GGML_METAL_DEL_KERNEL(get_rows_q4_0);
|
|
299
341
|
GGML_METAL_DEL_KERNEL(get_rows_q4_1);
|
|
@@ -305,6 +347,7 @@ void ggml_metal_free(struct ggml_metal_context * ctx) {
|
|
|
305
347
|
GGML_METAL_DEL_KERNEL(get_rows_q6_K);
|
|
306
348
|
GGML_METAL_DEL_KERNEL(rms_norm);
|
|
307
349
|
GGML_METAL_DEL_KERNEL(norm);
|
|
350
|
+
GGML_METAL_DEL_KERNEL(mul_mat_f32_f32);
|
|
308
351
|
GGML_METAL_DEL_KERNEL(mul_mat_f16_f32);
|
|
309
352
|
GGML_METAL_DEL_KERNEL(mul_mat_f16_f32_1row);
|
|
310
353
|
GGML_METAL_DEL_KERNEL(mul_mat_f16_f32_l4);
|
|
@@ -316,6 +359,7 @@ void ggml_metal_free(struct ggml_metal_context * ctx) {
|
|
|
316
359
|
GGML_METAL_DEL_KERNEL(mul_mat_q4_K_f32);
|
|
317
360
|
GGML_METAL_DEL_KERNEL(mul_mat_q5_K_f32);
|
|
318
361
|
GGML_METAL_DEL_KERNEL(mul_mat_q6_K_f32);
|
|
362
|
+
GGML_METAL_DEL_KERNEL(mul_mm_f32_f32);
|
|
319
363
|
GGML_METAL_DEL_KERNEL(mul_mm_f16_f32);
|
|
320
364
|
GGML_METAL_DEL_KERNEL(mul_mm_q4_0_f32);
|
|
321
365
|
GGML_METAL_DEL_KERNEL(mul_mm_q8_0_f32);
|
|
@@ -325,7 +369,8 @@ void ggml_metal_free(struct ggml_metal_context * ctx) {
|
|
|
325
369
|
GGML_METAL_DEL_KERNEL(mul_mm_q4_K_f32);
|
|
326
370
|
GGML_METAL_DEL_KERNEL(mul_mm_q5_K_f32);
|
|
327
371
|
GGML_METAL_DEL_KERNEL(mul_mm_q6_K_f32);
|
|
328
|
-
GGML_METAL_DEL_KERNEL(
|
|
372
|
+
GGML_METAL_DEL_KERNEL(rope_f32);
|
|
373
|
+
GGML_METAL_DEL_KERNEL(rope_f16);
|
|
329
374
|
GGML_METAL_DEL_KERNEL(alibi_f32);
|
|
330
375
|
GGML_METAL_DEL_KERNEL(cpy_f32_f16);
|
|
331
376
|
GGML_METAL_DEL_KERNEL(cpy_f32_f32);
|
|
@@ -350,7 +395,7 @@ void * ggml_metal_host_malloc(size_t n) {
|
|
|
350
395
|
void * data = NULL;
|
|
351
396
|
const int result = posix_memalign((void **) &data, sysconf(_SC_PAGESIZE), n);
|
|
352
397
|
if (result != 0) {
|
|
353
|
-
|
|
398
|
+
GGML_METAL_LOG_ERROR("%s: error: posix_memalign failed\n", __func__);
|
|
354
399
|
return NULL;
|
|
355
400
|
}
|
|
356
401
|
|
|
@@ -378,7 +423,7 @@ int * ggml_metal_get_concur_list(struct ggml_metal_context * ctx) {
|
|
|
378
423
|
// Metal buffer based on the host memory pointer
|
|
379
424
|
//
|
|
380
425
|
static id<MTLBuffer> ggml_metal_get_buffer(struct ggml_metal_context * ctx, struct ggml_tensor * t, size_t * offs) {
|
|
381
|
-
//
|
|
426
|
+
//GGML_METAL_LOG_INFO("%s: data tensor '%16s', offs_data = %8ld, offs_eval = %8ld, offs_cach = %8ld\n", __func__, t->name, offs_data, offs_eval, offs_cach);
|
|
382
427
|
|
|
383
428
|
const int64_t tsize = ggml_nbytes(t);
|
|
384
429
|
|
|
@@ -386,16 +431,17 @@ static id<MTLBuffer> ggml_metal_get_buffer(struct ggml_metal_context * ctx, stru
|
|
|
386
431
|
for (int i = 0; i < ctx->n_buffers; ++i) {
|
|
387
432
|
const int64_t ioffs = (int64_t) t->data - (int64_t) ctx->buffers[i].data;
|
|
388
433
|
|
|
434
|
+
//metal_printf("ioffs = %10ld, tsize = %10ld, sum = %10ld, ctx->buffers[%d].size = %10ld, name = %s\n", ioffs, tsize, ioffs + tsize, i, ctx->buffers[i].size, ctx->buffers[i].name);
|
|
389
435
|
if (ioffs >= 0 && ioffs + tsize <= (int64_t) ctx->buffers[i].size) {
|
|
390
436
|
*offs = (size_t) ioffs;
|
|
391
437
|
|
|
392
|
-
//
|
|
438
|
+
//GGML_METAL_LOG_INFO("%s: '%s' tensor '%16s', offs = %8ld\n", __func__, ctx->buffers[i].name, t->name, *offs);
|
|
393
439
|
|
|
394
440
|
return ctx->buffers[i].metal;
|
|
395
441
|
}
|
|
396
442
|
}
|
|
397
443
|
|
|
398
|
-
|
|
444
|
+
GGML_METAL_LOG_ERROR("%s: error: buffer is nil\n", __func__);
|
|
399
445
|
|
|
400
446
|
return nil;
|
|
401
447
|
}
|
|
@@ -407,7 +453,7 @@ bool ggml_metal_add_buffer(
|
|
|
407
453
|
size_t size,
|
|
408
454
|
size_t max_size) {
|
|
409
455
|
if (ctx->n_buffers >= GGML_METAL_MAX_BUFFERS) {
|
|
410
|
-
|
|
456
|
+
GGML_METAL_LOG_ERROR("%s: error: too many buffers\n", __func__);
|
|
411
457
|
return false;
|
|
412
458
|
}
|
|
413
459
|
|
|
@@ -417,7 +463,7 @@ bool ggml_metal_add_buffer(
|
|
|
417
463
|
const int64_t ioffs = (int64_t) data - (int64_t) ctx->buffers[i].data;
|
|
418
464
|
|
|
419
465
|
if (ioffs >= 0 && ioffs < (int64_t) ctx->buffers[i].size) {
|
|
420
|
-
|
|
466
|
+
GGML_METAL_LOG_ERROR("%s: error: buffer '%s' overlaps with '%s'\n", __func__, name, ctx->buffers[i].name);
|
|
421
467
|
return false;
|
|
422
468
|
}
|
|
423
469
|
}
|
|
@@ -438,11 +484,11 @@ bool ggml_metal_add_buffer(
|
|
|
438
484
|
ctx->buffers[ctx->n_buffers].metal = [ctx->device newBufferWithBytesNoCopy:data length:size_aligned options:MTLResourceStorageModeShared deallocator:nil];
|
|
439
485
|
|
|
440
486
|
if (ctx->buffers[ctx->n_buffers].metal == nil) {
|
|
441
|
-
|
|
487
|
+
GGML_METAL_LOG_ERROR("%s: error: failed to allocate '%-16s' buffer, size = %8.2f MB\n", __func__, name, size_aligned / 1024.0 / 1024.0);
|
|
442
488
|
return false;
|
|
443
489
|
}
|
|
444
490
|
|
|
445
|
-
|
|
491
|
+
GGML_METAL_LOG_INFO("%s: allocated '%-16s' buffer, size = %8.2f MB", __func__, name, size_aligned / 1024.0 / 1024.0);
|
|
446
492
|
|
|
447
493
|
++ctx->n_buffers;
|
|
448
494
|
} else {
|
|
@@ -462,13 +508,13 @@ bool ggml_metal_add_buffer(
|
|
|
462
508
|
ctx->buffers[ctx->n_buffers].metal = [ctx->device newBufferWithBytesNoCopy:(void *) ((uint8_t *) data + i) length:size_step_aligned options:MTLResourceStorageModeShared deallocator:nil];
|
|
463
509
|
|
|
464
510
|
if (ctx->buffers[ctx->n_buffers].metal == nil) {
|
|
465
|
-
|
|
511
|
+
GGML_METAL_LOG_ERROR("%s: error: failed to allocate '%-16s' buffer, size = %8.2f MB\n", __func__, name, size_step_aligned / 1024.0 / 1024.0);
|
|
466
512
|
return false;
|
|
467
513
|
}
|
|
468
514
|
|
|
469
|
-
|
|
515
|
+
GGML_METAL_LOG_INFO("%s: allocated '%-16s' buffer, size = %8.2f MB, offs = %12ld", __func__, name, size_step_aligned / 1024.0 / 1024.0, i);
|
|
470
516
|
if (i + size_step < size) {
|
|
471
|
-
|
|
517
|
+
GGML_METAL_LOG_INFO("\n");
|
|
472
518
|
}
|
|
473
519
|
|
|
474
520
|
++ctx->n_buffers;
|
|
@@ -476,17 +522,17 @@ bool ggml_metal_add_buffer(
|
|
|
476
522
|
}
|
|
477
523
|
|
|
478
524
|
#if TARGET_OS_OSX
|
|
479
|
-
|
|
525
|
+
GGML_METAL_LOG_INFO(", (%8.2f / %8.2f)",
|
|
480
526
|
ctx->device.currentAllocatedSize / 1024.0 / 1024.0,
|
|
481
527
|
ctx->device.recommendedMaxWorkingSetSize / 1024.0 / 1024.0);
|
|
482
528
|
|
|
483
529
|
if (ctx->device.currentAllocatedSize > ctx->device.recommendedMaxWorkingSetSize) {
|
|
484
|
-
|
|
530
|
+
GGML_METAL_LOG_WARN(", warning: current allocated size is greater than the recommended max working set size\n", __func__);
|
|
485
531
|
} else {
|
|
486
|
-
|
|
532
|
+
GGML_METAL_LOG_INFO("\n");
|
|
487
533
|
}
|
|
488
534
|
#else
|
|
489
|
-
|
|
535
|
+
GGML_METAL_LOG_INFO(", (%8.2f)\n", ctx->device.currentAllocatedSize / 1024.0 / 1024.0);
|
|
490
536
|
#endif
|
|
491
537
|
}
|
|
492
538
|
|
|
@@ -599,7 +645,7 @@ void ggml_metal_graph_find_concurrency(
|
|
|
599
645
|
}
|
|
600
646
|
|
|
601
647
|
if (ctx->concur_list_len > GGML_MAX_CONCUR) {
|
|
602
|
-
|
|
648
|
+
GGML_METAL_LOG_WARN("%s: too many elements for metal ctx->concur_list!\n", __func__);
|
|
603
649
|
}
|
|
604
650
|
}
|
|
605
651
|
|
|
@@ -653,7 +699,7 @@ void ggml_metal_graph_compute(
|
|
|
653
699
|
continue;
|
|
654
700
|
}
|
|
655
701
|
|
|
656
|
-
//
|
|
702
|
+
//GGML_METAL_LOG_INFO("%s: encoding node %3d, op = %8s\n", __func__, i, ggml_op_name(gf->nodes[i]->op));
|
|
657
703
|
|
|
658
704
|
struct ggml_tensor * src0 = gf->nodes[i]->src[0];
|
|
659
705
|
struct ggml_tensor * src1 = gf->nodes[i]->src[1];
|
|
@@ -697,17 +743,17 @@ void ggml_metal_graph_compute(
|
|
|
697
743
|
id<MTLBuffer> id_src1 = src1 ? ggml_metal_get_buffer(ctx, src1, &offs_src1) : nil;
|
|
698
744
|
id<MTLBuffer> id_dst = dst ? ggml_metal_get_buffer(ctx, dst, &offs_dst) : nil;
|
|
699
745
|
|
|
700
|
-
//
|
|
746
|
+
//GGML_METAL_LOG_INFO("%s: op - %s\n", __func__, ggml_op_name(dst->op));
|
|
701
747
|
//if (src0) {
|
|
702
|
-
//
|
|
748
|
+
// GGML_METAL_LOG_INFO("%s: src0 - %4s [%5lld, %5lld, %5lld], %d, %s\n", __func__, ggml_type_name(src0t), ne00, ne01, ne02,
|
|
703
749
|
// ggml_is_contiguous(src0), src0->name);
|
|
704
750
|
//}
|
|
705
751
|
//if (src1) {
|
|
706
|
-
//
|
|
752
|
+
// GGML_METAL_LOG_INFO("%s: src1 - %4s [%5lld, %5lld, %5lld], %d, %s\n", __func__, ggml_type_name(src1t), ne10, ne11, ne12,
|
|
707
753
|
// ggml_is_contiguous(src1), src1->name);
|
|
708
754
|
//}
|
|
709
755
|
//if (dst) {
|
|
710
|
-
//
|
|
756
|
+
// GGML_METAL_LOG_INFO("%s: dst - %4s [%5lld, %5lld, %5lld], 1, %s\n", __func__, ggml_type_name(dstt), ne0, ne1, ne2,
|
|
711
757
|
// dst->name);
|
|
712
758
|
//}
|
|
713
759
|
|
|
@@ -723,29 +769,66 @@ void ggml_metal_graph_compute(
|
|
|
723
769
|
case GGML_OP_ADD:
|
|
724
770
|
{
|
|
725
771
|
GGML_ASSERT(ggml_is_contiguous(src0));
|
|
772
|
+
GGML_ASSERT(ggml_is_contiguous(src1));
|
|
726
773
|
|
|
727
|
-
|
|
728
|
-
GGML_ASSERT(ne00 % 4 == 0);
|
|
729
|
-
const int64_t nb = ne00/4;
|
|
774
|
+
bool bcast_row = false;
|
|
730
775
|
|
|
731
|
-
|
|
776
|
+
int64_t nb = ne00;
|
|
777
|
+
|
|
778
|
+
if (ggml_nelements(src1) == ne10 && ne00 % 4 == 0) {
|
|
732
779
|
// src1 is a row
|
|
780
|
+
GGML_ASSERT(ne11 == 1);
|
|
781
|
+
|
|
782
|
+
nb = ne00 / 4;
|
|
733
783
|
[encoder setComputePipelineState:ctx->pipeline_add_row];
|
|
784
|
+
|
|
785
|
+
bcast_row = true;
|
|
734
786
|
} else {
|
|
735
787
|
[encoder setComputePipelineState:ctx->pipeline_add];
|
|
736
788
|
}
|
|
737
789
|
[encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
|
|
738
790
|
[encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];
|
|
739
791
|
[encoder setBuffer:id_dst offset:offs_dst atIndex:2];
|
|
740
|
-
[encoder setBytes:&
|
|
741
|
-
|
|
742
|
-
|
|
792
|
+
[encoder setBytes:&ne00 length:sizeof(ne00) atIndex:3];
|
|
793
|
+
[encoder setBytes:&ne01 length:sizeof(ne01) atIndex:4];
|
|
794
|
+
[encoder setBytes:&ne02 length:sizeof(ne02) atIndex:5];
|
|
795
|
+
[encoder setBytes:&ne03 length:sizeof(ne03) atIndex:6];
|
|
796
|
+
[encoder setBytes:&nb00 length:sizeof(nb00) atIndex:7];
|
|
797
|
+
[encoder setBytes:&nb01 length:sizeof(nb01) atIndex:8];
|
|
798
|
+
[encoder setBytes:&nb02 length:sizeof(nb02) atIndex:9];
|
|
799
|
+
[encoder setBytes:&nb03 length:sizeof(nb03) atIndex:10];
|
|
800
|
+
[encoder setBytes:&ne10 length:sizeof(ne10) atIndex:11];
|
|
801
|
+
[encoder setBytes:&ne11 length:sizeof(ne11) atIndex:12];
|
|
802
|
+
[encoder setBytes:&ne12 length:sizeof(ne12) atIndex:13];
|
|
803
|
+
[encoder setBytes:&ne13 length:sizeof(ne13) atIndex:14];
|
|
804
|
+
[encoder setBytes:&nb10 length:sizeof(nb10) atIndex:15];
|
|
805
|
+
[encoder setBytes:&nb11 length:sizeof(nb11) atIndex:16];
|
|
806
|
+
[encoder setBytes:&nb12 length:sizeof(nb12) atIndex:17];
|
|
807
|
+
[encoder setBytes:&nb13 length:sizeof(nb13) atIndex:18];
|
|
808
|
+
[encoder setBytes:&ne0 length:sizeof(ne0) atIndex:19];
|
|
809
|
+
[encoder setBytes:&ne1 length:sizeof(ne1) atIndex:20];
|
|
810
|
+
[encoder setBytes:&ne2 length:sizeof(ne2) atIndex:21];
|
|
811
|
+
[encoder setBytes:&ne3 length:sizeof(ne3) atIndex:22];
|
|
812
|
+
[encoder setBytes:&nb0 length:sizeof(nb0) atIndex:23];
|
|
813
|
+
[encoder setBytes:&nb1 length:sizeof(nb1) atIndex:24];
|
|
814
|
+
[encoder setBytes:&nb2 length:sizeof(nb2) atIndex:25];
|
|
815
|
+
[encoder setBytes:&nb3 length:sizeof(nb3) atIndex:26];
|
|
816
|
+
[encoder setBytes:&nb length:sizeof(nb) atIndex:27];
|
|
817
|
+
|
|
818
|
+
if (bcast_row) {
|
|
819
|
+
const int64_t n = ggml_nelements(dst)/4;
|
|
820
|
+
|
|
821
|
+
[encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
|
|
822
|
+
} else {
|
|
823
|
+
const int nth = MIN(1024, ne0);
|
|
743
824
|
|
|
744
|
-
|
|
825
|
+
[encoder dispatchThreadgroups:MTLSizeMake(ne01, ne02, ne03) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];
|
|
826
|
+
}
|
|
745
827
|
} break;
|
|
746
828
|
case GGML_OP_MUL:
|
|
747
829
|
{
|
|
748
830
|
GGML_ASSERT(ggml_is_contiguous(src0));
|
|
831
|
+
GGML_ASSERT(ggml_is_contiguous(src1));
|
|
749
832
|
|
|
750
833
|
// utilize float4
|
|
751
834
|
GGML_ASSERT(ne00 % 4 == 0);
|
|
@@ -753,6 +836,7 @@ void ggml_metal_graph_compute(
|
|
|
753
836
|
|
|
754
837
|
if (ggml_nelements(src1) == ne10) {
|
|
755
838
|
// src1 is a row
|
|
839
|
+
GGML_ASSERT(ne11 == 1);
|
|
756
840
|
[encoder setComputePipelineState:ctx->pipeline_mul_row];
|
|
757
841
|
} else {
|
|
758
842
|
[encoder setComputePipelineState:ctx->pipeline_mul];
|
|
@@ -768,6 +852,8 @@ void ggml_metal_graph_compute(
|
|
|
768
852
|
} break;
|
|
769
853
|
case GGML_OP_SCALE:
|
|
770
854
|
{
|
|
855
|
+
GGML_ASSERT(ggml_is_contiguous(src0));
|
|
856
|
+
|
|
771
857
|
const float scale = *(const float *) src1->data;
|
|
772
858
|
|
|
773
859
|
[encoder setComputePipelineState:ctx->pipeline_scale];
|
|
@@ -813,13 +899,13 @@ void ggml_metal_graph_compute(
|
|
|
813
899
|
} break;
|
|
814
900
|
default:
|
|
815
901
|
{
|
|
816
|
-
|
|
902
|
+
GGML_METAL_LOG_WARN("%s: node %3d, op = %8s not implemented\n", __func__, i, ggml_op_name(dst->op));
|
|
817
903
|
GGML_ASSERT(false);
|
|
818
904
|
}
|
|
819
905
|
} break;
|
|
820
906
|
case GGML_OP_SOFT_MAX:
|
|
821
907
|
{
|
|
822
|
-
const int nth = 32;
|
|
908
|
+
const int nth = MIN(32, ne00);
|
|
823
909
|
|
|
824
910
|
if (ne00%4 == 0) {
|
|
825
911
|
[encoder setComputePipelineState:ctx->pipeline_soft_max_4];
|
|
@@ -867,13 +953,14 @@ void ggml_metal_graph_compute(
|
|
|
867
953
|
|
|
868
954
|
// for now the matrix-matrix multiplication kernel only works on A14+/M1+ SoCs
|
|
869
955
|
// AMD GPU and older A-chips will reuse matrix-vector multiplication kernel
|
|
870
|
-
if (
|
|
871
|
-
|
|
956
|
+
if (!ggml_is_transposed(src0) &&
|
|
957
|
+
!ggml_is_transposed(src1) &&
|
|
872
958
|
src1t == GGML_TYPE_F32 &&
|
|
873
959
|
[ctx->device supportsFamily:MTLGPUFamilyApple7] &&
|
|
874
960
|
ne00%32 == 0 &&
|
|
875
|
-
ne11 >
|
|
961
|
+
ne11 > 2) {
|
|
876
962
|
switch (src0->type) {
|
|
963
|
+
case GGML_TYPE_F32: [encoder setComputePipelineState:ctx->pipeline_mul_mm_f32_f32]; break;
|
|
877
964
|
case GGML_TYPE_F16: [encoder setComputePipelineState:ctx->pipeline_mul_mm_f16_f32]; break;
|
|
878
965
|
case GGML_TYPE_Q4_0: [encoder setComputePipelineState:ctx->pipeline_mul_mm_q4_0_f32]; break;
|
|
879
966
|
case GGML_TYPE_Q4_1: [encoder setComputePipelineState:ctx->pipeline_mul_mm_q4_1_f32]; break;
|
|
@@ -893,9 +980,12 @@ void ggml_metal_graph_compute(
|
|
|
893
980
|
[encoder setBytes:&nb01 length:sizeof(nb01) atIndex:5];
|
|
894
981
|
[encoder setBytes:&nb02 length:sizeof(nb02) atIndex:6];
|
|
895
982
|
[encoder setBytes:&ne12 length:sizeof(ne12) atIndex:7];
|
|
896
|
-
[encoder setBytes:&
|
|
897
|
-
[encoder setBytes:&
|
|
898
|
-
[encoder setBytes:&
|
|
983
|
+
[encoder setBytes:&nb10 length:sizeof(nb10) atIndex:8];
|
|
984
|
+
[encoder setBytes:&nb11 length:sizeof(nb11) atIndex:9];
|
|
985
|
+
[encoder setBytes:&nb12 length:sizeof(nb12) atIndex:10];
|
|
986
|
+
[encoder setBytes:&ne0 length:sizeof(ne0) atIndex:11];
|
|
987
|
+
[encoder setBytes:&ne1 length:sizeof(ne1) atIndex:12];
|
|
988
|
+
[encoder setBytes:&gqa length:sizeof(gqa) atIndex:13];
|
|
899
989
|
[encoder setThreadgroupMemoryLength:8192 atIndex:0];
|
|
900
990
|
[encoder dispatchThreadgroups:MTLSizeMake( (ne11+31)/32, (ne01+63) / 64, ne12) threadsPerThreadgroup:MTLSizeMake(128, 1, 1)];
|
|
901
991
|
} else {
|
|
@@ -905,6 +995,11 @@ void ggml_metal_graph_compute(
|
|
|
905
995
|
|
|
906
996
|
// use custom matrix x vector kernel
|
|
907
997
|
switch (src0t) {
|
|
998
|
+
case GGML_TYPE_F32:
|
|
999
|
+
{
|
|
1000
|
+
[encoder setComputePipelineState:ctx->pipeline_mul_mat_f32_f32];
|
|
1001
|
+
nrows = 4;
|
|
1002
|
+
} break;
|
|
908
1003
|
case GGML_TYPE_F16:
|
|
909
1004
|
{
|
|
910
1005
|
nth0 = 32;
|
|
@@ -993,7 +1088,7 @@ void ggml_metal_graph_compute(
|
|
|
993
1088
|
} break;
|
|
994
1089
|
default:
|
|
995
1090
|
{
|
|
996
|
-
|
|
1091
|
+
GGML_METAL_LOG_ERROR("Asserting on type %d\n", (int)src0t);
|
|
997
1092
|
GGML_ASSERT(false && "not implemented");
|
|
998
1093
|
}
|
|
999
1094
|
};
|
|
@@ -1045,6 +1140,7 @@ void ggml_metal_graph_compute(
|
|
|
1045
1140
|
case GGML_OP_GET_ROWS:
|
|
1046
1141
|
{
|
|
1047
1142
|
switch (src0->type) {
|
|
1143
|
+
case GGML_TYPE_F32: [encoder setComputePipelineState:ctx->pipeline_get_rows_f32]; break;
|
|
1048
1144
|
case GGML_TYPE_F16: [encoder setComputePipelineState:ctx->pipeline_get_rows_f16]; break;
|
|
1049
1145
|
case GGML_TYPE_Q4_0: [encoder setComputePipelineState:ctx->pipeline_get_rows_q4_0]; break;
|
|
1050
1146
|
case GGML_TYPE_Q4_1: [encoder setComputePipelineState:ctx->pipeline_get_rows_q4_1]; break;
|
|
@@ -1060,9 +1156,9 @@ void ggml_metal_graph_compute(
|
|
|
1060
1156
|
[encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
|
|
1061
1157
|
[encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];
|
|
1062
1158
|
[encoder setBuffer:id_dst offset:offs_dst atIndex:2];
|
|
1063
|
-
[encoder setBytes:&
|
|
1064
|
-
[encoder setBytes:&
|
|
1065
|
-
[encoder setBytes:&
|
|
1159
|
+
[encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:3];
|
|
1160
|
+
[encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:4];
|
|
1161
|
+
[encoder setBytes:&nb1 length:sizeof(uint64_t) atIndex:5];
|
|
1066
1162
|
|
|
1067
1163
|
const int64_t n = ggml_nelements(src1);
|
|
1068
1164
|
|
|
@@ -1073,7 +1169,7 @@ void ggml_metal_graph_compute(
|
|
|
1073
1169
|
float eps;
|
|
1074
1170
|
memcpy(&eps, dst->op_params, sizeof(float));
|
|
1075
1171
|
|
|
1076
|
-
const int nth = 512;
|
|
1172
|
+
const int nth = MIN(512, ne00);
|
|
1077
1173
|
|
|
1078
1174
|
[encoder setComputePipelineState:ctx->pipeline_rms_norm];
|
|
1079
1175
|
[encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
|
|
@@ -1092,7 +1188,7 @@ void ggml_metal_graph_compute(
|
|
|
1092
1188
|
float eps;
|
|
1093
1189
|
memcpy(&eps, dst->op_params, sizeof(float));
|
|
1094
1190
|
|
|
1095
|
-
const int nth = 256;
|
|
1191
|
+
const int nth = MIN(256, ne00);
|
|
1096
1192
|
|
|
1097
1193
|
[encoder setComputePipelineState:ctx->pipeline_norm];
|
|
1098
1194
|
[encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
|
|
@@ -1110,6 +1206,8 @@ void ggml_metal_graph_compute(
|
|
|
1110
1206
|
{
|
|
1111
1207
|
GGML_ASSERT((src0t == GGML_TYPE_F32));
|
|
1112
1208
|
|
|
1209
|
+
const int nth = MIN(1024, ne00);
|
|
1210
|
+
|
|
1113
1211
|
const int n_past = ((int32_t *) dst->op_params)[0]; UNUSED(n_past);
|
|
1114
1212
|
const int n_head = ((int32_t *) dst->op_params)[1];
|
|
1115
1213
|
float max_bias;
|
|
@@ -1143,12 +1241,14 @@ void ggml_metal_graph_compute(
|
|
|
1143
1241
|
[encoder setBytes:&nb3 length:sizeof(uint64_t) atIndex:17];
|
|
1144
1242
|
[encoder setBytes:&m0 length:sizeof( float) atIndex:18];
|
|
1145
1243
|
|
|
1146
|
-
const int nth = 32;
|
|
1147
|
-
|
|
1148
1244
|
[encoder dispatchThreadgroups:MTLSizeMake(ne01, ne02, ne03) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];
|
|
1149
1245
|
} break;
|
|
1150
1246
|
case GGML_OP_ROPE:
|
|
1151
1247
|
{
|
|
1248
|
+
GGML_ASSERT(ne10 == ne02);
|
|
1249
|
+
|
|
1250
|
+
const int nth = MIN(1024, ne00);
|
|
1251
|
+
|
|
1152
1252
|
const int n_past = ((int32_t *) dst->op_params)[0];
|
|
1153
1253
|
const int n_dims = ((int32_t *) dst->op_params)[1];
|
|
1154
1254
|
const int mode = ((int32_t *) dst->op_params)[2];
|
|
@@ -1158,38 +1258,44 @@ void ggml_metal_graph_compute(
|
|
|
1158
1258
|
memcpy(&freq_base, (int32_t *) dst->op_params + 4, sizeof(float));
|
|
1159
1259
|
memcpy(&freq_scale, (int32_t *) dst->op_params + 5, sizeof(float));
|
|
1160
1260
|
|
|
1161
|
-
|
|
1261
|
+
switch (src0->type) {
|
|
1262
|
+
case GGML_TYPE_F32: [encoder setComputePipelineState:ctx->pipeline_rope_f32]; break;
|
|
1263
|
+
case GGML_TYPE_F16: [encoder setComputePipelineState:ctx->pipeline_rope_f16]; break;
|
|
1264
|
+
default: GGML_ASSERT(false);
|
|
1265
|
+
};
|
|
1266
|
+
|
|
1162
1267
|
[encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
|
|
1163
|
-
[encoder setBuffer:
|
|
1164
|
-
[encoder
|
|
1165
|
-
[encoder setBytes:&
|
|
1166
|
-
[encoder setBytes:&
|
|
1167
|
-
[encoder setBytes:&
|
|
1168
|
-
[encoder setBytes:&
|
|
1169
|
-
[encoder setBytes:&
|
|
1170
|
-
[encoder setBytes:&
|
|
1171
|
-
[encoder setBytes:&
|
|
1172
|
-
[encoder setBytes:&
|
|
1173
|
-
[encoder setBytes:&
|
|
1174
|
-
[encoder setBytes:&
|
|
1175
|
-
[encoder setBytes:&
|
|
1176
|
-
[encoder setBytes:&
|
|
1177
|
-
[encoder setBytes:&
|
|
1178
|
-
[encoder setBytes:&
|
|
1179
|
-
[encoder setBytes:&
|
|
1180
|
-
[encoder setBytes:&
|
|
1181
|
-
[encoder setBytes:&
|
|
1182
|
-
[encoder setBytes:&
|
|
1183
|
-
[encoder setBytes:&
|
|
1184
|
-
[encoder setBytes:&
|
|
1268
|
+
[encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];
|
|
1269
|
+
[encoder setBuffer:id_dst offset:offs_dst atIndex:2];
|
|
1270
|
+
[encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:3];
|
|
1271
|
+
[encoder setBytes:&ne01 length:sizeof( int64_t) atIndex:4];
|
|
1272
|
+
[encoder setBytes:&ne02 length:sizeof( int64_t) atIndex:5];
|
|
1273
|
+
[encoder setBytes:&ne03 length:sizeof( int64_t) atIndex:6];
|
|
1274
|
+
[encoder setBytes:&nb00 length:sizeof(uint64_t) atIndex:7];
|
|
1275
|
+
[encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:8];
|
|
1276
|
+
[encoder setBytes:&nb02 length:sizeof(uint64_t) atIndex:9];
|
|
1277
|
+
[encoder setBytes:&nb03 length:sizeof(uint64_t) atIndex:10];
|
|
1278
|
+
[encoder setBytes:&ne0 length:sizeof( int64_t) atIndex:11];
|
|
1279
|
+
[encoder setBytes:&ne1 length:sizeof( int64_t) atIndex:12];
|
|
1280
|
+
[encoder setBytes:&ne2 length:sizeof( int64_t) atIndex:13];
|
|
1281
|
+
[encoder setBytes:&ne3 length:sizeof( int64_t) atIndex:14];
|
|
1282
|
+
[encoder setBytes:&nb0 length:sizeof(uint64_t) atIndex:15];
|
|
1283
|
+
[encoder setBytes:&nb1 length:sizeof(uint64_t) atIndex:16];
|
|
1284
|
+
[encoder setBytes:&nb2 length:sizeof(uint64_t) atIndex:17];
|
|
1285
|
+
[encoder setBytes:&nb3 length:sizeof(uint64_t) atIndex:18];
|
|
1286
|
+
[encoder setBytes:&n_past length:sizeof( int) atIndex:19];
|
|
1287
|
+
[encoder setBytes:&n_dims length:sizeof( int) atIndex:20];
|
|
1288
|
+
[encoder setBytes:&mode length:sizeof( int) atIndex:21];
|
|
1289
|
+
[encoder setBytes:&freq_base length:sizeof(float) atIndex:22];
|
|
1290
|
+
[encoder setBytes:&freq_scale length:sizeof(float) atIndex:23];
|
|
1185
1291
|
|
|
1186
|
-
[encoder dispatchThreadgroups:MTLSizeMake(ne01, ne02, ne03) threadsPerThreadgroup:MTLSizeMake(
|
|
1292
|
+
[encoder dispatchThreadgroups:MTLSizeMake(ne01, ne02, ne03) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];
|
|
1187
1293
|
} break;
|
|
1188
1294
|
case GGML_OP_DUP:
|
|
1189
1295
|
case GGML_OP_CPY:
|
|
1190
1296
|
case GGML_OP_CONT:
|
|
1191
1297
|
{
|
|
1192
|
-
const int nth =
|
|
1298
|
+
const int nth = MIN(1024, ne00);
|
|
1193
1299
|
|
|
1194
1300
|
switch (src0t) {
|
|
1195
1301
|
case GGML_TYPE_F32:
|
|
@@ -1234,7 +1340,7 @@ void ggml_metal_graph_compute(
|
|
|
1234
1340
|
} break;
|
|
1235
1341
|
default:
|
|
1236
1342
|
{
|
|
1237
|
-
|
|
1343
|
+
GGML_METAL_LOG_ERROR("%s: error: node %3d, op = %8s not implemented\n", __func__, i, ggml_op_name(dst->op));
|
|
1238
1344
|
GGML_ASSERT(false);
|
|
1239
1345
|
}
|
|
1240
1346
|
}
|
|
@@ -1259,7 +1365,7 @@ void ggml_metal_graph_compute(
|
|
|
1259
1365
|
|
|
1260
1366
|
MTLCommandBufferStatus status = (MTLCommandBufferStatus) [ctx->command_buffers[i] status];
|
|
1261
1367
|
if (status != MTLCommandBufferStatusCompleted) {
|
|
1262
|
-
|
|
1368
|
+
GGML_METAL_LOG_INFO("%s: command buffer %d failed with status %lu\n", __func__, i, status);
|
|
1263
1369
|
GGML_ASSERT(false);
|
|
1264
1370
|
}
|
|
1265
1371
|
}
|