llama_cpp 0.1.2 → 0.1.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1028 @@
1
+ #include "ggml-opencl.h"
2
+
3
+ #include <array>
4
+ #include <atomic>
5
+ #include <sstream>
6
+
7
+ #define CL_TARGET_OPENCL_VERSION 110
8
+ #include <clblast.h>
9
+
10
+ #include <stdlib.h>
11
+ #include <stdio.h>
12
+ #include <string.h>
13
+
14
+ #include "ggml.h"
15
+
16
+ #define CL_DMMV_BLOCK_SIZE 32;
17
+
18
+ #define MULTILINE_QUOTE(...) #__VA_ARGS__
19
+ static std::string program_source = MULTILINE_QUOTE(
20
+
21
+ typedef char int8_t;
22
+ typedef uchar uint8_t;
23
+ typedef int int32_t;
24
+ typedef uint uint32_t;
25
+
26
+ struct __attribute__ ((packed)) block_q4_0
27
+ {
28
+ half d;
29
+ uint8_t qs[QK4_0 / 2];
30
+ };
31
+
32
+ struct __attribute__ ((packed)) block_q4_1
33
+ {
34
+ half d;
35
+ half m;
36
+ uint8_t qs[QK4_1 / 2];
37
+ };
38
+
39
+ struct __attribute__ ((packed)) block_q5_0
40
+ {
41
+ half d;
42
+ uint32_t qh;
43
+ uint8_t qs[QK5_0 / 2];
44
+ };
45
+
46
+ struct __attribute__ ((packed)) block_q5_1
47
+ {
48
+ half d;
49
+ half m;
50
+ uint32_t qh;
51
+ uint8_t qs[QK5_1 / 2];
52
+ };
53
+
54
+ struct __attribute__ ((packed)) block_q8_0
55
+ {
56
+ half d;
57
+ int8_t qs[QK8_0];
58
+ };
59
+
60
+
61
+ __kernel void convert_fp16_to_fp32(__global half* x, __global float* y) {
62
+ const uint i = get_global_id(0);
63
+
64
+ y[i] = vload_half(0, &x[i]);
65
+ }
66
+
67
+ void dequantize_q4_0(__global const struct block_q4_0* x, const int ib, const int iqs, float* v0, float* v1) {
68
+ const float d = vload_half(0, &x[ib].d);
69
+
70
+ const uint8_t vui = x[ib].qs[iqs];
71
+
72
+ const int8_t vi0 = vui & 0xF;
73
+ const int8_t vi1 = vui >> 4;
74
+
75
+ *v0 = (vi0 - 8)*d;
76
+ *v1 = (vi1 - 8)*d;
77
+ }
78
+ void dequantize_q4_1(__global const struct block_q4_1* x, const int ib, const int iqs, float* v0, float* v1) {
79
+ const float d = vload_half(0, &x[ib].d);
80
+ const float m = vload_half(0, &x[ib].m);
81
+
82
+ const uint8_t vui = x[ib].qs[iqs];
83
+
84
+ const int8_t vi0 = vui & 0xF;
85
+ const int8_t vi1 = vui >> 4;
86
+
87
+ *v0 = vi0*d + m;
88
+ *v1 = vi1*d + m;
89
+ }
90
+ void dequantize_q5_0(__global const struct block_q5_0* x, const int ib, const int iqs, float* v0, float* v1) {
91
+ const float d = vload_half(0, &x[ib].d);
92
+
93
+ uint32_t qh = x[ib].qh;
94
+
95
+ const uint8_t xh_0 = ((qh >> (iqs + 0)) << 4) & 0x10;
96
+ const uint8_t xh_1 = ((qh >> (iqs + 12)) ) & 0x10;
97
+
98
+ const int32_t x0 = ((x[ib].qs[iqs] & 0xf) | xh_0) - 16;
99
+ const int32_t x1 = ((x[ib].qs[iqs] >> 4) | xh_1) - 16;
100
+
101
+ *v0 = x0*d;
102
+ *v1 = x1*d;
103
+ }
104
+ void dequantize_q5_1(__global const struct block_q5_1* x, const int ib, const int iqs, float* v0, float* v1) {
105
+ const float d = vload_half(0, &x[ib].d);
106
+ const float m = vload_half(0, &x[ib].m);
107
+
108
+ uint32_t qh = x[ib].qh;
109
+
110
+ const uint8_t xh_0 = ((qh >> (iqs + 0)) << 4) & 0x10;
111
+ const uint8_t xh_1 = ((qh >> (iqs + 12)) ) & 0x10;
112
+
113
+ const int32_t x0 = ((x[ib].qs[iqs] & 0xf) | xh_0);
114
+ const int32_t x1 = ((x[ib].qs[iqs] >> 4) | xh_1);
115
+
116
+ *v0 = x0*d + m;
117
+ *v1 = x1*d + m;
118
+ }
119
+ void dequantize_q8_0(__global const struct block_q8_0* x, const int ib, const int iqs, float* v0, float* v1) {
120
+ const float d = vload_half(0, &x[ib].d);
121
+
122
+ const int8_t vi0 = x[ib].qs[iqs + 0];
123
+ const int8_t vi1 = x[ib].qs[iqs + 1];
124
+
125
+ *v0 = vi0*d;
126
+ *v1 = vi1*d;
127
+ }
128
+ void convert_f16(__global half* x, const int ib, const int iqs, float* v0, float* v1){
129
+ *v0 = vload_half(0, &x[ib + 0]);
130
+ *v1 = vload_half(0, &x[ib + 1]);
131
+ }
132
+ );
133
+
134
+ std::string dequant_template = MULTILINE_QUOTE(
135
+ __kernel void KERNEL_NAME(__global X_TYPE* x, __global float* y) {
136
+ const int i = get_group_id(0)*get_local_size(0) + get_local_id(0)*2;
137
+
138
+ if (i >= get_global_size(0)) {
139
+ return;
140
+ }
141
+
142
+ const uint qk = QUANT_K;
143
+ const uint qr = QUANT_R;
144
+
145
+ const int ib = i/qk; // block index
146
+ const int iqs = (i%qk)/qr; // quant index
147
+ const int iybs = i - i%qk; // y block start index
148
+ const int y_offset = qr == 1 ? 1 : qk/2;
149
+
150
+ // dequantize
151
+ float v0, v1;
152
+ DEQUANT_FUNC(x, ib, iqs, &v0, &v1);
153
+ y[iybs + iqs + 0] = v0;
154
+ y[iybs + iqs + y_offset] = v1;
155
+ }
156
+ );
157
+
158
+ std::string dequant_mul_mat_vec_template = MULTILINE_QUOTE(
159
+ __kernel void KERNEL_NAME(__global X_TYPE* x, __local float* tmp, __global float* y, __global float* dst, const int ncols) {
160
+ const int block_size = get_local_size(0);
161
+ const int row = get_global_id(0) / block_size;
162
+ const int tid = get_local_id(0);
163
+
164
+ const uint qk = QUANT_K;
165
+ const uint qr = QUANT_R;
166
+
167
+ const int y_offset = qr == 1 ? 1 : qk/2;
168
+
169
+ tmp[tid] = 0;
170
+
171
+ for (int i = 0; i < ncols/block_size; i += 2) {
172
+ const int col = i*block_size + 2*tid;
173
+ const int ib = (row*ncols + col)/qk; // block index
174
+ const int iqs = (col%qk)/qr; // quant index
175
+ const int iybs = col - col%qk; // y block start index
176
+
177
+ // dequantize
178
+ float v0, v1;
179
+ DEQUANT_FUNC(x, ib, iqs, &v0, &v1);
180
+
181
+ // matrix multiplication
182
+ tmp[tid] += v0 * y[iybs + iqs + 0];
183
+ tmp[tid] += v1 * y[iybs + iqs + y_offset];
184
+ }
185
+
186
+ // sum up partial sums and write back result
187
+ barrier(CLK_LOCAL_MEM_FENCE);
188
+ for (int s=block_size/2; s>0; s>>=1) {
189
+ if (tid < s) {
190
+ tmp[tid] += tmp[tid + s];
191
+ }
192
+ barrier(CLK_LOCAL_MEM_FENCE);
193
+ }
194
+ if (tid == 0) {
195
+ dst[row] = tmp[0];
196
+ }
197
+ }
198
+ );
199
+
200
+ #define CL_CHECK(err) \
201
+ do { \
202
+ cl_int err_ = (err); \
203
+ if (err_ != CL_SUCCESS) { \
204
+ fprintf(stderr, "ggml_opencl: %s error %d at %s:%d\n", \
205
+ #err, err_, __FILE__, __LINE__); \
206
+ exit(1); \
207
+ } \
208
+ } while (0)
209
+
210
+ #define CLBLAST_CHECK(err) \
211
+ do { \
212
+ CLBlastStatusCode err_ = (err); \
213
+ if (err_ != CLBlastSuccess) { \
214
+ fprintf(stderr, "ggml_opencl: %s error %d at %s:%d\n", \
215
+ #err, err_, __FILE__, __LINE__); \
216
+ exit(1); \
217
+ } \
218
+ } while (0)
219
+
220
+ std::array<std::string, 5> dequant_str_keys = {
221
+ "KERNEL_NAME", "X_TYPE", "QUANT_K", "QUANT_R", "DEQUANT_FUNC"
222
+ };
223
+
224
+ std::array<std::string, 30> dequant_str_values = {
225
+ "dequantize_row_q4_0", "struct block_q4_0", "QK4_0", "QR4_0", "dequantize_q4_0",
226
+ "dequantize_row_q4_1", "struct block_q4_1", "QK4_1", "QR4_1", "dequantize_q4_1",
227
+ "dequantize_row_q5_0", "struct block_q5_0", "QK5_0", "QR5_0", "dequantize_q5_0",
228
+ "dequantize_row_q5_1", "struct block_q5_1", "QK5_1", "QR5_1", "dequantize_q5_1",
229
+ "dequantize_row_q8_0", "struct block_q8_0", "QK8_0", "QR8_0", "dequantize_q8_0",
230
+ "convert_row_f16", "half", "1", "1", "convert_f16"
231
+ };
232
+
233
+ std::array<std::string, 30> dequant_mul_mat_vec_str_values = {
234
+ "dequantize_mul_mat_vec_q4_0", "struct block_q4_0", "QK4_0", "QR4_0", "dequantize_q4_0",
235
+ "dequantize_mul_mat_vec_q4_1", "struct block_q4_1", "QK4_1", "QR4_1", "dequantize_q4_1",
236
+ "dequantize_mul_mat_vec_q5_0", "struct block_q5_0", "QK5_0", "QR5_0", "dequantize_q5_0",
237
+ "dequantize_mul_mat_vec_q5_1", "struct block_q5_1", "QK5_1", "QR5_1", "dequantize_q5_1",
238
+ "dequantize_mul_mat_vec_q8_0", "struct block_q8_0", "QK8_0", "QR8_0", "dequantize_q8_0",
239
+ "convert_mul_mat_vec_f16", "half", "1", "1", "convert_f16"
240
+ };
241
+
242
+ std::string& replace(std::string& s, const std::string& from, const std::string& to) {
243
+ size_t pos = 0;
244
+ while ((pos = s.find(from, pos)) != std::string::npos) {
245
+ s.replace(pos, from.length(), to);
246
+ pos += to.length();
247
+ }
248
+ return s;
249
+ }
250
+
251
+ std::string generate_kernels() {
252
+ std::stringstream src;
253
+ src << program_source << '\n';
254
+ for (size_t i = 0; i < dequant_str_values.size(); i += dequant_str_keys.size()) {
255
+ std::string dequant_kernel = dequant_template;
256
+ std::string dmmv_kernel = dequant_mul_mat_vec_template;
257
+ for (size_t j = 0; j < dequant_str_keys.size(); j++) {
258
+ replace(dequant_kernel, dequant_str_keys[j], dequant_str_values[i + j]);
259
+ replace(dmmv_kernel, dequant_str_keys[j], dequant_mul_mat_vec_str_values[i + j]);
260
+ }
261
+ src << dequant_kernel << '\n';
262
+ src << dmmv_kernel << '\n';
263
+ }
264
+ return src.str();
265
+ }
266
+
267
+ static cl_platform_id platform;
268
+ static cl_device_id device;
269
+ static cl_context context;
270
+ static cl_command_queue queue;
271
+ static cl_program program;
272
+ static cl_kernel convert_row_f16_cl;
273
+ static cl_kernel dequantize_row_q4_0_cl, dequantize_row_q4_1_cl, dequantize_row_q5_0_cl, dequantize_row_q5_1_cl, dequantize_row_q8_0_cl;
274
+ static cl_kernel dequantize_mul_mat_vec_q4_0_cl, dequantize_mul_mat_vec_q4_1_cl, dequantize_mul_mat_vec_q5_0_cl, dequantize_mul_mat_vec_q5_1_cl, dequantize_mul_mat_vec_q8_0_cl, convert_mul_mat_vec_f16_cl;
275
+ static bool fp16_support;
276
+
277
+ static cl_program build_program_from_source(cl_context ctx, cl_device_id dev, const char* program_buffer) {
278
+ cl_program p;
279
+ char *program_log;
280
+ size_t program_size;
281
+ size_t log_size;
282
+ int err;
283
+
284
+ program_size = strlen(program_buffer);
285
+
286
+ p = clCreateProgramWithSource(ctx, 1, (const char**)&program_buffer, &program_size, &err);
287
+ if(err < 0) {
288
+ fprintf(stderr, "OpenCL error creating program");
289
+ exit(1);
290
+ }
291
+
292
+ const char* compile_opts = "-cl-mad-enable -cl-unsafe-math-optimizations -cl-finite-math-only -cl-fast-relaxed-math "
293
+ "-DQK4_0=32 -DQR4_0=2 -DQK4_1=32 -DQR4_1=2 -DQK5_0=32 -DQR5_0=2 -DQK5_1=32 -DQR5_1=2 -DQK8_0=32 -DQR8_0=1";
294
+
295
+ err = clBuildProgram(p, 0, NULL, compile_opts, NULL, NULL);
296
+ if(err < 0) {
297
+
298
+ clGetProgramBuildInfo(p, dev, CL_PROGRAM_BUILD_LOG, 0, NULL, &log_size);
299
+ program_log = (char*) malloc(log_size + 1);
300
+ program_log[log_size] = '\0';
301
+ clGetProgramBuildInfo(p, dev, CL_PROGRAM_BUILD_LOG, log_size + 1, program_log, NULL);
302
+ fprintf(stderr, "ggml_opencl: kernel compile error:\n\n%s\n", program_log);
303
+ free(program_log);
304
+ exit(1);
305
+ }
306
+
307
+ return p;
308
+ }
309
+
310
+ void ggml_cl_init(void) {
311
+ cl_int err;
312
+
313
+ struct cl_device;
314
+ struct cl_platform {
315
+ cl_platform_id id;
316
+ unsigned number;
317
+ char name[128];
318
+ char vendor[128];
319
+ struct cl_device * devices;
320
+ unsigned n_devices;
321
+ struct cl_device * default_device;
322
+ };
323
+
324
+ struct cl_device {
325
+ struct cl_platform * platform;
326
+ cl_device_id id;
327
+ unsigned number;
328
+ cl_device_type type;
329
+ char name[128];
330
+ };
331
+
332
+ enum { NPLAT = 16, NDEV = 16 };
333
+
334
+ struct cl_platform platforms[NPLAT];
335
+ unsigned n_platforms = 0;
336
+ struct cl_device devices[NDEV];
337
+ unsigned n_devices = 0;
338
+ struct cl_device * default_device = NULL;
339
+
340
+ platform = NULL;
341
+ device = NULL;
342
+
343
+ cl_platform_id platform_ids[NPLAT];
344
+ CL_CHECK(clGetPlatformIDs(NPLAT, platform_ids, &n_platforms));
345
+
346
+ for (unsigned i = 0; i < n_platforms; i++) {
347
+ struct cl_platform * p = &platforms[i];
348
+ p->number = i;
349
+ p->id = platform_ids[i];
350
+ CL_CHECK(clGetPlatformInfo(p->id, CL_PLATFORM_NAME, sizeof(p->name), &p->name, NULL));
351
+ CL_CHECK(clGetPlatformInfo(p->id, CL_PLATFORM_VENDOR, sizeof(p->vendor), &p->vendor, NULL));
352
+
353
+ cl_device_id device_ids[NDEV];
354
+ cl_int clGetDeviceIDsError = clGetDeviceIDs(p->id, CL_DEVICE_TYPE_ALL, NDEV, device_ids, &p->n_devices);
355
+ if (clGetDeviceIDsError == CL_DEVICE_NOT_FOUND) {
356
+ p->n_devices = 0;
357
+ } else {
358
+ CL_CHECK(clGetDeviceIDsError);
359
+ }
360
+ p->devices = p->n_devices > 0 ? &devices[n_devices] : NULL;
361
+ p->default_device = NULL;
362
+
363
+ for (unsigned j = 0; j < p->n_devices; j++) {
364
+ struct cl_device * d = &devices[n_devices];
365
+ d->number = n_devices++;
366
+ d->id = device_ids[j];
367
+ d->platform = p;
368
+ CL_CHECK(clGetDeviceInfo(d->id, CL_DEVICE_NAME, sizeof(d->name), &d->name, NULL));
369
+ CL_CHECK(clGetDeviceInfo(d->id, CL_DEVICE_TYPE, sizeof(d->type), &d->type, NULL));
370
+
371
+ if (p->default_device == NULL && d->type == CL_DEVICE_TYPE_GPU) {
372
+ p->default_device = d;
373
+ }
374
+ }
375
+
376
+ if (default_device == NULL && p->default_device != NULL) {
377
+ default_device = p->default_device;
378
+ }
379
+ }
380
+
381
+ if (n_devices == 0) {
382
+ fprintf(stderr, "ggml_opencl: could find any OpenCL devices.\n");
383
+ exit(1);
384
+ }
385
+
386
+ char * user_platform_string = getenv("GGML_OPENCL_PLATFORM");
387
+ char * user_device_string = getenv("GGML_OPENCL_DEVICE");
388
+ int user_platform_number = -1;
389
+ int user_device_number = -1;
390
+
391
+ unsigned n;
392
+ if (user_platform_string != NULL && sscanf(user_platform_string, " %u", &n) == 1 && n < n_platforms) {
393
+ user_platform_number = (int)n;
394
+ }
395
+ if (user_device_string != NULL && sscanf(user_device_string, " %u", &n) == 1 && n < n_devices) {
396
+ user_device_number = (int)n;
397
+ }
398
+ if (user_platform_number != -1 && user_device_number != -1) {
399
+ cl_platform* platform = &platforms[user_platform_number];
400
+ if ((unsigned)user_device_number >= platform->n_devices) {
401
+ fprintf(stderr, "ggml_opencl: invalid device number %d\n", user_device_number);
402
+ exit(1);
403
+ }
404
+ default_device = &platform->devices[user_device_number];
405
+ } else {
406
+
407
+ struct cl_device * selected_devices = devices;
408
+ unsigned n_selected_devices = n_devices;
409
+
410
+ if (user_platform_number == -1 && user_platform_string != NULL && user_platform_string[0] != 0) {
411
+ for (unsigned i = 0; i < n_platforms; i++) {
412
+ struct cl_platform * p = &platforms[i];
413
+ if (strstr(p->name, user_platform_string) != NULL ||
414
+ strstr(p->vendor, user_platform_string) != NULL) {
415
+ user_platform_number = (int)i;
416
+ break;
417
+ }
418
+ }
419
+ if (user_platform_number == -1) {
420
+ fprintf(stderr, "ggml_opencl: no platform matching '%s' was found.\n", user_platform_string);
421
+ exit(1);
422
+ }
423
+ }
424
+ if (user_platform_number != -1) {
425
+ struct cl_platform * p = &platforms[user_platform_number];
426
+ selected_devices = p->devices;
427
+ n_selected_devices = p->n_devices;
428
+ default_device = p->default_device;
429
+ if (n_selected_devices == 0) {
430
+ fprintf(stderr, "ggml_opencl: selected platform '%s' does not have any devices.\n", p->name);
431
+ exit(1);
432
+ }
433
+ }
434
+
435
+ if (user_device_number == -1 && user_device_string != NULL && user_device_string[0] != 0) {
436
+ for (unsigned i = 0; i < n_selected_devices; i++) {
437
+ struct cl_device * d = &selected_devices[i];
438
+ if (strstr(d->name, user_device_string) != NULL) {
439
+ user_device_number = d->number;
440
+ break;
441
+ }
442
+ }
443
+ if (user_device_number == -1) {
444
+ fprintf(stderr, "ggml_opencl: no device matching '%s' was found.\n", user_device_string);
445
+ exit(1);
446
+ }
447
+ }
448
+ if (user_device_number != -1) {
449
+ selected_devices = &devices[user_device_number];
450
+ n_selected_devices = 1;
451
+ default_device = &selected_devices[0];
452
+ }
453
+
454
+ GGML_ASSERT(n_selected_devices > 0);
455
+
456
+ if (default_device == NULL) {
457
+ default_device = &selected_devices[0];
458
+ }
459
+ }
460
+
461
+ fprintf(stderr, "ggml_opencl: selecting platform: '%s'\n", default_device->platform->name);
462
+ fprintf(stderr, "ggml_opencl: selecting device: '%s'\n", default_device->name);
463
+ if (default_device->type != CL_DEVICE_TYPE_GPU) {
464
+ fprintf(stderr, "ggml_opencl: warning, not a GPU: '%s'.\n", default_device->name);
465
+ }
466
+
467
+ platform = default_device->platform->id;
468
+ device = default_device->id;
469
+
470
+ size_t ext_str_size;
471
+ clGetDeviceInfo(device, CL_DEVICE_EXTENSIONS, 0, NULL, &ext_str_size);
472
+ char *ext_buffer = (char *)alloca(ext_str_size + 1);
473
+ clGetDeviceInfo(device, CL_DEVICE_EXTENSIONS, ext_str_size, ext_buffer, NULL);
474
+ ext_buffer[ext_str_size] = '\0'; // ensure it is null terminated
475
+ // Check if ext_buffer contains cl_khr_fp16
476
+ fp16_support = strstr(ext_buffer, "cl_khr_fp16") != NULL;
477
+ fprintf(stderr, "ggml_opencl: device FP16 support: %s\n", fp16_support ? "true" : "false");
478
+
479
+ cl_context_properties properties[] = {
480
+ (intptr_t)CL_CONTEXT_PLATFORM, (intptr_t)platform, 0
481
+ };
482
+
483
+ CL_CHECK((context = clCreateContext(properties, 1, &device, NULL, NULL, &err), err));
484
+
485
+ CL_CHECK((queue = clCreateCommandQueue(context, device, CL_QUEUE_OUT_OF_ORDER_EXEC_MODE_ENABLE, &err),
486
+ (err != CL_INVALID_QUEUE_PROPERTIES && err != CL_INVALID_VALUE ? err :
487
+ (queue = clCreateCommandQueue(context, device, 0, &err), err)
488
+ )));
489
+
490
+ const std::string kernel_src = generate_kernels();
491
+
492
+ program = build_program_from_source(context, device, kernel_src.c_str());
493
+
494
+ // FP16 to FP32 kernel
495
+ CL_CHECK((convert_row_f16_cl = clCreateKernel(program, "convert_row_f16", &err), err));
496
+
497
+ // Dequantize kernels
498
+ CL_CHECK((dequantize_row_q4_0_cl = clCreateKernel(program, "dequantize_row_q4_0", &err), err));
499
+ CL_CHECK((dequantize_row_q4_1_cl = clCreateKernel(program, "dequantize_row_q4_1", &err), err));
500
+ CL_CHECK((dequantize_row_q5_0_cl = clCreateKernel(program, "dequantize_row_q5_0", &err), err));
501
+ CL_CHECK((dequantize_row_q5_1_cl = clCreateKernel(program, "dequantize_row_q5_1", &err), err));
502
+ CL_CHECK((dequantize_row_q8_0_cl = clCreateKernel(program, "dequantize_row_q8_0", &err), err));
503
+
504
+ // dequant mul mat kernel
505
+ CL_CHECK((dequantize_mul_mat_vec_q4_0_cl = clCreateKernel(program, "dequantize_mul_mat_vec_q4_0", &err), err));
506
+ CL_CHECK((dequantize_mul_mat_vec_q4_1_cl = clCreateKernel(program, "dequantize_mul_mat_vec_q4_1", &err), err));
507
+ CL_CHECK((dequantize_mul_mat_vec_q5_0_cl = clCreateKernel(program, "dequantize_mul_mat_vec_q5_0", &err), err));
508
+ CL_CHECK((dequantize_mul_mat_vec_q5_1_cl = clCreateKernel(program, "dequantize_mul_mat_vec_q5_1", &err), err));
509
+ CL_CHECK((dequantize_mul_mat_vec_q8_0_cl = clCreateKernel(program, "dequantize_mul_mat_vec_q8_0", &err), err));
510
+ CL_CHECK((convert_mul_mat_vec_f16_cl = clCreateKernel(program, "convert_mul_mat_vec_f16", &err), err));
511
+ }
512
+
513
+ static cl_kernel* ggml_get_to_fp32_cl(ggml_type type) {
514
+ switch (type) {
515
+ case GGML_TYPE_Q4_0:
516
+ return &dequantize_row_q4_0_cl;
517
+ case GGML_TYPE_Q4_1:
518
+ return &dequantize_row_q4_1_cl;
519
+ case GGML_TYPE_Q5_0:
520
+ return &dequantize_row_q5_0_cl;
521
+ case GGML_TYPE_Q5_1:
522
+ return &dequantize_row_q5_1_cl;
523
+ case GGML_TYPE_Q8_0:
524
+ return &dequantize_row_q8_0_cl;
525
+ case GGML_TYPE_F16:
526
+ return &convert_row_f16_cl;
527
+ default:
528
+ return nullptr;
529
+ }
530
+ }
531
+
532
+ static cl_kernel* ggml_get_dequantize_mul_mat_vec_cl(ggml_type type) {
533
+ switch (type) {
534
+ case GGML_TYPE_Q4_0:
535
+ return &dequantize_mul_mat_vec_q4_0_cl;
536
+ case GGML_TYPE_Q4_1:
537
+ return &dequantize_mul_mat_vec_q4_1_cl;
538
+ case GGML_TYPE_Q5_0:
539
+ return &dequantize_mul_mat_vec_q5_0_cl;
540
+ case GGML_TYPE_Q5_1:
541
+ return &dequantize_mul_mat_vec_q5_1_cl;
542
+ case GGML_TYPE_Q8_0:
543
+ return &dequantize_mul_mat_vec_q8_0_cl;
544
+ case GGML_TYPE_F16:
545
+ return &convert_mul_mat_vec_f16_cl;
546
+ default:
547
+ return nullptr;
548
+ }
549
+ }
550
+
551
+ // buffer pool for cl
552
+ #define MAX_CL_BUFFERS 256
553
+
554
+ struct scoped_spin_lock {
555
+ std::atomic_flag& lock;
556
+ scoped_spin_lock(std::atomic_flag& lock) : lock(lock) {
557
+ while (lock.test_and_set(std::memory_order_acquire)) {
558
+ ; // spin
559
+ }
560
+ }
561
+ ~scoped_spin_lock() {
562
+ lock.clear(std::memory_order_release);
563
+ }
564
+ scoped_spin_lock(const scoped_spin_lock&) = delete;
565
+ scoped_spin_lock& operator=(const scoped_spin_lock&) = delete;
566
+ };
567
+
568
+ struct cl_buffer {
569
+ cl_mem mem;
570
+ size_t size = 0;
571
+ };
572
+
573
+ static cl_buffer g_cl_buffer_pool[MAX_CL_BUFFERS];
574
+ static std::atomic_flag g_cl_pool_lock = ATOMIC_FLAG_INIT;
575
+
576
+ static cl_mem ggml_cl_pool_malloc(size_t size, size_t * actual_size, cl_mem_flags flags) {
577
+ scoped_spin_lock lock(g_cl_pool_lock);
578
+ cl_int err;
579
+
580
+ for (int i = 0; i < MAX_CL_BUFFERS; ++i) {
581
+ cl_buffer& b = g_cl_buffer_pool[i];
582
+ if (b.size > 0 && b.size >= size) {
583
+ cl_mem mem = b.mem;
584
+ *actual_size = b.size;
585
+ b.size = 0;
586
+ return mem;
587
+ }
588
+ }
589
+ cl_mem mem;
590
+ CL_CHECK((mem = clCreateBuffer(context, flags, size, NULL, &err), err));
591
+ *actual_size = size;
592
+ return mem;
593
+ }
594
+
595
+ static void ggml_cl_pool_free(cl_mem mem, size_t size) {
596
+ scoped_spin_lock lock(g_cl_pool_lock);
597
+
598
+ for (int i = 0; i < MAX_CL_BUFFERS; ++i) {
599
+ cl_buffer& b = g_cl_buffer_pool[i];
600
+ if (b.size == 0) {
601
+ b.mem = mem;
602
+ b.size = size;
603
+ return;
604
+ }
605
+ }
606
+ fprintf(stderr, "WARNING: cl buffer pool full, increase MAX_CL_BUFFERS\n");
607
+ clReleaseMemObject(mem);
608
+ }
609
+
610
+ static cl_int ggml_cl_h2d_tensor_2d(cl_command_queue queue, cl_mem dst, size_t offset, const struct ggml_tensor * src, uint64_t i3, uint64_t i2, cl_event* ev) {
611
+ cl_int err;
612
+ const uint64_t ne0 = src->ne[0];
613
+ const uint64_t ne1 = src->ne[1];
614
+ const uint64_t nb0 = src->nb[0];
615
+ const uint64_t nb1 = src->nb[1];
616
+ const uint64_t nb2 = src->nb[2];
617
+ const uint64_t nb3 = src->nb[3];
618
+ const enum ggml_type type = src->type;
619
+ const size_t ts = ggml_type_size(type);
620
+ const size_t bs = ggml_blck_size(type);
621
+
622
+ const void * x = (const void *) ((const char *) src->data + i2*nb2 + i3*nb3);
623
+ if (nb0 == ts && nb1 == ts*ne0/bs) {
624
+ err = clEnqueueWriteBuffer(queue, dst, CL_FALSE, offset, ne1*nb1, x, 0, NULL, ev);
625
+ return err;
626
+ }
627
+ if (nb0 == ts) {
628
+ const size_t buffer_origin[3] = { offset, 0, 0 };
629
+ const size_t host_origin[3] = { 0, 0, 0 };
630
+ const size_t region[3] = { ts*ne0/bs, ne1, 1 };
631
+ err = clEnqueueWriteBufferRect(queue, dst, CL_FALSE, buffer_origin, host_origin, region, ts*ne0/bs, 0, nb1, 0, x, 0, NULL, ev);
632
+ return err;
633
+ }
634
+ for (uint64_t i1 = 0; i1 < ne1; i1++) {
635
+ // pretend the row is a matrix with cols=1
636
+ const size_t buffer_origin[3] = { offset, i1, 0 };
637
+ const size_t host_origin[3] = { 0, 0, 0 };
638
+ const size_t region[3] = { ts/bs, ne0, 1 };
639
+ err = clEnqueueWriteBufferRect(queue, dst, CL_FALSE, buffer_origin, host_origin, region, 0, 0, nb0, 0, ((const char *)x) + i1*nb0, 0, NULL, ev);
640
+ if (err != CL_SUCCESS) {
641
+ break;
642
+ }
643
+ }
644
+ return err;
645
+ }
646
+
647
+ static void ggml_cl_mul_mat_f32(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
648
+ const int64_t ne00 = src0->ne[0];
649
+ const int64_t ne01 = src0->ne[1];
650
+ const int64_t ne02 = src0->ne[2];
651
+ const int64_t ne03 = src0->ne[3];
652
+
653
+ const int64_t ne10 = src1->ne[0];
654
+ const int64_t ne11 = src1->ne[1];
655
+
656
+ const int nb2 = dst->nb[2];
657
+ const int nb3 = dst->nb[3];
658
+
659
+ const float alpha = 1.0f;
660
+ const float beta = 0.0f;
661
+ const int x_ne = ne01 * ne00;
662
+ const int y_ne = ne11 * ne10;
663
+ const int d_ne = ne11 * ne01;
664
+
665
+ size_t x_size;
666
+ size_t y_size;
667
+ size_t d_size;
668
+ cl_mem d_X;
669
+ if (src0->backend == GGML_BACKEND_CL) {
670
+ d_X = (cl_mem) src0->data;
671
+ } else {
672
+ d_X = ggml_cl_pool_malloc(sizeof(ggml_fp16_t) * x_ne, &x_size, CL_MEM_READ_ONLY);
673
+ }
674
+ cl_mem d_Y = ggml_cl_pool_malloc(sizeof(float) * y_ne, &y_size, CL_MEM_READ_ONLY);
675
+ cl_mem d_D = ggml_cl_pool_malloc(sizeof(float) * d_ne, &d_size, CL_MEM_WRITE_ONLY);
676
+
677
+ for (int64_t i03 = 0; i03 < ne03; i03++) {
678
+ for (int64_t i02 = 0; i02 < ne02; i02++) {
679
+ // copy data to device
680
+ if (src0->backend != GGML_BACKEND_CL) {
681
+ CL_CHECK(ggml_cl_h2d_tensor_2d(queue, d_X, 0, src0, i03, i02, NULL));
682
+ }
683
+ CL_CHECK(ggml_cl_h2d_tensor_2d(queue, d_Y, 0, src1, i03, i02, NULL));
684
+
685
+ CL_CHECK(clFinish(queue));
686
+
687
+ // compute
688
+ cl_event ev_sgemm;
689
+ clblast::StatusCode status = clblast::Gemm<cl_float>(clblast::Layout::kColMajor,
690
+ clblast::Transpose::kYes, clblast::Transpose::kNo,
691
+ ne01, ne11, ne10,
692
+ alpha,
693
+ d_X, 0, ne00,
694
+ d_Y, 0, ne10,
695
+ beta,
696
+ d_D, 0, ne01,
697
+ &queue, &ev_sgemm);
698
+
699
+ if (status != clblast::StatusCode::kSuccess) {
700
+ GGML_ASSERT(false);
701
+ }
702
+
703
+ // copy dst to host
704
+ float * d = (float *) ((char *) dst->data + i02*nb2 + i03*nb3);
705
+ CL_CHECK(clEnqueueReadBuffer(queue, d_D, true, 0, sizeof(float) * d_ne, d, 1, &ev_sgemm, NULL));
706
+ }
707
+ }
708
+
709
+ if (src0->backend != GGML_BACKEND_CL) {
710
+ ggml_cl_pool_free(d_X, x_size);
711
+ }
712
+ ggml_cl_pool_free(d_Y, y_size);
713
+ ggml_cl_pool_free(d_D, d_size);
714
+ }
715
+
716
+ static void ggml_cl_mul_mat_f16(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, void * wdata, size_t /* wsize */) {
717
+ GGML_ASSERT(fp16_support);
718
+
719
+ const int64_t ne00 = src0->ne[0];
720
+ const int64_t ne01 = src0->ne[1];
721
+ const int64_t ne02 = src0->ne[2];
722
+ const int64_t ne03 = src0->ne[3];
723
+
724
+ const int64_t ne10 = src1->ne[0];
725
+ const int64_t ne11 = src1->ne[1];
726
+
727
+ const int nb10 = src1->nb[0];
728
+ const int nb11 = src1->nb[1];
729
+ const int nb12 = src1->nb[2];
730
+ const int nb13 = src1->nb[3];
731
+
732
+ const int nb2 = dst->nb[2];
733
+ const int nb3 = dst->nb[3];
734
+
735
+ const ggml_fp16_t alpha = ggml_fp32_to_fp16(1.0f);
736
+ const ggml_fp16_t beta = ggml_fp32_to_fp16(0.0f);
737
+ const int x_ne = ne01 * ne00;
738
+ const int y_ne = ne11 * ne10;
739
+ const int d_ne = ne11 * ne01;
740
+
741
+ size_t x_size;
742
+ size_t y_size;
743
+ size_t d_size;
744
+ cl_mem d_X;
745
+ if (src0->backend == GGML_BACKEND_CL) {
746
+ d_X = (cl_mem) src0->data;
747
+ } else {
748
+ d_X = ggml_cl_pool_malloc(sizeof(ggml_fp16_t) * x_ne, &x_size, CL_MEM_READ_ONLY);
749
+ }
750
+ cl_mem d_Y = ggml_cl_pool_malloc(sizeof(ggml_fp16_t) * y_ne, &y_size, CL_MEM_READ_ONLY);
751
+ cl_mem d_D = ggml_cl_pool_malloc(sizeof(ggml_fp16_t) * d_ne, &d_size, CL_MEM_WRITE_ONLY);
752
+
753
+ bool src1_cont_rows = nb10 == sizeof(float);
754
+ bool src1_cont_cols = (size_t)nb11 == ne11*sizeof(float);
755
+
756
+ for (int64_t i03 = 0; i03 < ne03; i03++) {
757
+ for (int64_t i02 = 0; i02 < ne02; i02++) {
758
+ // copy src0 to device
759
+ if (src0->backend != GGML_BACKEND_CL) {
760
+ CL_CHECK(ggml_cl_h2d_tensor_2d(queue, d_X, 0, src0, i03, i02, NULL));
761
+ }
762
+
763
+ // convert src1 to fp16
764
+ // TODO: use multiple threads
765
+ ggml_fp16_t * const tmp = (ggml_fp16_t *) wdata + (ne11 * ne10) * (i03 * ne02 + i02);
766
+ char * src1i = (char *) src1->data + i03*nb13 + i02*nb12;
767
+ if (src1_cont_rows) {
768
+ if (src1_cont_cols) {
769
+ ggml_fp32_to_fp16_row((float *) src1i, tmp, ne10*ne11);
770
+ }
771
+ else {
772
+ for (int64_t i01 = 0; i01 < ne11; i01++) {
773
+ ggml_fp32_to_fp16_row((float *) (src1i + i01*nb11), tmp + i01*ne10, ne10);
774
+ }
775
+ }
776
+ }
777
+ else {
778
+ for (int64_t i01 = 0; i01 < ne11; i01++) {
779
+ for (int64_t i00 = 0; i00 < ne10; i00++) {
780
+ // very slow due to no inlining
781
+ tmp[i01*ne10 + i00] = ggml_fp32_to_fp16(*(float *) (src1i + i01*nb11 + i00*nb10));
782
+ }
783
+ }
784
+ }
785
+
786
+ // copy src1 to device
787
+ CL_CHECK(clEnqueueWriteBuffer(queue, d_Y, false, 0, sizeof(ggml_fp16_t) * y_ne, tmp, 0, NULL, NULL));
788
+
789
+ CL_CHECK(clFinish(queue));
790
+
791
+ // compute
792
+ cl_event ev_sgemm;
793
+ clblast::StatusCode status = clblast::Gemm<cl_half>(clblast::Layout::kColMajor,
794
+ clblast::Transpose::kYes, clblast::Transpose::kNo,
795
+ ne01, ne11, ne10,
796
+ alpha,
797
+ d_X, 0, ne00,
798
+ d_Y, 0, ne10,
799
+ beta,
800
+ d_D, 0, ne01,
801
+ &queue, &ev_sgemm);
802
+
803
+ if (status != clblast::StatusCode::kSuccess) {
804
+ GGML_ASSERT(false);
805
+ }
806
+
807
+ // copy dst to host, then convert to float
808
+ CL_CHECK(clEnqueueReadBuffer(queue, d_D, true, 0, sizeof(ggml_fp16_t) * d_ne, tmp, 1, &ev_sgemm, NULL));
809
+
810
+ float * d = (float *) ((char *) dst->data + i02*nb2 + i03*nb3);
811
+
812
+ ggml_fp16_to_fp32_row(tmp, d, d_ne);
813
+ }
814
+ }
815
+
816
+ if (src0->backend != GGML_BACKEND_CL) {
817
+ ggml_cl_pool_free(d_X, x_size);
818
+ }
819
+ ggml_cl_pool_free(d_Y, y_size);
820
+ ggml_cl_pool_free(d_D, d_size);
821
+ }
822
+
823
+ static void ggml_cl_mul_mat_q_f32(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
824
+ const int64_t ne00 = src0->ne[0];
825
+ const int64_t ne01 = src0->ne[1];
826
+ const int64_t ne02 = src0->ne[2];
827
+ const int64_t ne03 = src0->ne[3];
828
+
829
+ const int64_t ne10 = src1->ne[0];
830
+ const int64_t ne11 = src1->ne[1];
831
+
832
+ const int nb2 = dst->nb[2];
833
+ const int nb3 = dst->nb[3];
834
+ const ggml_type type = src0->type;
835
+ const bool mul_mat_vec = ne11 == 1;
836
+
837
+ const float alpha = 1.0f;
838
+ const float beta = 0.0f;
839
+ const int x_ne = ne01 * ne00;
840
+ const int y_ne = ne11 * ne10;
841
+ const int d_ne = ne11 * ne01;
842
+ const size_t q_sz = ggml_type_size(type) * x_ne / ggml_blck_size(type);
843
+
844
+ size_t x_size;
845
+ size_t y_size;
846
+ size_t d_size;
847
+ size_t q_size;
848
+ cl_mem d_X;
849
+ if (!mul_mat_vec) {
850
+ d_X = ggml_cl_pool_malloc(sizeof(float) * x_ne, &x_size, CL_MEM_READ_WRITE);
851
+ }
852
+ cl_mem d_Y = ggml_cl_pool_malloc(sizeof(float) * y_ne, &y_size, CL_MEM_READ_ONLY);
853
+ cl_mem d_D = ggml_cl_pool_malloc(sizeof(float) * d_ne, &d_size, CL_MEM_WRITE_ONLY);
854
+ cl_mem d_Q;
855
+ if (src0->backend == GGML_BACKEND_CPU) {
856
+ d_Q = ggml_cl_pool_malloc(q_sz, &q_size, CL_MEM_READ_ONLY);
857
+ }
858
+
859
+ cl_kernel* to_fp32_cl = ggml_get_to_fp32_cl(type);
860
+ cl_kernel* dmmv = ggml_get_dequantize_mul_mat_vec_cl(type);
861
+ GGML_ASSERT(to_fp32_cl != nullptr);
862
+
863
+ for (int64_t i03 = 0; i03 < ne03; i03++) {
864
+ for (int64_t i02 = 0; i02 < ne02; i02++) {
865
+ cl_event ev_sgemm;
866
+
867
+ // copy src0 to device if necessary
868
+ if (src0->backend == GGML_BACKEND_CPU) {
869
+ CL_CHECK(ggml_cl_h2d_tensor_2d(queue, d_Q, 0, src0, i03, i02, NULL));
870
+ } else if (src0->backend == GGML_BACKEND_CL) {
871
+ d_Q = (cl_mem) src0->data;
872
+ } else {
873
+ GGML_ASSERT(false);
874
+ }
875
+ if (mul_mat_vec) { // specialized dequantize_mul_mat_vec kernel
876
+ // copy src1 to device
877
+ CL_CHECK(ggml_cl_h2d_tensor_2d(queue, d_Y, 0, src1, i03, i02, NULL));
878
+
879
+ // compute
880
+ const size_t global = ne01 * CL_DMMV_BLOCK_SIZE;
881
+ const size_t local = CL_DMMV_BLOCK_SIZE;
882
+ const cl_int ncols = ne00;
883
+ CL_CHECK(clSetKernelArg(*dmmv, 0, sizeof(cl_mem), &d_Q));
884
+ CL_CHECK(clSetKernelArg(*dmmv, 1, sizeof(float) * local, NULL));
885
+ CL_CHECK(clSetKernelArg(*dmmv, 2, sizeof(cl_mem), &d_Y));
886
+ CL_CHECK(clSetKernelArg(*dmmv, 3, sizeof(cl_mem), &d_D));
887
+ CL_CHECK(clSetKernelArg(*dmmv, 4, sizeof(cl_int), &ncols));
888
+ CL_CHECK(clFinish(queue));
889
+ CL_CHECK(clEnqueueNDRangeKernel(queue, *dmmv, 1, NULL, &global, &local, 0, NULL, &ev_sgemm));
890
+ } else { // general dequantization kernel + CLBlast matrix matrix multiplication
891
+ // convert src0 to fp32 on device
892
+ const size_t global = x_ne;
893
+ CL_CHECK(clSetKernelArg(*to_fp32_cl, 0, sizeof(cl_mem), &d_Q));
894
+ CL_CHECK(clSetKernelArg(*to_fp32_cl, 1, sizeof(cl_mem), &d_X));
895
+ CL_CHECK(clFinish(queue));
896
+ CL_CHECK(clEnqueueNDRangeKernel(queue, *to_fp32_cl, 1, NULL, &global, NULL, 0, NULL, NULL));
897
+
898
+ // copy src1 to device
899
+ CL_CHECK(ggml_cl_h2d_tensor_2d(queue, d_Y, 0, src1, i03, i02, NULL));
900
+
901
+ // wait for conversion
902
+ CL_CHECK(clFinish(queue));
903
+
904
+ // compute
905
+ clblast::StatusCode status = clblast::Gemm<cl_float>(clblast::Layout::kColMajor,
906
+ clblast::Transpose::kYes, clblast::Transpose::kNo,
907
+ ne01, ne11, ne10,
908
+ alpha,
909
+ d_X, 0, ne00,
910
+ d_Y, 0, ne10,
911
+ beta,
912
+ d_D, 0, ne01,
913
+ &queue, &ev_sgemm);
914
+
915
+ if (status != clblast::StatusCode::kSuccess) {
916
+ GGML_ASSERT(false);
917
+ }
918
+ }
919
+
920
+ // copy dst to host
921
+ float * d = (float *) ((char *) dst->data + i02*nb2 + i03*nb3);
922
+ CL_CHECK(clEnqueueReadBuffer(queue, d_D, true, 0, sizeof(float) * d_ne, d, 1, &ev_sgemm, NULL));
923
+ clReleaseEvent(ev_sgemm);
924
+ }
925
+ }
926
+
927
+ if (!mul_mat_vec) {
928
+ ggml_cl_pool_free(d_X, x_size);
929
+ }
930
+ ggml_cl_pool_free(d_Y, y_size);
931
+ ggml_cl_pool_free(d_D, d_size);
932
+ if (src0->backend == GGML_BACKEND_CPU) {
933
+ ggml_cl_pool_free(d_Q, q_size);
934
+ }
935
+ }
936
+
937
+
938
+ bool ggml_cl_can_mul_mat(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst) {
939
+ const int64_t ne10 = src1->ne[0];
940
+
941
+ const int64_t ne0 = dst->ne[0];
942
+ const int64_t ne1 = dst->ne[1];
943
+
944
+ // TODO: find the optimal values for these
945
+ if ((src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || ggml_is_quantized(src0->type)) &&
946
+ src1->type == GGML_TYPE_F32 &&
947
+ dst->type == GGML_TYPE_F32 &&
948
+ ((ne0 >= 32 && ne1 >= 32 && ne10 >= 32) || src0->backend == GGML_BACKEND_CL)) {
949
+ return true;
950
+ }
951
+
952
+ return false;
953
+ }
954
+
955
+ bool ggml_cl_mul_mat_use_f16(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * /* dst */) {
956
+ // If device doesn't support FP16
957
+ if (!fp16_support) {
958
+ return false;
959
+ }
960
+
961
+ size_t src0_sz = ggml_nbytes(src0);
962
+ size_t src1_sz = ggml_nbytes(src1);
963
+
964
+ // mul_mat_q: src0 is converted to fp32 on device
965
+ size_t mul_mat_q_transfer = src0_sz + src1_sz;
966
+
967
+ // mul_mat_f16: src1 is converted to fp16 on cpu
968
+ size_t mul_mat_f16_transfer = src0_sz + sizeof(ggml_fp16_t) * ggml_nelements(src1);
969
+
970
+ // choose the smaller one to transfer to the device
971
+ // TODO: this is not always the best choice due to the overhead of converting to fp16
972
+ return mul_mat_f16_transfer < mul_mat_q_transfer;
973
+ }
974
+
975
+ void ggml_cl_mul_mat(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst, void * wdata, size_t wsize) {
976
+ GGML_ASSERT(ggml_cl_can_mul_mat(src0, src1, dst));
977
+
978
+ if (src0->type == GGML_TYPE_F32) {
979
+ ggml_cl_mul_mat_f32(src0, src1, dst);
980
+ }
981
+ else if (src0->type == GGML_TYPE_F16) {
982
+ if (ggml_cl_mul_mat_use_f16(src0, src1, dst)) {
983
+ ggml_cl_mul_mat_f16(src0, src1, dst, wdata, wsize);
984
+ }
985
+ else {
986
+ ggml_cl_mul_mat_q_f32(src0, src1, dst);
987
+ }
988
+ }
989
+ else if (ggml_is_quantized(src0->type)) {
990
+ ggml_cl_mul_mat_q_f32(src0, src1, dst);
991
+ }
992
+ else {
993
+ GGML_ASSERT(false);
994
+ }
995
+ }
996
+
997
+ size_t ggml_cl_mul_mat_get_wsize(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst) {
998
+ if (ggml_cl_mul_mat_use_f16(src0, src1, dst)) {
999
+ return ggml_nelements(src1) * sizeof(ggml_fp16_t);
1000
+ }
1001
+ return 0;
1002
+ }
1003
+
1004
+ void ggml_cl_transform_tensor(ggml_tensor * tensor) {
1005
+ const int64_t ne0 = tensor->ne[0];
1006
+ const int64_t ne1 = tensor->ne[1];
1007
+ const int64_t ne2 = tensor->ne[2];
1008
+ const int64_t ne3 = tensor->ne[3];
1009
+
1010
+ const ggml_type type = tensor->type;
1011
+ const size_t q_sz = ggml_type_size(type) * ne0 * ne1 * ne2 * ne3 / ggml_blck_size(type);
1012
+
1013
+ size_t q_size;
1014
+ cl_mem dst = ggml_cl_pool_malloc(q_sz, &q_size, CL_MEM_READ_ONLY);
1015
+
1016
+ // copy tensor to device
1017
+ for (int64_t i3 = 0; i3 < ne3; i3++) {
1018
+ for (int64_t i2 = 0; i2 < ne2; i2++) {
1019
+ int i = i3*ne2 + i2;
1020
+ CL_CHECK(ggml_cl_h2d_tensor_2d(queue, dst, i*ne0*ne1, tensor, i3, i2, NULL));
1021
+ }
1022
+ }
1023
+
1024
+ CL_CHECK(clFinish(queue));
1025
+
1026
+ tensor->data = dst;
1027
+ tensor->backend = GGML_BACKEND_CL;
1028
+ }