llama_cpp 0.1.2 → 0.1.3

Sign up to get free protection for your applications and to get access to all the features.
@@ -1,474 +0,0 @@
1
- #include "ggml-opencl.h"
2
-
3
- #define CL_TARGET_OPENCL_VERSION 110
4
- #include <clblast_c.h>
5
-
6
- #include <stdlib.h>
7
- #include <stdio.h>
8
- #include <string.h>
9
-
10
- #include "ggml.h"
11
-
12
- #define MULTILINE_QUOTE(...) #__VA_ARGS__
13
- static const char * program_source = MULTILINE_QUOTE(
14
-
15
- typedef char int8_t;
16
- typedef uchar uint8_t;
17
- typedef int int32_t;
18
- typedef uint uint32_t;
19
-
20
- struct __attribute__ ((packed)) block_q4_0
21
- {
22
- half d;
23
- uint8_t qs[16]; /* QK4_0 / 2 */
24
- };
25
-
26
- struct __attribute__ ((packed)) block_q4_1
27
- {
28
- half d;
29
- half m;
30
- uint8_t qs[16]; /* QK4_1 / 2 */
31
- };
32
-
33
- struct __attribute__ ((packed)) block_q5_0
34
- {
35
- half d;
36
- uint32_t qh;
37
- uint8_t qs[16]; /* QK5_0 / 2 */
38
- };
39
-
40
- struct __attribute__ ((packed)) block_q5_1
41
- {
42
- half d;
43
- half m;
44
- uint32_t qh;
45
- uint8_t qs[16]; /* QK5_1 / 2 */
46
- };
47
-
48
- struct __attribute__ ((packed)) block_q8_0
49
- {
50
- half d;
51
- int8_t qs[32]; /* QK8_0 */
52
- };
53
-
54
-
55
- __kernel void dequantize_row_q4_0(__global struct block_q4_0* x, __global float* y) {
56
- const uint i = get_global_id(0) / 32; /* QK4_0 */
57
- const uint j = get_local_id(0);
58
-
59
- const float d = vload_half(0, (__global half*) &x[i].d);
60
-
61
- const int x0 = (x[i].qs[j] & 0xf) - 8;
62
- const int x1 = (x[i].qs[j] >> 4) - 8;
63
-
64
- y[i*32 + j + 0 ] = x0*d;
65
- y[i*32 + j + 16] = x1*d;
66
- }
67
-
68
- __kernel void dequantize_row_q4_1(__global struct block_q4_1* x, __global float* y) {
69
- const uint i = get_global_id(0) / 32; /* QK4_1 */
70
- const uint j = get_local_id(0);
71
-
72
- const float d = vload_half(0, (__global half*) &x[i].d);
73
- const float m = vload_half(0, (__global half*) &x[i].m);
74
-
75
- const int x0 = (x[i].qs[j] & 0xf);
76
- const int x1 = (x[i].qs[j] >> 4);
77
-
78
- y[i*32 + j + 0 ] = x0*d + m;
79
- y[i*32 + j + 16] = x1*d + m;
80
- }
81
-
82
- __kernel void dequantize_row_q5_0(__global struct block_q5_0* x, __global float* y) {
83
- const uint i = get_global_id(0) / 32; /* QK5_0 */
84
- const uint j = get_local_id(0);
85
-
86
- const float d = vload_half(0, (__global half*) &x[i].d);
87
-
88
- uint32_t qh = x[i].qh;
89
-
90
- const uint8_t xh_0 = ((qh >> (j + 0)) << 4) & 0x10;
91
- const uint8_t xh_1 = ((qh >> (j + 12)) ) & 0x10;
92
-
93
- const int32_t x0 = ((x[i].qs[j] & 0xf) | xh_0) - 16;
94
- const int32_t x1 = ((x[i].qs[j] >> 4) | xh_1) - 16;
95
-
96
- y[i*32 + j + 0 ] = x0*d;
97
- y[i*32 + j + 16] = x1*d;
98
- }
99
-
100
- __kernel void dequantize_row_q5_1(__global struct block_q5_1* x, __global float* y) {
101
- const uint i = get_global_id(0) / 32; /* QK5_1 */
102
- const uint j = get_local_id(0);
103
-
104
- const float d = vload_half(0, (__global half*) &x[i].d);
105
- const float m = vload_half(0, (__global half*) &x[i].m);
106
-
107
- uint32_t qh = x[i].qh;
108
-
109
- const uint8_t xh_0 = ((qh >> (j + 0)) << 4) & 0x10;
110
- const uint8_t xh_1 = ((qh >> (j + 12)) ) & 0x10;
111
-
112
- const int x0 = (x[i].qs[j] & 0xf) | xh_0;
113
- const int x1 = (x[i].qs[j] >> 4) | xh_1;
114
-
115
- y[i*32 + j + 0 ] = x0*d + m;
116
- y[i*32 + j + 16] = x1*d + m;
117
- }
118
-
119
- __kernel void dequantize_row_q8_0(__global struct block_q8_0* x, __global float* y) {
120
- const uint i = get_global_id(0) / 32; /* QK8_0 */
121
- const uint j = get_local_id(0);
122
-
123
- const float d = vload_half(0, (__global half*) &x[i].d);
124
- y[i*32 + j] = x[i].qs[j]*d;
125
- }
126
-
127
- );
128
-
129
- #define CL_CHECK(err) \
130
- do { \
131
- cl_int err_ = (err); \
132
- if (err_ != CL_SUCCESS) { \
133
- fprintf(stderr, "ggml_opencl: %s error %d at %s:%d\n", \
134
- #err, err_, __FILE__, __LINE__); \
135
- exit(1); \
136
- } \
137
- } while (0)
138
-
139
- #define CLBLAST_CHECK(err) \
140
- do { \
141
- CLBlastStatusCode err_ = (err); \
142
- if (err_ != CLBlastSuccess) { \
143
- fprintf(stderr, "ggml_opencl: %s error %d at %s:%d\n", \
144
- #err, err_, __FILE__, __LINE__); \
145
- exit(1); \
146
- } \
147
- } while (0)
148
-
149
- static cl_platform_id platform;
150
- static cl_device_id device;
151
- static cl_context context;
152
- static cl_command_queue queue;
153
- static cl_program program;
154
- static cl_kernel kernel_q4_0, kernel_q4_1, kernel_q5_0, kernel_q5_1, kernel_q8_0;
155
- static cl_mem cl_buffer_a, cl_buffer_qb, cl_buffer_b, cl_buffer_c;
156
- static size_t cl_size_a = 0, cl_size_qb = 0, cl_size_b = 0, cl_size_c = 0;
157
-
158
- static cl_program build_program_from_source(cl_context ctx, cl_device_id dev, const char* program_buffer) {
159
- cl_program p;
160
- char *program_log;
161
- size_t program_size, log_size;
162
- int err;
163
-
164
- program_size = strlen(program_buffer);
165
-
166
- p = clCreateProgramWithSource(ctx, 1, (const char**)&program_buffer, &program_size, &err);
167
- if(err < 0) {
168
- fprintf(stderr, "OpenCL error creating program");
169
- exit(1);
170
- }
171
-
172
- err = clBuildProgram(p, 0, NULL, NULL, NULL, NULL);
173
- if(err < 0) {
174
-
175
- clGetProgramBuildInfo(p, dev, CL_PROGRAM_BUILD_LOG, 0, NULL, &log_size);
176
- program_log = (char*) malloc(log_size + 1);
177
- program_log[log_size] = '\0';
178
- clGetProgramBuildInfo(p, dev, CL_PROGRAM_BUILD_LOG, log_size + 1, program_log, NULL);
179
- printf("%s\n", program_log);
180
- free(program_log);
181
- exit(1);
182
- }
183
-
184
- return p;
185
- }
186
-
187
- void ggml_cl_init(void) {
188
- cl_int err = 0;
189
-
190
- struct cl_device;
191
- struct cl_platform {
192
- cl_platform_id id;
193
- unsigned number;
194
- char name[128];
195
- char vendor[128];
196
- struct cl_device * devices;
197
- unsigned n_devices;
198
- struct cl_device * default_device;
199
- };
200
-
201
- struct cl_device {
202
- struct cl_platform * platform;
203
- cl_device_id id;
204
- unsigned number;
205
- cl_device_type type;
206
- char name[128];
207
- };
208
-
209
- enum { NPLAT = 16, NDEV = 16 };
210
-
211
- struct cl_platform platforms[NPLAT];
212
- unsigned n_platforms = 0;
213
- struct cl_device devices[NDEV];
214
- unsigned n_devices = 0;
215
- struct cl_device * default_device = NULL;
216
-
217
- platform = NULL;
218
- device = NULL;
219
-
220
- cl_platform_id platform_ids[NPLAT];
221
- CL_CHECK(clGetPlatformIDs(NPLAT, platform_ids, &n_platforms));
222
-
223
- for (unsigned i = 0; i < n_platforms; i++) {
224
- struct cl_platform * p = &platforms[i];
225
- p->number = i;
226
- p->id = platform_ids[i];
227
- CL_CHECK(clGetPlatformInfo(p->id, CL_PLATFORM_NAME, sizeof(p->name), &p->name, NULL));
228
- CL_CHECK(clGetPlatformInfo(p->id, CL_PLATFORM_VENDOR, sizeof(p->vendor), &p->vendor, NULL));
229
-
230
- cl_device_id device_ids[NDEV];
231
- cl_int clGetDeviceIDsError = clGetDeviceIDs(p->id, CL_DEVICE_TYPE_ALL, NDEV, device_ids, &p->n_devices);
232
- if (clGetDeviceIDsError == CL_DEVICE_NOT_FOUND) {
233
- p->n_devices = 0;
234
- } else {
235
- CL_CHECK(clGetDeviceIDsError);
236
- }
237
- p->devices = p->n_devices > 0 ? &devices[n_devices] : NULL;
238
- p->default_device = NULL;
239
-
240
- for (unsigned j = 0; j < p->n_devices; j++) {
241
- struct cl_device * d = &devices[n_devices];
242
- d->number = n_devices++;
243
- d->id = device_ids[j];
244
- d->platform = p;
245
- CL_CHECK(clGetDeviceInfo(d->id, CL_DEVICE_NAME, sizeof(d->name), &d->name, NULL));
246
- CL_CHECK(clGetDeviceInfo(d->id, CL_DEVICE_TYPE, sizeof(d->type), &d->type, NULL));
247
-
248
- if (p->default_device == NULL && d->type == CL_DEVICE_TYPE_GPU) {
249
- p->default_device = d;
250
- }
251
- }
252
-
253
- if (default_device == NULL && p->default_device != NULL) {
254
- default_device = p->default_device;
255
- }
256
- }
257
-
258
- if (n_devices == 0) {
259
- fprintf(stderr, "ggml_opencl: could find any OpenCL devices.\n");
260
- exit(1);
261
- }
262
-
263
- char * user_platform_string = getenv("GGML_OPENCL_PLATFORM");
264
- char * user_device_string = getenv("GGML_OPENCL_DEVICE");
265
- int user_platform_number = -1;
266
- int user_device_number = -1;
267
-
268
- unsigned n;
269
- if (user_platform_string != NULL && sscanf(user_platform_string, " %u", &n) == 1 && n < n_platforms) {
270
- user_platform_number = (int)n;
271
- }
272
- if (user_device_string != NULL && sscanf(user_device_string, " %u", &n) == 1 && n < n_devices) {
273
- user_device_number = (int)n;
274
- }
275
-
276
- struct cl_device * selected_devices = devices;
277
- unsigned n_selected_devices = n_devices;
278
-
279
- if (user_platform_number == -1 && user_platform_string != NULL && user_platform_string[0] != 0) {
280
- for (unsigned i = 0; i < n_platforms; i++) {
281
- struct cl_platform * p = &platforms[i];
282
- if (strstr(p->name, user_platform_string) != NULL ||
283
- strstr(p->vendor, user_platform_string) != NULL) {
284
- user_platform_number = (int)i;
285
- break;
286
- }
287
- }
288
- if (user_platform_number == -1) {
289
- fprintf(stderr, "ggml_opencl: no platform matching '%s' was found.\n", user_platform_string);
290
- exit(1);
291
- }
292
- }
293
- if (user_platform_number != -1) {
294
- struct cl_platform * p = &platforms[user_platform_number];
295
- selected_devices = p->devices;
296
- n_selected_devices = p->n_devices;
297
- default_device = p->default_device;
298
- if (n_selected_devices == 0) {
299
- fprintf(stderr, "ggml_opencl: selected platform '%s' does not have any devices.\n", p->name);
300
- exit(1);
301
- }
302
- }
303
-
304
- if (user_device_number == -1 && user_device_string != NULL && user_device_string[0] != 0) {
305
- for (unsigned i = 0; i < n_selected_devices; i++) {
306
- struct cl_device * d = &selected_devices[i];
307
- if (strstr(d->name, user_device_string) != NULL) {
308
- user_device_number = d->number;
309
- break;
310
- }
311
- }
312
- if (user_device_number == -1) {
313
- fprintf(stderr, "ggml_opencl: no device matching '%s' was found.\n", user_device_string);
314
- exit(1);
315
- }
316
- }
317
- if (user_device_number != -1) {
318
- selected_devices = &devices[user_device_number];
319
- n_selected_devices = 1;
320
- default_device = &selected_devices[0];
321
- }
322
-
323
- GGML_ASSERT(n_selected_devices > 0);
324
-
325
- if (default_device == NULL) {
326
- default_device = &selected_devices[0];
327
- }
328
-
329
- fprintf(stderr, "ggml_opencl: selecting platform: '%s'\n", default_device->platform->name);
330
- fprintf(stderr, "ggml_opencl: selecting device: '%s'\n", default_device->name);
331
- if (default_device->type != CL_DEVICE_TYPE_GPU) {
332
- fprintf(stderr, "ggml_opencl: warning, not a GPU: '%s'.\n", default_device->name);
333
- }
334
-
335
- platform = default_device->platform->id;
336
- device = default_device->id;
337
-
338
- cl_context_properties properties[] = {
339
- (intptr_t)CL_CONTEXT_PLATFORM, (intptr_t)platform, 0
340
- };
341
-
342
- CL_CHECK((context = clCreateContext(properties, 1, &device, NULL, NULL, &err), err));
343
-
344
- CL_CHECK((queue = clCreateCommandQueue(context, device, CL_QUEUE_OUT_OF_ORDER_EXEC_MODE_ENABLE, &err),
345
- (err != CL_INVALID_PROPERTY && err != CL_INVALID_VALUE ? err :
346
- (queue = clCreateCommandQueue(context, device, 0, &err), err)
347
- )));
348
-
349
- program = build_program_from_source(context, device, program_source);
350
-
351
- // Prepare dequantize kernels
352
- CL_CHECK((kernel_q4_0 = clCreateKernel(program, "dequantize_row_q4_0", &err), err));
353
- CL_CHECK((kernel_q4_1 = clCreateKernel(program, "dequantize_row_q4_1", &err), err));
354
- CL_CHECK((kernel_q5_0 = clCreateKernel(program, "dequantize_row_q5_0", &err), err));
355
- CL_CHECK((kernel_q5_1 = clCreateKernel(program, "dequantize_row_q5_1", &err), err));
356
- CL_CHECK((kernel_q8_0 = clCreateKernel(program, "dequantize_row_q8_0", &err), err));
357
- }
358
-
359
- static void ggml_cl_malloc(size_t req_size, size_t* cur_size, cl_mem_flags flags, cl_mem* buf) {
360
- if (req_size <= *cur_size) {
361
- return;
362
- }
363
-
364
- // Reallocate buffer with enough space
365
- if (*cur_size > 0) {
366
- clReleaseMemObject(*buf);
367
- }
368
- cl_int err;
369
- CL_CHECK((*buf = clCreateBuffer(context, flags, req_size, NULL, &err), err));
370
- *cur_size = req_size;
371
- }
372
-
373
- void ggml_cl_sgemm_wrapper(
374
- const enum ggml_blas_order order, const enum ggml_blas_op trans_a, const enum ggml_blas_op trans_b,
375
- const int m, const int n, const int k,
376
- const float alpha, const void *host_a, const int lda,
377
- const float *host_b, const int ldb, const float beta,
378
- float *host_c, const int ldc, const int btype) {
379
-
380
- cl_kernel kernel;
381
- size_t global = n * k, local, size_qb;
382
- bool dequant;
383
-
384
- switch (btype) {
385
- case GGML_TYPE_F32:
386
- dequant = false;
387
- break;
388
- case GGML_TYPE_Q4_0:
389
- dequant = true;
390
- kernel = kernel_q4_0;
391
- local = 16;
392
- size_qb = global * (sizeof(ggml_fp16_t) + local) / 32;
393
- break;
394
- case GGML_TYPE_Q4_1:
395
- dequant = true;
396
- kernel = kernel_q4_1;
397
- local = 16;
398
- size_qb = global * (sizeof(ggml_fp16_t) * 2 + local) / 32;
399
- break;
400
- case GGML_TYPE_Q5_0:
401
- dequant = true;
402
- kernel = kernel_q5_0;
403
- local = 16;
404
- size_qb = global * (sizeof(ggml_fp16_t) + sizeof(uint32_t) + local) / 32;
405
- break;
406
- case GGML_TYPE_Q5_1:
407
- dequant = true;
408
- kernel = kernel_q5_1;
409
- local = 16;
410
- size_qb = global * (sizeof(ggml_fp16_t) * 2 + sizeof(uint32_t) + local) / 32;
411
- break;
412
- case GGML_TYPE_Q8_0:
413
- dequant = true;
414
- kernel = kernel_q8_0;
415
- local = 32;
416
- size_qb = global * (sizeof(ggml_fp16_t) + local) / 32;
417
- break;
418
- default:
419
- fprintf(stderr, "Error: Unsupported OpenCL btype %d\n", btype);
420
- abort();
421
- }
422
-
423
- const size_t size_a = m * k * sizeof(float);
424
- const size_t size_b = n * k * sizeof(float);
425
- const size_t size_c = m * n * sizeof(float);
426
-
427
- // Prepare buffers
428
- ggml_cl_malloc(size_a, &cl_size_a, CL_MEM_READ_ONLY, &cl_buffer_a);
429
- if (dequant) {
430
- ggml_cl_malloc(size_qb, &cl_size_qb, CL_MEM_READ_ONLY, &cl_buffer_qb);
431
- }
432
- ggml_cl_malloc(size_b, &cl_size_b, CL_MEM_READ_WRITE, &cl_buffer_b);
433
- ggml_cl_malloc(size_c, &cl_size_c, CL_MEM_WRITE_ONLY, &cl_buffer_c);
434
-
435
- cl_event ev_a, ev_qb, ev_b;
436
-
437
- if (dequant) {
438
- CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &cl_buffer_qb));
439
- CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &cl_buffer_b));
440
- CL_CHECK(clEnqueueWriteBuffer(queue, cl_buffer_qb, CL_FALSE, 0, size_qb, host_b, 0, NULL, &ev_qb));
441
- } else {
442
- CL_CHECK(clEnqueueWriteBuffer(queue, cl_buffer_b, CL_FALSE, 0, size_b, host_b, 0, NULL, &ev_b));
443
- }
444
-
445
- CL_CHECK(clEnqueueWriteBuffer(queue, cl_buffer_a, CL_FALSE, 0, size_a, host_a, 0, NULL, &ev_a));
446
- if (dequant) {
447
- CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 1, NULL, &global, &local, 1, &ev_qb, &ev_b));
448
- CL_CHECK(clReleaseEvent(ev_qb));
449
- }
450
- CL_CHECK(clWaitForEvents(1, &ev_a));
451
- CL_CHECK(clWaitForEvents(1, &ev_b));
452
- CL_CHECK(clReleaseEvent(ev_a));
453
- CL_CHECK(clReleaseEvent(ev_b));
454
-
455
- cl_event ev_sgemm;
456
- CLBLAST_CHECK(CLBlastSgemm(
457
- (CLBlastLayout)order,
458
- (CLBlastTranspose)trans_a, (CLBlastTranspose)trans_b,
459
- m, n, k,
460
- alpha,
461
- cl_buffer_a, 0, lda,
462
- cl_buffer_b, 0, ldb,
463
- beta,
464
- cl_buffer_c, 0, ldc,
465
- &queue, &ev_sgemm));
466
-
467
- cl_event ev_c;
468
- CL_CHECK(clEnqueueReadBuffer(queue, cl_buffer_c, CL_TRUE, 0, size_c, host_c, 1, &ev_sgemm, &ev_c));
469
-
470
- // Wait for completion
471
- CL_CHECK(clWaitForEvents(1, &ev_c));
472
- CL_CHECK(clReleaseEvent(ev_sgemm));
473
- CL_CHECK(clReleaseEvent(ev_c));
474
- }