llama_cpp 0.16.0 → 0.16.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +6 -0
- data/ext/llama_cpp/extconf.rb +2 -0
- data/ext/llama_cpp/llama_cpp.cpp +2 -0
- data/lib/llama_cpp/version.rb +2 -2
- data/sig/llama_cpp.rbs +2 -0
- data/vendor/tmp/llama.cpp/Makefile +110 -53
- data/vendor/tmp/llama.cpp/ggml-alloc.c +78 -22
- data/vendor/tmp/llama.cpp/ggml-backend-impl.h +20 -8
- data/vendor/tmp/llama.cpp/ggml-backend.c +178 -64
- data/vendor/tmp/llama.cpp/ggml-backend.h +3 -3
- data/vendor/tmp/llama.cpp/ggml-blas.cpp +363 -0
- data/vendor/tmp/llama.cpp/ggml-blas.h +23 -0
- data/vendor/tmp/llama.cpp/ggml-common.h +6 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/argsort.cu +1 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/dmmv.cu +21 -9
- data/vendor/tmp/llama.cpp/ggml-cuda/fattn-tile-f16.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/mmq.cu +15 -1491
- data/vendor/tmp/llama.cpp/ggml-cuda/mmvq.cu +76 -61
- data/vendor/tmp/llama.cpp/ggml-cuda/quantize.cu +77 -10
- data/vendor/tmp/llama.cpp/ggml-cuda/softmax.cu +1 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-f16.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q4_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q4_1.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q5_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q5_1.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q8_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-f16.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q4_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q4_1.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q5_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q5_1.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q8_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-f16.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q4_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q4_1.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q5_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q5_1.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q8_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-f16.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q4_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q4_1.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q5_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q5_1.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q8_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-f16.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q4_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q4_1.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q5_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q5_1.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q8_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-f16.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q4_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q4_1.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q5_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q5_1.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q8_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs256-f16-f16.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-f16.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q4_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q4_1.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q5_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q5_1.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q8_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-f16.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q4_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q4_1.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q5_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q5_1.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q8_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-f16.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q4_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q4_1.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q5_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q5_1.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q8_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-f16.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q4_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q4_1.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q5_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q5_1.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q8_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-f16.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q4_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q4_1.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q5_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q5_1.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q8_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-f16.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q4_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q4_1.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q5_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q5_1.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q8_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-f16.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q4_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q4_1.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q5_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q5_1.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q8_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs256-f16-f16.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-f16.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q4_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q4_1.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q5_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q5_1.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q8_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqfloat-cpb16.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqfloat-cpb32.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqhalf-cpb16.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqhalf-cpb32.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqhalf-cpb8.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q2_k.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q3_k.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q4_0.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q4_1.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q4_k.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q5_0.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q5_1.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q5_k.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q6_k.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q8_0.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/unary.cu +20 -0
- data/vendor/tmp/llama.cpp/ggml-cuda.cu +95 -129
- data/vendor/tmp/llama.cpp/ggml-kompute.cpp +8 -7
- data/vendor/tmp/llama.cpp/ggml-metal.m +11 -9
- data/vendor/tmp/llama.cpp/ggml-rpc.cpp +13 -12
- data/vendor/tmp/llama.cpp/ggml-sycl.cpp +19 -23
- data/vendor/tmp/llama.cpp/ggml-vulkan-shaders.hpp +1230 -1129
- data/vendor/tmp/llama.cpp/ggml-vulkan.cpp +181 -148
- data/vendor/tmp/llama.cpp/ggml.c +102 -275
- data/vendor/tmp/llama.cpp/llama.cpp +103 -47
- data/vendor/tmp/llama.cpp/llama.h +4 -0
- metadata +15 -3
|
@@ -148,6 +148,8 @@ void ggml_cuda_op_gelu(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
|
|
148
148
|
float * dst_d = (float *)dst->data;
|
|
149
149
|
cudaStream_t stream = ctx.stream();
|
|
150
150
|
|
|
151
|
+
GGML_ASSERT(ggml_is_contiguous(src0));
|
|
152
|
+
|
|
151
153
|
GGML_ASSERT(src0->type == GGML_TYPE_F32);
|
|
152
154
|
GGML_ASSERT( dst->type == GGML_TYPE_F32);
|
|
153
155
|
|
|
@@ -160,6 +162,8 @@ void ggml_cuda_op_silu(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
|
|
160
162
|
float * dst_d = (float *)dst->data;
|
|
161
163
|
cudaStream_t stream = ctx.stream();
|
|
162
164
|
|
|
165
|
+
GGML_ASSERT(ggml_is_contiguous(src0));
|
|
166
|
+
|
|
163
167
|
GGML_ASSERT(src0->type == GGML_TYPE_F32);
|
|
164
168
|
GGML_ASSERT( dst->type == GGML_TYPE_F32);
|
|
165
169
|
|
|
@@ -172,6 +176,8 @@ void ggml_cuda_op_gelu_quick(ggml_backend_cuda_context & ctx, ggml_tensor * dst)
|
|
|
172
176
|
float * dst_d = (float *)dst->data;
|
|
173
177
|
cudaStream_t stream = ctx.stream();
|
|
174
178
|
|
|
179
|
+
GGML_ASSERT(ggml_is_contiguous(src0));
|
|
180
|
+
|
|
175
181
|
GGML_ASSERT(src0->type == GGML_TYPE_F32);
|
|
176
182
|
GGML_ASSERT( dst->type == GGML_TYPE_F32);
|
|
177
183
|
|
|
@@ -184,6 +190,8 @@ void ggml_cuda_op_tanh(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
|
|
184
190
|
float * dst_d = (float *)dst->data;
|
|
185
191
|
cudaStream_t stream = ctx.stream();
|
|
186
192
|
|
|
193
|
+
GGML_ASSERT(ggml_is_contiguous(src0));
|
|
194
|
+
|
|
187
195
|
GGML_ASSERT(src0->type == GGML_TYPE_F32);
|
|
188
196
|
GGML_ASSERT( dst->type == GGML_TYPE_F32);
|
|
189
197
|
|
|
@@ -196,6 +204,8 @@ void ggml_cuda_op_relu(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
|
|
196
204
|
float * dst_d = (float *)dst->data;
|
|
197
205
|
cudaStream_t stream = ctx.stream();
|
|
198
206
|
|
|
207
|
+
GGML_ASSERT(ggml_is_contiguous(src0));
|
|
208
|
+
|
|
199
209
|
GGML_ASSERT(src0->type == GGML_TYPE_F32);
|
|
200
210
|
GGML_ASSERT( dst->type == GGML_TYPE_F32);
|
|
201
211
|
|
|
@@ -208,6 +218,8 @@ void ggml_cuda_op_sigmoid(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
|
|
208
218
|
float * dst_d = (float *)dst->data;
|
|
209
219
|
cudaStream_t stream = ctx.stream();
|
|
210
220
|
|
|
221
|
+
GGML_ASSERT(ggml_is_contiguous(src0));
|
|
222
|
+
|
|
211
223
|
GGML_ASSERT(src0->type == GGML_TYPE_F32);
|
|
212
224
|
GGML_ASSERT( dst->type == GGML_TYPE_F32);
|
|
213
225
|
|
|
@@ -220,6 +232,8 @@ void ggml_cuda_op_hardsigmoid(ggml_backend_cuda_context & ctx, ggml_tensor * dst
|
|
|
220
232
|
float * dst_d = (float *)dst->data;
|
|
221
233
|
cudaStream_t stream = ctx.stream();
|
|
222
234
|
|
|
235
|
+
GGML_ASSERT(ggml_is_contiguous(src0));
|
|
236
|
+
|
|
223
237
|
GGML_ASSERT(src0->type == GGML_TYPE_F32);
|
|
224
238
|
GGML_ASSERT( dst->type == GGML_TYPE_F32);
|
|
225
239
|
|
|
@@ -232,6 +246,8 @@ void ggml_cuda_op_hardswish(ggml_backend_cuda_context & ctx, ggml_tensor * dst)
|
|
|
232
246
|
float * dst_d = (float *)dst->data;
|
|
233
247
|
cudaStream_t stream = ctx.stream();
|
|
234
248
|
|
|
249
|
+
GGML_ASSERT(ggml_is_contiguous(src0));
|
|
250
|
+
|
|
235
251
|
GGML_ASSERT(src0->type == GGML_TYPE_F32);
|
|
236
252
|
GGML_ASSERT( dst->type == GGML_TYPE_F32);
|
|
237
253
|
|
|
@@ -244,6 +260,8 @@ void ggml_cuda_op_leaky_relu(ggml_backend_cuda_context & ctx, ggml_tensor * dst)
|
|
|
244
260
|
float * dst_d = (float *)dst->data;
|
|
245
261
|
cudaStream_t stream = ctx.stream();
|
|
246
262
|
|
|
263
|
+
GGML_ASSERT(ggml_is_contiguous(src0));
|
|
264
|
+
|
|
247
265
|
GGML_ASSERT(src0->type == GGML_TYPE_F32);
|
|
248
266
|
GGML_ASSERT( dst->type == GGML_TYPE_F32);
|
|
249
267
|
|
|
@@ -259,6 +277,8 @@ void ggml_cuda_op_sqr(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
|
|
259
277
|
float * dst_d = (float *)dst->data;
|
|
260
278
|
cudaStream_t stream = ctx.stream();
|
|
261
279
|
|
|
280
|
+
GGML_ASSERT(ggml_is_contiguous(src0));
|
|
281
|
+
|
|
262
282
|
GGML_ASSERT(src0->type == GGML_TYPE_F32);
|
|
263
283
|
GGML_ASSERT( dst->type == GGML_TYPE_F32);
|
|
264
284
|
|
|
@@ -188,13 +188,15 @@ static ggml_cuda_device_info ggml_cuda_init() {
|
|
|
188
188
|
info.default_tensor_split[id] = total_vram;
|
|
189
189
|
total_vram += prop.totalGlobalMem;
|
|
190
190
|
|
|
191
|
+
info.devices[id].nsm = prop.multiProcessorCount;
|
|
192
|
+
info.devices[id].smpb = prop.sharedMemPerBlock;
|
|
191
193
|
#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)
|
|
194
|
+
info.devices[id].smpbo = prop.sharedMemPerBlock;
|
|
192
195
|
info.devices[id].cc = 100*prop.major + 10*prop.minor + CC_OFFSET_AMD;
|
|
193
196
|
#else
|
|
197
|
+
info.devices[id].smpbo = prop.sharedMemPerBlockOptin;
|
|
194
198
|
info.devices[id].cc = 100*prop.major + 10*prop.minor;
|
|
195
199
|
#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)
|
|
196
|
-
info.devices[id].smpb = prop.sharedMemPerBlock;
|
|
197
|
-
info.devices[id].nsm = prop.multiProcessorCount;
|
|
198
200
|
}
|
|
199
201
|
|
|
200
202
|
for (int id = 0; id < info.device_count; ++id) {
|
|
@@ -543,6 +545,10 @@ GGML_CALL static const char * ggml_backend_cuda_buffer_type_name(ggml_backend_bu
|
|
|
543
545
|
return ctx->name.c_str();
|
|
544
546
|
}
|
|
545
547
|
|
|
548
|
+
static bool ggml_backend_buft_is_cuda(ggml_backend_buffer_type_t buft) {
|
|
549
|
+
return buft->iface.get_name == ggml_backend_cuda_buffer_type_name;
|
|
550
|
+
}
|
|
551
|
+
|
|
546
552
|
GGML_CALL static ggml_backend_buffer_t ggml_backend_cuda_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {
|
|
547
553
|
ggml_backend_cuda_buffer_type_context * buft_ctx = (ggml_backend_cuda_buffer_type_context *)buft->context;
|
|
548
554
|
|
|
@@ -585,24 +591,12 @@ GGML_CALL static size_t ggml_backend_cuda_buffer_type_get_alloc_size(ggml_backen
|
|
|
585
591
|
GGML_UNUSED(buft);
|
|
586
592
|
}
|
|
587
593
|
|
|
588
|
-
GGML_CALL static bool ggml_backend_cuda_buffer_type_supports_backend(ggml_backend_buffer_type_t buft, ggml_backend_t backend) {
|
|
589
|
-
if (!ggml_backend_is_cuda(backend)) {
|
|
590
|
-
return false;
|
|
591
|
-
}
|
|
592
|
-
|
|
593
|
-
ggml_backend_cuda_buffer_type_context * buft_ctx = (ggml_backend_cuda_buffer_type_context *)buft->context;
|
|
594
|
-
ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context;
|
|
595
|
-
|
|
596
|
-
return buft_ctx->device == cuda_ctx->device;
|
|
597
|
-
}
|
|
598
|
-
|
|
599
594
|
static ggml_backend_buffer_type_i ggml_backend_cuda_buffer_type_interface = {
|
|
600
595
|
/* .get_name = */ ggml_backend_cuda_buffer_type_name,
|
|
601
596
|
/* .alloc_buffer = */ ggml_backend_cuda_buffer_type_alloc_buffer,
|
|
602
597
|
/* .get_alignment = */ ggml_backend_cuda_buffer_type_get_alignment,
|
|
603
598
|
/* .get_max_size = */ NULL, // defaults to SIZE_MAX
|
|
604
599
|
/* .get_alloc_size = */ ggml_backend_cuda_buffer_type_get_alloc_size,
|
|
605
|
-
/* .supports_backend = */ ggml_backend_cuda_buffer_type_supports_backend,
|
|
606
600
|
/* .is_host = */ NULL,
|
|
607
601
|
};
|
|
608
602
|
|
|
@@ -633,88 +627,22 @@ GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_buffer_type(int device) {
|
|
|
633
627
|
|
|
634
628
|
// cuda split buffer
|
|
635
629
|
|
|
636
|
-
static int64_t get_row_rounding(
|
|
637
|
-
int64_t
|
|
638
|
-
int64_t max_compute_capability = INT_MIN;
|
|
630
|
+
static int64_t get_row_rounding(const std::array<float, GGML_CUDA_MAX_DEVICES> & tensor_split) {
|
|
631
|
+
int64_t row_rounding = 0;
|
|
639
632
|
for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) {
|
|
640
|
-
if (tensor_split[id]
|
|
641
|
-
|
|
642
|
-
min_compute_capability = ggml_cuda_info().devices[id].cc;
|
|
643
|
-
}
|
|
644
|
-
if (max_compute_capability < ggml_cuda_info().devices[id].cc) {
|
|
645
|
-
max_compute_capability = ggml_cuda_info().devices[id].cc;
|
|
646
|
-
}
|
|
633
|
+
if (tensor_split[id] >= (id + 1 < ggml_backend_cuda_get_device_count() ? tensor_split[id + 1] : 1.0f)) {
|
|
634
|
+
continue;
|
|
647
635
|
}
|
|
648
|
-
}
|
|
649
636
|
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
case GGML_TYPE_Q4_0:
|
|
653
|
-
case GGML_TYPE_Q4_1:
|
|
654
|
-
case GGML_TYPE_Q5_0:
|
|
655
|
-
case GGML_TYPE_Q5_1:
|
|
656
|
-
case GGML_TYPE_Q8_0:
|
|
657
|
-
return max_compute_capability >= CC_RDNA2 ? 128 : 64;
|
|
658
|
-
case GGML_TYPE_F16:
|
|
659
|
-
case GGML_TYPE_F32:
|
|
660
|
-
return 1;
|
|
661
|
-
case GGML_TYPE_Q2_K:
|
|
662
|
-
return max_compute_capability >= CC_RDNA2 ? 128 : 32;
|
|
663
|
-
case GGML_TYPE_Q3_K:
|
|
664
|
-
return min_compute_capability < CC_RDNA2 ? 128 : 64;
|
|
665
|
-
case GGML_TYPE_Q4_K:
|
|
666
|
-
case GGML_TYPE_Q5_K:
|
|
667
|
-
case GGML_TYPE_Q6_K:
|
|
668
|
-
case GGML_TYPE_IQ2_XXS:
|
|
669
|
-
case GGML_TYPE_IQ2_XS:
|
|
670
|
-
case GGML_TYPE_IQ2_S:
|
|
671
|
-
case GGML_TYPE_IQ3_XXS:
|
|
672
|
-
case GGML_TYPE_IQ1_S:
|
|
673
|
-
case GGML_TYPE_IQ1_M:
|
|
674
|
-
case GGML_TYPE_IQ4_NL:
|
|
675
|
-
case GGML_TYPE_IQ4_XS:
|
|
676
|
-
case GGML_TYPE_IQ3_S:
|
|
677
|
-
return max_compute_capability >= CC_RDNA2 ? 128 : 64;
|
|
678
|
-
default:
|
|
679
|
-
GGML_ASSERT(false);
|
|
637
|
+
const int cc = ggml_cuda_info().devices[id].cc;
|
|
638
|
+
row_rounding = std::max(row_rounding, (int64_t)get_mmq_y_host(cc, get_mmq_x_max_host(cc)));
|
|
680
639
|
}
|
|
681
|
-
|
|
682
|
-
switch(type) {
|
|
683
|
-
case GGML_TYPE_Q4_0:
|
|
684
|
-
case GGML_TYPE_Q4_1:
|
|
685
|
-
return max_compute_capability >= CC_VOLTA ? 128 : 64;
|
|
686
|
-
case GGML_TYPE_Q5_0:
|
|
687
|
-
case GGML_TYPE_Q5_1:
|
|
688
|
-
case GGML_TYPE_Q8_0:
|
|
689
|
-
return 64;
|
|
690
|
-
case GGML_TYPE_F16:
|
|
691
|
-
case GGML_TYPE_F32:
|
|
692
|
-
return 1;
|
|
693
|
-
case GGML_TYPE_Q2_K:
|
|
694
|
-
case GGML_TYPE_Q3_K:
|
|
695
|
-
case GGML_TYPE_Q4_K:
|
|
696
|
-
case GGML_TYPE_Q5_K:
|
|
697
|
-
case GGML_TYPE_IQ2_XXS:
|
|
698
|
-
case GGML_TYPE_IQ2_XS:
|
|
699
|
-
case GGML_TYPE_IQ2_S:
|
|
700
|
-
case GGML_TYPE_IQ3_XXS:
|
|
701
|
-
case GGML_TYPE_IQ1_S:
|
|
702
|
-
case GGML_TYPE_IQ1_M:
|
|
703
|
-
case GGML_TYPE_IQ4_NL:
|
|
704
|
-
case GGML_TYPE_IQ4_XS:
|
|
705
|
-
case GGML_TYPE_IQ3_S:
|
|
706
|
-
return max_compute_capability >= CC_VOLTA ? 128 : 64;
|
|
707
|
-
case GGML_TYPE_Q6_K:
|
|
708
|
-
return 64;
|
|
709
|
-
default:
|
|
710
|
-
GGML_ASSERT(false);
|
|
711
|
-
}
|
|
712
|
-
#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)
|
|
640
|
+
return row_rounding;
|
|
713
641
|
}
|
|
714
642
|
|
|
715
643
|
static void get_row_split(int64_t * row_low, int64_t * row_high, const ggml_tensor * tensor, const std::array<float, GGML_CUDA_MAX_DEVICES> & tensor_split, int id) {
|
|
716
644
|
const int64_t nrows = ggml_nrows(tensor);
|
|
717
|
-
const int64_t rounding = get_row_rounding(
|
|
645
|
+
const int64_t rounding = get_row_rounding(tensor_split);
|
|
718
646
|
|
|
719
647
|
*row_low = id == 0 ? 0 : nrows*tensor_split[id];
|
|
720
648
|
*row_low -= *row_low % rounding;
|
|
@@ -929,6 +857,10 @@ GGML_CALL static const char * ggml_backend_cuda_split_buffer_type_name(ggml_back
|
|
|
929
857
|
GGML_UNUSED(buft);
|
|
930
858
|
}
|
|
931
859
|
|
|
860
|
+
static bool ggml_backend_buft_is_cuda_split(ggml_backend_buffer_type_t buft) {
|
|
861
|
+
return buft->iface.get_name == ggml_backend_cuda_split_buffer_type_name;
|
|
862
|
+
}
|
|
863
|
+
|
|
932
864
|
GGML_CALL static ggml_backend_buffer_t ggml_backend_cuda_split_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {
|
|
933
865
|
// since we don't know the exact split after rounding, we cannot allocate the device buffers at this point
|
|
934
866
|
// instead, we allocate them for each tensor separately in init_tensor
|
|
@@ -972,12 +904,6 @@ GGML_CALL static size_t ggml_backend_cuda_split_buffer_type_get_alloc_size(ggml_
|
|
|
972
904
|
return total_size;
|
|
973
905
|
}
|
|
974
906
|
|
|
975
|
-
GGML_CALL static bool ggml_backend_cuda_split_buffer_type_supports_backend(ggml_backend_buffer_type_t buft, ggml_backend_t backend) {
|
|
976
|
-
return ggml_backend_is_cuda(backend);
|
|
977
|
-
|
|
978
|
-
GGML_UNUSED(buft);
|
|
979
|
-
}
|
|
980
|
-
|
|
981
907
|
GGML_CALL static bool ggml_backend_cuda_split_buffer_type_is_host(ggml_backend_buffer_type_t buft) {
|
|
982
908
|
return false;
|
|
983
909
|
|
|
@@ -990,7 +916,6 @@ static ggml_backend_buffer_type_i ggml_backend_cuda_split_buffer_type_interface
|
|
|
990
916
|
/* .get_alignment = */ ggml_backend_cuda_split_buffer_type_get_alignment,
|
|
991
917
|
/* .get_max_size = */ NULL, // defaults to SIZE_MAX
|
|
992
918
|
/* .get_alloc_size = */ ggml_backend_cuda_split_buffer_type_get_alloc_size,
|
|
993
|
-
/* .supports_backend = */ ggml_backend_cuda_split_buffer_type_supports_backend,
|
|
994
919
|
/* .is_host = */ ggml_backend_cuda_split_buffer_type_is_host,
|
|
995
920
|
};
|
|
996
921
|
|
|
@@ -1090,7 +1015,6 @@ GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_host_buffer_type() {
|
|
|
1090
1015
|
/* .get_alignment = */ ggml_backend_cpu_buffer_type()->iface.get_alignment,
|
|
1091
1016
|
/* .get_max_size = */ NULL, // defaults to SIZE_MAX
|
|
1092
1017
|
/* .get_alloc_size = */ ggml_backend_cpu_buffer_type()->iface.get_alloc_size,
|
|
1093
|
-
/* .supports_backend = */ ggml_backend_cpu_buffer_type()->iface.supports_backend,
|
|
1094
1018
|
/* .is_host = */ ggml_backend_cpu_buffer_type()->iface.is_host,
|
|
1095
1019
|
},
|
|
1096
1020
|
/* .context = */ nullptr,
|
|
@@ -1413,10 +1337,30 @@ static void ggml_cuda_set_peer_access(const int n_tokens, int main_device) {
|
|
|
1413
1337
|
GGML_UNUSED(main_device);
|
|
1414
1338
|
}
|
|
1415
1339
|
|
|
1340
|
+
static cudaError_t ggml_cuda_Memcpy2DPeerAsync(
|
|
1341
|
+
void * dst, int dstDevice, size_t dpitch, void * src, int srcDevice, size_t spitch, size_t width, size_t height, cudaStream_t stream) {
|
|
1342
|
+
|
|
1343
|
+
#if !defined(GGML_USE_HIPBLAS)
|
|
1344
|
+
// cudaMemcpy2DAsync may fail with copies between vmm pools of different devices
|
|
1345
|
+
cudaMemcpy3DPeerParms p = {};
|
|
1346
|
+
p.dstDevice = dstDevice;
|
|
1347
|
+
p.dstPtr = make_cudaPitchedPtr(dst, dpitch, dpitch, height);
|
|
1348
|
+
p.srcDevice = srcDevice;
|
|
1349
|
+
p.srcPtr = make_cudaPitchedPtr(src, spitch, spitch, height);
|
|
1350
|
+
p.extent = make_cudaExtent(width, height, 1);
|
|
1351
|
+
return cudaMemcpy3DPeerAsync(&p, stream);
|
|
1352
|
+
#else
|
|
1353
|
+
// HIP does not support cudaMemcpy3DPeerAsync or vmm pools
|
|
1354
|
+
GGML_UNUSED(dstDevice);
|
|
1355
|
+
GGML_UNUSED(srcDevice);
|
|
1356
|
+
return cudaMemcpy2DAsync(dst, dpitch, src, spitch, width, height, cudaMemcpyDeviceToDevice, stream);
|
|
1357
|
+
#endif // !defined(GGML_USE_HIPBLAS)
|
|
1358
|
+
}
|
|
1359
|
+
|
|
1416
1360
|
static void ggml_cuda_op_mul_mat(
|
|
1417
1361
|
ggml_backend_cuda_context & ctx,
|
|
1418
1362
|
const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, ggml_cuda_op_mul_mat_t op,
|
|
1419
|
-
|
|
1363
|
+
quantize_cuda_t quantize_src1) {
|
|
1420
1364
|
|
|
1421
1365
|
const int64_t ne00 = src0->ne[0];
|
|
1422
1366
|
const int64_t ne01 = src0->ne[1];
|
|
@@ -1473,7 +1417,9 @@ static void ggml_cuda_op_mul_mat(
|
|
|
1473
1417
|
}
|
|
1474
1418
|
|
|
1475
1419
|
struct dev_data {
|
|
1476
|
-
|
|
1420
|
+
int cc;
|
|
1421
|
+
|
|
1422
|
+
ggml_cuda_pool_alloc<char> src0_dd_alloc;
|
|
1477
1423
|
ggml_cuda_pool_alloc<float> src1_ddf_alloc;
|
|
1478
1424
|
ggml_cuda_pool_alloc<char> src1_ddq_alloc;
|
|
1479
1425
|
ggml_cuda_pool_alloc<float> dst_dd_alloc;
|
|
@@ -1492,6 +1438,8 @@ static void ggml_cuda_op_mul_mat(
|
|
|
1492
1438
|
int used_devices = 0;
|
|
1493
1439
|
|
|
1494
1440
|
for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) {
|
|
1441
|
+
dev[id].cc = ggml_cuda_info().devices[id].cc;
|
|
1442
|
+
|
|
1495
1443
|
// by default, use all rows
|
|
1496
1444
|
dev[id].row_low = 0;
|
|
1497
1445
|
dev[id].row_high = ne01;
|
|
@@ -1499,7 +1447,7 @@ static void ggml_cuda_op_mul_mat(
|
|
|
1499
1447
|
// for multi GPU, get the row boundaries from tensor split
|
|
1500
1448
|
// and round to mul_mat_q tile sizes
|
|
1501
1449
|
if (split) {
|
|
1502
|
-
const int64_t rounding = get_row_rounding(
|
|
1450
|
+
const int64_t rounding = get_row_rounding(tensor_split);
|
|
1503
1451
|
|
|
1504
1452
|
if (id != 0) {
|
|
1505
1453
|
dev[id].row_low = ne01*tensor_split[id];
|
|
@@ -1542,11 +1490,15 @@ static void ggml_cuda_op_mul_mat(
|
|
|
1542
1490
|
dev[id].src1_ddf = dev[id].src1_ddf_alloc.alloc(ctx.pool(id), ggml_nelements(src1));
|
|
1543
1491
|
}
|
|
1544
1492
|
|
|
1545
|
-
if (
|
|
1546
|
-
|
|
1493
|
+
if (quantize_src1) {
|
|
1494
|
+
size_t src_1_ddq_size = nrows1*src1_padded_col_size*q8_1_ts/q8_1_bs;
|
|
1495
|
+
if (quantize_src1 == quantize_mmq_q8_1_cuda) {
|
|
1496
|
+
src_1_ddq_size += get_mmq_x_max_host(dev[id].cc)*sizeof(block_q8_1_mmq);
|
|
1497
|
+
}
|
|
1498
|
+
dev[id].src1_ddq = dev[id].src1_ddq_alloc.alloc(ctx.pool(id), src_1_ddq_size);
|
|
1547
1499
|
|
|
1548
1500
|
if (src1_on_device && src1_is_contiguous) {
|
|
1549
|
-
|
|
1501
|
+
quantize_src1(dev[id].src1_ddf, dev[id].src1_ddq, ne10, ne11, ne12*ne13, src1_padded_col_size, src0->type, stream);
|
|
1550
1502
|
CUDA_CHECK(cudaGetLastError());
|
|
1551
1503
|
}
|
|
1552
1504
|
}
|
|
@@ -1592,7 +1544,12 @@ static void ggml_cuda_op_mul_mat(
|
|
|
1592
1544
|
const int64_t i03 = i0 / ne12;
|
|
1593
1545
|
const int64_t i02 = i0 % ne12;
|
|
1594
1546
|
|
|
1595
|
-
|
|
1547
|
+
size_t src1_ddq_i_offset = i0*ne11 * src1_padded_col_size*q8_1_ts/q8_1_bs;
|
|
1548
|
+
if (quantize_src1 == quantize_mmq_q8_1_cuda) {
|
|
1549
|
+
src1_ddq_i_offset += src1_col_0 * sizeof(block_q8_1_mmq);
|
|
1550
|
+
} else {
|
|
1551
|
+
src1_ddq_i_offset += src1_col_0 * src1_padded_col_size*q8_1_ts/q8_1_bs;
|
|
1552
|
+
}
|
|
1596
1553
|
|
|
1597
1554
|
// for split tensors the data begins at i0 == i0_offset_low
|
|
1598
1555
|
char * src0_dd_i = dev[id].src0_dd + (i0/i02_divisor) * (ne01*ne00*src0_ts)/src0_bs;
|
|
@@ -1609,10 +1566,17 @@ static void ggml_cuda_op_mul_mat(
|
|
|
1609
1566
|
// copy src0, src1 to device if necessary
|
|
1610
1567
|
if (src1_is_contiguous) {
|
|
1611
1568
|
if (id != ctx.device) {
|
|
1612
|
-
if (
|
|
1569
|
+
if (quantize_src1) {
|
|
1613
1570
|
char * src1_ddq_i_source = dev[ctx.device].src1_ddq + src1_ddq_i_offset;
|
|
1614
|
-
|
|
1615
|
-
|
|
1571
|
+
if (quantize_src1 == quantize_mmq_q8_1_cuda) {
|
|
1572
|
+
const size_t pitch = ne11*sizeof(block_q8_1_mmq);
|
|
1573
|
+
const size_t width = src1_ncols*sizeof(block_q8_1_mmq);
|
|
1574
|
+
const size_t height = src1_padded_col_size/(4*QK8_1);
|
|
1575
|
+
CUDA_CHECK(ggml_cuda_Memcpy2DPeerAsync(src1_ddq_i, id, pitch, src1_ddq_i_source, ctx.device, pitch, width, height, stream));
|
|
1576
|
+
} else {
|
|
1577
|
+
CUDA_CHECK(cudaMemcpyPeerAsync(
|
|
1578
|
+
src1_ddq_i, id, src1_ddq_i_source, ctx.device, src1_ncols*src1_padded_col_size*q8_1_ts/q8_1_bs, stream));
|
|
1579
|
+
}
|
|
1616
1580
|
} else {
|
|
1617
1581
|
float * src1_ddf_i_source = (float *) src1->data;
|
|
1618
1582
|
src1_ddf_i_source += (i0*ne11 + src1_col_0) * ne10;
|
|
@@ -1627,8 +1591,8 @@ static void ggml_cuda_op_mul_mat(
|
|
|
1627
1591
|
GGML_ASSERT(false);
|
|
1628
1592
|
}
|
|
1629
1593
|
|
|
1630
|
-
if (
|
|
1631
|
-
|
|
1594
|
+
if (quantize_src1 && !src1_is_contiguous) {
|
|
1595
|
+
quantize_src1(src1_ddf_i, src1_ddq_i, ne10, src1_ncols, 1, src1_padded_col_size, src0->type, stream);
|
|
1632
1596
|
CUDA_CHECK(cudaGetLastError());
|
|
1633
1597
|
}
|
|
1634
1598
|
|
|
@@ -1653,22 +1617,8 @@ static void ggml_cuda_op_mul_mat(
|
|
|
1653
1617
|
float * dhf_dst_i = (float *) ((char *) dst_off_device + i02*nb2 + i03*nb3);
|
|
1654
1618
|
GGML_ASSERT(dst->nb[1] == ne0*sizeof(float));
|
|
1655
1619
|
dhf_dst_i += src1_col_0*ne0 + dev[id].row_low;
|
|
1656
|
-
|
|
1657
|
-
|
|
1658
|
-
cudaMemcpy3DPeerParms p = {};
|
|
1659
|
-
p.dstDevice = ctx.device;
|
|
1660
|
-
p.dstPtr = make_cudaPitchedPtr(dhf_dst_i, ne0*sizeof(float), row_diff, src1_ncols);
|
|
1661
|
-
p.srcDevice = id;
|
|
1662
|
-
p.srcPtr = make_cudaPitchedPtr(dst_dd_i, row_diff*sizeof(float), row_diff, src1_ncols);
|
|
1663
|
-
p.extent = make_cudaExtent(row_diff*sizeof(float), src1_ncols, 1);
|
|
1664
|
-
CUDA_CHECK(cudaMemcpy3DPeerAsync(&p, stream));
|
|
1665
|
-
#else
|
|
1666
|
-
// HIP does not support cudaMemcpy3DPeerAsync or vmm pools
|
|
1667
|
-
CUDA_CHECK(cudaMemcpy2DAsync(dhf_dst_i, ne0*sizeof(float),
|
|
1668
|
-
dst_dd_i, row_diff*sizeof(float),
|
|
1669
|
-
row_diff*sizeof(float), src1_ncols,
|
|
1670
|
-
cudaMemcpyDeviceToDevice, stream));
|
|
1671
|
-
#endif
|
|
1620
|
+
CUDA_CHECK(ggml_cuda_Memcpy2DPeerAsync(
|
|
1621
|
+
dhf_dst_i, ctx.device, ne0*sizeof(float), dst_dd_i, id, row_diff*sizeof(float), row_diff*sizeof(float), src1_ncols, stream));
|
|
1672
1622
|
} else {
|
|
1673
1623
|
float * dhf_dst_i = (float *) ((char *) dst_off_device + i02*nb2 + i03*nb3);
|
|
1674
1624
|
GGML_ASSERT(dst->nb[1] == ne0*sizeof(float));
|
|
@@ -2007,13 +1957,13 @@ static void ggml_cuda_mul_mat(ggml_backend_cuda_context & ctx, const ggml_tensor
|
|
|
2007
1957
|
// KQ + KQV multi-batch
|
|
2008
1958
|
ggml_cuda_mul_mat_batched_cublas(ctx, src0, src1, dst);
|
|
2009
1959
|
} else if (use_dequantize_mul_mat_vec) {
|
|
2010
|
-
ggml_cuda_op_mul_mat(ctx, src0, src1, dst, ggml_cuda_op_dequantize_mul_mat_vec,
|
|
1960
|
+
ggml_cuda_op_mul_mat(ctx, src0, src1, dst, ggml_cuda_op_dequantize_mul_mat_vec, nullptr);
|
|
2011
1961
|
} else if (use_mul_mat_vec_q) {
|
|
2012
|
-
ggml_cuda_op_mul_mat(ctx, src0, src1, dst, ggml_cuda_op_mul_mat_vec_q,
|
|
1962
|
+
ggml_cuda_op_mul_mat(ctx, src0, src1, dst, ggml_cuda_op_mul_mat_vec_q, quantize_row_q8_1_cuda);
|
|
2013
1963
|
} else if (use_mul_mat_q) {
|
|
2014
|
-
ggml_cuda_op_mul_mat(ctx, src0, src1, dst, ggml_cuda_op_mul_mat_q,
|
|
1964
|
+
ggml_cuda_op_mul_mat(ctx, src0, src1, dst, ggml_cuda_op_mul_mat_q, quantize_mmq_q8_1_cuda);
|
|
2015
1965
|
} else {
|
|
2016
|
-
ggml_cuda_op_mul_mat(ctx, src0, src1, dst, ggml_cuda_op_mul_mat_cublas,
|
|
1966
|
+
ggml_cuda_op_mul_mat(ctx, src0, src1, dst, ggml_cuda_op_mul_mat_cublas, nullptr);
|
|
2017
1967
|
}
|
|
2018
1968
|
}
|
|
2019
1969
|
|
|
@@ -2780,7 +2730,7 @@ GGML_CALL static bool ggml_backend_cuda_supports_op(ggml_backend_t backend, cons
|
|
|
2780
2730
|
case GGML_UNARY_OP_HARDSWISH:
|
|
2781
2731
|
case GGML_UNARY_OP_GELU_QUICK:
|
|
2782
2732
|
case GGML_UNARY_OP_TANH:
|
|
2783
|
-
return
|
|
2733
|
+
return ggml_is_contiguous(op->src[0]);
|
|
2784
2734
|
default:
|
|
2785
2735
|
return false;
|
|
2786
2736
|
}
|
|
@@ -2919,6 +2869,20 @@ GGML_CALL static bool ggml_backend_cuda_supports_op(ggml_backend_t backend, cons
|
|
|
2919
2869
|
GGML_UNUSED(backend);
|
|
2920
2870
|
}
|
|
2921
2871
|
|
|
2872
|
+
GGML_CALL static bool ggml_backend_cuda_supports_buft(ggml_backend_t backend, ggml_backend_buffer_type_t buft) {
|
|
2873
|
+
if (ggml_backend_buft_is_cuda_split(buft)) {
|
|
2874
|
+
return true;
|
|
2875
|
+
}
|
|
2876
|
+
|
|
2877
|
+
if (ggml_backend_buft_is_cuda(buft)) {
|
|
2878
|
+
ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context;
|
|
2879
|
+
ggml_backend_cuda_buffer_type_context * buft_ctx = (ggml_backend_cuda_buffer_type_context *)buft->context;
|
|
2880
|
+
return buft_ctx->device == cuda_ctx->device;
|
|
2881
|
+
}
|
|
2882
|
+
|
|
2883
|
+
return false;
|
|
2884
|
+
}
|
|
2885
|
+
|
|
2922
2886
|
GGML_CALL static bool ggml_backend_cuda_offload_op(ggml_backend_t backend, const ggml_tensor * op) {
|
|
2923
2887
|
const int min_batch_size = 32;
|
|
2924
2888
|
|
|
@@ -2991,9 +2955,11 @@ static ggml_backend_i ggml_backend_cuda_interface = {
|
|
|
2991
2955
|
/* .synchronize = */ ggml_backend_cuda_synchronize,
|
|
2992
2956
|
/* .graph_plan_create = */ NULL,
|
|
2993
2957
|
/* .graph_plan_free = */ NULL,
|
|
2958
|
+
/* .graph_plan_update = */ NULL,
|
|
2994
2959
|
/* .graph_plan_compute = */ NULL,
|
|
2995
2960
|
/* .graph_compute = */ ggml_backend_cuda_graph_compute,
|
|
2996
2961
|
/* .supports_op = */ ggml_backend_cuda_supports_op,
|
|
2962
|
+
/* .supports_buft = */ ggml_backend_cuda_supports_buft,
|
|
2997
2963
|
/* .offload_op = */ ggml_backend_cuda_offload_op,
|
|
2998
2964
|
/* .event_new = */ ggml_backend_cuda_event_new,
|
|
2999
2965
|
/* .event_free = */ ggml_backend_cuda_event_free,
|
|
@@ -1340,7 +1340,7 @@ static bool ggml_vk_supports_op(const struct ggml_tensor * op) {
|
|
|
1340
1340
|
case GGML_UNARY_OP_RELU:
|
|
1341
1341
|
case GGML_UNARY_OP_GELU:
|
|
1342
1342
|
case GGML_UNARY_OP_SILU:
|
|
1343
|
-
return
|
|
1343
|
+
return ggml_is_contiguous(op->src[0]);
|
|
1344
1344
|
default:
|
|
1345
1345
|
;
|
|
1346
1346
|
}
|
|
@@ -1902,18 +1902,12 @@ static size_t ggml_backend_vk_buffer_type_get_max_size(ggml_backend_buffer_type_
|
|
|
1902
1902
|
return ctx->max_alloc;
|
|
1903
1903
|
}
|
|
1904
1904
|
|
|
1905
|
-
static bool ggml_backend_kompute_buffer_type_supports_backend(ggml_backend_buffer_type_t buft, ggml_backend_t backend) {
|
|
1906
|
-
GGML_UNUSED(buft);
|
|
1907
|
-
return ggml_backend_is_kompute(backend);
|
|
1908
|
-
}
|
|
1909
|
-
|
|
1910
1905
|
static ggml_backend_buffer_type_i ggml_backend_kompute_buffer_type_interface = {
|
|
1911
1906
|
/* .get_name = */ ggml_backend_kompute_buffer_type_get_name,
|
|
1912
1907
|
/* .alloc_buffer = */ ggml_backend_kompute_buffer_type_alloc_buffer,
|
|
1913
1908
|
/* .get_alignment = */ ggml_backend_kompute_buffer_type_get_alignment,
|
|
1914
1909
|
/* .get_max_size = */ ggml_backend_vk_buffer_type_get_max_size,
|
|
1915
1910
|
/* .get_alloc_size = */ NULL, // defaults to ggml_nbytes
|
|
1916
|
-
/* .supports_backend = */ ggml_backend_kompute_buffer_type_supports_backend,
|
|
1917
1911
|
/* .is_host = */ NULL,
|
|
1918
1912
|
};
|
|
1919
1913
|
|
|
@@ -1973,6 +1967,11 @@ static bool ggml_backend_kompute_supports_op(ggml_backend_t backend, const struc
|
|
|
1973
1967
|
return ggml_vk_supports_op(op);
|
|
1974
1968
|
}
|
|
1975
1969
|
|
|
1970
|
+
static bool ggml_backend_kompute_supports_buft(ggml_backend_t backend, ggml_backend_buffer_type_t buft) {
|
|
1971
|
+
GGML_UNUSED(backend);
|
|
1972
|
+
return buft->iface.get_name == ggml_backend_kompute_buffer_type_get_name;
|
|
1973
|
+
}
|
|
1974
|
+
|
|
1976
1975
|
static struct ggml_backend_i kompute_backend_i = {
|
|
1977
1976
|
/* .get_name = */ ggml_backend_kompute_name,
|
|
1978
1977
|
/* .free = */ ggml_backend_kompute_free,
|
|
@@ -1983,9 +1982,11 @@ static struct ggml_backend_i kompute_backend_i = {
|
|
|
1983
1982
|
/* .synchronize = */ NULL,
|
|
1984
1983
|
/* .graph_plan_create = */ NULL,
|
|
1985
1984
|
/* .graph_plan_free = */ NULL,
|
|
1985
|
+
/* .graph_plan_update = */ NULL,
|
|
1986
1986
|
/* .graph_plan_compute = */ NULL,
|
|
1987
1987
|
/* .graph_compute = */ ggml_backend_kompute_graph_compute,
|
|
1988
1988
|
/* .supports_op = */ ggml_backend_kompute_supports_op,
|
|
1989
|
+
/* .supports_buft = */ ggml_backend_kompute_supports_buft,
|
|
1989
1990
|
/* .offload_op = */ NULL,
|
|
1990
1991
|
/* .event_new = */ NULL,
|
|
1991
1992
|
/* .event_free = */ NULL,
|
|
@@ -744,7 +744,7 @@ static bool ggml_metal_supports_op(const struct ggml_metal_context * ctx, const
|
|
|
744
744
|
case GGML_UNARY_OP_GELU:
|
|
745
745
|
case GGML_UNARY_OP_GELU_QUICK:
|
|
746
746
|
case GGML_UNARY_OP_SILU:
|
|
747
|
-
return
|
|
747
|
+
return ggml_is_contiguous(op->src[0]);
|
|
748
748
|
default:
|
|
749
749
|
return false;
|
|
750
750
|
}
|
|
@@ -1862,9 +1862,10 @@ static enum ggml_status ggml_metal_graph_compute(
|
|
|
1862
1862
|
// ne21 = n_rows
|
|
1863
1863
|
const int dst_rows = ne20*ne21;
|
|
1864
1864
|
const int dst_rows_min = n_as;
|
|
1865
|
+
const int dst_rows_max = (ctx->device.maxThreadgroupMemoryLength - 32 - 8192)/4;
|
|
1865
1866
|
|
|
1866
1867
|
// max size of the rowids array in the kernel shared buffer
|
|
1867
|
-
GGML_ASSERT(dst_rows <=
|
|
1868
|
+
GGML_ASSERT(dst_rows <= dst_rows_max);
|
|
1868
1869
|
|
|
1869
1870
|
// for now the matrix-matrix multiplication kernel only works on A14+/M1+ SoCs
|
|
1870
1871
|
// AMD GPU and older A-chips will reuse matrix-vector multiplication kernel
|
|
@@ -3044,12 +3045,6 @@ GGML_CALL static size_t ggml_backend_metal_buffer_type_get_max_size(ggml_backend
|
|
|
3044
3045
|
UNUSED(buft);
|
|
3045
3046
|
}
|
|
3046
3047
|
|
|
3047
|
-
GGML_CALL static bool ggml_backend_metal_buffer_type_supports_backend(ggml_backend_buffer_type_t buft, ggml_backend_t backend) {
|
|
3048
|
-
return ggml_backend_is_metal(backend) || ggml_backend_is_cpu(backend);
|
|
3049
|
-
|
|
3050
|
-
UNUSED(buft);
|
|
3051
|
-
}
|
|
3052
|
-
|
|
3053
3048
|
GGML_CALL static bool ggml_backend_metal_buffer_type_is_host(ggml_backend_buffer_type_t buft) {
|
|
3054
3049
|
return true;
|
|
3055
3050
|
|
|
@@ -3064,7 +3059,6 @@ GGML_CALL ggml_backend_buffer_type_t ggml_backend_metal_buffer_type(void) {
|
|
|
3064
3059
|
/* .get_alignment = */ ggml_backend_metal_buffer_type_get_alignment,
|
|
3065
3060
|
/* .get_max_size = */ ggml_backend_metal_buffer_type_get_max_size,
|
|
3066
3061
|
/* .get_alloc_size = */ NULL, // defaults to ggml_nbytes
|
|
3067
|
-
/* .supports_backend = */ ggml_backend_metal_buffer_type_supports_backend,
|
|
3068
3062
|
/* .is_host = */ ggml_backend_metal_buffer_type_is_host,
|
|
3069
3063
|
},
|
|
3070
3064
|
/* .context = */ NULL,
|
|
@@ -3179,6 +3173,12 @@ GGML_CALL static bool ggml_backend_metal_supports_op(ggml_backend_t backend, con
|
|
|
3179
3173
|
return ggml_metal_supports_op(metal_ctx, op);
|
|
3180
3174
|
}
|
|
3181
3175
|
|
|
3176
|
+
GGML_CALL static bool ggml_backend_metal_supports_buft(ggml_backend_t backend, ggml_backend_buffer_type_t buft) {
|
|
3177
|
+
return buft->iface.get_name == ggml_backend_metal_buffer_type_get_name;
|
|
3178
|
+
|
|
3179
|
+
UNUSED(backend);
|
|
3180
|
+
}
|
|
3181
|
+
|
|
3182
3182
|
static struct ggml_backend_i ggml_backend_metal_i = {
|
|
3183
3183
|
/* .get_name = */ ggml_backend_metal_name,
|
|
3184
3184
|
/* .free = */ ggml_backend_metal_free,
|
|
@@ -3189,9 +3189,11 @@ static struct ggml_backend_i ggml_backend_metal_i = {
|
|
|
3189
3189
|
/* .synchronize = */ NULL,
|
|
3190
3190
|
/* .graph_plan_create = */ NULL,
|
|
3191
3191
|
/* .graph_plan_free = */ NULL,
|
|
3192
|
+
/* .graph_plan_update = */ NULL,
|
|
3192
3193
|
/* .graph_plan_compute = */ NULL,
|
|
3193
3194
|
/* .graph_compute = */ ggml_backend_metal_graph_compute,
|
|
3194
3195
|
/* .supports_op = */ ggml_backend_metal_supports_op,
|
|
3196
|
+
/* .supports_buft = */ ggml_backend_metal_supports_buft,
|
|
3195
3197
|
/* .offload_op = */ NULL,
|
|
3196
3198
|
/* .event_new = */ NULL,
|
|
3197
3199
|
/* .event_free = */ NULL,
|
|
@@ -540,22 +540,12 @@ GGML_CALL static size_t ggml_backend_rpc_buffer_type_get_alloc_size(ggml_backend
|
|
|
540
540
|
return ggml_nbytes(tensor);
|
|
541
541
|
}
|
|
542
542
|
|
|
543
|
-
GGML_CALL static bool ggml_backend_rpc_buffer_type_supports_backend(ggml_backend_buffer_type_t buft, ggml_backend_t backend) {
|
|
544
|
-
if (!ggml_backend_is_rpc(backend)) {
|
|
545
|
-
return false;
|
|
546
|
-
}
|
|
547
|
-
ggml_backend_rpc_buffer_type_context * buft_ctx = (ggml_backend_rpc_buffer_type_context *)buft->context;
|
|
548
|
-
ggml_backend_rpc_context * rpc_ctx = (ggml_backend_rpc_context *)backend->context;
|
|
549
|
-
return buft_ctx->endpoint == rpc_ctx->endpoint;
|
|
550
|
-
}
|
|
551
|
-
|
|
552
543
|
static ggml_backend_buffer_type_i ggml_backend_rpc_buffer_type_interface = {
|
|
553
544
|
/* .get_name = */ ggml_backend_rpc_buffer_type_name,
|
|
554
545
|
/* .alloc_buffer = */ ggml_backend_rpc_buffer_type_alloc_buffer,
|
|
555
546
|
/* .get_alignment = */ ggml_backend_rpc_buffer_type_get_alignment,
|
|
556
547
|
/* .get_max_size = */ ggml_backend_rpc_get_max_size,
|
|
557
548
|
/* .get_alloc_size = */ ggml_backend_rpc_buffer_type_get_alloc_size,
|
|
558
|
-
/* .supports_backend = */ ggml_backend_rpc_buffer_type_supports_backend,
|
|
559
549
|
/* .is_host = */ NULL,
|
|
560
550
|
};
|
|
561
551
|
|
|
@@ -634,8 +624,17 @@ GGML_CALL static enum ggml_status ggml_backend_rpc_graph_compute(ggml_backend_t
|
|
|
634
624
|
GGML_CALL static bool ggml_backend_rpc_supports_op(ggml_backend_t backend, const ggml_tensor * op) {
|
|
635
625
|
UNUSED(backend);
|
|
636
626
|
UNUSED(op);
|
|
637
|
-
|
|
638
|
-
return
|
|
627
|
+
//TODO: call the remote backend and cache the results
|
|
628
|
+
return true;
|
|
629
|
+
}
|
|
630
|
+
|
|
631
|
+
GGML_CALL static bool ggml_backend_rpc_supports_buft(ggml_backend_t backend, ggml_backend_buffer_type_t buft) {
|
|
632
|
+
if (buft->iface.get_name != ggml_backend_rpc_buffer_type_name) {
|
|
633
|
+
return false;
|
|
634
|
+
}
|
|
635
|
+
ggml_backend_rpc_buffer_type_context * buft_ctx = (ggml_backend_rpc_buffer_type_context *)buft->context;
|
|
636
|
+
ggml_backend_rpc_context * rpc_ctx = (ggml_backend_rpc_context *)backend->context;
|
|
637
|
+
return buft_ctx->endpoint == rpc_ctx->endpoint;
|
|
639
638
|
}
|
|
640
639
|
|
|
641
640
|
static ggml_backend_i ggml_backend_rpc_interface = {
|
|
@@ -648,9 +647,11 @@ static ggml_backend_i ggml_backend_rpc_interface = {
|
|
|
648
647
|
/* .synchronize = */ ggml_backend_rpc_synchronize,
|
|
649
648
|
/* .graph_plan_create = */ NULL,
|
|
650
649
|
/* .graph_plan_free = */ NULL,
|
|
650
|
+
/* .graph_plan_update = */ NULL,
|
|
651
651
|
/* .graph_plan_compute = */ NULL,
|
|
652
652
|
/* .graph_compute = */ ggml_backend_rpc_graph_compute,
|
|
653
653
|
/* .supports_op = */ ggml_backend_rpc_supports_op,
|
|
654
|
+
/* .supports_buft = */ ggml_backend_rpc_supports_buft,
|
|
654
655
|
/* .offload_op = */ NULL,
|
|
655
656
|
/* .event_new = */ NULL,
|
|
656
657
|
/* .event_free = */ NULL,
|