llama_cpp 0.16.1 → 0.17.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +25 -0
- data/README.md +7 -12
- data/ext/llama_cpp/extconf.rb +2 -42
- data/ext/llama_cpp/llama_cpp.cpp +20 -0
- data/lib/llama_cpp/version.rb +3 -3
- data/sig/llama_cpp.rbs +5 -0
- metadata +2 -171
- data/vendor/include/.gitkeep +0 -0
- data/vendor/lib/.gitkeep +0 -0
- data/vendor/tmp/llama.cpp/LICENSE +0 -21
- data/vendor/tmp/llama.cpp/Makefile +0 -1116
- data/vendor/tmp/llama.cpp/ggml-alloc.c +0 -1041
- data/vendor/tmp/llama.cpp/ggml-alloc.h +0 -76
- data/vendor/tmp/llama.cpp/ggml-backend-impl.h +0 -153
- data/vendor/tmp/llama.cpp/ggml-backend.c +0 -2214
- data/vendor/tmp/llama.cpp/ggml-backend.h +0 -233
- data/vendor/tmp/llama.cpp/ggml-blas.cpp +0 -363
- data/vendor/tmp/llama.cpp/ggml-blas.h +0 -23
- data/vendor/tmp/llama.cpp/ggml-common.h +0 -1805
- data/vendor/tmp/llama.cpp/ggml-cuda/acc.cu +0 -47
- data/vendor/tmp/llama.cpp/ggml-cuda/arange.cu +0 -34
- data/vendor/tmp/llama.cpp/ggml-cuda/argsort.cu +0 -104
- data/vendor/tmp/llama.cpp/ggml-cuda/binbcast.cu +0 -280
- data/vendor/tmp/llama.cpp/ggml-cuda/clamp.cu +0 -34
- data/vendor/tmp/llama.cpp/ggml-cuda/concat.cu +0 -196
- data/vendor/tmp/llama.cpp/ggml-cuda/convert.cu +0 -686
- data/vendor/tmp/llama.cpp/ggml-cuda/cpy.cu +0 -490
- data/vendor/tmp/llama.cpp/ggml-cuda/diagmask.cu +0 -40
- data/vendor/tmp/llama.cpp/ggml-cuda/dmmv.cu +0 -674
- data/vendor/tmp/llama.cpp/ggml-cuda/fattn-tile-f16.cu +0 -319
- data/vendor/tmp/llama.cpp/ggml-cuda/fattn-tile-f32.cu +0 -312
- data/vendor/tmp/llama.cpp/ggml-cuda/fattn.cu +0 -345
- data/vendor/tmp/llama.cpp/ggml-cuda/getrows.cu +0 -178
- data/vendor/tmp/llama.cpp/ggml-cuda/im2col.cu +0 -104
- data/vendor/tmp/llama.cpp/ggml-cuda/mmq.cu +0 -88
- data/vendor/tmp/llama.cpp/ggml-cuda/mmvq.cu +0 -419
- data/vendor/tmp/llama.cpp/ggml-cuda/norm.cu +0 -221
- data/vendor/tmp/llama.cpp/ggml-cuda/pad.cu +0 -49
- data/vendor/tmp/llama.cpp/ggml-cuda/pool2d.cu +0 -94
- data/vendor/tmp/llama.cpp/ggml-cuda/quantize.cu +0 -112
- data/vendor/tmp/llama.cpp/ggml-cuda/rope.cu +0 -271
- data/vendor/tmp/llama.cpp/ggml-cuda/scale.cu +0 -31
- data/vendor/tmp/llama.cpp/ggml-cuda/softmax.cu +0 -206
- data/vendor/tmp/llama.cpp/ggml-cuda/sumrows.cu +0 -40
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-f16.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q4_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q4_1.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q5_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q5_1.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q8_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-f16.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q4_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q4_1.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q5_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q5_1.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q8_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-f16.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q4_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q4_1.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q5_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q5_1.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q8_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-f16.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q4_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q4_1.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q5_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q5_1.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q8_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-f16.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q4_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q4_1.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q5_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q5_1.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q8_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-f16.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q4_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q4_1.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q5_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q5_1.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q8_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs256-f16-f16.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-f16.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q4_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q4_1.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q5_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q5_1.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q8_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-f16.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q4_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q4_1.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q5_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q5_1.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q8_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-f16.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q4_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q4_1.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q5_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q5_1.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q8_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-f16.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q4_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q4_1.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q5_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q5_1.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q8_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-f16.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q4_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q4_1.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q5_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q5_1.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q8_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-f16.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q4_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q4_1.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q5_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q5_1.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q8_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-f16.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q4_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q4_1.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q5_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q5_1.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q8_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs256-f16-f16.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-f16.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q4_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q4_1.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q5_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q5_1.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q8_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqfloat-cpb16.cu +0 -10
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqfloat-cpb32.cu +0 -9
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqhalf-cpb16.cu +0 -10
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqhalf-cpb32.cu +0 -10
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqhalf-cpb8.cu +0 -8
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q2_k.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q3_k.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q4_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q4_1.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q4_k.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q5_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q5_1.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q5_k.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q6_k.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q8_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/tsembd.cu +0 -47
- data/vendor/tmp/llama.cpp/ggml-cuda/unary.cu +0 -286
- data/vendor/tmp/llama.cpp/ggml-cuda/upscale.cu +0 -51
- data/vendor/tmp/llama.cpp/ggml-cuda.cu +0 -3069
- data/vendor/tmp/llama.cpp/ggml-cuda.h +0 -44
- data/vendor/tmp/llama.cpp/ggml-impl.h +0 -651
- data/vendor/tmp/llama.cpp/ggml-kompute.cpp +0 -2038
- data/vendor/tmp/llama.cpp/ggml-kompute.h +0 -46
- data/vendor/tmp/llama.cpp/ggml-metal.h +0 -66
- data/vendor/tmp/llama.cpp/ggml-metal.m +0 -3267
- data/vendor/tmp/llama.cpp/ggml-metal.metal +0 -6540
- data/vendor/tmp/llama.cpp/ggml-quants.c +0 -14380
- data/vendor/tmp/llama.cpp/ggml-quants.h +0 -133
- data/vendor/tmp/llama.cpp/ggml-rpc.cpp +0 -1173
- data/vendor/tmp/llama.cpp/ggml-rpc.h +0 -24
- data/vendor/tmp/llama.cpp/ggml-sycl.cpp +0 -17429
- data/vendor/tmp/llama.cpp/ggml-sycl.h +0 -49
- data/vendor/tmp/llama.cpp/ggml-vulkan-shaders.hpp +0 -140820
- data/vendor/tmp/llama.cpp/ggml-vulkan.cpp +0 -7271
- data/vendor/tmp/llama.cpp/ggml-vulkan.h +0 -29
- data/vendor/tmp/llama.cpp/ggml.c +0 -22589
- data/vendor/tmp/llama.cpp/ggml.h +0 -2452
- data/vendor/tmp/llama.cpp/llama.cpp +0 -18692
- data/vendor/tmp/llama.cpp/llama.h +0 -1143
- data/vendor/tmp/llama.cpp/scripts/get-flags.mk +0 -38
- data/vendor/tmp/llama.cpp/sgemm.cpp +0 -1030
- data/vendor/tmp/llama.cpp/sgemm.h +0 -14
- data/vendor/tmp/llama.cpp/unicode-data.cpp +0 -6983
- data/vendor/tmp/llama.cpp/unicode-data.h +0 -20
- data/vendor/tmp/llama.cpp/unicode.cpp +0 -796
- data/vendor/tmp/llama.cpp/unicode.h +0 -63
|
@@ -1,3069 +0,0 @@
|
|
|
1
|
-
#include "ggml-cuda.h"
|
|
2
|
-
#include "ggml.h"
|
|
3
|
-
#include "ggml-backend-impl.h"
|
|
4
|
-
|
|
5
|
-
#include "ggml-cuda/common.cuh"
|
|
6
|
-
#include "ggml-cuda/acc.cuh"
|
|
7
|
-
#include "ggml-cuda/arange.cuh"
|
|
8
|
-
#include "ggml-cuda/argsort.cuh"
|
|
9
|
-
#include "ggml-cuda/binbcast.cuh"
|
|
10
|
-
#include "ggml-cuda/clamp.cuh"
|
|
11
|
-
#include "ggml-cuda/concat.cuh"
|
|
12
|
-
#include "ggml-cuda/convert.cuh"
|
|
13
|
-
#include "ggml-cuda/cpy.cuh"
|
|
14
|
-
#include "ggml-cuda/diagmask.cuh"
|
|
15
|
-
#include "ggml-cuda/dmmv.cuh"
|
|
16
|
-
#include "ggml-cuda/fattn.cuh"
|
|
17
|
-
#include "ggml-cuda/getrows.cuh"
|
|
18
|
-
#include "ggml-cuda/im2col.cuh"
|
|
19
|
-
#include "ggml-cuda/mmq.cuh"
|
|
20
|
-
#include "ggml-cuda/mmvq.cuh"
|
|
21
|
-
#include "ggml-cuda/norm.cuh"
|
|
22
|
-
#include "ggml-cuda/pad.cuh"
|
|
23
|
-
#include "ggml-cuda/pool2d.cuh"
|
|
24
|
-
#include "ggml-cuda/quantize.cuh"
|
|
25
|
-
#include "ggml-cuda/rope.cuh"
|
|
26
|
-
#include "ggml-cuda/scale.cuh"
|
|
27
|
-
#include "ggml-cuda/softmax.cuh"
|
|
28
|
-
#include "ggml-cuda/sumrows.cuh"
|
|
29
|
-
#include "ggml-cuda/tsembd.cuh"
|
|
30
|
-
#include "ggml-cuda/unary.cuh"
|
|
31
|
-
#include "ggml-cuda/upscale.cuh"
|
|
32
|
-
|
|
33
|
-
#include <algorithm>
|
|
34
|
-
#include <array>
|
|
35
|
-
#include <atomic>
|
|
36
|
-
#include <cinttypes>
|
|
37
|
-
#include <cstddef>
|
|
38
|
-
#include <cstdint>
|
|
39
|
-
#include <float.h>
|
|
40
|
-
#include <limits>
|
|
41
|
-
#include <map>
|
|
42
|
-
#include <memory>
|
|
43
|
-
#include <mutex>
|
|
44
|
-
#include <stdint.h>
|
|
45
|
-
#include <stdio.h>
|
|
46
|
-
#include <stdarg.h>
|
|
47
|
-
#include <stdlib.h>
|
|
48
|
-
#include <string>
|
|
49
|
-
#include <vector>
|
|
50
|
-
|
|
51
|
-
static_assert(sizeof(half) == sizeof(ggml_fp16_t), "wrong fp16 size");
|
|
52
|
-
|
|
53
|
-
static void ggml_cuda_default_log_callback(enum ggml_log_level level, const char * msg, void * user_data) {
|
|
54
|
-
GGML_UNUSED(level);
|
|
55
|
-
GGML_UNUSED(user_data);
|
|
56
|
-
fprintf(stderr, "%s", msg);
|
|
57
|
-
}
|
|
58
|
-
|
|
59
|
-
ggml_log_callback ggml_cuda_log_callback = ggml_cuda_default_log_callback;
|
|
60
|
-
void * ggml_cuda_log_user_data = NULL;
|
|
61
|
-
|
|
62
|
-
GGML_API void ggml_backend_cuda_log_set_callback(ggml_log_callback log_callback, void * user_data) {
|
|
63
|
-
ggml_cuda_log_callback = log_callback;
|
|
64
|
-
ggml_cuda_log_user_data = user_data;
|
|
65
|
-
}
|
|
66
|
-
|
|
67
|
-
#define GGML_CUDA_LOG_INFO(...) ggml_cuda_log(GGML_LOG_LEVEL_INFO, __VA_ARGS__)
|
|
68
|
-
#define GGML_CUDA_LOG_WARN(...) ggml_cuda_log(GGML_LOG_LEVEL_WARN, __VA_ARGS__)
|
|
69
|
-
#define GGML_CUDA_LOG_ERROR(...) ggml_cuda_log(GGML_LOG_LEVEL_ERROR, __VA_ARGS__)
|
|
70
|
-
|
|
71
|
-
GGML_ATTRIBUTE_FORMAT(2, 3)
|
|
72
|
-
static void ggml_cuda_log(enum ggml_log_level level, const char * format, ...) {
|
|
73
|
-
if (ggml_cuda_log_callback != NULL) {
|
|
74
|
-
va_list args;
|
|
75
|
-
va_start(args, format);
|
|
76
|
-
char buffer[128];
|
|
77
|
-
int len = vsnprintf(buffer, 128, format, args);
|
|
78
|
-
if (len < 128) {
|
|
79
|
-
ggml_cuda_log_callback(level, buffer, ggml_cuda_log_user_data);
|
|
80
|
-
} else {
|
|
81
|
-
std::vector<char> buffer2(len + 1); // vsnprintf adds a null terminator
|
|
82
|
-
va_end(args);
|
|
83
|
-
va_start(args, format);
|
|
84
|
-
vsnprintf(&buffer2[0], buffer2.size(), format, args);
|
|
85
|
-
ggml_cuda_log_callback(level, buffer2.data(), ggml_cuda_log_user_data);
|
|
86
|
-
}
|
|
87
|
-
va_end(args);
|
|
88
|
-
}
|
|
89
|
-
}
|
|
90
|
-
|
|
91
|
-
[[noreturn]]
|
|
92
|
-
void ggml_cuda_error(const char * stmt, const char * func, const char * file, int line, const char * msg) {
|
|
93
|
-
int id = -1; // in case cudaGetDevice fails
|
|
94
|
-
cudaGetDevice(&id);
|
|
95
|
-
|
|
96
|
-
GGML_CUDA_LOG_ERROR("CUDA error: %s\n", msg);
|
|
97
|
-
GGML_CUDA_LOG_ERROR(" current device: %d, in function %s at %s:%d\n", id, func, file, line);
|
|
98
|
-
GGML_CUDA_LOG_ERROR(" %s\n", stmt);
|
|
99
|
-
// abort with GGML_ASSERT to get a stack trace
|
|
100
|
-
GGML_ASSERT(!"CUDA error");
|
|
101
|
-
}
|
|
102
|
-
|
|
103
|
-
// this is faster on Windows
|
|
104
|
-
// probably because the Windows CUDA libraries forget to make this check before invoking the drivers
|
|
105
|
-
void ggml_cuda_set_device(int device) {
|
|
106
|
-
int current_device;
|
|
107
|
-
CUDA_CHECK(cudaGetDevice(¤t_device));
|
|
108
|
-
|
|
109
|
-
if (device == current_device) {
|
|
110
|
-
return;
|
|
111
|
-
}
|
|
112
|
-
|
|
113
|
-
CUDA_CHECK(cudaSetDevice(device));
|
|
114
|
-
}
|
|
115
|
-
|
|
116
|
-
int ggml_cuda_get_device() {
|
|
117
|
-
int id;
|
|
118
|
-
CUDA_CHECK(cudaGetDevice(&id));
|
|
119
|
-
return id;
|
|
120
|
-
}
|
|
121
|
-
|
|
122
|
-
static cudaError_t ggml_cuda_device_malloc(void ** ptr, size_t size, int device) {
|
|
123
|
-
ggml_cuda_set_device(device);
|
|
124
|
-
#if defined(GGML_USE_HIPBLAS) && defined(GGML_HIP_UMA)
|
|
125
|
-
auto res = hipMallocManaged(ptr, size);
|
|
126
|
-
if (res == hipSuccess) {
|
|
127
|
-
// if error we "need" to know why...
|
|
128
|
-
CUDA_CHECK(hipMemAdvise(*ptr, size, hipMemAdviseSetCoarseGrain, device));
|
|
129
|
-
}
|
|
130
|
-
return res;
|
|
131
|
-
#else
|
|
132
|
-
return cudaMalloc(ptr, size);
|
|
133
|
-
#endif
|
|
134
|
-
}
|
|
135
|
-
|
|
136
|
-
static ggml_cuda_device_info ggml_cuda_init() {
|
|
137
|
-
#ifdef __HIP_PLATFORM_AMD__
|
|
138
|
-
// Workaround for a rocBLAS bug when using multiple graphics cards:
|
|
139
|
-
// https://github.com/ROCmSoftwarePlatform/rocBLAS/issues/1346
|
|
140
|
-
rocblas_initialize();
|
|
141
|
-
CUDA_CHECK(cudaDeviceSynchronize());
|
|
142
|
-
#endif
|
|
143
|
-
|
|
144
|
-
ggml_cuda_device_info info = {};
|
|
145
|
-
|
|
146
|
-
cudaError_t err = cudaGetDeviceCount(&info.device_count);
|
|
147
|
-
if (err != cudaSuccess) {
|
|
148
|
-
GGML_CUDA_LOG_ERROR("%s: failed to initialize " GGML_CUDA_NAME ": %s\n", __func__, cudaGetErrorString(err));
|
|
149
|
-
return info;
|
|
150
|
-
}
|
|
151
|
-
|
|
152
|
-
GGML_ASSERT(info.device_count <= GGML_CUDA_MAX_DEVICES);
|
|
153
|
-
|
|
154
|
-
int64_t total_vram = 0;
|
|
155
|
-
#if defined(GGML_CUDA_FORCE_MMQ)
|
|
156
|
-
GGML_CUDA_LOG_INFO("%s: GGML_CUDA_FORCE_MMQ: yes\n", __func__);
|
|
157
|
-
#else
|
|
158
|
-
GGML_CUDA_LOG_INFO("%s: GGML_CUDA_FORCE_MMQ: no\n", __func__);
|
|
159
|
-
#endif
|
|
160
|
-
#if defined(CUDA_USE_TENSOR_CORES)
|
|
161
|
-
GGML_CUDA_LOG_INFO("%s: CUDA_USE_TENSOR_CORES: yes\n", __func__);
|
|
162
|
-
#else
|
|
163
|
-
GGML_CUDA_LOG_INFO("%s: CUDA_USE_TENSOR_CORES: no\n", __func__);
|
|
164
|
-
#endif
|
|
165
|
-
GGML_CUDA_LOG_INFO("%s: found %d " GGML_CUDA_NAME " devices:\n", __func__, info.device_count);
|
|
166
|
-
for (int id = 0; id < info.device_count; ++id) {
|
|
167
|
-
int device_vmm = 0;
|
|
168
|
-
|
|
169
|
-
#if !defined(GGML_USE_HIPBLAS) && !defined(GGML_CUDA_NO_VMM)
|
|
170
|
-
CUdevice device;
|
|
171
|
-
CU_CHECK(cuDeviceGet(&device, id));
|
|
172
|
-
CU_CHECK(cuDeviceGetAttribute(&device_vmm, CU_DEVICE_ATTRIBUTE_VIRTUAL_MEMORY_MANAGEMENT_SUPPORTED, device));
|
|
173
|
-
|
|
174
|
-
if (device_vmm) {
|
|
175
|
-
CUmemAllocationProp alloc_prop = {};
|
|
176
|
-
alloc_prop.type = CU_MEM_ALLOCATION_TYPE_PINNED;
|
|
177
|
-
alloc_prop.location.type = CU_MEM_LOCATION_TYPE_DEVICE;
|
|
178
|
-
alloc_prop.location.id = id;
|
|
179
|
-
CU_CHECK(cuMemGetAllocationGranularity(&info.devices[id].vmm_granularity, &alloc_prop, CU_MEM_ALLOC_GRANULARITY_RECOMMENDED));
|
|
180
|
-
}
|
|
181
|
-
#endif // !defined(GGML_USE_HIPBLAS)
|
|
182
|
-
info.devices[id].vmm = !!device_vmm;
|
|
183
|
-
|
|
184
|
-
cudaDeviceProp prop;
|
|
185
|
-
CUDA_CHECK(cudaGetDeviceProperties(&prop, id));
|
|
186
|
-
GGML_CUDA_LOG_INFO(" Device %d: %s, compute capability %d.%d, VMM: %s\n", id, prop.name, prop.major, prop.minor, device_vmm ? "yes" : "no");
|
|
187
|
-
|
|
188
|
-
info.default_tensor_split[id] = total_vram;
|
|
189
|
-
total_vram += prop.totalGlobalMem;
|
|
190
|
-
|
|
191
|
-
info.devices[id].nsm = prop.multiProcessorCount;
|
|
192
|
-
info.devices[id].smpb = prop.sharedMemPerBlock;
|
|
193
|
-
#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)
|
|
194
|
-
info.devices[id].smpbo = prop.sharedMemPerBlock;
|
|
195
|
-
info.devices[id].cc = 100*prop.major + 10*prop.minor + CC_OFFSET_AMD;
|
|
196
|
-
#else
|
|
197
|
-
info.devices[id].smpbo = prop.sharedMemPerBlockOptin;
|
|
198
|
-
info.devices[id].cc = 100*prop.major + 10*prop.minor;
|
|
199
|
-
#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)
|
|
200
|
-
}
|
|
201
|
-
|
|
202
|
-
for (int id = 0; id < info.device_count; ++id) {
|
|
203
|
-
info.default_tensor_split[id] /= total_vram;
|
|
204
|
-
}
|
|
205
|
-
|
|
206
|
-
// configure logging to stdout
|
|
207
|
-
// CUBLAS_CHECK(cublasLoggerConfigure(1, 1, 0, nullptr));
|
|
208
|
-
|
|
209
|
-
return info;
|
|
210
|
-
}
|
|
211
|
-
|
|
212
|
-
const ggml_cuda_device_info & ggml_cuda_info() {
|
|
213
|
-
static ggml_cuda_device_info info = ggml_cuda_init();
|
|
214
|
-
return info;
|
|
215
|
-
}
|
|
216
|
-
|
|
217
|
-
// #define DEBUG_CUDA_MALLOC
|
|
218
|
-
|
|
219
|
-
// buffer pool for cuda (legacy)
|
|
220
|
-
struct ggml_cuda_pool_leg : public ggml_cuda_pool {
|
|
221
|
-
static const int MAX_BUFFERS = 256;
|
|
222
|
-
|
|
223
|
-
int device;
|
|
224
|
-
struct ggml_cuda_buffer {
|
|
225
|
-
void * ptr = nullptr;
|
|
226
|
-
size_t size = 0;
|
|
227
|
-
};
|
|
228
|
-
|
|
229
|
-
ggml_cuda_buffer buffer_pool[MAX_BUFFERS] = {};
|
|
230
|
-
size_t pool_size = 0;
|
|
231
|
-
|
|
232
|
-
explicit ggml_cuda_pool_leg(int device) :
|
|
233
|
-
device(device) {
|
|
234
|
-
}
|
|
235
|
-
|
|
236
|
-
~ggml_cuda_pool_leg() {
|
|
237
|
-
ggml_cuda_set_device(device);
|
|
238
|
-
for (int i = 0; i < MAX_BUFFERS; ++i) {
|
|
239
|
-
ggml_cuda_buffer & b = buffer_pool[i];
|
|
240
|
-
if (b.ptr != nullptr) {
|
|
241
|
-
CUDA_CHECK(cudaFree(b.ptr));
|
|
242
|
-
pool_size -= b.size;
|
|
243
|
-
}
|
|
244
|
-
}
|
|
245
|
-
GGML_ASSERT(pool_size == 0);
|
|
246
|
-
}
|
|
247
|
-
|
|
248
|
-
void * alloc(size_t size, size_t * actual_size) override {
|
|
249
|
-
#ifdef DEBUG_CUDA_MALLOC
|
|
250
|
-
int nnz = 0;
|
|
251
|
-
size_t max_size = 0;
|
|
252
|
-
#endif
|
|
253
|
-
size_t best_diff = 1ull << 36;
|
|
254
|
-
int ibest = -1;
|
|
255
|
-
for (int i = 0; i < MAX_BUFFERS; ++i) {
|
|
256
|
-
ggml_cuda_buffer& b = buffer_pool[i];
|
|
257
|
-
if (b.ptr != nullptr) {
|
|
258
|
-
#ifdef DEBUG_CUDA_MALLOC
|
|
259
|
-
++nnz;
|
|
260
|
-
if (b.size > max_size) max_size = b.size;
|
|
261
|
-
#endif
|
|
262
|
-
if (b.size >= size) {
|
|
263
|
-
size_t diff = b.size - size;
|
|
264
|
-
if (diff < best_diff) {
|
|
265
|
-
best_diff = diff;
|
|
266
|
-
ibest = i;
|
|
267
|
-
if (!best_diff) {
|
|
268
|
-
void * ptr = b.ptr;
|
|
269
|
-
*actual_size = b.size;
|
|
270
|
-
b.ptr = nullptr;
|
|
271
|
-
b.size = 0;
|
|
272
|
-
return ptr;
|
|
273
|
-
}
|
|
274
|
-
}
|
|
275
|
-
}
|
|
276
|
-
}
|
|
277
|
-
}
|
|
278
|
-
if (ibest >= 0) {
|
|
279
|
-
ggml_cuda_buffer& b = buffer_pool[ibest];
|
|
280
|
-
void * ptr = b.ptr;
|
|
281
|
-
*actual_size = b.size;
|
|
282
|
-
b.ptr = nullptr;
|
|
283
|
-
b.size = 0;
|
|
284
|
-
return ptr;
|
|
285
|
-
}
|
|
286
|
-
void * ptr;
|
|
287
|
-
size_t look_ahead_size = (size_t) (1.05 * size);
|
|
288
|
-
look_ahead_size = 256 * ((look_ahead_size + 255)/256);
|
|
289
|
-
ggml_cuda_set_device(device);
|
|
290
|
-
CUDA_CHECK(ggml_cuda_device_malloc(&ptr, look_ahead_size, device));
|
|
291
|
-
*actual_size = look_ahead_size;
|
|
292
|
-
pool_size += look_ahead_size;
|
|
293
|
-
#ifdef DEBUG_CUDA_MALLOC
|
|
294
|
-
GGML_CUDA_LOG_INFO("%s[%d]: %d buffers, max_size = %u MB, pool_size = %u MB, requested %u MB\n", __func__, device, nnz,
|
|
295
|
-
(uint32_t)(max_size / 1024 / 1024), (uint32_t)(pool_size / 1024 / 1024), (uint32_t)(size / 1024 / 1024));
|
|
296
|
-
#endif
|
|
297
|
-
return ptr;
|
|
298
|
-
}
|
|
299
|
-
|
|
300
|
-
void free(void * ptr, size_t size) override {
|
|
301
|
-
for (int i = 0; i < MAX_BUFFERS; ++i) {
|
|
302
|
-
ggml_cuda_buffer& b = buffer_pool[i];
|
|
303
|
-
if (b.ptr == nullptr) {
|
|
304
|
-
b.ptr = ptr;
|
|
305
|
-
b.size = size;
|
|
306
|
-
return;
|
|
307
|
-
}
|
|
308
|
-
}
|
|
309
|
-
GGML_CUDA_LOG_WARN("Cuda buffer pool full, increase MAX_CUDA_BUFFERS\n");
|
|
310
|
-
ggml_cuda_set_device(device);
|
|
311
|
-
CUDA_CHECK(cudaFree(ptr));
|
|
312
|
-
pool_size -= size;
|
|
313
|
-
}
|
|
314
|
-
};
|
|
315
|
-
|
|
316
|
-
// pool with virtual memory
|
|
317
|
-
#if !defined(GGML_USE_HIPBLAS) && !defined(GGML_CUDA_NO_VMM)
|
|
318
|
-
struct ggml_cuda_pool_vmm : public ggml_cuda_pool {
|
|
319
|
-
static const size_t CUDA_POOL_VMM_MAX_SIZE = 1ull << 35; // 32 GB
|
|
320
|
-
|
|
321
|
-
int device;
|
|
322
|
-
CUdeviceptr pool_addr = 0;
|
|
323
|
-
size_t pool_used = 0;
|
|
324
|
-
size_t pool_size = 0;
|
|
325
|
-
size_t granularity;
|
|
326
|
-
|
|
327
|
-
explicit ggml_cuda_pool_vmm(int device) :
|
|
328
|
-
device(device),
|
|
329
|
-
granularity(ggml_cuda_info().devices[device].vmm_granularity) {
|
|
330
|
-
}
|
|
331
|
-
|
|
332
|
-
~ggml_cuda_pool_vmm() {
|
|
333
|
-
if (pool_addr != 0) {
|
|
334
|
-
CU_CHECK(cuMemUnmap(pool_addr, pool_size));
|
|
335
|
-
CU_CHECK(cuMemAddressFree(pool_addr, CUDA_POOL_VMM_MAX_SIZE));
|
|
336
|
-
}
|
|
337
|
-
}
|
|
338
|
-
|
|
339
|
-
void * alloc(size_t size, size_t * actual_size) override {
|
|
340
|
-
// round up the allocation size to the alignment to ensure that all allocations are aligned for all data types
|
|
341
|
-
const size_t alignment = 128;
|
|
342
|
-
size = alignment * ((size + alignment - 1) / alignment);
|
|
343
|
-
|
|
344
|
-
size_t avail = pool_size - pool_used;
|
|
345
|
-
|
|
346
|
-
if (size > avail) {
|
|
347
|
-
// round up to the next multiple of the granularity
|
|
348
|
-
size_t reserve_size = size - avail;
|
|
349
|
-
reserve_size = granularity * ((reserve_size + granularity - 1) / granularity);
|
|
350
|
-
|
|
351
|
-
GGML_ASSERT(pool_size + reserve_size <= CUDA_POOL_VMM_MAX_SIZE);
|
|
352
|
-
|
|
353
|
-
// allocate more physical memory
|
|
354
|
-
CUmemAllocationProp prop = {};
|
|
355
|
-
prop.type = CU_MEM_ALLOCATION_TYPE_PINNED;
|
|
356
|
-
prop.location.type = CU_MEM_LOCATION_TYPE_DEVICE;
|
|
357
|
-
prop.location.id = device;
|
|
358
|
-
CUmemGenericAllocationHandle handle;
|
|
359
|
-
CU_CHECK(cuMemCreate(&handle, reserve_size, &prop, 0));
|
|
360
|
-
|
|
361
|
-
// reserve virtual address space (if not already reserved)
|
|
362
|
-
if (pool_addr == 0) {
|
|
363
|
-
CU_CHECK(cuMemAddressReserve(&pool_addr, CUDA_POOL_VMM_MAX_SIZE, 0, 0, 0));
|
|
364
|
-
}
|
|
365
|
-
|
|
366
|
-
// map at the end of the pool
|
|
367
|
-
CU_CHECK(cuMemMap(pool_addr + pool_size, reserve_size, 0, handle, 0));
|
|
368
|
-
|
|
369
|
-
// the memory allocation handle is no longer needed after mapping
|
|
370
|
-
CU_CHECK(cuMemRelease(handle));
|
|
371
|
-
|
|
372
|
-
// set access
|
|
373
|
-
CUmemAccessDesc access = {};
|
|
374
|
-
access.location.type = CU_MEM_LOCATION_TYPE_DEVICE;
|
|
375
|
-
access.location.id = device;
|
|
376
|
-
access.flags = CU_MEM_ACCESS_FLAGS_PROT_READWRITE;
|
|
377
|
-
CU_CHECK(cuMemSetAccess(pool_addr + pool_size, reserve_size, &access, 1));
|
|
378
|
-
|
|
379
|
-
// add to the pool
|
|
380
|
-
pool_size += reserve_size;
|
|
381
|
-
|
|
382
|
-
//printf("cuda pool[%d]: size increased to %llu MB (reserved %llu MB)\n",
|
|
383
|
-
// device, (unsigned long long) (pool_size/1024/1024),
|
|
384
|
-
// (unsigned long long) (reserve_size/1024/1024));
|
|
385
|
-
}
|
|
386
|
-
|
|
387
|
-
GGML_ASSERT(pool_addr != 0);
|
|
388
|
-
|
|
389
|
-
void * ptr = (void *) (pool_addr + pool_used);
|
|
390
|
-
*actual_size = size;
|
|
391
|
-
pool_used += size;
|
|
392
|
-
|
|
393
|
-
#ifdef DEBUG_CUDA_MALLOC
|
|
394
|
-
printf("cuda pool[%d]: allocated %llu bytes at %llx\n", device, (unsigned long long) size, ptr);
|
|
395
|
-
#endif
|
|
396
|
-
|
|
397
|
-
return ptr;
|
|
398
|
-
}
|
|
399
|
-
|
|
400
|
-
void free(void * ptr, size_t size) override {
|
|
401
|
-
#ifdef DEBUG_CUDA_MALLOC
|
|
402
|
-
printf("cuda pool[%d]: freed %llu bytes at %llx\n", device, (unsigned long long) size, ptr);
|
|
403
|
-
#endif
|
|
404
|
-
|
|
405
|
-
pool_used -= size;
|
|
406
|
-
|
|
407
|
-
// all deallocations must be in reverse order of the allocations
|
|
408
|
-
GGML_ASSERT(ptr == (void *) (pool_addr + pool_used));
|
|
409
|
-
}
|
|
410
|
-
};
|
|
411
|
-
#endif // !defined(GGML_USE_HIPBLAS)
|
|
412
|
-
|
|
413
|
-
std::unique_ptr<ggml_cuda_pool> ggml_backend_cuda_context::new_pool_for_device(int device) {
|
|
414
|
-
#if !defined(GGML_USE_HIPBLAS) && !defined(GGML_CUDA_NO_VMM)
|
|
415
|
-
if (ggml_cuda_info().devices[device].vmm) {
|
|
416
|
-
return std::unique_ptr<ggml_cuda_pool>(new ggml_cuda_pool_vmm(device));
|
|
417
|
-
}
|
|
418
|
-
#endif
|
|
419
|
-
return std::unique_ptr<ggml_cuda_pool>(new ggml_cuda_pool_leg(device));
|
|
420
|
-
}
|
|
421
|
-
|
|
422
|
-
// cuda buffer
|
|
423
|
-
|
|
424
|
-
struct ggml_backend_cuda_buffer_context {
|
|
425
|
-
int device;
|
|
426
|
-
void * dev_ptr = nullptr;
|
|
427
|
-
std::string name;
|
|
428
|
-
|
|
429
|
-
ggml_backend_cuda_buffer_context(int device, void * dev_ptr) :
|
|
430
|
-
device(device), dev_ptr(dev_ptr),
|
|
431
|
-
name(GGML_CUDA_NAME + std::to_string(device)) {
|
|
432
|
-
}
|
|
433
|
-
|
|
434
|
-
~ggml_backend_cuda_buffer_context() {
|
|
435
|
-
CUDA_CHECK(cudaFree(dev_ptr));
|
|
436
|
-
}
|
|
437
|
-
};
|
|
438
|
-
|
|
439
|
-
GGML_CALL static const char * ggml_backend_cuda_buffer_get_name(ggml_backend_buffer_t buffer) {
|
|
440
|
-
ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;
|
|
441
|
-
return ctx->name.c_str();
|
|
442
|
-
}
|
|
443
|
-
|
|
444
|
-
GGML_CALL static bool ggml_backend_buffer_is_cuda(ggml_backend_buffer_t buffer) {
|
|
445
|
-
return buffer->iface.get_name == ggml_backend_cuda_buffer_get_name;
|
|
446
|
-
}
|
|
447
|
-
|
|
448
|
-
GGML_CALL static void ggml_backend_cuda_buffer_free_buffer(ggml_backend_buffer_t buffer) {
|
|
449
|
-
ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;
|
|
450
|
-
delete ctx;
|
|
451
|
-
}
|
|
452
|
-
|
|
453
|
-
GGML_CALL static void * ggml_backend_cuda_buffer_get_base(ggml_backend_buffer_t buffer) {
|
|
454
|
-
ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;
|
|
455
|
-
return ctx->dev_ptr;
|
|
456
|
-
}
|
|
457
|
-
|
|
458
|
-
GGML_CALL static void ggml_backend_cuda_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) {
|
|
459
|
-
ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;
|
|
460
|
-
|
|
461
|
-
if (tensor->view_src != NULL) {
|
|
462
|
-
assert(tensor->view_src->buffer->buft == buffer->buft);
|
|
463
|
-
return;
|
|
464
|
-
}
|
|
465
|
-
|
|
466
|
-
if (ggml_is_quantized(tensor->type)) {
|
|
467
|
-
// initialize padding to 0 to avoid possible NaN values
|
|
468
|
-
size_t original_size = ggml_nbytes(tensor);
|
|
469
|
-
size_t padded_size = ggml_backend_buft_get_alloc_size(buffer->buft, tensor);
|
|
470
|
-
|
|
471
|
-
if (padded_size > original_size && tensor->view_src == nullptr) {
|
|
472
|
-
ggml_cuda_set_device(ctx->device);
|
|
473
|
-
CUDA_CHECK(cudaMemset((char *)tensor->data + original_size, 0, padded_size - original_size));
|
|
474
|
-
}
|
|
475
|
-
}
|
|
476
|
-
}
|
|
477
|
-
|
|
478
|
-
GGML_CALL static void ggml_backend_cuda_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) {
|
|
479
|
-
ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;
|
|
480
|
-
|
|
481
|
-
ggml_cuda_set_device(ctx->device);
|
|
482
|
-
CUDA_CHECK(cudaMemcpyAsync((char *)tensor->data + offset, data, size, cudaMemcpyHostToDevice, cudaStreamPerThread));
|
|
483
|
-
CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));
|
|
484
|
-
}
|
|
485
|
-
|
|
486
|
-
GGML_CALL static void ggml_backend_cuda_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) {
|
|
487
|
-
ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;
|
|
488
|
-
|
|
489
|
-
ggml_cuda_set_device(ctx->device);
|
|
490
|
-
CUDA_CHECK(cudaMemcpyAsync(data, (const char *)tensor->data + offset, size, cudaMemcpyDeviceToHost, cudaStreamPerThread));
|
|
491
|
-
CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));
|
|
492
|
-
}
|
|
493
|
-
|
|
494
|
-
GGML_CALL static bool ggml_backend_cuda_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * src, ggml_tensor * dst) {
|
|
495
|
-
if (ggml_backend_buffer_is_cuda(src->buffer)) {
|
|
496
|
-
ggml_backend_cuda_buffer_context * src_ctx = (ggml_backend_cuda_buffer_context *)src->buffer->context;
|
|
497
|
-
ggml_backend_cuda_buffer_context * dst_ctx = (ggml_backend_cuda_buffer_context *)dst->buffer->context;
|
|
498
|
-
if (src_ctx->device == dst_ctx->device) {
|
|
499
|
-
CUDA_CHECK(cudaMemcpyAsync(dst->data, src->data, ggml_nbytes(src), cudaMemcpyDeviceToDevice, cudaStreamPerThread));
|
|
500
|
-
} else {
|
|
501
|
-
#ifdef GGML_CUDA_NO_PEER_COPY
|
|
502
|
-
return false;
|
|
503
|
-
#else
|
|
504
|
-
CUDA_CHECK(cudaMemcpyPeerAsync(dst->data, dst_ctx->device, src->data, src_ctx->device, ggml_nbytes(src), cudaStreamPerThread));
|
|
505
|
-
#endif
|
|
506
|
-
}
|
|
507
|
-
CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));
|
|
508
|
-
return true;
|
|
509
|
-
}
|
|
510
|
-
return false;
|
|
511
|
-
|
|
512
|
-
GGML_UNUSED(buffer);
|
|
513
|
-
}
|
|
514
|
-
|
|
515
|
-
GGML_CALL static void ggml_backend_cuda_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {
|
|
516
|
-
ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;
|
|
517
|
-
|
|
518
|
-
ggml_cuda_set_device(ctx->device);
|
|
519
|
-
CUDA_CHECK(cudaDeviceSynchronize());
|
|
520
|
-
CUDA_CHECK(cudaMemset(ctx->dev_ptr, value, buffer->size));
|
|
521
|
-
CUDA_CHECK(cudaDeviceSynchronize());
|
|
522
|
-
}
|
|
523
|
-
|
|
524
|
-
static ggml_backend_buffer_i ggml_backend_cuda_buffer_interface = {
|
|
525
|
-
/* .get_name = */ ggml_backend_cuda_buffer_get_name,
|
|
526
|
-
/* .free_buffer = */ ggml_backend_cuda_buffer_free_buffer,
|
|
527
|
-
/* .get_base = */ ggml_backend_cuda_buffer_get_base,
|
|
528
|
-
/* .init_tensor = */ ggml_backend_cuda_buffer_init_tensor,
|
|
529
|
-
/* .set_tensor = */ ggml_backend_cuda_buffer_set_tensor,
|
|
530
|
-
/* .get_tensor = */ ggml_backend_cuda_buffer_get_tensor,
|
|
531
|
-
/* .cpy_tensor = */ ggml_backend_cuda_buffer_cpy_tensor,
|
|
532
|
-
/* .clear = */ ggml_backend_cuda_buffer_clear,
|
|
533
|
-
/* .reset = */ NULL,
|
|
534
|
-
};
|
|
535
|
-
|
|
536
|
-
// cuda buffer type
|
|
537
|
-
struct ggml_backend_cuda_buffer_type_context {
|
|
538
|
-
int device;
|
|
539
|
-
std::string name;
|
|
540
|
-
};
|
|
541
|
-
|
|
542
|
-
GGML_CALL static const char * ggml_backend_cuda_buffer_type_name(ggml_backend_buffer_type_t buft) {
|
|
543
|
-
ggml_backend_cuda_buffer_type_context * ctx = (ggml_backend_cuda_buffer_type_context *)buft->context;
|
|
544
|
-
|
|
545
|
-
return ctx->name.c_str();
|
|
546
|
-
}
|
|
547
|
-
|
|
548
|
-
static bool ggml_backend_buft_is_cuda(ggml_backend_buffer_type_t buft) {
|
|
549
|
-
return buft->iface.get_name == ggml_backend_cuda_buffer_type_name;
|
|
550
|
-
}
|
|
551
|
-
|
|
552
|
-
GGML_CALL static ggml_backend_buffer_t ggml_backend_cuda_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {
|
|
553
|
-
ggml_backend_cuda_buffer_type_context * buft_ctx = (ggml_backend_cuda_buffer_type_context *)buft->context;
|
|
554
|
-
|
|
555
|
-
ggml_cuda_set_device(buft_ctx->device);
|
|
556
|
-
|
|
557
|
-
size = std::max(size, (size_t)1); // cudaMalloc returns null for size 0
|
|
558
|
-
|
|
559
|
-
void * dev_ptr;
|
|
560
|
-
cudaError_t err = ggml_cuda_device_malloc(&dev_ptr, size, buft_ctx->device);
|
|
561
|
-
if (err != cudaSuccess) {
|
|
562
|
-
// clear the error
|
|
563
|
-
cudaGetLastError();
|
|
564
|
-
GGML_CUDA_LOG_ERROR("%s: allocating %.2f MiB on device %d: cudaMalloc failed: %s\n", __func__, size / 1024.0 / 1024.0, buft_ctx->device, cudaGetErrorString(err));
|
|
565
|
-
return nullptr;
|
|
566
|
-
}
|
|
567
|
-
|
|
568
|
-
ggml_backend_cuda_buffer_context * ctx = new ggml_backend_cuda_buffer_context(buft_ctx->device, dev_ptr);
|
|
569
|
-
|
|
570
|
-
return ggml_backend_buffer_init(buft, ggml_backend_cuda_buffer_interface, ctx, size);
|
|
571
|
-
}
|
|
572
|
-
|
|
573
|
-
GGML_CALL static size_t ggml_backend_cuda_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) {
|
|
574
|
-
return 128;
|
|
575
|
-
|
|
576
|
-
GGML_UNUSED(buft);
|
|
577
|
-
}
|
|
578
|
-
|
|
579
|
-
GGML_CALL static size_t ggml_backend_cuda_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) {
|
|
580
|
-
size_t size = ggml_nbytes(tensor);
|
|
581
|
-
int64_t ne0 = tensor->ne[0];
|
|
582
|
-
|
|
583
|
-
if (ggml_is_quantized(tensor->type)) {
|
|
584
|
-
if (ne0 % MATRIX_ROW_PADDING != 0) {
|
|
585
|
-
size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING);
|
|
586
|
-
}
|
|
587
|
-
}
|
|
588
|
-
|
|
589
|
-
return size;
|
|
590
|
-
|
|
591
|
-
GGML_UNUSED(buft);
|
|
592
|
-
}
|
|
593
|
-
|
|
594
|
-
static ggml_backend_buffer_type_i ggml_backend_cuda_buffer_type_interface = {
|
|
595
|
-
/* .get_name = */ ggml_backend_cuda_buffer_type_name,
|
|
596
|
-
/* .alloc_buffer = */ ggml_backend_cuda_buffer_type_alloc_buffer,
|
|
597
|
-
/* .get_alignment = */ ggml_backend_cuda_buffer_type_get_alignment,
|
|
598
|
-
/* .get_max_size = */ NULL, // defaults to SIZE_MAX
|
|
599
|
-
/* .get_alloc_size = */ ggml_backend_cuda_buffer_type_get_alloc_size,
|
|
600
|
-
/* .is_host = */ NULL,
|
|
601
|
-
};
|
|
602
|
-
|
|
603
|
-
GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_buffer_type(int device) {
|
|
604
|
-
static std::mutex mutex;
|
|
605
|
-
std::lock_guard<std::mutex> lock(mutex);
|
|
606
|
-
|
|
607
|
-
if (device >= ggml_backend_cuda_get_device_count()) {
|
|
608
|
-
return nullptr;
|
|
609
|
-
}
|
|
610
|
-
|
|
611
|
-
static ggml_backend_buffer_type ggml_backend_cuda_buffer_types[GGML_CUDA_MAX_DEVICES];
|
|
612
|
-
|
|
613
|
-
static bool ggml_backend_cuda_buffer_type_initialized = false;
|
|
614
|
-
|
|
615
|
-
if (!ggml_backend_cuda_buffer_type_initialized) {
|
|
616
|
-
for (int i = 0; i < GGML_CUDA_MAX_DEVICES; i++) {
|
|
617
|
-
ggml_backend_cuda_buffer_types[i] = {
|
|
618
|
-
/* .iface = */ ggml_backend_cuda_buffer_type_interface,
|
|
619
|
-
/* .context = */ new ggml_backend_cuda_buffer_type_context{i, GGML_CUDA_NAME + std::to_string(i)},
|
|
620
|
-
};
|
|
621
|
-
}
|
|
622
|
-
ggml_backend_cuda_buffer_type_initialized = true;
|
|
623
|
-
}
|
|
624
|
-
|
|
625
|
-
return &ggml_backend_cuda_buffer_types[device];
|
|
626
|
-
}
|
|
627
|
-
|
|
628
|
-
// cuda split buffer
|
|
629
|
-
|
|
630
|
-
static int64_t get_row_rounding(const std::array<float, GGML_CUDA_MAX_DEVICES> & tensor_split) {
|
|
631
|
-
int64_t row_rounding = 0;
|
|
632
|
-
for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) {
|
|
633
|
-
if (tensor_split[id] >= (id + 1 < ggml_backend_cuda_get_device_count() ? tensor_split[id + 1] : 1.0f)) {
|
|
634
|
-
continue;
|
|
635
|
-
}
|
|
636
|
-
|
|
637
|
-
const int cc = ggml_cuda_info().devices[id].cc;
|
|
638
|
-
row_rounding = std::max(row_rounding, (int64_t)get_mmq_y_host(cc, get_mmq_x_max_host(cc)));
|
|
639
|
-
}
|
|
640
|
-
return row_rounding;
|
|
641
|
-
}
|
|
642
|
-
|
|
643
|
-
static void get_row_split(int64_t * row_low, int64_t * row_high, const ggml_tensor * tensor, const std::array<float, GGML_CUDA_MAX_DEVICES> & tensor_split, int id) {
|
|
644
|
-
const int64_t nrows = ggml_nrows(tensor);
|
|
645
|
-
const int64_t rounding = get_row_rounding(tensor_split);
|
|
646
|
-
|
|
647
|
-
*row_low = id == 0 ? 0 : nrows*tensor_split[id];
|
|
648
|
-
*row_low -= *row_low % rounding;
|
|
649
|
-
|
|
650
|
-
if (id == ggml_backend_cuda_get_device_count() - 1) {
|
|
651
|
-
*row_high = nrows;
|
|
652
|
-
} else {
|
|
653
|
-
*row_high = nrows*tensor_split[id + 1];
|
|
654
|
-
*row_high -= *row_high % rounding;
|
|
655
|
-
}
|
|
656
|
-
}
|
|
657
|
-
|
|
658
|
-
static size_t ggml_nbytes_split(const struct ggml_tensor * tensor, int nrows_split) {
|
|
659
|
-
static_assert(GGML_MAX_DIMS == 4, "GGML_MAX_DIMS is not 4 - update this function");
|
|
660
|
-
|
|
661
|
-
return nrows_split*ggml_row_size(tensor->type, tensor->ne[0]);
|
|
662
|
-
}
|
|
663
|
-
|
|
664
|
-
struct ggml_backend_cuda_split_buffer_type_context {
|
|
665
|
-
std::array<float, GGML_CUDA_MAX_DEVICES> tensor_split;
|
|
666
|
-
};
|
|
667
|
-
|
|
668
|
-
struct ggml_backend_cuda_split_buffer_context {
|
|
669
|
-
~ggml_backend_cuda_split_buffer_context() {
|
|
670
|
-
for (ggml_tensor_extra_gpu * extra : tensor_extras) {
|
|
671
|
-
for (int id = 0; id < GGML_CUDA_MAX_DEVICES; ++id) {
|
|
672
|
-
for (int64_t is = 0; is < GGML_CUDA_MAX_STREAMS; ++is) {
|
|
673
|
-
if (extra->events[id][is] != nullptr) {
|
|
674
|
-
CUDA_CHECK(cudaEventDestroy(extra->events[id][is]));
|
|
675
|
-
}
|
|
676
|
-
}
|
|
677
|
-
if (extra->data_device[id] != nullptr) {
|
|
678
|
-
CUDA_CHECK(cudaFree(extra->data_device[id]));
|
|
679
|
-
}
|
|
680
|
-
}
|
|
681
|
-
delete extra;
|
|
682
|
-
}
|
|
683
|
-
}
|
|
684
|
-
|
|
685
|
-
std::vector<ggml_tensor_extra_gpu *> tensor_extras;
|
|
686
|
-
};
|
|
687
|
-
|
|
688
|
-
GGML_CALL static const char * ggml_backend_cuda_split_buffer_get_name(ggml_backend_buffer_t buffer) {
|
|
689
|
-
return GGML_CUDA_NAME "_Split";
|
|
690
|
-
|
|
691
|
-
GGML_UNUSED(buffer);
|
|
692
|
-
}
|
|
693
|
-
|
|
694
|
-
static bool ggml_backend_buffer_is_cuda_split(ggml_backend_buffer_t buffer) {
|
|
695
|
-
return buffer->iface.get_name == ggml_backend_cuda_split_buffer_get_name;
|
|
696
|
-
GGML_UNUSED(ggml_backend_buffer_is_cuda_split); // only used in debug builds currently, avoid unused function warning in release builds
|
|
697
|
-
}
|
|
698
|
-
|
|
699
|
-
GGML_CALL static void ggml_backend_cuda_split_buffer_free_buffer(ggml_backend_buffer_t buffer) {
|
|
700
|
-
ggml_backend_cuda_split_buffer_context * ctx = (ggml_backend_cuda_split_buffer_context *)buffer->context;
|
|
701
|
-
delete ctx;
|
|
702
|
-
}
|
|
703
|
-
|
|
704
|
-
GGML_CALL static void * ggml_backend_cuda_split_buffer_get_base(ggml_backend_buffer_t buffer) {
|
|
705
|
-
// the pointers are stored in the tensor extras, this is just a dummy address and never dereferenced
|
|
706
|
-
return (void *)0x1000;
|
|
707
|
-
|
|
708
|
-
GGML_UNUSED(buffer);
|
|
709
|
-
}
|
|
710
|
-
|
|
711
|
-
GGML_CALL static void ggml_backend_cuda_split_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) {
|
|
712
|
-
GGML_ASSERT(tensor->view_src == nullptr); // views of split tensors are not supported
|
|
713
|
-
|
|
714
|
-
ggml_backend_cuda_split_buffer_context * ctx = (ggml_backend_cuda_split_buffer_context *)buffer->context;
|
|
715
|
-
ggml_backend_cuda_split_buffer_type_context * buft_ctx = (ggml_backend_cuda_split_buffer_type_context *)buffer->buft->context;
|
|
716
|
-
|
|
717
|
-
const int64_t ne0 = tensor->ne[0];
|
|
718
|
-
|
|
719
|
-
ggml_tensor_extra_gpu * extra = new ggml_tensor_extra_gpu{};
|
|
720
|
-
ctx->tensor_extras.push_back(extra);
|
|
721
|
-
|
|
722
|
-
for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) {
|
|
723
|
-
int64_t row_low, row_high;
|
|
724
|
-
get_row_split(&row_low, &row_high, tensor, buft_ctx->tensor_split, id);
|
|
725
|
-
|
|
726
|
-
int64_t nrows_split = row_high - row_low;
|
|
727
|
-
if (nrows_split == 0) {
|
|
728
|
-
continue;
|
|
729
|
-
}
|
|
730
|
-
|
|
731
|
-
size_t size = ggml_nbytes_split(tensor, nrows_split);
|
|
732
|
-
const size_t original_size = size;
|
|
733
|
-
|
|
734
|
-
// pad last row to a multiple of 512 elements to avoid out-of-bounds memory accesses
|
|
735
|
-
if (ne0 % MATRIX_ROW_PADDING != 0) {
|
|
736
|
-
size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING);
|
|
737
|
-
}
|
|
738
|
-
|
|
739
|
-
// FIXME: do not crash if cudaMalloc fails
|
|
740
|
-
// currently, init_tensor cannot fail, it needs to be fixed in ggml-backend first
|
|
741
|
-
ggml_cuda_set_device(id);
|
|
742
|
-
char * buf;
|
|
743
|
-
CUDA_CHECK(ggml_cuda_device_malloc((void**)&buf, size, id));
|
|
744
|
-
|
|
745
|
-
// set padding to 0 to avoid possible NaN values
|
|
746
|
-
if (size > original_size) {
|
|
747
|
-
CUDA_CHECK(cudaMemset(buf + original_size, 0, size - original_size));
|
|
748
|
-
}
|
|
749
|
-
|
|
750
|
-
extra->data_device[id] = buf;
|
|
751
|
-
|
|
752
|
-
for (int64_t is = 0; is < GGML_CUDA_MAX_STREAMS; ++is) {
|
|
753
|
-
CUDA_CHECK(cudaEventCreateWithFlags(&extra->events[id][is], cudaEventDisableTiming));
|
|
754
|
-
}
|
|
755
|
-
}
|
|
756
|
-
tensor->extra = extra;
|
|
757
|
-
}
|
|
758
|
-
|
|
759
|
-
GGML_CALL static void ggml_backend_cuda_split_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) {
|
|
760
|
-
// split tensors must always be set in their entirety at once
|
|
761
|
-
GGML_ASSERT(offset == 0);
|
|
762
|
-
GGML_ASSERT(size == ggml_nbytes(tensor));
|
|
763
|
-
|
|
764
|
-
ggml_backend_cuda_split_buffer_type_context * buft_ctx = (ggml_backend_cuda_split_buffer_type_context *)buffer->buft->context;
|
|
765
|
-
|
|
766
|
-
const int64_t ne0 = tensor->ne[0];
|
|
767
|
-
const size_t nb1 = tensor->nb[1];
|
|
768
|
-
ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *)tensor->extra;
|
|
769
|
-
|
|
770
|
-
for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) {
|
|
771
|
-
int64_t row_low, row_high;
|
|
772
|
-
get_row_split(&row_low, &row_high, tensor, buft_ctx->tensor_split, id);
|
|
773
|
-
|
|
774
|
-
int64_t nrows_split = row_high - row_low;
|
|
775
|
-
if (nrows_split == 0) {
|
|
776
|
-
continue;
|
|
777
|
-
}
|
|
778
|
-
|
|
779
|
-
const size_t offset_split = row_low*nb1;
|
|
780
|
-
size_t size = ggml_nbytes_split(tensor, nrows_split);
|
|
781
|
-
const size_t original_size = size;
|
|
782
|
-
|
|
783
|
-
// pad last row to a multiple of 512 elements to avoid out-of-bounds memory accesses
|
|
784
|
-
if (ne0 % MATRIX_ROW_PADDING != 0) {
|
|
785
|
-
size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING);
|
|
786
|
-
}
|
|
787
|
-
|
|
788
|
-
const char * buf_host = (const char *)data + offset_split;
|
|
789
|
-
CUDA_CHECK(cudaMemcpyAsync(extra->data_device[id], buf_host, original_size, cudaMemcpyHostToDevice, cudaStreamPerThread));
|
|
790
|
-
}
|
|
791
|
-
|
|
792
|
-
for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) {
|
|
793
|
-
CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));
|
|
794
|
-
}
|
|
795
|
-
}
|
|
796
|
-
|
|
797
|
-
GGML_CALL static void ggml_backend_cuda_split_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) {
|
|
798
|
-
// split tensors must always be set in their entirety at once
|
|
799
|
-
GGML_ASSERT(offset == 0);
|
|
800
|
-
GGML_ASSERT(size == ggml_nbytes(tensor));
|
|
801
|
-
|
|
802
|
-
ggml_backend_cuda_split_buffer_type_context * buft_ctx = (ggml_backend_cuda_split_buffer_type_context *)buffer->buft->context;
|
|
803
|
-
|
|
804
|
-
const int64_t ne0 = tensor->ne[0];
|
|
805
|
-
const size_t nb1 = tensor->nb[1];
|
|
806
|
-
ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *)tensor->extra;
|
|
807
|
-
|
|
808
|
-
for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) {
|
|
809
|
-
int64_t row_low, row_high;
|
|
810
|
-
get_row_split(&row_low, &row_high, tensor, buft_ctx->tensor_split, id);
|
|
811
|
-
|
|
812
|
-
int64_t nrows_split = row_high - row_low;
|
|
813
|
-
if (nrows_split == 0) {
|
|
814
|
-
continue;
|
|
815
|
-
}
|
|
816
|
-
|
|
817
|
-
const size_t offset_split = row_low*nb1;
|
|
818
|
-
size_t size = ggml_nbytes_split(tensor, nrows_split);
|
|
819
|
-
const size_t original_size = size;
|
|
820
|
-
|
|
821
|
-
// pad last row to a multiple of 512 elements to avoid out-of-bounds memory accesses
|
|
822
|
-
if (ne0 % MATRIX_ROW_PADDING != 0) {
|
|
823
|
-
size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING);
|
|
824
|
-
}
|
|
825
|
-
|
|
826
|
-
char * buf_host = (char *)data + offset_split;
|
|
827
|
-
CUDA_CHECK(cudaMemcpyAsync(buf_host, extra->data_device[id], original_size, cudaMemcpyDeviceToHost, cudaStreamPerThread));
|
|
828
|
-
}
|
|
829
|
-
|
|
830
|
-
for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) {
|
|
831
|
-
CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));
|
|
832
|
-
}
|
|
833
|
-
}
|
|
834
|
-
|
|
835
|
-
GGML_CALL static void ggml_backend_cuda_split_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {
|
|
836
|
-
GGML_UNUSED(buffer);
|
|
837
|
-
GGML_UNUSED(value);
|
|
838
|
-
}
|
|
839
|
-
|
|
840
|
-
static struct ggml_backend_buffer_i ggml_backend_cuda_split_buffer_interface = {
|
|
841
|
-
/* .get_name = */ ggml_backend_cuda_split_buffer_get_name,
|
|
842
|
-
/* .free_buffer = */ ggml_backend_cuda_split_buffer_free_buffer,
|
|
843
|
-
/* .get_base = */ ggml_backend_cuda_split_buffer_get_base,
|
|
844
|
-
/* .init_tensor = */ ggml_backend_cuda_split_buffer_init_tensor,
|
|
845
|
-
/* .set_tensor = */ ggml_backend_cuda_split_buffer_set_tensor,
|
|
846
|
-
/* .get_tensor = */ ggml_backend_cuda_split_buffer_get_tensor,
|
|
847
|
-
/* .cpy_tensor = */ NULL,
|
|
848
|
-
/* .clear = */ ggml_backend_cuda_split_buffer_clear,
|
|
849
|
-
/* .reset = */ NULL,
|
|
850
|
-
};
|
|
851
|
-
|
|
852
|
-
// cuda split buffer type
|
|
853
|
-
|
|
854
|
-
GGML_CALL static const char * ggml_backend_cuda_split_buffer_type_name(ggml_backend_buffer_type_t buft) {
|
|
855
|
-
return GGML_CUDA_NAME "_Split";
|
|
856
|
-
|
|
857
|
-
GGML_UNUSED(buft);
|
|
858
|
-
}
|
|
859
|
-
|
|
860
|
-
static bool ggml_backend_buft_is_cuda_split(ggml_backend_buffer_type_t buft) {
|
|
861
|
-
return buft->iface.get_name == ggml_backend_cuda_split_buffer_type_name;
|
|
862
|
-
}
|
|
863
|
-
|
|
864
|
-
GGML_CALL static ggml_backend_buffer_t ggml_backend_cuda_split_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {
|
|
865
|
-
// since we don't know the exact split after rounding, we cannot allocate the device buffers at this point
|
|
866
|
-
// instead, we allocate them for each tensor separately in init_tensor
|
|
867
|
-
// however, the size still represents the maximum cumulative size of all the device buffers after the tensors are allocated,
|
|
868
|
-
// as returned by get_alloc_size. this limit is enforced during tensor allocation by ggml-alloc, so it must be correct.
|
|
869
|
-
ggml_backend_cuda_split_buffer_context * ctx = new ggml_backend_cuda_split_buffer_context();
|
|
870
|
-
|
|
871
|
-
return ggml_backend_buffer_init(buft, ggml_backend_cuda_split_buffer_interface, ctx, size);
|
|
872
|
-
}
|
|
873
|
-
|
|
874
|
-
GGML_CALL static size_t ggml_backend_cuda_split_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) {
|
|
875
|
-
return 128;
|
|
876
|
-
|
|
877
|
-
GGML_UNUSED(buft);
|
|
878
|
-
}
|
|
879
|
-
|
|
880
|
-
GGML_CALL static size_t ggml_backend_cuda_split_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) {
|
|
881
|
-
ggml_backend_cuda_split_buffer_type_context * ctx = (ggml_backend_cuda_split_buffer_type_context *)buft->context;
|
|
882
|
-
|
|
883
|
-
size_t total_size = 0;
|
|
884
|
-
|
|
885
|
-
const int64_t ne0 = tensor->ne[0];
|
|
886
|
-
|
|
887
|
-
for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) {
|
|
888
|
-
int64_t row_low, row_high;
|
|
889
|
-
get_row_split(&row_low, &row_high, tensor, ctx->tensor_split, id);
|
|
890
|
-
|
|
891
|
-
int64_t nrows_split = row_high - row_low;
|
|
892
|
-
if (nrows_split == 0) {
|
|
893
|
-
continue;
|
|
894
|
-
}
|
|
895
|
-
|
|
896
|
-
total_size += ggml_nbytes_split(tensor, nrows_split);
|
|
897
|
-
|
|
898
|
-
// pad last row to a multiple of 512 elements to avoid out-of-bounds memory accesses
|
|
899
|
-
if (ne0 % MATRIX_ROW_PADDING != 0) {
|
|
900
|
-
total_size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING);
|
|
901
|
-
}
|
|
902
|
-
}
|
|
903
|
-
|
|
904
|
-
return total_size;
|
|
905
|
-
}
|
|
906
|
-
|
|
907
|
-
GGML_CALL static bool ggml_backend_cuda_split_buffer_type_is_host(ggml_backend_buffer_type_t buft) {
|
|
908
|
-
return false;
|
|
909
|
-
|
|
910
|
-
GGML_UNUSED(buft);
|
|
911
|
-
}
|
|
912
|
-
|
|
913
|
-
static ggml_backend_buffer_type_i ggml_backend_cuda_split_buffer_type_interface = {
|
|
914
|
-
/* .get_name = */ ggml_backend_cuda_split_buffer_type_name,
|
|
915
|
-
/* .alloc_buffer = */ ggml_backend_cuda_split_buffer_type_alloc_buffer,
|
|
916
|
-
/* .get_alignment = */ ggml_backend_cuda_split_buffer_type_get_alignment,
|
|
917
|
-
/* .get_max_size = */ NULL, // defaults to SIZE_MAX
|
|
918
|
-
/* .get_alloc_size = */ ggml_backend_cuda_split_buffer_type_get_alloc_size,
|
|
919
|
-
/* .is_host = */ ggml_backend_cuda_split_buffer_type_is_host,
|
|
920
|
-
};
|
|
921
|
-
|
|
922
|
-
GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_split_buffer_type(const float * tensor_split) {
|
|
923
|
-
static std::mutex mutex;
|
|
924
|
-
std::lock_guard<std::mutex> lock(mutex);
|
|
925
|
-
|
|
926
|
-
static std::map<std::array<float, GGML_CUDA_MAX_DEVICES>, struct ggml_backend_buffer_type> buft_map;
|
|
927
|
-
|
|
928
|
-
std::array<float, GGML_CUDA_MAX_DEVICES> tensor_split_arr = {};
|
|
929
|
-
|
|
930
|
-
bool all_zero = tensor_split == nullptr || std::all_of(tensor_split, tensor_split + GGML_CUDA_MAX_DEVICES, [](float x) { return x == 0.0f; });
|
|
931
|
-
if (all_zero) {
|
|
932
|
-
tensor_split_arr = ggml_cuda_info().default_tensor_split;
|
|
933
|
-
} else {
|
|
934
|
-
float split_sum = 0.0f;
|
|
935
|
-
for (int i = 0; i < ggml_backend_cuda_get_device_count(); ++i) {
|
|
936
|
-
tensor_split_arr[i] = split_sum;
|
|
937
|
-
split_sum += tensor_split[i];
|
|
938
|
-
}
|
|
939
|
-
for (int i = 0; i < ggml_backend_cuda_get_device_count(); ++i) {
|
|
940
|
-
tensor_split_arr[i] /= split_sum;
|
|
941
|
-
}
|
|
942
|
-
}
|
|
943
|
-
|
|
944
|
-
auto it = buft_map.find(tensor_split_arr);
|
|
945
|
-
if (it != buft_map.end()) {
|
|
946
|
-
return &it->second;
|
|
947
|
-
}
|
|
948
|
-
|
|
949
|
-
struct ggml_backend_buffer_type buft {
|
|
950
|
-
/* .iface = */ ggml_backend_cuda_split_buffer_type_interface,
|
|
951
|
-
/* .context = */ new ggml_backend_cuda_split_buffer_type_context{tensor_split_arr},
|
|
952
|
-
};
|
|
953
|
-
|
|
954
|
-
auto result = buft_map.emplace(tensor_split_arr, buft);
|
|
955
|
-
return &result.first->second;
|
|
956
|
-
}
|
|
957
|
-
|
|
958
|
-
// host buffer type
|
|
959
|
-
|
|
960
|
-
GGML_CALL static const char * ggml_backend_cuda_host_buffer_type_name(ggml_backend_buffer_type_t buft) {
|
|
961
|
-
return GGML_CUDA_NAME "_Host";
|
|
962
|
-
|
|
963
|
-
GGML_UNUSED(buft);
|
|
964
|
-
}
|
|
965
|
-
|
|
966
|
-
GGML_CALL static const char * ggml_backend_cuda_host_buffer_name(ggml_backend_buffer_t buffer) {
|
|
967
|
-
return GGML_CUDA_NAME "_Host";
|
|
968
|
-
|
|
969
|
-
GGML_UNUSED(buffer);
|
|
970
|
-
}
|
|
971
|
-
|
|
972
|
-
GGML_CALL static void ggml_backend_cuda_host_buffer_free_buffer(ggml_backend_buffer_t buffer) {
|
|
973
|
-
CUDA_CHECK(cudaFreeHost(buffer->context));
|
|
974
|
-
}
|
|
975
|
-
|
|
976
|
-
static void * ggml_cuda_host_malloc(size_t size) {
|
|
977
|
-
if (getenv("GGML_CUDA_NO_PINNED") != nullptr) {
|
|
978
|
-
return nullptr;
|
|
979
|
-
}
|
|
980
|
-
|
|
981
|
-
void * ptr = nullptr;
|
|
982
|
-
cudaError_t err = cudaMallocHost((void **) &ptr, size);
|
|
983
|
-
if (err != cudaSuccess) {
|
|
984
|
-
// clear the error
|
|
985
|
-
cudaGetLastError();
|
|
986
|
-
GGML_CUDA_LOG_WARN("%s: failed to allocate %.2f MiB of pinned memory: %s\n", __func__,
|
|
987
|
-
size / 1024.0 / 1024.0, cudaGetErrorString(err));
|
|
988
|
-
return nullptr;
|
|
989
|
-
}
|
|
990
|
-
|
|
991
|
-
return ptr;
|
|
992
|
-
}
|
|
993
|
-
|
|
994
|
-
GGML_CALL static ggml_backend_buffer_t ggml_backend_cuda_host_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {
|
|
995
|
-
void * ptr = ggml_cuda_host_malloc(size);
|
|
996
|
-
|
|
997
|
-
if (ptr == nullptr) {
|
|
998
|
-
// fallback to cpu buffer
|
|
999
|
-
return ggml_backend_buft_alloc_buffer(ggml_backend_cpu_buffer_type(), size);
|
|
1000
|
-
}
|
|
1001
|
-
|
|
1002
|
-
ggml_backend_buffer_t buffer = ggml_backend_cpu_buffer_from_ptr(ptr, size);
|
|
1003
|
-
buffer->buft = buft;
|
|
1004
|
-
buffer->iface.get_name = ggml_backend_cuda_host_buffer_name;
|
|
1005
|
-
buffer->iface.free_buffer = ggml_backend_cuda_host_buffer_free_buffer;
|
|
1006
|
-
|
|
1007
|
-
return buffer;
|
|
1008
|
-
}
|
|
1009
|
-
|
|
1010
|
-
GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_host_buffer_type() {
|
|
1011
|
-
static struct ggml_backend_buffer_type ggml_backend_cuda_buffer_type_host = {
|
|
1012
|
-
/* .iface = */ {
|
|
1013
|
-
/* .get_name = */ ggml_backend_cuda_host_buffer_type_name,
|
|
1014
|
-
/* .alloc_buffer = */ ggml_backend_cuda_host_buffer_type_alloc_buffer,
|
|
1015
|
-
/* .get_alignment = */ ggml_backend_cpu_buffer_type()->iface.get_alignment,
|
|
1016
|
-
/* .get_max_size = */ NULL, // defaults to SIZE_MAX
|
|
1017
|
-
/* .get_alloc_size = */ ggml_backend_cpu_buffer_type()->iface.get_alloc_size,
|
|
1018
|
-
/* .is_host = */ ggml_backend_cpu_buffer_type()->iface.is_host,
|
|
1019
|
-
},
|
|
1020
|
-
/* .context = */ nullptr,
|
|
1021
|
-
};
|
|
1022
|
-
|
|
1023
|
-
return &ggml_backend_cuda_buffer_type_host;
|
|
1024
|
-
}
|
|
1025
|
-
|
|
1026
|
-
//static bool ggml_backend_buffer_is_cuda_host(ggml_backend_buffer_t buffer) {
|
|
1027
|
-
// return buffer->buft->iface.get_name == ggml_backend_cuda_host_buffer_type_name;
|
|
1028
|
-
//}
|
|
1029
|
-
|
|
1030
|
-
/// kernels
|
|
1031
|
-
|
|
1032
|
-
typedef void (*ggml_cuda_op_mul_mat_t)(
|
|
1033
|
-
ggml_backend_cuda_context & ctx,
|
|
1034
|
-
const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i,
|
|
1035
|
-
const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols,
|
|
1036
|
-
const int64_t src1_padded_row_size, cudaStream_t stream);
|
|
1037
|
-
|
|
1038
|
-
#ifndef GGML_CUDA_PEER_MAX_BATCH_SIZE
|
|
1039
|
-
#define GGML_CUDA_PEER_MAX_BATCH_SIZE 128
|
|
1040
|
-
#endif // GGML_CUDA_PEER_MAX_BATCH_SIZE
|
|
1041
|
-
|
|
1042
|
-
#define MUL_MAT_SRC1_COL_STRIDE 128
|
|
1043
|
-
|
|
1044
|
-
static __global__ void mul_mat_p021_f16_f32(
|
|
1045
|
-
const void * __restrict__ vx, const float * __restrict__ y, float * __restrict__ dst,
|
|
1046
|
-
const int ncols_x, const int nrows_x, const int nchannels_x, const int nchannels_y) {
|
|
1047
|
-
|
|
1048
|
-
const half * x = (const half *) vx;
|
|
1049
|
-
|
|
1050
|
-
const int row_x = blockDim.y*blockIdx.y + threadIdx.y;
|
|
1051
|
-
const int channel = blockDim.z*blockIdx.z + threadIdx.z;
|
|
1052
|
-
const int channel_x = channel / (nchannels_y / nchannels_x);
|
|
1053
|
-
|
|
1054
|
-
const int nrows_y = ncols_x;
|
|
1055
|
-
const int nrows_dst = nrows_x;
|
|
1056
|
-
const int row_dst = row_x;
|
|
1057
|
-
|
|
1058
|
-
float tmp = 0.0f;
|
|
1059
|
-
|
|
1060
|
-
for (int col_x0 = 0; col_x0 < ncols_x; col_x0 += blockDim.x) {
|
|
1061
|
-
const int col_x = col_x0 + threadIdx.x;
|
|
1062
|
-
|
|
1063
|
-
if (col_x >= ncols_x) {
|
|
1064
|
-
break;
|
|
1065
|
-
}
|
|
1066
|
-
|
|
1067
|
-
// x is transposed and permuted
|
|
1068
|
-
const int ix = row_x*nchannels_x*ncols_x + channel_x*ncols_x + col_x;
|
|
1069
|
-
const float xi = __half2float(x[ix]);
|
|
1070
|
-
|
|
1071
|
-
const int row_y = col_x;
|
|
1072
|
-
|
|
1073
|
-
// y is not transposed but permuted
|
|
1074
|
-
const int iy = channel*nrows_y + row_y;
|
|
1075
|
-
|
|
1076
|
-
tmp += xi * y[iy];
|
|
1077
|
-
}
|
|
1078
|
-
|
|
1079
|
-
// dst is not transposed and not permuted
|
|
1080
|
-
const int idst = channel*nrows_dst + row_dst;
|
|
1081
|
-
|
|
1082
|
-
// sum up partial sums and write back result
|
|
1083
|
-
tmp = warp_reduce_sum(tmp);
|
|
1084
|
-
|
|
1085
|
-
if (threadIdx.x == 0) {
|
|
1086
|
-
dst[idst] = tmp;
|
|
1087
|
-
}
|
|
1088
|
-
}
|
|
1089
|
-
|
|
1090
|
-
static __global__ void mul_mat_vec_nc_f16_f32( // nc == non-contiguous
|
|
1091
|
-
const void * __restrict__ vx, const float * __restrict__ y, float * __restrict__ dst, const int ncols_x, const int nrows_x,
|
|
1092
|
-
const int row_stride_x, const int channel_stride_x, const int channel_x_divisor) {
|
|
1093
|
-
|
|
1094
|
-
const half * x = (const half *) vx;
|
|
1095
|
-
|
|
1096
|
-
const int row_x = blockDim.y*blockIdx.y + threadIdx.y;
|
|
1097
|
-
const int channel = blockDim.z*blockIdx.z + threadIdx.z;
|
|
1098
|
-
const int channel_x = channel / channel_x_divisor;
|
|
1099
|
-
|
|
1100
|
-
const int nrows_y = ncols_x;
|
|
1101
|
-
const int nrows_dst = nrows_x;
|
|
1102
|
-
const int row_dst = row_x;
|
|
1103
|
-
|
|
1104
|
-
const int idst = channel*nrows_dst + row_dst;
|
|
1105
|
-
|
|
1106
|
-
float tmp = 0.0f;
|
|
1107
|
-
|
|
1108
|
-
for (int col_x0 = 0; col_x0 < ncols_x; col_x0 += blockDim.x) {
|
|
1109
|
-
const int col_x = col_x0 + threadIdx.x;
|
|
1110
|
-
|
|
1111
|
-
if (col_x >= ncols_x) {
|
|
1112
|
-
break;
|
|
1113
|
-
}
|
|
1114
|
-
|
|
1115
|
-
const int row_y = col_x;
|
|
1116
|
-
|
|
1117
|
-
const int ix = channel_x*channel_stride_x + row_x*row_stride_x + col_x;
|
|
1118
|
-
const int iy = channel*nrows_y + row_y;
|
|
1119
|
-
|
|
1120
|
-
const float xi = __half2float(x[ix]);
|
|
1121
|
-
|
|
1122
|
-
tmp += xi * y[iy];
|
|
1123
|
-
}
|
|
1124
|
-
|
|
1125
|
-
// sum up partial sums and write back result
|
|
1126
|
-
tmp = warp_reduce_sum(tmp);
|
|
1127
|
-
|
|
1128
|
-
if (threadIdx.x == 0) {
|
|
1129
|
-
dst[idst] = tmp;
|
|
1130
|
-
}
|
|
1131
|
-
}
|
|
1132
|
-
|
|
1133
|
-
static void ggml_mul_mat_p021_f16_f32_cuda(
|
|
1134
|
-
const void * vx, const float * y, float * dst, const int ncols_x, const int nrows_x,
|
|
1135
|
-
const int nchannels_x, const int nchannels_y, cudaStream_t stream) {
|
|
1136
|
-
|
|
1137
|
-
const dim3 block_nums(1, nrows_x, nchannels_y);
|
|
1138
|
-
const dim3 block_dims(WARP_SIZE, 1, 1);
|
|
1139
|
-
mul_mat_p021_f16_f32<<<block_nums, block_dims, 0, stream>>>(vx, y, dst, ncols_x, nrows_x, nchannels_x, nchannels_y);
|
|
1140
|
-
}
|
|
1141
|
-
|
|
1142
|
-
static void ggml_mul_mat_vec_nc_f16_f32_cuda(
|
|
1143
|
-
const void * vx, const float * y, float * dst, const int ncols_x, const int nrows_x, const int row_stride_x,
|
|
1144
|
-
const int nchannels_x, const int nchannels_y, const int channel_stride_x, cudaStream_t stream) {
|
|
1145
|
-
|
|
1146
|
-
const dim3 block_nums(1, nrows_x, nchannels_y);
|
|
1147
|
-
const dim3 block_dims(WARP_SIZE, 1, 1);
|
|
1148
|
-
mul_mat_vec_nc_f16_f32<<<block_nums, block_dims, 0, stream>>>
|
|
1149
|
-
(vx, y, dst, ncols_x, nrows_x, row_stride_x, channel_stride_x, nchannels_y/nchannels_x);
|
|
1150
|
-
}
|
|
1151
|
-
|
|
1152
|
-
static cudaError_t ggml_cuda_cpy_tensor_2d(
|
|
1153
|
-
void * dst, const struct ggml_tensor * src, int64_t i3, int64_t i2, int64_t i1_low, int64_t i1_high, cudaStream_t stream) {
|
|
1154
|
-
|
|
1155
|
-
GGML_ASSERT(ggml_backend_buffer_is_cuda(src->buffer));
|
|
1156
|
-
char * src_ptr = (char *) src->data;
|
|
1157
|
-
char * dst_ptr = (char *) dst;
|
|
1158
|
-
|
|
1159
|
-
const int64_t ne0 = src->ne[0];
|
|
1160
|
-
const int64_t nb0 = src->nb[0];
|
|
1161
|
-
const int64_t nb1 = src->nb[1];
|
|
1162
|
-
const int64_t nb2 = src->nb[2];
|
|
1163
|
-
const int64_t nb3 = src->nb[3];
|
|
1164
|
-
const enum ggml_type type = src->type;
|
|
1165
|
-
const int64_t ts = ggml_type_size(type);
|
|
1166
|
-
const int64_t bs = ggml_blck_size(type);
|
|
1167
|
-
int64_t i1_diff = i1_high - i1_low;
|
|
1168
|
-
|
|
1169
|
-
const char * x = src_ptr + i1_low*nb1 + i2*nb2 + i3*nb3;
|
|
1170
|
-
if (nb0 == ts && nb1 == ts*ne0/bs) {
|
|
1171
|
-
return cudaMemcpyAsync(dst_ptr, x, i1_diff*nb1, cudaMemcpyDeviceToDevice, stream);
|
|
1172
|
-
} else if (nb0 == ts) {
|
|
1173
|
-
return cudaMemcpy2DAsync(dst_ptr, ts*ne0/bs, x, nb1, ts*ne0/bs, i1_diff, cudaMemcpyDeviceToDevice, stream);
|
|
1174
|
-
} else {
|
|
1175
|
-
for (int64_t i1 = 0; i1 < i1_diff; i1++) {
|
|
1176
|
-
const void * rx = (const void *) ((const char *) x + i1*nb1);
|
|
1177
|
-
void * rd = (void *) (dst_ptr + i1*ts*ne0/bs);
|
|
1178
|
-
// pretend the row is a matrix with cols=1
|
|
1179
|
-
cudaError_t r = cudaMemcpy2DAsync(rd, ts/bs, rx, nb0, ts/bs, ne0, cudaMemcpyDeviceToDevice, stream);
|
|
1180
|
-
if (r != cudaSuccess) {
|
|
1181
|
-
return r;
|
|
1182
|
-
}
|
|
1183
|
-
}
|
|
1184
|
-
return cudaSuccess;
|
|
1185
|
-
}
|
|
1186
|
-
}
|
|
1187
|
-
|
|
1188
|
-
static void ggml_cuda_op_mul_mat_cublas(
|
|
1189
|
-
ggml_backend_cuda_context & ctx,
|
|
1190
|
-
const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i,
|
|
1191
|
-
const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols,
|
|
1192
|
-
const int64_t src1_padded_row_size, cudaStream_t stream) {
|
|
1193
|
-
|
|
1194
|
-
GGML_ASSERT(src0_dd_i != nullptr);
|
|
1195
|
-
GGML_ASSERT(src1_ddf_i != nullptr);
|
|
1196
|
-
GGML_ASSERT(dst_dd_i != nullptr);
|
|
1197
|
-
|
|
1198
|
-
const int64_t ne00 = src0->ne[0];
|
|
1199
|
-
const int64_t ne10 = src1->ne[0];
|
|
1200
|
-
|
|
1201
|
-
const int64_t ne0 = dst->ne[0];
|
|
1202
|
-
|
|
1203
|
-
const int64_t row_diff = row_high - row_low;
|
|
1204
|
-
|
|
1205
|
-
int id = ggml_cuda_get_device();
|
|
1206
|
-
|
|
1207
|
-
// the main device has a larger memory buffer to hold the results from all GPUs
|
|
1208
|
-
// ldc == nrows of the matrix that cuBLAS writes into
|
|
1209
|
-
int64_t ldc = id == ctx.device ? ne0 : row_diff;
|
|
1210
|
-
|
|
1211
|
-
const int compute_capability = ggml_cuda_info().devices[id].cc;
|
|
1212
|
-
|
|
1213
|
-
if (compute_capability >= CC_VOLTA && (src0->type == GGML_TYPE_F16 || ggml_is_quantized(src0->type)) && ggml_is_contiguous(src0) && row_diff == src0->ne[1] && dst->op_params[0] == GGML_PREC_DEFAULT) {
|
|
1214
|
-
// convert src0 and src1 to fp16, multiply as fp16, convert dst to fp32
|
|
1215
|
-
ggml_cuda_pool_alloc<half> src0_as_f16(ctx.pool(id));
|
|
1216
|
-
if (src0->type != GGML_TYPE_F16) {
|
|
1217
|
-
const to_fp16_cuda_t to_fp16_cuda = ggml_get_to_fp16_cuda(src0->type);
|
|
1218
|
-
GGML_ASSERT(to_fp16_cuda != nullptr);
|
|
1219
|
-
size_t ne = row_diff*ne00;
|
|
1220
|
-
src0_as_f16.alloc(ne);
|
|
1221
|
-
to_fp16_cuda(src0_dd_i, src0_as_f16.get(), ne, stream);
|
|
1222
|
-
}
|
|
1223
|
-
const half * src0_ptr = src0->type == GGML_TYPE_F16 ? (const half *) src0_dd_i : src0_as_f16.get();
|
|
1224
|
-
|
|
1225
|
-
ggml_cuda_pool_alloc<half> src1_as_f16(ctx.pool(id));
|
|
1226
|
-
if (src1->type != GGML_TYPE_F16) {
|
|
1227
|
-
const to_fp16_cuda_t to_fp16_cuda = ggml_get_to_fp16_cuda(src1->type);
|
|
1228
|
-
GGML_ASSERT(to_fp16_cuda != nullptr);
|
|
1229
|
-
size_t ne = src1_ncols*ne10;
|
|
1230
|
-
src1_as_f16.alloc(ne);
|
|
1231
|
-
to_fp16_cuda(src1_ddf_i, src1_as_f16.get(), ne, stream);
|
|
1232
|
-
}
|
|
1233
|
-
const half * src1_ptr = src1->type == GGML_TYPE_F16 ? (const half *) src1_ddf_i : src1_as_f16.get();
|
|
1234
|
-
ggml_cuda_pool_alloc<half> dst_f16(ctx.pool(id), row_diff*src1_ncols);
|
|
1235
|
-
|
|
1236
|
-
const half alpha_f16 = 1.0f;
|
|
1237
|
-
const half beta_f16 = 0.0f;
|
|
1238
|
-
|
|
1239
|
-
CUBLAS_CHECK(cublasSetStream(ctx.cublas_handle(id), stream));
|
|
1240
|
-
CUBLAS_CHECK(
|
|
1241
|
-
cublasGemmEx(ctx.cublas_handle(id), CUBLAS_OP_T, CUBLAS_OP_N,
|
|
1242
|
-
row_diff, src1_ncols, ne10,
|
|
1243
|
-
&alpha_f16, src0_ptr, CUDA_R_16F, ne00,
|
|
1244
|
-
src1_ptr, CUDA_R_16F, ne10,
|
|
1245
|
-
&beta_f16, dst_f16.get(), CUDA_R_16F, ldc,
|
|
1246
|
-
CUBLAS_COMPUTE_16F,
|
|
1247
|
-
CUBLAS_GEMM_DEFAULT_TENSOR_OP));
|
|
1248
|
-
|
|
1249
|
-
const to_fp32_cuda_t to_fp32_cuda = ggml_get_to_fp32_cuda(GGML_TYPE_F16);
|
|
1250
|
-
to_fp32_cuda(dst_f16.get(), dst_dd_i, row_diff*src1_ncols, stream);
|
|
1251
|
-
} else {
|
|
1252
|
-
ggml_cuda_pool_alloc<float> src0_ddq_as_f32(ctx.pool(id));
|
|
1253
|
-
ggml_cuda_pool_alloc<float> src1_ddq_as_f32(ctx.pool(id));
|
|
1254
|
-
|
|
1255
|
-
if (src0->type != GGML_TYPE_F32) {
|
|
1256
|
-
const to_fp32_cuda_t to_fp32_cuda = ggml_get_to_fp32_cuda(src0->type);
|
|
1257
|
-
GGML_ASSERT(to_fp32_cuda != nullptr);
|
|
1258
|
-
src0_ddq_as_f32.alloc(row_diff*ne00);
|
|
1259
|
-
to_fp32_cuda(src0_dd_i, src0_ddq_as_f32.get(), row_diff*ne00, stream);
|
|
1260
|
-
}
|
|
1261
|
-
if (src1->type != GGML_TYPE_F32) {
|
|
1262
|
-
const to_fp32_cuda_t to_fp32_cuda = ggml_get_to_fp32_cuda(src1->type);
|
|
1263
|
-
GGML_ASSERT(to_fp32_cuda != nullptr);
|
|
1264
|
-
src1_ddq_as_f32.alloc(src1_ncols*ne10);
|
|
1265
|
-
to_fp32_cuda(src1_ddf_i, src1_ddq_as_f32.get(), src1_ncols*ne10, stream);
|
|
1266
|
-
}
|
|
1267
|
-
|
|
1268
|
-
const float * src0_ddf_i = src0->type == GGML_TYPE_F32 ? (const float *) src0_dd_i : src0_ddq_as_f32.get();
|
|
1269
|
-
const float * src1_ddf1_i = src1->type == GGML_TYPE_F32 ? (const float *) src1_ddf_i : src1_ddq_as_f32.get();
|
|
1270
|
-
|
|
1271
|
-
const float alpha = 1.0f;
|
|
1272
|
-
const float beta = 0.0f;
|
|
1273
|
-
|
|
1274
|
-
CUBLAS_CHECK(cublasSetStream(ctx.cublas_handle(id), stream));
|
|
1275
|
-
CUBLAS_CHECK(
|
|
1276
|
-
cublasSgemm(ctx.cublas_handle(id), CUBLAS_OP_T, CUBLAS_OP_N,
|
|
1277
|
-
row_diff, src1_ncols, ne10,
|
|
1278
|
-
&alpha, src0_ddf_i, ne00,
|
|
1279
|
-
src1_ddf1_i, ne10,
|
|
1280
|
-
&beta, dst_dd_i, ldc));
|
|
1281
|
-
}
|
|
1282
|
-
|
|
1283
|
-
GGML_UNUSED(dst);
|
|
1284
|
-
GGML_UNUSED(src1_ddq_i);
|
|
1285
|
-
GGML_UNUSED(src1_padded_row_size);
|
|
1286
|
-
}
|
|
1287
|
-
|
|
1288
|
-
static void ggml_cuda_set_peer_access(const int n_tokens, int main_device) {
|
|
1289
|
-
static bool peer_access_enabled = false;
|
|
1290
|
-
|
|
1291
|
-
const bool enable_peer_access = n_tokens <= GGML_CUDA_PEER_MAX_BATCH_SIZE;
|
|
1292
|
-
|
|
1293
|
-
if (peer_access_enabled == enable_peer_access) {
|
|
1294
|
-
return;
|
|
1295
|
-
}
|
|
1296
|
-
|
|
1297
|
-
#ifdef NDEBUG
|
|
1298
|
-
for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) {
|
|
1299
|
-
ggml_cuda_set_device(id);
|
|
1300
|
-
CUDA_CHECK(cudaDeviceSynchronize());
|
|
1301
|
-
}
|
|
1302
|
-
|
|
1303
|
-
for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) {
|
|
1304
|
-
ggml_cuda_set_device(id);
|
|
1305
|
-
|
|
1306
|
-
for (int id_other = 0; id_other < ggml_backend_cuda_get_device_count(); ++id_other) {
|
|
1307
|
-
if (id == id_other) {
|
|
1308
|
-
continue;
|
|
1309
|
-
}
|
|
1310
|
-
if (id != main_device && id_other != main_device) {
|
|
1311
|
-
continue;
|
|
1312
|
-
}
|
|
1313
|
-
|
|
1314
|
-
int can_access_peer;
|
|
1315
|
-
CUDA_CHECK(cudaDeviceCanAccessPeer(&can_access_peer, id, id_other));
|
|
1316
|
-
if (can_access_peer) {
|
|
1317
|
-
if (enable_peer_access) {
|
|
1318
|
-
cudaError_t err = cudaDeviceEnablePeerAccess(id_other, 0);
|
|
1319
|
-
if (err != cudaErrorPeerAccessAlreadyEnabled) {
|
|
1320
|
-
CUDA_CHECK(err);
|
|
1321
|
-
}
|
|
1322
|
-
} else {
|
|
1323
|
-
cudaError_t err = cudaDeviceDisablePeerAccess(id_other);
|
|
1324
|
-
if (err != cudaErrorPeerAccessNotEnabled) {
|
|
1325
|
-
CUDA_CHECK(err);
|
|
1326
|
-
}
|
|
1327
|
-
}
|
|
1328
|
-
}
|
|
1329
|
-
}
|
|
1330
|
-
}
|
|
1331
|
-
|
|
1332
|
-
ggml_cuda_set_device(main_device);
|
|
1333
|
-
#endif // NDEBUG
|
|
1334
|
-
|
|
1335
|
-
peer_access_enabled = enable_peer_access;
|
|
1336
|
-
|
|
1337
|
-
GGML_UNUSED(main_device);
|
|
1338
|
-
}
|
|
1339
|
-
|
|
1340
|
-
static cudaError_t ggml_cuda_Memcpy2DPeerAsync(
|
|
1341
|
-
void * dst, int dstDevice, size_t dpitch, void * src, int srcDevice, size_t spitch, size_t width, size_t height, cudaStream_t stream) {
|
|
1342
|
-
|
|
1343
|
-
#if !defined(GGML_USE_HIPBLAS)
|
|
1344
|
-
// cudaMemcpy2DAsync may fail with copies between vmm pools of different devices
|
|
1345
|
-
cudaMemcpy3DPeerParms p = {};
|
|
1346
|
-
p.dstDevice = dstDevice;
|
|
1347
|
-
p.dstPtr = make_cudaPitchedPtr(dst, dpitch, dpitch, height);
|
|
1348
|
-
p.srcDevice = srcDevice;
|
|
1349
|
-
p.srcPtr = make_cudaPitchedPtr(src, spitch, spitch, height);
|
|
1350
|
-
p.extent = make_cudaExtent(width, height, 1);
|
|
1351
|
-
return cudaMemcpy3DPeerAsync(&p, stream);
|
|
1352
|
-
#else
|
|
1353
|
-
// HIP does not support cudaMemcpy3DPeerAsync or vmm pools
|
|
1354
|
-
GGML_UNUSED(dstDevice);
|
|
1355
|
-
GGML_UNUSED(srcDevice);
|
|
1356
|
-
return cudaMemcpy2DAsync(dst, dpitch, src, spitch, width, height, cudaMemcpyDeviceToDevice, stream);
|
|
1357
|
-
#endif // !defined(GGML_USE_HIPBLAS)
|
|
1358
|
-
}
|
|
1359
|
-
|
|
1360
|
-
static void ggml_cuda_op_mul_mat(
|
|
1361
|
-
ggml_backend_cuda_context & ctx,
|
|
1362
|
-
const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, ggml_cuda_op_mul_mat_t op,
|
|
1363
|
-
quantize_cuda_t quantize_src1) {
|
|
1364
|
-
|
|
1365
|
-
const int64_t ne00 = src0->ne[0];
|
|
1366
|
-
const int64_t ne01 = src0->ne[1];
|
|
1367
|
-
const int64_t ne02 = src0->ne[2];
|
|
1368
|
-
const int64_t ne03 = src0->ne[3];
|
|
1369
|
-
|
|
1370
|
-
const int64_t ne10 = src1->ne[0];
|
|
1371
|
-
const int64_t ne11 = src1->ne[1];
|
|
1372
|
-
const int64_t ne12 = src1->ne[2];
|
|
1373
|
-
const int64_t ne13 = src1->ne[3];
|
|
1374
|
-
const int64_t nrows1 = ggml_nrows(src1);
|
|
1375
|
-
|
|
1376
|
-
GGML_ASSERT(ne03 == ne13);
|
|
1377
|
-
|
|
1378
|
-
const int64_t ne0 = dst->ne[0];
|
|
1379
|
-
const int64_t ne1 = dst->ne[1];
|
|
1380
|
-
|
|
1381
|
-
const int64_t nb2 = dst->nb[2];
|
|
1382
|
-
const int64_t nb3 = dst->nb[3];
|
|
1383
|
-
|
|
1384
|
-
GGML_ASSERT(ggml_backend_buffer_is_cuda(dst->buffer));
|
|
1385
|
-
GGML_ASSERT(ggml_backend_buffer_is_cuda(src1->buffer));
|
|
1386
|
-
ggml_backend_cuda_buffer_context * src1_ctx = (ggml_backend_cuda_buffer_context *) src1->buffer->context;
|
|
1387
|
-
ggml_backend_cuda_buffer_context * dst_ctx = (ggml_backend_cuda_buffer_context *) dst->buffer->context;
|
|
1388
|
-
|
|
1389
|
-
GGML_ASSERT(src1->type == GGML_TYPE_F32 || (src1->ne[2] == 1 && src1->ne[3] == 1));
|
|
1390
|
-
|
|
1391
|
-
GGML_ASSERT(ne12 >= ne02 && ne12 % ne02 == 0);
|
|
1392
|
-
|
|
1393
|
-
const int64_t i02_divisor = ne12 / ne02;
|
|
1394
|
-
|
|
1395
|
-
const size_t src0_ts = ggml_type_size(src0->type);
|
|
1396
|
-
const size_t src0_bs = ggml_blck_size(src0->type);
|
|
1397
|
-
const size_t q8_1_ts = sizeof(block_q8_1);
|
|
1398
|
-
const size_t q8_1_bs = QK8_1;
|
|
1399
|
-
|
|
1400
|
-
const bool src0_is_contiguous = ggml_is_contiguous(src0);
|
|
1401
|
-
const bool src1_is_contiguous = ggml_is_contiguous(src1);
|
|
1402
|
-
|
|
1403
|
-
const int64_t src1_padded_col_size = GGML_PAD(ne10, MATRIX_ROW_PADDING);
|
|
1404
|
-
|
|
1405
|
-
const bool split = ggml_backend_buffer_is_cuda_split(src0->buffer);
|
|
1406
|
-
GGML_ASSERT(!(split && ne02 > 1));
|
|
1407
|
-
GGML_ASSERT(!(split && ne03 > 1));
|
|
1408
|
-
GGML_ASSERT(!(split && ne02 < ne12));
|
|
1409
|
-
|
|
1410
|
-
ggml_tensor_extra_gpu * src0_extra = split ? (ggml_tensor_extra_gpu *) src0->extra : nullptr;
|
|
1411
|
-
|
|
1412
|
-
|
|
1413
|
-
std::array<float, GGML_CUDA_MAX_DEVICES> tensor_split;
|
|
1414
|
-
if (split) {
|
|
1415
|
-
ggml_backend_cuda_split_buffer_type_context * buft_ctx = (ggml_backend_cuda_split_buffer_type_context *) src0->buffer->buft->context;
|
|
1416
|
-
tensor_split = buft_ctx->tensor_split;
|
|
1417
|
-
}
|
|
1418
|
-
|
|
1419
|
-
struct dev_data {
|
|
1420
|
-
int cc;
|
|
1421
|
-
|
|
1422
|
-
ggml_cuda_pool_alloc<char> src0_dd_alloc;
|
|
1423
|
-
ggml_cuda_pool_alloc<float> src1_ddf_alloc;
|
|
1424
|
-
ggml_cuda_pool_alloc<char> src1_ddq_alloc;
|
|
1425
|
-
ggml_cuda_pool_alloc<float> dst_dd_alloc;
|
|
1426
|
-
|
|
1427
|
-
char * src0_dd = nullptr;
|
|
1428
|
-
float * src1_ddf = nullptr; // float
|
|
1429
|
-
char * src1_ddq = nullptr; // q8_1
|
|
1430
|
-
float * dst_dd = nullptr;
|
|
1431
|
-
|
|
1432
|
-
int64_t row_low;
|
|
1433
|
-
int64_t row_high;
|
|
1434
|
-
};
|
|
1435
|
-
|
|
1436
|
-
dev_data dev[GGML_CUDA_MAX_DEVICES];
|
|
1437
|
-
|
|
1438
|
-
int used_devices = 0;
|
|
1439
|
-
|
|
1440
|
-
for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) {
|
|
1441
|
-
dev[id].cc = ggml_cuda_info().devices[id].cc;
|
|
1442
|
-
|
|
1443
|
-
// by default, use all rows
|
|
1444
|
-
dev[id].row_low = 0;
|
|
1445
|
-
dev[id].row_high = ne01;
|
|
1446
|
-
|
|
1447
|
-
// for multi GPU, get the row boundaries from tensor split
|
|
1448
|
-
// and round to mul_mat_q tile sizes
|
|
1449
|
-
if (split) {
|
|
1450
|
-
const int64_t rounding = get_row_rounding(tensor_split);
|
|
1451
|
-
|
|
1452
|
-
if (id != 0) {
|
|
1453
|
-
dev[id].row_low = ne01*tensor_split[id];
|
|
1454
|
-
if (dev[id].row_low < ne01) {
|
|
1455
|
-
dev[id].row_low -= dev[id].row_low % rounding;
|
|
1456
|
-
}
|
|
1457
|
-
}
|
|
1458
|
-
|
|
1459
|
-
if (id != ggml_backend_cuda_get_device_count() - 1) {
|
|
1460
|
-
dev[id].row_high = ne01*tensor_split[id + 1];
|
|
1461
|
-
if (dev[id].row_high < ne01) {
|
|
1462
|
-
dev[id].row_high -= dev[id].row_high % rounding;
|
|
1463
|
-
}
|
|
1464
|
-
}
|
|
1465
|
-
}
|
|
1466
|
-
}
|
|
1467
|
-
|
|
1468
|
-
for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) {
|
|
1469
|
-
if ((!split && id != ctx.device) || dev[id].row_low == dev[id].row_high) {
|
|
1470
|
-
continue;
|
|
1471
|
-
}
|
|
1472
|
-
|
|
1473
|
-
used_devices++;
|
|
1474
|
-
|
|
1475
|
-
const bool src1_on_device = id == src1_ctx->device;
|
|
1476
|
-
const bool dst_on_device = id == dst_ctx->device;
|
|
1477
|
-
|
|
1478
|
-
ggml_cuda_set_device(id);
|
|
1479
|
-
cudaStream_t stream = ctx.stream(id, 0);
|
|
1480
|
-
|
|
1481
|
-
if (src0_is_contiguous) {
|
|
1482
|
-
dev[id].src0_dd = split ? (char *) src0_extra->data_device[id] : (char *) src0->data;
|
|
1483
|
-
} else {
|
|
1484
|
-
dev[id].src0_dd = dev[id].src0_dd_alloc.alloc(ctx.pool(id), ggml_nbytes(src0));
|
|
1485
|
-
}
|
|
1486
|
-
|
|
1487
|
-
if (src1_on_device && src1_is_contiguous) {
|
|
1488
|
-
dev[id].src1_ddf = (float *) src1->data;
|
|
1489
|
-
} else {
|
|
1490
|
-
dev[id].src1_ddf = dev[id].src1_ddf_alloc.alloc(ctx.pool(id), ggml_nelements(src1));
|
|
1491
|
-
}
|
|
1492
|
-
|
|
1493
|
-
if (quantize_src1) {
|
|
1494
|
-
size_t src_1_ddq_size = nrows1*src1_padded_col_size*q8_1_ts/q8_1_bs;
|
|
1495
|
-
if (quantize_src1 == quantize_mmq_q8_1_cuda) {
|
|
1496
|
-
src_1_ddq_size += get_mmq_x_max_host(dev[id].cc)*sizeof(block_q8_1_mmq);
|
|
1497
|
-
}
|
|
1498
|
-
dev[id].src1_ddq = dev[id].src1_ddq_alloc.alloc(ctx.pool(id), src_1_ddq_size);
|
|
1499
|
-
|
|
1500
|
-
if (src1_on_device && src1_is_contiguous) {
|
|
1501
|
-
quantize_src1(dev[id].src1_ddf, dev[id].src1_ddq, ne10, ne11, ne12*ne13, src1_padded_col_size, src0->type, stream);
|
|
1502
|
-
CUDA_CHECK(cudaGetLastError());
|
|
1503
|
-
}
|
|
1504
|
-
}
|
|
1505
|
-
|
|
1506
|
-
if (dst_on_device) {
|
|
1507
|
-
dev[id].dst_dd = (float *) dst->data;
|
|
1508
|
-
} else {
|
|
1509
|
-
const size_t size_dst_ddf = split ? (dev[id].row_high - dev[id].row_low)*ne1 : ggml_nelements(dst);
|
|
1510
|
-
dev[id].dst_dd = dev[id].dst_dd_alloc.alloc(ctx.pool(id), size_dst_ddf);
|
|
1511
|
-
}
|
|
1512
|
-
}
|
|
1513
|
-
|
|
1514
|
-
// if multiple devices are used they need to wait for the main device
|
|
1515
|
-
// here an event is recorded that signals that the main device has finished calculating the input data
|
|
1516
|
-
if (split && used_devices > 1) {
|
|
1517
|
-
ggml_cuda_set_device(ctx.device);
|
|
1518
|
-
CUDA_CHECK(cudaEventRecord(src0_extra->events[ctx.device][0], ctx.stream()));
|
|
1519
|
-
}
|
|
1520
|
-
|
|
1521
|
-
const int64_t src1_col_stride = split && used_devices > 1 ? MUL_MAT_SRC1_COL_STRIDE : ne11;
|
|
1522
|
-
for (int64_t src1_col_0 = 0; src1_col_0 < ne11; src1_col_0 += src1_col_stride) {
|
|
1523
|
-
const int64_t is = split ? (src1_col_0/src1_col_stride) % GGML_CUDA_MAX_STREAMS : 0;
|
|
1524
|
-
const int64_t src1_ncols = src1_col_0 + src1_col_stride > ne11 ? ne11 - src1_col_0 : src1_col_stride;
|
|
1525
|
-
|
|
1526
|
-
for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) {
|
|
1527
|
-
if ((!split && id != ctx.device) || dev[id].row_low == dev[id].row_high) {
|
|
1528
|
-
continue;
|
|
1529
|
-
}
|
|
1530
|
-
|
|
1531
|
-
const bool src1_on_device = id == src1_ctx->device;
|
|
1532
|
-
const bool dst_on_device = id == dst_ctx->device;
|
|
1533
|
-
const int64_t row_diff = dev[id].row_high - dev[id].row_low;
|
|
1534
|
-
|
|
1535
|
-
ggml_cuda_set_device(id);
|
|
1536
|
-
cudaStream_t stream = ctx.stream(id, is);
|
|
1537
|
-
|
|
1538
|
-
// wait for main GPU data if necessary
|
|
1539
|
-
if (split && (id != ctx.device || is != 0)) {
|
|
1540
|
-
CUDA_CHECK(cudaStreamWaitEvent(stream, src0_extra->events[ctx.device][0], 0));
|
|
1541
|
-
}
|
|
1542
|
-
|
|
1543
|
-
for (int64_t i0 = 0; i0 < ne13*ne12; ++i0) {
|
|
1544
|
-
const int64_t i03 = i0 / ne12;
|
|
1545
|
-
const int64_t i02 = i0 % ne12;
|
|
1546
|
-
|
|
1547
|
-
size_t src1_ddq_i_offset = i0*ne11 * src1_padded_col_size*q8_1_ts/q8_1_bs;
|
|
1548
|
-
if (quantize_src1 == quantize_mmq_q8_1_cuda) {
|
|
1549
|
-
src1_ddq_i_offset += src1_col_0 * sizeof(block_q8_1_mmq);
|
|
1550
|
-
} else {
|
|
1551
|
-
src1_ddq_i_offset += src1_col_0 * src1_padded_col_size*q8_1_ts/q8_1_bs;
|
|
1552
|
-
}
|
|
1553
|
-
|
|
1554
|
-
// for split tensors the data begins at i0 == i0_offset_low
|
|
1555
|
-
char * src0_dd_i = dev[id].src0_dd + (i0/i02_divisor) * (ne01*ne00*src0_ts)/src0_bs;
|
|
1556
|
-
float * src1_ddf_i = dev[id].src1_ddf + (i0*ne11 + src1_col_0) * ne10;
|
|
1557
|
-
char * src1_ddq_i = dev[id].src1_ddq + src1_ddq_i_offset;
|
|
1558
|
-
float * dst_dd_i = dev[id].dst_dd + (i0*ne1 + src1_col_0) * (dst_on_device ? ne0 : row_diff);
|
|
1559
|
-
|
|
1560
|
-
// the main device memory buffer can be on VRAM scratch, with space for all partial results
|
|
1561
|
-
// in that case an offset on dst_ddf_i is needed
|
|
1562
|
-
if (id == ctx.device) {
|
|
1563
|
-
dst_dd_i += dev[id].row_low; // offset is 0 if no tensor split
|
|
1564
|
-
}
|
|
1565
|
-
|
|
1566
|
-
// copy src0, src1 to device if necessary
|
|
1567
|
-
if (src1_is_contiguous) {
|
|
1568
|
-
if (id != ctx.device) {
|
|
1569
|
-
if (quantize_src1) {
|
|
1570
|
-
char * src1_ddq_i_source = dev[ctx.device].src1_ddq + src1_ddq_i_offset;
|
|
1571
|
-
if (quantize_src1 == quantize_mmq_q8_1_cuda) {
|
|
1572
|
-
const size_t pitch = ne11*sizeof(block_q8_1_mmq);
|
|
1573
|
-
const size_t width = src1_ncols*sizeof(block_q8_1_mmq);
|
|
1574
|
-
const size_t height = src1_padded_col_size/(4*QK8_1);
|
|
1575
|
-
CUDA_CHECK(ggml_cuda_Memcpy2DPeerAsync(src1_ddq_i, id, pitch, src1_ddq_i_source, ctx.device, pitch, width, height, stream));
|
|
1576
|
-
} else {
|
|
1577
|
-
CUDA_CHECK(cudaMemcpyPeerAsync(
|
|
1578
|
-
src1_ddq_i, id, src1_ddq_i_source, ctx.device, src1_ncols*src1_padded_col_size*q8_1_ts/q8_1_bs, stream));
|
|
1579
|
-
}
|
|
1580
|
-
} else {
|
|
1581
|
-
float * src1_ddf_i_source = (float *) src1->data;
|
|
1582
|
-
src1_ddf_i_source += (i0*ne11 + src1_col_0) * ne10;
|
|
1583
|
-
CUDA_CHECK(cudaMemcpyPeerAsync(src1_ddf_i, id, src1_ddf_i_source, ctx.device,
|
|
1584
|
-
src1_ncols*ne10*sizeof(float), stream));
|
|
1585
|
-
}
|
|
1586
|
-
}
|
|
1587
|
-
} else if (src1_on_device && !src1_is_contiguous) {
|
|
1588
|
-
CUDA_CHECK(ggml_cuda_cpy_tensor_2d(
|
|
1589
|
-
src1_ddf_i, src1, i03, i02, src1_col_0, src1_col_0+src1_ncols, stream));
|
|
1590
|
-
} else {
|
|
1591
|
-
GGML_ASSERT(false);
|
|
1592
|
-
}
|
|
1593
|
-
|
|
1594
|
-
if (quantize_src1 && !src1_is_contiguous) {
|
|
1595
|
-
quantize_src1(src1_ddf_i, src1_ddq_i, ne10, src1_ncols, 1, src1_padded_col_size, src0->type, stream);
|
|
1596
|
-
CUDA_CHECK(cudaGetLastError());
|
|
1597
|
-
}
|
|
1598
|
-
|
|
1599
|
-
if (src1_col_0 == 0 && !src0_is_contiguous && i02 % i02_divisor == 0) {
|
|
1600
|
-
CUDA_CHECK(ggml_cuda_cpy_tensor_2d(src0_dd_i, src0, i03, i02/i02_divisor, dev[id].row_low, dev[id].row_high, stream));
|
|
1601
|
-
}
|
|
1602
|
-
|
|
1603
|
-
// do the computation
|
|
1604
|
-
op(ctx, src0, src1, dst, src0_dd_i, src1_ddf_i, src1_ddq_i, dst_dd_i,
|
|
1605
|
-
dev[id].row_low, dev[id].row_high, src1_ncols, src1_padded_col_size, stream);
|
|
1606
|
-
CUDA_CHECK(cudaGetLastError());
|
|
1607
|
-
|
|
1608
|
-
// copy dst to host or other device if necessary
|
|
1609
|
-
if (!dst_on_device) {
|
|
1610
|
-
void * dst_off_device = dst->data;
|
|
1611
|
-
if (split) {
|
|
1612
|
-
// src0 = weight matrix is saved as a transposed matrix for better memory layout.
|
|
1613
|
-
// dst is NOT transposed.
|
|
1614
|
-
// The outputs of matrix matrix multiplications can therefore NOT simply be concatenated for >1 GPU.
|
|
1615
|
-
// Instead they need to be copied to the correct slice in ne0 = dst row index.
|
|
1616
|
-
// If dst is a vector with ne0 == 1 then you don't have to do this but it still produces correct results.
|
|
1617
|
-
float * dhf_dst_i = (float *) ((char *) dst_off_device + i02*nb2 + i03*nb3);
|
|
1618
|
-
GGML_ASSERT(dst->nb[1] == ne0*sizeof(float));
|
|
1619
|
-
dhf_dst_i += src1_col_0*ne0 + dev[id].row_low;
|
|
1620
|
-
CUDA_CHECK(ggml_cuda_Memcpy2DPeerAsync(
|
|
1621
|
-
dhf_dst_i, ctx.device, ne0*sizeof(float), dst_dd_i, id, row_diff*sizeof(float), row_diff*sizeof(float), src1_ncols, stream));
|
|
1622
|
-
} else {
|
|
1623
|
-
float * dhf_dst_i = (float *) ((char *) dst_off_device + i02*nb2 + i03*nb3);
|
|
1624
|
-
GGML_ASSERT(dst->nb[1] == ne0*sizeof(float));
|
|
1625
|
-
dhf_dst_i += src1_col_0*ne0;
|
|
1626
|
-
CUDA_CHECK(cudaMemcpyAsync(dhf_dst_i, dst_dd_i, src1_ncols*ne0*sizeof(float), cudaMemcpyDeviceToDevice, stream));
|
|
1627
|
-
}
|
|
1628
|
-
}
|
|
1629
|
-
|
|
1630
|
-
// add event for the main device to wait on until other device is done
|
|
1631
|
-
if (split && (id != ctx.device || is != 0)) {
|
|
1632
|
-
CUDA_CHECK(cudaEventRecord(src0_extra->events[id][is], stream));
|
|
1633
|
-
}
|
|
1634
|
-
}
|
|
1635
|
-
}
|
|
1636
|
-
}
|
|
1637
|
-
|
|
1638
|
-
// main device waits for all other devices to be finished
|
|
1639
|
-
if (split && ggml_backend_cuda_get_device_count() > 1) {
|
|
1640
|
-
int64_t is_max = (ne11 + MUL_MAT_SRC1_COL_STRIDE - 1) / MUL_MAT_SRC1_COL_STRIDE;
|
|
1641
|
-
is_max = is_max <= GGML_CUDA_MAX_STREAMS ? is_max : GGML_CUDA_MAX_STREAMS;
|
|
1642
|
-
|
|
1643
|
-
ggml_cuda_set_device(ctx.device);
|
|
1644
|
-
for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) {
|
|
1645
|
-
if (dev[id].row_low == dev[id].row_high) {
|
|
1646
|
-
continue;
|
|
1647
|
-
}
|
|
1648
|
-
for (int64_t is = 0; is < is_max; ++is) {
|
|
1649
|
-
CUDA_CHECK(cudaStreamWaitEvent(ctx.stream(), src0_extra->events[id][is], 0));
|
|
1650
|
-
}
|
|
1651
|
-
}
|
|
1652
|
-
}
|
|
1653
|
-
}
|
|
1654
|
-
|
|
1655
|
-
static void ggml_cuda_mul_mat_vec_p021(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
|
|
1656
|
-
GGML_ASSERT(ggml_is_permuted(src0) && ggml_is_permuted(src1));
|
|
1657
|
-
GGML_ASSERT(ggml_backend_buffer_is_cuda(src0->buffer));
|
|
1658
|
-
GGML_ASSERT(src0->nb[0] <= src0->nb[1] && src0->nb[2] <= src0->nb[3]); // 0213 permutation
|
|
1659
|
-
GGML_ASSERT(src1->nb[0] <= src1->nb[1] && src1->nb[2] <= src1->nb[3]); // 0213 permutation
|
|
1660
|
-
GGML_ASSERT(src0->type == GGML_TYPE_F16);
|
|
1661
|
-
GGML_ASSERT(src1->type == GGML_TYPE_F32);
|
|
1662
|
-
|
|
1663
|
-
const int64_t ne00 = src0->ne[0];
|
|
1664
|
-
const int64_t ne01 = src0->ne[1];
|
|
1665
|
-
const int64_t ne02 = src0->ne[2];
|
|
1666
|
-
|
|
1667
|
-
const int64_t ne12 = src1->ne[2];
|
|
1668
|
-
|
|
1669
|
-
cudaStream_t main_stream = ctx.stream();
|
|
1670
|
-
|
|
1671
|
-
void * src0_ddq = src0->data;
|
|
1672
|
-
float * src1_ddf = (float *) src1->data;
|
|
1673
|
-
float * dst_ddf = (float *) dst->data;
|
|
1674
|
-
|
|
1675
|
-
ggml_mul_mat_p021_f16_f32_cuda(src0_ddq, src1_ddf, dst_ddf, ne00, ne01, ne02, ne12, main_stream);
|
|
1676
|
-
}
|
|
1677
|
-
|
|
1678
|
-
static void ggml_cuda_mul_mat_vec_nc(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
|
|
1679
|
-
GGML_ASSERT(!ggml_is_transposed(src0));
|
|
1680
|
-
GGML_ASSERT(!ggml_is_transposed(src1));
|
|
1681
|
-
GGML_ASSERT(!ggml_is_permuted(src0));
|
|
1682
|
-
GGML_ASSERT(ggml_backend_buffer_is_cuda(src0->buffer));
|
|
1683
|
-
GGML_ASSERT(src0->type == GGML_TYPE_F16);
|
|
1684
|
-
GGML_ASSERT(src1->type == GGML_TYPE_F32);
|
|
1685
|
-
|
|
1686
|
-
const int64_t ne00 = src0->ne[0];
|
|
1687
|
-
const int64_t ne01 = src0->ne[1];
|
|
1688
|
-
const int64_t ne02 = src0->ne[2];
|
|
1689
|
-
|
|
1690
|
-
const int64_t nb01 = src0->nb[1];
|
|
1691
|
-
const int64_t nb02 = src0->nb[2];
|
|
1692
|
-
|
|
1693
|
-
const int64_t ne12 = src1->ne[2];
|
|
1694
|
-
|
|
1695
|
-
cudaStream_t main_stream = ctx.stream();
|
|
1696
|
-
|
|
1697
|
-
void * src0_ddq = src0->data;
|
|
1698
|
-
float * src1_ddf = (float *) src1->data;
|
|
1699
|
-
float * dst_ddf = (float *) dst->data;
|
|
1700
|
-
|
|
1701
|
-
const int64_t row_stride_x = nb01 / sizeof(half);
|
|
1702
|
-
const int64_t channel_stride_x = nb02 / sizeof(half);
|
|
1703
|
-
|
|
1704
|
-
ggml_mul_mat_vec_nc_f16_f32_cuda(src0_ddq, src1_ddf, dst_ddf, ne00, ne01, row_stride_x, ne02, ne12, channel_stride_x, main_stream);
|
|
1705
|
-
}
|
|
1706
|
-
|
|
1707
|
-
static __global__ void k_compute_batched_ptrs(
|
|
1708
|
-
const half * src0_as_f16, const half * src1_as_f16, char * dst,
|
|
1709
|
-
const void ** ptrs_src, void ** ptrs_dst,
|
|
1710
|
-
int64_t ne12, int64_t ne13,
|
|
1711
|
-
int64_t ne23,
|
|
1712
|
-
size_t nb02, size_t nb03,
|
|
1713
|
-
size_t nb12, size_t nb13,
|
|
1714
|
-
size_t nbd2, size_t nbd3,
|
|
1715
|
-
int64_t r2, int64_t r3) {
|
|
1716
|
-
int64_t i13 = blockIdx.x * blockDim.x + threadIdx.x;
|
|
1717
|
-
int64_t i12 = blockIdx.y * blockDim.y + threadIdx.y;
|
|
1718
|
-
|
|
1719
|
-
if (i13 >= ne13 || i12 >= ne12) {
|
|
1720
|
-
return;
|
|
1721
|
-
}
|
|
1722
|
-
|
|
1723
|
-
int64_t i03 = i13 / r3;
|
|
1724
|
-
int64_t i02 = i12 / r2;
|
|
1725
|
-
|
|
1726
|
-
ptrs_src[0*ne23 + i12 + i13*ne12] = (const char *) src0_as_f16 + i02*nb02 + i03*nb03;
|
|
1727
|
-
ptrs_src[1*ne23 + i12 + i13*ne12] = (const char *) src1_as_f16 + i12*nb12 + i13*nb13;
|
|
1728
|
-
ptrs_dst[0*ne23 + i12 + i13*ne12] = ( char *) dst + i12*nbd2 + i13*nbd3;
|
|
1729
|
-
}
|
|
1730
|
-
|
|
1731
|
-
static void ggml_cuda_mul_mat_batched_cublas(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
|
|
1732
|
-
GGML_ASSERT(!ggml_is_transposed(src0));
|
|
1733
|
-
GGML_ASSERT(!ggml_is_transposed(src1));
|
|
1734
|
-
|
|
1735
|
-
GGML_ASSERT(ggml_backend_buffer_is_cuda(src0->buffer));
|
|
1736
|
-
GGML_ASSERT(src0->type == GGML_TYPE_F16);
|
|
1737
|
-
|
|
1738
|
-
GGML_TENSOR_BINARY_OP_LOCALS
|
|
1739
|
-
|
|
1740
|
-
const int64_t ne_dst = ggml_nelements(dst);
|
|
1741
|
-
|
|
1742
|
-
cudaStream_t main_stream = ctx.stream();
|
|
1743
|
-
|
|
1744
|
-
CUBLAS_CHECK(cublasSetStream(ctx.cublas_handle(), main_stream));
|
|
1745
|
-
|
|
1746
|
-
void * src0_ddq = src0->data;
|
|
1747
|
-
half * src0_f16 = (half *) src0_ddq;
|
|
1748
|
-
float * src1_ddf = (float *) src1->data;
|
|
1749
|
-
float * dst_ddf = (float *) dst->data;
|
|
1750
|
-
|
|
1751
|
-
// convert src1 to fp16
|
|
1752
|
-
ggml_cuda_pool_alloc<half> src1_f16_alloc(ctx.pool());
|
|
1753
|
-
if (src1->type != GGML_TYPE_F16) {
|
|
1754
|
-
const to_fp16_cuda_t to_fp16_cuda = ggml_get_to_fp16_cuda(src1->type);
|
|
1755
|
-
const int64_t ne_src1 = ggml_nelements(src1);
|
|
1756
|
-
src1_f16_alloc.alloc(ne_src1);
|
|
1757
|
-
GGML_ASSERT(to_fp16_cuda != nullptr);
|
|
1758
|
-
to_fp16_cuda(src1_ddf, src1_f16_alloc.get(), ne_src1, main_stream);
|
|
1759
|
-
}
|
|
1760
|
-
half * src1_f16 = src1->type == GGML_TYPE_F16 ? (half *) src1_ddf : src1_f16_alloc.get();
|
|
1761
|
-
|
|
1762
|
-
ggml_cuda_pool_alloc<half> dst_f16(ctx.pool());
|
|
1763
|
-
char * dst_t;
|
|
1764
|
-
|
|
1765
|
-
cublasComputeType_t cu_compute_type = CUBLAS_COMPUTE_16F;
|
|
1766
|
-
cudaDataType_t cu_data_type = CUDA_R_16F;
|
|
1767
|
-
|
|
1768
|
-
// dst strides
|
|
1769
|
-
size_t nbd2 = dst->nb[2];
|
|
1770
|
-
size_t nbd3 = dst->nb[3];
|
|
1771
|
-
|
|
1772
|
-
const half alpha_f16 = 1.0f;
|
|
1773
|
-
const half beta_f16 = 0.0f;
|
|
1774
|
-
|
|
1775
|
-
const float alpha_f32 = 1.0f;
|
|
1776
|
-
const float beta_f32 = 0.0f;
|
|
1777
|
-
|
|
1778
|
-
const void * alpha = &alpha_f16;
|
|
1779
|
-
const void * beta = &beta_f16;
|
|
1780
|
-
|
|
1781
|
-
if (dst->op_params[0] == GGML_PREC_DEFAULT) {
|
|
1782
|
-
dst_t = (char *) dst_f16.alloc(ne_dst);
|
|
1783
|
-
|
|
1784
|
-
nbd2 /= sizeof(float) / sizeof(half);
|
|
1785
|
-
nbd3 /= sizeof(float) / sizeof(half);
|
|
1786
|
-
} else {
|
|
1787
|
-
dst_t = (char *) dst_ddf;
|
|
1788
|
-
|
|
1789
|
-
cu_compute_type = CUBLAS_COMPUTE_32F;
|
|
1790
|
-
cu_data_type = CUDA_R_32F;
|
|
1791
|
-
|
|
1792
|
-
alpha = &alpha_f32;
|
|
1793
|
-
beta = &beta_f32;
|
|
1794
|
-
}
|
|
1795
|
-
|
|
1796
|
-
GGML_ASSERT(ne12 % ne02 == 0);
|
|
1797
|
-
GGML_ASSERT(ne13 % ne03 == 0);
|
|
1798
|
-
|
|
1799
|
-
// broadcast factors
|
|
1800
|
-
const int64_t r2 = ne12/ne02;
|
|
1801
|
-
const int64_t r3 = ne13/ne03;
|
|
1802
|
-
|
|
1803
|
-
#if 0
|
|
1804
|
-
// use cublasGemmEx
|
|
1805
|
-
{
|
|
1806
|
-
for (int i13 = 0; i13 < ne13; ++i13) {
|
|
1807
|
-
for (int i12 = 0; i12 < ne12; ++i12) {
|
|
1808
|
-
int i03 = i13 / r3;
|
|
1809
|
-
int i02 = i12 / r2;
|
|
1810
|
-
|
|
1811
|
-
CUBLAS_CHECK(
|
|
1812
|
-
cublasGemmEx(g_cublas_handles[g_main_device], CUBLAS_OP_T, CUBLAS_OP_N,
|
|
1813
|
-
ne01, ne11, ne10,
|
|
1814
|
-
alpha, (const char *) src0_as_f16 + i02*src0->nb[2] + i03*src0->nb[3] , CUDA_R_16F, nb01/sizeof(half),
|
|
1815
|
-
(const char *) src1_as_f16 + i12*src1->nb[2]/2 + i13*src1->nb[3]/2, CUDA_R_16F, nb11/sizeof(float),
|
|
1816
|
-
beta, ( char *) dst_t + i12*nbd2 + i13*nbd3, cu_data_type, ne01,
|
|
1817
|
-
cu_compute_type,
|
|
1818
|
-
CUBLAS_GEMM_DEFAULT_TENSOR_OP));
|
|
1819
|
-
}
|
|
1820
|
-
}
|
|
1821
|
-
}
|
|
1822
|
-
#else
|
|
1823
|
-
if (r2 == 1 && r3 == 1 && ggml_is_contiguous_2(src0) && ggml_is_contiguous_2(src1)) {
|
|
1824
|
-
// there is no broadcast and src0, src1 are contiguous across dims 2, 3
|
|
1825
|
-
// use cublasGemmStridedBatchedEx
|
|
1826
|
-
CUBLAS_CHECK(
|
|
1827
|
-
cublasGemmStridedBatchedEx(ctx.cublas_handle(), CUBLAS_OP_T, CUBLAS_OP_N,
|
|
1828
|
-
ne01, ne11, ne10,
|
|
1829
|
-
alpha, (const char *) src0_f16, CUDA_R_16F, nb01/nb00, nb02/nb00, // strideA
|
|
1830
|
-
(const char *) src1_f16, CUDA_R_16F, nb11/nb10, nb12/nb10, // strideB
|
|
1831
|
-
beta, ( char *) dst_t, cu_data_type, ne01, nb2/nb0, // strideC
|
|
1832
|
-
ne12*ne13,
|
|
1833
|
-
cu_compute_type,
|
|
1834
|
-
CUBLAS_GEMM_DEFAULT_TENSOR_OP));
|
|
1835
|
-
} else {
|
|
1836
|
-
// use cublasGemmBatchedEx
|
|
1837
|
-
const int ne23 = ne12*ne13;
|
|
1838
|
-
|
|
1839
|
-
ggml_cuda_pool_alloc<const void *> ptrs_src(ctx.pool(), 2*ne23);
|
|
1840
|
-
ggml_cuda_pool_alloc< void *> ptrs_dst(ctx.pool(), 1*ne23);
|
|
1841
|
-
|
|
1842
|
-
dim3 block_dims(ne13, ne12);
|
|
1843
|
-
k_compute_batched_ptrs<<<1, block_dims, 0, main_stream>>>(
|
|
1844
|
-
src0_f16, src1_f16, dst_t,
|
|
1845
|
-
ptrs_src.get(), ptrs_dst.get(),
|
|
1846
|
-
ne12, ne13,
|
|
1847
|
-
ne23,
|
|
1848
|
-
nb02, nb03,
|
|
1849
|
-
src1->type == GGML_TYPE_F16 ? nb12 : nb12/2,
|
|
1850
|
-
src1->type == GGML_TYPE_F16 ? nb13 : nb13/2,
|
|
1851
|
-
nbd2, nbd3,
|
|
1852
|
-
r2, r3);
|
|
1853
|
-
CUDA_CHECK(cudaGetLastError());
|
|
1854
|
-
|
|
1855
|
-
CUBLAS_CHECK(
|
|
1856
|
-
cublasGemmBatchedEx(ctx.cublas_handle(), CUBLAS_OP_T, CUBLAS_OP_N,
|
|
1857
|
-
ne01, ne11, ne10,
|
|
1858
|
-
alpha, (const void **) (ptrs_src.get() + 0*ne23), CUDA_R_16F, nb01/nb00,
|
|
1859
|
-
(const void **) (ptrs_src.get() + 1*ne23), CUDA_R_16F, nb11/nb10,
|
|
1860
|
-
beta, ( void **) (ptrs_dst.get() + 0*ne23), cu_data_type, ne01,
|
|
1861
|
-
ne23,
|
|
1862
|
-
cu_compute_type,
|
|
1863
|
-
CUBLAS_GEMM_DEFAULT_TENSOR_OP));
|
|
1864
|
-
}
|
|
1865
|
-
#endif
|
|
1866
|
-
|
|
1867
|
-
if (dst->op_params[0] == GGML_PREC_DEFAULT) {
|
|
1868
|
-
const to_fp32_cuda_t to_fp32_cuda = ggml_get_to_fp32_cuda(GGML_TYPE_F16);
|
|
1869
|
-
to_fp32_cuda(dst_f16.get(), dst_ddf, ne_dst, main_stream);
|
|
1870
|
-
}
|
|
1871
|
-
}
|
|
1872
|
-
|
|
1873
|
-
static void ggml_cuda_mul_mat(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
|
|
1874
|
-
const bool split = ggml_backend_buffer_is_cuda_split(src0->buffer);
|
|
1875
|
-
|
|
1876
|
-
int64_t min_compute_capability = INT_MAX;
|
|
1877
|
-
|
|
1878
|
-
bool any_pascal_with_slow_fp16 = false;
|
|
1879
|
-
if (split) {
|
|
1880
|
-
ggml_backend_cuda_split_buffer_type_context * buft_ctx = (ggml_backend_cuda_split_buffer_type_context *) src0->buffer->buft->context;
|
|
1881
|
-
auto & tensor_split = buft_ctx->tensor_split;
|
|
1882
|
-
for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) {
|
|
1883
|
-
// skip devices that are not going to do any work:
|
|
1884
|
-
if (tensor_split[id] >= (id + 1 < ggml_backend_cuda_get_device_count() ? tensor_split[id + 1] : 1.0f)) {
|
|
1885
|
-
continue;
|
|
1886
|
-
}
|
|
1887
|
-
|
|
1888
|
-
if (min_compute_capability > ggml_cuda_info().devices[id].cc) {
|
|
1889
|
-
min_compute_capability = ggml_cuda_info().devices[id].cc;
|
|
1890
|
-
}
|
|
1891
|
-
if (ggml_cuda_info().devices[id].cc == 610) {
|
|
1892
|
-
any_pascal_with_slow_fp16 = true;
|
|
1893
|
-
}
|
|
1894
|
-
}
|
|
1895
|
-
} else {
|
|
1896
|
-
min_compute_capability = ggml_cuda_info().devices[ctx.device].cc;
|
|
1897
|
-
any_pascal_with_slow_fp16 = ggml_cuda_info().devices[ctx.device].cc == 610;
|
|
1898
|
-
}
|
|
1899
|
-
|
|
1900
|
-
// check data types and tensor shapes for custom matrix multiplication kernels:
|
|
1901
|
-
bool use_dequantize_mul_mat_vec = (ggml_is_quantized(src0->type) || src0->type == GGML_TYPE_F16)
|
|
1902
|
-
&& src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32
|
|
1903
|
-
&& src0->ne[0] % GGML_CUDA_DMMV_X == 0 && src1->ne[1] == 1;
|
|
1904
|
-
|
|
1905
|
-
bool use_mul_mat_vec_q = ggml_is_quantized(src0->type)
|
|
1906
|
-
&& src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32
|
|
1907
|
-
&& src1->ne[1] <= MMVQ_MAX_BATCH_SIZE;
|
|
1908
|
-
|
|
1909
|
-
bool use_mul_mat_q = ggml_cuda_supports_mmq(src0->type)
|
|
1910
|
-
&& src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32;
|
|
1911
|
-
|
|
1912
|
-
#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)
|
|
1913
|
-
|
|
1914
|
-
const bool fp16_performance_good = min_compute_capability >= CC_RDNA1;
|
|
1915
|
-
|
|
1916
|
-
#ifdef CUDA_USE_TENSOR_CORES
|
|
1917
|
-
use_mul_mat_q = use_mul_mat_q && min_compute_capability < CC_RDNA3;
|
|
1918
|
-
#endif // CUDA_USE_TENSOR_CORES
|
|
1919
|
-
|
|
1920
|
-
#else
|
|
1921
|
-
|
|
1922
|
-
// fp16 performance is good on Volta or newer and on P100 (compute capability 6.0)
|
|
1923
|
-
const bool fp16_performance_good = min_compute_capability >= CC_PASCAL && !any_pascal_with_slow_fp16;
|
|
1924
|
-
|
|
1925
|
-
// mmvq and mmq need the __dp4a instruction which on NVIDIA is only available for CC >= 6.1
|
|
1926
|
-
use_mul_mat_vec_q = use_mul_mat_vec_q && min_compute_capability >= MIN_CC_DP4A;
|
|
1927
|
-
use_mul_mat_q = use_mul_mat_q && min_compute_capability >= MIN_CC_DP4A;
|
|
1928
|
-
|
|
1929
|
-
#ifdef CUDA_USE_TENSOR_CORES
|
|
1930
|
-
// when tensor cores are available, use them for large batch size
|
|
1931
|
-
// ref: https://github.com/ggerganov/llama.cpp/pull/3776
|
|
1932
|
-
use_mul_mat_q = use_mul_mat_q && (!fp16_performance_good || src1->ne[1] <= MMQ_MAX_BATCH_SIZE);
|
|
1933
|
-
#endif // CUDA_USE_TENSOR_CORES
|
|
1934
|
-
|
|
1935
|
-
#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)
|
|
1936
|
-
|
|
1937
|
-
// if mmvq is available it's a better choice than dmmv:
|
|
1938
|
-
#ifndef GGML_CUDA_FORCE_DMMV
|
|
1939
|
-
use_dequantize_mul_mat_vec = use_dequantize_mul_mat_vec && !use_mul_mat_vec_q;
|
|
1940
|
-
#endif // GGML_CUDA_FORCE_DMMV
|
|
1941
|
-
|
|
1942
|
-
// debug helpers
|
|
1943
|
-
//printf("src0: %8d %8d %8d %8d\n", src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3]);
|
|
1944
|
-
//printf(" %8d %8d %8d %8d\n", src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3]);
|
|
1945
|
-
//printf("src1: %8d %8d %8d %8d\n", src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3]);
|
|
1946
|
-
//printf(" %8d %8d %8d %8d\n", src1->nb[0], src1->nb[1], src1->nb[2], src1->nb[3]);
|
|
1947
|
-
//printf("src0 is contiguous %d, transposed %d, type = %s, name = %s\n", ggml_is_contiguous(src0), ggml_is_transposed(src0), ggml_type_name(src0->type), src0->name);
|
|
1948
|
-
//printf("src1 is contiguous %d, transposed %d, type = %s, name = %s\n", ggml_is_contiguous(src1), ggml_is_transposed(src1), ggml_type_name(src1->type), src1->name);
|
|
1949
|
-
|
|
1950
|
-
if (!split && !fp16_performance_good && src0->type == GGML_TYPE_F16 && ggml_is_permuted(src0) && ggml_is_permuted(src1) && src1->ne[1] == 1) {
|
|
1951
|
-
// KQ single-batch
|
|
1952
|
-
ggml_cuda_mul_mat_vec_p021(ctx, src0, src1, dst);
|
|
1953
|
-
} else if (!split && !fp16_performance_good && src0->type == GGML_TYPE_F16 && !ggml_is_contiguous(src0) && !ggml_is_transposed(src1) && src1->ne[1] == 1) {
|
|
1954
|
-
// KQV single-batch
|
|
1955
|
-
ggml_cuda_mul_mat_vec_nc(ctx, src0, src1, dst);
|
|
1956
|
-
} else if (!split && src0->type == GGML_TYPE_F16 && (src1->type == GGML_TYPE_F16 || fp16_performance_good) && !ggml_is_transposed(src0) && !ggml_is_transposed(src1) && src1->ne[2]*src1->ne[3] > 1) {
|
|
1957
|
-
// KQ + KQV multi-batch
|
|
1958
|
-
ggml_cuda_mul_mat_batched_cublas(ctx, src0, src1, dst);
|
|
1959
|
-
} else if (use_dequantize_mul_mat_vec) {
|
|
1960
|
-
ggml_cuda_op_mul_mat(ctx, src0, src1, dst, ggml_cuda_op_dequantize_mul_mat_vec, nullptr);
|
|
1961
|
-
} else if (use_mul_mat_vec_q) {
|
|
1962
|
-
ggml_cuda_op_mul_mat(ctx, src0, src1, dst, ggml_cuda_op_mul_mat_vec_q, quantize_row_q8_1_cuda);
|
|
1963
|
-
} else if (use_mul_mat_q) {
|
|
1964
|
-
ggml_cuda_op_mul_mat(ctx, src0, src1, dst, ggml_cuda_op_mul_mat_q, quantize_mmq_q8_1_cuda);
|
|
1965
|
-
} else {
|
|
1966
|
-
ggml_cuda_op_mul_mat(ctx, src0, src1, dst, ggml_cuda_op_mul_mat_cublas, nullptr);
|
|
1967
|
-
}
|
|
1968
|
-
}
|
|
1969
|
-
|
|
1970
|
-
struct mmid_row_mapping {
|
|
1971
|
-
int32_t i1;
|
|
1972
|
-
int32_t i2;
|
|
1973
|
-
};
|
|
1974
|
-
|
|
1975
|
-
static __global__ void k_copy_src1_to_contiguous(const char * __restrict__ src1_original, char * __restrict__ src1_contiguous,
|
|
1976
|
-
int * __restrict__ cur_src1_row, mmid_row_mapping * __restrict__ row_mapping,
|
|
1977
|
-
const char * __restrict ids, int64_t i02, size_t ids_nb1, size_t ids_nb0,
|
|
1978
|
-
int64_t ne11, int64_t ne10,
|
|
1979
|
-
size_t nb11, size_t nb12) {
|
|
1980
|
-
int32_t iid1 = blockIdx.x;
|
|
1981
|
-
int32_t id = blockIdx.y;
|
|
1982
|
-
|
|
1983
|
-
const int32_t row_id_i = *(const int32_t *) (ids + iid1*ids_nb1 + id*ids_nb0);
|
|
1984
|
-
|
|
1985
|
-
if (row_id_i != i02) {
|
|
1986
|
-
return;
|
|
1987
|
-
}
|
|
1988
|
-
|
|
1989
|
-
const int64_t i11 = id % ne11;
|
|
1990
|
-
const int64_t i12 = iid1;
|
|
1991
|
-
|
|
1992
|
-
__shared__ int src1_row;
|
|
1993
|
-
if (threadIdx.x == 0) {
|
|
1994
|
-
src1_row = atomicAdd(cur_src1_row, 1);
|
|
1995
|
-
row_mapping[src1_row] = {id, iid1};
|
|
1996
|
-
}
|
|
1997
|
-
__syncthreads();
|
|
1998
|
-
|
|
1999
|
-
const float * src1_row_original = (const float *)(src1_original + i11*nb11 + i12*nb12);
|
|
2000
|
-
float * src1_row_contiguous = (float *)(src1_contiguous + src1_row*nb11);
|
|
2001
|
-
|
|
2002
|
-
for (int i = threadIdx.x; i < ne10; i += blockDim.x) {
|
|
2003
|
-
src1_row_contiguous[i] = src1_row_original[i];
|
|
2004
|
-
}
|
|
2005
|
-
}
|
|
2006
|
-
|
|
2007
|
-
static __global__ void k_copy_dst_from_contiguous(char * __restrict__ dst_original, const char * __restrict__ dst_contiguous,
|
|
2008
|
-
const mmid_row_mapping * __restrict__ row_mapping,
|
|
2009
|
-
int64_t ne0,
|
|
2010
|
-
size_t nb1, size_t nb2) {
|
|
2011
|
-
int32_t i = blockIdx.x;
|
|
2012
|
-
|
|
2013
|
-
const int32_t i1 = row_mapping[i].i1;
|
|
2014
|
-
const int32_t i2 = row_mapping[i].i2;
|
|
2015
|
-
|
|
2016
|
-
const float * dst_row_contiguous = (const float *)(dst_contiguous + i*nb1);
|
|
2017
|
-
float * dst_row_original = (float *)(dst_original + i1*nb1 + i2*nb2);
|
|
2018
|
-
|
|
2019
|
-
for (int j = threadIdx.x; j < ne0; j += blockDim.x) {
|
|
2020
|
-
dst_row_original[j] = dst_row_contiguous[j];
|
|
2021
|
-
}
|
|
2022
|
-
}
|
|
2023
|
-
|
|
2024
|
-
static void ggml_cuda_mul_mat_id(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
|
2025
|
-
const ggml_tensor * src0 = dst->src[0];
|
|
2026
|
-
const ggml_tensor * src1 = dst->src[1];
|
|
2027
|
-
const ggml_tensor * ids = dst->src[2];
|
|
2028
|
-
|
|
2029
|
-
GGML_TENSOR_BINARY_OP_LOCALS
|
|
2030
|
-
|
|
2031
|
-
GGML_ASSERT(!ggml_backend_buffer_is_cuda_split(src0->buffer) && "mul_mat_id does not support split buffers");
|
|
2032
|
-
|
|
2033
|
-
cudaStream_t stream = ctx.stream();
|
|
2034
|
-
|
|
2035
|
-
const int64_t n_as = ne02;
|
|
2036
|
-
const int64_t n_ids = ids->ne[0];
|
|
2037
|
-
|
|
2038
|
-
std::vector<char> ids_host(ggml_nbytes(ids));
|
|
2039
|
-
const char * ids_dev = (const char *) ids->data;
|
|
2040
|
-
CUDA_CHECK(cudaMemcpyAsync(ids_host.data(), ids_dev, ggml_nbytes(ids), cudaMemcpyDeviceToHost, stream));
|
|
2041
|
-
CUDA_CHECK(cudaStreamSynchronize(stream));
|
|
2042
|
-
|
|
2043
|
-
ggml_tensor src0_row = *src0;
|
|
2044
|
-
ggml_tensor src1_row = *src1;
|
|
2045
|
-
ggml_tensor dst_row = *dst;
|
|
2046
|
-
|
|
2047
|
-
char * src0_original = (char *) src0->data;
|
|
2048
|
-
char * src1_original = (char *) src1->data;
|
|
2049
|
-
char * dst_original = (char *) dst->data;
|
|
2050
|
-
|
|
2051
|
-
src0_row.ne[2] = 1;
|
|
2052
|
-
src0_row.ne[3] = 1;
|
|
2053
|
-
src0_row.nb[3] = nb02;
|
|
2054
|
-
|
|
2055
|
-
src1_row.ne[1] = 1;
|
|
2056
|
-
src1_row.ne[2] = 1;
|
|
2057
|
-
src1_row.ne[3] = 1;
|
|
2058
|
-
src1_row.nb[2] = nb11;
|
|
2059
|
-
src1_row.nb[3] = nb11;
|
|
2060
|
-
|
|
2061
|
-
dst_row.ne[1] = 1;
|
|
2062
|
-
dst_row.ne[2] = 1;
|
|
2063
|
-
dst_row.ne[3] = 1;
|
|
2064
|
-
dst_row.nb[2] = nb1;
|
|
2065
|
-
dst_row.nb[3] = nb1;
|
|
2066
|
-
|
|
2067
|
-
if (ne12 == 1) {
|
|
2068
|
-
for (int64_t iid1 = 0; iid1 < ids->ne[1]; iid1++) {
|
|
2069
|
-
for (int64_t id = 0; id < n_ids; id++) {
|
|
2070
|
-
const int32_t i02 = *(const int32_t *) (ids_host.data() + iid1*ids->nb[1] + id*ids->nb[0]);
|
|
2071
|
-
|
|
2072
|
-
GGML_ASSERT(i02 >= 0 && i02 < n_as);
|
|
2073
|
-
|
|
2074
|
-
const int64_t i11 = id % ne11;
|
|
2075
|
-
const int64_t i12 = iid1;
|
|
2076
|
-
|
|
2077
|
-
const int64_t i1 = id;
|
|
2078
|
-
const int64_t i2 = i12;
|
|
2079
|
-
|
|
2080
|
-
src0_row.data = src0_original + i02*nb02;
|
|
2081
|
-
src1_row.data = src1_original + i11*nb11 + i12*nb12;
|
|
2082
|
-
dst_row.data = dst_original + i1*nb1 + i2*nb2;
|
|
2083
|
-
|
|
2084
|
-
ggml_cuda_mul_mat(ctx, &src0_row, &src1_row, &dst_row);
|
|
2085
|
-
}
|
|
2086
|
-
}
|
|
2087
|
-
} else {
|
|
2088
|
-
ggml_cuda_pool_alloc<char> src1_contiguous(ctx.pool(), sizeof(float)*ggml_nelements(src1));
|
|
2089
|
-
ggml_cuda_pool_alloc<char> dst_contiguous(ctx.pool(), sizeof(float)*ggml_nelements(dst));
|
|
2090
|
-
|
|
2091
|
-
src1_row.data = src1_contiguous.get();
|
|
2092
|
-
dst_row.data = dst_contiguous.get();
|
|
2093
|
-
|
|
2094
|
-
for (int64_t i02 = 0; i02 < n_as; i02++) {
|
|
2095
|
-
int64_t num_src1_rows = 0;
|
|
2096
|
-
|
|
2097
|
-
for (int64_t iid1 = 0; iid1 < ids->ne[1]; iid1++) {
|
|
2098
|
-
for (int64_t id = 0; id < n_ids; id++) {
|
|
2099
|
-
const int32_t row_id_i = *(const int32_t *) (ids_host.data() + iid1*ids->nb[1] + id*ids->nb[0]);
|
|
2100
|
-
|
|
2101
|
-
GGML_ASSERT(row_id_i >= 0 && row_id_i < n_as);
|
|
2102
|
-
|
|
2103
|
-
if (row_id_i != i02) {
|
|
2104
|
-
continue;
|
|
2105
|
-
}
|
|
2106
|
-
|
|
2107
|
-
num_src1_rows++;
|
|
2108
|
-
}
|
|
2109
|
-
}
|
|
2110
|
-
|
|
2111
|
-
if (num_src1_rows == 0) {
|
|
2112
|
-
continue;
|
|
2113
|
-
}
|
|
2114
|
-
|
|
2115
|
-
ggml_cuda_pool_alloc<int> dev_cur_src1_row(ctx.pool(), 1);
|
|
2116
|
-
ggml_cuda_pool_alloc<mmid_row_mapping> dev_row_mapping(ctx.pool(), num_src1_rows);
|
|
2117
|
-
CUDA_CHECK(cudaMemsetAsync(dev_cur_src1_row.get(), 0, sizeof(int), stream));
|
|
2118
|
-
|
|
2119
|
-
{
|
|
2120
|
-
dim3 block_dims(std::min((unsigned int)ne10, 768u));
|
|
2121
|
-
dim3 grid_dims(ids->ne[1], n_ids);
|
|
2122
|
-
k_copy_src1_to_contiguous<<<grid_dims, block_dims, 0, stream>>>(
|
|
2123
|
-
src1_original, src1_contiguous.get(),
|
|
2124
|
-
dev_cur_src1_row.get(), dev_row_mapping.get(),
|
|
2125
|
-
ids_dev, i02, ids->nb[1], ids->nb[0],
|
|
2126
|
-
ne11, ne10,
|
|
2127
|
-
nb11, nb12);
|
|
2128
|
-
CUDA_CHECK(cudaGetLastError());
|
|
2129
|
-
}
|
|
2130
|
-
|
|
2131
|
-
src0_row.data = src0_original + i02*nb02;
|
|
2132
|
-
|
|
2133
|
-
GGML_ASSERT(nb11 == sizeof(float)*ne10);
|
|
2134
|
-
GGML_ASSERT(nb1 == sizeof(float)*ne0);
|
|
2135
|
-
|
|
2136
|
-
src1_row.ne[1] = num_src1_rows;
|
|
2137
|
-
src1_row.nb[1] = nb11;
|
|
2138
|
-
src1_row.nb[2] = num_src1_rows*nb11;
|
|
2139
|
-
src1_row.nb[3] = num_src1_rows*nb11;
|
|
2140
|
-
|
|
2141
|
-
dst_row.ne[1] = num_src1_rows;
|
|
2142
|
-
dst_row.nb[1] = nb1;
|
|
2143
|
-
dst_row.nb[2] = num_src1_rows*nb1;
|
|
2144
|
-
dst_row.nb[3] = num_src1_rows*nb1;
|
|
2145
|
-
|
|
2146
|
-
ggml_cuda_mul_mat(ctx, &src0_row, &src1_row, &dst_row);
|
|
2147
|
-
|
|
2148
|
-
{
|
|
2149
|
-
dim3 block_dims(std::min((unsigned int)ne0, 768u));
|
|
2150
|
-
dim3 grid_dims(num_src1_rows);
|
|
2151
|
-
k_copy_dst_from_contiguous<<<grid_dims, block_dims, 0, stream>>>(
|
|
2152
|
-
dst_original, dst_contiguous.get(),
|
|
2153
|
-
dev_row_mapping.get(),
|
|
2154
|
-
ne0,
|
|
2155
|
-
nb1, nb2);
|
|
2156
|
-
CUDA_CHECK(cudaGetLastError());
|
|
2157
|
-
}
|
|
2158
|
-
}
|
|
2159
|
-
}
|
|
2160
|
-
}
|
|
2161
|
-
|
|
2162
|
-
static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct ggml_tensor * dst) {
|
|
2163
|
-
// why is this here instead of mul_mat?
|
|
2164
|
-
if (dst->src[0] != nullptr && ggml_backend_buffer_is_cuda_split(dst->src[0]->buffer)) {
|
|
2165
|
-
ggml_cuda_set_peer_access(dst->src[1]->ne[1], ctx.device);
|
|
2166
|
-
}
|
|
2167
|
-
|
|
2168
|
-
switch (dst->op) {
|
|
2169
|
-
case GGML_OP_REPEAT:
|
|
2170
|
-
ggml_cuda_op_repeat(ctx, dst);
|
|
2171
|
-
break;
|
|
2172
|
-
case GGML_OP_GET_ROWS:
|
|
2173
|
-
ggml_cuda_op_get_rows(ctx, dst);
|
|
2174
|
-
break;
|
|
2175
|
-
case GGML_OP_DUP:
|
|
2176
|
-
ggml_cuda_dup(ctx, dst);
|
|
2177
|
-
break;
|
|
2178
|
-
case GGML_OP_CPY:
|
|
2179
|
-
ggml_cuda_cpy(ctx, dst->src[0], dst->src[1]);
|
|
2180
|
-
break;
|
|
2181
|
-
case GGML_OP_CONT:
|
|
2182
|
-
ggml_cuda_dup(ctx, dst);
|
|
2183
|
-
break;
|
|
2184
|
-
case GGML_OP_ADD:
|
|
2185
|
-
ggml_cuda_op_add(ctx, dst);
|
|
2186
|
-
break;
|
|
2187
|
-
case GGML_OP_ACC:
|
|
2188
|
-
ggml_cuda_op_acc(ctx, dst);
|
|
2189
|
-
break;
|
|
2190
|
-
case GGML_OP_MUL:
|
|
2191
|
-
ggml_cuda_op_mul(ctx, dst);
|
|
2192
|
-
break;
|
|
2193
|
-
case GGML_OP_DIV:
|
|
2194
|
-
ggml_cuda_op_div(ctx, dst);
|
|
2195
|
-
break;
|
|
2196
|
-
case GGML_OP_UNARY:
|
|
2197
|
-
switch (ggml_get_unary_op(dst)) {
|
|
2198
|
-
case GGML_UNARY_OP_GELU:
|
|
2199
|
-
ggml_cuda_op_gelu(ctx, dst);
|
|
2200
|
-
break;
|
|
2201
|
-
case GGML_UNARY_OP_SILU:
|
|
2202
|
-
ggml_cuda_op_silu(ctx, dst);
|
|
2203
|
-
break;
|
|
2204
|
-
case GGML_UNARY_OP_GELU_QUICK:
|
|
2205
|
-
ggml_cuda_op_gelu_quick(ctx, dst);
|
|
2206
|
-
break;
|
|
2207
|
-
case GGML_UNARY_OP_TANH:
|
|
2208
|
-
ggml_cuda_op_tanh(ctx, dst);
|
|
2209
|
-
break;
|
|
2210
|
-
case GGML_UNARY_OP_RELU:
|
|
2211
|
-
ggml_cuda_op_relu(ctx, dst);
|
|
2212
|
-
break;
|
|
2213
|
-
case GGML_UNARY_OP_SIGMOID:
|
|
2214
|
-
ggml_cuda_op_sigmoid(ctx, dst);
|
|
2215
|
-
break;
|
|
2216
|
-
case GGML_UNARY_OP_HARDSIGMOID:
|
|
2217
|
-
ggml_cuda_op_hardsigmoid(ctx, dst);
|
|
2218
|
-
break;
|
|
2219
|
-
case GGML_UNARY_OP_HARDSWISH:
|
|
2220
|
-
ggml_cuda_op_hardswish(ctx, dst);
|
|
2221
|
-
break;
|
|
2222
|
-
default:
|
|
2223
|
-
return false;
|
|
2224
|
-
}
|
|
2225
|
-
break;
|
|
2226
|
-
case GGML_OP_NORM:
|
|
2227
|
-
ggml_cuda_op_norm(ctx, dst);
|
|
2228
|
-
break;
|
|
2229
|
-
case GGML_OP_GROUP_NORM:
|
|
2230
|
-
ggml_cuda_op_group_norm(ctx, dst);
|
|
2231
|
-
break;
|
|
2232
|
-
case GGML_OP_CONCAT:
|
|
2233
|
-
ggml_cuda_op_concat(ctx, dst);
|
|
2234
|
-
break;
|
|
2235
|
-
case GGML_OP_UPSCALE:
|
|
2236
|
-
ggml_cuda_op_upscale(ctx, dst);
|
|
2237
|
-
break;
|
|
2238
|
-
case GGML_OP_PAD:
|
|
2239
|
-
ggml_cuda_op_pad(ctx, dst);
|
|
2240
|
-
break;
|
|
2241
|
-
case GGML_OP_ARANGE:
|
|
2242
|
-
ggml_cuda_op_arange(ctx, dst);
|
|
2243
|
-
break;
|
|
2244
|
-
case GGML_OP_TIMESTEP_EMBEDDING:
|
|
2245
|
-
ggml_cuda_op_timestep_embedding(ctx, dst);
|
|
2246
|
-
break;
|
|
2247
|
-
case GGML_OP_LEAKY_RELU:
|
|
2248
|
-
ggml_cuda_op_leaky_relu(ctx, dst);
|
|
2249
|
-
break;
|
|
2250
|
-
case GGML_OP_RMS_NORM:
|
|
2251
|
-
ggml_cuda_op_rms_norm(ctx, dst);
|
|
2252
|
-
break;
|
|
2253
|
-
case GGML_OP_MUL_MAT:
|
|
2254
|
-
if (dst->src[0]->ne[3] != dst->src[1]->ne[3]) {
|
|
2255
|
-
GGML_CUDA_LOG_ERROR("%s: cannot compute %s: src0->ne[3] = %" PRId64 ", src1->ne[3] = %" PRId64 " - fallback to CPU\n", __func__, dst->name, dst->src[0]->ne[3], dst->src[1]->ne[3]);
|
|
2256
|
-
return false;
|
|
2257
|
-
} else {
|
|
2258
|
-
ggml_cuda_mul_mat(ctx, dst->src[0], dst->src[1], dst);
|
|
2259
|
-
}
|
|
2260
|
-
break;
|
|
2261
|
-
case GGML_OP_MUL_MAT_ID:
|
|
2262
|
-
ggml_cuda_mul_mat_id(ctx, dst);
|
|
2263
|
-
break;
|
|
2264
|
-
case GGML_OP_SCALE:
|
|
2265
|
-
ggml_cuda_op_scale(ctx, dst);
|
|
2266
|
-
break;
|
|
2267
|
-
case GGML_OP_SQR:
|
|
2268
|
-
ggml_cuda_op_sqr(ctx, dst);
|
|
2269
|
-
break;
|
|
2270
|
-
case GGML_OP_CLAMP:
|
|
2271
|
-
ggml_cuda_op_clamp(ctx, dst);
|
|
2272
|
-
break;
|
|
2273
|
-
case GGML_OP_NONE:
|
|
2274
|
-
case GGML_OP_RESHAPE:
|
|
2275
|
-
case GGML_OP_VIEW:
|
|
2276
|
-
case GGML_OP_PERMUTE:
|
|
2277
|
-
case GGML_OP_TRANSPOSE:
|
|
2278
|
-
break;
|
|
2279
|
-
case GGML_OP_DIAG_MASK_INF:
|
|
2280
|
-
ggml_cuda_op_diag_mask_inf(ctx, dst);
|
|
2281
|
-
break;
|
|
2282
|
-
case GGML_OP_SOFT_MAX:
|
|
2283
|
-
ggml_cuda_op_soft_max(ctx, dst);
|
|
2284
|
-
break;
|
|
2285
|
-
case GGML_OP_ROPE:
|
|
2286
|
-
ggml_cuda_op_rope(ctx, dst);
|
|
2287
|
-
break;
|
|
2288
|
-
case GGML_OP_IM2COL:
|
|
2289
|
-
ggml_cuda_op_im2col(ctx, dst);
|
|
2290
|
-
break;
|
|
2291
|
-
case GGML_OP_POOL_2D:
|
|
2292
|
-
ggml_cuda_op_pool2d(ctx, dst);
|
|
2293
|
-
break;
|
|
2294
|
-
case GGML_OP_SUM_ROWS:
|
|
2295
|
-
ggml_cuda_op_sum_rows(ctx, dst);
|
|
2296
|
-
break;
|
|
2297
|
-
case GGML_OP_ARGSORT:
|
|
2298
|
-
ggml_cuda_op_argsort(ctx, dst);
|
|
2299
|
-
break;
|
|
2300
|
-
case GGML_OP_FLASH_ATTN_EXT:
|
|
2301
|
-
ggml_cuda_flash_attn_ext(ctx, dst);
|
|
2302
|
-
break;
|
|
2303
|
-
default:
|
|
2304
|
-
return false;
|
|
2305
|
-
}
|
|
2306
|
-
|
|
2307
|
-
cudaError_t err = cudaGetLastError();
|
|
2308
|
-
if (err != cudaSuccess) {
|
|
2309
|
-
GGML_CUDA_LOG_ERROR("%s: %s failed\n", __func__, ggml_op_desc(dst));
|
|
2310
|
-
CUDA_CHECK(err);
|
|
2311
|
-
}
|
|
2312
|
-
|
|
2313
|
-
return true;
|
|
2314
|
-
}
|
|
2315
|
-
|
|
2316
|
-
////////////////////////////////////////////////////////////////////////////////
|
|
2317
|
-
|
|
2318
|
-
// backend
|
|
2319
|
-
|
|
2320
|
-
GGML_CALL static const char * ggml_backend_cuda_name(ggml_backend_t backend) {
|
|
2321
|
-
ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context;
|
|
2322
|
-
|
|
2323
|
-
return cuda_ctx->name.c_str();
|
|
2324
|
-
}
|
|
2325
|
-
|
|
2326
|
-
GGML_CALL static void ggml_backend_cuda_free(ggml_backend_t backend) {
|
|
2327
|
-
ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context;
|
|
2328
|
-
|
|
2329
|
-
delete cuda_ctx;
|
|
2330
|
-
delete backend;
|
|
2331
|
-
}
|
|
2332
|
-
|
|
2333
|
-
GGML_CALL static ggml_backend_buffer_type_t ggml_backend_cuda_get_default_buffer_type(ggml_backend_t backend) {
|
|
2334
|
-
ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context;
|
|
2335
|
-
|
|
2336
|
-
return ggml_backend_cuda_buffer_type(cuda_ctx->device);
|
|
2337
|
-
}
|
|
2338
|
-
|
|
2339
|
-
GGML_CALL static void ggml_backend_cuda_set_tensor_async(ggml_backend_t backend, ggml_tensor * tensor, const void * data, size_t offset, size_t size) {
|
|
2340
|
-
ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context;
|
|
2341
|
-
ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;
|
|
2342
|
-
|
|
2343
|
-
GGML_ASSERT(buf->buft == ggml_backend_cuda_buffer_type(cuda_ctx->device) && "unsupported buffer type");
|
|
2344
|
-
|
|
2345
|
-
CUDA_CHECK(cudaMemcpyAsync((char *)tensor->data + offset, data, size, cudaMemcpyHostToDevice, cuda_ctx->stream()));
|
|
2346
|
-
}
|
|
2347
|
-
|
|
2348
|
-
GGML_CALL static void ggml_backend_cuda_get_tensor_async(ggml_backend_t backend, const ggml_tensor * tensor, void * data, size_t offset, size_t size) {
|
|
2349
|
-
ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context;
|
|
2350
|
-
ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;
|
|
2351
|
-
|
|
2352
|
-
GGML_ASSERT(buf->buft == ggml_backend_cuda_buffer_type(cuda_ctx->device) && "unsupported buffer type");
|
|
2353
|
-
|
|
2354
|
-
CUDA_CHECK(cudaMemcpyAsync(data, (const char *)tensor->data + offset, size, cudaMemcpyDeviceToHost, cuda_ctx->stream()));
|
|
2355
|
-
}
|
|
2356
|
-
|
|
2357
|
-
GGML_CALL static bool ggml_backend_cuda_cpy_tensor_async(ggml_backend_t backend_src, ggml_backend_t backend_dst, const ggml_tensor * src, ggml_tensor * dst) {
|
|
2358
|
-
GGML_ASSERT(ggml_backend_is_cuda(backend_src) || ggml_backend_is_cuda(backend_dst));
|
|
2359
|
-
|
|
2360
|
-
ggml_backend_buffer_t buf_src = src->view_src ? src->view_src->buffer : src->buffer;
|
|
2361
|
-
ggml_backend_buffer_t buf_dst = dst->view_src ? dst->view_src->buffer : dst->buffer;
|
|
2362
|
-
|
|
2363
|
-
if (!ggml_backend_buffer_is_cuda(src->buffer)) {
|
|
2364
|
-
return false;
|
|
2365
|
-
}
|
|
2366
|
-
|
|
2367
|
-
if (!ggml_backend_buffer_is_cuda(dst->buffer)) {
|
|
2368
|
-
return false;
|
|
2369
|
-
}
|
|
2370
|
-
|
|
2371
|
-
// device -> device
|
|
2372
|
-
ggml_backend_cuda_context * cuda_ctx_src = (ggml_backend_cuda_context *)backend_src->context;
|
|
2373
|
-
ggml_backend_cuda_context * cuda_ctx_dst = (ggml_backend_cuda_context *)backend_dst->context;
|
|
2374
|
-
|
|
2375
|
-
if (backend_src != backend_dst) {
|
|
2376
|
-
ggml_backend_cuda_buffer_context * buf_ctx_src = (ggml_backend_cuda_buffer_context *)buf_src->context;
|
|
2377
|
-
ggml_backend_cuda_buffer_context * buf_ctx_dst = (ggml_backend_cuda_buffer_context *)buf_dst->context;
|
|
2378
|
-
|
|
2379
|
-
GGML_ASSERT(cuda_ctx_src->device == buf_ctx_src->device);
|
|
2380
|
-
GGML_ASSERT(cuda_ctx_dst->device == buf_ctx_dst->device);
|
|
2381
|
-
|
|
2382
|
-
// copy on src stream
|
|
2383
|
-
if (cuda_ctx_src->device == cuda_ctx_dst->device) {
|
|
2384
|
-
CUDA_CHECK(cudaMemcpyAsync(dst->data, src->data, ggml_nbytes(dst), cudaMemcpyDeviceToDevice, cuda_ctx_dst->stream()));
|
|
2385
|
-
} else {
|
|
2386
|
-
#ifdef GGML_CUDA_NO_PEER_COPY
|
|
2387
|
-
return false;
|
|
2388
|
-
#else
|
|
2389
|
-
CUDA_CHECK(cudaMemcpyPeerAsync(dst->data, cuda_ctx_dst->device, src->data, cuda_ctx_src->device, ggml_nbytes(dst), cuda_ctx_src->stream()));
|
|
2390
|
-
#endif
|
|
2391
|
-
}
|
|
2392
|
-
|
|
2393
|
-
// record event on src stream
|
|
2394
|
-
if (!cuda_ctx_src->copy_event) {
|
|
2395
|
-
ggml_cuda_set_device(cuda_ctx_src->device);
|
|
2396
|
-
CUDA_CHECK(cudaEventCreateWithFlags(&cuda_ctx_src->copy_event, cudaEventDisableTiming));
|
|
2397
|
-
}
|
|
2398
|
-
|
|
2399
|
-
CUDA_CHECK(cudaEventRecord(cuda_ctx_src->copy_event, cuda_ctx_src->stream()));
|
|
2400
|
-
|
|
2401
|
-
// wait on dst stream for the copy to complete
|
|
2402
|
-
CUDA_CHECK(cudaStreamWaitEvent(cuda_ctx_dst->stream(), cuda_ctx_src->copy_event, 0));
|
|
2403
|
-
} else {
|
|
2404
|
-
// src and dst are on the same backend
|
|
2405
|
-
CUDA_CHECK(cudaMemcpyAsync(dst->data, src->data, ggml_nbytes(dst), cudaMemcpyDeviceToDevice, cuda_ctx_dst->stream()));
|
|
2406
|
-
}
|
|
2407
|
-
return true;
|
|
2408
|
-
}
|
|
2409
|
-
|
|
2410
|
-
GGML_CALL static void ggml_backend_cuda_synchronize(ggml_backend_t backend) {
|
|
2411
|
-
ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context;
|
|
2412
|
-
|
|
2413
|
-
CUDA_CHECK(cudaStreamSynchronize(cuda_ctx->stream()));
|
|
2414
|
-
|
|
2415
|
-
GGML_UNUSED(backend);
|
|
2416
|
-
}
|
|
2417
|
-
|
|
2418
|
-
static void set_ggml_graph_node_properties(ggml_tensor * node, ggml_graph_node_properties * graph_node_properties) {
|
|
2419
|
-
graph_node_properties->node_address = node->data;
|
|
2420
|
-
graph_node_properties->node_op = node->op;
|
|
2421
|
-
for (int i = 0; i < GGML_MAX_DIMS; i++) {
|
|
2422
|
-
graph_node_properties->ne[i] = node->ne[i];
|
|
2423
|
-
graph_node_properties->nb[i] = node->nb[i];
|
|
2424
|
-
}
|
|
2425
|
-
for (int i = 0; i < GGML_MAX_SRC; i++) {
|
|
2426
|
-
graph_node_properties->src_address[i] = node->src[i] ? node->src[i]->data : nullptr;
|
|
2427
|
-
}
|
|
2428
|
-
}
|
|
2429
|
-
|
|
2430
|
-
static bool ggml_graph_node_has_matching_properties(ggml_tensor * node, ggml_graph_node_properties * graph_node_properties) {
|
|
2431
|
-
if (node->data != graph_node_properties->node_address &&
|
|
2432
|
-
node->op != GGML_OP_CPY &&
|
|
2433
|
-
node->op != GGML_OP_VIEW) {
|
|
2434
|
-
return false;
|
|
2435
|
-
}
|
|
2436
|
-
|
|
2437
|
-
if (node->op != graph_node_properties->node_op) {
|
|
2438
|
-
return false;
|
|
2439
|
-
}
|
|
2440
|
-
|
|
2441
|
-
for (int i = 0; i < GGML_MAX_DIMS; i++) {
|
|
2442
|
-
if (node->ne[i] != graph_node_properties->ne[i]) {
|
|
2443
|
-
return false;
|
|
2444
|
-
}
|
|
2445
|
-
if (node->nb[i] != graph_node_properties->nb[i]) {
|
|
2446
|
-
return false;
|
|
2447
|
-
}
|
|
2448
|
-
}
|
|
2449
|
-
|
|
2450
|
-
for (int i = 0; i < GGML_MAX_SRC; i++) {
|
|
2451
|
-
if (node->src[i] &&
|
|
2452
|
-
node->src[i]->data != graph_node_properties->src_address[i] &&
|
|
2453
|
-
node->op != GGML_OP_CPY &&
|
|
2454
|
-
node->op != GGML_OP_VIEW
|
|
2455
|
-
) {
|
|
2456
|
-
return false;
|
|
2457
|
-
}
|
|
2458
|
-
}
|
|
2459
|
-
return true;
|
|
2460
|
-
}
|
|
2461
|
-
|
|
2462
|
-
GGML_CALL static enum ggml_status ggml_backend_cuda_graph_compute(ggml_backend_t backend, ggml_cgraph * cgraph) {
|
|
2463
|
-
ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context;
|
|
2464
|
-
|
|
2465
|
-
ggml_cuda_set_device(cuda_ctx->device);
|
|
2466
|
-
|
|
2467
|
-
#ifdef USE_CUDA_GRAPH
|
|
2468
|
-
static const bool disable_cuda_graphs_due_to_env = (getenv("GGML_CUDA_DISABLE_GRAPHS") != nullptr);
|
|
2469
|
-
|
|
2470
|
-
// Objects required for CUDA Graph
|
|
2471
|
-
if (cuda_ctx->cuda_graph == nullptr) {
|
|
2472
|
-
cuda_ctx->cuda_graph.reset(new ggml_cuda_graph());
|
|
2473
|
-
}
|
|
2474
|
-
|
|
2475
|
-
bool use_cuda_graph = true;
|
|
2476
|
-
bool cuda_graph_update_required = false;
|
|
2477
|
-
// vector of pointers to CUDA cpy kernels, which are required to identify
|
|
2478
|
-
// kernel parameters which need updated in the graph for each token
|
|
2479
|
-
std::vector<void *> ggml_cuda_cpy_fn_ptrs;
|
|
2480
|
-
|
|
2481
|
-
if (cuda_ctx->cuda_graph->graph == nullptr) {
|
|
2482
|
-
if (ggml_cuda_info().devices[cuda_ctx->device].cc < CC_AMPERE) {
|
|
2483
|
-
cuda_ctx->cuda_graph->disable_due_to_gpu_arch = true;
|
|
2484
|
-
#ifndef NDEBUG
|
|
2485
|
-
GGML_CUDA_LOG_WARN("%s: disabling CUDA graphs due to GPU architecture\n", __func__);
|
|
2486
|
-
#endif
|
|
2487
|
-
}
|
|
2488
|
-
}
|
|
2489
|
-
|
|
2490
|
-
// Disable CUDA graphs in presence of env var, old GPU, use-case which is changing too rapidly,
|
|
2491
|
-
// or previous graph capture failure.
|
|
2492
|
-
// Also disable for multi-gpu for now. TO DO investigate
|
|
2493
|
-
if (disable_cuda_graphs_due_to_env
|
|
2494
|
-
|| cuda_ctx->cuda_graph->disable_due_to_gpu_arch
|
|
2495
|
-
|| cuda_ctx->cuda_graph->disable_due_to_too_many_updates
|
|
2496
|
-
|| cuda_ctx->cuda_graph->disable_due_to_failed_graph_capture) {
|
|
2497
|
-
use_cuda_graph = false;
|
|
2498
|
-
}
|
|
2499
|
-
|
|
2500
|
-
if (use_cuda_graph) {
|
|
2501
|
-
if (cuda_ctx->cuda_graph->instance == nullptr) {
|
|
2502
|
-
cuda_graph_update_required = true;
|
|
2503
|
-
}
|
|
2504
|
-
|
|
2505
|
-
// Check if the graph size has changed
|
|
2506
|
-
if (cuda_ctx->cuda_graph->ggml_graph_properties.size() != (size_t)cgraph->n_nodes) {
|
|
2507
|
-
cuda_graph_update_required = true;
|
|
2508
|
-
cuda_ctx->cuda_graph->ggml_graph_properties.resize(cgraph->n_nodes);
|
|
2509
|
-
}
|
|
2510
|
-
|
|
2511
|
-
// Loop over nodes in GGML graph to determine if CUDA graph update is required
|
|
2512
|
-
// and store properties to allow this comparison for the next token
|
|
2513
|
-
for (int i = 0; i < cgraph->n_nodes; i++) {
|
|
2514
|
-
bool has_matching_properties = true;
|
|
2515
|
-
if (!cuda_graph_update_required) {
|
|
2516
|
-
has_matching_properties = ggml_graph_node_has_matching_properties(cgraph->nodes[i], &cuda_ctx->cuda_graph->ggml_graph_properties[i]);
|
|
2517
|
-
}
|
|
2518
|
-
if (!has_matching_properties) {
|
|
2519
|
-
cuda_graph_update_required = true;
|
|
2520
|
-
}
|
|
2521
|
-
set_ggml_graph_node_properties(cgraph->nodes[i], &cuda_ctx->cuda_graph->ggml_graph_properties[i]);
|
|
2522
|
-
}
|
|
2523
|
-
|
|
2524
|
-
// Loop over nodes in GGML graph to obtain info needed for CUDA graph
|
|
2525
|
-
cuda_ctx->cuda_graph->updated_kernel_arg.clear();
|
|
2526
|
-
for (int i = 0; i < cgraph->n_nodes; i++) {
|
|
2527
|
-
ggml_tensor * node = cgraph->nodes[i];
|
|
2528
|
-
|
|
2529
|
-
if (node->src[0] && ggml_backend_buffer_is_cuda_split(node->src[0]->buffer)) {
|
|
2530
|
-
use_cuda_graph = false; // Split buffers are not supported by CUDA graph capture
|
|
2531
|
-
#ifndef NDEBUG
|
|
2532
|
-
GGML_CUDA_LOG_WARN("%s: disabling CUDA graphs due to split buffer\n", __func__);
|
|
2533
|
-
#endif
|
|
2534
|
-
}
|
|
2535
|
-
|
|
2536
|
-
if (node->op == GGML_OP_MUL_MAT_ID) {
|
|
2537
|
-
use_cuda_graph = false; // This node type is not supported by CUDA graph capture
|
|
2538
|
-
#ifndef NDEBUG
|
|
2539
|
-
GGML_CUDA_LOG_WARN("%s: disabling CUDA graphs due to mul_mat_id\n", __func__);
|
|
2540
|
-
#endif
|
|
2541
|
-
}
|
|
2542
|
-
|
|
2543
|
-
if (node->op == GGML_OP_ADD && node->src[1] && node->src[1]->ne[1] > 1) {
|
|
2544
|
-
// disable CUDA graphs for batch size > 1 for now.
|
|
2545
|
-
// Changes in batch size or context size can cause changes to the grid size of some kernels.
|
|
2546
|
-
use_cuda_graph = false;
|
|
2547
|
-
#ifndef NDEBUG
|
|
2548
|
-
GGML_CUDA_LOG_WARN("%s: disabling CUDA graphs due to batch size > 1 [%s] [%ld %ld %ld %ld]\n", __func__, node->name, node->ne[0], node->ne[1], node->ne[2], node->ne[3]);
|
|
2549
|
-
#endif
|
|
2550
|
-
}
|
|
2551
|
-
|
|
2552
|
-
if (node->op == GGML_OP_CPY) {
|
|
2553
|
-
// store the copy op parameter which changes with each token.
|
|
2554
|
-
cuda_ctx->cuda_graph->updated_kernel_arg.push_back((char **) &(node->src[1]->data));
|
|
2555
|
-
// store a pointer to each copy op CUDA kernel to identify it later
|
|
2556
|
-
void * ptr = ggml_cuda_cpy_fn(node->src[0], node->src[1]);
|
|
2557
|
-
if (std::find(ggml_cuda_cpy_fn_ptrs.begin(), ggml_cuda_cpy_fn_ptrs.end(), ptr) == ggml_cuda_cpy_fn_ptrs.end()) {
|
|
2558
|
-
ggml_cuda_cpy_fn_ptrs.push_back(ptr);
|
|
2559
|
-
}
|
|
2560
|
-
}
|
|
2561
|
-
|
|
2562
|
-
if (!use_cuda_graph) {
|
|
2563
|
-
break;
|
|
2564
|
-
}
|
|
2565
|
-
}
|
|
2566
|
-
|
|
2567
|
-
// Disable CUDA graphs (from the next token) if the use-case is demanding too many consecutive graph updates.
|
|
2568
|
-
if (use_cuda_graph && cuda_graph_update_required) {
|
|
2569
|
-
cuda_ctx->cuda_graph->number_consecutive_updates++;
|
|
2570
|
-
} else {
|
|
2571
|
-
cuda_ctx->cuda_graph->number_consecutive_updates = 0;
|
|
2572
|
-
}
|
|
2573
|
-
|
|
2574
|
-
if (cuda_ctx->cuda_graph->number_consecutive_updates >= 4) {
|
|
2575
|
-
cuda_ctx->cuda_graph->disable_due_to_too_many_updates = true;
|
|
2576
|
-
#ifndef NDEBUG
|
|
2577
|
-
GGML_CUDA_LOG_WARN("%s: disabling CUDA graphs due to too many consecutive updates\n", __func__);
|
|
2578
|
-
#endif
|
|
2579
|
-
}
|
|
2580
|
-
}
|
|
2581
|
-
|
|
2582
|
-
if (use_cuda_graph && cuda_graph_update_required) { // Start CUDA graph capture
|
|
2583
|
-
CUDA_CHECK(cudaStreamBeginCapture(cuda_ctx->stream(), cudaStreamCaptureModeRelaxed));
|
|
2584
|
-
}
|
|
2585
|
-
|
|
2586
|
-
#else
|
|
2587
|
-
bool use_cuda_graph = false;
|
|
2588
|
-
bool cuda_graph_update_required = false;
|
|
2589
|
-
#endif // USE_CUDA_GRAPH
|
|
2590
|
-
|
|
2591
|
-
bool graph_evaluated_or_captured = false;
|
|
2592
|
-
|
|
2593
|
-
while (!graph_evaluated_or_captured) {
|
|
2594
|
-
// Only perform the graph execution if CUDA graphs are not enabled, or we are capturing the graph.
|
|
2595
|
-
// With the use of CUDA graphs, the execution will be performed by the graph launch.
|
|
2596
|
-
if (!use_cuda_graph || cuda_graph_update_required) {
|
|
2597
|
-
for (int i = 0; i < cgraph->n_nodes; i++) {
|
|
2598
|
-
ggml_tensor * node = cgraph->nodes[i];
|
|
2599
|
-
|
|
2600
|
-
if (ggml_is_empty(node) || node->op == GGML_OP_RESHAPE || node->op == GGML_OP_TRANSPOSE || node->op == GGML_OP_VIEW || node->op == GGML_OP_PERMUTE || node->op == GGML_OP_NONE) {
|
|
2601
|
-
continue;
|
|
2602
|
-
}
|
|
2603
|
-
|
|
2604
|
-
#ifndef NDEBUG
|
|
2605
|
-
assert(node->buffer->buft == ggml_backend_cuda_buffer_type(cuda_ctx->device));
|
|
2606
|
-
for (int j = 0; j < GGML_MAX_SRC; j++) {
|
|
2607
|
-
if (node->src[j] != nullptr) {
|
|
2608
|
-
assert(node->src[j]->buffer->buft == ggml_backend_cuda_buffer_type(cuda_ctx->device) || ggml_backend_buffer_is_cuda_split(node->src[j]->buffer));
|
|
2609
|
-
}
|
|
2610
|
-
}
|
|
2611
|
-
#endif
|
|
2612
|
-
|
|
2613
|
-
bool ok = ggml_cuda_compute_forward(*cuda_ctx, node);
|
|
2614
|
-
if (!ok) {
|
|
2615
|
-
GGML_CUDA_LOG_ERROR("%s: op not supported %s (%s)\n", __func__, node->name, ggml_op_name(node->op));
|
|
2616
|
-
}
|
|
2617
|
-
GGML_ASSERT(ok);
|
|
2618
|
-
}
|
|
2619
|
-
}
|
|
2620
|
-
|
|
2621
|
-
#ifdef USE_CUDA_GRAPH
|
|
2622
|
-
if (use_cuda_graph && cuda_graph_update_required) { // End CUDA graph capture
|
|
2623
|
-
if (cuda_ctx->cuda_graph->graph != nullptr) {
|
|
2624
|
-
CUDA_CHECK(cudaGraphDestroy(cuda_ctx->cuda_graph->graph));
|
|
2625
|
-
cuda_ctx->cuda_graph->graph = nullptr;
|
|
2626
|
-
}
|
|
2627
|
-
CUDA_CHECK(cudaStreamEndCapture(cuda_ctx->stream(), &cuda_ctx->cuda_graph->graph));
|
|
2628
|
-
|
|
2629
|
-
#if 0
|
|
2630
|
-
if (disable_cuda_graphs_due_to_failed_capture) {
|
|
2631
|
-
use_cuda_graph = false;
|
|
2632
|
-
cuda_ctx->cuda_graph->disable_due_to_failed_graph_capture = true;
|
|
2633
|
-
#ifndef NDEBUG
|
|
2634
|
-
GGML_CUDA_LOG_WARN("%s: disabling CUDA graphs due to failed graph capture\n", __func__);
|
|
2635
|
-
#endif
|
|
2636
|
-
} else {
|
|
2637
|
-
graph_evaluated_or_captured = true; // CUDA graph has been captured
|
|
2638
|
-
}
|
|
2639
|
-
#endif
|
|
2640
|
-
graph_evaluated_or_captured = true; // CUDA graph has been captured
|
|
2641
|
-
} else {
|
|
2642
|
-
graph_evaluated_or_captured = true; // ggml graph has been directly evaluated
|
|
2643
|
-
}
|
|
2644
|
-
}
|
|
2645
|
-
|
|
2646
|
-
if (use_cuda_graph) {
|
|
2647
|
-
if (cuda_ctx->cuda_graph->instance == nullptr) { // Create executable graph from captured graph.
|
|
2648
|
-
CUDA_CHECK(cudaGraphInstantiate(&cuda_ctx->cuda_graph->instance, cuda_ctx->cuda_graph->graph, NULL, NULL, 0));
|
|
2649
|
-
}
|
|
2650
|
-
|
|
2651
|
-
// Perform update to graph (if required for this token), and change copy parameter (required for every token)
|
|
2652
|
-
|
|
2653
|
-
if (cuda_graph_update_required) {
|
|
2654
|
-
// Extract nodes from graph
|
|
2655
|
-
// First call with null argument gets number of nodes in graph
|
|
2656
|
-
CUDA_CHECK(cudaGraphGetNodes(cuda_ctx->cuda_graph->graph, nullptr, &cuda_ctx->cuda_graph->num_nodes));
|
|
2657
|
-
// Subsequent call with non-null argument gets nodes
|
|
2658
|
-
cuda_ctx->cuda_graph->nodes.resize(cuda_ctx->cuda_graph->num_nodes);
|
|
2659
|
-
cuda_ctx->cuda_graph->params.resize(cuda_ctx->cuda_graph->num_nodes);
|
|
2660
|
-
if (cuda_ctx->cuda_graph->num_nodes > 0) {
|
|
2661
|
-
CUDA_CHECK(cudaGraphGetNodes(cuda_ctx->cuda_graph->graph, cuda_ctx->cuda_graph->nodes.data(), &cuda_ctx->cuda_graph->num_nodes));
|
|
2662
|
-
|
|
2663
|
-
// Loop over nodes, and extract kernel parameters from each node
|
|
2664
|
-
for (size_t i = 0; i < cuda_ctx->cuda_graph->num_nodes; i++) {
|
|
2665
|
-
cudaGraphNodeType node_type;
|
|
2666
|
-
CUDA_CHECK(cudaGraphNodeGetType(cuda_ctx->cuda_graph->nodes[i], &node_type));
|
|
2667
|
-
if (node_type == cudaGraphNodeTypeKernel) {
|
|
2668
|
-
cudaError_t stat = cudaGraphKernelNodeGetParams(cuda_ctx->cuda_graph->nodes[i], &cuda_ctx->cuda_graph->params[i]); // Get params using runtime
|
|
2669
|
-
if (stat == cudaErrorInvalidDeviceFunction) {
|
|
2670
|
-
// Fails due to incorrect handling by CUDA runtime of CUDA BLAS node.
|
|
2671
|
-
// We don't need to update blas nodes, so clear error and move on.
|
|
2672
|
-
cudaGetLastError();
|
|
2673
|
-
} else {
|
|
2674
|
-
GGML_ASSERT(stat == cudaSuccess);
|
|
2675
|
-
}
|
|
2676
|
-
}
|
|
2677
|
-
}
|
|
2678
|
-
}
|
|
2679
|
-
}
|
|
2680
|
-
|
|
2681
|
-
// One of the arguments to the copy kernel is updated for each token, hence we need to
|
|
2682
|
-
// replace that argument with the updated value in the CUDA graph
|
|
2683
|
-
if (!cuda_graph_update_required) { // on update steps, the live parameters will already be captured
|
|
2684
|
-
int k = 0;
|
|
2685
|
-
for (size_t i = 0; i < cuda_ctx->cuda_graph->num_nodes; i++) {
|
|
2686
|
-
if(count(ggml_cuda_cpy_fn_ptrs.begin(), ggml_cuda_cpy_fn_ptrs.end(), cuda_ctx->cuda_graph->params[i].func) > 0) {
|
|
2687
|
-
char ** updated_kernel_arg_ptr = cuda_ctx->cuda_graph->updated_kernel_arg.at(k++);
|
|
2688
|
-
cuda_ctx->cuda_graph->params[i].kernelParams[1] = updated_kernel_arg_ptr;
|
|
2689
|
-
CUDA_CHECK(cudaGraphKernelNodeSetParams(cuda_ctx->cuda_graph->nodes[i], &cuda_ctx->cuda_graph->params[i]));
|
|
2690
|
-
}
|
|
2691
|
-
}
|
|
2692
|
-
}
|
|
2693
|
-
|
|
2694
|
-
// Update graph executable
|
|
2695
|
-
cudaGraphExecUpdateResultInfo result_info;
|
|
2696
|
-
cudaError_t stat = cudaGraphExecUpdate(cuda_ctx->cuda_graph->instance, cuda_ctx->cuda_graph->graph, &result_info);
|
|
2697
|
-
if (stat == cudaErrorGraphExecUpdateFailure) {
|
|
2698
|
-
#ifndef NDEBUG
|
|
2699
|
-
GGML_CUDA_LOG_ERROR("%s: CUDA graph update failed\n", __func__);
|
|
2700
|
-
#endif
|
|
2701
|
-
// The pre-existing graph exec cannot be updated due to violated constraints
|
|
2702
|
-
// so instead clear error and re-instantiate
|
|
2703
|
-
cudaGetLastError();
|
|
2704
|
-
CUDA_CHECK(cudaGraphExecDestroy(cuda_ctx->cuda_graph->instance));
|
|
2705
|
-
cuda_ctx->cuda_graph->instance = nullptr;
|
|
2706
|
-
CUDA_CHECK(cudaGraphInstantiate(&cuda_ctx->cuda_graph->instance, cuda_ctx->cuda_graph->graph, NULL, NULL, 0));
|
|
2707
|
-
} else {
|
|
2708
|
-
GGML_ASSERT(stat == cudaSuccess);
|
|
2709
|
-
}
|
|
2710
|
-
// Launch graph
|
|
2711
|
-
CUDA_CHECK(cudaGraphLaunch(cuda_ctx->cuda_graph->instance, cuda_ctx->stream()));
|
|
2712
|
-
#else
|
|
2713
|
-
graph_evaluated_or_captured = true;
|
|
2714
|
-
#endif // USE_CUDA_GRAPH
|
|
2715
|
-
}
|
|
2716
|
-
|
|
2717
|
-
return GGML_STATUS_SUCCESS;
|
|
2718
|
-
}
|
|
2719
|
-
|
|
2720
|
-
GGML_CALL static bool ggml_backend_cuda_supports_op(ggml_backend_t backend, const ggml_tensor * op) {
|
|
2721
|
-
ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *) backend->context;
|
|
2722
|
-
switch (op->op) {
|
|
2723
|
-
case GGML_OP_UNARY:
|
|
2724
|
-
switch (ggml_get_unary_op(op)) {
|
|
2725
|
-
case GGML_UNARY_OP_GELU:
|
|
2726
|
-
case GGML_UNARY_OP_SILU:
|
|
2727
|
-
case GGML_UNARY_OP_RELU:
|
|
2728
|
-
case GGML_UNARY_OP_SIGMOID:
|
|
2729
|
-
case GGML_UNARY_OP_HARDSIGMOID:
|
|
2730
|
-
case GGML_UNARY_OP_HARDSWISH:
|
|
2731
|
-
case GGML_UNARY_OP_GELU_QUICK:
|
|
2732
|
-
case GGML_UNARY_OP_TANH:
|
|
2733
|
-
return ggml_is_contiguous(op->src[0]);
|
|
2734
|
-
default:
|
|
2735
|
-
return false;
|
|
2736
|
-
}
|
|
2737
|
-
break;
|
|
2738
|
-
case GGML_OP_MUL_MAT:
|
|
2739
|
-
case GGML_OP_MUL_MAT_ID:
|
|
2740
|
-
{
|
|
2741
|
-
struct ggml_tensor * a;
|
|
2742
|
-
struct ggml_tensor * b;
|
|
2743
|
-
if (op->op == GGML_OP_MUL_MAT) {
|
|
2744
|
-
a = op->src[0];
|
|
2745
|
-
b = op->src[1];
|
|
2746
|
-
} else {
|
|
2747
|
-
a = op->src[2];
|
|
2748
|
-
b = op->src[1];
|
|
2749
|
-
}
|
|
2750
|
-
if (a->ne[3] != b->ne[3]) {
|
|
2751
|
-
return false;
|
|
2752
|
-
}
|
|
2753
|
-
ggml_type a_type = a->type;
|
|
2754
|
-
if (a_type == GGML_TYPE_IQ2_XXS || a_type == GGML_TYPE_IQ2_XS || a_type == GGML_TYPE_IQ3_XXS ||
|
|
2755
|
-
a_type == GGML_TYPE_IQ1_S || a_type == GGML_TYPE_IQ4_NL || a_type == GGML_TYPE_IQ3_S ||
|
|
2756
|
-
a_type == GGML_TYPE_IQ1_M || a_type == GGML_TYPE_IQ2_S || a_type == GGML_TYPE_IQ4_XS) {
|
|
2757
|
-
if (b->ne[1] == 1 && ggml_nrows(b) > 1) {
|
|
2758
|
-
return false;
|
|
2759
|
-
}
|
|
2760
|
-
}
|
|
2761
|
-
return true;
|
|
2762
|
-
} break;
|
|
2763
|
-
case GGML_OP_GET_ROWS:
|
|
2764
|
-
{
|
|
2765
|
-
switch (op->src[0]->type) {
|
|
2766
|
-
case GGML_TYPE_F16:
|
|
2767
|
-
case GGML_TYPE_F32:
|
|
2768
|
-
case GGML_TYPE_Q4_0:
|
|
2769
|
-
case GGML_TYPE_Q4_1:
|
|
2770
|
-
case GGML_TYPE_Q5_0:
|
|
2771
|
-
case GGML_TYPE_Q5_1:
|
|
2772
|
-
case GGML_TYPE_Q8_0:
|
|
2773
|
-
return true;
|
|
2774
|
-
default:
|
|
2775
|
-
return false;
|
|
2776
|
-
}
|
|
2777
|
-
} break;
|
|
2778
|
-
case GGML_OP_CPY:
|
|
2779
|
-
{
|
|
2780
|
-
ggml_type src0_type = op->src[0]->type;
|
|
2781
|
-
ggml_type src1_type = op->src[1]->type;
|
|
2782
|
-
if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_F32) {
|
|
2783
|
-
return true;
|
|
2784
|
-
}
|
|
2785
|
-
if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_F16) {
|
|
2786
|
-
return true;
|
|
2787
|
-
}
|
|
2788
|
-
if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_Q8_0) {
|
|
2789
|
-
return true;
|
|
2790
|
-
}
|
|
2791
|
-
if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_Q4_0) {
|
|
2792
|
-
return true;
|
|
2793
|
-
}
|
|
2794
|
-
if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_Q4_1) {
|
|
2795
|
-
return true;
|
|
2796
|
-
}
|
|
2797
|
-
if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_Q5_0) {
|
|
2798
|
-
return true;
|
|
2799
|
-
}
|
|
2800
|
-
if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_Q5_1) {
|
|
2801
|
-
return true;
|
|
2802
|
-
}
|
|
2803
|
-
if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_IQ4_NL) {
|
|
2804
|
-
return true;
|
|
2805
|
-
}
|
|
2806
|
-
if (src0_type == GGML_TYPE_F16 && src1_type == GGML_TYPE_F16) {
|
|
2807
|
-
return true;
|
|
2808
|
-
}
|
|
2809
|
-
if (src0_type == GGML_TYPE_F16 && src1_type == GGML_TYPE_F32) {
|
|
2810
|
-
return true;
|
|
2811
|
-
}
|
|
2812
|
-
return false;
|
|
2813
|
-
} break;
|
|
2814
|
-
case GGML_OP_DUP:
|
|
2815
|
-
case GGML_OP_REPEAT:
|
|
2816
|
-
case GGML_OP_CONCAT:
|
|
2817
|
-
{
|
|
2818
|
-
ggml_type src0_type = op->src[0]->type;
|
|
2819
|
-
return src0_type != GGML_TYPE_I32 && src0_type != GGML_TYPE_I16;
|
|
2820
|
-
} break;
|
|
2821
|
-
case GGML_OP_NONE:
|
|
2822
|
-
case GGML_OP_RESHAPE:
|
|
2823
|
-
case GGML_OP_VIEW:
|
|
2824
|
-
case GGML_OP_PERMUTE:
|
|
2825
|
-
case GGML_OP_TRANSPOSE:
|
|
2826
|
-
case GGML_OP_NORM:
|
|
2827
|
-
case GGML_OP_ADD:
|
|
2828
|
-
case GGML_OP_MUL:
|
|
2829
|
-
case GGML_OP_DIV:
|
|
2830
|
-
case GGML_OP_RMS_NORM:
|
|
2831
|
-
case GGML_OP_SCALE:
|
|
2832
|
-
case GGML_OP_SQR:
|
|
2833
|
-
case GGML_OP_CLAMP:
|
|
2834
|
-
case GGML_OP_CONT:
|
|
2835
|
-
case GGML_OP_DIAG_MASK_INF:
|
|
2836
|
-
case GGML_OP_SOFT_MAX:
|
|
2837
|
-
return true;
|
|
2838
|
-
case GGML_OP_ROPE:
|
|
2839
|
-
return ggml_is_contiguous(op->src[0]);
|
|
2840
|
-
case GGML_OP_IM2COL:
|
|
2841
|
-
case GGML_OP_POOL_2D:
|
|
2842
|
-
case GGML_OP_SUM_ROWS:
|
|
2843
|
-
case GGML_OP_ARGSORT:
|
|
2844
|
-
case GGML_OP_ACC:
|
|
2845
|
-
case GGML_OP_GROUP_NORM:
|
|
2846
|
-
case GGML_OP_UPSCALE:
|
|
2847
|
-
case GGML_OP_PAD:
|
|
2848
|
-
case GGML_OP_ARANGE:
|
|
2849
|
-
case GGML_OP_TIMESTEP_EMBEDDING:
|
|
2850
|
-
case GGML_OP_LEAKY_RELU:
|
|
2851
|
-
return true;
|
|
2852
|
-
case GGML_OP_FLASH_ATTN_EXT:
|
|
2853
|
-
#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)
|
|
2854
|
-
return op->src[0]->ne[0] == 64 || op->src[0]->ne[0] == 128;
|
|
2855
|
-
#else
|
|
2856
|
-
if (op->src[0]->ne[0] == 128) {
|
|
2857
|
-
return true;
|
|
2858
|
-
}
|
|
2859
|
-
if (op->src[0]->ne[0] == 64 && op->src[1]->type == GGML_TYPE_F16) {
|
|
2860
|
-
return true;
|
|
2861
|
-
}
|
|
2862
|
-
return ggml_cuda_info().devices[cuda_ctx->device].cc >= CC_VOLTA &&
|
|
2863
|
-
op->src[1]->type == GGML_TYPE_F16 && op->src[2]->type == GGML_TYPE_F16;
|
|
2864
|
-
#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)
|
|
2865
|
-
default:
|
|
2866
|
-
return false;
|
|
2867
|
-
}
|
|
2868
|
-
|
|
2869
|
-
GGML_UNUSED(backend);
|
|
2870
|
-
}
|
|
2871
|
-
|
|
2872
|
-
GGML_CALL static bool ggml_backend_cuda_supports_buft(ggml_backend_t backend, ggml_backend_buffer_type_t buft) {
|
|
2873
|
-
if (ggml_backend_buft_is_cuda_split(buft)) {
|
|
2874
|
-
return true;
|
|
2875
|
-
}
|
|
2876
|
-
|
|
2877
|
-
if (ggml_backend_buft_is_cuda(buft)) {
|
|
2878
|
-
ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context;
|
|
2879
|
-
ggml_backend_cuda_buffer_type_context * buft_ctx = (ggml_backend_cuda_buffer_type_context *)buft->context;
|
|
2880
|
-
return buft_ctx->device == cuda_ctx->device;
|
|
2881
|
-
}
|
|
2882
|
-
|
|
2883
|
-
return false;
|
|
2884
|
-
}
|
|
2885
|
-
|
|
2886
|
-
GGML_CALL static bool ggml_backend_cuda_offload_op(ggml_backend_t backend, const ggml_tensor * op) {
|
|
2887
|
-
const int min_batch_size = 32;
|
|
2888
|
-
|
|
2889
|
-
return (op->ne[1] >= min_batch_size && op->op != GGML_OP_GET_ROWS) ||
|
|
2890
|
-
(op->ne[2] >= min_batch_size && op->op == GGML_OP_MUL_MAT_ID);
|
|
2891
|
-
|
|
2892
|
-
GGML_UNUSED(backend);
|
|
2893
|
-
}
|
|
2894
|
-
|
|
2895
|
-
static ggml_backend_event_t ggml_backend_cuda_event_new(ggml_backend_t backend) {
|
|
2896
|
-
#ifdef GGML_CUDA_NO_PEER_COPY
|
|
2897
|
-
return nullptr;
|
|
2898
|
-
#else
|
|
2899
|
-
ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context;
|
|
2900
|
-
|
|
2901
|
-
ggml_cuda_set_device(cuda_ctx->device);
|
|
2902
|
-
|
|
2903
|
-
cudaEvent_t event;
|
|
2904
|
-
CUDA_CHECK(cudaEventCreateWithFlags(&event, cudaEventDisableTiming));
|
|
2905
|
-
|
|
2906
|
-
return new ggml_backend_event {
|
|
2907
|
-
/* .backend = */ backend,
|
|
2908
|
-
/* .context = */ event,
|
|
2909
|
-
};
|
|
2910
|
-
#endif
|
|
2911
|
-
}
|
|
2912
|
-
|
|
2913
|
-
static void ggml_backend_cuda_event_free(ggml_backend_event_t event) {
|
|
2914
|
-
CUDA_CHECK(cudaEventDestroy((cudaEvent_t)event->context));
|
|
2915
|
-
|
|
2916
|
-
delete event;
|
|
2917
|
-
}
|
|
2918
|
-
|
|
2919
|
-
static void ggml_backend_cuda_event_record(ggml_backend_event_t event) {
|
|
2920
|
-
ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)event->backend->context;
|
|
2921
|
-
|
|
2922
|
-
CUDA_CHECK(cudaEventRecord((cudaEvent_t)event->context, cuda_ctx->stream()));
|
|
2923
|
-
}
|
|
2924
|
-
|
|
2925
|
-
static void ggml_backend_cuda_event_wait(ggml_backend_t backend, ggml_backend_event_t event) {
|
|
2926
|
-
ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context;
|
|
2927
|
-
|
|
2928
|
-
if (ggml_backend_is_cuda(event->backend)) {
|
|
2929
|
-
CUDA_CHECK(cudaStreamWaitEvent(cuda_ctx->stream(), (cudaEvent_t)event->context, 0));
|
|
2930
|
-
} else {
|
|
2931
|
-
#if 0
|
|
2932
|
-
// untested
|
|
2933
|
-
auto wait_fn = [](void * user_data) {
|
|
2934
|
-
ggml_backend_event_t event = (ggml_backend_event_t)user_data;
|
|
2935
|
-
ggml_backend_event_synchronize(event);
|
|
2936
|
-
};
|
|
2937
|
-
|
|
2938
|
-
CUDA_CHECK(cudaLaunchHostFunc(cuda_ctx->stream(), wait_fn, event));
|
|
2939
|
-
#endif
|
|
2940
|
-
GGML_ASSERT(false);
|
|
2941
|
-
}
|
|
2942
|
-
}
|
|
2943
|
-
|
|
2944
|
-
static void ggml_backend_cuda_event_synchronize(ggml_backend_event_t event) {
|
|
2945
|
-
CUDA_CHECK(cudaEventSynchronize((cudaEvent_t)event->context));
|
|
2946
|
-
}
|
|
2947
|
-
|
|
2948
|
-
static ggml_backend_i ggml_backend_cuda_interface = {
|
|
2949
|
-
/* .get_name = */ ggml_backend_cuda_name,
|
|
2950
|
-
/* .free = */ ggml_backend_cuda_free,
|
|
2951
|
-
/* .get_default_buffer_type = */ ggml_backend_cuda_get_default_buffer_type,
|
|
2952
|
-
/* .set_tensor_async = */ ggml_backend_cuda_set_tensor_async,
|
|
2953
|
-
/* .get_tensor_async = */ ggml_backend_cuda_get_tensor_async,
|
|
2954
|
-
/* .cpy_tensor_async = */ ggml_backend_cuda_cpy_tensor_async,
|
|
2955
|
-
/* .synchronize = */ ggml_backend_cuda_synchronize,
|
|
2956
|
-
/* .graph_plan_create = */ NULL,
|
|
2957
|
-
/* .graph_plan_free = */ NULL,
|
|
2958
|
-
/* .graph_plan_update = */ NULL,
|
|
2959
|
-
/* .graph_plan_compute = */ NULL,
|
|
2960
|
-
/* .graph_compute = */ ggml_backend_cuda_graph_compute,
|
|
2961
|
-
/* .supports_op = */ ggml_backend_cuda_supports_op,
|
|
2962
|
-
/* .supports_buft = */ ggml_backend_cuda_supports_buft,
|
|
2963
|
-
/* .offload_op = */ ggml_backend_cuda_offload_op,
|
|
2964
|
-
/* .event_new = */ ggml_backend_cuda_event_new,
|
|
2965
|
-
/* .event_free = */ ggml_backend_cuda_event_free,
|
|
2966
|
-
/* .event_record = */ ggml_backend_cuda_event_record,
|
|
2967
|
-
/* .event_wait = */ ggml_backend_cuda_event_wait,
|
|
2968
|
-
/* .event_synchronize = */ ggml_backend_cuda_event_synchronize,
|
|
2969
|
-
};
|
|
2970
|
-
|
|
2971
|
-
static ggml_guid_t ggml_backend_cuda_guid() {
|
|
2972
|
-
static ggml_guid guid = { 0x2c, 0xdd, 0xe8, 0x1c, 0x65, 0xb3, 0x65, 0x73, 0x6a, 0x12, 0x88, 0x61, 0x1c, 0xc9, 0xdc, 0x25 };
|
|
2973
|
-
return &guid;
|
|
2974
|
-
}
|
|
2975
|
-
|
|
2976
|
-
GGML_CALL ggml_backend_t ggml_backend_cuda_init(int device) {
|
|
2977
|
-
if (device < 0 || device >= ggml_backend_cuda_get_device_count()) {
|
|
2978
|
-
GGML_CUDA_LOG_ERROR("%s: invalid device %d\n", __func__, device);
|
|
2979
|
-
return nullptr;
|
|
2980
|
-
}
|
|
2981
|
-
|
|
2982
|
-
ggml_backend_cuda_context * ctx = new ggml_backend_cuda_context(device);
|
|
2983
|
-
if (ctx == nullptr) {
|
|
2984
|
-
GGML_CUDA_LOG_ERROR("%s: failed to allocate context\n", __func__);
|
|
2985
|
-
return nullptr;
|
|
2986
|
-
}
|
|
2987
|
-
|
|
2988
|
-
ggml_backend_t cuda_backend = new ggml_backend {
|
|
2989
|
-
/* .guid = */ ggml_backend_cuda_guid(),
|
|
2990
|
-
/* .interface = */ ggml_backend_cuda_interface,
|
|
2991
|
-
/* .context = */ ctx
|
|
2992
|
-
};
|
|
2993
|
-
|
|
2994
|
-
return cuda_backend;
|
|
2995
|
-
}
|
|
2996
|
-
|
|
2997
|
-
GGML_CALL bool ggml_backend_is_cuda(ggml_backend_t backend) {
|
|
2998
|
-
return backend != NULL && ggml_guid_matches(backend->guid, ggml_backend_cuda_guid());
|
|
2999
|
-
}
|
|
3000
|
-
|
|
3001
|
-
GGML_CALL int ggml_backend_cuda_get_device_count() {
|
|
3002
|
-
return ggml_cuda_info().device_count;
|
|
3003
|
-
}
|
|
3004
|
-
|
|
3005
|
-
GGML_CALL void ggml_backend_cuda_get_device_description(int device, char * description, size_t description_size) {
|
|
3006
|
-
cudaDeviceProp prop;
|
|
3007
|
-
CUDA_CHECK(cudaGetDeviceProperties(&prop, device));
|
|
3008
|
-
snprintf(description, description_size, "%s", prop.name);
|
|
3009
|
-
}
|
|
3010
|
-
|
|
3011
|
-
GGML_CALL void ggml_backend_cuda_get_device_memory(int device, size_t * free, size_t * total) {
|
|
3012
|
-
ggml_cuda_set_device(device);
|
|
3013
|
-
|
|
3014
|
-
CUDA_CHECK(cudaMemGetInfo(free, total));
|
|
3015
|
-
}
|
|
3016
|
-
|
|
3017
|
-
GGML_CALL bool ggml_backend_cuda_register_host_buffer(void * buffer, size_t size) {
|
|
3018
|
-
if (getenv("GGML_CUDA_REGISTER_HOST") == nullptr) {
|
|
3019
|
-
return false;
|
|
3020
|
-
}
|
|
3021
|
-
|
|
3022
|
-
#if CUDART_VERSION >= 11100
|
|
3023
|
-
cudaError_t err = cudaHostRegister(buffer, size, cudaHostRegisterPortable | cudaHostRegisterReadOnly);
|
|
3024
|
-
if (err != cudaSuccess) {
|
|
3025
|
-
// clear the error
|
|
3026
|
-
cudaGetLastError();
|
|
3027
|
-
|
|
3028
|
-
GGML_CUDA_LOG_WARN("%s: failed to register %.2f MiB of pinned memory: %s\n", __func__,
|
|
3029
|
-
size / 1024.0 / 1024.0, cudaGetErrorString(err));
|
|
3030
|
-
return false;
|
|
3031
|
-
}
|
|
3032
|
-
return true;
|
|
3033
|
-
#else
|
|
3034
|
-
return false;
|
|
3035
|
-
#endif
|
|
3036
|
-
}
|
|
3037
|
-
|
|
3038
|
-
GGML_CALL void ggml_backend_cuda_unregister_host_buffer(void * buffer) {
|
|
3039
|
-
if (getenv("GGML_CUDA_REGISTER_HOST") == nullptr) {
|
|
3040
|
-
return;
|
|
3041
|
-
}
|
|
3042
|
-
|
|
3043
|
-
cudaError_t err = cudaHostUnregister(buffer);
|
|
3044
|
-
if (err != cudaSuccess) {
|
|
3045
|
-
// clear the error
|
|
3046
|
-
cudaGetLastError();
|
|
3047
|
-
}
|
|
3048
|
-
}
|
|
3049
|
-
|
|
3050
|
-
// backend registry
|
|
3051
|
-
GGML_CALL static ggml_backend_t ggml_backend_reg_cuda_init(const char * params, void * user_data) {
|
|
3052
|
-
ggml_backend_t cuda_backend = ggml_backend_cuda_init((int) (intptr_t) user_data);
|
|
3053
|
-
return cuda_backend;
|
|
3054
|
-
|
|
3055
|
-
GGML_UNUSED(params);
|
|
3056
|
-
}
|
|
3057
|
-
|
|
3058
|
-
extern "C" GGML_CALL int ggml_backend_cuda_reg_devices();
|
|
3059
|
-
|
|
3060
|
-
GGML_CALL int ggml_backend_cuda_reg_devices() {
|
|
3061
|
-
int device_count = ggml_backend_cuda_get_device_count();
|
|
3062
|
-
//int device_count = 1; // DEBUG: some tools require delaying CUDA initialization
|
|
3063
|
-
for (int i = 0; i < device_count; i++) {
|
|
3064
|
-
char name[128];
|
|
3065
|
-
snprintf(name, sizeof(name), "%s%d", GGML_CUDA_NAME, i);
|
|
3066
|
-
ggml_backend_register(name, ggml_backend_reg_cuda_init, ggml_backend_cuda_buffer_type(i), (void *) (intptr_t) i);
|
|
3067
|
-
}
|
|
3068
|
-
return device_count;
|
|
3069
|
-
}
|