llama_cpp 0.16.2 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +18 -0
- data/README.md +7 -12
- data/ext/llama_cpp/extconf.rb +2 -43
- data/ext/llama_cpp/llama_cpp.cpp +8 -0
- data/lib/llama_cpp/version.rb +3 -3
- data/sig/llama_cpp.rbs +3 -0
- metadata +2 -171
- data/vendor/include/.gitkeep +0 -0
- data/vendor/lib/.gitkeep +0 -0
- data/vendor/tmp/llama.cpp/LICENSE +0 -21
- data/vendor/tmp/llama.cpp/Makefile +0 -1124
- data/vendor/tmp/llama.cpp/ggml-alloc.c +0 -1041
- data/vendor/tmp/llama.cpp/ggml-alloc.h +0 -76
- data/vendor/tmp/llama.cpp/ggml-backend-impl.h +0 -153
- data/vendor/tmp/llama.cpp/ggml-backend.c +0 -2225
- data/vendor/tmp/llama.cpp/ggml-backend.h +0 -236
- data/vendor/tmp/llama.cpp/ggml-blas.cpp +0 -363
- data/vendor/tmp/llama.cpp/ggml-blas.h +0 -23
- data/vendor/tmp/llama.cpp/ggml-common.h +0 -1805
- data/vendor/tmp/llama.cpp/ggml-cuda/acc.cu +0 -47
- data/vendor/tmp/llama.cpp/ggml-cuda/arange.cu +0 -34
- data/vendor/tmp/llama.cpp/ggml-cuda/argsort.cu +0 -104
- data/vendor/tmp/llama.cpp/ggml-cuda/binbcast.cu +0 -280
- data/vendor/tmp/llama.cpp/ggml-cuda/clamp.cu +0 -34
- data/vendor/tmp/llama.cpp/ggml-cuda/concat.cu +0 -196
- data/vendor/tmp/llama.cpp/ggml-cuda/convert.cu +0 -686
- data/vendor/tmp/llama.cpp/ggml-cuda/cpy.cu +0 -490
- data/vendor/tmp/llama.cpp/ggml-cuda/diagmask.cu +0 -40
- data/vendor/tmp/llama.cpp/ggml-cuda/dmmv.cu +0 -674
- data/vendor/tmp/llama.cpp/ggml-cuda/fattn-tile-f16.cu +0 -319
- data/vendor/tmp/llama.cpp/ggml-cuda/fattn-tile-f32.cu +0 -312
- data/vendor/tmp/llama.cpp/ggml-cuda/fattn.cu +0 -345
- data/vendor/tmp/llama.cpp/ggml-cuda/getrows.cu +0 -178
- data/vendor/tmp/llama.cpp/ggml-cuda/im2col.cu +0 -104
- data/vendor/tmp/llama.cpp/ggml-cuda/mmq.cu +0 -88
- data/vendor/tmp/llama.cpp/ggml-cuda/mmvq.cu +0 -419
- data/vendor/tmp/llama.cpp/ggml-cuda/norm.cu +0 -221
- data/vendor/tmp/llama.cpp/ggml-cuda/pad.cu +0 -49
- data/vendor/tmp/llama.cpp/ggml-cuda/pool2d.cu +0 -94
- data/vendor/tmp/llama.cpp/ggml-cuda/quantize.cu +0 -112
- data/vendor/tmp/llama.cpp/ggml-cuda/rope.cu +0 -271
- data/vendor/tmp/llama.cpp/ggml-cuda/scale.cu +0 -31
- data/vendor/tmp/llama.cpp/ggml-cuda/softmax.cu +0 -206
- data/vendor/tmp/llama.cpp/ggml-cuda/sumrows.cu +0 -40
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-f16.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q4_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q4_1.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q5_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q5_1.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q8_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-f16.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q4_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q4_1.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q5_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q5_1.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q8_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-f16.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q4_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q4_1.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q5_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q5_1.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q8_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-f16.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q4_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q4_1.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q5_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q5_1.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q8_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-f16.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q4_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q4_1.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q5_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q5_1.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q8_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-f16.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q4_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q4_1.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q5_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q5_1.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q8_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs256-f16-f16.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-f16.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q4_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q4_1.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q5_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q5_1.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q8_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-f16.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q4_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q4_1.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q5_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q5_1.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q8_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-f16.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q4_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q4_1.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q5_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q5_1.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q8_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-f16.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q4_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q4_1.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q5_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q5_1.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q8_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-f16.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q4_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q4_1.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q5_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q5_1.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q8_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-f16.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q4_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q4_1.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q5_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q5_1.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q8_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-f16.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q4_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q4_1.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q5_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q5_1.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q8_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs256-f16-f16.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-f16.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q4_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q4_1.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q5_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q5_1.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q8_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqfloat-cpb16.cu +0 -10
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqfloat-cpb32.cu +0 -9
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqhalf-cpb16.cu +0 -10
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqhalf-cpb32.cu +0 -10
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqhalf-cpb8.cu +0 -8
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q2_k.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q3_k.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q4_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q4_1.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q4_k.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q5_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q5_1.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q5_k.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q6_k.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q8_0.cu +0 -5
- data/vendor/tmp/llama.cpp/ggml-cuda/tsembd.cu +0 -47
- data/vendor/tmp/llama.cpp/ggml-cuda/unary.cu +0 -314
- data/vendor/tmp/llama.cpp/ggml-cuda/upscale.cu +0 -51
- data/vendor/tmp/llama.cpp/ggml-cuda.cu +0 -3069
- data/vendor/tmp/llama.cpp/ggml-cuda.h +0 -44
- data/vendor/tmp/llama.cpp/ggml-impl.h +0 -651
- data/vendor/tmp/llama.cpp/ggml-kompute.cpp +0 -2038
- data/vendor/tmp/llama.cpp/ggml-kompute.h +0 -46
- data/vendor/tmp/llama.cpp/ggml-metal.h +0 -66
- data/vendor/tmp/llama.cpp/ggml-metal.m +0 -3273
- data/vendor/tmp/llama.cpp/ggml-metal.metal +0 -6540
- data/vendor/tmp/llama.cpp/ggml-quants.c +0 -14994
- data/vendor/tmp/llama.cpp/ggml-quants.h +0 -133
- data/vendor/tmp/llama.cpp/ggml-rpc.cpp +0 -1178
- data/vendor/tmp/llama.cpp/ggml-rpc.h +0 -24
- data/vendor/tmp/llama.cpp/ggml-sycl.cpp +0 -6351
- data/vendor/tmp/llama.cpp/ggml-sycl.h +0 -40
- data/vendor/tmp/llama.cpp/ggml-vulkan-shaders.hpp +0 -144508
- data/vendor/tmp/llama.cpp/ggml-vulkan.cpp +0 -7183
- data/vendor/tmp/llama.cpp/ggml-vulkan.h +0 -29
- data/vendor/tmp/llama.cpp/ggml.c +0 -22506
- data/vendor/tmp/llama.cpp/ggml.h +0 -2458
- data/vendor/tmp/llama.cpp/llama.cpp +0 -18985
- data/vendor/tmp/llama.cpp/llama.h +0 -1147
- data/vendor/tmp/llama.cpp/scripts/get-flags.mk +0 -38
- data/vendor/tmp/llama.cpp/sgemm.cpp +0 -1032
- data/vendor/tmp/llama.cpp/sgemm.h +0 -14
- data/vendor/tmp/llama.cpp/unicode-data.cpp +0 -7033
- data/vendor/tmp/llama.cpp/unicode-data.h +0 -20
- data/vendor/tmp/llama.cpp/unicode.cpp +0 -810
- data/vendor/tmp/llama.cpp/unicode.h +0 -63
|
@@ -1,2038 +0,0 @@
|
|
|
1
|
-
#include "ggml.h"
|
|
2
|
-
#include "ggml-backend.h"
|
|
3
|
-
#include "ggml-backend-impl.h"
|
|
4
|
-
#include "ggml-kompute.h"
|
|
5
|
-
|
|
6
|
-
// These are generated at build time by cmake custom command
|
|
7
|
-
#include "shaderop_scale.h"
|
|
8
|
-
#include "shaderop_scale_8.h"
|
|
9
|
-
#include "shaderop_add.h"
|
|
10
|
-
#include "shaderop_addrow.h"
|
|
11
|
-
#include "shaderop_mul.h"
|
|
12
|
-
#include "shaderop_silu.h"
|
|
13
|
-
#include "shaderop_relu.h"
|
|
14
|
-
#include "shaderop_gelu.h"
|
|
15
|
-
#include "shaderop_softmax.h"
|
|
16
|
-
#include "shaderop_norm.h"
|
|
17
|
-
#include "shaderop_rmsnorm.h"
|
|
18
|
-
#include "shaderop_diagmask.h"
|
|
19
|
-
#include "shaderop_mul_mat_f16.h"
|
|
20
|
-
#include "shaderop_mul_mat_q8_0.h"
|
|
21
|
-
#include "shaderop_mul_mat_q4_0.h"
|
|
22
|
-
#include "shaderop_mul_mat_q4_1.h"
|
|
23
|
-
#include "shaderop_mul_mat_q6_k.h"
|
|
24
|
-
#include "shaderop_mul_mat_mat_f32.h"
|
|
25
|
-
#include "shaderop_getrows_f32.h"
|
|
26
|
-
#include "shaderop_getrows_f16.h"
|
|
27
|
-
#include "shaderop_getrows_q4_0.h"
|
|
28
|
-
#include "shaderop_getrows_q4_1.h"
|
|
29
|
-
#include "shaderop_getrows_q6_k.h"
|
|
30
|
-
#include "shaderop_rope_f16.h"
|
|
31
|
-
#include "shaderop_rope_f32.h"
|
|
32
|
-
#include "shaderop_cpy_f16_f16.h"
|
|
33
|
-
#include "shaderop_cpy_f16_f32.h"
|
|
34
|
-
#include "shaderop_cpy_f32_f16.h"
|
|
35
|
-
#include "shaderop_cpy_f32_f32.h"
|
|
36
|
-
|
|
37
|
-
#include <algorithm>
|
|
38
|
-
#include <array>
|
|
39
|
-
#include <cassert>
|
|
40
|
-
#include <cstdint>
|
|
41
|
-
#include <cstdio>
|
|
42
|
-
#include <cstring>
|
|
43
|
-
#include <iostream>
|
|
44
|
-
#include <memory>
|
|
45
|
-
#include <stdexcept>
|
|
46
|
-
#include <string>
|
|
47
|
-
#include <unordered_map>
|
|
48
|
-
#include <utility>
|
|
49
|
-
#include <vector>
|
|
50
|
-
|
|
51
|
-
#include <kompute/Kompute.hpp>
|
|
52
|
-
#include <vulkan/vulkan.hpp>
|
|
53
|
-
|
|
54
|
-
#ifdef __linux__
|
|
55
|
-
#include <cstdlib> // for setenv
|
|
56
|
-
#endif
|
|
57
|
-
|
|
58
|
-
#define QK4_0 32
|
|
59
|
-
#define QR4_0 2
|
|
60
|
-
#define QK4_1 32
|
|
61
|
-
#define QK_NL 16
|
|
62
|
-
|
|
63
|
-
typedef ggml_fp16_t half;
|
|
64
|
-
|
|
65
|
-
static std::string ggml_kompute_format_name(int device) {
|
|
66
|
-
return "Kompute" + std::to_string(device);
|
|
67
|
-
}
|
|
68
|
-
|
|
69
|
-
struct ggml_kompute_context {
|
|
70
|
-
int device;
|
|
71
|
-
std::string name;
|
|
72
|
-
std::shared_ptr<vk::DescriptorPool> pool;
|
|
73
|
-
|
|
74
|
-
ggml_kompute_context(int device)
|
|
75
|
-
: device(device), name(ggml_kompute_format_name(device)) {}
|
|
76
|
-
};
|
|
77
|
-
|
|
78
|
-
// FIXME: It would be good to consolidate the kompute manager and the kompute context into one object
|
|
79
|
-
// and consolidate the init functions and simplify object lifetime management. As it currently stands,
|
|
80
|
-
// we *have* to have the kompute manager no matter what for device discovery, but the kompute context
|
|
81
|
-
// is only created when a device is set and vulkan is explicitly turned on.
|
|
82
|
-
static ggml_kompute_context *s_kompute_context = nullptr;
|
|
83
|
-
|
|
84
|
-
class kompute_manager {
|
|
85
|
-
kp::Manager *s_mgr = nullptr;
|
|
86
|
-
|
|
87
|
-
public:
|
|
88
|
-
kp::Manager *operator()() {
|
|
89
|
-
if (s_mgr && !s_mgr->hasInstance()) {
|
|
90
|
-
destroy();
|
|
91
|
-
}
|
|
92
|
-
if (!s_mgr) {
|
|
93
|
-
s_mgr = new kp::Manager;
|
|
94
|
-
}
|
|
95
|
-
return s_mgr;
|
|
96
|
-
}
|
|
97
|
-
|
|
98
|
-
void destroy() {
|
|
99
|
-
delete s_mgr;
|
|
100
|
-
s_mgr = nullptr;
|
|
101
|
-
}
|
|
102
|
-
};
|
|
103
|
-
|
|
104
|
-
static kompute_manager komputeManager;
|
|
105
|
-
|
|
106
|
-
struct ggml_vk_memory {
|
|
107
|
-
void *data = nullptr;
|
|
108
|
-
size_t size = 0;
|
|
109
|
-
vk::DeviceMemory *primaryMemory = nullptr;
|
|
110
|
-
vk::Buffer *primaryBuffer = nullptr;
|
|
111
|
-
vk::DeviceMemory *stagingMemory = nullptr;
|
|
112
|
-
vk::Buffer *stagingBuffer = nullptr;
|
|
113
|
-
};
|
|
114
|
-
|
|
115
|
-
#ifdef __linux__
|
|
116
|
-
__attribute__((constructor))
|
|
117
|
-
static void enable_sam() {
|
|
118
|
-
setenv("RADV_PERFTEST", "sam", false);
|
|
119
|
-
}
|
|
120
|
-
#endif
|
|
121
|
-
|
|
122
|
-
static bool ggml_vk_checkPhysicalDeviceFeatures(vk::PhysicalDevice physical_device) {
|
|
123
|
-
vk::PhysicalDeviceFeatures availableFeatures;
|
|
124
|
-
physical_device.getFeatures(&availableFeatures);
|
|
125
|
-
|
|
126
|
-
if (!availableFeatures.shaderInt16)
|
|
127
|
-
return false;
|
|
128
|
-
|
|
129
|
-
vk::PhysicalDeviceVulkan11Features availableFeatures11;
|
|
130
|
-
vk::PhysicalDeviceVulkan12Features availableFeatures12;
|
|
131
|
-
|
|
132
|
-
availableFeatures11.pNext = &availableFeatures12;
|
|
133
|
-
availableFeatures12.pNext = nullptr;
|
|
134
|
-
|
|
135
|
-
vk::PhysicalDeviceFeatures2 features2;
|
|
136
|
-
features2.pNext = &availableFeatures11;
|
|
137
|
-
|
|
138
|
-
physical_device.getFeatures2(&features2);
|
|
139
|
-
|
|
140
|
-
if (!availableFeatures11.uniformAndStorageBuffer16BitAccess ||
|
|
141
|
-
!availableFeatures11.storageBuffer16BitAccess) {
|
|
142
|
-
return false;
|
|
143
|
-
}
|
|
144
|
-
|
|
145
|
-
if (!availableFeatures12.storageBuffer8BitAccess ||
|
|
146
|
-
!availableFeatures12.uniformAndStorageBuffer8BitAccess ||
|
|
147
|
-
!availableFeatures12.shaderFloat16 ||
|
|
148
|
-
!availableFeatures12.shaderInt8) {
|
|
149
|
-
return false;
|
|
150
|
-
}
|
|
151
|
-
|
|
152
|
-
return true;
|
|
153
|
-
}
|
|
154
|
-
|
|
155
|
-
static const char * ggml_vk_getVendorName(uint32_t vendorID) {
|
|
156
|
-
switch (vendorID) {
|
|
157
|
-
case 0x10DE:
|
|
158
|
-
return "nvidia";
|
|
159
|
-
case 0x1002:
|
|
160
|
-
return "amd";
|
|
161
|
-
case 0x8086:
|
|
162
|
-
return "intel";
|
|
163
|
-
default:
|
|
164
|
-
return "unknown";
|
|
165
|
-
}
|
|
166
|
-
}
|
|
167
|
-
|
|
168
|
-
static std::vector<ggml_vk_device> ggml_vk_available_devices_internal(size_t memoryRequired) {
|
|
169
|
-
std::vector<ggml_vk_device> results;
|
|
170
|
-
if (!komputeManager()->hasVulkan() || !komputeManager()->hasInstance())
|
|
171
|
-
return results;
|
|
172
|
-
|
|
173
|
-
std::vector<vk::PhysicalDevice> physical_devices;
|
|
174
|
-
try {
|
|
175
|
-
physical_devices = komputeManager()->listDevices();
|
|
176
|
-
} catch (vk::SystemError & err) {
|
|
177
|
-
std::cerr << __func__ << ": ignoring Vulkan exception: " << err.what() << "\n";
|
|
178
|
-
return results;
|
|
179
|
-
}
|
|
180
|
-
|
|
181
|
-
uint32_t deviceCount = physical_devices.size();
|
|
182
|
-
if (deviceCount == 0)
|
|
183
|
-
return results;
|
|
184
|
-
|
|
185
|
-
std::unordered_map<std::string, size_t> count_by_name;
|
|
186
|
-
|
|
187
|
-
for (uint32_t i = 0; i < deviceCount; i++) {
|
|
188
|
-
const auto & physical_device = physical_devices[i];
|
|
189
|
-
|
|
190
|
-
VkPhysicalDeviceProperties dev_props = physical_device.getProperties();
|
|
191
|
-
VkPhysicalDeviceMemoryProperties memoryProperties = physical_device.getMemoryProperties();
|
|
192
|
-
const uint32_t major = VK_VERSION_MAJOR(dev_props.apiVersion);
|
|
193
|
-
const uint32_t minor = VK_VERSION_MINOR(dev_props.apiVersion);
|
|
194
|
-
if (major < 1 || minor < 2)
|
|
195
|
-
continue;
|
|
196
|
-
|
|
197
|
-
if (!ggml_vk_checkPhysicalDeviceFeatures(physical_device))
|
|
198
|
-
continue;
|
|
199
|
-
|
|
200
|
-
size_t heapSize = 0;
|
|
201
|
-
for (uint32_t j = 0; j < memoryProperties.memoryHeapCount; ++j) {
|
|
202
|
-
VkMemoryHeap heap = memoryProperties.memoryHeaps[j];
|
|
203
|
-
if (heap.flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) {
|
|
204
|
-
heapSize = heap.size;
|
|
205
|
-
break;
|
|
206
|
-
}
|
|
207
|
-
}
|
|
208
|
-
|
|
209
|
-
if (heapSize < memoryRequired)
|
|
210
|
-
continue;
|
|
211
|
-
|
|
212
|
-
auto ext_props = physical_device.enumerateDeviceExtensionProperties();
|
|
213
|
-
bool has_maintenance4 = false;
|
|
214
|
-
|
|
215
|
-
// Check if maintenance4 is supported
|
|
216
|
-
for (const auto & properties : ext_props) {
|
|
217
|
-
if (strcmp("VK_KHR_maintenance4", properties.extensionName) == 0) {
|
|
218
|
-
has_maintenance4 = true;
|
|
219
|
-
}
|
|
220
|
-
}
|
|
221
|
-
|
|
222
|
-
vk::PhysicalDeviceSubgroupProperties subgroup_props;
|
|
223
|
-
vk::PhysicalDeviceProperties2 dev_props2;
|
|
224
|
-
vk::PhysicalDeviceMaintenance3Properties dev_props3;
|
|
225
|
-
vk::PhysicalDeviceMaintenance4Properties dev_props4;
|
|
226
|
-
dev_props2.pNext = &dev_props3;
|
|
227
|
-
dev_props3.pNext = &subgroup_props;
|
|
228
|
-
if (has_maintenance4) {
|
|
229
|
-
subgroup_props.pNext = &dev_props4;
|
|
230
|
-
}
|
|
231
|
-
physical_device.getProperties2(&dev_props2);
|
|
232
|
-
|
|
233
|
-
if (subgroup_props.subgroupSize < 32)
|
|
234
|
-
continue;
|
|
235
|
-
|
|
236
|
-
ggml_vk_device d;
|
|
237
|
-
d.index = i;
|
|
238
|
-
d.type = dev_props.deviceType;
|
|
239
|
-
d.heapSize = heapSize;
|
|
240
|
-
d.vendor = strdup(ggml_vk_getVendorName(dev_props.vendorID));
|
|
241
|
-
d.subgroupSize = subgroup_props.subgroupSize;
|
|
242
|
-
d.bufferAlignment = dev_props.limits.minStorageBufferOffsetAlignment;
|
|
243
|
-
|
|
244
|
-
if (has_maintenance4) {
|
|
245
|
-
d.maxAlloc = std::min(dev_props3.maxMemoryAllocationSize, dev_props4.maxBufferSize);
|
|
246
|
-
} else {
|
|
247
|
-
d.maxAlloc = dev_props3.maxMemoryAllocationSize;
|
|
248
|
-
}
|
|
249
|
-
|
|
250
|
-
std::string name(dev_props.deviceName);
|
|
251
|
-
size_t n_idx = ++count_by_name[name];
|
|
252
|
-
if (n_idx > 1) {
|
|
253
|
-
name += " (" + std::to_string(n_idx) + ")";
|
|
254
|
-
}
|
|
255
|
-
d.name = strdup(name.c_str());
|
|
256
|
-
|
|
257
|
-
results.push_back(d);
|
|
258
|
-
}
|
|
259
|
-
|
|
260
|
-
std::stable_sort(results.begin(), results.end(),
|
|
261
|
-
[](const ggml_vk_device& lhs, const ggml_vk_device& rhs) -> bool {
|
|
262
|
-
if (lhs.type != rhs.type) {
|
|
263
|
-
if (lhs.type == VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU) return true;
|
|
264
|
-
if (rhs.type == VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU) return false;
|
|
265
|
-
|
|
266
|
-
if (lhs.type == VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU) return true;
|
|
267
|
-
if (rhs.type == VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU) return false;
|
|
268
|
-
}
|
|
269
|
-
return lhs.heapSize < rhs.heapSize;
|
|
270
|
-
}
|
|
271
|
-
);
|
|
272
|
-
|
|
273
|
-
return results;
|
|
274
|
-
}
|
|
275
|
-
|
|
276
|
-
// public API returns a C-style array
|
|
277
|
-
ggml_vk_device * ggml_vk_available_devices(size_t memoryRequired, size_t * count) {
|
|
278
|
-
auto devices = ggml_vk_available_devices_internal(memoryRequired);
|
|
279
|
-
*count = devices.size();
|
|
280
|
-
if (devices.empty()) {
|
|
281
|
-
return nullptr;
|
|
282
|
-
}
|
|
283
|
-
|
|
284
|
-
size_t nbytes = sizeof (ggml_vk_device) * (devices.size());
|
|
285
|
-
auto * arr = static_cast<ggml_vk_device *>(malloc(nbytes));
|
|
286
|
-
memcpy(arr, devices.data(), nbytes);
|
|
287
|
-
return arr;
|
|
288
|
-
}
|
|
289
|
-
|
|
290
|
-
static void ggml_vk_filterByVendor(std::vector<ggml_vk_device>& devices, const std::string& targetVendor) {
|
|
291
|
-
devices.erase(
|
|
292
|
-
std::remove_if(devices.begin(), devices.end(),
|
|
293
|
-
[&targetVendor](const ggml_vk_device& device) {
|
|
294
|
-
return device.vendor != targetVendor;
|
|
295
|
-
}),
|
|
296
|
-
devices.end()
|
|
297
|
-
);
|
|
298
|
-
}
|
|
299
|
-
|
|
300
|
-
static void ggml_vk_filterByName(std::vector<ggml_vk_device>& devices, const std::string& targetName) {
|
|
301
|
-
devices.erase(
|
|
302
|
-
std::remove_if(devices.begin(), devices.end(),
|
|
303
|
-
[&targetName](const ggml_vk_device& device) {
|
|
304
|
-
return device.name != targetName;
|
|
305
|
-
}),
|
|
306
|
-
devices.end()
|
|
307
|
-
);
|
|
308
|
-
}
|
|
309
|
-
|
|
310
|
-
static bool ggml_vk_get_device(ggml_vk_device * device, size_t memoryRequired, const std::string & name) {
|
|
311
|
-
if (name.empty())
|
|
312
|
-
return false;
|
|
313
|
-
|
|
314
|
-
auto devices = ggml_vk_available_devices_internal(memoryRequired);
|
|
315
|
-
if (name == "amd" || name == "nvidia" || name == "intel") {
|
|
316
|
-
ggml_vk_filterByVendor(devices, name);
|
|
317
|
-
} else if (name != "gpu") {
|
|
318
|
-
ggml_vk_filterByName(devices, name);
|
|
319
|
-
}
|
|
320
|
-
|
|
321
|
-
if (devices.empty())
|
|
322
|
-
return false;
|
|
323
|
-
|
|
324
|
-
*device = devices.front();
|
|
325
|
-
return true;
|
|
326
|
-
}
|
|
327
|
-
|
|
328
|
-
bool ggml_vk_get_device(ggml_vk_device * device, size_t memoryRequired, const char * name) {
|
|
329
|
-
return ggml_vk_get_device(device, memoryRequired, std::string(name));
|
|
330
|
-
}
|
|
331
|
-
|
|
332
|
-
bool ggml_vk_has_vulkan() {
|
|
333
|
-
return komputeManager()->hasVulkan();
|
|
334
|
-
}
|
|
335
|
-
|
|
336
|
-
bool ggml_vk_has_device() {
|
|
337
|
-
return komputeManager()->hasDevice();
|
|
338
|
-
}
|
|
339
|
-
|
|
340
|
-
ggml_vk_device ggml_vk_current_device() {
|
|
341
|
-
if (!komputeManager()->hasDevice())
|
|
342
|
-
return ggml_vk_device();
|
|
343
|
-
|
|
344
|
-
auto devices = ggml_vk_available_devices_internal(0);
|
|
345
|
-
ggml_vk_filterByName(devices, komputeManager()->physicalDevice()->getProperties().deviceName.data());
|
|
346
|
-
GGML_ASSERT(!devices.empty());
|
|
347
|
-
return devices.front();
|
|
348
|
-
}
|
|
349
|
-
|
|
350
|
-
static
|
|
351
|
-
void ggml_vk_allocate_descriptor_pool(struct ggml_kompute_context * ctx, size_t size) {
|
|
352
|
-
std::vector<vk::DescriptorPoolSize> descriptorPoolSizes = {
|
|
353
|
-
vk::DescriptorPoolSize(
|
|
354
|
-
vk::DescriptorType::eStorageBuffer,
|
|
355
|
-
3 * size // Descriptor count is number of possible tensors to pass into an algorithm
|
|
356
|
-
)
|
|
357
|
-
};
|
|
358
|
-
|
|
359
|
-
vk::DescriptorPoolCreateInfo descriptorPoolInfo(
|
|
360
|
-
vk::DescriptorPoolCreateFlags(),
|
|
361
|
-
size, // Max sets
|
|
362
|
-
static_cast<uint32_t>(descriptorPoolSizes.size()),
|
|
363
|
-
descriptorPoolSizes.data());
|
|
364
|
-
|
|
365
|
-
ctx->pool = std::make_shared<vk::DescriptorPool>();
|
|
366
|
-
vk::Result r = komputeManager()->device()->createDescriptorPool(
|
|
367
|
-
&descriptorPoolInfo, nullptr, ctx->pool.get());
|
|
368
|
-
if (r != vk::Result::eSuccess)
|
|
369
|
-
std::cerr << "Error allocating descriptor pool" << vk::to_string(r);
|
|
370
|
-
}
|
|
371
|
-
|
|
372
|
-
static
|
|
373
|
-
void ggml_vk_free_descriptor_pool(struct ggml_kompute_context * ctx) {
|
|
374
|
-
if (ctx->pool) {
|
|
375
|
-
komputeManager()->device()->destroy(
|
|
376
|
-
*ctx->pool,
|
|
377
|
-
(vk::Optional<const vk::AllocationCallbacks>)nullptr);
|
|
378
|
-
ctx->pool = nullptr;
|
|
379
|
-
}
|
|
380
|
-
}
|
|
381
|
-
|
|
382
|
-
static
|
|
383
|
-
vk::Buffer *ggml_vk_allocate_buffer(size_t size) {
|
|
384
|
-
vk::BufferCreateInfo bufferCreateInfo;
|
|
385
|
-
bufferCreateInfo.size = size;
|
|
386
|
-
bufferCreateInfo.usage = vk::BufferUsageFlagBits::eStorageBuffer |
|
|
387
|
-
vk::BufferUsageFlagBits::eTransferSrc |
|
|
388
|
-
vk::BufferUsageFlagBits::eTransferDst;
|
|
389
|
-
bufferCreateInfo.sharingMode = vk::SharingMode::eExclusive;
|
|
390
|
-
|
|
391
|
-
vk::Buffer *vkBuffer = new vk::Buffer;
|
|
392
|
-
vk::Result r = komputeManager()->device()->createBuffer(&bufferCreateInfo, nullptr, vkBuffer);
|
|
393
|
-
if (r != vk::Result::eSuccess)
|
|
394
|
-
std::cerr << "Error allocating buffer " << vk::to_string(r) << std::endl;
|
|
395
|
-
return vkBuffer;
|
|
396
|
-
}
|
|
397
|
-
|
|
398
|
-
static
|
|
399
|
-
vk::DeviceMemory *ggml_vk_allocate(size_t size, vk::MemoryPropertyFlags flags, vk::MemoryRequirements requirements, bool *isHostVisible) {
|
|
400
|
-
|
|
401
|
-
uint32_t memoryTypeIndex = -1;
|
|
402
|
-
bool memoryTypeIndexFound = false;
|
|
403
|
-
vk::PhysicalDeviceMemoryProperties memoryProperties = komputeManager()->physicalDevice()->getMemoryProperties();
|
|
404
|
-
for (uint32_t i = 0; i < memoryProperties.memoryTypeCount; i++) {
|
|
405
|
-
const vk::MemoryType &memoryType = memoryProperties.memoryTypes[i];
|
|
406
|
-
const vk::MemoryHeap &memoryHeap = memoryProperties.memoryHeaps[memoryType.heapIndex];
|
|
407
|
-
if (memoryHeap.size < size) {
|
|
408
|
-
continue;
|
|
409
|
-
}
|
|
410
|
-
|
|
411
|
-
if (requirements.memoryTypeBits & (1 << i)) {
|
|
412
|
-
if (((memoryProperties.memoryTypes[i]).propertyFlags &
|
|
413
|
-
flags) == flags) {
|
|
414
|
-
memoryTypeIndex = i;
|
|
415
|
-
memoryTypeIndexFound = true;
|
|
416
|
-
if (isHostVisible && (memoryProperties.memoryTypes[i].propertyFlags & vk::MemoryPropertyFlagBits::eHostVisible)) {
|
|
417
|
-
*isHostVisible = true;
|
|
418
|
-
}
|
|
419
|
-
break;
|
|
420
|
-
}
|
|
421
|
-
}
|
|
422
|
-
}
|
|
423
|
-
if (!memoryTypeIndexFound) {
|
|
424
|
-
throw std::runtime_error(
|
|
425
|
-
"Memory type index for buffer creation not found");
|
|
426
|
-
}
|
|
427
|
-
|
|
428
|
-
vk::MemoryAllocateInfo allocInfo;
|
|
429
|
-
allocInfo.allocationSize = size;
|
|
430
|
-
allocInfo.memoryTypeIndex = memoryTypeIndex;
|
|
431
|
-
vk::DeviceMemory *vkDeviceMemory = new vk::DeviceMemory;
|
|
432
|
-
vk::Result r = komputeManager()->device()->allocateMemory(&allocInfo, nullptr, vkDeviceMemory);
|
|
433
|
-
if (r != vk::Result::eSuccess) {
|
|
434
|
-
std::cerr << "Error allocating memory " << vk::to_string(r) << std::endl;
|
|
435
|
-
throw std::runtime_error("Error allocating vulkan memory.");
|
|
436
|
-
}
|
|
437
|
-
return vkDeviceMemory;
|
|
438
|
-
}
|
|
439
|
-
|
|
440
|
-
static size_t ggml_vk_aligned_offset(ggml_backend_buffer_t buffer, size_t offset) {
|
|
441
|
-
size_t minStorageBufferOffsetAlignment = ggml_backend_buffer_get_alignment(buffer);
|
|
442
|
-
|
|
443
|
-
// If offset is already aligned, return it directly
|
|
444
|
-
if (offset % minStorageBufferOffsetAlignment == 0) {
|
|
445
|
-
return offset;
|
|
446
|
-
}
|
|
447
|
-
|
|
448
|
-
// Otherwise, return the largest multiple of minStorageBufferOffsetAlignment less than offset
|
|
449
|
-
return (offset / minStorageBufferOffsetAlignment) * minStorageBufferOffsetAlignment;
|
|
450
|
-
}
|
|
451
|
-
|
|
452
|
-
static ggml_vk_memory ggml_vk_allocate(size_t size) {
|
|
453
|
-
ggml_vk_memory memory;
|
|
454
|
-
bool isHostVisible = false;
|
|
455
|
-
{
|
|
456
|
-
memory.primaryBuffer = ggml_vk_allocate_buffer(size);
|
|
457
|
-
vk::MemoryRequirements memoryRequirements = komputeManager()->device()->getBufferMemoryRequirements(*memory.primaryBuffer);
|
|
458
|
-
vk::MemoryPropertyFlags memoryPropertyFlags = vk::MemoryPropertyFlagBits::eDeviceLocal;
|
|
459
|
-
memory.primaryMemory = ggml_vk_allocate(size, memoryPropertyFlags, memoryRequirements, &isHostVisible);
|
|
460
|
-
komputeManager()->device()->bindBufferMemory(*memory.primaryBuffer, *memory.primaryMemory, 0);
|
|
461
|
-
if (isHostVisible) {
|
|
462
|
-
vk::Result r = komputeManager()->device()->mapMemory(*memory.primaryMemory, 0, size, vk::MemoryMapFlags(), &memory.data);
|
|
463
|
-
if (r != vk::Result::eSuccess)
|
|
464
|
-
std::cerr << "Error mapping memory" << vk::to_string(r);
|
|
465
|
-
}
|
|
466
|
-
}
|
|
467
|
-
|
|
468
|
-
if (!isHostVisible) {
|
|
469
|
-
memory.stagingBuffer = ggml_vk_allocate_buffer(size);
|
|
470
|
-
vk::MemoryRequirements memoryRequirements = komputeManager()->device()->getBufferMemoryRequirements(*memory.stagingBuffer);
|
|
471
|
-
vk::MemoryPropertyFlags memoryPropertyFlags = vk::MemoryPropertyFlagBits::eHostVisible |
|
|
472
|
-
vk::MemoryPropertyFlagBits::eHostCoherent |
|
|
473
|
-
vk::MemoryPropertyFlagBits::eHostCached;
|
|
474
|
-
memory.stagingMemory = ggml_vk_allocate(size, memoryPropertyFlags, memoryRequirements, &isHostVisible);
|
|
475
|
-
komputeManager()->device()->bindBufferMemory(*memory.stagingBuffer, *memory.stagingMemory, 0);
|
|
476
|
-
vk::Result r = komputeManager()->device()->mapMemory(*memory.stagingMemory, 0, size, vk::MemoryMapFlags(), &memory.data);
|
|
477
|
-
if (r != vk::Result::eSuccess)
|
|
478
|
-
std::cerr << "Error mapping memory" << vk::to_string(r);
|
|
479
|
-
}
|
|
480
|
-
|
|
481
|
-
memory.size = size;
|
|
482
|
-
return memory;
|
|
483
|
-
}
|
|
484
|
-
|
|
485
|
-
static void ggml_vk_free_memory(ggml_vk_memory &memory)
|
|
486
|
-
{
|
|
487
|
-
komputeManager()->device()->destroy(
|
|
488
|
-
*memory.primaryBuffer,
|
|
489
|
-
(vk::Optional<const vk::AllocationCallbacks>)nullptr);
|
|
490
|
-
if (memory.stagingBuffer) {
|
|
491
|
-
komputeManager()->device()->destroy(
|
|
492
|
-
*memory.stagingBuffer,
|
|
493
|
-
(vk::Optional<const vk::AllocationCallbacks>)nullptr);
|
|
494
|
-
}
|
|
495
|
-
komputeManager()->device()->freeMemory(
|
|
496
|
-
*memory.primaryMemory,
|
|
497
|
-
(vk::Optional<const vk::AllocationCallbacks>)nullptr);
|
|
498
|
-
if (memory.stagingMemory) {
|
|
499
|
-
komputeManager()->device()->freeMemory(
|
|
500
|
-
*memory.stagingMemory,
|
|
501
|
-
(vk::Optional<const vk::AllocationCallbacks>)nullptr);
|
|
502
|
-
}
|
|
503
|
-
}
|
|
504
|
-
|
|
505
|
-
static const char * ggml_backend_kompute_buffer_type_get_name(ggml_backend_buffer_type_t buft);
|
|
506
|
-
|
|
507
|
-
static
|
|
508
|
-
ggml_vk_memory * ggml_vk_find_tensor(const struct ggml_tensor * t, uint64_t & offset) {
|
|
509
|
-
ggml_backend_buffer_t buffer = t->view_src ? t->view_src->buffer : t->buffer;
|
|
510
|
-
|
|
511
|
-
// compatibility with ggml-backend
|
|
512
|
-
GGML_ASSERT(buffer && buffer->buft->iface.get_name == ggml_backend_kompute_buffer_type_get_name);
|
|
513
|
-
|
|
514
|
-
ggml_vk_memory * buf_ctx = static_cast<ggml_vk_memory *>(buffer->context);
|
|
515
|
-
|
|
516
|
-
const intptr_t ioffs = intptr_t(t->data) - intptr_t(buf_ctx->data);
|
|
517
|
-
|
|
518
|
-
GGML_ASSERT(ioffs >= 0 && ioffs + int64_t(ggml_nbytes(t)) <= int64_t(buffer->size));
|
|
519
|
-
|
|
520
|
-
offset = uint64_t(ioffs);
|
|
521
|
-
return buf_ctx;
|
|
522
|
-
}
|
|
523
|
-
|
|
524
|
-
static
|
|
525
|
-
const std::shared_ptr<kp::Tensor> ggml_vk_get_tensor(const struct ggml_tensor * t, uint32_t * alignedOffset = nullptr) {
|
|
526
|
-
uint64_t originalOffset = 0;
|
|
527
|
-
auto * res = ggml_vk_find_tensor(t, originalOffset);
|
|
528
|
-
if (!res) {
|
|
529
|
-
static std::shared_ptr<kp::Tensor> nullTensor = nullptr;
|
|
530
|
-
return nullTensor;
|
|
531
|
-
}
|
|
532
|
-
|
|
533
|
-
// Create a tensor whose memory will be composed of our buffers at the correct offset
|
|
534
|
-
const size_t nelements = ggml_nelements(t);
|
|
535
|
-
size_t nbytes = ggml_nbytes(t);
|
|
536
|
-
|
|
537
|
-
size_t vulkanOffset = ggml_vk_aligned_offset(t->buffer, originalOffset);
|
|
538
|
-
if (alignedOffset) {
|
|
539
|
-
*alignedOffset = originalOffset - vulkanOffset;
|
|
540
|
-
nbytes += *alignedOffset;
|
|
541
|
-
}
|
|
542
|
-
|
|
543
|
-
return komputeManager()->tensor(
|
|
544
|
-
t->data,
|
|
545
|
-
nelements,
|
|
546
|
-
nbytes, kp::Tensor::TensorDataTypes::eFloat,
|
|
547
|
-
res->primaryMemory, res->primaryBuffer,
|
|
548
|
-
res->stagingMemory, res->stagingBuffer,
|
|
549
|
-
vulkanOffset);
|
|
550
|
-
}
|
|
551
|
-
|
|
552
|
-
static std::vector<uint32_t> getSpirvShader(const unsigned char* rawData, size_t size) {
|
|
553
|
-
if (size % sizeof(uint32_t) != 0) {
|
|
554
|
-
throw std::runtime_error("Invalid size: must be divisible by sizeof(uint32_t)");
|
|
555
|
-
}
|
|
556
|
-
|
|
557
|
-
const uint32_t* data_ptr = reinterpret_cast<const uint32_t*>(rawData);
|
|
558
|
-
size_t count = size / sizeof(uint32_t);
|
|
559
|
-
return std::vector<uint32_t>(data_ptr, data_ptr + count);
|
|
560
|
-
}
|
|
561
|
-
|
|
562
|
-
inline static
|
|
563
|
-
uint32_t safe_divide(uint32_t a, uint32_t b) {
|
|
564
|
-
if (b <= 1) {
|
|
565
|
-
return a;
|
|
566
|
-
}
|
|
567
|
-
if ((a % b) != 0) {
|
|
568
|
-
fprintf(stderr, "((%u %% %u) == %u) != 0\n", a, b, a % b);
|
|
569
|
-
GGML_ASSERT(!"safe_divide result would've had remainder");
|
|
570
|
-
}
|
|
571
|
-
return a / b;
|
|
572
|
-
}
|
|
573
|
-
|
|
574
|
-
static void ggml_vk_add(
|
|
575
|
-
kp::Sequence& seq,
|
|
576
|
-
const std::shared_ptr<kp::Tensor>& inA,
|
|
577
|
-
const std::shared_ptr<kp::Tensor>& inB,
|
|
578
|
-
const std::shared_ptr<kp::Tensor>& out,
|
|
579
|
-
uint32_t inAOff, uint32_t inBOff, uint32_t outOff,
|
|
580
|
-
int32_t ne00, int32_t ne01, int32_t ne02, int32_t ne03,
|
|
581
|
-
int32_t nb00, int32_t nb01, int32_t nb02, int32_t nb03,
|
|
582
|
-
int32_t ne10, int32_t ne11, int32_t ne12, int32_t ne13,
|
|
583
|
-
int32_t nb10, int32_t nb11, int32_t nb12, int32_t nb13,
|
|
584
|
-
int32_t ne0,
|
|
585
|
-
int32_t nb0, int32_t nb1, int32_t nb2, int32_t nb3
|
|
586
|
-
) {
|
|
587
|
-
const static auto spirv = getSpirvShader(kp::shader_data::op_add_comp_spv,
|
|
588
|
-
kp::shader_data::op_add_comp_spv_len);
|
|
589
|
-
|
|
590
|
-
struct PushConstants {
|
|
591
|
-
uint32_t inAOff, inBOff, outOff;
|
|
592
|
-
int32_t ne00;
|
|
593
|
-
int32_t nb00, nb01, nb02, nb03;
|
|
594
|
-
int32_t ne10, ne11, ne12, ne13;
|
|
595
|
-
int32_t nb10, nb11, nb12, nb13;
|
|
596
|
-
int32_t ne0;
|
|
597
|
-
int32_t nb0, nb1, nb2, nb3;
|
|
598
|
-
} const pushConsts {
|
|
599
|
-
safe_divide(inAOff, 4), safe_divide(inBOff, 4), safe_divide(outOff, 4),
|
|
600
|
-
ne00,
|
|
601
|
-
nb00, nb01, nb02, nb03,
|
|
602
|
-
ne10, ne11, ne12, ne13,
|
|
603
|
-
nb10, nb11, nb12, nb13,
|
|
604
|
-
ne0,
|
|
605
|
-
nb0, nb1, nb2, nb3
|
|
606
|
-
};
|
|
607
|
-
|
|
608
|
-
std::shared_ptr<kp::Algorithm> s_algo = nullptr;
|
|
609
|
-
if (!komputeManager()->hasAlgorithm(__func__)) {
|
|
610
|
-
s_algo = komputeManager()->algorithm<float, PushConstants>(__func__, s_kompute_context->pool.get(), {inA, inB, out}, spirv, {unsigned(ne01), unsigned(ne02), unsigned(ne03)}, {}, {pushConsts});
|
|
611
|
-
} else {
|
|
612
|
-
s_algo = komputeManager()->getAlgorithm(__func__);
|
|
613
|
-
s_algo->setTensors({inA, inB, out});
|
|
614
|
-
s_algo->setWorkgroup({unsigned(ne01), unsigned(ne02), unsigned(ne03)});
|
|
615
|
-
s_algo->setPushConstants<PushConstants>({pushConsts});
|
|
616
|
-
s_algo->updateDescriptors(s_kompute_context->pool.get());
|
|
617
|
-
}
|
|
618
|
-
seq.record<kp::OpAlgoDispatch>(s_algo);
|
|
619
|
-
}
|
|
620
|
-
|
|
621
|
-
static void ggml_vk_addrow(kp::Sequence& seq,
|
|
622
|
-
const std::shared_ptr<kp::Tensor>& inA,
|
|
623
|
-
const std::shared_ptr<kp::Tensor>& inB,
|
|
624
|
-
const std::shared_ptr<kp::Tensor>& out,
|
|
625
|
-
uint32_t inAOff, uint32_t inBOff, uint32_t outOff,
|
|
626
|
-
uint32_t size, uint32_t row = 0) {
|
|
627
|
-
|
|
628
|
-
const static auto spirv = getSpirvShader(kp::shader_data::op_addrow_comp_spv,
|
|
629
|
-
kp::shader_data::op_addrow_comp_spv_len);
|
|
630
|
-
|
|
631
|
-
struct PushConstants {
|
|
632
|
-
uint32_t inAOff, inBOff, outOff;
|
|
633
|
-
uint32_t row;
|
|
634
|
-
} const pushConsts {
|
|
635
|
-
safe_divide(inAOff, 4), safe_divide(inBOff, 4), safe_divide(outOff, 4),
|
|
636
|
-
row
|
|
637
|
-
};
|
|
638
|
-
|
|
639
|
-
std::shared_ptr<kp::Algorithm> s_algo = nullptr;
|
|
640
|
-
if (!komputeManager()->hasAlgorithm(__func__))
|
|
641
|
-
s_algo = komputeManager()->algorithm<float, PushConstants>(__func__, s_kompute_context->pool.get(), {inA, inB, out}, spirv, {size}, {}, {pushConsts});
|
|
642
|
-
else {
|
|
643
|
-
s_algo = komputeManager()->getAlgorithm(__func__);
|
|
644
|
-
s_algo->setTensors({inA, inB, out});
|
|
645
|
-
s_algo->setWorkgroup({size});
|
|
646
|
-
s_algo->setPushConstants<PushConstants>({pushConsts});
|
|
647
|
-
s_algo->updateDescriptors(s_kompute_context->pool.get());
|
|
648
|
-
}
|
|
649
|
-
seq.record<kp::OpAlgoDispatch>(s_algo);
|
|
650
|
-
}
|
|
651
|
-
|
|
652
|
-
static void ggml_vk_mul(
|
|
653
|
-
kp::Sequence& seq,
|
|
654
|
-
const std::shared_ptr<kp::Tensor>& inA,
|
|
655
|
-
const std::shared_ptr<kp::Tensor>& inB,
|
|
656
|
-
const std::shared_ptr<kp::Tensor>& out,
|
|
657
|
-
uint32_t inAOff, uint32_t inBOff, uint32_t outOff,
|
|
658
|
-
int32_t ne00, int32_t ne01, int32_t ne02, int32_t ne03,
|
|
659
|
-
int32_t nb00, int32_t nb01, int32_t nb02, int32_t nb03,
|
|
660
|
-
int32_t ne10, int32_t ne11, int32_t ne12, int32_t ne13,
|
|
661
|
-
int32_t nb10, int32_t nb11, int32_t nb12, int32_t nb13,
|
|
662
|
-
int32_t ne0,
|
|
663
|
-
int32_t nb0, int32_t nb1, int32_t nb2, int32_t nb3
|
|
664
|
-
) {
|
|
665
|
-
const static auto spirv = getSpirvShader(kp::shader_data::op_mul_comp_spv,
|
|
666
|
-
kp::shader_data::op_mul_comp_spv_len);
|
|
667
|
-
|
|
668
|
-
struct PushConstants {
|
|
669
|
-
uint32_t inAOff, inBOff, outOff;
|
|
670
|
-
int32_t ne00;
|
|
671
|
-
int32_t nb00, nb01, nb02, nb03;
|
|
672
|
-
int32_t ne10, ne11, ne12, ne13;
|
|
673
|
-
int32_t nb10, nb11, nb12, nb13;
|
|
674
|
-
int32_t ne0;
|
|
675
|
-
int32_t nb0, nb1, nb2, nb3;
|
|
676
|
-
} const pushConsts {
|
|
677
|
-
safe_divide(inAOff, 4), safe_divide(inBOff, 4), safe_divide(outOff, 4),
|
|
678
|
-
ne00,
|
|
679
|
-
nb00, nb01, nb02, nb03,
|
|
680
|
-
ne10, ne11, ne12, ne13,
|
|
681
|
-
nb10, nb11, nb12, nb13,
|
|
682
|
-
ne0,
|
|
683
|
-
nb0, nb1, nb2, nb3
|
|
684
|
-
};
|
|
685
|
-
|
|
686
|
-
std::shared_ptr<kp::Algorithm> s_algo = nullptr;
|
|
687
|
-
if (!komputeManager()->hasAlgorithm(__func__)) {
|
|
688
|
-
s_algo = komputeManager()->algorithm<float, PushConstants>(__func__, s_kompute_context->pool.get(), {inA, inB, out}, spirv, {unsigned(ne01), unsigned(ne02), unsigned(ne03)}, {}, {pushConsts});
|
|
689
|
-
} else {
|
|
690
|
-
s_algo = komputeManager()->getAlgorithm(__func__);
|
|
691
|
-
s_algo->setTensors({inA, inB, out});
|
|
692
|
-
s_algo->setWorkgroup({unsigned(ne01), unsigned(ne02), unsigned(ne03)});
|
|
693
|
-
s_algo->setPushConstants<PushConstants>({pushConsts});
|
|
694
|
-
s_algo->updateDescriptors(s_kompute_context->pool.get());
|
|
695
|
-
}
|
|
696
|
-
seq.record<kp::OpAlgoDispatch>(s_algo);
|
|
697
|
-
}
|
|
698
|
-
|
|
699
|
-
static void ggml_vk_scale(kp::Sequence& seq,
|
|
700
|
-
const std::shared_ptr<kp::Tensor>& in,
|
|
701
|
-
const std::shared_ptr<kp::Tensor>& out,
|
|
702
|
-
uint32_t inOff, uint32_t outOff,
|
|
703
|
-
uint32_t size, float scale) {
|
|
704
|
-
const static auto spirv_1 = getSpirvShader(
|
|
705
|
-
kp::shader_data::op_scale_comp_spv, kp::shader_data::op_scale_comp_spv_len
|
|
706
|
-
);
|
|
707
|
-
const static auto spirv_8 = getSpirvShader(
|
|
708
|
-
kp::shader_data::op_scale_8_comp_spv, kp::shader_data::op_scale_8_comp_spv_len
|
|
709
|
-
);
|
|
710
|
-
|
|
711
|
-
struct PushConstants {
|
|
712
|
-
uint32_t inOff, outOff;
|
|
713
|
-
float scale;
|
|
714
|
-
} const pushConsts {
|
|
715
|
-
safe_divide(inOff, 4), safe_divide(outOff, 4),
|
|
716
|
-
scale
|
|
717
|
-
};
|
|
718
|
-
|
|
719
|
-
const auto * spirv = &spirv_1;
|
|
720
|
-
std::string name(__func__);
|
|
721
|
-
if (size % 8 == 0) {
|
|
722
|
-
size /= 8;
|
|
723
|
-
name += "_8";
|
|
724
|
-
spirv = &spirv_8;
|
|
725
|
-
}
|
|
726
|
-
|
|
727
|
-
std::shared_ptr<kp::Algorithm> s_algo = nullptr;
|
|
728
|
-
if (!komputeManager()->hasAlgorithm(name)) {
|
|
729
|
-
s_algo = komputeManager()->algorithm<float, PushConstants>(name, s_kompute_context->pool.get(), {in, out}, *spirv, {size}, {}, {pushConsts});
|
|
730
|
-
} else {
|
|
731
|
-
s_algo = komputeManager()->getAlgorithm(name);
|
|
732
|
-
s_algo->setTensors({in, out});
|
|
733
|
-
s_algo->setWorkgroup({size});
|
|
734
|
-
s_algo->setPushConstants<PushConstants>({pushConsts});
|
|
735
|
-
s_algo->updateDescriptors(s_kompute_context->pool.get());
|
|
736
|
-
}
|
|
737
|
-
seq.record<kp::OpAlgoDispatch>(s_algo);
|
|
738
|
-
}
|
|
739
|
-
|
|
740
|
-
static void ggml_vk_xxlu(
|
|
741
|
-
const std::vector<uint32_t>& spirv, const char * suffix, kp::Sequence& seq,
|
|
742
|
-
const std::shared_ptr<kp::Tensor>& in,
|
|
743
|
-
const std::shared_ptr<kp::Tensor>& out,
|
|
744
|
-
uint32_t inOff, uint32_t outOff,
|
|
745
|
-
uint32_t size
|
|
746
|
-
) {
|
|
747
|
-
struct PushConstants {
|
|
748
|
-
uint32_t inOff, outOff;
|
|
749
|
-
} const pushConsts {
|
|
750
|
-
safe_divide(inOff, 4), safe_divide(outOff, 4),
|
|
751
|
-
};
|
|
752
|
-
|
|
753
|
-
auto name = std::string(__func__) + "_" + suffix;
|
|
754
|
-
std::shared_ptr<kp::Algorithm> s_algo = nullptr;
|
|
755
|
-
if (!komputeManager()->hasAlgorithm(name)) {
|
|
756
|
-
s_algo = komputeManager()->algorithm<float, PushConstants>(name, s_kompute_context->pool.get(), {in, out}, spirv, {size}, {}, {pushConsts});
|
|
757
|
-
} else {
|
|
758
|
-
s_algo = komputeManager()->getAlgorithm(name);
|
|
759
|
-
s_algo->setTensors({in, out});
|
|
760
|
-
s_algo->setWorkgroup({size});
|
|
761
|
-
s_algo->setPushConstants<PushConstants>({pushConsts});
|
|
762
|
-
s_algo->updateDescriptors(s_kompute_context->pool.get());
|
|
763
|
-
}
|
|
764
|
-
seq.record<kp::OpAlgoDispatch>(s_algo);
|
|
765
|
-
}
|
|
766
|
-
|
|
767
|
-
template <typename... Args>
|
|
768
|
-
static void ggml_vk_silu(Args&&... args) {
|
|
769
|
-
const static auto spirv = getSpirvShader(kp::shader_data::op_silu_comp_spv,
|
|
770
|
-
kp::shader_data::op_silu_comp_spv_len);
|
|
771
|
-
|
|
772
|
-
ggml_vk_xxlu(spirv, "silu", std::forward<Args>(args)...);
|
|
773
|
-
}
|
|
774
|
-
|
|
775
|
-
template <typename... Args>
|
|
776
|
-
static void ggml_vk_relu(Args&&... args) {
|
|
777
|
-
const static auto spirv = getSpirvShader(kp::shader_data::op_relu_comp_spv,
|
|
778
|
-
kp::shader_data::op_relu_comp_spv_len);
|
|
779
|
-
|
|
780
|
-
ggml_vk_xxlu(spirv, "relu", std::forward<Args>(args)...);
|
|
781
|
-
}
|
|
782
|
-
|
|
783
|
-
template <typename... Args>
|
|
784
|
-
static void ggml_vk_gelu(Args&&... args) {
|
|
785
|
-
const static auto spirv = getSpirvShader(kp::shader_data::op_gelu_comp_spv,
|
|
786
|
-
kp::shader_data::op_gelu_comp_spv_len);
|
|
787
|
-
|
|
788
|
-
ggml_vk_xxlu(spirv, "gelu", std::forward<Args>(args)...);
|
|
789
|
-
}
|
|
790
|
-
|
|
791
|
-
static void ggml_vk_soft_max(
|
|
792
|
-
kp::Sequence& seq,
|
|
793
|
-
const std::shared_ptr<kp::Tensor>& inA,
|
|
794
|
-
const std::shared_ptr<kp::Tensor>& inB,
|
|
795
|
-
const std::shared_ptr<kp::Tensor>& out,
|
|
796
|
-
uint32_t inAOff, uint32_t inBOff, uint32_t outOff,
|
|
797
|
-
int32_t ne00, int32_t ne01, int32_t ne02, uint32_t ne03,
|
|
798
|
-
float scale
|
|
799
|
-
) {
|
|
800
|
-
const static auto spirv = getSpirvShader(kp::shader_data::op_softmax_comp_spv,
|
|
801
|
-
kp::shader_data::op_softmax_comp_spv_len);
|
|
802
|
-
|
|
803
|
-
struct PushConstants {
|
|
804
|
-
uint32_t inAOff, inBOff, outOff;
|
|
805
|
-
int32_t ne00, ne01, ne02;
|
|
806
|
-
float scale;
|
|
807
|
-
int32_t mask;
|
|
808
|
-
} pushConsts {
|
|
809
|
-
safe_divide(inAOff, 4), safe_divide(inBOff, 4), safe_divide(outOff, 4),
|
|
810
|
-
ne00, ne01, ne02,
|
|
811
|
-
scale,
|
|
812
|
-
bool(inB)
|
|
813
|
-
};
|
|
814
|
-
|
|
815
|
-
auto & inB_ = inB ? inB : inA;
|
|
816
|
-
|
|
817
|
-
std::shared_ptr<kp::Algorithm> s_algo = nullptr;
|
|
818
|
-
if (!komputeManager()->hasAlgorithm(__func__)) {
|
|
819
|
-
// FIXME: The softmax kernel needs to be fixed to use the subgroupsize which can vary by device
|
|
820
|
-
const uint32_t local_x = 32;
|
|
821
|
-
s_algo = komputeManager()->algorithm<uint32_t, PushConstants>(__func__, s_kompute_context->pool.get(), {inA, inB_, out}, spirv, {unsigned(ne01), unsigned(ne02), unsigned(ne03)}, {local_x}, {pushConsts});
|
|
822
|
-
} else {
|
|
823
|
-
s_algo = komputeManager()->getAlgorithm(__func__);
|
|
824
|
-
s_algo->setTensors({inA, inB_, out});
|
|
825
|
-
s_algo->setWorkgroup({unsigned(ne01), unsigned(ne02), unsigned(ne03)});
|
|
826
|
-
s_algo->setPushConstants<PushConstants>({pushConsts});
|
|
827
|
-
s_algo->updateDescriptors(s_kompute_context->pool.get());
|
|
828
|
-
}
|
|
829
|
-
seq.record<kp::OpAlgoDispatch>(s_algo);
|
|
830
|
-
}
|
|
831
|
-
|
|
832
|
-
static void ggml_vk_norm_(
|
|
833
|
-
const std::vector<uint32_t>& spirv, const char * suffix, kp::Sequence& seq,
|
|
834
|
-
const std::shared_ptr<kp::Tensor>& in,
|
|
835
|
-
const std::shared_ptr<kp::Tensor>& out,
|
|
836
|
-
uint32_t inOff, uint32_t outOff,
|
|
837
|
-
int32_t ne00, int32_t nb01,
|
|
838
|
-
int32_t nrows, float epsilon
|
|
839
|
-
) {
|
|
840
|
-
GGML_ASSERT(nb01%sizeof(float) == 0);
|
|
841
|
-
GGML_ASSERT(ne00%sizeof(float) == 0);
|
|
842
|
-
|
|
843
|
-
struct PushConstants {
|
|
844
|
-
uint32_t inOff, outOff;
|
|
845
|
-
uint32_t ne00, nb01;
|
|
846
|
-
float eps;
|
|
847
|
-
} pushConsts {
|
|
848
|
-
safe_divide(inOff, 4), safe_divide(outOff, 4),
|
|
849
|
-
(uint32_t)ne00, (uint32_t)nb01, epsilon
|
|
850
|
-
};
|
|
851
|
-
|
|
852
|
-
auto name = std::string(__func__) + "_" + suffix;
|
|
853
|
-
std::shared_ptr<kp::Algorithm> s_algo = nullptr;
|
|
854
|
-
if (!komputeManager()->hasAlgorithm(name)) {
|
|
855
|
-
s_algo = komputeManager()->algorithm<float, PushConstants>(name, s_kompute_context->pool.get(), {in, out}, spirv, {(uint32_t)nrows}, {}, {pushConsts});
|
|
856
|
-
} else {
|
|
857
|
-
s_algo = komputeManager()->getAlgorithm(name);
|
|
858
|
-
s_algo->setTensors({in, out});
|
|
859
|
-
s_algo->setWorkgroup({(uint32_t)nrows});
|
|
860
|
-
s_algo->setPushConstants<PushConstants>({pushConsts});
|
|
861
|
-
s_algo->updateDescriptors(s_kompute_context->pool.get());
|
|
862
|
-
}
|
|
863
|
-
seq.record<kp::OpAlgoDispatch>(s_algo);
|
|
864
|
-
}
|
|
865
|
-
|
|
866
|
-
template <typename... Args>
|
|
867
|
-
static void ggml_vk_norm(Args&&... args) {
|
|
868
|
-
const static auto spirv = getSpirvShader(kp::shader_data::op_norm_comp_spv,
|
|
869
|
-
kp::shader_data::op_norm_comp_spv_len);
|
|
870
|
-
|
|
871
|
-
ggml_vk_norm_(spirv, "norm", std::forward<Args>(args)...);
|
|
872
|
-
}
|
|
873
|
-
|
|
874
|
-
template <typename... Args>
|
|
875
|
-
static void ggml_vk_rms_norm(Args&&... args) {
|
|
876
|
-
const static auto spirv = getSpirvShader(kp::shader_data::op_rmsnorm_comp_spv,
|
|
877
|
-
kp::shader_data::op_rmsnorm_comp_spv_len);
|
|
878
|
-
|
|
879
|
-
ggml_vk_norm_(spirv, "rms", std::forward<Args>(args)...);
|
|
880
|
-
}
|
|
881
|
-
|
|
882
|
-
static void ggml_vk_diag_mask_inf(kp::Sequence& seq,
|
|
883
|
-
const std::shared_ptr<kp::Tensor>& in,
|
|
884
|
-
const std::shared_ptr<kp::Tensor>& out,
|
|
885
|
-
uint32_t inOff, uint32_t outOff,
|
|
886
|
-
uint32_t n_past,
|
|
887
|
-
int32_t ne00, int32_t ne01, int32_t ne02) {
|
|
888
|
-
const static auto spirv = getSpirvShader(kp::shader_data::op_diagmask_comp_spv,
|
|
889
|
-
kp::shader_data::op_diagmask_comp_spv_len);
|
|
890
|
-
|
|
891
|
-
struct PushConstants {
|
|
892
|
-
uint32_t inOff, outOff;
|
|
893
|
-
uint32_t n_past;
|
|
894
|
-
int32_t ne00, ne01;
|
|
895
|
-
} pushConsts {
|
|
896
|
-
safe_divide(inOff, 4), safe_divide(outOff, 4),
|
|
897
|
-
n_past,
|
|
898
|
-
ne00, ne01
|
|
899
|
-
};
|
|
900
|
-
|
|
901
|
-
std::shared_ptr<kp::Algorithm> s_algo = nullptr;
|
|
902
|
-
if (!komputeManager()->hasAlgorithm(__func__))
|
|
903
|
-
s_algo = komputeManager()->algorithm<float, PushConstants>(__func__, s_kompute_context->pool.get(), {in, out}, spirv, {unsigned(ne00), unsigned(ne01), unsigned(ne02)}, {}, {pushConsts});
|
|
904
|
-
else {
|
|
905
|
-
s_algo = komputeManager()->getAlgorithm(__func__);
|
|
906
|
-
s_algo->setTensors({in, out});
|
|
907
|
-
s_algo->setWorkgroup({unsigned(ne00), unsigned(ne01), unsigned(ne02)});
|
|
908
|
-
s_algo->setPushConstants<PushConstants>({pushConsts});
|
|
909
|
-
s_algo->updateDescriptors(s_kompute_context->pool.get());
|
|
910
|
-
}
|
|
911
|
-
seq.record<kp::OpAlgoDispatch>(s_algo);
|
|
912
|
-
}
|
|
913
|
-
|
|
914
|
-
static void ggml_vk_mul_mat_f16(
|
|
915
|
-
kp::Sequence& seq,
|
|
916
|
-
const std::shared_ptr<kp::Tensor>& inA,
|
|
917
|
-
const std::shared_ptr<kp::Tensor>& inB,
|
|
918
|
-
const std::shared_ptr<kp::Tensor>& out,
|
|
919
|
-
uint32_t inAOff, uint32_t inBOff, uint32_t outOff,
|
|
920
|
-
int32_t ne00, int32_t ne01, int32_t ne02,
|
|
921
|
-
uint32_t nb00, uint32_t nb01, uint32_t nb02,
|
|
922
|
-
int32_t ne10, int32_t ne11, int32_t ne12, int32_t ne13,
|
|
923
|
-
uint32_t nb10, uint32_t nb11, uint32_t nb12,
|
|
924
|
-
int32_t ne0, int32_t ne1,
|
|
925
|
-
uint32_t r2, uint32_t r3
|
|
926
|
-
) {
|
|
927
|
-
const static auto spirv = getSpirvShader(kp::shader_data::op_mul_mat_f16_comp_spv,
|
|
928
|
-
kp::shader_data::op_mul_mat_f16_comp_spv_len);
|
|
929
|
-
|
|
930
|
-
struct PushConstants {
|
|
931
|
-
uint32_t inAOff, inBOff, outOff;
|
|
932
|
-
int32_t ne00, ne01, ne02;
|
|
933
|
-
uint32_t nb00, nb01, nb02;
|
|
934
|
-
int32_t ne10, ne11, ne12;
|
|
935
|
-
uint32_t nb10, nb11, nb12;
|
|
936
|
-
int32_t ne0, ne1;
|
|
937
|
-
uint32_t r2, r3;
|
|
938
|
-
} pushConsts {
|
|
939
|
-
safe_divide(inAOff, 2), safe_divide(inBOff, 4), safe_divide(outOff, 4),
|
|
940
|
-
ne00, ne01, ne02,
|
|
941
|
-
nb00, nb01, nb02,
|
|
942
|
-
ne10, ne11, ne12,
|
|
943
|
-
nb10, nb11, nb12,
|
|
944
|
-
ne0, ne1,
|
|
945
|
-
r2, r3
|
|
946
|
-
};
|
|
947
|
-
|
|
948
|
-
const unsigned ny = unsigned((ne11 + 4 - 1)/4);
|
|
949
|
-
|
|
950
|
-
std::shared_ptr<kp::Algorithm> s_algo = nullptr;
|
|
951
|
-
if (!komputeManager()->hasAlgorithm(__func__)) {
|
|
952
|
-
const uint32_t local_x = ggml_vk_current_device().subgroupSize * 2;
|
|
953
|
-
s_algo = komputeManager()->algorithm<uint32_t, PushConstants>(__func__, s_kompute_context->pool.get(), {inA, inB, out}, spirv, {unsigned(ne01), ny, unsigned(ne12*ne13)}, {local_x}, {pushConsts});
|
|
954
|
-
} else {
|
|
955
|
-
s_algo = komputeManager()->getAlgorithm(__func__);
|
|
956
|
-
s_algo->setTensors({inA, inB, out});
|
|
957
|
-
s_algo->setWorkgroup({unsigned(ne01), ny, unsigned(ne12*ne13)});
|
|
958
|
-
s_algo->setPushConstants<PushConstants>({pushConsts});
|
|
959
|
-
s_algo->updateDescriptors(s_kompute_context->pool.get());
|
|
960
|
-
}
|
|
961
|
-
seq.record<kp::OpAlgoDispatch>(s_algo);
|
|
962
|
-
}
|
|
963
|
-
|
|
964
|
-
static void ggml_vk_mul_mat_mat_f32(kp::Sequence& seq,
|
|
965
|
-
const std::shared_ptr<kp::Tensor>& inA,
|
|
966
|
-
const std::shared_ptr<kp::Tensor>& inB,
|
|
967
|
-
const std::shared_ptr<kp::Tensor>& out,
|
|
968
|
-
uint32_t inAOff, uint32_t inBOff, uint32_t outOff,
|
|
969
|
-
int32_t ne00, int32_t ne01, int32_t ne02,
|
|
970
|
-
uint32_t nb01, uint32_t nb02,
|
|
971
|
-
int32_t ne11, int32_t ne12,
|
|
972
|
-
uint32_t nb11, uint32_t nb12,
|
|
973
|
-
uint32_t nb1, uint32_t nb2) {
|
|
974
|
-
const static auto spirv = getSpirvShader(kp::shader_data::op_mul_mat_mat_f32_comp_spv,
|
|
975
|
-
kp::shader_data::op_mul_mat_mat_f32_comp_spv_len);
|
|
976
|
-
|
|
977
|
-
struct PushConstants {
|
|
978
|
-
uint32_t inAOff, inBOff, outOff;
|
|
979
|
-
int32_t ne00, ne01, ne02, ne11, ne12;
|
|
980
|
-
uint32_t nb01, nb02;
|
|
981
|
-
uint32_t nb11, nb12;
|
|
982
|
-
uint32_t nb1, nb2;
|
|
983
|
-
} pushConsts {
|
|
984
|
-
safe_divide(inAOff, 4), safe_divide(inBOff, 4), safe_divide(outOff, 4),
|
|
985
|
-
ne00, ne01, ne02, ne11, ne12,
|
|
986
|
-
nb01, nb02, nb11, nb12,
|
|
987
|
-
nb1, nb2
|
|
988
|
-
};
|
|
989
|
-
|
|
990
|
-
const uint32_t local_x = ggml_vk_current_device().subgroupSize;
|
|
991
|
-
std::shared_ptr<kp::Algorithm> s_algo = nullptr;
|
|
992
|
-
if (!komputeManager()->hasAlgorithm(__func__)) {
|
|
993
|
-
s_algo = komputeManager()->algorithm<uint32_t, PushConstants>(__func__, s_kompute_context->pool.get(),
|
|
994
|
-
{inA, inB, out}, spirv,
|
|
995
|
-
{unsigned(ne01),
|
|
996
|
-
unsigned(ne11),
|
|
997
|
-
unsigned(std::max(ne12, ne02))
|
|
998
|
-
},
|
|
999
|
-
{local_x},
|
|
1000
|
-
{pushConsts});
|
|
1001
|
-
} else {
|
|
1002
|
-
s_algo = komputeManager()->getAlgorithm(__func__);
|
|
1003
|
-
s_algo->setTensors({inA, inB, out});
|
|
1004
|
-
s_algo->setWorkgroup({unsigned(ne01),
|
|
1005
|
-
unsigned(ne11),
|
|
1006
|
-
unsigned(std::max(ne12, ne02)),
|
|
1007
|
-
});
|
|
1008
|
-
s_algo->setPushConstants<PushConstants>({pushConsts});
|
|
1009
|
-
s_algo->updateDescriptors(s_kompute_context->pool.get());
|
|
1010
|
-
}
|
|
1011
|
-
seq.record<kp::OpAlgoDispatch>(s_algo);
|
|
1012
|
-
}
|
|
1013
|
-
|
|
1014
|
-
static void ggml_vk_mul_mat_impl(
|
|
1015
|
-
const std::vector<uint32_t>& spirv, const char * suffix, uint32_t block_size, kp::Sequence& seq,
|
|
1016
|
-
const std::shared_ptr<kp::Tensor>& inA,
|
|
1017
|
-
const std::shared_ptr<kp::Tensor>& inB,
|
|
1018
|
-
const std::shared_ptr<kp::Tensor>& out,
|
|
1019
|
-
uint32_t inAOff, uint32_t inBOff, uint32_t outOff,
|
|
1020
|
-
int32_t ne00, int32_t ne01, int32_t ne02,
|
|
1021
|
-
int32_t ne10, int32_t ne11, int32_t ne12, int32_t ne13,
|
|
1022
|
-
int32_t ne0, int32_t ne1,
|
|
1023
|
-
uint32_t r2, uint32_t r3
|
|
1024
|
-
) {
|
|
1025
|
-
struct PushConstants {
|
|
1026
|
-
uint32_t inAOff, inBOff, outOff;
|
|
1027
|
-
int32_t ne00, ne01, ne02;
|
|
1028
|
-
int32_t ne10, ne12;
|
|
1029
|
-
int32_t ne0, ne1;
|
|
1030
|
-
uint32_t r2, r3;
|
|
1031
|
-
} pushConsts {
|
|
1032
|
-
safe_divide(inAOff, block_size), safe_divide(inBOff, 4), safe_divide(outOff, 4),
|
|
1033
|
-
ne00, ne01, ne02,
|
|
1034
|
-
ne10, ne12,
|
|
1035
|
-
ne0, ne1,
|
|
1036
|
-
r2, r3
|
|
1037
|
-
};
|
|
1038
|
-
|
|
1039
|
-
auto name = std::string(__func__) + "_" + suffix;
|
|
1040
|
-
std::shared_ptr<kp::Algorithm> s_algo = nullptr;
|
|
1041
|
-
if (!komputeManager()->hasAlgorithm(name)) {
|
|
1042
|
-
const uint32_t local_x = ggml_vk_current_device().subgroupSize * 2;
|
|
1043
|
-
s_algo = komputeManager()->algorithm<uint32_t, PushConstants>(name, s_kompute_context->pool.get(), {inA, inB, out}, spirv, {unsigned((ne01 + 7)/8), unsigned(ne11), unsigned(ne12*ne13)}, {local_x}, {pushConsts});
|
|
1044
|
-
} else {
|
|
1045
|
-
s_algo = komputeManager()->getAlgorithm(name);
|
|
1046
|
-
s_algo->setTensors({inA, inB, out});
|
|
1047
|
-
s_algo->setWorkgroup({unsigned((ne01 + 7)/8), unsigned(ne11), unsigned(ne12*ne13)});
|
|
1048
|
-
s_algo->setPushConstants<PushConstants>({pushConsts});
|
|
1049
|
-
s_algo->updateDescriptors(s_kompute_context->pool.get());
|
|
1050
|
-
}
|
|
1051
|
-
seq.record<kp::OpAlgoDispatch>(s_algo);
|
|
1052
|
-
}
|
|
1053
|
-
|
|
1054
|
-
template <typename... Args>
|
|
1055
|
-
static void ggml_vk_mul_mat_q4_0(Args&&... args) {
|
|
1056
|
-
const static auto spirv = getSpirvShader(kp::shader_data::op_mul_mat_q4_0_comp_spv,
|
|
1057
|
-
kp::shader_data::op_mul_mat_q4_0_comp_spv_len);
|
|
1058
|
-
|
|
1059
|
-
ggml_vk_mul_mat_impl(spirv, "q4_0", 1/*We access blocks unaligned*/, std::forward<Args>(args)...);
|
|
1060
|
-
}
|
|
1061
|
-
|
|
1062
|
-
template <typename... Args>
|
|
1063
|
-
static void ggml_vk_mul_mat_q4_1(Args&&... args) {
|
|
1064
|
-
const static auto spirv = getSpirvShader(kp::shader_data::op_mul_mat_q4_1_comp_spv,
|
|
1065
|
-
kp::shader_data::op_mul_mat_q4_1_comp_spv_len);
|
|
1066
|
-
|
|
1067
|
-
ggml_vk_mul_mat_impl(spirv, "q4_1", 1/*We access blocks unaligned*/, std::forward<Args>(args)...);
|
|
1068
|
-
}
|
|
1069
|
-
|
|
1070
|
-
template <typename... Args>
|
|
1071
|
-
static void ggml_vk_mul_mat_q8_0(Args&&... args) {
|
|
1072
|
-
const static auto spirv = getSpirvShader(kp::shader_data::op_mul_mat_q8_0_comp_spv,
|
|
1073
|
-
kp::shader_data::op_mul_mat_q8_0_comp_spv_len);
|
|
1074
|
-
|
|
1075
|
-
ggml_vk_mul_mat_impl(spirv, "q8_0", 1/*We access blocks unaligned*/, std::forward<Args>(args)...);
|
|
1076
|
-
}
|
|
1077
|
-
|
|
1078
|
-
static void ggml_vk_mul_mat_q6_k(
|
|
1079
|
-
kp::Sequence& seq,
|
|
1080
|
-
const std::shared_ptr<kp::Tensor>& inA,
|
|
1081
|
-
const std::shared_ptr<kp::Tensor>& inB,
|
|
1082
|
-
const std::shared_ptr<kp::Tensor>& out,
|
|
1083
|
-
uint32_t inAOff, uint32_t inBOff, uint32_t outOff,
|
|
1084
|
-
int32_t ne00, int32_t ne10, int32_t ne0, int32_t ne1,
|
|
1085
|
-
int32_t ne01, int32_t ne11, int32_t ne12, int32_t ne02
|
|
1086
|
-
) {
|
|
1087
|
-
const static auto spirv = getSpirvShader(kp::shader_data::op_mul_mat_q6_k_comp_spv,
|
|
1088
|
-
kp::shader_data::op_mul_mat_q6_k_comp_spv_len);
|
|
1089
|
-
|
|
1090
|
-
struct PushConstants {
|
|
1091
|
-
uint32_t inAOff, inBOff, outOff;
|
|
1092
|
-
int32_t ne00, ne10, ne0, ne1, ne01, gqa;
|
|
1093
|
-
} pushConsts {
|
|
1094
|
-
inAOff, safe_divide(inBOff, 4), safe_divide(outOff, 4),
|
|
1095
|
-
ne00, ne10, ne0, ne1, ne01, ne12/ne02
|
|
1096
|
-
};
|
|
1097
|
-
|
|
1098
|
-
std::shared_ptr<kp::Algorithm> s_algo = nullptr;
|
|
1099
|
-
if (!komputeManager()->hasAlgorithm(__func__)) {
|
|
1100
|
-
const uint32_t local_x = ggml_vk_current_device().subgroupSize * 2;
|
|
1101
|
-
s_algo = komputeManager()->algorithm<uint32_t, PushConstants>(__func__, s_kompute_context->pool.get(), {inA, inB, out}, spirv, {unsigned((ne01 + 1)/2), unsigned(ne11), unsigned(ne12)}, {local_x}, {pushConsts});
|
|
1102
|
-
} else {
|
|
1103
|
-
s_algo = komputeManager()->getAlgorithm(__func__);
|
|
1104
|
-
s_algo->setTensors({inA, inB, out});
|
|
1105
|
-
s_algo->setWorkgroup({unsigned((ne01 + 1)/2), unsigned(ne11), unsigned(ne12)});
|
|
1106
|
-
s_algo->setPushConstants<PushConstants>({pushConsts});
|
|
1107
|
-
s_algo->updateDescriptors(s_kompute_context->pool.get());
|
|
1108
|
-
}
|
|
1109
|
-
seq.record<kp::OpAlgoDispatch>(s_algo);
|
|
1110
|
-
}
|
|
1111
|
-
|
|
1112
|
-
static void ggml_vk_get_rows(
|
|
1113
|
-
const std::vector<uint32_t>& spirv,
|
|
1114
|
-
const char * suffix,
|
|
1115
|
-
unsigned element_size, unsigned qk,
|
|
1116
|
-
kp::Sequence& seq,
|
|
1117
|
-
const std::shared_ptr<kp::Tensor>& inA,
|
|
1118
|
-
const std::shared_ptr<kp::Tensor>& inB,
|
|
1119
|
-
const std::shared_ptr<kp::Tensor>& out,
|
|
1120
|
-
uint32_t inAOff, uint32_t inBOff, uint32_t outOff,
|
|
1121
|
-
int32_t ne00, int32_t nb01, int32_t nb1,
|
|
1122
|
-
uint32_t size
|
|
1123
|
-
) {
|
|
1124
|
-
GGML_ASSERT(nb01%element_size == 0);
|
|
1125
|
-
GGML_ASSERT(nb1%sizeof(float) == 0);
|
|
1126
|
-
if (qk) GGML_ASSERT(ne00%qk == 0);
|
|
1127
|
-
|
|
1128
|
-
struct PushConstants {
|
|
1129
|
-
uint32_t inAOff, inBOff, outOff;
|
|
1130
|
-
int32_t ne00, nb01, nb1;
|
|
1131
|
-
} pushConsts {
|
|
1132
|
-
safe_divide(inAOff, element_size), safe_divide(inBOff, 4), safe_divide(outOff, 4),
|
|
1133
|
-
ne00, nb01, nb1
|
|
1134
|
-
};
|
|
1135
|
-
|
|
1136
|
-
auto name = std::string(__func__) + "_" + suffix;
|
|
1137
|
-
std::shared_ptr<kp::Algorithm> s_algo = nullptr;
|
|
1138
|
-
if (!komputeManager()->hasAlgorithm(name)) {
|
|
1139
|
-
s_algo = komputeManager()->algorithm<float, PushConstants>(name, s_kompute_context->pool.get(), {inA, inB, out}, spirv, {size}, {}, {pushConsts});
|
|
1140
|
-
} else {
|
|
1141
|
-
s_algo = komputeManager()->getAlgorithm(name);
|
|
1142
|
-
s_algo->setTensors({inA, inB, out});
|
|
1143
|
-
s_algo->setWorkgroup({size});
|
|
1144
|
-
s_algo->setPushConstants<PushConstants>({pushConsts});
|
|
1145
|
-
s_algo->updateDescriptors(s_kompute_context->pool.get());
|
|
1146
|
-
}
|
|
1147
|
-
seq.record<kp::OpAlgoDispatch>(s_algo);
|
|
1148
|
-
}
|
|
1149
|
-
|
|
1150
|
-
template <typename... Args>
|
|
1151
|
-
static void ggml_vk_get_rows_f32(Args&&... args) {
|
|
1152
|
-
const static auto spirv = getSpirvShader(kp::shader_data::op_getrows_f32_comp_spv,
|
|
1153
|
-
kp::shader_data::op_getrows_f32_comp_spv_len);
|
|
1154
|
-
|
|
1155
|
-
ggml_vk_get_rows(spirv, "f32", sizeof(float), 0, std::forward<Args>(args)...);
|
|
1156
|
-
}
|
|
1157
|
-
|
|
1158
|
-
template <typename... Args>
|
|
1159
|
-
static void ggml_vk_get_rows_f16(Args&&... args) {
|
|
1160
|
-
const static auto spirv = getSpirvShader(kp::shader_data::op_getrows_f16_comp_spv,
|
|
1161
|
-
kp::shader_data::op_getrows_f16_comp_spv_len);
|
|
1162
|
-
|
|
1163
|
-
ggml_vk_get_rows(spirv, "f16", sizeof(half), 0, std::forward<Args>(args)...);
|
|
1164
|
-
}
|
|
1165
|
-
|
|
1166
|
-
template <typename... Args>
|
|
1167
|
-
static void ggml_vk_get_rows_q4_0(Args&&... args) {
|
|
1168
|
-
const static auto spirv = getSpirvShader(kp::shader_data::op_getrows_q4_0_comp_spv,
|
|
1169
|
-
kp::shader_data::op_getrows_q4_0_comp_spv_len);
|
|
1170
|
-
|
|
1171
|
-
ggml_vk_get_rows(spirv, "q4_0", 1/*We access blocks unaligned*/, QK4_0, std::forward<Args>(args)...);
|
|
1172
|
-
}
|
|
1173
|
-
|
|
1174
|
-
template <typename... Args>
|
|
1175
|
-
static void ggml_vk_get_rows_q4_1(Args&&... args) {
|
|
1176
|
-
const static auto spirv = getSpirvShader(kp::shader_data::op_getrows_q4_1_comp_spv,
|
|
1177
|
-
kp::shader_data::op_getrows_q4_1_comp_spv_len);
|
|
1178
|
-
|
|
1179
|
-
ggml_vk_get_rows(spirv, "q4_1", 1/*We access blocks unaligned*/, QK4_1, std::forward<Args>(args)...);
|
|
1180
|
-
}
|
|
1181
|
-
|
|
1182
|
-
template <typename... Args>
|
|
1183
|
-
static void ggml_vk_get_rows_q6_k(Args&&... args) {
|
|
1184
|
-
const static auto spirv = getSpirvShader(kp::shader_data::op_getrows_q6_k_comp_spv,
|
|
1185
|
-
kp::shader_data::op_getrows_q6_k_comp_spv_len);
|
|
1186
|
-
ggml_vk_get_rows(spirv, "q6_k", 1/*We access blocks unaligned*/, QK_NL, std::forward<Args>(args)...);
|
|
1187
|
-
}
|
|
1188
|
-
|
|
1189
|
-
static void ggml_vk_rope(
|
|
1190
|
-
kp::Sequence& seq,
|
|
1191
|
-
const std::shared_ptr<kp::Tensor>& inA,
|
|
1192
|
-
const std::shared_ptr<kp::Tensor>& inB,
|
|
1193
|
-
const std::shared_ptr<kp::Tensor>& out,
|
|
1194
|
-
uint32_t inAOff, uint32_t inBOff, uint32_t outOff,
|
|
1195
|
-
ggml_type src0t, int32_t n_dims, int32_t mode, int32_t n_ctx_orig,
|
|
1196
|
-
float freq_base, float freq_scale, float ext_factor, float attn_factor, float beta_fast, float beta_slow,
|
|
1197
|
-
int32_t ne01, int32_t ne02, int32_t ne03,
|
|
1198
|
-
uint32_t nb00, uint32_t nb01, uint32_t nb02, uint32_t nb03,
|
|
1199
|
-
int32_t ne0,
|
|
1200
|
-
uint32_t nb0, uint32_t nb1, uint32_t nb2, uint32_t nb3
|
|
1201
|
-
) {
|
|
1202
|
-
GGML_ASSERT(src0t == GGML_TYPE_F16 || src0t == GGML_TYPE_F32);
|
|
1203
|
-
|
|
1204
|
-
static const auto spirv_f16 = getSpirvShader(
|
|
1205
|
-
kp::shader_data::op_rope_f16_comp_spv, kp::shader_data::op_rope_f16_comp_spv_len
|
|
1206
|
-
);
|
|
1207
|
-
static const auto spirv_f32 = getSpirvShader(
|
|
1208
|
-
kp::shader_data::op_rope_f32_comp_spv, kp::shader_data::op_rope_f32_comp_spv_len
|
|
1209
|
-
);
|
|
1210
|
-
|
|
1211
|
-
int type_size = src0t == GGML_TYPE_F16 ? 2 : 4;
|
|
1212
|
-
|
|
1213
|
-
GGML_ASSERT(nb03 % type_size == 0);
|
|
1214
|
-
GGML_ASSERT(nb02 % type_size == 0);
|
|
1215
|
-
GGML_ASSERT(nb01 % type_size == 0);
|
|
1216
|
-
GGML_ASSERT(nb00 % type_size == 0);
|
|
1217
|
-
GGML_ASSERT(nb3 % type_size == 0);
|
|
1218
|
-
GGML_ASSERT(nb2 % type_size == 0);
|
|
1219
|
-
GGML_ASSERT(nb1 % type_size == 0);
|
|
1220
|
-
GGML_ASSERT(nb0 % type_size == 0);
|
|
1221
|
-
|
|
1222
|
-
struct PushConstants {
|
|
1223
|
-
uint32_t inAOff, inBOff, outOff;
|
|
1224
|
-
int32_t n_dims, mode, n_ctx_orig;
|
|
1225
|
-
float freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow;
|
|
1226
|
-
uint32_t nb00, nb01, nb02, nb03;
|
|
1227
|
-
int32_t ne0;
|
|
1228
|
-
uint32_t nb0, nb1, nb2, nb3;
|
|
1229
|
-
} pushConsts {
|
|
1230
|
-
safe_divide(inAOff, type_size), safe_divide(inBOff, 4), safe_divide(outOff, type_size),
|
|
1231
|
-
n_dims, mode, n_ctx_orig,
|
|
1232
|
-
freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow,
|
|
1233
|
-
nb00, nb01, nb02, nb03,
|
|
1234
|
-
ne0,
|
|
1235
|
-
nb0, nb1, nb2, nb3
|
|
1236
|
-
};
|
|
1237
|
-
|
|
1238
|
-
auto name = std::string(__func__) + (src0t == GGML_TYPE_F16 ? "_f16" : "_f32");
|
|
1239
|
-
std::shared_ptr<kp::Algorithm> s_algo = nullptr;
|
|
1240
|
-
if (!komputeManager()->hasAlgorithm(name)) {
|
|
1241
|
-
s_algo = komputeManager()->algorithm<float, PushConstants>(
|
|
1242
|
-
name, s_kompute_context->pool.get(), {inA, inB, out},
|
|
1243
|
-
src0t == GGML_TYPE_F16 ? spirv_f16 : spirv_f32,
|
|
1244
|
-
{unsigned(ne01), unsigned(ne02), unsigned(ne03)}, {}, {pushConsts}
|
|
1245
|
-
);
|
|
1246
|
-
} else {
|
|
1247
|
-
s_algo = komputeManager()->getAlgorithm(name);
|
|
1248
|
-
s_algo->setTensors({inA, inB, out});
|
|
1249
|
-
s_algo->setWorkgroup({unsigned(ne01), unsigned(ne02), unsigned(ne03)});
|
|
1250
|
-
s_algo->setPushConstants<PushConstants>({pushConsts});
|
|
1251
|
-
s_algo->updateDescriptors(s_kompute_context->pool.get());
|
|
1252
|
-
}
|
|
1253
|
-
seq.record<kp::OpAlgoDispatch>(s_algo);
|
|
1254
|
-
}
|
|
1255
|
-
|
|
1256
|
-
static void ggml_vk_cpy(
|
|
1257
|
-
const std::vector<uint32_t>& spirv,
|
|
1258
|
-
uint32_t in_element_size, uint32_t out_element_size,
|
|
1259
|
-
kp::Sequence& seq,
|
|
1260
|
-
const std::shared_ptr<kp::Tensor>& in,
|
|
1261
|
-
const std::shared_ptr<kp::Tensor>& out,
|
|
1262
|
-
uint32_t inOff, uint32_t outOff,
|
|
1263
|
-
int32_t ne00, int32_t ne01, int32_t ne02, int32_t ne03,
|
|
1264
|
-
uint32_t nb00, uint32_t nb01, uint32_t nb02, uint32_t nb03,
|
|
1265
|
-
int32_t ne0, int32_t ne1, int32_t ne2,
|
|
1266
|
-
uint32_t nb0, uint32_t nb1, uint32_t nb2, uint32_t nb3
|
|
1267
|
-
) {
|
|
1268
|
-
struct PushConstants {
|
|
1269
|
-
uint32_t inOff, outOff;
|
|
1270
|
-
int32_t ne00, ne01, ne02;
|
|
1271
|
-
uint32_t nb00, nb01, nb02, nb03;
|
|
1272
|
-
int32_t ne0, ne1, ne2;
|
|
1273
|
-
uint32_t nb0, nb1, nb2, nb3;
|
|
1274
|
-
} pushConsts {
|
|
1275
|
-
safe_divide(inOff, in_element_size), safe_divide(outOff, out_element_size),
|
|
1276
|
-
ne00, ne01, ne02,
|
|
1277
|
-
nb00, nb01, nb02, nb03,
|
|
1278
|
-
ne0, ne1, ne2,
|
|
1279
|
-
nb0, nb1, nb2, nb3
|
|
1280
|
-
};
|
|
1281
|
-
|
|
1282
|
-
std::string name = std::string(__func__)
|
|
1283
|
-
+ "_i_" + std::to_string(in_element_size)
|
|
1284
|
-
+ "_o_" + std::to_string(out_element_size);
|
|
1285
|
-
std::shared_ptr<kp::Algorithm> s_algo = nullptr;
|
|
1286
|
-
if (!komputeManager()->hasAlgorithm(name))
|
|
1287
|
-
s_algo = komputeManager()->algorithm<float, PushConstants>(name, s_kompute_context->pool.get(), {in, out}, spirv, {unsigned(ne01), unsigned(ne02), unsigned(ne03)}, {}, {pushConsts});
|
|
1288
|
-
else {
|
|
1289
|
-
s_algo = komputeManager()->getAlgorithm(name);
|
|
1290
|
-
s_algo->setTensors({in, out});
|
|
1291
|
-
s_algo->setWorkgroup({unsigned(ne01), unsigned(ne02), unsigned(ne03)});
|
|
1292
|
-
s_algo->setPushConstants<PushConstants>({pushConsts});
|
|
1293
|
-
s_algo->updateDescriptors(s_kompute_context->pool.get());
|
|
1294
|
-
}
|
|
1295
|
-
seq.record<kp::OpAlgoDispatch>(s_algo);
|
|
1296
|
-
}
|
|
1297
|
-
|
|
1298
|
-
template <typename... Args>
|
|
1299
|
-
static void ggml_vk_cpy_f32_f16(Args&&... args) {
|
|
1300
|
-
const static auto spirv = getSpirvShader(kp::shader_data::op_cpy_f32_f16_comp_spv,
|
|
1301
|
-
kp::shader_data::op_cpy_f32_f16_comp_spv_len);
|
|
1302
|
-
ggml_vk_cpy(spirv, 4, 2, std::forward<Args>(args)...);
|
|
1303
|
-
}
|
|
1304
|
-
|
|
1305
|
-
template <typename... Args>
|
|
1306
|
-
static void ggml_vk_cpy_f32_f32(Args&&... args) {
|
|
1307
|
-
const static auto spirv = getSpirvShader(kp::shader_data::op_cpy_f32_f32_comp_spv,
|
|
1308
|
-
kp::shader_data::op_cpy_f32_f32_comp_spv_len);
|
|
1309
|
-
ggml_vk_cpy(spirv, 4, 4, std::forward<Args>(args)...);
|
|
1310
|
-
}
|
|
1311
|
-
|
|
1312
|
-
template <typename... Args>
|
|
1313
|
-
static void ggml_vk_cpy_f16_f16(Args&&... args) {
|
|
1314
|
-
const static auto spirv = getSpirvShader(kp::shader_data::op_cpy_f16_f16_comp_spv,
|
|
1315
|
-
kp::shader_data::op_cpy_f16_f16_comp_spv_len);
|
|
1316
|
-
ggml_vk_cpy(spirv, 2, 2, std::forward<Args>(args)...);
|
|
1317
|
-
}
|
|
1318
|
-
|
|
1319
|
-
template <typename... Args>
|
|
1320
|
-
static void ggml_vk_cpy_f16_f32(Args&&... args) {
|
|
1321
|
-
const static auto spirv = getSpirvShader(kp::shader_data::op_cpy_f16_f32_comp_spv,
|
|
1322
|
-
kp::shader_data::op_cpy_f16_f32_comp_spv_len);
|
|
1323
|
-
ggml_vk_cpy(spirv, 2, 4, std::forward<Args>(args)...);
|
|
1324
|
-
}
|
|
1325
|
-
|
|
1326
|
-
static bool ggml_vk_supports_op(const struct ggml_tensor * op) {
|
|
1327
|
-
switch (op->type) {
|
|
1328
|
-
case GGML_TYPE_F16:
|
|
1329
|
-
case GGML_TYPE_F32:
|
|
1330
|
-
case GGML_TYPE_Q4_0:
|
|
1331
|
-
case GGML_TYPE_Q4_1:
|
|
1332
|
-
break;
|
|
1333
|
-
default:
|
|
1334
|
-
return false;
|
|
1335
|
-
}
|
|
1336
|
-
|
|
1337
|
-
switch (op->op) {
|
|
1338
|
-
case GGML_OP_UNARY:
|
|
1339
|
-
switch (ggml_get_unary_op(op)) {
|
|
1340
|
-
case GGML_UNARY_OP_RELU:
|
|
1341
|
-
case GGML_UNARY_OP_GELU:
|
|
1342
|
-
case GGML_UNARY_OP_SILU:
|
|
1343
|
-
return ggml_is_contiguous(op->src[0]);
|
|
1344
|
-
default:
|
|
1345
|
-
;
|
|
1346
|
-
}
|
|
1347
|
-
break;
|
|
1348
|
-
case GGML_OP_NONE:
|
|
1349
|
-
case GGML_OP_RESHAPE:
|
|
1350
|
-
case GGML_OP_VIEW:
|
|
1351
|
-
case GGML_OP_TRANSPOSE:
|
|
1352
|
-
case GGML_OP_PERMUTE:
|
|
1353
|
-
case GGML_OP_ADD:
|
|
1354
|
-
case GGML_OP_MUL:
|
|
1355
|
-
case GGML_OP_SCALE:
|
|
1356
|
-
case GGML_OP_SOFT_MAX:
|
|
1357
|
-
case GGML_OP_RMS_NORM:
|
|
1358
|
-
case GGML_OP_NORM:
|
|
1359
|
-
case GGML_OP_ROPE:
|
|
1360
|
-
return true;
|
|
1361
|
-
case GGML_OP_DUP:
|
|
1362
|
-
case GGML_OP_CPY:
|
|
1363
|
-
case GGML_OP_CONT:
|
|
1364
|
-
switch (op->src[0]->type) {
|
|
1365
|
-
case GGML_TYPE_F32:
|
|
1366
|
-
case GGML_TYPE_F16:
|
|
1367
|
-
break;
|
|
1368
|
-
default:
|
|
1369
|
-
return false;
|
|
1370
|
-
}
|
|
1371
|
-
switch (op->type) {
|
|
1372
|
-
case GGML_TYPE_F32:
|
|
1373
|
-
case GGML_TYPE_F16:
|
|
1374
|
-
break;
|
|
1375
|
-
default:
|
|
1376
|
-
return false;
|
|
1377
|
-
}
|
|
1378
|
-
return true;
|
|
1379
|
-
case GGML_OP_DIAG_MASK_INF:
|
|
1380
|
-
return op->ne[3] == 1;
|
|
1381
|
-
case GGML_OP_GET_ROWS:
|
|
1382
|
-
switch (op->src[0]->type) {
|
|
1383
|
-
case GGML_TYPE_F32:
|
|
1384
|
-
case GGML_TYPE_F16:
|
|
1385
|
-
case GGML_TYPE_Q4_0:
|
|
1386
|
-
case GGML_TYPE_Q4_1:
|
|
1387
|
-
case GGML_TYPE_Q6_K:
|
|
1388
|
-
return op->ne[2] == 1 && op->ne[3] == 1;
|
|
1389
|
-
default:
|
|
1390
|
-
;
|
|
1391
|
-
}
|
|
1392
|
-
return false;
|
|
1393
|
-
case GGML_OP_MUL_MAT:
|
|
1394
|
-
if (op->src[1]->type != GGML_TYPE_F32 || ggml_is_transposed(op->src[0]) || ggml_is_transposed(op->src[1]))
|
|
1395
|
-
return false;
|
|
1396
|
-
|
|
1397
|
-
switch (op->src[0]->type) {
|
|
1398
|
-
case GGML_TYPE_F32:
|
|
1399
|
-
case GGML_TYPE_Q6_K:
|
|
1400
|
-
return op->ne[3] == 1;
|
|
1401
|
-
case GGML_TYPE_F16:
|
|
1402
|
-
case GGML_TYPE_Q8_0:
|
|
1403
|
-
case GGML_TYPE_Q4_0:
|
|
1404
|
-
case GGML_TYPE_Q4_1:
|
|
1405
|
-
return true;
|
|
1406
|
-
default:
|
|
1407
|
-
;
|
|
1408
|
-
}
|
|
1409
|
-
default:
|
|
1410
|
-
;
|
|
1411
|
-
}
|
|
1412
|
-
return false;
|
|
1413
|
-
}
|
|
1414
|
-
|
|
1415
|
-
static void ggml_vk_graph_compute(struct ggml_kompute_context * ctx, struct ggml_cgraph * gf) {
|
|
1416
|
-
const int n_seq = 8;
|
|
1417
|
-
|
|
1418
|
-
// FIXME: Figure out if we can somehow optimize the size of the pool... right now we're setting
|
|
1419
|
-
// it to the size of the graph, but I think it can be made smaller?
|
|
1420
|
-
ggml_vk_allocate_descriptor_pool(ctx, gf->n_nodes);
|
|
1421
|
-
|
|
1422
|
-
std::vector<std::shared_ptr<kp::Sequence>> sequences(n_seq);
|
|
1423
|
-
|
|
1424
|
-
for (auto& sequence : sequences) {
|
|
1425
|
-
sequence = komputeManager()->sequence();
|
|
1426
|
-
}
|
|
1427
|
-
for (int seq_idx = 0; seq_idx < n_seq; ++seq_idx) {
|
|
1428
|
-
const int n_nodes_per_seq = (gf->n_nodes + n_seq - 1) / n_seq;
|
|
1429
|
-
|
|
1430
|
-
auto& seq = *sequences[seq_idx];
|
|
1431
|
-
|
|
1432
|
-
const int node_start = (seq_idx + 0) * n_nodes_per_seq;
|
|
1433
|
-
const int node_end = std::min((seq_idx == n_seq - 1) ? gf->n_nodes : (seq_idx + 1) * n_nodes_per_seq, gf->n_nodes);
|
|
1434
|
-
|
|
1435
|
-
bool any_commands_recorded = false;
|
|
1436
|
-
|
|
1437
|
-
for (int i = node_start; i < node_end; ++i) {
|
|
1438
|
-
struct ggml_tensor * src0 = gf->nodes[i]->src[0];
|
|
1439
|
-
struct ggml_tensor * src1 = gf->nodes[i]->src[1];
|
|
1440
|
-
struct ggml_tensor * src2 = gf->nodes[i]->src[2]; GGML_UNUSED(src2);
|
|
1441
|
-
struct ggml_tensor * dst = gf->nodes[i];
|
|
1442
|
-
GGML_ASSERT(dst->data != nullptr);
|
|
1443
|
-
|
|
1444
|
-
if (ggml_is_empty(dst)) {
|
|
1445
|
-
continue;
|
|
1446
|
-
}
|
|
1447
|
-
|
|
1448
|
-
switch (dst->op) {
|
|
1449
|
-
case GGML_OP_NONE:
|
|
1450
|
-
case GGML_OP_RESHAPE:
|
|
1451
|
-
case GGML_OP_VIEW:
|
|
1452
|
-
case GGML_OP_TRANSPOSE:
|
|
1453
|
-
case GGML_OP_PERMUTE:
|
|
1454
|
-
continue; // noop -> next node
|
|
1455
|
-
default:
|
|
1456
|
-
break;
|
|
1457
|
-
}
|
|
1458
|
-
|
|
1459
|
-
any_commands_recorded = true;
|
|
1460
|
-
|
|
1461
|
-
if (!ggml_vk_supports_op(dst)) {
|
|
1462
|
-
fprintf(stderr, "%s: error: unsupported op '%s'\n", __func__, ggml_op_desc(dst));
|
|
1463
|
-
GGML_ASSERT(!"unsupported op");
|
|
1464
|
-
}
|
|
1465
|
-
|
|
1466
|
-
const int32_t ne00 = src0 ? src0->ne[0] : 0;
|
|
1467
|
-
const int32_t ne01 = src0 ? src0->ne[1] : 0;
|
|
1468
|
-
const int32_t ne02 = src0 ? src0->ne[2] : 0;
|
|
1469
|
-
const int32_t ne03 = src0 ? src0->ne[3] : 0;
|
|
1470
|
-
|
|
1471
|
-
const uint32_t nb00 = src0 ? src0->nb[0] : 0;
|
|
1472
|
-
const uint32_t nb01 = src0 ? src0->nb[1] : 0;
|
|
1473
|
-
const uint32_t nb02 = src0 ? src0->nb[2] : 0;
|
|
1474
|
-
const uint32_t nb03 = src0 ? src0->nb[3] : 0;
|
|
1475
|
-
|
|
1476
|
-
const int32_t ne10 = src1 ? src1->ne[0] : 0;
|
|
1477
|
-
const int32_t ne11 = src1 ? src1->ne[1] : 0;
|
|
1478
|
-
const int32_t ne12 = src1 ? src1->ne[2] : 0;
|
|
1479
|
-
const int32_t ne13 = src1 ? src1->ne[3] : 0;
|
|
1480
|
-
|
|
1481
|
-
const uint32_t nb10 = src1 ? src1->nb[0] : 0;
|
|
1482
|
-
const uint32_t nb11 = src1 ? src1->nb[1] : 0;
|
|
1483
|
-
const uint32_t nb12 = src1 ? src1->nb[2] : 0;
|
|
1484
|
-
const uint32_t nb13 = src1 ? src1->nb[3] : 0;
|
|
1485
|
-
|
|
1486
|
-
const int32_t ne0 = dst ? dst->ne[0] : 0;
|
|
1487
|
-
const int32_t ne1 = dst ? dst->ne[1] : 0;
|
|
1488
|
-
const int32_t ne2 = dst ? dst->ne[2] : 0;
|
|
1489
|
-
// const int32_t ne3 = dst ? dst->ne[3] : 0;
|
|
1490
|
-
|
|
1491
|
-
const uint32_t nb0 = dst ? dst->nb[0] : 0;
|
|
1492
|
-
const uint32_t nb1 = dst ? dst->nb[1] : 0;
|
|
1493
|
-
const uint32_t nb2 = dst ? dst->nb[2] : 0;
|
|
1494
|
-
const uint32_t nb3 = dst ? dst->nb[3] : 0;
|
|
1495
|
-
|
|
1496
|
-
const enum ggml_type src0t = src0 ? src0->type : GGML_TYPE_COUNT;
|
|
1497
|
-
const enum ggml_type src1t = src1 ? src1->type : GGML_TYPE_COUNT;
|
|
1498
|
-
const enum ggml_type dstt = dst ? dst->type : GGML_TYPE_COUNT;
|
|
1499
|
-
|
|
1500
|
-
const static std::shared_ptr<kp::Tensor> nullTensor = nullptr;
|
|
1501
|
-
uint32_t off_src0 = 0;
|
|
1502
|
-
uint32_t off_src1 = 0;
|
|
1503
|
-
uint32_t off_dst = 0;
|
|
1504
|
-
const std::shared_ptr<kp::Tensor>& id_src0 = src0 ? ggml_vk_get_tensor(src0, &off_src0) : nullTensor;
|
|
1505
|
-
const std::shared_ptr<kp::Tensor>& id_src1 = src1 ? ggml_vk_get_tensor(src1, &off_src1) : nullTensor;
|
|
1506
|
-
const std::shared_ptr<kp::Tensor>& id_dst = dst ? ggml_vk_get_tensor(dst, &off_dst) : nullTensor;
|
|
1507
|
-
|
|
1508
|
-
switch (dst->op) {
|
|
1509
|
-
case GGML_OP_ADD:
|
|
1510
|
-
{
|
|
1511
|
-
if (ggml_nelements(src1) == ne10 && ggml_is_contiguous(src1) && ne00 % 4 == 0 && ne10 % 4 == 0) {
|
|
1512
|
-
// src1 is a row
|
|
1513
|
-
ggml_vk_addrow(seq, id_src0, id_src1, id_dst, off_src0, off_src1, off_dst, ggml_nelements(dst)/4, ne00);
|
|
1514
|
-
} else {
|
|
1515
|
-
ggml_vk_add(
|
|
1516
|
-
seq, id_src0, id_src1, id_dst, off_src0, off_src1, off_dst,
|
|
1517
|
-
ne00, ne01, ne02, ne03,
|
|
1518
|
-
nb00, nb01, nb02, nb03,
|
|
1519
|
-
ne10, ne11, ne12, ne13,
|
|
1520
|
-
nb10, nb11, nb12, nb13,
|
|
1521
|
-
ne0,
|
|
1522
|
-
nb0, nb1, nb2, nb3
|
|
1523
|
-
);
|
|
1524
|
-
}
|
|
1525
|
-
} break;
|
|
1526
|
-
case GGML_OP_MUL:
|
|
1527
|
-
{
|
|
1528
|
-
ggml_vk_mul(
|
|
1529
|
-
seq, id_src0, id_src1, id_dst, off_src0, off_src1, off_dst,
|
|
1530
|
-
ne00, ne01, ne02, ne03,
|
|
1531
|
-
nb00, nb01, nb02, nb03,
|
|
1532
|
-
ne10, ne11, ne12, ne13,
|
|
1533
|
-
nb10, nb11, nb12, nb13,
|
|
1534
|
-
ne0,
|
|
1535
|
-
nb0, nb1, nb2, nb3
|
|
1536
|
-
);
|
|
1537
|
-
} break;
|
|
1538
|
-
case GGML_OP_SCALE:
|
|
1539
|
-
{
|
|
1540
|
-
float scale; memcpy(&scale, dst->op_params, sizeof(float));
|
|
1541
|
-
|
|
1542
|
-
ggml_vk_scale(seq, id_src0, id_dst, off_src0, off_dst, ggml_nelements(dst), scale);
|
|
1543
|
-
} break;
|
|
1544
|
-
case GGML_OP_UNARY:
|
|
1545
|
-
{
|
|
1546
|
-
int64_t n = ggml_nelements(dst);
|
|
1547
|
-
GGML_ASSERT(n % 4 == 0);
|
|
1548
|
-
switch (ggml_get_unary_op(gf->nodes[i])) {
|
|
1549
|
-
case GGML_UNARY_OP_SILU:
|
|
1550
|
-
{
|
|
1551
|
-
ggml_vk_silu(seq, id_src0, id_dst, off_src0, off_dst, n/4);
|
|
1552
|
-
} break;
|
|
1553
|
-
case GGML_UNARY_OP_RELU:
|
|
1554
|
-
{
|
|
1555
|
-
ggml_vk_relu(seq, id_src0, id_dst, off_src0, off_dst, n/4);
|
|
1556
|
-
} break;
|
|
1557
|
-
case GGML_UNARY_OP_GELU:
|
|
1558
|
-
{
|
|
1559
|
-
GGML_ASSERT(n % 8 == 0);
|
|
1560
|
-
ggml_vk_gelu(seq, id_src0, id_dst, off_src0, off_dst, n/8);
|
|
1561
|
-
} break;
|
|
1562
|
-
default:
|
|
1563
|
-
{
|
|
1564
|
-
fprintf(stderr, "%s: node %3d, op = %8s not implemented\n", __func__, i, ggml_op_name(dst->op));
|
|
1565
|
-
GGML_ASSERT(false);
|
|
1566
|
-
}
|
|
1567
|
-
}
|
|
1568
|
-
} break;
|
|
1569
|
-
case GGML_OP_SOFT_MAX:
|
|
1570
|
-
{
|
|
1571
|
-
float scale;
|
|
1572
|
-
float max_bias;
|
|
1573
|
-
|
|
1574
|
-
memcpy(&scale, (float *)dst->op_params + 0, sizeof(float));
|
|
1575
|
-
memcpy(&max_bias, (float *)dst->op_params + 1, sizeof(float));
|
|
1576
|
-
|
|
1577
|
-
#pragma message("TODO: add ggml_vk_soft_max() F16 src1 support")
|
|
1578
|
-
#pragma message("ref: https://github.com/ggerganov/llama.cpp/pull/5021")
|
|
1579
|
-
GGML_ASSERT(!src1 || src1t == GGML_TYPE_F32);
|
|
1580
|
-
|
|
1581
|
-
#pragma message("TODO: add ALiBi support")
|
|
1582
|
-
#pragma message("ref: https://github.com/ggerganov/llama.cpp/pull/7192")
|
|
1583
|
-
GGML_ASSERT(max_bias == 0.0f);
|
|
1584
|
-
|
|
1585
|
-
ggml_vk_soft_max(seq, id_src0, id_src1, id_dst, off_src0, off_src1, off_dst, ne00, ne01, ne02, ne03, scale);
|
|
1586
|
-
} break;
|
|
1587
|
-
case GGML_OP_DIAG_MASK_INF:
|
|
1588
|
-
{
|
|
1589
|
-
const int n_past = ((int32_t *)(dst->op_params))[0];
|
|
1590
|
-
ggml_vk_diag_mask_inf(seq, id_src0, id_dst, off_src0, off_dst, n_past, ne00, ne01, ne02);
|
|
1591
|
-
} break;
|
|
1592
|
-
case GGML_OP_NORM:
|
|
1593
|
-
{
|
|
1594
|
-
float eps;
|
|
1595
|
-
memcpy(&eps, dst->op_params, sizeof(float));
|
|
1596
|
-
ggml_vk_norm(seq, id_src0, id_dst, off_src0, off_dst, ne00, nb01, ggml_nrows(src0), eps);
|
|
1597
|
-
} break;
|
|
1598
|
-
case GGML_OP_RMS_NORM:
|
|
1599
|
-
{
|
|
1600
|
-
GGML_ASSERT(ne00 % 4 == 0);
|
|
1601
|
-
|
|
1602
|
-
float eps;
|
|
1603
|
-
memcpy(&eps, dst->op_params, sizeof(float));
|
|
1604
|
-
ggml_vk_rms_norm(seq, id_src0, id_dst, off_src0, off_dst, ne00, nb01, ggml_nrows(src0), eps);
|
|
1605
|
-
} break;
|
|
1606
|
-
case GGML_OP_MUL_MAT:
|
|
1607
|
-
{
|
|
1608
|
-
GGML_ASSERT(ne00 == ne10);
|
|
1609
|
-
|
|
1610
|
-
GGML_ASSERT(ne12 % ne02 == 0);
|
|
1611
|
-
GGML_ASSERT(ne13 % ne03 == 0);
|
|
1612
|
-
|
|
1613
|
-
const uint32_t r2 = ne12/ne02;
|
|
1614
|
-
const uint32_t r3 = ne13/ne03;
|
|
1615
|
-
|
|
1616
|
-
if (src1t != GGML_TYPE_F32) {
|
|
1617
|
-
fprintf(stderr, "%s: %s: Unsupported src1 type: %u/%u\n", __func__, ggml_op_name(dst->op), src0t, src1t);
|
|
1618
|
-
goto not_implemented;
|
|
1619
|
-
}
|
|
1620
|
-
|
|
1621
|
-
if (ggml_is_transposed(src0) ||
|
|
1622
|
-
ggml_is_transposed(src1)) {
|
|
1623
|
-
fprintf(stderr, "%s: %s: matmul on tranposed tensor not supported: %u/%u\n", __func__, ggml_op_name(dst->op), src0t, src1t);
|
|
1624
|
-
goto not_implemented;
|
|
1625
|
-
}
|
|
1626
|
-
|
|
1627
|
-
switch (src0t) {
|
|
1628
|
-
case GGML_TYPE_F32:
|
|
1629
|
-
ggml_vk_mul_mat_mat_f32(
|
|
1630
|
-
seq, id_src0, id_src1, id_dst, off_src0, off_src1, off_dst,
|
|
1631
|
-
ne00, ne01, ne02, nb01, nb02, ne11, ne12, nb11, nb12, nb1, nb2
|
|
1632
|
-
);
|
|
1633
|
-
break;
|
|
1634
|
-
case GGML_TYPE_F16:
|
|
1635
|
-
ggml_vk_mul_mat_f16(
|
|
1636
|
-
seq, id_src0, id_src1, id_dst, off_src0, off_src1, off_dst,
|
|
1637
|
-
ne00, ne01, ne02, nb00, nb01, nb02, ne10, ne11, ne12, ne13, nb10, nb11, nb12,
|
|
1638
|
-
ne0, ne1, r2, r3
|
|
1639
|
-
);
|
|
1640
|
-
break;
|
|
1641
|
-
case GGML_TYPE_Q8_0:
|
|
1642
|
-
ggml_vk_mul_mat_q8_0(
|
|
1643
|
-
seq, id_src0, id_src1, id_dst, off_src0, off_src1, off_dst,
|
|
1644
|
-
ne00, ne01, ne02, ne10, ne11, ne12, ne13, ne0, ne1, r2, r3
|
|
1645
|
-
);
|
|
1646
|
-
break;
|
|
1647
|
-
case GGML_TYPE_Q4_0:
|
|
1648
|
-
ggml_vk_mul_mat_q4_0(
|
|
1649
|
-
seq, id_src0, id_src1, id_dst, off_src0, off_src1, off_dst,
|
|
1650
|
-
ne00, ne01, ne02, ne10, ne11, ne12, ne13, ne0, ne1, r2, r3
|
|
1651
|
-
);
|
|
1652
|
-
break;
|
|
1653
|
-
case GGML_TYPE_Q4_1:
|
|
1654
|
-
ggml_vk_mul_mat_q4_1(
|
|
1655
|
-
seq, id_src0, id_src1, id_dst, off_src0, off_src1, off_dst,
|
|
1656
|
-
ne00, ne01, ne02, ne10, ne11, ne12, ne13, ne0, ne1, r2, r3
|
|
1657
|
-
);
|
|
1658
|
-
break;
|
|
1659
|
-
case GGML_TYPE_Q6_K:
|
|
1660
|
-
ggml_vk_mul_mat_q6_k(
|
|
1661
|
-
seq, id_src0, id_src1, id_dst, off_src0, off_src1, off_dst,
|
|
1662
|
-
ne00, ne10, ne0, ne1, ne01, ne11, ne12, ne02
|
|
1663
|
-
);
|
|
1664
|
-
break;
|
|
1665
|
-
default: {
|
|
1666
|
-
fprintf(stderr, "%s: %s: Unsupported quantization: %u/%u\n", __func__, ggml_op_name(dst->op), src0t, src1t);
|
|
1667
|
-
goto not_implemented;
|
|
1668
|
-
}
|
|
1669
|
-
}
|
|
1670
|
-
|
|
1671
|
-
} break;
|
|
1672
|
-
case GGML_OP_GET_ROWS:
|
|
1673
|
-
{
|
|
1674
|
-
if (src0t == GGML_TYPE_F32) {
|
|
1675
|
-
ggml_vk_get_rows_f32(seq, id_src0, id_src1, id_dst, off_src0, off_src1, off_dst, ne00, nb01, nb1, ggml_nelements(src1));
|
|
1676
|
-
} else if (src0t == GGML_TYPE_F16) {
|
|
1677
|
-
ggml_vk_get_rows_f16(seq, id_src0, id_src1, id_dst, off_src0, off_src1, off_dst, ne00, nb01, nb1, ggml_nelements(src1));
|
|
1678
|
-
} else if (src0t == GGML_TYPE_Q4_0) {
|
|
1679
|
-
ggml_vk_get_rows_q4_0(seq, id_src0, id_src1, id_dst, off_src0, off_src1, off_dst, ne00, nb01, nb1, ggml_nelements(src1));
|
|
1680
|
-
} else if (src0t == GGML_TYPE_Q4_1) {
|
|
1681
|
-
ggml_vk_get_rows_q4_1(seq, id_src0, id_src1, id_dst, off_src0, off_src1, off_dst, ne00, nb01, nb1, ggml_nelements(src1));
|
|
1682
|
-
} else if (src0t == GGML_TYPE_Q6_K) {
|
|
1683
|
-
ggml_vk_get_rows_q6_k(seq, id_src0, id_src1, id_dst, off_src0, off_src1, off_dst, ne00, nb01, nb1, ggml_nelements(src1));
|
|
1684
|
-
} else {
|
|
1685
|
-
fprintf(stderr, "%s: %s: Unsupported quantization: %u\n", __func__, ggml_op_name(dst->op), src0t);
|
|
1686
|
-
goto not_implemented;
|
|
1687
|
-
}
|
|
1688
|
-
} break;
|
|
1689
|
-
case GGML_OP_ROPE:
|
|
1690
|
-
{
|
|
1691
|
-
#pragma message("TODO: implement phi3 frequency factors support")
|
|
1692
|
-
#pragma message(" https://github.com/ggerganov/llama.cpp/pull/7225")
|
|
1693
|
-
GGML_ASSERT(dst->src[2] == nullptr && "phi3 frequency factors not implemented yet");
|
|
1694
|
-
|
|
1695
|
-
#pragma message("TODO: update rope NORM mode to match NEOX mode")
|
|
1696
|
-
#pragma message(" https://github.com/ggerganov/llama.cpp/pull/7634")
|
|
1697
|
-
|
|
1698
|
-
GGML_ASSERT(ne10 == ne02);
|
|
1699
|
-
GGML_ASSERT(src0t == dstt);
|
|
1700
|
-
// const int n_past = ((int32_t *) dst->op_params)[0];
|
|
1701
|
-
const int n_dims = ((int32_t *) dst->op_params)[1];
|
|
1702
|
-
const int mode = ((int32_t *) dst->op_params)[2];
|
|
1703
|
-
// skip 3, n_ctx used in GLM RoPE, unimplemented in Vulkan
|
|
1704
|
-
const int n_ctx_orig = ((int32_t *) dst->op_params)[4];
|
|
1705
|
-
|
|
1706
|
-
float freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow;
|
|
1707
|
-
memcpy(&freq_base, (int32_t *) dst->op_params + 5, sizeof(float));
|
|
1708
|
-
memcpy(&freq_scale, (int32_t *) dst->op_params + 6, sizeof(float));
|
|
1709
|
-
memcpy(&ext_factor, (int32_t *) dst->op_params + 7, sizeof(float));
|
|
1710
|
-
memcpy(&attn_factor, (int32_t *) dst->op_params + 8, sizeof(float));
|
|
1711
|
-
memcpy(&beta_fast, (int32_t *) dst->op_params + 9, sizeof(float));
|
|
1712
|
-
memcpy(&beta_slow, (int32_t *) dst->op_params + 10, sizeof(float));
|
|
1713
|
-
ggml_vk_rope(
|
|
1714
|
-
seq, id_src0, id_src1, id_dst, off_src0, off_src1, off_dst, src0t, n_dims, mode, n_ctx_orig,
|
|
1715
|
-
freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow,
|
|
1716
|
-
ne01, ne02, ne03, nb00, nb01, nb02, nb03, ne0, nb0, nb1, nb2, nb3
|
|
1717
|
-
);
|
|
1718
|
-
} break;
|
|
1719
|
-
case GGML_OP_DUP:
|
|
1720
|
-
case GGML_OP_CPY:
|
|
1721
|
-
case GGML_OP_CONT:
|
|
1722
|
-
{
|
|
1723
|
-
switch (src0t) {
|
|
1724
|
-
case GGML_TYPE_F32:
|
|
1725
|
-
{
|
|
1726
|
-
switch (dstt) {
|
|
1727
|
-
case GGML_TYPE_F16: ggml_vk_cpy_f32_f16(seq, id_src0, id_dst, off_src0, off_dst, ne00, ne01, ne02, ne03, nb00, nb01, nb02, nb03, ne0, ne1, ne2, nb0, nb1, nb2, nb3); break;
|
|
1728
|
-
case GGML_TYPE_F32: ggml_vk_cpy_f32_f32(seq, id_src0, id_dst, off_src0, off_dst, ne00, ne01, ne02, ne03, nb00, nb01, nb02, nb03, ne0, ne1, ne2, nb0, nb1, nb2, nb3); break;
|
|
1729
|
-
default: goto not_implemented;
|
|
1730
|
-
}
|
|
1731
|
-
} break;
|
|
1732
|
-
case GGML_TYPE_F16:
|
|
1733
|
-
{
|
|
1734
|
-
switch (dstt) {
|
|
1735
|
-
case GGML_TYPE_F16: ggml_vk_cpy_f16_f16(seq, id_src0, id_dst, off_src0, off_dst, ne00, ne01, ne02, ne03, nb00, nb01, nb02, nb03, ne0, ne1, ne2, nb0, nb1, nb2, nb3); break;
|
|
1736
|
-
case GGML_TYPE_F32: ggml_vk_cpy_f16_f32(seq, id_src0, id_dst, off_src0, off_dst, ne00, ne01, ne02, ne03, nb00, nb01, nb02, nb03, ne0, ne1, ne2, nb0, nb1, nb2, nb3); break;
|
|
1737
|
-
default: goto not_implemented;
|
|
1738
|
-
} break;
|
|
1739
|
-
default: goto not_implemented;
|
|
1740
|
-
}
|
|
1741
|
-
}
|
|
1742
|
-
} break;
|
|
1743
|
-
default: goto not_implemented;
|
|
1744
|
-
}
|
|
1745
|
-
continue;
|
|
1746
|
-
not_implemented: {}
|
|
1747
|
-
fprintf(stderr, "%s: node %3d, op = %8s not implemented\n", __func__, i, ggml_op_name(dst->op));
|
|
1748
|
-
//GGML_ASSERT(false);
|
|
1749
|
-
}
|
|
1750
|
-
|
|
1751
|
-
// Evaluate sequence
|
|
1752
|
-
if (any_commands_recorded) {
|
|
1753
|
-
seq.evalAsync();
|
|
1754
|
-
}
|
|
1755
|
-
}
|
|
1756
|
-
|
|
1757
|
-
// Wait for all sequences to finish
|
|
1758
|
-
for (auto& sequence : sequences) {
|
|
1759
|
-
if (sequence->isRunning())
|
|
1760
|
-
sequence->evalAwait();
|
|
1761
|
-
}
|
|
1762
|
-
|
|
1763
|
-
ggml_vk_free_descriptor_pool(ctx);
|
|
1764
|
-
}
|
|
1765
|
-
|
|
1766
|
-
template<>
|
|
1767
|
-
kp::Tensor::TensorDataTypes
|
|
1768
|
-
kp::TensorT<half>::dataType()
|
|
1769
|
-
{
|
|
1770
|
-
return TensorDataTypes::eFloat;
|
|
1771
|
-
}
|
|
1772
|
-
|
|
1773
|
-
template<>
|
|
1774
|
-
kp::Tensor::TensorDataTypes
|
|
1775
|
-
kp::TensorT<uint8_t>::dataType()
|
|
1776
|
-
{
|
|
1777
|
-
return TensorDataTypes::eUnsignedInt;
|
|
1778
|
-
}
|
|
1779
|
-
|
|
1780
|
-
////////////////////////////////////////////////////////////////////////////////
|
|
1781
|
-
|
|
1782
|
-
// backend interface
|
|
1783
|
-
|
|
1784
|
-
struct ggml_backend_kompute_buffer_type_context {
|
|
1785
|
-
int device;
|
|
1786
|
-
int device_ref = 0;
|
|
1787
|
-
uint64_t buffer_alignment;
|
|
1788
|
-
uint64_t max_alloc;
|
|
1789
|
-
std::string name;
|
|
1790
|
-
|
|
1791
|
-
ggml_backend_kompute_buffer_type_context(int device, uint64_t buffer_alignment, uint64_t max_alloc)
|
|
1792
|
-
: device(device), buffer_alignment(buffer_alignment), max_alloc(max_alloc), name(ggml_kompute_format_name(device)) {}
|
|
1793
|
-
};
|
|
1794
|
-
|
|
1795
|
-
static void ggml_backend_kompute_device_ref(ggml_backend_buffer_type_t buft) {
|
|
1796
|
-
auto * ctx = static_cast<ggml_backend_kompute_buffer_type_context *>(buft->context);
|
|
1797
|
-
|
|
1798
|
-
if (!ctx->device_ref) {
|
|
1799
|
-
komputeManager()->initializeDevice(
|
|
1800
|
-
ctx->device, {}, {
|
|
1801
|
-
"VK_KHR_shader_float16_int8", "VK_KHR_8bit_storage",
|
|
1802
|
-
"VK_KHR_16bit_storage", "VK_KHR_shader_non_semantic_info"
|
|
1803
|
-
}
|
|
1804
|
-
);
|
|
1805
|
-
}
|
|
1806
|
-
|
|
1807
|
-
assert(ggml_vk_has_device());
|
|
1808
|
-
ctx->device_ref++;
|
|
1809
|
-
}
|
|
1810
|
-
|
|
1811
|
-
static void ggml_backend_kompute_device_unref(ggml_backend_buffer_type_t buft) {
|
|
1812
|
-
auto * ctx = static_cast<ggml_backend_kompute_buffer_type_context *>(buft->context);
|
|
1813
|
-
|
|
1814
|
-
assert(ctx->device_ref > 0);
|
|
1815
|
-
|
|
1816
|
-
ctx->device_ref--;
|
|
1817
|
-
|
|
1818
|
-
if (!ctx->device_ref) {
|
|
1819
|
-
komputeManager.destroy();
|
|
1820
|
-
}
|
|
1821
|
-
}
|
|
1822
|
-
|
|
1823
|
-
static const char * ggml_backend_kompute_buffer_get_name(ggml_backend_buffer_t buffer) {
|
|
1824
|
-
auto * ctx = static_cast<ggml_backend_kompute_buffer_type_context *>(buffer->buft->context);
|
|
1825
|
-
return ctx->name.c_str();
|
|
1826
|
-
}
|
|
1827
|
-
|
|
1828
|
-
static void ggml_backend_kompute_buffer_free_buffer(ggml_backend_buffer_t buffer) {
|
|
1829
|
-
auto * memory = (ggml_vk_memory *)buffer->context;
|
|
1830
|
-
if (ggml_vk_has_device()) {
|
|
1831
|
-
ggml_vk_free_memory(*memory);
|
|
1832
|
-
}
|
|
1833
|
-
delete memory;
|
|
1834
|
-
}
|
|
1835
|
-
|
|
1836
|
-
static void * ggml_backend_kompute_buffer_get_base(ggml_backend_buffer_t buffer) {
|
|
1837
|
-
return ((ggml_vk_memory *)buffer->context)->data;
|
|
1838
|
-
}
|
|
1839
|
-
|
|
1840
|
-
static void ggml_backend_kompute_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) {
|
|
1841
|
-
GGML_UNUSED(buffer);
|
|
1842
|
-
|
|
1843
|
-
const auto res = ggml_vk_get_tensor(tensor);
|
|
1844
|
-
GGML_ASSERT(res);
|
|
1845
|
-
|
|
1846
|
-
memcpy((char *)tensor->data + offset, data, size);
|
|
1847
|
-
|
|
1848
|
-
komputeManager()->sequence()->eval<kp::OpTensorSyncDevice>({res});
|
|
1849
|
-
}
|
|
1850
|
-
|
|
1851
|
-
static void ggml_backend_kompute_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) {
|
|
1852
|
-
GGML_UNUSED(buffer);
|
|
1853
|
-
|
|
1854
|
-
const auto res = ggml_vk_get_tensor(tensor);
|
|
1855
|
-
GGML_ASSERT(res);
|
|
1856
|
-
|
|
1857
|
-
komputeManager()->sequence()->eval<kp::OpTensorSyncLocal>({res});
|
|
1858
|
-
|
|
1859
|
-
memcpy(data, (const char *)tensor->data + offset, size);
|
|
1860
|
-
}
|
|
1861
|
-
|
|
1862
|
-
static void ggml_backend_kompute_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {
|
|
1863
|
-
auto * memory = (ggml_vk_memory *)buffer->context;
|
|
1864
|
-
memset(memory->data, value, buffer->size);
|
|
1865
|
-
|
|
1866
|
-
if (memory->stagingBuffer)
|
|
1867
|
-
komputeManager()->sequence()->eval<kp::OpBufferSyncDevice>(memory->primaryBuffer, memory->stagingBuffer, memory->size);
|
|
1868
|
-
}
|
|
1869
|
-
|
|
1870
|
-
static ggml_backend_buffer_i ggml_backend_kompute_buffer_i = {
|
|
1871
|
-
/* .get_name = */ ggml_backend_kompute_buffer_get_name,
|
|
1872
|
-
/* .free_buffer = */ ggml_backend_kompute_buffer_free_buffer,
|
|
1873
|
-
/* .get_base = */ ggml_backend_kompute_buffer_get_base,
|
|
1874
|
-
/* .init_tensor = */ NULL,
|
|
1875
|
-
/* .set_tensor = */ ggml_backend_kompute_buffer_set_tensor,
|
|
1876
|
-
/* .get_tensor = */ ggml_backend_kompute_buffer_get_tensor,
|
|
1877
|
-
/* .cpy_tensor = */ NULL,
|
|
1878
|
-
/* .clear = */ ggml_backend_kompute_buffer_clear,
|
|
1879
|
-
/* .reset = */ NULL,
|
|
1880
|
-
};
|
|
1881
|
-
|
|
1882
|
-
// default buffer type
|
|
1883
|
-
|
|
1884
|
-
static const char * ggml_backend_kompute_buffer_type_get_name(ggml_backend_buffer_type_t buft) {
|
|
1885
|
-
auto * ctx = static_cast<ggml_backend_kompute_buffer_type_context *>(buft->context);
|
|
1886
|
-
return ctx->name.c_str();
|
|
1887
|
-
}
|
|
1888
|
-
|
|
1889
|
-
static ggml_backend_buffer_t ggml_backend_kompute_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {
|
|
1890
|
-
ggml_backend_kompute_device_ref(buft);
|
|
1891
|
-
auto * ctx = new ggml_vk_memory(ggml_vk_allocate(size));
|
|
1892
|
-
return ggml_backend_buffer_init(buft, ggml_backend_kompute_buffer_i, ctx, size);
|
|
1893
|
-
}
|
|
1894
|
-
|
|
1895
|
-
static size_t ggml_backend_kompute_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) {
|
|
1896
|
-
auto * ctx = static_cast<ggml_backend_kompute_buffer_type_context *>(buft->context);
|
|
1897
|
-
return ctx->buffer_alignment;
|
|
1898
|
-
}
|
|
1899
|
-
|
|
1900
|
-
static size_t ggml_backend_vk_buffer_type_get_max_size(ggml_backend_buffer_type_t buft) {
|
|
1901
|
-
auto * ctx = static_cast<ggml_backend_kompute_buffer_type_context *>(buft->context);
|
|
1902
|
-
return ctx->max_alloc;
|
|
1903
|
-
}
|
|
1904
|
-
|
|
1905
|
-
static ggml_backend_buffer_type_i ggml_backend_kompute_buffer_type_interface = {
|
|
1906
|
-
/* .get_name = */ ggml_backend_kompute_buffer_type_get_name,
|
|
1907
|
-
/* .alloc_buffer = */ ggml_backend_kompute_buffer_type_alloc_buffer,
|
|
1908
|
-
/* .get_alignment = */ ggml_backend_kompute_buffer_type_get_alignment,
|
|
1909
|
-
/* .get_max_size = */ ggml_backend_vk_buffer_type_get_max_size,
|
|
1910
|
-
/* .get_alloc_size = */ NULL, // defaults to ggml_nbytes
|
|
1911
|
-
/* .is_host = */ NULL,
|
|
1912
|
-
};
|
|
1913
|
-
|
|
1914
|
-
ggml_backend_buffer_type_t ggml_backend_kompute_buffer_type(int device) {
|
|
1915
|
-
static std::vector<ggml_backend_buffer_type> bufts = []() {
|
|
1916
|
-
std::vector<ggml_backend_buffer_type> vec;
|
|
1917
|
-
auto devices = ggml_vk_available_devices_internal(0);
|
|
1918
|
-
vec.reserve(devices.size());
|
|
1919
|
-
|
|
1920
|
-
for (const auto & dev : devices) {
|
|
1921
|
-
vec.push_back({
|
|
1922
|
-
/* .iface = */ ggml_backend_kompute_buffer_type_interface,
|
|
1923
|
-
/* .context = */ new ggml_backend_kompute_buffer_type_context(dev.index, dev.bufferAlignment, dev.maxAlloc)
|
|
1924
|
-
});
|
|
1925
|
-
}
|
|
1926
|
-
return vec;
|
|
1927
|
-
}();
|
|
1928
|
-
|
|
1929
|
-
auto it = std::find_if(bufts.begin(), bufts.end(), [device](const ggml_backend_buffer_type & t) {
|
|
1930
|
-
return device == static_cast<ggml_backend_kompute_buffer_type_context *>(t.context)->device;
|
|
1931
|
-
});
|
|
1932
|
-
return it < bufts.end() ? &*it : nullptr;
|
|
1933
|
-
}
|
|
1934
|
-
|
|
1935
|
-
// backend
|
|
1936
|
-
|
|
1937
|
-
static const char * ggml_backend_kompute_name(ggml_backend_t backend) {
|
|
1938
|
-
auto * ctx = static_cast<ggml_kompute_context *>(backend->context);
|
|
1939
|
-
return ctx->name.c_str();
|
|
1940
|
-
}
|
|
1941
|
-
|
|
1942
|
-
static void ggml_backend_kompute_free(ggml_backend_t backend) {
|
|
1943
|
-
auto * ctx = static_cast<ggml_kompute_context *>(backend->context);
|
|
1944
|
-
|
|
1945
|
-
assert(ctx == s_kompute_context);
|
|
1946
|
-
s_kompute_context = nullptr;
|
|
1947
|
-
if (ctx != nullptr) {
|
|
1948
|
-
delete ctx;
|
|
1949
|
-
}
|
|
1950
|
-
|
|
1951
|
-
delete backend;
|
|
1952
|
-
}
|
|
1953
|
-
|
|
1954
|
-
static ggml_backend_buffer_type_t ggml_backend_kompute_get_default_buffer_type(ggml_backend_t backend) {
|
|
1955
|
-
auto * ctx = static_cast<ggml_kompute_context *>(backend->context);
|
|
1956
|
-
return ggml_backend_kompute_buffer_type(ctx->device);
|
|
1957
|
-
}
|
|
1958
|
-
|
|
1959
|
-
static ggml_status ggml_backend_kompute_graph_compute(ggml_backend_t backend, struct ggml_cgraph * cgraph) {
|
|
1960
|
-
auto * ctx = static_cast<ggml_kompute_context *>(backend->context);
|
|
1961
|
-
ggml_vk_graph_compute(ctx, cgraph);
|
|
1962
|
-
return GGML_STATUS_SUCCESS;
|
|
1963
|
-
}
|
|
1964
|
-
|
|
1965
|
-
static bool ggml_backend_kompute_supports_op(ggml_backend_t backend, const struct ggml_tensor * op) {
|
|
1966
|
-
GGML_UNUSED(backend);
|
|
1967
|
-
return ggml_vk_supports_op(op);
|
|
1968
|
-
}
|
|
1969
|
-
|
|
1970
|
-
static bool ggml_backend_kompute_supports_buft(ggml_backend_t backend, ggml_backend_buffer_type_t buft) {
|
|
1971
|
-
GGML_UNUSED(backend);
|
|
1972
|
-
return buft->iface.get_name == ggml_backend_kompute_buffer_type_get_name;
|
|
1973
|
-
}
|
|
1974
|
-
|
|
1975
|
-
static struct ggml_backend_i kompute_backend_i = {
|
|
1976
|
-
/* .get_name = */ ggml_backend_kompute_name,
|
|
1977
|
-
/* .free = */ ggml_backend_kompute_free,
|
|
1978
|
-
/* .get_default_buffer_type = */ ggml_backend_kompute_get_default_buffer_type,
|
|
1979
|
-
/* .set_tensor_async = */ NULL,
|
|
1980
|
-
/* .get_tensor_async = */ NULL,
|
|
1981
|
-
/* .cpy_tensor_async = */ NULL,
|
|
1982
|
-
/* .synchronize = */ NULL,
|
|
1983
|
-
/* .graph_plan_create = */ NULL,
|
|
1984
|
-
/* .graph_plan_free = */ NULL,
|
|
1985
|
-
/* .graph_plan_update = */ NULL,
|
|
1986
|
-
/* .graph_plan_compute = */ NULL,
|
|
1987
|
-
/* .graph_compute = */ ggml_backend_kompute_graph_compute,
|
|
1988
|
-
/* .supports_op = */ ggml_backend_kompute_supports_op,
|
|
1989
|
-
/* .supports_buft = */ ggml_backend_kompute_supports_buft,
|
|
1990
|
-
/* .offload_op = */ NULL,
|
|
1991
|
-
/* .event_new = */ NULL,
|
|
1992
|
-
/* .event_free = */ NULL,
|
|
1993
|
-
/* .event_record = */ NULL,
|
|
1994
|
-
/* .event_wait = */ NULL,
|
|
1995
|
-
/* .event_synchronize = */ NULL,
|
|
1996
|
-
};
|
|
1997
|
-
|
|
1998
|
-
static ggml_guid_t ggml_backend_kompute_guid() {
|
|
1999
|
-
static ggml_guid guid = { 0x7b, 0x57, 0xdc, 0xaf, 0xde, 0x12, 0x1d, 0x49, 0xfb, 0x35, 0xfa, 0x9b, 0x18, 0x31, 0x1d, 0xca };
|
|
2000
|
-
return &guid;
|
|
2001
|
-
}
|
|
2002
|
-
|
|
2003
|
-
ggml_backend_t ggml_backend_kompute_init(int device) {
|
|
2004
|
-
GGML_ASSERT(s_kompute_context == nullptr);
|
|
2005
|
-
s_kompute_context = new ggml_kompute_context(device);
|
|
2006
|
-
|
|
2007
|
-
ggml_backend_t kompute_backend = new ggml_backend {
|
|
2008
|
-
/* .guid = */ ggml_backend_kompute_guid(),
|
|
2009
|
-
/* .interface = */ kompute_backend_i,
|
|
2010
|
-
/* .context = */ s_kompute_context,
|
|
2011
|
-
};
|
|
2012
|
-
|
|
2013
|
-
return kompute_backend;
|
|
2014
|
-
}
|
|
2015
|
-
|
|
2016
|
-
bool ggml_backend_is_kompute(ggml_backend_t backend) {
|
|
2017
|
-
return backend != NULL && ggml_guid_matches(backend->guid, ggml_backend_kompute_guid());
|
|
2018
|
-
}
|
|
2019
|
-
|
|
2020
|
-
static ggml_backend_t ggml_backend_reg_kompute_init(const char * params, void * user_data) {
|
|
2021
|
-
GGML_UNUSED(params);
|
|
2022
|
-
return ggml_backend_kompute_init(intptr_t(user_data));
|
|
2023
|
-
}
|
|
2024
|
-
|
|
2025
|
-
extern "C" int ggml_backend_kompute_reg_devices();
|
|
2026
|
-
|
|
2027
|
-
int ggml_backend_kompute_reg_devices() {
|
|
2028
|
-
auto devices = ggml_vk_available_devices_internal(0);
|
|
2029
|
-
for (const auto & device : devices) {
|
|
2030
|
-
ggml_backend_register(
|
|
2031
|
-
ggml_kompute_format_name(device.index).c_str(),
|
|
2032
|
-
ggml_backend_reg_kompute_init,
|
|
2033
|
-
ggml_backend_kompute_buffer_type(device.index),
|
|
2034
|
-
reinterpret_cast<void *>(intptr_t(device.index))
|
|
2035
|
-
);
|
|
2036
|
-
}
|
|
2037
|
-
return devices.size();
|
|
2038
|
-
}
|