llama_cpp 0.12.1 → 0.12.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (25) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +8 -0
  3. data/ext/llama_cpp/llama_cpp.cpp +64 -0
  4. data/lib/llama_cpp/version.rb +2 -2
  5. data/sig/llama_cpp.rbs +7 -0
  6. data/vendor/tmp/llama.cpp/Makefile +0 -9
  7. data/vendor/tmp/llama.cpp/ggml-alloc.c +28 -6
  8. data/vendor/tmp/llama.cpp/ggml-alloc.h +3 -1
  9. data/vendor/tmp/llama.cpp/ggml-backend-impl.h +36 -36
  10. data/vendor/tmp/llama.cpp/ggml-backend.c +510 -263
  11. data/vendor/tmp/llama.cpp/ggml-backend.h +42 -32
  12. data/vendor/tmp/llama.cpp/ggml-cuda.cu +692 -476
  13. data/vendor/tmp/llama.cpp/ggml-cuda.h +18 -30
  14. data/vendor/tmp/llama.cpp/ggml-impl.h +2 -0
  15. data/vendor/tmp/llama.cpp/ggml-metal.h +4 -56
  16. data/vendor/tmp/llama.cpp/ggml-metal.m +1860 -2073
  17. data/vendor/tmp/llama.cpp/ggml-opencl.cpp +321 -14
  18. data/vendor/tmp/llama.cpp/ggml-opencl.h +13 -3
  19. data/vendor/tmp/llama.cpp/ggml-quants.c +1638 -134
  20. data/vendor/tmp/llama.cpp/ggml-quants.h +15 -4
  21. data/vendor/tmp/llama.cpp/ggml.c +142 -64
  22. data/vendor/tmp/llama.cpp/ggml.h +47 -29
  23. data/vendor/tmp/llama.cpp/llama.cpp +1219 -1615
  24. data/vendor/tmp/llama.cpp/llama.h +30 -8
  25. metadata +2 -2
data/vendor/tmp/llama.cpp/ggml-metal.m CHANGED
@@ -24,7 +24,7 @@
24
24
 
25
25
  #define UNUSED(x) (void)(x)
26
26
 
27
- #define GGML_MAX_CONCUR (2*GGML_DEFAULT_GRAPH_SIZE)
27
+ #define GGML_METAL_MAX_KERNELS 256
28
28
 
29
29
  struct ggml_metal_buffer {
30
30
  const char * name;
@@ -35,6 +35,134 @@ struct ggml_metal_buffer {
35
35
  id<MTLBuffer> metal;
36
36
  };
37
37
 
38
+ struct ggml_metal_kernel {
39
+ id<MTLFunction> function;
40
+ id<MTLComputePipelineState> pipeline;
41
+ };
42
+
43
+ enum ggml_metal_kernel_type {
44
+ GGML_METAL_KERNEL_TYPE_ADD,
45
+ GGML_METAL_KERNEL_TYPE_ADD_ROW,
46
+ GGML_METAL_KERNEL_TYPE_MUL,
47
+ GGML_METAL_KERNEL_TYPE_MUL_ROW,
48
+ GGML_METAL_KERNEL_TYPE_DIV,
49
+ GGML_METAL_KERNEL_TYPE_DIV_ROW,
50
+ GGML_METAL_KERNEL_TYPE_SCALE,
51
+ GGML_METAL_KERNEL_TYPE_SCALE_4,
52
+ GGML_METAL_KERNEL_TYPE_TANH,
53
+ GGML_METAL_KERNEL_TYPE_RELU,
54
+ GGML_METAL_KERNEL_TYPE_GELU,
55
+ GGML_METAL_KERNEL_TYPE_GELU_QUICK,
56
+ GGML_METAL_KERNEL_TYPE_SILU,
57
+ GGML_METAL_KERNEL_TYPE_SOFT_MAX,
58
+ GGML_METAL_KERNEL_TYPE_SOFT_MAX_4,
59
+ GGML_METAL_KERNEL_TYPE_DIAG_MASK_INF,
60
+ GGML_METAL_KERNEL_TYPE_DIAG_MASK_INF_8,
61
+ GGML_METAL_KERNEL_TYPE_GET_ROWS_F32,
62
+ GGML_METAL_KERNEL_TYPE_GET_ROWS_F16,
63
+ GGML_METAL_KERNEL_TYPE_GET_ROWS_Q4_0,
64
+ GGML_METAL_KERNEL_TYPE_GET_ROWS_Q4_1,
65
+ GGML_METAL_KERNEL_TYPE_GET_ROWS_Q5_0,
66
+ GGML_METAL_KERNEL_TYPE_GET_ROWS_Q5_1,
67
+ GGML_METAL_KERNEL_TYPE_GET_ROWS_Q8_0,
68
+ GGML_METAL_KERNEL_TYPE_GET_ROWS_Q2_K,
69
+ GGML_METAL_KERNEL_TYPE_GET_ROWS_Q3_K,
70
+ GGML_METAL_KERNEL_TYPE_GET_ROWS_Q4_K,
71
+ GGML_METAL_KERNEL_TYPE_GET_ROWS_Q5_K,
72
+ GGML_METAL_KERNEL_TYPE_GET_ROWS_Q6_K,
73
+ GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_XXS,
74
+ GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_XS,
75
+ GGML_METAL_KERNEL_TYPE_GET_ROWS_I32,
76
+ GGML_METAL_KERNEL_TYPE_RMS_NORM,
77
+ GGML_METAL_KERNEL_TYPE_GROUP_NORM,
78
+ GGML_METAL_KERNEL_TYPE_NORM,
79
+ GGML_METAL_KERNEL_TYPE_MUL_MV_F32_F32,
80
+ GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F16,
81
+ GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F32,
82
+ GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F32_1ROW,
83
+ GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F32_L4,
84
+ GGML_METAL_KERNEL_TYPE_MUL_MV_Q4_0_F32,
85
+ GGML_METAL_KERNEL_TYPE_MUL_MV_Q4_1_F32,
86
+ GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_0_F32,
87
+ GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_1_F32,
88
+ GGML_METAL_KERNEL_TYPE_MUL_MV_Q8_0_F32,
89
+ GGML_METAL_KERNEL_TYPE_MUL_MV_Q2_K_F32,
90
+ GGML_METAL_KERNEL_TYPE_MUL_MV_Q3_K_F32,
91
+ GGML_METAL_KERNEL_TYPE_MUL_MV_Q4_K_F32,
92
+ GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_K_F32,
93
+ GGML_METAL_KERNEL_TYPE_MUL_MV_Q6_K_F32,
94
+ GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_XXS_F32,
95
+ GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_XS_F32,
96
+ GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F32_F32,
97
+ //GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F16,
98
+ GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F32,
99
+ //GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F32_1ROW,
100
+ //GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F32_L4,
101
+ GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q4_0_F32,
102
+ GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q4_1_F32,
103
+ GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q5_0_F32,
104
+ GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q5_1_F32,
105
+ GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q8_0_F32,
106
+ GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q2_K_F32,
107
+ GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q3_K_F32,
108
+ GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q4_K_F32,
109
+ GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q5_K_F32,
110
+ GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q6_K_F32,
111
+ GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_XXS_F32,
112
+ GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_XS_F32,
113
+ GGML_METAL_KERNEL_TYPE_MUL_MM_F32_F32,
114
+ GGML_METAL_KERNEL_TYPE_MUL_MM_F16_F32,
115
+ GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_0_F32,
116
+ GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_1_F32,
117
+ GGML_METAL_KERNEL_TYPE_MUL_MM_Q5_0_F32,
118
+ GGML_METAL_KERNEL_TYPE_MUL_MM_Q5_1_F32,
119
+ GGML_METAL_KERNEL_TYPE_MUL_MM_Q8_0_F32,
120
+ GGML_METAL_KERNEL_TYPE_MUL_MM_Q2_K_F32,
121
+ GGML_METAL_KERNEL_TYPE_MUL_MM_Q3_K_F32,
122
+ GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_K_F32,
123
+ GGML_METAL_KERNEL_TYPE_MUL_MM_Q5_K_F32,
124
+ GGML_METAL_KERNEL_TYPE_MUL_MM_Q6_K_F32,
125
+ GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_XXS_F32,
126
+ GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_XS_F32,
127
+ GGML_METAL_KERNEL_TYPE_MUL_MM_ID_F32_F32,
128
+ GGML_METAL_KERNEL_TYPE_MUL_MM_ID_F16_F32,
129
+ GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_0_F32,
130
+ GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_1_F32,
131
+ GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q5_0_F32,
132
+ GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q5_1_F32,
133
+ GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q8_0_F32,
134
+ GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q2_K_F32,
135
+ GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q3_K_F32,
136
+ GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_K_F32,
137
+ GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q5_K_F32,
138
+ GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q6_K_F32,
139
+ GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_XXS_F32,
140
+ GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_XS_F32,
141
+ GGML_METAL_KERNEL_TYPE_ROPE_F32,
142
+ GGML_METAL_KERNEL_TYPE_ROPE_F16,
143
+ GGML_METAL_KERNEL_TYPE_ALIBI_F32,
144
+ GGML_METAL_KERNEL_TYPE_IM2COL_F16,
145
+ GGML_METAL_KERNEL_TYPE_UPSCALE_F32,
146
+ GGML_METAL_KERNEL_TYPE_PAD_F32,
147
+ GGML_METAL_KERNEL_TYPE_ARGSORT_F32_I32_ASC,
148
+ GGML_METAL_KERNEL_TYPE_ARGSORT_F32_I32_DESC,
149
+ GGML_METAL_KERNEL_TYPE_LEAKY_RELU_F32,
150
+ GGML_METAL_KERNEL_TYPE_CPY_F32_F16,
151
+ GGML_METAL_KERNEL_TYPE_CPY_F32_F32,
152
+ GGML_METAL_KERNEL_TYPE_CPY_F32_Q8_0,
153
+ GGML_METAL_KERNEL_TYPE_CPY_F32_Q4_0,
154
+ GGML_METAL_KERNEL_TYPE_CPY_F32_Q4_1,
155
+ //GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_0,
156
+ //GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_1,
157
+ GGML_METAL_KERNEL_TYPE_CPY_F16_F16,
158
+ GGML_METAL_KERNEL_TYPE_CPY_F16_F32,
159
+ GGML_METAL_KERNEL_TYPE_CONCAT,
160
+ GGML_METAL_KERNEL_TYPE_SQR,
161
+ GGML_METAL_KERNEL_TYPE_SUM_ROWS,
162
+
163
+ GGML_METAL_KERNEL_TYPE_COUNT
164
+ };
165
+
38
166
  struct ggml_metal_context {
39
167
  int n_cb;
40
168
 
@@ -42,142 +170,15 @@ struct ggml_metal_context {
42
170
  id<MTLCommandQueue> queue;
43
171
  id<MTLLibrary> library;
44
172
 
45
- id<MTLCommandBuffer> command_buffers [GGML_METAL_MAX_COMMAND_BUFFERS];
46
- id<MTLComputeCommandEncoder> command_encoders[GGML_METAL_MAX_COMMAND_BUFFERS];
47
-
48
173
  dispatch_queue_t d_queue;
49
174
 
50
175
  int n_buffers;
51
176
  struct ggml_metal_buffer buffers[GGML_METAL_MAX_BUFFERS];
52
177
 
53
- int concur_list[GGML_MAX_CONCUR];
54
- int concur_list_len;
55
-
56
- // custom kernels
57
- #define GGML_METAL_DECL_KERNEL(name) \
58
- id<MTLFunction> function_##name; \
59
- id<MTLComputePipelineState> pipeline_##name
60
-
61
- GGML_METAL_DECL_KERNEL(add);
62
- GGML_METAL_DECL_KERNEL(add_row); // TODO: avoid this extra kernel, instead extend the "add" kernel to support broadcast
63
- GGML_METAL_DECL_KERNEL(mul);
64
- GGML_METAL_DECL_KERNEL(mul_row); // TODO: avoid this extra kernel, instead extend the "mul" kernel to support broadcast
65
- GGML_METAL_DECL_KERNEL(div);
66
- GGML_METAL_DECL_KERNEL(div_row);
67
- GGML_METAL_DECL_KERNEL(scale);
68
- GGML_METAL_DECL_KERNEL(scale_4);
69
- GGML_METAL_DECL_KERNEL(tanh);
70
- GGML_METAL_DECL_KERNEL(relu);
71
- GGML_METAL_DECL_KERNEL(gelu);
72
- GGML_METAL_DECL_KERNEL(gelu_quick);
73
- GGML_METAL_DECL_KERNEL(silu);
74
- GGML_METAL_DECL_KERNEL(soft_max);
75
- GGML_METAL_DECL_KERNEL(soft_max_4);
76
- GGML_METAL_DECL_KERNEL(diag_mask_inf);
77
- GGML_METAL_DECL_KERNEL(diag_mask_inf_8);
78
- GGML_METAL_DECL_KERNEL(get_rows_f32);
79
- GGML_METAL_DECL_KERNEL(get_rows_f16);
80
- GGML_METAL_DECL_KERNEL(get_rows_q4_0);
81
- GGML_METAL_DECL_KERNEL(get_rows_q4_1);
82
- GGML_METAL_DECL_KERNEL(get_rows_q5_0);
83
- GGML_METAL_DECL_KERNEL(get_rows_q5_1);
84
- GGML_METAL_DECL_KERNEL(get_rows_q8_0);
85
- GGML_METAL_DECL_KERNEL(get_rows_q2_K);
86
- GGML_METAL_DECL_KERNEL(get_rows_q3_K);
87
- GGML_METAL_DECL_KERNEL(get_rows_q4_K);
88
- GGML_METAL_DECL_KERNEL(get_rows_q5_K);
89
- GGML_METAL_DECL_KERNEL(get_rows_q6_K);
90
- GGML_METAL_DECL_KERNEL(get_rows_i32);
91
- GGML_METAL_DECL_KERNEL(get_rows_iq2_xxs);
92
- GGML_METAL_DECL_KERNEL(get_rows_iq2_xs);
93
- GGML_METAL_DECL_KERNEL(rms_norm);
94
- GGML_METAL_DECL_KERNEL(group_norm);
95
- GGML_METAL_DECL_KERNEL(norm);
96
- GGML_METAL_DECL_KERNEL(mul_mv_f32_f32);
97
- GGML_METAL_DECL_KERNEL(mul_mv_f16_f16);
98
- GGML_METAL_DECL_KERNEL(mul_mv_f16_f32);
99
- GGML_METAL_DECL_KERNEL(mul_mv_f16_f32_1row);
100
- GGML_METAL_DECL_KERNEL(mul_mv_f16_f32_l4);
101
- GGML_METAL_DECL_KERNEL(mul_mv_q4_0_f32);
102
- GGML_METAL_DECL_KERNEL(mul_mv_q4_1_f32);
103
- GGML_METAL_DECL_KERNEL(mul_mv_q5_0_f32);
104
- GGML_METAL_DECL_KERNEL(mul_mv_q5_1_f32);
105
- GGML_METAL_DECL_KERNEL(mul_mv_q8_0_f32);
106
- GGML_METAL_DECL_KERNEL(mul_mv_q2_K_f32);
107
- GGML_METAL_DECL_KERNEL(mul_mv_q3_K_f32);
108
- GGML_METAL_DECL_KERNEL(mul_mv_q4_K_f32);
109
- GGML_METAL_DECL_KERNEL(mul_mv_q5_K_f32);
110
- GGML_METAL_DECL_KERNEL(mul_mv_q6_K_f32);
111
- GGML_METAL_DECL_KERNEL(mul_mv_iq2_xxs_f32);
112
- GGML_METAL_DECL_KERNEL(mul_mv_iq2_xs_f32);
113
- GGML_METAL_DECL_KERNEL(mul_mv_id_f32_f32);
114
- //GGML_METAL_DECL_KERNEL(mul_mv_id_f16_f16);
115
- GGML_METAL_DECL_KERNEL(mul_mv_id_f16_f32);
116
- //GGML_METAL_DECL_KERNEL(mul_mv_id_f16_f32_1row);
117
- //GGML_METAL_DECL_KERNEL(mul_mv_id_f16_f32_l4);
118
- GGML_METAL_DECL_KERNEL(mul_mv_id_q4_0_f32);
119
- GGML_METAL_DECL_KERNEL(mul_mv_id_q4_1_f32);
120
- GGML_METAL_DECL_KERNEL(mul_mv_id_q5_0_f32);
121
- GGML_METAL_DECL_KERNEL(mul_mv_id_q5_1_f32);
122
- GGML_METAL_DECL_KERNEL(mul_mv_id_q8_0_f32);
123
- GGML_METAL_DECL_KERNEL(mul_mv_id_q2_K_f32);
124
- GGML_METAL_DECL_KERNEL(mul_mv_id_q3_K_f32);
125
- GGML_METAL_DECL_KERNEL(mul_mv_id_q4_K_f32);
126
- GGML_METAL_DECL_KERNEL(mul_mv_id_q5_K_f32);
127
- GGML_METAL_DECL_KERNEL(mul_mv_id_q6_K_f32);
128
- GGML_METAL_DECL_KERNEL(mul_mv_id_iq2_xxs_f32);
129
- GGML_METAL_DECL_KERNEL(mul_mv_id_iq2_xs_f32);
130
- GGML_METAL_DECL_KERNEL(mul_mm_f32_f32);
131
- GGML_METAL_DECL_KERNEL(mul_mm_f16_f32);
132
- GGML_METAL_DECL_KERNEL(mul_mm_q4_0_f32);
133
- GGML_METAL_DECL_KERNEL(mul_mm_q4_1_f32);
134
- GGML_METAL_DECL_KERNEL(mul_mm_q5_0_f32);
135
- GGML_METAL_DECL_KERNEL(mul_mm_q5_1_f32);
136
- GGML_METAL_DECL_KERNEL(mul_mm_q8_0_f32);
137
- GGML_METAL_DECL_KERNEL(mul_mm_q2_K_f32);
138
- GGML_METAL_DECL_KERNEL(mul_mm_q3_K_f32);
139
- GGML_METAL_DECL_KERNEL(mul_mm_q4_K_f32);
140
- GGML_METAL_DECL_KERNEL(mul_mm_q5_K_f32);
141
- GGML_METAL_DECL_KERNEL(mul_mm_q6_K_f32);
142
- GGML_METAL_DECL_KERNEL(mul_mm_iq2_xxs_f32);
143
- GGML_METAL_DECL_KERNEL(mul_mm_iq2_xs_f32);
144
- GGML_METAL_DECL_KERNEL(mul_mm_id_f32_f32);
145
- GGML_METAL_DECL_KERNEL(mul_mm_id_f16_f32);
146
- GGML_METAL_DECL_KERNEL(mul_mm_id_q4_0_f32);
147
- GGML_METAL_DECL_KERNEL(mul_mm_id_q4_1_f32);
148
- GGML_METAL_DECL_KERNEL(mul_mm_id_q5_0_f32);
149
- GGML_METAL_DECL_KERNEL(mul_mm_id_q5_1_f32);
150
- GGML_METAL_DECL_KERNEL(mul_mm_id_q8_0_f32);
151
- GGML_METAL_DECL_KERNEL(mul_mm_id_q2_K_f32);
152
- GGML_METAL_DECL_KERNEL(mul_mm_id_q3_K_f32);
153
- GGML_METAL_DECL_KERNEL(mul_mm_id_q4_K_f32);
154
- GGML_METAL_DECL_KERNEL(mul_mm_id_q5_K_f32);
155
- GGML_METAL_DECL_KERNEL(mul_mm_id_q6_K_f32);
156
- GGML_METAL_DECL_KERNEL(mul_mm_id_iq2_xxs_f32);
157
- GGML_METAL_DECL_KERNEL(mul_mm_id_iq2_xs_f32);
158
- GGML_METAL_DECL_KERNEL(rope_f32);
159
- GGML_METAL_DECL_KERNEL(rope_f16);
160
- GGML_METAL_DECL_KERNEL(alibi_f32);
161
- GGML_METAL_DECL_KERNEL(im2col_f16);
162
- GGML_METAL_DECL_KERNEL(upscale_f32);
163
- GGML_METAL_DECL_KERNEL(pad_f32);
164
- GGML_METAL_DECL_KERNEL(argsort_f32_i32_asc);
165
- GGML_METAL_DECL_KERNEL(argsort_f32_i32_desc);
166
- GGML_METAL_DECL_KERNEL(leaky_relu_f32);
167
- GGML_METAL_DECL_KERNEL(cpy_f32_f16);
168
- GGML_METAL_DECL_KERNEL(cpy_f32_f32);
169
- GGML_METAL_DECL_KERNEL(cpy_f32_q8_0);
170
- GGML_METAL_DECL_KERNEL(cpy_f32_q4_0);
171
- GGML_METAL_DECL_KERNEL(cpy_f32_q4_1);
172
- //GGML_METAL_DECL_KERNEL(cpy_f32_q5_0);
173
- //GGML_METAL_DECL_KERNEL(cpy_f32_q5_1);
174
- GGML_METAL_DECL_KERNEL(cpy_f16_f16);
175
- GGML_METAL_DECL_KERNEL(cpy_f16_f32);
176
- GGML_METAL_DECL_KERNEL(concat);
177
- GGML_METAL_DECL_KERNEL(sqr);
178
- GGML_METAL_DECL_KERNEL(sum_rows);
179
-
180
- #undef GGML_METAL_DECL_KERNEL
178
+ struct ggml_metal_kernel kernels[GGML_METAL_MAX_KERNELS];
179
+
180
+ bool support_simdgroup_reduction;
181
+ bool support_simdgroup_mm;
181
182
  };
182
183
 
183
184
  // MSL code
@@ -191,7 +192,6 @@ struct ggml_metal_context {
191
192
  @implementation GGMLMetalClass
192
193
  @end
193
194
 
194
-
195
195
  static void ggml_metal_default_log_callback(enum ggml_log_level level, const char * msg, void * user_data) {
196
196
  fprintf(stderr, "%s", msg);
197
197
 
@@ -202,11 +202,6 @@ static void ggml_metal_default_log_callback(enum ggml_log_level level, const cha
202
202
  ggml_log_callback ggml_metal_log_callback = ggml_metal_default_log_callback;
203
203
  void * ggml_metal_log_user_data = NULL;
204
204
 
205
- void ggml_metal_log_set_callback(ggml_log_callback log_callback, void * user_data) {
206
- ggml_metal_log_callback = log_callback;
207
- ggml_metal_log_user_data = user_data;
208
- }
209
-
210
205
  GGML_ATTRIBUTE_FORMAT(2, 3)
211
206
  static void ggml_metal_log(enum ggml_log_level level, const char * format, ...){
212
207
  if (ggml_metal_log_callback != NULL) {
@@ -229,7 +224,18 @@ static void ggml_metal_log(enum ggml_log_level level, const char * format, ...){
229
224
  }
230
225
  }
231
226
 
232
- struct ggml_metal_context * ggml_metal_init(int n_cb) {
227
+ static void * ggml_metal_host_malloc(size_t n) {
228
+ void * data = NULL;
229
+ const int result = posix_memalign((void **) &data, sysconf(_SC_PAGESIZE), n);
230
+ if (result != 0) {
231
+ GGML_METAL_LOG_ERROR("%s: error: posix_memalign failed\n", __func__);
232
+ return NULL;
233
+ }
234
+
235
+ return data;
236
+ }
237
+
238
+ static struct ggml_metal_context * ggml_metal_init(int n_cb) {
233
239
  GGML_METAL_LOG_INFO("%s: allocating\n", __func__);
234
240
 
235
241
  id<MTLDevice> device;
@@ -255,7 +261,6 @@ struct ggml_metal_context * ggml_metal_init(int n_cb) {
255
261
  ctx->n_cb = MIN(n_cb, GGML_METAL_MAX_BUFFERS);
256
262
  ctx->queue = [ctx->device newCommandQueue];
257
263
  ctx->n_buffers = 0;
258
- ctx->concur_list_len = 0;
259
264
 
260
265
  ctx->d_queue = dispatch_queue_create("ggml-metal", DISPATCH_QUEUE_CONCURRENT);
261
266
 
@@ -298,19 +303,22 @@ struct ggml_metal_context * ggml_metal_init(int n_cb) {
298
303
  return NULL;
299
304
  }
300
305
 
301
- MTLCompileOptions* options = nil;
306
+ // dictionary of preprocessor macros
307
+ NSMutableDictionary * prep = [NSMutableDictionary dictionary];
308
+
302
309
  #ifdef GGML_QKK_64
303
- options = [MTLCompileOptions new];
304
- options.preprocessorMacros = @{ @"QK_K" : @(64) };
310
+ prep[@"QK_K"] = @(64);
305
311
  #endif
306
- // try to disable fast-math
307
- // NOTE: this seems to have no effect whatsoever
308
- // instead, in order to disable fast-math, we have to build default.metallib from the command line
309
- // using xcrun -sdk macosx metal -fno-fast-math -c ggml-metal.metal -o ggml-metal.air
310
- // and go through the "pre-compiled library found" path above
312
+
313
+ MTLCompileOptions* options = [MTLCompileOptions new];
314
+ options.preprocessorMacros = prep;
315
+
311
316
  //[options setFastMathEnabled:false];
312
317
 
313
318
  ctx->library = [ctx->device newLibraryWithSource:src options:options error:&error];
319
+
320
+ [options release];
321
+ [prep release];
314
322
  }
315
323
 
316
324
  if (error) {
@@ -319,22 +327,51 @@ struct ggml_metal_context * ggml_metal_init(int n_cb) {
319
327
  }
320
328
  }
321
329
 
322
- #if TARGET_OS_OSX
323
330
  // print MTL GPU family:
324
331
  GGML_METAL_LOG_INFO("%s: GPU name: %s\n", __func__, [[ctx->device name] UTF8String]);
325
332
 
333
+ const NSInteger MTLGPUFamilyMetal3 = 5001;
334
+
326
335
  // determine max supported GPU family
327
336
  // https://developer.apple.com/metal/Metal-Shading-Language-Specification.pdf
328
337
  // https://developer.apple.com/metal/Metal-Feature-Set-Tables.pdf
329
- for (int i = MTLGPUFamilyApple1 + 20; i >= MTLGPUFamilyApple1; --i) {
330
- if ([ctx->device supportsFamily:i]) {
331
- GGML_METAL_LOG_INFO("%s: GPU family: MTLGPUFamilyApple%d (%d)\n", __func__, i - (int) MTLGPUFamilyApple1 + 1, i);
332
- break;
338
+ {
339
+ for (int i = MTLGPUFamilyApple1 + 20; i >= MTLGPUFamilyApple1; --i) {
340
+ if ([ctx->device supportsFamily:i]) {
341
+ GGML_METAL_LOG_INFO("%s: GPU family: MTLGPUFamilyApple%d (%d)\n", __func__, i - (int) MTLGPUFamilyApple1 + 1, i);
342
+ break;
343
+ }
344
+ }
345
+
346
+ for (int i = MTLGPUFamilyCommon1 + 5; i >= MTLGPUFamilyCommon1; --i) {
347
+ if ([ctx->device supportsFamily:i]) {
348
+ GGML_METAL_LOG_INFO("%s: GPU family: MTLGPUFamilyCommon%d (%d)\n", __func__, i - (int) MTLGPUFamilyCommon1 + 1, i);
349
+ break;
350
+ }
351
+ }
352
+
353
+ for (int i = MTLGPUFamilyMetal3 + 5; i >= MTLGPUFamilyMetal3; --i) {
354
+ if ([ctx->device supportsFamily:i]) {
355
+ GGML_METAL_LOG_INFO("%s: GPU family: MTLGPUFamilyMetal%d (%d)\n", __func__, i - (int) MTLGPUFamilyMetal3 + 3, i);
356
+ break;
357
+ }
333
358
  }
334
359
  }
335
360
 
361
+ ctx->support_simdgroup_reduction = [ctx->device supportsFamily:MTLGPUFamilyApple7];
362
+ ctx->support_simdgroup_reduction |= [ctx->device supportsFamily:MTLGPUFamilyMetal3];
363
+
364
+ ctx->support_simdgroup_mm = [ctx->device supportsFamily:MTLGPUFamilyApple7];
365
+
366
+ GGML_METAL_LOG_INFO("%s: simdgroup reduction support = %s\n", __func__, ctx->support_simdgroup_reduction ? "true" : "false");
367
+ GGML_METAL_LOG_INFO("%s: simdgroup matrix mul. support = %s\n", __func__, ctx->support_simdgroup_mm ? "true" : "false");
336
368
  GGML_METAL_LOG_INFO("%s: hasUnifiedMemory = %s\n", __func__, ctx->device.hasUnifiedMemory ? "true" : "false");
337
- GGML_METAL_LOG_INFO("%s: recommendedMaxWorkingSetSize = %8.2f MB\n", __func__, ctx->device.recommendedMaxWorkingSetSize / 1e6);
369
+
370
+ #if TARGET_OS_OSX || (TARGET_OS_IOS && __clang_major__ >= 15)
371
+ if (@available(macOS 10.12, iOS 16.0, *)) {
372
+ GGML_METAL_LOG_INFO("%s: recommendedMaxWorkingSetSize = %8.2f MB\n", __func__, ctx->device.recommendedMaxWorkingSetSize / 1e6);
373
+ }
374
+ #elif TARGET_OS_OSX
338
375
  if (ctx->device.maxTransferRate != 0) {
339
376
  GGML_METAL_LOG_INFO("%s: maxTransferRate = %8.2f MB/s\n", __func__, ctx->device.maxTransferRate / 1e6);
340
377
  } else {
@@ -346,279 +383,171 @@ struct ggml_metal_context * ggml_metal_init(int n_cb) {
346
383
  {
347
384
  NSError * error = nil;
348
385
 
386
+ for (int i = 0; i < GGML_METAL_MAX_KERNELS; ++i) {
387
+ ctx->kernels[i].function = nil;
388
+ ctx->kernels[i].pipeline = nil;
389
+ }
390
+
349
391
  /*
350
- GGML_METAL_LOG_INFO("%s: loaded %-32s %16p | th_max = %4d | th_width = %4d\n", __func__, "kernel_"#name, (void *) ctx->pipeline_##name, \
351
- (int) ctx->pipeline_##name.maxTotalThreadsPerThreadgroup, \
352
- (int) ctx->pipeline_##name.threadExecutionWidth); \
392
+ GGML_METAL_LOG_INFO("%s: loaded %-32s %16p | th_max = %4d | th_width = %4d\n", __func__, "kernel_"#name, (void *) kernel->pipeline, \
393
+ (int) kernel->pipeline.maxTotalThreadsPerThreadgroup, \
394
+ (int) kernel->pipeline.threadExecutionWidth); \
353
395
  */
354
- #define GGML_METAL_ADD_KERNEL(name) \
355
- ctx->function_##name = [ctx->library newFunctionWithName:@"kernel_"#name]; \
356
- ctx->pipeline_##name = [ctx->device newComputePipelineStateWithFunction:ctx->function_##name error:&error]; \
357
- if (error) { \
358
- GGML_METAL_LOG_ERROR("%s: error: load pipeline error: %s\n", __func__, [[error description] UTF8String]); \
359
- return NULL; \
396
+ #define GGML_METAL_ADD_KERNEL(e, name, supported) \
397
+ if (supported) { \
398
+ struct ggml_metal_kernel * kernel = &ctx->kernels[e]; \
399
+ kernel->function = [ctx->library newFunctionWithName:@"kernel_"#name]; \
400
+ kernel->pipeline = [ctx->device newComputePipelineStateWithFunction:kernel->function error:&error]; \
401
+ if (error) { \
402
+ GGML_METAL_LOG_ERROR("%s: error: load pipeline error: %s\n", __func__, [[error description] UTF8String]); \
403
+ return NULL; \
404
+ } \
405
+ } else { \
406
+ GGML_METAL_LOG_WARN("%s: skipping %-32s (not supported)\n", __func__, "kernel_"#name); \
360
407
  }
361
408
 
362
- GGML_METAL_ADD_KERNEL(add);
363
- GGML_METAL_ADD_KERNEL(add_row);
364
- GGML_METAL_ADD_KERNEL(mul);
365
- GGML_METAL_ADD_KERNEL(mul_row);
366
- GGML_METAL_ADD_KERNEL(div);
367
- GGML_METAL_ADD_KERNEL(div_row);
368
- GGML_METAL_ADD_KERNEL(scale);
369
- GGML_METAL_ADD_KERNEL(scale_4);
370
- GGML_METAL_ADD_KERNEL(tanh);
371
- GGML_METAL_ADD_KERNEL(relu);
372
- GGML_METAL_ADD_KERNEL(gelu);
373
- GGML_METAL_ADD_KERNEL(gelu_quick);
374
- GGML_METAL_ADD_KERNEL(silu);
375
- GGML_METAL_ADD_KERNEL(soft_max);
376
- GGML_METAL_ADD_KERNEL(soft_max_4);
377
- GGML_METAL_ADD_KERNEL(diag_mask_inf);
378
- GGML_METAL_ADD_KERNEL(diag_mask_inf_8);
379
- GGML_METAL_ADD_KERNEL(get_rows_f32);
380
- GGML_METAL_ADD_KERNEL(get_rows_f16);
381
- GGML_METAL_ADD_KERNEL(get_rows_q4_0);
382
- GGML_METAL_ADD_KERNEL(get_rows_q4_1);
383
- GGML_METAL_ADD_KERNEL(get_rows_q5_0);
384
- GGML_METAL_ADD_KERNEL(get_rows_q5_1);
385
- GGML_METAL_ADD_KERNEL(get_rows_q8_0);
386
- GGML_METAL_ADD_KERNEL(get_rows_q2_K);
387
- GGML_METAL_ADD_KERNEL(get_rows_q3_K);
388
- GGML_METAL_ADD_KERNEL(get_rows_q4_K);
389
- GGML_METAL_ADD_KERNEL(get_rows_q5_K);
390
- GGML_METAL_ADD_KERNEL(get_rows_q6_K);
391
- GGML_METAL_ADD_KERNEL(get_rows_i32);
392
- GGML_METAL_ADD_KERNEL(get_rows_iq2_xxs);
393
- GGML_METAL_ADD_KERNEL(get_rows_iq2_xs);
394
- GGML_METAL_ADD_KERNEL(rms_norm);
395
- GGML_METAL_ADD_KERNEL(group_norm);
396
- GGML_METAL_ADD_KERNEL(norm);
397
- GGML_METAL_ADD_KERNEL(mul_mv_f32_f32);
398
- GGML_METAL_ADD_KERNEL(mul_mv_f16_f16);
399
- GGML_METAL_ADD_KERNEL(mul_mv_f16_f32);
400
- GGML_METAL_ADD_KERNEL(mul_mv_f16_f32_1row);
401
- GGML_METAL_ADD_KERNEL(mul_mv_f16_f32_l4);
402
- GGML_METAL_ADD_KERNEL(mul_mv_q4_0_f32);
403
- GGML_METAL_ADD_KERNEL(mul_mv_q4_1_f32);
404
- GGML_METAL_ADD_KERNEL(mul_mv_q5_0_f32);
405
- GGML_METAL_ADD_KERNEL(mul_mv_q5_1_f32);
406
- GGML_METAL_ADD_KERNEL(mul_mv_q8_0_f32);
407
- GGML_METAL_ADD_KERNEL(mul_mv_q2_K_f32);
408
- GGML_METAL_ADD_KERNEL(mul_mv_q3_K_f32);
409
- GGML_METAL_ADD_KERNEL(mul_mv_q4_K_f32);
410
- GGML_METAL_ADD_KERNEL(mul_mv_q5_K_f32);
411
- GGML_METAL_ADD_KERNEL(mul_mv_q6_K_f32);
412
- GGML_METAL_ADD_KERNEL(mul_mv_iq2_xxs_f32);
413
- GGML_METAL_ADD_KERNEL(mul_mv_iq2_xs_f32);
414
- GGML_METAL_ADD_KERNEL(mul_mv_id_f32_f32);
415
- //GGML_METAL_ADD_KERNEL(mul_mv_id_f16_f16);
416
- GGML_METAL_ADD_KERNEL(mul_mv_id_f16_f32);
417
- //GGML_METAL_ADD_KERNEL(mul_mv_id_f16_f32_1row);
418
- //GGML_METAL_ADD_KERNEL(mul_mv_id_f16_f32_l4);
419
- GGML_METAL_ADD_KERNEL(mul_mv_id_q4_0_f32);
420
- GGML_METAL_ADD_KERNEL(mul_mv_id_q4_1_f32);
421
- GGML_METAL_ADD_KERNEL(mul_mv_id_q5_0_f32);
422
- GGML_METAL_ADD_KERNEL(mul_mv_id_q5_1_f32);
423
- GGML_METAL_ADD_KERNEL(mul_mv_id_q8_0_f32);
424
- GGML_METAL_ADD_KERNEL(mul_mv_id_q2_K_f32);
425
- GGML_METAL_ADD_KERNEL(mul_mv_id_q3_K_f32);
426
- GGML_METAL_ADD_KERNEL(mul_mv_id_q4_K_f32);
427
- GGML_METAL_ADD_KERNEL(mul_mv_id_q5_K_f32);
428
- GGML_METAL_ADD_KERNEL(mul_mv_id_q6_K_f32);
429
- GGML_METAL_ADD_KERNEL(mul_mv_id_iq2_xxs_f32);
430
- GGML_METAL_ADD_KERNEL(mul_mv_id_iq2_xs_f32);
431
- if ([ctx->device supportsFamily:MTLGPUFamilyApple7]) {
432
- GGML_METAL_ADD_KERNEL(mul_mm_f32_f32);
433
- GGML_METAL_ADD_KERNEL(mul_mm_f16_f32);
434
- GGML_METAL_ADD_KERNEL(mul_mm_q4_0_f32);
435
- GGML_METAL_ADD_KERNEL(mul_mm_q4_1_f32);
436
- GGML_METAL_ADD_KERNEL(mul_mm_q5_0_f32);
437
- GGML_METAL_ADD_KERNEL(mul_mm_q5_1_f32);
438
- GGML_METAL_ADD_KERNEL(mul_mm_q8_0_f32);
439
- GGML_METAL_ADD_KERNEL(mul_mm_q2_K_f32);
440
- GGML_METAL_ADD_KERNEL(mul_mm_q3_K_f32);
441
- GGML_METAL_ADD_KERNEL(mul_mm_q4_K_f32);
442
- GGML_METAL_ADD_KERNEL(mul_mm_q5_K_f32);
443
- GGML_METAL_ADD_KERNEL(mul_mm_q6_K_f32);
444
- GGML_METAL_ADD_KERNEL(mul_mm_iq2_xxs_f32);
445
- GGML_METAL_ADD_KERNEL(mul_mm_iq2_xs_f32);
446
- GGML_METAL_ADD_KERNEL(mul_mm_id_f32_f32);
447
- GGML_METAL_ADD_KERNEL(mul_mm_id_f16_f32);
448
- GGML_METAL_ADD_KERNEL(mul_mm_id_q4_0_f32);
449
- GGML_METAL_ADD_KERNEL(mul_mm_id_q4_1_f32);
450
- GGML_METAL_ADD_KERNEL(mul_mm_id_q5_0_f32);
451
- GGML_METAL_ADD_KERNEL(mul_mm_id_q5_1_f32);
452
- GGML_METAL_ADD_KERNEL(mul_mm_id_q8_0_f32);
453
- GGML_METAL_ADD_KERNEL(mul_mm_id_q2_K_f32);
454
- GGML_METAL_ADD_KERNEL(mul_mm_id_q3_K_f32);
455
- GGML_METAL_ADD_KERNEL(mul_mm_id_q4_K_f32);
456
- GGML_METAL_ADD_KERNEL(mul_mm_id_q5_K_f32);
457
- GGML_METAL_ADD_KERNEL(mul_mm_id_q6_K_f32);
458
- GGML_METAL_ADD_KERNEL(mul_mm_id_iq2_xxs_f32);
459
- GGML_METAL_ADD_KERNEL(mul_mm_id_iq2_xs_f32);
460
- }
461
- GGML_METAL_ADD_KERNEL(rope_f32);
462
- GGML_METAL_ADD_KERNEL(rope_f16);
463
- GGML_METAL_ADD_KERNEL(alibi_f32);
464
- GGML_METAL_ADD_KERNEL(im2col_f16);
465
- GGML_METAL_ADD_KERNEL(upscale_f32);
466
- GGML_METAL_ADD_KERNEL(pad_f32);
467
- GGML_METAL_ADD_KERNEL(argsort_f32_i32_asc);
468
- GGML_METAL_ADD_KERNEL(argsort_f32_i32_desc);
469
- GGML_METAL_ADD_KERNEL(leaky_relu_f32);
470
- GGML_METAL_ADD_KERNEL(cpy_f32_f16);
471
- GGML_METAL_ADD_KERNEL(cpy_f32_f32);
472
- GGML_METAL_ADD_KERNEL(cpy_f32_q8_0);
473
- GGML_METAL_ADD_KERNEL(cpy_f32_q4_0);
474
- GGML_METAL_ADD_KERNEL(cpy_f32_q4_1);
475
- //GGML_METAL_ADD_KERNEL(cpy_f32_q5_0);
476
- //GGML_METAL_ADD_KERNEL(cpy_f32_q5_1);
477
- GGML_METAL_ADD_KERNEL(cpy_f16_f16);
478
- GGML_METAL_ADD_KERNEL(cpy_f16_f32);
479
- GGML_METAL_ADD_KERNEL(concat);
480
- GGML_METAL_ADD_KERNEL(sqr);
481
- GGML_METAL_ADD_KERNEL(sum_rows);
482
-
483
- #undef GGML_METAL_ADD_KERNEL
409
+ // simd_sum and simd_max requires MTLGPUFamilyApple7
410
+
411
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ADD, add, true);
412
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ADD_ROW, add_row, true);
413
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL, mul, true);
414
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_ROW, mul_row, true);
415
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_DIV, div, true);
416
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_DIV_ROW, div_row, true);
417
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SCALE, scale, true);
418
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SCALE_4, scale_4, true);
419
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_TANH, tanh, true);
420
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_RELU, relu, true);
421
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GELU, gelu, true);
422
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GELU_QUICK, gelu_quick, true);
423
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SILU, silu, true);
424
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SOFT_MAX, soft_max, ctx->support_simdgroup_reduction);
425
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SOFT_MAX_4, soft_max_4, ctx->support_simdgroup_reduction);
426
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_DIAG_MASK_INF, diag_mask_inf, true);
427
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_DIAG_MASK_INF_8, diag_mask_inf_8, true);
428
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_F32, get_rows_f32, true);
429
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_F16, get_rows_f16, true);
430
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q4_0, get_rows_q4_0, true);
431
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q4_1, get_rows_q4_1, true);
432
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q5_0, get_rows_q5_0, true);
433
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q5_1, get_rows_q5_1, true);
434
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q8_0, get_rows_q8_0, true);
435
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q2_K, get_rows_q2_K, true);
436
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q3_K, get_rows_q3_K, true);
437
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q4_K, get_rows_q4_K, true);
438
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q5_K, get_rows_q5_K, true);
439
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q6_K, get_rows_q6_K, true);
440
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_XXS, get_rows_iq2_xxs, true);
441
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_XS, get_rows_iq2_xs, true);
442
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_I32, get_rows_i32, true);
443
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_RMS_NORM, rms_norm, ctx->support_simdgroup_reduction);
444
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GROUP_NORM, group_norm, ctx->support_simdgroup_reduction);
445
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_NORM, norm, true);
446
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_F32_F32, mul_mv_f32_f32, ctx->support_simdgroup_reduction);
447
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F16, mul_mv_f16_f16, ctx->support_simdgroup_reduction);
448
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F32, mul_mv_f16_f32, ctx->support_simdgroup_reduction);
449
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F32_1ROW, mul_mv_f16_f32_1row, ctx->support_simdgroup_reduction);
450
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F32_L4, mul_mv_f16_f32_l4, ctx->support_simdgroup_reduction);
451
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q4_0_F32, mul_mv_q4_0_f32, ctx->support_simdgroup_reduction);
452
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q4_1_F32, mul_mv_q4_1_f32, ctx->support_simdgroup_reduction);
453
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_0_F32, mul_mv_q5_0_f32, ctx->support_simdgroup_reduction);
454
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_1_F32, mul_mv_q5_1_f32, ctx->support_simdgroup_reduction);
455
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q8_0_F32, mul_mv_q8_0_f32, ctx->support_simdgroup_reduction);
456
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q2_K_F32, mul_mv_q2_K_f32, ctx->support_simdgroup_reduction);
457
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q3_K_F32, mul_mv_q3_K_f32, ctx->support_simdgroup_reduction);
458
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q4_K_F32, mul_mv_q4_K_f32, ctx->support_simdgroup_reduction);
459
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_K_F32, mul_mv_q5_K_f32, ctx->support_simdgroup_reduction);
460
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q6_K_F32, mul_mv_q6_K_f32, ctx->support_simdgroup_reduction);
461
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_XXS_F32, mul_mv_iq2_xxs_f32, ctx->support_simdgroup_reduction);
462
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_XS_F32, mul_mv_iq2_xs_f32, ctx->support_simdgroup_reduction);
463
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F32_F32, mul_mv_id_f32_f32, ctx->support_simdgroup_reduction);
464
+ //GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F16, mul_mv_id_f16_f16, ctx->support_simdgroup_reduction);
465
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F32, mul_mv_id_f16_f32, ctx->support_simdgroup_reduction);
466
+ //GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F32_1ROW, mul_mv_id_f16_f32_1row, ctx->support_simdgroup_reduction);
467
+ //GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F32_L4, mul_mv_id_f16_f32_l4, ctx->support_simdgroup_reduction);
468
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q4_0_F32, mul_mv_id_q4_0_f32, ctx->support_simdgroup_reduction);
469
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q4_1_F32, mul_mv_id_q4_1_f32, ctx->support_simdgroup_reduction);
470
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q5_0_F32, mul_mv_id_q5_0_f32, ctx->support_simdgroup_reduction);
471
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q5_1_F32, mul_mv_id_q5_1_f32, ctx->support_simdgroup_reduction);
472
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q8_0_F32, mul_mv_id_q8_0_f32, ctx->support_simdgroup_reduction);
473
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q2_K_F32, mul_mv_id_q2_K_f32, ctx->support_simdgroup_reduction);
474
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q3_K_F32, mul_mv_id_q3_K_f32, ctx->support_simdgroup_reduction);
475
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q4_K_F32, mul_mv_id_q4_K_f32, ctx->support_simdgroup_reduction);
476
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q5_K_F32, mul_mv_id_q5_K_f32, ctx->support_simdgroup_reduction);
477
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q6_K_F32, mul_mv_id_q6_K_f32, ctx->support_simdgroup_reduction);
478
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_XXS_F32, mul_mv_id_iq2_xxs_f32, ctx->support_simdgroup_reduction);
479
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_XS_F32, mul_mv_id_iq2_xs_f32, ctx->support_simdgroup_reduction);
480
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_F32_F32, mul_mm_f32_f32, ctx->support_simdgroup_mm);
481
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_F16_F32, mul_mm_f16_f32, ctx->support_simdgroup_mm);
482
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_0_F32, mul_mm_q4_0_f32, ctx->support_simdgroup_mm);
483
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_1_F32, mul_mm_q4_1_f32, ctx->support_simdgroup_mm);
484
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q5_0_F32, mul_mm_q5_0_f32, ctx->support_simdgroup_mm);
485
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q5_1_F32, mul_mm_q5_1_f32, ctx->support_simdgroup_mm);
486
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q8_0_F32, mul_mm_q8_0_f32, ctx->support_simdgroup_mm);
487
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q2_K_F32, mul_mm_q2_K_f32, ctx->support_simdgroup_mm);
488
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q3_K_F32, mul_mm_q3_K_f32, ctx->support_simdgroup_mm);
489
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_K_F32, mul_mm_q4_K_f32, ctx->support_simdgroup_mm);
490
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q5_K_F32, mul_mm_q5_K_f32, ctx->support_simdgroup_mm);
491
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q6_K_F32, mul_mm_q6_K_f32, ctx->support_simdgroup_mm);
492
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_XXS_F32, mul_mm_iq2_xxs_f32, ctx->support_simdgroup_mm);
493
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_XS_F32, mul_mm_iq2_xs_f32, ctx->support_simdgroup_mm);
494
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_F32_F32, mul_mm_id_f32_f32, ctx->support_simdgroup_mm);
495
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_F16_F32, mul_mm_id_f16_f32, ctx->support_simdgroup_mm);
496
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_0_F32, mul_mm_id_q4_0_f32, ctx->support_simdgroup_mm);
497
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_1_F32, mul_mm_id_q4_1_f32, ctx->support_simdgroup_mm);
498
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q5_0_F32, mul_mm_id_q5_0_f32, ctx->support_simdgroup_mm);
499
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q5_1_F32, mul_mm_id_q5_1_f32, ctx->support_simdgroup_mm);
500
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q8_0_F32, mul_mm_id_q8_0_f32, ctx->support_simdgroup_mm);
501
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q2_K_F32, mul_mm_id_q2_K_f32, ctx->support_simdgroup_mm);
502
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q3_K_F32, mul_mm_id_q3_K_f32, ctx->support_simdgroup_mm);
503
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_K_F32, mul_mm_id_q4_K_f32, ctx->support_simdgroup_mm);
504
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q5_K_F32, mul_mm_id_q5_K_f32, ctx->support_simdgroup_mm);
505
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q6_K_F32, mul_mm_id_q6_K_f32, ctx->support_simdgroup_mm);
506
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_XXS_F32, mul_mm_id_iq2_xxs_f32, ctx->support_simdgroup_mm);
507
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_XS_F32, mul_mm_id_iq2_xs_f32, ctx->support_simdgroup_mm);
508
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ROPE_F32, rope_f32, true);
509
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ROPE_F16, rope_f16, true);
510
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ALIBI_F32, alibi_f32, true);
511
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_IM2COL_F16, im2col_f16, true);
512
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_UPSCALE_F32, upscale_f32, true);
513
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_PAD_F32, pad_f32, true);
514
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ARGSORT_F32_I32_ASC, argsort_f32_i32_asc, true);
515
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ARGSORT_F32_I32_DESC, argsort_f32_i32_desc, true);
516
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_LEAKY_RELU_F32, leaky_relu_f32, true);
517
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_F16, cpy_f32_f16, true);
518
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_F32, cpy_f32_f32, true);
519
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_Q8_0, cpy_f32_q8_0, true);
520
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_Q4_0, cpy_f32_q4_0, true);
521
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_Q4_1, cpy_f32_q4_1, true);
522
+ //GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_0, cpy_f32_q5_0, true);
523
+ //GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_1, cpy_f32_q5_1, true);
524
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F16_F16, cpy_f16_f16, true);
525
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F16_F32, cpy_f16_f32, true);
526
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CONCAT, concat, true);
527
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SQR, sqr, true);
528
+ GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SUM_ROWS, sum_rows, true);
484
529
  }
485
530
 
486
531
  return ctx;
487
532
  }
488
533
 
489
- void ggml_metal_free(struct ggml_metal_context * ctx) {
534
+ static void ggml_metal_free(struct ggml_metal_context * ctx) {
490
535
  GGML_METAL_LOG_INFO("%s: deallocating\n", __func__);
491
- #define GGML_METAL_DEL_KERNEL(name) \
492
- [ctx->function_##name release]; \
493
- [ctx->pipeline_##name release];
494
-
495
- GGML_METAL_DEL_KERNEL(add);
496
- GGML_METAL_DEL_KERNEL(add_row);
497
- GGML_METAL_DEL_KERNEL(mul);
498
- GGML_METAL_DEL_KERNEL(mul_row);
499
- GGML_METAL_DEL_KERNEL(div);
500
- GGML_METAL_DEL_KERNEL(div_row);
501
- GGML_METAL_DEL_KERNEL(scale);
502
- GGML_METAL_DEL_KERNEL(scale_4);
503
- GGML_METAL_DEL_KERNEL(tanh);
504
- GGML_METAL_DEL_KERNEL(relu);
505
- GGML_METAL_DEL_KERNEL(gelu);
506
- GGML_METAL_DEL_KERNEL(gelu_quick);
507
- GGML_METAL_DEL_KERNEL(silu);
508
- GGML_METAL_DEL_KERNEL(soft_max);
509
- GGML_METAL_DEL_KERNEL(soft_max_4);
510
- GGML_METAL_DEL_KERNEL(diag_mask_inf);
511
- GGML_METAL_DEL_KERNEL(diag_mask_inf_8);
512
- GGML_METAL_DEL_KERNEL(get_rows_f32);
513
- GGML_METAL_DEL_KERNEL(get_rows_f16);
514
- GGML_METAL_DEL_KERNEL(get_rows_q4_0);
515
- GGML_METAL_DEL_KERNEL(get_rows_q4_1);
516
- GGML_METAL_DEL_KERNEL(get_rows_q5_0);
517
- GGML_METAL_DEL_KERNEL(get_rows_q5_1);
518
- GGML_METAL_DEL_KERNEL(get_rows_q8_0);
519
- GGML_METAL_DEL_KERNEL(get_rows_q2_K);
520
- GGML_METAL_DEL_KERNEL(get_rows_q3_K);
521
- GGML_METAL_DEL_KERNEL(get_rows_q4_K);
522
- GGML_METAL_DEL_KERNEL(get_rows_q5_K);
523
- GGML_METAL_DEL_KERNEL(get_rows_q6_K);
524
- GGML_METAL_DEL_KERNEL(get_rows_i32);
525
- GGML_METAL_DEL_KERNEL(get_rows_iq2_xxs);
526
- GGML_METAL_DEL_KERNEL(get_rows_iq2_xs);
527
- GGML_METAL_DEL_KERNEL(rms_norm);
528
- GGML_METAL_DEL_KERNEL(group_norm);
529
- GGML_METAL_DEL_KERNEL(norm);
530
- GGML_METAL_DEL_KERNEL(mul_mv_f32_f32);
531
- GGML_METAL_DEL_KERNEL(mul_mv_f16_f16);
532
- GGML_METAL_DEL_KERNEL(mul_mv_f16_f32);
533
- GGML_METAL_DEL_KERNEL(mul_mv_f16_f32_1row);
534
- GGML_METAL_DEL_KERNEL(mul_mv_f16_f32_l4);
535
- GGML_METAL_DEL_KERNEL(mul_mv_q4_0_f32);
536
- GGML_METAL_DEL_KERNEL(mul_mv_q4_1_f32);
537
- GGML_METAL_DEL_KERNEL(mul_mv_q5_0_f32);
538
- GGML_METAL_DEL_KERNEL(mul_mv_q5_1_f32);
539
- GGML_METAL_DEL_KERNEL(mul_mv_q8_0_f32);
540
- GGML_METAL_DEL_KERNEL(mul_mv_q2_K_f32);
541
- GGML_METAL_DEL_KERNEL(mul_mv_q3_K_f32);
542
- GGML_METAL_DEL_KERNEL(mul_mv_q4_K_f32);
543
- GGML_METAL_DEL_KERNEL(mul_mv_q5_K_f32);
544
- GGML_METAL_DEL_KERNEL(mul_mv_q6_K_f32);
545
- GGML_METAL_DEL_KERNEL(mul_mv_iq2_xxs_f32);
546
- GGML_METAL_DEL_KERNEL(mul_mv_iq2_xs_f32);
547
- GGML_METAL_DEL_KERNEL(mul_mv_id_f32_f32);
548
- //GGML_METAL_DEL_KERNEL(mul_mv_id_f16_f16);
549
- GGML_METAL_DEL_KERNEL(mul_mv_id_f16_f32);
550
- //GGML_METAL_DEL_KERNEL(mul_mv_id_f16_f32_1row);
551
- //GGML_METAL_DEL_KERNEL(mul_mv_id_f16_f32_l4);
552
- GGML_METAL_DEL_KERNEL(mul_mv_id_q4_0_f32);
553
- GGML_METAL_DEL_KERNEL(mul_mv_id_q4_1_f32);
554
- GGML_METAL_DEL_KERNEL(mul_mv_id_q5_0_f32);
555
- GGML_METAL_DEL_KERNEL(mul_mv_id_q5_1_f32);
556
- GGML_METAL_DEL_KERNEL(mul_mv_id_q8_0_f32);
557
- GGML_METAL_DEL_KERNEL(mul_mv_id_q2_K_f32);
558
- GGML_METAL_DEL_KERNEL(mul_mv_id_q3_K_f32);
559
- GGML_METAL_DEL_KERNEL(mul_mv_id_q4_K_f32);
560
- GGML_METAL_DEL_KERNEL(mul_mv_id_q5_K_f32);
561
- GGML_METAL_DEL_KERNEL(mul_mv_id_q6_K_f32);
562
- GGML_METAL_DEL_KERNEL(mul_mv_id_iq2_xxs_f32);
563
- GGML_METAL_DEL_KERNEL(mul_mv_id_iq2_xs_f32);
564
- if ([ctx->device supportsFamily:MTLGPUFamilyApple7]) {
565
- GGML_METAL_DEL_KERNEL(mul_mm_f32_f32);
566
- GGML_METAL_DEL_KERNEL(mul_mm_f16_f32);
567
- GGML_METAL_DEL_KERNEL(mul_mm_q4_0_f32);
568
- GGML_METAL_DEL_KERNEL(mul_mm_q4_1_f32);
569
- GGML_METAL_DEL_KERNEL(mul_mm_q5_0_f32);
570
- GGML_METAL_DEL_KERNEL(mul_mm_q5_1_f32);
571
- GGML_METAL_DEL_KERNEL(mul_mm_q8_0_f32);
572
- GGML_METAL_DEL_KERNEL(mul_mm_q2_K_f32);
573
- GGML_METAL_DEL_KERNEL(mul_mm_q3_K_f32);
574
- GGML_METAL_DEL_KERNEL(mul_mm_q4_K_f32);
575
- GGML_METAL_DEL_KERNEL(mul_mm_q5_K_f32);
576
- GGML_METAL_DEL_KERNEL(mul_mm_q6_K_f32);
577
- GGML_METAL_DEL_KERNEL(mul_mm_iq2_xxs_f32);
578
- GGML_METAL_DEL_KERNEL(mul_mm_iq2_xs_f32);
579
- GGML_METAL_DEL_KERNEL(mul_mm_id_f32_f32);
580
- GGML_METAL_DEL_KERNEL(mul_mm_id_f16_f32);
581
- GGML_METAL_DEL_KERNEL(mul_mm_id_q4_0_f32);
582
- GGML_METAL_DEL_KERNEL(mul_mm_id_q4_1_f32);
583
- GGML_METAL_DEL_KERNEL(mul_mm_id_q5_0_f32);
584
- GGML_METAL_DEL_KERNEL(mul_mm_id_q5_1_f32);
585
- GGML_METAL_DEL_KERNEL(mul_mm_id_q8_0_f32);
586
- GGML_METAL_DEL_KERNEL(mul_mm_id_q2_K_f32);
587
- GGML_METAL_DEL_KERNEL(mul_mm_id_q3_K_f32);
588
- GGML_METAL_DEL_KERNEL(mul_mm_id_q4_K_f32);
589
- GGML_METAL_DEL_KERNEL(mul_mm_id_q5_K_f32);
590
- GGML_METAL_DEL_KERNEL(mul_mm_id_q6_K_f32);
591
- GGML_METAL_DEL_KERNEL(mul_mm_id_iq2_xxs_f32);
592
- GGML_METAL_DEL_KERNEL(mul_mm_id_iq2_xs_f32);
593
- }
594
- GGML_METAL_DEL_KERNEL(rope_f32);
595
- GGML_METAL_DEL_KERNEL(rope_f16);
596
- GGML_METAL_DEL_KERNEL(alibi_f32);
597
- GGML_METAL_DEL_KERNEL(im2col_f16);
598
- GGML_METAL_DEL_KERNEL(upscale_f32);
599
- GGML_METAL_DEL_KERNEL(pad_f32);
600
- GGML_METAL_DEL_KERNEL(argsort_f32_i32_asc);
601
- GGML_METAL_DEL_KERNEL(argsort_f32_i32_desc);
602
- GGML_METAL_DEL_KERNEL(leaky_relu_f32);
603
- GGML_METAL_DEL_KERNEL(cpy_f32_f16);
604
- GGML_METAL_DEL_KERNEL(cpy_f32_f32);
605
- GGML_METAL_DEL_KERNEL(cpy_f32_q8_0);
606
- GGML_METAL_DEL_KERNEL(cpy_f32_q4_0);
607
- GGML_METAL_DEL_KERNEL(cpy_f32_q4_1);
608
- //GGML_METAL_DEL_KERNEL(cpy_f32_q5_0);
609
- //GGML_METAL_DEL_KERNEL(cpy_f32_q5_1);
610
- GGML_METAL_DEL_KERNEL(cpy_f16_f16);
611
- GGML_METAL_DEL_KERNEL(cpy_f16_f32);
612
- GGML_METAL_DEL_KERNEL(concat);
613
- GGML_METAL_DEL_KERNEL(sqr);
614
- GGML_METAL_DEL_KERNEL(sum_rows);
615
-
616
- #undef GGML_METAL_DEL_KERNEL
617
536
 
618
537
  for (int i = 0; i < ctx->n_buffers; ++i) {
619
538
  [ctx->buffers[i].metal release];
620
539
  }
621
540
 
541
+ for (int i = 0; i < GGML_METAL_MAX_KERNELS; ++i) {
542
+ if (ctx->kernels[i].pipeline) {
543
+ [ctx->kernels[i].pipeline release];
544
+ }
545
+
546
+ if (ctx->kernels[i].function) {
547
+ [ctx->kernels[i].function release];
548
+ }
549
+ }
550
+
622
551
  [ctx->library release];
623
552
  [ctx->queue release];
624
553
  [ctx->device release];
@@ -628,33 +557,6 @@ void ggml_metal_free(struct ggml_metal_context * ctx) {
628
557
  free(ctx);
629
558
  }
630
559
 
631
- void * ggml_metal_host_malloc(size_t n) {
632
- void * data = NULL;
633
- const int result = posix_memalign((void **) &data, sysconf(_SC_PAGESIZE), n);
634
- if (result != 0) {
635
- GGML_METAL_LOG_ERROR("%s: error: posix_memalign failed\n", __func__);
636
- return NULL;
637
- }
638
-
639
- return data;
640
- }
641
-
642
- void ggml_metal_host_free(void * data) {
643
- free(data);
644
- }
645
-
646
- void ggml_metal_set_n_cb(struct ggml_metal_context * ctx, int n_cb) {
647
- ctx->n_cb = MIN(n_cb, GGML_METAL_MAX_BUFFERS);
648
- }
649
-
650
- int ggml_metal_if_optimized(struct ggml_metal_context * ctx) {
651
- return ctx->concur_list_len;
652
- }
653
-
654
- int * ggml_metal_get_concur_list(struct ggml_metal_context * ctx) {
655
- return ctx->concur_list;
656
- }
657
-
658
560
  // temporarily defined here for compatibility between ggml-backend and the old API
659
561
 
660
562
  struct ggml_backend_metal_buffer {
@@ -727,210 +629,7 @@ static id<MTLBuffer> ggml_metal_get_buffer(struct ggml_metal_context * ctx, stru
727
629
  return nil;
728
630
  }
729
631
 
730
- bool ggml_metal_add_buffer(
731
- struct ggml_metal_context * ctx,
732
- const char * name,
733
- void * data,
734
- size_t size,
735
- size_t max_size) {
736
- if (ctx->n_buffers >= GGML_METAL_MAX_BUFFERS) {
737
- GGML_METAL_LOG_ERROR("%s: error: too many buffers\n", __func__);
738
- return false;
739
- }
740
-
741
- if (data) {
742
- // verify that the buffer does not overlap with any of the existing buffers
743
- for (int i = 0; i < ctx->n_buffers; ++i) {
744
- const int64_t ioffs = (int64_t) data - (int64_t) ctx->buffers[i].data;
745
-
746
- if (ioffs >= 0 && ioffs < (int64_t) ctx->buffers[i].size) {
747
- GGML_METAL_LOG_ERROR("%s: error: buffer '%s' overlaps with '%s'\n", __func__, name, ctx->buffers[i].name);
748
- return false;
749
- }
750
- }
751
-
752
- const size_t size_page = sysconf(_SC_PAGESIZE);
753
-
754
- size_t size_aligned = size;
755
- if ((size_aligned % size_page) != 0) {
756
- size_aligned += (size_page - (size_aligned % size_page));
757
- }
758
-
759
- // the buffer fits into the max buffer size allowed by the device
760
- if (size_aligned <= ctx->device.maxBufferLength) {
761
- ctx->buffers[ctx->n_buffers].name = name;
762
- ctx->buffers[ctx->n_buffers].data = data;
763
- ctx->buffers[ctx->n_buffers].size = size;
764
-
765
- ctx->buffers[ctx->n_buffers].metal = [ctx->device newBufferWithBytesNoCopy:data length:size_aligned options:MTLResourceStorageModeShared deallocator:nil];
766
-
767
- if (ctx->buffers[ctx->n_buffers].metal == nil) {
768
- GGML_METAL_LOG_ERROR("%s: error: failed to allocate '%-16s' buffer, size = %8.2f MiB\n", __func__, name, size_aligned / 1024.0 / 1024.0);
769
- return false;
770
- }
771
-
772
- GGML_METAL_LOG_INFO("%s: allocated '%-16s' buffer, size = %8.2f MiB", __func__, name, size_aligned / 1024.0 / 1024.0);
773
-
774
- ++ctx->n_buffers;
775
- } else {
776
- // this overlap between the views will guarantee that the tensor with the maximum size will fully fit into
777
- // one of the views
778
- const size_t size_ovlp = ((max_size + size_page - 1) / size_page + 1) * size_page; // round-up 2 pages just in case
779
- const size_t size_step = ctx->device.maxBufferLength - size_ovlp;
780
- const size_t size_view = ctx->device.maxBufferLength;
781
-
782
- for (size_t i = 0; i < size; i += size_step) {
783
- const size_t size_step_aligned = (i + size_view <= size) ? size_view : (size_aligned - i);
784
-
785
- ctx->buffers[ctx->n_buffers].name = name;
786
- ctx->buffers[ctx->n_buffers].data = (void *) ((uint8_t *) data + i);
787
- ctx->buffers[ctx->n_buffers].size = size_step_aligned;
788
-
789
- ctx->buffers[ctx->n_buffers].metal = [ctx->device newBufferWithBytesNoCopy:(void *) ((uint8_t *) data + i) length:size_step_aligned options:MTLResourceStorageModeShared deallocator:nil];
790
-
791
- if (ctx->buffers[ctx->n_buffers].metal == nil) {
792
- GGML_METAL_LOG_ERROR("%s: error: failed to allocate '%-16s' buffer, size = %8.2f MiB\n", __func__, name, size_step_aligned / 1024.0 / 1024.0);
793
- return false;
794
- }
795
-
796
- GGML_METAL_LOG_INFO("%s: allocated '%-16s' buffer, size = %8.2f MiB, offs = %12ld", __func__, name, size_step_aligned / 1024.0 / 1024.0, i);
797
- if (i + size_step < size) {
798
- GGML_METAL_LOG_INFO("\n");
799
- }
800
-
801
- ++ctx->n_buffers;
802
- }
803
- }
804
-
805
- #if TARGET_OS_OSX
806
- GGML_METAL_LOG_INFO(", (%8.2f / %8.2f)",
807
- ctx->device.currentAllocatedSize / 1024.0 / 1024.0,
808
- ctx->device.recommendedMaxWorkingSetSize / 1024.0 / 1024.0);
809
-
810
- if (ctx->device.currentAllocatedSize > ctx->device.recommendedMaxWorkingSetSize) {
811
- GGML_METAL_LOG_WARN("%s: warning: current allocated size is greater than the recommended max working set size\n", __func__);
812
- } else {
813
- GGML_METAL_LOG_INFO("\n");
814
- }
815
- #else
816
- GGML_METAL_LOG_INFO(", (%8.2f)\n", ctx->device.currentAllocatedSize / 1024.0 / 1024.0);
817
- #endif
818
- }
819
-
820
- return true;
821
- }
822
-
823
- void ggml_metal_set_tensor(
824
- struct ggml_metal_context * ctx,
825
- struct ggml_tensor * t) {
826
- size_t offs;
827
- id<MTLBuffer> id_dst = ggml_metal_get_buffer(ctx, t, &offs);
828
-
829
- memcpy((void *) ((uint8_t *) id_dst.contents + offs), t->data, ggml_nbytes(t));
830
- }
831
-
832
- void ggml_metal_get_tensor(
833
- struct ggml_metal_context * ctx,
834
- struct ggml_tensor * t) {
835
- size_t offs;
836
- id<MTLBuffer> id_src = ggml_metal_get_buffer(ctx, t, &offs);
837
-
838
- memcpy(t->data, (void *) ((uint8_t *) id_src.contents + offs), ggml_nbytes(t));
839
- }
840
-
841
- void ggml_metal_graph_find_concurrency(
842
- struct ggml_metal_context * ctx,
843
- struct ggml_cgraph * gf, bool check_mem) {
844
- int search_depth = gf->n_nodes; //we only find concurrency in this range to avoid wasting too much time
845
- int nodes_unused[GGML_MAX_CONCUR];
846
-
847
- for (int i = 0; i < GGML_MAX_CONCUR; i++) { ctx->concur_list[i] = 0; }
848
- for (int i = 0; i < gf->n_nodes; i++) { nodes_unused[i] = 1; }
849
- ctx->concur_list_len = 0;
850
-
851
- int n_left = gf->n_nodes;
852
- int n_start = 0; // all nodes before n_start at nodes_unused array have been sorted and store back to ctx->concur_list
853
- int level_pos = 0; // at ctx->concur_list, the last layer (level) ends at level_pos
854
-
855
- while (n_left > 0) {
856
- // number of nodes at a layer (that can be issued concurrently)
857
- int concurrency = 0;
858
- for (int i = n_start; i < ((n_start + search_depth > gf->n_nodes) ? gf->n_nodes : n_start + search_depth); i++) {
859
- if (nodes_unused[i]) {
860
- // if the requirements for gf->nodes[i] are satisfied
861
- int exe_flag = 1;
862
-
863
- // scan all srcs
864
- for (int src_ind = 0; src_ind < GGML_MAX_SRC; src_ind++) {
865
- struct ggml_tensor * src_cur = gf->nodes[i]->src[src_ind];
866
- if (src_cur) {
867
- // if is leaf nodes it's satisfied.
868
- // TODO: ggml_is_leaf()
869
- if (src_cur->op == GGML_OP_NONE && src_cur->grad == NULL) {
870
- continue;
871
- }
872
-
873
- // otherwise this src should be the output from previous nodes.
874
- int is_found = 0;
875
-
876
- // scan 2*search_depth back because we inserted barrier.
877
- //for (int j = ((level_pos - 2*search_depth) < 0 ? 0 : (level_pos - 2*search_depth)); j < level_pos; j++) {
878
- for (int j = MAX(0, level_pos - 2*search_depth); j < level_pos; j++) {
879
- if (ctx->concur_list[j] >= 0 && gf->nodes[ctx->concur_list[j]] == src_cur) {
880
- is_found = 1;
881
- break;
882
- }
883
- }
884
- if (is_found == 0) {
885
- exe_flag = 0;
886
- break;
887
- }
888
- }
889
- }
890
- if (exe_flag && check_mem) {
891
- // check if nodes[i]'s data will be overwritten by a node before nodes[i].
892
- // if node[5] and node[3] write to the same memory region, then we can't issue node[5] before node[3]
893
- int64_t data_start = (int64_t) gf->nodes[i]->data;
894
- int64_t length = (int64_t) ggml_nbytes(gf->nodes[i]);
895
- for (int j = n_start; j < i; j++) {
896
- if (nodes_unused[j] && gf->nodes[j]->op != GGML_OP_RESHAPE \
897
- && gf->nodes[j]->op != GGML_OP_VIEW \
898
- && gf->nodes[j]->op != GGML_OP_TRANSPOSE \
899
- && gf->nodes[j]->op != GGML_OP_PERMUTE) {
900
- if (((int64_t)gf->nodes[j]->data) >= data_start + length || \
901
- ((int64_t)gf->nodes[j]->data) + (int64_t) ggml_nbytes(gf->nodes[j]) <= data_start) {
902
- continue;
903
- }
904
-
905
- exe_flag = 0;
906
- }
907
- }
908
- }
909
- if (exe_flag) {
910
- ctx->concur_list[level_pos + concurrency] = i;
911
- nodes_unused[i] = 0;
912
- concurrency++;
913
- ctx->concur_list_len++;
914
- }
915
- }
916
- }
917
- n_left -= concurrency;
918
- // adding a barrier different layer
919
- ctx->concur_list[level_pos + concurrency] = -1;
920
- ctx->concur_list_len++;
921
- // jump all sorted nodes at nodes_bak
922
- while (!nodes_unused[n_start]) {
923
- n_start++;
924
- }
925
- level_pos += concurrency + 1;
926
- }
927
-
928
- if (ctx->concur_list_len > GGML_MAX_CONCUR) {
929
- GGML_METAL_LOG_WARN("%s: too many elements for metal ctx->concur_list!\n", __func__);
930
- }
931
- }
932
-
933
- static bool ggml_metal_supports_op(const struct ggml_tensor * op) {
632
+ static bool ggml_metal_supports_op(const struct ggml_metal_context * ctx, const struct ggml_tensor * op) {
934
633
  switch (op->op) {
935
634
  case GGML_OP_UNARY:
936
635
  switch (ggml_get_unary_op(op)) {
@@ -956,9 +655,11 @@ static bool ggml_metal_supports_op(const struct ggml_tensor * op) {
956
655
  case GGML_OP_SCALE:
957
656
  case GGML_OP_SQR:
958
657
  case GGML_OP_SUM_ROWS:
658
+ return true;
959
659
  case GGML_OP_SOFT_MAX:
960
660
  case GGML_OP_RMS_NORM:
961
661
  case GGML_OP_GROUP_NORM:
662
+ return ctx->support_simdgroup_reduction;
962
663
  case GGML_OP_NORM:
963
664
  case GGML_OP_ALIBI:
964
665
  case GGML_OP_ROPE:
@@ -967,9 +668,10 @@ static bool ggml_metal_supports_op(const struct ggml_tensor * op) {
967
668
  case GGML_OP_PAD:
968
669
  case GGML_OP_ARGSORT:
969
670
  case GGML_OP_LEAKY_RELU:
671
+ return true;
970
672
  case GGML_OP_MUL_MAT:
971
673
  case GGML_OP_MUL_MAT_ID:
972
- return true;
674
+ return ctx->support_simdgroup_reduction;
973
675
  case GGML_OP_CPY:
974
676
  case GGML_OP_DUP:
975
677
  case GGML_OP_CONT:
@@ -1007,1480 +709,1549 @@ static bool ggml_metal_supports_op(const struct ggml_tensor * op) {
1007
709
  return false;
1008
710
  }
1009
711
  }
1010
- bool ggml_metal_graph_compute(
712
+
713
+ static bool ggml_metal_graph_compute(
1011
714
  struct ggml_metal_context * ctx,
1012
715
  struct ggml_cgraph * gf) {
1013
716
  @autoreleasepool {
1014
717
 
1015
- // if there is ctx->concur_list, dispatch concurrently
1016
- // else fallback to serial dispatch
1017
718
  MTLComputePassDescriptor * edesc = MTLComputePassDescriptor.computePassDescriptor;
1018
-
1019
- const bool has_concur = ctx->concur_list_len && ctx->concur_list_len <= GGML_MAX_CONCUR;
1020
-
1021
- const int n_nodes = has_concur ? ctx->concur_list_len : gf->n_nodes;
1022
- edesc.dispatchType = has_concur ? MTLDispatchTypeConcurrent : MTLDispatchTypeSerial;
719
+ edesc.dispatchType = MTLDispatchTypeSerial;
1023
720
 
1024
721
  // create multiple command buffers and enqueue them
1025
722
  // then, we encode the graph into the command buffers in parallel
1026
723
 
724
+ const int n_nodes = gf->n_nodes;
1027
725
  const int n_cb = ctx->n_cb;
726
+ const int n_nodes_per_cb = (n_nodes + n_cb - 1) / n_cb;
1028
727
 
1029
- for (int i = 0; i < n_cb; ++i) {
1030
- ctx->command_buffers[i] = [ctx->queue commandBuffer];
728
+ id<MTLCommandBuffer> command_buffer_builder[n_cb];
729
+ for (int cb_idx = 0; cb_idx < n_cb; ++cb_idx) {
730
+ id<MTLCommandBuffer> command_buffer = [ctx->queue commandBufferWithUnretainedReferences];
731
+ command_buffer_builder[cb_idx] = command_buffer;
1031
732
 
1032
733
  // enqueue the command buffers in order to specify their execution order
1033
- [ctx->command_buffers[i] enqueue];
1034
-
1035
- ctx->command_encoders[i] = [ctx->command_buffers[i] computeCommandEncoderWithDescriptor: edesc];
734
+ [command_buffer enqueue];
1036
735
  }
736
+ const id<MTLCommandBuffer> *command_buffers = command_buffer_builder;
1037
737
 
1038
- for (int cb_idx = 0; cb_idx < n_cb; ++cb_idx) {
1039
- const int n_nodes_per_cb = (n_nodes + n_cb - 1) / n_cb;
1040
-
1041
- dispatch_async(ctx->d_queue, ^{
1042
- size_t offs_src0 = 0;
1043
- size_t offs_src1 = 0;
1044
- size_t offs_dst = 0;
1045
-
1046
- id<MTLCommandBuffer> command_buffer = ctx->command_buffers[cb_idx];
1047
- id<MTLComputeCommandEncoder> encoder = ctx->command_encoders[cb_idx];
1048
-
1049
- const int node_start = (cb_idx + 0) * n_nodes_per_cb;
1050
- const int node_end = MIN((cb_idx == n_cb - 1) ? n_nodes : (cb_idx + 1) * n_nodes_per_cb, n_nodes);
1051
-
1052
- for (int ind = node_start; ind < node_end; ++ind) {
1053
- const int i = has_concur ? ctx->concur_list[ind] : ind;
1054
-
1055
- if (i == -1) {
1056
- [encoder memoryBarrierWithScope:MTLBarrierScopeBuffers];
1057
- continue;
1058
- }
1059
-
1060
- //GGML_METAL_LOG_INFO("%s: encoding node %3d, op = %8s\n", __func__, i, ggml_op_name(gf->nodes[i]->op));
1061
-
1062
- struct ggml_tensor * src0 = gf->nodes[i]->src[0];
1063
- struct ggml_tensor * src1 = gf->nodes[i]->src[1];
1064
- struct ggml_tensor * dst = gf->nodes[i];
1065
-
1066
- switch (dst->op) {
1067
- case GGML_OP_NONE:
1068
- case GGML_OP_RESHAPE:
1069
- case GGML_OP_VIEW:
1070
- case GGML_OP_TRANSPOSE:
1071
- case GGML_OP_PERMUTE:
1072
- {
1073
- // noop -> next node
1074
- } continue;
1075
- default:
1076
- {
1077
- } break;
1078
- }
738
+ dispatch_apply(n_cb, ctx->d_queue, ^(size_t iter) {
739
+ const int cb_idx = iter;
1079
740
 
1080
- if (!ggml_metal_supports_op(dst)) {
1081
- GGML_METAL_LOG_ERROR("%s: error: unsupported op '%s'\n", __func__, ggml_op_desc(dst));
1082
- GGML_ASSERT(!"unsupported op");
1083
- }
1084
-
1085
- #ifndef GGML_METAL_NDEBUG
1086
- [encoder pushDebugGroup:[NSString stringWithCString:ggml_op_desc(dst) encoding:NSUTF8StringEncoding]];
1087
- #endif
741
+ size_t offs_src0 = 0;
742
+ size_t offs_src1 = 0;
743
+ size_t offs_dst = 0;
1088
744
 
1089
- const int64_t ne00 = src0 ? src0->ne[0] : 0;
1090
- const int64_t ne01 = src0 ? src0->ne[1] : 0;
1091
- const int64_t ne02 = src0 ? src0->ne[2] : 0;
1092
- const int64_t ne03 = src0 ? src0->ne[3] : 0;
1093
-
1094
- const uint64_t nb00 = src0 ? src0->nb[0] : 0;
1095
- const uint64_t nb01 = src0 ? src0->nb[1] : 0;
1096
- const uint64_t nb02 = src0 ? src0->nb[2] : 0;
1097
- const uint64_t nb03 = src0 ? src0->nb[3] : 0;
1098
-
1099
- const int64_t ne10 = src1 ? src1->ne[0] : 0;
1100
- const int64_t ne11 = src1 ? src1->ne[1] : 0;
1101
- const int64_t ne12 = src1 ? src1->ne[2] : 0;
1102
- const int64_t ne13 = src1 ? src1->ne[3] : 0; UNUSED(ne13);
1103
-
1104
- const uint64_t nb10 = src1 ? src1->nb[0] : 0;
1105
- const uint64_t nb11 = src1 ? src1->nb[1] : 0;
1106
- const uint64_t nb12 = src1 ? src1->nb[2] : 0;
1107
- const uint64_t nb13 = src1 ? src1->nb[3] : 0; UNUSED(nb13);
1108
-
1109
- const int64_t ne0 = dst ? dst->ne[0] : 0;
1110
- const int64_t ne1 = dst ? dst->ne[1] : 0;
1111
- const int64_t ne2 = dst ? dst->ne[2] : 0;
1112
- const int64_t ne3 = dst ? dst->ne[3] : 0;
1113
-
1114
- const uint64_t nb0 = dst ? dst->nb[0] : 0;
1115
- const uint64_t nb1 = dst ? dst->nb[1] : 0;
1116
- const uint64_t nb2 = dst ? dst->nb[2] : 0;
1117
- const uint64_t nb3 = dst ? dst->nb[3] : 0;
1118
-
1119
- const enum ggml_type src0t = src0 ? src0->type : GGML_TYPE_COUNT;
1120
- const enum ggml_type src1t = src1 ? src1->type : GGML_TYPE_COUNT;
1121
- const enum ggml_type dstt = dst ? dst->type : GGML_TYPE_COUNT;
1122
-
1123
- id<MTLBuffer> id_src0 = src0 ? ggml_metal_get_buffer(ctx, src0, &offs_src0) : nil;
1124
- id<MTLBuffer> id_src1 = src1 ? ggml_metal_get_buffer(ctx, src1, &offs_src1) : nil;
1125
- id<MTLBuffer> id_dst = dst ? ggml_metal_get_buffer(ctx, dst, &offs_dst) : nil;
1126
-
1127
- //GGML_METAL_LOG_INFO("%s: op - %s\n", __func__, ggml_op_name(dst->op));
1128
- //if (src0) {
1129
- // GGML_METAL_LOG_INFO("%s: src0 - %4s [%5lld, %5lld, %5lld], %d, %s\n", __func__, ggml_type_name(src0t), ne00, ne01, ne02,
1130
- // ggml_is_contiguous(src0), src0->name);
1131
- //}
1132
- //if (src1) {
1133
- // GGML_METAL_LOG_INFO("%s: src1 - %4s [%5lld, %5lld, %5lld], %d, %s\n", __func__, ggml_type_name(src1t), ne10, ne11, ne12,
1134
- // ggml_is_contiguous(src1), src1->name);
1135
- //}
1136
- //if (dst) {
1137
- // GGML_METAL_LOG_INFO("%s: dst - %4s [%5lld, %5lld, %5lld], 1, %s\n", __func__, ggml_type_name(dstt), ne0, ne1, ne2,
1138
- // dst->name);
1139
- //}
1140
-
1141
- switch (dst->op) {
1142
- case GGML_OP_CONCAT:
1143
- {
1144
- const int64_t nb = ne00;
1145
-
1146
- [encoder setComputePipelineState:ctx->pipeline_concat];
1147
- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
1148
- [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];
1149
- [encoder setBuffer:id_dst offset:offs_dst atIndex:2];
1150
- [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:3];
1151
- [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:4];
1152
- [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:5];
1153
- [encoder setBytes:&ne03 length:sizeof(ne03) atIndex:6];
1154
- [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:7];
1155
- [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:8];
1156
- [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:9];
1157
- [encoder setBytes:&nb03 length:sizeof(nb03) atIndex:10];
1158
- [encoder setBytes:&ne10 length:sizeof(ne10) atIndex:11];
1159
- [encoder setBytes:&ne11 length:sizeof(ne11) atIndex:12];
1160
- [encoder setBytes:&ne12 length:sizeof(ne12) atIndex:13];
1161
- [encoder setBytes:&ne13 length:sizeof(ne13) atIndex:14];
1162
- [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:15];
1163
- [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:16];
1164
- [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:17];
1165
- [encoder setBytes:&nb13 length:sizeof(nb13) atIndex:18];
1166
- [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:19];
1167
- [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:20];
1168
- [encoder setBytes:&ne2 length:sizeof(ne2) atIndex:21];
1169
- [encoder setBytes:&ne3 length:sizeof(ne3) atIndex:22];
1170
- [encoder setBytes:&nb0 length:sizeof(nb0) atIndex:23];
1171
- [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:24];
1172
- [encoder setBytes:&nb2 length:sizeof(nb2) atIndex:25];
1173
- [encoder setBytes:&nb3 length:sizeof(nb3) atIndex:26];
1174
- [encoder setBytes:&nb length:sizeof(nb) atIndex:27];
1175
-
1176
- const int nth = MIN(1024, ne0);
1177
-
1178
- [encoder dispatchThreadgroups:MTLSizeMake(ne1, ne2, ne3) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];
1179
- } break;
1180
- case GGML_OP_ADD:
1181
- case GGML_OP_MUL:
1182
- case GGML_OP_DIV:
1183
- {
1184
- const size_t offs = 0;
1185
-
1186
- bool bcast_row = false;
1187
-
1188
- int64_t nb = ne00;
745
+ id<MTLCommandBuffer> command_buffer = command_buffers[cb_idx];
746
+ id<MTLComputeCommandEncoder> encoder = [command_buffer computeCommandEncoderWithDescriptor: edesc];
1189
747
 
1190
- id<MTLComputePipelineState> pipeline = nil;
748
+ const int node_start = (cb_idx + 0) * n_nodes_per_cb;
749
+ const int node_end = MIN((cb_idx == n_cb - 1) ? n_nodes : (cb_idx + 1) * n_nodes_per_cb, n_nodes);
1191
750
 
1192
- if (ggml_nelements(src1) == ne10 && ggml_is_contiguous(src1) && ne00 % 4 == 0 && ne10 % 4 == 0) {
1193
- GGML_ASSERT(ggml_is_contiguous(src0));
751
+ for (int i = node_start; i < node_end; ++i) {
752
+ if (i == -1) {
753
+ [encoder memoryBarrierWithScope:MTLBarrierScopeBuffers];
754
+ continue;
755
+ }
1194
756
 
1195
- // src1 is a row
1196
- GGML_ASSERT(ne11 == 1);
757
+ //GGML_METAL_LOG_INFO("%s: encoding node %3d, op = %8s\n", __func__, i, ggml_op_name(gf->nodes[i]->op));
758
+
759
+ struct ggml_tensor * src0 = gf->nodes[i]->src[0];
760
+ struct ggml_tensor * src1 = gf->nodes[i]->src[1];
761
+ struct ggml_tensor * dst = gf->nodes[i];
762
+
763
+ switch (dst->op) {
764
+ case GGML_OP_NONE:
765
+ case GGML_OP_RESHAPE:
766
+ case GGML_OP_VIEW:
767
+ case GGML_OP_TRANSPOSE:
768
+ case GGML_OP_PERMUTE:
769
+ {
770
+ // noop -> next node
771
+ } continue;
772
+ default:
773
+ {
774
+ } break;
775
+ }
1197
776
 
1198
- nb = ne00 / 4;
1199
- switch (dst->op) {
1200
- case GGML_OP_ADD: pipeline = ctx->pipeline_add_row; break;
1201
- case GGML_OP_MUL: pipeline = ctx->pipeline_mul_row; break;
1202
- case GGML_OP_DIV: pipeline = ctx->pipeline_div_row; break;
1203
- default: GGML_ASSERT(false);
1204
- }
777
+ if (!ggml_metal_supports_op(ctx, dst)) {
778
+ GGML_METAL_LOG_ERROR("%s: error: unsupported op '%s'\n", __func__, ggml_op_desc(dst));
779
+ GGML_ASSERT(!"unsupported op");
780
+ }
1205
781
 
1206
- bcast_row = true;
1207
- } else {
1208
- switch (dst->op) {
1209
- case GGML_OP_ADD: pipeline = ctx->pipeline_add; break;
1210
- case GGML_OP_MUL: pipeline = ctx->pipeline_mul; break;
1211
- case GGML_OP_DIV: pipeline = ctx->pipeline_div; break;
1212
- default: GGML_ASSERT(false);
1213
- }
1214
- }
782
+ #ifndef GGML_METAL_NDEBUG
783
+ [encoder pushDebugGroup:[NSString stringWithCString:ggml_op_desc(dst) encoding:NSUTF8StringEncoding]];
784
+ #endif
1215
785
 
1216
- [encoder setComputePipelineState:pipeline];
1217
- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
1218
- [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];
1219
- [encoder setBuffer:id_dst offset:offs_dst atIndex:2];
1220
- [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:3];
1221
- [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:4];
1222
- [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:5];
1223
- [encoder setBytes:&ne03 length:sizeof(ne03) atIndex:6];
1224
- [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:7];
1225
- [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:8];
1226
- [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:9];
1227
- [encoder setBytes:&nb03 length:sizeof(nb03) atIndex:10];
1228
- [encoder setBytes:&ne10 length:sizeof(ne10) atIndex:11];
1229
- [encoder setBytes:&ne11 length:sizeof(ne11) atIndex:12];
1230
- [encoder setBytes:&ne12 length:sizeof(ne12) atIndex:13];
1231
- [encoder setBytes:&ne13 length:sizeof(ne13) atIndex:14];
1232
- [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:15];
1233
- [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:16];
1234
- [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:17];
1235
- [encoder setBytes:&nb13 length:sizeof(nb13) atIndex:18];
1236
- [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:19];
1237
- [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:20];
1238
- [encoder setBytes:&ne2 length:sizeof(ne2) atIndex:21];
1239
- [encoder setBytes:&ne3 length:sizeof(ne3) atIndex:22];
1240
- [encoder setBytes:&nb0 length:sizeof(nb0) atIndex:23];
1241
- [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:24];
1242
- [encoder setBytes:&nb2 length:sizeof(nb2) atIndex:25];
1243
- [encoder setBytes:&nb3 length:sizeof(nb3) atIndex:26];
1244
- [encoder setBytes:&offs length:sizeof(offs) atIndex:27];
1245
- [encoder setBytes:&nb length:sizeof(nb) atIndex:28];
1246
-
1247
- if (bcast_row) {
1248
- const int64_t n = ggml_nelements(dst)/4;
786
+ const int64_t ne00 = src0 ? src0->ne[0] : 0;
787
+ const int64_t ne01 = src0 ? src0->ne[1] : 0;
788
+ const int64_t ne02 = src0 ? src0->ne[2] : 0;
789
+ const int64_t ne03 = src0 ? src0->ne[3] : 0;
790
+
791
+ const uint64_t nb00 = src0 ? src0->nb[0] : 0;
792
+ const uint64_t nb01 = src0 ? src0->nb[1] : 0;
793
+ const uint64_t nb02 = src0 ? src0->nb[2] : 0;
794
+ const uint64_t nb03 = src0 ? src0->nb[3] : 0;
795
+
796
+ const int64_t ne10 = src1 ? src1->ne[0] : 0;
797
+ const int64_t ne11 = src1 ? src1->ne[1] : 0;
798
+ const int64_t ne12 = src1 ? src1->ne[2] : 0;
799
+ const int64_t ne13 = src1 ? src1->ne[3] : 0; UNUSED(ne13);
800
+
801
+ const uint64_t nb10 = src1 ? src1->nb[0] : 0;
802
+ const uint64_t nb11 = src1 ? src1->nb[1] : 0;
803
+ const uint64_t nb12 = src1 ? src1->nb[2] : 0;
804
+ const uint64_t nb13 = src1 ? src1->nb[3] : 0; UNUSED(nb13);
805
+
806
+ const int64_t ne0 = dst ? dst->ne[0] : 0;
807
+ const int64_t ne1 = dst ? dst->ne[1] : 0;
808
+ const int64_t ne2 = dst ? dst->ne[2] : 0;
809
+ const int64_t ne3 = dst ? dst->ne[3] : 0;
810
+
811
+ const uint64_t nb0 = dst ? dst->nb[0] : 0;
812
+ const uint64_t nb1 = dst ? dst->nb[1] : 0;
813
+ const uint64_t nb2 = dst ? dst->nb[2] : 0;
814
+ const uint64_t nb3 = dst ? dst->nb[3] : 0;
815
+
816
+ const enum ggml_type src0t = src0 ? src0->type : GGML_TYPE_COUNT;
817
+ const enum ggml_type src1t = src1 ? src1->type : GGML_TYPE_COUNT;
818
+ const enum ggml_type dstt = dst ? dst->type : GGML_TYPE_COUNT;
819
+
820
+ id<MTLBuffer> id_src0 = src0 ? ggml_metal_get_buffer(ctx, src0, &offs_src0) : nil;
821
+ id<MTLBuffer> id_src1 = src1 ? ggml_metal_get_buffer(ctx, src1, &offs_src1) : nil;
822
+ id<MTLBuffer> id_dst = dst ? ggml_metal_get_buffer(ctx, dst, &offs_dst) : nil;
823
+
824
+ //GGML_METAL_LOG_INFO("%s: op - %s\n", __func__, ggml_op_name(dst->op));
825
+ //if (src0) {
826
+ // GGML_METAL_LOG_INFO("%s: src0 - %4s [%5lld, %5lld, %5lld], %d, %s\n", __func__, ggml_type_name(src0t), ne00, ne01, ne02,
827
+ // ggml_is_contiguous(src0), src0->name);
828
+ //}
829
+ //if (src1) {
830
+ // GGML_METAL_LOG_INFO("%s: src1 - %4s [%5lld, %5lld, %5lld], %d, %s\n", __func__, ggml_type_name(src1t), ne10, ne11, ne12,
831
+ // ggml_is_contiguous(src1), src1->name);
832
+ //}
833
+ //if (dst) {
834
+ // GGML_METAL_LOG_INFO("%s: dst - %4s [%5lld, %5lld, %5lld], 1, %s\n", __func__, ggml_type_name(dstt), ne0, ne1, ne2,
835
+ // dst->name);
836
+ //}
837
+
838
+ switch (dst->op) {
839
+ case GGML_OP_CONCAT:
840
+ {
841
+ const int64_t nb = ne00;
842
+
843
+ id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CONCAT].pipeline;
844
+
845
+ [encoder setComputePipelineState:pipeline];
846
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
847
+ [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];
848
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:2];
849
+ [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:3];
850
+ [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:4];
851
+ [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:5];
852
+ [encoder setBytes:&ne03 length:sizeof(ne03) atIndex:6];
853
+ [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:7];
854
+ [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:8];
855
+ [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:9];
856
+ [encoder setBytes:&nb03 length:sizeof(nb03) atIndex:10];
857
+ [encoder setBytes:&ne10 length:sizeof(ne10) atIndex:11];
858
+ [encoder setBytes:&ne11 length:sizeof(ne11) atIndex:12];
859
+ [encoder setBytes:&ne12 length:sizeof(ne12) atIndex:13];
860
+ [encoder setBytes:&ne13 length:sizeof(ne13) atIndex:14];
861
+ [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:15];
862
+ [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:16];
863
+ [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:17];
864
+ [encoder setBytes:&nb13 length:sizeof(nb13) atIndex:18];
865
+ [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:19];
866
+ [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:20];
867
+ [encoder setBytes:&ne2 length:sizeof(ne2) atIndex:21];
868
+ [encoder setBytes:&ne3 length:sizeof(ne3) atIndex:22];
869
+ [encoder setBytes:&nb0 length:sizeof(nb0) atIndex:23];
870
+ [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:24];
871
+ [encoder setBytes:&nb2 length:sizeof(nb2) atIndex:25];
872
+ [encoder setBytes:&nb3 length:sizeof(nb3) atIndex:26];
873
+ [encoder setBytes:&nb length:sizeof(nb) atIndex:27];
874
+
875
+ const int nth = MIN(1024, ne0);
876
+
877
+ [encoder dispatchThreadgroups:MTLSizeMake(ne1, ne2, ne3) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];
878
+ } break;
879
+ case GGML_OP_ADD:
880
+ case GGML_OP_MUL:
881
+ case GGML_OP_DIV:
882
+ {
883
+ const size_t offs = 0;
884
+
885
+ bool bcast_row = false;
886
+
887
+ int64_t nb = ne00;
888
+
889
+ id<MTLComputePipelineState> pipeline = nil;
890
+
891
+ if (ggml_nelements(src1) == ne10 && ggml_is_contiguous(src1) && ne00 % 4 == 0 && ne10 % 4 == 0) {
892
+ GGML_ASSERT(ggml_is_contiguous(src0));
1249
893
 
1250
- [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
1251
- } else {
1252
- const int nth = MIN((int) pipeline.maxTotalThreadsPerThreadgroup, ne0);
894
+ // src1 is a row
895
+ GGML_ASSERT(ne11 == 1);
1253
896
 
1254
- [encoder dispatchThreadgroups:MTLSizeMake(ne01, ne02, ne03) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];
897
+ nb = ne00 / 4;
898
+ switch (dst->op) {
899
+ case GGML_OP_ADD: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_ADD_ROW].pipeline; break;
900
+ case GGML_OP_MUL: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_ROW].pipeline; break;
901
+ case GGML_OP_DIV: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_DIV_ROW].pipeline; break;
902
+ default: GGML_ASSERT(false);
1255
903
  }
1256
- } break;
1257
- case GGML_OP_ACC:
1258
- {
1259
- GGML_ASSERT(src0t == GGML_TYPE_F32);
1260
- GGML_ASSERT(src1t == GGML_TYPE_F32);
1261
- GGML_ASSERT(dstt == GGML_TYPE_F32);
1262
904
 
1263
- GGML_ASSERT(ggml_is_contiguous(src0));
1264
- GGML_ASSERT(ggml_is_contiguous(src1));
1265
-
1266
- const size_t pnb1 = ((int32_t *) dst->op_params)[0];
1267
- const size_t pnb2 = ((int32_t *) dst->op_params)[1];
1268
- const size_t pnb3 = ((int32_t *) dst->op_params)[2];
1269
- const size_t offs = ((int32_t *) dst->op_params)[3];
1270
-
1271
- const bool inplace = (bool) ((int32_t *) dst->op_params)[4];
1272
-
1273
- if (!inplace) {
1274
- // run a separete kernel to cpy src->dst
1275
- // not sure how to avoid this
1276
- // TODO: make a simpler cpy_bytes kernel
1277
-
1278
- const int nth = MIN((int) ctx->pipeline_cpy_f32_f32.maxTotalThreadsPerThreadgroup, ne00);
1279
-
1280
- [encoder setComputePipelineState:ctx->pipeline_cpy_f32_f32];
1281
- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
1282
- [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
1283
- [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:2];
1284
- [encoder setBytes:&ne01 length:sizeof( int64_t) atIndex:3];
1285
- [encoder setBytes:&ne02 length:sizeof( int64_t) atIndex:4];
1286
- [encoder setBytes:&ne03 length:sizeof( int64_t) atIndex:5];
1287
- [encoder setBytes:&nb00 length:sizeof(uint64_t) atIndex:6];
1288
- [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:7];
1289
- [encoder setBytes:&nb02 length:sizeof(uint64_t) atIndex:8];
1290
- [encoder setBytes:&nb03 length:sizeof(uint64_t) atIndex:9];
1291
- [encoder setBytes:&ne0 length:sizeof( int64_t) atIndex:10];
1292
- [encoder setBytes:&ne1 length:sizeof( int64_t) atIndex:11];
1293
- [encoder setBytes:&ne2 length:sizeof( int64_t) atIndex:12];
1294
- [encoder setBytes:&ne3 length:sizeof( int64_t) atIndex:13];
1295
- [encoder setBytes:&nb0 length:sizeof(uint64_t) atIndex:14];
1296
- [encoder setBytes:&nb1 length:sizeof(uint64_t) atIndex:15];
1297
- [encoder setBytes:&nb2 length:sizeof(uint64_t) atIndex:16];
1298
- [encoder setBytes:&nb3 length:sizeof(uint64_t) atIndex:17];
1299
-
1300
- [encoder dispatchThreadgroups:MTLSizeMake(ne01, ne02, ne03) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];
905
+ bcast_row = true;
906
+ } else {
907
+ switch (dst->op) {
908
+ case GGML_OP_ADD: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_ADD].pipeline; break;
909
+ case GGML_OP_MUL: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL].pipeline; break;
910
+ case GGML_OP_DIV: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_DIV].pipeline; break;
911
+ default: GGML_ASSERT(false);
1301
912
  }
913
+ }
1302
914
 
1303
- [encoder setComputePipelineState:ctx->pipeline_add];
1304
- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
1305
- [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];
1306
- [encoder setBuffer:id_dst offset:offs_dst atIndex:2];
1307
- [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:3];
1308
- [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:4];
1309
- [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:5];
1310
- [encoder setBytes:&ne03 length:sizeof(ne03) atIndex:6];
1311
- [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:7];
1312
- [encoder setBytes:&pnb1 length:sizeof(pnb1) atIndex:8];
1313
- [encoder setBytes:&pnb2 length:sizeof(pnb2) atIndex:9];
1314
- [encoder setBytes:&pnb3 length:sizeof(pnb3) atIndex:10];
1315
- [encoder setBytes:&ne10 length:sizeof(ne10) atIndex:11];
1316
- [encoder setBytes:&ne11 length:sizeof(ne11) atIndex:12];
1317
- [encoder setBytes:&ne12 length:sizeof(ne12) atIndex:13];
1318
- [encoder setBytes:&ne13 length:sizeof(ne13) atIndex:14];
1319
- [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:15];
1320
- [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:16];
1321
- [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:17];
1322
- [encoder setBytes:&nb13 length:sizeof(nb13) atIndex:18];
1323
- [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:19];
1324
- [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:20];
1325
- [encoder setBytes:&ne2 length:sizeof(ne2) atIndex:21];
1326
- [encoder setBytes:&ne3 length:sizeof(ne3) atIndex:22];
1327
- [encoder setBytes:&nb0 length:sizeof(nb0) atIndex:23];
1328
- [encoder setBytes:&pnb1 length:sizeof(pnb1) atIndex:24];
1329
- [encoder setBytes:&pnb2 length:sizeof(pnb2) atIndex:25];
1330
- [encoder setBytes:&pnb3 length:sizeof(pnb3) atIndex:26];
1331
- [encoder setBytes:&offs length:sizeof(offs) atIndex:27];
1332
-
1333
- const int nth = MIN((int) ctx->pipeline_add.maxTotalThreadsPerThreadgroup, ne00);
1334
-
1335
- [encoder dispatchThreadgroups:MTLSizeMake(ne11, ne12, ne13) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];
1336
- } break;
1337
- case GGML_OP_SCALE:
1338
- {
1339
- GGML_ASSERT(ggml_is_contiguous(src0));
915
+ [encoder setComputePipelineState:pipeline];
916
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
917
+ [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];
918
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:2];
919
+ [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:3];
920
+ [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:4];
921
+ [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:5];
922
+ [encoder setBytes:&ne03 length:sizeof(ne03) atIndex:6];
923
+ [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:7];
924
+ [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:8];
925
+ [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:9];
926
+ [encoder setBytes:&nb03 length:sizeof(nb03) atIndex:10];
927
+ [encoder setBytes:&ne10 length:sizeof(ne10) atIndex:11];
928
+ [encoder setBytes:&ne11 length:sizeof(ne11) atIndex:12];
929
+ [encoder setBytes:&ne12 length:sizeof(ne12) atIndex:13];
930
+ [encoder setBytes:&ne13 length:sizeof(ne13) atIndex:14];
931
+ [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:15];
932
+ [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:16];
933
+ [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:17];
934
+ [encoder setBytes:&nb13 length:sizeof(nb13) atIndex:18];
935
+ [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:19];
936
+ [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:20];
937
+ [encoder setBytes:&ne2 length:sizeof(ne2) atIndex:21];
938
+ [encoder setBytes:&ne3 length:sizeof(ne3) atIndex:22];
939
+ [encoder setBytes:&nb0 length:sizeof(nb0) atIndex:23];
940
+ [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:24];
941
+ [encoder setBytes:&nb2 length:sizeof(nb2) atIndex:25];
942
+ [encoder setBytes:&nb3 length:sizeof(nb3) atIndex:26];
943
+ [encoder setBytes:&offs length:sizeof(offs) atIndex:27];
944
+ [encoder setBytes:&nb length:sizeof(nb) atIndex:28];
945
+
946
+ if (bcast_row) {
947
+ const int64_t n = ggml_nelements(dst)/4;
1340
948
 
1341
- const float scale = *(const float *) dst->op_params;
949
+ [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
950
+ } else {
951
+ const int nth = MIN((int) pipeline.maxTotalThreadsPerThreadgroup, ne0);
1342
952
 
1343
- int64_t n = ggml_nelements(dst);
953
+ [encoder dispatchThreadgroups:MTLSizeMake(ne01, ne02, ne03) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];
954
+ }
955
+ } break;
956
+ case GGML_OP_ACC:
957
+ {
958
+ GGML_ASSERT(src0t == GGML_TYPE_F32);
959
+ GGML_ASSERT(src1t == GGML_TYPE_F32);
960
+ GGML_ASSERT(dstt == GGML_TYPE_F32);
1344
961
 
1345
- if (n % 4 == 0) {
1346
- n /= 4;
1347
- [encoder setComputePipelineState:ctx->pipeline_scale_4];
1348
- } else {
1349
- [encoder setComputePipelineState:ctx->pipeline_scale];
1350
- }
962
+ GGML_ASSERT(ggml_is_contiguous(src0));
963
+ GGML_ASSERT(ggml_is_contiguous(src1));
1351
964
 
1352
- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
1353
- [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
1354
- [encoder setBytes:&scale length:sizeof(scale) atIndex:2];
965
+ const size_t pnb1 = ((int32_t *) dst->op_params)[0];
966
+ const size_t pnb2 = ((int32_t *) dst->op_params)[1];
967
+ const size_t pnb3 = ((int32_t *) dst->op_params)[2];
968
+ const size_t offs = ((int32_t *) dst->op_params)[3];
1355
969
 
1356
- [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
1357
- } break;
1358
- case GGML_OP_UNARY:
1359
- switch (ggml_get_unary_op(gf->nodes[i])) {
1360
- case GGML_UNARY_OP_TANH:
1361
- {
1362
- [encoder setComputePipelineState:ctx->pipeline_tanh];
1363
- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
1364
- [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
970
+ const bool inplace = (bool) ((int32_t *) dst->op_params)[4];
1365
971
 
1366
- const int64_t n = ggml_nelements(dst);
972
+ if (!inplace) {
973
+ // run a separete kernel to cpy src->dst
974
+ // not sure how to avoid this
975
+ // TODO: make a simpler cpy_bytes kernel
1367
976
 
1368
- [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
1369
- } break;
1370
- case GGML_UNARY_OP_RELU:
1371
- {
1372
- [encoder setComputePipelineState:ctx->pipeline_relu];
1373
- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
1374
- [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
977
+ const id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_F32].pipeline;
1375
978
 
1376
- const int64_t n = ggml_nelements(dst);
979
+ [encoder setComputePipelineState:pipeline];
980
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
981
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
982
+ [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:2];
983
+ [encoder setBytes:&ne01 length:sizeof( int64_t) atIndex:3];
984
+ [encoder setBytes:&ne02 length:sizeof( int64_t) atIndex:4];
985
+ [encoder setBytes:&ne03 length:sizeof( int64_t) atIndex:5];
986
+ [encoder setBytes:&nb00 length:sizeof(uint64_t) atIndex:6];
987
+ [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:7];
988
+ [encoder setBytes:&nb02 length:sizeof(uint64_t) atIndex:8];
989
+ [encoder setBytes:&nb03 length:sizeof(uint64_t) atIndex:9];
990
+ [encoder setBytes:&ne0 length:sizeof( int64_t) atIndex:10];
991
+ [encoder setBytes:&ne1 length:sizeof( int64_t) atIndex:11];
992
+ [encoder setBytes:&ne2 length:sizeof( int64_t) atIndex:12];
993
+ [encoder setBytes:&ne3 length:sizeof( int64_t) atIndex:13];
994
+ [encoder setBytes:&nb0 length:sizeof(uint64_t) atIndex:14];
995
+ [encoder setBytes:&nb1 length:sizeof(uint64_t) atIndex:15];
996
+ [encoder setBytes:&nb2 length:sizeof(uint64_t) atIndex:16];
997
+ [encoder setBytes:&nb3 length:sizeof(uint64_t) atIndex:17];
1377
998
 
1378
- [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
1379
- } break;
1380
- case GGML_UNARY_OP_GELU:
1381
- {
1382
- [encoder setComputePipelineState:ctx->pipeline_gelu];
1383
- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
1384
- [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
999
+ const int nth = MIN((int) pipeline.maxTotalThreadsPerThreadgroup, ne00);
1385
1000
 
1386
- const int64_t n = ggml_nelements(dst);
1387
- GGML_ASSERT(n % 4 == 0);
1001
+ [encoder dispatchThreadgroups:MTLSizeMake(ne01, ne02, ne03) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];
1002
+ }
1388
1003
 
1389
- [encoder dispatchThreadgroups:MTLSizeMake(n/4, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
1390
- } break;
1391
- case GGML_UNARY_OP_GELU_QUICK:
1392
- {
1393
- [encoder setComputePipelineState:ctx->pipeline_gelu_quick];
1394
- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
1395
- [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
1004
+ const id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_ADD].pipeline;
1005
+
1006
+ [encoder setComputePipelineState:pipeline];
1007
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
1008
+ [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];
1009
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:2];
1010
+ [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:3];
1011
+ [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:4];
1012
+ [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:5];
1013
+ [encoder setBytes:&ne03 length:sizeof(ne03) atIndex:6];
1014
+ [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:7];
1015
+ [encoder setBytes:&pnb1 length:sizeof(pnb1) atIndex:8];
1016
+ [encoder setBytes:&pnb2 length:sizeof(pnb2) atIndex:9];
1017
+ [encoder setBytes:&pnb3 length:sizeof(pnb3) atIndex:10];
1018
+ [encoder setBytes:&ne10 length:sizeof(ne10) atIndex:11];
1019
+ [encoder setBytes:&ne11 length:sizeof(ne11) atIndex:12];
1020
+ [encoder setBytes:&ne12 length:sizeof(ne12) atIndex:13];
1021
+ [encoder setBytes:&ne13 length:sizeof(ne13) atIndex:14];
1022
+ [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:15];
1023
+ [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:16];
1024
+ [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:17];
1025
+ [encoder setBytes:&nb13 length:sizeof(nb13) atIndex:18];
1026
+ [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:19];
1027
+ [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:20];
1028
+ [encoder setBytes:&ne2 length:sizeof(ne2) atIndex:21];
1029
+ [encoder setBytes:&ne3 length:sizeof(ne3) atIndex:22];
1030
+ [encoder setBytes:&nb0 length:sizeof(nb0) atIndex:23];
1031
+ [encoder setBytes:&pnb1 length:sizeof(pnb1) atIndex:24];
1032
+ [encoder setBytes:&pnb2 length:sizeof(pnb2) atIndex:25];
1033
+ [encoder setBytes:&pnb3 length:sizeof(pnb3) atIndex:26];
1034
+ [encoder setBytes:&offs length:sizeof(offs) atIndex:27];
1035
+
1036
+ const int nth = MIN((int) pipeline.maxTotalThreadsPerThreadgroup, ne00);
1037
+
1038
+ [encoder dispatchThreadgroups:MTLSizeMake(ne11, ne12, ne13) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];
1039
+ } break;
1040
+ case GGML_OP_SCALE:
1041
+ {
1042
+ GGML_ASSERT(ggml_is_contiguous(src0));
1043
+
1044
+ const float scale = *(const float *) dst->op_params;
1045
+
1046
+ int64_t n = ggml_nelements(dst);
1047
+
1048
+ id<MTLComputePipelineState> pipeline = nil;
1049
+
1050
+ if (n % 4 == 0) {
1051
+ n /= 4;
1052
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SCALE_4].pipeline;
1053
+ } else {
1054
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SCALE].pipeline;
1055
+ }
1396
1056
 
1397
- const int64_t n = ggml_nelements(dst);
1398
- GGML_ASSERT(n % 4 == 0);
1057
+ [encoder setComputePipelineState:pipeline];
1058
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
1059
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
1060
+ [encoder setBytes:&scale length:sizeof(scale) atIndex:2];
1399
1061
 
1400
- [encoder dispatchThreadgroups:MTLSizeMake(n/4, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
1401
- } break;
1402
- case GGML_UNARY_OP_SILU:
1403
- {
1404
- [encoder setComputePipelineState:ctx->pipeline_silu];
1405
- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
1406
- [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
1062
+ [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
1063
+ } break;
1064
+ case GGML_OP_UNARY:
1065
+ switch (ggml_get_unary_op(gf->nodes[i])) {
1066
+ case GGML_UNARY_OP_TANH:
1067
+ {
1068
+ id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_TANH].pipeline;
1407
1069
 
1408
- const int64_t n = ggml_nelements(dst);
1409
- GGML_ASSERT(n % 4 == 0);
1070
+ [encoder setComputePipelineState:pipeline];
1071
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
1072
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
1410
1073
 
1411
- [encoder dispatchThreadgroups:MTLSizeMake(n/4, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
1412
- } break;
1413
- default:
1414
- {
1415
- GGML_METAL_LOG_WARN("%s: node %3d, op = %8s not implemented\n", __func__, i, ggml_op_name(dst->op));
1416
- GGML_ASSERT(false);
1417
- }
1418
- } break;
1419
- case GGML_OP_SQR:
1420
- {
1421
- GGML_ASSERT(ggml_is_contiguous(src0));
1074
+ const int64_t n = ggml_nelements(dst);
1422
1075
 
1423
- [encoder setComputePipelineState:ctx->pipeline_sqr];
1424
- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
1425
- [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
1076
+ [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
1077
+ } break;
1078
+ case GGML_UNARY_OP_RELU:
1079
+ {
1080
+ id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_RELU].pipeline;
1426
1081
 
1427
- const int64_t n = ggml_nelements(dst);
1428
- [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
1429
- } break;
1430
- case GGML_OP_SUM_ROWS:
1431
- {
1432
- GGML_ASSERT(src0->nb[0] == ggml_type_size(src0->type));
1082
+ [encoder setComputePipelineState:pipeline];
1083
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
1084
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
1433
1085
 
1434
- [encoder setComputePipelineState:ctx->pipeline_sum_rows];
1435
- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
1436
- [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
1437
- [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:2];
1438
- [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:3];
1439
- [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:4];
1440
- [encoder setBytes:&ne03 length:sizeof(ne03) atIndex:5];
1441
- [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:6];
1442
- [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:7];
1443
- [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:8];
1444
- [encoder setBytes:&nb03 length:sizeof(nb03) atIndex:9];
1445
- [encoder setBytes:&ne10 length:sizeof(ne10) atIndex:10];
1446
- [encoder setBytes:&ne11 length:sizeof(ne11) atIndex:11];
1447
- [encoder setBytes:&ne12 length:sizeof(ne12) atIndex:12];
1448
- [encoder setBytes:&ne13 length:sizeof(ne13) atIndex:13];
1449
- [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:14];
1450
- [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:15];
1451
- [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:16];
1452
- [encoder setBytes:&nb13 length:sizeof(nb13) atIndex:17];
1453
- [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:18];
1454
- [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:19];
1455
- [encoder setBytes:&ne2 length:sizeof(ne2) atIndex:20];
1456
- [encoder setBytes:&ne3 length:sizeof(ne3) atIndex:21];
1457
- [encoder setBytes:&nb0 length:sizeof(nb0) atIndex:22];
1458
- [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:23];
1459
- [encoder setBytes:&nb2 length:sizeof(nb2) atIndex:24];
1460
- [encoder setBytes:&nb3 length:sizeof(nb3) atIndex:25];
1461
-
1462
- [encoder dispatchThreadgroups:MTLSizeMake(ne01, ne02, ne03) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
1463
- } break;
1464
- case GGML_OP_SOFT_MAX:
1465
- {
1466
- int nth = 32; // SIMD width
1467
-
1468
- if (ne00%4 == 0) {
1469
- while (nth < ne00/4 && nth < 256) {
1470
- nth *= 2;
1471
- }
1472
- [encoder setComputePipelineState:ctx->pipeline_soft_max_4];
1473
- } else {
1474
- while (nth < ne00 && nth < 1024) {
1475
- nth *= 2;
1476
- }
1477
- [encoder setComputePipelineState:ctx->pipeline_soft_max];
1478
- }
1086
+ const int64_t n = ggml_nelements(dst);
1479
1087
 
1480
- const float scale = ((float *) dst->op_params)[0];
1088
+ [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
1089
+ } break;
1090
+ case GGML_UNARY_OP_GELU:
1091
+ {
1092
+ id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GELU].pipeline;
1481
1093
 
1482
- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
1483
- if (id_src1) {
1484
- [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];
1485
- } else {
1486
- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:1];
1487
- }
1488
- [encoder setBuffer:id_dst offset:offs_dst atIndex:2];
1489
- [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:3];
1490
- [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:4];
1491
- [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:5];
1492
- [encoder setBytes:&scale length:sizeof(scale) atIndex:6];
1493
- [encoder setThreadgroupMemoryLength:32*sizeof(float) atIndex:0];
1494
-
1495
- [encoder dispatchThreadgroups:MTLSizeMake(ne01*ne02*ne03, 1, 1) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];
1496
- } break;
1497
- case GGML_OP_DIAG_MASK_INF:
1498
- {
1499
- const int n_past = ((int32_t *)(dst->op_params))[0];
1500
-
1501
- if (ne00%8 == 0) {
1502
- [encoder setComputePipelineState:ctx->pipeline_diag_mask_inf_8];
1503
- } else {
1504
- [encoder setComputePipelineState:ctx->pipeline_diag_mask_inf];
1505
- }
1506
- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
1507
- [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
1508
- [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:2];
1509
- [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:3];
1510
- [encoder setBytes:&n_past length:sizeof(int) atIndex:4];
1094
+ [encoder setComputePipelineState:pipeline];
1095
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
1096
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
1511
1097
 
1512
- if (ne00%8 == 0) {
1513
- [encoder dispatchThreadgroups:MTLSizeMake(ne00*ne01*ne02/8, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
1514
- }
1515
- else {
1516
- [encoder dispatchThreadgroups:MTLSizeMake(ne00, ne01, ne02) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
1517
- }
1518
- } break;
1519
- case GGML_OP_MUL_MAT:
1520
- {
1521
- GGML_ASSERT(ne00 == ne10);
1098
+ const int64_t n = ggml_nelements(dst);
1099
+ GGML_ASSERT(n % 4 == 0);
1522
1100
 
1523
- // TODO: assert that dim2 and dim3 are contiguous
1524
- GGML_ASSERT(ne12 % ne02 == 0);
1525
- GGML_ASSERT(ne13 % ne03 == 0);
1101
+ [encoder dispatchThreadgroups:MTLSizeMake(n/4, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
1102
+ } break;
1103
+ case GGML_UNARY_OP_GELU_QUICK:
1104
+ {
1105
+ id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GELU_QUICK].pipeline;
1526
1106
 
1527
- const uint r2 = ne12/ne02;
1528
- const uint r3 = ne13/ne03;
1107
+ [encoder setComputePipelineState:pipeline];
1108
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
1109
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
1529
1110
 
1530
- // find the break-even point where the matrix-matrix kernel becomes more efficient compared
1531
- // to the matrix-vector kernel
1532
- int ne11_mm_min = 1;
1111
+ const int64_t n = ggml_nelements(dst);
1112
+ GGML_ASSERT(n % 4 == 0);
1113
+
1114
+ [encoder dispatchThreadgroups:MTLSizeMake(n/4, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
1115
+ } break;
1116
+ case GGML_UNARY_OP_SILU:
1117
+ {
1118
+ id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SILU].pipeline;
1119
+
1120
+ [encoder setComputePipelineState:pipeline];
1121
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
1122
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
1123
+
1124
+ const int64_t n = ggml_nelements(dst);
1125
+ GGML_ASSERT(n % 4 == 0);
1126
+
1127
+ [encoder dispatchThreadgroups:MTLSizeMake(n/4, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
1128
+ } break;
1129
+ default:
1130
+ {
1131
+ GGML_METAL_LOG_WARN("%s: node %3d, op = %8s not implemented\n", __func__, i, ggml_op_name(dst->op));
1132
+ GGML_ASSERT(false);
1133
+ }
1134
+ } break;
1135
+ case GGML_OP_SQR:
1136
+ {
1137
+ GGML_ASSERT(ggml_is_contiguous(src0));
1138
+
1139
+ id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SQR].pipeline;
1140
+
1141
+ [encoder setComputePipelineState:pipeline];
1142
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
1143
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
1144
+
1145
+ const int64_t n = ggml_nelements(dst);
1146
+
1147
+ [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
1148
+ } break;
1149
+ case GGML_OP_SUM_ROWS:
1150
+ {
1151
+ GGML_ASSERT(src0->nb[0] == ggml_type_size(src0->type));
1152
+
1153
+ id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SUM_ROWS].pipeline;
1154
+
1155
+ [encoder setComputePipelineState:pipeline];
1156
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
1157
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
1158
+ [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:2];
1159
+ [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:3];
1160
+ [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:4];
1161
+ [encoder setBytes:&ne03 length:sizeof(ne03) atIndex:5];
1162
+ [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:6];
1163
+ [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:7];
1164
+ [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:8];
1165
+ [encoder setBytes:&nb03 length:sizeof(nb03) atIndex:9];
1166
+ [encoder setBytes:&ne10 length:sizeof(ne10) atIndex:10];
1167
+ [encoder setBytes:&ne11 length:sizeof(ne11) atIndex:11];
1168
+ [encoder setBytes:&ne12 length:sizeof(ne12) atIndex:12];
1169
+ [encoder setBytes:&ne13 length:sizeof(ne13) atIndex:13];
1170
+ [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:14];
1171
+ [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:15];
1172
+ [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:16];
1173
+ [encoder setBytes:&nb13 length:sizeof(nb13) atIndex:17];
1174
+ [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:18];
1175
+ [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:19];
1176
+ [encoder setBytes:&ne2 length:sizeof(ne2) atIndex:20];
1177
+ [encoder setBytes:&ne3 length:sizeof(ne3) atIndex:21];
1178
+ [encoder setBytes:&nb0 length:sizeof(nb0) atIndex:22];
1179
+ [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:23];
1180
+ [encoder setBytes:&nb2 length:sizeof(nb2) atIndex:24];
1181
+ [encoder setBytes:&nb3 length:sizeof(nb3) atIndex:25];
1182
+
1183
+ [encoder dispatchThreadgroups:MTLSizeMake(ne01, ne02, ne03) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
1184
+ } break;
1185
+ case GGML_OP_SOFT_MAX:
1186
+ {
1187
+ int nth = 32; // SIMD width
1188
+
1189
+ id<MTLComputePipelineState> pipeline = nil;
1190
+
1191
+ if (ne00%4 == 0) {
1192
+ while (nth < ne00/4 && nth < 256) {
1193
+ nth *= 2;
1194
+ }
1195
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SOFT_MAX_4].pipeline;
1196
+ } else {
1197
+ while (nth < ne00 && nth < 1024) {
1198
+ nth *= 2;
1199
+ }
1200
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SOFT_MAX].pipeline;
1201
+ }
1202
+
1203
+ const float scale = ((float *) dst->op_params)[0];
1204
+
1205
+ [encoder setComputePipelineState:pipeline];
1206
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
1207
+ if (id_src1) {
1208
+ [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];
1209
+ } else {
1210
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:1];
1211
+ }
1212
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:2];
1213
+ [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:3];
1214
+ [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:4];
1215
+ [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:5];
1216
+ [encoder setBytes:&scale length:sizeof(scale) atIndex:6];
1217
+ [encoder setThreadgroupMemoryLength:32*sizeof(float) atIndex:0];
1218
+
1219
+ [encoder dispatchThreadgroups:MTLSizeMake(ne01*ne02*ne03, 1, 1) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];
1220
+ } break;
1221
+ case GGML_OP_DIAG_MASK_INF:
1222
+ {
1223
+ const int n_past = ((int32_t *)(dst->op_params))[0];
1224
+
1225
+ id<MTLComputePipelineState> pipeline = nil;
1226
+
1227
+ if (ne00%8 == 0) {
1228
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_DIAG_MASK_INF_8].pipeline;
1229
+ } else {
1230
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_DIAG_MASK_INF].pipeline;
1231
+ }
1232
+
1233
+ [encoder setComputePipelineState:pipeline];
1234
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
1235
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
1236
+ [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:2];
1237
+ [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:3];
1238
+ [encoder setBytes:&n_past length:sizeof(int) atIndex:4];
1239
+
1240
+ if (ne00%8 == 0) {
1241
+ [encoder dispatchThreadgroups:MTLSizeMake(ne00*ne01*ne02/8, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
1242
+ }
1243
+ else {
1244
+ [encoder dispatchThreadgroups:MTLSizeMake(ne00, ne01, ne02) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
1245
+ }
1246
+ } break;
1247
+ case GGML_OP_MUL_MAT:
1248
+ {
1249
+ GGML_ASSERT(ne00 == ne10);
1250
+
1251
+ // TODO: assert that dim2 and dim3 are contiguous
1252
+ GGML_ASSERT(ne12 % ne02 == 0);
1253
+ GGML_ASSERT(ne13 % ne03 == 0);
1254
+
1255
+ const uint r2 = ne12/ne02;
1256
+ const uint r3 = ne13/ne03;
1257
+
1258
+ // find the break-even point where the matrix-matrix kernel becomes more efficient compared
1259
+ // to the matrix-vector kernel
1260
+ int ne11_mm_min = 1;
1533
1261
 
1534
1262
  #if 0
1535
- // the numbers below are measured on M2 Ultra for 7B and 13B models
1536
- // these numbers do not translate to other devices or model sizes
1537
- // TODO: need to find a better approach
1538
- if ([ctx->device.name isEqualToString:@"Apple M2 Ultra"]) {
1539
- switch (src0t) {
1540
- case GGML_TYPE_F16: ne11_mm_min = 2; break;
1541
- case GGML_TYPE_Q8_0: ne11_mm_min = 7; break;
1542
- case GGML_TYPE_Q2_K: ne11_mm_min = 15; break;
1543
- case GGML_TYPE_Q3_K: ne11_mm_min = 7; break;
1544
- case GGML_TYPE_Q4_0:
1545
- case GGML_TYPE_Q4_1: ne11_mm_min = 15; break;
1546
- case GGML_TYPE_Q4_K: ne11_mm_min = 11; break;
1547
- case GGML_TYPE_Q5_0: // not tested yet
1548
- case GGML_TYPE_Q5_1: ne11_mm_min = 13; break; // not tested yet
1549
- case GGML_TYPE_Q5_K: ne11_mm_min = 7; break;
1550
- case GGML_TYPE_Q6_K: ne11_mm_min = 7; break;
1551
- default: ne11_mm_min = 1; break;
1552
- }
1263
+ // the numbers below are measured on M2 Ultra for 7B and 13B models
1264
+ // these numbers do not translate to other devices or model sizes
1265
+ // TODO: need to find a better approach
1266
+ if ([ctx->device.name isEqualToString:@"Apple M2 Ultra"]) {
1267
+ switch (src0t) {
1268
+ case GGML_TYPE_F16: ne11_mm_min = 2; break;
1269
+ case GGML_TYPE_Q8_0: ne11_mm_min = 7; break;
1270
+ case GGML_TYPE_Q2_K: ne11_mm_min = 15; break;
1271
+ case GGML_TYPE_Q3_K: ne11_mm_min = 7; break;
1272
+ case GGML_TYPE_Q4_0:
1273
+ case GGML_TYPE_Q4_1: ne11_mm_min = 15; break;
1274
+ case GGML_TYPE_Q4_K: ne11_mm_min = 11; break;
1275
+ case GGML_TYPE_Q5_0: // not tested yet
1276
+ case GGML_TYPE_Q5_1: ne11_mm_min = 13; break; // not tested yet
1277
+ case GGML_TYPE_Q5_K: ne11_mm_min = 7; break;
1278
+ case GGML_TYPE_Q6_K: ne11_mm_min = 7; break;
1279
+ default: ne11_mm_min = 1; break;
1553
1280
  }
1281
+ }
1554
1282
  #endif
1555
1283
 
1556
- // for now the matrix-matrix multiplication kernel only works on A14+/M1+ SoCs
1557
- // AMD GPU and older A-chips will reuse matrix-vector multiplication kernel
1558
- if ([ctx->device supportsFamily:MTLGPUFamilyApple7] &&
1559
- !ggml_is_transposed(src0) &&
1560
- !ggml_is_transposed(src1) &&
1561
- src1t == GGML_TYPE_F32 &&
1562
- ne00 % 32 == 0 && ne00 >= 64 &&
1563
- (ne11 > ne11_mm_min || (ggml_is_quantized(src0t) && ne12 > 1))) {
1564
- //printf("matrix: ne00 = %6d, ne01 = %6d, ne02 = %6d, ne11 = %6d, ne12 = %6d\n", ne00, ne01, ne02, ne11, ne12);
1565
- switch (src0->type) {
1566
- case GGML_TYPE_F32: [encoder setComputePipelineState:ctx->pipeline_mul_mm_f32_f32]; break;
1567
- case GGML_TYPE_F16: [encoder setComputePipelineState:ctx->pipeline_mul_mm_f16_f32]; break;
1568
- case GGML_TYPE_Q4_0: [encoder setComputePipelineState:ctx->pipeline_mul_mm_q4_0_f32]; break;
1569
- case GGML_TYPE_Q4_1: [encoder setComputePipelineState:ctx->pipeline_mul_mm_q4_1_f32]; break;
1570
- case GGML_TYPE_Q5_0: [encoder setComputePipelineState:ctx->pipeline_mul_mm_q5_0_f32]; break;
1571
- case GGML_TYPE_Q5_1: [encoder setComputePipelineState:ctx->pipeline_mul_mm_q5_1_f32]; break;
1572
- case GGML_TYPE_Q8_0: [encoder setComputePipelineState:ctx->pipeline_mul_mm_q8_0_f32]; break;
1573
- case GGML_TYPE_Q2_K: [encoder setComputePipelineState:ctx->pipeline_mul_mm_q2_K_f32]; break;
1574
- case GGML_TYPE_Q3_K: [encoder setComputePipelineState:ctx->pipeline_mul_mm_q3_K_f32]; break;
1575
- case GGML_TYPE_Q4_K: [encoder setComputePipelineState:ctx->pipeline_mul_mm_q4_K_f32]; break;
1576
- case GGML_TYPE_Q5_K: [encoder setComputePipelineState:ctx->pipeline_mul_mm_q5_K_f32]; break;
1577
- case GGML_TYPE_Q6_K: [encoder setComputePipelineState:ctx->pipeline_mul_mm_q6_K_f32]; break;
1578
- case GGML_TYPE_IQ2_XXS: [encoder setComputePipelineState:ctx->pipeline_mul_mm_iq2_xxs_f32]; break;
1579
- case GGML_TYPE_IQ2_XS : [encoder setComputePipelineState:ctx->pipeline_mul_mm_iq2_xs_f32]; break;
1580
- default: GGML_ASSERT(false && "MUL MAT-MAT not implemented");
1581
- }
1582
- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
1583
- [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];
1584
- [encoder setBuffer:id_dst offset:offs_dst atIndex:2];
1585
- [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:3];
1586
- [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:4];
1587
- [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:5];
1588
- [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:6];
1589
- [encoder setBytes:&ne12 length:sizeof(ne12) atIndex:7];
1590
- [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:8];
1591
- [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:9];
1592
- [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:10];
1593
- [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:11];
1594
- [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:12];
1595
- [encoder setBytes:&r2 length:sizeof(r2) atIndex:13];
1596
- [encoder setBytes:&r3 length:sizeof(r3) atIndex:14];
1597
- [encoder setThreadgroupMemoryLength:8192 atIndex:0];
1598
- [encoder dispatchThreadgroups:MTLSizeMake( (ne11 + 31)/32, (ne01 + 63)/64, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(128, 1, 1)];
1599
- } else {
1600
- int nth0 = 32;
1601
- int nth1 = 1;
1602
- int nrows = 1;
1603
- //printf("vector: ne00 = %6d, ne01 = %6d, ne02 = %6d, ne11 = %6d, ne12 = %6d\n", ne00, ne01, ne02, ne11, ne12);
1604
-
1605
- // use custom matrix x vector kernel
1606
- switch (src0t) {
1607
- case GGML_TYPE_F32:
1608
- {
1609
- GGML_ASSERT(src1t == GGML_TYPE_F32);
1610
- [encoder setComputePipelineState:ctx->pipeline_mul_mv_f32_f32];
1611
- nrows = 4;
1612
- } break;
1613
- case GGML_TYPE_F16:
1614
- {
1615
- nth0 = 32;
1616
- nth1 = 1;
1617
- if (src1t == GGML_TYPE_F32) {
1618
- if (ne11 * ne12 < 4) {
1619
- [encoder setComputePipelineState:ctx->pipeline_mul_mv_f16_f32_1row];
1620
- } else if (ne00 >= 128 && ne01 >= 8 && ne00%4 == 0) {
1621
- [encoder setComputePipelineState:ctx->pipeline_mul_mv_f16_f32_l4];
1622
- nrows = ne11;
1623
- } else {
1624
- [encoder setComputePipelineState:ctx->pipeline_mul_mv_f16_f32];
1625
- nrows = 4;
1626
- }
1284
+ // for now the matrix-matrix multiplication kernel only works on A14+/M1+ SoCs
1285
+ // AMD GPU and older A-chips will reuse matrix-vector multiplication kernel
1286
+ if ([ctx->device supportsFamily:MTLGPUFamilyApple7] &&
1287
+ !ggml_is_transposed(src0) &&
1288
+ !ggml_is_transposed(src1) &&
1289
+ src1t == GGML_TYPE_F32 &&
1290
+ ne00 % 32 == 0 && ne00 >= 64 &&
1291
+ (ne11 > ne11_mm_min || (ggml_is_quantized(src0t) && ne12 > 1))) {
1292
+ //printf("matrix: ne00 = %6d, ne01 = %6d, ne02 = %6d, ne11 = %6d, ne12 = %6d\n", ne00, ne01, ne02, ne11, ne12);
1293
+
1294
+ id<MTLComputePipelineState> pipeline = nil;
1295
+
1296
+ switch (src0->type) {
1297
+ case GGML_TYPE_F32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_F32_F32 ].pipeline; break;
1298
+ case GGML_TYPE_F16: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_F16_F32 ].pipeline; break;
1299
+ case GGML_TYPE_Q4_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_0_F32 ].pipeline; break;
1300
+ case GGML_TYPE_Q4_1: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_1_F32 ].pipeline; break;
1301
+ case GGML_TYPE_Q5_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q5_0_F32 ].pipeline; break;
1302
+ case GGML_TYPE_Q5_1: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q5_1_F32 ].pipeline; break;
1303
+ case GGML_TYPE_Q8_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q8_0_F32 ].pipeline; break;
1304
+ case GGML_TYPE_Q2_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q2_K_F32 ].pipeline; break;
1305
+ case GGML_TYPE_Q3_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q3_K_F32 ].pipeline; break;
1306
+ case GGML_TYPE_Q4_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_K_F32 ].pipeline; break;
1307
+ case GGML_TYPE_Q5_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q5_K_F32 ].pipeline; break;
1308
+ case GGML_TYPE_Q6_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q6_K_F32 ].pipeline; break;
1309
+ case GGML_TYPE_IQ2_XXS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_XXS_F32].pipeline; break;
1310
+ case GGML_TYPE_IQ2_XS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_XS_F32 ].pipeline; break;
1311
+ default: GGML_ASSERT(false && "MUL MAT-MAT not implemented");
1312
+ }
1313
+
1314
+ [encoder setComputePipelineState:pipeline];
1315
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
1316
+ [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];
1317
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:2];
1318
+ [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:3];
1319
+ [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:4];
1320
+ [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:5];
1321
+ [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:6];
1322
+ [encoder setBytes:&ne12 length:sizeof(ne12) atIndex:7];
1323
+ [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:8];
1324
+ [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:9];
1325
+ [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:10];
1326
+ [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:11];
1327
+ [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:12];
1328
+ [encoder setBytes:&r2 length:sizeof(r2) atIndex:13];
1329
+ [encoder setBytes:&r3 length:sizeof(r3) atIndex:14];
1330
+ [encoder setThreadgroupMemoryLength:8192 atIndex:0];
1331
+ [encoder dispatchThreadgroups:MTLSizeMake( (ne11 + 31)/32, (ne01 + 63)/64, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(128, 1, 1)];
1332
+ } else {
1333
+ int nth0 = 32;
1334
+ int nth1 = 1;
1335
+ int nrows = 1;
1336
+ //printf("vector: ne00 = %6d, ne01 = %6d, ne02 = %6d, ne11 = %6d, ne12 = %6d\n", ne00, ne01, ne02, ne11, ne12);
1337
+
1338
+ id<MTLComputePipelineState> pipeline = nil;
1339
+
1340
+ // use custom matrix x vector kernel
1341
+ switch (src0t) {
1342
+ case GGML_TYPE_F32:
1343
+ {
1344
+ GGML_ASSERT(src1t == GGML_TYPE_F32);
1345
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_F32_F32].pipeline;
1346
+ nrows = 4;
1347
+ } break;
1348
+ case GGML_TYPE_F16:
1349
+ {
1350
+ nth0 = 32;
1351
+ nth1 = 1;
1352
+ if (src1t == GGML_TYPE_F32) {
1353
+ if (ne11 * ne12 < 4) {
1354
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F32_1ROW].pipeline;
1355
+ } else if (ne00 >= 128 && ne01 >= 8 && ne00%4 == 0) {
1356
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F32_L4].pipeline;
1357
+ nrows = ne11;
1627
1358
  } else {
1628
- [encoder setComputePipelineState:ctx->pipeline_mul_mv_f16_f16];
1359
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F32].pipeline;
1629
1360
  nrows = 4;
1630
1361
  }
1631
- } break;
1632
- case GGML_TYPE_Q4_0:
1633
- {
1634
- nth0 = 8;
1635
- nth1 = 8;
1636
- [encoder setComputePipelineState:ctx->pipeline_mul_mv_q4_0_f32];
1637
- } break;
1638
- case GGML_TYPE_Q4_1:
1639
- {
1640
- nth0 = 8;
1641
- nth1 = 8;
1642
- [encoder setComputePipelineState:ctx->pipeline_mul_mv_q4_1_f32];
1643
- } break;
1644
- case GGML_TYPE_Q5_0:
1645
- {
1646
- nth0 = 8;
1647
- nth1 = 8;
1648
- [encoder setComputePipelineState:ctx->pipeline_mul_mv_q5_0_f32];
1649
- } break;
1650
- case GGML_TYPE_Q5_1:
1651
- {
1652
- nth0 = 8;
1653
- nth1 = 8;
1654
- [encoder setComputePipelineState:ctx->pipeline_mul_mv_q5_1_f32];
1655
- } break;
1656
- case GGML_TYPE_Q8_0:
1657
- {
1658
- nth0 = 8;
1659
- nth1 = 8;
1660
- [encoder setComputePipelineState:ctx->pipeline_mul_mv_q8_0_f32];
1661
- } break;
1662
- case GGML_TYPE_Q2_K:
1663
- {
1664
- nth0 = 2;
1665
- nth1 = 32;
1666
- [encoder setComputePipelineState:ctx->pipeline_mul_mv_q2_K_f32];
1667
- } break;
1668
- case GGML_TYPE_Q3_K:
1669
- {
1670
- nth0 = 2;
1671
- nth1 = 32;
1672
- [encoder setComputePipelineState:ctx->pipeline_mul_mv_q3_K_f32];
1673
- } break;
1674
- case GGML_TYPE_Q4_K:
1675
- {
1676
- nth0 = 4; //1;
1677
- nth1 = 8; //32;
1678
- [encoder setComputePipelineState:ctx->pipeline_mul_mv_q4_K_f32];
1679
- } break;
1680
- case GGML_TYPE_Q5_K:
1681
- {
1682
- nth0 = 2;
1683
- nth1 = 32;
1684
- [encoder setComputePipelineState:ctx->pipeline_mul_mv_q5_K_f32];
1685
- } break;
1686
- case GGML_TYPE_Q6_K:
1687
- {
1688
- nth0 = 2;
1689
- nth1 = 32;
1690
- [encoder setComputePipelineState:ctx->pipeline_mul_mv_q6_K_f32];
1691
- } break;
1692
- case GGML_TYPE_IQ2_XXS:
1693
- {
1694
- nth0 = 4;
1695
- nth1 = 16;
1696
- [encoder setComputePipelineState:ctx->pipeline_mul_mv_iq2_xxs_f32];
1697
- } break;
1698
- case GGML_TYPE_IQ2_XS:
1699
- {
1700
- nth0 = 4;
1701
- nth1 = 16;
1702
- [encoder setComputePipelineState:ctx->pipeline_mul_mv_iq2_xs_f32];
1703
- } break;
1704
- default:
1705
- {
1706
- GGML_METAL_LOG_ERROR("Asserting on type %d\n", (int)src0t);
1707
- GGML_ASSERT(false && "not implemented");
1362
+ } else {
1363
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F16].pipeline;
1364
+ nrows = 4;
1708
1365
  }
1709
- };
1710
-
1711
- if (ggml_is_quantized(src0t)) {
1712
- GGML_ASSERT(ne00 >= nth0*nth1);
1713
- }
1366
+ } break;
1367
+ case GGML_TYPE_Q4_0:
1368
+ {
1369
+ nth0 = 8;
1370
+ nth1 = 8;
1371
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q4_0_F32].pipeline;
1372
+ } break;
1373
+ case GGML_TYPE_Q4_1:
1374
+ {
1375
+ nth0 = 8;
1376
+ nth1 = 8;
1377
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q4_1_F32].pipeline;
1378
+ } break;
1379
+ case GGML_TYPE_Q5_0:
1380
+ {
1381
+ nth0 = 8;
1382
+ nth1 = 8;
1383
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_0_F32].pipeline;
1384
+ } break;
1385
+ case GGML_TYPE_Q5_1:
1386
+ {
1387
+ nth0 = 8;
1388
+ nth1 = 8;
1389
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_1_F32].pipeline;
1390
+ } break;
1391
+ case GGML_TYPE_Q8_0:
1392
+ {
1393
+ nth0 = 8;
1394
+ nth1 = 8;
1395
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q8_0_F32].pipeline;
1396
+ } break;
1397
+ case GGML_TYPE_Q2_K:
1398
+ {
1399
+ nth0 = 2;
1400
+ nth1 = 32;
1401
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q2_K_F32].pipeline;
1402
+ } break;
1403
+ case GGML_TYPE_Q3_K:
1404
+ {
1405
+ nth0 = 2;
1406
+ nth1 = 32;
1407
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q3_K_F32].pipeline;
1408
+ } break;
1409
+ case GGML_TYPE_Q4_K:
1410
+ {
1411
+ nth0 = 4; //1;
1412
+ nth1 = 8; //32;
1413
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q4_K_F32].pipeline;
1414
+ } break;
1415
+ case GGML_TYPE_Q5_K:
1416
+ {
1417
+ nth0 = 2;
1418
+ nth1 = 32;
1419
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_K_F32].pipeline;
1420
+ } break;
1421
+ case GGML_TYPE_Q6_K:
1422
+ {
1423
+ nth0 = 2;
1424
+ nth1 = 32;
1425
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q6_K_F32].pipeline;
1426
+ } break;
1427
+ case GGML_TYPE_IQ2_XXS:
1428
+ {
1429
+ nth0 = 4;
1430
+ nth1 = 16;
1431
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_XXS_F32].pipeline;
1432
+ } break;
1433
+ case GGML_TYPE_IQ2_XS:
1434
+ {
1435
+ nth0 = 4;
1436
+ nth1 = 16;
1437
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_XS_F32].pipeline;
1438
+ } break;
1439
+ default:
1440
+ {
1441
+ GGML_METAL_LOG_ERROR("Asserting on type %d\n", (int)src0t);
1442
+ GGML_ASSERT(false && "not implemented");
1443
+ }
1444
+ };
1714
1445
 
1715
- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
1716
- [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];
1717
- [encoder setBuffer:id_dst offset:offs_dst atIndex:2];
1718
- [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:3];
1719
- [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:4];
1720
- [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:5];
1721
- [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:6];
1722
- [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:7];
1723
- [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:8];
1724
- [encoder setBytes:&ne10 length:sizeof(ne10) atIndex:9];
1725
- [encoder setBytes:&ne11 length:sizeof(ne11) atIndex:10];
1726
- [encoder setBytes:&ne12 length:sizeof(ne12) atIndex:11];
1727
- [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:12];
1728
- [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:13];
1729
- [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:14];
1730
- [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:15];
1731
- [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:16];
1732
- [encoder setBytes:&r2 length:sizeof(r2) atIndex:17];
1733
- [encoder setBytes:&r3 length:sizeof(r3) atIndex:18];
1734
-
1735
- if (src0t == GGML_TYPE_Q4_0 || src0t == GGML_TYPE_Q4_1 ||
1736
- src0t == GGML_TYPE_Q5_0 || src0t == GGML_TYPE_Q5_1 || src0t == GGML_TYPE_Q8_0 ||
1737
- src0t == GGML_TYPE_Q2_K) { // || src0t == GGML_TYPE_Q4_K) {
1738
- [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 7)/8, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];
1739
- }
1740
- else if (src0t == GGML_TYPE_IQ2_XXS || src0t == GGML_TYPE_IQ2_XS) {
1741
- const int mem_size = src0t == GGML_TYPE_IQ2_XXS ? 256*8+128 : 512*8+128;
1742
- [encoder setThreadgroupMemoryLength:mem_size atIndex:0];
1743
- [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 7)/8, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];
1744
- }
1745
- else if (src0t == GGML_TYPE_Q4_K) {
1746
- [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 3)/4, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];
1747
- }
1748
- else if (src0t == GGML_TYPE_Q3_K) {
1749
- #ifdef GGML_QKK_64
1750
- [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 1)/2, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];
1751
- #else
1752
- [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 3)/4, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];
1753
- #endif
1754
- }
1755
- else if (src0t == GGML_TYPE_Q5_K) {
1756
- [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 3)/4, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];
1757
- }
1758
- else if (src0t == GGML_TYPE_Q6_K) {
1759
- [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 1)/2, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];
1760
- } else {
1761
- const int64_t ny = (ne11 + nrows - 1)/nrows;
1762
- [encoder dispatchThreadgroups:MTLSizeMake(ne01, ny, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];
1763
- }
1446
+ if (ggml_is_quantized(src0t)) {
1447
+ GGML_ASSERT(ne00 >= nth0*nth1);
1764
1448
  }
1765
- } break;
1766
- case GGML_OP_MUL_MAT_ID:
1767
- {
1768
- //GGML_ASSERT(ne00 == ne10);
1769
- //GGML_ASSERT(ne03 == ne13);
1770
-
1771
- GGML_ASSERT(src0t == GGML_TYPE_I32);
1772
-
1773
- const int n_as = ((int32_t *) dst->op_params)[1];
1774
-
1775
- // TODO: make this more general
1776
- GGML_ASSERT(n_as <= 8);
1777
-
1778
- // max size of the src1ids array in the kernel stack
1779
- GGML_ASSERT(ne11 <= 512);
1780
-
1781
- struct ggml_tensor * src2 = gf->nodes[i]->src[2];
1782
-
1783
- const int64_t ne20 = src2 ? src2->ne[0] : 0;
1784
- const int64_t ne21 = src2 ? src2->ne[1] : 0;
1785
- const int64_t ne22 = src2 ? src2->ne[2] : 0;
1786
- const int64_t ne23 = src2 ? src2->ne[3] : 0; GGML_UNUSED(ne23);
1787
-
1788
- const uint64_t nb20 = src2 ? src2->nb[0] : 0; GGML_UNUSED(nb20);
1789
- const uint64_t nb21 = src2 ? src2->nb[1] : 0;
1790
- const uint64_t nb22 = src2 ? src2->nb[2] : 0;
1791
- const uint64_t nb23 = src2 ? src2->nb[3] : 0; GGML_UNUSED(nb23);
1792
-
1793
- const enum ggml_type src2t = src2 ? src2->type : GGML_TYPE_COUNT; GGML_UNUSED(src2t);
1794
-
1795
- GGML_ASSERT(!ggml_is_transposed(src2));
1796
- GGML_ASSERT(!ggml_is_transposed(src1));
1797
-
1798
- GGML_ASSERT(src1t == GGML_TYPE_F32);
1799
-
1800
- const uint r2 = ne12/ne22;
1801
- const uint r3 = ne13/ne23;
1802
-
1803
- // find the break-even point where the matrix-matrix kernel becomes more efficient compared
1804
- // to the matrix-vector kernel
1805
- int ne11_mm_min = n_as;
1806
-
1807
- const int idx = ((int32_t *) dst->op_params)[0];
1808
-
1809
- // batch size
1810
- GGML_ASSERT(ne01 == ne11);
1811
-
1812
- // for now the matrix-matrix multiplication kernel only works on A14+/M1+ SoCs
1813
- // AMD GPU and older A-chips will reuse matrix-vector multiplication kernel
1814
- // !!!
1815
- // TODO: for now, always use mat-vec kernels until we figure out how to improve the
1816
- // indirect matrix multiplication
1817
- // !!!
1818
- if ([ctx->device supportsFamily:MTLGPUFamilyApple7] &&
1819
- ne20 % 32 == 0 && ne20 >= 64 &&
1820
- ne11 > ne11_mm_min) {
1821
- switch (src2->type) {
1822
- case GGML_TYPE_F32: [encoder setComputePipelineState:ctx->pipeline_mul_mm_id_f32_f32]; break;
1823
- case GGML_TYPE_F16: [encoder setComputePipelineState:ctx->pipeline_mul_mm_id_f16_f32]; break;
1824
- case GGML_TYPE_Q4_0: [encoder setComputePipelineState:ctx->pipeline_mul_mm_id_q4_0_f32]; break;
1825
- case GGML_TYPE_Q4_1: [encoder setComputePipelineState:ctx->pipeline_mul_mm_id_q4_1_f32]; break;
1826
- case GGML_TYPE_Q5_0: [encoder setComputePipelineState:ctx->pipeline_mul_mm_id_q5_0_f32]; break;
1827
- case GGML_TYPE_Q5_1: [encoder setComputePipelineState:ctx->pipeline_mul_mm_id_q5_1_f32]; break;
1828
- case GGML_TYPE_Q8_0: [encoder setComputePipelineState:ctx->pipeline_mul_mm_id_q8_0_f32]; break;
1829
- case GGML_TYPE_Q2_K: [encoder setComputePipelineState:ctx->pipeline_mul_mm_id_q2_K_f32]; break;
1830
- case GGML_TYPE_Q3_K: [encoder setComputePipelineState:ctx->pipeline_mul_mm_id_q3_K_f32]; break;
1831
- case GGML_TYPE_Q4_K: [encoder setComputePipelineState:ctx->pipeline_mul_mm_id_q4_K_f32]; break;
1832
- case GGML_TYPE_Q5_K: [encoder setComputePipelineState:ctx->pipeline_mul_mm_id_q5_K_f32]; break;
1833
- case GGML_TYPE_Q6_K: [encoder setComputePipelineState:ctx->pipeline_mul_mm_id_q6_K_f32]; break;
1834
- case GGML_TYPE_IQ2_XXS: [encoder setComputePipelineState:ctx->pipeline_mul_mm_id_iq2_xxs_f32]; break;
1835
- case GGML_TYPE_IQ2_XS : [encoder setComputePipelineState:ctx->pipeline_mul_mm_id_iq2_xs_f32]; break;
1836
- default: GGML_ASSERT(false && "MUL_MAT_ID not implemented");
1837
- }
1838
- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
1839
- [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];
1840
- [encoder setBuffer:id_dst offset:offs_dst atIndex:2];
1841
- [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:3];
1842
- [encoder setBytes:&ne20 length:sizeof(ne20) atIndex:4];
1843
- [encoder setBytes:&ne22 length:sizeof(ne22) atIndex:5];
1844
- [encoder setBytes:&nb21 length:sizeof(nb21) atIndex:6];
1845
- [encoder setBytes:&nb22 length:sizeof(nb22) atIndex:7];
1846
- [encoder setBytes:&ne12 length:sizeof(ne12) atIndex:8];
1847
- [encoder setBytes:&ne13 length:sizeof(ne13) atIndex:9];
1848
- [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:10];
1849
- [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:11];
1850
- [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:12];
1851
- [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:13];
1852
- [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:14];
1853
- [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:15];
1854
- [encoder setBytes:&r2 length:sizeof(r2) atIndex:16];
1855
- [encoder setBytes:&r3 length:sizeof(r3) atIndex:17];
1856
- [encoder setBytes:&idx length:sizeof(idx) atIndex:18];
1857
- // TODO: how to make this an array? read Metal docs
1858
- for (int j = 0; j < 8; ++j) {
1859
- // NOTE: this is done like this to avoid uninitialized kernel arguments when n_as < 8
1860
- struct ggml_tensor * src_cur = dst->src[2 + (j % n_as)];
1861
-
1862
- size_t offs_src_cur = 0;
1863
- id<MTLBuffer> id_src_cur = ggml_metal_get_buffer(ctx, src_cur, &offs_src_cur);
1864
-
1865
- [encoder setBuffer:id_src_cur offset:offs_src_cur atIndex:19 + j];
1866
- }
1867
-
1868
- [encoder setThreadgroupMemoryLength:8192 atIndex:0];
1869
-
1870
- [encoder dispatchThreadgroups:MTLSizeMake((ne11 + 31)/32, (ne21 + 63)/64, n_as*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(128, 1, 1)];
1871
- } else {
1872
- int nth0 = 32;
1873
- int nth1 = 1;
1874
- int nrows = 1;
1875
- //printf("vector: ne00 = %6d, ne01 = %6d, ne02 = %6d, ne11 = %6d, ne12 = %6d\n", ne00, ne01, ne02, ne11, ne12);
1876
-
1877
- // use custom matrix x vector kernel
1878
- switch (src2t) {
1879
- case GGML_TYPE_F32:
1880
- {
1881
- GGML_ASSERT(src1t == GGML_TYPE_F32);
1882
- [encoder setComputePipelineState:ctx->pipeline_mul_mv_id_f32_f32];
1883
- } break;
1884
- case GGML_TYPE_F16:
1885
- {
1886
- GGML_ASSERT(src1t == GGML_TYPE_F32);
1887
- nth0 = 32;
1888
- nth1 = 1;
1889
- [encoder setComputePipelineState:ctx->pipeline_mul_mv_id_f16_f32];
1890
- } break;
1891
- case GGML_TYPE_Q4_0:
1892
- {
1893
- nth0 = 8;
1894
- nth1 = 8;
1895
- [encoder setComputePipelineState:ctx->pipeline_mul_mv_id_q4_0_f32];
1896
- } break;
1897
- case GGML_TYPE_Q4_1:
1898
- {
1899
- nth0 = 8;
1900
- nth1 = 8;
1901
- [encoder setComputePipelineState:ctx->pipeline_mul_mv_id_q4_1_f32];
1902
- } break;
1903
- case GGML_TYPE_Q5_0:
1904
- {
1905
- nth0 = 8;
1906
- nth1 = 8;
1907
- [encoder setComputePipelineState:ctx->pipeline_mul_mv_id_q5_0_f32];
1908
- } break;
1909
- case GGML_TYPE_Q5_1:
1910
- {
1911
- nth0 = 8;
1912
- nth1 = 8;
1913
- [encoder setComputePipelineState:ctx->pipeline_mul_mv_id_q5_1_f32];
1914
- } break;
1915
- case GGML_TYPE_Q8_0:
1916
- {
1917
- nth0 = 8;
1918
- nth1 = 8;
1919
- [encoder setComputePipelineState:ctx->pipeline_mul_mv_id_q8_0_f32];
1920
- } break;
1921
- case GGML_TYPE_Q2_K:
1922
- {
1923
- nth0 = 2;
1924
- nth1 = 32;
1925
- [encoder setComputePipelineState:ctx->pipeline_mul_mv_id_q2_K_f32];
1926
- } break;
1927
- case GGML_TYPE_Q3_K:
1928
- {
1929
- nth0 = 2;
1930
- nth1 = 32;
1931
- [encoder setComputePipelineState:ctx->pipeline_mul_mv_id_q3_K_f32];
1932
- } break;
1933
- case GGML_TYPE_Q4_K:
1934
- {
1935
- nth0 = 4; //1;
1936
- nth1 = 8; //32;
1937
- [encoder setComputePipelineState:ctx->pipeline_mul_mv_id_q4_K_f32];
1938
- } break;
1939
- case GGML_TYPE_Q5_K:
1940
- {
1941
- nth0 = 2;
1942
- nth1 = 32;
1943
- [encoder setComputePipelineState:ctx->pipeline_mul_mv_id_q5_K_f32];
1944
- } break;
1945
- case GGML_TYPE_Q6_K:
1946
- {
1947
- nth0 = 2;
1948
- nth1 = 32;
1949
- [encoder setComputePipelineState:ctx->pipeline_mul_mv_id_q6_K_f32];
1950
- } break;
1951
- case GGML_TYPE_IQ2_XXS:
1952
- {
1953
- nth0 = 4;
1954
- nth1 = 16;
1955
- [encoder setComputePipelineState:ctx->pipeline_mul_mv_id_iq2_xxs_f32];
1956
- } break;
1957
- case GGML_TYPE_IQ2_XS:
1958
- {
1959
- nth0 = 4;
1960
- nth1 = 16;
1961
- [encoder setComputePipelineState:ctx->pipeline_mul_mv_id_iq2_xs_f32];
1962
- } break;
1963
- default:
1964
- {
1965
- GGML_METAL_LOG_ERROR("Asserting on type %d\n", (int)src2t);
1966
- GGML_ASSERT(false && "not implemented");
1967
- }
1968
- };
1969
-
1970
- if (ggml_is_quantized(src2t)) {
1971
- GGML_ASSERT(ne20 >= nth0*nth1);
1972
- }
1973
1449
 
1974
- const int64_t _ne1 = 1; // kernels needs a reference in constant memory
1975
-
1976
- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
1977
- [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];
1978
- [encoder setBuffer:id_dst offset:offs_dst atIndex:2];
1979
- [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:3];
1980
- [encoder setBytes:&ne20 length:sizeof(ne20) atIndex:4];
1981
- [encoder setBytes:&ne21 length:sizeof(ne21) atIndex:5];
1982
- [encoder setBytes:&ne22 length:sizeof(ne22) atIndex:6];
1983
- [encoder setBytes:&nb20 length:sizeof(nb20) atIndex:7];
1984
- [encoder setBytes:&nb21 length:sizeof(nb21) atIndex:8];
1985
- [encoder setBytes:&nb22 length:sizeof(nb22) atIndex:9];
1986
- [encoder setBytes:&ne10 length:sizeof(ne10) atIndex:10];
1987
- [encoder setBytes:&_ne1 length:sizeof(_ne1) atIndex:11];
1988
- [encoder setBytes:&ne12 length:sizeof(ne12) atIndex:12];
1989
- [encoder setBytes:&ne13 length:sizeof(ne13) atIndex:13];
1990
- [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:14];
1991
- [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:15];
1992
- [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:16];
1993
- [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:17];
1994
- [encoder setBytes:&_ne1 length:sizeof(_ne1) atIndex:18];
1995
- [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:19];
1996
- [encoder setBytes:&r2 length:sizeof(r2) atIndex:20];
1997
- [encoder setBytes:&r3 length:sizeof(r3) atIndex:21];
1998
- [encoder setBytes:&idx length:sizeof(idx) atIndex:22];
1999
- // TODO: how to make this an array? read Metal docs
2000
- for (int j = 0; j < 8; ++j) {
2001
- // NOTE: this is done like this to avoid uninitialized kernel arguments when n_as < 8
2002
- struct ggml_tensor * src_cur = dst->src[2 + (j % n_as)];
2003
-
2004
- size_t offs_src_cur = 0;
2005
- id<MTLBuffer> id_src_cur = ggml_metal_get_buffer(ctx, src_cur, &offs_src_cur);
2006
-
2007
- [encoder setBuffer:id_src_cur offset:offs_src_cur atIndex:23 + j];
2008
- }
2009
-
2010
- if (src2t == GGML_TYPE_Q4_0 || src2t == GGML_TYPE_Q4_1 ||
2011
- src2t == GGML_TYPE_Q5_0 || src2t == GGML_TYPE_Q5_1 || src2t == GGML_TYPE_Q8_0 ||
2012
- src2t == GGML_TYPE_Q2_K) { // || src2t == GGML_TYPE_Q4_K) {
2013
- [encoder dispatchThreadgroups:MTLSizeMake((ne21 + 7)/8, _ne1, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];
2014
- }
2015
- else if (src2t == GGML_TYPE_IQ2_XXS || src2t == GGML_TYPE_IQ2_XS) {
2016
- const int mem_size = src2t == GGML_TYPE_IQ2_XXS ? 256*8+128 : 512*8+128;
2017
- [encoder setThreadgroupMemoryLength:mem_size atIndex:0];
2018
- [encoder dispatchThreadgroups:MTLSizeMake((ne21 + 7)/8, _ne1, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];
2019
- }
2020
- else if (src2t == GGML_TYPE_Q4_K) {
2021
- [encoder dispatchThreadgroups:MTLSizeMake((ne21 + 3)/4, _ne1, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];
2022
- }
2023
- else if (src2t == GGML_TYPE_Q3_K) {
1450
+ [encoder setComputePipelineState:pipeline];
1451
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
1452
+ [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];
1453
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:2];
1454
+ [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:3];
1455
+ [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:4];
1456
+ [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:5];
1457
+ [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:6];
1458
+ [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:7];
1459
+ [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:8];
1460
+ [encoder setBytes:&ne10 length:sizeof(ne10) atIndex:9];
1461
+ [encoder setBytes:&ne11 length:sizeof(ne11) atIndex:10];
1462
+ [encoder setBytes:&ne12 length:sizeof(ne12) atIndex:11];
1463
+ [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:12];
1464
+ [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:13];
1465
+ [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:14];
1466
+ [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:15];
1467
+ [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:16];
1468
+ [encoder setBytes:&r2 length:sizeof(r2) atIndex:17];
1469
+ [encoder setBytes:&r3 length:sizeof(r3) atIndex:18];
1470
+
1471
+ if (src0t == GGML_TYPE_Q4_0 || src0t == GGML_TYPE_Q4_1 ||
1472
+ src0t == GGML_TYPE_Q5_0 || src0t == GGML_TYPE_Q5_1 || src0t == GGML_TYPE_Q8_0 ||
1473
+ src0t == GGML_TYPE_Q2_K) { // || src0t == GGML_TYPE_Q4_K) {
1474
+ [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 7)/8, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];
1475
+ }
1476
+ else if (src0t == GGML_TYPE_IQ2_XXS || src0t == GGML_TYPE_IQ2_XS) {
1477
+ const int mem_size = src0t == GGML_TYPE_IQ2_XXS ? 256*8+128 : 512*8+128;
1478
+ [encoder setThreadgroupMemoryLength:mem_size atIndex:0];
1479
+ [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 7)/8, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];
1480
+ }
1481
+ else if (src0t == GGML_TYPE_Q4_K) {
1482
+ [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 3)/4, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];
1483
+ }
1484
+ else if (src0t == GGML_TYPE_Q3_K) {
2024
1485
  #ifdef GGML_QKK_64
2025
- [encoder dispatchThreadgroups:MTLSizeMake((ne21 + 1)/2, _ne1, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];
1486
+ [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 1)/2, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];
2026
1487
  #else
2027
- [encoder dispatchThreadgroups:MTLSizeMake((ne21 + 3)/4, _ne1, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];
1488
+ [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 3)/4, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];
2028
1489
  #endif
2029
- }
2030
- else if (src2t == GGML_TYPE_Q5_K) {
2031
- [encoder dispatchThreadgroups:MTLSizeMake((ne21 + 3)/4, _ne1, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];
2032
- }
2033
- else if (src2t == GGML_TYPE_Q6_K) {
2034
- [encoder dispatchThreadgroups:MTLSizeMake((ne21 + 1)/2, _ne1, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];
2035
- } else {
2036
- const int64_t ny = (_ne1 + nrows - 1)/nrows;
2037
- [encoder dispatchThreadgroups:MTLSizeMake(ne21, ny, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];
2038
- }
2039
1490
  }
2040
- } break;
2041
- case GGML_OP_GET_ROWS:
2042
- {
2043
- switch (src0->type) {
2044
- case GGML_TYPE_F32: [encoder setComputePipelineState:ctx->pipeline_get_rows_f32]; break;
2045
- case GGML_TYPE_F16: [encoder setComputePipelineState:ctx->pipeline_get_rows_f16]; break;
2046
- case GGML_TYPE_Q4_0: [encoder setComputePipelineState:ctx->pipeline_get_rows_q4_0]; break;
2047
- case GGML_TYPE_Q4_1: [encoder setComputePipelineState:ctx->pipeline_get_rows_q4_1]; break;
2048
- case GGML_TYPE_Q5_0: [encoder setComputePipelineState:ctx->pipeline_get_rows_q5_0]; break;
2049
- case GGML_TYPE_Q5_1: [encoder setComputePipelineState:ctx->pipeline_get_rows_q5_1]; break;
2050
- case GGML_TYPE_Q8_0: [encoder setComputePipelineState:ctx->pipeline_get_rows_q8_0]; break;
2051
- case GGML_TYPE_Q2_K: [encoder setComputePipelineState:ctx->pipeline_get_rows_q2_K]; break;
2052
- case GGML_TYPE_Q3_K: [encoder setComputePipelineState:ctx->pipeline_get_rows_q3_K]; break;
2053
- case GGML_TYPE_Q4_K: [encoder setComputePipelineState:ctx->pipeline_get_rows_q4_K]; break;
2054
- case GGML_TYPE_Q5_K: [encoder setComputePipelineState:ctx->pipeline_get_rows_q5_K]; break;
2055
- case GGML_TYPE_Q6_K: [encoder setComputePipelineState:ctx->pipeline_get_rows_q6_K]; break;
2056
- case GGML_TYPE_I32: [encoder setComputePipelineState:ctx->pipeline_get_rows_i32]; break;
2057
- case GGML_TYPE_IQ2_XXS: [encoder setComputePipelineState:ctx->pipeline_get_rows_iq2_xxs]; break;
2058
- case GGML_TYPE_IQ2_XS : [encoder setComputePipelineState:ctx->pipeline_get_rows_iq2_xs]; break;
2059
- default: GGML_ASSERT(false && "not implemented");
1491
+ else if (src0t == GGML_TYPE_Q5_K) {
1492
+ [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 3)/4, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];
2060
1493
  }
2061
-
2062
- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
2063
- [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];
2064
- [encoder setBuffer:id_dst offset:offs_dst atIndex:2];
2065
- [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:3];
2066
- [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:4];
2067
- [encoder setBytes:&nb02 length:sizeof(uint64_t) atIndex:5];
2068
- [encoder setBytes:&ne10 length:sizeof( int64_t) atIndex:6];
2069
- [encoder setBytes:&nb10 length:sizeof( int64_t) atIndex:7];
2070
- [encoder setBytes:&nb11 length:sizeof( int64_t) atIndex:8];
2071
- [encoder setBytes:&nb1 length:sizeof(uint64_t) atIndex:9];
2072
- [encoder setBytes:&nb2 length:sizeof(uint64_t) atIndex:10];
2073
-
2074
- [encoder dispatchThreadgroups:MTLSizeMake(ne10, ne11, 1) threadsPerThreadgroup:MTLSizeMake(32, 1, 1)];
2075
- } break;
2076
- case GGML_OP_RMS_NORM:
2077
- {
2078
- GGML_ASSERT(ne00 % 4 == 0);
2079
-
2080
- float eps;
2081
- memcpy(&eps, dst->op_params, sizeof(float));
2082
-
2083
- int nth = 32; // SIMD width
2084
-
2085
- while (nth < ne00/4 && nth < 1024) {
2086
- nth *= 2;
1494
+ else if (src0t == GGML_TYPE_Q6_K) {
1495
+ [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 1)/2, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];
1496
+ } else {
1497
+ const int64_t ny = (ne11 + nrows - 1)/nrows;
1498
+ [encoder dispatchThreadgroups:MTLSizeMake(ne01, ny, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];
2087
1499
  }
1500
+ }
1501
+ } break;
1502
+ case GGML_OP_MUL_MAT_ID:
1503
+ {
1504
+ //GGML_ASSERT(ne00 == ne10);
1505
+ //GGML_ASSERT(ne03 == ne13);
2088
1506
 
2089
- [encoder setComputePipelineState:ctx->pipeline_rms_norm];
2090
- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
2091
- [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
2092
- [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:2];
2093
- [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:3];
2094
- [encoder setBytes:&eps length:sizeof( float) atIndex:4];
2095
- [encoder setThreadgroupMemoryLength:32*sizeof(float) atIndex:0];
2096
-
2097
- const int64_t nrows = ggml_nrows(src0);
2098
-
2099
- [encoder dispatchThreadgroups:MTLSizeMake(nrows, 1, 1) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];
2100
- } break;
2101
- case GGML_OP_GROUP_NORM:
2102
- {
2103
- GGML_ASSERT(ne00 % 4 == 0);
2104
-
2105
- //float eps;
2106
- //memcpy(&eps, dst->op_params, sizeof(float));
2107
-
2108
- const float eps = 1e-6f; // TODO: temporarily hardcoded
2109
-
2110
- const int32_t n_groups = ((int32_t *) dst->op_params)[0];
2111
-
2112
- int nth = 32; // SIMD width
2113
-
2114
- //while (nth < ne00/4 && nth < 1024) {
2115
- // nth *= 2;
2116
- //}
2117
-
2118
- [encoder setComputePipelineState:ctx->pipeline_group_norm];
2119
- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
2120
- [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
2121
- [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:2];
2122
- [encoder setBytes:&ne01 length:sizeof( int64_t) atIndex:3];
2123
- [encoder setBytes:&ne02 length:sizeof( int64_t) atIndex:4];
2124
- [encoder setBytes:&nb00 length:sizeof(uint64_t) atIndex:5];
2125
- [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:6];
2126
- [encoder setBytes:&nb02 length:sizeof(uint64_t) atIndex:7];
2127
- [encoder setBytes:&n_groups length:sizeof( int32_t) atIndex:8];
2128
- [encoder setBytes:&eps length:sizeof( float) atIndex:9];
2129
- [encoder setThreadgroupMemoryLength:32*sizeof(float) atIndex:0];
2130
-
2131
- [encoder dispatchThreadgroups:MTLSizeMake(n_groups, 1, 1) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];
2132
- } break;
2133
- case GGML_OP_NORM:
2134
- {
2135
- float eps;
2136
- memcpy(&eps, dst->op_params, sizeof(float));
2137
-
2138
- const int nth = MIN(256, ne00);
2139
-
2140
- [encoder setComputePipelineState:ctx->pipeline_norm];
2141
- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
2142
- [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
2143
- [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:2];
2144
- [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:3];
2145
- [encoder setBytes:&eps length:sizeof( float) atIndex:4];
2146
- [encoder setThreadgroupMemoryLength:GGML_PAD(nth*sizeof(float), 16) atIndex:0];
2147
-
2148
- const int64_t nrows = ggml_nrows(src0);
2149
-
2150
- [encoder dispatchThreadgroups:MTLSizeMake(nrows, 1, 1) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];
2151
- } break;
2152
- case GGML_OP_ALIBI:
2153
- {
2154
- GGML_ASSERT((src0t == GGML_TYPE_F32));
2155
-
2156
- const int nth = MIN(1024, ne00);
2157
-
2158
- //const int n_past = ((int32_t *) dst->op_params)[0];
2159
- const int n_head = ((int32_t *) dst->op_params)[1];
2160
- float max_bias;
2161
- memcpy(&max_bias, (int32_t *) dst->op_params + 2, sizeof(float));
1507
+ GGML_ASSERT(src0t == GGML_TYPE_I32);
2162
1508
 
2163
- const int n_heads_log2_floor = 1 << (int) floor(log2(n_head));
2164
- const float m0 = powf(2.0f, -(max_bias) / n_heads_log2_floor);
2165
- const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_heads_log2_floor);
1509
+ const int n_as = ((int32_t *) dst->op_params)[1];
2166
1510
 
2167
- [encoder setComputePipelineState:ctx->pipeline_alibi_f32];
2168
- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
2169
- [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
2170
- [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:2];
2171
- [encoder setBytes:&ne01 length:sizeof( int64_t) atIndex:3];
2172
- [encoder setBytes:&ne02 length:sizeof( int64_t) atIndex:4];
2173
- [encoder setBytes:&ne03 length:sizeof( int64_t) atIndex:5];
2174
- [encoder setBytes:&nb00 length:sizeof(uint64_t) atIndex:6];
2175
- [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:7];
2176
- [encoder setBytes:&nb02 length:sizeof(uint64_t) atIndex:8];
2177
- [encoder setBytes:&nb03 length:sizeof(uint64_t) atIndex:9];
2178
- [encoder setBytes:&ne0 length:sizeof( int64_t) atIndex:10];
2179
- [encoder setBytes:&ne1 length:sizeof( int64_t) atIndex:11];
2180
- [encoder setBytes:&ne2 length:sizeof( int64_t) atIndex:12];
2181
- [encoder setBytes:&ne3 length:sizeof( int64_t) atIndex:13];
2182
- [encoder setBytes:&nb0 length:sizeof(uint64_t) atIndex:14];
2183
- [encoder setBytes:&nb1 length:sizeof(uint64_t) atIndex:15];
2184
- [encoder setBytes:&nb2 length:sizeof(uint64_t) atIndex:16];
2185
- [encoder setBytes:&nb3 length:sizeof(uint64_t) atIndex:17];
2186
- [encoder setBytes:&m0 length:sizeof( float) atIndex:18];
2187
- [encoder setBytes:&m1 length:sizeof( float) atIndex:19];
2188
- [encoder setBytes:&n_heads_log2_floor length:sizeof(int) atIndex:20];
1511
+ // TODO: make this more general
1512
+ GGML_ASSERT(n_as <= 8);
2189
1513
 
2190
- [encoder dispatchThreadgroups:MTLSizeMake(ne01, ne02, ne03) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];
2191
- } break;
2192
- case GGML_OP_ROPE:
2193
- {
2194
- GGML_ASSERT(ne10 == ne02);
2195
-
2196
- const int nth = MIN(1024, ne00);
2197
-
2198
- const int n_past = ((int32_t *) dst->op_params)[0];
2199
- const int n_dims = ((int32_t *) dst->op_params)[1];
2200
- const int mode = ((int32_t *) dst->op_params)[2];
2201
- // skip 3, n_ctx, used in GLM RoPE, unimplemented in metal
2202
- const int n_orig_ctx = ((int32_t *) dst->op_params)[4];
2203
-
2204
- float freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow;
2205
- memcpy(&freq_base, (int32_t *) dst->op_params + 5, sizeof(float));
2206
- memcpy(&freq_scale, (int32_t *) dst->op_params + 6, sizeof(float));
2207
- memcpy(&ext_factor, (int32_t *) dst->op_params + 7, sizeof(float));
2208
- memcpy(&attn_factor, (int32_t *) dst->op_params + 8, sizeof(float));
2209
- memcpy(&beta_fast, (int32_t *) dst->op_params + 9, sizeof(float));
2210
- memcpy(&beta_slow, (int32_t *) dst->op_params + 10, sizeof(float));
1514
+ // max size of the src1ids array in the kernel stack
1515
+ GGML_ASSERT(ne11 <= 512);
2211
1516
 
2212
- switch (src0->type) {
2213
- case GGML_TYPE_F32: [encoder setComputePipelineState:ctx->pipeline_rope_f32]; break;
2214
- case GGML_TYPE_F16: [encoder setComputePipelineState:ctx->pipeline_rope_f16]; break;
2215
- default: GGML_ASSERT(false);
2216
- };
1517
+ struct ggml_tensor * src2 = gf->nodes[i]->src[2];
2217
1518
 
2218
- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
2219
- [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];
2220
- [encoder setBuffer:id_dst offset:offs_dst atIndex:2];
2221
- [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:3];
2222
- [encoder setBytes:&ne01 length:sizeof( int64_t) atIndex:4];
2223
- [encoder setBytes:&ne02 length:sizeof( int64_t) atIndex:5];
2224
- [encoder setBytes:&ne03 length:sizeof( int64_t) atIndex:6];
2225
- [encoder setBytes:&nb00 length:sizeof(uint64_t) atIndex:7];
2226
- [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:8];
2227
- [encoder setBytes:&nb02 length:sizeof(uint64_t) atIndex:9];
2228
- [encoder setBytes:&nb03 length:sizeof(uint64_t) atIndex:10];
2229
- [encoder setBytes:&ne0 length:sizeof( int64_t) atIndex:11];
2230
- [encoder setBytes:&ne1 length:sizeof( int64_t) atIndex:12];
2231
- [encoder setBytes:&ne2 length:sizeof( int64_t) atIndex:13];
2232
- [encoder setBytes:&ne3 length:sizeof( int64_t) atIndex:14];
2233
- [encoder setBytes:&nb0 length:sizeof(uint64_t) atIndex:15];
2234
- [encoder setBytes:&nb1 length:sizeof(uint64_t) atIndex:16];
2235
- [encoder setBytes:&nb2 length:sizeof(uint64_t) atIndex:17];
2236
- [encoder setBytes:&nb3 length:sizeof(uint64_t) atIndex:18];
2237
- [encoder setBytes:&n_past length:sizeof( int) atIndex:19];
2238
- [encoder setBytes:&n_dims length:sizeof( int) atIndex:20];
2239
- [encoder setBytes:&mode length:sizeof( int) atIndex:21];
2240
- [encoder setBytes:&n_orig_ctx length:sizeof( int) atIndex:22];
2241
- [encoder setBytes:&freq_base length:sizeof( float) atIndex:23];
2242
- [encoder setBytes:&freq_scale length:sizeof( float) atIndex:24];
2243
- [encoder setBytes:&ext_factor length:sizeof( float) atIndex:25];
2244
- [encoder setBytes:&attn_factor length:sizeof( float) atIndex:26];
2245
- [encoder setBytes:&beta_fast length:sizeof( float) atIndex:27];
2246
- [encoder setBytes:&beta_slow length:sizeof( float) atIndex:28];
1519
+ const int64_t ne20 = src2 ? src2->ne[0] : 0;
1520
+ const int64_t ne21 = src2 ? src2->ne[1] : 0;
1521
+ const int64_t ne22 = src2 ? src2->ne[2] : 0;
1522
+ const int64_t ne23 = src2 ? src2->ne[3] : 0; GGML_UNUSED(ne23);
2247
1523
 
2248
- [encoder dispatchThreadgroups:MTLSizeMake(ne01, ne02, ne03) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];
2249
- } break;
2250
- case GGML_OP_IM2COL:
2251
- {
2252
- GGML_ASSERT(src0->type == GGML_TYPE_F16);
2253
- GGML_ASSERT(src1->type == GGML_TYPE_F32);
2254
- GGML_ASSERT( dst->type == GGML_TYPE_F16);
1524
+ const uint64_t nb20 = src2 ? src2->nb[0] : 0; GGML_UNUSED(nb20);
1525
+ const uint64_t nb21 = src2 ? src2->nb[1] : 0;
1526
+ const uint64_t nb22 = src2 ? src2->nb[2] : 0;
1527
+ const uint64_t nb23 = src2 ? src2->nb[3] : 0; GGML_UNUSED(nb23);
2255
1528
 
2256
- const int32_t s0 = ((const int32_t *)(dst->op_params))[0];
2257
- const int32_t s1 = ((const int32_t *)(dst->op_params))[1];
2258
- const int32_t p0 = ((const int32_t *)(dst->op_params))[2];
2259
- const int32_t p1 = ((const int32_t *)(dst->op_params))[3];
2260
- const int32_t d0 = ((const int32_t *)(dst->op_params))[4];
2261
- const int32_t d1 = ((const int32_t *)(dst->op_params))[5];
2262
- const bool is_2D = ((const int32_t *)(dst->op_params))[6] == 1;
1529
+ const enum ggml_type src2t = src2 ? src2->type : GGML_TYPE_COUNT; GGML_UNUSED(src2t);
2263
1530
 
2264
- const int32_t N = src1->ne[is_2D ? 3 : 2];
2265
- const int32_t IC = src1->ne[is_2D ? 2 : 1];
2266
- const int32_t IH = is_2D ? src1->ne[1] : 1;
2267
- const int32_t IW = src1->ne[0];
1531
+ GGML_ASSERT(!ggml_is_transposed(src2));
1532
+ GGML_ASSERT(!ggml_is_transposed(src1));
2268
1533
 
2269
- const int32_t KH = is_2D ? src0->ne[1] : 1;
2270
- const int32_t KW = src0->ne[0];
1534
+ GGML_ASSERT(src1t == GGML_TYPE_F32);
2271
1535
 
2272
- const int32_t OH = is_2D ? dst->ne[2] : 1;
2273
- const int32_t OW = dst->ne[1];
1536
+ const uint r2 = ne12/ne22;
1537
+ const uint r3 = ne13/ne23;
2274
1538
 
2275
- const int32_t CHW = IC * KH * KW;
1539
+ // find the break-even point where the matrix-matrix kernel becomes more efficient compared
1540
+ // to the matrix-vector kernel
1541
+ int ne11_mm_min = n_as;
2276
1542
 
2277
- const int32_t ofs0 = src1->nb[is_2D ? 3 : 2] / 4;
2278
- const int32_t ofs1 = src1->nb[is_2D ? 2 : 1] / 4;
1543
+ const int idx = ((int32_t *) dst->op_params)[0];
2279
1544
 
2280
- switch (src0->type) {
2281
- case GGML_TYPE_F32: GGML_ASSERT(false && "not implemented"); break;
2282
- case GGML_TYPE_F16: [encoder setComputePipelineState:ctx->pipeline_im2col_f16]; break;
2283
- default: GGML_ASSERT(false);
2284
- };
1545
+ // batch size
1546
+ GGML_ASSERT(ne01 == ne11);
2285
1547
 
2286
- [encoder setBuffer:id_src1 offset:offs_src1 atIndex:0];
2287
- [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
2288
- [encoder setBytes:&ofs0 length:sizeof( int32_t) atIndex:2];
2289
- [encoder setBytes:&ofs1 length:sizeof( int32_t) atIndex:3];
2290
- [encoder setBytes:&IW length:sizeof( int32_t) atIndex:4];
2291
- [encoder setBytes:&IH length:sizeof( int32_t) atIndex:5];
2292
- [encoder setBytes:&CHW length:sizeof( int32_t) atIndex:6];
2293
- [encoder setBytes:&s0 length:sizeof( int32_t) atIndex:7];
2294
- [encoder setBytes:&s1 length:sizeof( int32_t) atIndex:8];
2295
- [encoder setBytes:&p0 length:sizeof( int32_t) atIndex:9];
2296
- [encoder setBytes:&p1 length:sizeof( int32_t) atIndex:10];
2297
- [encoder setBytes:&d0 length:sizeof( int32_t) atIndex:11];
2298
- [encoder setBytes:&d1 length:sizeof( int32_t) atIndex:12];
2299
-
2300
- [encoder dispatchThreadgroups:MTLSizeMake(IC, OH, OW) threadsPerThreadgroup:MTLSizeMake(N, KH, KW)];
2301
- } break;
2302
- case GGML_OP_UPSCALE:
2303
- {
2304
- GGML_ASSERT(src0->type == GGML_TYPE_F32);
2305
-
2306
- const int sf = dst->op_params[0];
2307
-
2308
- [encoder setComputePipelineState:ctx->pipeline_upscale_f32];
2309
- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
2310
- [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
2311
- [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:2];
2312
- [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:3];
2313
- [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:4];
2314
- [encoder setBytes:&ne03 length:sizeof(ne03) atIndex:5];
2315
- [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:6];
2316
- [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:7];
2317
- [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:8];
2318
- [encoder setBytes:&nb03 length:sizeof(nb03) atIndex:9];
2319
- [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:10];
2320
- [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:11];
2321
- [encoder setBytes:&ne2 length:sizeof(ne2) atIndex:12];
2322
- [encoder setBytes:&ne3 length:sizeof(ne3) atIndex:13];
2323
- [encoder setBytes:&nb0 length:sizeof(nb0) atIndex:14];
2324
- [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:15];
2325
- [encoder setBytes:&nb2 length:sizeof(nb2) atIndex:16];
2326
- [encoder setBytes:&nb3 length:sizeof(nb3) atIndex:17];
2327
- [encoder setBytes:&sf length:sizeof(sf) atIndex:18];
2328
-
2329
- const int nth = MIN((int) ctx->pipeline_upscale_f32.maxTotalThreadsPerThreadgroup, ne0);
2330
-
2331
- [encoder dispatchThreadgroups:MTLSizeMake(ne1, ne2, ne3) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];
2332
- } break;
2333
- case GGML_OP_PAD:
2334
- {
2335
- GGML_ASSERT(src0->type == GGML_TYPE_F32);
2336
-
2337
- [encoder setComputePipelineState:ctx->pipeline_pad_f32];
2338
- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
2339
- [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
2340
- [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:2];
2341
- [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:3];
2342
- [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:4];
2343
- [encoder setBytes:&ne03 length:sizeof(ne03) atIndex:5];
2344
- [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:6];
2345
- [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:7];
2346
- [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:8];
2347
- [encoder setBytes:&nb03 length:sizeof(nb03) atIndex:9];
2348
- [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:10];
2349
- [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:11];
2350
- [encoder setBytes:&ne2 length:sizeof(ne2) atIndex:12];
2351
- [encoder setBytes:&ne3 length:sizeof(ne3) atIndex:13];
2352
- [encoder setBytes:&nb0 length:sizeof(nb0) atIndex:14];
2353
- [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:15];
2354
- [encoder setBytes:&nb2 length:sizeof(nb2) atIndex:16];
2355
- [encoder setBytes:&nb3 length:sizeof(nb3) atIndex:17];
2356
-
2357
- const int nth = MIN(1024, ne0);
2358
-
2359
- [encoder dispatchThreadgroups:MTLSizeMake(ne1, ne2, ne3) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];
2360
- } break;
2361
- case GGML_OP_ARGSORT:
2362
- {
2363
- GGML_ASSERT(src0->type == GGML_TYPE_F32);
2364
- GGML_ASSERT( dst->type == GGML_TYPE_I32);
2365
-
2366
- const int nrows = ggml_nrows(src0);
2367
-
2368
- enum ggml_sort_order order = (enum ggml_sort_order) dst->op_params[0];
2369
-
2370
- switch (order) {
2371
- case GGML_SORT_ASC: [encoder setComputePipelineState:ctx->pipeline_argsort_f32_i32_asc]; break;
2372
- case GGML_SORT_DESC: [encoder setComputePipelineState:ctx->pipeline_argsort_f32_i32_desc]; break;
2373
- default: GGML_ASSERT(false);
2374
- };
1548
+ // for now the matrix-matrix multiplication kernel only works on A14+/M1+ SoCs
1549
+ // AMD GPU and older A-chips will reuse matrix-vector multiplication kernel
1550
+ // !!!
1551
+ // TODO: for now, always use mat-vec kernels until we figure out how to improve the
1552
+ // indirect matrix multiplication
1553
+ // !!!
1554
+ if ([ctx->device supportsFamily:MTLGPUFamilyApple7] &&
1555
+ ne20 % 32 == 0 && ne20 >= 64 &&
1556
+ ne11 > ne11_mm_min) {
2375
1557
 
2376
- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
2377
- [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
2378
- [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:2];
2379
-
2380
- [encoder dispatchThreadgroups:MTLSizeMake(1, nrows, 1) threadsPerThreadgroup:MTLSizeMake(ne00, 1, 1)];
2381
- } break;
2382
- case GGML_OP_LEAKY_RELU:
2383
- {
2384
- GGML_ASSERT(src0->type == GGML_TYPE_F32);
1558
+ id<MTLComputePipelineState> pipeline = nil;
2385
1559
 
2386
- float slope;
2387
- memcpy(&slope, dst->op_params, sizeof(float));
1560
+ switch (src2->type) {
1561
+ case GGML_TYPE_F32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_F32_F32 ].pipeline; break;
1562
+ case GGML_TYPE_F16: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_F16_F32 ].pipeline; break;
1563
+ case GGML_TYPE_Q4_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_0_F32 ].pipeline; break;
1564
+ case GGML_TYPE_Q4_1: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_1_F32 ].pipeline; break;
1565
+ case GGML_TYPE_Q5_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q5_0_F32 ].pipeline; break;
1566
+ case GGML_TYPE_Q5_1: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q5_1_F32 ].pipeline; break;
1567
+ case GGML_TYPE_Q8_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q8_0_F32 ].pipeline; break;
1568
+ case GGML_TYPE_Q2_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q2_K_F32 ].pipeline; break;
1569
+ case GGML_TYPE_Q3_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q3_K_F32 ].pipeline; break;
1570
+ case GGML_TYPE_Q4_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_K_F32 ].pipeline; break;
1571
+ case GGML_TYPE_Q5_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q5_K_F32 ].pipeline; break;
1572
+ case GGML_TYPE_Q6_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q6_K_F32 ].pipeline; break;
1573
+ case GGML_TYPE_IQ2_XXS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_XXS_F32].pipeline; break;
1574
+ case GGML_TYPE_IQ2_XS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_XS_F32 ].pipeline; break;
1575
+ default: GGML_ASSERT(false && "MUL_MAT_ID not implemented");
1576
+ }
2388
1577
 
2389
- [encoder setComputePipelineState:ctx->pipeline_leaky_relu_f32];
2390
- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
2391
- [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
2392
- [encoder setBytes:&slope length:sizeof(slope) atIndex:2];
1578
+ [encoder setComputePipelineState:pipeline];
1579
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
1580
+ [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];
1581
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:2];
1582
+ [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:3];
1583
+ [encoder setBytes:&ne20 length:sizeof(ne20) atIndex:4];
1584
+ [encoder setBytes:&ne22 length:sizeof(ne22) atIndex:5];
1585
+ [encoder setBytes:&nb21 length:sizeof(nb21) atIndex:6];
1586
+ [encoder setBytes:&nb22 length:sizeof(nb22) atIndex:7];
1587
+ [encoder setBytes:&ne12 length:sizeof(ne12) atIndex:8];
1588
+ [encoder setBytes:&ne13 length:sizeof(ne13) atIndex:9];
1589
+ [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:10];
1590
+ [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:11];
1591
+ [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:12];
1592
+ [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:13];
1593
+ [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:14];
1594
+ [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:15];
1595
+ [encoder setBytes:&r2 length:sizeof(r2) atIndex:16];
1596
+ [encoder setBytes:&r3 length:sizeof(r3) atIndex:17];
1597
+ [encoder setBytes:&idx length:sizeof(idx) atIndex:18];
1598
+ // TODO: how to make this an array? read Metal docs
1599
+ for (int j = 0; j < 8; ++j) {
1600
+ // NOTE: this is done like this to avoid uninitialized kernel arguments when n_as < 8
1601
+ struct ggml_tensor * src_cur = dst->src[2 + (j % n_as)];
1602
+
1603
+ size_t offs_src_cur = 0;
1604
+ id<MTLBuffer> id_src_cur = ggml_metal_get_buffer(ctx, src_cur, &offs_src_cur);
1605
+
1606
+ [encoder setBuffer:id_src_cur offset:offs_src_cur atIndex:19 + j];
1607
+ }
2393
1608
 
2394
- const int64_t n = ggml_nelements(dst);
1609
+ [encoder setThreadgroupMemoryLength:8192 atIndex:0];
2395
1610
 
2396
- [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
2397
- } break;
2398
- case GGML_OP_DUP:
2399
- case GGML_OP_CPY:
2400
- case GGML_OP_CONT:
2401
- {
2402
- GGML_ASSERT(ne00 % ggml_blck_size(src0->type) == 0);
1611
+ [encoder dispatchThreadgroups:MTLSizeMake((ne11 + 31)/32, (ne21 + 63)/64, n_as*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(128, 1, 1)];
1612
+ } else {
1613
+ int nth0 = 32;
1614
+ int nth1 = 1;
1615
+ int nrows = 1;
1616
+ //printf("vector: ne00 = %6d, ne01 = %6d, ne02 = %6d, ne11 = %6d, ne12 = %6d\n", ne00, ne01, ne02, ne11, ne12);
2403
1617
 
2404
- int nth = MIN(1024, ne00/ggml_blck_size(src0->type));
1618
+ id<MTLComputePipelineState> pipeline = nil;
2405
1619
 
2406
- switch (src0t) {
1620
+ // use custom matrix x vector kernel
1621
+ switch (src2t) {
2407
1622
  case GGML_TYPE_F32:
2408
1623
  {
2409
- GGML_ASSERT(ne0 % ggml_blck_size(dst->type) == 0);
2410
-
2411
- switch (dstt) {
2412
- case GGML_TYPE_F16: [encoder setComputePipelineState:ctx->pipeline_cpy_f32_f16]; break;
2413
- case GGML_TYPE_F32: [encoder setComputePipelineState:ctx->pipeline_cpy_f32_f32]; break;
2414
- case GGML_TYPE_Q8_0: [encoder setComputePipelineState:ctx->pipeline_cpy_f32_q8_0]; break;
2415
- case GGML_TYPE_Q4_0: [encoder setComputePipelineState:ctx->pipeline_cpy_f32_q4_0]; break;
2416
- case GGML_TYPE_Q4_1: [encoder setComputePipelineState:ctx->pipeline_cpy_f32_q4_1]; break;
2417
- //case GGML_TYPE_Q5_0: [encoder setComputePipelineState:ctx->pipeline_cpy_f32_q5_0]; break;
2418
- //case GGML_TYPE_Q5_1: [encoder setComputePipelineState:ctx->pipeline_cpy_f32_q5_1]; break;
2419
- default: GGML_ASSERT(false && "not implemented");
2420
- };
1624
+ GGML_ASSERT(src1t == GGML_TYPE_F32);
1625
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F32_F32].pipeline;
2421
1626
  } break;
2422
1627
  case GGML_TYPE_F16:
2423
1628
  {
2424
- switch (dstt) {
2425
- case GGML_TYPE_F16: [encoder setComputePipelineState:ctx->pipeline_cpy_f16_f16]; break;
2426
- case GGML_TYPE_F32: [encoder setComputePipelineState:ctx->pipeline_cpy_f16_f32]; break;
2427
- default: GGML_ASSERT(false && "not implemented");
2428
- };
1629
+ GGML_ASSERT(src1t == GGML_TYPE_F32);
1630
+ nth0 = 32;
1631
+ nth1 = 1;
1632
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F32].pipeline;
1633
+ } break;
1634
+ case GGML_TYPE_Q4_0:
1635
+ {
1636
+ nth0 = 8;
1637
+ nth1 = 8;
1638
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q4_0_F32].pipeline;
2429
1639
  } break;
2430
- default: GGML_ASSERT(false && "not implemented");
1640
+ case GGML_TYPE_Q4_1:
1641
+ {
1642
+ nth0 = 8;
1643
+ nth1 = 8;
1644
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q4_1_F32].pipeline;
1645
+ } break;
1646
+ case GGML_TYPE_Q5_0:
1647
+ {
1648
+ nth0 = 8;
1649
+ nth1 = 8;
1650
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q5_0_F32].pipeline;
1651
+ } break;
1652
+ case GGML_TYPE_Q5_1:
1653
+ {
1654
+ nth0 = 8;
1655
+ nth1 = 8;
1656
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q5_1_F32].pipeline;
1657
+ } break;
1658
+ case GGML_TYPE_Q8_0:
1659
+ {
1660
+ nth0 = 8;
1661
+ nth1 = 8;
1662
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q8_0_F32].pipeline;
1663
+ } break;
1664
+ case GGML_TYPE_Q2_K:
1665
+ {
1666
+ nth0 = 2;
1667
+ nth1 = 32;
1668
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q2_K_F32].pipeline;
1669
+ } break;
1670
+ case GGML_TYPE_Q3_K:
1671
+ {
1672
+ nth0 = 2;
1673
+ nth1 = 32;
1674
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q3_K_F32].pipeline;
1675
+ } break;
1676
+ case GGML_TYPE_Q4_K:
1677
+ {
1678
+ nth0 = 4; //1;
1679
+ nth1 = 8; //32;
1680
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q4_K_F32].pipeline;
1681
+ } break;
1682
+ case GGML_TYPE_Q5_K:
1683
+ {
1684
+ nth0 = 2;
1685
+ nth1 = 32;
1686
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q5_K_F32].pipeline;
1687
+ } break;
1688
+ case GGML_TYPE_Q6_K:
1689
+ {
1690
+ nth0 = 2;
1691
+ nth1 = 32;
1692
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q6_K_F32].pipeline;
1693
+ } break;
1694
+ case GGML_TYPE_IQ2_XXS:
1695
+ {
1696
+ nth0 = 4;
1697
+ nth1 = 16;
1698
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_XXS_F32].pipeline;
1699
+ } break;
1700
+ case GGML_TYPE_IQ2_XS:
1701
+ {
1702
+ nth0 = 4;
1703
+ nth1 = 16;
1704
+ pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_XS_F32].pipeline;
1705
+ } break;
1706
+ default:
1707
+ {
1708
+ GGML_METAL_LOG_ERROR("Asserting on type %d\n", (int)src2t);
1709
+ GGML_ASSERT(false && "not implemented");
1710
+ }
1711
+ };
1712
+
1713
+ if (ggml_is_quantized(src2t)) {
1714
+ GGML_ASSERT(ne20 >= nth0*nth1);
2431
1715
  }
2432
1716
 
2433
- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
2434
- [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
2435
- [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:2];
2436
- [encoder setBytes:&ne01 length:sizeof( int64_t) atIndex:3];
2437
- [encoder setBytes:&ne02 length:sizeof( int64_t) atIndex:4];
2438
- [encoder setBytes:&ne03 length:sizeof( int64_t) atIndex:5];
2439
- [encoder setBytes:&nb00 length:sizeof(uint64_t) atIndex:6];
2440
- [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:7];
2441
- [encoder setBytes:&nb02 length:sizeof(uint64_t) atIndex:8];
2442
- [encoder setBytes:&nb03 length:sizeof(uint64_t) atIndex:9];
2443
- [encoder setBytes:&ne0 length:sizeof( int64_t) atIndex:10];
2444
- [encoder setBytes:&ne1 length:sizeof( int64_t) atIndex:11];
2445
- [encoder setBytes:&ne2 length:sizeof( int64_t) atIndex:12];
2446
- [encoder setBytes:&ne3 length:sizeof( int64_t) atIndex:13];
2447
- [encoder setBytes:&nb0 length:sizeof(uint64_t) atIndex:14];
2448
- [encoder setBytes:&nb1 length:sizeof(uint64_t) atIndex:15];
2449
- [encoder setBytes:&nb2 length:sizeof(uint64_t) atIndex:16];
2450
- [encoder setBytes:&nb3 length:sizeof(uint64_t) atIndex:17];
1717
+ const int64_t _ne1 = 1; // kernels needs a reference in constant memory
2451
1718
 
2452
- [encoder dispatchThreadgroups:MTLSizeMake(ne01, ne02, ne03) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];
2453
- } break;
2454
- default:
2455
- {
2456
- GGML_METAL_LOG_ERROR("%s: error: node %3d, op = %8s not implemented\n", __func__, i, ggml_op_name(dst->op));
2457
- GGML_ASSERT(false);
2458
- }
2459
- }
1719
+ [encoder setComputePipelineState:pipeline];
1720
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
1721
+ [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];
1722
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:2];
1723
+ [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:3];
1724
+ [encoder setBytes:&ne20 length:sizeof(ne20) atIndex:4];
1725
+ [encoder setBytes:&ne21 length:sizeof(ne21) atIndex:5];
1726
+ [encoder setBytes:&ne22 length:sizeof(ne22) atIndex:6];
1727
+ [encoder setBytes:&nb20 length:sizeof(nb20) atIndex:7];
1728
+ [encoder setBytes:&nb21 length:sizeof(nb21) atIndex:8];
1729
+ [encoder setBytes:&nb22 length:sizeof(nb22) atIndex:9];
1730
+ [encoder setBytes:&ne10 length:sizeof(ne10) atIndex:10];
1731
+ [encoder setBytes:&_ne1 length:sizeof(_ne1) atIndex:11];
1732
+ [encoder setBytes:&ne12 length:sizeof(ne12) atIndex:12];
1733
+ [encoder setBytes:&ne13 length:sizeof(ne13) atIndex:13];
1734
+ [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:14];
1735
+ [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:15];
1736
+ [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:16];
1737
+ [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:17];
1738
+ [encoder setBytes:&_ne1 length:sizeof(_ne1) atIndex:18];
1739
+ [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:19];
1740
+ [encoder setBytes:&r2 length:sizeof(r2) atIndex:20];
1741
+ [encoder setBytes:&r3 length:sizeof(r3) atIndex:21];
1742
+ [encoder setBytes:&idx length:sizeof(idx) atIndex:22];
1743
+ // TODO: how to make this an array? read Metal docs
1744
+ for (int j = 0; j < 8; ++j) {
1745
+ // NOTE: this is done like this to avoid uninitialized kernel arguments when n_as < 8
1746
+ struct ggml_tensor * src_cur = dst->src[2 + (j % n_as)];
1747
+
1748
+ size_t offs_src_cur = 0;
1749
+ id<MTLBuffer> id_src_cur = ggml_metal_get_buffer(ctx, src_cur, &offs_src_cur);
1750
+
1751
+ [encoder setBuffer:id_src_cur offset:offs_src_cur atIndex:23 + j];
1752
+ }
2460
1753
 
2461
- #ifndef GGML_METAL_NDEBUG
2462
- [encoder popDebugGroup];
1754
+ if (src2t == GGML_TYPE_Q4_0 || src2t == GGML_TYPE_Q4_1 ||
1755
+ src2t == GGML_TYPE_Q5_0 || src2t == GGML_TYPE_Q5_1 || src2t == GGML_TYPE_Q8_0 ||
1756
+ src2t == GGML_TYPE_Q2_K) { // || src2t == GGML_TYPE_Q4_K) {
1757
+ [encoder dispatchThreadgroups:MTLSizeMake((ne21 + 7)/8, _ne1, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];
1758
+ }
1759
+ else if (src2t == GGML_TYPE_IQ2_XXS || src2t == GGML_TYPE_IQ2_XS) {
1760
+ const int mem_size = src2t == GGML_TYPE_IQ2_XXS ? 256*8+128 : 512*8+128;
1761
+ [encoder setThreadgroupMemoryLength:mem_size atIndex:0];
1762
+ [encoder dispatchThreadgroups:MTLSizeMake((ne21 + 7)/8, _ne1, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];
1763
+ }
1764
+ else if (src2t == GGML_TYPE_Q4_K) {
1765
+ [encoder dispatchThreadgroups:MTLSizeMake((ne21 + 3)/4, _ne1, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];
1766
+ }
1767
+ else if (src2t == GGML_TYPE_Q3_K) {
1768
+ #ifdef GGML_QKK_64
1769
+ [encoder dispatchThreadgroups:MTLSizeMake((ne21 + 1)/2, _ne1, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];
1770
+ #else
1771
+ [encoder dispatchThreadgroups:MTLSizeMake((ne21 + 3)/4, _ne1, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];
2463
1772
  #endif
2464
- }
1773
+ }
1774
+ else if (src2t == GGML_TYPE_Q5_K) {
1775
+ [encoder dispatchThreadgroups:MTLSizeMake((ne21 + 3)/4, _ne1, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];
1776
+ }
1777
+ else if (src2t == GGML_TYPE_Q6_K) {
1778
+ [encoder dispatchThreadgroups:MTLSizeMake((ne21 + 1)/2, _ne1, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];
1779
+ } else {
1780
+ const int64_t ny = (_ne1 + nrows - 1)/nrows;
1781
+ [encoder dispatchThreadgroups:MTLSizeMake(ne21, ny, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];
1782
+ }
1783
+ }
1784
+ } break;
1785
+ case GGML_OP_GET_ROWS:
1786
+ {
1787
+ id<MTLComputePipelineState> pipeline = nil;
1788
+
1789
+ switch (src0->type) {
1790
+ case GGML_TYPE_F32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_F32 ].pipeline; break;
1791
+ case GGML_TYPE_F16: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_F16 ].pipeline; break;
1792
+ case GGML_TYPE_Q4_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q4_0 ].pipeline; break;
1793
+ case GGML_TYPE_Q4_1: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q4_1 ].pipeline; break;
1794
+ case GGML_TYPE_Q5_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q5_0 ].pipeline; break;
1795
+ case GGML_TYPE_Q5_1: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q5_1 ].pipeline; break;
1796
+ case GGML_TYPE_Q8_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q8_0 ].pipeline; break;
1797
+ case GGML_TYPE_Q2_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q2_K ].pipeline; break;
1798
+ case GGML_TYPE_Q3_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q3_K ].pipeline; break;
1799
+ case GGML_TYPE_Q4_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q4_K ].pipeline; break;
1800
+ case GGML_TYPE_Q5_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q5_K ].pipeline; break;
1801
+ case GGML_TYPE_Q6_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q6_K ].pipeline; break;
1802
+ case GGML_TYPE_IQ2_XXS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_XXS].pipeline; break;
1803
+ case GGML_TYPE_IQ2_XS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_XS ].pipeline; break;
1804
+ case GGML_TYPE_I32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_I32 ].pipeline; break;
1805
+ default: GGML_ASSERT(false && "not implemented");
1806
+ }
1807
+
1808
+ [encoder setComputePipelineState:pipeline];
1809
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
1810
+ [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];
1811
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:2];
1812
+ [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:3];
1813
+ [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:4];
1814
+ [encoder setBytes:&nb02 length:sizeof(uint64_t) atIndex:5];
1815
+ [encoder setBytes:&ne10 length:sizeof( int64_t) atIndex:6];
1816
+ [encoder setBytes:&nb10 length:sizeof( int64_t) atIndex:7];
1817
+ [encoder setBytes:&nb11 length:sizeof( int64_t) atIndex:8];
1818
+ [encoder setBytes:&nb1 length:sizeof(uint64_t) atIndex:9];
1819
+ [encoder setBytes:&nb2 length:sizeof(uint64_t) atIndex:10];
1820
+
1821
+ [encoder dispatchThreadgroups:MTLSizeMake(ne10, ne11, 1) threadsPerThreadgroup:MTLSizeMake(32, 1, 1)];
1822
+ } break;
1823
+ case GGML_OP_RMS_NORM:
1824
+ {
1825
+ GGML_ASSERT(ne00 % 4 == 0);
1826
+
1827
+ float eps;
1828
+ memcpy(&eps, dst->op_params, sizeof(float));
1829
+
1830
+ int nth = 32; // SIMD width
1831
+
1832
+ while (nth < ne00/4 && nth < 1024) {
1833
+ nth *= 2;
1834
+ }
2465
1835
 
2466
- if (encoder != nil) {
2467
- [encoder endEncoding];
2468
- encoder = nil;
1836
+ id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_RMS_NORM].pipeline;
1837
+
1838
+ [encoder setComputePipelineState:pipeline];
1839
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
1840
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
1841
+ [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:2];
1842
+ [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:3];
1843
+ [encoder setBytes:&eps length:sizeof( float) atIndex:4];
1844
+ [encoder setThreadgroupMemoryLength:32*sizeof(float) atIndex:0];
1845
+
1846
+ const int64_t nrows = ggml_nrows(src0);
1847
+
1848
+ [encoder dispatchThreadgroups:MTLSizeMake(nrows, 1, 1) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];
1849
+ } break;
1850
+ case GGML_OP_GROUP_NORM:
1851
+ {
1852
+ GGML_ASSERT(ne00 % 4 == 0);
1853
+
1854
+ //float eps;
1855
+ //memcpy(&eps, dst->op_params, sizeof(float));
1856
+
1857
+ const float eps = 1e-6f; // TODO: temporarily hardcoded
1858
+
1859
+ const int32_t n_groups = ((int32_t *) dst->op_params)[0];
1860
+
1861
+ int nth = 32; // SIMD width
1862
+
1863
+ //while (nth < ne00/4 && nth < 1024) {
1864
+ // nth *= 2;
1865
+ //}
1866
+
1867
+ id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GROUP_NORM].pipeline;
1868
+
1869
+ [encoder setComputePipelineState:pipeline];
1870
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
1871
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
1872
+ [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:2];
1873
+ [encoder setBytes:&ne01 length:sizeof( int64_t) atIndex:3];
1874
+ [encoder setBytes:&ne02 length:sizeof( int64_t) atIndex:4];
1875
+ [encoder setBytes:&nb00 length:sizeof(uint64_t) atIndex:5];
1876
+ [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:6];
1877
+ [encoder setBytes:&nb02 length:sizeof(uint64_t) atIndex:7];
1878
+ [encoder setBytes:&n_groups length:sizeof( int32_t) atIndex:8];
1879
+ [encoder setBytes:&eps length:sizeof( float) atIndex:9];
1880
+ [encoder setThreadgroupMemoryLength:32*sizeof(float) atIndex:0];
1881
+
1882
+ [encoder dispatchThreadgroups:MTLSizeMake(n_groups, 1, 1) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];
1883
+ } break;
1884
+ case GGML_OP_NORM:
1885
+ {
1886
+ float eps;
1887
+ memcpy(&eps, dst->op_params, sizeof(float));
1888
+
1889
+ const int nth = MIN(256, ne00);
1890
+
1891
+ id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_NORM].pipeline;
1892
+
1893
+ [encoder setComputePipelineState:pipeline];
1894
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
1895
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
1896
+ [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:2];
1897
+ [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:3];
1898
+ [encoder setBytes:&eps length:sizeof( float) atIndex:4];
1899
+ [encoder setThreadgroupMemoryLength:GGML_PAD(nth*sizeof(float), 16) atIndex:0];
1900
+
1901
+ const int64_t nrows = ggml_nrows(src0);
1902
+
1903
+ [encoder dispatchThreadgroups:MTLSizeMake(nrows, 1, 1) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];
1904
+ } break;
1905
+ case GGML_OP_ALIBI:
1906
+ {
1907
+ GGML_ASSERT((src0t == GGML_TYPE_F32));
1908
+
1909
+ const int nth = MIN(1024, ne00);
1910
+
1911
+ //const int n_past = ((int32_t *) dst->op_params)[0];
1912
+ const int n_head = ((int32_t *) dst->op_params)[1];
1913
+ float max_bias;
1914
+ memcpy(&max_bias, (int32_t *) dst->op_params + 2, sizeof(float));
1915
+
1916
+ const int n_heads_log2_floor = 1 << (int) floor(log2(n_head));
1917
+ const float m0 = powf(2.0f, -(max_bias) / n_heads_log2_floor);
1918
+ const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_heads_log2_floor);
1919
+
1920
+ id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_ALIBI_F32].pipeline;
1921
+
1922
+ [encoder setComputePipelineState:pipeline];
1923
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
1924
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
1925
+ [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:2];
1926
+ [encoder setBytes:&ne01 length:sizeof( int64_t) atIndex:3];
1927
+ [encoder setBytes:&ne02 length:sizeof( int64_t) atIndex:4];
1928
+ [encoder setBytes:&ne03 length:sizeof( int64_t) atIndex:5];
1929
+ [encoder setBytes:&nb00 length:sizeof(uint64_t) atIndex:6];
1930
+ [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:7];
1931
+ [encoder setBytes:&nb02 length:sizeof(uint64_t) atIndex:8];
1932
+ [encoder setBytes:&nb03 length:sizeof(uint64_t) atIndex:9];
1933
+ [encoder setBytes:&ne0 length:sizeof( int64_t) atIndex:10];
1934
+ [encoder setBytes:&ne1 length:sizeof( int64_t) atIndex:11];
1935
+ [encoder setBytes:&ne2 length:sizeof( int64_t) atIndex:12];
1936
+ [encoder setBytes:&ne3 length:sizeof( int64_t) atIndex:13];
1937
+ [encoder setBytes:&nb0 length:sizeof(uint64_t) atIndex:14];
1938
+ [encoder setBytes:&nb1 length:sizeof(uint64_t) atIndex:15];
1939
+ [encoder setBytes:&nb2 length:sizeof(uint64_t) atIndex:16];
1940
+ [encoder setBytes:&nb3 length:sizeof(uint64_t) atIndex:17];
1941
+ [encoder setBytes:&m0 length:sizeof( float) atIndex:18];
1942
+ [encoder setBytes:&m1 length:sizeof( float) atIndex:19];
1943
+ [encoder setBytes:&n_heads_log2_floor length:sizeof(int) atIndex:20];
1944
+
1945
+ [encoder dispatchThreadgroups:MTLSizeMake(ne01, ne02, ne03) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];
1946
+ } break;
1947
+ case GGML_OP_ROPE:
1948
+ {
1949
+ GGML_ASSERT(ne10 == ne02);
1950
+
1951
+ const int nth = MIN(1024, ne00);
1952
+
1953
+ const int n_past = ((int32_t *) dst->op_params)[0];
1954
+ const int n_dims = ((int32_t *) dst->op_params)[1];
1955
+ const int mode = ((int32_t *) dst->op_params)[2];
1956
+ // skip 3, n_ctx, used in GLM RoPE, unimplemented in metal
1957
+ const int n_orig_ctx = ((int32_t *) dst->op_params)[4];
1958
+
1959
+ float freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow;
1960
+ memcpy(&freq_base, (int32_t *) dst->op_params + 5, sizeof(float));
1961
+ memcpy(&freq_scale, (int32_t *) dst->op_params + 6, sizeof(float));
1962
+ memcpy(&ext_factor, (int32_t *) dst->op_params + 7, sizeof(float));
1963
+ memcpy(&attn_factor, (int32_t *) dst->op_params + 8, sizeof(float));
1964
+ memcpy(&beta_fast, (int32_t *) dst->op_params + 9, sizeof(float));
1965
+ memcpy(&beta_slow, (int32_t *) dst->op_params + 10, sizeof(float));
1966
+
1967
+ id<MTLComputePipelineState> pipeline = nil;
1968
+
1969
+ switch (src0->type) {
1970
+ case GGML_TYPE_F32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_ROPE_F32].pipeline; break;
1971
+ case GGML_TYPE_F16: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_ROPE_F16].pipeline; break;
1972
+ default: GGML_ASSERT(false);
1973
+ };
1974
+
1975
+ [encoder setComputePipelineState:pipeline];
1976
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
1977
+ [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];
1978
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:2];
1979
+ [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:3];
1980
+ [encoder setBytes:&ne01 length:sizeof( int64_t) atIndex:4];
1981
+ [encoder setBytes:&ne02 length:sizeof( int64_t) atIndex:5];
1982
+ [encoder setBytes:&ne03 length:sizeof( int64_t) atIndex:6];
1983
+ [encoder setBytes:&nb00 length:sizeof(uint64_t) atIndex:7];
1984
+ [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:8];
1985
+ [encoder setBytes:&nb02 length:sizeof(uint64_t) atIndex:9];
1986
+ [encoder setBytes:&nb03 length:sizeof(uint64_t) atIndex:10];
1987
+ [encoder setBytes:&ne0 length:sizeof( int64_t) atIndex:11];
1988
+ [encoder setBytes:&ne1 length:sizeof( int64_t) atIndex:12];
1989
+ [encoder setBytes:&ne2 length:sizeof( int64_t) atIndex:13];
1990
+ [encoder setBytes:&ne3 length:sizeof( int64_t) atIndex:14];
1991
+ [encoder setBytes:&nb0 length:sizeof(uint64_t) atIndex:15];
1992
+ [encoder setBytes:&nb1 length:sizeof(uint64_t) atIndex:16];
1993
+ [encoder setBytes:&nb2 length:sizeof(uint64_t) atIndex:17];
1994
+ [encoder setBytes:&nb3 length:sizeof(uint64_t) atIndex:18];
1995
+ [encoder setBytes:&n_past length:sizeof( int) atIndex:19];
1996
+ [encoder setBytes:&n_dims length:sizeof( int) atIndex:20];
1997
+ [encoder setBytes:&mode length:sizeof( int) atIndex:21];
1998
+ [encoder setBytes:&n_orig_ctx length:sizeof( int) atIndex:22];
1999
+ [encoder setBytes:&freq_base length:sizeof( float) atIndex:23];
2000
+ [encoder setBytes:&freq_scale length:sizeof( float) atIndex:24];
2001
+ [encoder setBytes:&ext_factor length:sizeof( float) atIndex:25];
2002
+ [encoder setBytes:&attn_factor length:sizeof( float) atIndex:26];
2003
+ [encoder setBytes:&beta_fast length:sizeof( float) atIndex:27];
2004
+ [encoder setBytes:&beta_slow length:sizeof( float) atIndex:28];
2005
+
2006
+ [encoder dispatchThreadgroups:MTLSizeMake(ne01, ne02, ne03) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];
2007
+ } break;
2008
+ case GGML_OP_IM2COL:
2009
+ {
2010
+ GGML_ASSERT(src0->type == GGML_TYPE_F16);
2011
+ GGML_ASSERT(src1->type == GGML_TYPE_F32);
2012
+ GGML_ASSERT( dst->type == GGML_TYPE_F16);
2013
+
2014
+ const int32_t s0 = ((const int32_t *)(dst->op_params))[0];
2015
+ const int32_t s1 = ((const int32_t *)(dst->op_params))[1];
2016
+ const int32_t p0 = ((const int32_t *)(dst->op_params))[2];
2017
+ const int32_t p1 = ((const int32_t *)(dst->op_params))[3];
2018
+ const int32_t d0 = ((const int32_t *)(dst->op_params))[4];
2019
+ const int32_t d1 = ((const int32_t *)(dst->op_params))[5];
2020
+ const bool is_2D = ((const int32_t *)(dst->op_params))[6] == 1;
2021
+
2022
+ const int32_t N = src1->ne[is_2D ? 3 : 2];
2023
+ const int32_t IC = src1->ne[is_2D ? 2 : 1];
2024
+ const int32_t IH = is_2D ? src1->ne[1] : 1;
2025
+ const int32_t IW = src1->ne[0];
2026
+
2027
+ const int32_t KH = is_2D ? src0->ne[1] : 1;
2028
+ const int32_t KW = src0->ne[0];
2029
+
2030
+ const int32_t OH = is_2D ? dst->ne[2] : 1;
2031
+ const int32_t OW = dst->ne[1];
2032
+
2033
+ const int32_t CHW = IC * KH * KW;
2034
+
2035
+ const int32_t ofs0 = src1->nb[is_2D ? 3 : 2] / 4;
2036
+ const int32_t ofs1 = src1->nb[is_2D ? 2 : 1] / 4;
2037
+
2038
+ id<MTLComputePipelineState> pipeline = nil;
2039
+
2040
+ switch (src0->type) {
2041
+ case GGML_TYPE_F32: GGML_ASSERT(false && "not implemented"); break;
2042
+ case GGML_TYPE_F16: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_IM2COL_F16].pipeline; break;
2043
+ default: GGML_ASSERT(false);
2044
+ };
2045
+
2046
+ [encoder setComputePipelineState:pipeline];
2047
+ [encoder setBuffer:id_src1 offset:offs_src1 atIndex:0];
2048
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
2049
+ [encoder setBytes:&ofs0 length:sizeof( int32_t) atIndex:2];
2050
+ [encoder setBytes:&ofs1 length:sizeof( int32_t) atIndex:3];
2051
+ [encoder setBytes:&IW length:sizeof( int32_t) atIndex:4];
2052
+ [encoder setBytes:&IH length:sizeof( int32_t) atIndex:5];
2053
+ [encoder setBytes:&CHW length:sizeof( int32_t) atIndex:6];
2054
+ [encoder setBytes:&s0 length:sizeof( int32_t) atIndex:7];
2055
+ [encoder setBytes:&s1 length:sizeof( int32_t) atIndex:8];
2056
+ [encoder setBytes:&p0 length:sizeof( int32_t) atIndex:9];
2057
+ [encoder setBytes:&p1 length:sizeof( int32_t) atIndex:10];
2058
+ [encoder setBytes:&d0 length:sizeof( int32_t) atIndex:11];
2059
+ [encoder setBytes:&d1 length:sizeof( int32_t) atIndex:12];
2060
+
2061
+ [encoder dispatchThreadgroups:MTLSizeMake(IC, OH, OW) threadsPerThreadgroup:MTLSizeMake(N, KH, KW)];
2062
+ } break;
2063
+ case GGML_OP_UPSCALE:
2064
+ {
2065
+ GGML_ASSERT(src0->type == GGML_TYPE_F32);
2066
+
2067
+ const int sf = dst->op_params[0];
2068
+
2069
+ const id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_UPSCALE_F32].pipeline;
2070
+
2071
+ [encoder setComputePipelineState:pipeline];
2072
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
2073
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
2074
+ [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:2];
2075
+ [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:3];
2076
+ [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:4];
2077
+ [encoder setBytes:&ne03 length:sizeof(ne03) atIndex:5];
2078
+ [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:6];
2079
+ [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:7];
2080
+ [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:8];
2081
+ [encoder setBytes:&nb03 length:sizeof(nb03) atIndex:9];
2082
+ [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:10];
2083
+ [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:11];
2084
+ [encoder setBytes:&ne2 length:sizeof(ne2) atIndex:12];
2085
+ [encoder setBytes:&ne3 length:sizeof(ne3) atIndex:13];
2086
+ [encoder setBytes:&nb0 length:sizeof(nb0) atIndex:14];
2087
+ [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:15];
2088
+ [encoder setBytes:&nb2 length:sizeof(nb2) atIndex:16];
2089
+ [encoder setBytes:&nb3 length:sizeof(nb3) atIndex:17];
2090
+ [encoder setBytes:&sf length:sizeof(sf) atIndex:18];
2091
+
2092
+ const int nth = MIN((int) pipeline.maxTotalThreadsPerThreadgroup, ne0);
2093
+
2094
+ [encoder dispatchThreadgroups:MTLSizeMake(ne1, ne2, ne3) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];
2095
+ } break;
2096
+ case GGML_OP_PAD:
2097
+ {
2098
+ GGML_ASSERT(src0->type == GGML_TYPE_F32);
2099
+
2100
+ id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_PAD_F32].pipeline;
2101
+
2102
+ [encoder setComputePipelineState:pipeline];
2103
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
2104
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
2105
+ [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:2];
2106
+ [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:3];
2107
+ [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:4];
2108
+ [encoder setBytes:&ne03 length:sizeof(ne03) atIndex:5];
2109
+ [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:6];
2110
+ [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:7];
2111
+ [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:8];
2112
+ [encoder setBytes:&nb03 length:sizeof(nb03) atIndex:9];
2113
+ [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:10];
2114
+ [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:11];
2115
+ [encoder setBytes:&ne2 length:sizeof(ne2) atIndex:12];
2116
+ [encoder setBytes:&ne3 length:sizeof(ne3) atIndex:13];
2117
+ [encoder setBytes:&nb0 length:sizeof(nb0) atIndex:14];
2118
+ [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:15];
2119
+ [encoder setBytes:&nb2 length:sizeof(nb2) atIndex:16];
2120
+ [encoder setBytes:&nb3 length:sizeof(nb3) atIndex:17];
2121
+
2122
+ const int nth = MIN(1024, ne0);
2123
+
2124
+ [encoder dispatchThreadgroups:MTLSizeMake(ne1, ne2, ne3) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];
2125
+ } break;
2126
+ case GGML_OP_ARGSORT:
2127
+ {
2128
+ GGML_ASSERT(src0->type == GGML_TYPE_F32);
2129
+ GGML_ASSERT( dst->type == GGML_TYPE_I32);
2130
+
2131
+ const int nrows = ggml_nrows(src0);
2132
+
2133
+ enum ggml_sort_order order = (enum ggml_sort_order) dst->op_params[0];
2134
+
2135
+ id<MTLComputePipelineState> pipeline = nil;
2136
+
2137
+ switch (order) {
2138
+ case GGML_SORT_ASC: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_ARGSORT_F32_I32_ASC].pipeline; break;
2139
+ case GGML_SORT_DESC: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_ARGSORT_F32_I32_DESC].pipeline; break;
2140
+ default: GGML_ASSERT(false);
2141
+ };
2142
+
2143
+ [encoder setComputePipelineState:pipeline];
2144
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
2145
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
2146
+ [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:2];
2147
+
2148
+ [encoder dispatchThreadgroups:MTLSizeMake(1, nrows, 1) threadsPerThreadgroup:MTLSizeMake(ne00, 1, 1)];
2149
+ } break;
2150
+ case GGML_OP_LEAKY_RELU:
2151
+ {
2152
+ GGML_ASSERT(src0->type == GGML_TYPE_F32);
2153
+
2154
+ float slope;
2155
+ memcpy(&slope, dst->op_params, sizeof(float));
2156
+
2157
+ id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_LEAKY_RELU_F32].pipeline;
2158
+
2159
+ [encoder setComputePipelineState:pipeline];
2160
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
2161
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
2162
+ [encoder setBytes:&slope length:sizeof(slope) atIndex:2];
2163
+
2164
+ const int64_t n = ggml_nelements(dst);
2165
+
2166
+ [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
2167
+ } break;
2168
+ case GGML_OP_DUP:
2169
+ case GGML_OP_CPY:
2170
+ case GGML_OP_CONT:
2171
+ {
2172
+ GGML_ASSERT(ne00 % ggml_blck_size(src0->type) == 0);
2173
+
2174
+ int nth = MIN(1024, ne00/ggml_blck_size(src0->type));
2175
+
2176
+ id<MTLComputePipelineState> pipeline = nil;
2177
+
2178
+ switch (src0t) {
2179
+ case GGML_TYPE_F32:
2180
+ {
2181
+ GGML_ASSERT(ne0 % ggml_blck_size(dst->type) == 0);
2182
+
2183
+ switch (dstt) {
2184
+ case GGML_TYPE_F16: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_F16].pipeline; break;
2185
+ case GGML_TYPE_F32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_F32].pipeline; break;
2186
+ case GGML_TYPE_Q8_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_Q8_0].pipeline; break;
2187
+ case GGML_TYPE_Q4_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_Q4_0].pipeline; break;
2188
+ case GGML_TYPE_Q4_1: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_Q4_1].pipeline; break;
2189
+ //case GGML_TYPE_Q5_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_0].pipeline; break;
2190
+ //case GGML_TYPE_Q5_1: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_1].pipeline; break;
2191
+ default: GGML_ASSERT(false && "not implemented");
2192
+ };
2193
+ } break;
2194
+ case GGML_TYPE_F16:
2195
+ {
2196
+ switch (dstt) {
2197
+ case GGML_TYPE_F16: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F16_F16].pipeline; break;
2198
+ case GGML_TYPE_F32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F16_F32].pipeline; break;
2199
+ default: GGML_ASSERT(false && "not implemented");
2200
+ };
2201
+ } break;
2202
+ default: GGML_ASSERT(false && "not implemented");
2203
+ }
2204
+
2205
+ [encoder setComputePipelineState:pipeline];
2206
+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
2207
+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
2208
+ [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:2];
2209
+ [encoder setBytes:&ne01 length:sizeof( int64_t) atIndex:3];
2210
+ [encoder setBytes:&ne02 length:sizeof( int64_t) atIndex:4];
2211
+ [encoder setBytes:&ne03 length:sizeof( int64_t) atIndex:5];
2212
+ [encoder setBytes:&nb00 length:sizeof(uint64_t) atIndex:6];
2213
+ [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:7];
2214
+ [encoder setBytes:&nb02 length:sizeof(uint64_t) atIndex:8];
2215
+ [encoder setBytes:&nb03 length:sizeof(uint64_t) atIndex:9];
2216
+ [encoder setBytes:&ne0 length:sizeof( int64_t) atIndex:10];
2217
+ [encoder setBytes:&ne1 length:sizeof( int64_t) atIndex:11];
2218
+ [encoder setBytes:&ne2 length:sizeof( int64_t) atIndex:12];
2219
+ [encoder setBytes:&ne3 length:sizeof( int64_t) atIndex:13];
2220
+ [encoder setBytes:&nb0 length:sizeof(uint64_t) atIndex:14];
2221
+ [encoder setBytes:&nb1 length:sizeof(uint64_t) atIndex:15];
2222
+ [encoder setBytes:&nb2 length:sizeof(uint64_t) atIndex:16];
2223
+ [encoder setBytes:&nb3 length:sizeof(uint64_t) atIndex:17];
2224
+
2225
+ [encoder dispatchThreadgroups:MTLSizeMake(ne01, ne02, ne03) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];
2226
+ } break;
2227
+ default:
2228
+ {
2229
+ GGML_METAL_LOG_ERROR("%s: error: node %3d, op = %8s not implemented\n", __func__, i, ggml_op_name(dst->op));
2230
+ GGML_ASSERT(false);
2231
+ }
2469
2232
  }
2470
2233
 
2471
- [command_buffer commit];
2472
- });
2473
- }
2234
+ #ifndef GGML_METAL_NDEBUG
2235
+ [encoder popDebugGroup];
2236
+ #endif
2237
+ }
2238
+
2239
+ if (encoder != nil) {
2240
+ [encoder endEncoding];
2241
+ encoder = nil;
2242
+ }
2474
2243
 
2475
- // wait for all threads to finish
2476
- dispatch_barrier_sync(ctx->d_queue, ^{});
2244
+ [command_buffer commit];
2245
+ });
2477
2246
 
2478
- // check status of command buffers
2247
+ // Wait for completion and check status of each command buffer
2479
2248
  // needed to detect if the device ran out-of-memory for example (#1881)
2480
- for (int i = 0; i < n_cb; i++) {
2481
- [ctx->command_buffers[i] waitUntilCompleted];
2482
2249
 
2483
- MTLCommandBufferStatus status = (MTLCommandBufferStatus) [ctx->command_buffers[i] status];
2250
+ for (int i = 0; i < n_cb; ++i) {
2251
+ id<MTLCommandBuffer> command_buffer = command_buffers[i];
2252
+ [command_buffer waitUntilCompleted];
2253
+
2254
+ MTLCommandBufferStatus status = [command_buffer status];
2484
2255
  if (status != MTLCommandBufferStatusCompleted) {
2485
2256
  GGML_METAL_LOG_INFO("%s: command buffer %d failed with status %lu\n", __func__, i, status);
2486
2257
  return false;
@@ -2520,13 +2291,13 @@ static void ggml_backend_metal_free_device(void) {
2520
2291
  }
2521
2292
  }
2522
2293
 
2523
- static void * ggml_backend_metal_buffer_get_base(ggml_backend_buffer_t buffer) {
2524
- struct ggml_backend_metal_buffer_context * ctx = (struct ggml_backend_metal_buffer_context *)buffer->context;
2294
+ GGML_CALL static const char * ggml_backend_metal_buffer_get_name(ggml_backend_buffer_t buffer) {
2295
+ return "Metal";
2525
2296
 
2526
- return ctx->all_data;
2297
+ UNUSED(buffer);
2527
2298
  }
2528
2299
 
2529
- static void ggml_backend_metal_buffer_free_buffer(ggml_backend_buffer_t buffer) {
2300
+ GGML_CALL static void ggml_backend_metal_buffer_free_buffer(ggml_backend_buffer_t buffer) {
2530
2301
  struct ggml_backend_metal_buffer_context * ctx = (struct ggml_backend_metal_buffer_context *)buffer->context;
2531
2302
 
2532
2303
  for (int i = 0; i < ctx->n_buffers; i++) {
@@ -2541,50 +2312,80 @@ static void ggml_backend_metal_buffer_free_buffer(ggml_backend_buffer_t buffer)
2541
2312
  free(ctx);
2542
2313
  }
2543
2314
 
2544
- static void ggml_backend_metal_buffer_set_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size) {
2545
- memcpy((char *)tensor->data + offset, data, size);
2315
+ GGML_CALL static void * ggml_backend_metal_buffer_get_base(ggml_backend_buffer_t buffer) {
2316
+ struct ggml_backend_metal_buffer_context * ctx = (struct ggml_backend_metal_buffer_context *)buffer->context;
2546
2317
 
2547
- UNUSED(buffer);
2318
+ return ctx->all_data;
2548
2319
  }
2549
2320
 
2550
- static void ggml_backend_metal_buffer_get_tensor(ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size) {
2551
- memcpy(data, (const char *)tensor->data + offset, size);
2321
+ GGML_CALL static void ggml_backend_metal_buffer_set_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size) {
2322
+ memcpy((char *)tensor->data + offset, data, size);
2552
2323
 
2553
2324
  UNUSED(buffer);
2554
2325
  }
2555
2326
 
2556
- static void ggml_backend_metal_buffer_cpy_tensor_from(ggml_backend_buffer_t buffer, struct ggml_tensor * src, struct ggml_tensor * dst) {
2557
- ggml_backend_tensor_get(src, dst->data, 0, ggml_nbytes(src));
2327
+ GGML_CALL static void ggml_backend_metal_buffer_get_tensor(ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size) {
2328
+ memcpy(data, (const char *)tensor->data + offset, size);
2558
2329
 
2559
2330
  UNUSED(buffer);
2560
2331
  }
2561
2332
 
2562
- static void ggml_backend_metal_buffer_cpy_tensor_to(ggml_backend_buffer_t buffer, struct ggml_tensor * src, struct ggml_tensor * dst) {
2563
- ggml_backend_tensor_set(dst, src->data, 0, ggml_nbytes(src));
2333
+ GGML_CALL static bool ggml_backend_metal_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const struct ggml_tensor * src, struct ggml_tensor * dst) {
2334
+ if (ggml_backend_buffer_is_host(src->buffer)) {
2335
+ memcpy(dst->data, src->data, ggml_nbytes(src));
2336
+ return true;
2337
+ }
2338
+ return false;
2564
2339
 
2565
2340
  UNUSED(buffer);
2566
2341
  }
2567
2342
 
2568
- static void ggml_backend_metal_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {
2343
+ GGML_CALL static void ggml_backend_metal_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {
2569
2344
  struct ggml_backend_metal_buffer_context * ctx = (struct ggml_backend_metal_buffer_context *)buffer->context;
2570
2345
 
2571
2346
  memset(ctx->all_data, value, ctx->all_size);
2572
2347
  }
2573
2348
 
2574
2349
  static struct ggml_backend_buffer_i ggml_backend_metal_buffer_i = {
2350
+ /* .get_name = */ ggml_backend_metal_buffer_get_name,
2575
2351
  /* .free_buffer = */ ggml_backend_metal_buffer_free_buffer,
2576
2352
  /* .get_base = */ ggml_backend_metal_buffer_get_base,
2577
2353
  /* .init_tensor = */ NULL,
2578
2354
  /* .set_tensor = */ ggml_backend_metal_buffer_set_tensor,
2579
2355
  /* .get_tensor = */ ggml_backend_metal_buffer_get_tensor,
2580
- /* .cpy_tensor_from = */ ggml_backend_metal_buffer_cpy_tensor_from,
2581
- /* .cpy_tensor_to = */ ggml_backend_metal_buffer_cpy_tensor_to,
2356
+ /* .cpy_tensor = */ ggml_backend_metal_buffer_cpy_tensor,
2582
2357
  /* .clear = */ ggml_backend_metal_buffer_clear,
2358
+ /* .reset = */ NULL,
2583
2359
  };
2584
2360
 
2585
2361
  // default buffer type
2586
2362
 
2587
- static ggml_backend_buffer_t ggml_backend_metal_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {
2363
+ GGML_CALL static const char * ggml_backend_metal_buffer_type_get_name(ggml_backend_buffer_type_t buft) {
2364
+ return "Metal";
2365
+
2366
+ UNUSED(buft);
2367
+ }
2368
+
2369
+ static void ggml_backend_metal_log_allocated_size(id<MTLDevice> device) {
2370
+ #if TARGET_OS_OSX || (TARGET_OS_IOS && __clang_major__ >= 15)
2371
+ if (@available(macOS 10.12, iOS 16.0, *)) {
2372
+ GGML_METAL_LOG_INFO(", (%8.2f / %8.2f)",
2373
+ device.currentAllocatedSize / 1024.0 / 1024.0,
2374
+ device.recommendedMaxWorkingSetSize / 1024.0 / 1024.0);
2375
+
2376
+ if (device.currentAllocatedSize > device.recommendedMaxWorkingSetSize) {
2377
+ GGML_METAL_LOG_WARN("%s: warning: current allocated size is greater than the recommended max working set size\n", __func__);
2378
+ } else {
2379
+ GGML_METAL_LOG_INFO("\n");
2380
+ }
2381
+ } else {
2382
+ GGML_METAL_LOG_INFO(", (%8.2f)\n", device.currentAllocatedSize / 1024.0 / 1024.0);
2383
+ }
2384
+ #endif
2385
+ UNUSED(device);
2386
+ }
2387
+
2388
+ GGML_CALL static ggml_backend_buffer_t ggml_backend_metal_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {
2588
2389
  struct ggml_backend_metal_buffer_context * ctx = malloc(sizeof(struct ggml_backend_metal_buffer_context));
2589
2390
 
2590
2391
  const size_t size_page = sysconf(_SC_PAGESIZE);
@@ -2616,46 +2417,32 @@ static ggml_backend_buffer_t ggml_backend_metal_buffer_type_alloc_buffer(ggml_ba
2616
2417
  }
2617
2418
 
2618
2419
  GGML_METAL_LOG_INFO("%s: allocated buffer, size = %8.2f MiB", __func__, size_aligned / 1024.0 / 1024.0);
2619
-
2620
-
2621
- #if TARGET_OS_OSX
2622
- GGML_METAL_LOG_INFO(", (%8.2f / %8.2f)",
2623
- device.currentAllocatedSize / 1024.0 / 1024.0,
2624
- device.recommendedMaxWorkingSetSize / 1024.0 / 1024.0);
2625
-
2626
- if (device.currentAllocatedSize > device.recommendedMaxWorkingSetSize) {
2627
- GGML_METAL_LOG_WARN("%s: warning: current allocated size is greater than the recommended max working set size\n", __func__);
2628
- } else {
2629
- GGML_METAL_LOG_INFO("\n");
2630
- }
2631
- #else
2632
- GGML_METAL_LOG_INFO(", (%8.2f)\n", device.currentAllocatedSize / 1024.0 / 1024.0);
2633
- #endif
2634
-
2420
+ ggml_backend_metal_log_allocated_size(device);
2635
2421
 
2636
2422
  return ggml_backend_buffer_init(buft, ggml_backend_metal_buffer_i, ctx, size);
2637
2423
  }
2638
2424
 
2639
- static size_t ggml_backend_metal_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) {
2425
+ GGML_CALL static size_t ggml_backend_metal_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) {
2640
2426
  return 32;
2641
2427
  UNUSED(buft);
2642
2428
  }
2643
2429
 
2644
- static bool ggml_backend_metal_buffer_type_supports_backend(ggml_backend_buffer_type_t buft, ggml_backend_t backend) {
2430
+ GGML_CALL static bool ggml_backend_metal_buffer_type_supports_backend(ggml_backend_buffer_type_t buft, ggml_backend_t backend) {
2645
2431
  return ggml_backend_is_metal(backend) || ggml_backend_is_cpu(backend);
2646
2432
 
2647
2433
  UNUSED(buft);
2648
2434
  }
2649
2435
 
2650
- static bool ggml_backend_metal_buffer_type_is_host(ggml_backend_buffer_type_t buft) {
2436
+ GGML_CALL static bool ggml_backend_metal_buffer_type_is_host(ggml_backend_buffer_type_t buft) {
2651
2437
  return true;
2652
2438
 
2653
2439
  UNUSED(buft);
2654
2440
  }
2655
2441
 
2656
- ggml_backend_buffer_type_t ggml_backend_metal_buffer_type(void) {
2442
+ GGML_CALL ggml_backend_buffer_type_t ggml_backend_metal_buffer_type(void) {
2657
2443
  static struct ggml_backend_buffer_type ggml_backend_buffer_type_metal = {
2658
2444
  /* .iface = */ {
2445
+ /* .get_name = */ ggml_backend_metal_buffer_type_get_name,
2659
2446
  /* .alloc_buffer = */ ggml_backend_metal_buffer_type_alloc_buffer,
2660
2447
  /* .get_alignment = */ ggml_backend_metal_buffer_type_get_alignment,
2661
2448
  /* .get_alloc_size = */ NULL, // defaults to ggml_nbytes
@@ -2670,7 +2457,7 @@ ggml_backend_buffer_type_t ggml_backend_metal_buffer_type(void) {
2670
2457
 
2671
2458
  // buffer from ptr
2672
2459
 
2673
- ggml_backend_buffer_t ggml_backend_metal_buffer_from_ptr(void * data, size_t size, size_t max_size) {
2460
+ GGML_CALL ggml_backend_buffer_t ggml_backend_metal_buffer_from_ptr(void * data, size_t size, size_t max_size) {
2674
2461
  struct ggml_backend_metal_buffer_context * ctx = malloc(sizeof(struct ggml_backend_metal_buffer_context));
2675
2462
 
2676
2463
  ctx->all_data = data;
@@ -2679,6 +2466,14 @@ ggml_backend_buffer_t ggml_backend_metal_buffer_from_ptr(void * data, size_t siz
2679
2466
  ctx->n_buffers = 0;
2680
2467
 
2681
2468
  const size_t size_page = sysconf(_SC_PAGESIZE);
2469
+
2470
+ // page-align the data ptr
2471
+ {
2472
+ const uintptr_t offs = (uintptr_t) data % size_page;
2473
+ data = (void *) ((char *) data - offs);
2474
+ size += offs;
2475
+ }
2476
+
2682
2477
  size_t size_aligned = size;
2683
2478
  if ((size_aligned % size_page) != 0) {
2684
2479
  size_aligned += (size_page - (size_aligned % size_page));
@@ -2730,63 +2525,50 @@ ggml_backend_buffer_t ggml_backend_metal_buffer_from_ptr(void * data, size_t siz
2730
2525
  }
2731
2526
  }
2732
2527
 
2733
- #if TARGET_OS_OSX
2734
- GGML_METAL_LOG_INFO(", (%8.2f / %8.2f)",
2735
- device.currentAllocatedSize / 1024.0 / 1024.0,
2736
- device.recommendedMaxWorkingSetSize / 1024.0 / 1024.0);
2737
-
2738
- if (device.currentAllocatedSize > device.recommendedMaxWorkingSetSize) {
2739
- GGML_METAL_LOG_WARN("%s: warning: current allocated size is greater than the recommended max working set size\n", __func__);
2740
- } else {
2741
- GGML_METAL_LOG_INFO("\n");
2742
- }
2743
- #else
2744
- GGML_METAL_LOG_INFO(", (%8.2f)\n", device.currentAllocatedSize / 1024.0 / 1024.0);
2745
- #endif
2528
+ ggml_backend_metal_log_allocated_size(device);
2746
2529
 
2747
2530
  return ggml_backend_buffer_init(ggml_backend_metal_buffer_type(), ggml_backend_metal_buffer_i, ctx, size);
2748
2531
  }
2749
2532
 
2750
2533
  // backend
2751
2534
 
2752
- static const char * ggml_backend_metal_name(ggml_backend_t backend) {
2535
+ GGML_CALL static const char * ggml_backend_metal_name(ggml_backend_t backend) {
2753
2536
  return "Metal";
2754
2537
 
2755
2538
  UNUSED(backend);
2756
2539
  }
2757
2540
 
2758
- static void ggml_backend_metal_free(ggml_backend_t backend) {
2541
+ GGML_CALL static void ggml_backend_metal_free(ggml_backend_t backend) {
2759
2542
  struct ggml_metal_context * ctx = (struct ggml_metal_context *)backend->context;
2760
2543
  ggml_metal_free(ctx);
2761
2544
  free(backend);
2762
2545
  }
2763
2546
 
2764
- static ggml_backend_buffer_type_t ggml_backend_metal_get_default_buffer_type(ggml_backend_t backend) {
2547
+ GGML_CALL static ggml_backend_buffer_type_t ggml_backend_metal_get_default_buffer_type(ggml_backend_t backend) {
2765
2548
  return ggml_backend_metal_buffer_type();
2766
2549
 
2767
2550
  UNUSED(backend);
2768
2551
  }
2769
2552
 
2770
- static bool ggml_backend_metal_graph_compute(ggml_backend_t backend, struct ggml_cgraph * cgraph) {
2553
+ GGML_CALL static bool ggml_backend_metal_graph_compute(ggml_backend_t backend, struct ggml_cgraph * cgraph) {
2771
2554
  struct ggml_metal_context * metal_ctx = (struct ggml_metal_context *)backend->context;
2772
2555
 
2773
2556
  return ggml_metal_graph_compute(metal_ctx, cgraph);
2774
2557
  }
2775
2558
 
2776
- static bool ggml_backend_metal_supports_op(ggml_backend_t backend, const struct ggml_tensor * op) {
2777
- return ggml_metal_supports_op(op);
2559
+ GGML_CALL static bool ggml_backend_metal_supports_op(ggml_backend_t backend, const struct ggml_tensor * op) {
2560
+ struct ggml_metal_context * metal_ctx = (struct ggml_metal_context *)backend->context;
2778
2561
 
2779
- UNUSED(backend);
2562
+ return ggml_metal_supports_op(metal_ctx, op);
2780
2563
  }
2781
2564
 
2782
- static struct ggml_backend_i metal_backend_i = {
2565
+ static struct ggml_backend_i ggml_backend_metal_i = {
2783
2566
  /* .get_name = */ ggml_backend_metal_name,
2784
2567
  /* .free = */ ggml_backend_metal_free,
2785
2568
  /* .get_default_buffer_type = */ ggml_backend_metal_get_default_buffer_type,
2786
2569
  /* .set_tensor_async = */ NULL,
2787
2570
  /* .get_tensor_async = */ NULL,
2788
- /* .cpy_tensor_from_async = */ NULL,
2789
- /* .cpy_tensor_to_async = */ NULL,
2571
+ /* .cpy_tensor_async = */ NULL,
2790
2572
  /* .synchronize = */ NULL,
2791
2573
  /* .graph_plan_create = */ NULL,
2792
2574
  /* .graph_plan_free = */ NULL,
@@ -2795,6 +2577,11 @@ static struct ggml_backend_i metal_backend_i = {
2795
2577
  /* .supports_op = */ ggml_backend_metal_supports_op,
2796
2578
  };
2797
2579
 
2580
+ void ggml_backend_metal_log_set_callback(ggml_log_callback log_callback, void * user_data) {
2581
+ ggml_metal_log_callback = log_callback;
2582
+ ggml_metal_log_user_data = user_data;
2583
+ }
2584
+
2798
2585
  ggml_backend_t ggml_backend_metal_init(void) {
2799
2586
  struct ggml_metal_context * ctx = ggml_metal_init(GGML_DEFAULT_N_THREADS);
2800
2587
 
@@ -2805,7 +2592,7 @@ ggml_backend_t ggml_backend_metal_init(void) {
2805
2592
  ggml_backend_t metal_backend = malloc(sizeof(struct ggml_backend));
2806
2593
 
2807
2594
  *metal_backend = (struct ggml_backend) {
2808
- /* .interface = */ metal_backend_i,
2595
+ /* .interface = */ ggml_backend_metal_i,
2809
2596
  /* .context = */ ctx,
2810
2597
  };
2811
2598
 
@@ -2813,7 +2600,7 @@ ggml_backend_t ggml_backend_metal_init(void) {
2813
2600
  }
2814
2601
 
2815
2602
  bool ggml_backend_is_metal(ggml_backend_t backend) {
2816
- return backend->iface.get_name == ggml_backend_metal_name;
2603
+ return backend && backend->iface.get_name == ggml_backend_metal_name;
2817
2604
  }
2818
2605
 
2819
2606
  void ggml_backend_metal_set_n_cb(ggml_backend_t backend, int n_cb) {
@@ -2821,7 +2608,7 @@ void ggml_backend_metal_set_n_cb(ggml_backend_t backend, int n_cb) {
2821
2608
 
2822
2609
  struct ggml_metal_context * ctx = (struct ggml_metal_context *)backend->context;
2823
2610
 
2824
- ggml_metal_set_n_cb(ctx, n_cb);
2611
+ ctx->n_cb = MIN(n_cb, GGML_METAL_MAX_BUFFERS);
2825
2612
  }
2826
2613
 
2827
2614
  bool ggml_backend_metal_supports_family(ggml_backend_t backend, int family) {
@@ -2832,9 +2619,9 @@ bool ggml_backend_metal_supports_family(ggml_backend_t backend, int family) {
2832
2619
  return [ctx->device supportsFamily:(MTLGPUFamilyApple1 + family - 1)];
2833
2620
  }
2834
2621
 
2835
- ggml_backend_t ggml_backend_reg_metal_init(const char * params, void * user_data); // silence warning
2622
+ GGML_CALL ggml_backend_t ggml_backend_reg_metal_init(const char * params, void * user_data); // silence warning
2836
2623
 
2837
- ggml_backend_t ggml_backend_reg_metal_init(const char * params, void * user_data) {
2624
+ GGML_CALL ggml_backend_t ggml_backend_reg_metal_init(const char * params, void * user_data) {
2838
2625
  return ggml_backend_metal_init();
2839
2626
 
2840
2627
  GGML_UNUSED(params);