llama_cpp 0.16.2 → 0.17.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (177) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +18 -0
  3. data/README.md +7 -12
  4. data/ext/llama_cpp/extconf.rb +2 -43
  5. data/ext/llama_cpp/llama_cpp.cpp +8 -0
  6. data/lib/llama_cpp/version.rb +3 -3
  7. data/sig/llama_cpp.rbs +3 -0
  8. metadata +2 -171
  9. data/vendor/include/.gitkeep +0 -0
  10. data/vendor/lib/.gitkeep +0 -0
  11. data/vendor/tmp/llama.cpp/LICENSE +0 -21
  12. data/vendor/tmp/llama.cpp/Makefile +0 -1124
  13. data/vendor/tmp/llama.cpp/ggml-alloc.c +0 -1041
  14. data/vendor/tmp/llama.cpp/ggml-alloc.h +0 -76
  15. data/vendor/tmp/llama.cpp/ggml-backend-impl.h +0 -153
  16. data/vendor/tmp/llama.cpp/ggml-backend.c +0 -2225
  17. data/vendor/tmp/llama.cpp/ggml-backend.h +0 -236
  18. data/vendor/tmp/llama.cpp/ggml-blas.cpp +0 -363
  19. data/vendor/tmp/llama.cpp/ggml-blas.h +0 -23
  20. data/vendor/tmp/llama.cpp/ggml-common.h +0 -1805
  21. data/vendor/tmp/llama.cpp/ggml-cuda/acc.cu +0 -47
  22. data/vendor/tmp/llama.cpp/ggml-cuda/arange.cu +0 -34
  23. data/vendor/tmp/llama.cpp/ggml-cuda/argsort.cu +0 -104
  24. data/vendor/tmp/llama.cpp/ggml-cuda/binbcast.cu +0 -280
  25. data/vendor/tmp/llama.cpp/ggml-cuda/clamp.cu +0 -34
  26. data/vendor/tmp/llama.cpp/ggml-cuda/concat.cu +0 -196
  27. data/vendor/tmp/llama.cpp/ggml-cuda/convert.cu +0 -686
  28. data/vendor/tmp/llama.cpp/ggml-cuda/cpy.cu +0 -490
  29. data/vendor/tmp/llama.cpp/ggml-cuda/diagmask.cu +0 -40
  30. data/vendor/tmp/llama.cpp/ggml-cuda/dmmv.cu +0 -674
  31. data/vendor/tmp/llama.cpp/ggml-cuda/fattn-tile-f16.cu +0 -319
  32. data/vendor/tmp/llama.cpp/ggml-cuda/fattn-tile-f32.cu +0 -312
  33. data/vendor/tmp/llama.cpp/ggml-cuda/fattn.cu +0 -345
  34. data/vendor/tmp/llama.cpp/ggml-cuda/getrows.cu +0 -178
  35. data/vendor/tmp/llama.cpp/ggml-cuda/im2col.cu +0 -104
  36. data/vendor/tmp/llama.cpp/ggml-cuda/mmq.cu +0 -88
  37. data/vendor/tmp/llama.cpp/ggml-cuda/mmvq.cu +0 -419
  38. data/vendor/tmp/llama.cpp/ggml-cuda/norm.cu +0 -221
  39. data/vendor/tmp/llama.cpp/ggml-cuda/pad.cu +0 -49
  40. data/vendor/tmp/llama.cpp/ggml-cuda/pool2d.cu +0 -94
  41. data/vendor/tmp/llama.cpp/ggml-cuda/quantize.cu +0 -112
  42. data/vendor/tmp/llama.cpp/ggml-cuda/rope.cu +0 -271
  43. data/vendor/tmp/llama.cpp/ggml-cuda/scale.cu +0 -31
  44. data/vendor/tmp/llama.cpp/ggml-cuda/softmax.cu +0 -206
  45. data/vendor/tmp/llama.cpp/ggml-cuda/sumrows.cu +0 -40
  46. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-f16.cu +0 -5
  47. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q4_0.cu +0 -5
  48. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q4_1.cu +0 -5
  49. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q5_0.cu +0 -5
  50. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q5_1.cu +0 -5
  51. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q8_0.cu +0 -5
  52. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-f16.cu +0 -5
  53. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q4_0.cu +0 -5
  54. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q4_1.cu +0 -5
  55. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q5_0.cu +0 -5
  56. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q5_1.cu +0 -5
  57. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q8_0.cu +0 -5
  58. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-f16.cu +0 -5
  59. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q4_0.cu +0 -5
  60. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q4_1.cu +0 -5
  61. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q5_0.cu +0 -5
  62. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q5_1.cu +0 -5
  63. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q8_0.cu +0 -5
  64. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-f16.cu +0 -5
  65. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q4_0.cu +0 -5
  66. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q4_1.cu +0 -5
  67. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q5_0.cu +0 -5
  68. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q5_1.cu +0 -5
  69. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q8_0.cu +0 -5
  70. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-f16.cu +0 -5
  71. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q4_0.cu +0 -5
  72. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q4_1.cu +0 -5
  73. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q5_0.cu +0 -5
  74. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q5_1.cu +0 -5
  75. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q8_0.cu +0 -5
  76. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-f16.cu +0 -5
  77. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q4_0.cu +0 -5
  78. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q4_1.cu +0 -5
  79. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q5_0.cu +0 -5
  80. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q5_1.cu +0 -5
  81. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q8_0.cu +0 -5
  82. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs256-f16-f16.cu +0 -5
  83. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-f16.cu +0 -5
  84. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q4_0.cu +0 -5
  85. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q4_1.cu +0 -5
  86. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q5_0.cu +0 -5
  87. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q5_1.cu +0 -5
  88. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q8_0.cu +0 -5
  89. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-f16.cu +0 -5
  90. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q4_0.cu +0 -5
  91. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q4_1.cu +0 -5
  92. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q5_0.cu +0 -5
  93. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q5_1.cu +0 -5
  94. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q8_0.cu +0 -5
  95. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-f16.cu +0 -5
  96. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q4_0.cu +0 -5
  97. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q4_1.cu +0 -5
  98. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q5_0.cu +0 -5
  99. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q5_1.cu +0 -5
  100. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q8_0.cu +0 -5
  101. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-f16.cu +0 -5
  102. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q4_0.cu +0 -5
  103. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q4_1.cu +0 -5
  104. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q5_0.cu +0 -5
  105. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q5_1.cu +0 -5
  106. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q8_0.cu +0 -5
  107. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-f16.cu +0 -5
  108. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q4_0.cu +0 -5
  109. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q4_1.cu +0 -5
  110. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q5_0.cu +0 -5
  111. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q5_1.cu +0 -5
  112. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q8_0.cu +0 -5
  113. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-f16.cu +0 -5
  114. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q4_0.cu +0 -5
  115. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q4_1.cu +0 -5
  116. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q5_0.cu +0 -5
  117. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q5_1.cu +0 -5
  118. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q8_0.cu +0 -5
  119. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-f16.cu +0 -5
  120. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q4_0.cu +0 -5
  121. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q4_1.cu +0 -5
  122. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q5_0.cu +0 -5
  123. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q5_1.cu +0 -5
  124. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q8_0.cu +0 -5
  125. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs256-f16-f16.cu +0 -5
  126. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-f16.cu +0 -5
  127. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q4_0.cu +0 -5
  128. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q4_1.cu +0 -5
  129. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q5_0.cu +0 -5
  130. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q5_1.cu +0 -5
  131. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q8_0.cu +0 -5
  132. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqfloat-cpb16.cu +0 -10
  133. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqfloat-cpb32.cu +0 -9
  134. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqhalf-cpb16.cu +0 -10
  135. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqhalf-cpb32.cu +0 -10
  136. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqhalf-cpb8.cu +0 -8
  137. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q2_k.cu +0 -5
  138. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q3_k.cu +0 -5
  139. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q4_0.cu +0 -5
  140. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q4_1.cu +0 -5
  141. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q4_k.cu +0 -5
  142. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q5_0.cu +0 -5
  143. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q5_1.cu +0 -5
  144. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q5_k.cu +0 -5
  145. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q6_k.cu +0 -5
  146. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q8_0.cu +0 -5
  147. data/vendor/tmp/llama.cpp/ggml-cuda/tsembd.cu +0 -47
  148. data/vendor/tmp/llama.cpp/ggml-cuda/unary.cu +0 -314
  149. data/vendor/tmp/llama.cpp/ggml-cuda/upscale.cu +0 -51
  150. data/vendor/tmp/llama.cpp/ggml-cuda.cu +0 -3069
  151. data/vendor/tmp/llama.cpp/ggml-cuda.h +0 -44
  152. data/vendor/tmp/llama.cpp/ggml-impl.h +0 -651
  153. data/vendor/tmp/llama.cpp/ggml-kompute.cpp +0 -2038
  154. data/vendor/tmp/llama.cpp/ggml-kompute.h +0 -46
  155. data/vendor/tmp/llama.cpp/ggml-metal.h +0 -66
  156. data/vendor/tmp/llama.cpp/ggml-metal.m +0 -3273
  157. data/vendor/tmp/llama.cpp/ggml-metal.metal +0 -6540
  158. data/vendor/tmp/llama.cpp/ggml-quants.c +0 -14994
  159. data/vendor/tmp/llama.cpp/ggml-quants.h +0 -133
  160. data/vendor/tmp/llama.cpp/ggml-rpc.cpp +0 -1178
  161. data/vendor/tmp/llama.cpp/ggml-rpc.h +0 -24
  162. data/vendor/tmp/llama.cpp/ggml-sycl.cpp +0 -6351
  163. data/vendor/tmp/llama.cpp/ggml-sycl.h +0 -40
  164. data/vendor/tmp/llama.cpp/ggml-vulkan-shaders.hpp +0 -144508
  165. data/vendor/tmp/llama.cpp/ggml-vulkan.cpp +0 -7183
  166. data/vendor/tmp/llama.cpp/ggml-vulkan.h +0 -29
  167. data/vendor/tmp/llama.cpp/ggml.c +0 -22506
  168. data/vendor/tmp/llama.cpp/ggml.h +0 -2458
  169. data/vendor/tmp/llama.cpp/llama.cpp +0 -18985
  170. data/vendor/tmp/llama.cpp/llama.h +0 -1147
  171. data/vendor/tmp/llama.cpp/scripts/get-flags.mk +0 -38
  172. data/vendor/tmp/llama.cpp/sgemm.cpp +0 -1032
  173. data/vendor/tmp/llama.cpp/sgemm.h +0 -14
  174. data/vendor/tmp/llama.cpp/unicode-data.cpp +0 -7033
  175. data/vendor/tmp/llama.cpp/unicode-data.h +0 -20
  176. data/vendor/tmp/llama.cpp/unicode.cpp +0 -810
  177. data/vendor/tmp/llama.cpp/unicode.h +0 -63
data/vendor/tmp/llama.cpp/ggml-metal.m DELETED
@@ -1,3273 +0,0 @@
1
- #import "ggml-metal.h"
2
-
3
- #import "ggml-backend-impl.h"
4
- #import "ggml.h"
5
-
6
- #import <Foundation/Foundation.h>
7
-
8
- #import <Metal/Metal.h>
9
-
10
- #undef MIN
11
- #undef MAX
12
- #define MIN(a, b) ((a) < (b) ? (a) : (b))
13
- #define MAX(a, b) ((a) > (b) ? (a) : (b))
14
-
15
- #ifdef GGML_METAL_NDEBUG
16
- #define GGML_METAL_LOG_INFO(...)
17
- #define GGML_METAL_LOG_WARN(...)
18
- #define GGML_METAL_LOG_ERROR(...)
19
- #else
20
- #define GGML_METAL_LOG_INFO(...) ggml_metal_log(GGML_LOG_LEVEL_INFO, __VA_ARGS__)
21
- #define GGML_METAL_LOG_WARN(...) ggml_metal_log(GGML_LOG_LEVEL_WARN, __VA_ARGS__)
22
- #define GGML_METAL_LOG_ERROR(...) ggml_metal_log(GGML_LOG_LEVEL_ERROR, __VA_ARGS__)
23
- #endif
24
-
25
- #define UNUSED(x) (void)(x)
26
-
27
- struct ggml_metal_kernel {
28
- id<MTLComputePipelineState> pipeline;
29
- };
30
-
31
- enum ggml_metal_kernel_type {
32
- GGML_METAL_KERNEL_TYPE_ADD,
33
- GGML_METAL_KERNEL_TYPE_ADD_ROW,
34
- GGML_METAL_KERNEL_TYPE_MUL,
35
- GGML_METAL_KERNEL_TYPE_MUL_ROW,
36
- GGML_METAL_KERNEL_TYPE_DIV,
37
- GGML_METAL_KERNEL_TYPE_DIV_ROW,
38
- GGML_METAL_KERNEL_TYPE_REPEAT_F32,
39
- GGML_METAL_KERNEL_TYPE_REPEAT_F16,
40
- GGML_METAL_KERNEL_TYPE_REPEAT_I32,
41
- GGML_METAL_KERNEL_TYPE_REPEAT_I16,
42
- GGML_METAL_KERNEL_TYPE_SCALE,
43
- GGML_METAL_KERNEL_TYPE_SCALE_4,
44
- GGML_METAL_KERNEL_TYPE_CLAMP,
45
- GGML_METAL_KERNEL_TYPE_TANH,
46
- GGML_METAL_KERNEL_TYPE_RELU,
47
- GGML_METAL_KERNEL_TYPE_SIGMOID,
48
- GGML_METAL_KERNEL_TYPE_GELU,
49
- GGML_METAL_KERNEL_TYPE_GELU_4,
50
- GGML_METAL_KERNEL_TYPE_GELU_QUICK,
51
- GGML_METAL_KERNEL_TYPE_GELU_QUICK_4,
52
- GGML_METAL_KERNEL_TYPE_SILU,
53
- GGML_METAL_KERNEL_TYPE_SILU_4,
54
- GGML_METAL_KERNEL_TYPE_SOFT_MAX_F16,
55
- GGML_METAL_KERNEL_TYPE_SOFT_MAX_F16_4,
56
- GGML_METAL_KERNEL_TYPE_SOFT_MAX_F32,
57
- GGML_METAL_KERNEL_TYPE_SOFT_MAX_F32_4,
58
- GGML_METAL_KERNEL_TYPE_DIAG_MASK_INF,
59
- GGML_METAL_KERNEL_TYPE_DIAG_MASK_INF_8,
60
- GGML_METAL_KERNEL_TYPE_GET_ROWS_F32,
61
- GGML_METAL_KERNEL_TYPE_GET_ROWS_F16,
62
- GGML_METAL_KERNEL_TYPE_GET_ROWS_Q4_0,
63
- GGML_METAL_KERNEL_TYPE_GET_ROWS_Q4_1,
64
- GGML_METAL_KERNEL_TYPE_GET_ROWS_Q5_0,
65
- GGML_METAL_KERNEL_TYPE_GET_ROWS_Q5_1,
66
- GGML_METAL_KERNEL_TYPE_GET_ROWS_Q8_0,
67
- GGML_METAL_KERNEL_TYPE_GET_ROWS_Q2_K,
68
- GGML_METAL_KERNEL_TYPE_GET_ROWS_Q3_K,
69
- GGML_METAL_KERNEL_TYPE_GET_ROWS_Q4_K,
70
- GGML_METAL_KERNEL_TYPE_GET_ROWS_Q5_K,
71
- GGML_METAL_KERNEL_TYPE_GET_ROWS_Q6_K,
72
- GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_XXS,
73
- GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_XS,
74
- GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ3_XXS,
75
- GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ3_S,
76
- GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_S,
77
- GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ1_S,
78
- GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ1_M,
79
- GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ4_NL,
80
- GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ4_XS,
81
- GGML_METAL_KERNEL_TYPE_GET_ROWS_I32,
82
- GGML_METAL_KERNEL_TYPE_RMS_NORM,
83
- GGML_METAL_KERNEL_TYPE_GROUP_NORM,
84
- GGML_METAL_KERNEL_TYPE_NORM,
85
- GGML_METAL_KERNEL_TYPE_MUL_MV_F32_F32,
86
- GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F16,
87
- GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F32,
88
- GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F32_1ROW,
89
- GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F32_L4,
90
- GGML_METAL_KERNEL_TYPE_MUL_MV_Q4_0_F32,
91
- GGML_METAL_KERNEL_TYPE_MUL_MV_Q4_1_F32,
92
- GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_0_F32,
93
- GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_1_F32,
94
- GGML_METAL_KERNEL_TYPE_MUL_MV_Q8_0_F32,
95
- GGML_METAL_KERNEL_TYPE_MUL_MV_Q2_K_F32,
96
- GGML_METAL_KERNEL_TYPE_MUL_MV_Q3_K_F32,
97
- GGML_METAL_KERNEL_TYPE_MUL_MV_Q4_K_F32,
98
- GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_K_F32,
99
- GGML_METAL_KERNEL_TYPE_MUL_MV_Q6_K_F32,
100
- GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_XXS_F32,
101
- GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_XS_F32,
102
- GGML_METAL_KERNEL_TYPE_MUL_MV_IQ3_XXS_F32,
103
- GGML_METAL_KERNEL_TYPE_MUL_MV_IQ3_S_F32,
104
- GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_S_F32,
105
- GGML_METAL_KERNEL_TYPE_MUL_MV_IQ1_S_F32,
106
- GGML_METAL_KERNEL_TYPE_MUL_MV_IQ1_M_F32,
107
- GGML_METAL_KERNEL_TYPE_MUL_MV_IQ4_NL_F32,
108
- GGML_METAL_KERNEL_TYPE_MUL_MV_IQ4_XS_F32,
109
- GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F32_F32,
110
- //GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F16,
111
- GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F32,
112
- //GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F32_1ROW,
113
- //GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F32_L4,
114
- GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q4_0_F32,
115
- GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q4_1_F32,
116
- GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q5_0_F32,
117
- GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q5_1_F32,
118
- GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q8_0_F32,
119
- GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q2_K_F32,
120
- GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q3_K_F32,
121
- GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q4_K_F32,
122
- GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q5_K_F32,
123
- GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q6_K_F32,
124
- GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_XXS_F32,
125
- GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_XS_F32,
126
- GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ3_XXS_F32,
127
- GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ3_S_F32,
128
- GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_S_F32,
129
- GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ1_S_F32,
130
- GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ1_M_F32,
131
- GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ4_NL_F32,
132
- GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ4_XS_F32,
133
- GGML_METAL_KERNEL_TYPE_MUL_MM_F32_F32,
134
- GGML_METAL_KERNEL_TYPE_MUL_MM_F16_F32,
135
- GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_0_F32,
136
- GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_1_F32,
137
- GGML_METAL_KERNEL_TYPE_MUL_MM_Q5_0_F32,
138
- GGML_METAL_KERNEL_TYPE_MUL_MM_Q5_1_F32,
139
- GGML_METAL_KERNEL_TYPE_MUL_MM_Q8_0_F32,
140
- GGML_METAL_KERNEL_TYPE_MUL_MM_Q2_K_F32,
141
- GGML_METAL_KERNEL_TYPE_MUL_MM_Q3_K_F32,
142
- GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_K_F32,
143
- GGML_METAL_KERNEL_TYPE_MUL_MM_Q5_K_F32,
144
- GGML_METAL_KERNEL_TYPE_MUL_MM_Q6_K_F32,
145
- GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_XXS_F32,
146
- GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_XS_F32,
147
- GGML_METAL_KERNEL_TYPE_MUL_MM_IQ3_XXS_F32,
148
- GGML_METAL_KERNEL_TYPE_MUL_MM_IQ3_S_F32,
149
- GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_S_F32,
150
- GGML_METAL_KERNEL_TYPE_MUL_MM_IQ1_S_F32,
151
- GGML_METAL_KERNEL_TYPE_MUL_MM_IQ1_M_F32,
152
- GGML_METAL_KERNEL_TYPE_MUL_MM_IQ4_NL_F32,
153
- GGML_METAL_KERNEL_TYPE_MUL_MM_IQ4_XS_F32,
154
- GGML_METAL_KERNEL_TYPE_MUL_MM_ID_F32_F32,
155
- GGML_METAL_KERNEL_TYPE_MUL_MM_ID_F16_F32,
156
- GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_0_F32,
157
- GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_1_F32,
158
- GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q5_0_F32,
159
- GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q5_1_F32,
160
- GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q8_0_F32,
161
- GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q2_K_F32,
162
- GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q3_K_F32,
163
- GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_K_F32,
164
- GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q5_K_F32,
165
- GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q6_K_F32,
166
- GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_XXS_F32,
167
- GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_XS_F32,
168
- GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ3_XXS_F32,
169
- GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ3_S_F32,
170
- GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_S_F32,
171
- GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ1_S_F32,
172
- GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ1_M_F32,
173
- GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ4_NL_F32,
174
- GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ4_XS_F32,
175
- GGML_METAL_KERNEL_TYPE_ROPE_NORM_F32,
176
- GGML_METAL_KERNEL_TYPE_ROPE_NORM_F16,
177
- GGML_METAL_KERNEL_TYPE_ROPE_NEOX_F32,
178
- GGML_METAL_KERNEL_TYPE_ROPE_NEOX_F16,
179
- GGML_METAL_KERNEL_TYPE_IM2COL_F16,
180
- GGML_METAL_KERNEL_TYPE_IM2COL_F32,
181
- GGML_METAL_KERNEL_TYPE_UPSCALE_F32,
182
- GGML_METAL_KERNEL_TYPE_PAD_F32,
183
- GGML_METAL_KERNEL_TYPE_ARANGE_F32,
184
- GGML_METAL_KERNEL_TYPE_TIMESTEP_EMBEDDING_F32,
185
- GGML_METAL_KERNEL_TYPE_ARGSORT_F32_I32_ASC,
186
- GGML_METAL_KERNEL_TYPE_ARGSORT_F32_I32_DESC,
187
- GGML_METAL_KERNEL_TYPE_LEAKY_RELU_F32,
188
- GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H64,
189
- GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H80,
190
- GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H96,
191
- GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H112,
192
- GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H128,
193
- //GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H256, // https://github.com/ggerganov/llama.cpp/issues/7261
194
- GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_VEC_F16_H128,
195
- //GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_VEC_F16_H256, // https://github.com/ggerganov/llama.cpp/issues/7261
196
- GGML_METAL_KERNEL_TYPE_CPY_F32_F16,
197
- GGML_METAL_KERNEL_TYPE_CPY_F32_F32,
198
- GGML_METAL_KERNEL_TYPE_CPY_F32_Q8_0,
199
- GGML_METAL_KERNEL_TYPE_CPY_F32_Q4_0,
200
- GGML_METAL_KERNEL_TYPE_CPY_F32_Q4_1,
201
- GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_0,
202
- GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_1,
203
- GGML_METAL_KERNEL_TYPE_CPY_F32_IQ4_NL,
204
- GGML_METAL_KERNEL_TYPE_CPY_F16_F16,
205
- GGML_METAL_KERNEL_TYPE_CPY_F16_F32,
206
- GGML_METAL_KERNEL_TYPE_CONCAT,
207
- GGML_METAL_KERNEL_TYPE_SQR,
208
- GGML_METAL_KERNEL_TYPE_SUM_ROWS,
209
-
210
- GGML_METAL_KERNEL_TYPE_COUNT
211
- };
212
-
213
- struct ggml_metal_context {
214
- int n_cb;
215
-
216
- id<MTLDevice> device;
217
- id<MTLCommandQueue> queue;
218
-
219
- dispatch_queue_t d_queue;
220
-
221
- struct ggml_metal_kernel kernels[GGML_METAL_KERNEL_TYPE_COUNT];
222
-
223
- bool support_simdgroup_reduction;
224
- bool support_simdgroup_mm;
225
-
226
- bool should_capture_next_compute;
227
- };
228
-
229
- // MSL code
230
- // TODO: move the contents here when ready
231
- // for now it is easier to work in a separate file
232
- // static NSString * const msl_library_source = @"see metal.metal";
233
-
234
- // Here to assist with NSBundle Path Hack
235
- @interface GGMLMetalClass : NSObject
236
- @end
237
- @implementation GGMLMetalClass
238
- @end
239
-
240
- static void ggml_metal_default_log_callback(enum ggml_log_level level, const char * msg, void * user_data) {
241
- fprintf(stderr, "%s", msg);
242
-
243
- UNUSED(level);
244
- UNUSED(user_data);
245
- }
246
-
247
- ggml_log_callback ggml_metal_log_callback = ggml_metal_default_log_callback;
248
- void * ggml_metal_log_user_data = NULL;
249
-
250
- GGML_ATTRIBUTE_FORMAT(2, 3)
251
- static void ggml_metal_log(enum ggml_log_level level, const char * format, ...){
252
- if (ggml_metal_log_callback != NULL) {
253
- va_list args;
254
- va_start(args, format);
255
- char buffer[128];
256
- int len = vsnprintf(buffer, 128, format, args);
257
- if (len < 128) {
258
- ggml_metal_log_callback(level, buffer, ggml_metal_log_user_data);
259
- } else {
260
- char* buffer2 = malloc(len+1);
261
- va_end(args);
262
- va_start(args, format);
263
- vsnprintf(buffer2, len+1, format, args);
264
- buffer2[len] = 0;
265
- ggml_metal_log_callback(level, buffer2, ggml_metal_log_user_data);
266
- free(buffer2);
267
- }
268
- va_end(args);
269
- }
270
- }
271
-
272
- static void * ggml_metal_host_malloc(size_t n) {
273
- void * data = NULL;
274
-
275
- #if TARGET_OS_OSX
276
- kern_return_t err = vm_allocate((vm_map_t) mach_task_self(), (void *) &data, n, VM_FLAGS_ANYWHERE);
277
- if (err != KERN_SUCCESS) {
278
- GGML_METAL_LOG_ERROR("%s: error: vm_allocate failed\n", __func__);
279
- return NULL;
280
- }
281
- #else
282
- const int result = posix_memalign((void **) &data, sysconf(_SC_PAGESIZE), n);
283
- if (result != 0) {
284
- GGML_METAL_LOG_ERROR("%s: error: posix_memalign failed\n", __func__);
285
- return NULL;
286
- }
287
- #endif
288
-
289
- return data;
290
- }
291
-
292
- static struct ggml_metal_context * ggml_metal_init(int n_cb) {
293
- GGML_METAL_LOG_INFO("%s: allocating\n", __func__);
294
-
295
- #if TARGET_OS_OSX && !GGML_METAL_NDEBUG
296
- // Show all the Metal device instances in the system
297
- NSArray * devices = MTLCopyAllDevices();
298
- for (id<MTLDevice> device in devices) {
299
- GGML_METAL_LOG_INFO("%s: found device: %s\n", __func__, [[device name] UTF8String]);
300
- }
301
- [devices release]; // since it was created by a *Copy* C method
302
- #endif
303
-
304
- // Pick and show default Metal device
305
- id<MTLDevice> device = MTLCreateSystemDefaultDevice();
306
- GGML_METAL_LOG_INFO("%s: picking default device: %s\n", __func__, [[device name] UTF8String]);
307
-
308
- // Configure context
309
- struct ggml_metal_context * ctx = malloc(sizeof(struct ggml_metal_context));
310
- ctx->device = device;
311
- ctx->n_cb = MIN(n_cb, GGML_METAL_MAX_BUFFERS);
312
- ctx->queue = [ctx->device newCommandQueue];
313
- ctx->d_queue = dispatch_queue_create("ggml-metal", DISPATCH_QUEUE_CONCURRENT);
314
-
315
- id<MTLLibrary> metal_library;
316
-
317
- // load library
318
- //
319
- // - first check if the library is embedded
320
- // - then check if the library is in the bundle
321
- // - if not found, load the source and compile it
322
- // - if that fails, return NULL
323
- {
324
- NSBundle * bundle = nil;
325
- #ifdef SWIFT_PACKAGE
326
- bundle = SWIFTPM_MODULE_BUNDLE;
327
- #else
328
- bundle = [NSBundle bundleForClass:[GGMLMetalClass class]];
329
- #endif
330
-
331
- NSError * error = nil;
332
-
333
- #if GGML_METAL_EMBED_LIBRARY
334
- const bool try_metallib = false;
335
- #else
336
- const bool try_metallib = true;
337
- #endif
338
-
339
- NSString * path_lib = [bundle pathForResource:@"default" ofType:@"metallib"];
340
- if (try_metallib && path_lib != nil) {
341
- // pre-compiled library found
342
- NSURL * libURL = [NSURL fileURLWithPath:path_lib];
343
- GGML_METAL_LOG_INFO("%s: loading '%s'\n", __func__, [path_lib UTF8String]);
344
-
345
- metal_library = [ctx->device newLibraryWithURL:libURL error:&error];
346
- if (error) {
347
- GGML_METAL_LOG_ERROR("%s: error: %s\n", __func__, [[error description] UTF8String]);
348
- return NULL;
349
- }
350
- } else {
351
- #if GGML_METAL_EMBED_LIBRARY
352
- GGML_METAL_LOG_INFO("%s: using embedded metal library\n", __func__);
353
-
354
- extern const char ggml_metallib_start[];
355
- extern const char ggml_metallib_end[];
356
-
357
- NSString * src = [[NSString alloc] initWithBytes:ggml_metallib_start length:(ggml_metallib_end-ggml_metallib_start) encoding:NSUTF8StringEncoding];
358
- #else
359
- GGML_METAL_LOG_INFO("%s: default.metallib not found, loading from source\n", __func__);
360
-
361
- NSString * path_source;
362
- NSString * path_resource = [[NSProcessInfo processInfo].environment objectForKey:@"GGML_METAL_PATH_RESOURCES"];
363
-
364
- GGML_METAL_LOG_INFO("%s: GGML_METAL_PATH_RESOURCES = %s\n", __func__, path_resource ? [path_resource UTF8String] : "nil");
365
-
366
- if (path_resource) {
367
- path_source = [path_resource stringByAppendingPathComponent:@"ggml-metal.metal"];
368
- } else {
369
- path_source = [bundle pathForResource:@"ggml-metal" ofType:@"metal"];
370
- }
371
-
372
- if (path_source == nil) {
373
- GGML_METAL_LOG_WARN("%s: error: could not use bundle path to find ggml-metal.metal, falling back to trying cwd\n", __func__);
374
- path_source = @"ggml-metal.metal";
375
- }
376
-
377
- GGML_METAL_LOG_INFO("%s: loading '%s'\n", __func__, [path_source UTF8String]);
378
-
379
- NSString * src = [NSString stringWithContentsOfFile:path_source encoding:NSUTF8StringEncoding error:&error];
380
- if (error) {
381
- GGML_METAL_LOG_ERROR("%s: error: %s\n", __func__, [[error description] UTF8String]);
382
- return NULL;
383
- }
384
- #endif // GGML_METAL_EMBED_LIBRARY
385
-
386
- @autoreleasepool {
387
- // dictionary of preprocessor macros
388
- NSMutableDictionary * prep = [NSMutableDictionary dictionary];
389
-
390
- MTLCompileOptions* options = [MTLCompileOptions new];
391
- options.preprocessorMacros = prep;
392
-
393
- //[options setFastMathEnabled:false];
394
-
395
- metal_library = [ctx->device newLibraryWithSource:src options:options error:&error];
396
- if (error) {
397
- GGML_METAL_LOG_ERROR("%s: error: %s\n", __func__, [[error description] UTF8String]);
398
- return NULL;
399
- }
400
- }
401
- }
402
- }
403
-
404
- // print MTL GPU family:
405
- GGML_METAL_LOG_INFO("%s: GPU name: %s\n", __func__, [[ctx->device name] UTF8String]);
406
-
407
- const NSInteger MTLGPUFamilyMetal3 = 5001;
408
-
409
- // determine max supported GPU family
410
- // https://developer.apple.com/metal/Metal-Shading-Language-Specification.pdf
411
- // https://developer.apple.com/metal/Metal-Feature-Set-Tables.pdf
412
- {
413
- for (int i = MTLGPUFamilyApple1 + 20; i >= MTLGPUFamilyApple1; --i) {
414
- if ([ctx->device supportsFamily:i]) {
415
- GGML_METAL_LOG_INFO("%s: GPU family: MTLGPUFamilyApple%d (%d)\n", __func__, i - (int) MTLGPUFamilyApple1 + 1, i);
416
- break;
417
- }
418
- }
419
-
420
- for (int i = MTLGPUFamilyCommon1 + 5; i >= MTLGPUFamilyCommon1; --i) {
421
- if ([ctx->device supportsFamily:i]) {
422
- GGML_METAL_LOG_INFO("%s: GPU family: MTLGPUFamilyCommon%d (%d)\n", __func__, i - (int) MTLGPUFamilyCommon1 + 1, i);
423
- break;
424
- }
425
- }
426
-
427
- for (int i = MTLGPUFamilyMetal3 + 5; i >= MTLGPUFamilyMetal3; --i) {
428
- if ([ctx->device supportsFamily:i]) {
429
- GGML_METAL_LOG_INFO("%s: GPU family: MTLGPUFamilyMetal%d (%d)\n", __func__, i - (int) MTLGPUFamilyMetal3 + 3, i);
430
- break;
431
- }
432
- }
433
- }
434
-
435
- ctx->support_simdgroup_reduction = [ctx->device supportsFamily:MTLGPUFamilyApple7];
436
- ctx->support_simdgroup_reduction |= [ctx->device supportsFamily:MTLGPUFamilyMetal3];
437
-
438
- ctx->support_simdgroup_mm = [ctx->device supportsFamily:MTLGPUFamilyApple7];
439
-
440
- GGML_METAL_LOG_INFO("%s: simdgroup reduction support = %s\n", __func__, ctx->support_simdgroup_reduction ? "true" : "false");
441
- GGML_METAL_LOG_INFO("%s: simdgroup matrix mul. support = %s\n", __func__, ctx->support_simdgroup_mm ? "true" : "false");
442
- GGML_METAL_LOG_INFO("%s: hasUnifiedMemory = %s\n", __func__, ctx->device.hasUnifiedMemory ? "true" : "false");
443
-
444
- ctx->should_capture_next_compute = false;
445
-
446
- #if TARGET_OS_OSX || (TARGET_OS_IOS && __clang_major__ >= 15)
447
- if (@available(macOS 10.12, iOS 16.0, *)) {
448
- GGML_METAL_LOG_INFO("%s: recommendedMaxWorkingSetSize = %8.2f MB\n", __func__, ctx->device.recommendedMaxWorkingSetSize / 1e6);
449
- }
450
- #elif TARGET_OS_OSX
451
- if (ctx->device.maxTransferRate != 0) {
452
- GGML_METAL_LOG_INFO("%s: maxTransferRate = %8.2f MB/s\n", __func__, ctx->device.maxTransferRate / 1e6);
453
- } else {
454
- GGML_METAL_LOG_INFO("%s: maxTransferRate = built-in GPU\n", __func__);
455
- }
456
- #endif
457
-
458
- // load kernels
459
- {
460
- NSError * error = nil;
461
-
462
- for (int i = 0; i < GGML_METAL_KERNEL_TYPE_COUNT; ++i) {
463
- ctx->kernels[i].pipeline = nil;
464
- }
465
-
466
- /*
467
- GGML_METAL_LOG_INFO("%s: loaded %-40s %16p | th_max = %4d | th_width = %4d\n", __func__, "kernel_"#name, (void *) kernel->pipeline, \
468
- (int) kernel->pipeline.maxTotalThreadsPerThreadgroup, \
469
- (int) kernel->pipeline.threadExecutionWidth); \
470
- */
471
- #define GGML_METAL_ADD_KERNEL(e, name, supported) \
472
- if (supported) { \
473
- struct ggml_metal_kernel * kernel = &ctx->kernels[e]; \
474
- id<MTLFunction> metal_function = [metal_library newFunctionWithName:@"kernel_"#name]; \
475
- kernel->pipeline = [ctx->device newComputePipelineStateWithFunction:metal_function error:&error]; \
476
- [metal_function release]; \
477
- if (error) { \
478
- GGML_METAL_LOG_ERROR("%s: error: load pipeline error: %s\n", __func__, [[error description] UTF8String]); \
479
- [metal_library release]; \
480
- return NULL; \
481
- } \
482
- } else { \
483
- GGML_METAL_LOG_WARN("%s: skipping %-40s (not supported)\n", __func__, "kernel_"#name); \
484
- }
485
-
486
- // simd_sum and simd_max requires MTLGPUFamilyApple7
487
-
488
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ADD, add, true);
489
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ADD_ROW, add_row, true);
490
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL, mul, true);
491
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_ROW, mul_row, true);
492
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_DIV, div, true);
493
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_DIV_ROW, div_row, true);
494
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_REPEAT_F32, repeat_f32, true);
495
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_REPEAT_F16, repeat_f16, true);
496
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_REPEAT_I32, repeat_i32, true);
497
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_REPEAT_I16, repeat_i16, true);
498
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SCALE, scale, true);
499
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SCALE_4, scale_4, true);
500
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CLAMP, clamp, true);
501
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_TANH, tanh, true);
502
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_RELU, relu, true);
503
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SIGMOID, sigmoid, true);
504
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GELU, gelu, true);
505
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GELU_4, gelu_4, true);
506
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GELU_QUICK, gelu_quick, true);
507
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GELU_QUICK_4, gelu_quick_4, true);
508
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SILU, silu, true);
509
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SILU_4, silu_4, true);
510
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SOFT_MAX_F16, soft_max_f16, ctx->support_simdgroup_reduction);
511
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SOFT_MAX_F16_4, soft_max_f16_4, ctx->support_simdgroup_reduction);
512
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SOFT_MAX_F32, soft_max_f32, ctx->support_simdgroup_reduction);
513
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SOFT_MAX_F32_4, soft_max_f32_4, ctx->support_simdgroup_reduction);
514
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_DIAG_MASK_INF, diag_mask_inf, true);
515
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_DIAG_MASK_INF_8, diag_mask_inf_8, true);
516
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_F32, get_rows_f32, true);
517
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_F16, get_rows_f16, true);
518
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q4_0, get_rows_q4_0, true);
519
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q4_1, get_rows_q4_1, true);
520
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q5_0, get_rows_q5_0, true);
521
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q5_1, get_rows_q5_1, true);
522
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q8_0, get_rows_q8_0, true);
523
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q2_K, get_rows_q2_K, true);
524
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q3_K, get_rows_q3_K, true);
525
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q4_K, get_rows_q4_K, true);
526
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q5_K, get_rows_q5_K, true);
527
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_Q6_K, get_rows_q6_K, true);
528
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_XXS, get_rows_iq2_xxs, true);
529
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_XS, get_rows_iq2_xs, true);
530
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ3_XXS, get_rows_iq3_xxs, true);
531
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ3_S, get_rows_iq3_s, true);
532
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_S, get_rows_iq2_s, true);
533
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ1_S, get_rows_iq1_s, true);
534
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ1_M, get_rows_iq1_m, true);
535
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ4_NL, get_rows_iq4_nl, true);
536
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ4_XS, get_rows_iq4_xs, true);
537
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GET_ROWS_I32, get_rows_i32, true);
538
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_RMS_NORM, rms_norm, ctx->support_simdgroup_reduction);
539
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_GROUP_NORM, group_norm, ctx->support_simdgroup_reduction);
540
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_NORM, norm, true);
541
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_F32_F32, mul_mv_f32_f32, ctx->support_simdgroup_reduction);
542
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F16, mul_mv_f16_f16, ctx->support_simdgroup_reduction);
543
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F32, mul_mv_f16_f32, ctx->support_simdgroup_reduction);
544
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F32_1ROW, mul_mv_f16_f32_1row, ctx->support_simdgroup_reduction);
545
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F32_L4, mul_mv_f16_f32_l4, ctx->support_simdgroup_reduction);
546
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q4_0_F32, mul_mv_q4_0_f32, ctx->support_simdgroup_reduction);
547
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q4_1_F32, mul_mv_q4_1_f32, ctx->support_simdgroup_reduction);
548
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_0_F32, mul_mv_q5_0_f32, ctx->support_simdgroup_reduction);
549
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_1_F32, mul_mv_q5_1_f32, ctx->support_simdgroup_reduction);
550
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q8_0_F32, mul_mv_q8_0_f32, ctx->support_simdgroup_reduction);
551
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q2_K_F32, mul_mv_q2_K_f32, ctx->support_simdgroup_reduction);
552
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q3_K_F32, mul_mv_q3_K_f32, ctx->support_simdgroup_reduction);
553
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q4_K_F32, mul_mv_q4_K_f32, ctx->support_simdgroup_reduction);
554
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_K_F32, mul_mv_q5_K_f32, ctx->support_simdgroup_reduction);
555
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q6_K_F32, mul_mv_q6_K_f32, ctx->support_simdgroup_reduction);
556
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_XXS_F32, mul_mv_iq2_xxs_f32, ctx->support_simdgroup_reduction);
557
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_XS_F32, mul_mv_iq2_xs_f32, ctx->support_simdgroup_reduction);
558
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ3_XXS_F32, mul_mv_iq3_xxs_f32, ctx->support_simdgroup_reduction);
559
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ3_S_F32, mul_mv_iq3_s_f32, ctx->support_simdgroup_reduction);
560
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_S_F32, mul_mv_iq2_s_f32, ctx->support_simdgroup_reduction);
561
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ1_S_F32, mul_mv_iq1_s_f32, ctx->support_simdgroup_reduction);
562
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ1_M_F32, mul_mv_iq1_m_f32, ctx->support_simdgroup_reduction);
563
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ4_NL_F32, mul_mv_iq4_nl_f32, ctx->support_simdgroup_reduction);
564
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_IQ4_XS_F32, mul_mv_iq4_xs_f32, ctx->support_simdgroup_reduction);
565
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F32_F32, mul_mv_id_f32_f32, ctx->support_simdgroup_reduction);
566
- //GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F16, mul_mv_id_f16_f16, ctx->support_simdgroup_reduction);
567
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F32, mul_mv_id_f16_f32, ctx->support_simdgroup_reduction);
568
- //GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F32_1ROW, mul_mv_id_f16_f32_1row, ctx->support_simdgroup_reduction);
569
- //GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F32_L4, mul_mv_id_f16_f32_l4, ctx->support_simdgroup_reduction);
570
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q4_0_F32, mul_mv_id_q4_0_f32, ctx->support_simdgroup_reduction);
571
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q4_1_F32, mul_mv_id_q4_1_f32, ctx->support_simdgroup_reduction);
572
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q5_0_F32, mul_mv_id_q5_0_f32, ctx->support_simdgroup_reduction);
573
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q5_1_F32, mul_mv_id_q5_1_f32, ctx->support_simdgroup_reduction);
574
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q8_0_F32, mul_mv_id_q8_0_f32, ctx->support_simdgroup_reduction);
575
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q2_K_F32, mul_mv_id_q2_K_f32, ctx->support_simdgroup_reduction);
576
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q3_K_F32, mul_mv_id_q3_K_f32, ctx->support_simdgroup_reduction);
577
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q4_K_F32, mul_mv_id_q4_K_f32, ctx->support_simdgroup_reduction);
578
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q5_K_F32, mul_mv_id_q5_K_f32, ctx->support_simdgroup_reduction);
579
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q6_K_F32, mul_mv_id_q6_K_f32, ctx->support_simdgroup_reduction);
580
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_XXS_F32, mul_mv_id_iq2_xxs_f32, ctx->support_simdgroup_reduction);
581
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_XS_F32, mul_mv_id_iq2_xs_f32, ctx->support_simdgroup_reduction);
582
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ3_XXS_F32, mul_mv_id_iq3_xxs_f32, ctx->support_simdgroup_reduction);
583
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ3_S_F32, mul_mv_id_iq3_s_f32, ctx->support_simdgroup_reduction);
584
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_S_F32, mul_mv_id_iq2_s_f32, ctx->support_simdgroup_reduction);
585
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ1_S_F32, mul_mv_id_iq1_s_f32, ctx->support_simdgroup_reduction);
586
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ1_M_F32, mul_mv_id_iq1_m_f32, ctx->support_simdgroup_reduction);
587
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ4_NL_F32, mul_mv_id_iq4_nl_f32, ctx->support_simdgroup_reduction);
588
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ4_XS_F32, mul_mv_id_iq4_xs_f32, ctx->support_simdgroup_reduction);
589
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_F32_F32, mul_mm_f32_f32, ctx->support_simdgroup_mm);
590
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_F16_F32, mul_mm_f16_f32, ctx->support_simdgroup_mm);
591
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_0_F32, mul_mm_q4_0_f32, ctx->support_simdgroup_mm);
592
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_1_F32, mul_mm_q4_1_f32, ctx->support_simdgroup_mm);
593
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q5_0_F32, mul_mm_q5_0_f32, ctx->support_simdgroup_mm);
594
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q5_1_F32, mul_mm_q5_1_f32, ctx->support_simdgroup_mm);
595
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q8_0_F32, mul_mm_q8_0_f32, ctx->support_simdgroup_mm);
596
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q2_K_F32, mul_mm_q2_K_f32, ctx->support_simdgroup_mm);
597
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q3_K_F32, mul_mm_q3_K_f32, ctx->support_simdgroup_mm);
598
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_K_F32, mul_mm_q4_K_f32, ctx->support_simdgroup_mm);
599
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q5_K_F32, mul_mm_q5_K_f32, ctx->support_simdgroup_mm);
600
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_Q6_K_F32, mul_mm_q6_K_f32, ctx->support_simdgroup_mm);
601
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_XXS_F32, mul_mm_iq2_xxs_f32, ctx->support_simdgroup_mm);
602
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_XS_F32, mul_mm_iq2_xs_f32, ctx->support_simdgroup_mm);
603
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ3_XXS_F32, mul_mm_iq3_xxs_f32, ctx->support_simdgroup_mm);
604
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ3_S_F32, mul_mm_iq3_s_f32, ctx->support_simdgroup_mm);
605
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_S_F32, mul_mm_iq2_s_f32, ctx->support_simdgroup_mm);
606
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ1_S_F32, mul_mm_iq1_s_f32, ctx->support_simdgroup_mm);
607
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ1_M_F32, mul_mm_iq1_m_f32, ctx->support_simdgroup_mm);
608
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ4_NL_F32, mul_mm_iq4_nl_f32, ctx->support_simdgroup_mm);
609
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ4_XS_F32, mul_mm_iq4_xs_f32, ctx->support_simdgroup_mm);
610
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_F32_F32, mul_mm_id_f32_f32, ctx->support_simdgroup_mm);
611
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_F16_F32, mul_mm_id_f16_f32, ctx->support_simdgroup_mm);
612
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_0_F32, mul_mm_id_q4_0_f32, ctx->support_simdgroup_mm);
613
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_1_F32, mul_mm_id_q4_1_f32, ctx->support_simdgroup_mm);
614
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q5_0_F32, mul_mm_id_q5_0_f32, ctx->support_simdgroup_mm);
615
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q5_1_F32, mul_mm_id_q5_1_f32, ctx->support_simdgroup_mm);
616
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q8_0_F32, mul_mm_id_q8_0_f32, ctx->support_simdgroup_mm);
617
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q2_K_F32, mul_mm_id_q2_K_f32, ctx->support_simdgroup_mm);
618
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q3_K_F32, mul_mm_id_q3_K_f32, ctx->support_simdgroup_mm);
619
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_K_F32, mul_mm_id_q4_K_f32, ctx->support_simdgroup_mm);
620
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q5_K_F32, mul_mm_id_q5_K_f32, ctx->support_simdgroup_mm);
621
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q6_K_F32, mul_mm_id_q6_K_f32, ctx->support_simdgroup_mm);
622
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_XXS_F32, mul_mm_id_iq2_xxs_f32, ctx->support_simdgroup_mm);
623
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_XS_F32, mul_mm_id_iq2_xs_f32, ctx->support_simdgroup_mm);
624
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ3_XXS_F32, mul_mm_id_iq3_xxs_f32, ctx->support_simdgroup_mm);
625
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ3_S_F32, mul_mm_id_iq3_s_f32, ctx->support_simdgroup_mm);
626
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_S_F32, mul_mm_id_iq2_s_f32, ctx->support_simdgroup_mm);
627
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ1_S_F32, mul_mm_id_iq1_s_f32, ctx->support_simdgroup_mm);
628
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ1_M_F32, mul_mm_id_iq1_m_f32, ctx->support_simdgroup_mm);
629
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ4_NL_F32, mul_mm_id_iq4_nl_f32, ctx->support_simdgroup_mm);
630
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ4_XS_F32, mul_mm_id_iq4_xs_f32, ctx->support_simdgroup_mm);
631
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ROPE_NORM_F32, rope_norm_f32, true);
632
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ROPE_NORM_F16, rope_norm_f16, true);
633
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ROPE_NEOX_F32, rope_neox_f32, true);
634
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ROPE_NEOX_F16, rope_neox_f16, true);
635
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_IM2COL_F16, im2col_f16, true);
636
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_IM2COL_F32, im2col_f32, true);
637
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_UPSCALE_F32, upscale_f32, true);
638
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_PAD_F32, pad_f32, true);
639
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_TIMESTEP_EMBEDDING_F32, timestep_embedding_f32, true);
640
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ARANGE_F32, arange_f32, true);
641
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ARGSORT_F32_I32_ASC, argsort_f32_i32_asc, true);
642
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_ARGSORT_F32_I32_DESC, argsort_f32_i32_desc, true);
643
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_LEAKY_RELU_F32, leaky_relu_f32, true);
644
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H64, flash_attn_ext_f16_h64, ctx->support_simdgroup_mm);
645
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H80, flash_attn_ext_f16_h80, ctx->support_simdgroup_mm);
646
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H96, flash_attn_ext_f16_h96, ctx->support_simdgroup_mm);
647
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H112, flash_attn_ext_f16_h112, ctx->support_simdgroup_mm);
648
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H128, flash_attn_ext_f16_h128, ctx->support_simdgroup_mm);
649
- //GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H256, flash_attn_ext_f16_h256, ctx->support_simdgroup_mm);
650
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_VEC_F16_H128, flash_attn_ext_vec_f16_h128, ctx->support_simdgroup_reduction);
651
- //GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_VEC_F16_H256, flash_attn_ext_vec_f16_h256, ctx->support_simdgroup_reduction);
652
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_F16, cpy_f32_f16, true);
653
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_F32, cpy_f32_f32, true);
654
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_Q8_0, cpy_f32_q8_0, true);
655
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_Q4_0, cpy_f32_q4_0, true);
656
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_Q4_1, cpy_f32_q4_1, true);
657
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_0, cpy_f32_q5_0, true);
658
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_1, cpy_f32_q5_1, true);
659
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F32_IQ4_NL, cpy_f32_iq4_nl, true);
660
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F16_F16, cpy_f16_f16, true);
661
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CPY_F16_F32, cpy_f16_f32, true);
662
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_CONCAT, concat, true);
663
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SQR, sqr, true);
664
- GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SUM_ROWS, sum_rows, true);
665
- }
666
-
667
- [metal_library release];
668
- return ctx;
669
- }
670
-
671
- static void ggml_metal_free(struct ggml_metal_context * ctx) {
672
- GGML_METAL_LOG_INFO("%s: deallocating\n", __func__);
673
-
674
- for (int i = 0; i < GGML_METAL_KERNEL_TYPE_COUNT; ++i) {
675
- [ctx->kernels[i].pipeline release];
676
- }
677
-
678
- [ctx->queue release];
679
- [ctx->device release];
680
-
681
- dispatch_release(ctx->d_queue);
682
-
683
- free(ctx);
684
- }
685
-
686
- // temporarily defined here for compatibility between ggml-backend and the old API
687
-
688
- struct ggml_backend_metal_buffer {
689
- void * data;
690
- size_t size;
691
-
692
- id<MTLBuffer> metal;
693
- };
694
-
695
- struct ggml_backend_metal_buffer_context {
696
- void * all_data;
697
- size_t all_size;
698
- bool owned;
699
-
700
- // multiple buffers are used only to avoid the maximum buffer size limitation when using mmap
701
- int n_buffers;
702
- struct ggml_backend_metal_buffer buffers[GGML_METAL_MAX_BUFFERS];
703
- };
704
-
705
- // finds the Metal buffer that contains the tensor data on the GPU device
706
- // the assumption is that there is 1-to-1 mapping between the host and device memory buffers, so we can find the
707
- // Metal buffer based on the host memory pointer
708
- //
709
- static id<MTLBuffer> ggml_metal_get_buffer(struct ggml_tensor * t, size_t * offs) {
710
- //GGML_METAL_LOG_INFO("%s: data tensor '%16s', offs_data = %8ld, offs_eval = %8ld, offs_cach = %8ld\n", __func__, t->name, offs_data, offs_eval, offs_cach);
711
-
712
- const int64_t tsize = ggml_nbytes(t);
713
-
714
- ggml_backend_buffer_t buffer = t->view_src ? t->view_src->buffer : t->buffer;
715
-
716
- struct ggml_backend_metal_buffer_context * buf_ctx = (struct ggml_backend_metal_buffer_context *) buffer->context;
717
-
718
- // find the view that contains the tensor fully
719
- for (int i = 0; i < buf_ctx->n_buffers; ++i) {
720
- const int64_t ioffs = (int64_t) t->data - (int64_t) buf_ctx->buffers[i].data;
721
-
722
- //GGML_METAL_LOG_INFO("ioffs = %10ld, tsize = %10ld, sum = %10ld, buf_ctx->buffers[%d].size = %10ld\n", ioffs, tsize, ioffs + tsize, i, buf_ctx->buffers[i].size);
723
- if (ioffs >= 0 && ioffs + tsize <= (int64_t) buf_ctx->buffers[i].size) {
724
- *offs = (size_t) ioffs;
725
-
726
- //GGML_METAL_LOG_INFO("%s: tensor '%16s', offs = %8ld\n", __func__, t->name, *offs);
727
-
728
- return buf_ctx->buffers[i].metal;
729
- }
730
- }
731
-
732
- GGML_METAL_LOG_ERROR("%s: error: tensor '%s' buffer is nil\n", __func__, t->name);
733
-
734
- return nil;
735
- }
736
-
737
- static bool ggml_metal_supports_op(const struct ggml_metal_context * ctx, const struct ggml_tensor * op) {
738
- for (size_t i = 0, n = 3; i < n; ++i) {
739
- if (op->src[i] != NULL && op->src[i]->type == GGML_TYPE_BF16) {
740
- return false;
741
- }
742
- }
743
-
744
- switch (op->op) {
745
- case GGML_OP_UNARY:
746
- switch (ggml_get_unary_op(op)) {
747
- case GGML_UNARY_OP_TANH:
748
- case GGML_UNARY_OP_RELU:
749
- case GGML_UNARY_OP_SIGMOID:
750
- case GGML_UNARY_OP_GELU:
751
- case GGML_UNARY_OP_GELU_QUICK:
752
- case GGML_UNARY_OP_SILU:
753
- return ggml_is_contiguous(op->src[0]);
754
- default:
755
- return false;
756
- }
757
- case GGML_OP_NONE:
758
- case GGML_OP_RESHAPE:
759
- case GGML_OP_VIEW:
760
- case GGML_OP_TRANSPOSE:
761
- case GGML_OP_PERMUTE:
762
- case GGML_OP_CONCAT:
763
- case GGML_OP_ADD:
764
- case GGML_OP_ACC:
765
- case GGML_OP_MUL:
766
- case GGML_OP_DIV:
767
- case GGML_OP_REPEAT:
768
- case GGML_OP_SCALE:
769
- case GGML_OP_CLAMP:
770
- case GGML_OP_SQR:
771
- case GGML_OP_SUM_ROWS:
772
- return true;
773
- case GGML_OP_SOFT_MAX:
774
- case GGML_OP_RMS_NORM:
775
- case GGML_OP_GROUP_NORM:
776
- return ctx->support_simdgroup_reduction;
777
- case GGML_OP_NORM:
778
- case GGML_OP_ROPE:
779
- case GGML_OP_IM2COL:
780
- return true;
781
- case GGML_OP_POOL_1D:
782
- case GGML_OP_POOL_2D:
783
- return false;
784
- case GGML_OP_UPSCALE:
785
- case GGML_OP_PAD:
786
- case GGML_OP_ARANGE:
787
- case GGML_OP_TIMESTEP_EMBEDDING:
788
- case GGML_OP_ARGSORT:
789
- case GGML_OP_LEAKY_RELU:
790
- return true;
791
- case GGML_OP_FLASH_ATTN_EXT:
792
- if (op->src[1]->type != GGML_TYPE_F16) {
793
- return false;
794
- }
795
- if (op->src[2]->type != GGML_TYPE_F16) {
796
- return false;
797
- }
798
- if (op->src[0]->ne[0] == 256) {
799
- return false;
800
- }
801
- return ctx->support_simdgroup_mm; // TODO: over-restricted for vec-kernels
802
- case GGML_OP_MUL_MAT:
803
- case GGML_OP_MUL_MAT_ID:
804
- return ctx->support_simdgroup_reduction &&
805
- (op->src[0]->type != GGML_TYPE_F32 || op->src[1]->type == GGML_TYPE_F32);
806
- case GGML_OP_CPY:
807
- case GGML_OP_DUP:
808
- case GGML_OP_CONT:
809
- {
810
- switch (op->src[0]->type) {
811
- case GGML_TYPE_F32:
812
- switch (op->type) {
813
- case GGML_TYPE_F16:
814
- case GGML_TYPE_F32:
815
- case GGML_TYPE_Q8_0:
816
- case GGML_TYPE_Q4_0:
817
- case GGML_TYPE_Q4_1:
818
- case GGML_TYPE_Q5_0:
819
- case GGML_TYPE_Q5_1:
820
- case GGML_TYPE_IQ4_NL:
821
- return true;
822
- default:
823
- return false;
824
- }
825
- case GGML_TYPE_F16:
826
- switch (op->type) {
827
- case GGML_TYPE_F16:
828
- case GGML_TYPE_F32:
829
- return true;
830
- default:
831
- return false;
832
- }
833
- default:
834
- return false;
835
- };
836
- }
837
- case GGML_OP_DIAG_MASK_INF:
838
- case GGML_OP_GET_ROWS:
839
- {
840
- return op->src[0]->type != GGML_TYPE_BF16 && op->ne[3] == 1;
841
- }
842
- default:
843
- return false;
844
- }
845
- }
846
-
847
- static enum ggml_status ggml_metal_graph_compute(
848
- struct ggml_metal_context * ctx,
849
- struct ggml_cgraph * gf) {
850
-
851
- @autoreleasepool {
852
- MTLComputePassDescriptor * edesc = MTLComputePassDescriptor.computePassDescriptor;
853
- edesc.dispatchType = MTLDispatchTypeSerial;
854
-
855
- // create multiple command buffers and enqueue them
856
- // then, we encode the graph into the command buffers in parallel
857
-
858
- const int n_nodes = gf->n_nodes;
859
- const int n_cb = ctx->n_cb;
860
- const int n_nodes_per_cb = (n_nodes + n_cb - 1) / n_cb;
861
-
862
- const bool should_capture = ctx->should_capture_next_compute;
863
- if (should_capture) {
864
- ctx->should_capture_next_compute = false;
865
-
866
- MTLCaptureDescriptor * descriptor = [MTLCaptureDescriptor new];
867
- descriptor.captureObject = ctx->queue;
868
-
869
- NSError * error = nil;
870
- if (![[MTLCaptureManager sharedCaptureManager] startCaptureWithDescriptor:descriptor error:&error]) {
871
- GGML_METAL_LOG_ERROR("%s: error: unable to start capture '%s'\n", __func__, [[error localizedDescription] UTF8String]);
872
- GGML_ASSERT(!"capture failed");
873
- }
874
- }
875
-
876
- id<MTLCommandBuffer> command_buffer_builder[n_cb];
877
- for (int cb_idx = 0; cb_idx < n_cb; ++cb_idx) {
878
- id<MTLCommandBuffer> command_buffer = [ctx->queue commandBufferWithUnretainedReferences];
879
- command_buffer_builder[cb_idx] = command_buffer;
880
-
881
- // enqueue the command buffers in order to specify their execution order
882
- [command_buffer enqueue];
883
- }
884
-
885
- const id<MTLCommandBuffer> *command_buffers = command_buffer_builder;
886
-
887
- dispatch_apply(n_cb, ctx->d_queue, ^(size_t iter) {
888
- const int cb_idx = iter;
889
-
890
- size_t offs_src0 = 0;
891
- size_t offs_src1 = 0;
892
- size_t offs_src2 = 0;
893
- size_t offs_dst = 0;
894
-
895
- id<MTLCommandBuffer> command_buffer = command_buffers[cb_idx];
896
- id<MTLComputeCommandEncoder> encoder = [command_buffer computeCommandEncoderWithDescriptor: edesc];
897
-
898
- const int node_start = (cb_idx + 0) * n_nodes_per_cb;
899
- const int node_end = MIN((cb_idx == n_cb - 1) ? n_nodes : (cb_idx + 1) * n_nodes_per_cb, n_nodes);
900
-
901
- for (int i = node_start; i < node_end; ++i) {
902
- if (i == -1) {
903
- [encoder memoryBarrierWithScope:MTLBarrierScopeBuffers];
904
- continue;
905
- }
906
-
907
- //GGML_METAL_LOG_INFO("%s: encoding node %3d, op = %8s\n", __func__, i, ggml_op_name(gf->nodes[i]->op));
908
-
909
- struct ggml_tensor * src0 = gf->nodes[i]->src[0];
910
- struct ggml_tensor * src1 = gf->nodes[i]->src[1];
911
- struct ggml_tensor * src2 = gf->nodes[i]->src[2];
912
- struct ggml_tensor * dst = gf->nodes[i];
913
-
914
- if (ggml_is_empty(dst)) {
915
- continue;
916
- }
917
-
918
- switch (dst->op) {
919
- case GGML_OP_NONE:
920
- case GGML_OP_RESHAPE:
921
- case GGML_OP_VIEW:
922
- case GGML_OP_TRANSPOSE:
923
- case GGML_OP_PERMUTE:
924
- {
925
- // noop -> next node
926
- } continue;
927
- default:
928
- {
929
- } break;
930
- }
931
-
932
- if (!ggml_metal_supports_op(ctx, dst)) {
933
- GGML_METAL_LOG_ERROR("%s: error: unsupported op '%s'\n", __func__, ggml_op_desc(dst));
934
- GGML_ASSERT(!"unsupported op");
935
- }
936
-
937
- if (should_capture) {
938
- [encoder pushDebugGroup:[NSString stringWithCString:ggml_op_desc(dst) encoding:NSUTF8StringEncoding]];
939
- }
940
-
941
- const int64_t ne00 = src0 ? src0->ne[0] : 0;
942
- const int64_t ne01 = src0 ? src0->ne[1] : 0;
943
- const int64_t ne02 = src0 ? src0->ne[2] : 0;
944
- const int64_t ne03 = src0 ? src0->ne[3] : 0;
945
-
946
- const uint64_t nb00 = src0 ? src0->nb[0] : 0;
947
- const uint64_t nb01 = src0 ? src0->nb[1] : 0;
948
- const uint64_t nb02 = src0 ? src0->nb[2] : 0;
949
- const uint64_t nb03 = src0 ? src0->nb[3] : 0;
950
-
951
- const int64_t ne10 = src1 ? src1->ne[0] : 0;
952
- const int64_t ne11 = src1 ? src1->ne[1] : 0;
953
- const int64_t ne12 = src1 ? src1->ne[2] : 0;
954
- const int64_t ne13 = src1 ? src1->ne[3] : 0;
955
-
956
- const uint64_t nb10 = src1 ? src1->nb[0] : 0;
957
- const uint64_t nb11 = src1 ? src1->nb[1] : 0;
958
- const uint64_t nb12 = src1 ? src1->nb[2] : 0;
959
- const uint64_t nb13 = src1 ? src1->nb[3] : 0;
960
-
961
- const int64_t ne20 = src2 ? src2->ne[0] : 0;
962
- const int64_t ne21 = src2 ? src2->ne[1] : 0;
963
- const int64_t ne22 = src2 ? src2->ne[2] : 0; GGML_UNUSED(ne22);
964
- const int64_t ne23 = src2 ? src2->ne[3] : 0; GGML_UNUSED(ne23);
965
-
966
- const uint64_t nb20 = src2 ? src2->nb[0] : 0; GGML_UNUSED(nb20);
967
- const uint64_t nb21 = src2 ? src2->nb[1] : 0;
968
- const uint64_t nb22 = src2 ? src2->nb[2] : 0;
969
- const uint64_t nb23 = src2 ? src2->nb[3] : 0;
970
-
971
- const int64_t ne0 = dst ? dst->ne[0] : 0;
972
- const int64_t ne1 = dst ? dst->ne[1] : 0;
973
- const int64_t ne2 = dst ? dst->ne[2] : 0;
974
- const int64_t ne3 = dst ? dst->ne[3] : 0;
975
-
976
- const uint64_t nb0 = dst ? dst->nb[0] : 0;
977
- const uint64_t nb1 = dst ? dst->nb[1] : 0;
978
- const uint64_t nb2 = dst ? dst->nb[2] : 0;
979
- const uint64_t nb3 = dst ? dst->nb[3] : 0;
980
-
981
- const enum ggml_type src0t = src0 ? src0->type : GGML_TYPE_COUNT;
982
- const enum ggml_type src1t = src1 ? src1->type : GGML_TYPE_COUNT;
983
- const enum ggml_type dstt = dst ? dst->type : GGML_TYPE_COUNT;
984
-
985
- id<MTLBuffer> id_src0 = src0 ? ggml_metal_get_buffer(src0, &offs_src0) : nil;
986
- id<MTLBuffer> id_src1 = src1 ? ggml_metal_get_buffer(src1, &offs_src1) : nil;
987
- id<MTLBuffer> id_src2 = src2 ? ggml_metal_get_buffer(src2, &offs_src2) : nil;
988
- id<MTLBuffer> id_dst = dst ? ggml_metal_get_buffer(dst, &offs_dst) : nil;
989
-
990
- //GGML_METAL_LOG_INFO("%s: op - %s\n", __func__, ggml_op_name(dst->op));
991
- //if (src0) {
992
- // GGML_METAL_LOG_INFO("%s: src0 - %4s [%5lld, %5lld, %5lld], %d, %s\n", __func__, ggml_type_name(src0t), ne00, ne01, ne02,
993
- // ggml_is_contiguous(src0), src0->name);
994
- //}
995
- //if (src1) {
996
- // GGML_METAL_LOG_INFO("%s: src1 - %4s [%5lld, %5lld, %5lld], %d, %s\n", __func__, ggml_type_name(src1t), ne10, ne11, ne12,
997
- // ggml_is_contiguous(src1), src1->name);
998
- //}
999
- //if (dst) {
1000
- // GGML_METAL_LOG_INFO("%s: dst - %4s [%5lld, %5lld, %5lld], 1, %s\n", __func__, ggml_type_name(dstt), ne0, ne1, ne2,
1001
- // dst->name);
1002
- //}
1003
-
1004
- switch (dst->op) {
1005
- case GGML_OP_CONCAT:
1006
- {
1007
- id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CONCAT].pipeline;
1008
-
1009
- const int32_t dim = ((int32_t *) dst->op_params)[0];
1010
-
1011
- [encoder setComputePipelineState:pipeline];
1012
- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
1013
- [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];
1014
- [encoder setBuffer:id_dst offset:offs_dst atIndex:2];
1015
- [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:3];
1016
- [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:4];
1017
- [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:5];
1018
- [encoder setBytes:&ne03 length:sizeof(ne03) atIndex:6];
1019
- [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:7];
1020
- [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:8];
1021
- [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:9];
1022
- [encoder setBytes:&nb03 length:sizeof(nb03) atIndex:10];
1023
- [encoder setBytes:&ne10 length:sizeof(ne10) atIndex:11];
1024
- [encoder setBytes:&ne11 length:sizeof(ne11) atIndex:12];
1025
- [encoder setBytes:&ne12 length:sizeof(ne12) atIndex:13];
1026
- [encoder setBytes:&ne13 length:sizeof(ne13) atIndex:14];
1027
- [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:15];
1028
- [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:16];
1029
- [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:17];
1030
- [encoder setBytes:&nb13 length:sizeof(nb13) atIndex:18];
1031
- [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:19];
1032
- [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:20];
1033
- [encoder setBytes:&ne2 length:sizeof(ne2) atIndex:21];
1034
- [encoder setBytes:&ne3 length:sizeof(ne3) atIndex:22];
1035
- [encoder setBytes:&nb0 length:sizeof(nb0) atIndex:23];
1036
- [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:24];
1037
- [encoder setBytes:&nb2 length:sizeof(nb2) atIndex:25];
1038
- [encoder setBytes:&nb3 length:sizeof(nb3) atIndex:26];
1039
- [encoder setBytes:&dim length:sizeof(dim) atIndex:27];
1040
-
1041
- const int nth = MIN(1024, ne0);
1042
-
1043
- [encoder dispatchThreadgroups:MTLSizeMake(ne1, ne2, ne3) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];
1044
- } break;
1045
- case GGML_OP_ADD:
1046
- case GGML_OP_MUL:
1047
- case GGML_OP_DIV:
1048
- {
1049
- GGML_ASSERT(src0t == GGML_TYPE_F32);
1050
- GGML_ASSERT(src1t == GGML_TYPE_F32);
1051
-
1052
- const size_t offs = 0;
1053
-
1054
- bool bcast_row = false;
1055
-
1056
- int64_t nb = ne00; // used by the "row" kernels
1057
-
1058
- id<MTLComputePipelineState> pipeline = nil;
1059
-
1060
- if (ggml_nelements(src1) == ne10 && ggml_is_contiguous(src1) && ne00 % 4 == 0 && ne10 % 4 == 0) {
1061
- GGML_ASSERT(ggml_is_contiguous(src0));
1062
-
1063
- // src1 is a row
1064
- GGML_ASSERT(ne11 == 1);
1065
-
1066
- nb = ne00 / 4;
1067
- switch (dst->op) {
1068
- case GGML_OP_ADD: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_ADD_ROW].pipeline; break;
1069
- case GGML_OP_MUL: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_ROW].pipeline; break;
1070
- case GGML_OP_DIV: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_DIV_ROW].pipeline; break;
1071
- default: GGML_ASSERT(false);
1072
- }
1073
-
1074
- bcast_row = true;
1075
- } else {
1076
- switch (dst->op) {
1077
- case GGML_OP_ADD: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_ADD].pipeline; break;
1078
- case GGML_OP_MUL: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL].pipeline; break;
1079
- case GGML_OP_DIV: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_DIV].pipeline; break;
1080
- default: GGML_ASSERT(false);
1081
- }
1082
- }
1083
-
1084
- [encoder setComputePipelineState:pipeline];
1085
- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
1086
- [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];
1087
- [encoder setBuffer:id_dst offset:offs_dst atIndex:2];
1088
- [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:3];
1089
- [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:4];
1090
- [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:5];
1091
- [encoder setBytes:&ne03 length:sizeof(ne03) atIndex:6];
1092
- [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:7];
1093
- [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:8];
1094
- [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:9];
1095
- [encoder setBytes:&nb03 length:sizeof(nb03) atIndex:10];
1096
- [encoder setBytes:&ne10 length:sizeof(ne10) atIndex:11];
1097
- [encoder setBytes:&ne11 length:sizeof(ne11) atIndex:12];
1098
- [encoder setBytes:&ne12 length:sizeof(ne12) atIndex:13];
1099
- [encoder setBytes:&ne13 length:sizeof(ne13) atIndex:14];
1100
- [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:15];
1101
- [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:16];
1102
- [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:17];
1103
- [encoder setBytes:&nb13 length:sizeof(nb13) atIndex:18];
1104
- [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:19];
1105
- [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:20];
1106
- [encoder setBytes:&ne2 length:sizeof(ne2) atIndex:21];
1107
- [encoder setBytes:&ne3 length:sizeof(ne3) atIndex:22];
1108
- [encoder setBytes:&nb0 length:sizeof(nb0) atIndex:23];
1109
- [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:24];
1110
- [encoder setBytes:&nb2 length:sizeof(nb2) atIndex:25];
1111
- [encoder setBytes:&nb3 length:sizeof(nb3) atIndex:26];
1112
- [encoder setBytes:&offs length:sizeof(offs) atIndex:27];
1113
- [encoder setBytes:&nb length:sizeof(nb) atIndex:28];
1114
-
1115
- if (bcast_row) {
1116
- const int64_t n = ggml_nelements(dst)/4;
1117
-
1118
- [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
1119
- } else {
1120
- const int nth = MIN((int) pipeline.maxTotalThreadsPerThreadgroup, ne0);
1121
-
1122
- [encoder dispatchThreadgroups:MTLSizeMake(ne01, ne02, ne03) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];
1123
- }
1124
- } break;
1125
- case GGML_OP_REPEAT:
1126
- {
1127
- id<MTLComputePipelineState> pipeline;
1128
-
1129
- switch (src0t) {
1130
- case GGML_TYPE_F32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_REPEAT_F32].pipeline; break;
1131
- case GGML_TYPE_F16: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_REPEAT_F16].pipeline; break;
1132
- case GGML_TYPE_I32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_REPEAT_I32].pipeline; break;
1133
- case GGML_TYPE_I16: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_REPEAT_I16].pipeline; break;
1134
- default: GGML_ASSERT(false);
1135
- }
1136
-
1137
- [encoder setComputePipelineState:pipeline];
1138
- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
1139
- [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
1140
- [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:2];
1141
- [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:3];
1142
- [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:4];
1143
- [encoder setBytes:&ne03 length:sizeof(ne03) atIndex:5];
1144
- [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:6];
1145
- [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:7];
1146
- [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:8];
1147
- [encoder setBytes:&nb03 length:sizeof(nb03) atIndex:9];
1148
- [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:10];
1149
- [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:11];
1150
- [encoder setBytes:&ne2 length:sizeof(ne2) atIndex:12];
1151
- [encoder setBytes:&ne3 length:sizeof(ne3) atIndex:13];
1152
- [encoder setBytes:&nb0 length:sizeof(nb0) atIndex:14];
1153
- [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:15];
1154
- [encoder setBytes:&nb2 length:sizeof(nb2) atIndex:16];
1155
- [encoder setBytes:&nb3 length:sizeof(nb3) atIndex:17];
1156
-
1157
- const int nth = MIN((int) pipeline.maxTotalThreadsPerThreadgroup, ne0);
1158
-
1159
- [encoder dispatchThreadgroups:MTLSizeMake(ne1, ne2, ne3) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];
1160
- } break;
1161
- case GGML_OP_ACC:
1162
- {
1163
- GGML_ASSERT(src0t == GGML_TYPE_F32);
1164
- GGML_ASSERT(src1t == GGML_TYPE_F32);
1165
- GGML_ASSERT(dstt == GGML_TYPE_F32);
1166
-
1167
- GGML_ASSERT(ggml_is_contiguous(src0));
1168
- GGML_ASSERT(ggml_is_contiguous(src1));
1169
-
1170
- const size_t pnb1 = ((int32_t *) dst->op_params)[0];
1171
- const size_t pnb2 = ((int32_t *) dst->op_params)[1];
1172
- const size_t pnb3 = ((int32_t *) dst->op_params)[2];
1173
- const size_t offs = ((int32_t *) dst->op_params)[3];
1174
-
1175
- const bool inplace = (bool) ((int32_t *) dst->op_params)[4];
1176
-
1177
- if (!inplace) {
1178
- // run a separete kernel to cpy src->dst
1179
- // not sure how to avoid this
1180
- // TODO: make a simpler cpy_bytes kernel
1181
-
1182
- const id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_F32].pipeline;
1183
-
1184
- [encoder setComputePipelineState:pipeline];
1185
- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
1186
- [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
1187
- [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:2];
1188
- [encoder setBytes:&ne01 length:sizeof( int64_t) atIndex:3];
1189
- [encoder setBytes:&ne02 length:sizeof( int64_t) atIndex:4];
1190
- [encoder setBytes:&ne03 length:sizeof( int64_t) atIndex:5];
1191
- [encoder setBytes:&nb00 length:sizeof(uint64_t) atIndex:6];
1192
- [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:7];
1193
- [encoder setBytes:&nb02 length:sizeof(uint64_t) atIndex:8];
1194
- [encoder setBytes:&nb03 length:sizeof(uint64_t) atIndex:9];
1195
- [encoder setBytes:&ne0 length:sizeof( int64_t) atIndex:10];
1196
- [encoder setBytes:&ne1 length:sizeof( int64_t) atIndex:11];
1197
- [encoder setBytes:&ne2 length:sizeof( int64_t) atIndex:12];
1198
- [encoder setBytes:&ne3 length:sizeof( int64_t) atIndex:13];
1199
- [encoder setBytes:&nb0 length:sizeof(uint64_t) atIndex:14];
1200
- [encoder setBytes:&nb1 length:sizeof(uint64_t) atIndex:15];
1201
- [encoder setBytes:&nb2 length:sizeof(uint64_t) atIndex:16];
1202
- [encoder setBytes:&nb3 length:sizeof(uint64_t) atIndex:17];
1203
-
1204
- const int nth = MIN((int) pipeline.maxTotalThreadsPerThreadgroup, ne00);
1205
-
1206
- [encoder dispatchThreadgroups:MTLSizeMake(ne01, ne02, ne03) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];
1207
- }
1208
-
1209
- const id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_ADD].pipeline;
1210
-
1211
- [encoder setComputePipelineState:pipeline];
1212
- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
1213
- [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];
1214
- [encoder setBuffer:id_dst offset:offs_dst atIndex:2];
1215
- [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:3];
1216
- [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:4];
1217
- [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:5];
1218
- [encoder setBytes:&ne03 length:sizeof(ne03) atIndex:6];
1219
- [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:7];
1220
- [encoder setBytes:&pnb1 length:sizeof(pnb1) atIndex:8];
1221
- [encoder setBytes:&pnb2 length:sizeof(pnb2) atIndex:9];
1222
- [encoder setBytes:&pnb3 length:sizeof(pnb3) atIndex:10];
1223
- [encoder setBytes:&ne10 length:sizeof(ne10) atIndex:11];
1224
- [encoder setBytes:&ne11 length:sizeof(ne11) atIndex:12];
1225
- [encoder setBytes:&ne12 length:sizeof(ne12) atIndex:13];
1226
- [encoder setBytes:&ne13 length:sizeof(ne13) atIndex:14];
1227
- [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:15];
1228
- [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:16];
1229
- [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:17];
1230
- [encoder setBytes:&nb13 length:sizeof(nb13) atIndex:18];
1231
- [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:19];
1232
- [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:20];
1233
- [encoder setBytes:&ne2 length:sizeof(ne2) atIndex:21];
1234
- [encoder setBytes:&ne3 length:sizeof(ne3) atIndex:22];
1235
- [encoder setBytes:&nb0 length:sizeof(nb0) atIndex:23];
1236
- [encoder setBytes:&pnb1 length:sizeof(pnb1) atIndex:24];
1237
- [encoder setBytes:&pnb2 length:sizeof(pnb2) atIndex:25];
1238
- [encoder setBytes:&pnb3 length:sizeof(pnb3) atIndex:26];
1239
- [encoder setBytes:&offs length:sizeof(offs) atIndex:27];
1240
-
1241
- const int nth = MIN((int) pipeline.maxTotalThreadsPerThreadgroup, ne00);
1242
-
1243
- [encoder dispatchThreadgroups:MTLSizeMake(ne11, ne12, ne13) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];
1244
- } break;
1245
- case GGML_OP_SCALE:
1246
- {
1247
- GGML_ASSERT(ggml_is_contiguous(src0));
1248
-
1249
- float scale;
1250
- memcpy(&scale, dst->op_params, sizeof(scale));
1251
-
1252
- int64_t n = ggml_nelements(dst);
1253
-
1254
- id<MTLComputePipelineState> pipeline = nil;
1255
-
1256
- if (n % 4 == 0) {
1257
- n /= 4;
1258
- pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SCALE_4].pipeline;
1259
- } else {
1260
- pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SCALE].pipeline;
1261
- }
1262
-
1263
- [encoder setComputePipelineState:pipeline];
1264
- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
1265
- [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
1266
- [encoder setBytes:&scale length:sizeof(scale) atIndex:2];
1267
-
1268
- [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
1269
- } break;
1270
- case GGML_OP_CLAMP:
1271
- {
1272
- id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CLAMP].pipeline;
1273
-
1274
- float min;
1275
- float max;
1276
- memcpy(&min, ((int32_t *) dst->op_params) + 0, sizeof(float));
1277
- memcpy(&max, ((int32_t *) dst->op_params) + 1, sizeof(float));
1278
-
1279
- [encoder setComputePipelineState:pipeline];
1280
- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
1281
- [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
1282
- [encoder setBytes:&min length:sizeof(min) atIndex:2];
1283
- [encoder setBytes:&max length:sizeof(max) atIndex:3];
1284
-
1285
- const int64_t n = ggml_nelements(dst);
1286
-
1287
- [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
1288
- } break;
1289
- case GGML_OP_UNARY:
1290
- switch (ggml_get_unary_op(gf->nodes[i])) {
1291
- // we are not taking into account the strides, so for now require contiguous tensors
1292
- GGML_ASSERT(ggml_is_contiguous(src0));
1293
-
1294
- case GGML_UNARY_OP_TANH:
1295
- {
1296
- id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_TANH].pipeline;
1297
-
1298
- [encoder setComputePipelineState:pipeline];
1299
- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
1300
- [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
1301
-
1302
- const int64_t n = ggml_nelements(dst);
1303
-
1304
- [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
1305
- } break;
1306
- case GGML_UNARY_OP_RELU:
1307
- {
1308
- id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_RELU].pipeline;
1309
-
1310
- [encoder setComputePipelineState:pipeline];
1311
- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
1312
- [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
1313
-
1314
- const int64_t n = ggml_nelements(dst);
1315
-
1316
- [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
1317
- } break;
1318
- case GGML_UNARY_OP_SIGMOID:
1319
- {
1320
- id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SIGMOID].pipeline;
1321
-
1322
- [encoder setComputePipelineState:pipeline];
1323
- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
1324
- [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
1325
-
1326
- const int64_t n = ggml_nelements(dst);
1327
-
1328
- [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
1329
- } break;
1330
- case GGML_UNARY_OP_GELU:
1331
- {
1332
- int64_t n = ggml_nelements(dst);
1333
-
1334
- id<MTLComputePipelineState> pipeline = nil;
1335
-
1336
- if (n % 4 == 0) {
1337
- pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GELU_4].pipeline;
1338
- n /= 4;
1339
- } else {
1340
- pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GELU].pipeline;
1341
- }
1342
-
1343
- [encoder setComputePipelineState:pipeline];
1344
- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
1345
- [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
1346
-
1347
- [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
1348
- } break;
1349
- case GGML_UNARY_OP_GELU_QUICK:
1350
- {
1351
- int64_t n = ggml_nelements(dst);
1352
-
1353
- id<MTLComputePipelineState> pipeline = nil;
1354
-
1355
- if (n % 4 == 0) {
1356
- pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GELU_QUICK_4].pipeline;
1357
- n /= 4;
1358
- } else {
1359
- pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GELU_QUICK].pipeline;
1360
- }
1361
-
1362
- [encoder setComputePipelineState:pipeline];
1363
- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
1364
- [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
1365
-
1366
- [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
1367
- } break;
1368
- case GGML_UNARY_OP_SILU:
1369
- {
1370
- int64_t n = ggml_nelements(dst);
1371
-
1372
- id<MTLComputePipelineState> pipeline = nil;
1373
-
1374
- if (n % 4 == 0) {
1375
- pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SILU_4].pipeline;
1376
- n /= 4;
1377
- } else {
1378
- pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SILU].pipeline;
1379
- }
1380
-
1381
- [encoder setComputePipelineState:pipeline];
1382
- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
1383
- [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
1384
-
1385
- [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
1386
- } break;
1387
- default:
1388
- {
1389
- GGML_METAL_LOG_WARN("%s: node %3d, op = %8s not implemented\n", __func__, i, ggml_op_name(dst->op));
1390
- GGML_ASSERT(false);
1391
- }
1392
- } break;
1393
- case GGML_OP_SQR:
1394
- {
1395
- GGML_ASSERT(ggml_is_contiguous(src0));
1396
-
1397
- id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SQR].pipeline;
1398
-
1399
- [encoder setComputePipelineState:pipeline];
1400
- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
1401
- [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
1402
-
1403
- const int64_t n = ggml_nelements(dst);
1404
-
1405
- [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
1406
- } break;
1407
- case GGML_OP_SUM_ROWS:
1408
- {
1409
- GGML_ASSERT(src0->nb[0] == ggml_type_size(src0->type));
1410
-
1411
- id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SUM_ROWS].pipeline;
1412
-
1413
- [encoder setComputePipelineState:pipeline];
1414
- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
1415
- [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
1416
- [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:2];
1417
- [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:3];
1418
- [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:4];
1419
- [encoder setBytes:&ne03 length:sizeof(ne03) atIndex:5];
1420
- [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:6];
1421
- [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:7];
1422
- [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:8];
1423
- [encoder setBytes:&nb03 length:sizeof(nb03) atIndex:9];
1424
- [encoder setBytes:&ne10 length:sizeof(ne10) atIndex:10];
1425
- [encoder setBytes:&ne11 length:sizeof(ne11) atIndex:11];
1426
- [encoder setBytes:&ne12 length:sizeof(ne12) atIndex:12];
1427
- [encoder setBytes:&ne13 length:sizeof(ne13) atIndex:13];
1428
- [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:14];
1429
- [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:15];
1430
- [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:16];
1431
- [encoder setBytes:&nb13 length:sizeof(nb13) atIndex:17];
1432
- [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:18];
1433
- [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:19];
1434
- [encoder setBytes:&ne2 length:sizeof(ne2) atIndex:20];
1435
- [encoder setBytes:&ne3 length:sizeof(ne3) atIndex:21];
1436
- [encoder setBytes:&nb0 length:sizeof(nb0) atIndex:22];
1437
- [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:23];
1438
- [encoder setBytes:&nb2 length:sizeof(nb2) atIndex:24];
1439
- [encoder setBytes:&nb3 length:sizeof(nb3) atIndex:25];
1440
-
1441
- [encoder dispatchThreadgroups:MTLSizeMake(ne01, ne02, ne03) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
1442
- } break;
1443
- case GGML_OP_SOFT_MAX:
1444
- {
1445
- GGML_ASSERT(!src1 || src1->type == GGML_TYPE_F16 || src1->type == GGML_TYPE_F32);
1446
-
1447
- int nth = 32; // SIMD width
1448
-
1449
- id<MTLComputePipelineState> pipeline = nil;
1450
-
1451
- const bool use_f16 = (src1 && src1->type == GGML_TYPE_F16);
1452
-
1453
- if (ne00%4 == 0) {
1454
- while (nth < ne00/4 && nth*ne01*ne02*ne03 < 256) {
1455
- nth *= 2;
1456
- }
1457
- if (use_f16) {
1458
- pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SOFT_MAX_F16_4].pipeline;
1459
- } else {
1460
- pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SOFT_MAX_F32_4].pipeline;
1461
- }
1462
- } else {
1463
- while (nth < ne00 && nth*ne01*ne02*ne03 < 256) {
1464
- nth *= 2;
1465
- }
1466
- if (use_f16) {
1467
- pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SOFT_MAX_F16].pipeline;
1468
- } else {
1469
- pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SOFT_MAX_F32].pipeline;
1470
- }
1471
- }
1472
-
1473
- float scale;
1474
- float max_bias;
1475
-
1476
- memcpy(&scale, ((int32_t *) dst->op_params) + 0, sizeof(scale));
1477
- memcpy(&max_bias, ((int32_t *) dst->op_params) + 1, sizeof(max_bias));
1478
-
1479
- const int64_t nrows_x = ggml_nrows(src0);
1480
- const int64_t nrows_y = src0->ne[1];
1481
-
1482
- const uint32_t n_head = nrows_x/nrows_y;
1483
- const uint32_t n_head_log2 = 1u << (uint32_t) floorf(log2f((float) n_head));
1484
-
1485
- const float m0 = powf(2.0f, -(max_bias ) / n_head_log2);
1486
- const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2);
1487
-
1488
- [encoder setComputePipelineState:pipeline];
1489
- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
1490
- if (id_src1) {
1491
- [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];
1492
- } else {
1493
- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:1];
1494
- }
1495
- [encoder setBuffer:id_dst offset:offs_dst atIndex:2];
1496
- [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:3];
1497
- [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:4];
1498
- [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:5];
1499
- [encoder setBytes:&scale length:sizeof(scale) atIndex:6];
1500
- [encoder setBytes:&max_bias length:sizeof(max_bias) atIndex:7];
1501
- [encoder setBytes:&m0 length:sizeof(m0) atIndex:8];
1502
- [encoder setBytes:&m1 length:sizeof(m1) atIndex:9];
1503
- [encoder setBytes:&n_head_log2 length:sizeof(n_head_log2) atIndex:10];
1504
- [encoder setThreadgroupMemoryLength:32*sizeof(float) atIndex:0];
1505
-
1506
- [encoder dispatchThreadgroups:MTLSizeMake(ne01*ne02*ne03, 1, 1) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];
1507
- } break;
1508
- case GGML_OP_DIAG_MASK_INF:
1509
- {
1510
- const int n_past = ((int32_t *)(dst->op_params))[0];
1511
-
1512
- id<MTLComputePipelineState> pipeline = nil;
1513
-
1514
- if (ne00%8 == 0) {
1515
- pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_DIAG_MASK_INF_8].pipeline;
1516
- } else {
1517
- pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_DIAG_MASK_INF].pipeline;
1518
- }
1519
-
1520
- [encoder setComputePipelineState:pipeline];
1521
- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
1522
- [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
1523
- [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:2];
1524
- [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:3];
1525
- [encoder setBytes:&n_past length:sizeof(int) atIndex:4];
1526
-
1527
- if (ne00%8 == 0) {
1528
- [encoder dispatchThreadgroups:MTLSizeMake(ne00*ne01*ne02/8, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
1529
- }
1530
- else {
1531
- [encoder dispatchThreadgroups:MTLSizeMake(ne00, ne01, ne02) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
1532
- }
1533
- } break;
1534
- case GGML_OP_MUL_MAT:
1535
- {
1536
- GGML_ASSERT(ne00 == ne10);
1537
-
1538
- GGML_ASSERT(ne12 % ne02 == 0);
1539
- GGML_ASSERT(ne13 % ne03 == 0);
1540
-
1541
- const uint r2 = ne12/ne02;
1542
- const uint r3 = ne13/ne03;
1543
-
1544
- // find the break-even point where the matrix-matrix kernel becomes more efficient compared
1545
- // to the matrix-vector kernel
1546
- int ne11_mm_min = 1;
1547
-
1548
- #if 0
1549
- // the numbers below are measured on M2 Ultra for 7B and 13B models
1550
- // these numbers do not translate to other devices or model sizes
1551
- // TODO: need to find a better approach
1552
- if ([ctx->device.name isEqualToString:@"Apple M2 Ultra"]) {
1553
- switch (src0t) {
1554
- case GGML_TYPE_F16: ne11_mm_min = 2; break;
1555
- case GGML_TYPE_Q8_0: ne11_mm_min = 7; break;
1556
- case GGML_TYPE_Q2_K: ne11_mm_min = 15; break;
1557
- case GGML_TYPE_Q3_K: ne11_mm_min = 7; break;
1558
- case GGML_TYPE_Q4_0:
1559
- case GGML_TYPE_Q4_1: ne11_mm_min = 15; break;
1560
- case GGML_TYPE_Q4_K: ne11_mm_min = 11; break;
1561
- case GGML_TYPE_Q5_0: // not tested yet
1562
- case GGML_TYPE_Q5_1: ne11_mm_min = 13; break; // not tested yet
1563
- case GGML_TYPE_Q5_K: ne11_mm_min = 7; break;
1564
- case GGML_TYPE_Q6_K: ne11_mm_min = 7; break;
1565
- default: ne11_mm_min = 1; break;
1566
- }
1567
- }
1568
- #endif
1569
-
1570
- // for now the matrix-matrix multiplication kernel only works on A14+/M1+ SoCs
1571
- // AMD GPU and older A-chips will reuse matrix-vector multiplication kernel
1572
- if ([ctx->device supportsFamily:MTLGPUFamilyApple7] &&
1573
- !ggml_is_transposed(src0) &&
1574
- !ggml_is_transposed(src1) &&
1575
- src1t == GGML_TYPE_F32 &&
1576
- ne00 % 32 == 0 && ne00 >= 64 &&
1577
- (ne11 > ne11_mm_min || (ggml_is_quantized(src0t) && ne12 > 1))) {
1578
- //printf("matrix: ne00 = %6d, ne01 = %6d, ne02 = %6d, ne11 = %6d, ne12 = %6d\n", ne00, ne01, ne02, ne11, ne12);
1579
-
1580
- // some Metal matrix data types require aligned pointers
1581
- // ref: https://developer.apple.com/metal/Metal-Shading-Language-Specification.pdf (Table 2.5)
1582
- switch (src0->type) {
1583
- case GGML_TYPE_F32: GGML_ASSERT(nb01 % 16 == 0); break;
1584
- case GGML_TYPE_F16: GGML_ASSERT(nb01 % 8 == 0); break;
1585
- default: break;
1586
- }
1587
-
1588
- id<MTLComputePipelineState> pipeline = nil;
1589
-
1590
- switch (src0->type) {
1591
- case GGML_TYPE_F32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_F32_F32 ].pipeline; break;
1592
- case GGML_TYPE_F16: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_F16_F32 ].pipeline; break;
1593
- case GGML_TYPE_Q4_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_0_F32 ].pipeline; break;
1594
- case GGML_TYPE_Q4_1: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_1_F32 ].pipeline; break;
1595
- case GGML_TYPE_Q5_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q5_0_F32 ].pipeline; break;
1596
- case GGML_TYPE_Q5_1: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q5_1_F32 ].pipeline; break;
1597
- case GGML_TYPE_Q8_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q8_0_F32 ].pipeline; break;
1598
- case GGML_TYPE_Q2_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q2_K_F32 ].pipeline; break;
1599
- case GGML_TYPE_Q3_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q3_K_F32 ].pipeline; break;
1600
- case GGML_TYPE_Q4_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q4_K_F32 ].pipeline; break;
1601
- case GGML_TYPE_Q5_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q5_K_F32 ].pipeline; break;
1602
- case GGML_TYPE_Q6_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_Q6_K_F32 ].pipeline; break;
1603
- case GGML_TYPE_IQ2_XXS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_XXS_F32].pipeline; break;
1604
- case GGML_TYPE_IQ2_XS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_XS_F32 ].pipeline; break;
1605
- case GGML_TYPE_IQ3_XXS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_IQ3_XXS_F32].pipeline; break;
1606
- case GGML_TYPE_IQ3_S: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_IQ3_S_F32 ].pipeline; break;
1607
- case GGML_TYPE_IQ2_S: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_IQ2_S_F32 ].pipeline; break;
1608
- case GGML_TYPE_IQ1_S: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_IQ1_S_F32 ].pipeline; break;
1609
- case GGML_TYPE_IQ1_M: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_IQ1_M_F32 ].pipeline; break;
1610
- case GGML_TYPE_IQ4_NL: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_IQ4_NL_F32 ].pipeline; break;
1611
- case GGML_TYPE_IQ4_XS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_IQ4_XS_F32 ].pipeline; break;
1612
- default: GGML_ASSERT(false && "MUL MAT-MAT not implemented");
1613
- }
1614
-
1615
- [encoder setComputePipelineState:pipeline];
1616
- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
1617
- [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];
1618
- [encoder setBuffer:id_dst offset:offs_dst atIndex:2];
1619
- [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:3];
1620
- [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:4];
1621
- [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:5];
1622
- [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:6];
1623
- [encoder setBytes:&ne12 length:sizeof(ne12) atIndex:7];
1624
- [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:8];
1625
- [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:9];
1626
- [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:10];
1627
- [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:11];
1628
- [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:12];
1629
- [encoder setBytes:&r2 length:sizeof(r2) atIndex:13];
1630
- [encoder setBytes:&r3 length:sizeof(r3) atIndex:14];
1631
- [encoder setThreadgroupMemoryLength:8192 atIndex:0];
1632
- [encoder dispatchThreadgroups:MTLSizeMake( (ne11 + 31)/32, (ne01 + 63)/64, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(128, 1, 1)];
1633
- } else {
1634
- int nth0 = 32;
1635
- int nth1 = 1;
1636
- int nrows = 1;
1637
- //printf("vector: ne00 = %6d, ne01 = %6d, ne02 = %6d, ne11 = %6d, ne12 = %6d\n", ne00, ne01, ne02, ne11, ne12);
1638
-
1639
- id<MTLComputePipelineState> pipeline = nil;
1640
-
1641
- // use custom matrix x vector kernel
1642
- switch (src0t) {
1643
- case GGML_TYPE_F32:
1644
- {
1645
- GGML_ASSERT(src1t == GGML_TYPE_F32);
1646
- pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_F32_F32].pipeline;
1647
- nrows = 4;
1648
- } break;
1649
- case GGML_TYPE_F16:
1650
- {
1651
- nth0 = 32;
1652
- nth1 = 1;
1653
- if (src1t == GGML_TYPE_F32) {
1654
- if (ne11 * ne12 < 4) {
1655
- pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F32_1ROW].pipeline;
1656
- } else if (ne00 >= 128 && ne01 >= 8 && ne00%4 == 0) {
1657
- pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F32_L4].pipeline;
1658
- nrows = ne11;
1659
- } else {
1660
- pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F32].pipeline;
1661
- nrows = 4;
1662
- }
1663
- } else {
1664
- pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_F16_F16].pipeline;
1665
- nrows = 4;
1666
- }
1667
- } break;
1668
- case GGML_TYPE_Q4_0:
1669
- {
1670
- nth0 = 8;
1671
- nth1 = 8;
1672
- pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q4_0_F32].pipeline;
1673
- } break;
1674
- case GGML_TYPE_Q4_1:
1675
- {
1676
- nth0 = 8;
1677
- nth1 = 8;
1678
- pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q4_1_F32].pipeline;
1679
- } break;
1680
- case GGML_TYPE_Q5_0:
1681
- {
1682
- nth0 = 8;
1683
- nth1 = 8;
1684
- pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_0_F32].pipeline;
1685
- } break;
1686
- case GGML_TYPE_Q5_1:
1687
- {
1688
- nth0 = 8;
1689
- nth1 = 8;
1690
- pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_1_F32].pipeline;
1691
- } break;
1692
- case GGML_TYPE_Q8_0:
1693
- {
1694
- nth0 = 8;
1695
- nth1 = 8;
1696
- pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q8_0_F32].pipeline;
1697
- } break;
1698
- case GGML_TYPE_Q2_K:
1699
- {
1700
- nth0 = 2;
1701
- nth1 = 32;
1702
- pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q2_K_F32].pipeline;
1703
- } break;
1704
- case GGML_TYPE_Q3_K:
1705
- {
1706
- nth0 = 2;
1707
- nth1 = 32;
1708
- pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q3_K_F32].pipeline;
1709
- } break;
1710
- case GGML_TYPE_Q4_K:
1711
- {
1712
- nth0 = 4; //1;
1713
- nth1 = 8; //32;
1714
- pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q4_K_F32].pipeline;
1715
- } break;
1716
- case GGML_TYPE_Q5_K:
1717
- {
1718
- nth0 = 2;
1719
- nth1 = 32;
1720
- pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_K_F32].pipeline;
1721
- } break;
1722
- case GGML_TYPE_Q6_K:
1723
- {
1724
- nth0 = 2;
1725
- nth1 = 32;
1726
- pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_Q6_K_F32].pipeline;
1727
- } break;
1728
- case GGML_TYPE_IQ2_XXS:
1729
- {
1730
- nth0 = 4;
1731
- nth1 = 16;
1732
- pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_XXS_F32].pipeline;
1733
- } break;
1734
- case GGML_TYPE_IQ2_XS:
1735
- {
1736
- nth0 = 4;
1737
- nth1 = 16;
1738
- pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_XS_F32].pipeline;
1739
- } break;
1740
- case GGML_TYPE_IQ3_XXS:
1741
- {
1742
- nth0 = 4;
1743
- nth1 = 16;
1744
- pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_IQ3_XXS_F32].pipeline;
1745
- } break;
1746
- case GGML_TYPE_IQ3_S:
1747
- {
1748
- nth0 = 4;
1749
- nth1 = 16;
1750
- pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_IQ3_S_F32].pipeline;
1751
- } break;
1752
- case GGML_TYPE_IQ2_S:
1753
- {
1754
- nth0 = 4;
1755
- nth1 = 16;
1756
- pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_IQ2_S_F32].pipeline;
1757
- } break;
1758
- case GGML_TYPE_IQ1_S:
1759
- {
1760
- nth0 = 4;
1761
- nth1 = 16;
1762
- pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_IQ1_S_F32].pipeline;
1763
- } break;
1764
- case GGML_TYPE_IQ1_M:
1765
- {
1766
- nth0 = 4;
1767
- nth1 = 16;
1768
- pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_IQ1_M_F32].pipeline;
1769
- } break;
1770
- case GGML_TYPE_IQ4_NL:
1771
- {
1772
- nth0 = 4;
1773
- nth1 = 16;
1774
- pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_IQ4_NL_F32].pipeline;
1775
- } break;
1776
- case GGML_TYPE_IQ4_XS:
1777
- {
1778
- nth0 = 4;
1779
- nth1 = 16;
1780
- pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_IQ4_XS_F32].pipeline;
1781
- } break;
1782
- default:
1783
- {
1784
- GGML_METAL_LOG_ERROR("Asserting on type %d\n", (int)src0t);
1785
- GGML_ASSERT(false && "not implemented");
1786
- }
1787
- };
1788
-
1789
- if (ggml_is_quantized(src0t)) {
1790
- GGML_ASSERT(ne00 >= nth0*nth1);
1791
- }
1792
-
1793
- [encoder setComputePipelineState:pipeline];
1794
- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
1795
- [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];
1796
- [encoder setBuffer:id_dst offset:offs_dst atIndex:2];
1797
- [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:3];
1798
- [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:4];
1799
- [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:5];
1800
- [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:6];
1801
- [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:7];
1802
- [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:8];
1803
- [encoder setBytes:&ne10 length:sizeof(ne10) atIndex:9];
1804
- [encoder setBytes:&ne11 length:sizeof(ne11) atIndex:10];
1805
- [encoder setBytes:&ne12 length:sizeof(ne12) atIndex:11];
1806
- [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:12];
1807
- [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:13];
1808
- [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:14];
1809
- [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:15];
1810
- [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:16];
1811
- [encoder setBytes:&r2 length:sizeof(r2) atIndex:17];
1812
- [encoder setBytes:&r3 length:sizeof(r3) atIndex:18];
1813
-
1814
- if (src0t == GGML_TYPE_Q4_0 || src0t == GGML_TYPE_Q4_1 || src0t == GGML_TYPE_Q5_0 ||
1815
- src0t == GGML_TYPE_Q5_1 || src0t == GGML_TYPE_Q8_0 || src0t == GGML_TYPE_Q2_K ||
1816
- src0t == GGML_TYPE_IQ1_S || src0t == GGML_TYPE_IQ1_M || src0t == GGML_TYPE_IQ2_S) {
1817
- [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 7)/8, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];
1818
- }
1819
- else if (src0t == GGML_TYPE_IQ2_XXS || src0t == GGML_TYPE_IQ2_XS) {
1820
- const int mem_size = src0t == GGML_TYPE_IQ2_XXS ? 256*8+128 : 512*8+128;
1821
- [encoder setThreadgroupMemoryLength:mem_size atIndex:0];
1822
- [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 7)/8, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];
1823
- }
1824
- else if (src0t == GGML_TYPE_IQ3_XXS || src0t == GGML_TYPE_IQ3_S) {
1825
- const int mem_size = src0t == GGML_TYPE_IQ3_XXS ? 256*4+128 : 512*4;
1826
- [encoder setThreadgroupMemoryLength:mem_size atIndex:0];
1827
- [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 7)/8, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];
1828
- }
1829
- else if (src0t == GGML_TYPE_IQ4_NL || src0t == GGML_TYPE_IQ4_XS) {
1830
- const int mem_size = 32*sizeof(float);
1831
- [encoder setThreadgroupMemoryLength:mem_size atIndex:0];
1832
- [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 3)/4, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];
1833
- }
1834
- else if (src0t == GGML_TYPE_Q4_K) {
1835
- [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 3)/4, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];
1836
- }
1837
- else if (src0t == GGML_TYPE_Q3_K) {
1838
- [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 3)/4, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];
1839
- }
1840
- else if (src0t == GGML_TYPE_Q5_K) {
1841
- [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 3)/4, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];
1842
- }
1843
- else if (src0t == GGML_TYPE_Q6_K) {
1844
- [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 1)/2, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];
1845
- } else {
1846
- const int64_t ny = (ne11 + nrows - 1)/nrows;
1847
- [encoder dispatchThreadgroups:MTLSizeMake(ne01, ny, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];
1848
- }
1849
- }
1850
- } break;
1851
- case GGML_OP_MUL_MAT_ID:
1852
- {
1853
- const int n_as = src0->ne[2];
1854
-
1855
- // src2 = ids
1856
- const enum ggml_type src2t = src2->type; GGML_UNUSED(src2t);
1857
-
1858
- GGML_ASSERT(src2t == GGML_TYPE_I32);
1859
-
1860
- GGML_ASSERT(!ggml_is_transposed(src0));
1861
- GGML_ASSERT(!ggml_is_transposed(src1));
1862
-
1863
- GGML_ASSERT(src1t == GGML_TYPE_F32);
1864
-
1865
- // find the break-even point where the matrix-matrix kernel becomes more efficient compared
1866
- // to the matrix-vector kernel
1867
- // ne20 = n_used_experts
1868
- // ne21 = n_rows
1869
- const int dst_rows = ne20*ne21;
1870
- const int dst_rows_min = n_as;
1871
- const int dst_rows_max = (ctx->device.maxThreadgroupMemoryLength - 32 - 8192)/4;
1872
-
1873
- // max size of the rowids array in the kernel shared buffer
1874
- GGML_ASSERT(dst_rows <= dst_rows_max);
1875
-
1876
- // for now the matrix-matrix multiplication kernel only works on A14+/M1+ SoCs
1877
- // AMD GPU and older A-chips will reuse matrix-vector multiplication kernel
1878
- // !!!
1879
- // TODO: for now, always use mat-vec kernels until we figure out how to improve the
1880
- // indirect matrix multiplication
1881
- // !!!
1882
- if ([ctx->device supportsFamily:MTLGPUFamilyApple7] &&
1883
- ne00 % 32 == 0 && ne00 >= 64 &&
1884
- dst_rows > dst_rows_min) {
1885
-
1886
- // some Metal matrix data types require aligned pointers
1887
- // ref: https://developer.apple.com/metal/Metal-Shading-Language-Specification.pdf (Table 2.5)
1888
- switch (src0->type) {
1889
- case GGML_TYPE_F32: GGML_ASSERT(nb01 % 16 == 0); break;
1890
- case GGML_TYPE_F16: GGML_ASSERT(nb01 % 8 == 0); break;
1891
- default: break;
1892
- }
1893
-
1894
- id<MTLComputePipelineState> pipeline = nil;
1895
-
1896
- switch (src0->type) {
1897
- case GGML_TYPE_F32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_F32_F32 ].pipeline; break;
1898
- case GGML_TYPE_F16: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_F16_F32 ].pipeline; break;
1899
- case GGML_TYPE_Q4_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_0_F32 ].pipeline; break;
1900
- case GGML_TYPE_Q4_1: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_1_F32 ].pipeline; break;
1901
- case GGML_TYPE_Q5_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q5_0_F32 ].pipeline; break;
1902
- case GGML_TYPE_Q5_1: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q5_1_F32 ].pipeline; break;
1903
- case GGML_TYPE_Q8_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q8_0_F32 ].pipeline; break;
1904
- case GGML_TYPE_Q2_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q2_K_F32 ].pipeline; break;
1905
- case GGML_TYPE_Q3_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q3_K_F32 ].pipeline; break;
1906
- case GGML_TYPE_Q4_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q4_K_F32 ].pipeline; break;
1907
- case GGML_TYPE_Q5_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q5_K_F32 ].pipeline; break;
1908
- case GGML_TYPE_Q6_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_Q6_K_F32 ].pipeline; break;
1909
- case GGML_TYPE_IQ2_XXS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_XXS_F32].pipeline; break;
1910
- case GGML_TYPE_IQ2_XS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_XS_F32 ].pipeline; break;
1911
- case GGML_TYPE_IQ3_XXS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ3_XXS_F32].pipeline; break;
1912
- case GGML_TYPE_IQ3_S: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ3_S_F32 ].pipeline; break;
1913
- case GGML_TYPE_IQ2_S: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ2_S_F32 ].pipeline; break;
1914
- case GGML_TYPE_IQ1_S: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ1_S_F32 ].pipeline; break;
1915
- case GGML_TYPE_IQ1_M: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ1_M_F32 ].pipeline; break;
1916
- case GGML_TYPE_IQ4_NL: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ4_NL_F32 ].pipeline; break;
1917
- case GGML_TYPE_IQ4_XS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ4_XS_F32 ].pipeline; break;
1918
- default: GGML_ASSERT(false && "MUL_MAT_ID not implemented");
1919
- }
1920
-
1921
- [encoder setComputePipelineState:pipeline];
1922
- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
1923
- [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];
1924
- [encoder setBuffer:id_dst offset:offs_dst atIndex:2];
1925
- [encoder setBuffer:id_src2 offset:offs_src2 atIndex:3];
1926
- [encoder setBytes:&ne20 length:sizeof(ne20) atIndex:4];
1927
- [encoder setBytes:&ne21 length:sizeof(ne21) atIndex:5];
1928
- [encoder setBytes:&nb21 length:sizeof(nb21) atIndex:6];
1929
- [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:7];
1930
- [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:8];
1931
- [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:9];
1932
- [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:10];
1933
- [encoder setBytes:&ne11 length:sizeof(ne11) atIndex:11];
1934
- [encoder setBytes:&ne12 length:sizeof(ne12) atIndex:12];
1935
- [encoder setBytes:&ne13 length:sizeof(ne13) atIndex:13];
1936
- [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:14];
1937
- [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:15];
1938
- [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:16];
1939
- [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:17];
1940
- [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:18];
1941
- [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:19];
1942
-
1943
- [encoder setThreadgroupMemoryLength:GGML_PAD(8192 + dst_rows*4/*sizeof(ushort2)*/, 16) atIndex:0];
1944
-
1945
- [encoder dispatchThreadgroups:MTLSizeMake((ne21 + 31)/32, (ne01 + 63)/64, n_as) threadsPerThreadgroup:MTLSizeMake(128, 1, 1)];
1946
- } else {
1947
- int nth0 = 32;
1948
- int nth1 = 1;
1949
- int nrows = 1;
1950
- //printf("vector: ne00 = %6d, ne01 = %6d, ne02 = %6d, ne11 = %6d, ne12 = %6d\n", ne00, ne01, ne02, ne11, ne12);
1951
-
1952
- id<MTLComputePipelineState> pipeline = nil;
1953
-
1954
- // use custom matrix x vector kernel
1955
- switch (src0t) {
1956
- case GGML_TYPE_F32:
1957
- {
1958
- GGML_ASSERT(src1t == GGML_TYPE_F32);
1959
- pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F32_F32].pipeline;
1960
- } break;
1961
- case GGML_TYPE_F16:
1962
- {
1963
- GGML_ASSERT(src1t == GGML_TYPE_F32);
1964
- nth0 = 32;
1965
- nth1 = 1;
1966
- pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_F16_F32].pipeline;
1967
- } break;
1968
- case GGML_TYPE_Q4_0:
1969
- {
1970
- nth0 = 8;
1971
- nth1 = 8;
1972
- pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q4_0_F32].pipeline;
1973
- } break;
1974
- case GGML_TYPE_Q4_1:
1975
- {
1976
- nth0 = 8;
1977
- nth1 = 8;
1978
- pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q4_1_F32].pipeline;
1979
- } break;
1980
- case GGML_TYPE_Q5_0:
1981
- {
1982
- nth0 = 8;
1983
- nth1 = 8;
1984
- pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q5_0_F32].pipeline;
1985
- } break;
1986
- case GGML_TYPE_Q5_1:
1987
- {
1988
- nth0 = 8;
1989
- nth1 = 8;
1990
- pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q5_1_F32].pipeline;
1991
- } break;
1992
- case GGML_TYPE_Q8_0:
1993
- {
1994
- nth0 = 8;
1995
- nth1 = 8;
1996
- pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q8_0_F32].pipeline;
1997
- } break;
1998
- case GGML_TYPE_Q2_K:
1999
- {
2000
- nth0 = 2;
2001
- nth1 = 32;
2002
- pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q2_K_F32].pipeline;
2003
- } break;
2004
- case GGML_TYPE_Q3_K:
2005
- {
2006
- nth0 = 2;
2007
- nth1 = 32;
2008
- pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q3_K_F32].pipeline;
2009
- } break;
2010
- case GGML_TYPE_Q4_K:
2011
- {
2012
- nth0 = 4; //1;
2013
- nth1 = 8; //32;
2014
- pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q4_K_F32].pipeline;
2015
- } break;
2016
- case GGML_TYPE_Q5_K:
2017
- {
2018
- nth0 = 2;
2019
- nth1 = 32;
2020
- pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q5_K_F32].pipeline;
2021
- } break;
2022
- case GGML_TYPE_Q6_K:
2023
- {
2024
- nth0 = 2;
2025
- nth1 = 32;
2026
- pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_Q6_K_F32].pipeline;
2027
- } break;
2028
- case GGML_TYPE_IQ2_XXS:
2029
- {
2030
- nth0 = 4;
2031
- nth1 = 16;
2032
- pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_XXS_F32].pipeline;
2033
- } break;
2034
- case GGML_TYPE_IQ2_XS:
2035
- {
2036
- nth0 = 4;
2037
- nth1 = 16;
2038
- pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_XS_F32].pipeline;
2039
- } break;
2040
- case GGML_TYPE_IQ3_XXS:
2041
- {
2042
- nth0 = 4;
2043
- nth1 = 16;
2044
- pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ3_XXS_F32].pipeline;
2045
- } break;
2046
- case GGML_TYPE_IQ3_S:
2047
- {
2048
- nth0 = 4;
2049
- nth1 = 16;
2050
- pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ3_S_F32].pipeline;
2051
- } break;
2052
- case GGML_TYPE_IQ2_S:
2053
- {
2054
- nth0 = 4;
2055
- nth1 = 16;
2056
- pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ2_S_F32].pipeline;
2057
- } break;
2058
- case GGML_TYPE_IQ1_S:
2059
- {
2060
- nth0 = 4;
2061
- nth1 = 16;
2062
- pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ1_S_F32].pipeline;
2063
- } break;
2064
- case GGML_TYPE_IQ1_M:
2065
- {
2066
- nth0 = 4;
2067
- nth1 = 16;
2068
- pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ1_M_F32].pipeline;
2069
- } break;
2070
- case GGML_TYPE_IQ4_NL:
2071
- {
2072
- nth0 = 4;
2073
- nth1 = 16;
2074
- pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ4_NL_F32].pipeline;
2075
- } break;
2076
- case GGML_TYPE_IQ4_XS:
2077
- {
2078
- nth0 = 4;
2079
- nth1 = 16;
2080
- pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ4_XS_F32].pipeline;
2081
- } break;
2082
- default:
2083
- {
2084
- GGML_METAL_LOG_ERROR("Asserting on type %d\n", (int)src2t);
2085
- GGML_ASSERT(false && "not implemented");
2086
- }
2087
- };
2088
-
2089
- if (ggml_is_quantized(src0t)) {
2090
- GGML_ASSERT(ne00 >= nth0*nth1);
2091
- }
2092
-
2093
- [encoder setComputePipelineState:pipeline];
2094
- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
2095
- [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];
2096
- [encoder setBuffer:id_dst offset:offs_dst atIndex:2];
2097
- [encoder setBuffer:id_src2 offset:offs_src2 atIndex:3];
2098
- [encoder setBytes:&ne20 length:sizeof(ne20) atIndex:4];
2099
- [encoder setBytes:&ne21 length:sizeof(ne21) atIndex:5];
2100
- [encoder setBytes:&nb21 length:sizeof(nb21) atIndex:6];
2101
- [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:7];
2102
- [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:8];
2103
- [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:9];
2104
- [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:10];
2105
- [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:11];
2106
- [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:12];
2107
- [encoder setBytes:&ne10 length:sizeof(ne10) atIndex:13];
2108
- [encoder setBytes:&ne11 length:sizeof(ne11) atIndex:14];
2109
- [encoder setBytes:&ne12 length:sizeof(ne12) atIndex:15];
2110
- [encoder setBytes:&ne13 length:sizeof(ne13) atIndex:16];
2111
- [encoder setBytes:&nb10 length:sizeof(nb10) atIndex:17];
2112
- [encoder setBytes:&nb11 length:sizeof(nb11) atIndex:18];
2113
- [encoder setBytes:&nb12 length:sizeof(nb12) atIndex:19];
2114
- [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:20];
2115
- [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:21];
2116
- [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:22];
2117
-
2118
- const int64_t _ne1 = 1;
2119
- const int tgz = dst_rows;
2120
-
2121
- if (src0t == GGML_TYPE_Q4_0 || src0t == GGML_TYPE_Q4_1 || src0t == GGML_TYPE_Q5_0 ||
2122
- src0t == GGML_TYPE_Q5_1 || src0t == GGML_TYPE_Q8_0 || src0t == GGML_TYPE_Q2_K ||
2123
- src0t == GGML_TYPE_IQ1_S || src0t == GGML_TYPE_IQ1_M || src0t == GGML_TYPE_IQ2_S) {
2124
- [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 7)/8, _ne1, tgz) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];
2125
- }
2126
- else if (src0t == GGML_TYPE_IQ2_XXS || src0t == GGML_TYPE_IQ2_XS) {
2127
- const int mem_size = src0t == GGML_TYPE_IQ2_XXS ? 256*8+128 : 512*8+128;
2128
- [encoder setThreadgroupMemoryLength:mem_size atIndex:0];
2129
- [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 7)/8, _ne1, tgz) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];
2130
- }
2131
- else if (src0t == GGML_TYPE_IQ3_XXS || src0t == GGML_TYPE_IQ3_S) {
2132
- const int mem_size = src0t == GGML_TYPE_IQ3_XXS ? 256*4+128 : 512*4;
2133
- [encoder setThreadgroupMemoryLength:mem_size atIndex:0];
2134
- [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 7)/8, _ne1, tgz) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];
2135
- }
2136
- else if (src0t == GGML_TYPE_IQ4_NL || src0t == GGML_TYPE_IQ4_XS) {
2137
- const int mem_size = 32*sizeof(float);
2138
- [encoder setThreadgroupMemoryLength:mem_size atIndex:0];
2139
- [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 3)/4, _ne1, tgz) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];
2140
- }
2141
- else if (src0t == GGML_TYPE_Q4_K) {
2142
- [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 3)/4, _ne1, tgz) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];
2143
- }
2144
- else if (src0t == GGML_TYPE_Q3_K) {
2145
- [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 3)/4, _ne1, tgz) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];
2146
- }
2147
- else if (src0t == GGML_TYPE_Q5_K) {
2148
- [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 3)/4, _ne1, tgz) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];
2149
- }
2150
- else if (src0t == GGML_TYPE_Q6_K) {
2151
- [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 1)/2, _ne1, tgz) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];
2152
- } else {
2153
- const int64_t ny = (_ne1 + nrows - 1)/nrows; // = _ne1
2154
- [encoder dispatchThreadgroups:MTLSizeMake(ne01, ny, tgz) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)];
2155
- }
2156
- }
2157
- } break;
2158
- case GGML_OP_GET_ROWS:
2159
- {
2160
- id<MTLComputePipelineState> pipeline = nil;
2161
-
2162
- switch (src0->type) {
2163
- case GGML_TYPE_F32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_F32 ].pipeline; break;
2164
- case GGML_TYPE_F16: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_F16 ].pipeline; break;
2165
- case GGML_TYPE_Q4_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q4_0 ].pipeline; break;
2166
- case GGML_TYPE_Q4_1: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q4_1 ].pipeline; break;
2167
- case GGML_TYPE_Q5_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q5_0 ].pipeline; break;
2168
- case GGML_TYPE_Q5_1: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q5_1 ].pipeline; break;
2169
- case GGML_TYPE_Q8_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q8_0 ].pipeline; break;
2170
- case GGML_TYPE_Q2_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q2_K ].pipeline; break;
2171
- case GGML_TYPE_Q3_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q3_K ].pipeline; break;
2172
- case GGML_TYPE_Q4_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q4_K ].pipeline; break;
2173
- case GGML_TYPE_Q5_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q5_K ].pipeline; break;
2174
- case GGML_TYPE_Q6_K: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_Q6_K ].pipeline; break;
2175
- case GGML_TYPE_IQ2_XXS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_XXS].pipeline; break;
2176
- case GGML_TYPE_IQ2_XS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_XS ].pipeline; break;
2177
- case GGML_TYPE_IQ3_XXS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ3_XXS].pipeline; break;
2178
- case GGML_TYPE_IQ3_S: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ3_S ].pipeline; break;
2179
- case GGML_TYPE_IQ2_S: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ2_S ].pipeline; break;
2180
- case GGML_TYPE_IQ1_S: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ1_S ].pipeline; break;
2181
- case GGML_TYPE_IQ1_M: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ1_M ].pipeline; break;
2182
- case GGML_TYPE_IQ4_NL: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ4_NL ].pipeline; break;
2183
- case GGML_TYPE_IQ4_XS: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_IQ4_XS ].pipeline; break;
2184
- case GGML_TYPE_I32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GET_ROWS_I32 ].pipeline; break;
2185
- default: GGML_ASSERT(false && "not implemented");
2186
- }
2187
-
2188
- [encoder setComputePipelineState:pipeline];
2189
- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
2190
- [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];
2191
- [encoder setBuffer:id_dst offset:offs_dst atIndex:2];
2192
- [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:3];
2193
- [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:4];
2194
- [encoder setBytes:&nb02 length:sizeof(uint64_t) atIndex:5];
2195
- [encoder setBytes:&ne10 length:sizeof( int64_t) atIndex:6];
2196
- [encoder setBytes:&nb10 length:sizeof( int64_t) atIndex:7];
2197
- [encoder setBytes:&nb11 length:sizeof( int64_t) atIndex:8];
2198
- [encoder setBytes:&nb1 length:sizeof(uint64_t) atIndex:9];
2199
- [encoder setBytes:&nb2 length:sizeof(uint64_t) atIndex:10];
2200
-
2201
- [encoder dispatchThreadgroups:MTLSizeMake(ne10, ne11, 1) threadsPerThreadgroup:MTLSizeMake(32, 1, 1)];
2202
- } break;
2203
- case GGML_OP_RMS_NORM:
2204
- {
2205
- GGML_ASSERT(ne00 % 4 == 0);
2206
- GGML_ASSERT(ggml_is_contiguous_1(src0));
2207
-
2208
- float eps;
2209
- memcpy(&eps, dst->op_params, sizeof(float));
2210
-
2211
- int nth = 32; // SIMD width
2212
-
2213
- while (nth < ne00/4 && nth < 1024) {
2214
- nth *= 2;
2215
- }
2216
-
2217
- id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_RMS_NORM].pipeline;
2218
-
2219
- [encoder setComputePipelineState:pipeline];
2220
- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
2221
- [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
2222
- [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:2];
2223
- [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:3];
2224
- [encoder setBytes:&eps length:sizeof( float) atIndex:4];
2225
- [encoder setThreadgroupMemoryLength:32*sizeof(float) atIndex:0];
2226
-
2227
- const int64_t nrows = ggml_nrows(src0);
2228
-
2229
- [encoder dispatchThreadgroups:MTLSizeMake(nrows, 1, 1) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];
2230
- } break;
2231
- case GGML_OP_GROUP_NORM:
2232
- {
2233
- GGML_ASSERT(ne00 % 4 == 0);
2234
- GGML_ASSERT(ggml_is_contiguous(src0));
2235
-
2236
- //float eps;
2237
- //memcpy(&eps, dst->op_params, sizeof(float));
2238
-
2239
- const float eps = 1e-6f; // TODO: temporarily hardcoded
2240
-
2241
- const int32_t n_groups = ((int32_t *) dst->op_params)[0];
2242
-
2243
- int nth = 32; // SIMD width
2244
-
2245
- //while (nth < ne00/4 && nth < 1024) {
2246
- // nth *= 2;
2247
- //}
2248
-
2249
- id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_GROUP_NORM].pipeline;
2250
-
2251
- [encoder setComputePipelineState:pipeline];
2252
- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
2253
- [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
2254
- [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:2];
2255
- [encoder setBytes:&ne01 length:sizeof( int64_t) atIndex:3];
2256
- [encoder setBytes:&ne02 length:sizeof( int64_t) atIndex:4];
2257
- [encoder setBytes:&nb00 length:sizeof(uint64_t) atIndex:5];
2258
- [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:6];
2259
- [encoder setBytes:&nb02 length:sizeof(uint64_t) atIndex:7];
2260
- [encoder setBytes:&n_groups length:sizeof( int32_t) atIndex:8];
2261
- [encoder setBytes:&eps length:sizeof( float) atIndex:9];
2262
- [encoder setThreadgroupMemoryLength:32*sizeof(float) atIndex:0];
2263
-
2264
- [encoder dispatchThreadgroups:MTLSizeMake(n_groups, 1, 1) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];
2265
- } break;
2266
- case GGML_OP_NORM:
2267
- {
2268
- GGML_ASSERT(ggml_is_contiguous_1(src0));
2269
-
2270
- float eps;
2271
- memcpy(&eps, dst->op_params, sizeof(float));
2272
-
2273
- const int nth = MIN(256, ne00);
2274
-
2275
- id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_NORM].pipeline;
2276
-
2277
- [encoder setComputePipelineState:pipeline];
2278
- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
2279
- [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
2280
- [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:2];
2281
- [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:3];
2282
- [encoder setBytes:&eps length:sizeof( float) atIndex:4];
2283
- [encoder setThreadgroupMemoryLength:GGML_PAD(nth*sizeof(float), 16) atIndex:0];
2284
-
2285
- const int64_t nrows = ggml_nrows(src0);
2286
-
2287
- [encoder dispatchThreadgroups:MTLSizeMake(nrows, 1, 1) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];
2288
- } break;
2289
- case GGML_OP_ROPE:
2290
- {
2291
- GGML_ASSERT(ne10 == ne02);
2292
-
2293
- const int nth = MIN(1024, ne00);
2294
-
2295
- const int n_past = ((int32_t *) dst->op_params)[0];
2296
- const int n_dims = ((int32_t *) dst->op_params)[1];
2297
- const int mode = ((int32_t *) dst->op_params)[2];
2298
- // skip 3, n_ctx, used in GLM RoPE, unimplemented in metal
2299
- const int n_ctx_orig = ((int32_t *) dst->op_params)[4];
2300
-
2301
- float freq_base;
2302
- float freq_scale;
2303
- float ext_factor;
2304
- float attn_factor;
2305
- float beta_fast;
2306
- float beta_slow;
2307
-
2308
- memcpy(&freq_base, (int32_t *) dst->op_params + 5, sizeof(float));
2309
- memcpy(&freq_scale, (int32_t *) dst->op_params + 6, sizeof(float));
2310
- memcpy(&ext_factor, (int32_t *) dst->op_params + 7, sizeof(float));
2311
- memcpy(&attn_factor, (int32_t *) dst->op_params + 8, sizeof(float));
2312
- memcpy(&beta_fast, (int32_t *) dst->op_params + 9, sizeof(float));
2313
- memcpy(&beta_slow, (int32_t *) dst->op_params + 10, sizeof(float));
2314
-
2315
- const bool is_neox = mode & 2;
2316
-
2317
- id<MTLComputePipelineState> pipeline = nil;
2318
-
2319
- if (!is_neox) {
2320
- switch (src0->type) {
2321
- case GGML_TYPE_F32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_ROPE_NORM_F32].pipeline; break;
2322
- case GGML_TYPE_F16: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_ROPE_NORM_F16].pipeline; break;
2323
- default: GGML_ASSERT(false);
2324
- };
2325
- } else {
2326
- switch (src0->type) {
2327
- case GGML_TYPE_F32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_ROPE_NEOX_F32].pipeline; break;
2328
- case GGML_TYPE_F16: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_ROPE_NEOX_F16].pipeline; break;
2329
- default: GGML_ASSERT(false);
2330
- };
2331
- }
2332
-
2333
- [encoder setComputePipelineState:pipeline];
2334
- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
2335
- [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];
2336
- if (id_src2 != nil) {
2337
- [encoder setBuffer:id_src2 offset:offs_src2 atIndex:2];
2338
- } else {
2339
- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:2];
2340
- }
2341
- [encoder setBuffer:id_dst offset:offs_dst atIndex:3];
2342
- [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:4];
2343
- [encoder setBytes:&ne01 length:sizeof( int64_t) atIndex:5];
2344
- [encoder setBytes:&ne02 length:sizeof( int64_t) atIndex:6];
2345
- [encoder setBytes:&ne03 length:sizeof( int64_t) atIndex:7];
2346
- [encoder setBytes:&nb00 length:sizeof(uint64_t) atIndex:8];
2347
- [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:9];
2348
- [encoder setBytes:&nb02 length:sizeof(uint64_t) atIndex:10];
2349
- [encoder setBytes:&nb03 length:sizeof(uint64_t) atIndex:11];
2350
- [encoder setBytes:&ne0 length:sizeof( int64_t) atIndex:12];
2351
- [encoder setBytes:&ne1 length:sizeof( int64_t) atIndex:13];
2352
- [encoder setBytes:&ne2 length:sizeof( int64_t) atIndex:14];
2353
- [encoder setBytes:&ne3 length:sizeof( int64_t) atIndex:15];
2354
- [encoder setBytes:&nb0 length:sizeof(uint64_t) atIndex:16];
2355
- [encoder setBytes:&nb1 length:sizeof(uint64_t) atIndex:17];
2356
- [encoder setBytes:&nb2 length:sizeof(uint64_t) atIndex:18];
2357
- [encoder setBytes:&nb3 length:sizeof(uint64_t) atIndex:19];
2358
- [encoder setBytes:&n_past length:sizeof( int) atIndex:20];
2359
- [encoder setBytes:&n_dims length:sizeof( int) atIndex:21];
2360
- [encoder setBytes:&n_ctx_orig length:sizeof( int) atIndex:22];
2361
- [encoder setBytes:&freq_base length:sizeof( float) atIndex:23];
2362
- [encoder setBytes:&freq_scale length:sizeof( float) atIndex:24];
2363
- [encoder setBytes:&ext_factor length:sizeof( float) atIndex:25];
2364
- [encoder setBytes:&attn_factor length:sizeof( float) atIndex:26];
2365
- [encoder setBytes:&beta_fast length:sizeof( float) atIndex:27];
2366
- [encoder setBytes:&beta_slow length:sizeof( float) atIndex:28];
2367
-
2368
- [encoder dispatchThreadgroups:MTLSizeMake(ne01, ne02, ne03) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];
2369
- } break;
2370
- case GGML_OP_IM2COL:
2371
- {
2372
- GGML_ASSERT(src0->type == GGML_TYPE_F16);
2373
- GGML_ASSERT(src1->type == GGML_TYPE_F32);
2374
- GGML_ASSERT( dst->type == GGML_TYPE_F16 || dst->type == GGML_TYPE_F32);
2375
-
2376
- const int32_t s0 = ((const int32_t *)(dst->op_params))[0];
2377
- const int32_t s1 = ((const int32_t *)(dst->op_params))[1];
2378
- const int32_t p0 = ((const int32_t *)(dst->op_params))[2];
2379
- const int32_t p1 = ((const int32_t *)(dst->op_params))[3];
2380
- const int32_t d0 = ((const int32_t *)(dst->op_params))[4];
2381
- const int32_t d1 = ((const int32_t *)(dst->op_params))[5];
2382
-
2383
- const bool is_2D = ((const int32_t *)(dst->op_params))[6] == 1;
2384
-
2385
- const int32_t N = src1->ne[is_2D ? 3 : 2];
2386
- const int32_t IC = src1->ne[is_2D ? 2 : 1];
2387
- const int32_t IH = is_2D ? src1->ne[1] : 1;
2388
- const int32_t IW = src1->ne[0];
2389
-
2390
- const int32_t KH = is_2D ? src0->ne[1] : 1;
2391
- const int32_t KW = src0->ne[0];
2392
-
2393
- const int32_t OH = is_2D ? dst->ne[2] : 1;
2394
- const int32_t OW = dst->ne[1];
2395
-
2396
- const int32_t CHW = IC * KH * KW;
2397
-
2398
- const int32_t ofs0 = src1->nb[is_2D ? 3 : 2] / 4;
2399
- const int32_t ofs1 = src1->nb[is_2D ? 2 : 1] / 4;
2400
-
2401
- id<MTLComputePipelineState> pipeline = nil;
2402
-
2403
- switch (dst->type) {
2404
- case GGML_TYPE_F32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_IM2COL_F32].pipeline; break;
2405
- case GGML_TYPE_F16: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_IM2COL_F16].pipeline; break;
2406
- default: GGML_ASSERT(false);
2407
- };
2408
-
2409
- [encoder setComputePipelineState:pipeline];
2410
- [encoder setBuffer:id_src1 offset:offs_src1 atIndex:0];
2411
- [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
2412
- [encoder setBytes:&ofs0 length:sizeof( int32_t) atIndex:2];
2413
- [encoder setBytes:&ofs1 length:sizeof( int32_t) atIndex:3];
2414
- [encoder setBytes:&IW length:sizeof( int32_t) atIndex:4];
2415
- [encoder setBytes:&IH length:sizeof( int32_t) atIndex:5];
2416
- [encoder setBytes:&CHW length:sizeof( int32_t) atIndex:6];
2417
- [encoder setBytes:&s0 length:sizeof( int32_t) atIndex:7];
2418
- [encoder setBytes:&s1 length:sizeof( int32_t) atIndex:8];
2419
- [encoder setBytes:&p0 length:sizeof( int32_t) atIndex:9];
2420
- [encoder setBytes:&p1 length:sizeof( int32_t) atIndex:10];
2421
- [encoder setBytes:&d0 length:sizeof( int32_t) atIndex:11];
2422
- [encoder setBytes:&d1 length:sizeof( int32_t) atIndex:12];
2423
-
2424
- [encoder dispatchThreadgroups:MTLSizeMake(IC, OH, OW) threadsPerThreadgroup:MTLSizeMake(N, KH, KW)];
2425
- } break;
2426
- case GGML_OP_UPSCALE:
2427
- {
2428
- GGML_ASSERT(src0->type == GGML_TYPE_F32);
2429
-
2430
- const float sf0 = (float)ne0/src0->ne[0];
2431
- const float sf1 = (float)ne1/src0->ne[1];
2432
- const float sf2 = (float)ne2/src0->ne[2];
2433
- const float sf3 = (float)ne3/src0->ne[3];
2434
-
2435
- const id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_UPSCALE_F32].pipeline;
2436
-
2437
- [encoder setComputePipelineState:pipeline];
2438
- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
2439
- [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
2440
- [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:2];
2441
- [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:3];
2442
- [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:4];
2443
- [encoder setBytes:&ne03 length:sizeof(ne03) atIndex:5];
2444
- [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:6];
2445
- [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:7];
2446
- [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:8];
2447
- [encoder setBytes:&nb03 length:sizeof(nb03) atIndex:9];
2448
- [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:10];
2449
- [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:11];
2450
- [encoder setBytes:&ne2 length:sizeof(ne2) atIndex:12];
2451
- [encoder setBytes:&ne3 length:sizeof(ne3) atIndex:13];
2452
- [encoder setBytes:&nb0 length:sizeof(nb0) atIndex:14];
2453
- [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:15];
2454
- [encoder setBytes:&nb2 length:sizeof(nb2) atIndex:16];
2455
- [encoder setBytes:&nb3 length:sizeof(nb3) atIndex:17];
2456
- [encoder setBytes:&sf0 length:sizeof(sf0) atIndex:18];
2457
- [encoder setBytes:&sf1 length:sizeof(sf1) atIndex:19];
2458
- [encoder setBytes:&sf2 length:sizeof(sf2) atIndex:20];
2459
- [encoder setBytes:&sf3 length:sizeof(sf3) atIndex:21];
2460
-
2461
- const int nth = MIN((int) pipeline.maxTotalThreadsPerThreadgroup, ne0);
2462
-
2463
- [encoder dispatchThreadgroups:MTLSizeMake(ne1, ne2, ne3) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];
2464
- } break;
2465
- case GGML_OP_PAD:
2466
- {
2467
- GGML_ASSERT(src0->type == GGML_TYPE_F32);
2468
-
2469
- id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_PAD_F32].pipeline;
2470
-
2471
- [encoder setComputePipelineState:pipeline];
2472
- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
2473
- [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
2474
- [encoder setBytes:&ne00 length:sizeof(ne00) atIndex:2];
2475
- [encoder setBytes:&ne01 length:sizeof(ne01) atIndex:3];
2476
- [encoder setBytes:&ne02 length:sizeof(ne02) atIndex:4];
2477
- [encoder setBytes:&ne03 length:sizeof(ne03) atIndex:5];
2478
- [encoder setBytes:&nb00 length:sizeof(nb00) atIndex:6];
2479
- [encoder setBytes:&nb01 length:sizeof(nb01) atIndex:7];
2480
- [encoder setBytes:&nb02 length:sizeof(nb02) atIndex:8];
2481
- [encoder setBytes:&nb03 length:sizeof(nb03) atIndex:9];
2482
- [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:10];
2483
- [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:11];
2484
- [encoder setBytes:&ne2 length:sizeof(ne2) atIndex:12];
2485
- [encoder setBytes:&ne3 length:sizeof(ne3) atIndex:13];
2486
- [encoder setBytes:&nb0 length:sizeof(nb0) atIndex:14];
2487
- [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:15];
2488
- [encoder setBytes:&nb2 length:sizeof(nb2) atIndex:16];
2489
- [encoder setBytes:&nb3 length:sizeof(nb3) atIndex:17];
2490
-
2491
- const int nth = MIN(1024, ne0);
2492
-
2493
- [encoder dispatchThreadgroups:MTLSizeMake(ne1, ne2, ne3) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];
2494
- } break;
2495
- case GGML_OP_ARANGE:
2496
- {
2497
- GGML_ASSERT(dst->type == GGML_TYPE_F32);
2498
-
2499
- float start;
2500
- float step;
2501
-
2502
- memcpy(&start, ((int32_t *) dst->op_params) + 0, sizeof(float));
2503
- memcpy(&step, ((int32_t *) dst->op_params) + 2, sizeof(float));
2504
-
2505
- id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_ARANGE_F32].pipeline;
2506
-
2507
- [encoder setComputePipelineState:pipeline];
2508
- [encoder setBuffer:id_dst offset:offs_dst atIndex:0];
2509
- [encoder setBytes:&ne0 length:sizeof(ne0) atIndex:1];
2510
- [encoder setBytes:&start length:sizeof(start) atIndex:2];
2511
- [encoder setBytes:&step length:sizeof(step) atIndex:3];
2512
-
2513
- const int nth = MIN(1024, ne0);
2514
-
2515
- [encoder dispatchThreadgroups:MTLSizeMake(1, 1, 1) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];
2516
- } break;
2517
- case GGML_OP_TIMESTEP_EMBEDDING:
2518
- {
2519
- GGML_ASSERT(src0->type == GGML_TYPE_F32);
2520
-
2521
- const int dim = dst->op_params[0];
2522
- const int max_period = dst->op_params[1];
2523
-
2524
- const int half = dim / 2;
2525
-
2526
- id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_TIMESTEP_EMBEDDING_F32].pipeline;
2527
-
2528
- [encoder setComputePipelineState:pipeline];
2529
- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
2530
- [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
2531
- [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:2];
2532
- [encoder setBytes:&dim length:sizeof(dim) atIndex:3];
2533
- [encoder setBytes:&max_period length:sizeof(max_period) atIndex:4];
2534
-
2535
- const int nth = MIN(1024, half);
2536
-
2537
- [encoder dispatchThreadgroups:MTLSizeMake(ne00, 1, 1) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];
2538
- } break;
2539
- case GGML_OP_ARGSORT:
2540
- {
2541
- GGML_ASSERT(src0->type == GGML_TYPE_F32);
2542
- GGML_ASSERT( dst->type == GGML_TYPE_I32);
2543
-
2544
- const int nrows = ggml_nrows(src0);
2545
-
2546
- enum ggml_sort_order order = (enum ggml_sort_order) dst->op_params[0];
2547
-
2548
- // bitonic sort requires the number of elements to be power of 2
2549
- int64_t ne00_padded = 1;
2550
- while (ne00_padded < ne00) {
2551
- ne00_padded *= 2;
2552
- }
2553
-
2554
- // Metal kernels require the buffer size to be multiple of 16 bytes
2555
- // https://developer.apple.com/documentation/metal/mtlcomputecommandencoder/1443142-setthreadgroupmemorylength
2556
- const int mem_size = GGML_PAD(ne00_padded*sizeof(int32_t), 16);
2557
-
2558
- id<MTLComputePipelineState> pipeline = nil;
2559
-
2560
- switch (order) {
2561
- case GGML_SORT_ORDER_ASC: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_ARGSORT_F32_I32_ASC].pipeline; break;
2562
- case GGML_SORT_ORDER_DESC: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_ARGSORT_F32_I32_DESC].pipeline; break;
2563
- default: GGML_ASSERT(false);
2564
- };
2565
-
2566
- [encoder setComputePipelineState:pipeline];
2567
- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
2568
- [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
2569
- [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:2];
2570
- [encoder setBytes:&ne00_padded length:sizeof( int64_t) atIndex:3];
2571
- [encoder setThreadgroupMemoryLength:mem_size atIndex:0];
2572
-
2573
- [encoder dispatchThreadgroups:MTLSizeMake(1, nrows, 1) threadsPerThreadgroup:MTLSizeMake(ne00_padded, 1, 1)];
2574
- } break;
2575
- case GGML_OP_LEAKY_RELU:
2576
- {
2577
- GGML_ASSERT(src0->type == GGML_TYPE_F32);
2578
-
2579
- float slope;
2580
- memcpy(&slope, dst->op_params, sizeof(float));
2581
-
2582
- id<MTLComputePipelineState> pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_LEAKY_RELU_F32].pipeline;
2583
-
2584
- [encoder setComputePipelineState:pipeline];
2585
- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
2586
- [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
2587
- [encoder setBytes:&slope length:sizeof(slope) atIndex:2];
2588
-
2589
- const int64_t n = ggml_nelements(dst);
2590
-
2591
- [encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
2592
- } break;
2593
- case GGML_OP_FLASH_ATTN_EXT:
2594
- {
2595
- GGML_ASSERT(ne00 % 4 == 0);
2596
- GGML_ASSERT(ne11 % 32 == 0);
2597
-
2598
- GGML_ASSERT(src0->type == GGML_TYPE_F32);
2599
-
2600
- GGML_ASSERT(ggml_are_same_shape (src1, src2));
2601
-
2602
- struct ggml_tensor * src3 = gf->nodes[i]->src[3];
2603
-
2604
- size_t offs_src3 = 0;
2605
-
2606
- id<MTLBuffer> id_src3 = src3 ? ggml_metal_get_buffer(src3, &offs_src3) : nil;
2607
-
2608
- GGML_ASSERT(!src3 || src3->type == GGML_TYPE_F16);
2609
- GGML_ASSERT(!src3 || src3->ne[1] >= GGML_PAD(src0->ne[1], 8) &&
2610
- "the Flash-Attention Metal kernel requires the mask to be padded to 8 and at least n_queries big");
2611
-
2612
- const int64_t ne30 = src3 ? src3->ne[0] : 0; GGML_UNUSED(ne30);
2613
- //const int64_t ne31 = src3 ? src3->ne[1] : 0;
2614
- const int64_t ne32 = src3 ? src3->ne[2] : 0; GGML_UNUSED(ne32);
2615
- const int64_t ne33 = src3 ? src3->ne[3] : 0; GGML_UNUSED(ne33);
2616
-
2617
- const uint64_t nb30 = src3 ? src3->nb[0] : 0; GGML_UNUSED(nb30);
2618
- const uint64_t nb31 = src3 ? src3->nb[1] : 0;
2619
- const uint64_t nb32 = src3 ? src3->nb[2] : 0; GGML_UNUSED(nb32);
2620
- const uint64_t nb33 = src3 ? src3->nb[3] : 0; GGML_UNUSED(nb33);
2621
-
2622
- const enum ggml_type src2t = src2 ? src2->type : GGML_TYPE_COUNT; GGML_UNUSED(src2t);
2623
-
2624
- float scale;
2625
- float max_bias;
2626
-
2627
- memcpy(&scale, ((int32_t *) dst->op_params) + 0, sizeof(scale));
2628
- memcpy(&max_bias, ((int32_t *) dst->op_params) + 1, sizeof(max_bias));
2629
-
2630
- const uint32_t n_head = src0->ne[2];
2631
- const uint32_t n_head_log2 = 1u << (uint32_t) floorf(log2f((float) n_head));
2632
-
2633
- const float m0 = powf(2.0f, -(max_bias ) / n_head_log2);
2634
- const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2);
2635
-
2636
- id<MTLComputePipelineState> pipeline = nil;
2637
-
2638
- bool use_vec_kernel = false;
2639
-
2640
- if (ne01 >= 4 || (ne00%128 != 0)) {
2641
- switch (ne00) {
2642
- case 64: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H64 ].pipeline; break;
2643
- case 80: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H80 ].pipeline; break;
2644
- case 96: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H96 ].pipeline; break;
2645
- case 112: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H112].pipeline; break;
2646
- case 128: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H128].pipeline; break;
2647
- //case 256: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_F16_H256].pipeline; break;
2648
- default:
2649
- {
2650
- GGML_METAL_LOG_ERROR("unsupported size: %lld\n", ne00);
2651
- GGML_METAL_LOG_ERROR("add template specialization for this size\n");
2652
- GGML_ASSERT(false && "add template specialization for this size");
2653
- }
2654
- }
2655
- } else {
2656
- use_vec_kernel = true;
2657
-
2658
- switch (ne00) {
2659
- case 128: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_VEC_F16_H128].pipeline; break;
2660
- //case 256: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_FLASH_ATTN_EXT_VEC_F16_H256].pipeline; break;
2661
- default:
2662
- {
2663
- GGML_METAL_LOG_ERROR("unsupported size: %lld\n", ne00);
2664
- GGML_METAL_LOG_ERROR("add template specialization for this size\n");
2665
- GGML_ASSERT(false && "add template specialization for this size");
2666
- }
2667
- }
2668
- }
2669
-
2670
- [encoder setComputePipelineState:pipeline];
2671
- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
2672
- [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];
2673
- [encoder setBuffer:id_src2 offset:offs_src2 atIndex:2];
2674
- if (id_src3) {
2675
- [encoder setBuffer:id_src3 offset:offs_src3 atIndex:3];
2676
- } else {
2677
- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:3];
2678
- }
2679
- [encoder setBuffer:id_dst offset:offs_dst atIndex:4];
2680
- [encoder setBytes:&ne01 length:sizeof( int64_t) atIndex:5];
2681
- [encoder setBytes:&ne02 length:sizeof( int64_t) atIndex:6];
2682
- [encoder setBytes:&ne03 length:sizeof( int64_t) atIndex:7];
2683
- [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:8];
2684
- [encoder setBytes:&nb02 length:sizeof(uint64_t) atIndex:9];
2685
- [encoder setBytes:&nb03 length:sizeof(uint64_t) atIndex:10];
2686
- [encoder setBytes:&ne11 length:sizeof( int64_t) atIndex:11];
2687
- [encoder setBytes:&ne12 length:sizeof( int64_t) atIndex:12];
2688
- [encoder setBytes:&ne13 length:sizeof( int64_t) atIndex:13];
2689
- [encoder setBytes:&nb11 length:sizeof(uint64_t) atIndex:14];
2690
- [encoder setBytes:&nb12 length:sizeof(uint64_t) atIndex:15];
2691
- [encoder setBytes:&nb13 length:sizeof(uint64_t) atIndex:16];
2692
- [encoder setBytes:&nb21 length:sizeof(uint64_t) atIndex:17];
2693
- [encoder setBytes:&nb22 length:sizeof(uint64_t) atIndex:18];
2694
- [encoder setBytes:&nb23 length:sizeof(uint64_t) atIndex:19];
2695
- [encoder setBytes:&nb31 length:sizeof(uint64_t) atIndex:20];
2696
- [encoder setBytes:&ne1 length:sizeof( int64_t) atIndex:21];
2697
- [encoder setBytes:&ne2 length:sizeof( int64_t) atIndex:22];
2698
- [encoder setBytes:&scale length:sizeof( float) atIndex:23];
2699
- [encoder setBytes:&max_bias length:sizeof( float) atIndex:24];
2700
- [encoder setBytes:&m0 length:sizeof(m0) atIndex:25];
2701
- [encoder setBytes:&m1 length:sizeof(m1) atIndex:26];
2702
- [encoder setBytes:&n_head_log2 length:sizeof(n_head_log2) atIndex:27];
2703
-
2704
- if (!use_vec_kernel) {
2705
- // half8x8 kernel
2706
- const int64_t nqptg = 8; // queries per threadgroup !! sync with kernel template arguments !!
2707
- const int64_t ncpsg = 32; // cache values per simdgroup !! sync with kernel template arguments !!
2708
-
2709
- GGML_ASSERT(nqptg <= 32);
2710
- GGML_ASSERT(nqptg % 8 == 0);
2711
- GGML_ASSERT(ncpsg % 32 == 0);
2712
-
2713
- int64_t nsgmax = 2;
2714
-
2715
- while (true) {
2716
- const size_t smem = nqptg*(ne00 + 2*nsgmax*(ncpsg + nqptg))*(sizeof(float)/2);
2717
- if (smem > ctx->device.maxThreadgroupMemoryLength) {
2718
- break;
2719
- }
2720
- nsgmax *= 2;
2721
- }
2722
- nsgmax /= 2;
2723
-
2724
- // simdgroups per threadgroup (a.k.a. warps)
2725
- const int64_t nsg = ne01 <= nqptg ? MAX(4, MIN(nsgmax, MIN(ne11/ncpsg, (int64_t) pipeline.maxTotalThreadsPerThreadgroup/32))) : 4;
2726
-
2727
- const size_t smem = nqptg*(ne00 + 2*nsg*(ncpsg + nqptg))*(sizeof(float)/2);
2728
-
2729
- //printf("smem: %zu, max: %zu\n", smem, ctx->device.maxThreadgroupMemoryLength);
2730
- GGML_ASSERT(smem <= ctx->device.maxThreadgroupMemoryLength);
2731
-
2732
- [encoder setThreadgroupMemoryLength:GGML_PAD(smem, 16) atIndex:0];
2733
-
2734
- [encoder dispatchThreadgroups:MTLSizeMake((ne01 + nqptg - 1)/nqptg, ne02, ne03) threadsPerThreadgroup:MTLSizeMake(32, nsg, 1)];
2735
- } else {
2736
- // half1x4 kernel
2737
- const int64_t nqptg = 1; // queries per threadgroup !! sync with kernel template arguments !!
2738
- const int64_t ncpsg = 32; // cache values per simdgroup !! sync with kernel template arguments !!
2739
-
2740
- GGML_ASSERT(nqptg <= 32);
2741
- GGML_ASSERT(nqptg % 1 == 0);
2742
- GGML_ASSERT(ncpsg % 32 == 0);
2743
-
2744
- // simdgroups per threadgroup (a.k.a. warps)
2745
- const int64_t nsgt = MAX(2, MIN(ne11/ncpsg, (int64_t) pipeline.maxTotalThreadsPerThreadgroup/32));
2746
-
2747
- int64_t nsg = 1;
2748
- while (nsg <= nsgt) {
2749
- nsg *= 2;
2750
- }
2751
- nsg /= 2;
2752
-
2753
- const size_t smem = (nqptg*(ne00 + 2*nsg*(ncpsg + nqptg)) + nsg*ne00)*(sizeof(float)/2);
2754
-
2755
- //printf("smem: %zu, max: %zu\n", smem, ctx->device.maxThreadgroupMemoryLength);
2756
- GGML_ASSERT(smem <= ctx->device.maxThreadgroupMemoryLength);
2757
- [encoder setThreadgroupMemoryLength:GGML_PAD(smem, 16) atIndex:0];
2758
-
2759
- [encoder dispatchThreadgroups:MTLSizeMake((ne01 + nqptg - 1)/nqptg, ne02, ne03) threadsPerThreadgroup:MTLSizeMake(32, nsg, 1)];
2760
- }
2761
- } break;
2762
- case GGML_OP_DUP:
2763
- case GGML_OP_CPY:
2764
- case GGML_OP_CONT:
2765
- {
2766
- GGML_ASSERT(ne00 % ggml_blck_size(src0->type) == 0);
2767
-
2768
- int nth = MIN(1024, ne00/ggml_blck_size(src0->type));
2769
-
2770
- id<MTLComputePipelineState> pipeline = nil;
2771
-
2772
- switch (src0t) {
2773
- case GGML_TYPE_F32:
2774
- {
2775
- GGML_ASSERT(ne0 % ggml_blck_size(dst->type) == 0);
2776
-
2777
- switch (dstt) {
2778
- case GGML_TYPE_F16: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_F16].pipeline; break;
2779
- case GGML_TYPE_F32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_F32].pipeline; break;
2780
- case GGML_TYPE_Q8_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_Q8_0].pipeline; break;
2781
- case GGML_TYPE_Q4_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_Q4_0].pipeline; break;
2782
- case GGML_TYPE_Q4_1: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_Q4_1].pipeline; break;
2783
- case GGML_TYPE_Q5_0: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_0].pipeline; break;
2784
- case GGML_TYPE_Q5_1: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_Q5_1].pipeline; break;
2785
- case GGML_TYPE_IQ4_NL: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F32_IQ4_NL].pipeline; break;
2786
- default: GGML_ASSERT(false && "not implemented");
2787
- };
2788
- } break;
2789
- case GGML_TYPE_F16:
2790
- {
2791
- switch (dstt) {
2792
- case GGML_TYPE_F16: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F16_F16].pipeline; break;
2793
- case GGML_TYPE_F32: pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_CPY_F16_F32].pipeline; break;
2794
- default: GGML_ASSERT(false && "not implemented");
2795
- };
2796
- } break;
2797
- default: GGML_ASSERT(false && "not implemented");
2798
- }
2799
-
2800
- [encoder setComputePipelineState:pipeline];
2801
- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];
2802
- [encoder setBuffer:id_dst offset:offs_dst atIndex:1];
2803
- [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:2];
2804
- [encoder setBytes:&ne01 length:sizeof( int64_t) atIndex:3];
2805
- [encoder setBytes:&ne02 length:sizeof( int64_t) atIndex:4];
2806
- [encoder setBytes:&ne03 length:sizeof( int64_t) atIndex:5];
2807
- [encoder setBytes:&nb00 length:sizeof(uint64_t) atIndex:6];
2808
- [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:7];
2809
- [encoder setBytes:&nb02 length:sizeof(uint64_t) atIndex:8];
2810
- [encoder setBytes:&nb03 length:sizeof(uint64_t) atIndex:9];
2811
- [encoder setBytes:&ne0 length:sizeof( int64_t) atIndex:10];
2812
- [encoder setBytes:&ne1 length:sizeof( int64_t) atIndex:11];
2813
- [encoder setBytes:&ne2 length:sizeof( int64_t) atIndex:12];
2814
- [encoder setBytes:&ne3 length:sizeof( int64_t) atIndex:13];
2815
- [encoder setBytes:&nb0 length:sizeof(uint64_t) atIndex:14];
2816
- [encoder setBytes:&nb1 length:sizeof(uint64_t) atIndex:15];
2817
- [encoder setBytes:&nb2 length:sizeof(uint64_t) atIndex:16];
2818
- [encoder setBytes:&nb3 length:sizeof(uint64_t) atIndex:17];
2819
-
2820
- [encoder dispatchThreadgroups:MTLSizeMake(ne01, ne02, ne03) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];
2821
- } break;
2822
- default:
2823
- {
2824
- GGML_METAL_LOG_ERROR("%s: error: node %3d, op = %8s not implemented\n", __func__, i, ggml_op_name(dst->op));
2825
- GGML_ASSERT(false);
2826
- }
2827
- }
2828
-
2829
- if (should_capture) {
2830
- [encoder popDebugGroup];
2831
- }
2832
- }
2833
-
2834
- [encoder endEncoding];
2835
-
2836
- [command_buffer commit];
2837
- });
2838
-
2839
- // Wait for completion and check status of each command buffer
2840
- // needed to detect if the device ran out-of-memory for example (#1881)
2841
-
2842
- for (int i = 0; i < n_cb; ++i) {
2843
- id<MTLCommandBuffer> command_buffer = command_buffers[i];
2844
- [command_buffer waitUntilCompleted];
2845
-
2846
- MTLCommandBufferStatus status = [command_buffer status];
2847
- if (status != MTLCommandBufferStatusCompleted) {
2848
- GGML_METAL_LOG_INFO("%s: command buffer %d failed with status %lu\n", __func__, i, status);
2849
- if (status == MTLCommandBufferStatusError) {
2850
- NSString * error_code = [command_buffer error].localizedDescription;
2851
- GGML_METAL_LOG_INFO("error: %s\n", [error_code UTF8String]);
2852
- }
2853
-
2854
- return GGML_STATUS_FAILED;
2855
- }
2856
- }
2857
-
2858
- if (should_capture) {
2859
- [[MTLCaptureManager sharedCaptureManager] stopCapture];
2860
- }
2861
-
2862
- }
2863
- return GGML_STATUS_SUCCESS;
2864
- }
2865
-
2866
- ////////////////////////////////////////////////////////////////////////////////
2867
-
2868
- // backend interface
2869
-
2870
- // default buffer
2871
- static id<MTLDevice> g_backend_device = nil;
2872
- static int g_backend_device_ref_count = 0;
2873
-
2874
- static id<MTLDevice> ggml_backend_metal_get_device(void) {
2875
- if (g_backend_device == nil) {
2876
- g_backend_device = MTLCreateSystemDefaultDevice();
2877
- }
2878
-
2879
- g_backend_device_ref_count++;
2880
-
2881
- return g_backend_device;
2882
- }
2883
-
2884
- static void ggml_backend_metal_free_device(void) {
2885
- assert(g_backend_device_ref_count > 0);
2886
-
2887
- g_backend_device_ref_count--;
2888
-
2889
- if (g_backend_device_ref_count == 0) {
2890
- [g_backend_device release];
2891
- g_backend_device = nil;
2892
- }
2893
- }
2894
-
2895
- GGML_CALL static const char * ggml_backend_metal_buffer_get_name(ggml_backend_buffer_t buffer) {
2896
- return "Metal";
2897
-
2898
- UNUSED(buffer);
2899
- }
2900
-
2901
- GGML_CALL static void ggml_backend_metal_buffer_free_buffer(ggml_backend_buffer_t buffer) {
2902
- struct ggml_backend_metal_buffer_context * ctx = (struct ggml_backend_metal_buffer_context *)buffer->context;
2903
-
2904
- for (int i = 0; i < ctx->n_buffers; i++) {
2905
- [ctx->buffers[i].metal release];
2906
- }
2907
- ggml_backend_metal_free_device();
2908
-
2909
- if (ctx->owned) {
2910
- #if TARGET_OS_OSX
2911
- vm_deallocate((vm_map_t)mach_task_self(), (vm_address_t)ctx->all_data, ctx->all_size);
2912
- #else
2913
- free(ctx->all_data);
2914
- #endif
2915
- }
2916
-
2917
- free(ctx);
2918
- }
2919
-
2920
- GGML_CALL static void * ggml_backend_metal_buffer_get_base(ggml_backend_buffer_t buffer) {
2921
- struct ggml_backend_metal_buffer_context * ctx = (struct ggml_backend_metal_buffer_context *)buffer->context;
2922
-
2923
- return ctx->all_data;
2924
- }
2925
-
2926
- GGML_CALL static void ggml_backend_metal_buffer_set_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size) {
2927
- memcpy((char *)tensor->data + offset, data, size);
2928
-
2929
- UNUSED(buffer);
2930
- }
2931
-
2932
- GGML_CALL static void ggml_backend_metal_buffer_get_tensor(ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size) {
2933
- memcpy(data, (const char *)tensor->data + offset, size);
2934
-
2935
- UNUSED(buffer);
2936
- }
2937
-
2938
- GGML_CALL static bool ggml_backend_metal_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const struct ggml_tensor * src, struct ggml_tensor * dst) {
2939
- if (ggml_backend_buffer_is_host(src->buffer)) {
2940
- memcpy(dst->data, src->data, ggml_nbytes(src));
2941
- return true;
2942
- }
2943
- return false;
2944
-
2945
- UNUSED(buffer);
2946
- }
2947
-
2948
- GGML_CALL static void ggml_backend_metal_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {
2949
- struct ggml_backend_metal_buffer_context * ctx = (struct ggml_backend_metal_buffer_context *)buffer->context;
2950
-
2951
- memset(ctx->all_data, value, ctx->all_size);
2952
- }
2953
-
2954
- static struct ggml_backend_buffer_i ggml_backend_metal_buffer_i = {
2955
- /* .get_name = */ ggml_backend_metal_buffer_get_name,
2956
- /* .free_buffer = */ ggml_backend_metal_buffer_free_buffer,
2957
- /* .get_base = */ ggml_backend_metal_buffer_get_base,
2958
- /* .init_tensor = */ NULL,
2959
- /* .set_tensor = */ ggml_backend_metal_buffer_set_tensor,
2960
- /* .get_tensor = */ ggml_backend_metal_buffer_get_tensor,
2961
- /* .cpy_tensor = */ ggml_backend_metal_buffer_cpy_tensor,
2962
- /* .clear = */ ggml_backend_metal_buffer_clear,
2963
- /* .reset = */ NULL,
2964
- };
2965
-
2966
- // default buffer type
2967
-
2968
- GGML_CALL static const char * ggml_backend_metal_buffer_type_get_name(ggml_backend_buffer_type_t buft) {
2969
- return "Metal";
2970
-
2971
- UNUSED(buft);
2972
- }
2973
-
2974
- static void ggml_backend_metal_log_allocated_size(id<MTLDevice> device, size_t size_aligned) {
2975
- #ifndef GGML_METAL_NDEBUG
2976
- #if TARGET_OS_OSX || (TARGET_OS_IOS && __clang_major__ >= 15)
2977
- if (@available(macOS 10.12, iOS 16.0, *)) {
2978
- GGML_METAL_LOG_INFO("%s: allocated buffer, size = %8.2f MiB, (%8.2f / %8.2f)",
2979
- __func__,
2980
- size_aligned / 1024.0 / 1024.0,
2981
- device.currentAllocatedSize / 1024.0 / 1024.0,
2982
- device.recommendedMaxWorkingSetSize / 1024.0 / 1024.0);
2983
-
2984
- if (device.currentAllocatedSize > device.recommendedMaxWorkingSetSize) {
2985
- GGML_METAL_LOG_WARN("%s: warning: current allocated size is greater than the recommended max working set size\n", __func__);
2986
- } else {
2987
- GGML_METAL_LOG_INFO("\n");
2988
- }
2989
- } else {
2990
- GGML_METAL_LOG_INFO("%s: allocated buffer, size = %8.2f MiB, (%8.2f)\n",
2991
- __func__,
2992
- size_aligned / 1024.0 / 1024.0,
2993
- device.currentAllocatedSize / 1024.0 / 1024.0);
2994
- }
2995
- #endif
2996
- #endif
2997
- UNUSED(device);
2998
- UNUSED(size_aligned);
2999
- }
3000
-
3001
- GGML_CALL static ggml_backend_buffer_t ggml_backend_metal_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {
3002
- struct ggml_backend_metal_buffer_context * ctx = malloc(sizeof(struct ggml_backend_metal_buffer_context));
3003
-
3004
- const size_t size_page = sysconf(_SC_PAGESIZE);
3005
-
3006
- size_t size_aligned = size;
3007
- if ((size_aligned % size_page) != 0) {
3008
- size_aligned += (size_page - (size_aligned % size_page));
3009
- }
3010
-
3011
- id<MTLDevice> device = ggml_backend_metal_get_device();
3012
-
3013
- ctx->all_data = ggml_metal_host_malloc(size_aligned);
3014
- ctx->all_size = size_aligned;
3015
- ctx->owned = true;
3016
- ctx->n_buffers = 1;
3017
-
3018
- if (ctx->all_data != NULL) {
3019
- ctx->buffers[0].data = ctx->all_data;
3020
- ctx->buffers[0].size = size;
3021
- ctx->buffers[0].metal = [device newBufferWithBytesNoCopy:ctx->all_data
3022
- length:size_aligned
3023
- options:MTLResourceStorageModeShared
3024
- deallocator:nil];
3025
- }
3026
-
3027
- if (ctx->all_data == NULL || ctx->buffers[0].metal == nil) {
3028
- GGML_METAL_LOG_ERROR("%s: error: failed to allocate buffer, size = %8.2f MiB\n", __func__, size_aligned / 1024.0 / 1024.0);
3029
- free(ctx);
3030
- ggml_backend_metal_free_device();
3031
- return NULL;
3032
- }
3033
-
3034
- //ggml_backend_metal_log_allocated_size(device, size_aligned);
3035
-
3036
- return ggml_backend_buffer_init(buft, ggml_backend_metal_buffer_i, ctx, size);
3037
- }
3038
-
3039
- GGML_CALL static size_t ggml_backend_metal_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) {
3040
- return 32;
3041
- UNUSED(buft);
3042
- }
3043
-
3044
- GGML_CALL static size_t ggml_backend_metal_buffer_type_get_max_size(ggml_backend_buffer_type_t buft) {
3045
- id<MTLDevice> device = ggml_backend_metal_get_device();
3046
- size_t max_size = device.maxBufferLength;
3047
- ggml_backend_metal_free_device();
3048
-
3049
- return max_size;
3050
-
3051
- UNUSED(buft);
3052
- }
3053
-
3054
- GGML_CALL static bool ggml_backend_metal_buffer_type_is_host(ggml_backend_buffer_type_t buft) {
3055
- return true;
3056
-
3057
- UNUSED(buft);
3058
- }
3059
-
3060
- GGML_CALL ggml_backend_buffer_type_t ggml_backend_metal_buffer_type(void) {
3061
- static struct ggml_backend_buffer_type ggml_backend_buffer_type_metal = {
3062
- /* .iface = */ {
3063
- /* .get_name = */ ggml_backend_metal_buffer_type_get_name,
3064
- /* .alloc_buffer = */ ggml_backend_metal_buffer_type_alloc_buffer,
3065
- /* .get_alignment = */ ggml_backend_metal_buffer_type_get_alignment,
3066
- /* .get_max_size = */ ggml_backend_metal_buffer_type_get_max_size,
3067
- /* .get_alloc_size = */ NULL, // defaults to ggml_nbytes
3068
- /* .is_host = */ ggml_backend_metal_buffer_type_is_host,
3069
- },
3070
- /* .context = */ NULL,
3071
- };
3072
-
3073
- return &ggml_backend_buffer_type_metal;
3074
- }
3075
-
3076
- // buffer from ptr
3077
-
3078
- GGML_CALL ggml_backend_buffer_t ggml_backend_metal_buffer_from_ptr(void * data, size_t size, size_t max_size) {
3079
- struct ggml_backend_metal_buffer_context * ctx = malloc(sizeof(struct ggml_backend_metal_buffer_context));
3080
-
3081
- ctx->all_data = data;
3082
- ctx->all_size = size;
3083
- ctx->owned = false;
3084
- ctx->n_buffers = 0;
3085
-
3086
- const size_t size_page = sysconf(_SC_PAGESIZE);
3087
-
3088
- // page-align the data ptr
3089
- {
3090
- const uintptr_t offs = (uintptr_t) data % size_page;
3091
- data = (void *) ((char *) data - offs);
3092
- size += offs;
3093
- }
3094
-
3095
- size_t size_aligned = size;
3096
- if ((size_aligned % size_page) != 0) {
3097
- size_aligned += (size_page - (size_aligned % size_page));
3098
- }
3099
-
3100
- id<MTLDevice> device = ggml_backend_metal_get_device();
3101
-
3102
- // the buffer fits into the max buffer size allowed by the device
3103
- if (size_aligned <= device.maxBufferLength) {
3104
- ctx->buffers[ctx->n_buffers].data = data;
3105
- ctx->buffers[ctx->n_buffers].size = size;
3106
-
3107
- ctx->buffers[ctx->n_buffers].metal = [device newBufferWithBytesNoCopy:data length:size_aligned options:MTLResourceStorageModeShared deallocator:nil];
3108
-
3109
- if (ctx->buffers[ctx->n_buffers].metal == nil) {
3110
- GGML_METAL_LOG_ERROR("%s: error: failed to allocate buffer, size = %8.2f MiB\n", __func__, size_aligned / 1024.0 / 1024.0);
3111
- return false;
3112
- }
3113
-
3114
- ggml_backend_metal_log_allocated_size(device, size_aligned);
3115
-
3116
- ++ctx->n_buffers;
3117
- } else {
3118
- // this overlap between the views will guarantee that the tensor with the maximum size will fully fit into
3119
- // one of the views
3120
- const size_t size_ovlp = ((max_size + size_page - 1) / size_page + 1) * size_page; // round-up 2 pages just in case
3121
- const size_t size_step = device.maxBufferLength - size_ovlp;
3122
- const size_t size_view = device.maxBufferLength;
3123
-
3124
- for (size_t i = 0; i < size; i += size_step) {
3125
- const size_t size_step_aligned = (i + size_view <= size) ? size_view : (size_aligned - i);
3126
-
3127
- ctx->buffers[ctx->n_buffers].data = (void *) ((uint8_t *) data + i);
3128
- ctx->buffers[ctx->n_buffers].size = size_step_aligned;
3129
-
3130
- ctx->buffers[ctx->n_buffers].metal = [device newBufferWithBytesNoCopy:(void *) ((uint8_t *) data + i) length:size_step_aligned options:MTLResourceStorageModeShared deallocator:nil];
3131
-
3132
- if (ctx->buffers[ctx->n_buffers].metal == nil) {
3133
- GGML_METAL_LOG_ERROR("%s: error: failed to allocate buffer, size = %8.2f MiB\n", __func__, size_step_aligned / 1024.0 / 1024.0);
3134
- return false;
3135
- }
3136
-
3137
- ggml_backend_metal_log_allocated_size(device, size_step_aligned);
3138
-
3139
- if (i + size_step < size) {
3140
- GGML_METAL_LOG_INFO("\n");
3141
- }
3142
-
3143
- ++ctx->n_buffers;
3144
- }
3145
- }
3146
-
3147
- return ggml_backend_buffer_init(ggml_backend_metal_buffer_type(), ggml_backend_metal_buffer_i, ctx, size);
3148
- }
3149
-
3150
- // backend
3151
-
3152
- GGML_CALL static const char * ggml_backend_metal_name(ggml_backend_t backend) {
3153
- return "Metal";
3154
-
3155
- UNUSED(backend);
3156
- }
3157
-
3158
- GGML_CALL static void ggml_backend_metal_free(ggml_backend_t backend) {
3159
- struct ggml_metal_context * ctx = (struct ggml_metal_context *)backend->context;
3160
- ggml_metal_free(ctx);
3161
- free(backend);
3162
- }
3163
-
3164
- GGML_CALL static ggml_backend_buffer_type_t ggml_backend_metal_get_default_buffer_type(ggml_backend_t backend) {
3165
- return ggml_backend_metal_buffer_type();
3166
-
3167
- UNUSED(backend);
3168
- }
3169
-
3170
- GGML_CALL static enum ggml_status ggml_backend_metal_graph_compute(ggml_backend_t backend, struct ggml_cgraph * cgraph) {
3171
- struct ggml_metal_context * metal_ctx = (struct ggml_metal_context *)backend->context;
3172
-
3173
- return ggml_metal_graph_compute(metal_ctx, cgraph);
3174
- }
3175
-
3176
- GGML_CALL static bool ggml_backend_metal_supports_op(ggml_backend_t backend, const struct ggml_tensor * op) {
3177
- struct ggml_metal_context * metal_ctx = (struct ggml_metal_context *)backend->context;
3178
-
3179
- return ggml_metal_supports_op(metal_ctx, op);
3180
- }
3181
-
3182
- GGML_CALL static bool ggml_backend_metal_supports_buft(ggml_backend_t backend, ggml_backend_buffer_type_t buft) {
3183
- return buft->iface.get_name == ggml_backend_metal_buffer_type_get_name;
3184
-
3185
- UNUSED(backend);
3186
- }
3187
-
3188
- static struct ggml_backend_i ggml_backend_metal_i = {
3189
- /* .get_name = */ ggml_backend_metal_name,
3190
- /* .free = */ ggml_backend_metal_free,
3191
- /* .get_default_buffer_type = */ ggml_backend_metal_get_default_buffer_type,
3192
- /* .set_tensor_async = */ NULL,
3193
- /* .get_tensor_async = */ NULL,
3194
- /* .cpy_tensor_async = */ NULL,
3195
- /* .synchronize = */ NULL,
3196
- /* .graph_plan_create = */ NULL,
3197
- /* .graph_plan_free = */ NULL,
3198
- /* .graph_plan_update = */ NULL,
3199
- /* .graph_plan_compute = */ NULL,
3200
- /* .graph_compute = */ ggml_backend_metal_graph_compute,
3201
- /* .supports_op = */ ggml_backend_metal_supports_op,
3202
- /* .supports_buft = */ ggml_backend_metal_supports_buft,
3203
- /* .offload_op = */ NULL,
3204
- /* .event_new = */ NULL,
3205
- /* .event_free = */ NULL,
3206
- /* .event_record = */ NULL,
3207
- /* .event_wait = */ NULL,
3208
- /* .event_synchronize = */ NULL,
3209
- };
3210
-
3211
- void ggml_backend_metal_log_set_callback(ggml_log_callback log_callback, void * user_data) {
3212
- ggml_metal_log_callback = log_callback;
3213
- ggml_metal_log_user_data = user_data;
3214
- }
3215
-
3216
- static ggml_guid_t ggml_backend_metal_guid(void) {
3217
- static ggml_guid guid = { 0x81, 0xa1, 0x8b, 0x1e, 0x71, 0xec, 0x79, 0xed, 0x2b, 0x85, 0xdc, 0x8a, 0x61, 0x98, 0x30, 0xe6 };
3218
- return &guid;
3219
- }
3220
-
3221
- ggml_backend_t ggml_backend_metal_init(void) {
3222
- struct ggml_metal_context * ctx = ggml_metal_init(GGML_DEFAULT_N_THREADS);
3223
-
3224
- if (ctx == NULL) {
3225
- return NULL;
3226
- }
3227
-
3228
- ggml_backend_t metal_backend = malloc(sizeof(struct ggml_backend));
3229
-
3230
- *metal_backend = (struct ggml_backend) {
3231
- /* .guid = */ ggml_backend_metal_guid(),
3232
- /* .interface = */ ggml_backend_metal_i,
3233
- /* .context = */ ctx,
3234
- };
3235
-
3236
- return metal_backend;
3237
- }
3238
-
3239
- bool ggml_backend_is_metal(ggml_backend_t backend) {
3240
- return backend != NULL && ggml_guid_matches(backend->guid, ggml_backend_metal_guid());
3241
- }
3242
-
3243
- void ggml_backend_metal_set_n_cb(ggml_backend_t backend, int n_cb) {
3244
- GGML_ASSERT(ggml_backend_is_metal(backend));
3245
-
3246
- struct ggml_metal_context * ctx = (struct ggml_metal_context *)backend->context;
3247
-
3248
- ctx->n_cb = MIN(n_cb, GGML_METAL_MAX_BUFFERS);
3249
- }
3250
-
3251
- bool ggml_backend_metal_supports_family(ggml_backend_t backend, int family) {
3252
- GGML_ASSERT(ggml_backend_is_metal(backend));
3253
-
3254
- struct ggml_metal_context * ctx = (struct ggml_metal_context *)backend->context;
3255
-
3256
- return [ctx->device supportsFamily:(MTLGPUFamilyApple1 + family - 1)];
3257
- }
3258
-
3259
- void ggml_backend_metal_capture_next_compute(ggml_backend_t backend) {
3260
- GGML_ASSERT(ggml_backend_is_metal(backend));
3261
-
3262
- struct ggml_metal_context * ctx = (struct ggml_metal_context *)backend->context;
3263
- ctx->should_capture_next_compute = true;
3264
- }
3265
-
3266
- GGML_CALL ggml_backend_t ggml_backend_reg_metal_init(const char * params, void * user_data); // silence warning
3267
-
3268
- GGML_CALL ggml_backend_t ggml_backend_reg_metal_init(const char * params, void * user_data) {
3269
- return ggml_backend_metal_init();
3270
-
3271
- GGML_UNUSED(params);
3272
- GGML_UNUSED(user_data);
3273
- }