llama_cpp 0.15.3 → 0.16.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (149) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +16 -0
  3. data/ext/llama_cpp/extconf.rb +1 -2
  4. data/ext/llama_cpp/llama_cpp.cpp +27 -3
  5. data/lib/llama_cpp/version.rb +2 -2
  6. data/sig/llama_cpp.rbs +15 -1
  7. data/vendor/tmp/llama.cpp/Makefile +66 -36
  8. data/vendor/tmp/llama.cpp/ggml-alloc.c +4 -4
  9. data/vendor/tmp/llama.cpp/ggml-backend.c +5 -5
  10. data/vendor/tmp/llama.cpp/ggml-backend.h +1 -1
  11. data/vendor/tmp/llama.cpp/ggml-cuda/acc.cu +47 -0
  12. data/vendor/tmp/llama.cpp/ggml-cuda/arange.cu +34 -0
  13. data/vendor/tmp/llama.cpp/ggml-cuda/argsort.cu +103 -0
  14. data/vendor/tmp/llama.cpp/ggml-cuda/binbcast.cu +280 -0
  15. data/vendor/tmp/llama.cpp/ggml-cuda/clamp.cu +34 -0
  16. data/vendor/tmp/llama.cpp/ggml-cuda/concat.cu +196 -0
  17. data/vendor/tmp/llama.cpp/ggml-cuda/convert.cu +686 -0
  18. data/vendor/tmp/llama.cpp/ggml-cuda/cpy.cu +490 -0
  19. data/vendor/tmp/llama.cpp/ggml-cuda/diagmask.cu +40 -0
  20. data/vendor/tmp/llama.cpp/ggml-cuda/dmmv.cu +662 -0
  21. data/vendor/tmp/llama.cpp/ggml-cuda/fattn-tile-f16.cu +319 -0
  22. data/vendor/tmp/llama.cpp/ggml-cuda/fattn-tile-f32.cu +312 -0
  23. data/vendor/tmp/llama.cpp/ggml-cuda/fattn.cu +345 -0
  24. data/vendor/tmp/llama.cpp/ggml-cuda/getrows.cu +178 -0
  25. data/vendor/tmp/llama.cpp/ggml-cuda/im2col.cu +104 -0
  26. data/vendor/tmp/llama.cpp/ggml-cuda/mmq.cu +1564 -0
  27. data/vendor/tmp/llama.cpp/ggml-cuda/mmvq.cu +404 -0
  28. data/vendor/tmp/llama.cpp/ggml-cuda/norm.cu +221 -0
  29. data/vendor/tmp/llama.cpp/ggml-cuda/pad.cu +49 -0
  30. data/vendor/tmp/llama.cpp/ggml-cuda/pool2d.cu +94 -0
  31. data/vendor/tmp/llama.cpp/ggml-cuda/quantize.cu +45 -0
  32. data/vendor/tmp/llama.cpp/ggml-cuda/rope.cu +271 -0
  33. data/vendor/tmp/llama.cpp/ggml-cuda/scale.cu +31 -0
  34. data/vendor/tmp/llama.cpp/ggml-cuda/softmax.cu +205 -0
  35. data/vendor/tmp/llama.cpp/ggml-cuda/sumrows.cu +40 -0
  36. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-f16.cu +5 -0
  37. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q4_0.cu +5 -0
  38. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q4_1.cu +5 -0
  39. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q5_0.cu +5 -0
  40. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q5_1.cu +5 -0
  41. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q8_0.cu +5 -0
  42. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-f16.cu +5 -0
  43. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q4_0.cu +5 -0
  44. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q4_1.cu +5 -0
  45. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q5_0.cu +5 -0
  46. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q5_1.cu +5 -0
  47. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q8_0.cu +5 -0
  48. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-f16.cu +5 -0
  49. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q4_0.cu +5 -0
  50. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q4_1.cu +5 -0
  51. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q5_0.cu +5 -0
  52. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q5_1.cu +5 -0
  53. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q8_0.cu +5 -0
  54. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-f16.cu +5 -0
  55. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q4_0.cu +5 -0
  56. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q4_1.cu +5 -0
  57. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q5_0.cu +5 -0
  58. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q5_1.cu +5 -0
  59. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q8_0.cu +5 -0
  60. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-f16.cu +5 -0
  61. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q4_0.cu +5 -0
  62. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q4_1.cu +5 -0
  63. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q5_0.cu +5 -0
  64. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q5_1.cu +5 -0
  65. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q8_0.cu +5 -0
  66. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-f16.cu +5 -0
  67. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q4_0.cu +5 -0
  68. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q4_1.cu +5 -0
  69. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q5_0.cu +5 -0
  70. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q5_1.cu +5 -0
  71. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q8_0.cu +5 -0
  72. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs256-f16-f16.cu +5 -0
  73. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-f16.cu +5 -0
  74. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q4_0.cu +5 -0
  75. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q4_1.cu +5 -0
  76. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q5_0.cu +5 -0
  77. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q5_1.cu +5 -0
  78. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q8_0.cu +5 -0
  79. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-f16.cu +5 -0
  80. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q4_0.cu +5 -0
  81. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q4_1.cu +5 -0
  82. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q5_0.cu +5 -0
  83. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q5_1.cu +5 -0
  84. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q8_0.cu +5 -0
  85. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-f16.cu +5 -0
  86. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q4_0.cu +5 -0
  87. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q4_1.cu +5 -0
  88. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q5_0.cu +5 -0
  89. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q5_1.cu +5 -0
  90. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q8_0.cu +5 -0
  91. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-f16.cu +5 -0
  92. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q4_0.cu +5 -0
  93. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q4_1.cu +5 -0
  94. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q5_0.cu +5 -0
  95. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q5_1.cu +5 -0
  96. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q8_0.cu +5 -0
  97. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-f16.cu +5 -0
  98. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q4_0.cu +5 -0
  99. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q4_1.cu +5 -0
  100. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q5_0.cu +5 -0
  101. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q5_1.cu +5 -0
  102. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q8_0.cu +5 -0
  103. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-f16.cu +5 -0
  104. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q4_0.cu +5 -0
  105. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q4_1.cu +5 -0
  106. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q5_0.cu +5 -0
  107. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q5_1.cu +5 -0
  108. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q8_0.cu +5 -0
  109. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-f16.cu +5 -0
  110. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q4_0.cu +5 -0
  111. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q4_1.cu +5 -0
  112. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q5_0.cu +5 -0
  113. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q5_1.cu +5 -0
  114. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q8_0.cu +5 -0
  115. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs256-f16-f16.cu +5 -0
  116. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-f16.cu +5 -0
  117. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q4_0.cu +5 -0
  118. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q4_1.cu +5 -0
  119. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q5_0.cu +5 -0
  120. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q5_1.cu +5 -0
  121. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q8_0.cu +5 -0
  122. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqfloat-cpb16.cu +10 -0
  123. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqfloat-cpb32.cu +9 -0
  124. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqhalf-cpb16.cu +10 -0
  125. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqhalf-cpb32.cu +10 -0
  126. data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqhalf-cpb8.cu +8 -0
  127. data/vendor/tmp/llama.cpp/ggml-cuda/tsembd.cu +47 -0
  128. data/vendor/tmp/llama.cpp/ggml-cuda/unary.cu +266 -0
  129. data/vendor/tmp/llama.cpp/ggml-cuda/upscale.cu +51 -0
  130. data/vendor/tmp/llama.cpp/ggml-cuda.cu +35 -16
  131. data/vendor/tmp/llama.cpp/ggml-impl.h +4 -0
  132. data/vendor/tmp/llama.cpp/ggml-kompute.cpp +21 -7
  133. data/vendor/tmp/llama.cpp/ggml-metal.h +1 -1
  134. data/vendor/tmp/llama.cpp/ggml-metal.m +99 -35
  135. data/vendor/tmp/llama.cpp/ggml-metal.metal +146 -80
  136. data/vendor/tmp/llama.cpp/ggml-quants.c +101 -11
  137. data/vendor/tmp/llama.cpp/ggml-rpc.cpp +75 -58
  138. data/vendor/tmp/llama.cpp/ggml-sycl.cpp +345 -227
  139. data/vendor/tmp/llama.cpp/ggml-vulkan-shaders.hpp +99301 -39793
  140. data/vendor/tmp/llama.cpp/ggml-vulkan.cpp +458 -329
  141. data/vendor/tmp/llama.cpp/ggml.c +301 -409
  142. data/vendor/tmp/llama.cpp/ggml.h +19 -23
  143. data/vendor/tmp/llama.cpp/llama.cpp +855 -651
  144. data/vendor/tmp/llama.cpp/llama.h +28 -48
  145. metadata +121 -6
  146. data/vendor/tmp/llama.cpp/ggml-mpi.c +0 -216
  147. data/vendor/tmp/llama.cpp/ggml-mpi.h +0 -39
  148. data/vendor/tmp/llama.cpp/ggml-opencl.cpp +0 -2305
  149. data/vendor/tmp/llama.cpp/ggml-opencl.h +0 -36
data/vendor/tmp/llama.cpp/ggml.c CHANGED
@@ -5,6 +5,7 @@
5
5
  #include "ggml-quants.h"
6
6
  #include "ggml.h"
7
7
 
8
+
8
9
  #if defined(_MSC_VER) || defined(__MINGW32__)
9
10
  #include <malloc.h> // using malloc.h with MSC/MINGW
10
11
  #elif !defined(__FreeBSD__) && !defined(__NetBSD__) && !defined(__OpenBSD__)
@@ -28,6 +29,10 @@
28
29
  #include <syscall.h>
29
30
  #endif
30
31
 
32
+ #ifdef GGML_USE_OPENMP
33
+ #include <omp.h>
34
+ #endif
35
+
31
36
  #ifdef GGML_USE_METAL
32
37
  #include <unistd.h>
33
38
  #endif
@@ -60,6 +65,9 @@
60
65
 
61
66
  typedef volatile LONG atomic_int;
62
67
  typedef atomic_int atomic_bool;
68
+ typedef atomic_int atomic_flag;
69
+
70
+ #define ATOMIC_FLAG_INIT 0
63
71
 
64
72
  static void atomic_store(atomic_int * ptr, LONG val) {
65
73
  InterlockedExchange(ptr, val);
@@ -73,6 +81,12 @@ static LONG atomic_fetch_add(atomic_int * ptr, LONG inc) {
73
81
  static LONG atomic_fetch_sub(atomic_int * ptr, LONG dec) {
74
82
  return atomic_fetch_add(ptr, -(dec));
75
83
  }
84
+ static atomic_bool atomic_flag_test_and_set(atomic_flag * ptr) {
85
+ return InterlockedExchange(ptr, 1);
86
+ }
87
+ static void atomic_flag_clear(atomic_flag * ptr) {
88
+ InterlockedExchange(ptr, 0);
89
+ }
76
90
 
77
91
  typedef HANDLE pthread_t;
78
92
 
@@ -283,17 +297,12 @@ inline static void * ggml_calloc(size_t num, size_t size) {
283
297
 
284
298
  #if defined(GGML_USE_ACCELERATE)
285
299
  #include <Accelerate/Accelerate.h>
286
- #if defined(GGML_USE_CLBLAST) // allow usage of CLBlast alongside Accelerate functions
287
- #include "ggml-opencl.h"
288
- #endif
289
300
  #elif defined(GGML_USE_OPENBLAS)
290
301
  #if defined(GGML_BLAS_USE_MKL)
291
302
  #include <mkl.h>
292
303
  #else
293
304
  #include <cblas.h>
294
305
  #endif
295
- #elif defined(GGML_USE_CLBLAST)
296
- #include "ggml-opencl.h"
297
306
  #endif
298
307
 
299
308
  // floating point type used to accumulate sums
@@ -1567,11 +1576,11 @@ do { \
1567
1576
 
1568
1577
  // F16 arithmetic is not supported by AVX, so we use F32 instead
1569
1578
 
1570
- #define GGML_F32Cx8 __m256
1579
+ #define GGML_F32Cx8 __m256
1571
1580
  #define GGML_F32Cx8_ZERO (__m256)__lasx_xvldi(0)
1572
1581
  #define GGML_F32Cx8_SET1(x) (__m256)__lasx_xvreplgr2vr_w((x))
1573
1582
 
1574
- static inline __m256 __lasx_f32cx8_load(ggml_fp16_t *x) {
1583
+ static inline __m256 __lasx_f32cx8_load(const ggml_fp16_t * x) {
1575
1584
  float tmp[8];
1576
1585
 
1577
1586
  for (int i = 0; i < 8; i++) {
@@ -1580,13 +1589,14 @@ static inline __m256 __lasx_f32cx8_load(ggml_fp16_t *x) {
1580
1589
 
1581
1590
  return (__m256)__lasx_xvld(tmp, 0);
1582
1591
  }
1583
- static inline void __lasx_f32cx8_store(ggml_fp16_t *x, __m256 y) {
1592
+ static inline void __lasx_f32cx8_store(ggml_fp16_t * x, __m256 y) {
1584
1593
  float arr[8];
1585
1594
 
1586
1595
  __lasx_xvst(y, arr, 0);
1587
1596
 
1588
- for (int i = 0; i < 8; i++)
1597
+ for (int i = 0; i < 8; i++) {
1589
1598
  x[i] = GGML_FP32_TO_FP16(arr[i]);
1599
+ }
1590
1600
  }
1591
1601
  #define GGML_F32Cx8_LOAD(x) __lasx_f32cx8_load(x)
1592
1602
  #define GGML_F32Cx8_STORE(x, y) __lasx_f32cx8_store(x, y)
@@ -1662,7 +1672,7 @@ static inline void __lasx_f32cx8_store(ggml_fp16_t *x, __m256 y) {
1662
1672
  #define GGML_F16_STEP 32
1663
1673
  #define GGML_F16_EPR 4
1664
1674
 
1665
- static inline __m128 __lsx_f16x4_load(ggml_fp16_t *x) {
1675
+ static inline __m128 __lsx_f16x4_load(const ggml_fp16_t * x) {
1666
1676
  float tmp[4];
1667
1677
 
1668
1678
  tmp[0] = GGML_FP16_TO_FP32(x[0]);
@@ -1673,7 +1683,7 @@ static inline __m128 __lsx_f16x4_load(ggml_fp16_t *x) {
1673
1683
  return __lsx_vld(tmp, 0);
1674
1684
  }
1675
1685
 
1676
- static inline void __lsx_f16x4_store(ggml_fp16_t *x, __m128 y) {
1686
+ static inline void __lsx_f16x4_store(ggml_fp16_t * x, __m128 y) {
1677
1687
  float arr[4];
1678
1688
 
1679
1689
  __lsx_vst(y, arr, 0);
@@ -1746,7 +1756,7 @@ struct ggml_compute_state_shared {
1746
1756
  int64_t perf_node_start_cycles;
1747
1757
  int64_t perf_node_start_time_us;
1748
1758
 
1749
- const int n_threads;
1759
+ int n_threads;
1750
1760
 
1751
1761
  // synchronization primitives
1752
1762
  atomic_int n_active; // num active threads
@@ -2257,6 +2267,11 @@ inline static float ggml_silu_f32(float x) {
2257
2267
  return x/(1.0f + expf(-x));
2258
2268
  }
2259
2269
 
2270
+ #if __FINITE_MATH_ONLY__
2271
+ #error "some routines in ggml.c require non-finite math arithmetics -- pass -fno-finite-math-only to the compiler to fix"
2272
+ #error "ref: https://github.com/ggerganov/llama.cpp/pull/7154#issuecomment-2143844461"
2273
+ #endif
2274
+
2260
2275
  #if defined(__ARM_NEON) && defined(__aarch64__)
2261
2276
 
2262
2277
  // adapted from arm limited optimized routine
@@ -2306,32 +2321,27 @@ inline static __m512 ggml_v_expf(__m512 x) {
2306
2321
  const __m512 r = _mm512_set1_ps(0x1.8p23f);
2307
2322
  const __m512 z = _mm512_fmadd_ps(x, _mm512_set1_ps(0x1.715476p+0f), r);
2308
2323
  const __m512 n = _mm512_sub_ps(z, r);
2309
- const __m512 b = _mm512_fnmadd_ps(n, _mm512_set1_ps(0x1.7f7d1cp-20f),
2310
- _mm512_fnmadd_ps(n, _mm512_set1_ps(0x1.62e4p-1f), x));
2311
- const __m512i e = _mm512_slli_epi32(_mm512_castps_si512(z), 23);
2312
- const __m512 k = _mm512_castsi512_ps(_mm512_add_epi32(e, _mm512_castps_si512(_mm512_set1_ps(1))));
2313
- const __mmask16 c = _mm512_cmp_ps_mask(_mm512_abs_ps(n), _mm512_set1_ps(126), _CMP_GT_OQ);
2314
- const __m512 u = _mm512_mul_ps(b, b);
2315
- const __m512 j = _mm512_fmadd_ps(_mm512_fmadd_ps(_mm512_fmadd_ps(_mm512_set1_ps(0x1.0e4020p-7f), b,
2316
- _mm512_set1_ps(0x1.573e2ep-5f)), u,
2317
- _mm512_fmadd_ps(_mm512_set1_ps(0x1.555e66p-3f), b,
2318
- _mm512_set1_ps(0x1.fffdb6p-2f))),
2319
- u, _mm512_mul_ps(_mm512_set1_ps(0x1.ffffecp-1f), b));
2320
- if (_mm512_kortestz(c, c))
2321
- return _mm512_fmadd_ps(j, k, k);
2322
- const __m512i g = _mm512_and_si512(
2323
- _mm512_movm_epi32(_mm512_cmp_ps_mask(n, _mm512_setzero_ps(), _CMP_LE_OQ)),
2324
- _mm512_set1_epi32(0x82000000u));
2325
- const __m512 s1 =
2326
- _mm512_castsi512_ps(_mm512_add_epi32(g, _mm512_set1_epi32(0x7f000000u)));
2327
- const __m512 s2 = _mm512_castsi512_ps(_mm512_sub_epi32(e, g));
2324
+ const __m512 b =
2325
+ _mm512_fnmadd_ps(n, _mm512_set1_ps(0x1.7f7d1cp-20f),
2326
+ _mm512_fnmadd_ps(n, _mm512_set1_ps(0x1.62e4p-1f), x));
2328
2327
  const __mmask16 d =
2329
2328
  _mm512_cmp_ps_mask(_mm512_abs_ps(n), _mm512_set1_ps(192), _CMP_GT_OQ);
2330
- return _mm512_mask_blend_ps(
2331
- d, _mm512_mask_blend_ps(
2332
- c, _mm512_fmadd_ps(k, j, k),
2333
- _mm512_mul_ps(_mm512_fmadd_ps(s2, j, s2), s1)),
2334
- _mm512_mul_ps(s1, s1));
2329
+ const __m512 u = _mm512_mul_ps(b, b);
2330
+ const __m512 j = _mm512_fmadd_ps(
2331
+ _mm512_fmadd_ps(_mm512_fmadd_ps(_mm512_set1_ps(0x1.0e4020p-7f), b,
2332
+ _mm512_set1_ps(0x1.573e2ep-5f)),
2333
+ u,
2334
+ _mm512_fmadd_ps(_mm512_set1_ps(0x1.555e66p-3f), b,
2335
+ _mm512_set1_ps(0x1.fffdb6p-2f))),
2336
+ u,
2337
+ _mm512_fmadd_ps(_mm512_set1_ps(0x1.ffffecp-1f), b, _mm512_set1_ps(1.0F)));
2338
+ const __m512 res = _mm512_scalef_ps(j, n);
2339
+ if (_mm512_kortestz(d, d))
2340
+ return res;
2341
+ const __m512 zero = _mm512_setzero_ps();
2342
+ const __m512 alt = _mm512_mask_blend_ps(
2343
+ _mm512_cmp_ps_mask(n, zero, _CMP_LE_OQ), _mm512_set1_ps(INFINITY), zero);
2344
+ return _mm512_mask_blend_ps(d, res, alt);
2335
2345
  }
2336
2346
 
2337
2347
  // computes silu x/(1+exp(-x)) in single precision vector
@@ -2883,24 +2893,20 @@ struct ggml_state {
2883
2893
 
2884
2894
  // global state
2885
2895
  static struct ggml_state g_state;
2886
- static atomic_int g_state_barrier = 0;
2896
+ static atomic_flag g_state_critical = ATOMIC_FLAG_INIT;
2887
2897
 
2888
2898
  // barrier via spin lock
2889
2899
  inline static void ggml_critical_section_start(void) {
2890
- int processing = atomic_fetch_add(&g_state_barrier, 1);
2891
-
2892
- while (processing > 0) {
2893
- // wait for other threads to finish
2894
- atomic_fetch_sub(&g_state_barrier, 1);
2895
- sched_yield(); // TODO: reconsider this
2896
- processing = atomic_fetch_add(&g_state_barrier, 1);
2900
+ while (atomic_flag_test_and_set(&g_state_critical)) {
2901
+ // spin
2902
+ sched_yield();
2897
2903
  }
2898
2904
  }
2899
2905
 
2900
2906
  // TODO: make this somehow automatically executed
2901
2907
  // some sort of "sentry" mechanism
2902
2908
  inline static void ggml_critical_section_end(void) {
2903
- atomic_fetch_sub(&g_state_barrier, 1);
2909
+ atomic_flag_clear(&g_state_critical);
2904
2910
  }
2905
2911
 
2906
2912
  #if defined(__gnu_linux__)
@@ -3216,7 +3222,11 @@ GGML_CALL bool ggml_is_contiguous(const struct ggml_tensor * tensor) {
3216
3222
  tensor->nb[3] == tensor->nb[2]*tensor->ne[2];
3217
3223
  }
3218
3224
 
3219
- static inline bool ggml_is_contiguous_except_dim_1(const struct ggml_tensor * tensor) {
3225
+ GGML_CALL bool ggml_is_contiguous_0(const struct ggml_tensor * tensor) {
3226
+ return ggml_is_contiguous(tensor);
3227
+ }
3228
+
3229
+ GGML_CALL bool ggml_is_contiguous_1(const struct ggml_tensor * tensor) {
3220
3230
  static_assert(GGML_MAX_DIMS == 4, "GGML_MAX_DIMS is not 4 - update this function");
3221
3231
 
3222
3232
  return
@@ -3225,6 +3235,14 @@ static inline bool ggml_is_contiguous_except_dim_1(const struct ggml_tensor * te
3225
3235
  tensor->nb[3] == tensor->nb[2]*tensor->ne[2];
3226
3236
  }
3227
3237
 
3238
+ GGML_CALL bool ggml_is_contiguous_2(const struct ggml_tensor * tensor) {
3239
+ static_assert(GGML_MAX_DIMS == 4, "GGML_MAX_DIMS is not 4 - update this function");
3240
+
3241
+ return
3242
+ tensor->nb[0] == ggml_type_size(tensor->type) &&
3243
+ tensor->nb[3] == tensor->nb[2]*tensor->ne[2];
3244
+ }
3245
+
3228
3246
  GGML_CALL bool ggml_is_permuted(const struct ggml_tensor * tensor) {
3229
3247
  static_assert(GGML_MAX_DIMS == 4, "GGML_MAX_DIMS is not 4 - update this function");
3230
3248
 
@@ -3357,10 +3375,6 @@ struct ggml_context * ggml_init(struct ggml_init_params params) {
3357
3375
  GGML_PRINT_DEBUG("%s: g_state initialized in %f ms\n", __func__, (t_end - t_start)/1000.0f);
3358
3376
  }
3359
3377
 
3360
- #if defined(GGML_USE_CLBLAST)
3361
- ggml_cl_init();
3362
- #endif
3363
-
3364
3378
  ggml_setup_op_has_task_pass();
3365
3379
 
3366
3380
  is_first_call = false;
@@ -4882,10 +4896,21 @@ struct ggml_tensor * ggml_repeat_back(
4882
4896
  // ggml_concat
4883
4897
 
4884
4898
  struct ggml_tensor * ggml_concat(
4885
- struct ggml_context* ctx,
4886
- struct ggml_tensor* a,
4887
- struct ggml_tensor* b) {
4888
- GGML_ASSERT(a->ne[0] == b->ne[0] && a->ne[1] == b->ne[1] && a->ne[3] == b->ne[3]);
4899
+ struct ggml_context * ctx,
4900
+ struct ggml_tensor * a,
4901
+ struct ggml_tensor * b,
4902
+ int dim) {
4903
+ GGML_ASSERT(dim >= 0 && dim < GGML_MAX_DIMS);
4904
+
4905
+ int64_t ne[GGML_MAX_DIMS];
4906
+ for (int d = 0; d < GGML_MAX_DIMS; ++d) {
4907
+ if (d == dim) {
4908
+ ne[d] = a->ne[d] + b->ne[d];
4909
+ continue;
4910
+ }
4911
+ GGML_ASSERT(a->ne[d] == b->ne[d]);
4912
+ ne[d] = a->ne[d];
4913
+ }
4889
4914
 
4890
4915
  bool is_node = false;
4891
4916
 
@@ -4893,7 +4918,9 @@ struct ggml_tensor * ggml_concat(
4893
4918
  is_node = true;
4894
4919
  }
4895
4920
 
4896
- struct ggml_tensor * result = ggml_new_tensor_4d(ctx, a->type, a->ne[0], a->ne[1], a->ne[2] + b->ne[2], a->ne[3]);
4921
+ struct ggml_tensor * result = ggml_new_tensor(ctx, a->type, GGML_MAX_DIMS, ne);
4922
+
4923
+ ggml_set_op_params_i32(result, 0, dim);
4897
4924
 
4898
4925
  result->op = GGML_OP_CONCAT;
4899
4926
  result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
@@ -5013,6 +5040,7 @@ struct ggml_tensor * ggml_leaky_relu(
5013
5040
  }
5014
5041
 
5015
5042
  struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
5043
+
5016
5044
  ggml_set_op_params(result, &negative_slope, sizeof(negative_slope));
5017
5045
 
5018
5046
  result->op = GGML_OP_LEAKY_RELU;
@@ -6222,16 +6250,13 @@ static struct ggml_tensor * ggml_rope_impl(
6222
6250
  struct ggml_tensor * c,
6223
6251
  int n_dims,
6224
6252
  int mode,
6225
- int n_ctx,
6226
- int n_orig_ctx,
6253
+ int n_ctx_orig,
6227
6254
  float freq_base,
6228
6255
  float freq_scale,
6229
6256
  float ext_factor,
6230
6257
  float attn_factor,
6231
6258
  float beta_fast,
6232
6259
  float beta_slow,
6233
- float xpos_base,
6234
- bool xpos_down,
6235
6260
  bool inplace) {
6236
6261
  GGML_ASSERT((mode & 1) == 0 && "mode & 1 == 1 is no longer supported");
6237
6262
 
@@ -6252,15 +6277,13 @@ static struct ggml_tensor * ggml_rope_impl(
6252
6277
 
6253
6278
  struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
6254
6279
 
6255
- int32_t params[13] = { /*n_past*/ 0, n_dims, mode, n_ctx, n_orig_ctx };
6280
+ int32_t params[11] = { /*n_past*/ 0, n_dims, mode, /*n_ctx*/ 0, n_ctx_orig };
6256
6281
  memcpy(params + 5, &freq_base, sizeof(float));
6257
6282
  memcpy(params + 6, &freq_scale, sizeof(float));
6258
6283
  memcpy(params + 7, &ext_factor, sizeof(float));
6259
6284
  memcpy(params + 8, &attn_factor, sizeof(float));
6260
6285
  memcpy(params + 9, &beta_fast, sizeof(float));
6261
6286
  memcpy(params + 10, &beta_slow, sizeof(float));
6262
- memcpy(params + 11, &xpos_base, sizeof(float));
6263
- memcpy(params + 12, &xpos_down, sizeof(bool));
6264
6287
  ggml_set_op_params(result, params, sizeof(params));
6265
6288
 
6266
6289
  result->op = GGML_OP_ROPE;
@@ -6277,10 +6300,9 @@ struct ggml_tensor * ggml_rope(
6277
6300
  struct ggml_tensor * a,
6278
6301
  struct ggml_tensor * b,
6279
6302
  int n_dims,
6280
- int mode,
6281
- int n_ctx) {
6303
+ int mode) {
6282
6304
  return ggml_rope_impl(
6283
- ctx, a, b, NULL, n_dims, mode, n_ctx, 0, 10000.0f, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f, 0.0f, false, false
6305
+ ctx, a, b, NULL, n_dims, mode, 0, 10000.0f, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f, false
6284
6306
  );
6285
6307
  }
6286
6308
 
@@ -6289,10 +6311,9 @@ struct ggml_tensor * ggml_rope_inplace(
6289
6311
  struct ggml_tensor * a,
6290
6312
  struct ggml_tensor * b,
6291
6313
  int n_dims,
6292
- int mode,
6293
- int n_ctx) {
6314
+ int mode) {
6294
6315
  return ggml_rope_impl(
6295
- ctx, a, b, NULL, n_dims, mode, n_ctx, 0, 10000.0f, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f, 0.0f, false, true
6316
+ ctx, a, b, NULL, n_dims, mode, 0, 10000.0f, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f, true
6296
6317
  );
6297
6318
  }
6298
6319
 
@@ -6303,8 +6324,7 @@ struct ggml_tensor * ggml_rope_ext(
6303
6324
  struct ggml_tensor * c,
6304
6325
  int n_dims,
6305
6326
  int mode,
6306
- int n_ctx,
6307
- int n_orig_ctx,
6327
+ int n_ctx_orig,
6308
6328
  float freq_base,
6309
6329
  float freq_scale,
6310
6330
  float ext_factor,
@@ -6312,8 +6332,8 @@ struct ggml_tensor * ggml_rope_ext(
6312
6332
  float beta_fast,
6313
6333
  float beta_slow) {
6314
6334
  return ggml_rope_impl(
6315
- ctx, a, b, c, n_dims, mode, n_ctx, n_orig_ctx, freq_base, freq_scale,
6316
- ext_factor, attn_factor, beta_fast, beta_slow, 0.0f, false, false
6335
+ ctx, a, b, c, n_dims, mode, n_ctx_orig, freq_base, freq_scale,
6336
+ ext_factor, attn_factor, beta_fast, beta_slow, false
6317
6337
  );
6318
6338
  }
6319
6339
 
@@ -6324,8 +6344,7 @@ struct ggml_tensor * ggml_rope_ext_inplace(
6324
6344
  struct ggml_tensor * c,
6325
6345
  int n_dims,
6326
6346
  int mode,
6327
- int n_ctx,
6328
- int n_orig_ctx,
6347
+ int n_ctx_orig,
6329
6348
  float freq_base,
6330
6349
  float freq_scale,
6331
6350
  float ext_factor,
@@ -6333,8 +6352,8 @@ struct ggml_tensor * ggml_rope_ext_inplace(
6333
6352
  float beta_fast,
6334
6353
  float beta_slow) {
6335
6354
  return ggml_rope_impl(
6336
- ctx, a, b, c, n_dims, mode, n_ctx, n_orig_ctx, freq_base, freq_scale,
6337
- ext_factor, attn_factor, beta_fast, beta_slow, 0.0f, false, true
6355
+ ctx, a, b, c, n_dims, mode, n_ctx_orig, freq_base, freq_scale,
6356
+ ext_factor, attn_factor, beta_fast, beta_slow, true
6338
6357
  );
6339
6358
  }
6340
6359
 
@@ -6344,8 +6363,7 @@ struct ggml_tensor * ggml_rope_custom(
6344
6363
  struct ggml_tensor * b,
6345
6364
  int n_dims,
6346
6365
  int mode,
6347
- int n_ctx,
6348
- int n_orig_ctx,
6366
+ int n_ctx_orig,
6349
6367
  float freq_base,
6350
6368
  float freq_scale,
6351
6369
  float ext_factor,
@@ -6353,8 +6371,8 @@ struct ggml_tensor * ggml_rope_custom(
6353
6371
  float beta_fast,
6354
6372
  float beta_slow) {
6355
6373
  return ggml_rope_impl(
6356
- ctx, a, b, NULL, n_dims, mode, n_ctx, n_orig_ctx, freq_base, freq_scale,
6357
- ext_factor, attn_factor, beta_fast, beta_slow, 0.0f, false, false
6374
+ ctx, a, b, NULL, n_dims, mode, n_ctx_orig, freq_base, freq_scale,
6375
+ ext_factor, attn_factor, beta_fast, beta_slow, false
6358
6376
  );
6359
6377
  }
6360
6378
 
@@ -6364,8 +6382,7 @@ struct ggml_tensor * ggml_rope_custom_inplace(
6364
6382
  struct ggml_tensor * b,
6365
6383
  int n_dims,
6366
6384
  int mode,
6367
- int n_ctx,
6368
- int n_orig_ctx,
6385
+ int n_ctx_orig,
6369
6386
  float freq_base,
6370
6387
  float freq_scale,
6371
6388
  float ext_factor,
@@ -6373,8 +6390,8 @@ struct ggml_tensor * ggml_rope_custom_inplace(
6373
6390
  float beta_fast,
6374
6391
  float beta_slow) {
6375
6392
  return ggml_rope_impl(
6376
- ctx, a, b, NULL, n_dims, mode, n_ctx, n_orig_ctx, freq_base, freq_scale,
6377
- ext_factor, attn_factor, beta_fast, beta_slow, 0.0f, false, true
6393
+ ctx, a, b, NULL, n_dims, mode, n_ctx_orig, freq_base, freq_scale,
6394
+ ext_factor, attn_factor, beta_fast, beta_slow, true
6378
6395
  );
6379
6396
  }
6380
6397
 
@@ -6387,16 +6404,13 @@ struct ggml_tensor * ggml_rope_back(
6387
6404
  struct ggml_tensor * c,
6388
6405
  int n_dims,
6389
6406
  int mode,
6390
- int n_ctx,
6391
- int n_orig_ctx,
6407
+ int n_ctx_orig,
6392
6408
  float freq_base,
6393
6409
  float freq_scale,
6394
6410
  float ext_factor,
6395
6411
  float attn_factor,
6396
6412
  float beta_fast,
6397
- float beta_slow,
6398
- float xpos_base,
6399
- bool xpos_down) {
6413
+ float beta_slow) {
6400
6414
  GGML_ASSERT(ggml_is_vector(b));
6401
6415
  GGML_ASSERT(b->type == GGML_TYPE_I32);
6402
6416
  GGML_ASSERT(a->ne[2] == b->ne[0]);
@@ -6412,15 +6426,13 @@ struct ggml_tensor * ggml_rope_back(
6412
6426
 
6413
6427
  struct ggml_tensor * result = ggml_dup_tensor(ctx, a);
6414
6428
 
6415
- int32_t params[13] = { /*n_past*/ 0, n_dims, mode, n_ctx, n_orig_ctx };
6429
+ int32_t params[11] = { /*n_past*/ 0, n_dims, mode, /*n_ctx*/ 0, n_ctx_orig };
6416
6430
  memcpy(params + 5, &freq_base, sizeof(float));
6417
6431
  memcpy(params + 6, &freq_scale, sizeof(float));
6418
6432
  memcpy(params + 7, &ext_factor, sizeof(float));
6419
6433
  memcpy(params + 8, &attn_factor, sizeof(float));
6420
6434
  memcpy(params + 9, &beta_fast, sizeof(float));
6421
6435
  memcpy(params + 10, &beta_slow, sizeof(float));
6422
- memcpy(params + 11, &xpos_base, sizeof(float));
6423
- memcpy(params + 12, &xpos_down, sizeof(bool));
6424
6436
  ggml_set_op_params(result, params, sizeof(params));
6425
6437
 
6426
6438
  result->op = GGML_OP_ROPE_BACK;
@@ -9006,17 +9018,6 @@ static void ggml_compute_forward_add_f32(
9006
9018
  const int ith = params->ith;
9007
9019
  const int nth = params->nth;
9008
9020
 
9009
- #ifdef GGML_USE_CLBLAST
9010
- if (src1->backend == GGML_BACKEND_TYPE_GPU) {
9011
- // TODO: OpenCL kernel support full broadcast
9012
- GGML_ASSERT(ggml_can_repeat_rows(src1, src0));
9013
- if (ith == 0) {
9014
- ggml_cl_add(src0, src1, dst);
9015
- }
9016
- return;
9017
- }
9018
- #endif
9019
-
9020
9021
  const int nr = ggml_nrows(src0);
9021
9022
 
9022
9023
  GGML_TENSOR_BINARY_OP_LOCALS
@@ -10124,17 +10125,6 @@ static void ggml_compute_forward_mul_f32(
10124
10125
  const int ith = params->ith;
10125
10126
  const int nth = params->nth;
10126
10127
 
10127
- #if defined(GGML_USE_CLBLAST)
10128
- if (src1->backend == GGML_BACKEND_TYPE_GPU) {
10129
- // TODO: OpenCL kernel support full broadcast
10130
- GGML_ASSERT(ggml_can_repeat_rows(src1, src0));
10131
- if (ith == 0) {
10132
- ggml_cl_mul(src0, src1, dst);
10133
- }
10134
- return;
10135
- }
10136
- #endif
10137
-
10138
10128
  const int64_t nr = ggml_nrows(src0);
10139
10129
 
10140
10130
  GGML_TENSOR_BINARY_OP_LOCALS
@@ -10967,26 +10957,29 @@ static void ggml_compute_forward_concat_f32(
10967
10957
  GGML_ASSERT(nb00 == sizeof(float));
10968
10958
  GGML_ASSERT(nb10 == sizeof(float));
10969
10959
 
10960
+ const int32_t dim = ggml_get_op_params_i32(dst, 0);
10961
+
10962
+ GGML_ASSERT(dim >= 0 && dim < 4);
10963
+
10964
+ int64_t o[4] = {0, 0, 0, 0};
10965
+ o[dim] = src0->ne[dim];
10966
+
10967
+ const float * x;
10968
+
10969
+ // TODO: smarter multi-theading
10970
10970
  for (int i3 = 0; i3 < ne3; i3++) {
10971
10971
  for (int i2 = ith; i2 < ne2; i2 += nth) {
10972
- if (i2 < ne02) { // src0
10973
- for (int i1 = 0; i1 < ne1; i1++) {
10974
- for (int i0 = 0; i0 < ne0; i0++) {
10975
- const float * x = (float *)((char *) src0->data + i0 * nb00 + i1 * nb01 + i2 * nb02 + i3 * nb03);
10976
-
10977
- float * y = (float *)((char *)dst->data + i0 * nb0 + i1 * nb1 + i2 * nb2 + i3 * nb3);
10978
- *y = *x;
10979
- }
10980
- }
10981
- } // src1
10982
- else {
10983
- for (int i1 = 0; i1 < ne1; i1++) {
10984
- for (int i0 = 0; i0 < ne0; i0++) {
10985
- const float * x = (float *)((char *) src1->data + i0 * nb10 + i1 * nb11 + (i2 - ne02) * nb12 + i3 * nb13);
10986
-
10987
- float * y = (float *)((char *)dst->data + i0 * nb0 + i1 * nb1 + i2 * nb2 + i3 * nb3);
10988
- *y = *x;
10972
+ for (int i1 = 0; i1 < ne1; i1++) {
10973
+ for (int i0 = 0; i0 < ne0; i0++) {
10974
+ if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) {
10975
+ x = (const float *) ((const char *)src0->data + (i0 )*nb00 + (i1 )*nb01 + (i2 )*nb02 + (i3 )*nb03);
10976
+ } else {
10977
+ x = (const float *) ((const char *)src1->data + (i0 - o[0])*nb10 + (i1 - o[1])*nb11 + (i2 - o[2])*nb12 + (i3 - o[3])*nb13);
10989
10978
  }
10979
+
10980
+ float * y = (float *)((char *)dst->data + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3);
10981
+
10982
+ *y = *x;
10990
10983
  }
10991
10984
  }
10992
10985
  }
@@ -10994,8 +10987,8 @@ static void ggml_compute_forward_concat_f32(
10994
10987
  }
10995
10988
 
10996
10989
  static void ggml_compute_forward_concat(
10997
- const struct ggml_compute_params* params,
10998
- struct ggml_tensor* dst) {
10990
+ const struct ggml_compute_params * params,
10991
+ struct ggml_tensor * dst) {
10999
10992
 
11000
10993
  const struct ggml_tensor * src0 = dst->src[0];
11001
10994
 
@@ -11388,8 +11381,8 @@ static void ggml_compute_forward_gelu_f32(
11388
11381
 
11389
11382
  const struct ggml_tensor * src0 = dst->src[0];
11390
11383
 
11391
- GGML_ASSERT(ggml_is_contiguous_except_dim_1(src0));
11392
- GGML_ASSERT(ggml_is_contiguous_except_dim_1(dst));
11384
+ GGML_ASSERT(ggml_is_contiguous_1(src0));
11385
+ GGML_ASSERT(ggml_is_contiguous_1(dst));
11393
11386
  GGML_ASSERT(ggml_are_same_shape(src0, dst));
11394
11387
 
11395
11388
  if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
@@ -11451,8 +11444,8 @@ static void ggml_compute_forward_gelu_quick_f32(
11451
11444
 
11452
11445
  const struct ggml_tensor * src0 = dst->src[0];
11453
11446
 
11454
- GGML_ASSERT(ggml_is_contiguous_except_dim_1(src0));
11455
- GGML_ASSERT(ggml_is_contiguous_except_dim_1(dst));
11447
+ GGML_ASSERT(ggml_is_contiguous_1(src0));
11448
+ GGML_ASSERT(ggml_is_contiguous_1(dst));
11456
11449
  GGML_ASSERT(ggml_are_same_shape(src0, dst));
11457
11450
 
11458
11451
  if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
@@ -11514,8 +11507,8 @@ static void ggml_compute_forward_silu_f32(
11514
11507
 
11515
11508
  const struct ggml_tensor * src0 = dst->src[0];
11516
11509
 
11517
- GGML_ASSERT(ggml_is_contiguous_except_dim_1(src0));
11518
- GGML_ASSERT(ggml_is_contiguous_except_dim_1(dst));
11510
+ GGML_ASSERT(ggml_is_contiguous_1(src0));
11511
+ GGML_ASSERT(ggml_is_contiguous_1(dst));
11519
11512
  GGML_ASSERT(ggml_are_same_shape(src0, dst));
11520
11513
 
11521
11514
  if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
@@ -11626,9 +11619,9 @@ static void ggml_compute_forward_silu_back_f32(
11626
11619
  const struct ggml_tensor * src0 = dst->src[0];
11627
11620
  const struct ggml_tensor * grad = dst->src[1];
11628
11621
 
11629
- GGML_ASSERT(ggml_is_contiguous_except_dim_1(grad));
11630
- GGML_ASSERT(ggml_is_contiguous_except_dim_1(src0));
11631
- GGML_ASSERT(ggml_is_contiguous_except_dim_1(dst));
11622
+ GGML_ASSERT(ggml_is_contiguous_1(grad));
11623
+ GGML_ASSERT(ggml_is_contiguous_1(src0));
11624
+ GGML_ASSERT(ggml_is_contiguous_1(dst));
11632
11625
  GGML_ASSERT(ggml_are_same_shape(src0, dst));
11633
11626
  GGML_ASSERT(ggml_are_same_shape(src0, grad));
11634
11627
 
@@ -12367,15 +12360,6 @@ static void ggml_compute_forward_mul_mat(
12367
12360
  // nb01 >= nb00 - src0 is not transposed
12368
12361
  // compute by src0 rows
12369
12362
 
12370
- #if defined(GGML_USE_CLBLAST)
12371
- if (ggml_cl_can_mul_mat(src0, src1, dst)) {
12372
- if (params->ith == 0 && params->type == GGML_TASK_TYPE_COMPUTE) {
12373
- ggml_cl_mul_mat(src0, src1, dst, params->wdata, params->wsize);
12374
- }
12375
- return;
12376
- }
12377
- #endif
12378
-
12379
12363
  #if defined(GGML_USE_ACCELERATE) || defined(GGML_USE_OPENBLAS)
12380
12364
  if (ggml_compute_forward_mul_mat_use_blas(dst)) {
12381
12365
  const int64_t ne_plane = ne01*ne00;
@@ -12823,8 +12807,6 @@ static void ggml_compute_forward_out_prod_f32(
12823
12807
  // nb01 >= nb00 - src0 is not transposed
12824
12808
  // compute by src0 rows
12825
12809
 
12826
- // TODO: #if defined(GGML_USE_CLBLAST)
12827
-
12828
12810
  #if defined(GGML_USE_ACCELERATE) || defined(GGML_USE_OPENBLAS)
12829
12811
  bool use_blas = ggml_is_matrix(src0) &&
12830
12812
  ggml_is_matrix(src1) &&
@@ -13022,7 +13004,7 @@ static void ggml_compute_forward_out_prod_q_f32(
13022
13004
  // nb01 >= nb00 - src0 is not transposed
13023
13005
  // compute by src0 rows
13024
13006
 
13025
- // TODO: #if defined(GGML_USE_ACCELERATE) || defined(GGML_USE_OPENBLAS) || defined(GGML_USE_CLBLAST)
13007
+ // TODO: #if defined(GGML_USE_ACCELERATE) || defined(GGML_USE_OPENBLAS)
13026
13008
 
13027
13009
  if (params->type == GGML_TASK_TYPE_INIT) {
13028
13010
  if (ith != 0) {
@@ -14219,8 +14201,7 @@ static float rope_yarn_ramp(const float low, const float high, const int i0) {
14219
14201
  // MIT licensed. Copyright (c) 2023 Jeffrey Quesnelle and Bowen Peng.
14220
14202
  static void rope_yarn(
14221
14203
  float theta_extrap, float freq_scale, float corr_dims[2], int64_t i0, float ext_factor, float mscale,
14222
- float * cos_theta, float * sin_theta
14223
- ) {
14204
+ float * cos_theta, float * sin_theta) {
14224
14205
  // Get n-d rotational scaling corrected for extrapolation
14225
14206
  float theta_interp = freq_scale * theta_extrap;
14226
14207
  float theta = theta_interp;
@@ -14237,18 +14218,19 @@ static void rope_yarn(
14237
14218
 
14238
14219
  // Apparently solving `n_rot = 2pi * x * base^((2 * max_pos_emb) / n_dims)` for x, we get
14239
14220
  // `corr_dim(n_rot) = n_dims * log(max_pos_emb / (n_rot * 2pi)) / (2 * log(base))`
14240
- static float ggml_rope_yarn_corr_dim(int n_dims, int n_orig_ctx, float n_rot, float base) {
14241
- return n_dims * logf(n_orig_ctx / (n_rot * 2 * (float)M_PI)) / (2 * logf(base));
14221
+ static float ggml_rope_yarn_corr_dim(int n_dims, int n_ctx_orig, float n_rot, float base) {
14222
+ return n_dims * logf(n_ctx_orig / (n_rot * 2 * (float)M_PI)) / (2 * logf(base));
14242
14223
  }
14243
14224
 
14244
14225
  static void ggml_rope_cache_init(
14245
- float theta_base, float freq_scale, float corr_dims[2], int64_t ne0, float ext_factor, float mscale,
14246
- float * cache, float sin_sign, float theta_scale
14247
- ) {
14226
+ float theta_base, float freq_scale, const float * freq_factors, float corr_dims[2], int64_t ne0, float ext_factor, float mscale,
14227
+ float * cache, float sin_sign, float theta_scale) {
14228
+ // ref: https://github.com/jquesnelle/yarn/blob/master/scaled_rope/LlamaYaRNScaledRotaryEmbedding.py
14248
14229
  float theta = theta_base;
14249
14230
  for (int64_t i0 = 0; i0 < ne0; i0 += 2) {
14231
+ const float ff = freq_factors ? freq_factors[i0/2] : 1.0f;
14250
14232
  rope_yarn(
14251
- theta, freq_scale, corr_dims, i0, ext_factor, mscale, &cache[i0 + 0], &cache[i0 + 1]
14233
+ theta/ff, freq_scale, corr_dims, i0, ext_factor, mscale, &cache[i0 + 0], &cache[i0 + 1]
14252
14234
  );
14253
14235
  cache[i0 + 1] *= sin_sign;
14254
14236
 
@@ -14257,11 +14239,11 @@ static void ggml_rope_cache_init(
14257
14239
  }
14258
14240
 
14259
14241
  GGML_CALL void ggml_rope_yarn_corr_dims(
14260
- int n_dims, int n_orig_ctx, float freq_base, float beta_fast, float beta_slow, float dims[2]
14242
+ int n_dims, int n_ctx_orig, float freq_base, float beta_fast, float beta_slow, float dims[2]
14261
14243
  ) {
14262
14244
  // start and end correction dims
14263
- float start = floorf(ggml_rope_yarn_corr_dim(n_dims, n_orig_ctx, beta_fast, freq_base));
14264
- float end = ceilf(ggml_rope_yarn_corr_dim(n_dims, n_orig_ctx, beta_slow, freq_base));
14245
+ float start = floorf(ggml_rope_yarn_corr_dim(n_dims, n_ctx_orig, beta_fast, freq_base));
14246
+ float end = ceilf(ggml_rope_yarn_corr_dim(n_dims, n_ctx_orig, beta_slow, freq_base));
14265
14247
  dims[0] = MAX(0, start);
14266
14248
  dims[1] = MIN(n_dims - 1, end);
14267
14249
  }
@@ -14281,15 +14263,11 @@ static void ggml_compute_forward_rope_f32(
14281
14263
 
14282
14264
  float freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow;
14283
14265
 
14284
- // these two only relevant for xPos RoPE:
14285
- float xpos_base;
14286
- bool xpos_down;
14287
-
14288
14266
  //const int n_past = ((int32_t *) dst->op_params)[0];
14289
14267
  const int n_dims = ((int32_t *) dst->op_params)[1];
14290
14268
  const int mode = ((int32_t *) dst->op_params)[2];
14291
- const int n_ctx = ((int32_t *) dst->op_params)[3];
14292
- const int n_orig_ctx = ((int32_t *) dst->op_params)[4];
14269
+ //const int n_ctx = ((int32_t *) dst->op_params)[3];
14270
+ const int n_ctx_orig = ((int32_t *) dst->op_params)[4];
14293
14271
 
14294
14272
  memcpy(&freq_base, (int32_t *) dst->op_params + 5, sizeof(float));
14295
14273
  memcpy(&freq_scale, (int32_t *) dst->op_params + 6, sizeof(float));
@@ -14297,8 +14275,6 @@ static void ggml_compute_forward_rope_f32(
14297
14275
  memcpy(&attn_factor, (int32_t *) dst->op_params + 8, sizeof(float));
14298
14276
  memcpy(&beta_fast, (int32_t *) dst->op_params + 9, sizeof(float));
14299
14277
  memcpy(&beta_slow, (int32_t *) dst->op_params + 10, sizeof(float));
14300
- memcpy(&xpos_base, (int32_t *) dst->op_params + 11, sizeof(float));
14301
- memcpy(&xpos_down, (int32_t *) dst->op_params + 12, sizeof(bool));
14302
14278
 
14303
14279
  GGML_TENSOR_UNARY_OP_LOCALS
14304
14280
 
@@ -14326,22 +14302,17 @@ static void ggml_compute_forward_rope_f32(
14326
14302
  int ir = 0;
14327
14303
 
14328
14304
  const float theta_scale = powf(freq_base, -2.0f/n_dims);
14329
- const float inv_ndims = -1.f/n_dims;
14305
+
14330
14306
  float corr_dims[2];
14331
- ggml_rope_yarn_corr_dims(n_dims, n_orig_ctx, freq_base, beta_fast, beta_slow, corr_dims);
14307
+ ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, corr_dims);
14332
14308
 
14333
14309
  const bool is_neox = mode & 2;
14334
- const bool is_glm = mode & 4;
14335
14310
 
14336
14311
  const float * freq_factors = NULL;
14337
- if (is_neox) {
14338
- if (src2 != NULL) {
14339
- GGML_ASSERT(src2->type == GGML_TYPE_F32);
14340
- GGML_ASSERT(src2->ne[0] >= n_dims / 2);
14341
- freq_factors = (const float *) src2->data;
14342
- }
14343
- } else {
14344
- GGML_ASSERT(src2 == NULL && "TODO: freq_factors not implemented for !is_neox");
14312
+ if (src2 != NULL) {
14313
+ GGML_ASSERT(src2->type == GGML_TYPE_F32);
14314
+ GGML_ASSERT(src2->ne[0] >= n_dims / 2);
14315
+ freq_factors = (const float *) src2->data;
14345
14316
  }
14346
14317
 
14347
14318
  // backward process uses inverse rotation by cos and sin.
@@ -14356,101 +14327,50 @@ static void ggml_compute_forward_rope_f32(
14356
14327
  const int64_t p = pos[i2];
14357
14328
 
14358
14329
  float * cache = (float *) params->wdata + (ne0 + CACHE_LINE_SIZE_F32)*ith;
14359
- if (!is_glm && !is_neox) { // TODO: cache sin/cos for glm, neox
14360
- ggml_rope_cache_init(p, freq_scale, corr_dims, ne0, ext_factor, attn_factor, cache, sin_sign, theta_scale);
14361
- }
14330
+ ggml_rope_cache_init(p, freq_scale, freq_factors, corr_dims, ne0, ext_factor, attn_factor, cache, sin_sign, theta_scale);
14362
14331
 
14363
14332
  for (int64_t i1 = 0; i1 < ne1; i1++) {
14364
14333
  if (ir++ < ir0) continue;
14365
14334
  if (ir > ir1) break;
14366
14335
 
14367
- float theta_base = (float)p;
14368
-
14369
- if (is_glm) {
14370
- theta_base = MIN(p, n_ctx - 2);
14371
- float block_theta = MAX(p - (n_ctx - 2), 0);
14372
- for (int64_t i0 = 0; i0 < ne0 / 4; i0++) {
14373
- const float cos_theta = cosf(theta_base);
14374
- const float sin_theta = sinf(theta_base) * sin_sign;
14375
- const float cos_block_theta = cosf(block_theta);
14376
- const float sin_block_theta = sinf(block_theta) * sin_sign;
14377
-
14378
- theta_base *= theta_scale;
14379
- block_theta *= theta_scale;
14380
-
14381
- const float * const src = (float *)((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);
14382
- float * dst_data = (float *)((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);
14383
-
14384
- const float x0 = src[0];
14385
- const float x1 = src[n_dims/2];
14386
- const float x2 = src[n_dims];
14387
- const float x3 = src[n_dims/2*3];
14388
-
14389
- dst_data[0] = x0*cos_theta - x1*sin_theta;
14390
- dst_data[n_dims/2] = x0*sin_theta + x1*cos_theta;
14391
- dst_data[n_dims] = x2*cos_block_theta - x3*sin_block_theta;
14392
- dst_data[n_dims/2*3] = x2*sin_block_theta + x3*cos_block_theta;
14393
- }
14394
- } else if (!is_neox) {
14395
- for (int64_t i0 = 0; i0 < ne0; i0 += 2) {
14336
+ if (!is_neox) {
14337
+ for (int64_t i0 = 0; i0 < n_dims; i0 += 2) {
14396
14338
  const float cos_theta = cache[i0 + 0];
14397
14339
  const float sin_theta = cache[i0 + 1];
14398
14340
 
14399
- // zeta scaling for xPos only:
14400
- float zeta = xpos_base != 0.0f ? powf((i0 + 0.4f * ne0) / (1.4f * ne0), p / xpos_base) : 1.0f;
14401
- if (xpos_down) zeta = 1.0f / zeta;
14402
-
14403
14341
  const float * const src = (float *)((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);
14404
14342
  float * dst_data = (float *)((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);
14405
14343
 
14406
14344
  const float x0 = src[0];
14407
14345
  const float x1 = src[1];
14408
14346
 
14409
- dst_data[0] = x0*cos_theta*zeta - x1*sin_theta*zeta;
14410
- dst_data[1] = x0*sin_theta*zeta + x1*cos_theta*zeta;
14347
+ dst_data[0] = x0*cos_theta - x1*sin_theta;
14348
+ dst_data[1] = x0*sin_theta + x1*cos_theta;
14411
14349
  }
14412
14350
  } else {
14413
- // TODO: this might be wrong for ne0 != n_dims - need double check
14414
- // it seems we have to rope just the first n_dims elements and do nothing with the rest
14415
- // ref: https://github.com/ml-explore/mlx/blob/dc2edc762c797e3b8de50b1dad4dc0a131691033/benchmarks/python/llama_jax_bench.py#L11-L26
14416
- theta_base *= freq_scale;
14417
- for (int64_t ic = 0; ic < ne0; ic += 2) {
14418
- if (ic < n_dims) {
14419
- const int64_t ib = 0;
14420
-
14421
- // simplified from `(ib * n_dims + ic) * inv_ndims`
14422
- float cur_rot = inv_ndims * ic - ib;
14423
- float freq_factor = freq_factors ? freq_factors[ic/2] : 1.0f;
14351
+ for (int64_t i0 = 0; i0 < n_dims; i0 += 2) {
14352
+ const int64_t ic = i0/2;
14424
14353
 
14425
- float cos_theta, sin_theta;
14426
- rope_yarn(
14427
- theta_base/freq_factor, freq_scale, corr_dims, cur_rot, ext_factor, attn_factor,
14428
- &cos_theta, &sin_theta
14429
- );
14430
- sin_theta *= sin_sign;
14431
-
14432
- theta_base *= theta_scale;
14433
-
14434
- const int64_t i0 = ib*n_dims + ic/2;
14354
+ const float cos_theta = cache[i0 + 0];
14355
+ const float sin_theta = cache[i0 + 1];
14435
14356
 
14436
- const float * const src = (float *)((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);
14437
- float * dst_data = (float *)((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);
14357
+ const float * const src = (float *)((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01 + ic*nb00);
14358
+ float * dst_data = (float *)((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 + ic*nb0);
14438
14359
 
14439
- const float x0 = src[0];
14440
- const float x1 = src[n_dims/2];
14360
+ const float x0 = src[0];
14361
+ const float x1 = src[n_dims/2];
14441
14362
 
14442
- dst_data[0] = x0*cos_theta - x1*sin_theta;
14443
- dst_data[n_dims/2] = x0*sin_theta + x1*cos_theta;
14444
- } else {
14445
- const int64_t i0 = ic;
14363
+ dst_data[0] = x0*cos_theta - x1*sin_theta;
14364
+ dst_data[n_dims/2] = x0*sin_theta + x1*cos_theta;
14365
+ }
14366
+ }
14446
14367
 
14447
- const float * const src = (float *)((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);
14448
- float * dst_data = (float *)((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);
14368
+ for (int64_t i0 = n_dims; i0 < ne0; i0 += 2) {
14369
+ const float * const src = (float *)((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);
14370
+ float * dst_data = (float *)((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);
14449
14371
 
14450
- dst_data[0] = src[0];
14451
- dst_data[1] = src[1];
14452
- }
14453
- }
14372
+ dst_data[0] = src[0];
14373
+ dst_data[1] = src[1];
14454
14374
  }
14455
14375
  }
14456
14376
  }
@@ -14476,8 +14396,8 @@ static void ggml_compute_forward_rope_f16(
14476
14396
  //const int n_past = ((int32_t *) dst->op_params)[0];
14477
14397
  const int n_dims = ((int32_t *) dst->op_params)[1];
14478
14398
  const int mode = ((int32_t *) dst->op_params)[2];
14479
- const int n_ctx = ((int32_t *) dst->op_params)[3];
14480
- const int n_orig_ctx = ((int32_t *) dst->op_params)[4];
14399
+ //const int n_ctx = ((int32_t *) dst->op_params)[3];
14400
+ const int n_ctx_orig = ((int32_t *) dst->op_params)[4];
14481
14401
  memcpy(&freq_base, (int32_t *) dst->op_params + 5, sizeof(float));
14482
14402
  memcpy(&freq_scale, (int32_t *) dst->op_params + 6, sizeof(float));
14483
14403
  memcpy(&ext_factor, (int32_t *) dst->op_params + 7, sizeof(float));
@@ -14511,22 +14431,17 @@ static void ggml_compute_forward_rope_f16(
14511
14431
  int ir = 0;
14512
14432
 
14513
14433
  const float theta_scale = powf(freq_base, -2.0f/n_dims);
14514
- const float inv_ndims = -1.f/n_dims;
14434
+
14515
14435
  float corr_dims[2];
14516
- ggml_rope_yarn_corr_dims(n_dims, n_orig_ctx, freq_base, beta_fast, beta_slow, corr_dims);
14436
+ ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, corr_dims);
14517
14437
 
14518
14438
  const bool is_neox = mode & 2;
14519
- const bool is_glm = mode & 4;
14520
14439
 
14521
14440
  const float * freq_factors = NULL;
14522
- if (is_neox) {
14523
- if (src2 != NULL) {
14524
- GGML_ASSERT(src2->type == GGML_TYPE_F32);
14525
- GGML_ASSERT(src2->ne[0] >= n_dims / 2);
14526
- freq_factors = (const float *) src2->data;
14527
- }
14528
- } else {
14529
- GGML_ASSERT(src2 == NULL && "TODO: freq_factors not implemented for !is_neox");
14441
+ if (src2 != NULL) {
14442
+ GGML_ASSERT(src2->type == GGML_TYPE_F32);
14443
+ GGML_ASSERT(src2->ne[0] >= n_dims / 2);
14444
+ freq_factors = (const float *) src2->data;
14530
14445
  }
14531
14446
 
14532
14447
  // backward process uses inverse rotation by cos and sin.
@@ -14541,43 +14456,14 @@ static void ggml_compute_forward_rope_f16(
14541
14456
  const int64_t p = pos[i2];
14542
14457
 
14543
14458
  float * cache = (float *) params->wdata + (ne0 + CACHE_LINE_SIZE_F32)*ith;
14544
- if (!is_glm && !is_neox) { // TODO: cache sin/cos for glm, neox
14545
- ggml_rope_cache_init(p, freq_scale, corr_dims, ne0, ext_factor, attn_factor, cache, sin_sign, theta_scale);
14546
- }
14459
+ ggml_rope_cache_init(p, freq_scale, freq_factors, corr_dims, ne0, ext_factor, attn_factor, cache, sin_sign, theta_scale);
14547
14460
 
14548
14461
  for (int64_t i1 = 0; i1 < ne1; i1++) {
14549
14462
  if (ir++ < ir0) continue;
14550
14463
  if (ir > ir1) break;
14551
14464
 
14552
- float theta_base = (float)p;
14553
-
14554
- if (is_glm) {
14555
- theta_base = MIN(p, n_ctx - 2);
14556
- float block_theta = MAX(p - (n_ctx - 2), 0);
14557
- for (int64_t i0 = 0; i0 < ne0 / 4; i0++) {
14558
- const float cos_theta = cosf(theta_base);
14559
- const float sin_theta = sinf(theta_base) * sin_sign;
14560
- const float cos_block_theta = cosf(block_theta);
14561
- const float sin_block_theta = sinf(block_theta) * sin_sign;
14562
-
14563
- theta_base *= theta_scale;
14564
- block_theta *= theta_scale;
14565
-
14566
- const ggml_fp16_t * const src = (ggml_fp16_t *)((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);
14567
- ggml_fp16_t * dst_data = (ggml_fp16_t *)((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);
14568
-
14569
- const float x0 = GGML_FP16_TO_FP32(src[0]);
14570
- const float x1 = GGML_FP16_TO_FP32(src[n_dims/2]);
14571
- const float x2 = GGML_FP16_TO_FP32(src[n_dims]);
14572
- const float x3 = GGML_FP16_TO_FP32(src[n_dims/2*3]);
14573
-
14574
- dst_data[0] = GGML_FP32_TO_FP16(x0*cos_theta - x1*sin_theta);
14575
- dst_data[n_dims/2] = GGML_FP32_TO_FP16(x0*sin_theta + x1*cos_theta);
14576
- dst_data[n_dims] = GGML_FP32_TO_FP16(x2*cos_block_theta - x3*sin_block_theta);
14577
- dst_data[n_dims/2*3] = GGML_FP32_TO_FP16(x2*sin_block_theta + x3*cos_block_theta);
14578
- }
14579
- } else if (!is_neox) {
14580
- for (int64_t i0 = 0; i0 < ne0; i0 += 2) {
14465
+ if (!is_neox) {
14466
+ for (int64_t i0 = 0; i0 < n_dims; i0 += 2) {
14581
14467
  const float cos_theta = cache[i0 + 0];
14582
14468
  const float sin_theta = cache[i0 + 1];
14583
14469
 
@@ -14591,47 +14477,29 @@ static void ggml_compute_forward_rope_f16(
14591
14477
  dst_data[1] = GGML_FP32_TO_FP16(x0*sin_theta + x1*cos_theta);
14592
14478
  }
14593
14479
  } else {
14594
- // TODO: this might be wrong for ne0 != n_dims - need double check
14595
- // it seems we have to rope just the first n_dims elements and do nothing with the rest
14596
- // ref: https://github.com/ml-explore/mlx/blob/dc2edc762c797e3b8de50b1dad4dc0a131691033/benchmarks/python/llama_jax_bench.py#L11-L26
14597
- theta_base *= freq_scale;
14598
- for (int64_t ic = 0; ic < ne0; ic += 2) {
14599
- if (ic < n_dims) {
14600
- const int64_t ib = 0;
14601
-
14602
- // simplified from `(ib * n_dims + ic) * inv_ndims`
14603
- float cur_rot = inv_ndims * ic - ib;
14604
- float freq_factor = freq_factors ? freq_factors[ic/2] : 1.0f;
14605
-
14606
- float cos_theta, sin_theta;
14607
- rope_yarn(
14608
- theta_base/freq_factor, freq_scale, corr_dims, cur_rot, ext_factor, attn_factor,
14609
- &cos_theta, &sin_theta
14610
- );
14611
- sin_theta *= sin_sign;
14612
-
14613
- theta_base *= theta_scale;
14480
+ for (int64_t i0 = 0; i0 < n_dims; i0 += 2) {
14481
+ const int64_t ic = i0/2;
14614
14482
 
14615
- const int64_t i0 = ib*n_dims + ic/2;
14483
+ const float cos_theta = cache[i0 + 0];
14484
+ const float sin_theta = cache[i0 + 1];
14616
14485
 
14617
- const ggml_fp16_t * const src = (ggml_fp16_t *)((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);
14618
- ggml_fp16_t * dst_data = (ggml_fp16_t *)((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);
14486
+ const ggml_fp16_t * const src = (ggml_fp16_t *)((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01 + ic*nb00);
14487
+ ggml_fp16_t * dst_data = (ggml_fp16_t *)((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 + ic*nb0);
14619
14488
 
14620
- const float x0 = GGML_FP16_TO_FP32(src[0]);
14621
- const float x1 = GGML_FP16_TO_FP32(src[n_dims/2]);
14489
+ const float x0 = GGML_FP16_TO_FP32(src[0]);
14490
+ const float x1 = GGML_FP16_TO_FP32(src[n_dims/2]);
14622
14491
 
14623
- dst_data[0] = GGML_FP32_TO_FP16(x0*cos_theta - x1*sin_theta);
14624
- dst_data[n_dims/2] = GGML_FP32_TO_FP16(x0*sin_theta + x1*cos_theta);
14625
- } else {
14626
- const int64_t i0 = ic;
14492
+ dst_data[0] = GGML_FP32_TO_FP16(x0*cos_theta - x1*sin_theta);
14493
+ dst_data[n_dims/2] = GGML_FP32_TO_FP16(x0*sin_theta + x1*cos_theta);
14494
+ }
14495
+ }
14627
14496
 
14628
- const ggml_fp16_t * const src = (ggml_fp16_t *)((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);
14629
- ggml_fp16_t * dst_data = (ggml_fp16_t *)((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);
14497
+ for (int64_t i0 = n_dims; i0 < ne0; i0 += 2) {
14498
+ const ggml_fp16_t * const src = (ggml_fp16_t *)((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);
14499
+ ggml_fp16_t * dst_data = (ggml_fp16_t *)((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);
14630
14500
 
14631
- dst_data[0] = src[0];
14632
- dst_data[1] = src[1];
14633
- }
14634
- }
14501
+ dst_data[0] = src[0];
14502
+ dst_data[1] = src[1];
14635
14503
  }
14636
14504
  }
14637
14505
  }
@@ -18333,9 +18201,9 @@ static void ggml_compute_backward(struct ggml_context * ctx, struct ggml_tensor
18333
18201
  //const int n_past = ((int32_t *) tensor->op_params)[0];
18334
18202
  const int n_dims = ((int32_t *) tensor->op_params)[1];
18335
18203
  const int mode = ((int32_t *) tensor->op_params)[2];
18336
- const int n_ctx = ((int32_t *) tensor->op_params)[3];
18337
- const int n_orig_ctx = ((int32_t *) tensor->op_params)[4];
18338
- float freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow, xpos_base, xpos_down;
18204
+ //const int n_ctx = ((int32_t *) tensor->op_params)[3];
18205
+ const int n_ctx_orig = ((int32_t *) tensor->op_params)[4];
18206
+ float freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow;
18339
18207
 
18340
18208
  memcpy(&freq_base, (int32_t *) tensor->op_params + 5, sizeof(float));
18341
18209
  memcpy(&freq_scale, (int32_t *) tensor->op_params + 6, sizeof(float));
@@ -18343,8 +18211,6 @@ static void ggml_compute_backward(struct ggml_context * ctx, struct ggml_tensor
18343
18211
  memcpy(&attn_factor, (int32_t *) tensor->op_params + 8, sizeof(float));
18344
18212
  memcpy(&beta_fast, (int32_t *) tensor->op_params + 9, sizeof(float));
18345
18213
  memcpy(&beta_slow, (int32_t *) tensor->op_params + 10, sizeof(float));
18346
- memcpy(&xpos_base, (int32_t *) tensor->op_params + 11, sizeof(float));
18347
- memcpy(&xpos_down, (int32_t *) tensor->op_params + 12, sizeof(bool));
18348
18214
 
18349
18215
  src0->grad = ggml_add_or_set(ctx,
18350
18216
  src0->grad,
@@ -18354,16 +18220,13 @@ static void ggml_compute_backward(struct ggml_context * ctx, struct ggml_tensor
18354
18220
  src2,
18355
18221
  n_dims,
18356
18222
  mode,
18357
- n_ctx,
18358
- n_orig_ctx,
18223
+ n_ctx_orig,
18359
18224
  freq_base,
18360
18225
  freq_scale,
18361
18226
  ext_factor,
18362
18227
  attn_factor,
18363
18228
  beta_fast,
18364
- beta_slow,
18365
- xpos_base,
18366
- xpos_down),
18229
+ beta_slow),
18367
18230
  zero_table);
18368
18231
  }
18369
18232
  } break;
@@ -18373,9 +18236,9 @@ static void ggml_compute_backward(struct ggml_context * ctx, struct ggml_tensor
18373
18236
  //const int n_past = ((int32_t *) tensor->op_params)[0];
18374
18237
  const int n_dims = ((int32_t *) tensor->op_params)[1];
18375
18238
  const int mode = ((int32_t *) tensor->op_params)[2];
18376
- const int n_ctx = ((int32_t *) tensor->op_params)[3];
18377
- const int n_orig_ctx = ((int32_t *) tensor->op_params)[4];
18378
- float freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow, xpos_base, xpos_down;
18239
+ //const int n_ctx = ((int32_t *) tensor->op_params)[3];
18240
+ const int n_ctx_orig = ((int32_t *) tensor->op_params)[4];
18241
+ float freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow;
18379
18242
 
18380
18243
  memcpy(&freq_base, (int32_t *) tensor->op_params + 5, sizeof(float));
18381
18244
  memcpy(&freq_scale, (int32_t *) tensor->op_params + 6, sizeof(float));
@@ -18383,8 +18246,6 @@ static void ggml_compute_backward(struct ggml_context * ctx, struct ggml_tensor
18383
18246
  memcpy(&attn_factor, (int32_t *) tensor->op_params + 8, sizeof(float));
18384
18247
  memcpy(&beta_fast, (int32_t *) tensor->op_params + 9, sizeof(float));
18385
18248
  memcpy(&beta_slow, (int32_t *) tensor->op_params + 10, sizeof(float));
18386
- memcpy(&xpos_base, (int32_t *) tensor->op_params + 11, sizeof(float));
18387
- memcpy(&xpos_down, (int32_t *) tensor->op_params + 12, sizeof(bool));
18388
18249
 
18389
18250
  src0->grad = ggml_add_or_set(ctx,
18390
18251
  src0->grad,
@@ -18394,16 +18255,13 @@ static void ggml_compute_backward(struct ggml_context * ctx, struct ggml_tensor
18394
18255
  src2,
18395
18256
  n_dims,
18396
18257
  mode,
18397
- n_ctx,
18398
- n_orig_ctx,
18258
+ n_ctx_orig,
18399
18259
  freq_base,
18400
18260
  freq_scale,
18401
18261
  ext_factor,
18402
18262
  attn_factor,
18403
18263
  beta_fast,
18404
18264
  beta_slow,
18405
- xpos_base,
18406
- xpos_down,
18407
18265
  false),
18408
18266
  zero_table);
18409
18267
  }
@@ -19510,11 +19368,6 @@ struct ggml_cplan ggml_graph_plan(const struct ggml_cgraph * cgraph, int n_threa
19510
19368
  {
19511
19369
  const enum ggml_type vec_dot_type = type_traits[node->src[0]->type].vec_dot_type;
19512
19370
 
19513
- #if defined(GGML_USE_CLBLAST)
19514
- if (ggml_cl_can_mul_mat(node->src[0], node->src[1], node)) {
19515
- cur = ggml_cl_mul_mat_get_wsize(node->src[0], node->src[1], node);
19516
- } else
19517
- #endif
19518
19371
  #if defined(GGML_USE_ACCELERATE) || defined(GGML_USE_OPENBLAS)
19519
19372
  if (ggml_compute_forward_mul_mat_use_blas(node)) {
19520
19373
  if (node->src[0]->type != GGML_TYPE_F32) {
@@ -19644,6 +19497,59 @@ struct ggml_cplan ggml_graph_plan(const struct ggml_cgraph * cgraph, int n_threa
19644
19497
  return cplan;
19645
19498
  }
19646
19499
 
19500
+ static enum ggml_status ggml_graph_compute_parallel(struct ggml_compute_state * workers, int n_threads) {
19501
+ enum ggml_status compute_status = GGML_STATUS_SUCCESS;
19502
+
19503
+ #ifdef GGML_USE_OPENMP
19504
+ if (n_threads > 1) {
19505
+ #pragma omp parallel num_threads(n_threads)
19506
+ {
19507
+ #pragma omp single
19508
+ {
19509
+ // update the number of threads from the actual number of threads that we got from OpenMP
19510
+ n_threads = omp_get_num_threads();
19511
+ workers[0].shared->n_threads = n_threads;
19512
+ workers[0].shared->n_active = n_threads;
19513
+ }
19514
+ ggml_graph_compute_thread(&workers[omp_get_thread_num()]);
19515
+ }
19516
+ } else {
19517
+ ggml_graph_compute_thread(&workers[0]);
19518
+ }
19519
+ #else
19520
+ // create thread pool
19521
+ if (n_threads > 1) {
19522
+ for (int j = 1; j < n_threads; ++j) {
19523
+ const int rc = ggml_thread_create(&workers[j].thrd, NULL, ggml_graph_compute_thread, &workers[j]);
19524
+ GGML_ASSERT(rc == 0);
19525
+ UNUSED(rc);
19526
+ }
19527
+ }
19528
+
19529
+ // this is a work thread too
19530
+ ggml_graph_compute_thread(&workers[0]);
19531
+
19532
+ // join or kill thread pool
19533
+ if (n_threads > 1) {
19534
+ for (int j = 1; j < n_threads; j++) {
19535
+ const int rc = ggml_thread_join(workers[j].thrd, NULL);
19536
+ GGML_ASSERT(rc == 0);
19537
+ UNUSED(rc);
19538
+ }
19539
+ }
19540
+ #endif
19541
+ // don't leave affinity set on the main thread
19542
+ clear_numa_thread_affinity();
19543
+
19544
+ for (int j = 0; j < n_threads; j++) {
19545
+ if (workers[j].ec != GGML_STATUS_SUCCESS) {
19546
+ compute_status = workers[j].ec;
19547
+ break;
19548
+ }
19549
+ }
19550
+ return compute_status;
19551
+ }
19552
+
19647
19553
  enum ggml_status ggml_graph_compute(struct ggml_cgraph * cgraph, struct ggml_cplan * cplan) {
19648
19554
  {
19649
19555
  GGML_ASSERT(cplan);
@@ -19654,7 +19560,11 @@ enum ggml_status ggml_graph_compute(struct ggml_cgraph * cgraph, struct ggml_cpl
19654
19560
  }
19655
19561
  }
19656
19562
 
19657
- const int n_threads = cplan->n_threads;
19563
+ int n_threads = cplan->n_threads;
19564
+
19565
+ #if defined(GGML_USE_OPENMP)
19566
+ n_threads = MIN(n_threads, omp_get_max_threads());
19567
+ #endif
19658
19568
 
19659
19569
  struct ggml_compute_state_shared state_shared = {
19660
19570
  /*.cgraph =*/ cgraph,
@@ -19670,47 +19580,20 @@ enum ggml_status ggml_graph_compute(struct ggml_cgraph * cgraph, struct ggml_cpl
19670
19580
  /*.current_chunk; =*/ 0,
19671
19581
  };
19672
19582
  struct ggml_compute_state * workers = alloca(sizeof(struct ggml_compute_state)*n_threads);
19673
-
19674
- // create thread pool
19675
- if (n_threads > 1) {
19676
- for (int j = 1; j < n_threads; ++j) {
19677
- workers[j] = (struct ggml_compute_state) {
19678
- .thrd = 0,
19679
- .ith = j,
19680
- .shared = &state_shared,
19681
- .ec = GGML_STATUS_SUCCESS,
19682
- };
19683
-
19684
- const int rc = ggml_thread_create(&workers[j].thrd, NULL, ggml_graph_compute_thread, &workers[j]);
19685
- GGML_ASSERT(rc == 0);
19686
- UNUSED(rc);
19687
- }
19688
- }
19689
-
19690
- workers[0].ith = 0;
19691
- workers[0].shared = &state_shared;
19692
- workers[0].ec = GGML_STATUS_SUCCESS;
19693
-
19694
19583
  const int64_t perf_start_cycles = ggml_perf_cycles();
19695
19584
  const int64_t perf_start_time_us = ggml_perf_time_us();
19696
19585
 
19697
- // this is a work thread too
19698
- ggml_graph_compute_thread(&workers[0]);
19699
- enum ggml_status compute_status = workers[0].ec;
19700
-
19701
- // don't leave affinity set on the main thread
19702
- clear_numa_thread_affinity();
19703
-
19704
- // join or kill thread pool
19705
- if (n_threads > 1) {
19706
- for (int j = 1; j < n_threads; j++) {
19707
- const int rc = ggml_thread_join(workers[j].thrd, NULL);
19708
- GGML_ASSERT(rc == 0);
19709
- if (workers[j].ec != GGML_STATUS_SUCCESS)
19710
- compute_status = workers[j].ec;
19711
- }
19586
+ for (int j = 0; j < n_threads; ++j) {
19587
+ workers[j] = (struct ggml_compute_state) {
19588
+ .thrd = 0,
19589
+ .ith = j,
19590
+ .shared = &state_shared,
19591
+ .ec = GGML_STATUS_SUCCESS,
19592
+ };
19712
19593
  }
19713
19594
 
19595
+ enum ggml_status compute_status = ggml_graph_compute_parallel(workers, n_threads);
19596
+
19714
19597
  // performance stats (graph)
19715
19598
  {
19716
19599
  int64_t perf_cycles_cur = ggml_perf_cycles() - perf_start_cycles;
@@ -22742,6 +22625,16 @@ int ggml_cpu_has_neon(void) {
22742
22625
  #endif
22743
22626
  }
22744
22627
 
22628
+ int ggml_cpu_has_sve(void) {
22629
+ #if defined(__ARM_FEATURE_SVE)
22630
+ // TODO: Currently, SVE 256 bit is only supported.
22631
+ GGML_ASSERT(svcntb() == QK8_0);
22632
+ return 1;
22633
+ #else
22634
+ return 0;
22635
+ #endif
22636
+ }
22637
+
22745
22638
  int ggml_cpu_has_arm_fma(void) {
22746
22639
  #if defined(__ARM_FEATURE_FMA)
22747
22640
  return 1;
@@ -22783,7 +22676,7 @@ int ggml_cpu_has_wasm_simd(void) {
22783
22676
  }
22784
22677
 
22785
22678
  int ggml_cpu_has_blas(void) {
22786
- #if defined(GGML_USE_ACCELERATE) || defined(GGML_USE_OPENBLAS) || defined(GGML_USE_CUDA) || defined(GGML_USE_VULKAN) || defined(GGML_USE_CLBLAST) || defined(GGML_USE_SYCL)
22679
+ #if defined(GGML_USE_ACCELERATE) || defined(GGML_USE_OPENBLAS) || defined(GGML_USE_CUDA) || defined(GGML_USE_VULKAN) || defined(GGML_USE_SYCL)
22787
22680
  return 1;
22788
22681
  #else
22789
22682
  return 0;
@@ -22798,14 +22691,6 @@ int ggml_cpu_has_cuda(void) {
22798
22691
  #endif
22799
22692
  }
22800
22693
 
22801
- int ggml_cpu_has_clblast(void) {
22802
- #if defined(GGML_USE_CLBLAST)
22803
- return 1;
22804
- #else
22805
- return 0;
22806
- #endif
22807
- }
22808
-
22809
22694
  int ggml_cpu_has_vulkan(void) {
22810
22695
  #if defined(GGML_USE_VULKAN)
22811
22696
  return 1;
@@ -22830,9 +22715,16 @@ int ggml_cpu_has_sycl(void) {
22830
22715
  #endif
22831
22716
  }
22832
22717
 
22718
+ int ggml_cpu_has_rpc(void) {
22719
+ #if defined(GGML_USE_RPC)
22720
+ return 1;
22721
+ #else
22722
+ return 0;
22723
+ #endif
22724
+ }
22725
+
22833
22726
  int ggml_cpu_has_gpublas(void) {
22834
- return ggml_cpu_has_cuda() || ggml_cpu_has_clblast() || ggml_cpu_has_vulkan() || ggml_cpu_has_kompute() ||
22835
- ggml_cpu_has_sycl();
22727
+ return ggml_cpu_has_cuda() || ggml_cpu_has_vulkan() || ggml_cpu_has_kompute() || ggml_cpu_has_sycl();
22836
22728
  }
22837
22729
 
22838
22730
  int ggml_cpu_has_sse3(void) {