json 2.15.2 → 2.16.0

data/ext/json/ext/simd/simd.h

@@ -1,10 +1,14 @@
+#include "../json.h"
+
 typedef enum {
     SIMD_NONE,
     SIMD_NEON,
     SIMD_SSE2
 } SIMD_Implementation;
 
-#ifdef JSON_ENABLE_SIMD
+#ifndef __has_builtin         // Optional of course.
+  #define __has_builtin(x) 0  // Compatibility with non-clang compilers.
+#endif
 
 #ifdef __clang__
 # if __has_builtin(__builtin_ctzll)
@@ -20,6 +24,8 @@ typedef enum {
 
 static inline uint32_t trailing_zeros64(uint64_t input)
 {
+    JSON_ASSERT(input > 0); // __builtin_ctz(0) is undefined behavior
+
 #if HAVE_BUILTIN_CTZLL
     return __builtin_ctzll(input);
 #else
@@ -35,6 +41,8 @@ static inline uint32_t trailing_zeros64(uint64_t input)
 
 static inline int trailing_zeros(int input)
 {
+    JSON_ASSERT(input > 0); // __builtin_ctz(0) is undefined behavior
+
 #if HAVE_BUILTIN_CTZLL
     return __builtin_ctz(input);
 #else
@@ -48,12 +56,7 @@ static inline int trailing_zeros(int input)
 #endif
 }
 
-#if (defined(__GNUC__ ) || defined(__clang__))
-#define FORCE_INLINE __attribute__((always_inline))
-#else
-#define FORCE_INLINE
-#endif
-
+#ifdef JSON_ENABLE_SIMD
 
 #define SIMD_MINIMUM_THRESHOLD 6
 
@@ -70,14 +73,14 @@ static inline SIMD_Implementation find_simd_implementation(void)
 #define HAVE_SIMD_NEON 1
 
 // See: https://community.arm.com/arm-community-blogs/b/servers-and-cloud-computing-blog/posts/porting-x86-vector-bitmask-optimizations-to-arm-neon
-static inline FORCE_INLINE uint64_t neon_match_mask(uint8x16_t matches)
+static ALWAYS_INLINE() uint64_t neon_match_mask(uint8x16_t matches)
 {
     const uint8x8_t res = vshrn_n_u16(vreinterpretq_u16_u8(matches), 4);
     const uint64_t mask = vget_lane_u64(vreinterpret_u64_u8(res), 0);
     return mask & 0x8888888888888888ull;
 }
 
-static inline FORCE_INLINE uint64_t compute_chunk_mask_neon(const char *ptr)
+static ALWAYS_INLINE() uint64_t compute_chunk_mask_neon(const char *ptr)
 {
     uint8x16_t chunk = vld1q_u8((const unsigned char *)ptr);
 
@@ -90,7 +93,7 @@ static inline FORCE_INLINE uint64_t compute_chunk_mask_neon(const char *ptr)
     return neon_match_mask(needs_escape);
 }
 
-static inline FORCE_INLINE int string_scan_simd_neon(const char **ptr, const char *end, uint64_t *mask)
+static ALWAYS_INLINE() int string_scan_simd_neon(const char **ptr, const char *end, uint64_t *mask)
 {
     while (*ptr + sizeof(uint8x16_t) <= end) {
         uint64_t chunk_mask = compute_chunk_mask_neon(*ptr);
@@ -137,7 +140,7 @@ static inline uint8x16x4_t load_uint8x16_4(const unsigned char *table)
 #define _mm_cmpgt_epu8(a, b) _mm_xor_si128(_mm_cmple_epu8(a, b), _mm_set1_epi8(-1))
 #define _mm_cmplt_epu8(a, b) _mm_cmpgt_epu8(b, a)
 
-static inline TARGET_SSE2 FORCE_INLINE int compute_chunk_mask_sse2(const char *ptr)
+static TARGET_SSE2 ALWAYS_INLINE() int compute_chunk_mask_sse2(const char *ptr)
 {
     __m128i chunk         = _mm_loadu_si128((__m128i const*)ptr);
     // Trick: c < 32 || c == 34 can be factored as c ^ 2 < 33
@@ -148,7 +151,7 @@ static inline TARGET_SSE2 FORCE_INLINE int compute_chunk_mask_sse2(const char *p
     return _mm_movemask_epi8(needs_escape);
 }
 
-static inline TARGET_SSE2 FORCE_INLINE int string_scan_simd_sse2(const char **ptr, const char *end, int *mask)
+static TARGET_SSE2 ALWAYS_INLINE() int string_scan_simd_sse2(const char **ptr, const char *end, int *mask)
 {
     while (*ptr + sizeof(__m128i) <= end) {
         int chunk_mask = compute_chunk_mask_sse2(*ptr);