json 2.15.2 → 2.19.2

data/ext/json/ext/simd/simd.h

@@ -1,10 +1,14 @@
+#include "../json.h"
+
 typedef enum {
     SIMD_NONE,
     SIMD_NEON,
     SIMD_SSE2
 } SIMD_Implementation;
 
-#ifdef JSON_ENABLE_SIMD
+#ifndef __has_builtin         // Optional of course.
+  #define __has_builtin(x) 0  // Compatibility with non-clang compilers.
+#endif
 
 #ifdef __clang__
 # if __has_builtin(__builtin_ctzll)
@@ -20,6 +24,8 @@ typedef enum {
 
 static inline uint32_t trailing_zeros64(uint64_t input)
 {
+    JSON_ASSERT(input > 0); // __builtin_ctz(0) is undefined behavior
+
 #if HAVE_BUILTIN_CTZLL
     return __builtin_ctzll(input);
 #else
@@ -35,6 +41,8 @@ static inline uint32_t trailing_zeros64(uint64_t input)
 
 static inline int trailing_zeros(int input)
 {
+    JSON_ASSERT(input > 0); // __builtin_ctz(0) is undefined behavior
+
 #if HAVE_BUILTIN_CTZLL
     return __builtin_ctz(input);
 #else
@@ -48,14 +56,36 @@ static inline int trailing_zeros(int input)
 #endif
 }
 
-#if (defined(__GNUC__ ) || defined(__clang__))
-#define FORCE_INLINE __attribute__((always_inline))
-#else
-#define FORCE_INLINE
-#endif
+#ifdef JSON_ENABLE_SIMD
 
+#define SIMD_MINIMUM_THRESHOLD 4
 
-#define SIMD_MINIMUM_THRESHOLD 6
+ALWAYS_INLINE(static) void json_fast_memcpy16(char *dst, const char *src, size_t len)
+{
+    RBIMPL_ASSERT_OR_ASSUME(len < 16);
+    RBIMPL_ASSERT_OR_ASSUME(len >= SIMD_MINIMUM_THRESHOLD); // 4
+#if defined(__has_builtin) && __has_builtin(__builtin_memcpy)
+    // If __builtin_memcpy is available, use it to copy between SIMD_MINIMUM_THRESHOLD (4) and vec_len-1 (15) bytes.
+    // These copies overlap. The first copy will copy the first 8 (or 4) bytes. The second copy will copy
+    // the last 8 (or 4) bytes but overlap with the first copy. The overlapping bytes will be in the correct
+    // position in both copies.
+
+    // Please do not attempt to replace __builtin_memcpy with memcpy without profiling and/or looking at the
+    // generated assembly. On clang-specifically (tested on Apple clang version 17.0.0 (clang-1700.0.13.3)),
+    // when using memcpy, the compiler will notice the only difference is a 4 or 8 and generate a conditional
+    // select instruction instead of direct loads and stores with a branch. This ends up slower than the branch
+    // plus two loads and stores generated when using __builtin_memcpy.
+    if (len >= 8) {
+        __builtin_memcpy(dst, src, 8);
+        __builtin_memcpy(dst + len - 8, src + len - 8, 8);
+    } else {
+        __builtin_memcpy(dst, src, 4);
+        __builtin_memcpy(dst + len - 4, src + len - 4, 4);
+    }
+#else
+    MEMCPY(dst, src, char, len);
+#endif
+}
 
 #if defined(__ARM_NEON) || defined(__ARM_NEON__) || defined(__aarch64__) || defined(_M_ARM64)
 #include <arm_neon.h>
@@ -70,14 +100,14 @@ static inline SIMD_Implementation find_simd_implementation(void)
 #define HAVE_SIMD_NEON 1
 
 // See: https://community.arm.com/arm-community-blogs/b/servers-and-cloud-computing-blog/posts/porting-x86-vector-bitmask-optimizations-to-arm-neon
-static inline FORCE_INLINE uint64_t neon_match_mask(uint8x16_t matches)
+ALWAYS_INLINE(static) uint64_t neon_match_mask(uint8x16_t matches)
 {
     const uint8x8_t res = vshrn_n_u16(vreinterpretq_u16_u8(matches), 4);
     const uint64_t mask = vget_lane_u64(vreinterpret_u64_u8(res), 0);
     return mask & 0x8888888888888888ull;
 }
 
-static inline FORCE_INLINE uint64_t compute_chunk_mask_neon(const char *ptr)
+ALWAYS_INLINE(static) uint64_t compute_chunk_mask_neon(const char *ptr)
 {
     uint8x16_t chunk = vld1q_u8((const unsigned char *)ptr);
 
@@ -90,7 +120,7 @@ static inline FORCE_INLINE uint64_t compute_chunk_mask_neon(const char *ptr)
     return neon_match_mask(needs_escape);
 }
 
-static inline FORCE_INLINE int string_scan_simd_neon(const char **ptr, const char *end, uint64_t *mask)
+ALWAYS_INLINE(static) int string_scan_simd_neon(const char **ptr, const char *end, uint64_t *mask)
 {
     while (*ptr + sizeof(uint8x16_t) <= end) {
         uint64_t chunk_mask = compute_chunk_mask_neon(*ptr);
@@ -103,16 +133,6 @@ static inline FORCE_INLINE int string_scan_simd_neon(const char **ptr, const cha
     return 0;
 }
 
-static inline uint8x16x4_t load_uint8x16_4(const unsigned char *table)
-{
-    uint8x16x4_t tab;
-    tab.val[0] = vld1q_u8(table);
-    tab.val[1] = vld1q_u8(table+16);
-    tab.val[2] = vld1q_u8(table+32);
-    tab.val[3] = vld1q_u8(table+48);
-    return tab;
-}
-
 #endif /* ARM Neon Support.*/
 
 #if defined(__amd64__) || defined(__amd64) || defined(__x86_64__) || defined(__x86_64) || defined(_M_X64) || defined(_M_AMD64)
@@ -137,7 +157,7 @@ static inline uint8x16x4_t load_uint8x16_4(const unsigned char *table)
 #define _mm_cmpgt_epu8(a, b) _mm_xor_si128(_mm_cmple_epu8(a, b), _mm_set1_epi8(-1))
 #define _mm_cmplt_epu8(a, b) _mm_cmpgt_epu8(b, a)
 
-static inline TARGET_SSE2 FORCE_INLINE int compute_chunk_mask_sse2(const char *ptr)
+ALWAYS_INLINE(static) TARGET_SSE2 int compute_chunk_mask_sse2(const char *ptr)
 {
     __m128i chunk         = _mm_loadu_si128((__m128i const*)ptr);
     // Trick: c < 32 || c == 34 can be factored as c ^ 2 < 33
@@ -148,7 +168,7 @@ static inline TARGET_SSE2 FORCE_INLINE int compute_chunk_mask_sse2(const char *p
     return _mm_movemask_epi8(needs_escape);
 }
 
-static inline TARGET_SSE2 FORCE_INLINE int string_scan_simd_sse2(const char **ptr, const char *end, int *mask)
+ALWAYS_INLINE(static) TARGET_SSE2 int string_scan_simd_sse2(const char **ptr, const char *end, int *mask)
 {
     while (*ptr + sizeof(__m128i) <= end) {
         int chunk_mask = compute_chunk_mask_sse2(*ptr);

data/ext/json/ext/vendor/fpconv.c

@@ -29,7 +29,7 @@
 #include <string.h>
 #include <stdint.h>
 
-#ifdef JSON_DEBUG
+#if JSON_DEBUG
 #include <assert.h>
 #endif
 
@@ -449,7 +449,7 @@ static int filter_special(double fp, char* dest)
  * }
  *
  */
-static int fpconv_dtoa(double d, char dest[28])
+static int fpconv_dtoa(double d, char dest[32])
 {
     char digits[18];
 
@@ -472,7 +472,7 @@ static int fpconv_dtoa(double d, char dest[28])
     int ndigits = grisu2(d, digits, &K);
 
     str_len += emit_digits(digits, ndigits, dest + str_len, K, neg);
-#ifdef JSON_DEBUG
+#if JSON_DEBUG
     assert(str_len <= 32);
 #endif