oj 3.16.0 → 3.17.3

data/ext/oj/saj.c

@@ -94,8 +94,7 @@ inline static void call_add_value(VALUE handler, VALUE value, const char *key) {
     if (0 == key) {
         k = Qnil;
     } else {
-        k = rb_str_new2(key);
-        k = oj_encode(k);
+        k = rb_utf8_str_new_cstr(key);
     }
     rb_funcall(handler, oj_add_value_id, 2, value, k);
 }
@@ -106,8 +105,7 @@ inline static void call_no_value(VALUE handler, ID method, const char *key) {
     if (0 == key) {
         k = Qnil;
     } else {
-        k = rb_str_new2(key);
-        k = oj_encode(k);
+        k = rb_utf8_str_new_cstr(key);
     }
     rb_funcall(handler, method, 1, k);
 }
@@ -257,9 +255,8 @@ static void read_str(ParseInfo pi, const char *key) {
 
     text = read_quoted_value(pi);
     if (pi->has_add_value) {
-        VALUE s = rb_str_new2(text);
+        VALUE s = rb_utf8_str_new_cstr(text);
 
-        s = oj_encode(s);
         call_add_value(pi->handler, s, key);
     }
 }
@@ -578,7 +575,7 @@ static void saj_parse(VALUE handler, char *json) {
     /* initialize parse info */
     pi.str = json;
     pi.s   = json;
-#if IS_WINDOWS
+#if IS_WINDOWS || !defined(HAVE_GETRLIMIT)
     pi.stack_min = (void *)((char *)&obj - (512L * 1024L)); /* assume a 1M stack and give half to ruby */
 #else
     {
@@ -587,7 +584,7 @@ static void saj_parse(VALUE handler, char *json) {
         if (0 == getrlimit(RLIMIT_STACK, &lim) && RLIM_INFINITY != lim.rlim_cur) {
             pi.stack_min = (void *)((char *)&obj - (lim.rlim_cur / 4 * 3)); /* let 3/4ths of the stack be used only */
         } else {
-            pi.stack_min = 0;                                               /* indicates not to check stack limit */
+            pi.stack_min = 0; /* indicates not to check stack limit */
         }
     }
 #endif
@@ -651,8 +648,16 @@ oj_saj_parse(int argc, VALUE *argv, VALUE self) {
             len = lseek(fd, 0, SEEK_END);
             lseek(fd, 0, SEEK_SET);
             json = OJ_R_ALLOC_N(char, len + 1);
-            if (0 >= (cnt = read(fd, json, len)) || cnt != (ssize_t)len) {
-                rb_raise(rb_eIOError, "failed to read from IO Object.");
+            {
+                size_t total = 0;
+
+                while (total < len) {
+                    cnt = read(fd, json + total, len - total);
+                    if (cnt <= 0) {
+                        rb_raise(rb_eIOError, "failed to read from IO Object.");
+                    }
+                    total += cnt;
+                }
             }
             json[len] = '\0';
 #endif

data/ext/oj/scp.c

@@ -56,9 +56,8 @@ static void add_value(ParseInfo pi, VALUE val) {
 }
 
 static void add_cstr(ParseInfo pi, const char *str, size_t len, const char *orig) {
-    volatile VALUE rstr = rb_str_new(str, len);
+    volatile VALUE rstr = rb_utf8_str_new(str, len);
 
-    rstr = oj_encode(rstr);
     rb_funcall(pi->handler, oj_add_value_id, 1, rstr);
 }
 
@@ -87,9 +86,8 @@ static VALUE hash_key(ParseInfo pi, const char *key, size_t klen) {
 }
 
 static void hash_set_cstr(ParseInfo pi, Val kval, const char *str, size_t len, const char *orig) {
-    volatile VALUE rstr = rb_str_new(str, len);
+    volatile VALUE rstr = rb_utf8_str_new(str, len);
 
-    rstr = oj_encode(rstr);
     rb_funcall(pi->handler, oj_hash_set_id, 3, stack_peek(&pi->stack)->val, oj_calc_hash_key(pi, kval), rstr);
 }
 
@@ -107,9 +105,8 @@ static void hash_set_value(ParseInfo pi, Val kval, VALUE value) {
 }
 
 static void array_append_cstr(ParseInfo pi, const char *str, size_t len, const char *orig) {
-    volatile VALUE rstr = rb_str_new(str, len);
+    volatile VALUE rstr = rb_utf8_str_new(str, len);
 
-    rstr = oj_encode(rstr);
     rb_funcall(pi->handler, oj_array_append_id, 2, stack_peek(&pi->stack)->val, rstr);
 }
 

data/ext/oj/simd.h

@@ -0,0 +1,219 @@
+#ifndef OJ_SIMD_H
+#define OJ_SIMD_H
+
+// SIMD architecture detection and configuration
+// This header provides unified SIMD support across different CPU architectures
+// with cross-platform runtime detection (Windows/Linux/Mac)
+
+// SIMD implementation enum - used for runtime selection
+typedef enum _simd_implementation { SIMD_NONE, SIMD_NEON, SIMD_SSE2, SIMD_SSE42 } SIMD_Implementation;
+
+// Define in oj.c.
+extern SIMD_Implementation SIMD_Impl;
+
+// Runtime CPU detection function (implemented in oj.c)
+SIMD_Implementation oj_get_simd_implementation(void);
+
+// =============================================================================
+// Compiler compatibility macros
+// =============================================================================
+
+// Branch prediction hints
+#if defined(__GNUC__) || defined(__clang__)
+#define OJ_LIKELY(x) __builtin_expect(!!(x), 1)
+#define OJ_UNLIKELY(x) __builtin_expect(!!(x), 0)
+#else
+#define OJ_LIKELY(x) (x)
+#define OJ_UNLIKELY(x) (x)
+#endif
+
+// Prefetch hints
+#if defined(__GNUC__) || defined(__clang__)
+#define OJ_PREFETCH(addr) __builtin_prefetch(addr, 0, 0)
+#elif defined(_MSC_VER)
+#include <intrin.h>
+#define OJ_PREFETCH(addr) _mm_prefetch((const char *)(addr), _MM_HINT_T0)
+#else
+#define OJ_PREFETCH(addr) ((void)0)
+#endif
+
+// Count trailing zeros (for SSE2 mask scanning)
+#if defined(__GNUC__) || defined(__clang__)
+#define OJ_CTZ(x) __builtin_ctz(x)
+#define OJ_CTZ64(x) __builtin_ctzll(x)
+#elif defined(_MSC_VER)
+#include <intrin.h>
+static __inline int oj_ctz_msvc(unsigned int x) {
+    unsigned long index;
+    if (0 == x) {
+        return 32;
+    }
+    _BitScanForward(&index, x);
+    return (int)index;
+}
+static __inline int oj_ctz64_msvc(uint64_t x) {
+    unsigned long index;
+    if (_BitScanForward64(&index, x)) {
+        return (int)index;
+    }
+    return 64;
+}
+#define OJ_CTZ(x) oj_ctz_msvc(x)
+#define OJ_CTZ64(x) oj_ctz64_msvc(x)
+#else
+// Fallback: naive implementation
+static inline int oj_ctz_fallback(unsigned int x) {
+    int count = 0;
+    while ((x & 1) == 0 && count < 32) {
+        x >>= 1;
+        count++;
+    }
+    return count;
+}
+
+static inline int oj_ctz64_fallback(uint64_t x) {
+    int count = 0;
+    while ((x & 1) == 0 && count < 64) {
+        x >>= 1;
+        count++;
+    }
+    return count;
+}
+#define OJ_CTZ(x) oj_ctz_fallback(x)
+#define OJ_CTZ64(x) oj_ctz64_fallback(x)
+#endif
+
+// =============================================================================
+// x86/x86_64 SIMD detection
+// =============================================================================
+#if defined(__x86_64__) || defined(__i386__) || defined(_M_IX86) || defined(_M_X64)
+#define HAVE_SIMD_X86 1
+
+// Include appropriate SIMD headers
+#if defined(_MSC_VER)
+// MSVC: use intrin.h for all intrinsics
+#include <intrin.h>
+#define HAVE_SIMD_SSE4_2 1
+#define HAVE_SIMD_SSE2 1
+#elif defined(__GNUC__) || defined(__clang__)
+// GCC/Clang: check for header availability and include them
+// We include headers but use target attributes to enable instructions per-function
+// Include cpuid.h for __get_cpuid fallback when __builtin_cpu_supports is unavailable
+#if __has_include(<cpuid.h>)
+#include <cpuid.h>
+#endif
+#if defined(__SSE4_2__) || defined(__SSE2__)
+// If any SSE is enabled globally, x86intrin.h should be available
+#include <x86intrin.h>
+#define HAVE_SIMD_SSE4_2 1
+#define HAVE_SIMD_SSE2 1
+#else
+// Try to include headers anyway for target attribute functions
+#if __has_include(<x86intrin.h>)
+#include <x86intrin.h>
+#define HAVE_SIMD_SSE4_2 1
+#define HAVE_SIMD_SSE2 1
+#elif __has_include(<nmmintrin.h>)
+#include <nmmintrin.h>
+#define HAVE_SIMD_SSE4_2 1
+#define HAVE_SIMD_SSE2 1
+#elif __has_include(<emmintrin.h>)
+#include <emmintrin.h>
+#define HAVE_SIMD_SSE2 1
+#endif
+#endif
+#endif
+
+// Target attribute macros for function-level SIMD enabling
+#if defined(__clang__) || defined(__GNUC__)
+#define OJ_TARGET_SSE42 __attribute__((target("sse4.2")))
+#define OJ_TARGET_SSE2 __attribute__((target("sse2")))
+#else
+// MSVC doesn't need target attributes - intrinsics are always available
+#define OJ_TARGET_SSE42
+#define OJ_TARGET_SSE2
+#endif
+
+#endif  // x86/x86_64
+
+// =============================================================================
+// ARM NEON detection
+// =============================================================================
+#if defined(__ARM_NEON) || defined(__ARM_NEON__) || defined(__aarch64__) || defined(_M_ARM64)
+#define HAVE_SIMD_NEON 1
+#define SIMD_MINIMUM_THRESHOLD 6
+#include <arm_neon.h>
+#endif
+
+// =============================================================================
+// SIMD type string for debugging/logging
+// =============================================================================
+#if defined(HAVE_SIMD_SSE4_2) || defined(HAVE_SIMD_SSE2)
+#define HAVE_SIMD_STRING_SCAN 1
+#define SIMD_TYPE "x86 (runtime detected)"
+#elif defined(HAVE_SIMD_NEON)
+#define HAVE_SIMD_STRING_SCAN 1
+#define SIMD_TYPE "NEON"
+#else
+#define SIMD_TYPE "none"
+#endif
+
+#if defined(HAVE_SIMD_SSE4_2)
+
+#define SIMD_MINIMUM_THRESHOLD 6
+
+extern void initialize_sse42(void);
+
+static inline OJ_TARGET_SSE42 __m128i vector_lookup_sse42(__m128i input, __m128i *lookup_table, int tab_size) {
+    // Extract high 4 bits to determine which 16-byte chunk (0-15)
+    __m128i hi_index = _mm_and_si128(_mm_srli_epi32(input, 4), _mm_set1_epi8(0x0F));
+
+    // Extract low 4 bits for index within the chunk (0-15)
+    __m128i low_index = _mm_and_si128(input, _mm_set1_epi8(0x0F));
+
+    // Perform lookups in all 16 tables
+    __m128i results[16];
+    for (int i = 0; i < tab_size; i++) {
+        results[i] = _mm_shuffle_epi8(lookup_table[i], low_index);
+    }
+
+    // Create masks for each chunk and blend results
+    __m128i final_result = _mm_setzero_si128();
+
+    for (int i = 0; i < tab_size; i++) {
+        __m128i mask          = _mm_cmpeq_epi8(hi_index, _mm_set1_epi8(i));
+        __m128i masked_result = _mm_and_si128(mask, results[i]);
+        final_result          = _mm_or_si128(final_result, masked_result);
+    }
+
+    return final_result;
+}
+
+#endif
+
+#ifndef __has_builtin
+#define __has_builtin(x) 0
+#endif
+
+#if __has_builtin(__builtin_memcpy)
+#define HAVE_FAST_MEMCPY 1
+
+inline static void fast_memcpy16(void *dest, const void *src, size_t n) {
+    char *d = (char *)dest;
+    char *s = (char *)src;
+    if (n >= 8) {
+        __builtin_memcpy(d, s, 8);
+        __builtin_memcpy(d + n - 8, s + n - 8, 8);
+    } else if (n >= 4) {
+        __builtin_memcpy(d, s, 4);
+        __builtin_memcpy(d + n - 4, s + n - 4, 4);
+    } else if (n >= 2) {
+        __builtin_memcpy(d, s, 2);
+        __builtin_memcpy(d + n - 2, s + n - 2, 2);
+    } else if (n >= 1) {
+        *d = *s;
+    }
+}
+#endif
+
+#endif /* OJ_SIMD_H */

data/ext/oj/sparse.c

@@ -399,6 +399,7 @@ static void read_num(ParseInfo pi) {
     char            c;
 
     reader_protect(&pi->rd);
+    ni.pi       = pi;
     ni.i        = 0;
     ni.num      = 0;
     ni.div      = 1;
@@ -415,7 +416,7 @@ static void read_num(ParseInfo pi) {
         ni.bigdec_load = pi->options.compat_bigdec;
     } else {
         ni.no_big      = (FloatDec == pi->options.bigdec_load || FastDec == pi->options.bigdec_load ||
-                     RubyDec == pi->options.bigdec_load);
+                          RubyDec == pi->options.bigdec_load);
         ni.bigdec_load = pi->options.bigdec_load;
     }
 
@@ -549,6 +550,7 @@ static void read_nan(ParseInfo pi) {
     struct _numInfo ni;
     char            c;
 
+    ni.pi       = pi;
     ni.str      = pi->rd.str;
     ni.i        = 0;
     ni.num      = 0;
@@ -565,7 +567,7 @@ static void read_nan(ParseInfo pi) {
         ni.bigdec_load = pi->options.compat_bigdec;
     } else {
         ni.no_big      = (FloatDec == pi->options.bigdec_load || FastDec == pi->options.bigdec_load ||
-                     RubyDec == pi->options.bigdec_load);
+                          RubyDec == pi->options.bigdec_load);
         ni.bigdec_load = pi->options.bigdec_load;
     }
 
@@ -745,6 +747,7 @@ void oj_sparse2(ParseInfo pi) {
                     oj_set_error_at(pi, oj_parse_error_class, __FILE__, __LINE__, "expected NaN");
                     return;
                 }
+                ni.pi       = pi;
                 ni.str      = pi->rd.str;
                 ni.i        = 0;
                 ni.num      = 0;
@@ -761,7 +764,7 @@ void oj_sparse2(ParseInfo pi) {
                     ni.bigdec_load = pi->options.compat_bigdec;
                 } else {
                     ni.no_big      = (FloatDec == pi->options.bigdec_load || FastDec == pi->options.bigdec_load ||
-                                 RubyDec == pi->options.bigdec_load);
+                                      RubyDec == pi->options.bigdec_load);
                     ni.bigdec_load = pi->options.bigdec_load;
                 }
                 add_num_value(pi, &ni);

data/ext/oj/stream_writer.c

@@ -21,6 +21,17 @@ static void stream_writer_free(void *ptr) {
     OJ_R_FREE(ptr);
 }
 
+static const rb_data_type_t oj_stream_writer_type = {
+    "Oj/stream_writer",
+    {
+        NULL,
+        stream_writer_free,
+        NULL,
+    },
+    0,
+    0,
+};
+
 static void stream_writer_reset_buf(StreamWriter sw) {
     sw->sw.out.cur  = sw->sw.out.buf;
     *sw->sw.out.cur = '\0';
@@ -31,22 +42,12 @@ static void stream_writer_write(StreamWriter sw) {
 
     switch (sw->type) {
     case STRING_IO:
-    case STREAM_IO: {
-        volatile VALUE rs = rb_str_new(sw->sw.out.buf, size);
-
-        // Oddly enough, when pushing ASCII characters with UTF-8 encoding or
-        // even ASCII-8BIT does not change the output encoding. Pushing any
-        // non-ASCII no matter what the encoding changes the output encoding
-        // to ASCII-8BIT if it the string is not forced to UTF-8 here.
-        rs = oj_encode(rs);
+    case STREAM_IO:
+    case FILE_IO: {
+        volatile VALUE rs = rb_utf8_str_new(sw->sw.out.buf, size);
         rb_funcall(sw->stream, oj_write_id, 1, rs);
         break;
     }
-    case FILE_IO:
-        if (size != write(sw->fd, sw->sw.out.buf, size)) {
-            rb_raise(rb_eIOError, "Write failed. [_%d_:%s]\n", errno, strerror(errno));
-        }
-        break;
     default: rb_raise(rb_eArgError, "expected an IO Object.");
     }
     stream_writer_reset_buf(sw);
@@ -120,7 +121,7 @@ static VALUE stream_writer_new(int argc, VALUE *argv, VALUE self) {
     sw->type          = type;
     sw->fd            = fd;
 
-    return Data_Wrap_Struct(oj_stream_writer_class, 0, stream_writer_free, sw);
+    return TypedData_Wrap_Struct(oj_stream_writer_class, &oj_stream_writer_type, sw);
 }
 
 /* Document-method: push_key
@@ -133,7 +134,8 @@ static VALUE stream_writer_new(int argc, VALUE *argv, VALUE self) {
  * - *key* [_String_] the key pending for the next push
  */
 static VALUE stream_writer_push_key(VALUE self, VALUE key) {
-    StreamWriter sw = (StreamWriter)DATA_PTR(self);
+    StreamWriter sw;
+    TypedData_Get_Struct(self, struct _streamWriter, &oj_stream_writer_type, sw);
 
     oj_str_writer_push_key(&sw->sw, StringValuePtr(key));
     if (sw->flush_limit < sw->sw.out.cur - sw->sw.out.buf) {
@@ -151,7 +153,8 @@ static VALUE stream_writer_push_key(VALUE self, VALUE key) {
  * - *key* [_String_] the key if adding to an object in the JSON document
  */
 static VALUE stream_writer_push_object(int argc, VALUE *argv, VALUE self) {
-    StreamWriter sw = (StreamWriter)DATA_PTR(self);
+    StreamWriter sw;
+    TypedData_Get_Struct(self, struct _streamWriter, &oj_stream_writer_type, sw);
 
     switch (argc) {
     case 0: oj_str_writer_push_object(&sw->sw, 0); break;
@@ -179,7 +182,8 @@ static VALUE stream_writer_push_object(int argc, VALUE *argv, VALUE self) {
  * - *key* [_String_] the key if adding to an object in the JSON document
  */
 static VALUE stream_writer_push_array(int argc, VALUE *argv, VALUE self) {
-    StreamWriter sw = (StreamWriter)DATA_PTR(self);
+    StreamWriter sw;
+    TypedData_Get_Struct(self, struct _streamWriter, &oj_stream_writer_type, sw);
 
     switch (argc) {
     case 0: oj_str_writer_push_array(&sw->sw, 0); break;
@@ -206,15 +210,16 @@ static VALUE stream_writer_push_array(int argc, VALUE *argv, VALUE self) {
  * - *key* [_String_] the key if adding to an object in the JSON document
  */
 static VALUE stream_writer_push_value(int argc, VALUE *argv, VALUE self) {
-    StreamWriter sw = (StreamWriter)DATA_PTR(self);
+    StreamWriter sw;
+    TypedData_Get_Struct(self, struct _streamWriter, &oj_stream_writer_type, sw);
 
     switch (argc) {
-    case 1: oj_str_writer_push_value((StrWriter)DATA_PTR(self), *argv, 0); break;
+    case 1: oj_str_writer_push_value((StrWriter)sw, *argv, 0); break;
     case 2:
         if (Qnil == argv[1]) {
-            oj_str_writer_push_value((StrWriter)DATA_PTR(self), *argv, 0);
+            oj_str_writer_push_value((StrWriter)sw, *argv, 0);
         } else {
-            oj_str_writer_push_value((StrWriter)DATA_PTR(self), *argv, StringValuePtr(argv[1]));
+            oj_str_writer_push_value((StrWriter)sw, *argv, StringValuePtr(argv[1]));
         }
         break;
     default: rb_raise(rb_eArgError, "Wrong number of argument to 'push_value'."); break;
@@ -235,15 +240,16 @@ static VALUE stream_writer_push_value(int argc, VALUE *argv, VALUE self) {
  * - *key* [_String_] the key if adding to an object in the JSON document
  */
 static VALUE stream_writer_push_json(int argc, VALUE *argv, VALUE self) {
-    StreamWriter sw = (StreamWriter)DATA_PTR(self);
+    StreamWriter sw;
+    TypedData_Get_Struct(self, struct _streamWriter, &oj_stream_writer_type, sw);
 
     switch (argc) {
-    case 1: oj_str_writer_push_json((StrWriter)DATA_PTR(self), StringValuePtr(*argv), 0); break;
+    case 1: oj_str_writer_push_json((StrWriter)sw, StringValuePtr(*argv), 0); break;
     case 2:
         if (Qnil == argv[1]) {
-            oj_str_writer_push_json((StrWriter)DATA_PTR(self), StringValuePtr(*argv), 0);
+            oj_str_writer_push_json((StrWriter)sw, StringValuePtr(*argv), 0);
         } else {
-            oj_str_writer_push_json((StrWriter)DATA_PTR(self), StringValuePtr(*argv), StringValuePtr(argv[1]));
+            oj_str_writer_push_json((StrWriter)sw, StringValuePtr(*argv), StringValuePtr(argv[1]));
         }
         break;
     default: rb_raise(rb_eArgError, "Wrong number of argument to 'push_json'."); break;
@@ -261,7 +267,8 @@ static VALUE stream_writer_push_json(int argc, VALUE *argv, VALUE self) {
  * currently open.
  */
 static VALUE stream_writer_pop(VALUE self) {
-    StreamWriter sw = (StreamWriter)DATA_PTR(self);
+    StreamWriter sw;
+    TypedData_Get_Struct(self, struct _streamWriter, &oj_stream_writer_type, sw);
 
     oj_str_writer_pop(&sw->sw);
     if (sw->flush_limit < sw->sw.out.cur - sw->sw.out.buf) {
@@ -277,7 +284,8 @@ static VALUE stream_writer_pop(VALUE self) {
  * currently open.
  */
 static VALUE stream_writer_pop_all(VALUE self) {
-    StreamWriter sw = (StreamWriter)DATA_PTR(self);
+    StreamWriter sw;
+    TypedData_Get_Struct(self, struct _streamWriter, &oj_stream_writer_type, sw);
 
     oj_str_writer_pop_all(&sw->sw);
     stream_writer_write(sw);
@@ -291,7 +299,9 @@ static VALUE stream_writer_pop_all(VALUE self) {
  * Flush any remaining characters in the buffer.
  */
 static VALUE stream_writer_flush(VALUE self) {
-    stream_writer_write((StreamWriter)DATA_PTR(self));
+    StreamWriter sw;
+    TypedData_Get_Struct(self, struct _streamWriter, &oj_stream_writer_type, sw);
+    stream_writer_write(sw);
 
     return Qnil;
 }

data/ext/oj/strict.c

@@ -19,8 +19,7 @@ VALUE oj_cstr_to_value(const char *str, size_t len, size_t cache_str) {
     if (len < cache_str) {
         rstr = oj_str_intern(str, len);
     } else {
-        rstr = rb_str_new(str, len);
-        rstr = oj_encode(rstr);
+        rstr = rb_utf8_str_new(str, len);
     }
     return rstr;
 }
@@ -35,8 +34,7 @@ VALUE oj_calc_hash_key(ParseInfo pi, Val parent) {
         if (Yes == pi->options.sym_key) {
             rkey = ID2SYM(rb_intern3(parent->key, parent->klen, oj_utf8_encoding));
         } else {
-            rkey = rb_str_new(parent->key, parent->klen);
-            rkey = oj_encode(rkey);
+            rkey = rb_utf8_str_new(parent->key, parent->klen);
             OBJ_FREEZE(rkey);  // frozen when used as a Hash key anyway
         }
         return rkey;