oj 3.16.11 → 3.17.6

data/ext/oj/oj.h

@@ -104,61 +104,64 @@ typedef enum {
 } StreamWriterType;
 
 typedef struct _dumpOpts {
-    bool    use;
-    char    indent_str[16];
-    char    before_sep[16];
-    char    after_sep[16];
-    char    hash_nl[16];
-    char    array_nl[16];
-    uint8_t indent_size;
-    uint8_t before_size;
-    uint8_t after_size;
-    uint8_t hash_size;
-    uint8_t array_size;
-    char    nan_dump;  // NanDump
-    bool    omit_nil;
-    bool    omit_null_byte;
-    int     max_depth;
+    bool        use;
+    char        indent_str[16];
+    char        before_sep[16];
+    char        after_sep[16];
+    char        hash_nl[16];
+    char        array_nl[16];
+    uint8_t     indent_size;
+    uint8_t     before_size;
+    uint8_t     after_size;
+    uint8_t     hash_size;
+    uint8_t     array_size;
+    char        nan_dump;  // NanDump
+    bool        omit_nil;
+    bool        omit_null_byte;
+    int         max_depth;
+    const char *only;
+    const char *except;
 } *DumpOpts;
 
 typedef struct _options {
-    int              indent;         // indention for dump, default 2
-    char             circular;       // YesNo
-    char             auto_define;    // YesNo
-    char             sym_key;        // YesNo
-    char             escape_mode;    // Escape_Mode
-    char             mode;           // Mode
-    char             class_cache;    // YesNo
-    char             time_format;    // TimeFormat
-    char             bigdec_as_num;  // YesNo
-    char             bigdec_load;    // BigLoad
-    char             compat_bigdec;  // boolean (0 or 1)
-    char             to_hash;        // YesNo
-    char             to_json;        // YesNo
-    char             as_json;        // YesNo
-    char             raw_json;       // YesNo
-    char             nilnil;         // YesNo
-    char             empty_string;   // YesNo
-    char             allow_gc;       // allow GC during parse
-    char             quirks_mode;    // allow single JSON values instead of documents
-    char             allow_invalid;  // YesNo - allow invalid unicode
-    char             create_ok;      // YesNo allow create_id
-    char             allow_nan;      // YEsyNo for parsing only
-    char             trace;          // YesNo
-    char             safe;           // YesNo
-    char             sec_prec_set;   // boolean (0 or 1)
-    char             ignore_under;   // YesNo - ignore attrs starting with _ if true in object and custom modes
-    char             cache_keys;     // YesNo
-    char             cache_str;      // string short than or equal to this are cache
-    int64_t          int_range_min;  // dump numbers below as string
-    int64_t          int_range_max;  // dump numbers above as string
-    const char      *create_id;      // 0 or string
-    size_t           create_id_len;  // length of create_id
-    int              sec_prec;       // second precision when dumping time
-    char             float_prec;     // float precision, linked to float_fmt
-    char             float_fmt[7];   // float format for dumping, if empty use Ruby
-    VALUE            hash_class;     // class to use in place of Hash on load
-    VALUE            array_class;    // class to use in place of Array on load
+    int              indent;              // indention for dump, default 2
+    char             circular;            // YesNo
+    char             auto_define;         // YesNo
+    char             sym_key;             // YesNo
+    char             escape_mode;         // Escape_Mode
+    char             mode;                // Mode
+    char             class_cache;         // YesNo
+    char             time_format;         // TimeFormat
+    char             bigdec_as_num;       // YesNo
+    char             bigdec_load;         // BigLoad
+    char             compat_bigdec;       // boolean (0 or 1)
+    char             to_hash;             // YesNo
+    char             to_json;             // YesNo
+    char             as_json;             // YesNo
+    char             raw_json;            // YesNo
+    char             nilnil;              // YesNo
+    char             empty_string;        // YesNo
+    char             allow_gc;            // allow GC during parse
+    char             quirks_mode;         // allow single JSON values instead of documents
+    char             allow_invalid;       // YesNo - allow invalid unicode
+    char             create_ok;           // YesNo allow create_id
+    char             allow_nan;           // YesNo for parsing only
+    char             trace;               // YesNo
+    char             safe;                // YesNo
+    char             sec_prec_set;        // boolean (0 or 1)
+    char             ignore_under;        // YesNo - ignore attrs starting with _ if true in object and custom modes
+    char             cache_keys;          // YesNo
+    char             cache_str;           // string short than or equal to this are cache
+    int64_t          int_range_min;       // dump numbers below as string
+    int64_t          int_range_max;       // dump numbers above as string
+    size_t           max_integer_digits;  // 0 = unlimited; max decimal digits for parsed integers
+    const char      *create_id;           // 0 or string
+    size_t           create_id_len;       // length of create_id
+    int              sec_prec;            // second precision when dumping time
+    char             float_prec;          // float precision, linked to float_fmt
+    char             float_fmt[7];        // float format for dumping, if empty use Ruby
+    VALUE            hash_class;          // class to use in place of Hash on load
+    VALUE            array_class;         // class to use in place of Array on load
     struct _dumpOpts dump_opts;
     struct _rxClass  str_rx;
     VALUE           *ignore;  // Qnil terminated array of classes or NULL
@@ -194,6 +197,7 @@ typedef struct _out {
     bool      allocated;
     bool      omit_nil;
     bool      omit_null_byte;
+    bool      key_filter_off;  // rails: suppress the :only/:except key filter for an as_json result
     int       argc;
     VALUE    *argv;
     ROptTable ropts;
@@ -253,6 +257,11 @@ extern VALUE oj_custom_parse_cstr(int argc, VALUE *argv, char *json, size_t len)
 
 extern bool oj_hash_has_key(VALUE hash, VALUE key);
 extern void oj_parse_options(VALUE ropts, Options copts);
+extern bool oj_parse_options_consumed(VALUE ropts, Options copts);
+extern void oj_free_call_options(Options copts);
+extern void oj_options_take_ownership(Options copts);
+extern void oj_options_release(Options copts);
+extern void oj_options_mark(Options copts);
 
 extern void  oj_dump_obj_to_json(VALUE obj, Options copts, Out out);
 extern void  oj_dump_obj_to_json_using_params(VALUE obj, Options copts, Out out, int argc, VALUE *argv);
@@ -318,7 +327,6 @@ extern VALUE oj_max_nesting_sym;
 extern VALUE oj_object_class_sym;
 extern VALUE oj_object_nl_sym;
 extern VALUE oj_quirks_mode_sym;
-extern VALUE oj_skip_null_byte_sym;
 extern VALUE oj_space_before_sym;
 extern VALUE oj_space_sym;
 extern VALUE oj_symbolize_names_sym;

data/ext/oj/parse.c

@@ -15,12 +15,9 @@
 #include "mem.h"
 #include "oj.h"
 #include "rxclass.h"
+#include "simd.h"
 #include "val_stack.h"
 
-#ifdef OJ_USE_SSE4_2
-#include <nmmintrin.h>
-#endif
-
 // Workaround in case INFINITY is not defined in math.h or if the OS is CentOS
 #define OJ_INFINITY (1.0 / 0.0)
 
@@ -202,23 +199,170 @@ static inline const char *scan_string_noSIMD(const char *str, const char *end) {
     return str;
 }
 
-#ifdef OJ_USE_SSE4_2
-static inline const char *scan_string_SIMD(const char *str, const char *end) {
-    static const char chars[16] = "\x00\\\"";
-    const __m128i     terminate = _mm_loadu_si128((const __m128i *)&chars[0]);
-    const char       *_end      = (const char *)(end - 16);
+#ifdef HAVE_SIMD_NEON
+
+static inline const char *string_scan_neon(const char *str, const char *end) {
+    const uint8x16_t null_char = vdupq_n_u8(0);
+    const uint8x16_t backslash = vdupq_n_u8('\\');
+    const uint8x16_t quote     = vdupq_n_u8('"');
+
+    while (str + sizeof(uint8x16_t) <= end) {
+        uint8x16_t      chunk = vld1q_u8((const uint8_t *)str);
+        uint8x16_t      tmp   = vorrq_u8(vorrq_u8(vceqq_u8(chunk, null_char), vceqq_u8(chunk, backslash)),
+                                         vceqq_u8(chunk, quote));
+        const uint8x8_t res   = vshrn_n_u16(vreinterpretq_u16_u8(tmp), 4);
+        uint64_t        mask  = vget_lane_u64(vreinterpret_u64_u8(res), 0);
+        if (mask != 0) {
+            mask &= 0x8888888888888888ull;
+            return str + (OJ_CTZ64(mask) >> 2);
+        }
+        str += sizeof(uint8x16_t);
+    }
+
+    return scan_string_noSIMD(str, end);
+}
+
+#endif
 
-    for (; str <= _end; str += 16) {
+#ifdef HAVE_SIMD_SSE4_2
+// Optimized SIMD string scanner using SSE4.2 instructions
+// Uses prefetching and processes multiple chunks in parallel to reduce latency
+// Note: OJ_TARGET_SSE42 attribute allows this to compile even without global -msse4.2
+static OJ_TARGET_SSE42 const char *scan_string_SSE42(const char *str, const char *end) {
+    static const char chars[16]   = "\x00\\\"";
+    const __m128i     terminate   = _mm_loadu_si128((const __m128i *)&chars[0]);
+    const char       *safe_end_64 = end - 64;
+    const char       *safe_end_16 = end - 16;
+
+    // Process 64 bytes at a time with parallel SIMD operations
+    // This reduces pipeline stalls and improves instruction-level parallelism
+    while (str <= safe_end_64) {
+        // Prefetch next cache line for better memory throughput
+        OJ_PREFETCH(str + 64);
+
+        // Load and compare 4 chunks in parallel
+        const __m128i chunk0 = _mm_loadu_si128((const __m128i *)(str));
+        const __m128i chunk1 = _mm_loadu_si128((const __m128i *)(str + 16));
+        const __m128i chunk2 = _mm_loadu_si128((const __m128i *)(str + 32));
+        const __m128i chunk3 = _mm_loadu_si128((const __m128i *)(str + 48));
+
+        const int r0 = _mm_cmpestri(terminate,
+                                    3,
+                                    chunk0,
+                                    16,
+                                    _SIDD_UBYTE_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_LEAST_SIGNIFICANT);
+        if (OJ_UNLIKELY(r0 != 16))
+            return str + r0;
+
+        const int r1 = _mm_cmpestri(terminate,
+                                    3,
+                                    chunk1,
+                                    16,
+                                    _SIDD_UBYTE_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_LEAST_SIGNIFICANT);
+        if (OJ_UNLIKELY(r1 != 16))
+            return str + 16 + r1;
+
+        const int r2 = _mm_cmpestri(terminate,
+                                    3,
+                                    chunk2,
+                                    16,
+                                    _SIDD_UBYTE_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_LEAST_SIGNIFICANT);
+        if (OJ_UNLIKELY(r2 != 16))
+            return str + 32 + r2;
+
+        const int r3 = _mm_cmpestri(terminate,
+                                    3,
+                                    chunk3,
+                                    16,
+                                    _SIDD_UBYTE_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_LEAST_SIGNIFICANT);
+        if (OJ_UNLIKELY(r3 != 16))
+            return str + 48 + r3;
+
+        str += 64;
+    }
+
+    // Handle remaining 16-byte chunks
+    for (; str <= safe_end_16; str += 16) {
         const __m128i string = _mm_loadu_si128((const __m128i *)str);
         const int     r      = _mm_cmpestri(terminate,
-                                   3,
-                                   string,
-                                   16,
-                                   _SIDD_UBYTE_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_LEAST_SIGNIFICANT);
-        if (r != 16) {
-            str = (char *)(str + r);
-            return str;
-        }
+                                            3,
+                                            string,
+                                            16,
+                                            _SIDD_UBYTE_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_LEAST_SIGNIFICANT);
+        if (r != 16)
+            return str + r;
+    }
+
+    return scan_string_noSIMD(str, end);
+}
+#endif
+
+#ifdef HAVE_SIMD_SSE2
+// Optimized SSE2 string scanner (fallback for older x86_64 CPUs)
+// Uses SSE2 instructions with prefetching and parallel processing
+// Note: OJ_TARGET_SSE2 attribute allows this to compile even without global -msse2
+static OJ_TARGET_SSE2 const char *scan_string_SSE2(const char *str, const char *end) {
+    const char *safe_end_64 = end - 64;
+    const char *safe_end_16 = end - 16;
+
+    // Create comparison vectors for our three special characters
+    const __m128i null_char = _mm_setzero_si128();
+    const __m128i backslash = _mm_set1_epi8('\\');
+    const __m128i quote     = _mm_set1_epi8('"');
+
+    // Process 64 bytes at a time for better throughput
+    while (str <= safe_end_64) {
+        OJ_PREFETCH(str + 64);
+
+        // Load 4 chunks
+        const __m128i chunk0 = _mm_loadu_si128((const __m128i *)(str));
+        const __m128i chunk1 = _mm_loadu_si128((const __m128i *)(str + 16));
+        const __m128i chunk2 = _mm_loadu_si128((const __m128i *)(str + 32));
+        const __m128i chunk3 = _mm_loadu_si128((const __m128i *)(str + 48));
+
+        // Compare all chunks (allows CPU to parallelize)
+        const __m128i cmp0 = _mm_or_si128(
+            _mm_or_si128(_mm_cmpeq_epi8(chunk0, null_char), _mm_cmpeq_epi8(chunk0, backslash)),
+            _mm_cmpeq_epi8(chunk0, quote));
+        const __m128i cmp1 = _mm_or_si128(
+            _mm_or_si128(_mm_cmpeq_epi8(chunk1, null_char), _mm_cmpeq_epi8(chunk1, backslash)),
+            _mm_cmpeq_epi8(chunk1, quote));
+        const __m128i cmp2 = _mm_or_si128(
+            _mm_or_si128(_mm_cmpeq_epi8(chunk2, null_char), _mm_cmpeq_epi8(chunk2, backslash)),
+            _mm_cmpeq_epi8(chunk2, quote));
+        const __m128i cmp3 = _mm_or_si128(
+            _mm_or_si128(_mm_cmpeq_epi8(chunk3, null_char), _mm_cmpeq_epi8(chunk3, backslash)),
+            _mm_cmpeq_epi8(chunk3, quote));
+
+        // Convert to masks
+        int mask0 = _mm_movemask_epi8(cmp0);
+        if (OJ_UNLIKELY(mask0 != 0))
+            return str + OJ_CTZ(mask0);
+
+        int mask1 = _mm_movemask_epi8(cmp1);
+        if (OJ_UNLIKELY(mask1 != 0))
+            return str + 16 + OJ_CTZ(mask1);
+
+        int mask2 = _mm_movemask_epi8(cmp2);
+        if (OJ_UNLIKELY(mask2 != 0))
+            return str + 32 + OJ_CTZ(mask2);
+
+        int mask3 = _mm_movemask_epi8(cmp3);
+        if (OJ_UNLIKELY(mask3 != 0))
+            return str + 48 + OJ_CTZ(mask3);
+
+        str += 64;
+    }
+
+    // Handle remaining 16-byte chunks
+    for (; str <= safe_end_16; str += 16) {
+        const __m128i chunk   = _mm_loadu_si128((const __m128i *)str);
+        const __m128i matches = _mm_or_si128(
+            _mm_or_si128(_mm_cmpeq_epi8(chunk, null_char), _mm_cmpeq_epi8(chunk, backslash)),
+            _mm_cmpeq_epi8(chunk, quote));
+        int mask = _mm_movemask_epi8(matches);
+        if (mask != 0)
+            return str + OJ_CTZ(mask);
     }
 
     return scan_string_noSIMD(str, end);
@@ -228,16 +372,29 @@ static inline const char *scan_string_SIMD(const char *str, const char *end) {
 static const char *(*scan_func)(const char *str, const char *end) = scan_string_noSIMD;
 
 void oj_scanner_init(void) {
-#ifdef OJ_USE_SSE4_2
-    scan_func = scan_string_SIMD;
+    // Use runtime CPU detection to select the best SIMD implementation
+    // This ensures we don't crash on CPUs that don't support SSE4.2
+    SIMD_Implementation impl = oj_get_simd_implementation();
+
+    switch (impl) {
+#ifdef HAVE_SIMD_SSE4_2
+    case SIMD_SSE42: scan_func = scan_string_SSE42; break;
+#endif
+#ifdef HAVE_SIMD_SSE2
+    case SIMD_SSE2: scan_func = scan_string_SSE2; break;
 #endif
+#ifdef HAVE_SIMD_NEON
+    case SIMD_NEON: scan_func = string_scan_neon; break;
+#endif
+    default: scan_func = scan_string_noSIMD; break;
+    }
 }
 
 // entered at /
 static void read_escaped_str(ParseInfo pi, const char *start) {
     struct _buf buf;
     const char *s;
-    int         cnt = (int)(pi->cur - start);
+    size_t      cnt = pi->cur - start;
     uint32_t    code;
     Val         parent = stack_peek(&pi->stack);
 
@@ -257,6 +414,11 @@ static void read_escaped_str(ParseInfo pi, const char *start) {
 
         if ('\\' == *s) {
             s++;
+            if (pi->end <= s) {
+                oj_set_error_at(pi, oj_parse_error_class, __FILE__, __LINE__, "quoted string not terminated");
+                buf_cleanup(&buf);
+                return;
+            }
             switch (*s) {
             case 'n': buf_append(&buf, '\n'); break;
             case 'r': buf_append(&buf, '\r'); break;
@@ -454,7 +616,7 @@ static void read_num(ParseInfo pi) {
         ni.bigdec_load = pi->options.compat_bigdec;
     } else {
         ni.no_big      = (FloatDec == pi->options.bigdec_load || FastDec == pi->options.bigdec_load ||
-                     RubyDec == pi->options.bigdec_load);
+                          RubyDec == pi->options.bigdec_load);
         ni.bigdec_load = pi->options.bigdec_load;
     }
 
@@ -512,7 +674,7 @@ static void read_num(ParseInfo pi) {
             // A trailing . is not a valid decimal but if encountered allow it
             // except when mimicking the JSON gem or in strict mode.
             if (StrictMode == pi->options.mode || CompatMode == pi->options.mode) {
-                int pos = (int)(pi->cur - ni.str);
+                size_t pos = pi->cur - ni.str;
 
                 if (1 == pos || (2 == pos && ni.neg)) {
                     oj_set_error_at(pi, oj_parse_error_class, __FILE__, __LINE__, "not a number");
@@ -814,6 +976,20 @@ static long double exp_plus[] = {
     1.0e39, 1.0e40, 1.0e41, 1.0e42, 1.0e43, 1.0e44, 1.0e45, 1.0e46, 1.0e47, 1.0e48, 1.0e49,
 };
 
+static void validate_integer_size(size_t limit, NumInfo ni) {
+    size_t digit_count = ni->len - (ni->neg ? 1 : 0);
+
+    if (digit_count > limit) {
+        oj_set_error_at(ni->pi,
+                        (Qnil != ni->pi->err_class) ? ni->pi->err_class : oj_parse_error_class,
+                        __FILE__,
+                        __LINE__,
+                        "integer exceeds :max_integer_digits (%lu > %lu)",
+                        (unsigned long)digit_count,
+                        (unsigned long)limit);
+    }
+}
+
 VALUE
 oj_num_as_value(NumInfo ni) {
     VALUE rnum = Qnil;
@@ -827,6 +1003,12 @@ oj_num_as_value(NumInfo ni) {
     } else if (ni->nan) {
         rnum = rb_float_new(0.0 / 0.0);
     } else if (1 == ni->div && 0 == ni->exp && !ni->has_exp) {  // fixnum
+        size_t limit = (NULL != ni->pi) ? ni->pi->options.max_integer_digits : 0;
+
+        if (0 < limit) {
+            validate_integer_size(limit, ni);
+        }
+
         if (ni->big) {
             if (256 > ni->len) {
                 char buf[256];
@@ -919,7 +1101,9 @@ void oj_set_error_at(ParseInfo pi, VALUE err_clas, const char *file, int line, c
     }
     va_end(ap);
     pi->err.clas = err_clas;
-    if (p + 3 < end) {
+    // The eight bytes below, plus the ')' and the terminator that the two bytes
+    // held back from end are for.
+    if (p + 8 <= end) {
         *p++  = ' ';
         *p++  = '(';
         *p++  = 'a';
@@ -1036,11 +1220,19 @@ oj_pi_parse(int argc, VALUE *argv, ParseInfo pi, char *json, size_t len, int yie
             buf      = OJ_R_ALLOC_N(char, len + 1);
             pi->json = buf;
             pi->end  = buf + len;
-            if (0 >= (cnt = read(fd, (char *)pi->json, len)) || cnt != (ssize_t)len) {
-                if (0 != buf) {
-                    OJ_R_FREE(buf);
+            {
+                size_t total = 0;
+
+                while (total < len) {
+                    cnt = read(fd, (char *)pi->json + total, len - total);
+                    if (cnt <= 0) {
+                        if (0 != buf) {
+                            OJ_R_FREE(buf);
+                        }
+                        rb_raise(rb_eIOError, "failed to read from IO Object.");
+                    }
+                    total += cnt;
                 }
-                rb_raise(rb_eIOError, "failed to read from IO Object.");
             }
             ((char *)pi->json)[len] = '\0';
             /* skip UTF-8 BOM if present */
@@ -1121,9 +1313,16 @@ CLEANUP:
         OJ_R_FREE(json);
     }
     stack_cleanup(&pi->stack);
-    if (pi->str_rx.head != oj_default_options.str_rx.head) {
-        oj_rxclass_cleanup(&pi->str_rx);
+    // The parsers match against pi->options.str_rx. A :match_string option
+    // builds a chain of its own for this call, which has to be freed. Without
+    // the option the member still aliases the chain owned by the defaults and
+    // must be left alone.
+    if (pi->options.str_rx.head != oj_default_options.str_rx.head) {
+        oj_rxclass_cleanup(&pi->options.str_rx);
+        pi->options.str_rx.head = NULL;
+        pi->options.str_rx.tail = NULL;
     }
+    oj_free_call_options(&pi->options);
     if (err_has(&pi->err)) {
         rb_set_errinfo(Qnil);
         if (Qnil != pi->err_class) {

data/ext/oj/parse.h

@@ -49,7 +49,6 @@ typedef struct _parseInfo {
     VALUE            handler;
     struct _valStack stack;
     CircArray        circ_array;
-    struct _rxClass  str_rx;
     int              expect_value;
     int              max_depth;  // just for the json gem
     VALUE            proc;