oj 3.13.15 → 3.13.18

data/ext/oj/parse.c

@@ -183,6 +183,74 @@ static void unicode_to_chars(ParseInfo pi, Buf buf, uint32_t code) {
     }
 }
 
+static inline const char *scan_string_noSIMD(const char *str, const char *end) {
+    for (; '"' != *str; str++) {
+        if (end <= str || '\0' == *str || '\\' == *str) {
+            break;
+        }
+    }
+    return str;
+}
+
+// Taken from Tensorflow:
+// https://github.com/tensorflow/tensorflow/blob/5dcfc51118817f27fad5246812d83e5dccdc5f72/tensorflow/core/lib/hash/crc32c_accelerate.cc#L21-L38
+#ifdef __SSE4_2__
+#if defined(__x86_64__) && defined(__GNUC__) && \
+    (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8))
+#define USE_SSE_DETECT 1
+#elif defined(__x86_64__) && defined(__clang__)
+#if __has_builtin(__builtin_cpu_supports)
+#define USE_SSE_DETECT 1
+#endif
+#endif
+#endif /* __SSE4_2__ */
+
+// This version of Apple clang has a bug:
+// https://llvm.org/bugs/show_bug.cgi?id=25510
+#if defined(__APPLE__) && (__clang_major__ <= 8)
+#undef USE_SSE_DETECT
+#endif
+
+#ifdef USE_SSE_DETECT
+#include <nmmintrin.h>
+
+static inline const char *scan_string_SIMD(const char *str, const char *end) {
+    static const char chars[16] = "\x00\\\"";
+    const __m128i terminate = _mm_loadu_si128((const __m128i *)&chars[0]);
+    const char *_end = (const char *)(end - 16);
+
+    for (; str <= _end; str += 16) {
+        const __m128i string = _mm_loadu_si128((const __m128i *)str);
+        const int r = _mm_cmpestri(terminate, 3, string, 16, _SIDD_UBYTE_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_LEAST_SIGNIFICANT);
+        if (r != 16) {
+            str = (char*)(str + r);
+            return str;
+        }
+    }
+
+    return scan_string_noSIMD(str, end);
+}
+#endif
+
+static bool cpu_supports_sse42(void) {
+#if USE_SSE_DETECT
+    __builtin_cpu_init();
+    return (__builtin_cpu_supports("sse4.2"));
+#else
+    return false;
+#endif
+}
+
+static const char *(*scan_func) (const char *str, const char *end) = scan_string_noSIMD;
+
+void oj_scanner_init(void) {
+    if (cpu_supports_sse42()) {
+#if USE_SSE_DETECT
+        scan_func = scan_string_SIMD;
+#endif
+    }
+}
+
 // entered at /
 static void read_escaped_str(ParseInfo pi, const char *start) {
     struct _buf buf;
@@ -192,11 +260,11 @@ static void read_escaped_str(ParseInfo pi, const char *start) {
     Val         parent = stack_peek(&pi->stack);
 
     buf_init(&buf);
-    if (0 < cnt) {
-        buf_append_string(&buf, start, cnt);
-    }
-    for (s = pi->cur; '"' != *s; s++) {
-        if (s >= pi->end) {
+    buf_append_string(&buf, start, cnt);
+
+    for (s = pi->cur; '"' != *s;) {
+        const char *scanned = scan_func(s, pi->end);
+        if (scanned >= pi->end) {
             oj_set_error_at(pi,
                             oj_parse_error_class,
                             __FILE__,
@@ -204,7 +272,12 @@ static void read_escaped_str(ParseInfo pi, const char *start) {
                             "quoted string not terminated");
             buf_cleanup(&buf);
             return;
-        } else if ('\\' == *s) {
+        }
+
+        buf_append_string(&buf, s, (size_t)(scanned - s));
+        s = scanned;
+
+        if ('\\' == *s) {
             s++;
             switch (*s) {
             case 'n': buf_append(&buf, '\n'); break;
@@ -273,8 +346,7 @@ static void read_escaped_str(ParseInfo pi, const char *start) {
                 buf_cleanup(&buf);
                 return;
             }
-        } else {
-            buf_append(&buf, *s);
+            s++;
         }
     }
     if (0 == parent) {
@@ -327,44 +399,11 @@ static void read_escaped_str(ParseInfo pi, const char *start) {
     buf_cleanup(&buf);
 }
 
-static inline void scan_string_noSIMD(ParseInfo pi) {
-    for (; '"' != *pi->cur; pi->cur++) {
-        if (pi->end <= pi->cur || '\0' == *pi->cur || '\\' == *pi->cur) {
-            return;
-        }
-    }
-}
-
-#if defined(OJ_USE_SSE4_2)
-#include <nmmintrin.h>
-
-static inline void scan_string_SIMD(ParseInfo pi) {
-    static const char chars[16] = "\x00\\\"";
-    const __m128i terminate = _mm_loadu_si128((const __m128i *)&chars[0]);
-    const char *end = (const char *)(pi->end - 16);
-
-    for (; pi->cur <= end; pi->cur += 16) {
-        const __m128i string = _mm_loadu_si128((const __m128i *)pi->cur);
-        const int r = _mm_cmpestri(terminate, 3, string, 16, _SIDD_UBYTE_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_LEAST_SIGNIFICANT);
-        if (r != 16) {
-            pi->cur = (const char*)(pi->cur + r);
-            return;
-        }
-    }
-
-    scan_string_noSIMD(pi);
-}
-#endif
-
 static void read_str(ParseInfo pi) {
     const char *str    = pi->cur;
     Val         parent = stack_peek(&pi->stack);
 
-#if defined(OJ_USE_SSE4_2)
-    scan_string_SIMD(pi);
-#else
-    scan_string_noSIMD(pi);
-#endif
+    pi->cur = scan_func(pi->cur, pi->end);
     if (RB_UNLIKELY(pi->end <= pi->cur)) {
         oj_set_error_at(pi,
                         oj_parse_error_class,
@@ -494,33 +533,31 @@ static void read_num(ParseInfo pi) {
         int  dec_cnt = 0;
         bool zero1   = false;
 
+        // Skip leading zeros.
+        for (; '0' == *pi->cur; pi->cur++) {
+            zero1 = true;
+        }
+
         for (; '0' <= *pi->cur && *pi->cur <= '9'; pi->cur++) {
-            if (0 == ni.i && '0' == *pi->cur) {
-                zero1 = true;
-            }
-            if (0 < ni.i) {
-                dec_cnt++;
-            }
-            if (!ni.big) {
-                int d = (*pi->cur - '0');
+            int d = (*pi->cur - '0');
 
-                if (0 < d) {
-                    if (zero1 && CompatMode == pi->options.mode) {
-                        oj_set_error_at(pi,
-                                        oj_parse_error_class,
-                                        __FILE__,
-                                        __LINE__,
-                                        "not a number");
-                        return;
-                    }
-                    zero1 = false;
-                }
-                ni.i = ni.i * 10 + d;
-                if (INT64_MAX <= ni.i || DEC_MAX < dec_cnt) {
-                    ni.big = 1;
-                }
+            if (RB_LIKELY(0 != ni.i)) {
+                dec_cnt++;
             }
+            ni.i = ni.i * 10 + d;
+        }
+        if (RB_UNLIKELY(0 != ni.i && zero1 && CompatMode == pi->options.mode)) {
+            oj_set_error_at(pi,
+                            oj_parse_error_class,
+                            __FILE__,
+                            __LINE__,
+                            "not a number");
+            return;
+        }
+        if (INT64_MAX <= ni.i || DEC_MAX < dec_cnt) {
+            ni.big = true;
         }
+
         if ('.' == *pi->cur) {
             pi->cur++;
             // A trailing . is not a valid decimal but if encountered allow it
@@ -540,25 +577,20 @@ static void read_num(ParseInfo pi) {
             for (; '0' <= *pi->cur && *pi->cur <= '9'; pi->cur++) {
                 int d = (*pi->cur - '0');
 
-                if (0 < ni.num || 0 < ni.i) {
+                if (RB_LIKELY(0 != ni.num || 0 != ni.i)) {
                     dec_cnt++;
                 }
-                if (INT64_MAX <= ni.div) {
-                    if (!ni.no_big) {
-                        ni.big = true;
-                    }
-                } else {
-                    ni.num = ni.num * 10 + d;
-                    ni.div *= 10;
-                    ni.di++;
-                    if (INT64_MAX <= ni.div || DEC_MAX < dec_cnt) {
-                        if (!ni.no_big) {
-                            ni.big = true;
-                        }
-                    }
-                }
+                ni.num = ni.num * 10 + d;
+                ni.div *= 10;
+                ni.di++;
             }
         }
+        if (INT64_MAX <= ni.div || DEC_MAX < dec_cnt) {
+            if (!ni.no_big) {
+                ni.big = true;
+            }
+        }
+
         if ('e' == *pi->cur || 'E' == *pi->cur) {
             int eneg = 0;
 

data/ext/oj/parse.h

@@ -97,6 +97,8 @@ static inline void parse_info_init(ParseInfo pi) {
     memset(pi, 0, sizeof(struct _parseInfo));
 }
 
+extern void oj_scanner_init();
+
 static inline bool empty_ok(Options options) {
     switch (options->mode) {
     case ObjectMode:

data/ext/oj/parser.c

@@ -533,6 +533,7 @@ static void calc_num(ojParser p) {
         // nothing to do
         break;
     }
+    p->type = OJ_NONE;
 }
 
 static void big_change(ojParser p) {
@@ -598,6 +599,8 @@ static void parse(ojParser p, const byte *json) {
     const byte *b = json;
     int         i;
 
+    p->line = 1;
+    p->col  = -1;
 #if DEBUG
     printf("*** parse - mode: %c %s\n", p->map[256], (const char *)json);
 #endif
@@ -652,6 +655,7 @@ static void parse(ojParser p, const byte *json) {
             }
             buf_append_string(&p->buf, (const char *)start, b - start);
             if ('"' == *b) {
+                p->cur = b - json;
                 p->funcs[p->stack[p->depth]].add_str(p);
                 p->map = (0 == p->depth) ? value_map : after_map;
                 break;
@@ -661,12 +665,14 @@ static void parse(ojParser p, const byte *json) {
             p->next_map = (0 == p->depth) ? value_map : after_map;
             break;
         case OPEN_OBJECT:
+            p->cur = b - json;
             p->funcs[p->stack[p->depth]].open_object(p);
             p->depth++;
             p->stack[p->depth] = OBJECT_FUN;
             p->map             = key1_map;
             break;
         case NUM_CLOSE_OBJECT:
+            p->cur = b - json;
             calc_num(p);
             // flow through
         case CLOSE_OBJECT:
@@ -677,15 +683,18 @@ static void parse(ojParser p, const byte *json) {
                 return;
             }
             p->depth--;
+            p->cur = b - json;
             p->funcs[p->stack[p->depth]].close_object(p);
             break;
         case OPEN_ARRAY:
+            p->cur = b - json;
             p->funcs[p->stack[p->depth]].open_array(p);
             p->depth++;
             p->stack[p->depth] = ARRAY_FUN;
             p->map             = value_map;
             break;
         case NUM_CLOSE_ARRAY:
+            p->cur = b - json;
             calc_num(p);
             // flow through
         case CLOSE_ARRAY:
@@ -696,9 +705,11 @@ static void parse(ojParser p, const byte *json) {
                 return;
             }
             p->depth--;
+            p->cur = b - json;
             p->funcs[p->stack[p->depth]].close_array(p);
             break;
         case NUM_COMMA:
+            p->cur = b - json;
             calc_num(p);
             if (0 < p->depth && OBJECT_FUN == p->stack[p->depth]) {
                 p->map = key_map;
@@ -860,8 +871,14 @@ static void parse(ojParser p, const byte *json) {
             b--;
             p->map = big_exp_map;
             break;
-        case NUM_SPC: calc_num(p); break;
-        case NUM_NEWLINE: calc_num(p); b++;
+        case NUM_SPC:
+            p->cur = b - json;
+            calc_num(p);
+            break;
+        case NUM_NEWLINE:
+            p->cur = b - json;
+            calc_num(p);
+            b++;
 #ifdef SPACE_JUMP
             // for (uint32_t *sj = (uint32_t*)b; 0x20202020 == *sj; sj++) { b += 4; }
             for (uint16_t *sj = (uint16_t *)b; 0x2020 == *sj; sj++) {
@@ -882,6 +899,7 @@ static void parse(ojParser p, const byte *json) {
                 buf_append_string(&p->buf, (const char *)start, b - start);
             }
             if ('"' == *b) {
+                p->cur = b - json;
                 p->funcs[p->stack[p->depth]].add_str(p);
                 p->map = p->next_map;
                 break;
@@ -890,6 +908,7 @@ static void parse(ojParser p, const byte *json) {
             break;
         case STR_SLASH: p->map = esc_map; break;
         case STR_QUOTE:
+            p->cur = b - json;
             p->funcs[p->stack[p->depth]].add_str(p);
             p->map = p->next_map;
             break;
@@ -967,6 +986,7 @@ static void parse(ojParser p, const byte *json) {
         case VAL_NULL:
             if ('u' == b[1] && 'l' == b[2] && 'l' == b[3]) {
                 b += 3;
+                p->cur = b - json;
                 p->funcs[p->stack[p->depth]].add_null(p);
                 p->map = (0 == p->depth) ? value_map : after_map;
                 break;
@@ -992,6 +1012,7 @@ static void parse(ojParser p, const byte *json) {
         case VAL_TRUE:
             if ('r' == b[1] && 'u' == b[2] && 'e' == b[3]) {
                 b += 3;
+                p->cur = b - json;
                 p->funcs[p->stack[p->depth]].add_true(p);
                 p->map = (0 == p->depth) ? value_map : after_map;
                 break;
@@ -1017,6 +1038,7 @@ static void parse(ojParser p, const byte *json) {
         case VAL_FALSE:
             if ('a' == b[1] && 'l' == b[2] && 's' == b[3] && 'e' == b[4]) {
                 b += 4;
+                p->cur = b - json;
                 p->funcs[p->stack[p->depth]].add_false(p);
                 p->map = (0 == p->depth) ? value_map : after_map;
                 break;
@@ -1050,6 +1072,7 @@ static void parse(ojParser p, const byte *json) {
                         parse_error(p, "expected null");
                         return;
                     }
+                    p->cur = b - json;
                     p->funcs[p->stack[p->depth]].add_null(p);
                     p->map = (0 == p->depth) ? value_map : after_map;
                 }
@@ -1061,6 +1084,7 @@ static void parse(ojParser p, const byte *json) {
                         parse_error(p, "expected false");
                         return;
                     }
+                    p->cur = b - json;
                     p->funcs[p->stack[p->depth]].add_false(p);
                     p->map = (0 == p->depth) ? value_map : after_map;
                 }
@@ -1072,6 +1096,7 @@ static void parse(ojParser p, const byte *json) {
                         parse_error(p, "expected true");
                         return;
                     }
+                    p->cur = b - json;
                     p->funcs[p->stack[p->depth]].add_true(p);
                     p->map = (0 == p->depth) ? value_map : after_map;
                 }
@@ -1089,6 +1114,9 @@ static void parse(ojParser p, const byte *json) {
             p->map = trail_map;
         }
     }
+    if (0 < p->depth) {
+        parse_error(p, "parse error, not closed");
+    }
     if (0 == p->depth) {
         switch (p->map[256]) {
         case '0':
@@ -1099,7 +1127,10 @@ static void parse(ojParser p, const byte *json) {
         case 'D':
         case 'g':
         case 'B':
-        case 'Y': calc_num(p); break;
+        case 'Y':
+            p->cur = b - json;
+            calc_num(p);
+            break;
         }
     }
     return;
@@ -1456,7 +1487,7 @@ static VALUE saj_parser = Qundef;
 /* Document-method: saj
  * call-seq: saj
  *
- * Returns the default saj parser. Note the default SAJ parser can not be used
+ * Returns the default SAJ parser. Note the default SAJ parser can not be used
  * concurrently in more than one thread.
  */
 static VALUE parser_saj(VALUE self) {

data/ext/oj/parser.h

@@ -80,6 +80,7 @@ typedef struct _ojParser {
 
     char     token[8];
     long     line;
+    long     cur;  // only set before call to a function
     long     col;
     int      ri;
     uint32_t ucode;

data/ext/oj/rails.c

@@ -517,7 +517,7 @@ static void dump_as_string(VALUE obj, int depth, Out out, bool as_ok) {
 static void dump_as_json(VALUE obj, int depth, Out out, bool as_ok) {
     volatile VALUE ja;
 
-    if (Yes == out->opts->trace) {
+    if (RB_UNLIKELY(Yes == out->opts->trace)) {
         oj_trace("as_json", obj, __FILE__, __LINE__, depth + 1, TraceRubyIn);
     }
     // Some classes elect to not take an options argument so check the arity
@@ -527,7 +527,7 @@ static void dump_as_json(VALUE obj, int depth, Out out, bool as_ok) {
     } else {
         ja = rb_funcall2(obj, oj_as_json_id, out->argc, out->argv);
     }
-    if (Yes == out->opts->trace) {
+    if (RB_UNLIKELY(Yes == out->opts->trace)) {
         oj_trace("as_json", obj, __FILE__, __LINE__, depth + 1, TraceRubyOut);
     }
 
@@ -1464,7 +1464,7 @@ static DumpFunc rails_funcs[] = {
 static void dump_rails_val(VALUE obj, int depth, Out out, bool as_ok) {
     int type = rb_type(obj);
 
-    if (Yes == out->opts->trace) {
+    if (RB_UNLIKELY(Yes == out->opts->trace)) {
         oj_trace("dump", obj, __FILE__, __LINE__, depth, TraceIn);
     }
     if (MAX_DEPTH < depth) {
@@ -1475,14 +1475,14 @@ static void dump_rails_val(VALUE obj, int depth, Out out, bool as_ok) {
 
         if (NULL != f) {
             f(obj, depth, out, as_ok);
-            if (Yes == out->opts->trace) {
+            if (RB_UNLIKELY(Yes == out->opts->trace)) {
                 oj_trace("dump", obj, __FILE__, __LINE__, depth, TraceOut);
             }
             return;
         }
     }
     oj_dump_nil(Qnil, depth, out, false);
-    if (Yes == out->opts->trace) {
+    if (RB_UNLIKELY(Yes == out->opts->trace)) {
         oj_trace("dump", Qnil, __FILE__, __LINE__, depth, TraceOut);
     }
 }