oj 3.16.3 → 3.17.4

data/ext/oj/parser.c

@@ -5,6 +5,7 @@
 #include <fcntl.h>
 
 #include "oj.h"
+#include "simd.h"
 
 #define DEBUG 0
 
@@ -75,7 +76,7 @@ enum {
 
 /*
 0123456789abcdef0123456789abcdef */
-static const char value_map[257] = "\
+static const char value_map[258] = "\
 X........ab..a..................\
 a.i..........f..ghhhhhhhhh......\
 ...........................k.m..\
@@ -85,7 +86,7 @@ a.i..........f..ghhhhhhhhh......\
 ................................\
 ................................v";
 
-static const char null_map[257] = "\
+static const char null_map[258] = "\
 ................................\
 ............o...................\
 ................................\
@@ -95,7 +96,7 @@ static const char null_map[257] = "\
 ................................\
 ................................N";
 
-static const char true_map[257] = "\
+static const char true_map[258] = "\
 ................................\
 ............o...................\
 ................................\
@@ -105,7 +106,7 @@ static const char true_map[257] = "\
 ................................\
 ................................T";
 
-static const char false_map[257] = "\
+static const char false_map[258] = "\
 ................................\
 ............o...................\
 ................................\
@@ -115,7 +116,7 @@ static const char false_map[257] = "\
 ................................\
 ................................F";
 
-static const char comma_map[257] = "\
+static const char comma_map[258] = "\
 .........ab..a..................\
 a.i..........f..ghhhhhhhhh......\
 ...........................k....\
@@ -125,7 +126,7 @@ a.i..........f..ghhhhhhhhh......\
 ................................\
 ................................,";
 
-static const char after_map[257] = "\
+static const char after_map[258] = "\
 X........ab..a..................\
 a...........o...................\
 .............................m..\
@@ -135,7 +136,7 @@ a...........o...................\
 ................................\
 ................................a";
 
-static const char key1_map[257] = "\
+static const char key1_map[258] = "\
 .........ab..a..................\
 a.p.............................\
 ................................\
@@ -145,7 +146,7 @@ a.p.............................\
 ................................\
 ................................K";
 
-static const char key_map[257] = "\
+static const char key_map[258] = "\
 .........ab..a..................\
 a.p.............................\
 ................................\
@@ -155,7 +156,7 @@ a.p.............................\
 ................................\
 ................................k";
 
-static const char colon_map[257] = "\
+static const char colon_map[258] = "\
 .........ab..a..................\
 a.........................q.....\
 ................................\
@@ -165,7 +166,7 @@ a.........................q.....\
 ................................\
 ................................:";
 
-static const char neg_map[257] = "\
+static const char neg_map[258] = "\
 ................................\
 ................O---------......\
 ................................\
@@ -175,7 +176,7 @@ static const char neg_map[257] = "\
 ................................\
 ................................-";
 
-static const char zero_map[257] = "\
+static const char zero_map[258] = "\
 .........rs..r..................\
 r...........u.t.................\
 .............................H..\
@@ -185,7 +186,7 @@ r...........u.t.................\
 ................................\
 ................................0";
 
-static const char digit_map[257] = "\
+static const char digit_map[258] = "\
 .........rs..r..................\
 r...........u.t.NNNNNNNNNN......\
 .....w.......................H..\
@@ -195,7 +196,7 @@ r...........u.t.NNNNNNNNNN......\
 ................................\
 ................................d";
 
-static const char dot_map[257] = "\
+static const char dot_map[258] = "\
 ................................\
 ................vvvvvvvvvv......\
 ................................\
@@ -205,7 +206,7 @@ static const char dot_map[257] = "\
 ................................\
 .................................";
 
-static const char frac_map[257] = "\
+static const char frac_map[258] = "\
 .........rs..r..................\
 r...........u...vvvvvvvvvv......\
 .....w.......................H..\
@@ -215,7 +216,7 @@ r...........u...vvvvvvvvvv......\
 ................................\
 ................................f";
 
-static const char exp_sign_map[257] = "\
+static const char exp_sign_map[258] = "\
 ................................\
 ...........x.x..yyyyyyyyyy......\
 ................................\
@@ -225,7 +226,7 @@ static const char exp_sign_map[257] = "\
 ................................\
 ................................x";
 
-static const char exp_zero_map[257] = "\
+static const char exp_zero_map[258] = "\
 ................................\
 ................yyyyyyyyyy......\
 ................................\
@@ -235,7 +236,7 @@ static const char exp_zero_map[257] = "\
 ................................\
 ................................z";
 
-static const char exp_map[257] = "\
+static const char exp_map[258] = "\
 .........rs..r..................\
 r...........u...yyyyyyyyyy......\
 .............................H..\
@@ -245,7 +246,7 @@ r...........u...yyyyyyyyyy......\
 ................................\
 ................................X";
 
-static const char big_digit_map[257] = "\
+static const char big_digit_map[258] = "\
 .........rs..r..................\
 r...........u.D.CCCCCCCCCC......\
 .....J.......................H..\
@@ -255,7 +256,7 @@ r...........u.D.CCCCCCCCCC......\
 ................................\
 ................................D";
 
-static const char big_dot_map[257] = "\
+static const char big_dot_map[258] = "\
 ................................\
 ................IIIIIIIIII......\
 ................................\
@@ -265,7 +266,7 @@ static const char big_dot_map[257] = "\
 ................................\
 ................................o";
 
-static const char big_frac_map[257] = "\
+static const char big_frac_map[258] = "\
 .........rs..r..................\
 r...........u...IIIIIIIIII......\
 .....J.......................H..\
@@ -275,7 +276,7 @@ r...........u...IIIIIIIIII......\
 ................................\
 ................................g";
 
-static const char big_exp_sign_map[257] = "\
+static const char big_exp_sign_map[258] = "\
 ................................\
 ...........K.K..LLLLLLLLLL......\
 ................................\
@@ -285,7 +286,7 @@ static const char big_exp_sign_map[257] = "\
 ................................\
 ................................B";
 
-static const char big_exp_zero_map[257] = "\
+static const char big_exp_zero_map[258] = "\
 ................................\
 ................LLLLLLLLLL......\
 ................................\
@@ -295,7 +296,7 @@ static const char big_exp_zero_map[257] = "\
 ................................\
 ................................Z";
 
-static const char big_exp_map[257] = "\
+static const char big_exp_map[258] = "\
 .........rs..r..................\
 r...........u...LLLLLLLLLL......\
 .............................H..\
@@ -305,7 +306,7 @@ r...........u...LLLLLLLLLL......\
 ................................\
 ................................Y";
 
-static const char string_map[257] = "\
+static const char string_map[258] = "\
 ................................\
 RRzRRRRRRRRRRRRRRRRRRRRRRRRRRRRR\
 RRRRRRRRRRRRRRRRRRRRRRRRRRRRARRR\
@@ -315,7 +316,7 @@ RRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRR\
 MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM\
 PPPPPPPPPPPPPPPPQQQQQQQQ........s";
 
-static const char esc_map[257] = "\
+static const char esc_map[258] = "\
 ................................\
 ..B............B................\
 ............................B...\
@@ -325,7 +326,7 @@ static const char esc_map[257] = "\
 ................................\
 ................................~";
 
-static const char esc_byte_map[257] = "\
+static const char esc_byte_map[258] = "\
 ................................\
 ..\"............/................\
 ............................\\...\
@@ -335,7 +336,7 @@ static const char esc_byte_map[257] = "\
 ................................\
 ................................b";
 
-static const char u_map[257] = "\
+static const char u_map[258] = "\
 ................................\
 ................EEEEEEEEEE......\
 .EEEEEE.........................\
@@ -345,7 +346,7 @@ static const char u_map[257] = "\
 ................................\
 ................................u";
 
-static const char utf_map[257] = "\
+static const char utf_map[258] = "\
 ................................\
 ................................\
 ................................\
@@ -355,7 +356,7 @@ SSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSS\
 ................................\
 ................................8";
 
-static const char space_map[257] = "\
+static const char space_map[258] = "\
 .........ab..a..................\
 a...............................\
 ................................\
@@ -365,7 +366,7 @@ a...............................\
 ................................\
 ................................S";
 
-static const char trail_map[257] = "\
+static const char trail_map[258] = "\
 .........ab..a..................\
 a...............................\
 ................................\
@@ -375,7 +376,7 @@ a...............................\
 ................................\
 ................................R";
 
-static const byte hex_map[256] = "\
+static const byte hex_map[257] = "\
 ................................\
 ................\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09......\
 .\x0a\x0b\x0c\x0d\x0e\x0f.........................\
@@ -594,9 +595,64 @@ static void big_change(ojParser p) {
     }
 }
 
-static void parse(ojParser p, const byte *json) {
+// Scan forward over string content, returning the first byte that is not
+// STR_OK in string_map. This is a pure drop-in for the scalar loop
+//
+//     for (; STR_OK == string_map[*b]; b++) {}
+//
+// and must return the exact same stop position. The stop set (bytes whose
+// string_map class is not 'R'/STR_OK) is:
+//
+//     * control bytes  0x00..0x1F  (class '.', also stops at the NUL terminator)
+//     * the quote      0x22 '"'    (class 'z')
+//     * the backslash  0x5C '\'    (class 'A')
+//     * every high byte 0x80..0xFF (UTF-8 lead/continuation, classes M/P/Q/'.')
+//
+// Note this differs from parse.c's string_scan_neon, which stops only on
+// \0 \\ " -- parser.c hands multi-byte UTF-8 to its state machine, so the
+// scanner must stop on the high bytes too. The predictor below is derived from
+// string_map, not copied from parse.c.
+//
+// The NEON path only loads a full 16-byte vector when [b, b+16) stays within
+// [., end), so it never reads past the string's allocation; the sub-16-byte
+// tail (and the whole scan on non-NEON builds) uses the scalar loop, which
+// stops naturally at the guaranteed NUL terminator.
+static inline const byte *oj_scan_str_simd(const byte *b, const byte *end) {
+#ifdef HAVE_SIMD_NEON
+    if (SIMD_NEON == SIMD_Impl) {
+        const uint8x16_t quote  = vdupq_n_u8('"');
+        const uint8x16_t bslash = vdupq_n_u8('\\');
+        const uint8x16_t space  = vdupq_n_u8(0x20);
+        const uint8x16_t high   = vdupq_n_u8(0x80);
+
+        while (b + sizeof(uint8x16_t) <= end) {
+            const uint8x16_t chunk = vld1q_u8((const uint8_t *)b);
+            // special lane == 0xFF for any byte that stops the scan.
+            const uint8x16_t special = vorrq_u8(vorrq_u8(vceqq_u8(chunk, quote), vceqq_u8(chunk, bslash)),
+                                                vorrq_u8(vcltq_u8(chunk, space), vcgeq_u8(chunk, high)));
+            // Reduce to a 64-bit mask with 4 bits per lane (same idiom as
+            // parse.c's string_scan_neon) and locate the first set lane.
+            const uint8x8_t res  = vshrn_n_u16(vreinterpretq_u16_u8(special), 4);
+            uint64_t        mask = vget_lane_u64(vreinterpret_u64_u8(res), 0);
+            if (0 != mask) {
+                mask &= 0x8888888888888888ull;
+                return b + (OJ_CTZ64(mask) >> 2);
+            }
+            b += sizeof(uint8x16_t);
+        }
+    }
+#else
+    (void)end;
+#endif
+    for (; STR_OK == string_map[*b]; b++) {
+    }
+    return b;
+}
+
+static void parse(ojParser p, const byte *json, size_t len, bool more) {
     const byte *start;
-    const byte *b = json;
+    const byte *b   = json;
+    const byte *end = json + len;
     int         i;
 
     p->line = 1;
@@ -629,12 +685,14 @@ static void parse(ojParser p, const byte *json) {
             b++;
             p->key.tail = p->key.head;
             start       = b;
-            for (; STR_OK == string_map[*b]; b++) {
-            }
+            b           = oj_scan_str_simd(b, end);
             buf_append_string(&p->key, (const char *)start, b - start);
             if ('"' == *b) {
                 p->map = colon_map;
                 break;
+            } else if ('\0' == *b && !more) {
+                parse_error(p, "quoted string not terminated");
+                break;
             }
             b--;
             p->map      = string_map;
@@ -651,14 +709,16 @@ static void parse(ojParser p, const byte *json) {
             b++;
             start       = b;
             p->buf.tail = p->buf.head;
-            for (; STR_OK == string_map[*b]; b++) {
-            }
+            b           = oj_scan_str_simd(b, end);
             buf_append_string(&p->buf, (const char *)start, b - start);
             if ('"' == *b) {
                 p->cur = b - json;
                 p->funcs[p->stack[p->depth]].add_str(p);
                 p->map = (0 == p->depth) ? value_map : after_map;
                 break;
+            } else if ('\0' == *b && !more) {
+                parse_error(p, "quoted string not terminated");
+                break;
             }
             b--;
             p->map      = string_map;
@@ -668,6 +728,10 @@ static void parse(ojParser p, const byte *json) {
             p->cur = b - json;
             p->funcs[p->stack[p->depth]].open_object(p);
             p->depth++;
+            if ((int)sizeof(p->stack) <= p->depth) {
+                parse_error(p, "too deeply nested");
+                break;
+            }
             p->stack[p->depth] = OBJECT_FUN;
             p->map             = key1_map;
             break;
@@ -690,6 +754,10 @@ static void parse(ojParser p, const byte *json) {
             p->cur = b - json;
             p->funcs[p->stack[p->depth]].open_array(p);
             p->depth++;
+            if ((int)sizeof(p->stack) <= p->depth) {
+                parse_error(p, "too deeply nested");
+                break;
+            }
             p->stack[p->depth] = ARRAY_FUN;
             p->map             = value_map;
             break;
@@ -891,8 +959,7 @@ static void parse(ojParser p, const byte *json) {
             break;
         case STR_OK:
             start = b;
-            for (; STR_OK == string_map[*b]; b++) {
-            }
+            b     = oj_scan_str_simd(b, end);
             if (':' == p->next_map[256]) {
                 buf_append_string(&p->key, (const char *)start, b - start);
             } else {
@@ -1114,9 +1181,6 @@ static void parse(ojParser p, const byte *json) {
             p->map = trail_map;
         }
     }
-    if (0 < p->depth) {
-        parse_error(p, "parse error, not closed");
-    }
     if (0 == p->depth) {
         switch (p->map[256]) {
         case '0':
@@ -1164,7 +1228,7 @@ static void parser_mark(void *ptr) {
     }
 }
 
-static const rb_data_type_t oj_parser_type = {
+const rb_data_type_t oj_parser_type = {
     "Oj/parser",
     {
         parser_mark,
@@ -1179,6 +1243,8 @@ extern void oj_set_parser_validator(ojParser p);
 extern void oj_set_parser_saj(ojParser p);
 extern void oj_set_parser_usual(ojParser p);
 extern void oj_set_parser_debug(ojParser p);
+extern void oj_set_parser_safe(ojParser p, VALUE options);
+extern void oj_safe_init(VALUE parser_class);
 
 static int opt_cb(VALUE rkey, VALUE value, VALUE ptr) {
     ojParser    p   = (ojParser)ptr;
@@ -1357,6 +1423,34 @@ static VALUE parser_missing(int argc, VALUE *argv, VALUE self) {
     return p->option(p, key, rv);
 }
 
+static void validate_primitives_are_complete(ojParser p) {
+    if (0 >= p->ri) {
+        return;
+    }
+
+    switch (p->map[256]) {
+    case 'N': parse_error(p, "expected null"); break;
+    case 'F': parse_error(p, "expected false"); break;
+    case 'T': parse_error(p, "expected true"); break;
+    }
+}
+
+static void validate_non_primitives_are_complete(ojParser p) {
+    if (0 >= p->depth) {
+        return;
+    }
+    if (OBJECT_FUN == p->stack[p->depth]) {
+        parse_error(p, "Object is not closed");
+    } else {
+        parse_error(p, "Array is not closed");
+    }
+}
+
+static void validate_document_end(ojParser p) {
+    validate_primitives_are_complete(p);
+    validate_non_primitives_are_complete(p);
+}
+
 /* Document-method: parse(json)
  * call-seq: parse(json)
  *
@@ -1366,13 +1460,20 @@ static VALUE parser_missing(int argc, VALUE *argv, VALUE self) {
  */
 static VALUE parser_parse(VALUE self, VALUE json) {
     ojParser    p;
-    const byte *ptr = (const byte *)StringValuePtr(json);
+    int         frozen = OBJ_FROZEN(json);
+    const byte *ptr;
+
+    if (!frozen) {
+        rb_str_freeze(json);
+    }
+    ptr = (const byte *)StringValuePtr(json);
 
     TypedData_Get_Struct(self, struct _ojParser, &oj_parser_type, p);
 
     parser_reset(p);
     p->start(p);
-    parse(p, ptr);
+    parse(p, ptr, (size_t)RSTRING_LEN(json), false);
+    validate_document_end(p);
 
     return p->result(p);
 }
@@ -1392,7 +1493,13 @@ static VALUE load(VALUE self) {
     while (true) {
         rb_funcall(p->reader, oj_readpartial_id, 2, INT2NUM(16385), rbuf);
         if (0 < RSTRING_LEN(rbuf)) {
-            parse(p, (byte *)StringValuePtr(rbuf));
+            parse(p, (byte *)StringValuePtr(rbuf), (size_t)RSTRING_LEN(rbuf), true);
+        }
+        if (Qtrue == rb_funcall(p->reader, oj_eofq_id, 0)) {
+            if (0 < p->depth) {
+                parse_error(p, "parse error, not closed");
+            }
+            break;
         }
     }
     return Qtrue;
@@ -1456,7 +1563,7 @@ static VALUE parser_file(VALUE self, VALUE filename) {
     while (true) {
         if (0 < (rsize = read(fd, buf, size))) {
             buf[rsize] = '\0';
-            parse(p, buf);
+            parse(p, buf, rsize, true);
         }
         if (rsize <= 0) {
             if (0 != rsize) {
@@ -1567,6 +1674,28 @@ static VALUE parser_validate(VALUE self) {
     return validate_parser;
 }
 
+static VALUE parser_safe(int argc, VALUE *argv, VALUE self) {
+    VALUE options;
+
+    if (1 == argc) {
+        options = argv[0];
+
+        Check_Type(options, T_HASH);
+    } else {
+        options = rb_hash_new();
+    }
+
+    ojParser p = OJ_R_ALLOC(struct _ojParser);
+
+    memset(p, 0, sizeof(struct _ojParser));
+    buf_init(&p->key);
+    buf_init(&p->buf);
+    p->map = value_map;
+    oj_set_parser_safe(p, options);
+
+    return TypedData_Wrap_Struct(parser_class, &oj_parser_type, p);
+}
+
 /* Document-class: Oj::Parser
  *
  * A reusable parser that makes use of named delegates to determine the
@@ -1594,4 +1723,7 @@ void oj_parser_init(void) {
     rb_define_module_function(parser_class, "usual", parser_usual, 0);
     rb_define_module_function(parser_class, "saj", parser_saj, 0);
     rb_define_module_function(parser_class, "validate", parser_validate, 0);
+    rb_define_module_function(parser_class, "safe", parser_safe, -1);
+
+    oj_safe_init(parser_class);
 }

data/ext/oj/parser.h

@@ -32,9 +32,9 @@ typedef struct _num {
     long double dub;
     int64_t     fixnum;  // holds all digits
     uint32_t    len;
-    int16_t     div;     // 10^div
+    int16_t     div;  // 10^div
     int16_t     exp;
-    uint8_t     shift;   // shift of fixnum to get decimal
+    uint8_t     shift;  // shift of fixnum to get decimal
     bool        neg;
     bool        exp_neg;
     // for numbers as strings, reuse buf
@@ -42,6 +42,8 @@ typedef struct _num {
 
 struct _ojParser;
 
+extern const rb_data_type_t oj_parser_type;
+
 typedef struct _funcs {
     void (*add_null)(struct _ojParser *p);
     void (*add_true)(struct _ojParser *p);