oj 3.17.3 → 3.17.6

data/ext/oj/parse.c

@@ -414,6 +414,11 @@ static void read_escaped_str(ParseInfo pi, const char *start) {
 
         if ('\\' == *s) {
             s++;
+            if (pi->end <= s) {
+                oj_set_error_at(pi, oj_parse_error_class, __FILE__, __LINE__, "quoted string not terminated");
+                buf_cleanup(&buf);
+                return;
+            }
             switch (*s) {
             case 'n': buf_append(&buf, '\n'); break;
             case 'r': buf_append(&buf, '\r'); break;
@@ -1096,7 +1101,9 @@ void oj_set_error_at(ParseInfo pi, VALUE err_clas, const char *file, int line, c
     }
     va_end(ap);
     pi->err.clas = err_clas;
-    if (p + 3 < end) {
+    // The eight bytes below, plus the ')' and the terminator that the two bytes
+    // held back from end are for.
+    if (p + 8 <= end) {
         *p++  = ' ';
         *p++  = '(';
         *p++  = 'a';
@@ -1306,9 +1313,16 @@ CLEANUP:
         OJ_R_FREE(json);
     }
     stack_cleanup(&pi->stack);
-    if (pi->str_rx.head != oj_default_options.str_rx.head) {
-        oj_rxclass_cleanup(&pi->str_rx);
+    // The parsers match against pi->options.str_rx. A :match_string option
+    // builds a chain of its own for this call, which has to be freed. Without
+    // the option the member still aliases the chain owned by the defaults and
+    // must be left alone.
+    if (pi->options.str_rx.head != oj_default_options.str_rx.head) {
+        oj_rxclass_cleanup(&pi->options.str_rx);
+        pi->options.str_rx.head = NULL;
+        pi->options.str_rx.tail = NULL;
     }
+    oj_free_call_options(&pi->options);
     if (err_has(&pi->err)) {
         rb_set_errinfo(Qnil);
         if (Qnil != pi->err_class) {

data/ext/oj/parse.h

@@ -49,7 +49,6 @@ typedef struct _parseInfo {
     VALUE            handler;
     struct _valStack stack;
     CircArray        circ_array;
-    struct _rxClass  str_rx;
     int              expect_value;
     int              max_depth;  // just for the json gem
     VALUE            proc;

data/ext/oj/parser.c

@@ -5,6 +5,7 @@
 #include <fcntl.h>
 
 #include "oj.h"
+#include "simd.h"
 
 #define DEBUG 0
 
@@ -594,9 +595,64 @@ static void big_change(ojParser p) {
     }
 }
 
-static void parse(ojParser p, const byte *json, bool more) {
+// Scan forward over string content, returning the first byte that is not
+// STR_OK in string_map. This is a pure drop-in for the scalar loop
+//
+//     for (; STR_OK == string_map[*b]; b++) {}
+//
+// and must return the exact same stop position. The stop set (bytes whose
+// string_map class is not 'R'/STR_OK) is:
+//
+//     * control bytes  0x00..0x1F  (class '.', also stops at the NUL terminator)
+//     * the quote      0x22 '"'    (class 'z')
+//     * the backslash  0x5C '\'    (class 'A')
+//     * every high byte 0x80..0xFF (UTF-8 lead/continuation, classes M/P/Q/'.')
+//
+// Note this differs from parse.c's string_scan_neon, which stops only on
+// \0 \\ " -- parser.c hands multi-byte UTF-8 to its state machine, so the
+// scanner must stop on the high bytes too. The predictor below is derived from
+// string_map, not copied from parse.c.
+//
+// The NEON path only loads a full 16-byte vector when [b, b+16) stays within
+// [., end), so it never reads past the string's allocation; the sub-16-byte
+// tail (and the whole scan on non-NEON builds) uses the scalar loop, which
+// stops naturally at the guaranteed NUL terminator.
+static inline const byte *oj_scan_str_simd(const byte *b, const byte *end) {
+#ifdef HAVE_SIMD_NEON
+    if (SIMD_NEON == SIMD_Impl) {
+        const uint8x16_t quote  = vdupq_n_u8('"');
+        const uint8x16_t bslash = vdupq_n_u8('\\');
+        const uint8x16_t space  = vdupq_n_u8(0x20);
+        const uint8x16_t high   = vdupq_n_u8(0x80);
+
+        while (b + sizeof(uint8x16_t) <= end) {
+            const uint8x16_t chunk = vld1q_u8((const uint8_t *)b);
+            // special lane == 0xFF for any byte that stops the scan.
+            const uint8x16_t special = vorrq_u8(vorrq_u8(vceqq_u8(chunk, quote), vceqq_u8(chunk, bslash)),
+                                                vorrq_u8(vcltq_u8(chunk, space), vcgeq_u8(chunk, high)));
+            // Reduce to a 64-bit mask with 4 bits per lane (same idiom as
+            // parse.c's string_scan_neon) and locate the first set lane.
+            const uint8x8_t res  = vshrn_n_u16(vreinterpretq_u16_u8(special), 4);
+            uint64_t        mask = vget_lane_u64(vreinterpret_u64_u8(res), 0);
+            if (0 != mask) {
+                mask &= 0x8888888888888888ull;
+                return b + (OJ_CTZ64(mask) >> 2);
+            }
+            b += sizeof(uint8x16_t);
+        }
+    }
+#else
+    (void)end;
+#endif
+    for (; STR_OK == string_map[*b]; b++) {
+    }
+    return b;
+}
+
+static void parse(ojParser p, const byte *json, size_t len, bool more) {
     const byte *start;
-    const byte *b = json;
+    const byte *b   = json;
+    const byte *end = json + len;
     int         i;
 
     p->line = 1;
@@ -629,8 +685,7 @@ static void parse(ojParser p, const byte *json, bool more) {
             b++;
             p->key.tail = p->key.head;
             start       = b;
-            for (; STR_OK == string_map[*b]; b++) {
-            }
+            b           = oj_scan_str_simd(b, end);
             buf_append_string(&p->key, (const char *)start, b - start);
             if ('"' == *b) {
                 p->map = colon_map;
@@ -654,8 +709,7 @@ static void parse(ojParser p, const byte *json, bool more) {
             b++;
             start       = b;
             p->buf.tail = p->buf.head;
-            for (; STR_OK == string_map[*b]; b++) {
-            }
+            b           = oj_scan_str_simd(b, end);
             buf_append_string(&p->buf, (const char *)start, b - start);
             if ('"' == *b) {
                 p->cur = b - json;
@@ -838,10 +892,10 @@ static void parse(ojParser p, const byte *json, bool more) {
         case EXP_DIGIT:
             p->map = exp_map;
             for (; NUM_DIGIT == digit_map[*b]; b++) {
-                int16_t x = p->num.exp * 10 + (int16_t)(*b - '0');
+                int x = p->num.exp * 10 + (*b - '0');
 
                 if (x <= MAX_EXP) {
-                    p->num.exp = x;
+                    p->num.exp = (int16_t)x;
                 } else {
                     big_change(p);
                     p->map = big_exp_map;
@@ -905,8 +959,7 @@ static void parse(ojParser p, const byte *json, bool more) {
             break;
         case STR_OK:
             start = b;
-            for (; STR_OK == string_map[*b]; b++) {
-            }
+            b     = oj_scan_str_simd(b, end);
             if (':' == p->next_map[256]) {
                 buf_append_string(&p->key, (const char *)start, b - start);
             } else {
@@ -1229,7 +1282,8 @@ static int opt_cb(VALUE rkey, VALUE value, VALUE ptr) {
  * Oj::Parser.new(:usual, cache_keys: true).
  */
 static VALUE parser_new(int argc, VALUE *argv, VALUE self) {
-    ojParser p = OJ_R_ALLOC(struct _ojParser);
+    ojParser       p = OJ_R_ALLOC(struct _ojParser);
+    volatile VALUE pv;
 
 #if HAVE_RB_EXT_RACTOR_SAFE
     // This doesn't seem to do anything.
@@ -1239,6 +1293,10 @@ static VALUE parser_new(int argc, VALUE *argv, VALUE self) {
     buf_init(&p->key);
     buf_init(&p->buf);
     p->map = value_map;
+    // Until the parser is wrapped the GC can not free it, so the mode and the
+    // options, both of which raise on a value they do not take, are handled
+    // after the wrap.
+    pv = TypedData_Wrap_Struct(parser_class, &oj_parser_type, p);
 
     if (argc < 1) {
         oj_set_parser_validator(p);
@@ -1279,7 +1337,7 @@ static VALUE parser_new(int argc, VALUE *argv, VALUE self) {
             rb_hash_foreach(ropts, opt_cb, (VALUE)p);
         }
     }
-    return TypedData_Wrap_Struct(parser_class, &oj_parser_type, p);
+    return pv;
 }
 
 // Create a new parser without setting the delegate. The parser is
@@ -1419,7 +1477,7 @@ static VALUE parser_parse(VALUE self, VALUE json) {
 
     parser_reset(p);
     p->start(p);
-    parse(p, ptr, false);
+    parse(p, ptr, (size_t)RSTRING_LEN(json), false);
     validate_document_end(p);
 
     return p->result(p);
@@ -1440,7 +1498,7 @@ static VALUE load(VALUE self) {
     while (true) {
         rb_funcall(p->reader, oj_readpartial_id, 2, INT2NUM(16385), rbuf);
         if (0 < RSTRING_LEN(rbuf)) {
-            parse(p, (byte *)StringValuePtr(rbuf), true);
+            parse(p, (byte *)StringValuePtr(rbuf), (size_t)RSTRING_LEN(rbuf), true);
         }
         if (Qtrue == rb_funcall(p->reader, oj_eofq_id, 0)) {
             if (0 < p->depth) {
@@ -1471,6 +1529,39 @@ static VALUE parser_load(VALUE self, VALUE reader) {
     return p->result(p);
 }
 
+struct _fileParse {
+    ojParser    p;
+    const char *path;
+    int         fd;
+};
+
+static VALUE file_parse(VALUE x) {
+    struct _fileParse *fp = (struct _fileParse *)x;
+    byte               buf[16385];
+    size_t             size = sizeof(buf) - 1;
+    ssize_t            rsize;
+
+    while (true) {
+        if (0 > (rsize = read(fp->fd, buf, size))) {
+            rb_raise(rb_eIOError, "error reading from %s", fp->path);
+        }
+        if (0 == rsize) {
+            break;
+        }
+        buf[rsize] = '\0';
+        parse(fp->p, buf, rsize, true);
+    }
+    validate_document_end(fp->p);
+
+    return fp->p->result(fp->p);
+}
+
+static VALUE file_close(VALUE x) {
+    close(((struct _fileParse *)x)->fd);
+
+    return Qnil;
+}
+
 /* Document-method: file(filename)
  * call-seq: file(filename)
  *
@@ -1479,9 +1570,10 @@ static VALUE parser_load(VALUE self, VALUE reader) {
  * Returns the result according to the delegate of the parser.
  */
 static VALUE parser_file(VALUE self, VALUE filename) {
-    ojParser    p;
-    const char *path;
-    int         fd;
+    struct _fileParse fp;
+    ojParser          p;
+    const char       *path;
+    int               fd;
 
     TypedData_Get_Struct(self, struct _ojParser, &oj_parser_type, p);
 
@@ -1500,26 +1592,15 @@ static VALUE parser_file(VALUE self, VALUE filename) {
         // Use threaded version.
         // TBD only if has pthreads
         // TBD parse_large(p, fd);
+        close(fd);
         return p->result(p);
     }
 #endif
-    byte   buf[16385];
-    size_t size = sizeof(buf) - 1;
-    size_t rsize;
+    fp.p    = p;
+    fp.path = path;
+    fp.fd   = fd;
 
-    while (true) {
-        if (0 < (rsize = read(fd, buf, size))) {
-            buf[rsize] = '\0';
-            parse(p, buf, true);
-        }
-        if (rsize <= 0) {
-            if (0 != rsize) {
-                rb_raise(rb_eIOError, "error reading from %s", path);
-            }
-            break;
-        }
-    }
-    return p->result(p);
+    return rb_ensure(file_parse, (VALUE)&fp, file_close, (VALUE)&fp);
 }
 
 /* Document-method: just_one
@@ -1632,15 +1713,19 @@ static VALUE parser_safe(int argc, VALUE *argv, VALUE self) {
         options = rb_hash_new();
     }
 
-    ojParser p = OJ_R_ALLOC(struct _ojParser);
+    ojParser       p = OJ_R_ALLOC(struct _ojParser);
+    volatile VALUE pv;
 
     memset(p, 0, sizeof(struct _ojParser));
     buf_init(&p->key);
     buf_init(&p->buf);
     p->map = value_map;
+    // The limits raise on a value that is not an Integer, so the parser has to
+    // be wrapped before they are read.
+    pv = TypedData_Wrap_Struct(parser_class, &oj_parser_type, p);
     oj_set_parser_safe(p, options);
 
-    return TypedData_Wrap_Struct(parser_class, &oj_parser_type, p);
+    return pv;
 }
 
 /* Document-class: Oj::Parser