oj 3.13.16 → 3.13.17

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d07b4081b5dda88146594adba591542d11106f1c1eadcf7c56e38358d0ee0af3
-  data.tar.gz: 6bdc82c3812f45a8ae9d72ec0b2f43ee223d95ac250399fc8dae8daf34cd3b49
+  metadata.gz: b2ba7c6f3821ac59250c2893d01269cd45bb2ec5eb4b06babccf480a07fb8862
+  data.tar.gz: 666b047d4ac8f5637e6b1e8ca995bceb105197eabc4664936cc72986280684c2
 SHA512:
-  metadata.gz: 4358aa15b7956a5b577163e43fbdc0512212a218124878ceedc2932483f27acabed2d2b92eb888c8e54601f4f4aaf3dce3b959a36effbea989e7e7ea8956d1c1
-  data.tar.gz: 1e1710bce62aadfbcc1196357d2e692892b75cc18c982fb6f53c2e35782d9bd2256cb3b6ab98951691430f23e8c2d8db26987a95dcba29f87b472729613b8d2d
+  metadata.gz: 85005fe527745233220a680adf9bf3c24bb711abbe6b6758ccff236398c7f1218c9bc586eea7954388548772b17073e5540595d160948e19fffb94e535363ea4
+  data.tar.gz: 5b8b227c2d7e8156adb1adccff99ca18a8f2735ec3b7af9f8ee3f2b747427459c497ae738fe6c941d3df6e5131342677b1e8c08f208edeb7988697f80499339d

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 # CHANGELOG
 
+## 3.13.17 - 2022-07-15
+
+- Fixed Oj::Parser to detect unterminated arrays and objects.
+
 ## 3.13.16 - 2022-07-06
 
 - Added line and column as optional arguments to the Oj::Parser.saj parser.

data/ext/oj/buf.h

@@ -39,6 +39,10 @@ inline static const char *buf_str(Buf buf) {
 }
 
 inline static void buf_append_string(Buf buf, const char *s, size_t slen) {
+    if (0 == slen) {
+        return;
+    }
+
     if (buf->end <= buf->tail + slen) {
         size_t len     = buf->end - buf->head;
         size_t toff    = buf->tail - buf->head;

data/ext/oj/parse.c

@@ -183,6 +183,36 @@ static void unicode_to_chars(ParseInfo pi, Buf buf, uint32_t code) {
     }
 }
 
+static inline const char *scan_string_noSIMD(const char *str, const char *end) {
+    for (; '"' != *str; str++) {
+        if (end <= str || '\0' == *str || '\\' == *str) {
+            break;
+        }
+    }
+    return str;
+}
+
+#if defined(OJ_USE_SSE4_2)
+#include <nmmintrin.h>
+
+static inline const char *scan_string_SIMD(const char *str, const char *end) {
+    static const char chars[16] = "\x00\\\"";
+    const __m128i terminate = _mm_loadu_si128((const __m128i *)&chars[0]);
+    const char *_end = (const char *)(end - 16);
+
+    for (; str <= _end; str += 16) {
+        const __m128i string = _mm_loadu_si128((const __m128i *)str);
+        const int r = _mm_cmpestri(terminate, 3, string, 16, _SIDD_UBYTE_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_LEAST_SIGNIFICANT);
+        if (r != 16) {
+            str = (char*)(str + r);
+            return str;
+        }
+    }
+
+    return scan_string_noSIMD(str, end);
+}
+#endif
+
 // entered at /
 static void read_escaped_str(ParseInfo pi, const char *start) {
     struct _buf buf;
@@ -192,11 +222,15 @@ static void read_escaped_str(ParseInfo pi, const char *start) {
     Val         parent = stack_peek(&pi->stack);
 
     buf_init(&buf);
-    if (0 < cnt) {
-        buf_append_string(&buf, start, cnt);
-    }
-    for (s = pi->cur; '"' != *s; s++) {
-        if (s >= pi->end) {
+    buf_append_string(&buf, start, cnt);
+
+    for (s = pi->cur; '"' != *s;) {
+#if defined(OJ_USE_SSE4_2)
+        const char *scanned = scan_string_SIMD(s, pi->end);
+#else
+        const char *scanned = scan_string_noSIMD(s, pi->end);
+#endif
+        if (scanned >= pi->end) {
             oj_set_error_at(pi,
                             oj_parse_error_class,
                             __FILE__,
@@ -204,7 +238,12 @@ static void read_escaped_str(ParseInfo pi, const char *start) {
                             "quoted string not terminated");
             buf_cleanup(&buf);
             return;
-        } else if ('\\' == *s) {
+        }
+
+        buf_append_string(&buf, s, (size_t)(scanned - s));
+        s = scanned;
+
+        if ('\\' == *s) {
             s++;
             switch (*s) {
             case 'n': buf_append(&buf, '\n'); break;
@@ -273,8 +312,7 @@ static void read_escaped_str(ParseInfo pi, const char *start) {
                 buf_cleanup(&buf);
                 return;
             }
-        } else {
-            buf_append(&buf, *s);
+            s++;
         }
     }
     if (0 == parent) {
@@ -327,43 +365,14 @@ static void read_escaped_str(ParseInfo pi, const char *start) {
     buf_cleanup(&buf);
 }
 
-static inline void scan_string_noSIMD(ParseInfo pi) {
-    for (; '"' != *pi->cur; pi->cur++) {
-        if (pi->end <= pi->cur || '\0' == *pi->cur || '\\' == *pi->cur) {
-            return;
-        }
-    }
-}
-
-#if defined(OJ_USE_SSE4_2)
-#include <nmmintrin.h>
-
-static inline void scan_string_SIMD(ParseInfo pi) {
-    static const char chars[16] = "\x00\\\"";
-    const __m128i terminate = _mm_loadu_si128((const __m128i *)&chars[0]);
-    const char *end = (const char *)(pi->end - 16);
-
-    for (; pi->cur <= end; pi->cur += 16) {
-        const __m128i string = _mm_loadu_si128((const __m128i *)pi->cur);
-        const int r = _mm_cmpestri(terminate, 3, string, 16, _SIDD_UBYTE_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_LEAST_SIGNIFICANT);
-        if (r != 16) {
-            pi->cur = (const char*)(pi->cur + r);
-            return;
-        }
-    }
-
-    scan_string_noSIMD(pi);
-}
-#endif
-
 static void read_str(ParseInfo pi) {
     const char *str    = pi->cur;
     Val         parent = stack_peek(&pi->stack);
 
 #if defined(OJ_USE_SSE4_2)
-    scan_string_SIMD(pi);
+    pi->cur = scan_string_SIMD(pi->cur, pi->end);
 #else
-    scan_string_noSIMD(pi);
+    pi->cur = scan_string_noSIMD(pi->cur, pi->end);
 #endif
     if (RB_UNLIKELY(pi->end <= pi->cur)) {
         oj_set_error_at(pi,

data/ext/oj/parser.c

@@ -600,7 +600,7 @@ static void parse(ojParser p, const byte *json) {
     int         i;
 
     p->line = 1;
-    p->col = -1;
+    p->col  = -1;
 #if DEBUG
     printf("*** parse - mode: %c %s\n", p->map[256], (const char *)json);
 #endif
@@ -655,7 +655,7 @@ static void parse(ojParser p, const byte *json) {
             }
             buf_append_string(&p->buf, (const char *)start, b - start);
             if ('"' == *b) {
-		p->cur = b - json;
+                p->cur = b - json;
                 p->funcs[p->stack[p->depth]].add_str(p);
                 p->map = (0 == p->depth) ? value_map : after_map;
                 break;
@@ -665,14 +665,14 @@ static void parse(ojParser p, const byte *json) {
             p->next_map = (0 == p->depth) ? value_map : after_map;
             break;
         case OPEN_OBJECT:
-	    p->cur = b - json;
+            p->cur = b - json;
             p->funcs[p->stack[p->depth]].open_object(p);
             p->depth++;
             p->stack[p->depth] = OBJECT_FUN;
             p->map             = key1_map;
             break;
         case NUM_CLOSE_OBJECT:
-	    p->cur = b - json;
+            p->cur = b - json;
             calc_num(p);
             // flow through
         case CLOSE_OBJECT:
@@ -683,18 +683,18 @@ static void parse(ojParser p, const byte *json) {
                 return;
             }
             p->depth--;
-	    p->cur = b - json;
+            p->cur = b - json;
             p->funcs[p->stack[p->depth]].close_object(p);
             break;
         case OPEN_ARRAY:
-	    p->cur = b - json;
+            p->cur = b - json;
             p->funcs[p->stack[p->depth]].open_array(p);
             p->depth++;
             p->stack[p->depth] = ARRAY_FUN;
             p->map             = value_map;
             break;
         case NUM_CLOSE_ARRAY:
-	    p->cur = b - json;
+            p->cur = b - json;
             calc_num(p);
             // flow through
         case CLOSE_ARRAY:
@@ -705,11 +705,11 @@ static void parse(ojParser p, const byte *json) {
                 return;
             }
             p->depth--;
-	    p->cur = b - json;
+            p->cur = b - json;
             p->funcs[p->stack[p->depth]].close_array(p);
             break;
         case NUM_COMMA:
-	    p->cur = b - json;
+            p->cur = b - json;
             calc_num(p);
             if (0 < p->depth && OBJECT_FUN == p->stack[p->depth]) {
                 p->map = key_map;
@@ -872,13 +872,13 @@ static void parse(ojParser p, const byte *json) {
             p->map = big_exp_map;
             break;
         case NUM_SPC:
-	    p->cur = b - json;
-	    calc_num(p);
-	    break;
+            p->cur = b - json;
+            calc_num(p);
+            break;
         case NUM_NEWLINE:
-	    p->cur = b - json;
-	    calc_num(p);
-	    b++;
+            p->cur = b - json;
+            calc_num(p);
+            b++;
 #ifdef SPACE_JUMP
             // for (uint32_t *sj = (uint32_t*)b; 0x20202020 == *sj; sj++) { b += 4; }
             for (uint16_t *sj = (uint16_t *)b; 0x2020 == *sj; sj++) {
@@ -899,7 +899,7 @@ static void parse(ojParser p, const byte *json) {
                 buf_append_string(&p->buf, (const char *)start, b - start);
             }
             if ('"' == *b) {
-		p->cur = b - json;
+                p->cur = b - json;
                 p->funcs[p->stack[p->depth]].add_str(p);
                 p->map = p->next_map;
                 break;
@@ -908,7 +908,7 @@ static void parse(ojParser p, const byte *json) {
             break;
         case STR_SLASH: p->map = esc_map; break;
         case STR_QUOTE:
-	    p->cur = b - json;
+            p->cur = b - json;
             p->funcs[p->stack[p->depth]].add_str(p);
             p->map = p->next_map;
             break;
@@ -986,7 +986,7 @@ static void parse(ojParser p, const byte *json) {
         case VAL_NULL:
             if ('u' == b[1] && 'l' == b[2] && 'l' == b[3]) {
                 b += 3;
-		p->cur = b - json;
+                p->cur = b - json;
                 p->funcs[p->stack[p->depth]].add_null(p);
                 p->map = (0 == p->depth) ? value_map : after_map;
                 break;
@@ -1012,7 +1012,7 @@ static void parse(ojParser p, const byte *json) {
         case VAL_TRUE:
             if ('r' == b[1] && 'u' == b[2] && 'e' == b[3]) {
                 b += 3;
-		p->cur = b - json;
+                p->cur = b - json;
                 p->funcs[p->stack[p->depth]].add_true(p);
                 p->map = (0 == p->depth) ? value_map : after_map;
                 break;
@@ -1038,7 +1038,7 @@ static void parse(ojParser p, const byte *json) {
         case VAL_FALSE:
             if ('a' == b[1] && 'l' == b[2] && 's' == b[3] && 'e' == b[4]) {
                 b += 4;
-		p->cur = b - json;
+                p->cur = b - json;
                 p->funcs[p->stack[p->depth]].add_false(p);
                 p->map = (0 == p->depth) ? value_map : after_map;
                 break;
@@ -1072,7 +1072,7 @@ static void parse(ojParser p, const byte *json) {
                         parse_error(p, "expected null");
                         return;
                     }
-		    p->cur = b - json;
+                    p->cur = b - json;
                     p->funcs[p->stack[p->depth]].add_null(p);
                     p->map = (0 == p->depth) ? value_map : after_map;
                 }
@@ -1084,7 +1084,7 @@ static void parse(ojParser p, const byte *json) {
                         parse_error(p, "expected false");
                         return;
                     }
-		    p->cur = b - json;
+                    p->cur = b - json;
                     p->funcs[p->stack[p->depth]].add_false(p);
                     p->map = (0 == p->depth) ? value_map : after_map;
                 }
@@ -1096,7 +1096,7 @@ static void parse(ojParser p, const byte *json) {
                         parse_error(p, "expected true");
                         return;
                     }
-		    p->cur = b - json;
+                    p->cur = b - json;
                     p->funcs[p->stack[p->depth]].add_true(p);
                     p->map = (0 == p->depth) ? value_map : after_map;
                 }
@@ -1114,6 +1114,9 @@ static void parse(ojParser p, const byte *json) {
             p->map = trail_map;
         }
     }
+    if (0 < p->depth) {
+        parse_error(p, "parse error, not closed");
+    }
     if (0 == p->depth) {
         switch (p->map[256]) {
         case '0':
@@ -1125,9 +1128,9 @@ static void parse(ojParser p, const byte *json) {
         case 'g':
         case 'B':
         case 'Y':
-	    p->cur = b - json;
-	    calc_num(p);
-	    break;
+            p->cur = b - json;
+            calc_num(p);
+            break;
         }
     }
     return;

data/ext/oj/validate.c

@@ -2,50 +2,45 @@
 
 #include "parser.h"
 
-static void
-noop(ojParser p) {
+static void noop(ojParser p) {
 }
 
-static VALUE
-option(ojParser p, const char *key, VALUE value) {
+static VALUE option(ojParser p, const char *key, VALUE value) {
     rb_raise(rb_eArgError, "%s is not an option for the validate delegate", key);
     return Qnil;
 }
 
-static VALUE
-result(ojParser p) {
+static VALUE result(ojParser p) {
     return Qnil;
 }
 
-static void
-dfree(ojParser p) {
+static void dfree(ojParser p) {
 }
 
-static void
-mark(ojParser p) {
+static void mark(ojParser p) {
 }
 
 void oj_set_parser_validator(ojParser p) {
-    p->ctx = NULL;
-    Funcs	end = p->funcs + 3;
+    Funcs end = p->funcs + 3;
     Funcs f;
+    p->ctx = NULL;
 
     for (f = p->funcs; f < end; f++) {
-	f->add_null = noop;
-	f->add_true = noop;
-	f->add_false = noop;
-	f->add_int = noop;
-	f->add_float = noop;
-	f->add_big = noop;
-	f->add_str = noop;
-	f->open_array = noop;
-	f->close_array = noop;
-	f->open_object = noop;
-	f->close_object = noop;
+        f->add_null     = noop;
+        f->add_true     = noop;
+        f->add_false    = noop;
+        f->add_int      = noop;
+        f->add_float    = noop;
+        f->add_big      = noop;
+        f->add_str      = noop;
+        f->open_array   = noop;
+        f->close_array  = noop;
+        f->open_object  = noop;
+        f->close_object = noop;
     }
     p->option = option;
     p->result = result;
-    p->free = dfree;
-    p->mark = mark;
-    p->start = noop;
+    p->free   = dfree;
+    p->mark   = mark;
+    p->start  = noop;
 }

data/lib/oj/version.rb

@@ -1,5 +1,5 @@
 
 module Oj
   # Current version of the module.
-  VERSION = '3.13.16'
+  VERSION = '3.13.17'
 end

data/test/bar.rb

@@ -6,11 +6,6 @@ $: << File.join(File.dirname(__FILE__), "../ext")
 
 require 'oj'
 
-puts Oj.dump({
-  "float_test" => 0.25,
-  "nan_test" => Float::NAN,
-  "inf_test" => Float::INFINITY,
-  "minus_inf_test" => -Float::INFINITY,
-  "min_test" => Float::MIN,
-  "max_test" => Float::MAX,
-}, mode: :object) # => {"float_test":0.25,"nan_test":3.3e14159265358979323846
+p = Oj::Parser.validate
+# p = Oj::Parser.new(:debug)
+p.parse(%|{|)

data/test/test_compat.rb

@@ -504,6 +504,15 @@ class CompatJuice < Minitest::Test
     assert_equal("ぴーたー", Oj.load(json)['a'])
   end
 
+  def test_parse_large_escaped_string
+    invalid_json = %|{"a":\"aaaa\\nbbbb\\rcccc\\tddd\\feee\\bf\/\\\\\\u3074\\u30fc\\u305f\\u30fc                             }|
+    error = assert_raises() { Oj.load(invalid_json) }
+    assert(error.message.include?('quoted string not terminated'))
+
+    json = "\"aaaa\\nbbbb\\rcccc\\tddd\\feee\\bf\/\\\\\\u3074\\u30fc\\u305f\\u30fc             \""
+    assert_equal("aaaa\nbbbb\rcccc\tddd\feee\bf/\\ぴーたー             ", Oj.load(json))
+  end
+
   def dump_and_load(obj, trace=false)
     json = Oj.dump(obj)
     puts json if trace

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: oj
 version: !ruby/object:Gem::Version
-  version: 3.13.16
+  version: 3.13.17
 platform: ruby
 authors:
 - Peter Ohler
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-07-06 00:00:00.000000000 Z
+date: 2022-07-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake-compiler