json 2.12.0 → 2.13.2

data/ext/json/ext/parser/parser.c

@@ -20,6 +20,8 @@ typedef unsigned char _Bool;
 #endif
 #endif
 
+#include "../simd/simd.h"
+
 #ifndef RB_UNLIKELY
 #define RB_UNLIKELY(expr) expr
 #endif
@@ -35,7 +37,7 @@ static ID i_chr, i_aset, i_aref,
           i_leftshift, i_new, i_try_convert, i_uminus, i_encode;
 
 static VALUE sym_max_nesting, sym_allow_nan, sym_allow_trailing_comma, sym_symbolize_names, sym_freeze,
-             sym_decimal_class, sym_on_load;
+             sym_decimal_class, sym_on_load, sym_allow_duplicate_key;
 
 static int binary_encindex;
 static int utf8_encindex;
@@ -363,10 +365,17 @@ static int convert_UTF32_to_UTF8(char *buf, uint32_t ch)
     return len;
 }
 
+enum duplicate_key_action {
+    JSON_DEPRECATED = 0,
+    JSON_IGNORE,
+    JSON_RAISE,
+};
+
 typedef struct JSON_ParserStruct {
     VALUE on_load_proc;
     VALUE decimal_class;
     ID decimal_method_id;
+    enum duplicate_key_action on_duplicate_key;
     int max_nesting;
     bool allow_nan;
     bool allow_trailing_comma;
@@ -386,15 +395,8 @@ typedef struct JSON_ParserStateStruct {
     int current_nesting;
 } JSON_ParserState;
 
-
-#define PARSE_ERROR_FRAGMENT_LEN 32
-#ifdef RBIMPL_ATTR_NORETURN
-RBIMPL_ATTR_NORETURN()
-#endif
-static void raise_parse_error(const char *format, JSON_ParserState *state)
+static void cursor_position(JSON_ParserState *state, long *line_out, long *column_out)
 {
-    unsigned char buffer[PARSE_ERROR_FRAGMENT_LEN + 1];
-
     const char *cursor = state->cursor;
     long column = 0;
     long line = 1;
@@ -411,23 +413,58 @@ static void raise_parse_error(const char *format, JSON_ParserState *state)
             line++;
         }
     }
+    *line_out = line;
+    *column_out = column;
+}
 
-    const char *ptr = state->cursor;
-    size_t len = ptr ? strnlen(ptr, PARSE_ERROR_FRAGMENT_LEN) : 0;
+static void emit_parse_warning(const char *message, JSON_ParserState *state)
+{
+    long line, column;
+    cursor_position(state, &line, &column);
 
-    if (len == PARSE_ERROR_FRAGMENT_LEN) {
-        MEMCPY(buffer, ptr, char, PARSE_ERROR_FRAGMENT_LEN);
+    VALUE warning = rb_sprintf("%s at line %ld column %ld", message, line, column);
+    rb_funcall(mJSON, rb_intern("deprecation_warning"), 1, warning);
+}
 
-        while (buffer[len - 1] >= 0x80 && buffer[len - 1] < 0xC0) { // Is continuation byte
-            len--;
-        }
+#define PARSE_ERROR_FRAGMENT_LEN 32
 
-        if (buffer[len - 1] >= 0xC0) { // multibyte character start
-            len--;
+#ifdef RBIMPL_ATTR_NORETURN
+RBIMPL_ATTR_NORETURN()
+#endif
+static void raise_parse_error(const char *format, JSON_ParserState *state)
+{
+    unsigned char buffer[PARSE_ERROR_FRAGMENT_LEN + 3];
+    long line, column;
+    cursor_position(state, &line, &column);
+
+    const char *ptr = "EOF";
+    if (state->cursor && state->cursor < state->end) {
+        ptr = state->cursor;
+        size_t len = 0;
+        while (len < PARSE_ERROR_FRAGMENT_LEN) {
+            char ch = ptr[len];
+            if (!ch || ch == '\n' || ch == ' ' || ch == '\t' || ch == '\r') {
+                break;
+            }
+            len++;
         }
 
-        buffer[len] = '\0';
-        ptr = (const char *)buffer;
+        if (len) {
+            buffer[0] = '\'';
+            MEMCPY(buffer + 1, ptr, char, len);
+
+            while (buffer[len] >= 0x80 && buffer[len] < 0xC0) { // Is continuation byte
+                len--;
+            }
+
+            if (buffer[len] >= 0xC0) { // multibyte character start
+                len--;
+            }
+
+            buffer[len + 1] = '\'';
+            buffer[len + 2] = '\0';
+            ptr = (const char *)buffer;
+        }
     }
 
     VALUE msg = rb_sprintf(format, ptr);
@@ -473,16 +510,16 @@ static uint32_t unescape_unicode(JSON_ParserState *state, const unsigned char *p
     signed char b;
     uint32_t result = 0;
     b = digit_values[p[0]];
-    if (b < 0) raise_parse_error_at("incomplete unicode character escape sequence at '%s'", state, (char *)p - 2);
+    if (b < 0) raise_parse_error_at("incomplete unicode character escape sequence at %s", state, (char *)p - 2);
     result = (result << 4) | (unsigned char)b;
     b = digit_values[p[1]];
-    if (b < 0) raise_parse_error_at("incomplete unicode character escape sequence at '%s'", state, (char *)p - 2);
+    if (b < 0) raise_parse_error_at("incomplete unicode character escape sequence at %s", state, (char *)p - 2);
     result = (result << 4) | (unsigned char)b;
     b = digit_values[p[2]];
-    if (b < 0) raise_parse_error_at("incomplete unicode character escape sequence at '%s'", state, (char *)p - 2);
+    if (b < 0) raise_parse_error_at("incomplete unicode character escape sequence at %s", state, (char *)p - 2);
     result = (result << 4) | (unsigned char)b;
     b = digit_values[p[3]];
-    if (b < 0) raise_parse_error_at("incomplete unicode character escape sequence at '%s'", state, (char *)p - 2);
+    if (b < 0) raise_parse_error_at("incomplete unicode character escape sequence at %s", state, (char *)p - 2);
     result = (result << 4) | (unsigned char)b;
     return result;
 }
@@ -505,7 +542,7 @@ static void
 json_eat_comments(JSON_ParserState *state)
 {
     if (state->cursor + 1 < state->end) {
-        switch(state->cursor[1]) {
+        switch (state->cursor[1]) {
             case '/': {
                 state->cursor = memchr(state->cursor, '\n', state->end - state->cursor);
                 if (!state->cursor) {
@@ -532,11 +569,11 @@ json_eat_comments(JSON_ParserState *state)
                 break;
             }
             default:
-                raise_parse_error("unexpected token '%s'", state);
+                raise_parse_error("unexpected token %s", state);
                 break;
         }
     } else {
-        raise_parse_error("unexpected token '%s'", state);
+        raise_parse_error("unexpected token %s", state);
     }
 }
 
@@ -655,7 +692,7 @@ static VALUE json_string_unescape(JSON_ParserState *state, const char *string, c
                 break;
             case 'u':
                 if (pe > stringEnd - 5) {
-                    raise_parse_error_at("incomplete unicode character escape sequence at '%s'", state, p);
+                    raise_parse_error_at("incomplete unicode character escape sequence at %s", state, p);
                 } else {
                     uint32_t ch = unescape_unicode(state, (unsigned char *) ++pe);
                     pe += 3;
@@ -672,7 +709,7 @@ static VALUE json_string_unescape(JSON_ParserState *state, const char *string, c
                     if ((ch & 0xFC00) == 0xD800) {
                         pe++;
                         if (pe > stringEnd - 6) {
-                            raise_parse_error_at("incomplete surrogate pair at '%s'", state, p);
+                            raise_parse_error_at("incomplete surrogate pair at %s", state, p);
                         }
                         if (pe[0] == '\\' && pe[1] == 'u') {
                             uint32_t sur = unescape_unicode(state, (unsigned char *) pe + 2);
@@ -795,10 +832,67 @@ static inline VALUE json_decode_array(JSON_ParserState *state, JSON_ParserConfig
     return array;
 }
 
-static inline VALUE json_decode_object(JSON_ParserState *state, JSON_ParserConfig *config, long count)
+static VALUE json_find_duplicated_key(size_t count, const VALUE *pairs)
+{
+    VALUE set = rb_hash_new_capa(count / 2);
+    for (size_t index = 0; index < count; index += 2) {
+        size_t before = RHASH_SIZE(set);
+        VALUE key = pairs[index];
+        rb_hash_aset(set, key, Qtrue);
+        if (RHASH_SIZE(set) == before) {
+            if (RB_SYMBOL_P(key)) {
+                return rb_sym2str(key);
+            }
+            return key;
+        }
+    }
+    return Qfalse;
+}
+
+static void emit_duplicate_key_warning(JSON_ParserState *state, VALUE duplicate_key)
+{
+    VALUE message = rb_sprintf(
+        "detected duplicate key %"PRIsVALUE" in JSON object. This will raise an error in json 3.0 unless enabled via `allow_duplicate_key: true`",
+        rb_inspect(duplicate_key)
+    );
+
+    emit_parse_warning(RSTRING_PTR(message), state);
+    RB_GC_GUARD(message);
+}
+
+#ifdef RBIMPL_ATTR_NORETURN
+RBIMPL_ATTR_NORETURN()
+#endif
+static void raise_duplicate_key_error(JSON_ParserState *state, VALUE duplicate_key)
+{
+    VALUE message = rb_sprintf(
+        "duplicate key %"PRIsVALUE,
+        rb_inspect(duplicate_key)
+    );
+
+    raise_parse_error(RSTRING_PTR(message), state);
+    RB_GC_GUARD(message);
+}
+
+static inline VALUE json_decode_object(JSON_ParserState *state, JSON_ParserConfig *config, size_t count)
 {
-    VALUE object = rb_hash_new_capa(count);
-    rb_hash_bulk_insert(count, rvalue_stack_peek(state->stack, count), object);
+    size_t entries_count = count / 2;
+    VALUE object = rb_hash_new_capa(entries_count);
+    const VALUE *pairs = rvalue_stack_peek(state->stack, count);
+    rb_hash_bulk_insert(count, pairs, object);
+
+    if (RB_UNLIKELY(RHASH_SIZE(object) < entries_count)) {
+        switch (config->on_duplicate_key) {
+            case JSON_IGNORE:
+                break;
+            case JSON_DEPRECATED:
+                emit_duplicate_key_warning(state, json_find_duplicated_key(count, pairs));
+                break;
+            case JSON_RAISE:
+                raise_duplicate_key_error(state, json_find_duplicated_key(count, pairs));
+                break;
+        }
+    }
 
     rvalue_stack_pop(state->stack, count);
 
@@ -832,7 +926,7 @@ static inline VALUE json_push_value(JSON_ParserState *state, JSON_ParserConfig *
     return value;
 }
 
-static const bool string_scan[256] = {
+static const bool string_scan_table[256] = {
     // ASCII Control Characters
      1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
      1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
@@ -845,32 +939,71 @@ static const bool string_scan[256] = {
      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 };
 
+#if (defined(__GNUC__ ) || defined(__clang__))
+#define FORCE_INLINE __attribute__((always_inline))
+#else
+#define FORCE_INLINE
+#endif
+
+#ifdef HAVE_SIMD
+static SIMD_Implementation simd_impl = SIMD_NONE;
+#endif /* HAVE_SIMD */
+
+static inline bool FORCE_INLINE string_scan(JSON_ParserState *state)
+{
+#ifdef HAVE_SIMD
+#if defined(HAVE_SIMD_NEON)
+
+    uint64_t mask = 0;
+    if (string_scan_simd_neon(&state->cursor, state->end, &mask)) {
+        state->cursor += trailing_zeros64(mask) >> 2;
+        return 1;
+    }
+
+#elif defined(HAVE_SIMD_SSE2)
+    if (simd_impl == SIMD_SSE2) {
+        int mask = 0;
+        if (string_scan_simd_sse2(&state->cursor, state->end, &mask)) {
+            state->cursor += trailing_zeros(mask);
+            return 1;
+        }
+    }
+#endif /* HAVE_SIMD_NEON or HAVE_SIMD_SSE2 */
+#endif /* HAVE_SIMD */
+
+    while (state->cursor < state->end) {
+        if (RB_UNLIKELY(string_scan_table[(unsigned char)*state->cursor])) {
+            return 1;
+        }
+        *state->cursor++;
+    }
+    return 0;
+}
+
 static inline VALUE json_parse_string(JSON_ParserState *state, JSON_ParserConfig *config, bool is_name)
 {
     state->cursor++;
     const char *start = state->cursor;
     bool escaped = false;
 
-    while (state->cursor < state->end) {
-        if (RB_UNLIKELY(string_scan[(unsigned char)*state->cursor])) {
-            switch (*state->cursor) {
-                case '"': {
-                    VALUE string = json_decode_string(state, config, start, state->cursor, escaped, is_name);
-                    state->cursor++;
-                    return json_push_value(state, config, string);
-                }
-                case '\\': {
-                    state->cursor++;
-                    escaped = true;
-                    if ((unsigned char)*state->cursor < 0x20) {
-                        raise_parse_error("invalid ASCII control character in string: %s", state);
-                    }
-                    break;
-                }
-                default:
+    while (RB_UNLIKELY(string_scan(state))) {
+        switch (*state->cursor) {
+            case '"': {
+                VALUE string = json_decode_string(state, config, start, state->cursor, escaped, is_name);
+                state->cursor++;
+                return json_push_value(state, config, string);
+            }
+            case '\\': {
+                state->cursor++;
+                escaped = true;
+                if ((unsigned char)*state->cursor < 0x20) {
                     raise_parse_error("invalid ASCII control character in string: %s", state);
-                    break;
+                }
+                break;
             }
+            default:
+                raise_parse_error("invalid ASCII control character in string: %s", state);
+                break;
         }
 
         state->cursor++;
@@ -894,7 +1027,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config)
                 return json_push_value(state, config, Qnil);
             }
 
-            raise_parse_error("unexpected token '%s'", state);
+            raise_parse_error("unexpected token %s", state);
             break;
         case 't':
             if ((state->end - state->cursor >= 4) && (memcmp(state->cursor, "true", 4) == 0)) {
@@ -902,7 +1035,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config)
                 return json_push_value(state, config, Qtrue);
             }
 
-            raise_parse_error("unexpected token '%s'", state);
+            raise_parse_error("unexpected token %s", state);
             break;
         case 'f':
             // Note: memcmp with a small power of two compile to an integer comparison
@@ -911,7 +1044,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config)
                 return json_push_value(state, config, Qfalse);
             }
 
-            raise_parse_error("unexpected token '%s'", state);
+            raise_parse_error("unexpected token %s", state);
             break;
         case 'N':
             // Note: memcmp with a small power of two compile to an integer comparison
@@ -920,7 +1053,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config)
                 return json_push_value(state, config, CNaN);
             }
 
-            raise_parse_error("unexpected token '%s'", state);
+            raise_parse_error("unexpected token %s", state);
             break;
         case 'I':
             if (config->allow_nan && (state->end - state->cursor >= 8) && (memcmp(state->cursor, "Infinity", 8) == 0)) {
@@ -928,7 +1061,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config)
                 return json_push_value(state, config, CInfinity);
             }
 
-            raise_parse_error("unexpected token '%s'", state);
+            raise_parse_error("unexpected token %s", state);
             break;
         case '-':
             // Note: memcmp with a small power of two compile to an integer comparison
@@ -937,7 +1070,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config)
                     state->cursor += 9;
                     return json_push_value(state, config, CMinusInfinity);
                 } else {
-                    raise_parse_error("unexpected token '%s'", state);
+                    raise_parse_error("unexpected token %s", state);
                 }
             }
             // Fallthrough
@@ -1048,6 +1181,8 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config)
             break;
         }
         case '{': {
+            const char *object_start_cursor = state->cursor;
+
             state->cursor++;
             json_eat_whitespace(state);
             long stack_head = state->stack->head;
@@ -1062,7 +1197,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config)
                 }
 
                 if (*state->cursor != '"') {
-                    raise_parse_error("expected object key, got '%s", state);
+                    raise_parse_error("expected object key, got %s", state);
                 }
                 json_parse_string(state, config, true);
 
@@ -1082,8 +1217,15 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config)
                     if (*state->cursor == '}') {
                         state->cursor++;
                         state->current_nesting--;
-                        long count = state->stack->head - stack_head;
-                        return json_push_value(state, config, json_decode_object(state, config, count));
+                        size_t count = state->stack->head - stack_head;
+
+                        // Temporary rewind cursor in case an error is raised
+                        const char *final_cursor = state->cursor;
+                        state->cursor = object_start_cursor;
+                        VALUE object = json_decode_object(state, config, count);
+                        state->cursor = final_cursor;
+
+                        return json_push_value(state, config, object);
                     }
 
                     if (*state->cursor == ',') {
@@ -1097,13 +1239,13 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config)
                         }
 
                         if (*state->cursor != '"') {
-                            raise_parse_error("expected object key, got: '%s'", state);
+                            raise_parse_error("expected object key, got: %s", state);
                         }
                         json_parse_string(state, config, true);
 
                         json_eat_whitespace(state);
                         if ((state->cursor >= state->end) || (*state->cursor != ':')) {
-                            raise_parse_error("expected ':' after object key, got: '%s", state);
+                            raise_parse_error("expected ':' after object key, got: %s", state);
                         }
                         state->cursor++;
 
@@ -1113,24 +1255,24 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config)
                     }
                 }
 
-                raise_parse_error("expected ',' or '}' after object value, got: '%s'", state);
+                raise_parse_error("expected ',' or '}' after object value, got: %s", state);
             }
             break;
         }
 
         default:
-            raise_parse_error("unexpected character: '%s'", state);
+            raise_parse_error("unexpected character: %s", state);
             break;
     }
 
-    raise_parse_error("unreacheable: '%s'", state);
+    raise_parse_error("unreacheable: %s", state);
 }
 
 static void json_ensure_eof(JSON_ParserState *state)
 {
     json_eat_whitespace(state);
     if (state->cursor != state->end) {
-        raise_parse_error("unexpected token at end of stream '%s'", state);
+        raise_parse_error("unexpected token at end of stream %s", state);
     }
 }
 
@@ -1172,6 +1314,7 @@ static int parser_config_init_i(VALUE key, VALUE val, VALUE data)
     else if (key == sym_symbolize_names)      { config->symbolize_names = RTEST(val); }
     else if (key == sym_freeze)               { config->freeze = RTEST(val); }
     else if (key == sym_on_load)              { config->on_load_proc = RTEST(val) ? val : Qfalse; }
+    else if (key == sym_allow_duplicate_key) { config->on_duplicate_key = RTEST(val) ? JSON_IGNORE : JSON_RAISE; }
     else if (key == sym_decimal_class)        {
         if (RTEST(val)) {
             if (rb_respond_to(val, i_try_convert)) {
@@ -1388,6 +1531,7 @@ void Init_parser(void)
     sym_freeze = ID2SYM(rb_intern("freeze"));
     sym_on_load = ID2SYM(rb_intern("on_load"));
     sym_decimal_class = ID2SYM(rb_intern("decimal_class"));
+    sym_allow_duplicate_key = ID2SYM(rb_intern("allow_duplicate_key"));
 
     i_chr = rb_intern("chr");
     i_aset = rb_intern("[]=");
@@ -1401,4 +1545,8 @@ void Init_parser(void)
     binary_encindex = rb_ascii8bit_encindex();
     utf8_encindex = rb_utf8_encindex();
     enc_utf8 = rb_utf8_encoding();
+
+#ifdef HAVE_SIMD
+    simd_impl = find_simd_implementation();
+#endif
 }

data/ext/json/ext/simd/conf.rb

@@ -0,0 +1,24 @@
+case RbConfig::CONFIG['host_cpu']
+when /^(arm|aarch64)/
+  # Try to compile a small program using NEON instructions
+  header, type, init, extra = 'arm_neon.h', 'uint8x16_t', 'vdupq_n_u8(32)', nil
+when /^(x86_64|x64)/
+  header, type, init, extra = 'x86intrin.h', '__m128i', '_mm_set1_epi8(32)', 'if (__builtin_cpu_supports("sse2")) { printf("OK"); }'
+end
+if header
+  if have_header(header) && try_compile(<<~SRC, '-Werror=implicit-function-declaration')
+      #{cpp_include(header)}
+      int main(int argc, char **argv) {
+        #{type} test = #{init};
+        #{extra}
+        if (argc > 100000) printf("%p", &test);
+        return 0;
+      }
+    SRC
+    $defs.push("-DJSON_ENABLE_SIMD")
+  else
+    puts "Disable SIMD"
+  end
+end
+
+have_header('cpuid.h')