json 2.18.0 → 2.19.3

data/ext/json/ext/generator/generator.c

@@ -63,6 +63,8 @@ struct generate_json_data {
     long depth;
 };
 
+static SIMD_Implementation simd_impl;
+
 static VALUE cState_from_state_s(VALUE self, VALUE opts);
 static VALUE cState_partial_generate(VALUE self, VALUE obj, generator_func, VALUE io);
 static void generate_json(FBuffer *buffer, struct generate_json_data *data, VALUE obj);
@@ -72,9 +74,6 @@ static void generate_json_string(FBuffer *buffer, struct generate_json_data *dat
 static void generate_json_null(FBuffer *buffer, struct generate_json_data *data, VALUE obj);
 static void generate_json_false(FBuffer *buffer, struct generate_json_data *data, VALUE obj);
 static void generate_json_true(FBuffer *buffer, struct generate_json_data *data, VALUE obj);
-#ifdef RUBY_INTEGER_UNIFICATION
-static void generate_json_integer(FBuffer *buffer, struct generate_json_data *data, VALUE obj);
-#endif
 static void generate_json_fixnum(FBuffer *buffer, struct generate_json_data *data, VALUE obj);
 static void generate_json_bignum(FBuffer *buffer, struct generate_json_data *data, VALUE obj);
 static void generate_json_float(FBuffer *buffer, struct generate_json_data *data, VALUE obj);
@@ -155,8 +154,6 @@ static const unsigned char escape_table_basic[256] = {
      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 };
 
-static unsigned char (*search_escape_basic_impl)(search_state *);
-
 static inline unsigned char search_escape_basic(search_state *search)
 {
     while (search->ptr < search->end) {
@@ -212,11 +209,39 @@ ALWAYS_INLINE(static) void escape_UTF8_char_basic(search_state *search)
  * Everything else (should be UTF-8) is just passed through and
  * appended to the result.
  */
+
+
+#if defined(HAVE_SIMD_NEON)
+static inline unsigned char search_escape_basic_neon(search_state *search);
+#elif defined(HAVE_SIMD_SSE2)
+static inline unsigned char search_escape_basic_sse2(search_state *search);
+#endif
+
+static inline unsigned char search_escape_basic(search_state *search);
+
 static inline void convert_UTF8_to_JSON(search_state *search)
 {
-    while (search_escape_basic_impl(search)) {
+#ifdef HAVE_SIMD
+#if defined(HAVE_SIMD_NEON)
+    while (search_escape_basic_neon(search)) {
         escape_UTF8_char_basic(search);
     }
+#elif defined(HAVE_SIMD_SSE2)
+    if (simd_impl == SIMD_SSE2) {
+        while (search_escape_basic_sse2(search)) {
+            escape_UTF8_char_basic(search);
+        }
+        return;
+    }
+    while (search_escape_basic(search)) {
+        escape_UTF8_char_basic(search);
+    }
+#endif
+#else
+    while (search_escape_basic(search)) {
+        escape_UTF8_char_basic(search);
+    }
+#endif /* HAVE_SIMD */
 }
 
 static inline void escape_UTF8_char(search_state *search, unsigned char ch_len)
@@ -260,6 +285,8 @@ static inline void escape_UTF8_char(search_state *search, unsigned char ch_len)
 
 ALWAYS_INLINE(static) char *copy_remaining_bytes(search_state *search, unsigned long vec_len, unsigned long len)
 {
+    RBIMPL_ASSERT_OR_ASSUME(len < vec_len);
+
     // Flush the buffer so everything up until the last 'len' characters are unflushed.
     search_flush(search);
 
@@ -269,12 +296,18 @@ ALWAYS_INLINE(static) char *copy_remaining_bytes(search_state *search, unsigned
     char *s = (buf->ptr + buf->len);
 
     // Pad the buffer with dummy characters that won't need escaping.
-    // This seem wateful at first sight, but memset of vector length is very fast.
-    memset(s, 'X', vec_len);
+    // This seem wasteful at first sight, but memset of vector length is very fast.
+    // This is a space as it can be directly represented as an immediate on AArch64.
+    memset(s, ' ', vec_len);
 
     // Optimistically copy the remaining 'len' characters to the output FBuffer. If there are no characters
     // to escape, then everything ends up in the correct spot. Otherwise it was convenient temporary storage.
-    MEMCPY(s, search->ptr, char, len);
+    if (vec_len == 16) {
+        RBIMPL_ASSERT_OR_ASSUME(len >= SIMD_MINIMUM_THRESHOLD);
+        json_fast_memcpy16(s, search->ptr, len);
+    } else {
+        MEMCPY(s, search->ptr, char, len);
+    }
 
     return s;
 }
@@ -667,233 +700,6 @@ static void convert_UTF8_to_ASCII_only_JSON(search_state *search, const unsigned
     }
 }
 
-/*
- * Document-module: JSON::Ext::Generator
- *
- * This is the JSON generator implemented as a C extension. It can be
- * configured to be used by setting
- *
- *  JSON.generator = JSON::Ext::Generator
- *
- * with the method generator= in JSON.
- *
- */
-
-/* Explanation of the following: that's the only way to not pollute
- * standard library's docs with GeneratorMethods::<ClassName> which
- * are uninformative and take a large place in a list of classes
- */
-
-/*
- * Document-module: JSON::Ext::Generator::GeneratorMethods
- * :nodoc:
- */
-
-/*
- * Document-module: JSON::Ext::Generator::GeneratorMethods::Array
- * :nodoc:
- */
-
-/*
- * Document-module: JSON::Ext::Generator::GeneratorMethods::Bignum
- * :nodoc:
- */
-
-/*
- * Document-module: JSON::Ext::Generator::GeneratorMethods::FalseClass
- * :nodoc:
- */
-
-/*
- * Document-module: JSON::Ext::Generator::GeneratorMethods::Fixnum
- * :nodoc:
- */
-
-/*
- * Document-module: JSON::Ext::Generator::GeneratorMethods::Float
- * :nodoc:
- */
-
-/*
- * Document-module: JSON::Ext::Generator::GeneratorMethods::Hash
- * :nodoc:
- */
-
-/*
- * Document-module: JSON::Ext::Generator::GeneratorMethods::Integer
- * :nodoc:
- */
-
-/*
- * Document-module: JSON::Ext::Generator::GeneratorMethods::NilClass
- * :nodoc:
- */
-
-/*
- * Document-module: JSON::Ext::Generator::GeneratorMethods::Object
- * :nodoc:
- */
-
-/*
- * Document-module: JSON::Ext::Generator::GeneratorMethods::String
- * :nodoc:
- */
-
-/*
- * Document-module: JSON::Ext::Generator::GeneratorMethods::String::Extend
- * :nodoc:
- */
-
-/*
- * Document-module: JSON::Ext::Generator::GeneratorMethods::TrueClass
- * :nodoc:
- */
-
-/*
- * call-seq: to_json(state = nil)
- *
- * Returns a JSON string containing a JSON object, that is generated from
- * this Hash instance.
- * _state_ is a JSON::State object, that can also be used to configure the
- * produced JSON string output further.
- */
-static VALUE mHash_to_json(int argc, VALUE *argv, VALUE self)
-{
-    rb_check_arity(argc, 0, 1);
-    VALUE Vstate = cState_from_state_s(cState, argc == 1 ? argv[0] : Qnil);
-    return cState_partial_generate(Vstate, self, generate_json_object, Qfalse);
-}
-
-/*
- * call-seq: to_json(state = nil)
- *
- * Returns a JSON string containing a JSON array, that is generated from
- * this Array instance.
- * _state_ is a JSON::State object, that can also be used to configure the
- * produced JSON string output further.
- */
-static VALUE mArray_to_json(int argc, VALUE *argv, VALUE self)
-{
-    rb_check_arity(argc, 0, 1);
-    VALUE Vstate = cState_from_state_s(cState, argc == 1 ? argv[0] : Qnil);
-    return cState_partial_generate(Vstate, self, generate_json_array, Qfalse);
-}
-
-#ifdef RUBY_INTEGER_UNIFICATION
-/*
- * call-seq: to_json(*)
- *
- * Returns a JSON string representation for this Integer number.
- */
-static VALUE mInteger_to_json(int argc, VALUE *argv, VALUE self)
-{
-    rb_check_arity(argc, 0, 1);
-    VALUE Vstate = cState_from_state_s(cState, argc == 1 ? argv[0] : Qnil);
-    return cState_partial_generate(Vstate, self, generate_json_integer, Qfalse);
-}
-
-#else
-/*
- * call-seq: to_json(*)
- *
- * Returns a JSON string representation for this Integer number.
- */
-static VALUE mFixnum_to_json(int argc, VALUE *argv, VALUE self)
-{
-    rb_check_arity(argc, 0, 1);
-    VALUE Vstate = cState_from_state_s(cState, argc == 1 ? argv[0] : Qnil);
-    return cState_partial_generate(Vstate, self, generate_json_fixnum, Qfalse);
-}
-
-/*
- * call-seq: to_json(*)
- *
- * Returns a JSON string representation for this Integer number.
- */
-static VALUE mBignum_to_json(int argc, VALUE *argv, VALUE self)
-{
-    rb_check_arity(argc, 0, 1);
-    VALUE Vstate = cState_from_state_s(cState, argc == 1 ? argv[0] : Qnil);
-    return cState_partial_generate(Vstate, self, generate_json_bignum, Qfalse);
-}
-#endif
-
-/*
- * call-seq: to_json(*)
- *
- * Returns a JSON string representation for this Float number.
- */
-static VALUE mFloat_to_json(int argc, VALUE *argv, VALUE self)
-{
-    rb_check_arity(argc, 0, 1);
-    VALUE Vstate = cState_from_state_s(cState, argc == 1 ? argv[0] : Qnil);
-    return cState_partial_generate(Vstate, self, generate_json_float, Qfalse);
-}
-
-/*
- * call-seq: to_json(*)
- *
- * This string should be encoded with UTF-8 A call to this method
- * returns a JSON string encoded with UTF16 big endian characters as
- * \u????.
- */
-static VALUE mString_to_json(int argc, VALUE *argv, VALUE self)
-{
-    rb_check_arity(argc, 0, 1);
-    VALUE Vstate = cState_from_state_s(cState, argc == 1 ? argv[0] : Qnil);
-    return cState_partial_generate(Vstate, self, generate_json_string, Qfalse);
-}
-
-/*
- * call-seq: to_json(*)
- *
- * Returns a JSON string for true: 'true'.
- */
-static VALUE mTrueClass_to_json(int argc, VALUE *argv, VALUE self)
-{
-    rb_check_arity(argc, 0, 1);
-    return rb_utf8_str_new("true", 4);
-}
-
-/*
- * call-seq: to_json(*)
- *
- * Returns a JSON string for false: 'false'.
- */
-static VALUE mFalseClass_to_json(int argc, VALUE *argv, VALUE self)
-{
-    rb_check_arity(argc, 0, 1);
-    return rb_utf8_str_new("false", 5);
-}
-
-/*
- * call-seq: to_json(*)
- *
- * Returns a JSON string for nil: 'null'.
- */
-static VALUE mNilClass_to_json(int argc, VALUE *argv, VALUE self)
-{
-    rb_check_arity(argc, 0, 1);
-    return rb_utf8_str_new("null", 4);
-}
-
-/*
- * call-seq: to_json(*)
- *
- * Converts this object to a string (calling #to_s), converts
- * it to a JSON string, and returns the result. This is a fallback, if no
- * special method #to_json was defined for some object.
- */
-static VALUE mObject_to_json(int argc, VALUE *argv, VALUE self)
-{
-    VALUE state;
-    VALUE string = rb_funcall(self, i_to_s, 0);
-    rb_scan_args(argc, argv, "01", &state);
-    Check_Type(string, T_STRING);
-    state = cState_from_state_s(cState, state);
-    return cState_partial_generate(state, string, generate_json_string, Qfalse);
-}
-
 static void State_mark(void *ptr)
 {
     JSON_Generator_State *state = ptr;
@@ -1091,6 +897,7 @@ static void raw_generate_json_string(FBuffer *buffer, struct generate_json_data
     search.matches_mask = 0;
     search.has_matches = false;
     search.chunk_base = NULL;
+    search.chunk_end = NULL;
 #endif /* HAVE_SIMD */
 
     switch (rb_enc_str_coderange(obj)) {
@@ -1337,18 +1144,8 @@ static void generate_json_fixnum(FBuffer *buffer, struct generate_json_data *dat
 static void generate_json_bignum(FBuffer *buffer, struct generate_json_data *data, VALUE obj)
 {
     VALUE tmp = rb_funcall(obj, i_to_s, 0);
-    fbuffer_append_str(buffer, tmp);
-}
-
-#ifdef RUBY_INTEGER_UNIFICATION
-static void generate_json_integer(FBuffer *buffer, struct generate_json_data *data, VALUE obj)
-{
-    if (FIXNUM_P(obj))
-        generate_json_fixnum(buffer, data, obj);
-    else
-        generate_json_bignum(buffer, data, obj);
+    fbuffer_append_str(buffer, StringValue(tmp));
 }
-#endif
 
 static void generate_json_float(FBuffer *buffer, struct generate_json_data *data, VALUE obj)
 {
@@ -1393,7 +1190,7 @@ static void generate_json_fragment(FBuffer *buffer, struct generate_json_data *d
     fbuffer_append_str(buffer, fragment);
 }
 
-static void generate_json(FBuffer *buffer, struct generate_json_data *data, VALUE obj)
+static inline void generate_json_general(FBuffer *buffer, struct generate_json_data *data, VALUE obj, bool fallback)
 {
     bool as_json_called = false;
 start:
@@ -1420,15 +1217,15 @@ start:
                 generate_json_bignum(buffer, data, obj);
                 break;
             case T_HASH:
-                if (klass != rb_cHash) goto general;
+                if (fallback && klass != rb_cHash) goto general;
                 generate_json_object(buffer, data, obj);
                 break;
             case T_ARRAY:
-                if (klass != rb_cArray) goto general;
+                if (fallback && klass != rb_cArray) goto general;
                 generate_json_array(buffer, data, obj);
                 break;
             case T_STRING:
-                if (klass != rb_cString) goto general;
+                if (fallback && klass != rb_cString) goto general;
 
                 if (RB_LIKELY(valid_json_string_p(obj))) {
                     raw_generate_json_string(buffer, data, obj);
@@ -1444,7 +1241,7 @@ start:
                 generate_json_symbol(buffer, data, obj);
                 break;
             case T_FLOAT:
-                if (klass != rb_cFloat) goto general;
+                if (fallback && klass != rb_cFloat) goto general;
                 generate_json_float(buffer, data, obj);
                 break;
             case T_STRUCT:
@@ -1468,6 +1265,16 @@ start:
     }
 }
 
+static void generate_json(FBuffer *buffer, struct generate_json_data *data, VALUE obj)
+{
+    generate_json_general(buffer, data, obj, true);
+}
+
+static void generate_json_no_fallback(FBuffer *buffer, struct generate_json_data *data, VALUE obj)
+{
+    generate_json_general(buffer, data, obj, false);
+}
+
 static VALUE generate_json_try(VALUE d)
 {
     struct generate_json_data *data = (struct generate_json_data *)d;
@@ -1485,7 +1292,7 @@ static VALUE generate_json_ensure(VALUE d)
     return Qundef;
 }
 
-static VALUE cState_partial_generate(VALUE self, VALUE obj, generator_func func, VALUE io)
+static inline VALUE cState_partial_generate(VALUE self, VALUE obj, generator_func func, VALUE io)
 {
     GET_STATE(self);
 
@@ -1522,6 +1329,15 @@ static VALUE cState_generate(int argc, VALUE *argv, VALUE self)
     return cState_partial_generate(self, obj, generate_json, io);
 }
 
+/* :nodoc: */
+static VALUE cState_generate_no_fallback(int argc, VALUE *argv, VALUE self)
+{
+    rb_check_arity(argc, 1, 2);
+    VALUE obj = argv[0];
+    VALUE io = argc > 1 ? argv[1] : Qnil;
+    return cState_partial_generate(self, obj, generate_json_no_fallback, io);
+}
+
 static VALUE cState_initialize(int argc, VALUE *argv, VALUE self)
 {
     rb_warn("The json gem extension was loaded with the stdlib ruby code. You should upgrade rubygems with `gem update --system`");
@@ -1759,6 +1575,17 @@ static long long_config(VALUE num)
     return RTEST(num) ? FIX2LONG(num) : 0;
 }
 
+// depth must never be negative; reject early with a clear error.
+static long depth_config(VALUE num)
+{
+    if (!RTEST(num)) return 0;
+    long d = NUM2LONG(num);
+    if (RB_UNLIKELY(d < 0)) {
+        rb_raise(rb_eArgError, "depth must be >= 0 (got %ld)", d);
+    }
+    return d;
+}
+
 /*
  * call-seq: max_nesting=(depth)
  *
@@ -1915,7 +1742,7 @@ static VALUE cState_depth_set(VALUE self, VALUE depth)
 {
     rb_check_frozen(self);
     GET_STATE(self);
-    state->depth = long_config(depth);
+    state->depth = depth_config(depth);
     return Qnil;
 }
 
@@ -1980,7 +1807,7 @@ static int configure_state_i(VALUE key, VALUE val, VALUE _arg)
     else if (key == sym_max_nesting)           { state->max_nesting = long_config(val); }
     else if (key == sym_allow_nan)             { state->allow_nan = RTEST(val); }
     else if (key == sym_ascii_only)            { state->ascii_only = RTEST(val); }
-    else if (key == sym_depth)                 { state->depth = long_config(val); }
+    else if (key == sym_depth)                 { state->depth = depth_config(val); }
     else if (key == sym_buffer_initial_length) { buffer_initial_length_set(state, val); }
     else if (key == sym_script_safe)           { state->script_safe = RTEST(val); }
     else if (key == sym_escape_slash)          { state->script_safe = RTEST(val); }
@@ -2020,7 +1847,7 @@ static VALUE cState_configure(VALUE self, VALUE opts)
     return self;
 }
 
-static VALUE cState_m_generate(VALUE klass, VALUE obj, VALUE opts, VALUE io)
+static VALUE cState_m_do_generate(VALUE klass, VALUE obj, VALUE opts, VALUE io, generator_func func)
 {
     JSON_Generator_State state = {0};
     state_init(&state);
@@ -2038,14 +1865,21 @@ static VALUE cState_m_generate(VALUE klass, VALUE obj, VALUE opts, VALUE io)
         .state = &state,
         .depth = state.depth,
         .obj = obj,
-        .func = generate_json,
+        .func = func,
     };
     return rb_ensure(generate_json_try, (VALUE)&data, generate_json_ensure, (VALUE)&data);
 }
 
-/*
- *
- */
+static VALUE cState_m_generate(VALUE klass, VALUE obj, VALUE opts, VALUE io)
+{
+    return cState_m_do_generate(klass, obj, opts, io, generate_json);
+}
+
+static VALUE cState_m_generate_no_fallback(VALUE klass, VALUE obj, VALUE opts, VALUE io)
+{
+    return cState_m_do_generate(klass, obj, opts, io, generate_json_no_fallback);
+}
+
 void Init_generator(void)
 {
 #ifdef HAVE_RB_EXT_RACTOR_SAFE
@@ -2110,46 +1944,12 @@ void Init_generator(void)
     rb_define_method(cState, "buffer_initial_length", cState_buffer_initial_length, 0);
     rb_define_method(cState, "buffer_initial_length=", cState_buffer_initial_length_set, 1);
     rb_define_method(cState, "generate", cState_generate, -1);
+    rb_define_method(cState, "_generate_no_fallback", cState_generate_no_fallback, -1);
 
     rb_define_private_method(cState, "allow_duplicate_key?", cState_allow_duplicate_key_p, 0);
 
     rb_define_singleton_method(cState, "generate", cState_m_generate, 3);
-
-    VALUE mGeneratorMethods = rb_define_module_under(mGenerator, "GeneratorMethods");
-
-    VALUE mObject = rb_define_module_under(mGeneratorMethods, "Object");
-    rb_define_method(mObject, "to_json", mObject_to_json, -1);
-
-    VALUE mHash = rb_define_module_under(mGeneratorMethods, "Hash");
-    rb_define_method(mHash, "to_json", mHash_to_json, -1);
-
-    VALUE mArray = rb_define_module_under(mGeneratorMethods, "Array");
-    rb_define_method(mArray, "to_json", mArray_to_json, -1);
-
-#ifdef RUBY_INTEGER_UNIFICATION
-    VALUE mInteger = rb_define_module_under(mGeneratorMethods, "Integer");
-    rb_define_method(mInteger, "to_json", mInteger_to_json, -1);
-#else
-    VALUE mFixnum = rb_define_module_under(mGeneratorMethods, "Fixnum");
-    rb_define_method(mFixnum, "to_json", mFixnum_to_json, -1);
-
-    VALUE mBignum = rb_define_module_under(mGeneratorMethods, "Bignum");
-    rb_define_method(mBignum, "to_json", mBignum_to_json, -1);
-#endif
-    VALUE mFloat = rb_define_module_under(mGeneratorMethods, "Float");
-    rb_define_method(mFloat, "to_json", mFloat_to_json, -1);
-
-    VALUE mString = rb_define_module_under(mGeneratorMethods, "String");
-    rb_define_method(mString, "to_json", mString_to_json, -1);
-
-    VALUE mTrueClass = rb_define_module_under(mGeneratorMethods, "TrueClass");
-    rb_define_method(mTrueClass, "to_json", mTrueClass_to_json, -1);
-
-    VALUE mFalseClass = rb_define_module_under(mGeneratorMethods, "FalseClass");
-    rb_define_method(mFalseClass, "to_json", mFalseClass_to_json, -1);
-
-    VALUE mNilClass = rb_define_module_under(mGeneratorMethods, "NilClass");
-    rb_define_method(mNilClass, "to_json", mNilClass_to_json, -1);
+    rb_define_singleton_method(cState, "_generate_no_fallback", cState_m_generate_no_fallback, 3);
 
     rb_global_variable(&Encoding_UTF_8);
     Encoding_UTF_8 = rb_const_get(rb_path2class("Encoding"), rb_intern("UTF_8"));
@@ -2181,22 +1981,5 @@ void Init_generator(void)
 
     rb_require("json/ext/generator/state");
 
-
-    switch (find_simd_implementation()) {
-#ifdef HAVE_SIMD
-#ifdef HAVE_SIMD_NEON
-        case SIMD_NEON:
-            search_escape_basic_impl = search_escape_basic_neon;
-            break;
-#endif /* HAVE_SIMD_NEON */
-#ifdef HAVE_SIMD_SSE2
-        case SIMD_SSE2:
-            search_escape_basic_impl = search_escape_basic_sse2;
-            break;
-#endif /* HAVE_SIMD_SSE2 */
-#endif /* HAVE_SIMD */
-        default:
-            search_escape_basic_impl = search_escape_basic;
-            break;
-    }
+    simd_impl = find_simd_implementation();
 }

data/ext/json/ext/json.h

@@ -5,6 +5,10 @@
 #include "ruby/encoding.h"
 #include <stdint.h>
 
+#ifndef RBIMPL_ASSERT_OR_ASSUME
+# define RBIMPL_ASSERT_OR_ASSUME(x)
+#endif
+
 #if defined(RUBY_DEBUG) && RUBY_DEBUG
 # define JSON_ASSERT RUBY_ASSERT
 #else
@@ -51,8 +55,12 @@ typedef unsigned char _Bool;
 #endif
 
 #ifndef NORETURN
+#if defined(__has_attribute) && __has_attribute(noreturn)
+#define NORETURN(x) __attribute__((noreturn)) x
+#else
 #define NORETURN(x) x
 #endif
+#endif
 
 #ifndef NOINLINE
 #if defined(__has_attribute) && __has_attribute(noinline)