json 2.13.2 → 2.18.1

data/ext/json/ext/generator/generator.c

@@ -1,4 +1,4 @@
-#include "ruby.h"
+#include "../json.h"
 #include "../fbuffer/fbuffer.h"
 #include "../vendor/fpconv.c"
 
@@ -9,6 +9,12 @@
 
 /* ruby api and some helpers */
 
+enum duplicate_key_action {
+    JSON_DEPRECATED = 0,
+    JSON_IGNORE,
+    JSON_RAISE,
+};
+
 typedef struct JSON_Generator_StateStruct {
     VALUE indent;
     VALUE space;
@@ -21,20 +27,19 @@ typedef struct JSON_Generator_StateStruct {
     long depth;
     long buffer_initial_length;
 
+    enum duplicate_key_action on_duplicate_key;
+
+    bool as_json_single_arg;
     bool allow_nan;
     bool ascii_only;
     bool script_safe;
     bool strict;
 } JSON_Generator_State;
 
-#ifndef RB_UNLIKELY
-#define RB_UNLIKELY(cond) (cond)
-#endif
-
-static VALUE mJSON, cState, cFragment, mString_Extend, eGeneratorError, eNestingError, Encoding_UTF_8;
+static VALUE mJSON, cState, cFragment, eGeneratorError, eNestingError, Encoding_UTF_8;
 
-static ID i_to_s, i_to_json, i_new, i_pack, i_unpack, i_create_id, i_extend, i_encode;
-static VALUE sym_indent, sym_space, sym_space_before, sym_object_nl, sym_array_nl, sym_max_nesting, sym_allow_nan,
+static ID i_to_s, i_to_json, i_new, i_encode;
+static VALUE sym_indent, sym_space, sym_space_before, sym_object_nl, sym_array_nl, sym_max_nesting, sym_allow_nan, sym_allow_duplicate_key,
              sym_ascii_only, sym_depth, sym_buffer_initial_length, sym_script_safe, sym_escape_slash, sym_strict, sym_as_json;
 
 
@@ -55,8 +60,11 @@ struct generate_json_data {
     JSON_Generator_State *state;
     VALUE obj;
     generator_func func;
+    long depth;
 };
 
+static SIMD_Implementation simd_impl;
+
 static VALUE cState_from_state_s(VALUE self, VALUE opts);
 static VALUE cState_partial_generate(VALUE self, VALUE obj, generator_func, VALUE io);
 static void generate_json(FBuffer *buffer, struct generate_json_data *data, VALUE obj);
@@ -76,23 +84,18 @@ static void generate_json_fragment(FBuffer *buffer, struct generate_json_data *d
 
 static int usascii_encindex, utf8_encindex, binary_encindex;
 
-#ifdef RBIMPL_ATTR_NORETURN
-RBIMPL_ATTR_NORETURN()
-#endif
-static void raise_generator_error_str(VALUE invalid_object, VALUE str)
+NORETURN(static void) raise_generator_error_str(VALUE invalid_object, VALUE str)
 {
+    rb_enc_associate_index(str, utf8_encindex);
     VALUE exc = rb_exc_new_str(eGeneratorError, str);
     rb_ivar_set(exc, rb_intern("@invalid_object"), invalid_object);
     rb_exc_raise(exc);
 }
 
-#ifdef RBIMPL_ATTR_NORETURN
-RBIMPL_ATTR_NORETURN()
-#endif
 #ifdef RBIMPL_ATTR_FORMAT
 RBIMPL_ATTR_FORMAT(RBIMPL_PRINTF_FORMAT, 2, 3)
 #endif
-static void raise_generator_error(VALUE invalid_object, const char *fmt, ...)
+NORETURN(static void) raise_generator_error(VALUE invalid_object, const char *fmt, ...)
 {
     va_list args;
     va_start(args, fmt);
@@ -127,18 +130,12 @@ typedef struct _search_state {
 #endif /* HAVE_SIMD */
 } search_state;
 
-#if (defined(__GNUC__ ) || defined(__clang__))
-#define FORCE_INLINE __attribute__((always_inline))
-#else
-#define FORCE_INLINE
-#endif
-
-static inline FORCE_INLINE void search_flush(search_state *search)
+ALWAYS_INLINE(static) void search_flush(search_state *search)
 {
     // Do not remove this conditional without profiling, specifically escape-heavy text.
     // escape_UTF8_char_basic will advance search->ptr and search->cursor (effectively a search_flush).
-    // For back-to-back characters that need to be escaped, specifcally for the SIMD code paths, this method
-    // will be called just before calling escape_UTF8_char_basic. There will be no characers to append for the
+    // For back-to-back characters that need to be escaped, specifically for the SIMD code paths, this method
+    // will be called just before calling escape_UTF8_char_basic. There will be no characters to append for the
     // consecutive characters that need to be escaped. While the fbuffer_append is a no-op if
     // nothing needs to be flushed, we can save a few memory references with this conditional.
     if (search->ptr > search->cursor) {
@@ -160,8 +157,6 @@ static const unsigned char escape_table_basic[256] = {
      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 };
 
-static unsigned char (*search_escape_basic_impl)(search_state *);
-
 static inline unsigned char search_escape_basic(search_state *search)
 {
     while (search->ptr < search->end) {
@@ -176,7 +171,7 @@ static inline unsigned char search_escape_basic(search_state *search)
     return 0;
 }
 
-static inline FORCE_INLINE void escape_UTF8_char_basic(search_state *search)
+ALWAYS_INLINE(static) void escape_UTF8_char_basic(search_state *search)
 {
     const unsigned char ch = (unsigned char)*search->ptr;
     switch (ch) {
@@ -217,11 +212,39 @@ static inline FORCE_INLINE void escape_UTF8_char_basic(search_state *search)
  * Everything else (should be UTF-8) is just passed through and
  * appended to the result.
  */
+
+
+#if defined(HAVE_SIMD_NEON)
+static inline unsigned char search_escape_basic_neon(search_state *search);
+#elif defined(HAVE_SIMD_SSE2)
+static inline unsigned char search_escape_basic_sse2(search_state *search);
+#endif
+
+static inline unsigned char search_escape_basic(search_state *search);
+
 static inline void convert_UTF8_to_JSON(search_state *search)
 {
-    while (search_escape_basic_impl(search)) {
+#ifdef HAVE_SIMD
+#if defined(HAVE_SIMD_NEON)
+    while (search_escape_basic_neon(search)) {
         escape_UTF8_char_basic(search);
     }
+#elif defined(HAVE_SIMD_SSE2)
+    if (simd_impl == SIMD_SSE2) {
+        while (search_escape_basic_sse2(search)) {
+            escape_UTF8_char_basic(search);
+        }
+        return;
+    }
+    while (search_escape_basic(search)) {
+        escape_UTF8_char_basic(search);
+    }
+#endif
+#else
+    while (search_escape_basic(search)) {
+        escape_UTF8_char_basic(search);
+    }
+#endif /* HAVE_SIMD */
 }
 
 static inline void escape_UTF8_char(search_state *search, unsigned char ch_len)
@@ -263,8 +286,10 @@ static inline void escape_UTF8_char(search_state *search, unsigned char ch_len)
 
 #ifdef HAVE_SIMD
 
-static inline FORCE_INLINE char *copy_remaining_bytes(search_state *search, unsigned long vec_len, unsigned long len)
+ALWAYS_INLINE(static) char *copy_remaining_bytes(search_state *search, unsigned long vec_len, unsigned long len)
 {
+    RBIMPL_ASSERT_OR_ASSUME(len < vec_len);
+
     // Flush the buffer so everything up until the last 'len' characters are unflushed.
     search_flush(search);
 
@@ -274,19 +299,25 @@ static inline FORCE_INLINE char *copy_remaining_bytes(search_state *search, unsi
     char *s = (buf->ptr + buf->len);
 
     // Pad the buffer with dummy characters that won't need escaping.
-    // This seem wateful at first sight, but memset of vector length is very fast.
-    memset(s, 'X', vec_len);
+    // This seem wasteful at first sight, but memset of vector length is very fast.
+    // This is a space as it can be directly represented as an immediate on AArch64.
+    memset(s, ' ', vec_len);
 
     // Optimistically copy the remaining 'len' characters to the output FBuffer. If there are no characters
     // to escape, then everything ends up in the correct spot. Otherwise it was convenient temporary storage.
-    MEMCPY(s, search->ptr, char, len);
+    if (vec_len == 16) {
+        RBIMPL_ASSERT_OR_ASSUME(len >= SIMD_MINIMUM_THRESHOLD);
+        json_fast_memcpy16(s, search->ptr, len);
+    } else {
+        MEMCPY(s, search->ptr, char, len);
+    }
 
     return s;
 }
 
 #ifdef HAVE_SIMD_NEON
 
-static inline FORCE_INLINE unsigned char neon_next_match(search_state *search)
+ALWAYS_INLINE(static) unsigned char neon_next_match(search_state *search)
 {
     uint64_t mask = search->matches_mask;
     uint32_t index = trailing_zeros64(mask) >> 2;
@@ -400,7 +431,7 @@ static inline unsigned char search_escape_basic_neon(search_state *search)
 
 #ifdef HAVE_SIMD_SSE2
 
-static inline FORCE_INLINE unsigned char sse2_next_match(search_state *search)
+ALWAYS_INLINE(static) unsigned char sse2_next_match(search_state *search)
 {
     int mask = search->matches_mask;
     int index = trailing_zeros(mask);
@@ -424,7 +455,7 @@ static inline FORCE_INLINE unsigned char sse2_next_match(search_state *search)
 #define TARGET_SSE2
 #endif
 
-static inline TARGET_SSE2 FORCE_INLINE unsigned char search_escape_basic_sse2(search_state *search)
+ALWAYS_INLINE(static) TARGET_SSE2 unsigned char search_escape_basic_sse2(search_state *search)
 {
     if (RB_UNLIKELY(search->has_matches)) {
         // There are more matches if search->matches_mask > 0.
@@ -835,18 +866,6 @@ static VALUE mFloat_to_json(int argc, VALUE *argv, VALUE self)
     return cState_partial_generate(Vstate, self, generate_json_float, Qfalse);
 }
 
-/*
- * call-seq: String.included(modul)
- *
- * Extends _modul_ with the String::Extend module.
- */
-static VALUE mString_included_s(VALUE self, VALUE modul)
-{
-    VALUE result = rb_funcall(modul, i_extend, 1, mString_Extend);
-    rb_call_super(1, &modul);
-    return result;
-}
-
 /*
  * call-seq: to_json(*)
  *
@@ -861,51 +880,6 @@ static VALUE mString_to_json(int argc, VALUE *argv, VALUE self)
     return cState_partial_generate(Vstate, self, generate_json_string, Qfalse);
 }
 
-/*
- * call-seq: to_json_raw_object()
- *
- * This method creates a raw object hash, that can be nested into
- * other data structures and will be generated as a raw string. This
- * method should be used, if you want to convert raw strings to JSON
- * instead of UTF-8 strings, e. g. binary data.
- */
-static VALUE mString_to_json_raw_object(VALUE self)
-{
-    VALUE ary;
-    VALUE result = rb_hash_new();
-    rb_hash_aset(result, rb_funcall(mJSON, i_create_id, 0), rb_class_name(rb_obj_class(self)));
-    ary = rb_funcall(self, i_unpack, 1, rb_str_new2("C*"));
-    rb_hash_aset(result, rb_utf8_str_new_lit("raw"), ary);
-    return result;
-}
-
-/*
- * call-seq: to_json_raw(*args)
- *
- * This method creates a JSON text from the result of a call to
- * to_json_raw_object of this String.
- */
-static VALUE mString_to_json_raw(int argc, VALUE *argv, VALUE self)
-{
-    VALUE obj = mString_to_json_raw_object(self);
-    Check_Type(obj, T_HASH);
-    return mHash_to_json(argc, argv, obj);
-}
-
-/*
- * call-seq: json_create(o)
- *
- * Raw Strings are JSON Objects (the raw bytes are stored in an array for the
- * key "raw"). The Ruby String can be created by this module method.
- */
-static VALUE mString_Extend_json_create(VALUE self, VALUE o)
-{
-    VALUE ary;
-    Check_Type(o, T_HASH);
-    ary = rb_hash_aref(o, rb_str_new2("raw"));
-    return rb_funcall(ary, i_pack, 1, rb_str_new2("C*"));
-}
-
 /*
  * call-seq: to_json(*)
  *
@@ -989,11 +963,6 @@ static size_t State_memsize(const void *ptr)
     return sizeof(JSON_Generator_State);
 }
 
-#ifndef HAVE_RB_EXT_RACTOR_SAFE
-#   undef RUBY_TYPED_FROZEN_SHAREABLE
-#   define RUBY_TYPED_FROZEN_SHAREABLE 0
-#endif
-
 static const rb_data_type_t JSON_Generator_State_type = {
     "JSON/Generator/State",
     {
@@ -1035,18 +1004,24 @@ static void vstate_spill(struct generate_json_data *data)
     RB_OBJ_WRITTEN(vstate, Qundef, state->as_json);
 }
 
-static inline VALUE vstate_get(struct generate_json_data *data)
+static inline VALUE json_call_to_json(struct generate_json_data *data, VALUE obj)
 {
     if (RB_UNLIKELY(!data->vstate)) {
         vstate_spill(data);
     }
-    return data->vstate;
+    GET_STATE(data->vstate);
+    state->depth = data->depth;
+    VALUE tmp = rb_funcall(obj, i_to_json, 1, data->vstate);
+    // no need to restore state->depth, vstate is just a temporary State
+    return tmp;
 }
 
-struct hash_foreach_arg {
-    struct generate_json_data *data;
-    int iter;
-};
+static VALUE
+json_call_as_json(JSON_Generator_State *state, VALUE object, VALUE is_key)
+{
+    VALUE proc_args[2] = {object, is_key};
+    return rb_proc_call_with_block(state->as_json, 2, proc_args, Qnil);
+}
 
 static VALUE
 convert_string_subclass(VALUE key)
@@ -1063,6 +1038,145 @@ convert_string_subclass(VALUE key)
     return key_to_s;
 }
 
+static bool enc_utf8_compatible_p(int enc_idx)
+{
+    if (enc_idx == usascii_encindex) return true;
+    if (enc_idx == utf8_encindex) return true;
+    return false;
+}
+
+static VALUE encode_json_string_try(VALUE str)
+{
+    return rb_funcall(str, i_encode, 1, Encoding_UTF_8);
+}
+
+static VALUE encode_json_string_rescue(VALUE str, VALUE exception)
+{
+    raise_generator_error_str(str, rb_funcall(exception, rb_intern("message"), 0));
+    return Qundef;
+}
+
+static inline bool valid_json_string_p(VALUE str)
+{
+    int coderange = rb_enc_str_coderange(str);
+
+    if (RB_LIKELY(coderange == ENC_CODERANGE_7BIT)) {
+        return true;
+    }
+
+    if (RB_LIKELY(coderange == ENC_CODERANGE_VALID)) {
+        return enc_utf8_compatible_p(RB_ENCODING_GET_INLINED(str));
+    }
+
+    return false;
+}
+
+static inline VALUE ensure_valid_encoding(struct generate_json_data *data, VALUE str, bool as_json_called, bool is_key)
+{
+    if (RB_LIKELY(valid_json_string_p(str))) {
+        return str;
+    }
+
+    if (!as_json_called && data->state->strict && RTEST(data->state->as_json)) {
+        VALUE coerced_str = json_call_as_json(data->state, str, Qfalse);
+        if (coerced_str != str) {
+            if (RB_TYPE_P(coerced_str, T_STRING)) {
+                if (!valid_json_string_p(coerced_str)) {
+                    raise_generator_error(str, "source sequence is illegal/malformed utf-8");
+                }
+            } else {
+                // as_json could return another type than T_STRING
+                if (is_key) {
+                    raise_generator_error(coerced_str, "%"PRIsVALUE" not allowed as object key in JSON", CLASS_OF(coerced_str));
+                }
+            }
+
+            return coerced_str;
+        }
+    }
+
+    if (RB_ENCODING_GET_INLINED(str) == binary_encindex) {
+        VALUE utf8_string = rb_enc_associate_index(rb_str_dup(str), utf8_encindex);
+        switch (rb_enc_str_coderange(utf8_string)) {
+            case ENC_CODERANGE_7BIT:
+                return utf8_string;
+            case ENC_CODERANGE_VALID:
+                // For historical reason, we silently reinterpret binary strings as UTF-8 if it would work.
+                // TODO: Raise in 3.0.0
+                rb_warn("JSON.generate: UTF-8 string passed as BINARY, this will raise an encoding error in json 3.0");
+                return utf8_string;
+                break;
+        }
+    }
+
+    return rb_rescue(encode_json_string_try, str, encode_json_string_rescue, str);
+}
+
+static void raw_generate_json_string(FBuffer *buffer, struct generate_json_data *data, VALUE obj)
+{
+    fbuffer_append_char(buffer, '"');
+
+    long len;
+    search_state search;
+    search.buffer = buffer;
+    RSTRING_GETMEM(obj, search.ptr, len);
+    search.cursor = search.ptr;
+    search.end = search.ptr + len;
+
+#ifdef HAVE_SIMD
+    search.matches_mask = 0;
+    search.has_matches = false;
+    search.chunk_base = NULL;
+    search.chunk_end = NULL;
+#endif /* HAVE_SIMD */
+
+    switch (rb_enc_str_coderange(obj)) {
+        case ENC_CODERANGE_7BIT:
+        case ENC_CODERANGE_VALID:
+            if (RB_UNLIKELY(data->state->ascii_only)) {
+                convert_UTF8_to_ASCII_only_JSON(&search, data->state->script_safe ? script_safe_escape_table : ascii_only_escape_table);
+            } else if (RB_UNLIKELY(data->state->script_safe)) {
+                convert_UTF8_to_script_safe_JSON(&search);
+            } else {
+                convert_UTF8_to_JSON(&search);
+            }
+            break;
+        default:
+            raise_generator_error(obj, "source sequence is illegal/malformed utf-8");
+            break;
+    }
+    fbuffer_append_char(buffer, '"');
+}
+
+static void generate_json_string(FBuffer *buffer, struct generate_json_data *data, VALUE obj)
+{
+    obj = ensure_valid_encoding(data, obj, false, false);
+    raw_generate_json_string(buffer, data, obj);
+}
+
+struct hash_foreach_arg {
+    VALUE hash;
+    struct generate_json_data *data;
+    int first_key_type;
+    bool first;
+    bool mixed_keys_encountered;
+};
+
+NOINLINE(static) void
+json_inspect_hash_with_mixed_keys(struct hash_foreach_arg *arg)
+{
+    if (arg->mixed_keys_encountered) {
+        return;
+    }
+    arg->mixed_keys_encountered = true;
+
+    JSON_Generator_State *state = arg->data->state;
+    if (state->on_duplicate_key != JSON_IGNORE) {
+        VALUE do_raise = state->on_duplicate_key == JSON_RAISE ? Qtrue : Qfalse;
+        rb_funcall(mJSON, rb_intern("on_mixed_keys_hash"), 2, arg->hash, do_raise);
+    }
+}
+
 static int
 json_object_i(VALUE key, VALUE val, VALUE _arg)
 {
@@ -1072,22 +1186,34 @@ json_object_i(VALUE key, VALUE val, VALUE _arg)
     FBuffer *buffer = data->buffer;
     JSON_Generator_State *state = data->state;
 
-    long depth = state->depth;
-    int j;
+    long depth = data->depth;
+    int key_type = rb_type(key);
+
+    if (arg->first) {
+        arg->first = false;
+        arg->first_key_type = key_type;
+    }
+    else {
+        fbuffer_append_char(buffer, ',');
+    }
 
-    if (arg->iter > 0) fbuffer_append_char(buffer, ',');
     if (RB_UNLIKELY(data->state->object_nl)) {
         fbuffer_append_str(buffer, data->state->object_nl);
     }
     if (RB_UNLIKELY(data->state->indent)) {
-        for (j = 0; j < depth; j++) {
-            fbuffer_append_str(buffer, data->state->indent);
-        }
+        fbuffer_append_str_repeat(buffer, data->state->indent, depth);
     }
 
     VALUE key_to_s;
-    switch (rb_type(key)) {
+    bool as_json_called = false;
+
+  start:
+    switch (key_type) {
         case T_STRING:
+            if (RB_UNLIKELY(arg->first_key_type != T_STRING)) {
+                json_inspect_hash_with_mixed_keys(arg);
+            }
+
             if (RB_LIKELY(RBASIC_CLASS(key) == rb_cString)) {
                 key_to_s = key;
             } else {
@@ -1095,15 +1221,31 @@ json_object_i(VALUE key, VALUE val, VALUE _arg)
             }
             break;
         case T_SYMBOL:
+            if (RB_UNLIKELY(arg->first_key_type != T_SYMBOL)) {
+                json_inspect_hash_with_mixed_keys(arg);
+            }
+
             key_to_s = rb_sym2str(key);
             break;
         default:
+            if (data->state->strict) {
+                if (RTEST(data->state->as_json) && !as_json_called) {
+                    key = json_call_as_json(data->state, key, Qtrue);
+                    key_type = rb_type(key);
+                    as_json_called = true;
+                    goto start;
+                } else {
+                    raise_generator_error(key, "%"PRIsVALUE" not allowed as object key in JSON", CLASS_OF(key));
+                }
+            }
             key_to_s = rb_convert_type(key, T_STRING, "String", "to_s");
             break;
     }
 
+    key_to_s = ensure_valid_encoding(data, key_to_s, as_json_called, true);
+
     if (RB_LIKELY(RBASIC_CLASS(key_to_s) == rb_cString)) {
-        generate_json_string(buffer, data, key_to_s);
+        raw_generate_json_string(buffer, data, key_to_s);
     } else {
         generate_json(buffer, data, key_to_s);
     }
@@ -1112,46 +1254,43 @@ json_object_i(VALUE key, VALUE val, VALUE _arg)
     if (RB_UNLIKELY(state->space)) fbuffer_append_str(buffer, data->state->space);
     generate_json(buffer, data, val);
 
-    arg->iter++;
     return ST_CONTINUE;
 }
 
 static inline long increase_depth(struct generate_json_data *data)
 {
     JSON_Generator_State *state = data->state;
-    long depth = ++state->depth;
+    long depth = ++data->depth;
     if (RB_UNLIKELY(depth > state->max_nesting && state->max_nesting)) {
-        rb_raise(eNestingError, "nesting of %ld is too deep", --state->depth);
+        rb_raise(eNestingError, "nesting of %ld is too deep. Did you try to serialize objects with circular references?", --data->depth);
     }
     return depth;
 }
 
 static void generate_json_object(FBuffer *buffer, struct generate_json_data *data, VALUE obj)
 {
-    int j;
     long depth = increase_depth(data);
 
     if (RHASH_SIZE(obj) == 0) {
         fbuffer_append(buffer, "{}", 2);
-        --data->state->depth;
+        --data->depth;
         return;
     }
 
     fbuffer_append_char(buffer, '{');
 
     struct hash_foreach_arg arg = {
+        .hash = obj,
         .data = data,
-        .iter = 0,
+        .first = true,
     };
     rb_hash_foreach(obj, json_object_i, (VALUE)&arg);
 
-    depth = --data->state->depth;
+    depth = --data->depth;
     if (RB_UNLIKELY(data->state->object_nl)) {
         fbuffer_append_str(buffer, data->state->object_nl);
         if (RB_UNLIKELY(data->state->indent)) {
-            for (j = 0; j < depth; j++) {
-                fbuffer_append_str(buffer, data->state->indent);
-            }
+            fbuffer_append_str_repeat(buffer, data->state->indent, depth);
         }
     }
     fbuffer_append_char(buffer, '}');
@@ -1159,125 +1298,41 @@ static void generate_json_object(FBuffer *buffer, struct generate_json_data *dat
 
 static void generate_json_array(FBuffer *buffer, struct generate_json_data *data, VALUE obj)
 {
-    int i, j;
     long depth = increase_depth(data);
 
     if (RARRAY_LEN(obj) == 0) {
         fbuffer_append(buffer, "[]", 2);
-        --data->state->depth;
+        --data->depth;
         return;
     }
 
     fbuffer_append_char(buffer, '[');
     if (RB_UNLIKELY(data->state->array_nl)) fbuffer_append_str(buffer, data->state->array_nl);
-    for (i = 0; i < RARRAY_LEN(obj); i++) {
+    for (int i = 0; i < RARRAY_LEN(obj); i++) {
         if (i > 0) {
             fbuffer_append_char(buffer, ',');
             if (RB_UNLIKELY(data->state->array_nl)) fbuffer_append_str(buffer, data->state->array_nl);
         }
         if (RB_UNLIKELY(data->state->indent)) {
-            for (j = 0; j < depth; j++) {
-                fbuffer_append_str(buffer, data->state->indent);
-            }
+            fbuffer_append_str_repeat(buffer, data->state->indent, depth);
         }
         generate_json(buffer, data, RARRAY_AREF(obj, i));
     }
-    data->state->depth = --depth;
+    data->depth = --depth;
     if (RB_UNLIKELY(data->state->array_nl)) {
         fbuffer_append_str(buffer, data->state->array_nl);
         if (RB_UNLIKELY(data->state->indent)) {
-            for (j = 0; j < depth; j++) {
-                fbuffer_append_str(buffer, data->state->indent);
-            }
+            fbuffer_append_str_repeat(buffer, data->state->indent, depth);
         }
     }
     fbuffer_append_char(buffer, ']');
 }
 
-static inline int enc_utf8_compatible_p(int enc_idx)
-{
-    if (enc_idx == usascii_encindex) return 1;
-    if (enc_idx == utf8_encindex) return 1;
-    return 0;
-}
-
-static VALUE encode_json_string_try(VALUE str)
-{
-    return rb_funcall(str, i_encode, 1, Encoding_UTF_8);
-}
-
-static VALUE encode_json_string_rescue(VALUE str, VALUE exception)
-{
-    raise_generator_error_str(str, rb_funcall(exception, rb_intern("message"), 0));
-    return Qundef;
-}
-
-static inline VALUE ensure_valid_encoding(VALUE str)
-{
-    int encindex = RB_ENCODING_GET(str);
-    VALUE utf8_string;
-    if (RB_UNLIKELY(!enc_utf8_compatible_p(encindex))) {
-        if (encindex == binary_encindex) {
-            utf8_string = rb_enc_associate_index(rb_str_dup(str), utf8_encindex);
-            switch (rb_enc_str_coderange(utf8_string)) {
-                case ENC_CODERANGE_7BIT:
-                    return utf8_string;
-                case ENC_CODERANGE_VALID:
-                    // For historical reason, we silently reinterpret binary strings as UTF-8 if it would work.
-                    // TODO: Raise in 3.0.0
-                    rb_warn("JSON.generate: UTF-8 string passed as BINARY, this will raise an encoding error in json 3.0");
-                    return utf8_string;
-                    break;
-            }
-        }
-
-        str = rb_rescue(encode_json_string_try, str, encode_json_string_rescue, str);
-    }
-    return str;
-}
-
-static void generate_json_string(FBuffer *buffer, struct generate_json_data *data, VALUE obj)
-{
-    obj = ensure_valid_encoding(obj);
-
-    fbuffer_append_char(buffer, '"');
-
-    long len;
-    search_state search;
-    search.buffer = buffer;
-    RSTRING_GETMEM(obj, search.ptr, len);
-    search.cursor = search.ptr;
-    search.end = search.ptr + len;
-
-#ifdef HAVE_SIMD
-    search.matches_mask = 0;
-    search.has_matches = false;
-    search.chunk_base = NULL;
-#endif /* HAVE_SIMD */
-
-    switch (rb_enc_str_coderange(obj)) {
-        case ENC_CODERANGE_7BIT:
-        case ENC_CODERANGE_VALID:
-            if (RB_UNLIKELY(data->state->ascii_only)) {
-                convert_UTF8_to_ASCII_only_JSON(&search, data->state->script_safe ? script_safe_escape_table : ascii_only_escape_table);
-            } else if (RB_UNLIKELY(data->state->script_safe)) {
-                convert_UTF8_to_script_safe_JSON(&search);
-            } else {
-                convert_UTF8_to_JSON(&search);
-            }
-            break;
-        default:
-            raise_generator_error(obj, "source sequence is illegal/malformed utf-8");
-            break;
-    }
-    fbuffer_append_char(buffer, '"');
-}
-
 static void generate_json_fallback(FBuffer *buffer, struct generate_json_data *data, VALUE obj)
 {
     VALUE tmp;
     if (rb_respond_to(obj, i_to_json)) {
-        tmp = rb_funcall(obj, i_to_json, 1, vstate_get(data));
+        tmp = json_call_to_json(data, obj);
         Check_Type(tmp, T_STRING);
         fbuffer_append_str(buffer, tmp);
     } else {
@@ -1319,7 +1374,7 @@ static void generate_json_fixnum(FBuffer *buffer, struct generate_json_data *dat
 static void generate_json_bignum(FBuffer *buffer, struct generate_json_data *data, VALUE obj)
 {
     VALUE tmp = rb_funcall(obj, i_to_s, 0);
-    fbuffer_append_str(buffer, tmp);
+    fbuffer_append_str(buffer, StringValue(tmp));
 }
 
 #ifdef RUBY_INTEGER_UNIFICATION
@@ -1340,11 +1395,11 @@ static void generate_json_float(FBuffer *buffer, struct generate_json_data *data
         /* for NaN and Infinity values we either raise an error or rely on Float#to_s. */
         if (!allow_nan) {
             if (data->state->strict && data->state->as_json) {
-                VALUE casted_obj = rb_proc_call_with_block(data->state->as_json, 1, &obj, Qnil);
+                VALUE casted_obj = json_call_as_json(data->state, obj, Qfalse);
                 if (casted_obj != obj) {
                     increase_depth(data);
                     generate_json(buffer, data, casted_obj);
-                    data->state->depth--;
+                    data->depth--;
                     return;
                 }
             }
@@ -1357,12 +1412,11 @@ static void generate_json_float(FBuffer *buffer, struct generate_json_data *data
     }
 
     /* This implementation writes directly into the buffer. We reserve
-     * the 28 characters that fpconv_dtoa states as its maximum.
+     * the 32 characters that fpconv_dtoa states as its maximum.
      */
-    fbuffer_inc_capa(buffer, 28);
+    fbuffer_inc_capa(buffer, 32);
     char* d = buffer->ptr + buffer->len;
     int len = fpconv_dtoa(value, d);
-
     /* fpconv_dtoa converts a float to its shortest string representation,
      * but it adds a ".0" if this is a plain integer.
      */
@@ -1412,7 +1466,16 @@ start:
                 break;
             case T_STRING:
                 if (klass != rb_cString) goto general;
-                generate_json_string(buffer, data, obj);
+
+                if (RB_LIKELY(valid_json_string_p(obj))) {
+                    raw_generate_json_string(buffer, data, obj);
+                } else if (as_json_called) {
+                    raise_generator_error(obj, "source sequence is illegal/malformed utf-8");
+                } else {
+                    obj = ensure_valid_encoding(data, obj, false, false);
+                    as_json_called = true;
+                    goto start;
+                }
                 break;
             case T_SYMBOL:
                 generate_json_symbol(buffer, data, obj);
@@ -1429,7 +1492,7 @@ start:
             general:
                 if (data->state->strict) {
                     if (RTEST(data->state->as_json) && !as_json_called) {
-                        obj = rb_proc_call_with_block(data->state->as_json, 1, &obj, Qnil);
+                        obj = json_call_as_json(data->state, obj, Qfalse);
                         as_json_called = true;
                         goto start;
                     } else {
@@ -1448,16 +1511,14 @@ static VALUE generate_json_try(VALUE d)
 
     data->func(data->buffer, data, data->obj);
 
-    return Qnil;
+    return fbuffer_finalize(data->buffer);
 }
 
-static VALUE generate_json_rescue(VALUE d, VALUE exc)
+static VALUE generate_json_ensure(VALUE d)
 {
     struct generate_json_data *data = (struct generate_json_data *)d;
     fbuffer_free(data->buffer);
 
-    rb_exc_raise(exc);
-
     return Qundef;
 }
 
@@ -1473,14 +1534,15 @@ static VALUE cState_partial_generate(VALUE self, VALUE obj, generator_func func,
 
     struct generate_json_data data = {
         .buffer = &buffer,
-        .vstate = self,
+        .vstate = Qfalse, // don't use self as it may be frozen and its depth is mutated when calling to_json
         .state = state,
+        .depth = state->depth,
         .obj = obj,
         .func = func
     };
-    rb_rescue(generate_json_try, (VALUE)&data, generate_json_rescue, (VALUE)&data);
-
-    return fbuffer_finalize(&buffer);
+    VALUE result = rb_ensure(generate_json_try, (VALUE)&data, generate_json_ensure, (VALUE)&data);
+    RB_GC_GUARD(self);
+    return result;
 }
 
 /* call-seq:
@@ -1496,10 +1558,7 @@ static VALUE cState_generate(int argc, VALUE *argv, VALUE self)
     rb_check_arity(argc, 1, 2);
     VALUE obj = argv[0];
     VALUE io = argc > 1 ? argv[1] : Qnil;
-    VALUE result = cState_partial_generate(self, obj, generate_json, io);
-    GET_STATE(self);
-    (void)state;
-    return result;
+    return cState_partial_generate(self, obj, generate_json, io);
 }
 
 static VALUE cState_initialize(int argc, VALUE *argv, VALUE self)
@@ -1580,6 +1639,7 @@ static VALUE string_config(VALUE config)
  */
 static VALUE cState_indent_set(VALUE self, VALUE indent)
 {
+    rb_check_frozen(self);
     GET_STATE(self);
     RB_OBJ_WRITE(self, &state->indent, string_config(indent));
     return Qnil;
@@ -1605,6 +1665,7 @@ static VALUE cState_space(VALUE self)
  */
 static VALUE cState_space_set(VALUE self, VALUE space)
 {
+    rb_check_frozen(self);
     GET_STATE(self);
     RB_OBJ_WRITE(self, &state->space, string_config(space));
     return Qnil;
@@ -1628,6 +1689,7 @@ static VALUE cState_space_before(VALUE self)
  */
 static VALUE cState_space_before_set(VALUE self, VALUE space_before)
 {
+    rb_check_frozen(self);
     GET_STATE(self);
     RB_OBJ_WRITE(self, &state->space_before, string_config(space_before));
     return Qnil;
@@ -1653,6 +1715,7 @@ static VALUE cState_object_nl(VALUE self)
  */
 static VALUE cState_object_nl_set(VALUE self, VALUE object_nl)
 {
+    rb_check_frozen(self);
     GET_STATE(self);
     RB_OBJ_WRITE(self, &state->object_nl, string_config(object_nl));
     return Qnil;
@@ -1676,6 +1739,7 @@ static VALUE cState_array_nl(VALUE self)
  */
 static VALUE cState_array_nl_set(VALUE self, VALUE array_nl)
 {
+    rb_check_frozen(self);
     GET_STATE(self);
     RB_OBJ_WRITE(self, &state->array_nl, string_config(array_nl));
     return Qnil;
@@ -1699,6 +1763,7 @@ static VALUE cState_as_json(VALUE self)
  */
 static VALUE cState_as_json_set(VALUE self, VALUE as_json)
 {
+    rb_check_frozen(self);
     GET_STATE(self);
     RB_OBJ_WRITE(self, &state->as_json, rb_convert_type(as_json, T_DATA, "Proc", "to_proc"));
     return Qnil;
@@ -1741,6 +1806,7 @@ static long long_config(VALUE num)
  */
 static VALUE cState_max_nesting_set(VALUE self, VALUE depth)
 {
+    rb_check_frozen(self);
     GET_STATE(self);
     state->max_nesting = long_config(depth);
     return Qnil;
@@ -1766,6 +1832,7 @@ static VALUE cState_script_safe(VALUE self)
  */
 static VALUE cState_script_safe_set(VALUE self, VALUE enable)
 {
+    rb_check_frozen(self);
     GET_STATE(self);
     state->script_safe = RTEST(enable);
     return Qnil;
@@ -1797,6 +1864,7 @@ static VALUE cState_strict(VALUE self)
  */
 static VALUE cState_strict_set(VALUE self, VALUE enable)
 {
+    rb_check_frozen(self);
     GET_STATE(self);
     state->strict = RTEST(enable);
     return Qnil;
@@ -1821,6 +1889,7 @@ static VALUE cState_allow_nan_p(VALUE self)
  */
 static VALUE cState_allow_nan_set(VALUE self, VALUE enable)
 {
+    rb_check_frozen(self);
     GET_STATE(self);
     state->allow_nan = RTEST(enable);
     return Qnil;
@@ -1845,11 +1914,25 @@ static VALUE cState_ascii_only_p(VALUE self)
  */
 static VALUE cState_ascii_only_set(VALUE self, VALUE enable)
 {
+    rb_check_frozen(self);
     GET_STATE(self);
     state->ascii_only = RTEST(enable);
     return Qnil;
 }
 
+static VALUE cState_allow_duplicate_key_p(VALUE self)
+{
+    GET_STATE(self);
+    switch (state->on_duplicate_key) {
+        case JSON_IGNORE:
+            return Qtrue;
+        case JSON_DEPRECATED:
+            return Qnil;
+        default:
+            return Qfalse;
+    }
+}
+
 /*
  * call-seq: depth
  *
@@ -1869,6 +1952,7 @@ static VALUE cState_depth(VALUE self)
  */
 static VALUE cState_depth_set(VALUE self, VALUE depth)
 {
+    rb_check_frozen(self);
     GET_STATE(self);
     state->depth = long_config(depth);
     return Qnil;
@@ -1902,6 +1986,7 @@ static void buffer_initial_length_set(JSON_Generator_State *state, VALUE buffer_
  */
 static VALUE cState_buffer_initial_length_set(VALUE self, VALUE buffer_initial_length)
 {
+    rb_check_frozen(self);
     GET_STATE(self);
     buffer_initial_length_set(state, buffer_initial_length);
     return Qnil;
@@ -1939,8 +2024,10 @@ static int configure_state_i(VALUE key, VALUE val, VALUE _arg)
     else if (key == sym_script_safe)           { state->script_safe = RTEST(val); }
     else if (key == sym_escape_slash)          { state->script_safe = RTEST(val); }
     else if (key == sym_strict)                { state->strict = RTEST(val); }
+    else if (key == sym_allow_duplicate_key)   { state->on_duplicate_key = RTEST(val) ? JSON_IGNORE : JSON_RAISE; }
     else if (key == sym_as_json)               {
         VALUE proc = RTEST(val) ? rb_convert_type(val, T_DATA, "Proc", "to_proc") : Qfalse;
+        state->as_json_single_arg = proc && rb_proc_arity(proc) == 1;
         state_write_value(data, &state->as_json, proc);
     }
     return ST_CONTINUE;
@@ -1966,6 +2053,7 @@ static void configure_state(JSON_Generator_State *state, VALUE vstate, VALUE con
 
 static VALUE cState_configure(VALUE self, VALUE opts)
 {
+    rb_check_frozen(self);
     GET_STATE(self);
     configure_state(state, self, opts);
     return self;
@@ -1987,12 +2075,11 @@ static VALUE cState_m_generate(VALUE klass, VALUE obj, VALUE opts, VALUE io)
         .buffer = &buffer,
         .vstate = Qfalse,
         .state = &state,
+        .depth = state.depth,
         .obj = obj,
         .func = generate_json,
     };
-    rb_rescue(generate_json_try, (VALUE)&data, generate_json_rescue, (VALUE)&data);
-
-    return fbuffer_finalize(&buffer);
+    return rb_ensure(generate_json_try, (VALUE)&data, generate_json_ensure, (VALUE)&data);
 }
 
 /*
@@ -2062,7 +2149,8 @@ void Init_generator(void)
     rb_define_method(cState, "buffer_initial_length", cState_buffer_initial_length, 0);
     rb_define_method(cState, "buffer_initial_length=", cState_buffer_initial_length_set, 1);
     rb_define_method(cState, "generate", cState_generate, -1);
-    rb_define_alias(cState, "generate_new", "generate"); // :nodoc:
+
+    rb_define_private_method(cState, "allow_duplicate_key?", cState_allow_duplicate_key_p, 0);
 
     rb_define_singleton_method(cState, "generate", cState_m_generate, 3);
 
@@ -2091,13 +2179,7 @@ void Init_generator(void)
     rb_define_method(mFloat, "to_json", mFloat_to_json, -1);
 
     VALUE mString = rb_define_module_under(mGeneratorMethods, "String");
-    rb_define_singleton_method(mString, "included", mString_included_s, 1);
     rb_define_method(mString, "to_json", mString_to_json, -1);
-    rb_define_method(mString, "to_json_raw", mString_to_json_raw, -1);
-    rb_define_method(mString, "to_json_raw_object", mString_to_json_raw_object, 0);
-
-    mString_Extend = rb_define_module_under(mString, "Extend");
-    rb_define_method(mString_Extend, "json_create", mString_Extend_json_create, 1);
 
     VALUE mTrueClass = rb_define_module_under(mGeneratorMethods, "TrueClass");
     rb_define_method(mTrueClass, "to_json", mTrueClass_to_json, -1);
@@ -2114,10 +2196,6 @@ void Init_generator(void)
     i_to_s = rb_intern("to_s");
     i_to_json = rb_intern("to_json");
     i_new = rb_intern("new");
-    i_pack = rb_intern("pack");
-    i_unpack = rb_intern("unpack");
-    i_create_id = rb_intern("create_id");
-    i_extend = rb_intern("extend");
     i_encode = rb_intern("encode");
 
     sym_indent = ID2SYM(rb_intern("indent"));
@@ -2134,6 +2212,7 @@ void Init_generator(void)
     sym_escape_slash = ID2SYM(rb_intern("escape_slash"));
     sym_strict = ID2SYM(rb_intern("strict"));
     sym_as_json = ID2SYM(rb_intern("as_json"));
+    sym_allow_duplicate_key = ID2SYM(rb_intern("allow_duplicate_key"));
 
     usascii_encindex = rb_usascii_encindex();
     utf8_encindex = rb_utf8_encindex();
@@ -2141,22 +2220,5 @@ void Init_generator(void)
 
     rb_require("json/ext/generator/state");
 
-
-    switch (find_simd_implementation()) {
-#ifdef HAVE_SIMD
-#ifdef HAVE_SIMD_NEON
-        case SIMD_NEON:
-            search_escape_basic_impl = search_escape_basic_neon;
-            break;
-#endif /* HAVE_SIMD_NEON */
-#ifdef HAVE_SIMD_SSE2
-        case SIMD_SSE2:
-            search_escape_basic_impl = search_escape_basic_sse2;
-            break;
-#endif /* HAVE_SIMD_SSE2 */
-#endif /* HAVE_SIMD */
-        default:
-            search_escape_basic_impl = search_escape_basic;
-            break;
-    }
+    simd_impl = find_simd_implementation();
 }