json 2.7.3 → 2.12.0

data/ext/json/ext/generator/generator.c

@@ -1,13 +1,205 @@
+#include "ruby.h"
 #include "../fbuffer/fbuffer.h"
-#include "generator.h"
+#include "../vendor/fpconv.c"
+
+#include <math.h>
+#include <ctype.h>
+
+#include "simd.h"
+
+/* ruby api and some helpers */
+
+typedef struct JSON_Generator_StateStruct {
+    VALUE indent;
+    VALUE space;
+    VALUE space_before;
+    VALUE object_nl;
+    VALUE array_nl;
+    VALUE as_json;
+
+    long max_nesting;
+    long depth;
+    long buffer_initial_length;
+
+    bool allow_nan;
+    bool ascii_only;
+    bool script_safe;
+    bool strict;
+} JSON_Generator_State;
 
 #ifndef RB_UNLIKELY
 #define RB_UNLIKELY(cond) (cond)
 #endif
 
-static VALUE mJSON, cState, mString_Extend, eGeneratorError, eNestingError, Encoding_UTF_8;
+static VALUE mJSON, cState, cFragment, mString_Extend, eGeneratorError, eNestingError, Encoding_UTF_8;
 
 static ID i_to_s, i_to_json, i_new, i_pack, i_unpack, i_create_id, i_extend, i_encode;
+static VALUE sym_indent, sym_space, sym_space_before, sym_object_nl, sym_array_nl, sym_max_nesting, sym_allow_nan,
+             sym_ascii_only, sym_depth, sym_buffer_initial_length, sym_script_safe, sym_escape_slash, sym_strict, sym_as_json;
+
+
+#define GET_STATE_TO(self, state) \
+    TypedData_Get_Struct(self, JSON_Generator_State, &JSON_Generator_State_type, state)
+
+#define GET_STATE(self)                       \
+    JSON_Generator_State *state;              \
+    GET_STATE_TO(self, state)
+
+struct generate_json_data;
+
+typedef void (*generator_func)(FBuffer *buffer, struct generate_json_data *data, VALUE obj);
+
+struct generate_json_data {
+    FBuffer *buffer;
+    VALUE vstate;
+    JSON_Generator_State *state;
+    VALUE obj;
+    generator_func func;
+};
+
+static VALUE cState_from_state_s(VALUE self, VALUE opts);
+static VALUE cState_partial_generate(VALUE self, VALUE obj, generator_func, VALUE io);
+static void generate_json(FBuffer *buffer, struct generate_json_data *data, VALUE obj);
+static void generate_json_object(FBuffer *buffer, struct generate_json_data *data, VALUE obj);
+static void generate_json_array(FBuffer *buffer, struct generate_json_data *data, VALUE obj);
+static void generate_json_string(FBuffer *buffer, struct generate_json_data *data, VALUE obj);
+static void generate_json_null(FBuffer *buffer, struct generate_json_data *data, VALUE obj);
+static void generate_json_false(FBuffer *buffer, struct generate_json_data *data, VALUE obj);
+static void generate_json_true(FBuffer *buffer, struct generate_json_data *data, VALUE obj);
+#ifdef RUBY_INTEGER_UNIFICATION
+static void generate_json_integer(FBuffer *buffer, struct generate_json_data *data, VALUE obj);
+#endif
+static void generate_json_fixnum(FBuffer *buffer, struct generate_json_data *data, VALUE obj);
+static void generate_json_bignum(FBuffer *buffer, struct generate_json_data *data, VALUE obj);
+static void generate_json_float(FBuffer *buffer, struct generate_json_data *data, VALUE obj);
+static void generate_json_fragment(FBuffer *buffer, struct generate_json_data *data, VALUE obj);
+
+static int usascii_encindex, utf8_encindex, binary_encindex;
+
+#ifdef RBIMPL_ATTR_NORETURN
+RBIMPL_ATTR_NORETURN()
+#endif
+static void raise_generator_error_str(VALUE invalid_object, VALUE str)
+{
+    VALUE exc = rb_exc_new_str(eGeneratorError, str);
+    rb_ivar_set(exc, rb_intern("@invalid_object"), invalid_object);
+    rb_exc_raise(exc);
+}
+
+#ifdef RBIMPL_ATTR_NORETURN
+RBIMPL_ATTR_NORETURN()
+#endif
+#ifdef RBIMPL_ATTR_FORMAT
+RBIMPL_ATTR_FORMAT(RBIMPL_PRINTF_FORMAT, 2, 3)
+#endif
+static void raise_generator_error(VALUE invalid_object, const char *fmt, ...)
+{
+    va_list args;
+    va_start(args, fmt);
+    VALUE str = rb_vsprintf(fmt, args);
+    va_end(args);
+    raise_generator_error_str(invalid_object, str);
+}
+
+// 0 - single byte char that don't need to be escaped.
+// (x | 8) - char that needs to be escaped.
+static const unsigned char CHAR_LENGTH_MASK = 7;
+static const unsigned char ESCAPE_MASK = 8;
+
+typedef struct _search_state {
+    const char *ptr;
+    const char *end;
+    const char *cursor;
+    FBuffer *buffer;
+
+#ifdef HAVE_SIMD
+    const char *chunk_base;
+    const char *chunk_end;
+    bool has_matches;
+
+#if defined(HAVE_SIMD_NEON)
+    uint64_t matches_mask;
+#elif defined(HAVE_SIMD_SSE2)
+    int matches_mask;
+#else
+#error "Unknown SIMD Implementation."
+#endif /* HAVE_SIMD_NEON */
+#endif /* HAVE_SIMD */
+} search_state;
+
+#if (defined(__GNUC__ ) || defined(__clang__))
+#define FORCE_INLINE __attribute__((always_inline))
+#else
+#define FORCE_INLINE
+#endif
+
+static inline FORCE_INLINE void search_flush(search_state *search)
+{
+    // Do not remove this conditional without profiling, specifically escape-heavy text.
+    // escape_UTF8_char_basic will advance search->ptr and search->cursor (effectively a search_flush).
+    // For back-to-back characters that need to be escaped, specifcally for the SIMD code paths, this method
+    // will be called just before calling escape_UTF8_char_basic. There will be no characers to append for the
+    // consecutive characters that need to be escaped. While the fbuffer_append is a no-op if
+    // nothing needs to be flushed, we can save a few memory references with this conditional.
+    if (search->ptr > search->cursor) {
+        fbuffer_append(search->buffer, search->cursor, search->ptr - search->cursor);
+        search->cursor = search->ptr;
+    }
+}
+
+static const unsigned char escape_table_basic[256] = {
+    // ASCII Control Characters
+     9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
+     9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
+    // ASCII Characters
+     0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // '"'
+     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, // '\\'
+     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+};
+
+static unsigned char (*search_escape_basic_impl)(search_state *);
+
+static inline unsigned char search_escape_basic(search_state *search)
+{
+    while (search->ptr < search->end) {
+        if (RB_UNLIKELY(escape_table_basic[(const unsigned char)*search->ptr])) {
+            search_flush(search);
+            return 1;
+        } else {
+            search->ptr++;
+        }
+    }
+    search_flush(search);
+    return 0;
+}
+
+static inline FORCE_INLINE void escape_UTF8_char_basic(search_state *search)
+{
+    const unsigned char ch = (unsigned char)*search->ptr;
+    switch (ch) {
+        case '"':  fbuffer_append(search->buffer, "\\\"", 2); break;
+        case '\\': fbuffer_append(search->buffer, "\\\\", 2); break;
+        case '/':  fbuffer_append(search->buffer, "\\/", 2);  break;
+        case '\b': fbuffer_append(search->buffer, "\\b", 2);  break;
+        case '\f': fbuffer_append(search->buffer, "\\f", 2);  break;
+        case '\n': fbuffer_append(search->buffer, "\\n", 2);  break;
+        case '\r': fbuffer_append(search->buffer, "\\r", 2);  break;
+        case '\t': fbuffer_append(search->buffer, "\\t", 2);  break;
+        default: {
+            const char *hexdig = "0123456789abcdef";
+            char scratch[6] = { '\\', 'u', '0', '0', 0, 0 };
+            scratch[4] = hexdig[(ch >> 4) & 0xf];
+            scratch[5] = hexdig[ch & 0xf];
+            fbuffer_append(search->buffer, scratch, 6);
+            break;
+        }
+    }
+    search->ptr++;
+    search->cursor = search->ptr;
+}
 
 /* Converts in_string to a JSON string (without the wrapping '"'
  * characters) in FBuffer out_buffer.
@@ -19,296 +211,516 @@ static ID i_to_s, i_to_json, i_new, i_pack, i_unpack, i_create_id, i_extend, i_e
  *
  * - If out_ascii_only: non-ASCII characters (>0x7F)
  *
- * - If out_script_safe: forwardslash, line separator (U+2028), and
+ * - If script_safe: forwardslash (/), line separator (U+2028), and
  *   paragraph separator (U+2029)
  *
  * Everything else (should be UTF-8) is just passed through and
  * appended to the result.
  */
-static void convert_UTF8_to_JSON(FBuffer *out_buffer, VALUE str, const char escape_table[256], bool out_script_safe)
+static inline void convert_UTF8_to_JSON(search_state *search)
 {
-    const char *hexdig = "0123456789abcdef";
-    char scratch[12] = { '\\', 'u', 0, 0, 0, 0, '\\', 'u' };
+    while (search_escape_basic_impl(search)) {
+        escape_UTF8_char_basic(search);
+    }
+}
 
-    const char *ptr = RSTRING_PTR(str);
-    unsigned long len = RSTRING_LEN(str);
+static inline void escape_UTF8_char(search_state *search, unsigned char ch_len)
+{
+    const unsigned char ch = (unsigned char)*search->ptr;
+    switch (ch_len) {
+        case 1: {
+            switch (ch) {
+                case '"':  fbuffer_append(search->buffer, "\\\"", 2); break;
+                case '\\': fbuffer_append(search->buffer, "\\\\", 2); break;
+                case '/':  fbuffer_append(search->buffer, "\\/", 2);  break;
+                case '\b': fbuffer_append(search->buffer, "\\b", 2);  break;
+                case '\f': fbuffer_append(search->buffer, "\\f", 2);  break;
+                case '\n': fbuffer_append(search->buffer, "\\n", 2);  break;
+                case '\r': fbuffer_append(search->buffer, "\\r", 2);  break;
+                case '\t': fbuffer_append(search->buffer, "\\t", 2);  break;
+                default: {
+                    const char *hexdig = "0123456789abcdef";
+                    char scratch[6] = { '\\', 'u', '0', '0', 0, 0 };
+                    scratch[4] = hexdig[(ch >> 4) & 0xf];
+                    scratch[5] = hexdig[ch & 0xf];
+                    fbuffer_append(search->buffer, scratch, 6);
+                    break;
+                }
+            }
+            break;
+        }
+        case 3: {
+            if (search->ptr[2] & 1) {
+                fbuffer_append(search->buffer, "\\u2029", 6);
+            } else {
+                fbuffer_append(search->buffer, "\\u2028", 6);
+            }
+            break;
+        }
+    }
+    search->cursor = (search->ptr += ch_len);
+}
 
-    unsigned long beg = 0, pos = 0;
+#ifdef HAVE_SIMD
 
-#define FLUSH_POS(bytes) if (pos > beg) { fbuffer_append(out_buffer, &ptr[beg], pos - beg); } pos += bytes; beg = pos;
+static inline FORCE_INLINE char *copy_remaining_bytes(search_state *search, unsigned long vec_len, unsigned long len)
+{
+    // Flush the buffer so everything up until the last 'len' characters are unflushed.
+    search_flush(search);
 
-    while (pos < len) {
-        unsigned char ch = ptr[pos];
-        unsigned char ch_len = escape_table[ch];
-        /* JSON encoding */
+    FBuffer *buf = search->buffer;
+    fbuffer_inc_capa(buf, vec_len);
 
-        if (RB_UNLIKELY(ch_len)) {
-            switch (ch_len) {
-                case 0:
-                    pos++;
-                    break;
-                case 1: {
-                    FLUSH_POS(1);
-                    switch (ch) {
-                        case '"':  fbuffer_append(out_buffer, "\\\"", 2); break;
-                        case '\\': fbuffer_append(out_buffer, "\\\\", 2); break;
-                        case '/':  fbuffer_append(out_buffer, "\\/", 2); break;
-                        case '\b': fbuffer_append(out_buffer, "\\b", 2); break;
-                        case '\f': fbuffer_append(out_buffer, "\\f", 2); break;
-                        case '\n': fbuffer_append(out_buffer, "\\n", 2); break;
-                        case '\r': fbuffer_append(out_buffer, "\\r", 2); break;
-                        case '\t': fbuffer_append(out_buffer, "\\t", 2); break;
-                        default: {
-                            scratch[2] = hexdig[ch >> 12];
-                            scratch[3] = hexdig[(ch >> 8) & 0xf];
-                            scratch[4] = hexdig[(ch >> 4) & 0xf];
-                            scratch[5] = hexdig[ch & 0xf];
-                            fbuffer_append(out_buffer, scratch, 6);
-                            break;
-                        }
-                    }
-                    break;
-                }
-                case 3: {
-                    unsigned char b2 = ptr[pos + 1];
-                    if (RB_UNLIKELY(out_script_safe && b2 == 0x80)) {
-                        unsigned char b3 = ptr[pos + 2];
-                        if (b3 == 0xA8) {
-                            FLUSH_POS(3);
-                            fbuffer_append(out_buffer, "\\u2028", 6);
-                            break;
-                        } else if (b3 == 0xA9) {
-                            FLUSH_POS(3);
-                            fbuffer_append(out_buffer, "\\u2029", 6);
-                            break;
-                        }
-                    }
-                    // fallthrough
-                }
-                default:
-                    pos += ch_len;
-                    break;
-            }
+    char *s = (buf->ptr + buf->len);
+
+    // Pad the buffer with dummy characters that won't need escaping.
+    // This seem wateful at first sight, but memset of vector length is very fast.
+    memset(s, 'X', vec_len);
+
+    // Optimistically copy the remaining 'len' characters to the output FBuffer. If there are no characters
+    // to escape, then everything ends up in the correct spot. Otherwise it was convenient temporary storage.
+    MEMCPY(s, search->ptr, char, len);
+
+    return s;
+}
+
+#ifdef HAVE_SIMD_NEON
+
+static inline FORCE_INLINE unsigned char neon_next_match(search_state *search)
+{
+    uint64_t mask = search->matches_mask;
+    uint32_t index = trailing_zeros64(mask) >> 2;
+
+    // It is assumed escape_UTF8_char_basic will only ever increase search->ptr by at most one character.
+    // If we want to use a similar approach for full escaping we'll need to ensure:
+    //     search->chunk_base + index >= search->ptr
+    // However, since we know escape_UTF8_char_basic only increases search->ptr by one, if the next match
+    // is one byte after the previous match then:
+    //     search->chunk_base + index == search->ptr
+    search->ptr = search->chunk_base + index;
+    mask &= mask - 1;
+    search->matches_mask = mask;
+    search_flush(search);
+    return 1;
+}
+
+// See: https://community.arm.com/arm-community-blogs/b/servers-and-cloud-computing-blog/posts/porting-x86-vector-bitmask-optimizations-to-arm-neon
+static inline FORCE_INLINE uint64_t neon_match_mask(uint8x16_t matches)
+{
+    const uint8x8_t res = vshrn_n_u16(vreinterpretq_u16_u8(matches), 4);
+    const uint64_t mask = vget_lane_u64(vreinterpret_u64_u8(res), 0);
+    return mask & 0x8888888888888888ull;
+}
+
+static inline FORCE_INLINE uint64_t neon_rules_update(const char *ptr)
+{
+    uint8x16_t chunk = vld1q_u8((const unsigned char *)ptr);
+
+    // Trick: c < 32 || c == 34 can be factored as c ^ 2 < 33
+    // https://lemire.me/blog/2025/04/13/detect-control-characters-quotes-and-backslashes-efficiently-using-swar/
+    const uint8x16_t too_low_or_dbl_quote = vcltq_u8(veorq_u8(chunk, vdupq_n_u8(2)), vdupq_n_u8(33));
+
+    uint8x16_t has_backslash = vceqq_u8(chunk, vdupq_n_u8('\\'));
+    uint8x16_t needs_escape  = vorrq_u8(too_low_or_dbl_quote, has_backslash);
+
+    return neon_match_mask(needs_escape);
+}
+
+static inline unsigned char search_escape_basic_neon(search_state *search)
+{
+    if (RB_UNLIKELY(search->has_matches)) {
+        // There are more matches if search->matches_mask > 0.
+        if (search->matches_mask > 0) {
+            return neon_next_match(search);
         } else {
-            pos++;
+            // neon_next_match will only advance search->ptr up to the last matching character. 
+            // Skip over any characters in the last chunk that occur after the last match.
+            search->has_matches = false;
+            search->ptr = search->chunk_end;
+        }
+    }
+
+    /*
+    * The code below implements an SIMD-based algorithm to determine if N bytes at a time
+    * need to be escaped. 
+    * 
+    * Assume the ptr = "Te\sting!" (the double quotes are included in the string)
+    * 
+    * The explanation will be limited to the first 8 bytes of the string for simplicity. However
+    * the vector insructions may work on larger vectors.
+    * 
+    * First, we load three constants 'lower_bound', 'backslash' and 'dblquote" in vector registers.
+    * 
+    * lower_bound: [20 20 20 20 20 20 20 20] 
+    * backslash:   [5C 5C 5C 5C 5C 5C 5C 5C] 
+    * dblquote:    [22 22 22 22 22 22 22 22] 
+    * 
+    * Next we load the first chunk of the ptr: 
+    * [22 54 65 5C 73 74 69 6E] ("  T  e  \  s  t  i  n)
+    * 
+    * First we check if any byte in chunk is less than 32 (0x20). This returns the following vector
+    * as no bytes are less than 32 (0x20):
+    * [0 0 0 0 0 0 0 0]
+    * 
+    * Next, we check if any byte in chunk is equal to a backslash:
+    * [0 0 0 FF 0 0 0 0]
+    * 
+    * Finally we check if any byte in chunk is equal to a double quote:
+    * [FF 0 0 0 0 0 0 0] 
+    * 
+    * Now we have three vectors where each byte indicates if the corresponding byte in chunk
+    * needs to be escaped. We combine these vectors with a series of logical OR instructions.
+    * This is the needs_escape vector and it is equal to:
+    * [FF 0 0 FF 0 0 0 0] 
+    * 
+    * Next we compute the bitwise AND between each byte and 0x1 and compute the horizontal sum of
+    * the values in the vector. This computes how many bytes need to be escaped within this chunk.
+    * 
+    * Finally we compute a mask that indicates which bytes need to be escaped. If the mask is 0 then,
+    * no bytes need to be escaped and we can continue to the next chunk. If the mask is not 0 then we
+    * have at least one byte that needs to be escaped.
+    */
+    while (search->ptr + sizeof(uint8x16_t) <= search->end) {
+        uint64_t mask = neon_rules_update(search->ptr);
+
+        if (!mask) {
+            search->ptr += sizeof(uint8x16_t);
+            continue;
+        }
+        search->matches_mask = mask;
+        search->has_matches = true;
+        search->chunk_base = search->ptr;
+        search->chunk_end = search->ptr + sizeof(uint8x16_t);
+        return neon_next_match(search);
+    }
+
+    // There are fewer than 16 bytes left. 
+    unsigned long remaining = (search->end - search->ptr);
+    if (remaining >= SIMD_MINIMUM_THRESHOLD) {
+        char *s = copy_remaining_bytes(search, sizeof(uint8x16_t), remaining);
+
+        uint64_t mask = neon_rules_update(s);
+
+        if (!mask) {
+            // Nothing to escape, ensure search_flush doesn't do anything by setting 
+            // search->cursor to search->ptr.
+            search->buffer->len += remaining;
+            search->ptr = search->end;
+            search->cursor = search->end;
+            return 0;
         }
+
+        search->matches_mask = mask;
+        search->has_matches = true;
+        search->chunk_end = search->end;
+        search->chunk_base = search->ptr;
+        return neon_next_match(search);
     }
-#undef FLUSH_POS
 
-    if (beg < len) {
-        fbuffer_append(out_buffer, &ptr[beg], len - beg);
+    if (search->ptr < search->end) {
+        return search_escape_basic(search);
     }
 
-    RB_GC_GUARD(str);
+    search_flush(search);
+    return 0;
 }
+#endif /* HAVE_SIMD_NEON */
 
-static const char escape_table[256] = {
-    // ASCII Control Characters
-    1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
-    1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
-    // ASCII Characters
-    0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0, // '"'
-    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-    0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0, // '\\'
-    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-    // Continuation byte
-    1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
-    1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
-    1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
-    1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
-    // First byte of a 2-byte code point
-    2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
-    2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
-    // First byte of a 4-byte code point
-    3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,
-    //First byte of a 4+byte code point
-    4,4,4,4,4,4,4,4,5,5,5,5,6,6,1,1,
-};
+#ifdef HAVE_SIMD_SSE2
 
-static const char script_safe_escape_table[256] = {
-    // ASCII Control Characters
-    1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
-    1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
-    // ASCII Characters
-    0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1, // '"' and '/'
-    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-    0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0, // '\\'
-    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-    // Continuation byte
-    1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
-    1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
-    1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
-    1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
-    // First byte of a 2-byte code point
-    2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
-    2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
-    // First byte of a 4-byte code point
-    3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,
-    //First byte of a 4+byte code point
-    4,4,4,4,4,4,4,4,5,5,5,5,6,6,1,1,
-};
+#define _mm_cmpge_epu8(a, b) _mm_cmpeq_epi8(_mm_max_epu8(a, b), a)
+#define _mm_cmple_epu8(a, b) _mm_cmpge_epu8(b, a)
+#define _mm_cmpgt_epu8(a, b) _mm_xor_si128(_mm_cmple_epu8(a, b), _mm_set1_epi8(-1))
+#define _mm_cmplt_epu8(a, b) _mm_cmpgt_epu8(b, a)
 
-static void convert_ASCII_to_JSON(FBuffer *out_buffer, VALUE str, const char escape_table[256])
+static inline FORCE_INLINE unsigned char sse2_next_match(search_state *search)
 {
-    const char *hexdig = "0123456789abcdef";
-    char scratch[12] = { '\\', 'u', 0, 0, 0, 0, '\\', 'u' };
+    int mask = search->matches_mask;
+    int index = trailing_zeros(mask);
 
-    const char *ptr = RSTRING_PTR(str);
-    unsigned long len = RSTRING_LEN(str);
+    // It is assumed escape_UTF8_char_basic will only ever increase search->ptr by at most one character.
+    // If we want to use a similar approach for full escaping we'll need to ensure:
+    //     search->chunk_base + index >= search->ptr
+    // However, since we know escape_UTF8_char_basic only increases search->ptr by one, if the next match
+    // is one byte after the previous match then:
+    //     search->chunk_base + index == search->ptr
+    search->ptr = search->chunk_base + index;
+    mask &= mask - 1;
+    search->matches_mask = mask;
+    search_flush(search);
+    return 1;
+}
 
-    unsigned long beg = 0, pos;
+#if defined(__clang__) || defined(__GNUC__)
+#define TARGET_SSE2 __attribute__((target("sse2")))
+#else
+#define TARGET_SSE2
+#endif
 
-    for (pos = 0; pos < len;) {
-        unsigned char ch = ptr[pos];
-        /* JSON encoding */
-        if (escape_table[ch]) {
-            if (pos > beg) {
-                fbuffer_append(out_buffer, &ptr[beg], pos - beg);
-            }
+static inline TARGET_SSE2 FORCE_INLINE int sse2_update(const char *ptr)
+{
+    __m128i chunk         = _mm_loadu_si128((__m128i const*)ptr);
 
-            beg = pos + 1;
-            switch (ch) {
-                case '"':  fbuffer_append(out_buffer, "\\\"", 2); break;
-                case '\\': fbuffer_append(out_buffer, "\\\\", 2); break;
-                case '/':  fbuffer_append(out_buffer, "\\/", 2); break;
-                case '\b': fbuffer_append(out_buffer, "\\b", 2); break;
-                case '\f': fbuffer_append(out_buffer, "\\f", 2); break;
-                case '\n': fbuffer_append(out_buffer, "\\n", 2); break;
-                case '\r': fbuffer_append(out_buffer, "\\r", 2); break;
-                case '\t': fbuffer_append(out_buffer, "\\t", 2); break;
-                default:
-                    scratch[2] = hexdig[ch >> 12];
-                    scratch[3] = hexdig[(ch >> 8) & 0xf];
-                    scratch[4] = hexdig[(ch >> 4) & 0xf];
-                    scratch[5] = hexdig[ch & 0xf];
-                    fbuffer_append(out_buffer, scratch, 6);
+    // Trick: c < 32 || c == 34 can be factored as c ^ 2 < 33
+    // https://lemire.me/blog/2025/04/13/detect-control-characters-quotes-and-backslashes-efficiently-using-swar/
+    __m128i too_low_or_dbl_quote = _mm_cmplt_epu8(_mm_xor_si128(chunk, _mm_set1_epi8(2)), _mm_set1_epi8(33));
+    __m128i has_backslash = _mm_cmpeq_epi8(chunk, _mm_set1_epi8('\\'));
+    __m128i needs_escape  = _mm_or_si128(too_low_or_dbl_quote, has_backslash);
+    return _mm_movemask_epi8(needs_escape);
+}
+
+static inline TARGET_SSE2 FORCE_INLINE unsigned char search_escape_basic_sse2(search_state *search)
+{
+    if (RB_UNLIKELY(search->has_matches)) {
+        // There are more matches if search->matches_mask > 0.
+        if (search->matches_mask > 0) {
+            return sse2_next_match(search);
+        } else {
+            // sse2_next_match will only advance search->ptr up to the last matching character. 
+            // Skip over any characters in the last chunk that occur after the last match.
+            search->has_matches = false;
+            if (RB_UNLIKELY(search->chunk_base + sizeof(__m128i) >= search->end)) {
+                search->ptr = search->end;
+            } else {
+                search->ptr = search->chunk_base + sizeof(__m128i);
             }
         }
+    }
+
+    while (search->ptr + sizeof(__m128i) <= search->end) {
+        int needs_escape_mask = sse2_update(search->ptr);
+
+        if (needs_escape_mask == 0) {
+            search->ptr += sizeof(__m128i);
+            continue;
+        }
 
-        pos++;
+        search->has_matches = true;
+        search->matches_mask = needs_escape_mask;
+        search->chunk_base = search->ptr;
+        return sse2_next_match(search);
     }
 
-    if (beg < len) {
-        fbuffer_append(out_buffer, &ptr[beg], len - beg);
+    // There are fewer than 16 bytes left. 
+    unsigned long remaining = (search->end - search->ptr);
+    if (remaining >= SIMD_MINIMUM_THRESHOLD) {
+        char *s = copy_remaining_bytes(search, sizeof(__m128i), remaining);
+
+        int needs_escape_mask = sse2_update(s);
+
+        if (needs_escape_mask == 0) {
+            // Nothing to escape, ensure search_flush doesn't do anything by setting 
+            // search->cursor to search->ptr.
+            search->buffer->len += remaining;
+            search->ptr = search->end;
+            search->cursor = search->end;
+            return 0;
+        }
+
+        search->has_matches = true;
+        search->matches_mask = needs_escape_mask;
+        search->chunk_base = search->ptr;
+        return sse2_next_match(search);
     }
 
-    RB_GC_GUARD(str);
-}
+    if (search->ptr < search->end) {
+        return search_escape_basic(search);
+    }
 
-static void convert_UTF8_to_ASCII_only_JSON(FBuffer *out_buffer, VALUE str, const char escape_table[256], bool out_script_safe)
-{
-    const char *hexdig = "0123456789abcdef";
-    char scratch[12] = { '\\', 'u', 0, 0, 0, 0, '\\', 'u' };
+    search_flush(search);
+    return 0;
+}
 
-    const char *ptr = RSTRING_PTR(str);
-    unsigned long len = RSTRING_LEN(str);
+#endif /* HAVE_SIMD_SSE2 */
 
-    unsigned long beg = 0, pos = 0;
+#endif /* HAVE_SIMD */
 
-#define FLUSH_POS(bytes) if (pos > beg) { fbuffer_append(out_buffer, &ptr[beg], pos - beg); } pos += bytes; beg = pos;
+static const unsigned char script_safe_escape_table[256] = {
+    // ASCII Control Characters
+     9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
+     9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
+    // ASCII Characters
+     0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, // '"' and '/'
+     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, // '\\'
+     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    // Continuation byte
+     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+    // First byte of a 2-byte code point
+     2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+     2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+    // First byte of a 3-byte code point
+     3, 3,11, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, // 0xE2 is the start of \u2028 and \u2029
+    //First byte of a 4+ byte code point
+     4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 9, 9,
+};
 
-    while (pos < len) {
-        unsigned char ch = ptr[pos];
-        unsigned char ch_len = escape_table[ch];
+static inline unsigned char search_script_safe_escape(search_state *search)
+{
+    while (search->ptr < search->end) {
+        unsigned char ch = (unsigned char)*search->ptr;
+        unsigned char ch_len = script_safe_escape_table[ch];
 
         if (RB_UNLIKELY(ch_len)) {
-            switch (ch_len) {
-                case 0:
-                    pos++;
-                    break;
-                case 1: {
-                    FLUSH_POS(1);
-                    switch (ch) {
-                        case '"':  fbuffer_append(out_buffer, "\\\"", 2); break;
-                        case '\\': fbuffer_append(out_buffer, "\\\\", 2); break;
-                        case '/':  fbuffer_append(out_buffer, "\\/", 2); break;
-                        case '\b': fbuffer_append(out_buffer, "\\b", 2); break;
-                        case '\f': fbuffer_append(out_buffer, "\\f", 2); break;
-                        case '\n': fbuffer_append(out_buffer, "\\n", 2); break;
-                        case '\r': fbuffer_append(out_buffer, "\\r", 2); break;
-                        case '\t': fbuffer_append(out_buffer, "\\t", 2); break;
-                        default: {
-                            scratch[2] = hexdig[ch >> 12];
-                            scratch[3] = hexdig[(ch >> 8) & 0xf];
-                            scratch[4] = hexdig[(ch >> 4) & 0xf];
-                            scratch[5] = hexdig[ch & 0xf];
-                            fbuffer_append(out_buffer, scratch, 6);
-                            break;
-                        }
+            if (ch_len & ESCAPE_MASK) {
+                if (RB_UNLIKELY(ch_len == 11)) {
+                    const unsigned char *uptr = (const unsigned char *)search->ptr;
+                    if (!(uptr[1] == 0x80 && (uptr[2] >> 1) == 0x54)) {
+                        search->ptr += 3;
+                        continue;
                     }
-                    break;
                 }
-                default: {
-                    uint32_t wchar = 0;
-                    switch(ch_len) {
-                        case 2:
-                            wchar = ptr[pos] & 0x1F;
-                            break;
-                        case 3:
-                            wchar = ptr[pos] & 0x0F;
-                            break;
-                        case 4:
-                            wchar = ptr[pos] & 0x07;
-                            break;
-                    }
+                search_flush(search);
+                return ch_len & CHAR_LENGTH_MASK;
+            } else {
+                search->ptr += ch_len;
+            }
+        } else {
+            search->ptr++;
+        }
+    }
+    search_flush(search);
+    return 0;
+}
 
-                    for (short i = 1; i < ch_len; i++) {
-                        wchar = (wchar << 6) | (ptr[pos+i] & 0x3F);
-                    }
+static void convert_UTF8_to_script_safe_JSON(search_state *search)
+{
+    unsigned char ch_len;
+    while ((ch_len = search_script_safe_escape(search))) {
+        escape_UTF8_char(search, ch_len);
+    }
+}
 
-                    FLUSH_POS(ch_len);
+static const unsigned char ascii_only_escape_table[256] = {
+    // ASCII Control Characters
+     9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
+     9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
+    // ASCII Characters
+     0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // '"'
+     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, // '\\'
+     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    // Continuation byte
+     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+    // First byte of a  2-byte code point
+     2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+     2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+    // First byte of a 3-byte code point
+     3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
+    //First byte of a 4+ byte code point
+     4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 9, 9,
+};
 
-                    if (wchar <= 0xFFFF) {
-                        scratch[2] = hexdig[wchar >> 12];
-                        scratch[3] = hexdig[(wchar >> 8) & 0xf];
-                        scratch[4] = hexdig[(wchar >> 4) & 0xf];
-                        scratch[5] = hexdig[wchar & 0xf];
-                        fbuffer_append(out_buffer, scratch, 6);
-                    } else {
-                        uint16_t hi, lo;
-                        wchar -= 0x10000;
-                        hi = 0xD800 + (uint16_t)(wchar >> 10);
-                        lo = 0xDC00 + (uint16_t)(wchar & 0x3FF);
-
-                        scratch[2] = hexdig[hi >> 12];
-                        scratch[3] = hexdig[(hi >> 8) & 0xf];
-                        scratch[4] = hexdig[(hi >> 4) & 0xf];
-                        scratch[5] = hexdig[hi & 0xf];
-
-                        scratch[8] = hexdig[lo >> 12];
-                        scratch[9] = hexdig[(lo >> 8) & 0xf];
-                        scratch[10] = hexdig[(lo >> 4) & 0xf];
-                        scratch[11] = hexdig[lo & 0xf];
-
-                        fbuffer_append(out_buffer, scratch, 12);
-                    }
+static inline unsigned char search_ascii_only_escape(search_state *search, const unsigned char escape_table[256])
+{
+    while (search->ptr < search->end) {
+        unsigned char ch = (unsigned char)*search->ptr;
+        unsigned char ch_len = escape_table[ch];
 
+        if (RB_UNLIKELY(ch_len)) {
+            search_flush(search);
+            return ch_len & CHAR_LENGTH_MASK;
+        } else {
+            search->ptr++;
+        }
+    }
+    search_flush(search);
+    return 0;
+}
+
+static inline void full_escape_UTF8_char(search_state *search, unsigned char ch_len) {
+    const unsigned char ch = (unsigned char)*search->ptr;
+    switch (ch_len) {
+        case 1: {
+            switch (ch) {
+                case '"':  fbuffer_append(search->buffer, "\\\"", 2); break;
+                case '\\': fbuffer_append(search->buffer, "\\\\", 2); break;
+                case '/':  fbuffer_append(search->buffer, "\\/", 2);  break;
+                case '\b': fbuffer_append(search->buffer, "\\b", 2);  break;
+                case '\f': fbuffer_append(search->buffer, "\\f", 2);  break;
+                case '\n': fbuffer_append(search->buffer, "\\n", 2);  break;
+                case '\r': fbuffer_append(search->buffer, "\\r", 2);  break;
+                case '\t': fbuffer_append(search->buffer, "\\t", 2);  break;
+                default: {
+                    const char *hexdig = "0123456789abcdef";
+                    char scratch[6] = { '\\', 'u', '0', '0', 0, 0 };
+                    scratch[4] = hexdig[(ch >> 4) & 0xf];
+                    scratch[5] = hexdig[ch & 0xf];
+                    fbuffer_append(search->buffer, scratch, 6);
                     break;
                 }
             }
-        } else {
-            pos++;
+            break;
         }
-    }
-#undef FLUSH_POS
+        default: {
+            const char *hexdig = "0123456789abcdef";
+            char scratch[12] = { '\\', 'u', 0, 0, 0, 0, '\\', 'u' };
 
-    if (beg < len) {
-        fbuffer_append(out_buffer, &ptr[beg], len - beg);
-    }
+            uint32_t wchar = 0;
+
+            switch(ch_len) {
+                case 2:
+                    wchar = ch & 0x1F;
+                    break;
+                case 3:
+                    wchar = ch & 0x0F;
+                    break;
+                case 4:
+                    wchar = ch & 0x07;
+                    break;
+            }
+
+            for (short i = 1; i < ch_len; i++) {
+                wchar = (wchar << 6) | (search->ptr[i] & 0x3F);
+            }
+
+            if (wchar <= 0xFFFF) {
+                scratch[2] = hexdig[wchar >> 12];
+                scratch[3] = hexdig[(wchar >> 8) & 0xf];
+                scratch[4] = hexdig[(wchar >> 4) & 0xf];
+                scratch[5] = hexdig[wchar & 0xf];
+                fbuffer_append(search->buffer, scratch, 6);
+            } else {
+                uint16_t hi, lo;
+                wchar -= 0x10000;
+                hi = 0xD800 + (uint16_t)(wchar >> 10);
+                lo = 0xDC00 + (uint16_t)(wchar & 0x3FF);
+
+                scratch[2] = hexdig[hi >> 12];
+                scratch[3] = hexdig[(hi >> 8) & 0xf];
+                scratch[4] = hexdig[(hi >> 4) & 0xf];
+                scratch[5] = hexdig[hi & 0xf];
+
+                scratch[8] = hexdig[lo >> 12];
+                scratch[9] = hexdig[(lo >> 8) & 0xf];
+                scratch[10] = hexdig[(lo >> 4) & 0xf];
+                scratch[11] = hexdig[lo & 0xf];
+
+                fbuffer_append(search->buffer, scratch, 12);
+            }
 
-    RB_GC_GUARD(str);
+            break;
+        }
+    }
+    search->cursor = (search->ptr += ch_len);
 }
 
-static char *fstrndup(const char *ptr, unsigned long len) {
-  char *result;
-  if (len <= 0) return NULL;
-  result = ALLOC_N(char, len);
-  memcpy(result, ptr, len);
-  return result;
+static void convert_UTF8_to_ASCII_only_JSON(search_state *search, const unsigned char escape_table[256])
+{
+    unsigned char ch_len;
+    while ((ch_len = search_ascii_only_escape(search, escape_table))) {
+        full_escape_UTF8_char(search, ch_len);
+    }
 }
 
 /*
@@ -403,7 +815,9 @@ static char *fstrndup(const char *ptr, unsigned long len) {
  */
 static VALUE mHash_to_json(int argc, VALUE *argv, VALUE self)
 {
-    GENERATE_JSON(object);
+    rb_check_arity(argc, 0, 1);
+    VALUE Vstate = cState_from_state_s(cState, argc == 1 ? argv[0] : Qnil);
+    return cState_partial_generate(Vstate, self, generate_json_object, Qfalse);
 }
 
 /*
@@ -415,7 +829,9 @@ static VALUE mHash_to_json(int argc, VALUE *argv, VALUE self)
  * produced JSON string output further.
  */
 static VALUE mArray_to_json(int argc, VALUE *argv, VALUE self) {
-    GENERATE_JSON(array);
+    rb_check_arity(argc, 0, 1);
+    VALUE Vstate = cState_from_state_s(cState, argc == 1 ? argv[0] : Qnil);
+    return cState_partial_generate(Vstate, self, generate_json_array, Qfalse);
 }
 
 #ifdef RUBY_INTEGER_UNIFICATION
@@ -426,7 +842,9 @@ static VALUE mArray_to_json(int argc, VALUE *argv, VALUE self) {
  */
 static VALUE mInteger_to_json(int argc, VALUE *argv, VALUE self)
 {
-    GENERATE_JSON(integer);
+    rb_check_arity(argc, 0, 1);
+    VALUE Vstate = cState_from_state_s(cState, argc == 1 ? argv[0] : Qnil);
+    return cState_partial_generate(Vstate, self, generate_json_integer, Qfalse);
 }
 
 #else
@@ -437,7 +855,9 @@ static VALUE mInteger_to_json(int argc, VALUE *argv, VALUE self)
  */
 static VALUE mFixnum_to_json(int argc, VALUE *argv, VALUE self)
 {
-    GENERATE_JSON(fixnum);
+    rb_check_arity(argc, 0, 1);
+    VALUE Vstate = cState_from_state_s(cState, argc == 1 ? argv[0] : Qnil);
+    return cState_partial_generate(Vstate, self, generate_json_fixnum, Qfalse);
 }
 
 /*
@@ -447,7 +867,9 @@ static VALUE mFixnum_to_json(int argc, VALUE *argv, VALUE self)
  */
 static VALUE mBignum_to_json(int argc, VALUE *argv, VALUE self)
 {
-    GENERATE_JSON(bignum);
+    rb_check_arity(argc, 0, 1);
+    VALUE Vstate = cState_from_state_s(cState, argc == 1 ? argv[0] : Qnil);
+    return cState_partial_generate(Vstate, self, generate_json_bignum, Qfalse);
 }
 #endif
 
@@ -458,7 +880,9 @@ static VALUE mBignum_to_json(int argc, VALUE *argv, VALUE self)
  */
 static VALUE mFloat_to_json(int argc, VALUE *argv, VALUE self)
 {
-    GENERATE_JSON(float);
+    rb_check_arity(argc, 0, 1);
+    VALUE Vstate = cState_from_state_s(cState, argc == 1 ? argv[0] : Qnil);
+    return cState_partial_generate(Vstate, self, generate_json_float, Qfalse);
 }
 
 /*
@@ -481,7 +905,9 @@ static VALUE mString_included_s(VALUE self, VALUE modul) {
  */
 static VALUE mString_to_json(int argc, VALUE *argv, VALUE self)
 {
-    GENERATE_JSON(string);
+    rb_check_arity(argc, 0, 1);
+    VALUE Vstate = cState_from_state_s(cState, argc == 1 ? argv[0] : Qnil);
+    return cState_partial_generate(Vstate, self, generate_json_string, Qfalse);
 }
 
 /*
@@ -498,7 +924,7 @@ static VALUE mString_to_json_raw_object(VALUE self)
     VALUE result = rb_hash_new();
     rb_hash_aset(result, rb_funcall(mJSON, i_create_id, 0), rb_class_name(rb_obj_class(self)));
     ary = rb_funcall(self, i_unpack, 1, rb_str_new2("C*"));
-    rb_hash_aset(result, rb_str_new2("raw"), ary);
+    rb_hash_aset(result, rb_utf8_str_new_lit("raw"), ary);
     return result;
 }
 
@@ -536,7 +962,8 @@ static VALUE mString_Extend_json_create(VALUE self, VALUE o)
  */
 static VALUE mTrueClass_to_json(int argc, VALUE *argv, VALUE self)
 {
-    GENERATE_JSON(true);
+    rb_check_arity(argc, 0, 1);
+    return rb_utf8_str_new("true", 4);
 }
 
 /*
@@ -546,7 +973,8 @@ static VALUE mTrueClass_to_json(int argc, VALUE *argv, VALUE self)
  */
 static VALUE mFalseClass_to_json(int argc, VALUE *argv, VALUE self)
 {
-    GENERATE_JSON(false);
+    rb_check_arity(argc, 0, 1);
+    return rb_utf8_str_new("false", 5);
 }
 
 /*
@@ -556,7 +984,8 @@ static VALUE mFalseClass_to_json(int argc, VALUE *argv, VALUE self)
  */
 static VALUE mNilClass_to_json(int argc, VALUE *argv, VALUE self)
 {
-    GENERATE_JSON(null);
+    rb_check_arity(argc, 0, 1);
+    return rb_utf8_str_new("null", 4);
 }
 
 /*
@@ -573,30 +1002,40 @@ static VALUE mObject_to_json(int argc, VALUE *argv, VALUE self)
     rb_scan_args(argc, argv, "01", &state);
     Check_Type(string, T_STRING);
     state = cState_from_state_s(cState, state);
-    return cState_partial_generate(state, string);
+    return cState_partial_generate(state, string, generate_json_string, Qfalse);
+}
+
+static void State_mark(void *ptr)
+{
+    JSON_Generator_State *state = ptr;
+    rb_gc_mark_movable(state->indent);
+    rb_gc_mark_movable(state->space);
+    rb_gc_mark_movable(state->space_before);
+    rb_gc_mark_movable(state->object_nl);
+    rb_gc_mark_movable(state->array_nl);
+    rb_gc_mark_movable(state->as_json);
+}
+
+static void State_compact(void *ptr)
+{
+    JSON_Generator_State *state = ptr;
+    state->indent = rb_gc_location(state->indent);
+    state->space = rb_gc_location(state->space);
+    state->space_before = rb_gc_location(state->space_before);
+    state->object_nl = rb_gc_location(state->object_nl);
+    state->array_nl = rb_gc_location(state->array_nl);
+    state->as_json = rb_gc_location(state->as_json);
 }
 
 static void State_free(void *ptr)
 {
     JSON_Generator_State *state = ptr;
-    if (state->indent) ruby_xfree(state->indent);
-    if (state->space) ruby_xfree(state->space);
-    if (state->space_before) ruby_xfree(state->space_before);
-    if (state->object_nl) ruby_xfree(state->object_nl);
-    if (state->array_nl) ruby_xfree(state->array_nl);
     ruby_xfree(state);
 }
 
 static size_t State_memsize(const void *ptr)
 {
-    const JSON_Generator_State *state = ptr;
-    size_t size = sizeof(*state);
-    if (state->indent) size += state->indent_len + 1;
-    if (state->space) size += state->space_len + 1;
-    if (state->space_before) size += state->space_before_len + 1;
-    if (state->object_nl) size += state->object_nl_len + 1;
-    if (state->array_nl) size += state->array_nl_len + 1;
-    return size;
+    return sizeof(JSON_Generator_State);
 }
 
 #ifndef HAVE_RB_EXT_RACTOR_SAFE
@@ -606,52 +1045,103 @@ static size_t State_memsize(const void *ptr)
 
 static const rb_data_type_t JSON_Generator_State_type = {
     "JSON/Generator/State",
-    {NULL, State_free, State_memsize,},
+    {
+        .dmark = State_mark,
+        .dfree = State_free,
+        .dsize = State_memsize,
+        .dcompact = State_compact,
+    },
     0, 0,
     RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_FROZEN_SHAREABLE,
 };
 
+static void state_init(JSON_Generator_State *state)
+{
+    state->max_nesting = 100;
+    state->buffer_initial_length = FBUFFER_INITIAL_LENGTH_DEFAULT;
+}
+
 static VALUE cState_s_allocate(VALUE klass)
 {
     JSON_Generator_State *state;
     VALUE obj = TypedData_Make_Struct(klass, JSON_Generator_State, &JSON_Generator_State_type, state);
-    state->max_nesting = 100;
-    state->buffer_initial_length = FBUFFER_INITIAL_LENGTH_DEFAULT;
+    state_init(state);
     return obj;
 }
 
+static void vstate_spill(struct generate_json_data *data)
+{
+    VALUE vstate = cState_s_allocate(cState);
+    GET_STATE(vstate);
+    MEMCPY(state, data->state, JSON_Generator_State, 1);
+    data->state = state;
+    data->vstate = vstate;
+    RB_OBJ_WRITTEN(vstate, Qundef, state->indent);
+    RB_OBJ_WRITTEN(vstate, Qundef, state->space);
+    RB_OBJ_WRITTEN(vstate, Qundef, state->space_before);
+    RB_OBJ_WRITTEN(vstate, Qundef, state->object_nl);
+    RB_OBJ_WRITTEN(vstate, Qundef, state->array_nl);
+    RB_OBJ_WRITTEN(vstate, Qundef, state->as_json);
+}
+
+static inline VALUE vstate_get(struct generate_json_data *data)
+{
+    if (RB_UNLIKELY(!data->vstate)) {
+        vstate_spill(data);
+    }
+    return data->vstate;
+}
+
 struct hash_foreach_arg {
-    FBuffer *buffer;
-    JSON_Generator_State *state;
-    VALUE Vstate;
+    struct generate_json_data *data;
     int iter;
 };
 
+static VALUE
+convert_string_subclass(VALUE key)
+{
+    VALUE key_to_s = rb_funcall(key, i_to_s, 0);
+
+    if (RB_UNLIKELY(!RB_TYPE_P(key_to_s, T_STRING))) {
+        VALUE cname = rb_obj_class(key);
+        rb_raise(rb_eTypeError,
+                 "can't convert %"PRIsVALUE" to %s (%"PRIsVALUE"#%s gives %"PRIsVALUE")",
+                 cname, "String", cname, "to_s", rb_obj_class(key_to_s));
+    }
+
+    return key_to_s;
+}
+
 static int
 json_object_i(VALUE key, VALUE val, VALUE _arg)
 {
     struct hash_foreach_arg *arg = (struct hash_foreach_arg *)_arg;
-    FBuffer *buffer = arg->buffer;
-    JSON_Generator_State *state = arg->state;
-    VALUE Vstate = arg->Vstate;
+    struct generate_json_data *data = arg->data;
+
+    FBuffer *buffer = data->buffer;
+    JSON_Generator_State *state = data->state;
 
     long depth = state->depth;
     int j;
 
     if (arg->iter > 0) fbuffer_append_char(buffer, ',');
-    if (RB_UNLIKELY(state->object_nl)) {
-        fbuffer_append(buffer, state->object_nl, state->object_nl_len);
+    if (RB_UNLIKELY(data->state->object_nl)) {
+        fbuffer_append_str(buffer, data->state->object_nl);
     }
-    if (RB_UNLIKELY(state->indent)) {
+    if (RB_UNLIKELY(data->state->indent)) {
         for (j = 0; j < depth; j++) {
-            fbuffer_append(buffer, state->indent, state->indent_len);
+            fbuffer_append_str(buffer, data->state->indent);
         }
     }
 
     VALUE key_to_s;
     switch(rb_type(key)) {
         case T_STRING:
-            key_to_s = key;
+            if (RB_LIKELY(RBASIC_CLASS(key) == rb_cString)) {
+                key_to_s = key;
+            } else {
+                key_to_s = convert_string_subclass(key);
+            }
             break;
         case T_SYMBOL:
             key_to_s = rb_sym2str(key);
@@ -661,82 +1151,98 @@ json_object_i(VALUE key, VALUE val, VALUE _arg)
             break;
     }
 
-    generate_json_string(buffer, Vstate, state, key_to_s);
-    if (RB_UNLIKELY(state->space_before)) fbuffer_append(buffer, state->space_before, state->space_before_len);
+    if (RB_LIKELY(RBASIC_CLASS(key_to_s) == rb_cString)) {
+        generate_json_string(buffer, data, key_to_s);
+    } else {
+        generate_json(buffer, data, key_to_s);
+    }
+    if (RB_UNLIKELY(state->space_before)) fbuffer_append_str(buffer, data->state->space_before);
     fbuffer_append_char(buffer, ':');
-    if (RB_UNLIKELY(state->space)) fbuffer_append(buffer, state->space, state->space_len);
-    generate_json(buffer, Vstate, state, val);
+    if (RB_UNLIKELY(state->space)) fbuffer_append_str(buffer, data->state->space);
+    generate_json(buffer, data, val);
 
     arg->iter++;
     return ST_CONTINUE;
 }
 
-static void generate_json_object(FBuffer *buffer, VALUE Vstate, JSON_Generator_State *state, VALUE obj)
+static inline long increase_depth(struct generate_json_data *data)
 {
-    long max_nesting = state->max_nesting;
+    JSON_Generator_State *state = data->state;
     long depth = ++state->depth;
+    if (RB_UNLIKELY(depth > state->max_nesting && state->max_nesting)) {
+        rb_raise(eNestingError, "nesting of %ld is too deep", --state->depth);
+    }
+    return depth;
+}
+
+static void generate_json_object(FBuffer *buffer, struct generate_json_data *data, VALUE obj)
+{
     int j;
-    struct hash_foreach_arg arg;
+    long depth = increase_depth(data);
 
-    if (max_nesting != 0 && depth > max_nesting) {
-        rb_raise(eNestingError, "nesting of %ld is too deep", --state->depth);
+    if (RHASH_SIZE(obj) == 0) {
+        fbuffer_append(buffer, "{}", 2);
+        --data->state->depth;
+        return;
     }
+
     fbuffer_append_char(buffer, '{');
 
-    arg.buffer = buffer;
-    arg.state = state;
-    arg.Vstate = Vstate;
-    arg.iter = 0;
+    struct hash_foreach_arg arg = {
+        .data = data,
+        .iter = 0,
+    };
     rb_hash_foreach(obj, json_object_i, (VALUE)&arg);
 
-    depth = --state->depth;
-    if (RB_UNLIKELY(state->object_nl)) {
-        fbuffer_append(buffer, state->object_nl, state->object_nl_len);
-        if (RB_UNLIKELY(state->indent)) {
+    depth = --data->state->depth;
+    if (RB_UNLIKELY(data->state->object_nl)) {
+        fbuffer_append_str(buffer, data->state->object_nl);
+        if (RB_UNLIKELY(data->state->indent)) {
             for (j = 0; j < depth; j++) {
-                fbuffer_append(buffer, state->indent, state->indent_len);
+                fbuffer_append_str(buffer, data->state->indent);
             }
         }
     }
     fbuffer_append_char(buffer, '}');
 }
 
-static void generate_json_array(FBuffer *buffer, VALUE Vstate, JSON_Generator_State *state, VALUE obj)
+static void generate_json_array(FBuffer *buffer, struct generate_json_data *data, VALUE obj)
 {
-    long max_nesting = state->max_nesting;
-    long depth = ++state->depth;
     int i, j;
-    if (max_nesting != 0 && depth > max_nesting) {
-        rb_raise(eNestingError, "nesting of %ld is too deep", --state->depth);
+    long depth = increase_depth(data);
+
+    if (RARRAY_LEN(obj) == 0) {
+        fbuffer_append(buffer, "[]", 2);
+        --data->state->depth;
+        return;
     }
+
     fbuffer_append_char(buffer, '[');
-    if (RB_UNLIKELY(state->array_nl)) fbuffer_append(buffer, state->array_nl, state->array_nl_len);
+    if (RB_UNLIKELY(data->state->array_nl)) fbuffer_append_str(buffer, data->state->array_nl);
     for(i = 0; i < RARRAY_LEN(obj); i++) {
         if (i > 0) {
             fbuffer_append_char(buffer, ',');
-            if (RB_UNLIKELY(state->array_nl)) fbuffer_append(buffer, state->array_nl, state->array_nl_len);
+            if (RB_UNLIKELY(data->state->array_nl)) fbuffer_append_str(buffer, data->state->array_nl);
         }
-        if (RB_UNLIKELY(state->indent)) {
+        if (RB_UNLIKELY(data->state->indent)) {
             for (j = 0; j < depth; j++) {
-                fbuffer_append(buffer, state->indent, state->indent_len);
+                fbuffer_append_str(buffer, data->state->indent);
             }
         }
-        generate_json(buffer, Vstate, state, RARRAY_AREF(obj, i));
+        generate_json(buffer, data, RARRAY_AREF(obj, i));
     }
-    state->depth = --depth;
-    if (RB_UNLIKELY(state->array_nl)) {
-        fbuffer_append(buffer, state->array_nl, state->array_nl_len);
-        if (RB_UNLIKELY(state->indent)) {
+    data->state->depth = --depth;
+    if (RB_UNLIKELY(data->state->array_nl)) {
+        fbuffer_append_str(buffer, data->state->array_nl);
+        if (RB_UNLIKELY(data->state->indent)) {
             for (j = 0; j < depth; j++) {
-                fbuffer_append(buffer, state->indent, state->indent_len);
+                fbuffer_append_str(buffer, data->state->indent);
             }
         }
     }
     fbuffer_append_char(buffer, ']');
 }
 
-static int usascii_encindex, utf8_encindex, binary_encindex;
-
 static inline int enc_utf8_compatible_p(int enc_idx)
 {
     if (enc_idx == usascii_encindex) return 1;
@@ -744,117 +1250,199 @@ static inline int enc_utf8_compatible_p(int enc_idx)
     return 0;
 }
 
+static VALUE encode_json_string_try(VALUE str)
+{
+    return rb_funcall(str, i_encode, 1, Encoding_UTF_8);
+}
+
+static VALUE encode_json_string_rescue(VALUE str, VALUE exception)
+{
+    raise_generator_error_str(str, rb_funcall(exception, rb_intern("message"), 0));
+    return Qundef;
+}
+
 static inline VALUE ensure_valid_encoding(VALUE str)
 {
     int encindex = RB_ENCODING_GET(str);
     VALUE utf8_string;
     if (RB_UNLIKELY(!enc_utf8_compatible_p(encindex))) {
         if (encindex == binary_encindex) {
-            // For historical reason, we silently reinterpret binary strings as UTF-8 if it would work.
-            // TODO: Deprecate in 2.8.0
-            // TODO: Remove in 3.0.0
             utf8_string = rb_enc_associate_index(rb_str_dup(str), utf8_encindex);
             switch (rb_enc_str_coderange(utf8_string)) {
                 case ENC_CODERANGE_7BIT:
+                    return utf8_string;
                 case ENC_CODERANGE_VALID:
+                    // For historical reason, we silently reinterpret binary strings as UTF-8 if it would work.
+                    // TODO: Raise in 3.0.0
+                    rb_warn("JSON.generate: UTF-8 string passed as BINARY, this will raise an encoding error in json 3.0");
                     return utf8_string;
                     break;
             }
         }
 
-        str = rb_funcall(str, i_encode, 1, Encoding_UTF_8);
+        str = rb_rescue(encode_json_string_try, str, encode_json_string_rescue, str);
     }
     return str;
 }
 
-static void generate_json_string(FBuffer *buffer, VALUE Vstate, JSON_Generator_State *state, VALUE obj)
+static void generate_json_string(FBuffer *buffer, struct generate_json_data *data, VALUE obj)
 {
     obj = ensure_valid_encoding(obj);
 
     fbuffer_append_char(buffer, '"');
 
+    long len;
+    search_state search;
+    search.buffer = buffer;
+    RSTRING_GETMEM(obj, search.ptr, len);
+    search.cursor = search.ptr;
+    search.end = search.ptr + len;
+
+#ifdef HAVE_SIMD
+    search.matches_mask = 0;
+    search.has_matches = false;
+    search.chunk_base = NULL;
+#endif /* HAVE_SIMD */
+
     switch(rb_enc_str_coderange(obj)) {
         case ENC_CODERANGE_7BIT:
-            convert_ASCII_to_JSON(buffer, obj, state->script_safe ? script_safe_escape_table : escape_table);
-            break;
         case ENC_CODERANGE_VALID:
-            if (RB_UNLIKELY(state->ascii_only)) {
-                convert_UTF8_to_ASCII_only_JSON(buffer, obj, state->script_safe ? script_safe_escape_table : escape_table, state->script_safe);
+            if (RB_UNLIKELY(data->state->ascii_only)) {
+                convert_UTF8_to_ASCII_only_JSON(&search, data->state->script_safe ? script_safe_escape_table : ascii_only_escape_table);
+            } else if (RB_UNLIKELY(data->state->script_safe)) {
+                convert_UTF8_to_script_safe_JSON(&search);
             } else {
-                convert_UTF8_to_JSON(buffer, obj, state->script_safe ? script_safe_escape_table : escape_table, state->script_safe);
+                convert_UTF8_to_JSON(&search);
             }
             break;
         default:
-            rb_raise(rb_path2class("JSON::GeneratorError"), "source sequence is illegal/malformed utf-8");
+            raise_generator_error(obj, "source sequence is illegal/malformed utf-8");
             break;
     }
     fbuffer_append_char(buffer, '"');
 }
 
-static void generate_json_null(FBuffer *buffer, VALUE Vstate, JSON_Generator_State *state, VALUE obj)
+static void generate_json_fallback(FBuffer *buffer, struct generate_json_data *data, VALUE obj)
+{
+    VALUE tmp;
+    if (rb_respond_to(obj, i_to_json)) {
+        tmp = rb_funcall(obj, i_to_json, 1, vstate_get(data));
+        Check_Type(tmp, T_STRING);
+        fbuffer_append_str(buffer, tmp);
+    } else {
+        tmp = rb_funcall(obj, i_to_s, 0);
+        Check_Type(tmp, T_STRING);
+        generate_json_string(buffer, data, tmp);
+    }
+}
+
+static inline void generate_json_symbol(FBuffer *buffer, struct generate_json_data *data, VALUE obj)
+{
+    if (data->state->strict) {
+        generate_json_string(buffer, data, rb_sym2str(obj));
+    } else {
+        generate_json_fallback(buffer, data, obj);
+    }
+}
+
+static void generate_json_null(FBuffer *buffer, struct generate_json_data *data, VALUE obj)
 {
     fbuffer_append(buffer, "null", 4);
 }
 
-static void generate_json_false(FBuffer *buffer, VALUE Vstate, JSON_Generator_State *state, VALUE obj)
+static void generate_json_false(FBuffer *buffer, struct generate_json_data *data, VALUE obj)
 {
     fbuffer_append(buffer, "false", 5);
 }
 
-static void generate_json_true(FBuffer *buffer, VALUE Vstate, JSON_Generator_State *state, VALUE obj)
+static void generate_json_true(FBuffer *buffer, struct generate_json_data *data, VALUE obj)
 {
     fbuffer_append(buffer, "true", 4);
 }
 
-static void generate_json_fixnum(FBuffer *buffer, VALUE Vstate, JSON_Generator_State *state, VALUE obj)
+static void generate_json_fixnum(FBuffer *buffer, struct generate_json_data *data, VALUE obj)
 {
     fbuffer_append_long(buffer, FIX2LONG(obj));
 }
 
-static void generate_json_bignum(FBuffer *buffer, VALUE Vstate, JSON_Generator_State *state, VALUE obj)
+static void generate_json_bignum(FBuffer *buffer, struct generate_json_data *data, VALUE obj)
 {
     VALUE tmp = rb_funcall(obj, i_to_s, 0);
     fbuffer_append_str(buffer, tmp);
 }
 
 #ifdef RUBY_INTEGER_UNIFICATION
-static void generate_json_integer(FBuffer *buffer, VALUE Vstate, JSON_Generator_State *state, VALUE obj)
+static void generate_json_integer(FBuffer *buffer, struct generate_json_data *data, VALUE obj)
 {
     if (FIXNUM_P(obj))
-        generate_json_fixnum(buffer, Vstate, state, obj);
+        generate_json_fixnum(buffer, data, obj);
     else
-        generate_json_bignum(buffer, Vstate, state, obj);
+        generate_json_bignum(buffer, data, obj);
 }
 #endif
-static void generate_json_float(FBuffer *buffer, VALUE Vstate, JSON_Generator_State *state, VALUE obj)
+
+static void generate_json_float(FBuffer *buffer, struct generate_json_data *data, VALUE obj)
 {
     double value = RFLOAT_VALUE(obj);
-    char allow_nan = state->allow_nan;
-    VALUE tmp = rb_funcall(obj, i_to_s, 0);
-    if (!allow_nan) {
-        if (isinf(value)) {
-            rb_raise(eGeneratorError, "%"PRIsVALUE" not allowed in JSON", tmp);
-        } else if (isnan(value)) {
-            rb_raise(eGeneratorError, "%"PRIsVALUE" not allowed in JSON", tmp);
+    char allow_nan = data->state->allow_nan;
+    if (isinf(value) || isnan(value)) {
+        /* for NaN and Infinity values we either raise an error or rely on Float#to_s. */
+        if (!allow_nan) {
+            if (data->state->strict && data->state->as_json) {
+                VALUE casted_obj = rb_proc_call_with_block(data->state->as_json, 1, &obj, Qnil);
+                if (casted_obj != obj) {
+                    increase_depth(data);
+                    generate_json(buffer, data, casted_obj);
+                    data->state->depth--;
+                    return;
+                }
+            }
+            raise_generator_error(obj, "%"PRIsVALUE" not allowed in JSON", rb_funcall(obj, i_to_s, 0));
         }
+
+        VALUE tmp = rb_funcall(obj, i_to_s, 0);
+        fbuffer_append_str(buffer, tmp);
+        return;
     }
-    fbuffer_append_str(buffer, tmp);
+
+    /* This implementation writes directly into the buffer. We reserve
+     * the 24 characters that fpconv_dtoa states as its maximum, plus
+     * 2 more characters for the potential ".0" suffix.
+     */
+    fbuffer_inc_capa(buffer, 26);
+    char* d = buffer->ptr + buffer->len;
+    int len = fpconv_dtoa(value, d);
+
+    /* fpconv_dtoa converts a float to its shortest string representation,
+     * but it adds a ".0" if this is a plain integer.
+     */
+    buffer->len += len;
 }
 
-static void generate_json(FBuffer *buffer, VALUE Vstate, JSON_Generator_State *state, VALUE obj)
+static void generate_json_fragment(FBuffer *buffer, struct generate_json_data *data, VALUE obj)
 {
-    VALUE tmp;
+    VALUE fragment = RSTRUCT_GET(obj, 0);
+    Check_Type(fragment, T_STRING);
+    fbuffer_append_str(buffer, fragment);
+}
+
+static void generate_json(FBuffer *buffer, struct generate_json_data *data, VALUE obj)
+{
+    bool as_json_called = false;
+start:
     if (obj == Qnil) {
-        generate_json_null(buffer, Vstate, state, obj);
+        generate_json_null(buffer, data, obj);
     } else if (obj == Qfalse) {
-        generate_json_false(buffer, Vstate, state, obj);
+        generate_json_false(buffer, data, obj);
     } else if (obj == Qtrue) {
-        generate_json_true(buffer, Vstate, state, obj);
+        generate_json_true(buffer, data, obj);
     } else if (RB_SPECIAL_CONST_P(obj)) {
         if (RB_FIXNUM_P(obj)) {
-            generate_json_fixnum(buffer, Vstate, state, obj);
+            generate_json_fixnum(buffer, data, obj);
         } else if (RB_FLONUM_P(obj)) {
-            generate_json_float(buffer, Vstate, state, obj);
+            generate_json_float(buffer, data, obj);
+        } else if (RB_STATIC_SYM_P(obj)) {
+            generate_json_symbol(buffer, data, obj);
         } else {
             goto general;
         }
@@ -862,62 +1450,53 @@ static void generate_json(FBuffer *buffer, VALUE Vstate, JSON_Generator_State *s
         VALUE klass = RBASIC_CLASS(obj);
         switch (RB_BUILTIN_TYPE(obj)) {
             case T_BIGNUM:
-                generate_json_bignum(buffer, Vstate, state, obj);
+                generate_json_bignum(buffer, data, obj);
                 break;
             case T_HASH:
                 if (klass != rb_cHash) goto general;
-                generate_json_object(buffer, Vstate, state, obj);
+                generate_json_object(buffer, data, obj);
                 break;
             case T_ARRAY:
                 if (klass != rb_cArray) goto general;
-                generate_json_array(buffer, Vstate, state, obj);
+                generate_json_array(buffer, data, obj);
                 break;
             case T_STRING:
                 if (klass != rb_cString) goto general;
-                generate_json_string(buffer, Vstate, state, obj);
+                generate_json_string(buffer, data, obj);
+                break;
+            case T_SYMBOL:
+                generate_json_symbol(buffer, data, obj);
                 break;
             case T_FLOAT:
                 if (klass != rb_cFloat) goto general;
-                generate_json_float(buffer, Vstate, state, obj);
+                generate_json_float(buffer, data, obj);
+                break;
+            case T_STRUCT:
+                if (klass != cFragment) goto general;
+                generate_json_fragment(buffer, data, obj);
                 break;
             default:
             general:
-                if (state->strict) {
-                    rb_raise(eGeneratorError, "%"PRIsVALUE" not allowed in JSON", CLASS_OF(obj));
-                } else if (rb_respond_to(obj, i_to_json)) {
-                    tmp = rb_funcall(obj, i_to_json, 1, Vstate);
-                    Check_Type(tmp, T_STRING);
-                    fbuffer_append_str(buffer, tmp);
+                if (data->state->strict) {
+                    if (RTEST(data->state->as_json) && !as_json_called) {
+                        obj = rb_proc_call_with_block(data->state->as_json, 1, &obj, Qnil);
+                        as_json_called = true;
+                        goto start;
+                    } else {
+                        raise_generator_error(obj, "%"PRIsVALUE" not allowed in JSON", CLASS_OF(obj));
+                    }
                 } else {
-                    tmp = rb_funcall(obj, i_to_s, 0);
-                    Check_Type(tmp, T_STRING);
-                    generate_json_string(buffer, Vstate, state, tmp);
+                    generate_json_fallback(buffer, data, obj);
                 }
         }
     }
 }
 
-static FBuffer *cState_prepare_buffer(VALUE self)
-{
-    FBuffer *buffer;
-    GET_STATE(self);
-    buffer = fbuffer_alloc(state->buffer_initial_length);
-
-    return buffer;
-}
-
-struct generate_json_data {
-    FBuffer *buffer;
-    VALUE vstate;
-    JSON_Generator_State *state;
-    VALUE obj;
-};
-
 static VALUE generate_json_try(VALUE d)
 {
     struct generate_json_data *data = (struct generate_json_data *)d;
 
-    generate_json(data->buffer, data->vstate, data->state, data->obj);
+    data->func(data->buffer, data, data->obj);
 
     return Qnil;
 }
@@ -932,37 +1511,53 @@ static VALUE generate_json_rescue(VALUE d, VALUE exc)
     return Qundef;
 }
 
-static VALUE cState_partial_generate(VALUE self, VALUE obj)
+static VALUE cState_partial_generate(VALUE self, VALUE obj, generator_func func, VALUE io)
 {
-    FBuffer *buffer = cState_prepare_buffer(self);
     GET_STATE(self);
 
+    char stack_buffer[FBUFFER_STACK_SIZE];
+    FBuffer buffer = {
+        .io = RTEST(io) ? io : Qfalse,
+    };
+    fbuffer_stack_init(&buffer, state->buffer_initial_length, stack_buffer, FBUFFER_STACK_SIZE);
+
     struct generate_json_data data = {
-        .buffer = buffer,
+        .buffer = &buffer,
         .vstate = self,
         .state = state,
-        .obj = obj
+        .obj = obj,
+        .func = func
     };
     rb_rescue(generate_json_try, (VALUE)&data, generate_json_rescue, (VALUE)&data);
 
-    return fbuffer_to_s(buffer);
+    return fbuffer_finalize(&buffer);
 }
 
-/*
- * call-seq: generate(obj)
+/* call-seq:
+ *   generate(obj) -> String
+ *   generate(obj, anIO) -> anIO
  *
  * Generates a valid JSON document from object +obj+ and returns the
  * result. If no valid JSON document can be created this method raises a
  * GeneratorError exception.
  */
-static VALUE cState_generate(VALUE self, VALUE obj)
+static VALUE cState_generate(int argc, VALUE *argv, VALUE self)
 {
-    VALUE result = cState_partial_generate(self, obj);
+    rb_check_arity(argc, 1, 2);
+    VALUE obj = argv[0];
+    VALUE io = argc > 1 ? argv[1] : Qnil;
+    VALUE result = cState_partial_generate(self, obj, generate_json, io);
     GET_STATE(self);
     (void)state;
     return result;
 }
 
+static VALUE cState_initialize(int argc, VALUE *argv, VALUE self)
+{
+    rb_warn("The json gem extension was loaded with the stdlib ruby code. You should upgrade rubygems with `gem update --system`");
+    return self;
+}
+
 /*
  * call-seq: initialize_copy(orig)
  *
@@ -979,11 +1574,12 @@ static VALUE cState_init_copy(VALUE obj, VALUE orig)
     if (!objState) rb_raise(rb_eArgError, "unallocated JSON::State");
 
     MEMCPY(objState, origState, JSON_Generator_State, 1);
-    objState->indent = fstrndup(origState->indent, origState->indent_len);
-    objState->space = fstrndup(origState->space, origState->space_len);
-    objState->space_before = fstrndup(origState->space_before, origState->space_before_len);
-    objState->object_nl = fstrndup(origState->object_nl, origState->object_nl_len);
-    objState->array_nl = fstrndup(origState->array_nl, origState->array_nl_len);
+    objState->indent = origState->indent;
+    objState->space = origState->space;
+    objState->space_before = origState->space_before;
+    objState->object_nl = origState->object_nl;
+    objState->array_nl = origState->array_nl;
+    objState->as_json = origState->as_json;
     return obj;
 }
 
@@ -1013,7 +1609,18 @@ static VALUE cState_from_state_s(VALUE self, VALUE opts)
 static VALUE cState_indent(VALUE self)
 {
     GET_STATE(self);
-    return state->indent ? rb_str_new(state->indent, state->indent_len) : rb_str_new2("");
+    return state->indent ? state->indent : rb_str_freeze(rb_utf8_str_new("", 0));
+}
+
+static VALUE string_config(VALUE config)
+{
+    if (RTEST(config)) {
+        Check_Type(config, T_STRING);
+        if (RSTRING_LEN(config)) {
+            return rb_str_new_frozen(config);
+        }
+    }
+    return Qfalse;
 }
 
 /*
@@ -1023,21 +1630,8 @@ static VALUE cState_indent(VALUE self)
  */
 static VALUE cState_indent_set(VALUE self, VALUE indent)
 {
-    unsigned long len;
     GET_STATE(self);
-    Check_Type(indent, T_STRING);
-    len = RSTRING_LEN(indent);
-    if (len == 0) {
-        if (state->indent) {
-            ruby_xfree(state->indent);
-            state->indent = NULL;
-            state->indent_len = 0;
-        }
-    } else {
-        if (state->indent) ruby_xfree(state->indent);
-        state->indent = fstrndup(RSTRING_PTR(indent), len);
-        state->indent_len = len;
-    }
+    RB_OBJ_WRITE(self, &state->indent, string_config(indent));
     return Qnil;
 }
 
@@ -1050,7 +1644,7 @@ static VALUE cState_indent_set(VALUE self, VALUE indent)
 static VALUE cState_space(VALUE self)
 {
     GET_STATE(self);
-    return state->space ? rb_str_new(state->space, state->space_len) : rb_str_new2("");
+    return state->space ? state->space : rb_str_freeze(rb_utf8_str_new("", 0));
 }
 
 /*
@@ -1061,21 +1655,8 @@ static VALUE cState_space(VALUE self)
  */
 static VALUE cState_space_set(VALUE self, VALUE space)
 {
-    unsigned long len;
     GET_STATE(self);
-    Check_Type(space, T_STRING);
-    len = RSTRING_LEN(space);
-    if (len == 0) {
-        if (state->space) {
-            ruby_xfree(state->space);
-            state->space = NULL;
-            state->space_len = 0;
-        }
-    } else {
-        if (state->space) ruby_xfree(state->space);
-        state->space = fstrndup(RSTRING_PTR(space), len);
-        state->space_len = len;
-    }
+    RB_OBJ_WRITE(self, &state->space, string_config(space));
     return Qnil;
 }
 
@@ -1087,7 +1668,7 @@ static VALUE cState_space_set(VALUE self, VALUE space)
 static VALUE cState_space_before(VALUE self)
 {
     GET_STATE(self);
-    return state->space_before ? rb_str_new(state->space_before, state->space_before_len) : rb_str_new2("");
+    return state->space_before ? state->space_before : rb_str_freeze(rb_utf8_str_new("", 0));
 }
 
 /*
@@ -1097,21 +1678,8 @@ static VALUE cState_space_before(VALUE self)
  */
 static VALUE cState_space_before_set(VALUE self, VALUE space_before)
 {
-    unsigned long len;
     GET_STATE(self);
-    Check_Type(space_before, T_STRING);
-    len = RSTRING_LEN(space_before);
-    if (len == 0) {
-        if (state->space_before) {
-            ruby_xfree(state->space_before);
-            state->space_before = NULL;
-            state->space_before_len = 0;
-        }
-    } else {
-        if (state->space_before) ruby_xfree(state->space_before);
-        state->space_before = fstrndup(RSTRING_PTR(space_before), len);
-        state->space_before_len = len;
-    }
+    RB_OBJ_WRITE(self, &state->space_before, string_config(space_before));
     return Qnil;
 }
 
@@ -1124,7 +1692,7 @@ static VALUE cState_space_before_set(VALUE self, VALUE space_before)
 static VALUE cState_object_nl(VALUE self)
 {
     GET_STATE(self);
-    return state->object_nl ? rb_str_new(state->object_nl, state->object_nl_len) : rb_str_new2("");
+    return state->object_nl ? state->object_nl : rb_str_freeze(rb_utf8_str_new("", 0));
 }
 
 /*
@@ -1135,20 +1703,8 @@ static VALUE cState_object_nl(VALUE self)
  */
 static VALUE cState_object_nl_set(VALUE self, VALUE object_nl)
 {
-    unsigned long len;
     GET_STATE(self);
-    Check_Type(object_nl, T_STRING);
-    len = RSTRING_LEN(object_nl);
-    if (len == 0) {
-        if (state->object_nl) {
-            ruby_xfree(state->object_nl);
-            state->object_nl = NULL;
-        }
-    } else {
-        if (state->object_nl) ruby_xfree(state->object_nl);
-        state->object_nl = fstrndup(RSTRING_PTR(object_nl), len);
-        state->object_nl_len = len;
-    }
+    RB_OBJ_WRITE(self, &state->object_nl, string_config(object_nl));
     return Qnil;
 }
 
@@ -1160,7 +1716,7 @@ static VALUE cState_object_nl_set(VALUE self, VALUE object_nl)
 static VALUE cState_array_nl(VALUE self)
 {
     GET_STATE(self);
-    return state->array_nl ? rb_str_new(state->array_nl, state->array_nl_len) : rb_str_new2("");
+    return state->array_nl ? state->array_nl : rb_str_freeze(rb_utf8_str_new("", 0));
 }
 
 /*
@@ -1170,23 +1726,33 @@ static VALUE cState_array_nl(VALUE self)
  */
 static VALUE cState_array_nl_set(VALUE self, VALUE array_nl)
 {
-    unsigned long len;
     GET_STATE(self);
-    Check_Type(array_nl, T_STRING);
-    len = RSTRING_LEN(array_nl);
-    if (len == 0) {
-        if (state->array_nl) {
-            ruby_xfree(state->array_nl);
-            state->array_nl = NULL;
-        }
-    } else {
-        if (state->array_nl) ruby_xfree(state->array_nl);
-        state->array_nl = fstrndup(RSTRING_PTR(array_nl), len);
-        state->array_nl_len = len;
-    }
+    RB_OBJ_WRITE(self, &state->array_nl, string_config(array_nl));
     return Qnil;
 }
 
+/*
+ * call-seq: as_json()
+ *
+ * This string is put at the end of a line that holds a JSON array.
+ */
+static VALUE cState_as_json(VALUE self)
+{
+    GET_STATE(self);
+    return state->as_json;
+}
+
+/*
+ * call-seq: as_json=(as_json)
+ *
+ * This string is put at the end of a line that holds a JSON array.
+ */
+static VALUE cState_as_json_set(VALUE self, VALUE as_json)
+{
+    GET_STATE(self);
+    RB_OBJ_WRITE(self, &state->as_json, rb_convert_type(as_json, T_DATA, "Proc", "to_proc"));
+    return Qnil;
+}
 
 /*
 * call-seq: check_circular?
@@ -1212,6 +1778,11 @@ static VALUE cState_max_nesting(VALUE self)
     return LONG2FIX(state->max_nesting);
 }
 
+static long long_config(VALUE num)
+{
+    return RTEST(num) ? FIX2LONG(num) : 0;
+}
+
 /*
  * call-seq: max_nesting=(depth)
  *
@@ -1221,8 +1792,7 @@ static VALUE cState_max_nesting(VALUE self)
 static VALUE cState_max_nesting_set(VALUE self, VALUE depth)
 {
     GET_STATE(self);
-    Check_Type(depth, T_FIXNUM);
-    state->max_nesting = FIX2LONG(depth);
+    state->max_nesting = long_config(depth);
     return Qnil;
 }
 
@@ -1350,8 +1920,7 @@ static VALUE cState_depth(VALUE self)
 static VALUE cState_depth_set(VALUE self, VALUE depth)
 {
     GET_STATE(self);
-    Check_Type(depth, T_FIXNUM);
-    state->depth = FIX2LONG(depth);
+    state->depth = long_config(depth);
     return Qnil;
 }
 
@@ -1366,6 +1935,15 @@ static VALUE cState_buffer_initial_length(VALUE self)
     return LONG2FIX(state->buffer_initial_length);
 }
 
+static void buffer_initial_length_set(JSON_Generator_State *state, VALUE buffer_initial_length)
+{
+    Check_Type(buffer_initial_length, T_FIXNUM);
+    long initial_length = FIX2LONG(buffer_initial_length);
+    if (initial_length > 0) {
+        state->buffer_initial_length = initial_length;
+    }
+}
+
 /*
  * call-seq: buffer_initial_length=(length)
  *
@@ -1374,16 +1952,76 @@ static VALUE cState_buffer_initial_length(VALUE self)
  */
 static VALUE cState_buffer_initial_length_set(VALUE self, VALUE buffer_initial_length)
 {
-    long initial_length;
     GET_STATE(self);
-    Check_Type(buffer_initial_length, T_FIXNUM);
-    initial_length = FIX2LONG(buffer_initial_length);
-    if (initial_length > 0) {
-        state->buffer_initial_length = initial_length;
-    }
+    buffer_initial_length_set(state, buffer_initial_length);
     return Qnil;
 }
 
+static int configure_state_i(VALUE key, VALUE val, VALUE _arg)
+{
+    JSON_Generator_State *state = (JSON_Generator_State *)_arg;
+
+         if (key == sym_indent)                { state->indent = string_config(val); }
+    else if (key == sym_space)                 { state->space = string_config(val); }
+    else if (key == sym_space_before)          { state->space_before = string_config(val); }
+    else if (key == sym_object_nl)             { state->object_nl = string_config(val); }
+    else if (key == sym_array_nl)              { state->array_nl = string_config(val); }
+    else if (key == sym_max_nesting)           { state->max_nesting = long_config(val); }
+    else if (key == sym_allow_nan)             { state->allow_nan = RTEST(val); }
+    else if (key == sym_ascii_only)            { state->ascii_only = RTEST(val); }
+    else if (key == sym_depth)                 { state->depth = long_config(val); }
+    else if (key == sym_buffer_initial_length) { buffer_initial_length_set(state, val); }
+    else if (key == sym_script_safe)           { state->script_safe = RTEST(val); }
+    else if (key == sym_escape_slash)          { state->script_safe = RTEST(val); }
+    else if (key == sym_strict)                { state->strict = RTEST(val); }
+    else if (key == sym_as_json)               { state->as_json = RTEST(val) ? rb_convert_type(val, T_DATA, "Proc", "to_proc") : Qfalse; }
+    return ST_CONTINUE;
+}
+
+static void configure_state(JSON_Generator_State *state, VALUE config)
+{
+    if (!RTEST(config)) return;
+
+    Check_Type(config, T_HASH);
+
+    if (!RHASH_SIZE(config)) return;
+
+    // We assume in most cases few keys are set so it's faster to go over
+    // the provided keys than to check all possible keys.
+    rb_hash_foreach(config, configure_state_i, (VALUE)state);
+}
+
+static VALUE cState_configure(VALUE self, VALUE opts)
+{
+    GET_STATE(self);
+    configure_state(state, opts);
+    return self;
+}
+
+static VALUE cState_m_generate(VALUE klass, VALUE obj, VALUE opts, VALUE io)
+{
+    JSON_Generator_State state = {0};
+    state_init(&state);
+    configure_state(&state, opts);
+
+    char stack_buffer[FBUFFER_STACK_SIZE];
+    FBuffer buffer = {
+        .io = RTEST(io) ? io : Qfalse,
+    };
+    fbuffer_stack_init(&buffer, state.buffer_initial_length, stack_buffer, FBUFFER_STACK_SIZE);
+
+    struct generate_json_data data = {
+        .buffer = &buffer,
+        .vstate = Qfalse,
+        .state = &state,
+        .obj = obj,
+        .func = generate_json,
+    };
+    rb_rescue(generate_json_try, (VALUE)&data, generate_json_rescue, (VALUE)&data);
+
+    return fbuffer_finalize(&buffer);
+}
+
 /*
  *
  */
@@ -1397,17 +2035,26 @@ void Init_generator(void)
     rb_require("json/common");
 
     mJSON = rb_define_module("JSON");
+
+    rb_global_variable(&cFragment);
+    cFragment = rb_const_get(mJSON, rb_intern("Fragment"));
+
     VALUE mExt = rb_define_module_under(mJSON, "Ext");
     VALUE mGenerator = rb_define_module_under(mExt, "Generator");
 
+    rb_global_variable(&eGeneratorError);
     eGeneratorError = rb_path2class("JSON::GeneratorError");
+
+    rb_global_variable(&eNestingError);
     eNestingError = rb_path2class("JSON::NestingError");
-    rb_gc_register_mark_object(eGeneratorError);
-    rb_gc_register_mark_object(eNestingError);
 
     cState = rb_define_class_under(mGenerator, "State", rb_cObject);
     rb_define_alloc_func(cState, cState_s_allocate);
     rb_define_singleton_method(cState, "from_state", cState_from_state_s, 1);
+    rb_define_method(cState, "initialize", cState_initialize, -1);
+    rb_define_alias(cState, "initialize", "initialize"); // avoid method redefinition warnings
+    rb_define_private_method(cState, "_configure", cState_configure, 1);
+
     rb_define_method(cState, "initialize_copy", cState_init_copy, 1);
     rb_define_method(cState, "indent", cState_indent, 0);
     rb_define_method(cState, "indent=", cState_indent_set, 1);
@@ -1419,6 +2066,8 @@ void Init_generator(void)
     rb_define_method(cState, "object_nl=", cState_object_nl_set, 1);
     rb_define_method(cState, "array_nl", cState_array_nl, 0);
     rb_define_method(cState, "array_nl=", cState_array_nl_set, 1);
+    rb_define_method(cState, "as_json", cState_as_json, 0);
+    rb_define_method(cState, "as_json=", cState_as_json_set, 1);
     rb_define_method(cState, "max_nesting", cState_max_nesting, 0);
     rb_define_method(cState, "max_nesting=", cState_max_nesting_set, 1);
     rb_define_method(cState, "script_safe", cState_script_safe, 0);
@@ -1439,7 +2088,10 @@ void Init_generator(void)
     rb_define_method(cState, "depth=", cState_depth_set, 1);
     rb_define_method(cState, "buffer_initial_length", cState_buffer_initial_length, 0);
     rb_define_method(cState, "buffer_initial_length=", cState_buffer_initial_length_set, 1);
-    rb_define_method(cState, "generate", cState_generate, 1);
+    rb_define_method(cState, "generate", cState_generate, -1);
+    rb_define_alias(cState, "generate_new", "generate"); // :nodoc:
+
+    rb_define_singleton_method(cState, "generate", cState_m_generate, 3);
 
     VALUE mGeneratorMethods = rb_define_module_under(mGenerator, "GeneratorMethods");
 
@@ -1495,7 +2147,43 @@ void Init_generator(void)
     i_extend = rb_intern("extend");
     i_encode = rb_intern("encode");
 
+    sym_indent = ID2SYM(rb_intern("indent"));
+    sym_space = ID2SYM(rb_intern("space"));
+    sym_space_before = ID2SYM(rb_intern("space_before"));
+    sym_object_nl = ID2SYM(rb_intern("object_nl"));
+    sym_array_nl = ID2SYM(rb_intern("array_nl"));
+    sym_max_nesting = ID2SYM(rb_intern("max_nesting"));
+    sym_allow_nan = ID2SYM(rb_intern("allow_nan"));
+    sym_ascii_only = ID2SYM(rb_intern("ascii_only"));
+    sym_depth = ID2SYM(rb_intern("depth"));
+    sym_buffer_initial_length = ID2SYM(rb_intern("buffer_initial_length"));
+    sym_script_safe = ID2SYM(rb_intern("script_safe"));
+    sym_escape_slash = ID2SYM(rb_intern("escape_slash"));
+    sym_strict = ID2SYM(rb_intern("strict"));
+    sym_as_json = ID2SYM(rb_intern("as_json"));
+
     usascii_encindex = rb_usascii_encindex();
     utf8_encindex = rb_utf8_encindex();
     binary_encindex = rb_ascii8bit_encindex();
+
+    rb_require("json/ext/generator/state");
+
+
+    switch(find_simd_implementation()) {
+#ifdef HAVE_SIMD
+#ifdef HAVE_SIMD_NEON
+        case SIMD_NEON:
+            search_escape_basic_impl = search_escape_basic_neon;
+            break;
+#endif /* HAVE_SIMD_NEON */
+#ifdef HAVE_SIMD_SSE2
+        case SIMD_SSE2:
+            search_escape_basic_impl = search_escape_basic_sse2;
+            break;
+#endif /* HAVE_SIMD_SSE2 */
+#endif /* HAVE_SIMD */
+        default:
+            search_escape_basic_impl = search_escape_basic;
+            break;
+    }
 }