json 2.7.2 → 2.7.4

data/ext/json/ext/generator/generator.c

@@ -1,332 +1,306 @@
 #include "../fbuffer/fbuffer.h"
 #include "generator.h"
 
-static VALUE mJSON, mExt, mGenerator, cState, mGeneratorMethods, mObject,
-             mHash, mArray,
-#ifdef RUBY_INTEGER_UNIFICATION
-             mInteger,
-#else
-             mFixnum, mBignum,
+#ifndef RB_UNLIKELY
+#define RB_UNLIKELY(cond) (cond)
 #endif
-             mFloat, mString, mString_Extend,
-             mTrueClass, mFalseClass, mNilClass, eGeneratorError,
-             eNestingError;
 
-static ID i_to_s, i_to_json, i_new, i_indent, i_space, i_space_before,
-          i_object_nl, i_array_nl, i_max_nesting, i_allow_nan, i_ascii_only,
-          i_pack, i_unpack, i_create_id, i_extend, i_key_p,
-          i_aref, i_send, i_respond_to_p, i_match, i_keys, i_depth,
-          i_buffer_initial_length, i_dup, i_script_safe, i_escape_slash, i_strict;
+static VALUE mJSON, cState, mString_Extend, eGeneratorError, eNestingError, Encoding_UTF_8;
 
-/*
- * Copyright 2001-2004 Unicode, Inc.
+static ID i_to_s, i_to_json, i_new, i_pack, i_unpack, i_create_id, i_extend, i_encode;
+
+/* Converts in_string to a JSON string (without the wrapping '"'
+ * characters) in FBuffer out_buffer.
  *
- * Disclaimer
+ * Character are JSON-escaped according to:
  *
- * This source code is provided as is by Unicode, Inc. No claims are
- * made as to fitness for any particular purpose. No warranties of any
- * kind are expressed or implied. The recipient agrees to determine
- * applicability of information provided. If this file has been
- * purchased on magnetic or optical media from Unicode, Inc., the
- * sole remedy for any claim will be exchange of defective media
- * within 90 days of receipt.
+ * - Always: ASCII control characters (0x00-0x1F), dquote, and
+ *   backslash.
  *
- * Limitations on Rights to Redistribute This Code
+ * - If out_ascii_only: non-ASCII characters (>0x7F)
  *
- * Unicode, Inc. hereby grants the right to freely use the information
- * supplied in this file in the creation of products supporting the
- * Unicode Standard, and to make copies of this file in any form
- * for internal or external distribution as long as this notice
- * remains attached.
+ * - If out_script_safe: forwardslash, line separator (U+2028), and
+ *   paragraph separator (U+2029)
+ *
+ * Everything else (should be UTF-8) is just passed through and
+ * appended to the result.
  */
+static void convert_UTF8_to_JSON(FBuffer *out_buffer, VALUE str, const char escape_table[256], bool out_script_safe)
+{
+    const char *hexdig = "0123456789abcdef";
+    char scratch[12] = { '\\', 'u', 0, 0, 0, 0, '\\', 'u' };
 
-/*
- * Index into the table below with the first byte of a UTF-8 sequence to
- * get the number of trailing bytes that are supposed to follow it.
- * Note that *legal* UTF-8 values can't have 4 or 5-bytes. The table is
- * left as-is for anyone who may want to do such conversion, which was
- * allowed in earlier algorithms.
- */
-static const char trailingBytesForUTF8[256] = {
-    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-    1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
-    2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3,4,4,4,4,5,5,5,5
-};
+    const char *ptr = RSTRING_PTR(str);
+    unsigned long len = RSTRING_LEN(str);
 
-/*
- * Magic values subtracted from a buffer value during UTF8 conversion.
- * This table contains as many values as there might be trailing bytes
- * in a UTF-8 sequence.
- */
-static const UTF32 offsetsFromUTF8[6] = { 0x00000000UL, 0x00003080UL, 0x000E2080UL,
-    0x03C82080UL, 0xFA082080UL, 0x82082080UL };
+    unsigned long beg = 0, pos = 0;
 
-/*
- * Utility routine to tell whether a sequence of bytes is legal UTF-8.
- * This must be called with the length pre-determined by the first byte.
- * If not calling this from ConvertUTF8to*, then the length can be set by:
- *  length = trailingBytesForUTF8[*source]+1;
- * and the sequence is illegal right away if there aren't that many bytes
- * available.
- * If presented with a length > 4, this returns 0.  The Unicode
- * definition of UTF-8 goes up to 4-byte sequences.
- */
-static unsigned char isLegalUTF8(const UTF8 *source, unsigned long length)
-{
-    UTF8 a;
-    const UTF8 *srcptr = source+length;
-    switch (length) {
-        default: return 0;
-                 /* Everything else falls through when "1"... */
-        case 4: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return 0;
-        case 3: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return 0;
-        case 2: if ((a = (*--srcptr)) > 0xBF) return 0;
-
-                    switch (*source) {
-                        /* no fall-through in this inner switch */
-                        case 0xE0: if (a < 0xA0) return 0; break;
-                        case 0xED: if (a > 0x9F) return 0; break;
-                        case 0xF0: if (a < 0x90) return 0; break;
-                        case 0xF4: if (a > 0x8F) return 0; break;
-                        default:   if (a < 0x80) return 0;
+#define FLUSH_POS(bytes) if (pos > beg) { fbuffer_append(out_buffer, &ptr[beg], pos - beg); } pos += bytes; beg = pos;
+
+    while (pos < len) {
+        unsigned char ch = ptr[pos];
+        unsigned char ch_len = escape_table[ch];
+        /* JSON encoding */
+
+        if (RB_UNLIKELY(ch_len)) {
+            switch (ch_len) {
+                case 0:
+                    pos++;
+                    break;
+                case 1: {
+                    FLUSH_POS(1);
+                    switch (ch) {
+                        case '"':  fbuffer_append(out_buffer, "\\\"", 2); break;
+                        case '\\': fbuffer_append(out_buffer, "\\\\", 2); break;
+                        case '/':  fbuffer_append(out_buffer, "\\/", 2); break;
+                        case '\b': fbuffer_append(out_buffer, "\\b", 2); break;
+                        case '\f': fbuffer_append(out_buffer, "\\f", 2); break;
+                        case '\n': fbuffer_append(out_buffer, "\\n", 2); break;
+                        case '\r': fbuffer_append(out_buffer, "\\r", 2); break;
+                        case '\t': fbuffer_append(out_buffer, "\\t", 2); break;
+                        default: {
+                            scratch[2] = hexdig[ch >> 12];
+                            scratch[3] = hexdig[(ch >> 8) & 0xf];
+                            scratch[4] = hexdig[(ch >> 4) & 0xf];
+                            scratch[5] = hexdig[ch & 0xf];
+                            fbuffer_append(out_buffer, scratch, 6);
+                            break;
+                        }
                     }
+                    break;
+                }
+                case 3: {
+                    unsigned char b2 = ptr[pos + 1];
+                    if (RB_UNLIKELY(out_script_safe && b2 == 0x80)) {
+                        unsigned char b3 = ptr[pos + 2];
+                        if (b3 == 0xA8) {
+                            FLUSH_POS(3);
+                            fbuffer_append(out_buffer, "\\u2028", 6);
+                            break;
+                        } else if (b3 == 0xA9) {
+                            FLUSH_POS(3);
+                            fbuffer_append(out_buffer, "\\u2029", 6);
+                            break;
+                        }
+                    }
+                    // fallthrough
+                }
+                default:
+                    pos += ch_len;
+                    break;
+            }
+        } else {
+            pos++;
+        }
+    }
+#undef FLUSH_POS
 
-        case 1: if (*source >= 0x80 && *source < 0xC2) return 0;
+    if (beg < len) {
+        fbuffer_append(out_buffer, &ptr[beg], len - beg);
     }
-    if (*source > 0xF4) return 0;
-    return 1;
-}
 
-/* Escapes the UTF16 character and stores the result in the buffer buf. */
-static void unicode_escape(char *buf, UTF16 character)
-{
-    const char *digits = "0123456789abcdef";
+    RB_GC_GUARD(str);
+}
+
+static const char escape_table[256] = {
+    // ASCII Control Characters
+    1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
+    1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
+    // ASCII Characters
+    0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0, // '"'
+    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+    0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0, // '\\'
+    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+    // Continuation byte
+    1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
+    1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
+    1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
+    1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
+    // First byte of a 2-byte code point
+    2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
+    2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
+    // First byte of a 4-byte code point
+    3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,
+    //First byte of a 4+byte code point
+    4,4,4,4,4,4,4,4,5,5,5,5,6,6,1,1,
+};
 
-    buf[2] = digits[character >> 12];
-    buf[3] = digits[(character >> 8) & 0xf];
-    buf[4] = digits[(character >> 4) & 0xf];
-    buf[5] = digits[character & 0xf];
-}
+static const char script_safe_escape_table[256] = {
+    // ASCII Control Characters
+    1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
+    1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
+    // ASCII Characters
+    0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1, // '"' and '/'
+    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+    0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0, // '\\'
+    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+    // Continuation byte
+    1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
+    1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
+    1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
+    1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
+    // First byte of a 2-byte code point
+    2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
+    2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
+    // First byte of a 4-byte code point
+    3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,
+    //First byte of a 4+byte code point
+    4,4,4,4,4,4,4,4,5,5,5,5,6,6,1,1,
+};
 
-/* Escapes the UTF16 character and stores the result in the buffer buf, then
- * the buffer buf is appended to the FBuffer buffer. */
-static void unicode_escape_to_buffer(FBuffer *buffer, char buf[6], UTF16
-        character)
+static void convert_ASCII_to_JSON(FBuffer *out_buffer, VALUE str, const char escape_table[256])
 {
-    unicode_escape(buf, character);
-    fbuffer_append(buffer, buf, 6);
+    const char *hexdig = "0123456789abcdef";
+    char scratch[12] = { '\\', 'u', 0, 0, 0, 0, '\\', 'u' };
+
+    const char *ptr = RSTRING_PTR(str);
+    unsigned long len = RSTRING_LEN(str);
+
+    unsigned long beg = 0, pos;
+
+    for (pos = 0; pos < len;) {
+        unsigned char ch = ptr[pos];
+        /* JSON encoding */
+        if (escape_table[ch]) {
+            if (pos > beg) {
+                fbuffer_append(out_buffer, &ptr[beg], pos - beg);
+            }
+
+            beg = pos + 1;
+            switch (ch) {
+                case '"':  fbuffer_append(out_buffer, "\\\"", 2); break;
+                case '\\': fbuffer_append(out_buffer, "\\\\", 2); break;
+                case '/':  fbuffer_append(out_buffer, "\\/", 2); break;
+                case '\b': fbuffer_append(out_buffer, "\\b", 2); break;
+                case '\f': fbuffer_append(out_buffer, "\\f", 2); break;
+                case '\n': fbuffer_append(out_buffer, "\\n", 2); break;
+                case '\r': fbuffer_append(out_buffer, "\\r", 2); break;
+                case '\t': fbuffer_append(out_buffer, "\\t", 2); break;
+                default:
+                    scratch[2] = hexdig[ch >> 12];
+                    scratch[3] = hexdig[(ch >> 8) & 0xf];
+                    scratch[4] = hexdig[(ch >> 4) & 0xf];
+                    scratch[5] = hexdig[ch & 0xf];
+                    fbuffer_append(out_buffer, scratch, 6);
+            }
+        }
+
+        pos++;
+    }
+
+    if (beg < len) {
+        fbuffer_append(out_buffer, &ptr[beg], len - beg);
+    }
+
+    RB_GC_GUARD(str);
 }
 
-/* Converts string to a JSON string in FBuffer buffer, where all but the ASCII
- * and control characters are JSON escaped. */
-static void convert_UTF8_to_JSON_ASCII(FBuffer *buffer, VALUE string, char script_safe)
+static void convert_UTF8_to_ASCII_only_JSON(FBuffer *out_buffer, VALUE str, const char escape_table[256], bool out_script_safe)
 {
-    const UTF8 *source = (UTF8 *) RSTRING_PTR(string);
-    const UTF8 *sourceEnd = source + RSTRING_LEN(string);
-    char buf[6] = { '\\', 'u' };
+    const char *hexdig = "0123456789abcdef";
+    char scratch[12] = { '\\', 'u', 0, 0, 0, 0, '\\', 'u' };
 
-    while (source < sourceEnd) {
-        UTF32 ch = 0;
-        unsigned short extraBytesToRead = trailingBytesForUTF8[*source];
-        if (source + extraBytesToRead >= sourceEnd) {
-            rb_raise(rb_path2class("JSON::GeneratorError"),
-                    "partial character in source, but hit end");
-        }
-        if (!isLegalUTF8(source, extraBytesToRead+1)) {
-            rb_raise(rb_path2class("JSON::GeneratorError"),
-                    "source sequence is illegal/malformed utf-8");
-        }
-        /*
-         * The cases all fall through. See "Note A" below.
-         */
-        switch (extraBytesToRead) {
-            case 5: ch += *source++; ch <<= 6; /* remember, illegal UTF-8 */
-            case 4: ch += *source++; ch <<= 6; /* remember, illegal UTF-8 */
-            case 3: ch += *source++; ch <<= 6;
-            case 2: ch += *source++; ch <<= 6;
-            case 1: ch += *source++; ch <<= 6;
-            case 0: ch += *source++;
-        }
-        ch -= offsetsFromUTF8[extraBytesToRead];
-
-        if (ch <= UNI_MAX_BMP) { /* Target is a character <= 0xFFFF */
-            /* UTF-16 surrogate values are illegal in UTF-32 */
-            if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) {
-#if UNI_STRICT_CONVERSION
-                source -= (extraBytesToRead+1); /* return to the illegal value itself */
-                rb_raise(rb_path2class("JSON::GeneratorError"),
-                        "source sequence is illegal/malformed utf-8");
-#else
-                unicode_escape_to_buffer(buffer, buf, UNI_REPLACEMENT_CHAR);
-#endif
-            } else {
-                /* normal case */
-                if (ch >= 0x20 && ch <= 0x7f) {
+    const char *ptr = RSTRING_PTR(str);
+    unsigned long len = RSTRING_LEN(str);
+
+    unsigned long beg = 0, pos = 0;
+
+#define FLUSH_POS(bytes) if (pos > beg) { fbuffer_append(out_buffer, &ptr[beg], pos - beg); } pos += bytes; beg = pos;
+
+    while (pos < len) {
+        unsigned char ch = ptr[pos];
+        unsigned char ch_len = escape_table[ch];
+
+        if (RB_UNLIKELY(ch_len)) {
+            switch (ch_len) {
+                case 0:
+                    pos++;
+                    break;
+                case 1: {
+                    FLUSH_POS(1);
                     switch (ch) {
-                        case '\\':
-                            fbuffer_append(buffer, "\\\\", 2);
-                            break;
-                        case '"':
-                            fbuffer_append(buffer, "\\\"", 2);
-                            break;
-                        case '/':
-                            if(script_safe) {
-                                fbuffer_append(buffer, "\\/", 2);
-                                break;
-                            }
-                        default:
-                            fbuffer_append_char(buffer, (char)ch);
+                        case '"':  fbuffer_append(out_buffer, "\\\"", 2); break;
+                        case '\\': fbuffer_append(out_buffer, "\\\\", 2); break;
+                        case '/':  fbuffer_append(out_buffer, "\\/", 2); break;
+                        case '\b': fbuffer_append(out_buffer, "\\b", 2); break;
+                        case '\f': fbuffer_append(out_buffer, "\\f", 2); break;
+                        case '\n': fbuffer_append(out_buffer, "\\n", 2); break;
+                        case '\r': fbuffer_append(out_buffer, "\\r", 2); break;
+                        case '\t': fbuffer_append(out_buffer, "\\t", 2); break;
+                        default: {
+                            scratch[2] = hexdig[ch >> 12];
+                            scratch[3] = hexdig[(ch >> 8) & 0xf];
+                            scratch[4] = hexdig[(ch >> 4) & 0xf];
+                            scratch[5] = hexdig[ch & 0xf];
+                            fbuffer_append(out_buffer, scratch, 6);
                             break;
+                        }
                     }
-                } else {
-                    switch (ch) {
-                        case '\n':
-                            fbuffer_append(buffer, "\\n", 2);
-                            break;
-                        case '\r':
-                            fbuffer_append(buffer, "\\r", 2);
-                            break;
-                        case '\t':
-                            fbuffer_append(buffer, "\\t", 2);
-                            break;
-                        case '\f':
-                            fbuffer_append(buffer, "\\f", 2);
+                    break;
+                }
+                default: {
+                    uint32_t wchar = 0;
+                    switch(ch_len) {
+                        case 2:
+                            wchar = ptr[pos] & 0x1F;
                             break;
-                        case '\b':
-                            fbuffer_append(buffer, "\\b", 2);
+                        case 3:
+                            wchar = ptr[pos] & 0x0F;
                             break;
-                        default:
-                            unicode_escape_to_buffer(buffer, buf, (UTF16) ch);
+                        case 4:
+                            wchar = ptr[pos] & 0x07;
                             break;
                     }
-                }
-            }
-        } else if (ch > UNI_MAX_UTF16) {
-#if UNI_STRICT_CONVERSION
-            source -= (extraBytesToRead+1); /* return to the start */
-            rb_raise(rb_path2class("JSON::GeneratorError"),
-                    "source sequence is illegal/malformed utf8");
-#else
-            unicode_escape_to_buffer(buffer, buf, UNI_REPLACEMENT_CHAR);
-#endif
-        } else {
-            /* target is a character in range 0xFFFF - 0x10FFFF. */
-            ch -= halfBase;
-            unicode_escape_to_buffer(buffer, buf, (UTF16)((ch >> halfShift) + UNI_SUR_HIGH_START));
-            unicode_escape_to_buffer(buffer, buf, (UTF16)((ch & halfMask) + UNI_SUR_LOW_START));
-        }
-    }
-    RB_GC_GUARD(string);
-}
-
-/* Converts string to a JSON string in FBuffer buffer, where only the
- * characters required by the JSON standard are JSON escaped. The remaining
- * characters (should be UTF8) are just passed through and appended to the
- * result. */
-static void convert_UTF8_to_JSON(FBuffer *buffer, VALUE string, char script_safe)
-{
-    const char *ptr = RSTRING_PTR(string), *p;
-    unsigned long len = RSTRING_LEN(string), start = 0, end = 0;
-    const char *escape = NULL;
-    int escape_len;
-    unsigned char c;
-    char buf[6] = { '\\', 'u' };
-    int ascii_only = rb_enc_str_asciionly_p(string);
-
-    for (start = 0, end = 0; end < len;) {
-        p = ptr + end;
-        c = (unsigned char) *p;
-        if (c < 0x20) {
-            switch (c) {
-                case '\n':
-                    escape = "\\n";
-                    escape_len = 2;
-                    break;
-                case '\r':
-                    escape = "\\r";
-                    escape_len = 2;
-                    break;
-                case '\t':
-                    escape = "\\t";
-                    escape_len = 2;
-                    break;
-                case '\f':
-                    escape = "\\f";
-                    escape_len = 2;
-                    break;
-                case '\b':
-                    escape = "\\b";
-                    escape_len = 2;
-                    break;
-                default:
-                    unicode_escape(buf, (UTF16) *p);
-                    escape = buf;
-                    escape_len = 6;
-                    break;
-            }
-        } else {
-            switch (c) {
-                case '\\':
-                    escape = "\\\\";
-                    escape_len = 2;
-                    break;
-                case '"':
-                    escape =  "\\\"";
-                    escape_len = 2;
-                    break;
-                case '/':
-                    if(script_safe) {
-                        escape = "\\/";
-                        escape_len = 2;
-                        break;
+
+                    for (short i = 1; i < ch_len; i++) {
+                        wchar = (wchar << 6) | (ptr[pos+i] & 0x3F);
                     }
-                default:
-                    {
-                        unsigned short clen = 1;
-                        if (!ascii_only) {
-                            clen += trailingBytesForUTF8[c];
-                            if (end + clen > len) {
-                                rb_raise(rb_path2class("JSON::GeneratorError"),
-                                        "partial character in source, but hit end");
-                            }
-
-                            if (script_safe && c == 0xE2) {
-                                unsigned char c2 = (unsigned char) *(p+1);
-                                unsigned char c3 = (unsigned char) *(p+2);
-                                if (c2 == 0x80 && (c3 == 0xA8 || c3 == 0xA9)) {
-                                    fbuffer_append(buffer, ptr + start, end - start);
-                                    start = end = (end + clen);
-                                    if (c3 == 0xA8) {
-                                        fbuffer_append(buffer, "\\u2028", 6);
-                                    } else {
-                                        fbuffer_append(buffer, "\\u2029", 6);
-                                    }
-                                    continue;
-                                }
-                            }
-
-                            if (!isLegalUTF8((UTF8 *) p, clen)) {
-                                rb_raise(rb_path2class("JSON::GeneratorError"),
-                                        "source sequence is illegal/malformed utf-8");
-                            }
-                        }
-                        end += clen;
+
+                    FLUSH_POS(ch_len);
+
+                    if (wchar <= 0xFFFF) {
+                        scratch[2] = hexdig[wchar >> 12];
+                        scratch[3] = hexdig[(wchar >> 8) & 0xf];
+                        scratch[4] = hexdig[(wchar >> 4) & 0xf];
+                        scratch[5] = hexdig[wchar & 0xf];
+                        fbuffer_append(out_buffer, scratch, 6);
+                    } else {
+                        uint16_t hi, lo;
+                        wchar -= 0x10000;
+                        hi = 0xD800 + (uint16_t)(wchar >> 10);
+                        lo = 0xDC00 + (uint16_t)(wchar & 0x3FF);
+
+                        scratch[2] = hexdig[hi >> 12];
+                        scratch[3] = hexdig[(hi >> 8) & 0xf];
+                        scratch[4] = hexdig[(hi >> 4) & 0xf];
+                        scratch[5] = hexdig[hi & 0xf];
+
+                        scratch[8] = hexdig[lo >> 12];
+                        scratch[9] = hexdig[(lo >> 8) & 0xf];
+                        scratch[10] = hexdig[(lo >> 4) & 0xf];
+                        scratch[11] = hexdig[lo & 0xf];
+
+                        fbuffer_append(out_buffer, scratch, 12);
                     }
-                    continue;
+
                     break;
+                }
             }
+        } else {
+            pos++;
         }
-        fbuffer_append(buffer, ptr + start, end - start);
-        fbuffer_append(buffer, escape, escape_len);
-        start = ++end;
-        escape = NULL;
     }
-    fbuffer_append(buffer, ptr + start, end - start);
+#undef FLUSH_POS
+
+    if (beg < len) {
+        fbuffer_append(out_buffer, &ptr[beg], len - beg);
+    }
+
+    RB_GC_GUARD(str);
 }
 
 static char *fstrndup(const char *ptr, unsigned long len) {
@@ -610,9 +584,6 @@ static void State_free(void *ptr)
     if (state->space_before) ruby_xfree(state->space_before);
     if (state->object_nl) ruby_xfree(state->object_nl);
     if (state->array_nl) ruby_xfree(state->array_nl);
-    if (state->array_delim) fbuffer_free(state->array_delim);
-    if (state->object_delim) fbuffer_free(state->object_delim);
-    if (state->object_delim2) fbuffer_free(state->object_delim2);
     ruby_xfree(state);
 }
 
@@ -625,9 +596,6 @@ static size_t State_memsize(const void *ptr)
     if (state->space_before) size += state->space_before_len + 1;
     if (state->object_nl) size += state->object_nl_len + 1;
     if (state->array_nl) size += state->array_nl_len + 1;
-    if (state->array_delim) size += FBUFFER_CAPA(state->array_delim);
-    if (state->object_delim) size += FBUFFER_CAPA(state->object_delim);
-    if (state->object_delim2) size += FBUFFER_CAPA(state->object_delim2);
     return size;
 }
 
@@ -636,194 +604,20 @@ static size_t State_memsize(const void *ptr)
 #   define RUBY_TYPED_FROZEN_SHAREABLE 0
 #endif
 
-#ifdef NEW_TYPEDDATA_WRAPPER
 static const rb_data_type_t JSON_Generator_State_type = {
     "JSON/Generator/State",
     {NULL, State_free, State_memsize,},
-#ifdef RUBY_TYPED_FREE_IMMEDIATELY
     0, 0,
-    RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_FROZEN_SHAREABLE,
-#endif
+    RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_FROZEN_SHAREABLE,
 };
-#endif
 
 static VALUE cState_s_allocate(VALUE klass)
 {
     JSON_Generator_State *state;
-    return TypedData_Make_Struct(klass, JSON_Generator_State,
-				 &JSON_Generator_State_type, state);
-}
-
-/*
- * call-seq: configure(opts)
- *
- * Configure this State instance with the Hash _opts_, and return
- * itself.
- */
-static VALUE cState_configure(VALUE self, VALUE opts)
-{
-    VALUE tmp;
-    GET_STATE(self);
-    tmp = rb_check_convert_type(opts, T_HASH, "Hash", "to_hash");
-    if (NIL_P(tmp)) tmp = rb_convert_type(opts, T_HASH, "Hash", "to_h");
-    opts = tmp;
-    tmp = rb_hash_aref(opts, ID2SYM(i_indent));
-    if (RTEST(tmp)) {
-        unsigned long len;
-        Check_Type(tmp, T_STRING);
-        len = RSTRING_LEN(tmp);
-        state->indent = fstrndup(RSTRING_PTR(tmp), len + 1);
-        state->indent_len = len;
-    }
-    tmp = rb_hash_aref(opts, ID2SYM(i_space));
-    if (RTEST(tmp)) {
-        unsigned long len;
-        Check_Type(tmp, T_STRING);
-        len = RSTRING_LEN(tmp);
-        state->space = fstrndup(RSTRING_PTR(tmp), len + 1);
-        state->space_len = len;
-    }
-    tmp = rb_hash_aref(opts, ID2SYM(i_space_before));
-    if (RTEST(tmp)) {
-        unsigned long len;
-        Check_Type(tmp, T_STRING);
-        len = RSTRING_LEN(tmp);
-        state->space_before = fstrndup(RSTRING_PTR(tmp), len + 1);
-        state->space_before_len = len;
-    }
-    tmp = rb_hash_aref(opts, ID2SYM(i_array_nl));
-    if (RTEST(tmp)) {
-        unsigned long len;
-        Check_Type(tmp, T_STRING);
-        len = RSTRING_LEN(tmp);
-        state->array_nl = fstrndup(RSTRING_PTR(tmp), len + 1);
-        state->array_nl_len = len;
-    }
-    tmp = rb_hash_aref(opts, ID2SYM(i_object_nl));
-    if (RTEST(tmp)) {
-        unsigned long len;
-        Check_Type(tmp, T_STRING);
-        len = RSTRING_LEN(tmp);
-        state->object_nl = fstrndup(RSTRING_PTR(tmp), len + 1);
-        state->object_nl_len = len;
-    }
-    tmp = ID2SYM(i_max_nesting);
+    VALUE obj = TypedData_Make_Struct(klass, JSON_Generator_State, &JSON_Generator_State_type, state);
     state->max_nesting = 100;
-    if (option_given_p(opts, tmp)) {
-        VALUE max_nesting = rb_hash_aref(opts, tmp);
-        if (RTEST(max_nesting)) {
-            Check_Type(max_nesting, T_FIXNUM);
-            state->max_nesting = FIX2LONG(max_nesting);
-        } else {
-            state->max_nesting = 0;
-        }
-    }
-    tmp = ID2SYM(i_depth);
-    state->depth = 0;
-    if (option_given_p(opts, tmp)) {
-        VALUE depth = rb_hash_aref(opts, tmp);
-        if (RTEST(depth)) {
-            Check_Type(depth, T_FIXNUM);
-            state->depth = FIX2LONG(depth);
-        } else {
-            state->depth = 0;
-        }
-    }
-    tmp = ID2SYM(i_buffer_initial_length);
-    if (option_given_p(opts, tmp)) {
-        VALUE buffer_initial_length = rb_hash_aref(opts, tmp);
-        if (RTEST(buffer_initial_length)) {
-            long initial_length;
-            Check_Type(buffer_initial_length, T_FIXNUM);
-            initial_length = FIX2LONG(buffer_initial_length);
-            if (initial_length > 0) state->buffer_initial_length = initial_length;
-        }
-    }
-    tmp = rb_hash_aref(opts, ID2SYM(i_allow_nan));
-    state->allow_nan = RTEST(tmp);
-    tmp = rb_hash_aref(opts, ID2SYM(i_ascii_only));
-    state->ascii_only = RTEST(tmp);
-    tmp = rb_hash_aref(opts, ID2SYM(i_script_safe));
-    state->script_safe = RTEST(tmp);
-    if (!state->script_safe) {
-        tmp = rb_hash_aref(opts, ID2SYM(i_escape_slash));
-        state->script_safe = RTEST(tmp);
-    }
-    tmp = rb_hash_aref(opts, ID2SYM(i_strict));
-    state->strict = RTEST(tmp);
-    return self;
-}
-
-static void set_state_ivars(VALUE hash, VALUE state)
-{
-    VALUE ivars = rb_obj_instance_variables(state);
-    int i = 0;
-    for (i = 0; i < RARRAY_LEN(ivars); i++) {
-        VALUE key = rb_funcall(rb_ary_entry(ivars, i), i_to_s, 0);
-        long key_len = RSTRING_LEN(key);
-        VALUE value = rb_iv_get(state, StringValueCStr(key));
-        rb_hash_aset(hash, rb_str_intern(rb_str_substr(key, 1, key_len - 1)), value);
-    }
-}
-
-/*
- * call-seq: to_h
- *
- * Returns the configuration instance variables as a hash, that can be
- * passed to the configure method.
- */
-static VALUE cState_to_h(VALUE self)
-{
-    VALUE result = rb_hash_new();
-    GET_STATE(self);
-    set_state_ivars(result, self);
-    rb_hash_aset(result, ID2SYM(i_indent), rb_str_new(state->indent, state->indent_len));
-    rb_hash_aset(result, ID2SYM(i_space), rb_str_new(state->space, state->space_len));
-    rb_hash_aset(result, ID2SYM(i_space_before), rb_str_new(state->space_before, state->space_before_len));
-    rb_hash_aset(result, ID2SYM(i_object_nl), rb_str_new(state->object_nl, state->object_nl_len));
-    rb_hash_aset(result, ID2SYM(i_array_nl), rb_str_new(state->array_nl, state->array_nl_len));
-    rb_hash_aset(result, ID2SYM(i_allow_nan), state->allow_nan ? Qtrue : Qfalse);
-    rb_hash_aset(result, ID2SYM(i_ascii_only), state->ascii_only ? Qtrue : Qfalse);
-    rb_hash_aset(result, ID2SYM(i_max_nesting), LONG2FIX(state->max_nesting));
-    rb_hash_aset(result, ID2SYM(i_script_safe), state->script_safe ? Qtrue : Qfalse);
-    rb_hash_aset(result, ID2SYM(i_strict), state->strict ? Qtrue : Qfalse);
-    rb_hash_aset(result, ID2SYM(i_depth), LONG2FIX(state->depth));
-    rb_hash_aset(result, ID2SYM(i_buffer_initial_length), LONG2FIX(state->buffer_initial_length));
-    return result;
-}
-
-/*
-* call-seq: [](name)
-*
-* Returns the value returned by method +name+.
-*/
-static VALUE cState_aref(VALUE self, VALUE name)
-{
-    name = rb_funcall(name, i_to_s, 0);
-    if (RTEST(rb_funcall(self, i_respond_to_p, 1, name))) {
-        return rb_funcall(self, i_send, 1, name);
-    } else {
-        return rb_attr_get(self, rb_intern_str(rb_str_concat(rb_str_new2("@"), name)));
-    }
-}
-
-/*
-* call-seq: []=(name, value)
-*
-* Sets the attribute name to value.
-*/
-static VALUE cState_aset(VALUE self, VALUE name, VALUE value)
-{
-    VALUE name_writer;
-
-    name = rb_funcall(name, i_to_s, 0);
-    name_writer = rb_str_cat2(rb_str_dup(name), "=");
-    if (RTEST(rb_funcall(self, i_respond_to_p, 1, name_writer))) {
-        return rb_funcall(self, i_send, 2, name_writer, value);
-    } else {
-        rb_ivar_set(self, rb_intern_str(rb_str_concat(rb_str_new2("@"), name)), value);
-    }
-    return Qnil;
+    state->buffer_initial_length = FBUFFER_INITIAL_LENGTH_DEFAULT;
+    return obj;
 }
 
 struct hash_foreach_arg {
@@ -841,39 +635,36 @@ json_object_i(VALUE key, VALUE val, VALUE _arg)
     JSON_Generator_State *state = arg->state;
     VALUE Vstate = arg->Vstate;
 
-    char *object_nl = state->object_nl;
-    long object_nl_len = state->object_nl_len;
-    char *indent = state->indent;
-    long indent_len = state->indent_len;
-    char *delim = FBUFFER_PTR(state->object_delim);
-    long delim_len = FBUFFER_LEN(state->object_delim);
-    char *delim2 = FBUFFER_PTR(state->object_delim2);
-    long delim2_len = FBUFFER_LEN(state->object_delim2);
     long depth = state->depth;
     int j;
-    VALUE klass, key_to_s;
 
-    if (arg->iter > 0) fbuffer_append(buffer, delim, delim_len);
-    if (object_nl) {
-        fbuffer_append(buffer, object_nl, object_nl_len);
+    if (arg->iter > 0) fbuffer_append_char(buffer, ',');
+    if (RB_UNLIKELY(state->object_nl)) {
+        fbuffer_append(buffer, state->object_nl, state->object_nl_len);
     }
-    if (indent) {
+    if (RB_UNLIKELY(state->indent)) {
         for (j = 0; j < depth; j++) {
-            fbuffer_append(buffer, indent, indent_len);
+            fbuffer_append(buffer, state->indent, state->indent_len);
         }
     }
 
-    klass = CLASS_OF(key);
-    if (klass == rb_cString) {
-        key_to_s = key;
-    } else if (klass == rb_cSymbol) {
-        key_to_s = rb_sym2str(key);
-    } else {
-        key_to_s = rb_funcall(key, i_to_s, 0);
+    VALUE key_to_s;
+    switch(rb_type(key)) {
+        case T_STRING:
+            key_to_s = key;
+            break;
+        case T_SYMBOL:
+            key_to_s = rb_sym2str(key);
+            break;
+        default:
+            key_to_s = rb_convert_type(key, T_STRING, "String", "to_s");
+            break;
     }
-    Check_Type(key_to_s, T_STRING);
-    generate_json(buffer, Vstate, state, key_to_s);
-    fbuffer_append(buffer, delim2, delim2_len);
+
+    generate_json_string(buffer, Vstate, state, key_to_s);
+    if (RB_UNLIKELY(state->space_before)) fbuffer_append(buffer, state->space_before, state->space_before_len);
+    fbuffer_append_char(buffer, ':');
+    if (RB_UNLIKELY(state->space)) fbuffer_append(buffer, state->space, state->space_len);
     generate_json(buffer, Vstate, state, val);
 
     arg->iter++;
@@ -882,10 +673,6 @@ json_object_i(VALUE key, VALUE val, VALUE _arg)
 
 static void generate_json_object(FBuffer *buffer, VALUE Vstate, JSON_Generator_State *state, VALUE obj)
 {
-    char *object_nl = state->object_nl;
-    long object_nl_len = state->object_nl_len;
-    char *indent = state->indent;
-    long indent_len = state->indent_len;
     long max_nesting = state->max_nesting;
     long depth = ++state->depth;
     int j;
@@ -903,11 +690,11 @@ static void generate_json_object(FBuffer *buffer, VALUE Vstate, JSON_Generator_S
     rb_hash_foreach(obj, json_object_i, (VALUE)&arg);
 
     depth = --state->depth;
-    if (object_nl) {
-        fbuffer_append(buffer, object_nl, object_nl_len);
-        if (indent) {
+    if (RB_UNLIKELY(state->object_nl)) {
+        fbuffer_append(buffer, state->object_nl, state->object_nl_len);
+        if (RB_UNLIKELY(state->indent)) {
             for (j = 0; j < depth; j++) {
-                fbuffer_append(buffer, indent, indent_len);
+                fbuffer_append(buffer, state->indent, state->indent_len);
             }
         }
     }
@@ -916,62 +703,90 @@ static void generate_json_object(FBuffer *buffer, VALUE Vstate, JSON_Generator_S
 
 static void generate_json_array(FBuffer *buffer, VALUE Vstate, JSON_Generator_State *state, VALUE obj)
 {
-    char *array_nl = state->array_nl;
-    long array_nl_len = state->array_nl_len;
-    char *indent = state->indent;
-    long indent_len = state->indent_len;
     long max_nesting = state->max_nesting;
-    char *delim = FBUFFER_PTR(state->array_delim);
-    long delim_len = FBUFFER_LEN(state->array_delim);
     long depth = ++state->depth;
     int i, j;
     if (max_nesting != 0 && depth > max_nesting) {
         rb_raise(eNestingError, "nesting of %ld is too deep", --state->depth);
     }
     fbuffer_append_char(buffer, '[');
-    if (array_nl) fbuffer_append(buffer, array_nl, array_nl_len);
+    if (RB_UNLIKELY(state->array_nl)) fbuffer_append(buffer, state->array_nl, state->array_nl_len);
     for(i = 0; i < RARRAY_LEN(obj); i++) {
-        if (i > 0) fbuffer_append(buffer, delim, delim_len);
-        if (indent) {
+        if (i > 0) {
+            fbuffer_append_char(buffer, ',');
+            if (RB_UNLIKELY(state->array_nl)) fbuffer_append(buffer, state->array_nl, state->array_nl_len);
+        }
+        if (RB_UNLIKELY(state->indent)) {
             for (j = 0; j < depth; j++) {
-                fbuffer_append(buffer, indent, indent_len);
+                fbuffer_append(buffer, state->indent, state->indent_len);
             }
         }
-        generate_json(buffer, Vstate, state, rb_ary_entry(obj, i));
+        generate_json(buffer, Vstate, state, RARRAY_AREF(obj, i));
     }
     state->depth = --depth;
-    if (array_nl) {
-        fbuffer_append(buffer, array_nl, array_nl_len);
-        if (indent) {
+    if (RB_UNLIKELY(state->array_nl)) {
+        fbuffer_append(buffer, state->array_nl, state->array_nl_len);
+        if (RB_UNLIKELY(state->indent)) {
             for (j = 0; j < depth; j++) {
-                fbuffer_append(buffer, indent, indent_len);
+                fbuffer_append(buffer, state->indent, state->indent_len);
             }
         }
     }
     fbuffer_append_char(buffer, ']');
 }
 
-#ifdef HAVE_RUBY_ENCODING_H
-static int enc_utf8_compatible_p(rb_encoding *enc)
+static int usascii_encindex, utf8_encindex, binary_encindex;
+
+static inline int enc_utf8_compatible_p(int enc_idx)
 {
-    if (enc == rb_usascii_encoding()) return 1;
-    if (enc == rb_utf8_encoding()) return 1;
+    if (enc_idx == usascii_encindex) return 1;
+    if (enc_idx == utf8_encindex) return 1;
     return 0;
 }
-#endif
+
+static inline VALUE ensure_valid_encoding(VALUE str)
+{
+    int encindex = RB_ENCODING_GET(str);
+    VALUE utf8_string;
+    if (RB_UNLIKELY(!enc_utf8_compatible_p(encindex))) {
+        if (encindex == binary_encindex) {
+            // For historical reason, we silently reinterpret binary strings as UTF-8 if it would work.
+            // TODO: Deprecate in 2.8.0
+            // TODO: Remove in 3.0.0
+            utf8_string = rb_enc_associate_index(rb_str_dup(str), utf8_encindex);
+            switch (rb_enc_str_coderange(utf8_string)) {
+                case ENC_CODERANGE_7BIT:
+                case ENC_CODERANGE_VALID:
+                    return utf8_string;
+                    break;
+            }
+        }
+
+        str = rb_funcall(str, i_encode, 1, Encoding_UTF_8);
+    }
+    return str;
+}
 
 static void generate_json_string(FBuffer *buffer, VALUE Vstate, JSON_Generator_State *state, VALUE obj)
 {
+    obj = ensure_valid_encoding(obj);
+
     fbuffer_append_char(buffer, '"');
-#ifdef HAVE_RUBY_ENCODING_H
-    if (!enc_utf8_compatible_p(rb_enc_get(obj))) {
-        obj = rb_str_export_to_enc(obj, rb_utf8_encoding());
-    }
-#endif
-    if (state->ascii_only) {
-        convert_UTF8_to_JSON_ASCII(buffer, obj, state->script_safe);
-    } else {
-        convert_UTF8_to_JSON(buffer, obj, state->script_safe);
+
+    switch(rb_enc_str_coderange(obj)) {
+        case ENC_CODERANGE_7BIT:
+            convert_ASCII_to_JSON(buffer, obj, state->script_safe ? script_safe_escape_table : escape_table);
+            break;
+        case ENC_CODERANGE_VALID:
+            if (RB_UNLIKELY(state->ascii_only)) {
+                convert_UTF8_to_ASCII_only_JSON(buffer, obj, state->script_safe ? script_safe_escape_table : escape_table, state->script_safe);
+            } else {
+                convert_UTF8_to_JSON(buffer, obj, state->script_safe ? script_safe_escape_table : escape_table, state->script_safe);
+            }
+            break;
+        default:
+            rb_raise(rb_path2class("JSON::GeneratorError"), "source sequence is illegal/malformed utf-8");
+            break;
     }
     fbuffer_append_char(buffer, '"');
 }
@@ -1018,9 +833,9 @@ static void generate_json_float(FBuffer *buffer, VALUE Vstate, JSON_Generator_St
     VALUE tmp = rb_funcall(obj, i_to_s, 0);
     if (!allow_nan) {
         if (isinf(value)) {
-            rb_raise(eGeneratorError, "%"PRIsVALUE" not allowed in JSON", RB_OBJ_STRING(tmp));
+            rb_raise(eGeneratorError, "%"PRIsVALUE" not allowed in JSON", tmp);
         } else if (isnan(value)) {
-            rb_raise(eGeneratorError, "%"PRIsVALUE" not allowed in JSON", RB_OBJ_STRING(tmp));
+            rb_raise(eGeneratorError, "%"PRIsVALUE" not allowed in JSON", tmp);
         }
     }
     fbuffer_append_str(buffer, tmp);
@@ -1029,35 +844,56 @@ static void generate_json_float(FBuffer *buffer, VALUE Vstate, JSON_Generator_St
 static void generate_json(FBuffer *buffer, VALUE Vstate, JSON_Generator_State *state, VALUE obj)
 {
     VALUE tmp;
-    VALUE klass = CLASS_OF(obj);
-    if (klass == rb_cHash) {
-        generate_json_object(buffer, Vstate, state, obj);
-    } else if (klass == rb_cArray) {
-        generate_json_array(buffer, Vstate, state, obj);
-    } else if (klass == rb_cString) {
-        generate_json_string(buffer, Vstate, state, obj);
-    } else if (obj == Qnil) {
+    if (obj == Qnil) {
         generate_json_null(buffer, Vstate, state, obj);
     } else if (obj == Qfalse) {
         generate_json_false(buffer, Vstate, state, obj);
     } else if (obj == Qtrue) {
         generate_json_true(buffer, Vstate, state, obj);
-    } else if (FIXNUM_P(obj)) {
-        generate_json_fixnum(buffer, Vstate, state, obj);
-    } else if (RB_TYPE_P(obj, T_BIGNUM)) {
-        generate_json_bignum(buffer, Vstate, state, obj);
-    } else if (klass == rb_cFloat) {
-        generate_json_float(buffer, Vstate, state, obj);
-    } else if (state->strict) {
-        rb_raise(eGeneratorError, "%"PRIsVALUE" not allowed in JSON", RB_OBJ_STRING(CLASS_OF(obj)));
-    } else if (rb_respond_to(obj, i_to_json)) {
-        tmp = rb_funcall(obj, i_to_json, 1, Vstate);
-        Check_Type(tmp, T_STRING);
-        fbuffer_append_str(buffer, tmp);
+    } else if (RB_SPECIAL_CONST_P(obj)) {
+        if (RB_FIXNUM_P(obj)) {
+            generate_json_fixnum(buffer, Vstate, state, obj);
+        } else if (RB_FLONUM_P(obj)) {
+            generate_json_float(buffer, Vstate, state, obj);
+        } else {
+            goto general;
+        }
     } else {
-        tmp = rb_funcall(obj, i_to_s, 0);
-        Check_Type(tmp, T_STRING);
-        generate_json_string(buffer, Vstate, state, tmp);
+        VALUE klass = RBASIC_CLASS(obj);
+        switch (RB_BUILTIN_TYPE(obj)) {
+            case T_BIGNUM:
+                generate_json_bignum(buffer, Vstate, state, obj);
+                break;
+            case T_HASH:
+                if (klass != rb_cHash) goto general;
+                generate_json_object(buffer, Vstate, state, obj);
+                break;
+            case T_ARRAY:
+                if (klass != rb_cArray) goto general;
+                generate_json_array(buffer, Vstate, state, obj);
+                break;
+            case T_STRING:
+                if (klass != rb_cString) goto general;
+                generate_json_string(buffer, Vstate, state, obj);
+                break;
+            case T_FLOAT:
+                if (klass != rb_cFloat) goto general;
+                generate_json_float(buffer, Vstate, state, obj);
+                break;
+            default:
+            general:
+                if (state->strict) {
+                    rb_raise(eGeneratorError, "%"PRIsVALUE" not allowed in JSON", CLASS_OF(obj));
+                } else if (rb_respond_to(obj, i_to_json)) {
+                    tmp = rb_funcall(obj, i_to_json, 1, Vstate);
+                    Check_Type(tmp, T_STRING);
+                    fbuffer_append_str(buffer, tmp);
+                } else {
+                    tmp = rb_funcall(obj, i_to_s, 0);
+                    Check_Type(tmp, T_STRING);
+                    generate_json_string(buffer, Vstate, state, tmp);
+                }
+        }
     }
 }
 
@@ -1067,28 +903,6 @@ static FBuffer *cState_prepare_buffer(VALUE self)
     GET_STATE(self);
     buffer = fbuffer_alloc(state->buffer_initial_length);
 
-    if (state->object_delim) {
-        fbuffer_clear(state->object_delim);
-    } else {
-        state->object_delim = fbuffer_alloc(16);
-    }
-    fbuffer_append_char(state->object_delim, ',');
-    if (state->object_delim2) {
-        fbuffer_clear(state->object_delim2);
-    } else {
-        state->object_delim2 = fbuffer_alloc(16);
-    }
-    if (state->space_before) fbuffer_append(state->object_delim2, state->space_before, state->space_before_len);
-    fbuffer_append_char(state->object_delim2, ':');
-    if (state->space) fbuffer_append(state->object_delim2, state->space, state->space_len);
-
-    if (state->array_delim) {
-        fbuffer_clear(state->array_delim);
-    } else {
-        state->array_delim = fbuffer_alloc(16);
-    }
-    fbuffer_append_char(state->array_delim, ',');
-    if (state->array_nl) fbuffer_append(state->array_delim, state->array_nl, state->array_nl_len);
     return buffer;
 }
 
@@ -1149,34 +963,9 @@ static VALUE cState_generate(VALUE self, VALUE obj)
     return result;
 }
 
-/*
- * call-seq: new(opts = {})
- *
- * Instantiates a new State object, configured by _opts_.
- *
- * _opts_ can have the following keys:
- *
- * * *indent*: a string used to indent levels (default: ''),
- * * *space*: a string that is put after, a : or , delimiter (default: ''),
- * * *space_before*: a string that is put before a : pair delimiter (default: ''),
- * * *object_nl*: a string that is put at the end of a JSON object (default: ''),
- * * *array_nl*: a string that is put at the end of a JSON array (default: ''),
- * * *allow_nan*: true if NaN, Infinity, and -Infinity should be
- *   generated, otherwise an exception is thrown, if these values are
- *   encountered. This options defaults to false.
- * * *ascii_only*: true if only ASCII characters should be generated. This
- *   option defaults to false.
- * * *buffer_initial_length*: sets the initial length of the generator's
- *   internal buffer.
- */
 static VALUE cState_initialize(int argc, VALUE *argv, VALUE self)
 {
-    VALUE opts;
-    GET_STATE(self);
-    state->max_nesting = 100;
-    state->buffer_initial_length = FBUFFER_INITIAL_LENGTH_DEFAULT;
-    rb_scan_args(argc, argv, "01", &opts);
-    if (!NIL_P(opts)) cState_configure(self, opts);
+    rb_warn("The json gem extension was loaded with the stdlib ruby code. You should upgrade rubygems with `gem update --system`");
     return self;
 }
 
@@ -1201,9 +990,6 @@ static VALUE cState_init_copy(VALUE obj, VALUE orig)
     objState->space_before = fstrndup(origState->space_before, origState->space_before_len);
     objState->object_nl = fstrndup(origState->object_nl, origState->object_nl_len);
     objState->array_nl = fstrndup(origState->array_nl, origState->array_nl_len);
-    if (origState->array_delim) objState->array_delim = fbuffer_dup(origState->array_delim);
-    if (origState->object_delim) objState->object_delim = fbuffer_dup(origState->object_delim);
-    if (origState->object_delim2) objState->object_delim2 = fbuffer_dup(origState->object_delim2);
     return obj;
 }
 
@@ -1442,7 +1228,8 @@ static VALUE cState_max_nesting_set(VALUE self, VALUE depth)
 {
     GET_STATE(self);
     Check_Type(depth, T_FIXNUM);
-    return state->max_nesting = FIX2LONG(depth);
+    state->max_nesting = FIX2LONG(depth);
+    return Qnil;
 }
 
 /*
@@ -1513,6 +1300,18 @@ static VALUE cState_allow_nan_p(VALUE self)
     return state->allow_nan ? Qtrue : Qfalse;
 }
 
+/*
+ * call-seq: allow_nan=(enable)
+ *
+ * This sets whether or not to serialize NaN, Infinity, and -Infinity
+ */
+static VALUE cState_allow_nan_set(VALUE self, VALUE enable)
+{
+    GET_STATE(self);
+    state->allow_nan = RTEST(enable);
+    return Qnil;
+}
+
 /*
  * call-seq: ascii_only?
  *
@@ -1525,6 +1324,18 @@ static VALUE cState_ascii_only_p(VALUE self)
     return state->ascii_only ? Qtrue : Qfalse;
 }
 
+/*
+ * call-seq: ascii_only=(enable)
+ *
+ * This sets whether only ASCII characters should be generated.
+ */
+static VALUE cState_ascii_only_set(VALUE self, VALUE enable)
+{
+    GET_STATE(self);
+    state->ascii_only = RTEST(enable);
+    return Qnil;
+}
+
 /*
  * call-seq: depth
  *
@@ -1592,8 +1403,8 @@ void Init_generator(void)
     rb_require("json/common");
 
     mJSON = rb_define_module("JSON");
-    mExt = rb_define_module_under(mJSON, "Ext");
-    mGenerator = rb_define_module_under(mExt, "Generator");
+    VALUE mExt = rb_define_module_under(mJSON, "Ext");
+    VALUE mGenerator = rb_define_module_under(mExt, "Generator");
 
     eGeneratorError = rb_path2class("JSON::GeneratorError");
     eNestingError = rb_path2class("JSON::NestingError");
@@ -1604,6 +1415,8 @@ void Init_generator(void)
     rb_define_alloc_func(cState, cState_s_allocate);
     rb_define_singleton_method(cState, "from_state", cState_from_state_s, 1);
     rb_define_method(cState, "initialize", cState_initialize, -1);
+    rb_define_alias(cState, "initialize", "initialize"); // avoid method redefinition warnings
+
     rb_define_method(cState, "initialize_copy", cState_init_copy, 1);
     rb_define_method(cState, "indent", cState_indent, 0);
     rb_define_method(cState, "indent=", cState_indent_set, 1);
@@ -1628,76 +1441,72 @@ void Init_generator(void)
     rb_define_method(cState, "strict=", cState_strict_set, 1);
     rb_define_method(cState, "check_circular?", cState_check_circular_p, 0);
     rb_define_method(cState, "allow_nan?", cState_allow_nan_p, 0);
+    rb_define_method(cState, "allow_nan=", cState_allow_nan_set, 1);
     rb_define_method(cState, "ascii_only?", cState_ascii_only_p, 0);
+    rb_define_method(cState, "ascii_only=", cState_ascii_only_set, 1);
     rb_define_method(cState, "depth", cState_depth, 0);
     rb_define_method(cState, "depth=", cState_depth_set, 1);
     rb_define_method(cState, "buffer_initial_length", cState_buffer_initial_length, 0);
     rb_define_method(cState, "buffer_initial_length=", cState_buffer_initial_length_set, 1);
-    rb_define_method(cState, "configure", cState_configure, 1);
-    rb_define_alias(cState, "merge", "configure");
-    rb_define_method(cState, "to_h", cState_to_h, 0);
-    rb_define_alias(cState, "to_hash", "to_h");
-    rb_define_method(cState, "[]", cState_aref, 1);
-    rb_define_method(cState, "[]=", cState_aset, 2);
     rb_define_method(cState, "generate", cState_generate, 1);
 
-    mGeneratorMethods = rb_define_module_under(mGenerator, "GeneratorMethods");
-    mObject = rb_define_module_under(mGeneratorMethods, "Object");
+    VALUE mGeneratorMethods = rb_define_module_under(mGenerator, "GeneratorMethods");
+
+    VALUE mObject = rb_define_module_under(mGeneratorMethods, "Object");
     rb_define_method(mObject, "to_json", mObject_to_json, -1);
-    mHash = rb_define_module_under(mGeneratorMethods, "Hash");
+
+    VALUE mHash = rb_define_module_under(mGeneratorMethods, "Hash");
     rb_define_method(mHash, "to_json", mHash_to_json, -1);
-    mArray = rb_define_module_under(mGeneratorMethods, "Array");
+
+    VALUE mArray = rb_define_module_under(mGeneratorMethods, "Array");
     rb_define_method(mArray, "to_json", mArray_to_json, -1);
+
 #ifdef RUBY_INTEGER_UNIFICATION
-    mInteger = rb_define_module_under(mGeneratorMethods, "Integer");
+    VALUE mInteger = rb_define_module_under(mGeneratorMethods, "Integer");
     rb_define_method(mInteger, "to_json", mInteger_to_json, -1);
 #else
-    mFixnum = rb_define_module_under(mGeneratorMethods, "Fixnum");
+    VALUE mFixnum = rb_define_module_under(mGeneratorMethods, "Fixnum");
     rb_define_method(mFixnum, "to_json", mFixnum_to_json, -1);
-    mBignum = rb_define_module_under(mGeneratorMethods, "Bignum");
+
+    VALUE mBignum = rb_define_module_under(mGeneratorMethods, "Bignum");
     rb_define_method(mBignum, "to_json", mBignum_to_json, -1);
 #endif
-    mFloat = rb_define_module_under(mGeneratorMethods, "Float");
+    VALUE mFloat = rb_define_module_under(mGeneratorMethods, "Float");
     rb_define_method(mFloat, "to_json", mFloat_to_json, -1);
-    mString = rb_define_module_under(mGeneratorMethods, "String");
+
+    VALUE mString = rb_define_module_under(mGeneratorMethods, "String");
     rb_define_singleton_method(mString, "included", mString_included_s, 1);
     rb_define_method(mString, "to_json", mString_to_json, -1);
     rb_define_method(mString, "to_json_raw", mString_to_json_raw, -1);
     rb_define_method(mString, "to_json_raw_object", mString_to_json_raw_object, 0);
+
     mString_Extend = rb_define_module_under(mString, "Extend");
     rb_define_method(mString_Extend, "json_create", mString_Extend_json_create, 1);
-    mTrueClass = rb_define_module_under(mGeneratorMethods, "TrueClass");
+
+    VALUE mTrueClass = rb_define_module_under(mGeneratorMethods, "TrueClass");
     rb_define_method(mTrueClass, "to_json", mTrueClass_to_json, -1);
-    mFalseClass = rb_define_module_under(mGeneratorMethods, "FalseClass");
+
+    VALUE mFalseClass = rb_define_module_under(mGeneratorMethods, "FalseClass");
     rb_define_method(mFalseClass, "to_json", mFalseClass_to_json, -1);
-    mNilClass = rb_define_module_under(mGeneratorMethods, "NilClass");
+
+    VALUE mNilClass = rb_define_module_under(mGeneratorMethods, "NilClass");
     rb_define_method(mNilClass, "to_json", mNilClass_to_json, -1);
 
+    rb_global_variable(&Encoding_UTF_8);
+    Encoding_UTF_8 = rb_const_get(rb_path2class("Encoding"), rb_intern("UTF_8"));
+
     i_to_s = rb_intern("to_s");
     i_to_json = rb_intern("to_json");
     i_new = rb_intern("new");
-    i_indent = rb_intern("indent");
-    i_space = rb_intern("space");
-    i_space_before = rb_intern("space_before");
-    i_object_nl = rb_intern("object_nl");
-    i_array_nl = rb_intern("array_nl");
-    i_max_nesting = rb_intern("max_nesting");
-    i_script_safe = rb_intern("script_safe");
-    i_escape_slash = rb_intern("escape_slash");
-    i_strict = rb_intern("strict");
-    i_allow_nan = rb_intern("allow_nan");
-    i_ascii_only = rb_intern("ascii_only");
-    i_depth = rb_intern("depth");
-    i_buffer_initial_length = rb_intern("buffer_initial_length");
     i_pack = rb_intern("pack");
     i_unpack = rb_intern("unpack");
     i_create_id = rb_intern("create_id");
     i_extend = rb_intern("extend");
-    i_key_p = rb_intern("key?");
-    i_aref = rb_intern("[]");
-    i_send = rb_intern("__send__");
-    i_respond_to_p = rb_intern("respond_to?");
-    i_match = rb_intern("match");
-    i_keys = rb_intern("keys");
-    i_dup = rb_intern("dup");
+    i_encode = rb_intern("encode");
+
+    usascii_encindex = rb_usascii_encindex();
+    utf8_encindex = rb_utf8_encindex();
+    binary_encindex = rb_ascii8bit_encindex();
+
+    rb_require("json/ext/generator/state");
 }