oj 3.16.11 → 3.17.6

data/ext/oj/dump.c

@@ -40,7 +40,7 @@ static size_t ascii_friendly_size(const uint8_t *str, size_t len);
 static const char hex_chars[17] = "0123456789abcdef";
 
 // JSON standard except newlines are no escaped
-static char newline_friendly_chars[256] = "\
+static char newline_friendly_chars[257] = "\
 66666666221622666666666666666666\
 11211111111111111111111111111111\
 11111111111111111111111111112111\
@@ -51,7 +51,7 @@ static char newline_friendly_chars[256] = "\
 11111111111111111111111111111111";
 
 // JSON standard
-static char hibit_friendly_chars[256] = "\
+static char hibit_friendly_chars[257] = "\
 66666666222622666666666666666666\
 11211111111111111111111111111111\
 11111111111111111111111111112111\
@@ -62,7 +62,7 @@ static char hibit_friendly_chars[256] = "\
 11111111111111111111111111111111";
 
 // JSON standard but escape forward slashes `/`
-static char slash_friendly_chars[256] = "\
+static char slash_friendly_chars[257] = "\
 66666666222622666666666666666666\
 11211111111111121111111111111111\
 11111111111111111111111111112111\
@@ -74,7 +74,7 @@ static char slash_friendly_chars[256] = "\
 
 // High bit set characters are always encoded as unicode. Worse case is 3
 // bytes per character in the output. That makes this conservative.
-static char ascii_friendly_chars[256] = "\
+static char ascii_friendly_chars[257] = "\
 66666666222622666666666666666666\
 11211111111111111111111111111111\
 11111111111111111111111111112111\
@@ -85,7 +85,7 @@ static char ascii_friendly_chars[256] = "\
 33333333333333333333333333333333";
 
 // XSS safe mode
-static char xss_friendly_chars[256] = "\
+static char xss_friendly_chars[257] = "\
 66666666222622666666666666666666\
 11211161111111121111111111116161\
 11111111111111111111111111112111\
@@ -96,7 +96,7 @@ static char xss_friendly_chars[256] = "\
 33333333333333333333333333333333";
 
 // JSON XSS combo
-static char hixss_friendly_chars[256] = "\
+static char hixss_friendly_chars[257] = "\
 66666666222622666666666666666666\
 11211111111111111111111111111111\
 11111111111111111111111111112111\
@@ -107,7 +107,7 @@ static char hixss_friendly_chars[256] = "\
 11611111111111111111111111111111";
 
 // Rails XSS combo
-static char rails_xss_friendly_chars[256] = "\
+static char rails_xss_friendly_chars[257] = "\
 66666666222622666666666666666666\
 11211161111111111111111111116161\
 11111111111111111111111111112111\
@@ -118,7 +118,7 @@ static char rails_xss_friendly_chars[256] = "\
 11611111111111111111111111111111";
 
 // Rails HTML non-escape
-static char rails_friendly_chars[256] = "\
+static char rails_friendly_chars[257] = "\
 66666666222622666666666666666666\
 11211111111111111111111111111111\
 11111111111111111111111111112111\
@@ -201,6 +201,45 @@ void initialize_neon(void) {
 }
 #endif
 
+#ifdef HAVE_SIMD_SSE4_2
+
+static __m128i hibit_friendly_chars_sse42[8];
+
+// From: https://stackoverflow.com/questions/36998538/fastest-way-to-horizontally-sum-sse-unsigned-byte-vector
+inline static OJ_TARGET_SSE42 uint32_t _mm_sum_epu8(const __m128i v) {
+    __m128i vsum = _mm_sad_epu8(v, _mm_setzero_si128());
+    return _mm_cvtsi128_si32(vsum) + _mm_extract_epi16(vsum, 4);
+}
+
+inline static OJ_TARGET_SSE42 size_t hibit_friendly_size_sse42(const uint8_t *str, size_t len) {
+    size_t size = 0;
+    size_t i    = 0;
+
+    for (; i + sizeof(__m128i) <= len; i += sizeof(__m128i), str += sizeof(__m128i)) {
+        size += sizeof(__m128i);
+
+        __m128i chunk = _mm_loadu_si128((__m128i *)str);
+        __m128i tmp   = vector_lookup_sse42(chunk, hibit_friendly_chars_sse42, 8);
+        size += _mm_sum_epu8(tmp);
+    }
+    size_t total = size + calculate_string_size(str, len - i, hibit_friendly_chars);
+    return total;
+}
+
+void OJ_TARGET_SSE42 initialize_sse42(void) {
+    for (int i = 0; i < 8; i++) {
+        hibit_friendly_chars_sse42[i] = _mm_sub_epi8(
+            _mm_loadu_si128((__m128i *)(hibit_friendly_chars + i * sizeof(__m128i))),
+            _mm_set1_epi8('1'));
+    }
+}
+
+#else
+
+#define SIMD_TARGET
+
+#endif /* HAVE_SIMD_SSE4_2 */
+
 inline static size_t hibit_friendly_size(const uint8_t *str, size_t len) {
 #ifdef HAVE_SIMD_NEON
     size_t size = 0;
@@ -218,8 +257,38 @@ inline static size_t hibit_friendly_size(const uint8_t *str, size_t len) {
         size += tmp;
     }
 
+#ifdef HAVE_FAST_MEMCPY
+    size_t total = 0;
+    if (len - i > 4) {
+        size += (len - i);
+        unsigned char buf[sizeof(uint8x16_t)];
+        memset(buf, ' ', sizeof(buf));
+        fast_memcpy16(buf, str, len - i);
+
+        uint8x16_t chunk  = vld1q_u8((const unsigned char *)buf);
+        uint8x16_t tmp1   = vqtbl4q_u8(hibit_friendly_chars_neon[0], chunk);
+        uint8x16_t tmp2   = vqtbl4q_u8(hibit_friendly_chars_neon[1], veorq_u8(chunk, vdupq_n_u8(0x40)));
+        uint8x16_t result = vorrq_u8(tmp1, tmp2);
+        uint8_t    tmp    = vaddvq_u8(result);
+
+        size += tmp;
+
+        total = size;
+    } else {
+        total = size + calculate_string_size(str, len - i, hibit_friendly_chars);
+    }
+#else
     size_t total = size + calculate_string_size(str, len - i, hibit_friendly_chars);
+#endif
+
     return total;
+#elif defined(HAVE_SIMD_SSE4_2)
+    if (SIMD_Impl == SIMD_SSE42) {
+        if (len >= sizeof(__m128i)) {
+            return hibit_friendly_size_sse42(str, len);
+        }
+    }
+    return calculate_string_size(str, len, hibit_friendly_chars);
 #else
     return calculate_string_size(str, len, hibit_friendly_chars);
 #endif
@@ -252,38 +321,46 @@ inline static size_t hixss_friendly_size(const uint8_t *str, size_t len) {
 }
 
 inline static long rails_xss_friendly_size(const uint8_t *str, size_t len) {
-    long    size = 0;
-    uint8_t hi   = 0;
+    long     size = 0;
+    uint32_t hi   = 0;
 
 #ifdef HAVE_SIMD_NEON
     size_t i = 0;
 
-    uint8x16_t has_some_hibit = vdupq_n_u8(0);
-    uint8x16_t hibit          = vdupq_n_u8(0x80);
-    for (; i + sizeof(uint8x16_t) <= len; i += sizeof(uint8x16_t), str += sizeof(uint8x16_t)) {
-        size += sizeof(uint8x16_t);
+    if (len >= sizeof(uint8x16_t)) {
+        uint8x16_t has_some_hibit = vdupq_n_u8(0);
+        uint8x16_t hibit          = vdupq_n_u8(0x80);
 
-        uint8x16_t chunk = vld1q_u8(str);
+        for (; i + sizeof(uint8x16_t) <= len; i += sizeof(uint8x16_t), str += sizeof(uint8x16_t)) {
+            size += sizeof(uint8x16_t);
 
-        // Check to see if any of these bytes have the high bit set.
-        has_some_hibit = vorrq_u8(has_some_hibit, vandq_u8(chunk, hibit));
+            uint8x16_t chunk = vld1q_u8(str);
 
-        uint8x16_t tmp1   = vqtbl4q_u8(rails_xss_friendly_chars_neon[0], chunk);
-        uint8x16_t tmp2   = vqtbl4q_u8(rails_xss_friendly_chars_neon[1], veorq_u8(chunk, vdupq_n_u8(0x40)));
-        uint8x16_t tmp3   = vqtbl4q_u8(rails_xss_friendly_chars_neon[2], veorq_u8(chunk, vdupq_n_u8(0x80)));
-        uint8x16_t tmp4   = vqtbl4q_u8(rails_xss_friendly_chars_neon[3], veorq_u8(chunk, vdupq_n_u8(0xc0)));
-        uint8x16_t result = vorrq_u8(tmp4, vorrq_u8(tmp3, vorrq_u8(tmp1, tmp2)));
-        uint8_t    tmp    = vaddvq_u8(result);
-        size += tmp;
+            // Check to see if any of these bytes have the high bit set.
+            has_some_hibit = vorrq_u8(has_some_hibit, vandq_u8(chunk, hibit));
+
+            uint8x16_t tmp1   = vqtbl4q_u8(rails_xss_friendly_chars_neon[0], chunk);
+            uint8x16_t tmp2   = vqtbl4q_u8(rails_xss_friendly_chars_neon[1], veorq_u8(chunk, vdupq_n_u8(0x40)));
+            uint8x16_t tmp3   = vqtbl4q_u8(rails_xss_friendly_chars_neon[2], veorq_u8(chunk, vdupq_n_u8(0x80)));
+            uint8x16_t tmp4   = vqtbl4q_u8(rails_xss_friendly_chars_neon[3], veorq_u8(chunk, vdupq_n_u8(0xc0)));
+            uint8x16_t result = vorrq_u8(tmp4, vorrq_u8(tmp3, vorrq_u8(tmp1, tmp2)));
+            uint8_t    tmp    = vaddvq_u8(result);
+            size += tmp;
+        }
+
+        // 'hi' should be set if any of the bytes we processed have the high bit set. It doesn't matter which ones.
+        hi = vmaxvq_u8(has_some_hibit) != 0;
     }
 
-    // 'hi' should be set if any of the bytes we processed have the high bit set. It doesn't matter which ones.
-    hi = vmaxvq_u8(has_some_hibit) != 0;
+    size_t len_remaining = len - i;
 
     for (; i < len; str++, i++) {
-        size += rails_xss_friendly_chars[*str] - '0';
+        size += rails_xss_friendly_chars[*str];
         hi |= *str & 0x80;
     }
+
+    size -= (len_remaining * ((size_t)'0'));
+
     if (0 == hi) {
         return size;
     }
@@ -302,37 +379,43 @@ inline static long rails_xss_friendly_size(const uint8_t *str, size_t len) {
 }
 
 inline static size_t rails_friendly_size(const uint8_t *str, size_t len) {
-    long    size = 0;
-    uint8_t hi   = 0;
+    long     size = 0;
+    uint32_t hi   = 0;
 #ifdef HAVE_SIMD_NEON
-    size_t i = 0;
+    size_t i     = 0;
+    long   extra = 0;
 
-    uint8x16_t has_some_hibit = vdupq_n_u8(0);
-    uint8x16_t hibit          = vdupq_n_u8(0x80);
+    if (len >= sizeof(uint8x16_t)) {
+        uint8x16_t has_some_hibit = vdupq_n_u8(0);
+        uint8x16_t hibit          = vdupq_n_u8(0x80);
 
-    for (; i + sizeof(uint8x16_t) <= len; i += sizeof(uint8x16_t), str += sizeof(uint8x16_t)) {
-        size += sizeof(uint8x16_t);
+        for (; i + sizeof(uint8x16_t) <= len; i += sizeof(uint8x16_t), str += sizeof(uint8x16_t)) {
+            size += sizeof(uint8x16_t);
 
-        // See https://lemire.me/blog/2019/07/23/arbitrary-byte-to-byte-maps-using-arm-neon/
-        uint8x16_t chunk = vld1q_u8(str);
+            // See https://lemire.me/blog/2019/07/23/arbitrary-byte-to-byte-maps-using-arm-neon/
+            uint8x16_t chunk = vld1q_u8(str);
 
-        // Check to see if any of these bytes have the high bit set.
-        has_some_hibit = vorrq_u8(has_some_hibit, vandq_u8(chunk, hibit));
+            // Check to see if any of these bytes have the high bit set.
+            has_some_hibit = vorrq_u8(has_some_hibit, vandq_u8(chunk, hibit));
 
-        uint8x16_t tmp1   = vqtbl4q_u8(rails_friendly_chars_neon[0], chunk);
-        uint8x16_t tmp2   = vqtbl4q_u8(rails_friendly_chars_neon[1], veorq_u8(chunk, vdupq_n_u8(0x40)));
-        uint8x16_t result = vorrq_u8(tmp1, tmp2);
-        uint8_t    tmp    = vaddvq_u8(result);
-        size += tmp;
-    }
+            uint8x16_t tmp1   = vqtbl4q_u8(rails_friendly_chars_neon[0], chunk);
+            uint8x16_t tmp2   = vqtbl4q_u8(rails_friendly_chars_neon[1], veorq_u8(chunk, vdupq_n_u8(0x40)));
+            uint8x16_t result = vorrq_u8(tmp1, tmp2);
+            uint8_t    tmp    = vaddvq_u8(result);
+            size += tmp;
+        }
 
-    // 'hi' should be set if any of the bytes we processed have the high bit set. It doesn't matter which ones.
-    hi = vmaxvq_u8(has_some_hibit) != 0;
+        // 'hi' should be set if any of the bytes we processed have the high bit set. It doesn't matter which ones.
+        hi = vmaxvq_u8(has_some_hibit) != 0;
+    }
 
-    for (; i < len; str++, i++) {
-        size += rails_friendly_chars[*str] - '0';
+    for (; i < len; str++, i++, extra++) {
+        size += rails_friendly_chars[*str];
         hi |= *str & 0x80;
     }
+
+    size -= (extra * ((size_t)'0'));
+
     if (0 == hi) {
         return size;
     }
@@ -896,22 +979,23 @@ void oj_dump_raw_json(VALUE obj, int depth, Out out) {
     }
 }
 
+#if defined(__clang__) || defined(__GNUC__)
+#define FORCE_INLINE __attribute__((always_inline))
+#else
+#define FORCE_INLINE
+#endif
+
 #ifdef HAVE_SIMD_NEON
 typedef struct _neon_match_result {
     uint8x16_t needs_escape;
     bool       has_some_hibit;
     bool       do_unicode_validation;
+    uint64_t   escape_mask;
 } neon_match_result;
 
-#if defined(__clang__) || defined(__GNUC__)
-#define FORCE_INLINE __attribute__((always_inline))
-#else
-#define FORCE_INLINE
-#endif
-
 static inline FORCE_INLINE neon_match_result
 neon_update(const char *str, uint8x16x4_t *cmap_neon, int neon_table_size, bool do_unicode_validation, bool has_hi) {
-    neon_match_result result = {.has_some_hibit = false, .do_unicode_validation = false};
+    neon_match_result result = {.has_some_hibit = false, .do_unicode_validation = false, .escape_mask = 0};
 
     uint8x16_t chunk    = vld1q_u8((const unsigned char *)str);
     uint8x16_t tmp1     = vqtbl4q_u8(cmap_neon[0], chunk);
@@ -927,17 +1011,122 @@ neon_update(const char *str, uint8x16x4_t *cmap_neon, int neon_table_size, bool
         result.has_some_hibit        = vmaxvq_u8(has_some_hibit) != 0;
         result.do_unicode_validation = has_hi && do_unicode_validation && result.has_some_hibit;
     }
+    const uint8x8_t res  = vshrn_n_u16(vreinterpretq_u16_u8(vmvnq_u8(vceqq_u8(result.needs_escape, vdupq_n_u8(0)))), 4);
+    const uint64_t  mask = vget_lane_u64(vreinterpret_u64_u8(res), 0);
+    result.escape_mask   = mask & 0x8888888888888888ull;
+    return result;
+}
+
+#elif defined(HAVE_SIMD_SSE4_2)
+typedef struct _sse42_match_result {
+    __m128i actions;
+    bool    needs_escape;
+    int     escape_mask;
+    bool    has_some_hibit;
+    bool    do_unicode_validation;
+} sse42_match_result;
+
+static inline OJ_TARGET_SSE42 sse42_match_result
+sse42_update(const char *str, __m128i *cmap_sse42, int sse42_tab_size, bool do_unicode_validation, bool has_hi) {
+    sse42_match_result result = {.has_some_hibit = false, .do_unicode_validation = false};
+
+    __m128i chunk        = _mm_loadu_si128((__m128i *)str);
+    __m128i actions      = vector_lookup_sse42(chunk, cmap_sse42, sse42_tab_size);
+    __m128i needs_escape = _mm_xor_si128(_mm_cmpeq_epi8(actions, _mm_setzero_si128()), _mm_set1_epi8(0xFF));
+    result.actions       = _mm_add_epi8(actions, _mm_set1_epi8('1'));
+
+    result.escape_mask  = _mm_movemask_epi8(needs_escape);
+    result.needs_escape = result.escape_mask != 0;
+    if (has_hi && do_unicode_validation) {
+        __m128i has_some_hibit       = _mm_and_si128(chunk, _mm_set1_epi8(0x80));
+        result.has_some_hibit        = _mm_movemask_epi8(has_some_hibit) != 0;
+        result.do_unicode_validation = has_hi && do_unicode_validation && result.has_some_hibit;
+    }
     return result;
 }
 
 #endif /* HAVE_SIMD_NEON */
 
+static inline FORCE_INLINE const char *process_character(char         action,
+                                                         const char  *str,
+                                                         const char  *end,
+                                                         Out          out,
+                                                         const char  *orig,
+                                                         bool         do_unicode_validation,
+                                                         const char **check_start_) {
+    const char *check_start = *check_start_;
+    switch (action) {
+    case '1':
+        if (do_unicode_validation && check_start <= str) {
+            if (0 != (0x80 & (uint8_t)*str)) {
+                if (0xC0 == (0xC0 & (uint8_t)*str)) {
+                    *check_start_ = check_unicode(str, end, orig);
+                } else {
+                    raise_invalid_unicode(orig, (int)(end - orig), (int)(str - orig));
+                }
+            }
+        }
+        *out->cur++ = *str;
+        break;
+    case '2':
+        *out->cur++ = '\\';
+        switch (*str) {
+        case '\\': *out->cur++ = '\\'; break;
+        case '\b': *out->cur++ = 'b'; break;
+        case '\t': *out->cur++ = 't'; break;
+        case '\n': *out->cur++ = 'n'; break;
+        case '\f': *out->cur++ = 'f'; break;
+        case '\r': *out->cur++ = 'r'; break;
+        default: *out->cur++ = *str; break;
+        }
+        break;
+    case '3':  // Unicode
+        if (0xe2 == (uint8_t)*str && do_unicode_validation && 2 <= end - str) {
+            if (0x80 == (uint8_t)str[1] && (0xa8 == (uint8_t)str[2] || 0xa9 == (uint8_t)str[2])) {
+                str = dump_unicode(str, end, out, orig);
+            } else {
+                *check_start_ = check_unicode(str, end, orig);
+                *out->cur++   = *str;
+            }
+            break;
+        }
+        str = dump_unicode(str, end, out, orig);
+        break;
+    case '6':  // control characters
+        if (*(uint8_t *)str < 0x80) {
+            if (0 == (uint8_t)*str && out->opts->dump_opts.omit_null_byte) {
+                break;
+            }
+            APPEND_CHARS(out->cur, "\\u00", 4);
+            dump_hex((uint8_t)*str, out);
+        } else {
+            if (0xe2 == (uint8_t)*str && do_unicode_validation && 2 <= end - str) {
+                if (0x80 == (uint8_t)str[1] && (0xa8 == (uint8_t)str[2] || 0xa9 == (uint8_t)str[2])) {
+                    str = dump_unicode(str, end, out, orig);
+                } else {
+                    *check_start_ = check_unicode(str, end, orig);
+                    *out->cur++   = *str;
+                }
+                break;
+            }
+            str = dump_unicode(str, end, out, orig);
+        }
+        break;
+    default: break;  // ignore, should never happen if the table is correct
+    }
+
+    return str;
+}
+
 void oj_dump_cstr(const char *str, size_t cnt, bool is_sym, bool escape1, Out out) {
     size_t size;
     char  *cmap;
 #ifdef HAVE_SIMD_NEON
-    uint8x16x4_t *cmap_neon = NULL;
-    int           neon_table_size;
+    uint8x16x4_t *cmap_neon       = NULL;
+    int           neon_table_size = 0;
+#elif defined(HAVE_SIMD_SSE4_2)
+    __m128i *cmap_sse42 = NULL;
+    int      sse42_tab_size;
 #endif /* HAVE_SIMD_NEON */
     const char *orig                  = str;
     bool        has_hi                = false;
@@ -1006,6 +1195,9 @@ void oj_dump_cstr(const char *str, size_t cnt, bool is_sym, bool escape1, Out ou
 #ifdef HAVE_SIMD_NEON
         cmap_neon       = hibit_friendly_chars_neon;
         neon_table_size = 2;
+#elif defined(HAVE_SIMD_SSE4_2)
+        cmap_sse42     = hibit_friendly_chars_sse42;
+        sse42_tab_size = 8;
 #endif /* HAVE_NEON_SIMD */
         size = hibit_friendly_size((uint8_t *)str, cnt);
     }
@@ -1024,7 +1216,16 @@ void oj_dump_cstr(const char *str, size_t cnt, bool is_sym, bool escape1, Out ou
         if (is_sym) {
             *out->cur++ = ':';
         }
+#ifdef HAVE_FAST_MEMCPY
+        if (cnt <= 16) {
+            fast_memcpy16(out->cur, str, cnt);
+            out->cur += size;
+        } else {
+            APPEND_CHARS(out->cur, str, cnt);
+        }
+#else
         APPEND_CHARS(out->cur, str, cnt);
+#endif
         *out->cur++ = '"';
     } else {
         const char *end         = str + cnt;
@@ -1033,175 +1234,164 @@ void oj_dump_cstr(const char *str, size_t cnt, bool is_sym, bool escape1, Out ou
         if (is_sym) {
             *out->cur++ = ':';
         }
-#ifdef HAVE_SIMD_NEON
-        const char *chunk_start;
-        const char *chunk_end;
-        const char *cursor     = str;
-        int         neon_state = (cmap_neon != NULL) ? 1 : 4;
-        char        matches[16];
-        bool        do_hi_validation = false;
-        // uint64_t neon_match_mask = 0;
+
+#if defined(HAVE_SIMD_NEON) || defined(HAVE_SIMD_SSE4_2)
+
 #define SEARCH_FLUSH                                  \
     if (str > cursor) {                               \
         APPEND_CHARS(out->cur, cursor, str - cursor); \
         cursor = str;                                 \
     }
 
-    loop:
-#endif /* HAVE_SIMD_NEON */
-        for (; str < end; str++) {
-            char action = 0;
+        const char *chunk_start;
+        const char *chunk_end;
+        const char *cursor = str;
+        char        matches[16];
+#endif /* HAVE_SIMD_NEON || HAVE_SIMD_SSE4_2 */
+
+#if defined(HAVE_SIMD_NEON)
+        bool use_simd = (cmap_neon != NULL && cnt >= (sizeof(uint8x16_t))) ? true : false;
+#elif defined(HAVE_SIMD_SSE4_2)
+        bool use_simd = false;
+        if (SIMD_Impl == SIMD_SSE42) {
+            use_simd = (cmap_sse42 != NULL && cnt >= (sizeof(__m128i))) ? true : false;
+        }
+#endif
+
 #ifdef HAVE_SIMD_NEON
-            /* neon_state:
-             * 1: Scanning for matches. There must be at least
-                  sizeof(uint8x16_t) bytes of input data to use SIMD and
-                  cmap_neon must be non-null.
-             * 2: Matches have been found. Will set str to the position of the
-             *    next match and set the state to 3.
-             *    If there are no more matches it will transition to state 1.
-             * 4: Fallback to the scalar algorithm. Not enough data to use
-             *    SIMD.
-             */
-#define NEON_SET_STATE(state) \
-    neon_state = state;       \
-    goto loop;
-#define NEON_RETURN_TO_STATE(state) neon_state = state;
-            switch (neon_state) {
-            case 1: {
-                while (true) {
+
+#ifdef HAVE_FAST_MEMCPY
+#define APPEND_CHARS_SMALL(dst, src, length) \
+    fast_memcpy16((dst), (src), (length));   \
+    (dst) += (length);
+#define MEMCPY16 fast_memcpy16
+#else
+#define APPEND_CHARS_SMALL(dst, src, length) APPEND_CHARS(dst, str, length);
+#define MEMCPY16 memcpy
+#endif
+
+        if (use_simd) {
+            while (str < end) {
+                const char *chunk_ptr = NULL;
+                if (str + sizeof(uint8x16_t) <= end) {
+                    chunk_ptr   = str;
+                    chunk_start = str;
+                    chunk_end   = str + sizeof(uint8x16_t);
+                } else if ((end - str) >= SIMD_MINIMUM_THRESHOLD) {
+                    memset(out->cur, ' ', sizeof(uint8x16_t));
+                    MEMCPY16(out->cur, str, (end - str));
+                    chunk_ptr   = out->cur;
+                    chunk_start = str;
+                    chunk_end   = end;
+                } else {
+                    break;
+                }
+                neon_match_result result = neon_update(chunk_ptr,
+                                                       cmap_neon,
+                                                       neon_table_size,
+                                                       do_unicode_validation,
+                                                       has_hi);
+                if ((result.do_unicode_validation) || result.escape_mask != 0) {
+                    SEARCH_FLUSH;
+                    bool       process_each = result.do_unicode_validation;
+                    uint8x16_t actions      = vaddq_u8(result.needs_escape, vdupq_n_u8('1'));
+                    vst1q_u8((unsigned char *)matches, actions);
+                    // If no byte in this chunk had the high bit set then we can skip
+                    // all of the '1' bytes by directly copying them to the output.
+                    if (!process_each) {
+                        while (result.escape_mask) {
+                            int  esc_pos = OJ_CTZ64(result.escape_mask) >> 2;
+                            long run_len = esc_pos - (str - chunk_start);
+                            if (run_len > 0) {
+                                APPEND_CHARS_SMALL(out->cur, str, run_len);
+                                str += run_len;
+                            }
+                            str = process_character(matches[esc_pos],
+                                                    str,
+                                                    end,
+                                                    out,
+                                                    orig,
+                                                    do_unicode_validation,
+                                                    &check_start);
+                            str++;
+                            result.escape_mask &= result.escape_mask - 1;
+                        }
+                        if (str < chunk_end) {
+                            APPEND_CHARS_SMALL(out->cur, str, chunk_end - str);
+                            str = chunk_end;
+                        }
+                    } else {
+                        while (str < chunk_end) {
+                            long match_index = str - chunk_start;
+                            str              = process_character(matches[match_index],
+                                                                 str,
+                                                                 end,
+                                                                 out,
+                                                                 orig,
+                                                                 do_unicode_validation,
+                                                                 &check_start);
+                            str++;
+                        }
+                    }
+                    cursor = str;
+                    continue;
+                }
+                str = chunk_end;
+            }
+            SEARCH_FLUSH;
+        }
+#endif
+
+#ifdef HAVE_SIMD_SSE4_2
+        if (SIMD_Impl == SIMD_SSE42) {
+            if (use_simd) {
+                while (str < end) {
                     const char *chunk_ptr = NULL;
-                    if (str + sizeof(uint8x16_t) <= end) {
+                    if (str + sizeof(__m128i) <= end) {
                         chunk_ptr   = str;
                         chunk_start = str;
-                        chunk_end   = str + sizeof(uint8x16_t);
+                        chunk_end   = str + sizeof(__m128i);
                     } else if ((end - str) >= SIMD_MINIMUM_THRESHOLD) {
-                        memset(out->cur, 'A', sizeof(uint8x16_t));
+                        memset(out->cur, 'A', sizeof(__m128i));
                         memcpy(out->cur, str, (end - str));
                         chunk_ptr   = out->cur;
                         chunk_start = str;
                         chunk_end   = end;
                     } else {
-                        SEARCH_FLUSH;
-                        NEON_SET_STATE(4);
-                        break; /* Unreachable */
+                        break;
                     }
-                    neon_match_result result = neon_update(chunk_ptr,
-                                                           cmap_neon,
-                                                           neon_table_size,
-                                                           do_unicode_validation,
-                                                           has_hi);
-                    if ((result.do_unicode_validation) || vmaxvq_u8(result.needs_escape) != 0) {
+                    sse42_match_result result = sse42_update(chunk_ptr,
+                                                             cmap_sse42,
+                                                             sse42_tab_size,
+                                                             do_unicode_validation,
+                                                             has_hi);
+                    if ((result.do_unicode_validation) || result.needs_escape) {
                         SEARCH_FLUSH;
-                        uint8x16_t actions = vaddq_u8(result.needs_escape, vdupq_n_u8('1'));
-                        do_hi_validation   = result.do_unicode_validation;
-                        vst1q_u8((unsigned char *)matches, actions);
-                        NEON_SET_STATE(2);
-                        break; /* Unreachable */
+                        _mm_storeu_si128((__m128i *)matches, result.actions);
+                        while (str < chunk_end) {
+                            long match_index = str - chunk_start;
+                            str              = process_character(matches[match_index],
+                                                                 str,
+                                                                 end,
+                                                                 out,
+                                                                 orig,
+                                                                 do_unicode_validation,
+                                                                 &check_start);
+                            str++;
+                        }
+                        cursor = str;
+                        continue;
                     }
                     str = chunk_end;
                 }
-                // We must have run out of data to use SIMD. Go to state 4.
                 SEARCH_FLUSH;
-                NEON_SET_STATE(4);
-            } break;
-            case 3:
-                cursor = str;
-                // This fall through is intentional. We return to state 3 after we process
-                // a byte (or multiple). We return to this state to ensure the cursor is
-                // pointing to the correct location. We then resume looking for matches
-                // within the previously processed chunk.
-            case 2:
-                if (str >= chunk_end) {
-                    NEON_SET_STATE(1);
-                }
-                if (!do_hi_validation) {
-                    long i = str - chunk_start;
-                    for (; str < chunk_end; i++) {
-                        if ((action = matches[i]) != '1') {
-                            break;
-                        }
-                        *out->cur++ = *str++;
-                    }
-                    // The loop above may have advanced str and directly output them to out->cur.
-                    // Ensure cursor is set appropriately.
-                    cursor = str;
-                    if (str >= chunk_end) {
-                        // We must have advanced past the end... we are done.
-                        NEON_SET_STATE(1);
-                    }
-                } else {
-                    long match_index = str - chunk_start;
-                    action           = matches[match_index];
-                }
-                NEON_RETURN_TO_STATE(3);
-                break;
-            case 4: action = cmap[(uint8_t)*str];
-            }
-#undef NEON_SET_STATE
-#undef NEON_RETURN_TO_STATE
-#else
-            action = cmap[(uint8_t)*str];
-#endif /* HAVE_SIMD_NEON */
-            switch (action) {
-            case '1':
-                if (do_unicode_validation && check_start <= str) {
-                    if (0 != (0x80 & (uint8_t)*str)) {
-                        if (0xC0 == (0xC0 & (uint8_t)*str)) {
-                            check_start = check_unicode(str, end, orig);
-                        } else {
-                            raise_invalid_unicode(orig, (int)(end - orig), (int)(str - orig));
-                        }
-                    }
-                }
-                *out->cur++ = *str;
-                break;
-            case '2':
-                *out->cur++ = '\\';
-                switch (*str) {
-                case '\\': *out->cur++ = '\\'; break;
-                case '\b': *out->cur++ = 'b'; break;
-                case '\t': *out->cur++ = 't'; break;
-                case '\n': *out->cur++ = 'n'; break;
-                case '\f': *out->cur++ = 'f'; break;
-                case '\r': *out->cur++ = 'r'; break;
-                default: *out->cur++ = *str; break;
-                }
-                break;
-            case '3':  // Unicode
-                if (0xe2 == (uint8_t)*str && do_unicode_validation && 2 <= end - str) {
-                    if (0x80 == (uint8_t)str[1] && (0xa8 == (uint8_t)str[2] || 0xa9 == (uint8_t)str[2])) {
-                        str = dump_unicode(str, end, out, orig);
-                    } else {
-                        check_start = check_unicode(str, end, orig);
-                        *out->cur++ = *str;
-                    }
-                    break;
-                }
-                str = dump_unicode(str, end, out, orig);
-                break;
-            case '6':  // control characters
-                if (*(uint8_t *)str < 0x80) {
-                    if (0 == (uint8_t)*str && out->opts->dump_opts.omit_null_byte) {
-                        break;
-                    }
-                    APPEND_CHARS(out->cur, "\\u00", 4);
-                    dump_hex((uint8_t)*str, out);
-                } else {
-                    if (0xe2 == (uint8_t)*str && do_unicode_validation && 2 <= end - str) {
-                        if (0x80 == (uint8_t)str[1] && (0xa8 == (uint8_t)str[2] || 0xa9 == (uint8_t)str[2])) {
-                            str = dump_unicode(str, end, out, orig);
-                        } else {
-                            check_start = check_unicode(str, end, orig);
-                            *out->cur++ = *str;
-                        }
-                        break;
-                    }
-                    str = dump_unicode(str, end, out, orig);
-                }
-                break;
-            default: break;  // ignore, should never happen if the table is correct
             }
         }
+#endif /* HAVE_SIMD_SSE4_2 */
+
+        for (; str < end; str++) {
+            str = process_character(cmap[(uint8_t)*str], str, end, out, orig, do_unicode_validation, &check_start);
+        }
         *out->cur++ = '"';
     }
     if (do_unicode_validation && 0 < str - orig && 0 != (0x80 & *(str - 1))) {
@@ -1266,10 +1456,11 @@ void oj_dump_raw(const char *str, size_t cnt, Out out) {
 }
 
 void oj_out_init(Out out) {
-    out->buf       = out->stack_buffer;
-    out->cur       = out->buf;
-    out->end       = out->buf + sizeof(out->stack_buffer) - BUFFER_EXTRA;
-    out->allocated = false;
+    out->buf            = out->stack_buffer;
+    out->cur            = out->buf;
+    out->end            = out->buf + sizeof(out->stack_buffer) - BUFFER_EXTRA;
+    out->allocated      = false;
+    out->key_filter_off = false;
 }
 
 void oj_out_free(Out out) {
@@ -1541,6 +1732,11 @@ void oj_dump_float(VALUE obj, int depth, Out out, bool as_ok) {
 size_t oj_dump_float_printf(char *buf, size_t blen, VALUE obj, double d, const char *format) {
     size_t cnt = snprintf(buf, blen, format, d);
 
+    // snprintf() returns the length the output would have needed, which is
+    // more than it wrote when the format asks for more room than buf has.
+    if (blen <= cnt) {
+        cnt = blen - 1;
+    }
     // Round off issues at 16 significant digits so check for obvious ones of
     // 0001 and 9999.
     if (17 <= cnt && (0 == strcmp("0001", buf + cnt - 4) || 0 == strcmp("9999", buf + cnt - 4))) {
@@ -1551,3 +1747,34 @@ size_t oj_dump_float_printf(char *buf, size_t blen, VALUE obj, double d, const c
     }
     return cnt;
 }
+
+// The only and except lists are the keys joined with a ':' and wrapped in one
+// at each end, so a key is in a list when ":key:" is in it. Matching in place
+// finds what searching the list for a copy of the key with the colons on it
+// found, without a buffer sized by the key.
+static bool key_listed(const char *list, const char *key, size_t klen) {
+    const char *s;
+
+    for (s = list; NULL != (s = strchr(s, ':')); s++) {
+        if (0 == strncmp(s + 1, key, klen) && ':' == s[klen + 1]) {
+            return true;
+        }
+    }
+    return false;
+}
+
+bool oj_key_skip(VALUE key, const char *only, const char *except) {
+    const char *skey;
+
+    switch (rb_type(key)) {
+    case RUBY_T_STRING: skey = StringValueCStr(key); break;
+    case RUBY_T_SYMBOL: skey = rb_id2name(rb_sym2id(key)); break;
+    default: skey = NULL; break;
+    }
+    if (NULL != skey && '\0' != *skey) {
+        size_t klen = strlen(skey);
+
+        return ((NULL != only && !key_listed(only, skey, klen)) || (NULL != except && key_listed(except, skey, klen)));
+    }
+    return NULL != only;
+}