simdjson 0.1.0

data/vendor/simdjson/include/simdjson/stage1_find_marks_arm64.h

@@ -0,0 +1,210 @@
+#ifndef SIMDJSON_STAGE1_FIND_MARKS_ARM64_H
+#define SIMDJSON_STAGE1_FIND_MARKS_ARM64_H
+
+#include "simdjson/simdutf8check_arm64.h"
+#include "simdjson/stage1_find_marks.h"
+#include "simdjson/stage1_find_marks_flatten.h"
+#include "simdjson/stage1_find_marks_macros.h"
+
+#ifdef IS_ARM64
+namespace simdjson {
+template <> struct simd_input<Architecture::ARM64> {
+  uint8x16_t i0;
+  uint8x16_t i1;
+  uint8x16_t i2;
+  uint8x16_t i3;
+};
+
+template <>
+really_inline simd_input<Architecture::ARM64>
+fill_input<Architecture::ARM64>(const uint8_t *ptr) {
+  struct simd_input<Architecture::ARM64> in;
+  in.i0 = vld1q_u8(ptr + 0);
+  in.i1 = vld1q_u8(ptr + 16);
+  in.i2 = vld1q_u8(ptr + 32);
+  in.i3 = vld1q_u8(ptr + 48);
+  return in;
+}
+
+really_inline uint16_t neon_movemask(uint8x16_t input) {
+  const uint8x16_t bit_mask = {0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80,
+                               0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80};
+  uint8x16_t minput = vandq_u8(input, bit_mask);
+  uint8x16_t tmp = vpaddq_u8(minput, minput);
+  tmp = vpaddq_u8(tmp, tmp);
+  tmp = vpaddq_u8(tmp, tmp);
+  return vgetq_lane_u16(vreinterpretq_u16_u8(tmp), 0);
+}
+
+really_inline uint64_t neon_movemask_bulk(uint8x16_t p0, uint8x16_t p1,
+                                          uint8x16_t p2, uint8x16_t p3) {
+  const uint8x16_t bit_mask = {0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80,
+                               0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80};
+  uint8x16_t t0 = vandq_u8(p0, bit_mask);
+  uint8x16_t t1 = vandq_u8(p1, bit_mask);
+  uint8x16_t t2 = vandq_u8(p2, bit_mask);
+  uint8x16_t t3 = vandq_u8(p3, bit_mask);
+  uint8x16_t sum0 = vpaddq_u8(t0, t1);
+  uint8x16_t sum1 = vpaddq_u8(t2, t3);
+  sum0 = vpaddq_u8(sum0, sum1);
+  sum0 = vpaddq_u8(sum0, sum0);
+  return vgetq_lane_u64(vreinterpretq_u64_u8(sum0), 0);
+}
+
+template <>
+really_inline uint64_t
+compute_quote_mask<Architecture::ARM64>(uint64_t quote_bits) {
+#ifdef __ARM_FEATURE_CRYPTO // some ARM processors lack this extension
+  return vmull_p64(-1ULL, quote_bits);
+#else
+  return portable_compute_quote_mask(quote_bits);
+#endif
+}
+
+template <> struct utf8_checking_state<Architecture::ARM64> {
+  int8x16_t has_error{};
+  processed_utf_bytes previous{};
+};
+
+// Checks that all bytes are ascii
+really_inline bool check_ascii_neon(simd_input<Architecture::ARM64> in) {
+  // checking if the most significant bit is always equal to 0.
+  uint8x16_t high_bit = vdupq_n_u8(0x80);
+  uint8x16_t t0 = vorrq_u8(in.i0, in.i1);
+  uint8x16_t t1 = vorrq_u8(in.i2, in.i3);
+  uint8x16_t t3 = vorrq_u8(t0, t1);
+  uint8x16_t t4 = vandq_u8(t3, high_bit);
+  uint64x2_t v64 = vreinterpretq_u64_u8(t4);
+  uint32x2_t v32 = vqmovn_u64(v64);
+  uint64x1_t result = vreinterpret_u64_u32(v32);
+  return vget_lane_u64(result, 0) == 0;
+}
+
+template <>
+really_inline void check_utf8<Architecture::ARM64>(
+    simd_input<Architecture::ARM64> in,
+    utf8_checking_state<Architecture::ARM64> &state) {
+  if (check_ascii_neon(in)) {
+    // All bytes are ascii. Therefore the byte that was just before must be
+    // ascii too. We only check the byte that was just before simd_input. Nines
+    // are arbitrary values.
+    const int8x16_t verror =
+        (int8x16_t){9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 1};
+    state.has_error =
+        vorrq_s8(vreinterpretq_s8_u8(
+                     vcgtq_s8(state.previous.carried_continuations, verror)),
+                 state.has_error);
+  } else {
+    // it is not ascii so we have to do heavy work
+    state.previous = check_utf8_bytes(vreinterpretq_s8_u8(in.i0),
+                                      &(state.previous), &(state.has_error));
+    state.previous = check_utf8_bytes(vreinterpretq_s8_u8(in.i1),
+                                      &(state.previous), &(state.has_error));
+    state.previous = check_utf8_bytes(vreinterpretq_s8_u8(in.i2),
+                                      &(state.previous), &(state.has_error));
+    state.previous = check_utf8_bytes(vreinterpretq_s8_u8(in.i3),
+                                      &(state.previous), &(state.has_error));
+  }
+}
+
+template <>
+really_inline ErrorValues check_utf8_errors<Architecture::ARM64>(
+    utf8_checking_state<Architecture::ARM64> &state) {
+  uint64x2_t v64 = vreinterpretq_u64_s8(state.has_error);
+  uint32x2_t v32 = vqmovn_u64(v64);
+  uint64x1_t result = vreinterpret_u64_u32(v32);
+  return vget_lane_u64(result, 0) != 0 ? simdjson::UTF8_ERROR
+                                       : simdjson::SUCCESS;
+}
+
+template <>
+really_inline uint64_t cmp_mask_against_input<Architecture::ARM64>(
+    simd_input<Architecture::ARM64> in, uint8_t m) {
+  const uint8x16_t mask = vmovq_n_u8(m);
+  uint8x16_t cmp_res_0 = vceqq_u8(in.i0, mask);
+  uint8x16_t cmp_res_1 = vceqq_u8(in.i1, mask);
+  uint8x16_t cmp_res_2 = vceqq_u8(in.i2, mask);
+  uint8x16_t cmp_res_3 = vceqq_u8(in.i3, mask);
+  return neon_movemask_bulk(cmp_res_0, cmp_res_1, cmp_res_2, cmp_res_3);
+}
+
+template <>
+really_inline uint64_t unsigned_lteq_against_input<Architecture::ARM64>(
+    simd_input<Architecture::ARM64> in, uint8_t m) {
+  const uint8x16_t mask = vmovq_n_u8(m);
+  uint8x16_t cmp_res_0 = vcleq_u8(in.i0, mask);
+  uint8x16_t cmp_res_1 = vcleq_u8(in.i1, mask);
+  uint8x16_t cmp_res_2 = vcleq_u8(in.i2, mask);
+  uint8x16_t cmp_res_3 = vcleq_u8(in.i3, mask);
+  return neon_movemask_bulk(cmp_res_0, cmp_res_1, cmp_res_2, cmp_res_3);
+}
+
+template <>
+really_inline uint64_t find_odd_backslash_sequences<Architecture::ARM64>(
+    simd_input<Architecture::ARM64> in,
+    uint64_t &prev_iter_ends_odd_backslash) {
+  FIND_ODD_BACKSLASH_SEQUENCES(Architecture::ARM64, in,
+                               prev_iter_ends_odd_backslash);
+}
+
+template <>
+really_inline uint64_t find_quote_mask_and_bits<Architecture::ARM64>(
+    simd_input<Architecture::ARM64> in, uint64_t odd_ends,
+    uint64_t &prev_iter_inside_quote, uint64_t &quote_bits,
+    uint64_t &error_mask) {
+  FIND_QUOTE_MASK_AND_BITS(Architecture::ARM64, in, odd_ends,
+                           prev_iter_inside_quote, quote_bits, error_mask)
+}
+
+template <>
+really_inline void find_whitespace_and_structurals<Architecture::ARM64>(
+    simd_input<Architecture::ARM64> in, uint64_t &whitespace,
+    uint64_t &structurals) {
+  const uint8x16_t low_nibble_mask =
+      (uint8x16_t){16, 0, 0, 0, 0, 0, 0, 0, 0, 8, 12, 1, 2, 9, 0, 0};
+  const uint8x16_t high_nibble_mask =
+      (uint8x16_t){8, 0, 18, 4, 0, 1, 0, 1, 0, 0, 0, 3, 2, 1, 0, 0};
+  const uint8x16_t structural_shufti_mask = vmovq_n_u8(0x7);
+  const uint8x16_t whitespace_shufti_mask = vmovq_n_u8(0x18);
+  const uint8x16_t low_nib_and_mask = vmovq_n_u8(0xf);
+
+  uint8x16_t nib_0_lo = vandq_u8(in.i0, low_nib_and_mask);
+  uint8x16_t nib_0_hi = vshrq_n_u8(in.i0, 4);
+  uint8x16_t shuf_0_lo = vqtbl1q_u8(low_nibble_mask, nib_0_lo);
+  uint8x16_t shuf_0_hi = vqtbl1q_u8(high_nibble_mask, nib_0_hi);
+  uint8x16_t v_0 = vandq_u8(shuf_0_lo, shuf_0_hi);
+
+  uint8x16_t nib_1_lo = vandq_u8(in.i1, low_nib_and_mask);
+  uint8x16_t nib_1_hi = vshrq_n_u8(in.i1, 4);
+  uint8x16_t shuf_1_lo = vqtbl1q_u8(low_nibble_mask, nib_1_lo);
+  uint8x16_t shuf_1_hi = vqtbl1q_u8(high_nibble_mask, nib_1_hi);
+  uint8x16_t v_1 = vandq_u8(shuf_1_lo, shuf_1_hi);
+
+  uint8x16_t nib_2_lo = vandq_u8(in.i2, low_nib_and_mask);
+  uint8x16_t nib_2_hi = vshrq_n_u8(in.i2, 4);
+  uint8x16_t shuf_2_lo = vqtbl1q_u8(low_nibble_mask, nib_2_lo);
+  uint8x16_t shuf_2_hi = vqtbl1q_u8(high_nibble_mask, nib_2_hi);
+  uint8x16_t v_2 = vandq_u8(shuf_2_lo, shuf_2_hi);
+
+  uint8x16_t nib_3_lo = vandq_u8(in.i3, low_nib_and_mask);
+  uint8x16_t nib_3_hi = vshrq_n_u8(in.i3, 4);
+  uint8x16_t shuf_3_lo = vqtbl1q_u8(low_nibble_mask, nib_3_lo);
+  uint8x16_t shuf_3_hi = vqtbl1q_u8(high_nibble_mask, nib_3_hi);
+  uint8x16_t v_3 = vandq_u8(shuf_3_lo, shuf_3_hi);
+
+  uint8x16_t tmp_0 = vtstq_u8(v_0, structural_shufti_mask);
+  uint8x16_t tmp_1 = vtstq_u8(v_1, structural_shufti_mask);
+  uint8x16_t tmp_2 = vtstq_u8(v_2, structural_shufti_mask);
+  uint8x16_t tmp_3 = vtstq_u8(v_3, structural_shufti_mask);
+  structurals = neon_movemask_bulk(tmp_0, tmp_1, tmp_2, tmp_3);
+
+  uint8x16_t tmp_ws_0 = vtstq_u8(v_0, whitespace_shufti_mask);
+  uint8x16_t tmp_ws_1 = vtstq_u8(v_1, whitespace_shufti_mask);
+  uint8x16_t tmp_ws_2 = vtstq_u8(v_2, whitespace_shufti_mask);
+  uint8x16_t tmp_ws_3 = vtstq_u8(v_3, whitespace_shufti_mask);
+  whitespace = neon_movemask_bulk(tmp_ws_0, tmp_ws_1, tmp_ws_2, tmp_ws_3);
+}
+} // namespace simdjson
+
+#endif // IS_ARM64
+#endif // SIMDJSON_STAGE1_FIND_MARKS_ARM64_H

data/vendor/simdjson/include/simdjson/stage1_find_marks_flatten.h

@@ -0,0 +1,93 @@
+#ifndef SIMDJSON_STAGE1_FIND_MARKS_FLATTEN_H
+#define SIMDJSON_STAGE1_FIND_MARKS_FLATTEN_H
+
+namespace simdjson {
+
+#ifdef SIMDJSON_NAIVE_FLATTEN // useful for benchmarking
+//
+// This is just a naive implementation. It should be normally
+// disable, but can be used for research purposes to compare
+// again our optimized version.
+really_inline void flatten_bits(uint32_t *base_ptr, uint32_t &base,
+                                uint32_t idx, uint64_t bits) {
+  uint32_t *out_ptr = base_ptr + base;
+  idx -= 64;
+  while (bits != 0) {
+    out_ptr[0] = idx + trailing_zeroes(bits);
+    bits = bits & (bits - 1);
+    out_ptr++;
+  }
+  base = (out_ptr - base_ptr);
+}
+
+#else
+// flatten out values in 'bits' assuming that they are are to have values of idx
+// plus their position in the bitvector, and store these indexes at
+// base_ptr[base] incrementing base as we go
+// will potentially store extra values beyond end of valid bits, so base_ptr
+// needs to be large enough to handle this
+really_inline void flatten_bits(uint32_t *base_ptr, uint32_t &base,
+                                uint32_t idx, uint64_t bits) {
+  // In some instances, the next branch is expensive because it is mispredicted.
+  // Unfortunately, in other cases,
+  // it helps tremendously.
+  if (bits == 0)
+    return;
+  uint32_t cnt = hamming(bits);
+  uint32_t next_base = base + cnt;
+  idx -= 64;
+  base_ptr += base;
+  {
+    base_ptr[0] = idx + trailing_zeroes(bits);
+    bits = bits & (bits - 1);
+    base_ptr[1] = idx + trailing_zeroes(bits);
+    bits = bits & (bits - 1);
+    base_ptr[2] = idx + trailing_zeroes(bits);
+    bits = bits & (bits - 1);
+    base_ptr[3] = idx + trailing_zeroes(bits);
+    bits = bits & (bits - 1);
+    base_ptr[4] = idx + trailing_zeroes(bits);
+    bits = bits & (bits - 1);
+    base_ptr[5] = idx + trailing_zeroes(bits);
+    bits = bits & (bits - 1);
+    base_ptr[6] = idx + trailing_zeroes(bits);
+    bits = bits & (bits - 1);
+    base_ptr[7] = idx + trailing_zeroes(bits);
+    bits = bits & (bits - 1);
+    base_ptr += 8;
+  }
+  // We hope that the next branch is easily predicted.
+  if (cnt > 8) {
+    base_ptr[0] = idx + trailing_zeroes(bits);
+    bits = bits & (bits - 1);
+    base_ptr[1] = idx + trailing_zeroes(bits);
+    bits = bits & (bits - 1);
+    base_ptr[2] = idx + trailing_zeroes(bits);
+    bits = bits & (bits - 1);
+    base_ptr[3] = idx + trailing_zeroes(bits);
+    bits = bits & (bits - 1);
+    base_ptr[4] = idx + trailing_zeroes(bits);
+    bits = bits & (bits - 1);
+    base_ptr[5] = idx + trailing_zeroes(bits);
+    bits = bits & (bits - 1);
+    base_ptr[6] = idx + trailing_zeroes(bits);
+    bits = bits & (bits - 1);
+    base_ptr[7] = idx + trailing_zeroes(bits);
+    bits = bits & (bits - 1);
+    base_ptr += 8;
+  }
+  if (cnt > 16) { // unluckly: we rarely get here
+    // since it means having one structural or pseudo-structral element
+    // every 4 characters (possible with inputs like "","","",...).
+    do {
+      base_ptr[0] = idx + trailing_zeroes(bits);
+      bits = bits & (bits - 1);
+      base_ptr++;
+    } while (bits != 0);
+  }
+  base = next_base;
+}
+#endif // SIMDJSON_NAIVE_FLATTEN
+} // namespace simdjson
+
+#endif // SIMDJSON_STAGE1_FIND_MARKS_FLATTEN_H

data/vendor/simdjson/include/simdjson/stage1_find_marks_flatten_haswell.h

@@ -0,0 +1,95 @@
+#ifndef SIMDJSON_STAGE1_FIND_MARKS_FLATTEN_HASWELL_H
+#define SIMDJSON_STAGE1_FIND_MARKS_FLATTEN_HASWELL_H
+
+// This file provides the same function as
+// stage1_find_marks_flatten.h, but uses Intel intrinsics.
+// This should provide better performance on Visual Studio
+// and other compilers that do a conservative optimization.
+
+// Specifically, on x64 processors with BMI,
+// x & (x - 1) should be mapped to
+// the blsr instruction. By using the
+// _blsr_u64 intrinsic, we
+// ensure that this will happen.
+/////////
+
+#include "simdjson/common_defs.h"
+#include "simdjson/portability.h"
+
+#ifdef IS_X86_64
+
+TARGET_HASWELL
+namespace simdjson {
+namespace haswell {
+
+// flatten out values in 'bits' assuming that they are are to have values of idx
+// plus their position in the bitvector, and store these indexes at
+// base_ptr[base] incrementing base as we go
+// will potentially store extra values beyond end of valid bits, so base_ptr
+// needs to be large enough to handle this
+really_inline void flatten_bits(uint32_t *base_ptr, uint32_t &base,
+                                uint32_t idx, uint64_t bits) {
+  // In some instances, the next branch is expensive because it is mispredicted.
+  // Unfortunately, in other cases,
+  // it helps tremendously.
+  if (bits == 0)
+    return;
+  uint32_t cnt = _mm_popcnt_u64(bits);
+  uint32_t next_base = base + cnt;
+  idx -= 64;
+  base_ptr += base;
+  {
+    base_ptr[0] = idx + trailing_zeroes(bits);
+    bits = _blsr_u64(bits);
+    base_ptr[1] = idx + trailing_zeroes(bits);
+    bits = _blsr_u64(bits);
+    base_ptr[2] = idx + trailing_zeroes(bits);
+    bits = _blsr_u64(bits);
+    base_ptr[3] = idx + trailing_zeroes(bits);
+    bits = _blsr_u64(bits);
+    base_ptr[4] = idx + trailing_zeroes(bits);
+    bits = _blsr_u64(bits);
+    base_ptr[5] = idx + trailing_zeroes(bits);
+    bits = _blsr_u64(bits);
+    base_ptr[6] = idx + trailing_zeroes(bits);
+    bits = _blsr_u64(bits);
+    base_ptr[7] = idx + trailing_zeroes(bits);
+    bits = _blsr_u64(bits);
+    base_ptr += 8;
+  }
+  // We hope that the next branch is easily predicted.
+  if (cnt > 8) {
+    base_ptr[0] = idx + trailing_zeroes(bits);
+    bits = _blsr_u64(bits);
+    base_ptr[1] = idx + trailing_zeroes(bits);
+    bits = _blsr_u64(bits);
+    base_ptr[2] = idx + trailing_zeroes(bits);
+    bits = _blsr_u64(bits);
+    base_ptr[3] = idx + trailing_zeroes(bits);
+    bits = _blsr_u64(bits);
+    base_ptr[4] = idx + trailing_zeroes(bits);
+    bits = _blsr_u64(bits);
+    base_ptr[5] = idx + trailing_zeroes(bits);
+    bits = _blsr_u64(bits);
+    base_ptr[6] = idx + trailing_zeroes(bits);
+    bits = _blsr_u64(bits);
+    base_ptr[7] = idx + trailing_zeroes(bits);
+    bits = _blsr_u64(bits);
+    base_ptr += 8;
+  }
+  if (cnt > 16) { // unluckly: we rarely get here
+    // since it means having one structural or pseudo-structral element
+    // every 4 characters (possible with inputs like "","","",...).
+    do {
+      base_ptr[0] = idx + trailing_zeroes(bits);
+      bits = _blsr_u64(bits);
+      base_ptr++;
+    } while (bits != 0);
+  }
+  base = next_base;
+}
+} // namespace haswell
+} // namespace simdjson
+UNTARGET_REGION
+#endif // IS_X86_64
+#endif // SIMDJSON_STAGE1_FIND_MARKS_FLATTEN_H

data/vendor/simdjson/include/simdjson/stage1_find_marks_haswell.h

@@ -0,0 +1,210 @@
+#ifndef SIMDJSON_STAGE1_FIND_MARKS_HASWELL_H
+#define SIMDJSON_STAGE1_FIND_MARKS_HASWELL_H
+
+#include "simdjson/simdutf8check_haswell.h"
+#include "simdjson/stage1_find_marks.h"
+#include "simdjson/stage1_find_marks_flatten_haswell.h"
+#include "simdjson/stage1_find_marks_macros.h"
+
+#ifdef IS_X86_64
+
+TARGET_HASWELL
+namespace simdjson {
+template <> struct simd_input<Architecture::HASWELL> {
+  __m256i lo;
+  __m256i hi;
+};
+
+template <>
+really_inline simd_input<Architecture::HASWELL>
+fill_input<Architecture::HASWELL>(const uint8_t *ptr) {
+  struct simd_input<Architecture::HASWELL> in;
+  in.lo = _mm256_loadu_si256(reinterpret_cast<const __m256i *>(ptr + 0));
+  in.hi = _mm256_loadu_si256(reinterpret_cast<const __m256i *>(ptr + 32));
+  return in;
+}
+
+template <>
+really_inline uint64_t
+compute_quote_mask<Architecture::HASWELL>(uint64_t quote_bits) {
+  // There should be no such thing with a processing supporting avx2
+  // but not clmul.
+  uint64_t quote_mask = _mm_cvtsi128_si64(_mm_clmulepi64_si128(
+      _mm_set_epi64x(0ULL, quote_bits), _mm_set1_epi8(0xFFu), 0));
+  return quote_mask;
+}
+
+template <> struct utf8_checking_state<Architecture::HASWELL> {
+  __m256i has_error;
+  avx_processed_utf_bytes previous;
+  utf8_checking_state() {
+    has_error = _mm256_setzero_si256();
+    previous.raw_bytes = _mm256_setzero_si256();
+    previous.high_nibbles = _mm256_setzero_si256();
+    previous.carried_continuations = _mm256_setzero_si256();
+  }
+};
+
+template <>
+really_inline void check_utf8<Architecture::HASWELL>(
+    simd_input<Architecture::HASWELL> in,
+    utf8_checking_state<Architecture::HASWELL> &state) {
+  __m256i high_bit = _mm256_set1_epi8(0x80u);
+  if ((_mm256_testz_si256(_mm256_or_si256(in.lo, in.hi), high_bit)) == 1) {
+    // it is ascii, we just check continuation
+    state.has_error = _mm256_or_si256(
+        _mm256_cmpgt_epi8(state.previous.carried_continuations,
+                          _mm256_setr_epi8(9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
+                                           9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
+                                           9, 9, 9, 9, 9, 9, 9, 1)),
+        state.has_error);
+  } else {
+    // it is not ascii so we have to do heavy work
+    state.previous =
+        avx_check_utf8_bytes(in.lo, &(state.previous), &(state.has_error));
+    state.previous =
+        avx_check_utf8_bytes(in.hi, &(state.previous), &(state.has_error));
+  }
+}
+
+template <>
+really_inline ErrorValues check_utf8_errors<Architecture::HASWELL>(
+    utf8_checking_state<Architecture::HASWELL> &state) {
+  return _mm256_testz_si256(state.has_error, state.has_error) == 0
+             ? simdjson::UTF8_ERROR
+             : simdjson::SUCCESS;
+}
+
+template <>
+really_inline uint64_t cmp_mask_against_input<Architecture::HASWELL>(
+    simd_input<Architecture::HASWELL> in, uint8_t m) {
+  const __m256i mask = _mm256_set1_epi8(m);
+  __m256i cmp_res_0 = _mm256_cmpeq_epi8(in.lo, mask);
+  uint64_t res_0 = static_cast<uint32_t>(_mm256_movemask_epi8(cmp_res_0));
+  __m256i cmp_res_1 = _mm256_cmpeq_epi8(in.hi, mask);
+  uint64_t res_1 = _mm256_movemask_epi8(cmp_res_1);
+  return res_0 | (res_1 << 32);
+}
+
+template <>
+really_inline uint64_t unsigned_lteq_against_input<Architecture::HASWELL>(
+    simd_input<Architecture::HASWELL> in, uint8_t m) {
+  const __m256i maxval = _mm256_set1_epi8(m);
+  __m256i cmp_res_0 = _mm256_cmpeq_epi8(_mm256_max_epu8(maxval, in.lo), maxval);
+  uint64_t res_0 = static_cast<uint32_t>(_mm256_movemask_epi8(cmp_res_0));
+  __m256i cmp_res_1 = _mm256_cmpeq_epi8(_mm256_max_epu8(maxval, in.hi), maxval);
+  uint64_t res_1 = _mm256_movemask_epi8(cmp_res_1);
+  return res_0 | (res_1 << 32);
+}
+
+template <>
+really_inline uint64_t find_odd_backslash_sequences<Architecture::HASWELL>(
+    simd_input<Architecture::HASWELL> in,
+    uint64_t &prev_iter_ends_odd_backslash) {
+  FIND_ODD_BACKSLASH_SEQUENCES(Architecture::HASWELL, in,
+                               prev_iter_ends_odd_backslash);
+}
+
+template <>
+really_inline uint64_t find_quote_mask_and_bits<Architecture::HASWELL>(
+    simd_input<Architecture::HASWELL> in, uint64_t odd_ends,
+    uint64_t &prev_iter_inside_quote, uint64_t &quote_bits,
+    uint64_t &error_mask) {
+  FIND_QUOTE_MASK_AND_BITS(Architecture::HASWELL, in, odd_ends,
+                           prev_iter_inside_quote, quote_bits, error_mask)
+}
+
+template <>
+really_inline void find_whitespace_and_structurals<Architecture::HASWELL>(
+    simd_input<Architecture::HASWELL> in, uint64_t &whitespace,
+    uint64_t &structurals) {
+#ifdef SIMDJSON_NAIVE_STRUCTURAL
+  // You should never need this naive approach, but it can be useful
+  // for research purposes
+  const __m256i mask_open_brace = _mm256_set1_epi8(0x7b);
+  __m256i struct_lo = _mm256_cmpeq_epi8(in.lo, mask_open_brace);
+  __m256i struct_hi = _mm256_cmpeq_epi8(in.hi, mask_open_brace);
+  const __m256i mask_close_brace = _mm256_set1_epi8(0x7d);
+  struct_lo =
+      _mm256_or_si256(struct_lo, _mm256_cmpeq_epi8(in.lo, mask_close_brace));
+  struct_hi =
+      _mm256_or_si256(struct_hi, _mm256_cmpeq_epi8(in.hi, mask_close_brace));
+  const __m256i mask_open_bracket = _mm256_set1_epi8(0x5b);
+  struct_lo =
+      _mm256_or_si256(struct_lo, _mm256_cmpeq_epi8(in.lo, mask_open_bracket));
+  struct_hi =
+      _mm256_or_si256(struct_hi, _mm256_cmpeq_epi8(in.hi, mask_open_bracket));
+  const __m256i mask_close_bracket = _mm256_set1_epi8(0x5d);
+  struct_lo =
+      _mm256_or_si256(struct_lo, _mm256_cmpeq_epi8(in.lo, mask_close_bracket));
+  struct_hi =
+      _mm256_or_si256(struct_hi, _mm256_cmpeq_epi8(in.hi, mask_close_bracket));
+  const __m256i mask_column = _mm256_set1_epi8(0x3a);
+  struct_lo = _mm256_or_si256(struct_lo, _mm256_cmpeq_epi8(in.lo, mask_column));
+  struct_hi = _mm256_or_si256(struct_hi, _mm256_cmpeq_epi8(in.hi, mask_column));
+  const __m256i mask_comma = _mm256_set1_epi8(0x2c);
+  struct_lo = _mm256_or_si256(struct_lo, _mm256_cmpeq_epi8(in.lo, mask_comma));
+  struct_hi = _mm256_or_si256(struct_hi, _mm256_cmpeq_epi8(in.hi, mask_comma));
+  uint64_t structural_res_0 =
+      static_cast<uint32_t>(_mm256_movemask_epi8(struct_lo));
+  uint64_t structural_res_1 = _mm256_movemask_epi8(struct_hi);
+  structurals = (structural_res_0 | (structural_res_1 << 32));
+
+  const __m256i mask_space = _mm256_set1_epi8(0x20);
+  __m256i space_lo = _mm256_cmpeq_epi8(in.lo, mask_space);
+  __m256i space_hi = _mm256_cmpeq_epi8(in.hi, mask_space);
+  const __m256i mask_linefeed = _mm256_set1_epi8(0x0a);
+  space_lo = _mm256_or_si256(space_lo, _mm256_cmpeq_epi8(in.lo, mask_linefeed));
+  space_hi = _mm256_or_si256(space_hi, _mm256_cmpeq_epi8(in.hi, mask_linefeed));
+  const __m256i mask_tab = _mm256_set1_epi8(0x09);
+  space_lo = _mm256_or_si256(space_lo, _mm256_cmpeq_epi8(in.lo, mask_tab));
+  space_hi = _mm256_or_si256(space_hi, _mm256_cmpeq_epi8(in.hi, mask_tab));
+  const __m256i mask_carriage = _mm256_set1_epi8(0x0d);
+  space_lo = _mm256_or_si256(space_lo, _mm256_cmpeq_epi8(in.lo, mask_carriage));
+  space_hi = _mm256_or_si256(space_hi, _mm256_cmpeq_epi8(in.hi, mask_carriage));
+
+  uint64_t ws_res_0 = static_cast<uint32_t>(_mm256_movemask_epi8(space_lo));
+  uint64_t ws_res_1 = _mm256_movemask_epi8(space_hi);
+  whitespace = (ws_res_0 | (ws_res_1 << 32));
+  // end of naive approach
+
+#else  // SIMDJSON_NAIVE_STRUCTURAL
+  // clang-format off
+  const __m256i structural_table =
+      _mm256_setr_epi8(44, 125, 0, 0, 0xc0u, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 123,
+                       44, 125, 0, 0, 0xc0u, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 123);
+  const __m256i white_table = _mm256_setr_epi8(
+      32, 100, 100, 100, 17, 100, 113, 2, 100, 9, 10, 112, 100, 13, 100, 100,
+      32, 100, 100, 100, 17, 100, 113, 2, 100, 9, 10, 112, 100, 13, 100, 100);
+  // clang-format on
+  const __m256i struct_offset = _mm256_set1_epi8(0xd4u);
+  const __m256i struct_mask = _mm256_set1_epi8(32);
+
+  __m256i lo_white =
+      _mm256_cmpeq_epi8(in.lo, _mm256_shuffle_epi8(white_table, in.lo));
+  __m256i hi_white =
+      _mm256_cmpeq_epi8(in.hi, _mm256_shuffle_epi8(white_table, in.hi));
+  uint64_t ws_res_0 = static_cast<uint32_t>(_mm256_movemask_epi8(lo_white));
+  uint64_t ws_res_1 = _mm256_movemask_epi8(hi_white);
+  whitespace = (ws_res_0 | (ws_res_1 << 32));
+  __m256i lo_struct_r1 = _mm256_add_epi8(struct_offset, in.lo);
+  __m256i hi_struct_r1 = _mm256_add_epi8(struct_offset, in.hi);
+  __m256i lo_struct_r2 = _mm256_or_si256(in.lo, struct_mask);
+  __m256i hi_struct_r2 = _mm256_or_si256(in.hi, struct_mask);
+  __m256i lo_struct_r3 = _mm256_shuffle_epi8(structural_table, lo_struct_r1);
+  __m256i hi_struct_r3 = _mm256_shuffle_epi8(structural_table, hi_struct_r1);
+  __m256i lo_struct = _mm256_cmpeq_epi8(lo_struct_r2, lo_struct_r3);
+  __m256i hi_struct = _mm256_cmpeq_epi8(hi_struct_r2, hi_struct_r3);
+
+  uint64_t structural_res_0 =
+      static_cast<uint32_t>(_mm256_movemask_epi8(lo_struct));
+  uint64_t structural_res_1 = _mm256_movemask_epi8(hi_struct);
+  structurals = (structural_res_0 | (structural_res_1 << 32));
+#endif // SIMDJSON_NAIVE_STRUCTURAL
+}
+
+} // namespace simdjson
+UNTARGET_REGION
+
+#endif // IS_X86_64
+#endif // SIMDJSON_STAGE1_FIND_MARKS_HASWELL_H