simdjson 0.1.0

data/vendor/simdjson/src/CMakeLists.txt

@@ -0,0 +1,67 @@
+if(${CMAKE_C_COMPILER_ID} MATCHES "Intel") # icc / icpc
+  # prevent shared libraries from depending on Intel provided libraries
+  set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -static-intel")
+endif()
+
+include(GNUInstallDirs)
+
+# we default on a shared library.
+if(SIMDJSON_BUILD_STATIC)
+  set(SIMDJSON_LIB_TYPE STATIC)
+  MESSAGE( STATUS "Building a static library." )
+else()
+  MESSAGE( STATUS "Building a dynamic library (default)." )
+  set(SIMDJSON_LIB_TYPE SHARED)
+endif()
+
+MESSAGE( STATUS "SIMDJSON_LIB_TYPE: " ${SIMDJSON_LIB_TYPE})
+set(SIMDJSON_SRC
+    jsonioutil.cpp
+    jsonminifier.cpp
+    jsonparser.cpp
+    stage1_find_marks.cpp
+    stage2_build_tape.cpp
+    parsedjson.cpp
+    parsedjsoniterator.cpp
+    simdjson.cpp
+    )
+
+add_library(${SIMDJSON_LIB_NAME} ${SIMDJSON_LIB_TYPE} ${SIMDJSON_SRC})
+
+target_include_directories(${SIMDJSON_LIB_NAME}
+  PUBLIC
+  $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/include>
+  $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
+)
+
+install(TARGETS ${SIMDJSON_LIB_NAME}
+  EXPORT ${SIMDJSON_LIB_NAME}-config
+  ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
+  LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+  RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
+)
+
+install(EXPORT ${SIMDJSON_LIB_NAME}-config
+  FILE ${SIMDJSON_LIB_NAME}-config.cmake
+  NAMESPACE ${SIMDJSON_LIB_NAME}::
+  DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${SIMDJSON_LIB_NAME}
+)
+
+if(NOT MSVC)
+## We output the library at the root of the current directory where cmake is invoked
+## This is handy but Visual Studio will happily ignore us
+set_target_properties(${SIMDJSON_LIB_NAME} PROPERTIES
+  LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}
+  VERSION ${SIMDJSON_LIB_VERSION}
+  SOVERSION ${SIMDJSON_LIB_SOVERSION})
+MESSAGE( STATUS "Library output directory (does not apply to Visual Studio): " ${CMAKE_BINARY_DIR})
+endif()
+
+if(MSVC AND (SIMDJSON_LIB_TYPE STREQUAL "SHARED"))
+  if (CMAKE_VERSION VERSION_LESS 3.4)
+    MESSAGE( STATUS "To build  a Windows DLL using Visual Studio, you may need cmake 3.4 or better." )
+  endif()
+  MESSAGE( STATUS "Building a Windows DLL using Visual Studio, exporting all symbols automatically." )
+ set_target_properties(${SIMDJSON_LIB_NAME}
+    PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS 1)
+endif()

data/vendor/simdjson/src/jsonioutil.cpp

@@ -0,0 +1,35 @@
+#include "simdjson/jsonioutil.h"
+#include <cstdlib>
+#include <cstring>
+
+namespace simdjson {
+char *allocate_padded_buffer(size_t length) {
+  // we could do a simple malloc
+  // return (char *) malloc(length + SIMDJSON_PADDING);
+  // However, we might as well align to cache lines...
+  size_t totalpaddedlength = length + SIMDJSON_PADDING;
+  char *padded_buffer = aligned_malloc_char(64, totalpaddedlength);
+  return padded_buffer;
+}
+
+padded_string get_corpus(const std::string &filename) {
+  std::FILE *fp = std::fopen(filename.c_str(), "rb");
+  if (fp != nullptr) {
+    std::fseek(fp, 0, SEEK_END);
+    size_t len = std::ftell(fp);
+    padded_string s(len);
+    if (s.data() == nullptr) {
+      std::fclose(fp);
+      throw std::runtime_error("could not allocate memory");
+    }
+    std::rewind(fp);
+    size_t readb = std::fread(s.data(), 1, len, fp);
+    std::fclose(fp);
+    if (readb != len) {
+      throw std::runtime_error("could not read the data");
+    }
+    return s;
+  }
+  throw std::runtime_error("could not load corpus");
+}
+} // namespace simdjson

data/vendor/simdjson/src/jsonminifier.cpp

@@ -0,0 +1,285 @@
+#include "simdjson/portability.h"
+#include <cstdint>
+
+#ifndef __AVX2__
+
+namespace simdjson {
+static uint8_t jump_table[256 * 3] = {
+    0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0,
+    1, 1, 0, 1, 0, 0, 1, 0, 0, 1, 1, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 1, 1, 0, 1,
+    1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1,
+    0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 0, 0,
+    1, 1, 1, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1,
+    1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1,
+    0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0,
+    1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1,
+    1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1,
+    0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0,
+    1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1,
+    1, 0, 0, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1,
+    0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0,
+    1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1,
+    1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1,
+    0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0,
+    1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1,
+    1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1,
+    0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0,
+    1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1,
+    1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1,
+    0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0,
+    1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1,
+    1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1,
+    0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0,
+    1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1,
+    1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1,
+    0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0,
+    1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1,
+    1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1,
+    0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1,
+};
+
+size_t json_minify(const unsigned char *bytes, size_t how_many,
+                   unsigned char *out) {
+  size_t i = 0, pos = 0;
+  uint8_t quote = 0;
+  uint8_t nonescape = 1;
+
+  while (i < how_many) {
+    unsigned char c = bytes[i];
+    uint8_t *meta = jump_table + 3 * c;
+
+    quote = quote ^ (meta[0] & nonescape);
+    out[pos] = c;
+    pos += meta[2] | quote;
+
+    i += 1;
+    nonescape = (~nonescape) | (meta[1]);
+  }
+  return pos;
+}
+} // namespace simdjson
+#else
+#include "simdjson/simdprune_tables.h"
+#include <cstring>
+
+namespace simdjson {
+
+// some intrinsics are missing under GCC?
+#ifndef __clang__
+#ifndef _MSC_VER
+static __m256i inline _mm256_loadu2_m128i(__m128i const *__addr_hi,
+                                          __m128i const *__addr_lo) {
+  __m256i __v256 = _mm256_castsi128_si256(_mm_loadu_si128(__addr_lo));
+  return _mm256_insertf128_si256(__v256, _mm_loadu_si128(__addr_hi), 1);
+}
+
+static inline void _mm256_storeu2_m128i(__m128i *__addr_hi, __m128i *__addr_lo,
+                                        __m256i __a) {
+  __m128i __v128;
+  __v128 = _mm256_castsi256_si128(__a);
+  _mm_storeu_si128(__addr_lo, __v128);
+  __v128 = _mm256_extractf128_si256(__a, 1);
+  _mm_storeu_si128(__addr_hi, __v128);
+}
+#endif
+#endif
+
+// a straightforward comparison of a mask against input.
+static uint64_t cmp_mask_against_input_mini(__m256i input_lo, __m256i input_hi,
+                                            __m256i mask) {
+  __m256i cmp_res_0 = _mm256_cmpeq_epi8(input_lo, mask);
+  uint64_t res_0 = static_cast<uint32_t>(_mm256_movemask_epi8(cmp_res_0));
+  __m256i cmp_res_1 = _mm256_cmpeq_epi8(input_hi, mask);
+  uint64_t res_1 = _mm256_movemask_epi8(cmp_res_1);
+  return res_0 | (res_1 << 32);
+}
+
+// take input from buf and remove useless whitespace, input and output can be
+// the same, result is null terminated, return the string length (minus the null
+// termination)
+size_t json_minify(const uint8_t *buf, size_t len, uint8_t *out) {
+  // Useful constant masks
+  const uint64_t even_bits = 0x5555555555555555ULL;
+  const uint64_t odd_bits = ~even_bits;
+  uint8_t *initout(out);
+  uint64_t prev_iter_ends_odd_backslash =
+      0ULL;                               // either 0 or 1, but a 64-bit value
+  uint64_t prev_iter_inside_quote = 0ULL; // either all zeros or all ones
+  size_t idx = 0;
+  if (len >= 64) {
+    size_t avx_len = len - 63;
+
+    for (; idx < avx_len; idx += 64) {
+      __m256i input_lo =
+          _mm256_loadu_si256(reinterpret_cast<const __m256i *>(buf + idx + 0));
+      __m256i input_hi =
+          _mm256_loadu_si256(reinterpret_cast<const __m256i *>(buf + idx + 32));
+      uint64_t bs_bits = cmp_mask_against_input_mini(input_lo, input_hi,
+                                                     _mm256_set1_epi8('\\'));
+      uint64_t start_edges = bs_bits & ~(bs_bits << 1);
+      uint64_t even_start_mask = even_bits ^ prev_iter_ends_odd_backslash;
+      uint64_t even_starts = start_edges & even_start_mask;
+      uint64_t odd_starts = start_edges & ~even_start_mask;
+      uint64_t even_carries = bs_bits + even_starts;
+      uint64_t odd_carries;
+      bool iter_ends_odd_backslash =
+          add_overflow(bs_bits, odd_starts, &odd_carries);
+      odd_carries |= prev_iter_ends_odd_backslash;
+      prev_iter_ends_odd_backslash = iter_ends_odd_backslash ? 0x1ULL : 0x0ULL;
+      uint64_t even_carry_ends = even_carries & ~bs_bits;
+      uint64_t odd_carry_ends = odd_carries & ~bs_bits;
+      uint64_t even_start_odd_end = even_carry_ends & odd_bits;
+      uint64_t odd_start_even_end = odd_carry_ends & even_bits;
+      uint64_t odd_ends = even_start_odd_end | odd_start_even_end;
+      uint64_t quote_bits = cmp_mask_against_input_mini(input_lo, input_hi,
+                                                        _mm256_set1_epi8('"'));
+      quote_bits = quote_bits & ~odd_ends;
+      uint64_t quote_mask = _mm_cvtsi128_si64(_mm_clmulepi64_si128(
+          _mm_set_epi64x(0ULL, quote_bits), _mm_set1_epi8(0xFF), 0));
+      quote_mask ^= prev_iter_inside_quote;
+      prev_iter_inside_quote = static_cast<uint64_t>(
+          static_cast<int64_t>(quote_mask) >>
+          63); // might be undefined behavior, should be fully defined in C++20,
+               // ok according to John Regher from Utah University
+      const __m256i low_nibble_mask = _mm256_setr_epi8(
+          //  0                           9  a   b  c  d
+          16, 0, 0, 0, 0, 0, 0, 0, 0, 8, 12, 1, 2, 9, 0, 0, 16, 0, 0, 0, 0, 0,
+          0, 0, 0, 8, 12, 1, 2, 9, 0, 0);
+      const __m256i high_nibble_mask = _mm256_setr_epi8(
+          //  0     2   3     5     7
+          8, 0, 18, 4, 0, 1, 0, 1, 0, 0, 0, 3, 2, 1, 0, 0, 8, 0, 18, 4, 0, 1, 0,
+          1, 0, 0, 0, 3, 2, 1, 0, 0);
+      __m256i whitespace_shufti_mask = _mm256_set1_epi8(0x18);
+      __m256i v_lo = _mm256_and_si256(
+          _mm256_shuffle_epi8(low_nibble_mask, input_lo),
+          _mm256_shuffle_epi8(high_nibble_mask,
+                              _mm256_and_si256(_mm256_srli_epi32(input_lo, 4),
+                                               _mm256_set1_epi8(0x7f))));
+
+      __m256i v_hi = _mm256_and_si256(
+          _mm256_shuffle_epi8(low_nibble_mask, input_hi),
+          _mm256_shuffle_epi8(high_nibble_mask,
+                              _mm256_and_si256(_mm256_srli_epi32(input_hi, 4),
+                                               _mm256_set1_epi8(0x7f))));
+      __m256i tmp_ws_lo = _mm256_cmpeq_epi8(
+          _mm256_and_si256(v_lo, whitespace_shufti_mask), _mm256_set1_epi8(0));
+      __m256i tmp_ws_hi = _mm256_cmpeq_epi8(
+          _mm256_and_si256(v_hi, whitespace_shufti_mask), _mm256_set1_epi8(0));
+
+      uint64_t ws_res_0 =
+          static_cast<uint32_t>(_mm256_movemask_epi8(tmp_ws_lo));
+      uint64_t ws_res_1 = _mm256_movemask_epi8(tmp_ws_hi);
+      uint64_t whitespace = ~(ws_res_0 | (ws_res_1 << 32));
+      whitespace &= ~quote_mask;
+      int mask1 = whitespace & 0xFFFF;
+      int mask2 = (whitespace >> 16) & 0xFFFF;
+      int mask3 = (whitespace >> 32) & 0xFFFF;
+      int mask4 = (whitespace >> 48) & 0xFFFF;
+      int pop1 = hamming((~whitespace) & 0xFFFF);
+      int pop2 = hamming((~whitespace) & UINT64_C(0xFFFFFFFF));
+      int pop3 = hamming((~whitespace) & UINT64_C(0xFFFFFFFFFFFF));
+      int pop4 = hamming((~whitespace));
+      __m256i vmask1 = _mm256_loadu2_m128i(
+          reinterpret_cast<const __m128i *>(mask128_epi8) + (mask2 & 0x7FFF),
+          reinterpret_cast<const __m128i *>(mask128_epi8) + (mask1 & 0x7FFF));
+      __m256i vmask2 = _mm256_loadu2_m128i(
+          reinterpret_cast<const __m128i *>(mask128_epi8) + (mask4 & 0x7FFF),
+          reinterpret_cast<const __m128i *>(mask128_epi8) + (mask3 & 0x7FFF));
+      __m256i result1 = _mm256_shuffle_epi8(input_lo, vmask1);
+      __m256i result2 = _mm256_shuffle_epi8(input_hi, vmask2);
+      _mm256_storeu2_m128i(reinterpret_cast<__m128i *>(out + pop1),
+                           reinterpret_cast<__m128i *>(out), result1);
+      _mm256_storeu2_m128i(reinterpret_cast<__m128i *>(out + pop3),
+                           reinterpret_cast<__m128i *>(out + pop2), result2);
+      out += pop4;
+    }
+  }
+  // we finish off the job... copying and pasting the code is not ideal here,
+  // but it gets the job done.
+  if (idx < len) {
+    uint8_t buffer[64];
+    memset(buffer, 0, 64);
+    memcpy(buffer, buf + idx, len - idx);
+    __m256i input_lo =
+        _mm256_loadu_si256(reinterpret_cast<const __m256i *>(buffer));
+    __m256i input_hi =
+        _mm256_loadu_si256(reinterpret_cast<const __m256i *>(buffer + 32));
+    uint64_t bs_bits =
+        cmp_mask_against_input_mini(input_lo, input_hi, _mm256_set1_epi8('\\'));
+    uint64_t start_edges = bs_bits & ~(bs_bits << 1);
+    uint64_t even_start_mask = even_bits ^ prev_iter_ends_odd_backslash;
+    uint64_t even_starts = start_edges & even_start_mask;
+    uint64_t odd_starts = start_edges & ~even_start_mask;
+    uint64_t even_carries = bs_bits + even_starts;
+    uint64_t odd_carries;
+    // bool iter_ends_odd_backslash =
+    add_overflow(bs_bits, odd_starts, &odd_carries);
+    odd_carries |= prev_iter_ends_odd_backslash;
+    // prev_iter_ends_odd_backslash = iter_ends_odd_backslash ? 0x1ULL : 0x0ULL;
+    // // we never use it
+    uint64_t even_carry_ends = even_carries & ~bs_bits;
+    uint64_t odd_carry_ends = odd_carries & ~bs_bits;
+    uint64_t even_start_odd_end = even_carry_ends & odd_bits;
+    uint64_t odd_start_even_end = odd_carry_ends & even_bits;
+    uint64_t odd_ends = even_start_odd_end | odd_start_even_end;
+    uint64_t quote_bits =
+        cmp_mask_against_input_mini(input_lo, input_hi, _mm256_set1_epi8('"'));
+    quote_bits = quote_bits & ~odd_ends;
+    uint64_t quote_mask = _mm_cvtsi128_si64(_mm_clmulepi64_si128(
+        _mm_set_epi64x(0ULL, quote_bits), _mm_set1_epi8(0xFF), 0));
+    quote_mask ^= prev_iter_inside_quote;
+    // prev_iter_inside_quote = (uint64_t)((int64_t)quote_mask >> 63);// we
+    // don't need this anymore
+
+    __m256i mask_20 = _mm256_set1_epi8(0x20); // c==32
+    __m256i mask_70 =
+        _mm256_set1_epi8(0x70); // adding 0x70 does not check low 4-bits
+    // but moves any value >= 16 above 128
+
+    __m256i lut_cntrl = _mm256_setr_epi8(
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFF, 0xFF, 0x00,
+        0x00, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0xFF, 0xFF, 0x00, 0x00, 0xFF, 0x00, 0x00);
+
+    __m256i tmp_ws_lo = _mm256_or_si256(
+        _mm256_cmpeq_epi8(mask_20, input_lo),
+        _mm256_shuffle_epi8(lut_cntrl, _mm256_adds_epu8(mask_70, input_lo)));
+    __m256i tmp_ws_hi = _mm256_or_si256(
+        _mm256_cmpeq_epi8(mask_20, input_hi),
+        _mm256_shuffle_epi8(lut_cntrl, _mm256_adds_epu8(mask_70, input_hi)));
+    uint64_t ws_res_0 = static_cast<uint32_t>(_mm256_movemask_epi8(tmp_ws_lo));
+    uint64_t ws_res_1 = _mm256_movemask_epi8(tmp_ws_hi);
+    uint64_t whitespace = (ws_res_0 | (ws_res_1 << 32));
+    whitespace &= ~quote_mask;
+
+    if (len - idx < 64) {
+      whitespace |= UINT64_C(0xFFFFFFFFFFFFFFFF) << (len - idx);
+    }
+    int mask1 = whitespace & 0xFFFF;
+    int mask2 = (whitespace >> 16) & 0xFFFF;
+    int mask3 = (whitespace >> 32) & 0xFFFF;
+    int mask4 = (whitespace >> 48) & 0xFFFF;
+    int pop1 = hamming((~whitespace) & 0xFFFF);
+    int pop2 = hamming((~whitespace) & UINT64_C(0xFFFFFFFF));
+    int pop3 = hamming((~whitespace) & UINT64_C(0xFFFFFFFFFFFF));
+    int pop4 = hamming((~whitespace));
+    __m256i vmask1 = _mm256_loadu2_m128i(
+        reinterpret_cast<const __m128i *>(mask128_epi8) + (mask2 & 0x7FFF),
+        reinterpret_cast<const __m128i *>(mask128_epi8) + (mask1 & 0x7FFF));
+    __m256i vmask2 = _mm256_loadu2_m128i(
+        reinterpret_cast<const __m128i *>(mask128_epi8) + (mask4 & 0x7FFF),
+        reinterpret_cast<const __m128i *>(mask128_epi8) + (mask3 & 0x7FFF));
+    __m256i result1 = _mm256_shuffle_epi8(input_lo, vmask1);
+    __m256i result2 = _mm256_shuffle_epi8(input_hi, vmask2);
+    _mm256_storeu2_m128i(reinterpret_cast<__m128i *>(buffer + pop1),
+                         reinterpret_cast<__m128i *>(buffer), result1);
+    _mm256_storeu2_m128i(reinterpret_cast<__m128i *>(buffer + pop3),
+                         reinterpret_cast<__m128i *>(buffer + pop2), result2);
+    memcpy(out, buffer, pop4);
+    out += pop4;
+  }
+  *out = '\0'; // NULL termination
+  return out - initout;
+}
+} // namespace simdjson
+#endif

data/vendor/simdjson/src/jsonparser.cpp

@@ -0,0 +1,91 @@
+#include "simdjson/jsonparser.h"
+#include "simdjson/isadetection.h"
+#include "simdjson/portability.h"
+#include "simdjson/simdjson.h"
+
+namespace simdjson {
+
+// The function that users are expected to call is json_parse.
+// We have more than one such function because we want to support several
+// instruction sets.
+
+// function pointer type for json_parse
+using json_parse_functype = int(const uint8_t *buf, size_t len, ParsedJson &pj,
+                                bool realloc_if_needed);
+
+// Pointer that holds the json_parse implementation corresponding to the
+// available SIMD instruction set
+extern json_parse_functype *json_parse_ptr;
+
+int json_parse(const uint8_t *buf, size_t len, ParsedJson &pj,
+               bool realloc_if_needed) {
+  return json_parse_ptr(buf, len, pj, realloc_if_needed);
+}
+
+int json_parse(const char *buf, size_t len, ParsedJson &pj,
+               bool realloc_if_needed) {
+  return json_parse_ptr(reinterpret_cast<const uint8_t *>(buf), len, pj,
+                        realloc_if_needed);
+}
+
+Architecture find_best_supported_implementation() {
+  constexpr uint32_t haswell_flags =
+      instruction_set::AVX2 | instruction_set::PCLMULQDQ |
+      instruction_set::BMI1 | instruction_set::BMI2;
+  constexpr uint32_t westmere_flags =
+      instruction_set::SSE42 | instruction_set::PCLMULQDQ;
+
+  uint32_t supports = detect_supported_architectures();
+  // Order from best to worst (within architecture)
+  if ((haswell_flags & supports) == haswell_flags)
+    return Architecture::HASWELL;
+  if ((westmere_flags & supports) == westmere_flags)
+    return Architecture::WESTMERE;
+  if (instruction_set::NEON)
+    return Architecture::ARM64;
+
+  return Architecture::NONE;
+}
+
+// Responsible to select the best json_parse implementation
+int json_parse_dispatch(const uint8_t *buf, size_t len, ParsedJson &pj,
+                        bool realloc_if_needed) {
+  Architecture best_implementation = find_best_supported_implementation();
+  // Selecting the best implementation
+  switch (best_implementation) {
+#ifdef IS_X86_64
+  case Architecture::HASWELL:
+    json_parse_ptr = &json_parse_implementation<Architecture::HASWELL>;
+    break;
+  case Architecture::WESTMERE:
+    json_parse_ptr = &json_parse_implementation<Architecture::WESTMERE>;
+    break;
+#endif
+#ifdef IS_ARM64
+  case Architecture::ARM64:
+    json_parse_ptr = &json_parse_implementation<Architecture::ARM64>;
+    break;
+#endif
+  default:
+    std::cerr << "The processor is not supported by simdjson." << std::endl;
+    return simdjson::UNEXPECTED_ERROR;
+  }
+
+  return json_parse_ptr(buf, len, pj, realloc_if_needed);
+}
+
+json_parse_functype *json_parse_ptr = &json_parse_dispatch;
+
+WARN_UNUSED
+ParsedJson build_parsed_json(const uint8_t *buf, size_t len,
+                             bool realloc_if_needed) {
+  ParsedJson pj;
+  bool ok = pj.allocate_capacity(len);
+  if (ok) {
+    json_parse(buf, len, pj, realloc_if_needed);
+  } else {
+    std::cerr << "failure during memory allocation " << std::endl;
+  }
+  return pj;
+}
+} // namespace simdjson