simdjson 0.1.0

data/vendor/simdjson/include/simdjson/jsonformatutils.h

@@ -0,0 +1,202 @@
+#ifndef SIMDJSON_JSONFORMATUTILS_H
+#define SIMDJSON_JSONFORMATUTILS_H
+
+#include <cstdio>
+#include <iomanip>
+#include <iostream>
+
+namespace simdjson {
+// ends with zero char
+static inline void print_with_escapes(const unsigned char *src) {
+  while (*src) {
+    switch (*src) {
+    case '\b':
+      putchar('\\');
+      putchar('b');
+      break;
+    case '\f':
+      putchar('\\');
+      putchar('f');
+      break;
+    case '\n':
+      putchar('\\');
+      putchar('n');
+      break;
+    case '\r':
+      putchar('\\');
+      putchar('r');
+      break;
+    case '\"':
+      putchar('\\');
+      putchar('"');
+      break;
+    case '\t':
+      putchar('\\');
+      putchar('t');
+      break;
+    case '\\':
+      putchar('\\');
+      putchar('\\');
+      break;
+    default:
+      if (*src <= 0x1F) {
+        printf("\\u%04x", *src);
+      } else {
+        putchar(*src);
+      }
+    }
+    src++;
+  }
+}
+
+// ends with zero char
+static inline void print_with_escapes(const unsigned char *src,
+                                      std::ostream &os) {
+  while (*src) {
+    switch (*src) {
+    case '\b':
+      os << '\\';
+      os << 'b';
+      break;
+    case '\f':
+      os << '\\';
+      os << 'f';
+      break;
+    case '\n':
+      os << '\\';
+      os << 'n';
+      break;
+    case '\r':
+      os << '\\';
+      os << 'r';
+      break;
+    case '\"':
+      os << '\\';
+      os << '"';
+      break;
+    case '\t':
+      os << '\\';
+      os << 't';
+      break;
+    case '\\':
+      os << '\\';
+      os << '\\';
+      break;
+    default:
+      if (*src <= 0x1F) {
+        std::ios::fmtflags f(os.flags());
+        os << std::hex << std::setw(4) << std::setfill('0')
+           << static_cast<int>(*src);
+        os.flags(f);
+      } else {
+        os << *src;
+      }
+    }
+    src++;
+  }
+}
+
+// print len chars
+static inline void print_with_escapes(const unsigned char *src, size_t len) {
+  const unsigned char *finalsrc = src + len;
+  while (src < finalsrc) {
+    switch (*src) {
+    case '\b':
+      putchar('\\');
+      putchar('b');
+      break;
+    case '\f':
+      putchar('\\');
+      putchar('f');
+      break;
+    case '\n':
+      putchar('\\');
+      putchar('n');
+      break;
+    case '\r':
+      putchar('\\');
+      putchar('r');
+      break;
+    case '\"':
+      putchar('\\');
+      putchar('"');
+      break;
+    case '\t':
+      putchar('\\');
+      putchar('t');
+      break;
+    case '\\':
+      putchar('\\');
+      putchar('\\');
+      break;
+    default:
+      if (*src <= 0x1F) {
+        printf("\\u%04x", *src);
+      } else {
+        putchar(*src);
+      }
+    }
+    src++;
+  }
+}
+
+// print len chars
+static inline void print_with_escapes(const unsigned char *src,
+                                      std::ostream &os, size_t len) {
+  const unsigned char *finalsrc = src + len;
+  while (src < finalsrc) {
+    switch (*src) {
+    case '\b':
+      os << '\\';
+      os << 'b';
+      break;
+    case '\f':
+      os << '\\';
+      os << 'f';
+      break;
+    case '\n':
+      os << '\\';
+      os << 'n';
+      break;
+    case '\r':
+      os << '\\';
+      os << 'r';
+      break;
+    case '\"':
+      os << '\\';
+      os << '"';
+      break;
+    case '\t':
+      os << '\\';
+      os << 't';
+      break;
+    case '\\':
+      os << '\\';
+      os << '\\';
+      break;
+    default:
+      if (*src <= 0x1F) {
+        std::ios::fmtflags f(os.flags());
+        os << std::hex << std::setw(4) << std::setfill('0')
+           << static_cast<int>(*src);
+        os.flags(f);
+      } else {
+        os << *src;
+      }
+    }
+    src++;
+  }
+}
+
+static inline void print_with_escapes(const char *src, std::ostream &os) {
+  print_with_escapes(reinterpret_cast<const unsigned char *>(src), os);
+}
+
+static inline void print_with_escapes(const char *src, std::ostream &os,
+                                      size_t len) {
+  print_with_escapes(reinterpret_cast<const unsigned char *>(src), os, len);
+}
+} // namespace simdjson
+
+#
+#endif

data/vendor/simdjson/include/simdjson/jsonioutil.h

@@ -0,0 +1,32 @@
+#ifndef SIMDJSON_JSONIOUTIL_H
+#define SIMDJSON_JSONIOUTIL_H
+
+#include "simdjson/common_defs.h"
+#include <exception>
+#include <fstream>
+#include <iostream>
+#include <sstream>
+#include <string>
+
+#include "simdjson/padded_string.h"
+
+namespace simdjson {
+
+// load a file in memory...
+// get a corpus; pad out to cache line so we can always use SIMD
+// throws exceptions in case of failure
+// first element of the pair is a string (null terminated)
+// whereas the second element is the length.
+// caller is responsible to free (aligned_free((void*)result.data())))
+//
+// throws an exception if the file cannot be opened, use try/catch
+//      try {
+//        p = get_corpus(filename);
+//      } catch (const std::exception& e) {
+//        aligned_free((void*)p.data());
+//        std::cout << "Could not load the file " << filename << std::endl;
+//      }
+padded_string get_corpus(const std::string &filename);
+} // namespace simdjson
+
+#endif

data/vendor/simdjson/include/simdjson/jsonminifier.h

@@ -0,0 +1,30 @@
+#ifndef SIMDJSON_JSONMINIFIER_H
+#define SIMDJSON_JSONMINIFIER_H
+
+#include "simdjson/padded_string.h"
+#include <cstddef>
+#include <cstdint>
+#include <string_view>
+
+namespace simdjson {
+
+// Take input from buf and remove useless whitespace, write it to out; buf and
+// out can be the same pointer. Result is null terminated,
+// return the string length (minus the null termination).
+// The accelerated version of this function only runs on AVX2 hardware.
+size_t json_minify(const uint8_t *buf, size_t len, uint8_t *out);
+
+static inline size_t json_minify(const char *buf, size_t len, char *out) {
+  return json_minify(reinterpret_cast<const uint8_t *>(buf), len,
+                     reinterpret_cast<uint8_t *>(out));
+}
+
+static inline size_t json_minify(const std::string_view &p, char *out) {
+  return json_minify(p.data(), p.size(), out);
+}
+
+static inline size_t json_minify(const padded_string &p, char *out) {
+  return json_minify(p.data(), p.size(), out);
+}
+} // namespace simdjson
+#endif

data/vendor/simdjson/include/simdjson/jsonparser.h

@@ -0,0 +1,250 @@
+#ifndef SIMDJSON_JSONPARSER_H
+#define SIMDJSON_JSONPARSER_H
+#include "simdjson/common_defs.h"
+#include "simdjson/jsonioutil.h"
+#include "simdjson/padded_string.h"
+#include "simdjson/parsedjson.h"
+#include "simdjson/simdjson.h"
+#include "simdjson/stage1_find_marks.h"
+#include "simdjson/stage2_build_tape.h"
+#include <string>
+#ifdef _MSC_VER
+#include <windows.h>
+// must be included after windows.h
+#include <sysinfoapi.h>
+#else
+#include <unistd.h>
+#endif
+
+namespace simdjson {
+// json_parse_implementation is the generic function, it is specialized for
+// various architectures, e.g., as
+// json_parse_implementation<Architecture::HASWELL> or
+// json_parse_implementation<Architecture::ARM64>
+template <Architecture T>
+int json_parse_implementation(const uint8_t *buf, size_t len, ParsedJson &pj,
+                              bool realloc_if_needed = true) {
+  if (pj.byte_capacity < len) {
+    return simdjson::CAPACITY;
+  }
+  bool reallocated = false;
+  if (realloc_if_needed) {
+#if ALLOW_SAME_PAGE_BUFFER_OVERRUN
+// realloc is needed if the end of the memory crosses a page
+#ifdef _MSC_VER
+    SYSTEM_INFO sysInfo;
+    GetSystemInfo(&sysInfo);
+    long page_size = sysInfo.dwPageSize;
+#else
+    long page_size = sysconf(_SC_PAGESIZE);
+#endif
+    //////////////
+    // We want to check that buf + len - 1 and buf + len - 1 + SIMDJSON_PADDING
+    // are in the same page.
+    // That is, we want to check that
+    // (buf + len - 1) / page_size == (buf + len - 1 + SIMDJSON_PADDING) /
+    // page_size That's true if (buf + len - 1) % page_size + SIMDJSON_PADDING <
+    // page_size.
+    ///////////
+    if ((reinterpret_cast<uintptr_t>(buf + len - 1) % page_size) +
+            SIMDJSON_PADDING <
+        static_cast<uintptr_t>(page_size)) {
+#else // SIMDJSON_SAFE_SAME_PAGE_READ_OVERRUN
+    if (true) { // if not SIMDJSON_SAFE_SAME_PAGE_READ_OVERRUN, we always
+                // reallocate
+#endif
+      const uint8_t *tmp_buf = buf;
+      buf = (uint8_t *)allocate_padded_buffer(len);
+      if (buf == NULL)
+        return simdjson::MEMALLOC;
+      memcpy((void *)buf, tmp_buf, len);
+      reallocated = true;
+    } // if (true) OR if ( (reinterpret_cast<uintptr_t>(buf + len - 1) %
+      // page_size ) + SIMDJSON_PADDING < static_cast<uintptr_t>(page_size) ) {
+  }   // if(realloc_if_needed) {
+  int stage1_is_ok = simdjson::find_structural_bits<T>(buf, len, pj);
+  if (stage1_is_ok != simdjson::SUCCESS) {
+    pj.error_code = stage1_is_ok;
+    return pj.error_code;
+  }
+  int res = unified_machine<T>(buf, len, pj);
+  if (reallocated) {
+    aligned_free((void *)buf);
+  }
+  return res;
+}
+
+// Parse a document found in buf.
+//
+// The content should be a valid JSON document encoded as UTF-8. If there is a
+// UTF-8 BOM, the caller is responsible for omitting it, UTF-8 BOM are
+// discouraged.
+//
+// You need to preallocate ParsedJson with a capacity of len (e.g.,
+// pj.allocate_capacity(len)).
+//
+// The function returns simdjson::SUCCESS (an integer = 0) in case of a success
+// or an error code from simdjson/simdjson.h in case of failure such as
+// simdjson::CAPACITY, simdjson::MEMALLOC, simdjson::DEPTH_ERROR and so forth;
+// the simdjson::error_message function converts these error codes into a
+// string).
+//
+// You can also check validity by calling pj.is_valid(). The same ParsedJson can
+// be reused for other documents.
+//
+// If realloc_if_needed is true (default) then a temporary buffer is created
+// when needed during processing (a copy of the input string is made). The input
+// buf should be readable up to buf + len + SIMDJSON_PADDING if
+// realloc_if_needed is false, all bytes at and after buf + len  are ignored
+// (can be garbage). The ParsedJson object can be reused.
+
+int json_parse(const uint8_t *buf, size_t len, ParsedJson &pj,
+               bool realloc_if_needed = true);
+
+// Parse a document found in buf.
+//
+// The content should be a valid JSON document encoded as UTF-8. If there is a
+// UTF-8 BOM, the caller is responsible for omitting it, UTF-8 BOM are
+// discouraged.
+//
+// You need to preallocate ParsedJson with a capacity of len (e.g.,
+// pj.allocate_capacity(len)).
+//
+// The function returns simdjson::SUCCESS (an integer = 0) in case of a success
+// or an error code from simdjson/simdjson.h in case of failure such as
+// simdjson::CAPACITY, simdjson::MEMALLOC, simdjson::DEPTH_ERROR and so forth;
+// the simdjson::error_message function converts these error codes into a
+// string).
+//
+// You can also check validity
+// by calling pj.is_valid(). The same ParsedJson can be reused for other
+// documents.
+//
+// If realloc_if_needed is true (default) then a temporary buffer is created
+// when needed during processing (a copy of the input string is made). The input
+// buf should be readable up to buf + len + SIMDJSON_PADDING  if
+// realloc_if_needed is false, all bytes at and after buf + len  are ignored
+// (can be garbage). The ParsedJson object can be reused.
+int json_parse(const char *buf, size_t len, ParsedJson &pj,
+               bool realloc_if_needed = true);
+
+// We do not want to allow implicit conversion from C string to std::string.
+int json_parse(const char *buf, ParsedJson &pj) = delete;
+
+// Parse a document found in in string s.
+// You need to preallocate ParsedJson with a capacity of len (e.g.,
+// pj.allocate_capacity(len)).
+//
+// The function returns simdjson::SUCCESS (an integer = 0) in case of a success
+// or an error code from simdjson/simdjson.h in case of failure such as
+// simdjson::CAPACITY, simdjson::MEMALLOC, simdjson::DEPTH_ERROR and so forth;
+// the simdjson::error_message function converts these error codes into a
+// string).
+//
+// A temporary buffer is created when needed during processing
+// (a copy of the input string is made).
+inline int json_parse(const std::string &s, ParsedJson &pj) {
+  return json_parse(s.data(), s.length(), pj, true);
+}
+
+// Parse a document found in in string s.
+//
+// The content should be a valid JSON document encoded as UTF-8. If there is a
+// UTF-8 BOM, the caller is responsible for omitting it, UTF-8 BOM are
+// discouraged.
+//
+// You need to preallocate ParsedJson with a capacity of len (e.g.,
+// pj.allocate_capacity(len)).
+//
+// The function returns simdjson::SUCCESS (an integer = 0) in case of a success
+// or an error code from simdjson/simdjson.h in case of failure such as
+// simdjson::CAPACITY, simdjson::MEMALLOC, simdjson::DEPTH_ERROR and so forth;
+// the simdjson::error_message function converts these error codes into a
+// string).
+//
+// You can also check validity
+// by calling pj.is_valid(). The same ParsedJson can be reused for other
+// documents.
+inline int json_parse(const padded_string &s, ParsedJson &pj) {
+  return json_parse(s.data(), s.length(), pj, false);
+}
+
+// Build a ParsedJson object. You can check validity
+// by calling pj.is_valid(). This does the memory allocation needed for
+// ParsedJson. If realloc_if_needed is true (default) then a temporary buffer is
+// created when needed during processing (a copy of the input string is made).
+//
+// The input buf should be readable up to buf + len + SIMDJSON_PADDING  if
+// realloc_if_needed is false, all bytes at and after buf + len  are ignored
+// (can be garbage).
+//
+// The content should be a valid JSON document encoded as UTF-8. If there is a
+// UTF-8 BOM, the caller is responsible for omitting it, UTF-8 BOM are
+// discouraged.
+//
+// This is a convenience function which calls json_parse.
+WARN_UNUSED
+ParsedJson build_parsed_json(const uint8_t *buf, size_t len,
+                             bool realloc_if_needed = true);
+
+WARN_UNUSED
+// Build a ParsedJson object. You can check validity
+// by calling pj.is_valid(). This does the memory allocation needed for
+// ParsedJson. If realloc_if_needed is true (default) then a temporary buffer is
+// created when needed during processing (a copy of the input string is made).
+//
+// The input buf should be readable up to buf + len + SIMDJSON_PADDING if
+// realloc_if_needed is false, all bytes at and after buf + len  are ignored
+// (can be garbage).
+//
+//
+// The content should be a valid JSON document encoded as UTF-8. If there is a
+// UTF-8 BOM, the caller is responsible for omitting it, UTF-8 BOM are
+// discouraged.
+//
+// This is a convenience function which calls json_parse.
+inline ParsedJson build_parsed_json(const char *buf, size_t len,
+                                    bool realloc_if_needed = true) {
+  return build_parsed_json(reinterpret_cast<const uint8_t *>(buf), len,
+                           realloc_if_needed);
+}
+
+// We do not want to allow implicit conversion from C string to std::string.
+ParsedJson build_parsed_json(const char *buf) = delete;
+
+// Parse a document found in in string s.
+// You need to preallocate ParsedJson with a capacity of len (e.g.,
+// pj.allocate_capacity(len)). Return SUCCESS (an integer = 0) in case of a
+// success. You can also check validity by calling pj.is_valid(). The same
+// ParsedJson can be reused for other documents.
+//
+// A temporary buffer is created when needed during processing
+// (a copy of the input string is made).
+//
+// The content should be a valid JSON document encoded as UTF-8. If there is a
+// UTF-8 BOM, the caller is responsible for omitting it, UTF-8 BOM are
+// discouraged.
+//
+// This is a convenience function which calls json_parse.
+WARN_UNUSED
+inline ParsedJson build_parsed_json(const std::string &s) {
+  return build_parsed_json(s.data(), s.length(), true);
+}
+
+// Parse a document found in in string s.
+// You need to preallocate ParsedJson with a capacity of len (e.g.,
+// pj.allocate_capacity(len)). Return SUCCESS (an integer = 0) in case of a
+// success. You can also check validity by calling pj.is_valid(). The same
+// ParsedJson can be reused for other documents.
+//
+// The content should be a valid JSON document encoded as UTF-8. If there is a
+// UTF-8 BOM, the caller is responsible for omitting it, UTF-8 BOM are
+// discouraged.
+//
+// This is a convenience function which calls json_parse.
+WARN_UNUSED
+inline ParsedJson build_parsed_json(const padded_string &s) {
+  return build_parsed_json(s.data(), s.length(), false);
+}
+} // namespace simdjson
+#endif