simdjson 0.1.0

data/vendor/simdjson/src/parsedjson.cpp

@@ -0,0 +1,323 @@
+#include "simdjson/parsedjson.h"
+
+namespace simdjson {
+ParsedJson::ParsedJson()
+    : structural_indexes(nullptr), tape(nullptr),
+      containing_scope_offset(nullptr), ret_address(nullptr),
+      string_buf(nullptr), current_string_buf_loc(nullptr) {}
+
+ParsedJson::~ParsedJson() { deallocate(); }
+
+ParsedJson::ParsedJson(ParsedJson &&p)
+    : byte_capacity(p.byte_capacity), depth_capacity(p.depth_capacity),
+      tape_capacity(p.tape_capacity), string_capacity(p.string_capacity),
+      current_loc(p.current_loc), n_structural_indexes(p.n_structural_indexes),
+      structural_indexes(p.structural_indexes), tape(p.tape),
+      containing_scope_offset(p.containing_scope_offset),
+      ret_address(p.ret_address), string_buf(p.string_buf),
+      current_string_buf_loc(p.current_string_buf_loc), valid(p.valid) {
+  p.structural_indexes = nullptr;
+  p.tape = nullptr;
+  p.containing_scope_offset = nullptr;
+  p.ret_address = nullptr;
+  p.string_buf = nullptr;
+  p.current_string_buf_loc = nullptr;
+}
+
+WARN_UNUSED
+bool ParsedJson::allocate_capacity(size_t len, size_t max_depth) {
+  if (max_depth <= 0) {
+    max_depth = 1; // don't let the user allocate nothing
+  }
+  if (len <= 0) {
+    len = 64; // allocating 0 bytes is wasteful.
+  }
+  if (len > SIMDJSON_MAXSIZE_BYTES) {
+    return false;
+  }
+  if ((len <= byte_capacity) && (depth_capacity < max_depth)) {
+    return true;
+  }
+  deallocate();
+  valid = false;
+  byte_capacity = 0; // will only set it to len after allocations are a success
+  n_structural_indexes = 0;
+  uint32_t max_structures = ROUNDUP_N(len, 64) + 2 + 7;
+  structural_indexes = new (std::nothrow) uint32_t[max_structures];
+  // a pathological input like "[[[[..." would generate len tape elements, so
+  // need a capacity of len + 1
+  size_t local_tape_capacity = ROUNDUP_N(len + 1, 64);
+  // a document with only zero-length strings... could have len/3 string
+  // and we would need len/3 * 5 bytes on the string buffer
+  size_t local_string_capacity = ROUNDUP_N(5 * len / 3 + 32, 64);
+  string_buf = new (std::nothrow) uint8_t[local_string_capacity];
+  tape = new (std::nothrow) uint64_t[local_tape_capacity];
+  containing_scope_offset = new (std::nothrow) uint32_t[max_depth];
+#ifdef SIMDJSON_USE_COMPUTED_GOTO
+  ret_address = new (std::nothrow) void *[max_depth];
+#else
+  ret_address = new (std::nothrow) char[max_depth];
+#endif
+  if ((string_buf == nullptr) || (tape == nullptr) ||
+      (containing_scope_offset == nullptr) || (ret_address == nullptr) ||
+      (structural_indexes == nullptr)) {
+    std::cerr << "Could not allocate memory" << std::endl;
+    delete[] ret_address;
+    delete[] containing_scope_offset;
+    delete[] tape;
+    delete[] string_buf;
+    delete[] structural_indexes;
+
+    return false;
+  }
+  /*
+  // We do not need to initialize this content for parsing, though we could
+  // need to initialize it for safety.
+  memset(string_buf, 0 , local_string_capacity);
+  memset(structural_indexes, 0, max_structures * sizeof(uint32_t));
+  memset(tape, 0, local_tape_capacity * sizeof(uint64_t));
+  */
+  byte_capacity = len;
+  depth_capacity = max_depth;
+  tape_capacity = local_tape_capacity;
+  string_capacity = local_string_capacity;
+  return true;
+}
+
+bool ParsedJson::is_valid() const { return valid; }
+
+int ParsedJson::get_error_code() const { return error_code; }
+
+std::string ParsedJson::get_error_message() const {
+  return error_message(error_code);
+}
+
+void ParsedJson::deallocate() {
+  byte_capacity = 0;
+  depth_capacity = 0;
+  tape_capacity = 0;
+  string_capacity = 0;
+  delete[] ret_address;
+  delete[] containing_scope_offset;
+  delete[] tape;
+  delete[] string_buf;
+  delete[] structural_indexes;
+  valid = false;
+}
+
+void ParsedJson::init() {
+  current_string_buf_loc = string_buf;
+  current_loc = 0;
+  valid = false;
+}
+
+WARN_UNUSED
+bool ParsedJson::print_json(std::ostream &os) {
+  if (!valid) {
+    return false;
+  }
+  uint32_t string_length;
+  size_t tape_idx = 0;
+  uint64_t tape_val = tape[tape_idx];
+  uint8_t type = (tape_val >> 56);
+  size_t how_many = 0;
+  if (type == 'r') {
+    how_many = tape_val & JSON_VALUE_MASK;
+  } else {
+    fprintf(stderr, "Error: no starting root node?");
+    return false;
+  }
+  if (how_many > tape_capacity) {
+    fprintf(
+        stderr,
+        "We may be exceeding the tape capacity. Is this a valid document?\n");
+    return false;
+  }
+  tape_idx++;
+  bool *in_object = new bool[depth_capacity];
+  auto *in_object_idx = new size_t[depth_capacity];
+  int depth = 1; // only root at level 0
+  in_object_idx[depth] = 0;
+  in_object[depth] = false;
+  for (; tape_idx < how_many; tape_idx++) {
+    tape_val = tape[tape_idx];
+    uint64_t payload = tape_val & JSON_VALUE_MASK;
+    type = (tape_val >> 56);
+    if (!in_object[depth]) {
+      if ((in_object_idx[depth] > 0) && (type != ']')) {
+        os << ",";
+      }
+      in_object_idx[depth]++;
+    } else { // if (in_object) {
+      if ((in_object_idx[depth] > 0) && ((in_object_idx[depth] & 1) == 0) &&
+          (type != '}')) {
+        os << ",";
+      }
+      if (((in_object_idx[depth] & 1) == 1)) {
+        os << ":";
+      }
+      in_object_idx[depth]++;
+    }
+    switch (type) {
+    case '"': // we have a string
+      os << '"';
+      memcpy(&string_length, string_buf + payload, sizeof(uint32_t));
+      print_with_escapes(
+          (const unsigned char *)(string_buf + payload + sizeof(uint32_t)),
+          string_length);
+      os << '"';
+      break;
+    case 'l': // we have a long int
+      if (tape_idx + 1 >= how_many) {
+        delete[] in_object;
+        delete[] in_object_idx;
+        return false;
+      }
+      os << static_cast<int64_t>(tape[++tape_idx]);
+      break;
+    case 'd': // we have a double
+      if (tape_idx + 1 >= how_many) {
+        delete[] in_object;
+        delete[] in_object_idx;
+        return false;
+      }
+      double answer;
+      memcpy(&answer, &tape[++tape_idx], sizeof(answer));
+      os << answer;
+      break;
+    case 'n': // we have a null
+      os << "null";
+      break;
+    case 't': // we have a true
+      os << "true";
+      break;
+    case 'f': // we have a false
+      os << "false";
+      break;
+    case '{': // we have an object
+      os << '{';
+      depth++;
+      in_object[depth] = true;
+      in_object_idx[depth] = 0;
+      break;
+    case '}': // we end an object
+      depth--;
+      os << '}';
+      break;
+    case '[': // we start an array
+      os << '[';
+      depth++;
+      in_object[depth] = false;
+      in_object_idx[depth] = 0;
+      break;
+    case ']': // we end an array
+      depth--;
+      os << ']';
+      break;
+    case 'r': // we start and end with the root node
+      fprintf(stderr, "should we be hitting the root node?\n");
+      delete[] in_object;
+      delete[] in_object_idx;
+      return false;
+    default:
+      fprintf(stderr, "bug %c\n", type);
+      delete[] in_object;
+      delete[] in_object_idx;
+      return false;
+    }
+  }
+  delete[] in_object;
+  delete[] in_object_idx;
+  return true;
+}
+
+WARN_UNUSED
+bool ParsedJson::dump_raw_tape(std::ostream &os) {
+  if (!valid) {
+    return false;
+  }
+  uint32_t string_length;
+  size_t tape_idx = 0;
+  uint64_t tape_val = tape[tape_idx];
+  uint8_t type = (tape_val >> 56);
+  os << tape_idx << " : " << type;
+  tape_idx++;
+  size_t how_many = 0;
+  if (type == 'r') {
+    how_many = tape_val & JSON_VALUE_MASK;
+  } else {
+    fprintf(stderr, "Error: no starting root node?");
+    return false;
+  }
+  os << "\t// pointing to " << how_many << " (right after last node)\n";
+  uint64_t payload;
+  for (; tape_idx < how_many; tape_idx++) {
+    os << tape_idx << " : ";
+    tape_val = tape[tape_idx];
+    payload = tape_val & JSON_VALUE_MASK;
+    type = (tape_val >> 56);
+    switch (type) {
+    case '"': // we have a string
+      os << "string \"";
+      memcpy(&string_length, string_buf + payload, sizeof(uint32_t));
+      print_with_escapes(
+          (const unsigned char *)(string_buf + payload + sizeof(uint32_t)),
+          string_length);
+      os << '"';
+      os << '\n';
+      break;
+    case 'l': // we have a long int
+      if (tape_idx + 1 >= how_many) {
+        return false;
+      }
+      os << "integer " << static_cast<int64_t>(tape[++tape_idx]) << "\n";
+      break;
+    case 'd': // we have a double
+      os << "float ";
+      if (tape_idx + 1 >= how_many) {
+        return false;
+      }
+      double answer;
+      memcpy(&answer, &tape[++tape_idx], sizeof(answer));
+      os << answer << '\n';
+      break;
+    case 'n': // we have a null
+      os << "null\n";
+      break;
+    case 't': // we have a true
+      os << "true\n";
+      break;
+    case 'f': // we have a false
+      os << "false\n";
+      break;
+    case '{': // we have an object
+      os << "{\t// pointing to next tape location " << payload
+         << " (first node after the scope) \n";
+      break;
+    case '}': // we end an object
+      os << "}\t// pointing to previous tape location " << payload
+         << " (start of the scope) \n";
+      break;
+    case '[': // we start an array
+      os << "[\t// pointing to next tape location " << payload
+         << " (first node after the scope) \n";
+      break;
+    case ']': // we end an array
+      os << "]\t// pointing to previous tape location " << payload
+         << " (start of the scope) \n";
+      break;
+    case 'r': // we start and end with the root node
+      printf("end of root\n");
+      return false;
+    default:
+      return false;
+    }
+  }
+  tape_val = tape[tape_idx];
+  payload = tape_val & JSON_VALUE_MASK;
+  type = (tape_val >> 56);
+  os << tape_idx << " : " << type << "\t// pointing to " << payload
+     << " (start root)\n";
+  return true;
+}
+} // namespace simdjson

data/vendor/simdjson/src/parsedjsoniterator.cpp

@@ -0,0 +1,272 @@
+#include "simdjson/common_defs.h"
+#include "simdjson/parsedjson.h"
+#include <iterator>
+
+namespace simdjson {
+ParsedJson::Iterator::Iterator(ParsedJson &pj_)
+    : pj(pj_), depth(0), location(0), tape_length(0), depth_index(nullptr) {
+  if (!pj.is_valid()) {
+    throw InvalidJSON();
+  }
+  // we overallocate by "1" to silence a warning in Visual Studio
+  depth_index = new scopeindex_t[pj.depth_capacity + 1];
+  // memory allocation would throw
+  // if(depth_index == nullptr) {
+  //    return;
+  //}
+  depth_index[0].start_of_scope = location;
+  current_val = pj.tape[location++];
+  current_type = (current_val >> 56);
+  depth_index[0].scope_type = current_type;
+  if (current_type == 'r') {
+    tape_length = current_val & JSON_VALUE_MASK;
+    if (location < tape_length) {
+      // If we make it here, then depth_capacity must >=2, but the compiler
+      // may not know this.
+      current_val = pj.tape[location];
+      current_type = (current_val >> 56);
+      depth++;
+      depth_index[depth].start_of_scope = location;
+      depth_index[depth].scope_type = current_type;
+    }
+  } else {
+    // should never happen
+    throw InvalidJSON();
+  }
+}
+
+ParsedJson::Iterator::~Iterator() { delete[] depth_index; }
+
+ParsedJson::Iterator::Iterator(const Iterator &o) noexcept
+    : pj(o.pj), depth(o.depth), location(o.location), tape_length(0),
+      current_type(o.current_type), current_val(o.current_val),
+      depth_index(nullptr) {
+  depth_index = new scopeindex_t[pj.depth_capacity];
+  // allocation might throw
+  memcpy(depth_index, o.depth_index,
+         pj.depth_capacity * sizeof(depth_index[0]));
+  tape_length = o.tape_length;
+}
+
+ParsedJson::Iterator::Iterator(Iterator &&o) noexcept
+    : pj(o.pj), depth(o.depth), location(o.location),
+      tape_length(o.tape_length), current_type(o.current_type),
+      current_val(o.current_val), depth_index(o.depth_index) {
+  o.depth_index = nullptr; // we take ownership
+}
+
+bool ParsedJson::Iterator::print(std::ostream &os, bool escape_strings) const {
+  if (!is_ok()) {
+    return false;
+  }
+  switch (current_type) {
+  case '"': // we have a string
+    os << '"';
+    if (escape_strings) {
+      print_with_escapes(get_string(), os, get_string_length());
+    } else {
+      // was: os << get_string();, but given that we can include null chars, we
+      // have to do something crazier:
+      std::copy(get_string(), get_string() + get_string_length(),
+                std::ostream_iterator<char>(os));
+    }
+    os << '"';
+    break;
+  case 'l': // we have a long int
+    os << get_integer();
+    break;
+  case 'd':
+    os << get_double();
+    break;
+  case 'n': // we have a null
+    os << "null";
+    break;
+  case 't': // we have a true
+    os << "true";
+    break;
+  case 'f': // we have a false
+    os << "false";
+    break;
+  case '{': // we have an object
+  case '}': // we end an object
+  case '[': // we start an array
+  case ']': // we end an array
+    os << static_cast<char>(current_type);
+    break;
+  default:
+    return false;
+  }
+  return true;
+}
+
+bool ParsedJson::Iterator::move_to(const char *pointer, uint32_t length) {
+  char *new_pointer = nullptr;
+  if (pointer[0] == '#') {
+    // Converting fragment representation to string representation
+    new_pointer = new char[length];
+    uint32_t new_length = 0;
+    for (uint32_t i = 1; i < length; i++) {
+      if (pointer[i] == '%' && pointer[i + 1] == 'x') {
+        try {
+          int fragment =
+              std::stoi(std::string(&pointer[i + 2], 2), nullptr, 16);
+          if (fragment == '\\' || fragment == '"' || (fragment <= 0x1F)) {
+            // escaping the character
+            new_pointer[new_length] = '\\';
+            new_length++;
+          }
+          new_pointer[new_length] = fragment;
+          i += 3;
+        } catch (std::invalid_argument &) {
+          delete[] new_pointer;
+          return false; // the fragment is invalid
+        }
+      } else {
+        new_pointer[new_length] = pointer[i];
+      }
+      new_length++;
+    }
+    length = new_length;
+    pointer = new_pointer;
+  }
+
+  // saving the current state
+  size_t depth_s = depth;
+  size_t location_s = location;
+  uint8_t current_type_s = current_type;
+  uint64_t current_val_s = current_val;
+  scopeindex_t *depth_index_s = depth_index;
+
+  rewind(); // The json pointer is used from the root of the document.
+
+  bool found = relative_move_to(pointer, length);
+  delete[] new_pointer;
+
+  if (!found) {
+    // since the pointer has found nothing, we get back to the original
+    // position.
+    depth = depth_s;
+    location = location_s;
+    current_type = current_type_s;
+    current_val = current_val_s;
+    depth_index = depth_index_s;
+  }
+
+  return found;
+}
+
+bool ParsedJson::Iterator::relative_move_to(const char *pointer,
+                                            uint32_t length) {
+  if (length == 0) {
+    // returns the whole document
+    return true;
+  }
+
+  if (pointer[0] != '/') {
+    // '/' must be the first character
+    return false;
+  }
+
+  // finding the key in an object or the index in an array
+  std::string key_or_index;
+  uint32_t offset = 1;
+
+  // checking for the "-" case
+  if (is_array() && pointer[1] == '-') {
+    if (length != 2) {
+      // the pointer must be exactly "/-"
+      // there can't be anything more after '-' as an index
+      return false;
+    }
+    key_or_index = '-';
+    offset = length; // will skip the loop coming right after
+  }
+
+  // We either transform the first reference token to a valid json key
+  // or we make sure it is a valid index in an array.
+  for (; offset < length; offset++) {
+    if (pointer[offset] == '/') {
+      // beginning of the next key or index
+      break;
+    }
+    if (is_array() && (pointer[offset] < '0' || pointer[offset] > '9')) {
+      // the index of an array must be an integer
+      // we also make sure std::stoi won't discard whitespaces later
+      return false;
+    }
+    if (pointer[offset] == '~') {
+      // "~1" represents "/"
+      if (pointer[offset + 1] == '1') {
+        key_or_index += '/';
+        offset++;
+        continue;
+      }
+      // "~0" represents "~"
+      if (pointer[offset + 1] == '0') {
+        key_or_index += '~';
+        offset++;
+        continue;
+      }
+    }
+    if (pointer[offset] == '\\') {
+      if (pointer[offset + 1] == '\\' || pointer[offset + 1] == '"' ||
+          (pointer[offset + 1] <= 0x1F)) {
+        key_or_index += pointer[offset + 1];
+        offset++;
+        continue;
+      }
+      return false; // invalid escaped character
+    }
+    if (pointer[offset] == '\"') {
+      // unescaped quote character. this is an invalid case.
+      // lets do nothing and assume most pointers will be valid.
+      // it won't find any corresponding json key anyway.
+      // return false;
+    }
+    key_or_index += pointer[offset];
+  }
+
+  bool found = false;
+  if (is_object()) {
+    if (move_to_key(key_or_index.c_str(), key_or_index.length())) {
+      found = relative_move_to(pointer + offset, length - offset);
+    }
+  } else if (is_array()) {
+    if (key_or_index == "-") { // handling "-" case first
+      if (down()) {
+        while (next())
+          ; // moving to the end of the array
+        // moving to the nonexistent value right after...
+        size_t npos;
+        if ((current_type == '[') || (current_type == '{')) {
+          // we need to jump
+          npos = (current_val & JSON_VALUE_MASK);
+        } else {
+          npos =
+              location + ((current_type == 'd' || current_type == 'l') ? 2 : 1);
+        }
+        location = npos;
+        current_val = pj.tape[npos];
+        current_type = (current_val >> 56);
+        return true; // how could it fail ?
+      }
+    } else { // regular numeric index
+      // The index can't have a leading '0'
+      if (key_or_index[0] == '0' && key_or_index.length() > 1) {
+        return false;
+      }
+      // it cannot be empty
+      if (key_or_index.length() == 0) {
+        return false;
+      }
+      // we already checked the index contains only valid digits
+      uint32_t index = std::stoi(key_or_index);
+      if (move_to_index(index)) {
+        found = relative_move_to(pointer + offset, length - offset);
+      }
+    }
+  }
+
+  return found;
+}
+} // namespace simdjson

data/vendor/simdjson/src/simdjson.cpp

@@ -0,0 +1,30 @@
+#include "simdjson/simdjson.h"
+#include <map>
+
+namespace simdjson {
+const std::map<int, const std::string> error_strings = {
+    {SUCCESS, "No errors"},
+    {CAPACITY, "This ParsedJson can't support a document that big"},
+    {MEMALLOC, "Error allocating memory, we're most likely out of memory"},
+    {TAPE_ERROR, "Something went wrong while writing to the tape"},
+    {STRING_ERROR, "Problem while parsing a string"},
+    {T_ATOM_ERROR,
+     "Problem while parsing an atom starting with the letter 't'"},
+    {F_ATOM_ERROR,
+     "Problem while parsing an atom starting with the letter 'f'"},
+    {N_ATOM_ERROR,
+     "Problem while parsing an atom starting with the letter 'n'"},
+    {NUMBER_ERROR, "Problem while parsing a number"},
+    {UTF8_ERROR, "The input is not valid UTF-8"},
+    {UNITIALIZED, "Unitialized"},
+    {EMPTY, "Empty"},
+    {UNESCAPED_CHARS, "Within strings, some characters must be escapted, we "
+                      "found unescapted characters"},
+    {UNEXPECTED_ERROR, "Unexpected error, consider reporting this problem as "
+                       "you may have found a bug in simdjson"},
+};
+
+const std::string &error_message(const int error_code) {
+  return error_strings.at(error_code);
+}
+} // namespace simdjson

data/vendor/simdjson/src/stage1_find_marks.cpp

@@ -0,0 +1,41 @@
+#include "simdjson/portability.h"
+
+#ifdef IS_X86_64
+
+#include "simdjson/stage1_find_marks_haswell.h"
+#include "simdjson/stage1_find_marks_westmere.h"
+TARGET_HASWELL
+namespace simdjson {
+template <>
+int find_structural_bits<Architecture::HASWELL>(const uint8_t *buf, size_t len,
+                                                ParsedJson &pj) {
+  FIND_STRUCTURAL_BITS(Architecture::HASWELL, buf, len, pj,
+                       simdjson::haswell::flatten_bits);
+}
+} // namespace simdjson
+UNTARGET_REGION
+
+TARGET_WESTMERE
+namespace simdjson {
+template <>
+int find_structural_bits<Architecture::WESTMERE>(const uint8_t *buf, size_t len,
+                                                 ParsedJson &pj) {
+  FIND_STRUCTURAL_BITS(Architecture::WESTMERE, buf, len, pj,
+                       simdjson::flatten_bits);
+}
+} // namespace simdjson
+UNTARGET_REGION
+
+#endif
+
+#ifdef IS_ARM64
+#include "simdjson/stage1_find_marks_arm64.h"
+namespace simdjson {
+template <>
+int find_structural_bits<Architecture::ARM64>(const uint8_t *buf, size_t len,
+                                              ParsedJson &pj) {
+  FIND_STRUCTURAL_BITS(Architecture::ARM64, buf, len, pj,
+                       simdjson::flatten_bits);
+}
+} // namespace simdjson
+#endif