simdjson 0.1.0

data/vendor/simdjson/tests/stringparsingcheck.cpp

@@ -0,0 +1,408 @@
+#include <assert.h>
+#include <climits>
+#include <cstring>
+#include <dirent.h>
+#include <inttypes.h>
+#include <iostream>
+#include <math.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#ifndef JSON_TEST_STRINGS
+#define JSON_TEST_STRINGS
+#endif
+
+#include "simdjson/common_defs.h"
+
+char *fullpath;
+
+size_t bad_string;
+size_t good_string;
+size_t empty_string;
+
+size_t total_string_length;
+bool probable_bug;
+// borrowed code (sajson?)
+
+static inline bool read_hex(const char *p, unsigned &u) {
+  unsigned v = 0;
+  int i = 4;
+  while (i--) {
+    unsigned char c = *p++;
+    if (c >= '0' && c <= '9') {
+      c -= '0';
+    } else if (c >= 'a' && c <= 'f') {
+      c = c - 'a' + 10;
+    } else if (c >= 'A' && c <= 'F') {
+      c = c - 'A' + 10;
+    } else {
+      return false;
+    }
+    v = (v << 4) + c;
+  }
+
+  u = v;
+  return true;
+}
+
+static inline void write_utf8(unsigned codepoint, char *&end) {
+  if (codepoint < 0x80) {
+    *end++ = codepoint;
+  } else if (codepoint < 0x800) {
+    *end++ = 0xC0 | (codepoint >> 6);
+    *end++ = 0x80 | (codepoint & 0x3F);
+  } else if (codepoint < 0x10000) {
+    *end++ = 0xE0 | (codepoint >> 12);
+    *end++ = 0x80 | ((codepoint >> 6) & 0x3F);
+    *end++ = 0x80 | (codepoint & 0x3F);
+  } else {
+    assert(codepoint < 0x200000);
+    *end++ = 0xF0 | (codepoint >> 18);
+    *end++ = 0x80 | ((codepoint >> 12) & 0x3F);
+    *end++ = 0x80 | ((codepoint >> 6) & 0x3F);
+    *end++ = 0x80 | (codepoint & 0x3F);
+  }
+}
+
+static bool parse_string(const char *p, char *output, char **end) {
+  if (*p != '"')
+    return false;
+  p++;
+
+  for (;;) {
+#if (CHAR_MIN < 0) || (!defined(CHAR_MIN)) // the '!defined' is just paranoia
+    // in this path, char is *signed*
+    if ((*p >= 0 && *p < 0x20)) {
+      return false; // unescaped
+    }
+#else
+    // we have unsigned chars
+    if (*p < 0x20) {
+      return false; // unescaped
+    }
+#endif
+
+    switch (*p) {
+    case '"':
+      *output = '\0'; // end
+      *end = output;
+      return true;
+    case '\\':
+      ++p;
+
+      char replacement;
+      switch (*p) {
+      case '"':
+        replacement = '"';
+        goto replace;
+      case '\\':
+        replacement = '\\';
+        goto replace;
+      case '/':
+        replacement = '/';
+        goto replace;
+      case 'b':
+        replacement = '\b';
+        goto replace;
+      case 'f':
+        replacement = '\f';
+        goto replace;
+      case 'n':
+        replacement = '\n';
+        goto replace;
+      case 'r':
+        replacement = '\r';
+        goto replace;
+      case 't':
+        replacement = '\t';
+        goto replace;
+      replace:
+        *output++ = replacement;
+        ++p;
+        break;
+      case 'u': {
+        ++p;
+        unsigned u;
+        if (!read_hex(p, u))
+          return false;
+
+        p += 4;
+        if (u >= 0xD800 && u <= 0xDBFF) {
+          char p0 = p[0];
+          char p1 = p[1];
+          if (p0 != '\\' || p1 != 'u') {
+            return false;
+          }
+          p += 2;
+          unsigned v;
+          if (!read_hex(p, v))
+            return false;
+
+          p += 4;
+
+          if (v < 0xDC00 || v > 0xDFFF) {
+            return false;
+          }
+          u = 0x10000 + (((u - 0xD800) << 10) | (v - 0xDC00));
+        }
+        write_utf8(u, output);
+        break;
+      }
+      default:
+        return false;
+      }
+      break;
+
+    default:
+      // validate UTF-8
+      unsigned char c0 = p[0];
+      if (c0 < 128) {
+        *output++ = *p++;
+      } else if (c0 < 224) {
+        unsigned char c1 = p[1];
+        if (c1 < 128 || c1 >= 192) {
+          return false;
+        }
+        output[0] = c0;
+        output[1] = c1;
+        output += 2;
+        p += 2;
+      } else if (c0 < 240) {
+        unsigned char c1 = p[1];
+        if (c1 < 128 || c1 >= 192) {
+          return false;
+        }
+        unsigned char c2 = p[2];
+        if (c2 < 128 || c2 >= 192) {
+          return false;
+        }
+        output[0] = c0;
+        output[1] = c1;
+        output[2] = c2;
+        output += 3;
+        p += 3;
+      } else if (c0 < 248) {
+        unsigned char c1 = p[1];
+        if (c1 < 128 || c1 >= 192) {
+          return false;
+        }
+        unsigned char c2 = p[2];
+        if (c2 < 128 || c2 >= 192) {
+          return false;
+        }
+        unsigned char c3 = p[3];
+        if (c3 < 128 || c3 >= 192) {
+          return false;
+        }
+        output[0] = c0;
+        output[1] = c1;
+        output[2] = c2;
+        output[3] = c3;
+        output += 4;
+        p += 4;
+      } else {
+        return false;
+      }
+      break;
+    }
+  }
+}
+// end of borrowed code
+char *big_buffer; // global variable
+
+void found_bad_string(const uint8_t *buf) {
+  bad_string++;
+  char *end;
+  if (parse_string((const char *)buf, big_buffer, &end)) {
+    printf("WARNING: Sajson-like parser seems to think that the string is "
+           "valid %32s \n",
+           buf);
+    probable_bug = true;
+  }
+}
+
+void print_hex(const char *s, size_t len) {
+  for (size_t i = 0; i < len; i++) {
+    printf("%02x ", s[i] & 0xFF);
+  }
+}
+
+void print_cmp_hex(const char *s1, const char *s2, size_t len) {
+  for (size_t i = 0; i < len; i++) {
+    printf("%02x ", (s1[i] ^ s2[i]) & 0xFF);
+  }
+}
+
+void found_string(const uint8_t *buf, const uint8_t *parsed_begin,
+                  const uint8_t *parsed_end) {
+  size_t this_len = parsed_end - parsed_begin;
+  total_string_length += this_len;
+  good_string++;
+  char *end = NULL;
+  if (!parse_string((const char *)buf, big_buffer, &end)) {
+    printf("WARNING: reference parser seems to think that the string is NOT "
+           "valid %32s \n",
+           buf);
+  }
+  if (end == big_buffer) {
+    // we have a zero-length string
+    if (parsed_begin != parsed_end) {
+      printf("WARNING: We have a zero-length but gap is %zu \n",
+             (size_t)(parsed_end - parsed_begin));
+      probable_bug = true;
+    }
+    empty_string++;
+    return;
+  }
+  size_t len = end - big_buffer;
+  if (len != this_len) {
+    printf("WARNING: lengths on parsed strings disagree %zu %zu \n", this_len,
+           len);
+    printf("\nour parsed string  : '%*s'\n\n", (int)this_len,
+           (const char *)parsed_begin);
+    print_hex((const char *)parsed_begin, this_len);
+    printf("\n");
+
+    printf("reference parsing   :'%*s'\n\n", (int)len, big_buffer);
+    print_hex((const char *)big_buffer, len);
+    printf("\n");
+
+    probable_bug = true;
+  }
+  if (memcmp(big_buffer, parsed_begin, this_len) != 0) {
+    printf("WARNING: parsed strings disagree  \n");
+    printf("Lengths %zu %zu  \n", this_len, len);
+
+    printf("\nour parsed string  : '%*s'\n", (int)this_len,
+           (const char *)parsed_begin);
+    print_hex((const char *)parsed_begin, this_len);
+    printf("\n");
+
+    printf("reference parsing   :'%*s'\n", (int)len, big_buffer);
+    print_hex((const char *)big_buffer, len);
+    printf("\n");
+
+    print_cmp_hex((const char *)parsed_begin, big_buffer, this_len);
+
+    probable_bug = true;
+  }
+}
+
+#include "simdjson/jsonparser.h"
+#include "src/stage2_build_tape.cpp"
+
+/**
+ * Does the file filename ends with the given extension.
+ */
+static bool has_extension(const char *filename, const char *extension) {
+  const char *ext = strrchr(filename, '.');
+  return (ext && !strcmp(ext, extension));
+}
+
+bool starts_with(const char *pre, const char *str) {
+  size_t lenpre = strlen(pre), lenstr = strlen(str);
+  return lenstr < lenpre ? false : strncmp(pre, str, lenpre) == 0;
+}
+
+bool validate(const char *dirname) {
+  size_t total_strings = 0;
+  probable_bug = false;
+  const char *extension = ".json";
+  size_t dirlen = strlen(dirname);
+  struct dirent **entry_list;
+  int c = scandir(dirname, &entry_list, 0, alphasort);
+  if (c < 0) {
+    printf("error accessing %s \n", dirname);
+    return false;
+  }
+  if (c == 0) {
+    printf("nothing in dir %s \n", dirname);
+    return false;
+  }
+  bool needsep = (strlen(dirname) > 1) && (dirname[strlen(dirname) - 1] != '/');
+  for (int i = 0; i < c; i++) {
+    const char *name = entry_list[i]->d_name;
+    if (has_extension(name, extension)) {
+      size_t filelen = strlen(name);
+      fullpath = (char *)malloc(dirlen + filelen + 1 + 1);
+      strcpy(fullpath, dirname);
+      if (needsep) {
+        fullpath[dirlen] = '/';
+        strcpy(fullpath + dirlen + 1, name);
+      } else {
+        strcpy(fullpath + dirlen, name);
+      }
+      simdjson::padded_string p;
+      try {
+        simdjson::get_corpus(fullpath).swap(p);
+      } catch (const std::exception &e) {
+        std::cout << "Could not load the file " << fullpath << std::endl;
+        return EXIT_FAILURE;
+      }
+      simdjson::ParsedJson pj;
+      bool allocok = pj.allocate_capacity(p.size(), 1024);
+      if (!allocok) {
+        std::cerr << "can't allocate memory" << std::endl;
+        return false;
+      }
+      big_buffer = (char *)malloc(p.size());
+      if (big_buffer == NULL) {
+        std::cerr << "can't allocate memory" << std::endl;
+        return false;
+      }
+      bad_string = 0;
+      good_string = 0;
+      total_string_length = 0;
+      empty_string = 0;
+      bool isok = json_parse(p, pj);
+      free(big_buffer);
+      if (good_string > 0) {
+        printf("File %40s %s --- bad strings: %10zu \tgood strings: %10zu\t "
+               "empty strings: %10zu "
+               "\taverage string length: %.1f \n",
+               name, isok ? " is valid     " : " is not valid ", bad_string,
+               good_string, empty_string,
+               (double)total_string_length / good_string);
+      } else if (bad_string > 0) {
+        printf("File %40s %s --- bad strings: %10zu  \n", name,
+               isok ? " is valid     " : " is not valid ", bad_string);
+      }
+      total_strings += bad_string + good_string;
+      free(fullpath);
+    }
+  }
+  printf("%zu strings checked.\n", total_strings);
+  if (probable_bug) {
+    fprintf(stderr, "STRING PARSING FAILS?\n");
+  } else {
+    printf("All ok.\n");
+  }
+  for (int i = 0; i < c; ++i)
+    free(entry_list[i]);
+  free(entry_list);
+  return probable_bug == false;
+}
+
+int main(int argc, char *argv[]) {
+  if (argc != 2) {
+    std::cerr << "Usage: " << argv[0] << " <directorywithjsonfiles>"
+              << std::endl;
+#if defined(SIMDJSON_TEST_DATA_DIR) && defined(SIMDJSON_BENCHMARK_DATA_DIR)
+    std::cout << "We are going to assume you mean to use the '"
+              << SIMDJSON_TEST_DATA_DIR << "'  and  '"
+              << SIMDJSON_BENCHMARK_DATA_DIR << "'directories." << std::endl;
+    return validate(SIMDJSON_TEST_DATA_DIR) &&
+                   validate(SIMDJSON_BENCHMARK_DATA_DIR)
+               ? EXIT_SUCCESS
+               : EXIT_FAILURE;
+#else
+    std::cout << "We are going to assume you mean to use the 'jsonchecker' and "
+                 "'jsonexamples' directories."
+              << std::endl;
+    return validate("jsonchecker/") && validate("jsonexamples/") ? EXIT_SUCCESS
+                                                                 : EXIT_FAILURE;
+#endif
+  }
+  return validate(argv[1]) ? EXIT_SUCCESS : EXIT_FAILURE;
+}

data/vendor/simdjson/tools/CMakeLists.txt

@@ -0,0 +1,3 @@
+add_cpp_tool(json2json)
+add_cpp_tool(jsonstats)
+add_cpp_tool(minify)

data/vendor/simdjson/tools/cmake/FindCTargets.cmake

@@ -0,0 +1,15 @@
+function(add_cpp_test TEST_NAME)
+  add_executable(${TEST_NAME} ${TEST_NAME}.cpp)
+  target_link_libraries(${TEST_NAME} ${SIMDJSON_LIB_NAME})
+  add_test(${TEST_NAME} ${TEST_NAME})
+endfunction(add_cpp_test)
+
+function(add_cpp_benchmark BENCH_NAME)
+  add_executable(${BENCH_NAME} ${BENCH_NAME}.cpp)
+  target_link_libraries(${BENCH_NAME} ${SIMDJSON_LIB_NAME})
+endfunction(add_cpp_benchmark)
+
+function(add_cpp_tool TOOL_NAME)
+  add_executable(${TOOL_NAME} ${TOOL_NAME}.cpp)
+  target_link_libraries(${TOOL_NAME} ${SIMDJSON_LIB_NAME})
+endfunction(add_cpp_tool)

data/vendor/simdjson/tools/cmake/FindOptions.cmake

@@ -0,0 +1,52 @@
+macro(append var string)
+  set(${var} "${${var}} ${string}")
+endmacro(append)
+
+set(SANITIZE_FLAGS "")
+if(SIMDJSON_SANITIZE)
+  set(SIMDJSON_SANITIZE_FLAGS "-fsanitize=address -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=all")
+  if (CMAKE_COMPILER_IS_GNUCC)
+    # Ubuntu bug for GCC 5.0+ (safe for all versions)
+    append(CMAKE_EXE_LINKER_FLAGS "-fuse-ld=gold")
+    append(CMAKE_SHARED_LINKER_FLAGS "-fuse-ld=gold")
+  endif()
+endif()
+
+
+if(SIMDJSON_DISABLE_AVX) 
+  if(NOT MSVC)
+   set (OPT_FLAGS "${OPT_FLAGS} -mno-avx -mno-bmi -mno-pclmul -msse4.2")
+  else()
+   set (OPT_FLAGS "${OPT_FLAGS}")
+  endif()
+else() 
+  # some compilers like clang do not automagically define __AVX2__ and __BMI2__ even when the hardware supports it
+  if(NOT MSVC)
+   #set (OPT_FLAGS "${OPT_FLAGS} -mpclmul")
+  else()
+   #set (OPT_FLAGS "${OPT_FLAGS} /arch:AVX2")
+  endif()
+endif()
+
+if(NOT MSVC)
+set(CXXSTD_FLAGS "-std=c++17 -fPIC")
+endif()
+
+set(WARNING_FLAGS "-Wall")
+if(NOT MSVC)
+set(WARNING_FLAGS "${WARNING_FLAGS} -Wextra -Wsign-compare -Wshadow -Wwrite-strings -Wpointer-arith -Winit-self")
+set(CMAKE_C_FLAGS_DEBUG "-ggdb")
+set(CMAKE_C_FLAGS_RELEASE "-O3")
+endif()
+
+set(CMAKE_C_FLAGS "${STD_FLAGS} ${OPT_FLAGS} ${INCLUDE_FLAGS} ${WARNING_FLAGS} ${SIMDJSON_SANITIZE_FLAGS} ")
+
+if(NOT MSVC)
+set(CMAKE_CXX_FLAGS_DEBUG "-ggdb")
+set(CMAKE_CXX_FLAGS_RELEASE "-O3")
+endif()
+set(CMAKE_CXX_FLAGS "${CXXSTD_FLAGS} ${OPT_FLAGS} ${INCLUDE_FLAGS} ${WARNING_FLAGS} ${SIMDJSON_SANITIZE_FLAGS} ")
+
+if(MSVC)
+add_definitions( "${OPT_FLAGS} /W3 /D_CRT_SECURE_NO_WARNINGS /wd4005 /wd4996 /wd4267 /wd4244  /wd4113 /nologo")
+endif()

data/vendor/simdjson/tools/json2json.cpp

@@ -0,0 +1,112 @@
+#include <iostream>
+#ifndef _MSC_VER
+#include <unistd.h>
+#endif
+#include "simdjson/jsonioutil.h"
+#include "simdjson/jsonparser.h"
+
+void compute_dump(simdjson::ParsedJson::Iterator &pjh) {
+  if (pjh.is_object()) {
+    std::cout << "{";
+    if (pjh.down()) {
+      pjh.print(std::cout); // must be a string
+      std::cout << ":";
+      pjh.next();
+      compute_dump(pjh); // let us recurse
+      while (pjh.next()) {
+        std::cout << ",";
+        pjh.print(std::cout);
+        std::cout << ":";
+        pjh.next();
+        compute_dump(pjh); // let us recurse
+      }
+      pjh.up();
+    }
+    std::cout << "}";
+  } else if (pjh.is_array()) {
+    std::cout << "[";
+    if (pjh.down()) {
+      compute_dump(pjh); // let us recurse
+      while (pjh.next()) {
+        std::cout << ",";
+        compute_dump(pjh); // let us recurse
+      }
+      pjh.up();
+    }
+    std::cout << "]";
+  } else {
+    pjh.print(std::cout); // just print the lone value
+  }
+}
+
+int main(int argc, char *argv[]) {
+  bool rawdump = false;
+  bool apidump = false;
+
+#ifndef _MSC_VER
+  int c;
+
+  while ((c = getopt(argc, argv, "da")) != -1) {
+    switch (c) {
+    case 'd':
+      rawdump = true;
+      break;
+    case 'a':
+      apidump = true;
+      break;
+    default:
+      abort();
+    }
+  }
+#else
+  int optind = 1;
+#endif
+  if (optind >= argc) {
+    std::cerr << "Reads json in, out the result of the parsing. " << std::endl;
+    std::cerr << "Usage: " << argv[0] << " <jsonfile>" << std::endl;
+    std::cerr << "The -d flag dumps the raw content of the tape." << std::endl;
+
+    exit(1);
+  }
+  const char *filename = argv[optind];
+  if (optind + 1 < argc) {
+    std::cerr << "warning: ignoring everything after " << argv[optind + 1]
+              << std::endl;
+  }
+  simdjson::padded_string p;
+  try {
+    simdjson::get_corpus(filename).swap(p);
+  } catch (const std::exception &) { // caught by reference to base
+    std::cout << "Could not load the file " << filename << std::endl;
+    return EXIT_FAILURE;
+  }
+  simdjson::ParsedJson pj;
+  bool allocok = pj.allocate_capacity(p.size(), 1024);
+  if (!allocok) {
+    std::cerr << "failed to allocate memory" << std::endl;
+    return EXIT_FAILURE;
+  }
+  int res =
+      simdjson::json_parse(p, pj); // do the parsing, return false on error
+  if (res != simdjson::SUCCESS) {
+    std::cerr << " Parsing failed. Error is '" << simdjson::error_message(res)
+              << "'." << std::endl;
+    return EXIT_FAILURE;
+  }
+  if (apidump) {
+    simdjson::ParsedJson::Iterator pjh(pj);
+    if (!pjh.is_ok()) {
+      std::cerr << " Could not iterate parsed result. " << std::endl;
+      return EXIT_FAILURE;
+    }
+    compute_dump(pjh);
+  } else {
+    const bool is_ok =
+        rawdump ? pj.dump_raw_tape(std::cout) : pj.print_json(std::cout);
+    if (!is_ok) {
+      std::cerr << " Could not print out parsed result. " << std::endl;
+      return EXIT_FAILURE;
+    }
+  }
+  return EXIT_SUCCESS;
+}

data/vendor/simdjson/tools/jsonpointer.cpp

@@ -0,0 +1,93 @@
+#include "simdjson/jsonioutil.h"
+#include "simdjson/jsonparser.h"
+#include <iostream>
+
+void compute_dump(simdjson::ParsedJson::Iterator &pjh) {
+  if (pjh.is_object()) {
+    std::cout << "{";
+    if (pjh.down()) {
+      pjh.print(std::cout); // must be a string
+      std::cout << ":";
+      pjh.next();
+      compute_dump(pjh); // let us recurse
+      while (pjh.next()) {
+        std::cout << ",";
+        pjh.print(std::cout);
+        std::cout << ":";
+        pjh.next();
+        compute_dump(pjh); // let us recurse
+      }
+      pjh.up();
+    }
+    std::cout << "}";
+  } else if (pjh.is_array()) {
+    std::cout << "[";
+    if (pjh.down()) {
+      compute_dump(pjh); // let us recurse
+      while (pjh.next()) {
+        std::cout << ",";
+        compute_dump(pjh); // let us recurse
+      }
+      pjh.up();
+    }
+    std::cout << "]";
+  } else {
+    pjh.print(std::cout); // just print the lone value
+  }
+}
+
+int main(int argc, char *argv[]) {
+  if (argc < 3) {
+    std::cerr << "Usage: " << argv[0] << " <jsonfile> <jsonpath>" << std::endl;
+    std::cerr << "Follows the rfc6901 standard's syntax: "
+                 "https://tools.ietf.org/html/rfc6901"
+              << std::endl;
+    std::cerr << " Example: " << argv[0]
+              << " jsonexamples/small/demo.json /Image/Width /Image/Height "
+                 "/Image/IDs/2 "
+              << std::endl;
+    std::cerr << "Multiple <jsonpath> can be issued in the same command, but "
+                 "at least one is needed."
+              << std::endl;
+    exit(1);
+  }
+  const char *filename = argv[1];
+  simdjson::padded_string p;
+  try {
+    simdjson::get_corpus(filename).swap(p);
+  } catch (const std::exception &e) { // caught by reference to base
+    std::cout << "Could not load the file " << filename << std::endl;
+    return EXIT_FAILURE;
+  }
+  simdjson::ParsedJson pj;
+  bool allocok = pj.allocate_capacity(p.size(), 1024);
+  if (!allocok) {
+    std::cerr << "failed to allocate memory" << std::endl;
+    return EXIT_FAILURE;
+  }
+  int res =
+      simdjson::json_parse(p, pj); // do the parsing, return false on error
+  if (res) {
+    std::cerr << " Parsing failed with error " << simdjson::error_message(res)
+              << std::endl;
+    return EXIT_FAILURE;
+  }
+  std::cout << "[" << std::endl;
+  for (int idx = 2; idx < argc; idx++) {
+    const char *jsonpath = argv[idx];
+    simdjson::ParsedJson::Iterator it(pj);
+    if (it.move_to(std::string(jsonpath))) {
+      std::cout << "{\"jsonpath\": \"" << jsonpath << "\"," << std::endl;
+      std::cout << "\"value\":";
+      compute_dump(it);
+      std::cout << "}" << std::endl;
+    } else {
+      std::cout << "null" << std::endl;
+    }
+    if (idx + 1 < argc) {
+      std::cout << "," << std::endl;
+    }
+  }
+  std::cout << "]" << std::endl;
+  return EXIT_SUCCESS;
+}