simdjson 0.1.0

data/vendor/simdjson/tests/basictests.cpp

@@ -0,0 +1,75 @@
+#include <cassert>
+#include <cinttypes>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <iostream>
+#include <string>
+#include <vector>
+
+#include "simdjson/jsonparser.h"
+
+// returns true if successful
+bool skyprophet_test() {
+  const size_t n_records = 100000;
+  std::vector<std::string> data;
+  char buf[1024];
+  for (size_t i = 0; i < n_records; ++i) {
+    auto n = sprintf(buf,
+                     "{\"id\": %zu, \"name\": \"name%zu\", \"gender\": \"%s\", "
+                     "\"school\": {\"id\": %zu, \"name\": \"school%zu\"}}",
+                     i, i, (i % 2) ? "male" : "female", i % 10, i % 10);
+    data.emplace_back(std::string(buf, n));
+  }
+  for (size_t i = 0; i < n_records; ++i) {
+    auto n = sprintf(buf, "{\"counter\": %f, \"array\": [%s]}", i * 3.1416,
+                     (i % 2) ? "true" : "false");
+    data.emplace_back(std::string(buf, n));
+  }
+  for (size_t i = 0; i < n_records; ++i) {
+    auto n = sprintf(buf, "{\"number\": %e}", i * 10000.31321321);
+    data.emplace_back(std::string(buf, n));
+  }
+  data.emplace_back(std::string("true"));
+  data.emplace_back(std::string("false"));
+  data.emplace_back(std::string("null"));
+  data.emplace_back(std::string("0.1"));
+  size_t maxsize = 0;
+  for (auto &s : data) {
+    if (maxsize < s.size())
+      maxsize = s.size();
+  }
+  simdjson::ParsedJson pj;
+  if (!pj.allocate_capacity(maxsize)) {
+    printf("allocation failure in skyprophet_test\n");
+    return false;
+  }
+  size_t counter = 0;
+  for (auto &rec : data) {
+    if ((counter % 10000) == 0) {
+      printf(".");
+      fflush(NULL);
+    }
+    counter++;
+    auto ok1 = json_parse(rec.c_str(), rec.length(), pj);
+    if (ok1 != 0 || !pj.is_valid()) {
+      printf("Something is wrong in skyprophet_test: %s.\n", rec.c_str());
+      return false;
+    }
+    auto ok2 = json_parse(rec, pj);
+    if (ok2 != 0 || !pj.is_valid()) {
+      printf("Something is wrong in skyprophet_test: %s.\n", rec.c_str());
+      return false;
+    }
+  }
+  printf("\n");
+  return true;
+}
+
+int main() {
+  std::cout << "Running basic tests." << std::endl;
+  if (!skyprophet_test())
+    return EXIT_FAILURE;
+  std::cout << "Basic tests are ok." << std::endl;
+  return EXIT_SUCCESS;
+}

data/vendor/simdjson/tests/jsoncheck.cpp

@@ -0,0 +1,136 @@
+#include <cassert>
+#include <cstring>
+#ifndef _MSC_VER
+#include <dirent.h>
+#include <unistd.h>
+#else
+// Microsoft can't be bothered to provide standard utils.
+#include <dirent_portable.h>
+#endif
+#include <cinttypes>
+
+#include <cstdio>
+#include <cstdlib>
+
+#include "simdjson/jsonparser.h"
+
+/**
+ * Does the file filename ends with the given extension.
+ */
+static bool has_extension(const char *filename, const char *extension) {
+  const char *ext = strrchr(filename, '.');
+  return ((ext != nullptr) && (strcmp(ext, extension) == 0));
+}
+
+bool starts_with(const char *pre, const char *str) {
+  size_t len_pre = strlen(pre), len_str = strlen(str);
+  return len_str < len_pre ? false : strncmp(pre, str, len_pre) == 0;
+}
+
+bool contains(const char *pre, const char *str) {
+  return (strstr(str, pre) != nullptr);
+}
+
+bool validate(const char *dirname) {
+  bool everything_fine = true;
+  const char *extension = ".json";
+  size_t dirlen = strlen(dirname);
+  struct dirent **entry_list;
+  int c = scandir(dirname, &entry_list, nullptr, alphasort);
+  if (c < 0) {
+    fprintf(stderr, "error accessing %s \n", dirname);
+    return false;
+  }
+  if (c == 0) {
+    printf("nothing in dir %s \n", dirname);
+    return false;
+  }
+  bool *is_file_as_expected = new bool[c];
+  for (int i = 0; i < c; i++) {
+    is_file_as_expected[i] = true;
+  }
+  size_t how_many = 0;
+  bool needsep = (strlen(dirname) > 1) && (dirname[strlen(dirname) - 1] != '/');
+  for (int i = 0; i < c; i++) {
+    const char *name = entry_list[i]->d_name;
+    if (has_extension(name, extension)) {
+      printf("validating: file %s ", name);
+      fflush(nullptr);
+      size_t filelen = strlen(name);
+      char *fullpath = static_cast<char *>(malloc(dirlen + filelen + 1 + 1));
+      strcpy(fullpath, dirname);
+      if (needsep) {
+        fullpath[dirlen] = '/';
+        strcpy(fullpath + dirlen + 1, name);
+      } else {
+        strcpy(fullpath + dirlen, name);
+      }
+      simdjson::padded_string p;
+      try {
+        simdjson::get_corpus(fullpath).swap(p);
+      } catch (const std::exception &) {
+        std::cerr << "Could not load the file " << fullpath << std::endl;
+        return EXIT_FAILURE;
+      }
+      simdjson::ParsedJson pj;
+      bool allocok = pj.allocate_capacity(p.size(), 1024);
+      if (!allocok) {
+        std::cerr << "can't allocate memory" << std::endl;
+        return false;
+      }
+      ++how_many;
+      const int parse_res = json_parse(p, pj);
+      printf("%s\n", parse_res == 0 ? "ok" : "invalid");
+      if (contains("EXCLUDE", name)) {
+        // skipping
+        how_many--;
+      } else if (starts_with("pass", name) && parse_res != 0) {
+        is_file_as_expected[i] = false;
+        printf("warning: file %s should pass but it fails. Error is: %s\n",
+               name, simdjson::error_message(parse_res).data());
+        everything_fine = false;
+      } else if (starts_with("fail", name) && parse_res == 0) {
+        is_file_as_expected[i] = false;
+        printf("warning: file %s should fail but it passes.\n", name);
+        everything_fine = false;
+      }
+      free(fullpath);
+    }
+  }
+  printf("%zu files checked.\n", how_many);
+  if (everything_fine) {
+    printf("All ok!\n");
+  } else {
+    fprintf(stderr,
+            "There were problems! Consider reviewing the following files:\n");
+    for (int i = 0; i < c; i++) {
+      if (!is_file_as_expected[i]) {
+        fprintf(stderr, "%s \n", entry_list[i]->d_name);
+      }
+    }
+  }
+  for (int i = 0; i < c; ++i) {
+    free(entry_list[i]);
+  }
+  free(entry_list);
+  delete[] is_file_as_expected;
+  return everything_fine;
+}
+
+int main(int argc, char *argv[]) {
+  if (argc != 2) {
+    std::cerr << "Usage: " << argv[0] << " <directorywithjsonfiles>"
+              << std::endl;
+#ifndef SIMDJSON_TEST_DATA_DIR
+    std::cout
+        << "We are going to assume you mean to use the 'jsonchecker' directory."
+        << std::endl;
+    return validate("jsonchecker/") ? EXIT_SUCCESS : EXIT_FAILURE;
+#else
+    std::cout << "We are going to assume you mean to use the '"
+              << SIMDJSON_TEST_DATA_DIR << "' directory." << std::endl;
+    return validate(SIMDJSON_TEST_DATA_DIR) ? EXIT_SUCCESS : EXIT_FAILURE;
+#endif
+  }
+  return validate(argv[1]) ? EXIT_SUCCESS : EXIT_FAILURE;
+}

data/vendor/simdjson/tests/numberparsingcheck.cpp

@@ -0,0 +1,224 @@
+#include <assert.h>
+#include <cstring>
+#include <dirent.h>
+#include <inttypes.h>
+#include <math.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#ifndef JSON_TEST_NUMBERS
+#define JSON_TEST_NUMBERS
+#endif
+
+#include "simdjson/common_defs.h"
+
+// ulp distance
+// Marc B. Reynolds, 2016-2019
+// Public Domain under http://unlicense.org, see link for details.
+// adapted by D. Lemire
+inline uint32_t f32_ulp_dist(float a, float b) {
+  uint32_t ua, ub;
+  memcpy(&ua, &a, sizeof(ua));
+  memcpy(&ub, &b, sizeof(ub));
+  if ((int32_t)(ub ^ ua) >= 0)
+    return (int32_t)(ua - ub) >= 0 ? (ua - ub) : (ub - ua);
+  return ua + ub + 0x80000000;
+}
+
+// ulp distance
+// Marc B. Reynolds, 2016-2019
+// Public Domain under http://unlicense.org, see link for details.
+// adapted by D. Lemire
+inline uint64_t f64_ulp_dist(double a, double b) {
+  uint64_t ua, ub;
+  memcpy(&ua, &a, sizeof(ua));
+  memcpy(&ub, &b, sizeof(ub));
+  if ((int64_t)(ub ^ ua) >= 0)
+    return (int64_t)(ua - ub) >= 0 ? (ua - ub) : (ub - ua);
+  return ua + ub + 0x80000000;
+}
+
+int parse_error;
+char *fullpath;
+enum { PARSE_WARNING, PARSE_ERROR };
+
+size_t float_count;
+size_t int_count;
+size_t invalid_count;
+
+// strings that start with these should not be parsed as numbers
+const char *really_bad[] = {"013}", "0x14", "0e]", "0e+]", "0e+-1]"};
+
+bool starts_with(const char *pre, const char *str) {
+  size_t lenpre = strlen(pre);
+  return strncmp(pre, str, lenpre) == 0;
+}
+
+bool is_in_bad_list(const char *buf) {
+  if (buf[0] != '0')
+    return false;
+  for (size_t i = 0; i < sizeof(really_bad) / sizeof(really_bad[0]); i++)
+    if (starts_with(really_bad[i], buf))
+      return true;
+  return false;
+}
+
+void found_invalid_number(const uint8_t *buf) {
+  invalid_count++;
+  char *endptr;
+  double expected = strtod((const char *)buf, &endptr);
+  if (endptr != (const char *)buf) {
+    if (!is_in_bad_list((const char *)buf)) {
+      printf("Warning: found_invalid_number %.32s whereas strtod parses it to "
+             "%f, ",
+             buf, expected);
+      printf(" while parsing %s \n", fullpath);
+      parse_error |= PARSE_WARNING;
+    }
+  }
+}
+
+void found_integer(int64_t result, const uint8_t *buf) {
+  int_count++;
+  char *endptr;
+  long long expected = strtoll((const char *)buf, &endptr, 10);
+  if ((endptr == (const char *)buf) || (expected != result)) {
+    fprintf(stderr, "Error: parsed %" PRId64 " out of %.32s, ", result, buf);
+    fprintf(stderr, " while parsing %s \n", fullpath);
+    parse_error |= PARSE_ERROR;
+  }
+}
+
+void found_float(double result, const uint8_t *buf) {
+  char *endptr;
+  float_count++;
+  double expected = strtod((const char *)buf, &endptr);
+  if (endptr == (const char *)buf) {
+    fprintf(stderr,
+            "parsed %f from %.32s whereas strtod refuses to parse a float, ",
+            result, buf);
+    fprintf(stderr, " while parsing %s \n", fullpath);
+    parse_error |= PARSE_ERROR;
+  }
+  if (fpclassify(expected) != fpclassify(result)) {
+    fprintf(stderr,
+            "floats not in the same category expected: %f observed: %f \n",
+            expected, result);
+    fprintf(stderr, "%.32s\n", buf);
+    parse_error |= PARSE_ERROR;
+    return;
+  }
+  // we want to get some reasonable relative accuracy
+  uint64_t ULP = f64_ulp_dist(expected, result);
+  if (f64_ulp_dist(expected, result) > 1) {
+    fprintf(stderr, "parsed %.128e from \n", result);
+    fprintf(stderr, "       %.32s whereas strtod gives\n", buf);
+    fprintf(stderr, "       %.128e,", expected);
+    fprintf(stderr, " while parsing %s \n", fullpath);
+    fprintf(stderr, " ===========  ULP:  %u,", (unsigned int)ULP);
+    parse_error |= PARSE_ERROR;
+  }
+}
+
+#include "simdjson/jsonparser.h"
+#include "src/stage2_build_tape.cpp"
+
+/**
+ * Does the file filename ends with the given extension.
+ */
+static bool has_extension(const char *filename, const char *extension) {
+  const char *ext = strrchr(filename, '.');
+  return (ext && !strcmp(ext, extension));
+}
+
+bool validate(const char *dirname) {
+  parse_error = 0;
+  size_t total_count = 0;
+  const char *extension = ".json";
+  size_t dirlen = strlen(dirname);
+  struct dirent **entry_list;
+  int c = scandir(dirname, &entry_list, 0, alphasort);
+  if (c < 0) {
+    printf("error accessing %s \n", dirname);
+    return false;
+  }
+  if (c == 0) {
+    printf("nothing in dir %s \n", dirname);
+    return false;
+  }
+  bool needsep = (strlen(dirname) > 1) && (dirname[strlen(dirname) - 1] != '/');
+  for (int i = 0; i < c; i++) {
+    const char *name = entry_list[i]->d_name;
+    if (has_extension(name, extension)) {
+      size_t filelen = strlen(name);
+      fullpath = (char *)malloc(dirlen + filelen + 1 + 1);
+      strcpy(fullpath, dirname);
+      if (needsep) {
+        fullpath[dirlen] = '/';
+        strcpy(fullpath + dirlen + 1, name);
+      } else {
+        strcpy(fullpath + dirlen, name);
+      }
+      simdjson::padded_string p;
+      try {
+        simdjson::get_corpus(fullpath).swap(p);
+      } catch (const std::exception &e) {
+        std::cout << "Could not load the file " << fullpath << std::endl;
+        return EXIT_FAILURE;
+      }
+      // terrible hack but just to get it working
+      simdjson::ParsedJson pj;
+      bool allocok = pj.allocate_capacity(p.size(), 1024);
+      if (!allocok) {
+        std::cerr << "can't allocate memory" << std::endl;
+        return false;
+      }
+      float_count = 0;
+      int_count = 0;
+      invalid_count = 0;
+      total_count += float_count + int_count + invalid_count;
+      bool isok = json_parse(p, pj);
+      if (int_count + float_count + invalid_count > 0) {
+        printf("File %40s %s --- integers: %10zu floats: %10zu invalid: %10zu "
+               "total numbers: %10zu \n",
+               name, isok ? " is valid     " : " is not valid ", int_count,
+               float_count, invalid_count,
+               int_count + float_count + invalid_count);
+      }
+      free(fullpath);
+    }
+  }
+  if ((parse_error & PARSE_ERROR) != 0) {
+    printf("NUMBER PARSING FAILS?\n");
+  } else {
+    printf("All ok.\n");
+  }
+  for (int i = 0; i < c; ++i)
+    free(entry_list[i]);
+  free(entry_list);
+  return ((parse_error & PARSE_ERROR) == 0);
+}
+
+int main(int argc, char *argv[]) {
+  if (argc != 2) {
+    std::cerr << "Usage: " << argv[0] << " <directorywithjsonfiles>"
+              << std::endl;
+#if defined(SIMDJSON_TEST_DATA_DIR) && defined(SIMDJSON_BENCHMARK_DATA_DIR)
+    std::cout << "We are going to assume you mean to use the '"
+              << SIMDJSON_TEST_DATA_DIR << "'  and  '"
+              << SIMDJSON_BENCHMARK_DATA_DIR << "'directories." << std::endl;
+    return validate(SIMDJSON_TEST_DATA_DIR) &&
+                   validate(SIMDJSON_BENCHMARK_DATA_DIR)
+               ? EXIT_SUCCESS
+               : EXIT_FAILURE;
+#else
+    std::cout << "We are going to assume you mean to use the 'jsonchecker' and "
+                 "'jsonexamples' directories."
+              << std::endl;
+    return validate("jsonchecker/") && validate("jsonexamples/") ? EXIT_SUCCESS
+                                                                 : EXIT_FAILURE;
+#endif
+  }
+  return validate(argv[1]) ? EXIT_SUCCESS : EXIT_FAILURE;
+}

data/vendor/simdjson/tests/pointercheck.cpp

@@ -0,0 +1,38 @@
+#include <iostream>
+
+#include "simdjson/jsonparser.h"
+#include "simdjson/parsedjson.h"
+
+int main() {
+  // {"/~01abc": [0, {"\\\" 0": ["value0", "value1"]}]}"
+  std::string json =
+      "{\"/~01abc\": [0, {\"\\\\\\\" 0\": [\"value0\", \"value1\"]}]}";
+  simdjson::ParsedJson pj;
+  assert(pj.allocate_capacity(json.length()));
+  simdjson::json_parse(json.c_str(), json.length(), pj);
+  assert(pj.is_valid());
+  simdjson::ParsedJson::Iterator it(pj);
+
+  // valid JSON String Representation pointer
+  std::string pointer1("/~1~001abc/1/\\\\\\\" 0/0");
+  assert(it.move_to(pointer1.c_str(), pointer1.length()));
+  assert(it.is_string());
+  assert(it.get_string() == std::string("value0"));
+
+  // valid URI Fragment Identifier Representation pointer
+  std::string pointer2("#/~1~001abc/1/%x5C%x22%x200/1");
+  assert(it.move_to(pointer2.c_str(), pointer2.length()));
+  assert(it.is_string());
+  assert(it.get_string() == std::string("value1"));
+
+  // invalid pointer with leading 0 in index
+  std::string pointer3("#/~1~001abc/01");
+  assert(!it.move_to(pointer3.c_str(), pointer3.length())); // failed
+  assert(it.is_string());                           // has probably not moved
+  assert(it.get_string() == std::string("value1")); // has not move
+
+  // "the (nonexistent) member after the last array element"
+  std::string pointer4("/~1~001abc/-");
+  assert(it.move_to(pointer4.c_str(), pointer4.length()));
+  assert(it.get_type() == ']');
+}

data/vendor/simdjson/tests/singleheadertest.cpp

@@ -0,0 +1,22 @@
+#include "../singleheader/simdjson.h"
+#include <iostream>
+
+using namespace simdjson;
+
+int main() {
+  const char *filename = JSON_TEST_PATH;
+  padded_string p = get_corpus(filename);
+  ParsedJson pj = build_parsed_json(p); // do the parsing
+  if (!pj.is_valid()) {
+    return EXIT_FAILURE;
+  }
+  if (!pj.allocate_capacity(p.size())) {
+    return EXIT_FAILURE;
+  }
+  const int res = json_parse(p, pj);
+  if (res) {
+    std::cerr << error_message(res) << std::endl;
+    return EXIT_FAILURE;
+  }
+  return EXIT_SUCCESS;
+}