simdjson 0.1.0

data/vendor/simdjson/benchmark/parsingcompetition.cpp

@@ -0,0 +1,298 @@
+#include "simdjson/jsonparser.h"
+#ifndef _MSC_VER
+#include "linux-perf-events.h"
+#include <unistd.h>
+#ifdef __linux__
+#include <libgen.h>
+#endif //__linux__
+#endif // _MSC_VER
+
+#include <memory>
+
+#include "benchmark.h"
+
+// #define RAPIDJSON_SSE2 // bad for performance
+// #define RAPIDJSON_SSE42 // bad for performance
+#include "rapidjson/document.h"
+#include "rapidjson/reader.h"
+#include "rapidjson/stringbuffer.h"
+#include "rapidjson/writer.h"
+
+#include "sajson.h"
+
+#include <nlohmann/json.hpp>
+using json = nlohmann::json;
+
+#ifdef ALLPARSER
+
+#include "fastjson.cpp"
+#include "fastjson_dom.cpp"
+#include "gason.cpp"
+
+#include "json11.cpp"
+extern "C" {
+#include "cJSON.c"
+#include "cJSON.h"
+#include "jsmn.c"
+#include "jsmn.h"
+#include "ujdecode.h"
+#include "ultrajsondec.c"
+}
+
+#include "jsoncpp.cpp"
+#include "json/json.h"
+
+#endif
+
+using namespace rapidjson;
+
+#ifdef ALLPARSER
+// fastjson has a tricky interface
+void on_json_error(void *, const fastjson::ErrorContext &ec) {
+  // std::cerr<<"ERROR: "<<ec.mesg<<std::endl;
+}
+bool fastjson_parse(const char *input) {
+  fastjson::Token token;
+  fastjson::dom::Chunk chunk;
+  return fastjson::dom::parse_string(input, &token, &chunk, 0, &on_json_error,
+                                     NULL);
+}
+// end of fastjson stuff
+#endif
+
+int main(int argc, char *argv[]) {
+  bool verbose = false;
+  bool just_data = false;
+  int c;
+  while ((c = getopt(argc, argv, "vt")) != -1)
+    switch (c) {
+    case 't':
+      just_data = true;
+      break;
+    case 'v':
+      verbose = true;
+      break;
+    default:
+      abort();
+    }
+  if (optind >= argc) {
+    std::cerr << "Usage: " << argv[0] << " <jsonfile>" << std::endl;
+    std::cerr << "Or " << argv[0] << " -v <jsonfile>" << std::endl;
+    std::cerr << "The '-t' flag outputs a table. " << std::endl;
+    exit(1);
+  }
+  const char *filename = argv[optind];
+  if (optind + 1 < argc) {
+    std::cerr << "warning: ignoring everything after " << argv[optind + 1]
+              << std::endl;
+  }
+  simdjson::padded_string p;
+  try {
+    simdjson::get_corpus(filename).swap(p);
+  } catch (const std::exception &e) { // caught by reference to base
+    std::cout << "Could not load the file " << filename << std::endl;
+    return EXIT_FAILURE;
+  }
+
+  if (verbose) {
+    std::cout << "Input has ";
+    if (p.size() > 1024 * 1024)
+      std::cout << p.size() / (1024 * 1024) << " MB ";
+    else if (p.size() > 1024)
+      std::cout << p.size() / 1024 << " KB ";
+    else
+      std::cout << p.size() << " B ";
+    std::cout << std::endl;
+  }
+  simdjson::ParsedJson pj;
+  bool allocok = pj.allocate_capacity(p.size(), 1024);
+
+  if (!allocok) {
+    std::cerr << "can't allocate memory" << std::endl;
+    return EXIT_FAILURE;
+  }
+  int repeat = (p.size() < 1 * 1000 * 1000 ? 1000 : 10);
+  int volume = p.size();
+  if (just_data) {
+    printf("%-42s %20s %20s %20s %20s \n", "name", "cycles_per_byte",
+           "cycles_per_byte_err", "gb_per_s", "gb_per_s_err");
+  }
+  if (!just_data)
+    BEST_TIME("simdjson (dynamic mem) ", build_parsed_json(p).is_valid(), true,
+              , repeat, volume, !just_data);
+  // (static alloc)
+  BEST_TIME("simdjson ", json_parse(p, pj), simdjson::SUCCESS, , repeat, volume,
+            !just_data);
+
+  rapidjson::Document d;
+
+  char *buffer = (char *)malloc(p.size() + 1);
+  memcpy(buffer, p.data(), p.size());
+  buffer[p.size()] = '\0';
+#ifndef ALLPARSER
+  if (!just_data)
+#endif
+    BEST_TIME("RapidJSON  ",
+              d.Parse<kParseValidateEncodingFlag>((const char *)buffer)
+                  .HasParseError(),
+              false, memcpy(buffer, p.data(), p.size()), repeat, volume,
+              !just_data);
+  BEST_TIME("RapidJSON (insitu)",
+            d.ParseInsitu<kParseValidateEncodingFlag>(buffer).HasParseError(),
+            false,
+            memcpy(buffer, p.data(), p.size()) && (buffer[p.size()] = '\0'),
+            repeat, volume, !just_data);
+#ifndef ALLPARSER
+  if (!just_data)
+#endif
+    BEST_TIME("sajson (dynamic mem)",
+              sajson::parse(sajson::dynamic_allocation(),
+                            sajson::mutable_string_view(p.size(), buffer))
+                  .is_valid(),
+              true, memcpy(buffer, p.data(), p.size()), repeat, volume,
+              !just_data);
+
+  size_t ast_buffer_size = p.size();
+  size_t *ast_buffer = (size_t *)malloc(ast_buffer_size * sizeof(size_t));
+  //  (static alloc, insitu)
+  BEST_TIME(
+      "sajson",
+      sajson::parse(sajson::bounded_allocation(ast_buffer, ast_buffer_size),
+                    sajson::mutable_string_view(p.size(), buffer))
+          .is_valid(),
+      true, memcpy(buffer, p.data(), p.size()), repeat, volume, !just_data);
+  size_t expected = json::parse(p.data(), p.data() + p.size()).size();
+  BEST_TIME("nlohmann-json", json::parse(buffer, buffer + p.size()).size(),
+            expected, memcpy(buffer, p.data(), p.size()), repeat, volume,
+            !just_data);
+
+#ifdef ALLPARSER
+  std::string json11err;
+  BEST_TIME("dropbox (json11)     ",
+            ((json11::Json::parse(buffer, json11err).is_null()) ||
+             (!json11err.empty())),
+            false, memcpy(buffer, p.data(), p.size()), repeat, volume,
+            !just_data);
+
+  BEST_TIME("fastjson             ", fastjson_parse(buffer), true,
+            memcpy(buffer, p.data(), p.size()), repeat, volume, !just_data);
+  JsonValue value;
+  JsonAllocator allocator;
+  char *endptr;
+  BEST_TIME("gason             ", jsonParse(buffer, &endptr, &value, allocator),
+            JSON_OK, memcpy(buffer, p.data(), p.size()), repeat, volume,
+            !just_data);
+  void *state;
+  BEST_TIME("ultrajson         ",
+            (UJDecode(buffer, p.size(), NULL, &state) == NULL), false,
+            memcpy(buffer, p.data(), p.size()), repeat, volume, !just_data);
+
+  {
+    std::unique_ptr<jsmntok_t[]> tokens =
+        std::make_unique<jsmntok_t[]>(p.size());
+    jsmn_parser parser;
+    jsmn_init(&parser);
+    memcpy(buffer, p.data(), p.size());
+    buffer[p.size()] = '\0';
+    BEST_TIME(
+        "jsmn           ",
+        (jsmn_parse(&parser, buffer, p.size(), tokens.get(), p.size()) > 0),
+        true, jsmn_init(&parser), repeat, volume, !just_data);
+  }
+  memcpy(buffer, p.data(), p.size());
+  buffer[p.size()] = '\0';
+  cJSON *tree = cJSON_Parse(buffer);
+  BEST_TIME("cJSON           ", ((tree = cJSON_Parse(buffer)) != NULL), true,
+            cJSON_Delete(tree), repeat, volume, !just_data);
+  cJSON_Delete(tree);
+
+  Json::CharReaderBuilder b;
+  Json::CharReader *json_cpp_reader = b.newCharReader();
+  Json::Value root;
+  Json::String errs;
+  BEST_TIME("jsoncpp           ",
+            json_cpp_reader->parse(buffer, buffer + volume, &root, &errs), true,
+            , repeat, volume, !just_data);
+  delete json_cpp_reader;
+#endif
+  if (!just_data)
+    BEST_TIME("memcpy            ",
+              (memcpy(buffer, p.data(), p.size()) == buffer), true, , repeat,
+              volume, !just_data);
+#ifdef __linux__
+  if (!just_data) {
+    printf("\n \n <doing additional analysis with performance counters (Linux "
+           "only)>\n");
+    std::vector<int> evts;
+    evts.push_back(PERF_COUNT_HW_CPU_CYCLES);
+    evts.push_back(PERF_COUNT_HW_INSTRUCTIONS);
+    evts.push_back(PERF_COUNT_HW_BRANCH_MISSES);
+    evts.push_back(PERF_COUNT_HW_CACHE_REFERENCES);
+    evts.push_back(PERF_COUNT_HW_CACHE_MISSES);
+    LinuxEvents<PERF_TYPE_HARDWARE> unified(evts);
+    std::vector<unsigned long long> results;
+    std::vector<unsigned long long> stats;
+    results.resize(evts.size());
+    stats.resize(evts.size());
+    std::fill(stats.begin(), stats.end(), 0); // unnecessary
+    for (int i = 0; i < repeat; i++) {
+      unified.start();
+      if (json_parse(p, pj) != simdjson::SUCCESS)
+        printf("bug\n");
+      unified.end(results);
+      std::transform(stats.begin(), stats.end(), results.begin(), stats.begin(),
+                     std::plus<unsigned long long>());
+    }
+    printf("simdjson : cycles %10.0f instructions %10.0f branchmisses %10.0f "
+           "cacheref %10.0f cachemisses %10.0f  bytespercachemiss %10.0f "
+           "inspercycle %10.1f insperbyte %10.1f\n",
+           stats[0] * 1.0 / repeat, stats[1] * 1.0 / repeat,
+           stats[2] * 1.0 / repeat, stats[3] * 1.0 / repeat,
+           stats[4] * 1.0 / repeat, volume * repeat * 1.0 / stats[2],
+           stats[1] * 1.0 / stats[0], stats[1] * 1.0 / (volume * repeat));
+
+    std::fill(stats.begin(), stats.end(), 0);
+    for (int i = 0; i < repeat; i++) {
+      memcpy(buffer, p.data(), p.size());
+      buffer[p.size()] = '\0';
+      unified.start();
+      if (d.ParseInsitu<kParseValidateEncodingFlag>(buffer).HasParseError() !=
+          false)
+        printf("bug\n");
+      unified.end(results);
+      std::transform(stats.begin(), stats.end(), results.begin(), stats.begin(),
+                     std::plus<unsigned long long>());
+    }
+    printf("RapidJSON: cycles %10.0f instructions %10.0f branchmisses %10.0f "
+           "cacheref %10.0f cachemisses %10.0f  bytespercachemiss %10.0f "
+           "inspercycle %10.1f insperbyte %10.1f\n",
+           stats[0] * 1.0 / repeat, stats[1] * 1.0 / repeat,
+           stats[2] * 1.0 / repeat, stats[3] * 1.0 / repeat,
+           stats[4] * 1.0 / repeat, volume * repeat * 1.0 / stats[2],
+           stats[1] * 1.0 / stats[0], stats[1] * 1.0 / (volume * repeat));
+
+    std::fill(stats.begin(), stats.end(), 0); // unnecessary
+    for (int i = 0; i < repeat; i++) {
+      memcpy(buffer, p.data(), p.size());
+      unified.start();
+      if (sajson::parse(sajson::bounded_allocation(ast_buffer, ast_buffer_size),
+                        sajson::mutable_string_view(p.size(), buffer))
+              .is_valid() != true)
+        printf("bug\n");
+      unified.end(results);
+      std::transform(stats.begin(), stats.end(), results.begin(), stats.begin(),
+                     std::plus<unsigned long long>());
+    }
+    printf("sajson   : cycles %10.0f instructions %10.0f branchmisses %10.0f "
+           "cacheref %10.0f cachemisses %10.0f  bytespercachemiss %10.0f "
+           "inspercycle %10.1f insperbyte %10.1f\n",
+           stats[0] * 1.0 / repeat, stats[1] * 1.0 / repeat,
+           stats[2] * 1.0 / repeat, stats[3] * 1.0 / repeat,
+           stats[4] * 1.0 / repeat, volume * repeat * 1.0 / stats[2],
+           stats[1] * 1.0 / stats[0], stats[1] * 1.0 / (volume * repeat));
+  }
+#endif //  __linux__
+
+  free(ast_buffer);
+  free(buffer);
+}

data/vendor/simdjson/benchmark/statisticalmodel.cpp

@@ -0,0 +1,208 @@
+#include <iostream>
+#ifndef _MSC_VER
+#include <unistd.h>
+#endif
+#include "simdjson/jsonioutil.h"
+#include "simdjson/jsonparser.h"
+#ifdef __linux__
+#include "linux-perf-events.h"
+#endif
+
+size_t count_nonasciibytes(const uint8_t *input, size_t length) {
+  size_t count = 0;
+  for (size_t i = 0; i < length; i++) {
+    count += input[i] >> 7;
+  }
+  return count;
+}
+
+size_t count_backslash(const uint8_t *input, size_t length) {
+  size_t count = 0;
+  for (size_t i = 0; i < length; i++) {
+    count += (input[i] == '\\') ? 1 : 0;
+  }
+  return count;
+}
+
+struct stat_s {
+  size_t integer_count;
+  size_t float_count;
+  size_t string_count;
+  size_t backslash_count;
+  size_t non_ascii_byte_count;
+  size_t object_count;
+  size_t array_count;
+  size_t null_count;
+  size_t true_count;
+  size_t false_count;
+  size_t byte_count;
+  size_t structural_indexes_count;
+  bool valid;
+};
+
+using stat_t = struct stat_s;
+
+stat_t simdjson_compute_stats(const simdjson::padded_string &p) {
+  stat_t answer;
+  simdjson::ParsedJson pj = simdjson::build_parsed_json(p);
+  answer.valid = pj.is_valid();
+  if (!answer.valid) {
+    return answer;
+  }
+  answer.backslash_count =
+      count_backslash(reinterpret_cast<const uint8_t *>(p.data()), p.size());
+  answer.non_ascii_byte_count = count_nonasciibytes(
+      reinterpret_cast<const uint8_t *>(p.data()), p.size());
+  answer.byte_count = p.size();
+  answer.integer_count = 0;
+  answer.float_count = 0;
+  answer.object_count = 0;
+  answer.array_count = 0;
+  answer.null_count = 0;
+  answer.true_count = 0;
+  answer.false_count = 0;
+  answer.string_count = 0;
+  answer.structural_indexes_count = pj.n_structural_indexes;
+  size_t tape_idx = 0;
+  uint64_t tape_val = pj.tape[tape_idx++];
+  uint8_t type = (tape_val >> 56);
+  size_t how_many = 0;
+  assert(type == 'r');
+  how_many = tape_val & JSON_VALUE_MASK;
+  for (; tape_idx < how_many; tape_idx++) {
+    tape_val = pj.tape[tape_idx];
+    // uint64_t payload = tape_val & JSON_VALUE_MASK;
+    type = (tape_val >> 56);
+    switch (type) {
+    case 'l': // we have a long int
+      answer.integer_count++;
+      tape_idx++; // skipping the integer
+      break;
+    case 'd': // we have a double
+      answer.float_count++;
+      tape_idx++; // skipping the double
+      break;
+    case 'n': // we have a null
+      answer.null_count++;
+      break;
+    case 't': // we have a true
+      answer.true_count++;
+      break;
+    case 'f': // we have a false
+      answer.false_count++;
+      break;
+    case '{': // we have an object
+      answer.object_count++;
+      break;
+    case '}': // we end an object
+      break;
+    case '[': // we start an array
+      answer.array_count++;
+      break;
+    case ']': // we end an array
+      break;
+    case '"': // we have a string
+      answer.string_count++;
+      break;
+    default:
+      break; // ignore
+    }
+  }
+  return answer;
+}
+
+int main(int argc, char *argv[]) {
+#ifndef _MSC_VER
+  int c;
+  while ((c = getopt(argc, argv, "")) != -1) {
+    switch (c) {
+
+    default:
+      abort();
+    }
+  }
+#else
+  int optind = 1;
+#endif
+  if (optind >= argc) {
+    std::cerr << "Reads json, prints stats. " << std::endl;
+    std::cerr << "Usage: " << argv[0] << " <jsonfile>" << std::endl;
+
+    exit(1);
+  }
+  const char *filename = argv[optind];
+  if (optind + 1 < argc) {
+    std::cerr << "warning: ignoring everything after " << argv[optind + 1]
+              << std::endl;
+  }
+  simdjson::padded_string p;
+  try {
+    simdjson::get_corpus(filename).swap(p);
+  } catch (const std::exception &) { // caught by reference to base
+    std::cerr << "Could not load the file " << filename << std::endl;
+    return EXIT_FAILURE;
+  }
+  stat_t s = simdjson_compute_stats(p);
+  if (!s.valid) {
+    std::cerr << "not a valid JSON" << std::endl;
+    return EXIT_FAILURE;
+  }
+
+  printf("# integer_count float_count string_count backslash_count "
+         "non_ascii_byte_count object_count array_count null_count true_count "
+         "false_count byte_count structural_indexes_count ");
+#ifdef __linux__
+  printf("  stage1_cycle_count stage1_instruction_count  stage2_cycle_count "
+         " stage2_instruction_count  stage3_cycle_count "
+         "stage3_instruction_count  ");
+#else
+  printf("(you are not under linux, so perf counters are disaabled)");
+#endif
+  printf("\n");
+  printf("%zu %zu %zu %zu %zu %zu %zu %zu %zu %zu %zu %zu ", s.integer_count,
+         s.float_count, s.string_count, s.backslash_count,
+         s.non_ascii_byte_count, s.object_count, s.array_count, s.null_count,
+         s.true_count, s.false_count, s.byte_count, s.structural_indexes_count);
+#ifdef __linux__
+  simdjson::ParsedJson pj;
+  bool allocok = pj.allocate_capacity(p.size());
+  if (!allocok) {
+    std::cerr << "failed to allocate memory" << std::endl;
+    return EXIT_FAILURE;
+  }
+  const uint32_t iterations = p.size() < 1 * 1000 * 1000 ? 1000 : 50;
+  std::vector<int> evts;
+  evts.push_back(PERF_COUNT_HW_CPU_CYCLES);
+  evts.push_back(PERF_COUNT_HW_INSTRUCTIONS);
+  LinuxEvents<PERF_TYPE_HARDWARE> unified(evts);
+  unsigned long cy1 = 0, cy2 = 0;
+  unsigned long cl1 = 0, cl2 = 0;
+  std::vector<unsigned long long> results;
+  results.resize(evts.size());
+  for (uint32_t i = 0; i < iterations; i++) {
+    unified.start();
+    // The default template is simdjson::Architecture::NATIVE.
+    bool isok = (simdjson::find_structural_bits<>(p.data(), p.size(), pj) ==
+                 simdjson::SUCCESS);
+    unified.end(results);
+
+    cy1 += results[0];
+    cl1 += results[1];
+
+    unified.start();
+    isok =
+        isok && (simdjson::SUCCESS == unified_machine(p.data(), p.size(), pj));
+    unified.end(results);
+
+    cy2 += results[0];
+    cl2 += results[1];
+    if (!isok) {
+      std::cerr << "failure?" << std::endl;
+    }
+  }
+  printf("%f %f %f %f ", cy1 * 1.0 / iterations, cl1 * 1.0 / iterations,
+         cy2 * 1.0 / iterations, cl2 * 1.0 / iterations);
+#endif // __linux__
+  printf("\n");
+  return EXIT_SUCCESS;
+}