simdjson 0.1.0

data/vendor/simdjson/benchmark/parse.cpp

@@ -0,0 +1,393 @@
+#include <cassert>
+#include <cctype>
+#ifndef _MSC_VER
+#include <dirent.h>
+#include <unistd.h>
+#endif
+#include <cinttypes>
+
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+
+#include <algorithm>
+#include <chrono>
+#include <cstring>
+#include <fstream>
+#include <iomanip>
+#include <iostream>
+#include <map>
+#include <set>
+#include <sstream>
+#include <string>
+#include <vector>
+
+#include "linux-perf-events.h"
+#ifdef __linux__
+#include <libgen.h>
+#endif
+//#define DEBUG
+#include "simdjson/common_defs.h"
+#include "simdjson/isadetection.h"
+#include "simdjson/jsonioutil.h"
+#include "simdjson/jsonparser.h"
+#include "simdjson/parsedjson.h"
+#include "simdjson/stage1_find_marks.h"
+#include "simdjson/stage2_build_tape.h"
+namespace simdjson {
+Architecture _find_best_supported_implementation() {
+  constexpr uint32_t haswell_flags =
+      instruction_set::AVX2 | instruction_set::PCLMULQDQ |
+      instruction_set::BMI1 | instruction_set::BMI2;
+  constexpr uint32_t westmere_flags =
+      instruction_set::SSE42 | instruction_set::PCLMULQDQ;
+  uint32_t supports = detect_supported_architectures();
+  // Order from best to worst (within architecture)
+  if ((haswell_flags & supports) == haswell_flags) {
+    return Architecture::HASWELL;
+  }
+  if ((westmere_flags & supports) == westmere_flags) {
+    return Architecture::WESTMERE;
+  }
+  if (instruction_set::NEON)
+    return Architecture::ARM64;
+
+  return Architecture::NONE;
+}
+
+using unified_functype = int(const uint8_t *buf, size_t len, ParsedJson &pj);
+using stage1_functype = int(const uint8_t *buf, size_t len, ParsedJson &pj);
+
+extern unified_functype *unified_ptr;
+
+extern stage1_functype *stage1_ptr;
+
+int unified_machine_dispatch(const uint8_t *buf, size_t len, ParsedJson &pj) {
+  Architecture best_implementation = _find_best_supported_implementation();
+  // Selecting the best implementation
+  switch (best_implementation) {
+#ifdef IS_X86_64
+  case Architecture::HASWELL:
+    unified_ptr = &unified_machine<Architecture::HASWELL>;
+    break;
+  case Architecture::WESTMERE:
+    unified_ptr = &unified_machine<Architecture::WESTMERE>;
+    break;
+#endif
+#ifdef IS_ARM64
+  case Architecture::ARM64:
+    unified_ptr = &unified_machine<Architecture::ARM64>;
+    break;
+#endif
+  default:
+    std::cerr << "The processor is not supported by simdjson." << std::endl;
+    return simdjson::UNEXPECTED_ERROR;
+  }
+
+  return unified_ptr(buf, len, pj);
+}
+
+// Responsible to select the best json_parse implementation
+int find_structural_bits_dispatch(const uint8_t *buf, size_t len,
+                                  ParsedJson &pj) {
+  Architecture best_implementation = _find_best_supported_implementation();
+  // Selecting the best implementation
+  switch (best_implementation) {
+#ifdef IS_X86_64
+  case Architecture::HASWELL:
+    stage1_ptr = &find_structural_bits<Architecture::HASWELL>;
+    break;
+  case Architecture::WESTMERE:
+    stage1_ptr = &find_structural_bits<Architecture::WESTMERE>;
+    break;
+#endif
+#ifdef IS_ARM64
+  case Architecture::ARM64:
+    stage1_ptr = &find_structural_bits<Architecture::ARM64>;
+    break;
+#endif
+  default:
+    std::cerr << "The processor is not supported by simdjson." << std::endl;
+    return simdjson::UNEXPECTED_ERROR;
+  }
+
+  return stage1_ptr(buf, len, pj);
+}
+
+stage1_functype *stage1_ptr = &find_structural_bits_dispatch;
+unified_functype *unified_ptr = &unified_machine_dispatch;
+} // namespace simdjson
+
+int main(int argc, char *argv[]) {
+  bool verbose = false;
+  bool dump = false;
+  bool json_output = false;
+  bool force_one_iteration = false;
+  bool just_data = false;
+#ifndef _MSC_VER
+  int c;
+
+  while ((c = getopt(argc, argv, "1vdt")) != -1) {
+    switch (c) {
+    case 't':
+      just_data = true;
+      break;
+    case 'v':
+      verbose = true;
+      break;
+    case 'd':
+      dump = true;
+      break;
+    case 'j':
+      json_output = true;
+      break;
+    case '1':
+      force_one_iteration = true;
+      break;
+    default:
+      abort();
+    }
+  }
+#else
+  int optind = 1;
+#endif
+  if (optind >= argc) {
+    std::cerr << "Usage: " << argv[0] << " <jsonfile>" << std::endl;
+    exit(1);
+  }
+  const char *filename = argv[optind];
+  if (optind + 1 < argc) {
+    std::cerr << "warning: ignoring everything after " << argv[optind + 1]
+              << std::endl;
+  }
+  if (verbose) {
+    std::cout << "[verbose] loading " << filename << std::endl;
+  }
+  simdjson::padded_string p;
+  try {
+    simdjson::get_corpus(filename).swap(p);
+  } catch (const std::exception &) { // caught by reference to base
+    std::cout << "Could not load the file " << filename << std::endl;
+    return EXIT_FAILURE;
+  }
+  if (verbose) {
+    std::cout << "[verbose] loaded " << filename << " (" << p.size()
+              << " bytes)" << std::endl;
+  }
+#if defined(DEBUG)
+  const uint32_t iterations = 1;
+#else
+  const uint32_t iterations =
+      force_one_iteration ? 1 : (p.size() < 1 * 1000 * 1000 ? 1000 : 10);
+#endif
+  std::vector<double> res;
+  res.resize(iterations);
+  if (!just_data)
+    printf("number of iterations %u \n", iterations);
+#if !defined(__linux__)
+#define SQUASH_COUNTERS
+  if (just_data) {
+    printf("just_data (-t) flag only works under linux.\n");
+  }
+#endif
+  { // practice run
+    simdjson::ParsedJson pj;
+    bool allocok = pj.allocate_capacity(p.size());
+    if (allocok) {
+      simdjson::stage1_ptr((const uint8_t *)p.data(), p.size(), pj);
+      simdjson::unified_ptr(
+          (const uint8_t
+               *)(const uint8_t
+                      *)(const uint8_t
+                             *)(const uint8_t
+                                    *)(const uint8_t
+                                           *)(const uint8_t
+                                                  *)(const uint8_t
+                                                         *)(const uint8_t *)
+              p.data(),
+          p.size(), pj);
+    }
+  }
+#ifndef SQUASH_COUNTERS
+  std::vector<int> evts;
+  evts.push_back(PERF_COUNT_HW_CPU_CYCLES);
+  evts.push_back(PERF_COUNT_HW_INSTRUCTIONS);
+  evts.push_back(PERF_COUNT_HW_BRANCH_MISSES);
+  evts.push_back(PERF_COUNT_HW_CACHE_REFERENCES);
+  evts.push_back(PERF_COUNT_HW_CACHE_MISSES);
+  LinuxEvents<PERF_TYPE_HARDWARE> unified(evts);
+  std::vector<unsigned long long> results;
+  results.resize(evts.size());
+  unsigned long cy0 = 0, cy1 = 0, cy2 = 0;
+  unsigned long cl0 = 0, cl1 = 0, cl2 = 0;
+  unsigned long mis0 = 0, mis1 = 0, mis2 = 0;
+  unsigned long cref0 = 0, cref1 = 0, cref2 = 0;
+  unsigned long cmis0 = 0, cmis1 = 0, cmis2 = 0;
+#endif
+  bool isok = true;
+#ifndef SQUASH_COUNTERS
+  for (uint32_t i = 0; i < iterations; i++) {
+    if (verbose) {
+      std::cout << "[verbose] iteration # " << i << std::endl;
+    }
+    unified.start();
+    simdjson::ParsedJson pj;
+    bool allocok = pj.allocate_capacity(p.size());
+    if (!allocok) {
+      std::cerr << "failed to allocate memory" << std::endl;
+      return EXIT_FAILURE;
+    }
+    unified.end(results);
+    cy0 += results[0];
+    cl0 += results[1];
+    mis0 += results[2];
+    cref0 += results[3];
+    cmis0 += results[4];
+    if (verbose) {
+      std::cout << "[verbose] allocated memory for parsed JSON " << std::endl;
+    }
+    unified.start();
+    isok = (simdjson::stage1_ptr((const uint8_t *)p.data(), p.size(), pj) ==
+            simdjson::SUCCESS);
+    unified.end(results);
+    cy1 += results[0];
+    cl1 += results[1];
+    mis1 += results[2];
+    cref1 += results[3];
+    cmis1 += results[4];
+    if (!isok) {
+      std::cout << "Failed during stage 1" << std::endl;
+      break;
+    }
+    unified.start();
+    isok = isok &&
+           (simdjson::SUCCESS ==
+            simdjson::unified_ptr((const uint8_t *)p.data(), p.size(), pj));
+    unified.end(results);
+    cy2 += results[0];
+    cl2 += results[1];
+    mis2 += results[2];
+    cref2 += results[3];
+    cmis2 += results[4];
+    if (!isok) {
+      std::cout << "Failed during stage 2" << std::endl;
+      break;
+    }
+  }
+#endif
+  // we do it again, this time just measuring the elapsed time
+  for (uint32_t i = 0; i < iterations; i++) {
+    if (verbose) {
+      std::cout << "[verbose] iteration # " << i << std::endl;
+    }
+    simdjson::ParsedJson pj;
+    bool allocok = pj.allocate_capacity(p.size());
+    if (!allocok) {
+      std::cerr << "failed to allocate memory" << std::endl;
+      return EXIT_FAILURE;
+    }
+    if (verbose) {
+      std::cout << "[verbose] allocated memory for parsed JSON " << std::endl;
+    }
+
+    auto start = std::chrono::steady_clock::now();
+    isok = (simdjson::stage1_ptr((const uint8_t *)p.data(), p.size(), pj) ==
+            simdjson::SUCCESS);
+    isok = isok &&
+           (simdjson::SUCCESS ==
+            simdjson::unified_ptr((const uint8_t *)p.data(), p.size(), pj));
+    auto end = std::chrono::steady_clock::now();
+    std::chrono::duration<double> secs = end - start;
+    res[i] = secs.count();
+    if (!isok) {
+      std::cerr << pj.get_error_message() << std::endl;
+      std::cerr << "Could not parse. " << std::endl;
+      return EXIT_FAILURE;
+    }
+  }
+  simdjson::ParsedJson pj =
+      build_parsed_json(p); // do the parsing again to get the stats
+  if (!pj.is_valid()) {
+    std::cerr << pj.get_error_message() << std::endl;
+    std::cerr << "Could not parse. " << std::endl;
+    return EXIT_FAILURE;
+  }
+  double min_result = *min_element(res.begin(), res.end());
+  double speedinGBs = (p.size()) / (min_result * 1000000000.0);
+#ifndef SQUASH_COUNTERS
+  unsigned long total = cy0 + cy1 + cy2;
+  if (just_data) {
+    float cpb0 = (double)cy0 / (iterations * p.size());
+    float cpb1 = (double)cy1 / (iterations * p.size());
+    float cpb2 = (double)cy2 / (iterations * p.size());
+    float cpbtotal = (double)total / (iterations * p.size());
+    char *newfile = (char *)malloc(strlen(filename) + 1);
+    if (newfile == NULL) {
+      return EXIT_FAILURE;
+    }
+    ::strcpy(newfile, filename);
+    char *snewfile = ::basename(newfile);
+    size_t nl = strlen(snewfile);
+    for (size_t j = nl - 1; j > 0; j--) {
+      if (snewfile[j] == '.') {
+        snewfile[j] = '\0';
+        break;
+      }
+    }
+    printf("\"%s\"\t%f\t%f\t%f\t%f\t%f\n", snewfile, cpb0, cpb1, cpb2, cpbtotal,
+           speedinGBs);
+    free(newfile);
+  } else {
+    printf("number of bytes %ld number of structural chars %u ratio %.3f\n",
+           p.size(), pj.n_structural_indexes,
+           (double)pj.n_structural_indexes / p.size());
+    printf("mem alloc instructions: %10lu cycles: %10lu (%.2f %%) ins/cycles: "
+           "%.2f mis. branches: %10lu (cycles/mis.branch %.2f) cache accesses: "
+           "%10lu (failure %10lu)\n",
+           cl0 / iterations, cy0 / iterations, 100. * cy0 / total,
+           (double)cl0 / cy0, mis0 / iterations, (double)cy0 / mis0,
+           cref1 / iterations, cmis0 / iterations);
+    printf(" mem alloc runs at %.2f cycles per input byte.\n",
+           (double)cy0 / (iterations * p.size()));
+    printf("stage 1 instructions: %10lu cycles: %10lu (%.2f %%) ins/cycles: "
+           "%.2f mis. branches: %10lu (cycles/mis.branch %.2f) cache accesses: "
+           "%10lu (failure %10lu)\n",
+           cl1 / iterations, cy1 / iterations, 100. * cy1 / total,
+           (double)cl1 / cy1, mis1 / iterations, (double)cy1 / mis1,
+           cref1 / iterations, cmis1 / iterations);
+    printf(" stage 1 runs at %.2f cycles per input byte.\n",
+           (double)cy1 / (iterations * p.size()));
+
+    printf("stage 2 instructions: %10lu cycles: %10lu (%.2f %%) ins/cycles: "
+           "%.2f mis. branches: %10lu  (cycles/mis.branch %.2f)  cache "
+           "accesses: %10lu (failure %10lu)\n",
+           cl2 / iterations, cy2 / iterations, 100. * cy2 / total,
+           (double)cl2 / cy2, mis2 / iterations, (double)cy2 / mis2,
+           cref2 / iterations, cmis2 / iterations);
+    printf(" stage 2 runs at %.2f cycles per input byte and ",
+           (double)cy2 / (iterations * p.size()));
+    printf("%.2f cycles per structural character.\n",
+           (double)cy2 / (iterations * pj.n_structural_indexes));
+
+    printf(" all stages: %.2f cycles per input byte.\n",
+           (double)total / (iterations * p.size()));
+    printf("Estimated average frequency: %.3f GHz.\n",
+           (double)total / (iterations * min_result * 1000000000.0));
+  }
+#endif
+  if (!just_data) {
+    std::cout << "Min:  " << min_result << " bytes read: " << p.size()
+              << " Gigabytes/second: " << speedinGBs << std::endl;
+  }
+  if (json_output) {
+    isok = isok && pj.print_json(std::cout);
+  }
+  if (dump) {
+    isok = isok && pj.dump_raw_tape(std::cout);
+  }
+  if (!isok) {
+    fprintf(stderr, " Parsing failed. \n ");
+    return EXIT_FAILURE;
+  }
+  return EXIT_SUCCESS;
+}

data/vendor/simdjson/benchmark/parseandstatcompetition.cpp

@@ -0,0 +1,305 @@
+#include "simdjson/jsonparser.h"
+#include <unistd.h>
+
+#include "benchmark.h"
+// #define RAPIDJSON_SSE2 // bad for performance
+// #define RAPIDJSON_SSE42 // bad for performance
+#include "rapidjson/document.h"
+#include "rapidjson/reader.h"
+#include "rapidjson/stringbuffer.h"
+#include "rapidjson/writer.h"
+
+#include "sajson.h"
+
+using namespace rapidjson;
+using namespace simdjson;
+struct stat_s {
+  size_t number_count;
+  size_t object_count;
+  size_t array_count;
+  size_t null_count;
+  size_t true_count;
+  size_t false_count;
+  bool valid;
+};
+
+typedef struct stat_s stat_t;
+
+bool stat_equal(const stat_t &s1, const stat_t &s2) {
+  return (s1.valid == s2.valid) && (s1.number_count == s2.number_count) &&
+         (s1.object_count == s2.object_count) &&
+         (s1.array_count == s2.array_count) &&
+         (s1.null_count == s2.null_count) && (s1.true_count == s2.true_count) &&
+         (s1.false_count == s2.false_count);
+}
+
+void print_stat(const stat_t &s) {
+  if (!s.valid) {
+    printf("invalid\n");
+    return;
+  }
+  printf("number: %zu object: %zu array: %zu null: %zu true: %zu false: %zu\n",
+         s.number_count, s.object_count, s.array_count, s.null_count,
+         s.true_count, s.false_count);
+}
+
+__attribute__((noinline)) stat_t
+simdjson_compute_stats(const simdjson::padded_string &p) {
+  stat_t answer;
+  simdjson::ParsedJson pj = build_parsed_json(p);
+  answer.valid = pj.is_valid();
+  if (!answer.valid) {
+    return answer;
+  }
+  answer.number_count = 0;
+  answer.object_count = 0;
+  answer.array_count = 0;
+  answer.null_count = 0;
+  answer.true_count = 0;
+  answer.false_count = 0;
+  size_t tape_idx = 0;
+  uint64_t tape_val = pj.tape[tape_idx++];
+  uint8_t type = (tape_val >> 56);
+  size_t how_many = 0;
+  assert(type == 'r');
+  how_many = tape_val & JSON_VALUE_MASK;
+  for (; tape_idx < how_many; tape_idx++) {
+    tape_val = pj.tape[tape_idx];
+    // uint64_t payload = tape_val & JSON_VALUE_MASK;
+    type = (tape_val >> 56);
+    switch (type) {
+    case 'l': // we have a long int
+      answer.number_count++;
+      tape_idx++; // skipping the integer
+      break;
+    case 'd': // we have a double
+      answer.number_count++;
+      tape_idx++; // skipping the double
+      break;
+    case 'n': // we have a null
+      answer.null_count++;
+      break;
+    case 't': // we have a true
+      answer.true_count++;
+      break;
+    case 'f': // we have a false
+      answer.false_count++;
+      break;
+    case '{': // we have an object
+      answer.object_count++;
+      break;
+    case '}': // we end an object
+      break;
+    case '[': // we start an array
+      answer.array_count++;
+      break;
+    case ']': // we end an array
+      break;
+    default:
+      break; // ignore
+    }
+  }
+  return answer;
+}
+
+// see
+// https://github.com/miloyip/nativejson-benchmark/blob/master/src/tests/sajsontest.cpp
+void sajson_traverse(stat_t &stats, const sajson::value &node) {
+  using namespace sajson;
+  switch (node.get_type()) {
+  case TYPE_NULL:
+    stats.null_count++;
+    break;
+  case TYPE_FALSE:
+    stats.false_count++;
+    break;
+  case TYPE_TRUE:
+    stats.true_count++;
+    break;
+  case TYPE_ARRAY: {
+    stats.array_count++;
+    auto length = node.get_length();
+    for (size_t i = 0; i < length; ++i) {
+      sajson_traverse(stats, node.get_array_element(i));
+    }
+    break;
+  }
+  case TYPE_OBJECT: {
+    stats.object_count++;
+    auto length = node.get_length();
+    for (auto i = 0u; i < length; ++i) {
+      sajson_traverse(stats, node.get_object_value(i));
+    }
+    break;
+  }
+  case TYPE_STRING:
+    // skip
+    break;
+
+  case TYPE_DOUBLE:
+  case TYPE_INTEGER:
+    stats.number_count++; // node.get_number_value();
+    break;
+  default:
+    assert(false && "unknown node type");
+  }
+}
+
+__attribute__((noinline)) stat_t
+sasjon_compute_stats(const simdjson::padded_string &p) {
+  stat_t answer;
+  char *buffer = (char *)malloc(p.size());
+  memcpy(buffer, p.data(), p.size());
+  auto d = sajson::parse(sajson::dynamic_allocation(),
+                         sajson::mutable_string_view(p.size(), buffer));
+  answer.valid = d.is_valid();
+  if (!answer.valid) {
+    return answer;
+  }
+  answer.number_count = 0;
+  answer.object_count = 0;
+  answer.array_count = 0;
+  answer.null_count = 0;
+  answer.true_count = 0;
+  answer.false_count = 0;
+  sajson_traverse(answer, d.get_root());
+  free(buffer);
+  return answer;
+}
+
+void rapid_traverse(stat_t &stats, const rapidjson::Value &v) {
+  switch (v.GetType()) {
+  case kNullType:
+    stats.null_count++;
+    break;
+  case kFalseType:
+    stats.false_count++;
+    break;
+  case kTrueType:
+    stats.true_count++;
+    break;
+
+  case kObjectType:
+    for (Value::ConstMemberIterator m = v.MemberBegin(); m != v.MemberEnd();
+         ++m) {
+      rapid_traverse(stats, m->value);
+    }
+    stats.object_count++;
+    break;
+  case kArrayType:
+    for (Value::ConstValueIterator i = v.Begin(); i != v.End();
+         ++i) { // v.Size();
+      rapid_traverse(stats, *i);
+    }
+    stats.array_count++;
+    break;
+
+  case kStringType:
+    break;
+
+  case kNumberType:
+    stats.number_count++;
+    break;
+  }
+}
+
+__attribute__((noinline)) stat_t
+rapid_compute_stats(const simdjson::padded_string &p) {
+  stat_t answer;
+  char *buffer = (char *)malloc(p.size() + 1);
+  memcpy(buffer, p.data(), p.size());
+  buffer[p.size()] = '\0';
+  rapidjson::Document d;
+  d.ParseInsitu<kParseValidateEncodingFlag>(buffer);
+  answer.valid = !d.HasParseError();
+  if (!answer.valid) {
+    return answer;
+  }
+  answer.number_count = 0;
+  answer.object_count = 0;
+  answer.array_count = 0;
+  answer.null_count = 0;
+  answer.true_count = 0;
+  answer.false_count = 0;
+  rapid_traverse(answer, d);
+  free(buffer);
+  return answer;
+}
+
+int main(int argc, char *argv[]) {
+  bool verbose = false;
+  bool just_data = false;
+
+  int c;
+  while ((c = getopt(argc, argv, "vt")) != -1)
+    switch (c) {
+    case 't':
+      just_data = true;
+      break;
+    case 'v':
+      verbose = true;
+      break;
+    default:
+      abort();
+    }
+  if (optind >= argc) {
+    std::cerr
+        << "Using different parsers, we compute the content statistics of "
+           "JSON documents."
+        << std::endl;
+    std::cerr << "Usage: " << argv[0] << " <jsonfile>" << std::endl;
+    std::cerr << "Or " << argv[0] << " -v <jsonfile>" << std::endl;
+    exit(1);
+  }
+  const char *filename = argv[optind];
+  if (optind + 1 < argc) {
+    std::cerr << "warning: ignoring everything after " << argv[optind + 1]
+              << std::endl;
+  }
+  simdjson::padded_string p;
+  try {
+    simdjson::get_corpus(filename).swap(p);
+  } catch (const std::exception &e) { // caught by reference to base
+    std::cout << "Could not load the file " << filename << std::endl;
+    return EXIT_FAILURE;
+  }
+
+  if (verbose) {
+    std::cout << "Input has ";
+    if (p.size() > 1024 * 1024)
+      std::cout << p.size() / (1024 * 1024) << " MB ";
+    else if (p.size() > 1024)
+      std::cout << p.size() / 1024 << " KB ";
+    else
+      std::cout << p.size() << " B ";
+    std::cout << std::endl;
+  }
+  stat_t s1 = simdjson_compute_stats(p);
+  if (verbose) {
+    printf("simdjson: ");
+    print_stat(s1);
+  }
+  stat_t s2 = rapid_compute_stats(p);
+  if (verbose) {
+    printf("rapid:    ");
+    print_stat(s2);
+  }
+  stat_t s3 = sasjon_compute_stats(p);
+  if (verbose) {
+    printf("sasjon:   ");
+    print_stat(s3);
+  }
+  assert(stat_equal(s1, s2));
+  assert(stat_equal(s1, s3));
+  int repeat = 50;
+  int volume = p.size();
+  if (just_data) {
+    printf("name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err \n");
+  }
+  BEST_TIME("simdjson  ", simdjson_compute_stats(p).valid, true, , repeat,
+            volume, !just_data);
+  BEST_TIME("RapidJSON  ", rapid_compute_stats(p).valid, true, , repeat, volume,
+            !just_data);
+  BEST_TIME("sasjon  ", sasjon_compute_stats(p).valid, true, , repeat, volume,
+            !just_data);
+}