simdjson 0.1.0

data/vendor/simdjson/amalgamation.sh

@@ -0,0 +1,158 @@
+#!/bin/bash
+########################################################################
+# Generates an "amalgamation build" for roaring. Inspired by similar
+# script used by whefs.
+########################################################################
+SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )"
+
+echo "We are about to amalgamate all simdjson files into one source file. "
+echo "See https://www.sqlite.org/amalgamation.html and https://en.wikipedia.org/wiki/Single_Compilation_Unit for rationale. "
+
+AMAL_H="simdjson.h"
+AMAL_C="simdjson.cpp"
+
+# order does not matter
+ALLCFILES="
+$SCRIPTPATH/src/simdjson.cpp
+$SCRIPTPATH/src/jsonioutil.cpp
+$SCRIPTPATH/src/jsonminifier.cpp
+$SCRIPTPATH/src/jsonparser.cpp
+$SCRIPTPATH/src/stage1_find_marks.cpp
+$SCRIPTPATH/src/stage2_build_tape.cpp
+$SCRIPTPATH/src/parsedjson.cpp
+$SCRIPTPATH/src/parsedjsoniterator.cpp
+"
+
+# order matters
+ALLCHEADERS="
+$SCRIPTPATH/include/simdjson/simdjson_version.h
+$SCRIPTPATH/include/simdjson/portability.h
+$SCRIPTPATH/include/simdjson/isadetection.h
+$SCRIPTPATH/include/simdjson/simdjson.h
+$SCRIPTPATH/include/simdjson/common_defs.h
+$SCRIPTPATH/include/simdjson/padded_string.h
+$SCRIPTPATH/include/simdjson/jsoncharutils.h
+$SCRIPTPATH/include/simdjson/jsonformatutils.h
+$SCRIPTPATH/include/simdjson/jsonioutil.h
+$SCRIPTPATH/include/simdjson/simdprune_tables.h
+$SCRIPTPATH/include/simdjson/simdutf8check_haswell.h
+$SCRIPTPATH/include/simdjson/simdutf8check_westmere.h
+$SCRIPTPATH/include/simdjson/simdutf8check_arm64.h
+$SCRIPTPATH/include/simdjson/jsonminifier.h
+$SCRIPTPATH/include/simdjson/parsedjson.h
+$SCRIPTPATH/include/simdjson/stage1_find_marks.h
+$SCRIPTPATH/include/simdjson/stage1_find_marks_flatten.h
+$SCRIPTPATH/include/simdjson/stage1_find_marks_flatten_haswell.h
+$SCRIPTPATH/include/simdjson/stage1_find_marks_macros.h
+$SCRIPTPATH/include/simdjson/stage1_find_marks_westmere.h
+$SCRIPTPATH/include/simdjson/stage1_find_marks_haswell.h
+$SCRIPTPATH/include/simdjson/stage1_find_marks_arm64.h
+$SCRIPTPATH/include/simdjson/stringparsing.h
+$SCRIPTPATH/include/simdjson/stringparsing_macros.h
+$SCRIPTPATH/include/simdjson/stringparsing_westmere.h
+$SCRIPTPATH/include/simdjson/stringparsing_haswell.h
+$SCRIPTPATH/include/simdjson/stringparsing_arm64.h
+$SCRIPTPATH/include/simdjson/numberparsing.h
+$SCRIPTPATH/include/simdjson/stage2_build_tape.h
+$SCRIPTPATH/include/simdjson/jsonparser.h
+"
+
+for i in ${ALLCHEADERS} ${ALLCFILES}; do
+    test -e $i && continue
+    echo "FATAL: source file [$i] not found."
+    exit 127
+done
+
+
+function stripinc()
+{
+    sed -e '/# *include *"/d' -e '/# *include *<simdjson\//d'
+}
+function dofile()
+{
+    RELFILE=${1#"$SCRIPTPATH/"}
+    echo "/* begin file $RELFILE */"
+    # echo "#line 8 \"$1\"" ## redefining the line/file is not nearly as useful as it sounds for debugging. It breaks IDEs.
+    stripinc < $1
+    echo "/* end file $RELFILE */"
+}
+
+timestamp=$(date)
+echo "Creating ${AMAL_H}..."
+echo "/* auto-generated on ${timestamp}. Do not edit! */" > "${AMAL_H}"
+{
+    for h in ${ALLCHEADERS}; do
+        dofile $h
+    done
+} >> "${AMAL_H}"
+
+
+echo "Creating ${AMAL_C}..."
+echo "/* auto-generated on ${timestamp}. Do not edit! */" > "${AMAL_C}"
+{
+    echo "#include \"${AMAL_H}\""
+
+    echo ""
+    echo "/* used for http://dmalloc.com/ Dmalloc - Debug Malloc Library */"
+    echo "#ifdef DMALLOC"
+    echo "#include \"dmalloc.h\""
+    echo "#endif"
+    echo ""
+
+    for h in ${ALLCFILES}; do
+        dofile $h
+    done
+} >> "${AMAL_C}"
+
+
+
+DEMOCPP="amalgamation_demo.cpp"
+echo "Creating ${DEMOCPP}..."
+echo "/* auto-generated on ${timestamp}. Do not edit! */" > "${DEMOCPP}"
+cat <<< '
+#include <iostream>
+#include "simdjson.h"
+#include "simdjson.cpp"
+int main(int argc, char *argv[]) {
+  if(argc < 2) {
+    std::cerr << "Please specify a filename " << std::endl;
+  }
+  const char * filename = argv[1];
+  simdjson::padded_string p = simdjson::get_corpus(filename);
+  simdjson::ParsedJson pj = simdjson::build_parsed_json(p); // do the parsing
+  if( ! pj.is_valid() ) {
+    std::cout << "not valid" << std::endl;
+  } else {
+    std::cout << "valid" << std::endl;
+  }
+  return EXIT_SUCCESS;
+}
+' >>  "${DEMOCPP}"
+
+echo "Done with all files generation. "
+
+echo "Files have been written to  directory: $PWD "
+ls -la ${AMAL_C} ${AMAL_H}  ${DEMOCPP}
+
+echo "Giving final instructions:"
+
+
+CPPBIN=${DEMOCPP%%.*}
+
+echo "Try :"
+echo "c++ -O3 -std=c++17 -o ${CPPBIN} ${DEMOCPP}  && ./${CPPBIN} ../jsonexamples/twitter.json "
+
+SINGLEHDR=$SCRIPTPATH/singleheader
+echo "Copying files to $SCRIPTPATH/singleheader "
+mkdir -p $SINGLEHDR
+echo "c++ -O3 -std=c++17 -o ${CPPBIN} ${DEMOCPP}  && ./${CPPBIN} ../jsonexamples/twitter.json " > $SINGLEHDR/README.md
+cp ${AMAL_C} ${AMAL_H}  ${DEMOCPP} $SINGLEHDR
+ls $SINGLEHDR
+
+cd $SINGLEHDR && c++ -O3 -std=c++17 -o ${CPPBIN} ${DEMOCPP}  && ./${CPPBIN} ../jsonexamples/twitter.json
+
+lowercase(){
+    echo "$1" | tr 'A-Z' 'a-z'
+}
+
+OS=`lowercase \`uname\``

data/vendor/simdjson/benchmark/CMakeLists.txt

@@ -0,0 +1,8 @@
+target_include_directories(${SIMDJSON_LIB_NAME}
+  INTERFACE
+  $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/benchmark>
+  $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/benchmark/linux>
+)
+
+add_cpp_benchmark(parse)
+add_cpp_benchmark(statisticalmodel)

data/vendor/simdjson/benchmark/benchmark.h

@@ -0,0 +1,223 @@
+#ifndef _BENCHMARK_H_
+#define _BENCHMARK_H_
+#include <float.h>
+#include <stdint.h>
+#include <time.h>
+#ifdef __x86_64__
+
+const char *unitname = "cycles";
+
+#define RDTSC_START(cycles)                                                    \
+  do {                                                                         \
+    uint32_t cyc_high, cyc_low;                                                \
+    __asm volatile("cpuid\n"                                                   \
+                   "rdtsc\n"                                                   \
+                   "mov %%edx, %0\n"                                           \
+                   "mov %%eax, %1"                                             \
+                   : "=r"(cyc_high), "=r"(cyc_low)                             \
+                   :                                                           \
+                   :                              /* no read only */           \
+                   "%rax", "%rbx", "%rcx", "%rdx" /* clobbers */               \
+    );                                                                         \
+    (cycles) = ((uint64_t)cyc_high << 32) | cyc_low;                           \
+  } while (0)
+
+#define RDTSC_STOP(cycles)                                                     \
+  do {                                                                         \
+    uint32_t cyc_high, cyc_low;                                                \
+    __asm volatile("rdtscp\n"                                                  \
+                   "mov %%edx, %0\n"                                           \
+                   "mov %%eax, %1\n"                                           \
+                   "cpuid"                                                     \
+                   : "=r"(cyc_high), "=r"(cyc_low)                             \
+                   : /* no read only registers */                              \
+                   : "%rax", "%rbx", "%rcx", "%rdx" /* clobbers */             \
+    );                                                                         \
+    (cycles) = ((uint64_t)cyc_high << 32) | cyc_low;                           \
+  } while (0)
+
+#else
+const char *unitname = " (clock units) ";
+
+#define RDTSC_START(cycles)                                                    \
+  do {                                                                         \
+    cycles = clock();                                                          \
+  } while (0)
+
+#define RDTSC_STOP(cycles)                                                     \
+  do {                                                                         \
+    cycles = clock();                                                          \
+  } while (0)
+#endif
+
+static __attribute__((noinline)) uint64_t rdtsc_overhead_func(uint64_t dummy) {
+  return dummy;
+}
+
+uint64_t global_rdtsc_overhead = (uint64_t)UINT64_MAX;
+
+#define RDTSC_SET_OVERHEAD(test, repeat)                                       \
+  do {                                                                         \
+    uint64_t cycles_start, cycles_final, cycles_diff;                          \
+    uint64_t min_diff = UINT64_MAX;                                            \
+    for (int i = 0; i < repeat; i++) {                                         \
+      __asm volatile("" ::: /* pretend to clobber */ "memory");                \
+      RDTSC_START(cycles_start);                                               \
+      test;                                                                    \
+      RDTSC_STOP(cycles_final);                                                \
+      cycles_diff = (cycles_final - cycles_start);                             \
+      if (cycles_diff < min_diff)                                              \
+        min_diff = cycles_diff;                                                \
+    }                                                                          \
+    global_rdtsc_overhead = min_diff;                                          \
+  } while (0)
+
+double diff(timespec start, timespec end) {
+  return ((end.tv_nsec + 1000000000 * end.tv_sec) -
+          (start.tv_nsec + 1000000000 * start.tv_sec)) /
+         1000000000.0;
+}
+
+/*
+ * Prints the best number of operations per cycle where
+ * test is the function call, answer is the expected answer generated by
+ * test, repeat is the number of times we should repeat and size is the
+ * number of operations represented by test.
+ */
+#define BEST_TIME(name, test, expected, pre, repeat, size, verbose)            \
+  do {                                                                         \
+    if (global_rdtsc_overhead == UINT64_MAX) {                                 \
+      RDTSC_SET_OVERHEAD(rdtsc_overhead_func(1), repeat);                      \
+    }                                                                          \
+    if (verbose)                                                               \
+      printf("%-40s\t: ", name);                                               \
+    else                                                                       \
+      printf("\"%-40s\"", name);                                               \
+    fflush(NULL);                                                              \
+    uint64_t cycles_start, cycles_final, cycles_diff;                          \
+    uint64_t min_diff = (uint64_t)-1;                                          \
+    double min_sumclockdiff = DBL_MAX;                                         \
+    uint64_t sum_diff = 0;                                                     \
+    double sumclockdiff = 0;                                                   \
+    struct timespec time1, time2;                                              \
+    for (int i = 0; i < repeat; i++) {                                         \
+      pre;                                                                     \
+      __asm volatile("" ::: /* pretend to clobber */ "memory");                \
+      clock_gettime(CLOCK_REALTIME, &time1);                                   \
+      RDTSC_START(cycles_start);                                               \
+      if (test != expected) {                                                  \
+        fprintf(stderr, "not expected (%d , %d )", (int)test, (int)expected);  \
+        break;                                                                 \
+      }                                                                        \
+      RDTSC_STOP(cycles_final);                                                \
+      clock_gettime(CLOCK_REALTIME, &time2);                                   \
+      double thistiming = diff(time1, time2);                                  \
+      sumclockdiff += thistiming;                                              \
+      if (thistiming < min_sumclockdiff)                                       \
+        min_sumclockdiff = thistiming;                                         \
+      cycles_diff = (cycles_final - cycles_start - global_rdtsc_overhead);     \
+      if (cycles_diff < min_diff)                                              \
+        min_diff = cycles_diff;                                                \
+      sum_diff += cycles_diff;                                                 \
+    }                                                                          \
+    uint64_t S = size;                                                         \
+    float cycle_per_op = (min_diff) / (double)S;                               \
+    float avg_cycle_per_op = (sum_diff) / ((double)S * repeat);                \
+    double avg_gb_per_s =                                                      \
+        ((double)S * repeat) / ((sumclockdiff)*1000.0 * 1000.0 * 1000.0);      \
+    double max_gb_per_s =                                                      \
+        ((double)S) / ((min_sumclockdiff)*1000.0 * 1000.0 * 1000.0);           \
+    if (verbose)                                                               \
+      printf(" %7.3f %s per input byte (best) ", cycle_per_op, unitname);      \
+    if (verbose)                                                               \
+      printf(" %7.3f %s per input byte (avg) ", avg_cycle_per_op, unitname);   \
+    if (verbose)                                                               \
+      printf(" %7.3f GB/s (error margin: %.3f GB/s)", max_gb_per_s,            \
+             -avg_gb_per_s + max_gb_per_s);                                    \
+    if (!verbose)                                                              \
+      printf(" %20.3f %20.3f %20.3f %20.3f ", cycle_per_op,                    \
+             avg_cycle_per_op - cycle_per_op, max_gb_per_s,                    \
+             -avg_gb_per_s + max_gb_per_s);                                    \
+    printf("\n");                                                              \
+    fflush(NULL);                                                              \
+  } while (0)
+
+// like BEST_TIME, but no check
+#define BEST_TIME_NOCHECK(name, test, pre, repeat, size, verbose)              \
+  do {                                                                         \
+    if (global_rdtsc_overhead == UINT64_MAX) {                                 \
+      RDTSC_SET_OVERHEAD(rdtsc_overhead_func(1), repeat);                      \
+    }                                                                          \
+    if (verbose)                                                               \
+      printf("%-40s\t: ", name);                                               \
+    fflush(NULL);                                                              \
+    uint64_t cycles_start, cycles_final, cycles_diff;                          \
+    uint64_t min_diff = (uint64_t)-1;                                          \
+    uint64_t sum_diff = 0;                                                     \
+    for (int i = 0; i < repeat; i++) {                                         \
+      pre;                                                                     \
+      __asm volatile("" ::: /* pretend to clobber */ "memory");                \
+      RDTSC_START(cycles_start);                                               \
+      test;                                                                    \
+      RDTSC_STOP(cycles_final);                                                \
+      cycles_diff = (cycles_final - cycles_start - global_rdtsc_overhead);     \
+      if (cycles_diff < min_diff)                                              \
+        min_diff = cycles_diff;                                                \
+      sum_diff += cycles_diff;                                                 \
+    }                                                                          \
+    uint64_t S = size;                                                         \
+    float cycle_per_op = (min_diff) / (double)S;                               \
+    float avg_cycle_per_op = (sum_diff) / ((double)S * repeat);                \
+    if (verbose)                                                               \
+      printf(" %.3f %s per input byte (best) ", cycle_per_op, unitname);       \
+    if (verbose)                                                               \
+      printf(" %.3f %s per input byte (avg) ", avg_cycle_per_op, unitname);    \
+    if (verbose)                                                               \
+      printf("\n");                                                            \
+    if (!verbose)                                                              \
+      printf(" %.3f ", cycle_per_op);                                          \
+    fflush(NULL);                                                              \
+  } while (0)
+
+// like BEST_TIME except that we run a function to check the result
+#define BEST_TIME_CHECK(test, check, pre, repeat, size, verbose)               \
+  do {                                                                         \
+    if (global_rdtsc_overhead == UINT64_MAX) {                                 \
+      RDTSC_SET_OVERHEAD(rdtsc_overhead_func(1), repeat);                      \
+    }                                                                          \
+    if (verbose)                                                               \
+      printf("%-60s\t:\n", #test);                                             \
+    fflush(NULL);                                                              \
+    uint64_t cycles_start, cycles_final, cycles_diff;                          \
+    uint64_t min_diff = (uint64_t)-1;                                          \
+    uint64_t sum_diff = 0;                                                     \
+    for (int i = 0; i < repeat; i++) {                                         \
+      pre;                                                                     \
+      __asm volatile("" ::: /* pretend to clobber */ "memory");                \
+      RDTSC_START(cycles_start);                                               \
+      test;                                                                    \
+      RDTSC_STOP(cycles_final);                                                \
+      if (!check) {                                                            \
+        printf("error");                                                       \
+        break;                                                                 \
+      }                                                                        \
+      cycles_diff = (cycles_final - cycles_start - global_rdtsc_overhead);     \
+      if (cycles_diff < min_diff)                                              \
+        min_diff = cycles_diff;                                                \
+      sum_diff += cycles_diff;                                                 \
+    }                                                                          \
+    uint64_t S = size;                                                         \
+    float cycle_per_op = (min_diff) / (double)S;                               \
+    float avg_cycle_per_op = (sum_diff) / ((double)S * repeat);                \
+    if (verbose)                                                               \
+      printf(" %.3f cycles per operation (best) ", cycle_per_op);              \
+    if (verbose)                                                               \
+      printf("\t%.3f cycles per operation (avg) ", avg_cycle_per_op);          \
+    if (verbose)                                                               \
+      printf("\n");                                                            \
+    if (!verbose)                                                              \
+      printf(" %.3f ", cycle_per_op);                                          \
+    fflush(NULL);                                                              \
+  } while (0)
+
+#endif