simdjson 0.1.0

data/vendor/simdjson/style/run-clang-format.py

@@ -0,0 +1,326 @@
+#!/usr/bin/env python
+"""A wrapper script around clang-format, suitable for linting multiple files
+and to use for continuous integration.
+
+This is an alternative API for the clang-format command line.
+It runs over multiple files and directories in parallel.
+A diff output is produced and a sensible exit code is returned.
+
+"""
+
+from __future__ import print_function, unicode_literals
+
+import argparse
+import codecs
+import difflib
+import fnmatch
+import io
+import multiprocessing
+import os
+import signal
+import subprocess
+import sys
+import traceback
+
+from functools import partial
+
+DEFAULT_EXTENSIONS = 'c,h,C,H,cpp,hpp,cc,hh,c++,h++,cxx,hxx'
+
+
+class ExitStatus:
+    SUCCESS = 0
+    DIFF = 1
+    TROUBLE = 2
+
+
+def list_files(files, recursive=False, extensions=None, exclude=None):
+    if extensions is None:
+        extensions = []
+    if exclude is None:
+        exclude = []
+
+    out = []
+    for file in files:
+        if recursive and os.path.isdir(file):
+            for dirpath, dnames, fnames in os.walk(file):
+                fpaths = [os.path.join(dirpath, fname) for fname in fnames]
+                for pattern in exclude:
+                    # os.walk() supports trimming down the dnames list
+                    # by modifying it in-place,
+                    # to avoid unnecessary directory listings.
+                    dnames[:] = [
+                        x for x in dnames
+                        if
+                        not fnmatch.fnmatch(os.path.join(dirpath, x), pattern)
+                    ]
+                    fpaths = [
+                        x for x in fpaths if not fnmatch.fnmatch(x, pattern)
+                    ]
+                for f in fpaths:
+                    ext = os.path.splitext(f)[1][1:]
+                    if ext in extensions:
+                        out.append(f)
+        else:
+            out.append(file)
+    return out
+
+
+def make_diff(file, original, reformatted):
+    return list(
+        difflib.unified_diff(
+            original,
+            reformatted,
+            fromfile='{}\t(original)'.format(file),
+            tofile='{}\t(reformatted)'.format(file),
+            n=3))
+
+
+class DiffError(Exception):
+    def __init__(self, message, errs=None):
+        super(DiffError, self).__init__(message)
+        self.errs = errs or []
+
+
+class UnexpectedError(Exception):
+    def __init__(self, message, exc=None):
+        super(UnexpectedError, self).__init__(message)
+        self.formatted_traceback = traceback.format_exc()
+        self.exc = exc
+
+
+def run_clang_format_diff_wrapper(args, file):
+    try:
+        ret = run_clang_format_diff(args, file)
+        return ret
+    except DiffError:
+        raise
+    except Exception as e:
+        raise UnexpectedError('{}: {}: {}'.format(file, e.__class__.__name__,
+                                                  e), e)
+
+
+def run_clang_format_diff(args, file):
+    try:
+        with io.open(file, 'r', encoding='utf-8') as f:
+            original = f.readlines()
+    except IOError as exc:
+        raise DiffError(str(exc))
+    invocation = [args.clang_format_executable, file]
+
+    # Use of utf-8 to decode the process output.
+    #
+    # Hopefully, this is the correct thing to do.
+    #
+    # It's done due to the following assumptions (which may be incorrect):
+    # - clang-format will returns the bytes read from the files as-is,
+    #   without conversion, and it is already assumed that the files use utf-8.
+    # - if the diagnostics were internationalized, they would use utf-8:
+    #   > Adding Translations to Clang
+    #   >
+    #   > Not possible yet!
+    #   > Diagnostic strings should be written in UTF-8,
+    #   > the client can translate to the relevant code page if needed.
+    #   > Each translation completely replaces the format string
+    #   > for the diagnostic.
+    #   > -- http://clang.llvm.org/docs/InternalsManual.html#internals-diag-translation
+    #
+    # It's not pretty, due to Python 2 & 3 compatibility.
+    encoding_py3 = {}
+    if sys.version_info[0] >= 3:
+        encoding_py3['encoding'] = 'utf-8'
+
+    try:
+        proc = subprocess.Popen(
+            invocation,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            universal_newlines=True,
+            **encoding_py3)
+    except OSError as exc:
+        raise DiffError(str(exc))
+    proc_stdout = proc.stdout
+    proc_stderr = proc.stderr
+    if sys.version_info[0] < 3:
+        # make the pipes compatible with Python 3,
+        # reading lines should output unicode
+        encoding = 'utf-8'
+        proc_stdout = codecs.getreader(encoding)(proc_stdout)
+        proc_stderr = codecs.getreader(encoding)(proc_stderr)
+    # hopefully the stderr pipe won't get full and block the process
+    outs = list(proc_stdout.readlines())
+    errs = list(proc_stderr.readlines())
+    proc.wait()
+    if proc.returncode:
+        raise DiffError("clang-format exited with status {}: '{}'".format(
+            proc.returncode, file), errs)
+    return make_diff(file, original, outs), errs
+
+
+def bold_red(s):
+    return '\x1b[1m\x1b[31m' + s + '\x1b[0m'
+
+
+def colorize(diff_lines):
+    def bold(s):
+        return '\x1b[1m' + s + '\x1b[0m'
+
+    def cyan(s):
+        return '\x1b[36m' + s + '\x1b[0m'
+
+    def green(s):
+        return '\x1b[32m' + s + '\x1b[0m'
+
+    def red(s):
+        return '\x1b[31m' + s + '\x1b[0m'
+
+    for line in diff_lines:
+        if line[:4] in ['--- ', '+++ ']:
+            yield bold(line)
+        elif line.startswith('@@ '):
+            yield cyan(line)
+        elif line.startswith('+'):
+            yield green(line)
+        elif line.startswith('-'):
+            yield red(line)
+        else:
+            yield line
+
+
+def print_diff(diff_lines, use_color):
+    if use_color:
+        diff_lines = colorize(diff_lines)
+    if sys.version_info[0] < 3:
+        sys.stdout.writelines((l.encode('utf-8') for l in diff_lines))
+    else:
+        sys.stdout.writelines(diff_lines)
+
+
+def print_trouble(prog, message, use_colors):
+    error_text = 'error:'
+    if use_colors:
+        error_text = bold_red(error_text)
+    print("{}: {} {}".format(prog, error_text, message), file=sys.stderr)
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        '--clang-format-executable',
+        metavar='EXECUTABLE',
+        help='path to the clang-format executable',
+        default='clang-format')
+    parser.add_argument(
+        '--extensions',
+        help='comma separated list of file extensions (default: {})'.format(
+            DEFAULT_EXTENSIONS),
+        default=DEFAULT_EXTENSIONS)
+    parser.add_argument(
+        '-r',
+        '--recursive',
+        action='store_true',
+        help='run recursively over directories')
+    parser.add_argument('files', metavar='file', nargs='+')
+    parser.add_argument(
+        '-q',
+        '--quiet',
+        action='store_true')
+    parser.add_argument(
+        '-j',
+        metavar='N',
+        type=int,
+        default=0,
+        help='run N clang-format jobs in parallel'
+        ' (default number of cpus + 1)')
+    parser.add_argument(
+        '--color',
+        default='auto',
+        choices=['auto', 'always', 'never'],
+        help='show colored diff (default: auto)')
+    parser.add_argument(
+        '-e',
+        '--exclude',
+        metavar='PATTERN',
+        action='append',
+        default=[],
+        help='exclude paths matching the given glob-like pattern(s)'
+        ' from recursive search')
+
+    args = parser.parse_args()
+
+    # use default signal handling, like diff return SIGINT value on ^C
+    # https://bugs.python.org/issue14229#msg156446
+    signal.signal(signal.SIGINT, signal.SIG_DFL)
+    try:
+        signal.SIGPIPE
+    except AttributeError:
+        # compatibility, SIGPIPE does not exist on Windows
+        pass
+    else:
+        signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+
+    colored_stdout = False
+    colored_stderr = False
+    if args.color == 'always':
+        colored_stdout = True
+        colored_stderr = True
+    elif args.color == 'auto':
+        colored_stdout = sys.stdout.isatty()
+        colored_stderr = sys.stderr.isatty()
+
+    retcode = ExitStatus.SUCCESS
+    files = list_files(
+        args.files,
+        recursive=args.recursive,
+        exclude=args.exclude,
+        extensions=args.extensions.split(','))
+
+    if not files:
+        return
+
+    njobs = args.j
+    if njobs == 0:
+        njobs = multiprocessing.cpu_count() + 1
+    njobs = min(len(files), njobs)
+
+    if njobs == 1:
+        # execute directly instead of in a pool,
+        # less overhead, simpler stacktraces
+        it = (run_clang_format_diff_wrapper(args, file) for file in files)
+        pool = None
+    else:
+        pool = multiprocessing.Pool(njobs)
+        it = pool.imap_unordered(
+            partial(run_clang_format_diff_wrapper, args), files)
+    while True:
+        try:
+            outs, errs = next(it)
+        except StopIteration:
+            break
+        except DiffError as e:
+            print_trouble(parser.prog, str(e), use_colors=colored_stderr)
+            retcode = ExitStatus.TROUBLE
+            sys.stderr.writelines(e.errs)
+        except UnexpectedError as e:
+            print_trouble(parser.prog, str(e), use_colors=colored_stderr)
+            sys.stderr.write(e.formatted_traceback)
+            retcode = ExitStatus.TROUBLE
+            # stop at the first unexpected error,
+            # something could be very wrong,
+            # don't process all files unnecessarily
+            if pool:
+                pool.terminate()
+            break
+        else:
+            sys.stderr.writelines(errs)
+            if outs == []:
+                continue
+            if not args.quiet:
+                print_diff(outs, use_color=colored_stdout)
+            if retcode == ExitStatus.SUCCESS:
+                retcode = ExitStatus.DIFF
+    return retcode
+
+
+if __name__ == '__main__':
+    #sys.exit(main())
+    main() # we don't want a hard failure on a style check.

data/vendor/simdjson/tape.md

@@ -0,0 +1,134 @@
+
+# Tape structure in simdjson 
+
+We parse a JSON document to a tape. A tape is an array of 64-bit values. Each node encountered in the JSON document is written to the tape using one or more 64-bit tape elements; the layout of the tape is in "document order": elements are stored as they are encountered in the JSON document. 
+
+Throughout, little endian encoding is assumed. The tape is indexed starting at 0 (the first element is at index 0).
+
+## Example
+
+It is sometimes useful to start with an example. Consider the following JSON document:
+
+```json
+{
+	"Image": {
+		"Width": 800,
+		"Height": 600,
+		"Title": "View from 15th Floor",
+		"Thumbnail": {
+			"Url": "http://www.example.com/image/481989943",
+			"Height": 125,
+			"Width": 100
+		},
+		"Animated": false,
+		"IDs": [116, 943, 234, 38793]
+	}
+}
+```
+
+The following is a dump of the content of the tape, with the first number of each line representing the index of a tape element.
+
+```bash
+$ ./json2json -d jsonexamples/small/demo.json
+0 : r	// pointing to 38 (right after last node)
+1 : {	// pointing to next tape location 38 (first node after the scope)
+2 : string "Image"
+3 : {	// pointing to next tape location 37 (first node after the scope)
+4 : string "Width"
+5 : integer 800
+7 : string "Height"
+8 : integer 600
+10 : string "Title"
+11 : string "View from 15th Floor"
+12 : string "Thumbnail"
+13 : {	// pointing to next tape location 23 (first node after the scope)
+14 : string "Url"
+15 : string "http://www.example.com/image/481989943"
+16 : string "Height"
+17 : integer 125
+19 : string "Width"
+20 : integer 100
+22 : }	// pointing to previous tape location 13 (start of the scope)
+23 : string "Animated"
+24 : false
+25 : string "IDs"
+26 : [	// pointing to next tape location 36 (first node after the scope)
+27 : integer 116
+29 : integer 943
+31 : integer 234
+33 : integer 38793
+35 : ]	// pointing to previous tape location 26 (start of the scope)
+36 : }	// pointing to previous tape location 3 (start of the scope)
+37 : }	// pointing to previous tape location 1 (start of the scope)
+38 : r	// pointing to 0 (start root)
+
+```
+
+## General formal of the tape elements
+
+Most tape elements are written as `('c' << 56) + x` where `'c'` is some ASCII character determining the type of the element (out of 't', 'f', 'n', 'l', 'd', '"', '{', '}', '[', ']' ,'r') and where `x` is a 56-bit value called the payload. The payload is normally interpreted as an unsigned 56-bit integer. Note that 56-bit integers can be quite large.
+
+
+Performance consideration: We believe that accessing the tape in regular units of 64 bits is more important for performance than saving memory. 
+
+## Simple JSON values
+
+Simple JSON nodes are represented with one tape element:
+
+- null is  represented as the 64-bit value `('n' << 56)` where `'n'` is the 8-bit code point values (in ASCII) corresponding to the letter `'n'`.
+- true is  represented as the 64-bit value `('t' << 56)`.
+- false is  represented as the 64-bit value `('f' << 56)`.
+
+
+## Integer and Double values
+
+Integer values are represented as two 64-bit tape elements:
+- The 64-bit value `('l' << 56)` followed by the 64-bit integer value litterally. Integer values are assumed to be signed 64-bit values, using two's complement notation.
+
+Float values are represented as two 64-bit tape elements:
+- The 64-bit value `('d' << 56)` followed by the 64-bit double value litterally in standard IEEE 754 notation.
+
+Performance consideration: We store numbers of the main tape because we believe that locality of reference is helpful for performance. 
+
+## Root node
+
+Each JSON document will have two special 64-bit tape elements representing a root node, one at the beginning and one at the end.
+
+- The first 64-bit tape element contains the value `('r' << 56) + x` where `x` is the location on the tape of the last root element.
+- The last 64-bit tape element contains the value `('r' << 56)`.
+
+All of the parsed document is located between these two 64-bit tape elements.
+
+Hint: We can read the first tape element to determine the length of the tape.
+
+
+## Strings
+
+We prefix the string data itself by a 32-bit header to be interpreted as a 32-bit integer. It indicates the length of the string. The actual string data starts at an offset of 4 bytes.
+
+We store string values using UTF-8 encoding with null termination on a separate tape. A string value is represented on the main tape as the 64-bit tape element `('"' << 56) + x` where the payload `x` is the location on the string tape of the null-terminated string. 
+
+## Arrays 
+
+JSON arrays are represented using two 64-bit tape elements.
+
+- The first 64-bit tape element contains the value `('[' << 56) + x` where the payload `x` is 1 + the index of the second 64-bit tape element on the tape. 
+- The second 64-bit tape element contains the value `(']' << 56) + x` where the payload `x` contains the index of the first 64-bit tape element on the tape.
+
+All the content of the array is located between these two tape elements, including arrays and objects.
+
+Performance consideration: We can skip the content of an array entirely by accessing the first 64-bit tape element, reading the payload and moving to the corresponding index on the tape.
+
+## Objects 
+
+JSON objects are represented using two 64-bit tape elements.
+
+- The first 64-bit tape element contains the value `('{' << 56) + x` where the payload `x` is 1 + the index of the second 64-bit tape element on the tape.
+- The second 64-bit tape element contains the value `('}' << 56) + x` where the payload `x` contains the index of the first 64-bit tape element on the tape.
+
+In-between these two tape elements, we alternate between key (which must be strings) and values. A value could be an object or an array.
+
+All the content of the object is located between these two tape elements, including arrays and objects. 
+
+Performance consideration: We can skip the content of an object entirely by accessing the first 64-bit tape element, reading the payload and moving to the corresponding index on the tape.
+ 

data/vendor/simdjson/tests/CMakeLists.txt

@@ -0,0 +1,25 @@
+if(MSVC)
+  target_include_directories(${SIMDJSON_LIB_NAME}
+    INTERFACE $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/windows>
+  )
+endif()
+
+add_cpp_test(basictests)
+add_cpp_test(jsoncheck)
+add_cpp_test(pointercheck)
+
+## This causes problems
+# add_executable(singleheader ./singleheadertest.cpp ${PROJECT_SOURCE_DIR}/singleheader/simdjson.cpp)
+# target_compile_definitions(singleheader PRIVATE JSON_TEST_PATH="${PROJECT_SOURCE_DIR}/jsonexamples/twitter.json")
+# target_link_libraries(singleheader ${SIMDJSON_LIB_NAME})
+# add_test(singleheader singleheader)
+
+if(MSVC)
+  add_custom_command(TARGET basictests POST_BUILD        # Adds a post-build event
+    COMMAND ${CMAKE_COMMAND} -E echo "$<TARGET_FILE:simdjson>"
+    COMMAND ${CMAKE_COMMAND} -E echo "$<TARGET_FILE_DIR:basictests>"
+    COMMAND ${CMAKE_COMMAND} -E copy_if_different  # which executes "cmake -E copy_if_different..."
+        "$<TARGET_FILE:simdjson>"      # <--this is in-file
+        "$<TARGET_FILE_DIR:basictests>")                 # <--this is out-file path
+endif()
+

data/vendor/simdjson/tests/allparserscheckfile.cpp

@@ -0,0 +1,192 @@
+#include <unistd.h>
+
+#include "simdjson/jsonparser.h"
+
+// #define RAPIDJSON_SSE2 // bad
+// #define RAPIDJSON_SSE42 // bad
+#include "fastjson.cpp"
+#include "fastjson_dom.cpp"
+#include "gason.cpp"
+#include "json11.cpp"
+#include "rapidjson/document.h"
+#include "rapidjson/reader.h" // you have to check in the submodule
+#include "rapidjson/stringbuffer.h"
+#include "rapidjson/writer.h"
+#include "sajson.h"
+extern "C" {
+#include "cJSON.c"
+#include "cJSON.h"
+#include "jsmn.c"
+#include "jsmn.h"
+#include "ujdecode.h"
+#include "ultrajsondec.c"
+}
+#include "jsoncpp.cpp"
+#include "json/json.h"
+
+// fastjson has a tricky interface
+void on_json_error(void *, const fastjson::ErrorContext &ec) {
+  // std::cerr<<"ERROR: "<<ec.mesg<<std::endl;
+}
+bool fastjson_parse(const char *input) {
+  fastjson::Token token;
+  fastjson::dom::Chunk chunk;
+  return fastjson::dom::parse_string(input, &token, &chunk, 0, &on_json_error,
+                                     NULL);
+}
+// end of fastjson stuff
+
+using namespace rapidjson;
+
+int main(int argc, char *argv[]) {
+  bool verbose = false;
+  bool just_favorites = false;
+  int c;
+  while ((c = getopt(argc, argv, "vm")) != -1)
+    switch (c) {
+    case 'v':
+      verbose = true;
+      break;
+    case 'm':
+      just_favorites = true;
+      break;
+    default:
+      abort();
+    }
+  if (optind >= argc) {
+    std::cerr << "Usage: " << argv[0] << " <jsonfile>" << std::endl;
+    std::cerr << "Or " << argv[0] << " -v <jsonfile>" << std::endl;
+    exit(1);
+  }
+  const char *filename = argv[optind];
+  simdjson::padded_string p;
+  try {
+    simdjson::get_corpus(filename).swap(p);
+  } catch (const std::exception &e) { // caught by reference to base
+    std::cout << "Could not load the file " << filename << std::endl;
+    return EXIT_FAILURE;
+  }
+  if (verbose) {
+    std::cout << "Input has ";
+    if (p.size() > 1024 * 1024)
+      std::cout << p.size() / (1024 * 1024) << " MB ";
+    else if (p.size() > 1024)
+      std::cout << p.size() / 1024 << " KB ";
+    else
+      std::cout << p.size() << " B ";
+    std::cout << std::endl;
+  }
+  simdjson::ParsedJson pj;
+  size_t max_depth = 1024 * 4;
+  bool allocok = pj.allocate_capacity(p.size(), max_depth);
+  if (!allocok) {
+    std::cerr << "can't allocate memory" << std::endl;
+    return EXIT_FAILURE;
+  }
+  int oursreturn = json_parse(p, pj);
+  bool ours_correct = (oursreturn == 0); // returns 0 on success
+
+  rapidjson::Document d;
+
+  char *buffer = (char *)malloc(p.size() + 1);
+  memcpy(buffer, p.data(), p.size());
+  buffer[p.size()] = '\0';
+  bool rapid_correct_checkencoding =
+      (d.Parse<kParseValidateEncodingFlag>((const char *)buffer)
+           .HasParseError() == false);
+  bool sajson_correct =
+      sajson::parse(sajson::dynamic_allocation(),
+                    sajson::mutable_string_view(p.size(), buffer))
+          .is_valid();
+  if (just_favorites) {
+    printf("our parser                 : %s \n",
+           ours_correct ? "correct" : "invalid");
+    printf("rapid (check encoding)     : %s \n",
+           rapid_correct_checkencoding ? "correct" : "invalid");
+    printf("sajson                     : %s \n",
+           sajson_correct ? "correct" : "invalid");
+    if (oursreturn == simdjson::DEPTH_ERROR) {
+      printf("simdjson encountered a DEPTH_ERROR, it was parametrized to "
+             "reject documents with depth exceeding %zu.\n",
+             max_depth);
+    }
+    if ((ours_correct != rapid_correct_checkencoding) ||
+        (rapid_correct_checkencoding != sajson_correct) ||
+        (ours_correct != sajson_correct)) {
+      printf("WARNING: THEY DISAGREE\n\n");
+      return EXIT_FAILURE;
+    }
+    free(buffer);
+    return EXIT_SUCCESS;
+  }
+  bool rapid_correct = (d.Parse((const char *)buffer).HasParseError() == false);
+
+  std::string json11err;
+  bool dropbox_correct = ((json11::Json::parse(buffer, json11err).is_null()) ||
+                          (!json11err.empty())) == false;
+  bool fastjson_correct = fastjson_parse(buffer);
+  JsonValue value;
+  JsonAllocator allocator;
+  char *endptr;
+  bool gason_correct =
+      (jsonParse(buffer, &endptr, &value, allocator) == JSON_OK);
+  void *state;
+  bool ultrajson_correct =
+      ((UJDecode(buffer, p.size(), NULL, &state) == NULL) == false);
+
+  auto tokens = std::make_unique<jsmntok_t[]>(p.size());
+  bool jsmn_correct = false;
+  if (tokens == nullptr) {
+    printf("Failed to alloc memory for jsmn\n");
+  } else {
+    jsmn_parser parser;
+    jsmn_init(&parser);
+    memcpy(buffer, p.data(), p.size());
+    buffer[p.size()] = '\0';
+    int r = jsmn_parse(&parser, buffer, p.size(), tokens.get(), p.size());
+    tokens = nullptr;
+    jsmn_correct = (r > 0);
+  }
+
+  memcpy(buffer, p.data(), p.size());
+  buffer[p.size()] = '\0';
+  cJSON *tree = cJSON_Parse(buffer);
+  bool cjson_correct = (tree != NULL);
+  if (tree != NULL) {
+    cJSON_Delete(tree);
+  }
+
+  Json::CharReaderBuilder b;
+  Json::CharReader *json_cpp_reader = b.newCharReader();
+  Json::Value root;
+  Json::String errs;
+  bool is_json_cpp_ok =
+      json_cpp_reader->parse(buffer, buffer + p.size(), &root, &errs);
+  delete json_cpp_reader;
+
+  printf("our parser                 : %s \n",
+         ours_correct ? "correct" : "invalid");
+  printf("rapid                      : %s \n",
+         rapid_correct ? "correct" : "invalid");
+  printf("rapid (check encoding)     : %s \n",
+         rapid_correct_checkencoding ? "correct" : "invalid");
+  printf("sajson                     : %s \n",
+         sajson_correct ? "correct" : "invalid");
+  printf("dropbox                    : %s \n",
+         dropbox_correct ? "correct" : "invalid");
+  printf("fastjson                   : %s \n",
+         fastjson_correct ? "correct" : "invalid");
+  printf("gason                      : %s \n",
+         gason_correct ? "correct" : "invalid");
+  printf("ultrajson                  : %s \n",
+         ultrajson_correct ? "correct" : "invalid");
+  printf("jsmn                       : %s \n",
+         jsmn_correct ? "correct" : "invalid");
+  printf("cjson                      : %s \n",
+         cjson_correct ? "correct" : "invalid");
+  printf("jsoncpp                    : %s \n",
+         is_json_cpp_ok ? "correct" : "invalid");
+
+  free(buffer);
+  return EXIT_SUCCESS;
+}