simdjson 0.1.0

data/vendor/simdjson/tools/jsonstats.cpp

@@ -0,0 +1,143 @@
+#include <iostream>
+
+#include "simdjson/jsonioutil.h"
+#include "simdjson/jsonparser.h"
+
+size_t count_nonasciibytes(const uint8_t *input, size_t length) {
+  size_t count = 0;
+  for (size_t i = 0; i < length; i++) {
+    count += input[i] >> 7;
+  }
+  return count;
+}
+
+size_t count_backslash(const uint8_t *input, size_t length) {
+  size_t count = 0;
+  for (size_t i = 0; i < length; i++) {
+    count += (input[i] == '\\') ? 1 : 0;
+  }
+  return count;
+}
+
+struct stat_s {
+  size_t integer_count;
+  size_t float_count;
+  size_t string_count;
+  size_t backslash_count;
+  size_t non_ascii_byte_count;
+  size_t object_count;
+  size_t array_count;
+  size_t null_count;
+  size_t true_count;
+  size_t false_count;
+  size_t byte_count;
+  size_t structural_indexes_count;
+  bool valid;
+};
+
+using stat_t = struct stat_s;
+
+stat_t simdjson_compute_stats(const simdjson::padded_string &p) {
+  stat_t answer;
+  simdjson::ParsedJson pj = simdjson::build_parsed_json(p);
+  answer.valid = pj.is_valid();
+  if (!answer.valid) {
+    std::cerr << pj.get_error_message() << std::endl;
+    return answer;
+  }
+  answer.backslash_count =
+      count_backslash(reinterpret_cast<const uint8_t *>(p.data()), p.size());
+  answer.non_ascii_byte_count = count_nonasciibytes(
+      reinterpret_cast<const uint8_t *>(p.data()), p.size());
+  answer.byte_count = p.size();
+  answer.integer_count = 0;
+  answer.float_count = 0;
+  answer.object_count = 0;
+  answer.array_count = 0;
+  answer.null_count = 0;
+  answer.true_count = 0;
+  answer.false_count = 0;
+  answer.string_count = 0;
+  answer.structural_indexes_count = pj.n_structural_indexes;
+  size_t tape_idx = 0;
+  uint64_t tape_val = pj.tape[tape_idx++];
+  uint8_t type = (tape_val >> 56);
+  size_t how_many = 0;
+  assert(type == 'r');
+  how_many = tape_val & JSON_VALUE_MASK;
+  for (; tape_idx < how_many; tape_idx++) {
+    tape_val = pj.tape[tape_idx];
+    // uint64_t payload = tape_val & JSON_VALUE_MASK;
+    type = (tape_val >> 56);
+    switch (type) {
+    case 'l': // we have a long int
+      answer.integer_count++;
+      tape_idx++; // skipping the integer
+      break;
+    case 'd': // we have a double
+      answer.float_count++;
+      tape_idx++; // skipping the double
+      break;
+    case 'n': // we have a null
+      answer.null_count++;
+      break;
+    case 't': // we have a true
+      answer.true_count++;
+      break;
+    case 'f': // we have a false
+      answer.false_count++;
+      break;
+    case '{': // we have an object
+      answer.object_count++;
+      break;
+    case '}': // we end an object
+      break;
+    case '[': // we start an array
+      answer.array_count++;
+      break;
+    case ']': // we end an array
+      break;
+    case '"': // we have a string
+      answer.string_count++;
+      break;
+    default:
+      break; // ignore
+    }
+  }
+  return answer;
+}
+
+int main(int argc, char *argv[]) {
+  int myoptind = 1;
+  if (myoptind >= argc) {
+    std::cerr << "Reads json, prints stats. " << std::endl;
+    std::cerr << "Usage: " << argv[0] << " <jsonfile>" << std::endl;
+    exit(1);
+  }
+  const char *filename = argv[myoptind];
+  if (myoptind + 1 < argc) {
+    std::cerr << "warning: ignoring everything after " << argv[myoptind + 1]
+              << std::endl;
+  }
+  simdjson::padded_string p;
+  try {
+    simdjson::get_corpus(filename).swap(p);
+  } catch (const std::exception &) { // caught by reference to base
+    std::cerr << "Could not load the file " << filename << std::endl;
+    return EXIT_FAILURE;
+  }
+  stat_t s = simdjson_compute_stats(p);
+  if (!s.valid) {
+    std::cerr << "not a valid JSON" << std::endl;
+    return EXIT_FAILURE;
+  }
+
+  printf("# integer_count float_count string_count backslash_count "
+         "non_ascii_byte_count object_count array_count null_count true_count "
+         "false_count byte_count structural_indexes_count\n");
+  printf("%zu %zu %zu %zu %zu %zu %zu %zu %zu %zu %zu %zu\n", s.integer_count,
+         s.float_count, s.string_count, s.backslash_count,
+         s.non_ascii_byte_count, s.object_count, s.array_count, s.null_count,
+         s.true_count, s.false_count, s.byte_count, s.structural_indexes_count);
+  return EXIT_SUCCESS;
+}

data/vendor/simdjson/tools/minify.cpp

@@ -0,0 +1,21 @@
+#include <iostream>
+
+#include "simdjson/jsonioutil.h"
+#include "simdjson/jsonminifier.h"
+
+int main(int argc, char *argv[]) {
+  if (argc != 2) {
+    std::cerr << "Usage: " << argv[0] << " <jsonfile>\n";
+    exit(1);
+  }
+  simdjson::padded_string p;
+  std::string filename = argv[argc - 1];
+  try {
+    simdjson::get_corpus(filename).swap(p);
+  } catch (const std::exception &) {
+    std::cout << "Could not load the file " << filename << std::endl;
+    return EXIT_FAILURE;
+  }
+  simdjson::json_minify(p, p.data());
+  printf("%s", p.data());
+}

data/vendor/simdjson/tools/release.py

@@ -0,0 +1,125 @@
+#!/usr/bin/env python3
+########################################################################
+# Generates a new release.
+########################################################################
+import sys
+import re
+import subprocess
+import io
+import os
+
+
+def extractnumbers(s):
+    return tuple(map(int,re.findall("(\d+)\.(\d+)\.(\d+)",str(s))[0]))
+
+def toversionstring(major, minor, rev):
+    return str(major)+"."+str(minor)+"."+str(rev)
+
+def topaddedversionstring(major, minor, rev):
+    return str(major)+str(minor).zfill(3)+str(rev).zfill(3)
+
+pipe = subprocess.Popen(["git", "rev-parse", "--abbrev-ref", "HEAD"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+branchresult = pipe.communicate()[0].decode().strip()
+
+if(branchresult != "master"):
+    print("release on master, you are on '"+branchresult+"'")
+    sys.exit(-1)
+
+
+ret = subprocess.call(["git", "remote", "update"])
+
+if(ret != 0):
+    sys.exit(ret)
+
+
+
+pipe = subprocess.Popen(["git", "log", "HEAD..", "--oneline"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+uptodateresult = pipe.communicate()[0].decode().strip()
+
+if(len(uptodateresult) != 0):
+    print(uptodateresult)
+    sys.exit(-1)
+
+pipe = subprocess.Popen(["git", "rev-parse", "--show-toplevel"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+maindir = pipe.communicate()[0].decode().strip()
+
+
+print("repository: "+maindir)
+
+pipe = subprocess.Popen(["git", "describe", "--abbrev=0", "--tags"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+versionresult = pipe.communicate()[0].decode().strip()
+
+print("last version: "+versionresult )
+try:
+  currentv = extractnumbers(versionresult)
+except:
+  currentv = [0,0,0]
+if(len(sys.argv) != 2):
+    nextv = (currentv[0],currentv[1], currentv[2]+1)
+    print ("please specify version number, e.g. "+toversionstring(*nextv))
+    sys.exit(-1)
+try:
+    newversion = extractnumbers(sys.argv[1])
+except:
+    print("can't parse version number "+sys.argv[1])
+    sys.exit(-1)
+
+print("checking that new version is valid")
+
+if(newversion[0] !=  currentv[0]):
+    assert newversion[0] ==  currentv[0] + 1
+    assert newversion[1] == 0
+    assert newversion[2] == 0
+elif (newversion[1] !=  currentv[1]):
+    assert newversion[1] ==  currentv[1] + 1
+    assert newversion[2] == 0
+else :
+    assert newversion[2] ==  currentv[2] + 1
+
+
+
+versionfilerel = os.sep + "include" + os.sep + "simdjson" + os.sep + "simdjson_version.h"
+versionfile = maindir + versionfilerel
+
+with open(versionfile, 'w') as file:
+    file.write("// "+versionfilerel+" automatically generated by release.py,\n")
+    file.write("// do not change by hand\n")
+    file.write("#ifndef SIMDJSON_INCLUDE_SIMDJSON_VERSION\n")
+    file.write("#define SIMDJSON_INCLUDE_SIMDJSON_VERSION\n")
+    file.write("#define SIMDJSON_VERSION "+toversionstring(*newversion)+"\n")
+    file.write("namespace simdjson {\n")
+    file.write("enum {\n")
+    file.write("  SIMDJSON_VERSION_MAJOR = "+str(newversion[0])+",\n")
+    file.write("  SIMDJSON_VERSION_MINOR = "+str(newversion[1])+",\n")
+    file.write("  SIMDJSON_VERSION_REVISION = "+str(newversion[2])+"\n")
+    file.write("};\n")
+    file.write("}\n")
+    file.write("#endif // SIMDJSON_INCLUDE_SIMDJSON_VERSION\n")
+
+
+print(versionfile + " modified")
+
+import fileinput
+import re
+
+newmajorversionstring = str(newversion[0])
+mewminorversionstring = str(newversion[1])
+newrevversionstring = str(newversion[2])
+newversionstring = str(newversion[0]) + "." + str(newversion[1]) + "." + str(newversion[2])
+cmakefile = maindir + os.sep + "CMakeLists.txt"
+for line in fileinput.input(cmakefile, inplace=1, backup='.bak'):
+    line = re.sub('SIMDJSON_LIB_VERSION "\d+\.\d+\.\d+','SIMDJSON_LIB_VERSION "'+newversionstring, line.rstrip())
+    line = re.sub('SIMDJSON_LIB_SOVERSION "\d+','SIMDJSON_LIB_SOVERSION "'+newmajorversionstring, line)
+    line = re.sub('set\(PROJECT_VERSION_MAJOR \d+','set(PROJECT_VERSION_MAJOR '+newmajorversionstring, line)
+    line = re.sub('set\(PROJECT_VERSION_MINOR \d+','set(PROJECT_VERSION_MINOR '+mewminorversionstring, line)
+    line = re.sub('set\(PROJECT_VERSION_PATCH \d+','set(PROJECT_VERSION_PATCH '+newrevversionstring, line)
+    print(line)
+
+
+print("modified "+cmakefile+", a backup was made")
+
+scriptlocation = os.path.dirname(os.path.abspath(__file__))
+
+
+print("Please run the tests before issuing a release, do make test && make amalgamate \n")
+print("to issue release, enter \n git commit -a && git push  &&  git tag -a v"+toversionstring(*newversion)+" -m \"version "+toversionstring(*newversion)+"\" &&  git push --tags \n")