tree-sitter 0.0.1 → 0.1.0

data/ext/tree-sitter/tree-sitter/script/benchmark

@@ -0,0 +1,76 @@
+#!/usr/bin/env bash
+
+set -e
+
+function usage {
+  cat <<-EOF
+USAGE
+
+  $0  [-Ld] [-l language-name] [-f example-file-name]
+
+OPTIONS
+
+  -h  print this message
+
+  -l  run only the benchmarks for the given language
+
+  -f  run only the benchmarks that parse the file with the given name
+
+  -d  run tests in a debugger (either lldb or gdb)
+
+  -L  run benchmarks with parse logging turned on
+
+  -b  run make under the scan-build static analyzer
+
+EOF
+}
+
+if [ "$(uname -s)" == "Darwin" ]; then
+  export LINK="clang++ -fsanitize=address"
+fi
+
+mode=normal
+export BUILDTYPE=Release
+cmd=out/$BUILDTYPE/benchmarks
+run_scan_build=
+
+while getopts "bdhf:l:SL" option; do
+  case ${option} in
+    h)
+      usage
+      exit
+      ;;
+    d)
+      mode=debug
+      ;;
+    f)
+      export TREE_SITTER_BENCHMARK_FILE_NAME=${OPTARG}
+      ;;
+    l)
+      export TREE_SITTER_BENCHMARK_LANGUAGE=${OPTARG}
+      ;;
+    L)
+      export TREE_SITTER_BENCHMARK_LOG=1
+      ;;
+    b)
+      run_scan_build=true
+      ;;
+  esac
+done
+
+if [[ -n "$run_scan_build" ]]; then
+  . script/util/scan-build.sh
+  scan_build make -j2 benchmarks
+else
+  make -j2 benchmarks
+fi
+
+case $mode in
+  debug)
+    lldb $cmd
+    ;;
+
+  normal)
+    exec $cmd
+    ;;
+esac

data/ext/tree-sitter/tree-sitter/script/build-fuzzers

@@ -0,0 +1,66 @@
+#!/bin/bash
+set -e
+
+if [[ "$(uname -s)" != Linux ]]; then
+  echo "Fuzzing is only supported on Linux"
+  exit 1
+fi
+
+if [[ -z "$LIB_FUZZER_PATH" ]]; then
+  echo "LIB_FUZZER_PATH not set"
+  exit 1
+fi
+
+CC=${CC:-clang}
+CXX=${CXX:-clang++}
+LINK=${LINK:-clang++}
+
+default_fuzz_flags="-fsanitize=address,undefined -fsanitize-coverage=trace-pc-guard"
+
+CFLAGS=${CFLAGS:-"$default_fuzz_flags"}
+CXXFLAGS=${CXXFLAGS:-"$default_fuzz_flags"}
+
+CC=$CC CXX=$CXX LINK=$LINK CFLAGS=$CFLAGS CXXFLAGS=$CXXFLAGS ./script/configure
+
+export BUILDTYPE=Fuzz
+make runtime
+
+
+if [ -z "$@" ]; then
+  languages=$(ls test/fixtures/grammars)
+else
+  languages="$@"
+fi
+
+for lang in ${languages[@]}; do
+  echo "Building $lang fuzzer..."
+  lang_dir="test/fixtures/grammars/$lang"
+
+  # The following assumes each language is implemented as src/parser.c plus an
+  # optional scanner in src/scanner.{c,cc}
+  objects=()
+
+  lang_scanner="${lang_dir}/src/scanner"
+  if [ -e "${lang_scanner}.cc" ]; then
+    $CXX $CXXFLAGS -g -O1 "-I${lang_dir}/src" -c "${lang_scanner}.cc" -o "${lang_scanner}.o"
+    objects+=("${lang_scanner}.o")
+  elif [ -e "${lang_scanner}.c" ]; then
+    $CC $CFLAGS -std=c99 -g -O1 "-I${lang_dir}/src" -c "${lang_scanner}.c" -o "${lang_scanner}.o"
+    objects+=("${lang_scanner}.o")
+  fi
+
+
+  # Compiling with -O0 speeds up the build dramatically
+  $CC $CFLAGS -g -O0 "-I${lang_dir}/src" "${lang_dir}/src/parser.c" -c -o "${lang_dir}/src/parser.o"
+  objects+=("${lang_dir}/src/parser.o")
+
+  modes=(true halt false recover)
+  for i in 0 2; do
+    $CXX $CXXFLAGS -std=c++11 -Iinclude -D TS_HALT_ON_ERROR="${modes[i]}" -D TS_LANG="tree_sitter_$lang" \
+      "test/fuzz/fuzzer.cc" "${objects[@]}" \
+      out/Fuzz/obj.target/libruntime.a "$LIB_FUZZER_PATH" \
+      -o "out/${lang}_fuzzer_${modes[i+1]}"
+  done
+
+  python test/fuzz/gen-dict.py "${lang_dir}/src/grammar.json" > "out/$lang.dict"
+done

data/ext/tree-sitter/tree-sitter/script/ci

@@ -4,4 +4,5 @@ set -e
 
 script/fetch-fixtures
 script/check-mallocs
-script/test
+script/test -b
+script/benchmark -b

data/ext/tree-sitter/tree-sitter/script/configure.cmd

@@ -0,0 +1,3 @@
+git submodule update --init --recursive
+call .\externals\gyp\gyp.bat project.gyp --depth .
+call .\externals\gyp\gyp.bat tests.gyp --depth .

data/ext/tree-sitter/tree-sitter/script/fetch-fixtures

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-GRAMMARS_DIR=$(dirname $0)/../spec/fixtures/grammars
+GRAMMARS_DIR=$(dirname $0)/../test/fixtures/grammars
 
 fetch_grammar() {
   local grammar=$1
@@ -11,18 +11,22 @@ fetch_grammar() {
   echo "Updating ${grammar} grammar..."
 
   if [ ! -d $grammar_dir ]; then
-    git clone $grammar_url $grammar_dir
+    git clone $grammar_url $grammar_dir --depth=1
   fi
 
   (
     cd $grammar_dir;
-    git fetch origin
-    git reset --hard $ref;
+    git fetch origin $ref --depth=1
+    git reset --hard origin/$ref;
   )
 }
 
-fetch_grammar 'javascript' '76cd7dd5eb793db21640c725e58301bde83781f7'
-fetch_grammar 'json'       'origin/master'
-fetch_grammar 'c'          'origin/master'
-fetch_grammar 'cpp'        'origin/master'
-fetch_grammar 'python'     'origin/master'
+fetch_grammar javascript master
+fetch_grammar json       master
+fetch_grammar c          master
+fetch_grammar cpp        master
+fetch_grammar python     master
+fetch_grammar go         master
+fetch_grammar ruby       master
+fetch_grammar typescript master
+fetch_grammar bash       master

data/ext/tree-sitter/tree-sitter/script/fetch-fixtures.cmd

@@ -0,0 +1,26 @@
+@echo off
+
+call:fetch_grammar javascript master
+call:fetch_grammar json       master
+call:fetch_grammar c          master
+call:fetch_grammar cpp        master
+call:fetch_grammar python     master
+call:fetch_grammar go         master
+call:fetch_grammar ruby       master
+call:fetch_grammar typescript master
+call:fetch_grammar bash       master
+EXIT /B 0
+
+:fetch_grammar
+SETLOCAL
+SET grammar_dir=test\fixtures\grammars\%~1
+SET grammar_url=https://github.com/tree-sitter/tree-sitter-%~1
+SET grammar_branch=%~2
+@IF NOT EXIST %grammar_dir% (
+  git clone %grammar_url% %grammar_dir% --depth=1
+)
+pushd %grammar_dir%
+git fetch origin %2 --depth=1
+git reset --hard origin/%grammar_branch%
+popd
+EXIT /B 0

data/ext/tree-sitter/tree-sitter/script/reproduce

@@ -0,0 +1 @@
+ext/tree-sitter/tree-sitter/script/run-fuzzer

data/ext/tree-sitter/tree-sitter/script/run-fuzzer

@@ -0,0 +1,57 @@
+#!/bin/bash
+
+set -eux
+
+root=$(dirname "$0")/..
+export ASAN_OPTIONS="quarantine_size_mb=10:detect_leaks=1:symbolize=1"
+export UBSAN="print_stacktrace=1:halt_on_error=1:symbolize=1"
+
+declare -A mode_config=( ["halt"]="-timeout=1 -rss_limit_mb=256" ["recover"]="-timeout=10 -rss_limit_mb=256" )
+
+run_fuzzer() {
+  if [ "$#" -lt 2 ]; then
+    echo "usage: $0 <language> <halt|recover> <libFuzzer args...>"
+    exit 1
+  fi
+
+  lang="$1"
+  shift
+  mode="$1"
+  shift
+  # Treat remainder of arguments as libFuzzer arguments
+
+  # Fuzzing logs and testcases are always written to `pwd`, so `cd` there first
+  results="${root}/out/fuzz-results/${lang}_${mode}"
+  mkdir -p "${results}"
+  cd "${results}"
+
+  # Create a corpus directory, so new discoveries are stored on disk. These will
+  # then be loaded on subsequent fuzzing runs
+  mkdir -p corpus
+
+  "../../${lang}_fuzzer_${mode}" "-dict=../../${lang}.dict" "-artifact_prefix=${lang}_${mode}_" -max_len=2048 ${mode_config[$mode]} "./corpus" "$@"
+}
+
+reproduce() {
+  if [ "$#" -lt 3 ]; then
+    echo "usage: $0 <language> (halt|recover) <testcase> <libFuzzer args...>"
+    exit 1
+  fi
+
+  lang="$1"
+  shift
+  mode="$1"
+  shift
+  testcase="$1"
+  shift
+  # Treat remainder of arguments as libFuzzer arguments
+
+  "${root}/out/${lang}_fuzzer_${mode}" ${mode_config[$mode]} -runs=1 "${testcase}" "$@"
+}
+
+script=$(basename "$0")
+if [ "$script" == "run-fuzzer" ]; then
+  run_fuzzer "$@"
+elif [ "$script" == "reproduce" ]; then
+  reproduce "$@"
+fi

data/ext/tree-sitter/tree-sitter/script/test

@@ -12,6 +12,8 @@ OPTIONS
 
   -h  print this message
 
+  -b  run make under scan-build static analyzer
+
   -d  run tests in a debugger (either lldb or gdb)
 
   -g  run tests with valgrind's memcheck tool
@@ -24,7 +26,7 @@ OPTIONS
 
   -s  set the seed used to control random behavior
 
-  -z  pipe tests' stderr to \`dot(1)\` to render an SVG log
+  -D  pipe tests' stderr to \`dot(1)\` to render an SVG log
 
 EOF
 }
@@ -37,8 +39,13 @@ args=()
 target=tests
 export BUILDTYPE=Test
 cmd="out/${BUILDTYPE}/${target}"
+run_scan_build=
+
+if [ "$(uname -s)" == "Darwin" ]; then
+  export LINK="clang++ -fsanitize=address"
+fi
 
-while getopts "df:s:gGhpvS" option; do
+while getopts "bdf:s:gGhpvD" option; do
   case ${option} in
     h)
       usage
@@ -66,9 +73,13 @@ while getopts "df:s:gGhpvS" option; do
     s)
       export TREE_SITTER_SEED=${OPTARG}
       ;;
-    S)
+    D)
+      export TREE_SITTER_ENABLE_DEBUG_GRAPHS=1
       mode=SVG
       ;;
+    b)
+      run_scan_build=true
+      ;;
   esac
 done
 
@@ -78,7 +89,12 @@ else
   args+=("--reporter=singleline")
 fi
 
-make $target
+if [[ -n "$run_scan_build" ]]; then
+  . script/util/scan-build.sh
+  scan_build make -j2 $target
+else
+  make -j2 $target
+fi
 args=${args:-""}
 
 if [[ -n $profile ]]; then
@@ -92,7 +108,7 @@ case ${mode} in
       --dsymutil=yes                             \
       --leak-check=${leak_check}                 \
       $cmd "${args[@]}" 2>&1 |                   \
-      grep --color -E '\w+_specs?.cc:\d+|$'
+      grep --color -E '\w+_tests?.cc:\d+|$'
     ;;
 
   debug)
@@ -107,9 +123,32 @@ case ${mode} in
     ;;
 
   SVG)
-    echo "<!DOCTYPE html><style>svg { width: 100%; margin-bottom: 20px; }</style>" > index.html
-    $cmd "${args[@]}" 2> >(grep -v 'Assertion failed' | dot -Tsvg >> index.html)
-    echo "Wrote index.html"
+    html_file=log.html
+    dot_file=$html_file.dot
+
+    function write_log_file {
+      echo "<!DOCTYPE html><style>svg { width: 100%; margin-bottom: 20px; }</style>" > $html_file
+      line_count=$(grep -n '^$' $dot_file | tail -1 | cut -f1 -d:)
+      if [[ -n $line_count ]]; then
+        head -n $line_count $dot_file | dot -Tsvg >> $html_file
+      else
+        cat $dot_file | grep -v 'Assertion' | dot -Tsvg >> $html_file
+      fi
+      rm $dot_file
+      echo "Wrote $html_file - $line_count"
+    }
+
+    function handle_sigint {
+      trap '' SIGINT
+      echo
+      write_log_file
+      exit 0
+    }
+    trap handle_sigint SIGINT
+
+    $cmd "${args[@]}" 2> $dot_file || export status=$?
+    write_log_file
+    exit $status
     ;;
 
   normal)

data/ext/tree-sitter/tree-sitter/script/test.cmd

@@ -0,0 +1,2 @@
+msbuild /p:Configuration=Test tests.vcxproj
+.\test\tests.exe --reporter=singleline --no-color

data/ext/tree-sitter/tree-sitter/script/trim-whitespace

@@ -1,3 +1,3 @@
 #!/usr/bin/env bash
 
-find src spec include -type f | xargs perl -pi -e 's/ +$//'
+find src test include -type f | xargs perl -pi -e 's/ +$//'

data/ext/tree-sitter/tree-sitter/script/util/scan-build.sh

@@ -0,0 +1,24 @@
+function scan_build {
+  extra_args=()
+
+  # AFAICT, in the trusty travis container the scan-build tool is from the 3.4
+  # installation. Therefore, by default it will use clang-3.4 when analysing code
+  # which doesn't support the '-std=c++14' (it is available via '-std=c++1y').
+  # Use the system-wide installed clang instead which is 3.5 and does support
+  # '-std=c++14'.
+  extra_args+=("--use-analyzer=$(which clang)")
+
+  # scan-build will try to guess which CXX should be used to compile the actual
+  # code, which is usually g++ but we need g++5 in the CI. Explicitly pass
+  # $CC/$CXX to scan-build if they are set in the environment.
+
+  if [[ ! -z "$CC" ]]; then
+    extra_args+=("--use-cc=$CC")
+  fi
+
+  if [[ ! -z "$CXX" ]]; then
+    extra_args+=("--use-c++=$CXX")
+  fi
+
+  scan-build "${extra_args[@]}" --status-bugs -disable-checker deadcode.DeadStores "$@"
+}

data/ext/tree-sitter/tree-sitter/src/compiler/build_tables/lex_item.cc

@@ -2,12 +2,7 @@
 #include <unordered_set>
 #include "compiler/build_tables/lex_item_transitions.h"
 #include "compiler/build_tables/rule_can_be_blank.h"
-#include "compiler/rules/choice.h"
-#include "compiler/rules/metadata.h"
-#include "compiler/rules/seq.h"
-#include "compiler/rules/symbol.h"
-#include "compiler/rules/repeat.h"
-#include "compiler/rules/visitor.h"
+#include "compiler/rule.h"
 #include "compiler/util/hash_combine.h"
 
 namespace tree_sitter {
@@ -18,55 +13,65 @@ using std::string;
 using std::unordered_set;
 using rules::CharacterSet;
 using rules::Symbol;
+using rules::Metadata;
 
-LexItem::LexItem(const rules::Symbol &lhs, const rule_ptr rule)
+LexItem::LexItem(const rules::Symbol &lhs, const rules::Rule &rule)
     : lhs(lhs), rule(rule) {}
 
 bool LexItem::operator==(const LexItem &other) const {
-  return (other.lhs == lhs) && other.rule->operator==(*rule);
+  return lhs == other.lhs && rule == other.rule;
 }
 
-LexItem::CompletionStatus LexItem::completion_status() const {
-  class GetCompletionStatus : public rules::RuleFn<CompletionStatus> {
-   protected:
-    CompletionStatus apply_to(const rules::Choice *rule) {
-      for (const auto &element : rule->elements) {
-        CompletionStatus status = apply(element);
-        if (status.is_done)
-          return status;
+using CompletionStatus = LexItem::CompletionStatus;
+
+static CompletionStatus get_completion_status(const rules::Rule &rule) {
+  return rule.match(
+    [](rules::Choice choice) {
+      for (const auto &element : choice.elements) {
+        auto status = get_completion_status(element);
+        if (status.is_done) return status;
       }
-      return { false, PrecedenceRange(), false };
-    }
+      return CompletionStatus{false, PrecedenceRange()};
+    },
 
-    CompletionStatus apply_to(const rules::Metadata *rule) {
-      CompletionStatus result = apply(rule->rule);
-      if (result.is_done) {
-        if (result.precedence.empty && rule->params.has_precedence)
-          result.precedence.add(rule->params.precedence);
-        if (rule->params.is_string)
-          result.is_string = true;
+    [](rules::Metadata metadata) {
+      CompletionStatus result = get_completion_status(*metadata.rule);
+      if (result.is_done && result.precedence.empty && metadata.params.has_precedence) {
+        result.precedence.add(metadata.params.precedence);
       }
       return result;
-    }
+    },
+
+    [](rules::Repeat repeat) {
+      return get_completion_status(*repeat.rule);
+    },
+
+    [](rules::Seq sequence) {
+      CompletionStatus left_status = get_completion_status(*sequence.left);
+      if (left_status.is_done) {
+        return get_completion_status(*sequence.right);
+      } else {
+        return CompletionStatus{false, PrecedenceRange()};
+      }
+    },
 
-    CompletionStatus apply_to(const rules::Repeat *rule) {
-      return apply(rule->content);
-    }
+    [](rules::Blank blank) {
+      return CompletionStatus{true, PrecedenceRange()};
+    },
 
-    CompletionStatus apply_to(const rules::Blank *rule) {
-      return { true, PrecedenceRange(), false };
-    }
+    [](rules::CharacterSet) {
+      return CompletionStatus{false, PrecedenceRange()};
+    },
 
-    CompletionStatus apply_to(const rules::Seq *rule) {
-      CompletionStatus left_status = apply(rule->left);
-      if (left_status.is_done)
-        return apply(rule->right);
-      else
-        return { false, PrecedenceRange(), false };
+    [](auto) {
+      return CompletionStatus{false, PrecedenceRange()};
     }
-  };
+  );
+}
+
 
-  return GetCompletionStatus().apply(rule);
+LexItem::CompletionStatus LexItem::completion_status() const {
+  return get_completion_status(rule);
 }
 
 LexItemSet::LexItemSet() {}
@@ -78,10 +83,24 @@ bool LexItemSet::operator==(const LexItemSet &other) const {
   return entries == other.entries;
 }
 
+bool LexItem::is_in_separators() const {
+  if (!rule.is<Metadata>()) return false;
+  auto &metadata = rule.get_unchecked<Metadata>();
+  return !metadata.params.is_main_token;
+}
+
+bool LexItemSet::has_items_in_separators() const {
+  for (const LexItem &item : entries) {
+    if (item.is_in_separators()) return true;
+  }
+  return false;
+}
+
 LexItemSet::TransitionMap LexItemSet::transitions() const {
   TransitionMap result;
-  for (const LexItem &item : entries)
+  for (const LexItem &item : entries) {
     lex_item_transitions(&result, item);
+  }
   return result;
 }