tree-sitter 0.0.1 → 0.1.0

data/ext/tree-sitter/tree-sitter/test/fixtures/test_grammars/readme.md

@@ -0,0 +1,3 @@
+These small grammars demonstrate specific features or test for certain specific regressions.
+
+For some of them, compilation is expected to fail with a given error message. For others, the resulting parser is expected to produce certain trees.

data/ext/tree-sitter/tree-sitter/test/fixtures/test_grammars/readme_grammar/corpus.txt

@@ -0,0 +1,13 @@
+==================================
+the readme example
+==================================
+
+a + b * c
+
+---
+
+(expression (sum
+  (expression (variable))
+  (expression (product
+     (expression (variable))
+     (expression (variable))))))

data/{test/tree-sitter/fixtures/arithmetic → ext/tree-sitter/tree-sitter/test/fixtures/test_grammars/readme_grammar}/grammar.json

@@ -1,5 +1,5 @@
 {
-  "name": "arithmetic",
+  "name": "readme_grammar",
 
   // Things that can appear anywhere in the language, like comments
   // and whitespace, are expressed as 'extras'.
@@ -62,6 +62,6 @@
     // Tokens can be specified using ECMAScript regexps.
     "number": {"type": "PATTERN", "value": "\\d+"},
     "comment": {"type": "PATTERN", "value": "#.*"},
-    "variable": {"type": "PATTERN", "value": "[a-zA-Z]\\w*"},
+    "variable": {"type": "PATTERN", "value": "[a-zA-Z]\\w*"}
   }
-}
+}

data/ext/tree-sitter/tree-sitter/test/fixtures/test_grammars/start_rule_is_blank/corpus.txt

@@ -0,0 +1,7 @@
+========================
+the empty string
+=======================
+
+---
+
+(first_rule)

data/ext/tree-sitter/tree-sitter/test/fixtures/test_grammars/start_rule_is_blank/grammar.json

@@ -0,0 +1,6 @@
+{
+  "name": "start_rule_is_blank",
+  "rules": {
+    "first_rule": {"type": "BLANK"}
+  }
+}

data/ext/tree-sitter/tree-sitter/test/fixtures/test_grammars/start_rule_is_token/corpus.txt

@@ -0,0 +1,6 @@
+===========================
+the single token
+==========================
+the-value
+---
+(first_rule)

data/ext/tree-sitter/tree-sitter/test/fixtures/test_grammars/start_rule_is_token/grammar.json

@@ -0,0 +1,6 @@
+{
+  "name": "start_rule_is_token",
+  "rules": {
+    "first_rule": {"type": "STRING", "value": "the-value"}
+  }
+}

data/ext/tree-sitter/tree-sitter/test/fixtures/test_grammars/unused_rules/corpus.txt

@@ -0,0 +1,9 @@
+=========================
+the language
+=========================
+
+E F I J
+
+---
+
+(a (d (e) (f)) (h (i) (j)))

data/ext/tree-sitter/tree-sitter/test/fixtures/test_grammars/unused_rules/grammar.json

@@ -0,0 +1,73 @@
+{
+  "name": "unused_rules",
+
+  "extras": [
+    {"type": "PATTERN", "value": "\\s"}
+  ],
+
+  "rules": {
+    "a": {
+      "type": "SEQ",
+      "members": [
+        {"type": "SYMBOL", "name": "d"},
+        {"type": "SYMBOL", "name": "h"}
+      ]
+    },
+
+    "b": {
+      "type": "STRING",
+      "value": "B"
+    },
+
+    "c": {
+      "type": "STRING",
+      "value": "C"
+    },
+
+    "d": {
+      "type": "SEQ",
+      "members": [
+        {"type": "SYMBOL", "name": "e"},
+        {"type": "SYMBOL", "name": "f"}
+      ]
+    },
+
+    "e": {
+      "type": "STRING",
+      "value": "E"
+    },
+
+    "f": {
+      "type": "STRING",
+      "value": "F"
+    },
+
+    "g": {
+      "type": "STRING",
+      "value": "G"
+    },
+
+    "h": {
+      "type": "SEQ",
+      "members": [
+        {"type": "SYMBOL", "name": "i"},
+        {"type": "SYMBOL", "name": "j"}
+      ]
+    },
+
+    "i": {
+      "type": "STRING",
+      "value": "I"
+    },
+
+    "j": {
+      "type": "STRING",
+      "value": "J"
+    },
+
+    "k": {
+      "type": "STRING",
+      "value": "K"
+    }
+  }
+}

data/ext/tree-sitter/tree-sitter/test/fixtures/test_grammars/unused_rules/readme.md

@@ -0,0 +1 @@
+The generated parsers use the grammar's token count to distinguish between terminal and non-terminal symbols. When the grammar has unused tokens, these tokens don't appear in the parser, so they need to be omitted from the token count.

data/ext/tree-sitter/tree-sitter/test/fuzz/README.md

@@ -0,0 +1,43 @@
+# Fuzzing tree-sitter
+
+The tree-sitter fuzzing support requires 1) the `libFuzzer` runtime library and 2) a recent version of clang
+
+## libFuzzer
+
+The main fuzzing logic is implemented by `libFuzzer` which is part of the compiler-rt project but is not shipped by distros. `libFuzzer` will need to be built from source, e.g.:
+
+```
+cd ~/src
+git clone https://github.com/llvm-mirror/compiler-rt
+cd compiler-rt/lib/fuzzer
+./build.sh
+```
+
+## clang
+
+Using libFuzzer requires a reasonably new version of `clang` and will probably _not_ work with your system-installed version. The easiest way to get started is to use the version provided by the Chromium team. Instructions are available at [libFuzzer.info](http://libfuzzer.info).
+
+The fuzzers can then be built with:
+```
+export CLANG_DIR=$HOME/src/third_party/llvm-build/Release+Asserts/bin
+CC="$CLANG_DIR/clang" CXX="$CLANG_DIR/clang++" LINK="$CLANG_DIR/clang++" \
+  LIB_FUZZER_PATH=$HOME/src/compiler-rt/lib/fuzzer/libFuzzer.a \
+  ./script/build_fuzzers
+```
+
+This will generate a separate fuzzer for each grammar defined in `test/fixtures/grammars` and will be instrumented with [AddressSanitizer](https://clang.llvm.org/docs/AddressSanitizer.html) and [UndefinedBehaviorSanitizer](https://clang.llvm.org/docs/UndefinedBehaviorSanitizer.html). Individual fuzzers can be built with, for example, `./script/build_fuzzers python ruby`.
+
+The `run-fuzzer` script handles running an individual fuzzer with a sensible default set of arguments:
+```
+./script/run-fuzzer <grammar-name> (halt|recover) <extra libFuzzer arguments...>
+```
+
+which will log information to stdout. Failing testcases and a fuzz corpus will be saved to `fuzz-results/<grammar-name>`. The most important extra `libFuzzer` options are `-jobs` and `-workers` which allow parallel fuzzing. This is can done with, e.g.:
+```
+./script/run-fuzzer <grammer-name> halt -jobs=32 -workers=32
+```
+
+The testcase can be used to reproduce the crash by running:
+```
+./script/reproduce <grammar-name> (halt|recover) <path-to-testcase>
+```

data/ext/tree-sitter/tree-sitter/test/fuzz/fuzzer.cc

@@ -0,0 +1,27 @@
+#include <string.h>
+#include "tree_sitter/runtime.h"
+
+void test_log(void *payload, TSLogType type, const char *string) { }
+
+TSLogger logger = {
+  .log = test_log,
+};
+
+extern "C" const TSLanguage *TS_LANG();
+
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+  const char *str = reinterpret_cast<const char *>(data);
+
+  TSDocument *document = ts_document_new();
+  ts_document_set_language(document, TS_LANG());
+  ts_document_set_input_string_with_length(document, str, size);
+
+  TSParseOptions options = {};
+  options.halt_on_error = TS_HALT_ON_ERROR;
+  ts_document_parse_with_options(document, options);
+
+  TSNode root_node = ts_document_root_node(document);
+  ts_document_free(document);
+
+  return 0;
+}

data/ext/tree-sitter/tree-sitter/test/fuzz/gen-dict.py

@@ -0,0 +1,31 @@
+import json
+import sys
+
+def find_literals(literals, node):
+  '''Recursively find STRING literals in the grammar definition'''
+
+  if type(node) is dict:
+    if 'type' in node and node['type'] == 'STRING' and 'value' in node:
+      literals.add(node['value'])
+
+    for key, value in node.iteritems():
+      find_literals(literals, value)
+
+  elif type(node) is list:
+    for item in node:
+      find_literals(literals, item)
+
+def main():
+  '''Generate a libFuzzer / AFL dictionary from a tree-sitter grammar.json'''
+  with open(sys.argv[1]) as f:
+    grammar = json.load(f)
+
+  literals = set()
+  find_literals(literals, grammar)
+
+  for lit in sorted(literals):
+    if lit:
+      print '"%s"' % ''.join([(c if c.isalnum() else '\\x%02x' % ord(c)) for c in lit])
+
+if __name__ == '__main__':
+  main()

data/ext/tree-sitter/tree-sitter/{spec → test}/helpers/encoding_helpers.cc

@@ -4,10 +4,16 @@
 #include "utf8proc.h"
 
 static inline int string_iterate(TSInputEncoding encoding, const uint8_t *string, size_t length, int32_t *code_point) {
-  if (encoding == TSInputEncodingUTF8)
-    return utf8proc_iterate(string, length, code_point);
-  else
+  if (encoding == TSInputEncodingUTF8) {
+    int32_t character_size = utf8proc_iterate(string, length, code_point);
+    if (character_size < 0) {
+      return 1;
+    } else {
+      return character_size;
+    }
+  } else {
     return utf16_iterate(string, length, code_point);
+  }
 }
 
 size_t string_char_count(TSInputEncoding encoding, const std::string &input) {

data/ext/tree-sitter/tree-sitter/test/helpers/file_helpers.cc

@@ -0,0 +1,100 @@
+#include "helpers/file_helpers.h"
+#include <sys/stat.h>
+#include <errno.h>
+#include <fstream>
+
+using std::string;
+using std::ifstream;
+using std::istreambuf_iterator;
+using std::ofstream;
+using std::vector;
+
+bool file_exists(const string &path) {
+  struct stat file_stat;
+  return stat(path.c_str(), &file_stat) == 0;
+}
+
+int get_modified_time(const string &path) {
+  struct stat file_stat;
+  if (stat(path.c_str(), &file_stat) != 0) {
+    if (errno != ENOENT)
+      fprintf(stderr, "Error in stat() for path: %s\n", + path.c_str());
+    return 0;
+  }
+  return file_stat.st_mtime;
+}
+
+string read_file(const string &path) {
+  struct stat file_stat;
+  if (stat(path.c_str(), &file_stat) != 0 || (file_stat.st_mode & S_IFMT) != S_IFREG) return "";
+  ifstream file(path, std::ios::binary);
+  if (!file.good()) return "";
+  istreambuf_iterator<char> file_iterator(file), end_iterator;
+  string content(file_iterator, end_iterator);
+  file.close();
+  return content;
+}
+
+void write_file(const string &path, const string &content) {
+  ofstream file(path);
+  file << content;
+  file.close();
+}
+
+#ifdef _WIN32
+
+#include <windows.h>
+
+const char *path_separator = "\\";
+
+vector<string> list_directory(const string &path) {
+  vector<string> result;
+
+  WIN32_FIND_DATA search_data;
+  HANDLE handle = FindFirstFile((path + "\\*").c_str(), &search_data);
+  while (handle != INVALID_HANDLE_VALUE) {
+    string name(search_data.cFileName);
+    result.push_back(name);
+    if (FindNextFile(handle, &search_data) == FALSE) break;
+  }
+
+  return result;
+}
+
+#else
+
+#include <dirent.h>
+
+const char *path_separator = "/";
+
+vector<string> list_directory(const string &path) {
+  vector<string> result;
+
+  DIR *dir = opendir(path.c_str());
+  if (!dir) {
+    printf("\nTest error - no such directory '%s'", path.c_str());
+    return result;
+  }
+
+  struct dirent *dir_entry;
+  while ((dir_entry = readdir(dir))) {
+    string name(dir_entry->d_name);
+    if (name != "." && name != "..") {
+      result.push_back(name);
+    }
+  }
+
+  closedir(dir);
+  return result;
+}
+
+#endif
+
+string join_path(const vector<string> &parts) {
+  string result;
+  for (const string &part : parts) {
+    if (!result.empty()) result += path_separator;
+    result += part;
+  }
+  return result;
+}

data/ext/tree-sitter/tree-sitter/test/helpers/file_helpers.h

@@ -0,0 +1,15 @@
+#ifndef HELPERS_FILE_HELPERS_H_
+#define HELPERS_FILE_HELPERS_H_
+
+#include <string>
+#include <vector>
+#include <sys/stat.h>
+
+bool file_exists(const std::string &path);
+int get_modified_time(const std::string &path);
+std::string read_file(const std::string &path);
+void write_file(const std::string &path, const std::string &content);
+std::vector<std::string> list_directory(const std::string &path);
+std::string join_path(const std::vector<std::string> &parts);
+
+#endif  // HELPERS_FILE_HELPERS_H_

data/ext/tree-sitter/tree-sitter/test/helpers/load_language.cc

@@ -0,0 +1,239 @@
+#include "helpers/load_language.h"
+#include "helpers/file_helpers.h"
+#include <cassert>
+#include <sys/types.h>
+#include <map>
+#include <string>
+#include <fstream>
+#include <stdlib.h>
+#include "tree_sitter/compiler.h"
+
+using std::map;
+using std::string;
+using std::ifstream;
+using std::ofstream;
+using std::istreambuf_iterator;
+using std::vector;
+using std::to_string;
+
+map<string, const TSLanguage *> loaded_languages;
+int libcompiler_mtime = -1;
+int compile_result_count = 0;
+
+#ifdef _WIN32
+
+#include <windows.h>
+
+const char *libcompiler_path = "test\\lib\\compiler.lib";
+const char *dylib_extension = ".dll";
+
+static string get_cwd() {
+  string result(255, 0);
+  result.resize(GetCurrentDirectory(result.size(), &result[0]));
+  return result;
+}
+
+static int compile_parser(
+  string source_filename,
+  string scanner_source_filename,
+  string output_filename,
+  string header_dirname
+) {
+  CreateDirectory("out", nullptr);
+  CreateDirectory("out\\tmp", nullptr);
+
+  string command = "cl.exe";
+  command += " /nologo";
+  command += " /LD";
+  command += " /I " + header_dirname;
+  command += " /Od";
+  command += " " + source_filename;
+  command += " " + scanner_source_filename;
+  command += " /link /out:" + output_filename;
+  return system(command.c_str());
+}
+
+static void *load_function_from_library(string library_path, string function_name) {
+  HINSTANCE library = LoadLibrary(library_path.c_str());
+  if (!library) {
+    fputs(("Could not load library " + library_path).c_str(), stderr);
+    abort();
+  }
+
+  void *function = static_cast<void *>(GetProcAddress(library, function_name.c_str()));
+  if (!function) {
+    fputs(("Could not find function + " + function_name).c_str(), stderr);
+    abort();
+  }
+
+  return function;
+}
+
+#else // POSIX
+
+#ifdef __linux
+
+const char *libcompiler_path = "out/Test/obj.target/libcompiler.a";
+const char *dylib_extension = ".so";
+
+#else // macOS
+
+const char *libcompiler_path = "out/Test/libcompiler.a";
+const char *dylib_extension = ".dylib";
+
+#endif
+
+#include <unistd.h>
+#include <dlfcn.h>
+
+static string get_cwd() {
+  return string(getenv("PWD"));
+}
+
+static int compile_parser(
+  string source_filename,
+  string scanner_source_filename,
+  string output_filename,
+  string header_dirname
+) {
+  mkdir("out", 0777);
+  mkdir("out/tmp", 0777);
+
+  const char *compiler_name = getenv("CXX");
+  if (!compiler_name) compiler_name = "c++";
+
+  string command = compiler_name;
+  command += " -shared";
+  command += " -fPIC ";
+  command += " -I " + header_dirname;
+  command += " -o " + output_filename;
+  command += " -O0";
+  command += " -xc " + source_filename;
+
+  if (!scanner_source_filename.empty()) {
+    command += " -g";
+    string extension = scanner_source_filename.substr(scanner_source_filename.rfind("."));
+    if (extension == ".c") {
+      command += " -xc " + scanner_source_filename;
+    } else {
+      command += " -xc++ " + scanner_source_filename;
+    }
+  }
+
+  return system(command.c_str());
+}
+
+static void *load_function_from_library(string library_path, string function_name) {
+  void *parser_lib = dlopen(library_path.c_str(), RTLD_NOW);
+  if (!parser_lib) {
+    fputs(dlerror(), stderr);
+    abort();
+  }
+
+  void *language_function = dlsym(parser_lib, function_name.c_str());
+  if (!language_function) {
+    fputs(dlerror(), stderr);
+    abort();
+  }
+
+  return language_function;
+}
+
+#endif
+
+static const TSLanguage *load_language(const string &source_filename,
+                                       const string &lib_filename,
+                                       const string &language_name,
+                                       string external_scanner_filename = "") {
+  string language_function_name = "tree_sitter_" + language_name;
+  string header_dir = join_path({get_cwd(), "include"});
+  int source_mtime = get_modified_time(source_filename);
+  int header_mtime = get_modified_time(join_path({header_dir, "tree_sitter", "parser.h"}));
+  int lib_mtime = get_modified_time(lib_filename);
+  int external_scanner_mtime = get_modified_time(external_scanner_filename);
+
+  if (!header_mtime || lib_mtime < header_mtime || lib_mtime < source_mtime ||
+      lib_mtime < external_scanner_mtime) {
+    const char *compiler_name = getenv("CXX");
+    if (!compiler_name) compiler_name = "c++";
+
+    int status_code = compile_parser(
+      source_filename,
+      external_scanner_filename,
+      lib_filename,
+      header_dir
+    );
+
+    if (status_code != 0) abort();
+  }
+
+  void *language_function = load_function_from_library(lib_filename, language_function_name);
+
+  return reinterpret_cast<TSLanguage *(*)()>(language_function)();
+}
+
+const TSLanguage *load_test_language(const string &name,
+                                     const TSCompileResult &compile_result,
+                                     string external_scanner_path) {
+  if (compile_result.error_type != TSCompileErrorTypeNone) {
+    fputs((string("Compilation failed ") + compile_result.error_message).c_str(), stderr);
+    abort();
+  }
+
+  string source_filename = join_path({"out", "tmp", "compile-result-" + to_string(compile_result_count) + ".c"});
+  string lib_filename = source_filename + dylib_extension;
+  compile_result_count++;
+
+  ofstream source_file;
+  source_file.open(source_filename);
+  source_file << compile_result.code;
+  source_file.close();
+
+  auto language = load_language(source_filename, lib_filename, name, external_scanner_path);
+  free(compile_result.code);
+  return language;
+}
+
+const TSLanguage *load_real_language(const string &language_name) {
+  if (loaded_languages[language_name])
+    return loaded_languages[language_name];
+
+  string language_dir = join_path({"test", "fixtures", "grammars", language_name});
+  string grammar_filename = join_path({language_dir, "src", "grammar.json"});
+  string parser_filename = join_path({language_dir, "src", "parser.c"});
+  string external_scanner_filename = join_path({language_dir, "src", "scanner.cc"});
+  if (!file_exists(external_scanner_filename)) {
+    external_scanner_filename = join_path({language_dir, "src", "scanner.c"});
+    if (!file_exists(external_scanner_filename)) {
+      external_scanner_filename = "";
+    }
+  }
+
+  int grammar_mtime = get_modified_time(grammar_filename);
+  if (!grammar_mtime) return nullptr;
+
+  if (libcompiler_mtime == -1) {
+    libcompiler_mtime = get_modified_time(libcompiler_path);
+    if (!libcompiler_mtime) return nullptr;
+  }
+
+  int parser_mtime = get_modified_time(parser_filename);
+
+  if (parser_mtime <= grammar_mtime || parser_mtime <= libcompiler_mtime) {
+    printf("\n" "Regenerating the %s parser...\n", language_name.c_str());
+
+    string grammar_json = read_file(grammar_filename);
+    TSCompileResult result = ts_compile_grammar(grammar_json.c_str());
+    if (result.error_type != TSCompileErrorTypeNone) {
+      fprintf(stderr, "Failed to compile %s grammar: %s\n", language_name.c_str(), result.error_message);
+      return nullptr;
+    }
+
+    write_file(parser_filename, result.code);
+  }
+
+  string lib_filename = join_path({"out", "tmp", language_name + dylib_extension});
+  const TSLanguage *language = load_language(parser_filename, lib_filename, language_name, external_scanner_filename);
+  loaded_languages[language_name] = language;
+  return language;
+};