tree-sitter 0.0.1 → 0.1.0

data/ext/tree-sitter/tree-sitter/spec/compiler/rules/choice_spec.cc

@@ -1,53 +0,0 @@
-#include "spec_helper.h"
-#include "compiler/rules/choice.h"
-#include "helpers/rule_helpers.h"
-#include "helpers/equals_pointer.h"
-
-using namespace rules;
-
-START_TEST
-
-describe("Choice", []() {
-  describe("constructing choices", [&]() {
-    it("eliminates duplicate members", [&]() {
-      auto rule = Choice::build({
-        seq({ sym("one"), sym("two") }),
-        sym("three"),
-        seq({ sym("one"), sym("two") })
-      });
-
-      AssertThat(rule, EqualsPointer(choice({
-        seq({ sym("one"), sym("two") }),
-        sym("three"),
-      })));
-    });
-
-    it("eliminates duplicates within nested choices", [&]() {
-      auto rule = Choice::build({
-        seq({ sym("one"), sym("two") }),
-        Choice::build({
-          sym("three"),
-          seq({ sym("one"), sym("two") })
-        })
-      });
-
-      AssertThat(rule, EqualsPointer(choice({
-        seq({ sym("one"), sym("two") }),
-        sym("three"),
-      })));
-    });
-
-    it("doesn't construct a choice if there's only one unique member", [&]() {
-      auto rule = Choice::build({
-        sym("one"),
-        Choice::build({
-          sym("one"),
-        })
-      });
-
-      AssertThat(rule, EqualsPointer(sym("one")));
-    });
-  });
-});
-
-END_TEST

data/ext/tree-sitter/tree-sitter/spec/compiler/rules/repeat_spec.cc

@@ -1,22 +0,0 @@
-#include "spec_helper.h"
-#include "compiler/rules/repeat.h"
-#include "compiler/rules/symbol.h"
-
-using namespace rules;
-
-START_TEST
-
-describe("Repeat", []() {
-  describe("constructing repeats", [&]() {
-    it("doesn't create redundant repeats", [&]() {
-      auto sym = make_shared<Symbol>(1, Symbol::NonTerminal);
-      auto repeat = Repeat::build(sym);
-      auto outer_repeat = Repeat::build(repeat);
-
-      AssertThat(repeat, !Equals(sym));
-      AssertThat(outer_repeat, Equals(repeat));
-    });
-  });
-});
-
-END_TEST

data/ext/tree-sitter/tree-sitter/spec/fixtures/error_corpus/c_errors.txt

@@ -1,130 +0,0 @@
-========================================
-Errors inside ifdefs
-========================================
-
-#ifdef something
-int x // no semicolon
-#endif
-
-int a;
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-int b;
-
-#ifdef __cplusplus
-}
-#endif
-
-int c;
-
----
-
-(translation_unit
-  (preproc_ifdef (identifier)
-    (ERROR (identifier) (identifier))
-    (comment))
-
-  (declaration (identifier) (identifier))
-
-  (preproc_ifdef (identifier)
-    (ERROR (storage_class_specifier) (string_literal)))
-
-  (declaration (identifier) (identifier))
-
-  (preproc_ifdef (identifier)
-    (ERROR))
-
-  (declaration (identifier) (identifier)))
-
-========================================
-Errors inside blocks
-========================================
-
-int main() {
-  int x;
-  int %$#@
-}
-
----
-
-(translation_unit
-  (function_definition
-    (identifier)
-    (function_declarator (identifier))
-    (compound_statement
-      (declaration (identifier) (identifier))
-      (ERROR (identifier) (UNEXPECTED '$')))))
-
-========================================
-Errors inside expressions
-========================================
-
-int main() {
-  int x = (123 123);
-}
-
----
-
-(translation_unit
-  (function_definition
-    (identifier)
-    (function_declarator (identifier))
-    (compound_statement
-      (declaration (identifier) (init_declarator
-        (identifier)
-        (ERROR (number_literal))
-        (number_literal))))))
-
-========================================
-Errors in declarations
-========================================
-
-float x WTF;
-int y = 5;
-
----
-
-(translation_unit
-  (declaration (identifier) (ERROR (identifier)) (identifier))
-  (declaration (identifier) (init_declarator (identifier) (number_literal))))
-
-==========================================
-Errors at the beginnings of blocks
-==========================================
-
-int a() {
-  struct x = 1;
-  struct y = 2;
-}
-
-int b() {
-  w x y z = 3;
-  w x y z = 4;
-}
-
----
-
-(translation_unit
-  (function_definition
-    (identifier) (function_declarator (identifier))
-    (compound_statement
-      (ERROR (struct_specifier (identifier)))
-      (expression_statement (number_literal))
-      (ERROR (struct_specifier (identifier)))
-      (expression_statement (number_literal))))
-
-  (function_definition
-    (identifier) (function_declarator (identifier))
-    (compound_statement
-      (declaration
-        (identifier)
-        (init_declarator
-          (ERROR (identifier) (identifier))
-          (identifier) (number_literal)))
-      (declaration
-        (ERROR (identifier) (identifier))
-        (identifier)
-        (init_declarator (identifier) (number_literal))))))

data/ext/tree-sitter/tree-sitter/spec/fixtures/error_corpus/javascript_errors.txt

@@ -1,157 +0,0 @@
-===================================================
-one invalid token right after the viable prefix
-===================================================
-
-if (a b) {
-  c d;
-}
-e f;
-
----
-
-(program
-  (if_statement
-    (ERROR (identifier))
-    (identifier)
-    (statement_block
-      (ERROR (identifier))
-      (expression_statement (identifier))))
-  (ERROR (identifier))
-  (expression_statement (identifier)))
-
-=======================================================
-multiple invalid tokens right after the viable prefix
-=======================================================
-
-if (a b c) {
-  d e f g;
-}
-h i j k;
-
----
-
-(program
-  (if_statement
-    (ERROR (identifier) (identifier))
-    (identifier)
-    (statement_block
-      (ERROR (identifier) (identifier) (identifier))
-      (expression_statement (identifier))))
-  (expression_statement
-    (ERROR (identifier) (identifier) (identifier))
-    (identifier)))
-
-===================================================
-one invalid subtree right after the viable prefix
-===================================================
-
-if ({a: 'b'} {c: 'd'}) {
-  x = function(a) { b; } function(c) { d; }
-}
-
----
-
-(program
-  (if_statement
-    (object (pair (identifier) (string)))
-    (ERROR (object (pair (identifier) (string))))
-    (statement_block
-      (expression_statement (assignment
-        (identifier)
-        (ERROR (function
-          (formal_parameters (identifier))
-          (statement_block (expression_statement (identifier)))))
-        (function
-          (formal_parameters (identifier))
-          (statement_block (expression_statement (identifier)))))))))
-
-===================================================
-one invalid token at the end of the file
-===================================================
-
-// skip the equals sign
-a.b =
----
-
-(program
-  (comment)
-  (trailing_expression_statement
-    (member_access (identifier) (identifier)))
-  (ERROR))
-
-=================================================================
-An invalid token at the end of a construct with extra line breaks
-=================================================================
-
-a(
-  b,
-  c,,
-);
-
----
-
-(program
-  (expression_statement
-    (function_call (identifier) (arguments
-      (identifier)
-      (identifier)
-      (ERROR)))))
-
-===================================================
-Multi-line chained expressions in var declarations
-===================================================
-
-const one = two
-  .three(four)
-  .five()
-
----
-
-(program
-  (var_declaration (var_assignment
-    (identifier)
-    (function_call
-      (member_access
-        (function_call
-          (member_access (identifier) (identifier))
-          (arguments (identifier)))
-        (identifier))
-      (arguments)))))
-
-===================================================
-Errors after a sequence of function declarations
-===================================================
-
-/*
- * The JS grammar has an ambiguity such that these functions
- * can be parsed either as function declarations or as
- * function expressions. This ambiguity causes a lot of
- * splitting and merging in the parse stack. When iterating
- * the parse stack during an error repair, there would then
- * be a very large number (> 2^16) of paths through the parse
- * stack.
- */
-function a() {}
-function b() {}
-function c() {}
-function e() {}
-function f() {}
-function g() {}
-function h() {}
-function i() {}
-
-var x = !!!
-
----
-
-(program
-  (comment)
-  (expression_statement (function (identifier) (formal_parameters) (statement_block)))
-  (expression_statement (function (identifier) (formal_parameters) (statement_block)))
-  (expression_statement (function (identifier) (formal_parameters) (statement_block)))
-  (expression_statement (function (identifier) (formal_parameters) (statement_block)))
-  (expression_statement (function (identifier) (formal_parameters) (statement_block)))
-  (expression_statement (function (identifier) (formal_parameters) (statement_block)))
-  (expression_statement (function (identifier) (formal_parameters) (statement_block)))
-  (expression_statement (function (identifier) (formal_parameters) (statement_block)))
-  (trailing_var_declaration (identifier)) (ERROR))

data/ext/tree-sitter/tree-sitter/spec/fixtures/external_scanners/extra_external_tokens.c

@@ -1,42 +0,0 @@
-#include <tree_sitter/parser.h>
-
-enum {
-  COMMENT,
-};
-
-void *tree_sitter_extra_external_tokens_external_scanner_create() {
-  return NULL;
-}
-
-void tree_sitter_extra_external_tokens_external_scanner_reset(void *payload) {
-}
-
-bool tree_sitter_extra_external_tokens_external_scanner_serialize(void *payload, TSExternalTokenState state) {
-  return true;
-}
-
-void tree_sitter_extra_external_tokens_external_scanner_deserialize(void *payload, TSExternalTokenState state) {
-}
-
-bool tree_sitter_extra_external_tokens_external_scanner_scan(
-  void *payload, TSLexer *lexer, const bool *whitelist) {
-
-  while (lexer->lookahead == ' ') {
-    lexer->advance(lexer, true);
-  }
-
-  if (lexer->lookahead == '#') {
-    lexer->advance(lexer, false);
-    while (lexer->lookahead != '\n') {
-      lexer->advance(lexer, false);
-    }
-
-    lexer->result_symbol = COMMENT;
-    return true;
-  }
-
-  return false;
-}
-
-void tree_sitter_extra_external_tokens_external_scanner_destroy(void *payload) {
-}

data/ext/tree-sitter/tree-sitter/spec/helpers/equals_pointer.h

@@ -1,37 +0,0 @@
-#ifndef HELPERS_EQUALS_POINTER_H_
-#define HELPERS_EQUALS_POINTER_H_
-
-#include "bandit/bandit.h"
-#include <string>
-
-namespace snowhouse {
-  using namespace std;
-
-  template<typename ExpectedType>
-  struct EqualsPointerConstraint : Expression<EqualsPointerConstraint<ExpectedType>> {
-    EqualsPointerConstraint(const ExpectedType& expected) : expected(expected) {}
-
-    template<typename ActualType>
-    bool operator()(const ActualType& actual) const {
-      return *expected == *actual;
-    }
-
-    ExpectedType expected;
-  };
-
-  template<typename ExpectedType>
-  struct Stringizer<EqualsPointerConstraint<ExpectedType>> {
-    static string ToString(const EqualsPointerConstraint<ExpectedType>& constraint) {
-      ostringstream builder;
-      builder << "pointer to " << snowhouse::Stringize(constraint.expected);
-      return builder.str();
-    }
-  };
-
-  template<typename ExpectedType>
-  inline EqualsPointerConstraint<ExpectedType> EqualsPointer(const ExpectedType& expected) {
-    return EqualsPointerConstraint<ExpectedType>(expected);
-  }
-}
-
-#endif  // HELPERS_EQUALS_POINTER_H_

data/ext/tree-sitter/tree-sitter/spec/helpers/load_language.cc

@@ -1,206 +0,0 @@
-#include "spec_helper.h"
-#include "helpers/load_language.h"
-#include <unistd.h>
-#include <dlfcn.h>
-#include <sys/types.h>
-#include <sys/wait.h>
-#include <map>
-#include <string>
-#include <sys/stat.h>
-#include <fstream>
-#include <stdlib.h>
-#include "tree_sitter/compiler.h"
-
-using std::map;
-using std::string;
-using std::ifstream;
-using std::ofstream;
-using std::istreambuf_iterator;
-
-map<string, const TSLanguage *> loaded_languages;
-int libcompiler_mtime = -1;
-int compile_result_count = 0;
-
-const char *libcompiler_path =
-#if defined(__linux)
-  "out/Test/obj.target/libcompiler.a";
-#else
-  "out/Test/libcompiler.a";
-#endif
-
-static std::string run_command(const char *cmd, const char *args[]) {
-  int child_pid = fork();
-  if (child_pid < 0) {
-    return "fork failed";
-  }
-
-  if (child_pid == 0) {
-    close(0);
-    dup2(1, 0);
-    dup2(2, 1);
-    dup2(1, 2);
-    execvp(cmd, (char * const * )args);
-  }
-
-  int status;
-  do {
-    waitpid(child_pid, &status, 0);
-  } while (!WIFEXITED(status));
-
-  if (WEXITSTATUS(status) == 0) {
-    return "";
-  } else {
-    return "command failed";
-  }
-}
-
-static bool file_exists(const string &path) {
-  struct stat file_stat;
-  return stat(path.c_str(), &file_stat) == 0;
-}
-
-static int get_modified_time(const string &path) {
-  struct stat file_stat;
-  if (stat(path.c_str(), &file_stat) != 0) {
-    if (errno != ENOENT)
-      fprintf(stderr, "Error in stat() for path: %s\n", + path.c_str());
-    return 0;
-  }
-  return file_stat.st_mtime;
-}
-
-const TSLanguage *load_language(const string &source_filename,
-                                const string &lib_filename,
-                                const string &language_name,
-                                string external_scanner_filename = "") {
-  string language_function_name = "tree_sitter_" + language_name;
-  string header_dir = getenv("PWD") + string("/include");
-  int source_mtime = get_modified_time(source_filename);
-  int header_mtime = get_modified_time(header_dir + "/tree_sitter/parser.h");
-  int lib_mtime = get_modified_time(lib_filename);
-  int external_scanner_mtime = get_modified_time(external_scanner_filename);
-
-  if (!header_mtime || lib_mtime < header_mtime || lib_mtime < source_mtime ||
-      lib_mtime < external_scanner_mtime) {
-    const char *compiler_name = getenv("CXX");
-    if (!compiler_name) compiler_name = "c++";
-
-    vector<const char *> compile_args = {
-      compiler_name,
-      "-shared",
-      "-fPIC",
-      "-I", header_dir.c_str(),
-      "-o", lib_filename.c_str(),
-      "-x", "c",
-      source_filename.c_str()
-    };
-
-    if (!external_scanner_filename.empty()) {
-      compile_args.push_back("-g");
-      string extension = external_scanner_filename.substr(external_scanner_filename.rfind("."));
-      if (extension == ".c") {
-        compile_args.push_back("-xc");
-      } else {
-        compile_args.push_back("-xc++");
-      }
-      compile_args.push_back(external_scanner_filename.c_str());
-    }
-
-    compile_args.push_back(nullptr);
-
-    string compile_error = run_command(compiler_name, compile_args.data());
-    if (!compile_error.empty()) {
-      AssertThat(string(compile_error), IsEmpty());
-      return nullptr;
-    }
-  }
-
-  void *parser_lib = dlopen(lib_filename.c_str(), RTLD_NOW);
-  if (!parser_lib) {
-    std::string message(dlerror());
-    AssertThat(message, IsEmpty());
-    return nullptr;
-  }
-
-  void *language_function = dlsym(parser_lib, language_function_name.c_str());
-  if (!language_function) {
-    std::string message(dlerror());
-    AssertThat(message, IsEmpty());
-    return nullptr;
-  }
-
-  return reinterpret_cast<TSLanguage *(*)()>(language_function)();
-}
-
-const TSLanguage *load_compile_result(const string &name,
-                                      const TSCompileResult &compile_result,
-                                      string external_scanner_path) {
-  if (compile_result.error_type != TSCompileErrorTypeNone) {
-    Assert::Failure(string("Compilation failed ") + compile_result.error_message);
-    return nullptr;
-  }
-
-  mkdir("out/tmp", 0777);
-  string source_filename = "out/tmp/compile-result-" + to_string(compile_result_count) + ".c";
-  string lib_filename = source_filename + ".so";
-  compile_result_count++;
-
-  ofstream source_file;
-  source_file.open(source_filename);
-  source_file << compile_result.code;
-  source_file.close();
-
-  auto language = load_language(source_filename, lib_filename, name, external_scanner_path);
-  free(compile_result.code);
-  return language;
-}
-
-const TSLanguage *get_test_language(const string &language_name) {
-  if (loaded_languages[language_name])
-    return loaded_languages[language_name];
-
-  string language_dir = string("spec/fixtures/grammars/") + language_name;
-  string grammar_filename = language_dir + "/src/grammar.json";
-  string parser_filename = language_dir + "/src/parser.c";
-  string external_scanner_filename = language_dir + "/src/scanner.cc";
-  if (!file_exists(external_scanner_filename)) {
-    external_scanner_filename = "";
-  }
-
-  int grammar_mtime = get_modified_time(grammar_filename);
-  if (!grammar_mtime)
-    return nullptr;
-
-  if (libcompiler_mtime == -1) {
-    libcompiler_mtime = get_modified_time(libcompiler_path);
-    if (!libcompiler_mtime)
-      return nullptr;
-  }
-
-  int parser_mtime = get_modified_time(parser_filename);
-
-  if (parser_mtime < grammar_mtime || parser_mtime < libcompiler_mtime) {
-    printf("\n" "Regenerating the %s parser...\n", language_name.c_str());
-
-    ifstream grammar_file(grammar_filename);
-    istreambuf_iterator<char> grammar_file_iterator(grammar_file), end_iterator;
-    string grammar_json(grammar_file_iterator, end_iterator);
-    grammar_file.close();
-
-    TSCompileResult result = ts_compile_grammar(grammar_json.c_str());
-    if (result.error_type != TSCompileErrorTypeNone) {
-      fprintf(stderr, "Failed to compile %s grammar: %s\n", language_name.c_str(), result.error_message);
-      return nullptr;
-    }
-
-    ofstream parser_file(parser_filename);
-    parser_file << result.code;
-    parser_file.close();
-  }
-
-  mkdir("out/tmp", 0777);
-  string lib_filename = "out/tmp/" + language_name + ".so";
-  const TSLanguage *language = load_language(parser_filename, lib_filename, language_name, external_scanner_filename);
-  loaded_languages[language_name] = language;
-  return language;
-};