tree-sitter 0.0.1 → 0.1.0

data/ext/tree-sitter/tree-sitter/test/helpers/stream_methods.cc

@@ -0,0 +1,224 @@
+#include "helpers/stream_methods.h"
+#include "test_helper.h"
+#include "tree_sitter/compiler.h"
+#include "compiler/util/string_helpers.h"
+#include "compiler/parse_table.h"
+#include "compiler/syntax_grammar.h"
+#include "compiler/lexical_grammar.h"
+#include "compiler/build_tables/parse_item.h"
+#include "compiler/build_tables/lex_item.h"
+
+namespace tree_sitter {
+
+ostream &operator<<(ostream &stream, const InputGrammar &grammar) {
+  return stream << "(InputGrammar variables: " << grammar.variables << ")";
+}
+
+ostream &operator<<(ostream &stream, const CompileError &error) {
+  if (error.type) {
+    return stream << "(CompileError " << error.message << ")";
+  } else {
+    return stream << "(No CompileError)";
+  }
+}
+
+namespace rules {
+
+ostream &operator<<(ostream &stream, Associativity associativity) {
+  switch (associativity) {
+    case AssociativityLeft:
+      return stream << "AssociativityLeft";
+    case AssociativityRight:
+      return stream << "AssociativityRight";
+    default:
+      return stream << "AssociativityNone";
+  }
+}
+
+ostream &operator<<(ostream &stream, const Blank &) {
+  return stream << "(Blank)";
+}
+
+ostream &operator<<(ostream &stream, const CharacterRange &range) {
+  if (range.min == range.max) {
+    return stream << util::escape_char(range.min);
+  } else {
+    return stream << "(" + util::escape_char(range.min) << "-" << util::escape_char(range.max) << ")";
+  }
+}
+
+ostream &operator<<(ostream &stream, const CharacterSet &rule) {
+  stream << "(CharacterSet";
+  if (rule.includes_all) {
+    if (rule.excluded_chars.empty()) {
+      stream << " all";
+    } else {
+      stream << " exclude";
+      for (const auto &range : rule.excluded_ranges()) {
+        stream << " " << range;
+      }
+    }
+  } else {
+    for (const auto &range : rule.included_ranges()) {
+      stream << " " << range;
+    }
+  }
+  return stream << ")";
+}
+
+ostream &operator<<(ostream &stream, const Symbol &rule) {
+  stream << "(Symbol ";
+  switch (rule.type) {
+    case Symbol::External:
+      stream << "external";
+      break;
+    case Symbol::Terminal:
+      stream << "terminal";
+      break;
+    case Symbol::NonTerminal:
+      stream << "non-terminal";
+      break;
+  }
+  return stream << " " << rule.index << ")";
+}
+
+ostream &operator<<(ostream &stream, const NamedSymbol &rule) {
+  return stream << "(NamedSymbol " << rule.value << ")";
+}
+
+ostream &operator<<(ostream &stream, const String &rule) {
+  return stream << "(String " << rule.value << ")";
+}
+
+ostream &operator<<(ostream &stream, const Pattern &rule) {
+  return stream << "(Pattern " << rule.value << ")";
+}
+
+ostream &operator<<(ostream &stream, const Choice &rule) {
+  stream << "(Choice";
+  for (const auto &element : rule.elements) {
+    stream << " " << element;
+  }
+  return stream << ")";
+}
+
+ostream &operator<<(ostream &stream, const Seq &rule) {
+  return stream << "(Seq " << *rule.left << " " << *rule.right << ")";
+}
+
+ostream &operator<<(ostream &stream, const Repeat &rule) {
+  return stream << "(Repeat " << *rule.rule << ")";
+}
+
+ostream &operator<<(ostream &stream, const Metadata &rule) {
+  stream << "(Metadata";
+  if (rule.params.has_precedence) stream << " prec=" << to_string(rule.params.precedence);
+  if (rule.params.has_associativity) stream << " assoc=" << rule.params.associativity;
+  if (rule.params.is_token) stream << " token";
+  if (rule.params.is_main_token) stream << " main";
+  return stream << " " << *rule.rule << ")";
+}
+
+ostream &operator<<(ostream &stream, const Rule &rule) {
+  rule.match(
+    [&stream](Blank r) { stream << r; },
+    [&stream](NamedSymbol r) { stream << r; },
+    [&stream](Symbol r) { stream << r; },
+    [&stream](String r) { stream << r; },
+    [&stream](Pattern r) { stream << r; },
+    [&stream](CharacterSet r) { stream << r; },
+    [&stream](Choice r) { stream << r; },
+    [&stream](Seq r) { stream << r; },
+    [&stream](Repeat r) { stream << r; },
+    [&stream](Metadata r) { stream << r; }
+  );
+  return stream;
+}
+
+}  // namespace rules
+
+ostream &operator<<(ostream &stream, const Variable &variable) {
+  return stream << "(Variable " << variable.name << " " << variable.rule << ")";
+}
+
+ostream &operator<<(ostream &stream, const Production &production) {
+  return stream << "(Production " << production.steps << " " <<
+    to_string(production.dynamic_precedence) << ")";
+}
+
+ostream &operator<<(ostream &stream, const SyntaxVariable &variable) {
+  return stream << "(Variable " << variable.name << " " << variable.productions <<
+    " " << to_string(variable.type) << ")";
+}
+
+ostream &operator<<(ostream &stream, const LexicalVariable &variable) {
+  return stream << "(Variable " << variable.name << " " << to_string(variable.type) <<
+    " " << variable.rule << ")";
+}
+
+ostream &operator<<(ostream &stream, const ExternalToken &external_token) {
+  return stream << "(ExternalToken " << external_token.name << " " <<
+    external_token.type << " " << external_token.corresponding_internal_token << ")";
+}
+
+ostream &operator<<(ostream &stream, const ProductionStep &step) {
+  return stream << "(ProductionStep " << step.symbol << " precedence:" <<
+    to_string(step.precedence) << " associativity:" << step.associativity << ")";
+}
+
+ostream &operator<<(ostream &stream, const PrecedenceRange &range) {
+  if (range.empty) {
+    return stream << "(PrecedenceRange)";
+  } else {
+    return stream << "(PrecedenceRange " << to_string(range.min) << " " <<
+      to_string(range.max) << ")";
+  }
+}
+
+namespace build_tables {
+
+ostream &operator<<(ostream &stream, const LexItem &item) {
+  return stream << "(LexItem " << item.lhs << " " << item.rule << ")";
+}
+
+ostream &operator<<(ostream &stream, const LexItemSet &item_set) {
+  return stream << item_set.entries;
+}
+
+ostream &operator<<(ostream &stream, const ParseItem &item) {
+  stream << "(ParseItem " << item.lhs() << " ->";
+  for (size_t i = 0; i < item.production->size(); i++) {
+    if (i == item.step_index) {
+      stream << " •";
+    }
+    stream << " " << item.production->at(i).symbol << " " << item.production->at(i).precedence <<
+      " " << (int)item.production->at(i).associativity;
+  }
+
+  if (item.step_index == item.production->size()) {
+    stream << " • ";
+  }
+
+  return stream << ")";
+}
+
+ostream &operator<<(ostream &stream, const ParseItemSet &item_set) {
+  return stream << item_set.entries;
+}
+
+ostream &operator<<(ostream &stream, const LookaheadSet &lookaheads) {
+  stream << "(LookaheadSet";
+  lookaheads.for_each([&stream](Symbol symbol) {
+    stream << " " << symbol;
+    return true;
+  });
+  return stream << ")";
+}
+
+ostream &operator<<(ostream &stream, const LexItemSet::Transition &transition) {
+  return stream << "(Transition " << transition.destination << " prec:" << transition.precedence << ")";
+}
+
+}  // namespace build_tables
+
+} // namespace tree_sitter

data/ext/tree-sitter/tree-sitter/{spec → test}/helpers/stream_methods.h

@@ -8,6 +8,11 @@
 #include <unordered_set>
 #include <vector>
 #include "compiler/grammar.h"
+#include "compiler/prepare_grammar/interned_grammar.h"
+#include "compiler/prepare_grammar/initial_syntax_grammar.h"
+#include "compiler/lexical_grammar.h"
+#include "compiler/syntax_grammar.h"
+#include "compiler/rule.h"
 #include "compiler/compile_error.h"
 #include "compiler/build_tables/lex_item.h"
 
@@ -91,36 +96,48 @@ namespace tree_sitter {
 using std::ostream;
 using std::string;
 using std::to_string;
-struct Variable;
-struct SyntaxVariable;
+
+struct InputGrammar;
 struct AdvanceAction;
 struct AcceptTokenAction;
-class ParseAction;
-class ParseState;
+struct ParseAction;
+struct ParseState;
 struct ExternalToken;
 struct ProductionStep;
 struct PrecedenceRange;
 
-ostream &operator<<(ostream &, const Grammar &);
+ostream &operator<<(ostream &, const InputGrammar &);
 ostream &operator<<(ostream &, const CompileError &);
-ostream &operator<<(ostream &, const Rule &);
-ostream &operator<<(ostream &, const rule_ptr &);
-ostream &operator<<(ostream &, const Variable &);
-ostream &operator<<(ostream &, const SyntaxVariable &);
-ostream &operator<<(ostream &, const AdvanceAction &);
-ostream &operator<<(ostream &, const AcceptTokenAction &);
-ostream &operator<<(ostream &, const ParseAction &);
-ostream &operator<<(ostream &, const ParseState &);
 ostream &operator<<(ostream &, const ExternalToken &);
 ostream &operator<<(ostream &, const ProductionStep &);
+ostream &operator<<(ostream &, const Production &);
 ostream &operator<<(ostream &, const PrecedenceRange &);
+ostream &operator<<(ostream &, const Variable &);
+ostream &operator<<(ostream &, const LexicalVariable &);
+
+namespace rules {
+
+ostream &operator<<(ostream &, const Blank &);
+ostream &operator<<(ostream &, const CharacterRange &);
+ostream &operator<<(ostream &, const CharacterSet &);
+ostream &operator<<(ostream &, const Symbol &);
+ostream &operator<<(ostream &, const NamedSymbol &);
+ostream &operator<<(ostream &, const String &);
+ostream &operator<<(ostream &, const Pattern &);
+ostream &operator<<(ostream &stream, const Choice &rule);
+ostream &operator<<(ostream &stream, const Seq &rule);
+ostream &operator<<(ostream &stream, const Repeat &rule);
+ostream &operator<<(ostream &stream, const Metadata &rule);
+ostream &operator<<(ostream &stream, const Rule &rule);
+
+}  // namespace rules
 
 namespace build_tables {
 
 class LexItem;
 class LexItemSet;
-class ParseItem;
-class ParseItemSet;
+struct ParseItem;
+struct ParseItemSet;
 class LookaheadSet;
 
 ostream &operator<<(ostream &, const LexItem &);

data/ext/tree-sitter/tree-sitter/test/helpers/tree_helpers.cc

@@ -0,0 +1,99 @@
+#include "bandit/bandit.h"
+#include "helpers/tree_helpers.h"
+#include "helpers/point_helpers.h"
+#include "runtime/document.h"
+#include "runtime/node.h"
+#include <ostream>
+
+using std::string;
+using std::to_string;
+using std::ostream;
+
+const char *symbol_names[24] = {
+  "ERROR", "END",  "two", "three", "four", "five", "six", "seven", "eight",
+  "nine", "ten", "eleven", "twelve", "thirteen", "fourteen", "fifteen",
+  "sixteen", "seventeen", "eighteen", "nineteen", "twenty", "twenty-one",
+  "twenty-two", "twenty-three"
+};
+
+TreeArray *tree_array(std::vector<Tree *> trees) {
+  static TreeArray result;
+  result.capacity = trees.size();
+  result.size = trees.size();
+  result.contents = (Tree **)calloc(trees.size(), sizeof(Tree *));
+  for (size_t i = 0; i < trees.size(); i++)
+    result.contents[i] = trees[i];
+  return &result;
+}
+
+ostream &operator<<(std::ostream &stream, const Tree *tree) {
+  static TSLanguage DUMMY_LANGUAGE = {};
+  static TSDocument DUMMY_DOCUMENT = {};
+  DUMMY_DOCUMENT.parser.language = &DUMMY_LANGUAGE;
+  DUMMY_LANGUAGE.symbol_names = symbol_names;
+  TSNode node;
+  node.data = tree;
+  return stream << string(ts_node_string(node, &DUMMY_DOCUMENT));
+}
+
+ostream &operator<<(ostream &stream, const TSNode &node) {
+  return stream << string("{") << (const Tree *)node.data <<
+    string(", ") << to_string(ts_node_start_byte(node)) << string("}");
+}
+
+bool operator==(const TSNode &left, const TSNode &right) {
+  return ts_node_eq(left, right);
+}
+
+bool operator==(const std::vector<Tree *> &vec, const TreeArray &array) {
+  if (vec.size() != array.size)
+    return false;
+  for (size_t i = 0; i < array.size; i++)
+    if (array.contents[i] != vec[i])
+      return false;
+  return true;
+}
+
+void assert_consistent_tree_sizes(TSNode node) {
+  uint32_t child_count = ts_node_child_count(node);
+  uint32_t named_child_count = ts_node_named_child_count(node);
+  uint32_t start_byte = ts_node_start_byte(node);
+  uint32_t end_byte = ts_node_end_byte(node);
+  TSPoint start_point = ts_node_start_point(node);
+  TSPoint end_point = ts_node_end_point(node);
+
+  AssertThat(start_byte, !IsGreaterThan(end_byte));
+  AssertThat(start_point, !IsGreaterThan(end_point));
+
+  size_t last_child_end_byte = start_byte;
+  TSPoint last_child_end_point = start_point;
+
+  bool some_child_has_changes = false;
+  size_t actual_named_child_count = 0;
+  for (size_t i = 0; i < child_count; i++) {
+    TSNode child = ts_node_child(node, i);
+    uint32_t child_start_byte = ts_node_start_byte(child);
+    TSPoint child_start_point = ts_node_start_point(child);
+
+    AssertThat(child_start_byte, !IsLessThan(last_child_end_byte));
+    AssertThat(child_start_point, !IsLessThan(last_child_end_point));
+    assert_consistent_tree_sizes(child);
+
+    if (ts_node_has_changes(child)) some_child_has_changes = true;
+    if (ts_node_is_named(child)) actual_named_child_count++;
+
+    last_child_end_byte = ts_node_end_byte(child);
+    last_child_end_point = ts_node_end_point(child);
+  }
+
+  AssertThat(actual_named_child_count, Equals(named_child_count));
+
+  if (child_count > 0) {
+    AssertThat(end_byte, !IsLessThan(last_child_end_byte));
+    AssertThat(end_point, !IsLessThan(last_child_end_point));
+  }
+
+  if (some_child_has_changes) {
+    AssertThat(ts_node_has_changes(node), IsTrue());
+  }
+}

data/ext/tree-sitter/tree-sitter/{spec → test}/helpers/tree_helpers.h

@@ -6,11 +6,13 @@
 #include <string>
 
 extern const char *symbol_names[24];
-Tree ** tree_array(std::vector<Tree *> trees);
+TreeArray *tree_array(std::vector<Tree *> trees);
 
 std::ostream &operator<<(std::ostream &stream, const Tree *tree);
 std::ostream &operator<<(std::ostream &stream, const TSNode &node);
 bool operator==(const TSNode &left, const TSNode &right);
 bool operator==(const std::vector<Tree *> &right, const TreeArray &array);
 
+void assert_consistent_tree_sizes(TSNode node);
+
 #endif  // HELPERS_TREE_HELPERS_H_

data/ext/tree-sitter/tree-sitter/test/integration/fuzzing-examples.cc

@@ -0,0 +1,67 @@
+#include "test_helper.h"
+#include "base64.c"
+#include "helpers/load_language.h"
+#include "helpers/tree_helpers.h"
+#include "helpers/record_alloc.h"
+
+START_TEST
+
+vector<pair<string, string>> examples({
+  {
+    "javascript",
+    "Bi0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLXGK0i0vLS0tLS0tLS0tLS0tLS0tLS0tLS0tLXGK0i0vLS0tLS0tLS0tLS0tLS0xLS0tLTYtLfpZAA=="
+  },
+  {
+    "python",
+    "NWNvbogsKTMsLCwsY29uiCwqLDo1Y29uLA=="
+  },
+});
+
+describe("examples found via fuzzing", [&]() {
+  before_each([&]() {
+    record_alloc::start();
+  });
+
+  after_each([&]() {
+    AssertThat(record_alloc::outstanding_allocation_indices(), IsEmpty());
+  });
+
+  for (unsigned i = 0, n = examples.size(); i < n; i++) {
+
+    it(("parses example number " + to_string(i)).c_str(), [&]() {
+      TSDocument *document = ts_document_new();
+
+      if (getenv("TREE_SITTER_ENABLE_DEBUG_GRAPHS")) {
+        ts_document_print_debugging_graphs(document, true);
+      }
+
+      const string &language_name = examples[i].first;
+      ts_document_set_language(document, load_real_language(language_name));
+
+      string input;
+      const string &base64_input = examples[i].second;
+      input.resize(base64_input.size());
+      input.resize(base64_decode(
+        reinterpret_cast<const unsigned char *>(base64_input.c_str()),
+        reinterpret_cast<unsigned char *>(&input[0]),
+        base64_input.size()
+      ));
+
+      ts_document_set_input_string_with_length(
+        document,
+        input.c_str(),
+        input.size()
+      );
+
+      ts_document_parse(document);
+
+      TSNode node = ts_document_root_node(document);
+      assert_consistent_tree_sizes(node);
+
+      ts_document_free(document);
+    });
+
+  }
+});
+
+END_TEST