tree-sitter 0.0.1 → 0.1.0

data/ext/tree-sitter/tree-sitter/src/compiler/parse_grammar.h

@@ -2,6 +2,7 @@
 #define COMPILER_GRAMMAR_JSON_H_
 
 #include <string>
+#include <unordered_set>
 #include "tree_sitter/compiler.h"
 #include "compiler/grammar.h"
 
@@ -9,7 +10,7 @@ namespace tree_sitter {
 
 struct ParseGrammarResult {
   std::string name;
-  Grammar grammar;
+  InputGrammar grammar;
   std::string error_message;
 };
 

data/ext/tree-sitter/tree-sitter/src/compiler/parse_table.cc

@@ -1,37 +1,27 @@
 #include "compiler/parse_table.h"
 #include <string>
 #include "compiler/precedence_range.h"
-#include "compiler/rules/built_in_symbols.h"
+#include "compiler/rule.h"
 
 namespace tree_sitter {
 
 using std::string;
-using std::ostream;
 using std::to_string;
-using std::set;
 using std::vector;
 using std::function;
 using rules::Symbol;
 
-ParseAction::ParseAction(ParseActionType type, ParseStateId state_index,
-                         Symbol symbol, size_t consumed_symbol_count,
-                         const Production *production)
-    : type(type),
-      extra(false),
-      fragile(false),
-      state_index(state_index),
-      symbol(symbol),
-      consumed_symbol_count(consumed_symbol_count),
-      production(production) {}
-
-ParseAction::ParseAction()
-    : type(ParseActionTypeError),
-      extra(false),
-      fragile(false),
-      state_index(-1),
-      symbol(rules::NONE()),
-      consumed_symbol_count(0),
-      production(nullptr) {}
+ParseAction::ParseAction() :
+  type(ParseActionTypeError),
+  state_index(-1),
+  symbol(rules::NONE()),
+  consumed_symbol_count(0),
+  precedence(0),
+  dynamic_precedence(0),
+  associativity(rules::AssociativityNone),
+  alias_sequence_id(0),
+  extra(false),
+  repetition(false) {}
 
 ParseAction ParseAction::Error() {
   return ParseAction();
@@ -44,12 +34,16 @@ ParseAction ParseAction::Accept() {
 }
 
 ParseAction ParseAction::Shift(ParseStateId state_index) {
-  return ParseAction(ParseActionTypeShift, state_index, rules::NONE(), 0, nullptr);
+  ParseAction result;
+  result.type = ParseActionTypeShift;
+  result.state_index = state_index;
+  return result;
 }
 
-ParseAction ParseAction::Recover(ParseStateId state_index) {
-  return ParseAction(ParseActionTypeRecover, state_index, rules::NONE(), 0,
-                     nullptr);
+ParseAction ParseAction::Recover() {
+  ParseAction result;
+  result.type = ParseActionTypeRecover;
+  return result;
 }
 
 ParseAction ParseAction::ShiftExtra() {
@@ -60,82 +54,63 @@ ParseAction ParseAction::ShiftExtra() {
 }
 
 ParseAction ParseAction::Reduce(Symbol symbol, size_t consumed_symbol_count,
-                                const Production &production) {
-  return ParseAction(ParseActionTypeReduce, 0, symbol, consumed_symbol_count,
-                     &production);
-}
-
-int ParseAction::precedence() const {
-  if (consumed_symbol_count >= production->size()) {
-    if (production->empty()) {
-      return 0;
-    } else {
-      return production->back().precedence;
-    }
-  } else {
-    return production->at(consumed_symbol_count).precedence;
-  }
-}
-
-rules::Associativity ParseAction::associativity() const {
-  if (consumed_symbol_count >= production->size()) {
-    if (production->empty()) {
-      return rules::AssociativityNone;
-    } else {
-      return production->back().associativity;
-    }
-  } else {
-    return production->at(consumed_symbol_count).associativity;
-  }
+                                int precedence, int dynamic_precedence,
+                                rules::Associativity associativity, unsigned alias_sequence_id) {
+  ParseAction result;
+  result.type = ParseActionTypeReduce;
+  result.symbol = symbol;
+  result.consumed_symbol_count = consumed_symbol_count;
+  result.precedence = precedence;
+  result.dynamic_precedence = dynamic_precedence;
+  result.associativity = associativity;
+  result.alias_sequence_id = alias_sequence_id;
+  return result;
 }
 
 bool ParseAction::operator==(const ParseAction &other) const {
-  return (type == other.type && extra == other.extra &&
-          fragile == other.fragile && symbol == other.symbol &&
-          state_index == other.state_index && production == other.production &&
-          consumed_symbol_count == other.consumed_symbol_count);
+  return
+    type == other.type &&
+    state_index == other.state_index &&
+    symbol == other.symbol &&
+    consumed_symbol_count == other.consumed_symbol_count &&
+    precedence == other.precedence &&
+    dynamic_precedence == other.dynamic_precedence &&
+    associativity == other.associativity &&
+    alias_sequence_id == other.alias_sequence_id &&
+    extra == other.extra &&
+    repetition == other.repetition;
 }
 
 bool ParseAction::operator<(const ParseAction &other) const {
-  if (type < other.type)
-    return true;
-  if (other.type < type)
-    return false;
-  if (extra && !other.extra)
-    return true;
-  if (other.extra && !extra)
-    return false;
-  if (fragile && !other.fragile)
-    return true;
-  if (other.fragile && !fragile)
-    return false;
-  if (symbol < other.symbol)
-    return true;
-  if (other.symbol < symbol)
-    return false;
-  if (state_index < other.state_index)
-    return true;
-  if (other.state_index < state_index)
-    return false;
-  if (production < other.production)
-    return true;
-  if (other.production < production)
-    return false;
-  return consumed_symbol_count < other.consumed_symbol_count;
-}
-
-ParseTableEntry::ParseTableEntry()
-    : reusable(true), depends_on_lookahead(false) {}
-
-ParseTableEntry::ParseTableEntry(const vector<ParseAction> &actions,
-                                 bool reusable, bool depends_on_lookahead)
-    : actions(actions),
-      reusable(reusable),
-      depends_on_lookahead(depends_on_lookahead) {}
+  if (type < other.type) return true;
+  if (other.type < type) return false;
+  if (state_index < other.state_index) return true;
+  if (other.state_index < state_index) return false;
+  if (symbol < other.symbol) return true;
+  if (other.symbol < symbol) return false;
+  if (consumed_symbol_count < other.consumed_symbol_count) return true;
+  if (other.consumed_symbol_count < consumed_symbol_count) return false;
+  if (precedence < other.precedence) return true;
+  if (other.precedence < precedence) return false;
+  if (dynamic_precedence < other.dynamic_precedence) return true;
+  if (other.dynamic_precedence < dynamic_precedence) return false;
+  if (associativity < other.associativity) return true;
+  if (other.associativity < associativity) return false;
+  if (extra && !other.extra) return true;
+  if (other.extra && !extra) return false;
+  if (repetition && !other.repetition) return true;
+  if (other.repetition && !repetition) return false;
+  return alias_sequence_id < other.alias_sequence_id;
+}
+
+ParseTableEntry::ParseTableEntry() : reusable(true) {}
+
+ParseTableEntry::ParseTableEntry(const vector<ParseAction> &actions, bool reusable)
+  : actions(actions),
+    reusable(reusable) {}
 
 bool ParseTableEntry::operator==(const ParseTableEntry &other) const {
-  return actions == other.actions && reusable == other.reusable &&
-         depends_on_lookahead == other.depends_on_lookahead;
+  return actions == other.actions && reusable == other.reusable;
 }
 
 ParseState::ParseState() : lex_state_id(-1) {}
@@ -148,17 +123,10 @@ bool ParseState::has_shift_action() const {
   return (!nonterminal_entries.empty());
 }
 
-set<Symbol> ParseState::expected_inputs() const {
-  set<Symbol> result;
-  for (auto &entry : terminal_entries)
-    result.insert(entry.first);
-  return result;
-}
-
 void ParseState::each_referenced_state(function<void(ParseStateId *)> fn) {
   for (auto &entry : terminal_entries)
     for (ParseAction &action : entry.second.actions)
-      if (action.type == ParseActionTypeShift || ParseActionTypeRecover)
+      if (action.type == ParseActionTypeShift && !action.extra)
         fn(&action.state_index);
   for (auto &entry : nonterminal_entries)
     fn(&entry.second);
@@ -169,26 +137,9 @@ bool ParseState::operator==(const ParseState &other) const {
     nonterminal_entries == other.nonterminal_entries;
 }
 
-set<Symbol> ParseTable::all_symbols() const {
-  set<Symbol> result;
-  for (auto &pair : symbols)
-    result.insert(pair.first);
-  return result;
-}
-
-ParseStateId ParseTable::add_state() {
-  states.push_back(ParseState());
-  return states.size() - 1;
-}
-
 ParseAction &ParseTable::add_terminal_action(ParseStateId state_id,
                                              Symbol lookahead,
                                              ParseAction action) {
-  if (action.type == ParseActionTypeShift && action.extra)
-    symbols[lookahead].extra = true;
-  else
-    symbols[lookahead].structural = true;
-
   ParseTableEntry &entry = states[state_id].terminal_entries[lookahead];
   entry.actions.push_back(action);
   return *entry.actions.rbegin();
@@ -197,62 +148,8 @@ ParseAction &ParseTable::add_terminal_action(ParseStateId state_id,
 void ParseTable::set_nonterminal_action(ParseStateId state_id,
                                         Symbol::Index lookahead,
                                         ParseStateId next_state_id) {
-  symbols[Symbol(lookahead, Symbol::NonTerminal)].structural = true;
+  symbols.insert(Symbol::non_terminal(lookahead));
   states[state_id].nonterminal_entries[lookahead] = next_state_id;
 }
 
-static bool has_entry(const ParseState &state, const ParseTableEntry &entry) {
-  for (const auto &pair : state.terminal_entries)
-    if (pair.second == entry)
-      return true;
-  return false;
-}
-
-bool ParseTable::merge_state(size_t i, size_t j) {
-  ParseState &state = states[i];
-  ParseState &other = states[j];
-
-  if (state.nonterminal_entries != other.nonterminal_entries)
-    return false;
-
-  for (auto &entry : state.terminal_entries) {
-    Symbol lookahead = entry.first;
-    const vector<ParseAction> &actions = entry.second.actions;
-
-    const auto &other_entry = other.terminal_entries.find(lookahead);
-    if (other_entry == other.terminal_entries.end()) {
-      if (mergeable_symbols.count(lookahead) == 0 && !lookahead.is_built_in())
-        return false;
-      if (actions.back().type != ParseActionTypeReduce)
-        return false;
-      if (!has_entry(other, entry.second))
-        return false;
-    } else if (entry.second != other_entry->second) {
-      return false;
-    }
-  }
-
-  set<Symbol> symbols_to_merge;
-
-  for (auto &entry : other.terminal_entries) {
-    Symbol lookahead = entry.first;
-    const vector<ParseAction> &actions = entry.second.actions;
-
-    if (!state.terminal_entries.count(lookahead)) {
-      if (mergeable_symbols.count(lookahead) == 0 && !lookahead.is_built_in())
-        return false;
-      if (actions.back().type != ParseActionTypeReduce)
-        return false;
-      if (!has_entry(state, entry.second))
-        return false;
-      symbols_to_merge.insert(lookahead);
-    }
-  }
-
-  for (const Symbol &lookahead : symbols_to_merge)
-    state.terminal_entries[lookahead] = other.terminal_entries.find(lookahead)->second;
-
-  return true;
-}
-
 }  // namespace tree_sitter

data/ext/tree-sitter/tree-sitter/src/compiler/parse_table.h

@@ -6,8 +6,7 @@
 #include <utility>
 #include <vector>
 #include "compiler/lex_table.h"
-#include "compiler/rules/symbol.h"
-#include "compiler/rules/metadata.h"
+#include "compiler/rule.h"
 #include "compiler/precedence_range.h"
 #include "compiler/syntax_grammar.h"
 
@@ -23,54 +22,45 @@ enum ParseActionType {
   ParseActionTypeRecover,
 };
 
-class ParseAction {
-  ParseAction(ParseActionType type, ParseStateId state_index,
-              rules::Symbol symbol, size_t consumed_symbol_count,
-              const Production *);
-
- public:
+struct ParseAction {
   ParseAction();
   static ParseAction Accept();
   static ParseAction Error();
   static ParseAction Shift(ParseStateId state_index);
-  static ParseAction Recover(ParseStateId state_index);
-  static ParseAction Reduce(rules::Symbol symbol, size_t consumed_symbol_count,
-                            const Production &);
+  static ParseAction Recover();
+  static ParseAction Reduce(rules::Symbol symbol, size_t child_count,
+                            int precedence, int dynamic_precedence, rules::Associativity,
+                            unsigned alias_sequence_id);
   static ParseAction ShiftExtra();
   bool operator==(const ParseAction &) const;
   bool operator<(const ParseAction &) const;
 
-  rules::Associativity associativity() const;
-  int precedence() const;
-
   ParseActionType type;
-  bool extra;
-  bool fragile;
   ParseStateId state_index;
-
   rules::Symbol symbol;
-  size_t consumed_symbol_count;
-  const Production *production;
+  unsigned consumed_symbol_count;
+  int precedence;
+  int dynamic_precedence;
+  rules::Associativity associativity;
+  unsigned alias_sequence_id;
+  bool extra;
+  bool repetition;
 };
 
 struct ParseTableEntry {
-  std::vector<ParseAction> actions;
-  bool reusable;
-  bool depends_on_lookahead;
-
   ParseTableEntry();
-  ParseTableEntry(const std::vector<ParseAction> &, bool, bool);
+  ParseTableEntry(const std::vector<ParseAction> &, bool);
   bool operator==(const ParseTableEntry &other) const;
-
   inline bool operator!=(const ParseTableEntry &other) const {
     return !operator==(other);
   }
+
+  std::vector<ParseAction> actions;
+  bool reusable;
 };
 
-class ParseState {
- public:
+struct ParseState {
   ParseState();
-  std::set<rules::Symbol> expected_inputs() const;
   bool operator==(const ParseState &) const;
   bool merge(const ParseState &);
   void each_referenced_state(std::function<void(ParseStateId *)>);
@@ -79,26 +69,18 @@ class ParseState {
   std::map<rules::Symbol, ParseTableEntry> terminal_entries;
   std::map<rules::Symbol::Index, ParseStateId> nonterminal_entries;
   LexStateId lex_state_id;
-  size_t shift_actions_signature;
 };
 
-struct ParseTableSymbolMetadata {
-  bool extra;
-  bool structural;
-};
+using AliasSequence = std::vector<rules::Alias>;
 
-class ParseTable {
- public:
-  std::set<rules::Symbol> all_symbols() const;
-  ParseStateId add_state();
+struct ParseTable {
   ParseAction &add_terminal_action(ParseStateId state_id, rules::Symbol, ParseAction);
   void set_nonterminal_action(ParseStateId, rules::Symbol::Index, ParseStateId);
-  bool merge_state(size_t i, size_t j);
 
   std::vector<ParseState> states;
-  std::map<rules::Symbol, ParseTableSymbolMetadata> symbols;
-
-  std::set<rules::Symbol> mergeable_symbols;
+  std::set<rules::Symbol> symbols;
+  std::vector<AliasSequence> alias_sequences;
+  unsigned max_alias_sequence_length = 0;
 };
 
 }  // namespace tree_sitter

data/ext/tree-sitter/tree-sitter/src/compiler/prepare_grammar/expand_repeats.cc

@@ -2,14 +2,9 @@
 #include <vector>
 #include <string>
 #include <utility>
-#include "compiler/prepare_grammar/initial_syntax_grammar.h"
+#include <cassert>
+#include "compiler/grammar.h"
 #include "compiler/rule.h"
-#include "compiler/rules/visitor.h"
-#include "compiler/rules/seq.h"
-#include "compiler/rules/symbol.h"
-#include "compiler/rules/choice.h"
-#include "compiler/rules/blank.h"
-#include "compiler/rules/repeat.h"
 
 namespace tree_sitter {
 namespace prepare_grammar {
@@ -18,40 +13,72 @@ using std::string;
 using std::vector;
 using std::pair;
 using std::to_string;
-using std::make_shared;
-using rules::Blank;
-using rules::Choice;
-using rules::Repeat;
-using rules::Seq;
+using rules::Rule;
 using rules::Symbol;
 
-class ExpandRepeats : public rules::IdentityRuleFn {
+class ExpandRepeats {
   string rule_name;
   size_t offset;
   size_t repeat_count;
-  vector<pair<rule_ptr, Symbol>> existing_repeats;
-
-  rule_ptr apply_to(const Repeat *rule) {
-    for (const auto pair : existing_repeats)
-      if (pair.first->operator==(*rule))
-        return pair.second.copy();
-
-    rule_ptr inner_rule = apply(rule->content);
-    size_t index = aux_rules.size();
-    string helper_rule_name = rule_name + "_repeat" + to_string(++repeat_count);
-    Symbol repeat_symbol(offset + index, Symbol::NonTerminal);
-    existing_repeats.push_back({ rule->copy(), repeat_symbol });
-    aux_rules.push_back(
-      Variable(helper_rule_name, VariableTypeAuxiliary,
-               Choice::build({ Seq::build({ repeat_symbol.copy(), inner_rule }),
-                               inner_rule })));
-    return repeat_symbol.copy();
+  vector<pair<Rule, Symbol>> existing_repeats;
+
+  Rule apply(Rule rule) {
+    return rule.match(
+      [&](const rules::Blank &blank) -> Rule { return blank; },
+      [&](const rules::Symbol &symbol) { return symbol; },
+
+      [&](const rules::Choice &choice) {
+        vector<Rule> elements;
+        for (const auto &element : choice.elements) {
+          elements.push_back(apply(element));
+        }
+        return Rule::choice(elements);
+      },
+
+      [&](const rules::Seq &sequence) {
+        auto left = apply(*sequence.left);
+        auto right = apply(*sequence.right);
+        return rules::Seq{left, right};
+      },
+
+      [&](const rules::Repeat &repeat) {
+        for (const auto pair : existing_repeats) {
+          if (pair.first == rule) {
+            return pair.second;
+          }
+        }
+
+        Rule inner_rule = apply(*repeat.rule);
+        size_t index = aux_rules.size();
+        string helper_rule_name = rule_name + "_repeat" + to_string(++repeat_count);
+        Symbol repeat_symbol = Symbol::non_terminal(offset + index);
+        existing_repeats.push_back({repeat, repeat_symbol});
+        aux_rules.push_back({
+          helper_rule_name,
+          VariableTypeAuxiliary,
+          rules::Choice{{
+            rules::Seq{repeat_symbol, repeat_symbol},
+            inner_rule,
+          }}
+        });
+        return repeat_symbol;
+      },
+
+      [&](const rules::Metadata &metadata) {
+        return rules::Metadata{apply(*metadata.rule), metadata.params};
+      },
+
+      [](auto) {
+        assert(!"Unexpected rule type");
+        return rules::Blank{};
+      }
+    );
   }
 
  public:
   explicit ExpandRepeats(size_t offset) : offset(offset) {}
 
-  rule_ptr expand(const rule_ptr &rule, const string &name) {
+  Rule expand(const Rule &rule, const string &name) {
     rule_name = name;
     repeat_count = 0;
     return apply(rule);
@@ -66,13 +93,19 @@ InitialSyntaxGrammar expand_repeats(const InitialSyntaxGrammar &grammar) {
   result.extra_tokens = grammar.extra_tokens;
   result.expected_conflicts = grammar.expected_conflicts;
   result.external_tokens = grammar.external_tokens;
+  result.variables_to_inline = grammar.variables_to_inline;
 
   ExpandRepeats expander(result.variables.size());
-  for (auto &variable : result.variables)
+  for (auto &variable : result.variables) {
     variable.rule = expander.expand(variable.rule, variable.name);
+  }
+
+  result.variables.insert(
+    result.variables.end(),
+    expander.aux_rules.begin(),
+    expander.aux_rules.end()
+  );
 
-  result.variables.insert(result.variables.end(), expander.aux_rules.begin(),
-                          expander.aux_rules.end());
   return result;
 }