tree-sitter 0.0.1 → 0.1.0

data/ext/tree-sitter/tree-sitter/src/compiler/build_tables/parse_item.h

@@ -4,16 +4,14 @@
 #include <map>
 #include <utility>
 #include "compiler/build_tables/lookahead_set.h"
-#include "compiler/rules/symbol.h"
-#include "compiler/rules/metadata.h"
+#include "compiler/rule.h"
 #include "compiler/syntax_grammar.h"
 #include "compiler/precedence_range.h"
 
 namespace tree_sitter {
 namespace build_tables {
 
-class ParseItem {
- public:
+struct ParseItem {
   ParseItem();
   ParseItem(const rules::Symbol &, const Production &, unsigned int);
 
@@ -28,6 +26,7 @@ class ParseItem {
   rules::Symbol lhs() const;
   rules::Symbol next_symbol() const;
   int precedence() const;
+  int dynamic_precedence() const;
   rules::Associativity associativity() const;
   bool is_done() const;
 
@@ -36,11 +35,7 @@ class ParseItem {
   unsigned int step_index;
 };
 
-class ParseItemSet {
- public:
-  ParseItemSet();
-  explicit ParseItemSet(const std::map<ParseItem, LookaheadSet> &);
-
+struct ParseItemSet {
   bool operator==(const ParseItemSet &) const;
   void add(const ParseItemSet &);
   size_t unfinished_item_signature() const;

data/ext/tree-sitter/tree-sitter/src/compiler/build_tables/parse_item_set_builder.cc

@@ -1,48 +1,78 @@
 #include "compiler/build_tables/parse_item_set_builder.h"
+#include <algorithm>
+#include <cassert>
 #include <set>
+#include <unordered_map>
 #include <vector>
 #include <utility>
 #include "compiler/syntax_grammar.h"
 #include "compiler/lexical_grammar.h"
-#include "compiler/rules/built_in_symbols.h"
+#include "compiler/rule.h"
 
 namespace tree_sitter {
 namespace build_tables {
 
-using std::vector;
-using std::set;
-using std::map;
+using std::find;
 using std::get;
+using std::move;
 using std::pair;
-using std::tuple;
-using std::make_tuple;
-using std::shared_ptr;
-using std::make_shared;
+using std::set;
+using std::unordered_map;
+using std::vector;
 using rules::Symbol;
-using rules::NONE;
 
-ParseItemSetBuilder::ParseItemSetBuilder(const SyntaxGrammar &grammar,
-                                         const LexicalGrammar &lexical_grammar) {
-  vector<Symbol> symbols_to_process;
-  set<Symbol::Index> processed_non_terminals;
+struct FollowSetInfo {
+  LookaheadSet lookaheads;
+  bool propagates_lookaheads;
+};
 
-  for (size_t i = 0, n = lexical_grammar.variables.size(); i < n; i++) {
-    Symbol symbol(i, Symbol::Terminal);
-    first_sets.insert({symbol, LookaheadSet({ symbol })});
+struct NonTerminalQueueEntry {
+  Symbol::Index non_terminal;
+  LookaheadSet lookaheads;
+  bool propagates_lookaheads;
+};
+
+bool ParseItemSetBuilder::ParseItemSetComponent::operator==(
+  const ParseItemSetBuilder::ParseItemSetComponent &other) const {
+  return item == other.item &&
+    lookaheads == other.lookaheads &&
+    propagates_lookaheads == other.propagates_lookaheads;
+}
+
+template <typename T>
+inline void find_or_push(vector<T> &vector, const T &item) {
+  if (find(vector.begin(), vector.end(), item) == vector.end()) {
+    vector.push_back(item);
   }
+}
+
+ParseItemSetBuilder::ParseItemSetBuilder(
+  const SyntaxGrammar &grammar,
+  const LexicalGrammar &lexical_grammar
+) : grammar{grammar} {
 
+  // Populate the FIRST and LAST set of each terminal, which just contains the terminal itself.
+  for (size_t i = 0, n = lexical_grammar.variables.size(); i < n; i++) {
+    Symbol symbol = Symbol::terminal(i);
+    first_sets.insert({symbol, LookaheadSet({symbol})});
+    last_sets.insert({symbol, LookaheadSet({symbol})});
+  }
   for (size_t i = 0, n = grammar.external_tokens.size(); i < n; i++) {
-    Symbol symbol(i, Symbol::External);
-    first_sets.insert({symbol, LookaheadSet({ symbol })});
+    Symbol symbol = Symbol::external(i);
+    first_sets.insert({symbol, LookaheadSet({symbol})});
+    last_sets.insert({symbol, LookaheadSet({symbol})});
   }
 
+  // Populate the FIRST and LAST set of each non-terminal by recursively expanding non-terminals.
+  vector<Symbol> symbols_to_process;
+  set<Symbol::Index> processed_non_terminals;
   for (size_t i = 0, n = grammar.variables.size(); i < n; i++) {
-    Symbol symbol(i, Symbol::NonTerminal);
-    LookaheadSet first_set;
+    Symbol symbol = Symbol::non_terminal(i);
+    LookaheadSet &first_set = first_sets[symbol];
+    LookaheadSet &last_set = last_sets[symbol];
 
     processed_non_terminals.clear();
-    symbols_to_process.clear();
-    symbols_to_process.push_back(symbol);
+    symbols_to_process.assign({symbol});
     while (!symbols_to_process.empty()) {
       Symbol current_symbol = symbols_to_process.back();
       symbols_to_process.pop_back();
@@ -58,58 +88,64 @@ ParseItemSetBuilder::ParseItemSetBuilder(const SyntaxGrammar &grammar,
       }
     }
 
-    first_sets.insert({symbol, first_set});
-  }
-
-  vector<ParseItemSetComponent> components_to_process;
+    processed_non_terminals.clear();
+    symbols_to_process.assign({symbol});
+    while (!symbols_to_process.empty()) {
+      Symbol current_symbol = symbols_to_process.back();
+      symbols_to_process.pop_back();
 
-  for (size_t i = 0, n = grammar.variables.size(); i < n; i++) {
-    Symbol symbol(i, Symbol::NonTerminal);
-    map<ParseItem, pair<LookaheadSet, bool>> cache_entry;
-
-    components_to_process.clear();
-    for (const Production &production : grammar.variables[i].productions) {
-      components_to_process.push_back(ParseItemSetComponent{
-        ParseItem(symbol, production, 0),
-        LookaheadSet(),
-        true
-      });
+      if (!current_symbol.is_non_terminal()) {
+        last_set.insert(current_symbol);
+      } else if (processed_non_terminals.insert(current_symbol.index).second) {
+        for (const Production &production : grammar.variables[current_symbol.index].productions) {
+          if (!production.empty()) {
+            symbols_to_process.push_back(production.back().symbol);
+          }
+        }
+      }
     }
+  }
+
+  // Populate a cache of which ParseItems will be created when a given non-terminal is expanded.
+  vector<NonTerminalQueueEntry> non_terminal_queue;
+  for (Symbol::Index i = 0, n = grammar.variables.size(); i < n; i++) {
 
-    while (!components_to_process.empty()) {
-      ParseItemSetComponent component = components_to_process.back();
-      ParseItem &item = component.item;
-      LookaheadSet &lookaheads = component.lookaheads;
-      components_to_process.pop_back();
+    // Compute the follow set of each *other* non-terminal that the current non-terminal can
+    // start with.
+    unordered_map<Symbol::Index, FollowSetInfo> follow_set_info_by_non_terminal;
+    non_terminal_queue.assign({{i, LookaheadSet(), true}});
+    while (!non_terminal_queue.empty()) {
+      NonTerminalQueueEntry queue_entry = non_terminal_queue.back();
+      non_terminal_queue.pop_back();
 
-      bool component_is_new;
-      if (component.propagates_lookaheads) {
-        component_is_new = !cache_entry[item].second;
-        cache_entry[item].second = true;
+      bool queue_entry_is_new;
+      auto &follow_set_info = follow_set_info_by_non_terminal[queue_entry.non_terminal];
+      if (queue_entry.propagates_lookaheads) {
+        queue_entry_is_new = !follow_set_info.propagates_lookaheads;
+        follow_set_info.propagates_lookaheads = true;
       } else {
-        component_is_new = cache_entry[item].first.insert_all(lookaheads);
+        queue_entry_is_new = follow_set_info.lookaheads.insert_all(queue_entry.lookaheads);
       }
 
-      if (component_is_new) {
-        Symbol next_symbol = item.next_symbol();
-        if (!next_symbol.is_non_terminal() || next_symbol.is_built_in())
-          continue;
+      if (queue_entry_is_new) {
+        for (const Production &production : grammar.variables[queue_entry.non_terminal].productions) {
+          if (production.empty()) continue;
+          Symbol next_symbol = production.at(0).symbol;
+          if (!next_symbol.is_non_terminal() || next_symbol.is_built_in()) continue;
 
-        LookaheadSet next_lookaheads;
-        bool propagates_lookaheads;
-        size_t next_step = item.step_index + 1;
-        if (next_step == item.production->size()) {
-          next_lookaheads = lookaheads;
-          propagates_lookaheads = component.propagates_lookaheads;
-        } else {
-          Symbol symbol_after_next = item.production->at(next_step).symbol;
-          next_lookaheads = first_sets.find(symbol_after_next)->second;
-          propagates_lookaheads = false;
-        }
+          LookaheadSet next_lookaheads;
+          bool propagates_lookaheads;
+          if (production.size() == 1) {
+            next_lookaheads = queue_entry.lookaheads;
+            propagates_lookaheads = queue_entry.propagates_lookaheads;
+          } else {
+            Symbol symbol_after_next = production.at(1).symbol;
+            next_lookaheads = first_sets.find(symbol_after_next)->second;
+            propagates_lookaheads = false;
+          }
 
-        for (const Production &production : grammar.variables[next_symbol.index].productions) {
-          components_to_process.push_back(ParseItemSetComponent{
-            ParseItem(next_symbol, production, 0),
+          non_terminal_queue.push_back({
+            next_symbol.index,
             next_lookaheads,
             propagates_lookaheads
           });
@@ -117,45 +153,140 @@ ParseItemSetBuilder::ParseItemSetBuilder(const SyntaxGrammar &grammar,
       }
     }
 
-    for (auto &pair : cache_entry) {
-      component_cache[symbol.index].push_back(ParseItemSetComponent{
-        pair.first,
-        pair.second.first,
-        pair.second.second
-      });
+    // Use these follow sets to populate the cache of ParseItems for non-terminal `i`.
+    for (auto &pair : follow_set_info_by_non_terminal) {
+      Symbol non_terminal = Symbol::non_terminal(pair.first);
+
+      for (const Production &production : grammar.variables[non_terminal.index].productions) {
+        ParseItem item(non_terminal, production, 0);
+
+        if (grammar.variables_to_inline.count(item.next_symbol())) {
+          for (const Production &inlined_production : inline_production(item)) {
+            find_or_push(transitive_closure_component_cache[i], {
+              ParseItem(non_terminal, inlined_production, 0),
+              pair.second.lookaheads,
+              pair.second.propagates_lookaheads
+            });
+          }
+        } else if (!grammar.variables_to_inline.count(non_terminal)) {
+          find_or_push(transitive_closure_component_cache[i], {
+            item,
+            pair.second.lookaheads,
+            pair.second.propagates_lookaheads
+          });
+        }
+      }
+    }
+  }
+}
+
+const vector<Production> &ParseItemSetBuilder::inline_production(const ParseItem &item) {
+  vector<Production> &result = inlined_productions_by_original_production[item];
+  if (!result.empty()) return result;
+
+  auto &inlined_step = item.production->at(item.step_index);
+  vector<const Production *> productions_to_insert;
+  for (auto &production : grammar.variables[inlined_step.symbol.index].productions) {
+    productions_to_insert.push_back(&production);
+  }
+
+  for (auto iter = productions_to_insert.begin(); iter != productions_to_insert.end();) {
+    const Production *production = *iter;
+
+    if (!production->empty() && grammar.variables_to_inline.count(production->steps.front().symbol)) {
+      iter = productions_to_insert.erase(iter);
+      for (auto &inlined_production : inline_production(ParseItem(inlined_step.symbol, *production, 0))) {
+        iter = productions_to_insert.insert(iter, &inlined_production);
+      }
+    } else {
+      ++iter;
     }
   }
+
+  for (const Production *production_to_insert : productions_to_insert) {
+    auto begin = item.production->steps.begin();
+    auto end = item.production->steps.end();
+    auto step = begin + item.step_index;
+    Production production({begin, step}, item.production->dynamic_precedence);
+
+    for (auto &step : *production_to_insert) {
+      production.steps.push_back(step);
+      if (!inlined_step.alias.value.empty()) {
+        production.steps.back().alias = inlined_step.alias;
+      }
+    }
+
+    if (!production.back().precedence) {
+      production.back().precedence = inlined_step.precedence;
+    }
+    if (!production.back().associativity) {
+      production.back().associativity = inlined_step.associativity;
+    }
+    production.steps.insert(
+      production.steps.end(),
+      step + 1,
+      end
+    );
+
+    if (find(result.begin(), result.end(), production) == result.end()) {
+      result.push_back(move(production));
+    }
+  }
+
+  return result;
 }
 
 void ParseItemSetBuilder::apply_transitive_closure(ParseItemSet *item_set) {
-  item_set_buffer.clear();
-
-  for (const auto &pair : item_set->entries) {
-    const ParseItem &item = pair.first;
-    const LookaheadSet &lookaheads = pair.second;
-
-    const Symbol &next_symbol = item.next_symbol();
-    if (next_symbol.is_non_terminal() && !next_symbol.is_built_in()) {
-      LookaheadSet next_lookaheads;
-      size_t next_step = item.step_index + 1;
-      if (next_step == item.production->size()) {
-        next_lookaheads = lookaheads;
-      } else {
-        Symbol symbol_after_next = item.production->at(next_step).symbol;
-        next_lookaheads = first_sets.find(symbol_after_next)->second;
-      }
+  for (auto iter = item_set->entries.begin(), end = item_set->entries.end(); iter != end;) {
+    const ParseItem &item = iter->first;
+    const LookaheadSet &lookaheads = iter->second;
+
+    // Items whose `step_index` is 0 are not part of the item set's "kernel"; they have been
+    // added in previous iterations of this loop, and they don't need to be further processed.
+    if (item.lhs() == rules::START() || item.step_index > 0) {
+
+      // Kernel items whose next symbol is a non-terminal are expanded using the pre-computed
+      // parse item cache.
+      const Symbol &next_symbol = item.next_symbol();
+      if (next_symbol.is_non_terminal() && !next_symbol.is_built_in()) {
 
-      for (const ParseItemSetComponent &component : component_cache[next_symbol.index]) {
-        item_set_buffer.push_back({component.item, component.lookaheads});
-        if (component.propagates_lookaheads) {
-          item_set_buffer.push_back({component.item, next_lookaheads});
+        LookaheadSet next_lookaheads;
+        size_t next_step = item.step_index + 1;
+        if (next_step == item.production->size()) {
+          next_lookaheads = lookaheads;
+        } else {
+          Symbol symbol_after_next = item.production->at(next_step).symbol;
+          next_lookaheads = first_sets.find(symbol_after_next)->second;
+        }
+
+        for (const auto &component : transitive_closure_component_cache[next_symbol.index]) {
+          LookaheadSet &current_lookaheads = item_set->entries[component.item];
+          current_lookaheads.insert_all(component.lookaheads);
+          if (component.propagates_lookaheads) {
+            current_lookaheads.insert_all(next_lookaheads);
+          }
+        }
+
+        if (grammar.variables_to_inline.count(next_symbol)) {
+          for (const Production &inlined_production : inline_production(item)) {
+            item_set->entries.insert({
+              ParseItem(item.lhs(), inlined_production, item.step_index),
+              lookaheads
+            });
+          }
+
+          iter = item_set->entries.erase(iter);
+          continue;
         }
       }
     }
-  }
 
-  for (const auto &buffer_entry : item_set_buffer) {
-    item_set->entries[buffer_entry.first].insert_all(buffer_entry.second);
+    if (grammar.variables_to_inline.count(item.lhs())) {
+      iter = item_set->entries.erase(iter);
+      continue;
+    }
+
+    ++iter;
   }
 }
 
@@ -163,5 +294,9 @@ LookaheadSet ParseItemSetBuilder::get_first_set(const rules::Symbol &symbol) con
   return first_sets.find(symbol)->second;
 }
 
+LookaheadSet ParseItemSetBuilder::get_last_set(const rules::Symbol &symbol) const {
+  return last_sets.find(symbol)->second;
+}
+
 }  // namespace build_tables
 }  // namespace tree_sitter

data/ext/tree-sitter/tree-sitter/src/compiler/build_tables/parse_item_set_builder.h

@@ -2,8 +2,9 @@
 #define COMPILER_BUILD_TABLES_PARSE_ITEM_SET_BUILDER_H_
 
 #include "compiler/build_tables/parse_item.h"
-#include "compiler/rules/symbol.h"
+#include "compiler/rule.h"
 #include <map>
+#include <vector>
 
 namespace tree_sitter {
 
@@ -17,16 +18,21 @@ class ParseItemSetBuilder {
     ParseItem item;
     LookaheadSet lookaheads;
     bool propagates_lookaheads;
+    bool operator==(const ParseItemSetComponent &) const;
   };
 
+  const SyntaxGrammar &grammar;
   std::map<rules::Symbol, LookaheadSet> first_sets;
-  std::map<rules::Symbol::Index, std::vector<ParseItemSetComponent>> component_cache;
-  std::vector<std::pair<ParseItem, LookaheadSet>> item_set_buffer;
+  std::map<rules::Symbol, LookaheadSet> last_sets;
+  std::map<rules::Symbol::Index, std::vector<ParseItemSetComponent>> transitive_closure_component_cache;
+  std::map<ParseItem, std::vector<Production>> inlined_productions_by_original_production;
+  const std::vector<Production> &inline_production(const ParseItem &);
 
  public:
   ParseItemSetBuilder(const SyntaxGrammar &, const LexicalGrammar &);
   void apply_transitive_closure(ParseItemSet *);
   LookaheadSet get_first_set(const rules::Symbol &) const;
+  LookaheadSet get_last_set(const rules::Symbol &) const;
 };
 
 }  // namespace build_tables