tree-sitter 0.0.1 → 0.1.0

data/ext/tree-sitter/tree-sitter/src/compiler/build_tables/lex_table_builder.h

@@ -0,0 +1,57 @@
+#ifndef COMPILER_BUILD_TABLES_LEX_TABLE_BUILDER_H_
+#define COMPILER_BUILD_TABLES_LEX_TABLE_BUILDER_H_
+
+#include <memory>
+#include <vector>
+#include <unordered_map>
+#include <set>
+#include "compiler/lex_table.h"
+
+namespace tree_sitter {
+
+struct ParseTable;
+struct SyntaxGrammar;
+struct LexicalGrammar;
+
+namespace build_tables {
+
+class LookaheadSet;
+
+enum ConflictStatus {
+  DoesNotMatch = 0,
+  MatchesShorterStringWithinSeparators = 1 << 0,
+  MatchesSameString = 1 << 1,
+  MatchesLongerString = 1 << 2,
+  MatchesLongerStringWithValidNextChar = 1 << 3,
+  CannotDistinguish = (
+    MatchesShorterStringWithinSeparators |
+    MatchesSameString |
+    MatchesLongerStringWithValidNextChar
+  ),
+};
+
+class LexTableBuilder {
+ public:
+  static std::unique_ptr<LexTableBuilder> create(const SyntaxGrammar &,
+                                                 const LexicalGrammar &,
+                                                 const std::unordered_map<rules::Symbol, LookaheadSet> &,
+                                                 const std::vector<LookaheadSet> &);
+
+  struct BuildResult {
+    LexTable main_table;
+    LexTable keyword_table;
+    rules::Symbol keyword_capture_token;
+  };
+
+  BuildResult build(ParseTable *);
+
+  ConflictStatus get_conflict_status(rules::Symbol, rules::Symbol) const;
+
+ protected:
+  LexTableBuilder() = default;
+};
+
+}  // namespace build_tables
+}  // namespace tree_sitter
+
+#endif  // COMPILER_BUILD_TABLES_LEX_TABLE_BUILDER_H_

data/ext/tree-sitter/tree-sitter/src/compiler/build_tables/lookahead_set.cc

@@ -1,46 +1,120 @@
 #include "compiler/build_tables/lookahead_set.h"
 #include <set>
 #include <memory>
-#include "compiler/rules/symbol.h"
+#include "compiler/rule.h"
 
 namespace tree_sitter {
 namespace build_tables {
 
-using std::set;
-using std::make_shared;
+using std::vector;
 using rules::Symbol;
 
-LookaheadSet::LookaheadSet() : entries(nullptr) {}
+LookaheadSet::LookaheadSet() {}
 
-LookaheadSet::LookaheadSet(const set<Symbol> &symbols)
-    : entries(make_shared<set<Symbol>>(symbols)) {}
+LookaheadSet::LookaheadSet(const vector<Symbol> &symbols) {
+  for (auto symbol : symbols) insert(symbol);
+}
 
 bool LookaheadSet::empty() const {
-  return !entries.get() || entries->empty();
+  return terminal_bits.empty() && external_bits.empty() && !eof;
 }
 
 bool LookaheadSet::operator==(const LookaheadSet &other) const {
-  return *entries == *other.entries;
+  return
+    eof == other.eof &&
+    external_bits == other.external_bits &&
+    terminal_bits == other.terminal_bits;
 }
 
 bool LookaheadSet::contains(const Symbol &symbol) const {
-  return entries->find(symbol) != entries->end();
+  if (symbol == rules::END_OF_INPUT()) return eof;
+  auto &bits = symbol.is_external() ? external_bits : terminal_bits;
+  return bits.size() > static_cast<size_t>(symbol.index) && bits[symbol.index];
+}
+
+bool LookaheadSet::intersects(const LookaheadSet &other) const {
+  bool result = false;
+  for_each([&](Symbol symbol) {
+    if (other.contains(symbol)) {
+      result = true;
+      return false;
+    }
+    return true;
+  });
+  return result;
+}
+
+size_t LookaheadSet::size() const {
+  size_t result = 0;
+  for (bool bit : external_bits) if (bit) result++;
+  for (bool bit : terminal_bits) if (bit) result++;
+  if (eof) result++;
+  return result;
 }
 
 bool LookaheadSet::insert_all(const LookaheadSet &other) {
-  if (!other.entries.get())
-    return false;
-  if (!entries.get())
-    entries = make_shared<set<Symbol>>();
-  size_t previous_size = entries->size();
-  entries->insert(other.entries->begin(), other.entries->end());
-  return entries->size() > previous_size;
+  bool result = false;
+
+  if (other.eof) {
+    if (!eof) {
+      eof = true;
+      result = true;
+    }
+  }
+
+  if (other.external_bits.size() > external_bits.size()) {
+    external_bits.resize(other.external_bits.size());
+  }
+
+  auto iter = external_bits.begin();
+  auto other_iter = other.external_bits.begin();
+  auto other_end = other.external_bits.end();
+  while (other_iter != other_end) {
+    if (*other_iter && !*iter) {
+      result = true;
+      *iter = true;
+    }
+    ++iter;
+    ++other_iter;
+  }
+
+  if (other.terminal_bits.size() > terminal_bits.size()) {
+    terminal_bits.resize(other.terminal_bits.size());
+  }
+
+  iter = terminal_bits.begin();
+  other_iter = other.terminal_bits.begin();
+  other_end = other.terminal_bits.end();
+  while (other_iter != other_end) {
+    if (*other_iter && !*iter) {
+      result = true;
+      *iter = true;
+    }
+    ++iter;
+    ++other_iter;
+  }
+
+  return result;
 }
 
 bool LookaheadSet::insert(const Symbol &symbol) {
-  if (!entries.get())
-    entries = make_shared<set<Symbol>>();
-  return entries->insert(symbol).second;
+  if (symbol == rules::END_OF_INPUT()) {
+    if (!eof) {
+      eof = true;
+      return true;
+    }
+    return false;
+  }
+
+  auto &bits = symbol.is_external() ? external_bits : terminal_bits;
+  if (bits.size() <= static_cast<size_t>(symbol.index)) {
+    bits.resize(symbol.index + 1);
+  }
+  if (!bits[symbol.index]) {
+    bits[symbol.index] = true;
+    return true;
+  }
+  return false;
 }
 
 }  // namespace build_tables

data/ext/tree-sitter/tree-sitter/src/compiler/build_tables/lookahead_set.h

@@ -1,25 +1,110 @@
 #ifndef COMPILER_BUILD_TABLES_LOOKAHEAD_SET_H_
 #define COMPILER_BUILD_TABLES_LOOKAHEAD_SET_H_
 
-#include <set>
-#include <memory>
-#include "compiler/rules/symbol.h"
+#include <vector>
+#include "compiler/rule.h"
 
 namespace tree_sitter {
 namespace build_tables {
 
 class LookaheadSet {
+  std::vector<bool> terminal_bits;
+  std::vector<bool> external_bits;
+  bool eof = false;
+
  public:
   LookaheadSet();
-  explicit LookaheadSet(const std::set<rules::Symbol> &);
+  explicit LookaheadSet(const std::vector<rules::Symbol> &);
 
   bool empty() const;
+  size_t size() const;
   bool operator==(const LookaheadSet &) const;
   bool contains(const rules::Symbol &) const;
   bool insert_all(const LookaheadSet &);
   bool insert(const rules::Symbol &);
+  bool intersects(const LookaheadSet &) const;
+
+  template <typename Callback>
+  void for_each(const Callback &callback) const {
+    for (auto begin = external_bits.begin(),
+         end = external_bits.end(),
+         iter = begin;
+         iter != end;
+         ++iter) {
+      if (*iter) {
+        if (!callback(rules::Symbol::external(iter - begin))) return;
+      }
+    }
+
+    if (eof) {
+      if (!callback(rules::END_OF_INPUT())) return;
+    }
+
+    for (auto begin = terminal_bits.begin(),
+         end = terminal_bits.end(),
+         iter = begin;
+         iter != end;
+         ++iter) {
+      if (*iter) {
+        if (!callback(rules::Symbol::terminal(iter - begin))) return;
+      }
+    }
+  }
+
+  template <typename Callback>
+  void for_each_difference(const LookaheadSet &other, const Callback &callback) const {
+    auto end = external_bits.end();
+    auto begin = external_bits.begin();
+    auto other_end = other.external_bits.end();
+    auto other_begin = other.external_bits.begin();
+    auto common_end = other.external_bits.size() < external_bits.size() ?
+      begin + other.external_bits.size() :
+      end;
+    auto iter = begin;
+    auto other_iter = other_begin;
+    for (; iter != common_end; ++iter, ++other_iter) {
+      if (*iter) {
+        if (!*other_iter && !callback(true, rules::Symbol::external(iter - begin))) return;
+      } else if (*other_iter) {
+        if (!callback(false, rules::Symbol::external(iter - begin))) return;
+      }
+    }
+    for (; iter < end; ++iter) {
+      if (*iter && !callback(true, rules::Symbol::external(iter - begin))) return;
+    }
+    for (; other_iter < other_end; ++other_iter) {
+      if (*other_iter && !callback(false, rules::Symbol::external(other_iter - other_begin))) return;
+    }
+
+    if (eof) {
+      if (!other.eof && !callback(true, rules::END_OF_INPUT())) return;
+    } else if (other.eof) {
+      if (!callback(false, rules::END_OF_INPUT())) return;
+    }
 
-  std::shared_ptr<std::set<rules::Symbol>> entries;
+    end = terminal_bits.end();
+    begin = terminal_bits.begin();
+    other_end = other.terminal_bits.end();
+    other_begin = other.terminal_bits.begin();
+    common_end = other.terminal_bits.size() < terminal_bits.size() ?
+      begin + other.terminal_bits.size() :
+      end;
+    iter = begin;
+    other_iter = other_begin;
+    for (; iter != common_end; ++iter, ++other_iter) {
+      if (*iter) {
+        if (!*other_iter && !callback(true, rules::Symbol::terminal(iter - begin))) return;
+      } else if (*other_iter) {
+        if (!callback(false, rules::Symbol::terminal(iter - begin))) return;
+      }
+    }
+    for (; iter < end; ++iter) {
+      if (*iter && !callback(true, rules::Symbol::terminal(iter - begin))) return;
+    }
+    for (; other_iter < other_end; ++other_iter) {
+      if (*other_iter && !callback(false, rules::Symbol::terminal(other_iter - other_begin))) return;
+    }
+  }
 };
 
 }  // namespace build_tables

data/ext/tree-sitter/tree-sitter/src/compiler/build_tables/parse_item.cc

@@ -1,7 +1,7 @@
 #include "compiler/build_tables/parse_item.h"
 #include <string>
 #include "compiler/syntax_grammar.h"
-#include "compiler/rules/built_in_symbols.h"
+#include "compiler/rule.h"
 #include "compiler/util/hash_combine.h"
 
 namespace tree_sitter {
@@ -24,24 +24,54 @@ ParseItem::ParseItem(const Symbol &lhs, const Production &production,
       step_index(step_index) {}
 
 bool ParseItem::operator==(const ParseItem &other) const {
-  return ((variable_index == other.variable_index) &&
-          (step_index == other.step_index) && (production == other.production));
+  if (step_index != other.step_index) return false;
+  if (variable_index != other.variable_index) return false;
+  if (production->size() != other.production->size()) return false;
+  for (size_t i = 0; i < step_index; i++) {
+    if (production->at(i).alias != other.production->at(i).alias) return false;
+  }
+  if (is_done()) {
+    if (!production->empty()) {
+      if (production->back().precedence != other.production->back().precedence) return false;
+      if (production->back().associativity != other.production->back().associativity) return false;
+    }
+  } else {
+    for (size_t i = step_index, n = production->size(); i < n; i++) {
+      if (production->at(i) != other.production->at(i)) return false;
+    }
+  }
+  return true;
 }
 
 bool ParseItem::operator<(const ParseItem &other) const {
-  if (step_index < other.step_index)
-    return true;
-  if (step_index > other.step_index)
-    return false;
-  if (variable_index < other.variable_index)
-    return true;
-  if (variable_index > other.variable_index)
-    return false;
-  return production < other.production;
+  if (step_index < other.step_index) return true;
+  if (other.step_index < step_index) return false;
+  if (variable_index < other.variable_index) return true;
+  if (other.variable_index < variable_index) return false;
+  if (production->size() < other.production->size()) return true;
+  if (other.production->size() < production->size()) return false;
+  for (size_t i = 0; i < step_index; i++) {
+    if (production->at(i).alias < other.production->at(i).alias) return true;
+    if (other.production->at(i).alias < production->at(i).alias) return false;
+  }
+  if (is_done()) {
+    if (!production->empty()) {
+      if (production->back().precedence < other.production->back().precedence) return true;
+      if (other.production->back().precedence < production->back().precedence) return false;
+      if (production->back().associativity < other.production->back().associativity) return true;
+      if (other.production->back().associativity < production->back().associativity) return false;
+    }
+  } else {
+    for (size_t i = step_index, n = production->size(); i < n; i++) {
+      if (production->at(i) < other.production->at(i)) return true;
+      if (other.production->at(i) < production->at(i)) return false;
+    }
+  }
+  return false;
 }
 
 Symbol ParseItem::lhs() const {
-  return Symbol(variable_index, Symbol::NonTerminal);
+  return Symbol{variable_index, Symbol::NonTerminal};
 }
 
 bool ParseItem::is_done() const {
@@ -60,6 +90,10 @@ int ParseItem::precedence() const {
   }
 }
 
+int ParseItem::dynamic_precedence() const {
+  return production->dynamic_precedence;
+}
+
 rules::Associativity ParseItem::associativity() const {
   if (is_done()) {
     if (production->empty()) {
@@ -79,11 +113,6 @@ Symbol ParseItem::next_symbol() const {
     return production->at(step_index).symbol;
 }
 
-ParseItemSet::ParseItemSet() {}
-
-ParseItemSet::ParseItemSet(const map<ParseItem, LookaheadSet> &entries)
-    : entries(entries) {}
-
 bool ParseItemSet::operator==(const ParseItemSet &other) const {
   return entries == other.entries;
 }
@@ -93,13 +122,12 @@ size_t ParseItemSet::unfinished_item_signature() const {
   ParseItem previous_item;
   for (auto &pair : entries) {
     const ParseItem &item = pair.first;
-    if (item.step_index < item.production->size()) {
-      if (item.variable_index != previous_item.variable_index &&
-          item.step_index != previous_item.step_index) {
-        hash_combine(&result, item.variable_index);
-        hash_combine(&result, item.step_index);
-        previous_item = item;
-      }
+    if (item.step_index < item.production->size() &&
+        (item.variable_index != previous_item.variable_index ||
+         item.step_index != previous_item.step_index)) {
+      hash_combine(&result, item.variable_index);
+      hash_combine(&result, item.step_index);
+      previous_item = item;
     }
   }
   return result;
@@ -125,7 +153,25 @@ struct hash<ParseItem> {
     size_t result = 0;
     hash_combine(&result, item.variable_index);
     hash_combine(&result, item.step_index);
-    hash_combine(&result, item.production);
+    hash_combine(&result, item.production->dynamic_precedence);
+    hash_combine(&result, item.production->size());
+    for (size_t i = 0; i < item.step_index; i++) {
+      hash_combine(&result, item.production->at(i).alias.value);
+      hash_combine(&result, item.production->at(i).alias.is_named);
+    }
+    if (item.is_done()) {
+      if (!item.production->empty()) {
+        hash_combine(&result, item.production->back().precedence);
+        hash_combine<unsigned>(&result, item.production->back().associativity);
+      }
+    } else {
+      for (size_t i = item.step_index, n = item.production->size(); i < n; i++) {
+        auto &step = item.production->at(i);
+        hash_combine(&result, step.symbol);
+        hash_combine(&result, step.precedence);
+        hash_combine<unsigned>(&result, step.associativity);
+      }
+    }
     return result;
   }
 };
@@ -138,9 +184,11 @@ size_t hash<ParseItemSet>::operator()(const ParseItemSet &item_set) const {
     const auto &lookahead_set = pair.second;
 
     hash_combine(&result, item);
-    hash_combine(&result, lookahead_set.entries->size());
-    for (auto index : *pair.second.entries)
-      hash_combine(&result, index);
+    hash_combine(&result, lookahead_set.size());
+    lookahead_set.for_each([&result](Symbol symbol) {
+      hash_combine(&result, symbol);
+      return true;
+    });
   }
   return result;
 }