tree-sitter 0.0.1 → 0.1.0

data/ext/tree-sitter/tree-sitter/src/compiler/prepare_grammar/extract_tokens.h

@@ -4,14 +4,15 @@
 #include <tuple>
 #include "compiler/compile_error.h"
 #include "compiler/lexical_grammar.h"
-#include "compiler/prepare_grammar/initial_syntax_grammar.h"
 #include "compiler/prepare_grammar/interned_grammar.h"
+#include "compiler/prepare_grammar/initial_syntax_grammar.h"
 
 namespace tree_sitter {
 namespace prepare_grammar {
 
 std::tuple<InitialSyntaxGrammar, LexicalGrammar, CompileError> extract_tokens(
-  const InternedGrammar &);
+  const InternedGrammar &
+);
 
 }  // namespace prepare_grammar
 }  // namespace tree_sitter

data/ext/tree-sitter/tree-sitter/src/compiler/prepare_grammar/flatten_grammar.cc

@@ -1,75 +1,97 @@
 #include "compiler/prepare_grammar/flatten_grammar.h"
-#include <vector>
 #include <algorithm>
+#include <cassert>
+#include <cmath>
+#include <string>
+#include <vector>
 #include "compiler/prepare_grammar/extract_choices.h"
 #include "compiler/prepare_grammar/initial_syntax_grammar.h"
-#include "compiler/rules/visitor.h"
-#include "compiler/rules/seq.h"
-#include "compiler/rules/symbol.h"
-#include "compiler/rules/metadata.h"
-#include "compiler/rules/built_in_symbols.h"
+#include "compiler/grammar.h"
+#include "compiler/rule.h"
 
 namespace tree_sitter {
 namespace prepare_grammar {
 
 using std::find;
 using std::pair;
+using std::string;
 using std::vector;
+using rules::Rule;
+using rules::Symbol;
 
-class FlattenRule : public rules::RuleFn<void> {
+class FlattenRule {
  private:
   vector<int> precedence_stack;
   vector<rules::Associativity> associativity_stack;
-  int last_precedence;
-  rules::Associativity last_associativity;
+  vector<rules::Alias> alias_stack;
   Production production;
 
-  void apply_to(const rules::Symbol *sym) {
-    production.push_back(ProductionStep(*sym, precedence_stack.back(),
-                                        associativity_stack.back()));
-  }
+  void apply(const Rule &rule, bool at_end) {
+    rule.match(
+      [&](const rules::Symbol &symbol) {
+        production.steps.push_back(ProductionStep{
+          symbol,
+          precedence_stack.back(),
+          associativity_stack.back(),
+          alias_stack.back()
+        });
+      },
+
+      [&](const rules::Metadata &metadata) {
+        if (metadata.params.has_precedence) {
+          precedence_stack.push_back(metadata.params.precedence);
+        }
 
-  void apply_to(const rules::Metadata *metadata) {
-    if (metadata->params.has_precedence)
-      precedence_stack.push_back(metadata->params.precedence);
-    if (metadata->params.has_associativity)
-      associativity_stack.push_back(metadata->params.associativity);
+        if (metadata.params.has_associativity) {
+          associativity_stack.push_back(metadata.params.associativity);
+        }
 
-    apply(metadata->rule);
+        if (!metadata.params.alias.value.empty()) {
+          alias_stack.push_back(metadata.params.alias);
+        }
 
-    if (metadata->params.has_precedence) {
-      last_precedence = precedence_stack.back();
-      precedence_stack.pop_back();
-      production.back().precedence = precedence_stack.back();
-    }
+        if (abs(metadata.params.dynamic_precedence) > abs(production.dynamic_precedence)) {
+          production.dynamic_precedence = metadata.params.dynamic_precedence;
+        }
 
-    if (metadata->params.has_associativity) {
-      last_associativity = associativity_stack.back();
-      associativity_stack.pop_back();
-      production.back().associativity = associativity_stack.back();
-    }
-  }
+        apply(*metadata.rule, at_end);
+
+        if (metadata.params.has_precedence) {
+          precedence_stack.pop_back();
+          if (!at_end) production.back().precedence = precedence_stack.back();
+        }
+
+        if (metadata.params.has_associativity) {
+          associativity_stack.pop_back();
+          if (!at_end) production.back().associativity = associativity_stack.back();
+        }
+
+        if (!metadata.params.alias.value.empty()) {
+          alias_stack.pop_back();
+        }
+      },
+
+      [&](const rules::Seq &sequence) {
+        apply(*sequence.left, false);
+        apply(*sequence.right, at_end);
+      },
+
+      [&](const rules::Blank &blank) {},
 
-  void apply_to(const rules::Seq *seq) {
-    apply(seq->left);
-    last_precedence = 0;
-    last_associativity = rules::AssociativityNone;
-    apply(seq->right);
+      [&](auto) {
+        assert(!"Unexpected rule type");
+      }
+    );
   }
 
  public:
-  FlattenRule()
-      : precedence_stack({ 0 }),
-        associativity_stack({ rules::AssociativityNone }),
-        last_precedence(0),
-        last_associativity(rules::AssociativityNone) {}
-
-  Production flatten(const rule_ptr &rule) {
-    apply(rule);
-    if (!production.empty()) {
-      production.back().precedence = last_precedence;
-      production.back().associativity = last_associativity;
-    }
+  FlattenRule() :
+    precedence_stack({0}),
+    associativity_stack({rules::AssociativityNone}),
+    alias_stack({rules::Alias{}}) {}
+
+  Production flatten(const Rule &rule) {
+    apply(rule, true);
     return production;
   }
 };
@@ -77,7 +99,7 @@ class FlattenRule : public rules::RuleFn<void> {
 SyntaxVariable flatten_rule(const Variable &variable) {
   vector<Production> productions;
 
-  for (const rule_ptr &rule_component : extract_choices(variable.rule)) {
+  for (const Rule &rule_component : extract_choices(variable.rule)) {
     Production production = FlattenRule().flatten(rule_component);
     auto end = productions.end();
     if (find(productions.begin(), end, production) == end) {
@@ -85,36 +107,58 @@ SyntaxVariable flatten_rule(const Variable &variable) {
     }
   }
 
-  return SyntaxVariable(variable.name, variable.type, productions);
+  return SyntaxVariable{variable.name, variable.type, productions};
+}
+
+static bool variable_is_used(const SyntaxGrammar &grammar, Symbol::Index symbol_index) {
+  for (const SyntaxVariable &variable : grammar.variables) {
+    for (const Production &production : variable.productions) {
+      for (const auto &step : production) {
+        if (step.symbol == Symbol::non_terminal(symbol_index)) {
+          return true;
+        }
+      }
+    }
+  }
+  return false;
 }
 
 pair<SyntaxGrammar, CompileError> flatten_grammar(const InitialSyntaxGrammar &grammar) {
   SyntaxGrammar result;
-  result.expected_conflicts = grammar.expected_conflicts;
-  result.extra_tokens = grammar.extra_tokens;
   result.external_tokens = grammar.external_tokens;
+  result.variables_to_inline = grammar.variables_to_inline;
 
-  bool is_start = true;
-  for (const Variable &variable : grammar.variables) {
-    SyntaxVariable syntax_variable = flatten_rule(variable);
-
-    if (!is_start) {
-      for (const Production &production : syntax_variable.productions) {
-        if (production.empty()) {
-          return {
-            result,
-            CompileError(
-              TSCompileErrorTypeEpsilonRule,
-              "The rule `" + variable.name + "` matches the empty string.\n" +
-              "Tree-sitter currently does not support syntactic rules that match the empty string.\n"
-            )
-          };
-        }
+  for (const auto &expected_conflict : grammar.expected_conflicts) {
+    result.expected_conflicts.insert({
+      expected_conflict.begin(),
+      expected_conflict.end(),
+    });
+  }
+
+  for (const rules::Symbol &extra_token : grammar.extra_tokens) {
+    result.extra_tokens.insert(extra_token);
+  }
+
+  for (const auto &variable : grammar.variables) {
+    result.variables.push_back(flatten_rule(variable));
+  }
+
+  Symbol::Index i = 0;
+  for (const auto &variable : result.variables) {
+    for (const Production &production : variable.productions) {
+      if (production.empty() && variable_is_used(result, i)) {
+        return {
+          result,
+          CompileError(
+            TSCompileErrorTypeEpsilonRule,
+            "The rule `" + variable.name + "` matches the empty string.\n\n" +
+            "Tree-sitter does not support syntactic rules that match the empty string\n"
+            "unless they are used only as the grammar's start rule.\n"
+          )
+        };
       }
     }
-
-    result.variables.push_back(syntax_variable);
-    is_start = false;
+    i++;
   }
 
   return {result, CompileError::none()};

data/ext/tree-sitter/tree-sitter/src/compiler/prepare_grammar/flatten_grammar.h

@@ -4,13 +4,13 @@
 #include <string>
 #include "tree_sitter/compiler.h"
 #include "compiler/compile_error.h"
+#include "compiler/grammar.h"
+#include "compiler/prepare_grammar/initial_syntax_grammar.h"
 #include "compiler/syntax_grammar.h"
 
 namespace tree_sitter {
 namespace prepare_grammar {
 
-struct InitialSyntaxGrammar;
-
 SyntaxVariable flatten_rule(const Variable &variable);
 std::pair<SyntaxGrammar, CompileError> flatten_grammar(const InitialSyntaxGrammar &);
 

data/ext/tree-sitter/tree-sitter/src/compiler/prepare_grammar/initial_syntax_grammar.h

@@ -4,9 +4,9 @@
 #include <set>
 #include <vector>
 #include "tree_sitter/compiler.h"
-#include "compiler/rules/symbol.h"
+#include "compiler/grammar.h"
 #include "compiler/syntax_grammar.h"
-#include "compiler/variable.h"
+#include "compiler/rule.h"
 
 namespace tree_sitter {
 namespace prepare_grammar {
@@ -14,8 +14,9 @@ namespace prepare_grammar {
 struct InitialSyntaxGrammar {
   std::vector<Variable> variables;
   std::set<rules::Symbol> extra_tokens;
-  std::set<ConflictSet> expected_conflicts;
+  std::set<std::set<rules::Symbol>> expected_conflicts;
   std::vector<ExternalToken> external_tokens;
+  std::set<rules::Symbol> variables_to_inline;
 };
 
 }  // namespace prepare_grammar

data/ext/tree-sitter/tree-sitter/src/compiler/prepare_grammar/intern_symbols.cc

@@ -1,14 +1,11 @@
 #include "compiler/prepare_grammar/intern_symbols.h"
 #include <memory>
 #include <vector>
+#include <cassert>
 #include <set>
 #include "tree_sitter/compiler.h"
 #include "compiler/grammar.h"
-#include "compiler/rules/visitor.h"
-#include "compiler/rules/blank.h"
-#include "compiler/rules/named_symbol.h"
-#include "compiler/rules/symbol.h"
-#include "compiler/rules/built_in_symbols.h"
+#include "compiler/rule.h"
 
 namespace tree_sitter {
 namespace prepare_grammar {
@@ -17,34 +14,75 @@ using std::string;
 using std::vector;
 using std::set;
 using std::pair;
-using std::make_shared;
 using rules::Symbol;
+using rules::Rule;
 
-class SymbolInterner : public rules::IdentityRuleFn {
-  using rules::IdentityRuleFn::apply_to;
+class SymbolInterner {
+ public:
+  Rule apply(const Rule &rule) {
+    return rule.match(
+      [&](const rules::Blank &blank) -> Rule {
+        return blank;
+      },
+
+      [&](const rules::NamedSymbol &symbol) {
+        return intern_symbol(symbol);
+      },
+
+      [&](const rules::String &string) {
+        return string;
+      },
+
+      [&](const rules::Pattern &pattern) {
+        return pattern;
+      },
+
+      [&](const rules::Choice &choice) {
+        vector<rules::Rule> elements;
+        for (const auto &element : choice.elements) {
+          elements.push_back(apply(element));
+        }
+        return rules::Choice{elements};
+      },
+
+      [&](const rules::Seq &sequence) {
+        return rules::Seq{apply(*sequence.left), apply(*sequence.right)};
+      },
+
+      [&](const rules::Repeat &repeat) {
+        return rules::Repeat{apply(*repeat.rule)};
+      },
+
+      [&](const rules::Metadata &metadata) {
+        return rules::Metadata{apply(*metadata.rule), metadata.params};
+      },
+
+      [](auto) {
+        assert(!"Unexpected rule type");
+        return rules::Blank{};
+      }
+    );
+  }
 
-  rule_ptr apply_to(const rules::NamedSymbol *rule) {
-    auto result = symbol_for_rule_name(rule->name);
-    if (!result.get()) {
-      missing_rule_name = rule->name;
-      return rules::Blank::build();
+  Symbol intern_symbol(rules::NamedSymbol named_symbol) {
+    for (size_t i = 0; i < grammar.variables.size(); i++) {
+      if (grammar.variables[i].name == named_symbol.value) {
+        return Symbol::non_terminal(i);
+      }
     }
-    return result;
-  }
 
- public:
-  std::shared_ptr<rules::Symbol> symbol_for_rule_name(string rule_name) {
-    for (size_t i = 0; i < grammar.rules.size(); i++)
-      if (grammar.rules[i].first == rule_name)
-        return make_shared<Symbol>(i, Symbol::NonTerminal);
-    for (size_t i = 0; i < grammar.external_tokens.size(); i++)
-      if (grammar.external_tokens[i] == rule_name)
-        return make_shared<rules::Symbol>(i, Symbol::External);
-    return nullptr;
+    for (size_t i = 0; i < grammar.external_tokens.size(); i++) {
+      if (grammar.external_tokens[i] == named_symbol) {
+        return Symbol::external(i);
+      }
+    }
+
+    missing_rule_name = named_symbol.value;
+    return rules::NONE();
   }
 
-  explicit SymbolInterner(const Grammar &grammar) : grammar(grammar) {}
-  const Grammar grammar;
+  explicit SymbolInterner(const InputGrammar &grammar) : grammar(grammar) {}
+  const InputGrammar &grammar;
   string missing_rule_name;
 };
 
@@ -53,57 +91,82 @@ CompileError missing_rule_error(string rule_name) {
                       "Undefined rule '" + rule_name + "'");
 }
 
-pair<InternedGrammar, CompileError> intern_symbols(const Grammar &grammar) {
+pair<InternedGrammar, CompileError> intern_symbols(const InputGrammar &grammar) {
   InternedGrammar result;
 
-  for (auto &external_token_name : grammar.external_tokens) {
-    Symbol corresponding_internal_token = rules::NONE();
-    for (size_t i = 0, n = grammar.rules.size(); i < n; i++) {
-      if (grammar.rules[i].first == external_token_name) {
-        corresponding_internal_token = Symbol(i, Symbol::NonTerminal);
-        break;
-      }
+  SymbolInterner interner(grammar);
+
+  for (const Rule &external_token : grammar.external_tokens) {
+    string external_token_name;
+    VariableType external_token_type = VariableTypeAnonymous;
+    external_token.match(
+      [&](rules::NamedSymbol named_symbol) {
+        external_token_name = named_symbol.value;
+        if (external_token_name[0] == '_') {
+          external_token_type = VariableTypeHidden;
+        } else {
+          external_token_type =VariableTypeNamed;
+        }
+      },
+      [](auto rule) {}
+    );
+
+    auto new_rule = interner.apply(external_token);
+    if (!interner.missing_rule_name.empty()) {
+      return { result, missing_rule_error(interner.missing_rule_name) };
     }
 
-    result.external_tokens.push_back(ExternalToken{
+    result.external_tokens.push_back(Variable{
       external_token_name,
-      external_token_name[0] == '_' ? VariableTypeHidden : VariableTypeNamed,
-      corresponding_internal_token
+      external_token_type,
+      new_rule,
     });
   }
 
-  SymbolInterner interner(grammar);
+  if (grammar.variables[0].name[0] == '_') {
+    return {result, CompileError(TSCompileErrorTypeInvalidRuleName, "A grammar's start rule must be visible.")};
+  }
 
-  for (auto &pair : grammar.rules) {
-    auto new_rule = interner.apply(pair.second);
-    if (!interner.missing_rule_name.empty())
+  for (auto &variable : grammar.variables) {
+    auto new_rule = interner.apply(variable.rule);
+    if (!interner.missing_rule_name.empty()) {
       return { result, missing_rule_error(interner.missing_rule_name) };
+    }
 
     result.variables.push_back(Variable{
-      pair.first,
-      pair.first[0] == '_' ? VariableTypeHidden : VariableTypeNamed,
+      variable.name,
+      variable.name[0] == '_' ? VariableTypeHidden : VariableTypeNamed,
       new_rule
     });
   }
 
   for (auto &rule : grammar.extra_tokens) {
     auto new_rule = interner.apply(rule);
-    if (!interner.missing_rule_name.empty())
+    if (!interner.missing_rule_name.empty()) {
       return { result, missing_rule_error(interner.missing_rule_name) };
+    }
     result.extra_tokens.push_back(new_rule);
   }
 
-  for (auto &names : grammar.expected_conflicts) {
+  for (auto &expected_conflict : grammar.expected_conflicts) {
     set<rules::Symbol> entry;
-    for (auto &name : names) {
-      auto symbol = interner.symbol_for_rule_name(name);
-      if (symbol.get())
-        entry.insert(*symbol);
+    for (auto &named_symbol : expected_conflict) {
+      auto symbol = interner.intern_symbol(named_symbol);
+      if (symbol != rules::NONE()) {
+        entry.insert(symbol);
+      }
     }
     result.expected_conflicts.insert(entry);
   }
 
-  return { result, CompileError::none() };
+  for (auto &named_symbol : grammar.variables_to_inline) {
+    auto symbol = interner.intern_symbol(named_symbol);
+    if (symbol != rules::NONE()) {
+      result.variables_to_inline.insert(symbol);
+    }
+  }
+
+  return {result, CompileError::none()};
 }
 
 }  // namespace prepare_grammar