tree-sitter 0.0.1 → 0.1.0

data/ext/tree-sitter/tree-sitter/src/compiler/prepare_grammar/intern_symbols.h

@@ -8,11 +8,11 @@
 
 namespace tree_sitter {
 
-struct Grammar;
+struct InputGrammar;
 
 namespace prepare_grammar {
 
-std::pair<InternedGrammar, CompileError> intern_symbols(const Grammar &);
+std::pair<InternedGrammar, CompileError> intern_symbols(const InputGrammar &);
 
 }  // namespace prepare_grammar
 }  // namespace tree_sitter

data/ext/tree-sitter/tree-sitter/src/compiler/prepare_grammar/interned_grammar.h

@@ -4,18 +4,19 @@
 #include <set>
 #include <vector>
 #include "tree_sitter/compiler.h"
-#include "compiler/rules/symbol.h"
-#include "compiler/syntax_grammar.h"
-#include "compiler/variable.h"
+#include "compiler/grammar.h"
+#include "compiler/rule.h"
 
 namespace tree_sitter {
 namespace prepare_grammar {
 
 struct InternedGrammar {
   std::vector<Variable> variables;
-  std::vector<rule_ptr> extra_tokens;
-  std::set<ConflictSet> expected_conflicts;
-  std::vector<ExternalToken> external_tokens;
+  std::vector<rules::Rule> extra_tokens;
+  std::set<std::set<rules::Symbol>> expected_conflicts;
+  std::vector<Variable> external_tokens;
+  std::set<rules::Symbol> blank_external_tokens;
+  std::set<rules::Symbol> variables_to_inline;
 };
 
 }  // namespace prepare_grammar

data/ext/tree-sitter/tree-sitter/src/compiler/prepare_grammar/normalize_rules.cc

@@ -1,15 +1,17 @@
 #include "compiler/prepare_grammar/normalize_rules.h"
 #include "compiler/prepare_grammar/extract_choices.h"
-#include "compiler/rules/choice.h"
 
 namespace tree_sitter {
 namespace prepare_grammar {
 
+using std::vector;
+using rules::Rule;
+
 LexicalGrammar normalize_rules(const LexicalGrammar &input_grammar) {
   LexicalGrammar result(input_grammar);
 
-  for (Variable &variable : result.variables) {
-    variable.rule = rules::Choice::build(extract_choices(variable.rule));
+  for (LexicalVariable &variable : result.variables) {
+    variable.rule = Rule::choice(extract_choices(variable.rule));
   }
 
   return result;

data/ext/tree-sitter/tree-sitter/src/compiler/prepare_grammar/parse_regex.cc

@@ -2,11 +2,7 @@
 #include <string>
 #include <utility>
 #include <vector>
-#include "compiler/rules/choice.h"
-#include "compiler/rules/seq.h"
-#include "compiler/rules/repeat.h"
-#include "compiler/rules/character_set.h"
-#include "compiler/rules/blank.h"
+#include "compiler/rule.h"
 #include "compiler/util/string_helpers.h"
 #include "utf8proc.h"
 
@@ -16,12 +12,9 @@ namespace prepare_grammar {
 using std::string;
 using std::vector;
 using std::pair;
-using std::make_shared;
 using rules::CharacterSet;
-using rules::Seq;
 using rules::Blank;
-using rules::Choice;
-using rules::Repeat;
+using rules::Rule;
 
 class PatternParser {
  public:
@@ -32,103 +25,118 @@ class PatternParser {
     next();
   }
 
-  pair<rule_ptr, CompileError> rule(bool nested) {
-    vector<rule_ptr> choices = {};
+  pair<Rule, CompileError> rule(bool nested) {
+    vector<Rule> choices;
     do {
       if (!choices.empty()) {
-        if (peek() == '|')
+        if (peek() == '|') {
           next();
-        else
+        } else {
           break;
+        }
       }
       auto pair = term(nested);
-      if (pair.second.type)
-        return { Blank::build(), pair.second };
+      if (pair.second.type) {
+        return {Blank{}, pair.second };
+      }
       choices.push_back(pair.first);
     } while (has_more_input());
-    auto rule =
-      (choices.size() > 1) ? make_shared<Choice>(choices) : choices.front();
-    return { rule, CompileError::none() };
+    return {Rule::choice(choices), CompileError::none()};
   }
 
  private:
-  pair<rule_ptr, CompileError> term(bool nested) {
-    rule_ptr result = Blank::build();
+  pair<Rule, CompileError> term(bool nested) {
+    Rule result;
     do {
       if (peek() == '|')
         break;
       if (nested && peek() == ')')
         break;
       auto pair = factor();
-      if (pair.second.type)
-        return { Blank::build(), pair.second };
-      result = Seq::build({ result, pair.first });
+      if (pair.second) {
+        return {Blank{}, pair.second};
+      }
+      result = Rule::seq({result, pair.first});
     } while (has_more_input());
     return { result, CompileError::none() };
   }
 
-  pair<rule_ptr, CompileError> factor() {
+  pair<Rule, CompileError> factor() {
     auto pair = atom();
-    if (pair.second.type)
-      return { Blank::build(), pair.second };
-    rule_ptr result = pair.first;
+    if (pair.second.type) {
+      return {Blank{}, pair.second};
+    }
+
+    Rule result = pair.first;
     if (has_more_input()) {
       switch (peek()) {
         case '*':
           next();
-          result = Choice::build({ Repeat::build(result), Blank::build() });
+          result = Rule::choice({
+            Rule::repeat(result),
+            Blank{}
+          });
           break;
         case '+':
           next();
-          result = Repeat::build(result);
+          result = Rule::repeat(result);
           break;
         case '?':
           next();
-          result = Choice::build({ result, Blank::build() });
+          result = Rule::choice({result, Blank{}});
           break;
       }
     }
-    return { result, CompileError::none() };
+
+    return {result, CompileError::none()};
   }
 
-  pair<rule_ptr, CompileError> atom() {
+  pair<Rule, CompileError> atom() {
     switch (peek()) {
       case '(': {
         next();
         auto pair = rule(true);
-        if (pair.second.type)
-          return { Blank::build(), pair.second };
-        if (peek() != ')')
+        if (pair.second.type) {
+          return {Blank{}, pair.second};
+        }
+        if (peek() != ')') {
           return error("unmatched open paren");
+        }
         next();
-        return { pair.first, CompileError::none() };
+        return {pair.first, CompileError::none()};
       }
+
       case '[': {
         next();
         auto pair = char_set();
-        if (pair.second.type)
-          return { Blank::build(), pair.second };
-        if (peek() != ']')
+        if (pair.second.type) {
+          return {Blank{}, pair.second};
+        }
+        if (peek() != ']') {
           return error("unmatched open square bracket");
+        }
         next();
-        return { pair.first.copy(), CompileError::none() };
+        return {pair.first, CompileError::none()};
       }
+
       case ')': {
         return error("unmatched close paren");
       }
+
       case ']': {
         return error("unmatched close square bracket");
       }
+
       case '.': {
         next();
-        return { CharacterSet().include_all().exclude('\n').copy(),
-                 CompileError::none() };
+        return {
+          CharacterSet().include_all().exclude('\n'),
+          CompileError::none()
+        };
       }
+
       default: {
-        auto pair = single_char();
-        if (pair.second.type)
-          return { Blank::build(), pair.second };
-        return { pair.first.copy(), CompileError::none() };
+        return {single_char(), CompileError::none()};
       }
     }
   }
@@ -143,44 +151,50 @@ class PatternParser {
     }
 
     while (has_more_input() && (peek() != ']')) {
-      auto pair = single_char();
-      if (pair.second.type)
-        return { CharacterSet(), pair.second };
+      auto characters = single_char();
+
+      if (peek() == '-') {
+        next();
+        if (!characters.includes_all && characters.included_chars.size() == 1 && peek() != ']') {
+          auto next_characters = single_char();
+          if (!next_characters.includes_all && next_characters.included_chars.size() == 1) {
+            characters.include(
+              *characters.included_chars.begin(),
+              *next_characters.included_chars.begin()
+            );
+          } else {
+            characters.include('-');
+            characters.add_set(next_characters);
+          }
+        } else {
+          characters.include('-');
+        }
+      }
+
       if (is_affirmative)
-        result.add_set(pair.first);
+        result.add_set(characters);
       else
-        result.remove_set(pair.first);
+        result.remove_set(characters);
     }
 
     return { result, CompileError::none() };
   }
 
-  pair<CharacterSet, CompileError> single_char() {
+  CharacterSet single_char() {
     CharacterSet value;
-    switch (peek()) {
-      case '\\':
-        next();
-        value = escaped_char(peek());
-        next();
-        break;
-      default:
-        uint32_t first_char = peek();
-        next();
-        if (peek() == '-') {
-          next();
-          value = CharacterSet().include(first_char, peek());
-          next();
-        } else {
-          value = CharacterSet().include(first_char);
-        }
+    if (peek() == '\\') {
+      next();
+      value = escaped_char(peek());
+      next();
+    } else {
+      value = CharacterSet().include(peek());
+      next();
     }
-    return { value, CompileError::none() };
+    return value;
   }
 
   CharacterSet escaped_char(uint32_t value) {
     switch (value) {
-      case 'a':
-        return CharacterSet().include('a', 'z').include('A', 'Z');
       case 'w':
         return CharacterSet()
           .include('a', 'z')
@@ -199,8 +213,11 @@ class PatternParser {
       case 'D':
         return CharacterSet().include_all().exclude('0', '9');
       case 's':
-        return CharacterSet().include(' ').include('\t').include('\n').include(
-          '\r');
+        return CharacterSet()
+          .include(' ')
+          .include('\t')
+          .include('\n')
+          .include('\r');
       case 'S':
         return CharacterSet()
           .include_all()
@@ -208,6 +225,8 @@ class PatternParser {
           .exclude('\t')
           .exclude('\n')
           .exclude('\r');
+      case '0':
+        return CharacterSet().include('\0');
       case 't':
         return CharacterSet().include('\t');
       case 'n':
@@ -234,8 +253,8 @@ class PatternParser {
     return lookahead && iter <= end;
   }
 
-  pair<rule_ptr, CompileError> error(string msg) {
-    return { Blank::build(), CompileError(TSCompileErrorTypeInvalidRegex, msg) };
+  pair<Rule, CompileError> error(string msg) {
+    return { Blank{}, CompileError(TSCompileErrorTypeInvalidRegex, msg) };
   }
 
   string input;
@@ -244,7 +263,7 @@ class PatternParser {
   int32_t lookahead;
 };
 
-pair<rule_ptr, CompileError> parse_regex(const std::string &input) {
+pair<Rule, CompileError> parse_regex(const std::string &input) {
   return PatternParser(input.c_str()).rule(false);
 }
 

data/ext/tree-sitter/tree-sitter/src/compiler/prepare_grammar/parse_regex.h

@@ -9,7 +9,7 @@
 namespace tree_sitter {
 namespace prepare_grammar {
 
-std::pair<rule_ptr, CompileError> parse_regex(const std::string &);
+std::pair<rules::Rule, CompileError> parse_regex(const std::string &);
 
 }  // namespace prepare_grammar
 }  // namespace tree_sitter

data/ext/tree-sitter/tree-sitter/src/compiler/prepare_grammar/prepare_grammar.cc

@@ -17,7 +17,7 @@ using std::get;
 using std::make_tuple;
 
 tuple<SyntaxGrammar, LexicalGrammar, CompileError> prepare_grammar(
-  const Grammar &input_grammar) {
+  const InputGrammar &input_grammar) {
   /*
    * Convert all string-based `NamedSymbols` into numerical `Symbols`
    */
@@ -31,8 +31,9 @@ tuple<SyntaxGrammar, LexicalGrammar, CompileError> prepare_grammar(
    */
   auto extract_result = extract_tokens(intern_result.first);
   error = get<2>(extract_result);
-  if (error.type)
+  if (error.type) {
     return make_tuple(SyntaxGrammar(), LexicalGrammar(), error);
+  }
 
   /*
    * Replace `Repeat` rules with pairs of recursive rules
@@ -42,11 +43,12 @@ tuple<SyntaxGrammar, LexicalGrammar, CompileError> prepare_grammar(
   /*
    * Expand `String` and `Pattern` rules into full rule trees
    */
-  auto expand_tokens_result = expand_tokens(get<1>(extract_result));
-  LexicalGrammar lex_grammar = expand_tokens_result.first;
-  error = expand_tokens_result.second;
-  if (error.type)
-    return make_tuple(SyntaxGrammar(), LexicalGrammar(), error);
+  LexicalGrammar lex_grammar = get<1>(extract_result);
+  // auto expand_tokens_result = expand_tokens(get<1>(extract_result));
+  // LexicalGrammar lex_grammar = expand_tokens_result.first;
+  // error = expand_tokens_result.second;
+  // if (error.type)
+  //   return make_tuple(SyntaxGrammar(), LexicalGrammar(), error);
 
   /*
    * Flatten syntax rules into lists of productions.

data/ext/tree-sitter/tree-sitter/src/compiler/prepare_grammar/prepare_grammar.h

@@ -2,18 +2,15 @@
 #define COMPILER_PREPARE_GRAMMAR_PREPARE_GRAMMAR_H_
 
 #include <tuple>
+#include "compiler/grammar.h"
 #include "compiler/syntax_grammar.h"
 #include "compiler/lexical_grammar.h"
 #include "compiler/compile_error.h"
 
 namespace tree_sitter {
-
-struct Grammar;
-
 namespace prepare_grammar {
 
-std::tuple<SyntaxGrammar, LexicalGrammar, CompileError> prepare_grammar(
-  const Grammar &);
+std::tuple<SyntaxGrammar, LexicalGrammar, CompileError> prepare_grammar(const InputGrammar &);
 
 }  // namespace prepare_grammar
 }  // namespace tree_sitter

data/ext/tree-sitter/tree-sitter/src/compiler/prepare_grammar/token_description.cc

@@ -1,68 +1,82 @@
 #include "compiler/prepare_grammar/token_description.h"
-#include "compiler/rules/visitor.h"
-#include "compiler/rules/pattern.h"
-#include "compiler/rules/seq.h"
-#include "compiler/rules/choice.h"
-#include "compiler/rules/string.h"
-#include "compiler/rules/repeat.h"
-#include "compiler/rules/metadata.h"
+#include "compiler/rule.h"
 #include "compiler/util/string_helpers.h"
 
 namespace tree_sitter {
 namespace prepare_grammar {
 
 using std::string;
+using rules::Rule;
 
-class TokenDescription : public rules::RuleFn<string> {
-  string apply_to(const rules::Pattern *rule) {
-    is_trivial = false;
-    return rule->value;
-  }
+class TokenDescription {
+  bool is_trivial;
 
-  string apply_to(const rules::String *rule) {
-    return rule->value;
-  }
+  string apply(const Rule &rule) {
+    return rule.match(
+      [&](const rules::Blank) -> string {
+        return "";
+      },
 
-  string apply_to(const rules::Metadata *rule) {
-    return apply(rule->rule);
-  }
+      [&](const rules::Symbol) {
+        return "";
+      },
 
-  string apply_to(const rules::Seq *rule) {
-    is_trivial = false;
-    return apply(rule->left) + apply(rule->right);
-  }
+      [&](const rules::Pattern &rule) {
+        is_trivial = false;
+        return rule.value;
+      },
 
-  string apply_to(const rules::Repeat *rule) {
-    is_trivial = false;
-    return apply(rule->content) + "*";
-  }
+      [&](const rules::String &rule) {
+        return rule.value;
+      },
 
-  string apply_to(const rules::Choice *rule) {
-    is_trivial = false;
-    string result = "(";
-    bool started = false;
-    for (auto &element : rule->elements) {
-      if (started)
-        result += "|";
-      result += apply(element);
-      started = true;
-    }
-    return result + ")";
+      [&](const rules::Metadata &rule) {
+        return apply(*rule.rule);
+      },
+
+      [&](const rules::Seq &rule) {
+        is_trivial = false;
+        return apply(*rule.left) + apply(*rule.right);
+      },
+
+      [&](const rules::Repeat &rule) {
+        is_trivial = false;
+        return apply(*rule.rule) + "+";
+      },
+
+      [&](const rules::Choice &rule) {
+        is_trivial = false;
+        string result = "(";
+        bool started = false;
+        for (auto &element : rule.elements) {
+          if (started) result += "|";
+          result += apply(element);
+          started = true;
+        }
+        return result + ")";
+      },
+
+      [](auto) {
+        return "";
+      }
+    );
   }
 
  public:
-  bool is_trivial;
+  string describe(const Rule &rule) {
+    string result = apply(rule);
+    if (is_trivial) {
+      return result;
+    } else {
+      return "/" + result + "/";
+    }
+  }
 
   TokenDescription() : is_trivial(true) {}
 };
 
-string token_description(const rule_ptr &rule) {
-  TokenDescription description;
-  string result = description.apply(rule);
-  if (description.is_trivial)
-    return result;
-  else
-    return "/" + result + "/";
+string token_description(const Rule &rule) {
+  return TokenDescription().describe(rule);
 }
 
 }  // namespace prepare_grammar