tree-sitter 0.0.1 → 0.1.0

data/ext/tree-sitter/tree-sitter/src/compiler/build_tables/build_parse_table.h

@@ -1,22 +0,0 @@
-#ifndef COMPILER_BUILD_TABLES_BUILD_PARSE_TABLE_H_
-#define COMPILER_BUILD_TABLES_BUILD_PARSE_TABLE_H_
-
-#include <utility>
-#include <vector>
-#include "compiler/parse_table.h"
-#include "compiler/compile_error.h"
-
-namespace tree_sitter {
-
-struct SyntaxGrammar;
-struct LexicalGrammar;
-
-namespace build_tables {
-
-std::pair<ParseTable, CompileError> build_parse_table(const SyntaxGrammar &,
-                                                      const LexicalGrammar &);
-
-}  // namespace build_tables
-}  // namespace tree_sitter
-
-#endif  // COMPILER_BUILD_TABLES_BUILD_PARSE_TABLE_H_

data/ext/tree-sitter/tree-sitter/src/compiler/build_tables/build_tables.cc

@@ -1,27 +0,0 @@
-#include "compiler/build_tables/build_tables.h"
-#include <tuple>
-#include "compiler/build_tables/build_lex_table.h"
-#include "compiler/build_tables/build_parse_table.h"
-#include "compiler/syntax_grammar.h"
-#include "compiler/lexical_grammar.h"
-#include "compiler/compile_error.h"
-
-namespace tree_sitter {
-namespace build_tables {
-
-using std::string;
-using std::tuple;
-using std::vector;
-using std::make_tuple;
-
-tuple<ParseTable, LexTable, CompileError> build_tables(
-  const SyntaxGrammar &grammar, const LexicalGrammar &lex_grammar) {
-  auto parse_table_result = build_parse_table(grammar, lex_grammar);
-  ParseTable parse_table = parse_table_result.first;
-  const CompileError error = parse_table_result.second;
-  LexTable lex_table = build_lex_table(&parse_table, lex_grammar);
-  return make_tuple(parse_table, lex_table, error);
-}
-
-}  // namespace build_tables
-}  // namespace tree_sitter

data/ext/tree-sitter/tree-sitter/src/compiler/build_tables/build_tables.h

@@ -1,24 +0,0 @@
-#ifndef COMPILER_BUILD_TABLES_BUILD_TABLES_H_
-#define COMPILER_BUILD_TABLES_BUILD_TABLES_H_
-
-#include <string>
-#include <tuple>
-#include <vector>
-#include "compiler/parse_table.h"
-#include "compiler/lex_table.h"
-#include "compiler/compile_error.h"
-
-namespace tree_sitter {
-
-struct SyntaxGrammar;
-struct LexicalGrammar;
-
-namespace build_tables {
-
-std::tuple<ParseTable, LexTable, CompileError> build_tables(
-  const SyntaxGrammar &, const LexicalGrammar &);
-
-}  // namespace build_tables
-}  // namespace tree_sitter
-
-#endif  // COMPILER_BUILD_TABLES_BUILD_TABLES_H_

data/ext/tree-sitter/tree-sitter/src/compiler/build_tables/conflict_type.h

@@ -1,16 +0,0 @@
-#ifndef COMPILER_BUILD_TABLES_CONFLICT_TYPE_H_
-#define COMPILER_BUILD_TABLES_CONFLICT_TYPE_H_
-
-namespace tree_sitter {
-namespace build_tables {
-
-enum ConflictType {
-  ConflictTypeNone,
-  ConflictTypeResolved,
-  ConflictTypeUnresolved
-};
-
-}  // namespace build_tables
-}  // namespace tree_sitter
-
-#endif  // COMPILER_BUILD_TABLES_CONFLICT_TYPE_H_

data/ext/tree-sitter/tree-sitter/src/compiler/build_tables/lex_conflict_manager.cc

@@ -1,55 +0,0 @@
-#include "compiler/build_tables/lex_conflict_manager.h"
-#include <utility>
-#include "compiler/parse_table.h"
-#include "compiler/rules/built_in_symbols.h"
-#include "compiler/build_tables/lex_item.h"
-
-namespace tree_sitter {
-namespace build_tables {
-
-bool LexConflictManager::resolve(const LexItemSet &item_set,
-                                 const AdvanceAction &new_action,
-                                 const AcceptTokenAction &old_action) {
-  if (!old_action.is_present())
-    return true;
-  if (new_action.precedence_range.max >= old_action.precedence) {
-    for (const LexItem &item : item_set.entries)
-      possible_extensions[old_action.symbol.index].insert(item.lhs.index);
-    return true;
-  } else {
-    return false;
-  }
-}
-
-bool LexConflictManager::resolve(const AcceptTokenAction &new_action,
-                                 const AcceptTokenAction &old_action) {
-  if (!old_action.is_present())
-    return true;
-
-  int old_precedence = old_action.precedence;
-  int new_precedence = new_action.precedence;
-
-  bool result;
-  if (new_precedence > old_precedence)
-    result = true;
-  else if (new_precedence < old_precedence)
-    result = false;
-  else if (new_action.is_string && !old_action.is_string)
-    result = true;
-  else if (old_action.is_string && !new_action.is_string)
-    result = false;
-  else if (new_action.symbol.index < old_action.symbol.index)
-    result = true;
-  else
-    result = false;
-
-  if (result)
-    possible_homonyms[old_action.symbol.index].insert(new_action.symbol.index);
-  else
-    possible_homonyms[new_action.symbol.index].insert(old_action.symbol.index);
-
-  return result;
-}
-
-}  // namespace build_tables
-}  // namespace tree_sitter

data/ext/tree-sitter/tree-sitter/src/compiler/build_tables/lex_conflict_manager.h

@@ -1,31 +0,0 @@
-#ifndef COMPILER_BUILD_TABLES_LEX_CONFLICT_MANAGER_H_
-#define COMPILER_BUILD_TABLES_LEX_CONFLICT_MANAGER_H_
-
-#include <map>
-#include <set>
-#include "compiler/lexical_grammar.h"
-#include "compiler/rules/symbol.h"
-
-namespace tree_sitter {
-
-struct AdvanceAction;
-struct AcceptTokenAction;
-
-namespace build_tables {
-
-class LexItemSet;
-
-class LexConflictManager {
- public:
-  bool resolve(const LexItemSet &, const AdvanceAction &,
-               const AcceptTokenAction &);
-  bool resolve(const AcceptTokenAction &, const AcceptTokenAction &);
-
-  std::map<rules::Symbol::Index, std::set<rules::Symbol::Index>> possible_homonyms;
-  std::map<rules::Symbol::Index, std::set<rules::Symbol::Index>> possible_extensions;
-};
-
-}  // namespace build_tables
-}  // namespace tree_sitter
-
-#endif  // COMPILER_BUILD_TABLES_LEX_CONFLICT_MANAGER_H_

data/ext/tree-sitter/tree-sitter/src/compiler/build_tables/recovery_tokens.cc

@@ -1,89 +0,0 @@
-#include "compiler/build_tables/recovery_tokens.h"
-#include "compiler/lexical_grammar.h"
-#include "compiler/rules/choice.h"
-#include "compiler/rules/character_set.h"
-#include "compiler/rules/repeat.h"
-#include "compiler/rules/visitor.h"
-#include "compiler/rules/seq.h"
-#include "compiler/rules/metadata.h"
-
-namespace tree_sitter {
-namespace build_tables {
-
-using rules::Symbol;
-using std::set;
-
-template <bool left, bool right>
-class CharacterAggregator : public rules::RuleFn<void> {
-  void apply_to(const rules::Seq *rule) {
-    if (left)
-      apply(rule->left);
-    if (right)
-      apply(rule->right);
-  }
-
-  void apply_to(const rules::Choice *rule) {
-    for (const rule_ptr &element : rule->elements)
-      apply(element);
-  }
-
-  void apply_to(const rules::Repeat *rule) {
-    apply(rule->content);
-  }
-
-  void apply_to(const rules::Metadata *rule) {
-    apply(rule->rule);
-  }
-
-  void apply_to(const rules::CharacterSet *rule) {
-    result.add_set(*rule);
-  }
-
- public:
-  rules::CharacterSet result;
-};
-
-class FirstCharacters : public CharacterAggregator<true, false> {};
-class LastCharacters : public CharacterAggregator<false, true> {};
-class AllCharacters : public CharacterAggregator<true, true> {};
-
-set<Symbol> recovery_tokens(const LexicalGrammar &grammar) {
-  set<Symbol> result;
-
-  AllCharacters all_separator_characters;
-  for (const rule_ptr &separator : grammar.separators)
-    all_separator_characters.apply(separator);
-
-  for (size_t i = 0; i < grammar.variables.size(); i++) {
-    const Variable &variable = grammar.variables[i];
-    rule_ptr rule = variable.rule;
-
-    FirstCharacters first_characters;
-    first_characters.apply(variable.rule);
-
-    LastCharacters last_characters;
-    last_characters.apply(variable.rule);
-
-    AllCharacters all_characters;
-    all_characters.apply(variable.rule);
-
-    bool has_distinct_start =
-      !first_characters.result.includes_all &&
-      !first_characters.result.intersects(all_separator_characters.result);
-
-    bool has_distinct_end =
-      !last_characters.result.includes_all &&
-      !last_characters.result.intersects(all_separator_characters.result);
-
-    bool has_no_separators =
-      !all_characters.result.intersects(all_separator_characters.result);
-
-    if ((has_distinct_start && has_distinct_end) || has_no_separators)
-      result.insert(Symbol(i, Symbol::Terminal));
-  }
-
-  return result;
-}
-
-}  // namespace build_tables
-}  // namespace tree_sitter

data/ext/tree-sitter/tree-sitter/src/compiler/build_tables/recovery_tokens.h

@@ -1,19 +0,0 @@
-#ifndef COMPILER_BUILD_TABLES_DOES_MATCH_ANY_LINE_H_
-#define COMPILER_BUILD_TABLES_DOES_MATCH_ANY_LINE_H_
-
-#include "compiler/rule.h"
-#include "compiler/rules/symbol.h"
-#include <set>
-
-namespace tree_sitter {
-
-struct LexicalGrammar;
-
-namespace build_tables {
-
-std::set<rules::Symbol> recovery_tokens(const LexicalGrammar &);
-
-}  // namespace build_tables
-}  // namespace tree_sitter
-
-#endif  // COMPILER_BUILD_TABLES_DOES_MATCH_ANY_LINE_H_

data/ext/tree-sitter/tree-sitter/src/compiler/build_tables/remove_duplicate_states.h

@@ -1,65 +0,0 @@
-#ifndef COMPILER_BUILD_TABLES_REMOVE_DUPLICATE_STATES_H_
-#define COMPILER_BUILD_TABLES_REMOVE_DUPLICATE_STATES_H_
-
-#include <map>
-#include <vector>
-
-namespace tree_sitter {
-namespace build_tables {
-
-template <typename TableType>
-std::map<size_t, size_t> remove_duplicate_states(TableType *table) {
-  std::map<size_t, size_t> replacements;
-
-  while (true) {
-    std::map<size_t, size_t> duplicates;
-    for (size_t i = 0, size = table->states.size(); i < size; i++)
-      for (size_t j = 0; j < i; j++)
-        if (!duplicates.count(j) && table->merge_state(j, i)) {
-          duplicates.insert({ i, j });
-          break;
-        }
-
-    if (duplicates.empty())
-      break;
-
-    std::map<size_t, size_t> new_replacements;
-    for (size_t i = 0, size = table->states.size(); i < size; i++) {
-      size_t new_state_index = i;
-      auto duplicate = duplicates.find(i);
-      if (duplicate != duplicates.end())
-        new_state_index = duplicate->second;
-
-      size_t prior_removed = 0;
-      for (const auto &duplicate : duplicates) {
-        if (duplicate.first >= new_state_index)
-          break;
-        prior_removed++;
-      }
-
-      new_state_index -= prior_removed;
-      new_replacements.insert({ i, new_state_index });
-      replacements.insert({ i, new_state_index });
-      for (auto &replacement : replacements)
-        if (replacement.second == i)
-          replacement.second = new_state_index;
-    }
-
-    for (auto &state : table->states)
-      state.each_referenced_state([&new_replacements](int64_t *state_index) {
-        auto new_replacement = new_replacements.find(*state_index);
-        if (new_replacement != new_replacements.end())
-          *state_index = new_replacement->second;
-      });
-
-    for (auto i = duplicates.rbegin(); i != duplicates.rend(); ++i)
-      table->states.erase(table->states.begin() + i->first);
-  }
-
-  return replacements;
-}
-
-}  // namespace build_tables
-}  // namespace tree_sitter
-
-#endif  // COMPILER_BUILD_TABLES_REMOVE_DUPLICATE_STATES_H_

data/ext/tree-sitter/tree-sitter/src/compiler/compile.h

@@ -1,16 +0,0 @@
-#ifndef COMPILER_COMPILE_H_
-#define COMPILER_COMPILE_H_
-
-#include <string>
-#include <utility>
-#include "compiler/compile_error.h"
-
-namespace tree_sitter {
-
-struct Grammar;
-
-std::pair<std::string, CompileError> compile(const Grammar &, std::string);
-
-}  // namespace tree_sitter
-
-#endif  // COMPILER_COMPILE_H_

data/ext/tree-sitter/tree-sitter/src/compiler/prepare_grammar/is_token.cc

@@ -1,30 +0,0 @@
-#include "compiler/prepare_grammar/is_token.h"
-#include "tree_sitter/compiler.h"
-#include "compiler/rules/visitor.h"
-#include "compiler/rules/string.h"
-#include "compiler/rules/metadata.h"
-#include "compiler/rules/pattern.h"
-
-namespace tree_sitter {
-namespace prepare_grammar {
-
-class IsToken : public rules::RuleFn<bool> {
-  bool apply_to(const rules::String *rule) {
-    return true;
-  }
-
-  bool apply_to(const rules::Pattern *rule) {
-    return true;
-  }
-
-  bool apply_to(const rules::Metadata *rule) {
-    return rule->params.is_token;
-  }
-};
-
-bool is_token(const rule_ptr &rule) {
-  return IsToken().apply(rule);
-}
-
-}  // namespace prepare_grammar
-}  // namespace tree_sitter

data/ext/tree-sitter/tree-sitter/src/compiler/prepare_grammar/is_token.h

@@ -1,14 +0,0 @@
-#ifndef COMPILER_PREPARE_GRAMMAR_IS_TOKEN_H_
-#define COMPILER_PREPARE_GRAMMAR_IS_TOKEN_H_
-
-#include "compiler/rule.h"
-
-namespace tree_sitter {
-namespace prepare_grammar {
-
-bool is_token(const rule_ptr &);
-
-}  // namespace prepare_grammar
-}  // namespace tree_sitter
-
-#endif  // COMPILER_PREPARE_GRAMMAR_IS_TOKEN_H_

data/ext/tree-sitter/tree-sitter/src/compiler/rules.h

@@ -1,28 +0,0 @@
-#ifndef COMPILER_RULES_H_
-#define COMPILER_RULES_H_
-
-#include <string>
-#include <vector>
-#include <memory>
-#include "compiler/rule.h"
-
-namespace tree_sitter {
-
-rule_ptr blank();
-rule_ptr choice(const std::vector<rule_ptr> &);
-rule_ptr repeat(const rule_ptr &);
-rule_ptr repeat1(const rule_ptr &);
-rule_ptr seq(const std::vector<rule_ptr> &);
-rule_ptr sym(const std::string &);
-rule_ptr pattern(const std::string &);
-rule_ptr str(const std::string &);
-rule_ptr prec(int precedence, const rule_ptr &);
-rule_ptr prec_left(const rule_ptr &);
-rule_ptr prec_left(int precedence, const rule_ptr &);
-rule_ptr prec_right(const rule_ptr &);
-rule_ptr prec_right(int precedence, const rule_ptr &);
-rule_ptr token(const rule_ptr &rule);
-
-}  // namespace std
-
-#endif  // COMPILER_RULES_H_