tree-sitter 0.0.1 → 0.1.0

data/ext/tree-sitter/tree-sitter/spec/compiler/build_tables/parse_item_set_builder_spec.cc

@@ -1,133 +0,0 @@
-#include "spec_helper.h"
-#include "compiler/syntax_grammar.h"
-#include "compiler/lexical_grammar.h"
-#include "compiler/build_tables/parse_item_set_builder.h"
-#include "compiler/build_tables/lookahead_set.h"
-#include "compiler/rules/built_in_symbols.h"
-#include "helpers/rule_helpers.h"
-
-using namespace build_tables;
-using namespace rules;
-
-START_TEST
-
-describe("ParseItemSetBuilder", []() {
-  vector<Variable> lexical_variables;
-  for (size_t i = 0; i < 20; i++) {
-    lexical_variables.push_back(Variable{
-      "token_" + to_string(i),
-      VariableTypeNamed,
-      blank(),
-    });
-  }
-
-  LexicalGrammar lexical_grammar{lexical_variables, {}};
-
-  it("adds items at the beginnings of referenced rules", [&]() {
-    SyntaxGrammar grammar{{
-      SyntaxVariable("rule0", VariableTypeNamed, {
-        Production({
-          {Symbol(1, Symbol::NonTerminal), 0, AssociativityNone},
-          {Symbol(11, Symbol::Terminal), 0, AssociativityNone},
-        }),
-      }),
-      SyntaxVariable("rule1", VariableTypeNamed, {
-        Production({
-          {Symbol(12, Symbol::Terminal), 0, AssociativityNone},
-          {Symbol(13, Symbol::Terminal), 0, AssociativityNone},
-        }),
-        Production({
-          {Symbol(2, Symbol::NonTerminal), 0, AssociativityNone},
-        })
-      }),
-      SyntaxVariable("rule2", VariableTypeNamed, {
-        Production({
-          {Symbol(14, Symbol::Terminal), 0, AssociativityNone},
-          {Symbol(15, Symbol::Terminal), 0, AssociativityNone},
-        })
-      }),
-    }, {}, {}, {}};
-
-    auto production = [&](int variable_index, int production_index) -> const Production & {
-      return grammar.variables[variable_index].productions[production_index];
-    };
-
-    ParseItemSet item_set({
-      {
-        ParseItem(Symbol(0, Symbol::NonTerminal), production(0, 0), 0),
-        LookaheadSet({ Symbol(10, Symbol::Terminal) }),
-      }
-    });
-
-    ParseItemSetBuilder item_set_builder(grammar, lexical_grammar);
-    item_set_builder.apply_transitive_closure(&item_set);
-
-    AssertThat(item_set, Equals(ParseItemSet({
-      {
-        ParseItem(Symbol(0, Symbol::NonTerminal), production(0, 0), 0),
-        LookaheadSet({ Symbol(10, Symbol::Terminal) })
-        },
-      {
-        ParseItem(Symbol(1, Symbol::NonTerminal), production(1, 0), 0),
-        LookaheadSet({ Symbol(11, Symbol::Terminal) })
-      },
-      {
-        ParseItem(Symbol(1, Symbol::NonTerminal), production(1, 1), 0),
-        LookaheadSet({ Symbol(11, Symbol::Terminal) })
-      },
-      {
-        ParseItem(Symbol(2, Symbol::NonTerminal), production(2, 0), 0),
-        LookaheadSet({ Symbol(11, Symbol::Terminal) })
-      },
-    })));
-  });
-
-  it("handles rules with empty productions", [&]() {
-    SyntaxGrammar grammar{{
-      SyntaxVariable("rule0", VariableTypeNamed, {
-        Production({
-          {Symbol(1, Symbol::NonTerminal), 0, AssociativityNone},
-          {Symbol(11, Symbol::Terminal), 0, AssociativityNone},
-        }),
-      }),
-      SyntaxVariable("rule1", VariableTypeNamed, {
-        Production({
-          {Symbol(12, Symbol::Terminal), 0, AssociativityNone},
-          {Symbol(13, Symbol::Terminal), 0, AssociativityNone},
-        }),
-        Production({})
-      }),
-    }, {}, {}, {}};
-
-    auto production = [&](int variable_index, int production_index) -> const Production & {
-      return grammar.variables[variable_index].productions[production_index];
-    };
-
-    ParseItemSet item_set({
-      {
-        ParseItem(Symbol(0, Symbol::NonTerminal), production(0, 0), 0),
-        LookaheadSet({ Symbol(10, Symbol::Terminal) }),
-      }
-    });
-
-    ParseItemSetBuilder item_set_builder(grammar, lexical_grammar);
-    item_set_builder.apply_transitive_closure(&item_set);
-
-    AssertThat(item_set, Equals(ParseItemSet({
-      {
-        ParseItem(Symbol(0, Symbol::NonTerminal), production(0, 0), 0),
-        LookaheadSet({ Symbol(10, Symbol::Terminal) })
-      },
-      {
-        ParseItem(Symbol(1, Symbol::NonTerminal), production(1, 0), 0),
-        LookaheadSet({ Symbol(11, Symbol::Terminal) })
-      },
-      {
-        ParseItem(Symbol(1, Symbol::NonTerminal), production(1, 1), 0),
-        LookaheadSet({ Symbol(11, Symbol::Terminal) })
-      },
-    })));
-  });
-});
-
-END_TEST

data/ext/tree-sitter/tree-sitter/spec/compiler/prepare_grammar/expand_repeats_spec.cc

@@ -1,152 +0,0 @@
-#include "spec_helper.h"
-#include "compiler/prepare_grammar/initial_syntax_grammar.h"
-#include "compiler/prepare_grammar/expand_repeats.h"
-#include "helpers/rule_helpers.h"
-
-START_TEST
-
-using namespace rules;
-using prepare_grammar::InitialSyntaxGrammar;
-using prepare_grammar::expand_repeats;
-
-describe("expand_repeats", []() {
-  it("replaces repeat rules with pairs of recursive rules", [&]() {
-    InitialSyntaxGrammar grammar{{
-      Variable("rule0", VariableTypeNamed, repeat1(i_token(0))),
-    }, {}, {}, {}};
-
-    auto result = expand_repeats(grammar);
-
-    AssertThat(result.variables, Equals(vector<Variable>({
-      Variable("rule0", VariableTypeNamed, i_sym(1)),
-      Variable("rule0_repeat1", VariableTypeAuxiliary, choice({
-        seq({ i_sym(1), i_token(0) }),
-        i_token(0),
-      })),
-    })));
-  });
-
-  it("replaces repeats inside of sequences", [&]() {
-    InitialSyntaxGrammar grammar{{
-      Variable("rule0", VariableTypeNamed, seq({
-        i_token(10),
-        repeat1(i_token(11)),
-      })),
-    }, {}, {}, {}};
-
-    auto result = expand_repeats(grammar);
-
-    AssertThat(result.variables, Equals(vector<Variable>({
-      Variable("rule0", VariableTypeNamed, seq({
-        i_token(10),
-        i_sym(1),
-      })),
-      Variable("rule0_repeat1", VariableTypeAuxiliary, choice({
-        seq({ i_sym(1), i_token(11) }),
-        i_token(11)
-      })),
-    })));
-  });
-
-  it("replaces repeats inside of choices", [&]() {
-    InitialSyntaxGrammar grammar{{
-      Variable("rule0", VariableTypeNamed, choice({
-        i_token(10),
-        repeat1(i_token(11))
-      })),
-    }, {}, {}, {}};
-
-    auto result = expand_repeats(grammar);
-
-    AssertThat(result.variables, Equals(vector<Variable>({
-      Variable("rule0", VariableTypeNamed, choice({
-        i_token(10),
-        i_sym(1),
-      })),
-      Variable("rule0_repeat1", VariableTypeAuxiliary, choice({
-        seq({ i_sym(1), i_token(11) }),
-        i_token(11),
-      })),
-    })));
-  });
-
-  it("does not create redundant auxiliary rules", [&]() {
-    InitialSyntaxGrammar grammar{{
-      Variable("rule0", VariableTypeNamed, choice({
-        seq({ i_token(1), repeat1(i_token(4)) }),
-        seq({ i_token(2), repeat1(i_token(4)) }),
-      })),
-      Variable("rule1", VariableTypeNamed, seq({
-        i_token(3),
-        repeat1(i_token(4))
-      })),
-    }, {}, {}, {}};
-
-    auto result = expand_repeats(grammar);
-
-    AssertThat(result.variables, Equals(vector<Variable>({
-      Variable("rule0", VariableTypeNamed, choice({
-        seq({ i_token(1), i_sym(2) }),
-        seq({ i_token(2), i_sym(2) }),
-      })),
-      Variable("rule1", VariableTypeNamed, seq({
-        i_token(3),
-        i_sym(2),
-      })),
-      Variable("rule0_repeat1", VariableTypeAuxiliary, choice({
-        seq({ i_sym(2), i_token(4) }),
-        i_token(4),
-      })),
-    })));
-  });
-
-  it("can replace multiple repeats in the same rule", [&]() {
-    InitialSyntaxGrammar grammar{{
-      Variable("rule0", VariableTypeNamed, seq({
-        repeat1(i_token(10)),
-        repeat1(i_token(11)),
-      })),
-    }, {}, {}, {}};
-
-    auto result = expand_repeats(grammar);
-
-    AssertThat(result.variables, Equals(vector<Variable>({
-      Variable("rule0", VariableTypeNamed, seq({
-        i_sym(1),
-        i_sym(2),
-      })),
-      Variable("rule0_repeat1", VariableTypeAuxiliary, choice({
-        seq({ i_sym(1), i_token(10) }),
-        i_token(10),
-      })),
-      Variable("rule0_repeat2", VariableTypeAuxiliary, choice({
-        seq({ i_sym(2), i_token(11) }),
-        i_token(11),
-      })),
-    })));
-  });
-
-  it("can replace repeats in multiple rules", [&]() {
-    InitialSyntaxGrammar grammar{{
-      Variable("rule0", VariableTypeNamed, repeat1(i_token(10))),
-      Variable("rule1", VariableTypeNamed, repeat1(i_token(11))),
-    }, {}, {}, {}};
-
-    auto result = expand_repeats(grammar);
-
-    AssertThat(result.variables, Equals(vector<Variable>({
-      Variable("rule0", VariableTypeNamed, i_sym(2)),
-      Variable("rule1", VariableTypeNamed, i_sym(3)),
-      Variable("rule0_repeat1", VariableTypeAuxiliary, choice({
-        seq({ i_sym(2), i_token(10) }),
-        i_token(10),
-      })),
-      Variable("rule1_repeat1", VariableTypeAuxiliary, choice({
-        seq({ i_sym(3), i_token(11) }),
-        i_token(11),
-      })),
-    })));
-  });
-});
-
-END_TEST

data/ext/tree-sitter/tree-sitter/spec/compiler/prepare_grammar/expand_tokens_spec.cc

@@ -1,109 +0,0 @@
-#include "spec_helper.h"
-#include "compiler/lexical_grammar.h"
-#include "compiler/prepare_grammar/expand_tokens.h"
-#include "helpers/rule_helpers.h"
-
-START_TEST
-
-using namespace rules;
-using prepare_grammar::expand_tokens;
-
-describe("expand_tokens", []() {
-  MetadataParams string_token_params;
-  string_token_params.is_string = true;
-  string_token_params.is_token = true;
-
-  describe("string rules", [&]() {
-    it("replaces strings with sequences of character sets", [&]() {
-      LexicalGrammar grammar{{
-        Variable("rule_A", VariableTypeNamed, seq({
-          i_sym(10),
-          str("xyz"),
-          i_sym(11),
-        })),
-      }, {}};
-
-      auto result = expand_tokens(grammar);
-
-      AssertThat(result.second, Equals(CompileError::none()));
-      AssertThat(result.first.variables, Equals(vector<Variable>({
-        Variable("rule_A", VariableTypeNamed, seq({
-          i_sym(10),
-          metadata(seq({
-            character({ 'x' }),
-            character({ 'y' }),
-            character({ 'z' }),
-          }), string_token_params),
-          i_sym(11),
-        })),
-      })));
-    });
-
-    it("handles strings containing non-ASCII UTF8 characters", [&]() {
-      LexicalGrammar grammar{{
-        Variable("rule_A", VariableTypeNamed, str("\u03B1 \u03B2")),
-      }, {}};
-
-      auto result = expand_tokens(grammar);
-
-      AssertThat(result.first.variables, Equals(vector<Variable>({
-        Variable("rule_A", VariableTypeNamed, metadata(seq({
-          character({ 945 }),
-          character({ ' ' }),
-          character({ 946 }),
-        }), string_token_params)),
-      })));
-    });
-  });
-
-  describe("regexp rules", [&]() {
-    it("replaces regexps with the equivalent rule tree", [&]() {
-      LexicalGrammar grammar{{
-        Variable("rule_A", VariableTypeNamed, seq({
-          i_sym(10),
-          pattern("x*"),
-          i_sym(11),
-        })),
-      }, {}};
-
-      auto result = expand_tokens(grammar);
-
-      AssertThat(result.second, Equals(CompileError::none()));
-      AssertThat(result.first.variables, Equals(vector<Variable>({
-        Variable("rule_A", VariableTypeNamed, seq({
-          i_sym(10),
-          repeat(character({ 'x' })),
-          i_sym(11),
-        })),
-      })));
-    });
-
-    it("handles regexps containing non-ASCII UTF8 characters", [&]() {
-      LexicalGrammar grammar{{
-        Variable("rule_A", VariableTypeNamed, pattern("[^\u03B1-\u03B4]*")),
-      }, {}};
-
-      auto result = expand_tokens(grammar);
-
-      AssertThat(result.first.variables, Equals(vector<Variable>({
-        Variable("rule_A", VariableTypeNamed, repeat(character({ 945, 946, 947, 948 }, false))),
-      })));
-    });
-
-    it("returns an error when the grammar contains an invalid regex", [&]() {
-      LexicalGrammar grammar{{
-        Variable("rule_A", VariableTypeNamed, seq({
-          pattern("("),
-          str("xyz"),
-          pattern("["),
-        }))
-      }, {}};
-
-      auto result = expand_tokens(grammar);
-
-      AssertThat(result.second, Equals(CompileError(TSCompileErrorTypeInvalidRegex, "unmatched open paren")));
-    });
-  });
-});
-
-END_TEST

data/ext/tree-sitter/tree-sitter/spec/compiler/prepare_grammar/extract_choices_spec.cc

@@ -1,106 +0,0 @@
-#include "spec_helper.h"
-#include "compiler/prepare_grammar/extract_choices.h"
-#include "helpers/rule_helpers.h"
-
-START_TEST
-
-using namespace rules;
-using prepare_grammar::extract_choices;
-
-class rule_vector : public vector<rule_ptr> {
- public:
-  bool operator==(const vector<rule_ptr> &other) const {
-    if (this->size() != other.size()) return false;
-    for (size_t i = 0; i < this->size(); i++) {
-      auto rule = this->operator[](i);
-      auto other_rule = other[i];
-      if (!rule->operator==(*rule))
-        return false;
-    }
-    return true;
-  }
-
-  rule_vector(const initializer_list<rule_ptr> &list) :
-      vector<rule_ptr>(list) {}
-};
-
-describe("extract_choices", []() {
-  it("expands rules containing choices into multiple rules", [&]() {
-    auto rule = seq({
-      sym("a"),
-      choice({ sym("b"), sym("c"), sym("d") }),
-      sym("e")
-    });
-
-    AssertThat(extract_choices(rule), Equals(rule_vector({
-      seq({ sym("a"), sym("b"), sym("e") }),
-      seq({ sym("a"), sym("c"), sym("e") }),
-      seq({ sym("a"), sym("d"), sym("e") }),
-    })));
-  });
-
-  it("handles metadata rules", [&]() {
-    auto rule = prec(5, choice({ sym("b"), sym("c"), sym("d") }));
-
-    AssertThat(extract_choices(rule), Equals(rule_vector({
-      prec(5, sym("b")),
-      prec(5, sym("c")),
-      prec(5, sym("d")),
-    })));
-  });
-
-  it("handles nested choices", [&]() {
-    auto rule = choice({
-      seq({ choice({ sym("a"), sym("b") }), sym("c") }),
-      sym("d")
-    });
-
-    AssertThat(extract_choices(rule), Equals(rule_vector({
-      seq({ sym("a"), sym("c") }),
-      seq({ sym("b"), sym("c") }),
-      sym("d"),
-    })));
-  });
-
-  it("handles blank rules", [&]() {
-    AssertThat(extract_choices(blank()), Equals(rule_vector({
-      blank(),
-    })));
-  });
-
-  it("does not move choices outside of repeats", [&]() {
-    auto rule = seq({
-      choice({ sym("a"), sym("b") }),
-      repeat1(seq({
-        sym("c"),
-        choice({
-          sym("d"),
-          sym("e"),
-        }),
-        sym("f"),
-      })),
-      sym("g"),
-    });
-
-    AssertThat(extract_choices(rule), Equals(rule_vector({
-      seq({
-        sym("a"),
-        repeat1(choice({
-          seq({ sym("c"), sym("d"), sym("f") }),
-          seq({ sym("c"), sym("e"), sym("f") }),
-        })),
-        sym("g"),
-      }),
-      seq({
-        sym("b"),
-        repeat1(choice({
-          seq({ sym("c"), sym("d"), sym("f") }),
-          seq({ sym("c"), sym("e"), sym("f") }),
-        })),
-        sym("g"),
-      }),
-    })));
-  });
-});
-
-END_TEST