tree-sitter 0.0.1 → 0.1.0

data/ext/tree-sitter/tree-sitter/src/compiler/build_tables/parse_table_builder.h

@@ -0,0 +1,38 @@
+#ifndef COMPILER_BUILD_TABLES_PARSE_TABLE_BUILDER_H_
+#define COMPILER_BUILD_TABLES_PARSE_TABLE_BUILDER_H_
+
+#include <memory>
+#include "compiler/parse_table.h"
+#include "compiler/compile_error.h"
+
+namespace tree_sitter {
+
+struct ParseTable;
+struct LexTable;
+struct SyntaxGrammar;
+struct LexicalGrammar;
+
+namespace build_tables {
+
+class ParseTableBuilder {
+ public:
+  static std::unique_ptr<ParseTableBuilder> create(const SyntaxGrammar &, const LexicalGrammar &);
+
+  struct BuildResult {
+    ParseTable parse_table;
+    LexTable main_lex_table;
+    LexTable keyword_lex_table;
+    rules::Symbol keyword_capture_token;
+    CompileError error;
+  };
+
+  BuildResult build();
+
+ protected:
+  ParseTableBuilder() = default;
+};
+
+}  // namespace build_tables
+}  // namespace tree_sitter
+
+#endif  // COMPILER_BUILD_TABLES_PARSE_TABLE_BUILDER_H_

data/ext/tree-sitter/tree-sitter/src/compiler/build_tables/rule_can_be_blank.cc

@@ -1,43 +1,42 @@
 #include "compiler/build_tables/rule_can_be_blank.h"
-#include "compiler/rules/symbol.h"
-#include "compiler/rules/visitor.h"
-#include "compiler/rules/seq.h"
-#include "compiler/rules/choice.h"
-#include "compiler/rules/blank.h"
-#include "compiler/rules/metadata.h"
-#include "compiler/rules/repeat.h"
+#include "compiler/rule.h"
 
 namespace tree_sitter {
 namespace build_tables {
 
-class CanBeBlank : public rules::RuleFn<bool> {
- protected:
-  bool apply_to(const rules::Blank *) {
-    return true;
-  }
-
-  bool apply_to(const rules::Repeat *rule) {
-    return apply(rule->content);
-  }
-
-  bool apply_to(const rules::Choice *rule) {
-    for (const auto &element : rule->elements)
-      if (apply(element))
-        return true;
-    return false;
-  }
-
-  bool apply_to(const rules::Seq *rule) {
-    return apply(rule->left) && apply(rule->right);
-  }
-
-  bool apply_to(const rules::Metadata *rule) {
-    return apply(rule->rule);
-  }
-};
-
-bool rule_can_be_blank(const rule_ptr &rule) {
-  return CanBeBlank().apply(rule);
+bool rule_can_be_blank(const rules::Rule &rule) {
+  return rule.match(
+    [](rules::Blank) {
+      return true;
+    },
+
+    [](rules::CharacterSet) {
+      return false;
+    },
+
+    [](rules::Repeat repeat) {
+      return rule_can_be_blank(*repeat.rule);
+    },
+
+    [](rules::Metadata metadata) {
+      return rule_can_be_blank(*metadata.rule);
+    },
+
+    [](rules::Choice choice) {
+      for (const auto &element : choice.elements) {
+        if (rule_can_be_blank(element)) {
+          return true;
+        }
+      }
+      return false;
+    },
+
+    [](rules::Seq seq) {
+      return rule_can_be_blank(*seq.left) && rule_can_be_blank(*seq.right);
+    },
+
+    [](auto) { return false; }
+  );
 }
 
 }  // namespace build_tables

data/ext/tree-sitter/tree-sitter/src/compiler/build_tables/rule_can_be_blank.h

@@ -6,7 +6,7 @@
 namespace tree_sitter {
 namespace build_tables {
 
-bool rule_can_be_blank(const rule_ptr &rule);
+bool rule_can_be_blank(const rules::Rule &rule);
 
 }  // namespace build_tables
 }  // namespace tree_sitter

data/ext/tree-sitter/tree-sitter/src/compiler/compile.cc

@@ -1,6 +1,6 @@
 #include "tree_sitter/compiler.h"
 #include "compiler/prepare_grammar/prepare_grammar.h"
-#include "compiler/build_tables/build_tables.h"
+#include "compiler/build_tables/parse_table_builder.h"
 #include "compiler/generate_code/c_code.h"
 #include "compiler/syntax_grammar.h"
 #include "compiler/lexical_grammar.h"
@@ -9,6 +9,7 @@
 
 namespace tree_sitter {
 
+using std::move;
 using std::pair;
 using std::string;
 using std::vector;
@@ -22,51 +23,33 @@ extern "C" TSCompileResult ts_compile_grammar(const char *input) {
              TSCompileErrorTypeInvalidGrammar };
   }
 
-  auto prepare_grammar_result =
-    prepare_grammar::prepare_grammar(parse_result.grammar);
-  const SyntaxGrammar &syntax_grammar = get<0>(prepare_grammar_result);
-  const LexicalGrammar &lexical_grammar = get<1>(prepare_grammar_result);
+  auto prepare_grammar_result = prepare_grammar::prepare_grammar(parse_result.grammar);
+  SyntaxGrammar &syntax_grammar = get<0>(prepare_grammar_result);
+  LexicalGrammar &lexical_grammar = get<1>(prepare_grammar_result);
   CompileError error = get<2>(prepare_grammar_result);
   if (error.type) {
-    return { nullptr, strdup(error.message.c_str()), error.type };
+    return {nullptr, strdup(error.message.c_str()), error.type};
   }
 
-  auto table_build_result =
-    build_tables::build_tables(syntax_grammar, lexical_grammar);
-  const ParseTable &parse_table = get<0>(table_build_result);
-  const LexTable &lex_table = get<1>(table_build_result);
-  error = get<2>(table_build_result);
-  if (error.type) {
-    return { nullptr, strdup(error.message.c_str()), error.type };
+  auto builder = build_tables::ParseTableBuilder::create(syntax_grammar, lexical_grammar);
+  auto build_tables_result = builder->build();
+  error = build_tables_result.error;
+  if (error.type != 0) {
+    return {nullptr, strdup(error.message.c_str()), error.type};
   }
 
-  string code = generate_code::c_code(parse_result.name, parse_table, lex_table,
-                                      syntax_grammar, lexical_grammar);
-
-  return { strdup(code.c_str()), nullptr, TSCompileErrorTypeNone };
-}
-
-pair<string, const CompileError> compile(const Grammar &grammar,
-                                         std::string name) {
-  auto prepare_grammar_result = prepare_grammar::prepare_grammar(grammar);
-  const SyntaxGrammar &syntax_grammar = get<0>(prepare_grammar_result);
-  const LexicalGrammar &lexical_grammar = get<1>(prepare_grammar_result);
-  CompileError error = get<2>(prepare_grammar_result);
-  if (error.type)
-    return { "", error };
-
-  auto table_build_result =
-    build_tables::build_tables(syntax_grammar, lexical_grammar);
-  const ParseTable &parse_table = get<0>(table_build_result);
-  const LexTable &lex_table = get<1>(table_build_result);
-  error = get<2>(table_build_result);
-  if (error.type)
-    return { "", error };
-
-  string code = generate_code::c_code(name, parse_table, lex_table,
-                                      syntax_grammar, lexical_grammar);
-
-  return { code, CompileError::none() };
+  string code = generate_code::c_code(
+    parse_result.name,
+    move(build_tables_result.parse_table),
+    move(build_tables_result.main_lex_table),
+    move(build_tables_result.keyword_lex_table),
+    build_tables_result.keyword_capture_token,
+    move(syntax_grammar),
+    move(lexical_grammar)
+  );
+
+  return {
+    strdup(code.c_str()), nullptr, TSCompileErrorTypeNone };
 }
 
 }  // namespace tree_sitter

data/ext/tree-sitter/tree-sitter/src/compiler/compile_error.h

@@ -8,6 +8,8 @@ namespace tree_sitter {
 
 class CompileError {
  public:
+  CompileError() : type(TSCompileErrorTypeNone) {}
+
   CompileError(TSCompileErrorType type, std::string message)
       : type(type), message(message) {}
 
@@ -15,6 +17,10 @@ class CompileError {
     return CompileError(TSCompileErrorTypeNone, "");
   }
 
+  operator bool() const {
+    return type != TSCompileErrorTypeNone;
+  }
+
   bool operator==(const CompileError &other) const {
     return type == other.type && message == other.message;
   }

data/ext/tree-sitter/tree-sitter/src/compiler/generate_code/c_code.cc

@@ -9,7 +9,7 @@
 #include "compiler/parse_table.h"
 #include "compiler/syntax_grammar.h"
 #include "compiler/lexical_grammar.h"
-#include "compiler/rules/built_in_symbols.h"
+#include "compiler/rule.h"
 #include "compiler/util/string_helpers.h"
 #include "tree_sitter/runtime.h"
 
@@ -18,6 +18,7 @@ namespace generate_code {
 
 using std::function;
 using std::map;
+using std::move;
 using std::pair;
 using std::set;
 using std::string;
@@ -25,8 +26,7 @@ using std::to_string;
 using std::vector;
 using util::escape_char;
 using rules::Symbol;
-
-static Variable EOF_ENTRY("end", VariableTypeNamed, rule_ptr());
+using rules::Alias;
 
 static const map<char, string> REPLACEMENTS({
   { '~', "TILDE" },
@@ -71,24 +71,29 @@ class CCodeGenerator {
 
   const string name;
   const ParseTable parse_table;
-  const LexTable lex_table;
+  const LexTable main_lex_table;
+  const LexTable keyword_lex_table;
+  Symbol keyword_capture_token;
   const SyntaxGrammar syntax_grammar;
   const LexicalGrammar lexical_grammar;
   map<string, string> sanitized_names;
   vector<pair<size_t, ParseTableEntry>> parse_table_entries;
   vector<set<Symbol::Index>> external_scanner_states;
   size_t next_parse_action_list_index;
+  set<Alias> unique_aliases;
 
  public:
-  CCodeGenerator(string name, const ParseTable &parse_table,
-                 const LexTable &lex_table, const SyntaxGrammar &syntax_grammar,
-                 const LexicalGrammar &lexical_grammar)
+  CCodeGenerator(string name, ParseTable &&parse_table, LexTable &&main_lex_table,
+                 LexTable &&keyword_lex_table, Symbol keyword_capture_token,
+                 SyntaxGrammar &&syntax_grammar, LexicalGrammar &&lexical_grammar)
       : indent_level(0),
         name(name),
-        parse_table(parse_table),
-        lex_table(lex_table),
-        syntax_grammar(syntax_grammar),
-        lexical_grammar(lexical_grammar),
+        parse_table(move(parse_table)),
+        main_lex_table(move(main_lex_table)),
+        keyword_lex_table(move(keyword_lex_table)),
+        keyword_capture_token(keyword_capture_token),
+        syntax_grammar(move(syntax_grammar)),
+        lexical_grammar(move(lexical_grammar)),
         next_parse_action_list_index(0) {}
 
   string code() {
@@ -100,7 +105,17 @@ class CCodeGenerator {
     add_symbol_enum();
     add_symbol_names_list();
     add_symbol_metadata_list();
-    add_lex_function();
+
+    if (parse_table.alias_sequences.size() > 1) {
+      add_alias_sequences();
+    }
+
+    add_lex_function("ts_lex", main_lex_table);
+
+    if (keyword_capture_token != rules::NONE()) {
+      add_lex_function("ts_lex_keywords", keyword_lex_table);
+    }
+
     add_lex_modes_list();
 
     if (!syntax_grammar.external_tokens.empty()) {
@@ -122,24 +137,41 @@ class CCodeGenerator {
   }
 
   void add_warning_pragma() {
+    line("#if defined(__GNUC__) || defined(__clang__)");
     line("#pragma GCC diagnostic push");
     line("#pragma GCC diagnostic ignored \"-Wmissing-field-initializers\"");
+    line("#endif");
     line();
   }
 
   void add_stats() {
-    size_t token_count = 1 + lexical_grammar.variables.size();
-    for (const ExternalToken &external_token : syntax_grammar.external_tokens) {
-      if (external_token.corresponding_internal_token == rules::NONE()) {
+    size_t token_count = 0;
+    for (const Symbol &symbol : parse_table.symbols) {
+      if (symbol.is_terminal()) {
         token_count++;
+      } else if (symbol.is_external()) {
+        const ExternalToken &external_token = syntax_grammar.external_tokens[symbol.index];
+        if (external_token.corresponding_internal_token == rules::NONE()) {
+          token_count++;
+        }
+      }
+    }
+
+    for (const AliasSequence &alias_sequence : parse_table.alias_sequences) {
+      for (const Alias &alias : alias_sequence) {
+        if (!alias.value.empty()) {
+          unique_aliases.insert(alias);
+        }
       }
     }
 
     line("#define LANGUAGE_VERSION " + to_string(TREE_SITTER_LANGUAGE_VERSION));
     line("#define STATE_COUNT " + to_string(parse_table.states.size()));
     line("#define SYMBOL_COUNT " + to_string(parse_table.symbols.size()));
+    line("#define ALIAS_COUNT " + to_string(unique_aliases.size()));
     line("#define TOKEN_COUNT " + to_string(token_count));
     line("#define EXTERNAL_TOKEN_COUNT " + to_string(syntax_grammar.external_tokens.size()));
+    line("#define MAX_ALIAS_SEQUENCE_LENGTH " + to_string(parse_table.max_alias_sequence_length));
     line();
   }
 
@@ -147,13 +179,17 @@ class CCodeGenerator {
     line("enum {");
     indent([&]() {
       size_t i = 1;
-      for (const auto &entry : parse_table.symbols) {
-        const Symbol &symbol = entry.first;
+      for (const Symbol &symbol : parse_table.symbols) {
         if (!symbol.is_built_in()) {
           line(symbol_id(symbol) + " = " + to_string(i) + ",");
           i++;
         }
       }
+
+      for (const Alias &alias : unique_aliases) {
+        line(alias_id(alias) + " = " + to_string(i) + ",");
+        i++;
+      }
     });
     line("};");
     line();
@@ -162,19 +198,53 @@ class CCodeGenerator {
   void add_symbol_names_list() {
     line("static const char *ts_symbol_names[] = {");
     indent([&]() {
-      for (const auto &entry : parse_table.symbols)
-        line("[" + symbol_id(entry.first) + "] = \"" +
-             sanitize_name_for_string(symbol_name(entry.first)) + "\",");
+      for (const Symbol &symbol : parse_table.symbols) {
+        line(
+          "[" + symbol_id(symbol) + "] = \"" +
+          sanitize_name_for_string(symbol_name(symbol)) + "\","
+        );
+      }
+
+      for (const Alias &alias : unique_aliases) {
+        line(
+          "[" + alias_id(alias) + "] = \"" +
+          sanitize_name_for_string(alias.value) + "\","
+        );
+      }
+    });
+    line("};");
+    line();
+  }
+
+  void add_alias_sequences() {
+    line(
+      "static TSSymbol ts_alias_sequences[" +
+      to_string(parse_table.alias_sequences.size()) +
+      "][MAX_ALIAS_SEQUENCE_LENGTH] = {"
+    );
+
+    indent([&]() {
+      for (unsigned i = 1, n = parse_table.alias_sequences.size(); i < n; i++) {
+        const AliasSequence &sequence = parse_table.alias_sequences[i];
+        line("[" + to_string(i) + "] = {");
+        indent([&]() {
+          for (unsigned j = 0, n = sequence.size(); j < n; j++) {
+            if (!sequence[j].value.empty()) {
+              line("[" + to_string(j) + "] = " + alias_id(sequence[j]) + ",");
+            }
+          }
+        });
+        line("},");
+      }
     });
     line("};");
     line();
   }
 
   void add_symbol_metadata_list() {
-    line("static const TSSymbolMetadata ts_symbol_metadata[SYMBOL_COUNT] = {");
+    line("static const TSSymbolMetadata ts_symbol_metadata[] = {");
     indent([&]() {
-      for (const auto &entry : parse_table.symbols) {
-        const Symbol &symbol = entry.first;
+      for (const Symbol &symbol : parse_table.symbols) {
         line("[" + symbol_id(symbol) + "] = {");
         indent([&]() {
           switch (symbol_type(symbol)) {
@@ -195,27 +265,34 @@ class CCodeGenerator {
               line(".named = false,");
               break;
           }
-
-          line(".structural = " + _boolean(entry.second.structural) + ",");
-          line(".extra = " + _boolean(entry.second.extra) + ",");
         });
 
         line("},");
       }
+
+      for (const Alias &alias : unique_aliases) {
+        line("[" + alias_id(alias) + "] = {");
+        indent([&]() {
+          line(".visible = true,");
+          line(".named = " + _boolean(alias.is_named) + ",");
+        });
+        line("},");
+      }
     });
     line("};");
     line();
   }
 
-  void add_lex_function() {
-    line("static bool ts_lex(TSLexer *lexer, TSStateId state) {");
+  void add_lex_function(string name, const LexTable &lex_table) {
+    line("static bool " + name + "(TSLexer *lexer, TSStateId state) {");
     indent([&]() {
       line("START_LEXER();");
       _switch("state", [&]() {
         size_t i = 0;
-        for (const LexState &state : lex_table.states)
+        for (const LexState &state : lex_table.states) {
           _case(to_string(i++), [&]() { add_lex_state(state); });
-        _default([&]() { line("LEX_ERROR();"); });
+        }
+        _default([&]() { line("return false;"); });
       });
     });
     line("}");
@@ -244,14 +321,12 @@ class CCodeGenerator {
         line("[" + to_string(state_id++) + "] = {.lex_state = ");
         add(to_string(state.lex_state_id));
 
-        bool needs_external_scanner = false;
         set<Symbol::Index> external_token_indices;
         for (const auto &pair : state.terminal_entries) {
           Symbol symbol = pair.first;
           if (symbol.is_external()) {
-            needs_external_scanner = true;
             external_token_indices.insert(symbol.index);
-          } else if (symbol.is_token()) {
+          } else if (symbol.is_terminal()) {
             auto corresponding_external_token =
               external_tokens_by_corresponding_internal_token.find(symbol.index);
             if (corresponding_external_token != external_tokens_by_corresponding_internal_token.end()) {
@@ -260,7 +335,7 @@ class CCodeGenerator {
           }
         }
 
-        if (needs_external_scanner) {
+        if (!external_token_indices.empty()) {
           add(", .external_lex_state = " + add_external_scanner_state(external_token_indices));
         }
 
@@ -290,10 +365,10 @@ class CCodeGenerator {
   }
 
   void add_external_scanner_symbol_map() {
-    line("TSSymbol ts_external_scanner_symbol_map[EXTERNAL_TOKEN_COUNT] = {");
+    line("static TSSymbol ts_external_scanner_symbol_map[EXTERNAL_TOKEN_COUNT] = {");
     indent([&]() {
       for (size_t i = 0; i < syntax_grammar.external_tokens.size(); i++) {
-        line("[" + external_token_id(i) + "] = " + symbol_id(Symbol(i, Symbol::External)) + ",");
+        line("[" + external_token_id(i) + "] = " + symbol_id(Symbol::external(i)) + ",");
       }
     });
     line("};");
@@ -324,17 +399,17 @@ class CCodeGenerator {
   }
 
   void add_parse_table() {
-    add_parse_action_list_id(ParseTableEntry{ {}, false, false });
+    add_parse_action_list_id(ParseTableEntry{ {}, false });
 
     size_t state_id = 0;
-    line("static unsigned short ts_parse_table[STATE_COUNT][SYMBOL_COUNT] = {");
+    line("static uint16_t ts_parse_table[STATE_COUNT][SYMBOL_COUNT] = {");
 
     indent([&]() {
       for (const auto &state : parse_table.states) {
         line("[" + to_string(state_id++) + "] = {");
         indent([&]() {
           for (const auto &entry : state.nonterminal_entries) {
-            line("[" + symbol_id(Symbol(entry.first, Symbol::NonTerminal)) + "] = STATE(");
+            line("[" + symbol_id(Symbol::non_terminal(entry.first)) + "] = STATE(");
             add(to_string(entry.second));
             add("),");
           }
@@ -360,98 +435,182 @@ class CCodeGenerator {
 
     if (!syntax_grammar.external_tokens.empty()) {
       line("void *" + external_scanner_name + "_create();");
-      line("void " + external_scanner_name + "_destroy();");
-      line("void " + external_scanner_name + "_reset(void *);");
+      line("void " + external_scanner_name + "_destroy(void *);");
       line("bool " + external_scanner_name + "_scan(void *, TSLexer *, const bool *);");
-      line("bool " + external_scanner_name + "_serialize(void *, TSExternalTokenState);");
-      line("void " + external_scanner_name + "_deserialize(void *, const TSExternalTokenState);");
+      line("unsigned " + external_scanner_name + "_serialize(void *, char *);");
+      line("void " + external_scanner_name + "_deserialize(void *, const char *, unsigned);");
       line();
     }
 
-    line("const TSLanguage *" + language_function_name + "() {");
+    line("#ifdef _WIN32");
+    line("#define extern __declspec(dllexport)");
+    line("#endif");
+    line();
+
+    line("extern const TSLanguage *" + language_function_name + "() {");
     indent([&]() {
-      line("GET_LANGUAGE(");
-      if (syntax_grammar.external_tokens.empty()) {
-        add(");");
-      } else {
-        indent([&]() {
-          line("(const bool *)ts_external_scanner_states,");
-          line("ts_external_scanner_symbol_map,");
-          line(external_scanner_name + "_create,");
-          line(external_scanner_name + "_destroy,");
-          line(external_scanner_name + "_reset,");
-          line(external_scanner_name + "_scan,");
-          line(external_scanner_name + "_serialize,");
-          line(external_scanner_name + "_deserialize,");
-        });
-        line(");");
-      }
+      line("static TSLanguage language = {");
+      indent([&]() {
+        line(".version = LANGUAGE_VERSION,");
+        line(".symbol_count = SYMBOL_COUNT,");
+        line(".alias_count = ALIAS_COUNT,");
+        line(".token_count = TOKEN_COUNT,");
+        line(".symbol_metadata = ts_symbol_metadata,");
+        line(".parse_table = (const unsigned short *)ts_parse_table,");
+        line(".parse_actions = ts_parse_actions,");
+        line(".lex_modes = ts_lex_modes,");
+        line(".symbol_names = ts_symbol_names,");
+
+        if (parse_table.alias_sequences.size() > 1) {
+          line(".alias_sequences = (const TSSymbol *)ts_alias_sequences,");
+        }
+
+        line(".max_alias_sequence_length = MAX_ALIAS_SEQUENCE_LENGTH,");
+        line(".lex_fn = ts_lex,");
+
+        if (keyword_capture_token != rules::NONE()) {
+          line(".keyword_lex_fn = ts_lex_keywords,");
+          line(".keyword_capture_token = " + symbol_id(keyword_capture_token) + ",");
+        }
+
+        line(".external_token_count = EXTERNAL_TOKEN_COUNT,");
+
+        if (!syntax_grammar.external_tokens.empty()) {
+          line(".external_scanner = {");
+          indent([&]() {
+            line("(const bool *)ts_external_scanner_states,");
+            line("ts_external_scanner_symbol_map,");
+            line(external_scanner_name + "_create,");
+            line(external_scanner_name + "_destroy,");
+            line(external_scanner_name + "_scan,");
+            line(external_scanner_name + "_serialize,");
+            line(external_scanner_name + "_deserialize,");
+          });
+          line("},");
+        }
+      });
+
+      line("};");
+      line("return &language;");
     });
     line("}");
     line();
   }
 
   void add_lex_state(const LexState &lex_state) {
-    if (lex_state.is_token_start)
-      line("START_TOKEN();");
+    if (lex_state.accept_action.is_present()) {
+      add_accept_token_action(lex_state.accept_action);
+    }
 
-    for (const auto &pair : lex_state.advance_actions)
-      if (!pair.first.is_empty())
-        _if([&]() { add_character_set_condition(pair.first); },
-            [&]() { add_advance_action(pair.second); });
+    set<uint32_t> ruled_out_characters;
+    for (const auto &pair : lex_state.advance_actions) {
+      if (pair.first.is_empty()) continue;
 
-    if (lex_state.accept_action.is_present())
-      add_accept_token_action(lex_state.accept_action);
-    else
-      line("LEX_ERROR();");
+      size_t current_length = buffer.size();
+
+      line("if (");
+      if (add_character_set_condition(pair.first, ruled_out_characters)) {
+        add(")");
+        indent([&]() { add_advance_action(pair.second); });
+        ruled_out_characters.insert(pair.first.included_chars.begin(), pair.first.included_chars.end());
+      } else {
+        buffer.resize(current_length);
+        add_advance_action(pair.second);
+      }
+    }
+
+    line("END_STATE();");
   }
 
-  void add_character_set_condition(const rules::CharacterSet &rule) {
+  bool add_character_set_condition(const rules::CharacterSet &rule, const set<uint32_t> &ruled_out_characters) {
     if (rule.includes_all) {
-      add("!(");
-      add_character_range_conditions(rule.excluded_ranges());
-      add(")");
+      return add_character_range_conditions(rule.excluded_ranges(), ruled_out_characters, true);
     } else {
-      add_character_range_conditions(rule.included_ranges());
+      return add_character_range_conditions(rule.included_ranges(), ruled_out_characters, false);
     }
   }
 
-  void add_character_range_conditions(const vector<rules::CharacterRange> &ranges) {
-    if (ranges.size() == 1) {
-      add_character_range_condition(*ranges.begin());
-    } else {
-      bool first = true;
-      for (const auto &range : ranges) {
-        if (!first) {
-          add(" ||");
-          line();
-          add_padding();
+  bool add_character_range_conditions(const vector<rules::CharacterRange> &ranges,
+                                      const set<uint32_t> &ruled_out_characters,
+                                      bool is_negated) {
+    bool first = true;
+    for (auto iter = ranges.begin(), end = ranges.end(); iter != end;) {
+      auto range = *iter;
+
+      bool range_is_ruled_out = true;
+      for (uint32_t c = range.min; c <= range.max; c++) {
+        if (!ruled_out_characters.count(c)) {
+          range_is_ruled_out = false;
+          break;
         }
+      }
 
-        add("(");
-        add_character_range_condition(range);
-        add(")");
+      if (range_is_ruled_out) {
+        ++iter;
+        continue;
+      }
+
+      auto next_iter = iter + 1;
+      while (next_iter != end) {
+        bool can_join_ranges = true;
+        for (uint32_t character = range.max + 1; character < next_iter->min; character++) {
+          if (!ruled_out_characters.count(character)) {
+            can_join_ranges = false;
+            break;
+          }
+        }
 
-        first = false;
+        if (can_join_ranges) {
+          range.max = next_iter->max;
+          ++next_iter;
+        } else {
+          break;
+        }
+      }
+
+      if (!first) {
+        add(is_negated ? " &&" : " ||");
+        line("    ");
       }
+
+      add_character_range_condition(range, is_negated);
+      first = false;
+      iter = next_iter;
     }
+
+    return !first;
   }
 
-  void add_character_range_condition(const rules::CharacterRange &range) {
-    string lookahead("lookahead");
-    if (range.min == range.max) {
-      add(lookahead + " == " + escape_char(range.min));
+  void add_character_range_condition(const rules::CharacterRange &range, bool is_negated) {
+    auto min = escape_char(range.min);
+    auto max = escape_char(range.max);
+    if (is_negated) {
+      if (range.max == range.min) {
+        add("lookahead != " + min);
+      } else if (range.max == range.min + 1) {
+        add("lookahead != " + min + " &&");
+        line("    lookahead != " + max);
+      } else {
+        add("(lookahead < " + min + " || lookahead > " + max + ")");
+      }
     } else {
-      add(escape_char(range.min) + string(" <= ") + lookahead + " && " +
-          lookahead + " <= " + escape_char(range.max));
+      if (range.max == range.min) {
+        add("lookahead == " + min);
+      } else if (range.max == range.min + 1) {
+        add("lookahead == " + min + " ||");
+        line("    lookahead == " + max);
+      } else {
+        add("(" + min + " <= lookahead && lookahead <= " + max + ")");
+      }
     }
   }
 
   void add_advance_action(const AdvanceAction &action) {
-    if (action.in_main_token)
+    if (action.in_main_token) {
       line("ADVANCE(" + to_string(action.state_index) + ");");
-    else
+    } else {
       line("SKIP(" + to_string(action.state_index) + ");");
+    }
   }
 
   void add_accept_token_action(const AcceptTokenAction &action) {
@@ -464,10 +623,12 @@ class CCodeGenerator {
     indent([&]() {
       for (const auto &pair : parse_table_entries) {
         size_t index = pair.first;
-        line("[" + to_string(index) + "] = {.count = " +
-             to_string(pair.second.actions.size()) + ", .reusable = " +
-             _boolean(pair.second.reusable) + ", .depends_on_lookahead = " +
-             _boolean(pair.second.depends_on_lookahead) + "},");
+        line(
+          "[" + to_string(index) + "] = {"
+            ".count = " + to_string(pair.second.actions.size()) + ", "
+            ".reusable = " + _boolean(pair.second.reusable) +
+          "},"
+        );
 
         for (const ParseAction &action : pair.second.actions) {
           add(" ");
@@ -480,21 +641,30 @@ class CCodeGenerator {
             case ParseActionTypeShift:
               if (action.extra) {
                 add("SHIFT_EXTRA()");
+              } else if (action.repetition) {
+                add("SHIFT_REPEAT(" + to_string(action.state_index) + ")");
               } else {
                 add("SHIFT(" + to_string(action.state_index) + ")");
               }
               break;
             case ParseActionTypeReduce:
-              if (action.fragile) {
-                add("REDUCE_FRAGILE(" + symbol_id(action.symbol) + ", " +
-                    to_string(action.consumed_symbol_count) + ")");
-              } else {
-                add("REDUCE(" + symbol_id(action.symbol) + ", " +
-                    to_string(action.consumed_symbol_count) + ")");
+              add("REDUCE(");
+              add(symbol_id(action.symbol));
+              add(", ");
+              add(to_string(action.consumed_symbol_count));
+
+              if (action.dynamic_precedence != 0) {
+                add(", .dynamic_precedence = " + to_string(action.dynamic_precedence));
+              }
+
+              if (action.alias_sequence_id != 0) {
+                add(", .alias_sequence_id = " + to_string(action.alias_sequence_id));
               }
+
+              add(")");
               break;
             case ParseActionTypeRecover:
-              add("RECOVER(" + to_string(action.state_index) + ")");
+              add("RECOVER()");
               break;
             default: {}
           }
@@ -522,7 +692,7 @@ class CCodeGenerator {
   // Helper functions
 
   string external_token_id(Symbol::Index index) {
-    return "ts_external_token_" + syntax_grammar.external_tokens[index].name;
+    return "ts_external_token_" + sanitize_name(syntax_grammar.external_tokens[index].name);
   }
 
   string symbol_id(const Symbol &symbol) {
@@ -542,6 +712,14 @@ class CCodeGenerator {
     }
   }
 
+  string alias_id(const Alias &alias) {
+    if (alias.is_named) {
+      return "alias_sym_" + sanitize_name(alias.value);
+    } else {
+      return "anon_alias_sym_" + sanitize_name(alias.value);
+    }
+  }
+
   string symbol_name(const Symbol &symbol) {
     if (symbol == rules::END_OF_INPUT())
       return "END";
@@ -561,7 +739,7 @@ class CCodeGenerator {
         return { variable.name, variable.type };
       }
       case Symbol::Terminal: {
-        const Variable &variable = lexical_grammar.variables[symbol.index];
+        const LexicalVariable &variable = lexical_grammar.variables[symbol.index];
         return { variable.name, variable.type };
       }
       case Symbol::External:
@@ -590,13 +768,6 @@ class CCodeGenerator {
     indent(body);
   }
 
-  void _if(function<void()> condition, function<void()> body) {
-    line("if (");
-    indent(condition);
-    add(")");
-    indent(body);
-  }
-
   string sanitize_name_for_string(string name) {
     util::str_replace(&name, "\\", "\\\\");
     util::str_replace(&name, "\n", "\\n");
@@ -665,7 +836,7 @@ class CCodeGenerator {
 
   void add_padding() {
     for (size_t i = 0; i < indent_level; i++)
-      add("    ");
+      add("  ");
   }
 
   void indent(function<void()> body) {
@@ -679,12 +850,18 @@ class CCodeGenerator {
   }
 };
 
-string c_code(string name, const ParseTable &parse_table,
-              const LexTable &lex_table, const SyntaxGrammar &syntax_grammar,
-              const LexicalGrammar &lexical_grammar) {
-  return CCodeGenerator(name, parse_table, lex_table, syntax_grammar,
-                        lexical_grammar)
-    .code();
+string c_code(string name, ParseTable &&parse_table, LexTable &&lex_table,
+              LexTable &&keyword_lex_table, Symbol keyword_capture_token,
+              SyntaxGrammar &&syntax_grammar, LexicalGrammar &&lexical_grammar) {
+  return CCodeGenerator(
+    name,
+    move(parse_table),
+    move(lex_table),
+    move(keyword_lex_table),
+    keyword_capture_token,
+    move(syntax_grammar),
+    move(lexical_grammar)
+  ).code();
 }
 
 }  // namespace generate_code