tree-sitter 0.0.1 → 0.1.0

data/ext/tree-sitter/tree-sitter/src/compiler/prepare_grammar/expand_repeats.h

@@ -6,8 +6,6 @@
 namespace tree_sitter {
 namespace prepare_grammar {
 
-struct InitialSyntaxGrammar;
-
 InitialSyntaxGrammar expand_repeats(const InitialSyntaxGrammar &);
 
 }  // namespace prepare_grammar

data/ext/tree-sitter/tree-sitter/src/compiler/prepare_grammar/expand_tokens.cc

@@ -2,15 +2,8 @@
 #include <vector>
 #include <string>
 #include <utility>
-#include <map>
 #include "compiler/lexical_grammar.h"
-#include "compiler/rules/visitor.h"
-#include "compiler/rules/pattern.h"
-#include "compiler/rules/string.h"
-#include "compiler/rules/blank.h"
-#include "compiler/rules/seq.h"
-#include "compiler/rules/metadata.h"
-#include "compiler/rules/character_set.h"
+#include "compiler/rule.h"
 #include "compiler/prepare_grammar/parse_regex.h"
 #include "utf8proc.h"
 
@@ -19,70 +12,69 @@ namespace prepare_grammar {
 
 using std::string;
 using std::vector;
-using std::map;
-using std::pair;
-using std::make_shared;
-using rules::String;
-using rules::Pattern;
-using rules::Metadata;
+using rules::Rule;
 
-class ExpandTokens : public rules::IdentityRuleFn {
-  using rules::IdentityRuleFn::apply_to;
+ExpandTokenResult expand_token(const rules::Rule &rule) {
+  return rule.match(
+    [](const rules::Blank &blank) -> ExpandTokenResult { return Rule(blank); },
 
-  rule_ptr apply_to(const String *rule) {
-    vector<rule_ptr> elements;
-    const uint8_t *iter = reinterpret_cast<const uint8_t *>(rule->value.data());
-    const uint8_t *end = iter + rule->value.size();
+    [](const rules::String &string) {
+      vector<Rule> elements;
+      const uint8_t *iter = reinterpret_cast<const uint8_t *>(string.value.data());
+      const uint8_t *end = iter + string.value.size();
 
-    while (iter < end) {
-      int32_t el;
-      size_t size = utf8proc_iterate(iter, (end - iter), &el);
-      if (!size)
-        break;
-      iter += size;
+      while (iter < end) {
+        int32_t el;
+        size_t size = utf8proc_iterate(iter, (end - iter), &el);
+        if (!size)
+          break;
+        iter += size;
 
-      elements.push_back(rules::CharacterSet().include(el).copy());
-    }
+        elements.push_back(rules::CharacterSet().include(el));
+      }
 
-    rules::MetadataParams params;
-    params.is_token = true;
-    params.is_string = true;
+      return Rule::seq(elements);
+    },
 
-    return rules::Metadata::build(rules::Seq::build(elements), params);
-  }
+    [](const rules::Pattern &pattern) -> ExpandTokenResult {
+      auto result = parse_regex(pattern.value);
+      if (result.second) return result.second;
+      return result.first;
+    },
 
-  rule_ptr apply_to(const Pattern *rule) {
-    auto pair = parse_regex(rule->value);
-    if (!error.type)
-      error = pair.second;
-    return pair.first;
-  }
+    [](const rules::Repeat &rule) -> ExpandTokenResult {
+      auto result = expand_token(*rule.rule);
+      if (result.error) return result.error;
+      return Rule::repeat(result.rule);
+    },
 
- public:
-  CompileError error;
-  ExpandTokens() : error(CompileError::none()) {}
-};
-
-pair<LexicalGrammar, CompileError> expand_tokens(const LexicalGrammar &grammar) {
-  LexicalGrammar result;
-  ExpandTokens expander;
+    [](const rules::Metadata &rule) -> ExpandTokenResult {
+      auto result = expand_token(*rule.rule);
+      if (result.error) return result.error;
+      return Rule(rules::Metadata{result.rule, rule.params});
+    },
 
-  for (const Variable &variable : grammar.variables) {
-    auto rule = expander.apply(variable.rule);
-    if (expander.error.type)
-      return { result, expander.error };
-    result.variables.push_back(Variable(variable.name, variable.type, rule));
-  }
+    [](const rules::Seq &rule) -> ExpandTokenResult {
+      auto left_result = expand_token(*rule.left);
+      if (left_result.error) return left_result.error;
+      auto right_result = expand_token(*rule.right);
+      if (right_result.error) return right_result.error;
+      return Rule(rules::Seq{left_result.rule, right_result.rule});
+    },
 
-  for (auto &sep : grammar.separators) {
-    auto rule = expander.apply(sep);
-    if (expander.error.type)
-      return { result, expander.error };
-    result.separators.push_back(rule);
-  }
+    [](const rules::Choice &rule) -> ExpandTokenResult {
+      std::vector<Rule> elements;
+      for (const auto &element : rule.elements) {
+        auto result = expand_token(element);
+        if (result.error) return result.error;
+        elements.push_back(result.rule);
+      }
+      return Rule(rules::Choice{elements});
+    },
 
-  return { result, CompileError::none() };
-}
+    [](auto) { return CompileError(TSCompileErrorTypeInvalidTokenContents, "Symbols inside tokens are not allowed."); }
+  );
+};
 
 }  // namespace prepare_grammar
 }  // namespace tree_sitter

data/ext/tree-sitter/tree-sitter/src/compiler/prepare_grammar/expand_tokens.h

@@ -2,15 +2,21 @@
 #define COMPILER_PREPARE_GRAMMAR_EXPAND_TOKENS_H_
 
 #include <utility>
+#include "compiler/rule.h"
 #include "compiler/compile_error.h"
 
 namespace tree_sitter {
+namespace prepare_grammar {
 
-struct LexicalGrammar;
+struct ExpandTokenResult {
+  rules::Rule rule;
+  CompileError error;
 
-namespace prepare_grammar {
+  ExpandTokenResult(const rules::Rule &rule) : rule(rule) {}
+  ExpandTokenResult(const CompileError &error) : error(error) {}
+};
 
-std::pair<LexicalGrammar, CompileError> expand_tokens(const LexicalGrammar &);
+ExpandTokenResult expand_token(const rules::Rule &);
 
 }  // namespace prepare_grammar
 }  // namespace tree_sitter

data/ext/tree-sitter/tree-sitter/src/compiler/prepare_grammar/extract_choices.cc

@@ -1,54 +1,48 @@
 #include "compiler/prepare_grammar/extract_choices.h"
 #include <vector>
 #include <memory>
-#include "compiler/rules/visitor.h"
-#include "compiler/rules/seq.h"
-#include "compiler/rules/choice.h"
-#include "compiler/rules/metadata.h"
-#include "compiler/rules/repeat.h"
+#include "compiler/rule.h"
 
 namespace tree_sitter {
 namespace prepare_grammar {
 
 using std::vector;
-
-class ExtractChoices : public rules::RuleFn<vector<rule_ptr>> {
-  vector<rule_ptr> default_apply(const Rule *rule) {
-    return vector<rule_ptr>({ rule->copy() });
-  }
-
-  vector<rule_ptr> apply_to(const rules::Seq *rule) {
-    vector<rule_ptr> result;
-    for (auto left_entry : apply(rule->left))
-      for (auto right_entry : apply(rule->right))
-        result.push_back(rules::Seq::build({ left_entry, right_entry }));
-    return result;
-  }
-
-  vector<rule_ptr> apply_to(const rules::Metadata *rule) {
-    vector<rule_ptr> result;
-    for (auto entry : apply(rule->rule))
-      result.push_back(rules::Metadata::build(entry, rule->params));
-    return result;
-  }
-
-  vector<rule_ptr> apply_to(const rules::Choice *rule) {
-    vector<rule_ptr> result;
-    for (auto element : rule->elements)
-      for (auto entry : apply(element))
-        result.push_back(entry);
-    return result;
-  }
-
-  vector<rule_ptr> apply_to(const rules::Repeat *rule) {
-    return vector<rule_ptr>({
-      rules::Repeat::build(rules::Choice::build(apply(rule->content))),
-    });
-  }
-};
-
-std::vector<rule_ptr> extract_choices(const rule_ptr &rule) {
-  return ExtractChoices().apply(rule);
+using rules::Rule;
+
+vector<Rule> extract_choices(const Rule &rule) {
+  return rule.match(
+    [](const rules::Seq &sequence) {
+      vector<Rule> result;
+      for (auto &left_entry : extract_choices(*sequence.left)) {
+        for (auto &right_entry : extract_choices(*sequence.right)) {
+          result.push_back(rules::Rule::seq({left_entry, right_entry}));
+        }
+      }
+      return result;
+    },
+
+    [](const rules::Metadata &rule) {
+      vector<Rule> result;
+      for (auto &entry : extract_choices(*rule.rule)) {
+        result.push_back(rules::Metadata{entry, rule.params});
+      }
+      return result;
+    },
+
+    [](const rules::Choice &choice) {
+      vector<Rule> result;
+      for (auto &element : choice.elements) {
+        for (auto &entry : extract_choices(element)) {
+          result.push_back(entry);
+        }
+      }
+      return result;
+    },
+
+    [](const auto &rule) {
+      return vector<Rule>({rule});
+    }
+  );
 }
 
 }  // namespace prepare_grammar

data/ext/tree-sitter/tree-sitter/src/compiler/prepare_grammar/extract_choices.h

@@ -7,7 +7,7 @@
 namespace tree_sitter {
 namespace prepare_grammar {
 
-std::vector<rule_ptr> extract_choices(const rule_ptr &);
+std::vector<rules::Rule> extract_choices(const rules::Rule &);
 
 }  // namespace prepare_grammar
 }  // namespace tree_sitter

data/ext/tree-sitter/tree-sitter/src/compiler/prepare_grammar/extract_tokens.cc

@@ -1,24 +1,19 @@
 #include "compiler/prepare_grammar/extract_tokens.h"
 #include <map>
 #include <vector>
+#include <cassert>
 #include <set>
 #include <string>
 #include <tuple>
 #include "tree_sitter/compiler.h"
 #include "compiler/lexical_grammar.h"
-#include "compiler/prepare_grammar/initial_syntax_grammar.h"
-#include "compiler/rules/visitor.h"
-#include "compiler/rules/symbol.h"
-#include "compiler/rules/string.h"
-#include "compiler/rules/metadata.h"
-#include "compiler/rules/pattern.h"
+#include "compiler/rule.h"
 #include "compiler/prepare_grammar/token_description.h"
-#include "compiler/prepare_grammar/is_token.h"
+#include "compiler/prepare_grammar/expand_tokens.h"
 
 namespace tree_sitter {
 namespace prepare_grammar {
 
-using std::make_shared;
 using std::make_tuple;
 using std::map;
 using std::set;
@@ -26,169 +21,289 @@ using std::string;
 using std::tuple;
 using std::vector;
 using rules::Symbol;
+using rules::Rule;
+using rules::Rule;
 
-class SymbolReplacer : public rules::IdentityRuleFn {
-  using rules::IdentityRuleFn::apply_to;
-
-  rule_ptr apply_to(const Symbol *rule) {
-    return replace_symbol(*rule).copy();
-  }
-
+class SymbolReplacer {
  public:
   map<Symbol, Symbol> replacements;
 
+  Rule apply(const Rule &rule) {
+    return rule.match(
+      [this](const rules::Blank &blank) -> Rule {
+        return blank;
+      },
+
+      [this](const rules::Symbol &symbol) {
+        return replace_symbol(symbol);
+      },
+
+      [this](const rules::Choice &choice) {
+        vector<rules::Rule> elements;
+        for (const auto &element : choice.elements) {
+          elements.push_back(apply(element));
+        }
+        return Rule::choice(elements);
+      },
+
+      [this](const rules::Seq &sequence) {
+        return rules::Seq{
+          apply(*sequence.left),
+          apply(*sequence.right)
+        };
+      },
+
+      [this](const rules::Repeat &repeat) {
+        return Rule::repeat(apply(*repeat.rule));
+      },
+
+      [this](const rules::Metadata &metadata) {
+        return rules::Metadata{apply(*metadata.rule), metadata.params};
+      },
+
+      [](auto) {
+        assert(!"Unexpected rule type");
+        return rules::Blank{};
+      }
+    );
+  }
+
   Symbol replace_symbol(const Symbol &symbol) {
-    if (!symbol.is_non_terminal())
-      return symbol;
+    if (!symbol.is_non_terminal()) return symbol;
 
     auto replacement_pair = replacements.find(symbol);
-    if (replacement_pair != replacements.end())
+    if (replacement_pair != replacements.end()) {
       return replacement_pair->second;
+    }
 
     int new_index = symbol.index;
-    for (const auto &pair : replacements)
-      if (pair.first.index < symbol.index)
+    for (const auto &pair : replacements) {
+      if (pair.first.index < symbol.index) {
         new_index--;
-    return Symbol(new_index, Symbol::NonTerminal);
+      }
+    }
+
+    return Symbol::non_terminal(new_index);
   }
 };
 
-class TokenExtractor : public rules::IdentityRuleFn {
-  using rules::IdentityRuleFn::apply_to;
-
-  rule_ptr apply_to_token(const Rule *input, VariableType entry_type) {
-    for (size_t i = 0; i < tokens.size(); i++)
-      if (tokens[i].rule->operator==(*input)) {
+class TokenExtractor {
+  Symbol extract_token(const rules::Rule &input, VariableType entry_type) {
+    for (size_t i = 0; i < tokens.size(); i++) {
+      if (tokens[i].rule == input) {
         token_usage_counts[i]++;
-        return make_shared<Symbol>(i, Symbol::Terminal);
+        return Symbol::terminal(i);
       }
+    }
 
-    rule_ptr rule = input->copy();
     size_t index = tokens.size();
-    tokens.push_back(Variable(token_description(rule), entry_type, rule));
+    tokens.push_back({
+      token_description(input),
+      entry_type,
+      input
+    });
     token_usage_counts.push_back(1);
-    return make_shared<Symbol>(index, Symbol::Terminal);
-  }
 
-  rule_ptr apply_to(const rules::String *rule) {
-    return apply_to_token(rule, VariableTypeAnonymous);
+    return Symbol::terminal(index);
   }
 
-  rule_ptr apply_to(const rules::Pattern *rule) {
-    return apply_to_token(rule, VariableTypeAuxiliary);
-  }
-
-  rule_ptr apply_to(const rules::Metadata *rule) {
-    if (rule->params.is_token)
-      return apply_to_token(rule->rule.get(), VariableTypeAuxiliary);
-    else
-      return rules::IdentityRuleFn::apply_to(rule);
+ public:
+  Rule apply(const rules::Rule &rule) {
+    return rule.match(
+      [this](const rules::Blank &blank) -> Rule { return blank; },
+
+      [this](const rules::Metadata &rule) -> Rule {
+        if (rule.params.is_token) {
+          rules::Metadata metadata{*rule.rule, rule.params};
+          metadata.params.is_token = false;
+          if (metadata.params == rules::MetadataParams{}) {
+            return extract_token(*metadata.rule, VariableTypeAuxiliary);
+          } else {
+            return extract_token(metadata, VariableTypeAuxiliary);
+          }
+        } else {
+          return rules::Metadata{apply(*rule.rule), rule.params};
+        }
+      },
+
+      [this](const rules::String &rule) {
+        return extract_token(rule, VariableTypeAnonymous);
+      },
+
+      [this](const rules::Pattern &rule) {
+        return extract_token(rule, VariableTypeAuxiliary);
+      },
+
+      [this](const rules::Repeat &rule) {
+        return Rule::repeat(apply(*rule.rule));
+      },
+
+      [this](const rules::Seq &rule) {
+        return Rule::seq({apply(*rule.left), apply(*rule.right)});
+      },
+
+      [this](const rules::Choice &rule) {
+        std::vector<Rule> elements;
+        for (const auto &element : rule.elements) {
+          elements.push_back(apply(element));
+        }
+        return Rule::choice(elements);
+      },
+
+      [](const rules::Symbol &symbol) {
+        return symbol;
+      },
+
+      [](auto) {
+        assert(!"Unexpected rule type");
+        return rules::Blank{};
+      }
+    );
   }
 
- public:
   vector<size_t> token_usage_counts;
   vector<Variable> tokens;
 };
 
-static CompileError extra_token_error(const string &message) {
-  return CompileError(TSCompileErrorTypeInvalidExtraToken, "Not a token: " + message);
-}
-
 tuple<InitialSyntaxGrammar, LexicalGrammar, CompileError> extract_tokens(
-  const InternedGrammar &grammar) {
+  const InternedGrammar &grammar
+) {
   InitialSyntaxGrammar syntax_grammar;
   LexicalGrammar lexical_grammar;
   SymbolReplacer symbol_replacer;
   TokenExtractor extractor;
 
-  /*
-   *  First, extract all of the grammar's tokens into the lexical grammar.
-   */
+  // Extract all of the grammar's tokens into the lexical grammar.
   vector<Variable> processed_variables;
-  for (const Variable &variable : grammar.variables)
-    processed_variables.push_back(
-      Variable(variable.name, variable.type, extractor.apply(variable.rule)));
-  lexical_grammar.variables = extractor.tokens;
-
-  /*
-   *  If a variable's entire rule was extracted as a token and that token didn't
-   *  appear within any other rule, then remove that variable from the syntax
-   *  grammar, giving its name to the token in the lexical grammar. Any symbols
-   *  that pointed to that variable will need to be updated to point to the
-   *  variable in the lexical grammar. Symbols that pointed to later variables
-   *  will need to have their indices decremented.
-   */
-  size_t i = 0;
-  for (const Variable &variable : processed_variables) {
-    auto symbol = variable.rule->as<Symbol>();
-    if (symbol && symbol->is_token() && extractor.token_usage_counts[symbol->index] == 1) {
-      lexical_grammar.variables[symbol->index].type = variable.type;
-      lexical_grammar.variables[symbol->index].name = variable.name;
-      symbol_replacer.replacements.insert({ Symbol(i, Symbol::NonTerminal), *symbol });
-    } else {
-      syntax_grammar.variables.push_back(variable);
-    }
-    i++;
+  for (const auto &variable : grammar.variables) {
+    processed_variables.push_back({
+      variable.name,
+      variable.type,
+      extractor.apply(variable.rule)
+    });
   }
 
-  /*
-   *  Perform any replacements of symbols needed based on the previous step.
-   */
-  for (Variable &variable : syntax_grammar.variables)
-    variable.rule = symbol_replacer.apply(variable.rule);
+  vector<Variable> processed_external_tokens;
+  for (const auto &external_token : grammar.external_tokens) {
+    processed_external_tokens.push_back(Variable{
+      external_token.name,
+      external_token.type,
+      extractor.apply(external_token.rule),
+    });
+  }
 
-  for (const ConflictSet &conflict_set : grammar.expected_conflicts) {
-    ConflictSet new_conflict_set;
-    for (const Symbol &symbol : conflict_set)
-      new_conflict_set.insert(symbol_replacer.replace_symbol(symbol));
-    syntax_grammar.expected_conflicts.insert(new_conflict_set);
+  for (const auto &extracted_token : extractor.tokens) {
+    auto expansion = expand_token(extracted_token.rule);
+    if (expansion.error) return make_tuple(
+      syntax_grammar,
+      lexical_grammar,
+      expansion.error
+    );
+    lexical_grammar.variables.push_back({
+      extracted_token.name,
+      extracted_token.type,
+      expansion.rule,
+      extracted_token.type == VariableTypeAnonymous
+    });
   }
 
-  /*
-   *  The grammar's extra tokens can be either token rules or symbols
-   *  pointing to token rules. If they are symbols, then they'll be handled by
-   *  the parser; add them to the syntax grammar's ubiqutous tokens. If they
-   *  are anonymous rules, they can be handled by the lexer; add them to the
-   *  lexical grammar's separator rules.
-   */
-  for (const rule_ptr &rule : grammar.extra_tokens) {
-    int i = 0;
-    bool used_elsewhere_in_grammar = false;
-    for (const Variable &variable : lexical_grammar.variables) {
-      if (variable.rule->operator==(*rule)) {
-        syntax_grammar.extra_tokens.insert(Symbol(i, Symbol::Terminal));
-        used_elsewhere_in_grammar = true;
+  // If a variable's entire rule was extracted as a token and that token didn't
+  // appear within any other rule, then remove that variable from the syntax
+  // grammar, giving its name to the token in the lexical grammar. Any symbols
+  // that pointed to that variable will need to be updated to point to the
+  // variable in the lexical grammar. Symbols that pointed to later variables
+  // will need to have their indices decremented.
+  size_t i = -1;
+  for (const auto &variable : processed_variables) {
+    i++;
+    if (i > 0 && variable.rule.is<Symbol>()) {
+      auto symbol = variable.rule.get_unchecked<Symbol>();
+      if (symbol.is_terminal() && extractor.token_usage_counts[symbol.index] == 1) {
+        lexical_grammar.variables[symbol.index].type = variable.type;
+        lexical_grammar.variables[symbol.index].name = variable.name;
+        symbol_replacer.replacements[Symbol::non_terminal(i)] = symbol;
+        continue;
       }
-      i++;
     }
+    syntax_grammar.variables.push_back(variable);
+  }
 
-    if (used_elsewhere_in_grammar)
-      continue;
+  // Perform any replacements of symbols needed based on the previous step.
+  for (auto &variable : syntax_grammar.variables) {
+    variable.rule = symbol_replacer.apply(variable.rule);
+  }
 
-    if (is_token(rule)) {
-      lexical_grammar.separators.push_back(rule);
-      continue;
+  for (const auto &conflict_set : grammar.expected_conflicts) {
+    set<Symbol> new_conflict_set;
+    for (const Symbol &symbol : conflict_set) {
+      new_conflict_set.insert(symbol_replacer.replace_symbol(symbol));
     }
+    syntax_grammar.expected_conflicts.insert(new_conflict_set);
+  }
 
-    auto symbol = rule->as<Symbol>();
-    if (!symbol)
-      return make_tuple(syntax_grammar, lexical_grammar,
-                        extra_token_error(rule->to_string()));
+  for (const Symbol &symbol : grammar.variables_to_inline) {
+    syntax_grammar.variables_to_inline.insert(symbol_replacer.replace_symbol(symbol));
+  }
 
-    Symbol new_symbol = symbol_replacer.replace_symbol(*symbol);
-    if (new_symbol.is_non_terminal()) {
-      return make_tuple(
-        syntax_grammar, lexical_grammar,
-        extra_token_error(syntax_grammar.variables[new_symbol.index].name));
-    }
+  // The grammar's extra tokens can be either token rules or symbols
+  // pointing to token rules. If they are symbols, then they'll be handled by
+  // the parser; add them to the syntax grammar's extra tokens. If they
+  // are anonymous rules, they can be handled by the lexer; add them to the
+  // lexical grammar's separator rules.
+  for (const auto &rule : grammar.extra_tokens) {
+    CompileError error = rule.match(
+      [&](const Symbol &symbol) {
+        Symbol new_symbol = symbol_replacer.replace_symbol(symbol);
+        if (new_symbol.is_non_terminal()) {
+          return CompileError(
+            TSCompileErrorTypeInvalidExtraToken,
+            "Non-token symbol " + syntax_grammar.variables[new_symbol.index].name + " can't be used as an extra token"
+          );
+        } else {
+          syntax_grammar.extra_tokens.insert(new_symbol);
+          return CompileError::none();
+        }
+      },
+
+      [&](auto non_symbol) {
+        auto expansion = expand_token(non_symbol);
+        if (expansion.error) return CompileError(
+          TSCompileErrorTypeInvalidExtraToken,
+          "Non-token rule expression can't be used as an extra token"
+        );
+        int i = 0;
+        for (const LexicalVariable &variable : lexical_grammar.variables) {
+          if (variable.rule == expansion.rule) {
+            syntax_grammar.extra_tokens.insert(Symbol::terminal(i));
+            return CompileError::none();
+          }
+          i++;
+        }
+
+        lexical_grammar.separators.push_back(expansion.rule);
+        return CompileError::none();
+      }
+    );
 
-    syntax_grammar.extra_tokens.insert(new_symbol);
+    if (error) return make_tuple(syntax_grammar, lexical_grammar, error);
   }
 
-  for (const ExternalToken &external_token : grammar.external_tokens) {
-    Symbol internal_token = symbol_replacer.replace_symbol(external_token.corresponding_internal_token);
+  for (const auto &external_token : processed_external_tokens) {
+    Rule new_rule = symbol_replacer.apply(external_token.rule);
 
-    if (internal_token.is_non_terminal()) {
+    if (!new_rule.is<Symbol>()) {
+      return make_tuple(
+        syntax_grammar,
+        lexical_grammar,
+        CompileError(
+          TSCompileErrorTypeInvalidExternalToken,
+          "Non-symbol rule expressions can't be used as external tokens"
+        )
+      );
+    }
+
+    Symbol symbol = new_rule.get_unchecked<Symbol>();
+    if (symbol.is_non_terminal()) {
       return make_tuple(
         syntax_grammar,
         lexical_grammar,
@@ -199,11 +314,19 @@ tuple<InitialSyntaxGrammar, LexicalGrammar, CompileError> extract_tokens(
       );
     }
 
-    syntax_grammar.external_tokens.push_back({
-      external_token.name,
-      external_token.type,
-      internal_token
-    });
+    if (symbol.is_external()) {
+      syntax_grammar.external_tokens.push_back(ExternalToken{
+        external_token.name,
+        external_token.type,
+        rules::NONE(),
+      });
+    } else {
+      syntax_grammar.external_tokens.push_back(ExternalToken{
+        lexical_grammar.variables[symbol.index].name,
+        external_token.type,
+        symbol,
+      });
+    }
   }
 
   return make_tuple(syntax_grammar, lexical_grammar, CompileError::none());