tree-sitter 0.0.1 → 0.1.0

data/ext/tree-sitter/tree-sitter/src/compiler/generate_code/c_code.h

@@ -2,18 +2,26 @@
 #define COMPILER_GENERATE_CODE_C_CODE_H_
 
 #include <string>
+#include "compiler/rule.h"
 
 namespace tree_sitter {
 
 struct LexicalGrammar;
 struct SyntaxGrammar;
-class LexTable;
-class ParseTable;
+struct LexTable;
+struct ParseTable;
 
 namespace generate_code {
 
-std::string c_code(std::string, const ParseTable &, const LexTable &,
-                   const SyntaxGrammar &, const LexicalGrammar &);
+std::string c_code(
+  std::string,
+  ParseTable &&,
+  LexTable &&,
+  LexTable &&,
+  rules::Symbol,
+  SyntaxGrammar &&,
+  LexicalGrammar &&
+);
 
 }  // namespace generate_code
 }  // namespace tree_sitter

data/ext/tree-sitter/tree-sitter/{doc → src/compiler}/grammar-schema.json

@@ -31,23 +31,30 @@
       }
     },
 
-    "conflicts": {
+    "externals": {
       "type": "array",
       "items": {
-        "type": "array",
-        "items": {
-          "type": "string",
-          "pattern": "^[a-zA-Z_]\\w*$"
-        }
+        "$ref": "#/definitions/rule"
       }
     },
 
-    "externals": {
+    "inline": {
       "type": "array",
       "items": {
         "type": "string",
         "pattern": "^[a-zA-Z_]\\w*$"
       }
+    },
+
+    "conflicts": {
+      "type": "array",
+      "items": {
+        "type": "array",
+        "items": {
+          "type": "string",
+          "pattern": "^[a-zA-Z_]\\w*$"
+        }
+      }
     }
   },
 
@@ -136,40 +143,32 @@
       "required": ["type", "members"]
     },
 
-    "repeat-rule": {
+    "alias-rule": {
       "type": "object",
       "properties": {
         "type": {
           "type": "string",
-          "pattern": "^REPEAT$"
+          "pattern": "^ALIAS$"
         },
-        "content": {
-          "$ref": "#/definitions/rule"
-        }
-      },
-      "required": ["type", "content"]
-    },
-
-    "repeat1-rule": {
-      "type": "object",
-      "properties": {
-        "type": {
-          "type": "string",
-          "pattern": "^REPEAT1$"
+        "value": {
+          "type": "string"
+        },
+        "named": {
+          "type": "boolean"
         },
         "content": {
           "$ref": "#/definitions/rule"
         }
       },
-      "required": ["type", "content"]
+      "required": ["type", "named", "content", "value"]
     },
 
-    "token-rule": {
+    "repeat-rule": {
       "type": "object",
       "properties": {
         "type": {
           "type": "string",
-          "pattern": "^TOKEN$"
+          "pattern": "^REPEAT$"
         },
         "content": {
           "$ref": "#/definitions/rule"
@@ -178,46 +177,40 @@
       "required": ["type", "content"]
     },
 
-    "prec-rule": {
+    "repeat1-rule": {
       "type": "object",
       "properties": {
         "type": {
           "type": "string",
-          "pattern": "^PREC$"
-        },
-        "value": {
-          "type": "integer"
+          "pattern": "^REPEAT1$"
         },
         "content": {
           "$ref": "#/definitions/rule"
         }
       },
-      "required": ["type", "content", "value"]
+      "required": ["type", "content"]
     },
 
-    "prec-left-rule": {
+    "token-rule": {
       "type": "object",
       "properties": {
         "type": {
           "type": "string",
-          "pattern": "^PREC_LEFT$"
-        },
-        "value": {
-          "type": "integer"
+          "pattern": "^TOKEN$"
         },
         "content": {
           "$ref": "#/definitions/rule"
         }
       },
-      "required": ["type", "content", "value"]
+      "required": ["type", "content"]
     },
 
-    "prec-right-rule": {
+    "prec-rule": {
       "type": "object",
       "properties": {
         "type": {
           "type": "string",
-          "pattern": "^PREC_RIGHT$"
+          "pattern": "^(PREC|PREC_LEFT|PREC_RIGHT|PREC_DYNAMIC)$"
         },
         "value": {
           "type": "integer"
@@ -231,6 +224,7 @@
 
     "rule": {
       "oneOf": [
+        { "$ref": "#/definitions/alias-rule" },
         { "$ref": "#/definitions/blank-rule" },
         { "$ref": "#/definitions/string-rule" },
         { "$ref": "#/definitions/pattern-rule" },
@@ -240,9 +234,7 @@
         { "$ref": "#/definitions/repeat1-rule" },
         { "$ref": "#/definitions/repeat-rule" },
         { "$ref": "#/definitions/token-rule" },
-        { "$ref": "#/definitions/prec-rule" },
-        { "$ref": "#/definitions/prec-left-rule" },
-        { "$ref": "#/definitions/prec-right-rule" }
+        { "$ref": "#/definitions/prec-rule" }
       ]
     }
   }

data/ext/tree-sitter/tree-sitter/src/compiler/grammar.h

@@ -2,17 +2,36 @@
 #define COMPILER_GRAMMAR_H_
 
 #include <vector>
+#include <unordered_set>
 #include <string>
 #include <utility>
 #include "compiler/rule.h"
 
 namespace tree_sitter {
 
-struct Grammar {
-  std::vector<std::pair<std::string, rule_ptr>> rules;
-  std::vector<rule_ptr> extra_tokens;
-  std::vector<std::vector<std::string>> expected_conflicts;
-  std::vector<std::string> external_tokens;
+enum VariableType {
+  VariableTypeHidden,
+  VariableTypeAuxiliary,
+  VariableTypeAnonymous,
+  VariableTypeNamed,
+};
+
+struct Variable {
+  std::string name;
+  VariableType type;
+  rules::Rule rule;
+
+  inline bool operator==(const Variable &other) const {
+    return name == other.name && rule == other.rule && type == other.type;
+  }
+};
+
+struct InputGrammar {
+  std::vector<Variable> variables;
+  std::vector<rules::Rule> extra_tokens;
+  std::vector<std::unordered_set<rules::NamedSymbol>> expected_conflicts;
+  std::vector<rules::Rule> external_tokens;
+  std::unordered_set<rules::NamedSymbol> variables_to_inline;
 };
 
 }  // namespace tree_sitter

data/ext/tree-sitter/tree-sitter/src/compiler/lex_table.cc

@@ -1,6 +1,5 @@
 #include "compiler/lex_table.h"
-#include "compiler/rules/symbol.h"
-#include "compiler/rules/built_in_symbols.h"
+#include "compiler/rule.h"
 
 namespace tree_sitter {
 
@@ -42,37 +41,9 @@ bool AcceptTokenAction::operator==(const AcceptTokenAction &other) const {
          (is_string == other.is_string);
 }
 
-LexState::LexState() : is_token_start(false) {}
-
-set<CharacterSet> LexState::expected_inputs() const {
-  set<CharacterSet> result;
-  for (auto &pair : advance_actions)
-    result.insert(pair.first);
-  return result;
-}
-
 bool LexState::operator==(const LexState &other) const {
   return advance_actions == other.advance_actions &&
-         accept_action == other.accept_action &&
-         is_token_start == other.is_token_start;
-}
-
-void LexState::each_referenced_state(function<void(LexStateId *)> fn) {
-  for (auto &entry : advance_actions)
-    fn(&entry.second.state_index);
-}
-
-LexStateId LexTable::add_state() {
-  states.push_back(LexState());
-  return states.size() - 1;
-}
-
-LexState &LexTable::state(LexStateId id) {
-  return states[id];
-}
-
-bool LexTable::merge_state(size_t i, size_t j) {
-  return states[i] == states[j];
+         accept_action == other.accept_action;
 }
 
 }  // namespace tree_sitter

data/ext/tree-sitter/tree-sitter/src/compiler/lex_table.h

@@ -6,25 +6,17 @@
 #include <set>
 #include <string>
 #include "compiler/precedence_range.h"
-#include "compiler/rules/symbol.h"
-#include "compiler/rules/character_set.h"
+#include "compiler/rule.h"
 
 namespace tree_sitter {
 
 typedef int64_t LexStateId;
 
-typedef enum {
-  LexActionTypeError,
-  LexActionTypeAccept,
-  LexActionTypeAcceptFragile,
-  LexActionTypeAdvance
-} LexActionType;
-
 struct AdvanceAction {
   AdvanceAction();
   AdvanceAction(size_t, PrecedenceRange, bool);
-
   bool operator==(const AdvanceAction &other) const;
+  inline bool operator!=(const AdvanceAction &other) const { return !operator==(other); }
 
   LexStateId state_index;
   PrecedenceRange precedence_range;
@@ -34,40 +26,24 @@ struct AdvanceAction {
 struct AcceptTokenAction {
   AcceptTokenAction();
   AcceptTokenAction(rules::Symbol, int, bool);
-
   bool is_present() const;
-  bool operator==(const AcceptTokenAction &action) const;
+  bool operator==(const AcceptTokenAction &other) const;
+  inline bool operator!=(const AcceptTokenAction &other) const { return !operator==(other); }
 
   rules::Symbol symbol;
   int precedence;
   bool is_string;
 };
 
-}  // namespace tree_sitter
-
-namespace std {}  // namespace std
-
-namespace tree_sitter {
-
-class LexState {
- public:
-  LexState();
-  std::set<rules::CharacterSet> expected_inputs() const;
+struct LexState {
   bool operator==(const LexState &) const;
-  void each_referenced_state(std::function<void(LexStateId *)>);
 
   std::map<rules::CharacterSet, AdvanceAction> advance_actions;
   AcceptTokenAction accept_action;
-  bool is_token_start;
 };
 
-class LexTable {
- public:
-  LexStateId add_state();
-  LexState &state(LexStateId state_id);
+struct LexTable {
   std::vector<LexState> states;
-
-  bool merge_state(size_t i, size_t j);
 };
 
 }  // namespace tree_sitter

data/ext/tree-sitter/tree-sitter/src/compiler/lexical_grammar.h

@@ -5,13 +5,25 @@
 #include <string>
 #include <set>
 #include "compiler/rule.h"
-#include "compiler/variable.h"
+#include "compiler/grammar.h"
 
 namespace tree_sitter {
 
+struct LexicalVariable {
+  std::string name;
+  VariableType type;
+  rules::Rule rule;
+  bool is_string;
+
+  inline bool operator==(const LexicalVariable &other) const {
+    return other.name == name && other.type == type && other.rule == rule &&
+      other.is_string == is_string;
+  }
+};
+
 struct LexicalGrammar {
-  std::vector<Variable> variables;
-  std::vector<rule_ptr> separators;
+  std::vector<LexicalVariable> variables;
+  std::vector<rules::Rule> separators;
 };
 
 }  // namespace tree_sitter

data/ext/tree-sitter/tree-sitter/src/compiler/parse_grammar.cc

@@ -1,20 +1,32 @@
 #include "compiler/parse_grammar.h"
 #include <string>
 #include <vector>
+#include <unordered_set>
 #include <utility>
 #include "json.h"
 #include "compiler/rule.h"
-#include "compiler/rules.h"
 
 namespace tree_sitter {
 
+using std::move;
 using std::string;
 using std::vector;
+using std::unordered_set;
 using std::pair;
+using rules::Rule;
+using rules::Blank;
+using rules::Metadata;
+using rules::Pattern;
+using rules::String;
+using rules::NamedSymbol;
 
 struct ParseRuleResult {
-  rule_ptr rule;
+  Rule rule;
   string error_message;
+
+  ParseRuleResult(const string &error_message) : error_message(error_message) {}
+  ParseRuleResult(const char *error_message) : error_message(error_message) {}
+  ParseRuleResult(Rule rule) : rule(rule) {}
 };
 
 ParseRuleResult parse_rule(json_value *rule_json) {
@@ -23,194 +35,201 @@ ParseRuleResult parse_rule(json_value *rule_json) {
   string type;
 
   if (!rule_json) {
-    error_message = "Rule cannot be null";
-    goto error;
+    return "Rule cannot be null";
   }
 
   if (rule_json->type != json_object) {
-    error_message = "Rule type must be an object";
-    goto error;
+    return "Rule type must be an object";
   }
 
   rule_type_json = rule_json->operator[]("type");
   if (rule_type_json.type != json_string) {
-    error_message = "Rule type must be a string";
-    goto error;
+    return "Rule type must be a string";
   }
 
   type = rule_type_json.u.string.ptr;
 
   if (type == "BLANK") {
-    return { blank(), "" };
+    return Rule(Blank{});
   }
 
   if (type == "CHOICE") {
     json_value members_json = rule_json->operator[]("members");
     if (members_json.type != json_array) {
-      error_message = "Choice members must be an array";
-      goto error;
+      return "Choice members must be an array";
     }
 
-    vector<rule_ptr> members;
+    vector<Rule> members;
     for (size_t i = 0, length = members_json.u.array.length; i < length; i++) {
       json_value *member_json = members_json.u.array.values[i];
-      ParseRuleResult member = parse_rule(member_json);
-      if (member.rule.get()) {
-        members.push_back(member.rule);
-      } else {
-        error_message = "Invalid choice member: " + member.error_message;
-        goto error;
+      auto result = parse_rule(member_json);
+      if (!result.error_message.empty()) {
+        return "Invalid choice member: " + result.error_message;
       }
+      members.push_back(result.rule);
     }
-    return { choice(members), "" };
+    return Rule::choice(members);
   }
 
   if (type == "SEQ") {
     json_value members_json = rule_json->operator[]("members");
     if (members_json.type != json_array) {
-      error_message = "Seq members must be an array";
-      goto error;
+      return "Seq members must be an array";
     }
 
-    vector<rule_ptr> members;
+    vector<Rule> members;
     for (size_t i = 0, length = members_json.u.array.length; i < length; i++) {
       json_value *member_json = members_json.u.array.values[i];
-      ParseRuleResult member = parse_rule(member_json);
-      if (member.rule.get()) {
-        members.push_back(member.rule);
-      } else {
-        error_message = "Invalid seq member: " + member.error_message;
-        goto error;
+      auto result = parse_rule(member_json);
+      if (!result.error_message.empty()) {
+        return "Invalid choice member: " + result.error_message;
       }
+      members.push_back(result.rule);
     }
-    return { seq(members), "" };
+    return Rule::seq(members);
   }
 
   if (type == "REPEAT") {
     json_value content_json = rule_json->operator[]("content");
-    ParseRuleResult content = parse_rule(&content_json);
-    if (content.rule.get()) {
-      return { repeat(content.rule), "" };
-    } else {
-      error_message = "Invalid repeat content: " + content.error_message;
-      goto error;
+    auto result = parse_rule(&content_json);
+    if (!result.error_message.empty()) {
+      return "Invalid repeat content: " + result.error_message;
     }
+    return Rule::choice({Rule::repeat(result.rule), Blank{}});
   }
 
   if (type == "REPEAT1") {
     json_value content_json = rule_json->operator[]("content");
-    ParseRuleResult content = parse_rule(&content_json);
-    if (content.rule.get()) {
-      return { repeat1(content.rule), "" };
-    } else {
-      error_message = "Invalid repeat1 content: " + content.error_message;
-      goto error;
+    auto result = parse_rule(&content_json);
+    if (!result.error_message.empty()) {
+      return "Invalid repeat content: " + result.error_message;
     }
+    return Rule::repeat(result.rule);
   }
 
   if (type == "TOKEN") {
     json_value content_json = rule_json->operator[]("content");
-    ParseRuleResult content = parse_rule(&content_json);
-    if (content.rule.get()) {
-      return { token(content.rule), "" };
-    } else {
-      error_message = "Invalid token content: " + content.error_message;
-      goto error;
+    auto result = parse_rule(&content_json);
+    if (!result.error_message.empty()) {
+      return "Invalid token content: " + result.error_message;
     }
+      return Rule(Metadata::token(move(result.rule)));
   }
 
   if (type == "PATTERN") {
     json_value value_json = rule_json->operator[]("value");
     if (value_json.type == json_string) {
-      return { pattern(value_json.u.string.ptr), "" };
+      return Rule(Pattern{value_json.u.string.ptr});
     } else {
-      error_message = "Pattern value must be a string";
-      goto error;
+      return "Pattern value must be a string";
     }
   }
 
   if (type == "STRING") {
     json_value value_json = rule_json->operator[]("value");
     if (value_json.type == json_string) {
-      return { str(value_json.u.string.ptr), "" };
+      return Rule(String{value_json.u.string.ptr});
     } else {
-      error_message = "String rule value must be a string";
-      goto error;
+      return "String rule value must be a string";
     }
   }
 
   if (type == "SYMBOL") {
     json_value value_json = rule_json->operator[]("name");
     if (value_json.type == json_string) {
-      return { sym(value_json.u.string.ptr), "" };
+      return Rule(NamedSymbol{value_json.u.string.ptr});
     } else {
-      error_message = "Symbol value must be a string";
-      goto error;
+      return "Symbol value must be a string";
     }
   }
 
   if (type == "PREC") {
     json_value precedence_json = rule_json->operator[]("value");
     if (precedence_json.type != json_integer) {
-      error_message = "Precedence value must be an integer";
-      goto error;
+      return "Precedence value must be an integer";
     }
 
     json_value content_json = rule_json->operator[]("content");
-    ParseRuleResult content = parse_rule(&content_json);
-    if (!content.rule.get()) {
-      error_message = "Invalid precedence content: " + content.error_message;
-      goto error;
+    auto result = parse_rule(&content_json);
+    if (!result.error_message.empty()) {
+      return "Invalid precedence content: " + result.error_message;
     }
-
-    return { prec(precedence_json.u.integer, content.rule), "" };
+    return Rule(Metadata::prec(precedence_json.u.integer, move(result.rule)));
   }
 
   if (type == "PREC_LEFT") {
     json_value precedence_json = rule_json->operator[]("value");
     if (precedence_json.type != json_integer) {
-      error_message = "Precedence value must be an integer";
-      goto error;
+      return "Precedence value must be an integer";
     }
 
     json_value content_json = rule_json->operator[]("content");
-    ParseRuleResult content = parse_rule(&content_json);
-    if (!content.rule.get()) {
-      error_message = "Invalid precedence content: " + content.error_message;
-      goto error;
+    auto result = parse_rule(&content_json);
+    if (!result.error_message.empty()) {
+      return "Invalid precedence content: " + result.error_message;
     }
-
-    return { prec_left(precedence_json.u.integer, content.rule), "" };
+    return Rule(Metadata::prec_left(precedence_json.u.integer, move(result.rule)));
   }
 
   if (type == "PREC_RIGHT") {
     json_value precedence_json = rule_json->operator[]("value");
     if (precedence_json.type != json_integer) {
-      error_message = "Precedence value must be an integer";
-      goto error;
+      return "Precedence value must be an integer";
     }
 
     json_value content_json = rule_json->operator[]("content");
-    ParseRuleResult content = parse_rule(&content_json);
-    if (!content.rule.get()) {
-      error_message = "Invalid precedence content: " + content.error_message;
-      goto error;
+    auto result = parse_rule(&content_json);
+    if (!result.error_message.empty()) {
+      return "Invalid precedence content: " + result.error_message;
+    }
+    return Rule(Metadata::prec_right(precedence_json.u.integer, move(result.rule)));
+  }
+
+  if (type == "PREC_DYNAMIC") {
+    json_value precedence_json = rule_json->operator[]("value");
+    if (precedence_json.type != json_integer) {
+      return "Precedence value must be an integer";
     }
 
-    return { prec_right(precedence_json.u.integer, content.rule), "" };
+    json_value content_json = rule_json->operator[]("content");
+    auto result = parse_rule(&content_json);
+    if (!result.error_message.empty()) {
+      return "Invalid precedence content: " + result.error_message;
+    }
+    return Rule(Metadata::prec_dynamic(precedence_json.u.integer, move(result.rule)));
   }
 
-  error_message = "Unknown rule type " + type;
+  if (type == "ALIAS") {
+    json_value value_json = rule_json->operator[]("value");
+    if (value_json.type != json_string) {
+      return "Rename value must be a string";
+    }
+
+    json_value is_named_json = rule_json->operator[]("named");
+    if (is_named_json.type != json_boolean) {
+      return "Rename named value must be a boolean";
+    }
 
-error:
-  return { rule_ptr(), error_message };
+    json_value content_json = rule_json->operator[]("content");
+    auto result = parse_rule(&content_json);
+    if (!result.error_message.empty()) {
+      return "Invalid rename content: " + result.error_message;
+    }
+    return Rule(Metadata::alias(
+      string(value_json.u.string.ptr),
+      is_named_json.u.boolean,
+      move(result.rule)
+    ));
+  }
+
+  return "Unknown rule type: " + type;
 }
 
 ParseGrammarResult parse_grammar(const string &input) {
   string error_message;
   string name;
-  Grammar grammar;
-  json_value name_json, rules_json, extras_json, conflicts_json, external_tokens_json;
+  InputGrammar grammar;
+  json_value name_json, rules_json, extras_json, conflicts_json, external_tokens_json, inline_rules_json;
 
   json_settings settings = { 0, json_enable_comments, 0, 0, 0, 0 };
   char parse_error[json_error_max];
@@ -242,15 +261,16 @@ ParseGrammarResult parse_grammar(const string &input) {
 
   for (size_t i = 0, length = rules_json.u.object.length; i < length; i++) {
     json_object_entry entry_json = rules_json.u.object.values[i];
-    ParseRuleResult entry = parse_rule(entry_json.value);
-
-    if (!entry.rule.get()) {
-      error_message =
-        string("Invalid rule '") + entry_json.name + "' " + entry.error_message;
+    auto result = parse_rule(entry_json.value);
+    if (!result.error_message.empty()) {
+      error_message = result.error_message;
       goto error;
     }
-
-    grammar.rules.push_back({ string(entry_json.name), entry.rule });
+    grammar.variables.push_back(Variable{
+      string(entry_json.name),
+      VariableTypeNamed,
+      result.rule
+    });
   }
 
   extras_json = grammar_json->operator[]("extras");
@@ -262,13 +282,12 @@ ParseGrammarResult parse_grammar(const string &input) {
 
     for (size_t i = 0, length = extras_json.u.array.length; i < length; i++) {
       json_value *extra_json = extras_json.u.array.values[i];
-      ParseRuleResult extra = parse_rule(extra_json);
-      if (!extra.rule.get()) {
-        error_message = string("Invalid extra token: ") + extra.error_message;
+      auto result = parse_rule(extra_json);
+      if (!result.error_message.empty()) {
+        error_message = "Invalid extra token: " + result.error_message;
         goto error;
       }
-
-      grammar.extra_tokens.push_back(extra.rule);
+      grammar.extra_tokens.push_back(result.rule);
     }
   }
 
@@ -286,7 +305,7 @@ ParseGrammarResult parse_grammar(const string &input) {
         goto error;
       }
 
-      vector<string> conflict;
+      unordered_set<NamedSymbol> conflict;
       for (size_t j = 0, conflict_length = conflict_json->u.array.length;
            j < conflict_length; j++) {
         json_value *conflict_entry_json = conflict_json->u.array.values[j];
@@ -295,13 +314,33 @@ ParseGrammarResult parse_grammar(const string &input) {
           goto error;
         }
 
-        conflict.push_back(string(conflict_entry_json->u.string.ptr));
+        conflict.insert(rules::NamedSymbol{
+          string(conflict_entry_json->u.string.ptr)
+        });
       }
 
       grammar.expected_conflicts.push_back(conflict);
     }
   }
 
+  inline_rules_json = grammar_json->operator[]("inline");
+  if (inline_rules_json.type != json_none) {
+    if (inline_rules_json.type != json_array) {
+      error_message = "Inline rules must be an array";
+      goto error;
+    }
+
+    for (size_t i = 0, length = inline_rules_json.u.array.length; i < length; i++) {
+      json_value *inline_rule_json = inline_rules_json.u.array.values[i];
+      if (inline_rule_json->type != json_string) {
+        error_message = "Inline rules must be an array of rule names";
+        goto error;
+      }
+
+      grammar.variables_to_inline.insert(rules::NamedSymbol{string(inline_rule_json->u.string.ptr)});
+    }
+  }
+
   external_tokens_json = grammar_json->operator[]("externals");
   if (external_tokens_json.type != json_none) {
     if (external_tokens_json.type != json_array) {
@@ -310,14 +349,13 @@ ParseGrammarResult parse_grammar(const string &input) {
     }
 
     for (size_t i = 0, length = external_tokens_json.u.array.length; i < length; i++) {
-      json_value *token_name_json = external_tokens_json.u.array.values[i];
-      if (token_name_json->type != json_string) {
-        error_message = "External token values must be strings";
+      json_value *external_token_json = external_tokens_json.u.array.values[i];
+      auto result = parse_rule(external_token_json);
+      if (!result.error_message.empty()) {
+        error_message = "Invalid external token: " + result.error_message;
         goto error;
       }
-
-      string token_name = token_name_json->u.string.ptr;
-      grammar.external_tokens.push_back(token_name);
+      grammar.external_tokens.push_back(result.rule);
     }
   }
 
@@ -329,7 +367,7 @@ error:
     json_value_free(grammar_json);
   }
 
-  return { "", Grammar(), error_message };
+  return { "", InputGrammar(), error_message };
 }
 
 }  // namespace tree_sitter