ibex 0.1.0

data/lib/ibex/lalr/default_reductions.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module Ibex
+  module LALR
+    # Selects size-reducing default actions without changing any terminal cell.
+    module DefaultReductions
+      # @rbs!
+      #   private def select_default: (Hash[Integer, IR::parser_action] actions, Array[Integer] terminal_ids) -> IR::parser_action?
+      #   private def self.select_default: (Hash[Integer, IR::parser_action] actions, Array[Integer] terminal_ids) -> IR::parser_action?
+
+      ERROR_ACTION = { type: :error }.freeze #: IR::error_action
+
+      # @rbs (Array[IR::AutomatonState] states, terminal_ids: Array[Integer]) -> Array[IR::AutomatonState]
+      def apply(states, terminal_ids:)
+        states.map { |state| optimize(state, terminal_ids: terminal_ids) }
+      end
+      module_function :apply
+
+      # @rbs (IR::AutomatonState state, terminal_ids: Array[Integer]) -> IR::AutomatonState
+      def optimize(state, terminal_ids:)
+        return state if state.default_action
+
+        default_action = select_default(state.actions, terminal_ids)
+        return state unless default_action
+
+        actions = {} #: Hash[Integer, IR::parser_action]
+        terminal_ids.each_with_object(actions) do |token_id, result|
+          action = state.actions[token_id]
+          result[token_id] = action || ERROR_ACTION unless action == default_action
+        end
+        IR::AutomatonState.new(id: state.id, items: state.items, transitions: state.transitions,
+                               actions: actions, gotos: state.gotos, default_action: default_action,
+                               conflicts: state.conflicts)
+      end
+      module_function :optimize
+
+      # @rbs skip
+      private
+
+      # @rbs skip
+      def select_default(actions, terminal_ids)
+        candidates = [] #: Array[IR::reduce_action]
+        actions.each_value do |action|
+          next unless action[:type] == :reduce
+
+          reduction = action #: IR::reduce_action
+          candidates << reduction
+        end
+        candidates.uniq!
+        candidates.sort_by! { |action| action.fetch(:production) }
+        candidate = candidates.max_by do |action|
+          saved_entries = actions.count { |_token_id, candidate| candidate == action }
+          [saved_entries, -action.fetch(:production)]
+        end
+        return unless candidate
+
+        optimized_size = terminal_ids.count { |token_id| actions[token_id] != candidate } + 1
+        candidate if optimized_size < actions.length
+      end
+      module_function :select_default
+
+      class << self
+        private :select_default
+      end
+    end
+  end
+end

data/lib/ibex/lalr.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require_relative "lalr/conflict"
+require_relative "lalr/default_reductions"
+require_relative "lalr/builder"
+require_relative "lalr/conflict_search_limits"
+require_relative "lalr/conflict_search"
+require_relative "lalr/counterexample"
+
+module Ibex
+  module LALR
+    # @rbs!
+    #   type lr_item = [Integer, Integer, Integer]
+    #   type item_core = [Integer, Integer]
+    #   type item_set = Set[lr_item]
+    #   type packed_items = Hash[item_core, Set[Integer]]
+    #   type transitions = Array[Hash[Integer, Integer]]
+    #   type derivation_node = Hash[Symbol, untyped]
+    #   type search_status = :conflict | :shifted | :accepted
+    #   type search_entry = [search_status, ConflictSearch::Configuration]
+    #   type search_result = {
+    #     sentence_ids: Array[Integer],
+    #     lookahead_index: Integer,
+    #     interpretations: Array[IR::interpretation]
+    #   }
+  end
+end

data/lib/ibex/normalize/declarations.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module Ibex
+  # Declaration extraction used by Normalizer.
+  module NormalizeDeclarations
+    private
+
+    # @rbs () -> void
+    def read_declarations
+      # @type self: Normalizer
+      @declared_tokens = {} #: Hash[String, IR::location]
+      @precedence = {} #: Hash[String, IR::precedence]
+      @precedence_locations = {} #: Hash[String, IR::location]
+      @options = { result_var: true, omit_action_call: true }
+      @expected_conflicts = 0
+      @conversions = {} #: Hash[String, String]
+      @ast.declarations.each { |declaration| read_declaration(declaration) }
+    end
+
+    # @rbs (Frontend::AST::declaration declaration) -> void
+    def read_declaration(declaration)
+      # @type self: Normalizer
+      case declaration
+      when Frontend::AST::Tokens then declaration.names.each { |name| @declared_tokens[name] = declaration.loc.to_h }
+      when Frontend::AST::Precedence then read_precedence(declaration)
+      when Frontend::AST::Options then declaration.names.each { |name| read_option(name, declaration.loc) }
+      when Frontend::AST::Expect then @expected_conflicts = declaration.conflicts
+      when Frontend::AST::Start
+        @explicit_start = declaration.name
+        @start_location = declaration.loc
+      when Frontend::AST::Convert then declaration.pairs.each { |pair| @conversions[pair.name] = pair.expression }
+      end
+    end
+
+    # @rbs (Frontend::AST::Precedence declaration) -> void
+    def read_precedence(declaration)
+      # @type self: Normalizer
+      count = declaration.levels.length
+      declaration.levels.each_with_index do |level, index|
+        numeric_level = declaration.direction == :high_to_low ? count - index : index + 1
+        level.symbols.each do |name|
+          @precedence[name] = { associativity: level.associativity, level: numeric_level }
+          @precedence_locations[name] = level.loc.to_h
+        end
+      end
+    end
+
+    # @rbs (String name, Frontend::Location location) -> void
+    def read_option(name, location)
+      # @type self: Normalizer
+      case name
+      when "no_result_var" then @options[:result_var] = false
+      when "result_var" then @options[:result_var] = true
+      when "omit_action_call" then @options[:omit_action_call] = true
+      when "no_omit_action_call" then @options[:omit_action_call] = false
+      else fail_at(location, "unknown option #{name}")
+      end
+    end
+  end
+end

data/lib/ibex/normalize/diagnostics.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+require "set"
+
+module Ibex
+  # Static grammar diagnostics used by Normalizer.
+  module NormalizeDiagnostics
+    private
+
+    # @rbs () -> void
+    def validate_grammar
+      # @type self: Normalizer
+      warn_duplicate_productions
+      warn_unreachable_nonterminals
+      warn_unused_terminals
+      warn_empty_language
+    end
+
+    # @rbs () -> void
+    def warn_duplicate_productions
+      # @type self: Normalizer
+      seen = {} #: Hash[[Integer, Array[Integer]], Integer]
+      @productions.each do |production|
+        signature = [production.lhs, production.rhs] #: [Integer, Array[Integer]]
+        if seen.key?(signature)
+          @warnings << { type: :duplicate_production, production: production.id, original: seen[signature],
+                         loc: production.origin[:loc] }
+        else
+          seen[signature] = production.id
+        end
+      end
+    end
+
+    # @rbs () -> void
+    def warn_unreachable_nonterminals
+      # @type self: Normalizer
+      reachable = reachable_symbol_ids
+      @symbols.select(&:nonterminal?).each do |grammar_symbol|
+        next if reachable.include?(grammar_symbol.id) || grammar_symbol.name.start_with?("$")
+
+        @warnings << { type: :unreachable_nonterminal, symbol: grammar_symbol.name, loc: grammar_symbol.location }
+      end
+    end
+
+    # @rbs () -> Set[Integer]
+    def reachable_symbol_ids
+      # @type self: Normalizer
+      start = required_symbol(@start_name).id
+      reachable = Set[start]
+      loop do
+        before = reachable.length
+        @productions.select { |production| reachable.include?(production.lhs) }.each do |production|
+          production.rhs.each { |id| reachable << id }
+        end
+        return reachable if reachable.length == before
+      end
+    end
+
+    # @rbs () -> void
+    def warn_unused_terminals
+      # @type self: Normalizer
+      used = @productions.flat_map(&:rhs).to_set
+      @symbols.select(&:terminal?).each do |grammar_symbol|
+        next if grammar_symbol.reserved || used.include?(grammar_symbol.id) || @precedence.key?(grammar_symbol.name)
+
+        @warnings << { type: :unused_terminal, symbol: grammar_symbol.name, loc: grammar_symbol.location }
+      end
+    end
+
+    # @rbs () -> void
+    def warn_empty_language
+      # @type self: Normalizer
+      productive = productive_terminal_ids
+      loop do
+        before = productive.length
+        @productions.each do |production|
+          productive << production.lhs if production.rhs.all? { |id| productive.include?(id) }
+        end
+        break if productive.length == before
+      end
+      return if productive.include?(required_symbol(@start_name).id)
+
+      start_symbol = required_symbol(@start_name)
+      @warnings << { type: :empty_language, symbol: @start_name, loc: start_symbol.location }
+    end
+
+    # @rbs () -> Set[Integer]
+    def productive_terminal_ids
+      # @type self: Normalizer
+      @symbols.select { |grammar_symbol| productive_terminal?(grammar_symbol) }.to_set(&:id)
+    end
+
+    # @rbs (IR::GrammarSymbol grammar_symbol) -> bool
+    def productive_terminal?(grammar_symbol)
+      # @type self: Normalizer
+      grammar_symbol.terminal? && !grammar_symbol.reserved
+    end
+  end
+end

data/lib/ibex/normalize/expander.rb

@@ -0,0 +1,218 @@
+# frozen_string_literal: true
+
+module Ibex
+  # Production and EBNF expansion used by Normalizer.
+  module NormalizeExpander
+    private
+
+    # @rbs () -> void
+    def normalize_user_productions
+      # @type self: Normalizer
+      @ast.rules.each do |rule|
+        rule.alternatives.each { |alternative| normalize_alternative(rule, alternative) }
+      end
+    end
+
+    # @rbs (Frontend::AST::Rule rule, Frontend::AST::Alternative alternative) -> void
+    def normalize_alternative(rule, alternative)
+      # @type self: Normalizer
+      rhs = [] #: Array[String]
+      named_refs = [] #: Array[IR::named_ref]
+      alternative.items.each do |item|
+        if item.is_a?(Frontend::AST::InlineAction)
+          rhs << expand_inline_action(item, rhs.length, named_refs)
+          next
+        end
+
+        rhs << normalize_item(item)
+        add_named_reference(item, named_refs, rhs.length - 1)
+      end
+      action = normalize_action(alternative.action, named_refs)
+      add_production(rule.lhs, rhs, action, alternative.precedence,
+                     { kind: :user, loc: alternative.loc.to_h })
+    end
+
+    # @rbs (Frontend::AST::item item) -> String
+    def normalize_item(item)
+      # @type self: Normalizer
+      return symbol_for_reference(item).name if item.is_a?(Frontend::AST::SymbolReference)
+      return expand_group(item) if item.is_a?(Frontend::AST::Group)
+      return expand_optional(item) if item.is_a?(Frontend::AST::Optional)
+      return expand_star(item) if item.is_a?(Frontend::AST::Star)
+      return expand_plus(item) if item.is_a?(Frontend::AST::Plus)
+      return expand_separated_list(item) if item.is_a?(Frontend::AST::SeparatedList)
+
+      fail_at(item.loc, "unsupported nested EBNF expression")
+    end
+
+    # @rbs (Frontend::AST::InlineAction item, Integer context_length, Array[IR::named_ref] named_refs) -> String
+    def expand_inline_action(item, context_length, named_refs)
+      # @type self: Normalizer
+      helper = new_helper("inline", item.loc)
+      action = IR::Action.new(code: item.code, location: item.loc.to_h, named_refs: named_refs.map(&:dup),
+                              context_length: context_length)
+      add_production(helper, [], action, nil, { kind: :inline_action, loc: item.loc.to_h })
+      helper
+    end
+
+    # @rbs (Frontend::AST::Optional item) -> String
+    def expand_optional(item)
+      # @type self: Normalizer
+      base = normalize_item(item.item)
+      helper = new_helper("optional", item.loc)
+      add_production(helper, [], synthetic_action("nil", item.loc), nil, synthetic_origin(:optional, item))
+      add_production(helper, [base], synthetic_action("val[0]", item.loc), nil, synthetic_origin(:optional, item))
+      helper
+    end
+
+    # @rbs (Frontend::AST::Star item) -> String
+    def expand_star(item)
+      # @type self: Normalizer
+      base = normalize_item(item.item)
+      helper = new_helper("star", item.loc)
+      add_production(helper, [], synthetic_action("[]", item.loc), nil, synthetic_origin(:star, item))
+      add_production(helper, [helper, base], synthetic_action("val[0] + [val[1]]", item.loc), nil,
+                     synthetic_origin(:star, item))
+      helper
+    end
+
+    # @rbs (Frontend::AST::Plus item) -> String
+    def expand_plus(item)
+      # @type self: Normalizer
+      base = normalize_item(item.item)
+      helper = new_helper("plus", item.loc)
+      add_production(helper, [base], synthetic_action("[val[0]]", item.loc), nil, synthetic_origin(:plus, item))
+      add_production(helper, [helper, base], synthetic_action("val[0] + [val[1]]", item.loc), nil,
+                     synthetic_origin(:plus, item))
+      helper
+    end
+
+    # @rbs (Frontend::AST::SeparatedList item) -> String
+    def expand_separated_list(item)
+      # @type self: Normalizer
+      base = normalize_item(item.item)
+      separator = normalize_item(item.separator)
+      helper = new_helper("separated_list", item.loc)
+      unless item.nonempty
+        add_production(helper, [], synthetic_action("[]", item.loc), nil, synthetic_origin(:separated_list, item))
+      end
+      add_production(helper, [base], synthetic_action("[val[0]]", item.loc), nil,
+                     synthetic_origin(:separated_list, item))
+      add_production(helper, [helper, separator, base], synthetic_action("val[0] + [val[2]]", item.loc), nil,
+                     synthetic_origin(:separated_list, item))
+      helper
+    end
+
+    # @rbs (Frontend::AST::Group item) -> String
+    def expand_group(item)
+      # @type self: Normalizer
+      reject_group_named_references(item)
+      helper = new_helper("group", item.loc)
+      item.alternatives.each do |alternative|
+        rhs = alternative.map { |child| normalize_item(child) }
+        expression = group_value_expression(rhs.length)
+        add_production(helper, rhs, synthetic_action(expression, item.loc), nil, synthetic_origin(:group, item))
+      end
+      helper
+    end
+
+    # @rbs (Integer length) -> String
+    def group_value_expression(length)
+      # @type self: Normalizer
+      return "nil" if length.zero?
+      return "val[0]" if length == 1
+
+      "val"
+    end
+
+    # @rbs (Frontend::AST::Group group) -> void
+    def reject_group_named_references(group)
+      # @type self: Normalizer
+      reference = group.alternatives.flatten.filter_map { |item| named_reference_in(item) }.first
+      fail_at(reference.loc, "named references inside EBNF groups are not supported") if reference
+    end
+
+    # @rbs (Frontend::AST::item item) -> Frontend::AST::SymbolReference?
+    def named_reference_in(item)
+      # @type self: Normalizer
+      return item if item.is_a?(Frontend::AST::SymbolReference) && item.named_reference
+      if item.is_a?(Frontend::AST::Group)
+        return item.alternatives.flatten.filter_map { |child| named_reference_in(child) }.first
+      end
+      if item.is_a?(Frontend::AST::Optional) || item.is_a?(Frontend::AST::Star) ||
+         item.is_a?(Frontend::AST::Plus) || item.is_a?(Frontend::AST::SeparatedList)
+        return named_reference_in(item.item)
+      end
+
+      nil
+    end
+
+    # @rbs (String kind, Frontend::Location location) -> String
+    def new_helper(kind, location)
+      # @type self: Normalizer
+      @helper_sequence += 1
+      name = "$#{kind}_#{@helper_sequence}"
+      intern(name, :nonterminal, location: location.to_h)
+      name
+    end
+
+    # @rbs (String expression, Frontend::Location location) -> IR::Action
+    def synthetic_action(expression, location)
+      # @type self: Normalizer
+      code = @options[:result_var] ? " result = #{expression} " : " #{expression} "
+      IR::Action.new(code: code, location: location.to_h)
+    end
+
+    # @rbs (Symbol kind, Frontend::AST::item item) -> Hash[Symbol, untyped]
+    def synthetic_origin(kind, item)
+      # @type self: Normalizer
+      { kind: :"#{kind}_expansion", expression: NormalizeExpression.render(item), loc: item.loc.to_h }
+    end
+
+    # @rbs (Frontend::AST::InlineAction? action, Array[IR::named_ref] named_refs) -> IR::Action?
+    def normalize_action(action, named_refs)
+      # @type self: Normalizer
+      return nil unless action
+
+      IR::Action.new(code: action.code, location: action.loc.to_h, named_refs: named_refs)
+    end
+
+    # @rbs (Frontend::AST::item item, Array[IR::named_ref] refs, Integer index) -> void
+    def add_named_reference(item, refs, index)
+      # @type self: Normalizer
+      reference = unwrap_reference(item)
+      return unless reference
+
+      name = reference.named_reference
+      return unless name
+
+      fail_at(reference.loc, "reserved named reference #{name}") if Normalizer::RESERVED_NAMES.include?(name)
+      fail_at(reference.loc, "duplicate named reference #{name}") if refs.any? { |entry| entry[:name] == name }
+      refs << { name: name, index: index }
+    end
+
+    # @rbs (Frontend::AST::item item) -> Frontend::AST::SymbolReference?
+    def unwrap_reference(item)
+      # @type self: Normalizer
+      return item if item.is_a?(Frontend::AST::SymbolReference)
+      if item.is_a?(Frontend::AST::Optional) || item.is_a?(Frontend::AST::Star) ||
+         item.is_a?(Frontend::AST::Plus) || item.is_a?(Frontend::AST::SeparatedList)
+        return unwrap_reference(item.item)
+      end
+
+      nil
+    end
+
+    # @rbs (String lhs_name, Array[String] rhs_names, IR::Action? action, String? precedence_name,
+    #   Hash[Symbol, untyped] origin) -> void
+    def add_production(lhs_name, rhs_names, action, precedence_name, origin)
+      # @type self: Normalizer
+      lhs = symbol(lhs_name) || intern(lhs_name, :nonterminal, location: origin[:loc])
+      rhs = rhs_names.map { |name| symbol(name)&.id || fail_hash(origin[:loc], "undefined symbol #{name}") }
+      precedence = precedence_name && symbol(precedence_name)
+      fail_hash(origin[:loc], "undefined precedence symbol #{precedence_name}") if precedence_name && !precedence
+      @productions << IR::Production.new(id: @productions.length, lhs: lhs.id, rhs: rhs, action: action,
+                                         precedence_override: precedence&.id, origin: origin)
+    end
+  end
+end

data/lib/ibex/normalize/expression.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module Ibex
+  # Deterministically renders frontend EBNF items for production origin metadata.
+  module NormalizeExpression
+    # @rbs!
+    #   private def render_reference: (Frontend::AST::SymbolReference item) -> String
+    #   private def self.render_reference: (Frontend::AST::SymbolReference item) -> String
+    #   private def render_group: (Frontend::AST::Group item) -> String
+    #   private def self.render_group: (Frontend::AST::Group item) -> String
+    #   private def render_separated_list: (Frontend::AST::SeparatedList item) -> String
+    #   private def self.render_separated_list: (Frontend::AST::SeparatedList item) -> String
+
+    # @rbs (Frontend::AST::item item) -> String
+    def render(item)
+      case item
+      when Frontend::AST::SymbolReference then render_reference(item)
+      when Frontend::AST::Group then render_group(item)
+      when Frontend::AST::Optional then "#{render(item.item)}?"
+      when Frontend::AST::Star then "#{render(item.item)}*"
+      when Frontend::AST::Plus then "#{render(item.item)}+"
+      when Frontend::AST::SeparatedList then render_separated_list(item)
+      else raise Ibex::Error, "#{item.loc}: cannot render unsupported EBNF expression"
+      end
+    end
+    module_function :render
+
+    # @rbs skip
+    private
+
+    # @rbs skip
+    def render_reference(item)
+      suffix = item.named_reference ? ":#{item.named_reference}" : ""
+      "#{item.name}#{suffix}"
+    end
+
+    # @rbs skip
+    def render_group(item)
+      alternatives = item.alternatives.map { |items| items.map { |child| render(child) }.join(" ") }
+      "(#{alternatives.join(' | ')})"
+    end
+
+    # @rbs skip
+    def render_separated_list(item)
+      name = item.nonempty ? "separated_nonempty_list" : "separated_list"
+      "#{name}(#{render(item.item)}, #{render(item.separator)})"
+    end
+    module_function :render_reference, :render_group, :render_separated_list
+
+    class << self
+      private :render_reference, :render_group, :render_separated_list
+    end
+  end
+end

data/lib/ibex/normalize.rb

@@ -0,0 +1,166 @@
+# frozen_string_literal: true
+
+require_relative "normalize/declarations"
+require_relative "normalize/expression"
+require_relative "normalize/expander"
+require_relative "normalize/diagnostics"
+
+module Ibex
+  # Converts a frontend AST into immutable Grammar IR.
+  class Normalizer
+    include NormalizeDeclarations
+    include NormalizeExpander
+    include NormalizeDiagnostics
+
+    RESERVED_NAMES = %w[result val _values].freeze #: Array[String]
+
+    # @rbs @ast: Frontend::AST::Root
+    # @rbs @mode: Symbol
+    # @rbs @symbols: Array[IR::GrammarSymbol]
+    # @rbs @symbols_by_name: Hash[String, IR::GrammarSymbol]
+    # @rbs @productions: Array[IR::Production]
+    # @rbs @warnings: Array[IR::grammar_warning]
+    # @rbs @helper_sequence: Integer
+    # @rbs @declared_tokens: Hash[String, IR::location]
+    # @rbs @precedence: Hash[String, IR::precedence]
+    # @rbs @precedence_locations: Hash[String, IR::location]
+    # @rbs @options: IR::grammar_options
+    # @rbs @expected_conflicts: Integer
+    # @rbs @conversions: Hash[String, String]
+    # @rbs @explicit_start: String?
+    # @rbs @start_name: String
+    # @rbs @start_location: Frontend::Location?
+
+    # @rbs (Frontend::AST::Root ast, ?mode: Symbol | String) -> void
+    def initialize(ast, mode: :racc)
+      @ast = ast
+      @mode = mode
+      @symbols = [] #: Array[IR::GrammarSymbol]
+      @symbols_by_name = {} #: Hash[String, IR::GrammarSymbol]
+      @productions = [] #: Array[IR::Production]
+      @warnings = [] #: Array[IR::grammar_warning]
+      @helper_sequence = 0
+    end
+
+    # @rbs () -> IR::Grammar
+    def normalize
+      read_declarations
+      intern_reserved_symbols
+      intern_declared_terminals
+      intern_user_nonterminals
+      normalize_user_productions
+      validate_grammar
+      IR::Grammar.new(class_name: @ast.class_name, superclass: @ast.superclass, start: @start_name,
+                      expect: @expected_conflicts, options: @options, symbols: @symbols,
+                      productions: @productions, user_code: normalized_user_code,
+                      conversions: @conversions, warnings: @warnings, user_code_chunks: normalized_user_code_chunks)
+    end
+
+    private
+
+    # @rbs () -> void
+    def intern_reserved_symbols
+      intern("$eof", :terminal, reserved: true)
+      intern("error", :terminal, reserved: true)
+    end
+
+    # @rbs () -> void
+    def intern_declared_terminals
+      @declared_tokens.each { |name, loc| intern(name, :terminal, location: loc) }
+      @precedence.each_key { |name| intern(name, :terminal, location: @precedence_locations[name]) }
+    end
+
+    # @rbs () -> void
+    def intern_user_nonterminals
+      @ast.rules.each { |rule| intern(rule.lhs, :nonterminal, location: rule.loc.to_h) }
+      @start_name = @explicit_start || @ast.rules.first&.lhs
+      fail_at(@ast.loc, "grammar has no start rule") unless @start_name
+      return if symbol(@start_name)&.nonterminal?
+
+      fail_at(@start_location || @ast.loc, "undefined start symbol #{@start_name}")
+    end
+
+    # @rbs (String name, Symbol kind, ?reserved: bool, ?location: IR::location?) -> IR::GrammarSymbol
+    def intern(name, kind, reserved: false, location: nil)
+      existing = symbol(name)
+      if existing
+        fail_hash(location, "symbol #{name} is both terminal and nonterminal") if existing.kind != kind
+        return existing
+      end
+
+      precedence = @precedence[name]
+      definition = IR::GrammarSymbol.new(id: @symbols.length, name: name, kind: kind, reserved: reserved,
+                                         precedence: precedence, location: location)
+      @symbols << definition
+      @symbols_by_name[name] = definition
+      definition
+    end
+
+    # @rbs (String name) -> IR::GrammarSymbol?
+    def symbol(name)
+      @symbols_by_name[name]
+    end
+
+    # @rbs (String name) -> IR::GrammarSymbol
+    def required_symbol(name)
+      symbol(name) || raise(Ibex::Error, "missing normalized symbol #{name}")
+    end
+
+    # @rbs (Frontend::AST::SymbolReference reference) -> IR::GrammarSymbol
+    def symbol_for_reference(reference)
+      existing = symbol(reference.name)
+      return existing if existing
+      return undefined_nonterminal(reference) if nonterminal_name?(reference.name)
+
+      warn_undeclared_terminal(reference)
+      intern(reference.name, :terminal, location: reference.loc.to_h)
+    end
+
+    # @rbs (String name) -> bool
+    def nonterminal_name?(name)
+      name.match?(/\A[a-z_]/) && name != "error"
+    end
+
+    # @rbs (Frontend::AST::SymbolReference reference) -> bot
+    def undefined_nonterminal(reference)
+      fail_at(reference.loc, "undefined nonterminal #{reference.name}")
+    end
+
+    # @rbs (Frontend::AST::SymbolReference reference) -> void
+    def warn_undeclared_terminal(reference)
+      return if @declared_tokens.empty? || reference.name.start_with?("'", '"')
+
+      @warnings << { type: :undeclared_terminal, symbol: reference.name, loc: reference.loc.to_h }
+    end
+
+    # @rbs () -> Hash[String, String]
+    def normalized_user_code
+      %w[header inner footer].to_h do |name|
+        [name, @ast.user_code.fetch(name, Array.new(0)).map(&:code).join]
+      end
+    end
+
+    # @rbs () -> IR::user_code_chunks
+    def normalized_user_code_chunks
+      chunks_by_name = %w[header inner footer].to_h do |name|
+        chunks = @ast.user_code.fetch(name, Array.new(0)).map do |block|
+          location = block.loc.to_h.merge(line: block.loc.line + 1, column: 1)
+          IR::UserCodeChunk.new(code: block.code, location: location)
+        end
+        [name, chunks]
+      end
+      chunks_by_name.reject { |_name, chunks| chunks.empty? }
+    end
+
+    # @rbs (Frontend::Location location, String message) -> bot
+    def fail_at(location, message)
+      raise Ibex::Error, "#{location}: #{message}"
+    end
+
+    # @rbs (IR::location? location, String message) -> bot
+    def fail_hash(location, message)
+      rendered = location ? "#{location[:file]}:#{location[:line]}:#{location[:column]}" : "(grammar):1:1"
+      raise Ibex::Error, "#{rendered}: #{message}"
+    end
+  end
+end