ibex 0.1.0 → 0.2.0

data/lib/ibex/lsp/workspace_analyzer.rb

@@ -0,0 +1,199 @@
+# frozen_string_literal: true
+
+module Ibex
+  module LSP
+    # Parses workspace sources, discovers safe include closures, and renders diagnostics.
+    class WorkspaceAnalyzer
+      GLOB_CHARACTERS = /[*?\[\]{}]/ #: Regexp
+      WINDOWS_ABSOLUTE = %r{\A(?:[A-Za-z]:[\\/]|\\\\)} #: Regexp
+
+      # @rbs!
+      #   type lsp_diagnostic = Hash[String, untyped]
+      #   type analyzed_document = {
+      #     source: String,
+      #     document: Frontend::SourceDocument?,
+      #     diagnostics: Array[lsp_diagnostic]
+      #   }
+
+      # @rbs (Workspace workspace, Frontend::SourceLoader loader) -> void
+      def initialize(workspace, loader)
+        @workspace = workspace
+        @loader = loader
+      end
+
+      # @rbs (String path) -> analyzed_document
+      def analyze(path)
+        source = @loader.read(path)
+        return analyze_fragment(path, source) if fragment_source?(source, path)
+
+        analyze_root(path, source)
+      rescue Ibex::Error, SystemCallError => e
+        source ||= readable_source(path)
+        { source: source, document: nil, diagnostics: [diagnostic(e.message, source, path)] }
+      end
+
+      # @rbs (String root) -> [Array[String], Hash[String, analyzed_document]]
+      def closure(root)
+        files = [] #: Array[String]
+        analyzed = {} #: Hash[String, analyzed_document]
+        visiting = [root]
+        until visiting.empty?
+          path = visiting.pop
+          next unless path
+          next if analyzed.key?(path)
+
+          current = analyze(path)
+          analyzed[path] = current
+          files << path
+          dependencies(current.fetch(:document)).reverse_each do |target|
+            visiting << target unless analyzed.key?(target)
+          end
+        end
+        [files, analyzed]
+      end
+
+      # @rbs (String root) -> [Frontend::Resolution?, Array[lsp_diagnostic]]
+      def resolve(root)
+        resolution = Frontend::Resolver.new(root, mode: :extended, loader: @loader).resolve
+        [resolution, []]
+      rescue Ibex::Error, SystemCallError => e
+        [nil, [diagnostic(e.message, source_for_error(e.message, root), file_for_error(e.message, root))]]
+      end
+
+      private
+
+      # @rbs (String path, String source) -> analyzed_document
+      def analyze_root(path, source)
+        result = Frontend::Parser.new(source, file: path, mode: :extended)
+                                 .parse_with_diagnostics(max_diagnostics: 20)
+        diagnostics = result.diagnostics.map { |entry| frontend_diagnostic(entry, source) }
+        { source: source, document: result.document, diagnostics: diagnostics }
+      end
+
+      # @rbs (String path, String source) -> analyzed_document
+      def analyze_fragment(path, source)
+        document = Frontend::Parser.new(source, file: path, mode: :extended).parse_source_document
+        { source: source, document: document, diagnostics: [] }
+      end
+
+      # @rbs (String source, String path) -> bool
+      def fragment_source?(source, path)
+        document, = Frontend::Lexer.new(source, file: path).tokenize_document_recovering(max_diagnostics: 1)
+        first = document.tokens.find { |token| token.type != :eof }
+        first&.value == "fragment"
+      rescue Ibex::Error
+        false
+      end
+
+      # @rbs (Frontend::SourceDocument? document) -> Array[String]
+      def dependencies(document)
+        ast = document&.ast
+        return [] unless ast
+
+        ast.declarations.filter_map do |declaration|
+          next unless declaration.is_a?(Frontend::AST::Include)
+
+          canonical_include(declaration)
+        rescue Ibex::Error, SystemCallError
+          nil
+        end
+      end
+
+      # @rbs (Frontend::AST::Include include_node) -> String
+      def canonical_include(include_node)
+        path = include_node.path
+        return invalid_include(include_node) if path.empty? || path.include?("\0")
+        return invalid_include(include_node) if path.start_with?("/") || path.match?(WINDOWS_ABSOLUTE)
+        return invalid_include(include_node) if path.split(%r{[\\/]}).include?("..")
+        return invalid_include(include_node) if path.match?(GLOB_CHARACTERS)
+
+        candidate = File.expand_path(path, File.dirname(include_node.loc.file))
+        canonical = @loader.canonical_path(candidate, allow_missing: true)
+        return canonical if @workspace.root_for(canonical)
+
+        invalid_include(include_node)
+      end
+
+      # @rbs (Frontend::AST::Include include_node) -> bot
+      def invalid_include(include_node)
+        raise Ibex::Error, "#{include_node.loc}: unsafe include path"
+      end
+
+      # @rbs (String message, String source, String fallback_path) -> lsp_diagnostic
+      def diagnostic(message, source, fallback_path)
+        file, line, column, text = diagnostic_parts(message, fallback_path)
+        {
+          "range" => diagnostic_range(source, line, column),
+          "severity" => 1,
+          "code" => "ibex.frontend",
+          "source" => "ibex",
+          "message" => text,
+          "data" => { "file" => file }
+        }
+      end
+
+      # @rbs (Frontend::Diagnostic diagnostic, String source) -> lsp_diagnostic
+      def frontend_diagnostic(diagnostic, source)
+        range = if diagnostic.span
+                  PositionCodec.new(source).range(diagnostic.span)
+                else
+                  diagnostic_range(source, diagnostic.location.line, diagnostic.location.column)
+                end
+        {
+          "range" => range,
+          "severity" => 1,
+          "code" => diagnostic.code,
+          "source" => "ibex",
+          "message" => diagnostic.message,
+          "data" => { "file" => diagnostic.location.file }
+        }
+      rescue ArgumentError, Ibex::Error
+        {
+          "range" => diagnostic_range("", 1, 1),
+          "severity" => 1,
+          "code" => diagnostic.code,
+          "source" => "ibex",
+          "message" => diagnostic.message,
+          "data" => { "file" => diagnostic.location.file }
+        }
+      end
+
+      # @rbs (String message, String fallback_path) -> [String, Integer, Integer, String]
+      def diagnostic_parts(message, fallback_path)
+        match = message.match(/\A(.*):(\d+):(\d+):\s*(.*)\z/m)
+        return [fallback_path, 1, 1, message] unless match
+
+        [match[1] || fallback_path, Integer(match[2]), Integer(match[3]), match[4] || message]
+      end
+
+      # @rbs (String source, Integer line, Integer column) -> Hash[String, Hash[String, Integer]]
+      def diagnostic_range(source, line, column)
+        line_text = source.lines.fetch(line - 1, "").delete_suffix("\n").delete_suffix("\r")
+        prefix = line_text.each_char.take([column - 1, 0].max).join
+        character = prefix.each_codepoint.sum { |codepoint| codepoint > 0xFFFF ? 2 : 1 }
+        point = { "line" => [line - 1, 0].max, "character" => character }
+        { "start" => point, "end" => point.dup }
+      end
+
+      # @rbs (String path) -> String
+      def readable_source(path)
+        @loader.read(path)
+      rescue SystemCallError
+        ""
+      end
+
+      # @rbs (String message, String fallback) -> String
+      def file_for_error(message, fallback)
+        match = message.match(/\A(.*):\d+:\d+:/)
+        return fallback unless match
+
+        match[1] || fallback
+      end
+
+      # @rbs (String message, String fallback) -> String
+      def source_for_error(message, fallback)
+        readable_source(file_for_error(message, fallback))
+      end
+    end
+  end
+end

data/lib/ibex/lsp.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require "json"
+require "uri"
+require_relative "version"
+require_relative "error"
+require_relative "frontend"
+require_relative "lsp/protocol_error"
+require_relative "lsp/transport"
+require_relative "lsp/position_codec"
+require_relative "lsp/workspace"
+require_relative "lsp/workspace_analyzer"
+require_relative "lsp/document_store_validation"
+require_relative "lsp/document_store_diagnostics"
+require_relative "lsp/document_store"
+require_relative "lsp/symbol_occurrence"
+require_relative "lsp/symbol_index_source_queries"
+require_relative "lsp/symbol_index_precedence_references"
+require_relative "lsp/symbol_index_builder"
+require_relative "lsp/symbol_index"
+require_relative "lsp/request_support"
+require_relative "lsp/initialization_handlers"
+require_relative "lsp/document_handlers"
+require_relative "lsp/navigation_handlers"
+require_relative "lsp/request_handlers"
+require_relative "lsp/server"
+
+module Ibex
+  # Language Server Protocol support for grammar workspaces.
+  module LSP
+  end
+end

data/lib/ibex/normalize/declarations.rb

@@ -11,27 +11,160 @@ module Ibex
       @declared_tokens = {} #: Hash[String, IR::location]
       @precedence = {} #: Hash[String, IR::precedence]
       @precedence_locations = {} #: Hash[String, IR::location]
+      @display_names = {} #: Hash[String, String]
+      @display_name_locations = {} #: Hash[String, IR::location]
+      @semantic_types = {} #: Hash[String, String]
+      @semantic_type_locations = {} #: Hash[String, IR::location]
       @options = { result_var: true, omit_action_call: true }
+      @options[:cst] = true if @ast.cst
       @expected_conflicts = 0
+      @expected_rr_conflicts = nil
       @conversions = {} #: Hash[String, String]
+      @parser_parameters = [] #: Array[IR::parser_parameter]
+      @value_printers = {} #: Hash[String, IR::value_printer]
+      @recovery_sync_tokens = [] #: Array[String]
+      @recovery_sync_location = nil #: Frontend::Location?
+      @on_error_reduce_groups = [] #: Array[Array[String]]
+      @on_error_reduce_locations = {} #: Hash[String, Frontend::Location]
+      @grammar_tests = [] #: Array[IR::grammar_test]
+      @lexer_declaration = nil #: Frontend::AST::Lexer?
+      @node_shapes = {} #: Hash[String, Array[String]]
       @ast.declarations.each { |declaration| read_declaration(declaration) }
     end
 
     # @rbs (Frontend::AST::declaration declaration) -> void
-    def read_declaration(declaration)
+    def read_declaration(declaration) # rubocop:disable Metrics/CyclomaticComplexity
       # @type self: Normalizer
       case declaration
-      when Frontend::AST::Tokens then declaration.names.each { |name| @declared_tokens[name] = declaration.loc.to_h }
+      when Frontend::AST::Tokens then read_tokens(declaration)
       when Frontend::AST::Precedence then read_precedence(declaration)
-      when Frontend::AST::Options then declaration.names.each { |name| read_option(name, declaration.loc) }
+      when Frontend::AST::Options then read_options(declaration)
       when Frontend::AST::Expect then @expected_conflicts = declaration.conflicts
-      when Frontend::AST::Start
-        @explicit_start = declaration.name
-        @start_location = declaration.loc
-      when Frontend::AST::Convert then declaration.pairs.each { |pair| @conversions[pair.name] = pair.expression }
+      when Frontend::AST::ExpectRR then @expected_rr_conflicts = declaration.conflicts
+      when Frontend::AST::Start, Frontend::AST::Recovery, Frontend::AST::OnErrorReduce, Frontend::AST::GrammarTest
+        read_parser_control_declaration(declaration)
+      when Frontend::AST::Lexer
+        fail_at(declaration.loc, "duplicate lexer declaration") if @lexer_declaration
+        @lexer_declaration = declaration
+      when Frontend::AST::Convert then read_conversions(declaration)
+      when Frontend::AST::DisplayName, Frontend::AST::SemanticType, Frontend::AST::Parameter, Frontend::AST::Printer
+        read_extended_declaration(declaration)
+      when Frontend::AST::Include then fail_at(declaration.loc, "includes must be resolved before normalization")
       end
     end
 
+    # @rbs (Frontend::AST::Start declaration) -> void
+    def read_start_declaration(declaration)
+      # @type self: Normalizer
+      fail_at(declaration.loc, "duplicate start declaration") if @explicit_starts
+      if declaration.names.length > 1 && @mode != :extended
+        fail_at(declaration.loc, "multiple start symbols require extended mode")
+      end
+      fail_at(declaration.loc, "start declaration requires at least one symbol") if declaration.names.empty?
+      unless declaration.names.uniq.length == declaration.names.length
+        fail_at(declaration.loc, "start symbols must be unique")
+      end
+
+      @explicit_starts = declaration.names
+      @start_location = declaration.loc
+    end
+
+    # @rbs (Frontend::AST::symbol_metadata | Frontend::AST::Parameter | Frontend::AST::Printer declaration) -> void
+    def read_extended_declaration(declaration)
+      if declaration.is_a?(Frontend::AST::Parameter)
+        read_parser_parameter(declaration)
+      elsif declaration.is_a?(Frontend::AST::Printer)
+        read_value_printer(declaration)
+      else
+        read_symbol_metadata_declaration(declaration)
+      end
+    end
+
+    # @rbs (Frontend::AST::Printer declaration) -> void
+    def read_value_printer(declaration)
+      # @type self: Normalizer
+      if @value_printers.key?(declaration.name)
+        fail_at(declaration.loc, "duplicate %printer declaration for #{declaration.name}")
+      end
+
+      @value_printers[declaration.name] = {
+        symbol: declaration.name, code: declaration.code, loc: declaration.loc.to_h
+      }
+    end
+
+    # @rbs () -> void
+    def validate_value_printers
+      @value_printers.each_value do |printer|
+        next if @symbols_by_name.key?(printer[:symbol])
+
+        location = printer[:loc]
+        raise Ibex::Error,
+              "#{location[:file]}:#{location[:line]}:#{location[:column]}: " \
+              "%printer references missing symbol #{printer[:symbol]}"
+      end
+    end
+
+    # @rbs (Frontend::AST::Parameter declaration) -> void
+    def read_parser_parameter(declaration)
+      # @type self: Normalizer
+      if @parser_parameters.any? { |parameter| parameter[:name] == declaration.name }
+        fail_at(declaration.loc, "duplicate %param declaration for #{declaration.name}")
+      end
+      unless declaration.name.match?(/\A[a-z_][a-zA-Z0-9_]*\z/)
+        fail_at(declaration.loc, "%param name #{declaration.name.inspect} must be a Ruby local identifier")
+      end
+      if Normalizer::RUBY_KEYWORDS.include?(declaration.name)
+        fail_at(declaration.loc, "%param name #{declaration.name.inspect} is a Ruby keyword")
+      end
+
+      @parser_parameters << { name: declaration.name, semantic_type: declaration.semantic_type }
+    end
+
+    # @rbs (Frontend::AST::Tokens declaration) -> void
+    def read_tokens(declaration)
+      # @type self: Normalizer
+      declaration.names.each { |name| @declared_tokens[name] = declaration.loc.to_h }
+      (declaration.aliases || {}).each do |name, value|
+        fail_at(declaration.loc, "duplicate display declaration for #{name}") if @display_names.key?(name)
+
+        @display_names[name] = value
+        @display_name_locations[name] = declaration.loc.to_h
+      end
+    end
+
+    # @rbs (Frontend::AST::Options declaration) -> void
+    def read_options(declaration)
+      declaration.names.each { |name| read_option(name, declaration.loc) }
+    end
+
+    # @rbs (Frontend::AST::Convert declaration) -> void
+    def read_conversions(declaration)
+      # @type self: Normalizer
+      declaration.pairs.each { |pair| @conversions[pair.name] = pair.expression }
+    end
+
+    # @rbs (Frontend::AST::symbol_metadata declaration) -> void
+    def read_symbol_metadata_declaration(declaration)
+      # @type self: Normalizer
+      if declaration.is_a?(Frontend::AST::DisplayName)
+        read_symbol_metadata(declaration, @display_names, @display_name_locations, "display")
+      else
+        read_symbol_metadata(declaration, @semantic_types, @semantic_type_locations, "type")
+      end
+    end
+
+    # @rbs (Frontend::AST::symbol_metadata declaration, Hash[String, String] values,
+    #   Hash[String, IR::location] locations, String label) -> void
+    def read_symbol_metadata(declaration, values, locations, label)
+      # @type self: Normalizer
+      if values.key?(declaration.name)
+        fail_at(declaration.loc, "duplicate #{label} declaration for #{declaration.name}")
+      end
+
+      values[declaration.name] = declaration.value
+      locations[declaration.name] = declaration.loc.to_h
+    end
+
     # @rbs (Frontend::AST::Precedence declaration) -> void
     def read_precedence(declaration)
       # @type self: Normalizer

data/lib/ibex/normalize/diagnostics.rb

@@ -10,12 +10,26 @@ module Ibex
     # @rbs () -> void
     def validate_grammar
       # @type self: Normalizer
+      validate_symbol_metadata
       warn_duplicate_productions
       warn_unreachable_nonterminals
+      warn_unreachable_terminals
       warn_unused_terminals
+      warn_unused_precedence
       warn_empty_language
     end
 
+    # @rbs () -> void
+    def validate_symbol_metadata
+      # @type self: Normalizer
+      { display: @display_name_locations, type: @semantic_type_locations }.each do |label, locations|
+        locations.each do |name, location|
+          fail_hash(location, "#{label} declaration references undefined symbol #{name}") unless
+            symbol(name) || parameter_template?(name) || (label == :type && @inline_rule_names.include?(name))
+        end
+      end
+    end
+
     # @rbs () -> void
     def warn_duplicate_productions
       # @type self: Normalizer
@@ -45,8 +59,8 @@ module Ibex
     # @rbs () -> Set[Integer]
     def reachable_symbol_ids
       # @type self: Normalizer
-      start = required_symbol(@start_name).id
-      reachable = Set[start]
+      starts = @start_names.map { |name| required_symbol(name).id }
+      reachable = Set.new(starts)
       loop do
         before = reachable.length
         @productions.select { |production| reachable.include?(production.lhs) }.each do |production|
@@ -67,6 +81,35 @@ module Ibex
       end
     end
 
+    # @rbs () -> void
+    def warn_unreachable_terminals
+      # @type self: Normalizer
+      reachable = reachable_symbol_ids
+      used = @productions.flat_map(&:rhs).to_set
+      @declared_tokens.each_key do |name|
+        grammar_symbol = required_symbol(name)
+        next unless used.include?(grammar_symbol.id)
+        next if reachable.include?(grammar_symbol.id)
+
+        @warnings << { type: :unreachable_terminal, symbol: name, loc: @declared_tokens[name] }
+      end
+    end
+
+    # @rbs () -> void
+    def warn_unused_precedence
+      # @type self: Normalizer
+      referenced = @productions.each_with_object(Set.new) do |production, symbol_ids|
+        production.rhs.each { |id| symbol_ids << id }
+        symbol_ids << production.precedence_override if production.precedence_override
+      end
+      @precedence.each_key do |name|
+        grammar_symbol = required_symbol(name)
+        next if referenced.include?(grammar_symbol.id)
+
+        @warnings << { type: :unused_precedence, symbol: name, loc: @precedence_locations[name] }
+      end
+    end
+
     # @rbs () -> void
     def warn_empty_language
       # @type self: Normalizer
@@ -78,10 +121,12 @@ module Ibex
         end
         break if productive.length == before
       end
-      return if productive.include?(required_symbol(@start_name).id)
+      @start_names.each do |name|
+        next if productive.include?(required_symbol(name).id)
 
-      start_symbol = required_symbol(@start_name)
-      @warnings << { type: :empty_language, symbol: @start_name, loc: start_symbol.location }
+        start_symbol = required_symbol(name)
+        @warnings << { type: :empty_language, symbol: name, loc: start_symbol.location }
+      end
     end
 
     # @rbs () -> Set[Integer]

data/lib/ibex/normalize/expander.rb

@@ -9,6 +9,9 @@ module Ibex
     def normalize_user_productions
       # @type self: Normalizer
       @ast.rules.each do |rule|
+        next if parameterized_rule?(rule)
+
+        @current_include_chain = @resolution&.include_chain_for(rule) || []
         rule.alternatives.each { |alternative| normalize_alternative(rule, alternative) }
       end
     end
@@ -18,7 +21,8 @@ module Ibex
       # @type self: Normalizer
       rhs = [] #: Array[String]
       named_refs = [] #: Array[IR::named_ref]
-      alternative.items.each do |item|
+      items = normalized_alternative_items(alternative)
+      items.each do |item|
         if item.is_a?(Frontend::AST::InlineAction)
           rhs << expand_inline_action(item, rhs.length, named_refs)
           next
@@ -28,14 +32,31 @@ module Ibex
         add_named_reference(item, named_refs, rhs.length - 1)
       end
       action = normalize_action(alternative.action, named_refs)
+      node = normalize_node_annotation(alternative, rhs)
       add_production(rule.lhs, rhs, action, alternative.precedence,
-                     { kind: :user, loc: alternative.loc.to_h })
+                     { kind: :user, loc: alternative.loc.to_h }, rule.documentation, node)
+    end
+
+    # @rbs (Frontend::AST::Alternative alternative) -> Array[Frontend::AST::item]
+    def normalized_alternative_items(alternative)
+      # @type self: Normalizer
+      explicit = alternative.items.grep(Frontend::AST::Empty)
+      if explicit.any?
+        fail_at(explicit.first.loc, "%empty must be the only item in an alternative") unless alternative.items.one?
+        return []
+      end
+
+      if alternative.items.empty? && (@mode.to_sym == :extended || @ast.extended)
+        @warnings << { type: :implicit_empty, loc: alternative.loc.to_h }
+      end
+      alternative.items
     end
 
     # @rbs (Frontend::AST::item item) -> String
     def normalize_item(item)
       # @type self: Normalizer
       return symbol_for_reference(item).name if item.is_a?(Frontend::AST::SymbolReference)
+      return specialize_parameterized_reference(item) if item.is_a?(Frontend::AST::ParameterizedReference)
       return expand_group(item) if item.is_a?(Frontend::AST::Group)
       return expand_optional(item) if item.is_a?(Frontend::AST::Optional)
       return expand_star(item) if item.is_a?(Frontend::AST::Star)
@@ -59,6 +80,12 @@ module Ibex
     def expand_optional(item)
       # @type self: Normalizer
       base = normalize_item(item.item)
+      build_optional(item, base)
+    end
+
+    # @rbs (Frontend::AST::Optional item, String base) -> String
+    def build_optional(item, base)
+      # @type self: Normalizer
       helper = new_helper("optional", item.loc)
       add_production(helper, [], synthetic_action("nil", item.loc), nil, synthetic_origin(:optional, item))
       add_production(helper, [base], synthetic_action("val[0]", item.loc), nil, synthetic_origin(:optional, item))
@@ -69,6 +96,12 @@ module Ibex
     def expand_star(item)
       # @type self: Normalizer
       base = normalize_item(item.item)
+      build_star(item, base)
+    end
+
+    # @rbs (Frontend::AST::Star item, String base) -> String
+    def build_star(item, base)
+      # @type self: Normalizer
       helper = new_helper("star", item.loc)
       add_production(helper, [], synthetic_action("[]", item.loc), nil, synthetic_origin(:star, item))
       add_production(helper, [helper, base], synthetic_action("val[0] + [val[1]]", item.loc), nil,
@@ -80,6 +113,12 @@ module Ibex
     def expand_plus(item)
       # @type self: Normalizer
       base = normalize_item(item.item)
+      build_plus(item, base)
+    end
+
+    # @rbs (Frontend::AST::Plus item, String base) -> String
+    def build_plus(item, base)
+      # @type self: Normalizer
       helper = new_helper("plus", item.loc)
       add_production(helper, [base], synthetic_action("[val[0]]", item.loc), nil, synthetic_origin(:plus, item))
       add_production(helper, [helper, base], synthetic_action("val[0] + [val[1]]", item.loc), nil,
@@ -92,6 +131,12 @@ module Ibex
       # @type self: Normalizer
       base = normalize_item(item.item)
       separator = normalize_item(item.separator)
+      build_separated_list(item, base, separator)
+    end
+
+    # @rbs (Frontend::AST::SeparatedList item, String base, String separator) -> String
+    def build_separated_list(item, base, separator)
+      # @type self: Normalizer
       helper = new_helper("separated_list", item.loc)
       unless item.nonempty
         add_production(helper, [], synthetic_action("[]", item.loc), nil, synthetic_origin(:separated_list, item))
@@ -110,12 +155,18 @@ module Ibex
       helper = new_helper("group", item.loc)
       item.alternatives.each do |alternative|
         rhs = alternative.map { |child| normalize_item(child) }
-        expression = group_value_expression(rhs.length)
-        add_production(helper, rhs, synthetic_action(expression, item.loc), nil, synthetic_origin(:group, item))
+        add_group_production(helper, rhs, item)
       end
       helper
     end
 
+    # @rbs (String helper, Array[String] rhs, Frontend::AST::Group item) -> void
+    def add_group_production(helper, rhs, item)
+      # @type self: Normalizer
+      expression = group_value_expression(rhs.length)
+      add_production(helper, rhs, synthetic_action(expression, item.loc), nil, synthetic_origin(:group, item))
+    end
+
     # @rbs (Integer length) -> String
     def group_value_expression(length)
       # @type self: Normalizer
@@ -125,28 +176,6 @@ module Ibex
       "val"
     end
 
-    # @rbs (Frontend::AST::Group group) -> void
-    def reject_group_named_references(group)
-      # @type self: Normalizer
-      reference = group.alternatives.flatten.filter_map { |item| named_reference_in(item) }.first
-      fail_at(reference.loc, "named references inside EBNF groups are not supported") if reference
-    end
-
-    # @rbs (Frontend::AST::item item) -> Frontend::AST::SymbolReference?
-    def named_reference_in(item)
-      # @type self: Normalizer
-      return item if item.is_a?(Frontend::AST::SymbolReference) && item.named_reference
-      if item.is_a?(Frontend::AST::Group)
-        return item.alternatives.flatten.filter_map { |child| named_reference_in(child) }.first
-      end
-      if item.is_a?(Frontend::AST::Optional) || item.is_a?(Frontend::AST::Star) ||
-         item.is_a?(Frontend::AST::Plus) || item.is_a?(Frontend::AST::SeparatedList)
-        return named_reference_in(item.item)
-      end
-
-      nil
-    end
-
     # @rbs (String kind, Frontend::Location location) -> String
     def new_helper(kind, location)
       # @type self: Normalizer
@@ -177,42 +206,17 @@ module Ibex
       IR::Action.new(code: action.code, location: action.loc.to_h, named_refs: named_refs)
     end
 
-    # @rbs (Frontend::AST::item item, Array[IR::named_ref] refs, Integer index) -> void
-    def add_named_reference(item, refs, index)
-      # @type self: Normalizer
-      reference = unwrap_reference(item)
-      return unless reference
-
-      name = reference.named_reference
-      return unless name
-
-      fail_at(reference.loc, "reserved named reference #{name}") if Normalizer::RESERVED_NAMES.include?(name)
-      fail_at(reference.loc, "duplicate named reference #{name}") if refs.any? { |entry| entry[:name] == name }
-      refs << { name: name, index: index }
-    end
-
-    # @rbs (Frontend::AST::item item) -> Frontend::AST::SymbolReference?
-    def unwrap_reference(item)
-      # @type self: Normalizer
-      return item if item.is_a?(Frontend::AST::SymbolReference)
-      if item.is_a?(Frontend::AST::Optional) || item.is_a?(Frontend::AST::Star) ||
-         item.is_a?(Frontend::AST::Plus) || item.is_a?(Frontend::AST::SeparatedList)
-        return unwrap_reference(item.item)
-      end
-
-      nil
-    end
-
     # @rbs (String lhs_name, Array[String] rhs_names, IR::Action? action, String? precedence_name,
-    #   Hash[Symbol, untyped] origin) -> void
-    def add_production(lhs_name, rhs_names, action, precedence_name, origin)
+    #   Hash[Symbol, untyped] origin, ?String? documentation, ?IR::node_annotation? node) -> void
+    def add_production(lhs_name, rhs_names, action, precedence_name, origin, documentation = nil, node = nil)
       # @type self: Normalizer
       lhs = symbol(lhs_name) || intern(lhs_name, :nonterminal, location: origin[:loc])
       rhs = rhs_names.map { |name| symbol(name)&.id || fail_hash(origin[:loc], "undefined symbol #{name}") }
       precedence = precedence_name && symbol(precedence_name)
       fail_hash(origin[:loc], "undefined precedence symbol #{precedence_name}") if precedence_name && !precedence
       @productions << IR::Production.new(id: @productions.length, lhs: lhs.id, rhs: rhs, action: action,
-                                         precedence_override: precedence&.id, origin: origin)
+                                         precedence_override: precedence&.id, origin: origin,
+                                         documentation: documentation, expansion: resolved_expansion, node: node)
     end
   end
 end