ibex 0.1.0 → 0.2.0

data/lib/ibex/normalize/parameter_substitution.rb

@@ -0,0 +1,125 @@
+# frozen_string_literal: true
+
+module Ibex
+  # Structural, sharing-preserving substitution for parameterized user rules.
+  module NormalizeParameterSubstitution
+    private
+
+    # @rbs (Frontend::AST::Alternative alternative, Hash[String, Frontend::AST::item] bindings) ->
+    #   Frontend::AST::Alternative
+    def substitute_parameter_alternative(alternative, bindings)
+      # @type self: Normalizer
+      Frontend::AST::Alternative.new(
+        items: alternative.items.map { |item| substitute_parameter_item(item, bindings) },
+        action: alternative.action, precedence: substitute_parameter_precedence(alternative, bindings),
+        node_annotation: alternative.node_annotation, loc: alternative.loc
+      )
+    end
+
+    # @rbs (Frontend::AST::Alternative alternative, Hash[String, Frontend::AST::item] bindings) -> String?
+    def substitute_parameter_precedence(alternative, bindings)
+      # @type self: Normalizer
+      precedence = alternative.precedence
+      return precedence unless precedence && bindings.key?(precedence)
+
+      argument = bindings.fetch(precedence)
+      unless argument.is_a?(Frontend::AST::SymbolReference) && !argument.named_reference
+        fail_at(alternative.loc, "formal precedence #{precedence} requires one plain symbol argument")
+      end
+      argument.name
+    end
+
+    # @rbs (Frontend::AST::item item, Hash[String, Frontend::AST::item] bindings) -> Frontend::AST::item
+    def substitute_parameter_item(item, bindings)
+      # @type self: Normalizer
+      if item.is_a?(Frontend::AST::SymbolReference) && bindings.key?(item.name)
+        return substitute_formal_reference(item, bindings.fetch(item.name))
+      end
+
+      clone_parameter_item(item, bindings)
+    end
+
+    # @rbs (Frontend::AST::SymbolReference formal, Frontend::AST::item argument) -> Frontend::AST::item
+    def substitute_formal_reference(formal, argument)
+      # @type self: Normalizer
+      name = formal.named_reference
+      return argument unless name
+
+      case argument
+      when Frontend::AST::SymbolReference
+        Frontend::AST::SymbolReference.new(name: argument.name, named_reference: name, loc: argument.loc)
+      when Frontend::AST::ParameterizedReference
+        Frontend::AST::ParameterizedReference.new(
+          name: argument.name, arguments: argument.arguments, named_reference: name, loc: argument.loc
+        )
+      else
+        fail_at(formal.loc, "named formal #{formal.name} requires a plain symbol or parameterized argument")
+      end
+    end
+
+    # @rbs (Frontend::AST::item item, ?Hash[String, Frontend::AST::item] bindings) -> Frontend::AST::item
+    def clone_parameter_item(item, bindings = {})
+      # @type self: Normalizer
+      case item
+      when Frontend::AST::SymbolReference
+        clone_parameter_symbol(item, bindings)
+      when Frontend::AST::ParameterizedReference
+        clone_parameter_call(item, bindings)
+      when Frontend::AST::Group
+        clone_parameter_group(item, bindings)
+      when Frontend::AST::Optional, Frontend::AST::Star, Frontend::AST::Plus
+        clone_parameter_suffix(item, bindings)
+      when Frontend::AST::SeparatedList
+        clone_parameter_separated_list(item, bindings)
+      else
+        item
+      end
+    end
+
+    # @rbs (Frontend::AST::SymbolReference item, Hash[String, Frontend::AST::item] bindings) -> Frontend::AST::item
+    def clone_parameter_symbol(item, bindings)
+      # @type self: Normalizer
+      return substitute_parameter_item(item, bindings) if bindings.key?(item.name)
+
+      Frontend::AST::SymbolReference.new(name: item.name, named_reference: item.named_reference, loc: item.loc)
+    end
+
+    # @rbs (Frontend::AST::ParameterizedReference item, Hash[String, Frontend::AST::item] bindings) ->
+    #   Frontend::AST::ParameterizedReference
+    def clone_parameter_call(item, bindings)
+      # @type self: Normalizer
+      fail_at(item.loc, "formal #{item.name} cannot be used as a parameterized rule name") if bindings.key?(item.name)
+      Frontend::AST::ParameterizedReference.new(
+        name: item.name, arguments: item.arguments.map { |argument| substitute_parameter_item(argument, bindings) },
+        named_reference: item.named_reference, loc: item.loc
+      )
+    end
+
+    # @rbs (Frontend::AST::Group item, Hash[String, Frontend::AST::item] bindings) -> Frontend::AST::Group
+    def clone_parameter_group(item, bindings)
+      # @type self: Normalizer
+      alternatives = item.alternatives.map do |items|
+        items.map { |child| substitute_parameter_item(child, bindings) }
+      end
+      Frontend::AST::Group.new(alternatives: alternatives, loc: item.loc)
+    end
+
+    # @rbs (Frontend::AST::Optional | Frontend::AST::Star | Frontend::AST::Plus item,
+    #   Hash[String, Frontend::AST::item] bindings) -> Frontend::AST::item
+    def clone_parameter_suffix(item, bindings)
+      # @type self: Normalizer
+      item.class.new(item: substitute_parameter_item(item.item, bindings), loc: item.loc)
+    end
+
+    # @rbs (Frontend::AST::SeparatedList item, Hash[String, Frontend::AST::item] bindings) ->
+    #   Frontend::AST::SeparatedList
+    def clone_parameter_separated_list(item, bindings)
+      # @type self: Normalizer
+      Frontend::AST::SeparatedList.new(
+        item: substitute_parameter_item(item.item, bindings),
+        separator: substitute_parameter_item(item.separator, bindings),
+        nonempty: item.nonempty, loc: item.loc
+      )
+    end
+  end
+end

data/lib/ibex/normalize/parameter_validation.rb

@@ -0,0 +1,140 @@
+# frozen_string_literal: true
+
+module Ibex
+  # Static validation and collection of parameterized user rule definitions.
+  module NormalizeParameterValidation
+    private
+
+    # @rbs () -> void
+    def gather_parameter_templates
+      # @type self: Normalizer
+      @parameter_templates = {} #: Hash[String, Array[Frontend::AST::Rule]]
+      @parameter_formals = {} #: Hash[String, Array[String]]
+      plain_rules = {} #: Hash[String, Frontend::AST::Rule]
+      @rule_documentation = validated_rule_documentation
+
+      @ast.rules.each do |rule|
+        if parameterized_rule?(rule)
+          gather_parameterized_rule(rule, plain_rules)
+        else
+          reject_mixed_parameterized_rule(rule) if @parameter_templates.key?(rule.lhs)
+          plain_rules[rule.lhs] ||= rule
+        end
+      end
+      validate_parameter_terminal_collisions
+      @ast.rules.each { |rule| validate_parameter_references(rule) }
+    end
+
+    # @rbs (Frontend::AST::Rule rule, Hash[String, Frontend::AST::Rule] plain_rules) -> void
+    def gather_parameterized_rule(rule, plain_rules)
+      # @type self: Normalizer
+      reject_mixed_parameterized_rule(rule) if plain_rules.key?(rule.lhs)
+      duplicate = rule.parameters.tally.find { |_name, count| count > 1 }&.first
+      fail_at(rule.loc, "duplicate parameter #{duplicate} in rule #{rule.lhs}") if duplicate
+
+      validate_parameter_formals(rule)
+      @parameter_formals[rule.lhs] ||= rule.parameters
+      (@parameter_templates[rule.lhs] ||= Array.new(0)) << rule
+    end
+
+    # @rbs (Frontend::AST::Rule rule) -> void
+    def validate_parameter_formals(rule)
+      # @type self: Normalizer
+      expected = @parameter_formals[rule.lhs]
+      return unless expected && expected != rule.parameters
+
+      fail_at(
+        rule.loc,
+        "parameterized rule #{rule.lhs} uses inconsistent parameters " \
+        "(expected #{expected.join(', ')}, got #{rule.parameters.join(', ')})"
+      )
+    end
+
+    # @rbs (Frontend::AST::Rule rule) -> bot
+    def reject_mixed_parameterized_rule(rule)
+      # @type self: Normalizer
+      fail_at(rule.loc, "rule #{rule.lhs} has both plain and parameterized definitions")
+    end
+
+    # @rbs () -> void
+    def validate_parameter_terminal_collisions
+      # @type self: Normalizer
+      @parameter_templates.each do |name, rules|
+        next unless @declared_tokens.key?(name) || @precedence.key?(name)
+
+        fail_at(rules.fetch(0).loc, "parameterized rule #{name} collides with terminal #{name}")
+      end
+    end
+
+    # @rbs (Frontend::AST::Rule rule) -> void
+    def validate_parameter_references(rule)
+      # @type self: Normalizer
+      rule.alternatives.each do |alternative|
+        alternative.items.each do |item|
+          validate_parameter_item(item, rule.parameters, inside_argument: false)
+        end
+      end
+    end
+
+    # @rbs (Frontend::AST::item item, Array[String] formals, inside_argument: bool) -> void
+    def validate_parameter_item(item, formals, inside_argument:)
+      # @type self: Normalizer
+      case item
+      when Frontend::AST::ParameterizedReference
+        validate_parameterized_item(item, formals, inside_argument: inside_argument)
+      when Frontend::AST::SymbolReference
+        reject_argument_named_reference(item) if inside_argument && item.named_reference
+      when Frontend::AST::Group
+        item.alternatives.flatten.each do |child|
+          validate_parameter_item(child, formals, inside_argument: inside_argument)
+        end
+      when Frontend::AST::Optional, Frontend::AST::Star, Frontend::AST::Plus
+        validate_parameter_item(item.item, formals, inside_argument: inside_argument)
+      when Frontend::AST::SeparatedList
+        validate_parameter_item(item.item, formals, inside_argument: inside_argument)
+        validate_parameter_item(item.separator, formals, inside_argument: inside_argument)
+      end
+    end
+
+    # @rbs (Frontend::AST::ParameterizedReference item, Array[String] formals, inside_argument: bool) -> void
+    def validate_parameterized_item(item, formals, inside_argument:)
+      # @type self: Normalizer
+      if formals.include?(item.name)
+        fail_at(item.loc, "formal #{item.name} cannot be used as a parameterized rule name")
+      end
+      validate_parameter_call(item)
+      reject_argument_named_reference(item) if inside_argument && item.named_reference
+      item.arguments.each { |argument| validate_parameter_item(argument, formals, inside_argument: true) }
+    end
+
+    # @rbs (Frontend::AST::SymbolReference | Frontend::AST::ParameterizedReference item) -> bot
+    def reject_argument_named_reference(item)
+      # @type self: Normalizer
+      fail_at(item.loc, "named references are not allowed in parameter arguments")
+    end
+
+    # @rbs (Frontend::AST::ParameterizedReference reference) -> void
+    def validate_parameter_call(reference)
+      # @type self: Normalizer
+      formals = @parameter_formals[reference.name]
+      fail_at(reference.loc, "undefined parameterized rule #{reference.name}") unless formals
+      return if reference.arguments.length == formals.length
+
+      fail_at(
+        reference.loc,
+        "parameterized rule #{reference.name} expects #{formals.length} arguments, " \
+        "got #{reference.arguments.length}"
+      )
+    end
+
+    # @rbs (Frontend::AST::Rule rule) -> bool
+    def parameterized_rule?(rule)
+      !rule.parameters.empty?
+    end
+
+    # @rbs (String name) -> bool
+    def parameter_template?(name)
+      @parameter_templates.key?(name)
+    end
+  end
+end

data/lib/ibex/normalize/parameters.rb

@@ -0,0 +1,199 @@
+# frozen_string_literal: true
+
+module Ibex
+  # Iterative, hygienic specialization of validated parameterized user rules.
+  module NormalizeParameters
+    private
+
+    # @rbs (Frontend::AST::ParameterizedReference reference) -> String
+    def specialize_parameterized_reference(reference)
+      # @type self: Normalizer
+      helper, scheduled = schedule_parameter_specialization(reference)
+      if scheduled && @parameter_worklist_active
+        raise Ibex::Error, "#{reference.loc}: internal parameter worklist ordering failure"
+      end
+
+      drain_parameter_worklist unless @parameter_worklist_active
+      helper
+    end
+
+    # @rbs (Frontend::AST::ParameterizedReference reference) -> [String, bool]
+    def schedule_parameter_specialization(reference)
+      # @type self: Normalizer
+      arguments = reference.arguments
+      rendered_arguments = arguments.map { |argument| NormalizeExpression.render(argument) }
+      key = [reference.name, rendered_arguments] #: [String, Array[String]]
+      existing = @parameter_specializations[key]
+      return [existing, false] if existing
+
+      enforce_parameter_limits!(reference)
+      helper = new_parameter_helper(reference)
+      @parameter_specializations[key] = helper
+      @parameter_worklist << parameter_frame(reference, helper, arguments, rendered_arguments)
+      [helper, true]
+    end
+
+    # @rbs (Frontend::AST::ParameterizedReference reference) -> void
+    def enforce_parameter_limits!(reference)
+      # @type self: Normalizer
+      cyclic = @parameter_worklist.reverse.find do |frame|
+        frame.fetch(:reference).name == reference.name &&
+          growing_parameter_arguments?(reference.arguments, frame.fetch(:arguments))
+      end
+      if cyclic
+        path = (@parameter_worklist.drop_while { |frame| frame != cyclic } + [{ reference: reference }])
+               .map { |frame| frame.fetch(:reference).name }
+        fail_at(reference.loc, "cyclic parameter specialization #{path.join(' -> ')}")
+      end
+      return unless @parameter_specializations.length >= @max_parameter_specializations
+
+      fail_at(
+        reference.loc,
+        "parameter specialization limit of #{@max_parameter_specializations} exceeded"
+      )
+    end
+
+    # A constructor-growing recursive call can create infinitely many distinct
+    # specializations. Calls whose arguments shrink or merely change are
+    # allowed; they may reach a memoized fixed point after finite expansion.
+    # @rbs (Array[Frontend::AST::item] candidates, Array[Frontend::AST::item] ancestors) -> bool
+    def growing_parameter_arguments?(candidates, ancestors)
+      return false unless candidates.length == ancestors.length
+
+      contained = candidates.each_index.all? do |index|
+        candidate = candidates.fetch(index)
+        ancestor = ancestors.fetch(index)
+        parameter_argument_contains?(candidate, NormalizeExpression.render(ancestor))
+      end
+      contained && candidates.each_index.any? do |index|
+        candidate = candidates.fetch(index)
+        ancestor = ancestors.fetch(index)
+        NormalizeExpression.render(candidate) != NormalizeExpression.render(ancestor)
+      end
+    end
+
+    # @rbs (Frontend::AST::item candidate, String target) -> bool
+    def parameter_argument_contains?(candidate, target)
+      pending = [candidate] #: Array[Frontend::AST::item]
+      until pending.empty?
+        item = pending.pop
+        raise Ibex::Error, "missing parameter argument" unless item
+
+        return true if NormalizeExpression.render(item) == target
+
+        case item
+        when Frontend::AST::ParameterizedReference then pending.concat(item.arguments)
+        when Frontend::AST::Group then item.alternatives.each { |alternative| pending.concat(alternative) }
+        when Frontend::AST::Optional, Frontend::AST::Star, Frontend::AST::Plus then pending << item.item
+        when Frontend::AST::SeparatedList then pending.push(item.item, item.separator)
+        end
+      end
+      false
+    end
+
+    # @rbs (Frontend::AST::ParameterizedReference reference) -> String
+    def new_parameter_helper(reference)
+      # @type self: Normalizer
+      @helper_sequence += 1
+      name = "$parameter_#{@helper_sequence}"
+      definition = intern(
+        name, :nonterminal,
+        location: reference.loc.to_h,
+        documentation: @rule_documentation[reference.name],
+        metadata_name: reference.name
+      )
+      if @inline_rule_names.include?(reference.name)
+        @inline_symbol_ids << definition.id
+        @inline_rule_by_symbol[definition.id] = reference.name
+      end
+      name
+    end
+
+    # @rbs (Frontend::AST::ParameterizedReference reference, String helper,
+    #   Array[Frontend::AST::item] arguments, Array[String] rendered_arguments) ->
+    #   Hash[Symbol, untyped]
+    def parameter_frame(reference, helper, arguments, rendered_arguments)
+      # @type self: Normalizer
+      formals = @parameter_formals.fetch(reference.name)
+      {
+        reference: reference,
+        arguments: arguments,
+        helper: helper,
+        bindings: formals.zip(arguments).to_h,
+        rendered_arguments: rendered_arguments,
+        templates: @parameter_templates.fetch(reference.name),
+        template_index: 0,
+        alternative_index: 0,
+        current: nil,
+        item_index: 0,
+        rhs: [],
+        named_refs: [],
+        operations: [],
+        values: []
+      }
+    end
+
+    # @rbs () -> void
+    def drain_parameter_worklist
+      # @type self: Normalizer
+      @parameter_worklist_active = true
+      until @parameter_worklist.empty?
+        frame = @parameter_worklist.last
+        unless frame[:current] || prepare_parameter_alternative?(frame)
+          @parameter_worklist.pop
+          next
+        end
+
+        advance_parameter_alternative(frame)
+      end
+    ensure
+      @parameter_worklist_active = false
+    end
+
+    # @rbs (Hash[Symbol, untyped] frame) -> bool
+    def prepare_parameter_alternative?(frame)
+      # @type self: Normalizer
+      templates = frame.fetch(:templates)
+      while frame.fetch(:template_index) < templates.length
+        template = templates.fetch(frame.fetch(:template_index))
+        index = frame.fetch(:alternative_index)
+        if index < template.alternatives.length
+          prepare_parameter_alternative_entry(frame, template, index)
+          return true
+        end
+        frame[:template_index] += 1
+        frame[:alternative_index] = 0
+      end
+      false
+    end
+
+    # @rbs (Hash[Symbol, untyped] frame, Frontend::AST::Rule template, Integer index) -> void
+    def prepare_parameter_alternative_entry(frame, template, index)
+      # @type self: Normalizer
+      alternative = substitute_parameter_alternative(
+        template.alternatives.fetch(index), frame.fetch(:bindings)
+      )
+      rule = Frontend::AST::Rule.new(
+        lhs: frame.fetch(:helper), parameters: [], alternatives: [alternative], loc: template.loc,
+        documentation: template.documentation, inline: false
+      )
+      frame[:alternative_index] = index + 1
+      frame[:current] = [template, rule, alternative]
+      frame[:item_index] = 0
+      frame[:rhs] = []
+      frame[:named_refs] = []
+      frame[:operations] = []
+      frame[:values] = []
+    end
+
+    # @rbs (Hash[Symbol, untyped] frame) -> void
+    def clear_parameter_alternative(frame)
+      frame[:current] = nil
+      frame[:item_index] = 0
+      frame[:rhs] = []
+      frame[:named_refs] = []
+      frame[:operations] = []
+      frame[:values] = []
+    end
+  end
+end

data/lib/ibex/normalize/recovery_declarations.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+module Ibex
+  # Recovery declaration extraction and post-interning symbol validation.
+  module NormalizeRecoveryDeclarations
+    private
+
+    # @rbs (Frontend::AST::Start | Frontend::AST::Recovery | Frontend::AST::OnErrorReduce |
+    #   Frontend::AST::GrammarTest declaration) -> void
+    def read_parser_control_declaration(declaration)
+      # @type self: Normalizer
+      case declaration
+      when Frontend::AST::Start then read_start_declaration(declaration)
+      when Frontend::AST::Recovery then read_recovery_declaration(declaration)
+      when Frontend::AST::OnErrorReduce then read_on_error_reduce_declaration(declaration)
+      when Frontend::AST::GrammarTest then read_grammar_test(declaration)
+      end
+    end
+
+    # @rbs (Frontend::AST::GrammarTest declaration) -> void
+    def read_grammar_test(declaration)
+      # @type self: Normalizer
+      fail_at(declaration.loc, "%test requires extended mode") unless @mode == :extended
+      unless %i[accept reject].include?(declaration.expectation)
+        fail_at(declaration.loc, "unknown %test expectation #{declaration.expectation.inspect}")
+      end
+      if @grammar_tests.any? do |test|
+           test[:expectation] == declaration.expectation && test[:source] == declaration.source
+         end
+        fail_at(declaration.loc, "duplicate %test #{declaration.expectation} source")
+      end
+
+      @grammar_tests << {
+        expectation: declaration.expectation,
+        source: declaration.source,
+        loc: declaration.loc.to_h
+      }
+    end
+
+    # @rbs (Frontend::AST::Recovery declaration) -> void
+    def read_recovery_declaration(declaration)
+      # @type self: Normalizer
+      fail_at(declaration.loc, "duplicate %recover declaration") if @recovery_sync_location
+      fail_at(declaration.loc, "%recover sync requires at least one token") if declaration.sync_tokens.empty?
+      unless declaration.sync_tokens.uniq.length == declaration.sync_tokens.length
+        fail_at(declaration.loc, "%recover sync tokens must be unique")
+      end
+
+      @recovery_sync_tokens = declaration.sync_tokens
+      @recovery_sync_location = declaration.loc
+    end
+
+    # @rbs (Frontend::AST::OnErrorReduce declaration) -> void
+    def read_on_error_reduce_declaration(declaration)
+      # @type self: Normalizer
+      fail_at(declaration.loc, "%on_error_reduce requires at least one nonterminal") if declaration.names.empty?
+      declaration.names.each do |name|
+        fail_at(declaration.loc, "duplicate %on_error_reduce symbol #{name}") if
+          @on_error_reduce_locations.key?(name)
+
+        @on_error_reduce_locations[name] = declaration.loc
+      end
+      @on_error_reduce_groups << declaration.names
+    end
+
+    # @rbs () -> void
+    def validate_recovery_declarations
+      # @type self: Normalizer
+      @recovery_sync_tokens.each do |name|
+        definition = @symbols_by_name[name]
+        unless definition&.terminal? && definition.name != "error"
+          fail_at(@recovery_sync_location || @ast.loc, "%recover sync references nonterminal or missing token #{name}")
+        end
+      end
+      @on_error_reduce_groups.flatten.each do |name|
+        definition = @symbols_by_name[name]
+        unless definition&.nonterminal?
+          location = @on_error_reduce_locations.fetch(name)
+          fail_at(location, "%on_error_reduce references terminal or missing nonterminal #{name}")
+        end
+      end
+    end
+  end
+end