ibex 0.1.0 → 0.2.0

data/lib/ibex/normalize/expression.rb

@@ -3,52 +3,73 @@
 module Ibex
   # Deterministically renders frontend EBNF items for production origin metadata.
   module NormalizeExpression
-    # @rbs!
-    #   private def render_reference: (Frontend::AST::SymbolReference item) -> String
-    #   private def self.render_reference: (Frontend::AST::SymbolReference item) -> String
-    #   private def render_group: (Frontend::AST::Group item) -> String
-    #   private def self.render_group: (Frontend::AST::Group item) -> String
-    #   private def render_separated_list: (Frontend::AST::SeparatedList item) -> String
-    #   private def self.render_separated_list: (Frontend::AST::SeparatedList item) -> String
+    class << self
+      # @rbs (Frontend::AST::item item) -> String
+      def render(item)
+        pending = [item] #: Array[untyped]
+        output = [] #: Array[String]
+        until pending.empty?
+          current = pending.pop
+          if current.is_a?(String)
+            output << current
+            next
+          end
 
-    # @rbs (Frontend::AST::item item) -> String
-    def render(item)
-      case item
-      when Frontend::AST::SymbolReference then render_reference(item)
-      when Frontend::AST::Group then render_group(item)
-      when Frontend::AST::Optional then "#{render(item.item)}?"
-      when Frontend::AST::Star then "#{render(item.item)}*"
-      when Frontend::AST::Plus then "#{render(item.item)}+"
-      when Frontend::AST::SeparatedList then render_separated_list(item)
-      else raise Ibex::Error, "#{item.loc}: cannot render unsupported EBNF expression"
+          push_render_item(pending, current)
+        end
+        output.join
       end
-    end
-    module_function :render
 
-    # @rbs skip
-    private
+      private
 
-    # @rbs skip
-    def render_reference(item)
-      suffix = item.named_reference ? ":#{item.named_reference}" : ""
-      "#{item.name}#{suffix}"
-    end
+      # @rbs (Array[untyped] pending, Frontend::AST::item item) -> void
+      def push_render_item(pending, item)
+        case item
+        when Frontend::AST::SymbolReference
+          pending << "#{item.name}#{":#{item.named_reference}" if item.named_reference}"
+        when Frontend::AST::ParameterizedReference
+          push_render_parameterized_reference(pending, item)
+        when Frontend::AST::Group
+          push_render_group(pending, item)
+        when Frontend::AST::Optional
+          pending.push("?", item.item)
+        when Frontend::AST::Star
+          pending.push("*", item.item)
+        when Frontend::AST::Plus
+          pending.push("+", item.item)
+        when Frontend::AST::SeparatedList
+          name = item.nonempty ? "separated_nonempty_list" : "separated_list"
+          pending.push(")", item.separator, ", ", item.item, "(", name)
+        else
+          raise Ibex::Error, "#{item.loc}: cannot render unsupported EBNF expression"
+        end
+      end
 
-    # @rbs skip
-    def render_group(item)
-      alternatives = item.alternatives.map { |items| items.map { |child| render(child) }.join(" ") }
-      "(#{alternatives.join(' | ')})"
-    end
+      # @rbs (Array[untyped] pending, Frontend::AST::ParameterizedReference item) -> void
+      def push_render_parameterized_reference(pending, item)
+        pending << ":#{item.named_reference}" if item.named_reference
+        pending << ")"
+        push_render_items(pending, item.arguments, ", ")
+        pending.push("(", item.name)
+      end
 
-    # @rbs skip
-    def render_separated_list(item)
-      name = item.nonempty ? "separated_nonempty_list" : "separated_list"
-      "#{name}(#{render(item.item)}, #{render(item.separator)})"
-    end
-    module_function :render_reference, :render_group, :render_separated_list
+      # @rbs (Array[untyped] pending, Frontend::AST::Group item) -> void
+      def push_render_group(pending, item)
+        pending << ")"
+        (item.alternatives.length - 1).downto(0) do |index|
+          pending << " | " if index < item.alternatives.length - 1
+          push_render_items(pending, item.alternatives.fetch(index), " ")
+        end
+        pending << "("
+      end
 
-    class << self
-      private :render_reference, :render_group, :render_separated_list
+      # @rbs (Array[untyped] pending, Array[Frontend::AST::item] items, String separator) -> void
+      def push_render_items(pending, items, separator)
+        (items.length - 1).downto(0) do |index|
+          pending << separator if index < items.length - 1
+          pending << items.fetch(index)
+        end
+      end
     end
   end
 end

data/lib/ibex/normalize/inline_expansion.rb

@@ -0,0 +1,414 @@
+# frozen_string_literal: true
+
+module Ibex
+  # Deterministically substitutes inline-rule productions before LR construction.
+  # rubocop:disable Metrics/ModuleLength -- expansion, action plans, and id remapping share one invariant.
+  module NormalizeInlineExpansion
+    private
+
+    # @rbs () -> void
+    def expand_inline_rules
+      # @type self: Normalizer
+      return if @inline_symbol_ids.empty?
+
+      definitions = @productions.group_by(&:lhs)
+      expanded = [] #: Array[IR::Production]
+      @productions.each do |production|
+        next if @inline_symbol_ids.include?(production.lhs)
+
+        variants = expand_inline_production(production, definitions)
+        variants.each do |variant|
+          if variant[:inline_used]
+            charge_inline_expansion!(production)
+            expanded << composed_inline_production(production, variant, expanded.length)
+          else
+            expanded << copy_inline_production(production, expanded.length)
+          end
+        end
+      end
+      rebuild_inline_symbols(expanded)
+    end
+
+    # Explicit heap frames avoid tying grammar depth to the Ruby call stack.
+    # The single-definition path reuses its accumulated arrays, so a long
+    # acyclic chain remains linear in the size of its final action plan.
+    # @rbs (IR::Production production, Hash[Integer, Array[IR::Production]] definitions) ->
+    #   Array[Hash[Symbol, untyped]]
+    def expand_inline_production(production, definitions)
+      # @type self: Normalizer
+      variants = [] #: Array[Hash[Symbol, untyped]]
+      worklist = [[inline_expansion_frame(production, nil)]] #: Array[Array[Hash[Symbol, untyped]]]
+      until worklist.empty?
+        frames = worklist.pop || raise(Ibex::Error, "internal inline expansion worklist underflow")
+        drain_inline_expansion_frames(frames, definitions, worklist, variants, production.origin[:loc])
+      end
+      variants
+    end
+
+    # @rbs (Array[Hash[Symbol, untyped]] frames, Hash[Integer, Array[IR::Production]] definitions,
+    #   Array[Array[Hash[Symbol, untyped]]] worklist, Array[Hash[Symbol, untyped]] variants,
+    #   IR::location? location) -> void
+    def drain_inline_expansion_frames(frames, definitions, worklist, variants, location)
+      loop do
+        frame = frames.fetch(-1)
+        current = frame.fetch(:production)
+        if frame.fetch(:index) < current.rhs.length
+          return if advance_inline_expansion_frame?(frames, definitions, worklist, variants, location)
+        elsif complete_inline_expansion_frame?(frames, variants, worklist, location)
+          return
+        end
+      end
+    end
+
+    # @rbs (Array[Hash[Symbol, untyped]] frames, Hash[Integer, Array[IR::Production]] definitions,
+    #   Array[Array[Hash[Symbol, untyped]]] worklist, Array[Hash[Symbol, untyped]] variants,
+    #   IR::location? location) -> bool
+    def advance_inline_expansion_frame?(frames, definitions, worklist, variants, location)
+      frame = frames.fetch(-1)
+      current = frame.fetch(:production)
+      index = frame.fetch(:index)
+      symbol_id = current.rhs.fetch(index)
+      frame[:index] = index + 1
+      unless @inline_symbol_ids.include?(symbol_id)
+        frame[:variant] = append_inline_symbol(frame.fetch(:variant), symbol_id)
+        return false
+      end
+
+      choices = definitions.fetch(symbol_id)
+      if choices.one?
+        frames << inline_expansion_frame(choices.fetch(0), symbol_id)
+        return false
+      end
+
+      schedule_inline_choice_branches(frames, choices, symbol_id, worklist)
+      enforce_inline_choice_limit!(variants.length + worklist.length, location)
+      true
+    end
+
+    # @rbs (Array[Hash[Symbol, untyped]] frames, Array[IR::Production] choices, Integer symbol_id,
+    #   Array[Array[Hash[Symbol, untyped]]] worklist) -> void
+    def schedule_inline_choice_branches(frames, choices, symbol_id, worklist)
+      choices.reverse_each do |choice|
+        branch = clone_inline_expansion_frames(frames)
+        branch << inline_expansion_frame(choice, symbol_id)
+        worklist << branch
+      end
+    end
+
+    # @rbs (Array[Hash[Symbol, untyped]] frames, Array[Hash[Symbol, untyped]] variants,
+    #   Array[Array[Hash[Symbol, untyped]]] worklist, IR::location? location) -> bool
+    def complete_inline_expansion_frame?(frames, variants, worklist, location)
+      frame = frames.fetch(-1)
+      current = frame.fetch(:production)
+      completed = finish_inline_precedence(frame.fetch(:variant), current)
+      frames.pop
+      if frames.empty?
+        variants << completed
+        enforce_inline_choice_limit!(variants.length + worklist.length, location)
+        return true
+      end
+
+      choice = finish_inline_choice(completed, current, frame.fetch(:symbol_id))
+      parent = frames.fetch(-1)
+      parent[:variant] = combine_inline_variant(parent.fetch(:variant), choice)
+      false
+    end
+
+    # @rbs (IR::Production production, Integer? symbol_id) -> Hash[Symbol, untyped]
+    def inline_expansion_frame(production, symbol_id)
+      { production: production, symbol_id: symbol_id, index: 0, variant: empty_inline_variant(production) }
+    end
+
+    # @rbs (Array[Hash[Symbol, untyped]] frames) -> Array[Hash[Symbol, untyped]]
+    def clone_inline_expansion_frames(frames)
+      frames.map do |frame|
+        variant = frame.fetch(:variant)
+        frame.merge(
+          variant: variant.merge(
+            rhs: variant.fetch(:rhs).dup,
+            steps: variant.fetch(:steps).dup,
+            logical_refs: variant.fetch(:logical_refs).dup
+          )
+        )
+      end
+    end
+
+    # @rbs (IR::Production production) -> Hash[Symbol, untyped]
+    def empty_inline_variant(production)
+      {
+        rhs: [],
+        steps: [],
+        logical_refs: [],
+        output_ref: nil,
+        inline_used: false,
+        inline_rule: nil,
+        precedence_override: nil,
+        precedence_contributes: false,
+        parameter: production.expansion&.dig(:parameter)
+      }
+    end
+
+    # @rbs (Hash[Symbol, untyped] variant, Integer symbol_id) -> Hash[Symbol, untyped]
+    def append_inline_symbol(variant, symbol_id)
+      # @type self: Normalizer
+      rhs = variant.fetch(:rhs)
+      terminal = @symbols.fetch(symbol_id).terminal?
+      variant.merge(
+        rhs: rhs + [symbol_id],
+        logical_refs: variant.fetch(:logical_refs) + [[:physical, rhs.length]],
+        precedence_override: terminal ? nil : variant[:precedence_override],
+        precedence_contributes: terminal || variant.fetch(:precedence_contributes)
+      )
+    end
+
+    # @rbs (Hash[Symbol, untyped] parent, Hash[Symbol, untyped] child) -> Hash[Symbol, untyped]
+    def combine_inline_variant(parent, child)
+      physical_offset = parent.fetch(:rhs).length
+      step_offset = parent.fetch(:steps).length
+      steps = combined_inline_steps(parent, child, physical_offset, step_offset)
+      parent.merge(
+        rhs: append_inline_array(parent.fetch(:rhs), child.fetch(:rhs)),
+        steps: append_inline_array(parent.fetch(:steps), steps),
+        logical_refs: parent.fetch(:logical_refs) + [
+          remap_inline_reference(child.fetch(:output_ref), physical_offset, step_offset)
+        ],
+        inline_used: true,
+        inline_rule: parent[:inline_rule] || child[:inline_rule],
+        precedence_override: combined_inline_precedence(parent, child),
+        precedence_contributes: parent.fetch(:precedence_contributes) ||
+          child.fetch(:precedence_contributes),
+        parameter: parent[:parameter] || child[:parameter]
+      )
+    end
+
+    # @rbs (Hash[Symbol, untyped] parent, Hash[Symbol, untyped] child,
+    #   Integer physical_offset, Integer step_offset) -> Array[Hash[Symbol, untyped]]
+    def combined_inline_steps(parent, child, physical_offset, step_offset)
+      return child.fetch(:steps) if physical_offset.zero? && step_offset.zero? && parent.fetch(:logical_refs).empty?
+
+      remap_inline_steps(parent, child, physical_offset, step_offset)
+    end
+
+    # @rbs (Array[untyped] left, Array[untyped] right) -> Array[untyped]
+    def append_inline_array(left, right)
+      left.empty? ? right : left + right
+    end
+
+    # @rbs (Hash[Symbol, untyped] parent, Hash[Symbol, untyped] child) -> Integer?
+    def combined_inline_precedence(parent, child)
+      return child[:precedence_override] if child.fetch(:precedence_contributes)
+
+      parent[:precedence_override]
+    end
+
+    # @rbs (Hash[Symbol, untyped] parent, Hash[Symbol, untyped] child,
+    #   Integer physical_offset, Integer step_offset) -> Array[Hash[Symbol, untyped]]
+    def remap_inline_steps(parent, child, physical_offset, step_offset)
+      child.fetch(:steps).map do |step|
+        inputs = step.fetch(:inputs).map do |reference|
+          remap_inline_reference(reference, physical_offset, step_offset)
+        end
+        stack_inputs = step.fetch(:stack_inputs).map do |reference|
+          remap_inline_reference(reference, physical_offset, step_offset)
+        end
+        step.merge(
+          inputs: inputs,
+          stack_inputs: parent.fetch(:logical_refs) + stack_inputs,
+          lookahead: step.fetch(:lookahead) + physical_offset
+        )
+      end
+    end
+
+    # @rbs (Array[Symbol | Integer] reference, Integer physical_offset, Integer step_offset) ->
+    #   Array[Symbol | Integer]
+    def remap_inline_reference(reference, physical_offset, step_offset)
+      kind, index = reference
+      raise Ibex::Error, "invalid inline slot kind" unless kind.is_a?(Symbol)
+      raise Ibex::Error, "invalid inline slot reference" unless index.is_a?(Integer)
+
+      [kind, index + (kind == :physical ? physical_offset : step_offset)]
+    end
+
+    # @rbs (Hash[Symbol, untyped] variant, IR::Production production) -> Hash[Symbol, untyped]
+    def finish_inline_precedence(variant, production)
+      explicit = production.precedence_override
+      variant.merge(
+        precedence_override: explicit || variant[:precedence_override],
+        precedence_contributes: !explicit.nil? || variant.fetch(:precedence_contributes)
+      )
+    end
+
+    # @rbs (Hash[Symbol, untyped] variant, IR::Production production, Integer symbol_id) ->
+    #   Hash[Symbol, untyped]
+    def finish_inline_choice(variant, production, symbol_id)
+      steps = variant.fetch(:steps)
+      rule = @inline_rule_by_symbol.fetch(symbol_id)
+      step = inline_action_step(production, variant.fetch(:logical_refs), variant.fetch(:rhs).length, :inline, rule)
+      steps << step
+      variant.merge(
+        steps: steps,
+        output_ref: [:step, steps.length - 1],
+        inline_used: true,
+        inline_rule: rule
+      )
+    end
+
+    # @rbs (IR::Production production, Array[Array[Symbol | Integer]] inputs, Integer lookahead,
+    #   Symbol kind, String? rule) -> Hash[Symbol, untyped]
+    def inline_action_step(production, inputs, lookahead, kind, rule)
+      action = production.action
+      {
+        kind: kind,
+        rule: rule,
+        code: action&.code,
+        loc: action&.location || production.origin.fetch(:loc),
+        named_refs: action&.named_refs || [],
+        context_length: action&.context_length || 0,
+        inputs: inputs,
+        stack_inputs: [],
+        lookahead: lookahead,
+        result_var: @options.fetch(:result_var),
+        result_type: @symbols.fetch(production.lhs).semantic_type
+      }
+    end
+
+    # @rbs (IR::Production production, Hash[Symbol, untyped] variant, Integer id) -> IR::Production
+    def composed_inline_production(production, variant, id)
+      steps = variant.fetch(:steps)
+      caller = inline_action_step(
+        production, variant.fetch(:logical_refs), variant.fetch(:rhs).length, :rule, nil
+      )
+      all_steps = steps + [caller]
+      action = composed_inline_action(production, variant.fetch(:rhs).length, all_steps)
+      IR::Production.new(
+        id: id, lhs: production.lhs, rhs: variant.fetch(:rhs), action: action,
+        precedence_override: variant[:precedence_override], origin: production.origin,
+        documentation: production.documentation,
+        expansion: inline_production_expansion(production, variant)
+      )
+    end
+
+    # @rbs (IR::Production production, Integer physical_length, Array[Hash[Symbol, untyped]]) -> IR::Action
+    def composed_inline_action(production, physical_length, steps)
+      action = production.action
+      fragments = steps.map do |step|
+        { kind: step.fetch(:kind), source: inline_source_provenance(step.fetch(:loc)) }
+      end
+      plan_steps = steps.map { |step| composition_plan_step(step, physical_length) }
+      composition = {
+        strategy: "sequence",
+        fragments: fragments,
+        plan: { version: 1, physical: physical_length, steps: plan_steps }
+      } #: IR::action_composition
+      IR::Action.new(
+        code: action&.code || inline_implicit_code(production.rhs.length),
+        location: action&.location || production.origin.fetch(:loc),
+        composition: composition
+      )
+    end
+
+    # @rbs (Hash[Symbol, untyped] step, Integer physical_length) -> Hash[Symbol, untyped]
+    def composition_plan_step(step, physical_length)
+      {
+        kind: step.fetch(:kind),
+        rule: step[:rule],
+        code: step[:code],
+        loc: step.fetch(:loc),
+        named_refs: step.fetch(:named_refs),
+        context_length: step.fetch(:context_length),
+        inputs: step.fetch(:inputs).map { |reference| resolve_inline_reference(reference, physical_length) },
+        stack_inputs: step.fetch(:stack_inputs).map do |reference|
+          resolve_inline_reference(reference, physical_length)
+        end,
+        lookahead: step.fetch(:lookahead) < physical_length ? step.fetch(:lookahead) : nil,
+        result_var: step.fetch(:result_var),
+        result_type: step[:result_type]
+      }
+    end
+
+    # @rbs (Array[Symbol | Integer] reference, Integer physical_length) -> Integer
+    def resolve_inline_reference(reference, physical_length)
+      kind, index = reference
+      raise Ibex::Error, "invalid inline slot reference" unless index.is_a?(Integer)
+
+      index + (kind == :step ? physical_length : 0)
+    end
+
+    # @rbs (Integer rhs_length) -> String
+    def inline_implicit_code(rhs_length)
+      expression = rhs_length.zero? ? "nil" : "val[0]"
+      @options.fetch(:result_var) ? " result = #{expression} " : " #{expression} "
+    end
+
+    # @rbs (IR::location location) -> IR::source_provenance
+    def inline_source_provenance(location)
+      { file: location[:file], root: @resolution&.root_directory, byte_span: nil }
+    end
+
+    # @rbs (IR::Production production, Hash[Symbol, untyped] variant) -> IR::production_expansion
+    def inline_production_expansion(production, variant)
+      existing = production.expansion
+      {
+        parameter: existing&.dig(:parameter) || variant[:parameter],
+        inline: { rule: variant.fetch(:inline_rule) },
+        include_chain: existing&.dig(:include_chain) || []
+      }
+    end
+
+    # @rbs (IR::Production production, Integer id) -> IR::Production
+    def copy_inline_production(production, id)
+      IR::Production.new(
+        id: id, lhs: production.lhs, rhs: production.rhs, action: production.action,
+        precedence_override: production.precedence_override, origin: production.origin,
+        documentation: production.documentation, expansion: production.expansion
+      )
+    end
+
+    # @rbs (IR::Production production) -> void
+    def charge_inline_expansion!(production)
+      @inline_expansion_count += 1
+      enforce_inline_choice_limit!(@inline_expansion_count, production.origin[:loc])
+    end
+
+    # @rbs (Integer count, IR::location? location) -> void
+    def enforce_inline_choice_limit!(count, location)
+      # @type self: Normalizer
+      return if count <= @max_inline_expansions
+
+      fail_hash(location, "inline expansion limit of #{@max_inline_expansions} exceeded")
+    end
+
+    # @rbs (Array[IR::Production] productions) -> void
+    def rebuild_inline_symbols(productions)
+      kept = @symbols.reject { |definition| @inline_symbol_ids.include?(definition.id) }
+      id_map = kept.each_with_index.to_h { |definition, id| [definition.id, id] }
+      @symbols = kept.each_with_index.map { |definition, id| remap_inline_symbol(definition, id) }
+      @symbols_by_name = @symbols.to_h { |definition| [definition.name, definition] }
+      @productions = productions.each_with_index.map do |production, id|
+        remap_inline_production(production, id, id_map)
+      end
+    end
+
+    # @rbs (IR::GrammarSymbol definition, Integer id) -> IR::GrammarSymbol
+    def remap_inline_symbol(definition, id)
+      IR::GrammarSymbol.new(
+        id: id, name: definition.name, kind: definition.kind, reserved: definition.reserved,
+        precedence: definition.precedence, location: definition.location,
+        display_name: definition.display_name, semantic_type: definition.semantic_type,
+        documentation: definition.documentation
+      )
+    end
+
+    # @rbs (IR::Production production, Integer id, Hash[Integer, Integer] id_map) -> IR::Production
+    def remap_inline_production(production, id, id_map)
+      IR::Production.new(
+        id: id, lhs: id_map.fetch(production.lhs),
+        rhs: production.rhs.map { |symbol_id| id_map.fetch(symbol_id) },
+        action: production.action,
+        precedence_override: production.precedence_override && id_map.fetch(production.precedence_override),
+        origin: production.origin, documentation: production.documentation, expansion: production.expansion
+      )
+    end
+  end
+  # rubocop:enable Metrics/ModuleLength
+end