ibex 0.2.0 → 0.4.0

data/lib/ibex/lalr/ielr/annotator.rb

@@ -0,0 +1,214 @@
+# frozen_string_literal: true
+# rbs_inline: enabled
+
+# steep:ignore:start
+
+require "set"
+require_relative "bits"
+require_relative "inadequacy"
+require_relative "item_lookaheads"
+require_relative "split_stability"
+
+# The methods follow the paper's matrix definitions directly.
+# rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+
+module Ibex
+  module LALR
+    module IELR
+      # Finds grammar-relative inadequacies and traces their possible
+      # manifestations through predecessor states.
+      class Annotator
+        attr_reader :annotation_lists #: Array[Array[Annotation]]
+        attr_reader :inadequacies #: Array[Array[Inadequacy]]
+        attr_reader :item_lookaheads #: ItemLookaheads
+        attr_reader :split_stable_discarded #: Integer
+
+        # @rbs (IR::Grammar grammar, Array[core_set] states, transitions transitions,
+        #   Array[packed_items] items, GotoFollows goto_follows, ?resolver: ConflictResolver?) -> void
+        def initialize(grammar, states, transitions, items, goto_follows, resolver: nil)
+          @grammar = grammar
+          @states = states
+          @transitions = transitions
+          @items = items
+          @goto_follows = goto_follows
+          @resolver = resolver || ConflictResolver.new(grammar)
+          @kernel_cores = states.map { |state| kernel_items(state) }
+          @item_lookaheads = ItemLookaheads.new(
+            grammar, goto_follows, @kernel_cores, goto_follows.predecessors,
+            start_states: (0...grammar.starts.length).to_a
+          )
+          @annotation_lists = Array.new(states.length) { [] }
+          @keys = Array.new(states.length) { Set.new }
+          @inadequacies = Array.new(states.length) { [] }
+          @next_id = 0
+          @split_stable_discarded = 0
+        end
+
+        # @rbs () -> Array[Array[Annotation]]
+        def build
+          @states.each_index do |state_id|
+            @inadequacies[state_id] = build_inadequacies(state_id)
+            @inadequacies.fetch(state_id).each do |inadequacy|
+              register?(state_id, annotate_manifestation(state_id, inadequacy))
+            end
+          end
+          worklist = @annotation_lists.each_with_index.flat_map do |annotations, state_id|
+            annotations.map { |annotation| [state_id, annotation] }
+          end
+          cursor = 0
+          while cursor < worklist.length
+            state_id, annotation = worklist.fetch(cursor)
+            cursor += 1
+            @goto_follows.predecessors.fetch(state_id).each do |previous|
+              derived = annotate_predecessor(previous, state_id, annotation)
+              worklist << [previous, derived] if register?(previous, derived)
+            end
+          end
+          @annotation_lists
+        end
+
+        private
+
+        # @rbs (Integer state_id) -> Array[Inadequacy]
+        def build_inadequacies(state_id)
+          @grammar.terminals.filter_map do |terminal|
+            contributions = contributions_for(state_id, terminal.id)
+            next if contributions.length < 2
+
+            inadequacy = Inadequacy.new(state: state_id, token: terminal.id, contributions: contributions, id: @next_id)
+            @next_id += 1
+            inadequacy
+          end
+        end
+
+        # @rbs (Integer state_id, Integer token_id) -> Array[[Symbol, Integer?]]
+        def contributions_for(state_id, token_id)
+          reductions = @items.fetch(state_id).filter_map do |(production_id, dot), lookaheads|
+            next unless dot == rhs_for(production_id).length && lookaheads.include?(token_id)
+
+            production_id
+          end.sort
+          contributions = reductions.select(&:negative?).map { |production_id| [:accept, production_id] }
+          symbol = @grammar.symbol_by_id(token_id)
+          contributions << [:shift, nil] if symbol&.terminal? && @transitions.fetch(state_id).key?(token_id)
+          contributions.concat(reductions.reject(&:negative?).map { |production_id| [:reduce, production_id] })
+          contributions
+        end
+
+        # @rbs (Integer state_id, Inadequacy inadequacy) -> Annotation
+        def annotate_manifestation(state_id, inadequacy)
+          matrix = inadequacy.contributions.map do |kind, production_id|
+            next nil if kind == :shift
+
+            rhs = rhs_for(production_id)
+            if rhs.empty?
+              compute_lhs_contributions(state_id, lhs_for(production_id), inadequacy.token)
+            else
+              completed_kernel_mask(state_id, production_id, rhs.length)
+            end
+          end
+          Annotation.new(inadequacy: inadequacy, matrix: matrix)
+        end
+
+        # @rbs (Integer state_id, Integer successor_id, Annotation annotation) -> Annotation
+        def annotate_predecessor(state_id, successor_id, annotation)
+          token = annotation.inadequacy.token
+          cores = @kernel_cores.fetch(successor_id)
+          matrix = annotation.matrix.map do |row|
+            next nil if row.nil?
+            next nil if always_via_lhs?(state_id, cores, row, token)
+
+            project_row(state_id, cores, row, token)
+          end
+          Annotation.new(inadequacy: annotation.inadequacy, matrix: matrix)
+        end
+
+        # @rbs (Integer state_id, Array[item_core], Integer, Integer) -> bool
+        def always_via_lhs?(state_id, cores, row, token)
+          Bits.each_set_bit(row).any? do |index|
+            production_id, dot = cores.fetch(index)
+            next false unless dot == 1
+
+            compute_lhs_contributions(state_id, lhs_for(production_id), token).nil?
+          end
+        end
+
+        # @rbs (Integer state_id, Array[item_core], Integer, Integer) -> Integer
+        def project_row(state_id, cores, row, token)
+          Bits.each_set_bit(row).reduce(0) do |mask, index|
+            production_id, dot = cores.fetch(index)
+            if dot > 1
+              previous = kernel_index_of(state_id, production_id, dot - 1)
+              next mask unless previous && item_lookaheads.fetch(state_id, previous).anybits?(1 << token)
+
+              mask | (1 << previous)
+            else
+              lhs_mask = compute_lhs_contributions(state_id, lhs_for(production_id), token)
+              mask | lhs_mask.to_i
+            end
+          end
+        end
+
+        # @rbs (Integer state_id, Integer production_id, Integer length) -> Integer
+        def completed_kernel_mask(state_id, production_id, length)
+          index = kernel_index_of(state_id, production_id, length)
+          index ? (1 << index) : 0
+        end
+
+        # @rbs (Integer state_id, Integer lhs_id, Integer token_id) -> Integer?
+        def compute_lhs_contributions(state_id, lhs_id, token_id)
+          goto_id = @goto_follows.goto_for(state_id, lhs_id)
+          raise Ibex::Error, "missing goto for #{lhs_id} from #{state_id}" unless goto_id
+          return nil if @goto_follows.always_follows.fetch(goto_id).anybits?(1 << token_id)
+
+          mask = 0
+          Bits.each_set_bit(@goto_follows.follow_kernel_items.fetch(goto_id)) do |index|
+            mask |= 1 << index if @item_lookaheads.fetch(state_id, index).anybits?(1 << token_id)
+          end
+          mask
+        end
+
+        # @rbs (Integer state_id, Annotation annotation) -> bool
+        def register?(state_id, annotation)
+          if SplitStability.split_stable?(annotation, @resolver)
+            @split_stable_discarded += 1
+            return false
+          end
+
+          key = annotation.key
+          return false unless @keys.fetch(state_id).add?(key)
+
+          @annotation_lists.fetch(state_id) << annotation
+          true
+        end
+
+        # @rbs (Integer state_id, Integer production_id) -> Integer
+        def lhs_for(production_id)
+          return @grammar.symbol(@grammar.starts.fetch(-production_id - 1)).id if production_id.negative?
+
+          @grammar.productions.fetch(production_id).lhs
+        end
+
+        # @rbs (Integer production_id) -> Array[Integer]
+        def rhs_for(production_id)
+          return [lhs_for(production_id)] if production_id.negative?
+
+          @grammar.productions.fetch(production_id).rhs
+        end
+
+        # @rbs (core_set items) -> Array[item_core]
+        def kernel_items(items)
+          items.select { |production_id, dot| production_id.negative? || dot.positive? }.to_a.sort
+        end
+
+        # @rbs (Integer state_id, Integer production_id, Integer dot) -> Integer?
+        def kernel_index_of(state_id, production_id, dot)
+          @kernel_cores.fetch(state_id).index([production_id, dot])
+        end
+      end
+    end
+  end
+end
+# steep:ignore:end
+
+# rubocop:enable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity

data/lib/ibex/lalr/ielr/bits.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+# rbs_inline: enabled
+
+# steep:ignore:start
+
+module Ibex
+  module LALR
+    module IELR
+      module Bits
+        module_function
+
+        # @rbs (Integer bits) { (Integer) -> void } -> void
+        # @rbs (Integer bits) -> Enumerator[Integer, void]
+        def each_set_bit(bits)
+          return enum_for(:each_set_bit, bits) unless block_given?
+
+          index = 0
+          while bits.positive?
+            yield index if bits.odd?
+            bits >>= 1
+            index += 1
+          end
+        end
+      end
+    end
+  end
+end
+# steep:ignore:end

data/lib/ibex/lalr/ielr/inadequacy.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+# rbs_inline: enabled
+
+# steep:ignore:start
+
+module Ibex
+  module LALR
+    module IELR
+      # @rbs skip
+      Inadequacy = Struct.new(:state, :token, :contributions, :id, keyword_init: true) do
+        def initialize(state:, token:, contributions:, id:)
+          super(state: state, token: token, contributions: contributions.freeze, id: id)
+          freeze
+        end
+      end
+
+      # @rbs skip
+      Annotation = Struct.new(:inadequacy, :matrix, keyword_init: true) do
+        def initialize(inadequacy:, matrix:)
+          super(inadequacy: inadequacy, matrix: matrix.freeze)
+          freeze
+        end
+
+        def key
+          [inadequacy.id, matrix]
+        end
+      end
+
+      # @rbs!
+      #   type contribution = [Symbol, Integer?]
+      #   class Inadequacy < Struct[Integer | Array[contribution]]
+      #     attr_accessor state: Integer
+      #     attr_accessor token: Integer
+      #     attr_accessor contributions: Array[contribution]
+      #     attr_accessor id: Integer
+      #     def self.new: (state: Integer, token: Integer, contributions: Array[contribution], id: Integer) -> instance
+      #   end
+      #   class Annotation < Struct[Inadequacy | Array[Integer?]]
+      #     attr_accessor inadequacy: Inadequacy
+      #     attr_accessor matrix: Array[Integer?]
+      #     def self.new: (inadequacy: Inadequacy, matrix: Array[Integer?]) -> instance
+      #     def key: () -> [Integer, Array[Integer?]]
+      #   end
+    end
+  end
+end
+# steep:ignore:end

data/lib/ibex/lalr/ielr/item_lookaheads.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+# rbs_inline: enabled
+
+# steep:ignore:start
+
+require_relative "bits"
+
+module Ibex
+  module LALR
+    module IELR
+      # Lazily derives the lookahead of a kernel item from predecessor states
+      # and goto-follow sets.  Memoisation is important: predecessor paths can
+      # share exponentially many suffixes in a real grammar.
+      class ItemLookaheads
+        # @rbs @memo: Hash[Integer, Integer]
+
+        # @rbs (IR::Grammar grammar, GotoFollows goto_follows, Array[Array[item_core]] kernel_cores,
+        #   Array[Array[Integer]] predecessors, ?start_states: Array[Integer]?) -> void
+        def initialize(grammar, goto_follows, kernel_cores, predecessors, start_states: nil)
+          @grammar = grammar
+          @goto_follows = goto_follows
+          @kernel_cores = kernel_cores
+          @predecessors = predecessors
+          @start_states = start_states || []
+          @index = {}
+          kernel_cores.each_with_index do |cores, state_id|
+            cores.each_with_index { |core, kernel_index| @index[[state_id, core]] = kernel_index }
+          end
+          @memo = {}
+        end
+
+        # @rbs (Integer state_id, Integer kernel_index) -> Integer
+        def fetch(state_id, kernel_index)
+          key = [state_id, kernel_index]
+          return @memo.fetch(key) if @memo.key?(key)
+
+          @memo[key] = compute(state_id, kernel_index)
+        end
+
+        # @rbs () -> Integer
+        def size
+          @memo.length
+        end
+
+        private
+
+        # @rbs (Integer state_id, Integer kernel_index) -> Integer
+        def compute(state_id, kernel_index)
+          production_id, dot = @kernel_cores.fetch(state_id).fetch(kernel_index)
+          return start_seed(production_id) if production_id.negative? && dot.zero?
+          return start_seed(production_id) if production_id.negative? && dot.positive?
+          return 0 if dot.zero?
+          return goto_follows_for(state_id, production_id) if dot == 1
+
+          @predecessors.fetch(state_id).reduce(0) do |bits, previous|
+            previous_index = @index[[previous, [production_id, dot - 1]]]
+            previous_index ? bits | fetch(previous, previous_index) : bits
+          end
+        end
+
+        # @rbs (Integer state_id, Integer production_id) -> Integer
+        def goto_follows_for(state_id, production_id)
+          lhs = @grammar.productions.fetch(production_id).lhs
+          @predecessors.fetch(state_id).reduce(0) do |bits, previous|
+            goto_id = @goto_follows.goto_for(previous, lhs)
+            bits | (goto_id ? @goto_follows.goto_follows.fetch(goto_id) : 0)
+          end
+        end
+
+        # @rbs (Integer production_id) -> Integer
+        def start_seed(production_id)
+          production_id.negative? ? 1 : 0
+        end
+      end
+    end
+  end
+end
+# steep:ignore:end

data/lib/ibex/lalr/ielr/pipeline.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+# rbs_inline: enabled
+
+# steep:ignore:start
+
+require_relative "../lr0_collection"
+require_relative "../goto_follows"
+require_relative "../lookahead_propagation"
+require_relative "state_splitter"
+
+# The method follows the fixed six-phase pipeline as one transaction.
+# rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+
+module Ibex
+  module LALR
+    module IELR
+      # Coordinates the canonical-free direct IELR phases.  The returned
+      # profile deliberately reports canonical metrics as unavailable.
+      class Pipeline
+        # @rbs (IR::Grammar grammar, Analysis::Sets sets, ?starts: Array[String]?, ?profile: bool) -> void
+        def initialize(grammar, sets, starts: nil, profile: false)
+          @grammar = grammar
+          @sets = sets
+          @starts = starts || grammar.starts
+          @profile = profile
+        end
+
+        # @rbs () -> [Array[packed_items], transitions, LALR::build_collection]
+        def build
+          states, transitions = LR0Collection.new(@grammar, starts: @starts).build
+          seeds = @starts.each_with_index.map do |name, index|
+            augmented = -1 - @grammar.starts.index(name)
+            [index, [augmented, 0], 0]
+          end
+          initial_propagation = LookaheadPropagation.new(
+            @grammar, @sets, states, transitions, seeds: seeds
+          )
+          items = initial_propagation.build
+          goto_follows = GotoFollows.new(
+            @grammar, @sets, states, transitions, @starts.each_index.to_a, start_names: @starts
+          )
+          splitter = StateSplitter.new(@grammar, states, transitions, items, goto_follows)
+          _split_items, split_transitions = splitter.build
+          split_states = splitter.states.map { |state| states.fetch(state.lalr_isocore) }
+          phase4_items = LookaheadPropagation.new(
+            @grammar, @sets, split_states, split_transitions, seeds: seeds
+          ).build
+          profile = {
+            construction_states: phase4_items.length,
+            canonical_states: nil,
+            strategy: :ielr_direct,
+            lr0_states: @profile ? states.length : nil,
+            lr0_items: @profile ? states.sum(&:length) : nil,
+            canonical_items: nil,
+            propagation_edges: @profile ? initial_propagation.propagation_edge_count : nil,
+            ielr_initial_partitions: nil,
+            ielr_final_partitions: nil,
+            ielr_annotations: splitter.annotations.sum(&:length),
+            ielr_annotated_states: splitter.annotations.count { |entries| !entries.empty? },
+            ielr_inadequacies: splitter.inadequacies.sum(&:length),
+            ielr_split_stable_discarded: splitter.split_stable_discarded,
+            ielr_lalr_states: states.length,
+            ielr_split_states: splitter.split_states,
+            ielr_unreachable_removed: 0,
+            ielr_remergeable_candidates: nil
+          }
+          [phase4_items, split_transitions, profile]
+        end
+      end
+    end
+  end
+end
+# steep:ignore:end
+
+# rubocop:enable Metrics/AbcSize, Metrics/MethodLength

data/lib/ibex/lalr/ielr/split_stability.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+# rbs_inline: enabled
+
+# steep:ignore:start
+
+require_relative "bits"
+
+module Ibex
+  module LALR
+    module IELR
+      # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+      module SplitStability
+        MAX_ENUMERATED_POTENTIALS = 12
+
+        module_function
+
+        # @rbs (Annotation) -> bool
+        def simple?(annotation)
+          annotation.matrix.all? { |row| row.nil? || row.zero? }
+        end
+
+        # @rbs (Annotation, ConflictResolver) -> bool
+        def split_stable?(annotation, resolver)
+          return true if simple?(annotation)
+
+          contributions = annotation.inadequacy.contributions
+          kept = contributions.each_index.reject { |index| annotation.matrix[index]&.zero? }
+          return true if kept.empty?
+
+          potential = kept.reject { |index| annotation.matrix[index].nil? }
+          return false if potential.length > MAX_ENUMERATED_POTENTIALS
+
+          base = resolve(resolver, annotation.inadequacy.token, kept.map { |index| contributions[index] })
+          subsets(potential).all? do |removed|
+            remaining = kept.reject { |index| removed.include?(index) }
+            remaining.empty? || resolve(resolver, annotation.inadequacy.token,
+                                        remaining.map { |index| contributions[index] }) == base
+          end
+        end
+
+        # @rbs (ConflictResolver, Integer, Array[Object]) -> Object?
+        def resolve(resolver, token, contributions)
+          actions = contributions.map do |kind, production_id|
+            case kind
+            when :shift then { type: :shift, state: 0 }
+            when :reduce then { type: :reduce, production: production_id }
+            when :accept then { type: :accept }
+            when :error then { type: :error }
+            else raise Ibex::Error, "unknown IELR contribution #{kind.inspect}"
+            end
+          end
+          action, = resolver.resolve(token, actions)
+          normalize(action)
+        end
+
+        # @rbs (Object?) -> Object?
+        def normalize(action)
+          return nil unless action
+          return [:shift] if action[:type] == :shift
+          return [:reduce, action[:production]] if action[:type] == :reduce
+          return [:accept] if action[:type] == :accept
+          return [:error] if action[:type] == :error
+
+          action
+        end
+
+        # @rbs (Array[Integer]) -> Array[Array[Integer]]
+        def subsets(values)
+          (0...(1 << values.length)).map do |bits|
+            values.each_index.filter_map { |index| values.fetch(index) if bits.anybits?(1 << index) }
+          end
+        end
+      end
+      # rubocop:enable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+    end
+  end
+end
+# steep:ignore:end

data/lib/ibex/lalr/ielr/split_state.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+# rbs_inline: enabled
+
+module Ibex
+  module LALR
+    module IELR
+      # @rbs skip
+      SplitState = Struct.new(:core, :transitions, :lalr_isocore, keyword_init: true)
+
+      # @rbs!
+      #   class SplitState < Struct[Array[item_core] | Array[[Integer, Integer]] | Integer]
+      #     attr_accessor core: Array[item_core]
+      #     attr_accessor transitions: Array[[Integer, Integer]]
+      #     attr_accessor lalr_isocore: Integer
+      #     def self.new: (core: Array[item_core], transitions: Array[[Integer, Integer]],
+      #       lalr_isocore: Integer) -> instance
+      #   end
+    end
+  end
+end