ibex 0.2.0 → 0.4.0

data/lib/ibex/lalr/direct_lookaheads.rb

@@ -3,6 +3,10 @@
 
 require "set"
 
+# This class intentionally owns collection and propagation state so the
+# single-start compatibility path remains byte-identical.
+# rubocop:disable Metrics/AbcSize, Metrics/ClassLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+
 module Ibex
   module LALR
     # Builds an LR(0) collection and propagates LALR(1) lookaheads directly
@@ -10,53 +14,88 @@ module Ibex
     class DirectLookaheads
       AUGMENTED_PRODUCTION = -1 #: Integer
       EMPTY_PRODUCTIONS = Array.new(0).freeze #: Array[IR::Production]
-      EMPTY_NODES = Array.new(0).freeze #: Array[lookahead_node]
+      EMPTY_NODES = Array.new(0).freeze #: Array[Integer]
       private_constant :EMPTY_PRODUCTIONS, :EMPTY_NODES
 
-      # @rbs @grammar: IR::Grammar
-      # @rbs @sets: Analysis::Sets
-      # @rbs @productions_by_lhs: Hash[Integer, Array[IR::Production]]
-      # @rbs @augmented_rhs: Array[Integer]
-      # @rbs @production_rhs: Array[Array[Integer]]
-      # @rbs @augmented_item_cores: Array[item_core]
-      # @rbs @production_item_cores: Array[Array[item_core]]
-      # @rbs @terminal_ids: Array[Integer]
-      # @rbs @terminal_masks: Array[Integer]
-
-      # @rbs (IR::Grammar grammar, Analysis::Sets sets) -> void
-      def initialize(grammar, sets)
+      attr_reader :lr0_state_count #: Integer?
+      attr_reader :lr0_item_count #: Integer?
+      attr_reader :propagation_edge_count #: Integer?
+      attr_reader :states #: Array[core_set]
+      attr_reader :transitions #: transitions
+
+      # @rbs (IR::Grammar grammar, Analysis::Sets sets, ?starts: Array[String]?, ?profile: bool) -> void
+      def initialize(grammar, sets, starts: nil, profile: false)
         @grammar = grammar
         @sets = sets
+        @grammar_starts = grammar_starts.freeze
+        @starts = (starts || @grammar_starts).dup
+        raise ArgumentError, "starts must be a nonempty subset of grammar starts" if
+          @starts.empty? || (@starts - @grammar_starts).any?
+
         @productions_by_lhs = grammar.productions.group_by(&:lhs)
-        start = grammar.symbol(grammar.start) || raise(Ibex::Error, "missing start symbol")
-        @augmented_rhs = [start.id].freeze
+        @augmented_production_ids = @starts.map { |name| AUGMENTED_PRODUCTION - start_index(name) }
+        @augmented_rhs = @starts.each_with_index.to_h do |name, index|
+          symbol = grammar.symbol(name) || raise(Ibex::Error, "missing start symbol #{name}")
+          [@augmented_production_ids.fetch(index), [symbol.id].freeze]
+        end.freeze
         @production_rhs = grammar.productions.map(&:rhs).freeze
-        @augmented_item_cores = item_cores_for(AUGMENTED_PRODUCTION, @augmented_rhs.length)
+        @augmented_item_cores = @augmented_rhs.to_h do |production_id, rhs|
+          [production_id, item_cores_for(production_id, rhs.length)]
+        end.freeze
         @production_item_cores = grammar.productions.map do |production|
           item_cores_for(production.id, production.rhs.length)
         end.freeze
+        initialize_item_encoding
         @terminal_ids = grammar.terminals.map(&:id).freeze
         @terminal_masks = @terminal_ids.map { |id| 1 << id }.freeze
+        @terminal_ids_by_bits = {} #: Hash[Integer, Array[Integer]]
+        @lr0_state_count = nil
+        @lr0_item_count = nil
+        @propagation_edge_count = nil
+        @profile = profile
       end
 
       # @rbs () -> [Array[packed_items], transitions]
       def build
         states, transitions = lr0_collection
+        @states = states
+        @transitions = transitions
         lookaheads = empty_lookaheads(states)
         propagation = propagation_graph(states, transitions, lookaheads)
-        lookaheads.fetch(0).fetch(item_core(AUGMENTED_PRODUCTION, 0)) << 0
+        if @profile
+          @lr0_state_count = states.length
+          @lr0_item_count = states.sum(&:length)
+          @propagation_edge_count = propagation.values.sum(&:length)
+        end
+        @starts.each_with_index do |_name, index|
+          lookaheads.fetch(index).fetch(item_core(augmented_production(index), 0)) << 0
+        end
         propagate(lookaheads, propagation)
         [lookaheads, transitions]
       end
 
+      # @rbs (Integer index) -> Integer
+      def augmented_production(index)
+        @augmented_production_ids.fetch(index)
+      end
+
       private
 
+      # @rbs () -> void
+      def initialize_item_encoding
+        @item_key_stride = [*@augmented_rhs.values, *@production_rhs].map(&:length).max.to_i + 1
+        # Negative augmented ids are global to Grammar#starts.  A builder
+        # may isolate a non-first entry, so reserve the full global range.
+        @production_offset = @grammar_starts.length
+        @node_stride = (@production_offset + @production_rhs.length) * @item_key_stride
+      end
+
       # @rbs () -> [Array[core_set], transitions]
       def lr0_collection
-        seed = Set[item_core(AUGMENTED_PRODUCTION, 0)] #: core_set
-        states = [closure(seed)]
+        states = @starts.each_index.map { |index| closure(Set[item_core(augmented_production(index), 0)]) }
         transitions = [] #: transitions
-        indexes = { item_key(states.first) => 0 }
+        indexes = {} #: Hash[Array[Integer], Integer]
+        states.each_with_index { |items, index| indexes[item_key(items)] = index }
         cursor = 0
         while cursor < states.length
           transitions[cursor] = {}
@@ -108,15 +147,13 @@ module Ibex
 
       # @rbs (Array[core_set] states) -> Array[packed_items]
       def empty_lookaheads(states)
-        states.map do |items|
-          items.to_h { |item| [item, Set.new] } #: packed_items
-        end
+        states.map { |items| items.to_h { |item| [item, Set.new] } }
       end
 
       # @rbs (Array[core_set] states, transitions transitions, Array[packed_items] lookaheads) ->
-      #   Hash[lookahead_node, Array[lookahead_node]]
+      #   Hash[Integer, Array[Integer]]
       def propagation_graph(states, transitions, lookaheads)
-        edges = Hash.new { |hash, key| hash[key] = [] } #: Hash[lookahead_node, Array[lookahead_node]]
+        edges = Hash.new { |hash, key| hash[key] = [] } #: Hash[Integer, Array[Integer]]
         states.each_with_index do |items, state_id|
           items.each do |production_id, dot|
             add_transition_edge(edges, transitions, state_id, production_id, dot)
@@ -127,19 +164,17 @@ module Ibex
         edges
       end
 
-      # @rbs (Hash[lookahead_node, Array[lookahead_node]] edges, transitions transitions,
+      # @rbs (Hash[Integer, Array[Integer]] edges, transitions transitions,
       #   Integer state_id, Integer production_id, Integer dot) -> void
       def add_transition_edge(edges, transitions, state_id, production_id, dot)
         symbol_id = rhs_for(production_id)[dot]
         return unless symbol_id
 
         target_state = transitions.fetch(state_id).fetch(symbol_id)
-        source = [state_id, production_id, dot] #: lookahead_node
-        target = [target_state, production_id, dot + 1] #: lookahead_node
-        edges[source] << target
+        edges[node_id(state_id, production_id, dot)] << node_id(target_state, production_id, dot + 1)
       end
 
-      # @rbs (Hash[lookahead_node, Array[lookahead_node]] edges, Array[packed_items] lookaheads,
+      # @rbs (Hash[Integer, Array[Integer]] edges, Array[packed_items] lookaheads,
       #   Integer state_id, Integer production_id, Integer dot) -> void
       def add_closure_relations(edges, lookaheads, state_id, production_id, dot)
         rhs = rhs_for(production_id)
@@ -148,27 +183,22 @@ module Ibex
 
         suffix = rhs.drop(dot + 1)
         spontaneous = terminal_ids(@sets.first_of_sequence(suffix))
-        source = [state_id, production_id, dot] #: lookahead_node
+        source = node_id(state_id, production_id, dot)
         @productions_by_lhs.fetch(symbol.id, EMPTY_PRODUCTIONS).each do |production|
           target_item = item_core(production.id, 0)
           lookaheads.fetch(state_id).fetch(target_item).merge(spontaneous)
-          if @sets.sequence_nullable?(suffix)
-            target = [state_id, production.id, 0] #: lookahead_node
-            edges[source] << target
-          end
+          edges[source] << node_id(state_id, production.id, 0) if @sets.sequence_nullable?(suffix)
         end
       end
 
-      # @rbs (Array[packed_items] lookaheads, Hash[lookahead_node, Array[lookahead_node]] edges) -> void
+      # @rbs (Array[packed_items] lookaheads, Hash[Integer, Array[Integer]] edges) -> void
       def propagate(lookaheads, edges)
         queue = lookaheads.each_with_index.flat_map do |items, state_id|
           items.filter_map do |(production_id, dot), tokens|
-            next if tokens.empty?
-
-            [state_id, production_id, dot] #: lookahead_node
+            node_id(state_id, production_id, dot) unless tokens.empty?
           end
-        end #: Array[lookahead_node]
-        queued = queue.to_h { |node| [node, true] } #: Hash[lookahead_node, bool]
+        end
+        queued = queue.to_h { |node| [node, true] }
         cursor = 0
         while cursor < queue.length
           source = queue.fetch(cursor)
@@ -187,50 +217,73 @@ module Ibex
         end
       end
 
-      # @rbs (Array[packed_items] lookaheads, lookahead_node node) -> Set[Integer]
+      # @rbs (Array[packed_items] lookaheads, Integer node) -> Set[Integer]
       def node_set(lookaheads, node)
-        state_id, production_id, dot = node
+        state_id = node / @node_stride
+        item = node % @node_stride
+        production_id = ((item / @item_key_stride) - @production_offset).to_i
+        dot = (item % @item_key_stride).to_i
         lookaheads.fetch(state_id).fetch(item_core(production_id, dot))
       end
 
+      # @rbs (Integer state_id, Integer production_id, Integer dot) -> Integer
+      def node_id(state_id, production_id, dot)
+        ((state_id * @node_stride) + ((production_id + @production_offset) * @item_key_stride) + dot).to_i
+      end
+
       # @rbs (Integer bits) -> Array[Integer]
       def terminal_ids(bits)
-        selected = [] #: Array[Integer]
-        index = 0
-        while index < @terminal_ids.length
-          selected << @terminal_ids[index] if bits.anybits?(@terminal_masks[index])
-          index += 1
+        @terminal_ids_by_bits.fetch(bits) do
+          selected = @terminal_ids.select { |id| bits.anybits?(@terminal_masks[@terminal_ids.index(id)]) }
+          @terminal_ids_by_bits[bits] = selected.freeze
         end
-        selected
       end
 
       # @rbs (Integer production_id) -> Array[Integer]
       def rhs_for(production_id)
-        return @augmented_rhs if production_id == AUGMENTED_PRODUCTION
+        return @augmented_rhs.fetch(production_id) if production_id.negative?
 
         @production_rhs.fetch(production_id)
       end
 
       # @rbs (Integer production_id, Integer rhs_length) -> Array[item_core]
       def item_cores_for(production_id, rhs_length)
-        Array.new(rhs_length + 1) do |dot|
-          [production_id, dot].freeze #: item_core
-        end.freeze
+        result = Array.new(rhs_length + 1) #: Array[item_core]
+        rhs_length.next.times do |dot|
+          core = [production_id, dot].freeze #: item_core
+          result[dot] = core
+        end
+        result.freeze
       end
 
       # @rbs (Integer production_id, Integer dot) -> item_core
       def item_core(production_id, dot)
         raise IndexError, "invalid item dot: #{dot}" if dot.negative?
-        return @augmented_item_cores.fetch(dot) if production_id == AUGMENTED_PRODUCTION
-        raise IndexError, "invalid production id: #{production_id}" if production_id.negative?
+        return @augmented_item_cores.fetch(production_id).fetch(dot) if production_id.negative?
 
         @production_item_cores.fetch(production_id).fetch(dot)
       end
 
-      # @rbs (core_set items) -> Array[item_core]
+      # @rbs (core_set items) -> Array[Integer]
       def item_key(items)
-        items.to_a.sort
+        items.map do |production_id, dot|
+          (((production_id + @production_offset) * @item_key_stride) + dot).to_i
+        end.sort!
+      end
+
+      # @rbs (String name) -> Integer
+      def start_index(name)
+        @grammar_starts.index(name) || raise(Ibex::Error, "missing start symbol #{name}")
+      end
+
+      # @rbs () -> Array[String]
+      def grammar_starts
+        return @grammar.starts if @grammar.respond_to?(:starts)
+
+        [@grammar.start]
       end
     end
   end
 end
+
+# rubocop:enable Metrics/AbcSize, Metrics/ClassLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity

data/lib/ibex/lalr/goto_follows.rb

@@ -0,0 +1,229 @@
+# frozen_string_literal: true
+# rbs_inline: enabled
+
+# steep:ignore:start
+
+# Dependency construction mirrors the three formal relation definitions.
+# rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
+
+require_relative "lookahead_propagation"
+
+module Ibex
+  module LALR
+    # Computes DeRemer–Pennello goto-follow sets over an LR(0) collection.
+    # The three dependency closures are kept separately because IELR relies on
+    # the distinction between stable (successor/internal) and predecessor
+    # dependencies when a state is split.
+    class GotoFollows
+      # @rbs @grammar: IR::Grammar
+      # @rbs @sets: Analysis::Sets
+      # @rbs @states: Array[core_set]
+      # @rbs @transitions: transitions
+      # @rbs @kernel_cores: Array[Array[item_core]]
+
+      attr_reader :from_state #: Array[Integer]
+      attr_reader :to_state #: Array[Integer]
+      attr_reader :goto_index #: Hash[[Integer, Integer], Integer]
+      attr_reader :direct_reads #: Array[Integer]
+      attr_reader :successor_edges #: Array[Array[Integer]]
+      attr_reader :internal_edges #: Array[Array[Integer]]
+      attr_reader :includes_edges #: Array[Array[Integer]]
+      attr_reader :predecessors #: Array[Array[Integer]]
+      attr_reader :successor_follows #: Array[Integer]
+      attr_reader :always_follows #: Array[Integer]
+      attr_reader :goto_follows #: Array[Integer]
+      attr_reader :follow_kernel_items #: Array[Integer]
+      attr_reader :kernel_cores #: Array[Array[item_core]]
+
+      # @rbs (IR::Grammar grammar, Analysis::Sets sets, Array[core_set] states, transitions transitions,
+      #   Array[Integer] start_states, ?start_names: Array[String]?) -> void
+      def initialize(grammar, sets, states, transitions, start_states, start_names: nil)
+        @grammar = grammar
+        @sets = sets
+        @states = states
+        @transitions = transitions
+        @start_states = start_states
+        @start_names = start_names || grammar.starts
+        @kernel_cores = states.map { |items| kernel_items(items) }
+        build_gotos
+        build_dependencies
+        build_follow_sets
+        @reduction_lookaheads = nil
+      end
+
+      # @rbs (Integer state_id, Integer nonterminal_id) -> Integer?
+      def goto_for(state_id, nonterminal_id)
+        @goto_index[[state_id, nonterminal_id]]
+      end
+
+      # @rbs () -> Array[Hash[Integer, Integer]]
+      def reduction_lookaheads
+        return @reduction_lookaheads if @reduction_lookaheads
+
+        result = Array.new(@states.length) { {} }
+        seeds = @start_states.each_with_index.map do |state_id, index|
+          augmented = -1 - @grammar.starts.index(@start_names.fetch(index))
+          [state_id, [augmented, 0], eof_id]
+        end
+        propagated = LookaheadPropagation.new(
+          @grammar, @sets, @states, @transitions, seeds: seeds
+        ).build
+        @states.each_with_index do |items, state_id|
+          items.each do |production_id, dot|
+            next unless dot == rhs_for(production_id).length
+
+            values = propagated.fetch(state_id).fetch([production_id, dot], [])
+            result.fetch(state_id)[production_id] = values.reduce(0) do |bits, token|
+              bits | (1 << token)
+            end
+          end
+        end
+        @reduction_lookaheads = result
+      end
+
+      private
+
+      # @rbs () -> void
+      def build_gotos
+        @from_state = []
+        @to_state = []
+        @goto_symbols = []
+        @goto_index = {}
+        @transitions.each_with_index do |edges, state_id|
+          edges.keys.sort.each do |symbol_id|
+            symbol = @grammar.symbol_by_id(symbol_id)
+            next unless symbol&.nonterminal?
+
+            goto_id = @from_state.length
+            @from_state << state_id
+            @to_state << edges.fetch(symbol_id)
+            @goto_symbols << symbol_id
+            @goto_index[[state_id, symbol_id]] = goto_id
+          end
+        end
+      end
+
+      # @rbs () -> void
+      def build_dependencies
+        count = @from_state.length
+        @direct_reads = Array.new(count, 0)
+        @successor_edges = Array.new(count) { [] }
+        @internal_edges = Array.new(count) { [] }
+        @includes_edges = Array.new(count) { [] }
+        @predecessors = Array.new(@states.length) { [] }
+        @transitions.each_with_index do |edges, state_id|
+          edges.each_value { |target| @predecessors.fetch(target) << state_id }
+        end
+
+        count.times do |goto_id|
+          destination = @to_state.fetch(goto_id)
+          @transitions.fetch(destination).each_key do |symbol_id|
+            symbol = @grammar.symbol_by_id(symbol_id)
+            if symbol&.terminal?
+              @direct_reads[goto_id] |= 1 << symbol_id
+            elsif symbol&.nonterminal? && @sets.nullable?(symbol_id)
+              successor = @goto_index.fetch([destination, symbol_id])
+              @successor_edges.fetch(goto_id) << successor
+            end
+          end
+        end
+
+        @start_states.each_with_index do |state_id, index|
+          start_name = @start_names.fetch(index)
+          start_symbol = @grammar.symbol(start_name) || raise(Ibex::Error, "missing start symbol #{start_name}")
+          goto_id = @goto_index[[state_id, start_symbol.id]]
+          @direct_reads[goto_id] |= 1 << eof_id if goto_id
+        end
+
+        @goto_index.each do |(inner_source, inner_symbol), inner_goto_id|
+          @grammar.productions.each do |production|
+            production.rhs.each_with_index do |symbol_id, position|
+              next unless symbol_id == inner_symbol
+              next unless @grammar.symbol_by_id(symbol_id)&.nonterminal?
+
+              suffix = production.rhs[(position + 1)..] || []
+              next unless @sets.sequence_nullable?(suffix)
+
+              @goto_index.each do |(outer_source, outer_symbol), outer_goto_id|
+                next unless outer_symbol == production.lhs
+
+                reached = advance(outer_source, production.rhs.take(position))
+                next unless reached == inner_source
+
+                @includes_edges.fetch(inner_goto_id) << outer_goto_id
+                @internal_edges.fetch(inner_goto_id) << outer_goto_id if position.zero?
+              end
+            end
+          end
+        end
+        @successor_edges.each(&:uniq!)
+        @internal_edges.each(&:uniq!)
+        @includes_edges.each(&:uniq!)
+        @predecessors.each(&:uniq!)
+      end
+
+      # @rbs () -> void
+      def build_follow_sets
+        @successor_follows = Analysis::Digraph.closure(@direct_reads, @successor_edges)
+        internal_initial = @from_state.each_index.map do |goto_id|
+          1 << symbol_for_goto(goto_id)
+        end
+        @internal_reachable_symbols = Analysis::Digraph.closure(internal_initial, @internal_edges)
+        @always_follows = Analysis::Digraph.closure(@direct_reads, merged_edges(@successor_edges, @internal_edges))
+        @goto_follows = Analysis::Digraph.closure(@always_follows, @includes_edges)
+        @follow_kernel_items = @from_state.each_index.map { |goto_id| compute_follow_kernel_items(goto_id) }
+      end
+
+      # @rbs (Array[Array[Integer]], Array[Array[Integer]]) -> Array[Array[Integer]]
+      def merged_edges(left, right)
+        left.each_index.map { |index| (left.fetch(index) + right.fetch(index)).uniq }
+      end
+
+      # @rbs (Integer goto_id) -> Integer
+      def compute_follow_kernel_items(goto_id)
+        state_id = @from_state.fetch(goto_id)
+        reachable = @internal_reachable_symbols.fetch(goto_id)
+        @kernel_cores.fetch(state_id).each_with_index.reduce(0) do |mask, ((production_id, dot), index)|
+          rhs = rhs_for(production_id)
+          symbol_id = rhs[dot]
+          next mask unless symbol_id && reachable.anybits?(1 << symbol_id)
+          next mask unless @sets.sequence_nullable?(rhs[(dot + 1)..] || [])
+
+          mask | (1 << index)
+        end
+      end
+
+      # @rbs (core_set items) -> Array[item_core]
+      def kernel_items(items)
+        items.select { |production_id, dot| production_id.negative? || dot.positive? }.to_a.sort
+      end
+
+      # @rbs (Integer goto_id) -> Integer
+      def symbol_for_goto(goto_id)
+        @goto_symbols.fetch(goto_id)
+      end
+
+      # @rbs (Integer state_id, Array[Integer]) -> Integer?
+      def advance(state_id, symbols)
+        symbols.reduce(state_id) do |current, symbol_id|
+          @transitions.fetch(current).fetch(symbol_id, nil) || break
+        end
+      end
+
+      # @rbs (Integer production_id) -> Array[Integer]
+      def rhs_for(production_id)
+        return [@grammar.symbol(@grammar.starts.fetch(-production_id - 1)).id] if production_id.negative?
+
+        @grammar.productions.fetch(production_id).rhs
+      end
+
+      # @rbs () -> Integer
+      def eof_id
+        0
+      end
+    end
+  end
+end
+
+# rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
+# steep:ignore:end