ibex 0.2.0 → 0.3.0

data/lib/ibex/lalr/builder.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+# rbs_inline: enabled
 
 require "set"
 
@@ -11,10 +12,12 @@ module Ibex
 
       ALGORITHMS = %i[slr lalr ielr lr1].freeze #: Array[Symbol]
       LALR_STRATEGIES = %i[direct canonical_merge].freeze #: Array[Symbol]
+      IELR_STRATEGIES = %i[direct partition].freeze #: Array[Symbol]
 
       # @rbs @grammar: IR::Grammar
       # @rbs @algorithm: Symbol
       # @rbs @lalr_strategy: Symbol
+      # @rbs @ielr_strategy: Symbol
       # @rbs @sets: Analysis::Sets
       # @rbs @productions_by_lhs: Hash[Integer, Array[IR::Production]]
       # @rbs @resolver: ConflictResolver
@@ -22,6 +25,8 @@ module Ibex
       # @rbs @start_names: Array[String]
       # @rbs @entry_isolation: bool
       # @rbs @attribute_entries: bool
+      # @rbs @profile: bool
+      # @rbs @remove_unreachable: bool
       # @rbs @canonical_suffix_lookahead_cache: Hash[Integer, Hash[Integer, Hash[Integer, Array[Integer]]]]
       # @rbs @canonical_item_cache: Hash[Integer, Array[Array[lr_item?]?]]?
       # @rbs @canonical_key_radices: [Integer, Integer, Integer]?
@@ -29,19 +34,26 @@ module Ibex
       attr_reader :metrics #: BuildMetrics?
 
       # @rbs (IR::Grammar grammar, ?algorithm: Symbol | String, ?lalr_strategy: Symbol | String,
-      #   ?entry_isolation: bool, ?starts: Array[String]?, ?attribute_entries: bool) -> void
+      #   ?ielr_strategy: Symbol | String,
+      #   ?entry_isolation: bool, ?starts: Array[String]?, ?attribute_entries: bool, ?profile: bool,
+      #   ?remove_unreachable: bool) -> void
       def initialize(grammar, algorithm: :lalr, lalr_strategy: :direct, entry_isolation: false,
-                     starts: nil, attribute_entries: true)
+                     ielr_strategy: :partition, starts: nil, attribute_entries: true, profile: false,
+                     remove_unreachable: false)
         unless ALGORITHMS.include?(algorithm.to_sym)
           raise ArgumentError, "unknown parser algorithm #{algorithm.inspect}"
         end
         unless LALR_STRATEGIES.include?(lalr_strategy.to_sym)
           raise ArgumentError, "unknown LALR construction strategy #{lalr_strategy.inspect}"
         end
+        unless IELR_STRATEGIES.include?(ielr_strategy.to_sym)
+          raise ArgumentError, "unknown IELR construction strategy #{ielr_strategy.inspect}"
+        end
 
         @grammar = grammar
         @algorithm = algorithm.to_sym
         @lalr_strategy = lalr_strategy.to_sym
+        @ielr_strategy = ielr_strategy.to_sym
         @sets = Analysis::Sets.new(grammar)
         @productions_by_lhs = grammar.productions.group_by(&:lhs)
         @resolver = ConflictResolver.new(grammar)
@@ -56,31 +68,43 @@ module Ibex
 
         @entry_isolation = entry_isolation
         @attribute_entries = attribute_entries
+        @profile = profile
+        @remove_unreachable = remove_unreachable
       end
 
       # @rbs () -> IR::Automaton
       def build
         return build_isolated_automaton if @entry_isolation && @start_names.length > 1
 
-        merged_items, merged_transitions, construction_states, canonical_states, strategy = automaton_collection
+        merged_items, merged_transitions, collection = automaton_collection
         states = build_states(merged_items, merged_transitions)
         states = OnErrorReductions.apply(@grammar, states)
         states = DefaultReductions.apply(states, terminal_ids: @grammar.terminals.map(&:id))
         entry_states = entry_states_for(merged_items)
+        if @remove_unreachable
+          states, mapping = UnreachableStates.remove(states, entry_states.values.uniq)
+          entry_states = entry_states.transform_values { |state_id| mapping.fetch(state_id) }
+          collection[:ielr_unreachable_removed] = mapping.length - states.length if collection.respond_to?(:[]=)
+        end
         states = attribute_entry_conflicts(states, entry_states) if @attribute_entries && @start_names.length > 1
         summary = conflict_summary(states)
-        @metrics = BuildMetrics.new(
-          construction_states: construction_states,
-          canonical_states: canonical_states,
-          final_states: states.length,
-          strategy: strategy
-        )
-        IR::Automaton.new(grammar: @grammar, states: states, conflict_summary: summary,
-                          algorithm: algorithm_name, entry_states: entry_states)
+        final_items, final_lookahead_items = profiled_final_item_counts(merged_items)
+        @metrics = build_metrics(collection, states.length, final_items, final_lookahead_items)
+        build_output_automaton(states: states, conflict_summary: summary, entry_states: entry_states)
       end
 
       private
 
+      # @rbs (states: Array[IR::AutomatonState], conflict_summary: IR::conflict_summary,
+      #   entry_states: Hash[String, Integer]) -> IR::Automaton
+      def build_output_automaton(states:, conflict_summary:, entry_states:)
+        IR::Automaton.new(
+          grammar: @grammar, states: states, conflict_summary: conflict_summary,
+          algorithm: algorithm_name, entry_states: entry_states,
+          entry_construction: output_entry_construction || raise(Ibex::Error, "missing entry construction")
+        )
+      end
+
       # @rbs (Array[IR::AutomatonState] states) -> IR::conflict_summary
       def conflict_summary(states)
         conflicts = states.flat_map(&:conflicts)
@@ -99,39 +123,112 @@ module Ibex
         summary
       end
 
-      # @rbs () -> [Array[packed_items], transitions, Integer, Integer?, Symbol]
+      # @rbs () -> [Array[packed_items], transitions, build_collection]
       def automaton_collection
-        if @algorithm == :ielr
-          states, transitions = canonical_collection
-          items, merged_transitions = IELRPartition.new(@grammar, states, transitions).build
-          return [items, merged_transitions, states.length, states.length, :ielr_partition]
-        end
+        return ielr_collection if @algorithm == :ielr
+        return canonical_lr1_collection if @algorithm == :lr1
+        return merged_canonical_collection(:canonical_merge) if @lalr_strategy == :canonical_merge
+        return merged_canonical_collection(:canonical_merge_multi_entry) if @grammar.starts.length > 1
 
-        if @algorithm == :lr1
-          states, transitions = canonical_collection
-          return [pack_canonical_items(states), transitions, states.length, states.length, :canonical_lr1]
-        end
+        direct_collection
+      ensure
+        @canonical_item_cache = nil
+      end
 
-        if @lalr_strategy == :canonical_merge
-          states, transitions = canonical_collection
-          items, merged_transitions = merge_lalr(states, transitions)
-          apply_slr_lookaheads(items) if @algorithm == :slr
-          return [items, merged_transitions, states.length, states.length, :canonical_merge]
-        end
+      # @rbs () -> [Array[packed_items], transitions, build_collection]
+      def ielr_collection
+        return ielr_direct_collection if @ielr_strategy == :direct
 
-        if @grammar.starts.length > 1
-          states, transitions = canonical_collection
-          items, merged_transitions = merge_lalr(states, transitions)
-          apply_slr_lookaheads(items) if @algorithm == :slr
-          return [items, merged_transitions, states.length, states.length, :canonical_merge_multi_entry]
-        end
+        states, transitions = canonical_collection
+        partition = IELRPartition.new(@grammar, states, transitions, profile: @profile)
+        items, merged_transitions = partition.build
+        profile = canonical_profile(states, strategy: :ielr_partition)
+        profile[:ielr_initial_partitions] = partition.initial_partition_count
+        profile[:ielr_final_partitions] = partition.final_partition_count
+        [items, merged_transitions, profile]
+      end
+
+      # @rbs () -> [Array[packed_items], transitions, build_collection]
+      def ielr_direct_collection
+        pipeline = IELR::Pipeline.new(@grammar, @sets, starts: @start_names, profile: @profile)
+        pipeline.build
+      end
 
-        items, transitions = DirectLookaheads.new(@grammar, @sets).build
+      # @rbs () -> [Array[packed_items], transitions, build_collection]
+      def canonical_lr1_collection
+        states, transitions = canonical_collection
+        [pack_canonical_items(states), transitions, canonical_profile(states, strategy: :canonical_lr1)]
+      end
+
+      # @rbs (Symbol strategy) -> [Array[packed_items], transitions, build_collection]
+      def merged_canonical_collection(strategy)
+        states, transitions = canonical_collection
+        items, merged_transitions = merge_lalr(states, transitions)
         apply_slr_lookaheads(items) if @algorithm == :slr
-        collection = [items, transitions, items.length, nil, :direct_lalr]
-        collection #: [Array[packed_items], transitions, Integer, Integer?, Symbol]
-      ensure
-        @canonical_item_cache = nil
+        [items, merged_transitions, canonical_profile(states, strategy: strategy)]
+      end
+
+      # @rbs () -> untyped
+      def direct_collection
+        direct = DirectLookaheads.new(@grammar, @sets, profile: @profile)
+        items, transitions = direct.build
+        apply_slr_lookaheads(items) if @algorithm == :slr
+        profile = {
+          construction_states: items.length, canonical_states: nil, strategy: :direct_lalr,
+          lr0_states: direct.lr0_state_count, lr0_items: direct.lr0_item_count, canonical_items: nil,
+          propagation_edges: direct.propagation_edge_count,
+          ielr_initial_partitions: nil, ielr_final_partitions: nil,
+          ielr_annotations: nil, ielr_annotated_states: nil, ielr_inadequacies: nil,
+          ielr_split_stable_discarded: nil, ielr_lalr_states: nil, ielr_split_states: nil,
+          ielr_unreachable_removed: nil, ielr_remergeable_candidates: nil
+        }
+        [items, transitions, profile]
+      end
+
+      # @rbs (Array[item_set] states, strategy: Symbol) -> build_collection
+      def canonical_profile(states, strategy:)
+        cores = states.to_h { |items| [core_key(items), true] }.keys if @profile
+        {
+          construction_states: states.length, canonical_states: states.length, strategy: strategy,
+          lr0_states: cores&.length, lr0_items: cores&.sum(&:length),
+          canonical_items: @profile ? states.sum(&:length) : nil,
+          propagation_edges: nil, ielr_initial_partitions: nil, ielr_final_partitions: nil,
+          ielr_annotations: nil, ielr_annotated_states: nil, ielr_inadequacies: nil,
+          ielr_split_stable_discarded: nil, ielr_lalr_states: nil, ielr_split_states: nil,
+          ielr_unreachable_removed: nil, ielr_remergeable_candidates: nil
+        }
+      end
+
+      # @rbs (Array[packed_items] items) -> [Integer?, Integer?]
+      def profiled_final_item_counts(items)
+        return [nil, nil] unless @profile
+
+        [items.sum(&:length), items.sum { |state| state.values.sum(&:length) }]
+      end
+
+      # @rbs (build_collection collection, Integer final_states, Integer? final_items,
+      #   Integer? final_lookahead_items) -> BuildMetrics
+      def build_metrics(collection, final_states, final_items, final_lookahead_items)
+        BuildMetrics.new(
+          construction_states: collection.fetch(:construction_states),
+          canonical_states: collection.fetch(:canonical_states),
+          strategy: collection.fetch(:strategy),
+          lr0_states: collection.fetch(:lr0_states),
+          lr0_items: collection.fetch(:lr0_items),
+          canonical_items: collection.fetch(:canonical_items),
+          propagation_edges: collection.fetch(:propagation_edges),
+          ielr_initial_partitions: collection.fetch(:ielr_initial_partitions),
+          ielr_final_partitions: collection.fetch(:ielr_final_partitions),
+          ielr_annotations: collection.fetch(:ielr_annotations, nil),
+          ielr_annotated_states: collection.fetch(:ielr_annotated_states, nil),
+          ielr_inadequacies: collection.fetch(:ielr_inadequacies, nil),
+          ielr_split_stable_discarded: collection.fetch(:ielr_split_stable_discarded, nil),
+          ielr_lalr_states: collection.fetch(:ielr_lalr_states, nil),
+          ielr_split_states: collection.fetch(:ielr_split_states, nil),
+          ielr_unreachable_removed: collection.fetch(:ielr_unreachable_removed, nil),
+          ielr_remergeable_candidates: collection.fetch(:ielr_remergeable_candidates, nil),
+          final_states: final_states, final_items: final_items, final_lookahead_items: final_lookahead_items
+        )
       end
 
       # @rbs () -> String
@@ -335,21 +432,21 @@ module Ibex
       end
 
       # Conflict hashes are produced locally immediately before this boundary.
-      # Keeping the parameter untyped avoids duplicating both discriminated
-      # hash variants solely to add their common optional provenance field.
-      # @rbs (untyped conflict) -> IR::conflict
+      # Both conflict variants carry the same optional midrule provenance field.
+      # @rbs (IR::conflict conflict) -> IR::conflict
       def attribute_midrule_conflict(conflict)
         production_ids = if conflict[:type] == :shift_reduce
                            [conflict[:reduce]]
                          else
                            conflict[:reductions]
-                         end
+                         end #: Array[Integer]
         origins = production_ids.filter_map do |production_id|
           production = @grammar.productions.fetch(production_id)
           production.origin[:loc] if production.origin[:kind] == :inline_action
         end
-        conflict[:midrule_origins] = origins.uniq unless origins.empty?
-        conflict
+        result = conflict.dup #: Hash[Symbol, untyped]
+        result[:midrule_origins] = origins.uniq unless origins.empty?
+        result #: IR::conflict
       end
 
       # @rbs (Array[IR::AutomatonItem] items, Hash[Integer, Array[IR::parser_action]] candidates) -> void
@@ -465,23 +562,60 @@ module Ibex
           canonical_counts << metrics.canonical_states
         end
         canonical_states = canonical_counts.compact.sum if canonical_counts.none?(&:nil?)
-        @metrics = BuildMetrics.new(
-          construction_states: construction_states,
-          canonical_states: canonical_states,
-          final_states: states.length,
-          strategy: :entry_isolation
+        @metrics = isolated_metrics(entries, construction_states, canonical_states, states.length)
+        build_output_automaton(
+          states: states, conflict_summary: conflict_summary(states), entry_states: entry_states
         )
-        IR::Automaton.new(
-          grammar: @grammar, states: states, conflict_summary: conflict_summary(states),
-          algorithm: algorithm_name, entry_states: entry_states
+      end
+
+      # @rbs (Array[[String, IR::Automaton, BuildMetrics]] entries, Integer construction_states,
+      #   Integer? canonical_states, Integer final_states) -> BuildMetrics
+      def isolated_metrics(entries, construction_states, canonical_states, final_states)
+        BuildMetrics.new(
+          construction_states: construction_states, canonical_states: canonical_states,
+          final_states: final_states, strategy: :entry_isolation,
+          lr0_states: sum_optional_metric(entries, :lr0_states),
+          lr0_items: sum_optional_metric(entries, :lr0_items),
+          canonical_items: sum_optional_metric(entries, :canonical_items),
+          final_items: sum_optional_metric(entries, :final_items),
+          final_lookahead_items: sum_optional_metric(entries, :final_lookahead_items),
+          propagation_edges: sum_optional_metric(entries, :propagation_edges),
+          ielr_initial_partitions: sum_optional_metric(entries, :ielr_initial_partitions),
+          ielr_final_partitions: sum_optional_metric(entries, :ielr_final_partitions),
+          ielr_annotations: sum_optional_metric(entries, :ielr_annotations),
+          ielr_annotated_states: sum_optional_metric(entries, :ielr_annotated_states),
+          ielr_inadequacies: sum_optional_metric(entries, :ielr_inadequacies),
+          ielr_split_stable_discarded: sum_optional_metric(entries, :ielr_split_stable_discarded),
+          ielr_lalr_states: sum_optional_metric(entries, :ielr_lalr_states),
+          ielr_split_states: sum_optional_metric(entries, :ielr_split_states),
+          ielr_unreachable_removed: sum_optional_metric(entries, :ielr_unreachable_removed),
+          ielr_remergeable_candidates: sum_optional_metric(entries, :ielr_remergeable_candidates)
         )
       end
 
+      # @rbs (Array[[String, IR::Automaton, BuildMetrics]] entries, Symbol method) -> Integer?
+      def sum_optional_metric(entries, method)
+        values = entries.map { |_name, _automaton, metrics| metrics.public_send(method) }
+        values.compact.sum if values.none?(&:nil?)
+      end
+
+      # @rbs () -> Integer
+      def output_schema_version
+        IR::SCHEMA_VERSION
+      end
+
+      # @rbs () -> String?
+      def output_entry_construction
+        @entry_isolation ? "isolated" : "shared"
+      end
+
       # @rbs (String name) -> [String, IR::Automaton, BuildMetrics]
       def isolated_entry(name)
         builder = self.class.new(
           @grammar, algorithm: @algorithm, lalr_strategy: @lalr_strategy,
-                    starts: [name], attribute_entries: false
+                    ielr_strategy: @ielr_strategy,
+                    starts: [name], entry_isolation: true, attribute_entries: false, profile: @profile,
+                    remove_unreachable: @remove_unreachable
         )
         automaton = builder.build
         metrics = builder.metrics
@@ -569,11 +703,12 @@ module Ibex
         reachable
       end
 
-      # @rbs () -> Set[Array[untyped]]
+      # @rbs () -> Set[conflict_fingerprint]
       def isolated_conflict_fingerprints
         @start_names.each_with_object(Set.new) do |name, fingerprints|
           isolated = self.class.new(
             @grammar, algorithm: @algorithm, lalr_strategy: @lalr_strategy,
+                      ielr_strategy: @ielr_strategy,
                       starts: [name], attribute_entries: false
           ).build
           isolated.states.each do |state|
@@ -582,13 +717,13 @@ module Ibex
         end
       end
 
-      # @rbs (IR::conflict conflict) -> Array[untyped]
+      # @rbs (IR::conflict conflict) -> conflict_fingerprint
       def conflict_fingerprint(conflict)
         reductions = if conflict[:type] == :shift_reduce
                        [conflict[:reduce]]
                      else
                        conflict[:reductions]
-                     end
+                     end #: Array[Integer]
         [conflict[:type], conflict[:symbol], reductions]
       end
     end

data/lib/ibex/lalr/conflict_search.rb

@@ -47,9 +47,9 @@ module Ibex
         @initial_state = automaton.entry_states.fetch(entry)
       end
 
-      # @rbs () -> search_result?
+      # @rbs () -> search_outcome
       def call
-        return unless @lookahead
+        return outcome(:not_found) unless @lookahead
 
         initial = configuration([@initial_state], Array.new(0))
         queue = [[initial, Array.new(0)]] #: Array[[Configuration, Array[Integer]]]
@@ -58,11 +58,11 @@ module Ibex
           current, prefix = queue.shift
           conflict_configurations(current).each do |candidate|
             result = unify_from(candidate, prefix)
-            return result if result
+            return outcome(:found, result) if result
           end
           enqueue_prefixes(queue, visited, current, prefix)
         end
-        nil
+        outcome(exhausted? ? :exhausted : :not_found)
       end
 
       # @rbs () -> bool
@@ -70,6 +70,17 @@ module Ibex
 
       private
 
+      # @rbs (:found | :not_found | :exhausted status, ?search_result? result) -> search_outcome
+      def outcome(status, result = nil)
+        {
+          status: status,
+          result: result,
+          explored: @explored,
+          exhausted: status == :exhausted,
+          bounds: { max_tokens: @max_tokens, max_configurations: @max_configurations }
+        }
+      end
+
       # @rbs (Configuration current) -> Array[Configuration]
       def conflict_configurations(current)
         advance(current, required_lookahead, stop_at: @state.id, branch_conflicts: true)

data/lib/ibex/lalr/counterexample.rb

@@ -17,14 +17,14 @@ module Ibex
         @shortest_yields = compute_shortest_yields
       end
 
-      # @rbs () -> Array[IR::counterexample]
+      # @rbs () -> Array[search_counterexample]
       def all
         @automaton.states.flat_map do |state|
           state.conflicts.map { |conflict| build_example(state, conflict) }
         end
       end
 
-      # @rbs (Integer state_id, Integer conflict_index) -> IR::counterexample
+      # @rbs (Integer state_id, Integer conflict_index) -> search_counterexample
       def for_conflict(state_id, conflict_index)
         state = @automaton.states.find { |candidate| candidate.id == state_id }
         raise ArgumentError, "unknown automaton state #{state_id}" unless state
@@ -39,31 +39,34 @@ module Ibex
 
       private
 
-      # @rbs (IR::AutomatonState state, IR::conflict conflict) -> IR::counterexample
+      # @rbs (IR::AutomatonState state, IR::conflict conflict) -> search_counterexample
       def build_example(state, conflict)
-        unifying = ConflictSearch.new(
+        outcome = ConflictSearch.new(
           @automaton, state, conflict, max_tokens: @max_tokens, max_configurations: @max_configurations
         ).call
-        return unifying_example(state, conflict, unifying) if unifying
+        result = outcome[:result]
+        return unifying_example(state, conflict, outcome, result) if outcome[:status] == :found && result
 
-        reachability_example(state, conflict)
+        reachability_example(state, conflict, outcome)
       end
 
-      # @rbs (IR::AutomatonState state, IR::conflict conflict, Hash[Symbol, untyped] result) -> IR::counterexample
-      def unifying_example(state, conflict, result)
+      # @rbs (IR::AutomatonState state, IR::conflict conflict, search_outcome outcome,
+      #   search_result result) -> search_counterexample
+      def unifying_example(state, conflict, outcome, result)
         { state: state.id, type: conflict[:type], symbol_path: Array.new(0),
           sentence: names(result[:sentence_ids]), lookahead_index: result[:lookahead_index], unifying: true,
-          interpretations: result[:interpretations] }
+          inconclusive: false, search: outcome, interpretations: result[:interpretations] }
       end
 
-      # @rbs (IR::AutomatonState state, IR::conflict conflict) -> IR::counterexample
-      def reachability_example(state, conflict)
+      # @rbs (IR::AutomatonState state, IR::conflict conflict, search_outcome outcome) -> search_counterexample
+      def reachability_example(state, conflict, outcome)
         path = shortest_state_path(state.id, conflict)
         lookahead = @grammar.symbol(conflict[:symbol])
         terminal_ids = path.flat_map { |symbol_id| @shortest_yields[symbol_id] || [] }
         terminal_ids << lookahead.id if lookahead
         { state: state.id, type: conflict[:type], symbol_path: names(path), sentence: names(terminal_ids),
-          lookahead_index: terminal_ids.length - 1, unifying: false, interpretations: interpretations(conflict) }
+          lookahead_index: terminal_ids.length - 1, unifying: false,
+          inconclusive: outcome[:status] == :exhausted, search: outcome, interpretations: interpretations(conflict) }
       end
 
       # @rbs (Integer target, IR::conflict conflict) -> Array[Integer]
@@ -125,12 +128,14 @@ module Ibex
         end
       end
 
-      # @rbs (IR::shift_reduce_conflict conflict) -> IR::interpretation
+      # The compact counterexample tree intentionally uses a lightweight shape
+      # whose children are symbol names rather than recursively expanded nodes.
+      # @rbs (IR::shift_reduce_conflict conflict) -> untyped
       def shift_interpretation(conflict)
         { kind: :shift, state: conflict[:shift_to], tree: { token: conflict[:symbol] } }
       end
 
-      # @rbs (Integer production_id) -> IR::interpretation
+      # @rbs (Integer production_id) -> untyped
       def reduce_interpretation(production_id)
         production = @grammar.productions.fetch(production_id)
         lhs = symbol_name(production.lhs)