lrama 0.6.10 → 0.7.0

data/lib/lrama/state/reduce.rb

@@ -26,9 +26,8 @@ module Lrama
       end
 
       def selected_look_ahead
-        if @look_ahead
-          # @type ivar @look_ahead: Array<Grammar::Symbol>
-          @look_ahead - @not_selected_symbols
+        if look_ahead
+          look_ahead - @not_selected_symbols
         else
           []
         end

data/lib/lrama/state.rb

@@ -10,7 +10,7 @@ module Lrama
   class State
     attr_reader :id, :accessing_symbol, :kernels, :conflicts, :resolved_conflicts,
                 :default_reduction_rule, :closure, :items
-    attr_accessor :shifts, :reduces
+    attr_accessor :shifts, :reduces, :ielr_isocores, :lalr_isocore
 
     def initialize(id, accessing_symbol, kernels)
       @id = id
@@ -23,6 +23,12 @@ module Lrama
       @conflicts = []
       @resolved_conflicts = []
       @default_reduction_rule = nil
+      @predecessors = []
+      @lalr_isocore = self
+      @ielr_isocores = [self]
+      @internal_dependencies = {}
+      @successor_dependencies = {}
+      @always_follows = {}
     end
 
     def closure=(closure)
@@ -84,6 +90,18 @@ module Lrama
       @transitions ||= shifts.map {|shift| [shift, @items_to_state[shift.next_items]] }
     end
 
+    def update_transition(shift, next_state)
+      set_items_to_state(shift.next_items, next_state)
+      next_state.append_predecessor(self)
+      clear_transitions_cache
+    end
+
+    def clear_transitions_cache
+      @nterm_transitions = nil
+      @term_transitions = nil
+      @transitions = nil
+    end
+
     def selected_term_transitions
       term_transitions.reject do |shift, next_state|
         shift.not_selected
@@ -142,5 +160,274 @@ module Lrama
         conflict.type == :reduce_reduce
       end
     end
+
+    def propagate_lookaheads(next_state)
+      next_state.kernels.map {|item|
+        lookahead_sets =
+          if item.position == 1
+            goto_follow_set(item.lhs)
+          else
+            kernel = kernels.find {|k| k.predecessor_item_of?(item) }
+            item_lookahead_set[kernel]
+          end
+
+        [item, lookahead_sets & next_state.lookahead_set_filters[item]]
+      }.to_h
+    end
+
+    def lookaheads_recomputed
+      !@item_lookahead_set.nil?
+    end
+
+    def compatible_lookahead?(filtered_lookahead)
+      !lookaheads_recomputed ||
+        @lalr_isocore.annotation_list.all? {|token, actions|
+          a = dominant_contribution(token, actions, item_lookahead_set)
+          b = dominant_contribution(token, actions, filtered_lookahead)
+          a.nil? || b.nil? || a == b
+        }
+    end
+
+    def lookahead_set_filters
+      kernels.map {|kernel|
+        [kernel,
+         @lalr_isocore.annotation_list.select {|token, actions|
+           token.term? && actions.any? {|action, contributions|
+             !contributions.nil? && contributions.key?(kernel) && contributions[kernel]
+           }
+         }.map {|token, _| token }
+        ]
+      }.to_h
+    end
+
+    def dominant_contribution(token, actions, lookaheads)
+      a = actions.select {|action, contributions|
+        contributions.nil? || contributions.any? {|item, contributed| contributed && lookaheads[item].include?(token) }
+      }.map {|action, _| action }
+      return nil if a.empty?
+      a.reject {|action|
+        if action.is_a?(State::Shift)
+          action.not_selected
+        elsif action.is_a?(State::Reduce)
+          action.not_selected_symbols.include?(token)
+        end
+      }
+    end
+
+    def inadequacy_list
+      return @inadequacy_list if @inadequacy_list
+
+      shift_contributions = shifts.map {|shift|
+        [shift.next_sym, [shift]]
+      }.to_h
+      reduce_contributions = reduces.map {|reduce|
+        (reduce.look_ahead || []).map {|sym|
+          [sym, [reduce]]
+        }.to_h
+      }.reduce(Hash.new([])) {|hash, cont|
+        hash.merge(cont) {|_, a, b| a | b }
+      }
+
+      list = shift_contributions.merge(reduce_contributions) {|_, a, b| a | b }
+      @inadequacy_list = list.select {|token, actions| token.term? && actions.size > 1 }
+    end
+
+    def annotation_list
+      return @annotation_list if @annotation_list
+
+      @annotation_list = annotate_manifestation
+      @annotation_list = @items_to_state.values.map {|next_state| next_state.annotate_predecessor(self) }
+        .reduce(@annotation_list) {|result, annotations|
+          result.merge(annotations) {|_, actions_a, actions_b|
+            if actions_a.nil? || actions_b.nil?
+              actions_a || actions_b
+            else
+              actions_a.merge(actions_b) {|_, contributions_a, contributions_b|
+                if contributions_a.nil? || contributions_b.nil?
+                  next contributions_a || contributions_b
+                end
+
+                contributions_a.merge(contributions_b) {|_, contributed_a, contributed_b|
+                  contributed_a || contributed_b
+                }
+              }
+            end
+          }
+        }
+    end
+
+    def annotate_manifestation
+      inadequacy_list.transform_values {|actions|
+        actions.map {|action|
+          if action.is_a?(Shift)
+            [action, nil]
+          elsif action.is_a?(Reduce)
+            if action.rule.empty_rule?
+              [action, lhs_contributions(action.rule.lhs, inadequacy_list.key(actions))]
+            else
+              contributions = kernels.map {|kernel| [kernel, kernel.rule == action.rule && kernel.end_of_rule?] }.to_h
+              [action, contributions]
+            end
+          end
+        }.to_h
+      }
+    end
+
+    def annotate_predecessor(predecessor)
+      annotation_list.transform_values {|actions|
+        token = annotation_list.key(actions)
+        actions.transform_values {|inadequacy|
+          next nil if inadequacy.nil?
+          lhs_adequacy = kernels.any? {|kernel|
+            inadequacy[kernel] && kernel.position == 1 && predecessor.lhs_contributions(kernel.lhs, token).nil?
+          }
+          if lhs_adequacy
+            next nil
+          else
+            predecessor.kernels.map {|pred_k|
+              [pred_k, kernels.any? {|k|
+                inadequacy[k] && (
+                  pred_k.predecessor_item_of?(k) && predecessor.item_lookahead_set[pred_k].include?(token) ||
+                  k.position == 1 && predecessor.lhs_contributions(k.lhs, token)[pred_k]
+                )
+              }]
+            }.to_h
+          end
+        }
+      }
+    end
+
+    def lhs_contributions(sym, token)
+      shift, next_state = nterm_transitions.find {|sh, _| sh.next_sym == sym }
+      if always_follows(shift, next_state).include?(token)
+        nil
+      else
+        kernels.map {|kernel| [kernel, follow_kernel_items(shift, next_state, kernel) && item_lookahead_set[kernel].include?(token)] }.to_h
+      end
+    end
+
+    def follow_kernel_items(shift, next_state, kernel)
+      queue = [[self, shift, next_state]]
+      until queue.empty?
+        st, sh, next_st = queue.pop
+        return true if kernel.next_sym == sh.next_sym && kernel.symbols_after_transition.all?(&:nullable)
+        st.internal_dependencies(sh, next_st).each {|v| queue << v }
+      end
+      false
+    end
+
+    def item_lookahead_set
+      return @item_lookahead_set if @item_lookahead_set
+
+      kernels.map {|item|
+        value =
+          if item.lhs.accept_symbol?
+            []
+          elsif item.position > 1
+            prev_items = predecessors_with_item(item)
+            prev_items.map {|st, i| st.item_lookahead_set[i] }.reduce([]) {|acc, syms| acc |= syms }
+          elsif item.position == 1
+            prev_state = @predecessors.find {|p| p.shifts.any? {|shift| shift.next_sym == item.lhs } }
+            shift, next_state = prev_state.nterm_transitions.find {|shift, _| shift.next_sym == item.lhs }
+            prev_state.goto_follows(shift, next_state)
+          end
+        [item, value]
+      }.to_h
+    end
+
+    def item_lookahead_set=(k)
+      @item_lookahead_set = k
+    end
+
+    def predecessors_with_item(item)
+      result = []
+      @predecessors.each do |pre|
+        pre.items.each do |i|
+          result << [pre, i] if i.predecessor_item_of?(item)
+        end
+      end
+      result
+    end
+
+    def append_predecessor(prev_state)
+      @predecessors << prev_state
+      @predecessors.uniq!
+    end
+
+    def goto_follow_set(nterm_token)
+      return [] if nterm_token.accept_symbol?
+      shift, next_state = @lalr_isocore.nterm_transitions.find {|sh, _| sh.next_sym == nterm_token }
+
+      @kernels
+        .select {|kernel| follow_kernel_items(shift, next_state, kernel) }
+        .map {|kernel| item_lookahead_set[kernel] }
+        .reduce(always_follows(shift, next_state)) {|result, terms| result |= terms }
+    end
+
+    def goto_follows(shift, next_state)
+      queue = internal_dependencies(shift, next_state) + predecessor_dependencies(shift, next_state)
+      terms = always_follows(shift, next_state)
+      until queue.empty?
+        st, sh, next_st = queue.pop
+        terms |= st.always_follows(sh, next_st)
+        st.internal_dependencies(sh, next_st).each {|v| queue << v }
+        st.predecessor_dependencies(sh, next_st).each {|v| queue << v }
+      end
+      terms
+    end
+
+    def always_follows(shift, next_state)
+      return @always_follows[[shift, next_state]] if @always_follows[[shift, next_state]]
+
+      queue = internal_dependencies(shift, next_state) + successor_dependencies(shift, next_state)
+      terms = []
+      until queue.empty?
+        st, sh, next_st = queue.pop
+        terms |= next_st.term_transitions.map {|sh, _| sh.next_sym }
+        st.internal_dependencies(sh, next_st).each {|v| queue << v }
+        st.successor_dependencies(sh, next_st).each {|v| queue << v }
+      end
+      @always_follows[[shift, next_state]] = terms
+    end
+
+    def internal_dependencies(shift, next_state)
+      return @internal_dependencies[[shift, next_state]] if @internal_dependencies[[shift, next_state]]
+
+      syms = @items.select {|i|
+        i.next_sym == shift.next_sym && i.symbols_after_transition.all?(&:nullable) && i.position == 0
+      }.map(&:lhs).uniq
+      @internal_dependencies[[shift, next_state]] = nterm_transitions.select {|sh, _| syms.include?(sh.next_sym) }.map {|goto| [self, *goto] }
+    end
+
+    def successor_dependencies(shift, next_state)
+      return @successor_dependencies[[shift, next_state]] if @successor_dependencies[[shift, next_state]]
+
+      @successor_dependencies[[shift, next_state]] =
+        next_state.nterm_transitions
+        .select {|next_shift, _| next_shift.next_sym.nullable }
+        .map {|transition| [next_state, *transition] }
+    end
+
+    def predecessor_dependencies(shift, next_state)
+      state_items = []
+      @kernels.select {|kernel|
+        kernel.next_sym == shift.next_sym && kernel.symbols_after_transition.all?(&:nullable)
+      }.each do |item|
+        queue = predecessors_with_item(item)
+        until queue.empty?
+          st, i = queue.pop
+          if i.position == 0
+            state_items << [st, i]
+          else
+            st.predecessors_with_item(i).each {|v| queue << v }
+          end
+        end
+      end
+
+      state_items.map {|state, item|
+        sh, next_st = state.nterm_transitions.find {|shi, _| shi.next_sym == item.lhs }
+        [state, sh, next_st]
+      }
+    end
   end
 end

data/lib/lrama/states/item.rb

@@ -64,6 +64,10 @@ module Lrama
         rhs[position..-1]
       end
 
+      def symbols_after_transition
+        rhs[position+1..-1]
+      end
+
       def to_s
         "#{lhs.id.s_value}: #{display_name}"
       end
@@ -78,6 +82,10 @@ module Lrama
         r = symbols_after_dot.map(&:display_name).join(" ")
         ". #{r}  (rule #{rule_id})"
       end
+
+      def predecessor_item_of?(other_item)
+        rule == other_item.rule && position == other_item.position - 1
+      end
     end
   end
 end

data/lib/lrama/states.rb

@@ -41,6 +41,8 @@ module Lrama
       # value is array of [state.id, nterm.token_id].
       @reads_relation = {}
 
+      # `Read(p, A) =s DR(p, A) ∪ ∪{Read(r, C) | (p, A) reads (r, C)}`
+      #
       # `@read_sets` is a hash whose
       # key is [state.id, nterm.token_id],
       # value is bitmap of term.
@@ -62,6 +64,8 @@ module Lrama
       # value is array of [state.id, nterm.token_id].
       @lookback_relation = {}
 
+      # `Follow(p, A) =s Read(p, A) ∪ ∪{Follow(p', B) | (p, A) includes (p', B)}`
+      #
       # `@follow_sets` is a hash whose
       # key is [state.id, rule.id],
       # value is bitmap of term.
@@ -92,6 +96,20 @@ module Lrama
       report_duration(:compute_default_reduction) { compute_default_reduction }
     end
 
+    def compute_ielr
+      report_duration(:split_states) { split_states }
+      report_duration(:compute_direct_read_sets) { compute_direct_read_sets }
+      report_duration(:compute_reads_relation) { compute_reads_relation }
+      report_duration(:compute_read_sets) { compute_read_sets }
+      report_duration(:compute_includes_relation) { compute_includes_relation }
+      report_duration(:compute_lookback_relation) { compute_lookback_relation }
+      report_duration(:compute_follow_sets) { compute_follow_sets }
+      report_duration(:compute_look_ahead_sets) { compute_look_ahead_sets }
+      report_duration(:compute_conflicts) { compute_conflicts }
+
+      report_duration(:compute_default_reduction) { compute_default_reduction }
+    end
+
     def reporter
       StatesReporter.new(self)
     end
@@ -235,7 +253,7 @@ module Lrama
       # Trace
       previous = state.kernels.first.previous_sym
       trace_state do |out|
-        out << sprintf("state_list_append (state = %d, symbol = %d (%s))",
+        out << sprintf("state_list_append (state = %d, symbol = %d (%s))\n",
           @states.count, previous.number, previous.display_name)
       end
 
@@ -265,7 +283,10 @@ module Lrama
         state.shifts.each do |shift|
           new_state, created = create_state(shift.next_sym, shift.next_items, states_created)
           state.set_items_to_state(shift.next_items, new_state)
-          enqueue_state(states, new_state) if created
+          if created
+            enqueue_state(states, new_state)
+            new_state.append_predecessor(state)
+          end
         end
       end
     end
@@ -524,5 +545,51 @@ module Lrama
         end.first
       end
     end
+
+    def split_states
+      @states.each do |state|
+        state.transitions.each do |shift, next_state|
+          compute_state(state, shift, next_state)
+        end
+      end
+    end
+
+    def merge_lookaheads(state, filtered_lookaheads)
+      return if state.kernels.all? {|item| (filtered_lookaheads[item] - state.item_lookahead_set[item]).empty? }
+
+      state.item_lookahead_set = state.item_lookahead_set.merge {|_, v1, v2| v1 | v2 }
+      state.transitions.each do |shift, next_state|
+        next if next_state.lookaheads_recomputed
+        compute_state(state, shift, next_state)
+      end
+    end
+
+    def compute_state(state, shift, next_state)
+      filtered_lookaheads = state.propagate_lookaheads(next_state)
+      s = next_state.ielr_isocores.find {|st| st.compatible_lookahead?(filtered_lookaheads) }
+
+      if s.nil?
+        s = next_state.ielr_isocores.last
+        new_state = State.new(@states.count, s.accessing_symbol, s.kernels)
+        new_state.closure = s.closure
+        new_state.compute_shifts_reduces
+        s.transitions.each do |sh, next_state|
+          new_state.set_items_to_state(sh.next_items, next_state)
+        end
+        @states << new_state
+        new_state.lalr_isocore = s
+        s.ielr_isocores << new_state
+        s.ielr_isocores.each do |st|
+          st.ielr_isocores = s.ielr_isocores
+        end
+        new_state.item_lookahead_set = filtered_lookaheads
+        state.update_transition(shift, new_state)
+      elsif(!s.lookaheads_recomputed)
+        s.item_lookahead_set = filtered_lookaheads
+      else
+        state.update_transition(shift, s)
+        merge_lookaheads(s, filtered_lookaheads)
+      end
+    end
   end
 end

data/lib/lrama/trace_reporter.rb

@@ -1,27 +1,42 @@
+# rbs_inline: enabled
 # frozen_string_literal: true
 
 module Lrama
   class TraceReporter
+    # @rbs (Lrama::Grammar grammar) -> void
     def initialize(grammar)
       @grammar = grammar
     end
 
+    # @rbs (**Hash[Symbol, bool] options) -> void
     def report(**options)
       _report(**options)
     end
 
     private
 
-    def _report(rules: false, actions: false, **_)
-      report_rules if rules
+    # @rbs rules: (bool rules, bool actions, bool only_explicit_rules, **untyped _) -> void
+    def _report(rules: false, actions: false, only_explicit_rules: false, **_)
+      report_rules if rules && !only_explicit_rules
+      report_only_explicit_rules if only_explicit_rules
       report_actions if actions
     end
 
+    # @rbs () -> void
     def report_rules
       puts "Grammar rules:"
       @grammar.rules.each { |rule| puts rule.display_name }
     end
 
+    # @rbs () -> void
+    def report_only_explicit_rules
+      puts "Grammar rules:"
+      @grammar.rules.each do |rule|
+        puts rule.display_name_without_action if rule.lhs.first_set.any?
+      end
+    end
+
+    # @rbs () -> void
     def report_actions
       puts "Grammar rules with actions:"
       @grammar.rules.each { |rule| puts rule.with_actions }

data/lib/lrama/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Lrama
-  VERSION = "0.6.10".freeze
+  VERSION = "0.7.0".freeze
 end

data/lrama.gemspec

@@ -21,7 +21,7 @@ Gem::Specification.new do |spec|
 
   spec.metadata["homepage_uri"]      = spec.homepage
   spec.metadata["source_code_uri"]   = spec.homepage
-  spec.metadata["documentation_uri"] = spec.homepage
+  spec.metadata["documentation_uri"] = "https://ruby.github.io/lrama/"
   spec.metadata["changelog_uri"]     = "#{spec.homepage}/releases"
   spec.metadata["bug_tracker_uri"]   = "#{spec.homepage}/issues"
 

data/parser.y

@@ -25,7 +25,7 @@ rule
 
   bison_declaration: grammar_declaration
                    | "%expect" INTEGER { @grammar.expect = val[1] }
-                   | "%define" variable value
+                   | "%define" variable value { @grammar.define[val[1].s_value] = val[2]&.s_value }
                    | "%param" param+
                    | "%lex-param" param+
                        {
@@ -61,26 +61,26 @@ rule
                      | symbol_declaration
                      | rule_declaration
                      | inline_declaration
-                     | "%destructor" param generic_symbol+
+                     | "%destructor" param (symbol | TAG)+
                          {
                            @grammar.add_destructor(
-                             ident_or_tags: val[2],
+                             ident_or_tags: val[2].flatten,
                              token_code: val[1],
                              lineno: val[1].line
                            )
                          }
-                     | "%printer" param generic_symbol+
+                     | "%printer" param (symbol | TAG)+
                          {
                            @grammar.add_printer(
-                             ident_or_tags: val[2],
+                             ident_or_tags: val[2].flatten,
                              token_code: val[1],
                              lineno: val[1].line
                            )
                          }
-                     | "%error-token" param generic_symbol+
+                     | "%error-token" param (symbol | TAG)+
                          {
                            @grammar.add_error_token(
-                             ident_or_tags: val[2],
+                             ident_or_tags: val[2].flatten,
                              token_code: val[1],
                              lineno: val[1].line
                            )
@@ -115,6 +115,18 @@ rule
                             }
                           }
                         }
+                    | "%nterm" symbol_declarations
+                        {
+                          val[1].each {|hash|
+                            hash[:tokens].each {|id|
+                              if @grammar.find_term_by_s_value(id.s_value)
+                                on_action_error("symbol #{id.s_value} redeclared as a nonterminal", id)
+                              else
+                                @grammar.add_type(id: id, tag: hash[:tag])
+                              end
+                            }
+                          }
+                        }
                     | "%left" token_declarations_for_precedence
                         {
                           val[1].each {|hash|
@@ -156,13 +168,7 @@ rule
                           @precedence_number += 1
                         }
 
-  token_declarations: token_declaration+
-                        {
-                          val[0].each {|token_declaration|
-                            @grammar.add_term(id: token_declaration[0], alias_name: token_declaration[2], token_id: token_declaration[1], tag: nil, replace: true)
-                          }
-                        }
-                    | TAG token_declaration+
+  token_declarations: TAG? token_declaration+
                         {
                           val[1].each {|token_declaration|
                             @grammar.add_term(id: token_declaration[0], alias_name: token_declaration[2], token_id: token_declaration[1], tag: val[0], replace: true)
@@ -208,7 +214,7 @@ rule
                   result = val[0].append(builder)
                 }
 
-  rule_rhs: empty
+  rule_rhs: "%empty"?
             {
               reset_precs
               result = Grammar::ParameterizingRule::Rhs.new
@@ -250,11 +256,16 @@ rule
               result = builder
             }
 
-  alias: # empty
-       | string_as_id { result = val[0].s_value }
+  alias: string_as_id? { result = val[0].s_value if val[0] }
 
-  symbol_declarations: symbol+ { result = [{tag: nil, tokens: val[0]}] }
-                     | TAG symbol+ { result = [{tag: val[0], tokens: val[1]}] }
+  symbol_declarations: TAG? symbol+
+                       {
+                          result = if val[0]
+                            [{tag: val[0], tokens: val[1]}]
+                          else
+                            [{tag: nil, tokens: val[1]}]
+                          end
+                       }
                      | symbol_declarations TAG symbol+ { result = val[0].append({tag: val[1], tokens: val[2]}) }
 
   symbol: id
@@ -311,7 +322,7 @@ rule
                 result = val[0].append(builder)
               }
 
-  rhs: empty
+  rhs: "%empty"?
          {
            reset_precs
            result = @grammar.create_rule_builder(@rule_counter, @midrule_action_counter)
@@ -362,9 +373,15 @@ rule
                        | "+" { result = "nonempty_list" }
                        | "*" { result = "list" }
 
-  parameterizing_args: symbol { result = [val[0]] }
+  parameterizing_args: symbol parameterizing_suffix?
+                          {
+                            result = if val[1]
+                              [Lrama::Lexer::Token::InstantiateRule.new(s_value: val[1].s_value, location: @lexer.location, args: val[0])]
+                            else
+                              [val[0]]
+                            end
+                          }
                      | parameterizing_args ',' symbol { result = val[0].append(val[2]) }
-                     | symbol parameterizing_suffix { result = [Lrama::Lexer::Token::InstantiateRule.new(s_value: val[1].s_value, location: @lexer.location, args: val[0])] }
                      | IDENTIFIER "(" parameterizing_args ")" { result = [Lrama::Lexer::Token::InstantiateRule.new(s_value: val[0].s_value, location: @lexer.location, args: val[2])] }
 
   midrule_action: "{"
@@ -404,12 +421,6 @@ rule
        | STRING
        | "{...}"
 
-  generic_symbol: symbol
-                | TAG
-
-  empty: /* empty */
-       | "%empty"
-
   string_as_id: STRING { result = Lrama::Lexer::Token::Ident.new(s_value: val[0]) }
 end
 
@@ -417,17 +428,18 @@ end
 
 include Lrama::Report::Duration
 
-def initialize(text, path, debug = false)
+def initialize(text, path, debug = false, define = {})
   @grammar_file = Lrama::Lexer::GrammarFile.new(path, text)
   @yydebug = debug
   @rule_counter = Lrama::Grammar::Counter.new(0)
   @midrule_action_counter = Lrama::Grammar::Counter.new(1)
+  @define = define
 end
 
 def parse
   report_duration(:parse) do
     @lexer = Lrama::Lexer.new(@grammar_file)
-    @grammar = Lrama::Grammar.new(@rule_counter)
+    @grammar = Lrama::Grammar.new(@rule_counter, @define)
     @precedence_number = 0
     reset_precs
     do_parse