lrama 0.6.9 → 0.6.11

data/lib/lrama/grammar/rule_builder.rb

@@ -1,12 +1,15 @@
+# frozen_string_literal: true
+
 module Lrama
   class Grammar
     class RuleBuilder
       attr_accessor :lhs, :line
       attr_reader :lhs_tag, :rhs, :user_code, :precedence_sym
 
-      def initialize(rule_counter, midrule_action_counter, position_in_original_rule_rhs = nil, lhs_tag: nil, skip_preprocess_references: false)
+      def initialize(rule_counter, midrule_action_counter, parameterizing_rule_resolver, position_in_original_rule_rhs = nil, lhs_tag: nil, skip_preprocess_references: false)
         @rule_counter = rule_counter
         @midrule_action_counter = midrule_action_counter
+        @parameterizing_rule_resolver = parameterizing_rule_resolver
         @position_in_original_rule_rhs = position_in_original_rule_rhs
         @skip_preprocess_references = skip_preprocess_references
 
@@ -19,16 +22,12 @@ module Lrama
         @rules = []
         @rule_builders_for_parameterizing_rules = []
         @rule_builders_for_derived_rules = []
-        @rule_builders_for_inline_rules = []
         @parameterizing_rules = []
-        @inline_rules = []
         @midrule_action_rules = []
       end
 
       def add_rhs(rhs)
-        if !@line
-          @line = rhs.line
-        end
+        @line ||= rhs.line
 
         flush_user_code
 
@@ -36,9 +35,7 @@ module Lrama
       end
 
       def user_code=(user_code)
-        if !@line
-          @line = user_code&.line
-        end
+        @line ||= user_code&.line
 
         flush_user_code
 
@@ -55,18 +52,41 @@ module Lrama
         freeze_rhs
       end
 
-      def setup_rules(parameterizing_rule_resolver)
+      def setup_rules
         preprocess_references unless @skip_preprocess_references
-        if rhs.any? { |token| parameterizing_rule_resolver.find_inline(token) }
-          resolve_inline(parameterizing_rule_resolver)
-        else
-          process_rhs(parameterizing_rule_resolver)
-        end
+        process_rhs
         build_rules
       end
 
       def rules
-        @parameterizing_rules + @inline_rules + @midrule_action_rules + @rules
+        @parameterizing_rules + @midrule_action_rules + @rules
+      end
+
+      def has_inline_rules?
+        rhs.any? { |token| @parameterizing_rule_resolver.find_inline(token) }
+      end
+
+      def resolve_inline_rules
+        resolved_builders = [] #: Array[RuleBuilder]
+        rhs.each_with_index do |token, i|
+          if (inline_rule = @parameterizing_rule_resolver.find_inline(token))
+            inline_rule.rhs_list.each do |inline_rhs|
+              rule_builder = RuleBuilder.new(@rule_counter, @midrule_action_counter, @parameterizing_rule_resolver, lhs_tag: lhs_tag)
+              if token.is_a?(Lexer::Token::InstantiateRule)
+                resolve_inline_rhs(rule_builder, inline_rhs, i, Binding.new(inline_rule, token.args))
+              else
+                resolve_inline_rhs(rule_builder, inline_rhs, i)
+              end
+              rule_builder.lhs = lhs
+              rule_builder.line = line
+              rule_builder.precedence_sym = precedence_sym
+              rule_builder.user_code = replace_inline_user_code(inline_rhs, i)
+              resolved_builders << rule_builder
+            end
+            break
+          end
+        end
+        resolved_builders
       end
 
       private
@@ -82,31 +102,25 @@ module Lrama
       def build_rules
         tokens = @replaced_rhs
 
-        if tokens
-          rule = Rule.new(
-            id: @rule_counter.increment, _lhs: lhs, _rhs: tokens, lhs_tag: lhs_tag, token_code: user_code,
-            position_in_original_rule_rhs: @position_in_original_rule_rhs, precedence_sym: precedence_sym, lineno: line
-          )
-          @rules = [rule]
-          @parameterizing_rules = @rule_builders_for_parameterizing_rules.map do |rule_builder|
-            rule_builder.rules
-          end.flatten
-          @midrule_action_rules = @rule_builders_for_derived_rules.map do |rule_builder|
-            rule_builder.rules
-          end.flatten
-          @midrule_action_rules.each do |r|
-            r.original_rule = rule
-          end
-        else
-          @inline_rules = @rule_builders_for_inline_rules.map do |rule_builder|
-            rule_builder.rules
-          end.flatten
+        rule = Rule.new(
+          id: @rule_counter.increment, _lhs: lhs, _rhs: tokens, lhs_tag: lhs_tag, token_code: user_code,
+          position_in_original_rule_rhs: @position_in_original_rule_rhs, precedence_sym: precedence_sym, lineno: line
+        )
+        @rules = [rule]
+        @parameterizing_rules = @rule_builders_for_parameterizing_rules.map do |rule_builder|
+          rule_builder.rules
+        end.flatten
+        @midrule_action_rules = @rule_builders_for_derived_rules.map do |rule_builder|
+          rule_builder.rules
+        end.flatten
+        @midrule_action_rules.each do |r|
+          r.original_rule = rule
         end
       end
 
       # rhs is a mixture of variety type of tokens like `Ident`, `InstantiateRule`, `UserCode` and so on.
       # `#process_rhs` replaces some kind of tokens to `Ident` so that all `@replaced_rhs` are `Ident` or `Char`.
-      def process_rhs(parameterizing_rule_resolver)
+      def process_rhs
         return if @replaced_rhs
 
         @replaced_rhs = []
@@ -118,26 +132,26 @@ module Lrama
           when Lrama::Lexer::Token::Ident
             @replaced_rhs << token
           when Lrama::Lexer::Token::InstantiateRule
-            parameterizing_rule = parameterizing_rule_resolver.find_rule(token)
+            parameterizing_rule = @parameterizing_rule_resolver.find_rule(token)
             raise "Unexpected token. #{token}" unless parameterizing_rule
 
             bindings = Binding.new(parameterizing_rule, token.args)
             lhs_s_value = lhs_s_value(token, bindings)
-            if (created_lhs = parameterizing_rule_resolver.created_lhs(lhs_s_value))
+            if (created_lhs = @parameterizing_rule_resolver.created_lhs(lhs_s_value))
               @replaced_rhs << created_lhs
             else
               lhs_token = Lrama::Lexer::Token::Ident.new(s_value: lhs_s_value, location: token.location)
               @replaced_rhs << lhs_token
-              parameterizing_rule_resolver.created_lhs_list << lhs_token
+              @parameterizing_rule_resolver.created_lhs_list << lhs_token
               parameterizing_rule.rhs_list.each do |r|
-                rule_builder = RuleBuilder.new(@rule_counter, @midrule_action_counter, lhs_tag: token.lhs_tag || parameterizing_rule.tag)
+                rule_builder = RuleBuilder.new(@rule_counter, @midrule_action_counter, @parameterizing_rule_resolver, lhs_tag: token.lhs_tag || parameterizing_rule.tag)
                 rule_builder.lhs = lhs_token
                 r.symbols.each { |sym| rule_builder.add_rhs(bindings.resolve_symbol(sym)) }
                 rule_builder.line = line
                 rule_builder.precedence_sym = r.precedence_sym
                 rule_builder.user_code = r.resolve_user_code(bindings)
                 rule_builder.complete_input
-                rule_builder.setup_rules(parameterizing_rule_resolver)
+                rule_builder.setup_rules
                 @rule_builders_for_parameterizing_rules << rule_builder
               end
             end
@@ -147,11 +161,11 @@ module Lrama
             new_token = Lrama::Lexer::Token::Ident.new(s_value: prefix + @midrule_action_counter.increment.to_s)
             @replaced_rhs << new_token
 
-            rule_builder = RuleBuilder.new(@rule_counter, @midrule_action_counter, i, lhs_tag: tag, skip_preprocess_references: true)
+            rule_builder = RuleBuilder.new(@rule_counter, @midrule_action_counter, @parameterizing_rule_resolver, i, lhs_tag: tag, skip_preprocess_references: true)
             rule_builder.lhs = new_token
             rule_builder.user_code = token
             rule_builder.complete_input
-            rule_builder.setup_rules(parameterizing_rule_resolver)
+            rule_builder.setup_rules
 
             @rule_builders_for_derived_rules << rule_builder
           else
@@ -172,27 +186,10 @@ module Lrama
         "#{token.rule_name}_#{s_values.join('_')}"
       end
 
-      def resolve_inline(parameterizing_rule_resolver)
-        rhs.each_with_index do |token, i|
-          if inline_rule = parameterizing_rule_resolver.find_inline(token)
-            inline_rule.rhs_list.each_with_index do |inline_rhs|
-              rule_builder = RuleBuilder.new(@rule_counter, @midrule_action_counter, lhs_tag: lhs_tag, skip_preprocess_references: true)
-              resolve_inline_rhs(rule_builder, inline_rhs, i)
-              rule_builder.lhs = lhs
-              rule_builder.line = line
-              rule_builder.user_code = replace_inline_user_code(inline_rhs, i)
-              rule_builder.complete_input
-              rule_builder.setup_rules(parameterizing_rule_resolver)
-              @rule_builders_for_inline_rules << rule_builder
-            end
-          end
-        end
-      end
-
-      def resolve_inline_rhs(rule_builder, inline_rhs, index)
+      def resolve_inline_rhs(rule_builder, inline_rhs, index, bindings = nil)
         rhs.each_with_index do |token, i|
           if index == i
-            inline_rhs.symbols.each { |sym| rule_builder.add_rhs(sym) }
+            inline_rhs.symbols.each { |sym| rule_builder.add_rhs(bindings.nil? ? sym : bindings.resolve_symbol(sym)) }
           else
             rule_builder.add_rhs(token)
           end
@@ -204,6 +201,11 @@ module Lrama
         return user_code if user_code.nil?
 
         code = user_code.s_value.gsub(/\$#{index + 1}/, inline_rhs.user_code.s_value)
+        user_code.references.each do |ref|
+          next if ref.index.nil? || ref.index <= index # nil is a case for `$$`
+          code = code.gsub(/\$#{ref.index}/, "$#{ref.index + (inline_rhs.symbols.count-1)}")
+          code = code.gsub(/@#{ref.index}/, "@#{ref.index + (inline_rhs.symbols.count-1)}")
+        end
         Lrama::Lexer::Token::UserCode.new(s_value: code, location: user_code.location)
       end
 
@@ -238,9 +240,6 @@ module Lrama
             end
 
             if ref.number
-              # TODO: When Inlining is implemented, for example, if `$1` is expanded to multiple RHS tokens,
-              #       `$2` needs to access `$2 + n` to actually access it. So, after the Inlining implementation,
-              #       it needs resolves from number to index.
               ref.index = ref.number
             end
 

data/lib/lrama/grammar/symbol.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # Symbol is both of nterm and term
 # `number` is both for nterm and term
 # `token_id` is tokentype for term, internal sequence number for nterm

data/lib/lrama/grammar/symbols/resolver.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Lrama
   class Grammar
     class Symbols
@@ -42,7 +44,9 @@ module Lrama
         end
 
         def add_nterm(id:, alias_name: nil, tag: nil)
-          return if find_symbol_by_id(id)
+          if (sym = find_symbol_by_id(id))
+            return sym
+          end
 
           @symbols = nil
           nterm = Symbol.new(
@@ -53,6 +57,10 @@ module Lrama
           nterm
         end
 
+        def find_term_by_s_value(s_value)
+          terms.find { |s| s.id.s_value == s_value }
+        end
+
         def find_symbol_by_s_value(s_value)
           symbols.find { |s| s.id.s_value == s_value }
         end

data/lib/lrama/grammar/symbols.rb

@@ -1 +1,3 @@
+# frozen_string_literal: true
+
 require_relative "symbols/resolver"

data/lib/lrama/grammar/type.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Lrama
   class Grammar
     class Type

data/lib/lrama/grammar/union.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Lrama
   class Grammar
     class Union < Struct.new(:code, :lineno, keyword_init: true)

data/lib/lrama/grammar.rb

@@ -1,43 +1,40 @@
+# frozen_string_literal: true
+
 require "forwardable"
-require "lrama/grammar/auxiliary"
-require "lrama/grammar/binding"
-require "lrama/grammar/code"
-require "lrama/grammar/counter"
-require "lrama/grammar/destructor"
-require "lrama/grammar/error_token"
-require "lrama/grammar/parameterizing_rule"
-require "lrama/grammar/percent_code"
-require "lrama/grammar/precedence"
-require "lrama/grammar/printer"
-require "lrama/grammar/reference"
-require "lrama/grammar/rule"
-require "lrama/grammar/rule_builder"
-require "lrama/grammar/symbol"
-require "lrama/grammar/symbols"
-require "lrama/grammar/type"
-require "lrama/grammar/union"
-require "lrama/lexer"
+require_relative "grammar/auxiliary"
+require_relative "grammar/binding"
+require_relative "grammar/code"
+require_relative "grammar/counter"
+require_relative "grammar/destructor"
+require_relative "grammar/error_token"
+require_relative "grammar/parameterizing_rule"
+require_relative "grammar/percent_code"
+require_relative "grammar/precedence"
+require_relative "grammar/printer"
+require_relative "grammar/reference"
+require_relative "grammar/rule"
+require_relative "grammar/rule_builder"
+require_relative "grammar/symbol"
+require_relative "grammar/symbols"
+require_relative "grammar/type"
+require_relative "grammar/union"
+require_relative "lexer"
 
 module Lrama
   # Grammar is the result of parsing an input grammar file
   class Grammar
     extend Forwardable
 
-    attr_reader :percent_codes, :eof_symbol, :error_symbol, :undef_symbol, :accept_symbol, :aux
-    attr_accessor :union, :expect,
-                  :printers, :error_tokens,
-                  :lex_param, :parse_param, :initial_action,
+    attr_reader :percent_codes, :eof_symbol, :error_symbol, :undef_symbol, :accept_symbol, :aux, :parameterizing_rule_resolver
+    attr_accessor :union, :expect, :printers, :error_tokens, :lex_param, :parse_param, :initial_action,
                   :after_shift, :before_reduce, :after_reduce, :after_shift_error_token, :after_pop_stack,
-                  :symbols_resolver, :types,
-                  :rules, :rule_builders,
-                  :sym_to_rules, :no_stdlib
+                  :symbols_resolver, :types, :rules, :rule_builders, :sym_to_rules, :no_stdlib, :locations
 
-    def_delegators "@symbols_resolver", :symbols, :nterms, :terms, :add_nterm, :add_term,
+    def_delegators "@symbols_resolver", :symbols, :nterms, :terms, :add_nterm, :add_term, :find_term_by_s_value,
                                         :find_symbol_by_number!, :find_symbol_by_id!, :token_to_symbol,
                                         :find_symbol_by_s_value!, :fill_symbol_number, :fill_nterm_type,
                                         :fill_printer, :fill_destructor, :fill_error_token, :sort_by_number!
 
-
     def initialize(rule_counter)
       @rule_counter = rule_counter
 
@@ -59,10 +56,15 @@ module Lrama
       @accept_symbol = nil
       @aux = Auxiliary.new
       @no_stdlib = false
+      @locations = false
 
       append_special_symbols
     end
 
+    def create_rule_builder(rule_counter, midrule_action_counter)
+      RuleBuilder.new(rule_counter, midrule_action_counter, @parameterizing_rule_resolver)
+    end
+
     def add_percent_code(id:, code:)
       @percent_codes << PercentCode.new(id.s_value, code.s_value)
     end
@@ -141,6 +143,7 @@ module Lrama
     end
 
     def prepare
+      resolve_inline_rules
       normalize_rules
       collect_symbols
       set_lhs_and_rhs
@@ -149,6 +152,7 @@ module Lrama
       fill_sym_to_rules
       compute_nullable
       compute_first_set
+      set_locations
     end
 
     # TODO: More validation methods
@@ -255,7 +259,7 @@ module Lrama
 
     def setup_rules
       @rule_builders.each do |builder|
-        builder.setup_rules(@parameterizing_rule_resolver)
+        builder.setup_rules
       end
     end
 
@@ -289,10 +293,23 @@ module Lrama
       @accept_symbol = term
     end
 
+    def resolve_inline_rules
+      while @rule_builders.any? {|r| r.has_inline_rules? } do
+        @rule_builders = @rule_builders.flat_map do |builder|
+          if builder.has_inline_rules?
+            builder.resolve_inline_rules
+          else
+            builder
+          end
+        end
+      end
+    end
+
     def normalize_rules
       # Add $accept rule to the top of rules
-      lineno = @rule_builders.first ? @rule_builders.first.line : 0
-      @rules << Rule.new(id: @rule_counter.increment, _lhs: @accept_symbol.id, _rhs: [@rule_builders.first.lhs, @eof_symbol.id], token_code: nil, lineno: lineno)
+      rule_builder = @rule_builders.first # : RuleBuilder
+      lineno = rule_builder ? rule_builder.line : 0
+      @rules << Rule.new(id: @rule_counter.increment, _lhs: @accept_symbol.id, _rhs: [rule_builder.lhs, @eof_symbol.id], token_code: nil, lineno: lineno)
 
       setup_rules
 
@@ -365,17 +382,21 @@ module Lrama
     end
 
     def validate_rule_lhs_is_nterm!
-      errors = []
+      errors = [] #: Array[String]
 
       rules.each do |rule|
         next if rule.lhs.nterm?
 
-        errors << "[BUG] LHS of #{rule} (line: #{rule.lineno}) is term. It should be nterm."
+        errors << "[BUG] LHS of #{rule.display_name} (line: #{rule.lineno}) is term. It should be nterm."
       end
 
       return if errors.empty?
 
       raise errors.join("\n")
     end
+
+    def set_locations
+      @locations = @locations || @rules.any? {|rule| rule.contains_at_reference? }
+    end
   end
 end

data/lib/lrama/grammar_validator.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Lrama
+  class GrammarValidator
+    def initialize(grammar, states, logger)
+      @grammar = grammar
+      @states = states
+      @logger = logger
+    end
+
+    def valid?
+      conflicts_within_threshold?
+    end
+
+    private
+
+    def conflicts_within_threshold?
+      return true unless @grammar.expect
+
+      [sr_conflicts_within_threshold(@grammar.expect), rr_conflicts_within_threshold(0)].all?
+    end
+
+    def sr_conflicts_within_threshold(expected)
+      return true if expected == @states.sr_conflicts_count
+
+      @logger.error("shift/reduce conflicts: #{@states.sr_conflicts_count} found, #{expected} expected")
+      false
+    end
+
+    def rr_conflicts_within_threshold(expected)
+      return true if expected == @states.rr_conflicts_count
+
+      @logger.error("reduce/reduce conflicts: #{@states.rr_conflicts_count} found, #{expected} expected")
+      false
+    end
+  end
+end

data/lib/lrama/lexer/grammar_file.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Lrama
   class Lexer
     class GrammarFile

data/lib/lrama/lexer/location.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Lrama
   class Lexer
     class Location

data/lib/lrama/lexer/token/char.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Lrama
   class Lexer
     class Token

data/lib/lrama/lexer/token/ident.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Lrama
   class Lexer
     class Token

data/lib/lrama/lexer/token/instantiate_rule.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Lrama
   class Lexer
     class Token

data/lib/lrama/lexer/token/tag.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Lrama
   class Lexer
     class Token

data/lib/lrama/lexer/token/user_code.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "strscan"
 
 module Lrama
@@ -14,9 +16,9 @@ module Lrama
 
         def _references
           scanner = StringScanner.new(s_value)
-          references = []
+          references = [] #: Array[Grammar::Reference]
 
-          while !scanner.eos? do
+          until scanner.eos? do
             case
             when reference = scan_reference(scanner)
               references << reference

data/lib/lrama/lexer/token.rb

@@ -1,8 +1,10 @@
-require 'lrama/lexer/token/char'
-require 'lrama/lexer/token/ident'
-require 'lrama/lexer/token/instantiate_rule'
-require 'lrama/lexer/token/tag'
-require 'lrama/lexer/token/user_code'
+# frozen_string_literal: true
+
+require_relative 'token/char'
+require_relative 'token/ident'
+require_relative 'token/instantiate_rule'
+require_relative 'token/tag'
+require_relative 'token/user_code'
 
 module Lrama
   class Lexer

data/lib/lrama/lexer.rb

@@ -1,19 +1,22 @@
+# frozen_string_literal: true
+
 require "strscan"
 
-require "lrama/lexer/grammar_file"
-require "lrama/lexer/location"
-require "lrama/lexer/token"
+require_relative "lexer/grammar_file"
+require_relative "lexer/location"
+require_relative "lexer/token"
 
 module Lrama
   class Lexer
     attr_reader :head_line, :head_column, :line
     attr_accessor :status, :end_symbol
 
-    SYMBOLS = ['%{', '%}', '%%', '{', '}', '\[', '\]', '\(', '\)', '\,', ':', '\|', ';']
+    SYMBOLS = ['%{', '%}', '%%', '{', '}', '\[', '\]', '\(', '\)', '\,', ':', '\|', ';'].freeze
     PERCENT_TOKENS = %w(
       %union
       %token
       %type
+      %nterm
       %left
       %right
       %nonassoc
@@ -38,7 +41,8 @@ module Lrama
       %rule
       %no-stdlib
       %inline
-    )
+      %locations
+    ).freeze
 
     def initialize(grammar_file)
       @grammar_file = grammar_file
@@ -71,7 +75,7 @@ module Lrama
     end
 
     def lex_token
-      while !@scanner.eos? do
+      until @scanner.eos? do
         case
         when @scanner.scan(/\n/)
           newline
@@ -126,7 +130,7 @@ module Lrama
       code = ''
       reset_first_position
 
-      while !@scanner.eos? do
+      until @scanner.eos? do
         case
         when @scanner.scan(/{/)
           code += @scanner.matched
@@ -163,7 +167,7 @@ module Lrama
     private
 
     def lex_comment
-      while !@scanner.eos? do
+      until @scanner.eos? do
         case
         when @scanner.scan(/\n/)
           newline

data/lib/lrama/{warning.rb → logger.rb}

@@ -1,25 +1,17 @@
-module Lrama
-  class Warning
-    attr_reader :errors, :warns
+# frozen_string_literal: true
 
+module Lrama
+  class Logger
     def initialize(out = STDERR)
       @out = out
-      @errors = []
-      @warns = []
-    end
-
-    def error(message)
-      @out << message << "\n"
-      @errors << message
     end
 
     def warn(message)
       @out << message << "\n"
-      @warns << message
     end
 
-    def has_error?
-      !@errors.empty?
+    def error(message)
+      @out << message << "\n"
     end
   end
 end