lrama 0.5.12 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '008f1a5df39b115123be84cb5e1edd379d88458bc95256b0e4fe4659481ceb51'
-  data.tar.gz: 02af30da8cf4cb2cad841f43b603433edafab09b0cf0e8993201c2e9aea12029
+  metadata.gz: 7b1a3e3181fc7060235e167ea97a92809b31aa5b74ef5978ab70415252bb87e3
+  data.tar.gz: 51a6ee0620321936fcd9d81c44ece3f9e444b977dc8fa940ddf920a5bb3b1dad
 SHA512:
-  metadata.gz: 12cf8b98035052e89f9060dd73ec2b32ad9c5bec2fe4076e6ee0416832039962cf973c7ca895b482f2f434500648638c01a717d67357aa055eccce851995adaf
-  data.tar.gz: a4cffc85bb5945a3d4d037c0eadee0df22e30236085e01f06bffe1646d0acafd09fdb606860752cdea9dbfdf45baa1fc8d244504b7a0ca29c37ab60f9a727367
+  metadata.gz: f2b8805ad79cc0c3f125eba2715d274387191e6a355910d11c88f918eae4906b9398444162210c7ce955b1ba3f67cabf3b0679569a65d63bdc52328f2d7dc2f7
+  data.tar.gz: 93de8cc0c7f9154f37825d59010a4850cc81b8a66f460d70f95aed44c0eb1224e583d1afb95155be2d9ae063d286a4fa719c133c993c6ec9bfba1c490dfc997b

data/.github/workflows/test.yaml

@@ -13,7 +13,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: ['head', '3.2', '3.1', '3.0', '2.7', '2.6', '2.5']
+        ruby: ['head', '3.3', '3.2', '3.1', '3.0', '2.7', '2.6', '2.5']
     steps:
       - uses: actions/checkout@v4
       - uses: ruby/setup-ruby@v1

data/Gemfile

@@ -12,6 +12,6 @@ gem "stackprof", platforms: [:ruby] # stackprof doesn't support Windows
 # Recent steep requires Ruby >= 3.0.0.
 # Then skip install on some CI jobs.
 if !ENV['GITHUB_ACTION'] || ENV['INSTALL_STEEP'] == 'true'
-  gem "rbs", "3.3.0", require: false
+  gem "rbs", "3.4.0", require: false
   gem "steep", "1.6.0", require: false
 end

data/NEWS.md

@@ -1,5 +1,25 @@
 # NEWS for Lrama
 
+## Lrama 0.6.0 (2023-12-25)
+
+### User defined parameterizing rules
+
+Allow to define parameterizing rule by `%rule` directive.
+
+```
+%rule pair(X, Y): X Y { $$ = $1 + $2; }
+                ;
+
+%%
+
+program: stmt
+       ;
+
+stmt: pair(ODD, EVEN) <num>
+    | pair(EVEN, ODD) <num>
+    ;
+```
+
 ## Lrama 0.5.11 (2023-12-02)
 
 ### Type specification of parameterizing rules

data/README.md

@@ -49,9 +49,13 @@ Enter the formula:
 
 ## Versions and Branches
 
-### v0_5 (`master` branch)
+### v0_6 (`master` branch)
 
-This branch is for Ruby 3.3. `lrama_0_5` branch is created from this branch, once Ruby 3.3 is released.
+This branch is for Ruby 3.4. `lrama_0_6` branch is created from this branch, once Ruby 3.4 is released.
+
+### v0_5 (`lrama_0_5` branch)
+
+This branch is for Ruby 3.3.
 
 ### v0_4 (`lrama_0_4` branch)
 
@@ -91,6 +95,13 @@ $ bundle exec rbs collection install
 $ bundle exec steep check
 ```
 
+Running both of them:
+
+```shell
+$ bundle install
+$ bundle exec rake
+```
+
 ### Profiling Lrama
 
 #### 1. Create parse.tmp.y in ruby/ruby
@@ -136,7 +147,7 @@ $ stackprof --d3-flamegraph tmp/stackprof-cpu-myapp.dump > tmp/flamegraph.html
 
 ## Release flow
 
-1. Update `Lrama::VERSION`
+1. Update `Lrama::VERSION` and NEWS.md
 2. Release as a gem by `rake release`
 3. Update Lrama in ruby/ruby by `cp -r LEGAL.md NEWS.md MIT exe lib template ruby/tool/lrama`
 4. Create new release on [GitHub](https://github.com/ruby/lrama/releases)

data/Steepfile

@@ -8,6 +8,9 @@ target :lib do
   check "lib/lrama/grammar/code.rb"
   check "lib/lrama/grammar/counter.rb"
   check "lib/lrama/grammar/error_token.rb"
+  check "lib/lrama/grammar/parameterizing_rule_builder.rb"
+  check "lib/lrama/grammar/parameterizing_rule_resolver.rb"
+  check "lib/lrama/grammar/parameterizing_rule_rhs_builder.rb"
   check "lib/lrama/grammar/parameterizing_rules"
   check "lib/lrama/grammar/percent_code.rb"
   check "lib/lrama/grammar/precedence.rb"

data/lib/lrama/grammar/code/printer_code.rb

@@ -2,7 +2,7 @@ module Lrama
   class Grammar
     class Code
       class PrinterCode < Code
-        def initialize(type: nil, token_code: nil, tag: nil)
+        def initialize(type:, token_code:, tag:)
           super(type: type, token_code: token_code)
           @tag = tag
         end

data/lib/lrama/grammar/code/rule_action.rb

@@ -2,7 +2,7 @@ module Lrama
   class Grammar
     class Code
       class RuleAction < Code
-        def initialize(type: nil, token_code: nil, rule: nil)
+        def initialize(type:, token_code:, rule:)
           super(type: type, token_code: token_code)
           @rule = rule
         end

data/lib/lrama/grammar/code.rb

@@ -1,12 +1,29 @@
 require "forwardable"
+require "lrama/grammar/code/initial_action_code"
+require "lrama/grammar/code/no_reference_code"
+require "lrama/grammar/code/printer_code"
+require "lrama/grammar/code/rule_action"
 
 module Lrama
   class Grammar
-    class Code < Struct.new(:type, :token_code, keyword_init: true)
+    class Code
       extend Forwardable
 
       def_delegators "token_code", :s_value, :line, :column, :references
 
+      attr_reader :type, :token_code
+
+      def initialize(type:, token_code:)
+        @type = type
+        @token_code = token_code
+      end
+
+      def ==(other)
+        self.class == other.class &&
+        self.type == other.type &&
+        self.token_code == other.token_code
+      end
+
       # $$, $n, @$, @n are translated to C code
       def translated_code
         t_code = s_value.dup
@@ -17,7 +34,7 @@ module Lrama
 
           str = reference_to_c(ref)
 
-          t_code[first_column..last_column] = str
+          t_code[first_column...last_column] = str
         end
 
         return t_code
@@ -31,8 +48,3 @@ module Lrama
     end
   end
 end
-
-require "lrama/grammar/code/initial_action_code"
-require "lrama/grammar/code/no_reference_code"
-require "lrama/grammar/code/printer_code"
-require "lrama/grammar/code/rule_action"

data/lib/lrama/grammar/parameterizing_rule.rb

@@ -0,0 +1,6 @@
+module Lrama
+  class Grammar
+    class ParameterizingRule < Struct.new(:rules, :token, keyword_init: true)
+    end
+  end
+end

data/lib/lrama/grammar/parameterizing_rule_builder.rb

@@ -0,0 +1,34 @@
+module Lrama
+  class Grammar
+    class ParameterizingRuleBuilder
+      attr_reader :name, :parameters, :rhs
+
+      def initialize(name, parameters, rhs)
+        @name = name
+        @parameters = parameters
+        @rhs = rhs
+        @required_parameters_count = parameters.count
+      end
+
+      def build_rules(token, actual_args, rule_counter, lhs_tag, line, rule_builders)
+        validate_argument_number!(token)
+        lhs = lhs(actual_args)
+        @rhs.map do |rhs|
+          rhs.build_rules(token, actual_args, parameters, rule_counter, lhs, lhs_tag, line, rule_builders)
+        end.flatten
+      end
+
+      private
+
+      def validate_argument_number!(token)
+        unless @required_parameters_count == token.args.count
+          raise "Invalid number of arguments. expect: #{@required_parameters_count} actual: #{token.args.count}"
+        end
+      end
+
+      def lhs(actual_args)
+        Lrama::Lexer::Token::Ident.new(s_value: "#{name}_#{actual_args.map(&:s_value).join('_')}")
+      end
+    end
+  end
+end

data/lib/lrama/grammar/parameterizing_rule_resolver.rb

@@ -0,0 +1,30 @@
+module Lrama
+  class Grammar
+    class ParameterizingRuleResolver
+      def initialize
+        @parameterizing_rule_builders = []
+      end
+
+      def add_parameterizing_rule_builder(builder)
+        @parameterizing_rule_builders << builder
+      end
+
+      def defined?(name)
+        !rule_builders(name).empty?
+      end
+
+      def build_rules(token, rule_counter, lhs_tag, line)
+        builder = rule_builders(token.s_value).last
+        raise "Unknown parameterizing rule #{token.s_value} at line #{token.line}" unless builder
+
+        builder.build_rules(token, token.args, rule_counter, lhs_tag, line, @parameterizing_rule_builders)
+      end
+
+      private
+
+      def rule_builders(name)
+        @parameterizing_rule_builders.select { |builder| builder.name == name }
+      end
+    end
+  end
+end

data/lib/lrama/grammar/parameterizing_rule_rhs_builder.rb

@@ -0,0 +1,53 @@
+module Lrama
+  class Grammar
+    class ParameterizingRuleRhsBuilder
+      attr_accessor :symbols, :user_code, :precedence_sym
+
+      def initialize
+        @symbols = []
+        @user_code = nil
+        @precedence_sym = nil
+      end
+
+      def build_rules(token, actual_args, parameters, rule_counter, lhs, lhs_tag, line, rule_builders)
+        nested_rules = build_nested_rules(token, actual_args, parameters, rule_counter, lhs_tag, line, rule_builders)
+        rule = Rule.new(id: rule_counter.increment, _lhs: lhs, _rhs: rhs(token, actual_args, parameters, nested_rules.last), lhs_tag: lhs_tag, token_code: user_code, precedence_sym: precedence_sym, lineno: line)
+        ParameterizingRule.new(rules: nested_rules.map(&:rules) + [rule], token: lhs)
+      end
+
+      private
+
+      def build_nested_rules(token, actual_args, parameters, rule_counter, lhs_tag, line, rule_builders)
+        symbols.each_with_index.map do |sym, i|
+          next unless sym.is_a?(Lexer::Token::InstantiateRule)
+
+          builder = rule_builders.select { |builder| builder.name == sym.s_value }.last
+          raise "Unknown parameterizing rule #{token.s_value} at line #{token.line}" unless builder
+
+          builder.build_rules(sym, nested_actual_args(actual_args, parameters, i), rule_counter, lhs_tag, line, rule_builders)
+        end.flatten.compact
+      end
+
+      def nested_actual_args(actual_args, parameters, idx)
+        symbols[idx].args.map do |arg|
+          i = parameters.index { |parameter| parameter.s_value == arg.s_value }
+          i.nil? ? arg : actual_args[i]
+        end
+      end
+
+      def rhs(token, actual_args, parameters, nested_rule)
+        symbols.map do |sym|
+          if sym.is_a?(Lexer::Token::InstantiateRule)
+            sym.args.map do |arg|
+              idx = parameters.index { |parameter| parameter.s_value == arg.s_value }
+              idx.nil? ? sym : nested_rule&.token
+            end
+          else
+            idx = parameters.index { |parameter| parameter.s_value == sym.s_value }
+            idx.nil? ? sym : actual_args[idx]
+          end
+        end.flatten
+      end
+    end
+  end
+end

data/lib/lrama/grammar/rule_builder.rb

@@ -51,22 +51,14 @@ module Lrama
         freeze_rhs
       end
 
-      def setup_rules
+      def setup_rules(parameterizing_resolver)
         preprocess_references unless @skip_preprocess_references
-        process_rhs
+        process_rhs(parameterizing_resolver)
         build_rules
       end
 
-      def parameterizing_rules
-        @parameterizing_rules
-      end
-
-      def midrule_action_rules
-        @midrule_action_rules
-      end
-
       def rules
-        @rules
+        @parameterizing_rules + @midrule_action_rules + @rules
       end
 
       private
@@ -95,9 +87,9 @@ module Lrama
         end
       end
 
-      # rhs is a mixture of variety type of tokens like `Ident`, `Parameterizing`, `UserCode` and so on.
+      # rhs is a mixture of variety type of tokens like `Ident`, `InstantiateRule`, `UserCode` and so on.
       # `#process_rhs` replaces some kind of tokens to `Ident` so that all `@replaced_rhs` are `Ident` or `Char`.
-      def process_rhs
+      def process_rhs(parameterizing_resolver)
         return if @replaced_rhs
 
         @replaced_rhs = []
@@ -109,12 +101,17 @@ module Lrama
             @replaced_rhs << token
           when Lrama::Lexer::Token::Ident
             @replaced_rhs << token
-          when Lrama::Lexer::Token::Parameterizing
-            parameterizing = ParameterizingRules::Builder.new(token, @rule_counter, @lhs_tag, user_code, precedence_sym, line)
-            parameterizing.build.each do |r|
-              @parameterizing_rules << r
+          when Lrama::Lexer::Token::InstantiateRule
+            if parameterizing_resolver.defined?(token.rule_name)
+              parameterizing = parameterizing_resolver.build_rules(token, @rule_counter, @lhs_tag, line)
+              @parameterizing_rules = @parameterizing_rules + parameterizing.map(&:rules).flatten
+              @replaced_rhs = @replaced_rhs + parameterizing.map(&:token).flatten.uniq
+            else
+              # TODO: Delete when the standard library will defined as a grammar file.
+              parameterizing = ParameterizingRules::Builder.new(token, @rule_counter, @lhs_tag, user_code, precedence_sym, line)
+              @parameterizing_rules = @parameterizing_rules + parameterizing.build
+              @replaced_rhs << parameterizing.build_token
             end
-            @replaced_rhs << parameterizing.build_token
           when Lrama::Lexer::Token::UserCode
             prefix = token.referred ? "@" : "$@"
             new_token = Lrama::Lexer::Token::Ident.new(s_value: prefix + @midrule_action_counter.increment.to_s)
@@ -124,7 +121,7 @@ module Lrama
             rule_builder.lhs = new_token
             rule_builder.user_code = token
             rule_builder.complete_input
-            rule_builder.setup_rules
+            rule_builder.setup_rules(parameterizing_resolver)
 
             @rule_builders_for_derived_rules << rule_builder
           else
@@ -146,8 +143,15 @@ module Lrama
               else
                 candidates = rhs.each_with_index.select {|token, i| token.referred_by?(ref_name) }
 
-                raise "Referring symbol `#{ref_name}` is duplicated. #{token}" if candidates.size >= 2
-                raise "Referring symbol `#{ref_name}` is not found. #{token}" unless referring_symbol = candidates.first
+                if candidates.size >= 2
+                  location = token.location.partial_location(ref.first_column, ref.last_column)
+                  raise location.generate_error_message("Referring symbol `#{ref_name}` is duplicated.")
+                end
+
+                unless (referring_symbol = candidates.first)
+                  location = token.location.partial_location(ref.first_column, ref.last_column)
+                  raise location.generate_error_message("Referring symbol `#{ref_name}` is not found.")
+                end
 
                 ref.index = referring_symbol[1] + 1
               end
@@ -167,7 +171,7 @@ module Lrama
       end
 
       def flush_user_code
-        if c = @user_code
+        if (c = @user_code)
           @rhs << c
           @user_code = nil
         end

data/lib/lrama/grammar.rb

@@ -8,6 +8,10 @@ require "lrama/grammar/printer"
 require "lrama/grammar/reference"
 require "lrama/grammar/rule"
 require "lrama/grammar/rule_builder"
+require "lrama/grammar/parameterizing_rule_builder"
+require "lrama/grammar/parameterizing_rule_resolver"
+require "lrama/grammar/parameterizing_rule_rhs_builder"
+require "lrama/grammar/parameterizing_rule"
 require "lrama/grammar/symbol"
 require "lrama/grammar/type"
 require "lrama/grammar/union"
@@ -36,6 +40,7 @@ module Lrama
       @rule_builders = []
       @rules = []
       @sym_to_rules = {}
+      @parameterizing_resolver = ParameterizingRuleResolver.new
       @empty_symbol = nil
       @eof_symbol = nil
       @error_symbol = nil
@@ -69,7 +74,7 @@ module Lrama
         return sym
       end
 
-      if sym = @symbols.find {|s| s.id == id }
+      if (sym = @symbols.find {|s| s.id == id })
         return sym
       end
 
@@ -129,6 +134,10 @@ module Lrama
       @rule_builders << builder
     end
 
+    def add_parameterizing_rule_builder(builder)
+      @parameterizing_resolver.add_parameterizing_rule_builder(builder)
+    end
+
     def prologue_first_lineno=(prologue_first_lineno)
       @aux.prologue_first_lineno = prologue_first_lineno
     end
@@ -310,7 +319,7 @@ module Lrama
 
     def setup_rules
       @rule_builders.each do |builder|
-        builder.setup_rules
+        builder.setup_rules(@parameterizing_resolver)
       end
     end
 
@@ -350,56 +359,21 @@ module Lrama
       @accept_symbol = term
     end
 
-    # 1. Add $accept rule to the top of rules
-    # 2. Extract action in the middle of RHS into new Empty rule
-    # 3. Append id and extract action then create Rule
-    #
-    # Bison 3.8.2 uses different orders for symbol number and rule number
-    # when a rule has actions in the middle of a rule.
-    #
-    # For example,
-    #
-    # `program: $@1 top_compstmt`
-    #
-    # Rules are ordered like below,
-    #
-    # 1 $@1: ε
-    # 2 program: $@1 top_compstmt
-    #
-    # Symbols are ordered like below,
-    #
-    # 164 program
-    # 165 $@1
-    #
     def normalize_rules
-      # 1. Add $accept rule to the top of rules
-      accept = @accept_symbol
-      eof = @eof_symbol
+      # Add $accept rule to the top of rules
       lineno = @rule_builders.first ? @rule_builders.first.line : 0
-      @rules << Rule.new(id: @rule_counter.increment, _lhs: accept.id, _rhs: [@rule_builders.first.lhs, eof.id], token_code: nil, lineno: lineno)
+      @rules << Rule.new(id: @rule_counter.increment, _lhs: @accept_symbol.id, _rhs: [@rule_builders.first.lhs, @eof_symbol.id], token_code: nil, lineno: lineno)
 
       setup_rules
 
       @rule_builders.each do |builder|
-        # Extract actions in the middle of RHS into new rules.
-        builder.midrule_action_rules.each do |rule|
-          @rules << rule
-        end
-
         builder.rules.each do |rule|
-          add_nterm(id: rule._lhs)
-          @rules << rule
-        end
-
-        builder.parameterizing_rules.each do |rule|
           add_nterm(id: rule._lhs, tag: rule.lhs_tag)
           @rules << rule
         end
-
-        builder.midrule_action_rules.each do |rule|
-          add_nterm(id: rule._lhs)
-        end
       end
+
+      @rules.sort_by!(&:id)
     end
 
     # Collect symbols from rules

data/lib/lrama/lexer/grammar_file.rb

@@ -0,0 +1,21 @@
+module Lrama
+  class Lexer
+    class GrammarFile
+      attr_reader :path, :text
+
+      def initialize(path, text)
+        @path = path
+        @text = text
+      end
+
+      def ==(other)
+        self.class == other.class &&
+        self.path == other.path
+      end
+
+      def lines
+        @lines ||= text.split("\n")
+      end
+    end
+  end
+end

data/lib/lrama/lexer/location.rb

@@ -1,9 +1,10 @@
 module Lrama
   class Lexer
     class Location
-      attr_reader :first_line, :first_column, :last_line, :last_column
+      attr_reader :grammar_file, :first_line, :first_column, :last_line, :last_column
 
-      def initialize(first_line:, first_column:, last_line:, last_column:)
+      def initialize(grammar_file:, first_line:, first_column:, last_line:, last_column:)
+        @grammar_file = grammar_file
         @first_line = first_line
         @first_column = first_column
         @last_line = last_line
@@ -12,11 +13,85 @@ module Lrama
 
       def ==(other)
         self.class == other.class &&
+        self.grammar_file == other.grammar_file &&
         self.first_line == other.first_line &&
         self.first_column == other.first_column &&
         self.last_line == other.last_line &&
         self.last_column == other.last_column
       end
+
+      def partial_location(left, right)
+        offset = -first_column
+        new_first_line = -1
+        new_first_column = -1
+        new_last_line = -1
+        new_last_column = -1
+
+        _text.each.with_index do |line, index|
+          new_offset = offset + line.length + 1
+
+          if offset <= left && left <= new_offset
+            new_first_line = first_line + index
+            new_first_column = left - offset
+          end
+
+          if offset <= right && right <= new_offset
+            new_last_line = first_line + index
+            new_last_column = right - offset
+          end
+
+          offset = new_offset
+        end
+
+        Location.new(
+          grammar_file: grammar_file,
+          first_line: new_first_line, first_column: new_first_column,
+          last_line: new_last_line, last_column: new_last_column
+        )
+      end
+
+      def to_s
+        "#{path} (#{first_line},#{first_column})-(#{last_line},#{last_column})"
+      end
+
+      def generate_error_message(error_message)
+        <<~ERROR.chomp
+          #{path}:#{first_line}:#{first_column}: #{error_message}
+          #{line_with_carets}
+        ERROR
+      end
+
+      def line_with_carets
+        <<~TEXT
+          #{text}
+          #{carets}
+        TEXT
+      end
+
+      private
+
+      def path
+        grammar_file.path
+      end
+
+      def blanks
+        (text[0...first_column] or raise "#{first_column} is invalid").gsub(/[^\t]/, ' ')
+      end
+
+      def carets
+        blanks + '^' * (last_column - first_column)
+      end
+
+      def text
+        @text ||= _text.join("\n")
+      end
+
+      def _text
+        @_text ||=begin
+          range = (first_line - 1)...last_line
+          grammar_file.lines[range] or raise "#{range} is invalid"
+        end
+      end
     end
   end
 end

data/lib/lrama/lexer/token/instantiate_rule.rb

@@ -0,0 +1,18 @@
+module Lrama
+  class Lexer
+    class Token
+      class InstantiateRule < Token
+        attr_accessor :args
+
+        def initialize(s_value:, alias_name: nil, location: nil, args: [])
+          super s_value: s_value, alias_name: alias_name, location: location
+          @args = args
+        end
+
+        def rule_name
+          s_value
+        end
+      end
+    end
+  end
+end