ruby_parser 3.19.2 → 3.20.1

data/lib/ruby3_parser.yy

@@ -4,6 +4,8 @@
 class Ruby30Parser
 #elif V == 31
 class Ruby31Parser
+#elif V == 32
+class Ruby32Parser
 #else
 fail "version not specified or supported on code generation"
 #endif
@@ -227,6 +229,7 @@ rule
                 | lhs tEQL mrhs
                     {
                       lhs, _, rhs = val
+
                       result = new_assign lhs, s(:svalue, rhs).line(rhs.line)
                     }
                 | mlhs tEQL mrhs_arg kRESCUE_MOD stmt
@@ -1146,8 +1149,10 @@ rule
 
       paren_args: tLPAREN2 opt_call_args rparen
                     {
-                      _, args, _ = val
+                      _, args, (_, line_max) = val
+
                       result = args
+                      result.line_max = line_max if args
                     }
                 | tLPAREN2 args tCOMMA args_forward rparen
                     {
@@ -1377,12 +1382,14 @@ rule
                     {
                       result = wrap :colon3, val[1]
                     }
-                | tLBRACK { result = lexer.lineno } aref_args tRBRACK
+                | tLBRACK { result = lexer.lineno } aref_args rbracket
                     {
-                      _, line, args, _ = val
+                      _, line, args, (_, line_max) = val
+
                       result = args || s(:array)
                       result.sexp_type = :array # aref_args is :args
                       result.line line
+                      result.line_max = line_max
                     }
                 | tLBRACE
                     {
@@ -1973,13 +1980,19 @@ opt_block_args_tail: tCOMMA block_args_tail
                     {
                       call, args = val
 
-                      result = call.concat args.sexp_body if args
+                      result = call
+
+                      if args then
+                        call.concat args.sexp_body
+                        result.line_max = args.line_max
+                      end
                     }
                 | primary_value call_op operation2 opt_paren_args
                     {
-                      recv, call_op, (op, _line), args = val
+                      recv, call_op, (op, op_line), args = val
 
                       result = new_call recv, op.to_sym, args, call_op
+                      result.line_max = op_line unless args
                     }
                 | primary_value tCOLON2 operation2 paren_args
                     {
@@ -2749,15 +2762,17 @@ opt_block_args_tail: tCOMMA block_args_tail
 
            words: tWORDS_BEG tSPACE tSTRING_END
                     {
-                      (_, line), _, _ = val
+                      (_, line), _, (_, line_max) = val
 
                       result = s(:array).line line
+                      result.line_max = line_max
                     }
                 | tWORDS_BEG word_list tSTRING_END
                     {
-                      (_, line), list, _ = val
+                      (_, line), list, (_, line_max) = val
 
                       result = list.line line
+                      result.line_max = line_max
                     }
 
        word_list: none
@@ -2777,15 +2792,17 @@ opt_block_args_tail: tCOMMA block_args_tail
 
          symbols: tSYMBOLS_BEG tSPACE tSTRING_END
                     {
-                      (_, line), _, _ = val
+                      (_, line), _, (_, line_max) = val
 
                       result = s(:array).line line
+                      result.line_max = line_max
                     }
                 | tSYMBOLS_BEG symbol_list tSTRING_END
                     {
-                      (_, line), list, _, = val
-                      list.line line
-                      result = list
+                      (_, line), list, (_, line_max), = val
+
+                      result = list.line line
+                      result.line_max = line_max
                     }
 
      symbol_list: none
@@ -2800,28 +2817,32 @@ opt_block_args_tail: tCOMMA block_args_tail
 
           qwords: tQWORDS_BEG tSPACE tSTRING_END
                     {
-                      (_, line), _, _ = val
+                      (_, line), _, (_, line_max) = val
 
                       result = s(:array).line line
+                      result.line_max = line_max
                     }
                 | tQWORDS_BEG qword_list tSTRING_END
                     {
-                      (_, line), list, _ = val
+                      (_, line), list, (_, line_max) = val
 
                       result = list.line line
+                      result.line_max = line_max
                     }
 
         qsymbols: tQSYMBOLS_BEG tSPACE tSTRING_END
                     {
-                      (_, line), _, _ = val
+                      (_, line), _, (_, line_max) = val
 
                       result = s(:array).line line
+                      result.line_max = line_max
                     }
                 | tQSYMBOLS_BEG qsym_list tSTRING_END
                     {
-                      (_, line), list, _ = val
+                      (_, line), list, (_, line_max) = val
 
                       result = list.line line
+                      result.line_max = line_max
                     }
 
       qword_list: none
@@ -3257,7 +3278,14 @@ f_opt_paren_args: f_paren_args
                         result = s(:args, list).line list.line
                       end
 
-                      result << (Sexp === item ? item : item.first)
+                      if Sexp === item then
+                        line_max = item.line_max
+                      else
+                        item, line_max = item
+                      end
+
+                      result << item
+                      result.line_max = line_max
                     }
 
          f_label: tLABEL

data/lib/ruby_parser.rb

@@ -81,10 +81,12 @@ require "ruby26_parser"
 require "ruby27_parser"
 require "ruby30_parser"
 require "ruby31_parser"
+require "ruby32_parser"
 
 class RubyParser # HACK
   VERSIONS.clear # also a HACK caused by racc namespace issues
 
+  class V32 < ::Ruby32Parser; end
   class V31 < ::Ruby31Parser; end
   class V30 < ::Ruby30Parser; end
   class V27 < ::Ruby27Parser; end

data/lib/ruby_parser.yy

@@ -253,6 +253,7 @@ rule
                 | lhs tEQL mrhs
                     {
                       lhs, _, rhs = val
+
                       result = new_assign lhs, s(:svalue, rhs).line(rhs.line)
                     }
 #if V == 20
@@ -1107,8 +1108,10 @@ rule
 
       paren_args: tLPAREN2 opt_call_args rparen
                     {
-                      _, args, _ = val
+                      _, args, (_, line_max) = val
+
                       result = args
+                      result.line_max = line_max if args
                     }
 #if V >= 27
                 | tLPAREN2 args tCOMMA args_forward rparen
@@ -1335,12 +1338,14 @@ rule
                     {
                       result = wrap :colon3, val[1]
                     }
-                | tLBRACK { result = lexer.lineno } aref_args tRBRACK
+                | tLBRACK { result = lexer.lineno } aref_args rbracket
                     {
-                      _, line, args, _ = val
+                      _, line, args, (_, line_max) = val
+
                       result = args || s(:array)
                       result.sexp_type = :array # aref_args is :args
                       result.line line
+                      result.line_max = line_max
                     }
                 | tLBRACE
                     {
@@ -1942,13 +1947,19 @@ opt_block_args_tail: tCOMMA block_args_tail
                     {
                       call, args = val
 
-                      result = call.concat args.sexp_body if args
+                      result = call
+
+                      if args then
+                        call.concat args.sexp_body
+                        result.line_max = args.line_max
+                      end
                     }
                 | primary_value call_op operation2 opt_paren_args
                     {
-                      recv, call_op, (op, _line), args = val
+                      recv, call_op, (op, op_line), args = val
 
                       result = new_call recv, op.to_sym, args, call_op
+                      result.line_max = op_line unless args
                     }
                 | primary_value tCOLON2 operation2 paren_args
                     {
@@ -2674,15 +2685,17 @@ opt_block_args_tail: tCOMMA block_args_tail
 
            words: tWORDS_BEG tSPACE tSTRING_END
                     {
-                      (_, line), _, _ = val
+                      (_, line), _, (_, line_max) = val
 
                       result = s(:array).line line
+                      result.line_max = line_max
                     }
                 | tWORDS_BEG word_list tSTRING_END
                     {
-                      (_, line), list, _ = val
+                      (_, line), list, (_, line_max) = val
 
                       result = list.line line
+                      result.line_max = line_max
                     }
 
        word_list: none
@@ -2702,15 +2715,17 @@ opt_block_args_tail: tCOMMA block_args_tail
 
          symbols: tSYMBOLS_BEG tSPACE tSTRING_END
                     {
-                      (_, line), _, _ = val
+                      (_, line), _, (_, line_max) = val
 
                       result = s(:array).line line
+                      result.line_max = line_max
                     }
                 | tSYMBOLS_BEG symbol_list tSTRING_END
                     {
-                      (_, line), list, _, = val
-                      list.line line
-                      result = list
+                      (_, line), list, (_, line_max), = val
+
+                      result = list.line line
+                      result.line_max = line_max
                     }
 
      symbol_list: none
@@ -2725,28 +2740,32 @@ opt_block_args_tail: tCOMMA block_args_tail
 
           qwords: tQWORDS_BEG tSPACE tSTRING_END
                     {
-                      (_, line), _, _ = val
+                      (_, line), _, (_, line_max) = val
 
                       result = s(:array).line line
+                      result.line_max = line_max
                     }
                 | tQWORDS_BEG qword_list tSTRING_END
                     {
-                      (_, line), list, _ = val
+                      (_, line), list, (_, line_max) = val
 
                       result = list.line line
+                      result.line_max = line_max
                     }
 
         qsymbols: tQSYMBOLS_BEG tSPACE tSTRING_END
                     {
-                      (_, line), _, _ = val
+                      (_, line), _, (_, line_max) = val
 
                       result = s(:array).line line
+                      result.line_max = line_max
                     }
                 | tQSYMBOLS_BEG qsym_list tSTRING_END
                     {
-                      (_, line), list, _ = val
+                      (_, line), list, (_, line_max) = val
 
                       result = list.line line
+                      result.line_max = line_max
                     }
 
       qword_list: none
@@ -3197,7 +3216,14 @@ keyword_variable: kNIL      { result = s(:nil).line lexer.lineno }
                         result = s(:args, list).line list.line
                       end
 
-                      result << (Sexp === item ? item : item.first)
+                      if Sexp === item then
+                        line_max = item.line_max
+                      else
+                        item, line_max = item
+                      end
+
+                      result << item
+                      result.line_max = line_max
                     }
 
 #if V == 20
@@ -3449,7 +3475,15 @@ keyword_variable: kNIL      { result = s(:nil).line lexer.lineno }
        opt_terms:  | terms
           opt_nl:  | tNL
           rparen: opt_nl tRPAREN
+                    {
+                      _, close = val
+                      result = [close, lexer.lineno]
+                    }
         rbracket: opt_nl tRBRACK
+                    {
+                      _, close = val
+                      result = [close, lexer.lineno]
+                    }
 #if V >= 27
           rbrace: opt_nl tRCURLY
 #endif

data/lib/ruby_parser_extras.rb

@@ -18,7 +18,7 @@ class Sexp
   end
 
   ##
-  # Returns the maximum line number of the children of self.
+  # Returns the minimum line number of the children of self.
 
   def line_min
     @line_min ||= [self.deep_each.map(&:line).min, self.line].compact.min
@@ -30,7 +30,7 @@ class Sexp
 end
 
 module RubyParserStuff
-  VERSION = "3.19.2"
+  VERSION = "3.20.1"
 
   attr_accessor :lexer, :in_def, :in_single, :file
   attr_accessor :in_kwarg
@@ -153,6 +153,7 @@ module RubyParserStuff
       result.line lexer.lineno
     else
       result.line ss.first.line
+      result.line_max = ss.first.line_max
     end
 
     args.each do |arg|
@@ -330,9 +331,8 @@ module RubyParserStuff
     end
 
     args.each do |arg|
-      if arg.instance_of? Array and arg.size == 2 and arg.last.is_a? Numeric then
-        arg = arg.first
-      end
+      # ruby 3.0+ TODO: next if arg in [String, Integer] # eg ["(", 1]
+      next if arg.class == Array && arg.map(&:class) == [String, Integer]
 
       case arg
       when Sexp then
@@ -794,6 +794,7 @@ module RubyParserStuff
     case lhs.sexp_type
     when :lasgn, :iasgn, :cdecl, :cvdecl, :gasgn, :cvasgn, :attrasgn, :safe_attrasgn then
       lhs << rhs
+      lhs.line_max = rhs.line_max
     when :const then
       lhs.sexp_type = :cdecl
       lhs << rhs
@@ -885,12 +886,13 @@ module RubyParserStuff
     # TODO: need a test with f(&b) to produce block_pass
     # TODO: need a test with f(&b) { } to produce warning
 
-    if args
+    if args then
       if ARG_TYPES[args.sexp_type] then
         result.concat args.sexp_body
       else
         result << args
       end
+      result.line_max = args.line_max
     end
 
     # line = result.grep(Sexp).map(&:line).compact.min
@@ -927,7 +929,7 @@ module RubyParserStuff
 
   def new_class val
     # TODO: get line from class keyword
-    line, path, superclass, body = val[1], val[2], val[3], val[5]
+    _, line, path, superclass, _, body, (_, line_max) = val
 
     path = path.first if path.instance_of? Array
 
@@ -942,6 +944,7 @@ module RubyParserStuff
     end
 
     result.line = line
+    result.line_max = line_max
     result.comments = self.comments.pop
     result
   end
@@ -970,13 +973,14 @@ module RubyParserStuff
   end
 
   def new_defn val
-    _, (name, line), in_def, args, body, _ = val
+    _, (name, line), in_def, args, body, (_, line_max) = val
 
     body ||= s(:nil).line line
 
     args.line line
 
     result = s(:defn, name.to_sym, args).line line
+    result.line_max = line_max
 
     if body.sexp_type == :block then
       result.push(*body.sexp_body)
@@ -1033,13 +1037,14 @@ module RubyParserStuff
   end
 
   def new_defs val
-    _, recv, (name, line), in_def, args, body, _ = val
+    _, recv, (name, line), in_def, args, body, (_, line_max) = val
 
     body ||= s(:nil).line line
 
     args.line line
 
     result = s(:defs, recv, name.to_sym, args).line line
+    result.line_max = line_max
 
     # TODO: remove_begin
     # TODO: reduce_nodes
@@ -1204,12 +1209,12 @@ module RubyParserStuff
   end
 
   def new_module val
-    # TODO: get line from module keyword
-    line, path, body = val[1], val[2], val[4]
+    (_, line_min), _, path, _, body, (_, line_max) = val
 
     path = path.first if path.instance_of? Array
 
-    result = s(:module, path).line line
+    result = s(:module, path).line line_min
+    result.line_max = line_max
 
     if body then # REFACTOR?
       if body.sexp_type == :block then
@@ -1291,9 +1296,10 @@ module RubyParserStuff
   end
 
   def new_regexp val
-    (_, line), node, (options, _) = val
+    (_, line), node, (options, line_max) = val
 
     node ||= s(:str, "").line line
+    node.line_max = line_max
 
     o, k = 0, nil
     options.split(//).uniq.each do |c| # FIX: this has a better home

data/test/test_ruby_parser.rb

@@ -15,11 +15,26 @@ class Sexp
   def == other # :nodoc:
     if other.class == self.class then
       super and
-        (line.nil? or other.line.nil? or line == other.line)
+        (line.nil?     or other.line.nil?     or line == other.line) and
+        (!defined?(@line_max) or @line_max.nil? or line_max == other.line_max)
+        # (line_max.nil? or other.line_max.nil? or line_max == other.line_max)
     else
       false
     end
   end
+
+  # convenience function just for testing
+  alias dead line_max
+  def line_max n = UNASSIGNED
+    if n != UNASSIGNED then
+      raise ArgumentError, "setting %p.line_max %p" % [self, n] unless Integer === n
+      @line_max = n
+      self
+    else
+      # raise "Accessing before @line_max defined" unless defined?(@line_max)
+      @line_max ||= self.deep_each.map(&:line).compact.max
+    end
+  end
 end
 
 module TestRubyParserShared
@@ -965,7 +980,12 @@ module TestRubyParserShared
   end
 
   def test_heredoc_with_extra_carriage_horrible_mix?
-    rb = "<<'eot'\r\nbody\r\neot\n"
+    rb = <<~RUBY
+      <<'eot'\r
+      body\r
+      eot
+    RUBY
+
     pt = s(:str, "body\r\n")
 
     assert_parse rb, pt
@@ -1051,9 +1071,9 @@ module TestRubyParserShared
   end
 
   def test_i_fucking_hate_line_numbers2
-    rb = <<-EOM.gsub(/^ {6}/, "")
+    rb = <<~EOM
       if true then
-        p('a')
+        p("a")
         b = 1
         p b
         c =1
@@ -1074,6 +1094,138 @@ module TestRubyParserShared
     assert_parse rb, pt
   end
 
+  line_max_array = s(:array,
+                     s(:lit, :line2).line(2),
+                     s(:lit, :line3).line(3)).line(1).line_max(4)
+  line_max_array_empty = s(:array).line(1).line_max(4)
+  [
+    [:plain_array,
+     "[\n:line2,\n:line3\n]",
+     line_max_array,
+    ],
+    [:pct_i,
+     "%i[\nline2\nline3\n]",
+     line_max_array,
+    ],
+    [:pct_i_empty,
+     "%i[\n\n\n]",
+     line_max_array_empty,
+    ],
+    [:pct_I,
+     "%I[\nline2\nline3\n]",
+     line_max_array,
+    ],
+    [:pct_I_empty,
+     "%I[\n\n\n]",
+     line_max_array_empty,
+    ],
+    [:call_parens,
+     "x(\n:line2,\n:line3\n)",
+     s(:call, nil, :x, *line_max_array.sexp_body).line(1).line_max(4),
+    ],
+    [:pct_w,
+     "%w[\nline2\nline3\n]",
+     s(:array,
+       s(:str, "line2").line(2),
+       s(:str, "line3").line(3)).line(1).line_max(4),
+    ],
+    [:pct_w_empty,
+     "%w[\n\n\n]",
+     line_max_array_empty,
+    ],
+    [:pct_W,
+     "%W[\nline2\nline3\n]",
+     s(:array,
+       s(:str, "line2").line(2),
+       s(:str, "line3").line(3)).line(1).line_max(4),
+    ],
+    [:pct_W_empty,
+     "%W[\n\n\n]",
+     line_max_array_empty,
+    ],
+    [:regexp,
+     "%r[\n\n\n]", # double-quotes to have the \n counted as lines on input
+     s(:lit, %r[#{"\n\n\n"}]).line(1).line_max(4),
+    ],
+    [:module,
+     <<~"RUBY",
+       module X   # line 1
+         module Y # line 2
+           Z = 42 # line 3
+         end      # line 4
+       end        # line 5
+     RUBY
+     s(:module, :X,
+       s(:module, :Y,
+         s(:cdecl, :Z, s(:lit, 42).line(3)).line(3).line_max(3)
+        ).line(2).line_max(4)
+      ).line(1).line_max(5)],
+    [:class,
+     <<~"RUBY",
+       class X    # line 1
+         class Y  # line 2
+           Z = 42 # line 3
+         end      # line 4
+       end        # line 5
+     RUBY
+     s(:class, :X, nil,
+       s(:class, :Y, nil,
+         s(:cdecl, :Z, s(:lit, 42).line(3)).line(3).line_max(3)
+        ).line(2).line_max(4)
+      ).line(1).line_max(5)],
+    [:cdecl,
+     <<~"RUBY",
+       module X
+         X = [
+           :line3,
+           :line4,
+         ]
+       end
+     RUBY
+     s(:module, :X,
+       s(:cdecl, :X,
+         s(:array,
+           s(:lit, :line3).line(3),
+           s(:lit, :line4).line(4)).line(2).line_max(5),
+        ).line(2).line_max(5),
+      ).line(1).line_max(6)
+    ],
+    [:defn,
+     <<~"RUBY",
+       class X    # line 1
+         def y(a, # line 2
+               b) # line 3
+           a + b  # line 4
+         end      # line 5
+       end        # line 6
+     RUBY
+     s(:class, :X, nil,
+       s(:defn, :y, s(:args, :a, :b).line(2).line_max(3),
+         s(:call, s(:lvar, :a).line(4), :+, s(:lvar, :b).line(4)).line(4)
+        ).line(2).line_max(5),
+      ).line(1).line_max(6),
+    ],
+    [:defs,
+     <<~"RUBY",
+       class X         # line 1
+         def self.y(a, # line 2
+               b)      # line 3
+           a + b       # line 4
+         end           # line 5
+       end             # line 6
+     RUBY
+     s(:class, :X, nil,
+       s(:defs, s(:self).line(2), :y, s(:args, :a, :b).line(2).line_max(3),
+         s(:call, s(:lvar, :a).line(4), :+, s(:lvar, :b).line(4)).line(4)
+        ).line(2).line_max(5),
+      ).line(1).line_max(6),
+    ],
+  ].each do |(name, rb, pt)|
+    define_method "test_line_numbers__max_line__#{name}" do
+      assert_parse rb, pt
+    end
+  end
+
   def test_if_elsif
     rb = "if 1; elsif 2; end"
     pt = s(:if, s(:lit, 1), nil, s(:if, s(:lit, 2), nil, nil))
@@ -5478,6 +5630,10 @@ module TestRubyParserShared31Plus
   end
 end
 
+module TestRubyParserShared32Plus
+  include TestRubyParserShared31Plus
+end
+
 class Minitest::Test
   def skip s = "blah"
     warn "ignoring skip for %s: %s" % [name, s]
@@ -5778,8 +5934,6 @@ class TestRubyParserV26 < RubyParserTestCase
 end
 
 class TestRubyParserV27 < RubyParserTestCase
-  make_my_diffs_pretty!
-
   include TestRubyParserShared27Plus
 
   def setup
@@ -5829,6 +5983,16 @@ class TestRubyParserV31 < RubyParserTestCase
   end
 end
 
+class TestRubyParserV32 < RubyParserTestCase
+  include TestRubyParserShared32Plus
+
+  def setup
+    super
+
+    self.processor = RubyParser::V32.new
+  end
+end
+
 RubyParser::VERSIONS.each do |klass|
   v = klass.version
   describe "block args arity #{v}" do