ruby_parser 3.19.1 → 3.20.0

data/lib/ruby3_parser.yy

@@ -4,6 +4,8 @@
 class Ruby30Parser
 #elif V == 31
 class Ruby31Parser
+#elif V == 32
+class Ruby32Parser
 #else
 fail "version not specified or supported on code generation"
 #endif
@@ -2591,9 +2593,21 @@ opt_block_args_tail: tCOMMA block_args_tail
                 | xstring
                 | regexp
                 | words
+                    {
+                      result = ary_to_pat val[0]
+                    }
                 | qwords
+                    {
+                      result = ary_to_pat val[0]
+                    }
                 | symbols
+                    {
+                      result = ary_to_pat val[0]
+                    }
                 | qsymbols
+                    {
+                      result = ary_to_pat val[0]
+                    }
                 | keyword_variable
                     {
                       # TODO? if (!($$ = gettable(p, $1, &@$))) $$ = NEW_BEGIN(0, &@$);
@@ -2607,7 +2621,7 @@ opt_block_args_tail: tCOMMA block_args_tail
                     {
                       # TODO: error_duplicate_pattern_variable(p, $1, &@1);
                       # TODO: assignable(p, $1, 0, &@$);
-                      result = wrap :lvar, val[0]
+                      result = wrap :lasgn, val[0]
                     }
 
        p_var_ref: tCARET tIDENTIFIER

data/lib/ruby_lexer.rb

@@ -232,7 +232,7 @@ class RubyLexer
     content = match[1]
 
     if text =~ check then
-      content.gsub(ESC) { unescape $1 }
+      unescape_string content
     else
       content.gsub(/\\\\/, "\\").gsub(/\\\'/, "'")
     end
@@ -590,9 +590,7 @@ class RubyLexer
     orig_line = lineno
     self.lineno += text.count("\n")
 
-    str = text[1..-2]
-      .gsub(ESC) { unescape($1).b.force_encoding Encoding::UTF_8 }
-    str = str.b unless str.valid_encoding?
+    str = unescape_string text[1..-2]
 
     result EXPR_END, :tSTRING, str, orig_line
   end
@@ -817,6 +815,15 @@ class RubyLexer
     end
   end
 
+  def unescape_string str
+    str = str.gsub(ESC) { unescape($1).b.force_encoding Encoding::UTF_8 }
+    if str.valid_encoding?
+      str
+    else
+      str.b
+    end
+  end
+
   def unescape s
     r = ESCAPES[s]
 

data/lib/ruby_parser.rb

@@ -81,10 +81,12 @@ require "ruby26_parser"
 require "ruby27_parser"
 require "ruby30_parser"
 require "ruby31_parser"
+require "ruby32_parser"
 
 class RubyParser # HACK
   VERSIONS.clear # also a HACK caused by racc namespace issues
 
+  class V32 < ::Ruby32Parser; end
   class V31 < ::Ruby31Parser; end
   class V30 < ::Ruby30Parser; end
   class V27 < ::Ruby27Parser; end

data/lib/ruby_parser.yy

@@ -2530,9 +2530,21 @@ opt_block_args_tail: tCOMMA block_args_tail
                 | xstring
                 | regexp
                 | words
+                    {
+                      result = ary_to_pat val[0]
+                    }
                 | qwords
+                    {
+                      result = ary_to_pat val[0]
+                    }
                 | symbols
+                    {
+                      result = ary_to_pat val[0]
+                    }
                 | qsymbols
+                    {
+                      result = ary_to_pat val[0]
+                    }
                 | keyword_variable
                     {
                       # TODO? if (!($$ = gettable(p, $1, &@$))) $$ = NEW_BEGIN(0, &@$);
@@ -2546,7 +2558,7 @@ opt_block_args_tail: tCOMMA block_args_tail
                     {
                       # TODO: error_duplicate_pattern_variable(p, $1, &@1);
                       # TODO: assignable(p, $1, 0, &@$);
-                      result = wrap :lvar, val[0]
+                      result = wrap :lasgn, val[0]
                     }
 
        p_var_ref: tCARET tIDENTIFIER

data/lib/ruby_parser_extras.rb

@@ -30,7 +30,7 @@ class Sexp
 end
 
 module RubyParserStuff
-  VERSION = "3.19.1"
+  VERSION = "3.20.0"
 
   attr_accessor :lexer, :in_def, :in_single, :file
   attr_accessor :in_kwarg
@@ -471,7 +471,7 @@ module RubyParserStuff
         end
       else
         warn "unprocessed: %p" % [s]
-      end.map { |l| whitespace_width l.chomp }
+      end.map { |l| whitespace_width l }
     }.compact.min
   end
 
@@ -731,6 +731,13 @@ module RubyParserStuff
     result
   end
 
+  def ary_to_pat ary
+    pat = ary.dup
+    pat.sexp_type = :array_TAIL
+
+    new_array_pattern nil, nil, pat, ary.line
+  end
+
   def new_array_pattern const, pre_arg, arypat, loc
     result = s(:array_pat, const).line loc
     result << pre_arg if pre_arg
@@ -1066,9 +1073,9 @@ module RubyParserStuff
     lhs_id = "*#{lhs_id}".to_sym
     rhs_id = "*#{rhs_id}".to_sym
 
-    mid.sexp_type = :array_pat # HACK?
+    raise "BAD?" unless mid.sexp_type == :array_TAIL
 
-    s(:find_pat_TAIL, lhs_id, mid, rhs_id).line line
+    s(:find_pat_TAIL, lhs_id, *mid.sexp_body, rhs_id).line line
   end
 
   def new_for expr, var, body
@@ -1651,7 +1658,7 @@ module RubyParserStuff
 
     if remove_width then
       line[idx..-1]
-    elsif line[idx].nil?
+    elsif line[idx] == "\n"
       nil
     else
       col

data/test/test_ruby_lexer.rb

@@ -3009,6 +3009,19 @@ class TestRubyLexer < Minitest::Test
                 :tSTRING_END,     "\"",     EXPR_LIT)
   end
 
+  def test_yylex_symbol_double_escape_octal
+    setup_lexer ":\"Variet\\303\\240\""
+
+    adv = @lex.next_token
+    act_token, act_value = adv
+    act_value = act_value.first
+
+    assert_equal :tSYMBOL, act_token
+    assert_match EXPR_LIT, @lex.lex_state
+    # Force comparison of encodings
+    assert_equal "Varietà", act_value
+  end
+
   def test_yylex_symbol_single
     assert_lex3(":'symbol'",
                 nil,

data/test/test_ruby_parser.rb

@@ -826,6 +826,13 @@ module TestRubyParserShared
     assert_parse rb, pt
   end
 
+  def test_dsym_esc_to_sym
+    rb = ':"Variet\303\240"'
+    pt = s(:lit, :Varietà)
+
+    assert_parse rb, pt
+  end
+
   def test_empty
     refute_parse ""
   end
@@ -3541,10 +3548,8 @@ module TestRubyParserShared20Plus
   end
 
   def test_regexp_esc_C_slash
-    skip "https://bugs.ruby-lang.org/issues/18449" if RUBY_VERSION == "3.1.0"
-
     rb = "/\\cC\\d/"
-    pt = s(:lit, /\cC\d/)
+    pt = s(:lit, Regexp.new('\cC\d')) # https://bugs.ruby-lang.org/issues/18449
 
     assert_parse rb, pt
   end
@@ -4270,6 +4275,21 @@ module TestRubyParserShared23Plus
     assert_parse rb, pt
   end
 
+  def test_heredoc_squiggly_blank_line_plus_interpolation
+    rb = "a = foo(<<~EOF.chop)\n\n    #\{bar}baz\n  EOF"
+    pt = s(:lasgn, :a,
+           s(:call,
+             nil,
+             :foo,
+             s(:call,
+               s(:dstr, "\n",
+                 s(:evstr, s(:call, nil, :bar).line(3)).line(3),
+                 s(:str, "baz\n").line(3)).line(1),
+               :chop).line(1)).line(1)).line(1)
+
+    assert_parse rb, pt
+  end
+
   def test_integer_with_if_modifier
     rb = "1_234if true"
     pt = s(:if, s(:true), s(:lit, 1234), nil)
@@ -4553,8 +4573,8 @@ module TestPatternMatching
     pt = s(:array_pat,
            nil,
            s(:lit, :a).line(2),
-           s(:lvar, :b).line(2),
-           s(:lvar, :c).line(2),
+           s(:lasgn, :b).line(2),
+           s(:lasgn, :c).line(2),
            s(:array_pat,
              nil,
              s(:lit, :d).line(2),
@@ -4647,19 +4667,19 @@ module TestPatternMatching
   end
 
   def test_case_in_78
-    assert_case_in "%W[a b]", s(:array, s(:str, "a").line(2), s(:str, "b").line(2)).line(2)
+    assert_case_in "%W[a b]", s(:array_pat, nil, s(:str, "a").line(2), s(:str, "b").line(2)).line(2)
   end
 
   def test_case_in_79
-    assert_case_in "%w[a b]", s(:array, s(:str, "a").line(2), s(:str, "b").line(2)).line(2)
+    assert_case_in "%w[a b]", s(:array_pat, nil, s(:str, "a").line(2), s(:str, "b").line(2)).line(2)
   end
 
   def test_case_in_80
-    assert_case_in "%I[a b]", s(:array, s(:lit, :a).line(2), s(:lit, :b).line(2)).line(2)
+    assert_case_in "%I[a b]", s(:array_pat, nil, s(:lit, :a).line(2), s(:lit, :b).line(2)).line(2)
   end
 
   def test_case_in_81
-    assert_case_in "%i[a b]", s(:array, s(:lit, :a).line(2), s(:lit, :b).line(2)).line(2)
+    assert_case_in "%i[a b]", s(:array_pat, nil, s(:lit, :a).line(2), s(:lit, :b).line(2)).line(2)
   end
 
   def test_case_in_83
@@ -4667,7 +4687,7 @@ module TestPatternMatching
     pt = s(:array_pat, nil,
            s(:iter, s(:lambda).line(2), s(:args, :b).line(2),
              s(:true).line(2)).line(2),
-           s(:lvar, :c).line(2)).line(2)
+           s(:lasgn, :c).line(2)).line(2)
 
     assert_case_in rb, pt
   end
@@ -4677,7 +4697,7 @@ module TestPatternMatching
     pt = s(:array_pat, nil,
            s(:array_pat, nil,
              s(:lit, :b).line(2),
-             s(:lvar, :c).line(2)).line(2),
+             s(:lasgn, :c).line(2)).line(2),
            s(:array_pat,
              nil,
              s(:lit, :d).line(2),
@@ -4723,7 +4743,7 @@ module TestPatternMatching
            s(:in,
              s(:array_pat,
                s(:const, :B).line(2),
-               s(:lvar, :c).line(2)).line(2),
+               s(:lasgn, :c).line(2)).line(2),
              s(:lit, :d).line(3)).line(2),
            nil)
 
@@ -4736,7 +4756,7 @@ module TestPatternMatching
            s(:in,
              s(:array_pat,
                s(:const, s(:colon2, s(:const, :B).line(2), :C).line(2)).line(2),
-               s(:lvar, :d).line(2)).line(2),
+               s(:lasgn, :d).line(2)).line(2),
              s(:lit, :e).line(3)).line(2),
           nil)
 
@@ -4841,7 +4861,7 @@ module TestPatternMatching
              s(:hash_pat,
                nil,
                s(:lit, :b).line(2),
-               s(:lvar, :c).line(2),
+               s(:lasgn, :c).line(2),
                s(:kwrest, :"**rest").line(2)).line(2),
              s(:lit, :d).line(2)).line(2),
            nil)
@@ -4934,12 +4954,12 @@ module TestPatternMatching
   end
 
   def test_parse_pattern_044
-      rb = <<~RUBY
-        case obj
-        in Object[]
-          true
-        end
-      RUBY
+    rb = <<~RUBY
+      case obj
+      in Object[]
+        true
+      end
+    RUBY
     pt = s(:case,
            s(:call, nil, :obj),
            s(:in, s(:array_pat, s(:const, :Object).line(2)).line(2),
@@ -5072,7 +5092,7 @@ module TestPatternMatching30
                    s(:find_pat,
                      s(:const, :Symbol).line(2),
                      :"*lhs",
-                     s(:array_pat, s(:lvar, :x).line(2)).line(2),
+                     s(:lasgn, :x).line(2),
                      :"*rhs").line(2))
   end
 
@@ -5080,7 +5100,7 @@ module TestPatternMatching30
     assert_case_in("Symbol[*lhs, x, *rhs]",
                    s(:find_pat, s(:const, :Symbol).line(2),
                      :"*lhs",
-                     s(:array_pat, s(:lvar, :x).line(2)).line(2),
+                     s(:lasgn, :x).line(2),
                      :"*rhs").line(2))
   end
 end
@@ -5196,7 +5216,7 @@ module TestRubyParserShared30Plus
     rb = "42 => n"
     pt = s(:case,
            s(:lit, 42),
-           s(:in, s(:lvar, :n), nil), nil)
+           s(:in, s(:lasgn, :n), nil), nil)
 
     assert_parse rb, pt
   end
@@ -5208,7 +5228,7 @@ module TestRubyParserShared30Plus
            s(:in,
              s(:find_pat, nil,
                :"*a",
-               s(:array_pat, s(:lit, :+).line(2)).line(2),
+               s(:lit, :+).line(2),
                :"*b").line(2),
              nil).line(2),
            nil)
@@ -5223,7 +5243,7 @@ module TestRubyParserShared30Plus
            s(:in,
              s(:find_pat, nil,
                :*,
-               s(:array_pat, s(:lit, :b).line(2), s(:lvar, :c).line(2)).line(2),
+               s(:lit, :b).line(2), s(:lasgn, :c).line(2),
                :*).line(2),
              nil).line(2),
            nil)
@@ -5458,6 +5478,10 @@ module TestRubyParserShared31Plus
   end
 end
 
+module TestRubyParserShared32Plus
+  include TestRubyParserShared31Plus
+end
+
 class Minitest::Test
   def skip s = "blah"
     warn "ignoring skip for %s: %s" % [name, s]
@@ -5524,6 +5548,8 @@ class RubyParserTestCase < ParseTreeTestCase
 
   attr_accessor :assert_parse_ran
 
+  require "ruby2ruby" if ENV["R2R"]
+
   def assert_parse rb, pt
     self.processor.reset if assert_parse_ran # allows multiple calls
     self.assert_parse_ran = true
@@ -5534,6 +5560,10 @@ class RubyParserTestCase < ParseTreeTestCase
     pt.line ||= 1
 
     self.result = processor.parse rb, "(string)", timeout
+
+    # just try it for now:
+    Ruby2Ruby.new.process(result.deep_clone) if ENV["R2R"]
+
     assert_equal pt, result
   end
 
@@ -5803,6 +5833,16 @@ class TestRubyParserV31 < RubyParserTestCase
   end
 end
 
+class TestRubyParserV32 < RubyParserTestCase
+  include TestRubyParserShared32Plus
+
+  def setup
+    super
+
+    self.processor = RubyParser::V32.new
+  end
+end
+
 RubyParser::VERSIONS.each do |klass|
   v = klass.version
   describe "block args arity #{v}" do

data.tar.gz.sig

@@ -1 +1 @@
-��C-]̓�˺��)5J��������I�[���Y5��S�t?���e��z�1�Q����e��i��ł��^���BU4�'b�����o!_�ٝ:�Q�"l�l�$c��5n�V6L5{І���	0�}���By�p3�^�H�{���5�����j�y���
I�N^�6�����IZ�DQ݆M�	2w|(|x�uaP�F��L�i���6��M�>5s��@_�%�|�P����[��GP��i���
+P��F�V�ʿx��
O��)(�
n��'�M�2$���x/멀�o�ѝ�/g�s�h�l���؉G

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ruby_parser
 version: !ruby/object:Gem::Version
-  version: 3.19.1
+  version: 3.20.0
 platform: ruby
 authors:
 - Ryan Davis
@@ -10,9 +10,9 @@ bindir: bin
 cert_chain:
 - |
   -----BEGIN CERTIFICATE-----
-  MIIDPjCCAiagAwIBAgIBBjANBgkqhkiG9w0BAQsFADBFMRMwEQYDVQQDDApyeWFu
+  MIIDPjCCAiagAwIBAgIBBzANBgkqhkiG9w0BAQsFADBFMRMwEQYDVQQDDApyeWFu
   ZC1ydWJ5MRkwFwYKCZImiZPyLGQBGRYJemVuc3BpZGVyMRMwEQYKCZImiZPyLGQB
-  GRYDY29tMB4XDTIxMTIyMzIzMTkwNFoXDTIyMTIyMzIzMTkwNFowRTETMBEGA1UE
+  GRYDY29tMB4XDTIzMDEwMTA3NTExN1oXDTI0MDEwMTA3NTExN1owRTETMBEGA1UE
   AwwKcnlhbmQtcnVieTEZMBcGCgmSJomT8ixkARkWCXplbnNwaWRlcjETMBEGCgmS
   JomT8ixkARkWA2NvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALda
   b9DCgK+627gPJkB6XfjZ1itoOQvpqH1EXScSaba9/S2VF22VYQbXU1xQXL/WzCkx
@@ -22,14 +22,14 @@ cert_chain:
   qhtV7HJxNKuPj/JFH0D2cswvzznE/a5FOYO68g+YCuFi5L8wZuuM8zzdwjrWHqSV
   gBEfoTEGr7Zii72cx+sCAwEAAaM5MDcwCQYDVR0TBAIwADALBgNVHQ8EBAMCBLAw
   HQYDVR0OBBYEFEfFe9md/r/tj/Wmwpy+MI8d9k/hMA0GCSqGSIb3DQEBCwUAA4IB
-  AQCKB5jfsuSnKb+t/Wrh3UpdkmX7TrEsjVmERC0pPqzQ5GQJgmEXDD7oMgaKXaAq
-  x2m+KSZDrqk7c8uho5OX6YMqg4KdxehfSLqqTZGoeV78qwf/jpPQZKTf+W9gUSJh
-  zsWpo4K50MP+QtdSbKXZwjAafpQ8hK0MnnZ/aeCsW9ov5vdXpYbf3dpg6ADXRGE7
-  lQY2y1tJ5/chqu6h7dQmnm2ABUqx9O+JcN9hbCYoA5i/EeubUEtFIh2w3SpO6YfB
-  JFmxn4h9YO/pVdB962BdBNNDia0kgIjI3ENnkLq0dKpYU3+F3KhEuTksLO0L6X/V
-  YsuyUzsMz6GQA4khyaMgKNSD
+  AQAkg3y+PBnBAPWdxxITm5sPHqdWQgSyCpRA20o4LTuWr8BWhSXBkfQNa7cY6fOn
+  xyM34VPzBFbExv6XOGDfOMFBVaYTHuN9peC/5/umL7kLl+nflXzL2QA7K6LYj5Bg
+  sM574Onr0dZDM6Vn69bzQ7rBIFDfK/OhlPzqKZad4nsdcsVH8ODCiT+ATMIZyz5K
+  WCnNtqlyiWXI8tdTpahDgcUwfcN/oN7v4K8iU5IbLJX6HQ5DKgmKjfb6XyMth16k
+  ROfWo9Uyp8ba/j9eVG14KkYRaLydAY1MNQk2yd3R5CGfeOpD1kttxjoypoUJ2dOG
+  nsNBRuQJ1UfiCG97a6DNm+Fr
   -----END CERTIFICATE-----
-date: 2022-04-06 00:00:00.000000000 Z
+date: 2023-03-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sexp_processor
@@ -119,14 +119,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.23'
+        version: '4.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.23'
+        version: '4.0'
 description: |-
   ruby_parser (RP) is a ruby parser written in pure ruby (utilizing
   racc--which does by default use a C extension). It outputs
@@ -204,6 +204,8 @@ files:
 - lib/ruby30_parser.y
 - lib/ruby31_parser.rb
 - lib/ruby31_parser.y
+- lib/ruby32_parser.rb
+- lib/ruby32_parser.y
 - lib/ruby3_parser.yy
 - lib/ruby_lexer.rb
 - lib/ruby_lexer.rex
@@ -233,7 +235,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.1'
+      version: '2.6'
   - - "<"
     - !ruby/object:Gem::Version
       version: '4'
@@ -243,7 +245,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.3
+rubygems_version: 3.4.6
 signing_key:
 specification_version: 4
 summary: ruby_parser (RP) is a ruby parser written in pure ruby (utilizing racc--which