ruby_parser 3.18.1 → 3.19.0

data/lib/ruby_parser.yy

@@ -767,8 +767,7 @@ rule
 
            cpath: tCOLON3 cname
                     {
-                      _, (name, line) = val
-                      result = s(:colon3, name.to_sym).line line
+                      result = wrap :colon3, val[1]
                     }
                 | cname
                     {
@@ -793,9 +792,7 @@ rule
 
            fitem: fname
                     {
-                      (id, line), = val
-
-                      result = s(:lit, id.to_sym).line line
+                      result = wrap :lit, val[0]
                     }
                 | symbol
 
@@ -864,9 +861,9 @@ rule
                     }
                 | tCOLON3 tCONSTANT tOP_ASGN arg_rhs
                     {
-                      _, (lhs, line), op, rhs = val
+                      _, lhs, op, rhs = val
 
-                      lhs = s(:colon3, lhs.to_sym).line line
+                      lhs = wrap :colon3, lhs
                       result = new_const_op_asgn [lhs, op, rhs]
                     }
                 | backref tOP_ASGN arg_rhs
@@ -1336,9 +1333,7 @@ rule
                     }
                 | tCOLON3 tCONSTANT
                     {
-                      _, (id, line) = val
-
-                      result = s(:colon3, id.to_sym).line line
+                      result = wrap :colon3, val[1]
                     }
                 | tLBRACK { result = lexer.lineno } aref_args tRBRACK
                     {
@@ -1846,8 +1841,7 @@ opt_block_args_tail: tCOMMA block_args_tail
 
             bvar: tIDENTIFIER
                     {
-                      (id, line), = val
-                      result = s(:shadow, id.to_sym).line line
+                      result = wrap :shadow, val[0]
                     }
                 | f_bad_arg
 
@@ -2458,9 +2452,7 @@ opt_block_args_tail: tCOMMA block_args_tail
 
       p_kw_label: tLABEL
                     {
-                      (id, line), = val
-
-                      result = s(:lit, id.to_sym).line line
+                      result = wrap :lit, val[0]
                     }
 
         p_kwrest: kwrest_mark tIDENTIFIER
@@ -2552,26 +2544,20 @@ opt_block_args_tail: tCOMMA block_args_tail
 
       p_variable: tIDENTIFIER
                     {
-                      (id, line), = val
-
                       # TODO: error_duplicate_pattern_variable(p, $1, &@1);
                       # TODO: assignable(p, $1, 0, &@$);
-                      result = s(:lvar, id.to_sym).line line
+                      result = wrap :lvar, val[0]
                     }
 
        p_var_ref: tCARET tIDENTIFIER
                     {
-                      _, (id, line) = val
-
                       # TODO: check id against env for lvar or dvar
-
-                      result = s(:lvar, id.to_sym).line line
+                      result = wrap :lvar, val[1]
                     }
 
          p_const: tCOLON3 cname
                     {
-                      _, (id, line) = val
-                      result = s(:colon3, id.to_sym).line line
+                      result = wrap :colon3, val[1]
                     }
                 | p_const tCOLON2 cname
                     {
@@ -2583,8 +2569,7 @@ opt_block_args_tail: tCOMMA block_args_tail
                 | tCONSTANT
                     {
                       # TODO $$ = gettable(p, $1, &@$);
-                      (id, line), = val
-                      result = s(:const, id.to_sym).line line
+                      result = wrap :const, val[0]
                     }
 ######################################################################
 #endif
@@ -2871,18 +2856,15 @@ regexp_contents: none
 
      string_dvar: tGVAR
                     {
-                      (id, line), = val
-                      result = s(:gvar, id.to_sym).line line
+                      result = wrap :gvar, val[0]
                     }
                 | tIVAR
                     {
-                      (id, line), = val
-                      result = s(:ivar, id.to_sym).line line
+                      result = wrap :ivar, val[0]
                     }
                 | tCVAR
                     {
-                      (id, line), = val
-                      result = s(:cvar, id.to_sym).line line
+                      result = wrap :cvar, val[0]
                     }
                 | backref
 
@@ -2891,17 +2873,13 @@ regexp_contents: none
 
             ssym: tSYMBEG sym
                     {
-                      _, (id, line) = val
-
                       lexer.lex_state = EXPR_END
-                      result = s(:lit, id.to_sym).line line
+                      result = wrap :lit, val[1]
                     }
                 | tSYMBOL
                     {
-                      (id, line), = val
-
                       lexer.lex_state = EXPR_END
-                      result = s(:lit, id.to_sym).line line
+                      result = wrap :lit, val[0]
                     }
 
              sym: fname | tIVAR | tGVAR | tCVAR
@@ -3422,10 +3400,10 @@ keyword_variable: kNIL      { result = s(:nil).line lexer.lineno }
                     }
                 | tLABEL arg_value
                     {
-                      (label, line), arg = val
+                      label, arg = val
 
-                      lit = s(:lit, label.to_sym).line line
-                      result = s(:array, lit, arg).line line
+                      lit = wrap :lit, label
+                      result = s(:array, lit, arg).line lit.line
                     }
 #if V >= 22
                 | tSTRING_BEG string_contents tLABEL_END arg_value

data/lib/ruby_parser_extras.rb

@@ -30,7 +30,7 @@ class Sexp
 end
 
 module RubyParserStuff
-  VERSION = "3.18.1"
+  VERSION = "3.19.0"
 
   attr_accessor :lexer, :in_def, :in_single, :file
   attr_accessor :in_kwarg
@@ -978,6 +978,45 @@ module RubyParserStuff
     [result, in_def]
   end
 
+  def new_endless_defn val
+    (name, line, in_def), args, _, body, _, resbody = val
+
+    result =
+      if resbody then
+        s(:defn, name, args,
+          new_rescue(body,
+                     new_resbody(s(:array).line(line),
+                                 resbody))).line line
+      else
+        s(:defn, name, args, body).line line
+      end
+
+    local_pop in_def
+    endless_method_name result
+
+    result
+  end
+
+  def new_endless_defs val
+    (recv, (name, line, in_def)), args, _, body, _, resbody = val
+
+    result =
+      if resbody then
+        s(:defs, recv, name, args,
+          new_rescue(body,
+                     new_resbody(s(:array).line(line),
+                                 resbody))).line line
+      else
+        s(:defs, recv, name, args, body).line(line)
+      end
+
+    self.in_single -= 1
+    local_pop in_def
+    endless_method_name result
+
+    result
+  end
+
   def new_defs val
     _, recv, (name, line), in_def, args, body, _ = val
 
@@ -1613,6 +1652,12 @@ module RubyParserStuff
 
   alias remove_whitespace_width whitespace_width
 
+  def wrap type, node
+    value, line = node
+    value = value.to_sym if value.respond_to? :to_sym
+    s(type, value).line line
+  end
+
   class Keyword
     include RubyLexer::State::Values
 

data/test/test_ruby_parser.rb

@@ -3541,6 +3541,8 @@ module TestRubyParserShared20Plus
   end
 
   def test_regexp_esc_C_slash
+    skip "https://bugs.ruby-lang.org/issues/18449" if RUBY_VERSION == "3.1.0"
+
     rb = "/\\cC\\d/"
     pt = s(:lit, /\cC\d/)
 
@@ -5306,6 +5308,110 @@ module TestRubyParserShared30Plus
   end
 end
 
+module TestRubyParserShared31Plus
+  include TestRubyParserShared30Plus
+
+  def test_assoc__bare
+    rb = "{ y: }"
+    pt = s(:hash, s(:lit, :y), nil)
+
+    assert_parse rb, pt
+  end
+
+  def test_block_arg__bare
+    rb = "def x(&); end"
+    pt = s(:defn, :x, s(:args, :&).line(1),
+           s(:nil).line(1)).line(1)
+
+    assert_parse rb, pt
+  end
+
+  def test_case_in_carat_parens
+    processor.env[:a] = :lvar
+
+    rb = "[^(a)]"
+    pt = s(:array_pat, nil,
+           s(:lvar, :a).line(2)).line(2)
+
+    assert_case_in rb, pt
+  end
+
+  def test_case_in_carat_nonlocal_vars
+    processor.env[:a] = :lvar
+
+    rb = "[^@a, ^$b, ^@@c]"
+    pt = s(:array_pat,
+           nil,
+           s(:ivar, :@a).line(2),
+           s(:gvar, :$b).line(2),
+           s(:cvar, :@@c).line(2)).line(2)
+
+    assert_case_in rb, pt
+  end
+
+  def test_case_in_quoted_label
+    rb = " \"b\": "
+    pt = s(:hash_pat, nil, s(:lit, :b).line(2), nil).line(2)
+
+    assert_case_in rb, pt
+  end
+
+  def test_call_block_arg_named
+    processor.env[:blk] = :lvar
+    rb = "x(&blk)"
+    pt = s(:call, nil, :x,
+           s(:block_pass, s(:lvar, :blk).line(1)).line(1)).line(1)
+
+    assert_parse rb, pt
+  end
+
+  def test_call_block_arg_unnamed
+    rb = "x(&)"
+    pt = s(:call, nil, :x,
+           s(:block_pass).line(1)).line(1)
+
+    assert_parse rb, pt
+  end
+
+  def test_defn_endless_command
+    rb = "def some_method = other_method 42"
+    pt = s(:defn, :some_method, s(:args).line(1),
+           s(:call, nil, :other_method, s(:lit, 42).line(1)).line(1)).line(1)
+
+    assert_parse rb, pt
+  end
+
+  def test_defn_endless_command_rescue
+    rb = "def some_method = other_method 42 rescue 24"
+    pt = s(:defn, :some_method, s(:args).line(1),
+           s(:rescue,
+             s(:call, nil, :other_method, s(:lit, 42).line(1)).line(1),
+             s(:resbody, s(:array).line(1),
+               s(:lit, 24).line(1)).line(1)).line(1)).line(1)
+
+    assert_parse rb, pt
+  end
+
+  def test_defs_endless_command
+    rb = "def x.some_method = other_method 42"
+    pt = s(:defs, s(:call, nil, :x).line(1), :some_method, s(:args).line(1),
+           s(:call, nil, :other_method, s(:lit, 42).line(1)).line(1)).line(1)
+
+    assert_parse rb, pt
+  end
+
+  def test_defs_endless_command_rescue
+    rb = "def x.some_method = other_method 42 rescue 24"
+    pt = s(:defs, s(:call, nil, :x).line(1), :some_method, s(:args).line(1),
+           s(:rescue,
+             s(:call, nil, :other_method, s(:lit, 42).line(1)).line(1),
+             s(:resbody, s(:array).line(1),
+               s(:lit, 24).line(1)).line(1)).line(1)).line(1)
+
+    assert_parse rb, pt
+  end
+end
+
 class Minitest::Test
   def skip s = "blah"
     warn "ignoring skip for %s: %s" % [name, s]
@@ -5641,6 +5747,16 @@ class TestRubyParserV30 < RubyParserTestCase
   end
 end
 
+class TestRubyParserV31 < RubyParserTestCase
+  include TestRubyParserShared31Plus
+
+  def setup
+    super
+
+    self.processor = RubyParser::V31.new
+  end
+end
+
 RubyParser::VERSIONS.each do |klass|
   v = klass.version
   describe "block args arity #{v}" do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ruby_parser
 version: !ruby/object:Gem::Version
-  version: 3.18.1
+  version: 3.19.0
 platform: ruby
 authors:
 - Ryan Davis
@@ -10,9 +10,9 @@ bindir: bin
 cert_chain:
 - |
   -----BEGIN CERTIFICATE-----
-  MIIDPjCCAiagAwIBAgIBBTANBgkqhkiG9w0BAQsFADBFMRMwEQYDVQQDDApyeWFu
+  MIIDPjCCAiagAwIBAgIBBjANBgkqhkiG9w0BAQsFADBFMRMwEQYDVQQDDApyeWFu
   ZC1ydWJ5MRkwFwYKCZImiZPyLGQBGRYJemVuc3BpZGVyMRMwEQYKCZImiZPyLGQB
-  GRYDY29tMB4XDTIwMTIyMjIwMzgzMFoXDTIxMTIyMjIwMzgzMFowRTETMBEGA1UE
+  GRYDY29tMB4XDTIxMTIyMzIzMTkwNFoXDTIyMTIyMzIzMTkwNFowRTETMBEGA1UE
   AwwKcnlhbmQtcnVieTEZMBcGCgmSJomT8ixkARkWCXplbnNwaWRlcjETMBEGCgmS
   JomT8ixkARkWA2NvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALda
   b9DCgK+627gPJkB6XfjZ1itoOQvpqH1EXScSaba9/S2VF22VYQbXU1xQXL/WzCkx
@@ -22,14 +22,14 @@ cert_chain:
   qhtV7HJxNKuPj/JFH0D2cswvzznE/a5FOYO68g+YCuFi5L8wZuuM8zzdwjrWHqSV
   gBEfoTEGr7Zii72cx+sCAwEAAaM5MDcwCQYDVR0TBAIwADALBgNVHQ8EBAMCBLAw
   HQYDVR0OBBYEFEfFe9md/r/tj/Wmwpy+MI8d9k/hMA0GCSqGSIb3DQEBCwUAA4IB
-  AQAE3XRm1YZcCVjAJy5yMZvTOFrS7B2SYErc+0QwmKYbHztTTDY2m5Bii+jhpuxh
-  H+ETcU1z8TUKLpsBUP4kUpIRowkVN1p/jKapV8T3Rbwq+VuYFe+GMKsf8wGZSecG
-  oMQ8DzzauZfbvhe2kDg7G9BBPU0wLQlY25rDcCy9bLnD7R0UK3ONqpwvsI5I7x5X
-  ZIMXR0a9/DG+55mawwdGzCQobDKiSNLK89KK7OcNTALKU0DfgdTkktdgKchzKHqZ
-  d/AHw/kcnU6iuMUoJEcGiJd4gVCTn1l3cDcIvxakGslCA88Jubw0Sqatan0TnC9g
-  KToW560QIey7SPfHWduzFJnV
+  AQCKB5jfsuSnKb+t/Wrh3UpdkmX7TrEsjVmERC0pPqzQ5GQJgmEXDD7oMgaKXaAq
+  x2m+KSZDrqk7c8uho5OX6YMqg4KdxehfSLqqTZGoeV78qwf/jpPQZKTf+W9gUSJh
+  zsWpo4K50MP+QtdSbKXZwjAafpQ8hK0MnnZ/aeCsW9ov5vdXpYbf3dpg6ADXRGE7
+  lQY2y1tJ5/chqu6h7dQmnm2ABUqx9O+JcN9hbCYoA5i/EeubUEtFIh2w3SpO6YfB
+  JFmxn4h9YO/pVdB962BdBNNDia0kgIjI3ENnkLq0dKpYU3+F3KhEuTksLO0L6X/V
+  YsuyUzsMz6GQA4khyaMgKNSD
   -----END CERTIFICATE-----
-date: 2021-11-10 00:00:00.000000000 Z
+date: 2022-03-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sexp_processor
@@ -119,14 +119,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.22'
+        version: '3.23'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.22'
+        version: '3.23'
 description: |-
   ruby_parser (RP) is a ruby parser written in pure ruby (utilizing
   racc--which does by default use a C extension). It outputs
@@ -155,6 +155,9 @@ description: |-
   * 1.9 parser is at 99.9940% accuracy, 4.013 sigma
   * 2.0 parser is at 99.9939% accuracy, 4.008 sigma
   * 2.6 parser is at 99.9972% accuracy, 4.191 sigma
+  * 3.0 parser has a 100% parse rate.
+    * Tested against 2,672,412 unique ruby files across 167k gems.
+    * As do all the others now, basically.
 email:
 - ryand-ruby@zenspider.com
 executables:
@@ -199,6 +202,8 @@ files:
 - lib/ruby27_parser.y
 - lib/ruby30_parser.rb
 - lib/ruby30_parser.y
+- lib/ruby31_parser.rb
+- lib/ruby31_parser.y
 - lib/ruby3_parser.yy
 - lib/ruby_lexer.rb
 - lib/ruby_lexer.rex
@@ -238,7 +243,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.16
+rubygems_version: 3.3.3
 signing_key:
 specification_version: 4
 summary: ruby_parser (RP) is a ruby parser written in pure ruby (utilizing racc--which