ruby_parser 3.7.3 → 3.8.0

data/lib/ruby_lexer.rb

@@ -680,9 +680,9 @@ class RubyLexer
   end
 
   def process_label text
-    result = process_symbol text
-    result[0] = :tLABEL
-    result
+    symbol = text[1..-3].gsub(ESC) { unescape $1 }
+
+    result(:expr_labelarg, :tLABEL, [symbol, self.lineno])
   end
 
   def process_token text
@@ -1074,7 +1074,8 @@ class RubyLexer
   end
 
   def ruby22?
-    Ruby22Parser === parser
+    Ruby22Parser === parser or
+      Ruby23Parser === parser
   end
 
   def process_string # TODO: rewrite / remove

data/lib/ruby_lexer.rex

@@ -61,7 +61,7 @@ rule
 | bol?          /\=begin(?=\s)/         process_begin
 |               /\=(?=begin\b)/         { result arg_state, TOKENS[text], text }
 
-ruby22_label?   /\"(#{SIMPLE_STRING})\":/o process_label
+ruby22_label?   /\"#{SIMPLE_STRING}\":/o process_label
                 /\"(#{SIMPLE_STRING})\"/o { result :expr_end, :tSTRING, text[1..-2].gsub(ESC) { unescape $1 } }
                 /\"/                    { string STR_DQUOTE; result nil, :tSTRING_BEG, text }
 
@@ -133,6 +133,7 @@ was_label?        /\'#{SSTRING}\':?/o   process_label_or_string
 |               /\&\&\=/                { result(:expr_beg, :tOP_ASGN, "&&") }
 |               /\&\&/                  { result(:expr_beg, :tANDOP,   "&&") }
 |               /\&\=/                  { result(:expr_beg, :tOP_ASGN, "&" ) }
+|               /\&\./                  { result(:expr_dot, :tLONELY,  "&.") }
 |               /\&/                    process_amper
 
                 /\//                    process_slash

data/lib/ruby_lexer.rex.rb

@@ -111,7 +111,7 @@ class RubyLexer
             when text = ss.scan(/\=(?=begin\b)/) then
               action { result arg_state, TOKENS[text], text }
             end # group /=/
-          when ruby22_label? && (text = ss.scan(/\"(#{SIMPLE_STRING})\":/o)) then
+          when ruby22_label? && (text = ss.scan(/\"#{SIMPLE_STRING}\":/o)) then
             process_label text
           when text = ss.scan(/\"(#{SIMPLE_STRING})\"/o) then
             action { result :expr_end, :tSTRING, text[1..-2].gsub(ESC) { unescape $1 } }
@@ -232,6 +232,8 @@ class RubyLexer
               action { result(:expr_beg, :tANDOP,   "&&") }
             when ss.skip(/\&\=/) then
               action { result(:expr_beg, :tOP_ASGN, "&" ) }
+            when ss.skip(/\&\./) then
+              action { result(:expr_dot, :tLONELY,  "&.") }
             when text = ss.scan(/\&/) then
               process_amper text
             end # group /&/

data/lib/ruby_parser.rb

@@ -3,4 +3,5 @@ require 'ruby19_parser'
 require 'ruby20_parser'
 require 'ruby21_parser'
 require 'ruby22_parser'
+require 'ruby23_parser'
 require 'ruby_parser_extras'

data/lib/ruby_parser.yy

@@ -6,6 +6,8 @@ class Ruby20Parser
 class Ruby21Parser
 #elif defined(RUBY22)
 class Ruby22Parser
+#elif defined(RUBY23)
+class Ruby23Parser
 #endif
 
 token kCLASS kMODULE kDEF kUNDEF kBEGIN kRESCUE kENSURE kEND kIF kUNLESS
@@ -25,12 +27,15 @@ token kCLASS kMODULE kDEF kUNDEF kBEGIN kRESCUE kENSURE kEND kIF kUNLESS
       tWORDS_BEG tQWORDS_BEG tSTRING_DBEG tSTRING_DVAR tSTRING_END
       tSTRING tSYMBOL tNL tEH tCOLON tCOMMA tSPACE tSEMI tLAMBDA
       tLAMBEG tDSTAR tCHAR tSYMBOLS_BEG tQSYMBOLS_BEG tSTRING_DEND tUBANG
-#if defined(RUBY21) || defined(RUBY22)
+#if defined(RUBY21) || defined(RUBY22) || defined(RUBY23))
       tRATIONAL tIMAGINARY
 #endif
-#if defined(RUBY22)
+#if defined(RUBY22 || defined(RUBY23))
       tLABEL_END
 #endif
+#if defined(RUBY23)
+       tLONELY
+#endif
 
 prechigh
   right    tBANG tTILDE tUPLUS
@@ -222,7 +227,7 @@ rule
                       result = new_masgn val[0], val[2], :wrap
                     }
                 | mlhs tEQL mrhs
-#elif defined(RUBY21) || defined(RUBY22)
+#elif defined(RUBY21) || defined(RUBY22 || defined(RUBY23))
                 | mlhs tEQL mrhs_arg
 #endif
                     {
@@ -312,6 +317,13 @@ rule
                     {
                       result = new_call val[0], val[2].to_sym, val[3]
                     }
+#if defined(RUBY23)
+                | primary_value tLONELY operation2 command_args =tLOWEST
+                    {
+                      result = new_call val[0], val[2].to_sym, val[3]
+                      result[0] = :safe_call
+                    }
+#endif
                 | primary_value tDOT operation2 command_args cmd_brace_block
                     {
                       recv, _, msg, args, block = val
@@ -509,6 +521,16 @@ rule
                     {
                       result = s(:attrasgn, val[0], :"#{val[2]}=")
                     }
+#if defined(RUBY23)
+                | primary_value tLONELY tIDENTIFIER
+                    {
+                      result = s(:safe_attrasgn, val[0], :"#{val[2]}=")
+                    }
+                | primary_value tLONELY tCONSTANT
+                    {
+                      result = s(:safe_attrasgn, val[0], :"#{val[2]}=")
+                    }
+#endif
                 | primary_value tCOLON2 tIDENTIFIER
                     {
                       result = s(:attrasgn, val[0], :"#{val[2]}=")
@@ -707,7 +729,7 @@ rule
                       result = new_call(new_call(s(:lit, val[1]), :"**", argl(val[3])), :"-@")
                     }
                 | tUMINUS_NUM tFLOAT tPOW arg
-#elif defined(RUBY21) || defined(RUBY22)
+#elif defined(RUBY21) || defined(RUBY22 || defined(RUBY23))
                 | tUMINUS_NUM simple_numeric tPOW arg
 #endif
                     {
@@ -926,7 +948,7 @@ rule
                       result = self.list_append val[0], s(:splat, val[3])
                     }
 
-#if defined(RUBY21) || defined(RUBY22)
+#if defined(RUBY21) || defined(RUBY22 || defined(RUBY23))
         mrhs_arg: mrhs
                     {
                       result = new_masgn_arg val[0]
@@ -1574,6 +1596,13 @@ opt_block_args_tail: tCOMMA block_args_tail
                     {
                       result = new_call val[0], val[2].to_sym, val[3]
                     }
+#if defined(RUBY23)
+                | primary_value tLONELY operation2 opt_paren_args
+                    {
+                      result = new_call val[0], val[2].to_sym, val[3]
+                      result[0] = :safe_call
+                    }
+#endif
                 | primary_value tCOLON2 operation2 paren_args
                     {
                       result = new_call val[0], val[2].to_sym, val[3]
@@ -1586,6 +1615,13 @@ opt_block_args_tail: tCOMMA block_args_tail
                     {
                       result = new_call val[0], :call, val[2]
                     }
+#if defined(RUBY23)
+                | primary_value tLONELY paren_args
+                    {
+                      result = new_call val[0], :call, val[2]
+                      result[0] = :safe_call
+                    }
+#endif
                 | primary_value tCOLON2 paren_args
                     {
                       result = new_call val[0], :call, val[2]
@@ -1888,7 +1924,7 @@ regexp_contents: none
                     {
 #if defined(RUBY20)
                       # TODO: tRCURLY -> tSTRING_DEND
-#elif defined(RUBY21) || defined(RUBY22)
+#elif defined(RUBY21) || defined(RUBY22 || defined(RUBY23))
                       # TODO: tRCURLY -> tSTRING_END
 #endif
                       _, memo, stmt, _ = val
@@ -1960,7 +1996,7 @@ regexp_contents: none
          numeric: tINTEGER
                 | tFLOAT
                 | tUMINUS_NUM tINTEGER =tLOWEST
-#elif defined(RUBY21) || defined(RUBY22)
+#elif defined(RUBY21) || defined(RUBY22 || defined(RUBY23))
          numeric: simple_numeric
                 | tUMINUS_NUM simple_numeric
 #endif
@@ -1974,7 +2010,7 @@ regexp_contents: none
 #endif
                     }
 
-#if defined(RUBY21) || defined(RUBY22)
+#if defined(RUBY21) || defined(RUBY22) || defined(RUBY23))
   simple_numeric: tINTEGER
                 | tFLOAT
                 | tRATIONAL
@@ -2173,7 +2209,7 @@ keyword_variable: kNIL      { result = s(:nil)   }
                       result = identifier
                     }
 
-#if defined(RUBY22)
+#if defined(RUBY22) || defined(RUBY23))
       f_arg_asgn: f_norm_arg
 
       f_arg_item: f_arg_asgn
@@ -2217,7 +2253,7 @@ keyword_variable: kNIL      { result = s(:nil)   }
 
 #if defined(RUBY20)
             f_kw: tLABEL arg_value
-#elif defined(RUBY21) || defined(RUBY22)
+#elif defined(RUBY21) || defined(RUBY22) || defined(RUBY23)
          f_label: tLABEL
 
             f_kw: f_label arg_value
@@ -2230,7 +2266,7 @@ keyword_variable: kNIL      { result = s(:nil)   }
 
                       result = s(:array, s(:kwarg, identifier, val[1]))
                     }
-#if defined(RUBY21) || defined(RUBY22)
+#if defined(RUBY21) || defined(RUBY22) || defined(RUBY23)
                 | f_label
                     {
                       label, _ = val[0] # TODO: fix lineno?
@@ -2243,7 +2279,7 @@ keyword_variable: kNIL      { result = s(:nil)   }
 
 #if defined(RUBY20)
       f_block_kw: tLABEL primary_value
-#elif defined(RUBY21) || defined(RUBY22)
+#elif defined(RUBY21) || defined(RUBY22) || defined(RUBY23)
       f_block_kw: f_label primary_value
 #endif
                     {
@@ -2254,7 +2290,7 @@ keyword_variable: kNIL      { result = s(:nil)   }
 
                       result = s(:array, s(:kwarg, identifier, val[1]))
                     }
-#if defined(RUBY21) || defined(RUBY22)
+#if defined(RUBY21) || defined(RUBY22) || defined(RUBY23)
                 | f_label
                     {
                       label, _ = val[0] # TODO: fix lineno?
@@ -2294,7 +2330,7 @@ keyword_variable: kNIL      { result = s(:nil)   }
            f_opt: tIDENTIFIER tEQL arg_value
 #elif defined(RUBY21)
            f_opt: f_norm_arg tEQL arg_value
-#elif defined(RUBY22)
+#elif defined(RUBY22) || defined(RUBY23)
            f_opt: f_arg_asgn tEQL arg_value
 #endif
                     {
@@ -2306,7 +2342,7 @@ keyword_variable: kNIL      { result = s(:nil)   }
      f_block_opt: tIDENTIFIER tEQL primary_value
 #elif defined(RUBY21)
      f_block_opt: f_norm_arg tEQL primary_value
-#elif defined(RUBY22)
+#elif defined(RUBY22) || defined(RUBY23)
      f_block_opt: f_arg_asgn tEQL primary_value
 #endif
                     {
@@ -2407,7 +2443,7 @@ keyword_variable: kNIL      { result = s(:nil)   }
                     {
                       result = s(:array, s(:lit, val[0][0].to_sym), val[1])
                     }
-#if defined(RUBY22)
+#if defined(RUBY22) || defined(RUBY23)
                 | tSTRING_BEG string_contents tLABEL_END arg_value
                     {
                       _, sym, _, value = val

data/lib/ruby_parser_extras.rb

@@ -91,7 +91,7 @@ class RPStringScanner < StringScanner
 end
 
 module RubyParserStuff
-  VERSION = "3.7.3" unless constants.include? "VERSION" # SIGH
+  VERSION = "3.8.0" unless constants.include? "VERSION" # SIGH
 
   attr_accessor :lexer, :in_def, :in_single, :file
   attr_reader :env, :comments
@@ -932,7 +932,7 @@ module RubyParserStuff
     rhs = value_expr rhs
 
     case lhs[0]
-    when :lasgn, :iasgn, :cdecl, :cvdecl, :gasgn, :cvasgn, :attrasgn then
+    when :lasgn, :iasgn, :cdecl, :cvdecl, :gasgn, :cvasgn, :attrasgn, :safe_attrasgn then
       lhs << rhs
     when :const then
       lhs[0] = :cdecl
@@ -1320,6 +1320,10 @@ module RubyParserStuff
   end
 end
 
+class Ruby23Parser < Racc::Parser
+  include RubyParserStuff
+end
+
 class Ruby22Parser < Racc::Parser
   include RubyParserStuff
 end
@@ -1354,11 +1358,12 @@ class RubyParser
     @p20 = Ruby20Parser.new
     @p21 = Ruby21Parser.new
     @p22 = Ruby22Parser.new
+    @p23 = Ruby23Parser.new
   end
 
   def process s, f = "(string)", t = 10
     e = nil
-    [@p22, @p21, @p20, @p19, @p18].each do |parser|
+    [@p23, @p22, @p21, @p20, @p19, @p18].each do |parser|
       begin
         return parser.process s, f, t
       rescue Racc::ParseError, RubyParser::SyntaxError => exc
@@ -1376,6 +1381,7 @@ class RubyParser
     @p20.reset
     @p21.reset
     @p22.reset
+    @p23.reset
   end
 
   def self.for_current_ruby
@@ -1390,6 +1396,8 @@ class RubyParser
       Ruby21Parser.new
     when /^2.2/ then
       Ruby22Parser.new
+    when /^2.3/ then
+      Ruby23Parser.new
     else
       raise "unrecognized RUBY_VERSION #{RUBY_VERSION}"
     end

data/test/test_ruby_lexer.rb

@@ -245,6 +245,21 @@ class TestRubyLexer < Minitest::Test
     assert_lex3("&&=", nil, :tOP_ASGN, "&&", :expr_beg)
   end
 
+  def test_yylex_and_dot
+    setup_lexer_class Ruby23Parser
+
+    assert_lex3("&.", nil, :tLONELY, "&.", :expr_dot)
+  end
+
+  def test_yylex_and_dot_call
+    setup_lexer_class Ruby23Parser
+
+    assert_lex3("x&.y", nil,
+                :tIDENTIFIER, "x", :expr_cmdarg,
+                :tLONELY, "&.", :expr_dot,
+                :tIDENTIFIER, "y")
+  end
+
   def test_yylex_and_arg
     self.lex_state = :expr_arg
 
@@ -2653,10 +2668,34 @@ class TestRubyLexer < Minitest::Test
     assert_lex("{'a':1}",
                s(:hash, s(:lit, :a), s(:lit, 1)),
 
-               :tLBRACE, "{", :expr_beg,    0, 1,
-               :tLABEL,  "a", :expr_end,    0, 1,
-               :tINTEGER, 1,  :expr_end,    0, 1,
-               :tRCURLY, "}", :expr_endarg, 0, 0)
+               :tLBRACE, "{", :expr_beg,      0, 1,
+               :tLABEL,  "a", :expr_labelarg, 0, 1,
+               :tINTEGER, 1,  :expr_end,      0, 1,
+               :tRCURLY, "}", :expr_endarg,   0, 0)
+  end
+
+  def test_yylex_hash_colon_quoted_symbol_22
+    setup_lexer_class Ruby22Parser
+
+    assert_lex("{'abc': :b}",
+               s(:hash, s(:lit, :abc), s(:lit, :b)),
+
+               :tLBRACE, "{", :expr_beg,      0, 1,
+               :tLABEL,  "abc", :expr_labelarg, 0, 1,
+               :tSYMBOL, "b",  :expr_end,     0, 1,
+               :tRCURLY, "}", :expr_endarg,   0, 0)
+  end
+
+  def test_yylex_hash_colon_double_quoted_symbol_22
+    setup_lexer_class Ruby22Parser
+
+    assert_lex('{"abc": :b}',
+               s(:hash, s(:lit, :abc), s(:lit, :b)),
+
+               :tLBRACE, "{", :expr_beg,      0, 1,
+               :tLABEL,  "abc", :expr_labelarg, 0, 1,
+               :tSYMBOL, "b",  :expr_end,     0, 1,
+               :tRCURLY, "}", :expr_endarg,   0, 0)
   end
 
   def test_ruby21_new_numbers

data/test/test_ruby_parser.rb

@@ -961,6 +961,10 @@ module TestRubyParserShared
     Ruby22Parser === self.processor
   end
 
+  def ruby23
+    Ruby23Parser === self.processor
+  end
+
   def test_bug_comma
     val = if ruby18 then
             s(:lit, 100)
@@ -1016,7 +1020,7 @@ module TestRubyParserShared
     rb = "not(a)"
     pt = if ruby18 then
            s(:not, s(:call, nil, :a))
-         elsif ruby19 or ruby20 or ruby21 or ruby22 then
+         elsif ruby19 or ruby20 or ruby21 or ruby22 or ruby23 then
            s(:call, s(:call, nil, :a), :"!")
          else
            raise "wtf"
@@ -3384,9 +3388,78 @@ class TestRuby22Parser < RubyParserTestCase
     rb = "a ? \"\": b"
     assert_parse rb, pt
   end
+
+  def test_quoted_symbol_keys
+    rb = "{ 'a': :b }"
+    pt = s(:hash, s(:lit, :a), s(:lit, :b))
+
+    assert_parse rb, pt
+  end
+
+  def test_quoted_symbol_hash_arg
+    rb = "puts 'a': {}"
+    pt = s(:call, nil, :puts, s(:hash, s(:lit, :a), s(:hash)))
+
+    assert_parse rb, pt
+  end
 end
 
-[18, 19, 20, 21, 22].each do |v|
+class TestRuby23Parser < RubyParserTestCase
+  include TestRubyParserShared
+  include TestRubyParserShared19to22
+  include TestRubyParserShared20to22
+
+  def setup
+    super
+
+    self.processor = Ruby23Parser.new
+  end
+
+  def test_safe_call
+    rb = "a&.b"
+    pt = s(:safe_call, s(:call, nil, :a), :b)
+
+    assert_parse rb, pt
+  end
+
+  def test_safe_calls
+    rb = "a&.b&.c(1)"
+    pt = s(:safe_call, s(:safe_call, s(:call, nil, :a), :b), :c, s(:lit, 1))
+
+    assert_parse rb, pt
+  end
+
+  def test_safe_attrasgn
+    rb = "a&.b = 1"
+    pt = s(:safe_attrasgn, s(:call, nil, :a), :"b=", s(:lit, 1))
+
+    assert_parse rb, pt
+  end
+
+  def test_safe_attrasgn_constant
+    rb = "a&.B = 1"
+    pt = s(:safe_attrasgn, s(:call, nil, :a), :"B=", s(:lit, 1))
+
+    assert_parse rb, pt
+  end
+
+  def test_safe_call_dot_parens
+    rb = "a&.()"
+    pt = s(:safe_call, s(:call, nil, :a), :call)
+
+    assert_parse rb, pt
+  end
+
+  def test_safe_call_operator
+    rb = "a&.> 1"
+    pt = s(:safe_call, s(:call, nil, :a), :>, s(:lit, 1))
+
+    assert_parse rb, pt
+  end
+end
+
+
+[18, 19, 20, 21, 22, 23].each do |v|
   describe "block args arity #{v}" do
     attr_accessor :parser