parser 2.6.3.0 → 2.6.4.0

data/lib/parser/lexer/max_numparam_stack.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Parser
+
+  class Lexer::MaxNumparamStack
+    def initialize
+      @stack = []
+    end
+
+    def cant_have_numparams!
+      set(-1)
+    end
+
+    def can_have_numparams?
+      top >= 0
+    end
+
+    def register(numparam)
+      set( [top, numparam].max )
+    end
+
+    def top
+      @stack.last
+    end
+
+    def push
+      @stack.push(0)
+    end
+
+    def pop
+      @stack.pop
+    end
+
+    private
+
+    def set(value)
+      @stack.pop
+      @stack.push(value)
+    end
+  end
+
+end

data/lib/parser/messages.rb

@@ -10,26 +10,31 @@ module Parser
   #
   MESSAGES = {
     # Lexer errors
-    :unicode_point_too_large => 'invalid Unicode codepoint (too large)',
-    :invalid_escape          => 'invalid escape character syntax',
-    :incomplete_escape       => 'incomplete character syntax',
-    :invalid_hex_escape      => 'invalid hex escape',
-    :invalid_unicode_escape  => 'invalid Unicode escape',
-    :unterminated_unicode    => 'unterminated Unicode escape',
-    :escape_eof              => 'escape sequence meets end of file',
-    :string_eof              => 'unterminated string meets end of file',
-    :regexp_options          => 'unknown regexp options: %{options}',
-    :cvar_name               => "`%{name}' is not allowed as a class variable name",
-    :ivar_name               => "`%{name}' is not allowed as an instance variable name",
-    :trailing_in_number      => "trailing `%{character}' in number",
-    :empty_numeric           => 'numeric literal without digits',
-    :invalid_octal           => 'invalid octal digit',
-    :no_dot_digit_literal    => 'no .<digit> floating literal anymore; put 0 before dot',
-    :bare_backslash          => 'bare backslash only allowed before newline',
-    :unexpected              => "unexpected `%{character}'",
-    :embedded_document       => 'embedded document meets end of file (and they embark on a romantic journey)',
-    :heredoc_id_has_newline  => 'here document identifier across newlines, never match',
-    :heredoc_id_ends_with_nl => 'here document identifier ends with a newline',
+    :unicode_point_too_large  => 'invalid Unicode codepoint (too large)',
+    :invalid_escape           => 'invalid escape character syntax',
+    :incomplete_escape        => 'incomplete character syntax',
+    :invalid_hex_escape       => 'invalid hex escape',
+    :invalid_unicode_escape   => 'invalid Unicode escape',
+    :unterminated_unicode     => 'unterminated Unicode escape',
+    :escape_eof               => 'escape sequence meets end of file',
+    :string_eof               => 'unterminated string meets end of file',
+    :regexp_options           => 'unknown regexp options: %{options}',
+    :cvar_name                => "`%{name}' is not allowed as a class variable name",
+    :ivar_name                => "`%{name}' is not allowed as an instance variable name",
+    :trailing_in_number       => "trailing `%{character}' in number",
+    :empty_numeric            => 'numeric literal without digits',
+    :invalid_octal            => 'invalid octal digit',
+    :no_dot_digit_literal     => 'no .<digit> floating literal anymore; put 0 before dot',
+    :bare_backslash           => 'bare backslash only allowed before newline',
+    :unexpected               => "unexpected `%{character}'",
+    :embedded_document        => 'embedded document meets end of file (and they embark on a romantic journey)',
+    :heredoc_id_has_newline   => 'here document identifier across newlines, never match',
+    :heredoc_id_ends_with_nl  => 'here document identifier ends with a newline',
+    :unterminated_heredoc_id  => 'unterminated heredoc id',
+    :leading_zero_in_numparam => 'leading zero is not allowed as a numbered parameter',
+    :numparam_outside_block   => 'numbered parameter outside block',
+    :too_large_numparam       => 'too large numbered parameter',
+    :ordinary_param_defined   => 'ordinary parameter is defined',
 
     # Lexer warnings
     :invalid_escape_use      => 'invalid character syntax; use ?%{escape}',
@@ -61,6 +66,7 @@ module Parser
     :block_given_to_yield    => 'block given to yield',
     :invalid_regexp          => '%{message}',
     :invalid_return          => 'Invalid return in class/module body',
+    :csend_in_lhs_of_masgn   => '&. inside multiple assignment destination',
 
     # Parser warnings
     :useless_else            => 'else without rescue is useless',

data/lib/parser/meta.rb

@@ -26,6 +26,7 @@ module Parser
         ident root lambda indexasgn index procarg0
         meth_ref restarg_expr blockarg_expr
         objc_kwarg objc_restarg objc_varargs
+        numargs numblock numparam
       ).map(&:to_sym).to_set.freeze
 
   end # Meta

data/lib/parser/ruby25.y

@@ -1885,7 +1885,7 @@ regexp_contents: # nothing
                       result = @builder.symbol(val[0])
                     }
 
-            dsym: tSYMBEG xstring_contents tSTRING_END
+            dsym: tSYMBEG string_contents tSTRING_END
                     {
                       @lexer.state = :expr_end
                       result = @builder.symbol_compose(val[0], val[1], val[2])

data/lib/parser/ruby27.y

@@ -17,7 +17,7 @@ token kCLASS kMODULE kDEF kUNDEF kBEGIN kRESCUE kENSURE kEND kIF kUNLESS
       tWORDS_BEG tQWORDS_BEG tSYMBOLS_BEG tQSYMBOLS_BEG tSTRING_DBEG
       tSTRING_DVAR tSTRING_END tSTRING_DEND tSTRING tSYMBOL
       tNL tEH tCOLON tCOMMA tSPACE tSEMI tLAMBDA tLAMBEG tCHARACTER
-      tRATIONAL tIMAGINARY tLABEL_END tANDDOT tMETHREF tBDOT2 tBDOT3
+      tRATIONAL tIMAGINARY tLABEL_END tANDDOT tMETHREF tBDOT2 tBDOT3 tNUMPARAM
 
 prechigh
   right    tBANG tTILDE tUPLUS
@@ -193,6 +193,15 @@ rule
                       result = @builder.assign(val[0], val[1],
                                   @builder.array(nil, val[2], nil))
                     }
+                | mlhs tEQL mrhs_arg kRESCUE_MOD stmt
+                    {
+                      rescue_body = @builder.rescue_body(val[3],
+                                                         nil, nil, nil,
+                                                         nil, val[4])
+                      begin_body = @builder.begin_body(val[2], [ rescue_body ])
+
+                      result = @builder.multi_assign(val[0], val[1], begin_body)
+                    }
                 | mlhs tEQL mrhs_arg
                     {
                       result = @builder.multi_assign(val[0], val[1], val[2])
@@ -480,6 +489,10 @@ rule
                     }
                 | primary_value call_op tIDENTIFIER
                     {
+                      if (val[1][0] == :anddot)
+                        diagnostic :error, :csend_in_lhs_of_masgn, nil, val[1]
+                      end
+
                       result = @builder.attr_asgn(val[0], val[1], val[2])
                     }
                 | primary_value tCOLON2 tIDENTIFIER
@@ -488,6 +501,10 @@ rule
                     }
                 | primary_value call_op tCONSTANT
                     {
+                      if (val[1][0] == :anddot)
+                        diagnostic :error, :csend_in_lhs_of_masgn, nil, val[1]
+                      end
+
                       result = @builder.attr_asgn(val[0], val[1], val[2])
                     }
                 | primary_value tCOLON2 tCONSTANT
@@ -1294,45 +1311,33 @@ rule
                     }
 
          f_margs: f_marg_list
-                | f_marg_list tCOMMA tSTAR f_norm_arg
+                | f_marg_list tCOMMA f_rest_marg
                     {
                       result = val[0].
-                                  push(@builder.restarg(val[2], val[3]))
+                                  push(val[2])
                     }
-                | f_marg_list tCOMMA tSTAR f_norm_arg tCOMMA f_marg_list
+                | f_marg_list tCOMMA f_rest_marg tCOMMA f_marg_list
                     {
                       result = val[0].
-                                  push(@builder.restarg(val[2], val[3])).
-                                  concat(val[5])
-                    }
-                | f_marg_list tCOMMA tSTAR
-                    {
-                      result = val[0].
-                                  push(@builder.restarg(val[2]))
-                    }
-                | f_marg_list tCOMMA tSTAR            tCOMMA f_marg_list
-                    {
-                      result = val[0].
-                                  push(@builder.restarg(val[2])).
+                                  push(val[2]).
                                   concat(val[4])
                     }
-                |                    tSTAR f_norm_arg
+                |                    f_rest_marg
                     {
-                      result = [ @builder.restarg(val[0], val[1]) ]
+                      result = [ val[0] ]
                     }
-                |                    tSTAR f_norm_arg tCOMMA f_marg_list
+                |                    f_rest_marg tCOMMA f_marg_list
                     {
-                      result = [ @builder.restarg(val[0], val[1]),
-                                 *val[3] ]
+                      result = [ val[0], *val[2] ]
                     }
-                |                    tSTAR
+
+     f_rest_marg: tSTAR f_norm_arg
                     {
-                      result = [ @builder.restarg(val[0]) ]
+                      result = @builder.restarg(val[0], val[1])
                     }
-                |                    tSTAR tCOMMA f_marg_list
+                | tSTAR
                     {
-                      result = [ @builder.restarg(val[0]),
-                                 *val[2] ]
+                      result = @builder.restarg(val[0])
                     }
 
  block_args_tail: f_block_kwarg tCOMMA f_kwrest opt_f_block_arg
@@ -1460,14 +1465,17 @@ opt_block_args_tail:
 
  block_param_def: tPIPE opt_bv_decl tPIPE
                     {
+                      @lexer.max_numparam_stack.cant_have_numparams!
                       result = @builder.args(val[0], val[1], val[2])
                     }
                 | tOROP
                     {
+                      @lexer.max_numparam_stack.cant_have_numparams!
                       result = @builder.args(val[0], [], val[0])
                     }
                 | tPIPE block_param opt_bv_decl tPIPE
                     {
+                      @lexer.max_numparam_stack.cant_have_numparams!
                       result = @builder.args(val[0], val[1].concat(val[2]), val[3])
                     }
 
@@ -1498,26 +1506,34 @@ opt_block_args_tail:
 
           lambda:   {
                       @static_env.extend_dynamic
+                      @lexer.max_numparam_stack.push
+                      @context.push(:lambda)
                     }
                   f_larglist
                     {
+                      @context.pop
                       @lexer.cmdarg.push(false)
                     }
                   lambda_body
                     {
-                      @lexer.cmdarg.pop
-
-                      result = [ val[1], val[3] ]
+                      args = @lexer.max_numparam > 0 ? @builder.numargs(@lexer.max_numparam) : val[1]
+                      result = [ args, val[3] ]
 
+                      @lexer.max_numparam_stack.pop
                       @static_env.unextend
+                      @lexer.cmdarg.pop
                     }
 
      f_larglist: tLPAREN2 f_args opt_bv_decl tRPAREN
                     {
+                      @lexer.max_numparam_stack.cant_have_numparams!
                       result = @builder.args(val[0], val[1].concat(val[2]), val[3])
                     }
                 | f_args
                     {
+                      if val[0].any?
+                        @lexer.max_numparam_stack.cant_have_numparams!
+                      end
                       result = @builder.args(nil, val[0], nil)
                     }
 
@@ -1652,24 +1668,30 @@ opt_block_args_tail:
 
       brace_body:   {
                       @static_env.extend_dynamic
+                      @lexer.max_numparam_stack.push
                     }
                     opt_block_param compstmt
                     {
-                      result = [ val[1], val[2] ]
+                      args = @lexer.max_numparam > 0 ? @builder.numargs(@lexer.max_numparam) : val[1]
+                      result = [ args, val[2] ]
 
+                      @lexer.max_numparam_stack.pop
                       @static_env.unextend
                     }
 
          do_body:   {
                       @static_env.extend_dynamic
+                      @lexer.max_numparam_stack.push
                     }
                     {
                       @lexer.cmdarg.push(false)
                     }
                     opt_block_param bodystmt
                     {
-                      result = [ val[2], val[3] ]
+                      args = @lexer.max_numparam > 0 ? @builder.numargs(@lexer.max_numparam) : val[2]
+                      result = [ args, val[3] ]
 
+                      @lexer.max_numparam_stack.pop
                       @static_env.unextend
                       @lexer.cmdarg.pop
                     }
@@ -1892,6 +1914,10 @@ regexp_contents: # nothing
                     {
                       result = @builder.cvar(val[0])
                     }
+                | tNUMPARAM
+                    {
+                      result = @builder.numparam(val[0])
+                    }
                 | backref
 
           symbol: ssym
@@ -1964,6 +1990,10 @@ regexp_contents: # nothing
                     {
                       result = @builder.cvar(val[0])
                     }
+                | tNUMPARAM
+                    {
+                      result = @builder.numparam(val[0])
+                    }
 
 keyword_variable: kNIL
                     {
@@ -2188,6 +2218,8 @@ keyword_variable: kNIL
                     {
                       @static_env.declare val[0][0]
 
+                      @lexer.max_numparam_stack.cant_have_numparams!
+
                       result = val[0]
                     }
 
@@ -2220,6 +2252,8 @@ keyword_variable: kNIL
 
                       @static_env.declare val[0][0]
 
+                      @lexer.max_numparam_stack.cant_have_numparams!
+
                       result = val[0]
                     }
 

data/lib/parser/runner/ruby_parse.rb

@@ -123,7 +123,7 @@ module Parser
       opts.on '--emit-ruby', 'Emit S-expressions as valid Ruby code' do
         @emit_ruby = true
       end
-      
+
       opts.on '--emit-json', 'Emit S-expressions as valid JSON' do
         @emit_json = true
       end
@@ -146,7 +146,7 @@ module Parser
         if @emit_ruby
           puts ast.inspect
         elsif @emit_json
-          puts JSON.generate(ast.to_sexp_array)
+          puts(ast ? JSON.generate(ast.to_sexp_array) : nil)
         else
           puts ast.to_s
         end

data/lib/parser/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Parser
-  VERSION = '2.6.3.0'
+  VERSION = '2.6.4.0'
 end

data/parser.gemspec

@@ -38,7 +38,7 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency 'bundler',   '>= 1.15', '< 3.0.0'
   spec.add_development_dependency 'rake',      '~> 10.0'
-  spec.add_development_dependency 'racc',      '= 1.4.14'
+  spec.add_development_dependency 'racc',      '= 1.4.15'
   spec.add_development_dependency 'cliver',    '~> 0.3.2'
 
   spec.add_development_dependency 'yard'

data/test/test_lexer.rb

@@ -162,6 +162,24 @@ class TestLexer < Minitest::Test
     refute_escape 'u{123 f0'
   end
 
+  def test_read_escape_whitespaces__27
+    setup_lexer 27
+
+    [ *(0..8), *(14..31) ].each do |code|
+      @lex.reset
+      refute_scanned "\"\\C-" + code.chr + "\""
+
+      @lex.reset
+      refute_scanned "\"\\M-" + code.chr + "\""
+
+      @lex.reset
+      refute_scanned "\"\\C-\\M-" + code.chr + "\""
+
+      @lex.reset
+      refute_scanned "\"\\M-\\C-" + code.chr + "\""
+    end
+  end
+
   def test_ambiguous_uminus
     assert_scanned("m -3",
                    :tIDENTIFIER, "m", [0, 1],
@@ -3581,4 +3599,61 @@ class TestLexer < Minitest::Test
                   :tPLUS,       '+',   [6, 7])
   end
 
+  def lex_numbered_parameter(input)
+    @lex.max_numparam_stack.push
+
+    @lex.context = Parser::Context.new
+    @lex.context.push(:block)
+
+    source_buffer = Parser::Source::Buffer.new('(assert_lex_numbered_parameter)')
+    source_buffer.source = input
+
+    @lex.source_buffer = source_buffer
+
+    @lex.advance
+  end
+
+  def assert_scanned_numbered_parameter(input)
+    lex_token, (lex_value, lex_range) = lex_numbered_parameter(input)
+
+    assert_equal(lex_token, :tNUMPARAM)
+    assert_equal(lex_value, input.tr('@', ''))
+    assert_equal(lex_range.begin_pos, 0)
+    assert_equal(lex_range.end_pos, input.length)
+  end
+
+  def refute_scanned_numbered_parameter(input, message = nil)
+    err = assert_raises Parser::SyntaxError do
+      lex_token, (lex_value, lex_range) = lex_numbered_parameter(input)
+    end
+
+    if message
+      assert_equal(err.message, Parser::MESSAGES[message])
+
+      assert_equal(err.diagnostic.location.begin_pos, 0)
+      assert_equal(err.diagnostic.location.end_pos, input.length)
+    end
+  end
+
+  def test_numbered_args_before_27
+    setup_lexer(26)
+    refute_scanned_numbered_parameter('@1')
+  end
+
+  def test_numbered_args_27
+    setup_lexer(27)
+    assert_scanned_numbered_parameter('@1')
+    assert_equal(@lex.max_numparam, 1)
+
+    setup_lexer(27)
+    assert_scanned_numbered_parameter('@100')
+    assert_equal(@lex.max_numparam, 100)
+
+    setup_lexer(27)
+    refute_scanned_numbered_parameter('@101', :too_large_numparam)
+
+    setup_lexer(27)
+    refute_scanned_numbered_parameter('@01', :leading_zero_in_numparam)
+  end
+
 end