parser 2.6.5.0 → 2.7.0.2

data/lib/parser/lexer.rl

@@ -89,16 +89,13 @@ class Parser::Lexer
 
   REGEXP_META_CHARACTERS = Regexp.union(*"\\$()*+.<>?[]^{|}".chars).freeze
 
-  NUMPARAM_MAX = 9
-
   attr_reader   :source_buffer
-  attr_reader   :max_numparam_stack
 
   attr_accessor :diagnostics
   attr_accessor :static_env
   attr_accessor :force_utf32
 
-  attr_accessor :cond, :cmdarg, :in_kwarg, :context
+  attr_accessor :cond, :cmdarg, :in_kwarg, :context, :command_start
 
   attr_accessor :tokens, :comments
 
@@ -179,9 +176,6 @@ class Parser::Lexer
 
     # State before =begin / =end block comment
     @cs_before_block_comment = self.class.lex_en_line_begin
-
-    # Maximum numbered parameters stack
-    @max_numparam_stack = MaxNumparamStack.new
   end
 
   def source_buffer=(source_buffer)
@@ -255,10 +249,6 @@ class Parser::Lexer
     @cond = @cond_stack.pop
   end
 
-  def max_numparam
-    @max_numparam_stack.top
-  end
-
   def dedent_level
     # We erase @dedent_level as a precaution to avoid accidentally
     # using a stale value.
@@ -457,7 +447,7 @@ class Parser::Lexer
     '=>'  => :tASSOC,   '::'  => :tCOLON2,  '===' => :tEQQ,
     '<=>' => :tCMP,     '[]'  => :tAREF,    '[]=' => :tASET,
     '{'   => :tLCURLY,  '}'   => :tRCURLY,  '`'   => :tBACK_REF2,
-    '!@'  => :tBANG,    '&.'  => :tANDDOT,  '.:'  => :tMETHREF
+    '!@'  => :tBANG,    '&.'  => :tANDDOT,
   }
 
   PUNCTUATION_BEGIN = {
@@ -1029,6 +1019,20 @@ class Parser::Lexer
     fcall expr_variable;
   }
 
+  # Special case for Ruby > 2.7
+  # If interpolated instance/class variable starts with a digit we parse it as a plain substring
+  # However, "#$1" is still a regular interpolation
+  interp_digit_var = '#' ('@' | '@@') digit c_alpha*;
+
+  action extend_interp_digit_var {
+    if @version >= 27
+      literal.extend_string(tok, @ts, @te)
+    else
+      message = tok.start_with?('#@@') ? :cvar_name : :ivar_name
+      diagnostic :error, message, { :name => tok(@ts + 1, @te) }, range(@ts + 1, @te)
+    end
+  }
+
   # Interpolations with code blocks must match nested curly braces, as
   # interpolation ending is ambiguous with a block ending. So, every
   # opening and closing brace should be matched with e_[lr]brace rules,
@@ -1074,6 +1078,8 @@ class Parser::Lexer
         fbreak;
       end
     end
+
+    @paren_nest -= 1
   };
 
   action extend_interp_code {
@@ -1098,60 +1104,64 @@ class Parser::Lexer
   # above.
 
   interp_words := |*
-      interp_code => extend_interp_code;
-      interp_var  => extend_interp_var;
-      e_bs escape => extend_string_escaped;
-      c_space+    => extend_string_space;
-      c_eol       => extend_string_eol;
-      c_any       => extend_string;
+      interp_code      => extend_interp_code;
+      interp_digit_var => extend_interp_digit_var;
+      interp_var       => extend_interp_var;
+      e_bs escape      => extend_string_escaped;
+      c_space+         => extend_string_space;
+      c_eol            => extend_string_eol;
+      c_any            => extend_string;
   *|;
 
   interp_string := |*
-      interp_code => extend_interp_code;
-      interp_var  => extend_interp_var;
-      e_bs escape => extend_string_escaped;
-      c_eol       => extend_string_eol;
-      c_any       => extend_string;
+      interp_code      => extend_interp_code;
+      interp_digit_var => extend_interp_digit_var;
+      interp_var       => extend_interp_var;
+      e_bs escape      => extend_string_escaped;
+      c_eol            => extend_string_eol;
+      c_any            => extend_string;
   *|;
 
   plain_words := |*
-      e_bs c_any  => extend_string_escaped;
-      c_space+    => extend_string_space;
-      c_eol       => extend_string_eol;
-      c_any       => extend_string;
+      e_bs c_any       => extend_string_escaped;
+      c_space+         => extend_string_space;
+      c_eol            => extend_string_eol;
+      c_any            => extend_string;
   *|;
 
   plain_string := |*
-      '\\' c_nl   => extend_string_eol;
-      e_bs c_any  => extend_string_escaped;
-      c_eol       => extend_string_eol;
-      c_any       => extend_string;
+      '\\' c_nl        => extend_string_eol;
+      e_bs c_any       => extend_string_escaped;
+      c_eol            => extend_string_eol;
+      c_any            => extend_string;
   *|;
 
   interp_backslash_delimited := |*
-      interp_code => extend_interp_code;
-      interp_var  => extend_interp_var;
-      c_eol       => extend_string_eol;
-      c_any       => extend_string;
+      interp_code      => extend_interp_code;
+      interp_digit_var => extend_interp_digit_var;
+      interp_var       => extend_interp_var;
+      c_eol            => extend_string_eol;
+      c_any            => extend_string;
   *|;
 
   plain_backslash_delimited := |*
-      c_eol       => extend_string_eol;
-      c_any       => extend_string;
+      c_eol            => extend_string_eol;
+      c_any            => extend_string;
   *|;
 
   interp_backslash_delimited_words := |*
-      interp_code => extend_interp_code;
-      interp_var  => extend_interp_var;
-      c_space+    => extend_string_space;
-      c_eol       => extend_string_eol;
-      c_any       => extend_string;
+      interp_code      => extend_interp_code;
+      interp_digit_var => extend_interp_digit_var;
+      interp_var       => extend_interp_var;
+      c_space+         => extend_string_space;
+      c_eol            => extend_string_eol;
+      c_any            => extend_string;
   *|;
 
   plain_backslash_delimited_words := |*
-      c_space+    => extend_string_space;
-      c_eol       => extend_string_eol;
-      c_any       => extend_string;
+      c_space+         => extend_string_space;
+      c_eol            => extend_string_eol;
+      c_any            => extend_string;
   *|;
 
   regexp_modifiers := |*
@@ -1267,6 +1277,12 @@ class Parser::Lexer
 
   e_lbrack = '[' % {
     @cond.push(false); @cmdarg.push(false)
+
+    @paren_nest += 1
+  };
+
+  e_rbrack = ']' % {
+    @paren_nest -= 1
   };
 
   # Ruby 1.9 lambdas require parentheses counting in order to
@@ -1324,36 +1340,6 @@ class Parser::Lexer
         fnext *stack_pop; fbreak;
       };
 
-      '@' [0-9]+
-      => {
-        if @version < 27
-          diagnostic :error, :ivar_name, { :name => tok }
-        end
-
-        value = tok[1..-1]
-
-        if value[0] == '0'
-          diagnostic :error, :leading_zero_in_numparam, nil, range(@ts, @te)
-        end
-
-        if value.to_i > NUMPARAM_MAX
-          diagnostic :error, :too_large_numparam, nil, range(@ts, @te)
-        end
-
-        if !@context.in_block? && !@context.in_lambda?
-          diagnostic :error, :numparam_outside_block, nil, range(@ts, @te)
-        end
-
-        if !@max_numparam_stack.can_have_numparams?
-          diagnostic :error, :ordinary_param_defined, nil, range(@ts, @te)
-        end
-
-        @max_numparam_stack.register(value.to_i)
-
-        emit(:tNUMPARAM, tok[1..-1])
-        fnext *stack_pop; fbreak;
-      };
-
       instance_var_v
       => {
         if tok =~ /^@[0-9]/
@@ -1519,6 +1505,7 @@ class Parser::Lexer
           emit(:tLCURLY, '{'.freeze, @te - 1, @te)
         end
         @command_start = true
+        @paren_nest += 1
         fnext expr_value; fbreak;
       };
 
@@ -1679,6 +1666,7 @@ class Parser::Lexer
         else
           emit(:tLBRACE_ARG, '{'.freeze)
         end
+        @paren_nest += 1
         @command_start = true
         fnext expr_value; fbreak;
       };
@@ -1927,6 +1915,24 @@ class Parser::Lexer
         fgoto expr_end;
       };
 
+      #
+      # AMBIGUOUS EMPTY BLOCK ARGUMENTS
+      #
+
+      # Ruby >= 2.7 emits it as two tPIPE terminals
+      # while Ruby < 2.7 as a single tOROP (like in `a || b`)
+      '||'
+      => {
+        if @version >= 27
+          emit(:tPIPE, tok(@ts, @ts + 1), @ts, @ts + 1)
+          fhold;
+          fnext expr_beg; fbreak;
+        else
+          p -= 2
+          fgoto expr_end;
+        end
+      };
+
       #
       # KEYWORDS AND PUNCTUATION
       #
@@ -1941,6 +1947,7 @@ class Parser::Lexer
         else
           emit(:tLBRACE, '{'.freeze)
         end
+        @paren_nest += 1
         fbreak;
       };
 
@@ -2159,6 +2166,9 @@ class Parser::Lexer
             emit_do
           end
         end
+        if tok == '{'.freeze
+          @paren_nest += 1
+        end
         @command_start = true
 
         fnext expr_value; fbreak;
@@ -2333,24 +2343,6 @@ class Parser::Lexer
       # METHOD CALLS
       #
 
-      '.:' w_space+
-      => { emit(:tDOT, '.', @ts, @ts + 1)
-           emit(:tCOLON, ':', @ts + 1, @ts + 2)
-           p = p - tok.length + 2
-           fnext expr_dot; fbreak; };
-
-      '.:'
-      => {
-        if @version >= 27
-          emit_table(PUNCTUATION)
-        else
-          emit(:tDOT, tok(@ts, @ts + 1), @ts, @ts + 1)
-          fhold;
-        end
-
-        fnext expr_dot; fbreak;
-      };
-
       '.' | '&.' | '::'
       => { emit_table(PUNCTUATION)
            fnext expr_dot; fbreak; };
@@ -2394,7 +2386,7 @@ class Parser::Lexer
       => { emit_table(PUNCTUATION)
            fnext expr_beg; fbreak; };
 
-      e_rbrace | e_rparen | ']'
+      e_rbrace | e_rparen | e_rbrack
       => {
         emit_table(PUNCTUATION)
 
@@ -2431,6 +2423,17 @@ class Parser::Lexer
       => { emit(:tLBRACK2, '['.freeze)
            fnext expr_beg; fbreak; };
 
+      '...' c_nl
+      => {
+        if @paren_nest == 0
+          diagnostic :warning, :triple_dot_at_eol, nil, range(@ts, @te - 1)
+        end
+
+        emit(:tDOT3, '...'.freeze, @ts, @te - 1)
+        fhold;
+        fnext expr_beg; fbreak;
+      };
+
       punctuation_end
       => { emit_table(PUNCTUATION)
            fnext expr_beg; fbreak; };

data/lib/parser/lexer/dedenter.rb

@@ -3,72 +3,75 @@
 module Parser
 
   class Lexer::Dedenter
+    # Tab (\t) counts as 8 spaces
+    TAB_WIDTH = 8
+
     def initialize(dedent_level)
       @dedent_level = dedent_level
       @at_line_begin = true
       @indent_level  = 0
     end
 
+    # For a heredoc like
+    #   <<-HERE
+    #     a
+    #     b
+    #   HERE
+    # this method gets called with "  a\n" and "  b\n"
+    #
+    # However, the following heredoc:
+    #
+    #   <<-HERE
+    #     a\
+    #     b
+    #   HERE
+    # calls this method only once with a string "  a\\\n  b\n"
+    #
+    # This is important because technically it's a single line,
+    # but it has to be concatenated __after__ dedenting.
+    #
+    # It has no effect for non-squiggly heredocs, i.e. it simply removes "\\\n"
+    # Of course, lexer could do it but once again: it's all because of dedenting.
+    #
     def dedent(string)
-      space_begin = space_end = offset = 0
-      last_index  = string.length - 1
-      escape = false
-      _at_line_begin = nil
+      original_encoding = string.encoding
+      # Prevent the following error when processing binary encoded source.
+      # "\xC0".split # => ArgumentError (invalid byte sequence in UTF-8)
+      lines = string.force_encoding(Encoding::BINARY).split("\\\n")
+      lines.map! {|s| s.force_encoding(original_encoding) }
 
-      string.chars.each_with_index do |char, index|
-        if char == '\\'
-          # entering escape mode
-          escape = true
-          string.slice!(index - offset)
-          offset += 1
-          _at_line_begin = @at_line_begin
-          @at_line_begin = false
-        elsif escape
-          if char == ?\n
-            # trimming \n, starting a new line
-            string.slice!(index - offset)
-            offset += 1
-            @at_line_begin = true
-            space_begin = space_end = index - offset
-            @indent_level = 0
-          elsif char == ?n
-            # replacing \\n to \n
-            string.slice!(index - offset)
-            string.insert(index - offset, ?\n)
-          else
-            # exiting escape mode as it's not an escape sequence
-            @at_line_begin = _at_line_begin
-            escape = false
-            redo
-          end
-          escape = false
-        elsif @at_line_begin
-          if char == ?\n || @indent_level >= @dedent_level
-            string.slice!(space_begin...space_end)
-            offset += space_end - space_begin
-            @at_line_begin = false
-          end
+      if @at_line_begin
+        lines_to_dedent = lines
+      else
+        _first, *lines_to_dedent = lines
+      end
 
+      lines_to_dedent.each do |line|
+        left_to_remove = @dedent_level
+        remove = 0
+
+        line.each_char do |char|
+          break if left_to_remove <= 0
           case char
           when ?\s
-            @indent_level += 1
-            space_end += 1
+            remove += 1
+            left_to_remove -= 1
           when ?\t
-            @indent_level += 8 - @indent_level % 8
-            space_end += 1
+            break if TAB_WIDTH * (remove / TAB_WIDTH + 1) > @dedent_level
+            remove += 1
+            left_to_remove -= TAB_WIDTH
+          else
+            # no more spaces or tabs
+            break
           end
-        elsif char == ?\n && index == last_index
-          @at_line_begin = true
-          @indent_level  = 0
-          space_begin = space_end = index - offset + 1
         end
-      end
 
-      if @at_line_begin
-        string.slice!(space_begin..space_end)
+        line.slice!(0, remove)
       end
 
-      nil
+      string.replace(lines.join)
+
+      @at_line_begin = string.end_with?("\n")
     end
 
     def interrupt