parser 2.4.0.2 → 2.5.0.0

data/lib/parser/current.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Parser
   class << self
     def warn_syntax_deviation(feature, version)
@@ -9,24 +11,6 @@ module Parser
   end
 
   case RUBY_VERSION
-  when /^1\.8\./
-    current_version = '1.8.7'
-    if RUBY_VERSION != current_version
-      warn_syntax_deviation 'parser/ruby18', current_version
-    end
-
-    require 'parser/ruby18'
-    CurrentRuby = Ruby18
-
-  when /^1\.9\./
-    current_version = '1.9.3'
-    if RUBY_VERSION != current_version
-      warn_syntax_deviation 'parser/ruby19', current_version
-    end
-
-    require 'parser/ruby19'
-    CurrentRuby = Ruby19
-
   when /^2\.0\./
     current_version = '2.0.0'
     if RUBY_VERSION != current_version
@@ -46,7 +30,7 @@ module Parser
     CurrentRuby = Ruby21
 
   when /^2\.2\./
-    current_version = '2.2.8'
+    current_version = '2.2.9'
     if RUBY_VERSION != current_version
       warn_syntax_deviation 'parser/ruby22', current_version
     end
@@ -55,7 +39,7 @@ module Parser
     CurrentRuby = Ruby22
 
   when /^2\.3\./
-    current_version = '2.3.5'
+    current_version = '2.3.6'
     if RUBY_VERSION != current_version
       warn_syntax_deviation 'parser/ruby23', current_version
     end

data/lib/parser/deprecation.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Parser
+  ##
+  # @api private
+  #
+  module Deprecation
+    attr_writer :warned_of_deprecation
+    def warn_of_deprecation
+      @warned_of_deprecation ||= warn(self::DEPRECATION_WARNING) || true
+    end
+  end
+end

data/lib/parser/diagnostic.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Parser
 
   ##
@@ -152,9 +154,7 @@ module Parser
     #
     def last_line_only(range)
       if range.line != range.last_line
-        Source::Range.new(range.source_buffer,
-                          range.begin_pos + (range.source =~ /[^\n]*\z/),
-                          range.end_pos)
+        range.adjust(begin_pos: range.source =~ /[^\n]*\z/)
       else
         range
       end

data/lib/parser/diagnostic/engine.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Parser
 
   ##

data/lib/parser/lexer.rl

@@ -89,8 +89,6 @@ class Parser::Lexer
 
   REGEXP_META_CHARACTERS = Regexp.union(*"\\$()*+.<>?[]^{|}".chars).freeze
 
-  RBRACE_OR_RBRACK = %w"} ]".freeze
-
   attr_reader   :source_buffer
 
   attr_accessor :diagnostics
@@ -174,6 +172,9 @@ class Parser::Lexer
 
     # True at the end of "def foo a:"
     @in_kwarg      = false
+
+    # State before =begin / =end block comment
+    @cs_before_block_comment = self.class.lex_en_line_begin
   end
 
   def source_buffer=(source_buffer)
@@ -182,7 +183,7 @@ class Parser::Lexer
     if @source_buffer
       source = @source_buffer.source
 
-      if defined?(Encoding) && source.encoding == Encoding::UTF_8
+      if source.encoding == Encoding::UTF_8
         @source_pts = source.unpack('U*')
       else
         @source_pts = source.unpack('C*')
@@ -308,14 +309,8 @@ class Parser::Lexer
     @stack[@top]
   end
 
-  if defined?(Encoding)
-    def encode_escape(ord)
-      ord.chr.force_encoding(@source_buffer.source.encoding)
-    end
-  else
-    def encode_escape(ord)
-      ord.chr
-    end
+  def encode_escape(ord)
+    ord.chr.force_encoding(@source_buffer.source.encoding)
   end
 
   def tok(s = @ts, e = @te)
@@ -622,19 +617,23 @@ class Parser::Lexer
   flo_pow  = [eE] [+\-]? ( digit+ '_' )* digit+;
 
   int_suffix =
-    ''   % { @num_xfrm = lambda { |chars| emit(:tINTEGER,   chars) } }
-  | 'r'  % { @num_xfrm = lambda { |chars| emit(:tRATIONAL,  Rational(chars)) } }
-  | 'i'  % { @num_xfrm = lambda { |chars| emit(:tIMAGINARY, Complex(0, chars)) } }
-  | 'ri' % { @num_xfrm = lambda { |chars| emit(:tIMAGINARY, Complex(0, Rational(chars))) } };
+    ''       % { @num_xfrm = lambda { |chars| emit(:tINTEGER,   chars) } }
+  | 'r'      % { @num_xfrm = lambda { |chars| emit(:tRATIONAL,  Rational(chars)) } }
+  | 'i'      % { @num_xfrm = lambda { |chars| emit(:tIMAGINARY, Complex(0, chars)) } }
+  | 'ri'     % { @num_xfrm = lambda { |chars| emit(:tIMAGINARY, Complex(0, Rational(chars))) } }
+  | 'if'     % { @num_xfrm = lambda { |chars| emit(:tINTEGER,   chars, @ts, @te - 2); p -= 2 } }
+  | 'rescue' % { @num_xfrm = lambda { |chars| emit(:tINTEGER,   chars, @ts, @te - 6); p -= 6 } };
 
   flo_pow_suffix =
     ''   % { @num_xfrm = lambda { |chars| emit(:tFLOAT,     Float(chars)) } }
-  | 'i'  % { @num_xfrm = lambda { |chars| emit(:tIMAGINARY, Complex(0, Float(chars))) } };
+  | 'i'  % { @num_xfrm = lambda { |chars| emit(:tIMAGINARY, Complex(0, Float(chars))) } }
+  | 'if' % { @num_xfrm = lambda { |chars| emit(:tFLOAT,     Float(chars), @ts, @te - 2); p -= 2 } };
 
   flo_suffix =
     flo_pow_suffix
-  | 'r'  % { @num_xfrm = lambda { |chars| emit(:tRATIONAL,  Rational(chars)) } }
-  | 'ri' % { @num_xfrm = lambda { |chars| emit(:tIMAGINARY, Complex(0, Rational(chars))) } };
+  | 'r'      % { @num_xfrm = lambda { |chars| emit(:tRATIONAL,  Rational(chars)) } }
+  | 'ri'     % { @num_xfrm = lambda { |chars| emit(:tIMAGINARY, Complex(0, Rational(chars))) } }
+  | 'rescue' % { @num_xfrm = lambda { |chars| emit(:tFLOAT,     Float(chars), @ts, @te - 6); p -= 6 } };
 
   #
   # === ESCAPE SEQUENCE PARSING ===
@@ -655,17 +654,37 @@ class Parser::Lexer
     codepoints  = tok(@escape_s + 2, p - 1)
     codepoint_s = @escape_s + 2
 
-    codepoints.split(/[ \t]/).each do |codepoint_str|
-      codepoint = codepoint_str.to_i(16)
+    if @version < 24
+      if codepoints.start_with?(" ") || codepoints.start_with?("\t")
+        diagnostic :fatal, :invalid_unicode_escape, nil,
+          range(@escape_s + 2, @escape_s + 3)
+      end
 
-      if codepoint >= 0x110000
-        diagnostic :error, :unicode_point_too_large, nil,
-                   range(codepoint_s, codepoint_s + codepoint_str.length)
-        break
+      if spaces_p = codepoints.index(/[ \t]{2}/)
+        diagnostic :fatal, :invalid_unicode_escape, nil,
+          range(codepoint_s + spaces_p + 1, codepoint_s + spaces_p + 2)
       end
 
-      @escape     += codepoint.chr(Encoding::UTF_8)
-      codepoint_s += codepoint_str.length + 1
+      if codepoints.end_with?(" ") || codepoints.end_with?("\t")
+        diagnostic :fatal, :invalid_unicode_escape, nil, range(p - 1, p)
+      end
+    end
+
+    codepoints.scan(/([0-9a-fA-F]+)|([ \t]+)/).each do |(codepoint_str, spaces)|
+      if spaces
+        codepoint_s += spaces.length
+      else
+        codepoint = codepoint_str.to_i(16)
+
+        if codepoint >= 0x110000
+          diagnostic :error, :unicode_point_too_large, nil,
+                     range(codepoint_s, codepoint_s + codepoint_str.length)
+          break
+        end
+
+        @escape     += codepoint.chr(Encoding::UTF_8)
+        codepoint_s += codepoint_str.length
+      end
     end
   }
 
@@ -708,35 +727,38 @@ class Parser::Lexer
     | 'x' xdigit{1,2}
         % { @escape = encode_escape(tok(@escape_s + 1, p).to_i(16)) }
 
+      # %q[\x]
+    | 'x' ( c_any - xdigit )
+      % {
+        diagnostic :fatal, :invalid_hex_escape, nil, range(@escape_s - 1, p + 2)
+      }
+
       # \u263a
     | 'u' xdigit{4}
       % { @escape = tok(@escape_s + 1, p).to_i(16).chr(Encoding::UTF_8) }
 
-      # %q[\x]
-    | 'x' ( c_any - xdigit )
+      # \u123
+    | 'u' xdigit{0,3}
       % {
-        diagnostic :fatal, :invalid_hex_escape, nil, range(@escape_s - 1, p + 2)
+        diagnostic :fatal, :invalid_unicode_escape, nil, range(@escape_s - 1, p)
       }
 
-      # %q[\u123] %q[\u{12]
-    | 'u' ( c_any{0,4}  -
-            xdigit{4}   -            # \u1234 is valid
-            ( '{' xdigit{1,3}        # \u{1 \u{12 \u{123 are valid
-            | '{' xdigit [ \t}] any? # \u{1. \u{1} are valid
-            | '{' xdigit{2} [ \t}]   # \u{12. \u{12} are valid
-            )
-          )
+      # u{not hex} or u{}
+    | 'u{' ( c_any - xdigit - [ \t}] )* '}'
       % {
         diagnostic :fatal, :invalid_unicode_escape, nil, range(@escape_s - 1, p)
       }
 
-      # \u{123 456}
-    | 'u{' ( xdigit{1,6} [ \t] )*
-      ( xdigit{1,6} '}'
-        %unicode_points
-      | ( xdigit* ( c_any - xdigit - '}' )+ '}'
-        | ( c_any - '}' )* c_eof
-        | xdigit{7,}
+      # \u{  \t  123  \t 456   \t\t }
+    | 'u{' [ \t]* ( xdigit{1,6} [ \t]+ )*
+      (
+        ( xdigit{1,6} [ \t]* '}'
+          %unicode_points
+        )
+        |
+        ( xdigit* ( c_any - xdigit - [ \t}] )+ '}'
+          | ( c_any - [ \t}] )* c_eof
+          | xdigit{7,}
         ) % {
           diagnostic :fatal, :unterminated_unicode, nil, range(p - 1, p)
         }
@@ -1096,13 +1118,15 @@ class Parser::Lexer
         end
 
         emit(:tREGEXP_OPT)
-        fnext expr_end; fbreak;
+        fnext expr_end;
+        fbreak;
       };
 
       any
       => {
         emit(:tREGEXP_OPT, tok(@ts, @te - 1), @ts, @te - 1)
-        fhold; fgoto expr_end;
+        fhold;
+        fgoto expr_end;
       };
   *|;
 
@@ -1669,19 +1693,30 @@ class Parser::Lexer
       # <<-END | <<-'END' | <<-"END" | <<-`END` |
       # <<~END | <<~'END' | <<~"END" | <<~`END`
       '<<' [~\-]?
-        ( '"' ( c_line - '"' )* '"'
-        | "'" ( c_line - "'" )* "'"
-        | "`" ( c_line - "`" )* "`"
+        ( '"' ( any - '"' )* '"'
+        | "'" ( any - "'" )* "'"
+        | "`" ( any - "`" )* "`"
         | bareword ) % { heredoc_e      = p }
         c_line* c_nl % { new_herebody_s = p }
       => {
-        tok(@ts, heredoc_e) =~ /^<<(-?)(~?)(["'`]?)(.*)\3$/
+        tok(@ts, heredoc_e) =~ /^<<(-?)(~?)(["'`]?)(.*)\3$/m
 
         indent      = !$1.empty? || !$2.empty?
         dedent_body = !$2.empty?
         type        =  $3.empty? ? '<<"'.freeze : ('<<'.freeze + $3)
         delimiter   =  $4
 
+        if @version >= 24
+          if delimiter.count("\n") > 0
+            if delimiter.end_with?("\n")
+              diagnostic :warning, :heredoc_id_ends_with_nl, nil, range(@ts, @ts + 1)
+              delimiter = delimiter.rstrip
+            else
+              diagnostic :fatal, :heredoc_id_has_newline, nil, range(@ts, @ts + 1)
+            end
+          end
+        end
+
         if dedent_body && version?(18, 19, 20, 21, 22)
           emit(:tLSHFT, '<<'.freeze, @ts, @ts + 2)
           p = @ts + 1
@@ -1698,6 +1733,13 @@ class Parser::Lexer
       # SYMBOL LITERALS
       #
 
+      # :&&, :||
+      ':' ('&&' | '||') => {
+        fhold; fhold;
+        emit(:tSYMBEG, tok(@ts, @ts + 1), @ts, @ts + 1)
+        fgoto expr_fname;
+      };
+
       # :"bar", :'baz'
       ':' ['"] # '
       => {
@@ -1732,11 +1774,7 @@ class Parser::Lexer
         value = @escape || tok(@ts + 1)
 
         if version?(18)
-          if defined?(Encoding)
-            emit(:tINTEGER, value.dup.force_encoding(Encoding::BINARY)[0].ord)
-          else
-            emit(:tINTEGER, value[0].ord)
-          end
+          emit(:tINTEGER, value.dup.force_encoding(Encoding::BINARY)[0].ord)
         else
           emit(:tCHARACTER, value)
         end
@@ -1853,6 +1891,21 @@ class Parser::Lexer
       call_or_var
       => local_ident;
 
+      (call_or_var - keyword)
+        % { ident_tok = tok; ident_ts = @ts; ident_te = @te; }
+      w_space+ '('
+      => {
+        emit(:tIDENTIFIER, ident_tok, ident_ts, ident_te)
+        p = ident_te - 1
+
+        if !@static_env.nil? && @static_env.declared?(ident_tok) && @version < 25
+          fnext expr_endfn;
+        else
+          fnext expr_cmdarg;
+        end
+        fbreak;
+      };
+
       #
       # WHITESPACE
       #
@@ -1860,8 +1913,11 @@ class Parser::Lexer
       w_any;
 
       e_heredoc_nl '=begin' ( c_space | c_nl_zlen )
-      => { p = @ts - 1
-           fgoto line_begin; };
+      => {
+        p = @ts - 1
+        @cs_before_block_comment = @cs
+        fgoto line_begin;
+      };
 
       #
       # DEFAULT TRANSITION
@@ -2159,10 +2215,16 @@ class Parser::Lexer
       e_rbrace | e_rparen | ']'
       => {
         emit_table(PUNCTUATION)
-        @cond.lexpop; @cmdarg.lexpop
+        @cond.lexpop; @cmdarg.pop
 
-        if RBRACE_OR_RBRACK.include?(tok)
+        if tok == '}'.freeze
           fnext expr_endarg;
+        elsif tok == ']'
+          if @version >= 24
+            fnext expr_end;
+          else
+            fnext expr_endarg;
+          end
         else # )
           # fnext expr_endfn; ?
         end
@@ -2232,7 +2294,7 @@ class Parser::Lexer
       '=end' c_line* c_nl_zlen
       => {
         emit_comment(@eq_begin_s, @te)
-        fgoto line_begin;
+        fgoto *@cs_before_block_comment;
       };
 
       c_line* c_nl;

data/lib/parser/lexer/dedenter.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Parser
 
   class Lexer::Dedenter

data/lib/parser/lexer/explanation.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Parser
 
   module Lexer::Explanation

data/lib/parser/lexer/literal.rb

@@ -1,4 +1,5 @@
 # encoding: binary
+# frozen_string_literal: true
 
 module Parser
 
@@ -229,21 +230,15 @@ module Parser
     end
 
     def coerce_encoding(string)
-      if defined?(Encoding)
-        string.dup.force_encoding(Encoding::BINARY)
-      else
-        string
-      end
+      string.dup.force_encoding(Encoding::BINARY)
     end
 
     def clear_buffer
-      @buffer = ''
+      @buffer = ''.dup
 
       # Prime the buffer with lexer encoding; otherwise,
       # concatenation will produce varying results.
-      if defined?(Encoding)
-        @buffer.force_encoding(@lexer.source_buffer.source.encoding)
-      end
+      @buffer.force_encoding(@lexer.source_buffer.source.encoding)
 
       @buffer_s = nil
       @buffer_e = nil

data/lib/parser/lexer/stack_state.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Parser
 
   class Lexer::StackState
@@ -25,7 +27,8 @@ module Parser
     end
 
     def lexpop
-      push(pop || pop)
+      @stack = ((@stack >> 1) | (@stack & 1))
+      @stack[0] == 1
     end
 
     def active?