parser 0.9.1 → 0.9.2

data/lib/parser/lexer.rl

@@ -265,14 +265,18 @@ class Parser::Lexer
     new_literal = Literal.new(self, *args)
     @literal_stack.push(new_literal)
 
-    if new_literal.type == :tWORDS_BEG
-      self.class.lex_en_interp_words
-    elsif new_literal.type == :tQWORDS_BEG
-      self.class.lex_en_plain_words
-    elsif new_literal.interpolate?
-      self.class.lex_en_interp_string
+    if new_literal.words?
+      if new_literal.interpolate?
+        self.class.lex_en_interp_words
+      else
+        self.class.lex_en_plain_words
+      end
     else
-      self.class.lex_en_plain_string
+      if new_literal.interpolate?
+        self.class.lex_en_interp_string
+      else
+        self.class.lex_en_plain_string
+      end
     end
   end
 
@@ -313,9 +317,9 @@ class Parser::Lexer
   }
 
   PUNCTUATION_BEGIN = {
-    '&'   => :tAMPER,   '*'   => :tSTAR,   '+'   => :tUPLUS,
-    '-'   => :tUMINUS,  '::'  => :tCOLON3, '('   => :tLPAREN,
-    '{'   => :tLBRACE,  '['   => :tLBRACK,
+    '&'   => :tAMPER,   '*'   => :tSTAR,    '**'  => :tDSTAR,
+    '+'   => :tUPLUS,   '-'   => :tUMINUS,  '::'  => :tCOLON3,
+    '('   => :tLPAREN,  '{'   => :tLBRACE,  '['   => :tLBRACK,
   }
 
   KEYWORDS = {
@@ -410,7 +414,8 @@ class Parser::Lexer
 
   # A list of punctuation which has different meaning when used at the
   # beginning of expression.
-  punctuation_begin   = '-'  | '+'  | '::' | '('  | '['  | '*'   | '&' ;
+  punctuation_begin   = '-'  | '+'  | '::' | '('  | '['  |
+                        '*'  | '**' | '&'  ;
 
   # A list of all punctuation except punctuation_begin.
   punctuation_end     = ','  | '='  | '->' | '('  | '['  | ']'   |
@@ -748,6 +753,10 @@ class Parser::Lexer
     literal.flush_string
   }
 
+  action extend_string_space {
+    literal.extend_space @ts, @te
+  }
+
   #
   # === INTERPOLATION PARSING ===
   #
@@ -759,6 +768,8 @@ class Parser::Lexer
 
   action extend_interp_var {
     literal.flush_string
+    literal.extend_content
+
     emit(:tSTRING_DVAR, nil, @ts, @ts + 1)
 
     p = @ts
@@ -790,10 +801,10 @@ class Parser::Lexer
   e_rbrace = '}' % {
     if literal
       if literal.end_interp_brace_and_try_closing
-        emit(:tRCURLY, '}')
-
-        if literal.words?
-          emit(:tSPACE, nil)
+        if version?(18, 19)
+          emit(:tRCURLY, '}')
+        else
+          emit(:tSTRING_DEND, '}')
         end
 
         if literal.saved_herebody_s
@@ -809,6 +820,8 @@ class Parser::Lexer
 
   action extend_interp_code {
     literal.flush_string
+    literal.extend_content
+
     emit(:tSTRING_DBEG, '#{')
 
     literal.saved_herebody_s = @herebody_s
@@ -825,7 +838,7 @@ class Parser::Lexer
       interp_code => extend_interp_code;
       interp_var  => extend_interp_var;
       e_bs escape => extend_string_escaped;
-      c_space_nl  => { literal.flush_string };
+      c_space_nl+ => extend_string_space;
       c_eol       => extend_string_eol;
       c_any       => extend_string;
   *|;
@@ -840,7 +853,7 @@ class Parser::Lexer
 
   plain_words := |*
       e_bs c_any  => extend_string_escaped;
-      c_space_nl  => { literal.flush_string };
+      c_space_nl+ => extend_string_space;
       c_eol       => extend_string_eol;
       c_any       => extend_string;
   *|;
@@ -999,7 +1012,9 @@ class Parser::Lexer
       => { p = @ts - 1
            fcall expr_variable; };
 
-      c_space_nl+;
+      # TODO whitespace rule
+      c_space;
+      e_heredoc_nl;
 
       c_any
       => { fhold; fgoto expr_end; };
@@ -1030,9 +1045,11 @@ class Parser::Lexer
       => { emit_table(PUNCTUATION)
            fnext expr_arg; fbreak; };
 
-      c_space_nl+;
-
-      '#' c_line* c_nl;
+      # TODO whitespace rule
+      c_space;
+      e_heredoc_nl;
+      '\\' e_heredoc_nl;
+      '#' c_line*;
 
       c_any
       => { fhold; fgoto expr_end; };
@@ -1108,12 +1125,12 @@ class Parser::Lexer
       };
 
       # x *1
-      # Ambiguous splat or block-pass.
-      c_space+ [*&]
+      # Ambiguous splat, kwsplat or block-pass.
+      c_space+ %{ tm = p } ( '*' | '&' | '**' )
       => {
-        message = Parser::ERRORS[:ambiguous_prefix] % { :prefix => tok(@te - 1, @te) }
+        message = Parser::ERRORS[:ambiguous_prefix] % { :prefix => tok(tm, @te) }
         diagnostic :warning, message,
-                   range(@te - 1, @te)
+                   range(tm, @te)
 
         fhold; fgoto expr_beg;
       };
@@ -1182,7 +1199,7 @@ class Parser::Lexer
       => { emit(:kDO_BLOCK)
            fnext expr_value; };
 
-      c_space*;
+      c_space;
 
       c_any
       => { fhold; fgoto expr_end; };
@@ -1200,11 +1217,11 @@ class Parser::Lexer
       => { emit_table(KEYWORDS)
            fnext expr_beg; fbreak; };
 
-      c_space+;
-
+      # TODO whitespace rule
+      c_space;
       '#' c_line*;
 
-      c_nl
+      e_heredoc_nl
       => { fhold; fgoto expr_end; };
 
       c_any
@@ -1430,7 +1447,8 @@ class Parser::Lexer
       => {
         emit(:tIDENTIFIER)
 
-        if @static_env && @static_env.declared?(tok)
+        if !version?(18) &&
+              !@static_env.nil? && @static_env.declared?(tok)
           fnext expr_end; fbreak;
         else
           fnext expr_arg; fbreak;
@@ -1441,15 +1459,16 @@ class Parser::Lexer
       # WHITESPACE
       #
 
-      c_space_nl;
-
-      '\\\n';
+      # TODO whitespace rule
+      c_space;
+      e_heredoc_nl;
+      '\\' e_heredoc_nl;
 
       '#' c_line* c_eol
       => { @comments << tok
            fhold; };
 
-      c_nl '=begin' ( c_space | c_eol )
+      e_heredoc_nl '=begin' ( c_space | c_eol )
       => { p = @ts - 1
            fgoto line_begin; };
 
@@ -1476,7 +1495,9 @@ class Parser::Lexer
       => { p = @ts - 1
            fgoto expr_end; };
 
-      c_space_nl+;
+      # TODO whitespace rule
+      c_space;
+      e_heredoc_nl;
 
       c_any
       => { fhold; fgoto expr_beg; };
@@ -1489,9 +1510,9 @@ class Parser::Lexer
       # STABBY LAMBDA
       #
 
-      '->'
+      '->' c_space*
       => {
-        emit_table(PUNCTUATION)
+        emit_table(PUNCTUATION, @ts, @ts + 2)
 
         @lambda_stack.push @paren_nest
         fbreak;
@@ -1535,7 +1556,7 @@ class Parser::Lexer
       'class' c_space_nl* '<<'
       => { emit(:kCLASS, 'class', @ts, @ts + 5)
            emit(:tLSHFT, '<<',    @te - 2, @te)
-           fnext expr_beg; fbreak; };
+           fnext expr_value; fbreak; };
 
       # a if b:c: Syntax error.
       keyword_modifier
@@ -1717,6 +1738,7 @@ class Parser::Lexer
       # WHITESPACE
       #
 
+      # TODO whitespace rule
       '\\' e_heredoc_nl;
 
       '\\' c_line {
@@ -1779,6 +1801,7 @@ class Parser::Lexer
   *|;
 
   line_begin := |*
+      # TODO whitespace rule
       c_space_nl+;
 
       '#' c_line* c_eol