ruby_parser 2.0.5 → 3.0.0.a1

data/lib/ruby_lexer.rb

@@ -2,10 +2,17 @@ class RubyLexer
   attr_accessor :command_start
   attr_accessor :cmdarg
   attr_accessor :cond
+  attr_accessor :tern
   attr_accessor :nest
 
   ESC_RE = /\\([0-7]{1,3}|x[0-9a-fA-F]{1,2}|M-[^\\]|(C-|c)[^\\]|[^0-7xMCc])/
 
+  ##
+  # What version of ruby to parse. 18 and 19 are the only valid values
+  # currently supported.
+
+  attr_accessor :version
+
   # Additional context surrounding tokens that both the lexer and
   # grammar use.
   attr_reader :lex_state
@@ -35,7 +42,7 @@ class RubyLexer
   STR_FUNC_ESCAPE = 0x01 # TODO: remove and replace with REGEXP
   STR_FUNC_EXPAND = 0x02
   STR_FUNC_REGEXP = 0x04
-  STR_FUNC_AWORDS = 0x08
+  STR_FUNC_QWORDS = 0x08
   STR_FUNC_SYMBOL = 0x10
   STR_FUNC_INDENT = 0x20 # <<-HEREDOC
 
@@ -58,6 +65,7 @@ class RubyLexer
     "===" => :tEQQ,
     "=>"  => :tASSOC,
     "=~"  => :tMATCH,
+    "->"  => :tLAMBDA,
   }
 
   # How the parser advances to the next token.
@@ -130,7 +138,7 @@ class RubyLexer
         string_buffer << '#'
       end
 
-      until src.scan(eos_re) do
+      until src.check(eos_re) do
         c = tokadd_string func, "\n", nil
 
         rb_compile_error err_msg if
@@ -146,9 +154,6 @@ class RubyLexer
         rb_compile_error err_msg if
           src.eos?
       end
-
-      # tack on a NL after the heredoc token - FIX NL should not be needed
-      src.unread_many(eos + "\n") # TODO: remove this... stupid stupid stupid
     else
       until src.check(eos_re) do
         string_buffer << src.scan(/.*(\n|\z)/)
@@ -195,11 +200,10 @@ class RubyLexer
       return nil
     end
 
-    if src.check(/.*\n/) then
+    if src.scan(/.*\n/) then
       # TODO: think about storing off the char range instead
-      line = src.string[src.pos, src.matched_size]
-      src.string[src.pos, src.matched_size] = "\n"
-      src.pos += 1
+      line = src.matched
+      src.extra_lines_added += 1
     else
       line = nil
     end
@@ -215,9 +219,11 @@ class RubyLexer
     end
   end
 
-  def initialize
-    self.cond = StackState.new(:cond)
-    self.cmdarg = StackState.new(:cmdarg)
+  def initialize v = 18
+    self.version = v
+    self.cond = RubyParser::StackState.new(:cond)
+    self.cmdarg = RubyParser::StackState.new(:cmdarg)
+    self.tern = RubyParser::StackState.new(:tern)
     self.nest = 0
     @comments = []
 
@@ -231,7 +237,7 @@ class RubyLexer
   end
 
   def lex_state= o
-    raise "wtf?" unless Symbol === o
+    raise "wtf\?" unless Symbol === o
     @lex_state = o
   end
 
@@ -308,10 +314,10 @@ class RubyLexer
                                 [:tSTRING_BEG,   STR_SQUOTE]
                               when 'W' then
                                 src.scan(/\s*/)
-                                [:tWORDS_BEG,    STR_DQUOTE | STR_FUNC_AWORDS]
+                                [:tWORDS_BEG,    STR_DQUOTE | STR_FUNC_QWORDS]
                               when 'w' then
                                 src.scan(/\s*/)
-                                [:tAWORDS_BEG,   STR_SQUOTE | STR_FUNC_AWORDS]
+                                [:tQWORDS_BEG,   STR_SQUOTE | STR_FUNC_QWORDS]
                               when 'x' then
                                 [:tXSTRING_BEG,  STR_XQUOTE]
                               when 'r' then
@@ -337,7 +343,7 @@ class RubyLexer
     paren = open
     term_re = Regexp.escape term
 
-    awords = (func & STR_FUNC_AWORDS) != 0
+    qwords = (func & STR_FUNC_QWORDS) != 0
     regexp = (func & STR_FUNC_REGEXP) != 0
     expand = (func & STR_FUNC_EXPAND) != 0
 
@@ -346,10 +352,10 @@ class RubyLexer
       return :tSTRING_END
     end
 
-    space = true if awords and src.scan(/\s+/)
+    space = true if qwords and src.scan(/\s+/)
 
     if self.nest == 0 && src.scan(/#{term_re}/) then
-      if awords then
+      if qwords then
         quote[1] = nil
         return :tSPACE
       elsif regexp then
@@ -386,7 +392,6 @@ class RubyLexer
 
     self.yacc_value = string_buffer.join
 
-
     return :tSTRING_CONTENT
   end
 
@@ -473,6 +478,10 @@ class RubyLexer
     @lex_state = nil
   end
 
+  def ruby18
+    Ruby18Parser === parser
+  end
+
   def src= src
     raise "bad src: #{src.inspect}" unless String === src
     @src = RPStringScanner.new(src)
@@ -499,7 +508,7 @@ class RubyLexer
   end
 
   def tokadd_string(func, term, paren) # 105 lines
-    awords = (func & STR_FUNC_AWORDS) != 0
+    qwords = (func & STR_FUNC_QWORDS) != 0
     escape = (func & STR_FUNC_ESCAPE) != 0
     expand = (func & STR_FUNC_EXPAND) != 0
     regexp = (func & STR_FUNC_REGEXP) != 0
@@ -519,7 +528,7 @@ class RubyLexer
         self.nest += 1
       when src.scan(term_re) then
         self.nest -= 1
-      when awords && src.scan(/\s/) then
+      when qwords && src.scan(/\s/) then
         src.pos -= 1
         break
       when expand && src.scan(/#(?=[\$\@\{])/) then
@@ -529,10 +538,10 @@ class RubyLexer
         # do nothing
       when src.check(/\\/) then
         case
-        when awords && src.scan(/\\\n/) then
+        when qwords && src.scan(/\\\n/) then
           string_buffer << "\n"
           next
-        when awords && src.scan(/\\\s/) then
+        when qwords && src.scan(/\\\s/) then
           c = ' '
         when expand && src.scan(/\\\n/) then
           next
@@ -561,7 +570,7 @@ class RubyLexer
 
         t = Regexp.escape term
         x = Regexp.escape(paren) if paren && paren != "\000"
-        re = if awords then
+        re = if qwords then
                /[^#{t}#{x}\#\0\\\n\ ]+|./ # |. to pick up whatever
              else
                /[^#{t}#{x}\#\0\\]+|./
@@ -630,7 +639,6 @@ class RubyLexer
   # @return Description of the Returned Value
 
   def yylex # 826 lines
-
     c = ''
     space_seen = false
     command_state = false
@@ -655,7 +663,7 @@ class RubyLexer
           self.lineno = nil
           c = src.matched
           if c == '#' then
-            src.unread c # ok
+            src.pos -= 1
 
             while src.scan(/\s*#.*(\n+|\z)/) do
               @comments << src.matched.gsub(/^ +#/, '#').gsub(/^ +$/, '')
@@ -687,6 +695,7 @@ class RubyLexer
             "]" => :tRBRACK,
             "}" => :tRCURLY
           }[src.matched]
+          self.tern.lexpop if [:tRBRACK, :tRCURLY].include?(result)
           return result
         elsif src.scan(/\.\.\.?|,|![=~]?/) then
           self.lex_state = :expr_beg
@@ -701,19 +710,11 @@ class RubyLexer
             return :tDOT
           end
         elsif src.scan(/\(/) then
-          result = :tLPAREN2
-          self.command_start = true
-
-          if lex_state == :expr_beg || lex_state == :expr_mid then
-            result = :tLPAREN
-          elsif space_seen then
-            if lex_state == :expr_cmdarg then
-              result = :tLPAREN_ARG
-            elsif lex_state == :expr_arg then
-              warning("don't put space before argument parentheses")
-              result = :tLPAREN2
-            end
-          end
+          result = if ruby18 then
+                     yylex_paren18 space_seen
+                   else
+                     yylex_paren19 space_seen
+                   end
 
           self.expr_beg_push "("
 
@@ -723,11 +724,11 @@ class RubyLexer
             self.fix_arg_lex_state
             tok = self.yacc_value = src.matched
             return TOKENS[tok]
-          elsif src.was_begin_of_line and src.scan(/\=begin(?=\s)/) then
+          elsif src.scan(/\=begin(?=\s)/) then
             # @comments << '=' << src.matched
             @comments << src.matched
 
-            unless src.scan(/.*?\n=end\s*(\n|\z)/m) then
+            unless src.scan(/.*?\n=end( |\t|\f)*[^\n]*(\n|\z)/m) then
               @comments.clear
               rb_compile_error("embedded document meets end of file")
             end
@@ -807,9 +808,13 @@ class RubyLexer
               rb_compile_error "unexpected '['"
             end
           elsif lex_state == :expr_beg || lex_state == :expr_mid then
+            self.tern.push false
             result = :tLBRACK
           elsif lex_state.is_argument && space_seen then
+            self.tern.push false
             result = :tLBRACK
+          else
+            result = :tLBRACK2
           end
 
           self.expr_beg_push "["
@@ -838,17 +843,29 @@ class RubyLexer
             return :tPIPE
           end
         elsif src.scan(/\{/) then
+          if defined?(@hack_expects_lambda) && @hack_expects_lambda
+            @hack_expects_lambda = false
+            self.lex_state = :expr_beg
+            return :tLAMBEG
+          end
+
           result = if lex_state.is_argument || lex_state == :expr_end then
                      :tLCURLY      #  block (primary)
                    elsif lex_state == :expr_endarg then
                      :tLBRACE_ARG  #  block (expr)
                    else
+                     self.tern.push false
                      :tLBRACE      #  hash
                    end
 
           self.expr_beg_push "{"
+          self.command_start = true unless result == :tLBRACE
 
           return result
+        elsif src.scan(/->/) then
+          @hack_expects_lambda = true
+          self.lex_state = :expr_arg
+          return :tLAMBDA
         elsif src.scan(/[+-]/) then
           sign = src.matched
           utype, type = if sign == "+" then
@@ -994,6 +1011,7 @@ class RubyLexer
         elsif src.scan(/\?/) then
           if lex_state == :expr_end || lex_state == :expr_endarg then
             self.lex_state = :expr_beg
+            self.tern.push true
             self.yacc_value = "?"
             return :tEH
           end
@@ -1018,10 +1036,12 @@ class RubyLexer
 
             # ternary
             self.lex_state = :expr_beg
+            self.tern.push true
             self.yacc_value = "?"
             return :tEH
           elsif src.check(/\w(?=\w)/) then # ternary, also
             self.lex_state = :expr_beg
+            self.tern.push true
             self.yacc_value = "?"
             return :tEH
           end
@@ -1032,8 +1052,14 @@ class RubyLexer
                 src.getch
               end
           self.lex_state = :expr_end
-          self.yacc_value = c[0].ord & 0xff
-          return :tINTEGER
+
+          if version == 18 then
+            self.yacc_value = c[0].ord & 0xff
+            return :tINTEGER
+          else
+            self.yacc_value = c
+            return :tSTRING
+          end
         elsif src.check(/\&/) then
           if src.scan(/\&\&\=/) then
             self.yacc_value = "&&"
@@ -1207,6 +1233,53 @@ class RubyLexer
     end
   end
 
+  def yylex_paren18 space_seen
+    self.command_start = true
+    result = :tLPAREN2
+
+    if lex_state == :expr_beg || lex_state == :expr_mid then
+      result = :tLPAREN
+    elsif space_seen then
+      if lex_state == :expr_cmdarg then
+        result = :tLPAREN_ARG
+      elsif lex_state == :expr_arg then
+        self.tern.push false
+        warning "don't put space before argument parentheses"
+      end
+    else
+      self.tern.push false
+    end
+
+    result
+  end
+
+  def yylex_paren19 space_seen
+    if (lex_state == :expr_beg || lex_state == :expr_mid ||
+        lex_state == :expr_value || lex_state == :expr_class) then
+      result = :tLPAREN
+    elsif ((lex_state == :expr_arg || lex_state == :expr_cmdarg) and
+           space_seen) then
+      result = :tLPAREN_ARG
+    else
+      self.tern.push false
+      result = :tLPAREN2
+    end
+    # HACK paren_nest++;
+
+    # HACK: this is a mess, but it makes the tests pass, so suck it
+    # (stolen from the 1.8 side)
+    if lex_state == :expr_beg || lex_state == :expr_mid then
+      # do nothing
+    elsif space_seen then
+      if lex_state == :expr_arg then
+        self.tern.push false
+      end
+    else
+      self.tern.push false
+    end
+    result
+  end
+
   def process_token(command_state)
 
     token << src.matched if token =~ /^\w/ && src.scan(/[\!\?](?!=)/)
@@ -1214,7 +1287,6 @@ class RubyLexer
     result = nil
     last_state = lex_state
 
-
     case token
     when /^\$/ then
       self.lex_state, result = :expr_end, :tGVAR
@@ -1242,24 +1314,51 @@ class RubyLexer
                    end
       end
 
+      unless self.tern.is_in_state
+        if (lex_state == :expr_beg && (ruby18 || !command_state)) ||
+            lex_state == :expr_arg ||
+            lex_state == :expr_cmdarg then
+          colon = src.scan(/:/)
+
+          if colon && src.peek(1) != ":"
+            src.unscan
+            self.lex_state = :expr_beg
+            src.scan(/:/)
+            self.yacc_value = [token, src.lineno]
+            return :tLABEL
+          end
+
+          src.unscan if colon
+        end
+      end unless ruby18
+
       unless lex_state == :expr_dot then
         # See if it is a reserved word.
-        keyword = Keyword.keyword token
+        keyword = RubyParser::Keyword.keyword token
 
         if keyword then
           state           = lex_state
           self.lex_state  = keyword.state
-          self.yacc_value = token
+          self.yacc_value = [token, src.lineno]
+
+          if state == :expr_fname then
+            self.yacc_value = keyword.name
+            return keyword.id0
+          end
 
           if keyword.id0 == :kDO then
             self.command_start = true
             return :kDO_COND  if cond.is_in_state
             return :kDO_BLOCK if cmdarg.is_in_state && state != :expr_cmdarg
             return :kDO_BLOCK if state == :expr_endarg
+            if defined?(@hack_expects_lambda) && @hack_expects_lambda
+              @hack_expects_lambda = false
+              return :kDO_LAMBDA
+            end
             return :kDO
           end
 
-          return keyword.id0 if state == :expr_beg
+          return keyword.id0 if state == :expr_beg or state == :expr_value
 
           self.lex_state = :expr_beg if keyword.id0 != keyword.id1