opal 0.5.5 → 0.6.0

data/lib/opal/parser/lexer.rb

@@ -4,21 +4,51 @@ require 'opal/parser/keywords'
 module Opal
   class Lexer
 
-    attr_reader :line, :scope_line, :scope
-
-    attr_accessor :lex_state, :strterm, :scanner
+    STR_FUNC_ESCAPE = 0x01
+    STR_FUNC_EXPAND = 0x02
+    STR_FUNC_REGEXP = 0x04
+    STR_FUNC_QWORDS = 0x08
+    STR_FUNC_SYMBOL = 0x10
+    STR_FUNC_INDENT = 0x20
+    STR_FUNC_XQUOTE = 0x40
+
+    STR_SQUOTE = 0x00
+    STR_DQUOTE = STR_FUNC_EXPAND
+    STR_XQUOTE = STR_FUNC_EXPAND | STR_FUNC_XQUOTE
+    STR_REGEXP = STR_FUNC_REGEXP | STR_FUNC_ESCAPE | STR_FUNC_EXPAND
+    STR_SWORD  = STR_FUNC_QWORDS
+    STR_DWORD  = STR_FUNC_QWORDS | STR_FUNC_EXPAND
+    STR_SSYM   = STR_FUNC_SYMBOL
+    STR_DSYM   = STR_FUNC_SYMBOL | STR_FUNC_EXPAND
+
+    attr_reader :line
+    attr_reader :scope
+    attr_reader :eof_content
+
+    attr_accessor :lex_state
+    attr_accessor :strterm
+    attr_accessor :scanner
+    attr_accessor :yylval
+    attr_accessor :parser
 
     def initialize(source, file)
       @lex_state  = :expr_beg
       @cond       = 0
       @cmdarg     = 0
       @line       = 1
+      @tok_line   = 1
+      @column     = 0
+      @tok_column = 0
       @file       = file
 
       @scanner = StringScanner.new(source)
       @scanner_stack = [@scanner]
     end
 
+    def has_local?(local)
+      parser.scope.has_local?(local.to_sym)
+    end
+
     def cond_push(n)
       @cond = (@cond << 1) | (n & 1)
     end
@@ -75,153 +105,258 @@ module Opal
       @scanner.check(/\s/)
     end
 
+    def set_arg_state
+      @lex_state = after_operator? ? :expr_arg : :expr_beg
+    end
+
     def scan(regexp)
-      @scanner.scan regexp
+      if result = @scanner.scan(regexp)
+        @column += result.length
+        @yylval += @scanner.matched
+      end
+
+      result
+    end
+
+    def skip(regexp)
+      if result = @scanner.scan(regexp)
+        @column += result.length
+        @tok_column = @column
+      end
+
+      result
     end
 
     def check(regexp)
       @scanner.check regexp
     end
 
+    def pushback(n)
+      @scanner.pos -= n
+    end
+
     def matched
       @scanner.matched
     end
 
-    def next_token
-      self.yylex
+    def line=(line)
+      @column = @tok_column = 0
+      @line = @tok_line = line
     end
 
-    def strterm_expand?(strterm)
-      type = strterm[:type]
+    def next_token
+      token     = self.yylex
+      value     = self.yylval
+      location  = [@tok_line, @tok_column]
 
-      [:dquote, :dsym, :dword, :heredoc, :xquote, :regexp].include? type
+      # once location is stored, ensure next token starts in correct place
+      @tok_column = @column
+      @tok_line = @line
+
+      [token, [value, location]]
     end
 
-    def new_strterm(type, start, finish)
-      { :type => type, :beg => start, :end => finish }
+    def new_strterm(func, term, paren)
+      { :type => :string, :func => func, :term => term, :paren => paren }
     end
 
-    def new_strterm2(type, start, finish)
-      term = new_strterm(type, start, finish)
+    def new_strterm2(func, term, paren)
+      term = new_strterm(func, term, paren)
       term.merge({ :balance => true, :nesting => 0 })
     end
 
+    def new_op_asgn(value)
+      self.yylval = value
+      :tOP_ASGN
+    end
+
     def process_numeric
       @lex_state = :expr_end
-      scanner = @scanner
 
       if scan(/0b?(0|1|_)+/)
-        return [:tINTEGER, scanner.matched.to_i(2)]
+        self.yylval = scanner.matched.to_i(2)
+        return :tINTEGER
       elsif scan(/0o?([0-7]|_)+/)
-        return [:tINTEGER, scanner.matched.to_i(8)]
+        self.yylval = scanner.matched.to_i(8)
+        return :tINTEGER
       elsif scan(/[\d_]+\.[\d_]+\b|[\d_]+(\.[\d_]+)?[eE][-+]?[\d_]+\b/)
-        return [:tFLOAT, scanner.matched.gsub(/_/, '').to_f]
+        self.yylval = scanner.matched.gsub(/_/, '').to_f
+        return :tFLOAT
       elsif scan(/[\d_]+\b/)
-        return [:tINTEGER, scanner.matched.gsub(/_/, '').to_i]
+        self.yylval = scanner.matched.gsub(/_/, '').to_i
+        return :tINTEGER
       elsif scan(/0(x|X)(\d|[a-f]|[A-F]|_)+/)
-        return [:tINTEGER, scanner.matched.to_i(16)]
+        self.yylval = scanner.matched.to_i(16)
+        return :tINTEGER
       else
         raise "Lexing error on numeric type: `#{scanner.peek 5}`"
       end
     end
 
-    def next_string_token
-      str_parse = self.strterm
-      scanner = @scanner
-      space = false
+    def read_escape
+      if scan(/\\/)
+        "\\"
+      elsif scan(/n/)
+        "\n"
+      elsif scan(/t/)
+        "\t"
+      elsif scan(/r/)
+        "\r"
+      elsif scan(/f/)
+        "\f"
+      elsif scan(/v/)
+        "\v"
+      elsif scan(/a/)
+        "\a"
+      elsif scan(/e/)
+        "\e"
+      elsif scan(/s/)
+        " "
+      elsif scan(/[0-7]{1,3}/)
+        (matched.to_i(8) % 0x100).chr
+      elsif scan(/x([0-9a-fA-F]{1,2})/)
+        scanner[1].to_i(16).chr
+      elsif scan(/u([0-9a-zA-Z]{1,4})/)
+        if defined?(::Encoding)
+          scanner[1].to_i(16).chr(Encoding::UTF_8)
+        else
+          # FIXME: no encoding on 1.8
+          ""
+        end
+      else
+        # escaped char doesnt need escaping, so just return it
+        scan(/./)
+      end
+    end
 
-      expand = strterm_expand?(str_parse)
+    def peek_variable_name
+      if check(/[@$]/)
+        :tSTRING_DVAR
+      elsif scan(/\{/)
+        :tSTRING_DBEG
+      end
+    end
 
-      words = ['w', 'W'].include? str_parse[:beg]
+    def here_document(str_parse)
+      eos_regx = /[ \t]*#{Regexp.escape(str_parse[:term])}(\r*\n|$)/
+      expand = true
 
-      space = true if ['w', 'W'].include?(str_parse[:beg]) and scan(/\s+/)
+      if check(eos_regx)
+        scan(/[ \t]*#{Regexp.escape(str_parse[:term])}/)
+
+        if str_parse[:scanner]
+          @scanner_stack << str_parse[:scanner]
+          @scanner = str_parse[:scanner]
+        end
+
+        return :tSTRING_END
+      end
 
-      # if not end of string, so we must be parsing contents
       str_buffer = []
 
-      if str_parse[:type] == :heredoc
-        eos_regx = /[ \t]*#{Regexp.escape(str_parse[:end])}(\r*\n|$)/
+      if scan(/#/)
+        if tok = peek_variable_name
+          return tok
+        end
 
-        if check(eos_regx)
-          scan(/[ \t]*#{Regexp.escape(str_parse[:end])}/)
-          self.strterm = nil
+        str_buffer << '#'
+      end
 
-          if str_parse[:scanner]
-            @scanner_stack << str_parse[:scanner]
-            @scanner = str_parse[:scanner]
-          end
+      until check(eos_regx) && scanner.bol?
+        if scanner.eos?
+          raise "reached EOF while in heredoc"
+        end
 
-          @lex_state = :expr_end
-          return :tSTRING_END, scanner.matched
+        if scan(/\n/)
+          str_buffer << scanner.matched
+        elsif expand && check(/#(?=[\$\@\{])/)
+          break
+        elsif scan(/\\/)
+          str_buffer << self.read_escape
+        else
+          reg = Regexp.new("[^\#\0\\\\\n]+|.")
+
+          scan reg
+          str_buffer << scanner.matched
         end
       end
 
-      # see if we can read end of string/xstring/regexp markers
-      # if scan /#{str_parse[:end]}/
-      if scan Regexp.new(Regexp.escape(str_parse[:end]))
-        if words && !str_parse[:done_last_space]#&& space
+      complete_str = str_buffer.join ''
+      @line += complete_str.count("\n")
+
+      self.yylval = complete_str
+      return :tSTRING_CONTENT
+    end
+
+    def parse_string
+      str_parse = self.strterm
+      func = str_parse[:func]
+
+      space = false
+
+      qwords = (func & STR_FUNC_QWORDS) != 0
+      expand = (func & STR_FUNC_EXPAND) != 0
+      regexp = (func & STR_FUNC_REGEXP) != 0
+
+      space = true if qwords and scan(/\s+/)
+
+      # if not end of string, so we must be parsing contents
+      str_buffer = []
+
+      if scan Regexp.new(Regexp.escape(str_parse[:term]))
+        if qwords && !str_parse[:done_last_space]#&& space
           str_parse[:done_last_space] = true
-          scanner.pos -= 1
-          return :tSPACE, ' '
+          pushback(1)
+          self.yylval = ' '
+          return :tSPACE
         end
-        self.strterm = nil
 
         if str_parse[:balance]
           if str_parse[:nesting] == 0
-            @lex_state = :expr_end
 
-            if str_parse[:type] == :regexp
-              result = scan(/\w+/)
-              return :tREGEXP_END, result
+            if regexp
+              self.yylval = scan(/\w+/)
+              return :tREGEXP_END
             end
-            return :tSTRING_END, scanner.matched
+            return :tSTRING_END
           else
             str_buffer << scanner.matched
             str_parse[:nesting] -= 1
             self.strterm = str_parse
           end
-
-        elsif ['"', "'"].include? str_parse[:beg]
+        elsif regexp
           @lex_state = :expr_end
-          return :tSTRING_END, scanner.matched
-
-        elsif str_parse[:beg] == '`'
-          @lex_state = :expr_end
-          return :tSTRING_END, scanner.matched
-
-        elsif str_parse[:beg] == '/' || str_parse[:type] == :regexp
-          result = scan(/\w+/)
-          @lex_state = :expr_end
-          return :tREGEXP_END, result
-
+          self.yylval = scan(/\w+/)
+          return :tREGEXP_END
         else
           if str_parse[:scanner]
             @scanner_stack << str_parse[:scanner]
             @scanner = str_parse[:scanner]
           end
 
-          @lex_state = :expr_end
-          return :tSTRING_END, scanner.matched
+          return :tSTRING_END
         end
       end
 
-      return :tSPACE, ' ' if space
+      if space
+        self.yylval = ' '
+        return :tSPACE
+      end
 
-      if str_parse[:balance] and scan Regexp.new(Regexp.escape(str_parse[:beg]))
+      if str_parse[:balance] and scan Regexp.new(Regexp.escape(str_parse[:paren]))
         str_buffer << scanner.matched
         str_parse[:nesting] += 1
       elsif check(/#[@$]/)
         scan(/#/)
         if expand
-          return :tSTRING_DVAR, scanner.matched
+          return :tSTRING_DVAR
         else
           str_buffer << scanner.matched
         end
 
       elsif scan(/#\{/)
         if expand
-          # we are into ruby code, so stop parsing content (for now)
-          return :tSTRING_DBEG, scanner.matched
+          return :tSTRING_DBEG
         else
           str_buffer << scanner.matched
         end
@@ -231,80 +366,25 @@ module Opal
         str_buffer << '#'
       end
 
-      if str_parse[:type] == :heredoc
-        add_heredoc_content str_buffer, str_parse
-      else
-        add_string_content str_buffer, str_parse
-      end
+      add_string_content str_buffer, str_parse
 
       complete_str = str_buffer.join ''
       @line += complete_str.count("\n")
-      return :tSTRING_CONTENT, complete_str
-    end
-
-    def add_heredoc_content(str_buffer, str_parse)
-      scanner = @scanner
-
-      eos_regx = /[ \t]*#{Regexp.escape(str_parse[:end])}(\r*\n|$)/
-      expand = true
-
-      until scanner.eos?
-        c = nil
-        handled = true
 
-        if scan(/\n/)
-          c = scanner.matched
-        elsif check(eos_regx) && scanner.bol?
-          break # eos!
-        elsif expand && check(/#(?=[\$\@\{])/)
-          break
-        elsif scan(/\\/)
-          if str_parse[:type] == :regexp
-            if scan(/(.)/)
-              c = "\\" + scanner.matched
-            end
-          else
-            c = if scan(/n/)
-              "\n"
-            elsif scan(/r/)
-              "\r"
-            elsif scan(/\n/)
-              "\n"
-            elsif scan(/t/)
-              "\t"
-            else
-              # escaped char doesnt need escaping, so just return it
-              scan(/./)
-              scanner.matched
-            end
-          end
-        else
-          handled = false
-        end
-
-        unless handled
-          reg = Regexp.new("[^#{Regexp.escape str_parse[:end]}\#\0\\\\\n]+|.")
-
-          scan reg
-          c = scanner.matched
-        end
-
-        c ||= scanner.matched
-        str_buffer << c
-      end
-
-      raise "reached EOF while in string" if scanner.eos?
+      self.yylval = complete_str
+      return :tSTRING_CONTENT
     end
 
     def add_string_content(str_buffer, str_parse)
-      scanner = @scanner
-      # regexp for end of string/regexp
-      # end_str_re = /#{str_parse[:end]}/
-      end_str_re = Regexp.new(Regexp.escape(str_parse[:end]))
+      func = str_parse[:func]
 
-      expand = strterm_expand?(str_parse)
+      end_str_re = Regexp.new(Regexp.escape(str_parse[:term]))
 
-      words = ['W', 'w'].include? str_parse[:beg]
+      qwords = (func & STR_FUNC_QWORDS) != 0
+      expand = (func & STR_FUNC_EXPAND) != 0
+      regexp = (func & STR_FUNC_REGEXP) != 0
+      escape = (func & STR_FUNC_ESCAPE) != 0
+      xquote = (func == STR_XQUOTE)
 
       until scanner.eos?
         c = nil
@@ -323,51 +403,60 @@ module Opal
             break
           end
 
-        elsif str_parse[:balance] and scan Regexp.new(Regexp.escape(str_parse[:beg]))
+        elsif str_parse[:balance] and scan Regexp.new(Regexp.escape(str_parse[:paren]))
           str_parse[:nesting] += 1
           c = scanner.matched
 
-        elsif words && scan(/\s/)
-          scanner.pos -= 1
+        elsif qwords && scan(/\s/)
+          pushback(1)
           break
-
         elsif expand && check(/#(?=[\$\@\{])/)
           break
-
-        #elsif scan(/\\\\/)
-          #c = scanner.matched
-
+        elsif qwords and scan(/\s/)
+          pushback(1)
+          break
         elsif scan(/\\/)
-          if str_parse[:type] == :regexp
+          if xquote # opal - treat xstrings as dquotes? forces us to double escape
+            c = "\\" + scan(/./)
+          elsif qwords and scan(/\n/)
+            str_buffer << "\n"
+            next
+          elsif expand and scan(/\n/)
+            next
+          elsif qwords and scan(/\s/)
+            c = ' '
+          elsif regexp
             if scan(/(.)/)
               c = "\\" + scanner.matched
             end
-          else
-            c = if scan(/n/)
-              "\n"
-            elsif scan(/r/)
-              "\r"
-            elsif scan(/\n/)
-              "\n"
-            elsif scan(/t/)
-              "\t"
+          elsif expand
+            c = self.read_escape
+          elsif scan(/\n/)
+            # nothing..
+          elsif scan(/\\/)
+            if escape
+              c = "\\\\"
+            else
+              c = scanner.matched
+            end
+          else # \\
+            unless scan(end_str_re)
+              str_buffer << "\\"
             else
-              # escaped char doesnt need escaping, so just return it
-              scan(/./)
-              scanner.matched
-            end 
+              #c = scanner.matched
+            end
           end
         else
           handled = false
         end
 
         unless handled
-          reg = if words
-                  Regexp.new("[^#{Regexp.escape str_parse[:end]}\#\0\n\ \\\\]+|.")
+          reg = if qwords
+                  Regexp.new("[^#{Regexp.escape str_parse[:term]}\#\0\n\ \\\\]+|.")
                 elsif str_parse[:balance]
-                  Regexp.new("[^#{Regexp.escape str_parse[:end]}#{Regexp.escape str_parse[:beg]}\#\0\\\\]+|.")
+                  Regexp.new("[^#{Regexp.escape str_parse[:term]}#{Regexp.escape str_parse[:paren]}\#\0\\\\]+|.")
                 else
-                  Regexp.new("[^#{Regexp.escape str_parse[:end]}\#\0\\\\]+|.")
+                  Regexp.new("[^#{Regexp.escape str_parse[:term]}\#\0\\\\]+|.")
                 end
 
           scan reg
@@ -382,36 +471,39 @@ module Opal
     end
 
     def heredoc_identifier
-      if @scanner.scan(/(-?)['"]?(\w+)['"]?/)
+      if scan(/(-?)['"]?(\w+)['"]?/)
         heredoc = @scanner[2]
-        self.strterm = new_strterm(:heredoc, heredoc, heredoc)
+        self.strterm = new_strterm(STR_DQUOTE, heredoc, heredoc)
+        self.strterm[:type] = :heredoc
 
         # if ruby code at end of line after heredoc, we have to store it to
         # parse after heredoc is finished parsing
-        end_of_line = @scanner.scan(/.*\n/)
+        end_of_line = scan(/.*\n/)
         self.strterm[:scanner] = StringScanner.new(end_of_line) if end_of_line != "\n"
 
-        return :tSTRING_BEG, heredoc
+        self.line += 1
+        self.yylval = heredoc
+        return :tSTRING_BEG
       end
     end
 
     def process_identifier(matched, cmd_start)
-      scanner = @scanner
-      matched = scanner.matched
+      last_state = @lex_state
 
-      if scanner.peek(2) != '::' && scan(/:/)
+      if !check(/::/) and scan(/:/)
         @lex_state = :expr_beg
-        return :tLABEL, "#{matched}"
+        self.yylval = matched
+        return :tLABEL
       end
 
       if matched == 'defined?'
         if after_operator?
           @lex_state = :expr_end
-          return :tIDENTIFIER, matched
+          return :tIDENTIFIER
         end
 
         @lex_state = :expr_arg
-        return :kDEFINED, 'defined?'
+        return :kDEFINED
       end
 
       if matched.end_with? '?', '!'
@@ -435,7 +527,8 @@ module Opal
         @lex_state = kw.state
 
         if old_state == :expr_fname
-          return [kw.id[0], kw.name]
+          self.yylval = kw.name
+          return kw.id[0]
         end
 
         if @lex_state == :expr_beg
@@ -445,34 +538,36 @@ module Opal
         if matched == "do"
           if after_operator?
             @lex_state = :expr_end
-            return :tIDENTIFIER, matched
+            return :tIDENTIFIER
           end
 
           if @start_of_lambda
             @start_of_lambda = false
             @lex_state = :expr_beg
-            return [:kDO_LAMBDA, scanner.matched]
+            return :kDO_LAMBDA
           elsif cond?
             @lex_state = :expr_beg
-            return :kDO_COND, matched
+            return :kDO_COND
           elsif cmdarg? && @lex_state != :expr_cmdarg
             @lex_state = :expr_beg
-            return :kDO_BLOCK, matched
+            return :kDO_BLOCK
           elsif @lex_state == :expr_endarg
-            return :kDO_BLOCK, matched
+            return :kDO_BLOCK
           else
             @lex_state = :expr_beg
-            return :kDO, matched
+            return :kDO
           end
         else
           if old_state == :expr_beg or old_state == :expr_value
-            return [kw.id[0], matched]
+            self.yylval = matched
+            return kw.id[0]
           else
             if kw.id[0] != kw.id[1]
               @lex_state = :expr_beg
             end
 
-            return [kw.id[1], matched]
+            self.yylval = matched
+            return kw.id[1]
           end
         end
       end
@@ -483,123 +578,116 @@ module Opal
         @lex_state = :expr_end
       end
 
-      return [matched =~ /^[A-Z]/ ? :tCONSTANT : :tIDENTIFIER, matched]
+      if ![:expr_dot, :expr_fname].include?(last_state) and has_local?(matched)
+        @lex_state = :expr_end
+      end
+
+      return matched =~ /^[A-Z]/ ? :tCONSTANT : :tIDENTIFIER
     end
 
     def yylex
+      @yylval = ''
       @space_seen = false
       cmd_start = false
       c = ''
 
       if self.strterm
-        return next_string_token
+        if self.strterm[:type] == :heredoc
+          token = here_document(self.strterm)
+        else
+          token = parse_string
+        end
+
+        if token == :tSTRING_END or token == :tREGEXP_END
+          self.strterm = nil
+          @lex_state = :expr_end
+        end
+
+        return token
       end
 
       while true
-        if scan(/\ |\t|\r/)
+        if skip(/\ |\t|\r/)
           @space_seen = true
           next
 
-        elsif scan(/(\n|#)/)
+        elsif skip(/(\n|#)/)
           c = scanner.matched
-          if c == '#' then scan(/(.*)/) else @line += 1; end
+          if c == '#'
+            skip(/(.*)/)
+          else
+            self.line += 1
+          end
+
+          skip(/(\n+)/)
 
-          scan(/(\n+)/)
-          @line += scanner.matched.length if scanner.matched
+          if scanner.matched
+            self.line += scanner.matched.length
+          end
 
           next if [:expr_beg, :expr_dot].include? @lex_state
 
-          if scan(/([\ \t\r\f\v]*)\./)
+          if skip(/([\ \t\r\f\v]*)\./)
             @space_seen = true unless scanner[1].empty?
-            scanner.pos = scanner.pos - 1
+            pushback(1)
 
             next unless check(/\.\./)
           end
 
           cmd_start = true
           @lex_state = :expr_beg
-          return :tNL, '\\n'
+          self.yylval = '\\n'
+          return :tNL
 
         elsif scan(/\;/)
           @lex_state = :expr_beg
-          return :tSEMI, ';'
+          return :tSEMI
 
-        elsif scan(/\*/)
-          if scan(/\*/)
-            if scan(/\=/)
-              @lex_state = :expr_beg
-              return :tOP_ASGN, '**'
-            end
-
-            if @lex_state == :expr_fname or @lex_state == :expr_dot
-              @lex_state = :expr_arg
-            else
-              @lex_state = :expr_beg
-            end
-
-            return :tPOW, '**'
-
-          else
-            if scan(/\=/)
-              @lex_state = :expr_beg
-              return :tOP_ASGN, '*'
-            end
-          end
-
-          if scan(/\*\=/)
+        elsif check(/\*/)
+          if scan(/\*\*\=/)
             @lex_state = :expr_beg
-            return :tOP_ASGN, '**'
-          end
-
-          if scan(/\*/)
-            if after_operator?
-              @lex_state = :expr_arg
-            else
-              @lex_state = :expr_beg
-            end
-
-            return :tPOW, '**'
-          end
-
-          if scan(/\=/)
+            return new_op_asgn('**')
+          elsif scan(/\*\*/)
+            self.set_arg_state
+            return :tPOW
+          elsif scan(/\*\=/)
             @lex_state = :expr_beg
-            return :tOP_ASGN, '*'
+            return new_op_asgn('*')
           else
-            result = '*'
-            if @lex_state == :expr_fname or @lex_state == :expr_dot
+            scan(/\*/)
+
+            if after_operator?
               @lex_state = :expr_arg
-              return :tSTAR2, result
+              return :tSTAR2
             elsif @space_seen && check(/\S/)
               @lex_state = :expr_beg
-              return :tSTAR, result
+              return :tSTAR
             elsif [:expr_beg, :expr_mid].include? @lex_state
               @lex_state = :expr_beg
-              return :tSTAR, result
+              return :tSTAR
             else
               @lex_state = :expr_beg
-              return :tSTAR2, result
+              return :tSTAR2
             end
           end
 
         elsif scan(/\!/)
-          c = scan(/./)
           if after_operator?
             @lex_state = :expr_arg
-            if c == "@"
+            if scan(/@/)
               return :tBANG, '!'
             end
           else
             @lex_state = :expr_beg
           end
 
-          if c == '='
-            return :tNEQ, '!='
-          elsif c == '~'
-            return :tNMATCH, '!~'
+          if scan(/\=/)
+            return :tNEQ
+          elsif scan(/\~/)
+            return :tNMATCH
           end
 
-          scanner.pos = scanner.pos - 1
-          return :tBANG, '!'
+          return :tBANG
 
         elsif scan(/\=/)
           if @lex_state == :expr_beg and !@space_seen
@@ -614,7 +702,7 @@ module Opal
 
                 if scan(/\=end/) and space?
                   @line += line_count
-                  return next_token
+                  return yylex
                 end
 
                 if scan(/\n/)
@@ -627,53 +715,49 @@ module Opal
             end
           end
 
-          @lex_state = if after_operator?
-                         :expr_arg
-                       else
-                         :expr_beg
-                       end
+          self.set_arg_state
 
           if scan(/\=/)
             if scan(/\=/)
-              return :tEQQ, '==='
+              return :tEQQ
             end
 
-            return :tEQ, '=='
+            return :tEQ
           end
 
           if scan(/\~/)
-            return :tMATCH, '=~'
+            return :tMATCH
           elsif scan(/\>/)
-            return :tASSOC, '=>'
+            return :tASSOC
           end
 
-          return :tEQL, '='
+          return :tEQL
 
         elsif scan(/\"/)
-          self.strterm = new_strterm(:dquote, '"', '"')
-          return :tSTRING_BEG, scanner.matched
+          self.strterm = new_strterm(STR_DQUOTE, '"', "\0")
+          return :tSTRING_BEG
 
         elsif scan(/\'/)
-          self.strterm = new_strterm(:squote, "'", "'")
-          return :tSTRING_BEG, scanner.matched
+          self.strterm = new_strterm(STR_SQUOTE, "'", "\0")
+          return :tSTRING_BEG
 
         elsif scan(/\`/)
-          self.strterm = new_strterm(:xquote, '`', '`')
-          return :tXSTRING_BEG, scanner.matched
+          self.strterm = new_strterm(STR_XQUOTE, "`", "\0")
+          return :tXSTRING_BEG
 
         elsif scan(/\&/)
           if scan(/\&/)
             @lex_state = :expr_beg
 
             if scan(/\=/)
-              return :tOP_ASGN, '&&'
+              return new_op_asgn('&&')
             end
 
-            return :tANDOP, '&&'
+            return :tANDOP
 
           elsif scan(/\=/)
             @lex_state = :expr_beg
-            return :tOP_ASGN, '&'
+            return new_op_asgn('&')
           end
 
           if spcarg?
@@ -686,95 +770,93 @@ module Opal
             result = :tAMPER2
           end
 
-          @lex_state = after_operator? ? :expr_arg : :expr_beg
-          return result, '&'
+          self.set_arg_state
+          return result
 
         elsif scan(/\|/)
           if scan(/\|/)
             @lex_state = :expr_beg
             if scan(/\=/)
-              return :tOP_ASGN, '||'
+              return new_op_asgn('||')
             end
 
-            return :tOROP, '||'
+            return :tOROP
 
           elsif scan(/\=/)
-            return :tOP_ASGN, '|'
+            return new_op_asgn('|')
           end
 
-          @lex_state = after_operator?() ? :expr_arg : :expr_beg
-          return :tPIPE, '|'
+          self.set_arg_state
+          return :tPIPE
 
         elsif scan(/\%[QqWwixr]/)
           str_type = scanner.matched[1, 1]
-          paren = scan(/./)
-
-          term = case paren
-                 when '(' then ')'
-                 when '[' then ']'
-                 when '{' then '}'
-                 else paren
-                 end
-
-          case str_type
-          when 'Q'
-            self.strterm = new_strterm2(:dquote, paren, term)
-            return :tSTRING_BEG, scanner.matched
-          when 'q'
-            self.strterm = new_strterm2(:squote, paren, term)
-            return :tSTRING_BEG, scanner.matched
-          when 'W'
-            self.strterm = new_strterm(:dword, 'W', term)
-            scan(/\s*/)
-            return :tWORDS_BEG, scanner.matched
-          when 'w', 'i'
-            self.strterm = new_strterm(:sword, 'w', term)
-            scan(/\s*/)
-            return :tAWORDS_BEG, scanner.matched
-          when 'x'
-            self.strterm = new_strterm2(:xquote, paren, term)
-            return :tXSTRING_BEG, scanner.matched
-          when 'r'
-            self.strterm = new_strterm2(:regexp, paren, term)
-            return :tREGEXP_BEG, scanner.matched
+          paren = term = scan(/./)
+
+          case term
+          when '(' then term = ')'
+          when '[' then term = ']'
+          when '{' then term = '}'
+          when '<' then term = '>'
+          else paren = "\0"
           end
 
+          token, func = case str_type
+                        when 'Q'
+                          [:tSTRING_BEG, STR_DQUOTE]
+                        when 'q'
+                          [:tSTRING_BEG, STR_SQUOTE]
+                        when 'W'
+                          skip(/\s*/)
+                          [:tWORDS_BEG, STR_DWORD]
+                        when 'w', 'i'
+                          skip(/\s*/)
+                          [:tAWORDS_BEG, STR_SWORD]
+                        when 'x'
+                          [:tXSTRING_BEG, STR_XQUOTE]
+                        when 'r'
+                          [:tREGEXP_BEG, STR_REGEXP]
+                        end
+
+          self.strterm = new_strterm2(func, term, paren)
+          return token
+
         elsif scan(/\//)
-          if [:expr_beg, :expr_mid].include? @lex_state
-            self.strterm = new_strterm(:regexp, '/', '/')
-            return :tREGEXP_BEG, scanner.matched
+          if beg?
+            self.strterm = new_strterm(STR_REGEXP, '/', '/')
+            return :tREGEXP_BEG
           elsif scan(/\=/)
             @lex_state = :expr_beg
-            return :tOP_ASGN, '/'
-          elsif @lex_state == :expr_fname or @lex_state == :expr_dot
+            return new_op_asgn('/')
+          elsif after_operator?
             @lex_state = :expr_arg
-          elsif @lex_state == :expr_cmdarg || @lex_state == :expr_arg
+          elsif arg?
             if !check(/\s/) && @space_seen
-              self.strterm = new_strterm(:regexp, '/', '/')
-              return :tREGEXP_BEG, scanner.matched
+              self.strterm = new_strterm(STR_REGEXP, '/', '/')
+              return :tREGEXP_BEG
             end
           else
             @lex_state = :expr_beg
           end
 
-          return :tDIVIDE, '/'
+          return :tDIVIDE
 
         elsif scan(/\%/)
           if scan(/\=/)
             @lex_state = :expr_beg
-            return :tOP_ASGN, '%'
+            return new_op_asgn('%')
           elsif check(/[^\s]/)
             if @lex_state == :expr_beg or (@lex_state == :expr_arg && @space_seen)
               start_word  = scan(/./)
               end_word    = { '(' => ')', '[' => ']', '{' => '}' }[start_word] || start_word
-              self.strterm = new_strterm2(:dquote, start_word, end_word)
-              return :tSTRING_BEG, scanner.matched
+              self.strterm = new_strterm2(STR_DQUOTE, end_word, start_word)
+              return :tSTRING_BEG
             end
           end
 
-          @lex_state = after_operator? ? :expr_arg : :expr_beg
+          self.set_arg_state
 
-          return :tPERCENT, '%'
+          return :tPERCENT
 
         elsif scan(/\\/)
           if scan(/\r?\n/)
@@ -786,9 +868,9 @@ module Opal
 
         elsif scan(/\(/)
           result = scanner.matched
-          if [:expr_beg, :expr_mid].include? @lex_state
+          if beg?
             result = :tLPAREN
-          elsif @space_seen && [:expr_arg, :expr_cmdarg].include?(@lex_state)
+          elsif @space_seen && arg?
             result = :tLPAREN_ARG
           else
             result = :tLPAREN2
@@ -798,125 +880,123 @@ module Opal
           cond_push 0
           cmdarg_push 0
 
-          return result, scanner.matched
+          return result
 
         elsif scan(/\)/)
           cond_lexpop
           cmdarg_lexpop
           @lex_state = :expr_end
-          return :tRPAREN, scanner.matched
+          return :tRPAREN
 
         elsif scan(/\[/)
           result = scanner.matched
 
-          if [:expr_fname, :expr_dot].include? @lex_state
+          if after_operator?
             @lex_state = :expr_arg
             if scan(/\]=/)
-              return :tASET, '[]='
+              return :tASET
             elsif scan(/\]/)
-              return :tAREF, '[]'
+              return :tAREF
             else
               raise "Unexpected '[' token"
             end
-          elsif [:expr_beg, :expr_mid].include?(@lex_state) || @space_seen
-            @lex_state = :expr_beg
-            cond_push 0
-            cmdarg_push 0
-            return :tLBRACK, scanner.matched
+          elsif beg?
+            result = :tLBRACK
+          elsif arg? && @space_seen
+            result =  :tLBRACK
           else
-            @lex_state = :expr_beg
-            cond_push 0
-            cmdarg_push 0
-            return :tLBRACK2, scanner.matched
+            result = :tLBRACK2
           end
 
+          @lex_state = :expr_beg
+          cond_push 0
+          cmdarg_push 0
+          return result
+
         elsif scan(/\]/)
           cond_lexpop
           cmdarg_lexpop
           @lex_state = :expr_end
-          return :tRBRACK, scanner.matched
+          return :tRBRACK
 
         elsif scan(/\}/)
           cond_lexpop
           cmdarg_lexpop
           @lex_state = :expr_end
 
-          return :tRCURLY, scanner.matched
+          return :tRCURLY
 
         elsif scan(/\.\.\./)
           @lex_state = :expr_beg
-          return :tDOT3, scanner.matched
+          return :tDOT3
 
         elsif scan(/\.\./)
           @lex_state = :expr_beg
-          return :tDOT2, scanner.matched
+          return :tDOT2
 
         elsif scan(/\./)
           @lex_state = :expr_dot unless @lex_state == :expr_fname
-          return :tDOT, scanner.matched
+          return :tDOT
 
         elsif scan(/\:\:/)
-          if [:expr_beg, :expr_mid, :expr_class].include? @lex_state
+          if beg?
             @lex_state = :expr_beg
-            return :tCOLON3, scanner.matched
-          elsif @space_seen && @lex_state == :expr_arg
+            return :tCOLON3
+          elsif spcarg?
             @lex_state = :expr_beg
-            return :tCOLON3, scanner.matched
+            return :tCOLON3
           end
 
           @lex_state = :expr_dot
-          return :tCOLON2, scanner.matched
+          return :tCOLON2
 
         elsif scan(/\:/)
           if end? || check(/\s/)
             unless check(/\w/)
               @lex_state = :expr_beg
-              return :tCOLON, ':'
+              return :tCOLON
             end
 
             @lex_state = :expr_fname
-            return :tSYMBEG, ':'
+            return :tSYMBEG
           end
 
           if scan(/\'/)
-            self.strterm = new_strterm(:ssym, "'", "'")
+            self.strterm = new_strterm(STR_SSYM, "'", "\0")
           elsif scan(/\"/)
-            self.strterm = new_strterm(:dsym, '"', '"')
+            self.strterm = new_strterm(STR_DSYM, '"', "\0")
           end
 
           @lex_state = :expr_fname
-          return :tSYMBEG, ':'
+          return :tSYMBEG
 
         elsif scan(/\^\=/)
           @lex_state = :expr_beg
-          return :tOP_ASGN, '^'
-        elsif scan(/\^/)
-          if @lex_state == :expr_fname or @lex_state == :expr_dot
-            @lex_state = :expr_arg
-            return :tCARET, scanner.matched
-          end
+          return new_op_asgn('^')
 
-          @lex_state = :expr_beg
-          return :tCARET, scanner.matched
+        elsif scan(/\^/)
+          self.set_arg_state
+          return :tCARET
 
         elsif check(/\</)
           if scan(/\<\<\=/)
             @lex_state = :expr_beg
-            return :tOP_ASGN, '<<'
+            return new_op_asgn('<<')
+
           elsif scan(/\<\</)
-            if @lex_state == :expr_fname or @lex_state == :expr_dot
+            if after_operator?
               @lex_state = :expr_arg
-              return :tLSHFT, '<<'
-            elsif ![:expr_dot, :expr_class].include?(@lex_state) && !end? && (!arg? || @space_seen)
+              return :tLSHFT
+            elsif !after_operator? && !end? && (!arg? || @space_seen)
               if token = heredoc_identifier
                 return token
               end
 
               @lex_state = :expr_beg
-              return :tLSHFT, '<<'
+              return :tLSHFT
             end
             @lex_state = :expr_beg
-            return :tLSHFT, '<<'
+            return :tLSHFT
           elsif scan(/\<\=\>/)
             if after_operator?
               @lex_state = :expr_arg
@@ -928,54 +1008,38 @@ module Opal
               @lex_state = :expr_beg
             end
 
-            return :tCMP, '<=>'
+            return :tCMP
           elsif scan(/\<\=/)
-            if @lex_state == :expr_fname or @lex_state == :expr_dot
-              @lex_state = :expr_arg
-            else
-              @lex_state = :expr_beg
-            end
-            return :tLEQ, '<='
+            self.set_arg_state
+            return :tLEQ
+
           elsif scan(/\</)
-            if @lex_state == :expr_fname or @lex_state == :expr_dot
-              @lex_state = :expr_arg
-            else
-              @lex_state = :expr_beg
-            end
-            return :tLT, '<'
+            self.set_arg_state
+            return :tLT
           end
 
         elsif check(/\>/)
           if scan(/\>\>\=/)
-            return :tOP_ASGN, '>>'
+            return new_op_asgn('>>')
+
           elsif scan(/\>\>/)
-            if @lex_state == :expr_fname or @lex_state == :expr_dot
-              @lex_state = :expr_arg
-            else
-              @lex_state = :expr_beg
-            end
-            return :tRSHFT, '>>'
+            self.set_arg_state
+            return :tRSHFT
+
           elsif scan(/\>\=/)
-            if @lex_state == :expr_fname or @lex_state == :expr_dot
-              @lex_state = :expr_end
-            else
-              @lex_state = :expr_beg
-            end
-            return :tGEQ, scanner.matched
+            self.set_arg_state
+            return :tGEQ
+
           elsif scan(/\>/)
-            if @lex_state == :expr_fname or @lex_state == :expr_dot
-              @lex_state = :expr_arg
-            else
-              @lex_state = :expr_beg
-            end
-            return :tGT, '>'
+            self.set_arg_state
+            return :tGT
           end
 
         elsif scan(/->/)
           # FIXME: # should be :expr_arg, but '(' breaks it...
           @lex_state = :expr_end
           @start_of_lambda = true
-          return [:tLAMBDA, scanner.matched]
+          return :tLAMBDA
 
         elsif scan(/[+-]/)
           matched = scanner.matched
@@ -987,103 +1051,122 @@ module Opal
 
           if beg?
             @lex_state = :expr_mid
-            return [utype, matched]
+            self.yylval = matched
+            return utype
           elsif after_operator?
             @lex_state = :expr_arg
-            return [:tIDENTIFIER, matched + '@'] if scan(/@/)
-            return [sign, matched]
+            if scan(/@/)
+              self.yylval = matched + '@'
+              return :tIDENTIFIER
+            end
+
+            self.yylval = matched
+            return sign
           end
 
           if scan(/\=/)
             @lex_state = :expr_beg
-            return [:tOP_ASGN, matched]
+            return new_op_asgn(matched)
           end
 
-          if arg?
-            if !space? && @space_seen
-              @lex_state = :expr_mid
-              return [utype, matched]
-            end
+          if spcarg?
+            @lex_state = :expr_mid
+            self.yylval = matched
+            return utype
           end
 
           @lex_state = :expr_beg
-          return [sign, sign]
+          self.yylval = matched
+          return sign
 
         elsif scan(/\?/)
           if end?
             @lex_state = :expr_beg
-            return :tEH, scanner.matched
+            return :tEH
           end
 
-          unless check(/\ |\t|\r|\s/)
+          if check(/\ |\t|\r|\s/)
+            @lex_state = :expr_beg
+            return :tEH
+          elsif scan(/\\/)
             @lex_state = :expr_end
-            return :tSTRING, scan(/./)
+            self.yylval = self.read_escape
+            return :tSTRING
           end
 
-          @lex_state = :expr_beg
-          return :tEH, scanner.matched
+          @lex_state = :expr_end
+          self.yylval = scan(/./)
+          return :tSTRING
 
         elsif scan(/\~/)
-          if @lex_state == :expr_fname
-            @lex_state = :expr_end
-            return :tTILDE, '~'
-          end
-          @lex_state = :expr_beg
-          return :tTILDE, '~'
+          self.set_arg_state
+          return :tTILDE
 
         elsif check(/\$/)
           if scan(/\$([1-9]\d*)/)
             @lex_state = :expr_end
-            return :tNTH_REF, scanner.matched.sub('$', '')
+            self.yylval = scanner.matched.sub('$', '')
+            return :tNTH_REF
 
           elsif scan(/(\$_)(\w+)/)
             @lex_state = :expr_end
-            return :tGVAR, scanner.matched
+            return :tGVAR
 
           elsif scan(/\$[\+\'\`\&!@\"~*$?\/\\:;=.,<>_]/)
             @lex_state = :expr_end
-            return :tGVAR, scanner.matched
+            return :tGVAR
           elsif scan(/\$\w+/)
             @lex_state = :expr_end
-            return :tGVAR, scanner.matched
+            return :tGVAR
           else
             raise "Bad gvar name: #{scanner.peek(5).inspect}"
           end
 
         elsif scan(/\$\w+/)
           @lex_state = :expr_end
-          return :tGVAR, scanner.matched
+          return :tGVAR
 
         elsif scan(/\@\@\w*/)
           @lex_state = :expr_end
-          return :tCVAR, scanner.matched
+          return :tCVAR
 
         elsif scan(/\@\w*/)
           @lex_state = :expr_end
-          return :tIVAR, scanner.matched
+          return :tIVAR
 
         elsif scan(/\,/)
           @lex_state = :expr_beg
-          return :tCOMMA, scanner.matched
+          return :tCOMMA
 
         elsif scan(/\{/)
           if @start_of_lambda
             @start_of_lambda = false
             @lex_state = :expr_beg
-            return [:tLAMBEG, scanner.matched]
+            return :tLAMBEG
 
-          elsif [:expr_end, :expr_arg, :expr_cmdarg].include? @lex_state
+          elsif arg? or @lex_state == :expr_end
             result = :tLCURLY
           elsif @lex_state == :expr_endarg
             result = :LBRACE_ARG
           else
-            result = '{'
+            result = :tLBRACE
           end
 
           @lex_state = :expr_beg
           cond_push 0
           cmdarg_push 0
-          return result, scanner.matched
+          return result
+
+        elsif scanner.bol? and skip(/\__END__(\n|$)/)
+          while true
+            if scanner.eos?
+              @eof_content = self.yylval
+              return false
+            end
+
+            scan(/(.*)/)
+            scan(/\n/)
+          end
 
         elsif check(/[0-9]/)
           return process_numeric
@@ -1094,11 +1177,12 @@ module Opal
 
         if scanner.eos?
           if @scanner_stack.size == 1 # our main scanner, we cant pop this
-            return [false, false]
+            self.yylval = false
+            return false
           else # we were probably parsing a heredoc, so pop that parser and continue
             @scanner_stack.pop
             @scanner = @scanner_stack.last
-            return next_token
+            return yylex
           end
         end