opal 0.4.4 → 0.5.0

data/lib/opal/parser/keywords.rb

@@ -0,0 +1,64 @@
+module Opal
+  module Keywords
+
+    class KeywordTable
+      attr_accessor :name, :id, :state
+
+      def initialize(name, id, state)
+        @name = name
+        @id = id
+        @state = state
+      end
+    end
+
+    KEYWORDS = [
+      ["__LINE__",  [:LINE,     :LINE],       :expr_end],
+      ["__FILE__",  [:FILE,     :FILE],       :expr_end],
+      ["alias",     [:ALIAS,    :ALIAS],      :expr_fname],
+      ["and",       [:AND,      :AND],        :expr_beg],
+      ["begin",     [:BEGIN,    :BEGIN],      :expr_beg],
+      ["break",     [:BREAK,    :BREAK],      :expr_mid],
+      ["case",      [:CASE,     :CASE],       :expr_beg],
+      ["class",     [:CLASS,    :CLASS],      :expr_class],
+      ["def",       [:DEF,      :DEF],        :expr_fname],
+      ["defined?",  [:DEFINED,  :DEFINED],    :expr_arg],
+      ["do",        [:DO,       :DO],         :expr_beg],
+      ["else",      [:ELSE,     :ELSE],       :expr_beg],
+      ["elsif",     [:ELSIF,    :ELSIF],      :expr_beg],
+      ["end",       [:END,      :END],        :expr_end],
+      ["ensure",    [:ENSURE,   :ENSURE],     :expr_beg],
+      ["false",     [:FALSE,    :FALSE],      :expr_end],
+      ["if",        [:IF,       :IF_MOD],     :expr_beg],
+      ["module",    [:MODULE,   :MODULE],     :expr_beg],
+      ["nil",       [:NIL,      :NIL],        :expr_end],
+      ["next",      [:NEXT,     :NEXT],       :expr_mid],
+      ["not",       [:NOT,      :NOT],        :expr_beg],
+      ["or",        [:OR,       :OR],         :expr_beg],
+      ["redo",      [:REDO,     :REDO],       :expr_end],
+      ["rescue",    [:RESCUE,   :RESCUE_MOD], :expr_mid],
+      ["return",    [:RETURN,   :RETURN],     :expr_mid],
+      ["self",      [:SELF,     :SELF],       :expr_end],
+      ["super",     [:SUPER,    :SUPER],      :expr_arg],
+      ["then",      [:THEN,     :THEN],       :expr_beg],
+      ["true",      [:TRUE,     :TRUE],       :expr_end],
+      ["undef",     [:UNDEF,    :UNDEF],      :expr_fname],
+      ["unless",    [:UNLESS,   :UNLESS_MOD], :expr_beg],
+      ["until",     [:UNTIL,    :UNTIL_MOD],  :expr_beg],
+      ["when",      [:WHEN,     :WHEN],       :expr_beg],
+      ["while",     [:WHILE,    :WHILE_MOD],  :expr_beg],
+      ["yield",     [:YIELD,    :YIELD],      :expr_arg]
+    ].map { |decl| KeywordTable.new(*decl) }
+
+    def self.map
+      unless @map
+        @map = {}
+        KEYWORDS.each { |k| @map[k.name] = k }
+      end
+      @map
+    end
+
+    def self.keyword(kw)
+      map[kw]
+    end
+  end
+end

data/lib/opal/{lexer.rb → parser/lexer.rb}

@@ -1,59 +1,24 @@
-require 'opal/core_ext'
-require 'opal/grammar'
-require 'opal/grammar_helpers'
-require 'opal/lexer_scope'
 require 'strscan'
+require 'opal/parser/keywords'
 
 module Opal
+  class Lexer
 
-  class Grammar < Racc::Parser
+    attr_reader :line, :scope_line, :scope
 
-    attr_reader :line
+    attr_accessor :lex_state, :strterm
 
-    def initialize
+    def initialize(source, file)
       @lex_state  = :expr_beg
       @cond       = 0
       @cmdarg     = 0
       @line       = 1
-      @scopes     = []
+      @file       = file
 
-      @string_parse_stack = []
+      @scanner = StringScanner.new(source)
+      @scanner_stack = [@scanner]
     end
 
-    def s(*parts)
-      sexp = Array.new(parts)
-      sexp.line = @line
-      sexp
-    end
-
-    def parse(source, file = '(string)')
-      @file = file
-      @scanner = StringScanner.new source
-      push_scope
-      result = do_parse
-      pop_scope
-
-      result
-    end
-
-    def on_error(t, val, vstack)
-      raise "parse error on value #{val.inspect} (#{token_to_str(t) || '?'}) :#{@file}:#{@line}"
-    end
-
-    def push_scope(type = nil)
-      top = @scopes.last
-      scope = LexerScope.new type
-      scope.parent = top
-      @scopes << scope
-      @scope = scope
-    end
-
-    def pop_scope
-      @scopes.pop
-      @scope = @scopes.last
-    end
-
-
     def cond_push(n)
       @cond = (@cond << 1) | (n & 1)
     end
@@ -110,14 +75,22 @@ module Opal
       @scanner.check(/\s/)
     end
 
+    def next_token
+      self.yylex
+    end
+
+    def strterm_expand?(strterm)
+      type = strterm[:type]
+
+      [:dquote, :dsym, :dword, :heredoc, :xquote, :regexp].include? type
+    end
+
     def next_string_token
-      # str_parse, scanner = current_string_parse, @scanner
-      str_parse = @string_parse
+      str_parse = self.strterm
       scanner = @scanner
       space = false
 
-      # everything bar single quote and lower case bare wrds can interpolate
-      interpolate = str_parse[:interpolate]
+      expand = strterm_expand?(str_parse)
 
       words = ['w', 'W'].include? str_parse[:beg]
 
@@ -126,6 +99,23 @@ module Opal
       # if not end of string, so we must be parsing contents
       str_buffer = []
 
+      if str_parse[:type] == :heredoc
+        eos_regx = /[ \t]*#{Regexp.escape(str_parse[:end])}(\r*\n|$)/
+
+        if scanner.check(eos_regx)
+          scanner.scan(/[ \t]*#{Regexp.escape(str_parse[:end])}/)
+          self.strterm = nil
+
+          if str_parse[:scanner]
+            @scanner_stack << str_parse[:scanner]
+            @scanner = str_parse[:scanner]
+          end
+
+          @lex_state = :expr_end
+          return :STRING_END, scanner.matched
+        end
+      end
+
       # see if we can read end of string/xstring/regecp markers
       # if scanner.scan /#{str_parse[:end]}/
       if scanner.scan Regexp.new(Regexp.escape(str_parse[:end]))
@@ -134,7 +124,7 @@ module Opal
           scanner.pos -= 1
           return :SPACE, ' '
         end
-        @string_parse = nil
+        self.strterm = nil
 
         if str_parse[:balance]
           if str_parse[:nesting] == 0
@@ -148,7 +138,7 @@ module Opal
           else
             str_buffer << scanner.matched
             str_parse[:nesting] -= 1
-            @string_parse = str_parse
+            self.strterm = str_parse
           end
 
         elsif ['"', "'"].include? str_parse[:beg]
@@ -165,6 +155,11 @@ module Opal
           return :REGEXP_END, result
 
         else
+          if str_parse[:scanner]
+            @scanner_stack << str_parse[:scanner]
+            @scanner = str_parse[:scanner]
+          end
+
           @lex_state = :expr_end
           return :STRING_END, scanner.matched
         end
@@ -177,14 +172,14 @@ module Opal
         str_parse[:nesting] += 1
       elsif scanner.check(/#[@$]/)
         scanner.scan(/#/)
-        if interpolate
+        if expand
           return :STRING_DVAR, scanner.matched
         else
           str_buffer << scanner.matched
         end
 
       elsif scanner.scan(/#\{/)
-        if interpolate
+        if expand
           # we are into ruby code, so stop parsing content (for now)
           return :STRING_DBEG, scanner.matched
         else
@@ -196,19 +191,78 @@ module Opal
         str_buffer << '#'
       end
 
-      add_string_content str_buffer, str_parse
+      if str_parse[:type] == :heredoc
+        add_heredoc_content str_buffer, str_parse
+      else
+        add_string_content str_buffer, str_parse
+      end
+
       complete_str = str_buffer.join ''
       @line += complete_str.count("\n")
       return :STRING_CONTENT, complete_str
     end
 
+    def add_heredoc_content(str_buffer, str_parse)
+      scanner = @scanner
+
+      eos_regx = /[ \t]*#{Regexp.escape(str_parse[:end])}(\r*\n|$)/
+      expand = true
+
+      until scanner.eos?
+        c = nil
+        handled = true
+
+        if scanner.scan(/\n/)
+          c = scanner.matched
+        elsif scanner.check(eos_regx) && scanner.bol?
+          break # eos!
+        elsif expand && scanner.check(/#(?=[\$\@\{])/)
+          break
+        elsif scanner.scan(/\\/)
+          if str_parse[:regexp]
+            if scanner.scan(/(.)/)
+              c = "\\" + scanner.matched
+            end
+          else
+            c = if scanner.scan(/n/)
+              "\n"
+            elsif scanner.scan(/r/)
+              "\r"
+            elsif scanner.scan(/\n/)
+              "\n"
+            elsif scanner.scan(/t/)
+              "\t"
+            else
+              # escaped char doesnt need escaping, so just return it
+              scanner.scan(/./)
+              scanner.matched
+            end
+          end
+        else
+          handled = false
+        end
+
+        unless handled
+          reg = Regexp.new("[^#{Regexp.escape str_parse[:end]}\#\0\\\\\n]+|.")
+
+          scanner.scan reg
+          c = scanner.matched
+        end
+
+        c ||= scanner.matched
+        str_buffer << c
+      end
+
+      raise "reached EOF while in string" if scanner.eos?
+    end
+
     def add_string_content(str_buffer, str_parse)
       scanner = @scanner
       # regexp for end of string/regexp
       # end_str_re = /#{str_parse[:end]}/
       end_str_re = Regexp.new(Regexp.escape(str_parse[:end]))
-      # can be interpolate
-      interpolate = str_parse[:interpolate]
+
+      expand = strterm_expand?(str_parse)
 
       words = ['W', 'w'].include? str_parse[:beg]
 
@@ -237,7 +291,7 @@ module Opal
           scanner.pos -= 1
           break
 
-        elsif interpolate && scanner.check(/#(?=[\$\@\{])/)
+        elsif expand && scanner.check(/#(?=[\$\@\{])/)
           break
 
         #elsif scanner.scan(/\\\\/)
@@ -271,14 +325,12 @@ module Opal
           reg = if words
                   Regexp.new("[^#{Regexp.escape str_parse[:end]}\#\0\n\ \\\\]+|.")
                 elsif str_parse[:balance]
-                  #puts "using tis regexp"
                   Regexp.new("[^#{Regexp.escape str_parse[:end]}#{Regexp.escape str_parse[:beg]}\#\0\\\\]+|.")
                 else
                   Regexp.new("[^#{Regexp.escape str_parse[:end]}\#\0\\\\]+|.")
                 end
 
           scanner.scan reg
-          #puts scanner.matched
           c = scanner.matched
         end
 
@@ -289,16 +341,121 @@ module Opal
       raise "reached EOF while in string" if scanner.eos?
     end
 
-    def next_token
-      # if we are trying to parse a string, then delegate to that
-      return next_string_token if @string_parse
+    def heredoc_identifier
+      if @scanner.scan(/(-?)['"]?(\w+)['"]?/)
+        heredoc = @scanner[2]
+        self.strterm = { :type => :heredoc, :beg => heredoc, :end => heredoc }
+
+        # if ruby code at end of line after heredoc, we have to store it to
+        # parse after heredoc is finished parsing
+        end_of_line = @scanner.scan(/.*\n/)
+        self.strterm[:scanner] = StringScanner.new(end_of_line) if end_of_line != "\n"
+
+        return :STRING_BEG, heredoc
+      end
+    end
+
+    def process_identifier(matched, cmd_start)
+      scanner = @scanner
+      matched = scanner.matched
+
+      if scanner.peek(2) != '::' && scanner.scan(/:/)
+        @lex_state = :expr_beg
+        return :LABEL, "#{matched}"
+      end
+
+      if matched == 'defined?'
+        if after_operator?
+          @lex_state = :expr_end
+          return :IDENTIFIER, matched
+        end
+
+        @lex_state = :expr_arg
+        return :DEFINED, 'defined?'
+      end
+
+      if matched.end_with? '?', '!'
+        result = :IDENTIFIER
+      else
+        if @lex_state == :expr_fname
+          if scanner.scan(/\=/)
+            result = :IDENTIFIER
+            matched += scanner.matched
+          end
+
+        elsif matched =~ /^[A-Z]/
+          result = :CONSTANT
+        else
+          result = :IDENTIFIER
+        end
+      end
 
-      # scanner, @space_seen, cmd_start, c = @scanner, false, false, ''
+      if @lex_state != :expr_dot and kw = Keywords.keyword(matched)
+        old_state = @lex_state
+        @lex_state = kw.state
+
+        if old_state == :expr_fname
+          return [kw.id[0], kw.name]
+        end
+
+        if @lex_state == :expr_beg
+          cmd_start = true
+        end
+
+        if matched == "do"
+          if after_operator?
+            @lex_state = :expr_end
+            return :IDENTIFIER, matched
+          end
+
+          if @start_of_lambda
+            @start_of_lambda = false
+            @lex_state = :expr_beg
+            return [:DO_LAMBDA, scanner.matched]
+          elsif cond?
+            @lex_state = :expr_beg
+            return :DO_COND, matched
+          elsif cmdarg? && @lex_state != :expr_cmdarg
+            @lex_state = :expr_beg
+            return :DO_BLOCK, matched
+          elsif @lex_state == :expr_endarg
+            return :DO_BLOCK, matched
+          else
+            @lex_state = :expr_beg
+            return :DO, matched
+          end
+        else
+          if old_state == :expr_beg or old_state == :expr_value
+            return [kw.id[0], matched]
+          else
+            if kw.id[0] != kw.id[1]
+              @lex_state = :expr_beg
+            end
+
+            return [kw.id[1], matched]
+          end
+        end
+      end
+
+      if [:expr_beg, :expr_dot, :expr_mid, :expr_arg, :expr_cmdarg].include? @lex_state
+        @lex_state = cmd_start ? :expr_cmdarg : :expr_arg
+      else
+        @lex_state = :expr_end
+      end
+
+      return [matched =~ /^[A-Z]/ ? :CONSTANT : :IDENTIFIER, matched]
+    end
+
+    def yylex
       scanner = @scanner
       @space_seen = false
       cmd_start = false
       c = ''
 
+      if self.strterm
+        return next_string_token
+      end
+
       while true
         if scanner.scan(/\ |\t|\r/)
           @space_seen = true
@@ -370,8 +527,8 @@ module Opal
             return :OP_ASGN, '*'
           else
             result = '*'
-            if @lex_state == :expr_fname
-              @lex_state = :expr_end
+            if @lex_state == :expr_fname or @lex_state == :expr_dot
+              @lex_state = :expr_arg
               return '*', result
             elsif @space_seen && scanner.check(/\S/)
               @lex_state = :expr_beg
@@ -409,6 +566,7 @@ module Opal
           if @lex_state == :expr_beg and !@space_seen
             if scanner.scan(/begin/) and space?
               scanner.scan(/(.*)/) # end of line
+              line_count = 0
 
               while true
                 if scanner.eos?
@@ -416,10 +574,12 @@ module Opal
                 end
 
                 if scanner.scan(/\=end/) and space?
+                  @line += line_count
                   return next_token
                 end
 
                 if scanner.scan(/\n/)
+                  line_count += 1
                   next
                 end
 
@@ -451,15 +611,15 @@ module Opal
           return '=', '='
 
         elsif scanner.scan(/\"/)
-          @string_parse = { :beg => '"', :end => '"', :interpolate => true }
+          self.strterm = { :type => :dquote, :beg => '"', :end => '"' }
           return :STRING_BEG, scanner.matched
 
         elsif scanner.scan(/\'/)
-          @string_parse = { :beg => "'", :end => "'" }
+          self.strterm = { :type => :squote, :beg => "'", :end => "'" }
           return :STRING_BEG, scanner.matched
 
         elsif scanner.scan(/\`/)
-          @string_parse = { :beg => "`", :end => "`", :interpolate => true }
+          self.strterm = { :type => :xquote, :beg => "`", :end => "`" }
           return :XSTRING_BEG, scanner.matched
 
         elsif scanner.scan(/\&/)
@@ -509,48 +669,48 @@ module Opal
         elsif scanner.scan(/\%W/)
           start_word  = scanner.scan(/./)
           end_word    = { '(' => ')', '[' => ']', '{' => '}' }[start_word] || start_word
-          @string_parse = { :beg => 'W', :end => end_word, :interpolate => true }
+          self.strterm = { :type => :dword, :beg => 'W', :end => end_word  }
           scanner.scan(/\s*/)
           return :WORDS_BEG, scanner.matched
 
-        elsif scanner.scan(/\%w/)
+        elsif scanner.scan(/\%w/) or scanner.scan(/\%i/)
           start_word  = scanner.scan(/./)
           end_word    = { '(' => ')', '[' => ']', '{' => '}' }[start_word] || start_word
-          @string_parse = { :beg => 'w', :end => end_word }
+          self.strterm = { :type => :sword, :beg => 'w', :end => end_word }
           scanner.scan(/\s*/)
           return :AWORDS_BEG, scanner.matched
 
         elsif scanner.scan(/\%[Qq]/)
-          interpolate = scanner.matched.end_with? 'Q'
+          type = scanner.matched.end_with?('Q') ? :dquote : :squote
           start_word  = scanner.scan(/./)
           end_word    = { '(' => ')', '[' => ']', '{' => '}' }[start_word] || start_word
-          @string_parse = { :beg => start_word, :end => end_word, :balance => true, :nesting => 0, :interpolate => interpolate }
+          self.strterm = { :type => type, :beg => start_word, :end => end_word, :balance => true, :nesting => 0 }
           return :STRING_BEG, scanner.matched
 
         elsif scanner.scan(/\%x/)
           start_word = scanner.scan(/./)
           end_word   = { '(' => ')', '[' => ']', '{' => '}' }[start_word] || start_word
-          @string_parse = { :beg => start_word, :end => end_word, :balance => true, :nesting => 0, :interpolate => true }
+          self.strterm = { :type => :xquote, :beg => start_word, :end => end_word, :balance => true, :nesting => 0 }
           return :XSTRING_BEG, scanner.matched
 
         elsif scanner.scan(/\%r/)
           start_word = scanner.scan(/./)
           end_word   = { '(' => ')', '[' => ']', '{' => '}' }[start_word] || start_word
-          @string_parse = { :beg => start_word, :end => end_word, :regexp => true, :balance => true, :nesting => 0, :interpolate => true }
+          self.strterm = { :type => :regexp, :beg => start_word, :end => end_word, :regexp => true, :balance => true, :nesting => 0 }
           return :REGEXP_BEG, scanner.matched
 
         elsif scanner.scan(/\//)
           if [:expr_beg, :expr_mid].include? @lex_state
-            @string_parse = { :beg => '/', :end => '/', :interpolate => true, :regexp => true }
+            self.strterm = { :type => :regexp, :beg => '/', :end => '/', :regexp => true }
             return :REGEXP_BEG, scanner.matched
           elsif scanner.scan(/\=/)
             @lex_state = :expr_beg
             return :OP_ASGN, '/'
-          elsif @lex_state == :expr_fname
-            @lex_state = :expr_end
+          elsif @lex_state == :expr_fname or @lex_state == :expr_dot
+            @lex_state = :expr_arg
           elsif @lex_state == :expr_cmdarg || @lex_state == :expr_arg
             if !scanner.check(/\s/) && @space_seen
-              @string_parse = { :beg => '/', :end => '/', :interpolate => true, :regexp => true }
+              self.strterm = { :type => :regexp, :beg => '/', :end => '/', :regexp => true }
               return :REGEXP_BEG, scanner.matched
             end
           else
@@ -565,14 +725,15 @@ module Opal
             return :OP_ASGN, '%'
           elsif scanner.check(/[^\s]/)
             if @lex_state == :expr_beg or (@lex_state == :expr_arg && @space_seen)
-              interpolate = true
               start_word  = scanner.scan(/./)
               end_word    = { '(' => ')', '[' => ']', '{' => '}' }[start_word] || start_word
-              @string_parse = { :beg => start_word, :end => end_word, :balance => true, :nesting => 0, :interpolate => interpolate }
+              self.strterm = { :type => :dquote, :beg => start_word, :end => end_word, :balance => true, :nesting => 0 }
               return :STRING_BEG, scanner.matched
             end
           end
-          @lex_state = @lex_state == :expr_fname ? :expr_end : :expr_beg
+
+          @lex_state = after_operator? ? :expr_arg : :expr_beg
+
           return '%', '%'
 
         elsif scanner.scan(/\\/)
@@ -667,7 +828,7 @@ module Opal
           return '::', scanner.matched
 
         elsif scanner.scan(/\:/)
-          if [:expr_end, :expr_endarg].include?(@lex_state) || scanner.check(/\s/)
+          if end? || scanner.check(/\s/)
             unless scanner.check(/\w/)
               @lex_state = :expr_beg
               return ':', ':'
@@ -678,9 +839,9 @@ module Opal
           end
 
           if scanner.scan(/\'/)
-            @string_parse = { :beg => "'", :end => "'" }
+            self.strterm = { :type => :ssym, :beg => "'", :end => "'" }
           elsif scanner.scan(/\"/)
-            @string_parse = { :beg => '"', :end => '"', :interpolate => true }
+            self.strterm = { :type => :dsym, :beg => '"', :end => '"' }
           end
 
           @lex_state = :expr_fname
@@ -690,8 +851,8 @@ module Opal
           @lex_state = :expr_beg
           return :OP_ASGN, '^'
         elsif scanner.scan(/\^/)
-          if @lex_state == :expr_fname
-            @lex_state = :expr_end
+          if @lex_state == :expr_fname or @lex_state == :expr_dot
+            @lex_state = :expr_arg
             return '^', scanner.matched
           end
 
@@ -703,40 +864,41 @@ module Opal
             @lex_state = :expr_beg
             return :OP_ASGN, '<<'
           elsif scanner.scan(/\<\</)
-            if @lex_state == :expr_fname
-              @lex_state = :expr_end
+            if @lex_state == :expr_fname or @lex_state == :expr_dot
+              @lex_state = :expr_arg
               return '<<', '<<'
-            elsif ![:expr_end, :expr_dot, :expr_endarg, :expr_class].include?(@lex_state) && @space_seen
-              if scanner.scan(/(-?)['"]?(\w+)['"]?/)
-                heredoc = scanner[2]
-                # for now just scrap rest of line + skip down one line for
-                # string content
-                scanner.scan(/.*\n/)
-                @string_parse = { :beg => heredoc, :end => heredoc, :interpolate => true }
-                return :STRING_BEG, heredoc
+            elsif ![:expr_dot, :expr_class].include?(@lex_state) && !end? && (!arg? || @space_seen)
+              if token = heredoc_identifier
+                return token
               end
+
               @lex_state = :expr_beg
               return '<<', '<<'
             end
             @lex_state = :expr_beg
             return '<<', '<<'
           elsif scanner.scan(/\<\=\>/)
-            if @lex_state == :expr_fname
-              @lex_state = :expr_end
+            if after_operator?
+              @lex_state = :expr_arg
             else
+              if @lex_state == :expr_class
+                cmd_start = true
+              end
+
               @lex_state = :expr_beg
             end
+
             return '<=>', '<=>'
           elsif scanner.scan(/\<\=/)
-            if @lex_state == :expr_fname
-              @lex_state = :expr_end
+            if @lex_state == :expr_fname or @lex_state == :expr_dot
+              @lex_state = :expr_arg
             else
               @lex_state = :expr_beg
             end
             return '<=', '<='
           elsif scanner.scan(/\</)
-            if @lex_state == :expr_fname
-              @lex_state = :expr_end
+            if @lex_state == :expr_fname or @lex_state == :expr_dot
+              @lex_state = :expr_arg
             else
               @lex_state = :expr_beg
             end
@@ -754,14 +916,14 @@ module Opal
             end
             return '>>', '>>'
           elsif scanner.scan(/\>\=/)
-            if @lex_state == :expr_fname
+            if @lex_state == :expr_fname or @lex_state == :expr_dot
               @lex_state = :expr_end
             else
               @lex_state = :expr_beg
             end
             return '>=', scanner.matched
           elsif scanner.scan(/\>/)
-            if @lex_state == :expr_fname
+            if @lex_state == :expr_fname or @lex_state == :expr_dot
               @lex_state = :expr_arg
             else
               @lex_state = :expr_beg
@@ -782,8 +944,8 @@ module Opal
           if @lex_state == :expr_beg || @lex_state == :expr_mid
             @lex_state = :expr_mid
             return [sign, sign]
-          elsif @lex_state == :expr_fname
-            @lex_state = :expr_end
+          elsif @lex_state == :expr_fname or @lex_state == :expr_dot
+            @lex_state = :expr_arg
             return [:IDENTIFIER, result + scanner.matched] if scanner.scan(/@/)
             return [result, result]
           end
@@ -804,12 +966,12 @@ module Opal
           return [result, result]
 
         elsif scanner.scan(/\?/)
-          if [:expr_end, :expr_endarg].include?(@lex_state)
+          if end?
             @lex_state = :expr_beg
             return '?', scanner.matched
           end
 
-          unless scanner.check(/\ |\t|\r/)
+          unless scanner.check(/\ |\t|\r|\s/)
             @lex_state = :expr_end
             return :STRING, scanner.scan(/./)
           end
@@ -896,229 +1058,18 @@ module Opal
           end
 
         elsif scanner.scan(/(\w)+[\?\!]?/)
-          matched = scanner.matched
-          if scanner.peek(2) != '::' && scanner.scan(/:/)
-            @lex_state = :expr_beg
-            return :LABEL, "#{matched}"
-          end
-
-          case matched
-          when 'class'
-            if @lex_state == :expr_dot
-              @lex_state = :expr_end
-              return :IDENTIFIER, matched
-            end
-            @lex_state = :expr_class
-            return :CLASS, matched
-
-          when 'module'
-            return :IDENTIFIER, matched if @lex_state == :expr_dot
-            @lex_state = :expr_class
-            return :MODULE, matched
-
-          when 'defined?'
-            return :IDENTIFIER, matched if @lex_state == :expr_dot
-            @lex_state = :expr_arg
-            return :DEFINED, 'defined?'
-
-          when 'def'
-            @lex_state = :expr_fname
-            @scope_line = @line
-            return :DEF, matched
-
-          when 'undef'
-            @lex_state = :expr_fname
-            return :UNDEF, matched
-
-          when 'end'
-            if [:expr_dot, :expr_fname].include? @lex_state
-              @lex_state = :expr_end
-              return :IDENTIFIER, matched
-            end
-
-            @lex_state = :expr_end
-            return :END, matched
-
-          when 'do'
-            #puts cond?
-            #puts cmdarg?
-            #nputs @lex_state
-            if @start_of_lambda
-              @start_of_lambda = false
-              @lex_state = :expr_beg
-              return [:DO_LAMBDA, scanner.matched]
-            elsif cond?
-              @lex_state = :expr_beg
-              return :DO_COND, matched
-            elsif cmdarg? && @lex_state != :expr_cmdarg
-              @lex_state = :expr_beg
-              return :DO_BLOCK, matched
-            elsif @lex_state == :expr_endarg
-              return :DO_BLOCK, matched
-            else
-              @lex_state = :expr_beg
-              return :DO, matched
-            end
-
-          when 'if'
-            return :IF, matched if @lex_state == :expr_beg
-            @lex_state = :expr_beg
-            return :IF_MOD, matched
-
-          when 'unless'
-            return :UNLESS, matched if @lex_state == :expr_beg
-            @lex_state = :expr_beg
-            return :UNLESS_MOD, matched
-
-          when 'else'
-            return :ELSE, matched
-
-          when 'elsif'
-            return :ELSIF, matched
-
-          when 'self'
-            @lex_state = :expr_end unless @lex_state == :expr_fname
-            return :SELF, matched
-
-          when 'true'
-            @lex_state = :expr_end
-            return :TRUE, matched
-
-          when 'false'
-            @lex_state = :expr_end
-            return :FALSE, matched
-
-          when 'nil'
-            @lex_state = :expr_end
-            return :NIL, matched
-
-          when '__LINE__'
-            @lex_state = :expr_end
-            return :LINE, @line.to_s
-
-          when '__FILE__'
-            @lex_state = :expr_end
-            return :FILE, matched
-
-          when 'begin'
-            if [:expr_dot, :expr_fname].include? @lex_state
-              @lex_state = :expr_end
-              return :IDENTIFIER, matched
-            end
-            @lex_state = :expr_beg
-            return :BEGIN, matched
-
-          when 'rescue'
-            return :IDENTIFIER, matched if [:expr_dot, :expr_fname].include? @lex_state
-            if @lex_state == :expr_beg
-              @lex_state = :expr_mid
-              return :RESCUE, matched
-            end
-            @lex_state = :expr_beg
-            return :RESCUE_MOD, matched
-
-          when 'ensure'
-            @lex_state = :expr_beg
-            return :ENSURE, matched
-
-          when 'case'
-            @lex_state = :expr_beg
-            return :CASE, matched
-
-          when 'when'
-            @lex_state = :expr_beg
-            return :WHEN, matched
-
-          when 'or'
-            @lex_state = :expr_beg
-            return :OR, matched
-
-          when 'and'
-            @lex_state = :expr_beg
-            return :AND, matched
-
-          when 'not'
-            @lex_state = :expr_arg
-            return :NOT, matched
-
-          when 'return'
-            @lex_state = :expr_mid
-            return :RETURN, matched
-
-          when 'next'
-            if @lex_state == :expr_dot || @lex_state == :expr_fname
-              @lex_state = :expr_end
-              return :IDENTIFIER, matched
-            end
-
-            @lex_state = :expr_mid
-            return :NEXT, matched
-
-          when 'redo'
-            if @lex_state == :expr_dot || @lex_state == :expr_fname
-              @lex_state = :expr_end
-              return :IDENTIFIER, matched
-            end
-
-            @lex_state = :expr_mid
-            return :REDO, matched
-
-          when 'break'
-            @lex_state = :expr_mid
-            return :BREAK, matched
-
-          when 'super'
-            @lex_state = :expr_arg
-            return :SUPER, matched
-
-          when 'then'
-            @lex_state = :expr_beg
-            return :THEN, matched
-
-          when 'while'
-            return :WHILE, matched if @lex_state == :expr_beg
-            @lex_state = :expr_beg
-            return :WHILE_MOD, matched
-
-          when 'until'
-            return :UNTIL, matched if @lex_state == :expr_beg
-            @lex_state = :expr_beg
-            return :UNTIL_MOD, matched
-
-          when 'yield'
-            @lex_state = :expr_arg
-            return :YIELD, matched
-
-          when 'alias'
-            @lex_state = :expr_fname
-            return :ALIAS, matched
-          end
-
-          matched = matched
-          if scanner.peek(2) != '::' && scanner.scan(/\:/)
-            return :LABEL, matched 
-          end
-
-          if @lex_state == :expr_fname
-            if scanner.scan(/\=/)
-              @lex_state = :expr_end
-              return :IDENTIFIER, matched + scanner.matched
-            end
-          end
+          return process_identifier scanner.matched, cmd_start
+        end
 
-          if [:expr_beg, :expr_dot, :expr_mid, :expr_arg, :expr_cmdarg].include? @lex_state
-            # old:
-            #@lex_state = :expr_cmdarg
-            # new:
-            @lex_state = cmd_start ? :expr_cmdarg : :expr_arg
-          else
-            @lex_state = :expr_end
+        if scanner.eos?
+          if @scanner_stack.size == 1 # our main scanner, we cant pop this
+            return [false, false]
+          else # we were probably parsing a heredoc, so pop that parser and continue
+            @scanner_stack.pop
+            @scanner = @scanner_stack.last
+            return next_token
           end
-
-          return [matched =~ /^[A-Z]/ ? :CONSTANT : :IDENTIFIER, matched]
-
         end
-        return [false, false] if scanner.eos?
 
         raise "Unexpected content in parsing stream `#{scanner.peek 5}` :#{@file}:#{@line}"
       end