opal 0.5.2 → 0.5.4

data/lib/opal/parser/keywords.rb

@@ -12,41 +12,41 @@ module Opal
     end
 
     KEYWORDS = [
-      ["__LINE__",  [:LINE,     :LINE],       :expr_end],
-      ["__FILE__",  [:FILE,     :FILE],       :expr_end],
-      ["alias",     [:ALIAS,    :ALIAS],      :expr_fname],
-      ["and",       [:AND,      :AND],        :expr_beg],
-      ["begin",     [:BEGIN,    :BEGIN],      :expr_beg],
-      ["break",     [:BREAK,    :BREAK],      :expr_mid],
-      ["case",      [:CASE,     :CASE],       :expr_beg],
-      ["class",     [:CLASS,    :CLASS],      :expr_class],
-      ["def",       [:DEF,      :DEF],        :expr_fname],
-      ["defined?",  [:DEFINED,  :DEFINED],    :expr_arg],
-      ["do",        [:DO,       :DO],         :expr_beg],
-      ["else",      [:ELSE,     :ELSE],       :expr_beg],
-      ["elsif",     [:ELSIF,    :ELSIF],      :expr_beg],
-      ["end",       [:END,      :END],        :expr_end],
-      ["ensure",    [:ENSURE,   :ENSURE],     :expr_beg],
-      ["false",     [:FALSE,    :FALSE],      :expr_end],
-      ["if",        [:IF,       :IF_MOD],     :expr_beg],
-      ["module",    [:MODULE,   :MODULE],     :expr_beg],
-      ["nil",       [:NIL,      :NIL],        :expr_end],
-      ["next",      [:NEXT,     :NEXT],       :expr_mid],
-      ["not",       [:NOT,      :NOT],        :expr_beg],
-      ["or",        [:OR,       :OR],         :expr_beg],
-      ["redo",      [:REDO,     :REDO],       :expr_end],
-      ["rescue",    [:RESCUE,   :RESCUE_MOD], :expr_mid],
-      ["return",    [:RETURN,   :RETURN],     :expr_mid],
-      ["self",      [:SELF,     :SELF],       :expr_end],
-      ["super",     [:SUPER,    :SUPER],      :expr_arg],
-      ["then",      [:THEN,     :THEN],       :expr_beg],
-      ["true",      [:TRUE,     :TRUE],       :expr_end],
-      ["undef",     [:UNDEF,    :UNDEF],      :expr_fname],
-      ["unless",    [:UNLESS,   :UNLESS_MOD], :expr_beg],
-      ["until",     [:UNTIL,    :UNTIL_MOD],  :expr_beg],
-      ["when",      [:WHEN,     :WHEN],       :expr_beg],
-      ["while",     [:WHILE,    :WHILE_MOD],  :expr_beg],
-      ["yield",     [:YIELD,    :YIELD],      :expr_arg]
+      ["__LINE__",  [:k__LINE__, :k__LINE__],   :expr_end],
+      ["__FILE__",  [:k__FILE__, :k__FILE__],   :expr_end],
+      ["alias",     [:kALIAS,    :kALIAS],      :expr_fname],
+      ["and",       [:kAND,      :kAND],        :expr_beg],
+      ["begin",     [:kBEGIN,    :kBEGIN],      :expr_beg],
+      ["break",     [:kBREAK,    :kBREAK],      :expr_mid],
+      ["case",      [:kCASE,     :kCASE],       :expr_beg],
+      ["class",     [:kCLASS,    :kCLASS],      :expr_class],
+      ["def",       [:kDEF,      :kDEF],        :expr_fname],
+      ["defined?",  [:kDEFINED,  :kDEFINED],    :expr_arg],
+      ["do",        [:kDO,       :kDO],         :expr_beg],
+      ["else",      [:kELSE,     :kELSE],       :expr_beg],
+      ["elsif",     [:kELSIF,    :kELSIF],      :expr_beg],
+      ["end",       [:kEND,      :kEND],        :expr_end],
+      ["ensure",    [:kENSURE,   :kENSURE],     :expr_beg],
+      ["false",     [:kFALSE,    :kFALSE],      :expr_end],
+      ["if",        [:kIF,       :kIF_MOD],     :expr_beg],
+      ["module",    [:kMODULE,   :kMODULE],     :expr_beg],
+      ["nil",       [:kNIL,      :kNIL],        :expr_end],
+      ["next",      [:kNEXT,     :kNEXT],       :expr_mid],
+      ["not",       [:kNOT,      :kNOT],        :expr_beg],
+      ["or",        [:kOR,       :kOR],         :expr_beg],
+      ["redo",      [:kREDO,     :kREDO],       :expr_end],
+      ["rescue",    [:kRESCUE,   :kRESCUE_MOD], :expr_mid],
+      ["return",    [:kRETURN,   :kRETURN],     :expr_mid],
+      ["self",      [:kSELF,     :kSELF],       :expr_end],
+      ["super",     [:kSUPER,    :kSUPER],      :expr_arg],
+      ["then",      [:kTHEN,     :kTHEN],       :expr_beg],
+      ["true",      [:kTRUE,     :kTRUE],       :expr_end],
+      ["undef",     [:kUNDEF,    :kUNDEF],      :expr_fname],
+      ["unless",    [:kUNLESS,   :kUNLESS_MOD], :expr_beg],
+      ["until",     [:kUNTIL,    :kUNTIL_MOD],  :expr_beg],
+      ["when",      [:kWHEN,     :kWHEN],       :expr_beg],
+      ["while",     [:kWHILE,    :kWHILE_MOD],  :expr_beg],
+      ["yield",     [:kYIELD,    :kYIELD],      :expr_arg]
     ].map { |decl| KeywordTable.new(*decl) }
 
     def self.map

data/lib/opal/parser/lexer.rb

@@ -6,7 +6,7 @@ module Opal
 
     attr_reader :line, :scope_line, :scope
 
-    attr_accessor :lex_state, :strterm
+    attr_accessor :lex_state, :strterm, :scanner
 
     def initialize(source, file)
       @lex_state  = :expr_beg
@@ -75,6 +75,18 @@ module Opal
       @scanner.check(/\s/)
     end
 
+    def scan(regexp)
+      @scanner.scan regexp
+    end
+
+    def check(regexp)
+      @scanner.check regexp
+    end
+
+    def matched
+      @scanner.matched
+    end
+
     def next_token
       self.yylex
     end
@@ -85,6 +97,34 @@ module Opal
       [:dquote, :dsym, :dword, :heredoc, :xquote, :regexp].include? type
     end
 
+    def new_strterm(type, start, finish)
+      { :type => type, :beg => start, :end => finish }
+    end
+
+    def new_strterm2(type, start, finish)
+      term = new_strterm(type, start, finish)
+      term.merge({ :balance => true, :nesting => 0 })
+    end
+
+    def process_numeric
+      @lex_state = :expr_end
+      scanner = @scanner
+
+      if scan(/0b?(0|1|_)+/)
+        return [:tINTEGER, scanner.matched.to_i(2)]
+      elsif scan(/0o?([0-7]|_)+/)
+        return [:tINTEGER, scanner.matched.to_i(8)]
+      elsif scan(/[\d_]+\.[\d_]+\b|[\d_]+(\.[\d_]+)?[eE][-+]?[\d_]+\b/)
+        return [:tFLOAT, scanner.matched.gsub(/_/, '').to_f]
+      elsif scan(/[\d_]+\b/)
+        return [:tINTEGER, scanner.matched.gsub(/_/, '').to_i]
+      elsif scan(/0(x|X)(\d|[a-f]|[A-F]|_)+/)
+        return [:tINTEGER, scanner.matched.to_i(16)]
+      else
+        raise "Lexing error on numeric type: `#{scanner.peek 5}`"
+      end
+    end
+
     def next_string_token
       str_parse = self.strterm
       scanner = @scanner
@@ -94,7 +134,7 @@ module Opal
 
       words = ['w', 'W'].include? str_parse[:beg]
 
-      space = true if ['w', 'W'].include?(str_parse[:beg]) and scanner.scan(/\s+/)
+      space = true if ['w', 'W'].include?(str_parse[:beg]) and scan(/\s+/)
 
       # if not end of string, so we must be parsing contents
       str_buffer = []
@@ -102,8 +142,8 @@ module Opal
       if str_parse[:type] == :heredoc
         eos_regx = /[ \t]*#{Regexp.escape(str_parse[:end])}(\r*\n|$)/
 
-        if scanner.check(eos_regx)
-          scanner.scan(/[ \t]*#{Regexp.escape(str_parse[:end])}/)
+        if check(eos_regx)
+          scan(/[ \t]*#{Regexp.escape(str_parse[:end])}/)
           self.strterm = nil
 
           if str_parse[:scanner]
@@ -112,17 +152,17 @@ module Opal
           end
 
           @lex_state = :expr_end
-          return :STRING_END, scanner.matched
+          return :tSTRING_END, scanner.matched
         end
       end
 
-      # see if we can read end of string/xstring/regecp markers
-      # if scanner.scan /#{str_parse[:end]}/
-      if scanner.scan Regexp.new(Regexp.escape(str_parse[:end]))
+      # see if we can read end of string/xstring/regexp markers
+      # if scan /#{str_parse[:end]}/
+      if scan Regexp.new(Regexp.escape(str_parse[:end]))
         if words && !str_parse[:done_last_space]#&& space
           str_parse[:done_last_space] = true
           scanner.pos -= 1
-          return :SPACE, ' '
+          return :tSPACE, ' '
         end
         self.strterm = nil
 
@@ -130,11 +170,11 @@ module Opal
           if str_parse[:nesting] == 0
             @lex_state = :expr_end
 
-            if str_parse[:regexp]
-              result = scanner.scan(/\w+/)
-              return :REGEXP_END, result
+            if str_parse[:type] == :regexp
+              result = scan(/\w+/)
+              return :tREGEXP_END, result
             end
-            return :STRING_END, scanner.matched
+            return :tSTRING_END, scanner.matched
           else
             str_buffer << scanner.matched
             str_parse[:nesting] -= 1
@@ -143,16 +183,16 @@ module Opal
 
         elsif ['"', "'"].include? str_parse[:beg]
           @lex_state = :expr_end
-          return :STRING_END, scanner.matched
+          return :tSTRING_END, scanner.matched
 
         elsif str_parse[:beg] == '`'
           @lex_state = :expr_end
-          return :STRING_END, scanner.matched
+          return :tSTRING_END, scanner.matched
 
-        elsif str_parse[:beg] == '/' || str_parse[:regexp]
-          result = scanner.scan(/\w+/)
+        elsif str_parse[:beg] == '/' || str_parse[:type] == :regexp
+          result = scan(/\w+/)
           @lex_state = :expr_end
-          return :REGEXP_END, result
+          return :tREGEXP_END, result
 
         else
           if str_parse[:scanner]
@@ -161,33 +201,33 @@ module Opal
           end
 
           @lex_state = :expr_end
-          return :STRING_END, scanner.matched
+          return :tSTRING_END, scanner.matched
         end
       end
 
-      return :SPACE, ' ' if space
+      return :tSPACE, ' ' if space
 
-      if str_parse[:balance] and scanner.scan Regexp.new(Regexp.escape(str_parse[:beg]))
+      if str_parse[:balance] and scan Regexp.new(Regexp.escape(str_parse[:beg]))
         str_buffer << scanner.matched
         str_parse[:nesting] += 1
-      elsif scanner.check(/#[@$]/)
-        scanner.scan(/#/)
+      elsif check(/#[@$]/)
+        scan(/#/)
         if expand
-          return :STRING_DVAR, scanner.matched
+          return :tSTRING_DVAR, scanner.matched
         else
           str_buffer << scanner.matched
         end
 
-      elsif scanner.scan(/#\{/)
+      elsif scan(/#\{/)
         if expand
           # we are into ruby code, so stop parsing content (for now)
-          return :STRING_DBEG, scanner.matched
+          return :tSTRING_DBEG, scanner.matched
         else
           str_buffer << scanner.matched
         end
 
       # causes error, so we will just collect it later on with other text
-      elsif scanner.scan(/\#/)
+      elsif scan(/\#/)
         str_buffer << '#'
       end
 
@@ -199,7 +239,7 @@ module Opal
 
       complete_str = str_buffer.join ''
       @line += complete_str.count("\n")
-      return :STRING_CONTENT, complete_str
+      return :tSTRING_CONTENT, complete_str
     end
 
     def add_heredoc_content(str_buffer, str_parse)
@@ -212,29 +252,29 @@ module Opal
         c = nil
         handled = true
 
-        if scanner.scan(/\n/)
+        if scan(/\n/)
           c = scanner.matched
-        elsif scanner.check(eos_regx) && scanner.bol?
+        elsif check(eos_regx) && scanner.bol?
           break # eos!
-        elsif expand && scanner.check(/#(?=[\$\@\{])/)
+        elsif expand && check(/#(?=[\$\@\{])/)
           break
-        elsif scanner.scan(/\\/)
-          if str_parse[:regexp]
-            if scanner.scan(/(.)/)
+        elsif scan(/\\/)
+          if str_parse[:type] == :regexp
+            if scan(/(.)/)
               c = "\\" + scanner.matched
             end
           else
-            c = if scanner.scan(/n/)
+            c = if scan(/n/)
               "\n"
-            elsif scanner.scan(/r/)
+            elsif scan(/r/)
               "\r"
-            elsif scanner.scan(/\n/)
+            elsif scan(/\n/)
               "\n"
-            elsif scanner.scan(/t/)
+            elsif scan(/t/)
               "\t"
             else
               # escaped char doesnt need escaping, so just return it
-              scanner.scan(/./)
+              scan(/./)
               scanner.matched
             end
           end
@@ -245,7 +285,7 @@ module Opal
         unless handled
           reg = Regexp.new("[^#{Regexp.escape str_parse[:end]}\#\0\\\\\n]+|.")
 
-          scanner.scan reg
+          scan reg
           c = scanner.matched
         end
 
@@ -270,12 +310,12 @@ module Opal
         c = nil
         handled = true
 
-        if scanner.check end_str_re
+        if check end_str_re
           # eos
           # if its just balancing, add it ass normal content..
           if str_parse[:balance] && (str_parse[:nesting] != 0)
             # we only checked above, so actually scan it
-            scanner.scan end_str_re
+            scan end_str_re
             c = scanner.matched
             str_parse[:nesting] -= 1
           else
@@ -283,37 +323,37 @@ module Opal
             break
           end
 
-        elsif str_parse[:balance] and scanner.scan Regexp.new(Regexp.escape(str_parse[:beg]))
+        elsif str_parse[:balance] and scan Regexp.new(Regexp.escape(str_parse[:beg]))
           str_parse[:nesting] += 1
           c = scanner.matched
 
-        elsif words && scanner.scan(/\s/)
+        elsif words && scan(/\s/)
           scanner.pos -= 1
           break
 
-        elsif expand && scanner.check(/#(?=[\$\@\{])/)
+        elsif expand && check(/#(?=[\$\@\{])/)
           break
 
-        #elsif scanner.scan(/\\\\/)
+        #elsif scan(/\\\\/)
           #c = scanner.matched
 
-        elsif scanner.scan(/\\/)
-          if str_parse[:regexp]
-            if scanner.scan(/(.)/)
+        elsif scan(/\\/)
+          if str_parse[:type] == :regexp
+            if scan(/(.)/)
               c = "\\" + scanner.matched
             end
           else
-            c = if scanner.scan(/n/)
+            c = if scan(/n/)
               "\n"
-            elsif scanner.scan(/r/)
+            elsif scan(/r/)
               "\r"
-            elsif scanner.scan(/\n/)
+            elsif scan(/\n/)
               "\n"
-            elsif scanner.scan(/t/)
+            elsif scan(/t/)
               "\t"
             else
               # escaped char doesnt need escaping, so just return it
-              scanner.scan(/./)
+              scan(/./)
               scanner.matched
             end 
           end
@@ -330,7 +370,7 @@ module Opal
                   Regexp.new("[^#{Regexp.escape str_parse[:end]}\#\0\\\\]+|.")
                 end
 
-          scanner.scan reg
+          scan reg
           c = scanner.matched
         end
 
@@ -344,14 +384,14 @@ module Opal
     def heredoc_identifier
       if @scanner.scan(/(-?)['"]?(\w+)['"]?/)
         heredoc = @scanner[2]
-        self.strterm = { :type => :heredoc, :beg => heredoc, :end => heredoc }
+        self.strterm = new_strterm(:heredoc, heredoc, heredoc)
 
         # if ruby code at end of line after heredoc, we have to store it to
         # parse after heredoc is finished parsing
         end_of_line = @scanner.scan(/.*\n/)
         self.strterm[:scanner] = StringScanner.new(end_of_line) if end_of_line != "\n"
 
-        return :STRING_BEG, heredoc
+        return :tSTRING_BEG, heredoc
       end
     end
 
@@ -359,34 +399,34 @@ module Opal
       scanner = @scanner
       matched = scanner.matched
 
-      if scanner.peek(2) != '::' && scanner.scan(/:/)
+      if scanner.peek(2) != '::' && scan(/:/)
         @lex_state = :expr_beg
-        return :LABEL, "#{matched}"
+        return :tLABEL, "#{matched}"
       end
 
       if matched == 'defined?'
         if after_operator?
           @lex_state = :expr_end
-          return :IDENTIFIER, matched
+          return :tIDENTIFIER, matched
         end
 
         @lex_state = :expr_arg
-        return :DEFINED, 'defined?'
+        return :kDEFINED, 'defined?'
       end
 
       if matched.end_with? '?', '!'
-        result = :IDENTIFIER
+        result = :tIDENTIFIER
       else
         if @lex_state == :expr_fname
-          if scanner.scan(/\=/)
-            result = :IDENTIFIER
+          if scan(/\=/)
+            result = :tIDENTIFIER
             matched += scanner.matched
           end
 
         elsif matched =~ /^[A-Z]/
-          result = :CONSTANT
+          result = :tCONSTANT
         else
-          result = :IDENTIFIER
+          result = :tIDENTIFIER
         end
       end
 
@@ -405,24 +445,24 @@ module Opal
         if matched == "do"
           if after_operator?
             @lex_state = :expr_end
-            return :IDENTIFIER, matched
+            return :tIDENTIFIER, matched
           end
 
           if @start_of_lambda
             @start_of_lambda = false
             @lex_state = :expr_beg
-            return [:DO_LAMBDA, scanner.matched]
+            return [:kDO_LAMBDA, scanner.matched]
           elsif cond?
             @lex_state = :expr_beg
-            return :DO_COND, matched
+            return :kDO_COND, matched
           elsif cmdarg? && @lex_state != :expr_cmdarg
             @lex_state = :expr_beg
-            return :DO_BLOCK, matched
+            return :kDO_BLOCK, matched
           elsif @lex_state == :expr_endarg
-            return :DO_BLOCK, matched
+            return :kDO_BLOCK, matched
           else
             @lex_state = :expr_beg
-            return :DO, matched
+            return :kDO, matched
           end
         else
           if old_state == :expr_beg or old_state == :expr_value
@@ -443,11 +483,10 @@ module Opal
         @lex_state = :expr_end
       end
 
-      return [matched =~ /^[A-Z]/ ? :CONSTANT : :IDENTIFIER, matched]
+      return [matched =~ /^[A-Z]/ ? :tCONSTANT : :tIDENTIFIER, matched]
     end
 
     def yylex
-      scanner = @scanner
       @space_seen = false
       cmd_start = false
       c = ''
@@ -457,39 +496,39 @@ module Opal
       end
 
       while true
-        if scanner.scan(/\ |\t|\r/)
+        if scan(/\ |\t|\r/)
           @space_seen = true
           next
 
-        elsif scanner.scan(/(\n|#)/)
+        elsif scan(/(\n|#)/)
           c = scanner.matched
-          if c == '#' then scanner.scan(/(.*)/) else @line += 1; end
+          if c == '#' then scan(/(.*)/) else @line += 1; end
 
-          scanner.scan(/(\n+)/)
+          scan(/(\n+)/)
           @line += scanner.matched.length if scanner.matched
 
           next if [:expr_beg, :expr_dot].include? @lex_state
 
-          if scanner.scan(/([\ \t\r\f\v]*)\./)
+          if scan(/([\ \t\r\f\v]*)\./)
             @space_seen = true unless scanner[1].empty?
             scanner.pos = scanner.pos - 1
 
-            next unless scanner.check(/\.\./)
+            next unless check(/\.\./)
           end
 
           cmd_start = true
           @lex_state = :expr_beg
-          return '\\n', '\\n'
+          return :tNL, '\\n'
 
-        elsif scanner.scan(/\;/)
+        elsif scan(/\;/)
           @lex_state = :expr_beg
-          return ';', ';'
+          return :tSEMI, ';'
 
-        elsif scanner.scan(/\*/)
-          if scanner.scan(/\*/)
-            if scanner.scan(/\=/)
+        elsif scan(/\*/)
+          if scan(/\*/)
+            if scan(/\=/)
               @lex_state = :expr_beg
-              return :OP_ASGN, '**'
+              return :tOP_ASGN, '**'
             end
 
             if @lex_state == :expr_fname or @lex_state == :expr_dot
@@ -498,74 +537,74 @@ module Opal
               @lex_state = :expr_beg
             end
 
-            return '**', '**'
+            return :tPOW, '**'
 
           else
-            if scanner.scan(/\=/)
+            if scan(/\=/)
               @lex_state = :expr_beg
-              return :OP_ASGN, '*'
+              return :tOP_ASGN, '*'
             end
           end
 
-          if scanner.scan(/\*\=/)
+          if scan(/\*\=/)
             @lex_state = :expr_beg
-            return :OP_ASGN, '**'
+            return :tOP_ASGN, '**'
           end
 
-          if scanner.scan(/\*/)
+          if scan(/\*/)
             if after_operator?
               @lex_state = :expr_arg
             else
               @lex_state = :expr_beg
             end
 
-            return '**', '**'
+            return :tPOW, '**'
           end
 
-          if scanner.scan(/\=/)
+          if scan(/\=/)
             @lex_state = :expr_beg
-            return :OP_ASGN, '*'
+            return :tOP_ASGN, '*'
           else
             result = '*'
             if @lex_state == :expr_fname or @lex_state == :expr_dot
               @lex_state = :expr_arg
-              return '*', result
-            elsif @space_seen && scanner.check(/\S/)
+              return :tSTAR2, result
+            elsif @space_seen && check(/\S/)
               @lex_state = :expr_beg
-              return :SPLAT, result
+              return :tSTAR, result
             elsif [:expr_beg, :expr_mid].include? @lex_state
               @lex_state = :expr_beg
-              return :SPLAT, result
+              return :tSTAR, result
             else
               @lex_state = :expr_beg
-              return '*', result
+              return :tSTAR2, result
             end
           end
 
-        elsif scanner.scan(/\!/)
-          c = scanner.scan(/./)
+        elsif scan(/\!/)
+          c = scan(/./)
           if after_operator?
             @lex_state = :expr_arg
             if c == "@"
-              return '!', '!'
+              return :tBANG, '!'
             end
           else
             @lex_state = :expr_beg
           end
 
           if c == '='
-            return '!=', '!='
+            return :tNEQ, '!='
           elsif c == '~'
-            return '!~', '!~'
+            return :tNMATCH, '!~'
           end
 
           scanner.pos = scanner.pos - 1
-          return '!', '!'
+          return :tBANG, '!'
 
-        elsif scanner.scan(/\=/)
+        elsif scan(/\=/)
           if @lex_state == :expr_beg and !@space_seen
-            if scanner.scan(/begin/) and space?
-              scanner.scan(/(.*)/) # end of line
+            if scan(/begin/) and space?
+              scan(/(.*)/) # end of line
               line_count = 0
 
               while true
@@ -573,17 +612,17 @@ module Opal
                   raise "embedded document meets end of file"
                 end
 
-                if scanner.scan(/\=end/) and space?
+                if scan(/\=end/) and space?
                   @line += line_count
                   return next_token
                 end
 
-                if scanner.scan(/\n/)
+                if scan(/\n/)
                   line_count += 1
                   next
                 end
 
-                scanner.scan(/(.*)/)
+                scan(/(.*)/)
               end
             end
           end
@@ -594,164 +633,165 @@ module Opal
                          :expr_beg
                        end
 
-          if scanner.scan(/\=/)
-            if scanner.scan(/\=/)
-              return '===', '==='
+          if scan(/\=/)
+            if scan(/\=/)
+              return :tEQQ, '==='
             end
 
-            return '==', '=='
+            return :tEQ, '=='
           end
 
-          if scanner.scan(/\~/)
-            return '=~', '=~'
-          elsif scanner.scan(/\>/)
-            return '=>', '=>'
+          if scan(/\~/)
+            return :tMATCH, '=~'
+          elsif scan(/\>/)
+            return :tASSOC, '=>'
           end
 
-          return '=', '='
+          return :tEQL, '='
 
-        elsif scanner.scan(/\"/)
-          self.strterm = { :type => :dquote, :beg => '"', :end => '"' }
-          return :STRING_BEG, scanner.matched
+        elsif scan(/\"/)
+          self.strterm = new_strterm(:dquote, '"', '"')
+          return :tSTRING_BEG, scanner.matched
 
-        elsif scanner.scan(/\'/)
-          self.strterm = { :type => :squote, :beg => "'", :end => "'" }
-          return :STRING_BEG, scanner.matched
+        elsif scan(/\'/)
+          self.strterm = new_strterm(:squote, "'", "'")
+          return :tSTRING_BEG, scanner.matched
 
-        elsif scanner.scan(/\`/)
-          self.strterm = { :type => :xquote, :beg => "`", :end => "`" }
-          return :XSTRING_BEG, scanner.matched
+        elsif scan(/\`/)
+          self.strterm = new_strterm(:xquote, '`', '`')
+          return :tXSTRING_BEG, scanner.matched
 
-        elsif scanner.scan(/\&/)
-          if scanner.scan(/\&/)
+        elsif scan(/\&/)
+          if scan(/\&/)
             @lex_state = :expr_beg
 
-            if scanner.scan(/\=/)
-              return :OP_ASGN, '&&'
+            if scan(/\=/)
+              return :tOP_ASGN, '&&'
             end
 
-            return '&&', '&&'
+            return :tANDOP, '&&'
 
-          elsif scanner.scan(/\=/)
+          elsif scan(/\=/)
             @lex_state = :expr_beg
-            return :OP_ASGN, '&'
+            return :tOP_ASGN, '&'
           end
 
           if spcarg?
             #puts "warning: `&' interpreted as argument prefix"
-            result = '&@'
+            result = :tAMPER
           elsif beg?
-            result = '&@'
+            result = :tAMPER
           else
             #puts "warn_balanced: & argument prefix"
-            result = '&'
+            result = :tAMPER2
           end
 
           @lex_state = after_operator? ? :expr_arg : :expr_beg
           return result, '&'
 
-        elsif scanner.scan(/\|/)
-          if scanner.scan(/\|/)
+        elsif scan(/\|/)
+          if scan(/\|/)
             @lex_state = :expr_beg
-            if scanner.scan(/\=/)
-              return :OP_ASGN, '||'
+            if scan(/\=/)
+              return :tOP_ASGN, '||'
             end
 
-            return '||', '||'
+            return :tOROP, '||'
 
-          elsif scanner.scan(/\=/)
-            return :OP_ASGN, '|'
+          elsif scan(/\=/)
+            return :tOP_ASGN, '|'
           end
 
           @lex_state = after_operator?() ? :expr_arg : :expr_beg
-          return '|', '|'
-
-        elsif scanner.scan(/\%W/)
-          start_word  = scanner.scan(/./)
-          end_word    = { '(' => ')', '[' => ']', '{' => '}' }[start_word] || start_word
-          self.strterm = { :type => :dword, :beg => 'W', :end => end_word  }
-          scanner.scan(/\s*/)
-          return :WORDS_BEG, scanner.matched
-
-        elsif scanner.scan(/\%w/) or scanner.scan(/\%i/)
-          start_word  = scanner.scan(/./)
-          end_word    = { '(' => ')', '[' => ']', '{' => '}' }[start_word] || start_word
-          self.strterm = { :type => :sword, :beg => 'w', :end => end_word }
-          scanner.scan(/\s*/)
-          return :AWORDS_BEG, scanner.matched
-
-        elsif scanner.scan(/\%[Qq]/)
-          type = scanner.matched.end_with?('Q') ? :dquote : :squote
-          start_word  = scanner.scan(/./)
-          end_word    = { '(' => ')', '[' => ']', '{' => '}' }[start_word] || start_word
-          self.strterm = { :type => type, :beg => start_word, :end => end_word, :balance => true, :nesting => 0 }
-          return :STRING_BEG, scanner.matched
-
-        elsif scanner.scan(/\%x/)
-          start_word = scanner.scan(/./)
-          end_word   = { '(' => ')', '[' => ']', '{' => '}' }[start_word] || start_word
-          self.strterm = { :type => :xquote, :beg => start_word, :end => end_word, :balance => true, :nesting => 0 }
-          return :XSTRING_BEG, scanner.matched
-
-        elsif scanner.scan(/\%r/)
-          start_word = scanner.scan(/./)
-          end_word   = { '(' => ')', '[' => ']', '{' => '}' }[start_word] || start_word
-          self.strterm = { :type => :regexp, :beg => start_word, :end => end_word, :regexp => true, :balance => true, :nesting => 0 }
-          return :REGEXP_BEG, scanner.matched
-
-        elsif scanner.scan(/\//)
+          return :tPIPE, '|'
+
+        elsif scan(/\%[QqWwxr]/)
+          str_type = scanner.matched[1, 1]
+          paren = scan(/./)
+
+          term = case paren
+                 when '(' then ')'
+                 when '[' then ']'
+                 when '{' then '}'
+                 else paren
+                 end
+
+          case str_type
+          when 'Q'
+            self.strterm = new_strterm2(:dquote, paren, term)
+            return :tSTRING_BEG, scanner.matched
+          when 'q'
+            self.strterm = new_strterm2(:squote, paren, term)
+            return :tSTRING_BEG, scanner.matched
+          when 'W'
+            self.strterm = new_strterm(:dword, 'W', term)
+            scan(/\s*/)
+            return :tWORDS_BEG, scanner.matched
+          when 'w', 'i'
+            self.strterm = new_strterm(:sword, 'w', term)
+            scan(/\s*/)
+            return :tAWORDS_BEG, scanner.matched
+          when 'x'
+            self.strterm = new_strterm2(:xquote, paren, term)
+            return :tXSTRING_BEG, scanner.matched
+          when 'r'
+            self.strterm = new_strterm2(:regexp, paren, term)
+            return :tREGEXP_BEG, scanner.matched
+          end
+
+        elsif scan(/\//)
           if [:expr_beg, :expr_mid].include? @lex_state
-            self.strterm = { :type => :regexp, :beg => '/', :end => '/', :regexp => true }
-            return :REGEXP_BEG, scanner.matched
-          elsif scanner.scan(/\=/)
+            self.strterm = new_strterm(:regexp, '/', '/')
+            return :tREGEXP_BEG, scanner.matched
+          elsif scan(/\=/)
             @lex_state = :expr_beg
-            return :OP_ASGN, '/'
+            return :tOP_ASGN, '/'
           elsif @lex_state == :expr_fname or @lex_state == :expr_dot
             @lex_state = :expr_arg
           elsif @lex_state == :expr_cmdarg || @lex_state == :expr_arg
-            if !scanner.check(/\s/) && @space_seen
-              self.strterm = { :type => :regexp, :beg => '/', :end => '/', :regexp => true }
-              return :REGEXP_BEG, scanner.matched
+            if !check(/\s/) && @space_seen
+              self.strterm = new_strterm(:regexp, '/', '/')
+              return :tREGEXP_BEG, scanner.matched
             end
           else
             @lex_state = :expr_beg
           end
 
-          return '/', '/'
+          return :tDIVIDE, '/'
 
-        elsif scanner.scan(/\%/)
-          if scanner.scan(/\=/)
+        elsif scan(/\%/)
+          if scan(/\=/)
             @lex_state = :expr_beg
-            return :OP_ASGN, '%'
-          elsif scanner.check(/[^\s]/)
+            return :tOP_ASGN, '%'
+          elsif check(/[^\s]/)
             if @lex_state == :expr_beg or (@lex_state == :expr_arg && @space_seen)
-              start_word  = scanner.scan(/./)
+              start_word  = scan(/./)
               end_word    = { '(' => ')', '[' => ']', '{' => '}' }[start_word] || start_word
-              self.strterm = { :type => :dquote, :beg => start_word, :end => end_word, :balance => true, :nesting => 0 }
-              return :STRING_BEG, scanner.matched
+              self.strterm = new_strterm2(:dquote, start_word, end_word)
+              return :tSTRING_BEG, scanner.matched
             end
           end
 
           @lex_state = after_operator? ? :expr_arg : :expr_beg
 
-          return '%', '%'
+          return :tPERCENT, '%'
 
-        elsif scanner.scan(/\\/)
-          if scanner.scan(/\r?\n/)
+        elsif scan(/\\/)
+          if scan(/\r?\n/)
             @space_seen = true
             next
           end
 
           raise SyntaxError, "backslash must appear before newline :#{@file}:#{@line}"
 
-        elsif scanner.scan(/\(/)
+        elsif scan(/\(/)
           result = scanner.matched
           if [:expr_beg, :expr_mid].include? @lex_state
-            result = :PAREN_BEG
+            result = :tLPAREN
           elsif @space_seen && [:expr_arg, :expr_cmdarg].include?(@lex_state)
             result = :tLPAREN_ARG
           else
-            result = '('
+            result = :tLPAREN2
           end
 
           @lex_state = :expr_beg
@@ -760,21 +800,21 @@ module Opal
 
           return result, scanner.matched
 
-        elsif scanner.scan(/\)/)
+        elsif scan(/\)/)
           cond_lexpop
           cmdarg_lexpop
           @lex_state = :expr_end
-          return ')', scanner.matched
+          return :tRPAREN, scanner.matched
 
-        elsif scanner.scan(/\[/)
+        elsif scan(/\[/)
           result = scanner.matched
 
           if [:expr_fname, :expr_dot].include? @lex_state
             @lex_state = :expr_arg
-            if scanner.scan(/\]=/)
-              return '[]=', '[]='
-            elsif scanner.scan(/\]/)
-              return '[]', '[]'
+            if scan(/\]=/)
+              return :tASET, '[]='
+            elsif scan(/\]/)
+              return :tAREF, '[]'
             else
               raise "Unexpected '[' token"
             end
@@ -782,102 +822,102 @@ module Opal
             @lex_state = :expr_beg
             cond_push 0
             cmdarg_push 0
-            return '[', scanner.matched
+            return :tLBRACK, scanner.matched
           else
             @lex_state = :expr_beg
             cond_push 0
             cmdarg_push 0
-            return '[@', scanner.matched
+            return :tLBRACK2, scanner.matched
           end
 
-        elsif scanner.scan(/\]/)
+        elsif scan(/\]/)
           cond_lexpop
           cmdarg_lexpop
           @lex_state = :expr_end
-          return ']', scanner.matched
+          return :tRBRACK, scanner.matched
 
-        elsif scanner.scan(/\}/)
+        elsif scan(/\}/)
           cond_lexpop
           cmdarg_lexpop
           @lex_state = :expr_end
 
-          return '}', scanner.matched
+          return :tRCURLY, scanner.matched
 
-        elsif scanner.scan(/\.\.\./)
+        elsif scan(/\.\.\./)
           @lex_state = :expr_beg
-          return '...', scanner.matched
+          return :tDOT3, scanner.matched
 
-        elsif scanner.scan(/\.\./)
+        elsif scan(/\.\./)
           @lex_state = :expr_beg
-          return '..', scanner.matched
+          return :tDOT2, scanner.matched
 
-        elsif scanner.scan(/\./)
+        elsif scan(/\./)
           @lex_state = :expr_dot unless @lex_state == :expr_fname
-          return '.', scanner.matched
+          return :tDOT, scanner.matched
 
-        elsif scanner.scan(/\:\:/)
+        elsif scan(/\:\:/)
           if [:expr_beg, :expr_mid, :expr_class].include? @lex_state
             @lex_state = :expr_beg
-            return '::@', scanner.matched
+            return :tCOLON3, scanner.matched
           elsif @space_seen && @lex_state == :expr_arg
             @lex_state = :expr_beg
-            return '::@', scanner.matched
+            return :tCOLON3, scanner.matched
           end
 
           @lex_state = :expr_dot
-          return '::', scanner.matched
+          return :tCOLON2, scanner.matched
 
-        elsif scanner.scan(/\:/)
-          if end? || scanner.check(/\s/)
-            unless scanner.check(/\w/)
+        elsif scan(/\:/)
+          if end? || check(/\s/)
+            unless check(/\w/)
               @lex_state = :expr_beg
-              return ':', ':'
+              return :tCOLON, ':'
             end
 
             @lex_state = :expr_fname
-            return :SYMBOL_BEG, ':'
+            return :tSYMBEG, ':'
           end
 
-          if scanner.scan(/\'/)
-            self.strterm = { :type => :ssym, :beg => "'", :end => "'" }
-          elsif scanner.scan(/\"/)
-            self.strterm = { :type => :dsym, :beg => '"', :end => '"' }
+          if scan(/\'/)
+            self.strterm = new_strterm(:ssym, "'", "'")
+          elsif scan(/\"/)
+            self.strterm = new_strterm(:dsym, '"', '"')
           end
 
           @lex_state = :expr_fname
-          return :SYMBOL_BEG, ':'
+          return :tSYMBEG, ':'
 
-        elsif scanner.scan(/\^\=/)
+        elsif scan(/\^\=/)
           @lex_state = :expr_beg
-          return :OP_ASGN, '^'
-        elsif scanner.scan(/\^/)
+          return :tOP_ASGN, '^'
+        elsif scan(/\^/)
           if @lex_state == :expr_fname or @lex_state == :expr_dot
             @lex_state = :expr_arg
-            return '^', scanner.matched
+            return :tCARET, scanner.matched
           end
 
           @lex_state = :expr_beg
-          return '^', scanner.matched
+          return :tCARET, scanner.matched
 
-        elsif scanner.check(/\</)
-          if scanner.scan(/\<\<\=/)
+        elsif check(/\</)
+          if scan(/\<\<\=/)
             @lex_state = :expr_beg
-            return :OP_ASGN, '<<'
-          elsif scanner.scan(/\<\</)
+            return :tOP_ASGN, '<<'
+          elsif scan(/\<\</)
             if @lex_state == :expr_fname or @lex_state == :expr_dot
               @lex_state = :expr_arg
-              return '<<', '<<'
+              return :tLSHFT, '<<'
             elsif ![:expr_dot, :expr_class].include?(@lex_state) && !end? && (!arg? || @space_seen)
               if token = heredoc_identifier
                 return token
               end
 
               @lex_state = :expr_beg
-              return '<<', '<<'
+              return :tLSHFT, '<<'
             end
             @lex_state = :expr_beg
-            return '<<', '<<'
-          elsif scanner.scan(/\<\=\>/)
+            return :tLSHFT, '<<'
+          elsif scan(/\<\=\>/)
             if after_operator?
               @lex_state = :expr_arg
             else
@@ -888,148 +928,152 @@ module Opal
               @lex_state = :expr_beg
             end
 
-            return '<=>', '<=>'
-          elsif scanner.scan(/\<\=/)
+            return :tCMP, '<=>'
+          elsif scan(/\<\=/)
             if @lex_state == :expr_fname or @lex_state == :expr_dot
               @lex_state = :expr_arg
             else
               @lex_state = :expr_beg
             end
-            return '<=', '<='
-          elsif scanner.scan(/\</)
+            return :tLEQ, '<='
+          elsif scan(/\</)
             if @lex_state == :expr_fname or @lex_state == :expr_dot
               @lex_state = :expr_arg
             else
               @lex_state = :expr_beg
             end
-            return '<', '<'
+            return :tLT, '<'
           end
 
-        elsif scanner.check(/\>/)
-          if scanner.scan(/\>\>\=/)
-            return :OP_ASGN, '>>'
-          elsif scanner.scan(/\>\>/)
+        elsif check(/\>/)
+          if scan(/\>\>\=/)
+            return :tOP_ASGN, '>>'
+          elsif scan(/\>\>/)
             if @lex_state == :expr_fname or @lex_state == :expr_dot
               @lex_state = :expr_arg
             else
               @lex_state = :expr_beg
             end
-            return '>>', '>>'
-          elsif scanner.scan(/\>\=/)
+            return :tRSHFT, '>>'
+          elsif scan(/\>\=/)
             if @lex_state == :expr_fname or @lex_state == :expr_dot
               @lex_state = :expr_end
             else
               @lex_state = :expr_beg
             end
-            return '>=', scanner.matched
-          elsif scanner.scan(/\>/)
+            return :tGEQ, scanner.matched
+          elsif scan(/\>/)
             if @lex_state == :expr_fname or @lex_state == :expr_dot
               @lex_state = :expr_arg
             else
               @lex_state = :expr_beg
             end
-            return '>', '>'
+            return :tGT, '>'
           end
 
-        elsif scanner.scan(/->/)
+        elsif scan(/->/)
           # FIXME: # should be :expr_arg, but '(' breaks it...
           @lex_state = :expr_end
           @start_of_lambda = true
-          return [:LAMBDA, scanner.matched]
+          return [:tLAMBDA, scanner.matched]
 
-        elsif scanner.scan(/[+-]/)
-          result  = scanner.matched
-          sign    = result + '@'
+        elsif scan(/[+-]/)
+          matched = scanner.matched
+          sign, utype = if matched == '+'
+                          [:tPLUS, :tUPLUS]
+                        else
+                          [:tMINUS, :tUMINUS]
+                        end
 
-          if @lex_state == :expr_beg || @lex_state == :expr_mid
+          if beg?
             @lex_state = :expr_mid
-            return [sign, sign]
-          elsif @lex_state == :expr_fname or @lex_state == :expr_dot
+            return [utype, matched]
+          elsif after_operator?
             @lex_state = :expr_arg
-            return [:IDENTIFIER, result + scanner.matched] if scanner.scan(/@/)
-            return [result, result]
+            return [:tIDENTIFIER, matched + '@'] if scan(/@/)
+            return [sign, matched]
           end
 
-          if scanner.scan(/\=/)
+          if scan(/\=/)
             @lex_state = :expr_beg
-            return [:OP_ASGN, result]
+            return [:tOP_ASGN, matched]
           end
 
-          if @lex_state == :expr_cmdarg || @lex_state == :expr_arg
-            if !scanner.check(/\s/) && @space_seen
+          if arg?
+            if !space? && @space_seen
               @lex_state = :expr_mid
-              return [sign, sign]
+              return [utype, matched]
             end
           end
 
           @lex_state = :expr_beg
-          return [result, result]
+          return [sign, sign]
 
-        elsif scanner.scan(/\?/)
+        elsif scan(/\?/)
           if end?
             @lex_state = :expr_beg
-            return '?', scanner.matched
+            return :tEH, scanner.matched
           end
 
-          unless scanner.check(/\ |\t|\r|\s/)
+          unless check(/\ |\t|\r|\s/)
             @lex_state = :expr_end
-            return :STRING, scanner.scan(/./)
+            return :tSTRING, scan(/./)
           end
 
           @lex_state = :expr_beg
-          return '?', scanner.matched
+          return :tEH, scanner.matched
 
-        elsif scanner.scan(/\~/)
+        elsif scan(/\~/)
           if @lex_state == :expr_fname
             @lex_state = :expr_end
-            return '~', '~'
+            return :tTILDE, '~'
           end
           @lex_state = :expr_beg
-          return '~', '~'
+          return :tTILDE, '~'
 
-        elsif scanner.check(/\$/)
-          if scanner.scan(/\$([1-9]\d*)/)
+        elsif check(/\$/)
+          if scan(/\$([1-9]\d*)/)
             @lex_state = :expr_end
-            return :NTH_REF, scanner.matched.sub('$', '')
+            return :tNTH_REF, scanner.matched.sub('$', '')
 
-          elsif scanner.scan(/(\$_)(\w+)/)
+          elsif scan(/(\$_)(\w+)/)
             @lex_state = :expr_end
-            return :GVAR, scanner.matched
+            return :tGVAR, scanner.matched
 
-          elsif scanner.scan(/\$[\+\'\`\&!@\"~*$?\/\\:;=.,<>_]/)
+          elsif scan(/\$[\+\'\`\&!@\"~*$?\/\\:;=.,<>_]/)
             @lex_state = :expr_end
-            return :GVAR, scanner.matched
-          elsif scanner.scan(/\$\w+/)
+            return :tGVAR, scanner.matched
+          elsif scan(/\$\w+/)
             @lex_state = :expr_end
-            return :GVAR, scanner.matched
+            return :tGVAR, scanner.matched
           else
             raise "Bad gvar name: #{scanner.peek(5).inspect}"
           end
 
-        elsif scanner.scan(/\$\w+/)
+        elsif scan(/\$\w+/)
           @lex_state = :expr_end
-          return :GVAR, scanner.matched
+          return :tGVAR, scanner.matched
 
-        elsif scanner.scan(/\@\@\w*/)
+        elsif scan(/\@\@\w*/)
           @lex_state = :expr_end
-          return :CVAR, scanner.matched
+          return :tCVAR, scanner.matched
 
-        elsif scanner.scan(/\@\w*/)
+        elsif scan(/\@\w*/)
           @lex_state = :expr_end
-          return :IVAR, scanner.matched
+          return :tIVAR, scanner.matched
 
-        elsif scanner.scan(/\,/)
+        elsif scan(/\,/)
           @lex_state = :expr_beg
-          return ',', scanner.matched
+          return :tCOMMA, scanner.matched
 
-        elsif scanner.scan(/\{/)
+        elsif scan(/\{/)
           if @start_of_lambda
             @start_of_lambda = false
             @lex_state = :expr_beg
-            return [:LAMBEG, scanner.matched]
+            return [:tLAMBEG, scanner.matched]
 
           elsif [:expr_end, :expr_arg, :expr_cmdarg].include? @lex_state
-            result = :LCURLY
+            result = :tLCURLY
           elsif @lex_state == :expr_endarg
             result = :LBRACE_ARG
           else
@@ -1041,23 +1085,10 @@ module Opal
           cmdarg_push 0
           return result, scanner.matched
 
-        elsif scanner.check(/[0-9]/)
-          @lex_state = :expr_end
-          if scanner.scan(/0b?(0|1|_)+/)
-            return [:INTEGER, scanner.matched.to_i(2)]
-          elsif scanner.scan(/0o?([0-7]|_)+/)
-            return [:INTEGER, scanner.matched.to_i(8)]
-          elsif scanner.scan(/[\d_]+\.[\d_]+\b|[\d_]+(\.[\d_]+)?[eE][-+]?[\d_]+\b/)
-            return [:FLOAT, scanner.matched.gsub(/_/, '').to_f]
-          elsif scanner.scan(/[\d_]+\b/)
-            return [:INTEGER, scanner.matched.gsub(/_/, '').to_i]
-          elsif scanner.scan(/0(x|X)(\d|[a-f]|[A-F]|_)+/)
-            return [:INTEGER, scanner.matched.to_i(16)]
-          else
-            raise "Lexing error on numeric type: `#{scanner.peek 5}`"
-          end
+        elsif check(/[0-9]/)
+          return process_numeric
 
-        elsif scanner.scan(/(\w)+[\?\!]?/)
+        elsif scan(/(\w)+[\?\!]?/)
           return process_identifier scanner.matched, cmd_start
         end