opal 0.3.9 → 0.3.10

data/README.md

@@ -20,8 +20,9 @@ browsers - including older versions of IE and mobile devices.
 Using opal
 ----------
 
-Opal can currently be used in two ways: through a distributed ruby gem,
-or directly in the web browser.
+Opal can currently be used in three ways: through a distributed ruby gem,
+directly in the web browser or with rbp - a new package manager designed
+for opal but usable for any ruby project.
 
 ### Using the gem
 
@@ -46,6 +47,28 @@ $ cd opal
 $ bin/opal
 ```
 
+### Using rbp
+
+rbp installs dependencies locally to your project, so edit your
+package.yml file to add the following dependency:
+
+```yaml
+dev_dependencies:
+  opal: "0.3.9"
+  therubyracer: "0.9.4"
+```
+
+Install them with:
+
+```
+$ rbp install
+```
+
+This will install them into vendor/ ready to use. therubyracer is only
+needed if you want to run your ruby code, compiled into javascript,
+directly on the commandline. If you are just compiling ruby then just
+the opal package is sufficient.
+
 Running tests
 -------------
 
@@ -54,14 +77,14 @@ based on minitest. To get opaltest, run the following in the opal
 directory:
 
 ```
-$ rake vendor
+$ rbp install
 ```
 
-This will put opaltest into `vendor/opaltest` so it will be available
+This will put opaltest into `packages/opaltest` so it will be available
 for running. To test `array.rb` for example, run:
 
 ```
-$ bin/opal test/array.rb
+$ rbp exec opal test/array.rb
 ```
 
 The results should be printed to the console.

data/bin/opal

@@ -1,8 +1,5 @@
 #!/usr/bin/env ruby
 
-opal_lib = File.expand_path('../../lib', __FILE__)
-$:.unshift opal_lib unless $:.include? opal_lib
-
 require 'opal'
 require 'opal/command'
 

data/lib/opal.rb

@@ -1,7 +1,12 @@
-require 'rbp'
-
 require 'opal/parser'
 require 'opal/builder'
 require 'opal/context'
-require 'opal/browserify'
+
+# Opal is a set of build tools and runtime utilies for compiling ruby
+# source code into javascript. Opal can use therubyracer to provide a
+# ruby context for evaluating the generated javascript against the
+# provided runtime.
+module Opal
+  OPAL_DIR = File.expand_path('../..', __FILE__)
+end
 

data/lib/opal/builder.rb

@@ -1,4 +1,5 @@
 require 'fileutils'
+require 'opal/parser'
 
 module Opal
 
@@ -15,6 +16,14 @@ module Opal
 
     CORE_PATH = File.join OPAL_PATH, 'corelib'
 
+    def initialize
+      @parser = Parser.new
+    end
+
+    def parse(source, options = {})
+      @parser.parse source, options
+    end
+
     # Returns the result of the compiled file ready for opal to load.
     #
     # `relative_path` is used for the name the built file should have.
@@ -76,7 +85,7 @@ module Opal
         File.read File.join(CORE_PATH, o + '.rb')
       end
 
-      code += 'var core_lib = function($rb, self, __FILE__) { ' + Opal::Parser.new(core.join).parse!.generate_top + '};'
+      code += "var core_lib = #{parse core.join};"
 
       code + File.read(File.join RUNTIME_PATH, 'post.js')
     end

data/lib/opal/bundle.rb

@@ -0,0 +1,48 @@
+require 'opal/builder'
+
+begin
+  require 'rbp/package'
+rescue LoadError
+  abort "You need to install rbp. `gem install rbp`."
+end
+
+module Opal
+  # Takes a package and builds it ready for the browser
+  class Bundle
+    # @return [Rbp::Package] the package this is bundling
+    attr_reader :package
+
+    attr_accessor :options
+
+    def initialize(package)
+      @package        = package
+      @builder        = Builder.new
+      @options = {}
+    end
+
+    # Simple build - returns a string which can be written to a file
+    # FIXME: hardcoded lib directory to './lib'
+    def build
+      package_dir = @package.package_dir
+
+      lib_files = @package.relative_lib_files.map do |lib|
+        path = File.join package_dir, lib
+        code = @builder.parse File.read(path), options
+
+        "\"#{lib}\": #{code}"
+      end
+
+      bundle = []
+      bundle << %[opal.package({\n]
+      bundle << %[  name: "#{@package.name}",\n]
+      bundle << %[  version: "#{@package.version}",\n]
+      bundle << %[  libs: {\n]
+      bundle << %[    #{lib_files.join ",\n    "}\n]
+      bundle << %[  }\n]
+      bundle << %[});\n]
+
+      bundle.join ''
+    end
+  end
+end
+

data/lib/opal/command.rb

@@ -3,7 +3,7 @@ module Opal
   class Command
 
     # Valid command line arguments
-    COMMANDS = [:help, :irb, :compile, :bundle, :exec, :eval]
+    COMMANDS = [:help, :irb, :compile, :bundle, :exec, :eval, :install]
 
     def initialize(args)
       command = args.shift
@@ -40,12 +40,24 @@ module Opal
       puts Opal::Parser.new(File.read(path)).parse!.generate_top
     end
 
-    # desc "bundle", "Bundle the gem in the given directory ready for browser"
-    # method_options :out => :string
+    def install
+      install = RBP::Install.new
+      install.install
+    end
+
+    # Bundle the gem (browserify) ready for the browser
     def bundle
-      opts = options
-      bundle = Opal::Bundle.new(Opal::Gem.new(Dir.getwd))
-      bundle.build opts
+      # lazy load incase user does not have rbp installed
+      require 'opal/bundle'
+
+      path    = File.join Dir.getwd, 'package.yml'
+      package = Rbp::Package.load_path path
+      bundle  = Bundle.new package
+
+      puts bundle
+      puts bundle.package
+
+      puts bundle.build
     end
   end
 end

data/lib/opal/context.rb

@@ -4,64 +4,56 @@ module Opal
     def initialize(root_dir = Dir.getwd)
       @root_dir = root_dir
       @builder = Opal::Builder.new
+      @loaded_paths = false
+
+      # special case: if we are running in opal root, then we dont want
+      # setup.rb to load the opal lib itself, so we do some "magic"
+      if @root_dir == OPAL_DIR
+        def self.setup_load_paths
+          return if @loaded_paths
+          Dir['packages/*/package.yml'].map do |package|
+            path = File.expand_path File.join(File.dirname(package), 'lib')
+            @v8.eval "opal.loader.paths.push('#{path}')"
+          end
+        end
+      end
 
-      @load_paths = resolve_load_paths
+      setup_v8
     end
 
+    ##
     # Looks through vendor/ directory and adds all relevant load paths
-    def resolve_load_paths
-      Dir['vendor/*/package.yml'].map do |package|
-        File.expand_path File.join(File.dirname(package), 'lib')
-      end
-    end
-
-    # Setup the context. This basically loads opal.js into our context, and
-    # replace the loader etc with our custom loader for a Ruby environment. The
-    # default "browser" loader cannot access files from disk.
-    def setup_v8
-      return if @v8
-
-      begin
-        require 'v8'
-      rescue LoadError => e
-        abort "therubyracer is required for running javascript. Install it with `gem install therubyracer`"
-      end
-
-      @v8 = V8::Context.new
-      @v8['console'] = Console.new
-
-      eval @builder.build_core, '(opal)'
-      opal = @v8['opal']
-      opal['fs'] = FileSystem.new self
 
-      # FIXME: we cant use a ruby array as a js array :(
-      opal['loader'] = Loader.new self, eval("[]")
+    def setup_load_paths
+      return if @loaded_paths
 
-      @load_paths.each do |path|
-        eval "opal.loader.paths.push('#{path}')"
-      end
+      setup = File.join @root_dir, 'packages', 'init.rb'
+      return [] unless File.exists? setup
 
+      @v8.eval "opal.run(function() {opal.require('#{setup}');});", setup
     end
 
-    def eval(code, file = nil)
-      @v8.eval code, file
-    end
-
+    ##
     # Require the given id as if it was required in the context. This simply
     # passes the require through to the underlying context.
+
     def require_file(path)
       setup_v8
-      eval "opal.run(function() {opal.require('#{path}');});", path
+      @v8.eval "opal.run(function() {opal.require('#{path}');});", path
       finish
     end
 
+    ##
     # Set ARGV for the context
+
     def argv=(args)
       puts "setting argv to #{args.inspect}"
-      eval "opal.runtime.cs(opal.runtime.Object, 'ARGV', #{args.inspect});"
+      @v8.eval "opal.runtime.cs(opal.runtime.Object, 'ARGV', #{args.inspect});"
     end
 
+    ##
     # Start normal js repl
+
     def start_repl
       require 'readline'
       setup_v8
@@ -76,38 +68,68 @@ module Opal
           break
         end
 
-        puts "=> #{eval_ruby line, '(opal)'}"
+        puts "=> #{eval line, '(opal)'}"
       end
 
       finish
     end
 
-    def eval_ruby(content, line = "")
+    def eval(content, file = nil, line = "")
       begin
-        code = Opal::Parser.new(content).parse!.generate_top
-        code = "opal.run(function() {var $rb = opal.runtime, self = $rb.top, __FILE__ = '(opal)';" + code + "});"
+        js = @builder.parse content
+        code = "opal.run(function() { var $rb = opal.runtime, self = $rb.top"
+        code += ", __FILE__ = '(opal)'; return (#{js})($rb, self, __FILE__); })"
         # puts code
-        @v8['$opal_irb_result'] = eval code, line
-        eval "!($opal_irb_result == null || !$opal_irb_result.m$inspect) ? $opal_irb_result.m$inspect() : '(Object does not support #inspect)'"
+        @v8['$opal_irb_result'] = @v8.eval(code, file)
+        @v8.eval "!($opal_irb_result == null || !$opal_irb_result.m$inspect) ? $opal_irb_result.m$inspect() : '(Object does not support #inspect)'"
       rescue => e
         puts e
         puts("\t" + e.backtrace.join("\n\t"))
       end
     end
 
+    ##
     # Finishes the context, i.e. tidy everything up. This will cause
     # the opal runtime to do it's at_exit() calls (if applicable) and
     # then the v8 context will de removed. It can be reset by calling
     # #setup_v8
+
     def finish
       return unless @v8
-      eval "opal.runtime.do_at_exit()", "(opal)"
+      @v8.eval "opal.runtime.do_at_exit()", "(opal)"
 
       @v8 = nil
     end
 
+    # Setup the context. This basically loads opal.js into our context, and
+    # replace the loader etc with our custom loader for a Ruby environment. The
+    # default "browser" loader cannot access files from disk.
+    def setup_v8
+      return if @v8
+
+      begin
+        require 'v8'
+      rescue LoadError => e
+        abort "therubyracer is required for running javascript. Install it with `gem install therubyracer`"
+      end
+
+      @v8 = V8::Context.new
+      @v8['console'] = Console.new
+
+      @v8.eval @builder.build_core, '(opal)'
+      opal = @v8['opal']
+      opal['fs'] = FileSystem.new self
+
+      # FIXME: we cant use a ruby array as a js array :(
+      opal['loader'] = Loader.new self, @v8.eval("[]")
+
+      setup_load_paths
+    end
+
+    ##
     # Console class is used to mimic the console object in web browsers
     # to allow simple debugging to the stdout.
+
     class Console
       def log(*str)
         puts str.join("\n")
@@ -115,9 +137,11 @@ module Opal
       end
     end
 
+    ##
     # FileSystem is used to interact with the file system from the ruby
     # version of opal. The methods on this class replace the default ones
     # made available in the web browser.
+
     class FileSystem
 
       def initialize(context)
@@ -182,7 +206,7 @@ module Opal
       end
 
       def ruby_file_contents(filename)
-        Opal::Parser.new(File.read(filename)).parse!.generate_top
+        Opal::Parser.new.parse File.read(filename)
       end
 
       def wrap(content, filename)

data/lib/opal/lexer.rb

@@ -1,863 +1,883 @@
-
 require 'opal/parser'
 require 'opal/nodes'
 
-
 require 'strscan'
 
 module Opal
+  # This class is used for parsing ruby content and then generating
+  # javascript content.
+  #
+  # Usage:
+  #
+  #   parser = Opal::Parser.new
+  #
+  #   parser.parse "self.do_something 1, 2, 3"
+  #   # => "some ruby content"
   class Parser < Racc::Parser
 
-  class RubyLexingError < StandardError
-
-  end
-
-  def initialize(source, options = {})
-    @lex_state = :expr_beg
+    # Thrown on parsing error
+    class RubyLexingError < StandardError; end
+
+    # Parse the given ruby code in `source` and returns a string of
+    # compiled javascript. `options` may be passed as a hash, and
+    # include a set of various lexing and generator options.
+    #
+    # All options:
+    #
+    #   `:method_missing` - dictates if the generated javascript has
+    #       support for rubys method_missing. Defaults to `false`. It
+    #       is recomended only to use method_missing in debug mode as
+    #       it adds overhead to **every** method call.
+    #
+    #   `:main_scope` - whether the code is to run in the main scope.
+    #       `true` means that all local variables will be available
+    #       in future bindings. `false` means its just a normal scope,
+    #       i.e. top. This is only needed in [Context] for irb.
+    #
+    # @param [String] source ruby source code to parse
+    # @param [Hash] options parsing options to use
+    def parse(source, options = {})
+      @lex_state          = :expr_beg
+      @cond               = 0
+      @cmdarg             = 0
+      @line_number        = 1
+      @string_parse_stack = []
+      @scanner            = StringScanner.new source
+      nodes               = do_parse
+
+      return nodes.generate_top
+    end
 
-    @cond = 0
-    @cmdarg = 0
-    @line_number = 1
+    def next_token
+      t = get_next_token
+      t[1] = { :value => t[1], :line => @line_number }
+      t
+    end
 
-    @string_parse_stack = []
+    def cond_push(n)
+      @cond = (@cond << 1) | (n & 1)
+    end
 
-    @scanner = StringScanner.new source
-  end
+    def cond_pop
+      @cond = @cond >> 1
+    end
 
-  def parse!
-    do_parse
-  end
+    def cond_lexpop
+      @cond = (@cond >> 1) | (@cond & 1)
+    end
 
-  def next_token
-    t = get_next_token
-    t[1] = { :value => t[1], :line => @line_number }
-    t
-  end
+    def cond?
+      (@cond & 1) != 0
+    end
 
-  def cond_push(n)
-    @cond = (@cond << 1) | (n & 1)
-  end
+    def cmdarg_push(n)
+      @cmdarg = (@cmdarg << 1) | (n & 1)
+    end
 
-  def cond_pop
-    @cond = @cond >> 1
-  end
+    def cmdarg_pop
+      @cmdarg = @cmdarg >> 1
+    end
 
-  def cond_lexpop
-    @cond = (@cond >> 1) | (@cond & 1)
-  end
+    def cmdarg_lexpop
+      @cmdarg = (@cmdarg >> 1) | (@cmdarg & 1)
+    end
 
-  def cond?
-    (@cond & 1) != 0
-  end
+    def cmdarg?
+      (@cmdarg & 1) != 0
+    end
 
-  def cmdarg_push(n)
-    @cmdarg = (@cmdarg << 1) | (n & 1)
-  end
+    def push_string_parse(hash)
+      @string_parse_stack << hash
+    end
 
-  def cmdarg_pop
-    @cmdarg = @cmdarg >> 1
-  end
+    def pop_string_parse
+      @string_parse_stack.pop
+    end
 
-  def cmdarg_lexpop
-    @cmdarg = (@cmdarg >> 1) | (@cmdarg & 1)
-  end
+    def current_string_parse
+      @string_parse_stack.last
+    end
 
-  def cmdarg?
-    (@cmdarg & 1) != 0
-  end
+    def next_string_token
+      # str_parse, scanner = current_string_parse, @scanner
+      str_parse = current_string_parse
+      scanner = @scanner
+      space = false
 
-  def push_string_parse(hash)
-    @string_parse_stack << hash
-  end
+      # everything bar single quote and lower case bare wrds can interpolate
+      interpolate = (str_parse[:beg] != "'" && str_parse[:beg] != 'w')
 
-  def pop_string_parse
-    @string_parse_stack.pop
-  end
+      words = ['w', 'W'].include? str_parse[:beg]
 
-  def current_string_parse
-    @string_parse_stack.last
-  end
+      space = true if ['w', 'W'].include?(str_parse[:beg]) and scanner.scan(/\s+/)
 
-  def next_string_token
-    # str_parse, scanner = current_string_parse, @scanner
-    str_parse = current_string_parse
-    scanner = @scanner
-    space = false
+      # see if we can read end of string/xstring/regecp markers
+      # if scanner.scan /#{str_parse[:end]}/
+      if scanner.scan Regexp.new(Regexp.escape(str_parse[:end]))
+        if words && !str_parse[:done_last_space]#&& space
+          str_parse[:done_last_space] = true
+          scanner.pos -= 1
+          return :SPACE, ' '
+        end
+        pop_string_parse
 
-    # everything bar single quote and lower case bare wrds can interpolate
-    interpolate = (str_parse[:beg] != "'" && str_parse[:beg] != 'w')
+        # return :SPACE, ' ' if words && space
+        
+        if ['"', "'"].include? str_parse[:beg]
+          @lex_state = :expr_end
+          return :STRING_END, scanner.matched
 
-    words = ['w', 'W'].include? str_parse[:beg]
+        elsif str_parse[:beg] == '`'
+          @lex_state = :expr_end
+          return :STRING_END, scanner.matched
 
-    space = true if ['w', 'W'].include?(str_parse[:beg]) and scanner.scan(/\s+/)
+        elsif str_parse[:beg] == '/'
+          result = scanner.matched if scanner.scan(/\w+/)
+          @lex_state = :expr_end
+          return :REGEXP_END, result
 
-    # see if we can read end of string/xstring/regecp markers
-    # if scanner.scan /#{str_parse[:end]}/
-    if scanner.scan Regexp.new(Regexp.escape(str_parse[:end]))
-      if words && !str_parse[:done_last_space]#&& space
-        str_parse[:done_last_space] = true
-        scanner.pos -= 1
-        return :SPACE, ' '
-      end
-      pop_string_parse
-
-      # return :SPACE, ' ' if words && space
-      
-      if ['"', "'"].include? str_parse[:beg]
-        @lex_state = :expr_end
-        return :STRING_END, scanner.matched
-
-      elsif str_parse[:beg] == '`'
-        @lex_state = :expr_end
-        return :STRING_END, scanner.matched
-
-      elsif str_parse[:beg] == '/'
-        result = scanner.matched if scanner.scan(/\w+/)
-        @lex_state = :expr_end
-        return :REGEXP_END, result
-
-      else
-        @lex_state = :expr_end
-        return :STRING_END, scanner.matched
+        else
+          @lex_state = :expr_end
+          return :STRING_END, scanner.matched
+        end
       end
-    end
 
-    return :SPACE, ' ' if space
+      return :SPACE, ' ' if space
 
-    # not end of string, so we must be parsing contents
-    str_buffer = []
+      # not end of string, so we must be parsing contents
+      str_buffer = []
 
-    if scanner.scan(/#(\$\@)\w+/)
-      if interpolate
-        return :STRING_DVAR, scanner.matched.slice(2)
-      else
-        str_buffer << scanner.matched
-      end
+      if scanner.scan(/#(\$\@)\w+/)
+        if interpolate
+          return :STRING_DVAR, scanner.matched.slice(2)
+        else
+          str_buffer << scanner.matched
+        end
 
-    elsif scanner.scan(/#\{/)
-      if interpolate
-        # we are into ruby code, so stop parsing content (for now)
-        str_parse[:content] = false
-        return :STRING_DBEG, scanner.matched
-      else
-        str_buffer << scanner.matched
+      elsif scanner.scan(/#\{/)
+        if interpolate
+          # we are into ruby code, so stop parsing content (for now)
+          str_parse[:content] = false
+          return :STRING_DBEG, scanner.matched
+        else
+          str_buffer << scanner.matched
+        end
+
+      # causes error, so we will just collect it later on with other text
+      elsif scanner.scan(/\#/)
+        str_buffer << '#'
       end
 
-    # causes error, so we will just collect it later on with other text
-    elsif scanner.scan(/\#/)
-      str_buffer << '#'
+      add_string_content str_buffer, str_parse
+      complete_str = str_buffer.join ''
+      return :STRING_CONTENT, complete_str
     end
 
-    add_string_content str_buffer, str_parse
-    complete_str = str_buffer.join ''
-    return :STRING_CONTENT, complete_str
-  end
+    def add_string_content(str_buffer, str_parse)
+      scanner = @scanner
+      # regexp for end of string/regexp
+      # end_str_re = /#{str_parse[:end]}/
+      end_str_re = Regexp.new(Regexp.escape(str_parse[:end]))
+      # can be interpolate
+      interpolate = ['"', 'W', '/', '`'].include? str_parse[:beg]
 
-  def add_string_content(str_buffer, str_parse)
-    scanner = @scanner
-    # regexp for end of string/regexp
-    # end_str_re = /#{str_parse[:end]}/
-    end_str_re = Regexp.new(Regexp.escape(str_parse[:end]))
-    # can be interpolate
-    interpolate = ['"', 'W', '/', '`'].include? str_parse[:beg]
+      words = ['W', 'w'].include? str_parse[:beg]
 
-    words = ['W', 'w'].include? str_parse[:beg]
+      until scanner.eos?
+        c = nil
+        handled = true
 
-    until scanner.eos?
-      c = nil
-      handled = true
+        if scanner.check end_str_re
+          # eos
+          break
 
-      if scanner.check end_str_re
-        # eos
-        break
+        elsif words && scanner.scan(/\s/)
+          scanner.pos -= 1
+          break
 
-      elsif words && scanner.scan(/\s/)
-        scanner.pos -= 1
-        break
+        elsif interpolate && scanner.check(/#(?=[\@\{])/)
+          break
 
-      elsif interpolate && scanner.check(/#(?=[\@\{])/)
-        break
+        elsif scanner.scan(/\\\\/)
+          c = scanner.matched
 
-      elsif scanner.scan(/\\\\/)
-        c = scanner.matched
+        elsif scanner.scan(/\\/)
+          c = scanner.matched
+          c += scanner.matched if scanner.scan end_str_re
 
-      elsif scanner.scan(/\\/)
-        c = scanner.matched
-        c += scanner.matched if scanner.scan end_str_re
+        else
+          handled = false
+        end
 
-      else
-        handled = false
-      end
+        unless handled
+          reg = words ? Regexp.new("[^#{Regexp.escape str_parse[:end]}\#\0\n\ \\\\]+|.") : Regexp.new("[^#{Regexp.escape str_parse[:end]}\#\0\\\\]+|.")
+          scanner.scan reg
+          c = scanner.matched
+        end
 
-      unless handled
-        reg = words ? Regexp.new("[^#{Regexp.escape str_parse[:end]}\#\0\n\ \\\\]+|.") : Regexp.new("[^#{Regexp.escape str_parse[:end]}\#\0\\\\]+|.")
-        scanner.scan reg
-        c = scanner.matched
+        c ||= scanner.matched
+        str_buffer << c
       end
 
-      c ||= scanner.matched
-      str_buffer << c
+      raise "reached EOF while in string" if scanner.eos?
     end
 
-    raise "reached EOF while in string" if scanner.eos?
-  end
+    def get_next_token
+      string_scanner = current_string_parse
 
-  def get_next_token
-    string_scanner = current_string_parse
+      if string_scanner && string_scanner[:content]
+        return next_string_token
+      end
 
-    if string_scanner && string_scanner[:content]
-      return next_string_token
-    end
+      # scanner, space_seen, cmd_start, c = @scanner, false, false, ''
+      scanner = @scanner
+      space_seen = false
+      cmd_start = false
+      c = ''
 
-    # scanner, space_seen, cmd_start, c = @scanner, false, false, ''
-    scanner = @scanner
-    space_seen = false
-    cmd_start = false
-    c = ''
-
-    while true
-      if scanner.scan(/\ |\t|\r/)
-        space_seen = true
-        next
-
-      elsif scanner.scan(/(\n|#)/)
-        c = scanner.matched
-        if c == '#' then scanner.scan(/(.*)/) else @line_number += 1; end
-
-        scanner.scan(/(\n+)/)
-        @line_number += scanner.matched.length if scanner.matched
-
-        next if [:expr_beg, :expr_dot].include? @lex_state
-
-        cmd_start = true
-        @lex_state = :expr_beg
-        return '\\n', '\\n'
-
-      elsif scanner.scan(/\;/)
-        @lex_state = :expr_beg
-        return ';', ';'
-
-      elsif scanner.scan(/\"/)
-        push_string_parse :beg => '"', :content => true, :end => '"'
-        return :STRING_BEG, scanner.matched
-
-      elsif scanner.scan(/\'/)
-        push_string_parse :beg => "'", :content => true, :end => "'"
-        return :STRING_BEG, scanner.matched
-
-      elsif scanner.scan(/\`/)
-        push_string_parse :beg => "`", :content => true, :end => "`"
-        return :XSTRING_BEG, scanner.matched
-
-      elsif scanner.scan(/\%W/)
-        start_word  = scanner.scan(/./)
-        end_word    = { '(' => ')', '[' => ']', '{' => '}' }[start_word] || start_word
-        push_string_parse :beg => 'W', :content => true, :end => end_word
-        return :WORDS_BEG, scanner.matched
-
-      elsif scanner.scan(/\%w/)
-        start_word  = scanner.scan(/./)
-        end_word    = { '(' => ')', '[' => ']', '{' => '}' }[start_word] || start_word
-        push_string_parse :beg => 'w', :content => true, :end => end_word
-        return :AWORDS_BEG, scanner.matched
-
-      elsif scanner.scan(/\%[Qq]/)
-        start_word  = scanner.scan(/./)
-        end_word    = { '(' => ')', '[' => ']', '{' => '}' }[start_word] || start_word
-        push_string_parse :beg => start_word, :content => true, :end => end_word
-        return :STRING_BEG, scanner.matched
-
-      elsif scanner.scan(/\//)
-        if [:expr_beg, :expr_mid].include? @lex_state
-          push_string_parse :beg => '/', :content => true, :end => '/'
-          return :REGEXP_BEG, scanner.matched
-        elsif scanner.scan(/\=/)
-          @lex_state = :expr_beg
-          return :OP_ASGN, '/'
-        elsif @lex_state == :expr_fname
-          @lex_state = :expr_end
-        end
+      while true
+        if scanner.scan(/\ |\t|\r/)
+          space_seen = true
+          next
 
-        return '/', '/'
+        elsif scanner.scan(/(\n|#)/)
+          c = scanner.matched
+          if c == '#' then scanner.scan(/(.*)/) else @line_number += 1; end
 
-      elsif scanner.scan(/\%/)
-        @lex_state = @lex_state == :expr_fname ? :expr_end : :expr_beg
-        return '%', '%'
+          scanner.scan(/(\n+)/)
+          @line_number += scanner.matched.length if scanner.matched
 
-      elsif scanner.scan(/\(/)
-        result = scanner.matched
-        if [:expr_beg, :expr_mid].include? @lex_state
-          result = :PAREN_BEG
-        elsif space_seen
-          result = '('
-        end
+          next if [:expr_beg, :expr_dot].include? @lex_state
+
+          cmd_start = true
+          @lex_state = :expr_beg
+          return '\\n', '\\n'
 
-        @lex_state = :expr_beg
-        cond_push 0
-        cmdarg_push 0
+        elsif scanner.scan(/\;/)
+          @lex_state = :expr_beg
+          return ';', ';'
+
+        elsif scanner.scan(/\"/)
+          push_string_parse :beg => '"', :content => true, :end => '"'
+          return :STRING_BEG, scanner.matched
 
-        return result, scanner.matched
+        elsif scanner.scan(/\'/)
+          push_string_parse :beg => "'", :content => true, :end => "'"
+          return :STRING_BEG, scanner.matched
+
+        elsif scanner.scan(/\`/)
+          push_string_parse :beg => "`", :content => true, :end => "`"
+          return :XSTRING_BEG, scanner.matched
+
+        elsif scanner.scan(/\%W/)
+          start_word  = scanner.scan(/./)
+          end_word    = { '(' => ')', '[' => ']', '{' => '}' }[start_word] || start_word
+          push_string_parse :beg => 'W', :content => true, :end => end_word
+          return :WORDS_BEG, scanner.matched
+
+        elsif scanner.scan(/\%w/)
+          start_word  = scanner.scan(/./)
+          end_word    = { '(' => ')', '[' => ']', '{' => '}' }[start_word] || start_word
+          push_string_parse :beg => 'w', :content => true, :end => end_word
+          return :AWORDS_BEG, scanner.matched
+
+        elsif scanner.scan(/\%[Qq]/)
+          start_word  = scanner.scan(/./)
+          end_word    = { '(' => ')', '[' => ']', '{' => '}' }[start_word] || start_word
+          push_string_parse :beg => start_word, :content => true, :end => end_word
+          return :STRING_BEG, scanner.matched
+
+        elsif scanner.scan(/\//)
+          if [:expr_beg, :expr_mid].include? @lex_state
+            push_string_parse :beg => '/', :content => true, :end => '/'
+            return :REGEXP_BEG, scanner.matched
+          elsif scanner.scan(/\=/)
+            @lex_state = :expr_beg
+            return :OP_ASGN, '/'
+          elsif @lex_state == :expr_fname
+            @lex_state = :expr_end
+          end
 
-      elsif scanner.scan(/\)/)
-        cond_lexpop
-        cmdarg_lexpop
-        @lex_state = :expr_end
-        return ')', scanner.matched
+          return '/', '/'
 
-      elsif scanner.scan(/\[/)
-        result = scanner.matched
+        elsif scanner.scan(/\%/)
+          @lex_state = @lex_state == :expr_fname ? :expr_end : :expr_beg
+          return '%', '%'
 
-        if [:expr_fname, :expr_dot].include? @lex_state
-          @lex_state = :expr_arg
-          if scanner.scan(/\]=/)
-            return '[]=', '[]='
-          elsif scanner.scan(/\]/)
-            return '[]', '[]'
-          else
-            raise "Unexpected '[' token"
+        elsif scanner.scan(/\(/)
+          result = scanner.matched
+          if [:expr_beg, :expr_mid].include? @lex_state
+            result = :PAREN_BEG
+          elsif space_seen
+            result = '('
           end
-        elsif [:expr_beg, :expr_mid].include?(@lex_state) || space_seen
-          @lex_state = :expr_beg
-          cond_push 0
-          cmdarg_push 0
-          return '[', scanner.matched
-        else
+
           @lex_state = :expr_beg
           cond_push 0
           cmdarg_push 0
-          return '[@', scanner.matched
-        end
-
-      elsif scanner.scan(/\]/)
-        cond_lexpop
-        cmdarg_lexpop
-        @lex_state = :expr_end
-        return ']', scanner.matched
 
-      elsif scanner.scan(/\}/)
-        cond_lexpop
-        cmdarg_lexpop
-        @lex_state = :expr_end
+          return result, scanner.matched
 
-        current_string_parse[:content] = true if current_string_parse
-        return '}', scanner.matched
+        elsif scanner.scan(/\)/)
+          cond_lexpop
+          cmdarg_lexpop
+          @lex_state = :expr_end
+          return ')', scanner.matched
+
+        elsif scanner.scan(/\[/)
+          result = scanner.matched
+
+          if [:expr_fname, :expr_dot].include? @lex_state
+            @lex_state = :expr_arg
+            if scanner.scan(/\]=/)
+              return '[]=', '[]='
+            elsif scanner.scan(/\]/)
+              return '[]', '[]'
+            else
+              raise "Unexpected '[' token"
+            end
+          elsif [:expr_beg, :expr_mid].include?(@lex_state) || space_seen
+            @lex_state = :expr_beg
+            cond_push 0
+            cmdarg_push 0
+            return '[', scanner.matched
+          else
+            @lex_state = :expr_beg
+            cond_push 0
+            cmdarg_push 0
+            return '[@', scanner.matched
+          end
 
-      elsif scanner.scan(/\.\.\./)
-        @lex_state = :expr_beg
-        return '...', scanner.matched
+        elsif scanner.scan(/\]/)
+          cond_lexpop
+          cmdarg_lexpop
+          @lex_state = :expr_end
+          return ']', scanner.matched
 
-      elsif scanner.scan(/\.\./)
-        @lex_state = :expr_beg
-        return '..', scanner.matched
+        elsif scanner.scan(/\}/)
+          cond_lexpop
+          cmdarg_lexpop
+          @lex_state = :expr_end
 
-      elsif scanner.scan(/\./)
-        @lex_state = :expr_dot unless @lex_state == :expr_fname
-        return '.', scanner.matched
+          current_string_parse[:content] = true if current_string_parse
+          return '}', scanner.matched
 
-      elsif scanner.scan(/\*\*\=/)
-        @lex_state = :expr_beg
-        return :OP_ASGN, '**'
+        elsif scanner.scan(/\.\.\./)
+          @lex_state = :expr_beg
+          return '...', scanner.matched
 
-      elsif scanner.scan(/\*\*/)
-        return '**', '**'
+        elsif scanner.scan(/\.\./)
+          @lex_state = :expr_beg
+          return '..', scanner.matched
 
-      elsif scanner.scan(/\*\=/)
-        @lex_state = :expr_beg
-        return :OP_ASGN, '*'
+        elsif scanner.scan(/\./)
+          @lex_state = :expr_dot unless @lex_state == :expr_fname
+          return '.', scanner.matched
 
-      elsif scanner.scan(/\*/)
-        result = scanner.matched
-        if @lex_state == :expr_fname
-          @lex_state = :expr_end
-          return '*', result
-        elsif space_seen && scanner.check(/\S/)
+        elsif scanner.scan(/\*\*\=/)
           @lex_state = :expr_beg
-          return :SPLAT, result
-        elsif [:expr_beg, :expr_mid].include? @lex_state
-          @lex_state = :expr_beg
-          return :SPLAT, result
-        else
-          @lex_state = :expr_beg
-          return '*', result
-        end
+          return :OP_ASGN, '**'
 
-      elsif scanner.scan(/\:\:/)
-        if [:expr_beg, :expr_mid, :expr_class].include? @lex_state
+        elsif scanner.scan(/\*\*/)
+          return '**', '**'
+
+        elsif scanner.scan(/\*\=/)
           @lex_state = :expr_beg
-          return '::@', scanner.matched
-        end
+          return :OP_ASGN, '*'
 
-        @lex_state = :expr_dot
-        return '::', scanner.matched
+        elsif scanner.scan(/\*/)
+          result = scanner.matched
+          if @lex_state == :expr_fname
+            @lex_state = :expr_end
+            return '*', result
+          elsif space_seen && scanner.check(/\S/)
+            @lex_state = :expr_beg
+            return :SPLAT, result
+          elsif [:expr_beg, :expr_mid].include? @lex_state
+            @lex_state = :expr_beg
+            return :SPLAT, result
+          else
+            @lex_state = :expr_beg
+            return '*', result
+          end
 
-      elsif scanner.scan(/\:/)
-        if [:expr_end, :expr_endarg].include?(@lex_state) || scanner.check(/\s/)
-          unless scanner.check(/\w/)
+        elsif scanner.scan(/\:\:/)
+          if [:expr_beg, :expr_mid, :expr_class].include? @lex_state
             @lex_state = :expr_beg
-            return ':', ':'
+            return '::@', scanner.matched
+          end
+
+          @lex_state = :expr_dot
+          return '::', scanner.matched
+
+        elsif scanner.scan(/\:/)
+          if [:expr_end, :expr_endarg].include?(@lex_state) || scanner.check(/\s/)
+            unless scanner.check(/\w/)
+              @lex_state = :expr_beg
+              return ':', ':'
+            end
+
+            @lex_state = :expr_fname
+            return :SYMBOL_BEG, ':'
+          end
+
+          if scanner.scan(/\'/)
+            push_string_parse :beg => "'", :content => true, :end => "'"
+          elsif scanner.scan(/\"/)
+            push_string_parse :beg => '"', :content => true, :end => '"'
           end
 
           @lex_state = :expr_fname
           return :SYMBOL_BEG, ':'
-        end
-
-        if scanner.scan(/\'/)
-          push_string_parse :beg => "'", :content => true, :end => "'"
-        elsif scanner.scan(/\"/)
-          push_string_parse :beg => '"', :content => true, :end => '"'
-        end
 
-        @lex_state = :expr_fname
-        return :SYMBOL_BEG, ':'
+        elsif scanner.check(/\|/)
+          if scanner.scan(/\|\|\=/)
+            @lex_state = :expr_beg
+            return :OP_ASGN, '||'
+          elsif scanner.scan(/\|\|/)
+            @lex_state = :expr_beg
+            return '||', '||'
+          elsif scanner.scan(/\|\=/)
+            @lex_state = :expr_beg
+            return :OP_ASGN, '|'
+          elsif scanner.scan(/\|/)
+            if @lex_state == :expr_fname
+              @lex_state = :expr_end
+              return '|', scanner.matched
+            else
+              @lex_state = :expr_beg
+              return '|', scanner.matched
+            end
+          end
 
-      elsif scanner.check(/\|/)
-        if scanner.scan(/\|\|\=/)
-          @lex_state = :expr_beg
-          return :OP_ASGN, '||'
-        elsif scanner.scan(/\|\|/)
-          @lex_state = :expr_beg
-          return '||', '||'
-        elsif scanner.scan(/\|\=/)
-          @lex_state = :expr_beg
-          return :OP_ASGN, '|'
-        elsif scanner.scan(/\|/)
+        elsif scanner.scan(/\^/)
           if @lex_state == :expr_fname
             @lex_state = :expr_end
-            return '|', scanner.matched
-          else
-            @lex_state = :expr_beg
-            return '|', scanner.matched
+            return '^', scanner.matched
           end
-        end
 
-      elsif scanner.scan(/\^/)
-        if @lex_state == :expr_fname
-          @lex_state = :expr_end
+          @lex_state = :expr_beg
           return '^', scanner.matched
-        end
 
-        @lex_state = :expr_beg
-        return '^', scanner.matched
-
-      elsif scanner.check(/\&/)
-        if scanner.scan(/\&\&\=/)
-          @lex_state = :expr_beg
-          return :OP_ASGN, '&&'
-        elsif scanner.scan(/\&\&/)
-          @lex_state = :expr_beg
-          return '&&', scanner.matched
-        elsif scanner.scan(/\&\=/)
-          @lex_state = :expr_beg
-          return :OP_ASGN, '&'
-        elsif scanner.scan(/\&/)
-          if space_seen && !scanner.check(/\s/) && @lex_state == :expr_cmdarg
-            return '&@', '&'
-          elsif [:expr_beg, :expr_mid].include? @lex_state
-            return '&@', '&'
-          else
-            return '&', '&'
+        elsif scanner.check(/\&/)
+          if scanner.scan(/\&\&\=/)
+            @lex_state = :expr_beg
+            return :OP_ASGN, '&&'
+          elsif scanner.scan(/\&\&/)
+            @lex_state = :expr_beg
+            return '&&', scanner.matched
+          elsif scanner.scan(/\&\=/)
+            @lex_state = :expr_beg
+            return :OP_ASGN, '&'
+          elsif scanner.scan(/\&/)
+            if space_seen && !scanner.check(/\s/) && @lex_state == :expr_cmdarg
+              return '&@', '&'
+            elsif [:expr_beg, :expr_mid].include? @lex_state
+              return '&@', '&'
+            else
+              return '&', '&'
+            end
           end
-        end
 
-      elsif scanner.check(/\</)
-        if scanner.scan(/\<\<\=/)
-          @lex_state = :expr_beg
-          return :OP_ASGN, '<<'
-        elsif scanner.scan(/\<\</)
-          if @lex_state == :expr_fname
-            @lex_state = :expr_end
-            return '<<', '<<'
-          elsif ![:expr_end, :expr_dot, :expr_endarg, :expr_class].include?(@lex_state) && space_seen
+        elsif scanner.check(/\</)
+          if scanner.scan(/\<\<\=/)
+            @lex_state = :expr_beg
+            return :OP_ASGN, '<<'
+          elsif scanner.scan(/\<\</)
+            if @lex_state == :expr_fname
+              @lex_state = :expr_end
+              return '<<', '<<'
+            elsif ![:expr_end, :expr_dot, :expr_endarg, :expr_class].include?(@lex_state) && space_seen
+              @lex_state = :expr_beg
+              return '<<', '<<'
+            end
             @lex_state = :expr_beg
             return '<<', '<<'
+          elsif scanner.scan(/\<\=\>/)
+            if @lex_state == :expr_fname
+              @lex_state = :expr_end
+            else
+              @lex_state = :expr_beg
+            end
+            return '<=>', '<=>'
+          elsif scanner.scan(/\<\=/)
+            if @lex_state == :expr_fname
+              @lex_state = :expr_end
+            else
+              @lex_state = :expr_beg
+            end
+            return '<=', '<='
+          elsif scanner.scan(/\</)
+            if @lex_state == :expr_fname
+              @lex_state = :expr_end
+            else
+              @lex_state = :expr_beg
+            end
+            return '<', '<'
           end
-          @lex_state = :expr_beg
-          return '<<', '<<'
-        elsif scanner.scan(/\<\=\>/)
-          if @lex_state == :expr_fname
-            @lex_state = :expr_end
-          else
-            @lex_state = :expr_beg
+
+        elsif scanner.check(/\>/)
+          if scanner.scan(/\>\>\=/)
+            return :OP_ASGN, '>>'
+          elsif scanner.scan(/\>\>/)
+            return '>>', '>>'
+          elsif scanner.scan(/\>\=/)
+            if @lex_state == :expr_fname
+              @lex_state = :expr_end
+            else
+              @lex_state = :expr_beg
+            end
+            return '>=', scanner.matched
+          elsif scanner.scan(/\>/)
+            if @lex_state == :expr_fname
+              @lex_state = :expr_end
+            else
+              @lex_state = :expr_beg
+            end
+            return '>', '>'
           end
-          return '<=>', '<=>'
-        elsif scanner.scan(/\<\=/)
-          if @lex_state == :expr_fname
+
+        elsif scanner.scan(/[+-]/)
+          result  = scanner.matched
+          sign    = result + '@'
+
+          if @lex_state == :expr_beg || @lex_state == :expr_mid
             @lex_state = :expr_end
-          else
+            return [sign, sign]
+          elsif @lex_state == :expr_fname
+            @lex_state = :expr_end
+            return [:IDENTIFIER, result + scanner.matched] if scanner.scan(/@/)
+            return [result, result]
+          end
+
+          if scanner.scan(/\=/)
             @lex_state = :expr_beg
+            return [:OP_ASGN, result]
           end
-          return '<=', '<='
-        elsif scanner.scan(/\</)
+
+          @lex_state = :expr_beg
+          return [result, result]
+
+        elsif scanner.scan(/\?/)
+          @lex_state = :expr_beg if [:expr_end, :expr_endarg].include?(@lex_state)
+          return '?', scanner.matched
+
+        elsif scanner.scan(/\=\=\=/)
           if @lex_state == :expr_fname
             @lex_state = :expr_end
-          else
-            @lex_state = :expr_beg
+            return '===', '==='
           end
-          return '<', '<'
-        end
+          @lex_state = :expr_beg
+          return '===', '==='
 
-      elsif scanner.check(/\>/)
-        if scanner.scan(/\>\>\=/)
-          return :OP_ASGN, '>>'
-        elsif scanner.scan(/\>\>/)
-          return '>>', '>>'
-        elsif scanner.scan(/\>\=/)
+        elsif scanner.scan(/\=\=/)
           if @lex_state == :expr_fname
             @lex_state = :expr_end
-          else
-            @lex_state = :expr_beg
+            return '==', '=='
           end
-          return '>=', scanner.matched
-        elsif scanner.scan(/\>/)
+          @lex_state = :expr_beg
+          return '==', '=='
+
+        elsif scanner.scan(/\=\~/)
           if @lex_state == :expr_fname
             @lex_state = :expr_end
-          else
-            @lex_state = :expr_beg
+            return '=~', '=~'
           end
-          return '>', '>'
-        end
+          @lex_state = :expr_beg
+          return '=~', '=~'
 
-      elsif scanner.scan(/[+-]/)
-        result  = scanner.matched
-        sign    = result + '@'
+        elsif scanner.scan(/\=\>/)
+          @lex_state = :expr_beg
+          return '=>', '=>'
 
-        if @lex_state == :expr_beg || @lex_state == :expr_mid
-          @lex_state = :expr_end
-          return [sign, sign]
-        elsif @lex_state == :expr_fname
-          @lex_state = :expr_end
-          return [:IDENTIFIER, result + scanner.matched] if scanner.scan(/@/)
-          return [result, result]
-        end
+        elsif scanner.scan(/\=/)
+          @lex_state = :expr_beg
+          return '=', '='
 
-        if scanner.scan(/\=/)
+        elsif scanner.scan(/\!\=/)
+          if @lex_state == :expr_fname
+            @lex_state == :expr_end
+            return '!=', '!='
+          end
           @lex_state = :expr_beg
-          return [:OP_ASGN, result]
-        end
+          return '!=', '!='
 
-        @lex_state = :expr_beg
-        return [result, result]
+        elsif scanner.scan(/\!\~/)
+          @lex_state = :expr_beg
+          return '!~', '!~'
 
-      elsif scanner.scan(/\?/)
-        @lex_state = :expr_beg if [:expr_end, :expr_endarg].include?(@lex_state)
-        return '?', scanner.matched
+        elsif scanner.scan(/\!/)
+          if @lex_state == :expr_fname
+            @lex_state = :expr_end
+            return '!', '!'
+          end
+          @lex_state = :expr_beg
+          return '!', '!'
 
-      elsif scanner.scan(/\=\=\=/)
-        if @lex_state == :expr_fname
-          @lex_state = :expr_end
-          return '===', '==='
-        end
-        @lex_state = :expr_beg
-        return '===', '==='
+        elsif scanner.scan(/\~/)
+          if @lex_state == :expr_fname
+            @lex_state = :expr_end
+            return '~', '~'
+          end
+          @lex_state = :expr_beg
+          return '~', '~'
 
-      elsif scanner.scan(/\=\=/)
-        if @lex_state == :expr_fname
+        elsif scanner.scan(/\$[\+\'\`\&!@\"~*$?\/\\:;=.,<>_]/)
           @lex_state = :expr_end
-          return '==', '=='
-        end
-        @lex_state = :expr_beg
-        return '==', '=='
+          return :GVAR, scanner.matched
 
-      elsif scanner.scan(/\=\~/)
-        if @lex_state == :expr_fname
+        elsif scanner.scan(/\$\w+/)
           @lex_state = :expr_end
-          return '=~', '=~'
-        end
-        @lex_state = :expr_beg
-        return '=~', '=~'
-
-      elsif scanner.scan(/\=\>/)
-        @lex_state = :expr_beg
-        return '=>', '=>'
+          return :GVAR, scanner.matched
 
-      elsif scanner.scan(/\=/)
-        @lex_state = :expr_beg
-        return '=', '='
-
-      elsif scanner.scan(/\!\=/)
-        if @lex_state == :expr_fname
-          @lex_state == :expr_end
-          return '!=', '!='
-        end
-        @lex_state = :expr_beg
-        return '!=', '!='
-
-      elsif scanner.scan(/\!\~/)
-        @lex_state = :expr_beg
-        return '!~', '!~'
-
-      elsif scanner.scan(/\!/)
-        if @lex_state == :expr_fname
+        elsif scanner.scan(/\@\@\w*/)
           @lex_state = :expr_end
-          return '!', '!'
-        end
-        @lex_state = :expr_beg
-        return '!', '!'
+          return :CVAR, scanner.matched
 
-      elsif scanner.scan(/\~/)
-        if @lex_state == :expr_fname
+        elsif scanner.scan(/\@\w*/)
           @lex_state = :expr_end
-          return '~', '~'
-        end
-        @lex_state = :expr_beg
-        return '~', '~'
-
-      elsif scanner.scan(/\$[\+\'\`\&!@\"~*$?\/\\:;=.,<>_]/)
-        @lex_state = :expr_end
-        return :GVAR, scanner.matched
-
-      elsif scanner.scan(/\$\w+/)
-        @lex_state = :expr_end
-        return :GVAR, scanner.matched
-
-      elsif scanner.scan(/\@\@\w*/)
-        @lex_state = :expr_end
-        return :CVAR, scanner.matched
-
-      elsif scanner.scan(/\@\w*/)
-        @lex_state = :expr_end
-        return :IVAR, scanner.matched
-
-      elsif scanner.scan(/\,/)
-        @lex_state = :expr_beg
-        return ',', scanner.matched
-
-      elsif scanner.scan(/\{/)
-        if [:expr_end, :expr_cmdarg].include? @lex_state
-          result = '{@'
-        elsif @lex_state == :expr_endarg
-          result = 'LBRACE_ARG'
-        else
-          result = '{'
-        end
+          return :IVAR, scanner.matched
 
-        @lex_state = :expr_beg
-        cond_push 0
-        cmdarg_push 0
-        return result, scanner.matched
-
-      elsif scanner.check(/[0-9]/)
-        @lex_state = :expr_end
-        if scanner.scan(/[\d_]+\.[\d_]+\b/)
-          return [:FLOAT, scanner.matched.gsub(/_/, '')]
-        elsif scanner.scan(/[\d_]+\b/)
-          return [:INTEGER, scanner.matched.gsub(/_/, '')]
-        elsif scanner.scan(/0(x|X)(\d|[a-f]|[A-F])+/)
-          return [:INTEGER, scanner.matched]
-        else
-          raise "Lexing error on numeric type: `#{scanner.peek 5}`"
-        end
+        elsif scanner.scan(/\,/)
+          @lex_state = :expr_beg
+          return ',', scanner.matched
 
-      elsif scanner.scan(/(\w)+[\?\!]?/)
-        case scanner.matched
-        when 'class'
-          if @lex_state == :expr_dot
-            @lex_state = :expr_end
-            return :IDENTIFIER, scanner.matched
+        elsif scanner.scan(/\{/)
+          if [:expr_end, :expr_cmdarg].include? @lex_state
+            result = '{@'
+          elsif @lex_state == :expr_endarg
+            result = 'LBRACE_ARG'
+          else
+            result = '{'
           end
-          @lex_state = :expr_class
-          return :CLASS, scanner.matched
-
-        when 'module'
-          return :IDENTIFIER, scanner.matched if @lex_state == :expr_dot
-          @lex_state = :expr_class
-          return :MODULE, scanner.matched
 
-        when 'def'
-          @lex_state = :expr_fname
-          return :DEF, scanner.matched
-
-        when 'undef'
-          @lex_state = :expr_fname
-          return :UNDEF, scanner.matched
-
-        when 'end'
-          if [:expr_dot, :expr_fname].include? @lex_state
-            @lex_state = :expr_end
-            return :IDENTIFIER, scanner.matched
-          end
+          @lex_state = :expr_beg
+          cond_push 0
+          cmdarg_push 0
+          return result, scanner.matched
 
+        elsif scanner.check(/[0-9]/)
           @lex_state = :expr_end
-          return :END, scanner.matched
-
-        when 'do'
-          if cond?
-            @lex_state = :expr_beg
-            return :DO_COND, scanner.matched
-          elsif cmdarg? && @lex_state != :expr_cmdarg
-            @lex_state = :expr_beg
-            return :DO_BLOCK, scanner.matched
-          elsif @lex_state == :expr_endarg
-            return :DO_BLOCK, scanner.matched
+          if scanner.scan(/[\d_]+\.[\d_]+\b/)
+            return [:FLOAT, scanner.matched.gsub(/_/, '')]
+          elsif scanner.scan(/[\d_]+\b/)
+            return [:INTEGER, scanner.matched.gsub(/_/, '')]
+          elsif scanner.scan(/0(x|X)(\d|[a-f]|[A-F])+/)
+            return [:INTEGER, scanner.matched]
           else
-            @lex_state = :expr_beg
-            return :DO, scanner.matched
+            raise "Lexing error on numeric type: `#{scanner.peek 5}`"
           end
 
-        when 'if'
-          return :IF, scanner.matched if @lex_state == :expr_beg
-          @lex_state = :expr_beg
-          return :IF_MOD, scanner.matched
+        elsif scanner.scan(/(\w)+[\?\!]?/)
+          case scanner.matched
+          when 'class'
+            if @lex_state == :expr_dot
+              @lex_state = :expr_end
+              return :IDENTIFIER, scanner.matched
+            end
+            @lex_state = :expr_class
+            return :CLASS, scanner.matched
+
+          when 'module'
+            return :IDENTIFIER, scanner.matched if @lex_state == :expr_dot
+            @lex_state = :expr_class
+            return :MODULE, scanner.matched
+
+          when 'def'
+            @lex_state = :expr_fname
+            return :DEF, scanner.matched
+
+          when 'undef'
+            @lex_state = :expr_fname
+            return :UNDEF, scanner.matched
+
+          when 'end'
+            if [:expr_dot, :expr_fname].include? @lex_state
+              @lex_state = :expr_end
+              return :IDENTIFIER, scanner.matched
+            end
 
-        when 'unless'
-          return :UNLESS, scanner.matched if @lex_state == :expr_beg
-          return :UNLESS_MOD, scanner.matched
+            @lex_state = :expr_end
+            return :END, scanner.matched
+
+          when 'do'
+            if cond?
+              @lex_state = :expr_beg
+              return :DO_COND, scanner.matched
+            elsif cmdarg? && @lex_state != :expr_cmdarg
+              @lex_state = :expr_beg
+              return :DO_BLOCK, scanner.matched
+            elsif @lex_state == :expr_endarg
+              return :DO_BLOCK, scanner.matched
+            else
+              @lex_state = :expr_beg
+              return :DO, scanner.matched
+            end
+
+          when 'if'
+            return :IF, scanner.matched if @lex_state == :expr_beg
+            @lex_state = :expr_beg
+            return :IF_MOD, scanner.matched
 
-        when 'else'
-          return :ELSE, scanner.matched
+          when 'unless'
+            return :UNLESS, scanner.matched if @lex_state == :expr_beg
+            return :UNLESS_MOD, scanner.matched
 
-        when 'elsif'
-          return :ELSIF, scanner.matched
+          when 'else'
+            return :ELSE, scanner.matched
 
-        when 'self'
-          @lex_state = :expr_end unless @lex_state == :expr_fname
-          return :SELF, scanner.matched
+          when 'elsif'
+            return :ELSIF, scanner.matched
 
-        when 'true'
-          @lex_state = :expr_end
-          return :TRUE, scanner.matched
+          when 'self'
+            @lex_state = :expr_end unless @lex_state == :expr_fname
+            return :SELF, scanner.matched
 
-        when 'false'
-          @lex_state = :expr_end
-          return :FALSE, scanner.matched
+          when 'true'
+            @lex_state = :expr_end
+            return :TRUE, scanner.matched
 
-        when 'nil'
-          @lex_state = :expr_end
-          return :NIL, scanner.matched
+          when 'false'
+            @lex_state = :expr_end
+            return :FALSE, scanner.matched
 
-        when 'undefined'
-          @lex_state = :expr_end
-          return :UNDEFINED, scanner.matched
+          when 'nil'
+            @lex_state = :expr_end
+            return :NIL, scanner.matched
 
-        when 'null'
-          @lex_state = :expr_end
-          return :NULL, scanner.matched
+          when 'undefined'
+            @lex_state = :expr_end
+            return :UNDEFINED, scanner.matched
 
-        when '__LINE__'
-          @lex_state = :expr_end
-          return :LINE, @line_number.to_s
+          when 'null'
+            @lex_state = :expr_end
+            return :NULL, scanner.matched
 
-        when '__FILE__'
-          @lex_state = :expr_end
-          return :FILE, scanner.matched
+          when '__LINE__'
+            @lex_state = :expr_end
+            return :LINE, @line_number.to_s
 
-        when 'begin'
-          if [:expr_dot, :expr_fname].include? @lex_state
+          when '__FILE__'
             @lex_state = :expr_end
-            return :IDENTIFIER, scanner.matched
-          end
-          @lex_state = :expr_beg
-          return :BEGIN, scanner.matched
+            return :FILE, scanner.matched
 
-        when 'rescue'
-          return :IDENTIFIER, scanner.matched if [:expr_dot, :expr_fname].include? @lex_state
-          return :RESCUE, scanner.matched if @lex_state == :expr_beg
-          @lex_state = :expr_beg
-          return :RESCUE_MOD, scanner.matched
+          when 'begin'
+            if [:expr_dot, :expr_fname].include? @lex_state
+              @lex_state = :expr_end
+              return :IDENTIFIER, scanner.matched
+            end
+            @lex_state = :expr_beg
+            return :BEGIN, scanner.matched
 
-        when 'ensure'
-          @lex_state = :expr_beg
-          return :ENSURE, scanner.matched
+          when 'rescue'
+            return :IDENTIFIER, scanner.matched if [:expr_dot, :expr_fname].include? @lex_state
+            return :RESCUE, scanner.matched if @lex_state == :expr_beg
+            @lex_state = :expr_beg
+            return :RESCUE_MOD, scanner.matched
 
-        when 'case'
-          @lex_state = :expr_beg
-          return :CASE, scanner.matched
+          when 'ensure'
+            @lex_state = :expr_beg
+            return :ENSURE, scanner.matched
 
-        when 'when'
-          @lex_state = :expr_beg
-          return :WHEN, scanner.matched
+          when 'case'
+            @lex_state = :expr_beg
+            return :CASE, scanner.matched
 
-        when 'or'
-          @lex_state = :expr_beg
-          return :OR, scanner.matched
+          when 'when'
+            @lex_state = :expr_beg
+            return :WHEN, scanner.matched
 
-        when 'and'
-          @lex_state = :expr_beg
-          return :AND, scanner.matched
+          when 'or'
+            @lex_state = :expr_beg
+            return :OR, scanner.matched
 
-        when 'not'
-          @lex_state = :expr_beg
-          return :NOT, scanner.matched
+          when 'and'
+            @lex_state = :expr_beg
+            return :AND, scanner.matched
 
-        when 'return'
-          @lex_state = :expr_mid
-          return :RETURN, scanner.matched
+          when 'not'
+            @lex_state = :expr_beg
+            return :NOT, scanner.matched
 
-        when 'next'
-          if @lex_state == :expr_dot || @lex_state == :expr_fname
-            @lex_state = :expr_end
-            return :IDENTIFIER, scanner.matched
-          end
+          when 'return'
+            @lex_state = :expr_mid
+            return :RETURN, scanner.matched
 
-          @lex_state = :expr_mid
-          return :NEXT, scanner.matched
+          when 'next'
+            if @lex_state == :expr_dot || @lex_state == :expr_fname
+              @lex_state = :expr_end
+              return :IDENTIFIER, scanner.matched
+            end
 
-        when 'redo'
-          if @lex_state == :expr_dot || @lex_state == :expr_fname
-            @lex_state = :expr_end
-            return :IDENTIFIER, scanner.matched
-          end
+            @lex_state = :expr_mid
+            return :NEXT, scanner.matched
 
-          @lex_state = :expr_mid
-          return :REDO, scanner.matched
+          when 'redo'
+            if @lex_state == :expr_dot || @lex_state == :expr_fname
+              @lex_state = :expr_end
+              return :IDENTIFIER, scanner.matched
+            end
 
-        when 'break'
-          @lex_state = :expr_mid
-          return :BREAK, scanner.matched
+            @lex_state = :expr_mid
+            return :REDO, scanner.matched
 
-        when 'super'
-          @lex_state = :expr_arg
-          return :SUPER, scanner.matched
+          when 'break'
+            @lex_state = :expr_mid
+            return :BREAK, scanner.matched
 
-        when 'then'
-          return :THEN, scanner.matched
+          when 'super'
+            @lex_state = :expr_arg
+            return :SUPER, scanner.matched
 
-        when 'while'
-          return :WHILE, scanner.matched if @lex_state == :expr_beg
-          @lex_state = :expr_beg
-          return :WHILE_MOD, scanner.matched
+          when 'then'
+            return :THEN, scanner.matched
 
-        when 'for'
-          @lex_state = :expr_beg
-          return :FOR, scanner.matched
+          when 'while'
+            return :WHILE, scanner.matched if @lex_state == :expr_beg
+            @lex_state = :expr_beg
+            return :WHILE_MOD, scanner.matched
 
-        when 'in'
-          @lex_state = :expr_beg
-          return :IN, scanner.matched
+          when 'for'
+            @lex_state = :expr_beg
+            return :FOR, scanner.matched
 
-        when 'until'
-          return :WHILE, scanner.matched if @lex_state == :expr_beg
-          @lex_state = :expr_beg
-          return :UNTIL_MOD, scanner.matched
+          when 'in'
+            @lex_state = :expr_beg
+            return :IN, scanner.matched
 
-        when 'yield'
-          @lex_state = :expr_arg
-          return :YIELD, scanner.matched
-        end
+          when 'until'
+            return :WHILE, scanner.matched if @lex_state == :expr_beg
+            @lex_state = :expr_beg
+            return :UNTIL_MOD, scanner.matched
 
-        matched = scanner.matched
-        return :LABEL, matched if scanner.peek(2) != '::' && scanner.scan(/\:/)
+          when 'yield'
+            @lex_state = :expr_arg
+            return :YIELD, scanner.matched
+          end
 
-        if @lex_state == :expr_fname
-          if scanner.scan(/\=/)
+          matched = scanner.matched
+          return :LABEL, matched if scanner.peek(2) != '::' && scanner.scan(/\:/)
+
+          if @lex_state == :expr_fname
+            if scanner.scan(/\=/)
+              @lex_state = :expr_end
+              return :IDENTIFIER, matched + scanner.matched
+            end
+          end
+
+          if [:expr_beg, :expr_dot, :expr_mid, :expr_arg, :expr_cmdarg].include? @lex_state
+            @lex_state = :expr_cmdarg
+          else
             @lex_state = :expr_end
-            return :IDENTIFIER, matched + scanner.matched
           end
-        end
 
-        if [:expr_beg, :expr_dot, :expr_mid, :expr_arg, :expr_cmdarg].include? @lex_state
-          @lex_state = :expr_cmdarg
-        else
-          @lex_state = :expr_end
-        end
+          return [matched =~ /[A-Z]/ ? :CONSTANT : :IDENTIFIER, matched]
 
-        return [matched =~ /[A-Z]/ ? :CONSTANT : :IDENTIFIER, matched]
+        end
+        return [false, false] if scanner.eos?
 
+        raise RubyLexingError, "Unexpected content in parsing stream `#{scanner.peek 5}`"
       end
-      return [false, false] if scanner.eos?
-
-      raise RubyLexingError, "Unexpected content in parsing stream `#{scanner.peek 5}`"
     end
   end
 end
-  end