rogue_parser 1.0.1

data/.autotest

@@ -0,0 +1,38 @@
+# -*- ruby -*-
+
+Autotest.add_hook :initialize do |at|
+  at.extra_files << "../../ParseTree/dev/test/pt_testcase.rb"
+  at.libs << ":../../ParseTree/dev/lib:../../ParseTree/dev/test"
+  at.add_exception 'unit'
+  at.add_exception 'coverage'
+  at.add_exception '.diff'
+  at.add_exception 'coverage.info'
+
+  at.unit_diff = "unit_diff -u -b"
+
+  at.add_mapping(/^lib\/.*\.y$/) do |f, _|
+    at.files_matching %r%^test/.*#{File.basename(f, '.y').gsub '_', '_?'}.rb$%
+  end
+
+  at.add_mapping(/pt_testcase.rb/) do |f, _|
+    at.files_matching(/^test.*rb$/)
+  end
+
+  %w(TestEnvironment TestStackState).each do |klass|
+    at.extra_class_map[klass] = "test/test_ruby_parser_extras.rb"
+  end
+end
+
+Autotest.add_hook :run_command do |at|
+  system "rake parser"
+end
+
+class Autotest
+  def ruby
+    File.expand_path "~/.multiruby/install/1.9.0-0/bin/ruby"
+  end
+end if ENV['ONENINE']
+
+require 'autotest/rcov'
+Autotest::RCov.command = 'rcov_info'
+# Autotest::RCov.pattern = 'test/test_ruby_lexer.rb'

data/History.txt

@@ -0,0 +1,5 @@
+== 1.0.0 / 2007-12-20
+
+* 1 major enhancement
+  * Birthday!
+

data/Manifest.txt

@@ -0,0 +1,9 @@
+.autotest
+History.txt
+Manifest.txt
+README.txt
+Rakefile
+lib/ruby_lexer.rb
+lib/ruby_parser.y
+test/test_ruby_lexer.rb
+test/test_ruby_parser.rb

data/README.txt

@@ -0,0 +1,76 @@
+
+
+'rogue_parser' is 'ruby_parser' with this patch applied :
+
+http://rubyforge.org/tracker/?func=detail&aid=20106&group_id=439&atid=1780
+
+
+I hope I can get rid of this temporary repo by the next release of ruby_parser
+
+
+
+
+ruby_parser
+    by Ryan Davis
+    http://parsetree.rubyforge.org/
+
+== DESCRIPTION:
+
+ruby_parser (RP) is a ruby parser written in pure ruby (utilizing
+racc--which does by default use a C extension). RP's output is
+the same as ParseTree's output: s-expressions using ruby's arrays and
+base types.
+
+== FEATURES/PROBLEMS:
+
+* Pure ruby, no compiles.
+* Includes preceding comment data for defn/defs/class/module nodes!
+* Incredibly simple interface.
+* Output is 100% equivalent to ParseTree.
+  * Can utilize PT's SexpProcessor and UnifiedRuby for language processing.
+* Known Issue: Speed sucks currently. 5500 tests currently run in 21 min.
+* Known Issue: Code is waaay ugly. Port of a port. Not my fault. Will fix RSN.
+* Known Issue: I don't currently support newline nodes.
+* Known Issue: Totally awesome.
+* Known Issue: dasgn_curr decls can be out of order from ParseTree's.
+
+== SYNOPSIS:
+
+  RubyParser.new.parse "1+1"
+  # => s(:call, s(:lit, 1), :+, s(:array, s(:lit, 1)))
+
+== REQUIREMENTS:
+
+* ruby. woot.
+* ParseTree is needed for Sexp class... crap. I might break that out.
+* ParseTree for testing.
+* racc full package for parser development.
+
+== INSTALL:
+
+* sudo gem install ruby_parser
+
+== LICENSE:
+
+(The MIT License)
+
+Copyright (c) 2007 Ryan Davis
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,134 @@
+# -*- ruby -*-
+
+require 'rubygems'
+require 'hoe'
+
+$: << '../../ParseTree/dev/lib/'
+require './lib/ruby_parser_extras.rb'
+
+hoe = Hoe.new('rogue_parser', RubyParser::VERSION) do |parser|
+  parser.rubyforge_name = 'parsetree'
+  parser.developer('Ryan Davis', 'ryand-ruby@zenspider.com')
+  parser.extra_deps << 'ParseTree'
+end
+
+hoe.spec.files += ['lib/ruby_parser.rb'] # jim.... cmon man
+hoe.spec.files += ['lib/ruby_parser_extras.rb']
+
+module Rake::TaskManager
+  def all_tasks
+    @tasks
+  end
+end
+
+Rake.application.all_tasks["default"].prerequisites.clear
+
+[:default, :multi, :test].each do |t|
+  task t => :parser
+end
+
+path = "pkg/ruby_parser-#{RubyParser::VERSION}"
+task path => :parser do
+  Dir.chdir path do
+    sh "rake parser"
+  end
+end
+
+task :parser => ["lib/ruby_parser.rb"]
+
+rule '.rb' => '.y' do |t|
+  #sh "racc -l -t -o #{t.name} #{t.source}"
+  sh "racc -l -o #{t.name} #{t.source}"
+    # using racc 1.4.5... no -t...
+end
+
+task :clean do
+  rm_rf(Dir["**/*~"] +
+        Dir["**/*.diff"] +
+        Dir["coverage.info"] +
+        Dir["coverage"] +
+        Dir["lib/ruby_parser.rb"] +
+        Dir["lib/*.output"])
+end
+
+def next_num(glob)
+  num = Dir[glob].max[/\d+/].to_i + 1
+end
+
+def profile(type)
+  num = next_num("profile_#{type}*.txt")
+  sh "zenprofile -w -Ilib:ext:bin:test -rtest/unit test/test_ruby_#{type}.rb &> profile_#{type}_%03d.txt" % num
+end
+
+task :profile do
+  profile 'lexer'
+  profile 'parser'
+end
+
+begin
+  require 'rcov/rcovtask'
+  Rcov::RcovTask.new do |t|
+    pattern = ENV['PATTERN'] || 'test/test_ruby_*.rb'
+
+    t.test_files = FileList[pattern]
+    t.verbose = true
+    t.rcov_opts << "--threshold 80"
+    t.rcov_opts << "--no-color"
+  end
+rescue LoadError
+  # skip
+end
+
+desc "Compares PT to RP and deletes all files that match"
+task :compare do
+  files = Dir["unit/**/*.rb"]
+  puts "Parsing #{files.size} files"
+  files.each do |file|
+    puts file
+    system "./cmp.rb -q #{file} && rm #{file}"
+  end
+  system 'find -d unit -type d -empty -exec rmdir {} \;'
+end
+
+desc "Compares PT to RP and stops on first failure"
+task :find_bug do
+  files = Dir["unit/**/*.rb"]
+  puts "Parsing #{files.size} files"
+  files.each do |file|
+    puts file
+    sh "./cmp.rb -q #{file}"
+  end
+end
+
+task :sort do
+  sh 'grepsort "^ +def" lib/ruby_lexer.rb'
+  sh 'grepsort "^ +def (test|util)" test/test_ruby_lexer.rb'
+end
+
+task :rcov_info => :parser do
+  pattern = ENV['PATTERN'] || "test/test_*.rb"
+  ruby "-Ilib -S rcov --text-report --save coverage.info #{pattern}"
+end
+
+task :rcov_overlay do
+  rcov, eol = Marshal.load(File.read("coverage.info")).last[ENV["FILE"]], 1
+  puts rcov[:lines].zip(rcov[:coverage]).map { |line, coverage|
+    bol, eol = eol, eol + line.length
+    [bol, eol, "#ffcccc"] unless coverage
+  }.compact.inspect
+end
+
+task :loc do
+  loc1  = `wc -l ../1.0.0/lib/ruby_lexer.rb`[/\d+/]
+  flog1 = `flog -s ../1.0.0/lib/ruby_lexer.rb`[/\d+\.\d+/]
+  loc2  = `cat lib/ruby_lexer.rb lib/ruby_parser_extras.rb | wc -l`[/\d+/]
+  flog2 = `flog -s lib/ruby_lexer.rb lib/ruby_parser_extras.rb`[/\d+\.\d+/]
+
+  loc1, loc2, flog1, flog2 = loc1.to_i, loc2.to_i, flog1.to_f, flog2.to_f
+
+  puts "1.0.0: loc = #{loc1} flog = #{flog1}"
+  puts "dev  : loc = #{loc2} flog = #{flog2}"
+  puts "delta: loc = #{loc2-loc1} flog = #{flog2-flog1}"
+end
+
+# vim: syntax=Ruby

data/lib/ruby_lexer.rb

@@ -0,0 +1,1329 @@
+#$: << File.expand_path("~/Work/p4/zss/src/ParseTree/dev/lib") # for me, not you.
+require 'sexp'
+require 'ruby_parser_extras'
+
+class RubyLexer
+  attr_accessor :command_start
+  attr_accessor :cmdarg
+  attr_accessor :cond
+  attr_accessor :nest
+
+  # Additional context surrounding tokens that both the lexer and
+  # grammar use.
+  attr_reader :lex_state
+
+  attr_accessor :lex_strterm
+
+  # Stream of data that yylex examines.
+  attr_reader :src
+
+  # Last token read via yylex.
+  attr_accessor :token
+
+  # Tempory buffer to build up a potential token.  Consumer takes
+  # responsibility to reset this before use.
+  attr_accessor :token_buffer
+
+  # Value of last token which had a value associated with it.
+  attr_accessor :yacc_value
+
+  # What handles warnings
+  attr_accessor :warnings
+
+  EOF = :eof_haha!
+
+  # ruby constants for strings (should this be moved somewhere else?)
+  STR_FUNC_BORING = 0x00
+  STR_FUNC_ESCAPE = 0x01 # TODO: remove and replace with REGEXP
+  STR_FUNC_EXPAND = 0x02
+  STR_FUNC_REGEXP = 0x04
+  STR_FUNC_AWORDS = 0x08
+  STR_FUNC_SYMBOL = 0x10
+  STR_FUNC_INDENT = 0x20 # <<-HEREDOC
+
+  STR_SQUOTE = STR_FUNC_BORING
+  STR_DQUOTE = STR_FUNC_BORING | STR_FUNC_EXPAND
+  STR_XQUOTE = STR_FUNC_BORING | STR_FUNC_EXPAND
+  STR_REGEXP = STR_FUNC_REGEXP | STR_FUNC_ESCAPE | STR_FUNC_EXPAND
+  STR_SSYM   = STR_FUNC_SYMBOL
+  STR_DSYM   = STR_FUNC_SYMBOL | STR_FUNC_EXPAND
+
+  # How the parser advances to the next token.
+  #
+  # @return true if not at end of file (EOF).
+
+#   if ENV['SPY'] then
+#     @@stats = Hash.new 0
+#
+#     def self.stats
+#       @@stats
+#     end
+#
+#     at_exit {
+#       require 'pp'
+#       pp RubyLexer.stats.sort_by {|k,v| -v}.first(20)
+#     }
+#   end
+
+  def advance
+    r = yylex
+    self.token = r
+
+    @@stats[r] += 1 if ENV['SPY']
+
+    return r != RubyLexer::EOF
+  end
+
+  def arg_ambiguous
+    self.warning("Ambiguous first argument. make sure.")
+  end
+
+  def comments
+    c = @comments.join
+    @comments.clear
+    c
+  end
+
+  def expr_beg_push val
+    cond.push false
+    cmdarg.push false
+    self.lex_state = :expr_beg
+    self.yacc_value = s(val)
+  end
+
+  def fix_arg_lex_state
+    self.lex_state = if lex_state == :expr_fname || lex_state == :expr_dot
+                       :expr_arg
+                     else
+                       :expr_beg
+                     end
+  end
+
+  def heredoc here # Region has 63 lines, 1595 characters
+    _, eos, func, last_line = here
+
+    indent  = (func & STR_FUNC_INDENT) != 0
+    expand  = (func & STR_FUNC_EXPAND) != 0
+    eos_re  = indent ? /[ \t]*#{eos}(\r?\n|\z)/ : /#{eos}(\r?\n|\z)/
+    err_msg = "can't match #{eos_re.inspect} anywhere in "
+
+    rb_compile_error err_msg if
+      src.eos?
+
+    if src.beginning_of_line? && src.scan(eos_re) then
+      src.unread_many last_line # TODO: figure out how to remove this
+      self.yacc_value = s(eos)
+      return :tSTRING_END
+    end
+
+    token_buffer.clear
+
+    if expand then
+      case
+      when src.scan(/#[$@]/) then
+        src.pos -= 1 # FIX omg stupid
+        self.yacc_value = s(src.matched)
+        return :tSTRING_DVAR
+      when src.scan(/#[{]/) then
+        self.yacc_value = s(src.matched)
+        return :tSTRING_DBEG
+      when src.scan(/#/) then
+        token_buffer << '#'
+      end
+
+      until src.scan(eos_re) do
+        c = tokadd_string func, "\n", nil
+
+        rb_compile_error err_msg if
+          c == RubyLexer::EOF
+
+        if c != "\n" then
+          self.yacc_value = s(:str, token_buffer.join.delete("\r"))
+          return :tSTRING_CONTENT
+        else
+          token_buffer << src.scan(/\n/)
+        end
+
+        rb_compile_error err_msg if
+          src.eos?
+      end
+
+      # tack on a NL after the heredoc token - FIX NL should not be needed
+      src.unread_many(eos + "\n") # TODO: remove this... stupid stupid stupid
+    else
+      until src.check(eos_re) do
+        token_buffer << src.scan(/.*(\n|\z)/)
+        rb_compile_error err_msg if
+          src.eos?
+      end
+    end
+
+    self.lex_strterm = s(:heredoc, eos, func, last_line)
+    self.yacc_value = s(:str, token_buffer.join.delete("\r"))
+
+    return :tSTRING_CONTENT
+  end
+
+  def heredoc_identifier
+    term, func = nil, STR_FUNC_BORING
+    token_buffer.clear
+
+    case
+    when src.scan(/(-?)(['"`])(.*?)\2/) then
+      term = src[2]
+      unless src[1].empty? then
+        func |= STR_FUNC_INDENT
+      end
+      func |= case term
+              when "\'" then
+                STR_SQUOTE
+              when '"' then
+                STR_DQUOTE
+              else
+                STR_XQUOTE
+              end
+      token_buffer << src[3]
+    when src.scan(/-?(['"`])(?!\1*\Z)/) then
+      rb_compile_error "unterminated here document identifier"
+    when src.scan(/(-?)(\w+)/) then
+      term = '"'
+      func |= STR_DQUOTE
+      unless src[1].empty? then
+        func |= STR_FUNC_INDENT
+      end
+      token_buffer << src[2]
+    else
+      return nil
+    end
+
+    if src.check(/.*\n/) then
+      # TODO: think about storing off the char range instead
+      line = src.string[src.pos, src.matched_size]
+      src.string[src.pos, src.matched_size] = ''
+    else
+      line = nil
+    end
+
+    self.lex_strterm = s(:heredoc, token_buffer.join, func, line)
+
+    if term == '`' then
+      self.yacc_value = s("`")
+      return :tXSTRING_BEG
+    else
+      self.yacc_value = s("\"")
+      return :tSTRING_BEG
+    end
+  end
+
+  def initialize
+    self.token_buffer = []
+    self.cond = StackState.new(:cond)
+    self.cmdarg = StackState.new(:cmdarg)
+    self.nest = 0
+    @comments = []
+
+    reset
+  end
+
+  def int_with_base base
+    if src.matched =~ /__/ then
+      rb_compile_error "Invalid numeric format"
+    end
+    self.yacc_value = src.matched.to_i(base)
+    return :tINTEGER
+  end
+
+  def lex_state= o
+    raise "wtf?" unless Symbol === o
+    @lex_state = o
+  end
+
+  ##
+  #  Parse a number from the input stream.
+  #
+  # @param c The first character of the number.
+  # @return A int constant wich represents a token.
+
+  def parse_number
+    self.lex_state = :expr_end
+
+    case
+    when src.scan(/[+-]?0[xbd]\b/) then
+      rb_compile_error "Invalid numeric format"
+    when src.scan(/[+-]?0x[a-f0-9_]+/i) then
+      return int_with_base(16)
+    when src.scan(/[+-]?0b[01_]+/) then
+      return int_with_base(2)
+    when src.scan(/[+-]?0d[0-9_]+/) then
+      return int_with_base(10)
+    when src.scan(/[+-]?0o?[0-7_]*[89]/) then
+      rb_compile_error "Illegal octal digit."
+    when src.scan(/[+-]?0o?[0-7_]+|0o/) then
+      return int_with_base(8)
+    when src.scan(/[+-]?[\d_]+_(e|\.)/) then
+      rb_compile_error "Trailing '_' in number."
+    when src.scan(/[+-]?[\d_]+\.[\d_]+(e[+-]?[\d_]+)?\b|[+-]?[\d_]+e[+-]?[\d_]+\b/i) then
+      number = src.matched
+      if number =~ /__/ then
+        rb_compile_error "Invalid numeric format"
+      end
+      self.yacc_value = number.to_f
+      return :tFLOAT
+    when src.scan(/[+-]?0\b/) then
+      return int_with_base(10)
+    when src.scan(/[+-]?[\d_]+\b/) then
+      return int_with_base(10)
+    else
+      rb_compile_error "Bad number format"
+    end
+  end
+
+  def parse_quote
+    beg, nnd, short_hand, c = nil, nil, false, nil
+
+    if src.scan(/[a-z0-9]{1,2}/i) then # Long-hand (e.g. %Q{}).
+      rb_compile_error "unknown type of %string" if
+        src.matched_size == 2
+      c = src.matched
+      beg = src.getch
+      short_hand = false
+      # HACK: stupid rubinius
+      # c, beg, short_hand = src.matched, src.getch, false
+    else                               # Short-hand (e.g. %{, %., %!, etc)
+      c = 'Q'
+      beg = src.getch
+      short_hand = true
+      # HACK: stupid rubinius
+      # c, beg, short_hand = 'Q', src.getch, true
+    end
+
+    if src.eos? or c == RubyLexer::EOF or beg == RubyLexer::EOF then
+      rb_compile_error "unterminated quoted string meets end of file"
+    end
+
+    # Figure nnd-char.  "\0" is special to indicate beg=nnd and that no nesting?
+    nnd = { "(" => ")", "[" => "]", "{" => "}", "<" => ">" }[beg]
+    nnd, beg = beg, "\0" if nnd.nil?
+
+    token_type, self.yacc_value = nil, s("%#{c}#{beg}")
+    token_type, string_type = case c
+                              when 'Q' then
+                                ch = short_hand ? nnd : c + beg
+                                self.yacc_value = s("%#{ch}")
+                                [:tSTRING_BEG,   STR_DQUOTE]
+                              when 'q' then
+                                [:tSTRING_BEG,   STR_SQUOTE]
+                              when 'W' then
+                                src.scan(/\s*/)
+                                [:tWORDS_BEG,    STR_DQUOTE | STR_FUNC_AWORDS]
+                              when 'w' then
+                                src.scan(/\s*/)
+                                [:tAWORDS_BEG,   STR_SQUOTE | STR_FUNC_AWORDS]
+                              when 'x' then
+                                [:tXSTRING_BEG,  STR_XQUOTE]
+                              when 'r' then
+                                [:tREGEXP_BEG,   STR_REGEXP]
+                              when 's' then
+                                self.lex_state  = :expr_fname
+                                [:tSYMBEG,       STR_SSYM]
+                              end
+
+    rb_compile_error "Bad %string type. Expected [Qqwxr\W], found '#{c}'." if
+      token_type.nil?
+
+    self.lex_strterm = s(:strterm, string_type, nnd, beg)
+
+    return token_type
+  end
+
+  def parse_string(quote)
+    _, string_type, term, open = quote
+
+    space = false # FIX: remove these
+    func = string_type
+    paren = open
+    term_re = Regexp.escape term
+
+    awords = (func & STR_FUNC_AWORDS) != 0
+    regexp = (func & STR_FUNC_REGEXP) != 0
+    expand = (func & STR_FUNC_EXPAND) != 0
+
+    unless func then
+      return :tSTRING_END
+    end
+
+    space = true if awords and src.scan(/\s+/)
+
+    if self.nest == 0 && src.scan(/#{term_re}/) then
+      if awords then
+        quote[1] = nil
+        return ' '
+      elsif regexp then
+        self.yacc_value = self.regx_options
+        return :tREGEXP_END
+      else
+        self.yacc_value = s(term)
+        return :tSTRING_END
+      end
+    end
+
+    if space then
+      return ' '
+    end
+
+    self.token_buffer.clear
+
+    if expand
+      case
+      when src.scan(/#(?=[$@])/) then
+        return :tSTRING_DVAR
+      when src.scan(/#[{]/) then
+        return :tSTRING_DBEG
+      when src.scan(/#/) then
+        token_buffer << '#'
+      end
+    end
+
+    if tokadd_string(func, term, paren) == RubyLexer::EOF then
+      rb_compile_error "unterminated string meets end of file"
+    end
+
+    self.yacc_value = s(:str, token_buffer.join)
+    return :tSTRING_CONTENT
+  end
+
+  def rb_compile_error msg
+    msg += ". near line #{src.lineno}: #{src.rest[/^.*/].inspect}"
+    raise SyntaxError, msg
+  end
+
+  def read_escape
+    case
+    when src.scan(/\\/) then                   # Backslash
+      '\\'
+    when src.scan(/n/) then                    # newline
+      "\n"
+    when src.scan(/t/) then                    # horizontal tab
+      "\t"
+    when src.scan(/r/) then                    # carriage-return
+      "\r"
+    when src.scan(/f/) then                    # form-feed
+      "\f"
+    when src.scan(/v/) then                    # vertical tab
+      "\13"
+    when src.scan(/a/) then                    # alarm(bell)
+      "\007"
+    when src.scan(/e/) then                    # escape
+      "\033"
+    when src.scan(/b/) then                    # backspace
+      "\010"
+    when src.scan(/s/) then                    # space
+      " "
+    when src.scan(/[0-7]{1,3}/) then           # octal constant
+      src.matched.to_i(8).chr
+    when src.scan(/x([0-9a-fA-Fa-f]{2})/) then # hex constant
+      src[1].to_i(16).chr
+    when src.scan(/M-\\/) then
+      c = self.read_escape
+      c[0] = (c[0].ord | 0x80).chr
+      c
+    when src.scan(/M-(.)/) then
+      c = src[1]
+      c[0] = (c[0].ord | 0x80).chr
+      c
+    when src.scan(/C-\\|c\\/) then
+      c = self.read_escape
+      c[0] = (c[0].ord & 0x9f).chr
+      c
+    when src.scan(/C-\?|c\?/) then
+      0177.chr
+    when src.scan(/(C-|c)(.)/) then
+      c = src[2]
+      c[0] = (c[0].ord & 0x9f).chr
+      c
+    when src.scan(/[McCx0-9]/) || src.eos? then
+      rb_compile_error("Invalid escape character syntax")
+    else
+      src.getch
+    end
+  end
+
+  def regx_options
+    good, bad = [], []
+
+    if src.scan(/[a-z]+/) then
+      good, bad = src.matched.split(//).partition { |s| s =~ /^[ixmonesu]$/ }
+    end
+
+    unless bad.empty? then
+      rb_compile_error("unknown regexp option%s - %s" %
+                       [(bad.size > 1 ? "s" : ""), bad.join.inspect])
+    end
+
+    return good.join
+  end
+
+  def reset
+    self.command_start = true
+    self.lex_strterm = nil
+    self.token = nil
+    self.yacc_value = nil
+
+    @src = nil
+    @lex_state = nil
+  end
+
+  def src= src
+    raise "bad src: #{src.inspect}" unless String === src
+    @src = StringScanner.new src
+  end
+
+  def store_comment
+    @comments.push(*self.token_buffer)
+    self.token_buffer.clear
+  end
+
+  def tokadd_escape term
+    case
+    when src.scan(/\\\n/) then
+      # just ignore
+    when src.scan(/\\([0-7]{1,3}|x[0-9a-fA-F]{1,2})/) then
+      self.token_buffer << src.matched
+    when src.scan(/\\([MC]-|c)(?=\\)/) then
+      self.token_buffer << src.matched
+      self.tokadd_escape term
+    when src.scan(/\\([MC]-|c)(.)/) then
+      self.token_buffer << src.matched
+    when src.scan(/\\[McCx]/) then
+      rb_compile_error "Invalid escape character syntax"
+    when src.scan(/\\(.)/m) then
+      self.token_buffer << src.matched
+    else
+      rb_compile_error "Invalid escape character syntax"
+    end
+  end
+
+  def tokadd_string(func, term, paren)
+    awords = (func & STR_FUNC_AWORDS) != 0
+    escape = (func & STR_FUNC_ESCAPE) != 0
+    expand = (func & STR_FUNC_EXPAND) != 0
+    regexp = (func & STR_FUNC_REGEXP) != 0
+    symbol = (func & STR_FUNC_SYMBOL) != 0
+
+    paren_re = paren.nil? ? nil : Regexp.new(Regexp.escape(paren))
+    term_re  = Regexp.new(Regexp.escape(term))
+
+    until src.eos? do
+      c = nil
+      case
+      when paren_re && src.scan(paren_re) then
+        self.nest += 1
+      when self.nest == 0 && src.scan(term_re) then
+        src.pos -= 1
+        break
+      when src.scan(term_re) then
+        self.nest -= 1
+      when ((awords && src.scan(/\s/)) ||
+            (expand && src.scan(/#(?=[\$\@\{])/))) then
+        src.pos -= 1
+        break
+      when awords && src.scan(/\\\n/) then
+        token_buffer << "\n"
+        next
+      when expand && src.scan(/\\\n/) then
+        next
+      when awords && src.scan(/\\\s/) then
+        c = ' '
+      when (expand && src.scan(/#(?!\n)/)) || src.scan(/\\\n/) then
+        # do nothing
+      when src.scan(/\\\\/) then
+        if escape then
+          token_buffer << '\\'
+        end
+        c = '\\'
+      when regexp && src.check(/\\/) then
+        self.tokadd_escape term
+        next
+      when expand && src.scan(/\\/) then
+        c = self.read_escape
+      when src.scan(/\\/) then
+        unless src.scan(term_re) || paren.nil? || src.scan(paren_re) then
+          token_buffer << "\\"
+        end
+        # \\ case:
+        # else if (ismbchar(c)) {
+        #   int i, len = mbclen(c)-1;
+        #   for (i = 0; i < len; i++) {
+        #     tokadd(c);
+        #     c = nextc();
+        #   }
+        # }
+      else
+        c = src.getch # FIX: I don't like this style
+        if symbol && src.scan(/\0/) then
+          rb_compile_error "symbol cannot contain '\\0'"
+        end
+      end
+
+      c = src.matched unless c
+      token_buffer << c
+    end # until
+
+    c = src.matched unless c
+    c = RubyLexer::EOF if src.eos?
+
+    return c
+  end
+
+  def warning s
+    # do nothing for now
+  end
+
+  ##
+  # Returns the next token. Also sets yy_val is needed.
+  #
+  # @return Description of the Returned Value
+  # TODO: remove ALL sexps coming from here and move up to grammar
+  # TODO: only literal values should come up from the lexer.
+
+  def yylex
+    c = ''
+    space_seen = false
+    command_state = false
+    token_buffer.clear
+    src = self.src
+
+    self.token = nil
+    self.yacc_value = nil
+
+    if lex_strterm then
+      token = nil
+
+      if lex_strterm[0] == :heredoc then
+        token = self.heredoc(lex_strterm)
+        if token == :tSTRING_END then
+          self.lex_strterm = nil
+          self.lex_state = :expr_end
+        end
+      else
+        token = self.parse_string(lex_strterm)
+
+        if token == :tSTRING_END || token == :tREGEXP_END then
+          self.lex_strterm = nil
+          self.lex_state = :expr_end
+        end
+      end
+
+      return token
+    end
+
+    command_state = self.command_start
+    self.command_start = false
+
+    last_state = lex_state
+
+    loop do
+      case
+      when src.scan(/\004|\032|\000/), src.eos? then # ^D, ^Z, EOF
+        return RubyLexer::EOF
+      when src.scan(/\ |\t|\f|\r|\13/) then # white spaces, 13 = '\v
+        space_seen = true
+        next
+      when src.scan(/#|\n/) then
+        c = src.matched
+        if c == '#' then
+          src.unread c # ok
+
+          while src.scan(/\s*#.*(\n+|\z)/) do
+            token_buffer << src.matched.gsub(/^ +#/, '#').gsub(/^ +$/, '')
+          end
+
+          self.store_comment
+
+          if src.eos? then
+            return RubyLexer::EOF
+          end
+        end
+        # Replace a string of newlines with a single one
+
+        src.scan(/\n+/)
+
+        if [:expr_beg, :expr_fname,
+            :expr_dot, :expr_class].include? lex_state then
+          next
+        end
+
+        self.command_start = true
+        self.lex_state = :expr_beg
+        return "\n"
+      when src.scan(/\*\*=/) then
+        self.lex_state = :expr_beg
+        self.yacc_value = s("**")
+        return :tOP_ASGN
+      when src.scan(/\*\*/) then
+        self.yacc_value = s("**")
+        self.fix_arg_lex_state
+        return :tPOW
+      when src.scan(/\*\=/) then
+        self.lex_state = :expr_beg
+        self.yacc_value = s("*")
+        return :tOP_ASGN
+      when src.scan(/\*/) then
+        result = if lex_state.is_argument && space_seen && !src.check(/\s/) then
+                   warning("`*' interpreted as argument prefix")
+                   :tSTAR
+                 elsif lex_state == :expr_beg || lex_state == :expr_mid then
+                   :tSTAR
+                 else
+                   :tSTAR2
+                 end
+
+        self.yacc_value = s("*")
+
+        self.fix_arg_lex_state
+
+        return result
+      when src.scan(/\!\=/) then
+        self.lex_state = :expr_beg
+        self.yacc_value = s("!=")
+        return :tNEQ
+      when src.scan(/\!~/) then
+        self.lex_state = :expr_beg
+        self.yacc_value = s("!~")
+        return :tNMATCH
+      when src.scan(/\!/) then
+        self.lex_state = :expr_beg
+        self.yacc_value = s("!")
+        return :tBANG
+      when src.scan(/\=\=\=/) then
+        self.fix_arg_lex_state
+        self.yacc_value = s("===")
+        return :tEQQ
+      when src.scan(/\=\=/) then
+        self.fix_arg_lex_state
+        self.yacc_value = s("==")
+        return :tEQ
+      when src.scan(/\=~/) then
+        self.fix_arg_lex_state
+        self.yacc_value = s("=~")
+        return :tMATCH
+      when src.scan(/\=>/) then
+        self.fix_arg_lex_state
+        self.yacc_value = s("=>")
+        return :tASSOC
+      when src.scan(/\=/) then
+        # documentation nodes
+        if src.was_begin_of_line and src.scan(/begin(?=\s)/) then
+          self.token_buffer << '=' # FIX merge up
+          self.token_buffer << src.matched
+
+          unless src.scan(/.*?\n=end(\n|\z)/m) then
+            rb_compile_error("embedded document meets end of file")
+          end
+
+          self.token_buffer << src.matched
+          self.store_comment
+
+          next
+        else
+          self.fix_arg_lex_state
+          self.yacc_value = s("=")
+          return '='
+        end
+      when src.scan(/\<\=\>/) then
+        self.fix_arg_lex_state
+        self.yacc_value = s("<=>")
+        return :tCMP
+      when src.scan(/\<\=/) then
+        self.fix_arg_lex_state
+        self.yacc_value = s("<=")
+        return :tLEQ
+      when src.scan(/\<\<\=/) then
+        self.fix_arg_lex_state
+        self.lex_state = :expr_beg
+        self.yacc_value = s("\<\<")
+        return :tOP_ASGN
+      when src.scan(/\<\</) then
+        if (! [:expr_end,    :expr_dot,
+               :expr_endarg, :expr_class].include?(lex_state) &&
+            (!lex_state.is_argument || space_seen)) then
+          tok = self.heredoc_identifier
+          if tok then
+            return tok
+          end
+        end
+
+        self.fix_arg_lex_state
+        self.yacc_value = s("\<\<")
+        return :tLSHFT
+      when src.scan(/\</) then
+        self.fix_arg_lex_state
+        self.yacc_value = s("<")
+        return :tLT
+      when src.scan(/\>\=/) then
+        self.fix_arg_lex_state
+        self.yacc_value = s(">=")
+        return :tGEQ
+      when src.scan(/\>\>=/) then
+        self.fix_arg_lex_state
+        self.lex_state = :expr_beg
+        self.yacc_value = s(">>")
+        return :tOP_ASGN
+      when src.scan(/\>\>/) then
+        self.fix_arg_lex_state
+        self.yacc_value = s(">>")
+        return :tRSHFT
+      when src.scan(/\>/) then
+        self.fix_arg_lex_state
+        self.yacc_value = s(">")
+        return :tGT
+      when src.scan(/\"/) then
+        self.lex_strterm = s(:strterm, STR_DQUOTE, '"', "\0") # TODO: question this
+        self.yacc_value = s("\"")
+        return :tSTRING_BEG
+      when src.scan(/\`/) then
+        self.yacc_value = s("`")
+        case lex_state
+        when :expr_fname then
+          self.lex_state = :expr_end
+          return :tBACK_REF2
+        when :expr_dot then
+          self.lex_state = if command_state then
+                             :expr_cmdarg
+                           else
+                             :expr_arg
+                           end
+          return :tBACK_REF2
+        end
+        self.lex_strterm = s(:strterm, STR_XQUOTE, '`', "\0")
+        return :tXSTRING_BEG
+      when src.scan(/\'/) then
+        self.lex_strterm = s(:strterm, STR_SQUOTE, "\'", "\0")
+        self.yacc_value = s("'")
+        return :tSTRING_BEG
+      when src.scan(/\?/) then
+        if lex_state == :expr_end || lex_state == :expr_endarg then
+          self.lex_state = :expr_beg
+          self.yacc_value = s("?")
+          return '?'
+        end
+
+        if src.eos? then
+          rb_compile_error "incomplete character syntax"
+        end
+
+        if src.check(/\s|\v/) then
+          unless lex_state.is_argument then
+            c2 = case src.matched
+                 when " " then
+                   's'
+                 when "\n" then
+                   'n'
+                 when "\t" then
+                   't'
+                 when "\v" then
+                   'v'
+                 when "\r" then
+                   'r'
+                 when "\f" then
+                   'f'
+                 end
+
+            if c2 then
+              warning("invalid character syntax; use ?\\" + c2)
+            end
+          end
+
+          # ternary
+          self.lex_state = :expr_beg
+          self.yacc_value = s("?")
+          return '?'
+          # elsif ismbchar(c) then # ternary, also
+          # rb_warn "multibyte character literal not supported yet; use ?\\#{c}"
+          # support.unread c
+          # self.lex_state = :expr_beg
+          # return '?'
+        elsif src.check(/\w(?=\w)/) then # ternary, also
+          self.lex_state = :expr_beg
+          self.yacc_value = s("?")
+          return '?'
+        end
+
+        c = if src.scan(/\\/) then
+              self.read_escape
+            else
+              src.getch
+            end
+        c[0] = (c[0].ord & 0xff).chr
+        self.lex_state = :expr_end
+        self.yacc_value = c[0].ord
+        return :tINTEGER
+      when src.scan(/\&&=/) then
+        self.yacc_value = s("&&")
+        self.lex_state = :expr_beg
+        return :tOP_ASGN
+      when src.scan(/\&&/) then
+        self.lex_state = :expr_beg
+        self.yacc_value = s("&&")
+        return :tANDOP
+      when src.scan(/\&\=/) then
+        self.yacc_value = s("&")
+        self.lex_state = :expr_beg
+        return :tOP_ASGN
+      when src.scan(/&/) then
+        result = if lex_state.is_argument && space_seen && !src.check(/\s/) then
+                   warning("`&' interpreted as argument prefix")
+                   :tAMPER
+                 elsif lex_state == :expr_beg || lex_state == :expr_mid then
+                   :tAMPER
+                 else
+                   :tAMPER2
+                 end
+
+        self.fix_arg_lex_state
+        self.yacc_value = s("&")
+        return result
+      when src.scan(/\|\|\=/) then
+        self.lex_state = :expr_beg
+        self.yacc_value = s("||")
+        return :tOP_ASGN
+      when src.scan(/\|\|/) then
+        self.lex_state = :expr_beg
+        self.yacc_value = s("||")
+        return :tOROP
+      when src.scan(/\|\=/) then
+        self.lex_state = :expr_beg
+        self.yacc_value = s("|")
+        return :tOP_ASGN
+      when src.scan(/\|/) then
+        self.fix_arg_lex_state
+        self.yacc_value = s("|")
+        return :tPIPE
+      when src.scan(/[+-]/) then
+        sign = src.matched
+        utype, type = if sign == "+" then
+                        [:tUPLUS, :tPLUS]
+                      else
+                        [:tUMINUS, :tMINUS]
+                      end
+
+        if lex_state == :expr_fname || lex_state == :expr_dot then
+          self.lex_state = :expr_arg
+          if src.scan(/@/) then
+            self.yacc_value = s("#{sign}@")
+            return utype
+          else
+            self.yacc_value = s(sign)
+            return type
+          end
+        end
+
+        if src.scan(/\=/) then
+          self.lex_state = :expr_beg
+          self.yacc_value = s(sign)
+          return :tOP_ASGN
+        end
+
+        if (lex_state == :expr_beg || lex_state == :expr_mid ||
+            (lex_state.is_argument && space_seen && !src.check(/\s/))) then
+          if lex_state.is_argument then
+            arg_ambiguous
+          end
+
+          self.lex_state = :expr_beg
+          self.yacc_value = s(sign)
+
+          if src.check(/\d/) then
+            if utype == :tUPLUS then
+              return self.parse_number
+            else
+              return :tUMINUS_NUM
+            end
+          end
+
+          return utype
+        end
+
+        self.lex_state = :expr_beg
+        self.yacc_value = s(sign)
+        return type
+      when src.scan(/\.\.\./) then
+        self.lex_state = :expr_beg
+        self.yacc_value = s("...")
+        return :tDOT3
+      when src.scan(/\.\./) then
+        self.lex_state = :expr_beg
+        self.yacc_value = s("..")
+        return :tDOT2
+      when src.scan(/\.\d/) then
+        rb_compile_error "no .<digit> floating literal anymore put 0 before dot"
+      when src.scan(/\./) then
+        self.lex_state = :expr_dot
+        self.yacc_value = s(".")
+        return :tDOT
+      when src.check(/[0-9]/) then
+        return parse_number
+      when src.scan(/[\)\]\}]/) then
+        cond.lexpop
+        cmdarg.lexpop
+        self.lex_state = :expr_end
+        self.yacc_value = s(src.matched)
+        result = {
+          ")" => :tRPAREN,
+          "]" => :tRBRACK,
+          "}" => :tRCURLY
+        }[src.matched]
+        return result
+      when src.scan(/::/) then
+        if (lex_state == :expr_beg ||
+            lex_state == :expr_mid ||
+            lex_state == :expr_class ||
+            (lex_state.is_argument && space_seen)) then
+          self.lex_state = :expr_beg
+          self.yacc_value = s("::")
+          return :tCOLON3
+        end
+
+        self.lex_state = :expr_dot
+        self.yacc_value = s("::")
+        return :tCOLON2
+      when src.scan(/\:/) then
+        if (lex_state == :expr_end || lex_state == :expr_endarg ||
+            src.check(/\s/)) then
+          self.lex_state = :expr_beg
+          self.yacc_value = s(":")
+          return ':'
+        end
+
+        case
+        when src.scan(/\'/) then
+          self.lex_strterm = s(:strterm, STR_SSYM, src.matched, "\0")
+        when src.scan(/\"/) then
+          self.lex_strterm = s(:strterm, STR_DSYM, src.matched, "\0")
+        end
+
+        self.lex_state = :expr_fname
+        self.yacc_value = s(":")
+        return :tSYMBEG
+      when src.scan(/\//) then
+        if lex_state == :expr_beg || lex_state == :expr_mid then
+          self.lex_strterm = s(:strterm, STR_REGEXP, '/', "\0")
+          self.yacc_value = s("/")
+          return :tREGEXP_BEG
+        end
+
+        if src.scan(/\=/) then
+          self.yacc_value = s("/")
+          self.lex_state = :expr_beg
+          return :tOP_ASGN
+        end
+
+        if lex_state.is_argument && space_seen then
+          unless src.scan(/\s/) then
+            arg_ambiguous
+            self.lex_strterm = s(:strterm, STR_REGEXP, '/', "\0")
+            self.yacc_value = s("/")
+            return :tREGEXP_BEG
+          end
+        end
+
+        self.fix_arg_lex_state
+
+        self.yacc_value = s("/")
+        return :tDIVIDE
+      when src.scan(/\^=/) then
+        self.lex_state = :expr_beg
+        self.yacc_value = s("^")
+        return :tOP_ASGN
+      when src.scan(/\^/) then
+        self.fix_arg_lex_state
+        self.yacc_value = s("^")
+        return :tCARET
+      when src.scan(/\;/) then
+        self.command_start = true
+        self.lex_state = :expr_beg
+        self.yacc_value = s(";")
+        return src.matched
+      when src.scan(/\,/) then
+        self.lex_state = :expr_beg
+        self.yacc_value = s(",")
+        return src.matched
+      when src.scan(/\~/) then
+        if lex_state == :expr_fname || lex_state == :expr_dot then
+          src.scan(/@/)
+        end
+
+        self.fix_arg_lex_state
+        self.yacc_value = s("~")
+
+        return :tTILDE
+      when src.scan(/\(/) then
+        result = :tLPAREN2
+        self.command_start = true
+        if lex_state == :expr_beg || lex_state == :expr_mid then
+          result = :tLPAREN
+        elsif space_seen then
+          if lex_state == :expr_cmdarg then
+            result = :tLPAREN_ARG
+          elsif lex_state == :expr_arg then
+            warning("don't put space before argument parentheses")
+            result = :tLPAREN2
+          end
+        end
+
+        self.expr_beg_push "("
+
+        return result
+      when src.scan(/\[/) then
+        result = src.matched
+        if lex_state == :expr_fname || lex_state == :expr_dot then
+          self.lex_state = :expr_arg
+          case
+          when src.scan(/\]\=/) then
+            self.yacc_value = s("[]=")
+            return :tASET
+          when src.scan(/\]/) then
+            self.yacc_value = s("[]")
+            return :tAREF
+          else
+            rb_compile_error "unexpected '['"
+          end
+        elsif lex_state == :expr_beg || lex_state == :expr_mid then
+          result = :tLBRACK
+        elsif lex_state.is_argument && space_seen then
+          result = :tLBRACK
+        end
+
+        self.expr_beg_push("[")
+
+        return result
+      when src.scan(/\{/) then
+        result = if lex_state.is_argument || lex_state == :expr_end then
+                   :tLCURLY      #  block (primary)
+                 elsif lex_state == :expr_endarg then
+                   :tLBRACE_ARG  #  block (expr)
+                 else
+                   :tLBRACE      #  hash
+                 end
+
+        self.expr_beg_push("{")
+
+        return result
+      when src.scan(/\\/) then
+        if src.scan(/\n/) then
+          space_seen = true
+          next
+        end
+        rb_compile_error "bare backslash only allowed before newline"
+      when src.scan(/\%/) then
+        if lex_state == :expr_beg || lex_state == :expr_mid then
+          return parse_quote
+        end
+
+        if src.scan(/\=/) then
+          self.lex_state = :expr_beg
+          self.yacc_value = s("%")
+          return :tOP_ASGN
+        end
+
+        if lex_state.is_argument && space_seen && ! src.check(/\s/) then
+          return parse_quote
+        end
+
+        self.fix_arg_lex_state
+        self.yacc_value = s("%")
+
+        return :tPERCENT
+      when src.scan(/(\$_)(\w)/) then
+        self.lex_state = :expr_end
+        token_buffer << src[1]
+        # HACK? c = src[2]
+        # pass through
+      when src.scan(/\$_/) then
+        self.lex_state = :expr_end
+        token_buffer << src.matched
+        self.yacc_value = s(src.matched)
+        return :tGVAR
+      when src.scan(/\$[~*$?!@\/\\;,.=:<>\"]|\$-\w?/) then
+        self.lex_state = :expr_end
+        token_buffer << src.matched
+        self.yacc_value = s(token_buffer.join)
+        return :tGVAR
+      when src.scan(/\$([\&\`\'\+])/) then
+        self.lex_state = :expr_end
+        # Explicit reference to these vars as symbols...
+        if last_state == :expr_fname then
+          token_buffer << src.matched
+          self.yacc_value = s(token_buffer.join)
+          return :tGVAR
+        else
+          self.yacc_value = s(:back_ref, src[1].to_sym)
+          return :tBACK_REF
+        end
+      when src.scan(/\$[1-9]\d*/) then
+        self.lex_state = :expr_end
+        token_buffer.push(*src.matched.split(//))
+        if last_state == :expr_fname then
+          self.yacc_value = s(token_buffer.join)
+          return :tGVAR
+        else
+          self.yacc_value = s(:nth_ref, token_buffer.join[1..-1].to_i)
+          return :tNTH_REF
+        end
+      when src.scan(/\$0/) then
+        self.lex_state = :expr_end
+        token_buffer << '$' # why just this?!?
+        # pass through
+      when src.scan(/\$\W|\$\z/) then # TODO: remove?
+        self.lex_state = :expr_end
+        self.yacc_value = s("$")
+        return '$'
+      when src.scan(/\$/)
+        self.lex_state = :expr_end
+        token_buffer << src.matched
+        src.getch
+        # pass through
+      when src.scan(/\@/) then
+        token_buffer << '@'
+
+        if src.scan(/(@)?\d/) then
+          if src[1] then
+            rb_compile_error "`@@#{c}` is not allowed as a class variable name"
+          else
+            rb_compile_error "`@#{c}' is not allowed as an instance variable name"
+          end
+        end
+
+        if src.scan(/@/) then
+          token_buffer << src.matched
+        end
+
+        unless src.scan(/\w/) then
+          self.yacc_value = s("@")
+          return '@'
+        end
+      when src.scan(/\_/) then
+        if src.was_begin_of_line && src.scan(/_END__(\n|\Z)/) then
+          return RubyLexer::EOF
+        end
+      else
+        c = src.getch # FIX: I really hate this
+        if c =~ /\W/ then
+          rb_compile_error "Invalid char '#{c.inspect}' in expression"
+        end
+      end
+
+      src.pos -= 1 # HACK
+      if src.scan(/\w+/) then
+        token_buffer.push(*src.matched.split(//)) # TODO: that split is tarded.
+      end
+
+      if token_buffer[0] =~ /\w/ && src.scan(/[\!\?](?!=)/) then
+        token_buffer << src.matched
+      end
+
+      result = nil
+      last_state = lex_state
+
+      case token_buffer[0]
+      when /^\$/ then
+        self.lex_state = :expr_end
+        result = :tGVAR
+      when '@' then
+        self.lex_state = :expr_end
+        if token_buffer[1] == '@' then
+          result = :tCVAR
+        else
+          result = :tIVAR
+        end
+      else
+        if token_buffer[-1] =~ /[!?]/ then
+          result = :tFID
+        else
+          if lex_state == :expr_fname then
+            # ident=, not =~ => == or followed by =>
+            if src.scan(/=(?:(?![~>=])|(?==>))/) then
+              result = :tIDENTIFIER
+              token_buffer << src.matched
+            end
+          end
+
+          if result.nil? && token_buffer[0] =~ /[A-Z]/ then
+            result = :tCONSTANT
+          else
+            result = :tIDENTIFIER
+          end
+        end
+
+        unless lex_state == :expr_dot then
+          # See if it is a reserved word.
+          keyword = Keyword.keyword(token_buffer.join, token_buffer.length)
+
+          unless keyword.nil? then
+            state = lex_state
+            self.lex_state = keyword.state
+
+            self.yacc_value = if state == :expr_fname then
+                                s(keyword.name)
+                              else
+                                s(token_buffer.join)
+                              end
+
+            if keyword.id0 == :kDO then
+              self.command_start = true
+              if cond.is_in_state then
+                return :kDO_COND
+              end
+              if cmdarg.is_in_state && state != :expr_cmdarg then
+                return :kDO_BLOCK
+              end
+              if state == :expr_endarg then
+                return :kDO_BLOCK
+              end
+              return :kDO
+            end
+
+            if state == :expr_beg then
+              return keyword.id0
+            end
+
+            if keyword.id0 != keyword.id1 then
+              self.lex_state = :expr_beg
+            end
+
+            return keyword.id1
+          end
+        end # lex_state == :expr_dot
+
+        if (lex_state == :expr_beg ||
+            lex_state == :expr_mid ||
+            lex_state == :expr_dot ||
+            lex_state == :expr_arg ||
+            lex_state == :expr_cmdarg) then
+          if command_state then
+            self.lex_state = :expr_cmdarg
+          else
+            self.lex_state = :expr_arg
+          end
+        else
+          self.lex_state = :expr_end
+        end
+      end
+
+      # Lame: parsing logic made it into lexer in ruby...So we
+      # are emulating
+      # FIXME:  I believe this is much simpler now...
+      # HACK
+      # if (IdUtil.var_type(temp_val) == IdUtil.LOCAL_VAR &&
+      #     last_state != :expr_dot &&
+      #     (BlockStaticScope === scope && (scope.is_defined(temp_val) >= 0)) ||
+      #     (scope.local_scope.is_defined(temp_val) >= 0)) then
+      #   self.lex_state = :expr_end
+      # end
+
+      self.yacc_value = s(token_buffer.join)
+
+      return result
+    end
+  end
+end