ruby_parser 3.2.2 → 3.3.0

data/lib/ruby20_parser.y

@@ -263,7 +263,7 @@ rule
                     }
                     opt_block_param
                     {
-                      result = self.env.dynamic.keys
+                      result = nil # self.env.dynamic.keys
                     }
                     compstmt tRCURLY
                     {
@@ -785,7 +785,6 @@ rule
                     }
                 | arg tEH arg opt_nl tCOLON arg
                     {
-                      lexer.tern.pop
                       result = s(:if, val[0], val[2], val[5])
                     }
                 | primary
@@ -1136,7 +1135,7 @@ rule
                     }
                 | kDEF fname
                     {
-                      result = [lexer.lineno, self.in_def]
+                      result = self.in_def
 
                       self.comments.push self.lexer.comments
                       self.in_def = true
@@ -1144,10 +1143,9 @@ rule
                     }
                     f_arglist bodystmt kEND
                     {
-                      line, in_def = val[2]
+                      in_def = val[2]
 
                       result = new_defn val
-                      result[2].line line
 
                       self.env.unextend
                       self.in_def = in_def
@@ -1429,7 +1427,9 @@ opt_block_args_tail: tCOMMA block_args_tail
                 | f_bad_arg
 
           lambda:   {
-                      # TODO: dyna_push ? hrm
+                      self.env.extend :dynamic
+                      result = self.lexer.lineno
+
                       result = lexer.lpar_beg
                       lexer.paren_nest += 1
                       lexer.lpar_beg = lexer.paren_nest
@@ -1443,6 +1443,7 @@ opt_block_args_tail: tCOMMA block_args_tail
 
                       call = new_call nil, :lambda
                       result = new_iter call, args, body
+                      self.env.unextend
                     }
 
      f_larglist: tLPAREN2 f_args opt_bv_decl rparen
@@ -1470,7 +1471,7 @@ opt_block_args_tail: tCOMMA block_args_tail
                     }
                     opt_block_param
                     {
-                      result = self.env.dynamic.keys
+                      result = nil # self.env.dynamic.keys
                     }
                     compstmt kEND
                     {
@@ -1556,7 +1557,7 @@ opt_block_args_tail: tCOMMA block_args_tail
                     }
                     opt_block_param
                     {
-                      result = self.env.dynamic.keys
+                      result = nil # self.env.dynamic.keys
                     }
                     compstmt tRCURLY
                     {
@@ -1574,7 +1575,7 @@ opt_block_args_tail: tCOMMA block_args_tail
                     }
                  opt_block_param
                     {
-                      result = self.env.dynamic.keys
+                      result = nil # self.env.dynamic.keys
                     }
                     compstmt kEND
                     {
@@ -1972,13 +1973,14 @@ keyword_variable: kNIL      { result = s(:nil)   }
        f_arglist: tLPAREN2 f_args rparen
                     {
                       result = val[1]
-                      lexer.lex_state = :expr_beg
+                      self.lexer.lex_state = :expr_beg
                       self.lexer.command_start = true
                     }
                 | f_args term
                     {
-                      self.lexer.lex_state = :expr_beg
                       result = val[0]
+                      self.lexer.lex_state = :expr_beg
+                      self.lexer.command_start = true
                     }
 
        args_tail: f_kwarg tCOMMA f_kwrest opt_f_block_arg
@@ -2127,14 +2129,20 @@ keyword_variable: kNIL      { result = s(:nil)   }
                     {
                       # TODO: call_args
                       label, _ = val[0] # TODO: fix lineno?
-                      result = s(:array, s(:kwarg, label.to_sym, val[1]))
+                      identifier = label.to_sym
+                      self.env[identifier] = :lvar
+
+                      result = s(:array, s(:kwarg, identifier, val[1]))
                     }
 
       f_block_kw: tLABEL primary_value
                     {
                       # TODO: call_args
                       label, _ = val[0] # TODO: fix lineno?
-                      result = s(:array, s(:kwarg, label.to_sym, val[1]))
+                      identifier = label.to_sym
+                      self.env[identifier] = :lvar
+
+                      result = s(:array, s(:kwarg, identifier, val[1]))
                     }
 
    f_block_kwarg: f_block_kw

data/lib/ruby_lexer.rb

@@ -5,61 +5,21 @@ class RubyLexer
   # :stopdoc:
   RUBY19 = "".respond_to? :encoding
 
-  IDENT_CHAR_RE = if RUBY19 then
-                    /[\w\u0080-\u{10ffff}]/u
-                  else
-                    /[\w\x80-\xFF]/n
-                  end
-
-  IDENT_RE = /^#{IDENT_CHAR_RE}+/o
-
-  attr_accessor :command_start
-  attr_accessor :cmdarg
-  attr_accessor :cond
-  attr_accessor :tern # TODO: rename ternary damnit... wtf
-  attr_accessor :string_nest
-
-  ESC_RE = /\\((?>[0-7]{1,3}|x[0-9a-fA-F]{1,2}|M-[^\\]|(C-|c)[^\\]|[^0-7xMCc]))/u
-  # :startdoc:
-
-  ##
-  # What version of ruby to parse. 18 and 19 are the only valid values
-  # currently supported.
-
-  attr_accessor :version
-
-  # Additional context surrounding tokens that both the lexer and
-  # grammar use.
-  attr_reader :lex_state
-
-  attr_accessor :lex_strterm
-
-  attr_accessor :parser # HACK for very end of lexer... *sigh*
-
-  # Stream of data that yylex examines.
-  attr_reader :src
-
-  # Last token read via yylex.
-  attr_accessor :token
-
-  attr_accessor :string_buffer
-
-  # Value of last token which had a value associated with it.
-  attr_accessor :yacc_value
-
-  # What handles warnings
-  attr_accessor :warnings
+  IDENT_CHAR = if RUBY19 then
+                 /[\w\u0080-\u{10ffff}]/u
+               else
+                 /[\w\x80-\xFF]/n
+               end
 
-  attr_accessor :space_seen
-  attr_accessor :paren_nest
-  attr_accessor :brace_nest
-  attr_accessor :lpar_beg
+  IDENT = /^#{IDENT_CHAR}+/o
+  ESC = /\\((?>[0-7]{1,3}|x[0-9a-fA-F]{1,2}|M-[^\\]|(C-|c)[^\\]|u[0-9a-fA-F]+|u\{[0-9a-fA-F]+\}|[^0-7xMCc]))/u
+  SIMPLE_STRING = /(#{ESC}|#(#{ESC}|[^\{\#\@\$\"\\])|[^\"\\\#])*/o
+  SIMPLE_SSTRING = /(\\.|[^\'])*/
 
   EOF = :eof_haha!
 
   # ruby constants for strings (should this be moved somewhere else?)
 
-  # :stopdoc:
   STR_FUNC_BORING = 0x00
   STR_FUNC_ESCAPE = 0x01 # TODO: remove and replace with REGEXP
   STR_FUNC_EXPAND = 0x02
@@ -75,6 +35,22 @@ class RubyLexer
   STR_SSYM   = STR_FUNC_SYMBOL
   STR_DSYM   = STR_FUNC_SYMBOL | STR_FUNC_EXPAND
 
+  ESCAPES = {
+    "a"    => "\007",
+    "b"    => "\010",
+    "e"    => "\033",
+    "f"    => "\f",
+    "n"    => "\n",
+    "r"    => "\r",
+    "s"    => " ",
+    "t"    => "\t",
+    "v"    => "\13",
+    "\\"   => '\\',
+    "\n"   => "",
+    "C-\?" => 127.chr,
+    "c\?"  => 127.chr,
+  }
+
   TOKENS = {
     "!"   => :tBANG,
     "!="  => :tNEQ,
@@ -90,8 +66,58 @@ class RubyLexer
     "=~"  => :tMATCH,
     "->"  => :tLAMBDA,
   }
+
+  @@regexp_cache = Hash.new { |h,k| h[k] = Regexp.new(Regexp.escape(k)) }
+  @@regexp_cache[nil] = nil
+
   # :startdoc:
 
+  attr_accessor :brace_nest
+  attr_accessor :cmdarg
+  attr_accessor :command_start
+  attr_accessor :cond
+
+  ##
+  # Additional context surrounding tokens that both the lexer and
+  # grammar use.
+
+  attr_accessor :lex_state
+
+  attr_accessor :lex_strterm
+  attr_accessor :lpar_beg
+  attr_accessor :paren_nest
+  attr_accessor :parser # HACK for very end of lexer... *sigh*
+  attr_accessor :space_seen
+  attr_accessor :string_buffer
+  attr_accessor :string_nest
+
+  # Stream of data that yylex examines.
+  attr_reader :src
+  alias :ss :src
+
+  # Last token read via yylex.
+  attr_accessor :token
+
+  ##
+  # What version of ruby to parse. 18 and 19 are the only valid values
+  # currently supported.
+
+  attr_accessor :version
+
+  # Value of last token which had a value associated with it.
+  attr_accessor :yacc_value
+
+  attr_writer :lineno # reader is lazy initalizer
+
+  attr_writer :comments
+
+  def initialize v = 18
+    self.version = v
+
+    reset
+  end
+
+  ##
   # How the parser advances to the next token.
   #
   # @return true if not at end of file (EOF).
@@ -100,7 +126,7 @@ class RubyLexer
     r = yylex
     self.token = r
 
-    raise "yylex returned nil" unless r
+    raise "yylex returned nil, near #{ss.rest[0,10].inspect}" unless r
 
     return RubyLexer::EOF != r
   end
@@ -109,28 +135,35 @@ class RubyLexer
     self.warning("Ambiguous first argument. make sure.")
   end
 
-  def comments
+  def arg_state
+    in_arg_state? ? :expr_arg : :expr_beg
+  end
+
+  def beginning_of_line?
+    ss.bol?
+  end
+
+  def check re
+    ss.check re
+  end
+
+  def comments # TODO: remove this... maybe comment_string + attr_accessor
     c = @comments.join
     @comments.clear
     c
   end
 
-  def expr_beg_push val
-    cond.push false
-    cmdarg.push false
-    self.lex_state = :expr_beg
-    self.yacc_value = val
+  def end_of_stream?
+    ss.eos?
   end
 
-  def fix_arg_lex_state
-    self.lex_state = if in_lex_state? :expr_fname, :expr_dot then
-                       :expr_arg
-                     else
-                       :expr_beg
-                     end
+  def expr_result token, text
+    cond.push false
+    cmdarg.push false
+    result :expr_beg, token, text
   end
 
-  def heredoc here # 63 lines
+  def heredoc here # TODO: rewrite / remove
     _, eos, func, last_line = here
 
     indent  = (func & STR_FUNC_INDENT) != 0 ? "[ \t]*" : nil
@@ -138,11 +171,10 @@ class RubyLexer
     eos_re  = /#{indent}#{Regexp.escape eos}(\r*\n|\z)/
     err_msg = "can't match #{eos_re.inspect} anywhere in "
 
-    rb_compile_error err_msg if
-      src.eos?
+    rb_compile_error err_msg if end_of_stream?
 
-    if src.beginning_of_line? && src.scan(eos_re) then
-      src.unread_many last_line # TODO: figure out how to remove this
+    if beginning_of_line? && scan(eos_re) then
+      ss.unread_many last_line # TODO: figure out how to remove this
       self.yacc_value = eos
       return :tSTRING_END
     end
@@ -151,14 +183,14 @@ class RubyLexer
 
     if expand then
       case
-      when src.scan(/#[$@]/) then
-        src.pos -= 1 # FIX omg stupid
-        self.yacc_value = src.matched
+      when scan(/#[$@]/) then
+        ss.pos -= 1 # FIX omg stupid
+        self.yacc_value = matched
         return :tSTRING_DVAR
-      when src.scan(/#[{]/) then
-        self.yacc_value = src.matched
+      when scan(/#[{]/) then
+        self.yacc_value = matched
         return :tSTRING_DBEG
-      when src.scan(/#/) then
+      when scan(/#/) then
         string_buffer << '#'
       end
 
@@ -172,34 +204,32 @@ class RubyLexer
           self.yacc_value = string_buffer.join.delete("\r")
           return :tSTRING_CONTENT
         else
-          string_buffer << src.scan(/\n/)
+          string_buffer << scan(/\n/)
         end
 
-        rb_compile_error err_msg if
-          src.eos?
-      end until src.check(eos_re)
+        rb_compile_error err_msg if end_of_stream?
+      end until check(eos_re)
     else
-      until src.check(eos_re) do
-        string_buffer << src.scan(/.*(\n|\z)/)
-        rb_compile_error err_msg if
-          src.eos?
+      until check(eos_re) do
+        string_buffer << scan(/.*(\n|\z)/)
+        rb_compile_error err_msg if end_of_stream?
       end
     end
 
     self.lex_strterm = [:heredoc, eos, func, last_line]
-    self.yacc_value = string_buffer.join.delete("\r")
 
+    self.yacc_value = string_buffer.join.delete("\r")
     return :tSTRING_CONTENT
   end
 
-  def heredoc_identifier # 51 lines
+  def heredoc_identifier # TODO: remove / rewrite
     term, func = nil, STR_FUNC_BORING
     self.string_buffer = []
 
     case
-    when src.scan(/(-?)([\'\"\`])(.*?)\2/) then
-      term = src[2]
-      func |= STR_FUNC_INDENT unless src[1].empty?
+    when scan(/(-?)([\'\"\`])(.*?)\2/) then
+      term = ss[2]
+      func |= STR_FUNC_INDENT unless ss[1].empty?
       func |= case term
               when "\'" then
                 STR_SQUOTE
@@ -208,24 +238,24 @@ class RubyLexer
               else
                 STR_XQUOTE
               end
-      string_buffer << src[3]
-    when src.scan(/-?([\'\"\`])(?!\1*\Z)/) then
+      string_buffer << ss[3]
+    when scan(/-?([\'\"\`])(?!\1*\Z)/) then
       rb_compile_error "unterminated here document identifier"
-    when src.scan(/(-?)(#{IDENT_CHAR_RE}+)/) then
+    when scan(/(-?)(#{IDENT_CHAR}+)/) then
       term = '"'
       func |= STR_DQUOTE
-      unless src[1].empty? then
+      unless ss[1].empty? then
         func |= STR_FUNC_INDENT
       end
-      string_buffer << src[2]
+      string_buffer << ss[2]
     else
       return nil
     end
 
-    if src.scan(/.*\n/) then
+    if scan(/.*\n/) then
       # TODO: think about storing off the char range instead
-      line = src.matched
-      src.extra_lines_added += 1
+      line = matched
+      ss.extra_lines_added += 1 # FIX: ugh
     else
       line = nil
     end
@@ -241,41 +271,47 @@ class RubyLexer
     end
   end
 
+  def in_arg_state? # TODO: rename is_after_operator?
+    in_lex_state? :expr_fname, :expr_dot
+  end
+
   def in_lex_state?(*states)
     states.include? lex_state
   end
 
-  def initialize v = 18
-    self.version = v
-    self.cond   = RubyParserStuff::StackState.new(:cond)
-    self.cmdarg = RubyParserStuff::StackState.new(:cmdarg)
-    self.tern   = RubyParserStuff::StackState.new(:tern)
-    self.string_nest = 0
-    self.paren_nest = 0
-    self.brace_nest = 0
-    self.lpar_beg = nil
+  def int_with_base base
+    rb_compile_error "Invalid numeric format" if matched =~ /__/
 
-    @comments = []
+    self.yacc_value = matched.to_i(base)
+    return :tINTEGER
+  end
 
-    reset
+  def is_arg?
+    in_lex_state? :expr_arg, :expr_cmdarg
   end
 
-  def int_with_base base
-    rb_compile_error "Invalid numeric format" if src.matched =~ /__/
+  def is_beg?
+    in_lex_state? :expr_beg, :expr_value, :expr_mid, :expr_class
+  end
 
-    self.yacc_value = src.matched.to_i(base)
-    return :tINTEGER
+  def is_end?
+    in_lex_state? :expr_end, :expr_endarg, :expr_endfn
   end
 
-  def lex_state= o
-    # warn "wtf lex_state = #{o.inspect} from #{caller.first}"
-    raise "wtf\?" unless Symbol === o
-    @lex_state = o
+  def is_label_possible? command_state
+    (in_lex_state?(:expr_beg, :expr_endfn) && !command_state) || is_arg?
+  end
+
+  def is_space_arg? c = "x"
+    is_arg? and space_seen and c !~ /\s/
   end
 
-  attr_writer :lineno
   def lineno
-    @lineno ||= src.lineno
+    @lineno ||= ss.lineno
+  end
+
+  def matched
+    ss.matched
   end
 
   ##
@@ -288,45 +324,45 @@ class RubyLexer
     self.lex_state = :expr_end
 
     case
-    when src.scan(/[+-]?0[xXbBdD]\b/) then
+    when scan(/[+-]?0[xXbBdD]\b/) then
       rb_compile_error "Invalid numeric format"
-    when src.scan(/[+-]?(?:(?:[1-9][\d_]*|0)(?!\.\d)\b|0[Dd][0-9_]+)/) then
+    when scan(/[+-]?(?:(?:[1-9][\d_]*|0)(?!\.\d)\b|0[Dd][0-9_]+)/) then
       int_with_base(10)
-    when src.scan(/[+-]?0x[a-f0-9_]+/i) then
+    when scan(/[+-]?0x[a-f0-9_]+/i) then
       int_with_base(16)
-    when src.scan(/[+-]?0[Bb][01_]+/) then
+    when scan(/[+-]?0[Bb][01_]+/) then
       int_with_base(2)
-    when src.scan(/[+-]?0[Oo]?[0-7_]*[89]/) then
+    when scan(/[+-]?0[Oo]?[0-7_]*[89]/) then
       rb_compile_error "Illegal octal digit."
-    when src.scan(/[+-]?0[Oo]?[0-7_]+|0[Oo]/) then
+    when scan(/[+-]?0[Oo]?[0-7_]+|0[Oo]/) then
       int_with_base(8)
-    when src.scan(/[+-]?[\d_]+_(e|\.)/) then
+    when scan(/[+-]?[\d_]+_(e|\.)/) then
       rb_compile_error "Trailing '_' in number."
-    when src.scan(/[+-]?[\d_]+\.[\d_]+(e[+-]?[\d_]+)?\b|[+-]?[\d_]+e[+-]?[\d_]+\b/i) then
-      number = src.matched
+    when scan(/[+-]?[\d_]+\.[\d_]+(e[+-]?[\d_]+)?\b|[+-]?[\d_]+e[+-]?[\d_]+\b/i) then
+      number = matched
       if number =~ /__/ then
         rb_compile_error "Invalid numeric format"
       end
       self.yacc_value = number.to_f
       :tFLOAT
-    when src.scan(/[+-]?[0-9_]+(?![e])/) then
+    when scan(/[+-]?[0-9_]+(?![e])/) then
       int_with_base(10)
     else
       rb_compile_error "Bad number format"
     end
   end
 
-  def parse_quote # 58 lines
+  def parse_quote # TODO: remove / rewrite
     beg, nnd, short_hand, c = nil, nil, false, nil
 
-    if src.scan(/[a-z0-9]{1,2}/i) then # Long-hand (e.g. %Q{}).
-      rb_compile_error "unknown type of %string" if src.matched_size == 2
-      c, beg, short_hand = src.matched, src.getch, false
+    if scan(/[a-z0-9]{1,2}/i) then # Long-hand (e.g. %Q{}).
+      rb_compile_error "unknown type of %string" if ss.matched_size == 2
+      c, beg, short_hand = matched, ss.getch, false
     else                               # Short-hand (e.g. %{, %., %!, etc)
-      c, beg, short_hand = 'Q', src.getch, true
+      c, beg, short_hand = 'Q', ss.getch, true
     end
 
-    if src.eos? or c == RubyLexer::EOF or beg == RubyLexer::EOF then
+    if end_of_stream? or c == RubyLexer::EOF or beg == RubyLexer::EOF then
       rb_compile_error "unterminated quoted string meets end of file"
     end
 
@@ -334,19 +370,19 @@ class RubyLexer
     nnd = { "(" => ")", "[" => "]", "{" => "}", "<" => ">" }[beg]
     nnd, beg = beg, "\0" if nnd.nil?
 
-    token_type, self.yacc_value = nil, "%#{c}#{beg}"
+    token_type, text = nil, "%#{c}#{beg}"
     token_type, string_type = case c
                               when 'Q' then
                                 ch = short_hand ? nnd : c + beg
-                                self.yacc_value = "%#{ch}"
+                                text = "%#{ch}"
                                 [:tSTRING_BEG,   STR_DQUOTE]
                               when 'q' then
                                 [:tSTRING_BEG,   STR_SQUOTE]
                               when 'W' then
-                                src.scan(/\s*/)
+                                scan(/\s*/)
                                 [:tWORDS_BEG,    STR_DQUOTE | STR_FUNC_QWORDS]
                               when 'w' then
-                                src.scan(/\s*/)
+                                scan(/\s*/)
                                 [:tQWORDS_BEG,   STR_SQUOTE | STR_FUNC_QWORDS]
                               when 'x' then
                                 [:tXSTRING_BEG,  STR_XQUOTE]
@@ -356,20 +392,25 @@ class RubyLexer
                                 self.lex_state  = :expr_fname
                                 [:tSYMBEG,       STR_SSYM]
                               when 'I' then
+                                src.scan(/\s*/)
                                 [:tSYMBOLS_BEG, STR_DQUOTE | STR_FUNC_QWORDS]
                               when 'i' then
+                                src.scan(/\s*/)
                                 [:tQSYMBOLS_BEG, STR_SQUOTE | STR_FUNC_QWORDS]
                               end
 
-    rb_compile_error "Bad %string type. Expected [Qq\Wwxrs], found '#{c}'." if
+    rb_compile_error "Bad %string type. Expected [QqWwIixrs], found '#{c}'." if
       token_type.nil?
 
-    self.lex_strterm = [:strterm, string_type, nnd, beg]
+    raise "huh" unless string_type
+
+    string string_type, nnd, beg
 
+    self.yacc_value = text
     return token_type
   end
 
-  def parse_string(quote) # 65 lines
+  def parse_string quote # TODO: rewrite / remove
     _, string_type, term, open = quote
 
     space = false # FIX: remove these
@@ -381,41 +422,39 @@ class RubyLexer
     regexp = (func & STR_FUNC_REGEXP) != 0
     expand = (func & STR_FUNC_EXPAND) != 0
 
-    unless func then # FIX: impossible, prolly needs == 0
+    unless func then # nil'ed from qwords below. *sigh*
       self.lineno = nil
       return :tSTRING_END
     end
 
-    space = true if qwords and src.scan(/\s+/)
+    space = true if qwords and scan(/\s+/)
 
-    if self.string_nest == 0 && src.scan(/#{term_re}/) then
+    if self.string_nest == 0 && scan(/#{term_re}/) then
       if qwords then
-        quote[1] = nil # TODO: make struct
+        quote[1] = nil
         return :tSPACE
       elsif regexp then
-        self.yacc_value = self.regx_options
         self.lineno = nil
+        self.yacc_value = self.regx_options
         return :tREGEXP_END
       else
-        self.yacc_value = term
         self.lineno = nil
+        self.yacc_value = term
         return :tSTRING_END
       end
     end
 
-    if space then
-      return :tSPACE
-    end
+    return :tSPACE if space
 
     self.string_buffer = []
 
     if expand
       case
-      when src.scan(/#(?=[$@])/) then
+      when scan(/#(?=[$@])/) then
         return :tSTRING_DVAR
-      when src.scan(/#[{]/) then
+      when scan(/#[{]/) then
         return :tSTRING_DBEG
-      when src.scan(/#/) then
+      when scan(/#/) then
         string_buffer << '#'
       end
     end
@@ -429,71 +468,159 @@ class RubyLexer
     return :tSTRING_CONTENT
   end
 
+  def process_token command_state, last_state
+    token = self.token
+    token << matched if scan(/[\!\?](?!=)/)
+
+    tok_id =
+      case
+      when token =~ /[!?]$/ then
+        :tFID
+      when in_lex_state?(:expr_fname) && scan(/=(?:(?![~>=])|(?==>))/) then
+        # ident=, not =~ => == or followed by =>
+        # TODO test lexing of a=>b vs a==>b
+        token << matched
+        :tIDENTIFIER
+      when token =~ /^[A-Z]/ then
+        :tCONSTANT
+      else
+        :tIDENTIFIER
+      end
+
+    if !ruby18 and is_label_possible?(command_state) and scan(/:(?!:)/) then
+      return result(:expr_beg, :tLABEL, [token, ss.lineno]) # HACK: array? TODO: self.lineno
+    end
+
+    unless in_lex_state? :expr_dot then
+      # See if it is a reserved word.
+      keyword = if ruby18 then # REFACTOR need 18/19 lexer subclasses
+                  RubyParserStuff::Keyword.keyword18 token
+                else
+                  RubyParserStuff::Keyword.keyword19 token
+                end
+
+      return process_token_keyword keyword if keyword
+    end # unless in_lex_state? :expr_dot
+
+    # TODO:
+    # if (mb == ENC_CODERANGE_7BIT && lex_state != EXPR_DOT) {
+
+    state = if is_beg? or is_arg? or in_lex_state? :expr_dot then
+              command_state ? :expr_cmdarg : :expr_arg
+            elsif not ruby18 and in_lex_state? :expr_fname then
+              :expr_endfn
+            else
+              :expr_end
+            end
+
+    if not [:expr_dot, :expr_fname].include? last_state and
+        self.parser.env[token.to_sym] == :lvar then
+      state = :expr_end
+    end
+
+    return result(state, tok_id, token)
+  end
+
+  def process_token_keyword keyword
+    state = keyword.state
+    value = [token, ss.lineno] # TODO: use self.lineno ?
+
+    self.command_start = true if state == :expr_beg and lex_state != :expr_fname
+
+    case
+    when lex_state == :expr_fname then
+      result(state, keyword.id0, keyword.name)
+    when keyword.id0 == :kDO then
+      case
+      when lpar_beg && lpar_beg == paren_nest then
+        self.lpar_beg = nil
+        self.paren_nest -= 1
+        result(state, :kDO_LAMBDA, value)
+      when cond.is_in_state then
+        result(state, :kDO_COND, value)
+      when cmdarg.is_in_state && lex_state != :expr_cmdarg then
+        result(state, :kDO_BLOCK, value)
+      when in_lex_state?(:expr_beg, :expr_endarg) then
+        result(state, :kDO_BLOCK, value)
+      else
+        result(state, :kDO, value)
+      end
+    when in_lex_state?(:expr_beg, :expr_value) then
+      result(state, keyword.id0, value)
+    when keyword.id0 != keyword.id1 then
+      result(:expr_beg, keyword.id1, value)
+    else
+      result(state, keyword.id1, value)
+    end
+  end
+
   def rb_compile_error msg
-    msg += ". near line #{self.lineno}: #{src.rest[/^.*/].inspect}"
+    msg += ". near line #{self.lineno}: #{ss.rest[/^.*/].inspect}"
     raise RubyParser::SyntaxError, msg
   end
 
-  def read_escape # 51 lines
+  def read_escape # TODO: remove / rewrite
     case
-    when src.scan(/\\/) then                  # Backslash
+    when scan(/\\/) then                  # Backslash
       '\\'
-    when src.scan(/n/) then                   # newline
+    when scan(/n/) then                   # newline
       "\n"
-    when src.scan(/t/) then                   # horizontal tab
+    when scan(/t/) then                   # horizontal tab
       "\t"
-    when src.scan(/r/) then                   # carriage-return
+    when scan(/r/) then                   # carriage-return
       "\r"
-    when src.scan(/f/) then                   # form-feed
+    when scan(/f/) then                   # form-feed
       "\f"
-    when src.scan(/v/) then                   # vertical tab
+    when scan(/v/) then                   # vertical tab
       "\13"
-    when src.scan(/a/) then                   # alarm(bell)
+    when scan(/a/) then                   # alarm(bell)
       "\007"
-    when src.scan(/e/) then                   # escape
+    when scan(/e/) then                   # escape
       "\033"
-    when src.scan(/b/) then                   # backspace
+    when scan(/b/) then                   # backspace
       "\010"
-    when src.scan(/s/) then                   # space
+    when scan(/s/) then                   # space
       " "
-    when src.scan(/[0-7]{1,3}/) then          # octal constant
-      (src.matched.to_i(8) & 0xFF).chr
-    when src.scan(/x([0-9a-fA-F]{1,2})/) then # hex constant
-      src[1].to_i(16).chr
-    when src.check(/M-\\[\\MCc]/) then
-      src.scan(/M-\\/) # eat it
+    when scan(/[0-7]{1,3}/) then          # octal constant
+      (matched.to_i(8) & 0xFF).chr
+    when scan(/x([0-9a-fA-F]{1,2})/) then # hex constant
+      ss[1].to_i(16).chr
+    when check(/M-\\[\\MCc]/) then
+      scan(/M-\\/) # eat it
       c = self.read_escape
       c[0] = (c[0].ord | 0x80).chr
       c
-    when src.scan(/M-(.)/) then
-      c = src[1]
+    when scan(/M-(.)/) then
+      c = ss[1]
       c[0] = (c[0].ord | 0x80).chr
       c
-    when src.check(/(C-|c)\\[\\MCc]/) then
-      src.scan(/(C-|c)\\/) # eat it
+    when check(/(C-|c)\\[\\MCc]/) then
+      scan(/(C-|c)\\/) # eat it
       c = self.read_escape
       c[0] = (c[0].ord & 0x9f).chr
       c
-    when src.scan(/C-\?|c\?/) then
+    when scan(/C-\?|c\?/) then
       127.chr
-    when src.scan(/(C-|c)(.)/) then
-      c = src[2]
+    when scan(/(C-|c)(.)/) then
+      c = ss[2]
       c[0] = (c[0].ord & 0x9f).chr
       c
-    when src.scan(/^[89]/i) then # bad octal or hex... MRI ignores them :(
-      src.matched
-    when src.scan(/[McCx0-9]/) || src.eos? then
+    when scan(/^[89]/i) then # bad octal or hex... MRI ignores them :(
+      matched
+    when scan(/u([0-9a-fA-F]+|\{[0-9a-fA-F]+\})/) then
+      [ss[1].delete("{}").to_i(16)].pack("U")
+    when scan(/[McCx0-9]/) || end_of_stream? then
       rb_compile_error("Invalid escape character syntax")
     else
-      src.getch
+      ss.getch
     end
   end
 
-  def regx_options # 15 lines
+  def regx_options # TODO: rewrite / remove
     good, bad = [], []
 
-    if src.scan(/[a-z]+/) then
-      good, bad = src.matched.split(//).partition { |s| s =~ /^[ixmonesu]$/ }
+    if scan(/[a-z]+/) then
+      good, bad = matched.split(//).partition { |s| s =~ /^[ixmonesu]$/ }
     end
 
     unless bad.empty? then
@@ -505,13 +632,30 @@ class RubyLexer
   end
 
   def reset
+    self.brace_nest    = 0
     self.command_start = true
+    self.comments      = []
+    self.lex_state     = nil
     self.lex_strterm   = nil
+    self.lineno        = 1
+    self.lpar_beg      = nil
+    self.paren_nest    = 0
+    self.space_seen    = false
+    self.string_nest   = 0
     self.token         = nil
     self.yacc_value    = nil
 
-    @src       = nil
-    @lex_state = nil
+    self.cmdarg = RubyParserStuff::StackState.new(:cmdarg)
+    self.cond   = RubyParserStuff::StackState.new(:cond)
+
+    @src = nil
+  end
+
+  def result lex_state, token, text # :nodoc:
+    lex_state = self.arg_state if lex_state == :arg_state
+    self.lex_state = lex_state if lex_state
+    self.yacc_value = text
+    token
   end
 
   def ruby18
@@ -522,35 +666,52 @@ class RubyLexer
     Ruby19Parser === parser
   end
 
+  def scan re
+    ss.scan re
+  end
+
+  def space_vs_beginning space_type, beg_type, fallback
+    if is_space_arg? check(/./m) then
+      warning "`**' interpreted as argument prefix"
+      space_type
+    elsif is_beg? then
+      beg_type
+    else
+      # TODO: warn_balanced("**", "argument prefix");
+      fallback
+    end
+  end
+
+  def string type, beg = matched, nnd = "\0"
+    self.lex_strterm = [:strterm, type, beg, nnd]
+  end
+
   def src= src
     raise "bad src: #{src.inspect}" unless String === src
     @src = RPStringScanner.new(src)
   end
 
-  def tokadd_escape term # 20 lines
+  def tokadd_escape term # TODO: rewrite / remove
     case
-    when src.scan(/\\\n/) then
+    when scan(/\\\n/) then
       # just ignore
-    when src.scan(/\\([0-7]{1,3}|x[0-9a-fA-F]{1,2})/) then
-      self.string_buffer << src.matched
-    when src.scan(/\\([MC]-|c)(?=\\)/) then
-      self.string_buffer << src.matched
+    when scan(/\\([0-7]{1,3}|x[0-9a-fA-F]{1,2})/) then
+      self.string_buffer << matched
+    when scan(/\\([MC]-|c)(?=\\)/) then
+      self.string_buffer << matched
       self.tokadd_escape term
-    when src.scan(/\\([MC]-|c)(.)/) then
-      self.string_buffer << src.matched
-    when src.scan(/\\[McCx]/) then
+    when scan(/\\([MC]-|c)(.)/) then
+      self.string_buffer << matched
+    when scan(/\\[McCx]/) then
       rb_compile_error "Invalid escape character syntax"
-    when src.scan(/\\(.)/m) then
-      self.string_buffer << src.matched
+    when scan(/\\(.)/m) then
+      self.string_buffer << matched
     else
       rb_compile_error "Invalid escape character syntax"
     end
   end
 
-  @@regexp_cache = Hash.new { |h,k| h[k] = Regexp.new(Regexp.escape(k)) }
-  @@regexp_cache[nil] = nil
-
-  def tokadd_string(func, term, paren) # 105 lines
+  def tokadd_string(func, term, paren) # TODO: rewrite / remove
     qwords = (func & STR_FUNC_QWORDS) != 0
     escape = (func & STR_FUNC_ESCAPE) != 0
     expand = (func & STR_FUNC_EXPAND) != 0
@@ -560,49 +721,49 @@ class RubyLexer
     paren_re = @@regexp_cache[paren]
     term_re  = @@regexp_cache[term]
 
-    until src.eos? do
+    until end_of_stream? do
       c = nil
       handled = true
 
       case
-      when paren_re && src.scan(paren_re) then
+      when paren_re && scan(paren_re) then
         self.string_nest += 1
-      when src.scan(term_re) then
+      when scan(term_re) then
         if self.string_nest == 0 then
-          src.pos -= 1
+          ss.pos -= 1
           break
         else
           self.string_nest -= 1
         end
-      when expand && src.scan(/#(?=[\$\@\{])/) then
-        src.pos -= 1
+      when expand && scan(/#(?=[\$\@\{])/) then
+        ss.pos -= 1
         break
-      when qwords && src.scan(/\s/) then
-        src.pos -= 1
+      when qwords && scan(/\s/) then
+        ss.pos -= 1
         break
-      when expand && src.scan(/#(?!\n)/) then
+      when expand && scan(/#(?!\n)/) then
         # do nothing
-      when src.check(/\\/) then
+      when check(/\\/) then
         case
-        when qwords && src.scan(/\\\n/) then
+        when qwords && scan(/\\\n/) then
           string_buffer << "\n"
           next
-        when qwords && src.scan(/\\\s/) then
+        when qwords && scan(/\\\s/) then
           c = ' '
-        when expand && src.scan(/\\\n/) then
+        when expand && scan(/\\\n/) then
           next
-        when regexp && src.check(/\\/) then
+        when regexp && check(/\\/) then
           self.tokadd_escape term
           next
-        when expand && src.scan(/\\/) then
+        when expand && scan(/\\/) then
           c = self.read_escape
-        when src.scan(/\\\n/) then
+        when scan(/\\\n/) then
           # do nothing
-        when src.scan(/\\\\/) then
+        when scan(/\\\\/) then
           string_buffer << '\\' if escape
           c = '\\'
-        when src.scan(/\\/) then
-          unless src.scan(term_re) || paren.nil? || src.scan(paren_re) then
+        when scan(/\\/) then
+          unless scan(term_re) || paren.nil? || scan(paren_re) then
             string_buffer << "\\"
           end
         else
@@ -625,38 +786,22 @@ class RubyLexer
                /[^#{t}#{x}\#\0\\]+|./
              end
 
-        src.scan re
-        c = src.matched
+        scan re
+        c = matched
 
         rb_compile_error "symbol cannot contain '\\0'" if symbol && c =~ /\0/
       end # unless handled
 
-      c ||= src.matched
+      c ||= matched
       string_buffer << c
     end # until
 
-    c ||= src.matched
-    c = RubyLexer::EOF if src.eos?
+    c ||= matched
+    c = RubyLexer::EOF if end_of_stream?
 
     return c
   end
 
-  ESCAPES = {
-    "a"    => "\007",
-    "b"    => "\010",
-    "e"    => "\033",
-    "f"    => "\f",
-    "n"    => "\n",
-    "r"    => "\r",
-    "s"    => " ",
-    "t"    => "\t",
-    "v"    => "\13",
-    "\\"   => '\\',
-    "\n"   => "",
-    "C-\?" => 127.chr,
-    "c\?"  => 127.chr,
-  }
-
   def unescape s
     r = ESCAPES[s]
 
@@ -675,6 +820,8 @@ class RubyLexer
           s
         when /^[McCx0-9]/ then
           rb_compile_error("Invalid escape character syntax")
+        when /u([0-9a-fA-F]+|\{[0-9a-fA-F]+\})/ then
+          [$1.delete("{}").to_i(16)].pack("U")
         else
           s
         end
@@ -691,11 +838,11 @@ class RubyLexer
   #
   # @return Description of the Returned Value
 
-  def yylex # 826 lines
+  def yylex # 461 lines
     c = ''
     self.space_seen = false
     command_state = false
-    src = self.src
+    ss = self.src
 
     self.token = nil
     self.yacc_value = nil
@@ -708,41 +855,42 @@ class RubyLexer
     last_state = lex_state
 
     loop do # START OF CASE
-      if src.scan(/[\ \t\r\f\v]/) then # \s - \n + \v
+      if scan(/[\ \t\r\f\v]/) then # \s - \n + \v
         self.space_seen = true
         next
-      elsif src.check(/[^a-zA-Z]/) then
-        if src.scan(/\n|#/) then
+      elsif check(/[^a-zA-Z]/) then
+        if scan(/\n|\#/) then
           self.lineno = nil
-          c = src.matched
+          c = matched
           if c == '#' then
-            src.pos -= 1
+            ss.pos -= 1
 
-            while src.scan(/\s*#.*(\n+|\z)/) do
-              @comments << src.matched.gsub(/^ +#/, '#').gsub(/^ +$/, '')
+            while scan(/\s*#.*(\n+|\z)/) do
+              # TODO: self.lineno += matched.lines.to_a.size
+              @comments << matched.gsub(/^ +#/, '#').gsub(/^ +$/, '')
             end
 
-            return RubyLexer::EOF if src.eos?
+            return RubyLexer::EOF if end_of_stream?
           end
 
           # Replace a string of newlines with a single one
-          src.scan(/\n+/)
+          scan(/\n+/)
 
           next if in_lex_state?(:expr_beg, :expr_value, :expr_class,
                                 :expr_fname, :expr_dot)
 
-          if src.scan(/([\ \t\r\f\v]*)\./) then
-            self.space_seen = true unless src[1].empty?
+          if scan(/([\ \t\r\f\v]*)\./) then
+            self.space_seen = true unless ss[1].empty?
 
-            src.pos -= 1
-            next unless src.check(/\.\./)
+            ss.pos -= 1
+            next unless check(/\.\./)
           end
 
           self.command_start = true
-          self.lex_state = :expr_beg
-          return :tNL
-        elsif src.scan(/[\]\)\}]/) then
-          if src.matched == "}" then
+
+          return result(:expr_beg, :tNL, nil)
+        elsif scan(/[\]\)\}]/) then
+          if matched == "}" then
             self.brace_nest -= 1
           else
             self.paren_nest -= 1
@@ -750,54 +898,34 @@ class RubyLexer
 
           cond.lexpop
           cmdarg.lexpop
-          tern.lexpop
-
-          self.lex_state = if src.matched == ")" then
-                             :expr_endfn
-                           else
-                             :expr_endarg
-                           end
 
-          self.yacc_value = src.matched
-          result = {
+          text  = matched
+          state = text == ")" ? :expr_endfn : :expr_endarg
+          token = {
             ")" => :tRPAREN,
             "]" => :tRBRACK,
             "}" => :tRCURLY
-          }[src.matched]
-          return result
-        elsif src.scan(/\!/) then
-          if in_lex_state?(:expr_fname, :expr_dot) then
-            self.lex_state = :expr_arg
-
-            if src.scan(/@/) then
-              self.yacc_value = "!@"
-              return :tUBANG
-            end
-          else
-            self.lex_state = :expr_beg
-          end
+          }[text]
 
-          if src.scan(/[=~]/) then
-            self.yacc_value = "!#{src.matched}"
-          else
-            self.yacc_value = "!"
+          return result(state, token, text)
+        elsif scan(/\!/) then
+          if in_arg_state? then
+            return result(:expr_arg, :tUBANG, "!@") if scan(/@/)
           end
 
-          return TOKENS[self.yacc_value]
-        elsif src.scan(/\.\.\.?|,|![=~]?/) then
-          self.lex_state = :expr_beg
-          tok = self.yacc_value = src.matched
-          return TOKENS[tok]
-        elsif src.check(/\./) then
-          if src.scan(/\.\d/) then
+          text = scan(/[=~]/) ? "!#{matched}" : "!"
+
+          return result(arg_state, TOKENS[text], text)
+        elsif scan(/\.\.\.?|,|![=~]?/) then
+          return result(:expr_beg, TOKENS[matched], matched)
+        elsif check(/\./) then
+          if scan(/\.\d/) then
             rb_compile_error "no .<digit> floating literal anymore put 0 before dot"
-          elsif src.scan(/\./) then
-            self.lex_state = :expr_dot
-            self.yacc_value = "."
-            return :tDOT
+          elsif scan(/\./) then
+            return result(:expr_dot, :tDOT, ".")
           end
-        elsif src.scan(/\(/) then
-          result = if ruby18 then
+        elsif scan(/\(/) then
+          token = if ruby18 then
                      yylex_paren18
                    else
                      yylex_paren19
@@ -805,259 +933,188 @@ class RubyLexer
 
           self.paren_nest += 1
 
-          self.expr_beg_push "("
+          return expr_result(token, "(")
+        elsif check(/\=/) then
+          if scan(/\=\=\=|\=\=|\=~|\=>|\=(?!begin\b)/) then
+            tok = matched
+            return result(:arg_state, TOKENS[tok], tok)
+          elsif beginning_of_line? and scan(/\=begin(?=\s)/) then
+            @comments << matched
 
-          return result
-        elsif src.check(/\=/) then
-          if src.scan(/\=\=\=|\=\=|\=~|\=>|\=(?!begin\b)/) then
-            self.fix_arg_lex_state
-            tok = self.yacc_value = src.matched
-            return TOKENS[tok]
-          elsif src.scan(/\=begin(?=\s)/) then
-            @comments << src.matched
-
-            unless src.scan(/.*?\n=end( |\t|\f)*[^\n]*(\n|\z)/m) then
+            unless scan(/.*?\n=end( |\t|\f)*[^\n]*(\n|\z)/m) then
               @comments.clear
               rb_compile_error("embedded document meets end of file")
             end
 
-            @comments << src.matched
+            @comments << matched
 
             next
+          elsif scan(/\=(?=begin\b)/) then # h[k]=begin ... end
+            tok = matched
+            return result(:arg_state, TOKENS[tok], tok)
           else
             raise "you shouldn't be able to get here"
           end
-        elsif src.scan(/\"(#{ESC_RE}|#(#{ESC_RE}|[^\{\#\@\$\"\\])|[^\"\\\#])*\"/o) then
-          self.yacc_value = src.matched[1..-2].gsub(ESC_RE) { unescape $1 }
-          self.lex_state = :expr_end
-          return :tSTRING
-        elsif src.scan(/\"/) then # FALLBACK
-          self.lex_strterm = [:strterm, STR_DQUOTE, '"', "\0"] # TODO: question this
-          self.yacc_value = "\""
-          return :tSTRING_BEG
-        elsif src.scan(/\@\@?#{IDENT_CHAR_RE}+/o) then
-          self.token = src.matched
-
-          rb_compile_error "`#{token}` is not allowed as a variable name" if
-            token =~ /\@\d/
-
-          return process_token(command_state)
-        elsif src.scan(/\:\:/) then
+        elsif scan(/\"(#{SIMPLE_STRING})\"/o) then
+          string = matched[1..-2].gsub(ESC) { unescape $1 }
+          return result(:expr_end, :tSTRING, string)
+        elsif scan(/\"/) then # FALLBACK
+          string STR_DQUOTE, '"' # TODO: question this
+          return result(nil, :tSTRING_BEG, '"')
+        elsif scan(/\@\@?#{IDENT_CHAR}+/o) then
+          self.token = matched
+
+          rb_compile_error "`#{self.token}` is not allowed as a variable name" if
+            self.token =~ /\@\d/
+
+          tok_id = matched =~ /^@@/ ? :tCVAR : :tIVAR
+          return result(:expr_end, tok_id, self.token)
+        elsif scan(/\:\:/) then
           if is_beg? || in_lex_state?(:expr_class) || is_space_arg? then
-            self.lex_state = :expr_beg
-            self.yacc_value = "::"
-            return :tCOLON3
+            return result(:expr_beg, :tCOLON3, "::")
           end
 
-          self.lex_state = :expr_dot
-          self.yacc_value = "::"
-          return :tCOLON2
-        elsif ! is_end? && src.scan(/:([a-zA-Z_]#{IDENT_CHAR_RE}*(?:[?!]|=(?==>)|=(?![=>]))?)/) then
+          return result(:expr_dot, :tCOLON2, "::")
+        elsif ! is_end? && scan(/:([a-zA-Z_]#{IDENT_CHAR}*(?:[?!]|=(?==>)|=(?![=>]))?)/) then
           # scanning shortcut to symbols
-          self.yacc_value = src[1]
-          self.lex_state = :expr_end
-          return :tSYMBOL
-        elsif src.scan(/\:/) then
+          return result(:expr_end, :tSYMBOL, ss[1])
+        elsif ! is_end? && (scan(/\:\"(#{SIMPLE_STRING})\"/) ||
+                            scan(/\:\'(#{SIMPLE_SSTRING})\'/)) then
+          symbol = ss[1].gsub(ESC) { unescape $1 }
+
+          rb_compile_error "symbol cannot contain '\\0'" if
+            ruby18 && symbol =~ /\0/
+
+          return result(:expr_end, :tSYMBOL, symbol)
+        elsif scan(/\:/) then
           # ?: / then / when
-          if is_end? || src.check(/\s/) then
-            self.lex_state = :expr_beg
+          if is_end? || check(/\s/) then
             # TODO warn_balanced(":", "symbol literal");
-            self.yacc_value = ":"
-            return :tCOLON
+            return result(:expr_beg, :tCOLON, ":")
           end
 
           case
-          when src.scan(/\'/) then
-            self.lex_strterm = [:strterm, STR_SSYM, src.matched, "\0"]
-          when src.scan(/\"/) then
-            self.lex_strterm = [:strterm, STR_DSYM, src.matched, "\0"]
+          when scan(/\'/) then
+            string STR_SSYM, matched
+          when scan(/\"/) then
+            string STR_DSYM, matched
           end
 
-          self.lex_state = :expr_fname
-          self.yacc_value = ":"
-          return :tSYMBEG
-        elsif src.check(/[0-9]/) then
+          return result(:expr_fname, :tSYMBEG, ":")
+        elsif check(/[0-9]/) then
           return parse_number
-        elsif src.scan(/\[/) then
+        elsif scan(/\[/) then
           self.paren_nest += 1
 
-          result = src.matched
+          token = nil
 
           if in_lex_state? :expr_fname, :expr_dot then
-            self.lex_state = :expr_arg
             case
-            when src.scan(/\]\=/) then
+            when scan(/\]\=/) then
               self.paren_nest -= 1 # HACK? I dunno, or bug in MRI
-              self.yacc_value = "[]="
-              return :tASET
-            when src.scan(/\]/) then
+              return result(:expr_arg, :tASET, "[]=")
+            when scan(/\]/) then
               self.paren_nest -= 1 # HACK? I dunno, or bug in MRI
-              self.yacc_value = "[]"
-              return :tAREF
+              return result(:expr_arg, :tAREF, "[]")
             else
               rb_compile_error "unexpected '['"
             end
           elsif is_beg? then
-            self.tern.push false
-            result = :tLBRACK
+            token = :tLBRACK
           elsif is_arg? && space_seen then
-            self.tern.push false
-            result = :tLBRACK
+            token = :tLBRACK
           else
-            result = :tLBRACK2
+            token = :tLBRACK2
           end
 
-          self.expr_beg_push "["
-
-          return result
-        elsif src.scan(/\'(\\.|[^\'])*\'/) then
-          self.yacc_value = src.matched[1..-2].gsub(/\\\\/, "\\").gsub(/\\'/, "'") # "
-          self.lex_state = :expr_end
-          return :tSTRING
-        elsif src.check(/\|/) then
-          if src.scan(/\|\|\=/) then
-            self.lex_state = :expr_beg
-            self.yacc_value = "||"
-            return :tOP_ASGN
-          elsif src.scan(/\|\|/) then
-            self.lex_state = :expr_beg
-            self.yacc_value = "||"
-            return :tOROP
-          elsif src.scan(/\|\=/) then
-            self.lex_state = :expr_beg
-            self.yacc_value = "|"
-            return :tOP_ASGN
-          elsif src.scan(/\|/) then
-            self.fix_arg_lex_state
-            self.yacc_value = "|"
-            return :tPIPE
+          return expr_result(token, "[")
+        elsif scan(/\'#{SIMPLE_SSTRING}\'/) then
+          text = matched[1..-2].gsub(/\\\\/, "\\").gsub(/\\'/, "'") # "
+          return result(:expr_end, :tSTRING, text)
+        elsif check(/\|/) then
+          if scan(/\|\|\=/) then
+            return result(:expr_beg, :tOP_ASGN, "||")
+          elsif scan(/\|\|/) then
+            return result(:expr_beg, :tOROP, "||")
+          elsif scan(/\|\=/) then
+            return result(:expr_beg, :tOP_ASGN, "|")
+          elsif scan(/\|/) then
+            return result(:arg_state, :tPIPE, "|")
           end
-        elsif src.scan(/\{/) then
+        elsif scan(/\{/) then
           self.brace_nest += 1
           if lpar_beg && lpar_beg == paren_nest then
             self.lpar_beg = nil
             self.paren_nest -= 1
 
-            expr_beg_push "{"
-
-            return :tLAMBEG
+            return expr_result(:tLAMBEG, "{")
           end
 
-          result = if is_arg? || in_lex_state?(:expr_end, :expr_endfn) then
+          token = if is_arg? || in_lex_state?(:expr_end, :expr_endfn) then
                      :tLCURLY      #  block (primary)
                    elsif in_lex_state?(:expr_endarg) then
                      :tLBRACE_ARG  #  block (expr)
                    else
-                     self.tern.push false
                      :tLBRACE      #  hash
                    end
 
-          self.expr_beg_push "{"
-          self.command_start = true unless result == :tLBRACE
+          self.command_start = true unless token == :tLBRACE
 
-          return result
-        elsif src.scan(/->/) then
-          self.lex_state = :expr_endfn
-          return :tLAMBDA
-        elsif src.scan(/[+-]/) then
-          sign = src.matched
+          return expr_result(token, "{")
+        elsif scan(/->/) then
+          return result(:expr_endfn, :tLAMBDA, nil)
+        elsif scan(/[+-]/) then
+          sign = matched
           utype, type = if sign == "+" then
                           [:tUPLUS, :tPLUS]
                         else
                           [:tUMINUS, :tMINUS]
                         end
 
-          if in_lex_state? :expr_fname, :expr_dot then
-            self.lex_state = :expr_arg
-            if src.scan(/@/) then
-              self.yacc_value = "#{sign}@"
-              return utype
+          if in_arg_state? then
+            if scan(/@/) then
+              return result(:expr_arg, utype, "#{sign}@")
             else
-              self.yacc_value = sign
-              return type
+              return result(:expr_arg, type, sign)
             end
           end
 
-          if src.scan(/\=/) then
-            self.lex_state = :expr_beg
-            self.yacc_value = sign
-            return :tOP_ASGN
-          end
+          return result(:expr_beg, :tOP_ASGN, sign) if scan(/\=/)
 
-          if (is_beg? || (is_arg? && space_seen && !src.check(/\s/))) then
-            if is_arg? then
-              arg_ambiguous
-            end
+          if (is_beg? || (is_arg? && space_seen && !check(/\s/))) then
+            arg_ambiguous if is_arg?
 
-            self.lex_state = :expr_beg
-            self.yacc_value = sign
-
-            if src.check(/\d/) then
-              if utype == :tUPLUS then
-                return self.parse_number
-              else
-                return :tUMINUS_NUM
-              end
+            if check(/\d/) then
+              return self.parse_number if utype == :tUPLUS
+              return result(:expr_beg, :tUMINUS_NUM, sign)
             end
 
-            return utype
+            return result(:expr_beg, utype, sign)
           end
 
-          self.lex_state = :expr_beg
-          self.yacc_value = sign
-          return type
-        elsif src.check(/\*/) then
-          if src.scan(/\*\*=/) then
-            self.lex_state = :expr_beg
-            self.yacc_value = "**"
-            return :tOP_ASGN
-          elsif src.scan(/\*\*/) then
-            result = if is_space_arg? src.check(/./m) then
-                       warning "`**' interpreted as argument prefix"
-                       :tDSTAR
-                     elsif is_beg? then
-                       :tDSTAR
-                     else
-                       # TODO: warn_balanced("**", "argument prefix");
-                       :tPOW
-                     end
-            self.yacc_value = "**"
-            self.fix_arg_lex_state
-            return result
-          elsif src.scan(/\*\=/) then
-            self.lex_state = :expr_beg
-            self.yacc_value = "*"
-            return :tOP_ASGN
-          elsif src.scan(/\*/) then
-            result = if is_space_arg? src.check(/./m) then
-                       warning("`*' interpreted as argument prefix")
-                       :tSTAR
-                     elsif is_beg? then
-                       :tSTAR
-                     else
-                       # TODO: warn_balanced("*", "argument prefix");
-                       :tSTAR2 # TODO: rename
-                     end
+          return result(:expr_beg, type, sign)
+        elsif check(/\*/) then
+          if scan(/\*\*=/) then
+            return result(:expr_beg, :tOP_ASGN, "**")
+          elsif scan(/\*\*/) then
+            token = space_vs_beginning :tDSTAR, :tDSTAR, :tPOW
+
+            return result(:arg_state, token, "**")
+          elsif scan(/\*\=/) then
+            return result(:expr_beg, :tOP_ASGN, "*")
+          elsif scan(/\*/) then
+            token = space_vs_beginning :tSTAR, :tSTAR, :tSTAR2
 
-            self.yacc_value = "*"
-            self.fix_arg_lex_state
-            return result
+            return result(:arg_state, token, "*")
           end
-        elsif src.check(/\</) then
-          if src.scan(/\<\=\>/) then
-            self.fix_arg_lex_state
-            self.yacc_value = "<=>"
-            return :tCMP
-          elsif src.scan(/\<\=/) then
-            self.fix_arg_lex_state
-            self.yacc_value = "<="
-            return :tLEQ
-          elsif src.scan(/\<\<\=/) then
-            self.fix_arg_lex_state
-            self.lex_state = :expr_beg
-            self.yacc_value = "\<\<"
-            return :tOP_ASGN
-          elsif src.scan(/\<\</) then
+        elsif check(/\</) then
+          if scan(/\<\=\>/) then
+            return result(:arg_state, :tCMP, "<=>")
+          elsif scan(/\<\=/) then
+            return result(:arg_state, :tLEQ, "<=")
+          elsif scan(/\<\<\=/) then
+            return result(:arg_state, :tOP_ASGN, "<<")
+          elsif scan(/\<\</) then
             if (!in_lex_state?(:expr_dot, :expr_class) &&
                 !is_end? &&
                 (!is_arg? || space_seen)) then
@@ -1065,70 +1122,49 @@ class RubyLexer
               return tok if tok
             end
 
-            self.fix_arg_lex_state
-            self.yacc_value = "\<\<"
-            return :tLSHFT
-          elsif src.scan(/\</) then
-            self.fix_arg_lex_state
-            self.yacc_value = "<"
-            return :tLT
+            return result(:arg_state, :tLSHFT, "\<\<")
+          elsif scan(/\</) then
+            return result(:arg_state, :tLT, "<")
           end
-        elsif src.check(/\>/) then
-          if src.scan(/\>\=/) then
-            self.fix_arg_lex_state
-            self.yacc_value = ">="
-            return :tGEQ
-          elsif src.scan(/\>\>=/) then
-            self.fix_arg_lex_state
-            self.lex_state = :expr_beg
-            self.yacc_value = ">>"
-            return :tOP_ASGN
-          elsif src.scan(/\>\>/) then
-            self.fix_arg_lex_state
-            self.yacc_value = ">>"
-            return :tRSHFT
-          elsif src.scan(/\>/) then
-            self.fix_arg_lex_state
-            self.yacc_value = ">"
-            return :tGT
+        elsif check(/\>/) then
+          if scan(/\>\=/) then
+            return result(:arg_state, :tGEQ, ">=")
+          elsif scan(/\>\>=/) then
+            return result(:arg_state, :tOP_ASGN, ">>")
+          elsif scan(/\>\>/) then
+            return result(:arg_state, :tRSHFT, ">>")
+          elsif scan(/\>/) then
+            return result(:arg_state, :tGT, ">")
           end
-        elsif src.scan(/\`/) then
-          self.yacc_value = "`"
+        elsif scan(/\`/) then
           case lex_state
           when :expr_fname then
-            self.lex_state = :expr_end
-            return :tBACK_REF2
+            return result(:expr_end, :tBACK_REF2, "`")
           when :expr_dot then
-            self.lex_state = if command_state then
-                               :expr_cmdarg
-                             else
-                               :expr_arg
-                             end
-            return :tBACK_REF2
+            state = command_state ? :expr_cmdarg : :expr_arg
+            return result(state, :tBACK_REF2, "`")
+          else
+            string STR_XQUOTE, '`'
+            return result(nil, :tXSTRING_BEG, "`")
           end
-          self.lex_strterm = [:strterm, STR_XQUOTE, '`', "\0"]
-          return :tXSTRING_BEG
-        elsif src.scan(/\?/) then
-
+        elsif scan(/\?/) then
           if is_end? then
-            self.lex_state = ruby18 ? :expr_beg : :expr_value # HACK?
-            self.tern.push true
-            self.yacc_value = "?"
-            return :tEH
+            state = ruby18 ? :expr_beg : :expr_value # HACK?
+            return result(state, :tEH, "?")
           end
 
-          if src.eos? then
+          if end_of_stream? then
             rb_compile_error "incomplete character syntax"
           end
 
-          if src.check(/\s|\v/) then
+          if check(/\s|\v/) then
             unless is_arg? then
               c2 = { " " => 's',
                     "\n" => 'n',
                     "\t" => 't',
                     "\v" => 'v',
                     "\r" => 'r',
-                    "\f" => 'f' }[src.matched]
+                    "\f" => 'f' }[matched]
 
               if c2 then
                 warning("invalid character syntax; use ?\\" + c2)
@@ -1136,47 +1172,32 @@ class RubyLexer
             end
 
             # ternary
-            self.lex_state = ruby18 ? :expr_beg : :expr_value # HACK?
-            self.tern.push true
-            self.yacc_value = "?"
-            return :tEH
-          elsif src.check(/\w(?=\w)/) then # ternary, also
-            self.lex_state = :expr_beg
-            self.tern.push true
-            self.yacc_value = "?"
-            return :tEH
+            state = ruby18 ? :expr_beg : :expr_value # HACK?
+            return result(state, :tEH, "?")
+          elsif check(/\w(?=\w)/) then # ternary, also
+            return result(:expr_beg, :tEH, "?")
           end
 
-          c = if src.scan(/\\/) then
+          c = if scan(/\\/) then
                 self.read_escape
               else
-                src.getch
+                ss.getch
               end
-          self.lex_state = :expr_end
 
           if version == 18 then
-            self.yacc_value = c[0].ord & 0xff
-            return :tINTEGER
+            return result(:expr_end, :tINTEGER, c[0].ord & 0xff)
           else
-            self.yacc_value = c
-            return :tSTRING
+            return result(:expr_end, :tSTRING, c)
           end
-        elsif src.check(/\&/) then
-          if src.scan(/\&\&\=/) then
-            self.yacc_value = "&&"
-            self.lex_state = :expr_beg
-            return :tOP_ASGN
-          elsif src.scan(/\&\&/) then
-            self.lex_state = :expr_beg
-            self.yacc_value = "&&"
-            return :tANDOP
-          elsif src.scan(/\&\=/) then
-            self.yacc_value = "&"
-            self.lex_state = :expr_beg
-            return :tOP_ASGN
-          elsif src.scan(/&/) then
-            result = if is_arg? && space_seen &&
-                         !src.check(/\s/) then
+        elsif check(/\&/) then
+          if scan(/\&\&\=/) then
+            return result(:expr_beg, :tOP_ASGN, "&&")
+          elsif scan(/\&\&/) then
+            return result(:expr_beg, :tANDOP, "&&")
+          elsif scan(/\&\=/) then
+            return result(:expr_beg, :tOP_ASGN, "&")
+          elsif scan(/&/) then
+            token = if is_arg? && space_seen && !check(/\s/) then
                        warning("`&' interpreted as argument prefix")
                        :tAMPER
                      elsif in_lex_state? :expr_beg, :expr_mid then
@@ -1185,170 +1206,121 @@ class RubyLexer
                        :tAMPER2
                      end
 
-            self.fix_arg_lex_state
-            self.yacc_value = "&"
-            return result
+            return result(:arg_state, token, "&")
           end
-        elsif src.scan(/\//) then
+        elsif scan(/\//) then
           if is_beg? then
-            self.lex_strterm = [:strterm, STR_REGEXP, '/', "\0"]
-            self.yacc_value = "/"
-            return :tREGEXP_BEG
+            string STR_REGEXP, '/'
+            return result(nil, :tREGEXP_BEG, "/")
           end
 
-          if src.scan(/\=/) then
-            self.yacc_value = "/"
-            self.lex_state = :expr_beg
-            return :tOP_ASGN
+          if scan(/\=/) then
+            return result(:expr_beg, :tOP_ASGN, "/")
           end
 
           if is_arg? && space_seen then
-            unless src.scan(/\s/) then
+            unless scan(/\s/) then
               arg_ambiguous
-              self.lex_strterm = [:strterm, STR_REGEXP, '/', "\0"]
-              self.yacc_value = "/"
-              return :tREGEXP_BEG
+              string STR_REGEXP, '/'
+              return result(nil, :tREGEXP_BEG, "/")
             end
           end
 
-          self.fix_arg_lex_state
-          self.yacc_value = "/"
-
-          return :tDIVIDE
-        elsif src.scan(/\^=/) then
-          self.lex_state = :expr_beg
-          self.yacc_value = "^"
-          return :tOP_ASGN
-        elsif src.scan(/\^/) then
-          self.fix_arg_lex_state
-          self.yacc_value = "^"
-          return :tCARET
-        elsif src.scan(/\;/) then
+          return result(:arg_state, :tDIVIDE, "/")
+        elsif scan(/\^=/) then
+          return result(:expr_beg, :tOP_ASGN, "^")
+        elsif scan(/\^/) then
+          return result(:arg_state, :tCARET, "^")
+        elsif scan(/\;/) then
           self.command_start = true
-          self.lex_state = :expr_beg
-          self.yacc_value = ";"
-          return :tSEMI
-        elsif src.scan(/\~/) then
-          if in_lex_state? :expr_fname, :expr_dot then
-            src.scan(/@/)
-          end
-
-          self.fix_arg_lex_state
-          self.yacc_value = "~"
-
-          return :tTILDE
-        elsif src.scan(/\\/) then
-          if src.scan(/\r?\n/) then
+          return result(:expr_beg, :tSEMI, ";")
+        elsif scan(/\~/) then
+          scan(/@/) if in_lex_state? :expr_fname, :expr_dot
+          return result(:arg_state, :tTILDE, "~")
+        elsif scan(/\\/) then
+          if scan(/\r?\n/) then
             self.lineno = nil
             self.space_seen = true
             next
           end
           rb_compile_error "bare backslash only allowed before newline"
-        elsif src.scan(/\%/) then
-          if is_beg? then
-            return parse_quote
-          end
-
-          if src.scan(/\=/) then
-            self.lex_state = :expr_beg
-            self.yacc_value = "%"
-            return :tOP_ASGN
-          end
-
-          return parse_quote if is_arg? && space_seen && ! src.check(/\s/)
-
-          self.fix_arg_lex_state
-          self.yacc_value = "%"
-
-          return :tPERCENT
-        elsif src.check(/\$/) then
-          if src.scan(/(\$_)(\w+)/) then
-            self.lex_state = :expr_end
-            self.token = src.matched
-            return process_token(command_state)
-          elsif src.scan(/\$_/) then
-            self.lex_state = :expr_end
-            self.token = src.matched
-            self.yacc_value = src.matched
-            return :tGVAR
-          elsif src.scan(/\$[~*$?!@\/\\;,.=:<>\"]|\$-\w?/) then
-            self.lex_state = :expr_end
-            self.yacc_value = src.matched
-            return :tGVAR
-          elsif src.scan(/\$([\&\`\'\+])/) then
-            self.lex_state = :expr_end
+        elsif scan(/\%/) then
+          return parse_quote if is_beg?
+
+          return result(:expr_beg, :tOP_ASGN, "%") if scan(/\=/)
+
+          return parse_quote if is_arg? && space_seen && ! check(/\s/)
+
+          return result(:arg_state, :tPERCENT, "%")
+        elsif check(/\$/) then
+          if scan(/(\$_)(\w+)/) then
+            self.token = matched
+            return result(:expr_end, :tGVAR, matched)
+          elsif scan(/\$_/) then
+            return result(:expr_end, :tGVAR, matched)
+          elsif scan(/\$[~*$?!@\/\\;,.=:<>\"]|\$-\w?/) then
+            return result(:expr_end, :tGVAR, matched)
+          elsif scan(/\$([\&\`\'\+])/) then
             # Explicit reference to these vars as symbols...
-            if last_state == :expr_fname then
-              self.yacc_value = src.matched
-              return :tGVAR
+            if lex_state == :expr_fname then
+              return result(:expr_end, :tGVAR, matched)
             else
-              self.yacc_value = src[1].to_sym
-              return :tBACK_REF
+              return result(:expr_end, :tBACK_REF, ss[1].to_sym)
             end
-          elsif src.scan(/\$([1-9]\d*)/) then
-            self.lex_state = :expr_end
-            if last_state == :expr_fname then
-              self.yacc_value = src.matched
-              return :tGVAR
+          elsif scan(/\$([1-9]\d*)/) then
+            if lex_state == :expr_fname then
+              return result(:expr_end, :tGVAR, matched)
             else
-              self.yacc_value = src[1].to_i
-              return :tNTH_REF
+              return result(:expr_end, :tNTH_REF, ss[1].to_i)
             end
-          elsif src.scan(/\$0/) then
-            self.lex_state = :expr_end
-            self.token = src.matched
-            return process_token(command_state)
-          elsif src.scan(/\$\W|\$\z/) then # TODO: remove?
-            self.lex_state = :expr_end
-            self.yacc_value = "$"
-            return "$"
-          elsif src.scan(/\$\w+/)
-            self.lex_state = :expr_end
-            self.token = src.matched
-            return process_token(command_state)
+          elsif scan(/\$0/) then
+            return result(:expr_end, :tGVAR, matched)
+          elsif scan(/\$\W|\$\z/) then # TODO: remove?
+            return result(:expr_end, "$", "$") # FIX: "$"??
+          elsif scan(/\$\w+/)
+            return result(:expr_end, :tGVAR, matched)
           end
-        elsif src.check(/\_/) then
-          if src.beginning_of_line? && src.scan(/\__END__(\r?\n|\Z)/) then
+        elsif check(/\_/) then
+          if beginning_of_line? && scan(/\__END__(\r?\n|\Z)/) then
             self.lineno = nil
             return RubyLexer::EOF
-          elsif src.scan(/\_\w*/) then
-            self.token = src.matched
-            return process_token(command_state)
+          elsif scan(/\_\w*/) then
+            self.token = matched
+            return process_token command_state, last_state
           end
         end
       end # END OF CASE
 
-      if src.scan(/\004|\032|\000/) || src.eos? then # ^D, ^Z, EOF
+      if scan(/\004|\032|\000/) || end_of_stream? then # ^D, ^Z, EOF
         return RubyLexer::EOF
       else # alpha check
-        rb_compile_error "Invalid char #{src.rest[0].chr} in expression" unless
-          src.check IDENT_RE
+        rb_compile_error "Invalid char #{ss.rest[0].chr} in expression" unless
+          check IDENT
       end
 
-      self.token = src.matched if self.src.scan IDENT_RE
+      self.token = matched if self.scan IDENT
 
-      return process_token(command_state)
+      return process_token command_state, last_state
     end
   end
 
   def yylex_paren18
     self.command_start = true
-    result = :tLPAREN2
+    token = :tLPAREN2
 
     if in_lex_state? :expr_beg, :expr_mid then
-      result = :tLPAREN
+      token = :tLPAREN
     elsif space_seen then
       if in_lex_state? :expr_cmdarg then
-        result = :tLPAREN_ARG
+        token = :tLPAREN_ARG
       elsif in_lex_state? :expr_arg then
-        self.tern.push false
         warning "don't put space before argument parentheses"
       end
     else
-      self.tern.push false
+      # not a ternary -- do nothing?
     end
 
-    result
+    token
   end
 
   def yylex_paren19
@@ -1361,146 +1333,7 @@ class RubyLexer
     end
   end
 
-  def is_arg?
-    in_lex_state? :expr_arg, :expr_cmdarg
-  end
-
-  def is_end?
-    in_lex_state? :expr_end, :expr_endarg, :expr_endfn
-  end
-
-  def is_beg?
-    in_lex_state? :expr_beg, :expr_value, :expr_mid, :expr_class
-  end
-
-  # TODO #define IS_AFTER_OPERATOR() IS_lex_state(EXPR_FNAME | EXPR_DOT)
-
-  def is_space_arg? c = "x"
-    is_arg? and space_seen and c !~ /\s/
-  end
-
-  def is_label_possible? command_state
-    (in_lex_state?(:expr_beg) && !command_state) || is_arg?
-  end
-
-  def process_token(command_state)
-    token << src.matched if token =~ IDENT_RE && src.scan(/[\!\?](?!=)/)
-
-    result = nil
-    last_state = lex_state
-
-    case token
-    when /^\$/ then
-      self.lex_state, result = :expr_end, :tGVAR
-    when /^@@/ then
-      self.lex_state, result = :expr_end, :tCVAR
-    when /^@/ then
-      self.lex_state, result = :expr_end, :tIVAR
-    else
-      if token =~ /[!?]$/ then
-        result = :tFID
-      else
-        if in_lex_state? :expr_fname then
-          # ident=, not =~ => == or followed by =>
-          # TODO test lexing of a=>b vs a==>b
-          if src.scan(/=(?:(?![~>=])|(?==>))/) then
-            result = :tIDENTIFIER
-            token << src.matched
-          end
-        end
-
-        result ||= if token =~ /^[A-Z]/ then
-                     :tCONSTANT
-                   else
-                     :tIDENTIFIER
-                   end
-      end
-
-      unless ruby18
-        if is_label_possible? command_state then
-          colon = src.scan(/:/)
-
-          if colon && src.peek(1) != ":" then
-            self.lex_state = :expr_beg
-            self.yacc_value = [token, src.lineno]
-            return :tLABEL
-          end
-
-          src.unscan if colon
-        end
-      end
-
-      unless in_lex_state? :expr_dot then
-        # See if it is a reserved word.
-        keyword = if ruby18 then # REFACTOR need 18/19 lexer subclasses
-                    RubyParserStuff::Keyword.keyword18 token
-                  else
-                    RubyParserStuff::Keyword.keyword19 token
-                  end
-
-        if keyword then
-          state           = lex_state
-          self.lex_state  = keyword.state
-          self.yacc_value = [token, src.lineno]
-
-          if state == :expr_fname then
-            self.yacc_value = keyword.name
-            return keyword.id0
-          end
-
-          self.command_start = true if lex_state == :expr_beg
-
-          if keyword.id0 == :kDO then
-            if lpar_beg && lpar_beg == paren_nest then
-              self.lpar_beg = nil
-              self.paren_nest -= 1
-
-              return :kDO_LAMBDA
-            end
-
-            return :kDO_COND  if cond.is_in_state
-            return :kDO_BLOCK if cmdarg.is_in_state && state != :expr_cmdarg
-            return :kDO_BLOCK if [:expr_beg, :expr_endarg].include? state
-            return :kDO
-          end
-
-          return keyword.id0 if [:expr_beg, :expr_value].include? state
-
-          self.lex_state = :expr_beg if keyword.id0 != keyword.id1
-
-          return keyword.id1
-        end
-      end
-
-      # TODO:
-      # if (mb == ENC_CODERANGE_7BIT && lex_state != EXPR_DOT) {
-
-      self.lex_state =
-        if is_beg? || is_arg? || in_lex_state?(:expr_dot) then
-          if command_state then
-            :expr_cmdarg
-          else
-            :expr_arg
-          end
-        elsif !ruby18 && in_lex_state?(:expr_fname) then
-          :expr_endfn
-        else
-          :expr_end
-        end
-
-    end
-
-    self.yacc_value = token
-
-    if (![:expr_dot, :expr_fname].include?(last_state) &&
-        self.parser.env[token.to_sym] == :lvar) then
-      self.lex_state = :expr_end
-    end
-
-    return result
-  end
-
-  def yylex_string # 23 lines
+  def yylex_string # TODO: rewrite / remove
     token = if lex_strterm[0] == :heredoc then
               self.heredoc lex_strterm
             else