ruby_parser 3.0.0 → 3.19.1

data/lib/ruby_parser_extras.rb

@@ -1,222 +1,261 @@
 # encoding: ASCII-8BIT
+# frozen_string_literal: true
+# TODO: remove encoding comment
 
-require 'stringio'
-require 'racc/parser'
-require 'sexp'
-require 'strscan'
-require 'ruby_lexer'
+require "sexp"
+require "ruby_lexer"
 require "timeout"
+require "rp_extensions"
+require "rp_stringscanner"
 
-# WHY do I have to do this?!?
-class Regexp
-  ONCE = 0 unless defined? ONCE # FIX: remove this - it makes no sense
+class Sexp
+  def check_line_numbers
+    raise "bad nil line for:\n%s" % [self.pretty_inspect] if nil_line?
+    raise "bad line number for:\n%s" % [self.pretty_inspect] unless
+      Integer === self.line &&
+      self.line >= 1 &&
+      self.line <= self.line_min
+  end
+
+  ##
+  # Returns the maximum line number of the children of self.
 
-  unless defined? ENC_NONE then
-    ENC_NONE = /x/n.options
-    ENC_EUC  = /x/e.options
-    ENC_SJIS = /x/s.options
-    ENC_UTF8 = /x/u.options
+  def line_min
+    @line_min ||= [self.deep_each.map(&:line).min, self.line].compact.min
   end
-end
 
-# I hate ruby 1.9 string changes
-class Fixnum
-  def ord
-    self
+  def nil_line?
+    self.deep_each.map(&:line).any?(&:nil?)
   end
-end unless "a"[0] == "a"
+end
 
-class RPStringScanner < StringScanner
-#   if ENV['TALLY'] then
-#     alias :old_getch :getch
-#     def getch
-#       warn({:getch => caller[0]}.inspect)
-#       old_getch
-#     end
-#   end
+module RubyParserStuff
+  VERSION = "3.19.1"
 
-  if "".respond_to? :encoding then
-    def string_to_pos
-      string.byteslice(0, pos)
-    end
+  attr_accessor :lexer, :in_def, :in_single, :file
+  attr_accessor :in_kwarg
+  attr_reader :env, :comments
 
-    def charpos
-      string_to_pos.length
-    end
-  else
-    alias :charpos :pos
+  ##
+  # Canonicalize conditionals. Eg:
+  #
+  #   not x ? a : b
+  #
+  # becomes:
+  #
+  #   x ? b : a
 
-    def string_to_pos
-      string[0..pos]
-    end
-  end
+  attr_accessor :canonicalize_conditions
 
-  def current_line # HAHA fuck you (HACK)
-    string_to_pos[/\A.*__LINE__/m].split(/\n/).size
-  end
+  ##
+  # The last token type returned from #next_token
 
-  def extra_lines_added
-    @extra_lines_added ||= 0
-  end
+  attr_accessor :last_token_type
 
-  def extra_lines_added= val
-    @extra_lines_added = val
-  end
+  $good20 = []
 
-  def lineno
-    string[0...pos].count("\n") + 1 - extra_lines_added
+  %w[
+  ].map(&:to_i).each do |n|
+    $good20[n] = n
   end
 
-  # TODO: once we get rid of these, we can make things like
-  # TODO: current_line and lineno much more accurate and easy to do
+  def debug20 n, v = nil, r = nil
+    raise "not yet #{n} #{v.inspect} => #{r.inspect}" unless $good20[n]
+  end
 
-  def unread_many str # TODO: remove this entirely - we should not need it
-    warn({:unread_many => caller[0]}.inspect) if ENV['TALLY']
-    self.extra_lines_added += str.count("\n")
-    begin
-      string[charpos, 0] = str
-    rescue IndexError
-      # HACK -- this is a bandaid on a dirty rag on an open festering wound
+  def self.deprecate old, new
+    define_method old do |*args|
+      warn "DEPRECATED: #{old} -> #{new} from #{caller.first}"
+      send new, *args
     end
   end
 
-  if ENV['DEBUG'] then
-    alias :old_getch :getch
-    def getch
-      c = self.old_getch
-      p :getch => [c, caller.first]
-      c
-    end
+  ##
+  # for pure ruby systems only
+
+  def do_parse
+    _racc_do_parse_rb(_racc_setup, false)
+  end if ENV["PURE_RUBY"] || ENV["CHECK_LINE_NUMS"]
 
-    alias :old_scan :scan
-    def scan re
-      s = old_scan re
-      d :scan => [s, caller.first] if s
-      s
+  if ENV["CHECK_LINE_NUMS"] then
+    def _racc_do_reduce arg, act
+      x = super
+
+      @racc_vstack.grep(Sexp).each do |sexp|
+        sexp.check_line_numbers
+      end
+      x
     end
   end
 
-  def d o
-    $stderr.puts o.inspect
-  end
-end
+  ARG_TYPES = [:arglist, :call_args, :array, :args].map { |k|
+    [k, true]
+  }.to_h
 
-module RubyParserStuff
-  VERSION = '3.0.0' unless constants.include? "VERSION" # SIGH
+  has_enc = "".respond_to? :encoding
 
-  attr_accessor :lexer, :in_def, :in_single, :file
-  attr_reader :env, :comments
+  # This is in sorted order of occurrence according to
+  # charlock_holmes against 500k files, with UTF_8 forced
+  # to the top.
+  #
+  # Overwrite this contstant if you need something different.
+  ENCODING_ORDER = [
+    Encoding::UTF_8, # moved to top to reflect default in 2.0
+    Encoding::ISO_8859_1,
+    Encoding::ISO_8859_2,
+    Encoding::ISO_8859_9,
+    Encoding::SHIFT_JIS,
+    Encoding::WINDOWS_1252,
+    Encoding::EUC_JP
+  ] if has_enc
 
-  def syntax_error msg
-    raise RubyParser::SyntaxError, msg
-  end
+  JUMP_TYPE = [:return, :next, :break, :yield].map { |k| [k, true] }.to_h
 
-  def arg_add(node1, node2) # TODO: nuke
-    return s(:arglist, node2) unless node1
+  TAB_WIDTH = 8
 
-    node1[0] = :arglist if node1[0] == :array
-    return node1 << node2 if node1[0] == :arglist
+  def initialize(options = {})
+    super()
 
-    return s(:arglist, node1, node2)
-  end
+    v = self.class.name[/[23]\d/]
+    raise "Bad Class name #{self.class}" unless v
 
-  def arg_blk_pass node1, node2 # TODO: nuke
-    node1 = s(:arglist, node1) unless [:arglist, :array].include? node1.first
-    node1 << node2 if node2
-    node1
-  end
+    self.lexer = RubyLexer.new v && v.to_i
+    self.lexer.parser = self
+    self.in_kwarg = false
 
-  def arg_concat node1, node2 # TODO: nuke
-    raise "huh" unless node2
-    node1 << s(:splat, node2).compact
-    node1
-  end
+    @env = RubyParserStuff::Environment.new
+    @comments = []
 
-  def clean_mlhs sexp
-    case sexp.sexp_type
-    when :masgn then
-      if sexp.size == 2 and sexp[1].sexp_type == :array then
-        s(:masgn, *sexp[1][1..-1].map { |sub| clean_mlhs sub })
-      else
-        sexp
-      end
-    when :gasgn, :iasgn, :lasgn, :cvasgn then
-      if sexp.size == 2 then
-        sexp.last
-      else
-        sexp # optional value
-      end
-    else
-      raise "unsupported type: #{sexp.inspect}"
-    end
-  end
+    @canonicalize_conditions = true
 
-  def block_var *args
-    result = self.args args
-    result[0] = :masgn
-    result
+    self.reset
   end
 
-  def block_var18 ary, splat, block
-    ary ||= s(:array)
+  def arg_concat node1, node2 # TODO: nuke
+    raise "huh" unless node2
 
-    if splat then
-      splat = splat[1] unless Symbol === splat
-      ary << "*#{splat}".to_sym
-    end
+    splat = s(:splat, node2)
+    splat.line node2.line
 
-    ary << "&#{block[1]}".to_sym if block
+    node1 << splat
+  end
 
-    if ary.length > 2 or ary.splat then # HACK
-      s(:masgn, *ary[1..-1])
-    else
-      ary.last
-    end
+  def argl x
+    x = s(:arglist, x) if x and x.sexp_type == :array
+    x
   end
 
   def args args
     result = s(:args)
 
+    ss = args.grep Sexp
+    if ss.empty? then
+      result.line lexer.lineno
+    else
+      result.line ss.first.line
+    end
+
     args.each do |arg|
+      if arg.instance_of? Array and arg.size == 2 and arg.last.is_a? Numeric then
+        arg = arg.first
+      end
+
       case arg
       when Sexp then
         case arg.sexp_type
-        when :args, :block, :array then
-          result.concat arg[1..-1]
+        when :args, :block, :array, :call_args then # HACK call_args mismatch
+          rest = arg.sexp_body
+
+          rest.map! { |x|
+            if x.instance_of? Array and x.size == 2 and Numeric === x.last then
+              x.first
+            else
+              x
+            end
+          }
+
+          result.concat rest
+        when :forward_args then
+          self.env[:*]  = :lvar # TODO: arg_var(p, idFWD_REST) ?
+          self.env[:**] = :lvar
+          self.env[:&]  = :lvar
+
+          result << arg
         when :block_arg then
           result << :"&#{arg.last}"
-        when :masgn then
+        when :shadow then
+          name = arg.last
+          self.env[name] = :lvar
+          if Sexp === result.last and result.last.sexp_type == :shadow then
+            result.last << name
+          else
+            result << arg
+          end
+        when :masgn, :block_pass, :hash then # HACK: remove. prolly call_args
           result << arg
         else
-          raise "unhandled: #{arg.inspect}"
+          raise "unhandled: #{arg.sexp_type} in #{args.inspect}"
         end
       when Symbol then
+        name = arg.to_s.delete("&*")
+        self.env[name.to_sym] = :lvar unless name.empty?
         result << arg
-      when ",", nil then
+      when true, false then
+        self.in_kwarg = arg
+      when ",", "|", ";", "(", ")", nil then
         # ignore
       else
-        raise "unhandled: #{arg.inspect}"
+        raise "unhandled: #{arg.inspect} in #{args.inspect}"
       end
     end
 
     result
   end
 
-  def aryset receiver, index
-    index ||= []
-    s(:attrasgn, receiver, :"[]=", *index[1..-1])
+  def end_args args
+    lexer.lex_state     = RubyLexer::State::Values::EXPR_BEG
+    lexer.command_start = true
+    self.args args
   end
 
-  def assignable(lhs, value = nil)
-    id = lhs.to_sym
-    id = id.to_sym if Sexp === id
+  def attrset_id? id
+    id =~ /^\[\]=$|^\w+=$/
+  end
 
-    raise "write a test 1" if id.to_s =~ /^(?:self|nil|true|false|__LINE__|__FILE__)$/
+  def endless_method_name defn_or_defs
+    name = defn_or_defs[1]
+    name = defn_or_defs[2] unless Symbol === name
 
-    raise SyntaxError, "Can't change the value of #{id}" if
-      id.to_s =~ /^(?:self|nil|true|false|__LINE__|__FILE__)$/
+    if attrset_id? name then
+      yyerror "setter method cannot be defined in an endless method definition"
+    end
 
-    result = case id.to_s
+    # TODO? token_info_drop(p, "def", loc->beg_pos);
+  end
+
+  def array_to_hash array
+    case array.sexp_type
+    when :kwsplat then
+      array
+    else
+      s(:hash, *array.sexp_body).line array.line
+    end
+  end
+
+  def aryset receiver, index
+    index ||= s()
+    l = receiver.line
+    result = s(:attrasgn, receiver, :"[]=",
+               *index.sexp_body).compact # [].sexp_body => nil
+    result.line = l
+    result
+  end
+
+  def assignable(lhs, value = nil)
+    id, line = lhs
+    id = id.to_sym
+
+    result = case id
              when /^@@/ then
                asgn = in_def || in_single > 0
                s((asgn ? :cvasgn : :cvdecl), id)
@@ -228,104 +267,231 @@ module RubyParserStuff
                s(:cdecl, id)
              else
                case self.env[id]
-               when :lvar then
+               when :lvar, :dvar, nil then
                  s(:lasgn, id)
-               when :dvar, nil then
-                 if self.env.current[id] == :dvar then
-                   s(:lasgn, id)
-                 elsif self.env[id] == :dvar then
-                   self.env.use(id)
-                   s(:lasgn, id)
-                 elsif ! self.env.dynamic? then
-                   s(:lasgn, id)
-                 else
-                   s(:lasgn, id)
-                 end
                else
                  raise "wtf? unknown type: #{self.env[id]}"
                end
              end
 
-    self.env[id] ||= :lvar
+    self.env[id] ||= :lvar if result.sexp_type == :lasgn
 
     result << value if value
+    result.line line
+    result
+  end
 
-    return result
+  def backref_assign_error ref
+    # TODO: need a test for this... obviously
+    case ref.sexp_type
+    when :nth_ref then
+      raise "write a test 2"
+      raise SyntaxError, "Can't set variable %p" % ref.last
+    when :back_ref then
+      raise "write a test 3"
+      raise SyntaxError, "Can't set back reference %p" % ref.last
+    else
+      raise "Unknown backref type: #{ref.inspect}"
+    end
   end
 
   def block_append(head, tail)
     return head if tail.nil?
     return tail if head.nil?
 
-    case head[0]
-    when :lit, :str then
-      return tail
-    end
-
     line = [head.line, tail.line].compact.min
 
     head = remove_begin(head)
-    head = s(:block, head) unless head.node_type == :block
+    head = s(:block, head).line(line) unless head.sexp_type == :block
 
-    head.line = line
+    # head.line = line
     head << tail
   end
 
+  def block_dup_check call_or_args, block
+    syntax_error "Both block arg and actual block given." if
+      block and call_or_args.block_pass?
+  end
+
+  def block_var *args
+    result = self.args args
+    result.sexp_type = :masgn
+    result
+  end
+
+  def call_args args
+    result = s(:call_args)
+
+    a = args.grep(Sexp).first
+    if a then
+      result.line a.line
+    else
+      result.line lexer.lineno
+    end
+
+    args.each do |arg|
+      if arg.instance_of? Array and arg.size == 2 and arg.last.is_a? Numeric then
+        arg = arg.first
+      end
+
+      case arg
+      when Sexp then
+        case arg.sexp_type
+        when :array, :args, :call_args then # HACK? remove array at some point
+          result.concat arg.sexp_body
+        else
+          result << arg
+        end
+      when Symbol then
+        result << arg
+      when Array then
+        id, _line = arg
+        result << id
+      when ",", nil, "(" then
+        # ignore
+      else
+        raise "unhandled: #{arg.inspect} in #{args.inspect}"
+      end
+    end
+
+    result
+  end
+
+  def clean_mlhs sexp
+    case sexp.sexp_type
+    when :masgn then
+      if sexp.size == 2 and sexp[1].sexp_type == :array then
+        s(:masgn, *sexp[1].sexp_body.map { |sub| clean_mlhs sub })
+      else
+        debug20 5
+        sexp
+      end
+    when :gasgn, :iasgn, :lasgn, :cvasgn then
+      if sexp.size == 2 then
+        sexp.last
+      else
+        debug20 7
+        sexp # optional value
+      end
+    else
+      raise "unsupported type: #{sexp.inspect}"
+    end
+  end
+
   def cond node
     return nil if node.nil?
     node = value_expr node
 
-    case node.first
+    case node.sexp_type
     when :lit then
       if Regexp === node.last then
-        return s(:match, node)
+        s(:match, node)
       else
-        return node
+        node
       end
     when :and then
-      return s(:and, cond(node[1]), cond(node[2]))
+      _, lhs, rhs = node
+      s(:and,  cond(lhs), cond(rhs))
     when :or then
-      return s(:or,  cond(node[1]), cond(node[2]))
+      _, lhs, rhs = node
+      s(:or,  cond(lhs), cond(rhs))
     when :dot2 then
       label = "flip#{node.hash}"
       env[label] = :lvar
-      return s(:flip2, node[1], node[2])
+      _, lhs, rhs = node
+      s(:flip2, lhs, rhs) # TODO: recurse?
     when :dot3 then
       label = "flip#{node.hash}"
       env[label] = :lvar
-      return s(:flip3, node[1], node[2])
+      _, lhs, rhs = node
+      s(:flip3, lhs, rhs)
     else
-      return node
-    end
+      node
+    end.line node.line
   end
 
-  ##
-  # for pure ruby systems only
+  def dedent sexp
+    dedent_count = dedent_size sexp
 
-  def do_parse
-    _racc_do_parse_rb(_racc_setup, false)
-  end if ENV['PURE_RUBY']
+    skip_one = false
+    sexp.map { |obj|
+      case obj
+      when Symbol then
+        obj
+      when String then
+        obj.lines.map { |l| remove_whitespace_width l, dedent_count }.join
+      when Sexp then
+        case obj.sexp_type
+        when :evstr then
+          skip_one = true
+          obj
+        when :str then
+          _, str = obj
+          str = if skip_one then
+                  skip_one = false
+                  s1, *rest = str.lines
+                  s1 + rest.map { |l| remove_whitespace_width l, dedent_count }.join
+                else
+                  str.lines.map { |l| remove_whitespace_width l, dedent_count }.join
+                end
 
-  def get_match_node lhs, rhs # TODO: rename to new_match
-    if lhs then
-      case lhs[0]
-      when :dregx, :dregx_once then
-        return s(:match2, lhs, rhs).line(lhs.line)
-      when :lit then
-        return s(:match2, lhs, rhs).line(lhs.line) if Regexp === lhs.last
+          s(:str, str).line obj.line
+        else
+          warn "unprocessed sexp %p" % [obj]
+        end
+      else
+        warn "unprocessed: %p" % [obj]
       end
-    end
+    }
+  end
 
-    if rhs then
-      case rhs[0]
-      when :dregx, :dregx_once then
-        return s(:match3, rhs, lhs).line(lhs.line)
-      when :lit then
-        return s(:match3, rhs, lhs).line(lhs.line) if Regexp === rhs.last
+  def dedent_size sexp
+    skip_one = false
+    sexp.flat_map { |s|
+      case s
+      when Symbol then
+        next
+      when String then
+        s.lines
+      when Sexp then
+        case s.sexp_type
+        when :evstr then
+          skip_one = true
+          next
+        when :str then
+          _, str = s
+          lines = str.lines
+          if skip_one then
+            skip_one = false
+            lines.shift
+          end
+          lines
+        else
+          warn "unprocessed sexp %p" % [s]
+        end
+      else
+        warn "unprocessed: %p" % [s]
+      end.map { |l| whitespace_width l.chomp }
+    }.compact.min
+  end
+
+  def dedent_string string, width
+    characters_skipped = 0
+    indentation_skipped = 0
+
+    string.chars.each do |char|
+      break if indentation_skipped >= width
+      if char == " "
+        characters_skipped += 1
+        indentation_skipped += 1
+      elsif char == "\t"
+        proposed = TAB_WIDTH * (indentation_skipped / TAB_WIDTH + 1)
+        break if proposed > width
+        characters_skipped += 1
+        indentation_skipped = proposed
       end
     end
-
-    return new_call(lhs, :"=~", argl(rhs)).line(lhs.line)
+    string[characters_skipped..-1]
   end
 
   def gettable(id)
@@ -354,76 +520,151 @@ module RubyParserStuff
     result
   end
 
+  def hack_encoding str, extra = nil
+    encodings = ENCODING_ORDER.dup
+    encodings.unshift(extra) unless extra.nil?
+
+    # terrible, horrible, no good, very bad, last ditch effort.
+    encodings.each do |enc|
+      begin
+        str.force_encoding enc
+        if str.valid_encoding? then
+          str.encode! Encoding::UTF_8
+          break
+        end
+      rescue ArgumentError # unknown encoding name
+        # do nothing
+      rescue Encoding::InvalidByteSequenceError
+        # do nothing
+      rescue Encoding::UndefinedConversionError
+        # do nothing
+      end
+    end
+
+    # no amount of pain is enough for you.
+    raise "Bad encoding. Need a magic encoding comment." unless
+      str.encoding.name == "UTF-8"
+  end
+
   ##
-  # Canonicalize conditionals. Eg:
-  #
-  #   not x ? a : b
+  # Returns a UTF-8 encoded string after processing BOMs and magic
+  # encoding comments.
   #
-  # becomes:
+  # Holy crap... ok. Here goes:
   #
-  #   x ? b : a
+  # Ruby's file handling and encoding support is insane. We need to be
+  # able to lex a file. The lexer file is explicitly UTF-8 to make
+  # things cleaner. This allows us to deal with extended chars in
+  # class and method names. In order to do this, we need to encode all
+  # input source files as UTF-8. First, we look for a UTF-8 BOM by
+  # looking at the first line while forcing its encoding to
+  # ASCII-8BIT. If we find a BOM, we strip it and set the expected
+  # encoding to UTF-8. Then, we search for a magic encoding comment.
+  # If found, it overrides the BOM. Finally, we force the encoding of
+  # the input string to whatever was found, and then encode that to
+  # UTF-8 for compatibility with the lexer.
 
-  attr_accessor :canonicalize_conditions
+  def handle_encoding str
+    str = str.dup
+    has_enc = str.respond_to? :encoding
+    encoding = nil
 
-  def initialize(options = {})
-    super()
+    header = str.each_line.first(2)
+    header.map! { |s| s.force_encoding "ASCII-8BIT" } if has_enc
 
-    v = self.class.name[/1[89]/]
-    self.lexer = RubyLexer.new v && v.to_i
-    self.lexer.parser = self
-    @env = RubyParserStuff::Environment.new
-    @comments = []
+    first = header.first || ""
+    encoding, str = +"utf-8", str.b[3..-1] if first =~ /\A\xEF\xBB\xBF/
 
-    @canonicalize_conditions = true
+    encoding = $1.strip if header.find { |s|
+      s[/^#.*?-\*-.*?coding:\s*([^ ;]+).*?-\*-/, 1] ||
+      s[/^#.*(?:en)?coding(?:\s*[:=])\s*([\w-]+)/, 1]
+    }
 
-    self.reset
+    if encoding then
+      if has_enc then
+        encoding.sub!(/utf-8-.+$/, "utf-8") # HACK for stupid emacs formats
+        hack_encoding str, encoding
+      else
+        warn "Skipping magic encoding comment"
+      end
+    else
+      # nothing specified... ugh. try to encode as utf-8
+      hack_encoding str if has_enc
+    end
+
+    str
+  end
+
+  def invert_block_call val
+    ret, iter = val
+    type, call = ret
+
+    iter.insert 1, call
+
+    ret = s(type).line ret.line
+
+    [iter, ret]
+  end
+
+  def inverted? val
+    JUMP_TYPE[val[0].sexp_type]
   end
 
   def list_append list, item # TODO: nuke me *sigh*
     return s(:array, item) unless list
-    list = s(:array, list) unless Sexp === list && list.first == :array
+    list = s(:array, list) unless Sexp === list && list.sexp_type == :array
     list << item
   end
 
   def list_prepend item, list # TODO: nuke me *sigh*
-    list = s(:array, list) unless Sexp === list && list[0] == :array
+    list = s(:array, list) unless Sexp === list && list.sexp_type == :array
     list.insert 1, item
     list
   end
 
-  def literal_concat head, tail
+  def literal_concat head, tail # TODO: ugh. rewrite
     return tail unless head
     return head unless tail
 
-    htype, ttype = head[0], tail[0]
+    htype, ttype = head.sexp_type, tail.sexp_type
 
-    head = s(:dstr, '', head) if htype == :evstr
+    head = s(:dstr, "", head).line head.line if htype == :evstr
 
     case ttype
     when :str then
       if htype == :str
-        head[-1] << tail[-1]
+        a, b = head.last, tail.last
+        b = b.dup.force_encoding a.encoding unless Encoding.compatible?(a, b)
+        a << b
       elsif htype == :dstr and head.size == 2 then
-        head[-1] << tail[-1]
+        head.last << tail.last
       else
         head << tail
       end
     when :dstr then
       if htype == :str then
-        tail[1] = head[-1] + tail[1]
+        lineno = head.line
+        tail[1] = head.last + tail[1]
         head = tail
+        head.line = lineno
       else
-        tail[0] = :array
-        tail[1] = s(:str, tail[1])
-        tail.delete_at 1 if tail[1] == s(:str, '')
+        tail.sexp_type = :array
+        tail[1] = s(:str, tail[1]).line tail.line
+        tail.delete_at 1 if tail[1] == s(:str, "")
 
-        head.push(*tail[1..-1])
+        head.push(*tail.sexp_body)
       end
     when :evstr then
-      head[0] = :dstr if htype == :str
-      if head.size == 2 and tail.size > 1 and tail[1][0] == :str then
-        head[-1] << tail[1][-1]
-        head[0] = :str if head.size == 2 # HACK ?
+      if htype == :str then
+        f, l = head.file, head.line
+        head = s(:dstr, *head.sexp_body)
+        head.file = f
+        head.line = l
+      end
+
+      if head.size == 2 and tail.size > 1 and tail[1].sexp_type == :str then
+        head.last << tail[1].last
+        head.sexp_type = :str if head.size == 2 # HACK ?
       else
         head.push(tail)
       end
@@ -435,109 +676,238 @@ module RubyParserStuff
     return head
   end
 
-  def logop(type, left, right) # TODO: rename logical_op
+  def local_pop in_def
+    lexer.cond.pop # group = local_pop
+    lexer.cmdarg.pop
+    self.env.unextend
+    self.in_def = in_def
+  end
+
+  def logical_op type, left, right
     left = value_expr left
 
-    if left and left[0] == type and not left.paren then
-      node, second = left, nil
+    if left and left.sexp_type == type and not left.paren then
+      node, rhs = left, nil
 
-      while (second = node[2]) && second[0] == type and not second.paren do
-        node = second
+      loop do
+        _, _lhs, rhs = node
+        break unless rhs && rhs.sexp_type == type and not rhs.paren
+        node = rhs
       end
 
-      node[2] = s(type, second, right)
+      node.pop
+      node << s(type, rhs, right).line(rhs.line)
 
       return left
     end
 
-    return s(type, left, right)
+    result = s(type, left, right)
+    result.line left.line if left.line
+    result
   end
 
   def new_aref val
     val[2] ||= s(:arglist)
-    val[2][0] = :arglist if val[2][0] == :array # REFACTOR
-    if val[0].node_type == :self then
-      result = new_call nil, :"[]", val[2]
+    val[2].sexp_type = :arglist if val[2].sexp_type == :array # REFACTOR
+    new_call val[0], :"[]", val[2]
+  end
+
+  def new_arg val
+    arg, = val
+
+    case arg
+    when Symbol then
+      result = s(:args, arg).line line
+    when Sexp then
+      result = arg
+    when Array then
+      (arg, line), = val
+      result = s(:args, arg).line line
+    else
+      debug20 32
+      raise "Unknown f_arg type: #{val.inspect}"
+    end
+
+    result
+  end
+
+  def new_array_pattern const, pre_arg, arypat, loc
+    result = s(:array_pat, const).line loc
+    result << pre_arg if pre_arg
+
+    if arypat && arypat.sexp_type == :array_TAIL then
+      result.concat arypat.sexp_body
+    else
+      raise "NO?: %p" % [arypat]
+    end
+
+    result
+  end
+
+  def array_pat_concat lhs, rhs
+    case lhs.sexp_type
+    when :PATTERN then
+      lhs.sexp_type = :array_pat
+    end
+
+    if rhs then
+      case rhs.sexp_type
+      when :array_pat, :array_TAIL, :PATTERN then
+        lhs.concat rhs.sexp_body
+      else
+        lhs << rhs
+      end
+    end
+  end
+
+  def new_array_pattern_tail pre_args, has_rest, rest_arg, post_args
+    # TODO: remove has_rest once all tests pass !!!
+    rest_arg = if has_rest then
+                 :"*#{rest_arg}"
+               else
+                 nil
+               end
+
+    result = s(:array_TAIL).line 666
+
+    array_pat_concat result, pre_args
+
+    result << rest_arg if rest_arg
+
+    array_pat_concat result, post_args
+
+    result
+  end
+
+  def new_assign lhs, rhs
+    return nil unless lhs
+
+    rhs = value_expr rhs
+
+    case lhs.sexp_type
+    when :lasgn, :iasgn, :cdecl, :cvdecl, :gasgn, :cvasgn, :attrasgn, :safe_attrasgn then
+      lhs << rhs
+    when :const then
+      lhs.sexp_type = :cdecl
+      lhs << rhs
     else
-      result = new_call val[0], :"[]", val[2]
+      raise "unknown lhs #{lhs.inspect} w/ #{rhs.inspect}"
     end
+
+    lhs
+  end
+
+  def new_attrasgn recv, meth, call_op = :"."
+    call_op = call_op.first if Array === call_op
+
+    meth = :"#{meth}="
+
+    result = case call_op.to_sym
+             when :"."
+               s(:attrasgn, recv, meth)
+             when :"&."
+               s(:safe_attrasgn, recv, meth)
+             else
+               raise "unknown call operator: `#{type.inspect}`"
+             end
+
+    result.line = recv.line
+    result
+  end
+
+  def new_begin val
+    _, lineno, body, _ = val
+
+    result = body ? s(:begin, body) : s(:nil)
+    result.line lineno
+
     result
   end
 
   def new_body val
-    result = val[0]
+    body, resbody, elsebody, ensurebody = val
+
+    result = body
 
-    if val[1] then
+    if resbody then
       result = s(:rescue)
-      result << val[0] if val[0]
+      result << body if body
 
-      resbody = val[1]
+      res = resbody
 
-      while resbody do
-        result << resbody
-        resbody = resbody.resbody(true)
+      while res do
+        result << res
+        res = res.resbody(true)
       end
 
-      result << val[2] if val[2]
+      result << elsebody if elsebody
+
+      result.line = (body || resbody).line
+    end
 
-      result.line = (val[0] || val[1]).line
-    elsif not val[2].nil? then
+    if elsebody and not resbody then
       warning("else without rescue is useless")
-      result = block_append(result, val[2])
+      result = s(:begin, result).line result.line if result
+      result = block_append(result, elsebody)
     end
 
-    result = s(:ensure, result, val[3]).compact if val[3]
-    return result
-  end
+    if ensurebody
+      lineno = (result || ensurebody).line
+      result = s(:ensure, result, ensurebody).compact.line lineno
+    end
 
-  def argl x
-    x = s(:arglist, x) if x and x[0] != :arglist
-    x
+    result
   end
 
-  def backref_assign_error ref
-    # TODO: need a test for this... obviously
-    case ref.first
-    when :nth_ref then
-      raise "write a test 2"
-      raise SyntaxError, "Can't set variable %p" % ref.last
-    when :back_ref then
-      raise "write a test 3"
-      raise SyntaxError, "Can't set back reference %p" % ref.last
-    else
-      raise "Unknown backref type: #{ref.inspect}"
-    end
+  def new_brace_body args, body, lineno
+    new_iter(nil, args, body).line lineno
   end
 
-  def new_call recv, meth, args = nil
-    result = s(:call, recv, meth)
-    result.line = recv.line if recv
+  def new_call recv, meth, args = nil, call_op = :"."
+    call_op = call_op.first if Array === call_op
+
+    result = case call_op.to_sym
+             when :"."
+               s(:call, recv, meth)
+             when :"&."
+               s(:safe_call, recv, meth)
+             else
+               raise "unknown call operator: `#{type.inspect}`"
+             end
 
     # TODO: need a test with f(&b) to produce block_pass
     # TODO: need a test with f(&b) { } to produce warning
 
-    args ||= s(:arglist)
-    args[0] = :arglist if args.first == :array
-    args = s(:arglist, args) unless args.first == :arglist
+    if args
+      if ARG_TYPES[args.sexp_type] then
+        result.concat args.sexp_body
+      else
+        result << args
+      end
+    end
 
-    # HACK quick hack to make this work quickly... easy to clean up above
-    result.concat args[1..-1]
+    # line = result.grep(Sexp).map(&:line).compact.min
+    result.line = recv.line if recv
+    result.line ||= lexer.lineno
 
     result
   end
 
-  def new_case expr, body
+  def new_in pat, body, cases, line
+    s(:in, pat, body, cases).line line
+  end
+
+  def new_case expr, body, line
     result = s(:case, expr)
-    line = (expr || body).line
 
-    while body and body.node_type == :when
+    while body and [:when, :in].include? body.sexp_type
       result << body
       body = body.delete_at 3
     end
 
     result[2..-1].each do |node|
       block = node.block(:delete)
-      node.concat block[1..-1] if block
+      node.concat block.sexp_body if block
     end
 
     # else
@@ -549,13 +919,16 @@ module RubyParserStuff
   end
 
   def new_class val
+    # TODO: get line from class keyword
     line, path, superclass, body = val[1], val[2], val[3], val[5]
 
+    path = path.first if path.instance_of? Array
+
     result = s(:class, path, superclass)
 
     if body then
-      if body.first == :block then
-        result.push(*body[1..-1])
+      if body.sexp_type == :block then
+        result.push(*body.sexp_body)
       else
         result.push body
       end
@@ -572,53 +945,188 @@ module RubyParserStuff
     result
   end
 
+  def new_const_op_asgn val
+    lhs, (asgn_op, _), rhs = val
+    asgn_op = asgn_op.to_sym
+
+    result = case asgn_op
+             when :"||" then
+               s(:op_asgn_or, lhs, rhs)
+             when :"&&" then
+               s(:op_asgn_and, lhs, rhs)
+             else
+               s(:op_asgn, lhs, asgn_op, rhs)
+             end
+
+    result.line = lhs.line
+    result
+  end
+
   def new_defn val
-    (_, line), name, args, body = val[0], val[1], val[3], val[4]
-    body ||= s(:nil)
+    _, (name, line), in_def, args, body, _ = val
 
-    result = s(:defn, name.to_sym, args)
+    body ||= s(:nil).line line
 
-    if body then
-      if body.first == :block then
-        result.push(*body[1..-1])
+    args.line line
+
+    result = s(:defn, name.to_sym, args).line line
+
+    if body.sexp_type == :block then
+      result.push(*body.sexp_body)
+    else
+      result.push body
+    end
+
+    result.comments = self.comments.pop
+
+    [result, in_def]
+  end
+
+  def new_endless_defn val
+    (name, line, in_def), args, _, body, _, resbody = val
+
+    result =
+      if resbody then
+        s(:defn, name, args,
+          new_rescue(body,
+                     new_resbody(s(:array).line(line),
+                                 resbody))).line line
       else
-        result.push body
+        s(:defn, name, args, body).line line
       end
-    end
 
-    result.line = line
+    local_pop in_def
+    endless_method_name result
+
     result.comments = self.comments.pop
+
     result
   end
 
-  def new_defs val
-    recv, name, args, body = val[1], val[4], val[6], val[7]
-
-    result = s(:defs, recv, name.to_sym, args)
+  def new_endless_defs val
+    (recv, (name, line, in_def)), args, _, body, _, resbody = val
 
-    if body then
-      if body.first == :block then
-        result.push(*body[1..-1])
+    result =
+      if resbody then
+        s(:defs, recv, name, args,
+          new_rescue(body,
+                     new_resbody(s(:array).line(line),
+                                 resbody))).line line
       else
-        result.push body
+        s(:defs, recv, name, args, body).line(line)
       end
-    end
 
-    result.line = recv.line
+    self.in_single -= 1
+    local_pop in_def
+    endless_method_name result
+
     result.comments = self.comments.pop
+
     result
   end
 
+  def new_defs val
+    _, recv, (name, line), in_def, args, body, _ = val
+
+    body ||= s(:nil).line line
+
+    args.line line
+
+    result = s(:defs, recv, name.to_sym, args).line line
+
+    # TODO: remove_begin
+    # TODO: reduce_nodes
+
+    if body.sexp_type == :block then
+      result.push(*body.sexp_body)
+    else
+      result.push body
+    end
+
+    result.comments = self.comments.pop
+
+    [result, in_def]
+  end
+
+  def new_do_body args, body, lineno
+    new_iter(nil, args, body).line(lineno)
+  end
+
+  def new_find_pattern const, pat
+    pat.sexp_type = :find_pat
+    pat.insert 1, const
+  end
+
+  def new_find_pattern_tail lhs, mid, rhs
+    lhs_id, line = lhs
+    rhs_id, _line = rhs
+
+    # TODO: fpinfo->pre_rest_arg = pre_rest_arg ? assignable(p, pre_rest_arg, 0, loc) : NODE_SPECIAL_NO_NAME_REST;
+
+    lhs_id = "*#{lhs_id}".to_sym
+    rhs_id = "*#{rhs_id}".to_sym
+
+    mid.sexp_type = :array_pat # HACK?
+
+    s(:find_pat_TAIL, lhs_id, mid, rhs_id).line line
+  end
+
   def new_for expr, var, body
     result = s(:for, expr, var).line(var.line)
     result << body if body
     result
   end
 
+  def new_hash val
+    _, line, assocs = val
+
+    s(:hash).line(line).concat assocs.sexp_body
+  end
+
+  def new_hash_pattern const, hash_pat, loc
+    _, pat, kw_args, kw_rest_arg = hash_pat
+
+    line = (const||hash_pat).line
+
+    result = s(:hash_pat, const).line line
+    result.concat pat.sexp_body if pat
+    result << kw_args     if kw_args
+    result << kw_rest_arg if kw_rest_arg
+    result
+  end
+
+  def new_hash_pattern_tail kw_args, kw_rest_arg, line # TODO: remove line arg
+    # kw_rest_arg = assignable(kw_rest_arg, nil).line line if kw_rest_arg
+
+    result = s(:hash_pat).line line
+    result << kw_args
+
+    if kw_rest_arg then
+      name = kw_rest_arg.value
+      # TODO: I _hate_ this:
+      assignable [name, kw_rest_arg.line] if name != :**
+      result << kw_rest_arg
+    end
+
+    result
+  end
+
+  def push_pktbl
+  end
+
+  def pop_pktbl
+  end
+
+  def push_pvtbl
+  end
+
+  def pop_pvtbl
+  end
+
   def new_if c, t, f
     l = [c.line, t && t.line, f && f.line].compact.min
     c = cond c
-    c, t, f = c.last, f, t if c[0] == :not and canonicalize_conditions
+    c, t, f = c.last, f, t if c.sexp_type == :not and canonicalize_conditions
     s(:if, c, t, f).line(l)
   end
 
@@ -633,98 +1141,198 @@ module RubyParserStuff
     result << args
     result << body if body
 
-    args[0] = :args unless args == 0
+    result.line call.line if call
+
+    unless args == 0 then
+      args.line call.line if call
+      args.sexp_type = :args
+    end
 
     result
   end
 
   def new_masgn lhs, rhs, wrap = false
+    _, ary = lhs
+
+    line = rhs.line
     rhs = value_expr(rhs)
-    rhs = lhs[1] ? s(:to_ary, rhs) : s(:array, rhs) if wrap
+    rhs = ary ? s(:to_ary, rhs) : s(:array, rhs) if wrap
+    rhs.line line if wrap
 
-    lhs.delete_at 1 if lhs[1].nil?
+    lhs.delete_at 1 if ary.nil?
     lhs << rhs
 
     lhs
   end
 
+  def new_masgn_arg rhs, wrap = false
+    rhs = value_expr(rhs)
+    # HACK: could be array if lhs isn't right
+    rhs = s(:to_ary, rhs).line rhs.line if wrap
+    rhs
+  end
+
+  def new_match lhs, rhs
+    if lhs then
+      case lhs.sexp_type
+      when :dregx, :dregx_once then
+        # TODO: no test coverage
+        return s(:match2, lhs, rhs).line(lhs.line)
+      when :lit then
+        return s(:match2, lhs, rhs).line(lhs.line) if Regexp === lhs.last
+      end
+    end
+
+    if rhs then
+      case rhs.sexp_type
+      when :dregx, :dregx_once then
+        # TODO: no test coverage
+        return s(:match3, rhs, lhs).line(lhs.line)
+      when :lit then
+        return s(:match3, rhs, lhs).line(lhs.line) if Regexp === rhs.last
+      end
+    end
+
+    new_call(lhs, :"=~", argl(rhs)).line lhs.line
+  end
+
   def new_module val
+    # TODO: get line from module keyword
     line, path, body = val[1], val[2], val[4]
 
-    result = s(:module, path)
+    path = path.first if path.instance_of? Array
+
+    result = s(:module, path).line line
 
     if body then # REFACTOR?
-      if body.first == :block then
-        result.push(*body[1..-1])
+      if body.sexp_type == :block then
+        result.push(*body.sexp_body)
       else
         result.push body
       end
     end
 
-    result.line = line
     result.comments = self.comments.pop
     result
   end
 
   def new_op_asgn val
-    lhs, asgn_op, arg = val[0], val[1].to_sym, val[2]
-    name = lhs.value
-    arg = remove_begin(arg)
-    result = case asgn_op # REFACTOR
+    lhs, (op, _line), rhs = val
+    op = op.to_sym
+
+    name = gettable(lhs.last).line lhs.line
+    arg = remove_begin rhs
+    result = case op # REFACTOR
              when :"||" then
                lhs << arg
-               s(:op_asgn_or, self.gettable(name), lhs)
+               s(:op_asgn_or, name, lhs).line lhs.line
              when :"&&" then
                lhs << arg
-               s(:op_asgn_and, self.gettable(name), lhs)
+               s(:op_asgn_and, name, lhs).line lhs.line
              else
-               # TODO: why [2] ?
-               lhs[2] = new_call(self.gettable(name), asgn_op, argl(arg))
+               lhs << new_call(name, op, argl(arg))
                lhs
              end
-    result.line = lhs.line
+
+    result
+  end
+
+  def new_op_asgn1 val
+    lhs, _, args, _, (op, _), rhs = val
+
+    args.sexp_type = :arglist if args
+
+    result = s(:op_asgn1, lhs, args, op.to_sym, rhs)
+    result.line lhs.line
+    result
+  end
+
+  def new_op_asgn2 val
+    recv, (call_op, _), (meth, _), (op, _), arg = val
+    meth = :"#{meth}="
+
+    result = case call_op.to_sym
+             when :"."
+               s(:op_asgn2, recv, meth, op.to_sym, arg)
+             when :"&."
+               s(:safe_op_asgn2, recv, meth, op.to_sym, arg)
+             else
+               raise "unknown call operator: `#{type.inspect}`"
+             end
+
+    result.line = recv.line
     result
   end
 
+  def new_qsym_list
+    s(:array).line lexer.lineno
+  end
+
+  def new_qsym_list_entry val
+    _, (str, line), _ = val
+    s(:lit, str.to_sym).line line
+  end
+
+  def new_qword_list
+    s(:array).line lexer.lineno
+  end
+
+  def new_qword_list_entry val
+    _, (str, line), _ = val
+    str.force_encoding("ASCII-8BIT") unless str.valid_encoding?
+    s(:str, str).line line
+  end
+
   def new_regexp val
-    node = val[1] || s(:str, '')
-    options = val[2]
+    (_, line), node, (options, _) = val
+
+    node ||= s(:str, "").line line
 
     o, k = 0, nil
     options.split(//).uniq.each do |c| # FIX: this has a better home
       v = {
-        'x' => Regexp::EXTENDED,
-        'i' => Regexp::IGNORECASE,
-        'm' => Regexp::MULTILINE,
-        'o' => Regexp::ONCE,
-        'n' => Regexp::ENC_NONE,
-        'e' => Regexp::ENC_EUC,
-        's' => Regexp::ENC_SJIS,
-        'u' => Regexp::ENC_UTF8,
+        "x" => Regexp::EXTENDED,
+        "i" => Regexp::IGNORECASE,
+        "m" => Regexp::MULTILINE,
+        "o" => Regexp::ONCE,
+        "n" => Regexp::ENC_NONE,
+        "e" => Regexp::ENC_EUC,
+        "s" => Regexp::ENC_SJIS,
+        "u" => Regexp::ENC_UTF8,
       }[c]
       raise "unknown regexp option: #{c}" unless v
       o += v
-      k = c if c =~ /[esu]/
     end
 
-    case node[0]
+    case node.sexp_type
     when :str then
-      node[0] = :lit
+      node.sexp_type = :lit
       node[1] = if k then
                   Regexp.new(node[1], o, k)
                 else
-                  Regexp.new(node[1], o)
-                end rescue node[1] # HACK - drops options
+                  begin
+                    Regexp.new(node[1], o)
+                  rescue RegexpError => e
+                    warn "WARNING: #{e.message} for #{node[1].inspect} #{options.inspect}"
+                    begin
+                      warn "WARNING: trying to recover with ENC_UTF8"
+                      Regexp.new(node[1], Regexp::ENC_UTF8)
+                    rescue RegexpError => e
+                      warn "WARNING: trying to recover with ENC_NONE"
+                      Regexp.new(node[1], Regexp::ENC_NONE)
+                    end
+                  end
+                end
     when :dstr then
       if options =~ /o/ then
-        node[0] = :dregx_once
+        node.sexp_type = :dregx_once
       else
-        node[0] = :dregx
+        node.sexp_type = :dregx
       end
       node << o if o and o != 0
     else
-      node = s(:dregx, '', node);
-      node[0] = :dregx_once if options =~ /o/
+      node = s(:dregx, "", node).line line
+      node.sexp_type = :dregx_once if options =~ /o/
       node << o if o and o != 0
     end
 
@@ -732,12 +1340,17 @@ module RubyParserStuff
   end
 
   def new_resbody cond, body
-    if body && body.first == :block then
+    if body && body.sexp_type == :block then
       body.shift # remove block and splat it in directly
     else
       body = [body]
     end
-    s(:resbody, cond, *body)
+
+    s(:resbody, cond, *body).line cond.line
+  end
+
+  def new_rescue body, resbody
+    s(:rescue, body, resbody).line body.line
   end
 
   def new_sclass val
@@ -746,8 +1359,8 @@ module RubyParserStuff
     result = s(:sclass, recv)
 
     if body then
-      if body.first == :block then
-        result.push(*body[1..-1])
+      if body.sexp_type == :block then
+        result.push(*body.sexp_body)
       else
         result.push body
       end
@@ -759,20 +1372,56 @@ module RubyParserStuff
     result
   end
 
+  def new_string val
+    (str, line), = val
+
+    str.force_encoding("UTF-8")
+    # TODO: remove:
+    str.force_encoding("ASCII-8BIT") unless str.valid_encoding?
+    s(:str, str).line line
+  end
+
   def new_super args
-    if args && args.node_type == :block_pass then
-      s(:super, args)
+    if args && args.sexp_type == :block_pass then
+      s(:super, args).line args.line
+    else
+      args ||= s(:arglist).line lexer.lineno
+      s(:super, *args.sexp_body).line args.line
+    end
+  end
+
+  def new_symbol val
+    name = val.last
+    s(:lit, name.to_sym).line lexer.lineno
+  end
+
+  def new_symbol_list
+    # TODO: hunt down and try to remove ALL lexer.lineno usage!
+    s(:array).line lexer.lineno
+  end
+
+  def new_symbol_list_entry val
+    _, sym, _ = val
+
+    sym ||= s(:str, "").line lexer.lineno
+
+    case sym.sexp_type
+    when :dstr then
+      sym.sexp_type = :dsym
+    when :str then
+      sym = s(:lit, sym.last.to_sym).line sym.line
     else
-      args ||= s(:arglist)
-      s(:super, *args[1..-1])
+      sym = s(:dsym, "", sym).line sym.line
     end
+
+    sym
   end
 
   def new_undef n, m = nil
     if m then
-      block_append(n, s(:undef, m))
+      block_append(n, s(:undef, m).line(m.line))
     else
-      s(:undef, n)
+      s(:undef, n).line n.line
     end
   end
 
@@ -783,11 +1432,11 @@ module RubyParserStuff
   def new_until_or_while type, block, expr, pre
     other = type == :until ? :while : :until
     line = [block && block.line, expr.line].compact.min
-    block, pre = block.last, false if block && block[0] == :begin
+    block, pre = block.last, false if block && block.sexp_type == :begin
 
     expr = cond expr
 
-    result = unless expr.first == :not and canonicalize_conditions then
+    result = unless expr.sexp_type == :not and canonicalize_conditions then
                s(type,  expr,      block, pre)
              else
                s(other, expr.last, block, pre)
@@ -805,147 +1454,78 @@ module RubyParserStuff
     new_until_or_while :while, block, expr, pre
   end
 
-  def new_xstring str
-    if str then
-      case str[0]
+  def new_word_list
+    s(:array).line lexer.lineno
+  end
+
+  def new_word_list_entry val
+    _, word, _ = val
+    word.sexp_type == :evstr ? s(:dstr, "", word).line(word.line) : word
+  end
+
+  def new_xstring val
+    _, node = val
+
+    node ||= s(:str, "").line lexer.lineno
+
+    if node then
+      case node.sexp_type
       when :str
-        str[0] = :xstr
+        node.sexp_type = :xstr
       when :dstr
-        str[0] = :dxstr
+        node.sexp_type = :dxstr
       else
-        str = s(:dxstr, '', str)
+        node = s(:dxstr, "", node).line node.line
       end
-      str
-    else
-      s(:xstr, '')
     end
+
+    node
   end
 
   def new_yield args = nil
     # TODO: raise args.inspect unless [:arglist].include? args.first # HACK
-    raise "write a test 4" if args && args.node_type == :block_pass
+    raise "write a test 4" if args && args.sexp_type == :block_pass
     raise SyntaxError, "Block argument should not be given." if
-      args && args.node_type == :block_pass
+      args && args.sexp_type == :block_pass
 
-    args ||= s(:arglist)
+    args ||= s(:arglist).line lexer.lineno
 
-    # TODO: I can prolly clean this up
-    args[0] = :arglist       if args.first == :array
-    args = s(:arglist, args) unless args.first == :arglist
-
-    return s(:yield, *args[1..-1])
-  end
+    args.sexp_type = :arglist if [:call_args, :array].include? args.sexp_type
+    args = s(:arglist, args).line args.line unless args.sexp_type == :arglist
 
-  def next_token
-    if self.lexer.advance then
-      return self.lexer.token, self.lexer.yacc_value
-    else
-      return [false, '$end']
-    end
+    s(:yield, *args.sexp_body).line args.line
   end
 
-  def node_assign(lhs, rhs) # TODO: rename new_assign
-    return nil unless lhs
-
-    rhs = value_expr rhs
-
-    case lhs[0]
-    when :gasgn, :iasgn, :lasgn, :masgn, :cdecl, :cvdecl, :cvasgn then
-      lhs << rhs
-    when :attrasgn, :call then
-      args = lhs.pop unless Symbol === lhs.last
-      lhs.concat arg_add(args, rhs)[1..-1]
-    when :const then
-      lhs[0] = :cdecl
-      lhs << rhs
+  def prev_value_to_lineno v
+    s, n = v
+    if String === s then
+      n
     else
-      raise "unknown lhs #{lhs.inspect}"
+      lexer.lineno
     end
-
-    lhs
   end
 
-  ##
-  # Returns a UTF-8 encoded string after processing BOMs and magic
-  # encoding comments.
-  #
-  # Holy crap... ok. Here goes:
-  #
-  # Ruby's file handling and encoding support is insane. We need to be
-  # able to lex a file. The lexer file is explicitly UTF-8 to make
-  # things cleaner. This allows us to deal with extended chars in
-  # class and method names. In order to do this, we need to encode all
-  # input source files as UTF-8. First, we look for a UTF-8 BOM by
-  # looking at the first line while forcing its encoding to
-  # ASCII-8BIT. If we find a BOM, we strip it and set the expected
-  # encoding to UTF-8. Then, we search for a magic encoding comment.
-  # If found, it overrides the BOM. Finally, we force the encoding of
-  # the input string to whatever was found, and then encode that to
-  # UTF-8 for compatibility with the lexer.
-
-  def handle_encoding str
-    str = str.dup
-    ruby19 = str.respond_to? :encoding
-    encoding = nil
-
-    header = str.lines.first(2)
-    header.map! { |s| s.force_encoding "ASCII-8BIT" } if ruby19
-
-    first = header.first || ""
-    encoding, str = "utf-8", str[3..-1] if first =~ /\A\xEF\xBB\xBF/
-
-    encoding = $1.strip if header.find { |s|
-      s[/^#.*?-\*-.*?coding:\s*([^ ;]+).*?-\*-/, 1] ||
-      s[/^#.*(?:en)?coding(?:\s*[:=])\s*([\w-]+)/, 1]
-    }
+  def next_token
+    token = self.lexer.next_token
 
-    if encoding then
-      if ruby19 then
-        encoding.sub!(/utf-8-.+$/, 'utf-8') # HACK for stupid emacs formats
-        hack_encoding str, encoding
-      else
-        warn "Skipping magic encoding comment"
-      end
+    if token and token.first != RubyLexer::EOF then
+      self.last_token_type = token
+      return token
+    elsif !token
+      return self.lexer.next_token
     else
-      # nothing specified... ugh. try to encode as utf-8
-      hack_encoding str if ruby19
+      return [false, false]
     end
-
-    str
   end
 
-  def hack_encoding str, extra = nil
-    # this is in sorted order of occurrence according to
-    # charlock_holmes against 500k files
-    encodings = [
-                 extra,
-                 Encoding::ISO_8859_1,
-                 Encoding::UTF_8,
-                 Encoding::ISO_8859_2,
-                 Encoding::ISO_8859_9,
-                 Encoding::SHIFT_JIS,
-                 Encoding::WINDOWS_1252,
-                 Encoding::EUC_JP,
-                ].compact
-
-    # terrible, horrible, no good, very bad, last ditch effort.
-    encodings.each do |enc|
-      begin
-        str.force_encoding enc
-        if str.valid_encoding? then
-          str.encode! Encoding::UTF_8
-          break
-        end
-      rescue Encoding::InvalidByteSequenceError
-        # do nothing
-      rescue Encoding::UndefinedConversionError
-        # do nothing
-      end
-    end
-
-    # no amount of pain is enough for you.
-    raise "Bad encoding. Need a magic encoding comment." unless
-      str.encoding.name == "UTF-8"
+  def on_error(et, ev, values)
+    ev = ev.first if ev.instance_of?(Array) && ev.size == 2 && ev.last.is_a?(Integer)
+    super
+  rescue Racc::ParseError => e
+    # I don't like how the exception obscures the error message
+    e.message.replace "%s:%p :: %s" % [self.file, lexer.lineno, e.message.strip]
+    warn e.message if $DEBUG
+    raise
   end
 
   ##
@@ -953,56 +1533,60 @@ module RubyParserStuff
   # Timeout::Error if it runs for more than +time+ seconds.
 
   def process(str, file = "(string)", time = 10)
+    str.freeze
+
     Timeout.timeout time do
       raise "bad val: #{str.inspect}" unless String === str
 
-      str = handle_encoding str
+      self.lexer.string = handle_encoding str
 
       self.file = file.dup
-      self.lexer.src = str
 
-      @yydebug = ENV.has_key? 'DEBUG'
+      @yydebug = ENV.has_key? "DEBUG"
 
       do_parse
     end
   end
 
-  alias :parse :process
+  alias parse process
 
   def remove_begin node
-    oldnode = node
-    if node and :begin == node[0] and node.size == 2 then
-      node = node[-1]
-      node.line = oldnode.line
-    end
+    line = node.line
+
+    node = node.last while node and node.sexp_type == :begin and node.size == 2
+
+    node = s(:nil) if node == s(:begin)
+
+    node.line ||= line
+
     node
   end
 
+  alias value_expr remove_begin # TODO: for now..? could check the tree, but meh?
+
   def reset
     lexer.reset
     self.in_def = false
     self.in_single = 0
     self.env.reset
     self.comments.clear
-  end
-
-  def block_dup_check call_or_args, block
-    syntax_error "Both block arg and actual block given." if
-      block and call_or_args.block_pass?
+    self.last_token_type = nil
   end
 
   def ret_args node
     if node then
-      raise "write a test 5" if node[0] == :block_pass
+      raise "write a test 5" if node.sexp_type == :block_pass
 
       raise SyntaxError, "block argument should not be given" if
-        node[0] == :block_pass
+        node.sexp_type == :block_pass
+
+      node.sexp_type = :array if node.sexp_type == :call_args
+      node = node.last if node.sexp_type == :array && node.size == 2
 
-      node = node.last if node[0] == :array && node.size == 2
       # HACK matz wraps ONE of the FOUR splats in a newline to
       # distinguish. I use paren for now. ugh
-      node = s(:svalue, node) if node[0] == :splat and not node.paren
-      node[0] = :svalue if node[0] == :arglist && node[1][0] == :splat
+      node = s(:svalue, node).line node.line if node.sexp_type == :splat and not node.paren
+      node.sexp_type = :svalue if node.sexp_type == :arglist && node[1].sexp_type == :splat
     end
 
     node
@@ -1010,23 +1594,35 @@ module RubyParserStuff
 
   def s(*args)
     result = Sexp.new(*args)
-    result.line ||= lexer.lineno if lexer.src          # otherwise...
+    # result.line ||= lexer.lineno if lexer.ss unless ENV["CHECK_LINE_NUMS"] # otherwise...
     result.file = self.file
     result
   end
 
-  def value_expr oldnode # HACK
-    node = remove_begin oldnode
-    node.line = oldnode.line if oldnode
-    node[2] = value_expr(node[2]) if node and node[0] == :if
-    node
+  def debug n
+    if ENV["PRY"] then
+      require "pry"; binding.pry
+    end
+
+    raise RubyParser::SyntaxError, "debug #{n}"
+  end
+
+  def syntax_error msg
+    raise RubyParser::SyntaxError, msg
   end
 
+  alias yyerror syntax_error
+
   def void_stmts node
     return nil unless node
-    return node unless node[0] == :block
+    return node unless node.sexp_type == :block
+
+    if node.respond_to? :sexp_body= then
+      node.sexp_body = node.sexp_body.map { |n| remove_begin n }
+    else
+      node[1..-1] = node[1..-1].map { |n| remove_begin(n) }
+    end
 
-    node[1..-1] = node[1..-1].map { |n| remove_begin(n) }
     node
   end
 
@@ -1034,21 +1630,45 @@ module RubyParserStuff
     # do nothing for now
   end
 
-  def yyerror msg
-    warn msg
-    super()
+  def whitespace_width line, remove_width = nil
+    col = 0
+    idx = 0
+
+    line.chars.each do |c|
+      break if remove_width && col >= remove_width
+      case c
+      when " " then
+        col += 1
+      when "\t" then
+        n = TAB_WIDTH * (col / TAB_WIDTH + 1)
+        break if remove_width && n > remove_width
+        col = n
+      else
+        break
+      end
+      idx += 1
+    end
+
+    if remove_width then
+      line[idx..-1]
+    elsif line[idx].nil?
+      nil
+    else
+      col
+    end
   end
 
-  def on_error(et, ev, values)
-    super
-  rescue Racc::ParseError => e
-    # I don't like how the exception obscures the error message
-    msg = "# ERROR: %s:%p :: %s" % [self.file, lexer.lineno, e.message.strip]
-    warn msg
-    raise
+  alias remove_whitespace_width whitespace_width
+
+  def wrap type, node
+    value, line = node
+    value = value.to_sym if value.respond_to? :to_sym
+    s(type, value).line line
   end
 
   class Keyword
+    include RubyLexer::State::Values
+
     class KWtable
       attr_accessor :name, :state, :id0, :id1
       def initialize(name, id=[], state=nil)
@@ -1061,80 +1681,75 @@ module RubyParserStuff
     ##
     # :stopdoc:
     #
-    # :expr_beg    = ignore newline, +/- is a sign.
-    # :expr_end    = newline significant, +/- is a operator.
-    # :expr_arg    = newline significant, +/- is a operator.
-    # :expr_cmdarg = newline significant, +/- is a operator.
-    # :expr_endarg = newline significant, +/- is a operator.
-    # :expr_mid    = newline significant, +/- is a operator.
-    # :expr_fname  = ignore newline, no reserved words.
-    # :expr_dot    = right after . or ::, no reserved words.
-    # :expr_class  = immediate after class, no here document.
+    # :expr_beg     = ignore newline, +/- is a sign.
+    # :expr_end     = newline significant, +/- is an operator.
+    # :expr_endarg  = ditto, and unbound braces.
+    # :expr_endfn   = ditto, and unbound braces.
+    # :expr_arg     = newline significant, +/- is an operator.
+    # :expr_cmdarg  = ditto
+    # :expr_mid     = ditto
+    # :expr_fname   = ignore newline, no reserved words.
+    # :expr_dot     = right after . or ::, no reserved words.
+    # :expr_class   = immediate after class, no here document.
+    # :expr_label   = flag bit, label is allowed.
+    # :expr_labeled = flag bit, just after a label.
+    # :expr_fitem   = symbol literal as FNAME.
+    # :expr_value   = :expr_beg -- work to remove. Need multi-state support.
+
+    expr_woot = EXPR_FNAME|EXPR_FITEM
 
     wordlist = [
-                ["end",      [:kEND,      :kEND        ], :expr_end   ],
-                ["else",     [:kELSE,     :kELSE       ], :expr_beg   ],
-                ["case",     [:kCASE,     :kCASE       ], :expr_beg   ],
-                ["ensure",   [:kENSURE,   :kENSURE     ], :expr_beg   ],
-                ["module",   [:kMODULE,   :kMODULE     ], :expr_beg   ],
-                ["elsif",    [:kELSIF,    :kELSIF      ], :expr_beg   ],
-                ["def",      [:kDEF,      :kDEF        ], :expr_fname ],
-                ["rescue",   [:kRESCUE,   :kRESCUE_MOD ], :expr_mid   ],
-                ["not",      [:kNOT,      :kNOT        ], :expr_beg   ],
-                ["then",     [:kTHEN,     :kTHEN       ], :expr_beg   ],
-                ["yield",    [:kYIELD,    :kYIELD      ], :expr_arg   ],
-                ["for",      [:kFOR,      :kFOR        ], :expr_beg   ],
-                ["self",     [:kSELF,     :kSELF       ], :expr_end   ],
-                ["false",    [:kFALSE,    :kFALSE      ], :expr_end   ],
-                ["retry",    [:kRETRY,    :kRETRY      ], :expr_end   ],
-                ["return",   [:kRETURN,   :kRETURN     ], :expr_mid   ],
-                ["true",     [:kTRUE,     :kTRUE       ], :expr_end   ],
-                ["if",       [:kIF,       :kIF_MOD     ], :expr_beg   ],
-                ["defined?", [:kDEFINED,  :kDEFINED    ], :expr_arg   ],
-                ["super",    [:kSUPER,    :kSUPER      ], :expr_arg   ],
-                ["undef",    [:kUNDEF,    :kUNDEF      ], :expr_fname ],
-                ["break",    [:kBREAK,    :kBREAK      ], :expr_mid   ],
-                ["in",       [:kIN,       :kIN         ], :expr_beg   ],
-                ["do",       [:kDO,       :kDO         ], :expr_beg   ],
-                ["nil",      [:kNIL,      :kNIL        ], :expr_end   ],
-                ["until",    [:kUNTIL,    :kUNTIL_MOD  ], :expr_beg   ],
-                ["unless",   [:kUNLESS,   :kUNLESS_MOD ], :expr_beg   ],
-                ["or",       [:kOR,       :kOR         ], :expr_beg   ],
-                ["next",     [:kNEXT,     :kNEXT       ], :expr_mid   ],
-                ["when",     [:kWHEN,     :kWHEN       ], :expr_beg   ],
-                ["redo",     [:kREDO,     :kREDO       ], :expr_end   ],
-                ["and",      [:kAND,      :kAND        ], :expr_beg   ],
-                ["begin",    [:kBEGIN,    :kBEGIN      ], :expr_beg   ],
-                ["__LINE__", [:k__LINE__, :k__LINE__   ], :expr_end   ],
-                ["class",    [:kCLASS,    :kCLASS      ], :expr_class ],
-                ["__FILE__", [:k__FILE__, :k__FILE__   ], :expr_end   ],
-                ["END",      [:klEND,     :klEND       ], :expr_end   ],
-                ["BEGIN",    [:klBEGIN,   :klBEGIN     ], :expr_end   ],
-                ["while",    [:kWHILE,    :kWHILE_MOD  ], :expr_beg   ],
-                ["alias",    [:kALIAS,    :kALIAS      ], :expr_fname ],
-                ["__ENCODING__", [:k__ENCODING__, :k__ENCODING__], :expr_end],
-               ].map { |args| KWtable.new(*args) }
+                ["alias",    [:kALIAS,    :kALIAS      ], expr_woot  ],
+                ["and",      [:kAND,      :kAND        ], EXPR_BEG   ],
+                ["begin",    [:kBEGIN,    :kBEGIN      ], EXPR_BEG   ],
+                ["break",    [:kBREAK,    :kBREAK      ], EXPR_MID   ],
+                ["case",     [:kCASE,     :kCASE       ], EXPR_BEG   ],
+                ["class",    [:kCLASS,    :kCLASS      ], EXPR_CLASS ],
+                ["def",      [:kDEF,      :kDEF        ], EXPR_FNAME ],
+                ["defined?", [:kDEFINED,  :kDEFINED    ], EXPR_ARG   ],
+                ["do",       [:kDO,       :kDO         ], EXPR_BEG   ],
+                ["else",     [:kELSE,     :kELSE       ], EXPR_BEG   ],
+                ["elsif",    [:kELSIF,    :kELSIF      ], EXPR_BEG   ],
+                ["end",      [:kEND,      :kEND        ], EXPR_END   ],
+                ["ensure",   [:kENSURE,   :kENSURE     ], EXPR_BEG   ],
+                ["false",    [:kFALSE,    :kFALSE      ], EXPR_END   ],
+                ["for",      [:kFOR,      :kFOR        ], EXPR_BEG   ],
+                ["if",       [:kIF,       :kIF_MOD     ], EXPR_BEG   ],
+                ["in",       [:kIN,       :kIN         ], EXPR_BEG   ],
+                ["module",   [:kMODULE,   :kMODULE     ], EXPR_BEG   ],
+                ["next",     [:kNEXT,     :kNEXT       ], EXPR_MID   ],
+                ["nil",      [:kNIL,      :kNIL        ], EXPR_END   ],
+                ["not",      [:kNOT,      :kNOT        ], EXPR_ARG   ],
+                ["or",       [:kOR,       :kOR         ], EXPR_BEG   ],
+                ["redo",     [:kREDO,     :kREDO       ], EXPR_END   ],
+                ["rescue",   [:kRESCUE,   :kRESCUE_MOD ], EXPR_MID   ],
+                ["retry",    [:kRETRY,    :kRETRY      ], EXPR_END   ],
+                ["return",   [:kRETURN,   :kRETURN     ], EXPR_MID   ],
+                ["self",     [:kSELF,     :kSELF       ], EXPR_END   ],
+                ["super",    [:kSUPER,    :kSUPER      ], EXPR_ARG   ],
+                ["then",     [:kTHEN,     :kTHEN       ], EXPR_BEG   ],
+                ["true",     [:kTRUE,     :kTRUE       ], EXPR_END   ],
+                ["undef",    [:kUNDEF,    :kUNDEF      ], expr_woot  ],
+                ["unless",   [:kUNLESS,   :kUNLESS_MOD ], EXPR_BEG   ],
+                ["until",    [:kUNTIL,    :kUNTIL_MOD  ], EXPR_BEG   ],
+                ["when",     [:kWHEN,     :kWHEN       ], EXPR_BEG   ],
+                ["while",    [:kWHILE,    :kWHILE_MOD  ], EXPR_BEG   ],
+                ["yield",    [:kYIELD,    :kYIELD      ], EXPR_ARG   ],
+                ["BEGIN",    [:klBEGIN,   :klBEGIN     ], EXPR_END   ],
+                ["END",      [:klEND,     :klEND       ], EXPR_END   ],
+                ["__FILE__", [:k__FILE__, :k__FILE__   ], EXPR_END   ],
+                ["__LINE__", [:k__LINE__, :k__LINE__   ], EXPR_END   ],
+                ["__ENCODING__", [:k__ENCODING__, :k__ENCODING__], EXPR_END],
+               ].map { |args|
+      KWtable.new(*args)
+    }
 
     # :startdoc:
 
-    WORDLIST18 = Hash[*wordlist.map { |o| [o.name, o] }.flatten]
-    WORDLIST19 = Hash[*wordlist.map { |o| [o.name, o] }.flatten]
+    WORDLIST = Hash[*wordlist.map { |o| [o.name, o] }.flatten]
 
-    %w[and case elsif for if in module or unless until when while].each do |k|
-      WORDLIST19[k] = WORDLIST19[k].dup
-      WORDLIST19[k].state = :expr_value
-    end
-    %w[not].each do |k|
-      WORDLIST19[k] = WORDLIST19[k].dup
-      WORDLIST19[k].state = :expr_arg
-    end
-
-    def self.keyword18 str # REFACTOR
-      WORDLIST18[str]
-    end
-
-    def self.keyword19 str
-      WORDLIST19[str]
+    def self.keyword str
+      WORDLIST[str]
     end
   end
 
@@ -1159,55 +1774,28 @@ module RubyParserStuff
       @env.first
     end
 
-    def dynamic
-      idx = @dyn.index false
-      @env[0...idx].reverse.inject { |env, scope| env.merge scope } || {}
-    end
-
-    def dynamic?
-      @dyn[0] != false
-    end
-
     def extend dyn = false
       @dyn.unshift dyn
       @env.unshift({})
-      @use.unshift({})
     end
 
     def initialize dyn = false
       @dyn = []
       @env = []
-      @use = []
       self.reset
     end
 
     def reset
       @dyn.clear
       @env.clear
-      @use.clear
       self.extend
     end
 
     def unextend
       @dyn.shift
       @env.shift
-      @use.shift
       raise "You went too far unextending env" if @env.empty?
     end
-
-    def use id
-      @env.each_with_index do |env, i|
-        if env[id] then
-          @use[i][id] = true
-        end
-      end
-    end
-
-    def used? id
-      idx = @dyn.index false # REFACTOR
-      u = @use[0...idx].reverse.inject { |env, scope| env.merge scope } || {}
-      u[id]
-    end
   end
 
   class StackState
@@ -1215,10 +1803,10 @@ module RubyParserStuff
     attr_reader :stack
     attr_accessor :debug
 
-    def initialize(name)
+    def initialize name, debug=false
       @name = name
       @stack = [false]
-      @debug = false
+      @debug = debug
     end
 
     def inspect
@@ -1226,111 +1814,52 @@ module RubyParserStuff
     end
 
     def is_in_state
-      p :stack_is_in_state => [name, @stack.last, caller.first] if debug
+      log :is_in_state if debug
       @stack.last
     end
 
     def lexpop
-      p :stack_lexpop => caller.first if debug
       raise if @stack.size == 0
       a = @stack.pop
       b = @stack.pop
       @stack.push(a || b)
+      log :lexpop if debug
+    end
+
+    def log action
+      c = caller[1]
+      c = caller[2] if c =~ /expr_result/
+      warn "%s_stack.%s: %p at %s" % [name, action, @stack, c.clean_caller]
+      nil
     end
 
     def pop
       r = @stack.pop
-      p :stack_pop => [name, r, @stack, caller.first] if debug
-      @stack.push false if @stack.size == 0
+      @stack.push false if @stack.empty?
+      log :pop if debug
       r
     end
 
     def push val
       @stack.push val
-      p :stack_push => [name, @stack, caller.first] if debug
-      nil
+      log :push if debug
     end
-  end
-end
-
-class Ruby19Parser < Racc::Parser
-  include RubyParserStuff
-end
-
-class Ruby18Parser < Racc::Parser
-  include RubyParserStuff
-end
-
-##
-# RubyParser is a compound parser that first attempts to parse using
-# the 1.9 syntax parser and falls back to the 1.8 syntax parser on a
-# parse error.
-
-class RubyParser
-  class SyntaxError < RuntimeError; end
-
-  def initialize
-    @p18 = Ruby18Parser.new
-    @p19 = Ruby19Parser.new
-  end
-
-  def process(s, f = "(string)") # parens for emacs *sigh*
-    Ruby19Parser.new.process s, f
-  rescue Racc::ParseError
-    Ruby18Parser.new.process s, f
-  end
-
-  alias :parse :process
 
-  def reset
-    @p18.reset
-    @p19.reset
-  end
-end
-
-############################################################
-# HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK
-
-unless "".respond_to?(:grep) then
-  class String
-    def grep re
-      lines.grep re
+    def reset
+      @stack = [false]
+      log :reset if debug
     end
-  end
-end
-
-class Sexp
-  attr_writer :paren
-
-  def paren
-    @paren ||= false
-  end
-
-  def value
-    raise "multi item sexp" if size > 2
-    last
-  end
-
-  def to_sym
-    raise "no"
-    self.value.to_sym
-  end
-
-  def add x
-    concat x
-  end
 
-  def add_all x
-    raise "no: #{self.inspect}.add_all #{x.inspect}" # TODO: need a test to trigger this
-  end
+    def restore oldstate
+      @stack.replace oldstate
+      log :restore if debug
+    end
 
-  def block_pass?
-    any? { |s| Sexp === s && s[0] == :block_pass }
+    def store base = false
+      result = @stack.dup
+      @stack.replace [base]
+      log :store if debug
+      result
+    end
   end
-
-  alias :node_type :sexp_type
-  alias :values :sexp_body # TODO: retire
 end
-
-# END HACK
-############################################################