redparse 0.8.4 → 1.0.0

data/lib/redparse.rb

@@ -1,6 +1,6 @@
 =begin
     redparse - a ruby parser written in ruby
-    Copyright (C) 2008,2009  Caleb Clausen
+    Copyright (C) 2008,2009, 2012, 2016  Caleb Clausen
 
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU Lesser General Public License as published by
@@ -20,6 +20,8 @@
 
 require 'forwardable'
 
+require 'digest/sha2'
+
 begin
   require 'rubygems'
 rescue LoadError=>e
@@ -33,14 +35,39 @@ require 'reglookab'
 require "redparse/node"
 #require "redparse/decisiontree"
 require "redparse/reg_more_sugar"
-require "redparse/generate"
+#require "redparse/generate"
 require "redparse/cache"
 #require "redparse/compile"
 
 class RedParse
+
+
+
+  alias :dump :inspect # preserve old inspect functionality
   
-####### generic stuff for parsing any(?) language
+  # irb friendly #inspect/#to_s
+  def to_s
+    mods=class<<self; ancestors; end.reject{|k| !k.name }-self.class.ancestors
+    mods=mods.map{|mod| mod.name }.join('+')
+    mods="+"<<mods unless mods.empty?
+    input=@input||@lexer.input
+    "#<#{self.class.name}#{mods}: [#{input.inspect}]>"
+  end
+  
+  alias :inspect :to_s
+  
+  def pretty_stack max=nil
+    target=@stack
+    target=target[-max..-1] if max and max<target.size
+
+    target.map{|n| 
+      res=n.inspect
+      res<<"\n" unless res[-1]=="\n"
+      res
+    }
+  end
 
+####### generic stuff for parsing any(?) language
 #  include Nodes
   class StackMonkey
     def initialize(name,first_changed_index,and_expect_node,options={},&monkey_code)
@@ -119,12 +146,16 @@ end
     #index of data at which to start matching
     i=@stack.size-1   #-1 because last element of @stack is always lookahead
 
+=begin was, but now done by expanded_RULES
     #I could call this a JIT compiler, but that's a bit grandiose....
     #more of a JIT pre-processor
     compiled_rule=@compiled_rules[rule]||=
       rule.map{|pattern| 
         String|Regexp===pattern ? KW(pattern) : pattern 
       }
+=end
+    assert(rule.grep(String|Regexp|Reg::Subseq|Reg::LookAhead|Reg::LookBack|Proc).empty?)
+    compiled_rule=rule
 
     #what's the minimum @stack size this rule could match?
     rule_min_size=@min_sizes[compiled_rule]||=
@@ -198,12 +229,417 @@ end
     return true #let caller know we found a match
 
     
-  rescue Exception=>e
+  rescue Exception #=>e
     #puts "error (#{e}) while executing rule: #{rule.inspect}"
     #puts e.backtrace.join("\n")
     raise
   end
 
+
+  def coalesce_loop(klass=nil,ident=nil,klass2=nil,ident2=nil)
+    eligible=rules.reverse.map!{|rule| can_coalesce?(rule,klass,ident,klass2,ident2)&&rule }
+    i=rules.size
+    eligible.map!{|rule| 
+      i-=1
+      next unless rule
+      if @size_cache
+        @size_cache[[i,rule.right]]||=1
+        @size_cache[[i,rule.right]]+=1
+      end
+      coalesce rule, i, klass,ident,klass2,ident2
+    }
+    eligible.compact!
+    @size_cache[klass2 ? [klass,ident,klass2,ident2] : ident ? ident : klass]= eligible.size if @size_cache
+
+    @empty_reduce_withs+=1 if defined? @empty_reduce_withs and eligible.size.zero?
+
+    return eligible
+  end
+
+  def can_coalesce? rule,klass=nil,ident=nil,klass2=nil,ident2=nil
+    Reg::Transform===rule or fail
+    node_type= rule.right
+    rule=rule.left.subregs.dup
+    rule.pop if Proc|::Reg::LookAhead===rule.last
+    rule[0]=rule[0].subregs[0] if ::Reg::LookBack===rule[0]
+
+=begin was, but now done by expanded_RULES
+    #I could call this a JIT compiler, but that's a bit grandiose....
+    #more of a JIT pre-processor
+    compiled_rule=@compiled_rules[rule]||=
+      rule.map{|pattern| 
+        String|Regexp===pattern ? KW(pattern) : pattern 
+      }
+=end
+    assert(rule.grep(String|Regexp|Reg::Subseq|Reg::LookAhead|Reg::LookBack|Proc).empty?)
+
+
+    return false if klass && !can_combine?(rule,klass,ident)
+    return false if klass2 && !can_combine2?(rule,klass2,ident2,-2)
+    warn "plain lit matches #{node_type}" if klass==LiteralNode and klass2.nil?
+    return true
+  end
+
+  def coalesce rule,rulenum,klass=nil,ident=nil,klass2=nil,ident2=nil
+    #last 4 params aren't actually neeeded anymore
+
+    @coalesce_result||=[]
+    result=@coalesce_result[rulenum]
+    return result if result
+
+    #dissect the rule
+    Reg::Transform===rule or fail
+    node_type= rule.right
+    rule=rule.left.subregs.dup
+    lookahead_processor=(rule.pop if Proc|::Reg::LookAhead===rule.last)
+    lookback=rule[0]=rule[0].subregs[0] if ::Reg::LookBack===rule[0]
+
+    assert @rules[rulenum].right==node_type
+
+    if klass==VarNode and klass2==KeywordToken
+      #warn "can_combine2? about to fail"
+    end
+
+    needends=0
+    result=["\n##{mui node_type}\n"]
+    
+    #index of data at which to start matching
+    result<<"i=@stack.size-1   ##{mui node_type}\n#-1 because last element of @stack is always lookahead\n"
+
+=begin was, but now done by expanded_RULES
+    #I could call this a JIT compiler, but that's a bit grandiose....
+    #more of a JIT pre-processor
+    compiled_rule=@compiled_rules[rule]||=
+      rule.map{|pattern| 
+        String|Regexp===pattern ? KW(pattern) : pattern 
+      }
+=end
+    assert(rule.grep(String|Regexp|Reg::Subseq|Reg::LookAhead|Reg::LookBack|Proc).empty?)
+    compiled_rule=rule
+
+    return if klass && !can_combine?(compiled_rule,klass,ident) #should never happen
+    return if klass2 && !can_combine2?(compiled_rule,klass2,ident2,-2) #should never happen
+
+    #what's the minimum @stack size this rule could match?
+    rule_min_size=@min_sizes[compiled_rule]||=
+      compiled_rule.inject(0){|sum,pattern| 
+        sum + pattern.itemrange.begin 
+      }
+    if rule_min_size > 1
+      needends+=1
+      result<<"if i>=#{rule_min_size}\n"
+      min_i=rule_min_size
+    end
+    #@@has_loop||=[]
+    #@@has_optional||=[]
+    has_loop=#@@has_loop[rulenum]||=
+      compiled_rule.find{|x| x.itemrange.last.to_f.infinite? }
+    has_optional=#@@has_optional[rulenum]||=
+      compiled_rule.find{|x| x.itemrange.first.zero? }
+
+    if Class===node_type and has_loop||has_optional
+      result<<"matching=[]\n"
+      need_matching=true
+    end
+
+    j=compiled_rule.size
+    #actually try to match rule elements against each @stack element in turn
+    first1=true
+    compiled_rule.reverse_each{|matcher|
+      j-=1
+      result<<"i.zero? and fail\n" unless min_i && min_i>0 or first1
+      first1=false
+      #is this matcher optional? looping?
+      maximum= matcher.itemrange.last
+      minimum= matcher.itemrange.first
+      loop= maximum.to_f.infinite?
+      optional=minimum.zero?
+      fail "looping matcher with finite maximum not supported" if maximum>1 and !loop
+      if need_matching
+        success="matching.unshift item"
+        loopsuccess="target.unshift item"
+        optfail="matching.unshift nil"
+
+        result<<"matching.unshift target=[]\n" if loop
+      end
+      is_lookback=matcher .equal? lookback
+      if loop or optional
+        matcher=matcher.subregs[0]
+        fail "lookback is not a scalar" if is_lookback
+      end
+
+      itemget="@stack[i-=1]"
+      itemget="(item=#{itemget})" if success
+      test="#{ref_to matcher,rulenum,j}===#{itemget}  #try match of #{mui matcher}"
+      p [:misparse_start, matcher] if node_type===MisparsedNode and j.zero?
+      matcher= ~ (matcher.subregs[0]|NilClass) if Reg::Not===matcher
+      if matcher===nil and j.zero?
+        warn "rule ##{rulenum}(>>#{node_type}) can match nil at start; might match emptiness before start of stack"
+      end
+      if !loop
+        fail unless maximum==1
+        min_i-=1 if min_i
+        result<<<<-END
+            if #{test}
+              #{success if !is_lookback}
+                   END
+        optional ? result<<<<-END : needends+=1
+            else
+              #ignore optional match fail
+              #but bump the data position back up, since the latest datum
+              #didn't actually match anything.
+              i+=1
+              #{optfail}
+            end
+                   END
+      else
+        min_i=nil
+        if minimum<10
+          needends+=minimum
+          result<<<<-END*minimum
+          if #{test}
+            #{loopsuccess}
+          END
+          result<<<<-END
+            while #{test}
+              #{loopsuccess}
+            end
+              #but bump the data position back up, since the latest datum
+              #didn't actually match anything.
+              i+=1
+          END
+        else
+          needends+=1
+          result<<<<-END
+            #{"n=#{minimum}" unless need_matching}
+            while #{test}
+              #{loopsuccess || "n-=1"}
+            end
+            if #{need_matching ? "target.size>=minimum" : "n<=0"} then
+              #but bump the data position back up, since the latest datum
+              #didn't actually match anything.
+              i+=1
+          END
+        end
+
+      end
+    } 
+
+    #give lookahead matcher (if any) a chance to fail the match
+    result<<case lookahead_processor
+    when ::Reg::LookAhead
+      action_idx=compiled_rule.size+1
+      needends+=1
+      "if #{ref_to lookahead_processor.subregs[0],rulenum,compiled_rule.size}===@stack.last ##{mui lookahead_processor.subregs[0] }\n"
+    when Proc
+      action_idx=compiled_rule.size+1
+      needends+=1
+      "if #{ref_to lookahead_processor,rulenum,compiled_rule.size}[self,@stack.last] ##{mui lookahead_processor}\n"
+    else ''
+    end
+
+    #if there was a lookback item, don't include it in the matched set
+    #result<<"matching.shift\n"    if lookback and need_matching
+
+    need_return=true
+
+    #replace matching elements in @stack with node type found
+    result<<
+    case node_type
+    when Class
+      #if there was a lookback item, don't include it in the new node
+      <<-END
+        #{"i+=1" if lookback}
+        matchrange= i...-1  #what elems in @stack were matched?
+        #{"matching=@stack.slice! matchrange" unless need_matching}
+        node=#{ref_to node_type,rulenum,action_idx||rule.size}.create(*matching) ##{mui node_type}
+        node.startline||=#{need_matching ? "@stack[i]" : "matching.first"}.startline
+        node.endline=@endline
+        #{need_matching ? "@stack[matchrange]=[node]" : "@stack.insert i,node" }
+      END
+    when Proc,StackMonkey;   ref_to(node_type,rulenum,action_idx||rule.size)+"[@stack] ##{mui node_type}\n"
+    when :shift; need_return=false; "return 0\n"
+    when :accept,:error; need_return=false; "throw :ParserDone\n"
+    else fail
+    end
+    
+    result<<"return true #let caller know we found a match\n" if need_return
+    result<<"end;"*needends
+    result<<"\n"
+
+    return @coalesce_result[rulenum]=result
+  rescue Exception  #=>e
+    #puts "error (#{e}) while executing rule: #{rule.inspect}"
+    #puts e.backtrace.join("\n")
+    raise
+  end
+
+  @@ref_to_cache={}
+  @@ref_to_cache_by_id={}
+  @@ref_to_idx=-1
+  def ref_to obj,i,j
+    assert j<=0x3FF
+    if Module===obj and obj.name
+      return obj.name
+    elsif ref=@@ref_to_cache_by_id[obj.__id__] || @@ref_to_cache[(i<<10)+j] 
+      return ref
+    else
+      @@ref_to_rules||=
+          rules.map{|rule|
+            rule.left.subregs.map{|pat|
+              case pat
+              when String,Regexp #not needed anymore...?
+                RedParse::KW(pat)
+              when Reg::LookBack,Reg::LookAhead,Reg::Repeat #Reg::Repeat should be handled already by now
+                pat.subregs[0]
+              #subseqs handled already
+              else pat
+              end
+            }<<rule.right
+          }
+
+      @ref_to_code||=[]
+      name="@@ref_#{@@ref_to_idx+=1}"
+      #eval "#{name}=obj"
+      unless @@ref_to_rules[i][j]==obj
+        warn "ref_to mismatch"
+      end
+      @ref_to_code<<"#{name}=rules[#{i}][#{j}]"
+      @ref_to_code<<"warn_unless_equal #@@ref_to_idx,mui(#{name}),#{squote mui( obj )}"
+      @@ref_to_cache[(i<<10)+j]=name
+      @@ref_to_cache_by_id[obj.__id__]=name
+    end
+  end
+
+  module ReduceWithUtils
+    #a version of inspect that is especially likely to be stable;
+    #no embedded addresses and ivar order is always the same
+    def matcher_unique_inspect(m)
+      result=m.inspect
+      return result unless /\A#<[A-Z]/===result
+      "#<#{m.class}: "+
+         m.instance_variables.sort.map{|iv| 
+           val=m.instance_variable_get(iv).inspect
+           val.gsub!(/#<(Proc|(?:Stack|Delete)Monkey):(?:0[xX])?[0-9a-fA-F]+/){ "#<#$1:" }
+           iv.to_s+"="+val
+         }.join(" ")+">"
+    end
+    alias mui matcher_unique_inspect
+
+    def squote(str)
+      "'#{str.gsub(/['\\]/){|ch| %[\\]+ch }}'"
+    end
+
+    @@unequal_parser_ref_vars=0
+    @@line_mismatch_parser_ref_vars=0
+    def warn_unless_equal i,ref,orig
+      return if ref==orig
+      msg="expected @ref_#{i} to == #{squote orig}, saw #{squote ref}"
+      ref=ref.gsub(/\.rb:\d+>/,".rb:X>")
+      orig=orig.gsub(/\.rb:\d+>/,".rb:X>")
+      count=
+        if ref==orig
+          msg="@ref_#{i} differed in line nums"
+          warn "more @ref_ vars differed in line nums..." if @@line_mismatch_parser_ref_vars==1
+          @@line_mismatch_parser_ref_vars+=1
+        else
+          @@unequal_parser_ref_vars+=1
+        end
+      warn msg if 1==count
+    end
+  end
+  include ReduceWithUtils
+
+  def classes_matched_by(matcher)
+    result=[]
+    worklist=[matcher]
+    begin
+      case x=worklist.shift
+      when Reg::And,Reg::Or; worklist.concat x.subregs
+      when Class; result<<x
+      end
+    end until worklist.empty?
+    return [Object] if result.empty?
+    return result
+  end
+ 
+
+
+  def can_combine? rule,klass,ident
+    rule.reverse_each{|matcher|
+      if Reg::Repeat===matcher
+        optional= matcher.times.first==0
+        matcher=matcher.subregs[0]
+      end
+      if ident
+        return true if matcher===klass.new(ident)
+        optional ? next : break
+      end
+
+=begin was
+      orlist= Reg::Or===matcher ? matcher.subregs : [matcher]
+      orlist.map!{|m| 
+        classes=(Reg::And===m ? m.subregs : [m]).grep(Class)
+        case classes.size
+        when 0; return true
+        when 1
+        else warn "multiple classes in matcher #{matcher}"
+        end
+        classes if classes.all?{|k| klass<=k }
+      }
+      return true if orlist.compact.flatten[0]
+=end
+      return true if classes_matched_by(matcher).any?{|k| klass<=k }
+
+      break unless optional
+    }
+    return false
+  end
+
+  def can_combine2? rule,klass,ident,index=-1
+  #very similar to can_combine?, just above
+  #i think can_combine2? with 3 params is equiv to can_combine?
+  #so, the two should be merged
+    index=-index
+    rule_max_size=rule.inject(0){|sum,pattern|
+        sum + pattern.itemrange.end
+    }
+    return true if rule_max_size<index
+    min=max=0
+    rule.reverse_each{|matcher|
+      break if index<min
+      if Reg::Repeat===matcher
+        #optional= matcher.times.first==0
+        min+=matcher.times.first
+        max+=matcher.times.last
+        matcher=matcher.subregs[0]
+      else
+        min+=1
+        max+=1
+      end
+      next if index>max
+      if ident
+        return true if matcher===klass.new(ident)
+        next #was: optional ? next : break
+      end
+=begin was
+      orlist= Reg::Or===matcher ? matcher.subregs : [matcher]
+      orlist.map!{|m| 
+        classes=(Reg::And===m ? m.subregs : [m]).grep(Class)
+        case classes.size
+        when 0; return true
+        when 1
+        else warn "multiple classes in matcher #{matcher}: #{classes.inspect}"
+        end
+        classes if classes.all?{|k| klass<=k }
+      }
+      return true if orlist.compact.flatten[0]
+=end
+      return true if classes_matched_by(matcher).any?{|k| klass<=k }
+    }
+    return false
+  end
+
   class ParseError<RuntimeError
     def initialize(msg,stack)
       super(msg)
@@ -234,7 +670,7 @@ end
   end
 
   #try all possible reductions
-  def reduce
+  def old_slow_reduce
       shift=nil
       @rules.reverse_each{|rule|
         shift=evaluate(rule) and break
@@ -242,20 +678,279 @@ end
       return shift
   end
 
-  def parse
+  HASHED_REDUCER=!ENV['REDUCE_INTERPRETER']
 
-    #hack, so StringToken can know what parser its called from
-    #so it can use it to parse inclusions
-    oldparser=Thread.current[:$RedParse_parser]
-    Thread.current[:$RedParse_parser]||=self
+  @@rules_compile_cache={}
 
-    return @cached_result if defined? @cached_result
+  #try all possible reductions
+  def reduce
+    code=@@rules_compile_cache[class<<self; ancestors end.reject{|k| !k.name}<<@rubyversion]||=coalesce_loop().join
+    code= <<-END
+      class RedParse
+      def (Thread.current['$RedParse_instance']).reduce
+      #{code}
+      return nil
+      end
+      end
+    END
 
-    @rules||=expanded_RULES()
-#    @inputs||=enumerate_exemplars
+    f=Tempfile.new("reduce")
+    Thread.current['$RedParse_instance']=self
+    p [:code_hash, code.hash]
+    f.write code
+    f.flush
+    load f.path
 
-    @stack=[StartToken.new, get_token] 
-           #last token on @stack is always implicitly the lookahead
+    reduce
+  ensure f.close if f
+  end if !HASHED_REDUCER
+
+
+#  include StackableClasses
+
+  Punc2name={
+    "("=>"lparen",    ")"=>"rparen",
+    "["=>"lbracket",    "]"=>"rbracket",
+    "{"=>"lbrace",    "}"=>"rbrace",
+    ","=>"comma",
+    ";"=>"semicolon",
+    "::"=>"double_colon",
+    "."=>"dot",
+    "?"=>"question_mark", ":"=>"colon",
+    "="=>"equals",
+    "|"=>"pipe",
+    "<<"=>"leftleft", ">>"=>"rightright",
+    "=>"=>"arrow",
+    "->"=>"stabby",
+    "rhs,"=>"rhs_comma",
+    "lhs,"=>"lhs_comma",
+    "||="=>"or_equals",
+    "&&="=>"and_equals",
+  }
+
+
+  RUBYUNOPERATORS=::RubyLexer::RUBYUNOPERATORS
+  RUBYBINOPERATORS=::RubyLexer::RUBYBINOPERATORS
+  RUBYSYMOPERATORS=::RubyLexer::RUBYSYMOPERATORS
+  RUBYNONSYMOPERATORS=::RubyLexer::RUBYNONSYMOPERATORS
+  OPERATORS=RUBYUNOPERATORS-%w[~@ !@]+RUBYBINOPERATORS+RUBYNONSYMOPERATORS+
+              %w[while until if unless rescue and or not unary* unary& rescue3 lhs* rhs*]
+  OPERATORS.uniq!
+  RUBYKEYWORDLIST=( 
+   RubyLexer::RUBYKEYWORDLIST+Punc2name.keys+
+   RUBYSYMOPERATORS+RUBYNONSYMOPERATORS
+  ).uniq
+
+  def rubyoperatorlist; OPERATORS end
+  def rubykeywordlist; RUBYKEYWORDLIST end
+
+  class KeywordToken
+    def reducer_method(stack)
+      :"reduce_with_tos_KeywordToken_#@ident"
+    end
+    def reducer_ident
+      :"KeywordToken_#@ident"
+    end
+  end
+
+  class OperatorToken
+    def reducer_ident
+      :"OperatorToken_#@ident"
+    end
+  end
+
+  class ValueNode
+    def reducer_method(stack)
+      :"reduce_with_tos_#{stack[-3].reducer_ident}_then_#{reducer_ident}"
+    end
+  end
+
+  def parser_identity
+  #what is the relationship between this method and #signature?
+  #can the two be combined?
+    result=class<<self; ancestors end.reject{|k| !k.name}
+    result.reject!{|k| !!((::RedParse<k)..false) }
+    result.reject!{|k| k.name[/^(?:RedParse::)?ReduceWiths/] }
+    result.reverse!
+    result.push @rubyversion
+    #@rubyversion in identity is a hack; should have RedParse1_9 module instead
+  end
+
+  def code_for_reduce_with ident, code
+     code=coalesce_loop(*code) if Array===code
+     ident.gsub!(/[\\']/){|x| "\\"+x}
+     code=code.join
+     @reduce_with_defns+=1
+     if name=@reduce_with_cache[code]
+       @reduce_with_aliases+=1
+       "alias :'reduce_with_tos_#{ident}' :'#{name}'\n"
+     else
+       @reduce_with_cache[code]=name="reduce_with_tos_#{ident}"
+       ["define_method('", name ,"') do\n", code ,"\nnil\nend\n"]
+     end
+  end
+
+  def addl_node_containers; [] end
+
+  def write_reduce_withs path=nil
+    return unless HASHED_REDUCER
+    start=Time.now
+    @size_cache={}
+    identity=parser_identity
+    @reduce_with_cache={}
+    @reduce_with_aliases=0
+    @empty_reduce_withs=@reduce_with_defns=0
+
+      expanded_RULES()
+      shortnames=[]   #[[],[]]
+      list=[self.class,*addl_node_containers].map{|mod| 
+        mod.constants.select{|k| 
+          /(?:Node|Token)$/===k.to_s 
+        }.map{|k| 
+          mod.const_get k
+        }
+      }.flatten.grep(Class).uniq
+      #list=STACKABLE_CLASSES()
+      list -= [KeywordToken,ImplicitParamListStartToken,ImplicitParamListEndToken,
+               Token,WToken,NewlineToken,DecoratorToken,Node,ValueNode]
+      list.reject!{|x| IgnoreToken>=x and not /(^|:)AssignmentRhs/===x.name}
+      exprclasses,list=list.partition{|k| k<=ValueNode }
+      fail unless list.include? StartToken
+      indexcode=list.map{|klass|
+        shortname=klass.to_s[/[^:]+$/]
+        warn "empty reducer_ident for ::#{klass}" if shortname.empty?
+        <<-END
+          class ::#{klass}
+            def reducer_method(stack)
+              :reduce_with_tos_#{shortname}
+            end if instance_methods(false).&(["reducer_method",:reducer_method]).empty?
+            def reducer_ident
+              :#{shortname}
+            end if instance_methods(false).&(["reducer_ident",:reducer_ident]).empty?
+          end
+        END
+      }.concat(exprclasses.map{|exprclass|
+        shec=exprclass.name[/[^:]+$/]
+        warn "empty reducer_ident for ::#{exprclass}" if shec.empty?
+        <<-END
+            class ::#{exprclass}
+              def reducer_ident
+                :#{shec}
+              end if instance_methods(false).&(["reducer_ident",:reducer_ident]).empty?
+            end
+        END
+      })
+      ruby=["#Copyright (C) #{Time.now.year} #{ENV['COPYRIGHT_OWNER']||'Caleb Clausen'}\n"+
+            "#Generated with ruby v#{RUBY_VERSION}\n"
+      ].concat list.map{|klass|
+        shortname=klass.to_s[/[^:]+$/]
+        shortnames<<[shortname,klass,nil]
+        code_for_reduce_with( shortname, [klass] )
+      }.concat(rubykeywordlist.map{|kw|
+        shortname="KeywordToken_#{kw}"
+        shortnames<<[shortname,KeywordToken,kw]
+        code_for_reduce_with( shortname, [KeywordToken, kw] )
+      }).concat({ImplicitParamListStartToken=>'(',ImplicitParamListEndToken=>')'}.map{|(k,v)|
+        shortnames<<[k.name,k,v]
+        code_for_reduce_with k.name, [k,v]
+      })
+      shortnames.delete ["OperatorToken",OperatorToken,nil]
+      record=shortnames.dup
+      ruby.concat(exprclasses.map{|exprclass|
+        shec=exprclass.name[/[^:]+$/]
+        shortnames.map{|(sn,snclass,snparam)|
+          warn "empty shortname for #{snclass}" if sn.empty?
+          record<<["#{sn}_then_#{shec}", exprclass, nil, snclass, snparam]
+          code_for_reduce_with "#{sn}_then_#{shec}", [exprclass, nil, snclass, snparam]
+        } 
+      })
+      ruby.concat(exprclasses.map{|exprclass|
+        shec=exprclass.name[/[^:]+$/]
+        rubyoperatorlist.map{|op|
+          record<<["OperatorToken_#{op}_then_#{shec}", exprclass, nil, OperatorToken, op]
+          code_for_reduce_with "OperatorToken_#{op}_then_#{shec}", [exprclass, nil, OperatorToken, op]
+        } 
+      }).concat([LiteralNode,VarNode].map{|k|
+          shec=k.name[/[^:]+$/]
+          record<<["#{shec}_then_#{shec}", k, nil, k, nil]
+          code_for_reduce_with "#{shec}_then_#{shec}", [k, nil, k, nil]
+      })
+
+      modname="ReduceWithsFor_#{parser_identity.join('_').tr(':.','_')}"
+
+      size_cache,rule_popularity=@size_cache.partition{|((i,action),size)| Integer===i }
+
+      ruby.unshift [<<-END,@ref_to_code.join("\n"),<<-END2]
+        #number of coalescences: #{size_cache.size}
+        #empty coalescences: #@empty_reduce_withs
+        #duplicate coalescences: #@reduce_with_aliases
+        #nonduplicate coalescences: #{@reduce_with_cache.size}
+        #reduce_with_defns: #@reduce_with_defns
+        extend RedParse::ReduceWithUtils
+        def self.redparse_modules_init(parser)
+          return if defined? @@ref_0
+          rules=parser.rules.map{|rule|
+            rule.left.subregs.map{|pat|
+              case pat
+              when String,Regexp #not needed anymore...?
+                RedParse::KW(pat)
+              when Reg::LookBack,Reg::LookAhead,Reg::Repeat #Reg::Repeat should be handled already by now
+                pat.subregs[0]
+              #subseqs handled already
+              else pat
+              end
+            }<<rule.right
+          }
+                                          END
+
+        end
+        def redparse_modules_init
+          ::RedParse::#{modname}.redparse_modules_init(self) 
+          super
+        end
+                                          END2
+
+      ruby.unshift( "#15 largest coalescences:\n", 
+        *size_cache.sort_by{|(k,size)| size}[-15..-1].map{ \
+          |(k,size)| "##{k.inspect}=#{size}\n" 
+      })
+ 
+      ruby.unshift("#10 most popular rules:\n",
+        *rule_popularity.sort_by{|(rule,pop)| pop}[-10..-1].map{ \
+          |((i,action),pop)| "##{i} #{action.inspect}=#{pop}\n" 
+      })
+
+      warn "15 largest coalescences:"
+      size_cache.sort_by{|(klass,size)| size}[-15..-1].each{ \
+        |(klass,size)| warn "#{klass.inspect}=#{size}" 
+      }
+
+      warn "10 most popular rules:"
+      rule_popularity.sort_by{|(rule,pop)| pop}[-10..-1].each{ \
+        |((i,action),pop)| warn "#{i} #{action.inspect}=#{pop}" 
+      }
+
+
+      @ref_to_code=nil
+      ruby=["module RedParse::#{modname}\n",ruby,"\nend\n",indexcode]
+      @@rules_compile_cache[identity]=ruby
+
+    path ||= $LOAD_PATH.find{|d| File.exist? File.join(d,"redparse.rb") }+"/redparse/"
+    #should use reduce_withs_directory here somehow instead...
+
+    path += modname+".rb" if path[-1]==?/
+    File.open(path,"wb") {|f| ruby.flatten.each{|frag| f.write frag } }
+
+    #warn "actual write_reduce_withs writing took #{Time.now-start}s"
+    warn "size of #{path}: #{File.size path}"
+
+  ensure 
+    warn "write_reduce_withs took #{Time.now-start}s" if start
+    @reduce_with_cache=nil if @reduce_with_cache
+    @size_cache=nil if @size_cache
+  end
+
+  def old_reduce_loop
     catch(:ParserDone){ loop {
       #try all possible reductions
       next if reduce==true 
@@ -269,6 +964,55 @@ end
       #shift our token onto the @stack
       @stack.push tok
     }}
+  end
+
+=begin should be
+  reduce_call= HASHED_REDUCER ? 
+      'send(@stack[-2].reducer_method(@stack))' : 
+      'reduce'
+  eval <<-END,__FILE__,__LINE__
+    def reduce_loop
+      catch(:ParserDone){ ( @stack.push(get_token||break) unless(#{reduce_call}==true) ) while true }
+    end
+  END
+=end
+  def reduce_loop
+    catch(:ParserDone){ while true
+      #try all possible reductions
+      #was: next if reduce==true
+      next if send(@stack[-2].reducer_method(@stack))==true
+
+      #no rule can match current @stack, get another token
+      tok=get_token  or break
+
+      #are we done yet?
+      #tok.nil? or EoiToken===tok && EoiToken===@stack.last and break
+
+      #shift our token onto the @stack
+      @stack.push tok
+    end }
+  end
+
+  if ENV['REDUCE_INTERPRETER']
+    alias reduce old_slow_reduce
+    alias reduce_loop old_reduce_loop
+  end
+
+  def parse
+
+    #hack, so StringToken can know what parser its called from
+    #so it can use it to parse inclusions
+    oldparser=Thread.current[:$RedParse_parser]
+    Thread.current[:$RedParse_parser]||=self
+
+    return @cached_result if defined? @cached_result
+
+    expanded_RULES()
+#    @inputs||=enumerate_exemplars
+
+    @stack=[StartToken.new, get_token] 
+           #last token on @stack is always implicitly the lookahead
+    reduce_loop
 
     @stack.size==2 and return result=NopNode.new #handle empty parse string
 
@@ -276,7 +1020,7 @@ end
     #with the last an Eoi, and first a StartToken
     #there was a parse error
     unless @stack.size==3
-      pp @stack[-[15,@stack.size].min..-1] if ENV['PRINT_STACK']
+      puts( pretty_stack( 15 ))if ENV['PRINT_STACK']
       top=MisparsedNode.new("(toplevel)", @stack[1...-1],'')
       raise ParseError.new(top.msg,@stack)
     end
@@ -314,17 +1058,19 @@ end
 
 =begin
   rescue Exception=>e
+    if ENV['PRINT_PARSE_ERRORS']
       input=@lexer
       if Array===input
-        puts "error while parsing:"
-        pp input
+        STDERR.puts "error while parsing:"
+        STDERR.write input.pretty_inspect
         input=nil
       else
         input=input.original_file
-        inputname=@lexer.filename
-        input.to_s.size>1000 and input=inputname
-        puts "error while parsing: <<<  #{input}  >>>"
+#        inputname=@lexer.filename
+        STDERR.puts "error while parsing #@filename:#@endline: <<<  #{input.inspect if input.inspect.size<=1000}  >>>"
       end
+      e.backtrace.each{|l| p l }
+    end
     raise
   else
 =end
@@ -354,18 +1100,23 @@ end
 
   #inline any subsequences in RULES right into the patterns
   #reg should do this already, but current release does not
+  #also expand regexp/string to keyword matcher
   def expanded_RULES
+    return @rules if defined? @rules
     result=RULES()
-    return result if (-[:foo, -[:bar]]).subregs.grep(Reg::Subseq).empty?
-    result.map!{|rule|
-      unless rule.left.subregs.grep(Reg::Subseq)
+    #return result if (-[:foo, -[:bar]]).subregs.grep(Reg::Subseq).empty?
+    @rules=result.map!{|rule|
+      if rule.left.subregs.grep(Reg::Subseq|String|Regexp).empty?
       then rule
       else
         right=rule.right
         rule=rule.left.subregs.dup
         (rule.size-1).downto(0){|i|
-          if Reg::Subseq===rule[i]
-            rule[i,1]=rule[i].subregs
+          case mtr=rule[i]
+          when Reg::Subseq
+            rule[i,1]=mtr.subregs
+          when String,Regexp
+            rule[i]=RedParse::KW(mtr)
           end
         }
         -rule>>right
@@ -482,8 +1233,8 @@ end
       #the 'precedence' of comma is somewhat controversial. it actually has
       #several different precedences depending on which kind of comma it is.
       #the precedence of , is higher than :, => and the assignment operators 
-      #in certain (lhs) contexts. therefore, the precedence of lhs, should 
-      #really be above =.
+      #in certain (lhs) contexts. therefore, the precedence of lhs-comma should 
+      #really be above "=".
 
     #"unary" prefix function names seen has operators have this precedence
     #but, rubylexer handles precedence of these and outputs fake parens 
@@ -514,21 +1265,6 @@ if defined? SPECIALIZED_KEYWORDS
   end
 
   KW2class={}
-
-  Punc2name={
-    "("=>"lparen",    ")"=>"rparen",
-    "["=>"lbracket",    "]"=>"rbracket",
-    "{"=>"lbrace",    "}"=>"rbrace",
-    ","=>"comma",
-    ";"=>"semicolon",
-    "::"=>"double_colon",
-    "."=>"dot",
-    "?"=>"question_mark", ":"=>"colon",
-    "="=>"equals",
-    "|"=>"pipe",
-    "<<"=>"leftleft", ">>"=>"rightright",
-    "=>"=>"arrow",
-  }
 end
 
   def self.KW(ident)
@@ -665,17 +1401,24 @@ end
   end
 
   def lower_op
+    @lower_op||=proc{|parser,op| 
+      LOWEST_OP===op or (!(parser.VALUELIKE_LA() === op) and 
+        parser.left_op_higher(parser.stack[-3],op)
+      ) 
+    }.extend LowerOp_inspect
+=begin was
     return @lower_op if defined? @lower_op
     lower_op=item_that{|op| left_op_higher(@stack[-3],op) }
-    lower_op=(LOWEST_OP|(~VALUELIKE_LA & lower_op)).la
+    lower_op=(LOWEST_OP|(~VALUELIKE_LA() & lower_op)).la
     lower_op.extend LowerOp_inspect
     @lower_op=lower_op
+=end
   end
 
   #this is a hack, should use graphcopy to search for Deferreds and replace with double-Deferred as below
   def item_that(*a,&b)
     if defined? @generating_parse_tables
-      huh unless b
+      fail unless b
       #double supers, one of them in a block executed after this method returns....
       #man that's weird
       super(*a){|ob| @saw_item_that[[super(*a,&b),ob]]=true}
@@ -741,21 +1484,31 @@ end
   LHS_COMMA=Op('lhs,',true)#&-{:tag => :lhs}
   RHS_COMMA=Op('rhs,',true)#&-{:tag => :rhs}
   #PARAM_COMMA=Op('param,',true)#&-{:tag => :param}
-  FUNCLIKE_KEYWORD=KeywordToken&-{:ident=>RubyLexer::FUNCLIKE_KEYWORDS}
+  def FUNCLIKE_KEYWORD
+    KeywordToken&-{:ident=>@funclikes} 
+  end
   IGN_SEMI_BEFORE=KW(/^(#{RubyLexer::INNERBOUNDINGWORDS.gsub(/(rescue|then)\|/,'')[1...-1]}|end|[)}\]])$/)|EoiToken
   IGN_SEMI_AFTER=KW(/^(begin|[;:({|]|then|do|else|ensure)$/)|BlockFormalsNode
 
   #for use in lookback patterns
   OPERATORLIKE_LB=OperatorToken|
-                 KW(/^(not | defined\? | rescue3 | .*[@,] | [ ~ ! ; \( \[ \{ ? : ] | \.{1,3} | :: | => | ![=~])$/x)|
+                 KW(/^(not | defined\? | rescue3 | .*[@,] | [~!;\(\[\{?:] | \.{1,3} | :: | => | ![=~])$/x)|
                  KW(%r{^( \*\*? | << | >> | &&? | \|\|? | \^ | % | / | - | \+ )?=$}x)|
                  KW(BEGINWORDS)|KW(/^#{INNERBOUNDINGWORDS}$/)|RescueHeaderNode|StartToken|
                  GoalPostToken|BlockFormalsNode|AssignmentRhsListStartToken
 
   #for use in lookahead patterns
-  VALUELIKE_LA=KW(RubyLexer::VARLIKE_KEYWORDS)|NumberToken|SymbolToken|StringToken|UNOP|DEFOP|
-               KW(/^[({]$/x)|VarNameToken|MethNameToken|HerePlaceholderToken|
-               KW(BEGINWORDS)|FUNCLIKE_KEYWORD|AssignmentRhsListStartToken
+  def VALUELIKE_LA
+    @valuelike_la ||=
+
+    KW(@varlikes)|NumberToken|SymbolToken|StringToken|UNOP|DEFOP|
+    KW(/^[({]$/x)|VarNameToken|MethNameToken|HerePlaceholderToken|
+    KW(BEGINWORDS)|FUNCLIKE_KEYWORD()|AssignmentRhsListStartToken
+
+    #why isn't this a sufficient implementation of this method:
+         # KW('(')
+    #in which case, '(' can be made the highest precedence operator instead
+  end
   LOWEST_OP=KW(/^(#{ENDWORDS})$/)|KW(/^#{INNERBOUNDINGWORDS.sub('rescue|','')}$/)|
             EoiToken|GoalPostToken|AssignmentRhsListEndToken
 
@@ -770,6 +1523,8 @@ end
   def RULES
     lower_op= lower_op()
 
+
+    result=
     [-[StartToken.lb, Expr.-, EoiToken.la]>>:accept,
      -[EoiToken]>>:error,
     ]+
@@ -786,12 +1541,11 @@ end
 
 #    -[Op('=',true)|KW(/^(rescue|when|\[)$/)|Op(/,$/,true),
 #      Op(/^(?:unary|rhs)\*$/), ValueNode, (MODIFYASSIGNOP|Op('=',true)).la]>>:shift,
-#    -[MethNameToken|FUNCLIKE_KEYWORD, KW('('), 
+#    -[MethNameToken|FUNCLIKE_KEYWORD(), KW('('), 
 #      Op(/^(?:unary|rhs)\*$/), ValueNode, (MODIFYASSIGNOP|Op('=',true)).la]>>:shift,
     #star should not be used in an lhs if an rhs or param list context is available to eat it.
     #(including param lists for keywords such as return,break,next,rescue,yield,when)
 
-    #hmmm.... | in char classes below looks useless (predates GoalPostToken)
     -[Op(/^(?:unary|lhs)\*$/), (GoalPostToken|Op(/,$/,true)|KW(/^(in|[=)|;])$/)).la]>>DanglingStarNode, #dangling *
     -[Op(/,$/,true), (GoalPostToken|KW(/^(in|[=)|;])$/)).la]>> #dangling ,
       stack_monkey("DanglingComma",1,DanglingCommaNode){|stack| 
@@ -799,9 +1553,12 @@ end
         dcomma.offset=stack.last.offset
         stack.push dcomma, stack.pop
       },
+    #hmmm.... | in char classes above looks useless (predates GoalPostToken)
+
+#    -[Expr, Op|KW_Op, Expr, lower_op]>>RawOpNode,  #most operators
+    -[Expr, Op, Expr, lower_op]>>RawOpNode,  #most operators
+    -[Expr, "=>", Expr, lower_op]>>ArrowOpNode,
 
-    -[Expr, Op|KW_Op, Expr, lower_op]>>RawOpNode,  #most operators
-    
     #assignment
     -[Lvalue, MODIFYASSIGNOP, Expr, lower_op]>>AssignNode,
     -[Lvalue, Op('=',true), AssignmentRhsNode, lower_op]>>AssignNode,
@@ -840,11 +1597,6 @@ end
         stack[-3].after_comma=true}, 
                #mebbe this should be a lexer hack?
 
-    -[#(OPERATORLIKE_LB&~(MethNameToken|FUNCLIKE_KEYWORD)).lb, 
-      '(', Expr, KW(')')&~(-{:callsite? =>true}|-{:not_real? =>true})]>>ParenedNode,
-    -[#(OPERATORLIKE_LB&~(MethNameToken|FUNCLIKE_KEYWORD)).lb, 
-      '(', KW(')')&~(-{:callsite? =>true}|-{:not_real? =>true})]>>VarLikeNode, #(), alias for nil
-
     -[#(OPERATORLIKE_LB&~Op('=',true)).lb, 
       Expr, RESCUE_OP, Expr, lower_op]>>RescueOpNode,
 
@@ -853,12 +1605,22 @@ end
     -[Expr, DotOp, CallNode, lower_op]>>DotCall,      #binary .
     -[Expr, DoubleColonOp, CallNode, lower_op]>>DotCall,    #binary ::
     -[Expr, DoubleColonOp, VarNode, lower_op]>>ConstantNode,#binary ::
+    #lower_op constaints on lookahead are unnecessary in above 4 (unless I give openparen a precedence)
 
     -[Expr, "?", Expr, ":", Expr, lower_op]>>TernaryNode,
 
 
     -[MethNameToken, '(', Expr.-, ')', BlockNode.-, KW('do').~.la]>>CallNode,
-    -[FUNCLIKE_KEYWORD, '(', Expr.-, ')', BlockNode.-, KW('do').~.la]>>KWCallNode,
+    -[FUNCLIKE_KEYWORD(), '(', Expr.-, ')', BlockNode.-, KW('do').~.la]>>KWCallNode,
+
+    -[#(OPERATORLIKE_LB&
+      (MethNameToken|FUNCLIKE_KEYWORD()).~.lb, 
+      '(', Expr, KW(')')&~(-{:callsite? =>true}|-{:not_real? =>true}), KW('do').~.la]>>ParenedNode,
+
+    -[#(OPERATORLIKE_LB&
+      (MethNameToken|FUNCLIKE_KEYWORD()).~.lb, 
+      '(', KW(')')&~(-{:callsite? =>true}|-{:not_real? =>true}), KW('do').~.la]>>VarLikeNode, #(), alias for nil
+    #constraint on do in above 2 rules is probably overkill
 
     -[ValueNode, Op(/,$/,true), ValueNode, lower_op]>>CommaOpNode,
 
@@ -996,17 +1758,26 @@ end
       },
  
    #treat these keywords like (rvalue) variables.
-   -[RubyLexer::VARLIKE_KEYWORDS]>>VarLikeNode,
+   -[@varlikes]>>VarLikeNode,
 
    #here docs
    -[HerePlaceholderToken]>>HereDocNode,
-   -[HereBodyToken.la]>>delete_monkey(1,"delete_here_body"),
-   ##this is rediculous. this should be a lexer hack?
+   #-[HereBodyToken.la]>>delete_monkey(1,"delete_here_body"),   ##this is rediculous. this should be a lexer hack?
 
    -[VarNameToken]>>VarNode,
 
 
   ]
+
+  if @rubyversion >= 1.9
+    result.concat [
+#      -['->', ParenedNode.-, 'do', Expr.-, 'end']>>ProcLiteralNode,
+#      -['->', VarLikeNode["nil",{:@value=>nil}].reg, 'do', Expr.-, 'end']>>ProcLiteralNode,
+      -[(DotOp|DoubleColonOp).lb, '(',Expr.-,')', BlockNode.-, KW('do').~.la]>>CallNode,
+     ]
+  end
+
+  return result
   end
 
 if defined? END_ATTACK
@@ -1014,32 +1785,62 @@ if defined? END_ATTACK
   include Reducer
 end
 
-  def initialize(input,name="(eval)",line=1,lvars=[],options={})
-    @rubyversion=options[:rubyversion]||1.8
+  def signature
+    ancs=class<<self; ancestors end.reject{|k| !k.name}
+    ancs.slice!(ancs.index(RedParse)..-1)
 
-    cache=Cache.new(name,line,lvars.sort.join(" "),@rubyversion,self.class.name)
-    cache_mode=options[:cache_mode]||:read_write
-    raise ArgumentError unless /^(?:read_(?:write|only)|write_only|none)$/===cache_mode.to_s    
-    read_cache= /read/===cache_mode.to_s
+    lancs=class<<@lexer; ancestors end.reject{|k| !k.name}
+    [RubyLexer,Array].each{|k|
+      if i=lancs.index(k)
+        lancs.slice!(i..-1)
+      end
+    }
+
+    RedParse.signature(@encoding,ancs+lancs)
+  end
+  def RedParse.signature(encoding,ancs=ancestors)
+    @@my_sha||=Digest::SHA256.file(__FILE__)
+    @@node_sha||=Digest::SHA256.file(__FILE__.sub(/\.rb\z/,"/node.rb"))
+    [ancs.map{|m| m.name}, encoding, @@my_sha, @@node_sha,]
+  end
+
+  def initialize(input,name=nil,line=nil,lvars=nil,options=nil)
+    line,name=name,nil if Hash===name or Array===name or Integer===name
+    lvars,line=line,nil if Hash===line or Array===line
+    options,lvars=lvars,nil if Hash===lvars
+    options||={}
+    name||=options[:name]||"(eval)"
+    line||=options[:line]||1
+    lvars||=options[:lvars]||[]
+    @rubyversion=options[:rubyversion]||1.8
     input.binmode if input.respond_to? :binmode
-    if read_cache and cache and result=cache.get(input)
-      @cached_result=result
-      @write_cache=nil
-      return
-    end
-    if /write/===cache_mode.to_s
-      @write_cache,@input= cache,input 
-    else
-      @write_cache=nil
-    end
+
+    @encoding=options[:encoding]||:ascii
+    @encoding=:binary if @rubyversion<=1.8
 
     if Array===input
       def input.get1token; shift end
       @lexer=input
+      if @rubyversion>=1.9
+        @funclikes=RubyLexer::RubyLexer1_9::FUNCLIKE_KEYWORDS
+        @varlikes=RubyLexer::RubyLexer1_9::VARLIKE_KEYWORDS
+      else
+        @funclikes=RubyLexer::FUNCLIKE_KEYWORDS
+        @varlikes=RubyLexer::VARLIKE_KEYWORDS
+      end
     else
-      @lexer=RubyLexer.new(name,input,line,0,:rubyversion=>@rubyversion)
+      if RubyLexer===input
+        @lexer=input
+      else
+        @lexer=RubyLexer.new(name,input,line,0,:rubyversion=>@rubyversion,:encoding=>@encoding)
+      end
+      @funclikes=@lexer::FUNCLIKE_KEYWORDS()
+      @varlikes=@lexer::VARLIKE_KEYWORDS()
       lvars.each{|lvar| @lexer.localvars[lvar]=true }
+      @encoding=@lexer.encoding_name_normalize(@encoding.to_s).to_sym
+      warn "#{@encoding} encoding won't really work right now" if RubyLexer::NONWORKING_ENCODINGS.include? @encoding
     end
+    @funclikes=/#@funclikes|^->$/ if @rubyversion>=1.9
     @filename=name
     @min_sizes={}
     @compiled_rules={}
@@ -1052,26 +1853,82 @@ if defined? END_ATTACK
     compile
 end
     @saw_item_that=nil
+    @print_filter=proc{true}
+ 
+    if modules=options[:modules]
+      modules.each{|m| extend m}
+    end
+    if modules=options[:lexer_modules]
+      modules.each{|m| @lexer.extend m}
+    end
+
+    dir=reduce_withs_directory
+    modname="ReduceWithsFor_#{parser_identity.join('_').tr(':.','_')}"
+
+    cache=Cache.new(
+      File===input,name, 
+        :line,line,:encoding,@encoding,:locals,lvars.sort.join(","), 
+      @rubyversion, :/, *signature
+    )
+    cache_mode=options[:cache_mode]||ENV['REDPARSE_CACHE']||:read_write
+    cache_mode=cache_mode.to_sym
+    raise ArgumentError,"bad cache mode #{cache_mode}" unless /^(?:read_(?:write|only)|write_only|none)$/===cache_mode.to_s    
+    read_cache= /read/===cache_mode.to_s
+    if read_cache and cache and result=cache.get(input)
+      @cached_result=result
+      @write_cache=nil
+      return
+    end
+    if /write/===cache_mode.to_s
+      @write_cache,@input= cache,input 
+    else
+      @write_cache=nil
+    end
+
+    #but, need to skip warning lines matching this regexp:
+    #  /(^|[/\\])#{modname}\.rb:\d+: warning: mismatched indentations at 'end' with 'if' at \d+$/
+
+    begin
+    require File.join(dir,modname)
+    rescue LoadError
+    else
+     extend self.class.const_get( modname )
+    end
+    redparse_modules_init
+  end
+
+  def self.parse(*args)
+    new(*args).parse
+  end
+
+  def redparse_modules_init
+
+  end
+
+  def reduce_withs_directory
+    "redparse"
   end
 
-  attr_accessor :lexer
-  attr :rubyversion
+  attr_accessor :lexer, :print_filter
+  attr_reader :rubyversion, :stack
+
+  alias rules expanded_RULES
 
   def get_token(recursing=false)
     unless @moretokens.empty? 
       @last_token=@moretokens.shift
-      p @last_token if ENV['PRINT_TOKENS'] unless recursing
+      p @last_token if ENV['PRINT_TOKENS'] && @print_filter[@last_token] and not recursing
       return @last_token
     end
 
     rpt=ENV['RAW_PRINT_TOKENS']
     begin
       result=@lexer.get1token or break
-      p result if rpt
+      p result if rpt and @print_filter[result]
 
       #set token's line
       result.startline= @endline||=1
-      result.endline||=@endline if result.respond_to? :endline=
+      #result.endline||=@endline if result.respond_to? :endline=
 
       if result.respond_to?(:as) and as=result.as
         #result=make_kw(as,result.offset)
@@ -1080,7 +1937,10 @@ end
           result=result.dup
           result.ident=as
         else
-          result=make_kw(as,result.offset)
+          result2=make_kw(as,result.offset)
+          result2.startline=result.startline
+          result2.endline=result.endline
+          result=result2
         end
         result.not_real! if result.respond_to? :not_real!
       else
@@ -1102,28 +1962,44 @@ end
           case name=result.ident
 
           when /^(#{BINOP_KEYWORDS.join '|'})$/o #should be like this in rubylexer
-            result=OperatorToken.new(name,result.offset) unless result.has_end?
-          when "|"; result=GoalPostToken.new(result.offset) #is this needed still?
+            unless result.has_end?
+              orig=result 
+              result=OperatorToken.new(name,result.offset)
+              result.endline=orig.endline
+            end
+          when "|";
+            orig=result 
+            result=GoalPostToken.new(result.offset) #is this needed still? (yes)
+            result.endline=orig.endline
           when "__FILE__"; #I wish rubylexer would handle this
-            class<<result; attr_accessor :value; end
-            result.value=@file.dup
+            #class<<result; attr_accessor :value; end
+            assert result.value==@file.dup
           when "__LINE__"; #I wish rubylexer would handle this
-            class<<result; attr_accessor :value; end
-            result.value=@endline
+            #class<<result; attr_accessor :value; end
+            assert result.value==@endline
           else 
             result=make_kw name,result.offset if defined? SPECIALIZED_KEYWORDS
             #warning, this may discard information stored in instance vars of result
           end
 
+      when StringToken,HerePlaceholderToken
+        @endline=result.endline
+
       when EoiToken; break
-      when HereBodyToken; break
+      when HereBodyToken;
+        @endline=result.endline
+        redo
       when AssignmentRhsListStartToken; break
       when AssignmentRhsListEndToken; break
       when IgnoreToken; redo
       end
       end
     end while false
-    p result if ENV['PRINT_TOKENS'] unless recursing
+    p result if ENV['PRINT_TOKENS'] && @print_filter[@last_token] unless recursing
+
+    #ugly weak assertion
+    assert result.endline==@endline unless result.ident==';' && result.endline-1==@endline or EoiToken===result
+
     return @last_token=result
   end
 
@@ -1135,21 +2011,11 @@ end
     @moretokens.unshift token
   end
 
-=begin
-  self.LOOKAHEAD_CLASSES.each_with_index{|classes,i|
-    case classes
-    when Class: huh
-    when Array: classes.flatten.each{huh}
-    else 
-    end
-  }
-=end
-
-#  def fixup_multiple_assignments!; end
 end
 
 
 if __FILE__==$0
+  #this code has moved to bin/redparse; really, all this should just go away
   require 'problemfiles'
   class NeverExecThis<RuntimeError; end