redparse 0.8.0

data/Rakefile

@@ -0,0 +1,26 @@
+# Copyright (C) 2008  Caleb Clausen
+# Distributed under the terms of Ruby's license.
+require 'rubygems'
+require 'hoe'
+require 'lib/redparse/version.rb'
+ 
+ 
+   readme=open("README.txt")
+   readme.readline("\n== DESCRIPTION:")
+   readme.readline("\n\n")
+   desc=readme.readline("\n\n")
+ 
+   hoe=Hoe.new("redparse", RedParse::VERSION) do |_|
+     _.author = "Caleb Clausen"
+     _.email = "redparse-owner @at@ inforadical .dot. net"
+     _.url = ["http://redparse.rubyforge.org/", "http://rubyforge.org/projects/redparse/"]
+     _.extra_deps << ['rubylexer', '>= 0.7.2']
+#     _.test_globs=["test/*"]
+     _.description=desc
+     _.summary=desc[/\A[^.]+\./]
+     _.spec_extras={:bindir=>''}
+     _.rdoc_pattern=/\A(README\.txt|lib\/.*\.rb)\Z/
+     _.remote_rdoc_dir="/"
+   end
+
+

data/lib/redparse.rb

@@ -0,0 +1,1083 @@
+=begin
+    redparse - a ruby parser written in ruby
+    Copyright (C) 2008  Caleb Clausen
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+=end
+
+#warn 'hacking up LOAD_PATH to include the latest RubyLexer!'
+#$:.unshift Dir.pwd+'/../rubylexer/lib', Dir.pwd+'/../rubylexer'
+
+# "faster rule compiler is untested"
+
+require 'rubygems'
+require 'rubylexer'
+require 'reg'
+
+require "redparse/node"
+#require "redparse/decisiontree"
+require "redparse/reg_more_sugar"
+class RedParse
+#  include Nodes
+
+  def self.has_return_hash_fix?
+    rl=RubyLexer.new("","return {}.size")
+    return(
+      FileAndLineToken===rl.get1token and
+      MethNameToken===rl.get1token and
+      ImplicitParamListStartToken===rl.get1token and
+      WsToken===rl.get1token and
+      KeywordToken===rl.get1token and
+      KeywordToken===rl.get1token and
+      KeywordToken===rl.get1token and
+      MethNameToken===rl.get1token and
+      ImplicitParamListStartToken===rl.get1token and
+      ImplicitParamListEndToken===rl.get1token and
+      ImplicitParamListEndToken===rl.get1token and
+      EoiToken===rl.get1token
+    )
+  end
+
+  #see pickaxe, 1st ed, page 221
+  def RIGHT_ASSOCIATIVE
+    {
+#    "defined?"=>120.5,
+    "**"=>118,   
+
+    "="=>105,    "%="=>105,   "/="=>105,   "-="=>105,    "+="=>105,
+    "|="=>105,   "&="=>105,   ">>="=>105,  "<<="=>105,   "*="=>105,
+    "&&="=>105,  "||="=>105,  "**="=>105,  "^="=>105,
+
+#    "and"=>99, "or"=>99,
+
+#   "if"=>98, "unless"=>98, "while"=>98, "until"=>98, "rescue"=>98, 
+
+#    "&&"=>109, "||"=>108,
+    }
+  end
+  
+  def PRECEDENCE
+    {
+
+  #  "("=>122,     #method param list
+  #  "{"=>122,    "do"=>122,    #blocks
+
+    "::"=>121,    "."=>121,
+
+ #   "defined?"=>120.5, 
+
+    "["=>120,     #[] []= methods
+
+    "!"=>119,    "~"=>119,
+    "+@"=>119, 
+
+    "**"=>118,   
+
+    "-@"=>117, 
+
+    "*"=>116,    "/"=>116,    "%"=>116,
+
+    "+"=>115,    "-"=>115,
+
+    "<<"=>114,   ">>"=>114,
+ 
+    "&"=>113,
+
+    "^"=>112,    "|"=>112,
+
+    "<="=>111,   ">="=>111,   "<"=>111,    ">"=>111,
+
+    "<=>"=>110,  "=="=>110,   "==="=>110,  
+    "!="=>110,   "=~"=>110,   "!~"=>110,
+
+    "&&"=>109,
+
+    "||"=>108,
+
+    ".."=>107, "..."=>107,
+
+    "?"=>106, # ":"=>106,    #not sure what to do with ":"
+  
+    "*@"=>105.5, "&@"=>105.5, #unary * and & operators
+
+    "="=>105,    "%="=>105,   "/="=>105,   "-="=>105,    "+="=>105,
+    "|="=>105,   "&="=>105,   ">>="=>105,  "<<="=>105,   "*="=>105,
+    "&&="=>105,  "||="=>105,  "**="=>105,  "^="=>105,
+
+    "defined?"=>103,
+    "not"=>103,
+    ":"=>102, #but not when used as a substitute for 'then'
+    "rescue3"=>102,
+
+    "=>"=>101,
+    ","=>100, 
+      #the 'precedence' of comma is somewhat controversial. it actually has
+      #several different precedences depending on which kind of comma it is.
+      #the precedence of , is higher than :, => and the assignment operators 
+      #in certain contexts.
+
+    #"unary" prefix function names seen has operators have this precedence
+    #but, rubylexer handles precedence of these and outputs fake parens 
+    #to tell us how its parsed
+
+    "or"=>99,   "and"=>99,
+
+    "if"=>98,    "unless"=>98,    "while"=>98,    "until"=>98,
+
+    "rescue"=>98,
+
+    ";"=>96,
+    }
+  end
+
+  module BracketsCall; end
+  
+  Value=     #NumberToken|SymbolToken|
+        #HerePlaceholderToken|
+        ((VarNameToken|ValueNode)&-{:lvalue? =>nil})
+  Expr=Value
+
+  def self.KW(ident)
+    ident=case ident
+          when Integer:        ident.chr
+          when String,Regexp:  ident
+          else                 ident.to_s
+          end
+
+    return KeywordToken&-{:ident=>ident}
+  end
+  def KW(ident); self.class.KW(ident) end
+  UNOP=
+  (OperatorToken|KeywordToken)&-{  #sppflt! KeywordToken here is a hack too
+    :ident=>/^[*&+-]@$/,
+    :unary =>true,
+  }|
+  (OperatorToken|KeywordToken)&-{  #sppflt! KeywordToken here is a hack too
+    :ident=>/^([~!]|not|defined\?)$/,
+  } #|
+  DEFOP=
+  (OperatorToken|KeywordToken)&-{  #sppflt! KeywordToken here is a hack too
+    :ident=>"defined?",
+  } 
+=begin
+  MethNameToken&-{ #hack, shouldn't be necessary
+     #rubylexer should know to generally treat "defined?" as a keyword
+     #or operator. (like most keywords, it can also be used as a method 
+     #               name....)
+    :ident=>"defined?"
+  }
+=end
+
+  def self.Op(ident=nil, allow_keyword=false)  
+    result=OperatorToken
+    result |= KeywordToken if allow_keyword
+    result &= -{:ident=>ident} if ident
+    #result[:infix?]=true
+    return result
+  end
+  def Op(*args); self.class.Op(*args); end
+  BINOP_KEYWORDS=%w[if unless while until and or && \|\|]
+ 
+  #HAS_PRECEDENCE=Op(/^#{PRECEDENCE.keys.map{|k| Regexp.quote k}.join('|')}$/,true)
+=begin
+  KeywordOp=
+    KeywordToken & -{
+      :ident=>/^(#{BINOP_KEYWORDS.join('|')})$/
+    }
+  KeywordOp2= 
+    KeywordToken & -{ 
+      :ident=>/^([\[({!+*?:,]|\.{1,3}|::|=>)$/ 
+    }
+=end
+  DotOp= KeywordToken & -{ :ident=>"." }
+  DoubleColonOp= KeywordToken & -{ :ident=>"::" }
+ 
+  Op=Op()
+  MODIFYASSIGNOP=Op( /^(([^=])\2|[^<>=!])=$/, true )
+  NONASSIGNOP=Op( /([^=]|[<>=!]=)$/)
+  KW_Op= #some of these ought to be regular operators, fer gosh sake
+    Op(/^((![=~])|\.\.\.?|=>)$/,true)|Op(/^(#{BINOP_KEYWORDS.join('|')})$/)
+
+  EPSILON=Float::EPSILON*10_000_000    #this should be <<1 and >0 
+  fail unless  1+EPSILON>1
+  fail unless EPSILON<0.1
+
+  def left_op_higher(op,op2)
+#    (Op|KeywordOp|KeywordOp2|ASSIGNOP===op2) or return true 
+    KeywordToken===op2 or OperatorToken===op2 or return true
+    rightprec=@precedence[op2.to_s] or return true
+#or fail "unrecognized right operator: #{op2.inspect}"
+    rightprec+=EPSILON if @RIGHT_ASSOCIATIVE[op2.to_s]
+    return @precedence[op.to_s]>=rightprec
+  end
+
+  LowerOp=     proc{|parser,op2| parser.left_op_higher(parser[-3],op2) }
+
+  def dont_postpone_semi
+    @dps||=~wants_semi_context
+  end
+  WANTS_SEMI=%w[while until if unless 
+          def case when in rescue 
+          elsif class module << => . ::
+          ]
+  def wants_semi_context
+    Op('<<')|KW(/^(#{WANTS_SEMI.map{|ws| Regexp.quote ws }.join('|')})$/)
+  end
+
+  NeverBlockKeyword=MethNameToken&-{:ident=>/^(return|break|next)$/}
+  FakeBegin=KW('(')&-{:not_real? =>true}
+  FakeEnd=KW(')')&-{:not_real? =>true}
+
+  #rule format: 
+  # -[syntax pattern_matchers.+, lookahead.-]>>node type
+
+  DotCall=proc{|stack|
+    right=stack[-2]
+    left,bogus=*stack.slice!(-4..-3)
+ 
+    right.set_receiver! left
+  }
+
+  Lvalue=(VarNameToken|CallSiteNode|BracketsGetNode|CommaOpNode|
+          ParenedNode|ConstantNode|UnaryStarNode)&-{:lvalue? =>true}
+
+  BareMethod=MethNameToken|LiteralNode&-{:val=>Symbol|StringNode}
+
+  BEGINWORDLIST=RubyLexer::BEGINWORDLIST + %w"( [ {"  
+  ENDWORDLIST=%w"end ) ] }"
+  BEGIN2END={"{"=>"}", "("=>")", "["=>"]", }
+    endword="end"
+    RubyLexer::BEGINWORDLIST.each{|bw| BEGIN2END[bw]=endword }
+  def beginsendsmatcher
+    @bem||=
+    /^(#{(BEGINWORDLIST+ENDWORDLIST).map{|x| Regexp.quote x}.join('|')})$/
+  end
+
+  MULTIASSIGN=UnaryStarNode|CommaOpNode|(ParenedNode&-{:size=>1})
+  WITHCOMMAS=UnaryStarNode|CommaOpNode|
+      (CallSiteNode&-{:real_parens=>false, :args=>-{:size=>~0.reg}}) #|(ParenedNode&-{:size=>~1.reg})
+
+  BEGINAFTEREQUALS=
+    ParenedNode&
+      -{:size =>~1.reg, :op? =>NilClass|FalseClass, :after_equals =>nil}&
+      (-{:body=>item_that.size>0}|-{:rescues=>item_that.size>0}|-{:ensures=>~NilClass})
+#      item_that{|x| x.body.size+x.rescues.size > 0 or x.ensures }
+  
+#  ASSIGN_COMMA=Op(',',true)&-{:comma_type=>Symbol}
+  LHS_COMMA=Op(',',true)&-{:comma_type => :lhs}
+  RHS_COMMA=Op(',',true)&-{:comma_type => :rhs}
+  PARAM_COMMA=Op(',',true)&-{:comma_type => :param}
+  FUNCLIKE_KEYWORD=KeywordToken&-{:ident=>RubyLexer::FUNCLIKE_KEYWORDS}
+
+  def RULES
+    #these must be the lowest possible priority, and hence first in the rules list
+    BEGIN2END.map{|_beg,_end| 
+      -[KW(_beg), KW(beginsendsmatcher).~.*, KW(_end), KW(/^(do|\{)$/).~.la]>>MisparsedNode
+    }+
+
+    [
+    -[UNOP, Value, LowerOp]>>UnOpNode,
+    -[DEFOP, ParenedNode&-{:size=>1}]>>UnOpNode,
+    -[Op('*@'), VarNameToken|ValueNode, LowerOp]>>UnaryStarNode,
+
+    -[Op('=',true)|KW(/^(rescue|when|\[)$/)|-{:comma_type=>:call.reg|:array|:param|:rhs}, 
+      Op('*@'), VarNameToken|ValueNode, (MODIFYASSIGNOP|Op('=',true)).la]>>:shift,
+    -[MethNameToken|FUNCLIKE_KEYWORD, KW('('), 
+      Op('*@'), VarNameToken|ValueNode, (MODIFYASSIGNOP|Op('=',true)).la]>>:shift,
+#    -[KW('[')|-{:comma_type=>:call.reg|:array}, 
+#      Op('*@'), VarNameToken|ValueNode, Op('=',true).la]>>:shift,
+    #star should not be used in an lhs if an rhs or param list context is available to eat it.
+    #(including param lists for keywords such as return,break,next,continue,rescue,yield,when)
+
+    -[Op('*@'), (GoalPostNode|KW(/^(in|[=)|,;])$/)).la]>>DanglingStarNode, #dangling *
+    -[',', (GoalPostNode|KW(/^(in|[=)|;])$/)).la]>> #dangling ,
+      proc{|stack| 
+        dcomma=DanglingCommaNode.new
+        dcomma.offset=stack.last.offset
+        stack.push dcomma, stack.pop
+      },
+
+    -[Value, Op|KW_Op, Value, LowerOp]>>RawOpNode,  #most operators
+    
+    #assignment
+    -[Lvalue, MODIFYASSIGNOP, Value, LowerOp]>>AssignNode,
+    -[Lvalue, Op('=',true), AssignmentRhsNode, LowerOp]>>AssignNode,
+    -[Op('=',true).lb, Value, LowerOp]>>AssignmentRhsNode,
+#was:    -[AssignmentRhsListStartToken, Value, AssignmentRhsListEndToken]>>AssignmentRhsNode,
+    -[Lvalue&~MULTIASSIGN, Op('=',true), AssignmentRhsNode&-{:val =>~WITHCOMMAS}, 
+           Op('rescue3',true), Value, LowerOp]>>AssignNode,
+    -[Lvalue&~MULTIASSIGN, Op('=',true), AssignmentRhsNode&-{:val =>~WITHCOMMAS}, 
+        Op('rescue3',true).la]>>:shift,
+    -[Lvalue&~MULTIASSIGN, Op('=',true), AssignmentRhsNode&-{:val =>~WITHCOMMAS}, 
+        Op('rescue',true).la] >>proc{|stack| 
+          resc=stack.last.dup
+          resc.ident += '3'
+          stack[-1]=resc
+        },
+    # a = b rescue c acts like a ternary,,,
+    #provided that both a and b are not multiple and b 
+    #(if it is a parenless callsite) has just 1 param
+
+#    -[Op('=',true), ~WITHCOMMAS, Op('rescue',true).la]>>:shift,  
+    #relative precedence of = and rescue are to be inverted if rescue
+    #is to the right and assignment is not multiple.
+
+    -[Op('=',true).~.lb, OB, Op('=',true), Value, RHS_COMMA.la]>>:shift,  
+    -[RHS_COMMA.lb, Lvalue, Op('=',true), Value, RHS_COMMA.la ]>>AssignNode,
+    -[ValueNode|VarNameToken, LHS_COMMA, ValueNode|VarNameToken, Op('=',true).la]>>CommaOpNode,
+    #relative precedence of = and lhs/rhs , are to be inverted.
+  
+    -[KW(',')&-{:comma_type=>:lhs}, (UnaryStarNode|ParenedNode)&~-{:after_comma =>true}, Op('=',true)]>>
+      proc{|stack| stack[-3].after_comma=true}, #mebbe this should be a lexer hack
+    #mark parentheses and unary stars that come after lhs commas
+
+    #-[Value, DotOp|DoubleColonOp, MethNameToken, 
+    #  ASSIGNOP, Value, LowerOp]>>AccessorAssignNode,
+
+    -[MethNameToken.~.lb, '(', Value, ')']>>ParenedNode,
+    -[MethNameToken.~.lb, '(', ')']>>VarLikeNode, #alias for nil
+
+#    -[Value, KeywordOp, Value, LowerOp]>>KeywordOpNode,
+    -[Op('=',true).~.lb, Value, Op('rescue',true), Value, LowerOp]>>ParenedNode,
+
+    #dot and double-colon
+    -[DoubleColonOp, VarNameToken,  LowerOp]>>ConstantNode,#unary ::
+    -[Value, DotOp, CallNode, LowerOp]>>DotCall,      #binary .
+    -[Value, DoubleColonOp, CallNode, LowerOp]>>DotCall,    #binary ::
+    -[Value, DoubleColonOp, VarNameToken, LowerOp]>>ConstantNode,#binary ::
+
+    -[Value, "?", Value, ":", Value, LowerOp]>>TernaryNode,
+
+#    -[Value, /^\.\.\.?$/, Value, LowerOp]>>RangeNode,
+
+    -[MethNameToken, '(', Value.-, ')', BlockNode.-, KW(/^(do|\{)$/).~.la]>>CallNode,
+    -[FUNCLIKE_KEYWORD, '(', Value.-, ')', 
+      BlockNode.-, KW(/^(do|\{)$/).~.la]>>KWCallNode,
+
+    -[ValueNode|VarNameToken, ',', ValueNode|VarNameToken, LowerOp]>>CommaOpNode,
+
+    -[dont_postpone_semi.lb, 
+      Value, ';', Value, LowerOp]>>SequenceNode,
+
+#    -[Value, '=>', Value, LowerOp]>>ArrowOpNode,
+
+    -[KW(')').~.lb, '{', (CommaOpNode|ArrowOpNode).-, '}']>>HashLiteralNode,
+
+#    -[CallSiteNode.~.lb, '{', Value, '}']>>HashLiteralNode,
+
+#    -[KW(')').lb, '{', BlockFormalsNode.-, Value.-, '}']>>BlockNode,
+    -[KW(')').lb, 'do', BlockFormalsNode.-, Value.-, 'end']>>BlockNode,
+    #rubylexer handles the 'low precedence' of do...end
+
+    -[GoalPostNode, Value.-, GoalPostNode]>>BlockFormalsNode,
+    #rubylexer disambiguated operator vs keyword '|'
+
+    -[/^(while|until)$/, Value, /^([:;]|do)$/, Value.-, 'end']>>LoopNode,
+
+    -[/^(if|unless)$/, Value, /^(;|then|:)$/, 
+      Value.-, ElsifNode.*, ElseNode.-, 'end'
+     ]>>IfNode,
+
+    -['else', Value.-, KW(/^(ensure|end)$/).la]>>ElseNode,
+
+    -['elsif', Value, /^(;|then|:)$/, Value.-,
+      KW(/^e(nd|ls(e|if))$/).la
+     ]>>ElsifNode,
+
+    -['module', ConstantNode|VarNameToken, KW(';'), Value.-, 'end']>>ModuleNode,
+#     -['module', ConstantNode|VarNameToken, KW(/^(;|::)$/).~.la]>>
+#       proc{|stack| #insert ; at end of module header if none was present
+#         stack.push KeywordToken.new(';'), stack.pop
+#       },
+    -['class', Value, ';', Value.-, 'end']>>ClassNode,
+    -['class', Value, Op('<'), Value, KW(';').~.la]>>:shift,
+    -['class', Op('<<'), Value, ';', Value.-, 'end']>>MetaClassNode,
+
+    -['alias', BareMethod|VarNameToken, BareMethod|VarNameToken]>>AliasNode,
+    -['undef', BareMethod]>>UndefNode,
+    -[UndefNode, ',', BareMethod]>>UndefNode,
+
+    -['def', CallSiteNode, Op('=').-, KW(';'), 
+        Value.-, RescueNode.*, ElseNode.-, EnsureNode.-, 
+      'end'
+    ]>>MethodNode,
+
+    -['begin', 
+        Value.-, RescueNode.*, ElseNode.-, EnsureNode.-, 
+      'end'
+    ]>>ParenedNode,
+
+    -[Op('=',true), BEGINAFTEREQUALS, Op('rescue',true).la]>>
+      proc{ |stack| stack[-2].after_equals=true }, 
+    #this is bs. all for an extra :begin in the parsetree
+
+    -[(KW(/^(;|begin)$/)|ParenedNode|RescueNode).lb,
+      'rescue', KW('=>').-, Value.-, /^([:;]|then)$/,
+    ]>>RescueHeaderNode,
+    -[ RescueHeaderNode, Value.-, KW(';').-, KW(/^(rescue|else|ensure|end)$/).la
+    ]>>RescueNode,
+
+    -['ensure', Value.-, KW('end').la]>>EnsureNode,
+
+    -['[', Value.-, ']']>>ArrayLiteralNode,
+
+    -[Value, '[', Value.-, ']']>>BracketsGetNode,
+
+    -[HereDocNode, StringToken.*, StringToken, StringToken.~.la]>>StringCatNode,  
+    -[(StringToken|HereDocNode).~.lb, StringToken.*, StringToken, StringToken, StringToken.~.la]>>StringCatNode,  
+    -[(StringToken|HereDocNode).~.lb, StringToken, StringToken.~.la]>>StringNode,  #includes regexp, wordlist, backquotes
+
+    -['case', Value.-, KW(/^[:;]$/).-, WhenNode.*, ElseNode.-, 'end']>>CaseNode,
+
+    -['when', Value, /^([:;]|then)$/, Value.-, 
+     KW(/^(when|else|end)$/).la
+    ]>>WhenNode,            
+
+    -['for', Value, 'in', Value, /^([:;]|do)$/, Value.-, 'end']>>ForNode,
+
+    #semicolon cleanup....
+    -[dont_postpone_semi.lb, 
+      Value, ';', 
+      (KW(/^(#{RubyLexer::INNERBOUNDINGWORDS.gsub(/(rescue|then)\|/,'')}|end|[)}\]])$/)|EoiToken).la
+    ]>>proc{|stack| stack.delete_at -2 },
+    -[Value, ';', KW('then').la
+    ]>>proc{|stack| stack.delete_at -2 },
+    -[dont_postpone_semi.lb, Value, ';', RescueNode
+    ]>>proc{|stack| stack.delete_at -3 },
+    -[(KW(/^(begin|[;:({|]|then|do|else|ensure)$/)|StartNode|RescueHeaderNode).lb, ';' 
+    ]>>proc{|stack| stack.delete_at -2 },
+     #this rule is somewhat more forgiving than matz' parser...
+     #not all semicolons after :, (, and { keywords should 
+     #be ignored. some should cause syntax errors.
+
+ 
+    #comma cleanup....
+    -[',', KW(/^[}\]]$/).la]>>proc{|stack| stack.delete_at -2},
+    #likewise, this is somewhat too forgiving.
+    #some commas before } or ] should cause syntax errors
+
+    #multiple assignment.... (handled in a subsequent stage?)
+    #(cause it requires that the relative priorities of = and , be reversed!)
+
+
+    #turn lvalues into rvalues if not followed by an assignop
+    -[-{:lvalue? =>true}, (Op('=',true)|MODIFYASSIGNOP).~.la]>>proc{|stack| stack[-2].lvalue=nil},
+    
+    #expand the = into a separate token in calls to settors (after . or ::).
+    #but not in method headers
+    -[KW('def').~.lb, Value, DotOp|DoubleColonOp, 
+      (MethNameToken&-{:ident=>/^[a-z_][a-z0-9_]*=$/i}).la]>>
+      proc{|stack| 
+        methname=stack.pop
+        methname.ident.chomp!('=')
+        offset=methname.offset+methname.ident.size
+        stack.push(
+          CallNode.new(methname,nil,nil,nil,nil),
+          OperatorToken.new('=',offset)
+        )
+      },
+
+   -[NumberToken|SymbolToken]>>LiteralNode,
+
+   #lexer does the wrong thing with -22**44.5, making the - part
+   #of the first number token. it's actually lower precedence than
+   #**... this rule fixes that problem.
+   -[NumberToken&-{:ident=>/\A-/}, Op('**').la]>>
+      proc{|stack|
+        neg_op=OperatorToken.new("-@",stack[-2].offset)
+        neg_op.unary=true
+        stack[-2,0]=neg_op
+        stack[-2].ident.sub!(/\A-/,'')
+        stack[-2].offset+=1
+      },
+ 
+   #treat these keywords like (rvalue) variables.
+   -[/^(nil|false|true|__FILE__|__LINE__|self)$/]>>VarLikeNode,
+
+   #here docs
+   -[HerePlaceholderToken]>>HereDocNode,
+   -[HereBodyToken]>>proc{|stack| 
+     stack.delete_at(-2)#.instance_eval{@headtok}.node.saw_body! 
+   },
+
+  ]
+  end
+
+
+
+  def initialize(input,name="(eval)",line=1,lvars=[])
+    if Array===input
+      def input.get1token; shift end
+      @lexer=input
+    else
+      @lexer=RubyLexer.new(name,input,line)
+      lvars.each{|lvar| @lexer.localvars[lvar]=true }
+    end
+    @filename=name
+    @min_sizes={}
+    @compiled_rules={}
+    @moretokens=[]
+    @unary_or_binary_op=/^[-+&*]$/
+    @rules=self.RULES
+    @precedence=self.PRECEDENCE
+    @RIGHT_ASSOCIATIVE=self.RIGHT_ASSOCIATIVE
+  end
+
+  attr_accessor :lexer
+
+  def get_token(recursing=false)
+    unless @moretokens.empty? 
+      @last_token=@moretokens.shift
+      p @last_token if ENV['PRINT_TOKENS'] unless recursing
+      return @last_token
+    end
+
+    begin
+      result=@lexer.get1token or break
+      p result if ENV['RAW_PRINT_TOKENS']
+
+      #set token's line if wanted
+      result.line||=@line if result.respond_to? :line=
+
+      if result.respond_to?(:as) and as=result.as
+        result=KeywordToken.new(as,result.offset)
+        result.not_real!
+      else
+
+      case result
+#=begin
+      when ImplicitParamListStartToken:  #treat it like (
+          result=KeywordToken.new('(', result.offset)
+          result.not_real!
+#=end
+#=begin
+      when ImplicitParamListEndToken:
+          result=KeywordToken.new(')', result.offset)
+          result.not_real!
+#=end
+#      when AssignmentRhsListStartToken, AssignmentRhsListEndToken:
+        #do nothing, pass it thru
+#=begin
+      when NewlineToken:
+        result=KeywordToken.new(';',result.offset)
+#=end
+      when FileAndLineToken:  #so __FILE__ and __LINE__ can know what their values are
+        @file=result.file
+        @line=result.line
+        redo
+      when NoWsToken:  
+        #rubylexer disambiguates array literal from
+        #call to [] or []= method with a preceding NoWsToken...
+        #kind of a dumb interface.         
+        result=get_token(true)
+        result.ident=='[' and result.extend BracketsCall
+
+      
+      when OperatorToken:
+        if @unary_or_binary_op===result.ident and result.unary
+          result=result.dup
+          result.ident+="@"
+        end
+
+      #more symbol table maintenance....
+      when KeywordToken:
+          case name=result.ident
+
+#=begin
+          when "do":
+            if result.has_end?
+            else
+              result=KeywordToken.new(';',result.offset)
+            end
+#=end
+          when /^(#{BINOP_KEYWORDS.join '|'})$/: #should be like this in rubylexer
+            result=OperatorToken.new(name,result.offset) unless result.has_end?
+          when "|": result=GoalPostNode.new(result.offset) #is this needed still?
+          when "__FILE__": #I wish rubylexer would handle this
+            class<<result; attr_accessor :value; end
+            result.value=@file.dup
+          when "__LINE__": #I wish rubylexer would handle this
+            class<<result; attr_accessor :value; end
+            result.value=@line
+          end
+
+      when EoiToken: break
+      when HereBodyToken: break
+      when IgnoreToken: redo
+      end
+      end
+    end while false
+    p result if ENV['PRINT_TOKENS'] unless recursing
+    return @last_token=result
+  end
+
+  def evaluate rule
+    #dissect the rule
+if false
+    rule=rule.dup
+    lookahead_processor=(rule.pop if Proc===rule.last)
+    node_type=rule.pop
+else
+    Reg::Transform===rule or fail
+    node_type= rule.right
+    rule=rule.left.subregs.dup
+    lookahead_processor=(rule.pop if Proc|::Reg::LookAhead===rule.last)
+    lookback=rule[0]=rule[0].regs(0) if ::Reg::LookBack===rule[0]
+end
+    
+    #index of data at which to start matching
+    i=@stack.size-1   #-1 because last element of @stack is always lookahead
+
+    #I could call this a JIT compiler, but that's a bit grandiose....
+    #more of a JIT pre-processor
+    compiled_rule=@compiled_rules[rule]||=
+      rule.map{|pattern| 
+        String|Regexp===pattern ? KW(pattern) : pattern 
+      }
+
+    #what's the minimum @stack size this rule could match?
+    rule_min_size=@min_sizes[compiled_rule]||=
+      compiled_rule.inject(0){|sum,pattern| 
+        sum + pattern.itemrange.begin 
+      }
+    i>=rule_min_size or return false
+
+    matching=[]
+
+    #actually try to match rule elements against each @stack element in turn
+    compiled_rule.reverse_each{|matcher|
+      i.zero? and fail
+      target=matching
+      #is this matcher optional? looping?
+      loop= matcher.itemrange.last.to_f.infinite?
+      optional=matcher.itemrange.first.zero?
+      matching.unshift target=[]  if loop
+      if loop or optional
+        matcher=matcher.regs(0) 
+      end
+
+      begin
+        if matcher===@stack[i-=1]  #try match
+          target.unshift @stack[i]
+        else
+          #if match failed, the whole rule fails
+          #unless this match was optional, in which case, ignore it
+          #but bump the data position back up, since the latest datum
+          #didn't actually match anything.
+          return false unless optional or loop&&!target.empty?
+          i+=1
+          matching.unshift nil unless loop
+          break
+        end
+      end while loop
+    } 
+
+    matchrange= i...-1  #what elems in @stack were matched?
+
+    #give lookahead matcher (if any) a chance to fail the match
+    case lookahead_processor
+    when ::Reg::LookAhead:
+      return false unless lookahead_processor.regs(0)===@stack.last
+    when Proc:
+      return false unless lookahead_processor[self,@stack.last] 
+    end
+
+    #if there was a lookback item, don't include it in the new node
+    if lookback
+      matchrange= i+1...-1  #what elems in @stack were matched?
+      matching.shift
+    end
+
+    #replace matching elements in @stack with node type found
+    case node_type
+    when Class
+        node=node_type.new(*matching)
+        node.line=@line
+        @stack[matchrange]=[node]
+    when Proc;   node_type[@stack]
+    when :shift; return 0
+    else fail
+    end
+    
+    return true #let caller know we found a match
+
+    
+  rescue Exception=>e
+    puts "error (#{e}) while executing rule: #{rule.inspect}"
+    puts e.backtrace.join("\n")
+    raise
+  end
+
+  class ParseError<RuntimeError
+    def initialize(msg,stack)
+      super(msg)
+      @stack=stack
+if false
+      ranges=(1..stack.size-2).map{|i|
+        node=stack[i]
+        if node.respond_to? :linerange
+          node.linerange
+        elsif node.respond_to? :line
+          node.line..node.line
+        end
+      }
+      types=(1..stack.size-2).map{|i| stack[i].class }
+      msg += "couldn't interpret #{types.inspect} at line ranges: #{ranges.inspect}"
+end
+      super(msg)
+    end
+    attr :stack
+  end
+
+  def [](*args)
+    @stack.[] *args
+  end
+
+  def []=(*args)
+    @stack.[]= *args
+  end
+
+  def parse
+    oldparser=   Thread.current[:$RedParse_parser]
+    Thread.current[:$RedParse_parser]||=self
+
+    @stack=[StartNode.new, get_token] 
+           #last token on @stack is always implicitly the lookahead
+    loop {
+      #try all possible reductions
+      shift=nil
+      @rules.reverse_each{|rule|
+        shift=evaluate(rule) and break
+      }
+      next if shift==true 
+      
+      #no rule can match current @stack, get another token
+      tok=get_token
+
+      #are we done yet?
+      tok.nil? or EoiToken===tok && EoiToken===@stack.last and break
+
+      #shift our token onto the @stack
+      @stack.push tok
+    }
+
+    @stack.size==2 and return NopNode.new #handle empty parse string
+
+    #unless the @stack is 3 tokens, 
+    #with the last an Eoi, and first a StartNode
+    #there was a parse error
+    unless @stack.size==3
+      pp @stack[-[15,@stack.size].min..-1] if ENV['PRINT_STACK']
+      top=MisparsedNode.new("(toplevel)", @stack[1...-1],'')
+      raise ParseError.new(top.msg,@stack)
+    end
+    EoiToken===@stack.last or fail
+    StartNode===@stack.first or fail
+
+    result= @stack[1]
+
+
+    #multiple assignment must be resolved 
+    #afterwards by walking the parse tree.
+    #(because the relative precedences of = and , 
+    #are reversed in multiple assignment.)
+#    result.respond_to? :fixup_multiple_assignments! and
+#      result=result.fixup_multiple_assignments!
+
+    #relative precedence of = and rescue are also inverted sometimes
+#    result.respond_to? :fixup_rescue_assignments! and 
+#      result=result.fixup_rescue_assignments!
+
+    #do something with error nodes
+    msgs=[]
+    result.walk{|parent,i,subi,node|
+      not if ErrorNode===node
+        msgs<< @filename+":"+node.blame.msg
+      end
+    } if result.respond_to? :walk #hack hack
+    result.errors=msgs unless msgs.empty?
+    #other types of errors (lexer errors, exceptions in lexer or parser actions)
+    #should be handled in the same way, but currently are not
+#    puts msgs.join("\n")
+
+  rescue Exception=>e
+#    input=@filename
+#    if input=="(eval)" 
+      input=@lexer
+      if Array===input
+        puts "error while parsing:"
+        pp input
+        input=nil
+      else
+        input=input.original_file
+        inputname=@lexer.filename
+        input.to_s.size>1000 and input=inputname
+      end
+#    end
+    puts "error while parsing: <<<  #{input}  >>>"
+    raise
+  else
+    unless msgs.empty?
+      pp @stack[-[15,@stack.size].min..-1] if ENV['PRINT_STACK']
+      raise RedParse::ParseError.new(msgs.join("\n"),@stack)
+    end
+
+    return result
+  ensure 
+    Thread.current[:$RedParse_parser]=oldparser
+  end
+
+  def LEFT_MATCHERS;self.RULES.map{|r| r.left.subregs }.flatten; end
+  def STACKABLE_CLASSES
+  
+  
+  _LEFT_MATCHERS.map!{|m| 
+    case m
+    when Reg::LookAhead,Reg::LookBack: m.regs(0)
+    else m
+    end
+  } #remove lookahead and lookback decoration
+  rule_juicer=proc{|m|
+    case m 
+    when Class: m
+    when Reg::And: m.subregs.map &rule_juicer
+    when Reg::Or: m.subregs.map &rule_juicer
+    else #fukit
+    end
+  }
+  _LEFT_CLASSES=_LEFT_MATCHERS.map{|m| rule_juicer[m] }.flatten.compact
+  _RIGHT_CLASSES=  self.RULES.map{|r| r.right }.grep(Class) #classes in productions
+  _LEFT_CLASSES+_RIGHT_CLASSES
+  end
+=begin
+  HIER=Class::FlattenedHierarchy.new *STACKABLE_CLASSES
+
+  LOOKAHEAD_MATCHERS=self.RULES.map{|r| r.left.subregs.last }.map{|la| Reg::LookAhead===la and la.regs(0) }
+
+  LOOKAHEAD_CLASSES=LOOKAHEAD_MATCHERS.map(&rule_juicer)
+  LOOKAHEAD_CLASSES.each_with_index{|classes,i|
+    case classes
+    when Class: huh
+    when Array: classes.flatten.each{huh}
+    else 
+    end
+  }
+=end
+#  def fixup_multiple_assignments!; end
+end
+
+
+if __FILE__==$0
+  require 'problemfiles'
+  class NeverExecThis<RuntimeError; end
+
+  def arraydiff(a,b)
+    a==b and return [a,false]
+    (Array===a or a=[a])
+    result= a.dup
+    diff=false
+    size= a.size >= b.size ? a.size : b.size
+    size.times{|i|
+      ai=a[i]
+      bi=b[i]
+      if Array===ai and Array===bi
+        result_i,diff_i= arraydiff(ai,bi)
+        diff||=diff_i
+        result[i]=result_i
+      elsif ai!=bi
+        next if Regexp===ai and ai.to_s==bi.to_s and 
+          ai.options==bi.options 
+        diff=true
+        result[i]={ai=>bi}
+      elsif ai.nil? 
+        result[i]={'size mismatch'=>"#{a.size} for #{b.size}"} if a.size!=b.size
+        diff=true
+      end
+      if i.nonzero? and Hash===result[i] and Hash===result[i-1]
+        old=result[i-1]
+        oldkeys=old.keys
+        oldvals=old.values
+        if Reg::Subseq===oldkeys.first
+          oldkeys=oldkeys.children
+          oldval=oldvals.children
+        end
+        result[i-1..i]=[ {-[*oldkeys+result[i].keys]=>-[*oldvals+result[i].values]} ]
+      end
+    }
+    return result,diff
+  end
+
+    output=:pp
+    quiet=true
+  while /^-/===ARGV.first
+    case opt=ARGV.shift
+      when "--": break
+      when "--pp": output=:pp
+      when "--lisp": output=:lisp
+      when "--parsetree": output=:parsetree
+      when "--vsparsetree": output=:vsparsetree
+      when "--vsparsetree2": output=:vsparsetree2
+      when "--update-problemfiles": problemfiles=ProblemFiles.new
+      when "-q": quiet=true
+      when "-v": quiet=false
+      when "-e": inputs=[ARGV.join(" ")]; names=["-e"]; break
+      else fail "unknown option: #{opt}"
+
+    end
+  end
+  
+  unless inputs
+    if ARGV.empty?
+      inputs=[STDIN.read]
+      names=["-"]
+    elsif ARGV.size==1 and (Dir.entries(ARGV.first) rescue false)
+      names=Dir[ARGV.first+"/**/*.rb"]
+    else
+      names=ARGV.dup
+    end
+    inputs||=names.map{|name| File.open(name).read rescue nil}
+  end
+
+  result=0
+ 
+  safety="BEGIN{raise NeverExecThis};BEGIN{throw :never_exec_this,1};\n"
+  nullsafety="\n"
+  safe_inputs=inputs.map{|input| safety+input}
+
+  inputs.each_index{|i| 
+    begin
+
+    input=inputs[i] or next
+    name=names[i]
+
+    input=nullsafety+input
+    #print name+"... "; STDOUT.flush
+
+    begin
+      tree=nil
+      if catch(:never_exec_this){
+        tree=RedParse.new(input,name).parse; nil
+      } #raise NeverExecThis
+#    rescue RedParse::ParseError=>e
+#      require 'pp'
+#      pp e.stack[-[15,e.stack.size].min..-1]
+#      raise
+#    rescue NeverExecThis:
+        puts "RedParse attempted to execute parse data in #{name}"
+        next
+      end
+    rescue Interrupt: exit 2
+    rescue Exception=>e
+#      puts e.backtrace.join("\n")
+      e.message << " during parse of #{name}"
+#      err=e.class.new(e.message+" during parse of #{name}")
+#      err.set_backtrace e.backtrace
+      problemfiles.push name if problemfiles
+      raise e
+    end
+    tree or fail "parsetree was nil for #{name}"
+
+    case output
+    when :pp
+      require 'pp'
+      pp tree
+    when :lisp
+      puts tree.to_lisp
+    when :parsetree
+      pp tree.to_parsetree
+    when :vsparsetree,:vsparsetree2
+      begin
+      require 'rubygems'
+      rescue Exception
+      end
+      require 'parse_tree'
+      #require 'algorithm/diff'
+      begin
+        mine=tree.to_parsetree(:quirks)
+        if IO===input 
+          input.rewind
+          input=input.read
+        end
+        ryans=nil
+        catch(:never_exec_this){
+          ryans=ParseTree.new.parse_tree_for_string(safe_inputs[i],name); nil
+        } and raise NeverExecThis
+        delta,is_diff=arraydiff(mine,ryans)
+      rescue NeverExecThis:
+        puts "ParseTree attempted to execute parse data in #{name}"
+        next
+      rescue Interrupt: exit 2
+      rescue Exception=>e
+        #raise( RuntimeError.new( "#{e} during to_parsetree of #{name}" ) )
+        puts "error during to_parsetree of #{name}"
+        problemfiles.push name if problemfiles
+        raise
+      end
+      if output==:vsparsetree2
+        if !quiet or is_diff
+          puts "mine:"
+          pp mine
+          puts "ryans:" if is_diff
+          pp ryans if is_diff
+        end
+      elsif !quiet or is_diff
+        puts 'differences in '+name if is_diff
+        pp delta
+      end
+      if is_diff
+        result=1
+        problemfiles.push name if problemfiles
+      else
+        puts "no differences in "+name
+        problemfiles.delete name if problemfiles
+      end
+    end
+
+    rescue NeverExecThis: 
+      puts "mysterious attempt to execute parse data in #{name}"
+      next
+    rescue Interrupt,SystemExit: exit 2
+    rescue Exception=>e
+      puts "#{e}:#{e.class}"
+      puts e.backtrace.join("\n")
+      #problemfiles.push name if problemfiles
+      #raise
+    ensure
+      STDOUT.flush
+    end
+  }
+  exit result
+end
+
+=begin todo:
+v merge DotCallNode and CallSiteNode and CallWithBlockNode
+remove actual Tokens from parse tree...
+instead, each node has a corresponding range of tokens 
+-in an (optional) array of all tokens printed by the tokenizer.
+split ParenedNode into ParenedNode + Rescue/EnsureNode
+'incomplete' subtrees such as ElseNode, ElsifNode, RescueNode 
+-should not appear in final output
+v split keywordopnode into loop and if varieties?
+=end
+
+=begin optimization opportunities:
+top of stack slot contains mostly keywords, specific node classes, and Value
+lookahead slot contains mostly LowerOp and keywords, with a few classes and inverted keywords
+-(LowerOp is hard to optimize)
+if top of stack matcher is Value, then the next matcher down is mostly keywords, with some operators
+class membership can be optimized to test of integer within a range
+keywords could be stored as symbols instead of strings
+a few rules may need exploding (eg, ensure) to spoon feed the optimizer
+make all Nodes descendants of Array
+=end