redparse 0.8.4 → 1.0.0

data/lib/redparse/decisiontree.rb

@@ -1,6 +1,6 @@
 =begin           
     redparse - a ruby parser written in ruby
-    Copyright (C) 2008  Caleb Clausen
+    Copyright (C) 2008, 2012, 2016  Caleb Clausen
 
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU Lesser General Public License as published by

data/lib/redparse/float_accurate_to_s.rb

@@ -1,20 +1,44 @@
+=begin
+    redparse - a ruby parser written in ruby
+    Copyright (C) 2012, 2016  Caleb Clausen
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+=end
+needed=  !defined?(RUBY_ENGINE) ||
+         (RUBY_ENGINE=='ruby' && (
+            RUBY_VERSION[/^\d+\.\d+/].to_f<=1.8 || #1.8 or below or
+            RUBY_VERSION[/^1\.9\.(\d+)/,1].to_i<2  #1.9 if < 1.9.2
+         ))
+
+
 class Float
   SIZE=[1.1].pack("d").size
   BITSIZE=SIZE*8
   BASE10_DIGITS=(2**BITSIZE-1).to_s.size
   def accurate_to_s
+  #this shouldn't be needed anymore after 1.9.2
     return "#{'-' if self<0}Infinity" if infinite?
     return "NaN" if nan?
-    return "0.0e0" if zero?
+    return to_s if zero? #-0.0 and +0.0
 
     as_str=sprintf("%.#{BASE10_DIGITS+2}e",self)
 
     #decompose self into sign, mantissa, and exponent (in string form)
-    all,sign,first,digits,exp=*as_str.match(/^([+-]?)(\d)\.(\d+)e(.*)$/)
-    digits=first<<digits
-    exp=exp.to_i+1
+    all,sign,first,digits,exp=*as_str.match(/^([+-]?)(\d+)(?:\.(\d+))?(?:[eE](.*))?$/)
+    digits=first<<(digits||'0')
+    exp=(exp||'0').to_i+1
     lead=sign<<"0."
-    return digits=digits if as_str.to_f.zero? #hopeless
 
     #recompose back to a float
     result=[lead,digits,"e",exp].join
@@ -92,7 +116,7 @@ class Float
       return [lead,digits<<last,"e",exp].join
     end
   end
-end
+end if needed
 
 =begin not quite accurate, tho
 class String

data/lib/redparse/generate.rb

@@ -1,3 +1,21 @@
+=begin
+    redparse - a ruby parser written in ruby
+    Copyright (C) 2012, 2016  Caleb Clausen
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+=end
+
 class RedParse
 =begin
 3 LR-Parsing Mechanics

data/lib/redparse/node.rb

@@ -1,6 +1,6 @@
 =begin           
     redparse - a ruby parser written in ruby
-    Copyright (C) 2008  Caleb Clausen
+    Copyright (C) 2008, 2012, 2016  Caleb Clausen
 
 
     This program is free software: you can redistribute it and/or modify
@@ -41,9 +41,9 @@ class RedParse
 
     module FlattenedIvars
       EXCLUDED_IVARS=%w[@data @offset @startline @endline]
-      EXCLUDED_IVARS.push(*EXCLUDED_IVARS.map{|iv| iv.to_sym })
+      #EXCLUDED_IVARS.push(*EXCLUDED_IVARS.map{|iv| iv.to_sym })
       def flattened_ivars
-        ivars=instance_variables
+        ivars=instance_variables.map{|v| v.to_s }
         ivars-=EXCLUDED_IVARS
         ivars.sort!
         result=ivars+ivars.map{|iv| 
@@ -165,6 +165,8 @@ class RedParse
       extend Stackable::Meta
 
       def image; "#{inspect}" end
+
+      def short_inspect; inspect+"\n" end
  
       def to_parsetree(*options) #this shouldn't be needed anymore
         o={}
@@ -184,13 +186,13 @@ class RedParse
       def rescue_parsetree(o); parsetree(o) end
       def begin_parsetree(o); parsetree(o) end
 
-      attr :line
-      alias endline line
+      #attr :line
+      #alias endline line
 
-      attr_writer :startline
-      def startline
-        @startline||=endline
-      end
+      #attr_writer :startline
+      #def startline
+      #  @startline||=endline
+      #end
     end
 
     class KeywordToken
@@ -314,7 +316,7 @@ class RedParse
         [varname2assigntype, ident.to_sym]
       end
 
-      def lvalue
+      def old_unused_lvalue #i think this is the correct way, but its overridded below
         return @lvalue if defined? @lvalue
         @lvalue=true
       end
@@ -346,15 +348,16 @@ class RedParse
     end   
 
     class StringToken
-      attr :char
+      attr :char unless allocate.respond_to? :char
     end
 
     class HerePlaceholderToken
       attr_accessor :node
-      attr :string
+      attr :string unless allocate.respond_to? :string
     end
 
     module ListInNode
+      def self.[](*args) args.extend ListInNode end
       def []=(*args)
         val=args.pop
         #inline symbols as callnodes
@@ -366,6 +369,9 @@ class RedParse
         end
         super( *args<<val )
       end
+      def ==(other)
+        ListInNode===other and super
+      end
     end
 
     class Node<Array
@@ -374,6 +380,12 @@ class RedParse
       include FlattenedIvars
 
       def initialize(*data)
+        if Hash===data.last
+          options=data.pop
+          options.each_pair{|name,val|
+            instance_variable_set name,val
+          }
+        end
         replace data
       end
 
@@ -414,6 +426,10 @@ class RedParse
 
       def image; "(#{inspect})" end
 
+      def short_inspect(cutoff=2)
+        inspect.gsub(/\n {#{cutoff*2},}.*$/,'')
+      end
+
       def error? x; false end
 
       @@data_warned=nil
@@ -456,8 +472,27 @@ class RedParse
         return result
       end
 
-      def inspect label=nil,indent=0
-        ivarnames=instance_variables-FlattenedIvars::EXCLUDED_IVARS
+      def to_s #mostly for form inside #{ }
+        unparse
+      end
+
+      def classic_inspect
+        self.class.name.dup+'['+
+          map{|elem| 
+            if elem.respond_to? :classic_inspect
+              elem.classic_inspect
+            else
+              elem.inspect
+            end
+          }.join(', ')+
+        ']'
+        #this skips ivars, tho...
+      end
+
+      def inspect label=nil,indent=0,verbose=false
+        ivarnames=instance_variables.map{|v| v.to_s }
+        ivarnames-=FlattenedIvars::EXCLUDED_IVARS
+        ivarnames-=noinspect_instance_variables if defined? noinspect_instance_variables
         ivarnodes=[]
         ivars=ivarnames.map{|ivarname|
           ivar=instance_variable_get(ivarname)
@@ -465,17 +500,21 @@ class RedParse
             ivarnodes.push [ivarname,ivar]
             nil
           else
-            ivarname[1..-1]+"="+ivar.inspect if ivar
+            ivarname[1..-1]+"="+ivar.inspect if ivar and verbose
           end
         }.compact.join(' ')
 
 
-        pos=@startline.to_s
-        pos<<"..#@endline" if @endline!=@startline
-        pos<<"@#@offset"
+        if verbose
+          pos=@startline.to_s
+          pos<<"..#@endline" if @endline!=@startline
+          pos<<"@#@offset"
+        end
         classname=self.class.name
         classname.sub!(/^(?:RedParse::)?(.*?)(?:Node)?$/){$1}
-        result= [' '*indent,"+",(label+': ' if label),classname," pos=",pos," ",ivars,"\n"]
+        result= [' '*indent,"+",(label.to_s+': ' if label),classname,]
+        result+=[" pos=",pos,] if pos
+        result+=[" ",ivars,"\n"]
         indent+=2
 
         namelist=self.class.namelist
@@ -483,10 +522,10 @@ class RedParse
           namelist.each{|name|
             val=send name rescue "{{ERROR INSPECTING ATTR #{name}}}"
             case val
-              when Node; result<< val.inspect(name,indent)
+              when Node; result<< val.inspect(name,indent,verbose)
               when ListInNode 
-                result.push ' '*indent,"#{name}:\n",*val.map{|v| 
-                  v.inspect(nil,indent+2) rescue ' '*(indent+2)+"-#{v.inspect}\n"
+                result.push ' '*indent,"+#{name}:\n",*val.map{|v| 
+                  v.inspect(nil,indent+2,verbose) rescue ' '*(indent+2)+"-#{v.inspect}\n"
                 }
               when nil;
               else ivars<< " #{name}=#{val.inspect}"
@@ -495,7 +534,7 @@ class RedParse
         else
           each{|val|
             case val
-            when Node; result<< val.inspect(nil,indent) 
+            when Node; result<< val.inspect(nil,indent,verbose) 
             else result<< ' '*indent+"-#{val.inspect}\n"
             end
           }
@@ -594,11 +633,14 @@ class RedParse
         #@namelist
         result=superclass.namelist||[] rescue []
         result.concat @namelist if defined? @namelist
+        result.uniq!
         return result
       end
 
       def lhs_unparse o; unparse(o) end
 
+      def to_ruby o={}; unparse(o) end
+
       def to_parsetree(*options)
         o={}
         [:newlines,:quirks,:ruby187].each{|opt| 
@@ -753,6 +795,26 @@ class RedParse
 =end
       end
 
+      def rgrep pattern
+        result=grep(pattern)
+        each{|subnode| result.concat subnode.rgrep(pattern) if subnode.respond_to? :rgrep}
+        return result
+      end
+
+      def rfind ifnone=nil, &block
+        result=find(proc{
+          find{|subnode| subnode.rfind(&block) if subnode.respond_to? :rfind}
+        },&block)
+        return result if result
+        return ifnone[] if ifnone
+      end
+
+      def rfind_all &block
+        result=find_all(&block)
+        each{|subnode| result.concat subnode.find_all(&block) if subnode.respond_to? :rfind_all}
+        return result
+      end
+
       def replace_ivars_and_self o,session,&replace_self_action
           o.instance_variables.each{|ovname|
             ov=o.instance_variable_get(ovname)
@@ -1086,21 +1148,43 @@ end
           when '&'; amp=val.parsetree(o)
           end
         }
-        return output,star,amp
+        return output,star
       end
 
       def unparse_nl(token,o,alt=';',nl="\n")
-
+        fail unless alt
         #should really only emit newlines 
         #to bring line count up to startline, not endline.
 
-        linenum= Integer===token ? token : token.startline rescue (return alt)
+        linenum=
+        case token 
+        when Integer; token 
+        when nil; return alt
+        else token.startline rescue (return alt)
+        end
         shy=(linenum||0)-o[:linenum]
+        #warn if shy<0 ???
         return alt if shy<=0
         o[:linenum]=linenum
         return nl*shy
       end
 
+      def unparse_maybe_parens(o,changed=[])
+        result=unparse(o)
+        if !o[:exact] and 
+          case self
+          when CallNode; params and params.size>0 and !real_parens
+          when KWCallNode,AliasNode,UndefNode,SequenceNode; true
+          when AssignNode; MultiAssign===left
+          end
+        then
+          result="(#{result})"
+          changed[0]=true
+        end
+        return result
+      end
+      protected :unparse_maybe_parens
+
       def default_unparse_options
         {:linenum=>1}
       end
@@ -1271,13 +1355,21 @@ end
 
     class RawOpNode<ValueNode
       param_names(:left,:op,:right)
-      def initialize(left,op,right=nil)
-        op,right=nil,op if right.nil?
+      def initialize(*args)
+        options=args.pop if Hash===args.last
+        if args.size==2
+          left,right=*args
+          op=nil
+        else
+          left,op,right=*args
+        end
         if op.respond_to? :ident
           @offset=op.offset
           op=op.ident
         end
-        super(left,op,right)
+        args=left,op,right
+        args.push options if options
+        super( *args )
       end
 
 #      def initialize_copy other
@@ -1441,12 +1533,15 @@ end
     class LiteralNode<ValueNode; end
     class StringNode<ValueNode; end
     class StringCatNode < ValueNode; end
-    class NopNode<ValueNode; end
     class VarLikeNode<ValueNode; end  #nil,false,true,__FILE__,__LINE__,self
+    class SequenceNode<ListOpNode; end
+    class NopNode<SequenceNode; end
+    class UnOpNode<ValueNode; end
 
     class SequenceNode<ListOpNode
       def initialize(*args)
-        super
+        return if args.empty?
+        super 
         @offset=self.first.offset
       end
       def +(other)
@@ -1480,11 +1575,18 @@ end
 
       LITFIX=LiteralNode&-{:val=>Fixnum}
       LITRANGE=RangeNode&-{:left=>LITFIX,:right=>LITFIX}
-      LITSTR=StringNode&-{:size=>1,:char=>/^[^`\[{]$/}
+      LITSTR=Recursive(ls={},StringNode&-{:char=>/^[^`\[{]$/}&+[(String|ls).+])
       #LITCAT=proc{|item| item.grep(~LITSTR).empty?}
       #class<<LITCAT; alias === call; end
       LITCAT=StringCatNode& item_that.grep(~LITSTR).empty? #+[LITSTR.+]
-      LITNODE=LiteralNode|NopNode|LITSTR|LITCAT|LITRANGE|(VarLikeNode&-{:name=>/^__/})
+      LITUPLUS=Recursive(lup={},
+                 UnOpNode&-{:op=>"+@", :val=>
+                   (LiteralNode&-{:val=>Symbol|Numeric})|
+                   (StringNode&-{:size=>1,:char=>"/"})|
+                   lup
+                 }
+               )
+      LITNODE=LiteralNode|NopNode|LITSTR|LITCAT|(VarLikeNode&-{:name=>/^__/})|LITUPLUS
             #VarNode|  #why not this too?
       def parsetree(o)
         data=compact
@@ -1509,6 +1611,7 @@ end
             (items[i,1] = *header[1..-1]) if header and header.first==:block
           }     
         end
+        result=
         if items.size>1
           items.unshift :block
         elsif items.size==1
@@ -1516,6 +1619,13 @@ end
         else
           items
         end
+
+        limit=o[:quirks] ? 1 : result.size-1
+        (0..limit).each{|i| 
+          result[i,1]=result[i][1..-1] if Array===result[i] and result[i][0]==:block
+        }
+
+        return result
       end
 
       def unparse o=default_unparse_options
@@ -1582,12 +1692,18 @@ end
 
     class RangeNode
       param_names(:first,:op,:last)
-      def initialize(left,op,right=nil)
-        op,right="..",op unless right
-        op=op.ident if op.respond_to? :ident
+      def initialize(*args)
+        options=Hash===args.last ? args.pop : {}
+        if args.size==2
+          left,right=*args
+          op=".."
+        else
+          left,op,right=*args
+          op=op.ident if op.respond_to? :ident
+        end
         @exclude_end=!!op[2]
         @as_flow_control=false
-        super(left,op,right)
+        super(left,op,right,options)
       end
       def begin; first end
       def end; last end
@@ -1651,15 +1767,26 @@ end
 
     class UnOpNode<ValueNode
       param_names(:op,:val)
-      def initialize(op,val)
+      def self.create(op,val)
+        return UnaryStarNode.new(op,val) if /\*$/===op.ident
+        return UnaryAmpNode.new(op,val) if /&$/===op.ident
+        return new(op,val)
+      end
+
+      def initialize(op,val,options=nil)
         @offset||=op.offset rescue val.offset
         op=op.ident if op.respond_to? :ident
         /([&*])$/===op and op=$1+"@"
         /^(?:!|not)$/===op and 
           val.respond_to? :special_conditions! and 
             val.special_conditions!
-        super(op,val)
+        args=[op,val]
+        args<<options if options
+        super(*args)
       end
+
+      def arg; val end
+      def arg= x; self.val=x end
    
       alias ident op
 
@@ -1700,16 +1827,29 @@ end
         op=op()
         op=op.chomp "@"
         result=op
-        result+=" " if /#{LETTER}/o===op or /^[+-]/===op && LiteralNode===val
+        result+=" " if /(?:#{LETTER}|[?!])$/o===op or /^[+-]/===op && LiteralNode===val
         result+=val.unparse(o)
       end
     end
+    UnaryOpNode=UnOpNode
+
+    class UnaryAmpNode<UnOpNode
+
+    end
 
     class UnaryStarNode<UnOpNode
-      def initialize(op,val=nil)
-        op,val="*@",op unless val
+      def initialize(*args)
+        options=args.pop if Hash===args.last
+        if args.size==1
+          op="*@"
+          val=args.first
+        else
+          op,val=*args
+        end
         op.ident="*@" if op.respond_to? :ident
-        super(op,val)
+        args=[op,val]
+        args.push options if options
+        super(*args)
       end
 
       class<<self
@@ -1744,6 +1884,10 @@ end
         @startline=@endline=star.startline if star.respond_to? :startline
         super('*@',var||VarNode[''])
       end
+      def self.create(star,var=nil)
+        DanglingStarNode.new(star,var)
+      end
+
       attr :offset
       def lvars_defined_in; [] end
       def parsetree(o); [:splat] end
@@ -1812,11 +1956,17 @@ end
       attr_writer :lvalue
       identity_param :lvalue, nil, true
 
-      def inspect label=nil,indent=0
+      def inspect label=nil,indent=0,verbose=false
         result=' '*indent
         result+="#{label}: " if label 
         result+='Constant '
-        result+=map{|name| name.inspect}.join(', ')+"\n"
+        unless String===first or nil==first
+          head=first
+          rest=self[1..-1]
+        end
+        result+=(rest||self).map{|name| name.inspect}.join(', ')+"\n"
+        result+=head.inspect("head",indent+2,verbose) if head
+        return result
       end
     end
     LookupNode=ConstantNode
@@ -1881,7 +2031,8 @@ end
       param_names :body 
       alias val body
       alias val= body=
-      def initialize(lparen,body,rparen)
+      def initialize(lparen=nil,body=nil,rparen=nil)
+        return unless body
         @offset=lparen.offset
         self[0]=body
       end
@@ -1903,11 +2054,11 @@ end
       def op?; false end
  
       def parsetree(o)
-          body.parsetree(o)
+        body ? body.parsetree(o) : [:nil]
       end
 
       def rescue_parsetree o
-        body.rescue_parsetree o
+        body ? body.rescue_parsetree(o) : [:nil]
 #        result.first==:begin and result=result.last unless o[:ruby187]
 #        result
       end
@@ -1923,13 +2074,14 @@ end
       identity_param :lvalue, nil, true
 
       def unparse(o=default_unparse_options)
-          "("+(body&&body.unparse(o))+")"
+          "(#{body&&body.unparse(o)})"
       end
     end
 
     module HasRescue
       def parsetree_and_rescues(o)
           body=body()
+          body=body.clone if body
           target=result=[]   #was: [:begin, ]
 
           #body,rescues,else_,ensure_=*self
@@ -1971,12 +2123,12 @@ end
       def unparse_and_rescues(o)
           result=" "
           result+= body.unparse(o) if body
-          result+=unparse_nl(rescues.first,o)
-          rescues.each{|resc| result+=resc.unparse(o) }
+          result+=unparse_nl(rescues.first,o) if rescues
+          rescues.each{|resc| result+=resc.unparse(o) } if rescues
           result+=unparse_nl(else_,o)+"else "+else_.unparse(o) if else_
-          result+=";else" if @empty_else
+          result+=";else" if defined? @empty_else and @empty_else
           result+=unparse_nl(ensure_,o)+"ensure "+ensure_.unparse(o) if ensure_
-          result+=";ensure" if @empty_ensure 
+          result+=";ensure" if defined? @empty_ensure and @empty_ensure
           return result
       end
 
@@ -2026,6 +2178,14 @@ end
  
       def parsetree(o)
         result=parsetree_and_rescues(o)
+
+=begin should be something like this:
+        limit=o[:quirks] ? 1 : result.size
+        (0...limit).each{|i| 
+          result[i,1]=result[i][1..-1] if Array===result[i] and result[i][0]==:block
+        }
+=end
+
         if o[:ruby187] and result.first==:begin
           result=result[1]
         else
@@ -2041,7 +2201,7 @@ end
       end
 
       def begin_parsetree(o)
-        body,rescues,else_,ensure_=*self
+        #body,rescues,else_,ensure_=*self
         needbegin=(rescues&&!rescues.empty?) || ensure_ || @empty_ensure
         result=parsetree(o)
         needbegin and result=[:begin, result] unless result.first==:begin
@@ -2073,11 +2233,18 @@ end
             replace [expr,[RescueNode[[],nil,backup]].extend(ListInNode)]
       end
 
+      def self.namelist
+        %w[left right]
+      end
+
       def else; nil end
       def ensure; nil end
 
       def left; body end
       def right; rescues[0].action end
+      def op; "rescue" end
+
+      alias rescue_with right
 
       alias ensure_ ensure
       alias else_ else
@@ -2104,7 +2271,7 @@ end
 
           rescues=rescues().map{|resc| resc.parsetree(o)}
           target.push newtarget=[:rescue, ]
-          else_=nil
+          #else_=nil
           needbegin=  (BeginNode===body and body.after_equals)
           huh if needbegin and RescueOpNode===body #need test case for this
           huh if needbegin and ParenedNode===body #need test case for this
@@ -2173,10 +2340,10 @@ end
             else
               rhs=RescueOpNode.new(rescuee.val,op2,rescuer)
             end
+            super(lhs,op,rhs)
           else
-            lhs,op,bogus1,rhs,bogus2=*args
+            super(args[0],args[1],args[3])
           end
-          super(lhs,op,rhs)
         else super
         end
       end
@@ -2193,7 +2360,10 @@ end
               rhs=RescueOpNode.new(rescuee.val,op2,rescuer)
             end
           else
-            lhs,op,bogus1,rhs,bogus2=*args
+            lhs=args.shift
+            op=args.shift
+            args.shift #bogus1
+            rhs=args.shift
           end
         else
           lhs,op,rhs=*args
@@ -2424,7 +2594,7 @@ end
       def unparse(o=default_unparse_options)
         result=lhs.lhs_unparse(o)
         result="(#{result})" if defined? @lhs_parens
-        result+op+
+        result+' '+op+' '+
           (rhs.class==Array ? 
             rhs.map{|rv| rv.unparse o}.join(',') :
             rhs.unparse(o) 
@@ -2594,6 +2764,7 @@ end
           data=CommaOpNode===item ? Array.new(item) : [item]
           @had_parens=true
         end
+        data= Array.new(data) if CommaOpNode===data
 
         super(data) unless data.empty?
       end
@@ -2639,7 +2810,7 @@ end
           [:op_asgn2, rcvr,prop,  op.to_sym, args]
         end
       end
-    end
+    end if false
  
     class LogicalNode
       include KeywordOpNode
@@ -2757,7 +2928,10 @@ end
 
       def while; condition end
       def do; consequent end
+      alias body do
       def op; "while" end
+      def reversed; false end
+      attr :test_first
 
       def parsetree(o)
         cond=condition.rescue_parsetree(o)
@@ -2795,7 +2969,9 @@ end
 
       def while; negate condition end
       def do; consequent end
+      alias body do
       def op; "until" end
+      def reversed; true end
 
       def parsetree(o)
         cond=condition.rescue_parsetree(o)
@@ -2881,13 +3057,48 @@ end
     end
 
     class CallSiteNode<ValueNode
-      param_names :receiver, :name, :params, :blockparams, :block
+      param_names :receiver, :name, :params, :blockparams, :locals, :block
+
       alias blockargs blockparams
       alias block_args blockargs
       alias block_params blockparams
 
-      def initialize(method,open_paren,param_list,close_paren,block)
-        @not_real_parens=!open_paren || open_paren.not_real?
+      alias block_locals locals
+      alias blocklocals locals
+      alias locals_in_block locals
+
+      def initialize(*args)
+        if KeywordToken===args.first and args.first.ident=='('
+          method,open_paren,param_list,close_paren,block="()",*args
+          parened=open_paren
+        else 
+          method,open_paren,param_list,close_paren,block=*args
+          @not_real_parens=!open_paren || open_paren.not_real?
+        end
+
+        if KeywordToken===method and method.ident=='->'
+          if SequenceNode===param_list
+            blockparams,blocklocals,* = param_list
+            if CommaOpNode===blocklocals
+              blocklocals=Array(blocklocals)
+            else
+              blocklocals=[blocklocals]
+            end
+            blocklocals.map!{|bl| bl.ident}
+            blocklocals.extend ListInNode
+          else
+            blockparams= param_list
+          end
+
+          if CommaOpNode===blockparams
+            blockparams=Array(blockparams)
+          elsif blockparams
+            blockparams=[blockparams]
+          end
+
+          blockparams=BlockParams.new blockparams if blockparams
+          param_list=nil
+        end
 
         case param_list
         when CommaOpNode
@@ -2914,28 +3125,39 @@ end
 
         if block
           @do_end=block.do_end
-          blockparams=block.params
+          blockparams||=block.params
+          blocklocals||=block.locals
           block=block.body #||[]
         end
-        @offset=method.offset
         if Token===method 
+          @offset=method.offset
           method=method.ident
           fail unless String===method
+        else
+          @offset=parened&&parened.offset
         end
 
-        super(nil,method,param_list,blockparams,block)
+        super(nil,method,param_list,blockparams,blocklocals,block)
         #receiver, if any, is tacked on later
       end
 
-      def real_parens; !@not_real_parens end
+      def real_parens; @not_real_parens||=nil; !@not_real_parens end
       def real_parens= x; @not_real_parens=!x end
+      alias has_parens? real_parens
 
       def unparse o=default_unparse_options
         fail if block==false
+        parensadded=[false]
+        params=params()
+        params&&=params.map{|param|  
+          unparse_nl(param,o,'',"\\\n")+param.unparse_maybe_parens(o,parensadded)  
+        }.join(', ')
+        real_parens=real_parens() || parensadded[0]
         result=[
-         receiver&&receiver.unparse(o)+'.',name,      
+         receiver&&receiver.unparse(o)+'.',
+         name=='()' ? '' : name,
          real_parens ? '(' : (' ' if params),
-         params&&params.map{|param|  unparse_nl(param,o,'',"\\\n")+param.unparse(o)  }.join(', '),
+         params,
          real_parens ? ')' : nil,
         
          block&&[
@@ -2951,7 +3173,7 @@ end
       end
 
       def image
-        result="(#{receiver.image if receiver}.#{name})"
+        "(#{receiver.image if receiver}.#{name})"
       end
      
       def with_commas
@@ -3155,12 +3377,12 @@ end
     end
 
     class CallNode<CallSiteNode #normal method calls
-      def initialize(method,open_paren,param_list,close_paren,block)
+      def initialize(*args)
         super
       end
     end
     class KWCallNode<CallSiteNode #keywords that look (more or less) like methods
-      def initialize(method,open_paren,param_list,close_paren,block)
+      def initialize(method,*args)
         KeywordToken===method or fail
         super
       end
@@ -3188,14 +3410,26 @@ end
     end
 
     class BlockNode<ValueNode #not to appear in final parse tree
-      param_names :params,:body
+      param_names :params,:locals,:body
       def initialize(open_brace,formals,stmts,close_brace)
         stmts||=SequenceNode[{:@offset => open_brace.offset, :@startline=>open_brace.startline}]
         stmts=SequenceNode[stmts,{:@offset => open_brace.offset, :@startline=>open_brace.startline}] unless SequenceNode===stmts
         
+        if formals and formals.size==1 and SequenceNode===formals[0]
+          formals,blocklocals,* = formals[0]
+          formals&&=BlockParams[formals]
+          if CommaOpNode===blocklocals
+            blocklocals=Array(blocklocals)
+          else
+            blocklocals=[blocklocals]
+          end
+          blocklocals.map!{|bl| bl.ident}
+          blocklocals.extend ListInNode
+        end
+
         formals&&=BlockParams.new(Array.new(formals))
         @do_end=true unless open_brace.not_real?
-        super(formals,stmts)
+        super(formals,blocklocals,stmts)
       end
 
       attr_reader :do_end
@@ -3225,12 +3459,31 @@ end
       end
     end
 
-    class NopNode<ValueNode
+    class ProcLiteralNode<ValueNode
+      def initialize(params, other_locals, body)
+        @params, @other_locals, @body= params, other_locals, body
+      end
+
+      def self.create(arrow, lparen, params, rparen, do_word, body, endword)
+        params,other_locals=*params if SequenceNode===params
+        new(params,other_locals,body)
+      end
+
+      def unparse o=default_parse_options
+        huh
+      end
+    end
+
+    class NopNode<SequenceNode        
       def initialize(*args)
         @startline=@endline=1
         super()
       end
+      def unparse(o=default_unparse_options)
+        ""
+      end
 
+=begin
       def unparse o=default_unparse_options
         ''      
       end
@@ -3244,6 +3497,7 @@ end
       def to_parsetree(*options)
         []
       end
+=end
     end
 
 =begin
@@ -3318,6 +3572,7 @@ end
         @open||='"' 
         @close||='"' 
         @bs_handler||=:dquote_esc_seq
+        @modifiers||=nil
         if /[\[{]/===@char
           @parses_like||=split_into_words(str)
         end
@@ -3328,7 +3583,6 @@ end
         result=str.dup
         seq=result.to_sequence
         rl.instance_eval{@file=seq}
-        repls=[]
         i=0
         #ugly ugly ugly... all so I can call @bs_handler
         while i<result.size and bs_at=result.index(/\\./m,i)
@@ -3371,6 +3625,7 @@ end
         o[:linenum]+=@open.count("\n")
         result=[@open,unparse_interior(o),@close,@modifiers].join
         o[:linenum]+=@close.count("\n")
+        result<<" " if /\r\z/===result
         return result
       end
 
@@ -3378,8 +3633,10 @@ end
         unless escapable=ESCAPABLES[open]
           maybe_crunch='\\#' if %r{\A["`/\{]\Z} === @char and open[1] != ?q and open != "'" #"
           #crunch (#) might need to be escaped too, depending on what @char is
+          quotes=open[-1,1]
+          quotes+=close unless quotes==close
           escapable=ESCAPABLES[open]=
-            /[#{Regexp.quote open[-1,1]+close}#{maybe_crunch}]/
+            /[#{Regexp.quote(quotes)}#{maybe_crunch}]/
         end
         escapable             
       end
@@ -3387,7 +3644,8 @@ end
       def unparse_interior o,open=@open,close=@close,escape=nil
         escapable=escapable(open,close)
         result=map{|substr|
-          if String===substr
+          case substr
+          when String
 
             #hack: this is needed for here documents only, because their
             #delimiter is changing.
@@ -3396,6 +3654,8 @@ end
             o[:linenum]+=substr.count("\n") if o[:linenum]
 
             substr
+          when NopNode
+            '#{}'
           else
             ['#{',substr.unparse(o),'}']
           end
@@ -3427,7 +3687,7 @@ end
 
 #        data=tokens.inject([]){|sum,token|
 #          data=elems=token.string.elems
-          data=elems=
+          data= #elems=
             case token
             when StringToken; token.elems
             when HerePlaceholderToken; token.string.elems
@@ -3439,7 +3699,6 @@ end
 #        endline=@endline
         1.step(data.length-1,2){|i|
           tokens=data[i].ident.dup
-          line=data[i].linenum
 
           #replace trailing } with EoiToken
           (tokens.size-1).downto(0){|j| 
@@ -3496,17 +3755,17 @@ end
       DQ_ESC=/(?>\\(?>[CM]-|c)?)/
       DQ_EVEN=%r[
                   (?:
-                   \A |
+                   \G |
                    [^\\c-] |
-                   (?>\A|[^\\])c |
-                   (?> [^CM] | (?>\A|[^\\])[CM] )-
+                   (?>\G|[^\\])c |
+                   (?> [^CM] | (?>\G|[^\\])[CM] )-
                   )              #not esc
                   #{DQ_ESC}{2}*  #an even number of esc
              ]omx
       DQ_ODD=/#{DQ_EVEN}#{DQ_ESC}/omx
       SQ_ESC=/\\/
       SQ_EVEN=%r[
-                  (?:  \A | [^\\]  )  #not esc
+                  (?:  \G | [^\\]  )  #not esc
                   #{SQ_ESC}{2}*       #an even number of esc
              ]omx
       SQ_ODD=/#{SQ_EVEN}#{SQ_ESC}/omx
@@ -3815,9 +4074,9 @@ end
 
     class LiteralNode<ValueNode
       param_names :val
-      attr_accessor :offset
-      def initialize(old_val)
-        @offset=old_val.offset
+      attr_accessor :offset, :string
+      def self.create(old_val)
+        offset=old_val.offset
         val=old_val.ident
         case old_val
         when SymbolToken
@@ -3842,12 +4101,15 @@ end
           end
         when NumberToken 
           case val
-          when /\A-?0([^.]|\Z)/; val=val.oct
-          when /[.e]/i; val=val.to_f
+          when /\A[+-]?0([^.]|\Z)/; val=val.oct
+          when /[.e]/i; orig=val; val=val.to_f
           else val=val.to_i
           end
         end
-        super(val)
+        result=LiteralNode.new(val)
+        result.string=orig if orig
+        result.offset=offset
+        result
       end
 
       def self.inline_symbols data
@@ -3881,16 +4143,19 @@ end
           ":"+
             val.unparse(o)
         when Float
-          s= val.accurate_to_s
-          #why must it be *2? I wouldn't think any fudge factor would be necessary
-          case s
-          when /-inf/i; s="-"+Inf
-          when /inf/i;  s=    Inf
-          when /nan/i;  s=    Nan
+          if defined? @string; 
+            @string
           else
-            fail unless [s.to_f].pack("d")==[val].pack("d")
+            s= val.accurate_to_s
+            case s
+            when /-inf/i; s="-"+Inf
+            when /inf/i;  s=    Inf
+            when /nan/i;  s=    Nan
+            else
+              fail "float coercion not stable" unless [s.to_f].pack("d")==[val].pack("d")
+            end
+            s
           end
-          s
         else val.inspect
         end
       end
@@ -3926,17 +4191,18 @@ end
 
     class VarLikeNode<ValueNode #nil,false,true,__FILE__,__LINE__,self
       param_names :name
-      def initialize(name,*more)
-        @offset=name.offset
-        if name.ident=='(' 
-          #simulate nil
-          replace ['nil']
-          @value=nil
+      def self.new(name,*more)
+        if name.ident=='(' #shouldn't happen now, AFAICT
+          ParenedNode.new
         else
-          replace [name.ident]
-          @value=name.respond_to?(:value) && name.value
+          super
         end
       end
+      def initialize(name,*more)
+        @offset=name.offset
+        replace [name.ident]
+        @value=name.respond_to?(:value) && name.value
+      end
 
       alias ident name
       
@@ -3986,12 +4252,12 @@ end
       def image; "([])" end
 
       def unparse o=default_unparse_options
-        "["+map{|item| unparse_nl(item,o,'')+item.unparse(o)}.join(', ')+"]"
+        "["+map{|item| unparse_nl(item,o,'')+item.unparse_maybe_parens(o)}.join(', ')+"]"
       end
 
       def parsetree(o)
         size.zero? and return [:zarray]
-        normals,star,amp=param_list_parse(self,o)
+        normals,star=param_list_parse(self,o)
         result=normals.unshift :array
         if star
           if size==1
@@ -4033,9 +4299,7 @@ end
       param_names :condition,:consequent,:elsifs,:otherwise
       def initialize(iftok,condition,thentok,consequent,elsifs,else_,endtok)
         @offset=iftok.offset
-        if else_ 
-          else_=else_.val or @empty_else=true
-        end
+        @empty_else= else_ && !(else_=else_.val)
         condition.special_conditions! if condition.respond_to? :special_conditions!
         elsifs.extend ListInNode if elsifs
         super(condition,consequent,elsifs,else_)
@@ -4060,7 +4324,7 @@ end
         result+=unparse_nl(consequent,o)+"#{consequent.unparse(o)}" if consequent
         result+=unparse_nl(elsifs.first,o)+elsifs.map{|n| n.unparse(o)}.join if elsifs
         result+=unparse_nl(else_,o)+"else "+else_.unparse(o) if else_
-        result+=";else " if defined? @empty_else
+        result+=";else " if defined? @empty_else and @empty_else
         result+=";end"
         return result
       end
@@ -4195,6 +4459,14 @@ end
         @reverse ? condition : negate(condition, @loopword_offset)
       end
 
+      def reversed
+        @reverse
+      end
+
+      def test_first
+        false
+      end
+
       def to_lisp
         body=body()
         "(#{@reverse ? :until : :while} #{condition.to_lisp}\n#{body.to_lisp})"
@@ -4334,7 +4606,7 @@ end
       def image; "(for)" end
 
       def unparse o=default_unparse_options
-        result=unparse_nl(self,o)+"         for #{iterator.lhs_unparse(o)[1...-1]} in #{enumerable.unparse o}"
+        result=unparse_nl(self,o,"")+"         for #{iterator.lhs_unparse(o)[1...-1]} in #{enumerable.unparse o}"
         result+=unparse_nl(body,o)+"           #{body.unparse(o)}" if body
         result+=";end"
       end
@@ -4399,7 +4671,7 @@ end
           super(*data)
         else fail
         end
-        @no_braces=!open
+        @no_braces=true unless open
       end
 
       attr :no_arrows
@@ -4410,15 +4682,15 @@ end
 
       def unparse o=default_unparse_options
         result=''
-        result << "{" unless @no_braces
+        result << "{" unless defined? @no_braces and @no_braces
         arrow= defined?(@no_arrows) ? " , " : " => "
         (0...size).step(2){|i| 
           result<< unparse_nl(self[i],o,'')+
-            self[i].unparse(o)+arrow+
-            self[i+1].unparse(o)+', '
+            self[i].unparse_maybe_parens(o)+arrow+
+            self[i+1].unparse_maybe_parens(o)+', '
         }
         result.chomp! ', '
-        result << "}" unless @no_braces
+        result << "}" unless defined? @no_braces and @no_braces
         return result
       end
 
@@ -4454,6 +4726,9 @@ end
         return true if defined?(@no_arrows) and rubyversion>=1.9
         return super
       end
+ 
+      def blame; self end
+      def msg; "comma not allowed within pairs in hashes anymore" end
     end
 
     class TernaryNode<ValueNode
@@ -4499,6 +4774,7 @@ end
 #          header=header.data[1]
 #        end
         if CallSiteNode===header
+          @parens=header.has_parens?
           receiver=header.receiver
           args=header.args
           header=header.name
@@ -4526,6 +4802,10 @@ end
         %w[receiver name args body rescues elses ensures]
       end
 
+      def has_parens?; 
+        @parens if defined? @parens
+      end
+
 #      def receiver= x
 #        self[0]=x      
 #      end
@@ -4548,8 +4828,9 @@ end
 
       def unparse o=default_unparse_options
         result=[
-         "def ",receiver&&receiver.unparse(o)+'.',name,
-           args && '('+args.map{|arg| arg.unparse o}.join(',')+')', unparse_nl(body||self,o)
+         "def ",receiver&&receiver.unparse(o)+'.',name, has_parens? ? '(' : ' ', 
+           args&&args.map{|arg| arg.unparse o}.join(','), 
+         (')' if has_parens?), unparse_nl(body||self,o)
         ]
         result<<unparse_and_rescues(o)
 =begin
@@ -4630,6 +4911,7 @@ end
 
         target.push ensuretarget=target=[:ensure, ] if ensures or @empty_ensure
         #simple dup won't work... won't copy extend'd modules
+        #should use clone here
         body=Marshal.load(Marshal.dump(body())) if body()
         elses=elses()
         if rescues.empty?
@@ -4647,6 +4929,11 @@ end
             body.rescues.empty? and !body.ensure and !body.empty_ensure and body.body and body.body.size>1
               wantblock=true
           end
+          if o[:quirks]
+            first=body
+            first=first.first if SequenceNode===first
+            wantblock=true if UndefNode===first and first.size>1
+          end
           body=body.parsetree(o)
           if body.first==:block and rescues.empty? and not ensures||@empty_ensure
             if wantblock
@@ -4783,6 +5070,7 @@ end
 
       alias else_ else
       alias ensure_ ensure
+      alias receiver name
 
       def image; "(module #{name})" end
 
@@ -4832,6 +5120,7 @@ end
 
       alias else_ else
       alias ensure_ ensure
+      alias receiver name
 
       def image; "(class #{name})" end
 
@@ -4943,6 +5232,7 @@ end
         exlist.extend ListInNode
         super(exlist,rescuehdr.varname,action)
       end
+      alias name varname
 
       def unparse o=default_unparse_options
         xx=exceptions.map{|exc| exc.unparse o}.join(',')
@@ -5013,7 +5303,7 @@ end
       def image; "(#{receiver.image}.[])" end
 
       def unparse o=default_unparse_options
-        [ receiver.unparse(o).sub(/\s+\Z/,''),
+        [ receiver.unparse(o).sub(/ \z/,''),
           '[',
           params&&params.map{|param| param.unparse o}.join(','),
           ']'
@@ -5029,7 +5319,7 @@ end
 
       def parsetree_no_fcall o
         params=params()
-        output,star,amp=param_list_parse(params,o)
+        output,star=param_list_parse(params,o)
 #        receiver=receiver.parsetree(o)
         result=[:call, receiver.rescue_parsetree(o), :[], output]
         if params
@@ -5096,9 +5386,9 @@ end
     module ErrorNode
       def error?(x=nil) @error end
       alias msg error?
+      def blame; fail end
     end
 
-
     class MisparsedNode<ValueNode
       include ErrorNode
       param_names :open,:middle,:close_
@@ -5116,8 +5406,9 @@ end
       end
  
       def error? x=nil
+        @endline||=nil
         inner=middle.grep(MisparsedNode).first and return inner.error?( x )
-        "#@endline: misparsed #{what}: #{middle.map{|node| node&&node.image}.join(' ')}" 
+        "#@endline: misparsed #{what}: #{middle.map{|node| node&&(node.short_inspect rescue node.inspect)}.join}" 
       end
       alias msg error?
     end