redparse 0.8.1 → 0.8.2

data/lib/redparse/reg_more_sugar.rb

@@ -16,6 +16,19 @@
     along with this program.  If not, see <http://www.gnu.org/licenses/>.
 =end
 
+  #this is actually fixing a bug in reg 0.4.7, grrrrr
+  unless Array===(Class*5).subregs
+    module ::Reg
+      class Repeat
+        undef subregs if instance_methods.include? "subregs"
+        def subregs
+          [@reg]
+        end
+      end
+    end
+  end
+
+
   unless defined? ::Reg::Transform and ::Reg::Transform.ancestors.include? ::Reg::HasCmatch
   #hack, until support for this syntax makes it into the release of reg
     module ::Reg
@@ -49,6 +62,10 @@
           @reg
         end
 
+        def subregs
+          [@reg]
+        end
+
         def itemrange
           0..0
         end
@@ -72,6 +89,10 @@
           @reg
         end
 
+        def subregs
+          [@reg]
+        end
+
         def itemrange
           0..0
         end

data/lib/redparse/version.rb

@@ -1,3 +1,3 @@
 class RedParse
-  VERSION='0.8.1'
+  VERSION='0.8.2'
 end

data/test/data/def_spec.rb

@@ -0,0 +1,2 @@
+def foo(a = 1)    end; def foo(a=b=c={})  end; def bar(a=b=c=1,d=2)  end
+

data/test/rp-locatetest.rb

@@ -39,7 +39,7 @@ $RUBY=ENV['RUBY']||'ruby'
 
 #$RUBYLEXERVSRUBY="#$RUBY test/code/rubylexervsruby.rb"
 
-RUBY_VERSION[/^1\.8\.7/] or raise 'need ruby>= 1.8.7'
+RUBY_VERSION[/^1\.8\.6/] or raise 'need ruby>= 1.8.6'
 
 
 
@@ -140,6 +140,7 @@ RUBYLIST=(pf.badlist+
     split("\n")
 ].sort_by{rand})-pf.goodlist + pf.goodlist
 RUBYLIST.reject!{|x| %r{japanese/zipcodes\.rb$}===x } #65M of ruby src!!!
+RUBYLIST.reject!{|x| /\A\s*\Z/===x }
 RUBYLIST.uniq!
 
 def self.main

data/test/test_redparse.rb

@@ -20,21 +20,277 @@
 require 'test/unit'
 require 'rubygems'
 require 'parse_tree'
+require 'tempfile'
 
 require "redparse"
 
-begin
 require "rubylexer/test/testcases"
-rescue Exception
-  puts "!!!!!!rubylexer/test/testcases not found!!!!"
-  module TestCases;TESTCASES=[];end
+
+$VERBOSE=1
+
+class Test::Unit::TestCase
+  def known_error
+    from=caller.first
+    from=from[/ in `.*'\Z/] || from[/\A[^:]*:[^:]*/]
+    yield
+  rescue Test::Unit::AssertionFailedError=>e
+    warn "an expected error occurred in #{from}: #{e.message}"
+    if defined? @@known_errors
+      @@known_errors+=1
+    else
+      @@known_errors=1
+      at_exit {warn "!!!UNFIXED KNOWN ERRORS!!!: #@@known_errors"}
+    end
+  rescue Exception=>e
+    raise
+  else
+    warn "expected error in #{from}, but was fixed(?!)"
+    if defined? @@known_errors_fixed
+      @@known_errors_fixed+=1
+    else
+      @@known_errors_fixed=1
+      at_exit {warn "unexpectedly fixed known errors: #@@known_errors_fixed"}
+    end
+  end
+
+  def known_failure
+    from=caller.first
+    from=from[/ in `.*'\Z/] || from[/\A[^:]*:[^:]*/]
+    yield
+  rescue Exception=>e
+    warn "an expected failure occurred in #{from}: #{e}"
+    if defined? @@known_failures
+      @@known_failures+=1
+    else
+      @@known_failures=1
+      at_exit {warn "!!!UNFIXED KNOWN FAILURES!!!: #@@known_failures"}
+    end
+  else
+    warn "expected failure in #{from}, but was fixed(?!)"
+    if defined? @@known_failures_fixed
+      @@known_failures_fixed+=1
+    else
+      @@known_failures_fixed=1
+      at_exit {warn "unexpectedly fixed known failures: #@@known_failures_fixed"}
+    end
+  end
+
+  def slow
+    if ENV['SLOW']
+      yield
+    else
+      if defined? @@slow_spots
+        @@slow_spots+=1
+      else
+        @@slow_spots=1
+        at_exit {warn "slow test code skipped in #@@slow_spots places. (set SLOW to enable)"}
+      end
+    end
+  end
+end
+
+require 'test/unit/ui/console/testrunner'
+class Test::Unit::UI::Console::TestRunner
+  alias add_fault__no_immed_output add_fault
+  def add_fault fault
+    @fault_count||=0
+    output("\n%3d) %s" % [@fault_count+=1, fault.long_display])
+    add_fault__no_immed_output fault
+  end
+end
+
+class ParseTree
+  def put o
+    o=Marshal.dump o
+    @@out.write [o.size].pack("N")+o
+  end
+  def get
+    Marshal.load @@in.read(@@in.read(4).unpack("N")[0])
+  end
+  def fork_server?
+    return if defined? @@out 
+    si,co=IO::pipe
+    ci,so=IO::pipe
+    fork{
+      begin
+      co.close; ci.close
+      @@out=so; @@in=si
+      warnstash=Tempfile.new "warnstash"
+      STDERR.reopen warnstash
+      while 1
+        str=get
+        exit! if str==:exit!
+
+        pos=STDERR.pos
+
+        tree=
+        begin
+          parse_tree_for_string(str) #tree
+        rescue Exception=>e; 
+          tree=e
+        end
+        put tree
+
+        open(STDERR.path){|f| 
+          f.pos=pos
+          put f.read.split #warnings
+        }
+      end
+      rescue Exception=>e; p e; raise
+      ensure exit!
+      end
+    }
+    si.close; so.close
+    @@out=co; @@in=ci
+    at_exit { put :exit! }
+  end
+
+  #returns +[parse_tree|Exception, +[String.*]]
+  def parse_tree_and_warnings str
+    fork_server?
+    put str
+    tree=get
+    warnings=get
+    raise tree if Exception===tree
+    return tree,warnings
+  end
+
+  #this way is bad enough, but there's a fd leak this way,
+  #so I have to use the even more complicated version above
+  def parse_tree_and_warnings_leaks_stderr(str)
+    oldSTDERR=STDERR.dup
+    warnstash=Tempfile.new "warnstash"
+    warnings=[]
+
+    STDERR.reopen warnstash
+      tree=parse_tree_for_string(str)
+
+    return tree,warnings
+  ensure
+    STDERR.reopen oldSTDERR
+
+    warnstash.rewind
+    warnings.replace warnstash.read.split
+    warnstash.close
+  end
 end
 
 class RedParseTest<Test::Unit::TestCase
+  ERROR_EXAMPLES=[
+   '%W"is #{x}#{"Slim #{2?"W":"S"} "}."',
+   '%W"is #{x}#{"Slim #{2?"W":"S"}"}#{xx}."',
+   '%W"is #{x}#{"Slim #{2?W: S} "}."',
+   '%W"is #{x}#{%Q\'Slim #{2?W: S} \'}."',
+   '%W"is #{x}#{%W"Slim #{2?"W":"S"}"}#{xx}."',
+   '%W"is #{x}#{%W\'Slim #{2?W: S} \'}."',
+   '%W"is #{x}#{%q\'Slim #{2?W: S} \'}."',
+   '%W"is #{x}#{%r\'Slim #{2?W: S} \'}."',
+   '%W"is #{x}#{%w\'Slim #{2?W: S} \'}."',
+   '%W"is #{x}#{%x\'Slim #{2?W: S} \'}."',
+   '%W"is #{x}#{/Slim #{2?W: S} /}."',
+   '%W"is #{x}#{`Slim #{2?W: S} `}."',
+   '%W"is_#{"Slim_#{2?"W":"S"}"}#{xx}."',
+   '%W"is_#{x}#{"Slim_#{2?"W":"S"}"}#{xx}."',
+   '%W"is_#{x}#{"Slim_#{2?"W":"S"}_"}."',
+  ]
+  FAILURE_EXAMPLES=[
+  ]
+  RUBYBUG_EXAMPLES=[
+    'def foo(a=b=c={}) end',
+    "$11111111111111111111111111111111111111111111111111111111111111111111",
+    "c do p (110).m end",
+    "case F;when G; else;case; when j; end;end",
+    "p = p m %(1)",
+    "p = p m %(1) ,&t",
+    "p = p m %(1) ,*t",
+    "p = p m %(1) ,t",
+    "p = p m %(1) do end",
+    "p=556;p (e) /a",
+    "z{|| p (1).m}",
+    'def sum(options = {:weights => weights = Hash.new(1)}); options.empty? or options.keys.size > 1; end',
+  ]
+
   ONELINERS=[
+    'z = valueo_s rescue "?"'...'',
+    '"#{publi}#{}>"'...'',
+    'return (@images = @old_imgs)'...'',
+    ' /\A#{__FILE__}tcase/n =~ i '...'',
+    'doc_status, err_args = Documeh_status{fcgi_state = 3; docespond do doc_response =fcgi_state =  1; end }'...'',
+    'print "coled: " + $! +" wiin #{@ray}\n";'...'',
+    'class A;def b;class <<self;@@p = false end;end;end'...'',
+    'def d; return (block_given? ? begin; yield f; ensure; f.close; end : f); end'...'',
+    'def sum(options = {:weights => weights = Hash.new(1)}); options.empty? or options.keys.size > 1; end'...'',
+    'def d;e install_dir;end'...'',
+    '(m).kk,(m+n).kk=3'...'',
+    '((m).kk,(m+n).kk)=3'...'',
+    '((m).kk,(m+n).kk),t=3'...'',
+    's,((m).kk,(m+n).kk)=3'...'',
+    's,((m).kk,(m+n).kk),t=3'...'',
+    'proc{|(m).kk,(m+n).kk| }'...'',
+    "p p:'b'"...'', #changes semantics in 1.9
+    "p:'b'"...'', #but this doesn't
+    't ?6e0 : 5'...'',
+    "super[]"...'',
+    'p end=5'...'', #illegal
+    'p or=5'...'', #illegal
+    'p else=5'...'', #illegal
+    'p if=5'...'', #illegal
+    "x{\x0afor i in (begin\n[44,55,66,77,88] end) do p i**Math.sqrt(i) end\n}\n"...'',
+    'e { |c|; print "%02X" % c }'...'',
+    %[p <<-heredoc "x y z" and 5\n       a b c\n     heredoc]...'',
+    "File.open() {|f|  ;  }"...'',
+    %[     p <<-heredoc "x y z" and 5\n       a b c\n     heredoc]...'',
+    'a rescue b until 1'...'',
+    'a rescue b while 1'...'',
+    '%W[r\\c     3]'...'',
+
+    "%w[a\\C- b]"...'',
+    "%w[a\\M- b]"...'',
+    "%w[a\\c b]"...'',
+    "%W[a\\c b]"...'',
+    "%w[a\\\\c b]"...'',
+    "%W[a\\\\c b]"...'',
+    "%w[a\\c  b]"...'',
+    "%W[a\\c  b]"...'',
+    "%w[a\\\\c  b]"...'',
+    "%W[a\\\\c  b]"...'',
+    "module A; b; rescue C=>d; e; else g; ensure f; end"...'',
+    "class<<A; b; rescue C=>d; e; else g; ensure f; end"...'',
+    "class A; b; rescue C=>d; e; else g; ensure f; end"...'',
+    'def i;"..#{@@c = 1}";end'...'',
+    "def wait; begin; ync; mup; end ;end"...'',
+    "def e; begin; y; g else t; m; end ;end"...'',
+    '"#{}"""'...'',
+    %[%W(white\\  \\  \\ \\  \\ space).should == ["white ", " ", "  ", " space"]]...'',
+    "%w[- \\\\ ]"...'',
+    "%w[- \\\\ e]"...'',
+    "begin begin; ync; p1; end;rr end"...'',
+    "begin;mode;rescue;o_chmod rescue nil;end"...'',
+    "%w![ ] { } ( ) | - * . \\\\ ? + ^ $ #!"...'',
+
+    'def foo(a = 1)    end; def foo(a=b=c={})  end; def bar(a=b=c=1,d=2)  end'...'',
+    '() until 1'...'',
+    '(a) until 1'...'',
+    '(a) while l 1'...'',
+    "module A; B='' end"...'',
+    "{'one' => {}}"...'',
+    "a[3, 2] = 'a', 'b'"...'',
+    'def st;  begin ire end end'...'',
+    'x{k=1; return-k}'...'',
+    'x{k=1; return -k}'...'',
+    'x{k=1; return- k}'...'',
+    'x{k=1; return - k}'...'',
+    'x{k=1; return + k}'...'',
+    'x{k=1; return * k}'...'',
+    'x{k=1; break - k}'...'',
+    'x{k=1; next - k}'...'',
+    "p\r44"...'',
+    'yield []'...'',
+    'yield [1]'...'',
+    'f{a rescue b}'...'',
     '%w[- \\\\\\\\ e]'...'',
     '%w[- \\\\\\\\ ]'...'',
-    'a=[]; a [42] = 24'...'',   #not legal unless a is a local var
+    'x{a=[]; a [42] = 24}'...'',   #not legal unless a is a local var
     '%w![ ] { } ( ) | - * . \\\\\\\\ ? + ^ $ #!'...'',
     '%W(white\  \  \ \  \ space).should == ["white ", " ", "  ", " space"]'...'',
     'module A; b; rescue C=>d; e; else g; ensure f; end'...'',
@@ -761,6 +1017,32 @@ class RedParseTest<Test::Unit::TestCase
     '%W"is #{"Slim #{2?"W":"S"}"}."'...'',
     '"is #{"Slim #{2?"W":"S"}"}#{xx}."'...'',
     '%W"is #{"Slim #{2?"W":"S"}"}#{xx}."'...'',
+    '"is #{x}#{"Slim #{2?"W":"S"}"}#{xx}."'...'',
+    '%W"is #{x}#{"Slim #{2?"W":"S"}"}#{xx}."'...'',
+    '"is #{x}#{%W"Slim #{2?"W":"S"}"}#{xx}."'...'',
+    '%W"is #{x}#{%W"Slim #{2?"W":"S"}"}#{xx}."'...'',
+
+    '%W"is_#{x}#{"Slim_#{2?"W":"S"}"}#{xx}."'...'',
+    '%W"is_#{"Slim_#{2?"W":"S"}"}#{xx}."'...'',
+    '%W"is_#{"Slim_#{2?"W":"S"}"}."'...'',
+    '%W"is_#{x}#{"Slim_#{2?"W":"S"}_"}."'...'',
+    '%W"is_#{"Slim_#{2?"W":"S"}_"}."'...'',
+    '%W"is_#{"Slim_#{2?"W":"S"}_" "rr"}."'...'',
+
+    "%W\"is \#{x}\#{%W'Slim \#{2?W: S} '}.\""...'',
+    "%W\"is \#{x}\#{%w'Slim \#{2?W: S} '}.\""...'',
+    "%W\"is \#{x}\#{%Q'Slim \#{2?W: S} '}.\""...'',
+    "%W\"is \#{x}\#{\"Slim \#{2?W: S} \"}.\""...'',
+    "%W\"is \#{x}\#{%q'Slim \#{2?W: S} '}.\""...'',
+    "%W\"is \#{x}\#{%x'Slim \#{2?W: S} '}.\""...'',
+    "%W\"is \#{x}\#{%r'Slim \#{2?W: S} '}.\""...'',
+    "%W\"is \#{x}\#{`Slim \#{2?W: S} `}.\""...'',
+    "%W\"is \#{x}\#{/Slim \#{2?W: S} /}.\""...'',
+    "%\"is \#{x}\#{%r'Slim \#{2?W: S} '}.\""...'',
+    "\"is \#{x}\#{%r'Slim \#{2?W: S} '}.\""...'',
+    "\"is \#{x}\#{%'Slim \#{2?W: S} '}.\""...'',
+    "\"is \#{x}\#{'Slim \#{2?W: S} '}.\""...'',
+
     'case $1; when 0,*[2,66]: p 1; when 3; 4 else p 2 end'...'',
     'case $1; when 0,*a: p 1; when 3; 4 else p 2 end'...'',
     ' p(String <<- Class)'...'',
@@ -2388,7 +2670,93 @@ class RedParseTest<Test::Unit::TestCase
     ]
 
 
-  STANZAS=%q[
+   PASSTHRU_BSLASHES_ENTIRE=<<'END'
+     <<-'foo'.count('\\')==0
+       '
+     foo
+
+     <<-'foo'.count('\\')==1
+       \'
+     foo
+
+      <<-'foo'.count('\\')==2
+      \\'
+      foo
+
+      <<-'foo'.count('\\')==3
+      \\\'
+      foo
+
+      <<-'foo'.count('\\')==4
+      \\\\'
+      foo
+
+      <<-'foo'.count('\\')==5
+      \\\\\'
+      foo
+
+      <<-"foo".count('\\')==0
+      "
+      foo
+
+      <<-"foo".count('\\')==0
+      \"
+      foo
+
+      <<-"foo".count('\\')==1
+      \\"
+      foo
+
+      <<-"foo".count('\\')==1
+      \\\"
+      foo
+
+      <<-"foo".count('\\')==2
+      \\\\"
+      foo
+
+      <<-"foo".count('\\')==2
+      \\\\\"
+      foo
+
+     <<-`foo`
+       \`
+     foo
+
+     <<-`foo`
+       \\`
+     foo
+
+     <<-"foo"
+       \"
+     foo
+
+     <<-"foo"
+       "
+     foo
+
+     <<-"foo"
+       \\"
+     foo
+
+END
+
+  STANZAS=PASSTHRU_BSLASHES_ENTIRE+%q[
+    return @senders[1] =
+      2
+
+    case
+    when 0
+      guecoding     
+    else case
+      when eucjp_match_length 
+        guing
+      end
+    end
+
+     %w[ ac
+         df]
+
      begin 
        a
      rescue B=>c
@@ -3003,11 +3371,11 @@ end
     end
 
         location_display = if(location.size == 1)
-          location[0].sub(/\A(.+:\d+).*/, ' [\\1]')
+          location[0].sub(/\\A(.+:\\d+).*/, ' [\\1]')
         else
-          "\n#{location.join("\n")}"
+          "\\n#{location.join("\\n")}"
         end
-        "Failure:\n#@test_name#{location_display}:\n#@message"
+        "Failure:\\n#@test_name#{location_display}:\\n#@message"
 
   def parse_date(aString)
     return Time.rfc822(aString) rescue Time.parse(aString)
@@ -3138,10 +3506,15 @@ EOS
   end
 
   EXAMPLES=ONELINERS.map{|rng| rng.first}+STANZAS.split(/\n( *\n)+/)+TestCases::TESTCASES+
-           from_rubinius #+ESC_SEQS
+           from_rubinius + #ESC_SEQS +
+           []-ERROR_EXAMPLES-FAILURE_EXAMPLES-RUBYBUG_EXAMPLES
+
 
   WRAPPERS=[ #enable at most 2 or tests take forever!!!
     '(...)',  #normal mode, should usually be enabled
+#    'a rescue (...)',
+#    "((...))",
+#    'def ((...)).foo; end',
 #    'a0 = (...) rescue b0',
 #    'a0 = ((...)) rescue b0',    
 #    '(...)  #with a comment',
@@ -3160,11 +3533,10 @@ EOS
 #    'a=b,c=d',
   ]
   puts "warning: most data fuzzing is disabled for now"
-  puts "warning: unparser tests disabled for now"
 
-  RUBYIDENT=/((?:$|@@?)?[a-z_][a-z_0-9]*[?!]?)/i
+  RUBYIDENT=/((?:$|@@?)?#{RubyLexer::LETTER}#{RubyLexer::LETTER_DIGIT}*[?!]?)/o
 
-  def self.snippet2testmethod(snippet)
+  def self.snippet2testmethod(snippet,wrap=nil)
     escaped=snippet.gsub(/[\\']/){"\\"+$&}
     safe=escaped.gsub(/([^ -~])/){
       x=$1[0].to_s(16)
@@ -3173,11 +3545,13 @@ EOS
     }
     safe.gsub! /\\\\/,"__"
     safe[/[^ -~]|\\\\/] and fail
+    cp="check_parsing '#{escaped}',pt"
+    cp="#{wrap}{#{cp}}" if wrap
     "
       define_method 'test_parsing_of_#{safe}' do
         #puts 'test_parsing_of_#{safe}'
         pt=ParseTree.new
-        check_parsing '#{escaped}',pt
+        #{cp}
       end
     "
   end
@@ -3190,25 +3564,83 @@ EOS
     /\A\s*x\s*\{(.*)\}\Z/===xmpl and xmpl=$1
 #    xmpl= (xmpl[/\A\s*x\s*\{(.*)\}\s*(#.*)?\Z/,1] rescue xmpl)
 
-  wrapped=WRAPPERS.map{|wrap|
-     #apply wrapper
-    snippet2testmethod(wrap.gsub('(...)', xmpl))
-  }
-  injected=INJECTABLES.map{|inj|
-    xlist=xmpl.split(RUBYIDENT)
-    if xlist.size>1
-      i=rand(xlist.size&~1)|1
-      i&1==1 or fail
-      /\A#{RUBYIDENT}\Z/o===xlist[i] or fail
-      xlist[i]=inj
-      snippet2testmethod(xlist.to_s)
-    end
-  }
-  wrapped+injected
+    wrapped=WRAPPERS.map{|wrap|
+       #apply wrapper
+      maybe_slow="slow" if xmpl.size>1000
+      wrap=wrap.dup
+      wrap['(...)']=xmpl
+      snippet2testmethod(wrap,maybe_slow)
+    }
+    injected=INJECTABLES.map{|inj|
+      xlist=xmpl.split(RUBYIDENT)
+      if xlist.size>1
+        i=rand(xlist.size&~1)|1
+        i&1==1 or fail
+        /\A#{RUBYIDENT}\Z/o===xlist[i] or fail
+        xlist[i]=inj
+        maybe_slow="slow" if xmpl.size>1000
+        snippet2testmethod(xlist.to_s,maybe_slow)
+      end
+    }
+    wrapped+injected
   }.to_s
   #puts code.split("\n")[5880..5890].join("\n")
   eval code
 
+  error_code=RUBYBUG_EXAMPLES.map{|xmpl| 
+    #remove comment if present
+    /\A(.*)\s*\Z/===xmpl and xmpl=$1
+    #remove x{...} wrapper if present
+    /\A\s*x\s*\{(.*)\}\Z/===xmpl and xmpl=$1
+
+    snippet2testmethod(xmpl, :known_ruby_bug)
+  }.to_s
+  eval error_code
+
+  error_code=ERROR_EXAMPLES.map{|xmpl| 
+    #remove comment if present
+    /\A(.*)\s*\Z/===xmpl and xmpl=$1
+    #remove x{...} wrapper if present
+    /\A\s*x\s*\{(.*)\}\Z/===xmpl and xmpl=$1
+
+    snippet2testmethod(xmpl, :known_error)
+  }.to_s
+  eval error_code
+
+  failure_code=FAILURE_EXAMPLES.map{|xmpl| 
+    #remove comment if present
+    /\A(.*)\s*\Z/===xmpl and xmpl=$1
+    #remove x{...} wrapper if present
+    /\A\s*x\s*\{(.*)\}\Z/===xmpl and xmpl=$1
+
+    snippet2testmethod(xmpl, :known_failure)
+  }.to_s
+  eval failure_code
+
+  def known_ruby_bug
+    from=caller.first
+    from=from[/ in `.*'\Z/] || from[/\A[^:]*:[^:]*/]
+    yield
+  rescue Test::Unit::AssertionFailedError=>e
+    warn "a known bug in MRI reared its head in #{from}: #{e.message}"
+    if defined? @@known_ruby_bugs
+      @@known_ruby_bugs+=1
+    else
+      @@known_ruby_bugs=1
+      at_exit {warn "unfixed bugs in MRI/ParseTree: #@@known_ruby_bugs"}
+    end
+  rescue Exception=>e
+    raise
+  else
+    warn "expected bug in MRI in #{from}, but was fixed(?!)"
+    if defined? @@known_ruby_bugs_fixed
+      @@known_ruby_bugs_fixed+=1
+    else
+      @@known_ruby_bugs_fixed=1
+      at_exit {warn "unexpectedly fixed known MRI/ParseTree bugs: #@@known_ruby_bugs_fixed"}
+    end
+  end
+
   def test_case_that_segfaults_ruby185
     assert_equal \
       [[:op_asgn1, [:call, [:vcall, :a], :b], [:zarray], :%, [:vcall, :d]]],
@@ -3227,13 +3659,91 @@ EOS
        RedParse.new('Proc{|&b|}','-').parse.to_parsetree(:quirks)
   end
 
+  def test_cases_misparsed_by_ruby186_slash_parsetree
+    {"$11111111111111111111111111111111111111111111111111111111111111111111"=>[
+       [:nth_ref,  11111111111111111111111111111111111111111111111111111111111111111111]],
+      "c do p (110).m end"=>[
+       [:iter, [:fcall, :c], nil, [:fcall, :p, [:array, [:call, [:lit, 110], :m]]]]],
+      "case F;when G; else;case; when j; end;end"=>[
+       [:case,  [:const, :F],  [:when, [:array, [:const, :G]], nil],
+         [:case, nil, [:when, [:array, [:vcall, :j]], nil], nil]]],
+      "p = p m %(1)"=>[
+       [:lasgn, :p, [:fcall, :p, [:array, [:fcall, :m, [:array, [:str, "1"]]]]]]],
+      "p = p m %(1) ,&t"=>[
+       [:lasgn,  :p,  [:fcall,   :p,   [:array,
+         [:block_pass, [:vcall, :t], [:fcall, :m, [:array, [:str, "1"]]]]]]]],
+      "p = p m %(1) ,*t"=>[
+       [:lasgn,  :p,  [:fcall,   :p,
+         [:array, [:fcall, :m, [:argscat, [:array, [:str, "1"]], [:vcall, :t]]]]]]],
+      "p = p m %(1) ,t"=>[
+       [:lasgn,  :p,  [:fcall, :p, [:array, [:fcall, :m, 
+         [:array, [:str, "1"], [:vcall, :t]]]]]]],
+      "p = p m %(1) do end"=>[
+       [:lasgn,  :p,  [:iter, [:fcall, :p, [:array, 
+         [:fcall, :m, [:array, [:str, "1"]]]]], nil]]],
+      "p=556;p (e) /a"=>[
+       [:block,  [:lasgn, :p, [:lit, 556]],
+         [:fcall, :p, [:array, [:call, [:vcall, :e], :/, [:array, [:vcall, :a]]]]]]],
+      "z{|| p (1).m}"=>[
+       [:iter, [:fcall, :z], 0, [:fcall, :p, [:array, [:call, [:lit, 1], :m]]]]],
+      "
+        <<-'foo'
+        \\'
+        foo
+      "=>[[:str, "        \\'\n"]],
+      "
+        <<-'foo'
+        \\\\'
+        foo
+      "=>[[:str, "        \\\\'\n"]],
+      "
+        <<-'foo'
+        \\\\\\'
+        foo
+      "=>[[:str, "        \\\\\\'\n"]],
+      "
+        <<-'foo'
+        \\\\\\\\'
+        foo
+      "=>[[:str, "        \\\\\\\\'\n"]],
+      "
+        <<-'foo'
+        \\\\\\\\\\'
+        foo
+      "=>[[:str, "        \\\\\\\\\\'\n"]],
+      "
+        <<-'foo'
+        \\\\\\\\\\\\'
+        foo
+      "=>[[:str, "        \\\\\\\\\\\\'\n"]],
+
+
+    }.each_pair{|code,tree| 
+      assert_equal tree,RedParse.new(code,'-').parse.to_parsetree(:quirks)
+    }
+  end
+
+  POWERS_OF_2={}
+  i=1
+  while i<1_000_000_000
+    POWERS_OF_2[i]=1
+    i*=2
+  end
+
   def assert_hopefully_raises_Exception xmpl
     begin
       yield
     rescue Exception
       assert true
     else
-      puts "warning: syntax error expected, but none was seen, expression: <<< #{xmpl} >>>"
+      if defined? @@missed_syntax_errors
+        @@missed_syntax_errors+=1
+      else
+        @@missed_syntax_errors=1
+        at_exit{warn "missed syntax errors: #@@missed_syntax_errors"}
+      end
+      #puts "warning: syntax error expected, but none was seen, expression: <<< #{xmpl} >>>" if 
+      #  POWERS_OF_2[@@missed_syntax_errors]
     end
   end
 
@@ -3248,11 +3758,12 @@ EOS
     @problem_exprs=$stdout
   end
 
-  @@differed_by_nil=0
-
   def check_parsing xmpl,pt=ParseTree.new
+    pt_opts=[:quirks]
+    pt_opts<<:ruby187 if ::VERSION["1.8.7"]
     /unparse/===xmpl and warn 'unparse in parser test data!'
     problem_exprs=problem_exprs()
+    nodes=warnings=warnings2=nil
 =begin
       xmpl=<<-prefix+xmpl+<<-suffix
         BEGIN{throw :never_exec_parse_data_try1,1}
@@ -3272,31 +3783,39 @@ EOS
 #        catch(:never_exec_parse_data_try1){
 #          catch(:never_exec_parse_data_try2){
 #            catch(:never_exec_parse_data_try3){
-              tree=pt.parse_tree_for_string(xmpl)
+              tree,warnings=pt.parse_tree_and_warnings(xmpl)
 #            }
 #          }
 #        }
 #        break if loops+=1 > 3
-      rescue Interrupt: raise
-      rescue Exception=>e:
+      rescue Interrupt; raise
+      rescue Exception=>e
+        #pp e
+        #pp e.backtrace
         #raise "last gasp ParseTree exec catcher failed!"
         tree=e
-        tree2=nodes=nil
+        tree2=nodes=h=nil
         assert_hopefully_raises_Exception(xmpl){
           nodes=RedParse.new(xmpl,"-").parse
-          tree2=nodes.to_parsetree(:quirks)
+          h=nodes.hash
+          tree2,warnings2=nodes.to_parsetree_and_warnings(*pt_opts)
         }
+        assert_equal h,nodes.hash if h
       else
         begin
           nodes=RedParse.new(xmpl,"-").parse
-          tree2=nodes.to_parsetree(:quirks)
-          if tree==tree2
-            assert true
-          else
-            assert_equal recursive_compact!(tree), recursive_compact!(tree2)
-            warn "parsetree differed by nil(s), expression #{xmpl}"
-            @@differed_by_nil+=1
-            END{ puts "#@@differed_by_nil expressions differed by nil(s)" }
+          h=nodes.hash
+          tree2,warnings2=nodes.to_parsetree_and_warnings(*pt_opts)
+          assert_equal h,nodes.hash
+          assert_equal tree, tree2
+          assert_equal warnings, warnings2 if ENV['WARN_PICKINESS']
+          if warnings != warnings2
+            if defined? @@mismatched_warnings
+              @@mismatched_warnings+=1
+            else
+              @@mismatched_warnings=1
+              at_exit{warn "mismatched warnings: #@@mismatched_warnings (set WARN_PICKINESS for details)"}
+            end
           end
         rescue Exception=>e
           if problem_exprs
@@ -3313,26 +3832,40 @@ EOS
         
       end #until output.equal? tree 
 
-      return #skip unparse tests for now
+#      puts "warning: unparser tests disabled for now"
+#      return #skip unparse tests for now
 
-      reparsed= RedParse.new(nodes.unparse({}),"-").parse
-      assert_equal nodes.delete_extraneous_ivars!,
-                   reparsed.delete_extraneous_ivars!
+      return unless nodes
+      begin
+        unparsed=nodes.unparse
+        if unparsed==xmpl
+          assert true
+          return
+        end
+        reparsed= RedParse.new(unparsed,"-").parse
+        if nodes.delete_extraneous_ivars! != reparsed.delete_extraneous_ivars!
+          assert_equal nodes.delete_linenums!, reparsed.delete_linenums!
+          warn "unparser doesn't preserve linenums perfectly in #{xmpl}"
+          if defined? @@unparse_mismatched_linenums
+              @@unparse_mismatched_linenums+=1
+          else
+              @@unparse_mismatched_linenums=1
+              at_exit{warn "unparse mismatched linenums: #@@unparse_mismatched_linenums"}
+          end
+        else
+          assert true 
+        end
+      rescue Exception
+        raise unless Exception===tree
+      end
 
-      tree3=reparsed.to_parsetree(:quirks)
-      assert_equal tree, tree3
+      unless Exception===tree
+        tree3=reparsed.to_parsetree(*pt_opts)
+        assert_equal tree, tree3
+      else #missing a syntax errr, but that's been noted already
+      end
 
 #  rescue Exception=>e:
 #      raise "error: #{e}:#{e.class} while testing '#{xmpl}'"
   end
-
-  def recursive_compact!(tree)
-    return tree unless Array===tree
-    recursive_compact_!(tree)
-  end
-  def recursive_compact_!(tree)
-    tree.compact!
-    tree.each{|subtree| recursive_compact_!(subtree) if Array===subtree}
-    return tree
-  end
 end