rubylexer 0.7.0 → 0.7.1

data/test/code/errscan

@@ -0,0 +1,2 @@
+#!/bin/sh
+egrep -B1 -v '^(\(expected\) |executing)' locatetest.out

data/test/code/isolate_error.rb

@@ -0,0 +1,72 @@
+require 'test/code/rubylexervsruby'
+opts = GetoptLong.new( [ '--ends', '-e', GetoptLong::OPTIONAL_ARGUMENT] )
+endcount=0
+opts.each {|opt,arg|
+  if opt=='--ends'
+    if arg==''
+      endcount=1
+    else
+      endcount=arg.to_i
+    end
+  end
+}
+
+ends="end\n"*endcount
+
+endcode=ends#+"__END__\n"
+
+lines=IO.readlines(ARGV.first)
+
+low=1
+high=lines.size-1
+
+class DontEvalThis<Exception; end
+
+def syntax_ok?(code)
+  caught=
+  begin
+    catch :dontevalthis do
+      eval "
+        BEGIN{throw :dontevalthis, :notevaled}
+        BEGIN{raise DontEvalThis}
+      "+code
+    end
+  rescue DontEvalThis:
+    puts "first level eval barrier failed!!!"
+    caught=:notevaled
+  rescue Exception:
+    caught=:unexpected
+  end
+  case caught
+  when :unexpected
+    return false
+  when :notevaled
+    return true
+  else #wtf?
+    puts "eval barriers totally bypassed?!?!?"
+  end
+end
+
+span=9999999999999999999999999999
+loop do
+  break if high-low>=span
+  span=high-low
+  adjust=1
+  mid=(low+high)/2
+  begin
+    text=lines.dup
+    realmid=mid+adjust
+    realmid=[[realmid,low].max,high].min
+    text=text[0...realmid] << endcode
+    text=text.to_s
+    adjust*=-2
+  end until syntax_ok?text
+  
+  if RubyLexerVsRuby.rubylexervsruby(ARGV.first+".chunk#{low..high}", text)
+    low=realmid
+  else
+    high=realmid
+  end
+end
+
+p low..high

data/test/code/lexloop

@@ -0,0 +1,14 @@
+ruby -Ilib test/code/tokentest.rb --loop --maxws \
+  /home/caleb/sandbox/rubyparser/babyparser.rb \
+  /home/caleb/sandbox/rubyparser/hd2.rb \
+  /home/caleb/sandbox/rubyparser/test_rubyparser.rb \
+  /home/caleb/sandbox/rubylexer/lib/rubylexer/rulexer.rb \
+  /home/caleb/sandbox/rubylexer/lib/rubylexer/tokenprinter.rb \
+  /home/caleb/sandbox/rubylexer/lib/rubylexer/0.6.2.rb \
+  /home/caleb/sandbox/rubylexer/lib/rubylexer/0.7.1.rb \
+  /home/caleb/sandbox/rubylexer/lib/rubylexer/charset.rb \
+  /home/caleb/sandbox/rubylexer/lib/rubylexer/symboltable.rb \
+  /home/caleb/sandbox/rubylexer/lib/rubylexer/charhandler.rb \
+  /home/caleb/sandbox/rubylexer/lib/rubylexer/0.7.0.rb \
+  /home/caleb/sandbox/rubylexer/lib/assert.rb \
+  >/dev/null 2>&1

data/test/code/locatetest.rb

@@ -1,6 +1,30 @@
+=begin legal crap
+    rubylexer - a ruby lexer written in ruby
+    Copyright (C) 2004,2005,2008  Caleb Clausen
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation; either
+    version 2.1 of the License, or (at your option) any later version.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+=end
+
 require 'test/code/rubylexervsruby'
+require 'test/code/strgen'
+
+class RubyLexer
+module LocateTest
 #ENV['RUBY']||='ruby'
 $RUBY=ENV['RUBY']||'ruby'
+skip_til=ENV['SKIP_TIL']
 #test $RUBY || export RUBY=ruby
 
 #$RUBYLEXERVSRUBY="#$RUBY test/code/rubylexervsruby.rb"
@@ -31,19 +55,137 @@ ruby -e 'print ($:.sort.uniq+[""]).join"\n"'|xargs -i ls "{}/*.rb" >> test/resul
 
 =end
 
-for i in [
-  RLROOT+"/test/data/p.rb", *Dir["test/data/*.rb"]+`(locate /tk.rb;
-  locate examples/examples_test.rb;locate .rb; locate rakefile; 
-  locate Rakefile; locate RAKEFILE)|egrep -v '/test/(results|data)/'`.
-    split("\n")
-] do
+bindirs=ENV['PATH'].split(':')+['/sbin','/usr/sbin']
+
+
+puts "hack hack hack: absolute path in ruby test file list"
+RUBYBINS= 
+bindirs.map{|dir| Dir[dir+"/*"].select{|prog| 
+  line1=File.open(prog){|f| f.readline} rescue next
+  %r{\A#!([^\s]*/env )?[^\s]*/j?ruby(?:1\.[6789])(?:\s|\n)}===line1
+}}.flatten
+TEST_THESE_FIRST=
+%w[
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   /home/caleb/sandbox/rubylexer/jewels/xrefresh-server-0.1.0/lib/xrefresh-server.rb
+   /home/caleb/sandbox/rubylexer/jewels/shattered-0.7.0/test/acceptance/lib/mesh_test.rb
+   /home/caleb/sandbox/rubylexer/jewels/rutils-0.2.3/lib/countries/countries.rb
+   /home/caleb/sandbox/rubylexer/jewels/ruport-1.6.1/examples/trac_ticket_status.rb
+   /home/caleb/sandbox/rubylexer/jewels/rubysdl-2.0.1/extconf.rb
+   /home/caleb/sandbox/rubylexer/jewels/rubysdl-1.1.0/extconf.rb
+   /home/caleb/sandbox/rubylexer/jewels/ruby-msg-1.3.1/lib/msg/rtf.rb
+   /home/caleb/sandbox/rubylexer/jewels/ruby-mediawiki-0.1/apps/iso_639_leecher.rb
+   /home/caleb/sandbox/rubylexer/jewels/ruby-finance-0.2.2/lib/finance/quote/yahoo/australia.rb
+   /home/caleb/sandbox/rubylexer/jewels/roby-0.7.2/test/test_task.rb
+   /home/caleb/sandbox/rubylexer/jewels/reve-0.0.94/test/test_reve.rb
+   /home/caleb/sandbox/rubylexer/jewels/remote_api-0.2.1/test/spec_test.rb
+   /home/caleb/sandbox/rubylexer/jewels/rbrainz-0.4.1/examples/getartist.rb
+   /home/caleb/sandbox/rubylexer/jewels/rbrainz-0.4.1/examples/getlabel.rb
+   /home/caleb/sandbox/rubylexer/jewels/rbrainz-0.4.1/examples/gettrack.rb
+   /home/caleb/sandbox/rubylexer/jewels/railscart-0.0.4/starter_app/vendor/plugins/engines/init.rb
+   /home/caleb/sandbox/rubylexer/jewels/ok-extensions-1.0.15/test/extensions/test_object.rb
+   /home/caleb/sandbox/rubylexer/jewels/oai-0.0.8/lib/oai/harvester/logging.rb
+   /home/caleb/sandbox/rubylexer/jewels/oai-0.0.8/examples/providers/dublin_core.rb
+   /home/caleb/sandbox/rubylexer/jewels/erubis-2.6.2/test/assert-text-equal.rbc
+   /home/caleb/sandbox/rubylexer/jewels/erubis-2.6.2/test/test-engines.rbc
+   /home/caleb/sandbox/rubylexer/jewels/erubis-2.6.2/test/test-erubis.rbc
+   /home/caleb/sandbox/rubylexer/jewels/erubis-2.6.2/test/test-users-guide.rbc
+   /home/caleb/sandbox/rubylexer/jewels/erubis-2.6.2/test/test.rbc
+   /home/caleb/sandbox/rubylexer/jewels/erubis-2.6.2/test/testutil.rbc
+   /home/caleb/sandbox/rubylexer/jewels/fb-0.5.5/test/CursorTestCases.rb
+   /home/caleb/sandbox/rubylexer/jewels/flickraw-0.4.5/examples/auth.rb
+   /home/caleb/sandbox/rubylexer/jewels/flickraw-0.4.5/examples/upload.rb
+   /home/caleb/sandbox/rubylexer/jewels/flickraw-0.4.5/test/test.rb
+   /home/caleb/sandbox/rubylexer/jewels/fox-tool-0.10.0-preview/fox-tool/examples/input.rbin
+   /home/caleb/sandbox/rubylexer/jewels/fox-tool-0.10.0-preview/fox-tool/examples/cvs/Base/print.rbin
+   /home/caleb/sandbox/rubylexer/jewels/foxGUIb_1.0.0/foxguib_1.0.0/foxguib/src/gui/_guib_genruby.rbin
+   /home/caleb/sandbox/rubylexer/jewels/hpricot_scrub-0.3.2/test/scrubber_data.rb
+   /home/caleb/sandbox/rubylexer/jewels/htmltools-1.10/test/tc_stacking-parser.rb
+   /home/caleb/sandbox/rubylexer/jewels/ludy-0.1.13/test/deprecated/ts_ludy.rb
+   /home/caleb/sandbox/rubylexer/jewels/menu_helper-0.0.5/test/unit/menu_test.rb
+   /home/caleb/sandbox/rubylexer/jewels/mod_spox-0.0.5/data/mod_spox/extras/PhpCli.rb
+   /home/caleb/sandbox/rubylexer/jewels/motiro-0.6.11/app/core/wiki_page_not_found.rb
+   /home/caleb/sandbox/rubylexer/jewels/motiro-0.6.11/vendor/plugins/globalize/generators/globalize/templates/migration.rb.gz
+
+]+[
+  "/home/caleb/sandbox/rubylexer/jewels/core_ex-0.6.6.3/lib/core_ex/numeric.rb",
+  "/home/caleb/sandbox/rubylexer/jewels/cerberus-0.3.6/test/bjam_builder_test.rb",
+  "/home/caleb/sandbox/rubylexer/jewels/cerberus-0.3.6/test/maven2_builer_test.rb",
+  "/home/caleb/sandbox/rubylexer/jewels/buildr-1.3.1.1/lib/buildr/java/groovyc.rb",
+  "/home/caleb/sandbox/rubylexer/jewels/adhearsion-0.7.7/apps/default/helpers/micromenus.rb",
+  "/home/caleb/rubies/ruby-1.8.7/instruby.rb",
+  "/home/caleb/sandbox/rubylexer/jewels/RuCodeGen-0.3.1/lib/rucodegen/value_incapsulator.rb",
+
+  "/home/caleb/sandbox/rubylexer/jewels/Wiki2Go-1.17.3/test/TestWiki2GoServlet.rb",
+  "/home/caleb/rubies/ruby-1.8.7/test/rss/test_parser_atom_entry.rb",
+  "/home/caleb/sandbox/rubylexer/jewels/active_form-0.0.8/test/elements/test_base_element.rb",
+  "/home/caleb/sandbox/rubylexer/jewels/dohruby-0.2.1/bin/create_database.rb",
+  "/home/caleb/sandbox/rubylexer/jewels/depager-0.2.2/examples/c89/c89.tab.rb",
+
+  "/home/caleb/sandbox/rubylexer/jewels/samizdat-0.6.1/samizdat/lib/samizdat/storage.rb",
+  "/home/caleb/sandbox/rubylexer/jewels/math3d-0.04/tests/make_tests.rb",
+  "/home/caleb/sandbox/rubylexer/jewels/QuickBaseClient.rb/quickbasecontactsappbuilder.rb",
+  "/home/caleb/sandbox/rubylexer/jewels/QuickBaseClient.rb/quickbaseclient.rb",
+  "/home/caleb/sandbox/rubylexer/jewels/ruby-ivy_0.1.0/ruby-ivy/examples/._000-IVYTranslater.rb",
+  "/home/caleb/sandbox/rubylexer/jewels/ruby-ivy_0.1.0/ruby-ivy/examples/._002-ApplicationList.rb",
+  "/home/caleb/sandbox/rubylexer/jewels/ruby-ivy_0.1.0/ruby-ivy/examples/._001-UnBind.rb",
+  "/home/caleb/sandbox/rubylexer/jewels/ruby-ivy_0.1.0/ruby-ivy/._extconf.rb",
+  "/home/caleb/sandbox/rubylexer/jewels/smf-0.15.10/sample/virtual-samp.rb",
+  "/home/caleb/sandbox/rubylexer/jewels/syntax/syntax.rb",
+  "/home/caleb/sandbox/rubylexer/jewels/rex-1.0rc1/rex/packages/rex/test/rex-20060511.rb",
+  "/home/caleb/sandbox/rubylexer/jewels/rex-1.0rc1/rex/packages/rex/test/rex-20060125.rb",
+#  "/home/caleb/sandbox/rubylexer/jewels/japanese-zipcodes-0.0.20080227/lib/japanese/zipcodes.rb", #huge!!!!!
+]
+RUBYLIST=TEST_THESE_FIRST+RUBYBINS+
+[RLROOT+"/test/data/p.rb", 
+  *Dir["test/data/*.rb"]+
+  `(
+    locate /inline.rb /tk.rb;
+    locate examples/examples_test.rb;
+    locate .rb; 
+    locate rakefile; locate Rakefile; locate RAKEFILE; 
+  )|egrep -v '(/test/(results|data)/|.broken$)'`.
+    split("\n").reject{|i| %r(japanese/zipcodes\.rb\Z)===i }
+]
+RUBYLIST.uniq!
+if skip_til
+  skip_til=RUBYLIST.index(skip_til)
+  skip_til or fail "SKIP_TIL not found in list of rubies"
+  RUBYLIST.slice! 0...skip_til
+end
+def self.main
+for i in RUBYLIST do
 #  system $RUBYLEXERVSRUBY, i
   #hmm, rubylexervsruby needs to be upgraded to not regard an output
   #consisting entirely of warnings as a failure.
   #if no 'warning' (in any capitalization) for 4 or more lines
-  RubyLexerVsRuby.rubylexervsruby i #or fail "failed in #{i}"
+  begin
+#  puts File.readlines("/proc/#{$$}/status").grep(/^VmSize:\s/)
+  File.exist? i or next
+  RubyLexerVsRuby.rubylexervsruby i,nil,nil,true #or puts "syntax error in #{i}"
+  rescue Interrupt; raise
+  rescue Exception=>e
+    puts "error in: "+i
+    puts e
+    puts e.backtrace.map{|s| "    from: "+s}.join("\n")
+  end
+end
 end
-
 #for i in test/data/p.rb `(locate /tk.rb;locate examples/examples_test.rb;#locate .rb; locate rakefile; locate Rakefile; locate RAKEFILE)|egrep -v '/#test/results/'`; do
 #  $RUBYLEXERVSRUBY $i
 #done
+end
+end
+
+RubyLexer::LocateTest.main if $0==__FILE__

data/test/code/regression.rb

@@ -0,0 +1,109 @@
+require 'open3'
+
+require 'test/code/testcases'
+require 'test/code/rubylexervsruby'
+  SEP='';
+  'caleb clausen'.each_byte{|ch| SEP<<ch.to_s(2).gsub('0','-').gsub('1','+')}
+  SEP<<'(0)'
+
+#require 'test/code/rubyoracle'
+class<<RubyLexerVsRuby
+
+  def progress ruby,input; end 
+=begin oracular version was just a bad idea....
+  def ruby_parsedump(input,output,ruby)
+    #todo: use ruby's md5 lib
+    #recursive ruby call here is unavoidable because -y flag has to be set
+
+    @oracle||= Open3.popen3("#{ruby} -w -y -c")
+    @oracle[0].write IO.read(input)
+    @oracle[0].flush
+    #timeout=Time.now+0.1
+    data=''
+    while data.empty? #  and Time.now<timeout
+      begin 
+        data<< chunk=@oracle[2].read_nonblock(1024) 
+        timeout=Time.now+0.02
+      rescue EOFError
+#        data<<"\nError: premature eof...\n"
+        break
+      rescue Errno::EAGAIN
+        break if data[/^Reading a token: \Z/] and Time.now>=timeout
+      end while chunk
+    end
+
+    status=0
+    lines=data.split("\n")
+    File.open(output,"w") { |outfd|
+      lines.each{|line| 
+        outfd.puts(line) if /^Shifting/===line
+        if /^#{DeleteWarns::WARNERRREX}|^Error|^(Now at end of input\.)/o===line
+          outfd.puts(line) 
+          if status.zero? and $2!="warning"
+            status=1 unless $4 #unless end of input seen
+            @oracle.each{|fd| fd.close} if @oracle
+            @oracle=nil
+          end
+        end
+      }
+    }
+    return status
+  end
+=end
+end
+
+if __FILE__==$0
+  require 'test/unit'
+  
+  class LexerTests<Test::Unit::TestCase
+    class LexerTestFailure<RuntimeError; end
+    class DifferencesFromMRILex<LexerTestFailure; end
+    
+    i=-1
+    test_code= TestCases::TESTCASES.map{|tc|
+      i+=1
+      name="testcase_#{i}__"
+      esctc=tc.gsub(/['\\]/){"\\"+$&}
+      %[  
+         define_method '#{name}' do 
+           difflines=[]
+           begin
+             res=RubyLexerVsRuby.rubylexervsruby('__#{name}','#{esctc}',difflines) 
+             difflines.empty? or raise DifferencesFromMRILex, difflines
+             res or raise LexerTestFailure, ''
+           rescue Interrupt: exit
+           rescue Exception=>e: 
+             e.message<<"\n"+'while lexing: #{esctc}'
+             raise e
+           end
+         end  
+      ]
+    }.to_s
+    
+    illegal_test_code= TestCases::ILLEGAL_TESTCASES.map{|tc|
+      i+=1
+      name="testcase_#{i}__"
+      esctc=tc.gsub(/['\\]/){"\\"+$&}
+      %[  
+         define_method '#{name}' do 
+           difflines=[]
+           begin
+             res=RubyLexerVsRuby.rubylexervsruby('__#{name}','#{esctc}',difflines) 
+             difflines.empty? or raise DifferencesFromMRILex, difflines
+             res or raise LexerTestFailure, ''
+           rescue LexerTestFailure: 
+             puts 'warning: test failure lexing "#{esctc}"' 
+           rescue Interrupt: exit
+           rescue Exception=>e: 
+             e.message<<"\n"+'while lexing: #{esctc}'
+             raise e
+           end
+         end  
+      ]
+    }.to_s
+
+
+#    puts test_code+illegal_test_code
+    eval test_code+illegal_test_code
+  end
+end

data/test/code/rubylexervsruby.rb

@@ -1,4 +1,22 @@
 #!/usr/bin/ruby
+=begin legal crap
+    rubylexer - a ruby lexer written in ruby
+    Copyright (C) 2004,2005,2008  Caleb Clausen
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation; either
+    version 2.1 of the License, or (at your option) any later version.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+=end
 #$DEBUG=$VERBOSE=true
 $Debug=true
 require "getoptlong"
@@ -39,15 +57,20 @@ def ruby_parsedump(input,output,ruby)
     } 
   }
   ENABLEMD5 and status==0 and system "md5sum #{input} > #{input}.md5" #compute sum only if no errors
+  return status>>8
 end
 
 def head(fname)
-  print "foobaaaaaaaaar\n\n\n\n\n\n"
   File.open(fname){|fd| print(fd.read(512)+"\n") }
 end
 
-def rubylexervsruby(input,stringdata=nil,&ignore_it)
 
+def progress ruby,input
+  print "executing: #{ruby} -Ilib test/code/tokentest.rb --keepws #{input}\n"
+  $stdout.flush
+end
+
+def rubylexervsruby(input,stringdata=nil,difflines=nil,bulk=nil,&ignore_it)
 #cmdpath= `which #$0`
 cmddir=Dir.getwd+"/test/code/"
 base='test/results/'+File.basename(input)
@@ -67,12 +90,14 @@ expected_failures=Dir.getwd+"/test/code/"+File.basename(input)+".expected_failur
 
 nop_ruby "#{input[/\.gz$/]&&'z'}cat", input, nopfile, stringdata
 
-print "executing: #{ruby} -Ilib test/code/tokentest.rb --keepws #{input}\n"
-
-ruby_parsedump nopfile, origfile, ruby
-
 
+legal=ruby_parsedump nopfile, origfile, ruby
+if false&&legal.nonzero?
+  puts "skipping #{input}; not legal"
+  return
+end
 
+progress ruby, input
 tokentest nopfile, RubyLexer, RubyLexer::KeepWsTokenPrinter.new, nil, _ttfile
 tokentest nopfile, RubyLexer, RubyLexer::KeepWsTokenPrinter.new(' '), nil, mttfile
 
@@ -81,7 +106,7 @@ ruby_parsedump _ttfile, p_ttfile, ruby
 ruby_parsedump mttfile, pmttfile, ruby
 
 if File.exists?(p_ttfile)
-  IO.popen("diff -u1 -b #{origfile} #{p_ttfile}"){ |pipe|
+  IO.popen("diff --unified=1 -b #{origfile} #{p_ttfile}"){ |pipe|
   File.open(p_ttdiff,"w") { |diff|
     DeleteWarns.deletewarns(pipe){|s| diff.print s}
   }
@@ -90,7 +115,7 @@ if File.exists?(p_ttfile)
 end
 
 if File.exists?(pmttfile)
-  IO.popen("diff -u1 -b #{origfile} #{pmttfile}"){ |pipe|
+  IO.popen("diff --unified=1 -b #{origfile} #{pmttfile}"){ |pipe|
   File.open(pmttdiff,"w") { |diff|
     DeleteWarns.deletewarns(pipe){|s| diff.print s}
   }
@@ -101,10 +126,11 @@ end
 list=[]
 #nonwarn4=/(^(?![^\n]*warning[^\n]*)[^\n]*\n){4,}/im
 #4 or more non-warning lines:
-nonwarn4=/^(?:(?![^\r\n]*warning)[^\r\n]+(?:\r\n?|\n\r?)){4,}/mi
+nonwarn4=/^(?:(?![^\r\n]*warning)[^\n]+\n){4,}/mi
 result=true
+(system "ruby -c #{input} >/dev/null 2>&1" or expected="(expected) ") unless stringdata
 for name in [p_ttdiff,pmttdiff] do
-  i=File.read(name)
+  i=File.read(name) rescue next
  # i.tr("\r","\n")
 #  i.gsub!(/^\n/m, '')
   i.sub!(/\A([^\r\n]+(\r\n?|\n\r?)){2}/, '')  #remove 1st 2 lines
@@ -116,9 +142,10 @@ for name in [p_ttdiff,pmttdiff] do
   
   unless list.empty?
   list=list.join("\n") +"\n"
-  unless (File.exists?(expected_failures) and File.read(expected_failures))==list
-    print list
-    result=false
+  unless (File.exists?(expected_failures) and File.read(expected_failures)==list)
+    result=!!expected
+    print "#{expected}error in: #{input}\n"
+    (difflines ? difflines.push(*list) : print(list)) unless expected
   end
     list=[]
   end
@@ -126,6 +153,8 @@ end
 
 #print( list.join("\n") +"\n")
 #Dir.chdir olddir
+File.unlink(*Dir[base+".{tt,mtt,nop}{,.prs}{,.diff}"]) if result and bulk
+
 return result
 
 =begin
@@ -147,10 +176,14 @@ end
 return false
 =end
 
+rescue Interrupt: raise
 rescue Exception
-  system "ruby -c #{input} >/dev/null 2>&1" or expected="(expected) "
+  File.exist? nopfile and
+    system "ruby -c #{nopfile} >/dev/null 2>&1" or expected="(expected) " unless stringdata
   print "#{expected}error in: #{input}\n"
-  raise unless expected
+  File.unlink(*Dir[base+".{tt,mtt,nop}{,.prs}{,.diff}"]) if expected and bulk
+  return true if expected
+  raise
 end
 end
 
@@ -165,6 +198,11 @@ opts.each{|opt,arg|
 }
 
 input||=ARGV[0]
+
+unless input
+  input="-"
+  stringdata=$stdin.read
+end
 RubyLexerVsRuby.rubylexervsruby(input,stringdata) and exit 0
 
 exit 1