rubylexer 0.6.2 → 0.7.0

data/test/code/locatetest

@@ -0,0 +1,30 @@
+test $RUBY || export RUBY=ruby
+export PWD=`pwd`
+export RUBYLEXERVSRUBY="$RUBY -Ilib test/code/rubylexervsruby.rb"
+
+if $RUBY --version|grep '^ruby 1\.6'; then
+  echo 'error: need ruby 1.8'; exit
+fi
+
+mkdir test/results
+
+ruby <<END > test/results/rubyscripts.txt
+require 'open3'
+binfiles=ENV['PATH'].split(':').map{|dir| Dir[dir+'/*']}.compact
+rubyscripts=[]
+Open3.popen3('file -f -'){|cin,cout,cerr|
+  cin.puts *binfiles
+  cin.flush
+  cin.close
+  cout.each{|line|
+    name=line[/\A([^:]+):.*ruby.*text/i,1] and rubyscripts<<name and p name
+  }
+}
+puts rubyscripts
+END
+
+#cd `dirname -- $0`
+
+for i in test/data/p.rb `(locate tk.rb;locate examples/examples_test.rb ron.rb /generator.rb ipaddr.rb date/format.rb /optparse.rb ferret/browser.rb;locate .rb; locate rakefile; locate Rakefile; locate RAKEFILE)|egrep -v '/test/results/'; cat test/results/rubyscripts.txt; find test/data/gems -name "*.rb"`; do
+  $RUBYLEXERVSRUBY $i
+done

data/test/code/locatetest.rb

@@ -0,0 +1,49 @@
+require 'test/code/rubylexervsruby'
+#ENV['RUBY']||='ruby'
+$RUBY=ENV['RUBY']||'ruby'
+#test $RUBY || export RUBY=ruby
+
+#$RUBYLEXERVSRUBY="#$RUBY test/code/rubylexervsruby.rb"
+
+RUBY_VERSION[/^1\.[0-7]\./] and raise 'need ruby>= 1.8'
+
+
+
+#if RUBY_VERSION --version|grep '^ruby 1\.6'; then
+#  echo 'error: need ruby 1.8'; exit
+#fi
+
+
+RLROOT= (File.dirname $0)+'/../..'
+#cd `dirname -- $0`
+
+=begin if locate fails, we should use the algorithm from this sh code
+
+#also look in bin and lib directories
+file -L `echo $PATH":/sbin:/usr/sbin"|tr : "\n"|sort -u|xargs -i echo "{}/*"`| \
+ grep "ruby[^:]*script"|cut -d: -f1 > test/results/rubyexelibs
+
+ruby -e 'print ($:.sort.uniq+[""]).join"\n"'|xargs -i ls "{}/*.rb" >> test/results/rubyexelibs
+
+   for i in `cat test/results/rubyexelibs`; do
+      $RUBYLEXERVSRUBY $i;
+   done
+
+=end
+
+for i in [
+  RLROOT+"/test/data/p.rb", *Dir["test/data/*.rb"]+`(locate /tk.rb;
+  locate examples/examples_test.rb;locate .rb; locate rakefile; 
+  locate Rakefile; locate RAKEFILE)|egrep -v '/test/(results|data)/'`.
+    split("\n")
+] do
+#  system $RUBYLEXERVSRUBY, i
+  #hmm, rubylexervsruby needs to be upgraded to not regard an output
+  #consisting entirely of warnings as a failure.
+  #if no 'warning' (in any capitalization) for 4 or more lines
+  RubyLexerVsRuby.rubylexervsruby i #or fail "failed in #{i}"
+end
+
+#for i in test/data/p.rb `(locate /tk.rb;locate examples/examples_test.rb;#locate .rb; locate rakefile; locate Rakefile; locate RAKEFILE)|egrep -v '/#test/results/'`; do
+#  $RUBYLEXERVSRUBY $i
+#done

data/test/code/rubylexervsruby.rb

@@ -0,0 +1,173 @@
+#!/usr/bin/ruby
+#$DEBUG=$VERBOSE=true
+$Debug=true
+require "getoptlong"
+require "test/code/tokentest"
+require "test/code/deletewarns"
+
+
+module RubyLexerVsRuby;end
+class<<RubyLexerVsRuby
+ENABLEMD5=false
+def nop_ruby(cmd,input,output,stringdata)
+#   system %[echo "BEGIN{exit};">#{output}]
+   File.open(output,'w'){|f| f.write "BEGIN{exit};\n" }
+   if stringdata 
+     File.open(output,'a'){|f| f.write stringdata }     
+   else
+     system [cmd,'"'+input+'"','>>',output].join(' ')
+   end
+end
+
+def ruby_parsedump(input,output,ruby)
+  #todo: use ruby's md5 lib
+  #recursive ruby call here is unavoidable because -y flag has to be set
+
+  #do nothing if input unchanged
+  ENABLEMD5 and system "md5sum -c #{input}.md5 2>/dev/null" and return
+
+  status=0
+  IO.popen("#{ruby} -w -y < #{input} 2>&1"){ |pipe| 
+    File.open(output,"w") { |outfd|
+      pipe.each{ |line|
+        outfd.print(line) \
+          if /^Shifting|^#{DeleteWarns::WARNERRREX}/o===line
+        #elsif /(warning|error)/i===line
+        #  raise("a warning or error, appearently, not caught by rex above: "+line)
+      }
+      pid,status=Process.waitpid2 pipe.pid #get err status of subprocess
+    } 
+  }
+  ENABLEMD5 and status==0 and system "md5sum #{input} > #{input}.md5" #compute sum only if no errors
+end
+
+def head(fname)
+  print "foobaaaaaaaaar\n\n\n\n\n\n"
+  File.open(fname){|fd| print(fd.read(512)+"\n") }
+end
+
+def rubylexervsruby(input,stringdata=nil,&ignore_it)
+
+#cmdpath= `which #$0`
+cmddir=Dir.getwd+"/test/code/"
+base='test/results/'+File.basename(input)
+_ttfile=base+'.tt'
+mttfile=base+'.mtt'
+p_ttfile=_ttfile+'.prs'
+pmttfile=mttfile+'.prs'
+p_ttdiff=p_ttfile+'.diff'
+pmttdiff=pmttfile+'.diff'
+nopfile=base+'.nop'
+origfile=nopfile+'.prs'
+ruby=ENV['RUBY'] || 'ruby'
+expected_failures=Dir.getwd+"/test/code/"+File.basename(input)+".expected_failures"
+
+#olddir=Dir.pwd
+#Dir.chdir cmddir + '/../..'
+
+nop_ruby "#{input[/\.gz$/]&&'z'}cat", input, nopfile, stringdata
+
+print "executing: #{ruby} -Ilib test/code/tokentest.rb --keepws #{input}\n"
+
+ruby_parsedump nopfile, origfile, ruby
+
+
+
+tokentest nopfile, RubyLexer, RubyLexer::KeepWsTokenPrinter.new, nil, _ttfile
+tokentest nopfile, RubyLexer, RubyLexer::KeepWsTokenPrinter.new(' '), nil, mttfile
+
+
+ruby_parsedump _ttfile, p_ttfile, ruby
+ruby_parsedump mttfile, pmttfile, ruby
+
+if File.exists?(p_ttfile)
+  IO.popen("diff -u1 -b #{origfile} #{p_ttfile}"){ |pipe|
+  File.open(p_ttdiff,"w") { |diff|
+    DeleteWarns.deletewarns(pipe){|s| diff.print s}
+  }
+  }
+#  File.unlink p_ttfile
+end
+
+if File.exists?(pmttfile)
+  IO.popen("diff -u1 -b #{origfile} #{pmttfile}"){ |pipe|
+  File.open(pmttdiff,"w") { |diff|
+    DeleteWarns.deletewarns(pipe){|s| diff.print s}
+  }
+  }
+#  File.unlink pmttfile
+end
+
+list=[]
+#nonwarn4=/(^(?![^\n]*warning[^\n]*)[^\n]*\n){4,}/im
+#4 or more non-warning lines:
+nonwarn4=/^(?:(?![^\r\n]*warning)[^\r\n]+(?:\r\n?|\n\r?)){4,}/mi
+result=true
+for name in [p_ttdiff,pmttdiff] do
+  i=File.read(name)
+ # i.tr("\r","\n")
+#  i.gsub!(/^\n/m, '')
+  i.sub!(/\A([^\r\n]+(\r\n?|\n\r?)){2}/, '')  #remove 1st 2 lines
+  i.scan nonwarn4 do |j|
+    unless ignore_it && ignore_it[j]
+      list.push( *j.split(/\r\n?|\n\r?/) ) #unless list.size>=10
+    end
+  end
+  
+  unless list.empty?
+  list=list.join("\n") +"\n"
+  unless (File.exists?(expected_failures) and File.read(expected_failures))==list
+    print list
+    result=false
+  end
+    list=[]
+  end
+end
+
+#print( list.join("\n") +"\n")
+#Dir.chdir olddir
+return result
+
+=begin
+case File.zero?(p_ttdiff).to_s +
+     File.zero?(pmttdiff).to_s
+  when 'falsefalse' then
+    head p_ttdiff
+    print "omitting #{pmttdiff}\n"
+  when 'falsetrue'
+    head p_ttdiff
+  when 'truefalse'
+    head pmttdiff
+  when 'truetrue'
+    #File.unlink origfile
+    return true
+  default
+    raise "unexpected 2bool val"
+end
+return false
+=end
+
+rescue Exception
+  system "ruby -c #{input} >/dev/null 2>&1" or expected="(expected) "
+  print "#{expected}error in: #{input}\n"
+  raise unless expected
+end
+end
+
+if __FILE__==$0
+#allow -e
+stringdata=input=nil
+opts=GetoptLong.new(["--eval", "-e", GetoptLong::REQUIRED_ARGUMENT])
+opts.each{|opt,arg|
+  opt=='--eval' or raise :impossible
+  stringdata=arg
+  input='-e'
+}
+
+input||=ARGV[0]
+RubyLexerVsRuby.rubylexervsruby(input,stringdata) and exit 0
+
+exit 1
+end
+
+

data/{testcode → test/code}/tokentest.rb

@@ -1,24 +1,31 @@
-#!/usr/bin/ruby -dw
+#!/usr/bin/ruby
+$Debug=true
 require "rubylexer"
-#  require "rumalexer"
-require "token"
-require "tokenprinter"
 require "getoptlong"
 require "pp"
 
+class RubyLexer
 class Token
   def verify_offset(fd); false end
   
   def check_for_error; end
 end
 
+class LexerError<Exception; end
+
 module ErrorToken
-  def check_for_error; raise @error end
+  def check_for_error; raise LexerError,@error end
 end
 
 class FileAndLineToken
   def verify_offset(fd); true  end
 end
+class ImplicitParamListStartToken
+  def verify_offset(fd); true  end
+end
+class ImplicitParamListEndToken
+  def verify_offset(fd); true  end
+end
 
 module SimpleVerify
   def verify_offset(fd)
@@ -33,8 +40,13 @@ class MethNameToken; include SimpleVerify; end
 
 class SymbolToken
   def verify_offset(fd)
-    readsym=fd.read(@ident.length)
-    @ident[1]==?" or @ident[1]==?' or readsym==@ident
+    la=fd.read(2)
+    case la
+      when '%s': #stay right here
+      when /^:/: fd.pos-=1
+      else raise 'unrecognized symbol type'
+    end
+    @ident[1]==?" or @ident[1]==?' or fd.read(@ident.length-1)==@ident[1..-1]
 
   end
 end
@@ -42,7 +54,7 @@ end
 class EoiToken
   def verify_offset(fd)
     result=super(fd)
-    fd.pos=fd.stat.size
+    fd.eof?
     return result
   end
 end
@@ -66,12 +78,12 @@ end
 class HerePlaceholderToken
   def verify_offset(fd)
     '<<'==fd.read(2) or return false
-    @dash and (?-==fd.getc or return false)
-    case ch=fd.getc
+    @dash and (?-==fd.read1 or return false)
+    case ch=fd.read1
       when ?', ?`, ?"
-        @quote==ch.chr or return false
-        fd.read(@ender.size)==@ender or return false
-        return fd.getc.chr==@quote
+        @quote==ch.chr and
+        fd.read(@ender.size)==@ender and
+        return fd.read(1)==@quote
       when ?a..?z, ?A..?Z, ?_, ?0..?9
         @quote=='"' or return false
         fd.pos-=1
@@ -93,7 +105,7 @@ class StringToken
 
   def verify_subtoken_offsets(fd)
     #verify offsets of subtokens
-    1.step(@elems.length-1,2) { |i| @elems[i].verify_offset(fd) }
+    1.step(@elems.length-1,2) { |i| @elems[i].verify_offset(fd) or raise LexerError}
     return true
   end
   
@@ -137,28 +149,28 @@ end
 #    $ShowImplicit ? explicit_form : super
 #  end
 #end
-
-class RuLexer
-  def check_offset(tok,file=@file)
-    endpos=(@moretokens.empty?)? file.pos : @moretokens[0].offset
-    super(tok,file,endpos)
-  end
 end
+public
 
-def check_offset(tok,file,endpos)
-    oldpos=file.pos
+def check_offset(tok,file=nil,endpos=nil)
+  file||=@file
+  endpos||=(@moretokens.empty?)? file.pos : @moretokens[0].offset
+  oldpos=file.pos
 
-    assert Integer===tok.offset
-    assert Integer===endpos
-    assert endpos>=tok.offset
+  assert Integer===tok.offset
+  assert Integer===endpos
+  endpos>=tok.offset or 
+    raise RubyLexer::LexerError, "expected >=#{tok.offset}, got #{endpos}, "\
+                      "token #{tok}:#{tok.class}"
 
-    file.pos=tok.offset
-    assert tok.verify_offset(file)
-    case tok
-      when StringToken,NumberToken,HereBodyToken
-      else assert(file.pos==endpos)
-    end
-    file.pos=oldpos
+  file.pos=tok.offset
+  tok.verify_offset(file) or raise RubyLexer::LexerError, "couldn't check offset of token #{tok.class}: #{tok}"
+  case tok
+    when RubyLexer::StringToken,RubyLexer::NumberToken,
+         RubyLexer::HereBodyToken,RubyLexer::SymbolToken: #do nothing
+    else (file.pos==endpos) or raise RubyLexer::LexerError, "positions don't line up, expected #{endpos}, got #{file.pos}, token: #{tok}"
+  end
+  file.pos=oldpos
 end
 
 
@@ -166,34 +178,33 @@ end
 
 
 
-def tokentest(name,lexertype,pprinter,input=File.open(name),output=nil)
+def tokentest(name,lexertype,pprinter,input=File.open(name),output=$stdout)
   input ||= File.open(name)
-  if output
-    old_stdout=$stdout
-    $stdout=File.open(output,'w')
+  if output!=$stdout
+    output=File.open(output,'w')
   end
-  File.open(name) {|fd|
+
+  fd=input
+  #File.open(name) {|fd|
     lxr=lexertype.new(name,fd,1)
 
     begin
       tok=lxr.get1token
       lxr.check_offset(tok)
       tok.check_for_error
-      pprinter.pprint(tok)
-    end until EoiToken===tok
+      pprinter.pprint(tok,output)
+    end until RubyLexer::EoiToken===tok
 
     #hack for SimpleTokenPrinter....
-    print "\n" if NewlineToken===lxr.last_operative_token and
-                  SimpleTokenPrinter===pprinter
+    print "\n" if RubyLexer::NewlineToken===lxr.last_operative_token and
+                  RubyLexer::SimpleTokenPrinter===pprinter
 
 #    unless lxr.balanced_braces? 
 #      raise "unbalanced braces at eof"
 #    end
-  }
-  if output
-    $stdout.close
-    $stdout=old_stdout
-  end
+  #}
+   output.close unless output==$stdout
+
 end
 
 #$ShowImplicit=false
@@ -203,7 +214,7 @@ if __FILE__==$0
 #  lexertype= RumaLexer if defined? RumaLexer
   lexertype=RubyLexer
   insertnils=fd=name=nil
-  pprinter=SimpleTokenPrinter
+  pprinter=RubyLexer::SimpleTokenPrinter
 
   opts=GetoptLong.new \
     ["--eval","-e", GetoptLong::REQUIRED_ARGUMENT],
@@ -213,15 +224,15 @@ if __FILE__==$0
     ["--implicit","-i", GetoptLong::NO_ARGUMENT],
     ["--implicit-all", GetoptLong::NO_ARGUMENT]
 
+  saweval=nil
   opts.each do|opt,arg|
     case opt
     when '--eval'   then 
-       pprinter =pprinter.new(sep,line,showzw)
-       tokentest('-e',lexertype,pprinter,arg)
+       tokentest('-e',lexertype,pprinter.new(sep,line,showzw),arg)
        saweval=true
 #    when '--ruby'   then lexertype=RubyLexer
-    when '--keepws' then pprinter= KeepWsTokenPrinter
-    when '--maxws'  then pprinter= KeepWsTokenPrinter;sep=' '
+    when '--keepws' then pprinter= RubyLexer::KeepWsTokenPrinter
+    when '--maxws'  then pprinter= RubyLexer::KeepWsTokenPrinter;sep=' '
     when '--implicit' then showzw=1
     when '--implicit-all' then showzw=2
     else raise :impossible