rubylexer 0.7.0 → 0.7.1

data/test/code/strgen.rb

@@ -0,0 +1,138 @@
+module Strgen
+  PAIRS=[
+    ['<','>'],
+    ['(',')'],
+    ['[',']'],
+    ['{','}']
+  ]
+  ALLOWED_UNNESTING_FANCY=/[^<>\[\]{}()a-z0-9_]/i
+  FANCY_TYPES=%w[q Q r s x w W]<<''
+  SIMPLE_QUOTES=%w[" ' / `]
+  SIMPLE_ESCAPES=%w[s n r t v f a b e]
+  MULTI_ESCAPES=%w[x c C M 0 1 2 3 4 5 6 7]
+  NON_ESCAPES=/[^#{SIMPLE_ESCAPES+MULTI_ESCAPES}]/
+
+  def Strgen.rand_char_including(allow,disallow='')
+    q=nil
+    q=rand(255).chr until ((allow===q) and not (disallow[q]))
+    q
+  end
+
+  def Strgen.rand_esc_seq(disallow,bsonly)
+    limit=4
+    bsonly[/\\/] and disallow+='\\'
+    (disallow['#'] or bsonly['#']) and limit=3
+    choice=rand limit
+    choice=3 if disallow[/\\/] or bsonly[/\#/]
+    case choice
+    when 0: "\\"+rand_char_including(NON_ESCAPES,disallow)
+    when 1: "\\"+SIMPLE_ESCAPES[rand(SIMPLE_ESCAPES.size)]
+    when 2:
+      "\\"+
+      case ch=MULTI_ESCAPES[rand(MULTI_ESCAPES.size)]
+      when "x": "x"+rand(256).to_s(16)
+      when "0".."7": rand(256).to_s(8)
+      when "c": 
+        "c"+
+        if rand(2).zero?
+          rand_char_including(/[^\\]/,disallow+bsonly)
+        else
+          rand_esc_seq disallow+"#",bsonly
+        end
+      when "C","M":
+        return rand_esc_seq(disallow,bsonly) if disallow['-'] or bsonly['-']
+        ch+"-"+
+        if rand(2).zero?
+          rand_char_including(/[^\\]/,disallow+bsonly)
+        else
+          rand_esc_seq disallow+"#",bsonly
+        end
+      end
+    when 3:
+      '#{'+rand(9999999999).to_s+'}'
+    end
+  end
+  
+  CACHE={}
+
+  def Strgen.strgen
+    must_be_escaped="#\\"
+    case rand(3)
+    when 0
+      starter=ender=SIMPLE_QUOTES[rand(SIMPLE_QUOTES.size)]
+      must_be_escaped<<starter
+    when 1
+      type=FANCY_TYPES[rand(FANCY_TYPES.size)]
+      pair=PAIRS[rand(PAIRS.size)]
+      starter= "%"+type+pair[0]
+      ender= pair[1]
+      must_be_escaped<<pair.to_s
+    when 2
+      type=FANCY_TYPES[rand(FANCY_TYPES.size)]
+      q=rand_char_including ALLOWED_UNNESTING_FANCY
+      /w/i===type and /\s|\v/===q and q='"'
+      starter= "%"+type+q
+      ender=q
+      must_be_escaped<<q
+    end
+  
+    if starter=="/" or type=='r'
+      must_be_escaped+="[]{}()?+*"
+    end
+    must_be_escaped+="\0" if type=='s'
+    ckey=must_be_escaped
+    ordinary=
+      CACHE[ckey]||=
+        /[^#{must_be_escaped.gsub(/./){"\\"+$&}}]/
+
+    interior=(1..rand(40)).map{|x|
+      rand_char_including ordinary
+    }.to_s
+
+    interior["\\"] and fail
+
+    disallow=''
+    bsonly=starter[-1,1]+ender
+#    disallow+='#' if /[\#\\\-]/===starter[-1,1]
+#    disallow+=starter[-1,1]+ender if type=='r' or starter=='/'
+    disallow+="\0" if type=='s'
+    disallow+=must_be_escaped.gsub('\\','') if type=='r' or starter=='/'
+
+    poslimit=interior.size+1
+    rand(5).times{
+      pos=rand poslimit
+      interior[pos,0]=rand_esc_seq disallow,bsonly
+      poslimit=pos
+    } unless starter[-1]==?\\
+
+    interior.gsub!(/\\[a-z]/i,'') if type=='r' or starter=='/'
+
+    starter[-1]==?\r and interior.gsub!(/\A\n+/,'')
+
+    starter[1]==?s and interior=='' and interior="x"
+
+    result= starter+interior+ender
+
+    begin
+      begin eval "BEGIN{break};proc() do #{result} end" end while false
+    rescue Exception
+      #puts %<failing string: eval "#{result.gsub(/./){'\\x'+$&[0].to_s(16)}}">
+      return strgen
+    end
+
+    return result
+  end
+end
+
+if __FILE__==$0
+start=Time.now
+i=0
+10_000_000.times{|i|
+  begin
+    ss=Strgen.strgen
+    RubyLexerVsRuby.rubylexervsruby "-e#{i}", ss
+  rescue Exception
+    puts %<failing string: eval "#{ss.gsub(/./){'\\x'+$&[0].to_s(16)}}">
+  end
+}
+end

data/test/code/tarball.rb

@@ -0,0 +1,144 @@
+require 'open-uri'
+
+class Tarball
+PROTOCOLS=%w[http https ftp]
+EXTENSIONS=%w[tar zip rb tgz tbz2 tbz gem]
+EXTRA_EXTENSIONS=%w[gz bz2 Z]
+EXTRA_EXTENSIONS_REX="\\.(?:#{EXTRA_EXTENSIONS.join'|'})"
+VERSIONTOO='' #was: "(?:[_-](.*))"
+ENDINGS="\\.(?:#{EXTENSIONS.join('|')})(?:#{EXTRA_EXTENSIONS_REX})?"
+TARBALL=%r<
+  \A(?:#{PROTOCOLS.join('|')})://
+  (?:[^/]+/)+
+  (.*)
+  #{VERSIONTOO}
+  #{ENDINGS}
+  \Z
+>ixo
+
+def Tarball.unpack1gem(gem)
+  dir=gem.dup
+  dir[/\.gem$/]=''
+  Dir.mkdir dir rescue nil
+  
+  system "tar x -f #{gem} -C #{dir}"
+  files_in_dir=Dir[dir+"/*"]-[dir+"/data.tar.gz",dir+"/metadata.gz"]
+  files_in_dir.empty? or puts "gem archive toplevel contains extra files: #{files_in_dir.join(' ')}"
+  system "gunzip -f #{dir}/metadata.gz -c > #{dir}/metadata"
+  system "tar xz -f #{dir}/data.tar.gz -C #{dir}"
+
+
+  (File.unlink gem rescue nil) if File.exist? dir+"/metadata.gz" and File.exist? dir+"/data.tar.gz"
+  (File.unlink dir+"/metadata.gz" rescue nil) if File.exist? dir+"/metadata"
+  files_in_dir=Dir[dir+"/*"]-[dir+"/data.tar.gz",dir+"/metadata.gz",dir+"/metadata"]-files_in_dir
+  (File.unlink dir+"/data.tar.gz" rescue nil) unless files_in_dir.empty?
+end
+
+def Tarball.dl_and_unpack(cachedir,url)
+  projectname=url[%r{[^/]+\Z}]
+  localname=cachedir+projectname
+  projectname.sub!(/#{ENDINGS}\Z/o,'')+"/"
+  localdir= cachedir+projectname
+  #localdir=localname.sub(/#{ENDINGS}\Z/o,'')+"/"
+
+  if File.exist? localdir 
+   puts "skipping already extant #{localdir}"
+   return
+  end
+  begin
+  open(localname,"w"){|disk|
+    open(url){|net|
+      while buf=net.read(40960)
+        disk.write buf
+      end
+    }
+  }
+  rescue Interrupt=>e
+    File.unlink localname rescue nil
+    raise if e.class==Interrupt
+    return
+  rescue Exception
+    File.unlink localname rescue nil
+    return
+  end
+
+  unpack(cachedir,localname,projectname,localdir)
+end
+
+def Tarball.unpack(cachedir,localname,
+                   projectname=localname[/\/(.*)#{ENDINGS}\Z/,1],
+                   localdir=localname.sub(/#{ENDINGS}\Z/o,'')+"/")
+  cachedir[-1]==?/ or fail
+  localname[0,cachedir.size]==cachedir or fail
+
+  if File.exist? localdir 
+   puts "skipping already extant #{localdir}"
+   return
+  end
+
+  #rename .tgz,.tbz2? to the full form
+  case localname
+  when /\.tgz\Z/: 
+    oldln=localname
+    localname=localname[0...-4]+".tar.gz"
+  when /\.tbz2?\Z/:
+    oldln=localname
+    localname=localname[0...-$&.size]+".tar.bz2"
+  end
+  File.rename oldln, localname if oldln
+
+  #remove any gz or bz2 whole-archive compression
+  case localname
+  when /\.bz2\Z/: 
+    system "bunzip2 "+localname or return
+    localname=localname[0...-$&.size]
+  when /\.(gz|Z)\Z/: 
+    system "gunzip -f "+localname or return
+    localname=localname[0...-$&.size]
+  end
+
+  #now actually unpack the archive
+  case localname
+  when /\.rb\Z/: 
+    Dir.mkdir localdir
+    File.rename localname, localdir+localname[%r{[^/]+\Z}]
+  when /\.gem\Z/: unpack1gem localname
+  when /\.zip\Z/:
+   filelist=`unzip -L -l #{localname}`
+   if $?>>8 > 1 
+     puts "invalid zip file #{localname}"
+     return
+   end
+   filelist=filelist.split("\n")[3...-2]
+   wellformed=!filelist.find{|entry|
+     entry[/\A\s*[^\s]+\s+[^\s]+\s+[^\s]+\s+\^?(.*)\Z/,1][0...projectname.size] != projectname
+   } 
+   if wellformed 
+     zipopts=" -d #{cachedir}"
+   else
+     zipopts=" -d #{localdir}"
+     Dir.mkdir localdir
+   end
+   puts "unzip -L #{localname} #{zipopts}"
+   system "unzip -L #{localname} #{zipopts}"
+   (File.unlink localname rescue nil) if $?>>8 <= 1
+
+  when /\.tar\Z/:
+   wellformed=!`tar tf #{localname}`.split("\n").find{|entry|
+     entry[0...projectname.size] != projectname
+   }
+   if wellformed
+     taropts=" -C #{cachedir}"
+   else
+     taropts=" -C #{localdir}"
+     Dir.mkdir localdir
+   end
+   system "tar xf #{localname} #{taropts}"
+   File.unlink localname rescue nil
+
+  else fail "unknown tarball type: #{localname}"
+  end
+end
+
+end
+

data/test/code/testcases.rb

@@ -0,0 +1,11 @@
+module TestCases
+#  fail unless File.exist 'test/data/oneliners.rb' and  File.exist 'test/data/stanzas.rb'
+  rldir=$:.find{|dir| File.exist? dir+'/test/data/oneliners.rb' and  File.exist? dir+'/test/data/stanzas.rb' }
+  ONELINERS=IO.readlines(rldir+'/test/data/oneliners.rb').map{|x| x.chomp}.grep(/\A\s*[^#\s\n]/).reverse
+  STANZAS=IO.read(rldir+'/test/data/stanzas.rb').split("\n\n").grep(/./).reverse
+  STANZAS.each{|stanza| stanza<<"\n" }
+  ILLEGAL_ONELINERS=IO.readlines(rldir+'/test/data/illegal_oneliners.rb').map{|x| x.chomp}.grep(/\A\s*[^#\s\n]/).reverse
+  ILLEGAL_STANZAS=IO.read(rldir+'/test/data/illegal_stanzas.rb').split("\n\n").grep(/./).reverse
+  TESTCASES=ONELINERS+STANZAS
+  ILLEGAL_TESTCASES=ILLEGAL_ONELINERS+ILLEGAL_STANZAS
+end

data/test/code/tokentest.rb

@@ -1,4 +1,22 @@
 #!/usr/bin/ruby
+=begin legal crap
+    rubylexer - a ruby lexer written in ruby
+    Copyright (C) 2004,2005,2008  Caleb Clausen
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation; either
+    version 2.1 of the License, or (at your option) any later version.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+=end
 $Debug=true
 require "rubylexer"
 require "getoptlong"
@@ -34,24 +52,52 @@ module SimpleVerify
 end
 
 class WToken;      include SimpleVerify; end
-class NewlineToken;     include SimpleVerify; end
 class IgnoreToken; include SimpleVerify; end
 class MethNameToken; include SimpleVerify; end
 
+class NewlineToken
+  include SimpleVerify
+  def verify_offset(fd)
+    super or fd.eof?
+  end
+end
+
 class SymbolToken
   def verify_offset(fd)
     la=fd.read(2)
     case la
-      when '%s': #stay right here
-      when /^:/: fd.pos-=1
+      when '%s': 
+        quote=fd.read(1)
+        ender=RubyLexer::PAIRS[quote] || quote
+        body=@ident[2...-1]
+      when /^:(['"])/: 
+        #stay right here
+        quote=ender=$1
+        body=@ident[2...-1]
+      when /^:/: 
+        fd.pos-=1
+        body=@ident[1..-1]
       else raise 'unrecognized symbol type'
     end
-    @ident[1]==?" or @ident[1]==?' or fd.read(@ident.length-1)==@ident[1..-1]
+    
+    bodyread=fd.read(body.length)
+
+    #punt if its too hard
+    if quote
+      bs="\\"
+      hardstuff= /[#{bs}#{quote}#{bs}#{ender}\#\\]/
+      return true if (body+bodyread).match(hardstuff)
+    end
 
+    if bodyread==body
+      return fd.read(1)==ender if ender 
+      return true 
+    end
   end
 end
 
 class EoiToken
+  include SimpleVerify
   def verify_offset(fd)
     result=super(fd)
     fd.eof?
@@ -78,8 +124,8 @@ end
 class HerePlaceholderToken
   def verify_offset(fd)
     '<<'==fd.read(2) or return false
-    @dash and (?-==fd.read1 or return false)
-    case ch=fd.read1
+    @dash and ('-'==fd.read(1) or return false)
+    case ch=fd.read(1)[0]
       when ?', ?`, ?"
         @quote==ch.chr and
         fd.read(@ender.size)==@ender and
@@ -98,14 +144,38 @@ class StringToken
   FANCY_QUOTE_BEGINNINGS= {'`'=>'%x', '['=>'%w', '{'=>'%W',
                            '"'=>/('|%[^a-pr-z0-9])/i, '/'=>'%r'}
   def verify_offset(fd)
-    str=fd.read(2)
-    @char==str[0,1] or FANCY_QUOTE_BEGINNINGS[@char]===str or return false
+    fd.read(open.size)==open  or return false
+#    str=fd.read(2)
+#    @char==str[0,1] or FANCY_QUOTE_BEGINNINGS[@char]===str or return false
     verify_subtoken_offsets(fd)
   end
 
   def verify_subtoken_offsets(fd)
     #verify offsets of subtokens
-    1.step(@elems.length-1,2) { |i| @elems[i].verify_offset(fd) or raise LexerError}
+    @elems.each{|elem|
+      case elem
+      when String: 
+        #get string data to compare against,
+        #translating dos newlines to unix.
+        #(buffer mgt is a PITA)
+        goal=elem.size
+        saw=fd.read(goal)
+        saw.gsub!("\r\n","\n")
+        now_at=nil
+        loop do
+          now_at=saw.size
+          saw.chomp!("\r") and fd.pos-=1 and now_at-=1
+          break if now_at>=goal
+          more=fd.read([goal-now_at,2].max)
+          more.gsub!("\r\n","\n")
+          saw<<more
+        end
+        #assert now_at<=goal+1 #not needed
+        saw[goal..-1]='' unless goal==now_at
+        saw==elem  or return false
+      else elem.verify_offset(fd) or raise LexerError
+      end
+    }
     return true
   end
   
@@ -127,7 +197,7 @@ class RubyCode
     }
     assert nexttok.nil?
     assert thistok.object_id==@ident.last.object_id
-    assert WToken===thistok
+    assert(( WToken===thistok or EoiToken===thistok&&thistok.error ))
     fd.pos=endpos
   end
   
@@ -150,40 +220,52 @@ end
 #  end
 #end
 end
+
 public
 
+
 def check_offset(tok,file=nil,endpos=nil)
-  file||=@file
-  endpos||=(@moretokens.empty?)? file.pos : @moretokens[0].offset
+  #the errors detected here are now reduced to warnings....
+  file||=@original_file
+  String===file and file=file.to_sequence
+  allow_ooo= @moretokens&&@moretokens[0]&&@moretokens[0].allow_ooo_offset unless endpos
+  endpos||=((@moretokens.empty?)? input_position : @moretokens[0].offset)
   oldpos=file.pos
 
   assert Integer===tok.offset
   assert Integer===endpos
-  endpos>=tok.offset or 
-    raise RubyLexer::LexerError, "expected >=#{tok.offset}, got #{endpos}, "\
-                      "token #{tok}:#{tok.class}"
+  if endpos<tok.offset and !allow_ooo
+    $stderr.puts "expected #{endpos} to be >= #{tok.offset} token #{tok.to_s.gsub("\n","\n  ")}:#{tok.class}"
+  end
 
   file.pos=tok.offset
-  tok.verify_offset(file) or raise RubyLexer::LexerError, "couldn't check offset of token #{tok.class}: #{tok}"
+  tok.verify_offset(file) or 
+     $stderr.puts "couldn't check offset of token #{tok.class}: #{tok.to_s.gsub("\n","\n  ")} at #{tok.offset}"
   case tok
     when RubyLexer::StringToken,RubyLexer::NumberToken,
-         RubyLexer::HereBodyToken,RubyLexer::SymbolToken: #do nothing
-    else (file.pos==endpos) or raise RubyLexer::LexerError, "positions don't line up, expected #{endpos}, got #{file.pos}, token: #{tok}"
+         RubyLexer::HereBodyToken,RubyLexer::SymbolToken,
+         RubyLexer::HerePlaceholderToken,
+         RubyLexer::FileAndLineToken: #do nothing
+    else 
+      file.pos==endpos or allow_ooo or 
+        $stderr.puts "positions don't line up, expected #{endpos}, got #{file.pos}, token: #{tok.to_s.gsub("\n","\n  ") }"
   end
   file.pos=oldpos
+  return
 end
 
 
 
 
 
-
 def tokentest(name,lexertype,pprinter,input=File.open(name),output=$stdout)
   input ||= File.open(name)
   if output!=$stdout
     output=File.open(output,'w')
   end
 
+  input=input.read if IO===input and not File===input
+
   fd=input
   #File.open(name) {|fd|
     lxr=lexertype.new(name,fd,1)
@@ -213,7 +295,7 @@ if __FILE__==$0
   sep,line,showzw='',1,0
 #  lexertype= RumaLexer if defined? RumaLexer
   lexertype=RubyLexer
-  insertnils=fd=name=nil
+  insertnils=fd=name=loop=nil
   pprinter=RubyLexer::SimpleTokenPrinter
 
   opts=GetoptLong.new \
@@ -222,27 +304,36 @@ if __FILE__==$0
     ["--keepws","-k", GetoptLong::NO_ARGUMENT],
     ["--maxws","-m", GetoptLong::NO_ARGUMENT],
     ["--implicit","-i", GetoptLong::NO_ARGUMENT],
-    ["--implicit-all", GetoptLong::NO_ARGUMENT]
+    ["--implicit-all", GetoptLong::NO_ARGUMENT],
+    ["--loop", GetoptLong::NO_ARGUMENT]
 
   saweval=nil
   opts.each do|opt,arg|
     case opt
     when '--eval'   then 
        tokentest('-e',lexertype,pprinter.new(sep,line,showzw),arg)
-       saweval=true
+       saweval=arg
 #    when '--ruby'   then lexertype=RubyLexer
     when '--keepws' then pprinter= RubyLexer::KeepWsTokenPrinter
     when '--maxws'  then pprinter= RubyLexer::KeepWsTokenPrinter;sep=' '
     when '--implicit' then showzw=1
     when '--implicit-all' then showzw=2
+    when '--loop' then loop=true
     else raise :impossible
     end
   end
 
   pprinter =pprinter.new(sep,line,showzw)
   
-    ARGV.empty?   ?    saweval || tokentest('-',lexertype,pprinter,$stdin) : 
-        ARGV.each{|fn| tokentest(fn,lexertype,pprinter) } 
+  begin
+    if ARGV.empty?     
+      saweval ? 
+        tokentest('-e',lexertype,pprinter,saweval) : 
+        tokentest('-',lexertype,pprinter,$stdin) 
+    else
+      ARGV.each{|fn| tokentest(fn,lexertype,pprinter) } 
+    end
 #  ARGV.first[/[_.]rb$/i] and lexertype=RubyLexer  #filename with _rb are special hack
+  end while loop
 
 end