rubylexer 0.7.0 → 0.7.1

data/lib/rubylexer/version.rb

@@ -1,3 +1,3 @@
 class RubyLexer
-  VERSION='0.7.0'
+  VERSION='0.7.1'
 end

data/rubylexer.vpj

@@ -0,0 +1,98 @@
+<!DOCTYPE Project SYSTEM "http://www.slickedit.com/dtd/vse/10.0/vpj.dtd">
+<Project
+	Version="10.0"
+	VendorName="SlickEdit"
+	WorkingDir=".">
+	<Config
+		Name="Release"
+		OutputFile=""
+		CompilerConfigName="Latest Version">
+		<Menu>
+			<Target
+				Name="Compile"
+				MenuCaption="&amp;Compile"
+				CaptureOutputWith="ProcessBuffer"
+				SaveOption="SaveCurrent"
+				RunFromDir="%rw">
+				<Exec/>
+			</Target>
+			<Target
+				Name="Build"
+				MenuCaption="&amp;Build"
+				CaptureOutputWith="ProcessBuffer"
+				SaveOption="SaveWorkspaceFiles"
+				RunFromDir="%rw">
+				<Exec/>
+			</Target>
+			<Target
+				Name="Rebuild"
+				MenuCaption="&amp;Rebuild"
+				CaptureOutputWith="ProcessBuffer"
+				SaveOption="SaveWorkspaceFiles"
+				RunFromDir="%rw">
+				<Exec/>
+			</Target>
+			<Target
+				Name="Debug"
+				MenuCaption="&amp;Debug"
+				SaveOption="SaveNone"
+				RunFromDir="%rw">
+				<Exec/>
+			</Target>
+			<Target
+				Name="Execute"
+				MenuCaption="E&amp;xecute"
+				SaveOption="SaveNone"
+				RunFromDir="%rw">
+				<Exec CmdLine='".exe"'/>
+			</Target>
+		</Menu>
+	</Config>
+	<Files>
+		<Folder
+			Name="Source Files"
+			Filters="*.c;*.C;*.cc;*.cpp;*.cp;*.cxx;*.prg;*.pas;*.dpr;*.asm;*.s;*.bas;*.java;*.cs;*.sc;*.e;*.cob;*.html;*.rc;*.tcl;*.py;*.pl;*.rb">
+			<F N="lib/rubylexer/0.6.2.rb"/>
+			<F N="lib/rubylexer/0.6.rb"/>
+			<F N="lib/rubylexer/0.7.0.rb"/>
+			<F N="lib/rubylexer/0.7.1.rb"/>
+			<F N="test/code/all_the_gems.rb"/>
+			<F N="test/code/all_the_raas.rb"/>
+			<F N="test/code/all_the_rubies.rb"/>
+			<F N="lib/assert.rb"/>
+			<F N="lib/rubylexer/charhandler.rb"/>
+			<F N="lib/rubylexer/charset.rb"/>
+			<F N="lib/rubylexer/context.rb"/>
+			<F N="test/code/deletewarns.rb"/>
+			<F N="test/code/dumptokens.rb"/>
+			<F N="test/code/isolate_error.rb"/>
+			<F N="test/code/locatetest.rb"/>
+			<F N="test/code/regression.rb"/>
+			<F N="lib/rubylexer/rubycode.rb"/>
+			<F N="lib/rubylexer.rb"/>
+			<F N="test/code/rubylexervsruby.rb"/>
+			<F N="lib/rubylexer/rulexer.rb"/>
+			<F N="lib/rubylexer/symboltable.rb"/>
+			<F N="test/code/tarball.rb"/>
+			<F N="test/code/testcases.rb"/>
+			<F N="lib/rubylexer/token.rb"/>
+			<F N="lib/rubylexer/tokenprinter.rb"/>
+			<F N="test/code/tokentest.rb"/>
+			<F N="lib/rubylexer/version.rb"/>
+		</Folder>
+		<Folder
+			Name="Header Files"
+			Filters="*.h;*.H;*.hh;*.hpp;*.hxx;*.inc;*.sh;*.cpy;*.if"/>
+		<Folder
+			Name="Resource Files"
+			Filters="*.ico;*.cur;*.dlg"/>
+		<Folder
+			Name="Bitmaps"
+			Filters="*.bmp"/>
+		<Folder
+			Name="Other Files"
+			Filters="">
+			<F N="testing.txt"/>
+		</Folder>
+	</Files>
+</Project>

data/test/code/all_the_gems.rb

@@ -0,0 +1,33 @@
+#http://gems.rubyforge.org/yaml
+#http://gems.rubyforge.org/gems/#{name}-#{version}.gem
+
+require 'rubygems'
+require 'yaml'
+require 'open-uri'
+
+require "test/code/tarball"
+
+limit=(ENV['LIMIT']||20).to_i
+offset=(ENV['OFFSET']||0).to_i
+
+specs=open(ARGV.first||"http://gems.rubyforge.org/yaml"){|net| YAML.load net }
+
+name2vers={}
+specs.each{|bogus,spec| 
+  name2vers[spec.name]||=[]
+  name2vers[spec.name]<<spec.version
+}
+specs=nil
+
+name2vers.each_key{|name|
+  name2vers[name]=name2vers[name].max
+}
+#name2vers=name2vers.to_a[limit,offset]
+
+name2vers.each{|name,version|
+  begin
+    Tarball.dl_and_unpack("jewels/","http://gems.rubyforge.org/gems/#{name}-#{version}.gem")
+  rescue Interrupt: exit
+  rescue Exception: #do nothing
+  end
+}

data/test/code/all_the_raas.rb

@@ -0,0 +1,226 @@
+require 'rubygems'
+require 'hpricot'
+require 'open-uri'
+require 'algorithm/diff'
+require 'pathname'
+
+#require "test/code/all_the_gems"
+require "test/code/tarball"
+
+def fetch_with_cache(base,path,cachedir)
+
+path[-1]==?/ and path+='index.html'
+
+#make sure right dir structure for this path exists in cachedir
+dnames=Pathname.new(path)
+dirs=[]
+while dnames.to_s["/"]
+  dnames=dnames.dirname
+  dirs.unshift cachedir+dnames.to_s
+end
+dirs.each{|dir| Dir.mkdir dir rescue nil}
+
+#find latest cached version of this file in cachedir
+latest_fname=nil
+latest_date=Time.mktime '1970'
+Dir[cachedir+path+".*"].each{|fname| 
+  mtime=File.mtime(fname)
+  if mtime>=latest_date
+    latest_date=mtime
+    latest_fname=fname
+  end
+}
+
+options={"User-Agent"=>"all_the_raas.rb"}
+
+#extract etag from latest name
+if latest_fname
+  latest_etag=latest_fname[%r{\A#{Regexp.quote cachedir+path}\.(.*)\Z},1]
+  options['If-None-Match']=latest_etag unless latest_etag==''
+end
+
+#refetch the file if it has changed, otherwise use cached copy
+begin
+  open(base+path, options){|net|
+    latest_fname=cachedir+path+"."+(net.meta['etag']||'')
+    File.open(latest_fname,'w'){|f| f.write net.read }
+  }
+  #puts "fetched a fresh #{base+path}"
+rescue OpenURI::HTTPError=>e
+  raise e unless e.io.status.first=='304' and /Not Modified/i===e.io.status.last
+  #puts "reusing latest #{latest_fname}"
+end
+
+return File.open latest_fname
+end
+
+def changeratio(s1,s2)
+  diffslen=s1.diff(s2).inject(0){|sum,(msg,pos,data)| 
+    sum+data.size 
+  }
+  return diffslen.to_f/(s1.size+s2.size)
+end
+
+def sameproject?(s1,s2)
+   s1.casecmp(s2).zero? or 
+   s1.index(s2)==0 or 
+   s2.index(s1)==0 or
+   changeratio(s1,s2)<0.5
+end
+
+
+if __FILE__==$0
+base="http://raa.ruby-lang.org/"
+
+offset=(ENV['OFFSET']||0).to_i
+limit=(ENV['LIMIT']||20).to_i
+
+cachedir="jewels/"
+
+Dir.mkdir cachedir rescue nil
+
+#fetch list of all projects from raa's all.html
+all_raas=fetch_with_cache(base,"all.html",cachedir)
+tree=Hpricot(all_raas)
+tree/=:table
+tree.search(:thead).remove
+tree/=:tr
+#tree=tree[offset,limit]
+urls=tree.map{|row|
+  begin
+    row.search(:td).first.search(:a).first[:href] 
+  rescue Exception=>e
+    puts "failure #{e} in row #{row}" #wank about it
+    nil
+  end
+}
+
+RUBYFORGE=%r{\Ahttp://(?:[a-z0-9_+-:]+\.)*rubyforge.org/}i
+
+PROTOCOLS=%w[http https ftp]
+EXTENSIONS=%w[tar zip rb tgz tbz2 tbz gem]
+EXTRA_EXTENSIONS=%w[gz bz2 Z]
+EXTRA_EXTENSIONS_REX="\\.(?:#{EXTRA_EXTENSIONS.join'|'})"
+VERSIONTOO='' #was: "(?:[_-](.*))"
+ENDINGS="\\.(?:#{EXTENSIONS.join('|')})(?:#{EXTRA_EXTENSIONS_REX})?"
+TARBALL=%r<
+  \A(?:#{PROTOCOLS.join('|')})://
+  (?:[^/]+/)+
+  (.*)
+  #{VERSIONTOO}
+  #{ENDINGS}
+  \Z
+>ixo
+
+$rubyforge_urls=0 
+
+#pp urls
+#crawl raa's individual page for each project, looking for dl link
+urls.map!{|url|
+  begin
+     tree=Hpricot(fetch_with_cache(base,url,cachedir))
+#    if url
+#      url=base+url
+#      tree=Hpricot(open(url))
+#    end
+    project=tree.search(:title).inner_html[/\ARAA - (.*)\Z/,1]
+    tree/=:table
+    trs=tree/:tr
+    dl=trs.find{|tr| 
+      !tr.search("th[text()^='Download']").empty?
+    }
+    newurl=dl.search('td/a').first
+    if newurl
+      url=newurl[:href]
+      %r{\A(#{PROTOCOLS.join('|')})://}io===url or url="http://"+url
+      if RUBYFORGE===url
+        unless TARBALL===url and not /\.gem\Z/===url
+          url=nil
+          $rubyforge_urls += 1
+        end
+      end
+      url.slice!(/\#.*\Z/) if url #trim off url section
+      [url,project] 
+    else
+      puts "couldn't find td/a in #{base+url}"
+      [nil,nil]
+    end
+  rescue Interrupt=>e 
+    raise if e.class==Interrupt #^c only, dammit! #$^$&%'n Timeout::Error
+  rescue Exception=>e
+    puts "error: #{e} during url #{url}"
+    #wank about it
+    [nil,nil]
+  end 
+}
+
+
+#pp urls
+#resolve list of dl urls into urls to 'tarballs' (which is meant to include zip, gem, etc)
+#dl urls found on raa might be a direct link to a tarball, or point to a page that
+#points to the tarball
+urls.map!{|url,project|
+  if TARBALL===url
+    urlproject=$1
+    versionstart=urlproject.rindex(/[_-]/)
+    urlproject.slice! versionstart..-1 if versionstart
+    if sameproject?(urlproject,project)
+      url
+    else
+      puts "uh-oh, project #{project} not found in url #{url}. urlproject was #{urlproject}"
+    end
+      
+  elsif url
+    href=nil
+    begin
+      tree=Hpricot(open(url))
+      tree/=:a
+      tarball_url=tree.find{|a| 
+         href=a[:href] or next
+         unless %r[\A(?:#{PROTOCOLS.join('|')})://]o===href #relative url?
+#           url+="/" unless url[-1]==?/
+           href="/"+href unless href[0]==?/
+           href=url[%r{\A[^/]+//[^/]+}]+href
+         end
+         TARBALL===href 
+      }
+      if tarball_url
+          urlproject=$1
+          versionstart=urlproject.rindex(/[_-]/)
+          urlproject.slice! versionstart..-1 if versionstart
+          if sameproject?(urlproject,project)
+            href
+          else
+            puts "uh-oh, project #{project} not found in url #{url}. urlproject was #{urlproject}"
+          end
+      end
+
+    rescue Interrupt=>e
+      raise if e.class==Interrupt
+      puts "error: #{e} during page scan of url #{url}"
+    rescue Exception=>e
+      puts "error: #{e} during page scan of url #{url}"
+      nil
+    end
+  end
+}
+
+/unzip ([^\s]+)[\s\n]/i===`unzip -v`
+unzip="unzip"
+unzip+=" -L" if $1[0..2].to_f>=5.5
+
+pp urls
+#for each tarball url actually found, dl the tarball
+urls.compact.each{|url| 
+  begin
+    Tarball.dl_and_unpack(cachedir,url) 
+  rescue Interrupt=>e
+    raise if e.class==Interrupt
+    #else do nothing
+  rescue Exception
+    #do nothing
+  end
+}
+
+p [:$rubyforge_urls, $rubyforge_urls]
+end

data/test/code/all_the_rubies.rb

@@ -0,0 +1,2 @@
+load 'all_the_gems'
+load 'all_the_raas'

data/test/code/deletewarns.rb

@@ -1,11 +1,29 @@
 #!/usr/bin/env ruby
+=begin legal crap
+    rubylexer - a ruby lexer written in ruby
+    Copyright (C) 2004,2005,2008  Caleb Clausen
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation; either
+    version 2.1 of the License, or (at your option) any later version.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+=end
 
 module DeleteWarns
 FN='[^\n]+'
 DATETIME='\d+-\d\d?-\d\d? \d\d:\d\d:\d\d\.\d+ -?\d+'
 INDENTLINE='(?: [^\n]*\n)'
 
-WARNERRREX='(?:Reading a token: )?-:(\d+): (warning|(?:syntax )error)(?:: ([^\n]+))?'
+WARNERRREX='(?:Reading a token: )?-:(\d+): (warning|(?:syntax )?error)(?:: ([^\n]+))?'
 
 RE=%r"(?#--- #{FN}	#{DATETIME}
 \+\+\+ #{FN}	#{DATETIME}

data/test/code/dumptokens.rb

@@ -1,4 +1,23 @@
 #!/usr/bin/env ruby
+=begin legal crap
+    rubylexer - a ruby lexer written in ruby
+    Copyright (C) 2004,2005,2008  Caleb Clausen
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation; either
+    version 2.1 of the License, or (at your option) any later version.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+=end
+
 $Debug=true
 require 'rubylexer'
 require 'getoptlong'
@@ -16,14 +35,21 @@ class Token
 end
 end
 
-file=nil
+silent=file=nil
 
 #allow -e
-opts=GetoptLong.new(["--eval", "-e", GetoptLong::REQUIRED_ARGUMENT])
+opts=GetoptLong.new(
+  ["--eval", "-e", GetoptLong::REQUIRED_ARGUMENT],
+  ["--silent", "-s", GetoptLong::NO_ARGUMENT]
+)
 opts.each{|opt,arg|
-  opt=='--eval' or raise :impossible
-  file=arg
-  name='-e'
+  case opt
+  when '--eval'
+    file=arg
+    name='-e'
+  when '--silent'
+    silent=true
+  end
 }
      
 #determine input file and its name if not already known
@@ -31,11 +57,16 @@ file||=if name=ARGV.first
     File.open(name)
   else 
     name='-'
-    $stdin
+    $stdin.read
   end
 
 lexer=RubyLexer.new(name, file) 
-until RubyLexer::EoiToken===(tok=lexer.get1token)
-  puts tok.strify
+if silent
+  until RubyLexer::EoiToken===(tok=lexer.get1token)
+  end
+else
+  until RubyLexer::EoiToken===(tok=lexer.get1token)
+    puts tok.strify
+  end
 end
 puts tok.strify #print eoi token