spidr 0.2.4 → 0.2.5

data/.gitignore

@@ -3,6 +3,7 @@ doc
 web
 tmp
 .DS_Store
+.bundle
 .yardoc
 *.swp
 *~

data/ChangeLog.md

@@ -1,3 +1,13 @@
+### 0.2.5 / 2010-07-02
+
+* Added {Spidr::Page#meta_redirect}.
+* Added {Spidr::Page#meta_redirect?}.
+* Manage development dependencies with Bundler.
+* Support following "old-school" meta-refresh redirects (thanks zapnap).
+* Allow {Spidr::CookieJar} inherit cookies set by a parent domain.
+* Fixed a constant lookup issue in {Spidr::Agent}.
+* Use `yield` instead of `block.call` when necessary.
+
 ### 0.2.4 / 2010-05-05
 
 * Added {Spidr::Filters#visit_urls}.

data/Gemfile

@@ -0,0 +1,27 @@
+source 'https://rubygems.org'
+
+group(:runtime) do
+  gem 'nokogiri',	'>= 1.3.0'
+end
+
+group(:development) do
+  gem 'rake',			'~> 0.8.7'
+  gem 'jeweler',		'~> 1.4.0', :git => 'git://github.com/technicalpickles/jeweler.git'
+end
+
+group(:doc) do
+  case RUBY_PLATFORM
+  when 'java'
+    gem 'maruku',	'~> 0.6.0'
+  else
+    gem 'rdiscount',	'~> 1.6.3'
+  end
+
+  gem 'yard',		'~> 0.5.3'
+end
+
+group(:test) do
+  gem 'wsoc',	'~> 0.1.3'
+end
+
+gem 'rspec',	'~> 1.3.0', :group => [:development, :test]

data/README.md

@@ -20,6 +20,7 @@ and easy to use.
   * frame tags.
   * Cookie protected links.
   * HTTP 300, 301, 302, 303 and 307 Redirects.
+  * Meta-Refresh Redirects.
   * HTTP Basic Auth protected links.
 * Black-list or white-list URLs based upon:
   * URL scheme.

data/Rakefile

@@ -1,27 +1,28 @@
 require 'rubygems'
+require 'bundler'
+
+begin
+  Bundler.setup(:development, :doc)
+rescue Bundler::BundlerError => e
+  STDERR.puts e.message
+  STDERR.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+
 require 'rake'
+require 'jeweler'
 require './lib/spidr/version.rb'
 
-begin
-  require 'jeweler'
-  Jeweler::Tasks.new do |gem|
-    gem.name = 'spidr'
-    gem.version = Spidr::VERSION
-    gem.license = 'MIT'
-    gem.summary = %Q{A versatile Ruby web spidering library}
-    gem.description = %Q{Spidr is a versatile Ruby web spidering library that can spider a site, multiple domains, certain links or infinitely. Spidr is designed to be fast and easy to use.}
-    gem.email = 'postmodern.mod3@gmail.com'
-    gem.homepage = 'http://github.com/postmodern/spidr'
-    gem.authors = ['Postmodern']
-    gem.add_dependency 'nokogiri', '>= 1.3.0'
-    gem.add_development_dependency 'rspec', '~> 1.3.0'
-    gem.add_development_dependency 'yard', '~> 0.5.3'
-    gem.add_development_dependency 'wsoc', '~> 0.1.1'
-    gem.has_rdoc = 'yard'
-  end
-  Jeweler::GemcutterTasks.new
-rescue LoadError
-  puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
+Jeweler::Tasks.new do |gem|
+  gem.name = 'spidr'
+  gem.version = Spidr::VERSION
+  gem.license = 'MIT'
+  gem.summary = %Q{A versatile Ruby web spidering library}
+  gem.description = %Q{Spidr is a versatile Ruby web spidering library that can spider a site, multiple domains, certain links or infinitely. Spidr is designed to be fast and easy to use.}
+  gem.email = 'postmodern.mod3@gmail.com'
+  gem.homepage = 'http://github.com/postmodern/spidr'
+  gem.authors = ['Postmodern']
+  gem.has_rdoc = 'yard'
 end
 
 require 'spec/rake/spectask'
@@ -31,15 +32,7 @@ Spec::Rake::SpecTask.new(:spec) do |spec|
   spec.spec_opts = ['--options', '.specopts']
 end
 
-task :spec => :check_dependencies
 task :default => :spec
 
-begin
-  require 'yard'
-
-  YARD::Rake::YardocTask.new
-rescue LoadError
-  task :yard do
-    abort "YARD is not available. In order to run yard, you must: gem install yard"
-  end
-end
+require 'yard'
+YARD::Rake::YardocTask.new

data/lib/spidr/agent.rb

@@ -98,7 +98,7 @@ module Spidr
     # @yieldparam [Agent] agent
     #   The newly created agent.
     #
-    def initialize(options={},&block)
+    def initialize(options={})
       @host_header = options[:host_header]
       @host_headers = {}
 
@@ -121,7 +121,7 @@ module Spidr
 
       super(options)
 
-      block.call(self) if block
+      yield self if block_given?
     end
 
     #
@@ -140,9 +140,9 @@ module Spidr
     # @yieldparam [Agent] agent
     #   The newly created agent.
     #
-    def self.start_at(url,options={},&block)
+    def self.start_at(url,options={})
       self.new(options) do |spider|
-        block.call(spider) if block
+        yield spider if block_given?
 
         spider.start_at(url)
       end
@@ -164,9 +164,9 @@ module Spidr
     # @yieldparam [Agent] agent
     #   The newly created agent.
     #
-    def self.host(name,options={},&block)
+    def self.host(name,options={})
       self.new(options.merge(:host => name)) do |spider|
-        block.call(spider) if block
+        yield spider if block_given?
 
         spider.start_at("http://#{name}/")
       end
@@ -188,11 +188,11 @@ module Spidr
     # @yieldparam [Agent] agent
     #   The newly created agent.
     #
-    def self.site(url,options={},&block)
+    def self.site(url,options={})
       url = URI(url.to_s)
 
       return self.new(options.merge(:host => url.host)) do |spider|
-        block.call(spider) if block
+        yield spider if block_given?
 
         spider.start_at(url)
       end
@@ -457,11 +457,18 @@ module Spidr
         link = url.to_s
 
         begin
-          @every_url_blocks.each { |block| block.call(url) }
+          @every_url_blocks.each { |url_block| url_block.call(url) }
 
-          @urls_like_blocks.each do |pattern,blocks|
-            if ((pattern.kind_of?(Regexp) && link =~ pattern) || pattern == link || pattern == url)
-              blocks.each { |url_block| url_block.call(url) }
+          @urls_like_blocks.each do |pattern,url_blocks|
+            match = case pattern
+                    when Regexp
+                      link =~ pattern
+                    else
+                      (pattern == link) || (pattern == url)
+                    end
+
+            if match
+              url_blocks.each { |url_block| url_block.call(url) }
             end
           end
         rescue Actions::Paused => action
@@ -494,7 +501,7 @@ module Spidr
     # @return [Page, nil]
     #   The page for the response, or `nil` if the request failed.
     #
-    def get_page(url,&block)
+    def get_page(url)
       url = URI(url.to_s)
 
       prepare_request(url) do |session,path,headers|
@@ -503,7 +510,7 @@ module Spidr
         # save any new cookies
         @cookies.from_page(new_page)
 
-        block.call(new_page) if block
+        yield new_page if block_given?
         return new_page
       end
     end
@@ -529,7 +536,7 @@ module Spidr
     #
     # @since 0.2.2
     #
-    def post_page(url,post_data='',&block)
+    def post_page(url,post_data='')
       url = URI(url.to_s)
 
       prepare_request(url) do |session,path,headers|
@@ -538,7 +545,7 @@ module Spidr
         # save any new cookies
         @cookies.from_page(new_page)
 
-        block.call(new_page) if block
+        yield new_page if block_given?
         return new_page
       end
     end
@@ -560,7 +567,7 @@ module Spidr
     #   The page that was visited. If `nil` is returned, either the request
     #   for the page failed, or the page was skipped.
     #
-    def visit_page(url,&block)
+    def visit_page(url)
       url = URI(url.to_s) unless url.kind_of?(URI)
 
       get_page(url) do |page|
@@ -569,7 +576,7 @@ module Spidr
         begin
           @every_page_blocks.each { |page_block| page_block.call(page) }
 
-          block.call(page) if block
+          yield page if block_given?
         rescue Actions::Paused => action
           raise(action)
         rescue Actions::SkipPage
@@ -668,7 +675,7 @@ module Spidr
       begin
         sleep(@delay) if @delay > 0
 
-        block.call(@sessions[url],path,headers)
+        yield @sessions[url], path, headers
       rescue SystemCallError,
              Timeout::Error,
              SocketError,
@@ -719,7 +726,7 @@ module Spidr
     #
     def failed(url)
       @failures << url
-      @every_failed_url_blocks.each { |block| block.call(url) }
+      @every_failed_url_blocks.each { |fail_block| fail_block.call(url) }
       return true
     end
 

data/lib/spidr/cookie_jar.rb

@@ -130,6 +130,22 @@ module Spidr
         @dirty.delete(host)
       end
 
+      hdomain = host.split('.')
+
+      if hdomain.length > 2
+        parent_cookies = for_host(hdomain[1..-1].join('.'))
+
+        unless (parent_cookies.nil? || parent_cookies.empty?)
+          @cookies[host] = if @cookies[host].nil?
+                             # inherit the parent cookies
+                             parent_cookies
+                           else
+                             # merge the parent cookies with any host-specific cookies
+                             "#{parent_cookies}; #{@cookies[host]}"
+                           end
+        end
+      end
+
       return @cookies[host]
     end
 

data/lib/spidr/events.rb

@@ -72,8 +72,8 @@ module Spidr
     # @yieldparam [Hash] headers
     #   The headers from a response.
     #
-    def all_headers(&block)
-      every_page { |page| block.call(page.headers) }
+    def all_headers
+      every_page { |page| yield page.headers }
     end
 
     #
@@ -99,9 +99,9 @@ module Spidr
     # @yieldparam [Page] page
     #   A visited page.
     #
-    def every_ok_page(&block)
+    def every_ok_page
       every_page do |page|
-        block.call(page) if (block && page.ok?)
+        yield page if (block_given? && page.ok?)
       end
     end
 
@@ -114,9 +114,9 @@ module Spidr
     # @yieldparam [Page] page
     #   A visited page.
     #
-    def every_redirect_page(&block)
+    def every_redirect_page
       every_page do |page|
-        block.call(page) if (block && page.redirect?)
+        yield page if (block_given? && page.redirect?)
       end
     end
 
@@ -129,9 +129,9 @@ module Spidr
     # @yieldparam [Page] page
     #   A visited page.
     #
-    def every_timedout_page(&block)
+    def every_timedout_page
       every_page do |page|
-        block.call(page) if (block && page.timedout?)
+        yield page if (block_given? && page.timedout?)
       end
     end
 
@@ -144,9 +144,9 @@ module Spidr
     # @yieldparam [Page] page
     #   A visited page.
     #
-    def every_bad_request_page(&block)
+    def every_bad_request_page
       every_page do |page|
-        block.call(page) if (block && page.bad_request?)
+        yield page if (block_given? && page.bad_request?)
       end
     end
 
@@ -159,9 +159,9 @@ module Spidr
     # @yieldparam [Page] page
     #   A visited page.
     #
-    def every_unauthorized_page(&block)
+    def every_unauthorized_page
       every_page do |page|
-        block.call(page) if (block && page.unauthorized?)
+        yield page if (block_given? && page.unauthorized?)
       end
     end
 
@@ -174,9 +174,9 @@ module Spidr
     # @yieldparam [Page] page
     #   A visited page.
     #
-    def every_forbidden_page(&block)
+    def every_forbidden_page
       every_page do |page|
-        block.call(page) if (block && page.forbidden?)
+        yield page if (block_given? && page.forbidden?)
       end
     end
 
@@ -189,9 +189,9 @@ module Spidr
     # @yieldparam [Page] page
     #   A visited page.
     #
-    def every_missing_page(&block)
+    def every_missing_page
       every_page do |page|
-        block.call(page) if (block && page.missing?)
+        yield page if (block_given? && page.missing?)
       end
     end
 
@@ -205,9 +205,9 @@ module Spidr
     # @yieldparam [Page] page
     #   A visited page.
     #
-    def every_internal_server_error_page(&block)
+    def every_internal_server_error_page
       every_page do |page|
-        block.call(page) if (block && page.had_internal_server_error?)
+        yield page if (block_given? && page.had_internal_server_error?)
       end
     end
 
@@ -220,9 +220,9 @@ module Spidr
     # @yieldparam [Page] page
     #   A visited page.
     #
-    def every_txt_page(&block)
+    def every_txt_page
       every_page do |page|
-        block.call(page) if (block && page.txt?)
+        yield page if (block_given? && page.txt?)
       end
     end
 
@@ -235,9 +235,9 @@ module Spidr
     # @yieldparam [Page] page
     #   A visited page.
     #
-    def every_html_page(&block)
+    def every_html_page
       every_page do |page|
-        block.call(page) if (block && page.html?)
+        yield page if (block_given? && page.html?)
       end
     end
 
@@ -250,9 +250,9 @@ module Spidr
     # @yieldparam [Page] page
     #   A visited page.
     #
-    def every_xml_page(&block)
+    def every_xml_page
       every_page do |page|
-        block.call(page) if (block && page.xml?)
+        yield page if (block_given? && page.xml?)
       end
     end
 
@@ -266,9 +266,9 @@ module Spidr
     # @yieldparam [Page] page
     #   A visited page.
     #
-    def every_xsl_page(&block)
+    def every_xsl_page
       every_page do |page|
-        block.call(page) if (block && page.xsl?)
+        yield page if (block_given? && page.xsl?)
       end
     end
 
@@ -285,11 +285,11 @@ module Spidr
     # @see http://nokogiri.rubyforge.org/nokogiri/Nokogiri/XML/Document.html
     # @see http://nokogiri.rubyforge.org/nokogiri/Nokogiri/HTML/Document.html
     #
-    def every_doc(&block)
+    def every_doc
       every_page do |page|
-        if block
+        if block_given?
           if (doc = page.doc)
-            block.call(doc)
+            yield doc
           end
         end
       end
@@ -306,11 +306,11 @@ module Spidr
     #
     # @see http://nokogiri.rubyforge.org/nokogiri/Nokogiri/HTML/Document.html
     #
-    def every_html_doc(&block)
+    def every_html_doc
       every_page do |page|
-        if (block && page.html?)
+        if (block_given? && page.html?)
           if (doc = page.doc)
-            block.call(doc)
+            yield doc
           end
         end
       end
@@ -327,11 +327,11 @@ module Spidr
     #
     # @see http://nokogiri.rubyforge.org/nokogiri/Nokogiri/XML/Document.html
     #
-    def every_xml_doc(&block)
+    def every_xml_doc
       every_page do |page|
-        if (block && page.xml?)
+        if (block_given? && page.xml?)
           if (doc = page.doc)
-            block.call(doc)
+            yield doc
           end
         end
       end
@@ -349,11 +349,11 @@ module Spidr
     #
     # @see http://nokogiri.rubyforge.org/nokogiri/Nokogiri/XML/Document.html
     #
-    def every_xsl_doc(&block)
+    def every_xsl_doc
       every_page do |page|
-        if (block && page.xsl?)
+        if (block_given? && page.xsl?)
           if (doc = page.doc)
-            block.call(doc)
+            yield doc
           end
         end
       end
@@ -370,11 +370,11 @@ module Spidr
     #
     # @see http://nokogiri.rubyforge.org/nokogiri/Nokogiri/XML/Document.html
     #
-    def every_rss_doc(&block)
+    def every_rss_doc
       every_page do |page|
-        if (block && page.rss?)
+        if (block_given? && page.rss?)
           if (doc = page.doc)
-            block.call(doc)
+            yield doc
           end
         end
       end
@@ -391,11 +391,11 @@ module Spidr
     #
     # @see http://nokogiri.rubyforge.org/nokogiri/Nokogiri/XML/Document.html
     #
-    def every_atom_doc(&block)
+    def every_atom_doc
       every_page do |page|
-        if (block && page.atom?)
+        if (block_given? && page.atom?)
           if (doc = page.doc)
-            block.call(doc)
+            yield doc
           end
         end
       end
@@ -410,9 +410,9 @@ module Spidr
     # @yieldparam [Page] page
     #   A visited page.
     #
-    def every_javascript_page(&block)
+    def every_javascript_page
       every_page do |page|
-        block.call(page) if (block && page.javascript?)
+        yield page if (block_given? && page.javascript?)
       end
     end
 
@@ -425,9 +425,9 @@ module Spidr
     # @yieldparam [Page] page
     #   A visited page.
     #
-    def every_css_page(&block)
+    def every_css_page
       every_page do |page|
-        block.call(page) if (block && page.css?)
+        yield page if (block_given? && page.css?)
       end
     end
 
@@ -440,9 +440,9 @@ module Spidr
     # @yieldparam [Page] feed
     #   A visited page.
     #
-    def every_rss_page(&block)
+    def every_rss_page
       every_page do |page|
-        block.call(page) if (block && page.rss?)
+        yield page if (block_given? && page.rss?)
       end
     end
 
@@ -455,9 +455,9 @@ module Spidr
     # @yieldparam [Page] feed
     #   A visited page.
     #
-    def every_atom_page(&block)
+    def every_atom_page
       every_page do |page|
-        block.call(page) if (block && page.atom?)
+        yield page if (block_given? && page.atom?)
       end
     end
 
@@ -470,9 +470,9 @@ module Spidr
     # @yieldparam [Page] page
     #   A visited page.
     #
-    def every_ms_word_page(&block)
+    def every_ms_word_page
       every_page do |page|
-        block.call(page) if (block && page.ms_word?)
+        yield page if (block_given? && page.ms_word?)
       end
     end
 
@@ -485,9 +485,9 @@ module Spidr
     # @yieldparam [Page] page
     #   A visited page.
     #
-    def every_pdf_page(&block)
+    def every_pdf_page
       every_page do |page|
-        block.call(page) if (block && page.pdf?)
+        yield page if (block_given? && page.pdf?)
       end
     end
 
@@ -500,9 +500,9 @@ module Spidr
     # @yieldparam [Page] page
     #   A visited page.
     #
-    def every_zip_page(&block)
+    def every_zip_page
       every_page do |page|
-        block.call(page) if (block && page.zip?)
+        yield page if (block_given? && page.zip?)
       end
     end
 

data/lib/spidr/page.rb

@@ -62,7 +62,8 @@ module Spidr
 
     #
     # Determines if the response code is `300`, `301`, `302`, `303`
-    # or `307`.
+    # or `307`. Also checks for "soft" redirects added at the page 
+    # level by a meta refresh tag.
     #
     # @return [Boolean]
     #   Specifies whether the response code is a HTTP Redirect code.
@@ -71,6 +72,8 @@ module Spidr
       case code
       when 300..303, 307
         true
+      when 200
+        meta_redirect?
       else
         false
       end
@@ -434,17 +437,7 @@ module Spidr
         urls << url unless (url.nil? || url.empty?)
       }
 
-      if self.is_redirect?
-        location = @headers['location']
-
-        if location.kind_of?(Array)
-          # handle multiple location URLs
-          location.each(&add_url)
-        else
-          # usually the location header contains a single String
-          add_url.call(location)
-        end
-      end
+      self.redirects_to.each(&add_url) if self.is_redirect?
 
       if (html? && doc)
         doc.search('a[@href]').each do |a|
@@ -471,6 +464,27 @@ module Spidr
       return urls
     end
 
+    #
+    # URL(s) that this document redirects to.
+    #
+    # @return [Array<String>]
+    #   The links that this page redirects to (usually found in a
+    #   location header or by way of a page-level meta redirect).
+    #
+    def redirects_to
+      location = @headers['location']
+
+      if location.nil?
+        # check page-level meta redirects if there isn't a location header
+        meta_redirect
+      elsif location.kind_of?(Array)
+        location
+      else
+        # usually the location header contains a single String
+        [location]
+      end
+    end
+
     #
     # Absolute URIs from within the page.
     #
@@ -507,6 +521,43 @@ module Spidr
       return url
     end
 
+    #
+    # Determines if a page-level "soft" redirect is present. If yes,
+    # returns an array of those redirects (usually a single URL).
+    # Otherwise, returns false.
+    #
+    # @return [Array<String>]
+    #   An array of redirect URLs
+    #
+    def meta_redirect
+      redirects = []
+
+      if (html? && doc)
+        search('//meta[@http-equiv and @content]').each do |node|
+          if node.attr('http-equiv') =~ /refresh/i
+            content = node.attr('content')
+
+            if (redirect = content.match(/url=(\S+)$/))
+              redirects << redirect[1]
+            end
+          end
+        end
+      end
+
+      return redirects.uniq
+    end
+
+    #
+    # Returns a boolean indicating whether or not page-level meta
+    # redirects are present in this page.
+    #
+    # @return [Boolean]
+    #   Specifies whether the page includes page-level redirects.
+    #
+    def meta_redirect?
+      !meta_redirect.empty?
+    end
+
     protected
 
     #

data/lib/spidr/version.rb

@@ -1,4 +1,4 @@
 module Spidr
   # Spidr version
-  VERSION = '0.2.4'
+  VERSION = '0.2.5'
 end

data/spec/cookie_jar_spec.rb

@@ -101,8 +101,21 @@ describe CookieJar do
     it "should encode multiple cookie params" do
       @cookie_jar['zerosum.org'] = {'admin' => 'ofcourseiam'}
       @cookie_jar['zerosum.org'] = {'other' => '1'}
+      cookie = @cookie_jar.for_host('zerosum.org')
 
-      @cookie_jar.for_host('zerosum.org').should == 'admin=ofcourseiam; other=1'
+      cookie.should include('admin=ofcourseiam')
+      cookie.should include('; ')
+      cookie.should include('other=1')
+    end
+
+    it "should include cookies for the parent domain" do
+      @cookie_jar['zerosum.org'] = {'admin' => 'ofcourseiam'}
+      @cookie_jar['sub.zerosum.org'] = {'other' => '1'}
+      cookie = @cookie_jar.for_host('sub.zerosum.org')
+
+      cookie.should include('admin=ofcourseiam')
+      cookie.should include('; ')
+      cookie.should include('other=1')
     end
   end
 end

data/spec/helpers/wsoc.rb

@@ -66,7 +66,7 @@ module Helpers
     end
 
     def run_course
-      Agent.start_at(COURSE_URL) do |agent|
+      Spidr::Agent.start_at(COURSE_URL) do |agent|
         course_auth_store.each do |path,auth|
           agent.authorized.add(
             COURSE_URL.merge(path),

data/spec/page_spec.rb

@@ -79,6 +79,21 @@ describe Page do
     end
   end
 
+  describe "redirects" do
+    before(:all) do
+      @page = get_page('http://spidr.rubyforge.org/course/start.html')
+      @page.stub!(:body).and_return('<meta HTTP-EQUIV="REFRESH" content="0; url=http://spidr.rubyforge.org/redirected">')
+    end
+
+    it "should provide access to page-level redirects" do
+      @page.redirects_to.should == ['http://spidr.rubyforge.org/redirected']
+    end 
+
+    it "should include meta refresh redirects in the list of links" do
+      @page.links.should include('http://spidr.rubyforge.org/redirected')
+    end
+  end
+
   describe "cookies" do
     before(:all) do
       @page = get_page('http://twitter.com/login')

data/spec/spec_helper.rb

@@ -1,7 +1,15 @@
 require 'rubygems'
-gem 'rspec', '>=1.2.8'
-require 'spec'
+require 'bundler'
+
+begin
+  Bundler.setup(:runtime, :test)
+rescue Bundler::BundlerError => e
+  STDERR.puts e.message
+  STDERR.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
 
+require 'spec'
 require 'spidr/version'
 
 include Spidr

data/spidr.gemspec

@@ -5,112 +5,112 @@
 
 Gem::Specification.new do |s|
   s.name = %q{spidr}
-  s.version = "0.2.4"
+  s.version = "0.2.5"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Postmodern"]
-  s.date = %q{2010-05-05}
+  s.date = %q{2010-07-02}
   s.description = %q{Spidr is a versatile Ruby web spidering library that can spider a site, multiple domains, certain links or infinitely. Spidr is designed to be fast and easy to use.}
   s.email = %q{postmodern.mod3@gmail.com}
   s.extra_rdoc_files = [
     "ChangeLog.md",
-     "LICENSE.txt",
-     "README.md"
+    "LICENSE.txt",
+    "README.md"
   ]
   s.files = [
     ".gitignore",
-     ".specopts",
-     ".yardopts",
-     "ChangeLog.md",
-     "LICENSE.txt",
-     "README.md",
-     "Rakefile",
-     "lib/spidr.rb",
-     "lib/spidr/actions.rb",
-     "lib/spidr/actions/actions.rb",
-     "lib/spidr/actions/exceptions.rb",
-     "lib/spidr/actions/exceptions/action.rb",
-     "lib/spidr/actions/exceptions/paused.rb",
-     "lib/spidr/actions/exceptions/skip_link.rb",
-     "lib/spidr/actions/exceptions/skip_page.rb",
-     "lib/spidr/agent.rb",
-     "lib/spidr/auth_credential.rb",
-     "lib/spidr/auth_store.rb",
-     "lib/spidr/cookie_jar.rb",
-     "lib/spidr/events.rb",
-     "lib/spidr/extensions.rb",
-     "lib/spidr/extensions/uri.rb",
-     "lib/spidr/filters.rb",
-     "lib/spidr/page.rb",
-     "lib/spidr/rules.rb",
-     "lib/spidr/sanitizers.rb",
-     "lib/spidr/session_cache.rb",
-     "lib/spidr/spidr.rb",
-     "lib/spidr/version.rb",
-     "spec/actions_spec.rb",
-     "spec/agent_spec.rb",
-     "spec/auth_store_spec.rb",
-     "spec/cookie_jar_spec.rb",
-     "spec/extensions/uri_spec.rb",
-     "spec/filters_spec.rb",
-     "spec/helpers/history.rb",
-     "spec/helpers/page.rb",
-     "spec/helpers/wsoc.rb",
-     "spec/page_examples.rb",
-     "spec/page_spec.rb",
-     "spec/rules_spec.rb",
-     "spec/sanitizers_spec.rb",
-     "spec/session_cache.rb",
-     "spec/spec_helper.rb",
-     "spec/spidr_spec.rb",
-     "spidr.gemspec"
+    ".specopts",
+    ".yardopts",
+    "ChangeLog.md",
+    "Gemfile",
+    "LICENSE.txt",
+    "README.md",
+    "Rakefile",
+    "lib/spidr.rb",
+    "lib/spidr/actions.rb",
+    "lib/spidr/actions/actions.rb",
+    "lib/spidr/actions/exceptions.rb",
+    "lib/spidr/actions/exceptions/action.rb",
+    "lib/spidr/actions/exceptions/paused.rb",
+    "lib/spidr/actions/exceptions/skip_link.rb",
+    "lib/spidr/actions/exceptions/skip_page.rb",
+    "lib/spidr/agent.rb",
+    "lib/spidr/auth_credential.rb",
+    "lib/spidr/auth_store.rb",
+    "lib/spidr/cookie_jar.rb",
+    "lib/spidr/events.rb",
+    "lib/spidr/extensions.rb",
+    "lib/spidr/extensions/uri.rb",
+    "lib/spidr/filters.rb",
+    "lib/spidr/page.rb",
+    "lib/spidr/rules.rb",
+    "lib/spidr/sanitizers.rb",
+    "lib/spidr/session_cache.rb",
+    "lib/spidr/spidr.rb",
+    "lib/spidr/version.rb",
+    "spec/actions_spec.rb",
+    "spec/agent_spec.rb",
+    "spec/auth_store_spec.rb",
+    "spec/cookie_jar_spec.rb",
+    "spec/extensions/uri_spec.rb",
+    "spec/filters_spec.rb",
+    "spec/helpers/history.rb",
+    "spec/helpers/page.rb",
+    "spec/helpers/wsoc.rb",
+    "spec/page_examples.rb",
+    "spec/page_spec.rb",
+    "spec/rules_spec.rb",
+    "spec/sanitizers_spec.rb",
+    "spec/session_cache.rb",
+    "spec/spec_helper.rb",
+    "spec/spidr_spec.rb",
+    "spidr.gemspec"
   ]
   s.has_rdoc = %q{yard}
   s.homepage = %q{http://github.com/postmodern/spidr}
   s.licenses = ["MIT"]
-  s.rdoc_options = ["--charset=UTF-8"]
   s.require_paths = ["lib"]
-  s.rubygems_version = %q{1.3.6}
+  s.rubygems_version = %q{1.3.7}
   s.summary = %q{A versatile Ruby web spidering library}
   s.test_files = [
+    "spec/actions_spec.rb",
+    "spec/agent_spec.rb",
     "spec/auth_store_spec.rb",
-     "spec/rules_spec.rb",
-     "spec/session_cache.rb",
-     "spec/spec_helper.rb",
-     "spec/sanitizers_spec.rb",
-     "spec/filters_spec.rb",
-     "spec/page_spec.rb",
-     "spec/spidr_spec.rb",
-     "spec/agent_spec.rb",
-     "spec/cookie_jar_spec.rb",
-     "spec/extensions/uri_spec.rb",
-     "spec/helpers/history.rb",
-     "spec/helpers/page.rb",
-     "spec/helpers/wsoc.rb",
-     "spec/page_examples.rb",
-     "spec/actions_spec.rb"
+    "spec/cookie_jar_spec.rb",
+    "spec/extensions/uri_spec.rb",
+    "spec/filters_spec.rb",
+    "spec/helpers/history.rb",
+    "spec/helpers/page.rb",
+    "spec/helpers/wsoc.rb",
+    "spec/page_examples.rb",
+    "spec/page_spec.rb",
+    "spec/rules_spec.rb",
+    "spec/sanitizers_spec.rb",
+    "spec/session_cache.rb",
+    "spec/spec_helper.rb",
+    "spec/spidr_spec.rb"
   ]
 
   if s.respond_to? :specification_version then
     current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
     s.specification_version = 3
 
-    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
       s.add_runtime_dependency(%q<nokogiri>, [">= 1.3.0"])
+      s.add_development_dependency(%q<rake>, ["~> 0.8.7"])
+      s.add_development_dependency(%q<jeweler>, ["~> 1.4.0"])
       s.add_development_dependency(%q<rspec>, ["~> 1.3.0"])
-      s.add_development_dependency(%q<yard>, ["~> 0.5.3"])
-      s.add_development_dependency(%q<wsoc>, ["~> 0.1.1"])
     else
       s.add_dependency(%q<nokogiri>, [">= 1.3.0"])
+      s.add_dependency(%q<rake>, ["~> 0.8.7"])
+      s.add_dependency(%q<jeweler>, ["~> 1.4.0"])
       s.add_dependency(%q<rspec>, ["~> 1.3.0"])
-      s.add_dependency(%q<yard>, ["~> 0.5.3"])
-      s.add_dependency(%q<wsoc>, ["~> 0.1.1"])
     end
   else
     s.add_dependency(%q<nokogiri>, [">= 1.3.0"])
+    s.add_dependency(%q<rake>, ["~> 0.8.7"])
+    s.add_dependency(%q<jeweler>, ["~> 1.4.0"])
     s.add_dependency(%q<rspec>, ["~> 1.3.0"])
-    s.add_dependency(%q<yard>, ["~> 0.5.3"])
-    s.add_dependency(%q<wsoc>, ["~> 0.1.1"])
   end
 end
 

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 2
-  - 4
-  version: 0.2.4
+  - 5
+  version: 0.2.5
 platform: ruby
 authors: 
 - Postmodern
@@ -14,13 +14,13 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-05-05 00:00:00 -07:00
+date: 2010-07-02 00:00:00 -07:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: nokogiri
-  prerelease: false
   requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
@@ -30,48 +30,52 @@ dependencies:
         - 0
         version: 1.3.0
   type: :runtime
+  prerelease: false
   version_requirements: *id001
 - !ruby/object:Gem::Dependency 
-  name: rspec
-  prerelease: false
+  name: rake
   requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
         segments: 
-        - 1
-        - 3
         - 0
-        version: 1.3.0
+        - 8
+        - 7
+        version: 0.8.7
   type: :development
+  prerelease: false
   version_requirements: *id002
 - !ruby/object:Gem::Dependency 
-  name: yard
-  prerelease: false
+  name: jeweler
   requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
         segments: 
+        - 1
+        - 4
         - 0
-        - 5
-        - 3
-        version: 0.5.3
+        version: 1.4.0
   type: :development
+  prerelease: false
   version_requirements: *id003
 - !ruby/object:Gem::Dependency 
-  name: wsoc
-  prerelease: false
+  name: rspec
   requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
         segments: 
-        - 0
         - 1
-        - 1
-        version: 0.1.1
+        - 3
+        - 0
+        version: 1.3.0
   type: :development
+  prerelease: false
   version_requirements: *id004
 description: Spidr is a versatile Ruby web spidering library that can spider a site, multiple domains, certain links or infinitely. Spidr is designed to be fast and easy to use.
 email: postmodern.mod3@gmail.com
@@ -88,6 +92,7 @@ files:
 - .specopts
 - .yardopts
 - ChangeLog.md
+- Gemfile
 - LICENSE.txt
 - README.md
 - Rakefile
@@ -135,18 +140,21 @@ homepage: http://github.com/postmodern/spidr
 licenses: 
 - MIT
 post_install_message: 
-rdoc_options: 
-- --charset=UTF-8
+rdoc_options: []
+
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 740918287
       segments: 
       - 0
       version: "0"
 required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
@@ -156,24 +164,24 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.3.6
+rubygems_version: 1.3.7
 signing_key: 
 specification_version: 3
 summary: A versatile Ruby web spidering library
 test_files: 
-- spec/auth_store_spec.rb
-- spec/rules_spec.rb
-- spec/session_cache.rb
-- spec/spec_helper.rb
-- spec/sanitizers_spec.rb
-- spec/filters_spec.rb
-- spec/page_spec.rb
-- spec/spidr_spec.rb
+- spec/actions_spec.rb
 - spec/agent_spec.rb
+- spec/auth_store_spec.rb
 - spec/cookie_jar_spec.rb
 - spec/extensions/uri_spec.rb
+- spec/filters_spec.rb
 - spec/helpers/history.rb
 - spec/helpers/page.rb
 - spec/helpers/wsoc.rb
 - spec/page_examples.rb
-- spec/actions_spec.rb
+- spec/page_spec.rb
+- spec/rules_spec.rb
+- spec/sanitizers_spec.rb
+- spec/session_cache.rb
+- spec/spec_helper.rb
+- spec/spidr_spec.rb