spidr 0.2.7 → 0.3.0

data/lib/spidr/sanitizers.rb

@@ -3,7 +3,7 @@ require 'uri'
 module Spidr
   #
   # The {Sanitizers} module adds methods to {Agent} which control the
-  # sanitization of incoming links.
+  # sanitation of incoming links.
   #
   module Sanitizers
     def self.included(base)
@@ -17,7 +17,7 @@ module Spidr
     end
 
     #
-    # Initializes the sanitization rules.
+    # Initializes the Sanitizer rules.
     #
     # @param [Hash] options
     #   Additional options.
@@ -52,7 +52,7 @@ module Spidr
     # @since 0.2.2
     #
     def sanitize_url(url)
-      url = URI(url.to_s)
+      url = URI(url.to_s) unless url.kind_of?(URI)
 
       url.fragment = nil if @strip_fragments
       url.query = nil if @strip_query

data/lib/spidr/session_cache.rb

@@ -84,6 +84,7 @@ module Spidr
         if url.scheme == 'https'
           session.use_ssl = true
           session.verify_mode = OpenSSL::SSL::VERIFY_NONE
+          session.start
         end
 
         @sessions[key] = session

data/lib/spidr/version.rb

@@ -1,4 +1,4 @@
 module Spidr
   # Spidr version
-  VERSION = '0.2.7'
+  VERSION = '0.3.0'
 end

data/spec/actions_spec.rb

@@ -4,9 +4,7 @@ require 'spidr/agent'
 require 'spec_helper'
 
 describe Actions do
-  before(:all) do
-    @url = URI('http://spidr.rubyforge.org/')
-  end
+  let(:url) { URI('http://spidr.rubyforge.org/') }
 
   it "should be able to pause spidering" do
     count = 0
@@ -28,10 +26,10 @@ describe Actions do
       end
     end
 
-    agent.enqueue(@url)
+    agent.enqueue(url)
     agent.continue!
 
-    agent.visited?(@url).should == true
+    agent.visited?(url).should == true
   end
 
   it "should allow skipping of enqueued links" do
@@ -41,7 +39,7 @@ describe Actions do
       end
     end
 
-    agent.enqueue(@url)
+    agent.enqueue(url)
 
     agent.queue.should be_empty
   end
@@ -53,9 +51,9 @@ describe Actions do
       end
     end
 
-    agent.visit_page(@url)
+    agent.visit_page(url)
 
-    agent.history.should == Set[@url]
+    agent.history.should == Set[url]
     agent.queue.should be_empty
   end
 end

data/spec/auth_store_spec.rb

@@ -3,10 +3,12 @@ require 'spidr/auth_store'
 require 'spec_helper'
 
 describe AuthStore do
+  let(:root_uri) { URI('http://zerosum.org/') }
+  let(:uri) { root_uri.merge('/course/auth') }
+
   before(:each) do
     @auth_store = AuthStore.new
-    @uri = URI('http://zerosum.org/course/auth')
-    @auth_store.add(@uri, 'admin', 'password')
+    @auth_store.add(uri, 'admin', 'password')
   end
 
   after(:each) do
@@ -14,58 +16,56 @@ describe AuthStore do
   end
 
   it 'should retrieve auth credentials for the URL' do
-    uri = @uri.merge('/')
-
-    @auth_store[uri] = AuthCredential.new('user1', 'pass1')
-    @auth_store[uri].username.should == 'user1'
-    @auth_store[uri].password.should == 'pass1'
+    @auth_store[root_uri] = AuthCredential.new('user1', 'pass1')
+    @auth_store[root_uri].username.should == 'user1'
+    @auth_store[root_uri].password.should == 'pass1'
   end 
 
   it 'should add auth credentials for the URL' do
-    uri = @uri.merge('/')
-
     lambda {
-      @auth_store.add(uri, 'user1', 'pass1')
+      @auth_store.add(root_uri, 'user1', 'pass1')
     }.should change(@auth_store, :size)
 
-    @auth_store[uri].username.should == 'user1'
-    @auth_store[uri].password.should == 'pass1'
+    @auth_store[root_uri].username.should == 'user1'
+    @auth_store[root_uri].password.should == 'pass1'
   end
 
   describe 'matching' do
-    it 'should match a longer URL to the base' do
-      uri = @uri.merge('/course/auth/protected.html')
+    let(:sub_uri) { uri.merge('/course/auth/protected.html') }
 
-      @auth_store[uri].username.should == 'admin'
-      @auth_store[uri].password.should == 'password'
+    it 'should match a longer URL to the base' do
+      @auth_store[sub_uri].username.should == 'admin'
+      @auth_store[sub_uri].password.should == 'password'
     end
 
     it 'should match the longest of all matching URLs' do
-      @auth_store.add(@uri.merge('/course'), 'user1', 'pass1')
-      @auth_store.add(@uri.merge('/course/auth/special'), 'user2', 'pass2')
-      @auth_store.add(@uri.merge('/course/auth/special/extra'), 'user3', 'pass3')
+      @auth_store.add(uri.merge('/course'), 'user1', 'pass1')
+      @auth_store.add(uri.merge('/course/auth/special'), 'user2', 'pass2')
+      @auth_store.add(uri.merge('/course/auth/special/extra'), 'user3', 'pass3')
 
-      auth = @auth_store[@uri.merge('/course/auth/special/1.html')]
+      auth = @auth_store[uri.merge('/course/auth/special/1.html')]
       auth.username.should == 'user2'
       auth.password.should == 'pass2'
     end
 
     it 'should not match a URL with a different host' do
-      uri = URI('http://spidr.rubyforge.org/course/auth')
-      @auth_store[uri].should be_nil
+      remote_uri = URI('http://spidr.rubyforge.org/course/auth')
+
+      @auth_store[remote_uri].should be_nil
     end
 
     it 'should not match a URL with an alternate path' do
-      uri = @uri.merge('/course/admin/protected.html')
-      @auth_store[uri].should be_nil
+      relative_uri = uri.merge('/course/admin/protected.html')
+
+      @auth_store[relative_uri].should be_nil
     end
   end
 
   it 'should override previous auth credentials' do
-    @auth_store.add(@uri, 'newuser', 'newpass')
+    @auth_store.add(uri, 'newuser', 'newpass')
 
-    @auth_store[@uri].username.should == 'newuser'
-    @auth_store[@uri].password.should == 'newpass'
+    @auth_store[uri].username.should == 'newuser'
+    @auth_store[uri].password.should == 'newpass'
   end
 
   it 'should clear all cookies' do
@@ -79,7 +79,7 @@ describe AuthStore do
     end
 
     it 'should create an encoded authorization string' do
-      @auth_store.for_url(@uri).should == "YWRtaW46cGFzc3dvcmQ=\n"
+      @auth_store.for_url(uri).should == "YWRtaW46cGFzc3dvcmQ=\n"
     end
   end
 end

data/spec/cookie_jar_spec.rb

@@ -3,108 +3,99 @@ require 'spidr/cookie_jar'
 require 'spec_helper'
 
 describe CookieJar do
-  before(:each) do
-    @cookie_jar = CookieJar.new
-  end
-
   it "should retrieve cookies for the named host" do
-    @cookie_jar['zerosum.org'] = {'admin' => 'ofcourseiam'}
+    subject['zerosum.org'] = {'admin' => 'ofcourseiam'}
 
-    @cookie_jar['zerosum.org'].should == {'admin' => 'ofcourseiam'}
+    subject['zerosum.org'].should == {'admin' => 'ofcourseiam'}
   end
 
   it "should add a cookie to the jar" do
-    @cookie_jar['zerosum.org'] = {'admin' => 'ofcourseiam'}
+    subject['zerosum.org'] = {'admin' => 'ofcourseiam'}
 
-    @cookie_jar['zerosum.org'].should == {'admin' => 'ofcourseiam'}
+    subject['zerosum.org'].should == {'admin' => 'ofcourseiam'}
   end
 
   it "should merge new cookies into the jar" do
-    @cookie_jar['zerosum.org'] = {'admin' => 'ofcourseiam'}
-    @cookie_jar['zerosum.org'] = {'other' => '1'}
+    subject['zerosum.org'] = {'admin' => 'ofcourseiam'}
+    subject['zerosum.org'] = {'other' => '1'}
 
-    @cookie_jar['zerosum.org'].should == {
+    subject['zerosum.org'].should == {
       'admin' => 'ofcourseiam',
       'other' => '1'
     }
   end
 
   it "should override previous cookies in the jar" do
-    @cookie_jar['zerosum.org'] = {'admin' => 'ofcourseiam'}
-    @cookie_jar['zerosum.org'] = {'admin' => 'somethingcompletelydifferent'}
+    subject['zerosum.org'] = {'admin' => 'ofcourseiam'}
+    subject['zerosum.org'] = {'admin' => 'somethingcompletelydifferent'}
 
-    @cookie_jar['zerosum.org'].should == {
+    subject['zerosum.org'].should == {
       'admin' => 'somethingcompletelydifferent'
     }
   end
 
   it "should clear all cookies" do
-    @cookie_jar['zerosum.org'] = {'cookie' => 'foobar'}
-    @cookie_jar.clear!
+    subject['zerosum.org'] = {'cookie' => 'foobar'}
+    subject.clear!
 
-    @cookie_jar.size.should == 0
+    subject.size.should == 0
   end
 
   describe "dirty" do
-    before(:each) do
-      @cookie_jar = CookieJar.new
-      @dirty = @cookie_jar.instance_variable_get('@dirty')
-    end
+    let(:dirty) { subject.instance_variable_get('@dirty') }
 
     it "should mark a cookie dirty after adding new params" do
-      @cookie_jar['zerosum.org'] = {'admin' => 'ofcourseiam'}
-      @cookie_jar['zerosum.org'] = {'other' => '1'}
+      subject['zerosum.org'] = {'admin' => 'ofcourseiam'}
+      subject['zerosum.org'] = {'other' => '1'}
 
-      @dirty.include?('zerosum.org').should == true
+      dirty.include?('zerosum.org').should == true
     end
 
     it "should mark a cookie dirty after overriding params" do
-      @cookie_jar['zerosum.org'] = {'admin' => 'ofcourseiam'}
-      @cookie_jar['zerosum.org'] = {'admin' => 'nope'}
+      subject['zerosum.org'] = {'admin' => 'ofcourseiam'}
+      subject['zerosum.org'] = {'admin' => 'nope'}
 
-      @dirty.include?('zerosum.org').should == true
+      dirty.include?('zerosum.org').should == true
     end
 
     it "should un-mark a cookie as dirty after re-encoding it" do
-      @cookie_jar['zerosum.org'] = {'admin' => 'ofcourseiam'}
-      @cookie_jar['zerosum.org'] = {'admin' => 'nope'}
+      subject['zerosum.org'] = {'admin' => 'ofcourseiam'}
+      subject['zerosum.org'] = {'admin' => 'nope'}
 
-      @dirty.include?('zerosum.org').should == true
+      dirty.include?('zerosum.org').should == true
 
-      @cookie_jar.for_host('zerosum.org')
+      subject.for_host('zerosum.org')
 
-      @dirty.include?('zerosum.org').should == false
+      dirty.include?('zerosum.org').should == false
     end
   end
 
   describe "cookies_for_host" do
-    before(:each) do
-      @cookie_jar = CookieJar.new
-    end
-
     it "should return an empty Hash for unknown hosts" do
-      @cookie_jar.cookies_for_host('lol.com').should be_empty
+      subject.cookies_for_host('lol.com').should be_empty
     end
 
     it "should return an empty Hash for hosts with no cookie params" do
-      @cookie_jar['lol.com'] = {}
+      subject['lol.com'] = {}
 
-      @cookie_jar.cookies_for_host('lol.com').should be_empty
+      subject.cookies_for_host('lol.com').should be_empty
     end
 
     it "should return cookie parameters for the host" do
-      @cookie_jar['zerosum.org'] = {'admin' => 'ofcourseiam'}
-      @cookie_jar['zerosum.org'] = {'other' => '1'}
-      cookie = @cookie_jar.cookies_for_host('zerosum.org')
+      subject['zerosum.org'] = {'admin' => 'ofcourseiam'}
+      subject['zerosum.org'] = {'other' => '1'}
+
+      cookie = subject.cookies_for_host('zerosum.org')
 
       cookie['admin'].should == 'ofcourseiam'
       cookie['other'].should == '1'
     end
 
     it "should include cookies for the parent domain" do
-      @cookie_jar['zerosum.org'] = {'admin' => 'ofcourseiam'}
-      @cookie_jar['sub.zerosum.org'] = {'other' => '1'}
-      cookie = @cookie_jar.cookies_for_host('sub.zerosum.org')
+      subject['zerosum.org'] = {'admin' => 'ofcourseiam'}
+      subject['sub.zerosum.org'] = {'other' => '1'}
+
+      cookie = subject.cookies_for_host('sub.zerosum.org')
 
       cookie['admin'].should == 'ofcourseiam'
       cookie['other'].should == '1'
@@ -112,30 +103,27 @@ describe CookieJar do
   end
 
   describe "for_host" do
-    before(:each) do
-      @cookie_jar = CookieJar.new
-    end
-
     it "should return nil for unknown hosts" do
-      @cookie_jar.for_host('lol.com').should be_nil
+      subject.for_host('lol.com').should be_nil
     end
 
     it "should return nil for hosts with no cookie params" do
-      @cookie_jar['lol.com'] = {}
+      subject['lol.com'] = {}
 
-      @cookie_jar.for_host('lol.com').should be_nil
+      subject.for_host('lol.com').should be_nil
     end
 
     it "should encode single cookie params" do
-      @cookie_jar['zerosum.org'] = {'admin' => 'ofcourseiam'}
+      subject['zerosum.org'] = {'admin' => 'ofcourseiam'}
 
-      @cookie_jar.for_host('zerosum.org').should == 'admin=ofcourseiam'
+      subject.for_host('zerosum.org').should == 'admin=ofcourseiam'
     end
 
     it "should encode multiple cookie params" do
-      @cookie_jar['zerosum.org'] = {'admin' => 'ofcourseiam'}
-      @cookie_jar['zerosum.org'] = {'other' => '1'}
-      cookie = @cookie_jar.for_host('zerosum.org')
+      subject['zerosum.org'] = {'admin' => 'ofcourseiam'}
+      subject['zerosum.org'] = {'other' => '1'}
+
+      cookie = subject.for_host('zerosum.org')
 
       cookie.should include('admin=ofcourseiam')
       cookie.should include('; ')
@@ -143,9 +131,10 @@ describe CookieJar do
     end
 
     it "should include cookies for the parent domain" do
-      @cookie_jar['zerosum.org'] = {'admin' => 'ofcourseiam'}
-      @cookie_jar['sub.zerosum.org'] = {'other' => '1'}
-      cookie = @cookie_jar.for_host('sub.zerosum.org')
+      subject['zerosum.org'] = {'admin' => 'ofcourseiam'}
+      subject['sub.zerosum.org'] = {'other' => '1'}
+
+      cookie = subject.for_host('sub.zerosum.org')
 
       cookie.should include('admin=ofcourseiam')
       cookie.should include('; ')

data/spec/extensions/uri_spec.rb

@@ -35,5 +35,9 @@ describe URI do
     it "should preserve the root path" do
       URI.expand_path('/').should == '/'
     end
+
+    it "should default empty paths to the root path" do
+      URI.expand_path('').should == '/'
+    end
   end
 end

data/spec/filters_spec.rb

@@ -13,41 +13,49 @@ describe Filters do
 
   it "should provide the hosts that will be visited" do
     agent = Agent.new(:hosts => ['spidr.rubyforge.org'])
+
     agent.visit_hosts.should == ['spidr.rubyforge.org']
   end
 
   it "should provide the hosts that will not be visited" do
     agent = Agent.new(:ignore_hosts => ['example.com'])
+
     agent.ignore_hosts.should == ['example.com']
   end
 
   it "should provide the ports that will be visited" do
     agent = Agent.new(:ports => [80, 443, 8000])
+
     agent.visit_ports.should == [80, 443, 8000]
   end
 
   it "should provide the ports that will not be visited" do
     agent = Agent.new(:ignore_ports => [8000, 8080])
+
     agent.ignore_ports.should == [8000, 8080]
   end
 
   it "should provide the links that will be visited" do
     agent = Agent.new(:links => ['index.php'])
+
     agent.visit_links.should == ['index.php']
   end
 
   it "should provide the links that will not be visited" do
     agent = Agent.new(:ignore_links => [/login/])
+
     agent.ignore_links.should == [/login/]
   end
 
   it "should provide the exts that will be visited" do
     agent = Agent.new(:exts => ['htm'])
+
     agent.visit_exts.should == ['htm']
   end
 
   it "should provide the exts that will not be visited" do
     agent = Agent.new(:ignore_exts => ['cfm'])
+
     agent.ignore_exts.should == ['cfm']
   end
 end