spidr 0.2.2 → 0.2.3

data/lib/spidr/rules.rb

@@ -1,4 +1,8 @@
 module Spidr
+  #
+  # The {Rules} class represents collections of acceptance and rejection
+  # rules, which are used to filter data.
+  #
   class Rules
 
     # Accept rules

data/lib/spidr/sanitizers.rb

@@ -1,6 +1,10 @@
 require 'uri'
 
 module Spidr
+  #
+  # The {Sanitizers} module adds methods to {Agent} which control the
+  # sanitization of incoming links.
+  #
   module Sanitizers
     def self.included(base)
       base.module_eval do

data/lib/spidr/session_cache.rb

@@ -3,6 +3,9 @@ require 'spidr/spidr'
 require 'net/http'
 
 module Spidr
+  #
+  # Stores active HTTP Sessions organized by scheme, host-name and port.
+  #
   class SessionCache
 
     # Proxy to use
@@ -33,6 +36,27 @@ module Spidr
       @sessions = {}
     end
 
+    #
+    # Determines if there is an active HTTP session for a given URL.
+    #
+    # @param [URI::HTTP, String] url
+    #   The URL that represents a session.
+    #
+    # @return [Boolean]
+    #   Specifies whether there is an active HTTP session.
+    #
+    # @since 0.2.3
+    #
+    def active?(url)
+      # normalize the url
+      url = URI(url.to_s) unless url.kind_of?(URI)
+
+      # session key
+      key = [url.scheme, url.host, url.port]
+
+      return @sessions.has_key?(key)
+    end
+
     #
     # Provides an active HTTP session for a given URL.
     #
@@ -46,6 +70,7 @@ module Spidr
       # normalize the url
       url = URI(url.to_s) unless url.kind_of?(URI)
 
+      # session key
       key = [url.scheme, url.host, url.port]
 
       unless @sessions[key]
@@ -81,10 +106,10 @@ module Spidr
       # normalize the url
       url = URI(url.to_s) unless url.kind_of?(URI)
 
+      # session key
       key = [url.scheme, url.host, url.port]
 
       if (sess = @sessions[key])
-
         begin 
           sess.finish
         rescue IOError

data/lib/spidr/version.rb

@@ -1,4 +1,4 @@
 module Spidr
   # Spidr version
-  VERSION = '0.2.2'
+  VERSION = '0.2.3'
 end

data/spec/auth_store_spec.rb

@@ -0,0 +1,85 @@
+require 'spidr/auth_store'
+
+require 'spec_helper'
+
+describe AuthStore do
+  before(:each) do
+    @auth_store = AuthStore.new
+    @uri = URI('http://zerosum.org/course/auth')
+    @auth_store.add(@uri, 'admin', 'password')
+  end
+
+  after(:each) do
+    @auth_store.clear!
+  end
+
+  it 'should retrieve auth credentials for the URL' do
+    uri = @uri.merge('/')
+
+    @auth_store[uri] = AuthCredential.new('user1', 'pass1')
+    @auth_store[uri].username.should == 'user1'
+    @auth_store[uri].password.should == 'pass1'
+  end 
+
+  it 'should add auth credentials for the URL' do
+    uri = @uri.merge('/')
+
+    lambda {
+      @auth_store.add(uri, 'user1', 'pass1')
+    }.should change(@auth_store, :size)
+
+    @auth_store[uri].username.should == 'user1'
+    @auth_store[uri].password.should == 'pass1'
+  end
+
+  describe 'matching' do
+    it 'should match a longer URL to the base' do
+      uri = @uri.merge('/course/auth/protected.html')
+
+      @auth_store[uri].username.should == 'admin'
+      @auth_store[uri].password.should == 'password'
+    end
+
+    it 'should match the longest of all matching URLs' do
+      @auth_store.add(@uri.merge('/course'), 'user1', 'pass1')
+      @auth_store.add(@uri.merge('/course/auth/special'), 'user2', 'pass2')
+      @auth_store.add(@uri.merge('/course/auth/special/extra'), 'user3', 'pass3')
+
+      auth = @auth_store[@uri.merge('/course/auth/special/1.html')]
+      auth.username.should == 'user2'
+      auth.password.should == 'pass2'
+    end
+
+    it 'should not match a URL with a different host' do
+      uri = URI('http://spidr.rubyforge.org/course/auth')
+      @auth_store[uri].should be_nil
+    end
+
+    it 'should not match a URL with an alternate path' do
+      uri = @uri.merge('/course/admin/protected.html')
+      @auth_store[uri].should be_nil
+    end
+  end
+
+  it 'should override previous auth credentials' do
+    @auth_store.add(@uri, 'newuser', 'newpass')
+
+    @auth_store[@uri].username.should == 'newuser'
+    @auth_store[@uri].password.should == 'newpass'
+  end
+
+  it 'should clear all cookies' do
+    @auth_store.clear!
+    @auth_store.size.should == 0
+  end
+
+  describe 'for_url' do
+    it 'should return nil if no authorization exists' do
+      @auth_store.for_url(URI('http://php.net')).should be_nil
+    end
+
+    it 'should create an encoded authorization string' do
+      @auth_store.for_url(@uri).should == "YWRtaW46cGFzc3dvcmQ=\n"
+    end
+  end
+end

data/spec/cookie_jar_spec.rb

@@ -0,0 +1,108 @@
+require 'spidr/cookie_jar'
+
+require 'spec_helper'
+
+describe CookieJar do
+  before(:each) do
+    @cookie_jar = CookieJar.new
+  end
+
+  it "should retrieve cookies for the named host" do
+    @cookie_jar['zerosum.org'] = {'admin' => 'ofcourseiam'}
+
+    @cookie_jar['zerosum.org'].should == {'admin' => 'ofcourseiam'}
+  end
+
+  it "should add a cookie to the jar" do
+    @cookie_jar['zerosum.org'] = {'admin' => 'ofcourseiam'}
+
+    @cookie_jar['zerosum.org'].should == {'admin' => 'ofcourseiam'}
+  end
+
+  it "should merge new cookies into the jar" do
+    @cookie_jar['zerosum.org'] = {'admin' => 'ofcourseiam'}
+    @cookie_jar['zerosum.org'] = {'other' => '1'}
+
+    @cookie_jar['zerosum.org'].should == {
+      'admin' => 'ofcourseiam',
+      'other' => '1'
+    }
+  end
+
+  it "should override previous cookies in the jar" do
+    @cookie_jar['zerosum.org'] = {'admin' => 'ofcourseiam'}
+    @cookie_jar['zerosum.org'] = {'admin' => 'somethingcompletelydifferent'}
+
+    @cookie_jar['zerosum.org'].should == {
+      'admin' => 'somethingcompletelydifferent'
+    }
+  end
+
+  it "should clear all cookies" do
+    @cookie_jar['zerosum.org'] = {'cookie' => 'foobar'}
+    @cookie_jar.clear!
+
+    @cookie_jar.size.should == 0
+  end
+
+  describe "dirty" do
+    before(:each) do
+      @cookie_jar = CookieJar.new
+      @dirty = @cookie_jar.instance_variable_get('@dirty')
+    end
+
+    it "should mark a cookie dirty after adding new params" do
+      @cookie_jar['zerosum.org'] = {'admin' => 'ofcourseiam'}
+      @cookie_jar['zerosum.org'] = {'other' => '1'}
+
+      @dirty.include?('zerosum.org').should == true
+    end
+
+    it "should mark a cookie dirty after overriding params" do
+      @cookie_jar['zerosum.org'] = {'admin' => 'ofcourseiam'}
+      @cookie_jar['zerosum.org'] = {'admin' => 'nope'}
+
+      @dirty.include?('zerosum.org').should == true
+    end
+
+    it "should un-mark a cookie as dirty after re-encoding it" do
+      @cookie_jar['zerosum.org'] = {'admin' => 'ofcourseiam'}
+      @cookie_jar['zerosum.org'] = {'admin' => 'nope'}
+
+      @dirty.include?('zerosum.org').should == true
+
+      @cookie_jar.for_host('zerosum.org')
+
+      @dirty.include?('zerosum.org').should == false
+    end
+  end
+
+  describe "for_host" do
+    before(:each) do
+      @cookie_jar = CookieJar.new
+    end
+
+    it "should return nil for unknown hosts" do
+      @cookie_jar.for_host('lol.com').should be_nil
+    end
+
+    it "should return nil for hosts with no cookie params" do
+      @cookie_jar['lol.com'] = {}
+
+      @cookie_jar.for_host('lol.com').should be_nil
+    end
+
+    it "should encode single cookie params" do
+      @cookie_jar['zerosum.org'] = {'admin' => 'ofcourseiam'}
+
+      @cookie_jar.for_host('zerosum.org').should == 'admin=ofcourseiam'
+    end
+
+    it "should encode multiple cookie params" do
+      @cookie_jar['zerosum.org'] = {'admin' => 'ofcourseiam'}
+      @cookie_jar['zerosum.org'] = {'other' => '1'}
+
+      @cookie_jar.for_host('zerosum.org').should == 'admin=ofcourseiam; other=1'
+    end
+  end
+end

data/spec/page_spec.rb

@@ -104,7 +104,6 @@ describe Page do
 
       params.each do |key,value|
         key.should_not be_empty
-        value.should_not be_empty
       end
     end
   end

data/spec/session_cache.rb

@@ -0,0 +1,58 @@
+require 'spidr/session_cache'
+
+require 'spec_helper'
+
+describe SessionCache do
+  describe "empty" do
+    before(:all) do
+      @sessions = SessionCache.new
+    end
+
+    it "should not have any active sessions" do
+      @sessions.should_not be_active(URI('http://example.com/'))
+    end
+
+    it "should start new sessions on-demand" do
+      @sessions[URI('http://example.com/')].should_not be_nil
+    end
+
+    after(:all) do
+      @sessions.clear
+    end
+  end
+
+  describe "not-empty" do
+    before(:all) do
+      @url = URI('http://example.com/')
+
+      @sessions = SessionCache.new
+      @sessions[@url]
+    end
+
+    it "should have active sessions" do
+      @sessions.should be_active(@url)
+    end
+
+    it "should provide access to sessions" do
+      @sessions[@url].should_not be_nil
+    end
+
+    it "should start new sessions on-demand" do
+      url2 = URI('http://www.w3c.org/')
+
+      @sessions[url2].should_not be_nil
+    end
+
+    it "should be able to kill sessions" do
+      url2 = URI('http://www.w3c.org/')
+
+      @sessions[url2].should_not be_nil
+      @sessions.kill!(url2)
+      @sessions.should_not be_active(url2)
+    end
+
+    after(:all) do
+      @sessions.clear
+    end
+  end
+end

data/spidr.gemspec

@@ -0,0 +1,115 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{spidr}
+  s.version = "0.2.3"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Postmodern"]
+  s.date = %q{2010-02-27}
+  s.description = %q{Spidr is a versatile Ruby web spidering library that can spider a site, multiple domains, certain links or infinitely. Spidr is designed to be fast and easy to use.}
+  s.email = %q{postmodern.mod3@gmail.com}
+  s.extra_rdoc_files = [
+    "ChangeLog.md",
+    "LICENSE.txt",
+    "README.md"
+  ]
+  s.files = [
+    ".gitignore",
+    ".specopts",
+    ".yardopts",
+    "ChangeLog.md",
+    "LICENSE.txt",
+    "README.md",
+    "Rakefile",
+    "lib/spidr.rb",
+    "lib/spidr/actions.rb",
+    "lib/spidr/actions/actions.rb",
+    "lib/spidr/actions/exceptions.rb",
+    "lib/spidr/actions/exceptions/action.rb",
+    "lib/spidr/actions/exceptions/paused.rb",
+    "lib/spidr/actions/exceptions/skip_link.rb",
+    "lib/spidr/actions/exceptions/skip_page.rb",
+    "lib/spidr/agent.rb",
+    "lib/spidr/auth_credential.rb",
+    "lib/spidr/auth_store.rb",
+    "lib/spidr/cookie_jar.rb",
+    "lib/spidr/events.rb",
+    "lib/spidr/extensions.rb",
+    "lib/spidr/extensions/uri.rb",
+    "lib/spidr/filters.rb",
+    "lib/spidr/page.rb",
+    "lib/spidr/rules.rb",
+    "lib/spidr/sanitizers.rb",
+    "lib/spidr/session_cache.rb",
+    "lib/spidr/spidr.rb",
+    "lib/spidr/version.rb",
+    "spec/actions_spec.rb",
+    "spec/agent_spec.rb",
+    "spec/auth_store_spec.rb",
+    "spec/cookie_jar_spec.rb",
+    "spec/extensions/uri_spec.rb",
+    "spec/filters_spec.rb",
+    "spec/helpers/history.rb",
+    "spec/helpers/page.rb",
+    "spec/helpers/wsoc.rb",
+    "spec/page_examples.rb",
+    "spec/page_spec.rb",
+    "spec/rules_spec.rb",
+    "spec/sanitizers_spec.rb",
+    "spec/session_cache.rb",
+    "spec/spec_helper.rb",
+    "spec/spidr_spec.rb",
+    "spidr.gemspec"
+  ]
+  s.has_rdoc = %q{yard}
+  s.homepage = %q{http://github.com/postmodern/spidr}
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.6}
+  s.summary = %q{A versatile Ruby web spidering library}
+  s.test_files = [
+    "spec/agent_spec.rb",
+    "spec/helpers/history.rb",
+    "spec/helpers/wsoc.rb",
+    "spec/helpers/page.rb",
+    "spec/spec_helper.rb",
+    "spec/extensions/uri_spec.rb",
+    "spec/page_spec.rb",
+    "spec/spidr_spec.rb",
+    "spec/sanitizers_spec.rb",
+    "spec/page_examples.rb",
+    "spec/filters_spec.rb",
+    "spec/actions_spec.rb",
+    "spec/rules_spec.rb",
+    "spec/auth_store_spec.rb",
+    "spec/cookie_jar_spec.rb",
+    "spec/session_cache.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<nokogiri>, [">= 1.2.0"])
+      s.add_development_dependency(%q<rspec>, [">= 1.3.0"])
+      s.add_development_dependency(%q<yard>, [">= 0.5.3"])
+      s.add_development_dependency(%q<wsoc>, [">= 0.1.1"])
+    else
+      s.add_dependency(%q<nokogiri>, [">= 1.2.0"])
+      s.add_dependency(%q<rspec>, [">= 1.3.0"])
+      s.add_dependency(%q<yard>, [">= 0.5.3"])
+      s.add_dependency(%q<wsoc>, [">= 0.1.1"])
+    end
+  else
+    s.add_dependency(%q<nokogiri>, [">= 1.2.0"])
+    s.add_dependency(%q<rspec>, [">= 1.3.0"])
+    s.add_dependency(%q<yard>, [">= 0.5.3"])
+    s.add_dependency(%q<wsoc>, [">= 0.1.1"])
+  end
+end
+