spidr 0.4.1 → 0.5.0

data/lib/spidr/actions.rb

@@ -1,2 +0,0 @@
-require 'spidr/actions/exceptions'
-require 'spidr/actions/actions'

data/lib/spidr/actions/exceptions.rb

@@ -1,4 +0,0 @@
-require 'spidr/actions/exceptions/action'
-require 'spidr/actions/exceptions/paused'
-require 'spidr/actions/exceptions/skip_link'
-require 'spidr/actions/exceptions/skip_page'

data/lib/spidr/actions/exceptions/action.rb

@@ -1,9 +0,0 @@
-module Spidr
-  module Actions
-    #
-    # The base {Actions} exception class.
-    #
-    class Action < RuntimeError
-    end
-  end
-end

data/lib/spidr/actions/exceptions/paused.rb

@@ -1,11 +0,0 @@
-require 'spidr/actions/exceptions/action'
-
-module Spidr
-  module Actions
-    #
-    # An {Actions} exception class used to pause a running {Agent}.
-    #
-    class Paused < Action
-    end
-  end
-end

data/lib/spidr/actions/exceptions/skip_link.rb

@@ -1,12 +0,0 @@
-require 'spidr/actions/exceptions/action'
-
-module Spidr
-  module Actions
-    #
-    # An {Actions} exception class which causes a running {Agent} to
-    # skip a link.
-    #
-    class SkipLink < Action
-    end
-  end
-end

data/lib/spidr/actions/exceptions/skip_page.rb

@@ -1,12 +0,0 @@
-require 'spidr/actions/exceptions/action'
-
-module Spidr
-  module Actions
-    #
-    # An {Actions} exception class which causes a running {Agent} to
-    # skip a {Page}, and all links within that page.
-    #
-    class SkipPage < Action
-    end
-  end
-end

data/spec/actions_spec.rb

@@ -1,59 +0,0 @@
-require 'spidr/actions'
-require 'spidr/agent'
-
-require 'spec_helper'
-
-describe Actions do
-  let(:url) { URI('http://spidr.rubyforge.org/') }
-
-  it "should be able to pause spidering" do
-    count = 0
-    agent = Agent.host('spidr.rubyforge.org') do |spider|
-      spider.every_page do |page|
-        count += 1
-        spider.pause! if count >= 2
-      end
-    end
-
-    agent.should be_paused
-    agent.history.length.should == 2
-  end
-
-  it "should be able to continue spidering after being paused" do
-    agent = Agent.new do |spider|
-      spider.every_page do |page|
-        spider.pause!
-      end
-    end
-
-    agent.enqueue(url)
-    agent.continue!
-
-    agent.visited?(url).should == true
-  end
-
-  it "should allow skipping of enqueued links" do
-    agent = Agent.new do |spider|
-      spider.every_url do |url|
-        spider.skip_link!
-      end
-    end
-
-    agent.enqueue(url)
-
-    agent.queue.should be_empty
-  end
-
-  it "should allow skipping of visited pages" do
-    agent = Agent.new do |spider|
-      spider.every_page do |url|
-        spider.skip_page!
-      end
-    end
-
-    agent.visit_page(url)
-
-    agent.history.should == Set[url]
-    agent.queue.should be_empty
-  end
-end

data/spec/filters_spec.rb

@@ -1,61 +0,0 @@
-require 'spidr/filters'
-require 'spidr/agent'
-
-require 'spec_helper'
-
-describe Filters do
-  it "should allow setting the acceptable schemes" do
-    agent = Agent.new
-
-    agent.schemes = [:http]
-    agent.schemes.should == ['http']
-  end
-
-  it "should provide the hosts that will be visited" do
-    agent = Agent.new(:hosts => ['spidr.rubyforge.org'])
-
-    agent.visit_hosts.should == ['spidr.rubyforge.org']
-  end
-
-  it "should provide the hosts that will not be visited" do
-    agent = Agent.new(:ignore_hosts => ['example.com'])
-
-    agent.ignore_hosts.should == ['example.com']
-  end
-
-  it "should provide the ports that will be visited" do
-    agent = Agent.new(:ports => [80, 443, 8000])
-
-    agent.visit_ports.should == [80, 443, 8000]
-  end
-
-  it "should provide the ports that will not be visited" do
-    agent = Agent.new(:ignore_ports => [8000, 8080])
-
-    agent.ignore_ports.should == [8000, 8080]
-  end
-
-  it "should provide the links that will be visited" do
-    agent = Agent.new(:links => ['index.php'])
-
-    agent.visit_links.should == ['index.php']
-  end
-
-  it "should provide the links that will not be visited" do
-    agent = Agent.new(:ignore_links => [/login/])
-
-    agent.ignore_links.should == [/login/]
-  end
-
-  it "should provide the exts that will be visited" do
-    agent = Agent.new(:exts => ['htm'])
-
-    agent.visit_exts.should == ['htm']
-  end
-
-  it "should provide the exts that will not be visited" do
-    agent = Agent.new(:ignore_exts => ['cfm'])
-
-    agent.ignore_exts.should == ['cfm']
-  end
-end

data/spec/sanitizers_spec.rb

@@ -1,61 +0,0 @@
-require 'spidr/sanitizers'
-require 'spidr/agent'
-
-require 'spec_helper'
-
-describe Sanitizers do
-  describe "sanitize_url" do
-    let(:url) { 'http://host.com' }
-    before(:all) { @agent = Agent.new }
-
-    it "should sanitize URLs" do
-      agent = Agent.new
-      clean_url = agent.sanitize_url(URI(url))
-
-      clean_url.host.should == 'host.com'
-    end
-
-    it "should sanitize URLs given as Strings" do
-      agent = Agent.new
-      clean_url = agent.sanitize_url(url)
-
-      clean_url.host.should == 'host.com'
-    end
-  end
-
-  describe "strip_fragments" do
-    let(:url) { URI("http://host.com/page#lol") }
-
-    it "should strip fragment components by default" do
-      agent = Agent.new
-      clean_url = agent.sanitize_url(url)
-
-      clean_url.fragment.should be_nil
-    end
-
-    it "should allow perserving fragment components" do
-      agent = Agent.new(:strip_fragments => false)
-      clean_url = agent.sanitize_url(url)
-
-      clean_url.fragment.should == 'lol'
-    end
-  end
-
-  describe "strip_query" do
-    let(:url) { URI("http://host.com/page?x=1") }
-
-    it "should not strip query components by default" do
-      agent = Agent.new
-      clean_url = agent.sanitize_url(url)
-
-      clean_url.query.should == 'x=1'
-    end
-
-    it "should allow stripping of query components" do
-      agent = Agent.new(:strip_query => true)
-      clean_url = agent.sanitize_url(url)
-
-      clean_url.query.should be_nil
-    end
-  end
-end