spidr 0.1.7 → 0.1.8

data/History.txt

@@ -1,3 +1,14 @@
+=== 0.1.8 / 2009-05-27
+
+* Added the Agent#pause! and Agent#continue! methods.
+* Added the Agent#running? and Agent#paused? methods.
+* Added an alias for pending_urls to the queue methods.
+* Added Agent#queue to provide read access to the queue.
+* Added Agent#queue= and Agent#history= for setting the queue and history.
+* Added Agent#to_hash which returns a Hash of the agents queue and history.
+* Made Agent#enqueue and Agent#queued? public.
+* Added more specs.
+
 === 0.1.7 / 2009-04-24
 
 * Added Agent#all_headers.

data/Manifest.txt

@@ -13,6 +13,7 @@ tasks/course.rb
 spec/spec_helper.rb
 spec/helpers/course.rb
 spec/agent_spec.rb
+spec/spidr_spec.rb
 static/course/index.html
 static/course/start.html
 static/course/fail.html

data/README.txt

@@ -27,6 +27,8 @@ and easy to use.
   * Every visited URL.
   * Every visited URL that matches a specified pattern.
   * Every URL that failed to be visited.
+* Pause and continue spidering.
+* Restore the spidering queue and history from a previous session.
 * Custom User-Agent strings.
 * Custom proxy settings.
 

data/lib/spidr/agent.rb

@@ -28,6 +28,9 @@ module Spidr
     # List of unreachable URLs
     attr_reader :failures
 
+    # Queue of URLs to visit
+    attr_reader :queue
+
     #
     # Creates a new Agent object with the given _options_ and _block_.
     # If a _block_ is given, it will be passed the newly created
@@ -82,6 +85,7 @@ module Spidr
       @history = []
       @failures = []
       @queue = []
+      @paused = true
 
       if options[:host]
         visit_hosts_like(options[:host])
@@ -361,22 +365,70 @@ module Spidr
     #
     def start_at(url)
       clear
-      return run(url)
+      enqueue(url)
+
+      return continue!
     end
 
     #
-    # Start spidering at the specified _url_.
+    # Start spidering until the queue becomes empty or the agent is
+    # paused.
     #
-    def run(url)
-      enqueue(url)
-
-      until @queue.empty?
+    def run
+      until (@queue.empty? || @paused == true)
         visit_page(dequeue)
       end
 
       return self
     end
 
+    #
+    # Continue spidering.
+    #
+    def continue!
+      @paused = false
+      return run
+    end
+
+    #
+    # Returns +true+ if the agent is still spidering, returns +false+
+    # otherwise.
+    #
+    def running?
+      @paused == false
+    end
+
+    #
+    # Returns +true+ if the agent is paused, returns +false+ otherwise.
+    #
+    def paused?
+      @paused == true
+    end
+
+    #
+    # Pauses the agent, causing spidering to temporarily stop.
+    #
+    def pause!
+      @paused = true
+      return self
+    end
+
+    #
+    # Sets the history of links that were previously visited to the
+    # specified _new_history_.
+    #
+    #   agent.history = ['http://tenderlovemaking.com/2009/05/06/ann-nokogiri-130rc1-has-been-released/']
+    #
+    def history=(new_history)
+      @history = new_history.map do |url|
+        unless url.kind_of?(URI)
+          URI(url.to_s)
+        else
+          url
+        end
+      end
+    end
+
     alias visited_urls history
 
     #
@@ -398,9 +450,7 @@ module Spidr
     # otherwise.
     #
     def visited?(url)
-      unless url.kind_of?(URI)
-        url = URI(url)
-      end
+      url = URI(url) unless url.kind_of?(URI)
 
       return @history.include?(url)
     end
@@ -410,13 +460,13 @@ module Spidr
     # returns +false+ otherwise.
     #
     def failed?(url)
-      unless url.kind_of?(URI)
-        url = URI(url)
-      end
+      url = URI(url) unless url.kind_of?(URI)
 
       return @failures.include?(url)
     end
 
+    alias pending_urls queue
+
     #
     # Creates a new Page object from the specified _url_. If a _block_ is
     # given, it will be passed the newly created Page object.
@@ -454,7 +504,28 @@ module Spidr
       end
     end
 
-    protected
+    #
+    # Returns the agent represented as a Hash containing the agents
+    # +history+ and +queue+ information.
+    #
+    def to_hash
+      {:history => @history, :queue => @queue}
+    end
+
+    #
+    # Sets the queue of links to visit to the specified _new_queue_.
+    #
+    #   agent.queue = ['http://www.vimeo.com/', 'http://www.reddit.com/']
+    #
+    def queue=(new_queue)
+      @queue = new_queue.map do |url|
+        unless url.kind_of?(URI)
+          URI(url.to_s)
+        else
+          url
+        end
+      end
+    end
 
     #
     # Returns +true+ if the specified _url_ is queued for visiting, returns
@@ -489,6 +560,8 @@ module Spidr
       return false
     end
 
+    protected
+
     #
     # Dequeues a URL that will later be visited.
     #
@@ -574,9 +647,7 @@ module Spidr
     # Adds the specified _url_ to the failures list.
     #
     def failed(url)
-      unless url.kind_of?(URI)
-        url = URI(url.to_s)
-      end
+      url = URI(url.to_s) unless url.kind_of?(URI)
 
       @every_failed_url_blocks.each { |block| block.call(url) }
       @failures << url

data/lib/spidr/version.rb

@@ -1,3 +1,3 @@
 module Spidr
-  VERSION = '0.1.7'
+  VERSION = '0.1.8'
 end

data/spec/agent_spec.rb

@@ -9,4 +9,80 @@ describe Agent do
   before(:all) do
     @agent = run_course
   end
+
+  it "should provide the history" do
+    @agent.history.should_not be_empty
+  end
+
+  it "should provide the queue" do
+    @agent.queue.should be_empty
+  end
+
+  it "should be able to restore the history" do
+    agent = Agent.new
+    previous_history = [URI('http://www.example.com')]
+
+    agent.history = previous_history
+    agent.history.should == previous_history
+  end
+
+  it "should convert new histories to an Array of URIs" do
+    agent = Agent.new
+    previous_history = ['http://www.example.com']
+
+    agent.history = previous_history
+    agent.history.should_not == previous_history
+    agent.history.should == previous_history.map { |url| URI(url) }
+  end
+
+  it "should be able to restore the queue" do
+    agent = Agent.new
+    previous_queue = [URI('http://www.example.com')]
+
+    agent.queue = previous_queue
+    agent.queue.should == previous_queue
+  end
+
+  it "should convert new queues to an Array of URIs" do
+    agent = Agent.new
+    previous_queue = ['http://www.example.com']
+
+    agent.queue = previous_queue
+    agent.queue.should_not == previous_queue
+    agent.queue.should == previous_queue.map { |url| URI(url) }
+  end
+
+  it "should be able to pause spidering" do
+    count = 0
+    agent = Agent.host('spidr.rubyforge.org') do |spider|
+      spider.every_page do |page|
+        count += 1
+        spider.pause! if count >= 2
+      end
+    end
+
+    agent.should be_paused
+    agent.history.length.should == 2
+  end
+
+  it "should be able to continue spidering after being paused" do
+    agent = Agent.new do |spider|
+      spider.enqueue('http://spidr.rubyforge.org/')
+      spider.every_page do |page|
+        spider.pause!
+      end
+    end
+
+    agent.pause!
+    agent.continue!
+
+    agent.visited?('http://spidr.rubyforge.org/').should == true
+  end
+
+  it "should provide a to_hash method that returns the queue and history" do
+    hash = @agent.to_hash
+
+    hash[:queue].should be_empty
+    hash[:history].should_not be_empty
+  end
 end

data/spec/spidr_spec.rb

@@ -0,0 +1,9 @@
+require 'spidr'
+
+require 'spec_helper'
+
+describe Spidr do
+  it "should have a VERSION constant" do
+    Spidr.const_defined?('VERSION').should == true
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: spidr
 version: !ruby/object:Gem::Version 
-  version: 0.1.7
+  version: 0.1.8
 platform: ruby
 authors: 
 - Postmodern
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-04-24 00:00:00 -07:00
+date: 2009-05-27 00:00:00 -07:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -59,6 +59,7 @@ files:
 - spec/spec_helper.rb
 - spec/helpers/course.rb
 - spec/agent_spec.rb
+- spec/spidr_spec.rb
 - static/course/index.html
 - static/course/start.html
 - static/course/fail.html