spidr 0.2.2 → 0.2.3

data/lib/spidr/actions/actions.rb

@@ -3,6 +3,10 @@ require 'spidr/actions/exceptions/skip_link'
 require 'spidr/actions/exceptions/skip_page'
 
 module Spidr
+  #
+  # The {Actions} module adds methods to {Agent} for controling the
+  # spidering of links.
+  #
   module Actions
     def initialize(options={})
       @paused = false

data/lib/spidr/actions/exceptions/action.rb

@@ -1,5 +1,8 @@
 module Spidr
   module Actions
+    #
+    # The base {Actions} exception class.
+    #
     class Action < RuntimeError
     end
   end

data/lib/spidr/actions/exceptions/paused.rb

@@ -2,6 +2,9 @@ require 'spidr/actions/exceptions/action'
 
 module Spidr
   module Actions
+    #
+    # An {Actions} exception class used to pause a running {Agent}.
+    #
     class Paused < Action
     end
   end

data/lib/spidr/actions/exceptions/skip_link.rb

@@ -2,6 +2,10 @@ require 'spidr/actions/exceptions/action'
 
 module Spidr
   module Actions
+    #
+    # An {Actions} exception class which causes a running {Agent} to
+    # skip a link.
+    #
     class SkipLink < Action
     end
   end

data/lib/spidr/actions/exceptions/skip_page.rb

@@ -2,6 +2,10 @@ require 'spidr/actions/exceptions/action'
 
 module Spidr
   module Actions
+    #
+    # An {Actions} exception class which causes a running {Agent} to
+    # skip a {Page}, and all links within that page.
+    #
     class SkipPage < Action
     end
   end

data/lib/spidr/agent.rb

@@ -19,6 +19,12 @@ module Spidr
     include Events
     include Actions
 
+    # HTTP Host Header to use
+    attr_accessor :host_header
+
+    # HTTP Host Headers to use for specific hosts
+    attr_reader :host_headers
+
     # User-Agent to use
     attr_accessor :user_agent
 
@@ -64,6 +70,12 @@ module Spidr
     # @option :proxy [String] :password
     #   The password to authenticate with.
     #
+    # @option options [String] :host_header
+    #   The HTTP Host header to use with each request.
+    #
+    # @option options [Hash{String,Regexp => String}] :host_headers
+    #   The HTTP Host headers to use for specific hosts.
+    #
     # @option options [String] :user_agent (Spidr.user_agent)
     #   The User-Agent string to send with each requests.
     #
@@ -87,6 +99,13 @@ module Spidr
     #   The newly created agent.
     #
     def initialize(options={},&block)
+      @host_header = options[:host_header]
+      @host_headers = {}
+
+      if options[:host_headers]
+        @host_headers.merge!(options[:host_headers])
+      end
+
       @user_agent = (options[:user_agent] || Spidr.user_agent)
       @referer = options[:referer]
 
@@ -473,7 +492,7 @@ module Spidr
     #   The page for the response.
     #
     # @return [Page, nil]
-    #   The page for the response, or +nil+ if the request failed.
+    #   The page for the response, or `nil` if the request failed.
     #
     def get_page(url,&block)
       url = URI(url.to_s)
@@ -506,7 +525,7 @@ module Spidr
     #   The page for the response.
     #
     # @return [Page, nil]
-    #   The page for the response, or +nil+ if the request failed.
+    #   The page for the response, or `nil` if the request failed.
     #
     # @since 0.2.2
     #
@@ -538,7 +557,7 @@ module Spidr
     #   The page which was visited.
     #
     # @return [Page, nil]
-    #   The page that was visited. If +nil+ is returned, either the request
+    #   The page that was visited. If `nil` is returned, either the request
     #   for the page failed, or the page was skipped.
     #
     def visit_page(url,&block)
@@ -558,7 +577,20 @@ module Spidr
         rescue Actions::Action
         end
 
-        page.urls.each { |next_url| enqueue(next_url) }
+        page.urls.each do |next_url|
+          begin
+            @every_link_blocks.each do |link_block|
+              link_block.call(page.url,next_url)
+            end
+          rescue Actions::Paused => action
+            raise(action)
+          rescue Actions::SkipLink
+            next
+          rescue Actions::Action
+          end
+
+          enqueue(next_url)
+        end
       end
     end
 
@@ -566,8 +598,8 @@ module Spidr
     # Converts the agent into a Hash.
     #
     # @return [Hash]
-    #   The agent represented as a Hash containing the +history+ and
-    #   the +queue+ of the agent.
+    #   The agent represented as a Hash containing the `history` and
+    #   the `queue` of the agent.
     #
     def to_hash
       {:history => @history, :queue => @queue}
@@ -609,20 +641,32 @@ module Spidr
       # append the URL query to the path
       path += "?#{url.query}" if url.query
 
-      begin
-        sleep(@delay) if @delay > 0
-
-        headers = {}
-        headers['User-Agent'] = @user_agent if @user_agent
-        headers['Referer'] = @referer if @referer
+      # set any additional HTTP headers
+      headers = {}
 
-        if (authorization = @authorized.for_url(url))
-          headers['Authorization'] = "Basic #{authorization}"
+      unless @host_headers.empty?
+        @host_headers.each do |name,header|
+          if host.match(name)
+            headers['Host'] = header
+            break
+          end
         end
+      end
 
-        if (header_cookies = @cookies.for_host(url.host))
-          headers['Cookie'] = header_cookies
-        end
+      headers['Host'] ||= @host_header if @host_header
+      headers['User-Agent'] = @user_agent if @user_agent
+      headers['Referer'] = @referer if @referer
+
+      if (authorization = @authorized.for_url(url))
+        headers['Authorization'] = "Basic #{authorization}"
+      end
+
+      if (header_cookies = @cookies.for_host(url.host))
+        headers['Cookie'] = header_cookies
+      end
+
+      begin
+        sleep(@delay) if @delay > 0
 
         block.call(@sessions[url],path,headers)
       rescue SystemCallError,

data/lib/spidr/auth_credential.rb

@@ -1,4 +1,7 @@
 module Spidr
+  #
+  # Represents HTTP Authentication credentials for a website.
+  #
   class AuthCredential
 
     # The username

data/lib/spidr/auth_store.rb

@@ -5,6 +5,10 @@ require 'spidr/page'
 require 'base64'
 
 module Spidr
+  #
+  # Stores {AuthCredential} objects organized by a website's scheme,
+  # host-name and sub-directory.
+  #
   class AuthStore
 
     #
@@ -24,13 +28,13 @@ module Spidr
     #
     # @return [AuthCredential, nil]
     #   Closest matching {AuthCredential} values for the URL,
-    #   or +nil+ if nothing matches.
+    #   or `nil` if nothing matches.
     #
     # @since 0.2.2
     #
     def [](url)
       # normalize the url
-      url = URI(url) unless url.kind_of?(URI)
+      url = URI(url.to_s) unless url.kind_of?(URI)
 
       key = [url.scheme, url.host, url.port]
       paths = @credentials[key]
@@ -64,9 +68,9 @@ module Spidr
     #
     # @since 0.2.2
     #
-    def []=(url, auth)
+    def []=(url,auth)
       # normalize the url
-      url = URI(url) unless url.kind_of?(URI)
+      url = URI(url.to_s) unless url.kind_of?(URI)
 
       # normalize the URL path
       path = URI.expand_path(url.path)
@@ -96,19 +100,19 @@ module Spidr
     #
     # @since 0.2.2
     #
-    def add(url, username, password)
-      self[url] = AuthCredential.new(username, password)
+    def add(url,username,password)
+      self[url] = AuthCredential.new(username,password)
     end
 
     #
     # Returns the base64 encoded authorization string for the URL
-    # or +nil+ if no authorization exists.
+    # or `nil` if no authorization exists.
     #
     # @param [URI] url
     #   The url.
     #
     # @return [String, nil]
-    #   The base64 encoded authorizatio string or +nil+.
+    #   The base64 encoded authorizatio string or `nil`.
     #
     # @since 0.2.2
     #

data/lib/spidr/cookie_jar.rb

@@ -3,6 +3,9 @@ require 'spidr/page'
 require 'set'
 
 module Spidr
+  #
+  # Stores HTTP Cookies organized by host-name.
+  #
   class CookieJar
 
     include Enumerable
@@ -47,7 +50,7 @@ module Spidr
     #   Host or domain name for cookies.
     #
     # @return [String, nil]
-    #   The cookie values or +nil+ if the host does not have a cookie in the
+    #   The cookie values or `nil` if the host does not have a cookie in the
     #   jar.
     #
     # @since 0.2.2

data/lib/spidr/events.rb

@@ -1,4 +1,9 @@
 module Spidr
+  #
+  # The {Events} module adds methods to {Agent} for registering
+  # callbacks which will receive URLs, links, headers and pages, when
+  # they are visited.
+  #
   module Events
     def initialize(options={})
       super(options)
@@ -8,6 +13,7 @@ module Spidr
       @urls_like_blocks = Hash.new { |hash,key| hash[key] = [] }
 
       @every_page_blocks = []
+      @every_link_blocks = []
     end
 
     #
@@ -499,5 +505,24 @@ module Spidr
         block.call(page) if (block && page.zip?)
       end
     end
+
+    #
+    # Passes every origin and destination URI of each link to a given
+    # block.
+    #
+    # @yield [origin,dest]
+    #   The block will be passed every origin and destination URI of
+    #   each link.
+    #
+    # @yieldparam [URI::HTTP] origin
+    #   The URI that a link originated from.
+    #
+    # @yieldparam [URI::HTTP] dest
+    #   The destination URI of a link.
+    #
+    def every_link(&block)
+      @every_link_blocks << block
+      return self
+    end
   end
 end

data/lib/spidr/filters.rb

@@ -1,6 +1,10 @@
 require 'spidr/rules'
 
 module Spidr
+  #
+  # The {Filters} module adds methods to {Agent} for controlling which
+  # URLs the agent will visit.
+  #
   module Filters
     def self.included(base)
       base.module_eval do
@@ -17,7 +21,7 @@ module Spidr
     #
     # @option options [Array] :schemes (['http', 'https'])
     #   The list of acceptable URI schemes to visit.
-    #   The +https+ scheme will be ignored if +net/https+ cannot be loaded.
+    #   The `https` scheme will be ignored if `net/https` cannot be loaded.
     #
     # @option options [String] :host
     #   The host-name to visit.

data/lib/spidr/page.rb

@@ -5,6 +5,9 @@ require 'uri'
 require 'nokogiri'
 
 module Spidr
+  #
+  # Represents a requested page from a website.
+  #
   class Page
 
     # Reserved names used within Cookie strings
@@ -46,10 +49,10 @@ module Spidr
     end
 
     #
-    # Determines if the response code is +200+.
+    # Determines if the response code is `200`.
     #
     # @return [Boolean]
-    #   Specifies whether the response code is +200+.
+    #   Specifies whether the response code is `200`.
     #
     def is_ok?
       code == 200
@@ -58,10 +61,10 @@ module Spidr
     alias ok? is_ok?
 
     #
-    # Determines if the response code is +301+ or +307+.
+    # Determines if the response code is `301` or `307`.
     #
     # @return [Boolean]
-    #   Specifies whether the response code is +301+ or +307+.
+    #   Specifies whether the response code is `301` or `307`.
     #
     def is_redirect?
       (code == 301 || code == 307)
@@ -70,30 +73,30 @@ module Spidr
     alias redirect? is_redirect?
 
     #
-    # Determines if the response code is +308+.
+    # Determines if the response code is `308`.
     #
     # @return [Boolean]
-    #   Specifies whether the response code is +308+.
+    #   Specifies whether the response code is `308`.
     #
     def timedout?
       code == 308
     end
 
     #
-    # Determines if the response code is +400+.
+    # Determines if the response code is `400`.
     #
     # @return [Boolean]
-    #   Specifies whether the response code is +400+.
+    #   Specifies whether the response code is `400`.
     #
     def bad_request?
       code == 400
     end
 
     #
-    # Determines if the response code is +401+.
+    # Determines if the response code is `401`.
     #
     # @return [Boolean]
-    #   Specifies whether the response code is +401+.
+    #   Specifies whether the response code is `401`.
     #
     def is_unauthorized?
       code == 401
@@ -102,10 +105,10 @@ module Spidr
     alias unauthorized? is_unauthorized?
 
     #
-    # Determines if the response code is +403+.
+    # Determines if the response code is `403`.
     #
     # @return [Boolean]
-    #   Specifies whether the response code is +403+.
+    #   Specifies whether the response code is `403`.
     #
     def is_forbidden?
       code == 403
@@ -114,10 +117,10 @@ module Spidr
     alias forbidden? is_forbidden?
 
     #
-    # Determines if the response code is +404+.
+    # Determines if the response code is `404`.
     #
     # @return [Boolean]
-    #   Specifies whether the response code is +404+.
+    #   Specifies whether the response code is `404`.
     #
     def is_missing?
       code == 404
@@ -126,10 +129,10 @@ module Spidr
     alias missing? is_missing?
 
     #
-    # Determines if the response code is +500+.
+    # Determines if the response code is `500`.
     #
     # @return [Boolean]
-    #   Specifies whether the response code is +500+.
+    #   Specifies whether the response code is `500`.
     #
     def had_internal_server_error?
       code == 500
@@ -306,12 +309,14 @@ module Spidr
     def cookie_params
       params = {}
 
-      cookies.each do |key_value|
-        key, value = key_value.split('=',2)
+      cookies.each do |cookie|
+        cookie.split('; ').each do |key_value|
+          key, value = key_value.split('=',2)
 
-        next if RESERVED_COOKIE_NAMES.include?(key)
+          next if RESERVED_COOKIE_NAMES.include?(key)
 
-        params[key] = (value || '')
+          params[key] = (value || '')
+        end
       end
 
       return params
@@ -332,7 +337,7 @@ module Spidr
     #
     # @return [Nokogiri::HTML::Document, Nokogiri::XML::Document, nil]
     #   The document that represents HTML or XML pages.
-    #   Returns +nil+ if the page is neither HTML, XML, RSS, Atom or if
+    #   Returns `nil` if the page is neither HTML, XML, RSS, Atom or if
     #   the page could not be parsed properly.
     #
     # @see http://nokogiri.rubyforge.org/nokogiri/Nokogiri/XML/Document.html
@@ -380,7 +385,7 @@ module Spidr
     # Searches for the first occurrence an XPath or CSS Path expression.
     #
     # @return [Nokogiri::HTML::Node, Nokogiri::XML::Node, nil]
-    #   The first matched node. Returns +nil+ if no nodes could be matched,
+    #   The first matched node. Returns `nil` if no nodes could be matched,
     #   or if the page is not a HTML or XML document.
     #
     # @example
@@ -416,7 +421,7 @@ module Spidr
     #
     # @return [Array<String>]
     #   All links within the HTML page, frame/iframe source URLs and any
-    #   links in the +Location+ header.
+    #   links in the `Location` header.
     #
     def links
       urls = []
@@ -502,7 +507,7 @@ module Spidr
     protected
 
     #
-    # Provides transparent access to the values in +headers+.
+    # Provides transparent access to the values in `headers`.
     #
     def method_missing(sym,*args,&block)
       if (args.empty? && block.nil?)