spidr 0.5.0 → 0.6.0

data/lib/spidr/page/status_codes.rb

@@ -0,0 +1,112 @@
+module Spidr
+  class Page
+    #
+    # The response code from the page.
+    #
+    # @return [Integer]
+    #   Response code from the page.
+    #
+    def code
+      @response.code.to_i
+    end
+
+    #
+    # Determines if the response code is `200`.
+    #
+    # @return [Boolean]
+    #   Specifies whether the response code is `200`.
+    #
+    def is_ok?
+      code == 200
+    end
+
+    alias ok? is_ok?
+
+    #
+    # Determines if the response code is `308`.
+    #
+    # @return [Boolean]
+    #   Specifies whether the response code is `308`.
+    #
+    def timedout?
+      code == 308
+    end
+
+    #
+    # Determines if the response code is `400`.
+    #
+    # @return [Boolean]
+    #   Specifies whether the response code is `400`.
+    #
+    def bad_request?
+      code == 400
+    end
+
+    #
+    # Determines if the response code is `401`.
+    #
+    # @return [Boolean]
+    #   Specifies whether the response code is `401`.
+    #
+    def is_unauthorized?
+      code == 401
+    end
+
+    alias unauthorized? is_unauthorized?
+
+    #
+    # Determines if the response code is `403`.
+    #
+    # @return [Boolean]
+    #   Specifies whether the response code is `403`.
+    #
+    def is_forbidden?
+      code == 403
+    end
+
+    alias forbidden? is_forbidden?
+
+    #
+    # Determines if the response code is `404`.
+    #
+    # @return [Boolean]
+    #   Specifies whether the response code is `404`.
+    #
+    def is_missing?
+      code == 404
+    end
+
+    alias missing? is_missing?
+
+    #
+    # Determines if the response code is `500`.
+    #
+    # @return [Boolean]
+    #   Specifies whether the response code is `500`.
+    #
+    def had_internal_server_error?
+      code == 500
+    end
+
+    #
+    # Determines if the response code is `300`, `301`, `302`, `303`
+    # or `307`. Also checks for "soft" redirects added at the page 
+    # level by a meta refresh tag.
+    #
+    # @return [Boolean]
+    #   Specifies whether the response code is a HTTP Redirect code.
+    #
+    def is_redirect?
+      case code
+      when 300..303, 307
+        true
+      when 200
+        meta_redirect?
+      else
+        false
+      end
+    end
+
+    alias redirect? is_redirect?
+  end
+end

data/lib/spidr/proxy.rb

@@ -0,0 +1,56 @@
+module Spidr
+  #
+  # @since 0.6.0
+  #
+  class Proxy < Struct.new(:host, :port, :user, :password)
+
+    # Default port to use.
+    DEFAULT_PORT = 8080
+
+    #
+    # Initializes the proxy.
+    #
+    # @param [Hash] attributes
+    #   Attributes for the proxy.
+    #
+    # @option attributes [String] :host
+    #   The host the proxy is running on.
+    #
+    # @option attributes [Integer] :port
+    #   The port the proxy is running on.
+    #
+    # @option attributes [String] :user
+    #   The user to authenticate as with the proxy.
+    #
+    # @option attributes [String] :password
+    #   The password to authenticate with.
+    #
+    def initialize(attributes={})
+      super(
+        attributes[:host],
+        attributes.fetch(:port,DEFAULT_PORT),
+        attributes[:user],
+        attributes[:password]
+      )
+    end
+
+    #
+    # Determines if the proxy settings are set.
+    #
+    # @return [Boolean]
+    #
+    def enabled?
+      !host.nil?
+    end
+
+    #
+    # Determines if the proxy is not set.
+    #
+    # @return [Boolean]
+    #
+    def disabled?
+      host.nil?
+    end
+
+  end
+end

data/lib/spidr/session_cache.rb

@@ -1,6 +1,9 @@
+require 'spidr/settings/proxy'
+require 'spidr/settings/timeouts'
 require 'spidr/spidr'
 
 require 'net/http'
+require 'openssl'
 
 module Spidr
   #
@@ -8,31 +11,44 @@ module Spidr
   #
   class SessionCache
 
-    # Proxy to use
-    attr_accessor :proxy
+    include Settings::Proxy
+    include Settings::Timeouts
 
     #
     # Creates a new session cache.
     #
-    # @param [Hash] proxy (Spidr.proxy)
+    # @param [Hash] options
+    #   Configuration options.
+    #
+    # @option [Hash] :proxy (Spidr.proxy)
     #   Proxy options.
     #
-    # @option proxy [String] :host
-    #   The host the proxy is running on.
+    # @option [Integer] :open_timeout (Spidr.open_timeout)
+    #   Optional open timeout.
     #
-    # @option proxy [Integer] :port
-    #   The port the proxy is running on.
+    # @option [Integer] :ssl_timeout (Spidr.ssl_timeout)
+    #   Optional ssl timeout.
     #
-    # @option proxy [String] :user
-    #   The user to authenticate as with the proxy.
+    # @option [Integer] :read_timeout (Spidr.read_timeout)
+    #   Optional read timeout.
     #
-    # @option proxy [String] :password
-    #   The password to authenticate with.
+    # @option [Integer] :continue_timeout (Spidr.continue_timeout)
+    #   Optional `Continue` timeout.
     #
-    # @since 0.2.2
+    # @option [Integer] :keep_alive_timeout (Spidr.keep_alive_timeout)
+    #   Optional `Keep-Alive` timeout.
     #
-    def initialize(proxy=Spidr.proxy)
-      @proxy    = proxy
+    # @since 0.6.0
+    #
+    def initialize(options={})
+      @proxy = options.fetch(:proxy,Spidr.proxy)
+
+      @open_timeout       = options.fetch(:open_timeout,Spidr.open_timeout)
+      @ssl_timeout        = options.fetch(:ssl_timeout,Spidr.ssl_timeout)
+      @read_timeout       = options.fetch(:read_timeout,Spidr.read_timeout)
+      @continue_timeout   = options.fetch(:continue_timeout,Spidr.continue_timeout)
+      @keep_alive_timeout = options.fetch(:keep_alive_timeout,Spidr.keep_alive_timeout)
+
       @sessions = {}
     end
 
@@ -52,7 +68,7 @@ module Spidr
       url = URI(url.to_s) unless url.kind_of?(URI)
 
       # session key
-      key = [url.scheme, url.host, url.port]
+      key = key_for(url)
 
       return @sessions.has_key?(key)
     end
@@ -71,19 +87,25 @@ module Spidr
       url = URI(url.to_s) unless url.kind_of?(URI)
 
       # session key
-      key = [url.scheme, url.host, url.port]
+      key = key_for(url)
 
       unless @sessions[key]
         session = Net::HTTP::Proxy(
-          @proxy[:host],
-          @proxy[:port],
-          @proxy[:user],
-          @proxy[:password]
+          @proxy.host,
+          @proxy.port,
+          @proxy.user,
+          @proxy.password
         ).new(url.host,url.port)
 
+        session.open_timeout       = @open_timeout       if @open_timeout
+        session.read_timeout       = @read_timeout       if @read_timeout
+        session.continue_timeout   = @continue_timeout   if @continue_timeout
+        session.keep_alive_timeout = @keep_alive_timeout if @keep_alive_timeout
+
         if url.scheme == 'https'
           session.use_ssl     = true
           session.verify_mode = OpenSSL::SSL::VERIFY_NONE
+          session.ssl_timeout = @ssl_timeout
           session.start
         end
 
@@ -108,7 +130,7 @@ module Spidr
       url = URI(url.to_s) unless url.kind_of?(URI)
 
       # session key
-      key = [url.scheme, url.host, url.port]
+      key = key_for(url)
 
       if (sess = @sessions[key])
         begin 
@@ -129,11 +151,10 @@ module Spidr
     # @since 0.2.2
     #
     def clear
-      @sessions.each_value do |sess|
+      @sessions.each_value do |session|
         begin
-          sess.finish
+          session.finish
         rescue IOError
-          nil
         end
       end
 
@@ -141,5 +162,20 @@ module Spidr
       return self
     end
 
+    private
+
+    #
+    # Creates a session key based on the URL.
+    #
+    # @param [URI::HTTP] url
+    #   The given URL.
+    #
+    # @return [Array]
+    #   The session key containing the scheme, host and port.
+    #
+    def key_for(url)
+      [url.scheme, url.host, url.port]
+    end
+
   end
 end

data/lib/spidr/settings.rb

@@ -0,0 +1,3 @@
+require 'spidr/settings/proxy'
+require 'spidr/settings/timeouts'
+require 'spidr/settings/user_agent'

data/lib/spidr/settings/proxy.rb

@@ -0,0 +1,61 @@
+require 'spidr/proxy'
+
+module Spidr
+  module Settings
+    #
+    # Methods for configuring a proxy.
+    #
+    # @since 0.6.0
+    #
+    module Proxy
+      #
+      # Proxy information used by all newly created Agent objects by default.
+      #
+      # @return [Spidr::Proxy]
+      #   The Spidr proxy information.
+      #
+      def proxy
+        @proxy ||= Spidr::Proxy.new
+      end
+
+      #
+      # Sets the proxy information used by Agent objects.
+      #
+      # @param [Spidr::Proxy, Hash, nil] new_proxy
+      #   The new proxy information.
+      #
+      # @option new_proxy [String] :host
+      #   The host-name of the proxy.
+      #
+      # @option new_proxy [Integer] :port (COMMON_PROXY_PORT)
+      #   The port of the proxy.
+      #
+      # @option new_proxy [String] :user
+      #   The user to authenticate with the proxy as.
+      #
+      # @option new_proxy [String] :password
+      #   The password to authenticate with the proxy.
+      #
+      # @return [Spidr::Proxy]
+      #   The new proxy information.
+      #
+      def proxy=(new_proxy)
+        @proxy = case new_proxy
+                 when Spidr::Proxy then new_proxy
+                 when Hash         then Spidr::Proxy.new(new_proxy)
+                 when nil          then Spidr::Proxy.new
+                 else
+                   raise(TypeError,"#{self.class}#{__method__} only accepts Proxy, Hash or nil")
+                 end
+      end
+
+      #
+      # Disables the proxy settings used by all newly created Agent objects.
+      #
+      def disable_proxy!
+        @proxy = Spidr::Proxy.new
+        return true
+      end
+    end
+  end
+end

data/lib/spidr/settings/timeouts.rb

@@ -0,0 +1,33 @@
+module Spidr
+  module Settings
+    #
+    # @since 0.6.0
+    #
+    module Timeouts
+      # Read timeout.
+      #
+      # @return [Integer, nil]
+      attr_accessor :read_timeout
+
+      # Open timeout.
+      #
+      # @return [Integer, nil]
+      attr_accessor :open_timeout
+
+      # SSL timeout.
+      #
+      # @return [Integer, nil]
+      attr_accessor :ssl_timeout
+
+      # `Continue` timeout.
+      #
+      # @return [Integer, nil]
+      attr_accessor :continue_timeout
+
+      # `Keep-Alive` timeout.
+      #
+      # @return [Integer, nil]
+      attr_accessor :keep_alive_timeout
+    end
+  end
+end

data/lib/spidr/settings/user_agent.rb

@@ -0,0 +1,14 @@
+module Spidr
+  module Settings
+    #
+    # @since 0.6.0
+    #
+    module UserAgent
+      # The User-Agent string used by all Agent objects by default.
+      #
+      # @return [String]
+      #   The Spidr User-Agent string.
+      attr_accessor :user_agent
+    end
+  end
+end

data/lib/spidr/spidr.rb

@@ -1,79 +1,12 @@
+require 'spidr/settings/proxy'
+require 'spidr/settings/timeouts'
+require 'spidr/settings/user_agent'
 require 'spidr/agent'
 
 module Spidr
-  # Common proxy port.
-  COMMON_PROXY_PORT = 8080
-
-  # Default proxy information.
-  DEFAULT_PROXY = {
-    host:     nil,
-    port:     COMMON_PROXY_PORT,
-    user:     nil,
-    password: nil
-  }
-
-  #
-  # Proxy information used by all newly created Agent objects by default.
-  #
-  # @return [Hash]
-  #   The Spidr proxy information.
-  #
-  def Spidr.proxy
-    @@spidr_proxy ||= DEFAULT_PROXY
-  end
-
-  #
-  # Sets the proxy information used by Agent objects.
-  #
-  # @param [Hash] new_proxy
-  #   The new proxy information.
-  #
-  # @option new_proxy [String] :host
-  #   The host-name of the proxy.
-  #
-  # @option new_proxy [Integer] :port (COMMON_PROXY_PORT)
-  #   The port of the proxy.
-  #
-  # @option new_proxy [String] :user
-  #   The user to authenticate with the proxy as.
-  #
-  # @option new_proxy [String] :password
-  #   The password to authenticate with the proxy.
-  #
-  # @return [Hash]
-  #   The new proxy information.
-  #
-  def Spidr.proxy=(new_proxy)
-    @@spidr_proxy = {port: COMMON_PROXY_PORT}.merge(new_proxy)
-  end
-
-  #
-  # Disables the proxy settings used by all newly created Agent objects.
-  #
-  def Spidr.disable_proxy!
-    @@spidr_proxy = DEFAULT_PROXY
-    return true
-  end
-
-  #
-  # The User-Agent string used by all Agent objects by default.
-  #
-  # @return [String]
-  #   The Spidr User-Agent string.
-  #
-  def Spidr.user_agent
-    @@spidr_user_agent ||= nil
-  end
-
-  #
-  # Sets the Spidr User-Agent string.
-  #
-  # @param [String] new_agent
-  #   The new User-Agent string.
-  #
-  def Spidr.user_agent=(new_agent)
-    @@spidr_user_agent = new_agent
-  end
+  extend Settings::Proxy
+  extend Settings::Timeouts
+  extend Settings::UserAgent
 
   #
   # Specifies whether `robots.txt` should be honored globally.
@@ -82,7 +15,7 @@ module Spidr
   #
   # @since 0.5.0
   #
-  def Spidr.robots?
+  def self.robots?
     @robots
   end
 
@@ -95,31 +28,34 @@ module Spidr
   #
   # @since 0.5.0
   #
-  def Spidr.robots=(mode)
+  def self.robots=(mode)
     @robots = mode
   end
 
   #
   # @see Agent.start_at
   #
-  def Spidr.start_at(url,options={},&block)
+  def self.start_at(url,options={},&block)
     Agent.start_at(url,options,&block)
   end
 
   #
   # @see Agent.host
   #
-  def Spidr.host(name,options={},&block)
+  def self.host(name,options={},&block)
     Agent.host(name,options,&block)
   end
 
   #
   # @see Agent.site
   #
-  def Spidr.site(url,options={},&block)
+  def self.site(url,options={},&block)
     Agent.site(url,options,&block)
   end
 
-  def Spidr.robots
+  # 
+  # @abstract
+  #
+  def self.robots
   end
 end