spidr 0.6.1 → 0.7.1

data/lib/spidr/agent.rb

@@ -1,14 +1,16 @@
-require 'spidr/settings/user_agent'
-require 'spidr/agent/sanitizers'
-require 'spidr/agent/filters'
-require 'spidr/agent/events'
-require 'spidr/agent/actions'
-require 'spidr/agent/robots'
-require 'spidr/page'
-require 'spidr/session_cache'
-require 'spidr/cookie_jar'
-require 'spidr/auth_store'
-require 'spidr/spidr'
+# frozen_string_literal: true
+
+require_relative 'settings/user_agent'
+require_relative 'agent/sanitizers'
+require_relative 'agent/filters'
+require_relative 'agent/events'
+require_relative 'agent/actions'
+require_relative 'agent/robots'
+require_relative 'page'
+require_relative 'session_cache'
+require_relative 'cookie_jar'
+require_relative 'auth_store'
+require_relative 'spidr'
 
 require 'openssl'
 require 'net/http'
@@ -19,12 +21,12 @@ module Spidr
 
     include Settings::UserAgent
 
-    # HTTP Host Header to use
+    # HTTP Host `Header` to use
     #
     # @return [String]
     attr_accessor :host_header
 
-    # HTTP Host Headers to use for specific hosts
+    # HTTP `Host` Headers to use for specific hosts
     #
     # @return [Hash{String,Regexp => String}]
     attr_reader :host_headers
@@ -96,70 +98,110 @@ module Spidr
     #
     # Creates a new Agent object.
     #
-    # @param [Hash] options
-    #   Additional options
+    # @param [String, nil] host_header
+    #   The HTTP `Host` header to use with each request.
     #
-    # @option options [Integer] :open_timeout (Spidr.open_timeout)
-    #   Optional open timeout.
+    # @param [Hash{String,Regexp => String}] host_headers
+    #   The HTTP `Host` headers to use for specific hosts.
     #
-    # @option options [Integer] :read_timeout (Spidr.read_timeout)
+    # @param [Hash{String => String}] default_headers
+    #   Default headers to set for every request.
+    #
+    # @param [String, nil] user_agent
+    #   The `User-Agent` string to send with each requests.
+    #
+    # @param [String, nil] referer
+    #   The `Referer` URL to send with each request.
+    #
+    # @param [Integer, nil] open_timeout
+    #   Optional open connection timeout.
+    #
+    # @param [Integer, nil] read_timeout
     #   Optional read timeout.
     #
-    # @option options [Integer] :ssl_timeout (Spidr.ssl_timeout)
-    #   Optional ssl timeout.
+    # @param [Integer, nil] ssl_timeout
+    #   Optional SSL connection timeout.
     #
-    # @option options [Integer] :continue_timeout (Spidr.continue_timeout)
+    # @param [Integer, nil] continue_timeout
     #   Optional continue timeout.
     #
-    # @option options [Integer] :keep_alive_timeout (Spidr.keep_alive_timeout)
-    #   Optional keep_alive timeout.
+    # @param [Integer, nil] keep_alive_timeout
+    #   Optional `Keep-Alive` timeout.
     #
-    # @option options [Hash] :proxy (Spidr.proxy)
+    # @param [Spidr::Proxy, Hash, URI::HTTP, String, nil] proxy
     #   The proxy information to use.
     #
-    # @option :proxy [String] :host
+    # @option proxy [String] :host
     #   The host the proxy is running on.
     #
-    # @option :proxy [Integer] :port
+    # @option proxy [Integer] :port (8080)
     #   The port the proxy is running on.
     #
-    # @option :proxy [String] :user
+    # @option proxy [String, nil] :user
     #   The user to authenticate as with the proxy.
     #
-    # @option :proxy [String] :password
+    # @option proxy [String, nil] :password
     #   The password to authenticate with.
     #
-    # @option options [Hash{String => String}] :default_headers
-    #   Default headers to set for every request.
+    # @param [Integer] delay
+    #   The number of seconds to pause between each request.
     #
-    # @option options [String] :host_header
-    #   The HTTP Host header to use with each request.
+    # @param [Integer, nil] limit
+    #   The maximum number of pages to visit.
     #
-    # @option options [Hash{String,Regexp => String}] :host_headers
-    #   The HTTP Host headers to use for specific hosts.
+    # @param [Integer, nil] max_depth
+    #   The maximum link depth to follow.
     #
-    # @option options [String] :user_agent (Spidr.user_agent)
-    #   The User-Agent string to send with each requests.
+    # @param [Set, Array, nil] queue
+    #   The initial queue of URLs to visit.
     #
-    # @option options [String] :referer
-    #   The Referer URL to send with each request.
+    # @param [Set, Array, nil] history
+    #   The initial list of visited URLs.
     #
-    # @option options [Integer] :delay (0)
-    #   The number of seconds to pause between each request.
+    # @param [Boolean] strip_fragments
+    #   Controls whether to strip the fragment components from the URLs.
     #
-    # @option options [Set, Array] :queue
-    #   The initial queue of URLs to visit.
+    # @param [Boolean] strip_query
+    #   Controls whether to strip the query components from the URLs.
     #
-    # @option options [Set, Array] :history
-    #   The initial list of visited URLs.
+    # @param [Array<String>] schemes
+    #   The list of acceptable URI schemes to visit.
+    #   The `https` scheme will be ignored if `net/https` cannot be loaded.
     #
-    # @option options [Integer] :limit
-    #   The maximum number of pages to visit.
+    # @param [String] host
+    #   The host-name to visit.
     #
-    # @option options [Integer] :max_depth
-    #   The maximum link depth to follow.
+    # @param [Array<String, Regexp, Proc>] hosts
+    #   The patterns which match the host-names to visit.
     #
-    # @option options [Boolean] :robots (Spidr.robots?)
+    # @param [Array<String, Regexp, Proc>] ignore_hosts
+    #   The patterns which match the host-names to not visit.
+    #
+    # @param [Array<Integer, Regexp, Proc>] ports
+    #   The patterns which match the ports to visit.
+    #
+    # @param [Array<Integer, Regexp, Proc>] ignore_ports
+    #   The patterns which match the ports to not visit.
+    #
+    # @param [Array<String, Regexp, Proc>] links
+    #   The patterns which match the links to visit.
+    #
+    # @param [Array<String, Regexp, Proc>] ignore_links
+    #   The patterns which match the links to not visit.
+    #
+    # @param [Array<String, Regexp, Proc>] urls
+    #   The patterns which match the URLs to visit.
+    #
+    # @param [Array<String, Regexp, Proc>] ignore_urls
+    #   The patterns which match the URLs to not visit.
+    #
+    # @param [Array<String, Regexp, Proc>] exts
+    #   The patterns which match the URI path extensions to visit.
+    #
+    # @param [Array<String, Regexp, Proc>] ignore_exts
+    #   The patterns which match the URI path extensions to not visit.
+    #
+    # @param [Boolean] robots
     #   Specifies whether `robots.txt` should be honored.
     #
     # @yield [agent]
@@ -169,58 +211,99 @@ module Spidr
     # @yieldparam [Agent] agent
     #   The newly created agent.
     #
-    # @see #initialize_sanitizers
-    # @see #initialize_filters
-    # @see #initialize_actions
-    # @see #initialize_events
-    #
-    def initialize(options={})
-      @host_header  = options[:host_header]
-      @host_headers = {}
-
-      if options[:host_headers]
-        @host_headers.merge!(options[:host_headers])
-      end
-
-      @default_headers = {}
-
-      if options[:default_headers]
-        @default_headers.merge!(options[:default_headers])
-      end
-
-      @user_agent = options.fetch(:user_agent,Spidr.user_agent)
-      @referer    = options[:referer]
-
-      @sessions   = SessionCache.new(options)
+    def initialize(# header keyword arguments
+                   host_header:        nil,
+                   host_headers:       {},
+                   default_headers:    {},
+                   user_agent:         Spidr.user_agent,
+                   referer:            nil,
+                   # session cache keyword arguments
+                   proxy:              Spidr.proxy,
+                   open_timeout:       Spidr.open_timeout,
+                   ssl_timeout:        Spidr.ssl_timeout,
+                   read_timeout:       Spidr.read_timeout,
+                   continue_timeout:   Spidr.continue_timeout,
+                   keep_alive_timeout: Spidr.keep_alive_timeout,
+                   # spidering controls keyword arguments
+                   delay:     0,
+                   limit:     nil,
+                   max_depth: nil,
+                   # history keyword arguments
+                   queue:   nil,
+                   history: nil,
+                   # sanitizer keyword arguments
+                   strip_fragments: true,
+                   strip_query:     false,
+                   # filtering keyword arguments
+                   schemes:      self.class.default_schemes,
+                   host:         nil,
+                   hosts:        nil,
+                   ignore_hosts: nil,
+                   ports:        nil,
+                   ignore_ports: nil,
+                   links:        nil,
+                   ignore_links: nil,
+                   urls:         nil,
+                   ignore_urls:  nil,
+                   exts:         nil,
+                   ignore_exts:  nil,
+                   # robots keyword arguments
+                   robots:       Spidr.robots?)
+      @host_header  = host_header
+      @host_headers = host_headers
+
+      @default_headers = default_headers
+
+      @user_agent = user_agent
+      @referer    = referer
+
+      @sessions   = SessionCache.new(
+        proxy:              proxy,
+        open_timeout:       open_timeout,
+        ssl_timeout:        ssl_timeout,
+        read_timeout:       read_timeout,
+        continue_timeout:   continue_timeout,
+        keep_alive_timeout: keep_alive_timeout
+      )
       @cookies    = CookieJar.new
       @authorized = AuthStore.new
 
       @running  = false
-      @delay    = options.fetch(:delay,0)
+      @delay    = delay
       @history  = Set[]
       @failures = Set[]
       @queue    = []
 
-      @limit     = options[:limit]
+      @limit     = limit
       @levels    = Hash.new(0)
-      @max_depth = options[:max_depth]
-
-      if options[:queue]
-        self.queue = options[:queue]
-      end
-
-      if options[:history]
-        self.history = options[:history]
-      end
-
-      initialize_sanitizers(options)
-      initialize_filters(options)
-      initialize_actions(options)
-      initialize_events(options)
-
-      if options.fetch(:robots,Spidr.robots?)
-        initialize_robots
-      end
+      @max_depth = max_depth
+
+      self.queue   = queue   if queue
+      self.history = history if history
+
+      initialize_sanitizers(
+        strip_fragments: strip_fragments,
+        strip_query:     strip_query
+      )
+
+      initialize_filters(
+        schemes:      schemes,
+        host:         host,
+        hosts:        hosts,
+        ignore_hosts: ignore_hosts,
+        ports:        ports,
+        ignore_ports: ignore_ports,
+        links:        links,
+        ignore_links: ignore_links,
+        urls:         urls,
+        ignore_urls:  ignore_urls,
+        exts:         exts,
+        ignore_exts:  ignore_exts
+      )
+      initialize_actions
+      initialize_events
+
+      initialize_robots if robots
 
       yield self if block_given?
     end
@@ -231,8 +314,8 @@ module Spidr
     # @param [URI::HTTP, String] url
     #   The URL to start spidering at.
     #
-    # @param [Hash] options
-    #   Additional options. See {Agent#initialize}.
+    # @param [Hash{Symbol => Object}] kwargs
+    #   Additional keyword arguments. See {Agent#initialize}.
     #
     # @yield [agent]
     #   If a block is given, it will be passed the newly created agent
@@ -241,12 +324,16 @@ module Spidr
     # @yieldparam [Agent] agent
     #   The newly created agent.
     #
+    # @return [Agent]
+    #   The created agent object.
+    #
     # @see #initialize
     # @see #start_at
     #
-    def self.start_at(url,options={},&block)
-      agent = new(options,&block)
+    def self.start_at(url,**kwargs,&block)
+      agent = new(**kwargs,&block)
       agent.start_at(url)
+      return agent
     end
 
     #
@@ -255,8 +342,8 @@ module Spidr
     # @param [URI::HTTP, String] url
     #   The web-site to spider.
     #
-    # @param [Hash] options
-    #   Additional options. See {Agent#initialize}.
+    # @param [Hash{Symbol => Object}] kwargs
+    #   Additional keyword arguments. See {Agent#initialize}.
     #
     # @yield [agent]
     #   If a block is given, it will be passed the newly created agent
@@ -265,13 +352,17 @@ module Spidr
     # @yieldparam [Agent] agent
     #   The newly created agent.
     #
+    # @return [Agent]
+    #   The created agent object.
+    #
     # @see #initialize
     #
-    def self.site(url,options={},&block)
+    def self.site(url,**kwargs,&block)
       url = URI(url)
 
-      agent = new(options.merge(host: url.host),&block)
+      agent = new(host: url.host, **kwargs, &block)
       agent.start_at(url)
+      return agent
     end
 
     #
@@ -280,8 +371,35 @@ module Spidr
     # @param [String] name
     #   The host-name to spider.
     #
-    # @param [Hash] options
-    #   Additional options. See {Agent#initialize}.
+    # @param [Hash{Symbol => Object}] kwargs
+    #   Additional keyword arguments. See {Agent#initialize}.
+    #
+    # @yield [agent]
+    #   If a block is given, it will be passed the newly created agent
+    #   before it begins spidering.
+    #
+    # @yieldparam [Agent] agent
+    #   The newly created agent.
+    #
+    # @return [Agent]
+    #   The created agent object.
+    #
+    # @see #initialize
+    #
+    def self.host(name,**kwargs,&block)
+      agent = new(host: name, **kwargs, &block)
+      agent.start_at(URI::HTTP.build(host: name, path: '/'))
+      return agent
+    end
+
+    #
+    # Creates a new agent and spiders the entire domain.
+    #
+    # @param [String] name
+    #   The top-level domain to spider.
+    #
+    # @param [Hash{Symbol => Object}] kwargs
+    #   Additional keyword arguments. See {Agent#initialize}.
     #
     # @yield [agent]
     #   If a block is given, it will be passed the newly created agent
@@ -290,11 +408,17 @@ module Spidr
     # @yieldparam [Agent] agent
     #   The newly created agent.
     #
+    # @return [Agent]
+    #   The created agent object.
+    #
     # @see #initialize
     #
-    def self.host(name,options={},&block)
-      agent = new(options.merge(host: name),&block)
+    # @since 0.7.0
+    #
+    def self.domain(name,**kwargs,&block)
+      agent = new(host: /(^|\.)#{Regexp.escape(name)}$/, **kwargs, &block)
       agent.start_at(URI::HTTP.build(host: name, path: '/'))
+      return agent
     end
 
     #
@@ -314,10 +438,10 @@ module Spidr
     #
     # Sets the proxy information that the agent uses.
     #
-    # @param [Proxy] new_proxy
+    # @param [Proxy, Hash, URI::HTTP, String, nil] new_proxy
     #   The new proxy information.
     #
-    # @return [Hash]
+    # @return [Proxy]
     #   The new proxy information.
     #
     # @see SessionCache#proxy=
@@ -534,7 +658,7 @@ module Spidr
     def enqueue(url,level=0)
       url = sanitize_url(url)
 
-      if (!(queued?(url)) && visit?(url))
+      if (!queued?(url) && visit?(url))
         link = url.to_s
 
         begin
@@ -633,7 +757,7 @@ module Spidr
     end
 
     #
-    # Visits a given URL, and enqueus the links recovered from the URL
+    # Visits a given URL, and enqueues the links recovered from the URL
     # to be visited later.
     #
     # @param [URI::HTTP, String] url

data/lib/spidr/auth_credential.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Spidr
   #
   # Represents HTTP Authentication credentials for a website.

data/lib/spidr/auth_store.rb

@@ -1,6 +1,8 @@
-require 'spidr/extensions/uri'
-require 'spidr/auth_credential'
-require 'spidr/page'
+# frozen_string_literal: true
+
+require_relative 'extensions/uri'
+require_relative 'auth_credential'
+require_relative 'page'
 
 require 'base64'
 
@@ -20,7 +22,7 @@ module Spidr
       @credentials = {}
     end
 
-    # 
+    #
     # Given a URL, return the most specific matching auth credential.
     #
     # @param [URI] url
@@ -54,7 +56,7 @@ module Spidr
       return nil
     end
 
-    # 
+    #
     # Add an auth credential to the store for supplied base URL.
     #
     # @param [URI] url
@@ -109,7 +111,7 @@ module Spidr
     # or `nil` if no authorization exists.
     #
     # @param [URI] url
-    #   The url.
+    #   The URL.
     #
     # @return [String, nil]
     #   The base64 encoded authorizatio string or `nil`.
@@ -122,7 +124,7 @@ module Spidr
       end
     end
 
-    # 
+    #
     # Clear the contents of the auth store.
     #
     # @return [AuthStore]

data/lib/spidr/cookie_jar.rb

@@ -1,4 +1,6 @@
-require 'spidr/page'
+# frozen_string_literal: true
+
+require_relative 'page'
 
 require 'set'
 
@@ -42,8 +44,8 @@ module Spidr
       @params.each(&block)
     end
 
-    # 
-    # Return all relevant cookies in a single string for the 
+    #
+    # Return all relevant cookies in a single string for the
     # named host or domain (in browser request format).
     #
     # @param [String] host
@@ -59,7 +61,7 @@ module Spidr
       @params[host] ||= {}
     end
 
-    # 
+    #
     # Add a cookie to the jar for a particular domain.
     #
     # @param [String] host
@@ -166,7 +168,7 @@ module Spidr
       return host_cookies
     end
 
-    # 
+    #
     # Clear out the jar, removing all stored cookies.
     #
     # @since 0.2.2

data/lib/spidr/extensions/uri.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'uri'
 require 'strscan'
 
@@ -58,7 +60,7 @@ module URI
     unless stack.empty?
       "#{leading_slash}#{stack.join('/')}#{trailing_slash}"
     else
-      '/'
+      String.new('/')
     end
   end
 end

data/lib/spidr/extensions.rb

@@ -1 +1,3 @@
-require 'spidr/extensions/uri'
+# frozen_string_literal: true
+
+require_relative 'extensions/uri'

data/lib/spidr/page/content_types.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Spidr
   class Page
     #
@@ -221,5 +223,56 @@ module Spidr
     def zip?
       is_content_type?('application/zip')
     end
+
+    #
+    # Determines if the page is a PNG image.
+    #
+    # @return [Boolean]
+    #   Specifies whether the page is a PNG image.
+    #
+    # @since 0.7.0
+    #
+    def png?
+      is_content_type?('image/png')
+    end
+
+    #
+    # Determines if the page is a GIF image.
+    #
+    # @return [Boolean]
+    #   Specifies whether the page is a GIF image.
+    #
+    # @since 0.7.0
+    #
+    def gif?
+      is_content_type?('image/gif')
+    end
+
+    #
+    # Determines if the page is a JPEG image.
+    #
+    # @return [Boolean]
+    #   Specifies whether the page is a JPEG image.
+    #
+    # @since 0.7.0
+    #
+    def jpeg?
+      is_content_type?('image/jpeg')
+    end
+
+    #
+    # Determines if the page is a ICO image.
+    #
+    # @return [Boolean]
+    #   Specifies whether the page is a ICO image.
+    #
+    # @since 0.7.0
+    #
+    def ico?
+      is_content_type?('image/x-icon') ||
+        is_content_type?('image/vnd.microsoft.icon')
+    end
+
+    alias icon? ico?
   end
 end

data/lib/spidr/page/cookies.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'set'
 
 module Spidr