spidr 0.6.0 → 0.7.0

data/lib/spidr/page/html.rb

@@ -105,7 +105,9 @@ module Spidr
     def each_redirect(&block)
       return enum_for(__method__) unless block
 
-      if (locations = @response.get_fields('Location'))
+      locations = @response.get_fields('Location')
+
+      unless (locations.nil? || locations.empty?)
         # Location headers override any meta-refresh redirects in the HTML
         locations.each(&block)
       else
@@ -175,34 +177,30 @@ module Spidr
     #
     # @since 0.3.0
     #
-    def each_link
+    def each_link(&block)
       return enum_for(__method__) unless block_given?
 
-      filter = lambda { |url|
-        yield url unless (url.nil? || url.empty?)
-      }
-
-      each_redirect(&filter) if is_redirect?
+      each_redirect(&block) if is_redirect?
 
       if (html? && doc)
-        doc.search('//a[@href]').each do |a|
-          filter.call(a.get_attribute('href'))
+        doc.search('//a[@href[string()]]').each do |a|
+          yield a.get_attribute('href')
         end
 
-        doc.search('//frame[@src]').each do |iframe|
-          filter.call(iframe.get_attribute('src'))
+        doc.search('//frame[@src[string()]]').each do |iframe|
+          yield iframe.get_attribute('src')
         end
 
-        doc.search('//iframe[@src]').each do |iframe|
-          filter.call(iframe.get_attribute('src'))
+        doc.search('//iframe[@src[string()]]').each do |iframe|
+          yield iframe.get_attribute('src')
         end
 
-        doc.search('//link[@href]').each do |link|
-          filter.call(link.get_attribute('href'))
+        doc.search('//link[@href[string()]]').each do |link|
+          yield link.get_attribute('href')
         end
 
-        doc.search('//script[@src]').each do |script|
-          filter.call(script.get_attribute('src'))
+        doc.search('//script[@src[string()]]').each do |script|
+          yield script.get_attribute('src')
         end
       end
     end
@@ -211,7 +209,7 @@ module Spidr
     # The links from within the page.
     #
     # @return [Array<String>]
-    #   All links within the HTML page, frame/iframe source URLs and any
+    #   All links within the HTML page, `frame`/`iframe` source URLs and any
     #   links in the `Location` header.
     #
     def links
@@ -271,7 +269,7 @@ module Spidr
                   return
                 end
 
-      if (path = new_url.path)
+      if (!new_url.opaque) && (path = new_url.path)
         # ensure that paths begin with a leading '/' for URI::FTP
         if (new_url.scheme == 'ftp' && !path.start_with?('/'))
           path.insert(0,'/')

data/lib/spidr/page/status_codes.rb

@@ -22,16 +22,6 @@ module Spidr
 
     alias ok? is_ok?
 
-    #
-    # Determines if the response code is `308`.
-    #
-    # @return [Boolean]
-    #   Specifies whether the response code is `308`.
-    #
-    def timedout?
-      code == 308
-    end
-
     #
     # Determines if the response code is `400`.
     #
@@ -78,6 +68,18 @@ module Spidr
 
     alias missing? is_missing?
 
+    #
+    # Determines if the response code is `408`.
+    #
+    # @return [Boolean]
+    #   Specifies whether the response code is `408`.
+    #
+    def is_timedout?
+      code == 408
+    end
+
+    alias timedout? is_timedout?
+
     #
     # Determines if the response code is `500`.
     #

data/lib/spidr/proxy.rb

@@ -10,28 +10,20 @@ module Spidr
     #
     # Initializes the proxy.
     #
-    # @param [Hash] attributes
-    #   Attributes for the proxy.
-    #
-    # @option attributes [String] :host
+    # @param [String] host
     #   The host the proxy is running on.
     #
-    # @option attributes [Integer] :port
+    # @param [Integer] port
     #   The port the proxy is running on.
     #
-    # @option attributes [String] :user
+    # @param [String] user
     #   The user to authenticate as with the proxy.
     #
-    # @option attributes [String] :password
+    # @param [String] password
     #   The password to authenticate with.
     #
-    def initialize(attributes={})
-      super(
-        attributes[:host],
-        attributes.fetch(:port,DEFAULT_PORT),
-        attributes[:user],
-        attributes[:password]
-      )
+    def initialize(host: nil, port: DEFAULT_PORT, user: nil, password: nil)
+      super(host,port,user,password)
     end
 
     #

data/lib/spidr/rules.rb

@@ -14,21 +14,18 @@ module Spidr
     #
     # Creates a new Rules object.
     #
-    # @param [Hash] options
-    #   Additional options.
-    #
-    # @option options [Array<String, Regexp, Proc>] :accept
+    # @param [Array<String, Regexp, Proc>, nil] accept
     #   The patterns to accept data with.
     #
-    # @option options [Array<String, Regexp, Proc>] :reject
+    # @param [Array<String, Regexp, Proc>, nil] reject
     #   The patterns to reject data with.
     #
-    def initialize(options={})
+    def initialize(accept: nil, reject: nil)
       @accept = []
       @reject = []
 
-      @accept += options[:accept] if options[:accept]
-      @reject += options[:reject] if options[:reject]
+      @accept += accept if accept
+      @reject += reject if reject
     end
 
     #

data/lib/spidr/session_cache.rb

@@ -17,37 +17,39 @@ module Spidr
     #
     # Creates a new session cache.
     #
-    # @param [Hash] options
-    #   Configuration options.
-    #
-    # @option [Hash] :proxy (Spidr.proxy)
+    # @param [Hash] proxy
     #   Proxy options.
     #
-    # @option [Integer] :open_timeout (Spidr.open_timeout)
-    #   Optional open timeout.
+    # @param [Integer] open_timeout
+    #   Optional open connection timeout.
     #
-    # @option [Integer] :ssl_timeout (Spidr.ssl_timeout)
-    #   Optional ssl timeout.
+    # @param [Integer] ssl_timeout
+    #   Optional SSL connection timeout.
     #
-    # @option [Integer] :read_timeout (Spidr.read_timeout)
+    # @param [Integer] read_timeout
     #   Optional read timeout.
     #
-    # @option [Integer] :continue_timeout (Spidr.continue_timeout)
+    # @param [Integer] continue_timeout
     #   Optional `Continue` timeout.
     #
-    # @option [Integer] :keep_alive_timeout (Spidr.keep_alive_timeout)
+    # @param [Integer] keep_alive_timeout
     #   Optional `Keep-Alive` timeout.
     #
     # @since 0.6.0
     #
-    def initialize(options={})
-      @proxy = options.fetch(:proxy,Spidr.proxy)
+    def initialize(proxy:              Spidr.proxy,
+                   open_timeout:       Spidr.open_timeout,
+                   ssl_timeout:        Spidr.ssl_timeout,
+                   read_timeout:       Spidr.read_timeout,
+                   continue_timeout:   Spidr.continue_timeout,
+                   keep_alive_timeout: Spidr.keep_alive_timeout)
+      self.proxy = proxy
 
-      @open_timeout       = options.fetch(:open_timeout,Spidr.open_timeout)
-      @ssl_timeout        = options.fetch(:ssl_timeout,Spidr.ssl_timeout)
-      @read_timeout       = options.fetch(:read_timeout,Spidr.read_timeout)
-      @continue_timeout   = options.fetch(:continue_timeout,Spidr.continue_timeout)
-      @keep_alive_timeout = options.fetch(:keep_alive_timeout,Spidr.keep_alive_timeout)
+      self.open_timeout       = open_timeout
+      self.ssl_timeout        = ssl_timeout
+      self.read_timeout       = read_timeout
+      self.continue_timeout   = continue_timeout
+      self.keep_alive_timeout = keep_alive_timeout
 
       @sessions = {}
     end
@@ -65,7 +67,7 @@ module Spidr
     #
     def active?(url)
       # normalize the url
-      url = URI(url.to_s) unless url.kind_of?(URI)
+      url = URI(url)
 
       # session key
       key = key_for(url)
@@ -84,7 +86,7 @@ module Spidr
     #
     def [](url)
       # normalize the url
-      url = URI(url.to_s) unless url.kind_of?(URI)
+      url = URI(url)
 
       # session key
       key = key_for(url)
@@ -127,7 +129,7 @@ module Spidr
     #
     def kill!(url)
       # normalize the url
-      url = URI(url.to_s) unless url.kind_of?(URI)
+      url = URI(url)
 
       # session key
       key = key_for(url)

data/lib/spidr/settings/proxy.rb

@@ -1,5 +1,7 @@
 require 'spidr/proxy'
 
+require 'uri/http'
+
 module Spidr
   module Settings
     #
@@ -21,7 +23,7 @@ module Spidr
       #
       # Sets the proxy information used by Agent objects.
       #
-      # @param [Spidr::Proxy, Hash, nil] new_proxy
+      # @param [Spidr::Proxy, Hash, URI::HTTP, String, nil] new_proxy
       #   The new proxy information.
       #
       # @option new_proxy [String] :host
@@ -41,11 +43,23 @@ module Spidr
       #
       def proxy=(new_proxy)
         @proxy = case new_proxy
-                 when Spidr::Proxy then new_proxy
-                 when Hash         then Spidr::Proxy.new(new_proxy)
-                 when nil          then Spidr::Proxy.new
+                 when Spidr::Proxy
+                   new_proxy
+                 when Hash
+                   Spidr::Proxy.new(**new_proxy)
+                 when String, URI::HTTP
+                   proxy_uri = URI(new_proxy)
+
+                   Spidr::Proxy.new(
+                      host:     proxy_uri.host,
+                      port:     proxy_uri.port,
+                      user:     proxy_uri.user,
+                      password: proxy_uri.password
+                   )
+                 when nil
+                   Spidr::Proxy.new
                  else
-                   raise(TypeError,"#{self.class}#{__method__} only accepts Proxy, Hash or nil")
+                   raise(TypeError,"#{self.class}#{__method__} only accepts Spidr::Proxy, URI::HTTP, Hash, or nil")
                  end
       end
 

data/lib/spidr/spidr.rb

@@ -16,6 +16,7 @@ module Spidr
   # @since 0.5.0
   #
   def self.robots?
+    @robots ||= false
     @robots
   end
 
@@ -35,22 +36,31 @@ module Spidr
   #
   # @see Agent.start_at
   #
-  def self.start_at(url,options={},&block)
-    Agent.start_at(url,options,&block)
+  def self.start_at(url,**kwargs,&block)
+    Agent.start_at(url,**kwargs,&block)
   end
 
   #
   # @see Agent.host
   #
-  def self.host(name,options={},&block)
-    Agent.host(name,options,&block)
+  def self.host(name,**kwargs,&block)
+    Agent.host(name,**kwargs,&block)
+  end
+
+  #
+  # @see Agent.domain
+  #
+  # @since 0.7.0
+  #
+  def self.domain(name,options={},&block)
+    Agent.domain(name,options,&block)
   end
 
   #
   # @see Agent.site
   #
-  def self.site(url,options={},&block)
-    Agent.site(url,options,&block)
+  def self.site(url,**kwargs,&block)
+    Agent.site(url,**kwargs,&block)
   end
 
   # 

data/lib/spidr/version.rb

@@ -1,4 +1,4 @@
 module Spidr
   # Spidr version
-  VERSION = '0.6.0'
+  VERSION = '0.7.0'
 end