spidr 0.6.1 → 0.7.1

data/lib/spidr/page/html.rb

@@ -1,5 +1,8 @@
+# frozen_string_literal: true
+
+require_relative '../extensions/uri'
+
 require 'nokogiri'
-require 'spidr/extensions/uri'
 
 module Spidr
   class Page
@@ -105,7 +108,9 @@ module Spidr
     def each_redirect(&block)
       return enum_for(__method__) unless block
 
-      if (locations = @response.get_fields('Location'))
+      locations = @response.get_fields('Location')
+
+      unless (locations.nil? || locations.empty?)
         # Location headers override any meta-refresh redirects in the HTML
         locations.each(&block)
       else
@@ -175,34 +180,30 @@ module Spidr
     #
     # @since 0.3.0
     #
-    def each_link
+    def each_link(&block)
       return enum_for(__method__) unless block_given?
 
-      filter = lambda { |url|
-        yield url unless (url.nil? || url.empty?)
-      }
-
-      each_redirect(&filter) if is_redirect?
+      each_redirect(&block) if is_redirect?
 
       if (html? && doc)
-        doc.search('//a[@href]').each do |a|
-          filter.call(a.get_attribute('href'))
+        doc.search('//a[@href[string()]]').each do |a|
+          yield a.get_attribute('href')
         end
 
-        doc.search('//frame[@src]').each do |iframe|
-          filter.call(iframe.get_attribute('src'))
+        doc.search('//frame[@src[string()]]').each do |iframe|
+          yield iframe.get_attribute('src')
         end
 
-        doc.search('//iframe[@src]').each do |iframe|
-          filter.call(iframe.get_attribute('src'))
+        doc.search('//iframe[@src[string()]]').each do |iframe|
+          yield iframe.get_attribute('src')
         end
 
-        doc.search('//link[@href]').each do |link|
-          filter.call(link.get_attribute('href'))
+        doc.search('//link[@href[string()]]').each do |link|
+          yield link.get_attribute('href')
         end
 
-        doc.search('//script[@src]').each do |script|
-          filter.call(script.get_attribute('src'))
+        doc.search('//script[@src[string()]]').each do |script|
+          yield script.get_attribute('src')
         end
       end
     end
@@ -211,7 +212,7 @@ module Spidr
     # The links from within the page.
     #
     # @return [Array<String>]
-    #   All links within the HTML page, frame/iframe source URLs and any
+    #   All links within the HTML page, `frame`/`iframe` source URLs and any
     #   links in the `Location` header.
     #
     def links
@@ -267,7 +268,7 @@ module Spidr
       link    = link.to_s
       new_url = begin
                   url.merge(link)
-                rescue Exception
+                rescue URI::Error
                   return
                 end
 

data/lib/spidr/page/status_codes.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Spidr
   class Page
     #
@@ -22,16 +24,6 @@ module Spidr
 
     alias ok? is_ok?
 
-    #
-    # Determines if the response code is `308`.
-    #
-    # @return [Boolean]
-    #   Specifies whether the response code is `308`.
-    #
-    def timedout?
-      code == 308
-    end
-
     #
     # Determines if the response code is `400`.
     #
@@ -78,6 +70,18 @@ module Spidr
 
     alias missing? is_missing?
 
+    #
+    # Determines if the response code is `408`.
+    #
+    # @return [Boolean]
+    #   Specifies whether the response code is `408`.
+    #
+    def is_timedout?
+      code == 408
+    end
+
+    alias timedout? is_timedout?
+
     #
     # Determines if the response code is `500`.
     #
@@ -90,7 +94,7 @@ module Spidr
 
     #
     # Determines if the response code is `300`, `301`, `302`, `303`
-    # or `307`. Also checks for "soft" redirects added at the page 
+    # or `307`. Also checks for "soft" redirects added at the page
     # level by a meta refresh tag.
     #
     # @return [Boolean]

data/lib/spidr/page.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Spidr
   #
   # Represents a requested page from a website.
@@ -142,7 +144,7 @@ module Spidr
 
       return super(name,*arguments,&block)
     end
-  
+
   end
 end
 

data/lib/spidr/proxy.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Spidr
   #
   # @since 0.6.0
@@ -10,28 +12,20 @@ module Spidr
     #
     # Initializes the proxy.
     #
-    # @param [Hash] attributes
-    #   Attributes for the proxy.
-    #
-    # @option attributes [String] :host
+    # @param [String] host
     #   The host the proxy is running on.
     #
-    # @option attributes [Integer] :port
+    # @param [Integer] port
     #   The port the proxy is running on.
     #
-    # @option attributes [String] :user
+    # @param [String] user
     #   The user to authenticate as with the proxy.
     #
-    # @option attributes [String] :password
+    # @param [String] password
     #   The password to authenticate with.
     #
-    def initialize(attributes={})
-      super(
-        attributes[:host],
-        attributes.fetch(:port,DEFAULT_PORT),
-        attributes[:user],
-        attributes[:password]
-      )
+    def initialize(host: nil, port: DEFAULT_PORT, user: nil, password: nil)
+      super(host,port,user,password)
     end
 
     #

data/lib/spidr/rules.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Spidr
   #
   # The {Rules} class represents collections of acceptance and rejection
@@ -14,21 +16,18 @@ module Spidr
     #
     # Creates a new Rules object.
     #
-    # @param [Hash] options
-    #   Additional options.
-    #
-    # @option options [Array<String, Regexp, Proc>] :accept
+    # @param [Array<String, Regexp, Proc>, nil] accept
     #   The patterns to accept data with.
     #
-    # @option options [Array<String, Regexp, Proc>] :reject
+    # @param [Array<String, Regexp, Proc>, nil] reject
     #   The patterns to reject data with.
     #
-    def initialize(options={})
+    def initialize(accept: nil, reject: nil)
       @accept = []
       @reject = []
 
-      @accept += options[:accept] if options[:accept]
-      @reject += options[:reject] if options[:reject]
+      @accept += accept if accept
+      @reject += reject if reject
     end
 
     #

data/lib/spidr/session_cache.rb

@@ -1,6 +1,8 @@
-require 'spidr/settings/proxy'
-require 'spidr/settings/timeouts'
-require 'spidr/spidr'
+# frozen_string_literal: true
+
+require_relative 'settings/proxy'
+require_relative 'settings/timeouts'
+require_relative 'spidr'
 
 require 'net/http'
 require 'openssl'
@@ -17,37 +19,39 @@ module Spidr
     #
     # Creates a new session cache.
     #
-    # @param [Hash] options
-    #   Configuration options.
-    #
-    # @option [Hash] :proxy (Spidr.proxy)
+    # @param [Hash] proxy
     #   Proxy options.
     #
-    # @option [Integer] :open_timeout (Spidr.open_timeout)
-    #   Optional open timeout.
+    # @param [Integer] open_timeout
+    #   Optional open connection timeout.
     #
-    # @option [Integer] :ssl_timeout (Spidr.ssl_timeout)
-    #   Optional ssl timeout.
+    # @param [Integer] ssl_timeout
+    #   Optional SSL connection timeout.
     #
-    # @option [Integer] :read_timeout (Spidr.read_timeout)
+    # @param [Integer] read_timeout
     #   Optional read timeout.
     #
-    # @option [Integer] :continue_timeout (Spidr.continue_timeout)
+    # @param [Integer] continue_timeout
     #   Optional `Continue` timeout.
     #
-    # @option [Integer] :keep_alive_timeout (Spidr.keep_alive_timeout)
+    # @param [Integer] keep_alive_timeout
     #   Optional `Keep-Alive` timeout.
     #
     # @since 0.6.0
     #
-    def initialize(options={})
-      @proxy = options.fetch(:proxy,Spidr.proxy)
+    def initialize(proxy:              Spidr.proxy,
+                   open_timeout:       Spidr.open_timeout,
+                   ssl_timeout:        Spidr.ssl_timeout,
+                   read_timeout:       Spidr.read_timeout,
+                   continue_timeout:   Spidr.continue_timeout,
+                   keep_alive_timeout: Spidr.keep_alive_timeout)
+      self.proxy = proxy
 
-      @open_timeout       = options.fetch(:open_timeout,Spidr.open_timeout)
-      @ssl_timeout        = options.fetch(:ssl_timeout,Spidr.ssl_timeout)
-      @read_timeout       = options.fetch(:read_timeout,Spidr.read_timeout)
-      @continue_timeout   = options.fetch(:continue_timeout,Spidr.continue_timeout)
-      @keep_alive_timeout = options.fetch(:keep_alive_timeout,Spidr.keep_alive_timeout)
+      self.open_timeout       = open_timeout
+      self.ssl_timeout        = ssl_timeout
+      self.read_timeout       = read_timeout
+      self.continue_timeout   = continue_timeout
+      self.keep_alive_timeout = keep_alive_timeout
 
       @sessions = {}
     end
@@ -133,7 +137,7 @@ module Spidr
       key = key_for(url)
 
       if (sess = @sessions[key])
-        begin 
+        begin
           sess.finish
         rescue IOError
         end

data/lib/spidr/settings/proxy.rb

@@ -1,4 +1,8 @@
-require 'spidr/proxy'
+# frozen_string_literal: true
+
+require_relative '../proxy'
+
+require 'uri/http'
 
 module Spidr
   module Settings
@@ -21,7 +25,7 @@ module Spidr
       #
       # Sets the proxy information used by Agent objects.
       #
-      # @param [Spidr::Proxy, Hash, nil] new_proxy
+      # @param [Spidr::Proxy, Hash, URI::HTTP, String, nil] new_proxy
       #   The new proxy information.
       #
       # @option new_proxy [String] :host
@@ -41,11 +45,23 @@ module Spidr
       #
       def proxy=(new_proxy)
         @proxy = case new_proxy
-                 when Spidr::Proxy then new_proxy
-                 when Hash         then Spidr::Proxy.new(new_proxy)
-                 when nil          then Spidr::Proxy.new
+                 when Spidr::Proxy
+                   new_proxy
+                 when Hash
+                   Spidr::Proxy.new(**new_proxy)
+                 when String, URI::HTTP
+                   proxy_uri = URI(new_proxy)
+
+                   Spidr::Proxy.new(
+                      host:     proxy_uri.host,
+                      port:     proxy_uri.port,
+                      user:     proxy_uri.user,
+                      password: proxy_uri.password
+                   )
+                 when nil
+                   Spidr::Proxy.new
                  else
-                   raise(TypeError,"#{self.class}#{__method__} only accepts Proxy, Hash or nil")
+                   raise(TypeError,"#{self.class}#{__method__} only accepts Spidr::Proxy, URI::HTTP, Hash, or nil")
                  end
       end
 

data/lib/spidr/settings/timeouts.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Spidr
   module Settings
     #

data/lib/spidr/settings/user_agent.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Spidr
   module Settings
     #

data/lib/spidr/settings.rb

@@ -1,3 +1,5 @@
-require 'spidr/settings/proxy'
-require 'spidr/settings/timeouts'
-require 'spidr/settings/user_agent'
+# frozen_string_literal: true
+
+require_relative 'settings/proxy'
+require_relative 'settings/timeouts'
+require_relative 'settings/user_agent'

data/lib/spidr/spidr.rb

@@ -1,7 +1,9 @@
-require 'spidr/settings/proxy'
-require 'spidr/settings/timeouts'
-require 'spidr/settings/user_agent'
-require 'spidr/agent'
+# frozen_string_literal: true
+
+require_relative 'settings/proxy'
+require_relative 'settings/timeouts'
+require_relative 'settings/user_agent'
+require_relative 'agent'
 
 module Spidr
   extend Settings::Proxy
@@ -36,25 +38,34 @@ module Spidr
   #
   # @see Agent.start_at
   #
-  def self.start_at(url,options={},&block)
-    Agent.start_at(url,options,&block)
+  def self.start_at(url,**kwargs,&block)
+    Agent.start_at(url,**kwargs,&block)
   end
 
   #
   # @see Agent.host
   #
-  def self.host(name,options={},&block)
-    Agent.host(name,options,&block)
+  def self.host(name,**kwargs,&block)
+    Agent.host(name,**kwargs,&block)
+  end
+
+  #
+  # @see Agent.domain
+  #
+  # @since 0.7.0
+  #
+  def self.domain(name,**kwargs,&block)
+    Agent.domain(name,**kwargs,&block)
   end
 
   #
   # @see Agent.site
   #
-  def self.site(url,options={},&block)
-    Agent.site(url,options,&block)
+  def self.site(url,**kwargs,&block)
+    Agent.site(url,**kwargs,&block)
   end
 
-  # 
+  #
   # @abstract
   #
   def self.robots

data/lib/spidr/version.rb

@@ -1,4 +1,6 @@
+# frozen_string_literal: true
+
 module Spidr
   # Spidr version
-  VERSION = '0.6.1'
+  VERSION = '0.7.1'
 end

data/lib/spidr.rb

@@ -1,3 +1,5 @@
-require 'spidr/agent'
-require 'spidr/spidr'
-require 'spidr/version'
+# frozen_string_literal: true
+
+require_relative 'spidr/agent'
+require_relative 'spidr/spidr'
+require_relative 'spidr/version'