wgit 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d9d045d2dd7f570db1811bafab1ac244103cc359033efd9279323c795a67bb9f
-  data.tar.gz: 996801763a6576ede812e2edd7d201ceb34b2135548a365b748f953e7df40db9
+  metadata.gz: 0db518346d1a939c13e689856a4ae9946ead17f188185529bce7fc1f97b84fba
+  data.tar.gz: aee59fe627767c45736e5ae518fe46c46ca9de13e1cd6bed23c0dc5b313aa1e6
 SHA512:
-  metadata.gz: e0dfe907c599c320377464aec927b24700d0e9e17d7bb37b4903715af63cbf031dc5983cd6749b1d90353cbcffc0d71e76eb2a0f8c0ba77b3b03f2d51ca9634f
-  data.tar.gz: bade693ab5b32bf8a16747233356307fe489798855133b8e16b3a907d38f8fd9ecfadadab0273d1c0767106bcf85027c64ebb6f86dc38660240200d5fef07377
+  metadata.gz: 5c29a7dc084742132241ec17118eb8dccf39ae57c57aa3364a01c948c4ed1a3304dd1510324b2f3b5d7c9d06ea04802524f6cd9eae00e2f68127e456b8cd9d17
+  data.tar.gz: 80cae11e47d806bcb14cc77e2628d533e9b466d7fc43d7e6e4fd79b3a65c51df5cbcd6886b62eab88ceb6176d8fdaaf91e45bddd5de3dabae00b245d2cb53968

data/lib/wgit/crawler.rb

@@ -4,8 +4,7 @@ require_relative 'url'
 require_relative 'document'
 require_relative 'utils'
 require_relative 'assertable'
-require 'net/http' # Requires 'uri'.
-require 'benchmark'
+require 'typhoeus'
 
 module Wgit
   # The Crawler class provides a means of crawling web based HTTP Wgit::Url's,
@@ -17,15 +16,24 @@ module Wgit
     # disable redirects completely.
     attr_accessor :redirect_limit
 
-    # The Net::HTTPResponse of the most recently crawled URL or nil.
+    # The maximum amount of time (in seconds) a crawl request has to complete
+    # before raising an error. Set to 0 to disable time outs completely.
+    attr_accessor :time_out
+
+    # The Typhoeus::Response of the most recently crawled URL or nil.
+    # See https://rubydoc.info/gems/typhoeus/Typhoeus/Response for more info.
     attr_reader :last_response
 
     # Initializes and returns a Wgit::Crawler instance.
     #
     # @param redirect_limit [Integer] The amount of allowed redirects before
     #   raising an error. Set to 0 to disable redirects completely.
-    def initialize(redirect_limit: 5)
+    # @param time_out [Integer, Float] The maximum amount of time (in seconds)
+    #   a crawl request has to complete before raising an error. Set to 0 to
+    #   disable time outs completely.
+    def initialize(redirect_limit: 5, time_out: 5)
       @redirect_limit = redirect_limit
+      @time_out       = time_out
     end
 
     # Crawls an entire website's HTML pages by recursively going through
@@ -103,7 +111,7 @@ module Wgit
       doc
     end
 
-    # Crawl the url returning the response Wgit::Document or nil if an error
+    # Crawl the url returning the response Wgit::Document or nil, if an error
     # occurs.
     #
     # @param url [Wgit::Url] The Url to crawl; which will likely be modified.
@@ -141,13 +149,12 @@ module Wgit
 
     protected
 
-    # This method calls Wgit::Crawler#resolve to obtain the page HTML, handling
-    # any errors that arise and setting the @last_response. Errors or any
-    # HTTP response that doesn't return a HTML body will be ignored and nil
-    # will be returned; otherwise, the HTML String is returned.
+    # Fetches the url HTML String or nil. Handles any errors that arise
+    # and sets the @last_response. Errors or any HTTP response that doesn't
+    # return a HTML body will be ignored, returning nil.
     #
-    # @param url [Wgit::Url] The URL to fetch the HTML for. This Url object
-    #   will likely be modified as a result of the fetch/crawl.
+    # @param url [Wgit::Url] The URL to fetch. This Url object is passed by
+    #   reference and gets modified as a result of the fetch/crawl.
     # @param follow_external_redirects [Boolean] Whether or not to follow
     #   an external redirect. False will return nil for such a crawl. If false,
     #   you must also provide a `host:` parameter.
@@ -159,16 +166,15 @@ module Wgit
     # @return [String, nil] The crawled HTML or nil if the crawl was
     #   unsuccessful.
     def fetch(url, follow_external_redirects: true, host: nil)
-      crawl_duration = nil
       response       = nil
+      crawl_duration = nil
 
-      crawl_duration = Benchmark.measure do
-        response = resolve(
-          url,
-          follow_external_redirects: follow_external_redirects,
-          host: host
-        )
-      end.real
+      response = resolve(
+        url,
+        follow_external_redirects: follow_external_redirects,
+        host: host
+      )
+      crawl_duration = response.total_time
 
       response.body.empty? ? nil : response.body
     rescue StandardError => e
@@ -181,10 +187,10 @@ module Wgit
       @last_response     = response
     end
 
-    # The resolve method performs a HTTP GET to obtain the HTML response. The
-    # Net::HTTPResponse will be returned or an error raised.
+    # Resolves the url by handling any redirects. The response object will be
+    # returned or an error raised.
     #
-    # @param url [Wgit::Url] The URL to fetch the HTML from.
+    # @param url [Wgit::Url] The URL to resolve.
     # @param follow_external_redirects [Boolean] Whether or not to follow
     #   an external redirect. If false, you must also provide a `host:`
     #   parameter.
@@ -193,37 +199,66 @@ module Wgit
     #   absolute and contain a protocol prefix. For example, a `host:` of
     #   'http://www.example.com' will only allow redirects for Urls with a
     #   `to_host` value of 'www.example.com'.
-    # @raise [StandardError] If !url.respond_to? :to_uri or a redirect isn't
-    #   allowed.
-    # @return [Net::HTTPResponse] The HTTP response of the GET request.
+    # @raise [StandardError] If a redirect isn't allowed etc.
+    # @return [Typhoeus::Response] The HTTP response of the GET request.
     def resolve(url, follow_external_redirects: true, host: nil)
-      raise 'url must respond to :to_uri' unless url.respond_to?(:to_uri)
-
+      response       = nil
       redirect_count = 0
-      response = nil
+      total_net_time = 0.0
 
       loop do
-        response = Net::HTTP.get_response(url.to_uri)
-        break unless response.is_a?(Net::HTTPRedirection)
+        response = get_response(url)
+        total_net_time += response.total_time if response.total_time
 
-        location = Wgit::Url.new(response.fetch('location', ''))
+        # Break unless it's a redirect.
+        break unless (response.code >= 300) && (response.code < 400)
+
+        # Handle response 'Location' header.
+        location = Wgit::Utils.fetch(response.headers, :location, '')
+        location = Wgit::Url.new(location)
         raise 'Encountered redirect without Location header' if location.empty?
 
         yield(url, response, location) if block_given?
 
+        # Handle redirect logic.
         raise "External redirect not allowed - Redirected to: \
 '#{location}', which is outside of host: '#{host}'" \
-        if !follow_external_redirects && !location.is_relative?(host: host)
+        if !follow_external_redirects && !location.relative?(host: host)
 
-        raise "Too many redirects: #{redirect_count}" \
+        raise "Too many redirects, exceeded: #{redirect_count}" \
         if redirect_count >= @redirect_limit
 
         redirect_count += 1
 
-        location = url.to_base.concat(location) if location.is_relative?
+        # Process the location to be crawled next.
+        location = url.to_base.concat(location) if location.relative?
         url.replace(location) # Update the url on redirect.
       end
 
+      response.options[:redirect_count] = redirect_count
+      response.options[:total_time]     = total_net_time
+
+      response
+    end
+
+    # Performs a HTTP GET request and returns the response.
+    #
+    # @param url [String] The url to GET. Will call url#normalize if possible.
+    # @raise [StandardError] If a response can't be obtained.
+    # @return [Typhoeus::Response] The HTTP response of the GET request.
+    def get_response(url)
+      url = url.normalize if url.respond_to?(:normalize)
+
+      opts = {
+        followlocation: false, timeout: @time_out, accept_encoding: 'gzip'
+      }
+
+      response = Typhoeus.get(url, opts)
+
+      # Handle response status code.
+      raise "No response (within timeout: #{@time_out} second(s))" \
+      if response.code == 0
+
       response
     end
 

data/lib/wgit/database/database.rb

@@ -23,6 +23,8 @@ module Wgit
     # Initializes a connected database client using the provided
     # connection_string or ENV['WGIT_CONNECTION_STRING'].
     #
+    # @param connection_string [String] The connection string needed to connect
+    #   to the database.
     # @raise [StandardError] If a connection string isn't provided, either as a
     #   parameter or via the environment.
     def initialize(connection_string = nil)
@@ -36,6 +38,10 @@ module Wgit
 
     # A class alias for Database.new.
     #
+    # @param connection_string [String] The connection string needed to connect
+    #   to the database.
+    # @raise [StandardError] If a connection string isn't provided, either as a
+    #   parameter or via the environment.
     # @return [Wgit::Database] The connected database client.
     def self.connect(connection_string = nil)
       new(connection_string)
@@ -43,6 +49,8 @@ module Wgit
 
     # Initializes a connected database client using the connection string.
     #
+    # @param connection_string [String] The connection string needed to connect
+    #   to the database.
     # @raise [StandardError] If a connection cannot be established.
     # @return [Mong::Client] The connected MongoDB client.
     def self.establish_connection(connection_string)

data/lib/wgit/document.rb

@@ -224,11 +224,11 @@ module Wgit
     # @return [Wgit::Url] The base URL of this Document e.g.
     #   'http://example.com/public'.
     def base_url(link: nil)
-      get_base = -> { @base.is_relative? ? @url.to_base.concat(@base) : @base }
+      get_base = -> { @base.relative? ? @url.to_base.concat(@base) : @base }
 
       if link
         link = Wgit::Url.new(link)
-        raise "link must be relative: #{link}" unless link.is_relative?
+        raise "link must be relative: #{link}" unless link.relative?
 
         if link.is_anchor? || link.is_query?
           base_url = @base ? get_base.call : @url
@@ -339,7 +339,7 @@ module Wgit
       return [] if @links.empty?
 
       links = @links
-              .select { |link| link.is_relative?(host: @url.to_base) }
+              .select { |link| link.relative?(host: @url.to_base) }
               .map(&:without_base)
               .map do |link| # Map @url.to_host into / as it's a duplicate.
         link.to_host == @url.to_host ? Wgit::Url.new('/') : link
@@ -365,7 +365,7 @@ module Wgit
       return [] if @links.empty?
 
       links = @links
-              .reject { |link| link.is_relative?(host: @url.to_base) }
+              .reject { |link| link.relative?(host: @url.to_base) }
               .map(&:without_trailing_slash)
 
       Wgit::Utils.process_arr(links)
@@ -597,6 +597,7 @@ module Wgit
       end
     end
 
+    alias statistics             stats
     alias internal_urls          internal_links
     alias internal_absolute_urls internal_absolute_links
     alias external_urls          external_links

data/lib/wgit/url.rb

@@ -189,7 +189,7 @@ protocol: #{url}" unless url.to_base
     # @return [Wgit::Url] self + separator + path, separator depends on path.
     def concat(path)
       path = Wgit::Url.new(path)
-      raise 'path must be relative' unless path.is_relative?
+      raise 'path must be relative' unless path.relative?
 
       path = path.without_leading_slash
       separator = path.start_with?('#') || path.start_with?('?') ? '' : '/'

data/lib/wgit/utils.rb

@@ -47,6 +47,32 @@ module Wgit
       obj_or_objs
     end
 
+    # An improved Hash :fetch method which checks for multiple formats of the
+    # given key and returns the value, or the default value (nil unless
+    # provided).
+    #
+    # For example, if key == :foo, hash is searched for:
+    # :foo, 'foo', 'Foo', 'FOO' in that order. The first value found is
+    # returned. If no value is found, the default value is returned.
+    #
+    # @param hash [Hash] The Hash to search within.
+    # @param key [Symbol, String] The key with which to search hash.
+    # @param default [Object] The default value to be returned if hash[key]
+    #   doesn't exist.
+    # @return [Object] The value found at hash[key] or the default value.
+    def self.fetch(hash, key, default = nil)
+      key = key.to_s.downcase
+
+      # Try (in order): :foo, 'foo', 'Foo', 'FOO'.
+      [key.to_sym, key, key.capitalize, key.upcase].each do |k|
+        value = hash[k]
+
+        return value if value
+      end
+
+      default
+    end
+
     # Formats the sentence (modifies the receiver) and returns its value.
     # The formatting is essentially to shorten the sentence and ensure that
     # the index is present somewhere in the sentence. Used for search query

data/lib/wgit/version.rb

@@ -5,5 +5,5 @@
 # @author Michael Telford
 module Wgit
   # The current gem version of Wgit.
-  VERSION = '0.3.0'
+  VERSION = '0.4.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: wgit
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Michael Telford
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-10-08 00:00:00.000000000 Z
+date: 2019-10-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -53,61 +53,47 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 1.10.3
 - !ruby/object:Gem::Dependency
-  name: byebug
+  name: typhoeus
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '10.0'
-- !ruby/object:Gem::Dependency
-  name: dotenv
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.5'
-  type: :development
+        version: 1.3.1
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.5'
+        version: 1.3.1
 - !ruby/object:Gem::Dependency
-  name: httplog
+  name: byebug
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '10.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '10.0'
 - !ruby/object:Gem::Dependency
-  name: inch
+  name: dotenv
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: '2.5'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: '2.5'
 - !ruby/object:Gem::Dependency
   name: maxitest
   requirement: !ruby/object:Gem::Requirement