wgit 0.4.1 → 0.5.0

data/lib/wgit/document_extensions.rb

@@ -34,8 +34,8 @@ Wgit::Document.define_extension(
   '//meta[@name="keywords"]/@content',
   singleton: true,
   text_content_only: true
-) do |keywords, source|
-  if keywords && (source == :html)
+) do |keywords, _source, type|
+  if keywords && (type == :document)
     keywords = keywords.split(',')
     Wgit::Utils.process_arr(keywords)
   end

data/lib/wgit/indexer.rb

@@ -44,12 +44,16 @@ module Wgit
   #   inserted into the database allowing for prior manipulation.
   # @return [Integer] The total number of pages crawled within the website.
   def self.index_site(
-    url, connection_string: nil, insert_externals: true, &block
+    url, connection_string: nil, insert_externals: true,
+    allow_paths: nil, disallow_paths: nil, &block
   )
     url = Wgit::Url.parse(url)
     db = Wgit::Database.new(connection_string)
     indexer = Wgit::Indexer.new(db)
-    indexer.index_site(url, insert_externals: insert_externals, &block)
+    indexer.index_site(
+      url, insert_externals: insert_externals,
+           allow_paths: allow_paths, disallow_paths: disallow_paths, &block
+    )
   end
 
   # Convience method to index a single webpage using
@@ -215,10 +219,13 @@ the next iteration.")
     #   nil or false from the block to prevent the document from being saved
     #   into the database.
     # @return [Integer] The total number of webpages/documents indexed.
-    def index_site(url, insert_externals: true)
+    def index_site(
+      url, insert_externals: true, allow_paths: nil, disallow_paths: nil
+    )
+      crawl_opts = { allow_paths: allow_paths, disallow_paths: disallow_paths }
       total_pages_indexed = 0
 
-      ext_urls = @crawler.crawl_site(url) do |doc|
+      ext_urls = @crawler.crawl_site(url, crawl_opts) do |doc|
         result = true
         result = yield(doc) if block_given?
 
@@ -231,8 +238,8 @@ the next iteration.")
       @db.url?(url) ? @db.update(url) : @db.insert(url)
 
       if insert_externals && ext_urls
-        write_urls_to_db(ext_urls)
-        Wgit.logger.info("Found and saved #{ext_urls.length} external url(s)")
+        num_inserted_urls = write_urls_to_db(ext_urls)
+        Wgit.logger.info("Found and saved #{num_inserted_urls} external url(s)")
       end
 
       Wgit.logger.info("Crawled and saved #{total_pages_indexed} docs for the \
@@ -266,8 +273,8 @@ site: #{url}")
 
       ext_urls = document&.external_links
       if insert_externals && ext_urls
-        write_urls_to_db(ext_urls)
-        Wgit.logger.info("Found and saved #{ext_urls.length} external url(s)")
+        num_inserted_urls = write_urls_to_db(ext_urls)
+        Wgit.logger.info("Found and saved #{num_inserted_urls} external url(s)")
       end
 
       nil
@@ -315,14 +322,19 @@ site: #{url}")
     def write_urls_to_db(urls)
       count = 0
 
-      if urls.respond_to?(:each)
-        urls.each do |url|
-          @db.insert(url)
-          count += 1
-          Wgit.logger.info("Inserted url: #{url}")
-        rescue Mongo::Error::OperationFailure
-          Wgit.logger.info("Url already exists: #{url}")
+      return count unless urls.respond_to?(:each)
+
+      urls.each do |url|
+        if url.invalid?
+          Wgit.logger.info("Ignoring invalid external url: #{url}")
+          next
         end
+
+        @db.insert(url)
+        count += 1
+        Wgit.logger.info("Inserted external url: #{url}")
+      rescue Mongo::Error::OperationFailure
+        Wgit.logger.info("External url already exists: #{url}")
       end
 
       count

data/lib/wgit/response.rb

@@ -0,0 +1,144 @@
+module Wgit
+  # Response class representing a generic HTTP crawl response.
+  class Response
+    # The underlying HTTP adapter/library response object.
+    attr_accessor :adapter_response
+
+    # The HTML response body.
+    attr_reader   :body
+
+    # The HTTP response headers.
+    attr_reader   :headers
+
+    # The servers IP address.
+    attr_accessor :ip_address
+
+    # The redirections of the response.
+    attr_reader   :redirections
+
+    # The number of redirections for the response.
+    attr_reader   :redirect_count
+
+    # The HTTP response status code.
+    attr_reader   :status
+
+    # The total crawl/network time for the response.
+    attr_reader   :total_time
+
+    # The HTTP request URL.
+    attr_accessor :url
+
+    # Defaults some values and returns a "blank" Wgit::Response object.
+    def initialize
+      @body         = ''
+      @headers      = {}
+      @redirections = {}
+      @total_time   = 0.0
+    end
+
+    # Adds time to @total_time (incrementally).
+    #
+    # @param time [Float] The time to add to @total_time.
+    # @return [Float] @total_time's new value.
+    def add_total_time(time)
+      @total_time += (time || 0.0)
+    end
+
+    # Sets the HTML response body.
+    #
+    # @param str [String] The new HTML body.
+    # @return [String] @body's new value.
+    def body=(str)
+      @body = (str || '')
+    end
+
+    # Returns the HTML response body or nil (if it's empty).
+    #
+    # @return [String, NilClass] The HTML body or nil if empty.
+    def body_or_nil
+      @body.empty? ? nil : @body
+    end
+
+    # Returns true if the response isn't a #success? or a #redirect?
+    #
+    # @return [Boolean] True if failed, false otherwise.
+    def failure?
+      !success? && !redirect?
+    end
+
+    # Sets the headers Hash to the given value. The header keys are mapped
+    # to snake_cased Symbols for consistency.
+    #
+    # @param headers [Hash] The new response headers.
+    # @return [Hash] @headers's new value.
+    def headers=(headers)
+      return @headers = {} unless headers
+
+      @headers = headers.map do |k, v|
+        k = k.downcase.gsub('-', '_').to_sym
+        [k, v]
+      end.to_h
+    end
+
+    # Returns whether or not the response is 404 Not Found.
+    #
+    # @return [Boolean] True if 404 Not Found, false otherwise.
+    def not_found?
+      @status == 404
+    end
+
+    # Returns whether or not the response is 200 OK.
+    #
+    # @return [Boolean] True if 200 OK, false otherwise.
+    def ok?
+      @status == 200
+    end
+
+    # Returns whether or not the response is a 3xx Redirect.
+    #
+    # @return [Boolean] True if 3xx Redirect, false otherwise.
+    def redirect?
+      return false unless @status
+
+      @status.between?(300, 399)
+    end
+
+    # Returns the number of redirects this response has had.
+    #
+    # @return [Integer] The number of response redirects.
+    def redirect_count
+      @redirections.size
+    end
+
+    # Returns the size of the response body.
+    #
+    # @return [Integer] The response body size in bytes.
+    def size
+      @body.size
+    end
+
+    # Sets the HTML response status.
+    #
+    # @param int [Integer] The new response status.
+    # @return [Integer] @status' new value.
+    def status=(int)
+      @status = int.positive? ? int : nil
+    end
+
+    # Returns whether or not the response is a 2xx Success.
+    #
+    # @return [Boolean] True if 2xx Success, false otherwise.
+    def success?
+      return false unless @status
+
+      @status.between?(200, 299)
+    end
+
+    alias code       status
+    alias content    body
+    alias crawl_time total_time
+    alias to_s       body
+    alias redirects  redirections
+    alias length     size
+  end
+end

data/lib/wgit/url.rb

@@ -8,15 +8,19 @@ require 'addressable/uri'
 module Wgit
   # Class modeling a web based HTTP URL.
   #
-  # Can be an internal/relative link e.g. "about.html" or a full URL
+  # Can be an internal/relative link e.g. "about.html" or an absolute URL
   # e.g. "http://www.google.co.uk". Is a subclass of String and uses 'uri' and
   # 'addressable/uri' internally.
+  #
+  # Most of the methods in this class return new Wgit::Url instances making the
+  # method calls chainable e.g. url.omit_base.omit_fragment etc. The methods
+  # also try to be idempotent where possible.
   class Url < String
     include Assertable
 
     # Whether or not the Url has been crawled or not. A custom crawled= method
     # is provided by this class, overridding the default one.
-    attr_accessor :crawled
+    attr_reader :crawled
 
     # The Time stamp of when this Url was crawled.
     attr_accessor :date_crawled
@@ -110,7 +114,7 @@ module Wgit
 
     # Returns true if self is a relative Url; false if absolute.
     #
-    # All external links in a page are expected to have a protocol prefix e.g.
+    # All external links in a page are expected to have a scheme prefix e.g.
     # 'http://', otherwise the link is treated as an internal link (regardless
     # of whether it's valid or not). The only exception is if an opts arg is
     # provided and self is a page belonging to that arg type e.g. host; then
@@ -118,7 +122,7 @@ module Wgit
     #
     # @param opts [Hash] The options with which to check relativity. Only one
     #   opts param should be provided. The provided opts param Url must be
-    #   absolute and be prefixed with a protocol. Consider using the output of
+    #   absolute and be prefixed with a scheme. Consider using the output of
     #   Wgit::Url#to_base which should work unless it's nil.
     # @option opts [Wgit::Url, String] :base The Url base e.g.
     #   http://www.google.com/how which gives a base of
@@ -147,8 +151,10 @@ module Wgit
 
       type, url = opts.first
       url = Wgit::Url.new(url)
-      raise "Invalid opts param value, Url must be absolute and contain \
-protocol: #{url}" unless url.to_base
+      unless url.to_base
+        raise "Invalid opts param value, Url must be absolute and contain \
+protocol scheme: #{url}"
+      end
 
       case type
       when :base   # http://www.google.com
@@ -182,19 +188,29 @@ protocol: #{url}" unless url.to_base
       true
     end
 
-    # Concats self and path together before returning a new Url. Self is not
+    # Returns if self is an invalid (relative) HTTP Url or not.
+    #
+    # @return [Boolean] True if invalid, otherwise false.
+    def invalid?
+      !valid?
+    end
+
+    # Concats self and other together before returning a new Url. Self is not
     # modified.
     #
-    # @param path [Wgit::Url, String] The path to concat onto the end of self.
-    # @return [Wgit::Url] self + separator + path, separator depends on path.
-    def concat(path)
-      path = Wgit::Url.new(path)
-      raise 'path must be relative' unless path.relative?
+    # @param other [Wgit::Url, String] The other to concat to the end of self.
+    # @return [Wgit::Url] self + separator + other, separator depends on other.
+    def concat(other)
+      other = Wgit::Url.new(other)
+      raise 'other must be relative' unless other.relative?
+
+      other = other.omit_leading_slash
+      separator = other.start_with?('#') || other.start_with?('?') ? '' : '/'
 
-      path = path.without_leading_slash
-      separator = path.start_with?('#') || path.start_with?('?') ? '' : '/'
+      # We use to_s below to call String#+, not Wgit::Url#+ (alias for concat).
+      concatted = omit_trailing_slash.to_s + separator.to_s + other.to_s
 
-      Wgit::Url.new(without_trailing_slash + separator + path)
+      Wgit::Url.new(concatted)
     end
 
     # Normalises/escapes self and returns a new Wgit::Url. Self isn't modified.
@@ -204,21 +220,47 @@ protocol: #{url}" unless url.to_base
       Wgit::Url.new(@uri.normalize.to_s)
     end
 
-    # Modifies self by prefixing it with a protocol. Returns the url whether
-    # its been modified or not. The default protocol prefix is http://.
+    # Returns an absolute form of self within the context of doc. Doesn't
+    # modify the receiver.
     #
-    # @param protocol [Symbol] Either :http or :https.
-    # @return [Wgit::Url] The url with protocol prefix (having been modified).
-    def prefix_protocol(protocol: :http)
-      unless %i[http https].include?(protocol)
-        raise 'protocol must be :http or :https'
-      end
+    # If self is absolute then it's returned as is, making this method
+    # idempotent. The doc's <base> element is used if present, otherwise
+    # doc.url is used as the base; which is concatted with self.
+    #
+    # Typically used to build an absolute link obtained from a document e.g.
+    #
+    #   link = Wgit::Url.new('/favicon.png')
+    #   doc  = Wgit::Document.new('http://example.com')
+    #
+    #   link.prefix_base(doc) # => "http://example.com/favicon.png"
+    #
+    # @param doc [Wgit::Document] The doc whose base Url is concatted with
+    #   self.
+    # @raise [StandardError] If doc isn't a Wgit::Document or if `doc.base_url`
+    #   raises an Exception.
+    # @return [Wgit::Url] Self in absolute form.
+    def prefix_base(doc)
+      assert_type(doc, Wgit::Document)
 
-      unless start_with?('http://') || start_with?('https://')
-        protocol == :http ? replace("http://#{url}") : replace("https://#{url}")
-      end
+      absolute? ? self : doc.base_url(link: self).concat(self)
+    end
 
-      self
+    # Returns self having prefixed a protocol scheme. Doesn't modify receiver.
+    # Returns self even if absolute (with scheme); therefore is idempotent.
+    #
+    # @param protocol [Symbol] Either :http or :https.
+    # @return [Wgit::Url] Self with a protocol scheme prefix.
+    def prefix_scheme(protocol: :http)
+      return self if absolute?
+
+      case protocol
+      when :http
+        Wgit::Url.new("http://#{url}")
+      when :https
+        Wgit::Url.new("https://#{url}")
+      else
+        raise "protocol must be :http or :https, not :#{protocol}"
+      end
     end
 
     # Returns a Hash containing this Url's instance vars excluding @uri.
@@ -238,6 +280,13 @@ protocol: #{url}" unless url.to_base
       URI(normalize)
     end
 
+    # Returns the Addressable::URI object for this URL.
+    #
+    # @return [Addressable::URI] The Addressable::URI object of self.
+    def to_addressable_uri
+      @uri
+    end
+
     # Returns self.
     #
     # @return [Wgit::Url] This (self) Url.
@@ -245,10 +294,10 @@ protocol: #{url}" unless url.to_base
       self
     end
 
-    # Returns a new Wgit::Url containing just the scheme/protocol of this URL
+    # Returns a new Wgit::Url containing just the scheme of this URL
     # e.g. Given http://www.google.co.uk, http is returned.
     #
-    # @return [Wgit::Url, nil] Containing just the scheme/protocol or nil.
+    # @return [Wgit::Url, nil] Containing just the scheme or nil.
     def to_scheme
       scheme = @uri.scheme
       scheme ? Wgit::Url.new(scheme) : nil
@@ -281,9 +330,11 @@ protocol: #{url}" unless url.to_base
       domain ? Wgit::Url.new(domain.split('.').first) : nil
     end
 
-    # Returns only the base of this URL e.g. the protocol and host combined.
+    # Returns only the base of this URL e.g. the protocol scheme and host
+    # combined.
     #
-    # @return [Wgit::Url, nil] Base of self e.g. http://www.google.co.uk or nil.
+    # @return [Wgit::Url, nil] The base of self e.g. http://www.google.co.uk or
+    #   nil.
     def to_base
       return nil if @uri.scheme.nil? || @uri.host.nil?
 
@@ -302,7 +353,7 @@ protocol: #{url}" unless url.to_base
       return nil if path.nil? || path.empty?
       return Wgit::Url.new('/') if path == '/'
 
-      Wgit::Url.new(path).without_slashes
+      Wgit::Url.new(path).omit_slashes
     end
 
     # Returns the endpoint of this URL e.g. the bit after the host with any
@@ -324,16 +375,16 @@ protocol: #{url}" unless url.to_base
     # @return [Wgit::Url, nil] Containing just the query string or nil.
     def to_query
       query = @uri.query
-      query ? Wgit::Url.new("?#{query}") : nil
+      query ? Wgit::Url.new(query) : nil
     end
 
-    # Returns a new Wgit::Url containing just the anchor string of this URL
+    # Returns a new Wgit::Url containing just the fragment string of this URL
     # e.g. Given http://google.com#about, #about is returned.
     #
-    # @return [Wgit::Url, nil] Containing just the anchor string or nil.
-    def to_anchor
-      anchor = @uri.fragment
-      anchor ? Wgit::Url.new("##{anchor}") : nil
+    # @return [Wgit::Url, nil] Containing just the fragment string or nil.
+    def to_fragment
+      fragment = @uri.fragment
+      fragment ? Wgit::Url.new(fragment) : nil
     end
 
     # Returns a new Wgit::Url containing just the file extension of this URL
@@ -348,12 +399,27 @@ protocol: #{url}" unless url.to_base
       segs.length > 1 ? Wgit::Url.new(segs.last) : nil
     end
 
+    # Omits the given URL components from self and returns a new Wgit::Url.
+    #
+    # Calls Addressable::URI#omit underneath and creates a new Wgit::Url from
+    # the output. See the Addressable::URI docs for more information.
+    #
+    # @param components [*Symbol] One or more Symbols representing the URL
+    #   components to omit. The following components are supported: :scheme,
+    #   :user, :password, :userinfo, :host, :port, :authority, :path, :query,
+    #   :fragment.
+    # @return [Wgit::Url] Self's URL value with the given components omitted.
+    def omit(*components)
+      omitted = @uri.omit(*components)
+      Wgit::Url.new(omitted.to_s)
+    end
+
     # Returns a new Wgit::Url containing self without a trailing slash. Is
     # idempotent meaning self will always be returned regardless of whether
     # there's a trailing slash or not.
     #
     # @return [Wgit::Url] Self without a trailing slash.
-    def without_leading_slash
+    def omit_leading_slash
       start_with?('/') ? Wgit::Url.new(self[1..-1]) : self
     end
 
@@ -362,7 +428,7 @@ protocol: #{url}" unless url.to_base
     # there's a trailing slash or not.
     #
     # @return [Wgit::Url] Self without a trailing slash.
-    def without_trailing_slash
+    def omit_trailing_slash
       end_with?('/') ? Wgit::Url.new(chop) : self
     end
 
@@ -371,9 +437,9 @@ protocol: #{url}" unless url.to_base
     # present or not.
     #
     # @return [Wgit::Url] Self without leading or trailing slashes.
-    def without_slashes
-      without_leading_slash
-      .without_trailing_slash
+    def omit_slashes
+      omit_leading_slash
+        .omit_trailing_slash
     end
 
     # Returns a new Wgit::Url with the base (proto and host) removed e.g. Given
@@ -382,13 +448,13 @@ protocol: #{url}" unless url.to_base
     # Leading and trailing slashes are always stripped from the return value.
     #
     # @return [Wgit::Url] Self containing everything after the base.
-    def without_base
+    def omit_base
       base_url = to_base
-      without_base = base_url ? gsub(base_url, '') : self
+      omit_base = base_url ? gsub(base_url, '') : self
 
-      return self if ['', '/'].include?(without_base)
+      return self if ['', '/'].include?(omit_base)
 
-      Wgit::Url.new(without_base).without_slashes
+      Wgit::Url.new(omit_base).omit_slashes
     end
 
     # Returns a new Wgit::Url with the query string portion removed e.g. Given
@@ -398,26 +464,26 @@ protocol: #{url}" unless url.to_base
     # URL.
     #
     # @return [Wgit::Url] Self with the query string portion removed.
-    def without_query
+    def omit_query
       query = to_query
-      without_query_string = query ? gsub(query, '') : self
+      omit_query_string = query ? gsub("?#{query}", '') : self
 
-      Wgit::Url.new(without_query_string)
+      Wgit::Url.new(omit_query_string)
     end
 
-    # Returns a new Wgit::Url with the anchor portion removed e.g. Given
+    # Returns a new Wgit::Url with the fragment portion removed e.g. Given
     # http://google.com/search#about, http://google.com/search is
-    # returned. Self is returned as is if no anchor is present. A URL
-    # consisting of only an anchor e.g. '#about' will return an empty URL.
-    # This method assumes that the anchor is correctly placed at the very end
+    # returned. Self is returned as is if no fragment is present. A URL
+    # consisting of only a fragment e.g. '#about' will return an empty URL.
+    # This method assumes that the fragment is correctly placed at the very end
     # of the URL.
     #
-    # @return [Wgit::Url] Self with the anchor portion removed.
-    def without_anchor
-      anchor = to_anchor
-      without_anchor = anchor ? gsub(anchor, '') : self
+    # @return [Wgit::Url] Self with the fragment portion removed.
+    def omit_fragment
+      fragment = to_fragment
+      omit_fragment = fragment ? gsub("##{fragment}", '') : self
 
-      Wgit::Url.new(without_anchor)
+      Wgit::Url.new(omit_fragment)
     end
 
     # Returns true if self is a URL query string e.g. ?q=hello etc. Note this
@@ -428,35 +494,33 @@ protocol: #{url}" unless url.to_base
       start_with?('?')
     end
 
-    # Returns true if self is a URL anchor/fragment e.g. #top etc. Note this
-    # shouldn't be used to determine if self contains an anchor/fragment.
+    # Returns true if self is a URL fragment e.g. #top etc. Note this
+    # shouldn't be used to determine if self contains a fragment.
     #
-    # @return [Boolean] True if self is a anchor/fragment, false otherwise.
-    def anchor?
+    # @return [Boolean] True if self is a fragment, false otherwise.
+    def fragment?
       start_with?('#')
     end
 
-    alias crawled?         crawled
-    alias is_relative?     relative?
-    alias is_absolute?     absolute?
-    alias is_valid?        valid?
-    alias normalise        normalize
-    alias uri              to_uri
-    alias url              to_url
-    alias scheme           to_scheme
-    alias host             to_host
-    alias domain           to_domain
-    alias brand            to_brand
-    alias base             to_base
-    alias path             to_path
-    alias endpoint         to_endpoint
-    alias query            to_query
-    alias anchor           to_anchor
-    alias fragment         to_anchor
-    alias extension        to_extension
-    alias without_fragment without_anchor
-    alias is_query?        query?
-    alias is_anchor?       anchor?
-    alias fragment?        anchor?
+    alias +            concat
+    alias crawled?     crawled
+    alias normalise    normalize
+    alias is_relative? relative?
+    alias is_absolute? absolute?
+    alias is_valid?    valid?
+    alias is_query?    query?
+    alias is_fragment? fragment?
+    alias uri          to_uri
+    alias url          to_url
+    alias scheme       to_scheme
+    alias host         to_host
+    alias domain       to_domain
+    alias brand        to_brand
+    alias base         to_base
+    alias path         to_path
+    alias endpoint     to_endpoint
+    alias query        to_query
+    alias fragment     to_fragment
+    alias extension    to_extension
   end
 end