wgit 0.5.1 → 0.10.0

data/lib/wgit/response.rb

@@ -1,5 +1,5 @@
 module Wgit
-  # Response class representing a generic HTTP crawl response.
+  # Response class modeling a generic HTTP GET response.
   class Response
     # The underlying HTTP adapter/library response object.
     attr_accessor :adapter_response
@@ -56,11 +56,11 @@ module Wgit
       @body.empty? ? nil : @body
     end
 
-    # Returns true if the response isn't a #success? or a #redirect?
+    # Returns whether or not a server response is absent.
     #
-    # @return [Boolean] True if failed, false otherwise.
+    # @return [Boolean] True if the status is nil or < 1, false otherwise.
     def failure?
-      !success? && !redirect?
+      !success?
     end
 
     # Sets the headers Hash to the given value. The header keys are mapped
@@ -69,7 +69,10 @@ module Wgit
     # @param headers [Hash] The new response headers.
     # @return [Hash] @headers's new value.
     def headers=(headers)
-      return @headers = {} unless headers
+      unless headers
+        @headers = {}
+        return
+      end
 
       @headers = headers.map do |k, v|
         k = k.downcase.gsub('-', '_').to_sym
@@ -122,20 +125,20 @@ module Wgit
       @status = int.positive? ? int : nil
     end
 
-    # Returns whether or not the response is a 2xx Success.
+    # Returns whether or not a server response is present.
     #
-    # @return [Boolean] True if 2xx Success, false otherwise.
+    # @return [Boolean] True if the status is > 0, false otherwise.
     def success?
       return false unless @status
 
-      @status.between?(200, 299)
+      @status.positive?
     end
 
-    alias code       status
-    alias content    body
-    alias crawl_time total_time
-    alias to_s       body
-    alias redirects  redirections
-    alias length     size
+    alias code           status
+    alias content        body
+    alias crawl_duration total_time
+    alias to_s           body
+    alias redirects      redirections
+    alias length         size
   end
 end

data/lib/wgit/url.rb

@@ -6,20 +6,20 @@ require 'uri'
 require 'addressable/uri'
 
 module Wgit
-  # Class modeling a web based HTTP URL.
+  # Class modeling/serialising a web based HTTP URL.
   #
   # Can be an internal/relative link e.g. "about.html" or an absolute URL
-  # e.g. "http://www.google.co.uk". Is a subclass of String and uses 'uri' and
-  # 'addressable/uri' internally.
+  # e.g. "http://www.google.co.uk". Is a subclass of String and uses `URI` and
+  # `addressable/uri` internally for parsing.
   #
-  # Most of the methods in this class return new Wgit::Url instances making the
-  # method calls chainable e.g. url.omit_base.omit_fragment etc. The methods
-  # also try to be idempotent where possible.
+  # Most of the methods in this class return new `Wgit::Url` instances making
+  # the method calls chainable e.g. `url.omit_base.omit_fragment` etc. The
+  # methods also try to be idempotent where possible.
   class Url < String
     include Assertable
 
     # Whether or not the Url has been crawled or not. A custom crawled= method
-    # is provided by this class, overridding the default one.
+    # is provided by this class.
     attr_reader :crawled
 
     # The Time stamp of when this Url was crawled.
@@ -28,10 +28,10 @@ module Wgit
     # The duration of the crawl for this Url (in seconds).
     attr_accessor :crawl_duration
 
-    # Initializes a new instance of Wgit::Url which represents a web based
+    # Initializes a new instance of Wgit::Url which models a web based
     # HTTP URL.
     #
-    # @param url_or_obj [String, Wgit::Url, Object#fetch#[]] Is either a String
+    # @param url_or_obj [String, Wgit::Url, #fetch#[]] Is either a String
     #   based URL or an object representing a Database record e.g. a MongoDB
     #   document/object.
     # @param crawled [Boolean] Whether or not the HTML of the URL's web page
@@ -90,6 +90,23 @@ module Wgit
       obj.is_a?(Wgit::Url) ? obj : new(obj)
     end
 
+    # Returns a Wgit::Url instance from Wgit::Url.parse, or nil if obj cannot
+    # be parsed successfully e.g. the String is invalid.
+    #
+    # Use this method when you can't gaurentee that obj is parsable as a URL.
+    # See Wgit::Url.parse for more information.
+    #
+    # @param obj [Object] The object to parse, which #is_a?(String).
+    # @raise [StandardError] If obj.is_a?(String) is false.
+    # @return [Wgit::Url] A Wgit::Url instance or nil (if obj is invalid).
+    def self.parse?(obj)
+      parse(obj)
+    rescue Addressable::URI::InvalidURIError
+      Wgit.logger.debug("Wgit::Url.parse?('#{obj}') exception: \
+Addressable::URI::InvalidURIError")
+      nil
+    end
+
     # Sets the @crawled instance var, also setting @date_crawled for
     # convenience.
     #
@@ -98,8 +115,6 @@ module Wgit
     def crawled=(bool)
       @crawled      = bool
       @date_crawled = bool ? Wgit::Utils.time_stamp : nil
-
-      bool
     end
 
     # Overrides String#replace setting the new_url @uri and String value.
@@ -114,33 +129,40 @@ module Wgit
 
     # Returns true if self is a relative Url; false if absolute.
     #
-    # All external links in a page are expected to have a scheme prefix e.g.
-    # 'http://', otherwise the link is treated as an internal link (regardless
+    # An absolute URL must have a scheme prefix e.g.
+    # 'http://', otherwise the URL is regarded as being relative (regardless
     # of whether it's valid or not). The only exception is if an opts arg is
     # provided and self is a page belonging to that arg type e.g. host; then
     # the link is relative.
     #
+    # @example
+    #   url = Wgit::Url.new('http://example.com/about')
+    #
+    #   url.relative? # => false
+    #   url.relative?(host: 'http://example.com') # => true
+    #
     # @param opts [Hash] The options with which to check relativity. Only one
     #   opts param should be provided. The provided opts param Url must be
     #   absolute and be prefixed with a scheme. Consider using the output of
-    #   Wgit::Url#to_base which should work unless it's nil.
-    # @option opts [Wgit::Url, String] :base The Url base e.g.
-    #   http://www.google.com/how which gives a base of
-    #   'http://www.google.com'.
+    #   Wgit::Url#to_origin which should work (unless it's nil).
+    # @option opts [Wgit::Url, String] :origin The Url origin e.g.
+    #   http://www.google.com:81/how which gives a origin of
+    #   'http://www.google.com:81'.
     # @option opts [Wgit::Url, String] :host The Url host e.g.
     #   http://www.google.com/how which gives a host of 'www.google.com'.
     # @option opts [Wgit::Url, String] :domain The Url domain e.g.
     #   http://www.google.com/how which gives a domain of 'google.com'.
     # @option opts [Wgit::Url, String] :brand The Url brand e.g.
     #   http://www.google.com/how which gives a domain of 'google'.
-    # @raise [StandardError] If self is invalid e.g. empty or an invalid opts
+    # @raise [StandardError] If self is invalid (e.g. empty) or an invalid opts
     #   param has been provided.
     # @return [Boolean] True if relative, false if absolute.
     def relative?(opts = {})
-      defaults = { base: nil, host: nil, domain: nil, brand: nil }
+      defaults = { origin: nil, host: nil, domain: nil, brand: nil }
       opts = defaults.merge(opts)
       raise 'Url (self) cannot be empty' if empty?
 
+      return false if scheme_relative?
       return true if @uri.relative?
 
       # Self is absolute but may be relative to the opts param e.g. host.
@@ -151,14 +173,14 @@ module Wgit
 
       type, url = opts.first
       url = Wgit::Url.new(url)
-      unless url.to_base
-        raise "Invalid opts param value, Url must be absolute and contain \
-protocol scheme: #{url}"
+      if url.invalid?
+        raise "Invalid opts param value, it must be absolute, containing a \
+protocol scheme and domain (e.g. http://example.com): #{url}"
       end
 
       case type
-      when :base   # http://www.google.com
-        to_base   == url.to_base
+      when :origin # http://www.google.com:81
+        to_origin == url.to_origin
       when :host   # www.google.com
         to_host   == url.to_host
       when :domain # google.com
@@ -177,18 +199,20 @@ protocol scheme: #{url}"
       @uri.absolute?
     end
 
-    # Returns if self is a valid and absolute HTTP Url or not.
+    # Returns if self is a valid and absolute HTTP URL or not. Self should
+    # always be crawlable if this method returns true.
     #
-    # @return [Boolean] True if valid and absolute, otherwise false.
+    # @return [Boolean] True if valid, absolute and crawable, otherwise false.
     def valid?
       return false if relative?
-      return false unless start_with?('http://') || start_with?('https://')
-      return false if URI::DEFAULT_PARSER.make_regexp.match(normalize).nil?
+      return false unless to_origin && to_domain
+      return false unless URI::DEFAULT_PARSER.make_regexp.match(normalize)
 
       true
     end
 
-    # Returns if self is an invalid (relative) HTTP Url or not.
+    # Returns if self is an invalid (e.g. relative) HTTP URL. See
+    # Wgit::Url#valid? for the inverse (and more information).
     #
     # @return [Boolean] True if invalid, otherwise false.
     def invalid?
@@ -213,7 +237,8 @@ protocol scheme: #{url}"
       Wgit::Url.new(concatted)
     end
 
-    # Normalises/escapes self and returns a new Wgit::Url. Self isn't modified.
+    # Normalizes/escapes self and returns a new Wgit::Url. Self isn't modified.
+    # This should be used before GET'ing the url, in case it has IRI chars.
     #
     # @return [Wgit::Url] An escaped version of self.
     def normalize
@@ -224,43 +249,46 @@ protocol scheme: #{url}"
     # modify the receiver.
     #
     # If self is absolute then it's returned as is, making this method
-    # idempotent. The doc's <base> element is used if present, otherwise
-    # doc.url is used as the base; which is concatted with self.
+    # idempotent. The doc's `<base>` element is used if present, otherwise
+    # `doc.url` is used as the base; which is concatted with self.
     #
-    # Typically used to build an absolute link obtained from a document e.g.
+    # Typically used to build an absolute link obtained from a document.
     #
+    # @example
     #   link = Wgit::Url.new('/favicon.png')
     #   doc  = Wgit::Document.new('http://example.com')
     #
-    #   link.prefix_base(doc) # => "http://example.com/favicon.png"
+    #   link.make_absolute(doc) # => "http://example.com/favicon.png"
     #
     # @param doc [Wgit::Document] The doc whose base Url is concatted with
     #   self.
     # @raise [StandardError] If doc isn't a Wgit::Document or if `doc.base_url`
     #   raises an Exception.
     # @return [Wgit::Url] Self in absolute form.
-    def prefix_base(doc)
+    def make_absolute(doc)
       assert_type(doc, Wgit::Document)
+      raise 'Cannot make absolute when Document @url is not valid' \
+      unless doc.url.valid?
+
+      return prefix_scheme(doc.url.to_scheme&.to_sym) if scheme_relative?
 
       absolute? ? self : doc.base_url(link: self).concat(self)
     end
 
-    # Returns self having prefixed a protocol scheme. Doesn't modify receiver.
+    # Returns self having prefixed a scheme/protocol. Doesn't modify receiver.
     # Returns self even if absolute (with scheme); therefore is idempotent.
     #
-    # @param protocol [Symbol] Either :http or :https.
-    # @return [Wgit::Url] Self with a protocol scheme prefix.
-    def prefix_scheme(protocol: :http)
-      return self if absolute?
-
-      case protocol
-      when :http
-        Wgit::Url.new("http://#{url}")
-      when :https
-        Wgit::Url.new("https://#{url}")
-      else
-        raise "protocol must be :http or :https, not :#{protocol}"
+    # @param scheme [Symbol] Either :http or :https.
+    # @return [Wgit::Url] Self with a scheme prefix.
+    def prefix_scheme(scheme = :http)
+      unless %i[http https].include?(scheme)
+        raise "scheme must be :http or :https, not :#{scheme}"
       end
+
+      return self if absolute? && !scheme_relative?
+
+      separator = scheme_relative? ? '' : '//'
+      Wgit::Url.new("#{scheme}:#{separator}#{self}")
     end
 
     # Returns a Hash containing this Url's instance vars excluding @uri.
@@ -268,8 +296,7 @@ protocol scheme: #{url}"
     #
     # @return [Hash] self's instance vars as a Hash.
     def to_h
-      ignore = ['@uri']
-      h = Wgit::Utils.to_h(self, ignore: ignore)
+      h = Wgit::Utils.to_h(self, ignore: ['@uri'])
       Hash[h.to_a.insert(0, ['url', self])] # Insert url at position 0.
     end
 
@@ -312,6 +339,20 @@ protocol scheme: #{url}"
       host ? Wgit::Url.new(host) : nil
     end
 
+    # Returns a new Wgit::Url containing just the port of this URL e.g.
+    # Given http://www.google.co.uk:443/about.html, '443' is returned.
+    #
+    # @return [Wgit::Url, nil] Containing just the port or nil.
+    def to_port
+      port = @uri.port
+
+      # @uri.port defaults port to 80/443 if missing, so we check for :#{port}.
+      return nil unless port
+      return nil unless include?(":#{port}")
+
+      Wgit::Url.new(port.to_s)
+    end
+
     # Returns a new Wgit::Url containing just the domain of this URL e.g.
     # Given http://www.google.co.uk/about.html, google.co.uk is returned.
     #
@@ -321,6 +362,20 @@ protocol scheme: #{url}"
       domain ? Wgit::Url.new(domain) : nil
     end
 
+    # Returns a new Wgit::Url containing just the sub domain of this URL e.g.
+    # Given http://scripts.dev.google.com, scripts.dev is returned.
+    #
+    # @return [Wgit::Url, nil] Containing just the sub domain or nil.
+    def to_sub_domain
+      return nil unless to_host
+
+      dot_domain = ".#{to_domain}"
+      return nil unless include?(dot_domain)
+
+      sub_domain = to_host.sub(dot_domain, '')
+      Wgit::Url.new(sub_domain)
+    end
+
     # Returns a new Wgit::Url containing just the brand of this URL e.g.
     # Given http://www.google.co.uk/about.html, google is returned.
     #
@@ -336,12 +391,24 @@ protocol scheme: #{url}"
     # @return [Wgit::Url, nil] The base of self e.g. http://www.google.co.uk or
     #   nil.
     def to_base
-      return nil if @uri.scheme.nil? || @uri.host.nil?
+      return nil unless @uri.scheme && @uri.host
 
       base = "#{@uri.scheme}://#{@uri.host}"
       Wgit::Url.new(base)
     end
 
+    # Returns only the origin of this URL e.g. the protocol scheme, host and
+    # port combined. For http://localhost:3000/api, http://localhost:3000 gets
+    # returned. If there's no port present, then to_base is returned.
+    #
+    # @return [Wgit::Url, nil] The origin of self or nil.
+    def to_origin
+      return nil unless to_base
+      return to_base unless to_port
+
+      Wgit::Url.new("#{to_base}:#{to_port}")
+    end
+
     # Returns the path of this URL e.g. the bit after the host without slashes.
     # For example:
     # Wgit::Url.new("http://www.google.co.uk/about.html/").to_path returns
@@ -370,7 +437,7 @@ protocol scheme: #{url}"
     end
 
     # Returns a new Wgit::Url containing just the query string of this URL
-    # e.g. Given http://google.com?q=ruby, '?q=ruby' is returned.
+    # e.g. Given http://google.com?q=foo&bar=1, 'q=ruby&bar=1' is returned.
     #
     # @return [Wgit::Url, nil] Containing just the query string or nil.
     def to_query
@@ -378,6 +445,24 @@ protocol scheme: #{url}"
       query ? Wgit::Url.new(query) : nil
     end
 
+    # Returns a Hash containing just the query string parameters of this URL
+    # e.g. Given http://google.com?q=ruby, "{ 'q' => 'ruby' }" is returned.
+    #
+    # @param symbolize_keys [Boolean] The returned Hash keys will be Symbols if
+    #   true, Strings otherwise.
+    # @return [Hash<String | Symbol, String>] Containing the query string
+    #   params or empty if the URL doesn't contain any query parameters.
+    def to_query_hash(symbolize_keys: false)
+      query_str = to_query
+      return {} unless query_str
+
+      query_str.split('&').each_with_object({}) do |param, hash|
+        k, v = param.split('=')
+        k = k.to_sym if symbolize_keys
+        hash[k] = v
+      end
+    end
+
     # Returns a new Wgit::Url containing just the fragment string of this URL
     # e.g. Given http://google.com#about, #about is returned.
     #
@@ -399,6 +484,24 @@ protocol scheme: #{url}"
       segs.length > 1 ? Wgit::Url.new(segs.last) : nil
     end
 
+    # Returns a new Wgit::Url containing just the username string of this URL
+    # e.g. Given http://me:pass1@example.com, me is returned.
+    #
+    # @return [Wgit::Url, nil] Containing just the user string or nil.
+    def to_user
+      user = @uri.user
+      user ? Wgit::Url.new(user) : nil
+    end
+
+    # Returns a new Wgit::Url containing just the password string of this URL
+    # e.g. Given http://me:pass1@example.com, pass1 is returned.
+    #
+    # @return [Wgit::Url, nil] Containing just the password string or nil.
+    def to_password
+      password = @uri.password
+      password ? Wgit::Url.new(password) : nil
+    end
+
     # Omits the given URL components from self and returns a new Wgit::Url.
     #
     # Calls Addressable::URI#omit underneath and creates a new Wgit::Url from
@@ -442,7 +545,7 @@ protocol scheme: #{url}"
         .omit_trailing_slash
     end
 
-    # Returns a new Wgit::Url with the base (proto and host) removed e.g. Given
+    # Returns a new Wgit::Url with the base (scheme and host) removed e.g. Given
     # http://google.com/search?q=something#about, search?q=something#about is
     # returned. If relative and base isn't present then self is returned.
     # Leading and trailing slashes are always stripped from the return value.
@@ -457,6 +560,21 @@ protocol scheme: #{url}"
       Wgit::Url.new(omit_base).omit_slashes
     end
 
+    # Returns a new Wgit::Url with the origin (base + port) removed e.g. Given
+    # http://google.com:81/search?q=something#about, search?q=something#about is
+    # returned. If relative and base isn't present then self is returned.
+    # Leading and trailing slashes are always stripped from the return value.
+    #
+    # @return [Wgit::Url] Self containing everything after the origin.
+    def omit_origin
+      origin = to_origin
+      omit_origin = origin ? gsub(origin, '') : self
+
+      return self if ['', '/'].include?(omit_origin)
+
+      Wgit::Url.new(omit_origin).omit_slashes
+    end
+
     # Returns a new Wgit::Url with the query string portion removed e.g. Given
     # http://google.com/search?q=hello, http://google.com/search is
     # returned. Self is returned as is if no query string is present. A URL
@@ -502,25 +620,47 @@ protocol scheme: #{url}"
       start_with?('#')
     end
 
-    alias +            concat
-    alias crawled?     crawled
-    alias normalise    normalize
-    alias is_relative? relative?
-    alias is_absolute? absolute?
-    alias is_valid?    valid?
-    alias is_query?    query?
-    alias is_fragment? fragment?
-    alias uri          to_uri
-    alias url          to_url
-    alias scheme       to_scheme
-    alias host         to_host
-    alias domain       to_domain
-    alias brand        to_brand
-    alias base         to_base
-    alias path         to_path
-    alias endpoint     to_endpoint
-    alias query        to_query
-    alias fragment     to_fragment
-    alias extension    to_extension
+    # Returns true if self equals '/' a.k.a. index.
+    #
+    # @return [Boolean] True if self equals '/', false otherwise.
+    def index?
+      self == '/'
+    end
+
+    # Returns true if self starts with '//' a.k.a a scheme/protocol relative
+    # path.
+    #
+    # @return [Boolean] True if self starts with '//', false otherwise.
+    def scheme_relative?
+      start_with?('//')
+    end
+
+    alias +                   concat
+    alias crawled?            crawled
+    alias is_relative?        relative?
+    alias is_absolute?        absolute?
+    alias is_valid?           valid?
+    alias is_query?           query?
+    alias is_fragment?        fragment?
+    alias is_index?           index?
+    alias is_scheme_relative? scheme_relative?
+    alias uri                 to_uri
+    alias url                 to_url
+    alias scheme              to_scheme
+    alias host                to_host
+    alias port                to_port
+    alias domain              to_domain
+    alias brand               to_brand
+    alias base                to_base
+    alias origin              to_origin
+    alias path                to_path
+    alias endpoint            to_endpoint
+    alias query               to_query
+    alias query_hash          to_query_hash
+    alias fragment            to_fragment
+    alias extension           to_extension
+    alias user                to_user
+    alias password            to_password
+    alias sub_domain          to_sub_domain
   end
 end