wgit 0.8.0 → 0.9.0

data/lib/wgit/response.rb

@@ -1,5 +1,5 @@
 module Wgit
-  # Response class representing a generic HTTP crawl response.
+  # Response class modeling a generic HTTP GET response.
   class Response
     # The underlying HTTP adapter/library response object.
     attr_accessor :adapter_response
@@ -69,7 +69,10 @@ module Wgit
     # @param headers [Hash] The new response headers.
     # @return [Hash] @headers's new value.
     def headers=(headers)
-      return @headers = {} unless headers
+      unless headers
+        @headers = {}
+        return
+      end
 
       @headers = headers.map do |k, v|
         k = k.downcase.gsub('-', '_').to_sym

data/lib/wgit/url.rb

@@ -6,15 +6,15 @@ require 'uri'
 require 'addressable/uri'
 
 module Wgit
-  # Class modeling a web based HTTP URL.
+  # Class modeling/serialising a web based HTTP URL.
   #
   # Can be an internal/relative link e.g. "about.html" or an absolute URL
-  # e.g. "http://www.google.co.uk". Is a subclass of String and uses 'uri' and
-  # 'addressable/uri' internally.
+  # e.g. "http://www.google.co.uk". Is a subclass of String and uses `URI` and
+  # `addressable/uri` internally for parsing.
   #
-  # Most of the methods in this class return new Wgit::Url instances making the
-  # method calls chainable e.g. url.omit_base.omit_fragment etc. The methods
-  # also try to be idempotent where possible.
+  # Most of the methods in this class return new `Wgit::Url` instances making
+  # the method calls chainable e.g. `url.omit_base.omit_fragment` etc. The
+  # methods also try to be idempotent where possible.
   class Url < String
     include Assertable
 
@@ -28,7 +28,7 @@ module Wgit
     # The duration of the crawl for this Url (in seconds).
     attr_accessor :crawl_duration
 
-    # Initializes a new instance of Wgit::Url which represents a web based
+    # Initializes a new instance of Wgit::Url which models a web based
     # HTTP URL.
     #
     # @param url_or_obj [String, Wgit::Url, #fetch#[]] Is either a String
@@ -99,10 +99,10 @@ module Wgit
     # @param obj [Object] The object to parse, which #is_a?(String).
     # @raise [StandardError] If obj.is_a?(String) is false.
     # @return [Wgit::Url] A Wgit::Url instance or nil (if obj is invalid).
-    def self.parse_or_nil(obj)
+    def self.parse?(obj)
       parse(obj)
     rescue Addressable::URI::InvalidURIError
-      Wgit.logger.debug("Wgit::Url.parse_or_nil('#{obj}') exception: \
+      Wgit.logger.debug("Wgit::Url.parse?('#{obj}') exception: \
 Addressable::URI::InvalidURIError")
       nil
     end
@@ -115,8 +115,6 @@ Addressable::URI::InvalidURIError")
     def crawled=(bool)
       @crawled      = bool
       @date_crawled = bool ? Wgit::Utils.time_stamp : nil
-
-      bool
     end
 
     # Overrides String#replace setting the new_url @uri and String value.
@@ -146,10 +144,10 @@ Addressable::URI::InvalidURIError")
     # @param opts [Hash] The options with which to check relativity. Only one
     #   opts param should be provided. The provided opts param Url must be
     #   absolute and be prefixed with a scheme. Consider using the output of
-    #   Wgit::Url#to_base which should work (unless it's nil).
-    # @option opts [Wgit::Url, String] :base The Url base e.g.
-    #   http://www.google.com/how which gives a base of
-    #   'http://www.google.com'.
+    #   Wgit::Url#to_origin which should work (unless it's nil).
+    # @option opts [Wgit::Url, String] :origin The Url origin e.g.
+    #   http://www.google.com:81/how which gives a origin of
+    #   'http://www.google.com:81'.
     # @option opts [Wgit::Url, String] :host The Url host e.g.
     #   http://www.google.com/how which gives a host of 'www.google.com'.
     # @option opts [Wgit::Url, String] :domain The Url domain e.g.
@@ -160,7 +158,7 @@ Addressable::URI::InvalidURIError")
     #   param has been provided.
     # @return [Boolean] True if relative, false if absolute.
     def relative?(opts = {})
-      defaults = { base: nil, host: nil, domain: nil, brand: nil }
+      defaults = { origin: nil, host: nil, domain: nil, brand: nil }
       opts = defaults.merge(opts)
       raise 'Url (self) cannot be empty' if empty?
 
@@ -180,8 +178,8 @@ protocol scheme and domain (e.g. http://example.com): #{url}"
       end
 
       case type
-      when :base   # http://www.google.com
-        to_base   == url.to_base
+      when :origin # http://www.google.com:81
+        to_origin == url.to_origin
       when :host   # www.google.com
         to_host   == url.to_host
       when :domain # google.com
@@ -206,8 +204,8 @@ protocol scheme and domain (e.g. http://example.com): #{url}"
     # @return [Boolean] True if valid, absolute and crawable, otherwise false.
     def valid?
       return false if relative?
-      return false unless to_base && to_domain
-      return false if URI::DEFAULT_PARSER.make_regexp.match(normalize).nil?
+      return false unless to_origin && to_domain
+      return false unless URI::DEFAULT_PARSER.make_regexp.match(normalize)
 
       true
     end
@@ -238,7 +236,8 @@ protocol scheme and domain (e.g. http://example.com): #{url}"
       Wgit::Url.new(concatted)
     end
 
-    # Normalises/escapes self and returns a new Wgit::Url. Self isn't modified.
+    # Normalizes/escapes self and returns a new Wgit::Url. Self isn't modified.
+    # This should be used before GET'ing the url, in case it has IRI chars.
     #
     # @return [Wgit::Url] An escaped version of self.
     def normalize
@@ -249,8 +248,8 @@ protocol scheme and domain (e.g. http://example.com): #{url}"
     # modify the receiver.
     #
     # If self is absolute then it's returned as is, making this method
-    # idempotent. The doc's <base> element is used if present, otherwise
-    # doc.url is used as the base; which is concatted with self.
+    # idempotent. The doc's `<base>` element is used if present, otherwise
+    # `doc.url` is used as the base; which is concatted with self.
     #
     # Typically used to build an absolute link obtained from a document.
     #
@@ -258,14 +257,14 @@ protocol scheme and domain (e.g. http://example.com): #{url}"
     #   link = Wgit::Url.new('/favicon.png')
     #   doc  = Wgit::Document.new('http://example.com')
     #
-    #   link.prefix_base(doc) # => "http://example.com/favicon.png"
+    #   link.make_absolute(doc) # => "http://example.com/favicon.png"
     #
     # @param doc [Wgit::Document] The doc whose base Url is concatted with
     #   self.
     # @raise [StandardError] If doc isn't a Wgit::Document or if `doc.base_url`
     #   raises an Exception.
     # @return [Wgit::Url] Self in absolute form.
-    def prefix_base(doc)
+    def make_absolute(doc)
       assert_type(doc, Wgit::Document)
 
       absolute? ? self : doc.base_url(link: self).concat(self)
@@ -294,8 +293,7 @@ protocol scheme and domain (e.g. http://example.com): #{url}"
     #
     # @return [Hash] self's instance vars as a Hash.
     def to_h
-      ignore = ['@uri']
-      h = Wgit::Utils.to_h(self, ignore: ignore)
+      h = Wgit::Utils.to_h(self, ignore: ['@uri'])
       Hash[h.to_a.insert(0, ['url', self])] # Insert url at position 0.
     end
 
@@ -338,6 +336,20 @@ protocol scheme and domain (e.g. http://example.com): #{url}"
       host ? Wgit::Url.new(host) : nil
     end
 
+    # Returns a new Wgit::Url containing just the port of this URL e.g.
+    # Given http://www.google.co.uk:443/about.html, '443' is returned.
+    #
+    # @return [Wgit::Url, nil] Containing just the port or nil.
+    def to_port
+      port = @uri.port
+
+      # @uri.port defaults port to 80/443 if missing, so we check for :#{port}.
+      return nil unless port
+      return nil unless include?(":#{port}")
+
+      Wgit::Url.new(port.to_s)
+    end
+
     # Returns a new Wgit::Url containing just the domain of this URL e.g.
     # Given http://www.google.co.uk/about.html, google.co.uk is returned.
     #
@@ -347,6 +359,20 @@ protocol scheme and domain (e.g. http://example.com): #{url}"
       domain ? Wgit::Url.new(domain) : nil
     end
 
+    # Returns a new Wgit::Url containing just the sub domain of this URL e.g.
+    # Given http://scripts.dev.google.com, scripts.dev is returned.
+    #
+    # @return [Wgit::Url, nil] Containing just the sub domain or nil.
+    def to_sub_domain
+      return nil unless to_host
+
+      dot_domain = ".#{to_domain}"
+      return nil unless include?(dot_domain)
+
+      sub_domain = to_host.sub(dot_domain, '')
+      Wgit::Url.new(sub_domain)
+    end
+
     # Returns a new Wgit::Url containing just the brand of this URL e.g.
     # Given http://www.google.co.uk/about.html, google is returned.
     #
@@ -362,12 +388,24 @@ protocol scheme and domain (e.g. http://example.com): #{url}"
     # @return [Wgit::Url, nil] The base of self e.g. http://www.google.co.uk or
     #   nil.
     def to_base
-      return nil if @uri.scheme.nil? || @uri.host.nil?
+      return nil unless @uri.scheme && @uri.host
 
       base = "#{@uri.scheme}://#{@uri.host}"
       Wgit::Url.new(base)
     end
 
+    # Returns only the origin of this URL e.g. the protocol scheme, host and
+    # port combined. For http://localhost:3000/api, http://localhost:3000 gets
+    # returned. If there's no port present, then to_base is returned.
+    #
+    # @return [Wgit::Url, nil] The origin of self or nil.
+    def to_origin
+      return nil unless to_base
+      return to_base unless to_port
+
+      Wgit::Url.new("#{to_base}:#{to_port}")
+    end
+
     # Returns the path of this URL e.g. the bit after the host without slashes.
     # For example:
     # Wgit::Url.new("http://www.google.co.uk/about.html/").to_path returns
@@ -396,7 +434,7 @@ protocol scheme and domain (e.g. http://example.com): #{url}"
     end
 
     # Returns a new Wgit::Url containing just the query string of this URL
-    # e.g. Given http://google.com?q=ruby, '?q=ruby' is returned.
+    # e.g. Given http://google.com?q=foo&bar=1, 'q=ruby&bar=1' is returned.
     #
     # @return [Wgit::Url, nil] Containing just the query string or nil.
     def to_query
@@ -404,6 +442,24 @@ protocol scheme and domain (e.g. http://example.com): #{url}"
       query ? Wgit::Url.new(query) : nil
     end
 
+    # Returns a Hash containing just the query string parameters of this URL
+    # e.g. Given http://google.com?q=ruby, "{ 'q' => 'ruby' }" is returned.
+    #
+    # @param symbolize_keys [Boolean] The returned Hash keys will be Symbols if
+    #   true, Strings otherwise.
+    # @return [Hash<String | Symbol, String>] Containing the query string
+    #   params or empty if the URL doesn't contain any query parameters.
+    def to_query_hash(symbolize_keys: false)
+      query_str = to_query
+      return {} unless query_str
+
+      query_str.split('&').each_with_object({}) do |param, hash|
+        k, v = param.split('=')
+        k = k.to_sym if symbolize_keys
+        hash[k] = v
+      end
+    end
+
     # Returns a new Wgit::Url containing just the fragment string of this URL
     # e.g. Given http://google.com#about, #about is returned.
     #
@@ -425,6 +481,24 @@ protocol scheme and domain (e.g. http://example.com): #{url}"
       segs.length > 1 ? Wgit::Url.new(segs.last) : nil
     end
 
+    # Returns a new Wgit::Url containing just the username string of this URL
+    # e.g. Given http://me:pass1@example.com, me is returned.
+    #
+    # @return [Wgit::Url, nil] Containing just the user string or nil.
+    def to_user
+      user = @uri.user
+      user ? Wgit::Url.new(user) : nil
+    end
+
+    # Returns a new Wgit::Url containing just the password string of this URL
+    # e.g. Given http://me:pass1@example.com, pass1 is returned.
+    #
+    # @return [Wgit::Url, nil] Containing just the password string or nil.
+    def to_password
+      password = @uri.password
+      password ? Wgit::Url.new(password) : nil
+    end
+
     # Omits the given URL components from self and returns a new Wgit::Url.
     #
     # Calls Addressable::URI#omit underneath and creates a new Wgit::Url from
@@ -468,7 +542,7 @@ protocol scheme and domain (e.g. http://example.com): #{url}"
         .omit_trailing_slash
     end
 
-    # Returns a new Wgit::Url with the base (proto and host) removed e.g. Given
+    # Returns a new Wgit::Url with the base (scheme and host) removed e.g. Given
     # http://google.com/search?q=something#about, search?q=something#about is
     # returned. If relative and base isn't present then self is returned.
     # Leading and trailing slashes are always stripped from the return value.
@@ -483,6 +557,21 @@ protocol scheme and domain (e.g. http://example.com): #{url}"
       Wgit::Url.new(omit_base).omit_slashes
     end
 
+    # Returns a new Wgit::Url with the origin (base + port) removed e.g. Given
+    # http://google.com:81/search?q=something#about, search?q=something#about is
+    # returned. If relative and base isn't present then self is returned.
+    # Leading and trailing slashes are always stripped from the return value.
+    #
+    # @return [Wgit::Url] Self containing everything after the origin.
+    def omit_origin
+      origin = to_origin
+      omit_origin = origin ? gsub(origin, '') : self
+
+      return self if ['', '/'].include?(omit_origin)
+
+      Wgit::Url.new(omit_origin).omit_slashes
+    end
+
     # Returns a new Wgit::Url with the query string portion removed e.g. Given
     # http://google.com/search?q=hello, http://google.com/search is
     # returned. Self is returned as is if no query string is present. A URL
@@ -528,25 +617,38 @@ protocol scheme and domain (e.g. http://example.com): #{url}"
       start_with?('#')
     end
 
+    # Returns true if self equals '/' a.k.a. index.
+    #
+    # @return [Boolean] True if self equals '/', false otherwise.
+    def index?
+      self == '/'
+    end
+
     alias +            concat
     alias crawled?     crawled
-    alias normalise    normalize
     alias is_relative? relative?
     alias is_absolute? absolute?
     alias is_valid?    valid?
     alias is_query?    query?
     alias is_fragment? fragment?
+    alias is_index?    index?
     alias uri          to_uri
     alias url          to_url
     alias scheme       to_scheme
     alias host         to_host
+    alias port         to_port
     alias domain       to_domain
     alias brand        to_brand
     alias base         to_base
+    alias origin       to_origin
     alias path         to_path
     alias endpoint     to_endpoint
     alias query        to_query
+    alias query_hash   to_query_hash
     alias fragment     to_fragment
     alias extension    to_extension
+    alias user         to_user
+    alias password     to_password
+    alias sub_domain   to_sub_domain
   end
 end

data/lib/wgit/utils.rb

@@ -145,7 +145,8 @@ module Wgit
     # @param keyword_limit [Integer] The max amount of keywords to be
     #   outputted to the stream.
     # @param stream [#puts] Any object that respond_to?(:puts). It is used
-    #   to output text somewhere e.g. a file or STDOUT.
+    #   to output text somewhere e.g. a file or STDERR.
+    # @return [Integer] The number of results.
     def self.printf_search_results(results, keyword_limit: 5, stream: STDOUT)
       raise 'stream must respond_to? :puts' unless stream.respond_to?(:puts)
 
@@ -162,18 +163,37 @@ module Wgit
         stream.puts
       end
 
-      nil
+      results.size
     end
 
-    # Processes a String to make it uniform. Strips any leading/trailing white
+    # Sanitises the obj to make it uniform by calling the correct sanitize_*
+    # method for its type e.g. if obj.is_a? String then sanitize(obj). Any type
+    # not in the case statement will be ignored and returned as is.
+    #
+    # @param obj [Object] The object to be sanitized.
+    # @param encode [Boolean] Whether or not to encode to UTF-8 replacing
+    #   invalid characters.
+    # @return [Object] The sanitized obj is both modified and then returned.
+    def self.sanitize(obj, encode: true)
+      case obj
+      when String
+        sanitize_str(obj, encode: encode)
+      when Array
+        sanitize_arr(obj, encode: encode)
+      else
+        obj
+      end
+    end
+
+    # Sanitises a String to make it uniform. Strips any leading/trailing white
     # space. Also applies UTF-8 encoding (replacing invalid characters) if
     # `encode: true`.
     #
-    # @param str [String] The String to process. str is modified.
+    # @param str [String] The String to sanitize. str is modified.
     # @param encode [Boolean] Whether or not to encode to UTF-8 replacing
     #   invalid characters.
-    # @return [String] The processed str is both modified and then returned.
-    def self.process_str(str, encode: true)
+    # @return [String] The sanitized str is both modified and then returned.
+    def self.sanitize_str(str, encode: true)
       if str.is_a?(String)
         str.encode!('UTF-8', undef: :replace, invalid: :replace) if encode
         str.strip!
@@ -182,15 +202,15 @@ module Wgit
       str
     end
 
-    # Processes an Array to make it uniform. Removes empty Strings and nils,
-    # processes non empty Strings using Wgit::Utils.process_str and removes
+    # Sanitises an Array to make it uniform. Removes empty Strings and nils,
+    # processes non empty Strings using Wgit::Utils.sanitize and removes
     # duplicates.
     #
-    # @param arr [Enumerable] The Array to process. arr is modified.
-    # @return [Enumerable] The processed arr is both modified and then returned.
-    def self.process_arr(arr, encode: true)
+    # @param arr [Enumerable] The Array to sanitize. arr is modified.
+    # @return [Enumerable] The sanitized arr is both modified and then returned.
+    def self.sanitize_arr(arr, encode: true)
       if arr.is_a?(Array)
-        arr.map! { |str| process_str(str, encode: encode) }
+        arr.map! { |str| sanitize(str, encode: encode) }
         arr.reject! { |str| str.is_a?(String) ? str.empty? : false }
         arr.compact!
         arr.uniq!
@@ -198,13 +218,5 @@ module Wgit
 
       arr
     end
-
-    # Returns the model having removed non bson types (for use with MongoDB).
-    #
-    # @param model_hash [Hash] The model Hash to process.
-    # @return [Hash] The model Hash with non bson types removed.
-    def self.remove_non_bson_types(model_hash)
-      model_hash.select { |_k, v| v.respond_to?(:bson_type) }
-    end
   end
 end

data/lib/wgit/version.rb

@@ -2,10 +2,11 @@
 
 # Wgit is a WWW indexer/scraper which crawls URL's and retrieves their page
 # contents for later use.
+#
 # @author Michael Telford
 module Wgit
   # The current gem version of Wgit.
-  VERSION = '0.8.0'
+  VERSION = '0.9.0'
 
   # Returns the current gem version of Wgit as a String.
   def self.version