wgit 0.10.8 → 0.11.0

data/lib/wgit/url.rb

@@ -28,6 +28,9 @@ module Wgit
     # The duration of the crawl for this Url (in seconds).
     attr_accessor :crawl_duration
 
+    # Record the redirects from the initial Url to the final Url.
+    attr_reader :redirects
+
     # Initializes a new instance of Wgit::Url which models a web based
     # HTTP URL.
     #
@@ -57,12 +60,14 @@ module Wgit
         crawled        = obj.fetch('crawled', false)
         date_crawled   = obj.fetch('date_crawled', nil)
         crawl_duration = obj.fetch('crawl_duration', nil)
+        redirects      = obj.fetch('redirects', {})
       end
 
       @uri            = Addressable::URI.parse(url)
       @crawled        = crawled
       @date_crawled   = date_crawled
       @crawl_duration = crawl_duration
+      @redirects      = redirects || {}
 
       super(url)
     end
@@ -107,16 +112,6 @@ Addressable::URI::InvalidURIError")
       nil
     end
 
-    # Sets the @crawled instance var, also setting @date_crawled for
-    # convenience.
-    #
-    # @param bool [Boolean] True if this Url has been crawled, false otherwise.
-    # @return [Boolean] The value of bool having been set.
-    def crawled=(bool)
-      @crawled      = bool
-      @date_crawled = bool ? Wgit::Utils.time_stamp : nil
-    end
-
     # Overrides String#inspect to distingiush this Url from a String.
     #
     # @return [String] A short textual representation of this Url.
@@ -134,6 +129,71 @@ Addressable::URI::InvalidURIError")
       super(new_url)
     end
 
+    # Overrides String#concat which oddly returns a Wgit::Url object, and
+    # instead returns a String. Therefore this method works the same as if
+    # you call String#concat, or its alias String#+, which is desired for
+    # this method. If you want to join two Urls, use Wgit::Url#join method.
+    #
+    # @param other [String] The String to concat onto this one.
+    # @return [String] The new concatted String, not a Wgit::Url.
+    def concat(other)
+      to_s.concat(other.to_s)
+    end
+
+    # Sets the @crawled instance var, also setting @date_crawled for
+    # convenience.
+    #
+    # @param bool [Boolean] True if this Url has been crawled, false otherwise.
+    # @return [Boolean] The value of bool having been set.
+    def crawled=(bool)
+      @crawled      = bool
+      @date_crawled = bool ? Wgit::Utils.time_stamp : nil
+    end
+
+    # Sets the @redirects instance var, mapping any Strings into Wgit::Urls.
+    #
+    # @param redirects [Hash] The redirects Hash to set for this Url.
+    def redirects=(redirects)
+      assert_type(redirects, Hash)
+
+      map_to_url = proc do |url|
+        Wgit::Url.new(url.to_s, crawled: @crawled, date_crawled: @date_crawled)
+      end
+
+      @redirects = redirects
+                   .map { |from, to| [map_to_url.call(from), map_to_url.call(to)] }
+                   .to_h
+    end
+
+    # Returns the Wgit::Url's starting with the originally requested Url to be
+    # crawled, followed by each redirected to Url, finishing with the final
+    # crawled Url e.g.
+    #
+    # Example Url redirects journey (dictated by the webserver):
+    #
+    # ```
+    # http://example.com   => 301 to https://example.com
+    # https://example.com  => 301 to https://example.com/
+    # https://example.com/ => 200 OK (no more redirects, crawl complete)
+    # ```
+    #
+    # Would return an Array of Wgit::Url's in the form of:
+    #
+    # ```
+    # %w(
+    #   http://example.com
+    #   https://example.com
+    #   https://example.com/
+    # )
+    # ```
+    #
+    # @return [Array<Wgit::Url>] Each redirected to Url's finishing with the
+    #   final (successfully) crawled Url. If no redirects took place, then just
+    #   the originally requested Url is returned inside the Array.
+    def redirects_journey
+      [redirects.keys, self].flatten
+    end
+
     # Returns true if self is a relative Url; false if absolute.
     #
     # An absolute URL must have a scheme prefix e.g.
@@ -170,7 +230,7 @@ Addressable::URI::InvalidURIError")
       raise 'Url (self) cannot be empty' if empty?
 
       return false if scheme_relative?
-      return true if @uri.relative?
+      return true  if @uri.relative?
 
       # Self is absolute but may be relative to the opts param e.g. host.
       opts.select! { |_k, v| v }
@@ -226,22 +286,23 @@ protocol scheme and domain (e.g. http://example.com): #{url}"
       !valid?
     end
 
-    # Concats self and other together before returning a new Url. Self is not
-    # modified.
+    # Joins self and other together before returning a new Url. Self is not
+    # modified. Some magic occurs depending on what is being joined, see
+    # the source code for more information.
     #
-    # @param other [Wgit::Url, String] The other to concat to the end of self.
+    # @param other [Wgit::Url, String] The other (relative) Url to join to the
+    #   end of self.
     # @return [Wgit::Url] self + separator + other, separator depends on other.
-    def concat(other)
+    def join(other)
       other = Wgit::Url.new(other)
       raise 'other must be relative' unless other.relative?
 
       other = other.omit_leading_slash
       separator = %w[# ? .].include?(other[0]) ? '' : '/'
+      separator = '' if end_with?('/')
+      joined = self + separator + other
 
-      # We use to_s below to call String#+, not Wgit::Url#+ (alias for concat).
-      concatted = omit_trailing_slash.to_s + separator.to_s + other.to_s
-
-      Wgit::Url.new(concatted)
+      Wgit::Url.new(joined)
     end
 
     # Normalizes/escapes self and returns a new Wgit::Url. Self isn't modified.
@@ -257,7 +318,7 @@ protocol scheme and domain (e.g. http://example.com): #{url}"
     #
     # If self is absolute then it's returned as is, making this method
     # idempotent. The doc's `<base>` element is used if present, otherwise
-    # `doc.url` is used as the base; which is concatted with self.
+    # `doc.url` is used as the base; which is joined with self.
     #
     # Typically used to build an absolute link obtained from a document.
     #
@@ -267,7 +328,7 @@ protocol scheme and domain (e.g. http://example.com): #{url}"
     #
     #   link.make_absolute(doc) # => "http://example.com/favicon.png"
     #
-    # @param doc [Wgit::Document] The doc whose base Url is concatted with
+    # @param doc [Wgit::Document] The doc whose base Url is joined with
     #   self.
     # @raise [StandardError] If doc isn't a Wgit::Document or if `doc.base_url`
     #   raises an Exception.
@@ -279,7 +340,7 @@ protocol scheme and domain (e.g. http://example.com): #{url}"
 
       return prefix_scheme(doc.url.to_scheme&.to_sym) if scheme_relative?
 
-      absolute? ? self : doc.base_url(link: self).concat(self)
+      absolute? ? self : doc.base_url(link: self).join(self)
     end
 
     # Returns self having prefixed a scheme/protocol. Doesn't modify receiver.
@@ -427,7 +488,7 @@ protocol scheme and domain (e.g. http://example.com): #{url}"
       return nil if path.nil? || path.empty?
       return Wgit::Url.new('/') if path == '/'
 
-      Wgit::Url.new(path).omit_slashes
+      Wgit::Url.new(path).omit_leading_slash
     end
 
     # Returns the endpoint of this URL e.g. the bit after the host with any
@@ -439,7 +500,7 @@ protocol scheme and domain (e.g. http://example.com): #{url}"
     #   an endpoint, / is returned.
     def to_endpoint
       endpoint = @uri.path
-      endpoint = '/' + endpoint unless endpoint.start_with?('/')
+      endpoint = "/#{endpoint}" unless endpoint.start_with?('/')
       Wgit::Url.new(endpoint)
     end
 
@@ -484,7 +545,7 @@ protocol scheme and domain (e.g. http://example.com): #{url}"
     #
     # @return [Wgit::Url, nil] Containing just the extension string or nil.
     def to_extension
-      path = to_path
+      path = to_path&.omit_trailing_slash
       return nil unless path
 
       segs = path.split('.')
@@ -530,7 +591,7 @@ protocol scheme and domain (e.g. http://example.com): #{url}"
     #
     # @return [Wgit::Url] Self without a trailing slash.
     def omit_leading_slash
-      start_with?('/') ? Wgit::Url.new(self[1..-1]) : self
+      start_with?('/') ? Wgit::Url.new(self[1..]) : self
     end
 
     # Returns a new Wgit::Url containing self without a trailing slash. Is
@@ -564,7 +625,7 @@ protocol scheme and domain (e.g. http://example.com): #{url}"
 
       return self if ['', '/'].include?(omit_base)
 
-      Wgit::Url.new(omit_base).omit_slashes
+      Wgit::Url.new(omit_base).omit_leading_slash
     end
 
     # Returns a new Wgit::Url with the origin (base + port) removed e.g. Given
@@ -579,7 +640,7 @@ protocol scheme and domain (e.g. http://example.com): #{url}"
 
       return self if ['', '/'].include?(omit_origin)
 
-      Wgit::Url.new(omit_origin).omit_slashes
+      Wgit::Url.new(omit_origin).omit_leading_slash
     end
 
     # Returns a new Wgit::Url with the query string portion removed e.g. Given
@@ -642,32 +703,31 @@ protocol scheme and domain (e.g. http://example.com): #{url}"
       start_with?('//')
     end
 
-    alias +                   concat
-    alias crawled?            crawled
-    alias is_relative?        relative?
-    alias is_absolute?        absolute?
-    alias is_valid?           valid?
-    alias is_query?           query?
-    alias is_fragment?        fragment?
-    alias is_index?           index?
-    alias is_scheme_relative? scheme_relative?
-    alias uri                 to_uri
-    alias url                 to_url
-    alias scheme              to_scheme
-    alias host                to_host
-    alias port                to_port
-    alias domain              to_domain
-    alias brand               to_brand
-    alias base                to_base
-    alias origin              to_origin
-    alias path                to_path
-    alias endpoint            to_endpoint
-    alias query               to_query
-    alias query_hash          to_query_hash
-    alias fragment            to_fragment
-    alias extension           to_extension
-    alias user                to_user
-    alias password            to_password
-    alias sub_domain          to_sub_domain
+    alias_method :crawled?,            :crawled
+    alias_method :is_relative?,        :relative?
+    alias_method :is_absolute?,        :absolute?
+    alias_method :is_valid?,           :valid?
+    alias_method :is_query?,           :query?
+    alias_method :is_fragment?,        :fragment?
+    alias_method :is_index?,           :index?
+    alias_method :is_scheme_relative?, :scheme_relative?
+    alias_method :uri,                 :to_uri
+    alias_method :url,                 :to_url
+    alias_method :scheme,              :to_scheme
+    alias_method :host,                :to_host
+    alias_method :port,                :to_port
+    alias_method :domain,              :to_domain
+    alias_method :brand,               :to_brand
+    alias_method :base,                :to_base
+    alias_method :origin,              :to_origin
+    alias_method :path,                :to_path
+    alias_method :endpoint,            :to_endpoint
+    alias_method :query,               :to_query
+    alias_method :query_hash,          :to_query_hash
+    alias_method :fragment,            :to_fragment
+    alias_method :extension,           :to_extension
+    alias_method :user,                :to_user
+    alias_method :password,            :to_password
+    alias_method :sub_domain,          :to_sub_domain
   end
 end

data/lib/wgit/utils.rb

@@ -23,7 +23,7 @@ module Wgit
       obj.instance_variables.each do |var|
         next if ignore.include?(var.to_s)
 
-        key = var.to_s[1..-1] # Remove the @ prefix.
+        key = var.to_s[1..] # Remove the @ prefix.
         key = key.to_sym unless use_strings_as_keys
         hash[key] = obj.instance_variable_get(var)
       end
@@ -37,9 +37,9 @@ module Wgit
     # @yield [el] Gives each element (Object) of obj_or_objects if it's
     #   Enumerable, otherwise obj_or_objs itself is given.
     # @return [Object] The obj_or_objs parameter is returned.
-    def self.each(obj_or_objs)
+    def self.each(obj_or_objs, &block)
       if obj_or_objs.respond_to?(:each)
-        obj_or_objs.each { |obj| yield(obj) }
+        obj_or_objs.each(&block)
       else
         yield(obj_or_objs)
       end
@@ -129,15 +129,13 @@ module Wgit
     # Prints out the search results in a search engine like format.
     # The format for each result looks like:
     #
+    # ```
     # Title
-    #
     # Keywords (if there are some)
-    #
     # Text Snippet (formatted to show the searched for query, if provided)
-    #
     # URL
-    #
     # <empty_line_seperator>
+    # ```
     #
     # @param results [Array<Wgit::Document>] Array of Wgit::Document's which
     #   each have had #search!(query) called (to update it's @text with the
@@ -147,7 +145,7 @@ module Wgit
     # @param stream [#puts] Any object that respond_to?(:puts). It is used
     #   to output text somewhere e.g. a file or STDERR.
     # @return [Integer] The number of results.
-    def self.printf_search_results(results, keyword_limit: 5, stream: STDOUT)
+    def self.pprint_search_results(results, keyword_limit: 5, stream: $stdout)
       raise 'stream must respond_to? :puts' unless stream.respond_to?(:puts)
 
       results.each do |doc|
@@ -167,56 +165,111 @@ module Wgit
     end
 
     # Sanitises the obj to make it uniform by calling the correct sanitize_*
-    # method for its type e.g. if obj.is_a? String then sanitize(obj). Any type
-    # not in the case statement will be ignored and returned as is.
+    # method for its type e.g. if obj.is_a? String then sanitize_str(obj) is called.
+    # Any type not in the case statement will be ignored and returned as is.
+    # Call this method if unsure what obj's type is.
     #
     # @param obj [Object] The object to be sanitized.
     # @param encode [Boolean] Whether or not to encode to UTF-8 replacing
     #   invalid characters.
-    # @return [Object] The sanitized obj is both modified and then returned.
+    # @return [Object] The sanitized obj.
     def self.sanitize(obj, encode: true)
       case obj
+      when Wgit::Url
+        sanitize_url(obj, encode:)
       when String
-        sanitize_str(obj, encode: encode)
+        sanitize_str(obj, encode:)
       when Array
-        sanitize_arr(obj, encode: encode)
+        sanitize_arr(obj, encode:)
       else
         obj
       end
     end
 
+    # Sanitises a Wgit::Url to make it uniform. First sanitizes the Url as a
+    # String before replacing the Url value with the sanitized version. This
+    # method therefore modifies the given url param and also returns it.
+    #
+    # @param url [Wgit::Url] The Wgit::Url to sanitize. url is modified.
+    # @param encode [Boolean] Whether or not to encode to UTF-8 replacing
+    #   invalid characters.
+    # @return [Wgit::Url] The sanitized url, which is also modified.
+    def self.sanitize_url(url, encode: true)
+      str = sanitize_str(url.to_s, encode:)
+      url.replace(str)
+    end
+
     # Sanitises a String to make it uniform. Strips any leading/trailing white
     # space. Also applies UTF-8 encoding (replacing invalid characters) if
     # `encode: true`.
     #
-    # @param str [String] The String to sanitize. str is modified.
+    # @param str [String] The String to sanitize. str is not modified.
     # @param encode [Boolean] Whether or not to encode to UTF-8 replacing
     #   invalid characters.
-    # @return [String] The sanitized str is both modified and then returned.
+    # @return [String] The sanitized str.
     def self.sanitize_str(str, encode: true)
-      if str.is_a?(String)
-        str.encode!('UTF-8', undef: :replace, invalid: :replace) if encode
-        str.strip!
-      end
+      return str unless str.is_a?(String)
 
-      str
+      str = str.encode('UTF-8', undef: :replace, invalid: :replace) if encode
+      str.strip
     end
 
     # Sanitises an Array to make it uniform. Removes empty Strings and nils,
     # processes non empty Strings using Wgit::Utils.sanitize and removes
     # duplicates.
     #
-    # @param arr [Enumerable] The Array to sanitize. arr is modified.
-    # @return [Enumerable] The sanitized arr is both modified and then returned.
+    # @param arr [Enumerable] The Array to sanitize. arr is not modified.
+    # @return [Enumerable] The sanitized arr.
     def self.sanitize_arr(arr, encode: true)
-      if arr.is_a?(Array)
-        arr.map! { |str| sanitize(str, encode: encode) }
-        arr.reject! { |str| str.is_a?(String) ? str.empty? : false }
-        arr.compact!
-        arr.uniq!
-      end
+      return arr unless arr.is_a?(Array)
 
       arr
+        .map { |str| sanitize(str, encode:) }
+        .reject { |str| str.is_a?(String) && str.empty? }
+        .compact
+        .uniq
+    end
+
+    # Pretty prints a log statement, used for debugging purposes.
+    #
+    # Use like:
+    #
+    # ```
+    # Wgit::Utils.pprint 1, include_html: include_html, ignore: ignore_vars
+    # ```
+    #
+    # Which produces a log like:
+    #
+    # ```
+    # DEBUG_1 - include_html: true | ignore: ['@html', '@parser']
+    # ```
+    #
+    # @param identifier [#to_s] A log identifier e.g. "START" or 1 etc.
+    # @param stream [#puts] Any object that respond_to? :puts and :print. It is
+    #   used to output the log text somewhere e.g. a file or STDERR.
+    # @param prefix [String] The log prefix, useful for visibility/greping.
+    # @param new_line [Boolean] Wether or not to use a new line (\n) as the
+    #   separator.
+    # @param vars [Hash<#inspect, #inspect>] The vars to inspect in the log.
+    def self.pprint(identifier, stream: $stdout, prefix: 'DEBUG', new_line: false, **vars)
+      sep1 = new_line ? "\n" : ' - '
+      sep2 = new_line ? "\n" : ' | '
+
+      stream.print "\n#{prefix}_#{identifier}#{sep1}"
+
+      vars.each_with_index do |arr, i|
+        last_item = (i + 1) == vars.size
+        sep3 = sep2
+        sep3 = new_line ? "\n" : '' if last_item
+        k, v = arr
+
+        stream.print "#{k}: #{v}#{sep3}"
+      end
+
+      stream.puts "\n"
+      stream.puts "\n" unless new_line
+
+      nil
     end
   end
 end

data/lib/wgit/version.rb

@@ -6,7 +6,7 @@
 # @author Michael Telford
 module Wgit
   # The current gem version of Wgit.
-  VERSION = '0.10.8'
+  VERSION = '0.11.0'
 
   # Returns the current gem version of Wgit as a String.
   def self.version

data/lib/wgit.rb

@@ -10,6 +10,7 @@ require_relative 'wgit/document_extractors'
 require_relative 'wgit/crawler'
 require_relative 'wgit/database/model'
 require_relative 'wgit/database/database'
+require_relative 'wgit/robots_parser'
 require_relative 'wgit/indexer'
 require_relative 'wgit/dsl'
 require_relative 'wgit/base'