wgit 0.11.0 → 0.12.1

data/lib/wgit/model.rb

@@ -0,0 +1,164 @@
+# frozen_string_literal: true
+
+require_relative "./utils"
+
+module Wgit
+  # Module used to build the Database collection objects, forming a data model.
+  # The models produced are Hash like and therefore DB agnostic. Each model
+  # will contain a unique field used for searching and avoiding duplicates,
+  # this is typically a `url` field. Also contained in the model are the
+  # search fields used in Database and Document #search calls.
+  module Model
+    # The default search fields used in Database and Document #search calls.
+    # The number of matches for each field is multiplied by the field weight,
+    # the total is the search score, used to sort the search results.
+    # Call Wgit::Model.set_default_search_fields` to revert to default.
+    DEFAULT_SEARCH_FIELDS = {
+      title: 2,
+      description: 2,
+      keywords: 2,
+      text: 1
+    }.freeze
+
+    # The search fields used in Database and Document #search calls.
+    # The number of matches for each field is multiplied by the field weight,
+    # the total is the search score, used to sort the search results.
+    # Call Wgit::Model.set_default_search_fields` to revert to default.
+    @search_fields = DEFAULT_SEARCH_FIELDS
+
+    # Whether or not to include the Document#html in the #document model.
+    @include_doc_html = false
+
+    # Whether or not to include the Document#score in the #document model.
+    @include_doc_score = false
+
+    class << self
+      # The search fields used in Database and Document #search calls.
+      # A custom setter method is also provided for changing these fields.
+      attr_reader :search_fields
+
+      # Whether or not to include the Document#html in the #document model.
+      attr_accessor :include_doc_html
+
+      # Whether or not to include the Document#score in the #document model.
+      attr_accessor :include_doc_score
+    end
+
+    # Sets the search fields used in Database and Document #search calls.
+    #
+    # You can pass the fields as an Array of Symbols which gives each field a
+    # weight of 1 meaning all fields are considered of equal value. Or you can
+    # pass a Hash of Symbol => Int and specify the weights yourself, allowing
+    # you to customise the search rankings.
+    #
+    # Use like:
+    # ```
+    # Wgit::Model.set_search_fields [:title, :text], db
+    # => { title: 1, text: 1 }
+    # Wgit::Model.set_search_fields {title: 2, text: 1}, db
+    # => { title: 2, text: 1 }
+    # ```
+    #
+    # If the given db (database) param responds to #search_fields= then it will
+    # be called and given the fields to set. This should perform whatever the
+    # database adapter needs in order to search using the given fields e.g.
+    # creating a search index. Calling the DB enables the search_fields to be
+    # set globally within Wgit by one method call, this one.
+    #
+    # @param fields [Array<Symbol>, Hash<Symbol, Integer>] The field names or
+    #   the field names with their coresponding search weights.
+    # @param db [Wgit::Database::DatabaseAdapter] A connected db instance. If
+    #   db responds to #search_fields=, it will be called and given the fields.
+    # @raise [StandardError] If fields is of an incorrect type.
+    # @return [Hash<Symbol, Integer>] The fields and their weights.
+    def self.set_search_fields(fields, db = nil)
+      # We need a Hash of fields => weights (Symbols => Integers).
+      case fields
+      when Array # of Strings/Symbols.
+        fields = fields.map { |field| [field.to_sym, 1] }
+      when Hash  # of Strings/Symbols and Integers.
+        fields = fields.map { |field, weight| [field.to_sym, weight.to_i] }
+      else
+        raise "fields must be an Array or Hash, not a #{fields.class}"
+      end
+
+      @search_fields = fields.to_h
+      db.search_fields = @search_fields if db.respond_to?(:search_fields=)
+
+      @search_fields
+    end
+
+    # Sets the search fields used in Database and Document #search calls.
+    #
+    # If the given db (database) param responds to #search_fields= then it will
+    # be called and given the fields to set. This should perform whatever the
+    # database adapter needs in order to search using the given fields e.g.
+    # creating a search index. Calling the DB enables the search_fields to be
+    # set globally within Wgit by one method call, this one.
+    #
+    # @param db [Wgit::Database::DatabaseAdapter] A connected db instance. If
+    #   db responds to #search_fields=, it will be called and given the fields.
+    # @return [Hash<Symbol, Integer>] The fields and their weights.
+    def self.set_default_search_fields(db = nil)
+      set_search_fields(DEFAULT_SEARCH_FIELDS, db)
+    end
+
+    # The data model for a Wgit::Url collection object and for an embedded
+    # 'url' inside a Wgit::Document collection object.
+    #
+    # The unique field for this model is `model['url']`.
+    #
+    # @param url [Wgit::Url] The Url data object.
+    # @return [Hash] The URL model ready for DB insertion.
+    def self.url(url)
+      raise "url must respond_to? :to_h" unless url.respond_to?(:to_h)
+
+      model = url.to_h
+      select_bson_types(model)
+    end
+
+    # The data model for a Wgit::Document collection object.
+    #
+    # The unique field for this model is `model['url']['url']`.
+    #
+    # @param doc [Wgit::Document] The Document data object.
+    # @return [Hash] The Document model ready for DB insertion.
+    def self.document(doc)
+      raise "doc must respond_to? :to_h" unless doc.respond_to?(:to_h)
+
+      model = doc.to_h(
+        include_html: @include_doc_html, include_score: @include_doc_score
+      )
+      model["url"] = url(doc.url) # Expand Url String into full object.
+
+      select_bson_types(model)
+    end
+
+    # Common fields when inserting a record into the DB.
+    #
+    # @return [Hash] Insertion fields common to all models.
+    def self.common_insert_data
+      {
+        date_added:    Wgit::Utils.time_stamp,
+        date_modified: Wgit::Utils.time_stamp
+      }
+    end
+
+    # Common fields when updating a record in the DB.
+    #
+    # @return [Hash] Update fields common to all models.
+    def self.common_update_data
+      {
+        date_modified: Wgit::Utils.time_stamp
+      }
+    end
+
+    # Returns the model having removed non bson types (for use with MongoDB).
+    #
+    # @param model_hash [Hash] The model Hash to sanitize.
+    # @return [Hash] The model Hash with non bson types removed.
+    def self.select_bson_types(model_hash)
+      model_hash.select { |_k, v| v.respond_to?(:bson_type) }
+    end
+  end
+end

data/lib/wgit/response.rb

@@ -27,7 +27,7 @@ module Wgit
 
     # Defaults some values and returns a "blank" Wgit::Response object.
     def initialize
-      @body         = ''
+      @body         = ""
       @headers      = {}
       @redirections = {}
       @total_time   = 0.0
@@ -45,7 +45,7 @@ module Wgit
     # @param time [Float] The time to add to @total_time.
     # @return [Float] @total_time's new value.
     def add_total_time(time)
-      @total_time += (time || 0.0)
+      @total_time += time || 0.0
     end
 
     # Sets the HTML response body.
@@ -53,7 +53,7 @@ module Wgit
     # @param str [String] The new HTML body.
     # @return [String] @body's new value.
     def body=(str)
-      @body = (str || '')
+      @body = str || ""
     end
 
     # Returns the HTML response body or nil (if it's empty).
@@ -81,10 +81,7 @@ module Wgit
         return
       end
 
-      @headers = headers.map do |k, v|
-        k = k.downcase.gsub('-', '_').to_sym
-        [k, v]
-      end.to_h
+      @headers = headers.transform_keys { |k| k.downcase.gsub("-", "_").to_sym }
     end
 
     # Returns whether or not the response is 404 Not Found.
@@ -146,7 +143,7 @@ module Wgit
     # @return [Boolean] True if Wgit should not index this site, false
     #   otherwise.
     def no_index?
-      headers.fetch(:x_robots_tag, '').downcase.strip == 'noindex'
+      headers.fetch(:x_robots_tag, "").downcase.strip == "noindex"
     end
 
     alias_method :code,           :status

data/lib/wgit/robots_parser.rb

@@ -7,15 +7,15 @@ module Wgit
     include Wgit::Assertable
 
     # Key representing the start of a comment.
-    KEY_COMMENT    = '#'
+    KEY_COMMENT    = "#"
     # Key value separator used in robots.txt files.
-    KEY_SEPARATOR  = ':'
+    KEY_SEPARATOR  = ":"
     # Key representing a user agent.
-    KEY_USER_AGENT = 'User-agent'
+    KEY_USER_AGENT = "User-agent"
     # Key representing an allow URL rule.
-    KEY_ALLOW      = 'Allow'
+    KEY_ALLOW      = "Allow"
     # Key representing a disallow URL rule.
-    KEY_DISALLOW   = 'Disallow'
+    KEY_DISALLOW   = "Disallow"
 
     # Value representing the Wgit user agent.
     USER_AGENT_WGIT = :wgit
@@ -143,7 +143,7 @@ module Wgit
       return line unless line.count(KEY_SEPARATOR) == 1
 
       segs = line.split(KEY_SEPARATOR)
-      return '' if segs.size == 1
+      return "" if segs.size == 1
 
       segs.last.strip
     end
@@ -176,13 +176,13 @@ module Wgit
 
     def parse_special_syntax(path)
       # Remove $ e.g. "/blah$" becomes "/blah"
-      path = path.gsub('$', '')
+      path = path.gsub("$", "")
 
       # Remove any inline comments e.g. "/blah # comment" becomes "/blah"
       path = path.split(" #{KEY_COMMENT}").first if path.include?(" #{KEY_COMMENT}")
 
       # Replace an empty path with * e.g. "Allow: " becomes "Allow: *"
-      path = '*' if path.empty?
+      path = "*" if path.empty?
 
       path
     end

data/lib/wgit/url.rb

@@ -1,9 +1,9 @@
 # frozen_string_literal: true
 
-require_relative 'utils'
-require_relative 'assertable'
-require 'uri'
-require 'addressable/uri'
+require_relative "utils"
+require_relative "assertable"
+require "uri"
+require "addressable/uri"
 
 module Wgit
   # Class modeling/serialising a web based HTTP URL.
@@ -56,11 +56,11 @@ module Wgit
         obj = url_or_obj
         assert_respond_to(obj, :fetch)
 
-        url            = obj.fetch('url') # Should always be present.
-        crawled        = obj.fetch('crawled', false)
-        date_crawled   = obj.fetch('date_crawled', nil)
-        crawl_duration = obj.fetch('crawl_duration', nil)
-        redirects      = obj.fetch('redirects', {})
+        url            = obj.fetch("url") # Should always be present.
+        crawled        = obj.fetch("crawled", false)
+        date_crawled   = obj.fetch("date_crawled", nil)
+        crawl_duration = obj.fetch("crawl_duration", nil)
+        redirects      = obj.fetch("redirects", {})
       end
 
       @uri            = Addressable::URI.parse(url)
@@ -89,7 +89,7 @@ module Wgit
     # @raise [StandardError] If obj.is_a?(String) is false.
     # @return [Wgit::Url] A Wgit::Url instance.
     def self.parse(obj)
-      raise 'Can only parse if obj#is_a?(String)' unless obj.is_a?(String)
+      raise "Can only parse if obj#is_a?(String)" unless obj.is_a?(String)
 
       # Return a Wgit::Url as is to avoid losing state e.g. date_crawled etc.
       obj.is_a?(Wgit::Url) ? obj : new(obj)
@@ -98,7 +98,7 @@ module Wgit
     # Returns a Wgit::Url instance from Wgit::Url.parse, or nil if obj cannot
     # be parsed successfully e.g. the String is invalid.
     #
-    # Use this method when you can't gaurentee that obj is parsable as a URL.
+    # Use this method when you can't guarantee that obj is parsable as a URL.
     # See Wgit::Url.parse for more information.
     #
     # @param obj [Object] The object to parse, which #is_a?(String).
@@ -227,7 +227,7 @@ Addressable::URI::InvalidURIError")
     def relative?(opts = {})
       defaults = { origin: nil, host: nil, domain: nil, brand: nil }
       opts = defaults.merge(opts)
-      raise 'Url (self) cannot be empty' if empty?
+      raise "Url (self) cannot be empty" if empty?
 
       return false if scheme_relative?
       return true  if @uri.relative?
@@ -295,11 +295,11 @@ protocol scheme and domain (e.g. http://example.com): #{url}"
     # @return [Wgit::Url] self + separator + other, separator depends on other.
     def join(other)
       other = Wgit::Url.new(other)
-      raise 'other must be relative' unless other.relative?
+      raise "other must be relative" unless other.relative?
 
       other = other.omit_leading_slash
-      separator = %w[# ? .].include?(other[0]) ? '' : '/'
-      separator = '' if end_with?('/')
+      separator = %w[# ? .].include?(other[0]) ? "" : "/"
+      separator = "" if end_with?("/")
       joined = self + separator + other
 
       Wgit::Url.new(joined)
@@ -335,7 +335,7 @@ protocol scheme and domain (e.g. http://example.com): #{url}"
     # @return [Wgit::Url] Self in absolute form.
     def make_absolute(doc)
       assert_type(doc, Wgit::Document)
-      raise 'Cannot make absolute when Document @url is not valid' \
+      raise "Cannot make absolute when Document @url is not valid" \
       unless doc.url.valid?
 
       return prefix_scheme(doc.url.to_scheme&.to_sym) if scheme_relative?
@@ -355,7 +355,7 @@ protocol scheme and domain (e.g. http://example.com): #{url}"
 
       return self if absolute? && !scheme_relative?
 
-      separator = scheme_relative? ? '' : '//'
+      separator = scheme_relative? ? "" : "//"
       Wgit::Url.new("#{scheme}:#{separator}#{self}")
     end
 
@@ -364,8 +364,8 @@ protocol scheme and domain (e.g. http://example.com): #{url}"
     #
     # @return [Hash] self's instance vars as a Hash.
     def to_h
-      h = Wgit::Utils.to_h(self, ignore: ['@uri'])
-      Hash[h.to_a.insert(0, ['url', self])] # Insert url at position 0.
+      h = Wgit::Utils.to_h(self, ignore: ["@uri"])
+      Hash[h.to_a.insert(0, ["url", to_s])] # Insert url at position 0.
     end
 
     # Returns a normalised URI object for this URL.
@@ -440,7 +440,7 @@ protocol scheme and domain (e.g. http://example.com): #{url}"
       dot_domain = ".#{to_domain}"
       return nil unless include?(dot_domain)
 
-      sub_domain = to_host.sub(dot_domain, '')
+      sub_domain = to_host.sub(dot_domain, "")
       Wgit::Url.new(sub_domain)
     end
 
@@ -450,7 +450,7 @@ protocol scheme and domain (e.g. http://example.com): #{url}"
     # @return [Wgit::Url, nil] Containing just the brand or nil.
     def to_brand
       domain = to_domain
-      domain ? Wgit::Url.new(domain.split('.').first) : nil
+      domain ? Wgit::Url.new(domain.split(".").first) : nil
     end
 
     # Returns only the base of this URL e.g. the protocol scheme and host
@@ -486,7 +486,7 @@ protocol scheme and domain (e.g. http://example.com): #{url}"
     def to_path
       path = @uri.path
       return nil if path.nil? || path.empty?
-      return Wgit::Url.new('/') if path == '/'
+      return Wgit::Url.new("/") if path == "/"
 
       Wgit::Url.new(path).omit_leading_slash
     end
@@ -500,7 +500,7 @@ protocol scheme and domain (e.g. http://example.com): #{url}"
     #   an endpoint, / is returned.
     def to_endpoint
       endpoint = @uri.path
-      endpoint = "/#{endpoint}" unless endpoint.start_with?('/')
+      endpoint = "/#{endpoint}" unless endpoint.start_with?("/")
       Wgit::Url.new(endpoint)
     end
 
@@ -524,8 +524,8 @@ protocol scheme and domain (e.g. http://example.com): #{url}"
       query_str = to_query
       return {} unless query_str
 
-      query_str.split('&').each_with_object({}) do |param, hash|
-        k, v = param.split('=')
+      query_str.split("&").each_with_object({}) do |param, hash|
+        k, v = param.split("=")
         k = k.to_sym if symbolize_keys
         hash[k] = v
       end
@@ -548,7 +548,7 @@ protocol scheme and domain (e.g. http://example.com): #{url}"
       path = to_path&.omit_trailing_slash
       return nil unless path
 
-      segs = path.split('.')
+      segs = path.split(".")
       segs.length > 1 ? Wgit::Url.new(segs.last) : nil
     end
 
@@ -591,7 +591,7 @@ protocol scheme and domain (e.g. http://example.com): #{url}"
     #
     # @return [Wgit::Url] Self without a trailing slash.
     def omit_leading_slash
-      start_with?('/') ? Wgit::Url.new(self[1..]) : self
+      start_with?("/") ? Wgit::Url.new(self[1..]) : self
     end
 
     # Returns a new Wgit::Url containing self without a trailing slash. Is
@@ -600,7 +600,7 @@ protocol scheme and domain (e.g. http://example.com): #{url}"
     #
     # @return [Wgit::Url] Self without a trailing slash.
     def omit_trailing_slash
-      end_with?('/') ? Wgit::Url.new(chop) : self
+      end_with?("/") ? Wgit::Url.new(chop) : self
     end
 
     # Returns a new Wgit::Url containing self without a leading or trailing
@@ -621,9 +621,9 @@ protocol scheme and domain (e.g. http://example.com): #{url}"
     # @return [Wgit::Url] Self containing everything after the base.
     def omit_base
       base_url = to_base
-      omit_base = base_url ? gsub(base_url, '') : self
+      omit_base = base_url ? gsub(base_url, "") : self
 
-      return self if ['', '/'].include?(omit_base)
+      return self if ["", "/"].include?(omit_base)
 
       Wgit::Url.new(omit_base).omit_leading_slash
     end
@@ -636,9 +636,9 @@ protocol scheme and domain (e.g. http://example.com): #{url}"
     # @return [Wgit::Url] Self containing everything after the origin.
     def omit_origin
       origin = to_origin
-      omit_origin = origin ? gsub(origin, '') : self
+      omit_origin = origin ? gsub(origin, "") : self
 
-      return self if ['', '/'].include?(omit_origin)
+      return self if ["", "/"].include?(omit_origin)
 
       Wgit::Url.new(omit_origin).omit_leading_slash
     end
@@ -652,7 +652,7 @@ protocol scheme and domain (e.g. http://example.com): #{url}"
     # @return [Wgit::Url] Self with the query string portion removed.
     def omit_query
       query = to_query
-      omit_query_string = query ? gsub("?#{query}", '') : self
+      omit_query_string = query ? gsub("?#{query}", "") : self
 
       Wgit::Url.new(omit_query_string)
     end
@@ -667,7 +667,7 @@ protocol scheme and domain (e.g. http://example.com): #{url}"
     # @return [Wgit::Url] Self with the fragment portion removed.
     def omit_fragment
       fragment = to_fragment
-      omit_fragment = fragment ? gsub("##{fragment}", '') : self
+      omit_fragment = fragment ? gsub("##{fragment}", "") : self
 
       Wgit::Url.new(omit_fragment)
     end
@@ -677,7 +677,7 @@ protocol scheme and domain (e.g. http://example.com): #{url}"
     #
     # @return [Boolean] True if self is a query string, false otherwise.
     def query?
-      start_with?('?')
+      start_with?("?")
     end
 
     # Returns true if self is a URL fragment e.g. #top etc. Note this
@@ -685,14 +685,14 @@ protocol scheme and domain (e.g. http://example.com): #{url}"
     #
     # @return [Boolean] True if self is a fragment, false otherwise.
     def fragment?
-      start_with?('#')
+      start_with?("#")
     end
 
     # Returns true if self equals '/' a.k.a. index.
     #
     # @return [Boolean] True if self equals '/', false otherwise.
     def index?
-      self == '/'
+      self == "/"
     end
 
     # Returns true if self starts with '//' a.k.a a scheme/protocol relative
@@ -700,7 +700,7 @@ protocol scheme and domain (e.g. http://example.com): #{url}"
     #
     # @return [Boolean] True if self starts with '//', false otherwise.
     def scheme_relative?
-      start_with?('//')
+      start_with?("//")
     end
 
     alias_method :crawled?,            :crawled