wgit 0.5.1 → 0.10.0

data/lib/wgit/database/model.rb

@@ -14,8 +14,7 @@ module Wgit
       raise 'url must respond_to? :to_h' unless url.respond_to?(:to_h)
 
       model = url.to_h
-
-      Wgit::Utils.remove_non_bson_types(model)
+      select_bson_types(model)
     end
 
     # The data model for a Wgit::Document collection object.
@@ -28,7 +27,7 @@ module Wgit
       model = doc.to_h(include_html: false, include_score: false)
       model['url'] = url(doc.url) # Expand Url String into full object.
 
-      Wgit::Utils.remove_non_bson_types(model)
+      select_bson_types(model)
     end
 
     # Common fields when inserting a record into the DB.
@@ -49,5 +48,13 @@ module Wgit
         date_modified: Wgit::Utils.time_stamp
       }
     end
+
+    # Returns the model having removed non bson types (for use with MongoDB).
+    #
+    # @param model_hash [Hash] The model Hash to sanitize.
+    # @return [Hash] The model Hash with non bson types removed.
+    def self.select_bson_types(model_hash)
+      model_hash.select { |_k, v| v.respond_to?(:bson_type) }
+    end
   end
 end

data/lib/wgit/document.rb

@@ -3,45 +3,56 @@ require_relative 'utils'
 require_relative 'assertable'
 require 'nokogiri'
 require 'json'
+require 'set'
 
 module Wgit
-  # Class primarily modeling a HTML web document, although other MIME types
+  # Class modeling/serialising a HTML web document, although other MIME types
   # will work e.g. images etc. Also doubles as a search result when
-  # loading Documents from the database via Wgit::Database#search.
+  # loading Documents from the database via `Wgit::Database#search`.
   #
   # The initialize method dynamically initializes instance variables from the
   # Document HTML / Database object e.g. text. This bit is dynamic so that the
-  # Document class can be easily extended allowing you to pull out the bits of
-  # a webpage that are important to you. See Wgit::Document.define_extension.
+  # Document class can be easily extended allowing you to extract the bits of
+  # a webpage that are important to you. See `Wgit::Document.define_extractor`.
   class Document
     include Assertable
 
-    # Regex for the allowed var names when defining an extension.
-    REGEX_EXTENSION_NAME = /[a-z0-9_]+/.freeze
+    # Regex for the allowed var names when defining an extractor.
+    REGEX_EXTRACTOR_NAME = /[a-z0-9_]+/.freeze
 
-    # The HTML elements that make up the visible text on a page.
-    # These elements are used to initialize the @text of the Document.
-    # See the README.md for how to add to this Array dynamically.
-    @text_elements = %i[
-      dd div dl dt figcaption figure hr li
-      main ol p pre span ul h1 h2 h3 h4 h5
-    ]
+    # Set of text elements used to build Document#text.
+    @text_elements = Set.new(%i[
+      a abbr address article aside b bdi bdo blockquote button caption cite
+      code data dd del details dfn div dl dt em figcaption figure footer h1 h2
+      h3 h4 h5 h6 header hr i input ins kbd label legend li main mark meter ol
+      option output p pre q rb rt ruby s samp section small span strong sub
+      summary sup td textarea th time u ul var wbr
+    ])
+
+    # Set of Symbols representing the defined Document extractors.
+    @extractors = Set.new
 
     class << self
-      # Class level instance reader method for @text_elements.
+      # Set of HTML elements that make up the visible text on a page. These
+      # elements are used to initialize the Wgit::Document#text. See the
+      # README.md for how to add to this Set dynamically.
       attr_reader :text_elements
+
+      # Set of Symbols representing the defined Document extractors. Is
+      # read-only. Use Wgit::Document.define_extractor for a new extractor.
+      attr_reader :extractors
     end
 
     # The URL of the webpage, an instance of Wgit::Url.
     attr_reader :url
 
-    # The HTML of the webpage, an instance of String.
+    # The content/HTML of the document, an instance of String.
     attr_reader :html
 
     # The Nokogiri::HTML document object initialized from @html.
-    attr_reader :doc
+    attr_reader :parser
 
-    # The score is only used following a Database#search and records matches.
+    # The score is only used following a `Database#search` and records matches.
     attr_reader :score
 
     # Initialize takes either two strings (representing the URL and HTML) or an
@@ -50,29 +61,31 @@ module Wgit
     # pages retrieved from the database.
     #
     # During initialisation, the Document will call any private
-    # 'init_*_from_html' and 'init_*_from_object' methods it can find. See the
-    # README.md and Wgit::Document.define_extension method for more details.
+    # `init_*_from_html` and `init_*_from_object` methods it can find. See the
+    # Wgit::Document.define_extractor method for more details.
     #
-    # @param url_or_obj [String, Wgit::Url, Object#fetch] Either a String
+    # @param url_or_obj [String, Wgit::Url, #fetch] Either a String
     #   representing a URL or a Hash-like object responding to :fetch. e.g. a
     #   MongoDB collection object. The Object's :fetch method should support
     #   Strings as keys.
-    # @param html [String, NilClass] The crawled web page's HTML. This param is
-    #   only used if url_or_obj is a String representing the web page's URL.
-    #   Otherwise, the HTML comes from the database object. A html of nil will
-    #   be defaulted to an empty String.
-    def initialize(url_or_obj, html = '', encode_html: true)
+    # @param html [String, NilClass] The crawled web page's content/HTML. This
+    #   param is only used if url_or_obj is a String representing the web
+    #   page's URL. Otherwise, the HTML comes from the database object. A html
+    #   of nil will be defaulted to an empty String.
+    # @param encode [Boolean] Whether or not to UTF-8 encode the html. Set to
+    #   false if the Document content is an image etc.
+    def initialize(url_or_obj, html = '', encode: true)
       if url_or_obj.is_a?(String)
-        init_from_strings(url_or_obj, html, encode_html: encode_html)
+        init_from_strings(url_or_obj, html, encode: encode)
       else
-        init_from_object(url_or_obj, encode_html: encode_html)
+        init_from_object(url_or_obj, encode: encode)
       end
     end
 
     ### Document Class Methods ###
 
     # Uses Document.text_elements to build an xpath String, used to obtain
-    # all of the combined text on a webpage.
+    # all of the combined visual text on a webpage.
     #
     # @return [String] An xpath String to obtain a webpage's text elements.
     def self.text_elements_xpath
@@ -88,86 +101,101 @@ module Wgit
       xpath
     end
 
-    # Defines an extension, which is a way to serialise HTML elements into
-    # instance variables upon Document initialization. See the default
-    # extensions defined in 'document_extensions.rb' as examples.
+    # Defines a content extractor, which extracts HTML elements/content
+    # into instance variables upon Document initialization. See the default
+    # extractors defined in 'document_extractors.rb' as examples. Defining an
+    # extractor means that every subsequently crawled/initialized document
+    # will attempt to extract the xpath's content. Use `#xpath` for a one off
+    # content extraction.
     #
-    # Note that defined extensions work for both Documents initialized from
+    # Note that defined extractors work for both Documents initialized from
     # HTML (via Wgit::Crawler methods) and from database objects.
-    # An extension once defined, initializes a private instance variable with
+    # An extractor once defined, initializes a private instance variable with
     # the xpath or database object result(s).
     #
     # When initialising from HTML, a singleton value of true will only
-    # ever return one result; otherwise all xpath results are returned in an
-    # Array. When initialising from a database object, the value is taken as
-    # is and singleton is only used to define the default empty value.
-    # If a value cannot be found (in either the HTML or database object), then
-    # a default will be used. The default value is: `singleton ? nil : []`.
-    #
-    # @param var [Symbol] The name of the variable to be initialised.
-    # @param xpath [String, Object#call] The xpath used to find the element(s)
+    # ever return the first result found; otherwise all the results are
+    # returned in an Array. When initialising from a database object, the value
+    # is taken as is and singleton is only used to define the default empty
+    # value. If a value cannot be found (in either the HTML or database
+    # object), then a default will be used. The default value is:
+    # `singleton ? nil : []`.
+    #
+    # @param var [Symbol] The name of the variable to be initialised, that will
+    #   contain the extracted content. A getter and setter method is defined
+    #   for the initialised variable.
+    # @param xpath [String, #call] The xpath used to find the element(s)
     #   of the webpage. Only used when initializing from HTML.
     #
     #   Pass a callable object (proc etc.) if you want the
     #   xpath value to be derived on Document initialisation (instead of when
-    #   the extension is defined). The call method must return a valid xpath
+    #   the extractor is defined). The call method must return a valid xpath
     #   String.
-    # @param options [Hash] The options to define an extension with. The
+    # @param opts [Hash] The options to define an extractor with. The
     #   options are only used when intializing from HTML, not the database.
-    # @option options [Boolean] :singleton The singleton option determines
+    # @option opts [Boolean] :singleton The singleton option determines
     #   whether or not the result(s) should be in an Array. If multiple
     #   results are found and singleton is true then the first result will be
     #   used. Defaults to true.
-    # @option options [Boolean] :text_content_only The text_content_only option
+    # @option opts [Boolean] :text_content_only The text_content_only option
     #   if true will use the text content of the Nokogiri result object,
     #   otherwise the Nokogiri object itself is returned. Defaults to true.
-    # @yield [value, source, type] Yields the value (Object) about to be
-    #   assigned to the new var, the source of the value (Wgit::Document or DB
-    #   Object) and the source type (Symbol of either :document or :object).
-    #
-    #   The return value of the block becomes the new var value, unless nil.
-    #   Return nil if you want to inspect but not change the var value. The
-    #   block is executed when a Wgit::Document is initialized.
+    # @yield The block is executed when a Wgit::Document is initialized,
+    #   regardless of the source. Use it (optionally) to process the result
+    #   value.
+    # @yieldparam value [Object] The result value to be assigned to the new
+    #   `var`.
+    # @yieldparam source [Wgit::Document, Object] The source of the `value`.
+    # @yieldparam type [Symbol] The `source` type, either `:document` or (DB)
+    #   `:object`.
+    # @yieldreturn [Object] The return value of the block becomes the new var's
+    #   value. Return the block's value param unchanged if you want to inspect.
     # @raise [StandardError] If the var param isn't valid.
-    # @return [Symbol] The given var Symbol.
-    def self.define_extension(var, xpath, options = {}, &block)
+    # @return [Symbol] The given var Symbol if successful.
+    def self.define_extractor(var, xpath, opts = {}, &block)
       var = var.to_sym
-      default_options = { singleton: true, text_content_only: true }
-      options = default_options.merge(options)
+      defaults = { singleton: true, text_content_only: true }
+      opts = defaults.merge(opts)
 
-      raise "var must match #{REGEX_EXTENSION_NAME}" unless \
-      var =~ REGEX_EXTENSION_NAME
+      raise "var must match #{REGEX_EXTRACTOR_NAME}" unless \
+      var =~ REGEX_EXTRACTOR_NAME
 
       # Define the private init_*_from_html method for HTML.
       # Gets the HTML's xpath value and creates a var for it.
       func_name = Document.send(:define_method, "init_#{var}_from_html") do
-        result = find_in_html(xpath, options, &block)
+        result = extract_from_html(xpath, **opts, &block)
         init_var(var, result)
       end
-      Document.send :private, func_name
+      Document.send(:private, func_name)
 
       # Define the private init_*_from_object method for a Database object.
       # Gets the Object's 'key' value and creates a var for it.
-      func_name = Document.send(:define_method, "init_#{var}_from_object") do |obj|
-        result = find_in_object(obj, var.to_s, singleton: options[:singleton], &block)
+      func_name = Document.send(
+        :define_method, "init_#{var}_from_object"
+      ) do |obj|
+        result = extract_from_object(
+          obj, var.to_s, singleton: opts[:singleton], &block
+        )
         init_var(var, result)
       end
-      Document.send :private, func_name
+      Document.send(:private, func_name)
 
+      @extractors << var
       var
     end
 
-    # Removes the init_* methods created when an extension is defined.
-    # Therefore, this is the opposing method to Document.define_extension.
+    # Removes the `init_*` methods created when an extractor is defined.
+    # Therefore, this is the opposing method to `Document.define_extractor`.
     # Returns true if successful or false if the method(s) cannot be found.
     #
-    # @param var [Symbol] The extension variable already defined.
-    # @return [Boolean] True if the extension var was found and removed;
+    # @param var [Symbol] The extractor variable to remove.
+    # @return [Boolean] True if the extractor `var` was found and removed;
     #   otherwise false.
-    def self.remove_extension(var)
+    def self.remove_extractor(var)
       Document.send(:remove_method, "init_#{var}_from_html")
       Document.send(:remove_method, "init_#{var}_from_object")
 
+      @extractors.delete(var.to_sym)
       true
     rescue NameError
       false
@@ -186,7 +214,7 @@ module Wgit
       (@url == other.url) && (@html == other.html)
     end
 
-    # Is a shortcut for calling Document#html[range].
+    # Shortcut for calling Document#html[range].
     #
     # @param range [Range] The range of @html to return.
     # @return [String] The given range of @html.
@@ -196,9 +224,9 @@ module Wgit
 
     # Returns the base URL of this Wgit::Document. The base URL is either the
     # <base> element's href value or @url (if @base is nil). If @base is
-    # present and relative, then @url.to_base + @base is returned. This method
-    # should be used instead of `doc.url.to_base` etc. when manually building
-    # absolute links from relative links; or use `link.prefix_base(doc)`.
+    # present and relative, then @url.to_origin + @base is returned. This method
+    # should be used instead of `doc.url.to_origin` etc. when manually building
+    # absolute links from relative links; or use `link.make_absolute(doc)`.
     #
     # Provide the `link:` parameter to get the correct base URL for that type
     # of link. For example, a link of `#top` would always return @url because
@@ -217,12 +245,16 @@ module Wgit
     # @return [Wgit::Url] The base URL of this Document e.g.
     #   'http://example.com/public'.
     def base_url(link: nil)
-      raise "Document @url ('#{@url}') cannot be relative if <base> is nil" \
       if @url.relative? && @base.nil?
-      raise "Document @url ('#{@url}') and <base> ('#{@base}') both can't be relative" \
+        raise "Document @url ('#{@url}') cannot be relative if <base> is nil"
+      end
+
       if @url.relative? && @base&.relative?
+        raise "Document @url ('#{@url}') and <base> ('#{@base}') both can't \
+be relative"
+      end
 
-      get_base = -> { @base.relative? ? @url.to_base.concat(@base) : @base }
+      get_base = -> { @base.relative? ? @url.to_origin.concat(@base) : @base }
 
       if link
         link = Wgit::Url.new(link)
@@ -234,7 +266,7 @@ module Wgit
         end
       end
 
-      base_url = @base ? get_base.call : @url.to_base
+      base_url = @base ? get_base.call : @url.to_origin
       base_url.omit_fragment.omit_query
     end
 
@@ -248,7 +280,7 @@ module Wgit
     def to_h(include_html: false, include_score: true)
       ignore = include_html ? [] : ['@html']
       ignore << '@score' unless include_score
-      ignore << '@doc' # Always ignore Nokogiri @doc.
+      ignore << '@parser' # Always ignore the Nokogiri object.
 
       Wgit::Utils.to_h(self, ignore: ignore)
     end
@@ -265,7 +297,7 @@ module Wgit
 
     # Returns a Hash containing this Document's instance variables and
     # their #length (if they respond to it). Works dynamically so that any
-    # user defined extensions (and their created instance vars) will appear in
+    # user defined extractors (and their created instance vars) will appear in
     # the returned Hash as well. The number of text snippets as well as total
     # number of textual bytes are always included in the returned Hash.
     #
@@ -275,8 +307,8 @@ module Wgit
       instance_variables.each do |var|
         # Add up the total bytes of text as well as the length.
         if var == :@text
-          hash[:text_snippets] = @text.length
-          hash[:text_bytes]    = @text.sum(&:length)
+          hash[:text]       = @text.length
+          hash[:text_bytes] = @text.sum(&:length)
         # Else take the var's #length method return value.
         else
           next unless instance_variable_get(var).respond_to?(:length)
@@ -305,25 +337,43 @@ module Wgit
     end
 
     # Uses Nokogiri's xpath method to search the doc's html and return the
-    # results.
+    # results. Use `#at_xpath` for returning the first result only.
     #
     # @param xpath [String] The xpath to search the @html with.
     # @return [Nokogiri::XML::NodeSet] The result set of the xpath search.
     def xpath(xpath)
-      @doc.xpath(xpath)
+      @parser.xpath(xpath)
+    end
+
+    # Uses Nokogiri's `at_xpath` method to search the doc's html and return the
+    # result. Use `#xpath` for returning several results.
+    #
+    # @param xpath [String] The xpath to search the @html with.
+    # @return [Nokogiri::XML::Element] The result of the xpath search.
+    def at_xpath(xpath)
+      @parser.at_xpath(xpath)
     end
 
-    # Uses Nokogiri's css method to search the doc's html and return the
-    # results.
+    # Uses Nokogiri's `css` method to search the doc's html and return the
+    # results. Use `#at_css` for returning the first result only.
     #
     # @param selector [String] The CSS selector to search the @html with.
     # @return [Nokogiri::XML::NodeSet] The result set of the CSS search.
     def css(selector)
-      @doc.css(selector)
+      @parser.css(selector)
     end
 
-    # Returns all internal links from this Document in relative form. Internal
-    # meaning a link to another document on the same host.
+    # Uses Nokogiri's `at_css` method to search the doc's html and return the
+    # result. Use `#css` for returning several results.
+    #
+    # @param selector [String] The CSS selector to search the @html with.
+    # @return [Nokogiri::XML::Element] The result of the CSS search.
+    def at_css(selector)
+      @parser.at_css(selector)
+    end
+
+    # Returns all unique internal links from this Document in relative form.
+    # Internal meaning a link to another document on the same host.
     #
     # This Document's host is used to determine if an absolute URL is actually
     # a relative link e.g. For a Document representing
@@ -332,41 +382,48 @@ module Wgit
     # as an internal link because both Documents live on the same host. Also
     # see Wgit::Document#internal_absolute_links.
     #
-    # @return [Array<Wgit::Url>] Self's internal Url's in relative form.
+    # @return [Array<Wgit::Url>] Self's unique internal Url's in relative form.
     def internal_links
       return [] if @links.empty?
 
       links = @links
-              .select { |link| link.relative?(host: @url.to_base) }
+              .select { |link| link.relative?(host: @url.to_origin) }
               .map(&:omit_base)
               .map do |link| # Map @url.to_host into / as it's a duplicate.
         link.to_host == @url.to_host ? Wgit::Url.new('/') : link
       end
 
-      Wgit::Utils.process_arr(links)
+      Wgit::Utils.sanitize(links)
     end
 
-    # Returns all internal links from this Document in absolute form by
+    # Returns all unique internal links from this Document in absolute form by
     # appending them to self's #base_url. Also see
     # Wgit::Document#internal_links.
     #
-    # @return [Array<Wgit::Url>] Self's internal Url's in absolute form.
+    # @return [Array<Wgit::Url>] Self's unique internal Url's in absolute form.
     def internal_absolute_links
-      internal_links.map { |link| link.prefix_base(self) }
+      internal_links.map { |link| link.make_absolute(self) }
     end
 
-    # Returns all external links from this Document in absolute form. External
-    # meaning a link to a different host.
+    # Returns all unique external links from this Document in absolute form.
+    # External meaning a link to a different host.
     #
-    # @return [Array<Wgit::Url>] Self's external Url's in absolute form.
+    # @return [Array<Wgit::Url>] Self's unique external Url's in absolute form.
     def external_links
       return [] if @links.empty?
 
       links = @links
-              .reject { |link| link.relative?(host: @url.to_base) }
+              .map do |link|
+                if link.scheme_relative?
+                  link.prefix_scheme(@url.to_scheme.to_sym)
+                else
+                  link
+                end
+              end
+              .reject { |link| link.relative?(host: @url.to_origin) }
               .map(&:omit_trailing_slash)
 
-      Wgit::Utils.process_arr(links)
+      Wgit::Utils.sanitize(links)
     end
 
     # Searches the @text for the given query and returns the results.
@@ -381,8 +438,8 @@ module Wgit
     # original sentence, which ever is less. The algorithm obviously ensures
     # that the search query is visible somewhere in the sentence.
     #
-    # @param query [String, Object#to_s] The value to search the document's
-    #   @text for.
+    # @param query [Regexp, #to_s] The regex or text value to search the
+    #   document's @text for.
     # @param case_sensitive [Boolean] Whether character case must match.
     # @param whole_sentence [Boolean] Whether multiple words should be searched
     #   for separately.
@@ -390,21 +447,27 @@ module Wgit
     #   sentence.
     # @return [Array<String>] A subset of @text, matching the query.
     def search(
-      query, case_sensitive: false, whole_sentence: false, sentence_limit: 80
+      query, case_sensitive: false, whole_sentence: true, sentence_limit: 80
     )
-      query = query.to_s
-      raise 'A search query must be provided' if query.empty?
       raise 'The sentence_limit value must be even' if sentence_limit.odd?
 
-      query   = query.gsub(' ', '|') unless whole_sentence
-      regex   = Regexp.new(query, !case_sensitive)
+      if query.is_a?(Regexp)
+        regex = query
+      else # respond_to? #to_s == true
+        query = query.to_s
+        query = query.gsub(' ', '|') unless whole_sentence
+        regex = Regexp.new(query, !case_sensitive)
+      end
+
       results = {}
 
       @text.each do |sentence|
+        sentence = sentence.strip
+        next if results[sentence]
+
         hits = sentence.scan(regex).count
         next unless hits.positive?
 
-        sentence.strip!
         index = sentence.index(regex) # Index of first match.
         Wgit::Utils.format_sentence_length(sentence, index, sentence_limit)
 
@@ -422,8 +485,8 @@ module Wgit
     # functionality. The original text is returned; no other reference to it
     # is kept thereafter.
     #
-    # @param query [String, Object#to_s] The value to search the document's
-    #   @text for.
+    # @param query [Regexp, #to_s] The regex or text value to search the
+    #   document's @text for.
     # @param case_sensitive [Boolean] Whether character case must match.
     # @param whole_sentence [Boolean] Whether multiple words should be searched
     #   for separately.
@@ -431,7 +494,7 @@ module Wgit
     #   sentence.
     # @return [String] This Document's original @text value.
     def search!(
-      query, case_sensitive: false, whole_sentence: false, sentence_limit: 80
+      query, case_sensitive: false, whole_sentence: true, sentence_limit: 80
     )
       orig_text = @text
       @text = search(
@@ -442,104 +505,114 @@ module Wgit
       orig_text
     end
 
+    # Extracts a value/object from this Document's @html using the given xpath
+    # parameter.
+    #
+    # @param xpath [String, #call] Used to find the value/object in @html.
+    # @param singleton [Boolean] singleton ? results.first (single Nokogiri
+    #   Object) : results (Array).
+    # @param text_content_only [Boolean] text_content_only ? result.content
+    #   (String) : result (Nokogiri Object).
+    # @return [String, Object] The value found in the html or the default value
+    #   (singleton ? nil : []).
+    def extract(xpath, singleton: true, text_content_only: true)
+      send(
+        :extract_from_html, xpath,
+        singleton: singleton, text_content_only: text_content_only
+      )
+    end
+
     protected
 
     # Initializes the nokogiri object using @html, which cannot be nil.
     # Override this method to custom configure the Nokogiri object returned.
     # Gets called from Wgit::Document.new upon initialization.
     #
+    # @yield [config] The given block is passed to Nokogiri::HTML for
+    #   initialisation.
     # @raise [StandardError] If @html isn't set.
     # @return [Nokogiri::HTML] The initialised Nokogiri HTML object.
-    def init_nokogiri
+    def init_nokogiri(&block)
       raise '@html must be set' unless @html
 
-      Nokogiri::HTML(@html) do |config|
-        # TODO: Remove #'s below when crawling in production.
-        # config.options = Nokogiri::XML::ParseOptions::STRICT |
-        #                 Nokogiri::XML::ParseOptions::NONET
-      end
+      Nokogiri::HTML(@html, &block)
     end
 
-    # Returns a value/object from this Document's @html using the given xpath
+    # Extracts a value/object from this Document's @html using the given xpath
     # parameter.
     #
-    # @param xpath [String] Used to find the value/object in @html.
+    # @param xpath [String, #call] Used to find the value/object in @html.
     # @param singleton [Boolean] singleton ? results.first (single Nokogiri
     #   Object) : results (Array).
     # @param text_content_only [Boolean] text_content_only ? result.content
     #   (String) : result (Nokogiri Object).
-    # @yield [value, source] Given the value (String/Object) before it's set as
-    #   an instance variable so that you can inspect/alter the value if
-    #   desired. Return nil from the block if you don't want to override the
-    #   value. Also given the source (Symbol) which is always :document.
+    # @yield The block is executed when a Wgit::Document is initialized,
+    #   regardless of the source. Use it (optionally) to process the result
+    #   value.
+    # @yieldparam value [Object] The result value to be returned.
+    # @yieldparam source [Wgit::Document, Object] The source of the `value`.
+    # @yieldparam type [Symbol] The `source` type, either `:document` or (DB)
+    #   `:object`.
+    # @yieldreturn [Object] The return value of the block gets returned. Return
+    #   the block's `value` param unchanged if you simply want to inspect it.
     # @return [String, Object] The value found in the html or the default value
     #   (singleton ? nil : []).
-    def find_in_html(xpath, singleton: true, text_content_only: true)
-      default = singleton ? nil : []
-      xpath   = xpath.call if xpath.respond_to?(:call)
-      results = @doc.xpath(xpath)
-
-      return default if results.nil? || results.empty?
-
-      result = if singleton
-                 text_content_only ? results.first.content : results.first
-               else
-                 text_content_only ? results.map(&:content) : results
-               end
-
-      singleton ? Wgit::Utils.process_str(result) : Wgit::Utils.process_arr(result)
+    def extract_from_html(xpath, singleton: true, text_content_only: true)
+      xpath  = xpath.call if xpath.respond_to?(:call)
+      result = singleton ? @parser.at_xpath(xpath) : @parser.xpath(xpath)
 
-      if block_given?
-        new_result = yield(result, self, :document)
-        result = new_result unless new_result.nil?
+      if text_content_only
+        result = singleton ? result&.content : result.map(&:content)
       end
 
+      Wgit::Utils.sanitize(result)
+      result = yield(result, self, :document) if block_given?
       result
     end
 
-    # Returns a value from the obj using the given key via obj#fetch.
+    # Returns a value from the obj using the given key via `obj#fetch`.
     #
-    # @param obj [Object#fetch] The object containing the key/value.
+    # @param obj [#fetch] The object containing the key/value.
     # @param key [String] Used to find the value in the obj.
     # @param singleton [Boolean] True if a single value, false otherwise.
-    # @yield [value, source] Given the value (String/Object) before it's set as
-    #   an instance variable so that you can inspect/alter the value if
-    #   desired. Return nil from the block if you don't want to override the
-    #   value. Also given the source (Symbol) which is always :object.
+    # @yield The block is executed when a Wgit::Document is initialized,
+    #   regardless of the source. Use it (optionally) to process the result
+    #   value.
+    # @yieldparam value [Object] The result value to be returned.
+    # @yieldparam source [Wgit::Document, Object] The source of the `value`.
+    # @yieldparam type [Symbol] The `source` type, either `:document` or (DB)
+    #   `:object`.
+    # @yieldreturn [Object] The return value of the block gets returned. Return
+    #   the block's `value` param unchanged if you simply want to inspect it.
     # @return [String, Object] The value found in the obj or the default value
     #   (singleton ? nil : []).
-    def find_in_object(obj, key, singleton: true)
+    def extract_from_object(obj, key, singleton: true)
       assert_respond_to(obj, :fetch)
 
       default = singleton ? nil : []
       result  = obj.fetch(key.to_s, default)
 
-      singleton ? Wgit::Utils.process_str(result) : Wgit::Utils.process_arr(result)
-
-      if block_given?
-        new_result = yield(result, obj, :object)
-        result = new_result unless new_result.nil?
-      end
-
+      Wgit::Utils.sanitize(result)
+      result = yield(result, obj, :object) if block_given?
       result
     end
 
     private
 
     # Initialise the Document from URL and HTML Strings.
-    def init_from_strings(url, html, encode_html: true)
+    def init_from_strings(url, html, encode: true)
       assert_types(html, [String, NilClass])
 
       # We already know url.is_a?(String) so parse into Url unless already so.
       url = Wgit::Url.parse(url)
       url.crawled = true unless url.crawled? # Avoid overriding date_crawled.
 
-      @url   = url
-      @html  = html || ''
-      @doc   = init_nokogiri
-      @score = 0.0
+      @url    = url
+      @html   = html || ''
+      @parser = init_nokogiri
+      @score  = 0.0
 
-      Wgit::Utils.process_str(@html, encode: encode_html)
+      Wgit::Utils.sanitize(@html, encode: encode)
 
       # Dynamically run the init_*_from_html methods.
       Document.private_instance_methods(false).each do |method|
@@ -552,15 +625,15 @@ module Wgit
 
     # Initialise the Document from a Hash like Object containing Strings as
     # keys e.g. database collection object or Hash.
-    def init_from_object(obj, encode_html: true)
+    def init_from_object(obj, encode: true)
       assert_respond_to(obj, :fetch)
 
-      @url   = Wgit::Url.new(obj.fetch('url')) # Should always be present.
-      @html  = obj.fetch('html', '')
-      @doc   = init_nokogiri
-      @score = obj.fetch('score', 0.0)
+      @url    = Wgit::Url.new(obj.fetch('url')) # Should always be present.
+      @html   = obj.fetch('html', '')
+      @parser = init_nokogiri
+      @score  = obj.fetch('score', 0.0)
 
-      Wgit::Utils.process_str(@html, encode: encode_html)
+      Wgit::Utils.sanitize(@html, encode: encode)
 
       # Dynamically run the init_*_from_object methods.
       Document.private_instance_methods(false).each do |method|
@@ -571,11 +644,11 @@ module Wgit
       end
     end
 
-    # Initialises an instance variable and defines a getter method for it.
+    # Initialises an instance variable and defines an accessor method for it.
     #
     # @param var [Symbol] The name of the variable to be initialized.
     # @param value [Object] The newly initialized variable's value.
-    # @return [Symbol] The name of the newly created getter method.
+    # @return [Symbol] The name of the defined getter method.
     def init_var(var, value)
       # instance_var_name starts with @, var_name doesn't.
       var = var.to_s
@@ -583,10 +656,9 @@ module Wgit
       instance_var_name = "@#{var_name}".to_sym
 
       instance_variable_set(instance_var_name, value)
+      Wgit::Document.attr_accessor(var_name)
 
-      Document.send(:define_method, var_name) do
-        instance_variable_get(instance_var_name)
-      end
+      var_name
     end
 
     alias content                html