wgit 0.7.0 → 0.10.1

data/lib/wgit/document.rb

@@ -6,29 +6,41 @@ require 'json'
 require 'set'
 
 module Wgit
-  # Class primarily modeling a HTML web document, although other MIME types
+  # Class modeling/serialising a HTML web document, although other MIME types
   # will work e.g. images etc. Also doubles as a search result when
   # loading Documents from the database via `Wgit::Database#search`.
   #
   # The initialize method dynamically initializes instance variables from the
   # Document HTML / Database object e.g. text. This bit is dynamic so that the
-  # Document class can be easily extended allowing you to pull out the bits of
-  # a webpage that are important to you. See `Wgit::Document.define_extension`.
+  # Document class can be easily extended allowing you to extract the bits of
+  # a webpage that are important to you. See `Wgit::Document.define_extractor`.
   class Document
     include Assertable
 
-    # Regex for the allowed var names when defining an extension.
-    REGEX_EXTENSION_NAME = /[a-z0-9_]+/.freeze
+    # Regex for the allowed var names when defining an extractor.
+    REGEX_EXTRACTOR_NAME = /[a-z0-9_]+/.freeze
 
-    # The xpath used to extract the visible text on a page.
-    TEXT_ELEMENTS_XPATH = '//*/text()'.freeze
+    # Set of text elements used to build Document#text.
+    @text_elements = Set.new(%i[
+      a abbr address article aside b bdi bdo blockquote button caption cite
+      code data dd del details dfn div dl dt em figcaption figure footer h1 h2
+      h3 h4 h5 h6 header hr i input ins kbd label legend li main mark meter ol
+      option output p pre q rb rt ruby s samp section small span strong sub
+      summary sup td textarea th time u ul var wbr
+    ])
 
-    # Set of Symbols representing the defined Document extensions.
-    @extensions = Set.new
+    # Set of Symbols representing the defined Document extractors.
+    @extractors = Set.new
 
     class << self
-      # Class level attr_reader for the Document defined extensions.
-      attr_reader :extensions
+      # Set of HTML elements that make up the visible text on a page. These
+      # elements are used to initialize the Wgit::Document#text. See the
+      # README.md for how to add to this Set dynamically.
+      attr_reader :text_elements
+
+      # Set of Symbols representing the defined Document extractors. Is
+      # read-only. Use Wgit::Document.define_extractor for a new extractor.
+      attr_reader :extractors
     end
 
     # The URL of the webpage, an instance of Wgit::Url.
@@ -38,7 +50,7 @@ module Wgit
     attr_reader :html
 
     # The Nokogiri::HTML document object initialized from @html.
-    attr_reader :doc
+    attr_reader :parser
 
     # The score is only used following a `Database#search` and records matches.
     attr_reader :score
@@ -50,7 +62,7 @@ module Wgit
     #
     # During initialisation, the Document will call any private
     # `init_*_from_html` and `init_*_from_object` methods it can find. See the
-    # README.md and Wgit::Document.define_extension method for more details.
+    # Wgit::Document.define_extractor method for more details.
     #
     # @param url_or_obj [String, Wgit::Url, #fetch] Either a String
     #   representing a URL or a Hash-like object responding to :fetch. e.g. a
@@ -72,31 +84,54 @@ module Wgit
 
     ### Document Class Methods ###
 
-    # Defines an extension, which is a way to serialise HTML elements into
-    # instance variables upon Document initialization. See the default
-    # extensions defined in 'document_extensions.rb' as examples.
+    # Uses Document.text_elements to build an xpath String, used to obtain
+    # all of the combined visual text on a webpage.
     #
-    # Note that defined extensions work for both Documents initialized from
+    # @return [String] An xpath String to obtain a webpage's text elements.
+    def self.text_elements_xpath
+      xpath = ''
+      return xpath if Wgit::Document.text_elements.empty?
+
+      el_xpath = '//%s/text()'
+      Wgit::Document.text_elements.each_with_index do |el, i|
+        xpath += ' | ' unless i.zero?
+        xpath += format(el_xpath, el)
+      end
+
+      xpath
+    end
+
+    # Defines a content extractor, which extracts HTML elements/content
+    # into instance variables upon Document initialization. See the default
+    # extractors defined in 'document_extractors.rb' as examples. Defining an
+    # extractor means that every subsequently crawled/initialized document
+    # will attempt to extract the xpath's content. Use `#xpath` for a one off
+    # content extraction.
+    #
+    # Note that defined extractors work for both Documents initialized from
     # HTML (via Wgit::Crawler methods) and from database objects.
-    # An extension once defined, initializes a private instance variable with
+    # An extractor once defined, initializes a private instance variable with
     # the xpath or database object result(s).
     #
     # When initialising from HTML, a singleton value of true will only
-    # ever return one result; otherwise all xpath results are returned in an
-    # Array. When initialising from a database object, the value is taken as
-    # is and singleton is only used to define the default empty value.
-    # If a value cannot be found (in either the HTML or database object), then
-    # a default will be used. The default value is: `singleton ? nil : []`.
-    #
-    # @param var [Symbol] The name of the variable to be initialised.
+    # ever return the first result found; otherwise all the results are
+    # returned in an Array. When initialising from a database object, the value
+    # is taken as is and singleton is only used to define the default empty
+    # value. If a value cannot be found (in either the HTML or database
+    # object), then a default will be used. The default value is:
+    # `singleton ? nil : []`.
+    #
+    # @param var [Symbol] The name of the variable to be initialised, that will
+    #   contain the extracted content. A getter and setter method is defined
+    #   for the initialised variable.
     # @param xpath [String, #call] The xpath used to find the element(s)
     #   of the webpage. Only used when initializing from HTML.
     #
     #   Pass a callable object (proc etc.) if you want the
     #   xpath value to be derived on Document initialisation (instead of when
-    #   the extension is defined). The call method must return a valid xpath
+    #   the extractor is defined). The call method must return a valid xpath
     #   String.
-    # @param opts [Hash] The options to define an extension with. The
+    # @param opts [Hash] The options to define an extractor with. The
     #   options are only used when intializing from HTML, not the database.
     # @option opts [Boolean] :singleton The singleton option determines
     #   whether or not the result(s) should be in an Array. If multiple
@@ -105,56 +140,62 @@ module Wgit
     # @option opts [Boolean] :text_content_only The text_content_only option
     #   if true will use the text content of the Nokogiri result object,
     #   otherwise the Nokogiri object itself is returned. Defaults to true.
-    # @yieldparam value [Object] The value to be assigned to the new var.
-    # @yieldparam source [Wgit::Document, Object] The source of the value.
-    # @yieldparam type [Symbol] The source type, either :document or (DB)
-    #   :object.
-    # @yieldreturn [Object] The return value of the block becomes the new var
-    #   value, unless nil. Return nil if you want to inspect but not change the
-    #   var value. The block is executed when a Wgit::Document is initialized,
-    #   regardless of the source.
+    # @yield The block is executed when a Wgit::Document is initialized,
+    #   regardless of the source. Use it (optionally) to process the result
+    #   value.
+    # @yieldparam value [Object] The result value to be assigned to the new
+    #   `var`.
+    # @yieldparam source [Wgit::Document, Object] The source of the `value`.
+    # @yieldparam type [Symbol] The `source` type, either `:document` or (DB)
+    #   `:object`.
+    # @yieldreturn [Object] The return value of the block becomes the new var's
+    #   value. Return the block's value param unchanged if you want to inspect.
     # @raise [StandardError] If the var param isn't valid.
     # @return [Symbol] The given var Symbol if successful.
-    def self.define_extension(var, xpath, opts = {}, &block)
+    def self.define_extractor(var, xpath, opts = {}, &block)
       var = var.to_sym
       defaults = { singleton: true, text_content_only: true }
       opts = defaults.merge(opts)
 
-      raise "var must match #{REGEX_EXTENSION_NAME}" unless \
-      var =~ REGEX_EXTENSION_NAME
+      raise "var must match #{REGEX_EXTRACTOR_NAME}" unless \
+      var =~ REGEX_EXTRACTOR_NAME
 
       # Define the private init_*_from_html method for HTML.
       # Gets the HTML's xpath value and creates a var for it.
       func_name = Document.send(:define_method, "init_#{var}_from_html") do
-        result = find_in_html(xpath, opts, &block)
+        result = extract_from_html(xpath, **opts, &block)
         init_var(var, result)
       end
       Document.send(:private, func_name)
 
       # Define the private init_*_from_object method for a Database object.
       # Gets the Object's 'key' value and creates a var for it.
-      func_name = Document.send(:define_method, "init_#{var}_from_object") do |obj|
-        result = find_in_object(obj, var.to_s, singleton: opts[:singleton], &block)
+      func_name = Document.send(
+        :define_method, "init_#{var}_from_object"
+      ) do |obj|
+        result = extract_from_object(
+          obj, var.to_s, singleton: opts[:singleton], &block
+        )
         init_var(var, result)
       end
       Document.send(:private, func_name)
 
-      @extensions << var
+      @extractors << var
       var
     end
 
-    # Removes the init_* methods created when an extension is defined.
-    # Therefore, this is the opposing method to Document.define_extension.
+    # Removes the `init_*` methods created when an extractor is defined.
+    # Therefore, this is the opposing method to `Document.define_extractor`.
     # Returns true if successful or false if the method(s) cannot be found.
     #
-    # @param var [Symbol] The extension variable already defined.
-    # @return [Boolean] True if the extension var was found and removed;
+    # @param var [Symbol] The extractor variable to remove.
+    # @return [Boolean] True if the extractor `var` was found and removed;
     #   otherwise false.
-    def self.remove_extension(var)
+    def self.remove_extractor(var)
       Document.send(:remove_method, "init_#{var}_from_html")
       Document.send(:remove_method, "init_#{var}_from_object")
 
-      @extensions.delete(var.to_sym)
+      @extractors.delete(var.to_sym)
       true
     rescue NameError
       false
@@ -173,7 +214,7 @@ module Wgit
       (@url == other.url) && (@html == other.html)
     end
 
-    # Is a shortcut for calling Document#html[range].
+    # Shortcut for calling Document#html[range].
     #
     # @param range [Range] The range of @html to return.
     # @return [String] The given range of @html.
@@ -183,9 +224,9 @@ module Wgit
 
     # Returns the base URL of this Wgit::Document. The base URL is either the
     # <base> element's href value or @url (if @base is nil). If @base is
-    # present and relative, then @url.to_base + @base is returned. This method
-    # should be used instead of `doc.url.to_base` etc. when manually building
-    # absolute links from relative links; or use `link.prefix_base(doc)`.
+    # present and relative, then @url.to_origin + @base is returned. This method
+    # should be used instead of `doc.url.to_origin` etc. when manually building
+    # absolute links from relative links; or use `link.make_absolute(doc)`.
     #
     # Provide the `link:` parameter to get the correct base URL for that type
     # of link. For example, a link of `#top` would always return @url because
@@ -204,12 +245,16 @@ module Wgit
     # @return [Wgit::Url] The base URL of this Document e.g.
     #   'http://example.com/public'.
     def base_url(link: nil)
-      raise "Document @url ('#{@url}') cannot be relative if <base> is nil" \
       if @url.relative? && @base.nil?
-      raise "Document @url ('#{@url}') and <base> ('#{@base}') both can't be relative" \
+        raise "Document @url ('#{@url}') cannot be relative if <base> is nil"
+      end
+
       if @url.relative? && @base&.relative?
+        raise "Document @url ('#{@url}') and <base> ('#{@base}') both can't \
+be relative"
+      end
 
-      get_base = -> { @base.relative? ? @url.to_base.concat(@base) : @base }
+      get_base = -> { @base.relative? ? @url.to_origin.concat(@base) : @base }
 
       if link
         link = Wgit::Url.new(link)
@@ -221,7 +266,7 @@ module Wgit
         end
       end
 
-      base_url = @base ? get_base.call : @url.to_base
+      base_url = @base ? get_base.call : @url.to_origin
       base_url.omit_fragment.omit_query
     end
 
@@ -235,7 +280,7 @@ module Wgit
     def to_h(include_html: false, include_score: true)
       ignore = include_html ? [] : ['@html']
       ignore << '@score' unless include_score
-      ignore << '@doc' # Always ignore Nokogiri @doc.
+      ignore << '@parser' # Always ignore the Nokogiri object.
 
       Wgit::Utils.to_h(self, ignore: ignore)
     end
@@ -252,7 +297,7 @@ module Wgit
 
     # Returns a Hash containing this Document's instance variables and
     # their #length (if they respond to it). Works dynamically so that any
-    # user defined extensions (and their created instance vars) will appear in
+    # user defined extractors (and their created instance vars) will appear in
     # the returned Hash as well. The number of text snippets as well as total
     # number of textual bytes are always included in the returned Hash.
     #
@@ -262,8 +307,8 @@ module Wgit
       instance_variables.each do |var|
         # Add up the total bytes of text as well as the length.
         if var == :@text
-          hash[:text_snippets] = @text.length
-          hash[:text_bytes]    = @text.sum(&:length)
+          hash[:text]       = @text.length
+          hash[:text_bytes] = @text.sum(&:length)
         # Else take the var's #length method return value.
         else
           next unless instance_variable_get(var).respond_to?(:length)
@@ -292,25 +337,43 @@ module Wgit
     end
 
     # Uses Nokogiri's xpath method to search the doc's html and return the
-    # results.
+    # results. Use `#at_xpath` for returning the first result only.
     #
     # @param xpath [String] The xpath to search the @html with.
     # @return [Nokogiri::XML::NodeSet] The result set of the xpath search.
     def xpath(xpath)
-      @doc.xpath(xpath)
+      @parser.xpath(xpath)
     end
 
-    # Uses Nokogiri's css method to search the doc's html and return the
-    # results.
+    # Uses Nokogiri's `at_xpath` method to search the doc's html and return the
+    # result. Use `#xpath` for returning several results.
+    #
+    # @param xpath [String] The xpath to search the @html with.
+    # @return [Nokogiri::XML::Element] The result of the xpath search.
+    def at_xpath(xpath)
+      @parser.at_xpath(xpath)
+    end
+
+    # Uses Nokogiri's `css` method to search the doc's html and return the
+    # results. Use `#at_css` for returning the first result only.
     #
     # @param selector [String] The CSS selector to search the @html with.
     # @return [Nokogiri::XML::NodeSet] The result set of the CSS search.
     def css(selector)
-      @doc.css(selector)
+      @parser.css(selector)
+    end
+
+    # Uses Nokogiri's `at_css` method to search the doc's html and return the
+    # result. Use `#css` for returning several results.
+    #
+    # @param selector [String] The CSS selector to search the @html with.
+    # @return [Nokogiri::XML::Element] The result of the CSS search.
+    def at_css(selector)
+      @parser.at_css(selector)
     end
 
-    # Returns all internal links from this Document in relative form. Internal
-    # meaning a link to another document on the same host.
+    # Returns all unique internal links from this Document in relative form.
+    # Internal meaning a link to another document on the same host.
     #
     # This Document's host is used to determine if an absolute URL is actually
     # a relative link e.g. For a Document representing
@@ -319,41 +382,48 @@ module Wgit
     # as an internal link because both Documents live on the same host. Also
     # see Wgit::Document#internal_absolute_links.
     #
-    # @return [Array<Wgit::Url>] Self's internal Url's in relative form.
+    # @return [Array<Wgit::Url>] Self's unique internal Url's in relative form.
     def internal_links
       return [] if @links.empty?
 
       links = @links
-              .select { |link| link.relative?(host: @url.to_base) }
+              .select { |link| link.relative?(host: @url.to_origin) }
               .map(&:omit_base)
               .map do |link| # Map @url.to_host into / as it's a duplicate.
         link.to_host == @url.to_host ? Wgit::Url.new('/') : link
       end
 
-      Wgit::Utils.process_arr(links)
+      Wgit::Utils.sanitize(links)
     end
 
-    # Returns all internal links from this Document in absolute form by
+    # Returns all unique internal links from this Document in absolute form by
     # appending them to self's #base_url. Also see
     # Wgit::Document#internal_links.
     #
-    # @return [Array<Wgit::Url>] Self's internal Url's in absolute form.
+    # @return [Array<Wgit::Url>] Self's unique internal Url's in absolute form.
     def internal_absolute_links
-      internal_links.map { |link| link.prefix_base(self) }
+      internal_links.map { |link| link.make_absolute(self) }
     end
 
-    # Returns all external links from this Document in absolute form. External
-    # meaning a link to a different host.
+    # Returns all unique external links from this Document in absolute form.
+    # External meaning a link to a different host.
     #
-    # @return [Array<Wgit::Url>] Self's external Url's in absolute form.
+    # @return [Array<Wgit::Url>] Self's unique external Url's in absolute form.
     def external_links
       return [] if @links.empty?
 
       links = @links
-              .reject { |link| link.relative?(host: @url.to_base) }
+              .map do |link|
+                if link.scheme_relative?
+                  link.prefix_scheme(@url.to_scheme.to_sym)
+                else
+                  link
+                end
+              end
+              .reject { |link| link.relative?(host: @url.to_origin) }
               .map(&:omit_trailing_slash)
 
-      Wgit::Utils.process_arr(links)
+      Wgit::Utils.sanitize(links)
     end
 
     # Searches the @text for the given query and returns the results.
@@ -368,8 +438,8 @@ module Wgit
     # original sentence, which ever is less. The algorithm obviously ensures
     # that the search query is visible somewhere in the sentence.
     #
-    # @param query [String, #to_s] The value to search the document's
-    #   @text for.
+    # @param query [Regexp, #to_s] The regex or text value to search the
+    #   document's @text for.
     # @param case_sensitive [Boolean] Whether character case must match.
     # @param whole_sentence [Boolean] Whether multiple words should be searched
     #   for separately.
@@ -379,12 +449,16 @@ module Wgit
     def search(
       query, case_sensitive: false, whole_sentence: true, sentence_limit: 80
     )
-      query = query.to_s
-      raise 'A search query must be provided' if query.empty?
       raise 'The sentence_limit value must be even' if sentence_limit.odd?
 
-      query   = query.gsub(' ', '|') unless whole_sentence
-      regex   = Regexp.new(query, !case_sensitive)
+      if query.is_a?(Regexp)
+        regex = query
+      else # respond_to? #to_s == true
+        query = query.to_s
+        query = query.gsub(' ', '|') unless whole_sentence
+        regex = Regexp.new(query, !case_sensitive)
+      end
+
       results = {}
 
       @text.each do |sentence|
@@ -411,8 +485,8 @@ module Wgit
     # functionality. The original text is returned; no other reference to it
     # is kept thereafter.
     #
-    # @param query [String, #to_s] The value to search the document's
-    #   @text for.
+    # @param query [Regexp, #to_s] The regex or text value to search the
+    #   document's @text for.
     # @param case_sensitive [Boolean] Whether character case must match.
     # @param whole_sentence [Boolean] Whether multiple words should be searched
     #   for separately.
@@ -431,85 +505,95 @@ module Wgit
       orig_text
     end
 
+    # Extracts a value/object from this Document's @html using the given xpath
+    # parameter.
+    #
+    # @param xpath [String, #call] Used to find the value/object in @html.
+    # @param singleton [Boolean] singleton ? results.first (single Nokogiri
+    #   Object) : results (Array).
+    # @param text_content_only [Boolean] text_content_only ? result.content
+    #   (String) : result (Nokogiri Object).
+    # @return [String, Object] The value found in the html or the default value
+    #   (singleton ? nil : []).
+    def extract(xpath, singleton: true, text_content_only: true)
+      send(
+        :extract_from_html, xpath,
+        singleton: singleton, text_content_only: text_content_only
+      )
+    end
+
     protected
 
     # Initializes the nokogiri object using @html, which cannot be nil.
     # Override this method to custom configure the Nokogiri object returned.
     # Gets called from Wgit::Document.new upon initialization.
     #
+    # @yield [config] The given block is passed to Nokogiri::HTML for
+    #   initialisation.
     # @raise [StandardError] If @html isn't set.
     # @return [Nokogiri::HTML] The initialised Nokogiri HTML object.
-    def init_nokogiri
+    def init_nokogiri(&block)
       raise '@html must be set' unless @html
 
-      Nokogiri::HTML(@html) do |config|
-        # TODO: Remove #'s below when crawling in production.
-        # config.options = Nokogiri::XML::ParseOptions::STRICT |
-        #                 Nokogiri::XML::ParseOptions::NONET
-      end
+      Nokogiri::HTML(@html, &block)
     end
 
-    # Returns a value/object from this Document's @html using the given xpath
+    # Extracts a value/object from this Document's @html using the given xpath
     # parameter.
     #
-    # @param xpath [String] Used to find the value/object in @html.
+    # @param xpath [String, #call] Used to find the value/object in @html.
     # @param singleton [Boolean] singleton ? results.first (single Nokogiri
     #   Object) : results (Array).
     # @param text_content_only [Boolean] text_content_only ? result.content
     #   (String) : result (Nokogiri Object).
-    # @yield [value, source] Given the value (String/Object) before it's set as
-    #   an instance variable so that you can inspect/alter the value if
-    #   desired. Return nil from the block if you don't want to override the
-    #   value. Also given the source (Symbol) which is always :document.
+    # @yield The block is executed when a Wgit::Document is initialized,
+    #   regardless of the source. Use it (optionally) to process the result
+    #   value.
+    # @yieldparam value [Object] The result value to be returned.
+    # @yieldparam source [Wgit::Document, Object] The source of the `value`.
+    # @yieldparam type [Symbol] The `source` type, either `:document` or (DB)
+    #   `:object`.
+    # @yieldreturn [Object] The return value of the block gets returned. Return
+    #   the block's `value` param unchanged if you simply want to inspect it.
     # @return [String, Object] The value found in the html or the default value
     #   (singleton ? nil : []).
-    def find_in_html(xpath, singleton: true, text_content_only: true)
-      default = singleton ? nil : []
-      xpath   = xpath.call if xpath.respond_to?(:call)
-      results = @doc.xpath(xpath)
-
-      return default if results.nil? || results.empty?
-
-      result = if singleton
-                 text_content_only ? results.first.content : results.first
-               else
-                 text_content_only ? results.map(&:content) : results
-               end
+    def extract_from_html(xpath, singleton: true, text_content_only: true)
+      xpath  = xpath.call if xpath.respond_to?(:call)
+      result = singleton ? @parser.at_xpath(xpath) : @parser.xpath(xpath)
 
-      singleton ? Wgit::Utils.process_str(result) : Wgit::Utils.process_arr(result)
-
-      if block_given?
-        new_result = yield(result, self, :document)
-        result = new_result unless new_result.nil?
+      if text_content_only
+        result = singleton ? result&.content : result.map(&:content)
       end
 
+      Wgit::Utils.sanitize(result)
+      result = yield(result, self, :document) if block_given?
       result
     end
 
-    # Returns a value from the obj using the given key via obj#fetch.
+    # Returns a value from the obj using the given key via `obj#fetch`.
     #
     # @param obj [#fetch] The object containing the key/value.
     # @param key [String] Used to find the value in the obj.
     # @param singleton [Boolean] True if a single value, false otherwise.
-    # @yield [value, source] Given the value (String/Object) before it's set as
-    #   an instance variable so that you can inspect/alter the value if
-    #   desired. Return nil from the block if you don't want to override the
-    #   value. Also given the source (Symbol) which is always :object.
+    # @yield The block is executed when a Wgit::Document is initialized,
+    #   regardless of the source. Use it (optionally) to process the result
+    #   value.
+    # @yieldparam value [Object] The result value to be returned.
+    # @yieldparam source [Wgit::Document, Object] The source of the `value`.
+    # @yieldparam type [Symbol] The `source` type, either `:document` or (DB)
+    #   `:object`.
+    # @yieldreturn [Object] The return value of the block gets returned. Return
+    #   the block's `value` param unchanged if you simply want to inspect it.
     # @return [String, Object] The value found in the obj or the default value
     #   (singleton ? nil : []).
-    def find_in_object(obj, key, singleton: true)
+    def extract_from_object(obj, key, singleton: true)
       assert_respond_to(obj, :fetch)
 
       default = singleton ? nil : []
       result  = obj.fetch(key.to_s, default)
 
-      singleton ? Wgit::Utils.process_str(result) : Wgit::Utils.process_arr(result)
-
-      if block_given?
-        new_result = yield(result, obj, :object)
-        result = new_result unless new_result.nil?
-      end
-
+      Wgit::Utils.sanitize(result)
+      result = yield(result, obj, :object) if block_given?
       result
     end
 
@@ -523,12 +607,12 @@ module Wgit
       url = Wgit::Url.parse(url)
       url.crawled = true unless url.crawled? # Avoid overriding date_crawled.
 
-      @url   = url
-      @html  = html || ''
-      @doc   = init_nokogiri
-      @score = 0.0
+      @url    = url
+      @html   = html || ''
+      @parser = init_nokogiri
+      @score  = 0.0
 
-      Wgit::Utils.process_str(@html, encode: encode)
+      Wgit::Utils.sanitize(@html, encode: encode)
 
       # Dynamically run the init_*_from_html methods.
       Document.private_instance_methods(false).each do |method|
@@ -544,12 +628,12 @@ module Wgit
     def init_from_object(obj, encode: true)
       assert_respond_to(obj, :fetch)
 
-      @url   = Wgit::Url.new(obj.fetch('url')) # Should always be present.
-      @html  = obj.fetch('html', '')
-      @doc   = init_nokogiri
-      @score = obj.fetch('score', 0.0)
+      @url    = Wgit::Url.new(obj.fetch('url')) # Should always be present.
+      @html   = obj.fetch('html', '')
+      @parser = init_nokogiri
+      @score  = obj.fetch('score', 0.0)
 
-      Wgit::Utils.process_str(@html, encode: encode)
+      Wgit::Utils.sanitize(@html, encode: encode)
 
       # Dynamically run the init_*_from_object methods.
       Document.private_instance_methods(false).each do |method|
@@ -560,11 +644,11 @@ module Wgit
       end
     end
 
-    # Initialises an instance variable and defines a getter method for it.
+    # Initialises an instance variable and defines an accessor method for it.
     #
     # @param var [Symbol] The name of the variable to be initialized.
     # @param value [Object] The newly initialized variable's value.
-    # @return [Symbol] The name of the newly created getter method.
+    # @return [Symbol] The name of the defined getter method.
     def init_var(var, value)
       # instance_var_name starts with @, var_name doesn't.
       var = var.to_s
@@ -572,10 +656,9 @@ module Wgit
       instance_var_name = "@#{var_name}".to_sym
 
       instance_variable_set(instance_var_name, value)
+      Wgit::Document.attr_accessor(var_name)
 
-      Document.send(:define_method, var_name) do
-        instance_variable_get(instance_var_name)
-      end
+      var_name
     end
 
     alias content                html