wgit 0.10.8 → 0.12.0

data/lib/wgit/database/database_adapter.rb

@@ -0,0 +1,147 @@
+# frozen_string_literal: true
+
+require_relative "../assertable"
+require_relative "../url"
+require_relative "../document"
+require_relative "../model"
+
+module Wgit::Database
+  # The parent DatabaseAdapter class that should be inherited from when
+  # creating an underlying Database adapter implementation class e.g.
+  # Wgit::Database::MongoDB.
+  #
+  # Listed in this class are the methods that an implementer class must
+  # implement to work with Wgit. Failure to do so will result in a
+  # NotImplementedError being raised.
+  #
+  # While not required, implementing the method `#search_fields=(fields)` in an
+  # adapter class will allow `Wgit::Model.set_search_fields` to call
+  # it. This allows the search fields to be set in one method call, from within
+  # the Wgit::Model class. See this method's docs for more info.
+  #
+  # Also listed in this class are common helper methods available to all
+  # Database implementer subclasses.
+  class DatabaseAdapter
+    include Wgit::Assertable
+
+    # The NotImplementedError message that gets raised if an implementor class
+    # doesn't implement a method required by Wgit.
+    NOT_IMPL_ERR = "The DatabaseAdapter class you're using hasn't \
+  implemented this method"
+
+    ###################### START OF INTERFACE METHODS ######################
+
+    # Initializes a DatabaseAdapter instance.
+    #
+    # The implementor class should establish a DB connection here using the
+    # given connection_string, falling back to `ENV['WGIT_CONNECTION_STRING']`.
+    # Don't forget to call `super`.
+    #
+    # @param connection_string [String] The connection string needed to connect
+    #   to the database.
+    # @raise [StandardError] If a connection string isn't provided, either as a
+    #   parameter or via the environment.
+    def initialize(connection_string = nil); end
+
+    # Returns the current size of the database.
+    #
+    # @return [Integer] The current size of the DB.
+    def size
+      raise NotImplementedError, NOT_IMPL_ERR
+    end
+
+    # Searches the database's Documents for the given query. The
+    # `Wgit::Model.search_fields` should be searched for matches
+    # against the given query. Documents should be sorted starting with the
+    # most relevant. Each returned Document should have it's `score` field set
+    # for relevance.
+    #
+    # @param query [String] The text query to search with.
+    # @param case_sensitive [Boolean] Whether character case must match.
+    # @param whole_sentence [Boolean] Whether multiple words should be searched
+    #   for separately.
+    # @param limit [Integer] The max number of results to return.
+    # @param skip [Integer] The number of results to skip.
+    # @yield [doc] Given each search result (Wgit::Document) returned from the
+    #   DB.
+    # @return [Array<Wgit::Document>] The search results obtained from the DB.
+    def search(
+      query, case_sensitive: false, whole_sentence: true, limit: 10, skip: 0
+    )
+      raise NotImplementedError, NOT_IMPL_ERR
+    end
+
+    # Deletes everything in the urls and documents collections.
+    #
+    # @return [Integer] The number of deleted records.
+    def empty
+      raise NotImplementedError, NOT_IMPL_ERR
+    end
+
+    # Returns Url records that haven't yet been crawled.
+    #
+    # @param limit [Integer] The max number of Url's to return. 0 returns all.
+    # @param skip [Integer] Skip n amount of Url's.
+    # @yield [url] Given each Url object (Wgit::Url) returned from the DB.
+    # @return [Array<Wgit::Url>] The uncrawled Urls obtained from the DB.
+    def uncrawled_urls(limit: 0, skip: 0)
+      raise NotImplementedError, NOT_IMPL_ERR
+    end
+
+    # Inserts or updates the object in the database.
+    #
+    # @param obj [Wgit::Url, Wgit::Document] The obj/record to insert/update.
+    # @return [Boolean] True if inserted, false if updated.
+    def upsert(obj)
+      raise NotImplementedError, NOT_IMPL_ERR
+    end
+
+    # Bulk upserts the objects in the database collection.
+    # You cannot mix collection objs types, all must be Urls or Documents.
+    #
+    # @param objs [Array<Wgit::Url>, Array<Wgit::Document>] The objs to be
+    #   inserted/updated.
+    # @return [Integer] The total number of newly inserted objects.
+    def bulk_upsert(objs)
+      raise NotImplementedError, NOT_IMPL_ERR
+    end
+
+    ###################### END OF INTERFACE METHODS ######################
+
+    private
+
+    # Returns the correct Wgit::Database:Model for the given obj type.
+    #
+    # @param obj [Wgit::Url, Wgit::Document] The obj to obtain a model for.
+    # @return [Hash] The obj model.
+    def build_model(obj)
+      assert_type(obj, [Wgit::Url, Wgit::Document])
+
+      if obj.is_a?(Wgit::Url)
+        Wgit::Model.url(obj)
+      else
+        Wgit::Model.document(obj)
+      end
+    end
+
+    # Map each DB hash object into a Wgit::Document. Each Document is yielded
+    # if a block is given before returning the mapped Array of Documents.
+    def map_documents(doc_hashes)
+      doc_hashes.map do |doc|
+        doc = Wgit::Document.new(doc)
+        yield(doc) if block_given?
+        doc
+      end
+    end
+
+    # Map each DB hash object into a Wgit::Url. Each Url is yielded
+    # if a block is given before returning the mapped Array of Urls.
+    def map_urls(url_hashes)
+      url_hashes.map do |url|
+        url = Wgit::Url.new(url)
+        yield(url) if block_given?
+        url
+      end
+    end
+  end
+end

data/lib/wgit/document.rb

@@ -3,12 +3,12 @@ require_relative 'utils'
 require_relative 'assertable'
 require 'nokogiri'
 require 'json'
-require 'set'
 
 module Wgit
   # Class modeling/serialising a HTML web document, although other MIME types
   # will work e.g. images etc. Also doubles as a search result when
-  # loading Documents from the database via `Wgit::Database#search`.
+  # loading Documents from the database via
+  # `Wgit::Database::DatabaseAdapter#search`.
   #
   # The initialize method dynamically initializes instance variables from the
   # Document HTML / Database object e.g. text. This bit is dynamic so that the
@@ -18,25 +18,23 @@ module Wgit
     include Assertable
 
     # Regex for the allowed var names when defining an extractor.
-    REGEX_EXTRACTOR_NAME = /[a-z0-9_]+/.freeze
+    REGEX_EXTRACTOR_NAME = /[a-z0-9_]+/
 
-    # Set of text elements used to build Document#text.
-    @text_elements = Set.new(%i[
-      a abbr address article aside b bdi bdo blockquote button caption cite
-      code data dd del details dfn div dl dt em figcaption figure footer h1 h2
-      h3 h4 h5 h6 header hr i input ins kbd label legend li main mark meter ol
-      option output p pre q rb rt ruby s samp section small span strong sub
-      summary sup td textarea th time u ul var wbr
-    ])
+    # Instance vars to be ignored by Document#to_h and in turn
+    # Wgit::Model.document.
+    @to_h_ignore_vars = [
+      '@parser' # Always ignore the Nokogiri object.
+    ]
 
     # Set of Symbols representing the defined Document extractors.
     @extractors = Set.new
 
     class << self
-      # Set of HTML elements that make up the visible text on a page. These
-      # elements are used to initialize the Wgit::Document#text. See the
-      # README.md for how to add to this Set dynamically.
-      attr_reader :text_elements
+      # Array of instance vars to ignore when Document#to_h and (in turn)
+      # Wgit::Model.document methods are called. Append your own defined extractor
+      # vars to omit them from the model (database object) when indexing.
+      # Each var should be a String starting with an '@' char e.g. "@data" etc.
+      attr_reader :to_h_ignore_vars
 
       # Set of Symbols representing the defined Document extractors. Is
       # read-only. Use Wgit::Document.define_extractor for a new extractor.
@@ -52,7 +50,7 @@ module Wgit
     # The Nokogiri::HTML document object initialized from @html.
     attr_reader :parser
 
-    # The score is only used following a `Database#search` and records matches.
+    # The score is set/used following a `Database#search` and records matches.
     attr_reader :score
 
     # Initialize takes either two strings (representing the URL and HTML) or an
@@ -76,25 +74,14 @@ module Wgit
     #   false if the Document content is an image etc.
     def initialize(url_or_obj, html = '', encode: true)
       if url_or_obj.is_a?(String)
-        init_from_strings(url_or_obj, html, encode: encode)
+        init_from_strings(url_or_obj, html, encode:)
       else
-        init_from_object(url_or_obj, encode: encode)
+        init_from_object(url_or_obj, encode:)
       end
     end
 
     ### Document Class Methods ###
 
-    # Uses Document.text_elements to build an xpath String, used to obtain
-    # all of the combined visual text on a webpage.
-    #
-    # @return [String] An xpath String to obtain a webpage's text elements.
-    def self.text_elements_xpath
-      Wgit::Document.text_elements.each_with_index.reduce('') do |xpath, (el, i)|
-        xpath += ' | ' unless i.zero?
-        xpath += format('//%s/text()', el)
-      end
-    end
-
     # Defines a content extractor, which extracts HTML elements/content
     # into instance variables upon Document initialization. See the default
     # extractors defined in 'document_extractors.rb' as examples. Defining an
@@ -118,8 +105,9 @@ module Wgit
     # @param var [Symbol] The name of the variable to be initialised, that will
     #   contain the extracted content. A getter and setter method is defined
     #   for the initialised variable.
-    # @param xpath [String, #call] The xpath used to find the element(s)
-    #   of the webpage. Only used when initializing from HTML.
+    # @param xpath [String, #call, nil] The xpath used to find the element(s)
+    #   of the webpage. Only used when initializing from HTML. Passing nil will
+    #   skip the HTML extraction, which sometimes isn't required.
     #
     #   Pass a callable object (proc etc.) if you want the
     #   xpath value to be derived on Document initialisation (instead of when
@@ -210,7 +198,7 @@ module Wgit
     #
     # @return [String] A short textual representation of this Document.
     def inspect
-      "#<Wgit::Document url=\"#{@url}\" html=#{size} bytes>"
+      "#<Wgit::Document url=\"#{@url}\" html_size=#{size}>"
     end
 
     # Determines if both the url and html match. Use
@@ -241,10 +229,10 @@ module Wgit
     # Provide the `link:` parameter to get the correct base URL for that type
     # of link. For example, a link of `#top` would always return @url because
     # it applies to that page, not a different one. Query strings work in the
-    # same way. Use this parameter if manually concatting Url's e.g.
+    # same way. Use this parameter if manually joining Url's e.g.
     #
     #   relative_link = Wgit::Url.new('?q=hello')
-    #   absolute_link = doc.base_url(link: relative_link).concat(relative_link)
+    #   absolute_link = doc.base_url(link: relative_link).join(relative_link)
     #
     # This is similar to how Wgit::Document#internal_absolute_links works.
     #
@@ -264,7 +252,7 @@ module Wgit
 be relative"
       end
 
-      get_base = -> { @base.relative? ? @url.to_origin.concat(@base) : @base }
+      get_base = -> { @base.relative? ? @url.to_origin.join(@base) : @base }
 
       if link
         link = Wgit::Url.new(link)
@@ -288,11 +276,11 @@ be relative"
     #   returned Hash.
     # @return [Hash] Containing self's instance vars.
     def to_h(include_html: false, include_score: true)
-      ignore = include_html ? [] : ['@html']
+      ignore = Wgit::Document.to_h_ignore_vars.dup
+      ignore << '@html' unless include_html
       ignore << '@score' unless include_score
-      ignore << '@parser' # Always ignore the Nokogiri object.
 
-      Wgit::Utils.to_h(self, ignore: ignore)
+      Wgit::Utils.to_h(self, ignore:)
     end
 
     # Converts this Document's #to_h return value to a JSON String.
@@ -301,7 +289,7 @@ be relative"
     #   returned JSON String.
     # @return [String] This Document represented as a JSON String.
     def to_json(include_html: false)
-      h = to_h(include_html: include_html)
+      h = to_h(include_html:)
       JSON.generate(h)
     end
 
@@ -323,7 +311,7 @@ be relative"
         else
           next unless instance_variable_get(var).respond_to?(:length)
 
-          hash[var[1..-1].to_sym] = instance_variable_get(var).send(:length)
+          hash[var[1..].to_sym] = instance_variable_get(var).send(:length)
         end
       end
 
@@ -431,17 +419,18 @@ be relative"
                 end
               end
               .reject { |link| link.relative?(host: @url.to_origin) }
-              .map(&:omit_trailing_slash)
 
       Wgit::Utils.sanitize(links)
     end
 
-    # Searches the @text for the given query and returns the results.
+    # Searches the Document's instance vars for the given query and returns
+    # the results. The `Wgit::Model.search_fields` denote the vars to be
+    # searched, unless overridden using the search_fields: param.
     #
-    # The number of search hits for each sentenence are recorded internally
+    # The number of matches for each search field is recorded internally
     # and used to rank/sort the search results before being returned. Where
-    # the Wgit::Database#search method search all documents for the most hits,
-    # this method searches each document's @text for the most hits.
+    # the Wgit::Database::DatabaseAdapter#search method searches all documents
+    # for matches, this method searches each individual Document for matches.
     #
     # Each search result comprises of a sentence of a given length. The length
     # will be based on the sentence_limit parameter or the full length of the
@@ -449,51 +438,86 @@ be relative"
     # that the search query is visible somewhere in the sentence.
     #
     # @param query [Regexp, #to_s] The regex or text value to search the
-    #   document's @text for.
+    #   document's instance vars (Wgit::Model.search_fields) for.
     # @param case_sensitive [Boolean] Whether character case must match.
     # @param whole_sentence [Boolean] Whether multiple words should be searched
     #   for separately.
     # @param sentence_limit [Integer] The max length of each search result
     #   sentence.
-    # @return [Array<String>] A subset of @text, matching the query.
+    # @param search_fields [Hash<Symbol, Integer>] The Document instance vars
+    #   to search and the weight for a match (used to determine relevence).
+    #   This should only be set for custom one-off Document searches. For
+    #   permanent changing of search fields, see Wgit::Model.set_search_fields.
+    # @yield [results_hash] Given the results_hash containing each search
+    #   result (String) and its score (num_matches * weight).
+    # @return [Array<String>] A subset of this document's instance vars,
+    #   matching the query for the search_fields: param.
     def search(
-      query, case_sensitive: false, whole_sentence: true, sentence_limit: 80
+      query, case_sensitive: false, whole_sentence: true,
+      sentence_limit: 80, search_fields: Wgit::Model.search_fields
     )
       raise 'The sentence_limit value must be even' if sentence_limit.odd?
+      assert_type(search_fields, Hash)
 
-      if query.is_a?(Regexp)
-        regex = query
-      else # query.respond_to? :to_s == true
-        query = query.to_s
-        query = query.gsub(' ', '|') unless whole_sentence
-        regex = Regexp.new(query, !case_sensitive)
-      end
-
+      regex = Wgit::Utils.build_search_regex(
+        query, case_sensitive:, whole_sentence:)
       results = {}
 
-      @text.each do |sentence|
-        sentence = sentence.strip
-        next if results[sentence]
+      search_fields.each do |field, weight|
+        doc_field = instance_variable_get("@#{field}".to_sym)
+        next unless doc_field
+
+        Wgit::Utils.each(doc_field) do |text|
+          assert_type(text, String)
 
-        hits = sentence.scan(regex).count
-        next unless hits.positive?
+          text = text.strip
+          matches = text.scan(regex).count
+          next unless matches.positive?
 
-        index = sentence.index(regex) # Index of first match.
-        Wgit::Utils.format_sentence_length(sentence, index, sentence_limit)
+          index = text.index(regex) # Index of first match.
+          Wgit::Utils.format_sentence_length(text, index, sentence_limit)
 
-        results[sentence] = hits
+          # For duplicate matching text, total the text score.
+          text_score = matches * weight
+          existing_score = results[text]
+          text_score += existing_score if existing_score
+
+          results[text] = text_score
+        end
       end
 
       return [] if results.empty?
 
-      results = Hash[results.sort_by { |_k, v| v }]
-      results.keys.reverse
+      yield results if block_given?
+
+      # Return only the matching text sentences, sorted by relevance.
+      Hash[results.sort_by { |_, score| -score }].keys
+    end
+
+    # Performs a text only search of the Document, instead of searching all
+    # search fields defined in Wgit::Model.search_fields.
+    #
+    # @param query [Regexp, #to_s] The regex or text value to search the
+    #   document's text for.
+    # @param case_sensitive [Boolean] Whether character case must match.
+    # @param whole_sentence [Boolean] Whether multiple words should be searched
+    #   for separately.
+    # @param sentence_limit [Integer] The max length of each search result
+    #   sentence.
+    # @return [Array<String>] A subset of this document's text fields that
+    #   match the query.
+    def search_text(
+      query, case_sensitive: false, whole_sentence: true, sentence_limit: 80
+    )
+      search(
+        query, case_sensitive:, whole_sentence:,
+        sentence_limit:, search_fields: { text: 1 })
     end
 
-    # Performs a text search (see Document#search for details) but assigns the
-    # results to the @text instance variable. This can be used for sub search
-    # functionality. The original text is returned; no other reference to it
-    # is kept thereafter.
+    # Performs a text only search (see Document#search_text for details) but
+    # assigns the results to the @text instance variable. This can be used
+    # for sub search functionality. The original text is returned; no other
+    # reference to it is kept thereafter.
     #
     # @param query [Regexp, #to_s] The regex or text value to search the
     #   document's @text for.
@@ -503,14 +527,11 @@ be relative"
     # @param sentence_limit [Integer] The max length of each search result
     #   sentence.
     # @return [String] This Document's original @text value.
-    def search!(
+    def search_text!(
       query, case_sensitive: false, whole_sentence: true, sentence_limit: 80
     )
       orig_text = @text
-      @text = search(
-        query, case_sensitive: case_sensitive,
-               whole_sentence: whole_sentence, sentence_limit: sentence_limit
-      )
+      @text = search_text(query, case_sensitive:, whole_sentence:, sentence_limit:)
 
       orig_text
     end
@@ -533,11 +554,74 @@ be relative"
     # @return [String, Object] The value found in the html or the default value
     #   (singleton ? nil : []).
     def extract(xpath, singleton: true, text_content_only: true, &block)
-      send(
-        :extract_from_html, xpath,
-        singleton: singleton, text_content_only: text_content_only,
-        &block
+      send(:extract_from_html, xpath, singleton:, text_content_only:, &block)
+    end
+
+    # Attempts to extract and check the HTML meta tags instructing Wgit not to
+    # index this document (save it to a Database).
+    #
+    # @return [Boolean] True if this document shouldn't be saved to a Database,
+    #   false otherwise.
+    def no_index?
+      meta_robots = extract_from_html(
+        '//meta[@name="robots"]/@content',
+        singleton: true,
+        text_content_only: true
+      )
+      meta_wgit = extract_from_html(
+        '//meta[@name="wgit"]/@content',
+        singleton: true,
+        text_content_only: true
       )
+
+      [meta_robots, meta_wgit].include?('noindex')
+    end
+
+    # Firstly finds the target element whose text contains el_text.
+    # Then finds the preceeding fragment element nearest to the target
+    # element and returns it's href value (starting with #). The search is
+    # performed against the @html so Documents loaded from a DB will need to
+    # contain the 'html' field in the Wgit::Model. See the
+    # `Wgit::Model#include_doc_html` documentation for more info.
+    #
+    # @param el_text [String] The element text of the target element.
+    # @param el_type [String] The element type, defaulting to any type.
+    # @yield [results] Given the results of the xpath query. Return the target
+    #   you want or nil to use the default (first) target in results.
+    # @return [String, nil] nil if no nearest fragment or the nearest
+    #   fragment's href e.g. '#about'.
+    # @raise [StandardError] Raises if no matching target element containg
+    #   el_text can be found or if @html is empty.
+    def nearest_fragment(el_text, el_type = "*")
+      raise "The @html is empty" if @html.empty?
+
+      xpath_query = "//#{el_type}[text()[contains(.,\"#{el_text}\")]]"
+      results = xpath(xpath_query)
+      return nil if results.empty?
+
+      target = results.first
+      if block_given?
+        result = yield(results)
+        target = result if result
+      end
+
+      target_index = html_index(target)
+      raise 'Failed to find target index' unless target_index
+
+      fragment_h = fragment_indices(fragments)
+
+      # Return the target href if the target is itself a fragment.
+      return fragment_h[target_index] if fragment_h.keys.include?(target_index)
+
+      # Find the target's nearest preceeding fragment href.
+      closest_index = 0
+      fragment_h.each do |fragment_index, href|
+        if fragment_index.between?(closest_index, target_index)
+          closest_index = fragment_index
+        end
+      end
+
+      fragment_h[closest_index]
     end
 
     protected
@@ -559,7 +643,8 @@ be relative"
     # Extracts a value/object from this Document's @html using the given xpath
     # parameter.
     #
-    # @param xpath [String, #call] Used to find the value/object in @html.
+    # @param xpath [String, #call, nil] Used to find the value/object in @html.
+    #   Passing nil will skip the HTML extraction which isn't always needed.
     # @param singleton [Boolean] singleton ? results.first (single Object) :
     #   results (Enumerable).
     # @param text_content_only [Boolean] text_content_only ? result.content
@@ -574,14 +659,18 @@ be relative"
     # @return [String, Object] The value found in the html or the default value
     #   (singleton ? nil : []).
     def extract_from_html(xpath, singleton: true, text_content_only: true)
-      xpath  = xpath.call if xpath.respond_to?(:call)
-      result = singleton ? at_xpath(xpath) : xpath(xpath)
+      result = nil
+
+      if xpath
+        xpath  = xpath.call if xpath.respond_to?(:call)
+        result = singleton ? at_xpath(xpath) : xpath(xpath)
+      end
 
       if result && text_content_only
         result = singleton ? result.content : result.map(&:content)
       end
 
-      Wgit::Utils.sanitize(result)
+      result = Wgit::Utils.sanitize(result)
       result = yield(result, self, :document) if block_given?
       result
     end
@@ -608,7 +697,7 @@ be relative"
       default = singleton ? nil : []
       result  = obj.fetch(key.to_s, default)
 
-      Wgit::Utils.sanitize(result)
+      result = Wgit::Utils.sanitize(result)
       result = yield(result, obj, :object) if block_given?
       result
     end
@@ -628,13 +717,14 @@ be relative"
       @parser = init_nokogiri
       @score  = 0.0
 
-      Wgit::Utils.sanitize(@html, encode: encode)
+      @html = Wgit::Utils.sanitize(@html, encode:)
 
       # Dynamically run the init_*_from_html methods.
       Document.private_instance_methods(false).each do |method|
         if method.to_s.start_with?('init_') &&
-           method.to_s.end_with?('_from_html')
-          send(method) unless method == __method__
+           method.to_s.end_with?('_from_html') &&
+           method != __method__
+          send(method)
         end
       end
     end
@@ -644,18 +734,20 @@ be relative"
     def init_from_object(obj, encode: true)
       assert_respond_to(obj, :fetch)
 
-      @url    = Wgit::Url.new(obj.fetch('url')) # Should always be present.
+      url = obj.fetch('url') # Should always be present.
+      raise "Missing 'url' field in doc object" unless url
+
+      @url    = Wgit::Url.new(url)
       @html   = obj.fetch('html', '')
       @parser = init_nokogiri
       @score  = obj.fetch('score', 0.0)
-
-      Wgit::Utils.sanitize(@html, encode: encode)
+      @html   = Wgit::Utils.sanitize(@html, encode:)
 
       # Dynamically run the init_*_from_object methods.
       Document.private_instance_methods(false).each do |method|
         if method.to_s.start_with?('init_') &&
-           method.to_s.end_with?('_from_object')
-          send(method, obj) unless method == __method__
+           method.to_s.end_with?('_from_object') && method != __method__
+          send(method, obj)
         end
       end
     end
@@ -668,7 +760,7 @@ be relative"
     def init_var(var, value)
       # instance_var_name starts with @, var_name doesn't.
       var = var.to_s
-      var_name = (var.start_with?('@') ? var[1..-1] : var).to_sym
+      var_name = (var.start_with?('@') ? var[1..] : var).to_sym
       instance_var_name = "@#{var_name}".to_sym
 
       instance_variable_set(instance_var_name, value)
@@ -677,10 +769,42 @@ be relative"
       var_name
     end
 
-    alias content                html
-    alias statistics             stats
-    alias internal_urls          internal_links
-    alias internal_absolute_urls internal_absolute_links
-    alias external_urls          external_links
+    # Returns all <a> fragment elements from within the HTML body e.g. #about.
+    def fragments
+      anchors = xpath("/html/body//a")
+
+      anchors.select do |anchor|
+        href = anchor.attributes['href']&.value
+        href&.start_with?('#')
+      end
+    end
+
+    # Returns a Hash{Int=>String} of <a> fragment positions and their href
+    # values. Only fragment anchors are returned e.g. <a> elements with a
+    # href starting with '#'.
+    def fragment_indices(fragments)
+      fragments.reduce({}) do |hash, fragment|
+        index = html_index(fragment)
+        next hash unless index
+
+        href = fragment.attributes['href']&.value
+        hash[index] = href
+
+        hash
+      end
+    end
+
+    # Takes a Nokogiri element or HTML substring and returns it's index in
+    # the html. Returns the index/position Int or nil if not found. The search
+    # is case insensitive because Nokogiri lower cases camelCase attributes.
+    def html_index(el_or_str)
+      @html.downcase.index(el_or_str.to_s.strip.downcase)
+    end
+
+    alias_method :content,                :html
+    alias_method :statistics,             :stats
+    alias_method :internal_urls,          :internal_links
+    alias_method :internal_absolute_urls, :internal_absolute_links
+    alias_method :external_urls,          :external_links
   end
 end