wgit 0.10.8 → 0.12.0

data/lib/wgit/document_extractors.rb

@@ -5,7 +5,7 @@
 # Base.
 Wgit::Document.define_extractor(
   :base,
-  '//base/@href',
+  "//base/@href",
   singleton: true,
   text_content_only: true
 ) do |base|
@@ -15,7 +15,7 @@ end
 # Title.
 Wgit::Document.define_extractor(
   :title,
-  '//title',
+  "//title",
   singleton: true,
   text_content_only: true
 )
@@ -43,17 +43,18 @@ Wgit::Document.define_extractor(
   singleton: true,
   text_content_only: true
 ) do |keywords, _source, type|
-  if keywords && (type == :document)
-    keywords = keywords.split(',')
-    Wgit::Utils.sanitize(keywords)
+  if keywords && type == :document
+    keywords = keywords.split(",")
+    keywords = Wgit::Utils.sanitize(keywords)
   end
+
   keywords
 end
 
 # Links.
 Wgit::Document.define_extractor(
   :links,
-  '//a/@href',
+  "//a/@href",
   singleton: false,
   text_content_only: true
 ) do |links|
@@ -65,7 +66,12 @@ end
 # Text.
 Wgit::Document.define_extractor(
   :text,
-  proc { Wgit::Document.text_elements_xpath },
-  singleton: false,
-  text_content_only: true
-)
+  nil # doc.parser contains all HTML so omit the xpath search.
+) do |text, doc, type|
+  if type == :document
+    html_to_text = Wgit::HTMLToText.new(doc.parser)
+    text = html_to_text.extract
+  end
+
+  text
+end

data/lib/wgit/dsl.rb

@@ -44,14 +44,14 @@ the 'start' function".freeze
       Wgit::Document.define_extractor(var, xpath, opts, &block)
     end
 
-    # Initializes a `Wgit::Crawler`. This crawler is then used in all crawl and
-    # index methods used by the DSL. See the Wgit::Crawler documentation for
-    # more details.
+    # Sets and returns the Wgit::Crawler used in subsequent crawls including
+    # indexing. Defaults to `Wgit::Crawler.new` if not given a param. See the
+    # Wgit::Crawler documentation for more details.
     #
-    # @yield [crawler] The created crawler; use the block to configure.
-    # @return [Wgit::Crawler] The created crawler used by the DSL.
-    def crawler
-      @dsl_crawler ||= Wgit::Crawler.new
+    # @yield [crawler] Given the DSL crawler; use the block to configure.
+    # @return [Wgit::Crawler] The crawler instance used by the DSL.
+    def use_crawler(crawler = nil)
+      @dsl_crawler = crawler || @dsl_crawler || Wgit::Crawler.new
       yield @dsl_crawler if block_given?
       @dsl_crawler
     end
@@ -66,7 +66,7 @@ the 'start' function".freeze
     # @yield [crawler] The crawler that'll be used in the subsequent
     #   crawl/index; use the block to configure.
     def start(*urls, &block)
-      crawler(&block)
+      use_crawler(&block) if block_given?
       @dsl_start = urls
     end
 
@@ -101,7 +101,7 @@ the 'start' function".freeze
       raise DSL_ERROR__NO_START_URL if urls.empty?
 
       urls.map! { |url| Wgit::Url.parse(url) }
-      crawler.crawl_urls(*urls, follow_redirects: follow_redirects, &block)
+      get_crawler.crawl_urls(*urls, follow_redirects:, &block)
     end
 
     # Crawls an entire site using `Wgit::Crawler#crawl_site` underneath. If no
@@ -135,47 +135,44 @@ the 'start' function".freeze
       raise DSL_ERROR__NO_START_URL if urls.empty?
 
       xpath = follow || :default
-      opts  = {
-        follow: xpath, allow_paths: allow_paths, disallow_paths: disallow_paths
-      }
+      opts  = { follow: xpath, allow_paths:, disallow_paths: }
 
       urls.reduce([]) do |externals, url|
-        externals + crawler.crawl_site(Wgit::Url.parse(url), **opts, &block)
+        externals + get_crawler.crawl_site(Wgit::Url.parse(url), **opts, &block)
       end
     end
 
-    # Returns the DSL's `crawler#last_response`.
+    # Returns the DSL's `Wgit::Crawler#last_response`.
     #
     # @return [Wgit::Response] The response from the last URL crawled.
     def last_response
-      crawler.last_response
+      get_crawler.last_response
     end
 
     # Nilifies the DSL instance variables.
     def reset
-      @dsl_crawler  = nil
-      @dsl_start    = nil
-      @dsl_follow   = nil
-      @dsl_conn_str = nil
+      @dsl_crawler = nil
+      @dsl_start   = nil
+      @dsl_follow  = nil
+      @dsl_db      = nil
     end
 
     ### INDEXER METHODS ###
 
-    # Defines the connection string to the database used in subsequent `index*`
-    # method calls. This method is optional as the connection string can be
-    # passed to the index method instead.
+    # Defines the connected database instance used in subsequent index and DB
+    # method calls. This method is optional however, as a new instance of the
+    # Wgit::Database.adapter_class will be initialised otherwise. Therefore
+    # if not calling this method, you should ensure
+    # ENV['WGIT_CONNECTION_STRING'] is set or the connection will fail.
     #
-    # @param conn_str [String] The connection string used to connect to the
-    #   database in subsequent `index*` method calls.
-    def connection_string(conn_str)
-      @dsl_conn_str = conn_str
+    # @param db [Wgit::Database::DatabaseAdapter] The connected database
+    #   instance used in subsequent `index*` method calls.
+    def use_database(db)
+      @dsl_db = db
     end
 
     # Indexes the World Wide Web using `Wgit::Indexer#index_www` underneath.
     #
-    # @param connection_string [String] The database connection string. Set as
-    #   nil to use ENV['WGIT_CONNECTION_STRING'] or set using
-    #   `connection_string`.
     # @param max_sites [Integer] The number of separate and whole
     #   websites to be crawled before the method exits. Defaults to -1 which
     #   means the crawl will occur until manually stopped (Ctrl+C etc).
@@ -183,22 +180,16 @@ the 'start' function".freeze
     #   scraped from the web (default is 1GB). Note, that this value is used to
     #   determine when to stop crawling; it's not a guarantee of the max data
     #   that will be obtained.
-    def index_www(
-      connection_string: @dsl_conn_str, max_sites: -1, max_data: 1_048_576_000
-    )
-      db      = Wgit::Database.new(connection_string)
-      indexer = Wgit::Indexer.new(db, crawler)
+    def index_www(max_sites: -1, max_data: 1_048_576_000)
+      indexer = Wgit::Indexer.new(get_db, get_crawler)
 
-      indexer.index_www(max_sites: max_sites, max_data: max_data)
+      indexer.index_www(max_sites:, max_data:)
     end
 
     # Indexes a single website using `Wgit::Indexer#index_site` underneath.
     #
     # @param urls [*String, *Wgit::Url] The base URL(s) of the website(s) to
     #   crawl. Can be set using `start`.
-    # @param connection_string [String] The database connection string. Set as
-    #   nil to use ENV['WGIT_CONNECTION_STRING'] or set using
-    #   `connection_string`.
     # @param insert_externals [Boolean] Whether or not to insert the website's
     #   external URL's into the database.
     # @param follow [String] The xpath extracting links to be followed during
@@ -215,19 +206,16 @@ the 'start' function".freeze
     #   set.
     # @return [Integer] The total number of pages crawled within the website.
     def index_site(
-      *urls, connection_string: @dsl_conn_str,
-      insert_externals: false, follow: @dsl_follow,
+      *urls, insert_externals: false, follow: @dsl_follow,
       allow_paths: nil, disallow_paths: nil, &block
     )
       urls = (@dsl_start || []) if urls.empty?
       raise DSL_ERROR__NO_START_URL if urls.empty?
 
-      db         = Wgit::Database.new(connection_string)
-      indexer    = Wgit::Indexer.new(db, crawler)
+      indexer    = Wgit::Indexer.new(get_db, get_crawler)
       xpath      = follow || :default
       crawl_opts = {
-        insert_externals: insert_externals, follow: xpath,
-        allow_paths: allow_paths, disallow_paths: disallow_paths
+        insert_externals:, follow: xpath, allow_paths:, disallow_paths:
       }
 
       urls.reduce(0) do |total, url|
@@ -239,9 +227,6 @@ the 'start' function".freeze
     #
     # @param urls [*Wgit::Url] The webpage URL's to crawl. Defaults to the
     #   `start` URL(s).
-    # @param connection_string [String] The database connection string. Set as
-    #   nil to use ENV['WGIT_CONNECTION_STRING'] or set using
-    #   `connection_string`.
     # @param insert_externals [Boolean] Whether or not to insert the website's
     #   external URL's into the database.
     # @yield [doc] Given the Wgit::Document of the crawled webpage,
@@ -250,28 +235,24 @@ the 'start' function".freeze
     #   document from being saved into the database.
     # @raise [StandardError] If no urls are provided and no `start` URL has
     #   been set.
-    def index(
-      *urls, connection_string: @dsl_conn_str,
-      insert_externals: false, &block
-    )
+    def index(*urls, insert_externals: false, &block)
       urls = (@dsl_start || []) if urls.empty?
       raise DSL_ERROR__NO_START_URL if urls.empty?
 
-      db      = Wgit::Database.new(connection_string)
-      indexer = Wgit::Indexer.new(db, crawler)
+      indexer = Wgit::Indexer.new(get_db, get_crawler)
 
       urls.map! { |url| Wgit::Url.parse(url) }
-      indexer.index_urls(*urls, insert_externals: insert_externals, &block)
+      indexer.index_urls(*urls, insert_externals:, &block)
     end
 
+    ### DATABASE METHODS ###
+
     # Performs a search of the database's indexed documents and pretty prints
-    # the results in a search engine-esque format. See `Wgit::Database#search!`
-    # and `Wgit::Document#search!` for details of how the search works.
+    # the results in a search engine-esque format. See
+    # `Wgit::Database::DatabaseAdapter#search` and `Wgit::Document#search!`
+    # for details of how the search methods work.
     #
     # @param query [String] The text query to search with.
-    # @param connection_string [String] The database connection string. Set as
-    #   nil to use ENV['WGIT_CONNECTION_STRING'] or set using
-    #   `connection_string`.
     # @param stream [nil, #puts] Any object that respond_to?(:puts). It is used
     #   to output text somewhere e.g. a file or STDERR. Use nil for no output.
     # @param case_sensitive [Boolean] Whether character case must match.
@@ -285,41 +266,53 @@ the 'start' function".freeze
     #   database containing only its matching `#text`.
     # @return [Array<Wgit::Document>] The search results with matching text.
     def search(
-      query, connection_string: @dsl_conn_str, stream: STDOUT,
+      query, stream: $stdout,
+      top_result_only: true, include_score: false,
       case_sensitive: false, whole_sentence: true,
-      limit: 10, skip: 0, sentence_limit: 80, &block
+      limit: 10, skip: 0, sentence_limit: 80
     )
       stream ||= File.open(File::NULL, 'w')
-      db = Wgit::Database.new(connection_string)
 
-      results = db.search!(
-        query,
-        case_sensitive: case_sensitive,
-        whole_sentence: whole_sentence,
-        limit: limit,
-        skip: skip,
-        sentence_limit: sentence_limit,
-        &block
-      )
+      results = get_db.search(
+        query, case_sensitive:, whole_sentence:, limit:, skip:)
+
+      results.each do |doc|
+        doc.search_text!(
+          query, case_sensitive:, whole_sentence:, sentence_limit:)
+        yield(doc) if block_given?
+      end
 
-      Wgit::Utils.printf_search_results(results, stream: stream)
+      if top_result_only
+        Wgit::Utils.pprint_top_search_results(results, include_score:, stream:)
+      else
+        Wgit::Utils.pprint_all_search_results(results, include_score:, stream:)
+      end
 
       results
     end
 
     # Deletes everything in the urls and documents collections by calling
-    # `Wgit::Database#clear_db` underneath. This will nuke the entire database
-    # so yeah... be careful.
+    # `Wgit::Database::DatabaseAdapter#empty` underneath.
     #
     # @return [Integer] The number of deleted records.
-    def clear_db!(connection_string: @dsl_conn_str)
-      db = Wgit::Database.new(connection_string)
-      db.clear_db
+    def empty_db!
+      get_db.empty
+    end
+
+    private
+
+    def get_crawler
+      @dsl_crawler ||= Wgit::Crawler.new
+    end
+
+    def get_db
+      @dsl_db ||= Wgit::Database.new
     end
 
-    alias crawl_url  crawl
-    alias crawl_r    crawl_site
-    alias index_r    index_site
-    alias start_urls start
+    alias_method :crawl_url,  :crawl
+    alias_method :crawl_r,    :crawl_site
+    alias_method :index_r,    :index_site
+    alias_method :index_url,  :index
+    alias_method :start_urls, :start
   end
 end

data/lib/wgit/html_to_text.rb

@@ -0,0 +1,277 @@
+require_relative "utils"
+require_relative "assertable"
+require "nokogiri"
+
+module Wgit
+  # Class used to extract the visible page text from a HTML string.
+  # This is in turn used to set the output of a Wgit::Document#text method.
+  class HTMLToText
+    include Assertable
+
+    # Set of text elements used to extract the visible text.
+    # The element's display (:inline or :block) is used to delimit sentences e.g.
+    # <div>foo</div><div>bar</div> will be extracted as ['foo', 'bar'] whereas
+    # <span>foo</span><span>bar</span> will be extracted as ['foobar'].
+    @text_elements = {
+      a:          :inline,
+      abbr:       :inline,
+      address:    :block,
+      article:    :block,
+      aside:      :block,
+      b:          :inline,
+      bdi:        :inline,
+      bdo:        :inline,
+      blockquote: :block,
+      br:         :block,
+      button:     :block, # Normally inline but Wgit treats as block.
+      caption:    :block,
+      cite:       :inline,
+      code:       :inline,
+      data:       :inline,
+      dd:         :block,
+      del:        :inline,
+      details:    :block,
+      dfn:        :inline,
+      div:        :block,
+      dl:         :block,
+      dt:         :block,
+      em:         :inline,
+      figcaption: :block,
+      figure:     :block,
+      footer:     :block,
+      h1:         :block,
+      h2:         :block,
+      h3:         :block,
+      h4:         :block,
+      h5:         :block,
+      h6:         :block,
+      header:     :block,
+      hr:         :block,
+      i:          :inline,
+      input:      :inline,
+      ins:        :block,
+      kbd:        :inline,
+      label:      :inline,
+      legend:     :block,
+      li:         :block,
+      main:       :block,
+      mark:       :inline,
+      meter:      :block,
+      ol:         :block,
+      option:     :block,
+      output:     :block,
+      p:          :block,
+      pre:        :block,
+      q:          :inline,
+      rb:         :inline,
+      rt:         :inline,
+      ruby:       :inline,
+      s:          :inline,
+      samp:       :inline,
+      section:    :block,
+      small:      :inline,
+      span:       :inline,
+      strong:     :inline,
+      sub:        :inline,
+      summary:    :block,
+      sup:        :inline,
+      td:         :block,
+      textarea:   :block,
+      th:         :block,
+      time:       :inline,
+      u:          :inline,
+      ul:         :block,
+      var:        :inline,
+      wbr:        :inline
+    }
+
+    class << self
+      # Set of HTML elements that make up the visible text on a page. These
+      # elements are used to initialize the Wgit::Document#text. See the
+      # README.md for how to add to this Hash dynamically.
+      attr_reader :text_elements
+    end
+
+    # The Nokogiri::HTML document object initialized from a HTML string.
+    attr_reader :parser
+
+    # Creates a new HTML to text extractor instance.
+    #
+    # @param parser [Nokogiri::HTML4::Document] The nokogiri parser object.
+    # @raise [StandardError] If the given parser is of an invalid type.
+    def initialize(parser)
+      assert_type(parser, Nokogiri::HTML4::Document)
+
+      @parser = parser
+    end
+
+    # Extracts and returns the text sentences from the @parser HTML.
+    #
+    # @return [Array<String>] An array of unique text sentences.
+    def extract_arr
+      return [] if @parser.to_s.empty?
+
+      text_str = extract_str
+
+      # Split the text_str into an Array of text sentences.
+      text_str
+        .split("\n")
+        .map(&:strip)
+        .reject(&:empty?)
+    end
+
+    # Extracts and returns a text string from the @parser HTML.
+    #
+    # @return [String] A string of text with \n delimiting sentences.
+    def extract_str
+      text_str = ""
+
+      iterate_child_nodes(@parser) do |node, display|
+        # Handle any special cases e.g. skip nodes we don't care about...
+        # <pre> nodes should have their contents displayed exactly as is.
+        if node_name(node) == :pre
+          text_str << "\n"
+          text_str << node.text
+          next
+        end
+
+        # Skip any child node of <pre> since they're handled as a special case above.
+        next if child_of?(:pre, node)
+
+        if node.text?
+          # Skip any text element that is purely whitespace.
+          next unless valid_text_content?(node.text)
+        else
+          # Skip a concrete node if it has other concrete child nodes as these
+          # will be iterated onto later.
+          #
+          # Process if node has no children or one child which is a valid text node.
+          next unless node.children.empty? || parent_of_text_node_only?(node)
+        end
+
+        # Apply display rules deciding if a new line is needed before node.text.
+        add_new_line = false
+        prev = prev_sibling_or_parent(node)
+
+        if node.text?
+          add_new_line = true unless prev && inline?(prev)
+        else
+          add_new_line = true if display == :block
+          add_new_line = true if prev && block?(prev)
+        end
+
+        text_str << "\n" if add_new_line
+        text_str << format_text(node.text)
+      end
+
+      text_str
+        .strip
+        .squeeze("\n")
+        .squeeze(" ")
+    end
+
+    private
+
+    def node_name(node)
+      node.name&.downcase&.to_sym
+    end
+
+    def display(node)
+      name = node_name(node)
+      Wgit::HTMLToText.text_elements[name]
+    end
+
+    def inline?(node)
+      display(node) == :inline
+    end
+
+    def block?(node)
+      display(node) == :block
+    end
+
+    # Returns the previous sibling of node or nil. Only valid text elements are
+    # returned i.e. non duplicates with valid text content.
+    def prev_sibling(node)
+      prev = node.previous
+
+      return nil unless prev
+      return prev unless prev.text?
+      return prev if valid_text_node?(prev) && !contains_new_line?(prev.text)
+      return prev if valid_text_node?(prev) && !format_text(prev.text).strip.empty?
+
+      prev.previous
+    end
+
+    # Returns node's previous sibling, parent or nil; in that order. Only valid
+    # text elements are returned i.e. non duplicates with valid text content.
+    def prev_sibling_or_parent(node)
+      prev = prev_sibling(node)
+      return prev if prev
+
+      node.parent
+    end
+
+    def child_of?(ancestor_name, node)
+      node.ancestors.any? { |ancestor| node_name(ancestor) == ancestor_name }
+    end
+
+    # Returns true if any of the child nodes contain a non empty :text node.
+    def parent_of_text_node?(node)
+      node.children.any? { |child| child.text? && valid_text_content?(child.text) }
+    end
+
+    def parent_of_text_node_only?(node)
+      node.children.size == 1 && parent_of_text_node?(node)
+    end
+
+    # Returns true if text is not empty having removed all new lines.
+    def valid_text_content?(text)
+      !format_text(text).empty?
+    end
+
+    # Returns true if node is a text node.
+    # Duplicate text nodes (that follow a concrete node) are omitted.
+    def valid_text_node?(node)
+      node.text? && node.text != node.parent.text
+    end
+
+    def contains_new_line?(text)
+      ["\n", '\\n'].any? { |new_line| text.include?(new_line) }
+    end
+
+    # Remove special characters including any new lines; as semantic HTML will
+    # typically use <br> and/or block elements to denote a line break.
+    def format_text(text)
+      text
+        .encode("UTF-8", undef: :replace, invalid: :replace)
+        .gsub("\n",       "")
+        .gsub('\\n',      "")
+        .gsub("\r",       "")
+        .gsub('\\r',      "")
+        .gsub("\f",       "")
+        .gsub('\\f',      "")
+        .gsub("\t",       "")
+        .gsub('\\t',      "")
+        .gsub("&zwnj;",   "")
+        .gsub("&nbsp;",   " ")
+        .gsub("&#160;",   " ")
+        .gsub("&thinsp;", " ")
+        .gsub("&ensp;",   " ")
+        .gsub("&emsp;",   " ")
+        .gsub('\u00a0',   " ")
+    end
+
+    # Iterate over node and it's child nodes, yielding each to &block.
+    # Only HTMLToText.text_elements or valid :text nodes will be yielded.
+    # Duplicate text nodes (that follow a concrete node) are omitted.
+    def iterate_child_nodes(node, &block)
+      display = display(node)
+      text_node = valid_text_node?(node)
+
+      yield(node, display) if display || text_node
+      node.children.each { |child| iterate_child_nodes(child, &block) }
+    end
+
+    alias_method :extract, :extract_arr
+  end
+end