wgit 0.10.3 → 0.10.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 720cf6b84698fbd54c109319f05557ee2e29bdbda59ec23278422dc5ddc77f2f
-  data.tar.gz: d4304bce849b404b9d2d7faa4d9a3f7969784f649a83152605b51b2e0bd21ac4
+  metadata.gz: 4598dcfc047ce3915ba5a871837be5efc54201d61b4967cf53070bec2af4dd52
+  data.tar.gz: 604010011024af6f2d4dfcc87e6c4c1d73f8e4811938281119fccb79792818c1
 SHA512:
-  metadata.gz: a8743ec17b3caaa9b6c5dd5c9b9b18902561927dfd992003f25db88334cc2b4364a4c6ce2dea34629f801d5d7dbe9761b15e7f2f034e00ba526db36ce828dcaf
-  data.tar.gz: 00cf954a86c8b0d96f2e694359c1c75e3193e0e6d146ffba19b3857bef4c15ca93d25f1310ebebf815de8da93ede1b97e325dc54aade699219b9ab35f2976e49
+  metadata.gz: 44b098e2a97191801787386e9d2060dcdeacc625c3453976679fc276a73b2bf0614713764a55f7074073018e898f2e43dc1a7f4f803339a86158052f59dcabcb
+  data.tar.gz: 8645c7095bb14590cf83c21905c9f5ed524e1047254e6526b8fe46a53f3989395472300d27fb65f899951a5f4b80ee9928accd23164b10e1a834975bf045db47

data/CHANGELOG.md

@@ -9,6 +9,33 @@
 - ...
 ---
 
+## v0.10.6
+### Added
+- `Wgit::DSL` method `#crawl_url` (aliased to `#crawl`).
+### Changed/Removed
+- Added a `&block` param to `Wgit::Document#extract`, which gets passed to `#extract_from_html`.
+### Fixed
+- ...
+---
+
+## v0.10.5
+### Added
+- `Database#last_result` getter method to return the most recent raw mongo result.
+### Changed/Removed
+- ...
+### Fixed
+- ...
+---
+
+## v0.10.4
+### Added
+- `Database#search_text` method which returns a Hash of `url => text_results` instead of `Wgit::Documents` (like `#search`).
+### Changed/Removed
+- ...
+### Fixed
+- ...
+---
+
 ## v0.10.3
 ### Added
 - ...

data/lib/wgit/database/database.rb

@@ -45,6 +45,9 @@ module Wgit
     # A custom setter method is also provided for changing the search logic.
     attr_reader :text_index
 
+    # The raw MongoDB result of the most recent operation.
+    attr_reader :last_result
+
     # Initializes a connected database client using the provided
     # connection_string or ENV['WGIT_CONNECTION_STRING'].
     #
@@ -185,6 +188,8 @@ module Wgit
       result = @client[collection].replace_one(query, data_hash, upsert: true)
 
       result.matched_count.zero?
+    ensure
+      @last_result = result
     end
 
     ### Retrieve Data ###
@@ -294,16 +299,12 @@ module Wgit
       results = retrieve(DOCUMENTS_COLLECTION, query,
                          sort: sort_proj, projection: sort_proj,
                          limit: limit, skip: skip)
-      return [] if results.count < 1 # respond_to? :empty? == false
 
-      # results.respond_to? :map! is false so we use map and overwrite the var.
-      results = results.map do |mongo_doc|
+      results.map do |mongo_doc|
         doc = Wgit::Document.new(mongo_doc)
         yield(doc) if block_given?
         doc
       end
-
-      results
     end
 
     # Searches the database's Documents for the given query and then searches
@@ -348,6 +349,58 @@ module Wgit
       results
     end
 
+    # Searches the database's Documents for the given query and then searches
+    # each result in turn using `doc.search`. Instead of an Array of Documents,
+    # this method returns a Hash of the docs url => search_results creating a
+    # search engine like result set for quick access to text matches.
+    #
+    # @param query [String] The text query to search with.
+    # @param case_sensitive [Boolean] Whether character case must match.
+    # @param whole_sentence [Boolean] Whether multiple words should be searched
+    #   for separately.
+    # @param limit [Integer] The max number of results to return.
+    # @param skip [Integer] The number of results to skip.
+    # @param sentence_limit [Integer] The max length of each search result
+    #   sentence.
+    # @param top_result_only [Boolean] Whether to return all of the documents
+    #   search results or just the top (most relavent) result.
+    # @yield [doc] Given each search result (Wgit::Document) returned from the
+    #   DB.
+    # @return [Hash<String, String | Array<String>>] The search results obtained
+    #   from the DB having mapped the docs url => search_results. The format of
+    #   search_results depends on the value of `top_result_only`.
+    def search_text(
+      query, case_sensitive: false, whole_sentence: true,
+      limit: 10, skip: 0, sentence_limit: 80, top_result_only: false
+    )
+      results = search(
+        query,
+        case_sensitive: case_sensitive,
+        whole_sentence: whole_sentence,
+        limit: limit,
+        skip: skip
+      )
+
+      results
+        .map do |doc|
+          yield(doc) if block_given?
+
+          results = doc.search(
+            query,
+            case_sensitive: case_sensitive,
+            whole_sentence: whole_sentence,
+            sentence_limit: sentence_limit
+          )
+
+          # Only return result if its text has a match - compact is called below.
+          next nil if results.empty?
+
+          [doc.url, (top_result_only ? results.first : results)]
+        end
+        .compact
+        .to_h
+    end
+
     # Returns statistics about the database.
     #
     # @return [BSON::Document#[]#fetch] Similar to a Hash instance.
@@ -454,21 +507,30 @@ module Wgit
     #   0 or 1 because urls are unique.
     def delete(obj)
       collection, query = get_type_info(obj)
-      @client[collection].delete_one(query).n
+      result = @client[collection].delete_one(query)
+      result.n
+    ensure
+      @last_result = result
     end
 
     # Deletes everything in the urls collection.
     #
     # @return [Integer] The number of deleted records.
     def clear_urls
-      @client[URLS_COLLECTION].delete_many({}).n
+      result = @client[URLS_COLLECTION].delete_many({})
+      result.n
+    ensure
+      @last_result = result
     end
 
     # Deletes everything in the documents collection.
     #
     # @return [Integer] The number of deleted records.
     def clear_docs
-      @client[DOCUMENTS_COLLECTION].delete_many({}).n
+      result = @client[DOCUMENTS_COLLECTION].delete_many({})
+      result.n
+    ensure
+      @last_result = result
     end
 
     # Deletes everything in the urls and documents collections. This will nuke
@@ -536,6 +598,8 @@ module Wgit
       else
         raise 'data must be a Hash or an Array of Hashes'
       end
+    ensure
+      @last_result = result
     end
 
     # Return if the write to the DB succeeded or not.
@@ -572,8 +636,8 @@ module Wgit
                  sort: {}, projection: {},
                  limit: 0, skip: 0)
       assert_type(query, Hash)
-      @client[collection.to_sym].find(query).projection(projection)
-                                .skip(skip).limit(limit).sort(sort)
+      @last_result = @client[collection.to_sym].find(query).projection(projection)
+                                               .skip(skip).limit(limit).sort(sort)
     end
 
     # Mutate/update one or more Url or Document records in the DB.
@@ -593,6 +657,8 @@ module Wgit
       raise 'DB write(s) (update) failed' unless write_succeeded?(result)
 
       result.n
+    ensure
+      @last_result = result
     end
 
     alias num_objects num_records

data/lib/wgit/document.rb

@@ -89,24 +89,18 @@ module Wgit
     #
     # @return [String] An xpath String to obtain a webpage's text elements.
     def self.text_elements_xpath
-      xpath = ''
-      return xpath if Wgit::Document.text_elements.empty?
-
-      el_xpath = '//%s/text()'
-      Wgit::Document.text_elements.each_with_index do |el, i|
-        xpath += ' | ' unless i.zero?
-        xpath += format(el_xpath, el)
+      Wgit::Document.text_elements.each_with_index.reduce("") do |xpath, (el, i)|
+        xpath += " | " unless i.zero?
+        xpath += format("//%s/text()", el)
       end
-
-      xpath
     end
 
     # Defines a content extractor, which extracts HTML elements/content
     # into instance variables upon Document initialization. See the default
     # extractors defined in 'document_extractors.rb' as examples. Defining an
     # extractor means that every subsequently crawled/initialized document
-    # will attempt to extract the xpath's content. Use `#xpath` for a one off
-    # content extraction.
+    # will attempt to extract the xpath's content. Use `#extract` for a one off
+    # content extraction on any document.
     #
     # Note that defined extractors work for both Documents initialized from
     # HTML (via Wgit::Crawler methods) and from database objects.
@@ -115,9 +109,9 @@ module Wgit
     #
     # When initialising from HTML, a singleton value of true will only
     # ever return the first result found; otherwise all the results are
-    # returned in an Array. When initialising from a database object, the value
-    # is taken as is and singleton is only used to define the default empty
-    # value. If a value cannot be found (in either the HTML or database
+    # returned in an Enumerable. When initialising from a database object, the
+    # value is taken as is and singleton is only used to define the default
+    # empty value. If a value cannot be found (in either the HTML or database
     # object), then a default will be used. The default value is:
     # `singleton ? nil : []`.
     #
@@ -134,12 +128,14 @@ module Wgit
     # @param opts [Hash] The options to define an extractor with. The
     #   options are only used when intializing from HTML, not the database.
     # @option opts [Boolean] :singleton The singleton option determines
-    #   whether or not the result(s) should be in an Array. If multiple
+    #   whether or not the result(s) should be in an Enumerable. If multiple
     #   results are found and singleton is true then the first result will be
     #   used. Defaults to true.
     # @option opts [Boolean] :text_content_only The text_content_only option
-    #   if true will use the text content of the Nokogiri result object,
-    #   otherwise the Nokogiri object itself is returned. Defaults to true.
+    #   if true will use the text #content of the Nokogiri result object,
+    #   otherwise the Nokogiri object itself is returned. The type of Nokogiri
+    #   object returned depends on the given xpath query. See the Nokogiri
+    #   documentation for more information. Defaults to true.
     # @yield The block is executed when a Wgit::Document is initialized,
     #   regardless of the source. Use it (optionally) to process the result
     #   value.
@@ -453,7 +449,7 @@ be relative"
 
       if query.is_a?(Regexp)
         regex = query
-      else # respond_to? #to_s == true
+      else # query.respond_to? :to_s == true
         query = query.to_s
         query = query.gsub(' ', '|') unless whole_sentence
         regex = Regexp.new(query, !case_sensitive)
@@ -509,16 +505,24 @@ be relative"
     # parameter.
     #
     # @param xpath [String, #call] Used to find the value/object in @html.
-    # @param singleton [Boolean] singleton ? results.first (single Nokogiri
-    #   Object) : results (Array).
+    # @param singleton [Boolean] singleton ? results.first (single Object) : 
+    #   results (Enumerable).
     # @param text_content_only [Boolean] text_content_only ? result.content
     #   (String) : result (Nokogiri Object).
+    # @yield (Optionally) Pass a block to read/write the result value before
+    #   it's returned.
+    # @yieldparam value [Object] The result value to be returned.
+    # @yieldparam source [Wgit::Document, Object] This Document instance.
+    # @yieldparam type [Symbol] The `source` type, which is `:document`.
+    # @yieldreturn [Object] The return value of the block gets returned. Return
+    #   the block's `value` param unchanged if you simply want to inspect it.
     # @return [String, Object] The value found in the html or the default value
     #   (singleton ? nil : []).
-    def extract(xpath, singleton: true, text_content_only: true)
+    def extract(xpath, singleton: true, text_content_only: true, &block)
       send(
         :extract_from_html, xpath,
-        singleton: singleton, text_content_only: text_content_only
+        singleton: singleton, text_content_only: text_content_only,
+        &block
       )
     end
 
@@ -542,27 +546,25 @@ be relative"
     # parameter.
     #
     # @param xpath [String, #call] Used to find the value/object in @html.
-    # @param singleton [Boolean] singleton ? results.first (single Nokogiri
-    #   Object) : results (Array).
+    # @param singleton [Boolean] singleton ? results.first (single Object) : 
+    #   results (Enumerable).
     # @param text_content_only [Boolean] text_content_only ? result.content
     #   (String) : result (Nokogiri Object).
-    # @yield The block is executed when a Wgit::Document is initialized,
-    #   regardless of the source. Use it (optionally) to process the result
-    #   value.
+    # @yield (Optionally) Pass a block to read/write the result value before
+    #   it's returned.
     # @yieldparam value [Object] The result value to be returned.
-    # @yieldparam source [Wgit::Document, Object] The source of the `value`.
-    # @yieldparam type [Symbol] The `source` type, either `:document` or (DB)
-    #   `:object`.
+    # @yieldparam source [Wgit::Document, Object] This Document instance.
+    # @yieldparam type [Symbol] The `source` type, which is `:document`.
     # @yieldreturn [Object] The return value of the block gets returned. Return
     #   the block's `value` param unchanged if you simply want to inspect it.
     # @return [String, Object] The value found in the html or the default value
     #   (singleton ? nil : []).
     def extract_from_html(xpath, singleton: true, text_content_only: true)
       xpath  = xpath.call if xpath.respond_to?(:call)
-      result = singleton ? @parser.at_xpath(xpath) : @parser.xpath(xpath)
+      result = singleton ? at_xpath(xpath) : xpath(xpath)
 
-      if text_content_only
-        result = singleton ? result&.content : result.map(&:content)
+      if result && text_content_only
+        result = singleton ? result.content : result.map(&:content)
       end
 
       Wgit::Utils.sanitize(result)

data/lib/wgit/dsl.rb

@@ -317,6 +317,7 @@ the 'start' function".freeze
       db.clear_db
     end
 
+    alias crawl_url  crawl
     alias crawl_r    crawl_site
     alias index_r    index_site
     alias start_urls start

data/lib/wgit/version.rb

@@ -6,7 +6,7 @@
 # @author Michael Telford
 module Wgit
   # The current gem version of Wgit.
-  VERSION = '0.10.3'
+  VERSION = '0.10.6'
 
   # Returns the current gem version of Wgit as a String.
   def self.version

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: wgit
 version: !ruby/object:Gem::Version
-  version: 0.10.3
+  version: 0.10.6
 platform: ruby
 authors:
 - Michael Telford
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-11-25 00:00:00.000000000 Z
+date: 2022-07-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable