wgit 0.10.5 → 0.10.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a359a5011cce717a84c0fa353cba658a4abcd11d2e8575da701b39eef35f641c
-  data.tar.gz: 730b2eee3c88d9cd99c1d9754744aa4bae71bc10019113e840f2914b3a7909e5
+  metadata.gz: 4598dcfc047ce3915ba5a871837be5efc54201d61b4967cf53070bec2af4dd52
+  data.tar.gz: 604010011024af6f2d4dfcc87e6c4c1d73f8e4811938281119fccb79792818c1
 SHA512:
-  metadata.gz: 4807d488cf03aa3dcf624249bd5169871bc17dd7e2de273cb841801147c040843912edc225a1f7346e427c022ecb2ce5360581323509221560ee99b31ea6a72b
-  data.tar.gz: ea3f1237116d05bbb24b2e85fdba1b6447821377a07ff5c5e6afea859afa5fb36d4e2c3f708c6f5dc201e0802aa61f0eaa56e63a62882330b5225dbbaa08721d
+  metadata.gz: 44b098e2a97191801787386e9d2060dcdeacc625c3453976679fc276a73b2bf0614713764a55f7074073018e898f2e43dc1a7f4f803339a86158052f59dcabcb
+  data.tar.gz: 8645c7095bb14590cf83c21905c9f5ed524e1047254e6526b8fe46a53f3989395472300d27fb65f899951a5f4b80ee9928accd23164b10e1a834975bf045db47

data/CHANGELOG.md

@@ -9,6 +9,15 @@
 - ...
 ---
 
+## v0.10.6
+### Added
+- `Wgit::DSL` method `#crawl_url` (aliased to `#crawl`).
+### Changed/Removed
+- Added a `&block` param to `Wgit::Document#extract`, which gets passed to `#extract_from_html`.
+### Fixed
+- ...
+---
+
 ## v0.10.5
 ### Added
 - `Database#last_result` getter method to return the most recent raw mongo result.

data/lib/wgit/document.rb

@@ -89,24 +89,18 @@ module Wgit
     #
     # @return [String] An xpath String to obtain a webpage's text elements.
     def self.text_elements_xpath
-      xpath = ''
-      return xpath if Wgit::Document.text_elements.empty?
-
-      el_xpath = '//%s/text()'
-      Wgit::Document.text_elements.each_with_index do |el, i|
-        xpath += ' | ' unless i.zero?
-        xpath += format(el_xpath, el)
+      Wgit::Document.text_elements.each_with_index.reduce("") do |xpath, (el, i)|
+        xpath += " | " unless i.zero?
+        xpath += format("//%s/text()", el)
       end
-
-      xpath
     end
 
     # Defines a content extractor, which extracts HTML elements/content
     # into instance variables upon Document initialization. See the default
     # extractors defined in 'document_extractors.rb' as examples. Defining an
     # extractor means that every subsequently crawled/initialized document
-    # will attempt to extract the xpath's content. Use `#xpath` for a one off
-    # content extraction.
+    # will attempt to extract the xpath's content. Use `#extract` for a one off
+    # content extraction on any document.
     #
     # Note that defined extractors work for both Documents initialized from
     # HTML (via Wgit::Crawler methods) and from database objects.
@@ -115,9 +109,9 @@ module Wgit
     #
     # When initialising from HTML, a singleton value of true will only
     # ever return the first result found; otherwise all the results are
-    # returned in an Array. When initialising from a database object, the value
-    # is taken as is and singleton is only used to define the default empty
-    # value. If a value cannot be found (in either the HTML or database
+    # returned in an Enumerable. When initialising from a database object, the
+    # value is taken as is and singleton is only used to define the default
+    # empty value. If a value cannot be found (in either the HTML or database
     # object), then a default will be used. The default value is:
     # `singleton ? nil : []`.
     #
@@ -134,12 +128,14 @@ module Wgit
     # @param opts [Hash] The options to define an extractor with. The
     #   options are only used when intializing from HTML, not the database.
     # @option opts [Boolean] :singleton The singleton option determines
-    #   whether or not the result(s) should be in an Array. If multiple
+    #   whether or not the result(s) should be in an Enumerable. If multiple
     #   results are found and singleton is true then the first result will be
     #   used. Defaults to true.
     # @option opts [Boolean] :text_content_only The text_content_only option
-    #   if true will use the text content of the Nokogiri result object,
-    #   otherwise the Nokogiri object itself is returned. Defaults to true.
+    #   if true will use the text #content of the Nokogiri result object,
+    #   otherwise the Nokogiri object itself is returned. The type of Nokogiri
+    #   object returned depends on the given xpath query. See the Nokogiri
+    #   documentation for more information. Defaults to true.
     # @yield The block is executed when a Wgit::Document is initialized,
     #   regardless of the source. Use it (optionally) to process the result
     #   value.
@@ -509,16 +505,24 @@ be relative"
     # parameter.
     #
     # @param xpath [String, #call] Used to find the value/object in @html.
-    # @param singleton [Boolean] singleton ? results.first (single Nokogiri
-    #   Object) : results (Array).
+    # @param singleton [Boolean] singleton ? results.first (single Object) : 
+    #   results (Enumerable).
     # @param text_content_only [Boolean] text_content_only ? result.content
     #   (String) : result (Nokogiri Object).
+    # @yield (Optionally) Pass a block to read/write the result value before
+    #   it's returned.
+    # @yieldparam value [Object] The result value to be returned.
+    # @yieldparam source [Wgit::Document, Object] This Document instance.
+    # @yieldparam type [Symbol] The `source` type, which is `:document`.
+    # @yieldreturn [Object] The return value of the block gets returned. Return
+    #   the block's `value` param unchanged if you simply want to inspect it.
     # @return [String, Object] The value found in the html or the default value
     #   (singleton ? nil : []).
-    def extract(xpath, singleton: true, text_content_only: true)
+    def extract(xpath, singleton: true, text_content_only: true, &block)
       send(
         :extract_from_html, xpath,
-        singleton: singleton, text_content_only: text_content_only
+        singleton: singleton, text_content_only: text_content_only,
+        &block
       )
     end
 
@@ -542,27 +546,25 @@ be relative"
     # parameter.
     #
     # @param xpath [String, #call] Used to find the value/object in @html.
-    # @param singleton [Boolean] singleton ? results.first (single Nokogiri
-    #   Object) : results (Array).
+    # @param singleton [Boolean] singleton ? results.first (single Object) : 
+    #   results (Enumerable).
     # @param text_content_only [Boolean] text_content_only ? result.content
     #   (String) : result (Nokogiri Object).
-    # @yield The block is executed when a Wgit::Document is initialized,
-    #   regardless of the source. Use it (optionally) to process the result
-    #   value.
+    # @yield (Optionally) Pass a block to read/write the result value before
+    #   it's returned.
     # @yieldparam value [Object] The result value to be returned.
-    # @yieldparam source [Wgit::Document, Object] The source of the `value`.
-    # @yieldparam type [Symbol] The `source` type, either `:document` or (DB)
-    #   `:object`.
+    # @yieldparam source [Wgit::Document, Object] This Document instance.
+    # @yieldparam type [Symbol] The `source` type, which is `:document`.
     # @yieldreturn [Object] The return value of the block gets returned. Return
     #   the block's `value` param unchanged if you simply want to inspect it.
     # @return [String, Object] The value found in the html or the default value
     #   (singleton ? nil : []).
     def extract_from_html(xpath, singleton: true, text_content_only: true)
       xpath  = xpath.call if xpath.respond_to?(:call)
-      result = singleton ? @parser.at_xpath(xpath) : @parser.xpath(xpath)
+      result = singleton ? at_xpath(xpath) : xpath(xpath)
 
-      if text_content_only
-        result = singleton ? result&.content : result.map(&:content)
+      if result && text_content_only
+        result = singleton ? result.content : result.map(&:content)
       end
 
       Wgit::Utils.sanitize(result)

data/lib/wgit/dsl.rb

@@ -317,6 +317,7 @@ the 'start' function".freeze
       db.clear_db
     end
 
+    alias crawl_url  crawl
     alias crawl_r    crawl_site
     alias index_r    index_site
     alias start_urls start

data/lib/wgit/version.rb

@@ -6,7 +6,7 @@
 # @author Michael Telford
 module Wgit
   # The current gem version of Wgit.
-  VERSION = '0.10.5'
+  VERSION = '0.10.6'
 
   # Returns the current gem version of Wgit as a String.
   def self.version

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: wgit
 version: !ruby/object:Gem::Version
-  version: 0.10.5
+  version: 0.10.6
 platform: ruby
 authors:
 - Michael Telford
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-07-11 00:00:00.000000000 Z
+date: 2022-07-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable