wgit 0.10.0 → 0.10.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b6719bb2015379133ef2c9b417cada1826deab254f6fa1adaa093314f8fece99
-  data.tar.gz: 5ced648c0dff501bf0191aebfc0188d535f4ee657a072e1dbccd68ebbc6ac881
+  metadata.gz: 13a092ae05f0338598d0cc26d85ed828a86d60240eb996bacdbd7511aaae25a2
+  data.tar.gz: 3835b8aa3c06d49c1230dd42316c30315e39b0b33f0c433ff8df60eb6be2ecaa
 SHA512:
-  metadata.gz: 4a7782b4ccf6fa69fad9bb63d7d421fa548603ad5a35304db554bdcdf6deafe305395aba1ac9f35bcd095bc6cf4049ce70e56645faf1457e2e1313d48d1eb7f8
-  data.tar.gz: 8b8bb1454a131201e262eda060c6ae8490266a7675910026a0dd6ae0b2b55f2accf140d473edf135078f68cbe1048c4bb86f2dc5a6d4cf08a006f8fc20ac49b5
+  metadata.gz: 41a88d424925e3ba670104e69d027a5656c66616ab59bc85c62ec3359afd78230073e5a8f50697cbfc553b8dc9a012cf2ad2f2d857f7840fc668e0e43cdbfb33
+  data.tar.gz: 17bd6ddaba4cb53bd5cff4f9b36d5930f4bf496ee588163d190e10274438844f1a5e764c08fc7f82dc6c5c6b2efebaa52dc624be134e38a7d9d3ff9b970d1430

data/CHANGELOG.md

@@ -9,6 +9,42 @@
 - ...
 ---
 
+## v0.10.4
+### Added
+- `Database#search_text` method which returns a Hash of `url => text_results` instead of `Wgit::Documents` (like `#search`).
+### Changed/Removed
+- ...
+### Fixed
+- ...
+---
+
+## v0.10.3
+### Added
+- ...
+### Changed/Removed
+- Changed `Database#create_collections` and `#create_unique_indexes` by removing `rescue nil` from their database operations. Now any underlying errors with the database client are not masked.
+### Fixed
+- ...
+---
+
+## v0.10.2
+### Added
+- `Wgit::Base#setup` and `#teardown` methods (lifecycle hooks) that can be overridden by subclasses.
+### Changed/Removed
+- ...
+### Fixed
+- ...
+---
+
+## v0.10.1
+### Added
+- Support for Ruby 3.
+### Changed/Removed
+- Removed support for Ruby 2.5 (as it's too old).
+### Fixed
+- ...
+---
+
 ## v0.10.0
 ### Added
 - `Wgit::Url#scheme_relative?` method.

data/README.md

@@ -160,7 +160,7 @@ Only MRI Ruby is tested and supported, but Wgit may work with other Ruby impleme
 
 Currently, the required MRI Ruby version is:
 
-`~> 2.5` (a.k.a.) `>= 2.5 && < 3`
+`ruby '>= 2.6', '< 4'`
 
 ### Using Bundler
 

data/lib/wgit/base.rb

@@ -4,16 +4,25 @@ module Wgit
   class Base
     extend Wgit::DSL
 
+    # Runs once before the crawl/index is run. Override as needed.
+    def setup; end
+
+    # Runs once after the crawl/index is complete. Override as needed.
+    def teardown; end
+
     # Runs the crawl/index passing each crawled `Wgit::Document` and the given
     # block to the subclass's `#parse` method.
     def self.run(&block)
+      crawl_method = @method || :crawl
       obj = new
+
       unless obj.respond_to?(:parse)
         raise "#{obj.class} must respond_to? #parse(doc, &block)"
       end
 
-      crawl_method = @method || :crawl
+      obj.setup
       send(crawl_method) { |doc| obj.parse(doc, &block) }
+      obj.teardown
 
       obj
     end

data/lib/wgit/database/database.rb

@@ -91,29 +91,27 @@ module Wgit
 
     ### DDL ###
 
-    # Creates the urls and documents collections if they don't already exist.
-    # This method is therefore idempotent.
+    # Creates the 'urls' and 'documents' collections.
     #
     # @return [nil] Always returns nil.
     def create_collections
-      db.client[URLS_COLLECTION].create rescue nil
-      db.client[DOCUMENTS_COLLECTION].create rescue nil
+      @client[URLS_COLLECTION].create
+      @client[DOCUMENTS_COLLECTION].create
 
       nil
     end
 
-    # Creates the urls and documents unique 'url' indexes if they don't already
-    # exist. This method is therefore idempotent.
+    # Creates the urls and documents unique 'url' indexes.
     #
     # @return [nil] Always returns nil.
     def create_unique_indexes
       @client[URLS_COLLECTION].indexes.create_one(
         { url: 1 }, name: UNIQUE_INDEX, unique: true
-      ) rescue nil
+      )
 
       @client[DOCUMENTS_COLLECTION].indexes.create_one(
         { 'url.url' => 1 }, name: UNIQUE_INDEX, unique: true
-      ) rescue nil
+      )
 
       nil
     end
@@ -350,6 +348,58 @@ module Wgit
       results
     end
 
+    # Searches the database's Documents for the given query and then searches
+    # each result in turn using `doc.search`. Instead of an Array of Documents,
+    # this method returns a Hash of the docs url => search_results creating a
+    # search engine like result set for quick access to text matches.
+    #
+    # @param query [String] The text query to search with.
+    # @param case_sensitive [Boolean] Whether character case must match.
+    # @param whole_sentence [Boolean] Whether multiple words should be searched
+    #   for separately.
+    # @param limit [Integer] The max number of results to return.
+    # @param skip [Integer] The number of results to skip.
+    # @param sentence_limit [Integer] The max length of each search result
+    #   sentence.
+    # @param top_result_only [Boolean] Whether to return all of the documents
+    #   search results or just the top (most relavent) result.
+    # @yield [doc] Given each search result (Wgit::Document) returned from the
+    #   DB.
+    # @return [Hash<String, String | Array<String>>] The search results obtained
+    #   from the DB having mapped the docs url => search_results. The format of
+    #   search_results depends on the value of `top_result_only`.
+    def search_text(
+      query, case_sensitive: false, whole_sentence: true,
+      limit: 10, skip: 0, sentence_limit: 80, top_result_only: false
+    )
+      results = search(
+        query,
+        case_sensitive: case_sensitive,
+        whole_sentence: whole_sentence,
+        limit: limit,
+        skip: skip
+      )
+
+      results
+        .map do |doc|
+          yield(doc) if block_given?
+
+          results = doc.search(
+            query,
+            case_sensitive: case_sensitive,
+            whole_sentence: whole_sentence,
+            sentence_limit: sentence_limit
+          )
+
+          # Only return result if its text has a match - compact is called below.
+          next nil if results.empty?
+
+          [doc.url, (top_result_only ? results.first : results)]
+        end
+        .compact
+        .to_h
+    end
+
     # Returns statistics about the database.
     #
     # @return [BSON::Document#[]#fetch] Similar to a Hash instance.

data/lib/wgit/document.rb

@@ -453,7 +453,7 @@ be relative"
 
       if query.is_a?(Regexp)
         regex = query
-      else # respond_to? #to_s == true
+      else # query.respond_to? :to_s == true
         query = query.to_s
         query = query.gsub(' ', '|') unless whole_sentence
         regex = Regexp.new(query, !case_sensitive)

data/lib/wgit/indexer.rb

@@ -80,8 +80,8 @@ database capacity, exiting.")
           urls_count += write_urls_to_db(ext_links)
         end
 
-        Wgit.logger.info("Crawled and indexed docs for #{docs_count} url(s) \
-overall for this iteration.")
+        Wgit.logger.info("Crawled and indexed documents for #{docs_count} \
+url(s) overall for this iteration.")
         Wgit.logger.info("Found and saved #{urls_count} external url(s) for \
 the next iteration.")
 
@@ -136,8 +136,8 @@ the next iteration.")
         Wgit.logger.info("Found and saved #{num_inserted_urls} external url(s)")
       end
 
-      Wgit.logger.info("Crawled and indexed #{total_pages_indexed} docs for \
-the site: #{url}")
+      Wgit.logger.info("Crawled and indexed #{total_pages_indexed} documents \
+for the site: #{url}")
 
       total_pages_indexed
     end

data/lib/wgit/version.rb

@@ -6,7 +6,7 @@
 # @author Michael Telford
 module Wgit
   # The current gem version of Wgit.
-  VERSION = '0.10.0'
+  VERSION = '0.10.4'
 
   # Returns the current gem version of Wgit as a String.
   def self.version

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: wgit
 version: !ruby/object:Gem::Version
-  version: 0.10.0
+  version: 0.10.4
 platform: ruby
 authors:
 - Michael Telford
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-04-20 00:00:00.000000000 Z
+date: 2021-11-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -249,16 +249,19 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - "~>"
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.6'
+  - - "<"
     - !ruby/object:Gem::Version
-      version: '2.5'
+      version: '4'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.2.22
 signing_key:
 specification_version: 4
 summary: Wgit is a HTML web crawler, written in Ruby, that allows you to programmatically