wgit 0.11.0 → 0.12.1

data/lib/wgit/utils.rb

@@ -18,17 +18,15 @@ module Wgit
     #   keys.
     # @return [Hash] A Hash created from obj's instance vars and values.
     def self.to_h(obj, ignore: [], use_strings_as_keys: true)
-      hash = {}
-
-      obj.instance_variables.each do |var|
-        next if ignore.include?(var.to_s)
+      obj.instance_variables.reduce({}) do |hash, var|
+        next hash if ignore.include?(var.to_s)
 
         key = var.to_s[1..] # Remove the @ prefix.
         key = key.to_sym unless use_strings_as_keys
         hash[key] = obj.instance_variable_get(var)
-      end
 
-      hash
+        hash
+      end
     end
 
     # An improved :each method which supports both singleton and Enumerable
@@ -127,13 +125,19 @@ module Wgit
     end
 
     # Prints out the search results in a search engine like format.
-    # The format for each result looks like:
+    #
+    # The given results should be matching documents from a DB and should have
+    # `doc.search_text!` called for each document - to turn doc.text into only
+    # matching text, which this method uses.
+    #
+    # The format for each result looks something like:
     #
     # ```
     # Title
     # Keywords (if there are some)
-    # Text Snippet (formatted to show the searched for query, if provided)
+    # Text Snippet (formatted to show the searched for query)
     # URL
+    # Score (if include_score: true)
     # <empty_line_seperator>
     # ```
     #
@@ -142,28 +146,91 @@ module Wgit
     #   the search results). The first @text sentence gets printed.
     # @param keyword_limit [Integer] The max amount of keywords to be
     #   outputted to the stream.
+    # @param include_score [Boolean] Wether or not to puts the document score.
     # @param stream [#puts] Any object that respond_to?(:puts). It is used
     #   to output text somewhere e.g. a file or STDERR.
     # @return [Integer] The number of results.
-    def self.pprint_search_results(results, keyword_limit: 5, stream: $stdout)
+    def self.pprint_top_search_results(
+      results, keyword_limit: 5, include_score: false, stream: $stdout
+    )
       raise 'stream must respond_to? :puts' unless stream.respond_to?(:puts)
 
       results.each do |doc|
-        title    = (doc.title || '<no title>')
+        title    = doc.title || '<no title>'
         keywords = doc.keywords&.take(keyword_limit)&.join(', ')
         sentence = doc.text.first
         url      = doc.url
+        score    = doc.score
 
         stream.puts title
         stream.puts keywords if keywords
         stream.puts sentence
         stream.puts url
+        stream.puts score if include_score
         stream.puts
       end
 
       results.size
     end
 
+    # Prints out the search results listing all of the matching text in each
+    # document.
+    #
+    # The given results should be matching documents from a DB and should have
+    # `doc.search_text!` called for each document - to turn doc.text into only
+    # matching text, which this method uses.
+    #
+    # The format for each result looks something like:
+    #
+    # ```
+    # Title
+    # Keywords (if there are some)
+    # URL
+    # Score (if include_score: true)
+    #
+    # "<text_snippet_1>"
+    # "<text_snippet_2>"
+    # ...
+    #
+    # <seperator>
+    # ```
+    #
+    # @param results [Array<Wgit::Document>] Array of Wgit::Document's which
+    #   each have had #search!(query) called (to update it's @text with the
+    #   the search results). The first @text sentence gets printed.
+    # @param keyword_limit [Integer] The max amount of keywords to be
+    #   outputted to the stream.
+    # @param include_score [Boolean] Wether or not to puts the document score.
+    # @param stream [#puts] Any object that respond_to?(:puts). It is used
+    #   to output text somewhere e.g. a file or STDERR.
+    # @return [Integer] The number of results.
+    def self.pprint_all_search_results(
+      results, keyword_limit: 5, include_score: false, stream: $stdout
+    )
+      raise 'stream must respond_to? :puts' unless stream.respond_to?(:puts)
+
+      results.each_with_index do |doc, i|
+        last_result = i == (results.size-1)
+
+        title    = doc.title || '<no title>'
+        keywords = doc.keywords&.take(keyword_limit)&.join(', ')
+        url      = doc.url
+        score    = doc.score
+
+        stream.puts title
+        stream.puts keywords if keywords
+        stream.puts url
+        stream.puts score if include_score
+        stream.puts
+        doc.text.each { |text| stream.puts text }
+        stream.puts
+        stream.puts "-----" unless last_result
+        stream.puts unless last_result
+      end
+
+      results.size
+    end
+
     # Sanitises the obj to make it uniform by calling the correct sanitize_*
     # method for its type e.g. if obj.is_a? String then sanitize_str(obj) is called.
     # Any type not in the case statement will be ignored and returned as is.
@@ -230,6 +297,42 @@ module Wgit
         .uniq
     end
 
+    # Build a regular expression from a query string, for searching text with.
+    #
+    # All searches using this regex are always whole word based while whole
+    # sentence searches are configurable using the whole_sentence: param. For
+    # example:
+    #
+    # ```
+    # text  = "hello world"
+    # query = "world hello", whole_sentence: true  # => No match
+    # query = "world hello", whole_sentence: false # => Match
+    # query = "he"                                 # => Never matches
+    # ```
+    #
+    # @param query [String, Regexp] The query string to build a regex from.
+    # @param case_sensitive [Boolean] Whether character case must match.
+    # @param whole_sentence [Boolean] Whether multiple words should be searched
+    #   for separately, matching in any order.
+    # @return [Regexp] The regex with which to search text.
+    def self.build_search_regex(
+      query, case_sensitive: false, whole_sentence: true
+    )
+      return query if query.is_a?(Regexp)
+
+      # query: "hello world", whole_sentence: false produces:
+      # (?<=^|\s|[^a-zA-Z0-9])hello(?=$|\s|[^a-zA-Z0-9])|(?<=^|\s|[^a-zA-Z0-9])world(?=$|\s|[^a-zA-Z0-9])
+
+      sep = whole_sentence ? " " : "|"
+      segs = query.split(" ").map do |word|
+        word = Regexp.escape(word)
+        "(?<=^|\\s|[^a-zA-Z0-9])#{word}(?=$|\\s|[^a-zA-Z0-9])"
+      end
+      query = segs.join(sep)
+
+      Regexp.new(query, !case_sensitive)
+    end
+
     # Pretty prints a log statement, used for debugging purposes.
     #
     # Use like:
@@ -245,25 +348,30 @@ module Wgit
     # ```
     #
     # @param identifier [#to_s] A log identifier e.g. "START" or 1 etc.
+    # @param display [Boolean] Setting as false will cause a noop, useful for
+    #   switching off several/all pprint statements at once e.g. via ENV var.
     # @param stream [#puts] Any object that respond_to? :puts and :print. It is
     #   used to output the log text somewhere e.g. a file or STDERR.
     # @param prefix [String] The log prefix, useful for visibility/greping.
     # @param new_line [Boolean] Wether or not to use a new line (\n) as the
     #   separator.
     # @param vars [Hash<#inspect, #inspect>] The vars to inspect in the log.
-    def self.pprint(identifier, stream: $stdout, prefix: 'DEBUG', new_line: false, **vars)
+    def self.pprint(identifier, display: true, stream: $stdout, prefix: 'DEBUG', new_line: false, **vars)
+      return unless display
+
       sep1 = new_line ? "\n" : ' - '
+      sep1 = '' if vars.empty?
       sep2 = new_line ? "\n" : ' | '
 
       stream.print "\n#{prefix}_#{identifier}#{sep1}"
 
       vars.each_with_index do |arr, i|
-        last_item = (i + 1) == vars.size
+        is_last_item = (i + 1) == vars.size
         sep3 = sep2
-        sep3 = new_line ? "\n" : '' if last_item
+        sep3 = new_line ? "\n" : '' if is_last_item
         k, v = arr
 
-        stream.print "#{k}: #{v}#{sep3}"
+        stream.print "#{k}: #{v.inspect}#{sep3}"
       end
 
       stream.puts "\n"
@@ -271,5 +379,7 @@ module Wgit
 
       nil
     end
+
+    self.singleton_class.alias_method :pprint_search_results, :pprint_top_search_results
   end
 end

data/lib/wgit/version.rb

@@ -6,7 +6,7 @@
 # @author Michael Telford
 module Wgit
   # The current gem version of Wgit.
-  VERSION = '0.11.0'
+  VERSION = "0.12.1"
 
   # Returns the current gem version of Wgit as a String.
   def self.version

data/lib/wgit.rb

@@ -1,17 +1,21 @@
 # frozen_string_literal: true
 
-require_relative 'wgit/version'
-require_relative 'wgit/logger'
-require_relative 'wgit/assertable'
-require_relative 'wgit/utils'
-require_relative 'wgit/url'
-require_relative 'wgit/document'
-require_relative 'wgit/document_extractors'
-require_relative 'wgit/crawler'
-require_relative 'wgit/database/model'
-require_relative 'wgit/database/database'
-require_relative 'wgit/robots_parser'
-require_relative 'wgit/indexer'
-require_relative 'wgit/dsl'
-require_relative 'wgit/base'
+require_relative "wgit/version"
+require_relative "wgit/logger"
+require_relative "wgit/assertable"
+require_relative "wgit/utils"
+require_relative "wgit/url"
+require_relative "wgit/html_to_text"
+require_relative "wgit/document"
+require_relative "wgit/document_extractors"
+require_relative "wgit/crawler"
+require_relative "wgit/model"
+require_relative "wgit/database/database"
+require_relative "wgit/database/database_adapter"
+require_relative "wgit/database/adapters/mongo_db"
+require_relative "wgit/database/adapters/in_memory"
+require_relative "wgit/robots_parser"
+require_relative "wgit/indexer"
+require_relative "wgit/dsl"
+require_relative "wgit/base"
 # require_relative 'wgit/core_ext' - Must be explicitly required.

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: wgit
 version: !ruby/object:Gem::Version
-  version: 0.11.0
+  version: 0.12.1
 platform: ruby
 authors:
 - Michael Telford
-autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-01-19 00:00:00.000000000 Z
+date: 2025-08-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -25,131 +24,173 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2.8'
 - !ruby/object:Gem::Dependency
-  name: mongo
+  name: base64
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.19'
+        version: '0.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.19'
+        version: '0.3'
 - !ruby/object:Gem::Dependency
-  name: nokogiri
+  name: benchmark
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.15'
+        version: '0.4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.15'
+        version: '0.4'
 - !ruby/object:Gem::Dependency
-  name: typhoeus
+  name: ferrum
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.4'
+        version: '0.17'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.4'
+        version: '0.17'
 - !ruby/object:Gem::Dependency
-  name: ferrum
+  name: logger
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.14'
+        version: '1.7'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.14'
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: mongo
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.21'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.21'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.18'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.18'
+- !ruby/object:Gem::Dependency
+  name: typhoeus
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
 - !ruby/object:Gem::Dependency
   name: byebug
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '11.1'
+        version: '12.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '11.1'
+        version: '12.0'
 - !ruby/object:Gem::Dependency
   name: dotenv
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.8'
+        version: '3.1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.8'
+        version: '3.1'
 - !ruby/object:Gem::Dependency
   name: maxitest
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.4'
+        version: '6.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.4'
+        version: '6.0'
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.14'
+        version: '0.15'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.14'
+        version: '0.15'
 - !ruby/object:Gem::Dependency
   name: rubocop
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.57'
+        version: '1.79'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.57'
+        version: '1.79'
 - !ruby/object:Gem::Dependency
   name: toys
   requirement: !ruby/object:Gem::Requirement
@@ -170,14 +211,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.19'
+        version: '3.25'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.19'
+        version: '3.25'
 - !ruby/object:Gem::Dependency
   name: yard
   requirement: !ruby/object:Gem::Requirement
@@ -209,13 +250,17 @@ files:
 - "./lib/wgit/base.rb"
 - "./lib/wgit/core_ext.rb"
 - "./lib/wgit/crawler.rb"
+- "./lib/wgit/database/adapters/in_memory.rb"
+- "./lib/wgit/database/adapters/mongo_db.rb"
 - "./lib/wgit/database/database.rb"
-- "./lib/wgit/database/model.rb"
+- "./lib/wgit/database/database_adapter.rb"
 - "./lib/wgit/document.rb"
 - "./lib/wgit/document_extractors.rb"
 - "./lib/wgit/dsl.rb"
+- "./lib/wgit/html_to_text.rb"
 - "./lib/wgit/indexer.rb"
 - "./lib/wgit/logger.rb"
+- "./lib/wgit/model.rb"
 - "./lib/wgit/response.rb"
 - "./lib/wgit/robots_parser.rb"
 - "./lib/wgit/url.rb"
@@ -256,8 +301,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.3
-signing_key:
+rubygems_version: 3.6.7
 specification_version: 4
 summary: Wgit is a HTML web crawler, written in Ruby, that allows you to programmatically
   extract the data you want from the web.

data/lib/wgit/database/model.rb

@@ -1,60 +0,0 @@
-# frozen_string_literal: true
-
-require_relative '../utils'
-
-module Wgit
-  # Module used to build the database collection objects, forming a data model.
-  module Model
-    # The data model for a Wgit::Url collection object and for an embedded
-    # 'url' inside a Wgit::Document collection object.
-    #
-    # @param url [Wgit::Url] The Url data object.
-    # @return [Hash] The URL model ready for DB insertion.
-    def self.url(url)
-      raise 'url must respond_to? :to_h' unless url.respond_to?(:to_h)
-
-      model = url.to_h
-      select_bson_types(model)
-    end
-
-    # The data model for a Wgit::Document collection object.
-    #
-    # @param doc [Wgit::Document] The Document data object.
-    # @return [Hash] The Document model ready for DB insertion.
-    def self.document(doc)
-      raise 'doc must respond_to? :to_h' unless doc.respond_to?(:to_h)
-
-      model = doc.to_h(include_html: false, include_score: false)
-      model['url'] = url(doc.url) # Expand Url String into full object.
-
-      select_bson_types(model)
-    end
-
-    # Common fields when inserting a record into the DB.
-    #
-    # @return [Hash] Insertion fields common to all models.
-    def self.common_insert_data
-      {
-        date_added: Wgit::Utils.time_stamp,
-        date_modified: Wgit::Utils.time_stamp
-      }
-    end
-
-    # Common fields when updating a record in the DB.
-    #
-    # @return [Hash] Update fields common to all models.
-    def self.common_update_data
-      {
-        date_modified: Wgit::Utils.time_stamp
-      }
-    end
-
-    # Returns the model having removed non bson types (for use with MongoDB).
-    #
-    # @param model_hash [Hash] The model Hash to sanitize.
-    # @return [Hash] The model Hash with non bson types removed.
-    def self.select_bson_types(model_hash)
-      model_hash.select { |_k, v| v.respond_to?(:bson_type) }
-    end
-  end
-end