wgit 0.10.8 → 0.12.0

data/lib/wgit/utils.rb

@@ -18,17 +18,15 @@ module Wgit
     #   keys.
     # @return [Hash] A Hash created from obj's instance vars and values.
     def self.to_h(obj, ignore: [], use_strings_as_keys: true)
-      hash = {}
+      obj.instance_variables.reduce({}) do |hash, var|
+        next hash if ignore.include?(var.to_s)
 
-      obj.instance_variables.each do |var|
-        next if ignore.include?(var.to_s)
-
-        key = var.to_s[1..-1] # Remove the @ prefix.
+        key = var.to_s[1..] # Remove the @ prefix.
         key = key.to_sym unless use_strings_as_keys
         hash[key] = obj.instance_variable_get(var)
-      end
 
-      hash
+        hash
+      end
     end
 
     # An improved :each method which supports both singleton and Enumerable
@@ -37,9 +35,9 @@ module Wgit
     # @yield [el] Gives each element (Object) of obj_or_objects if it's
     #   Enumerable, otherwise obj_or_objs itself is given.
     # @return [Object] The obj_or_objs parameter is returned.
-    def self.each(obj_or_objs)
+    def self.each(obj_or_objs, &block)
       if obj_or_objs.respond_to?(:each)
-        obj_or_objs.each { |obj| yield(obj) }
+        obj_or_objs.each(&block)
       else
         yield(obj_or_objs)
       end
@@ -127,96 +125,261 @@ module Wgit
     end
 
     # Prints out the search results in a search engine like format.
-    # The format for each result looks like:
-    #
-    # Title
     #
-    # Keywords (if there are some)
+    # The given results should be matching documents from a DB and should have
+    # `doc.search_text!` called for each document - to turn doc.text into only
+    # matching text, which this method uses.
     #
-    # Text Snippet (formatted to show the searched for query, if provided)
+    # The format for each result looks something like:
     #
+    # ```
+    # Title
+    # Keywords (if there are some)
+    # Text Snippet (formatted to show the searched for query)
     # URL
-    #
+    # Score (if include_score: true)
     # <empty_line_seperator>
+    # ```
     #
     # @param results [Array<Wgit::Document>] Array of Wgit::Document's which
     #   each have had #search!(query) called (to update it's @text with the
     #   the search results). The first @text sentence gets printed.
     # @param keyword_limit [Integer] The max amount of keywords to be
     #   outputted to the stream.
+    # @param include_score [Boolean] Wether or not to puts the document score.
     # @param stream [#puts] Any object that respond_to?(:puts). It is used
     #   to output text somewhere e.g. a file or STDERR.
     # @return [Integer] The number of results.
-    def self.printf_search_results(results, keyword_limit: 5, stream: STDOUT)
+    def self.pprint_top_search_results(
+      results, keyword_limit: 5, include_score: false, stream: $stdout
+    )
       raise 'stream must respond_to? :puts' unless stream.respond_to?(:puts)
 
       results.each do |doc|
-        title    = (doc.title || '<no title>')
+        title    = doc.title || '<no title>'
         keywords = doc.keywords&.take(keyword_limit)&.join(', ')
         sentence = doc.text.first
         url      = doc.url
+        score    = doc.score
 
         stream.puts title
         stream.puts keywords if keywords
         stream.puts sentence
         stream.puts url
+        stream.puts score if include_score
         stream.puts
       end
 
       results.size
     end
 
+    # Prints out the search results listing all of the matching text in each
+    # document.
+    #
+    # The given results should be matching documents from a DB and should have
+    # `doc.search_text!` called for each document - to turn doc.text into only
+    # matching text, which this method uses.
+    #
+    # The format for each result looks something like:
+    #
+    # ```
+    # Title
+    # Keywords (if there are some)
+    # URL
+    # Score (if include_score: true)
+    #
+    # "<text_snippet_1>"
+    # "<text_snippet_2>"
+    # ...
+    #
+    # <seperator>
+    # ```
+    #
+    # @param results [Array<Wgit::Document>] Array of Wgit::Document's which
+    #   each have had #search!(query) called (to update it's @text with the
+    #   the search results). The first @text sentence gets printed.
+    # @param keyword_limit [Integer] The max amount of keywords to be
+    #   outputted to the stream.
+    # @param include_score [Boolean] Wether or not to puts the document score.
+    # @param stream [#puts] Any object that respond_to?(:puts). It is used
+    #   to output text somewhere e.g. a file or STDERR.
+    # @return [Integer] The number of results.
+    def self.pprint_all_search_results(
+      results, keyword_limit: 5, include_score: false, stream: $stdout
+    )
+      raise 'stream must respond_to? :puts' unless stream.respond_to?(:puts)
+
+      results.each_with_index do |doc, i|
+        last_result = i == (results.size-1)
+
+        title    = doc.title || '<no title>'
+        keywords = doc.keywords&.take(keyword_limit)&.join(', ')
+        url      = doc.url
+        score    = doc.score
+
+        stream.puts title
+        stream.puts keywords if keywords
+        stream.puts url
+        stream.puts score if include_score
+        stream.puts
+        doc.text.each { |text| stream.puts text }
+        stream.puts
+        stream.puts "-----" unless last_result
+        stream.puts unless last_result
+      end
+
+      results.size
+    end
+
     # Sanitises the obj to make it uniform by calling the correct sanitize_*
-    # method for its type e.g. if obj.is_a? String then sanitize(obj). Any type
-    # not in the case statement will be ignored and returned as is.
+    # method for its type e.g. if obj.is_a? String then sanitize_str(obj) is called.
+    # Any type not in the case statement will be ignored and returned as is.
+    # Call this method if unsure what obj's type is.
     #
     # @param obj [Object] The object to be sanitized.
     # @param encode [Boolean] Whether or not to encode to UTF-8 replacing
     #   invalid characters.
-    # @return [Object] The sanitized obj is both modified and then returned.
+    # @return [Object] The sanitized obj.
     def self.sanitize(obj, encode: true)
       case obj
+      when Wgit::Url
+        sanitize_url(obj, encode:)
       when String
-        sanitize_str(obj, encode: encode)
+        sanitize_str(obj, encode:)
       when Array
-        sanitize_arr(obj, encode: encode)
+        sanitize_arr(obj, encode:)
       else
         obj
       end
     end
 
+    # Sanitises a Wgit::Url to make it uniform. First sanitizes the Url as a
+    # String before replacing the Url value with the sanitized version. This
+    # method therefore modifies the given url param and also returns it.
+    #
+    # @param url [Wgit::Url] The Wgit::Url to sanitize. url is modified.
+    # @param encode [Boolean] Whether or not to encode to UTF-8 replacing
+    #   invalid characters.
+    # @return [Wgit::Url] The sanitized url, which is also modified.
+    def self.sanitize_url(url, encode: true)
+      str = sanitize_str(url.to_s, encode:)
+      url.replace(str)
+    end
+
     # Sanitises a String to make it uniform. Strips any leading/trailing white
     # space. Also applies UTF-8 encoding (replacing invalid characters) if
     # `encode: true`.
     #
-    # @param str [String] The String to sanitize. str is modified.
+    # @param str [String] The String to sanitize. str is not modified.
     # @param encode [Boolean] Whether or not to encode to UTF-8 replacing
     #   invalid characters.
-    # @return [String] The sanitized str is both modified and then returned.
+    # @return [String] The sanitized str.
     def self.sanitize_str(str, encode: true)
-      if str.is_a?(String)
-        str.encode!('UTF-8', undef: :replace, invalid: :replace) if encode
-        str.strip!
-      end
+      return str unless str.is_a?(String)
 
-      str
+      str = str.encode('UTF-8', undef: :replace, invalid: :replace) if encode
+      str.strip
     end
 
     # Sanitises an Array to make it uniform. Removes empty Strings and nils,
     # processes non empty Strings using Wgit::Utils.sanitize and removes
     # duplicates.
     #
-    # @param arr [Enumerable] The Array to sanitize. arr is modified.
-    # @return [Enumerable] The sanitized arr is both modified and then returned.
+    # @param arr [Enumerable] The Array to sanitize. arr is not modified.
+    # @return [Enumerable] The sanitized arr.
     def self.sanitize_arr(arr, encode: true)
-      if arr.is_a?(Array)
-        arr.map! { |str| sanitize(str, encode: encode) }
-        arr.reject! { |str| str.is_a?(String) ? str.empty? : false }
-        arr.compact!
-        arr.uniq!
-      end
+      return arr unless arr.is_a?(Array)
 
       arr
+        .map { |str| sanitize(str, encode:) }
+        .reject { |str| str.is_a?(String) && str.empty? }
+        .compact
+        .uniq
+    end
+
+    # Build a regular expression from a query string, for searching text with.
+    #
+    # All searches using this regex are always whole word based while whole
+    # sentence searches are configurable using the whole_sentence: param. For
+    # example:
+    #
+    # ```
+    # text  = "hello world"
+    # query = "world hello", whole_sentence: true  # => No match
+    # query = "world hello", whole_sentence: false # => Match
+    # query = "he"                                 # => Never matches
+    # ```
+    #
+    # @param query [String, Regexp] The query string to build a regex from.
+    # @param case_sensitive [Boolean] Whether character case must match.
+    # @param whole_sentence [Boolean] Whether multiple words should be searched
+    #   for separately, matching in any order.
+    # @return [Regexp] The regex with which to search text.
+    def self.build_search_regex(
+      query, case_sensitive: false, whole_sentence: true
+    )
+      return query if query.is_a?(Regexp)
+
+      # query: "hello world", whole_sentence: false produces:
+      # (?<=^|\s|[^a-zA-Z0-9])hello(?=$|\s|[^a-zA-Z0-9])|(?<=^|\s|[^a-zA-Z0-9])world(?=$|\s|[^a-zA-Z0-9])
+
+      sep = whole_sentence ? " " : "|"
+      segs = query.split(" ").map do |word|
+        word = Regexp.escape(word)
+        "(?<=^|\\s|[^a-zA-Z0-9])#{word}(?=$|\\s|[^a-zA-Z0-9])"
+      end
+      query = segs.join(sep)
+
+      Regexp.new(query, !case_sensitive)
+    end
+
+    # Pretty prints a log statement, used for debugging purposes.
+    #
+    # Use like:
+    #
+    # ```
+    # Wgit::Utils.pprint 1, include_html: include_html, ignore: ignore_vars
+    # ```
+    #
+    # Which produces a log like:
+    #
+    # ```
+    # DEBUG_1 - include_html: true | ignore: ['@html', '@parser']
+    # ```
+    #
+    # @param identifier [#to_s] A log identifier e.g. "START" or 1 etc.
+    # @param display [Boolean] Setting as false will cause a noop, useful for
+    #   switching off several/all pprint statements at once e.g. via ENV var.
+    # @param stream [#puts] Any object that respond_to? :puts and :print. It is
+    #   used to output the log text somewhere e.g. a file or STDERR.
+    # @param prefix [String] The log prefix, useful for visibility/greping.
+    # @param new_line [Boolean] Wether or not to use a new line (\n) as the
+    #   separator.
+    # @param vars [Hash<#inspect, #inspect>] The vars to inspect in the log.
+    def self.pprint(identifier, display: true, stream: $stdout, prefix: 'DEBUG', new_line: false, **vars)
+      return unless display
+
+      sep1 = new_line ? "\n" : ' - '
+      sep1 = '' if vars.empty?
+      sep2 = new_line ? "\n" : ' | '
+
+      stream.print "\n#{prefix}_#{identifier}#{sep1}"
+
+      vars.each_with_index do |arr, i|
+        is_last_item = (i + 1) == vars.size
+        sep3 = sep2
+        sep3 = new_line ? "\n" : '' if is_last_item
+        k, v = arr
+
+        stream.print "#{k}: #{v.inspect}#{sep3}"
+      end
+
+      stream.puts "\n"
+      stream.puts "\n" unless new_line
+
+      nil
     end
+
+    self.singleton_class.alias_method :pprint_search_results, :pprint_top_search_results
   end
 end

data/lib/wgit/version.rb

@@ -6,7 +6,7 @@
 # @author Michael Telford
 module Wgit
   # The current gem version of Wgit.
-  VERSION = '0.10.8'
+  VERSION = "0.12.0"
 
   # Returns the current gem version of Wgit as a String.
   def self.version

data/lib/wgit.rb

@@ -1,16 +1,21 @@
 # frozen_string_literal: true
 
-require_relative 'wgit/version'
-require_relative 'wgit/logger'
-require_relative 'wgit/assertable'
-require_relative 'wgit/utils'
-require_relative 'wgit/url'
-require_relative 'wgit/document'
-require_relative 'wgit/document_extractors'
-require_relative 'wgit/crawler'
-require_relative 'wgit/database/model'
-require_relative 'wgit/database/database'
-require_relative 'wgit/indexer'
-require_relative 'wgit/dsl'
-require_relative 'wgit/base'
+require_relative "wgit/version"
+require_relative "wgit/logger"
+require_relative "wgit/assertable"
+require_relative "wgit/utils"
+require_relative "wgit/url"
+require_relative "wgit/html_to_text"
+require_relative "wgit/document"
+require_relative "wgit/document_extractors"
+require_relative "wgit/crawler"
+require_relative "wgit/model"
+require_relative "wgit/database/database"
+require_relative "wgit/database/database_adapter"
+require_relative "wgit/database/adapters/mongo_db"
+require_relative "wgit/database/adapters/in_memory"
+require_relative "wgit/robots_parser"
+require_relative "wgit/indexer"
+require_relative "wgit/dsl"
+require_relative "wgit/base"
 # require_relative 'wgit/core_ext' - Must be explicitly required.

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: wgit
 version: !ruby/object:Gem::Version
-  version: 0.10.8
+  version: 0.12.0
 platform: ruby
 authors:
 - Michael Telford
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-08-18 00:00:00.000000000 Z
+date: 2024-10-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -16,188 +16,196 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.6'
+        version: '2.8'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.6'
+        version: '2.8'
 - !ruby/object:Gem::Dependency
-  name: mongo
+  name: base64
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.9'
+        version: '0.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.9'
+        version: '0.2'
 - !ruby/object:Gem::Dependency
-  name: nokogiri
+  name: ferrum
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.10'
+        version: '0.14'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.10'
+        version: '0.14'
 - !ruby/object:Gem::Dependency
-  name: typhoeus
+  name: mongo
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '2.19'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '2.19'
 - !ruby/object:Gem::Dependency
-  name: ferrum
+  name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: '1.15'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: '1.15'
+- !ruby/object:Gem::Dependency
+  name: typhoeus
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
 - !ruby/object:Gem::Dependency
   name: byebug
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '11.1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '11.1'
 - !ruby/object:Gem::Dependency
   name: dotenv
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.5'
+        version: '2.8'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.5'
+        version: '2.8'
 - !ruby/object:Gem::Dependency
   name: maxitest
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.3'
+        version: '5.4'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.3'
+        version: '5.4'
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.12'
+        version: '0.14'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.12'
+        version: '0.14'
 - !ruby/object:Gem::Dependency
   name: rubocop
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.74'
+        version: '1.57'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.74'
+        version: '1.57'
 - !ruby/object:Gem::Dependency
   name: toys
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: '0.15'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: '0.15'
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.6'
+        version: '3.19'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.6'
+        version: '3.19'
 - !ruby/object:Gem::Dependency
   name: yard
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.9.20
-    - - "<"
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '0.9'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.9.20
-    - - "<"
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '0.9'
 description: 'Wgit was primarily designed to crawl static HTML websites to index and
   search their content - providing the basis of any search engine; but Wgit is suitable
   for many application domains including: URL parsing, data mining and statistical
@@ -215,14 +223,19 @@ files:
 - "./lib/wgit/base.rb"
 - "./lib/wgit/core_ext.rb"
 - "./lib/wgit/crawler.rb"
+- "./lib/wgit/database/adapters/in_memory.rb"
+- "./lib/wgit/database/adapters/mongo_db.rb"
 - "./lib/wgit/database/database.rb"
-- "./lib/wgit/database/model.rb"
+- "./lib/wgit/database/database_adapter.rb"
 - "./lib/wgit/document.rb"
 - "./lib/wgit/document_extractors.rb"
 - "./lib/wgit/dsl.rb"
+- "./lib/wgit/html_to_text.rb"
 - "./lib/wgit/indexer.rb"
 - "./lib/wgit/logger.rb"
+- "./lib/wgit/model.rb"
 - "./lib/wgit/response.rb"
+- "./lib/wgit/robots_parser.rb"
 - "./lib/wgit/url.rb"
 - "./lib/wgit/utils.rb"
 - "./lib/wgit/version.rb"
@@ -251,7 +264,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.6'
+      version: '3'
   - - "<"
     - !ruby/object:Gem::Version
       version: '4'
@@ -261,7 +274,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.22
+rubygems_version: 3.5.22
 signing_key:
 specification_version: 4
 summary: Wgit is a HTML web crawler, written in Ruby, that allows you to programmatically