wgit 0.5.1 → 0.10.0

data/lib/wgit/utils.rb

@@ -145,7 +145,8 @@ module Wgit
     # @param keyword_limit [Integer] The max amount of keywords to be
     #   outputted to the stream.
     # @param stream [#puts] Any object that respond_to?(:puts). It is used
-    #   to output text somewhere e.g. a file or STDOUT.
+    #   to output text somewhere e.g. a file or STDERR.
+    # @return [Integer] The number of results.
     def self.printf_search_results(results, keyword_limit: 5, stream: STDOUT)
       raise 'stream must respond_to? :puts' unless stream.respond_to?(:puts)
 
@@ -162,18 +163,37 @@ module Wgit
         stream.puts
       end
 
-      nil
+      results.size
     end
 
-    # Processes a String to make it uniform. Strips any leading/trailing white
+    # Sanitises the obj to make it uniform by calling the correct sanitize_*
+    # method for its type e.g. if obj.is_a? String then sanitize(obj). Any type
+    # not in the case statement will be ignored and returned as is.
+    #
+    # @param obj [Object] The object to be sanitized.
+    # @param encode [Boolean] Whether or not to encode to UTF-8 replacing
+    #   invalid characters.
+    # @return [Object] The sanitized obj is both modified and then returned.
+    def self.sanitize(obj, encode: true)
+      case obj
+      when String
+        sanitize_str(obj, encode: encode)
+      when Array
+        sanitize_arr(obj, encode: encode)
+      else
+        obj
+      end
+    end
+
+    # Sanitises a String to make it uniform. Strips any leading/trailing white
     # space. Also applies UTF-8 encoding (replacing invalid characters) if
     # `encode: true`.
     #
-    # @param str [String] The String to process. str is modified.
+    # @param str [String] The String to sanitize. str is modified.
     # @param encode [Boolean] Whether or not to encode to UTF-8 replacing
     #   invalid characters.
-    # @return [String] The processed str is both modified and then returned.
-    def self.process_str(str, encode: true)
+    # @return [String] The sanitized str is both modified and then returned.
+    def self.sanitize_str(str, encode: true)
       if str.is_a?(String)
         str.encode!('UTF-8', undef: :replace, invalid: :replace) if encode
         str.strip!
@@ -182,15 +202,15 @@ module Wgit
       str
     end
 
-    # Processes an Array to make it uniform. Removes empty Strings and nils,
-    # processes non empty Strings using Wgit::Utils.process_str and removes
+    # Sanitises an Array to make it uniform. Removes empty Strings and nils,
+    # processes non empty Strings using Wgit::Utils.sanitize and removes
     # duplicates.
     #
-    # @param arr [Enumerable] The Array to process. arr is modified.
-    # @return [Enumerable] The processed arr is both modified and then returned.
-    def self.process_arr(arr)
+    # @param arr [Enumerable] The Array to sanitize. arr is modified.
+    # @return [Enumerable] The sanitized arr is both modified and then returned.
+    def self.sanitize_arr(arr, encode: true)
       if arr.is_a?(Array)
-        arr.map! { |str| process_str(str) }
+        arr.map! { |str| sanitize(str, encode: encode) }
         arr.reject! { |str| str.is_a?(String) ? str.empty? : false }
         arr.compact!
         arr.uniq!
@@ -198,13 +218,5 @@ module Wgit
 
       arr
     end
-
-    # Returns the model having removed non bson types (for use with MongoDB).
-    #
-    # @param model_hash [Hash] The model Hash to process.
-    # @return [Hash] The model Hash with non bson types removed.
-    def self.remove_non_bson_types(model_hash)
-      model_hash.select { |_k, v| v.respond_to?(:bson_type) }
-    end
   end
 end

data/lib/wgit/version.rb

@@ -1,11 +1,12 @@
 # frozen_string_literal: true
 
 # Wgit is a WWW indexer/scraper which crawls URL's and retrieves their page
-# contents for later use by serialisation.
+# contents for later use.
+#
 # @author Michael Telford
 module Wgit
   # The current gem version of Wgit.
-  VERSION = '0.5.1'
+  VERSION = '0.10.0'
 
   # Returns the current gem version of Wgit as a String.
   def self.version

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: wgit
 version: !ruby/object:Gem::Version
-  version: 0.5.1
+  version: 0.10.0
 platform: ruby
 authors:
 - Michael Telford
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-11-22 00:00:00.000000000 Z
+date: 2021-04-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -66,6 +66,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: ferrum
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
 - !ruby/object:Gem::Dependency
   name: byebug
   requirement: !ruby/object:Gem::Requirement
@@ -184,35 +198,41 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '1.0'
-description: 'Fundamentally, Wgit is a HTTP indexer/scraper which crawls URL''s to
-  retrieve and serialise their page contents for later use. You can use Wgit to copy
-  entire websites if required. Wgit also provides a means to search indexed documents
-  stored in a database. Therefore, this library provides the main components of a
-  WWW search engine. The Wgit API is easily extended allowing you to pull out the
-  parts of a webpage that are important to you, the code snippets or tables for example.
-  As Wgit is a library, it supports many different use cases including data mining,
-  analytics, web indexing and URL parsing to name a few.
+description: 'Wgit was primarily designed to crawl static HTML websites to index and
+  search their content - providing the basis of any search engine; but Wgit is suitable
+  for many application domains including: URL parsing, data mining and statistical
+  analysis.
 
   '
 email: michael.telford@live.com
-executables: []
+executables:
+- wgit
 extensions: []
 extra_rdoc_files: []
 files:
 - "./lib/wgit.rb"
 - "./lib/wgit/assertable.rb"
+- "./lib/wgit/base.rb"
 - "./lib/wgit/core_ext.rb"
 - "./lib/wgit/crawler.rb"
 - "./lib/wgit/database/database.rb"
 - "./lib/wgit/database/model.rb"
 - "./lib/wgit/document.rb"
-- "./lib/wgit/document_extensions.rb"
+- "./lib/wgit/document_extractors.rb"
+- "./lib/wgit/dsl.rb"
 - "./lib/wgit/indexer.rb"
 - "./lib/wgit/logger.rb"
 - "./lib/wgit/response.rb"
 - "./lib/wgit/url.rb"
 - "./lib/wgit/utils.rb"
 - "./lib/wgit/version.rb"
+- ".yardopts"
+- CHANGELOG.md
+- CODE_OF_CONDUCT.md
+- CONTRIBUTING.md
+- LICENSE.txt
+- README.md
+- bin/wgit
 homepage: https://github.com/michaeltelford/wgit
 licenses:
 - MIT
@@ -223,7 +243,7 @@ metadata:
   bug_tracker_uri: https://github.com/michaeltelford/wgit/issues
   documentation_uri: https://www.rubydoc.info/gems/wgit
   allowed_push_host: https://rubygems.org
-post_install_message: 
+post_install_message: Added the 'wgit' executable to $PATH
 rdoc_options: []
 require_paths:
 - lib
@@ -238,10 +258,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
-signing_key: 
+rubygems_version: 3.1.2
+signing_key:
 specification_version: 4
-summary: Wgit is a Ruby gem similar in nature to GNU's `wget` tool. It provides an
-  easy to use API for programmatic URL parsing, HTML indexing and searching.
+summary: Wgit is a HTML web crawler, written in Ruby, that allows you to programmatically
+  extract the data you want from the web.
 test_files: []