wgit 0.0.18 → 0.2.0

data/lib/wgit/utils.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Wgit
-  # Utility module containing generic methods.
+  # Utility module containing generic methods that don't belong to a Class.
   module Utils
     # Returns the current time stamp.
     #
@@ -13,75 +13,70 @@ module Wgit
     # Returns a Hash created from obj's instance vars and values.
     #
     # @param obj [Object] The object to process.
-    # @param ignore [Array<String>] Attributes to ignore.
-    # @param use_strings_as_keys [Boolean] Whether or not to use strings as
-    #     the keys in the returned Hash. Symbols are used otherwise.
+    # @param ignore [Array<String>] Attributes to ignore e.g. [':@html'].
+    # @param use_strings_as_keys [Boolean] Whether to use Strings or Symbols as
+    #   keys.
     # @return [Hash] A Hash created from obj's instance vars and values.
-    def self.to_h(obj, ignore = [], use_strings_as_keys = true)
+    def self.to_h(obj, ignore: [], use_strings_as_keys: true)
       hash = {}
+
       obj.instance_variables.each do |var|
         next if ignore.include?(var.to_s)
 
-        key = var.to_s[1..-1]
+        key = var.to_s[1..-1] # Remove the @ prefix.
         key = key.to_sym unless use_strings_as_keys
         hash[key] = obj.instance_variable_get(var)
       end
-      hash
-    end
 
-    # Returns the model having removed non bson types (for use with MongoDB).
-    #
-    # @param model_hash [Hash] The model Hash to process.
-    # @return [Hash] The model Hash with non bson types removed.
-    def self.remove_non_bson_types(model_hash)
-      model_hash.select do |_k, v|
-        v.respond_to? :bson_type
-      end
+      hash
     end
 
-    # An improved :each method which accepts both singleton and Enumerable
+    # An improved :each method which supports both singleton and Enumerable
     # objects (as opposed to just an Enumerable object).
     #
-    # @yield [el] Gives each element of obj_or_objects if it's Enumerable,
-    #     otherwise obj_or_objs itself is given.
+    # @yield [el] Gives each element (Object) of obj_or_objects if it's
+    #   Enumerable, otherwise obj_or_objs itself is given.
+    # @return [Object] The obj_or_objs parameter is returned.
     def self.each(obj_or_objs)
       if obj_or_objs.respond_to?(:each)
         obj_or_objs.each { |obj| yield(obj) }
       else
         yield(obj_or_objs)
       end
+
+      obj_or_objs
     end
 
     # Formats the sentence (modifies the receiver) and returns its value.
     # The formatting is essentially to shorten the sentence and ensure that
     # the index is present somewhere in the sentence. Used for search query
-    # results.
+    # results with the index of the matching query.
     #
     # @param sentence [String] The sentence to be formatted.
     # @param index [Integer] The first index of a word in sentence. This is
-    #     usually a word in a search query.
+    #   usually a word in a search query.
     # @param sentence_limit [Integer] The max length of the formatted sentence
-    #     being returned. The length will be based on the sentence_limit
-    #     parameter or the full length of the original sentence, which ever
-    #     is less. The full sentence is returned if the sentence_limit is 0.
+    #   being returned. The length will be decided by the sentence_limit
+    #   parameter or the full length of the original sentence, which ever
+    #   is less. The full sentence is returned if the sentence_limit is 0.
     # @return [String] The sentence once formatted.
     def self.format_sentence_length(sentence, index, sentence_limit)
       raise 'A sentence value must be provided' if sentence.empty?
       raise 'The sentence length value must be even' if sentence_limit.odd?
-      if (index < 0) || (index > sentence.length)
+      if index.negative? || (index > sentence.length)
         raise "Incorrect index value: #{index}"
       end
 
-      return sentence if sentence_limit == 0
+      return sentence if sentence_limit.zero?
 
-      start = 0
+      start  = 0
       finish = sentence.length
 
       if sentence.length > sentence_limit
-        start = index - (sentence_limit / 2)
+        start  = index - (sentence_limit / 2)
         finish = index + (sentence_limit / 2)
 
-        if start < 0
+        if start.negative?
           diff = 0 - start
           if (finish + diff) > sentence.length
             finish = sentence.length
@@ -91,7 +86,7 @@ module Wgit
           start = 0
         elsif finish > sentence.length
           diff = finish - sentence.length
-          if (start - diff) < 0
+          if (start - diff).negative?
             start = 0
           else
             start -= diff
@@ -106,54 +101,38 @@ module Wgit
     end
 
     # Prints out the search results in a search engine like format.
-    # Most of the params are passed to Wgit::Document#search; see the docs.
     # The format for each result looks like:
     #
     # Title
     #
     # Keywords (if there are some)
     #
-    # Text Snippet (showing the searched for query if provided)
+    # Text Snippet (formatted to show the searched for query, if provided)
     #
     # URL
     #
     # <empty_line_seperator>
     #
-    # @param results [Array<Wgit::Document>] An Array whose
-    #     Wgit::Documents#text matches the query at least once.
-    # @param query [String] The text query to search for.
-    # @param _case_sensitive [Boolean] Whether or not the search should be
-    #     case sensitive or not.
-    # @param sentence_length [Integer] The length of the matching text of the
-    #     search results to be outputted to the stream.
-    # @param keyword_count [Integer] The max amount of keywords to be
-    #     outputted to the stream.
-    # @param stream [#puts] Any object that respond_to? :puts. It is used
-    #     to output text somewhere e.g. STDOUT (the default).
-    # @return [nil]
-    def self.printf_search_results(results, query = nil, _case_sensitive = false,
-                                   sentence_length = 80, keyword_count = 5,
-                                   stream = Kernel)
-      raise 'stream must respond_to? :puts' unless stream.respond_to? :puts
-
-      keyword_count -= 1 # Because Array's are zero indexed.
+    # @param results [Array<Wgit::Document>] Array of Wgit::Document's which
+    #   each have had #search!(query) called (to update it's @text with the
+    #   the search results). The first @text sentence gets printed.
+    # @param keyword_limit [Integer] The max amount of keywords to be
+    #   outputted to the stream.
+    # @param stream [#puts] Any object that respond_to?(:puts). It is used
+    #   to output text somewhere e.g. a file or STDOUT.
+    # @return [NilClass] Returns nil.
+    def self.printf_search_results(results, keyword_limit: 5, stream: STDOUT)
+      raise 'stream must respond_to? :puts' unless stream.respond_to?(:puts)
 
       results.each do |doc|
-        sentence =  if query.nil?
-                      nil
-                    else
-                      sentence = doc.search(query, sentence_length).first
-                      if sentence.nil?
-                        nil
-                      else
-                        sentence.strip.empty? ? nil : sentence
-                      end
-                    end
-        stream.puts doc.title
-        unless doc.keywords.nil? || doc.keywords.empty?
-          stream.puts doc.keywords[0..keyword_count].join(', ')
-        end
-        stream.puts sentence unless sentence.nil?
+        title = (doc.title || '<no title>')
+        missing_keywords = (doc.keywords.nil? || doc.keywords.empty?)
+        keywords = missing_keywords ? nil : doc.keywords.take(keyword_limit)
+        sentence = doc.text.first
+
+        stream.puts title
+        stream.puts keywords.join(', ') if keywords
+        stream.puts sentence if sentence
         stream.puts doc.url
         stream.puts
       end
@@ -161,8 +140,8 @@ module Wgit
       nil
     end
 
-    # Processes a String to make it uniform. Strips off any leading/trailing
-    # white space and converts to UTF-8.
+    # Processes a String to make it uniform. Strips any leading/trailing white
+    # space and converts to UTF-8.
     #
     # @param str [String] The String to process. str is modified.
     # @return [String] The processed str is both modified and then returned.
@@ -171,11 +150,12 @@ module Wgit
         str.encode!('UTF-8', 'UTF-8', invalid: :replace)
         str.strip!
       end
+
       str
     end
 
     # Processes an Array to make it uniform. Removes empty Strings and nils,
-    # processes non empty Strings using Wgit::Utils.process_str before removing
+    # processes non empty Strings using Wgit::Utils.process_str and removes
     # duplicates.
     #
     # @param arr [Enumerable] The Array to process. arr is modified.
@@ -187,7 +167,16 @@ module Wgit
         arr.compact!
         arr.uniq!
       end
+
       arr
     end
+
+    # Returns the model having removed non bson types (for use with MongoDB).
+    #
+    # @param model_hash [Hash] The model Hash to process.
+    # @return [Hash] The model Hash with non bson types removed.
+    def self.remove_non_bson_types(model_hash)
+      model_hash.select { |_k, v| v.respond_to?(:bson_type) }
+    end
   end
 end

data/lib/wgit/version.rb

@@ -5,5 +5,5 @@
 # @author Michael Telford
 module Wgit
   # The current gem version of Wgit.
-  VERSION = '0.0.18'
+  VERSION = '0.2.0'
 end

metadata

@@ -1,15 +1,57 @@
 --- !ruby/object:Gem::Specification
 name: wgit
 version: !ruby/object:Gem::Version
-  version: 0.0.18
+  version: 0.2.0
 platform: ruby
 authors:
 - Michael Telford
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-03-07 00:00:00.000000000 Z
+date: 2019-09-22 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: addressable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.6.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.6.0
+- !ruby/object:Gem::Dependency
+  name: mongo
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.9.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.9.0
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.10.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.10.3
 - !ruby/object:Gem::Dependency
   name: byebug
   requirement: !ruby/object:Gem::Requirement
@@ -53,19 +95,33 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.3'
 - !ruby/object:Gem::Dependency
-  name: minitest
+  name: inch
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+- !ruby/object:Gem::Dependency
+  name: maxitest
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.11'
+        version: '3.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.11'
+        version: '3.3'
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement
@@ -94,6 +150,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '12.3'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.74'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.74'
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
@@ -128,55 +198,16 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '1.0'
-- !ruby/object:Gem::Dependency
-  name: addressable
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 2.6.0
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 2.6.0
-- !ruby/object:Gem::Dependency
-  name: mongo
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 2.9.0
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 2.9.0
-- !ruby/object:Gem::Dependency
-  name: nokogiri
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 1.10.3
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 1.10.3
-description: Fundamentally, Wgit is a WWW indexer/scraper which crawls URL's, retrieves
-  and serialises their page contents for later use. You can use Wgit to copy entire
-  websites if required. Wgit also provides a means to search indexed documents stored
-  in a database. Therefore, this library provides the main components of a WWW search
-  engine. The Wgit API is easily extended allowing you to pull out the parts of a
-  webpage that are important to you, the code snippets or tables for example. As Wgit
-  is a library, it has uses in many different application types.
+description: 'Fundamentally, Wgit is a HTTP indexer/scraper which crawls URL''s to
+  retrieve and serialise their page contents for later use. You can use Wgit to copy
+  entire websites if required. Wgit also provides a means to search indexed documents
+  stored in a database. Therefore, this library provides the main components of a
+  WWW search engine. The Wgit API is easily extended allowing you to pull out the
+  parts of a webpage that are important to you, the code snippets or tables for example.
+  As Wgit is a library, it supports many different use cases including data mining,
+  analytics, web indexing and URL parsing to name a few.
+
+  '
 email: michael.telford@live.com
 executables: []
 extensions: []
@@ -186,7 +217,6 @@ files:
 - "./lib/wgit/assertable.rb"
 - "./lib/wgit/core_ext.rb"
 - "./lib/wgit/crawler.rb"
-- "./lib/wgit/database/connection_details.rb"
 - "./lib/wgit/database/database.rb"
 - "./lib/wgit/database/model.rb"
 - "./lib/wgit/document.rb"
@@ -196,10 +226,6 @@ files:
 - "./lib/wgit/url.rb"
 - "./lib/wgit/utils.rb"
 - "./lib/wgit/version.rb"
-- CHANGELOG.md
-- LICENSE.txt
-- README.md
-- TODO.txt
 homepage: https://github.com/michaeltelford/wgit
 licenses:
 - MIT
@@ -227,5 +253,5 @@ rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
 summary: Wgit is a Ruby gem similar in nature to GNU's `wget` tool. It provides an
-  easy to use API for programmatic web scraping, indexing and searching.
+  easy to use API for programmatic URL parsing, HTML indexing and searching.
 test_files: []

data/CHANGELOG.md

@@ -1,61 +0,0 @@
-# Wgit Change Log
-
-## v0.0.0 (TEMPLATE - DO NOT EDIT)
-### Added
-- ...
-### Changed/Removed
-- ...
-### Fixed
-- ...
----
-
-## v0.0.18
-### Added
-- `Wgit::Url#to_brand` method and updated `Wgit::Url#is_relative?` to support it.
-### Changed/Removed
-- Updated the documentation by changing some `private` methods to `protected`. These methods are now documented (on rubydocs) as a result.
-### Fixed
-- ...
----
-
-## v0.0.17
-### Added
-- Support for `<base>` element in `Wgit::Document`'s.
-- New `Wgit::Url` methods: `without_query_string`, `is_query_string?`, `is_anchor?`, `replace` (override of `String#replace`).
-### Changed/Removed
-- Breaking changes: Removed `Wgit::Document#internal_links_without_anchors` method.
-- Breaking changes (potentially): `Wgit::Url`'s are now replaced with the redirected to Url during a crawl.
-- Updated `Wgit::Document#base_url` to support an optional `link:` named parameter.
-- Updated `Wgit::Crawler#crawl_site` to allow the initial url to redirect to another host.
-- Updated `Wgit::Url#is_relative?` to support an optional `domain:` named parameter.
-### Fixed
-- Bug in `Wgit::Document#internal_full_links` affecting anchor and query string links including those used during `Wgit::Crawler#crawl_site`.
-- Bug causing an 'Invalid URL' error for `Wgit::Crawler#crawl_site`.
----
-
-## v0.0.16
-### Added
-- Added `Url.parse` class method as alias for `Url.new`.
-### Changed/Removed
-- Breaking changes: Removed `Wgit::Url.relative_link?` (class method). Use `Wgit::Url#is_relative?` (instance method) instead e.g. `Wgit::Url.new('/blah').is_relative?`.
-### Fixed
-- Several URI related bugs in `Wgit::Url` affecting crawls.
----
-
-## v0.0.15
-### Added
-- Support for IRI's (non ASCII based URL's).
-### Changed/Removed
-- Breaking changes: Removed `Document` and `Url#to_hash` aliases. Call `to_h` instead.
-### Fixed
-- Bug in `Crawler#crawl_site` where an internal redirect to an external site's page was being followed.
----
-
-## v0.0.14
-### Added
-- `Indexer#index_this_page` method.
-### Changed/Removed
-- Breaking Changes: `Wgit::CONNECTION_DETAILS` now only requires `DB_CONNECTION_STRING`.
-### Fixed
-- Found and fixed a bug in `Document#new`.
----