wgit 0.0.6 → 0.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4d081963b841bc76f0d02e823d40b1155b9ccf46b77b1611bc19297d9a415e36
-  data.tar.gz: 9568888d18bf1206c0fe082b7bf2176fe42bd1ea28e2e372d40295d1ae617408
+  metadata.gz: 81cc82cb5f9b408ca678b7b1731bac5531e72be11186d3c15d36bfbf61ed3838
+  data.tar.gz: b20d7c77389895b7a4dd303cb3a804aa0d1b099a6f98243144f6ae7f04094cd3
 SHA512:
-  metadata.gz: 44742705e9e853d587dd3a2daa45c25c0acbd2e0630ddb5ec112f77bf52bd018103998a5a39e37c7d26e3ea6b7580bfb05a75430c48f2c87f8caf806410fb490
-  data.tar.gz: b5a26fc1cf38c19c15cb75f56dca53327cf7e4c35da0d5a72d1dfa14c872d065714966c5a24936ed7a956f2ebbff27f06fff80b3395e7ea39308289c2c75f3f9
+  metadata.gz: c17a87d4bb10f750ea9d19faf16ed0a6de10caeab47d3586a3c2c633ec36a990c238ff9e119763c46868a81c111a4b19abd1cad8f09af7008c89474ff0bbf861
+  data.tar.gz: 3ad0950b664dc872bbc9394e6307974ace5a4ac35aac446aada4d470703360faac28efd911e48895708274581a82cb6c1b2ff1f02116122a2e25f817cd54b5c2

data/lib/wgit/assertable.rb

@@ -6,6 +6,7 @@ module Wgit
     DEFAULT_TYPE_FAIL_MSG = "Expected: %s, Actual: %s".freeze
     WRONG_METHOD_MSG = "arr must be Enumerable, use a different method".freeze
     DEFAULT_DUCK_FAIL_MSG = "%s doesn't respond_to? %s".freeze
+    DEFAULT_REQUIRED_KEYS_MSG = "Some or all of the required keys are not present: %s".freeze
   
     # Tests if the obj is of a given type.
     #
@@ -57,7 +58,20 @@ module Wgit
       end
       obj_or_objs
     end
-  
+
+    # The hash must include? the keys or a KeyError is raised.
+    #
+    # @param hash [Hash] The hash which should include the required keys.
+    # @param keys [Array<String, Symbol>] The keys whose presence to assert.
+    # @param msg [String] The raised KeyError message, if provided.
+    # @return [Hash] The given hash on successful assertion.
+    def assert_required_keys(hash, keys, msg = nil)
+      msg ||= DEFAULT_REQUIRED_KEYS_MSG % [keys.join(', ')]
+      all_present = keys.all? { |key| hash.keys.include? key }
+      raise KeyError.new(msg) unless all_present
+      hash
+    end
+
   private
   
     # obj must respond_to? all methods or an exception is raised.

data/lib/wgit/database/connection_details.rb

@@ -0,0 +1,47 @@
+require_relative '../assertable'
+
+module Wgit
+  extend Assertable
+
+  # The connection details for the database. This must be set if you want to
+  # store and access webpages in a database. Don't set the constant directly,
+  # instead use the funcs contained within the Wgit module.
+  CONNECTION_DETAILS = {}
+
+  # The keys required for a successful database connection.
+  CONNECTION_KEYS_REQUIRED = [
+    'DB_HOST', 'DB_PORT', 'DB_USERNAME', 'DB_PASSWORD', 'DB_DATABASE'
+  ]
+
+  # Set the database's connection details from the given hash. It is your
+  # responsibility to ensure the correct hash vars are present and set.
+  #
+  # @param hash [Hash] Containing the database connection details to use.
+  #   The hash should contain the following keys (of type String):
+  #   DB_HOST, DB_PORT, DB_USERNAME, DB_PASSWORD, DB_DATABASE
+  # @raise [KeyError] If any of the required connection details are missing.
+  # @return [Hash] Containing the database connection details from hash.
+  def self.set_connection_details(hash)
+    assert_required_keys(hash, CONNECTION_KEYS_REQUIRED)
+
+    CONNECTION_DETAILS[:host]  = hash.fetch('DB_HOST')
+    CONNECTION_DETAILS[:port]  = hash.fetch('DB_PORT')
+    CONNECTION_DETAILS[:uname] = hash.fetch('DB_USERNAME')
+    CONNECTION_DETAILS[:pword] = hash.fetch('DB_PASSWORD')
+    CONNECTION_DETAILS[:db]    = hash.fetch('DB_DATABASE')
+
+    CONNECTION_DETAILS
+  end
+
+  # Set the database's connection details from the ENV. It is your
+  # responsibility to ensure the correct ENV vars are present and set.
+  #
+  # The ENV should contain the following keys (of type String):
+  # DB_HOST, DB_PORT, DB_USERNAME, DB_PASSWORD, DB_DATABASE
+  #
+  # @raise [KeyError] If any of the required connection details are missing.
+  # @return [Hash] Containing the database connection details from the ENV.
+  def self.set_connection_details_from_env
+    self.set_connection_details(ENV)
+  end
+end

data/lib/wgit/database/database.rb

@@ -3,6 +3,7 @@ require_relative '../url'
 require_relative '../utils'
 require_relative '../assertable'
 require_relative 'model'
+require 'logger'
 require 'mongo'
 
 module Wgit
@@ -21,17 +22,19 @@ module Wgit
         raise "Wgit::CONNECTION_DETAILS must be defined and include :host, 
 :port, :db, :uname, :pword for a database connection to be established."
       end
-      
-      # Only log to STDOUT in fatal scenarios.
-      Mongo::Logger.logger.level = Logger::FATAL
-      
+
+      # Only log for error (or more severe) scenarios.
+      Mongo::Logger.logger          = Wgit.logger.clone
+      Mongo::Logger.logger.progname = 'mongo'
+      Mongo::Logger.logger.level    = Logger::ERROR
+
       address = "#{conn_details[:host]}:#{conn_details[:port]}"
       @@client = Mongo::Client.new([address], 
-                                   database:      conn_details[:db],
-                                   user:          conn_details[:uname],
-                                   password:      conn_details[:pword])
+                                   database: conn_details[:db],
+                                   user:     conn_details[:uname],
+                                   password: conn_details[:pword])
     end
-  
+
     ### Create Data ###
   
     # Insert one or more Url or Document objects into the DB.

data/lib/wgit/document.rb

@@ -24,7 +24,7 @@ module Wgit
       :main, :ol, :p, :pre, :span, :ul, :h1, :h2, :h3, :h4, :h5
     ]
     
-    # The URL of the webpage, an instance of Wgit:Url.
+    # The URL of the webpage, an instance of Wgit::Url.
     attr_reader :url
     
     # The HTML of the webpage, an instance of String.
@@ -56,7 +56,7 @@ module Wgit
       # Init from URL String and HTML String.
       if url_or_obj.is_a?(String)
         url = url_or_obj
-        assert_type(url, Url)
+        assert_type(url, Wgit::Url)
   
         @url = url
         @html = html ||= ""

data/lib/wgit/indexer.rb

@@ -8,8 +8,8 @@ module Wgit
   #
   # Retrieves uncrawled url's from the database and recursively crawls each
   # site storing their internal pages into the database and adding their
-  # external url's to be crawled at a later date. Puts out info on the crawl
-  # to STDOUT as it goes along.
+  # external url's to be crawled later on. Logs info on the crawl
+  # using Wgit.logger as it goes along.
   # 
   # @param max_sites_to_crawl [Integer] The number of separate and whole
   #   websites to be crawled before the method exits. Defaults to -1 which
@@ -81,8 +81,8 @@ module Wgit
 
     # Retrieves uncrawled url's from the database and recursively crawls each
     # site storing their internal pages into the database and adding their
-    # external url's to be crawled at a later date. Puts out info on the crawl
-    # to STDOUT as it goes along.
+    # external url's to be crawled later on. Logs info on the crawl
+    # using Wgit.logger as it goes along.
     #
     # @param max_sites_to_crawl [Integer] The number of separate and whole
     #   websites to be crawled before the method exits. Defaults to -1 which
@@ -93,28 +93,28 @@ module Wgit
     #   that will be obtained.
     def index_the_web(max_sites_to_crawl = -1, max_data_size = 1048576000)
       if max_sites_to_crawl < 0
-        puts "Indexing until the database has been filled or it runs out of \
-urls to crawl (which might be never)."
+        Wgit.logger.info("Indexing until the database has been filled or it runs out of \
+urls to crawl (which might be never).")
       end
       site_count = 0
       
       while keep_crawling?(site_count, max_sites_to_crawl, max_data_size) do
-        puts "Current database size: #{@db.size}"
+        Wgit.logger.info("Current database size: #{@db.size}")
         @crawler.urls = @db.uncrawled_urls
 
         if @crawler.urls.empty?
-          puts "No urls to crawl, exiting."
+          Wgit.logger.info("No urls to crawl, exiting.")
           return
         end
-        puts "Starting crawl loop for: #{@crawler.urls}"
+        Wgit.logger.info("Starting crawl loop for: #{@crawler.urls}")
     
         docs_count = 0
         urls_count = 0
     
         @crawler.urls.each do |url|
           unless keep_crawling?(site_count, max_sites_to_crawl, max_data_size)
-            puts "Reached max number of sites to crawl or database \
-capacity, exiting."
+            Wgit.logger.info("Reached max number of sites to crawl or database \
+capacity, exiting.")
             return
           end
           site_count += 1
@@ -133,20 +133,20 @@ capacity, exiting."
           end
       
           urls_count += write_urls_to_db(ext_links)
-          puts "Crawled and saved #{site_docs_count} docs for the \
-site: #{url}"
+          Wgit.logger.info("Crawled and saved #{site_docs_count} docs for the \
+site: #{url}")
         end
 
-        puts "Crawled and saved docs for #{docs_count} url(s) overall for \
-this iteration."
-        puts "Found and saved #{urls_count} external url(s) for the next \
-iteration."
+        Wgit.logger.info("Crawled and saved docs for #{docs_count} url(s) overall for \
+this iteration.")
+        Wgit.logger.info("Found and saved #{urls_count} external url(s) for the next \
+iteration.")
       end
     end
 
     # Crawls a single website's pages and stores them into the database.
     # There is no max download limit so be careful which sites you index.
-    # Puts out info on the crawl to STDOUT as it goes along.
+    # Logs info on the crawl using Wgit.logger as it goes along.
     #
     # @param url [Wgit::Url] The base Url of the website to crawl.
     # @param insert_externals [Boolean] Whether or not to insert the website's
@@ -168,7 +168,7 @@ iteration."
         if result
           if write_doc_to_db(doc)
             total_pages_indexed += 1
-            puts "Crawled and saved internal page: #{doc.url}"
+            Wgit.logger.info("Crawled and saved internal page: #{doc.url}")
           end
         end
       end
@@ -182,11 +182,11 @@ iteration."
       
       if insert_externals
         write_urls_to_db(ext_urls)
-        puts "Found and saved #{ext_urls.length} external url(s)"
+        Wgit.logger.info("Found and saved #{ext_urls.length} external url(s)")
       end
       
-      puts "Crawled and saved #{total_pages_indexed} docs for the \
-site: #{url}"
+      Wgit.logger.info("Crawled and saved #{total_pages_indexed} docs for the \
+site: #{url}")
 
       total_pages_indexed
     end
@@ -208,10 +208,10 @@ site: #{url}"
     # inserts.
     def write_doc_to_db(doc)
       @db.insert(doc)
-      puts "Saved document for url: #{doc.url}"
+      Wgit.logger.info("Saved document for url: #{doc.url}")
       true
     rescue Mongo::Error::OperationFailure
-      puts "Document already exists: #{doc.url}"
+      Wgit.logger.info("Document already exists: #{doc.url}")
       false
     end
 
@@ -223,9 +223,9 @@ site: #{url}"
           begin
             @db.insert(url)
             count += 1
-            puts "Inserted url: #{url}"
+            Wgit.logger.info("Inserted url: #{url}")
           rescue Mongo::Error::OperationFailure
-            puts "Url already exists: #{url}"
+            Wgit.logger.info("Url already exists: #{url}")
           end
         end
       end

data/lib/wgit/logger.rb

@@ -0,0 +1,36 @@
+# FYI: The default logger is set at the bottom of this file.
+
+require 'logger'
+
+module Wgit
+  # The Logger instance used by Wgit. Set your own custom logger after
+  # requiring this file if needed.
+  @logger = nil
+
+  # Returns the current Logger instance.
+  # @return [Logger] The current Logger instance.
+  def self.logger
+    @logger
+  end
+
+  # Sets the current Logger instance.
+  # @param logger [Logger] The Logger instance to use.
+  # @return [Logger] The current Logger instance having being set.
+  def self.logger=(logger)
+    @logger = logger
+  end
+
+  # Returns the default Logger instance.
+  # @return [Logger] The default Logger instance.
+  def self.default_logger
+    Logger.new(STDOUT, progname: 'wgit', level: :info)
+  end
+
+  # Sets the default Logger instance to be used by Wgit.
+  # @return [Logger] The default Logger instance.
+  def self.use_default_logger
+    @logger = self.default_logger
+  end
+end
+
+Wgit.use_default_logger

data/lib/wgit/version.rb

@@ -3,5 +3,5 @@
 # @author Michael Telford
 module Wgit
   # The current gem version of Wgit.
-  VERSION = "0.0.6".freeze
+  VERSION = "0.0.7".freeze
 end

data/lib/wgit.rb

@@ -1,11 +1,12 @@
 require_relative 'wgit/version'
-require_relative 'wgit/crawler'
-require_relative 'wgit/indexer'
+require_relative 'wgit/logger'
+require_relative 'wgit/assertable'
+require_relative 'wgit/utils'
 require_relative 'wgit/url'
 require_relative 'wgit/document'
-require_relative 'wgit/utils'
-require_relative 'wgit/assertable'
-require_relative 'wgit/database/database'
+require_relative 'wgit/crawler'
+require_relative 'wgit/database/connection_details'
 require_relative 'wgit/database/model'
-require_relative 'wgit/database/mongo_connection_details'
-#require_relative 'wgit/core_ext'
+require_relative 'wgit/database/database'
+require_relative 'wgit/indexer'
+#require_relative 'wgit/core_ext' - Must be explicitly required.

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: wgit
 version: !ruby/object:Gem::Version
-  version: 0.0.6
+  version: 0.0.7
 platform: ruby
 authors:
 - Michael Telford
@@ -94,6 +94,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '12.3'
+- !ruby/object:Gem::Dependency
+  name: httplog
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
@@ -138,11 +152,12 @@ files:
 - "./lib/wgit/assertable.rb"
 - "./lib/wgit/core_ext.rb"
 - "./lib/wgit/crawler.rb"
+- "./lib/wgit/database/connection_details.rb"
 - "./lib/wgit/database/database.rb"
 - "./lib/wgit/database/model.rb"
-- "./lib/wgit/database/mongo_connection_details.rb"
 - "./lib/wgit/document.rb"
 - "./lib/wgit/indexer.rb"
+- "./lib/wgit/logger.rb"
 - "./lib/wgit/url.rb"
 - "./lib/wgit/utils.rb"
 - "./lib/wgit/version.rb"

data/lib/wgit/database/mongo_connection_details.rb

@@ -1,48 +0,0 @@
-module Wgit
-  # The connection details for the database. This must be set if you want to
-  # store and access webpages in a database. Don't set the constant directly,
-  # instead use the funcs contained within the Wgit module.
-  CONNECTION_DETAILS = {}
-
-  # Set the database's connection details from the given hash and freeze them.
-  # It is your responsibility to ensure the correct hash vars are present and
-  # set. Due to the freezing of the CONNECTION_DETAILS, this func is designed
-  # to be called only once.
-  #
-  # @param hash [Hash] Containing the database connection details to use.
-  #   The hash should contain the following keys (of type String):
-  #   host, port, uname, pword, db
-  # @raise [KeyError, FrozenError] If any of the required connection
-  #   details are missing or if the connection details have already been set.
-  # @return [Hash] Containing the database connection details from hash.
-  def self.set_connection_details(hash)
-    CONNECTION_DETAILS[:host]   = hash.fetch('host')
-    CONNECTION_DETAILS[:port]   = hash.fetch('port')
-    CONNECTION_DETAILS[:uname]  = hash.fetch('uname')
-    CONNECTION_DETAILS[:pword]  = hash.fetch('pword')
-    CONNECTION_DETAILS[:db]     = hash.fetch('db')
-
-    CONNECTION_DETAILS.freeze
-  end
-
-  # Set the database's connection details from the ENV and freeze them. It is
-  # your responsibility to ensure the correct ENV vars are present and set.
-  # Due to the freezing of the CONNECTION_DETAILS, this func is designed to be
-  # called only once.
-  #
-  # The ENV should contain the following keys (of type String):
-  # DB_HOST, DB_PORT, DB_USERNAME, DB_PASSWORD, DB_DATABASE
-  #
-  # @raise [KeyError, FrozenError] If any of the required connection
-  #   details are missing or if the connection details have already been set.
-  # @return [Hash] Containing the database connection details from the ENV.
-  def self.set_connection_details_from_env
-    CONNECTION_DETAILS[:host]   = ENV.fetch('DB_HOST')
-    CONNECTION_DETAILS[:port]   = ENV.fetch('DB_PORT')
-    CONNECTION_DETAILS[:uname]  = ENV.fetch('DB_USERNAME')
-    CONNECTION_DETAILS[:pword]  = ENV.fetch('DB_PASSWORD')
-    CONNECTION_DETAILS[:db]     = ENV.fetch('DB_DATABASE')
-
-    CONNECTION_DETAILS.freeze
-  end
-end