site_mapper 0.0.9 → 0.0.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d2f7c3a2410e0cb508297f6fb229c1e5ec5e8f98
-  data.tar.gz: 6495918729c0e0d4c8788c4eb126c829b72dc17b
+  metadata.gz: 49d4c0ab215ac8872234e3c552275f89688f997d
+  data.tar.gz: 4b75bae288524a38fe3a8f7d6df9ab42e715211a
 SHA512:
-  metadata.gz: 80fceb2fd976b3c0e6ffe2f7237ffe9645257a271b99ca7f919e147a132fd09275862ba8c0563924e533e6b5049cd192726a06c48a8c4a4efa1b61b0af2c1e55
-  data.tar.gz: 9ff4ef13486b564bc9f8b3d60c42c248c26ed4245584af3e05a0a907b41ac7bf6bfc44ab5ece3cee91216c240201990c65e0adfa20960b28a0b37f8f6eb6ad4e
+  metadata.gz: e0cad8aedfb27fc5a4d56b297098b76a9abd43df86b6c20b9d6f228256091191bb020e620217a9bb40bdc68c981bf8d2ca4fdf2930c55158bb321fa11dfc960b
+  data.tar.gz: c3b1c1863e3de70793d127772b7a314b1e788483fe9151e441c1c209b7fed59c8b210cdae5d453017b0d49eff179f7b51eee47544f68a15c6a62c94783b7b24e

data/bin/site_mapper

@@ -3,4 +3,4 @@
 require 'site_mapper'
 
 url = ARGV[0]
-SiteMapper.map(url)
+SiteMapper.map(url, logger: :system)

data/lib/site_mapper.rb

@@ -2,6 +2,7 @@ require 'uri'
 require 'net/http'
 
 require 'site_mapper/version'
+require 'site_mapper/logger'
 require 'site_mapper/request'
 require 'site_mapper/robots'
 require 'site_mapper/crawler'
@@ -17,9 +18,28 @@ module SiteMapper
   # Map all links on a given site.
   # @return [Array] with links.
   # @param [String] link to domain
+  # @param [Hash] options hash
   # @example Collect all URLs from example.com
   #   SiteMapper.map('example.com')
-  def self.map(link)
+  def self.map(link, options = {})
+    set_logger(options[:logger])
     Crawler.collect_urls(link) { |url| yield(url) if block_given? }
   end
+
+  # Set logger.
+  # @param [Object] logger
+  # @example set system logger
+  #   SiteMapper.set_logger(:system)
+  # @example set nil logger
+  #   SiteMapper.set_logger(:nil)
+  # @example set your own logger
+  #   SiteMapper.set_logger(YourLogger)
+  def self.set_logger(logger)
+    return if logger.nil?
+    if logger.is_a?(Symbol)
+      Logger.use_logger_type(logger)
+    else
+      Logger.use_logger(logger)
+    end
+  end
 end

data/lib/site_mapper/crawler.rb

@@ -5,10 +5,10 @@ module SiteMapper
   # Crawls a given site.
   class Crawler
     # @param [String] url base url for crawler
-    # @param [Hash] resolve (optional false by default)
-    def initialize(url, resolve: false)
-      @base_url     = Request.resolve_url(url)
-      @options     = { resolve: resolve }
+    # @param [Hash] options hash, resolve key (optional false by default)
+    def initialize(url, options = {})
+      @base_url    = Request.resolve_url(url)
+      @options     = { resolve: false }.merge(options)
       @crawl_url   = CrawlUrl.new(@base_url)
       @fetch_queue = CrawlQueue.new
       @processed   = Set.new
@@ -37,17 +37,17 @@ module SiteMapper
         yield(url)
         page_links(url)
       end
-      puts "Crawling finished, #{@processed.length} links found"
+      Logger.log "Crawling finished, #{@processed.length} links found"
       @processed.to_a
     rescue Interrupt, IRB::Abort
-      puts 'Crawl interrupted.'
+      Logger.err_log 'Crawl interrupted.'
       @fetch_queue.to_a
     end
 
     private
 
     def page_links(get_url)
-      puts "Queue length: #{@fetch_queue.length}, Parsing: #{get_url}"
+      Logger.log "Queue length: #{@fetch_queue.length}, Parsing: #{get_url}"
       link_elements = Request.get_page(get_url).css('a') rescue []
       @processed << get_url
       link_elements.each do |page_link|
@@ -70,25 +70,25 @@ module SiteMapper
     def resolve(url)
       @options[:resolve] ? Request.resolve_url(url) : url
     end
-  end
 
-  # Queue of urls to be crawled.
-  class CrawlQueue
-    # @return [Set] that exends EnumerablePop module
-    def self.new
-      Set.new.extend(EnumerablePop)
-    end
-    
-    # Add pop method when added to class.
-    # The class that extends this module need to implement #first and #delete.
-    module EnumerablePop
-      # Pop first element from list.
-      # @return [Object] the first object in the list or nil
-      def pop
-        first_element = first
-        delete(first_element)
-        first_element
+    # Queue of urls to be crawled.
+    class CrawlQueue
+      # @return [Set] that exends EnumerablePop module
+      def self.new
+        Set.new.extend(EnumerablePop)
+      end
+      
+      # Add pop method when added to class.
+      # The class that extends this module need to implement #first and #delete.
+      module EnumerablePop
+        # Pop first element from list.
+        # @return [Object] the first object in the list or nil
+        def pop
+          first_element = first
+          delete(first_element)
+          first_element
+        end
       end
     end
-  end
+  end  
 end

data/lib/site_mapper/logger.rb

@@ -0,0 +1,57 @@
+module SiteMapper
+  # Handles logging
+  class Logger
+    # @param [Symbol] type of logger class to be used
+    def self.use_logger_type(type)
+      fail 'Logger already set' if defined?(@@log)
+      @@log = case type.to_s
+      when 'nil', 'default'
+        NilLogger
+      when 'system'
+        SystemOutLogger
+      else
+        fail ArgumentError, "Unknown logger type: '#{type}'"
+      end
+      @@log
+    end
+
+    # @param [Class, #log, #err_log] logger a logger class
+    def self.use_logger(logger)
+      fail 'Logger already set' if defined?(@@log)
+      @@log = logger
+    end
+
+    # @param [String] msg to be logged
+    def self.log(msg)
+      @@log ||= use_logger_type(:default)
+      @@log.log(msg)
+    end
+
+    # @param [String] err_msg to be logged
+    def self.err_log(err_msg)
+      @@log ||= use_logger_type(:default)
+      @@log.err_log(err_msg)
+    end
+
+    # Log to terminal.
+    module SystemOutLogger
+      # @param [String] msg to be logged to STDOUT
+      def self.log(msg)
+        STDOUT.puts(msg)
+      end
+
+      # @param [String] msg to be logged to STDERR
+      def self.err_log(msg)
+        STDERR.puts("[ERROR] #{msg}")
+      end
+    end
+
+    # Don't log
+    module NilLogger
+      # @param [String] msg to be ignored
+      def self.log(msg);end
+      # @param [String] msg to be ignored
+      def self.err_log(msg);end
+    end
+  end
+end

data/lib/site_mapper/request.rb

@@ -37,7 +37,7 @@ module SiteMapper
       end
 
       # Get response body, rescues with nil if an exception is raised.
-      # @see #get_response
+      # @see Request#get_response
       def get_response_body(*args)
         get_response(*args).body rescue nil
       end
@@ -45,14 +45,15 @@ module SiteMapper
       # Resolve an URL string and follows redirects.
       # if the URL can't be resolved the original URL is returned.
       # @param [String] url
-      # @param [Boolean] with_query (optional and true by default)
+      # @param [Hash] options hash, with_query key (optional and true by default)
       # @return [String] a URL string that potentially is a redirected URL
       # @example Resolve google.com
       #    resolve_url('google.com')
       #    # => 'https://www.google.com'
-      def resolve_url(url, with_query: true)
+      def resolve_url(url, options = {})
+        options  = { with_query: true }.merge(options)
         resolved = UrlResolver.resolve(url)
-        resolved = remove_query(resolved) unless with_query
+        resolved = remove_query(resolved) unless options[:with_query]
         resolved
       end
 

data/lib/site_mapper/robots.rb

@@ -1,5 +1,5 @@
-# Based on: https://rubygems.org/gems/robots, v0.10.1
 module SiteMapper
+  # Based on: https://rubygems.org/gems/robots, v0.10.1
   # Provided a base URL it checks whether a given URL is
   # allowed to be crawled according to /robots.txt.
   # @see https://rubygems.org/gems/robots
@@ -114,7 +114,9 @@ module SiteMapper
       end
     end
 
-    # @param [String] url to fetch /robots.txt from
+    # @param [String] robots_txt contents of /robots.txt
+    # @param [String] hostname for the passed robots_txt
+    # @param [String] user_agent to check
     def initialize(robots_txt, hostname, user_agent)
       @robots_txt = robots_txt
       @hostname   = hostname

data/lib/site_mapper/version.rb

@@ -1,4 +1,4 @@
 module SiteMapper
   # Gem version
-  VERSION = '0.0.9'
+  VERSION = '0.0.10'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: site_mapper
 version: !ruby/object:Gem::Version
-  version: 0.0.9
+  version: 0.0.10
 platform: ruby
 authors:
 - Jacob Burenstam
@@ -94,6 +94,20 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: '0.8'
+- !ruby/object:Gem::Dependency
+  name: redcarpet
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.2'
 - !ruby/object:Gem::Dependency
   name: coveralls
   requirement: !ruby/object:Gem::Requirement
@@ -119,6 +133,7 @@ files:
 - bin/site_mapper
 - lib/site_mapper/crawl_url.rb
 - lib/site_mapper/crawler.rb
+- lib/site_mapper/logger.rb
 - lib/site_mapper/request.rb
 - lib/site_mapper/robots.rb
 - lib/site_mapper/version.rb