site_mapper 0.0.12 → 0.0.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: dc4b21c14dc15f1cc6df4b6406d12acf5cb821d9
-  data.tar.gz: 34ef0ab2fcd0a74bbcdd53d9e47681d6440f951d
+  metadata.gz: f4c94ba470563b948533b4f9f39023ea0bef89bd
+  data.tar.gz: ce8e91a137ee13825fdd1b5ef0fe1220d48d85a1
 SHA512:
-  metadata.gz: 1d4da1f2753dfb5f06ea577c02183efbf4cb919b783ce128f07b46f29b6af7a330cc01d839895ea4d4fb53cf68db2a6b5adccba47530a0278aaca9bfe1fa4c02
-  data.tar.gz: c32dde9478240d63b63d6e521e04f3c914322b544c79eee7ee25e1b2ea46b5ad2529ddc7d778d30fe2c10cde074f88a72c53458a814fc5d2ab74bc87288e63ed
+  metadata.gz: 36171ef9626a3f905d2441582a95f81b36c2ac5cd068ea3015c32b6c540d8cebc50ba6f4ecec2ac9ea889b8d164c383d2f5cea86cc53401ea33e6183c0bdd1e2
+  data.tar.gz: cd9e6bd664c97b0f31dce8e7b56a2b99e92261f20e5c4876b37c89afbe8b616149284a2d579b8c9b15849e5fc0393ef97cf07d7c02e2120fa8b7a01456891f9c

data/lib/site_mapper/crawl_url.rb

@@ -11,7 +11,7 @@ module SiteMapper
     # @example Intitialize CrawlUrl with example.com as base_url
     #   CrawlUrl.new('example.com')
     def initialize(base_url)
-      @resolved_base_url = Request.resolve_url(base_url) # "#{protocol}#{host}"
+      @resolved_base_url = Request.resolve_url(base_url)
       @base_hostname     = URI.parse(@resolved_base_url).hostname
     end
 

data/lib/site_mapper/crawler.rb

@@ -6,18 +6,14 @@ module SiteMapper
   class Crawler
     # Default options
     OPTIONS = {
-      resolve:      false,
       sleep_length: 0.5,
       max_requests: Float::INFINITY
     }
 
     # @param [String] url base url for crawler
-    # @param [Hash] options hash, resolve key (optional false by default)
-    # add user_agent key to specify custom User-agent
-    # @example Create crawler with custom User-agent
+    # @param [Hash] options hash
+    # @example Create crawler with custom User-Agent
     #    Crawler.new('example.com', user_agent: 'MyUserAgent')
-    # @example Create crawler and resolve all urls
-    #    Crawler.new('example.com', resolve: true)
     # @example Create crawler and sleep 1 second between each request
     #    Crawler.new('example.com', sleep_length: 1)
     # @example Create crawler and perform max 3 requests
@@ -75,7 +71,7 @@ module SiteMapper
       @processed << current_url
       link_elements.each do |page_link|
         url = @crawl_url.absolute_url_from(page_link.attr('href'), current_url)
-        @fetch_queue << url if url && eligible_for_queue?(resolve(url))
+        @fetch_queue << url if url && eligible_for_queue?(url)
       end
     end
 
@@ -85,22 +81,19 @@ module SiteMapper
 
     def robots
       return @robots unless @robots.nil?
-      robots_body  = Request.response_body("#{@base_url}/robots.txt", user_agent: @options[:user_agent])
-      @robots      = Robots.new(robots_body, URI.parse(@base_url).host, SiteMapper::USER_AGENT)
+      robots_url  = URI.join(@base_url, '/robots.txt').to_s
+      robots_body = Request.response_body(robots_url, user_agent: @options[:user_agent])
+      @robots     = Robots.new(robots_body, URI.parse(@base_url).host, @options[:user_agent])
       @robots
     end
 
-    def resolve(url)
-      @options[:resolve] ? Request.resolve_url(url) : url
-    end
-
     def wait
       sleep @options[:sleep_length]
     end
 
     # Queue of urls to be crawled.
     class CrawlQueue
-      # @return [Set] that exends EnumerablePop module
+      # @return [Set] that extends EnumerablePop module
       def self.new
         Set.new.extend(EnumerablePop)
       end

data/lib/site_mapper/logger.rb

@@ -45,13 +45,13 @@ module SiteMapper
       # Log to STDOUT
       # @param [String] msg to be logged to STDOUT
       def self.log(msg)
-        STDOUT.puts(msg)
+        $stdout.puts(msg)
       end
 
       # Log to STDERR
       # @param [String] msg to be logged to STDERR
       def self.err_log(msg)
-        STDERR.puts("[ERROR] #{msg}")
+        $stderr.puts("[ERROR] #{msg}")
       end
     end
 

data/lib/site_mapper/version.rb

@@ -1,4 +1,4 @@
 module SiteMapper
   # Gem version
-  VERSION = '0.0.12'
+  VERSION = '0.0.13'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: site_mapper
 version: !ruby/object:Gem::Version
-  version: 0.0.12
+  version: 0.0.13
 platform: ruby
 authors:
 - Jacob Burenstam