organic-sitemap 0.2.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 758be2109c62c6cb65424a33b2cccfb42c0500b1
-  data.tar.gz: fb77334c80b26ce31a745e698dcc8752947314dc
+  metadata.gz: aa747ab73beae12ff458805daddb62de82a43440
+  data.tar.gz: 12a0e654afe1610fc3bf9ee976ae4a157db36874
 SHA512:
-  metadata.gz: 6b0f70bd7651db8adf134049a88b49f9ae20035b4bd55c44417a558cba13a5584b7c05cac91230eb0f28062f470ed7c71561fd4b67ecd8054555cac940081b93
-  data.tar.gz: 18b8bdad32a4b68c34c0344c143b32531ee677c9380c69670e4bf7c54e4c4f7913440b077059c824623b846c6daf1431a6387e6e4b1b118954e701fb464850c9
+  metadata.gz: de620e662c93b3f13901fe443cb2dd619381c951146f679449e0bc2d6484de7d17c5cfb59877b6a63463dfed3aa1ff7377ee68957a63f93748ffea537a588819
+  data.tar.gz: 513bd94dfdf1f313145fe33f7d3ab588a381c9e11402100c4a7ded6fcff2866ea83fd98cb966e427e1bd59a05397b84c5304c1153b7a2eb79bf6b37e4b5bb7e4

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.2.1
+
+### Changed
+
+- Improve response of crawler_manager warmup method
+
 ## 0.2.0
 
 ### Changed

data/lib/organic-sitemap/crawler_manager.rb

@@ -1,18 +1,30 @@
 module OrganicSitemap
   class CrawlerManager
     def self.warmup(urls, opts = {})
-      [*urls].each do |uri|
+      data = {urls: [*urls],
+              valids: 0,
+              invalids: {}}
+
+      data[:urls].each do |uri|
         p "OrganicSitemap::CrawlerManager GET: #{uri}"
         response = HTTParty.get(url_for(uri), opts)
 
         url_processed = OrganicSitemap::UrlProcessor.new(response.code, response.headers, response.request)
 
         unless url_processed.url_from_cache_valid?
-          p "OrganicSitemap::CrawlerManager REMOVING URL: #{uri} invalid url"
+          p "OrganicSitemap::CrawlerManager REMOVING URL: #{uri} response code: #{response.code}"
           OrganicSitemap::RedisManager.remove_key(key: uri)
+          if data[:invalids][response.code]
+            data[:invalids][response.code] << uri
+          else
+            data[:invalids][response.code] = [uri]
+          end
+        else
+          data[:valids] += 1
         end
         sleep crawler_delay if crawler_delay
       end
+      data
     end
 
     private
@@ -20,6 +32,7 @@ module OrganicSitemap
     def self.url_for(uri)
       "#{OrganicSitemap.configuration.crawler_domain}#{uri}"
     end
+
     def self.crawler_delay
       OrganicSitemap.configuration.crawler_delay
     end

data/lib/organic-sitemap/version.rb

@@ -1,3 +1,3 @@
 module OrganicSitemap
-  VERSION = "0.2.0" # Add cache_service and crawler_service
+  VERSION = "0.2.1" # Improve crawler_manager warmup response
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: organic-sitemap
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - Kaskito
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2015-11-25 00:00:00.000000000 Z
+date: 2015-11-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rack