organic-sitemap 0.1.3 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7e531fda12be1284f92a9281f34a951e1e38e8bc
-  data.tar.gz: 710e256a9584b0b5aaf05920f7d7758f32abc935
+  metadata.gz: 758be2109c62c6cb65424a33b2cccfb42c0500b1
+  data.tar.gz: fb77334c80b26ce31a745e698dcc8752947314dc
 SHA512:
-  metadata.gz: 6c17c4decabef8100527d31677d079ec99f03ad4ae342aeef75c53314593ace1266e1eb16c0887de9da6c2e7454c7e7b93d2478d0277caa56b24829491c00fac
-  data.tar.gz: 7abeeb1efc8acfcee36d4a4254b90a83e40255108ea1bc33bcc9b664817cd4f427f89e94d181eeb2d4c8ec2b8992bd8466712776fa35092749efb5d8410108c7
+  metadata.gz: 6b0f70bd7651db8adf134049a88b49f9ae20035b4bd55c44417a558cba13a5584b7c05cac91230eb0f28062f470ed7c71561fd4b67ecd8054555cac940081b93
+  data.tar.gz: 18b8bdad32a4b68c34c0344c143b32531ee677c9380c69670e4bf7c54e4c4f7913440b077059c824623b846c6daf1431a6387e6e4b1b118954e701fb464850c9

data/CHANGELOG.md

@@ -1,10 +1,16 @@
+## 0.2.0
+
+### Changed
+
+- Let crawler_manager accept Httparty options
+
 ## 0.1.3
 
 ### Changed
 
-- Add cache_service for get expiry cache url
+- Add cache_manager for get expiry cache url
 - Fix error with rspec gem versions
-- Add crawler_service to warmup urls
+- Add crawler_manager to warmup urls
 
 ## 0.1.2
 

data/README.md

@@ -90,12 +90,15 @@ OrganicSitemap::CacheManager.uncached_urls(expiration_time: 3.hours, url_pattern
 
 
 ```
-The with **CrawlerManager.warmup(urls)** we visit all this urls with a delay setted on configuration file (by default 5 sec). When we visit a url, *RedisManager* update score for this url and will be no more visited until not expire cache time
+The with **CrawlerManager.warmup(urls, opts={})** we visit all this urls. We can set a delay between each page load setting a delay on configuration file. When we visit a url, *RedisManager* update score for this url and will be no more visited until not expire cache time
 
 Example:
 ```
-# For a 1.day page cache
+# For a 3.hours page cache, get page with user-agent='Ruby'
 CrawlerManager.warmup(CacheManager.uncached_urls(expiration_time: 3.hours))
+
+# Get '/test' with user-agent='Crawler-bot'
+CrawlerManager.warmup('/test', {headers: {"User-Agent" => 'Crawler-bot'}})
 ```
 
 ## Rails config generator

data/lib/organic-sitemap/configuration.rb

@@ -20,7 +20,7 @@ module OrganicSitemap
       @skipped_urls     = []
       @redis_connection = Redis.new(url: 'redis://127.0.0.1:6379')
       @expiry_time      = 7
-      @crawler_delay    = 5
+      @crawler_delay    = nil
     end
   end
 end

data/lib/organic-sitemap/crawler_manager.rb

@@ -1,15 +1,17 @@
 module OrganicSitemap
   class CrawlerManager
-    def self.warmup(urls)
+    def self.warmup(urls, opts = {})
       [*urls].each do |uri|
-        response = HTTParty.get(url_for(uri))
+        p "OrganicSitemap::CrawlerManager GET: #{uri}"
+        response = HTTParty.get(url_for(uri), opts)
 
         url_processed = OrganicSitemap::UrlProcessor.new(response.code, response.headers, response.request)
 
         unless url_processed.url_from_cache_valid?
+          p "OrganicSitemap::CrawlerManager REMOVING URL: #{uri} invalid url"
           OrganicSitemap::RedisManager.remove_key(key: uri)
         end
-        sleep crawler_delay 
+        sleep crawler_delay if crawler_delay
       end
     end
 

data/lib/organic-sitemap/version.rb

@@ -1,3 +1,3 @@
 module OrganicSitemap
-  VERSION = "0.1.3" # Add cache_service and crawler_service
+  VERSION = "0.2.0" # Add cache_service and crawler_service
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: organic-sitemap
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.2.0
 platform: ruby
 authors:
 - Kaskito
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2015-11-19 00:00:00.000000000 Z
+date: 2015-11-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rack