organic-sitemap 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 550d7496372f4d7a51fab04c239469b01b7dd16e
-  data.tar.gz: 3ed2ed70b7af47dff51d9cb7dd9cab368ac50062
+  metadata.gz: 7e531fda12be1284f92a9281f34a951e1e38e8bc
+  data.tar.gz: 710e256a9584b0b5aaf05920f7d7758f32abc935
 SHA512:
-  metadata.gz: aed731809b5b558160b4d8ee24b861b38ec5a62438f3daeb4ecb281bd66d7c01d9f6c710165410da5b4de736570ee81f3e74d56404d4d472627fd578177a7ab6
-  data.tar.gz: 73a9bad75585e3c16d83dec8db8dacdb67e624a7625dfdd03c87d3b6fb41c94fcc698ebea5eb61fa8222a953139d745378e48f8d810645145fc733e5557f39c1
+  metadata.gz: 6c17c4decabef8100527d31677d079ec99f03ad4ae342aeef75c53314593ace1266e1eb16c0887de9da6c2e7454c7e7b93d2478d0277caa56b24829491c00fac
+  data.tar.gz: 7abeeb1efc8acfcee36d4a4254b90a83e40255108ea1bc33bcc9b664817cd4f427f89e94d181eeb2d4c8ec2b8992bd8466712776fa35092749efb5d8410108c7

data/CHANGELOG.md

@@ -1,3 +1,11 @@
+## 0.1.3
+
+### Changed
+
+- Add cache_service for get expiry cache url
+- Fix error with rspec gem versions
+- Add crawler_service to warmup urls
+
 ## 0.1.2
 
 ### Changed

data/README.md

@@ -36,16 +36,68 @@ OrganicSitemap.configure do |config|
   # OrganicSitemap ignore query_params to identify urls. You can add you allowed params
   # config.allowed_params = [...]
 
-  # By default, all urls are saved on Redis.new(url: 'redis://127.0.0.1:6379'), but you can set you own connection
+  # By default, all urls are saved on Redis.new(url: 'redis://127.0.0.1:6379'),
+  # but you can set you own connection
   # config.redis_connection = Your redis connection
 
   # url are saved on a set on redis called "sitemap-urls", but if you want you can change it
   # config.storage_key = your key
   
-  # By dafault all url have a expiry time in 7 days, after this time, if no one load this page, it will be removed from the set. To change it (expect number of days):
+  # By dafault all url have a expiry time in 7 days
+  # after this time, if no one load this page, it will be removed from the set.
+  # To change it (number of days):
   # config.expiry_time = X
+
+  # Crawler needs a domain to mount urls to visit
+  config.crawler_domain = "http://mi.domain.com"
+
+  # By default crawler_delay is 5sec. This is the time between get each url
+  # To change it (seconds of delay):
+  # config.crawler_delay = x
 end
 ```
+
+## Crawler and cache Services
+
+If have a front cache service, this services allow you to use your score sitemap to warmup this expire urls
+
+When a page is loaded, score is update with current time and this told as when last cache occurs. Using it we can get all expired urls, and visit they to warm it up.
+
+To configure it:
+
+### On initializer define:
+```
+  # Crawler needs a domain to mount urls to visit
+  config.crawler_domain = "http://mi.domain.com"
+
+  # By default crawler_delay is 5sec. This is the time between get each url
+  # To change it (seconds of delay):
+  # config.crawler_delay = x
+```
+
+With **CacheManager.uncached_urls(expiration_time: CacheExpirationTime, url_pattern: PATTERN)** we get all url not hitted on this time (all expired urls)
+
+Example:
+```
+# Return urls not visited between 1.week.ago(setted on config.expiry_time) and 3.hours.ago
+OrganicSitemap::CacheManager.uncached_urls(expiration_time: 3.hours)
+
+# Return urls not visited between 1.week.ago(setted on config.expiry_time) and 3.hours.ago and contains "/test/" string
+OrganicSitemap::CacheManager.uncached_urls(expiration_time: 3.hours, url_pattern: "/test/")
+
+# Return urls not visited between 1.week.ago(setted on config.expiry_time) and 3.hours.ago and match ^\/test\/ regexp
+OrganicSitemap::CacheManager.uncached_urls(expiration_time: 3.hours, url_pattern: /^\/test\//)
+
+
+```
+The with **CrawlerManager.warmup(urls)** we visit all this urls with a delay setted on configuration file (by default 5 sec). When we visit a url, *RedisManager* update score for this url and will be no more visited until not expire cache time
+
+Example:
+```
+# For a 1.day page cache
+CrawlerManager.warmup(CacheManager.uncached_urls(expiration_time: 3.hours))
+```
+
 ## Rails config generator
 
 Copy base config file on your Rails app by

data/lib/generators/organic_sitemap/templates/organic_sitemap.rb

@@ -19,4 +19,11 @@ OrganicSitemap.configure do |config|
   # after this time, if no one load this page, it will be removed from the set.
   # To change it (number of days):
   # config.expiry_time = X
+
+  # Crawler needs a domain to mount urls to visit
+  config.crawler_domain = "http://mi.domain.com"
+
+  # By default crawler_delay is 5sec. This is the time between get each url
+  # To change it (seconds of delay):
+  # config.crawler_delay = x 
 end

data/lib/organic-sitemap.rb

@@ -2,7 +2,10 @@ require 'organic-sitemap/version'
 require 'organic-sitemap/redis_manager'
 require 'organic-sitemap/configuration'
 require 'organic-sitemap/url_processor'
+require 'organic-sitemap/cache_manager'
+require 'organic-sitemap/crawler_manager'
 require 'redis'
+require 'httparty'
 
 if defined? Rails
   require 'organic-sitemap/railtie'

data/lib/organic-sitemap/cache_manager.rb

@@ -0,0 +1,14 @@
+module OrganicSitemap
+  class CacheManager
+    def self.uncached_urls(expiration_time:, url_pattern: "")
+      opts = {from: Time.now, to: to(expiration_time)}
+      urls = OrganicSitemap::RedisManager.sitemap_urls(opts)
+      urls.select{|x| x[url_pattern]}
+    end
+
+    private
+    def self.to(expiration_time)
+      Time.now + OrganicSitemap.configuration.expiry_time.days - expiration_time
+    end
+  end
+end

data/lib/organic-sitemap/configuration.rb

@@ -11,6 +11,7 @@ module OrganicSitemap
   class Configuration
     attr_accessor :storage, :storage_key, :domains, :allowed_params,
                   :skipped_urls, :redis_connection, :expiry_time
+    attr_accessor :crawler_domain, :crawler_delay
 
     def initialize
       @storage          = 'redis'
@@ -19,6 +20,7 @@ module OrganicSitemap
       @skipped_urls     = []
       @redis_connection = Redis.new(url: 'redis://127.0.0.1:6379')
       @expiry_time      = 7
+      @crawler_delay    = 5
     end
   end
 end

data/lib/organic-sitemap/crawler_manager.rb

@@ -0,0 +1,25 @@
+module OrganicSitemap
+  class CrawlerManager
+    def self.warmup(urls)
+      [*urls].each do |uri|
+        response = HTTParty.get(url_for(uri))
+
+        url_processed = OrganicSitemap::UrlProcessor.new(response.code, response.headers, response.request)
+
+        unless url_processed.url_from_cache_valid?
+          OrganicSitemap::RedisManager.remove_key(key: uri)
+        end
+        sleep crawler_delay 
+      end
+    end
+
+    private
+
+    def self.url_for(uri)
+      "#{OrganicSitemap.configuration.crawler_domain}#{uri}"
+    end
+    def self.crawler_delay
+      OrganicSitemap.configuration.crawler_delay
+    end
+  end
+end

data/lib/organic-sitemap/redis_manager.rb

@@ -26,6 +26,7 @@ module OrganicSitemap
         remove_key key
       end
     end
+
     private
     def self.redis_connection
       OrganicSitemap.configuration.redis_connection

data/lib/organic-sitemap/url_processor.rb

@@ -24,6 +24,10 @@ module OrganicSitemap
       success_response? && html_page? && request.get? && is_expected_domain? && is_allowed_url?
     end
 
+    def url_from_cache_valid?
+      success_response? && html_page? && is_expected_domain?
+    end
+
     private
 
     def success_response?

data/lib/organic-sitemap/version.rb

@@ -1,3 +1,3 @@
 module OrganicSitemap
-  VERSION = "0.1.2" # Add more redis Z methods
+  VERSION = "0.1.3" # Add cache_service and crawler_service
 end

data/organic_sitemap.gemspec

@@ -20,9 +20,11 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_dependency 'rack', '~> 1.0', '>= 1.0.0'
+  spec.add_dependency 'httparty'
 
   spec.add_development_dependency "bundler", "~> 1.9"
-  spec.add_development_dependency "rspec", '~> 0'
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "rspec-core"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency 'simplecov'
   spec.add_development_dependency 'redis'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: organic-sitemap
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - Kaskito
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2015-10-16 00:00:00.000000000 Z
+date: 2015-11-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rack
@@ -30,6 +30,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.0.0
+- !ruby/object:Gem::Dependency
+  name: httparty
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -48,14 +62,28 @@ dependencies:
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec-core
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
@@ -150,13 +178,14 @@ files:
 - lib/generators/organic_sitemap/config_generator.rb
 - lib/generators/organic_sitemap/templates/organic_sitemap.rb
 - lib/organic-sitemap.rb
+- lib/organic-sitemap/cache_manager.rb
 - lib/organic-sitemap/configuration.rb
+- lib/organic-sitemap/crawler_manager.rb
 - lib/organic-sitemap/middleware/url_capture.rb
 - lib/organic-sitemap/railtie.rb
 - lib/organic-sitemap/redis_manager.rb
 - lib/organic-sitemap/url_processor.rb
 - lib/organic-sitemap/version.rb
-- organic-sitemap-0.1.1.gem
 - organic_sitemap.gemspec
 homepage: https://github.com/abelardogilm/organic-sitemap/
 licenses: