organic-sitemap 0.2.1 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: aa747ab73beae12ff458805daddb62de82a43440
-  data.tar.gz: 12a0e654afe1610fc3bf9ee976ae4a157db36874
+  metadata.gz: d3e7fcd97003744b1142e0ffcb04cc1512f15128
+  data.tar.gz: e3c6405460286fd4e43d4f16132b938f195a3eb5
 SHA512:
-  metadata.gz: de620e662c93b3f13901fe443cb2dd619381c951146f679449e0bc2d6484de7d17c5cfb59877b6a63463dfed3aa1ff7377ee68957a63f93748ffea537a588819
-  data.tar.gz: 513bd94dfdf1f313145fe33f7d3ab588a381c9e11402100c4a7ded6fcff2866ea83fd98cb966e427e1bd59a05397b84c5304c1153b7a2eb79bf6b37e4b5bb7e4
+  metadata.gz: bbd93f06a83762feb6e1289866c608afa43daf0658b538238c22bab555f1b020292fb1b1319a90c4e2df2b69998900164167e17519a00518fcaa68ccacc51a2f
+  data.tar.gz: 1225d1f5cdf9b3d1b246f18dd7a8704c1b1f7ed1becfd56ea2932d7e2072b4cd411ed3aa7f50797ec83e79f75297f0de94337385ef68c384ec97ad9520a7d5b5

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.3.0
+
+### Changed
+
+- Add posibility to remove urls from collection if 404 or 301 response code
+
 ## 0.2.1
 
 ### Changed

data/README.md

@@ -13,13 +13,11 @@ OrganicSitemap is a gem that gives you a structure to manage your sitemap with h
 Uses a **Redis** connection to save sitemaps urls
 
 ## Installation
-OrganicSearch is not available as a gem yet, you can load it using my github account.
+Put this line in your Gemfile:
 
-Add in your Gemfile:
+```gem 'organic-sitemap'```
 
-```gem 'organic-sitemap', :git => 'git://github.com/abelardogilm/organic-sitemap.git'```
-
-Run ```bundle install.```
+Run ```bundle install```
 
 ## Configuration
 
@@ -54,6 +52,12 @@ OrganicSitemap.configure do |config|
   # By default crawler_delay is 5sec. This is the time between get each url
   # To change it (seconds of delay):
   # config.crawler_delay = x
+
+  # By default, do nothing with urls that not return 200.
+  # If you want remove automatically 301 urls from Redis
+  # config.clean_redirects = true
+  # If you want remove automatically 404 urls from Redis
+  # config.clean_not_found = true
 end
 ```
 
@@ -77,7 +81,7 @@ To configure it:
 
 With **CacheManager.uncached_urls(expiration_time: CacheExpirationTime, url_pattern: PATTERN)** we get all url not hitted on this time (all expired urls)
 
-Example:
+*** Examples
 ```
 # Return urls not visited between 1.week.ago(setted on config.expiry_time) and 3.hours.ago
 OrganicSitemap::CacheManager.uncached_urls(expiration_time: 3.hours)
@@ -88,11 +92,11 @@ OrganicSitemap::CacheManager.uncached_urls(expiration_time: 3.hours, url_pattern
 # Return urls not visited between 1.week.ago(setted on config.expiry_time) and 3.hours.ago and match ^\/test\/ regexp
 OrganicSitemap::CacheManager.uncached_urls(expiration_time: 3.hours, url_pattern: /^\/test\//)
 
-
 ```
+
 The with **CrawlerManager.warmup(urls, opts={})** we visit all this urls. We can set a delay between each page load setting a delay on configuration file. When we visit a url, *RedisManager* update score for this url and will be no more visited until not expire cache time
 
-Example:
+*** Examples
 ```
 # For a 3.hours page cache, get page with user-agent='Ruby'
 CrawlerManager.warmup(CacheManager.uncached_urls(expiration_time: 3.hours))

data/lib/generators/organic_sitemap/templates/organic_sitemap.rb

@@ -25,5 +25,11 @@ OrganicSitemap.configure do |config|
 
   # By default crawler_delay is 5sec. This is the time between get each url
   # To change it (seconds of delay):
-  # config.crawler_delay = x 
+  # config.crawler_delay = x
+
+  # By default, do nothing with urls that not return 200.
+  # If you want remove automatically 301 urls from Redis
+  # config.clean_redirects = true
+  # If you want remove automatically 404 urls from Redis
+  # config.clean_not_found = true
 end

data/lib/organic-sitemap/configuration.rb

@@ -10,7 +10,8 @@ module OrganicSitemap
 
   class Configuration
     attr_accessor :storage, :storage_key, :domains, :allowed_params,
-                  :skipped_urls, :redis_connection, :expiry_time
+                  :skipped_urls, :redis_connection, :expiry_time,
+                  :clean_redirects, :clean_not_found
     attr_accessor :crawler_domain, :crawler_delay
 
     def initialize
@@ -19,6 +20,8 @@ module OrganicSitemap
       @allowed_params   = []
       @skipped_urls     = []
       @redis_connection = Redis.new(url: 'redis://127.0.0.1:6379')
+      @clean_redirects  = false
+      @clean_not_found  = false
       @expiry_time      = 7
       @crawler_delay    = nil
     end

data/lib/organic-sitemap/middleware/url_capture.rb

@@ -10,6 +10,8 @@ module OrganicSitemap
         processor = OrganicSitemap::UrlProcessor.new(status, headers, Rack::Request.new(env))
         if processor.sitemap_url?
           OrganicSitemap::RedisManager.add(processor.sanitize_path_info)
+        elsif processor.cleanable_url?
+          OrganicSitemap::RedisManager.remove_key(key: processor.sanitize_path_info)
         end
         [status, headers, response]
       end

data/lib/organic-sitemap/url_processor.rb

@@ -28,6 +28,18 @@ module OrganicSitemap
       success_response? && html_page? && is_expected_domain?
     end
 
+    def cleanable_url?
+      redirect_response? || not_found_response?
+    end
+
+    def redirect_response?
+      OrganicSitemap.configuration.clean_redirects && status == 301
+    end
+
+    def not_found_response?
+      OrganicSitemap.configuration.clean_not_found && status == 404
+    end
+
     private
 
     def success_response?

data/lib/organic-sitemap/version.rb

@@ -1,3 +1,3 @@
 module OrganicSitemap
-  VERSION = "0.2.1" # Improve crawler_manager warmup response
+  VERSION = "0.3.0" # Add remove 301 and 404 functionality
 end

data/organic_sitemap.gemspec

@@ -9,8 +9,8 @@ Gem::Specification.new do |spec|
   spec.authors       = ["Kaskito"]
   spec.email         = ["abelardogilm@gmail.com"]
 
-  spec.summary       = %q{Create your Sitemap in a organic way.}
-  spec.description   = %q{Create your Sitemap in a organic way.}
+  spec.summary       = %q{Lets Users and Bots to create our Sitemap in a organic way.}
+  spec.description   = %q{Lets Users and Bots to create our Sitemap in a organic way.}
   spec.homepage      = "https://github.com/abelardogilm/organic-sitemap/"
   spec.license       = "MIT"
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: organic-sitemap
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.3.0
 platform: ruby
 authors:
 - Kaskito
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2015-11-27 00:00:00.000000000 Z
+date: 2016-05-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rack
@@ -156,7 +156,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: Create your Sitemap in a organic way.
+description: Lets Users and Bots to create our Sitemap in a organic way.
 email:
 - abelardogilm@gmail.com
 executables: []
@@ -210,5 +210,5 @@ rubyforge_project:
 rubygems_version: 2.4.3
 signing_key: 
 specification_version: 4
-summary: Create your Sitemap in a organic way.
+summary: Lets Users and Bots to create our Sitemap in a organic way.
 test_files: []