http_url_validation_improved 1.2.0 → 1.3.0

data/Rakefile

@@ -20,6 +20,7 @@ a specified type.}
     gem.email = "walter@katipo.co.nz"
     gem.homepage = "http://github.com/kete/http_url_validation_improved"
     gem.authors = ["Erik Gregg", "Walter McGinnis", "Kieran Pilkington"]
+    gem.add_dependency "addressable"
     gem.add_development_dependency "thoughtbot-shoulda", ">= 0"
     # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
   end

data/VERSION

@@ -1 +1 @@
-1.2.0
+1.3.0

data/lib/http_url_validation_improved.rb

@@ -1,5 +1,5 @@
 require 'net/http'
-require 'uri'
+require 'addressable/uri'
 require 'socket'
 
 module ActiveRecord
@@ -26,15 +26,20 @@ module ActiveRecord
             moved_retry ||= false
             not_allowed_retry ||= false
             retry_without_headers ||= false
+            # some domains will block requests that come in more frequently than 1 per second
+            sleepy_domains = ['wikipedia.org']
+            sleep_interval = 2 # 2 to be on the safe side
+            must_sleep ||= false
             response = nil
 
             # resolve to url escaped version of URL
-            # TODO: at some point hopefully URI lib
-            # be updated to allow unicode values
-            # escape for now
-            value = URI.escape(value)
+            # value = URI.escape(value)
+            # updated to allow unicode values
+            # escaping shouldn't be necessary
+            must_sleep = sleepy_domains.select { |d| value.include?(d) }.size > 0
 
-            url = URI.parse(value)
+
+            url = Addressable::URI.parse(value)
 
             # Check Formatting
             # moved to use the URI library's logic
@@ -50,23 +55,17 @@ module ActiveRecord
               http.use_ssl = true
               http.verify_mode = OpenSSL::SSL::VERIFY_NONE
             end
-            headers = Object.const_defined?('SITE_URL') ? { "User-Agent" => "#{SITE_URL} link checking mechanism via Ruby Net/HTTP" } : { "User-Agent" => "Ruby Net/HTTp used for link checking mechanism" }
+            headers = Object.const_defined?('SITE_URL') ? { "User-Agent" => "#{SITE_URL} link checking mechanism (http://github.com/kete/http_url_validation_improved) via Ruby Net/HTTP" } : { "User-Agent" => "Ruby Net/HTTp used for link checking mechanism (http://github.com/kete/http_url_validation_improved)" }
             response = if not_allowed_retry
+                         sleep sleep_interval if must_sleep
+
                          if retry_without_headers
                            http.request_get(url.path) {|r|}
                          else
                            http.request_get(url.path, headers) {|r|}
                          end
                        else
-                         # we know that *.wikipedia.org don't like the headers
-                         # and will treat 3 requests to get to the point
-                         # where we normally try without headers as DoS
-                         # if not wikipedia, try with headers
-                         if value.include?('wikipedia.org')
-                           http.request_head(url.path)
-                         else
-                           http.request_head(url.path, headers)
-                         end
+                         http.request_head(url.path, headers)
                        end
             # response = not_allowed_retry ? http.request_get(url.path) {|r|} : http.request_head(url.path)
             # Comment out as you need to

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: http_url_validation_improved
 version: !ruby/object:Gem::Version 
-  hash: 31
+  hash: 27
   prerelease: false
   segments: 
   - 1
-  - 2
+  - 3
   - 0
-  version: 1.2.0
+  version: 1.3.0
 platform: ruby
 authors: 
 - Erik Gregg
@@ -17,11 +17,11 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-06-15 00:00:00 +12:00
+date: 2011-02-04 00:00:00 +13:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
-  name: thoughtbot-shoulda
+  name: addressable
   prerelease: false
   requirement: &id001 !ruby/object:Gem::Requirement 
     none: false
@@ -32,8 +32,22 @@ dependencies:
         segments: 
         - 0
         version: "0"
-  type: :development
+  type: :runtime
   version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: thoughtbot-shoulda
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id002
 description: |-
   a Rails gem that allows you to validate a URL 
   entered in a form. It validates if the URL exists by hitting it with a HEAD