url_scrubber 0.7.4 → 0.7.5

data/lib/url_scrubber/version.rb

@@ -1,3 +1,3 @@
 module UrlScrubber
-  VERSION = "0.7.4"
+  VERSION = "0.7.5"
 end

data/lib/url_scrubber.rb

@@ -1,4 +1,8 @@
 require "url_scrubber/version"
+require 'net/http'
+require 'uri'
+
+USER_AGENT = 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_3; en-us) AppleWebKit/534.1+ (KHTML, like Gecko) Version/5.0 Safari/533.16'
 
 module UrlScrubber
   def self.scrub(url)
@@ -149,6 +153,10 @@ module UrlScrubber
   def self.facebook(url)
     if url.match("/media/albums") || url.match("/media/set")
       url = url.match('\&') ? url.split('&',2)[0] : url
+    elsif url.include?("facebook.com/profile.php?id=")
+      # these were being truncated, they do redirect, but typically a 301 response is generated 
+      # so the url is returned unchanged.  Better than truncation.
+      url, http_response = check_for_redirection(url)
     else
       url.sub!(/facebook\.com\/home\.php[\?#!\/]+/, 'facebook.com/')
       url = drop_url_query!(url)
@@ -188,4 +196,69 @@ module UrlScrubber
   def self.yelp(url)
     url
   end
+  
+  private
+  
+    def self.check_for_redirection(uri_str, limit = 5)
+      login_patterns = [
+        # pages that require user logins
+        %r{^.*/login[^/]*$}
+      ]
+
+      failure_patterns = [
+        # pages that give 200 codes but actually indicate a not found
+        %r{linkedin\.com/home\?report%2Efailure}i
+      ]
+
+      raise 'Too many HTTP redirects' if limit == 0
+      
+      begin
+        url = URI.parse(uri_str)        
+      rescue URI::InvalidURIError => e
+        return [uri_str, CustomError.new(786, "Invalid URI #{uri_str} : #{e.message}") ]
+      end
+
+      http = Net::HTTP.new(url.host, url.port)
+      if url.port == 443
+        http.use_ssl = true
+        http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      else
+        http.use_ssl = false
+      end
+      request = Net::HTTP::Get.new(url.request_uri, { 'User-Agent' => USER_AGENT })
+
+      begin
+        response = http.request(request)    
+      rescue Exception => e
+        failure_response = Net::HTTPClientError.new('1.1', '404', 'Not Found')
+        return [uri_str, failure_response]
+        end
+
+      if response.is_a? Net::HTTPRedirection
+        if response['location'][0,4] == "http"
+          if failure_patterns.any? { |pattern| response['location'].match(pattern) }
+            # got redirected to a page indicating failure, so act like it's a 404
+            failure_response = Net::HTTPClientError.new('1.1', '404', 'Not Found')
+            return [uri_str, failure_response]
+          end
+
+          redirected_url, base_response = check_for_redirection(response['location'], limit - 1)
+
+          if login_patterns.any? { |pattern| redirected_url.match(pattern) }
+            # got redirected to a login page. return the ultimate response, but the previous url
+            return [uri_str, base_response]
+          else
+            return [redirected_url, base_response]
+          end
+
+        else
+          redir_url = "http://#{url.host}#{response['location']}"
+          redirected_url, base_response = check_for_redirection(redir_url, limit - 1)
+          return [redirected_url, base_response]
+        end
+      else
+        return [uri_str, response]
+      end
+    end
+  
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: url_scrubber
 version: !ruby/object:Gem::Version
-  version: 0.7.4
+  version: 0.7.5
   prerelease: 
 platform: ruby
 authors:
@@ -11,7 +11,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-08-28 00:00:00.000000000 Z
+date: 2013-09-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec