url_scrubber 0.8.0 → 0.8.2

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    YjU1ZDhmYTczNWEzMDk2MjQ4M2U2NDkzMGNiZmQ1Y2U2YTMxZDk0MQ==
+    ZDlkYzlmMDBiOTY5MzU0N2ExYTZjMjI4YzUyZDQ0MzFiZWNiYmM1NA==
   data.tar.gz: !binary |-
-    NDE2YzlhOWFjMmU3NWFiNjIzNmE4OTJlYjQ3ZmYwMDExMTJjZGNmMw==
+    Y2NiYzUzMDAwZWRlZjA3MTYzZmM5ZTI2MjkyNDI2ZmQ3ZjRhMzcyMg==
 SHA512:
   metadata.gz: !binary |-
-    MDRlZTJlNzFiOTMwYWFkZWY4YzZiZWM5ZjAxMWVhMzMwZjNmYzg4ZmZkMjc2
-    NGRmZTc0Nzc0NGU4M2MzNTk2NGQ5YzhkODRhYTk1YTU3ZWE5YmY1MDMyMTQ4
-    ZjFiNGI4MTAyMmM0MzQ1Y2Y5NWMyMTYwMmQyZTQyNTE1ZTQ4ZDQ=
+    NTIyOGY2ZTc3YTNiMDQwMmE5NzNiZDhlOWMxMTcyYThhZDBmMTBhNDAxNTc5
+    MTIzM2RiMzVmM2E4ZTJmMDBlNjdjNmM2MDJiNWZhMWJjZjM0MDE5NjNlNTQ3
+    ZmFiYWYwYjJiNDRlMDg0MWUxMTdhNmY3MjdjYmQ2ZjI0YWFlNmQ=
   data.tar.gz: !binary |-
-    MTQ4MDcxMDFlM2QzM2Q2ODAwMWUxOGRhMzg2OTBmMjhiNTEzNmNmYTY5ZDg4
-    NjM4YjhmOGNkMTJmNzQ1ZGQxMDI3ODgxNDc0NGNiN2QyNmExMTk5NzRjOGUy
-    NWMzOWRjNTMwM2YxZjFjYjUzYmMyYjU1YjVkNGNjOWFiMTRkOTE=
+    YTQwYmIwMGYzOGU2ZGZjOTA1NGE3MTNkNDk4NGMzNDkyOTBlMmM3MWFjYTRi
+    ODdhMDhjNmM0ZTIwYWY3MzZkY2UyMjQ2N2I4YThjNDQ1OGFkMDFlODU1OTU5
+    NTY2YzQxNzk0ZTk0MTE5ZTdlZGY2MzNmY2VlNTg4NTA5NTA3ZmY=

data/lib/url_scrubber/version.rb

@@ -1,3 +1,3 @@
 module UrlScrubber
-  VERSION = "0.8.0"
+  VERSION = "0.8.2"
 end

data/lib/url_scrubber.rb

@@ -153,6 +153,7 @@ module UrlScrubber
   end
 
   def self.special_cases(url)
+    #puts "special_cases"
     case service_of(url)
     when :youtube   then return sc_youtube(url)
     when :twitter   then return sc_twitter(url)
@@ -191,6 +192,7 @@ module UrlScrubber
   end
 
   def self.drop_anchor!(url)
+    #puts "drop anchor"
     url.sub!(/#.*$/, '')
     url
   end
@@ -232,7 +234,7 @@ module UrlScrubber
   end
 
   def self.sc_facebook(url)
-    # puts "sc_facebook: #{url}"
+    #puts "sc_facebook: #{url}"
     regex1 = /^(?<url>(https?:\/\/)(www\.)?facebook\.com\/(pages\/)*(?<uname>.*)[\/-](?<uid>[0-9]+))($|\/|\/(about|timeline|info|app_)?)/i
     regex2 = /^(?<url>(https?:\/\/)(www\.)?facebook\.com\/profile.php\?id=(?<uid>[0-9]+))($|\/|\/.*|&.*)/i
     regex2a = /^(?<url>(https?:\/\/)(www\.)?facebook\.com\/profile.php\?_rdr=p&id=(?<uid>[0-9]+))($|\/|\/.*|&.*)/i
@@ -253,13 +255,13 @@ module UrlScrubber
     elsif mdata = url.match(regex2)
       # puts "regex2"
       # "https://www.facebook.com/profile.php?id=100009574328879"
-      url, http_response = check_for_redirection(mdata[:url])
+      url, http_response = check_for_facebook_redirection(mdata[:url])
       uid = mdata[:uid]
     elsif mdata = url.match(regex2a)
       # puts "regex2a"
       # "https://www.facebook.com/profile.php?_rdr=p&id=100009574328879"
       url = "http://facebook.com/profile.php?id=" + mdata[:uid]
-      url, http_response = check_for_redirection(url)
+      url, http_response = check_for_facebook_redirection(url)
       uid = mdata[:uid]
     elsif mdata = url.match(regex4)
       # puts "#{url} - #{mdata[:uname]}"
@@ -278,7 +280,7 @@ module UrlScrubber
       # puts "profile.php"
       # these were being truncated, they do redirect, but typically a 301 response is generated 
       # so the url is returned unchanged.  Better than truncation.
-      url, http_response = check_for_redirection(url)
+      url, http_response = check_for_facebook_redirection(url)
     else
       # puts "else"
       url = drop_url_query!(url)
@@ -355,7 +357,9 @@ module UrlScrubber
     url
   end
   
-  def self.check_for_redirection(uri_str, limit = 5)
+  def self.check_for_facebook_redirection(uri_str, limit = 5)
+    #puts "check_for_facebook_redirection called! uri=#{uri_str}, limit=#{limit.to_s}"
+    # finds any redirects   intended for facebook URLs only!!!!
     login_patterns = [
       # pages that require user logins
       %r{^.*/login[^/]*$}
@@ -367,14 +371,20 @@ module UrlScrubber
     ]
 
     raise 'Too many HTTP redirects' if limit == 0
+
+    uri_str_new = uri_str.sub('http://', 'https://')
+    uri_str_new = uri_str_new.sub('https://', 'https://www.') if !uri_str_new.include?("https://www.")
     
     begin
-      url = URI.parse(uri_str)        
+      url = URI.parse(uri_str_new)        
     rescue URI::InvalidURIError => e
-      return [uri_str, CustomError.new(786, "Invalid URI #{uri_str} : #{e.message}") ]
+      return [uri_str_new, CustomError.new(786, "Invalid URI #{uri_str_new} : #{e.message}") ]
     end
 
     http = Net::HTTP.new(url.host, url.port)
+    http = Net::HTTP.new(url.host, url.port)
+    http.open_timeout = 7 # only wait up to 7 seconds for a the connection to be established
+    http.read_timeout = 10 # and up to 10 seconds for a response 
     if url.port == 443
       http.use_ssl = true
       http.verify_mode = OpenSSL::SSL::VERIFY_NONE
@@ -385,35 +395,43 @@ module UrlScrubber
 
     begin
       response = http.request(request)    
+    rescue Timeout::Error
+      #Rails.logger.error("UrlScrubber.check_for_facebook_redirection - http.request Timeout, URL=#{uri_str_new}")
+      failure_response = Net::HTTPClientError.new('1.1', '400', 'Unreachable')
+      return [uri_str_new, failure_response]
     rescue Exception => e
       failure_response = Net::HTTPClientError.new('1.1', '404', 'Not Found')
-      return [uri_str, failure_response]
-      end
+      return [uri_str_new, failure_response]
+    end
 
     if response.is_a? Net::HTTPRedirection
       if response['location'][0,4] == "http"
         if failure_patterns.any? { |pattern| response['location'].match(pattern) }
           # got redirected to a page indicating failure, so act like it's a 404
           failure_response = Net::HTTPClientError.new('1.1', '404', 'Not Found')
-          return [uri_str, failure_response]
+          #puts "check_for_facebook_redirection 404"
+          return [uri_str_new, failure_response]
         end
 
-        redirected_url, base_response = check_for_redirection(response['location'], limit - 1)
-
         if login_patterns.any? { |pattern| redirected_url.match(pattern) }
           # got redirected to a login page. return the ultimate response, but the previous url
-          return [uri_str, base_response]
-        else
-          return [redirected_url, base_response]
+          failure_response = Net::HTTPClientError.new('1.1', '401', 'Inaccessible')
+          #puts "check_for_facebook_redirection 401"
+          return [uri_str_new, failure_response]
         end
+        #puts "check_for_facebook_redirection 1 limit=#{limit.to_s}"
+        redirected_url, base_response = check_for_facebook_redirection(response['location'], limit - 1)
+        return [redirected_url, base_response]
 
       else
         redir_url = "http://#{url.host}#{response['location']}"
-        redirected_url, base_response = check_for_redirection(redir_url, limit - 1)
+        #puts "check_for_facebook_redirection recalled limit =#{limit.to_s}"
+        redirected_url, base_response = check_for_facebook_redirection(redir_url, limit - 1)
         return [redirected_url, base_response]
       end
     else
-      return [uri_str, response]
+      #puts "check_for_facebook_redirection return code #{response.code.to_s}"
+      return [uri_str_new, response]
     end
   end
   

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: url_scrubber
 version: !ruby/object:Gem::Version
-  version: 0.8.0
+  version: 0.8.2
 platform: ruby
 authors:
 - Colin Langton
@@ -11,7 +11,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-04-08 00:00:00.000000000 Z
+date: 2016-09-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -124,7 +124,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.8
+rubygems_version: 2.6.6
 signing_key: 
 specification_version: 4
 summary: Clean up URLs.