http_utilities 1.0.2 → 1.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c8fb30e7ce4a8093f1bc2929fc0d94a68e289d40
-  data.tar.gz: 19cb1986479a7a2ae1957aec32180ecf98c330ac
+  metadata.gz: 1c3ac10c88e4301a9be712c06747987b7bbec37e
+  data.tar.gz: 85a711b2b5cc059682522bc9c03cd09140afe4a6
 SHA512:
-  metadata.gz: 4873c9349bbe2196f1f157ef972e16c0f9241b5e2bf9a67c96c4726b8f4a8a5b4194d207d66198886d4dc7af4ac70f44f152086337fbeba525563d221b138615
-  data.tar.gz: d2853749c566baf145dfde797ee07c983340d4d83142438bd641cffe3e326ddef73aed208e4d956c46d5f1781b4b34e82e8174a4b9a67ec4c0f5c80c76faf1a0
+  metadata.gz: 933e6381c41d5bfd6c0d1e337b936087eaf4e6ea8319caa0ab9878739ecfec1271a596fb551b02d98ada765caa56b5a73dbb8bb3fe337110d09a19bdf8e3a386
+  data.tar.gz: b4101592f6b9c6ced152ca5bf933ca74b51cb54bc7c80f486a095612d246a387d6d4ffd23cad4fbfa40f498861d0c6da49d4d6a2efa9f86b6813e92eb90c1a18

data/Gemfile

@@ -1,8 +1,9 @@
 source "http://rubygems.org"
 
-gem "nokogiri", ">= 1.5.5"
-gem "mechanize", ">= 2.5"
-gem "multi_xml", ">= 0.5"
+gem "nokogiri",   ">= 1.5.5"
+gem "mechanize",  ">= 2.5"
+gem "multi_xml",  ">= 0.5"
+gem "net-ssh",    ">= 2.8"
 
 gem "activerecord-import", :require => false
 

data/http_utilities.gemspec

@@ -3,10 +3,9 @@ Gem::Specification.new do |s|
   s.required_rubygems_version = Gem::Requirement.new(">= 1.3.5") if s.respond_to? :required_rubygems_version=
   
   s.name = "http_utilities"
-  s.version = "1.0.2"
+  s.version = "1.0.5"
 
   s.authors = ["Sebastian Johnsson"]
-  s.date = "2012-11-22"
   s.description = "Wrapper for common Http Libraries (Net:HTTP/Open URI/Curl)"
 
   s.homepage = "http://github.com/Agiley/http_utilities"
@@ -15,6 +14,7 @@ Gem::Specification.new do |s|
   s.add_dependency(%q<nokogiri>, [">= 1.5.5"])
   s.add_dependency(%q<mechanize>, [">= 2.5"])
   s.add_dependency(%q<multi_xml>, [">= 0.5"])
+  s.add_dependency(%q<net-ssh>, [">= 2.8"])
   s.add_dependency(%q<activerecord-import>, [">= 0"])
   
   s.add_development_dependency(%q<rails>, [">= 0"])

data/lib/http_utilities/http/proxy_support.rb

@@ -35,13 +35,15 @@ module HttpUtilities
             self.proxy            =   specific_proxy
 
           elsif (proxy_model_defined?)
-            proxy_object = Proxy.get_random_proxy(self.proxy[:protocol], self.proxy[:type])
+            proxy_object          =   Proxy.get_random_proxy(self.proxy[:protocol], self.proxy[:type])
             
             #log(:info, "[HttpUtilities::Http::ProxySupport] - Randomized Proxy object: #{proxy_object.inspect}")
 
             if (proxy_object)
               self.proxy[:host]   =   proxy_object.host
               self.proxy[:port]   =   proxy_object.port
+              proxy_username      =   proxy_object.username.present? ? proxy_object.username : nil
+              proxy_password      =   proxy_object.password.present? ? proxy_object.password : nil
             end
           end
         end
@@ -52,20 +54,20 @@ module HttpUtilities
       def set_proxy_credentials(proxy_username, proxy_password, proxy_credentials)
         if (self.using_proxy? && (!self.proxy[:username] || !self.proxy[:password]))
           if (proxy_username && proxy_password)
-            self.proxy[:username] = proxy_username
-            self.proxy[:password] = proxy_password
+            self.proxy[:username]       =   proxy_username
+            self.proxy[:password]       =   proxy_password
 
           elsif (proxy_credentials)
             if (proxy_credentials.is_a?(Hash))
-              self.proxy[:username] = proxy_credentials[:username]
-              self.proxy[:password] = proxy_credentials[:password]
+              self.proxy[:username]     =   proxy_credentials[:username]
+              self.proxy[:password]     =   proxy_credentials[:password]
 
             elsif (proxy_credentials.is_a?(String))
               parts = proxy_credentials.split(":")
 
               if (parts && parts.any? && parts.size >= 2)
-                self.proxy[:username] = parts.first
-                self.proxy[:password] = parts.second
+                self.proxy[:username]   =   parts.first
+                self.proxy[:password]   =   parts.second
               end
             end
           end

data/lib/http_utilities/proxies/proxy_checker.rb

@@ -1,9 +1,12 @@
 # -*- encoding : utf-8 -*-
 
+require 'socket'
+require 'net/ssh/proxy/socks5'
+require 'activerecord-import'
+
 module HttpUtilities
   module Proxies
     class ProxyChecker
-      require 'activerecord-import'
       attr_accessor :client, :processed_proxies
       attr_accessor :limit, :minimum_successful_attempts, :maximum_failed_attempts
 
@@ -43,20 +46,52 @@ module HttpUtilities
         end
       end
       
-      def check_proxy(proxy, timeout = 60)
+      def check_proxy(proxy)
+        Rails.logger.info "#{Time.now}: Will check if proxy #{proxy.proxy_address} is working."
+        
+        self.send("check_#{proxy.protocol}_proxy")
+      end
+      
+      def check_socks_proxy(proxy, test_host: "whois.verisign-grs.com", test_port: 43, test_query: "=google.com")
+        valid_proxy     =   false
+
+        begin
+          socks_proxy   =   Net::SSH::Proxy::SOCKS5.new(proxy.host, proxy.port, proxy.socks_proxy_credentials)
+          client        =   socks_proxy.open(test_host, test_port)
+  
+          client.write("#{test_query}\r\n")
+          response      =   client.read
+          
+          valid_proxy   =   (response && response.present?)
+        
+        rescue StandardError => e
+          Rails.logger.error "Exception occured while trying to check proxy #{proxy.proxy_address}. Error Class: #{e.class}. Error Message: #{e.message}"
+          valid_proxy   =   false
+        end
+        
+        if (valid_proxy)
+          Rails.logger.info "#{Time.now}: Proxy #{proxy.proxy_address} is working!"
+        else
+          Rails.logger.info "#{Time.now}: Proxy #{proxy.proxy_address} is not working!"
+        end
+
+        self.processed_proxies << {proxy: proxy, valid: valid_proxy}
+      end
+      
+      def check_http_proxy(proxy, timeout = 60)
         document      =   nil
         valid_proxy   =   false
 
-        options = {:method              =>  :net_http,
-                   :use_proxy           =>  true, 
-                   :proxy               =>  proxy.proxy_address, 
-                   :proxy_protocol      =>  proxy.protocol,
-                   :timeout             =>  timeout,
-                   :maximum_redirects   =>  1,
-                   :disable_auth        =>  true
+        options = {method:             :net_http,
+                   use_proxy:          true, 
+                   proxy:              proxy.proxy_address, 
+                   proxy_protocol:     proxy.protocol,
+                   timeout:            timeout,
+                   maximum_redirects:  1,
+                   disable_auth:       true
                   }
 
-        Rails.logger.info "#{Time.now}: Fetching Proxy #{proxy.proxy_address}."
+        Rails.logger.info "#{Time.now}: Fetching Google.com with proxy #{proxy.proxy_address}."
 
         response            =   self.client.retrieve_parsed_html("http://www.google.com/webhp?hl=en", options)
 
@@ -73,7 +108,7 @@ module HttpUtilities
               Rails.logger.info "Title is: #{title}. Proxy #{proxy.proxy_address}"
               
             rescue Exception => e
-              Rails.logger.error "Exception occured while trying to validate proxy. Error Class: #{e.class}. Error Message: #{e.message}"
+              Rails.logger.error "Exception occured while trying to check proxy #{proxy.proxy_address}. Error Class: #{e.class}. Error Message: #{e.message}"
               valid_proxy   =   false
             end
           end
@@ -85,10 +120,10 @@ module HttpUtilities
           Rails.logger.info "#{Time.now}: Proxy #{proxy.proxy_address} is not working!"
         end
 
-        self.processed_proxies << {:proxy => proxy, :valid => valid_proxy}
+        self.processed_proxies << {proxy: proxy, valid: valid_proxy}
       end
 
-      def update_proxies()
+      def update_proxies
         columns   =   [:host, :port, :last_checked_at, :valid_proxy, :successful_attempts, :failed_attempts]
         values    =   []
 

data/lib/http_utilities/proxies/proxy_module.rb

@@ -63,6 +63,18 @@ module HttpUtilities
         def proxy_credentials
           return ::Proxy.format_proxy_credentials(self.username, self.password)
         end
+        
+        def socks_proxy_credentials
+          credentials     =   {}
+          
+          if (!self.username.empty? && !self.password.empty?)
+            credentials   =   {user: self.username, password: self.password}
+          elsif (!self.username.empty? && self.password.empty?)
+            credentials   =   {user: self.username}
+          end
+          
+          return credentials
+        end
       end
       
     end

data/lib/http_utilities.rb

@@ -1,6 +1,6 @@
 # -*- encoding : utf-8 -*-
 module HttpUtilities
-  VERSION = "1.0.2"
+  VERSION = "1.0.5"
   
   require File.join(File.dirname(__FILE__), 'http_utilities/railtie') if defined?(Rails)
 

metadata

@@ -1,125 +1,139 @@
 --- !ruby/object:Gem::Specification
 name: http_utilities
 version: !ruby/object:Gem::Version
-  version: 1.0.2
+  version: 1.0.5
 platform: ruby
 authors:
 - Sebastian Johnsson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-11-22 00:00:00.000000000 Z
+date: 2014-06-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.5.5
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.5.5
 - !ruby/object:Gem::Dependency
   name: mechanize
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '2.5'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '2.5'
 - !ruby/object:Gem::Dependency
   name: multi_xml
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0.5'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0.5'
+- !ruby/object:Gem::Dependency
+  name: net-ssh
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.8'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.8'
 - !ruby/object:Gem::Dependency
   name: activerecord-import
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rails
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: sqlite3
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: mysql2
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.3.11
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.3.11
 description: Wrapper for common Http Libraries (Net:HTTP/Open URI/Curl)
@@ -184,17 +198,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: 1.3.5
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.3
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 2
 summary: Wrapper for common Http Libraries (Net:HTTP/Open URI/Curl)