http_utilities 1.1.2 → 1.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1a18294693d9cd95171156f4fccdf461a8c45127
-  data.tar.gz: 6d32ede9f59c390b64ebcfcc47ec00901fe6e77f
+  metadata.gz: 08cfb5e367ca9f757c3eb75ff8958a55af204bf6
+  data.tar.gz: ec280ce2d9e5bc1f0e1ad7fbd724a6abb438409c
 SHA512:
-  metadata.gz: 51c959c41979b8c1a253c1b5d43e019ac19e692b9fbb7f92b04da37be639099949bd540e2ed2b068f30d9943a2a3f46212401bc416e4a1a44730d1758e553f09
-  data.tar.gz: 0fe6a191e9f0e802622d5eb24a2529e0b6c9d1c1cdc1525347570492c1d27aae74baa15e5650566dbc69701397e58d9df47ea4fb8e105002aeea8b69ca98c930
+  metadata.gz: 0395ba295966ce1622021940273827ff880d691c78c3f8ed71f45edeebf418932e602bf9219e2d3ad6b245e3830de4a59aac73d363b394c9c73a631bab8f531d
+  data.tar.gz: 057783241fe6465bff927446dc03c31c10faa3a20784a34caff5edd6bdebc4617c6cb584d15ad0783d4c4babc4c4e2e39efa8566e0a6b83e90e08e2bc1613f81

data/Gemfile

@@ -1,16 +1,11 @@
 source "http://rubygems.org"
 
-gem "nokogiri",   ">= 1.5.5"
-gem "mechanize",  ">= 2.5"
-gem "multi_xml",  ">= 0.5"
-gem "net-ssh",    ">= 2.8"
+gem 'nokogiri'
+gem 'mechanize'
+gem 'net-ssh'
 
 gem "activerecord-import", :require => false
 
-platforms :ruby do
-  gem 'curb'
-end
-
 group :development, :test do
   gem 'rails'
   gem 'jeweler'
@@ -18,6 +13,6 @@ group :development, :test do
   gem 'sqlite3'
   
   platforms :ruby do
-    gem "mysql2", ">= 0.3.11"
+    gem "mysql2"
   end
 end

data/http_utilities.gemspec

@@ -3,24 +3,23 @@ Gem::Specification.new do |s|
   s.required_rubygems_version = Gem::Requirement.new(">= 1.3.5") if s.respond_to? :required_rubygems_version=
   
   s.name = "http_utilities"
-  s.version = "1.1.2"
+  s.version = "1.1.5"
 
   s.authors = ["Sebastian Johnsson"]
-  s.description = "Wrapper for common Http Libraries (Net:HTTP/Open URI/Curl)"
+  s.description = "Wrapper for Faraday with additional functionality"
 
   s.homepage = "http://github.com/Agiley/http_utilities"
-  s.summary = "Wrapper for common Http Libraries (Net:HTTP/Open URI/Curl)"
+  s.summary = "Wrapper for Faraday with additional functionality"
   
-  s.add_dependency(%q<nokogiri>, [">= 1.5.5"])
-  s.add_dependency(%q<mechanize>, [">= 2.5"])
-  s.add_dependency(%q<multi_xml>, [">= 0.5"])
-  s.add_dependency(%q<net-ssh>, [">= 2.8"])
-  s.add_dependency(%q<activerecord-import>, [">= 0"])
+  s.add_dependency(%q<nokogiri>, [">= 1.6"])
+  s.add_dependency(%q<mechanize>, [">= 2.7"])
+  s.add_dependency(%q<net-ssh>, [">= 2.9"])
   
   s.add_development_dependency(%q<rails>, [">= 0"])
   s.add_development_dependency(%q<rspec>, [">= 0"])
   s.add_development_dependency(%q<sqlite3>, [">= 0"])
-  s.add_development_dependency(%q<mysql2>, [">= 0.3.11"])
+  s.add_development_dependency(%q<mysql2>, [">= 0"])
+  s.add_development_dependency(%q<activerecord-import>, [">= 0"])
   
   # = MANIFEST =
  s.files = %w[
@@ -38,16 +37,9 @@ Gem::Specification.new do |s|
  lib/generators/templates/http_utilities.rb
  lib/generators/templates/user_agents.yml
  lib/http_utilities.rb
- lib/http_utilities/http/adapters/curb.rb
- lib/http_utilities/http/adapters/net_http.rb
- lib/http_utilities/http/adapters/open_uri.rb
  lib/http_utilities/http/client.rb
- lib/http_utilities/http/cookies.rb
- lib/http_utilities/http/format.rb
- lib/http_utilities/http/get.rb
  lib/http_utilities/http/logger.rb
  lib/http_utilities/http/mechanize/client.rb
- lib/http_utilities/http/post.rb
  lib/http_utilities/http/proxy_support.rb
  lib/http_utilities/http/request.rb
  lib/http_utilities/http/response.rb
@@ -57,8 +49,9 @@ Gem::Specification.new do |s|
  lib/http_utilities/jobs/resque/proxies/check_proxy_job.rb
  lib/http_utilities/jobs/sidekiq/proxies/check_proxies_job.rb
  lib/http_utilities/jobs/sidekiq/proxies/check_proxy_job.rb
+ lib/http_utilities/proxies/mongo/proxy_module.rb
+ lib/http_utilities/proxies/mysql/proxy_module.rb
  lib/http_utilities/proxies/proxy_checker.rb
- lib/http_utilities/proxies/proxy_module.rb
  lib/http_utilities/proxies/proxy_seeder.rb
  lib/http_utilities/railtie.rb
  lib/tasks/http_utilities_tasks.rake

data/lib/http_utilities/http/client.rb

@@ -1,21 +1,75 @@
 # -*- encoding : utf-8 -*-
 require 'open-uri'
-require 'net/http'
 require 'uri'
 require 'cgi'
 
 module HttpUtilities
   module Http
     class Client
-      include HttpUtilities::Http::Cookies
-      include HttpUtilities::Http::Url
-      include HttpUtilities::Http::Get
-      include HttpUtilities::Http::Post
       include HttpUtilities::Http::Logger
+      
+      def get(url, arguments: {}, options: {}, retries: 3)
+        response        =   nil
+        request         =   build_request(options)
+        
+        begin
+          response      =   request.interface.get(url, arguments)
+          response      =   HttpUtilities::Http::Response.new(response, request, options)
+    
+        rescue Faraday::TimeoutError, Net::ReadTimeout, Timeout::Error, StandardError => e
+          log(:error, "[HttpUtilities::Http::Client] - An error occurred while trying to fetch the response. Error Class: #{e.class.name}. Error Message: #{e.message}.")
+          retries      -=   1
+          retry if retries > 0
+        end
 
-      include HttpUtilities::Http::Adapters::NetHttp
-      include HttpUtilities::Http::Adapters::OpenUri
-      include HttpUtilities::Http::Adapters::Curb
+        return response
+      end
+      
+      def post(url, data: nil, options: {}, retries: 3)
+        response        =   nil
+        request         =   build_request(options)
+    
+        begin
+          response      =   request.interface.post(url, data)
+          response      =   HttpUtilities::Http::Response.new(response, request, options)
+    
+        rescue Faraday::TimeoutError, Net::ReadTimeout, Timeout::Error, StandardError => e
+          log(:error, "[HttpUtilities::Http::Client] - An error occurred while trying to fetch the response. Error Class: #{e.class.name}. Error Message: #{e.message}.")
+          retries          -=   1
+          retry if retries > 0
+        end
+
+        return response
+      end
+      
+      private
+      def build_request(options = {}, faraday_options = {})
+        options         =   options.dup
+        options         =   options.merge(ssl: {:verify => false})
+        
+        adapter         =   options.delete(:adapter)                { |opt| Faraday.default_adapter }
+        timeout         =   options.delete(:timeout)                { |opt| 60 }
+        open_timeout    =   options.delete(:open_timeout)           { |opt| 60 }
+        
+        request                             =   HttpUtilities::Http::Request.new
+        request.set_proxy_options(options)
+        
+        proxy_options                       =   request.generate_proxy_options
+    
+        connection      =   Faraday.new(faraday_options) do |builder|
+          builder.headers[:user_agent]      =   request.user_agent
+          builder.options[:timeout]         =   timeout
+          builder.options[:open_timeout]    =   open_timeout        
+          #builder.response  :logger
+          builder.proxy     proxy_options unless proxy_options.empty?
+          builder.adapter   adapter
+        end
+
+        request.interface                   =   connection
+        
+        return request
+      end
+      
     end
   end
 end

data/lib/http_utilities/http/mechanize/client.rb

@@ -30,7 +30,11 @@ module HttpUtilities
           self.agent.log            =   ::Logger.new(logger) if (verbose)
           
           self.set_proxy_options(options)
-          self.agent.set_proxy(self.proxy[:host], self.proxy[:port], self.proxy[:username], self.proxy[:password]) if (self.proxy[:host] && self.proxy[:port])
+          
+          if (self.proxy[:host] && self.proxy[:port])
+            log(:info, "[HttpUtilities::Http::Mechanize::Client] - Will use proxy #{self.proxy[:host]}:#{self.proxy[:port]} for Mechanize.")
+            self.agent.set_proxy(self.proxy[:host], self.proxy[:port], self.proxy[:username], self.proxy[:password])
+          end
           
           self.set_user_agent
           (self.user_agent) ? self.agent.user_agent = self.user_agent : self.agent.user_agent_alias = 'Mac Safari'

data/lib/http_utilities/http/proxy_support.rb

@@ -84,6 +84,16 @@ module HttpUtilities
       def using_proxy?
         return (self.proxy[:host] && self.proxy[:port] && self.proxy[:port] > 0)
       end
+      
+      def generate_proxy_options
+        proxy_options             =   {}
+        
+        proxy_options[:uri]       =   "http://#{self.proxy[:host]}:#{self.proxy[:port]}"
+        proxy_options[:user]      =   self.proxy[:username] if self.proxy[:username] && self.proxy[:username].present?
+        proxy_options[:password]  =   self.proxy[:password] if self.proxy[:password] && self.proxy[:password].present?
+        
+        return proxy_options
+      end
 
     end
   end

data/lib/http_utilities/http/request.rb

@@ -5,12 +5,11 @@ module HttpUtilities
       include HttpUtilities::Http::ProxySupport
       include HttpUtilities::Http::UserAgent
       
-      attr_accessor :interface, :proxy, :cookies, :user_agent
+      attr_accessor :interface, :proxy, :user_agent
       
-      def initialize(interface = nil, proxy = {}, cookies = [])
+      def initialize(interface = nil, proxy = {})
         self.interface  =   interface
         self.proxy      =   proxy
-        self.cookies    =   cookies
         
         self.set_user_agent
       end

data/lib/http_utilities/http/response.rb

@@ -1,15 +1,14 @@
 module HttpUtilities
   module Http
     class Response
-      include HttpUtilities::Http::Format
       include HttpUtilities::Http::Logger
 
       attr_accessor :body, :parsed_body, :page, :format, :request, :force_encoding
 
-      def initialize(body = nil, request = nil, options = {})
-        options               =   options.clone()
+      def initialize(response = nil, request = nil, options = {})
+        options               =   options.dup
 
-        self.body             =   body
+        self.body             =   (response && response.body) ? response.body : nil
         self.request          =   request
 
         self.parsed_body      =   nil
@@ -34,6 +33,18 @@ module HttpUtilities
       def parse_response
         self.send("as_#{self.format}".to_sym) if (self.body && self.format)
       end
+      
+      def as_html
+        self.parsed_body = (self.body && self.body != "") ? Nokogiri::HTML(self.body.to_s.force_encoding("utf-8"), nil, "utf-8") : nil
+      end
+
+      def as_xml
+        self.parsed_body = (self.body && self.body != "") ? Nokogiri::XML(self.body.to_s.force_encoding("utf-8"), nil, "utf-8") : nil
+      end
+
+      def as_json
+        self.parsed_body = (self.body && self.body != "") ? self.body.to_s.force_encoding("utf-8").to_json : nil
+      end
 
       def set_page(page)
         self.page = page

data/lib/http_utilities/http/user_agent.rb

@@ -5,7 +5,7 @@ module HttpUtilities
     module UserAgent
       
       def set_user_agent
-        user_agent        =   (USER_AGENTS && USER_AGENTS.any?) ? USER_AGENTS[rand(USER_AGENTS.size)] : ""
+        user_agent        =   (USER_AGENTS && USER_AGENTS.any?) ? USER_AGENTS.sample : nil
         self.user_agent   =   user_agent if (user_agent && self.respond_to?(:user_agent=))
       end
       

data/lib/http_utilities/proxies/mongo/proxy_module.rb

@@ -0,0 +1,97 @@
+module HttpUtilities
+  module Proxies
+    module Mongo
+      
+      module ProxyModule
+      
+        def self.included(base)
+          base.send :extend, ClassMethods
+          base.send :include, InstanceMethods
+        end
+
+        module ClassMethods
+          def should_be_checked(protocol: :all, proxy_type: :all, date: Time.now, limit: 10, maximum_failed_attempts: 10)
+            proxies     =   get_proxies_for_protocol_and_proxy_type(protocol, proxy_type)
+    
+            proxies     =   proxies.any_of(
+              {:last_checked_at.exists => false},
+              {:last_checked_at.ne => nil},
+              {:last_checked_at.exists => true, :last_checked_at.ne => nil, :last_checked_at.lt => date}
+            )
+    
+            proxies     =   proxies.any_of(
+              {:failed_attempts.exists => false},
+              {:failed_attempts.in => ["", nil]},
+              {:failed_attempts.exists => true, :failed_attempts.nin => ["", nil], :failed_attempts.lte => maximum_failed_attempts}
+            )
+    
+            proxies     =   proxies.order_by([[:valid_proxy, :asc], [:failed_attempts, :asc], [:last_checked_at, :asc]])
+            proxies     =   proxies.limit(limit)
+    
+            return proxies
+          end
+        
+          def get_random_proxy(protocol: :all, proxy_type: :all, maximum_failed_attempts: nil, retries: 3)
+            proxies     =   get_proxies_for_protocol_and_proxy_type(protocol, proxy_type)
+            proxies     =   proxies.where(valid_proxy: true)
+            proxies     =   proxies.where(:failed_attempts.lte => maximum_failed_attempts) if maximum_failed_attempts
+            proxy       =   nil
+            
+            begin
+              proxy     =   proxies.skip(rand(proxies.count)).first
+            
+            rescue StandardError
+              retries  -=   1
+              retry if retries > 0
+            end
+            
+            return proxy
+          end
+        
+          def get_proxies_for_protocol_and_proxy_type(protocol, proxy_type)
+            proxies     =   ::Proxy.where(nil)
+            proxies     =   proxies.where(protocol: protocol)     if (protocol && !protocol.downcase.to_sym.eql?(:all))
+            proxies     =   proxies.where(proxy_type: proxy_type) if (proxy_type && !proxy_type.downcase.to_sym.eql?(:all))
+    
+            return proxies
+          end
+
+          def format_proxy_address(proxy_host, proxy_port = 80, include_http = false)
+            proxy_address = "#{proxy_host}:#{proxy_port}"
+            proxy_address.insert(0, "http://") if (include_http && !proxy_address.start_with?("http://"))
+    
+            return proxy_address
+          end
+
+          def format_proxy_credentials(username, password)
+            return "#{username}:#{password}"
+          end
+        end
+
+        module InstanceMethods
+          def proxy_address(include_http = false)
+            return ::Proxy.format_proxy_address(self.host, self.port, include_http)
+          end
+        
+          def proxy_credentials
+            return ::Proxy.format_proxy_credentials(self.username, self.password)
+          end
+        
+          def socks_proxy_credentials
+            credentials     =   {}
+    
+            if (!self.username.empty? && !self.password.empty?)
+              credentials   =   {user: self.username, password: self.password}
+            elsif (!self.username.empty? && self.password.empty?)
+              credentials   =   {user: self.username}
+            end
+    
+            return credentials
+          end
+        end
+      
+      end
+      
+    end
+  end
+end

data/lib/http_utilities/proxies/mysql/proxy_module.rb

@@ -0,0 +1,93 @@
+module HttpUtilities
+  module Proxies
+    module Mysql
+      
+      module ProxyModule
+      
+        def self.included(base)
+          base.send :extend, ClassMethods
+          base.send :include, InstanceMethods
+        end
+
+        module ClassMethods
+          def should_be_checked(protocol: :all, proxy_type: :all, date: Time.now, limit: 10, maximum_failed_attempts: 10)
+            proxies     =   get_proxies_for_protocol_and_proxy_type(protocol, proxy_type)
+            proxies     =   proxies.where(["(last_checked_at IS NULL OR last_checked_at < ?)", date])
+            proxies     =   proxies.where(["failed_attempts <= ?", maximum_failed_attempts])
+            proxies     =   proxies.order("valid_proxy ASC, failed_attempts ASC, last_checked_at ASC")
+            proxies     =   proxies.limit(limit)
+          
+            return proxies
+          end
+        
+          def get_random_proxy(protocol: :all, proxy_type: :all, maximum_failed_attempts: nil)
+            proxies     =   get_proxies_for_protocol_and_proxy_type(protocol, proxy_type)
+            proxies     =   proxies.where(["valid_proxy = ? AND last_checked_at IS NOT NULL", true])
+            proxies     =   proxies.where(["failed_attempts <= ?", maximum_failed_attempts]) if maximum_failed_attempts
+          
+            order_clause = case ActiveRecord::Base.connection.class.name
+              when "ActiveRecord::ConnectionAdapters::MysqlAdapter", "ActiveRecord::ConnectionAdapters::Mysql2Adapter"
+                "RAND() DESC"
+              when "ActiveRecord::ConnectionAdapters::SQLite3Adapter"
+                "RANDOM() DESC"
+              else
+                "RAND() DESC"
+            end
+          
+            proxies     =   proxies.order(order_clause)
+          
+            proxy       =   nil
+          
+            uncached do
+              proxy     =   proxies.limit(1).first
+            end
+
+            return proxy
+          end
+        
+          def get_proxies_for_protocol_and_proxy_type(protocol, proxy_type)
+            proxies     =   ::Proxy.where(nil)
+            proxies     =   proxies.where(protocol: protocol)     if (protocol && !protocol.downcase.to_sym.eql?(:all))
+            proxies     =   proxies.where(proxy_type: proxy_type) if (proxy_type && !proxy_type.downcase.to_sym.eql?(:all))
+          
+            return proxies
+          end
+
+          def format_proxy_address(proxy_host, proxy_port = 80, include_http = false)
+            proxy_address = "#{proxy_host}:#{proxy_port}"
+            proxy_address.insert(0, "http://") if (include_http && !proxy_address.start_with?("http://"))
+            return proxy_address
+          end
+
+          def format_proxy_credentials(username, password)
+            return "#{username}:#{password}"
+          end
+        end
+
+        module InstanceMethods
+          def proxy_address(include_http = false)
+            return ::Proxy.format_proxy_address(self.host, self.port, include_http)
+          end
+        
+          def proxy_credentials
+            return ::Proxy.format_proxy_credentials(self.username, self.password)
+          end
+        
+          def socks_proxy_credentials
+            credentials     =   {}
+          
+            if (!self.username.empty? && !self.password.empty?)
+              credentials   =   {user: self.username, password: self.password}
+            elsif (!self.username.empty? && self.password.empty?)
+              credentials   =   {user: self.username}
+            end
+          
+            return credentials
+          end
+        end
+      
+      end
+      
+    end
+  end
+end

data/lib/http_utilities/proxies/proxy_checker.rb

@@ -2,7 +2,6 @@
 
 require 'socket'
 require 'net/ssh/proxy/socks5'
-require 'activerecord-import'
 
 module HttpUtilities
   module Proxies
@@ -11,9 +10,8 @@ module HttpUtilities
       attr_accessor :limit, :minimum_successful_attempts, :maximum_failed_attempts
 
       def initialize
-        self.client                       =   HttpUtilities::Http::Mechanize::Client.new(verbose: false)
-        self.client.agent.agent.http.verify_mode = OpenSSL::SSL::VERIFY_NONE
-        
+        self.client                       =   HttpUtilities::Http::Client.new
+
         self.processed_proxies            =   []
         
         self.limit                        =   1000
@@ -23,7 +21,6 @@ module HttpUtilities
 
       def check_and_update_proxies(protocol: :all, proxy_type: :all, mode: :synchronous, maximum_failed_attempts: self.maximum_failed_attempts)
         check_proxies(protocol: protocol, proxy_type: proxy_type, mode: mode, maximum_failed_attempts: maximum_failed_attempts)
-        update_proxies
       end
 
       def check_proxies(protocol: :all, proxy_type: :all, mode: :synchronous, maximum_failed_attempts: self.maximum_failed_attempts)
@@ -77,86 +74,57 @@ module HttpUtilities
           valid_proxy   =   false
         end
         
-        if (valid_proxy)
-          Rails.logger.info "#{Time.now}: Proxy #{proxy.proxy_address} is working!"
-        else
-          Rails.logger.info "#{Time.now}: Proxy #{proxy.proxy_address} is not working!"
-        end
-
-        self.processed_proxies << {proxy: proxy, valid: valid_proxy}
+        update_proxy(proxy, valid_proxy)
       end
       
-      def check_http_proxy(proxy, timeout = 60)
-        document      =   nil
-        valid_proxy   =   false
-
+      def check_http_proxy(proxy, test_url: "http://www.google.com/robots.txt", timeout: 10)
         options       =   {
                             use_proxy:       true,
-                            proxy:           proxy.proxy_address, 
+                            proxy:           {host: proxy.host, port: proxy.port}, 
                             proxy_protocol:  proxy.protocol,
-                            timeout:         timeout,
+                            timeout:         timeout
                           }
-
-        Rails.logger.info "#{Time.now}: Fetching Google.com with proxy #{proxy.proxy_address}."
         
-        page                =   self.client.get_page("https://www.google.com/webhp?hl=en&gws_rd=ssl", options)
-        
-        if (page)
-          parser            =   self.client.get_parser(page)
-          title             =   parser.at_css("head title")
-
-          if (title && title.content)
-            begin
-              title         =   title.content.encode("UTF-8").strip.downcase
-              body_content  =   page.body.to_s.encode("UTF-8").strip.downcase
-              
-              valid_proxy   =   (title.eql?("google") || !(body_content =~ /google home/i).nil?)
-              
-              Rails.logger.info "Title is: #{title}. Proxy #{proxy.proxy_address}"
-              
-            rescue Exception => e
-              Rails.logger.error "Exception occured while trying to check proxy #{proxy.proxy_address}. Error Class: #{e.class}. Error Message: #{e.message}"
-              valid_proxy   =   false
-            end
-          end
-        end
+        options.merge!(proxy_username: proxy.username) if proxy.username && proxy.username.present?
+        options.merge!(proxy_password: proxy.password) if proxy.password && proxy.password.present?
 
-        if (valid_proxy)
-          Rails.logger.info "#{Time.now}: Proxy #{proxy.proxy_address} is working!"
-        else
-          Rails.logger.info "#{Time.now}: Proxy #{proxy.proxy_address} is not working!"
-        end
+        Rails.logger.info "#{Time.now}: Fetching robots.txt for Google.com with proxy #{proxy.proxy_address}. Using authentication? #{options.has_key?(:proxy_username).to_s}"
+        
+        response       =   self.client.get(test_url, options: options)
+        valid_proxy    =   (response && response.body && response.body =~ /Allow: \/search\/about/i)
 
-        self.processed_proxies << {proxy: proxy, valid: valid_proxy}
+        update_proxy(proxy, valid_proxy)
       end
-
+      
       def update_proxies
-        columns   =   [:host, :port, :last_checked_at, :valid_proxy, :successful_attempts, :failed_attempts]
-        values    =   []
-
         Rails.logger.info "Updating/Importing #{self.processed_proxies.size} proxies"
 
         if (self.processed_proxies && self.processed_proxies.any?)
           self.processed_proxies.each do |value|
-            proxy                 =   value[:proxy]
-            valid                 =   value[:valid]
-            successful_attempts   =   proxy.successful_attempts
-            failed_attempts       =   proxy.failed_attempts
-
-            if (valid)
-              successful_attempts +=  1
-            else
-              failed_attempts     +=  1
-            end
-
-            is_valid              =   (successful_attempts >= self.minimum_successful_attempts && failed_attempts < self.maximum_failed_attempts)
-            value_arr             =   [proxy.host, proxy.port, Time.now, is_valid, successful_attempts, failed_attempts]
-            values                <<  value_arr
+            update_proxy(value[:proxy], value[:valid])
           end
+        end
+      end
+      
+      def update_proxy(proxy, valid)
+        Rails.logger.info "#{Time.now}: Proxy #{proxy.proxy_address} is #{valid ? "working" : "not working"}!"
+        
+        successful_attempts         =   proxy.successful_attempts || 0
+        failed_attempts             =   proxy.failed_attempts || 0
 
-          ::Proxy.import(columns, values, :on_duplicate_key_update => [:last_checked_at, :valid_proxy, :successful_attempts, :failed_attempts], :validate => false)
+        if (valid)
+          successful_attempts      +=  1
+        else
+          failed_attempts          +=  1
         end
 
+        is_valid                    =   (successful_attempts >= self.minimum_successful_attempts && failed_attempts < self.maximum_failed_attempts)
+        
+        proxy.valid_proxy           =   is_valid
+        proxy.successful_attempts   =   successful_attempts
+        proxy.failed_attempts       =   failed_attempts
+        proxy.last_checked_at       =   Time.now
+        proxy.save
       end
 
     end

data/lib/http_utilities/proxies/proxy_seeder.rb

@@ -1,7 +1,6 @@
 module HttpUtilities
   module Proxies
     class ProxySeeder
-      require 'activerecord-import'
       attr_accessor :protocols, :proxy_types, :categories
 
       def initialize
@@ -27,20 +26,23 @@ module HttpUtilities
       end
 
       def bulk_import_proxies(proxy_list, protocol, proxy_type, category)        
-        columns = [:host, :port, :protocol, :proxy_type, :category]
-        category = (category && !category.eql?('unspecified')) ? category : nil
+        columns     =   [:host, :port, :protocol, :proxy_type, :category]
+        category    =   (category && !category.eql?('unspecified')) ? category : nil
 
         begin
-          values = []
-
           proxy_list.slice!(0..1000).each do |proxy|
-            host = proxy[:host]
-            port = proxy[:port]
-            value_arr = [host, port, protocol, proxy_type, category]
-            values << value_arr
+            host              =   proxy[:host]
+            port              =   proxy[:port]
+            
+            proxy             =   ::Proxy.where(host: host, port: port).first || ::Proxy.new
+            proxy.host        =   host
+            proxy.port        =   port
+            proxy.protocol    =   protocol
+            proxy.proxy_type  =   proxy_type
+            proxy.category    =   category
+            proxy.save
           end
-
-          ::Proxy.import(columns, values, :on_duplicate_key_update => [:proxy_type], :validate => false) if (values && values.any?)
+          
         end while (proxy_list && proxy_list.any?)
       end