proxied 0.2.1 → 0.2.7

data/lib/generators/mongoid/proxied_generator.rb

@@ -27,6 +27,7 @@ module Mongoid
   
   field :username, type: String
   field :password, type: String
+  field :auth_mode, type: String, default: :credentials
   
   field :protocol, type: String, default: :http
   field :proxy_type, type: String, default: :public
@@ -39,6 +40,9 @@ module Mongoid
   field :successful_attempts, type: Integer, default: 0
   field :failed_attempts, type: Integer, default: 0
   
+  field :checkable, type: Boolean, default: true
+  field :asyncable, type: Boolean, default: true
+  
   field :last_checked_at, type: DateTime
   
   # Validations

data/lib/generators/templates/proxied.rb

@@ -1,21 +1,38 @@
 ::Proxied.configure do |config|
-  config.proxy_class                  =   "<%= class_name.camelize.to_s %>" # Must be set to the ActiveRecord or Mongoid model that will be used for managing proxies
+  # The ActiveRecord or Mongoid model that will be used for managing proxies - Must be set otherwise the gem won't work!
+  config.proxy_class                  =   "<%= class_name.camelize.to_s %>"
   
+  # The minimum successful attempts and maximum failed attempts before a proxy shouldn't be considered valid anymore
   config.minimum_successful_attempts  =   1
   config.maximum_failed_attempts      =   10
   
+  # The queue that Sidekiq/ActiveJob will use to check proxies
   config.job_queue                    =   :proxies
   
+  # Log settings - if Rails is available it will log to the Rails log, otherwise just puts
+  config.logger                       =   defined?(Rails) ? -> (message) { Rails.logger.info(message) } : -> (message) { puts(message) }
+  config.log_level                    =   :info
+  
+  # The settings below are for configuring the proxy checker service
+  config.faraday                      =   {
+    adapter:    :net_http,
+    user_agent: -> { "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.2 Safari/605.1.15" },
+    verbose:    false
+  }
+  
+  # If you're using Typhoeus as adapter, consider setting the following setting either here or in a separate config/initializer:
+  # Ethon.logger = Logger.new(nil)
+  
   config.http_test                    =   {
     url:      "http://ipinfo.io/ip",
-    evaluate: -> (proxy, response) { response.to_s.strip.eql?(proxy.ip_address.strip) },
-    timeout:  10,
+    evaluate: -> (proxy, response) { !(response&.to_s&.strip&.downcase =~ /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$/i).nil? },
+    timeout:  30,
   }
   
   config.socks_test                   =   {
     hostname: "whois.verisign-grs.com",
     port:     43,
     query:    "=google.com",
-    timeout:  10
+    timeout:  30
   }
 end

data/lib/proxied/checker.rb

@@ -11,13 +11,15 @@ module Proxied
       self.limit                        =   limit
     end
 
-    def check_proxies(protocol: :all, proxy_type: :all, update: true)
+    def check_proxies(protocol: :all, proxy_type: :all, category: nil, update: true)
       proxies                           =   ::Proxied.configuration.proxy_class.constantize.should_be_checked(
-        protocol:                 protocol,
-        proxy_type:               proxy_type,
-        date:                     Time.now,
-        limit:                    self.limit,
-        maximum_failed_attempts:  self.maximum_failed_attempts
+        mode:                    self.mode.to_sym,
+        protocol:                protocol,
+        proxy_type:              proxy_type,
+        category:                category,
+        date:                    Time.now,
+        limit:                   self.limit,
+        maximum_failed_attempts: self.maximum_failed_attempts
       )
 
       if proxies&.any?
@@ -31,56 +33,79 @@ module Proxied
               ::Proxied::Jobs::CheckProxyJob.perform_async(proxy.id.to_s)
           end
         end
-
       else
         ::Proxied::Logger.log "Couldn't find any proxies to check!"
       end
     end
-    
-    def check_proxy(proxy, update: true)
+
+    def check_proxy(proxy, protocol: nil, update: true)
       ::Proxied::Logger.log "#{Time.now}: Will check if proxy #{proxy.proxy_address} is working."
+      protocol ||= proxy.protocol
       
-      self.send("check_#{proxy.protocol}_proxy", proxy, update: update)
+      self.send("check_#{protocol}_proxy", proxy, update: update)
     end
-    
-    def check_socks_proxy(proxy, test_host: ::Proxied.configuration.socks_test[:hostname], test_port: ::Proxied.configuration.socks_test[:port], test_query: ::Proxied.configuration.socks_test[:query], timeout: ::Proxied.configuration.socks_test[:timeout], update: true)
+
+    def check_socks_proxy(
+      proxy, 
+      test_host:  ::Proxied.configuration.socks_test[:hostname], 
+      test_port:  ::Proxied.configuration.socks_test[:port], 
+      test_query: ::Proxied.configuration.socks_test[:query], 
+      timeout:    ::Proxied.configuration.socks_test[:timeout], 
+      update:     true
+    )
       valid_proxy     =   false
 
       begin
         socks_proxy   =   ::Net::SSH::Proxy::SOCKS5.new(proxy.host, proxy.port, proxy.socks_proxy_credentials)
-        client        =   socks_proxy.open(test_host, test_port, {timeout: timeout})
+        socket        =   socks_proxy.open(test_host, test_port, {timeout: timeout})
+
+        socket.write("#{test_query}\r\n")
+        response      =   socket.read
+        valid_proxy   =   !response.to_s.empty? && response.downcase.include?(test_query.gsub(/^=/, '').downcase)
+
+        socket.close
 
-        client.write("#{test_query}\r\n")
-        response      =   client.read
-        
-        valid_proxy   =   !response.to_s.empty?
-        
-        client.close
-      
       rescue StandardError => e
         ::Proxied::Logger.log "Exception occured while trying to check proxy #{proxy.proxy_address}. Error Class: #{e.class}. Error Message: #{e.message}"
         valid_proxy   =   false
       end
-      
+
       update_proxy(proxy, valid_proxy) if update
-      
+
       return valid_proxy
     end
-    
-    def check_http_proxy(proxy, test_url: ::Proxied.configuration.http_test[:url], evaluate: ::Proxied.configuration.http_test[:evaluate], timeout: ::Proxied.configuration.http_test[:timeout], update: true)
-      ::Proxied::Logger.log "#{Time.now}: Fetching #{::Proxied.configuration.http_test[:url]} with proxy #{proxy.proxy_address}."
-
-      response        =   request(test_url, proxy, options: {timeout: timeout})
-      valid_proxy     =   evaluate.call(proxy, response)
 
-      update_proxy(proxy, valid_proxy) if update
+    def check_https_proxy(
+      proxy,
+      test_url: ::Proxied.configuration.http_test[:url],
+      evaluate: ::Proxied.configuration.http_test[:evaluate],
+      timeout:  ::Proxied.configuration.http_test[:timeout],
+      update:   true)
       
+      return check_http_proxy(proxy, test_url: test_url, evaluate: evaluate, timeout: timeout, update: update)
+    end
+
+    def check_http_proxy(
+      proxy,
+      test_url: ::Proxied.configuration.http_test[:url],
+      evaluate: ::Proxied.configuration.http_test[:evaluate],
+      timeout:  ::Proxied.configuration.http_test[:timeout],
+      update:   true)
+
+      ::Proxied::Logger.log "#{Time.now}: Fetching #{::Proxied.configuration.http_test[:url]} with proxy #{proxy.proxy_address} (#{proxy.ip_address})."
+
+      response                    =   request(test_url, proxy, options: {timeout: timeout})
+      valid_proxy                 =   evaluate.call(proxy, response)
+
+      update_proxy(proxy, valid_proxy, response) if update
+
       return valid_proxy
     end
-    
-    def update_proxy(proxy, valid)
-      ::Proxied::Logger.log "#{Time.now}: Proxy #{proxy.proxy_address} is #{valid ? "working" : "not working"}!"
-      
+
+    def update_proxy(proxy, valid, response = nil)      
+      ::Proxied::Logger.info "#{Time.now}: Proxy #{proxy.proxy_address} (#{proxy.ip_address}) is #{valid ? "working" : "not working"}!"
+      ::Proxied::Logger.debug "Response: #{response}" if !valid && response
+
       successful_attempts         =   proxy.successful_attempts || 0
       failed_attempts             =   proxy.failed_attempts || 0
 
@@ -91,35 +116,48 @@ module Proxied
       end
 
       is_valid                    =   (successful_attempts >= self.minimum_successful_attempts && failed_attempts < self.maximum_failed_attempts)
-      
+
       proxy.valid_proxy           =   is_valid
       proxy.successful_attempts   =   successful_attempts
       proxy.failed_attempts       =   failed_attempts
       proxy.last_checked_at       =   Time.now
       proxy.save
     end
-    
+
     private
       def request(url, proxy, options: {})
         response                  =   nil
         
-        user_agent                =   options.fetch(:user_agent, ::Proxied.configuration.faraday.fetch(:user_agent, nil))
+        user_agent                =   options.fetch(:user_agent, ::Proxied.configuration.faraday.fetch(:user_agent, nil)&.call)
         timeout                   =   options.fetch(:timeout, ::Proxied.configuration.http_test[:timeout])
-        
+        auth_mode                 =   proxy.respond_to?(:auth_mode) ? proxy.auth_mode : 'credentials'
+
         begin
-          response                =   ::Faraday.new(url: url) do |builder|
+          connection              =   ::Faraday.new(url: url) do |builder|
             builder.headers["User-Agent"]   =   user_agent if !user_agent.to_s.empty?
             builder.options[:timeout]       =   timeout if timeout
             builder.proxy                   =   proxy.proxy_options_for_faraday
+            #builder.basic_auth(proxy.username, proxy.password) if using_basic_auth?(auth_mode)
             builder.response :logger if ::Proxied.configuration.verbose_faraday?
             builder.adapter ::Proxied.configuration.faraday.fetch(:adapter, :net_http)
-          end.get&.body
-        rescue Faraday::Error => e
+          end
+
+          response                =   connection.get&.body
+          
+        rescue Faraday::TimeoutError, Faraday::Error => e
           ::Proxied::Logger.log "Exception occured while trying to check proxy #{proxy.proxy_address}. Error Class: #{e.class}. Error Message: #{e.message}"
         end
         
         return response
       end
 
+      def using_credentials?(mode)
+        mode.to_s.eql?('credentials')
+      end
+
+      def using_basic_auth?(mode)
+        mode.to_s.eql?('basic') || mode.to_s.eql?('basic_auth')
+      end
+
   end
 end

data/lib/proxied/configuration.rb

@@ -2,39 +2,45 @@ module Proxied
   class Configuration
     attr_accessor :proxy_class
     attr_accessor :minimum_successful_attempts, :maximum_failed_attempts
+    attr_accessor :job_queue
+    attr_accessor :logger, :log_level
     attr_accessor :faraday
     attr_accessor :http_test, :socks_test
-    attr_accessor :job_queue
-    attr_accessor :logger
-  
-    def initialize      
-      self.proxy_class                  =   nil # Must be set to the ActiveRecord or Mongoid model that will be used for managing proxies
+    
+    def initialize
+      # The ActiveRecord or Mongoid model that will be used for managing proxies - Must be set otherwise the gem won't work!
+      self.proxy_class                  =   nil
       
+      # The minimum successful attempts and maximum failed attempts before a proxy shouldn't be considered valid anymore
       self.minimum_successful_attempts  =   1
       self.maximum_failed_attempts      =   10
       
+      # The queue that Sidekiq/ActiveJob will use to check proxies
+      self.job_queue        =   :proxies
+      
+      # Log settings - if Rails is available it will log to the Rails log, otherwise just puts
+      self.logger           =   defined?(Rails) ? -> (message) { Rails.logger.info(message) } : -> (message) { puts(message) }
+      self.log_level        =   :info
+      
+      # The settings below are for configuring the proxy checker service
       self.faraday          =   {
         adapter:    :net_http,
-        user_agent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/11.1.1 Safari/605.1.15",
+        user_agent: -> { "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.2 Safari/605.1.15" },
         verbose:    false
       }
       
       self.http_test        =   {
-        url:      "http://ipinfo.io/ip",
-        evaluate: -> (proxy, response) { response.to_s.strip.eql?(proxy.ip_address.strip) },
-        timeout:  10,
+        url:      "https://ipinfo.io/ip",
+        evaluate: -> (proxy, response) { !(response&.to_s&.strip&.downcase =~ /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$/i).nil? },
+        timeout:  30,
       }
       
       self.socks_test       =   {
         hostname: "whois.verisign-grs.com",
         port:     43,
         query:    "=google.com",
-        timeout:  10
+        timeout:  30
       }
-      
-      self.job_queue        =   :proxies
-      
-      self.logger           =   defined?(Rails) ? -> (message) { Rails.logger.info(message) } : -> (message) { puts(message) }
     end
     
     def verbose_faraday?

data/lib/proxied/importer.rb

@@ -21,19 +21,22 @@ module Proxied
         ip_address                  =   proxy_item.fetch(:ip_address, host)&.to_s&.strip&.downcase
         
         query                       =   {
-          host:       host,
-          port:       port
+          host:            host,
+          port:            port
         }
     
         parsed                      =   {
-          ip_address: ip_address,
-          protocol:   proxy_item.fetch(:protocol, "http")&.to_s&.strip&.downcase,
-          proxy_type: proxy_item.fetch(:type, :private)&.to_s&.strip,
-          category:   proxy_item.fetch(:category, nil)&.to_s&.strip&.downcase,
-          country:    proxy_item.fetch(:country, nil)&.to_s&.strip&.upcase,
-          city:       proxy_item.fetch(:city, nil)&.to_s&.strip&.downcase,
-          username:   proxy_item.fetch(:username, nil)&.to_s&.strip,
-          password:   proxy_item.fetch(:password, nil)&.to_s&.strip
+          ip_address:      ip_address,
+          protocol:        proxy_item.fetch(:protocol, "http")&.to_s&.strip&.downcase,
+          proxy_type:      proxy_item.fetch(:type, :private)&.to_s&.strip,
+          category:        proxy_item.fetch(:category, nil)&.to_s&.strip&.downcase,
+          country:         proxy_item.fetch(:country, nil)&.to_s&.strip&.upcase,
+          city:            proxy_item.fetch(:city, nil)&.to_s&.strip&.downcase,
+          username:        proxy_item.fetch(:username, nil)&.to_s&.strip,
+          password:        proxy_item.fetch(:password, nil)&.to_s&.strip,
+          auth_mode:       proxy_item.fetch(:auth_mode, :credentials)&.to_s&.strip,
+          checkable:       proxy_item.fetch(:checkable, true),
+          asyncable:       proxy_item.fetch(:asyncable, true),
         }.merge(query)
     
         proxy                       =   ::Proxied.configuration.proxy_class.constantize.where(query).first || ::Proxied.configuration.proxy_class.constantize.new

data/lib/proxied/logger.rb

@@ -1,9 +1,19 @@
 module Proxied
   class Logger
+    class << self
+      
+      def info(message)
+        log(message) if ::Proxied.configuration.log_level.to_sym.eql?(:info)
+      end
     
-    def self.log(message)
-      ::Proxied.configuration.logger.call(message)
-    end
+      def debug(message)
+        log(message) if ::Proxied.configuration.log_level.to_sym.eql?(:debug)
+      end
+    
+      def log(message)
+        ::Proxied.configuration.logger.call(message)
+      end
       
+    end
   end
 end

data/lib/proxied/nosql/proxy_methods.rb

@@ -11,8 +11,10 @@ module Proxied
       end
 
       module ClassMethods
-        def should_be_checked(protocol: :all, proxy_type: :all, date: Time.now, limit: 10, maximum_failed_attempts: 10)
-          proxies     =   get_proxies_for_protocol_and_proxy_type(protocol, proxy_type)
+        def should_be_checked(mode: :synchronous, protocol: :all, proxy_type: :all, category: nil, country: nil, date: Time.now, limit: nil, maximum_failed_attempts: 10)
+          proxies     =   get_proxies(protocol: protocol, proxy_type: proxy_type, category: category, country: country)
+          proxies     =   proxies.where(checkable: true)
+          proxies     =   proxies.where(asyncable: true) if mode.to_sym.eql?(:asynchronous)
   
           proxies     =   proxies.any_of(
             {:last_checked_at.exists => false},
@@ -27,16 +29,15 @@ module Proxied
           )
   
           proxies     =   proxies.order_by([[:valid_proxy, :asc], [:failed_attempts, :asc], [:last_checked_at, :asc]])
-          proxies     =   proxies.limit(limit)
+          proxies     =   proxies.limit(limit) if limit && !limit.zero?
   
           return proxies
         end
       
-        def get_valid_proxies(protocol: :all, proxy_type: :all, category: nil, maximum_failed_attempts: nil)
-          proxies     =   get_proxies_for_protocol_and_proxy_type(protocol, proxy_type)
+        def get_valid_proxies(protocol: :all, proxy_type: :all, category: nil, country: nil, maximum_failed_attempts: nil)
+          proxies     =   get_proxies(protocol: protocol, proxy_type: proxy_type, category: category, country: country)
           proxies     =   proxies.where(valid_proxy: true)
           proxies     =   proxies.where(:failed_attempts.lte => maximum_failed_attempts) if maximum_failed_attempts
-          proxies     =   proxies.where(category: category) unless category.to_s.empty?
         end
       
         def get_random_proxy(protocol: :all, proxy_type: :all, category: nil, maximum_failed_attempts: nil, retries: 3)

data/lib/proxied/shared.rb

@@ -2,17 +2,19 @@ module Proxied
   module Shared
 
     module ClassMethods    
-      def get_proxies_for_protocol_and_proxy_type(protocol, proxy_type)
+      def get_proxies(protocol: :all, proxy_type: :all, category: nil, country: nil)
         proxies     =   ::Proxy.where(nil)
-        proxies     =   proxies.where(protocol: protocol)     if (protocol && !protocol.downcase.to_sym.eql?(:all))
-        proxies     =   proxies.where(proxy_type: proxy_type) if (proxy_type && !proxy_type.downcase.to_sym.eql?(:all))
+        proxies     =   proxies.where(protocol: protocol)           if (protocol && !protocol.downcase.to_sym.eql?(:all))
+        proxies     =   proxies.where(proxy_type: proxy_type)       if (proxy_type && !proxy_type.downcase.to_sym.eql?(:all))
+        proxies     =   proxies.where(category: category)           unless category.to_s.empty?
+        proxies     =   proxies.where(country: country.to_s.upcase) unless country.to_s.empty?
       
         return proxies
       end
 
       # Keep these for compatibility for now, they just wrap the utility functions
-      def format_proxy_address(host, port = 80, include_http = false)
-        ::Proxied::Utilities.format_proxy_address(host: host, port: port, include_http: include_http)
+      def format_proxy_address(host, port = 80, include_protocol = false)
+        ::Proxied::Utilities.format_proxy_address(host: host, port: port, include_protocol: include_protocol)
       end
 
       def format_proxy_credentials(username, password)
@@ -21,8 +23,13 @@ module Proxied
     end
 
     module InstanceMethods
-      def proxy_address(include_http: false)
-        ::Proxied::Utilities.format_proxy_address(host: self.host, port: self.port, include_http: include_http)
+      def proxy_address(include_protocol: false)
+        case self.auth_mode.to_sym
+          when :credentials
+            ::Proxied::Utilities.format_proxy_address(host: self.host, port: self.port, protocol: self.protocol, include_protocol: include_protocol)
+          when :url
+            ::Proxied::Utilities.format_proxy_address(host: self.host, port: self.port, protocol: self.protocol, username: self.username, password: self.password, include_protocol: include_protocol)
+        end
       end
 
       def proxy_credentials
@@ -34,12 +41,16 @@ module Proxied
       end
       
       def proxy_options_for_faraday
-        ::Proxied::Utilities.proxy_options_for_faraday(host: self.host, port: self.port, username: self.username, password: self.password)
+        ::Proxied::Utilities.proxy_options_for_faraday(
+          host:      self.host,
+          port:      self.port,
+          protocol:  self.protocol,
+          username:  self.username,
+          password:  self.password,
+          auth_mode: self.auth_mode
+        )
       end
       
-      def proxy_switcher_import_format
-        ::Proxied::Utilities.proxy_switcher_import_format(host: self.host, port: self.port, username: self.username, password: self.password, country: self.country)
-      end
     end
         
   end