proxied 0.2.1 → 0.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 259fa06faa16224529c74da60047158116d019a7017972887cd9c91228f854b4
-  data.tar.gz: a9a83e6416cc5db560e4b3d90129eabc7e0c55aeea4150bba155f482957091bb
+  metadata.gz: 670ca19a9100f67ddca208cad0c90c729d47f79148739657e7f031f5ce732df1
+  data.tar.gz: 4adedbbef5ab9765610080ccd34a6df00a6158f37728af7f059c7190b71e68cc
 SHA512:
-  metadata.gz: 764f5d70013bf48d88064cf6bd90d5c29759f75b6672f26e4d443f3bebc7e49e18f71d4bd1dbd7628cfcbd3c9bd178027a2327d3f68732cf96b09a2be8234537
-  data.tar.gz: 1a79de19edcfb1bdef83ff8d110489ab43501732dbb04c38f6a312e382f69fbc63142c6a60fc1de9e081628952c5a3984bfd55e8e4373dbf4e99e0c0f5ec28c2
+  metadata.gz: 7de2420827e224b1e4d23cc595714a0a15b526d94d68481efb3b008122090cde99f0ef7a1820f14b8fdb70473ab6f0cf1de0c5cf093ac160bc0f3e0c16c68656
+  data.tar.gz: 2eda880c9e511393a3e0224d6863a31e85bb3eb7804a66623446aa7ae6a171bbe46245e2416786f586c091a009c61a75cf95e2aa18f85774ccfcf0ce8b23811a

data/lib/generators/active_record/templates/migration.rb

@@ -3,19 +3,23 @@
 class ProxiedCreate<%= table_name.camelize %> < ActiveRecord::Migration<%= migration_version %>
   def change
     create_table :<%= table_name %><%= primary_key_type %> do |t|
-      t.string  :host,                  null: false, index: true
-      t.string  :ip_address,            null: false, index: true
-      t.integer :port,                  null: false, index: true
-      t.string  :username
-      t.string  :password
+      t.string   :host,                 null: false, index: true
+      t.string   :ip_address,           null: false, index: true
+      t.integer  :port,                 null: false, index: true
       
-      t.string  :protocol,              null: false, default: 'http', index: true
-      t.string  :proxy_type,            null: false, default: 'public', index: true
-      t.string  :category
+      t.string   :username
+      t.string   :password
+      t.string   :auth_mode,            null: false, default: 'credentials', index: true
       
-      t.string  :country,               index: true
-      t.string  :city,                  index: true
+      t.string   :protocol,             null: false, default: 'http', index: true
+      t.string   :proxy_type,           null: false, default: 'public', index: true
+      t.string   :category
       
+      t.string   :country,              index: true
+      t.string   :city,                 index: true
+      
+      t.boolean  :checkable,            null: false, default: true, index: true
+      t.boolean  :asyncable,            null: false, default: true, index: true
       t.datetime :last_checked_at,      index: true
       
       t.boolean  :valid_proxy,          null:  false, default: false, index: true

data/lib/generators/mongoid/proxied_generator.rb

@@ -27,6 +27,7 @@ module Mongoid
   
   field :username, type: String
   field :password, type: String
+  field :auth_mode, type: String, default: :credentials
   
   field :protocol, type: String, default: :http
   field :proxy_type, type: String, default: :public
@@ -39,6 +40,9 @@ module Mongoid
   field :successful_attempts, type: Integer, default: 0
   field :failed_attempts, type: Integer, default: 0
   
+  field :checkable, type: Boolean, default: true
+  field :asyncable, type: Boolean, default: true
+  
   field :last_checked_at, type: DateTime
   
   # Validations

data/lib/generators/templates/proxied.rb

@@ -1,21 +1,38 @@
 ::Proxied.configure do |config|
-  config.proxy_class                  =   "<%= class_name.camelize.to_s %>" # Must be set to the ActiveRecord or Mongoid model that will be used for managing proxies
+  # The ActiveRecord or Mongoid model that will be used for managing proxies - Must be set otherwise the gem won't work!
+  config.proxy_class                  =   "<%= class_name.camelize.to_s %>"
   
+  # The minimum successful attempts and maximum failed attempts before a proxy shouldn't be considered valid anymore
   config.minimum_successful_attempts  =   1
   config.maximum_failed_attempts      =   10
   
+  # The queue that Sidekiq/ActiveJob will use to check proxies
   config.job_queue                    =   :proxies
   
+  # Log settings - if Rails is available it will log to the Rails log, otherwise just puts
+  config.logger                       =   defined?(Rails) ? -> (message) { Rails.logger.info(message) } : -> (message) { puts(message) }
+  config.log_level                    =   :info
+  
+  # The settings below are for configuring the proxy checker service
+  config.faraday                      =   {
+    adapter:    :net_http,
+    user_agent: -> { "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/11.1.1 Safari/605.1.15" },
+    verbose:    false
+  }
+  
+  # If you're using Typhoeus as adapter, consider setting the following setting either here or in a separate config/initializer:
+  # Ethon.logger = Logger.new(nil)
+  
   config.http_test                    =   {
     url:      "http://ipinfo.io/ip",
-    evaluate: -> (proxy, response) { response.to_s.strip.eql?(proxy.ip_address.strip) },
-    timeout:  10,
+    evaluate: -> (proxy, response) { !(response&.to_s&.strip&.downcase =~ /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$/i).nil? },
+    timeout:  30,
   }
   
   config.socks_test                   =   {
     hostname: "whois.verisign-grs.com",
     port:     43,
     query:    "=google.com",
-    timeout:  10
+    timeout:  30
   }
 end

data/lib/proxied/checker.rb

@@ -13,11 +13,12 @@ module Proxied
 
     def check_proxies(protocol: :all, proxy_type: :all, update: true)
       proxies                           =   ::Proxied.configuration.proxy_class.constantize.should_be_checked(
-        protocol:                 protocol,
-        proxy_type:               proxy_type,
-        date:                     Time.now,
-        limit:                    self.limit,
-        maximum_failed_attempts:  self.maximum_failed_attempts
+        mode:                    self.mode.to_sym,
+        protocol:                protocol,
+        proxy_type:              proxy_type,
+        date:                    Time.now,
+        limit:                   self.limit,
+        maximum_failed_attempts: self.maximum_failed_attempts
       )
 
       if proxies&.any?
@@ -68,18 +69,19 @@ module Proxied
     end
     
     def check_http_proxy(proxy, test_url: ::Proxied.configuration.http_test[:url], evaluate: ::Proxied.configuration.http_test[:evaluate], timeout: ::Proxied.configuration.http_test[:timeout], update: true)
-      ::Proxied::Logger.log "#{Time.now}: Fetching #{::Proxied.configuration.http_test[:url]} with proxy #{proxy.proxy_address}."
-
-      response        =   request(test_url, proxy, options: {timeout: timeout})
-      valid_proxy     =   evaluate.call(proxy, response)
+      ::Proxied::Logger.log "#{Time.now}: Fetching #{::Proxied.configuration.http_test[:url]} with proxy #{proxy.proxy_address} (#{proxy.ip_address})."
+      
+      response                    =   request(test_url, proxy, options: {timeout: timeout})
+      valid_proxy                 =   evaluate.call(proxy, response)
 
-      update_proxy(proxy, valid_proxy) if update
+      update_proxy(proxy, valid_proxy, response) if update
       
       return valid_proxy
     end
     
-    def update_proxy(proxy, valid)
-      ::Proxied::Logger.log "#{Time.now}: Proxy #{proxy.proxy_address} is #{valid ? "working" : "not working"}!"
+    def update_proxy(proxy, valid, response = nil)      
+      ::Proxied::Logger.info "#{Time.now}: Proxy #{proxy.proxy_address} (#{proxy.ip_address}) is #{valid ? "working" : "not working"}!"
+      ::Proxied::Logger.debug "Response: #{response}" if !valid && response
       
       successful_attempts         =   proxy.successful_attempts || 0
       failed_attempts             =   proxy.failed_attempts || 0
@@ -103,22 +105,22 @@ module Proxied
       def request(url, proxy, options: {})
         response                  =   nil
         
-        user_agent                =   options.fetch(:user_agent, ::Proxied.configuration.faraday.fetch(:user_agent, nil))
+        user_agent                =   options.fetch(:user_agent, ::Proxied.configuration.faraday.fetch(:user_agent, nil)&.call)
         timeout                   =   options.fetch(:timeout, ::Proxied.configuration.http_test[:timeout])
         
         begin
           response                =   ::Faraday.new(url: url) do |builder|
             builder.headers["User-Agent"]   =   user_agent if !user_agent.to_s.empty?
             builder.options[:timeout]       =   timeout if timeout
-            builder.proxy                   =   proxy.proxy_options_for_faraday
+            builder.proxy                   =   proxy.proxy_options_for_faraday            
             builder.response :logger if ::Proxied.configuration.verbose_faraday?
             builder.adapter ::Proxied.configuration.faraday.fetch(:adapter, :net_http)
-          end.get&.body
+          end
         rescue Faraday::Error => e
           ::Proxied::Logger.log "Exception occured while trying to check proxy #{proxy.proxy_address}. Error Class: #{e.class}. Error Message: #{e.message}"
         end
         
-        return response
+        return response.get&.body
       end
 
   end

data/lib/proxied/configuration.rb

@@ -2,39 +2,47 @@ module Proxied
   class Configuration
     attr_accessor :proxy_class
     attr_accessor :minimum_successful_attempts, :maximum_failed_attempts
+    attr_accessor :job_queue
+    attr_accessor :logger, :log_level
     attr_accessor :faraday
     attr_accessor :http_test, :socks_test
-    attr_accessor :job_queue
-    attr_accessor :logger
+    
+    
   
-    def initialize      
-      self.proxy_class                  =   nil # Must be set to the ActiveRecord or Mongoid model that will be used for managing proxies
+    def initialize
+      # The ActiveRecord or Mongoid model that will be used for managing proxies - Must be set otherwise the gem won't work!
+      self.proxy_class                  =   nil
       
+      # The minimum successful attempts and maximum failed attempts before a proxy shouldn't be considered valid anymore
       self.minimum_successful_attempts  =   1
       self.maximum_failed_attempts      =   10
       
+      # The queue that Sidekiq/ActiveJob will use to check proxies
+      self.job_queue        =   :proxies
+      
+      # Log settings - if Rails is available it will log to the Rails log, otherwise just puts
+      self.logger           =   defined?(Rails) ? -> (message) { Rails.logger.info(message) } : -> (message) { puts(message) }
+      self.log_level        =   :info
+      
+      # The settings below are for configuring the proxy checker service
       self.faraday          =   {
         adapter:    :net_http,
-        user_agent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/11.1.1 Safari/605.1.15",
+        user_agent: -> { "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/11.1.1 Safari/605.1.15" },
         verbose:    false
       }
       
       self.http_test        =   {
         url:      "http://ipinfo.io/ip",
-        evaluate: -> (proxy, response) { response.to_s.strip.eql?(proxy.ip_address.strip) },
-        timeout:  10,
+        evaluate: -> (proxy, response) { !(response&.to_s&.strip&.downcase =~ /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$/i).nil? },
+        timeout:  30,
       }
       
       self.socks_test       =   {
         hostname: "whois.verisign-grs.com",
         port:     43,
         query:    "=google.com",
-        timeout:  10
+        timeout:  30
       }
-      
-      self.job_queue        =   :proxies
-      
-      self.logger           =   defined?(Rails) ? -> (message) { Rails.logger.info(message) } : -> (message) { puts(message) }
     end
     
     def verbose_faraday?

data/lib/proxied/importer.rb

@@ -21,19 +21,22 @@ module Proxied
         ip_address                  =   proxy_item.fetch(:ip_address, host)&.to_s&.strip&.downcase
         
         query                       =   {
-          host:       host,
-          port:       port
+          host:            host,
+          port:            port
         }
     
         parsed                      =   {
-          ip_address: ip_address,
-          protocol:   proxy_item.fetch(:protocol, "http")&.to_s&.strip&.downcase,
-          proxy_type: proxy_item.fetch(:type, :private)&.to_s&.strip,
-          category:   proxy_item.fetch(:category, nil)&.to_s&.strip&.downcase,
-          country:    proxy_item.fetch(:country, nil)&.to_s&.strip&.upcase,
-          city:       proxy_item.fetch(:city, nil)&.to_s&.strip&.downcase,
-          username:   proxy_item.fetch(:username, nil)&.to_s&.strip,
-          password:   proxy_item.fetch(:password, nil)&.to_s&.strip
+          ip_address:      ip_address,
+          protocol:        proxy_item.fetch(:protocol, "http")&.to_s&.strip&.downcase,
+          proxy_type:      proxy_item.fetch(:type, :private)&.to_s&.strip,
+          category:        proxy_item.fetch(:category, nil)&.to_s&.strip&.downcase,
+          country:         proxy_item.fetch(:country, nil)&.to_s&.strip&.upcase,
+          city:            proxy_item.fetch(:city, nil)&.to_s&.strip&.downcase,
+          username:        proxy_item.fetch(:username, nil)&.to_s&.strip,
+          password:        proxy_item.fetch(:password, nil)&.to_s&.strip,
+          auth_mode:       proxy_item.fetch(:auth_mode, :credentials)&.to_s&.strip,
+          checkable:       proxy_item.fetch(:checkable, true),
+          asyncable:       proxy_item.fetch(:asyncable, true),
         }.merge(query)
     
         proxy                       =   ::Proxied.configuration.proxy_class.constantize.where(query).first || ::Proxied.configuration.proxy_class.constantize.new

data/lib/proxied/logger.rb

@@ -1,9 +1,19 @@
 module Proxied
   class Logger
+    class << self
+      
+      def info(message)
+        log(message) if ::Proxied.configuration.log_level.to_sym.eql?(:info)
+      end
     
-    def self.log(message)
-      ::Proxied.configuration.logger.call(message)
-    end
+      def debug(message)
+        log(message) if ::Proxied.configuration.log_level.to_sym.eql?(:debug)
+      end
+    
+      def log(message)
+        ::Proxied.configuration.logger.call(message)
+      end
       
+    end
   end
 end

data/lib/proxied/nosql/proxy_methods.rb

@@ -11,8 +11,10 @@ module Proxied
       end
 
       module ClassMethods
-        def should_be_checked(protocol: :all, proxy_type: :all, date: Time.now, limit: 10, maximum_failed_attempts: 10)
-          proxies     =   get_proxies_for_protocol_and_proxy_type(protocol, proxy_type)
+        def should_be_checked(mode: :synchronous, protocol: :all, proxy_type: :all, category: nil, date: Time.now, limit: nil, maximum_failed_attempts: 10)
+          proxies     =   get_proxies(protocol: protocol, proxy_type: proxy_type, category: category)
+          proxies     =   proxies.where(checkable: true)
+          proxies     =   proxies.where(asyncable: true) if mode.to_sym.eql?(:asynchronous)
   
           proxies     =   proxies.any_of(
             {:last_checked_at.exists => false},
@@ -27,16 +29,15 @@ module Proxied
           )
   
           proxies     =   proxies.order_by([[:valid_proxy, :asc], [:failed_attempts, :asc], [:last_checked_at, :asc]])
-          proxies     =   proxies.limit(limit)
+          proxies     =   proxies.limit(limit) if limit && !limit.zero?
   
           return proxies
         end
       
         def get_valid_proxies(protocol: :all, proxy_type: :all, category: nil, maximum_failed_attempts: nil)
-          proxies     =   get_proxies_for_protocol_and_proxy_type(protocol, proxy_type)
+          proxies     =   get_proxies(protocol: protocol, proxy_type: proxy_type, category: category)
           proxies     =   proxies.where(valid_proxy: true)
           proxies     =   proxies.where(:failed_attempts.lte => maximum_failed_attempts) if maximum_failed_attempts
-          proxies     =   proxies.where(category: category) unless category.to_s.empty?
         end
       
         def get_random_proxy(protocol: :all, proxy_type: :all, category: nil, maximum_failed_attempts: nil, retries: 3)

data/lib/proxied/shared.rb

@@ -2,10 +2,11 @@ module Proxied
   module Shared
 
     module ClassMethods    
-      def get_proxies_for_protocol_and_proxy_type(protocol, proxy_type)
+      def get_proxies(protocol: :all, proxy_type: :all, category: nil)
         proxies     =   ::Proxy.where(nil)
         proxies     =   proxies.where(protocol: protocol)     if (protocol && !protocol.downcase.to_sym.eql?(:all))
         proxies     =   proxies.where(proxy_type: proxy_type) if (proxy_type && !proxy_type.downcase.to_sym.eql?(:all))
+        proxies     =   proxies.where(category: category) unless category.to_s.empty?
       
         return proxies
       end
@@ -34,7 +35,7 @@ module Proxied
       end
       
       def proxy_options_for_faraday
-        ::Proxied::Utilities.proxy_options_for_faraday(host: self.host, port: self.port, username: self.username, password: self.password)
+        ::Proxied::Utilities.proxy_options_for_faraday(host: self.host, port: self.port, username: self.username, password: self.password, auth_mode: self.auth_mode)
       end
       
       def proxy_switcher_import_format

data/lib/proxied/sql/proxy_methods.rb

@@ -11,21 +11,22 @@ module Proxied
       end
 
       module ClassMethods
-        def should_be_checked(protocol: :all, proxy_type: :all, date: Time.now, limit: 10, maximum_failed_attempts: 10)
-          proxies     =   get_proxies_for_protocol_and_proxy_type(protocol, proxy_type)
+        def should_be_checked(mode: :synchronous, protocol: :all, proxy_type: :all, category: nil, date: Time.now, limit: nil, maximum_failed_attempts: 10)
+          proxies     =   get_proxies(protocol: protocol, proxy_type: proxy_type, category: category)
+          proxies     =   proxies.where(checkable: true)
+          proxies     =   proxies.where(asyncable: true) if mode.to_sym.eql?(:asynchronous)
           proxies     =   proxies.where(["(last_checked_at IS NULL OR last_checked_at < ?)", date])
           proxies     =   proxies.where(["failed_attempts <= ?", maximum_failed_attempts])
           proxies     =   proxies.order("valid_proxy ASC, failed_attempts ASC, last_checked_at ASC")
-          proxies     =   proxies.limit(limit)
+          proxies     =   proxies.limit(limit) if limit && !limit.zero?
         
           return proxies
         end
       
         def get_valid_proxies(protocol: :all, proxy_type: :all, category: nil, maximum_failed_attempts: nil)
-          proxies     =   get_proxies_for_protocol_and_proxy_type(protocol, proxy_type)
+          proxies     =   get_proxies(protocol: protocol, proxy_type: proxy_type, category: category)
           proxies     =   proxies.where(["valid_proxy = ? AND last_checked_at IS NOT NULL", true])
           proxies     =   proxies.where(["failed_attempts <= ?", maximum_failed_attempts]) if maximum_failed_attempts
-          proxies     =   proxies.where(category: category) unless category.to_s.empty?
 
           return proxies
         end
@@ -45,7 +46,7 @@ module Proxied
               "RAND() DESC"
           end
         
-          proxies     =   proxies.order(order_clause)
+          proxies     =   proxies.order(Arel.sql(order_clause))
         
           proxy       =   nil
         

data/lib/proxied/utilities.rb

@@ -1,11 +1,11 @@
 module Proxied
   class Utilities
-    
     class << self
       
-      def format_proxy_address(host:, port: 80, include_http: false)
+      def format_proxy_address(host:, port: 80, username: nil, password: nil, include_http: false)
         address                   =   "#{host.strip}:#{port}"
-        address                   =   "http://#{address}" if include_http && !address.start_with?("http://")
+        address                   =   "#{format_proxy_credentials(username, password)}@#{address}" if !username.to_s.empty? && !password.to_s.empty?
+        address                   =   (include_http && !address.start_with?("http://")) ? "http://#{address}" : address
         
         return address
       end
@@ -23,12 +23,18 @@ module Proxied
         return credentials
       end
       
-      def proxy_options_for_faraday(host:, port: 80, username: nil, password: nil)
+      def proxy_options_for_faraday(host:, port: 80, username: nil, password: nil, auth_mode: :credentials)
         proxy_options             =   {}
-    
-        proxy_options[:uri]       =   format_proxy_address(host: host, port: port, include_http: true)
-        proxy_options[:user]      =   username if !username.to_s.empty?
-        proxy_options[:password]  =   password if !password.to_s.empty?
+        username                  =   CGI::escape(username)
+        password                  =   CGI::escape(password)
+        
+        options                   =   {host: host, port: port, include_http: true}
+        options[:username]        =   username if !username.to_s.empty? && auth_mode&.to_sym&.eql?(:basic_auth)
+        options[:password]        =   password if !password.to_s.empty? && auth_mode&.to_sym&.eql?(:basic_auth)
+        
+        proxy_options[:uri]       =   format_proxy_address(options)
+        proxy_options[:user]      =   username if !username.to_s.empty? && auth_mode&.to_sym&.eql?(:credentials)
+        proxy_options[:password]  =   password if !password.to_s.empty? && auth_mode&.to_sym&.eql?(:credentials)
     
         return proxy_options
       end
@@ -51,6 +57,5 @@ module Proxied
       end
       
     end
-    
   end
 end

data/lib/proxied/version.rb

@@ -1,3 +1,3 @@
 module Proxied
-  VERSION = "0.2.1"
+  VERSION = "0.2.2"
 end

data/lib/tasks/tasks.rake

@@ -5,8 +5,7 @@ namespace :proxied do
     checker = ::Proxied::Checker.new(mode: args.mode.to_sym, maximum_failed_attempts: args.maximum_failed_attempts.to_i)
     checker.check_proxies(
       protocol:     args.protocol.to_sym, 
-      proxy_type:   args.proxy_type.to_sym, 
-      
+      proxy_type:   args.proxy_type.to_sym,
     )
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: proxied
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.2
 platform: ruby
 authors:
 - Sebastian