proxied 0.1.0

data/lib/proxied/configuration.rb

@@ -0,0 +1,41 @@
+module Proxied
+  class Configuration
+    attr_accessor :proxy_class
+    attr_accessor :minimum_successful_attempts, :maximum_failed_attempts
+    attr_accessor :faraday
+    attr_accessor :http_test, :socks_test
+    attr_accessor :logger
+  
+    def initialize      
+      self.proxy_class                  =   nil # Must be set to the ActiveRecord or Mongoid model that will be used for managing proxies
+      
+      self.minimum_successful_attempts  =   1
+      self.maximum_failed_attempts      =   10
+      
+      self.faraday          =   {
+        adapter:    :net_http,
+        user_agent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/11.1.1 Safari/605.1.15",
+        verbose:    true
+      }
+      
+      self.http_test        =   {
+        url:      "http://www.google.com/robots.txt",
+        timeout:  10,
+      }
+      
+      self.socks_test       =   {
+        hostname: "whois.verisign-grs.com",
+        port:     43,
+        query:    "=google.com",
+        timeout:  10
+      }
+      
+      self.logger           =   defined?(Rails) ? -> (message) { Rails.logger.info(message) } : -> (message) { puts(message) }
+    end
+    
+    def verbose_faraday?
+      self.faraday.fetch(:verbose, false)
+    end
+  
+  end
+end

data/lib/proxied/jobs/check_proxies_job.rb

@@ -0,0 +1,12 @@
+module Proxied
+  module Jobs
+    class CheckProxiesJob
+      include ::Sidekiq::Worker
+      sidekiq_options queue: :proxies
+
+      def perform(protocol = :all, proxy_type = :all, mode = :synchronous)
+        ::Proxied::Checker.new.check_proxies(protocol: protocol.to_sym, proxy_type: proxy_type.to_sym, mode: mode.to_sym)
+      end
+    end
+  end
+end

data/lib/proxied/jobs/check_proxy_job.rb

@@ -0,0 +1,18 @@
+module Proxied
+  module Jobs
+    class CheckProxyJob
+      include ::Sidekiq::Worker
+      sidekiq_options queue: :proxies
+
+      def perform(proxy_id)
+        proxy_object  =   ::Proxy.where(id: proxy_id).first
+
+        if proxy_object
+          checker     =   ::Proxied::Checker.new
+          checker.check_proxy(proxy_object)
+          checker.update_proxies
+        end
+      end
+    end
+  end
+end

data/lib/proxied/logger.rb

@@ -0,0 +1,9 @@
+module Proxied
+  class Logger
+    
+    def self.log(message)
+      ::Proxied.configuration.logger.call(message)
+    end
+      
+  end
+end

data/lib/proxied/nosql/proxy_methods.rb

@@ -0,0 +1,60 @@
+module Proxied
+  module Nosql
+    
+    module ProxyMethods
+    
+      def self.included(base)
+        base.send :extend ::Proxied::Shared::ClassMethods
+        base.send :extend, ClassMethods
+        
+        base.send :include ::Proxied::Shared::InstanceMethods
+      end
+
+      module ClassMethods
+        def should_be_checked(protocol: :all, proxy_type: :all, date: Time.now, limit: 10, maximum_failed_attempts: 10)
+          proxies     =   get_proxies_for_protocol_and_proxy_type(protocol, proxy_type)
+  
+          proxies     =   proxies.any_of(
+            {:last_checked_at.exists => false},
+            {:last_checked_at.ne => nil},
+            {:last_checked_at.exists => true, :last_checked_at.ne => nil, :last_checked_at.lt => date}
+          )
+  
+          proxies     =   proxies.any_of(
+            {:failed_attempts.exists => false},
+            {:failed_attempts.in => ["", nil]},
+            {:failed_attempts.exists => true, :failed_attempts.nin => ["", nil], :failed_attempts.lte => maximum_failed_attempts}
+          )
+  
+          proxies     =   proxies.order_by([[:valid_proxy, :asc], [:failed_attempts, :asc], [:last_checked_at, :asc]])
+          proxies     =   proxies.limit(limit)
+  
+          return proxies
+        end
+      
+        def get_valid_proxies(protocol: :all, proxy_type: :all, maximum_failed_attempts: nil, retries: 3)
+          proxies     =   get_proxies_for_protocol_and_proxy_type(protocol, proxy_type)
+          proxies     =   proxies.where(valid_proxy: true)
+          proxies     =   proxies.where(:failed_attempts.lte => maximum_failed_attempts) if maximum_failed_attempts
+        end
+      
+        def get_random_proxy(protocol: :all, proxy_type: :all, maximum_failed_attempts: nil, retries: 3)
+          proxies     =   get_valid_proxies(protocol: protocol, proxy_type: proxy_type, maximum_failed_attempts: maximum_failed_attempts)
+          proxy       =   nil
+          
+          begin
+            proxy     =   proxies.skip(rand(proxies.count)).first
+          
+          rescue StandardError
+            retries  -=   1
+            retry if retries > 0
+          end
+          
+          return proxy
+        end
+      end
+    
+    end
+    
+  end
+end

data/lib/proxied/railtie.rb

@@ -0,0 +1,11 @@
+require 'rails'
+
+module Proxied
+  class Railtie < Rails::Railtie
+    
+    rake_tasks do
+      Dir[File.join(File.dirname(__FILE__), '../tasks/*.rake')].each { |ext| load ext }
+    end
+    
+  end
+end

data/lib/proxied/shared.rb

@@ -0,0 +1,54 @@
+module Proxied
+  module Shared
+
+    module ClassMethods    
+      def get_proxies_for_protocol_and_proxy_type(protocol, proxy_type)
+        proxies     =   ::Proxy.where(nil)
+        proxies     =   proxies.where(protocol: protocol)     if (protocol && !protocol.downcase.to_sym.eql?(:all))
+        proxies     =   proxies.where(proxy_type: proxy_type) if (proxy_type && !proxy_type.downcase.to_sym.eql?(:all))
+      
+        return proxies
+      end
+
+      def format_proxy_address(proxy_host, proxy_port = 80, include_http = false)
+        proxy_address = "#{proxy_host}:#{proxy_port}"
+        proxy_address.insert(0, "http://") if (include_http && !proxy_address.start_with?("http://"))
+        return proxy_address
+      end
+
+      def format_proxy_credentials(username, password)
+        return "#{username}:#{password}"
+      end
+    end
+
+    module InstanceMethods
+      def proxy_address(include_http = false)
+        return self.class.format_proxy_address(self.host, self.port, include_http)
+      end
+
+      def proxy_credentials
+        return self.class.format_proxy_credentials(self.username, self.password)
+      end
+
+      def socks_proxy_credentials
+        credentials               =   {}
+        
+        credentials[:user]        =   self.username if !self.username.to_s.empty?
+        credentials[:password]    =   self.password if !self.password.to_s.empty?
+
+        return credentials
+      end
+      
+      def proxy_options_for_faraday
+        proxy_options             =   {}
+    
+        proxy_options[:uri]       =   self.class.format_proxy_address(self.host, self.port, true)
+        proxy_options[:user]      =   self.username if !self.username.to_s.empty?
+        proxy_options[:password]  =   self.password if !self.password.to_s.empty?
+    
+        return proxy_options
+      end
+    end
+        
+  end
+end

data/lib/proxied/sql/proxy_methods.rb

@@ -0,0 +1,61 @@
+module Proxied
+  module Sql
+    
+    module ProxyMethods
+    
+      def self.included(base)
+        base.send :extend ::Proxied::Shared::ClassMethods
+        base.send :extend, ClassMethods
+        
+        base.send :include ::Proxied::Shared::InstanceMethods
+      end
+
+      module ClassMethods
+        def should_be_checked(protocol: :all, proxy_type: :all, date: Time.now, limit: 10, maximum_failed_attempts: 10)
+          proxies     =   get_proxies_for_protocol_and_proxy_type(protocol, proxy_type)
+          proxies     =   proxies.where(["(last_checked_at IS NULL OR last_checked_at < ?)", date])
+          proxies     =   proxies.where(["failed_attempts <= ?", maximum_failed_attempts])
+          proxies     =   proxies.order("valid_proxy ASC, failed_attempts ASC, last_checked_at ASC")
+          proxies     =   proxies.limit(limit)
+        
+          return proxies
+        end
+      
+        def get_valid_proxies(protocol: :all, proxy_type: :all, maximum_failed_attempts: nil)
+          proxies     =   get_proxies_for_protocol_and_proxy_type(protocol, proxy_type)
+          proxies     =   proxies.where(["valid_proxy = ? AND last_checked_at IS NOT NULL", true])
+          proxies     =   proxies.where(["failed_attempts <= ?", maximum_failed_attempts]) if maximum_failed_attempts
+
+          return proxies
+        end
+      
+        def get_random_proxy(protocol: :all, proxy_type: :all, maximum_failed_attempts: nil)
+          proxies     =   get_valid_proxies(protocol: protocol, proxy_type: proxy_type, maximum_failed_attempts: maximum_failed_attempts)
+        
+          order_clause = case ActiveRecord::Base.connection.class.name
+            when "ActiveRecord::ConnectionAdapters::MysqlAdapter", "ActiveRecord::ConnectionAdapters::Mysql2Adapter"
+              "RAND() DESC"
+            when "ActiveRecord::ConnectionAdapters::PostgreSQLAdapter"
+              "RANDOM() DESC"
+            when "ActiveRecord::ConnectionAdapters::SQLite3Adapter"
+              "RANDOM() DESC"
+            else
+              "RAND() DESC"
+          end
+        
+          proxies     =   proxies.order(order_clause)
+        
+          proxy       =   nil
+        
+          uncached do
+            proxy     =   proxies.limit(1).first
+          end
+
+          return proxy
+        end
+      end
+    
+    end
+    
+  end
+end

data/lib/proxied/version.rb

@@ -0,0 +1,3 @@
+module Proxied
+  VERSION = "0.1.0"
+end

data/lib/tasks/tasks.rake

@@ -0,0 +1,13 @@
+namespace :proxied do
+  task :check_proxies, [:protocol, :proxy_type, :mode, :maximum_failed_attempts] => :environment do |task, args|
+    args.with_defaults(protocol: :http, proxy_type: :public, mode: :synchronous, maximum_failed_attempts: 10)
+    
+    checker = ::Proxied::Checker.new
+    checker.check_proxies(
+      protocol:                 args.protocol.to_sym, 
+      proxy_type:               args.proxy_type.to_sym, 
+      mode:                     args.mode.to_sym,
+      maximum_failed_attempts:  args.maximum_failed_attempts.to_i
+    )
+  end
+end

data/proxied.gemspec

@@ -0,0 +1,47 @@
+
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "proxied/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "proxied"
+  spec.version       = Proxied::VERSION
+  spec.authors       = ["Sebastian"]
+  spec.email         = ["sebastian.johnsson@gmail.com"]
+
+  spec.summary       = %q{Proxied is a proxy management gem for Ruby/Rails.}
+  spec.description   = %q{Manage HTTP(S) and SOCKS proxies for Rails apps. Supports ActiveRecord and Mongoid as ORM.}
+  spec.homepage      = "https://github.com/SebastianJ/proxied"
+  spec.license       = "MIT"
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+  
+  spec.add_dependency "faraday",                    "~> 0.15.2"
+  spec.add_dependency "net-ssh",                    "~> 5.0.2"
+  
+  spec.add_development_dependency "bundler",        "~> 1.16.2"
+  spec.add_development_dependency "rake",           "~> 12.3.1"
+  spec.add_development_dependency "appraisal",      "~> 2.2"
+  
+  spec.add_development_dependency "virtus",         "~> 1.0", ">= 1.0.5"
+  
+  spec.add_development_dependency "rspec-rails",    '~> 3.7', '>= 3.7.2'
+  spec.add_development_dependency "rspec-core",     '~> 3.7', '>= 3.7.1'
+  spec.add_development_dependency "rspec-expectations", '~> 3.7'
+  spec.add_development_dependency "rspec-mocks",    '~> 3.7'
+  spec.add_development_dependency "rspec-support",  '~> 3.7', '>= 3.7.1'
+  spec.add_development_dependency "generator_spec", "~> 0.9.4"
+  
+  spec.add_development_dependency "rdoc",           "~> 6.0.4"
+  spec.add_development_dependency "vcr",            "~> 4.0"
+  spec.add_development_dependency "webmock",        "~> 3.4.2"
+  
+  spec.add_development_dependency "pry",            "~> 0.11.3"  
+end

metadata

@@ -0,0 +1,332 @@
+--- !ruby/object:Gem::Specification
+name: proxied
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Sebastian
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2018-06-29 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: faraday
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.15.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.15.2
+- !ruby/object:Gem::Dependency
+  name: net-ssh
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 5.0.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 5.0.2
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.16.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.16.2
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 12.3.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 12.3.1
+- !ruby/object:Gem::Dependency
+  name: appraisal
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
+- !ruby/object:Gem::Dependency
+  name: virtus
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.5
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.5
+- !ruby/object:Gem::Dependency
+  name: rspec-rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.7'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.7.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.7'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.7.2
+- !ruby/object:Gem::Dependency
+  name: rspec-core
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.7'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.7.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.7'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.7.1
+- !ruby/object:Gem::Dependency
+  name: rspec-expectations
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.7'
+- !ruby/object:Gem::Dependency
+  name: rspec-mocks
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.7'
+- !ruby/object:Gem::Dependency
+  name: rspec-support
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.7'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.7.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.7'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.7.1
+- !ruby/object:Gem::Dependency
+  name: generator_spec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.9.4
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.9.4
+- !ruby/object:Gem::Dependency
+  name: rdoc
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 6.0.4
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 6.0.4
+- !ruby/object:Gem::Dependency
+  name: vcr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.4.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.4.2
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.11.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.11.3
+description: Manage HTTP(S) and SOCKS proxies for Rails apps. Supports ActiveRecord
+  and Mongoid as ORM.
+email:
+- sebastian.johnsson@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".ruby-gemset"
+- ".ruby-version"
+- ".travis.yml"
+- Appraisals
+- CODE_OF_CONDUCT.md
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- gemfiles/.bundle/config
+- gemfiles/activerecord5.gemfile
+- gemfiles/mongoid7.gemfile
+- lib/.DS_Store
+- lib/generators/active_record/proxied_generator.rb
+- lib/generators/active_record/templates/migration.rb
+- lib/generators/mongoid/proxied_generator.rb
+- lib/generators/proxied/install_generator.rb
+- lib/generators/proxied/orm_helpers.rb
+- lib/generators/proxied/proxied_generator.rb
+- lib/generators/templates/proxied.rb
+- lib/proxied.rb
+- lib/proxied/.DS_Store
+- lib/proxied/checker.rb
+- lib/proxied/configuration.rb
+- lib/proxied/jobs/check_proxies_job.rb
+- lib/proxied/jobs/check_proxy_job.rb
+- lib/proxied/logger.rb
+- lib/proxied/nosql/proxy_methods.rb
+- lib/proxied/railtie.rb
+- lib/proxied/shared.rb
+- lib/proxied/sql/proxy_methods.rb
+- lib/proxied/version.rb
+- lib/tasks/tasks.rake
+- proxied.gemspec
+homepage: https://github.com/SebastianJ/proxied
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.7
+signing_key: 
+specification_version: 4
+summary: Proxied is a proxy management gem for Ruby/Rails.
+test_files: []