proxied 0.1.0

data/bin/console

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "proxied"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+require "pry"
+Pry.config.history.file = File.join(__FILE__, "../.pry_history")
+Pry.start
+
+#require "irb"
+#IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/gemfiles/.bundle/config

@@ -0,0 +1,2 @@
+---
+BUNDLE_RETRY: "1"

data/gemfiles/activerecord5.gemfile

@@ -0,0 +1,11 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "rails", "~> 5.2"
+gem "activerecord", ">= 5.2", require: "active_record"
+gem "actionpack", ">= 5.2"
+gem "activemodel", ">= 5.2"
+gem "railties", ">= 5.2"
+
+gemspec path: "../"

data/gemfiles/mongoid7.gemfile

@@ -0,0 +1,9 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "rails", "~> 5.2"
+gem "mongoid", "~> 7", require: "mongoid"
+gem "bson_ext", platform: "ruby"
+
+gemspec path: "../"

data/lib/generators/active_record/proxied_generator.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+require "rails/generators/active_record"
+require "generators/proxied/orm_helpers"
+
+module ActiveRecord
+  module Generators
+    class ProxiedGenerator < ActiveRecord::Generators::Base
+      class_option :primary_key_type, type: :string, desc: "The type for primary key"
+
+      include Proxied::Generators::OrmHelpers
+      source_root File.expand_path("../templates", __FILE__)
+
+      def copy_proxy_migration
+        migration_template "migration.rb", "#{migration_path}/proxied_create_#{table_name}.rb", migration_version: migration_version
+      end
+
+      def generate_model
+        invoke "active_record:model", [name], migration: false unless model_exists? && behavior == :invoke
+      end
+      
+      def inject_proxied_content
+        content = model_contents
+
+        class_path = if namespaced?
+          class_name.to_s.split("::")
+        else
+          [class_name]
+        end
+
+        indent_depth = class_path.size - 1
+        content = content.split("\n").map { |line| "  " * indent_depth + line } .join("\n") << "\n"
+
+        inject_into_class(model_path, class_path.last, content) if model_exists?
+      end
+      
+      def model_contents
+<<RUBY
+  include ::Proxied::Sql::ProxyMethods
+RUBY
+      end
+
+      def rails5?
+        Rails.version.start_with? '5'
+      end
+
+      def postgresql?
+        config = ActiveRecord::Base.configurations[Rails.env]
+        config && config['adapter'] == 'postgresql'
+      end
+
+     def migration_version
+       if rails5?
+         "[#{Rails::VERSION::MAJOR}.#{Rails::VERSION::MINOR}]"
+       end
+     end
+
+     def primary_key_type
+       primary_key_string if rails5?
+     end
+
+     def primary_key_string
+       key_string = options[:primary_key_type]
+       ", id: :#{key_string}" if key_string
+     end
+    end
+  end
+end

data/lib/generators/active_record/templates/migration.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+class ProxiedCreate<%= table_name.camelize %> < ActiveRecord::Migration<%= migration_version %>
+  def change
+    create_table :<%= table_name %><%= primary_key_type %> do |t|
+      t.string  :host,                  null: false
+      t.integer :port,                  null: false
+      t.string  :username
+      t.string  :password
+      
+      t.string  :protocol,              null: false, default: 'http', index: true
+      t.string  :proxy_type,            null: false, default: 'public', index: true
+      t.string  :category
+      
+      t.datetime :last_checked_at,      index: true
+      
+      t.boolean  :valid_proxy,          null:  false, default: false, index: true
+      t.integer  :successful_attempts,  null:  false, default: 0, index: true
+      t.integer  :failed_attempts,      null:  false, default: 0, index: true
+    end
+    
+    add_index :<%= table_name %>, [:host, :port], unique: true, name: 'index_unique_<%= table_name.singularize %>'
+  end
+end

data/lib/generators/mongoid/proxied_generator.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require "rails/generators/named_base"
+require "generators/proxied/orm_helpers"
+
+module Mongoid
+  module Generators
+    class ProxiedGenerator < Rails::Generators::NamedBase
+      include Proxied::Generators::OrmHelpers
+
+      def generate_model
+        invoke "mongoid:model", [name] unless model_exists? && behavior == :invoke
+      end
+      
+      def inject_proxied_content
+        inject_into_file model_path, model_contents, after: inject_after_pattern(:mongoid) if model_exists?
+      end
+
+      def model_contents
+<<RUBY
+  include Mongoid::Timestamps
+  
+  # Fields
+  field :host, type: String
+  field :port, type: Integer
+  
+  field :username, type: String
+  field :password, type: String
+  
+  field :protocol, type: String, default: :http
+  field :proxy_type, type: String, default: :public
+  field :category, type: String
+  
+  field :valid_proxy, type: Boolean, default: false
+  field :successful_attempts, type: Integer, default: 0
+  field :failed_attempts, type: Integer, default: 0
+  
+  field :last_checked_at, type: DateTime
+  
+  # Validations
+  validates :host, presence: true, uniqueness: {scope: :port}
+  
+  # Indexes
+  index({ host: 1, port: 1 }, { unique: true, drop_dups: true })
+  index({ protocol: 1, proxy_type: 1, valid_proxy: -1, failed_attempts: 1 })
+  index({ valid_proxy: 1 })
+  
+  # Methods
+  include ::Proxied::Nosql::ProxyMethods
+RUBY
+      end
+    end
+  end
+end

data/lib/generators/proxied/install_generator.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require "rails/generators/base"
+
+module Proxied
+  module Generators
+    class InstallGenerator < Rails::Generators::NamedBase
+      include Rails::Generators::ResourceHelpers
+      
+      source_root File.expand_path("../../templates", __FILE__)
+
+      desc "Creates a Proxied initializer for your application."
+
+      def copy_initializer
+        template "proxied.rb", "config/initializers/proxied.rb"
+      end
+      
+    end
+  end
+end

data/lib/generators/proxied/orm_helpers.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Proxied
+  module Generators
+    module OrmHelpers
+
+      private
+      
+      def inject_after_pattern(orm = :active_record)
+        if orm == :active_record
+          /class #{table_name.camelize}\n|class #{table_name.camelize} .*\n|class #{table_name.demodulize.camelize}\n|class #{table_name.demodulize.camelize} .*\n/
+        else
+          /include Mongoid::Document\n|include Mongoid::Document .*\n/
+        end
+      end
+      
+      def model_exists?
+        File.exist?(File.join(destination_root, model_path))
+      end
+
+      def migration_path
+        if Rails.version >= '5.0.3'
+          db_migrate_path
+        else
+          @migration_path ||= File.join("db", "migrate")
+        end
+      end
+
+      def model_path
+        @model_path ||= File.join("app", "models", "#{file_path}.rb")
+      end
+    end
+  end
+end

data/lib/generators/proxied/proxied_generator.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require "rails/generators/named_base"
+
+module Proxied
+  module Generators
+    class ProxiedGenerator < Rails::Generators::NamedBase
+      include Rails::Generators::ResourceHelpers
+      
+      class_option :orm, type: :string, default: "active_record"
+
+      namespace "proxied"
+      source_root File.expand_path("../templates", __FILE__)
+
+      desc "Generates a model with the given NAME (if one does not exist) with devise " \
+           "configuration plus a migration file and devise routes."
+
+      hook_for :orm, required: true
+
+    end
+  end
+end

data/lib/generators/templates/proxied.rb

@@ -0,0 +1,26 @@
+::Proxied.configure do |config|
+  config.proxy_class                  =   <%= class_name.camelize.to_s %> # Must be set to the ActiveRecord or Mongoid model that will be used for managing proxies
+  
+  config.minimum_successful_attempts  =   1
+  config.maximum_failed_attempts      =   10
+  
+  config.faraday          =   {
+    adapter:    :net_http,
+    user_agent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/11.1.1 Safari/605.1.15",
+    verbose:    false
+  }
+  
+  config.http_test        =   {
+    url:      "http://www.google.com/robots.txt",
+    timeout:  10,
+  }
+  
+  config.socks_test       =   {
+    hostname: "whois.verisign-grs.com",
+    port:     43,
+    query:    "=google.com",
+    timeout:  10
+  }
+  
+  config.logger           =   defined?(Rails) ? -> (message) { Rails.logger.info(message) } : -> (message) { puts(message) }
+end

data/lib/proxied.rb

@@ -0,0 +1,43 @@
+# Gems
+require "faraday"
+require "net/ssh/proxy/socks5"
+
+# Standard library
+require "socket"
+
+# Gem
+require "proxied/version"
+
+require "proxied/configuration"
+require "proxied/logger"
+
+require "proxied/shared"
+require "proxied/sql/proxy_methods" if defined?(ActiveRecord)
+require "proxied/nosql/proxy_methods" if defined?(Mongoid)
+
+require "proxied/checker"
+
+if defined?(Sidekiq)
+  require "proxied/jobs/check_proxies_job"
+  require "proxied/jobs/check_proxy_job"
+end
+
+require "proxied/railtie" if defined?(Rails)
+
+module Proxied
+  class << self
+    attr_writer :configuration
+  end
+
+  def self.configuration
+    @configuration ||= ::Proxied::Configuration.new
+  end
+
+  def self.reset
+    @configuration = ::Proxied::Configuration.new
+  end
+
+  def self.configure
+    yield(configuration)
+  end
+end

data/lib/proxied/checker.rb

@@ -0,0 +1,123 @@
+module Proxied
+  class Checker
+    attr_accessor :minimum_successful_attempts, :maximum_failed_attempts
+    attr_accessor :limit
+
+    def initialize(minimum_successful_attempts: ::Proxied.configuration.minimum_successful_attempts, maximum_failed_attempts: ::Proxied.configuration.maximum_failed_attempts, limit: nil)
+      self.minimum_successful_attempts  =   minimum_successful_attempts
+      self.maximum_failed_attempts      =   maximum_failed_attempts
+      self.limit                        =   limit
+    end
+
+    def check_proxies(protocol: :all, proxy_type: :all, mode: :synchronous)
+      proxies                           =   ::Proxied.configuration.proxy_class.should_be_checked(
+        protocol:                 protocol,
+        proxy_type:               proxy_type,
+        date:                     Time.now,
+        limit:                    self.limit,
+        maximum_failed_attempts:  self.maximum_failed_attempts
+      )
+
+      if proxies&.any?
+        ::Proxied::Logger.log "Found #{proxies.size} #{proxy_type} proxies to check."
+
+        proxies.each do |proxy|
+          case mode
+            when :synchronous
+              check_proxy(proxy)
+            when :sidekiq
+              ::Proxied::Jobs::CheckProxyJob.perform_async(proxy.id.to_s)
+          end
+        end
+
+      else
+        ::Proxied::Logger.log "Couldn't find any proxies to check!"
+      end
+    end
+    
+    def check_proxy(proxy, update: true)
+      ::Proxied::Logger.log "#{Time.now}: Will check if proxy #{proxy.proxy_address} is working."
+      
+      self.send("check_#{proxy.protocol}_proxy", proxy, update: update)
+    end
+    
+    def check_socks_proxy(proxy, test_host: ::Proxied.configuration.socks_test[:hostname], test_port: ::Proxied.configuration.socks_test[:port], test_query: ::Proxied.configuration.socks_test[:query], timeout: ::Proxied.configuration.socks_test[:timeout], update: true)
+      valid_proxy     =   false
+
+      begin
+        socks_proxy   =   ::Net::SSH::Proxy::SOCKS5.new(proxy.host, proxy.port, proxy.socks_proxy_credentials)
+        client        =   socks_proxy.open(test_host, test_port, {timeout: timeout})
+
+        client.write("#{test_query}\r\n")
+        response      =   client.read
+        
+        valid_proxy   =   !response.to_s.empty?
+        
+        client.close
+      
+      rescue StandardError => e
+        ::Proxied::Logger.log "Exception occured while trying to check proxy #{proxy.proxy_address}. Error Class: #{e.class}. Error Message: #{e.message}"
+        valid_proxy   =   false
+      end
+      
+      update_proxy(proxy, valid_proxy) if update
+      
+      return valid_proxy
+    end
+    
+    def check_http_proxy(proxy, test_url: ::Proxied.configuration.http_test[:url], timeout: ::Proxied.configuration.http_test[:timeout], update: true)
+      ::Proxied::Logger.log "#{Time.now}: Fetching robots.txt for Google.com with proxy #{proxy.proxy_address}. Using authentication? #{proxy.proxy_credentials}"
+
+      response        =   request(test_url, proxy, options: {timeout: timeout})
+      valid_proxy     =   (response && !(response =~ /Allow: \/search\/about/i).nil?)
+
+      update_proxy(proxy, valid_proxy) if update
+      
+      return valid_proxy
+    end
+    
+    def update_proxy(proxy, valid)
+      ::Proxied::Logger.log "#{Time.now}: Proxy #{proxy.proxy_address} is #{valid ? "working" : "not working"}!"
+      
+      successful_attempts         =   proxy.successful_attempts || 0
+      failed_attempts             =   proxy.failed_attempts || 0
+
+      if valid
+        successful_attempts      +=  1
+      else
+        failed_attempts          +=  1
+      end
+
+      is_valid                    =   (successful_attempts >= self.minimum_successful_attempts && failed_attempts < self.maximum_failed_attempts)
+      
+      proxy.valid_proxy           =   is_valid
+      proxy.successful_attempts   =   successful_attempts
+      proxy.failed_attempts       =   failed_attempts
+      proxy.last_checked_at       =   Time.now
+      proxy.save
+    end
+    
+    private
+      def request(url, proxy, options: {})
+        response                  =   nil
+        
+        user_agent                =   options.fetch(:user_agent, ::Proxied.configuration.faraday.fetch(:user_agent, nil))
+        timeout                   =   options.fetch(:timeout, ::Proxied.configuration.http_test[:timeout])
+        
+        begin
+          response                =   ::Faraday.new(url: url) do |builder|
+            builder.headers["User-Agent"]   =   user_agent if !user_agent.to_s.empty?
+            builder.options[:timeout]       =   timeout if timeout
+            builder.proxy                   =   proxy.proxy_options_for_faraday
+            builder.response :logger if ::Proxied.configuration.verbose_faraday?
+            builder.adapter ::Proxied.configuration.faraday.fetch(:adapter, :net_http)
+          end.get&.body
+        rescue Faraday::Error => e
+          ::Proxied::Logger.log "Exception occured while trying to check proxy #{proxy.proxy_address}. Error Class: #{e.class}. Error Message: #{e.message}"
+        end
+        
+        return response
+      end
+
+  end
+end