proxied 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4b63589961f4cbc4d46a03c5c97d88171f5a86af86e6a58f47042336e38fd58d
-  data.tar.gz: e3ae83908238864b37d9d6d78ba1a914157898cf9430fc946d39e8480d7ff5ab
+  metadata.gz: c3225511d5ca688c62c4c41138f7dfa135b766a1ca22de15f638c76a4bc805e6
+  data.tar.gz: ddaa33c4df250b5b868eb98262b0c87dc3c0ad099fd9343989f77c18e16a8d74
 SHA512:
-  metadata.gz: feb8cbe47e0563025c8c497242a5ab3aa982be099a6b457d3bee2dae64f710ec2d67e368eb25de5f0fdaee0249c924c073a9c2a9d1b6d3c3dc9bcc2654eb7448
-  data.tar.gz: a0799134b33788cc919808258a3add09fd881fb7a082de3a62ca91182c831ac4a23fa90827673cd436802537c5f9b021219e32dab2fc196ad1c74a64070dc3e5
+  metadata.gz: 107c196e83c9a87d27674c89cb6f6384b5c69a689e3c2c0d818f0241bddec129180bbc39e0e16599abc15e1eb10e76e308e5aa038c13478fd2e74b5ba57b435d
+  data.tar.gz: 9d5d1cfed81dad98ec888d571e2e1cfcac30cc8e0d6898e60c4edad01538eada8936621363c016b6e0c75998d714771cd8e3bcd6a864794f332c1659a025063f

data/README.md

@@ -32,10 +32,27 @@ Generate the migration and model:
 
     $ rails generate proxied MODEL
 
+## Features
+
+Features:
+
+* ActiveRecord model and migration or Mongoid model for managing your proxy data.
+
+* Common model methods for selecting random proxies (based on protocol, etc.) for both ActiveRecord and Mongoid.
+
+* A proxy checker to check the status for HTTP(S) and Socks proxies (with configurable test setups).
+
+* An importer that will bulk import proxies for you.
+
+* Sidekiq jobs to asynchronously check proxies.
 
 ## Usage
 
-- TODO -
+1. Set up the gem according to the installation instructions.
+2. Import proxies.
+3. Schedule the rake task (proxied:check_proxies) using cron or schedule the Sidekiq job (Proxied::Jobs::CheckProxiesJob) using a scheduler for Sidekiq.
+4. Use the model methods to query for valid proxies (the maximum failed attempts etc. are configurable using the global Proxied.configure-method).
+5. Use the proxies/proxy from the resulting query to perform HTTP(S) or SOCKS requests using the proxy. Some helper methods, e.g. #proxy_options_for_faraday are included to help with integration with some common gems.
 
 ## Development
 
@@ -54,7 +71,7 @@ In order to test specific specs: ADAPTER=mongoid bundle exec appraisal mongoid7
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/proxied. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
+Bug reports and pull requests are welcome on GitHub at https://github.com/SebastianJ/proxied. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
 
 ## License
 
@@ -62,4 +79,4 @@ The gem is available as open source under the terms of the [MIT License](https:/
 
 ## Code of Conduct
 
-Everyone interacting in the Proxied project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/[USERNAME]/proxied/blob/master/CODE_OF_CONDUCT.md).
+Everyone interacting in the Proxied project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/SebastianJ/proxied/blob/master/CODE_OF_CONDUCT.md).

data/lib/proxied/checker.rb

@@ -1,15 +1,17 @@
 module Proxied
   class Checker
+    attr_accessor :mode
     attr_accessor :minimum_successful_attempts, :maximum_failed_attempts
     attr_accessor :limit
 
-    def initialize(minimum_successful_attempts: ::Proxied.configuration.minimum_successful_attempts, maximum_failed_attempts: ::Proxied.configuration.maximum_failed_attempts, limit: nil)
+    def initialize(mode: :synchronous, minimum_successful_attempts: ::Proxied.configuration.minimum_successful_attempts, maximum_failed_attempts: ::Proxied.configuration.maximum_failed_attempts, limit: nil)
+      self.mode                         =   mode
       self.minimum_successful_attempts  =   minimum_successful_attempts
       self.maximum_failed_attempts      =   maximum_failed_attempts
       self.limit                        =   limit
     end
 
-    def check_proxies(protocol: :all, proxy_type: :all, mode: :synchronous, update: true)
+    def check_proxies(protocol: :all, proxy_type: :all, update: true)
       proxies                           =   ::Proxied.configuration.proxy_class.constantize.should_be_checked(
         protocol:                 protocol,
         proxy_type:               proxy_type,
@@ -22,10 +24,10 @@ module Proxied
         ::Proxied::Logger.log "Found #{proxies.size} #{proxy_type} proxies to check."
 
         proxies.each do |proxy|
-          case mode
+          case self.mode.to_sym
             when :synchronous
               check_proxy(proxy, update: update)
-            when :sidekiq
+            when :asynchronous
               ::Proxied::Jobs::CheckProxyJob.perform_async(proxy.id.to_s)
           end
         end
@@ -66,7 +68,7 @@ module Proxied
     end
     
     def check_http_proxy(proxy, test_url: ::Proxied.configuration.http_test[:url], timeout: ::Proxied.configuration.http_test[:timeout], update: true)
-      ::Proxied::Logger.log "#{Time.now}: Fetching robots.txt for Google.com with proxy #{proxy.proxy_address}. Using authentication? #{proxy.proxy_credentials}"
+      ::Proxied::Logger.log "#{Time.now}: Fetching robots.txt for Google.com with proxy #{proxy.proxy_address}."
 
       response        =   request(test_url, proxy, options: {timeout: timeout})
       valid_proxy     =   (response && !(response =~ /Allow: \/search\/about/i).nil?)

data/lib/proxied/configuration.rb

@@ -16,7 +16,7 @@ module Proxied
       self.faraday          =   {
         adapter:    :net_http,
         user_agent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/11.1.1 Safari/605.1.15",
-        verbose:    true
+        verbose:    false
       }
       
       self.http_test        =   {

data/lib/proxied/jobs/check_proxies_job.rb

@@ -4,8 +4,8 @@ module Proxied
       include ::Sidekiq::Worker
       sidekiq_options queue: ::Proxied.configuration.job_queue
 
-      def perform(protocol = :all, proxy_type = :all, mode = :synchronous)
-        ::Proxied::Checker.new.check_proxies(protocol: protocol.to_sym, proxy_type: proxy_type.to_sym, mode: mode.to_sym, update: true)
+      def perform(protocol = :all, proxy_type = :all, mode = :asynchronous)
+        ::Proxied::Checker.new(mode: mode.to_sym).check_proxies(protocol: protocol.to_sym, proxy_type: proxy_type.to_sym, update: true)
       end
     end
   end

data/lib/proxied/version.rb

@@ -1,3 +1,3 @@
 module Proxied
-  VERSION = "0.1.2"
+  VERSION = "0.1.3"
 end

data/lib/tasks/tasks.rake

@@ -2,12 +2,11 @@ namespace :proxied do
   task :check_proxies, [:protocol, :proxy_type, :mode, :maximum_failed_attempts] => :environment do |task, args|
     args.with_defaults(protocol: :http, proxy_type: :public, mode: :synchronous, maximum_failed_attempts: 10)
     
-    checker = ::Proxied::Checker.new
+    checker = ::Proxied::Checker.new(mode: args.mode.to_sym, maximum_failed_attempts: args.maximum_failed_attempts.to_i)
     checker.check_proxies(
-      protocol:                 args.protocol.to_sym, 
-      proxy_type:               args.proxy_type.to_sym, 
-      mode:                     args.mode.to_sym,
-      maximum_failed_attempts:  args.maximum_failed_attempts.to_i
+      protocol:     args.protocol.to_sym, 
+      proxy_type:   args.proxy_type.to_sym, 
+      
     )
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: proxied
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - Sebastian