proxy_fetcher 0.10.2 → 0.15.0

data/lib/proxy_fetcher/utils/http_client.rb

@@ -70,22 +70,17 @@ module ProxyFetcher
     def fetch
       response = process_http_request
       response.body.to_s
-    rescue StandardError => error
-      ProxyFetcher.logger.warn("Failed to process request to #{url} (#{error.message})")
-      ''
-    end
-
-    def fetch_with_headers
-      process_http_request
-    rescue StandardError => error
-      ProxyFetcher.logger.warn("Failed to process request to #{url} (#{error.message})")
-       HTTP::Response.new(version: '1.1', status: 500, body: '')
+    rescue StandardError => e
+      ProxyFetcher.config.logger.warn("Failed to process request to #{url} (#{e.message})")
+      ""
     end
 
     protected
 
     def process_http_request(http_method: method, http_params: params)
-      raise ArgumentError, 'wrong http method name!' unless HTTP::Request::METHODS.include?(http_method)
+      unless HTTP::Request::METHODS.include?(http_method)
+        raise ArgumentError, "'#{http_method}' is a wrong HTTP method name!"
+      end
 
       http.public_send(
         http_method.to_sym, url,
@@ -101,7 +96,7 @@ module ProxyFetcher
     #
     def default_headers
       {
-        'User-Agent' => ProxyFetcher.config.user_agent
+        "User-Agent" => ProxyFetcher.config.user_agent
       }
     end
   end

data/lib/proxy_fetcher/utils/proxy_list_validator.rb

@@ -34,7 +34,9 @@ module ProxyFetcher
             proxy = target_proxies_lock.synchronize { target_proxies.shift }
             break unless proxy
 
-            connectable_proxies_lock.synchronize { connectable_proxies << proxy } if proxy.connectable?
+            if proxy.connectable?
+              connectable_proxies_lock.synchronize { connectable_proxies << proxy }
+            end
           end
         end
       end

data/lib/proxy_fetcher/utils/proxy_validator.rb

@@ -6,31 +6,43 @@ module ProxyFetcher
   # URL to check if proxy can be used (aka connectable?).
   class ProxyValidator
     # Default URL that will be used to check if proxy can be used.
-    URL_TO_CHECK = 'https://google.com'.freeze
+    URL_TO_CHECK = "https://google.com"
 
     # Short variant to validate proxy.
     #
-    # @param proxy_addr [String] proxy address or IP
-    # @param proxy_port [String, Integer] proxy port
+    # @param address [String] proxy address or IP
+    # @param port [String, Integer] proxy port
     #
     # @return [Boolean]
     #   true if connection to the server using proxy established, otherwise false
     #
-    def self.connectable?(proxy_addr, proxy_port)
-      new(proxy_addr, proxy_port).connectable?
+    def self.connectable?(address, port)
+      new(address, port).connectable?
     end
 
     # Initialize new ProxyValidator instance
     #
-    # @param proxy_addr [String] proxy address or IP
-    # @param proxy_port [String, Integer] proxy port
+    # @param address [String] Proxy address or IP
+    # @param port [String, Integer] Proxy port
+    # @param options [Hash] proxy options
+    #   @option username [String] Proxy authentication username
+    #   @option password [String] Proxy authentication password
+    #   @option headers [Hash] Proxy headers
     #
     # @return [ProxyValidator]
     #
-    def initialize(proxy_addr, proxy_port)
+    def initialize(address, port, options: {})
       timeout = ProxyFetcher.config.proxy_validation_timeout
+      proxy = [address, port.to_i]
 
-      @http = HTTP.follow.via(proxy_addr, proxy_port.to_i).timeout(connect: timeout, read: timeout)
+      if options[:username] && options[:password]
+        proxy << options[:username]
+        proxy << options[:password]
+      end
+
+      proxy << options[:headers].to_h if options[:headers]
+
+      @http = HTTP.follow.via(*proxy).timeout(connect: timeout, read: timeout)
     end
 
     # Checks if proxy is connectable (can be used to connect

data/lib/proxy_fetcher/version.rb

@@ -13,11 +13,11 @@ module ProxyFetcher
     # Major version number
     MAJOR = 0
     # Minor version number
-    MINOR = 10
+    MINOR = 15
     # Smallest version number
-    TINY  = 2
+    TINY  = 0
 
     # Full version number
-    STRING = [MAJOR, MINOR, TINY].compact.join('.')
+    STRING = [MAJOR, MINOR, TINY].compact.join(".")
   end
 end

data/proxy_fetcher.gemspec

@@ -1,25 +1,30 @@
-$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), 'lib'))
+# frozen_string_literal: true
 
-require 'proxy_fetcher/version'
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), "lib"))
+
+require "proxy_fetcher/version"
 
 Gem::Specification.new do |gem|
-  gem.name = 'proxy_fetcher'
+  gem.name = "proxy_fetcher"
   gem.version = ProxyFetcher.gem_version
-  gem.summary = 'Ruby gem for dealing with proxy lists from different providers'
-  gem.description = 'This gem can help your Ruby application to make HTTP(S) requests ' \
-                    'using proxies by fetching and validating proxy lists from the different providers.'
-  gem.authors = ['Nikita Bulai']
-  gem.email = 'bulajnikita@gmail.com'
-  gem.require_paths = ['lib']
-  gem.bindir = 'bin'
+  gem.summary = "Ruby gem for dealing with proxy lists from different providers"
+  gem.description = <<-TEXT.strip.gsub(/[\s\n]+/, " ")
+    This gem can help your Ruby application to make HTTP(S) requests 
+    using proxies by fetching and validating proxy lists from 
+    the different providers.
+  TEXT
+  gem.authors = ["Nikita Bulai"]
+  gem.email = "bulajnikita@gmail.com"
+  gem.require_paths = ["lib"]
+  gem.bindir = "bin"
   gem.files = `git ls-files`.split($RS) - %w[README.md .travis.yml .rubocop.yml]
   gem.executables = `git ls-files -- bin/*`.split("\n").map { |f| File.basename(f) }
-  gem.homepage = 'http://github.com/nbulaj/proxy_fetcher'
-  gem.license = 'MIT'
-  gem.required_ruby_version = '>= 2.0.0'
+  gem.homepage = "http://github.com/nbulaj/proxy_fetcher"
+  gem.license = "MIT"
+  gem.required_ruby_version = ">= 2.3.0"
 
-  gem.add_runtime_dependency 'http', '~> 3.0'
+  gem.add_runtime_dependency "http", ">= 3", "< 5"
 
-  gem.add_development_dependency 'rake', '>= 12.0'
-  gem.add_development_dependency 'rspec', '~> 3.5'
+  gem.add_development_dependency "rake", ">= 12.0"
+  gem.add_development_dependency "rspec", "~> 3.9"
 end

data/spec/fixtures/proxies.txt

@@ -0,0 +1,14 @@
+139.162.59.9:3128
+176.62.187.158:56351
+144.217.22.142:8080
+176.55.108.21:3128
+157.225.214.251:3128
+202.51.49.52:48298
+104.244.75.26:8080
+163.172.28.20:80
+187.56.191.12:3128
+129.194.12.26:80
+178.128.39.39:8080
+181.30.28.15:8080
+51.181.96.185:8080
+148.134.10.13

data/spec/proxy_fetcher/client/client_spec.rb

@@ -1,19 +1,22 @@
 # frozen_string_literal: true
 
-require 'spec_helper'
-require 'json'
+require "spec_helper"
+require "json"
 
-require 'evil-proxy'
-require 'evil-proxy/async'
+require "evil-proxy"
+require "evil-proxy/async"
 
-xdescribe ProxyFetcher::Client do
+describe ProxyFetcher::Client do
   before :all do
     ProxyFetcher.configure do |config|
       config.provider = :xroxy
       config.client_timeout = 5
+      config.logger = ProxyFetcher::NullLogger.new
     end
 
-    @server = EvilProxy::MITMProxyServer.new Port: 3128, Quiet: true
+    quiet = ENV.key?("LOG_MITM") ? ENV["LOG_MITM"] == "false" : true
+
+    @server = EvilProxy::MITMProxyServer.new Port: 3128, Quiet: quiet
     @server.start
   end
 
@@ -21,121 +24,133 @@ xdescribe ProxyFetcher::Client do
     @server.shutdown
   end
 
+  let(:local_proxy) { ProxyFetcher::Proxy.new(addr: "127.0.0.1", port: 3128, type: "HTTP, HTTPS") }
+
   # Use local proxy server in order to avoid side effects, non-working proxies, etc
   before :each do
-    proxy = ProxyFetcher::Proxy.new(addr: '127.0.0.1', port: 3128, type: 'HTTP, HTTPS')
-    ProxyFetcher::Client::ProxiesRegistry.manager.instance_variable_set(:'@proxies', [proxy])
-    allow_any_instance_of(ProxyFetcher::Providers::Base).to receive(:fetch_proxies!).and_return([proxy])
+    ProxyFetcher::Client::ProxiesRegistry.manager.instance_variable_set(:'@proxies', [local_proxy])
+    allow_any_instance_of(ProxyFetcher::Providers::Base).to receive(:fetch_proxies).and_return([local_proxy])
   end
 
-  context 'GET request with the valid proxy' do
-    it 'successfully returns page content for HTTP' do
-      content = ProxyFetcher::Client.get('http://httpbin.org')
+  context "GET request with the valid proxy" do
+    it "successfully returns page content for HTTP" do
+      content = ProxyFetcher::Client.get("http://httpbin.org/get")
 
       expect(content).not_to be_empty
     end
 
-    it 'successfully returns page content for HTTPS' do
-      content = ProxyFetcher::Client.get('https://httpbin.org')
+    # TODO: oh this SSL / MITM proxies ....
+    xit "successfully returns page content for HTTPS" do
+      content = ProxyFetcher::Client.get("https://httpbin.org/get")
 
       expect(content).not_to be_empty
     end
 
-    it 'successfully returns page content using custom proxy' do
-      manager = ProxyFetcher::Manager.new
-
-      proxy = manager.get! until proxy
-      content = ProxyFetcher::Client.get('http://httpbin.org', options: { proxy: proxy })
+    it "successfully returns page content using custom proxy" do
+      content = ProxyFetcher::Client.get("http://httpbin.org/get", options: { proxy: local_proxy })
 
       expect(content).not_to be_empty
     end
   end
 
-  context 'POST request with the valid proxy' do
-    it 'successfully returns page content for HTTP' do
+  context "POST request with the valid proxy" do
+    it "successfully returns page content for HTTP" do
       headers = {
-        'X-Proxy-Fetcher-Version' => ProxyFetcher::VERSION::STRING
+        "X-Proxy-Fetcher-Version" => ProxyFetcher::VERSION::STRING
       }
-      content = ProxyFetcher::Client.post('http://httpbin.org/post', { param: 'value' } , headers: headers)
+
+      content = ProxyFetcher::Client.post(
+        "http://httpbin.org/post",
+        { param: "value" },
+        headers: headers
+      )
 
       expect(content).not_to be_empty
 
       json = JSON.parse(content)
 
-      expect(json['headers']['X-Proxy-Fetcher-Version']).to eq(ProxyFetcher::VERSION::STRING)
-      expect(json['headers']['User-Agent']).to eq(ProxyFetcher.config.user_agent)
+      expect(json["headers"]["X-Proxy-Fetcher-Version"]).to eq(ProxyFetcher::VERSION::STRING)
+      expect(json["headers"]["User-Agent"]).to eq(ProxyFetcher.config.user_agent)
     end
   end
 
-  context 'PUT request with the valid proxy' do
-    it 'successfully returns page content for HTTP' do
-      content = ProxyFetcher::Client.put('http://httpbin.org/put', 'param=PutValue')
+  # TODO: EvilProxy incompatible with latest Ruby/Webrick
+  # @see https://github.com/bbtfr/evil-proxy/issues/10
+  if Gem::Version.new(RUBY_VERSION) < Gem::Version.new("2.6")
+    context "PUT request with the valid proxy" do
+      it "successfully returns page content for HTTP" do
+        content = ProxyFetcher::Client.put("http://httpbin.org/put", "param=PutValue")
 
-      expect(content).not_to be_empty
+        expect(content).not_to be_empty
 
-      json = JSON.parse(content)
+        json = JSON.parse(content)
 
-      expect(json['data']).to eq('param=PutValue')
+        expect(json["form"]["param"]).to eq("PutValue")
+      end
     end
-  end
 
-  context 'PATCH request with the valid proxy' do
-    it 'successfully returns page content for HTTP' do
-      content = ProxyFetcher::Client.patch('http://httpbin.org/patch', param: 'value')
+    context "PATCH request with the valid proxy" do
+      it "successfully returns page content for HTTP" do
+        content = ProxyFetcher::Client.patch("http://httpbin.org/patch", param: "value")
 
-      expect(content).not_to be_empty
+        expect(content).not_to be_empty
 
-      json = JSON.parse(content)
+        json = JSON.parse(content)
 
-      expect(json['form']['param']).to eq('value')
+        expect(json["form"]["param"]).to eq("value")
+      end
     end
   end
 
-  context 'DELETE request with the valid proxy' do
-    it 'successfully returns page content for HTTP' do
-      content = ProxyFetcher::Client.delete('http://httpbin.org/delete')
+  context "DELETE request with the valid proxy" do
+    it "successfully returns page content for HTTP" do
+      content = ProxyFetcher::Client.delete("http://httpbin.org/delete")
 
       expect(content).not_to be_empty
     end
   end
 
-  context 'HEAD request with the valid proxy' do
-    it 'successfully works' do
-      content = ProxyFetcher::Client.head('http://httpbin.org')
+  context "HEAD request with the valid proxy" do
+    it "successfully works" do
+      content = ProxyFetcher::Client.head("http://httpbin.org")
 
       expect(content).to be_empty
     end
   end
 
-  context 'retries' do
-    it 'raises an error when reaches max retries limit' do
+  context "retries" do
+    it "raises an error when reaches max retries limit" do
       allow(ProxyFetcher::Client::Request).to receive(:execute).and_raise(StandardError)
 
-      expect { ProxyFetcher::Client.get('http://httpbin.org') }.to raise_error(ProxyFetcher::Exceptions::MaximumRetriesReached)
+      expect { ProxyFetcher::Client.get("http://httpbin.org", options: { max_retries: 10 }) }
+        .to raise_error(ProxyFetcher::Exceptions::MaximumRetriesReached)
     end
 
-    it 'raises an error when http request returns an error' do
+    xit "raises an error when http request returns an error" do
       allow_any_instance_of(HTTP::Client).to receive(:get).and_return(StandardError.new)
 
-      expect { ProxyFetcher::Client.get('http://httpbin.org') }.to raise_error(ProxyFetcher::Exceptions::MaximumRetriesReached)
+      expect { ProxyFetcher::Client.get("http://httpbin.org") }
+        .to raise_error(ProxyFetcher::Exceptions::MaximumRetriesReached)
     end
 
-    it 'refreshes proxy lists if no proxy found' do
-      ProxyFetcher::Client::ProxiesRegistry.manager.instance_variable_set(:'@proxies', [])
+    it "refreshes proxy lists if no proxy found" do
+      allow(ProxyFetcher::Manager.new).to receive(:proxies).and_return([])
 
-      expect { ProxyFetcher::Client.get('http://httpbin.org') }.not_to raise_error
+      expect { ProxyFetcher::Client.get("http://httpbin.org") }
+        .not_to raise_error
     end
   end
 
-  context 'redirects' do
-    it 'follows redirect when present' do
-      content = ProxyFetcher::Client.get('http://httpbin.org/absolute-redirect/2')
+  xcontext "redirects" do
+    it "follows redirect when present" do
+      content = ProxyFetcher::Client.get("http://httpbin.org/absolute-redirect/2")
 
       expect(content).not_to be_empty
     end
 
-    it 'raises an error when reaches max redirects limit' do
-      expect { ProxyFetcher::Client.get('http://httpbin.org/absolute-redirect/11') }.to raise_error(ProxyFetcher::Exceptions::MaximumRedirectsReached)
+    it "raises an error when reaches max redirects limit" do
+      expect { ProxyFetcher::Client.get("http://httpbin.org/absolute-redirect/11") }
+        .to raise_error(ProxyFetcher::Exceptions::MaximumRedirectsReached)
     end
   end
 end

data/spec/proxy_fetcher/configuration_spec.rb

@@ -1,13 +1,13 @@
 # frozen_string_literal: true
 
-require 'spec_helper'
+require "spec_helper"
 
 describe ProxyFetcher::Configuration do
   before { ProxyFetcher.config.reset! }
   after { ProxyFetcher.config.reset! }
 
-  context 'custom HTTP client' do
-    it 'successfully setups if class has all the required methods' do
+  context "custom HTTP client" do
+    it "successfully setups if class has all the required methods" do
       class MyHTTPClient
         def self.fetch(url)
           url
@@ -17,7 +17,7 @@ describe ProxyFetcher::Configuration do
       expect { ProxyFetcher.config.http_client = MyHTTPClient }.not_to raise_error
     end
 
-    it 'failed on setup if required methods are missing' do
+    it "failed on setup if required methods are missing" do
       MyWrongHTTPClient = Class.new
 
       expect { ProxyFetcher.config.http_client = MyWrongHTTPClient }
@@ -25,8 +25,8 @@ describe ProxyFetcher::Configuration do
     end
   end
 
-  context 'custom proxy validator' do
-    it 'successfully setups if class has all the required methods' do
+  context "custom proxy validator" do
+    it "successfully setups if class has all the required methods" do
       class MyProxyValidator
         def self.connectable?(*)
           true
@@ -36,7 +36,7 @@ describe ProxyFetcher::Configuration do
       expect { ProxyFetcher.config.proxy_validator = MyProxyValidator }.not_to raise_error
     end
 
-    it 'failed on setup if required methods are missing' do
+    it "failed on setup if required methods are missing" do
       MyWrongProxyValidator = Class.new
 
       expect { ProxyFetcher.config.proxy_validator = MyWrongProxyValidator }
@@ -44,8 +44,8 @@ describe ProxyFetcher::Configuration do
     end
   end
 
-  context 'custom provider' do
-    it 'fails on registration if provider class already registered' do
+  context "custom provider" do
+    it "fails on registration if provider class already registered" do
       expect { ProxyFetcher::Configuration.register_provider(:xroxy, Class.new) }
         .to raise_error(ProxyFetcher::Exceptions::RegisteredProvider)
     end
@@ -58,13 +58,13 @@ describe ProxyFetcher::Configuration do
     end
   end
 
-  context 'custom HTML parsing adapter' do
+  context "custom HTML parsing adapter" do
     it "fails if adapter can't be installed" do
       old_adapter = ProxyFetcher.config.adapter.dup
 
       class CustomAdapter < ProxyFetcher::Document::AbstractAdapter
         def self.install_requirements!
-          require 'not_existing_gem'
+          require "not_existing_gem"
         end
       end