proxy_fetcher 0.11.0 → 0.15.1

data/lib/proxy_fetcher/utils/proxy_validator.rb

@@ -10,27 +10,39 @@ module ProxyFetcher
 
     # Short variant to validate proxy.
     #
-    # @param proxy_addr [String] proxy address or IP
-    # @param proxy_port [String, Integer] proxy port
+    # @param address [String] proxy address or IP
+    # @param port [String, Integer] proxy port
     #
     # @return [Boolean]
     #   true if connection to the server using proxy established, otherwise false
     #
-    def self.connectable?(proxy_addr, proxy_port)
-      new(proxy_addr, proxy_port).connectable?
+    def self.connectable?(address, port)
+      new(address, port).connectable?
     end
 
     # Initialize new ProxyValidator instance
     #
-    # @param proxy_addr [String] proxy address or IP
-    # @param proxy_port [String, Integer] proxy port
+    # @param address [String] Proxy address or IP
+    # @param port [String, Integer] Proxy port
+    # @param options [Hash] proxy options
+    #   @option username [String] Proxy authentication username
+    #   @option password [String] Proxy authentication password
+    #   @option headers [Hash] Proxy headers
     #
     # @return [ProxyValidator]
     #
-    def initialize(proxy_addr, proxy_port)
+    def initialize(address, port, options: {})
       timeout = ProxyFetcher.config.proxy_validation_timeout
+      proxy = [address, port.to_i]
 
-      @http = HTTP.follow.via(proxy_addr, proxy_port.to_i).timeout(connect: timeout, read: timeout)
+      if options[:username] && options[:password]
+        proxy << options[:username]
+        proxy << options[:password]
+      end
+
+      proxy << options[:headers].to_h if options[:headers]
+
+      @http = HTTP.follow.via(*proxy).timeout(connect: timeout, read: timeout)
     end
 
     # Checks if proxy is connectable (can be used to connect

data/lib/proxy_fetcher/version.rb

@@ -13,9 +13,9 @@ module ProxyFetcher
     # Major version number
     MAJOR = 0
     # Minor version number
-    MINOR = 11
+    MINOR = 15
     # Smallest version number
-    TINY  = 0
+    TINY  = 1
 
     # Full version number
     STRING = [MAJOR, MINOR, TINY].compact.join(".")

data/proxy_fetcher.gemspec

@@ -8,9 +8,11 @@ Gem::Specification.new do |gem|
   gem.name = "proxy_fetcher"
   gem.version = ProxyFetcher.gem_version
   gem.summary = "Ruby gem for dealing with proxy lists from different providers"
-  gem.description = "This gem can help your Ruby application to make HTTP(S) requests " \
-                    "using proxies by fetching and validating proxy lists from " \
-                    "the different providers."
+  gem.description = <<-TEXT.strip.gsub(/[\s\n]+/, " ")
+    This gem can help your Ruby application to make HTTP(S) requests
+    using proxies by fetching and validating proxy lists from
+    the different providers.
+  TEXT
   gem.authors = ["Nikita Bulai"]
   gem.email = "bulajnikita@gmail.com"
   gem.require_paths = ["lib"]
@@ -24,5 +26,5 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency "http", ">= 3", "< 5"
 
   gem.add_development_dependency "rake", ">= 12.0"
-  gem.add_development_dependency "rspec", "~> 3.5"
+  gem.add_development_dependency "rspec", "~> 3.9"
 end

data/spec/fixtures/proxies.txt

@@ -0,0 +1,14 @@
+139.162.59.9:3128
+176.62.187.158:56351
+144.217.22.142:8080
+176.55.108.21:3128
+157.225.214.251:3128
+202.51.49.52:48298
+104.244.75.26:8080
+163.172.28.20:80
+187.56.191.12:3128
+129.194.12.26:80
+178.128.39.39:8080
+181.30.28.15:8080
+51.181.96.185:8080
+148.134.10.13

data/spec/proxy_fetcher/client/client_spec.rb

@@ -3,6 +3,11 @@
 require "spec_helper"
 require "json"
 
+begin
+  require "webrick"
+rescue LoadError
+  # nop
+end
 require "evil-proxy"
 require "evil-proxy/async"
 
@@ -118,15 +123,15 @@ describe ProxyFetcher::Client do
     end
   end
 
-  xcontext "retries" do
+  context "retries" do
     it "raises an error when reaches max retries limit" do
       allow(ProxyFetcher::Client::Request).to receive(:execute).and_raise(StandardError)
 
-      expect { ProxyFetcher::Client.get("http://httpbin.org") }
+      expect { ProxyFetcher::Client.get("http://httpbin.org", options: { max_retries: 10 }) }
         .to raise_error(ProxyFetcher::Exceptions::MaximumRetriesReached)
     end
 
-    it "raises an error when http request returns an error" do
+    xit "raises an error when http request returns an error" do
       allow_any_instance_of(HTTP::Client).to receive(:get).and_return(StandardError.new)
 
       expect { ProxyFetcher::Client.get("http://httpbin.org") }
@@ -134,14 +139,14 @@ describe ProxyFetcher::Client do
     end
 
     it "refreshes proxy lists if no proxy found" do
-      ProxyFetcher::Client::ProxiesRegistry.manager.instance_variable_set(:"@proxies", [])
+      allow(ProxyFetcher::Manager.new).to receive(:proxies).and_return([])
 
       expect { ProxyFetcher::Client.get("http://httpbin.org") }
         .not_to raise_error
     end
   end
 
-  context "redirects" do
+  xcontext "redirects" do
     it "follows redirect when present" do
       content = ProxyFetcher::Client.get("http://httpbin.org/absolute-redirect/2")
 

data/spec/proxy_fetcher/manager_spec.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require "spec_helper"
+
+describe ProxyFetcher::Manager do
+  it "can initialize with a proxies from file(s)" do
+    manager = described_class.new(refresh: false, file: "spec/fixtures/proxies.txt")
+
+    expect(manager.proxies.size).to be(14)
+
+    manager = described_class.new(
+      refresh: false,
+      file: ["spec/fixtures/proxies.txt", "spec/fixtures/proxies.txt"]
+    )
+
+    expect(manager.proxies.size).to be(14)
+  end
+end

data/spec/proxy_fetcher/providers/proxy_classes_spec.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require "spec_helper"
+
+describe "Proxy classes" do
+  [
+    [:free_proxy_list,       "FreeProxyList"],
+    [:free_proxy_list_socks, "FreeProxyListSocks"],
+    [:free_proxy_list_ssl,   "FreeProxyListSSL"],
+    [:free_proxy_list_us,    "FreeProxyListUS"],
+    [:http_tunnel,           "HTTPTunnel"],
+    [:mtpro,                 "MTPro"],
+    [:proxy_list,            "ProxyList"],
+    [:proxypedia,            "Proxypedia"],
+    [:proxyscrape_http,      "ProxyscrapeHTTP"],
+    [:proxyscrape_socks4,    "ProxyscrapeSOCKS4"],
+    [:proxyscrape_socks5,    "ProxyscrapeSOCKS5"],
+    [:xroxy,                 "XRoxy"]
+  ].each do |(provider_name, provider_klass)|
+    describe Object.const_get("ProxyFetcher::Providers::#{provider_klass}") do
+      before :all do
+        ProxyFetcher.config.provider = provider_name
+      end
+
+      it_behaves_like "a manager"
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: proxy_fetcher
 version: !ruby/object:Gem::Version
-  version: 0.11.0
+  version: 0.15.1
 platform: ruby
 authors:
 - Nikita Bulai
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-10-24 00:00:00.000000000 Z
+date: 2021-02-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: http
@@ -50,14 +50,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.5'
+        version: '3.9'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.5'
+        version: '3.9'
 description: This gem can help your Ruby application to make HTTP(S) requests using
   proxies by fetching and validating proxy lists from the different providers.
 email: bulajnikita@gmail.com
@@ -94,10 +94,16 @@ files:
 - lib/proxy_fetcher/null_logger.rb
 - lib/proxy_fetcher/providers/base.rb
 - lib/proxy_fetcher/providers/free_proxy_list.rb
+- lib/proxy_fetcher/providers/free_proxy_list_socks.rb
 - lib/proxy_fetcher/providers/free_proxy_list_ssl.rb
-- lib/proxy_fetcher/providers/gather_proxy.rb
+- lib/proxy_fetcher/providers/free_proxy_list_us.rb
 - lib/proxy_fetcher/providers/http_tunnel.rb
+- lib/proxy_fetcher/providers/mtpro.rb
 - lib/proxy_fetcher/providers/proxy_list.rb
+- lib/proxy_fetcher/providers/proxypedia.rb
+- lib/proxy_fetcher/providers/proxyscrape_http.rb
+- lib/proxy_fetcher/providers/proxyscrape_socks4.rb
+- lib/proxy_fetcher/providers/proxyscrape_socks5.rb
 - lib/proxy_fetcher/providers/xroxy.rb
 - lib/proxy_fetcher/proxy.rb
 - lib/proxy_fetcher/utils/http_client.rb
@@ -105,18 +111,15 @@ files:
 - lib/proxy_fetcher/utils/proxy_validator.rb
 - lib/proxy_fetcher/version.rb
 - proxy_fetcher.gemspec
+- spec/fixtures/proxies.txt
 - spec/proxy_fetcher/client/client_spec.rb
 - spec/proxy_fetcher/configuration_spec.rb
 - spec/proxy_fetcher/document/adapters_spec.rb
 - spec/proxy_fetcher/document/node_spec.rb
+- spec/proxy_fetcher/manager_spec.rb
 - spec/proxy_fetcher/providers/base_spec.rb
-- spec/proxy_fetcher/providers/free_proxy_list_spec.rb
-- spec/proxy_fetcher/providers/free_proxy_list_ssl_spec.rb
-- spec/proxy_fetcher/providers/gather_proxy_spec.rb
-- spec/proxy_fetcher/providers/http_tunnel_spec.rb
 - spec/proxy_fetcher/providers/multiple_providers_spec.rb
-- spec/proxy_fetcher/providers/proxy_list_spec.rb
-- spec/proxy_fetcher/providers/xroxy_spec.rb
+- spec/proxy_fetcher/providers/proxy_classes_spec.rb
 - spec/proxy_fetcher/proxy_spec.rb
 - spec/proxy_fetcher/version_spec.rb
 - spec/spec_helper.rb
@@ -140,7 +143,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: Ruby gem for dealing with proxy lists from different providers

data/lib/proxy_fetcher/providers/gather_proxy.rb

@@ -1,50 +0,0 @@
-# frozen_string_literal: true
-
-require "json"
-
-module ProxyFetcher
-  module Providers
-    # GatherProxy provider class.
-    class GatherProxy < Base
-      # Provider URL to fetch proxy list
-      def provider_url
-        "http://www.gatherproxy.com/"
-      end
-
-      def xpath
-        '//div[@class="proxy-list"]/table/script'
-      end
-
-      # Converts HTML node (entry of N tags) to <code>ProxyFetcher::Proxy</code>
-      # object.
-      #
-      # @param html_node [Object]
-      #   HTML node from the <code>ProxyFetcher::Document</code> DOM model.
-      #
-      # @return [ProxyFetcher::Proxy]
-      #   Proxy object
-      #
-      def to_proxy(html_node)
-        json = parse_json(html_node)
-
-        ProxyFetcher::Proxy.new.tap do |proxy|
-          proxy.addr = json["PROXY_IP"]
-          proxy.port = json["PROXY_PORT"].to_i(16)
-          proxy.anonymity = json["PROXY_TYPE"]
-          proxy.country = json["PROXY_COUNTRY"]
-          proxy.response_time = json["PROXY_TIME"].to_i
-          proxy.type = ProxyFetcher::Proxy::HTTP
-        end
-      end
-
-      private
-
-      def parse_json(html_node)
-        javascript = html_node.content[/{.+}/im]
-        JSON.parse(javascript)
-      end
-    end
-
-    ProxyFetcher::Configuration.register_provider(:gather_proxy, GatherProxy)
-  end
-end

data/spec/proxy_fetcher/providers/free_proxy_list_spec.rb

@@ -1,13 +0,0 @@
-# frozen_string_literal: true
-
-require "spec_helper"
-
-describe ProxyFetcher::Providers::FreeProxyList do
-  before :all do
-    ProxyFetcher.configure do |config|
-      config.provider = :free_proxy_list
-    end
-  end
-
-  it_behaves_like "a manager"
-end

data/spec/proxy_fetcher/providers/free_proxy_list_ssl_spec.rb

@@ -1,11 +0,0 @@
-# frozen_string_literal: true
-
-require "spec_helper"
-
-describe ProxyFetcher::Providers::FreeProxyListSSL do
-  before :all do
-    ProxyFetcher.config.provider = :free_proxy_list_ssl
-  end
-
-  it_behaves_like "a manager"
-end

data/spec/proxy_fetcher/providers/gather_proxy_spec.rb

@@ -1,11 +0,0 @@
-# frozen_string_literal: true
-
-require "spec_helper"
-
-describe ProxyFetcher::Providers::GatherProxy do
-  before :all do
-    ProxyFetcher.config.provider = :gather_proxy
-  end
-
-  it_behaves_like "a manager"
-end

data/spec/proxy_fetcher/providers/http_tunnel_spec.rb

@@ -1,11 +0,0 @@
-# frozen_string_literal: true
-
-require "spec_helper"
-
-describe ProxyFetcher::Providers::HTTPTunnel do
-  before :all do
-    ProxyFetcher.config.provider = :http_tunnel
-  end
-
-  it_behaves_like "a manager"
-end

data/spec/proxy_fetcher/providers/proxy_list_spec.rb

@@ -1,11 +0,0 @@
-# frozen_string_literal: true
-
-require "spec_helper"
-
-describe ProxyFetcher::Providers::ProxyList do
-  before :all do
-    ProxyFetcher.config.provider = :proxy_list
-  end
-
-  it_behaves_like "a manager"
-end

data/spec/proxy_fetcher/providers/xroxy_spec.rb

@@ -1,11 +0,0 @@
-# frozen_string_literal: true
-
-require "spec_helper"
-
-describe ProxyFetcher::Providers::XRoxy do
-  before :all do
-    ProxyFetcher.config.provider = :xroxy
-  end
-
-  it_behaves_like "a manager"
-end