proxy_fetcher 0.14.0 → 0.15.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 46d91f40f7c61a9a14aa90530ece0ef42365566ed6055abb141925ac471126d9
-  data.tar.gz: 72f56ef68ee8f46f226bf6de07e5888f485cd533ec2b6b8a64047b8530ec6b3c
+  metadata.gz: bf6591a8ede2c142fca7f433f3edceb407ed884c1dee6a98e3ca33efbae392e2
+  data.tar.gz: 7d7259da718adec21b8d749d04faf594fa926389e570be6c55bbe9ebf3695783
 SHA512:
-  metadata.gz: a9f59f62fd5c731a1d15430faca81468f4a3368eb0287f5fad405ff3379af55886bd4a7608bba5a90f2117e42bb1d08a46ff6e63fe132c72a253f122164cbc4a
-  data.tar.gz: de8fa38b0895a6faf87c7ddac76b80a331bc6e6beb95a7972d94146d4ecaaf1025a4442bb3639d902e979cb57d1c1b6ba03f33137c56369692a5f2ca0b99b28e
+  metadata.gz: 90eed5f390d122ae58f9bd6c7553b7a1ff6bbca8e65c05406593f53b7a00a4ed160120ae62d0ce8289965810aaa3795287769ee9ef12f9404d805694a8ac0cff
+  data.tar.gz: 31a5b4fcd6199dfc75ca3e89c7bddcc82533ca171f86954e50d4ff44716d73f1ad014f921a05e9eb2fd38a83c5e603eec1fbbf9530b4c01153f6780877388501

data/Rakefile

@@ -3,6 +3,8 @@
 require "bundler/gem_tasks"
 
 require "rspec/core/rake_task"
-RSpec::Core::RakeTask.new(:spec)
+RSpec::Core::RakeTask.new(:spec) do |t|
+  t.rspec_opts = '--tag "~unreliable"'
+end
 
 task default: :spec

data/gemfiles/nokogiri.gemfile

@@ -9,5 +9,5 @@ gem "nokogiri", "~> 1.8"
 group :test do
   gem "coveralls", require: false
   gem "evil-proxy", "~> 0.2"
-  gem "rspec", "~> 3.6"
+  gem "rspec", "~> 3.9"
 end

data/gemfiles/oga.gemfile

@@ -4,10 +4,10 @@ source "https://rubygems.org"
 
 gemspec path: "../"
 
-gem "oga", "~> 2.0"
+gem "oga", "~> 3.0"
 
 group :test do
   gem "coveralls", require: false
   gem "evil-proxy", "~> 0.2"
-  gem "rspec", "~> 3.6"
+  gem "rspec", "~> 3.9"
 end

data/lib/proxy_fetcher.rb

@@ -33,13 +33,17 @@ module ProxyFetcher
   # ProxyFetcher providers namespace
   module Providers
     require File.dirname(__FILE__) + "/proxy_fetcher/providers/base"
-    require File.dirname(__FILE__) + "/proxy_fetcher/providers/proxypedia"
     require File.dirname(__FILE__) + "/proxy_fetcher/providers/free_proxy_list"
+    require File.dirname(__FILE__) + "/proxy_fetcher/providers/free_proxy_list_socks"
     require File.dirname(__FILE__) + "/proxy_fetcher/providers/free_proxy_list_ssl"
-    require File.dirname(__FILE__) + "/proxy_fetcher/providers/gather_proxy"
+    require File.dirname(__FILE__) + "/proxy_fetcher/providers/free_proxy_list_us"
     require File.dirname(__FILE__) + "/proxy_fetcher/providers/http_tunnel"
     require File.dirname(__FILE__) + "/proxy_fetcher/providers/mtpro"
     require File.dirname(__FILE__) + "/proxy_fetcher/providers/proxy_list"
+    require File.dirname(__FILE__) + "/proxy_fetcher/providers/proxypedia"
+    require File.dirname(__FILE__) + "/proxy_fetcher/providers/proxyscrape_http"
+    require File.dirname(__FILE__) + "/proxy_fetcher/providers/proxyscrape_socks4"
+    require File.dirname(__FILE__) + "/proxy_fetcher/providers/proxyscrape_socks5"
     require File.dirname(__FILE__) + "/proxy_fetcher/providers/xroxy"
   end
 

data/lib/proxy_fetcher/manager.rb

@@ -47,6 +47,8 @@ module ProxyFetcher
 
       ProxyFetcher.config.providers.each do |provider_name|
         threads << Thread.new do
+          Thread.current.report_on_exception = false
+
           provider = ProxyFetcher::Configuration.providers_registry.class_for(provider_name)
           provider_filters = filters && filters.fetch(provider_name.to_sym, filters)
           provider_proxies = provider.fetch_proxies!(provider_filters)

data/lib/proxy_fetcher/providers/free_proxy_list_socks.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module ProxyFetcher
+  module Providers
+    # FreeProxyListSocks provider class.
+    class FreeProxyListSocks < Base
+      # Provider URL to fetch proxy list
+      def provider_url
+        "https://www.socks-proxy.net/"
+      end
+
+      # [NOTE] Doesn't support filtering
+      def xpath
+        '//table[@id="proxylisttable"]/tbody/tr'
+      end
+
+      # Converts HTML node (entry of N tags) to <code>ProxyFetcher::Proxy</code>
+      # object.
+      #
+      # @param html_node [Object]
+      #   HTML node from the <code>ProxyFetcher::Document</code> DOM model.
+      #
+      # @return [ProxyFetcher::Proxy]
+      #   Proxy object
+      #
+      def to_proxy(html_node)
+        ProxyFetcher::Proxy.new.tap do |proxy|
+          proxy.addr = html_node.content_at("td[1]")
+          proxy.port = Integer(html_node.content_at("td[2]").gsub(/^0+/, ""))
+          proxy.country = html_node.content_at("td[4]")
+          proxy.type = parse_type(html_node)
+          proxy.anonymity = html_node.content_at("td[6]")
+        end
+      end
+
+      private
+
+      # Parses HTML node to extract proxy type.
+      #
+      # @param html_node [Object]
+      #   HTML node from the <code>ProxyFetcher::Document</code> DOM model.
+      #
+      # @return [String]
+      #   Proxy type
+      #
+      def parse_type(html_node)
+        https = html_node.content_at("td[5]")
+
+        return ProxyFetcher::Proxy::SOCKS4 if https&.casecmp("socks4")&.zero?
+        return ProxyFetcher::Proxy::SOCKS5 if https&.casecmp("socks5")&.zero?
+
+        "Unknown"
+      end
+    end
+
+    ProxyFetcher::Configuration.register_provider(:free_proxy_list_socks, FreeProxyListSocks)
+  end
+end

data/lib/proxy_fetcher/providers/free_proxy_list_ssl.rb

@@ -9,6 +9,7 @@ module ProxyFetcher
         "https://www.sslproxies.org/"
       end
 
+      # [NOTE] Doesn't support filtering
       def xpath
         '//table[@id="proxylisttable"]/tbody/tr'
       end

data/lib/proxy_fetcher/providers/free_proxy_list_us.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module ProxyFetcher
+  module Providers
+    # FreeProxyListUS provider class.
+    class FreeProxyListUS < Base
+      # Provider URL to fetch proxy list
+      def provider_url
+        "https://www.us-proxy.org/"
+      end
+
+      # [NOTE] Doesn't support filtering
+      def xpath
+        '//table[@id="proxylisttable"]/tbody/tr'
+      end
+
+      # Converts HTML node (entry of N tags) to <code>ProxyFetcher::Proxy</code>
+      # object.
+      #
+      # @param html_node [Object]
+      #   HTML node from the <code>ProxyFetcher::Document</code> DOM model.
+      #
+      # @return [ProxyFetcher::Proxy]
+      #   Proxy object
+      #
+      def to_proxy(html_node)
+        ProxyFetcher::Proxy.new.tap do |proxy|
+          proxy.addr = html_node.content_at("td[1]")
+          proxy.port = Integer(html_node.content_at("td[2]").gsub(/^0+/, ""))
+          proxy.country = html_node.content_at("td[4]")
+          proxy.anonymity = html_node.content_at("td[5]")
+          proxy.type = parse_type(html_node)
+        end
+      end
+
+      private
+
+      # Parses HTML node to extract proxy type.
+      #
+      # @param html_node [Object]
+      #   HTML node from the <code>ProxyFetcher::Document</code> DOM model.
+      #
+      # @return [String]
+      #   Proxy type
+      #
+      def parse_type(html_node)
+        https = html_node.content_at("td[7]")
+        https&.casecmp("yes")&.zero? ? ProxyFetcher::Proxy::HTTPS : ProxyFetcher::Proxy::HTTP
+      end
+    end
+
+    ProxyFetcher::Configuration.register_provider(:free_proxy_list_us, FreeProxyListUS)
+  end
+end

data/lib/proxy_fetcher/providers/mtpro.rb

@@ -4,7 +4,7 @@ require "json"
 
 module ProxyFetcher
   module Providers
-    # GatherProxy provider class.
+    # MTPro provider class.
     class MTPro < Base
       # Provider URL to fetch proxy list
       def provider_url

data/lib/proxy_fetcher/providers/proxyscrape_http.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require "csv"
+
+module ProxyFetcher
+  module Providers
+    # FreeProxyList provider class.
+    class ProxyscrapeHTTP < Base
+      # Provider URL to fetch proxy list
+      def provider_url
+        "https://api.proxyscrape.com/v2/?request=getproxies&protocol=http"
+      end
+
+      # Loads provider HTML and parses it with internal document object.
+      #
+      # @param url [String]
+      #   URL to fetch
+      #
+      # @param filters [Hash]
+      #   filters for proxy provider
+      #
+      # @return [Array]
+      #   Collection of extracted proxies with ports
+      #
+      def load_document(url, filters = {})
+        html = load_html(url, filters)
+
+        CSV.parse(html, col_sep: "\t").map(&:first)
+      end
+
+      # Fetches HTML content by sending HTTP request to the provider URL and
+      # parses the txt document to return all the proxy entries (ip addresses
+      # and ports).
+      #
+      # @return [Array]
+      #   Collection of extracted proxies with ports
+      #
+      def load_proxy_list(filters = {})
+        load_document(provider_url, filters)
+      end
+
+      # Converts String to <code>ProxyFetcher::Proxy</code> object.
+      #
+      # @param node [String]
+      #   String
+      #
+      # @return [ProxyFetcher::Proxy]
+      #   Proxy object
+      #
+      def to_proxy(node)
+        addr, port = node.split(":")
+
+        ProxyFetcher::Proxy.new.tap do |proxy|
+          proxy.addr = addr
+          proxy.port = Integer(port)
+          proxy.country = "Unknown"
+          proxy.anonymity = "Unknown"
+          proxy.type = ProxyFetcher::Proxy::HTTP
+        end
+      end
+    end
+
+    ProxyFetcher::Configuration.register_provider(:proxyscrape_http, ProxyscrapeHTTP)
+  end
+end

data/lib/proxy_fetcher/providers/proxyscrape_socks4.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require "csv"
+
+module ProxyFetcher
+  module Providers
+    # FreeProxyList provider class.
+    class ProxyscrapeSOCKS4 < Base
+      # Provider URL to fetch proxy list
+      def provider_url
+        "https://api.proxyscrape.com/v2/?request=getproxies&protocol=socks4"
+      end
+
+      # Loads provider HTML and parses it with internal document object.
+      #
+      # @param url [String]
+      #   URL to fetch
+      #
+      # @param filters [Hash]
+      #   filters for proxy provider
+      #
+      # @return [Array]
+      #   Collection of extracted proxies with ports
+      #
+      def load_document(url, filters = {})
+        html = load_html(url, filters)
+
+        CSV.parse(html, col_sep: "\t").map(&:first)
+      end
+
+      # Fetches HTML content by sending HTTP request to the provider URL and
+      # parses the txt document to return all the proxy entries (ip addresses
+      # and ports).
+      #
+      # @return [Array]
+      #   Collection of extracted proxies with ports
+      #
+      def load_proxy_list(filters = {})
+        load_document(provider_url, filters)
+      end
+
+      # Converts String to <code>ProxyFetcher::Proxy</code> object.
+      #
+      # @param node [String]
+      #   String
+      #
+      # @return [ProxyFetcher::Proxy]
+      #   Proxy object
+      #
+      def to_proxy(html_node)
+        addr, port = html_node.split(":")
+
+        ProxyFetcher::Proxy.new.tap do |proxy|
+          proxy.addr = addr
+          proxy.port = Integer(port)
+          proxy.country = "Unknown"
+          proxy.anonymity = "Unknown"
+          proxy.type = ProxyFetcher::Proxy::SOCKS4
+        end
+      end
+    end
+
+    ProxyFetcher::Configuration.register_provider(:proxyscrape_socks4, ProxyscrapeSOCKS4)
+  end
+end

data/lib/proxy_fetcher/providers/proxyscrape_socks5.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require "csv"
+
+module ProxyFetcher
+  module Providers
+    # FreeProxyList provider class.
+    class ProxyscrapeSOCKS5 < Base
+      # Provider URL to fetch proxy list
+      def provider_url
+        "https://api.proxyscrape.com/v2/?request=getproxies&protocol=socks5"
+      end
+
+      # Loads provider HTML and parses it with internal document object.
+      #
+      # @param url [String]
+      #   URL to fetch
+      #
+      # @param filters [Hash]
+      #   filters for proxy provider
+      #
+      # @return [Array]
+      #   Collection of extracted proxies with ports
+      #
+      def load_document(url, filters = {})
+        html = load_html(url, filters)
+
+        CSV.parse(html, col_sep: "\t").map(&:first)
+      end
+
+      # Fetches HTML content by sending HTTP request to the provider URL and
+      # parses the txt document to return all the proxy entries (ip addresses
+      # and ports).
+      #
+      # @return [Array]
+      #   Collection of extracted proxies with ports
+      #
+      def load_proxy_list(filters = {})
+        load_document(provider_url, filters)
+      end
+
+      # Converts String to <code>ProxyFetcher::Proxy</code> object.
+      #
+      # @param node [String]
+      #   String
+      #
+      # @return [ProxyFetcher::Proxy]
+      #   Proxy object
+      #
+      def to_proxy(html_node)
+        addr, port = html_node.split(":")
+
+        ProxyFetcher::Proxy.new.tap do |proxy|
+          proxy.addr = addr
+          proxy.port = Integer(port)
+          proxy.country = "Unknown"
+          proxy.anonymity = "Unknown"
+          proxy.type = ProxyFetcher::Proxy::SOCKS5
+        end
+      end
+    end
+
+    ProxyFetcher::Configuration.register_provider(:proxyscrape_socks5, ProxyscrapeSOCKS5)
+  end
+end

data/lib/proxy_fetcher/providers/xroxy.rb

@@ -6,7 +6,7 @@ module ProxyFetcher
     class XRoxy < Base
       # Provider URL to fetch proxy list
       def provider_url
-        "https://madison.xroxy.com/proxylist.html"
+        "https://www.xroxy.com/proxylist.htm"
       end
 
       def xpath

data/lib/proxy_fetcher/utils/http_client.rb

@@ -71,17 +71,10 @@ module ProxyFetcher
       response = process_http_request
       response.body.to_s
     rescue StandardError => e
-      ProxyFetcher.logger.warn("Failed to process request to #{url} (#{e.message})")
+      ProxyFetcher.config.logger.warn("Failed to process request to #{url} (#{e.message})")
       ""
     end
 
-    def fetch_with_headers
-      process_http_request
-    rescue StandardError => e
-      ProxyFetcher.logger.warn("Failed to process request to #{url} (#{e.message})")
-      HTTP::Response.new(version: "1.1", status: 500, body: "")
-    end
-
     protected
 
     def process_http_request(http_method: method, http_params: params)

data/lib/proxy_fetcher/utils/proxy_validator.rb

@@ -10,27 +10,39 @@ module ProxyFetcher
 
     # Short variant to validate proxy.
     #
-    # @param proxy_addr [String] proxy address or IP
-    # @param proxy_port [String, Integer] proxy port
+    # @param address [String] proxy address or IP
+    # @param port [String, Integer] proxy port
     #
     # @return [Boolean]
     #   true if connection to the server using proxy established, otherwise false
     #
-    def self.connectable?(proxy_addr, proxy_port)
-      new(proxy_addr, proxy_port).connectable?
+    def self.connectable?(address, port)
+      new(address, port).connectable?
     end
 
     # Initialize new ProxyValidator instance
     #
-    # @param proxy_addr [String] proxy address or IP
-    # @param proxy_port [String, Integer] proxy port
+    # @param address [String] Proxy address or IP
+    # @param port [String, Integer] Proxy port
+    # @param options [Hash] proxy options
+    #   @option username [String] Proxy authentication username
+    #   @option password [String] Proxy authentication password
+    #   @option headers [Hash] Proxy headers
     #
     # @return [ProxyValidator]
     #
-    def initialize(proxy_addr, proxy_port)
+    def initialize(address, port, options: {})
       timeout = ProxyFetcher.config.proxy_validation_timeout
+      proxy = [address, port.to_i]
 
-      @http = HTTP.follow.via(proxy_addr, proxy_port.to_i).timeout(connect: timeout, read: timeout)
+      if options[:username] && options[:password]
+        proxy << options[:username]
+        proxy << options[:password]
+      end
+
+      proxy << options[:headers].to_h if options[:headers]
+
+      @http = HTTP.follow.via(*proxy).timeout(connect: timeout, read: timeout)
     end
 
     # Checks if proxy is connectable (can be used to connect

data/lib/proxy_fetcher/version.rb

@@ -13,7 +13,7 @@ module ProxyFetcher
     # Major version number
     MAJOR = 0
     # Minor version number
-    MINOR = 14
+    MINOR = 15
     # Smallest version number
     TINY  = 0
 

data/proxy_fetcher.gemspec

@@ -8,9 +8,11 @@ Gem::Specification.new do |gem|
   gem.name = "proxy_fetcher"
   gem.version = ProxyFetcher.gem_version
   gem.summary = "Ruby gem for dealing with proxy lists from different providers"
-  gem.description = "This gem can help your Ruby application to make HTTP(S) requests " \
-                    "using proxies by fetching and validating proxy lists from " \
-                    "the different providers."
+  gem.description = <<-TEXT.strip.gsub(/[\s\n]+/, " ")
+    This gem can help your Ruby application to make HTTP(S) requests 
+    using proxies by fetching and validating proxy lists from 
+    the different providers.
+  TEXT
   gem.authors = ["Nikita Bulai"]
   gem.email = "bulajnikita@gmail.com"
   gem.require_paths = ["lib"]
@@ -24,5 +26,5 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency "http", ">= 3", "< 5"
 
   gem.add_development_dependency "rake", ">= 12.0"
-  gem.add_development_dependency "rspec", "~> 3.5"
+  gem.add_development_dependency "rspec", "~> 3.9"
 end

data/spec/proxy_fetcher/client/client_spec.rb

@@ -118,15 +118,15 @@ describe ProxyFetcher::Client do
     end
   end
 
-  xcontext "retries" do
+  context "retries" do
     it "raises an error when reaches max retries limit" do
       allow(ProxyFetcher::Client::Request).to receive(:execute).and_raise(StandardError)
 
-      expect { ProxyFetcher::Client.get("http://httpbin.org") }
+      expect { ProxyFetcher::Client.get("http://httpbin.org", options: { max_retries: 10 }) }
         .to raise_error(ProxyFetcher::Exceptions::MaximumRetriesReached)
     end
 
-    it "raises an error when http request returns an error" do
+    xit "raises an error when http request returns an error" do
       allow_any_instance_of(HTTP::Client).to receive(:get).and_return(StandardError.new)
 
       expect { ProxyFetcher::Client.get("http://httpbin.org") }
@@ -134,14 +134,14 @@ describe ProxyFetcher::Client do
     end
 
     it "refreshes proxy lists if no proxy found" do
-      ProxyFetcher::Client::ProxiesRegistry.manager.instance_variable_set(:"@proxies", [])
+      allow(ProxyFetcher::Manager.new).to receive(:proxies).and_return([])
 
       expect { ProxyFetcher::Client.get("http://httpbin.org") }
         .not_to raise_error
     end
   end
 
-  context "redirects" do
+  xcontext "redirects" do
     it "follows redirect when present" do
       content = ProxyFetcher::Client.get("http://httpbin.org/absolute-redirect/2")
 

data/spec/proxy_fetcher/providers/proxy_classes_spec.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require "spec_helper"
+
+describe "Proxy classes" do
+  [
+    [:free_proxy_list,       "FreeProxyList"],
+    [:free_proxy_list_socks, "FreeProxyListSocks"],
+    [:free_proxy_list_ssl,   "FreeProxyListSSL"],
+    [:free_proxy_list_us,    "FreeProxyListUS"],
+    [:http_tunnel,           "HTTPTunnel"],
+    [:mtpro,                 "MTPro"],
+    [:proxy_list,            "ProxyList"],
+    [:proxypedia,            "Proxypedia"],
+    [:proxyscrape_http,      "ProxyscrapeHTTP"],
+    [:proxyscrape_socks4,    "ProxyscrapeSOCKS4"],
+    [:proxyscrape_socks5,    "ProxyscrapeSOCKS5"],
+    [:xroxy,                 "XRoxy"]
+  ].each do |(provider_name, provider_klass)|
+    describe Object.const_get("ProxyFetcher::Providers::#{provider_klass}") do
+      before :all do
+        ProxyFetcher.config.provider = provider_name
+      end
+
+      it_behaves_like "a manager"
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: proxy_fetcher
 version: !ruby/object:Gem::Version
-  version: 0.14.0
+  version: 0.15.0
 platform: ruby
 authors:
 - Nikita Bulai
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-05-11 00:00:00.000000000 Z
+date: 2021-01-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: http
@@ -50,14 +50,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.5'
+        version: '3.9'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.5'
+        version: '3.9'
 description: This gem can help your Ruby application to make HTTP(S) requests using
   proxies by fetching and validating proxy lists from the different providers.
 email: bulajnikita@gmail.com
@@ -94,12 +94,16 @@ files:
 - lib/proxy_fetcher/null_logger.rb
 - lib/proxy_fetcher/providers/base.rb
 - lib/proxy_fetcher/providers/free_proxy_list.rb
+- lib/proxy_fetcher/providers/free_proxy_list_socks.rb
 - lib/proxy_fetcher/providers/free_proxy_list_ssl.rb
-- lib/proxy_fetcher/providers/gather_proxy.rb
+- lib/proxy_fetcher/providers/free_proxy_list_us.rb
 - lib/proxy_fetcher/providers/http_tunnel.rb
 - lib/proxy_fetcher/providers/mtpro.rb
 - lib/proxy_fetcher/providers/proxy_list.rb
 - lib/proxy_fetcher/providers/proxypedia.rb
+- lib/proxy_fetcher/providers/proxyscrape_http.rb
+- lib/proxy_fetcher/providers/proxyscrape_socks4.rb
+- lib/proxy_fetcher/providers/proxyscrape_socks5.rb
 - lib/proxy_fetcher/providers/xroxy.rb
 - lib/proxy_fetcher/proxy.rb
 - lib/proxy_fetcher/utils/http_client.rb
@@ -114,15 +118,8 @@ files:
 - spec/proxy_fetcher/document/node_spec.rb
 - spec/proxy_fetcher/manager_spec.rb
 - spec/proxy_fetcher/providers/base_spec.rb
-- spec/proxy_fetcher/providers/free_proxy_list_spec.rb
-- spec/proxy_fetcher/providers/free_proxy_list_ssl_spec.rb
-- spec/proxy_fetcher/providers/gather_proxy_spec.rb
-- spec/proxy_fetcher/providers/http_tunnel_spec.rb
-- spec/proxy_fetcher/providers/mtpro_spec.rb
 - spec/proxy_fetcher/providers/multiple_providers_spec.rb
-- spec/proxy_fetcher/providers/proxy_list_spec.rb
-- spec/proxy_fetcher/providers/proxypedia_spec.rb
-- spec/proxy_fetcher/providers/xroxy_spec.rb
+- spec/proxy_fetcher/providers/proxy_classes_spec.rb
 - spec/proxy_fetcher/proxy_spec.rb
 - spec/proxy_fetcher/version_spec.rb
 - spec/spec_helper.rb
@@ -146,7 +143,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.2
+rubyforge_project: 
+rubygems_version: 2.7.9
 signing_key: 
 specification_version: 4
 summary: Ruby gem for dealing with proxy lists from different providers

data/lib/proxy_fetcher/providers/gather_proxy.rb

@@ -1,50 +0,0 @@
-# frozen_string_literal: true
-
-require "json"
-
-module ProxyFetcher
-  module Providers
-    # GatherProxy provider class.
-    class GatherProxy < Base
-      # Provider URL to fetch proxy list
-      def provider_url
-        "https://proxygather.com"
-      end
-
-      def xpath
-        '//div[@class="proxy-list"]/table/script'
-      end
-
-      # Converts HTML node (entry of N tags) to <code>ProxyFetcher::Proxy</code>
-      # object.
-      #
-      # @param html_node [Object]
-      #   HTML node from the <code>ProxyFetcher::Document</code> DOM model.
-      #
-      # @return [ProxyFetcher::Proxy]
-      #   Proxy object
-      #
-      def to_proxy(html_node)
-        json = parse_json(html_node)
-
-        ProxyFetcher::Proxy.new.tap do |proxy|
-          proxy.addr = json["PROXY_IP"]
-          proxy.port = json["PROXY_PORT"].to_i(16)
-          proxy.anonymity = json["PROXY_TYPE"]
-          proxy.country = json["PROXY_COUNTRY"]
-          proxy.response_time = json["PROXY_TIME"].to_i
-          proxy.type = ProxyFetcher::Proxy::HTTP
-        end
-      end
-
-      private
-
-      def parse_json(html_node)
-        javascript = html_node.content[/{.+}/im]
-        JSON.parse(javascript)
-      end
-    end
-
-    ProxyFetcher::Configuration.register_provider(:gather_proxy, GatherProxy)
-  end
-end

data/spec/proxy_fetcher/providers/free_proxy_list_spec.rb

@@ -1,13 +0,0 @@
-# frozen_string_literal: true
-
-require "spec_helper"
-
-describe ProxyFetcher::Providers::FreeProxyList do
-  before :all do
-    ProxyFetcher.configure do |config|
-      config.provider = :free_proxy_list
-    end
-  end
-
-  it_behaves_like "a manager"
-end

data/spec/proxy_fetcher/providers/free_proxy_list_ssl_spec.rb

@@ -1,11 +0,0 @@
-# frozen_string_literal: true
-
-require "spec_helper"
-
-describe ProxyFetcher::Providers::FreeProxyListSSL do
-  before :all do
-    ProxyFetcher.config.provider = :free_proxy_list_ssl
-  end
-
-  it_behaves_like "a manager"
-end

data/spec/proxy_fetcher/providers/gather_proxy_spec.rb

@@ -1,11 +0,0 @@
-# frozen_string_literal: true
-
-require "spec_helper"
-
-describe ProxyFetcher::Providers::GatherProxy do
-  before :all do
-    ProxyFetcher.config.provider = :gather_proxy
-  end
-
-  it_behaves_like "a manager"
-end

data/spec/proxy_fetcher/providers/http_tunnel_spec.rb

@@ -1,11 +0,0 @@
-# frozen_string_literal: true
-
-require "spec_helper"
-
-describe ProxyFetcher::Providers::HTTPTunnel do
-  before :all do
-    ProxyFetcher.config.provider = :http_tunnel
-  end
-
-  it_behaves_like "a manager"
-end

data/spec/proxy_fetcher/providers/mtpro_spec.rb

@@ -1,11 +0,0 @@
-# frozen_string_literal: true
-
-require "spec_helper"
-
-describe ProxyFetcher::Providers::MTPro do
-  before :all do
-    ProxyFetcher.config.provider = :mtpro
-  end
-
-  it_behaves_like "a manager"
-end

data/spec/proxy_fetcher/providers/proxy_list_spec.rb

@@ -1,11 +0,0 @@
-# frozen_string_literal: true
-
-require "spec_helper"
-
-describe ProxyFetcher::Providers::ProxyList do
-  before :all do
-    ProxyFetcher.config.provider = :proxy_list
-  end
-
-  it_behaves_like "a manager"
-end

data/spec/proxy_fetcher/providers/proxypedia_spec.rb

@@ -1,11 +0,0 @@
-# frozen_string_literal: true
-
-require "spec_helper"
-
-describe ProxyFetcher::Providers::Proxypedia do
-  before :all do
-    ProxyFetcher.config.provider = :proxypedia
-  end
-
-  it_behaves_like "a manager"
-end

data/spec/proxy_fetcher/providers/xroxy_spec.rb

@@ -1,11 +0,0 @@
-# frozen_string_literal: true
-
-require "spec_helper"
-
-describe ProxyFetcher::Providers::XRoxy do
-  before :all do
-    ProxyFetcher.config.provider = :xroxy
-  end
-
-  it_behaves_like "a manager"
-end