proxy_fetcher 0.13.0 → 0.16.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '09772e9bc018d8accb01401b2e8b1897804e0a948c67a7eeeef8b1a2e9fcd245'
-  data.tar.gz: 16b49036c2ecdd06e23f53c4fe55982ec7ceb1531e9e223f31590eb83417c4e0
+  metadata.gz: 985e3e5cffffa62960dbe04510e232b5a8e652119acb2cc787824e0834f10870
+  data.tar.gz: beb50433ddf5d298ca1c7c45af357395cb24f87b695231159b3b52ad8429cd8b
 SHA512:
-  metadata.gz: 526c8fdcfb1171c09f2e9bf6a4e13dbbc6e837963ce1c1ef7e2e48ed1ec3052e3dbc6a4d04bb33a0b3f083914149c22af4b4238a6c558825a91e3f26e112a378
-  data.tar.gz: 8dddea87295d1825910e4eecb4a5a16836b06e0070cc07f823a4902d6952550ae3b016bfb2e29d0ce8cc16ba3add12ad28a6ab6fec8eccd8a1656e368d88be5d
+  metadata.gz: 28ee05704cdcf8dc48881119a208dad97408d3c50f34b691e6062ca6a8697f3ffe58ba5056c0c8d07f8fec9f5f977927877341572716fe6db3a28876b537264d
+  data.tar.gz: c0533aa9584e02300734385ae8552c5e50d9836f7647337eb797eb11aa50b0edd93bc6a54eb3b22059ac4eed26adbb8ea21dcca9285d9bceff019c67a797146c

data/.github/dependabot.yml

@@ -0,0 +1,8 @@
+version: 2
+updates:
+- package-ecosystem: bundler
+  directory: "/"
+  schedule:
+    interval: daily
+    time: "03:00"
+  open-pull-requests-limit: 10

data/.github/workflows/ci.yml

@@ -0,0 +1,58 @@
+name: CI
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    name: >-
+      Ruby ${{ matrix.ruby }} (${{ matrix.gemfile }})
+    env:
+      CI: true
+    runs-on: ${{ matrix.os }}
+    continue-on-error: ${{ endsWith(matrix.ruby, 'head') || matrix.ruby == 'debug' || matrix.experimental }}
+    if: |
+      !(   contains(github.event.pull_request.title,  '[ci skip]')
+        || contains(github.event.pull_request.title,  '[skip ci]'))
+    strategy:
+      fail-fast: true
+      matrix:
+        experimental: [false]
+        os: [ ubuntu-latest ]
+        ruby:
+          - 2.6
+          - 2.7
+          - '3.0'
+          - '3.1'
+        gemfile:
+          - gemfiles/oga.gemfile
+          - gemfiles/nokogiri.gemfile
+        include:
+          - ruby: head
+            os: ubuntu-latest
+            gemfile: gemfiles/nokogiri.gemfile
+            experimental: true
+          - ruby: head
+          os: ubuntu-latest
+          gemfile: gemfiles/oga.gemfile
+          experimental: true
+          - ruby: jruby
+            os: ubuntu-latest
+            gemfile: gemfiles/nokogiri.gemfile
+            experimental: true
+          - ruby: truffleruby
+            os: ubuntu-latest
+            gemfile: gemfiles/nokogiri.gemfile
+            experimental: true
+    steps:
+      - name: Repo checkout
+        uses: actions/checkout@v2
+
+      - name: Setup Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true
+
+      - name: Run tests
+        timeout-minutes: 10
+        run: bundle exec rake spec

data/CHANGELOG.md

@@ -4,7 +4,22 @@ Reverse Chronological Order:
 
 ## `master`
 
-* Add your description here
+...
+
+## `0.15.1` (2021-02-17)
+
+* Support for Ruby 3.0
+
+## `0.15.0` (2021-01-26)
+
+* Removed failing providers
+* Added new
+* Specs refactoring
+
+## `0.14.0` (2020-05-11)
+
+* Add MTPro provider
+* Add Proxypedia provider
 
 ## `0.13.0` (2020-03-09)
 

data/Gemfile

@@ -6,9 +6,11 @@ gemspec
 
 gem "nokogiri", "~> 1.8"
 gem "oga", "~> 3.2"
-gem "rubocop", "~> 0.74"
+gem "rubocop", "~> 1.0"
 
 group :test do
-  gem "coveralls", require: false
+  gem "coveralls_reborn", require: false
+  # Until I find a way to introduce other MITM proxy
+  gem "webrick", "1.4.2"
   gem "evil-proxy", "~> 0.2"
 end

data/Rakefile

@@ -3,6 +3,8 @@
 require "bundler/gem_tasks"
 
 require "rspec/core/rake_task"
-RSpec::Core::RakeTask.new(:spec)
+RSpec::Core::RakeTask.new(:spec) do |t|
+  t.rspec_opts = '--tag "~unreliable"'
+end
 
 task default: :spec

data/gemfiles/nokogiri.gemfile

@@ -7,7 +7,7 @@ gemspec path: "../"
 gem "nokogiri", "~> 1.8"
 
 group :test do
-  gem "coveralls", require: false
+  gem "coveralls_reborn", require: false
+  gem "webrick"
   gem "evil-proxy", "~> 0.2"
-  gem "rspec", "~> 3.6"
 end

data/gemfiles/oga.gemfile

@@ -4,10 +4,10 @@ source "https://rubygems.org"
 
 gemspec path: "../"
 
-gem "oga", "~> 2.0"
+gem "oga", "~> 3.0"
 
 group :test do
-  gem "coveralls", require: false
+  gem "coveralls_reborn", require: false
+  gem "webrick"
   gem "evil-proxy", "~> 0.2"
-  gem "rspec", "~> 3.6"
 end

data/lib/proxy_fetcher/client/request.rb

@@ -41,15 +41,15 @@ module ProxyFetcher
       # @return [String]
       #   response body (requested resource content)
       #
-      def self.execute(args)
-        new(args).execute
+      def self.execute(**args)
+        new(**args).execute
       end
 
       # Initialize new HTTP request
       #
       # @return [Request]
       #
-      def initialize(args)
+      def initialize(**args)
         raise ArgumentError, "args must be a Hash!" unless args.is_a?(Hash)
 
         @url = args.fetch(:url)

data/lib/proxy_fetcher/configuration.rb

@@ -114,7 +114,7 @@ module ProxyFetcher
 
     # Sets default configuration options
     def reset!
-      @logger = Logger.new(STDOUT)
+      @logger = Logger.new($stdout)
       @user_agent = DEFAULT_USER_AGENT
       @pool_size = 10
       @client_timeout = 3

data/lib/proxy_fetcher/document/node.rb

@@ -83,7 +83,7 @@ module ProxyFetcher
       def clear(text)
         return "" if text.nil? || text.empty?
 
-        text.strip.gsub(/[\t]/i, "")
+        text.strip.gsub(/\t/i, "")
       end
     end
   end

data/lib/proxy_fetcher/manager.rb

@@ -47,6 +47,8 @@ module ProxyFetcher
 
       ProxyFetcher.config.providers.each do |provider_name|
         threads << Thread.new do
+          Thread.current.report_on_exception = false
+
           provider = ProxyFetcher::Configuration.providers_registry.class_for(provider_name)
           provider_filters = filters && filters.fetch(provider_name.to_sym, filters)
           provider_proxies = provider.fetch_proxies!(provider_filters)

data/lib/proxy_fetcher/providers/free_proxy_list.rb

@@ -11,7 +11,7 @@ module ProxyFetcher
 
       # [NOTE] Doesn't support filtering
       def xpath
-        '//table[@id="proxylisttable"]/tbody/tr'
+        "//table[./thead/tr/th[contains(text(), 'IP')]]/tbody/tr"
       end
 
       # Converts HTML node (entry of N tags) to <code>ProxyFetcher::Proxy</code>
@@ -45,27 +45,6 @@ module ProxyFetcher
       #
       def parse_type(html_node)
         https = html_node.content_at("td[6]")
-        # frozen_string_literal: true
-        # FreeProxyList provider class.
-        # Provider URL to fetch proxy list
-        # [NOTE] Doesn't support filtering
-        # Converts HTML node (entry of N tags) to <code>ProxyFetcher::Proxy</code>
-        # object.
-        #
-        # @param html_node [Object]
-        #   HTML node from the <code>ProxyFetcher::Document</code> DOM model.
-        #
-        # @return [ProxyFetcher::Proxy]
-        #   Proxy object
-        #
-        # Parses HTML node to extract proxy type.
-        #
-        # @param html_node [Object]
-        #   HTML node from the <code>ProxyFetcher::Document</code> DOM model.
-        #
-        # @return [String]
-        #   Proxy type
-        #
         https&.casecmp("yes")&.zero? ? ProxyFetcher::Proxy::HTTPS : ProxyFetcher::Proxy::HTTP
       end
     end

data/lib/proxy_fetcher/providers/free_proxy_list_socks.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module ProxyFetcher
+  module Providers
+    # FreeProxyListSocks provider class.
+    class FreeProxyListSocks < Base
+      # Provider URL to fetch proxy list
+      def provider_url
+        "https://www.socks-proxy.net/"
+      end
+
+      # [NOTE] Doesn't support filtering
+      def xpath
+        "//table[./thead/tr/th[contains(text(), 'IP')]]/tbody/tr"
+      end
+
+      # Converts HTML node (entry of N tags) to <code>ProxyFetcher::Proxy</code>
+      # object.
+      #
+      # @param html_node [Object]
+      #   HTML node from the <code>ProxyFetcher::Document</code> DOM model.
+      #
+      # @return [ProxyFetcher::Proxy]
+      #   Proxy object
+      #
+      def to_proxy(html_node)
+        ProxyFetcher::Proxy.new.tap do |proxy|
+          proxy.addr = html_node.content_at("td[1]")
+          proxy.port = Integer(html_node.content_at("td[2]").gsub(/^0+/, ""))
+          proxy.country = html_node.content_at("td[4]")
+          proxy.type = parse_type(html_node)
+          proxy.anonymity = html_node.content_at("td[6]")
+        end
+      end
+
+      private
+
+      # Parses HTML node to extract proxy type.
+      #
+      # @param html_node [Object]
+      #   HTML node from the <code>ProxyFetcher::Document</code> DOM model.
+      #
+      # @return [String]
+      #   Proxy type
+      #
+      def parse_type(html_node)
+        https = html_node.content_at("td[5]")
+
+        return ProxyFetcher::Proxy::SOCKS4 if https&.casecmp("socks4")&.zero?
+        return ProxyFetcher::Proxy::SOCKS5 if https&.casecmp("socks5")&.zero?
+
+        "Unknown"
+      end
+    end
+
+    ProxyFetcher::Configuration.register_provider(:free_proxy_list_socks, FreeProxyListSocks)
+  end
+end

data/lib/proxy_fetcher/providers/free_proxy_list_ssl.rb

@@ -9,8 +9,9 @@ module ProxyFetcher
         "https://www.sslproxies.org/"
       end
 
+      # [NOTE] Doesn't support filtering
       def xpath
-        '//table[@id="proxylisttable"]/tbody/tr'
+        "//table[./thead/tr/th[contains(text(), 'IP')]]/tbody/tr"
       end
 
       # Converts HTML node (entry of N tags) to <code>ProxyFetcher::Proxy</code>

data/lib/proxy_fetcher/providers/free_proxy_list_us.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module ProxyFetcher
+  module Providers
+    # FreeProxyListUS provider class.
+    class FreeProxyListUS < Base
+      # Provider URL to fetch proxy list
+      def provider_url
+        "https://www.us-proxy.org/"
+      end
+
+      # [NOTE] Doesn't support filtering
+      def xpath
+        "//table[./thead/tr/th[contains(text(), 'IP')]]/tbody/tr"
+      end
+
+      # Converts HTML node (entry of N tags) to <code>ProxyFetcher::Proxy</code>
+      # object.
+      #
+      # @param html_node [Object]
+      #   HTML node from the <code>ProxyFetcher::Document</code> DOM model.
+      #
+      # @return [ProxyFetcher::Proxy]
+      #   Proxy object
+      #
+      def to_proxy(html_node)
+        ProxyFetcher::Proxy.new.tap do |proxy|
+          proxy.addr = html_node.content_at("td[1]")
+          proxy.port = Integer(html_node.content_at("td[2]").gsub(/^0+/, ""))
+          proxy.country = html_node.content_at("td[4]")
+          proxy.anonymity = html_node.content_at("td[5]")
+          proxy.type = parse_type(html_node)
+        end
+      end
+
+      private
+
+      # Parses HTML node to extract proxy type.
+      #
+      # @param html_node [Object]
+      #   HTML node from the <code>ProxyFetcher::Document</code> DOM model.
+      #
+      # @return [String]
+      #   Proxy type
+      #
+      def parse_type(html_node)
+        https = html_node.content_at("td[7]")
+        https&.casecmp("yes")&.zero? ? ProxyFetcher::Proxy::HTTPS : ProxyFetcher::Proxy::HTTP
+      end
+    end
+
+    ProxyFetcher::Configuration.register_provider(:free_proxy_list_us, FreeProxyListUS)
+  end
+end

data/lib/proxy_fetcher/providers/mtpro.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require "json"
+
+module ProxyFetcher
+  module Providers
+    # MTPro provider class.
+    class MTPro < Base
+      # Provider URL to fetch proxy list
+      def provider_url
+        "https://mtpro.xyz/api/?type=socks"
+      end
+
+      def load_proxy_list(filters = {})
+        html = load_html(provider_url, filters)
+        JSON.parse(html)
+      rescue JSON::ParserError
+        []
+      end
+
+      # Converts HTML node (entry of N tags) to <code>ProxyFetcher::Proxy</code>
+      # object.
+      #
+      # @param node [Object]
+      #   HTML node from the <code>ProxyFetcher::Document</code> DOM model.
+      #
+      # @return [ProxyFetcher::Proxy]
+      #   Proxy object
+      #
+      def to_proxy(node)
+        ProxyFetcher::Proxy.new.tap do |proxy|
+          proxy.addr = node["ip"]
+          proxy.port = Integer(node["port"])
+          proxy.country = node["country"]
+          proxy.anonymity = "Unknown"
+          proxy.type = ProxyFetcher::Proxy::SOCKS5
+        end
+      end
+    end
+
+    ProxyFetcher::Configuration.register_provider(:mtpro, MTPro)
+  end
+end

data/lib/proxy_fetcher/providers/proxypedia.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module ProxyFetcher
+  module Providers
+    # FreeProxyList provider class.
+    class Proxypedia < Base
+      # Provider URL to fetch proxy list
+      def provider_url
+        "https://proxypedia.org"
+      end
+
+      # [NOTE] Doesn't support filtering
+      def xpath
+        "//main/ul/li[position()>1]"
+      end
+
+      # Converts HTML node (entry of N tags) to <code>ProxyFetcher::Proxy</code>
+      # object.]
+      #
+      # @param html_node [Object]
+      #   HTML node from the <code>ProxyFetcher::Document</code> DOM model.
+      #
+      # @return [ProxyFetcher::Proxy]
+      #   Proxy object
+      #
+      def to_proxy(html_node)
+        addr, port = html_node.content_at("a").to_s.split(":")
+
+        ProxyFetcher::Proxy.new.tap do |proxy|
+          proxy.addr = addr
+          proxy.port = Integer(port)
+          proxy.country = parse_country(html_node)
+          proxy.anonymity = "Unknown"
+          proxy.type = ProxyFetcher::Proxy::HTTP
+        end
+      end
+
+      private
+
+      def parse_country(html_node)
+        text = html_node.content.to_s
+        text[/\((.+?)\)/, 1] || "Unknown"
+      end
+    end
+
+    ProxyFetcher::Configuration.register_provider(:proxypedia, Proxypedia)
+  end
+end

data/lib/proxy_fetcher/providers/proxyscrape_http.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require "csv"
+
+module ProxyFetcher
+  module Providers
+    # FreeProxyList provider class.
+    class ProxyscrapeHTTP < Base
+      # Provider URL to fetch proxy list
+      def provider_url
+        "https://api.proxyscrape.com/v2/?request=getproxies&protocol=http"
+      end
+
+      # Loads provider HTML and parses it with internal document object.
+      #
+      # @param url [String]
+      #   URL to fetch
+      #
+      # @param filters [Hash]
+      #   filters for proxy provider
+      #
+      # @return [Array]
+      #   Collection of extracted proxies with ports
+      #
+      def load_document(url, filters = {})
+        html = load_html(url, filters)
+
+        CSV.parse(html, col_sep: "\t").map(&:first)
+      end
+
+      # Fetches HTML content by sending HTTP request to the provider URL and
+      # parses the txt document to return all the proxy entries (ip addresses
+      # and ports).
+      #
+      # @return [Array]
+      #   Collection of extracted proxies with ports
+      #
+      def load_proxy_list(filters = {})
+        load_document(provider_url, filters)
+      end
+
+      # Converts String to <code>ProxyFetcher::Proxy</code> object.
+      #
+      # @param node [String]
+      #   String
+      #
+      # @return [ProxyFetcher::Proxy]
+      #   Proxy object
+      #
+      def to_proxy(node)
+        addr, port = node.split(":")
+
+        ProxyFetcher::Proxy.new.tap do |proxy|
+          proxy.addr = addr
+          proxy.port = Integer(port)
+          proxy.country = "Unknown"
+          proxy.anonymity = "Unknown"
+          proxy.type = ProxyFetcher::Proxy::HTTP
+        end
+      end
+    end
+
+    ProxyFetcher::Configuration.register_provider(:proxyscrape_http, ProxyscrapeHTTP)
+  end
+end

data/lib/proxy_fetcher/providers/proxyscrape_socks4.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require "csv"
+
+module ProxyFetcher
+  module Providers
+    # FreeProxyList provider class.
+    class ProxyscrapeSOCKS4 < Base
+      # Provider URL to fetch proxy list
+      def provider_url
+        "https://api.proxyscrape.com/v2/?request=getproxies&protocol=socks4"
+      end
+
+      # Loads provider HTML and parses it with internal document object.
+      #
+      # @param url [String]
+      #   URL to fetch
+      #
+      # @param filters [Hash]
+      #   filters for proxy provider
+      #
+      # @return [Array]
+      #   Collection of extracted proxies with ports
+      #
+      def load_document(url, filters = {})
+        html = load_html(url, filters)
+
+        CSV.parse(html, col_sep: "\t").map(&:first)
+      end
+
+      # Fetches HTML content by sending HTTP request to the provider URL and
+      # parses the txt document to return all the proxy entries (ip addresses
+      # and ports).
+      #
+      # @return [Array]
+      #   Collection of extracted proxies with ports
+      #
+      def load_proxy_list(filters = {})
+        load_document(provider_url, filters)
+      end
+
+      # Converts String to <code>ProxyFetcher::Proxy</code> object.
+      #
+      # @param node [String]
+      #   String
+      #
+      # @return [ProxyFetcher::Proxy]
+      #   Proxy object
+      #
+      def to_proxy(html_node)
+        addr, port = html_node.split(":")
+
+        ProxyFetcher::Proxy.new.tap do |proxy|
+          proxy.addr = addr
+          proxy.port = Integer(port)
+          proxy.country = "Unknown"
+          proxy.anonymity = "Unknown"
+          proxy.type = ProxyFetcher::Proxy::SOCKS4
+        end
+      end
+    end
+
+    ProxyFetcher::Configuration.register_provider(:proxyscrape_socks4, ProxyscrapeSOCKS4)
+  end
+end

data/lib/proxy_fetcher/providers/proxyscrape_socks5.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require "csv"
+
+module ProxyFetcher
+  module Providers
+    # FreeProxyList provider class.
+    class ProxyscrapeSOCKS5 < Base
+      # Provider URL to fetch proxy list
+      def provider_url
+        "https://api.proxyscrape.com/v2/?request=getproxies&protocol=socks5"
+      end
+
+      # Loads provider HTML and parses it with internal document object.
+      #
+      # @param url [String]
+      #   URL to fetch
+      #
+      # @param filters [Hash]
+      #   filters for proxy provider
+      #
+      # @return [Array]
+      #   Collection of extracted proxies with ports
+      #
+      def load_document(url, filters = {})
+        html = load_html(url, filters)
+
+        CSV.parse(html, col_sep: "\t").map(&:first)
+      end
+
+      # Fetches HTML content by sending HTTP request to the provider URL and
+      # parses the txt document to return all the proxy entries (ip addresses
+      # and ports).
+      #
+      # @return [Array]
+      #   Collection of extracted proxies with ports
+      #
+      def load_proxy_list(filters = {})
+        load_document(provider_url, filters)
+      end
+
+      # Converts String to <code>ProxyFetcher::Proxy</code> object.
+      #
+      # @param node [String]
+      #   String
+      #
+      # @return [ProxyFetcher::Proxy]
+      #   Proxy object
+      #
+      def to_proxy(html_node)
+        addr, port = html_node.split(":")
+
+        ProxyFetcher::Proxy.new.tap do |proxy|
+          proxy.addr = addr
+          proxy.port = Integer(port)
+          proxy.country = "Unknown"
+          proxy.anonymity = "Unknown"
+          proxy.type = ProxyFetcher::Proxy::SOCKS5
+        end
+      end
+    end
+
+    ProxyFetcher::Configuration.register_provider(:proxyscrape_socks5, ProxyscrapeSOCKS5)
+  end
+end

data/lib/proxy_fetcher/providers/xroxy.rb

@@ -6,7 +6,7 @@ module ProxyFetcher
     class XRoxy < Base
       # Provider URL to fetch proxy list
       def provider_url
-        "https://madison.xroxy.com/proxylist.html"
+        "https://www.xroxy.com/proxylist.htm"
       end
 
       def xpath

data/lib/proxy_fetcher/utils/http_client.rb

@@ -41,8 +41,8 @@ module ProxyFetcher
     # @return [String]
     #   resource content
     #
-    def self.fetch(*args)
-      new(*args).fetch
+    def self.fetch(*args, **kwargs, &block)
+      new(*args, **kwargs, &block).fetch
     end
 
     # Initialize HTTP client instance
@@ -51,15 +51,17 @@ module ProxyFetcher
     #
     def initialize(url, method: :get, params: {}, headers: {})
       @url = url.to_s
-      @method = method
+      @method = method.to_sym
       @params = params
       @headers = headers
 
-      @http = HTTP.headers(default_headers.merge(headers)).timeout(connect: timeout, read: timeout)
-      @timeout = ProxyFetcher.config.provider_proxies_load_timeout
+      unless HTTP::Request::METHODS.include?(@method)
+        raise ArgumentError, "'#{@method}' is a wrong HTTP method name"
+      end
 
-      @ssl_ctx = OpenSSL::SSL::SSLContext.new
-      @ssl_ctx.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      @timeout = ProxyFetcher.config.provider_proxies_load_timeout
+      @http = build_http_engine
+      @ssl_ctx = build_ssl_context
     end
 
     # Fetches resource content by sending HTTP request to it.
@@ -67,30 +69,32 @@ module ProxyFetcher
     # @return [String]
     #   response body
     #
-    def fetch
-      response = process_http_request
+    def fetch(**options)
+      response = perform_http_request
+      return response if options.fetch(:raw, false)
+
       response.body.to_s
     rescue StandardError => e
-      ProxyFetcher.logger.warn("Failed to process request to #{url} (#{e.message})")
+      ProxyFetcher.config.logger.warn("Failed to process request to #{url} (#{e.message})")
       ""
     end
 
-    def fetch_with_headers
-      process_http_request
-    rescue StandardError => e
-      ProxyFetcher.logger.warn("Failed to process request to #{url} (#{e.message})")
-      HTTP::Response.new(version: "1.1", status: 500, body: "")
-    end
-
     protected
 
-    def process_http_request(http_method: method, http_params: params)
-      unless HTTP::Request::METHODS.include?(http_method)
-        raise ArgumentError, "'#{http_method}' is a wrong HTTP method name!"
+    def build_ssl_context
+      OpenSSL::SSL::SSLContext.new.tap do |context|
+        context.verify_mode = OpenSSL::SSL::VERIFY_NONE
       end
+    end
+
+    def build_http_engine
+      HTTP.headers(default_headers.merge(headers)).timeout(connect: timeout, read: timeout)
+    end
 
+    def perform_http_request(http_method: method, http_params: params)
       http.public_send(
-        http_method.to_sym, url,
+        http_method,
+        url,
         form: http_params,
         ssl_context: ssl_ctx
       )

data/lib/proxy_fetcher/utils/proxy_validator.rb

@@ -10,27 +10,39 @@ module ProxyFetcher
 
     # Short variant to validate proxy.
     #
-    # @param proxy_addr [String] proxy address or IP
-    # @param proxy_port [String, Integer] proxy port
+    # @param address [String] proxy address or IP
+    # @param port [String, Integer] proxy port
     #
     # @return [Boolean]
     #   true if connection to the server using proxy established, otherwise false
     #
-    def self.connectable?(proxy_addr, proxy_port)
-      new(proxy_addr, proxy_port).connectable?
+    def self.connectable?(address, port)
+      new(address, port).connectable?
     end
 
     # Initialize new ProxyValidator instance
     #
-    # @param proxy_addr [String] proxy address or IP
-    # @param proxy_port [String, Integer] proxy port
+    # @param address [String] Proxy address or IP
+    # @param port [String, Integer] Proxy port
+    # @param options [Hash] proxy options
+    #   @option username [String] Proxy authentication username
+    #   @option password [String] Proxy authentication password
+    #   @option headers [Hash] Proxy headers
     #
     # @return [ProxyValidator]
     #
-    def initialize(proxy_addr, proxy_port)
+    def initialize(address, port, options: {})
       timeout = ProxyFetcher.config.proxy_validation_timeout
+      proxy = [address, port.to_i]
 
-      @http = HTTP.follow.via(proxy_addr, proxy_port.to_i).timeout(connect: timeout, read: timeout)
+      if options[:username] && options[:password]
+        proxy << options[:username]
+        proxy << options[:password]
+      end
+
+      proxy << options[:headers].to_h if options[:headers]
+
+      @http = HTTP.follow.via(*proxy).timeout(connect: timeout, read: timeout)
     end
 
     # Checks if proxy is connectable (can be used to connect

data/lib/proxy_fetcher/version.rb

@@ -13,7 +13,7 @@ module ProxyFetcher
     # Major version number
     MAJOR = 0
     # Minor version number
-    MINOR = 13
+    MINOR = 16
     # Smallest version number
     TINY  = 0
 

data/lib/proxy_fetcher.rb

@@ -4,41 +4,47 @@ require "uri"
 require "http"
 require "logger"
 
-require File.dirname(__FILE__) + "/proxy_fetcher/version"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/version"
 
-require File.dirname(__FILE__) + "/proxy_fetcher/exceptions"
-require File.dirname(__FILE__) + "/proxy_fetcher/configuration"
-require File.dirname(__FILE__) + "/proxy_fetcher/configuration/providers_registry"
-require File.dirname(__FILE__) + "/proxy_fetcher/proxy"
-require File.dirname(__FILE__) + "/proxy_fetcher/manager"
-require File.dirname(__FILE__) + "/proxy_fetcher/null_logger"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/exceptions"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/configuration"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/configuration/providers_registry"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/proxy"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/manager"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/null_logger"
 
-require File.dirname(__FILE__) + "/proxy_fetcher/utils/http_client"
-require File.dirname(__FILE__) + "/proxy_fetcher/utils/proxy_validator"
-require File.dirname(__FILE__) + "/proxy_fetcher/utils/proxy_list_validator"
-require File.dirname(__FILE__) + "/proxy_fetcher/client/client"
-require File.dirname(__FILE__) + "/proxy_fetcher/client/request"
-require File.dirname(__FILE__) + "/proxy_fetcher/client/proxies_registry"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/utils/http_client"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/utils/proxy_validator"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/utils/proxy_list_validator"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/client/client"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/client/request"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/client/proxies_registry"
 
-require File.dirname(__FILE__) + "/proxy_fetcher/document"
-require File.dirname(__FILE__) + "/proxy_fetcher/document/adapters"
-require File.dirname(__FILE__) + "/proxy_fetcher/document/node"
-require File.dirname(__FILE__) + "/proxy_fetcher/document/adapters/abstract_adapter"
-require File.dirname(__FILE__) + "/proxy_fetcher/document/adapters/nokogiri_adapter"
-require File.dirname(__FILE__) + "/proxy_fetcher/document/adapters/oga_adapter"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/document"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/document/adapters"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/document/node"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/document/adapters/abstract_adapter"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/document/adapters/nokogiri_adapter"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/document/adapters/oga_adapter"
 
 ##
 # Ruby / JRuby lib for managing proxies
 module ProxyFetcher
   # ProxyFetcher providers namespace
   module Providers
-    require File.dirname(__FILE__) + "/proxy_fetcher/providers/base"
-    require File.dirname(__FILE__) + "/proxy_fetcher/providers/free_proxy_list"
-    require File.dirname(__FILE__) + "/proxy_fetcher/providers/free_proxy_list_ssl"
-    require File.dirname(__FILE__) + "/proxy_fetcher/providers/gather_proxy"
-    require File.dirname(__FILE__) + "/proxy_fetcher/providers/http_tunnel"
-    require File.dirname(__FILE__) + "/proxy_fetcher/providers/proxy_list"
-    require File.dirname(__FILE__) + "/proxy_fetcher/providers/xroxy"
+    require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/base"
+    require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/free_proxy_list"
+    require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/free_proxy_list_socks"
+    require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/free_proxy_list_ssl"
+    require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/free_proxy_list_us"
+    require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/http_tunnel"
+    require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/mtpro"
+    require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/proxy_list"
+    require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/proxypedia"
+    require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/proxyscrape_http"
+    require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/proxyscrape_socks4"
+    require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/proxyscrape_socks5"
+    require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/xroxy"
   end
 
   @__config_access_lock__ = Mutex.new

data/proxy_fetcher.gemspec

@@ -8,9 +8,11 @@ Gem::Specification.new do |gem|
   gem.name = "proxy_fetcher"
   gem.version = ProxyFetcher.gem_version
   gem.summary = "Ruby gem for dealing with proxy lists from different providers"
-  gem.description = "This gem can help your Ruby application to make HTTP(S) requests " \
-                    "using proxies by fetching and validating proxy lists from " \
-                    "the different providers."
+  gem.description = <<-TEXT.strip.gsub(/[\s\n]+/, " ")
+    This gem can help your Ruby application to make HTTP(S) requests
+    using proxies by fetching and validating proxy lists from
+    the different providers.
+  TEXT
   gem.authors = ["Nikita Bulai"]
   gem.email = "bulajnikita@gmail.com"
   gem.require_paths = ["lib"]
@@ -21,8 +23,8 @@ Gem::Specification.new do |gem|
   gem.license = "MIT"
   gem.required_ruby_version = ">= 2.3.0"
 
-  gem.add_runtime_dependency "http", ">= 3", "< 5"
+  gem.add_runtime_dependency "http", ">= 3"
 
   gem.add_development_dependency "rake", ">= 12.0"
-  gem.add_development_dependency "rspec", "~> 3.5"
+  gem.add_development_dependency "rspec", "~> 3.9"
 end

data/spec/proxy_fetcher/client/client_spec.rb

@@ -3,6 +3,11 @@
 require "spec_helper"
 require "json"
 
+begin
+  require "webrick"
+rescue LoadError
+  # nop
+end
 require "evil-proxy"
 require "evil-proxy/async"
 
@@ -118,15 +123,15 @@ describe ProxyFetcher::Client do
     end
   end
 
-  xcontext "retries" do
+  context "retries" do
     it "raises an error when reaches max retries limit" do
       allow(ProxyFetcher::Client::Request).to receive(:execute).and_raise(StandardError)
 
-      expect { ProxyFetcher::Client.get("http://httpbin.org") }
+      expect { ProxyFetcher::Client.get("http://httpbin.org", options: { max_retries: 10 }) }
         .to raise_error(ProxyFetcher::Exceptions::MaximumRetriesReached)
     end
 
-    it "raises an error when http request returns an error" do
+    xit "raises an error when http request returns an error" do
       allow_any_instance_of(HTTP::Client).to receive(:get).and_return(StandardError.new)
 
       expect { ProxyFetcher::Client.get("http://httpbin.org") }
@@ -134,14 +139,14 @@ describe ProxyFetcher::Client do
     end
 
     it "refreshes proxy lists if no proxy found" do
-      ProxyFetcher::Client::ProxiesRegistry.manager.instance_variable_set(:"@proxies", [])
+      allow(ProxyFetcher::Manager.new).to receive(:proxies).and_return([])
 
       expect { ProxyFetcher::Client.get("http://httpbin.org") }
         .not_to raise_error
     end
   end
 
-  context "redirects" do
+  xcontext "redirects" do
     it "follows redirect when present" do
       content = ProxyFetcher::Client.get("http://httpbin.org/absolute-redirect/2")
 

data/spec/proxy_fetcher/providers/proxy_classes_spec.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require "spec_helper"
+
+describe "Proxy classes" do
+  [
+    [:free_proxy_list,       "FreeProxyList"],
+    [:free_proxy_list_socks, "FreeProxyListSocks"],
+    [:free_proxy_list_ssl,   "FreeProxyListSSL"],
+    [:free_proxy_list_us,    "FreeProxyListUS"],
+    [:http_tunnel,           "HTTPTunnel"],
+    [:mtpro,                 "MTPro"],
+    [:proxy_list,            "ProxyList"],
+    [:proxypedia,            "Proxypedia"],
+    [:proxyscrape_http,      "ProxyscrapeHTTP"],
+    [:proxyscrape_socks4,    "ProxyscrapeSOCKS4"],
+    [:proxyscrape_socks5,    "ProxyscrapeSOCKS5"],
+    [:xroxy,                 "XRoxy"]
+  ].each do |(provider_name, provider_klass)|
+    describe Object.const_get("ProxyFetcher::Providers::#{provider_klass}") do
+      before :all do
+        ProxyFetcher.config.provider = provider_name
+      end
+
+      it_behaves_like "a manager"
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: proxy_fetcher
 version: !ruby/object:Gem::Version
-  version: 0.13.0
+  version: 0.16.0
 platform: ruby
 authors:
 - Nikita Bulai
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-03-09 00:00:00.000000000 Z
+date: 2022-02-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: http
@@ -17,9 +17,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '3'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '5'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -27,9 +24,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '3'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '5'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -50,14 +44,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.5'
+        version: '3.9'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.5'
+        version: '3.9'
 description: This gem can help your Ruby application to make HTTP(S) requests using
   proxies by fetching and validating proxy lists from the different providers.
 email: bulajnikita@gmail.com
@@ -67,6 +61,8 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/dependabot.yml"
+- ".github/workflows/ci.yml"
 - ".gitignore"
 - CHANGELOG.md
 - CODE_OF_CONDUCT.md
@@ -94,10 +90,16 @@ files:
 - lib/proxy_fetcher/null_logger.rb
 - lib/proxy_fetcher/providers/base.rb
 - lib/proxy_fetcher/providers/free_proxy_list.rb
+- lib/proxy_fetcher/providers/free_proxy_list_socks.rb
 - lib/proxy_fetcher/providers/free_proxy_list_ssl.rb
-- lib/proxy_fetcher/providers/gather_proxy.rb
+- lib/proxy_fetcher/providers/free_proxy_list_us.rb
 - lib/proxy_fetcher/providers/http_tunnel.rb
+- lib/proxy_fetcher/providers/mtpro.rb
 - lib/proxy_fetcher/providers/proxy_list.rb
+- lib/proxy_fetcher/providers/proxypedia.rb
+- lib/proxy_fetcher/providers/proxyscrape_http.rb
+- lib/proxy_fetcher/providers/proxyscrape_socks4.rb
+- lib/proxy_fetcher/providers/proxyscrape_socks5.rb
 - lib/proxy_fetcher/providers/xroxy.rb
 - lib/proxy_fetcher/proxy.rb
 - lib/proxy_fetcher/utils/http_client.rb
@@ -112,13 +114,8 @@ files:
 - spec/proxy_fetcher/document/node_spec.rb
 - spec/proxy_fetcher/manager_spec.rb
 - spec/proxy_fetcher/providers/base_spec.rb
-- spec/proxy_fetcher/providers/free_proxy_list_spec.rb
-- spec/proxy_fetcher/providers/free_proxy_list_ssl_spec.rb
-- spec/proxy_fetcher/providers/gather_proxy_spec.rb
-- spec/proxy_fetcher/providers/http_tunnel_spec.rb
 - spec/proxy_fetcher/providers/multiple_providers_spec.rb
-- spec/proxy_fetcher/providers/proxy_list_spec.rb
-- spec/proxy_fetcher/providers/xroxy_spec.rb
+- spec/proxy_fetcher/providers/proxy_classes_spec.rb
 - spec/proxy_fetcher/proxy_spec.rb
 - spec/proxy_fetcher/version_spec.rb
 - spec/spec_helper.rb
@@ -142,7 +139,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.2
+rubygems_version: 3.0.8
 signing_key: 
 specification_version: 4
 summary: Ruby gem for dealing with proxy lists from different providers

data/lib/proxy_fetcher/providers/gather_proxy.rb

@@ -1,50 +0,0 @@
-# frozen_string_literal: true
-
-require "json"
-
-module ProxyFetcher
-  module Providers
-    # GatherProxy provider class.
-    class GatherProxy < Base
-      # Provider URL to fetch proxy list
-      def provider_url
-        "https://proxygather.com"
-      end
-
-      def xpath
-        '//div[@class="proxy-list"]/table/script'
-      end
-
-      # Converts HTML node (entry of N tags) to <code>ProxyFetcher::Proxy</code>
-      # object.
-      #
-      # @param html_node [Object]
-      #   HTML node from the <code>ProxyFetcher::Document</code> DOM model.
-      #
-      # @return [ProxyFetcher::Proxy]
-      #   Proxy object
-      #
-      def to_proxy(html_node)
-        json = parse_json(html_node)
-
-        ProxyFetcher::Proxy.new.tap do |proxy|
-          proxy.addr = json["PROXY_IP"]
-          proxy.port = json["PROXY_PORT"].to_i(16)
-          proxy.anonymity = json["PROXY_TYPE"]
-          proxy.country = json["PROXY_COUNTRY"]
-          proxy.response_time = json["PROXY_TIME"].to_i
-          proxy.type = ProxyFetcher::Proxy::HTTP
-        end
-      end
-
-      private
-
-      def parse_json(html_node)
-        javascript = html_node.content[/{.+}/im]
-        JSON.parse(javascript)
-      end
-    end
-
-    ProxyFetcher::Configuration.register_provider(:gather_proxy, GatherProxy)
-  end
-end

data/spec/proxy_fetcher/providers/free_proxy_list_spec.rb

@@ -1,13 +0,0 @@
-# frozen_string_literal: true
-
-require "spec_helper"
-
-describe ProxyFetcher::Providers::FreeProxyList do
-  before :all do
-    ProxyFetcher.configure do |config|
-      config.provider = :free_proxy_list
-    end
-  end
-
-  it_behaves_like "a manager"
-end

data/spec/proxy_fetcher/providers/free_proxy_list_ssl_spec.rb

@@ -1,11 +0,0 @@
-# frozen_string_literal: true
-
-require "spec_helper"
-
-describe ProxyFetcher::Providers::FreeProxyListSSL do
-  before :all do
-    ProxyFetcher.config.provider = :free_proxy_list_ssl
-  end
-
-  it_behaves_like "a manager"
-end

data/spec/proxy_fetcher/providers/gather_proxy_spec.rb

@@ -1,11 +0,0 @@
-# frozen_string_literal: true
-
-require "spec_helper"
-
-describe ProxyFetcher::Providers::GatherProxy do
-  before :all do
-    ProxyFetcher.config.provider = :gather_proxy
-  end
-
-  it_behaves_like "a manager"
-end

data/spec/proxy_fetcher/providers/http_tunnel_spec.rb

@@ -1,11 +0,0 @@
-# frozen_string_literal: true
-
-require "spec_helper"
-
-describe ProxyFetcher::Providers::HTTPTunnel do
-  before :all do
-    ProxyFetcher.config.provider = :http_tunnel
-  end
-
-  it_behaves_like "a manager"
-end

data/spec/proxy_fetcher/providers/proxy_list_spec.rb

@@ -1,11 +0,0 @@
-# frozen_string_literal: true
-
-require "spec_helper"
-
-describe ProxyFetcher::Providers::ProxyList do
-  before :all do
-    ProxyFetcher.config.provider = :proxy_list
-  end
-
-  it_behaves_like "a manager"
-end

data/spec/proxy_fetcher/providers/xroxy_spec.rb

@@ -1,11 +0,0 @@
-# frozen_string_literal: true
-
-require "spec_helper"
-
-describe ProxyFetcher::Providers::XRoxy do
-  before :all do
-    ProxyFetcher.config.provider = :xroxy
-  end
-
-  it_behaves_like "a manager"
-end