proxy_fetcher 0.10.2 → 0.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d256aaf841030823ef753d6c2b2726dd7e8e5306dbddae6b6079a26815df4224
-  data.tar.gz: a747a8d9ed567ffa6373bd0d30dc9a99e5320cb03b75dd3feb5ef45465144375
+  metadata.gz: 5a0d7b377ed3c25e50552e89ba76c0e73fad3923bf171e1cee2f592d777787c3
+  data.tar.gz: 83b594e04e03c74a63146a6907c99025d50607c88f8cf94f6d5ce044795243ad
 SHA512:
-  metadata.gz: '01997f86deda80383198be18ac9bca8309970fa38ea337a082093822838732038425b642a6806798fa35e2a891b63d679020fad33fd5f447efed00b426b85706'
-  data.tar.gz: 2fd10653fdf594e8847d4ebe34cb66c30b3d79d2900bd741768a5d4742ffa50923092f2c8f2bd95e3215f505a073e040e27cad8e2d868ff5837e81078c7d6f95
+  metadata.gz: 1f096c2473035255eb9492297b5641ab5caee62566eb20eb40d3b2f02eea5d06fa1279a2cadeb3266c0f52ce98040185a69678faf116b6398c2c75f79d5c4ebd
+  data.tar.gz: a9372ef8bdbb3c51c5060308cbc46c905df3819e682bceb858ba494f5f94722095f2a2bee94575606f628d091b87325425c218f8c31a2b807bcb159c59ba6e65

data/Gemfile

@@ -1,11 +1,14 @@
-source 'https://rubygems.org'
+# frozen_string_literal: true
+
+source "https://rubygems.org"
 
 gemspec
 
-gem 'nokogiri', '~> 1.8'
-gem 'oga', '~> 2.0'
+gem "nokogiri", "~> 1.8"
+gem "oga", "~> 2.0"
+gem "rubocop", "~> 0.74"
 
 group :test do
-  gem 'coveralls', require: false
-  gem 'evil-proxy', '~> 0.2'
+  gem "coveralls", require: false
+  gem "evil-proxy", "~> 0.2"
 end

data/Rakefile

@@ -1,6 +1,8 @@
-require 'bundler/gem_tasks'
+# frozen_string_literal: true
 
-require 'rspec/core/rake_task'
+require "bundler/gem_tasks"
+
+require "rspec/core/rake_task"
 RSpec::Core::RakeTask.new(:spec)
 
 task default: :spec

data/gemfiles/nokogiri.gemfile

@@ -1,11 +1,13 @@
-source 'https://rubygems.org'
+# frozen_string_literal: true
 
-gemspec path: '../'
+source "https://rubygems.org"
 
-gem 'nokogiri', '~> 1.8'
+gemspec path: "../"
+
+gem "nokogiri", "~> 1.8"
 
 group :test do
-  gem 'coveralls', require: false
-  gem 'evil-proxy', '~> 0.2'
-  gem 'rspec', '~> 3.6'
+  gem "coveralls", require: false
+  gem "evil-proxy", "~> 0.2"
+  gem "rspec", "~> 3.6"
 end

data/gemfiles/oga.gemfile

@@ -1,11 +1,13 @@
-source 'https://rubygems.org'
+# frozen_string_literal: true
 
-gemspec path: '../'
+source "https://rubygems.org"
 
-gem 'oga', '~> 2.0'
+gemspec path: "../"
+
+gem "oga", "~> 2.0"
 
 group :test do
-  gem 'coveralls', require: false
-  gem 'evil-proxy', '~> 0.2'
-  gem 'rspec', '~> 3.6'
+  gem "coveralls", require: false
+  gem "evil-proxy", "~> 0.2"
+  gem "rspec", "~> 3.6"
 end

data/lib/proxy_fetcher.rb

@@ -1,44 +1,44 @@
 # frozen_string_literal: true
 
-require 'uri'
-require 'http'
-require 'logger'
+require "uri"
+require "http"
+require "logger"
 
-require File.dirname(__FILE__) + '/proxy_fetcher/version'
+require File.dirname(__FILE__) + "/proxy_fetcher/version"
 
-require File.dirname(__FILE__) + '/proxy_fetcher/exceptions'
-require File.dirname(__FILE__) + '/proxy_fetcher/configuration'
-require File.dirname(__FILE__) + '/proxy_fetcher/configuration/providers_registry'
-require File.dirname(__FILE__) + '/proxy_fetcher/proxy'
-require File.dirname(__FILE__) + '/proxy_fetcher/manager'
-require File.dirname(__FILE__) + '/proxy_fetcher/null_logger'
+require File.dirname(__FILE__) + "/proxy_fetcher/exceptions"
+require File.dirname(__FILE__) + "/proxy_fetcher/configuration"
+require File.dirname(__FILE__) + "/proxy_fetcher/configuration/providers_registry"
+require File.dirname(__FILE__) + "/proxy_fetcher/proxy"
+require File.dirname(__FILE__) + "/proxy_fetcher/manager"
+require File.dirname(__FILE__) + "/proxy_fetcher/null_logger"
 
-require File.dirname(__FILE__) + '/proxy_fetcher/utils/http_client'
-require File.dirname(__FILE__) + '/proxy_fetcher/utils/proxy_validator'
-require File.dirname(__FILE__) + '/proxy_fetcher/utils/proxy_list_validator'
-require File.dirname(__FILE__) + '/proxy_fetcher/client/client'
-require File.dirname(__FILE__) + '/proxy_fetcher/client/request'
-require File.dirname(__FILE__) + '/proxy_fetcher/client/proxies_registry'
+require File.dirname(__FILE__) + "/proxy_fetcher/utils/http_client"
+require File.dirname(__FILE__) + "/proxy_fetcher/utils/proxy_validator"
+require File.dirname(__FILE__) + "/proxy_fetcher/utils/proxy_list_validator"
+require File.dirname(__FILE__) + "/proxy_fetcher/client/client"
+require File.dirname(__FILE__) + "/proxy_fetcher/client/request"
+require File.dirname(__FILE__) + "/proxy_fetcher/client/proxies_registry"
 
-require File.dirname(__FILE__) + '/proxy_fetcher/document'
-require File.dirname(__FILE__) + '/proxy_fetcher/document/adapters'
-require File.dirname(__FILE__) + '/proxy_fetcher/document/node'
-require File.dirname(__FILE__) + '/proxy_fetcher/document/adapters/abstract_adapter'
-require File.dirname(__FILE__) + '/proxy_fetcher/document/adapters/nokogiri_adapter'
-require File.dirname(__FILE__) + '/proxy_fetcher/document/adapters/oga_adapter'
+require File.dirname(__FILE__) + "/proxy_fetcher/document"
+require File.dirname(__FILE__) + "/proxy_fetcher/document/adapters"
+require File.dirname(__FILE__) + "/proxy_fetcher/document/node"
+require File.dirname(__FILE__) + "/proxy_fetcher/document/adapters/abstract_adapter"
+require File.dirname(__FILE__) + "/proxy_fetcher/document/adapters/nokogiri_adapter"
+require File.dirname(__FILE__) + "/proxy_fetcher/document/adapters/oga_adapter"
 
 ##
 # Ruby / JRuby lib for managing proxies
 module ProxyFetcher
   # ProxyFetcher providers namespace
   module Providers
-    require File.dirname(__FILE__) + '/proxy_fetcher/providers/base'
-    require File.dirname(__FILE__) + '/proxy_fetcher/providers/free_proxy_list'
-    require File.dirname(__FILE__) + '/proxy_fetcher/providers/free_proxy_list_ssl'
-    require File.dirname(__FILE__) + '/proxy_fetcher/providers/gather_proxy'
-    require File.dirname(__FILE__) + '/proxy_fetcher/providers/http_tunnel'
-    require File.dirname(__FILE__) + '/proxy_fetcher/providers/proxy_list'
-    require File.dirname(__FILE__) + '/proxy_fetcher/providers/xroxy'
+    require File.dirname(__FILE__) + "/proxy_fetcher/providers/base"
+    require File.dirname(__FILE__) + "/proxy_fetcher/providers/free_proxy_list"
+    require File.dirname(__FILE__) + "/proxy_fetcher/providers/free_proxy_list_ssl"
+    require File.dirname(__FILE__) + "/proxy_fetcher/providers/gather_proxy"
+    require File.dirname(__FILE__) + "/proxy_fetcher/providers/http_tunnel"
+    require File.dirname(__FILE__) + "/proxy_fetcher/providers/proxy_list"
+    require File.dirname(__FILE__) + "/proxy_fetcher/providers/xroxy"
   end
 
   # Main ProxyFetcher module.
@@ -75,7 +75,7 @@ module ProxyFetcher
 
     # Returns ProxyFetcher logger instance.
     #
-    # @return [Logger, NullLogger] logger object
+    # @return [Logger, ProxyFetcher::NullLogger] logger object
     #
     def logger
       return @logger if defined?(@logger)

data/lib/proxy_fetcher/client/client.rb

@@ -128,7 +128,11 @@ module ProxyFetcher
       #
       def request_with_payload(method, url, payload, headers, options)
         with_proxy_for(url, options.fetch(:max_retries, 1000)) do |proxy|
-          opts = options.merge(payload: payload, proxy: options.fetch(:proxy, proxy), headers: default_headers.merge(headers))
+          opts = options.merge(
+            payload: payload,
+            proxy: options.fetch(:proxy, proxy),
+            headers: default_headers.merge(headers)
+          )
 
           Request.execute(url: url, method: method, **opts)
         end
@@ -138,7 +142,10 @@ module ProxyFetcher
       #
       def request_without_payload(method, url, headers, options)
         with_proxy_for(url, options.fetch(:max_retries, 1000)) do |proxy|
-          opts = options.merge(proxy: options.fetch(:proxy, proxy), headers: default_headers.merge(headers))
+          opts = options.merge(
+            proxy: options.fetch(:proxy, proxy),
+            headers: default_headers.merge(headers)
+          )
 
           Request.execute(url: url, method: method, **opts)
         end
@@ -152,7 +159,7 @@ module ProxyFetcher
       #
       def default_headers
         {
-          'User-Agent' => ProxyFetcher.config.user_agent
+          "User-Agent" => ProxyFetcher.config.user_agent
         }
       end
 

data/lib/proxy_fetcher/client/request.rb

@@ -50,7 +50,7 @@ module ProxyFetcher
       # @return [Request]
       #
       def initialize(args)
-        raise ArgumentError, 'args must be a Hash!' unless args.is_a?(Hash)
+        raise ArgumentError, "args must be a Hash!" unless args.is_a?(Hash)
 
         @url = args.fetch(:url)
         @method = args.fetch(:method).to_s.downcase
@@ -86,9 +86,9 @@ module ProxyFetcher
       #
       def build_http_client
         HTTP.via(proxy.addr, proxy.port.to_i)
-            .headers(headers)
-            .timeout(connect: timeout, read: timeout)
-            .follow(max_hops: max_redirects)
+          .headers(headers)
+          .timeout(connect: timeout, read: timeout)
+          .follow(max_hops: max_redirects)
       end
 
       # Default SSL options that will be used for connecting to resources

data/lib/proxy_fetcher/configuration.rb

@@ -6,15 +6,20 @@ module ProxyFetcher
   #
   class Configuration
     # @!attribute client_timeout
-    #   @return [Integer] HTTP request timeout (connect / open) for [ProxyFetcher::Client]
+    #   @return [Integer]
+    #     HTTP request timeout (connect / open) for [ProxyFetcher::Client]
     attr_accessor :client_timeout
 
     # @!attribute provider_proxies_load_timeout
-    #   @return [Integer] HTTP request timeout (connect / open) for loading of proxies list by provider
+    #   @return [Integer]
+    #     HTTP request timeout (connect / open) for loading
+    #     of proxies list by provider
     attr_accessor :provider_proxies_load_timeout
 
     # @!attribute proxy_validation_timeout
-    #   @return [Integer] HTTP request timeout (connect / open) for proxy validation with [ProxyFetcher::ProxyValidator]
+    #   @return [Integer]
+    #     HTTP request timeout (connect / open) for proxy
+    #     validation with [ProxyFetcher::ProxyValidator]
     attr_accessor :proxy_validation_timeout
 
     # to save compatibility
@@ -30,16 +35,12 @@ module ProxyFetcher
     attr_accessor :user_agent
 
     # @!attribute [r] logger
-    #   @return [Object] Logger object
+    #   @return [Logger] Logger object
     attr_accessor :logger
 
     # @!attribute [r] adapter
     #   @return [Object] HTML parser adapter
-    attr_accessor :adapter
-
-    # @!attribute [r] adapter_class
-    #   @return [Object] HTML adapter class
-    attr_reader :adapter_class
+    attr_reader :adapter
 
     # @!attribute [r] http_client
     #   @return [Object] HTTP client class
@@ -58,8 +59,8 @@ module ProxyFetcher
     #
     # Default is Google Chrome 60, but can be changed in <code>ProxyFetcher.config</code>.
     #
-    DEFAULT_USER_AGENT = 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 ' \
-                         '(KHTML, like Gecko) Chrome/60.0.3112 Safari/537.36'.freeze
+    DEFAULT_USER_AGENT = "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 " \
+                         "(KHTML, like Gecko) Chrome/60.0.3112 Safari/537.36"
 
     # HTML parser adapter name.
     #

data/lib/proxy_fetcher/document.rb

@@ -36,14 +36,5 @@ module ProxyFetcher
     def xpath(*args)
       backend.xpath(*args).map { |node| backend.proxy_node.new(node) }
     end
-
-    # Searches elements by CSS selector.
-    #
-    # @return [Array<ProxyFetcher::Document::Node>]
-    #   collection of nodes
-    #
-    def css(*args)
-      backend.css(*args).map { |node| backend.proxy_node.new(node) }
-    end
   end
 end

data/lib/proxy_fetcher/document/adapters.rb

@@ -13,7 +13,7 @@ module ProxyFetcher
     # <code>ProxyFetcher::Document::AbstractAdapter</code>.
     class Adapters
       # Adapters class name suffix
-      ADAPTER = 'Adapter'.freeze
+      ADAPTER = "Adapter"
       private_constant :ADAPTER
 
       class << self

data/lib/proxy_fetcher/document/adapters/abstract_adapter.rb

@@ -26,15 +26,6 @@ module ProxyFetcher
         document.xpath(selector)
       end
 
-      # You can override this method in your own adapter class
-      #
-      # @param selector [String]
-      #   CSS selector
-      #
-      def css(selector)
-        document.css(selector)
-      end
-
       # Returns <code>Node</code> class that will handle HTML
       # nodes for particular adapter.
       #
@@ -42,7 +33,7 @@ module ProxyFetcher
       #   node
       #
       def proxy_node
-        self.class.const_get('Node')
+        self.class.const_get("Node")
       end
 
       # Installs adapter requirements.
@@ -53,8 +44,8 @@ module ProxyFetcher
       def self.setup!(*args)
         install_requirements!(*args)
         self
-      rescue LoadError, StandardError => error
-        raise Exceptions::AdapterSetupError.new(name, error.message)
+      rescue LoadError, StandardError => e
+        raise Exceptions::AdapterSetupError.new(name, e.message)
       end
     end
   end

data/lib/proxy_fetcher/document/adapters/nokogiri_adapter.rb

@@ -6,7 +6,7 @@ module ProxyFetcher
     class NokogiriAdapter < AbstractAdapter
       # Requires Nokogiri gem to the application.
       def self.install_requirements!
-        require 'nokogiri'
+        require "nokogiri"
       end
 
       # Parses raw HTML content with specific gem.

data/lib/proxy_fetcher/document/adapters/oga_adapter.rb

@@ -6,7 +6,7 @@ module ProxyFetcher
     class OgaAdapter < AbstractAdapter
       # Requires Oga gem to the application.
       def self.install_requirements!
-        require 'oga'
+        require "oga"
       end
 
       # Parses raw HTML content with specific gem.

data/lib/proxy_fetcher/document/node.rb

@@ -81,9 +81,9 @@ module ProxyFetcher
       #   clean text
       #
       def clear(text)
-        return '' if text.nil? || text.empty?
+        return "" if text.nil? || text.empty?
 
-        text.strip.gsub(/[ \t]/i, '')
+        text.strip.gsub(/[\t]/i, "")
       end
     end
   end

data/lib/proxy_fetcher/exceptions.rb

@@ -13,7 +13,7 @@ module ProxyFetcher
       # @return [WrongCustomClass]
       #
       def initialize(klass, methods)
-        required_methods = Array(methods).join(', ')
+        required_methods = Array(methods).join(", ")
         super("#{klass} must respond to [#{required_methods}] class methods!")
       end
     end
@@ -53,7 +53,7 @@ module ProxyFetcher
       # @return [MaximumRedirectsReached]
       #
       def initialize(*)
-        super('maximum redirects reached')
+        super("maximum redirects reached")
       end
     end
 
@@ -66,7 +66,7 @@ module ProxyFetcher
       # @return [MaximumRetriesReached]
       #
       def initialize(*)
-        super('reached the maximum number of retries')
+        super("reached the maximum number of retries")
       end
     end
 
@@ -95,7 +95,7 @@ module ProxyFetcher
         super(<<-MSG.strip.squeeze
           you need to specify adapter for HTML parsing: ProxyFetcher.config.adapter = :nokogiri.
           You can use one of the predefined adapters (:nokogiri or :oga) or your own implementation.
-          MSG
+        MSG
         )
       end
     end
@@ -111,7 +111,7 @@ module ProxyFetcher
       # @return [AdapterSetupError]
       #
       def initialize(adapter_name, error)
-        adapter = demodulize(adapter_name.gsub('Adapter', ''))
+        adapter = demodulize(adapter_name.gsub("Adapter", ""))
 
         super("can't setup '#{adapter}' adapter during the following error:\n\t#{error}'")
       end
@@ -127,7 +127,7 @@ module ProxyFetcher
       #
       def demodulize(path)
         path = path.to_s
-        index = path.rindex('::')
+        index = path.rindex("::")
 
         index ? path[(index + 2)..-1] : path
       end

data/lib/proxy_fetcher/manager.rb

@@ -55,7 +55,7 @@ module ProxyFetcher
 
     # Pop just first proxy (and back it to the end of the proxy list).
     #
-    # @return [Proxy]
+    # @return [ProxyFetcher::Proxy, NilClass]
     #   proxy object from the list
     #
     def get
@@ -72,7 +72,7 @@ module ProxyFetcher
     # Pop first valid proxy (and back it to the end of the proxy list)
     # Invalid proxies will be removed from the list
     #
-    # @return [Proxy]
+    # @return [ProxyFetcher::Proxy, NilClass]
     #   proxy object from the list
     #
     def get!

data/lib/proxy_fetcher/providers/base.rb

@@ -6,12 +6,15 @@ module ProxyFetcher
     class Base
       # Loads proxy provider page content, extract proxy list from it
       # and convert every entry to proxy object.
-      def fetch_proxies!(filters = {})
+      def fetch_proxies(filters = {})
         raw_proxies = load_proxy_list(filters)
         proxies = raw_proxies.map { |html_node| build_proxy(html_node) }.compact
         proxies.reject { |proxy| proxy.addr.nil? }
       end
 
+      # For retro-compatibility
+      alias fetch_proxies! fetch_proxies
+
       def provider_url
         raise NotImplementedError, "#{__method__} must be implemented in a descendant class!"
       end
@@ -24,10 +27,17 @@ module ProxyFetcher
         {}
       end
 
+      # @return [Hash]
+      #   Provider headers required to fetch the proxy list
+      #
       def provider_headers
         {}
       end
 
+      def xpath
+        raise NotImplementedError, "#{__method__} must be implemented in a descendant class!"
+      end
+
       # Just synthetic sugar to make it easier to call #fetch_proxies! method.
       def self.fetch_proxies!(*args)
         new.fetch_proxies!(*args)
@@ -37,18 +47,29 @@ module ProxyFetcher
 
       # Loads raw provider HTML with proxies.
       #
+      # @param url [String]
+      #   Provider URL
+      #
+      # @param filters [#to_h]
+      #   Provider filters (Hash-like object)
+      #
       # @return [String]
-      #   HTML body
+      #   HTML body from the response
       #
       def load_html(url, filters = {})
-        raise ArgumentError, 'filters must be a Hash' if filters && !filters.is_a?(Hash)
+        unless filters.respond_to?(:to_h)
+          raise ArgumentError, "filters must be a Hash or respond to #to_h"
+        end
 
-        uri = URI.parse(url)
-        # TODO: query for post request?
-        uri.query = URI.encode_www_form(provider_params.merge(filters)) if filters && filters.any?
+        if filters&.any?
+          # TODO: query for post request?
+          uri = URI.parse(url)
+          uri.query = URI.encode_www_form(provider_params.merge(filters.to_h))
+          url = uri.to_s
+        end
 
         ProxyFetcher.config.http_client.fetch(
-          uri.to_s,
+          url,
           method: provider_method,
           headers: provider_headers,
           params: provider_params
@@ -71,29 +92,28 @@ module ProxyFetcher
         ProxyFetcher::Document.parse(html)
       end
 
+      # Fetches HTML content by sending HTTP request to the provider URL and
+      # parses the document (built as abstract <code>ProxyFetcher::Document</code>)
+      # to return all the proxy entries (HTML nodes).
+      #
+      # @return [Array<ProxyFetcher::Document::Node>]
+      #   Collection of extracted HTML nodes with full proxy info
+      #
+      def load_proxy_list(filters = {})
+        doc = load_document(provider_url, filters)
+        doc.xpath(xpath)
+      end
+
       def build_proxy(*args)
         to_proxy(*args)
-      rescue StandardError => error
+      rescue StandardError => e
         ProxyFetcher.logger.warn(
-          "Failed to build Proxy object for #{self.class.name} due to error: #{error.message}"
+          "Failed to build Proxy object for #{self.class.name} due to error: #{e.message}"
         )
 
         nil
       end
 
-      # Fetches HTML content by sending HTTP request to the provider URL and
-      # parses the document (built as abstract <code>ProxyFetcher::Document</code>)
-      # to return all the proxy entries (HTML nodes).
-      #
-      # Abstract method. Must be implemented in a descendant class
-      #
-      # @return [Array<Document::Node>]
-      #   list of proxy elements from the providers HTML content
-      #
-      def load_proxy_list(*)
-        raise NotImplementedError, "#{__method__} must be implemented in a descendant class!"
-      end
-
       # Convert HTML element with proxy info to ProxyFetcher::Proxy instance.
       #
       # Abstract method. Must be implemented in a descendant class