proxy_fetcher 0.10.2 → 0.15.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d256aaf841030823ef753d6c2b2726dd7e8e5306dbddae6b6079a26815df4224
-  data.tar.gz: a747a8d9ed567ffa6373bd0d30dc9a99e5320cb03b75dd3feb5ef45465144375
+  metadata.gz: bf6591a8ede2c142fca7f433f3edceb407ed884c1dee6a98e3ca33efbae392e2
+  data.tar.gz: 7d7259da718adec21b8d749d04faf594fa926389e570be6c55bbe9ebf3695783
 SHA512:
-  metadata.gz: '01997f86deda80383198be18ac9bca8309970fa38ea337a082093822838732038425b642a6806798fa35e2a891b63d679020fad33fd5f447efed00b426b85706'
-  data.tar.gz: 2fd10653fdf594e8847d4ebe34cb66c30b3d79d2900bd741768a5d4742ffa50923092f2c8f2bd95e3215f505a073e040e27cad8e2d868ff5837e81078c7d6f95
+  metadata.gz: 90eed5f390d122ae58f9bd6c7553b7a1ff6bbca8e65c05406593f53b7a00a4ed160120ae62d0ce8289965810aaa3795287769ee9ef12f9404d805694a8ac0cff
+  data.tar.gz: 31a5b4fcd6199dfc75ca3e89c7bddcc82533ca171f86954e50d4ff44716d73f1ad014f921a05e9eb2fd38a83c5e603eec1fbbf9530b4c01153f6780877388501

data/CHANGELOG.md

@@ -4,7 +4,28 @@ Reverse Chronological Order:
 
 ## `master`
 
-* Add your description here
+...
+
+## `0.14.0` (2020-05-11)
+
+* Add MTPro provider
+* Add Proxypedia provider
+
+## `0.13.0` (2020-03-09)
+
+* Fix GatherProxy provider
+* Fix XRoxy provider
+* Allow ability to load proxies from files
+* Fix Proxy object comparators
+
+## `0.12.0` (2020-01-28)
+
+* Fix XRoxy provider
+* Fix multi-threading issues with config and adapter
+
+## `0.11.0` (2019-10-24)
+
+* Big gem refactoring
 
 ## `0.10.2` (2019-03-15)
 

data/Gemfile

@@ -1,11 +1,14 @@
-source 'https://rubygems.org'
+# frozen_string_literal: true
+
+source "https://rubygems.org"
 
 gemspec
 
-gem 'nokogiri', '~> 1.8'
-gem 'oga', '~> 2.0'
+gem "nokogiri", "~> 1.8"
+gem "oga", "~> 3.2"
+gem "rubocop", "~> 0.80"
 
 group :test do
-  gem 'coveralls', require: false
-  gem 'evil-proxy', '~> 0.2'
+  gem "coveralls", require: false
+  gem "evil-proxy", "~> 0.2"
 end

data/Rakefile

@@ -1,6 +1,10 @@
-require 'bundler/gem_tasks'
+# frozen_string_literal: true
 
-require 'rspec/core/rake_task'
-RSpec::Core::RakeTask.new(:spec)
+require "bundler/gem_tasks"
+
+require "rspec/core/rake_task"
+RSpec::Core::RakeTask.new(:spec) do |t|
+  t.rspec_opts = '--tag "~unreliable"'
+end
 
 task default: :spec

data/gemfiles/nokogiri.gemfile

@@ -1,11 +1,13 @@
-source 'https://rubygems.org'
+# frozen_string_literal: true
 
-gemspec path: '../'
+source "https://rubygems.org"
 
-gem 'nokogiri', '~> 1.8'
+gemspec path: "../"
+
+gem "nokogiri", "~> 1.8"
 
 group :test do
-  gem 'coveralls', require: false
-  gem 'evil-proxy', '~> 0.2'
-  gem 'rspec', '~> 3.6'
+  gem "coveralls", require: false
+  gem "evil-proxy", "~> 0.2"
+  gem "rspec", "~> 3.9"
 end

data/gemfiles/oga.gemfile

@@ -1,11 +1,13 @@
-source 'https://rubygems.org'
+# frozen_string_literal: true
 
-gemspec path: '../'
+source "https://rubygems.org"
 
-gem 'oga', '~> 2.0'
+gemspec path: "../"
+
+gem "oga", "~> 3.0"
 
 group :test do
-  gem 'coveralls', require: false
-  gem 'evil-proxy', '~> 0.2'
-  gem 'rspec', '~> 3.6'
+  gem "coveralls", require: false
+  gem "evil-proxy", "~> 0.2"
+  gem "rspec", "~> 3.9"
 end

data/lib/proxy_fetcher.rb

@@ -1,46 +1,55 @@
 # frozen_string_literal: true
 
-require 'uri'
-require 'http'
-require 'logger'
+require "uri"
+require "http"
+require "logger"
 
-require File.dirname(__FILE__) + '/proxy_fetcher/version'
+require File.dirname(__FILE__) + "/proxy_fetcher/version"
 
-require File.dirname(__FILE__) + '/proxy_fetcher/exceptions'
-require File.dirname(__FILE__) + '/proxy_fetcher/configuration'
-require File.dirname(__FILE__) + '/proxy_fetcher/configuration/providers_registry'
-require File.dirname(__FILE__) + '/proxy_fetcher/proxy'
-require File.dirname(__FILE__) + '/proxy_fetcher/manager'
-require File.dirname(__FILE__) + '/proxy_fetcher/null_logger'
+require File.dirname(__FILE__) + "/proxy_fetcher/exceptions"
+require File.dirname(__FILE__) + "/proxy_fetcher/configuration"
+require File.dirname(__FILE__) + "/proxy_fetcher/configuration/providers_registry"
+require File.dirname(__FILE__) + "/proxy_fetcher/proxy"
+require File.dirname(__FILE__) + "/proxy_fetcher/manager"
+require File.dirname(__FILE__) + "/proxy_fetcher/null_logger"
 
-require File.dirname(__FILE__) + '/proxy_fetcher/utils/http_client'
-require File.dirname(__FILE__) + '/proxy_fetcher/utils/proxy_validator'
-require File.dirname(__FILE__) + '/proxy_fetcher/utils/proxy_list_validator'
-require File.dirname(__FILE__) + '/proxy_fetcher/client/client'
-require File.dirname(__FILE__) + '/proxy_fetcher/client/request'
-require File.dirname(__FILE__) + '/proxy_fetcher/client/proxies_registry'
+require File.dirname(__FILE__) + "/proxy_fetcher/utils/http_client"
+require File.dirname(__FILE__) + "/proxy_fetcher/utils/proxy_validator"
+require File.dirname(__FILE__) + "/proxy_fetcher/utils/proxy_list_validator"
+require File.dirname(__FILE__) + "/proxy_fetcher/client/client"
+require File.dirname(__FILE__) + "/proxy_fetcher/client/request"
+require File.dirname(__FILE__) + "/proxy_fetcher/client/proxies_registry"
 
-require File.dirname(__FILE__) + '/proxy_fetcher/document'
-require File.dirname(__FILE__) + '/proxy_fetcher/document/adapters'
-require File.dirname(__FILE__) + '/proxy_fetcher/document/node'
-require File.dirname(__FILE__) + '/proxy_fetcher/document/adapters/abstract_adapter'
-require File.dirname(__FILE__) + '/proxy_fetcher/document/adapters/nokogiri_adapter'
-require File.dirname(__FILE__) + '/proxy_fetcher/document/adapters/oga_adapter'
+require File.dirname(__FILE__) + "/proxy_fetcher/document"
+require File.dirname(__FILE__) + "/proxy_fetcher/document/adapters"
+require File.dirname(__FILE__) + "/proxy_fetcher/document/node"
+require File.dirname(__FILE__) + "/proxy_fetcher/document/adapters/abstract_adapter"
+require File.dirname(__FILE__) + "/proxy_fetcher/document/adapters/nokogiri_adapter"
+require File.dirname(__FILE__) + "/proxy_fetcher/document/adapters/oga_adapter"
 
 ##
 # Ruby / JRuby lib for managing proxies
 module ProxyFetcher
   # ProxyFetcher providers namespace
   module Providers
-    require File.dirname(__FILE__) + '/proxy_fetcher/providers/base'
-    require File.dirname(__FILE__) + '/proxy_fetcher/providers/free_proxy_list'
-    require File.dirname(__FILE__) + '/proxy_fetcher/providers/free_proxy_list_ssl'
-    require File.dirname(__FILE__) + '/proxy_fetcher/providers/gather_proxy'
-    require File.dirname(__FILE__) + '/proxy_fetcher/providers/http_tunnel'
-    require File.dirname(__FILE__) + '/proxy_fetcher/providers/proxy_list'
-    require File.dirname(__FILE__) + '/proxy_fetcher/providers/xroxy'
+    require File.dirname(__FILE__) + "/proxy_fetcher/providers/base"
+    require File.dirname(__FILE__) + "/proxy_fetcher/providers/free_proxy_list"
+    require File.dirname(__FILE__) + "/proxy_fetcher/providers/free_proxy_list_socks"
+    require File.dirname(__FILE__) + "/proxy_fetcher/providers/free_proxy_list_ssl"
+    require File.dirname(__FILE__) + "/proxy_fetcher/providers/free_proxy_list_us"
+    require File.dirname(__FILE__) + "/proxy_fetcher/providers/http_tunnel"
+    require File.dirname(__FILE__) + "/proxy_fetcher/providers/mtpro"
+    require File.dirname(__FILE__) + "/proxy_fetcher/providers/proxy_list"
+    require File.dirname(__FILE__) + "/proxy_fetcher/providers/proxypedia"
+    require File.dirname(__FILE__) + "/proxy_fetcher/providers/proxyscrape_http"
+    require File.dirname(__FILE__) + "/proxy_fetcher/providers/proxyscrape_socks4"
+    require File.dirname(__FILE__) + "/proxy_fetcher/providers/proxyscrape_socks5"
+    require File.dirname(__FILE__) + "/proxy_fetcher/providers/xroxy"
   end
 
+  @__config_access_lock__ = Mutex.new
+  @__config_definition_lock__ = Mutex.new
+
   # Main ProxyFetcher module.
   class << self
     ##
@@ -53,12 +62,14 @@ module ProxyFetcher
     #   ProxyFetcher.config
     #
     #   #=> #<ProxyFetcher::Configuration:0x0000000241eec8 @user_agent="Mozilla/5.0, ...", @pool_size=10,
-    #           @client_timeout=3, @proxy_validation_timeout=3, @provider_proxies_load_timeout=30,
-    #           @http_client=ProxyFetcher::HTTPClient, @proxy_validator=ProxyFetcher::ProxyValidator,
-    #           @providers=[:free_proxy_list, ...], @adapter=ProxyFetcher::Document::NokogiriAdapter>
+    #          @client_timeout=3, @proxy_validation_timeout=3, @provider_proxies_load_timeout=30,
+    #          @http_client=ProxyFetcher::HTTPClient, @proxy_validator=ProxyFetcher::ProxyValidator,
+    #          @providers=[:free_proxy_list, ...], @adapter=ProxyFetcher::Document::NokogiriAdapter>
     #
     def config
-      @config ||= ProxyFetcher::Configuration.new
+      @__config_definition_lock__.synchronize do
+        @config ||= ProxyFetcher::Configuration.new
+      end
     end
 
     ##
@@ -70,12 +81,12 @@ module ProxyFetcher
     #   Configuration object.
     #
     def configure
-      yield config
+      @__config_access_lock__.synchronize { yield config }
     end
 
     # Returns ProxyFetcher logger instance.
     #
-    # @return [Logger, NullLogger] logger object
+    # @return [Logger, ProxyFetcher::NullLogger] logger object
     #
     def logger
       return @logger if defined?(@logger)

data/lib/proxy_fetcher/client/client.rb

@@ -128,7 +128,11 @@ module ProxyFetcher
       #
       def request_with_payload(method, url, payload, headers, options)
         with_proxy_for(url, options.fetch(:max_retries, 1000)) do |proxy|
-          opts = options.merge(payload: payload, proxy: options.fetch(:proxy, proxy), headers: default_headers.merge(headers))
+          opts = options.merge(
+            payload: payload,
+            proxy: options.fetch(:proxy, proxy),
+            headers: default_headers.merge(headers)
+          )
 
           Request.execute(url: url, method: method, **opts)
         end
@@ -138,7 +142,10 @@ module ProxyFetcher
       #
       def request_without_payload(method, url, headers, options)
         with_proxy_for(url, options.fetch(:max_retries, 1000)) do |proxy|
-          opts = options.merge(proxy: options.fetch(:proxy, proxy), headers: default_headers.merge(headers))
+          opts = options.merge(
+            proxy: options.fetch(:proxy, proxy),
+            headers: default_headers.merge(headers)
+          )
 
           Request.execute(url: url, method: method, **opts)
         end
@@ -152,7 +159,7 @@ module ProxyFetcher
       #
       def default_headers
         {
-          'User-Agent' => ProxyFetcher.config.user_agent
+          "User-Agent" => ProxyFetcher.config.user_agent
         }
       end
 

data/lib/proxy_fetcher/client/request.rb

@@ -50,7 +50,7 @@ module ProxyFetcher
       # @return [Request]
       #
       def initialize(args)
-        raise ArgumentError, 'args must be a Hash!' unless args.is_a?(Hash)
+        raise ArgumentError, "args must be a Hash!" unless args.is_a?(Hash)
 
         @url = args.fetch(:url)
         @method = args.fetch(:method).to_s.downcase
@@ -86,9 +86,9 @@ module ProxyFetcher
       #
       def build_http_client
         HTTP.via(proxy.addr, proxy.port.to_i)
-            .headers(headers)
-            .timeout(connect: timeout, read: timeout)
-            .follow(max_hops: max_redirects)
+          .headers(headers)
+          .timeout(connect: timeout, read: timeout)
+          .follow(max_hops: max_redirects)
       end
 
       # Default SSL options that will be used for connecting to resources

data/lib/proxy_fetcher/configuration.rb

@@ -6,15 +6,20 @@ module ProxyFetcher
   #
   class Configuration
     # @!attribute client_timeout
-    #   @return [Integer] HTTP request timeout (connect / open) for [ProxyFetcher::Client]
+    #   @return [Integer]
+    #     HTTP request timeout (connect / open) for [ProxyFetcher::Client]
     attr_accessor :client_timeout
 
     # @!attribute provider_proxies_load_timeout
-    #   @return [Integer] HTTP request timeout (connect / open) for loading of proxies list by provider
+    #   @return [Integer]
+    #     HTTP request timeout (connect / open) for loading
+    #     of proxies list by provider
     attr_accessor :provider_proxies_load_timeout
 
     # @!attribute proxy_validation_timeout
-    #   @return [Integer] HTTP request timeout (connect / open) for proxy validation with [ProxyFetcher::ProxyValidator]
+    #   @return [Integer]
+    #     HTTP request timeout (connect / open) for proxy
+    #     validation with [ProxyFetcher::ProxyValidator]
     attr_accessor :proxy_validation_timeout
 
     # to save compatibility
@@ -30,16 +35,12 @@ module ProxyFetcher
     attr_accessor :user_agent
 
     # @!attribute [r] logger
-    #   @return [Object] Logger object
+    #   @return [Logger] Logger object
     attr_accessor :logger
 
     # @!attribute [r] adapter
-    #   @return [Object] HTML parser adapter
-    attr_accessor :adapter
-
-    # @!attribute [r] adapter_class
-    #   @return [Object] HTML adapter class
-    attr_reader :adapter_class
+    #   @return [#to_s] HTML parser adapter
+    attr_reader :adapter
 
     # @!attribute [r] http_client
     #   @return [Object] HTTP client class
@@ -50,7 +51,7 @@ module ProxyFetcher
     attr_reader :proxy_validator
 
     # @!attribute [r] providers
-    #   @return [Array<String>, Array<Symbol>] proxy providers list to be used
+    #   @return [Array<String, Symbol>] proxy providers list to be used
     attr_reader :providers
 
     # User-Agent string that will be used by the ProxyFetcher HTTP client (to
@@ -58,8 +59,8 @@ module ProxyFetcher
     #
     # Default is Google Chrome 60, but can be changed in <code>ProxyFetcher.config</code>.
     #
-    DEFAULT_USER_AGENT = 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 ' \
-                         '(KHTML, like Gecko) Chrome/60.0.3112 Safari/537.36'.freeze
+    DEFAULT_USER_AGENT = "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 " \
+                         "(KHTML, like Gecko) Chrome/60.0.3112 Safari/537.36"
 
     # HTML parser adapter name.
     #
@@ -67,6 +68,8 @@ module ProxyFetcher
     #
     DEFAULT_ADAPTER = :nokogiri
 
+    @__adapter_lock__ = Mutex.new
+
     class << self
       # Registry for handling proxy providers.
       #
@@ -92,7 +95,7 @@ module ProxyFetcher
 
       # Returns registered providers names.
       #
-      # @return [Array<String>, Array<Symbol>]
+      # @return [Array<String, Symbol>]
       #   registered providers names
       #
       def registered_providers
@@ -130,16 +133,18 @@ module ProxyFetcher
     end
 
     def adapter_class
-      return @adapter_class if defined?(@adapter_class)
+      self.class.instance_variable_get(:@__adapter_lock__).synchronize do
+        return @adapter_class if defined?(@adapter_class)
 
-      @adapter_class = ProxyFetcher::Document::Adapters.lookup(adapter)
-      @adapter_class.setup!
-      @adapter_class
+        @adapter_class = ProxyFetcher::Document::Adapters.lookup(adapter)
+        @adapter_class.setup!
+        @adapter_class
+      end
     end
 
     # Setups collection of providers that will be used to fetch proxies.
     #
-    # @param value [String, Symbol, Array<String>, Array<Symbol>]
+    # @param value [String, Symbol, Array<String, Symbol>]
     #   provider names
     #
     def providers=(value)

data/lib/proxy_fetcher/document.rb

@@ -36,14 +36,5 @@ module ProxyFetcher
     def xpath(*args)
       backend.xpath(*args).map { |node| backend.proxy_node.new(node) }
     end
-
-    # Searches elements by CSS selector.
-    #
-    # @return [Array<ProxyFetcher::Document::Node>]
-    #   collection of nodes
-    #
-    def css(*args)
-      backend.css(*args).map { |node| backend.proxy_node.new(node) }
-    end
   end
 end

data/lib/proxy_fetcher/document/adapters.rb

@@ -13,7 +13,7 @@ module ProxyFetcher
     # <code>ProxyFetcher::Document::AbstractAdapter</code>.
     class Adapters
       # Adapters class name suffix
-      ADAPTER = 'Adapter'.freeze
+      ADAPTER = "Adapter"
       private_constant :ADAPTER
 
       class << self

data/lib/proxy_fetcher/document/adapters/abstract_adapter.rb

@@ -26,15 +26,6 @@ module ProxyFetcher
         document.xpath(selector)
       end
 
-      # You can override this method in your own adapter class
-      #
-      # @param selector [String]
-      #   CSS selector
-      #
-      def css(selector)
-        document.css(selector)
-      end
-
       # Returns <code>Node</code> class that will handle HTML
       # nodes for particular adapter.
       #
@@ -42,7 +33,7 @@ module ProxyFetcher
       #   node
       #
       def proxy_node
-        self.class.const_get('Node')
+        self.class.const_get("Node")
       end
 
       # Installs adapter requirements.
@@ -53,8 +44,8 @@ module ProxyFetcher
       def self.setup!(*args)
         install_requirements!(*args)
         self
-      rescue LoadError, StandardError => error
-        raise Exceptions::AdapterSetupError.new(name, error.message)
+      rescue LoadError, StandardError => e
+        raise Exceptions::AdapterSetupError.new(name, e.message)
       end
     end
   end