proxy_fetcher 0.11.0 → 0.15.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5a0d7b377ed3c25e50552e89ba76c0e73fad3923bf171e1cee2f592d777787c3
-  data.tar.gz: 83b594e04e03c74a63146a6907c99025d50607c88f8cf94f6d5ce044795243ad
+  metadata.gz: 071dd5bfeaf01174bf803981de6147f999e75b99a2d03c32e034b72dd5dc7c82
+  data.tar.gz: 1675ce07491e0cbce401c2a113ee7eccdcba0c4cf42ba7e380ad63be988ec91a
 SHA512:
-  metadata.gz: 1f096c2473035255eb9492297b5641ab5caee62566eb20eb40d3b2f02eea5d06fa1279a2cadeb3266c0f52ce98040185a69678faf116b6398c2c75f79d5c4ebd
-  data.tar.gz: a9372ef8bdbb3c51c5060308cbc46c905df3819e682bceb858ba494f5f94722095f2a2bee94575606f628d091b87325425c218f8c31a2b807bcb159c59ba6e65
+  metadata.gz: 42037b0b55109b535e25284815a096154c79bca2f58b107311f5c53da5b9ec0242ab39a6dd9bae48ef3e863c7f43251167180266578ac177887bd3a63215ef1e
+  data.tar.gz: 188dd298f8ea03e5985d89e97f99a12ef5d5384eae03a78f9b77e5172c28ae5990dec64795817d52ea6895395679a6c2f2deafafa3f8617154afb32fc06bc972

data/CHANGELOG.md

@@ -4,7 +4,38 @@ Reverse Chronological Order:
 
 ## `master`
 
-* Add your description here
+...
+
+## `0.15.1` (2021-02-17)
+
+* Support for Ruby 3.0
+
+## `0.15.0` (2021-01-26)
+
+* Removed failing providers
+* Added new
+* Specs refactoring
+
+## `0.14.0` (2020-05-11)
+
+* Add MTPro provider
+* Add Proxypedia provider
+
+## `0.13.0` (2020-03-09)
+
+* Fix GatherProxy provider
+* Fix XRoxy provider
+* Allow ability to load proxies from files
+* Fix Proxy object comparators
+
+## `0.12.0` (2020-01-28)
+
+* Fix XRoxy provider
+* Fix multi-threading issues with config and adapter
+
+## `0.11.0` (2019-10-24)
+
+* Big gem refactoring
 
 ## `0.10.2` (2019-03-15)
 

data/Gemfile

@@ -5,10 +5,12 @@ source "https://rubygems.org"
 gemspec
 
 gem "nokogiri", "~> 1.8"
-gem "oga", "~> 2.0"
-gem "rubocop", "~> 0.74"
+gem "oga", "~> 3.2"
+gem "rubocop", "~> 1.0"
 
 group :test do
   gem "coveralls", require: false
+  # Until I find a way to introduce other MITM proxy
+  gem "webrick", "1.4.2"
   gem "evil-proxy", "~> 0.2"
 end

data/Rakefile

@@ -3,6 +3,8 @@
 require "bundler/gem_tasks"
 
 require "rspec/core/rake_task"
-RSpec::Core::RakeTask.new(:spec)
+RSpec::Core::RakeTask.new(:spec) do |t|
+  t.rspec_opts = '--tag "~unreliable"'
+end
 
 task default: :spec

data/gemfiles/nokogiri.gemfile

@@ -8,6 +8,6 @@ gem "nokogiri", "~> 1.8"
 
 group :test do
   gem "coveralls", require: false
+  gem "webrick"
   gem "evil-proxy", "~> 0.2"
-  gem "rspec", "~> 3.6"
 end

data/gemfiles/oga.gemfile

@@ -4,10 +4,10 @@ source "https://rubygems.org"
 
 gemspec path: "../"
 
-gem "oga", "~> 2.0"
+gem "oga", "~> 3.0"
 
 group :test do
   gem "coveralls", require: false
+  gem "webrick"
   gem "evil-proxy", "~> 0.2"
-  gem "rspec", "~> 3.6"
 end

data/lib/proxy_fetcher.rb

@@ -4,43 +4,52 @@ require "uri"
 require "http"
 require "logger"
 
-require File.dirname(__FILE__) + "/proxy_fetcher/version"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/version"
 
-require File.dirname(__FILE__) + "/proxy_fetcher/exceptions"
-require File.dirname(__FILE__) + "/proxy_fetcher/configuration"
-require File.dirname(__FILE__) + "/proxy_fetcher/configuration/providers_registry"
-require File.dirname(__FILE__) + "/proxy_fetcher/proxy"
-require File.dirname(__FILE__) + "/proxy_fetcher/manager"
-require File.dirname(__FILE__) + "/proxy_fetcher/null_logger"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/exceptions"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/configuration"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/configuration/providers_registry"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/proxy"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/manager"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/null_logger"
 
-require File.dirname(__FILE__) + "/proxy_fetcher/utils/http_client"
-require File.dirname(__FILE__) + "/proxy_fetcher/utils/proxy_validator"
-require File.dirname(__FILE__) + "/proxy_fetcher/utils/proxy_list_validator"
-require File.dirname(__FILE__) + "/proxy_fetcher/client/client"
-require File.dirname(__FILE__) + "/proxy_fetcher/client/request"
-require File.dirname(__FILE__) + "/proxy_fetcher/client/proxies_registry"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/utils/http_client"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/utils/proxy_validator"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/utils/proxy_list_validator"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/client/client"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/client/request"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/client/proxies_registry"
 
-require File.dirname(__FILE__) + "/proxy_fetcher/document"
-require File.dirname(__FILE__) + "/proxy_fetcher/document/adapters"
-require File.dirname(__FILE__) + "/proxy_fetcher/document/node"
-require File.dirname(__FILE__) + "/proxy_fetcher/document/adapters/abstract_adapter"
-require File.dirname(__FILE__) + "/proxy_fetcher/document/adapters/nokogiri_adapter"
-require File.dirname(__FILE__) + "/proxy_fetcher/document/adapters/oga_adapter"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/document"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/document/adapters"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/document/node"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/document/adapters/abstract_adapter"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/document/adapters/nokogiri_adapter"
+require "#{File.dirname(__FILE__)}/proxy_fetcher/document/adapters/oga_adapter"
 
 ##
 # Ruby / JRuby lib for managing proxies
 module ProxyFetcher
   # ProxyFetcher providers namespace
   module Providers
-    require File.dirname(__FILE__) + "/proxy_fetcher/providers/base"
-    require File.dirname(__FILE__) + "/proxy_fetcher/providers/free_proxy_list"
-    require File.dirname(__FILE__) + "/proxy_fetcher/providers/free_proxy_list_ssl"
-    require File.dirname(__FILE__) + "/proxy_fetcher/providers/gather_proxy"
-    require File.dirname(__FILE__) + "/proxy_fetcher/providers/http_tunnel"
-    require File.dirname(__FILE__) + "/proxy_fetcher/providers/proxy_list"
-    require File.dirname(__FILE__) + "/proxy_fetcher/providers/xroxy"
+    require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/base"
+    require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/free_proxy_list"
+    require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/free_proxy_list_socks"
+    require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/free_proxy_list_ssl"
+    require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/free_proxy_list_us"
+    require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/http_tunnel"
+    require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/mtpro"
+    require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/proxy_list"
+    require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/proxypedia"
+    require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/proxyscrape_http"
+    require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/proxyscrape_socks4"
+    require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/proxyscrape_socks5"
+    require "#{File.dirname(__FILE__)}/proxy_fetcher/providers/xroxy"
   end
 
+  @__config_access_lock__ = Mutex.new
+  @__config_definition_lock__ = Mutex.new
+
   # Main ProxyFetcher module.
   class << self
     ##
@@ -53,12 +62,14 @@ module ProxyFetcher
     #   ProxyFetcher.config
     #
     #   #=> #<ProxyFetcher::Configuration:0x0000000241eec8 @user_agent="Mozilla/5.0, ...", @pool_size=10,
-    #           @client_timeout=3, @proxy_validation_timeout=3, @provider_proxies_load_timeout=30,
-    #           @http_client=ProxyFetcher::HTTPClient, @proxy_validator=ProxyFetcher::ProxyValidator,
-    #           @providers=[:free_proxy_list, ...], @adapter=ProxyFetcher::Document::NokogiriAdapter>
+    #          @client_timeout=3, @proxy_validation_timeout=3, @provider_proxies_load_timeout=30,
+    #          @http_client=ProxyFetcher::HTTPClient, @proxy_validator=ProxyFetcher::ProxyValidator,
+    #          @providers=[:free_proxy_list, ...], @adapter=ProxyFetcher::Document::NokogiriAdapter>
     #
     def config
-      @config ||= ProxyFetcher::Configuration.new
+      @__config_definition_lock__.synchronize do
+        @config ||= ProxyFetcher::Configuration.new
+      end
     end
 
     ##
@@ -70,7 +81,7 @@ module ProxyFetcher
     #   Configuration object.
     #
     def configure
-      yield config
+      @__config_access_lock__.synchronize { yield config }
     end
 
     # Returns ProxyFetcher logger instance.

data/lib/proxy_fetcher/client/request.rb

@@ -41,15 +41,15 @@ module ProxyFetcher
       # @return [String]
       #   response body (requested resource content)
       #
-      def self.execute(args)
-        new(args).execute
+      def self.execute(**args)
+        new(**args).execute
       end
 
       # Initialize new HTTP request
       #
       # @return [Request]
       #
-      def initialize(args)
+      def initialize(**args)
         raise ArgumentError, "args must be a Hash!" unless args.is_a?(Hash)
 
         @url = args.fetch(:url)

data/lib/proxy_fetcher/configuration.rb

@@ -39,7 +39,7 @@ module ProxyFetcher
     attr_accessor :logger
 
     # @!attribute [r] adapter
-    #   @return [Object] HTML parser adapter
+    #   @return [#to_s] HTML parser adapter
     attr_reader :adapter
 
     # @!attribute [r] http_client
@@ -51,7 +51,7 @@ module ProxyFetcher
     attr_reader :proxy_validator
 
     # @!attribute [r] providers
-    #   @return [Array<String>, Array<Symbol>] proxy providers list to be used
+    #   @return [Array<String, Symbol>] proxy providers list to be used
     attr_reader :providers
 
     # User-Agent string that will be used by the ProxyFetcher HTTP client (to
@@ -68,6 +68,8 @@ module ProxyFetcher
     #
     DEFAULT_ADAPTER = :nokogiri
 
+    @__adapter_lock__ = Mutex.new
+
     class << self
       # Registry for handling proxy providers.
       #
@@ -93,7 +95,7 @@ module ProxyFetcher
 
       # Returns registered providers names.
       #
-      # @return [Array<String>, Array<Symbol>]
+      # @return [Array<String, Symbol>]
       #   registered providers names
       #
       def registered_providers
@@ -112,7 +114,7 @@ module ProxyFetcher
 
     # Sets default configuration options
     def reset!
-      @logger = Logger.new(STDOUT)
+      @logger = Logger.new($stdout)
       @user_agent = DEFAULT_USER_AGENT
       @pool_size = 10
       @client_timeout = 3
@@ -131,16 +133,18 @@ module ProxyFetcher
     end
 
     def adapter_class
-      return @adapter_class if defined?(@adapter_class)
+      self.class.instance_variable_get(:@__adapter_lock__).synchronize do
+        return @adapter_class if defined?(@adapter_class)
 
-      @adapter_class = ProxyFetcher::Document::Adapters.lookup(adapter)
-      @adapter_class.setup!
-      @adapter_class
+        @adapter_class = ProxyFetcher::Document::Adapters.lookup(adapter)
+        @adapter_class.setup!
+        @adapter_class
+      end
     end
 
     # Setups collection of providers that will be used to fetch proxies.
     #
-    # @param value [String, Symbol, Array<String>, Array<Symbol>]
+    # @param value [String, Symbol, Array<String, Symbol>]
     #   provider names
     #
     def providers=(value)

data/lib/proxy_fetcher/document/node.rb

@@ -83,7 +83,7 @@ module ProxyFetcher
       def clear(text)
         return "" if text.nil? || text.empty?
 
-        text.strip.gsub(/[\t]/i, "")
+        text.strip.gsub(/\t/i, "")
       end
     end
   end

data/lib/proxy_fetcher/manager.rb

@@ -3,6 +3,16 @@
 module ProxyFetcher
   # ProxyFetcher Manager class for interacting with proxy lists from various providers.
   class Manager
+    REFRESHER_LOCK = Mutex.new
+
+    class << self
+      def from_files(files, **options)
+        new(**options.merge(files: Array(files)))
+      end
+
+      alias from_file from_files
+    end
+
     # @!attribute [r] proxies
     #   @return [Array<ProxyFetcher::Proxy>] An array of proxies
     attr_reader :proxies
@@ -14,14 +24,17 @@ module ProxyFetcher
     #
     # @return [Manager]
     #
-    def initialize(refresh: true, validate: false, filters: {})
-      if refresh
-        refresh_list!(filters)
+    def initialize(**options)
+      if options.fetch(:refresh, true)
+        refresh_list!(options.fetch(:filters, {}))
       else
         @proxies = []
       end
 
-      cleanup! if validate
+      files = Array(options.fetch(:file, options.fetch(:files, [])))
+      load_proxies_from_files!(files) if files&.any?
+
+      cleanup! if options.fetch(:validate, false)
     end
 
     # Update current proxy list using configured providers.
@@ -30,17 +43,17 @@ module ProxyFetcher
     #
     def refresh_list!(filters = nil)
       @proxies = []
-
       threads = []
-      lock = Mutex.new
 
       ProxyFetcher.config.providers.each do |provider_name|
         threads << Thread.new do
+          Thread.current.report_on_exception = false
+
           provider = ProxyFetcher::Configuration.providers_registry.class_for(provider_name)
           provider_filters = filters && filters.fetch(provider_name.to_sym, filters)
           provider_proxies = provider.fetch_proxies!(provider_filters)
 
-          lock.synchronize do
+          REFRESHER_LOCK.synchronize do
             @proxies.concat(provider_proxies)
           end
         end
@@ -89,6 +102,26 @@ module ProxyFetcher
 
     alias pop! get!
 
+    # Loads proxies from files.
+    #
+    # @param proxy_files [String, Array<String,Pathname>]
+    #   file path of list of files to load
+    #
+    def load_proxies_from_files!(proxy_files)
+      proxy_files = Array(proxy_files)
+      return if proxy_files.empty?
+
+      proxy_files.each do |proxy_file|
+        File.foreach(proxy_file, chomp: true) do |proxy_string|
+          addr, port = proxy_string.split(":", 2)
+          port = Integer(port) if port
+          @proxies << Proxy.new(addr: addr, port: port)
+        end
+      end
+
+      @proxies.uniq!
+    end
+
     # Clean current proxy list from dead proxies (that doesn't respond by timeout)
     #
     # @return [Array<ProxyFetcher::Proxy>]

data/lib/proxy_fetcher/providers/base.rb

@@ -108,7 +108,8 @@ module ProxyFetcher
         to_proxy(*args)
       rescue StandardError => e
         ProxyFetcher.logger.warn(
-          "Failed to build Proxy object for #{self.class.name} due to error: #{e.message}"
+          "Failed to build Proxy for #{self.class.name.split("::").last} " \
+          "due to error: #{e.message}"
         )
 
         nil

data/lib/proxy_fetcher/providers/free_proxy_list.rb

@@ -45,27 +45,6 @@ module ProxyFetcher
       #
       def parse_type(html_node)
         https = html_node.content_at("td[6]")
-        # frozen_string_literal: true
-        # FreeProxyList provider class.
-        # Provider URL to fetch proxy list
-        # [NOTE] Doesn't support filtering
-        # Converts HTML node (entry of N tags) to <code>ProxyFetcher::Proxy</code>
-        # object.
-        #
-        # @param html_node [Object]
-        #   HTML node from the <code>ProxyFetcher::Document</code> DOM model.
-        #
-        # @return [ProxyFetcher::Proxy]
-        #   Proxy object
-        #
-        # Parses HTML node to extract proxy type.
-        #
-        # @param html_node [Object]
-        #   HTML node from the <code>ProxyFetcher::Document</code> DOM model.
-        #
-        # @return [String]
-        #   Proxy type
-        #
         https&.casecmp("yes")&.zero? ? ProxyFetcher::Proxy::HTTPS : ProxyFetcher::Proxy::HTTP
       end
     end

data/lib/proxy_fetcher/providers/free_proxy_list_socks.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module ProxyFetcher
+  module Providers
+    # FreeProxyListSocks provider class.
+    class FreeProxyListSocks < Base
+      # Provider URL to fetch proxy list
+      def provider_url
+        "https://www.socks-proxy.net/"
+      end
+
+      # [NOTE] Doesn't support filtering
+      def xpath
+        '//table[@id="proxylisttable"]/tbody/tr'
+      end
+
+      # Converts HTML node (entry of N tags) to <code>ProxyFetcher::Proxy</code>
+      # object.
+      #
+      # @param html_node [Object]
+      #   HTML node from the <code>ProxyFetcher::Document</code> DOM model.
+      #
+      # @return [ProxyFetcher::Proxy]
+      #   Proxy object
+      #
+      def to_proxy(html_node)
+        ProxyFetcher::Proxy.new.tap do |proxy|
+          proxy.addr = html_node.content_at("td[1]")
+          proxy.port = Integer(html_node.content_at("td[2]").gsub(/^0+/, ""))
+          proxy.country = html_node.content_at("td[4]")
+          proxy.type = parse_type(html_node)
+          proxy.anonymity = html_node.content_at("td[6]")
+        end
+      end
+
+      private
+
+      # Parses HTML node to extract proxy type.
+      #
+      # @param html_node [Object]
+      #   HTML node from the <code>ProxyFetcher::Document</code> DOM model.
+      #
+      # @return [String]
+      #   Proxy type
+      #
+      def parse_type(html_node)
+        https = html_node.content_at("td[5]")
+
+        return ProxyFetcher::Proxy::SOCKS4 if https&.casecmp("socks4")&.zero?
+        return ProxyFetcher::Proxy::SOCKS5 if https&.casecmp("socks5")&.zero?
+
+        "Unknown"
+      end
+    end
+
+    ProxyFetcher::Configuration.register_provider(:free_proxy_list_socks, FreeProxyListSocks)
+  end
+end