proxy_fetcher 0.10.2 → 0.15.0

data/lib/proxy_fetcher/document/adapters/nokogiri_adapter.rb

@@ -6,7 +6,7 @@ module ProxyFetcher
     class NokogiriAdapter < AbstractAdapter
       # Requires Nokogiri gem to the application.
       def self.install_requirements!
-        require 'nokogiri'
+        require "nokogiri"
       end
 
       # Parses raw HTML content with specific gem.

data/lib/proxy_fetcher/document/adapters/oga_adapter.rb

@@ -6,7 +6,7 @@ module ProxyFetcher
     class OgaAdapter < AbstractAdapter
       # Requires Oga gem to the application.
       def self.install_requirements!
-        require 'oga'
+        require "oga"
       end
 
       # Parses raw HTML content with specific gem.

data/lib/proxy_fetcher/document/node.rb

@@ -81,9 +81,9 @@ module ProxyFetcher
       #   clean text
       #
       def clear(text)
-        return '' if text.nil? || text.empty?
+        return "" if text.nil? || text.empty?
 
-        text.strip.gsub(/[ \t]/i, '')
+        text.strip.gsub(/[\t]/i, "")
       end
     end
   end

data/lib/proxy_fetcher/exceptions.rb

@@ -13,7 +13,7 @@ module ProxyFetcher
       # @return [WrongCustomClass]
       #
       def initialize(klass, methods)
-        required_methods = Array(methods).join(', ')
+        required_methods = Array(methods).join(", ")
         super("#{klass} must respond to [#{required_methods}] class methods!")
       end
     end
@@ -53,7 +53,7 @@ module ProxyFetcher
       # @return [MaximumRedirectsReached]
       #
       def initialize(*)
-        super('maximum redirects reached')
+        super("maximum redirects reached")
       end
     end
 
@@ -66,7 +66,7 @@ module ProxyFetcher
       # @return [MaximumRetriesReached]
       #
       def initialize(*)
-        super('reached the maximum number of retries')
+        super("reached the maximum number of retries")
       end
     end
 
@@ -95,7 +95,7 @@ module ProxyFetcher
         super(<<-MSG.strip.squeeze
           you need to specify adapter for HTML parsing: ProxyFetcher.config.adapter = :nokogiri.
           You can use one of the predefined adapters (:nokogiri or :oga) or your own implementation.
-          MSG
+        MSG
         )
       end
     end
@@ -111,7 +111,7 @@ module ProxyFetcher
       # @return [AdapterSetupError]
       #
       def initialize(adapter_name, error)
-        adapter = demodulize(adapter_name.gsub('Adapter', ''))
+        adapter = demodulize(adapter_name.gsub("Adapter", ""))
 
         super("can't setup '#{adapter}' adapter during the following error:\n\t#{error}'")
       end
@@ -127,7 +127,7 @@ module ProxyFetcher
       #
       def demodulize(path)
         path = path.to_s
-        index = path.rindex('::')
+        index = path.rindex("::")
 
         index ? path[(index + 2)..-1] : path
       end

data/lib/proxy_fetcher/manager.rb

@@ -3,6 +3,16 @@
 module ProxyFetcher
   # ProxyFetcher Manager class for interacting with proxy lists from various providers.
   class Manager
+    REFRESHER_LOCK = Mutex.new
+
+    class << self
+      def from_files(files, **options)
+        new(**options.merge(files: Array(files)))
+      end
+
+      alias from_file from_files
+    end
+
     # @!attribute [r] proxies
     #   @return [Array<ProxyFetcher::Proxy>] An array of proxies
     attr_reader :proxies
@@ -14,14 +24,17 @@ module ProxyFetcher
     #
     # @return [Manager]
     #
-    def initialize(refresh: true, validate: false, filters: {})
-      if refresh
-        refresh_list!(filters)
+    def initialize(**options)
+      if options.fetch(:refresh, true)
+        refresh_list!(options.fetch(:filters, {}))
       else
         @proxies = []
       end
 
-      cleanup! if validate
+      files = Array(options.fetch(:file, options.fetch(:files, [])))
+      load_proxies_from_files!(files) if files&.any?
+
+      cleanup! if options.fetch(:validate, false)
     end
 
     # Update current proxy list using configured providers.
@@ -30,17 +43,17 @@ module ProxyFetcher
     #
     def refresh_list!(filters = nil)
       @proxies = []
-
       threads = []
-      lock = Mutex.new
 
       ProxyFetcher.config.providers.each do |provider_name|
         threads << Thread.new do
+          Thread.current.report_on_exception = false
+
           provider = ProxyFetcher::Configuration.providers_registry.class_for(provider_name)
           provider_filters = filters && filters.fetch(provider_name.to_sym, filters)
           provider_proxies = provider.fetch_proxies!(provider_filters)
 
-          lock.synchronize do
+          REFRESHER_LOCK.synchronize do
             @proxies.concat(provider_proxies)
           end
         end
@@ -55,7 +68,7 @@ module ProxyFetcher
 
     # Pop just first proxy (and back it to the end of the proxy list).
     #
-    # @return [Proxy]
+    # @return [ProxyFetcher::Proxy, NilClass]
     #   proxy object from the list
     #
     def get
@@ -72,7 +85,7 @@ module ProxyFetcher
     # Pop first valid proxy (and back it to the end of the proxy list)
     # Invalid proxies will be removed from the list
     #
-    # @return [Proxy]
+    # @return [ProxyFetcher::Proxy, NilClass]
     #   proxy object from the list
     #
     def get!
@@ -89,6 +102,26 @@ module ProxyFetcher
 
     alias pop! get!
 
+    # Loads proxies from files.
+    #
+    # @param proxy_files [String, Array<String,Pathname>]
+    #   file path of list of files to load
+    #
+    def load_proxies_from_files!(proxy_files)
+      proxy_files = Array(proxy_files)
+      return if proxy_files.empty?
+
+      proxy_files.each do |proxy_file|
+        File.foreach(proxy_file, chomp: true) do |proxy_string|
+          addr, port = proxy_string.split(":", 2)
+          port = Integer(port) if port
+          @proxies << Proxy.new(addr: addr, port: port)
+        end
+      end
+
+      @proxies.uniq!
+    end
+
     # Clean current proxy list from dead proxies (that doesn't respond by timeout)
     #
     # @return [Array<ProxyFetcher::Proxy>]

data/lib/proxy_fetcher/providers/base.rb

@@ -6,12 +6,15 @@ module ProxyFetcher
     class Base
       # Loads proxy provider page content, extract proxy list from it
       # and convert every entry to proxy object.
-      def fetch_proxies!(filters = {})
+      def fetch_proxies(filters = {})
         raw_proxies = load_proxy_list(filters)
         proxies = raw_proxies.map { |html_node| build_proxy(html_node) }.compact
         proxies.reject { |proxy| proxy.addr.nil? }
       end
 
+      # For retro-compatibility
+      alias fetch_proxies! fetch_proxies
+
       def provider_url
         raise NotImplementedError, "#{__method__} must be implemented in a descendant class!"
       end
@@ -24,10 +27,17 @@ module ProxyFetcher
         {}
       end
 
+      # @return [Hash]
+      #   Provider headers required to fetch the proxy list
+      #
       def provider_headers
         {}
       end
 
+      def xpath
+        raise NotImplementedError, "#{__method__} must be implemented in a descendant class!"
+      end
+
       # Just synthetic sugar to make it easier to call #fetch_proxies! method.
       def self.fetch_proxies!(*args)
         new.fetch_proxies!(*args)
@@ -37,18 +47,29 @@ module ProxyFetcher
 
       # Loads raw provider HTML with proxies.
       #
+      # @param url [String]
+      #   Provider URL
+      #
+      # @param filters [#to_h]
+      #   Provider filters (Hash-like object)
+      #
       # @return [String]
-      #   HTML body
+      #   HTML body from the response
       #
       def load_html(url, filters = {})
-        raise ArgumentError, 'filters must be a Hash' if filters && !filters.is_a?(Hash)
+        unless filters.respond_to?(:to_h)
+          raise ArgumentError, "filters must be a Hash or respond to #to_h"
+        end
 
-        uri = URI.parse(url)
-        # TODO: query for post request?
-        uri.query = URI.encode_www_form(provider_params.merge(filters)) if filters && filters.any?
+        if filters&.any?
+          # TODO: query for post request?
+          uri = URI.parse(url)
+          uri.query = URI.encode_www_form(provider_params.merge(filters.to_h))
+          url = uri.to_s
+        end
 
         ProxyFetcher.config.http_client.fetch(
-          uri.to_s,
+          url,
           method: provider_method,
           headers: provider_headers,
           params: provider_params
@@ -71,29 +92,29 @@ module ProxyFetcher
         ProxyFetcher::Document.parse(html)
       end
 
+      # Fetches HTML content by sending HTTP request to the provider URL and
+      # parses the document (built as abstract <code>ProxyFetcher::Document</code>)
+      # to return all the proxy entries (HTML nodes).
+      #
+      # @return [Array<ProxyFetcher::Document::Node>]
+      #   Collection of extracted HTML nodes with full proxy info
+      #
+      def load_proxy_list(filters = {})
+        doc = load_document(provider_url, filters)
+        doc.xpath(xpath)
+      end
+
       def build_proxy(*args)
         to_proxy(*args)
-      rescue StandardError => error
+      rescue StandardError => e
         ProxyFetcher.logger.warn(
-          "Failed to build Proxy object for #{self.class.name} due to error: #{error.message}"
+          "Failed to build Proxy for #{self.class.name.split("::").last} " \
+          "due to error: #{e.message}"
         )
 
         nil
       end
 
-      # Fetches HTML content by sending HTTP request to the provider URL and
-      # parses the document (built as abstract <code>ProxyFetcher::Document</code>)
-      # to return all the proxy entries (HTML nodes).
-      #
-      # Abstract method. Must be implemented in a descendant class
-      #
-      # @return [Array<Document::Node>]
-      #   list of proxy elements from the providers HTML content
-      #
-      def load_proxy_list(*)
-        raise NotImplementedError, "#{__method__} must be implemented in a descendant class!"
-      end
-
       # Convert HTML element with proxy info to ProxyFetcher::Proxy instance.
       #
       # Abstract method. Must be implemented in a descendant class

data/lib/proxy_fetcher/providers/free_proxy_list.rb

@@ -6,13 +6,12 @@ module ProxyFetcher
     class FreeProxyList < Base
       # Provider URL to fetch proxy list
       def provider_url
-        'https://free-proxy-list.net/'
+        "https://free-proxy-list.net/"
       end
 
       # [NOTE] Doesn't support filtering
-      def load_proxy_list(_filters = {})
-        doc = load_document(provider_url, {})
-        doc.xpath('//table[@id="proxylisttable"]/tbody/tr')
+      def xpath
+        '//table[@id="proxylisttable"]/tbody/tr'
       end
 
       # Converts HTML node (entry of N tags) to <code>ProxyFetcher::Proxy</code>
@@ -26,10 +25,10 @@ module ProxyFetcher
       #
       def to_proxy(html_node)
         ProxyFetcher::Proxy.new.tap do |proxy|
-          proxy.addr = html_node.content_at('td[1]')
-          proxy.port = Integer(html_node.content_at('td[2]').gsub(/^0+/, ''))
-          proxy.country = html_node.content_at('td[4]')
-          proxy.anonymity = html_node.content_at('td[5]')
+          proxy.addr = html_node.content_at("td[1]")
+          proxy.port = Integer(html_node.content_at("td[2]").gsub(/^0+/, ""))
+          proxy.country = html_node.content_at("td[4]")
+          proxy.anonymity = html_node.content_at("td[5]")
           proxy.type = parse_type(html_node)
         end
       end
@@ -45,8 +44,8 @@ module ProxyFetcher
       #   Proxy type
       #
       def parse_type(html_node)
-        https = html_node.content_at('td[6]')
-        https && https.casecmp('yes').zero? ? ProxyFetcher::Proxy::HTTPS : ProxyFetcher::Proxy::HTTP
+        https = html_node.content_at("td[6]")
+        https&.casecmp("yes")&.zero? ? ProxyFetcher::Proxy::HTTPS : ProxyFetcher::Proxy::HTTP
       end
     end
 

data/lib/proxy_fetcher/providers/free_proxy_list_socks.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module ProxyFetcher
+  module Providers
+    # FreeProxyListSocks provider class.
+    class FreeProxyListSocks < Base
+      # Provider URL to fetch proxy list
+      def provider_url
+        "https://www.socks-proxy.net/"
+      end
+
+      # [NOTE] Doesn't support filtering
+      def xpath
+        '//table[@id="proxylisttable"]/tbody/tr'
+      end
+
+      # Converts HTML node (entry of N tags) to <code>ProxyFetcher::Proxy</code>
+      # object.
+      #
+      # @param html_node [Object]
+      #   HTML node from the <code>ProxyFetcher::Document</code> DOM model.
+      #
+      # @return [ProxyFetcher::Proxy]
+      #   Proxy object
+      #
+      def to_proxy(html_node)
+        ProxyFetcher::Proxy.new.tap do |proxy|
+          proxy.addr = html_node.content_at("td[1]")
+          proxy.port = Integer(html_node.content_at("td[2]").gsub(/^0+/, ""))
+          proxy.country = html_node.content_at("td[4]")
+          proxy.type = parse_type(html_node)
+          proxy.anonymity = html_node.content_at("td[6]")
+        end
+      end
+
+      private
+
+      # Parses HTML node to extract proxy type.
+      #
+      # @param html_node [Object]
+      #   HTML node from the <code>ProxyFetcher::Document</code> DOM model.
+      #
+      # @return [String]
+      #   Proxy type
+      #
+      def parse_type(html_node)
+        https = html_node.content_at("td[5]")
+
+        return ProxyFetcher::Proxy::SOCKS4 if https&.casecmp("socks4")&.zero?
+        return ProxyFetcher::Proxy::SOCKS5 if https&.casecmp("socks5")&.zero?
+
+        "Unknown"
+      end
+    end
+
+    ProxyFetcher::Configuration.register_provider(:free_proxy_list_socks, FreeProxyListSocks)
+  end
+end

data/lib/proxy_fetcher/providers/free_proxy_list_ssl.rb

@@ -6,20 +6,12 @@ module ProxyFetcher
     class FreeProxyListSSL < Base
       # Provider URL to fetch proxy list
       def provider_url
-        'https://www.sslproxies.org/'
+        "https://www.sslproxies.org/"
       end
 
-      # Fetches HTML content by sending HTTP request to the provider URL and
-      # parses the document (built as abstract <code>ProxyFetcher::Document</code>)
-      # to return all the proxy entries (HTML nodes).
-      #
-      # @return [Array<ProxyFetcher::Document::Node>]
-      #   Collection of extracted HTML nodes with full proxy info
-      #
       # [NOTE] Doesn't support filtering
-      def load_proxy_list(_filters = {})
-        doc = load_document(provider_url, {})
-        doc.xpath('//table[@id="proxylisttable"]/tbody/tr')
+      def xpath
+        '//table[@id="proxylisttable"]/tbody/tr'
       end
 
       # Converts HTML node (entry of N tags) to <code>ProxyFetcher::Proxy</code>
@@ -33,10 +25,10 @@ module ProxyFetcher
       #
       def to_proxy(html_node)
         ProxyFetcher::Proxy.new.tap do |proxy|
-          proxy.addr = html_node.content_at('td[1]')
-          proxy.port = Integer(html_node.content_at('td[2]').gsub(/^0+/, ''))
-          proxy.country = html_node.content_at('td[4]')
-          proxy.anonymity = html_node.content_at('td[5]')
+          proxy.addr = html_node.content_at("td[1]")
+          proxy.port = Integer(html_node.content_at("td[2]").gsub(/^0+/, ""))
+          proxy.country = html_node.content_at("td[4]")
+          proxy.anonymity = html_node.content_at("td[5]")
           proxy.type = ProxyFetcher::Proxy::HTTPS
         end
       end

data/lib/proxy_fetcher/providers/free_proxy_list_us.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module ProxyFetcher
+  module Providers
+    # FreeProxyListUS provider class.
+    class FreeProxyListUS < Base
+      # Provider URL to fetch proxy list
+      def provider_url
+        "https://www.us-proxy.org/"
+      end
+
+      # [NOTE] Doesn't support filtering
+      def xpath
+        '//table[@id="proxylisttable"]/tbody/tr'
+      end
+
+      # Converts HTML node (entry of N tags) to <code>ProxyFetcher::Proxy</code>
+      # object.
+      #
+      # @param html_node [Object]
+      #   HTML node from the <code>ProxyFetcher::Document</code> DOM model.
+      #
+      # @return [ProxyFetcher::Proxy]
+      #   Proxy object
+      #
+      def to_proxy(html_node)
+        ProxyFetcher::Proxy.new.tap do |proxy|
+          proxy.addr = html_node.content_at("td[1]")
+          proxy.port = Integer(html_node.content_at("td[2]").gsub(/^0+/, ""))
+          proxy.country = html_node.content_at("td[4]")
+          proxy.anonymity = html_node.content_at("td[5]")
+          proxy.type = parse_type(html_node)
+        end
+      end
+
+      private
+
+      # Parses HTML node to extract proxy type.
+      #
+      # @param html_node [Object]
+      #   HTML node from the <code>ProxyFetcher::Document</code> DOM model.
+      #
+      # @return [String]
+      #   Proxy type
+      #
+      def parse_type(html_node)
+        https = html_node.content_at("td[7]")
+        https&.casecmp("yes")&.zero? ? ProxyFetcher::Proxy::HTTPS : ProxyFetcher::Proxy::HTTP
+      end
+    end
+
+    ProxyFetcher::Configuration.register_provider(:free_proxy_list_us, FreeProxyListUS)
+  end
+end