proxy_fetcher 0.11.0 → 0.15.1

data/lib/proxy_fetcher/providers/free_proxy_list_ssl.rb

@@ -9,6 +9,7 @@ module ProxyFetcher
         "https://www.sslproxies.org/"
       end
 
+      # [NOTE] Doesn't support filtering
       def xpath
         '//table[@id="proxylisttable"]/tbody/tr'
       end

data/lib/proxy_fetcher/providers/free_proxy_list_us.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module ProxyFetcher
+  module Providers
+    # FreeProxyListUS provider class.
+    class FreeProxyListUS < Base
+      # Provider URL to fetch proxy list
+      def provider_url
+        "https://www.us-proxy.org/"
+      end
+
+      # [NOTE] Doesn't support filtering
+      def xpath
+        '//table[@id="proxylisttable"]/tbody/tr'
+      end
+
+      # Converts HTML node (entry of N tags) to <code>ProxyFetcher::Proxy</code>
+      # object.
+      #
+      # @param html_node [Object]
+      #   HTML node from the <code>ProxyFetcher::Document</code> DOM model.
+      #
+      # @return [ProxyFetcher::Proxy]
+      #   Proxy object
+      #
+      def to_proxy(html_node)
+        ProxyFetcher::Proxy.new.tap do |proxy|
+          proxy.addr = html_node.content_at("td[1]")
+          proxy.port = Integer(html_node.content_at("td[2]").gsub(/^0+/, ""))
+          proxy.country = html_node.content_at("td[4]")
+          proxy.anonymity = html_node.content_at("td[5]")
+          proxy.type = parse_type(html_node)
+        end
+      end
+
+      private
+
+      # Parses HTML node to extract proxy type.
+      #
+      # @param html_node [Object]
+      #   HTML node from the <code>ProxyFetcher::Document</code> DOM model.
+      #
+      # @return [String]
+      #   Proxy type
+      #
+      def parse_type(html_node)
+        https = html_node.content_at("td[7]")
+        https&.casecmp("yes")&.zero? ? ProxyFetcher::Proxy::HTTPS : ProxyFetcher::Proxy::HTTP
+      end
+    end
+
+    ProxyFetcher::Configuration.register_provider(:free_proxy_list_us, FreeProxyListUS)
+  end
+end

data/lib/proxy_fetcher/providers/mtpro.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require "json"
+
+module ProxyFetcher
+  module Providers
+    # MTPro provider class.
+    class MTPro < Base
+      # Provider URL to fetch proxy list
+      def provider_url
+        "https://mtpro.xyz/api/?type=socks"
+      end
+
+      def load_proxy_list(filters = {})
+        html = load_html(provider_url, filters)
+        JSON.parse(html)
+      rescue JSON::ParserError
+        []
+      end
+
+      # Converts HTML node (entry of N tags) to <code>ProxyFetcher::Proxy</code>
+      # object.
+      #
+      # @param node [Object]
+      #   HTML node from the <code>ProxyFetcher::Document</code> DOM model.
+      #
+      # @return [ProxyFetcher::Proxy]
+      #   Proxy object
+      #
+      def to_proxy(node)
+        ProxyFetcher::Proxy.new.tap do |proxy|
+          proxy.addr = node["ip"]
+          proxy.port = Integer(node["port"])
+          proxy.country = node["country"]
+          proxy.anonymity = "Unknown"
+          proxy.type = ProxyFetcher::Proxy::SOCKS5
+        end
+      end
+    end
+
+    ProxyFetcher::Configuration.register_provider(:mtpro, MTPro)
+  end
+end

data/lib/proxy_fetcher/providers/proxypedia.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module ProxyFetcher
+  module Providers
+    # FreeProxyList provider class.
+    class Proxypedia < Base
+      # Provider URL to fetch proxy list
+      def provider_url
+        "https://proxypedia.org"
+      end
+
+      # [NOTE] Doesn't support filtering
+      def xpath
+        "//main/ul/li[position()>1]"
+      end
+
+      # Converts HTML node (entry of N tags) to <code>ProxyFetcher::Proxy</code>
+      # object.]
+      #
+      # @param html_node [Object]
+      #   HTML node from the <code>ProxyFetcher::Document</code> DOM model.
+      #
+      # @return [ProxyFetcher::Proxy]
+      #   Proxy object
+      #
+      def to_proxy(html_node)
+        addr, port = html_node.content_at("a").to_s.split(":")
+
+        ProxyFetcher::Proxy.new.tap do |proxy|
+          proxy.addr = addr
+          proxy.port = Integer(port)
+          proxy.country = parse_country(html_node)
+          proxy.anonymity = "Unknown"
+          proxy.type = ProxyFetcher::Proxy::HTTP
+        end
+      end
+
+      private
+
+      def parse_country(html_node)
+        text = html_node.content.to_s
+        text[/\((.+?)\)/, 1] || "Unknown"
+      end
+    end
+
+    ProxyFetcher::Configuration.register_provider(:proxypedia, Proxypedia)
+  end
+end

data/lib/proxy_fetcher/providers/proxyscrape_http.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require "csv"
+
+module ProxyFetcher
+  module Providers
+    # FreeProxyList provider class.
+    class ProxyscrapeHTTP < Base
+      # Provider URL to fetch proxy list
+      def provider_url
+        "https://api.proxyscrape.com/v2/?request=getproxies&protocol=http"
+      end
+
+      # Loads provider HTML and parses it with internal document object.
+      #
+      # @param url [String]
+      #   URL to fetch
+      #
+      # @param filters [Hash]
+      #   filters for proxy provider
+      #
+      # @return [Array]
+      #   Collection of extracted proxies with ports
+      #
+      def load_document(url, filters = {})
+        html = load_html(url, filters)
+
+        CSV.parse(html, col_sep: "\t").map(&:first)
+      end
+
+      # Fetches HTML content by sending HTTP request to the provider URL and
+      # parses the txt document to return all the proxy entries (ip addresses
+      # and ports).
+      #
+      # @return [Array]
+      #   Collection of extracted proxies with ports
+      #
+      def load_proxy_list(filters = {})
+        load_document(provider_url, filters)
+      end
+
+      # Converts String to <code>ProxyFetcher::Proxy</code> object.
+      #
+      # @param node [String]
+      #   String
+      #
+      # @return [ProxyFetcher::Proxy]
+      #   Proxy object
+      #
+      def to_proxy(node)
+        addr, port = node.split(":")
+
+        ProxyFetcher::Proxy.new.tap do |proxy|
+          proxy.addr = addr
+          proxy.port = Integer(port)
+          proxy.country = "Unknown"
+          proxy.anonymity = "Unknown"
+          proxy.type = ProxyFetcher::Proxy::HTTP
+        end
+      end
+    end
+
+    ProxyFetcher::Configuration.register_provider(:proxyscrape_http, ProxyscrapeHTTP)
+  end
+end

data/lib/proxy_fetcher/providers/proxyscrape_socks4.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require "csv"
+
+module ProxyFetcher
+  module Providers
+    # FreeProxyList provider class.
+    class ProxyscrapeSOCKS4 < Base
+      # Provider URL to fetch proxy list
+      def provider_url
+        "https://api.proxyscrape.com/v2/?request=getproxies&protocol=socks4"
+      end
+
+      # Loads provider HTML and parses it with internal document object.
+      #
+      # @param url [String]
+      #   URL to fetch
+      #
+      # @param filters [Hash]
+      #   filters for proxy provider
+      #
+      # @return [Array]
+      #   Collection of extracted proxies with ports
+      #
+      def load_document(url, filters = {})
+        html = load_html(url, filters)
+
+        CSV.parse(html, col_sep: "\t").map(&:first)
+      end
+
+      # Fetches HTML content by sending HTTP request to the provider URL and
+      # parses the txt document to return all the proxy entries (ip addresses
+      # and ports).
+      #
+      # @return [Array]
+      #   Collection of extracted proxies with ports
+      #
+      def load_proxy_list(filters = {})
+        load_document(provider_url, filters)
+      end
+
+      # Converts String to <code>ProxyFetcher::Proxy</code> object.
+      #
+      # @param node [String]
+      #   String
+      #
+      # @return [ProxyFetcher::Proxy]
+      #   Proxy object
+      #
+      def to_proxy(html_node)
+        addr, port = html_node.split(":")
+
+        ProxyFetcher::Proxy.new.tap do |proxy|
+          proxy.addr = addr
+          proxy.port = Integer(port)
+          proxy.country = "Unknown"
+          proxy.anonymity = "Unknown"
+          proxy.type = ProxyFetcher::Proxy::SOCKS4
+        end
+      end
+    end
+
+    ProxyFetcher::Configuration.register_provider(:proxyscrape_socks4, ProxyscrapeSOCKS4)
+  end
+end

data/lib/proxy_fetcher/providers/proxyscrape_socks5.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require "csv"
+
+module ProxyFetcher
+  module Providers
+    # FreeProxyList provider class.
+    class ProxyscrapeSOCKS5 < Base
+      # Provider URL to fetch proxy list
+      def provider_url
+        "https://api.proxyscrape.com/v2/?request=getproxies&protocol=socks5"
+      end
+
+      # Loads provider HTML and parses it with internal document object.
+      #
+      # @param url [String]
+      #   URL to fetch
+      #
+      # @param filters [Hash]
+      #   filters for proxy provider
+      #
+      # @return [Array]
+      #   Collection of extracted proxies with ports
+      #
+      def load_document(url, filters = {})
+        html = load_html(url, filters)
+
+        CSV.parse(html, col_sep: "\t").map(&:first)
+      end
+
+      # Fetches HTML content by sending HTTP request to the provider URL and
+      # parses the txt document to return all the proxy entries (ip addresses
+      # and ports).
+      #
+      # @return [Array]
+      #   Collection of extracted proxies with ports
+      #
+      def load_proxy_list(filters = {})
+        load_document(provider_url, filters)
+      end
+
+      # Converts String to <code>ProxyFetcher::Proxy</code> object.
+      #
+      # @param node [String]
+      #   String
+      #
+      # @return [ProxyFetcher::Proxy]
+      #   Proxy object
+      #
+      def to_proxy(html_node)
+        addr, port = html_node.split(":")
+
+        ProxyFetcher::Proxy.new.tap do |proxy|
+          proxy.addr = addr
+          proxy.port = Integer(port)
+          proxy.country = "Unknown"
+          proxy.anonymity = "Unknown"
+          proxy.type = ProxyFetcher::Proxy::SOCKS5
+        end
+      end
+    end
+
+    ProxyFetcher::Configuration.register_provider(:proxyscrape_socks5, ProxyscrapeSOCKS5)
+  end
+end

data/lib/proxy_fetcher/providers/xroxy.rb

@@ -6,11 +6,11 @@ module ProxyFetcher
     class XRoxy < Base
       # Provider URL to fetch proxy list
       def provider_url
-        "https://www.xroxy.com/free-proxy-lists/"
+        "https://www.xroxy.com/proxylist.htm"
       end
 
       def xpath
-        "//div/table/tbody/tr"
+        "//tr[@class='row1' or @class='row0']"
       end
 
       # Converts HTML node (entry of N tags) to <code>ProxyFetcher::Proxy</code>

data/lib/proxy_fetcher/proxy.rb

@@ -104,5 +104,17 @@ module ProxyFetcher
         URI::Generic.build(host: addr, port: port).to_s
       end
     end
+
+    def ==(other)
+      other.is_a?(Proxy) && addr == other.addr && port == other.port
+    end
+
+    def eql?(other)
+      hash.eql?(other.hash)
+    end
+
+    def hash
+      [addr.hash, port.hash].hash
+    end
   end
 end

data/lib/proxy_fetcher/utils/http_client.rb

@@ -41,8 +41,8 @@ module ProxyFetcher
     # @return [String]
     #   resource content
     #
-    def self.fetch(*args)
-      new(*args).fetch
+    def self.fetch(*args, **kwargs, &block)
+      new(*args, **kwargs, &block).fetch
     end
 
     # Initialize HTTP client instance
@@ -51,15 +51,17 @@ module ProxyFetcher
     #
     def initialize(url, method: :get, params: {}, headers: {})
       @url = url.to_s
-      @method = method
+      @method = method.to_sym
       @params = params
       @headers = headers
 
-      @http = HTTP.headers(default_headers.merge(headers)).timeout(connect: timeout, read: timeout)
-      @timeout = ProxyFetcher.config.provider_proxies_load_timeout
+      unless HTTP::Request::METHODS.include?(@method)
+        raise ArgumentError, "'#{@method}' is a wrong HTTP method name"
+      end
 
-      @ssl_ctx = OpenSSL::SSL::SSLContext.new
-      @ssl_ctx.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      @timeout = ProxyFetcher.config.provider_proxies_load_timeout
+      @http = build_http_engine
+      @ssl_ctx = build_ssl_context
     end
 
     # Fetches resource content by sending HTTP request to it.
@@ -67,30 +69,32 @@ module ProxyFetcher
     # @return [String]
     #   response body
     #
-    def fetch
-      response = process_http_request
+    def fetch(**options)
+      response = perform_http_request
+      return response if options.fetch(:raw, false)
+
       response.body.to_s
     rescue StandardError => e
-      ProxyFetcher.logger.warn("Failed to process request to #{url} (#{e.message})")
+      ProxyFetcher.config.logger.warn("Failed to process request to #{url} (#{e.message})")
       ""
     end
 
-    def fetch_with_headers
-      process_http_request
-    rescue StandardError => e
-      ProxyFetcher.logger.warn("Failed to process request to #{url} (#{e.message})")
-      HTTP::Response.new(version: "1.1", status: 500, body: "")
-    end
-
     protected
 
-    def process_http_request(http_method: method, http_params: params)
-      unless HTTP::Request::METHODS.include?(http_method)
-        raise ArgumentError, "'#{http_method}' is a wrong HTTP method name!"
+    def build_ssl_context
+      OpenSSL::SSL::SSLContext.new.tap do |context|
+        context.verify_mode = OpenSSL::SSL::VERIFY_NONE
       end
+    end
+
+    def build_http_engine
+      HTTP.headers(default_headers.merge(headers)).timeout(connect: timeout, read: timeout)
+    end
 
+    def perform_http_request(http_method: method, http_params: params)
       http.public_send(
-        http_method.to_sym, url,
+        http_method,
+        url,
         form: http_params,
         ssl_context: ssl_ctx
       )