proxy_fetcher 0.10.2 → 0.15.0

data/lib/proxy_fetcher/providers/http_tunnel.rb

@@ -6,19 +6,11 @@ module ProxyFetcher
     class HTTPTunnel < Base
       # Provider URL to fetch proxy list
       def provider_url
-        'http://www.httptunnel.ge/ProxyListForFree.aspx'
+        "http://www.httptunnel.ge/ProxyListForFree.aspx"
       end
 
-      # Fetches HTML content by sending HTTP request to the provider URL and
-      # parses the document (built as abstract <code>ProxyFetcher::Document</code>)
-      # to return all the proxy entries (HTML nodes).
-      #
-      # @return [Array<ProxyFetcher::Document::Node>]
-      #   Collection of extracted HTML nodes with full proxy info
-      #
-      def load_proxy_list(_filters = {})
-        doc = load_document(provider_url)
-        doc.xpath('//table[contains(@id, "GridView")]/tr[(count(td)>2)]')
+      def xpath
+        '//table[contains(@id, "GridView")]/tr[(count(td)>2)]'
       end
 
       # Converts HTML node (entry of N tags) to <code>ProxyFetcher::Proxy</code>
@@ -53,7 +45,7 @@ module ProxyFetcher
       #   URI object
       #
       def parse_proxy_uri(html_node)
-        full_addr = html_node.content_at('td[1]')
+        full_addr = html_node.content_at("td[1]")
         URI.parse("http://#{full_addr}")
       end
 
@@ -66,7 +58,7 @@ module ProxyFetcher
       #   Country code
       #
       def parse_country(html_node)
-        html_node.find('.//img').attr('title')
+        html_node.find(".//img").attr("title")
       end
 
       # Parses HTML node to extract proxy anonymity level.
@@ -78,14 +70,14 @@ module ProxyFetcher
       #   Anonymity level
       #
       def parse_anonymity(html_node)
-        transparency = html_node.content_at('td[5]').to_sym
+        transparency = html_node.content_at("td[5]").to_sym
 
         {
-          A: 'Anonymous',
-          E: 'Elite',
-          T: 'Transparent',
-          U: 'Unknown'
-        }.fetch(transparency, 'Unknown')
+          A: "Anonymous",
+          E: "Elite",
+          T: "Transparent",
+          U: "Unknown"
+        }.fetch(transparency, "Unknown")
       end
     end
 

data/lib/proxy_fetcher/providers/mtpro.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require "json"
+
+module ProxyFetcher
+  module Providers
+    # MTPro provider class.
+    class MTPro < Base
+      # Provider URL to fetch proxy list
+      def provider_url
+        "https://mtpro.xyz/api/?type=socks"
+      end
+
+      def load_proxy_list(filters = {})
+        html = load_html(provider_url, filters)
+        JSON.parse(html)
+      rescue JSON::ParserError
+        []
+      end
+
+      # Converts HTML node (entry of N tags) to <code>ProxyFetcher::Proxy</code>
+      # object.
+      #
+      # @param node [Object]
+      #   HTML node from the <code>ProxyFetcher::Document</code> DOM model.
+      #
+      # @return [ProxyFetcher::Proxy]
+      #   Proxy object
+      #
+      def to_proxy(node)
+        ProxyFetcher::Proxy.new.tap do |proxy|
+          proxy.addr = node["ip"]
+          proxy.port = Integer(node["port"])
+          proxy.country = node["country"]
+          proxy.anonymity = "Unknown"
+          proxy.type = ProxyFetcher::Proxy::SOCKS5
+        end
+      end
+    end
+
+    ProxyFetcher::Configuration.register_provider(:mtpro, MTPro)
+  end
+end

data/lib/proxy_fetcher/providers/proxy_list.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require 'base64'
+require "base64"
 
 module ProxyFetcher
   module Providers
@@ -8,19 +8,11 @@ module ProxyFetcher
     class ProxyList < Base
       # Provider URL to fetch proxy list
       def provider_url
-        'https://proxy-list.org/english/index.php'
+        "https://proxy-list.org/english/index.php"
       end
 
-      # Fetches HTML content by sending HTTP request to the provider URL and
-      # parses the document (built as abstract <code>ProxyFetcher::Document</code>)
-      # to return all the proxy entries (HTML nodes).
-      #
-      # @return [Array<ProxyFetcher::Document::Node>]
-      #   Collection of extracted HTML nodes with full proxy info
-      #
-      def load_proxy_list(filters = {})
-        doc = load_document(provider_url, filters)
-        doc.css('.table-wrap .table ul')
+      def xpath
+        '//div[@class="table-wrap"]/div[@class="table"]/ul'
       end
 
       # Converts HTML node (entry of N tags) to <code>ProxyFetcher::Proxy</code>
@@ -38,9 +30,9 @@ module ProxyFetcher
           proxy.addr = uri.host
           proxy.port = uri.port
 
-          proxy.type = html_node.content_at('li[2]')
-          proxy.anonymity = html_node.content_at('li[4]')
-          proxy.country = html_node.find("li[5]//span[@class='country']").attr('title')
+          proxy.type = html_node.content_at("li[2]")
+          proxy.anonymity = html_node.content_at("li[4]")
+          proxy.country = html_node.find("li[5]//span[@class='country']").attr("title")
         end
       end
 
@@ -55,7 +47,7 @@ module ProxyFetcher
       #   URI object
       #
       def parse_proxy_uri(html_node)
-        full_addr = ::Base64.decode64(html_node.at_css('li script').html.match(/'(.+)'/)[1])
+        full_addr = ::Base64.decode64(html_node.at_css("li script").html.match(/'(.+)'/)[1])
         URI.parse("http://#{full_addr}")
       end
     end

data/lib/proxy_fetcher/providers/proxypedia.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module ProxyFetcher
+  module Providers
+    # FreeProxyList provider class.
+    class Proxypedia < Base
+      # Provider URL to fetch proxy list
+      def provider_url
+        "https://proxypedia.org"
+      end
+
+      # [NOTE] Doesn't support filtering
+      def xpath
+        "//main/ul/li[position()>1]"
+      end
+
+      # Converts HTML node (entry of N tags) to <code>ProxyFetcher::Proxy</code>
+      # object.]
+      #
+      # @param html_node [Object]
+      #   HTML node from the <code>ProxyFetcher::Document</code> DOM model.
+      #
+      # @return [ProxyFetcher::Proxy]
+      #   Proxy object
+      #
+      def to_proxy(html_node)
+        addr, port = html_node.content_at("a").to_s.split(":")
+
+        ProxyFetcher::Proxy.new.tap do |proxy|
+          proxy.addr = addr
+          proxy.port = Integer(port)
+          proxy.country = parse_country(html_node)
+          proxy.anonymity = "Unknown"
+          proxy.type = ProxyFetcher::Proxy::HTTP
+        end
+      end
+
+      private
+
+      def parse_country(html_node)
+        text = html_node.content.to_s
+        text[/\((.+?)\)/, 1] || "Unknown"
+      end
+    end
+
+    ProxyFetcher::Configuration.register_provider(:proxypedia, Proxypedia)
+  end
+end

data/lib/proxy_fetcher/providers/proxyscrape_http.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require "csv"
+
+module ProxyFetcher
+  module Providers
+    # FreeProxyList provider class.
+    class ProxyscrapeHTTP < Base
+      # Provider URL to fetch proxy list
+      def provider_url
+        "https://api.proxyscrape.com/v2/?request=getproxies&protocol=http"
+      end
+
+      # Loads provider HTML and parses it with internal document object.
+      #
+      # @param url [String]
+      #   URL to fetch
+      #
+      # @param filters [Hash]
+      #   filters for proxy provider
+      #
+      # @return [Array]
+      #   Collection of extracted proxies with ports
+      #
+      def load_document(url, filters = {})
+        html = load_html(url, filters)
+
+        CSV.parse(html, col_sep: "\t").map(&:first)
+      end
+
+      # Fetches HTML content by sending HTTP request to the provider URL and
+      # parses the txt document to return all the proxy entries (ip addresses
+      # and ports).
+      #
+      # @return [Array]
+      #   Collection of extracted proxies with ports
+      #
+      def load_proxy_list(filters = {})
+        load_document(provider_url, filters)
+      end
+
+      # Converts String to <code>ProxyFetcher::Proxy</code> object.
+      #
+      # @param node [String]
+      #   String
+      #
+      # @return [ProxyFetcher::Proxy]
+      #   Proxy object
+      #
+      def to_proxy(node)
+        addr, port = node.split(":")
+
+        ProxyFetcher::Proxy.new.tap do |proxy|
+          proxy.addr = addr
+          proxy.port = Integer(port)
+          proxy.country = "Unknown"
+          proxy.anonymity = "Unknown"
+          proxy.type = ProxyFetcher::Proxy::HTTP
+        end
+      end
+    end
+
+    ProxyFetcher::Configuration.register_provider(:proxyscrape_http, ProxyscrapeHTTP)
+  end
+end

data/lib/proxy_fetcher/providers/proxyscrape_socks4.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require "csv"
+
+module ProxyFetcher
+  module Providers
+    # FreeProxyList provider class.
+    class ProxyscrapeSOCKS4 < Base
+      # Provider URL to fetch proxy list
+      def provider_url
+        "https://api.proxyscrape.com/v2/?request=getproxies&protocol=socks4"
+      end
+
+      # Loads provider HTML and parses it with internal document object.
+      #
+      # @param url [String]
+      #   URL to fetch
+      #
+      # @param filters [Hash]
+      #   filters for proxy provider
+      #
+      # @return [Array]
+      #   Collection of extracted proxies with ports
+      #
+      def load_document(url, filters = {})
+        html = load_html(url, filters)
+
+        CSV.parse(html, col_sep: "\t").map(&:first)
+      end
+
+      # Fetches HTML content by sending HTTP request to the provider URL and
+      # parses the txt document to return all the proxy entries (ip addresses
+      # and ports).
+      #
+      # @return [Array]
+      #   Collection of extracted proxies with ports
+      #
+      def load_proxy_list(filters = {})
+        load_document(provider_url, filters)
+      end
+
+      # Converts String to <code>ProxyFetcher::Proxy</code> object.
+      #
+      # @param node [String]
+      #   String
+      #
+      # @return [ProxyFetcher::Proxy]
+      #   Proxy object
+      #
+      def to_proxy(html_node)
+        addr, port = html_node.split(":")
+
+        ProxyFetcher::Proxy.new.tap do |proxy|
+          proxy.addr = addr
+          proxy.port = Integer(port)
+          proxy.country = "Unknown"
+          proxy.anonymity = "Unknown"
+          proxy.type = ProxyFetcher::Proxy::SOCKS4
+        end
+      end
+    end
+
+    ProxyFetcher::Configuration.register_provider(:proxyscrape_socks4, ProxyscrapeSOCKS4)
+  end
+end

data/lib/proxy_fetcher/providers/proxyscrape_socks5.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require "csv"
+
+module ProxyFetcher
+  module Providers
+    # FreeProxyList provider class.
+    class ProxyscrapeSOCKS5 < Base
+      # Provider URL to fetch proxy list
+      def provider_url
+        "https://api.proxyscrape.com/v2/?request=getproxies&protocol=socks5"
+      end
+
+      # Loads provider HTML and parses it with internal document object.
+      #
+      # @param url [String]
+      #   URL to fetch
+      #
+      # @param filters [Hash]
+      #   filters for proxy provider
+      #
+      # @return [Array]
+      #   Collection of extracted proxies with ports
+      #
+      def load_document(url, filters = {})
+        html = load_html(url, filters)
+
+        CSV.parse(html, col_sep: "\t").map(&:first)
+      end
+
+      # Fetches HTML content by sending HTTP request to the provider URL and
+      # parses the txt document to return all the proxy entries (ip addresses
+      # and ports).
+      #
+      # @return [Array]
+      #   Collection of extracted proxies with ports
+      #
+      def load_proxy_list(filters = {})
+        load_document(provider_url, filters)
+      end
+
+      # Converts String to <code>ProxyFetcher::Proxy</code> object.
+      #
+      # @param node [String]
+      #   String
+      #
+      # @return [ProxyFetcher::Proxy]
+      #   Proxy object
+      #
+      def to_proxy(html_node)
+        addr, port = html_node.split(":")
+
+        ProxyFetcher::Proxy.new.tap do |proxy|
+          proxy.addr = addr
+          proxy.port = Integer(port)
+          proxy.country = "Unknown"
+          proxy.anonymity = "Unknown"
+          proxy.type = ProxyFetcher::Proxy::SOCKS5
+        end
+      end
+    end
+
+    ProxyFetcher::Configuration.register_provider(:proxyscrape_socks5, ProxyscrapeSOCKS5)
+  end
+end

data/lib/proxy_fetcher/providers/xroxy.rb

@@ -6,19 +6,11 @@ module ProxyFetcher
     class XRoxy < Base
       # Provider URL to fetch proxy list
       def provider_url
-        'https://www.xroxy.com/free-proxy-lists/'
+        "https://www.xroxy.com/proxylist.htm"
       end
 
-      # Fetches HTML content by sending HTTP request to the provider URL and
-      # parses the document (built as abstract <code>ProxyFetcher::Document</code>)
-      # to return all the proxy entries (HTML nodes).
-      #
-      # @return [Array<ProxyFetcher::Document::Node>]
-      #   Collection of extracted HTML nodes with full proxy info
-      #
-      def load_proxy_list(filters = { type: 'All_http' })
-        doc = load_document(provider_url, filters)
-        doc.xpath('//div/table/tbody/tr')
+      def xpath
+        "//tr[@class='row1' or @class='row0']"
       end
 
       # Converts HTML node (entry of N tags) to <code>ProxyFetcher::Proxy</code>
@@ -32,12 +24,12 @@ module ProxyFetcher
       #
       def to_proxy(html_node)
         ProxyFetcher::Proxy.new.tap do |proxy|
-          proxy.addr = html_node.content_at('td[1]')
-          proxy.port = Integer(html_node.content_at('td[2]').gsub(/^0+/, ''))
-          proxy.anonymity = html_node.content_at('td[3]')
-          proxy.country = html_node.content_at('td[5]')
-          proxy.response_time = Integer(html_node.content_at('td[6]'))
-          proxy.type = html_node.content_at('td[3]')
+          proxy.addr = html_node.content_at("td[1]")
+          proxy.port = Integer(html_node.content_at("td[2]").gsub(/^0+/, ""))
+          proxy.anonymity = html_node.content_at("td[3]")
+          proxy.country = html_node.content_at("td[5]")
+          proxy.response_time = Integer(html_node.content_at("td[6]"))
+          proxy.type = html_node.content_at("td[3]")
         end
       end
     end

data/lib/proxy_fetcher/proxy.rb

@@ -29,10 +29,10 @@ module ProxyFetcher
 
     # Proxy types
     TYPES = [
-      HTTP = 'HTTP'.freeze,
-      HTTPS = 'HTTPS'.freeze,
-      SOCKS4 = 'SOCKS4'.freeze,
-      SOCKS5 = 'SOCKS5'.freeze
+      HTTP = "HTTP",
+      HTTPS = "HTTPS",
+      SOCKS4 = "SOCKS4",
+      SOCKS5 = "SOCKS5"
     ].freeze
 
     # Proxy type predicates (#socks4?, #https?)
@@ -104,5 +104,17 @@ module ProxyFetcher
         URI::Generic.build(host: addr, port: port).to_s
       end
     end
+
+    def ==(other)
+      other.is_a?(Proxy) && addr == other.addr && port == other.port
+    end
+
+    def eql?(other)
+      hash.eql?(other.hash)
+    end
+
+    def hash
+      [addr.hash, port.hash].hash
+    end
   end
 end