proxy_fetcher 0.9.0 → 0.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d5fee8c7b429b32d608388ddf15885d2354721d6de693396ad228248925cf2d6
-  data.tar.gz: '0250291040c511a723cdae4a22a78d4f765a0ca7b10325f331a8d119f8d25157'
+  metadata.gz: '01894f07ae28eafbb09934aa7a4e52188fb0bf02db7ab458df5c91a93b3e32af'
+  data.tar.gz: b487f3aeb6b833ab6b6395d0d4c9311294f7894e68f6a51c2e9943ccf2904d09
 SHA512:
-  metadata.gz: b7d12b75f0e80075d31832f8add9d1269ae48ff32714a0c0b481a85eb275094ccec62020911d55f74a2920211fc445f11ee3c31f43fdc7cc1f5ae2d175278eec
-  data.tar.gz: 138c464c0531fc12773b995d3101f5e952e2a4868e7e13d18274b27f7b0ea310a515d76e2066f36b25c2586c2046b9497f15a74e943cfbc8fdc8f0204796f93e
+  metadata.gz: aef23de20b41467dc2e1cadf65e6e728555f62e3cba0006df55ec62c8b877f8ca924dff9a81441897dc8e4f3efbf346ecefad62cf0b1e5a3bbcbd78272e1ea34
+  data.tar.gz: de41ae50f5bff9c8b8ae309057695661e76c47d2df703d369df6a2b3b40bffcfc881edc21b9833ca0f303392a480b7562b53b39d3f8c79e8e47cd0595a690366

data/gemfiles/nokogiri.gemfile

@@ -7,5 +7,5 @@ gem 'nokogiri', '~> 1.8'
 group :test do
   gem 'coveralls', require: false
   gem 'evil-proxy', '~> 0.2'
-  gem 'rspec-rails', '~> 3.6'
+  gem 'rspec', '~> 3.6'
 end

data/gemfiles/oga.gemfile

@@ -7,5 +7,5 @@ gem 'oga', '~> 2.0'
 group :test do
   gem 'coveralls', require: false
   gem 'evil-proxy', '~> 0.2'
-  gem 'rspec-rails', '~> 3.6'
+  gem 'rspec', '~> 3.6'
 end

data/lib/proxy_fetcher.rb

@@ -80,6 +80,7 @@ module ProxyFetcher
     #
     def logger
       return @logger if defined?(@logger)
+
       @logger = config.logger || NullLogger.new
     end
 

data/lib/proxy_fetcher/providers/base.rb

@@ -7,7 +7,25 @@ module ProxyFetcher
       # Loads proxy provider page content, extract proxy list from it
       # and convert every entry to proxy object.
       def fetch_proxies!(filters = {})
-        load_proxy_list(filters).map { |html_node| to_proxy(html_node) }
+        raw_proxies = load_proxy_list(filters)
+        proxies = raw_proxies.map { |html_node| build_proxy(html_node) }.compact
+        proxies.reject { |proxy| proxy.addr.nil? }
+      end
+
+      def provider_url
+        raise NotImplementedError, "#{__method__} must be implemented in a descendant class!"
+      end
+
+      def provider_method
+        :get
+      end
+
+      def provider_params
+        {}
+      end
+
+      def provider_headers
+        {}
       end
 
       # Just synthetic sugar to make it easier to call #fetch_proxies! method.
@@ -17,7 +35,27 @@ module ProxyFetcher
 
       protected
 
-      # Loads HTML document with Nokogiri by the URL combined with custom filters
+      # Loads raw provider HTML with proxies.
+      #
+      # @return [String]
+      #   HTML body
+      #
+      def load_html(url, filters = {})
+        raise ArgumentError, 'filters must be a Hash' if filters && !filters.is_a?(Hash)
+
+        uri = URI.parse(url)
+        # TODO: query for post request?
+        uri.query = URI.encode_www_form(provider_params.merge(filters)) if filters && filters.any?
+
+        ProxyFetcher.config.http_client.fetch(
+          uri.to_s,
+          method: provider_method,
+          headers: provider_headers,
+          params: provider_params
+        )
+      end
+
+      # Loads provider HTML and parses it with internal document object.
       #
       # @param url [String]
       #   URL to fetch
@@ -29,15 +67,17 @@ module ProxyFetcher
       #   ProxyFetcher document object
       #
       def load_document(url, filters = {})
-        raise ArgumentError, 'filters must be a Hash' if filters && !filters.is_a?(Hash)
-
-        uri = URI.parse(url)
-        uri.query = URI.encode_www_form(filters) if filters && filters.any?
-
-        html = ProxyFetcher.config.http_client.fetch(uri.to_s)
+        html = load_html(url, filters)
         ProxyFetcher::Document.parse(html)
       end
 
+      def build_proxy(*args)
+        to_proxy(*args)
+      rescue StandardError => error
+        ProxyFetcher.logger.warn("Failed to build Proxy object due to error: #{error.message}")
+        nil
+      end
+
       # Fetches HTML content by sending HTTP request to the provider URL and
       # parses the document (built as abstract <code>ProxyFetcher::Document</code>)
       # to return all the proxy entries (HTML nodes).

data/lib/proxy_fetcher/providers/free_proxy_list.rb

@@ -5,11 +5,13 @@ module ProxyFetcher
     # FreeProxyList provider class.
     class FreeProxyList < Base
       # Provider URL to fetch proxy list
-      PROVIDER_URL = 'https://free-proxy-list.net/'.freeze
+      def provider_url
+        'https://free-proxy-list.net/'
+      end
 
       # [NOTE] Doesn't support filtering
-      def load_proxy_list(*)
-        doc = load_document(PROVIDER_URL, {})
+      def load_proxy_list(_filters = {})
+        doc = load_document(provider_url, {})
         doc.xpath('//table[@id="proxylisttable"]/tbody/tr')
       end
 

data/lib/proxy_fetcher/providers/free_proxy_list_ssl.rb

@@ -5,7 +5,9 @@ module ProxyFetcher
     # FreeProxyListSSL provider class.
     class FreeProxyListSSL < Base
       # Provider URL to fetch proxy list
-      PROVIDER_URL = 'https://www.sslproxies.org/'.freeze
+      def provider_url
+        'https://www.sslproxies.org/'
+      end
 
       # Fetches HTML content by sending HTTP request to the provider URL and
       # parses the document (built as abstract <code>ProxyFetcher::Document</code>)
@@ -15,8 +17,8 @@ module ProxyFetcher
       #   Collection of extracted HTML nodes with full proxy info
       #
       # [NOTE] Doesn't support filtering
-      def load_proxy_list(*)
-        doc = load_document(PROVIDER_URL, {})
+      def load_proxy_list(_filters = {})
+        doc = load_document(provider_url, {})
         doc.xpath('//table[@id="proxylisttable"]/tbody/tr')
       end
 

data/lib/proxy_fetcher/providers/gather_proxy.rb

@@ -7,7 +7,9 @@ module ProxyFetcher
     # GatherProxy provider class.
     class GatherProxy < Base
       # Provider URL to fetch proxy list
-      PROVIDER_URL = 'http://www.gatherproxy.com/'.freeze
+      def provider_url
+        'http://www.gatherproxy.com/'
+      end
 
       # Fetches HTML content by sending HTTP request to the provider URL and
       # parses the document (built as abstract <code>ProxyFetcher::Document</code>)
@@ -17,7 +19,7 @@ module ProxyFetcher
       #   Collection of extracted HTML nodes with full proxy info
       #
       def load_proxy_list(*)
-        doc = load_document(PROVIDER_URL)
+        doc = load_document(provider_url)
         doc.xpath('//div[@class="proxy-list"]/table/script')
       end
 

data/lib/proxy_fetcher/providers/http_tunnel.rb

@@ -5,7 +5,9 @@ module ProxyFetcher
     # HTTPTunnel provider class.
     class HTTPTunnel < Base
       # Provider URL to fetch proxy list
-      PROVIDER_URL = 'http://www.httptunnel.ge/ProxyListForFree.aspx'.freeze
+      def provider_url
+        'http://www.httptunnel.ge/ProxyListForFree.aspx'
+      end
 
       # Fetches HTML content by sending HTTP request to the provider URL and
       # parses the document (built as abstract <code>ProxyFetcher::Document</code>)
@@ -14,8 +16,8 @@ module ProxyFetcher
       # @return [Array<ProxyFetcher::Document::Node>]
       #   Collection of extracted HTML nodes with full proxy info
       #
-      def load_proxy_list(*)
-        doc = load_document(PROVIDER_URL)
+      def load_proxy_list(_filters = {})
+        doc = load_document(provider_url)
         doc.xpath('//table[contains(@id, "GridView")]/tr[(count(td)>2)]')
       end
 

data/lib/proxy_fetcher/providers/proxy_docker.rb

@@ -1,11 +1,39 @@
 # frozen_string_literal: true
 
+require 'json'
+
 module ProxyFetcher
   module Providers
     # ProxyDocker provider class.
     class ProxyDocker < Base
       # Provider URL to fetch proxy list
-      PROVIDER_URL = 'https://www.proxydocker.com/en/proxylist/'.freeze
+      def provider_url
+        'https://www.proxydocker.com/en/api/proxylist/'
+      end
+
+      def provider_method
+        :post
+      end
+
+      def provider_params
+        {
+          token: 'GmZyl0OJmmgrWakdzO7AFf6AWfkdledR6xmKvGmwmJg',
+          country: 'all',
+          city: 'all',
+          state: 'all',
+          port: 'all',
+          type: 'all',
+          anonymity: 'all',
+          need: 'all',
+          page: '1'
+        }
+      end
+
+      def provider_headers
+        {
+          cookie: 'PHPSESSID=7f59558ee58b1e4352c4ab4c2f1a3c11'
+        }
+      end
 
       # Fetches HTML content by sending HTTP request to the provider URL and
       # parses the document (built as abstract <code>ProxyFetcher::Document</code>)
@@ -16,30 +44,42 @@ module ProxyFetcher
       #
       # [NOTE] Doesn't support direct filters
       def load_proxy_list(*)
-        doc = load_document(PROVIDER_URL, {})
-        doc.xpath('//table[contains(@class, "table")]/tbody/tr[(count(td)>2)]')
+        json = JSON.parse(load_html(provider_url, {}))
+        json.fetch('proxies', [])
+      rescue JSON::ParserError
+        []
       end
 
-      # Converts HTML node (entry of N tags) to <code>ProxyFetcher::Proxy</code>
+      # Converts JSON node  to <code>ProxyFetcher::Proxy</code>
       # object.
       #
-      # @param html_node [Object]
-      #   HTML node from the <code>ProxyFetcher::Document</code> DOM model.
+      # @param node [Hash]
+      #   JSON entry from the API response
       #
       # @return [ProxyFetcher::Proxy]
       #   Proxy object
       #
-      def to_proxy(html_node)
+      def to_proxy(node)
         ProxyFetcher::Proxy.new.tap do |proxy|
-          uri = URI("//#{html_node.content_at('td[1]')}")
-          proxy.addr = uri.host
-          proxy.port = uri.port
+          proxy.addr = node['ip']
+          proxy.port = node['port']
 
-          proxy.type = html_node.content_at('td[2]')
-          proxy.anonymity = html_node.content_at('td[3]')
-          proxy.country = html_node.content_at('td[5]')
+          proxy.type = types_mapping.fetch(node['type'], ProxyFetcher::Proxy::HTTP)
+          proxy.anonymity = "Lvl#{node['anonymity']}"
+          proxy.country = node['country']
         end
       end
+
+      def types_mapping
+        {
+          '16' => ProxyFetcher::Proxy::HTTP,
+          '26' => ProxyFetcher::Proxy::HTTPS,
+          '3' => ProxyFetcher::Proxy::SOCKS4,
+          '4' => ProxyFetcher::Proxy::SOCKS5,
+          '56' => ProxyFetcher::Proxy::HTTP, # CON25
+          '6' => ProxyFetcher::Proxy::HTTP # CON80
+        }
+      end
     end
 
     ProxyFetcher::Configuration.register_provider(:proxy_docker, ProxyDocker)

data/lib/proxy_fetcher/providers/proxy_list.rb

@@ -7,7 +7,9 @@ module ProxyFetcher
     # ProxyList provider class.
     class ProxyList < Base
       # Provider URL to fetch proxy list
-      PROVIDER_URL = 'https://proxy-list.org/english/index.php'.freeze
+      def provider_url
+        'https://proxy-list.org/english/index.php'
+      end
 
       # Fetches HTML content by sending HTTP request to the provider URL and
       # parses the document (built as abstract <code>ProxyFetcher::Document</code>)
@@ -17,7 +19,7 @@ module ProxyFetcher
       #   Collection of extracted HTML nodes with full proxy info
       #
       def load_proxy_list(filters = {})
-        doc = load_document(PROVIDER_URL, filters)
+        doc = load_document(provider_url, filters)
         doc.css('.table-wrap .table ul')
       end
 

data/lib/proxy_fetcher/providers/xroxy.rb

@@ -5,7 +5,9 @@ module ProxyFetcher
     # XRoxy provider class.
     class XRoxy < Base
       # Provider URL to fetch proxy list
-      PROVIDER_URL = 'https://www.xroxy.com/free-proxy-lists/'.freeze
+      def provider_url
+        'https://www.xroxy.com/free-proxy-lists/'
+      end
 
       # Fetches HTML content by sending HTTP request to the provider URL and
       # parses the document (built as abstract <code>ProxyFetcher::Document</code>)
@@ -15,7 +17,7 @@ module ProxyFetcher
       #   Collection of extracted HTML nodes with full proxy info
       #
       def load_proxy_list(filters = { type: 'All_http' })
-        doc = load_document(PROVIDER_URL, filters)
+        doc = load_document(provider_url, filters)
         doc.xpath('//div/table/tbody/tr')
       end
 

data/lib/proxy_fetcher/utils/http_client.rb

@@ -9,6 +9,18 @@ module ProxyFetcher
     #   @return [String] URL
     attr_reader :url
 
+    # @!attribute [r] HTTP method
+    #   @return [String] HTTP method verb
+    attr_reader :method
+
+    # @!attribute [r] HTTP params
+    #   @return [Hash] params
+    attr_reader :params
+
+    # @!attribute [r] HTTP headers
+    #   @return [Hash] headers
+    attr_reader :headers
+
     # @!attribute [r] http
     #   @return [Net::HTTP] HTTP client
     attr_reader :http
@@ -29,17 +41,21 @@ module ProxyFetcher
     # @return [String]
     #   resource content
     #
-    def self.fetch(url)
-      new(url).fetch
+    def self.fetch(*args)
+      new(*args).fetch
     end
 
     # Initialize HTTP client instance
     #
     # @return [HTTPClient]
     #
-    def initialize(url)
+    def initialize(url, method: :get, params: {}, headers: {})
       @url = url.to_s
-      @http = HTTP.headers(default_headers)
+      @method = method
+      @params = params
+      @headers = headers
+
+      @http = HTTP.headers(default_headers.merge(headers)).timeout(connect: timeout, read: timeout)
       @timeout = ProxyFetcher.config.provider_proxies_load_timeout
 
       @ssl_ctx = OpenSSL::SSL::SSLContext.new
@@ -52,11 +68,16 @@ module ProxyFetcher
     #   response body
     #
     def fetch
-      @http.timeout(connect: timeout, read: timeout)
-           .get(url, ssl_context: ssl_ctx)
-           .body.to_s
-    rescue StandardError
-      ProxyFetcher.logger.warn("Failed to load proxy list for #{url}")
+      # TODO: must be more generic
+      response = if method == :post
+                   http.post(url, form: params, ssl_context: ssl_ctx)
+                 else
+                   http.get(url, ssl_context: ssl_ctx)
+                 end
+
+      response.body.to_s
+    rescue StandardError => error
+      ProxyFetcher.logger.warn("Failed to load proxy list for #{url} (#{error.message})")
       ''
     end
 

data/lib/proxy_fetcher/version.rb

@@ -13,7 +13,7 @@ module ProxyFetcher
     # Major version number
     MAJOR = 0
     # Minor version number
-    MINOR = 9
+    MINOR = 10
     # Smallest version number
     TINY  = 0
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: proxy_fetcher
 version: !ruby/object:Gem::Version
-  version: 0.9.0
+  version: 0.10.0
 platform: ruby
 authors:
 - Nikita Bulai
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-01-22 00:00:00.000000000 Z
+date: 2019-03-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: http