proxy_fetcher 0.10.0 → 0.10.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '01894f07ae28eafbb09934aa7a4e52188fb0bf02db7ab458df5c91a93b3e32af'
-  data.tar.gz: b487f3aeb6b833ab6b6395d0d4c9311294f7894e68f6a51c2e9943ccf2904d09
+  metadata.gz: 90fedddf9296d31071b49be0c44736626987ed973b48395ccfed9b04f0c6ae4a
+  data.tar.gz: 900464a6378e2d72259e54022df01ec1e50ac95f76f9ec613df803ffd0cf2d78
 SHA512:
-  metadata.gz: aef23de20b41467dc2e1cadf65e6e728555f62e3cba0006df55ec62c8b877f8ca924dff9a81441897dc8e4f3efbf346ecefad62cf0b1e5a3bbcbd78272e1ea34
-  data.tar.gz: de41ae50f5bff9c8b8ae309057695661e76c47d2df703d369df6a2b3b40bffcfc881edc21b9833ca0f303392a480b7562b53b39d3f8c79e8e47cd0595a690366
+  metadata.gz: bc14dedb91c2a319aa27e3c03b3a881ac0861c378c5b1d8396ca79755f8a80ae86d28e904b999cddafc3f85ad7805ee6386e1fca327ded8a764d60ce572af4a3
+  data.tar.gz: 4d06f65bd36132ae0fbac2c77b12d34b31e5ec851e68995de86457e0d45f29777bcd06ff8c08c6b3e6830a18278f4984e4ca5c3573b7e564c84da155d60a8946

data/CHANGELOG.md

@@ -6,6 +6,11 @@ Reverse Chronological Order:
 
 * Add your description here
 
+## `0.10.1` (2019-03-07)
+
+* Fix broken ProxyDocker provider.
+* Refactor gem internals.
+
 ## `0.9.0` (2019-01-22)
 
 * Fix a problem with stuck of proxies list loading.

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+require 'proxy_fetcher'
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require 'irb'
+IRB.start(__FILE__)

data/lib/proxy_fetcher/configuration.rb

@@ -35,7 +35,11 @@ module ProxyFetcher
 
     # @!attribute [r] adapter
     #   @return [Object] HTML parser adapter
-    attr_reader :adapter
+    attr_accessor :adapter
+
+    # @!attribute [r] adapter_class
+    #   @return [Object] HTML adapter class
+    attr_reader :adapter_class
 
     # @!attribute [r] http_client
     #   @return [Object] HTTP client class
@@ -120,14 +124,11 @@ module ProxyFetcher
       self.providers = self.class.registered_providers
     end
 
-    # Setups HTML parser adapter for all the proxy providers.
-    #
-    # @param name_or_class [String, Symbol, Class]
-    #   name of the adapter or it's class
-    #
-    def adapter=(name_or_class)
-      @adapter = ProxyFetcher::Document::Adapters.lookup(name_or_class)
-      @adapter.setup!
+    def adapter_class
+      return @adapter_class if defined?(@adapter_class)
+
+      @adapter_class = ProxyFetcher::Document::Adapters.lookup(adapter)
+      @adapter_class.setup!
     end
 
     # Setups collection of providers that will be used to fetch proxies.

data/lib/proxy_fetcher/document.rb

@@ -17,7 +17,7 @@ module ProxyFetcher
     #   ProxyFetcher document model
     #
     def self.parse(data)
-      new(ProxyFetcher.config.adapter.parse(data))
+      new(ProxyFetcher.config.adapter_class.parse(data))
     end
 
     # Initialize abstract ProxyFetcher HTML Document

data/lib/proxy_fetcher/document/adapters.rb

@@ -30,7 +30,7 @@ module ProxyFetcher
 
           case name_or_class
           when Symbol, String
-            adapter_name = name_or_class.to_s.capitalize << ADAPTER
+            adapter_name = "#{name_or_class.to_s.capitalize}#{ADAPTER}"
             ProxyFetcher::Document.const_get(adapter_name)
           else
             name_or_class

data/lib/proxy_fetcher/document/adapters/abstract_adapter.rb

@@ -52,6 +52,7 @@ module ProxyFetcher
       #
       def self.setup!(*args)
         install_requirements!(*args)
+        self
       rescue LoadError => error
         raise Exceptions::AdapterSetupError.new(name, error.message)
       end

data/lib/proxy_fetcher/providers/base.rb

@@ -74,7 +74,10 @@ module ProxyFetcher
       def build_proxy(*args)
         to_proxy(*args)
       rescue StandardError => error
-        ProxyFetcher.logger.warn("Failed to build Proxy object due to error: #{error.message}")
+        ProxyFetcher.logger.warn(
+          "Failed to build Proxy object for #{self.class.name} due to error: #{error.message}"
+        )
+
         nil
       end
 

data/lib/proxy_fetcher/providers/proxy_docker.rb

@@ -6,6 +6,8 @@ module ProxyFetcher
   module Providers
     # ProxyDocker provider class.
     class ProxyDocker < Base
+      attr_reader :cookie, :token
+
       # Provider URL to fetch proxy list
       def provider_url
         'https://www.proxydocker.com/en/api/proxylist/'
@@ -17,7 +19,7 @@ module ProxyFetcher
 
       def provider_params
         {
-          token: 'GmZyl0OJmmgrWakdzO7AFf6AWfkdledR6xmKvGmwmJg',
+          token: @token,
           country: 'all',
           city: 'all',
           state: 'all',
@@ -31,7 +33,7 @@ module ProxyFetcher
 
       def provider_headers
         {
-          cookie: 'PHPSESSID=7f59558ee58b1e4352c4ab4c2f1a3c11'
+          cookie: @cookie
         }
       end
 
@@ -44,6 +46,8 @@ module ProxyFetcher
       #
       # [NOTE] Doesn't support direct filters
       def load_proxy_list(*)
+        load_dependencies
+
         json = JSON.parse(load_html(provider_url, {}))
         json.fetch('proxies', [])
       rescue JSON::ParserError
@@ -70,6 +74,8 @@ module ProxyFetcher
         end
       end
 
+      private
+
       def types_mapping
         {
           '16' => ProxyFetcher::Proxy::HTTP,
@@ -80,6 +86,24 @@ module ProxyFetcher
           '6' => ProxyFetcher::Proxy::HTTP # CON80
         }
       end
+
+      def load_dependencies
+        client = ProxyFetcher.config.http_client.new('https://www.proxydocker.com')
+        response = client.fetch_with_headers
+
+        @cookie = load_cookie_from(response)
+        @token = load_token_from(response)
+      end
+
+      def load_cookie_from(response)
+        cookie_headers = (response.headers['Set-Cookie'] || [])
+        cookie_headers.join('; ')
+      end
+
+      def load_token_from(response)
+        html = response.body.to_s
+        html[/meta\s+name\s*=["']_token["']\s+content.+["'](.+?)["']\s*>/i, 1]
+      end
     end
 
     ProxyFetcher::Configuration.register_provider(:proxy_docker, ProxyDocker)

data/lib/proxy_fetcher/utils/http_client.rb

@@ -68,21 +68,32 @@ module ProxyFetcher
     #   response body
     #
     def fetch
-      # TODO: must be more generic
-      response = if method == :post
-                   http.post(url, form: params, ssl_context: ssl_ctx)
-                 else
-                   http.get(url, ssl_context: ssl_ctx)
-                 end
-
+      response = process_http_request
       response.body.to_s
     rescue StandardError => error
-      ProxyFetcher.logger.warn("Failed to load proxy list for #{url} (#{error.message})")
+      ProxyFetcher.logger.warn("Failed to process request to #{url} (#{error.message})")
       ''
     end
 
+    def fetch_with_headers
+      process_http_request
+    rescue StandardError => error
+      ProxyFetcher.logger.warn("Failed to process request to #{url} (#{error.message})")
+       HTTP::Response.new(version: '1.1', status: 500, body: '')
+    end
+
     protected
 
+    def process_http_request(http_method: method, http_params: params)
+      raise ArgumentError, 'wrong http method name!' unless HTTP::Request::METHODS.include?(http_method)
+
+      http.public_send(
+        http_method.to_sym, url,
+        form: http_params,
+        ssl_context: ssl_ctx
+      )
+    end
+
     # Default HTTP client headers
     #
     # @return [Hash]

data/lib/proxy_fetcher/version.rb

@@ -15,7 +15,7 @@ module ProxyFetcher
     # Minor version number
     MINOR = 10
     # Smallest version number
-    TINY  = 0
+    TINY  = 1
 
     # Full version number
     STRING = [MAJOR, MINOR, TINY].compact.join('.')

data/spec/proxy_fetcher/configuration_spec.rb

@@ -68,7 +68,9 @@ describe ProxyFetcher::Configuration do
         end
       end
 
-      expect { ProxyFetcher.config.adapter = CustomAdapter }
+      ProxyFetcher.config.adapter = CustomAdapter
+
+      expect { ProxyFetcher::Manager.new }
         .to raise_error(ProxyFetcher::Exceptions::AdapterSetupError)
 
       ProxyFetcher.instance_variable_set('@config', old_config)

data/spec/proxy_fetcher/providers/base_spec.rb

@@ -44,7 +44,7 @@ describe ProxyFetcher::Providers::Base do
 
     allow_any_instance_of(HTTP::Client).to receive(:get).and_raise(StandardError)
 
-    expect(logger).to receive(:warn).with(/Failed to load proxy list for http[s:\/]/)
+    expect(logger).to receive(:warn).with(/Failed to process request to http[s:\/]/)
 
     ProxyFetcher::Manager.new
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: proxy_fetcher
 version: !ruby/object:Gem::Version
-  version: 0.10.0
+  version: 0.10.1
 platform: ruby
 authors:
 - Nikita Bulai
@@ -56,6 +56,7 @@ description: This gem can help your Ruby application to make HTTP(S) requests us
   proxies by fetching and validating proxy lists from the different providers.
 email: bulajnikita@gmail.com
 executables:
+- console
 - proxy_fetcher
 extensions: []
 extra_rdoc_files: []
@@ -66,6 +67,7 @@ files:
 - Gemfile
 - LICENSE
 - Rakefile
+- bin/console
 - bin/proxy_fetcher
 - gemfiles/nokogiri.gemfile
 - gemfiles/oga.gemfile