proxy_fetcher 0.6.2 → 0.6.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6c2d4076c9f73303b305c364d3dcf672b6e8eba7
-  data.tar.gz: b692847fab646ec4bd230909ac1cba9eb77c1c7f
+  metadata.gz: f7ade86090b2c8dca7aa7d20a47fdf63334507e2
+  data.tar.gz: 0256d5fa29f0ec451f2248ab280dc71878c3e05f
 SHA512:
-  metadata.gz: cd70341cd18eea64dad58d63b8888f36e403251bb08e9d0289c584de1bc2201b45ff97f48fbac26697b25ee52328611b1e37de8d393b3b50ed7b3885192d0885
-  data.tar.gz: aed0b7532628796cb2b9f397be8bd8d87108c4d5a4e4efd2d30a468e2b8c9c203fbe7ac1fd4b56c7e571a67748b681297c56e2c880fddddb20c463bc117789e6
+  metadata.gz: c598b1a286e45d96e40ee9772365ece447d9dd18054fbdf80e727b086181b8b0cac9c5933898d74364ed95301da35855a368ea6c81cd41a4e26e355a92b1f1b7
+  data.tar.gz: 3cc0c7c348d63abb9fde40aadf3730d176fb7de59dbd51036082d605a9bf50614eb7f19a103f9ee448078d73dabf05e666911ee97cd9c7f30e0d1fe64a57ef21

data/CHANGELOG.md

@@ -2,10 +2,19 @@
 
 Reverse Chronological Order:
 
+## `0.6.3` (2018-01-26)
+
+* Add ability to use own proxy for `ProxyFetcher::Client`
+* Improve specs
+
+## `0.6.2` (2017-12-27)
+
+* Fix ProxyDocker provider.
+
 ## `0.6.1` (2017-12-11)
 
 * Fix gem executable to check dependencies for adapters
-* Code clean
+* Code cleanup
 * Some new specs
 
 ## `0.6.0` (2017-12-08)

data/LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2017 Nikita Bulai
+Copyright (c) 2017—2018 Nikita Bulai
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -245,6 +245,17 @@ require 'proxy-fetcher'
 ProxyFetcher::Client.get 'https://example.com/resource', options: { max_retries: 10_000 }
 ```
 
+You can also use your own proxy object when using ProxyFetcher client:
+
+```ruby
+require 'proxy-fetcher'
+
+manager = ProxyFetcher::Manager.new # will immediately load proxy list from the server
+
+#random will return random proxy object from the list
+ProxyFetcher::Client.get 'https://example.com/resource', options: { proxy: manager.random }
+```
+
 Btw, if you need support of JavaScript or some other features, you need to implement your own client using, for example,
 `selenium-webdriver`.
 
@@ -469,4 +480,4 @@ Thanks.
 
 `proxy_fetcher` gem is released under the [MIT License](http://www.opensource.org/licenses/MIT).
 
-Copyright (c) 2017 Nikita Bulai (bulajnikita@gmail.com).
+Copyright (c) 2017—2018 Nikita Bulai (bulajnikita@gmail.com).

data/lib/proxy_fetcher.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'uri'
 require 'net/https'
 
@@ -70,6 +72,9 @@ module ProxyFetcher
 
     private
 
+    # Configures default adapter if it isn't defined by the user.
+    # @api private
+    #
     def configure_adapter!
       config.adapter = Configuration::DEFAULT_ADAPTER if config.adapter.nil?
     end

data/lib/proxy_fetcher/client/client.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module ProxyFetcher
   # ProxyFetcher HTTP client that encapsulates all the logic for sending
   # HTTP(S) requests using proxies, automatically fetched and validated by the gem.
@@ -122,29 +124,48 @@ module ProxyFetcher
 
       private
 
+      # Executes HTTP request with user payload.
+      #
       def request_with_payload(method, url, payload, headers, options)
-        safe_request_to(url, options.fetch(:max_retries, 1000)) do |proxy|
-          opts = options.merge(url: url, payload: payload, proxy: proxy, headers: default_headers.merge(headers))
+        with_proxy_for(url, options.fetch(:max_retries, 1000)) do |proxy|
+          opts = options.merge(payload: payload, proxy: options.fetch(:proxy, proxy), headers: default_headers.merge(headers))
 
-          Request.execute(method: method, **opts)
+          Request.execute(url: url, method: method, **opts)
         end
       end
 
+      # Executes HTTP request without user payload.
+      #
       def request_without_payload(method, url, headers, options)
-        safe_request_to(url, options.fetch(:max_retries, 1000)) do |proxy|
-          opts = options.merge(url: url, proxy: proxy, headers: default_headers.merge(headers))
+        with_proxy_for(url, options.fetch(:max_retries, 1000)) do |proxy|
+          opts = options.merge(proxy: options.fetch(:proxy, proxy), headers: default_headers.merge(headers))
 
-          Request.execute(method: method, **opts)
+          Request.execute(url: url, method: method, **opts)
         end
       end
 
+      # Default ProxyFetcher::Client http headers. Uses some options
+      # from the configuration object, such as User-Agent string.
+      #
+      # @return [Hash]
+      #   headers
+      #
       def default_headers
         {
           'User-Agent' => ProxyFetcher.config.user_agent
         }
       end
 
-      def safe_request_to(url, max_retries = 1000)
+      # Searches for valid proxy (suitable for URL type) using <code>ProxyFetcher::Manager</code>
+      # instance and executes the block with found proxy with retries (N times, default is 1000) if
+      # something goes wrong.
+      #
+      # @param url [String] request URL
+      # @param max_retries [Integer] maximum number of retries
+      #
+      # @raise [ProxyFetcher::Error] internal error happened during block execution
+      #
+      def with_proxy_for(url, max_retries = 1000)
         tries = 0
 
         begin

data/lib/proxy_fetcher/client/proxies_registry.rb

@@ -1,12 +1,33 @@
+# frozen_string_literal: true
+
 module ProxyFetcher
   module Client
+    # ProxyFetcher proxies registry for managing proxy lists used by the Client.
+    # It is used to fetch proxy lists and instantiate Manager object that will
+    # handle proxies.
     class ProxiesRegistry
       class << self
+        # Removes proxy from the list of the current proxy manager
+        # instance. If no more proxy available, refreshes the list.
+        #
+        # @param proxy [ProxyFetcher::Proxy]
+        #   proxy object to remove
+        #
         def invalidate_proxy!(proxy)
           manager.proxies.delete(proxy)
           manager.refresh_list! if manager.proxies.empty?
         end
 
+        # Searches for valid proxy or required type (HTTP or secure)
+        # for requested URL. If no proxy found, than it refreshes proxy list
+        # and tries again.
+        #
+        # @param url [String]
+        #   URL to process with proxy
+        #
+        # @return [ProxyFetcher::Proxy]
+        #   gems proxy object
+        #
         def find_proxy_for(url)
           proxy = if URI.parse(url).is_a?(URI::HTTPS)
                     manager.proxies.detect(&:ssl?)
@@ -20,6 +41,12 @@ module ProxyFetcher
           find_proxy_for(url)
         end
 
+        # Instantiates or returns <code>ProxyFetcher::Manager</code> instance
+        # for current <code>Thread</code>.
+        #
+        # @return [ProxyFetcher::Manager]
+        #   ProxyFetcher manager class
+        #
         def manager
           manager = Thread.current[:proxy_fetcher_manager]
           return manager unless manager.nil?

data/lib/proxy_fetcher/client/request.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module ProxyFetcher
   module Client
     # ProxyFetcher::Client HTTP request abstraction.
@@ -14,13 +16,55 @@ module ProxyFetcher
         verify_mode: OpenSSL::SSL::VERIFY_NONE
       }.freeze
 
-      attr_reader :http, :method, :uri, :headers, :timeout,
-                  :payload, :proxy, :max_redirects, :ssl_options
+      # @!attribute [r] http
+      #   @return [Class] HTTP client
+      attr_reader :http
+
+      # @!attribute [r] method
+      #   @return [String, Symbol] HTTP request method
+      attr_reader :method
+
+      # @!attribute [r] uri
+      #   @return [URI] Request URI
+      attr_reader :uri
+
+      # @!attribute [r] headers
+      #   @return [Hash] HTTP headers
+      attr_reader :headers
+
+      # @!attribute [r] timeout
+      #   @return [Integer] Request timeout
+      attr_reader :timeout
+
+      # @!attribute [r] payload
+      #   @return [String, Hash] Request payload
+      attr_reader :payload
+
+      # @!attribute [r] proxy
+      #   @return [Proxy] Proxy to process the request
+      attr_reader :proxy
+
+      # @!attribute [r] max_redirects
+      #   @return [Integer] Maximum count of requests (if fails)
+      attr_reader :max_redirects
+
+      # @!attribute [r] ssl_options
+      #   @return [Hash] SSL options
+      attr_reader :ssl_options
 
+      # Initializes a new HTTP request and processes it
+      #
+      # @return [String]
+      #   response body (requested resource content)
+      #
       def self.execute(args)
         new(args).execute
       end
 
+      # Initialize new HTTP request
+      #
+      # @return [Request]
+      #
       def initialize(args)
         raise ArgumentError, 'args must be a Hash!' unless args.is_a?(Hash)
 
@@ -37,6 +81,11 @@ module ProxyFetcher
         build_http_client
       end
 
+      # Executes HTTP request with defined options.
+      #
+      # @return [String]
+      #   response body (requested resource content)
+      #
       def execute
         request = request_class_for(method).new(uri, headers)
 
@@ -47,6 +96,9 @@ module ProxyFetcher
 
       private
 
+      # Converts payload to the required format, so <code>Hash</code>
+      # must be a WWW-Form encoded for example.
+      #
       def preprocess_payload(payload)
         return if payload.nil?
 
@@ -58,6 +110,11 @@ module ProxyFetcher
         end
       end
 
+      # Builds HTTP client based on stdlib Net::HTTP.
+      #
+      # @return [Net::HTTP]
+      #   HTTP client
+      #
       def build_http_client
         @http = Net::HTTP.new(uri.host, uri.port, proxy.addr, proxy.port)
 
@@ -67,6 +124,15 @@ module ProxyFetcher
         @http.read_timeout = timeout
       end
 
+      # Processes HTTP response: checks it status and follows redirect if required.
+      # If response returned an error, then throws it.
+      #
+      # @param http_response [Net::HTTPResponse]
+      #   HTTP response object
+      #
+      # @return [String]
+      #   requested resource content
+      #
       def process_response!(http_response)
         case http_response
         when Net::HTTPSuccess then http_response.read_body
@@ -76,6 +142,8 @@ module ProxyFetcher
         end
       end
 
+      # Follows redirection for response.
+      #
       def follow_redirection(http_response)
         raise ProxyFetcher::Exceptions::MaximumRedirectsReached if max_redirects <= 0
 
@@ -85,6 +153,9 @@ module ProxyFetcher
         Request.execute(method: :get, url: url, proxy: proxy, headers: headers, timeout: timeout, max_redirects: max_redirects - 1)
       end
 
+      # Returns particular Net::HTTP method object
+      # for processing required request.
+      #
       def request_class_for(method)
         Net::HTTP.const_get(method, false)
       end

data/lib/proxy_fetcher/configuration.rb

@@ -1,10 +1,37 @@
+# frozen_string_literal: true
+
 module ProxyFetcher
   # ProxyFetcher configuration. Stores all the options for dealing
   # with HTTP requests, adapters, custom classes.
   #
   class Configuration
-    attr_accessor :timeout, :pool_size, :user_agent
-    attr_reader :adapter, :http_client, :proxy_validator, :providers
+    # @!attribute timeout
+    #   @return [Integer] HTTP request connection / open timeout
+    attr_accessor :timeout
+
+    # @!attribute pool_size
+    #   @return [Integer] proxy validator pool size (max number of threads)
+    attr_accessor :pool_size
+
+    # @!attribute user_agent
+    #   @return [String] User-Agent string
+    attr_accessor :user_agent
+
+    # @!attribute [r] adapter
+    #   @return [Object] HTML parser adapter
+    attr_reader :adapter
+
+    # @!attribute [r] http_client
+    #   @return [Class] HTTP client class
+    attr_reader :http_client
+
+    # @!attribute [r] proxy_validator
+    #   @return [Class] proxy validator class
+    attr_reader :proxy_validator
+
+    # @!attribute [r] providers
+    #   @return [Array<String>, Array<Symbol>] proxy providers list to be used
+    attr_reader :providers
 
     # User-Agent string that will be used by the ProxyFetcher HTTP client (to
     # send requests via proxy) and to fetch proxy lists from the sources.
@@ -21,6 +48,11 @@ module ProxyFetcher
     DEFAULT_ADAPTER = :nokogiri
 
     class << self
+      # Registry for handling proxy providers.
+      #
+      # @return [ProxyFetcher::ProvidersRegistry]
+      #   providers registry
+      #
       def providers_registry
         @registry ||= ProvidersRegistry.new
       end
@@ -38,11 +70,21 @@ module ProxyFetcher
         providers_registry.register(name, klass)
       end
 
+      # Returns registered providers names.
+      #
+      # @return [Array<String>, Array<Symbol>]
+      #   registered providers names
+      #
       def registered_providers
         providers_registry.providers.keys
       end
     end
 
+    # Initialize ProxyFetcher configuration with default options.
+    #
+    # @return [ProxyFetcher::Configuration]
+    #   ProxyFetcher gem configuration object
+    #
     def initialize
       reset!
     end
@@ -58,11 +100,21 @@ module ProxyFetcher
       self.providers = self.class.registered_providers
     end
 
+    # Setups HTML parser adapter for all the proxy providers.
+    #
+    # @param name_or_class [String, Symbol, Class]
+    #   name of the adapter or it's class
+    #
     def adapter=(name_or_class)
       @adapter = ProxyFetcher::Document::Adapters.lookup(name_or_class)
       @adapter.setup!
     end
 
+    # Setups collection of providers that will be used to fetch proxies.
+    #
+    # @param value [String, Symbol, Array<String>, Array<Symbol>]
+    #   provider names
+    #
     def providers=(value)
       @providers = Array(value)
     end
@@ -70,10 +122,22 @@ module ProxyFetcher
     alias provider providers
     alias provider= providers=
 
+    # Setups HTTP client class that will be used to fetch proxy lists.
+    # Validates class for the required methods to be defined.
+    #
+    # @param klass [Class]
+    #   HTTP client class
+    #
     def http_client=(klass)
       @http_client = setup_custom_class(klass, required_methods: :fetch)
     end
 
+    # Setups class that will be used to validate proxy lists.
+    # Validates class for the required methods to be defined.
+    #
+    # @param klass [Class]
+    #   Proxy validator class
+    #
     def proxy_validator=(klass)
       @proxy_validator = setup_custom_class(klass, required_methods: :connectable?)
     end

data/lib/proxy_fetcher/configuration/providers_registry.rb

@@ -1,5 +1,14 @@
+# frozen_string_literal: true
+
 module ProxyFetcher
+  # ProxyFetcher providers registry that stores all registered proxy providers.
   class ProvidersRegistry
+    # Returns providers hash where <i>key</i> is the name of the provider
+    # and <i>value</i> is an associated class.
+    #
+    # @return [Hash]
+    #   registered providers
+    #
     def providers
       @providers ||= {}
     end
@@ -7,14 +16,34 @@ module ProxyFetcher
     # Add custom provider to common registry.
     # Requires proxy provider name ('proxy_docker' for example) and a class
     # that implements the parsing logic.
+    #
+    # @param name [String, Symbol]
+    #   provider name
+    #
+    # @param klass [Class]
+    #   provider class
+    #
+    # @raise [ProxyFetcher::Exceptions::RegisteredProvider]
+    #   provider already registered
+    #
     def register(name, klass)
       raise ProxyFetcher::Exceptions::RegisteredProvider, name if providers.key?(name.to_sym)
 
       providers[name.to_sym] = klass
     end
 
-    # Returns a class for specific provider if it is
-    # registered in the registry. Otherwise throws an exception.
+    # Returns a class for specific provider if it is registered
+    # in the registry. Otherwise throws an exception.
+    #
+    # @param provider_name [String, Symbol]
+    #   provider name
+    #
+    # @return [Class]
+    #   provider class
+    #
+    # @raise [ProxyFetcher::Exceptions::UnknownProvider]
+    #   provider is unknown
+    #
     def class_for(provider_name)
       provider_name = provider_name.to_sym