proxy_fetcher 0.8.0 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: ad122ebb9d6241313981ef764c41a1b940dfdc12
-  data.tar.gz: 2cf2d359bfd74122fbfdf9d97d94c1e70b008cd5
+SHA256:
+  metadata.gz: d5fee8c7b429b32d608388ddf15885d2354721d6de693396ad228248925cf2d6
+  data.tar.gz: '0250291040c511a723cdae4a22a78d4f765a0ca7b10325f331a8d119f8d25157'
 SHA512:
-  metadata.gz: 4ebe17a6ebfb52a14267856053ad750676d4b28e85e138aa172d778b85368ec85ccd04af55cdceb53e1aaad3db5cdab09d42ee87767d56b1fc5578c1c6ba114b
-  data.tar.gz: 6133dfc0b9810831ae7e79c2795ad3214a4b1c78df71e7a2a910429dd21d6b67a7a2f22c29b284fc30d68fff4ea9907fc39edefbb630027eb9a1e2cd4eb2ae24
+  metadata.gz: b7d12b75f0e80075d31832f8add9d1269ae48ff32714a0c0b481a85eb275094ccec62020911d55f74a2920211fc445f11ee3c31f43fdc7cc1f5ae2d175278eec
+  data.tar.gz: 138c464c0531fc12773b995d3101f5e952e2a4868e7e13d18274b27f7b0ea310a515d76e2066f36b25c2586c2046b9497f15a74e943cfbc8fdc8f0204796f93e

data/CHANGELOG.md

@@ -4,7 +4,18 @@ Reverse Chronological Order:
 
 ## `master`
 
-* Add your changelog here
+* Add your description here
+
+## `0.9.0` (2019-01-22)
+
+* Fix a problem with stuck of proxies list loading.
+
+* Add a possibility to configure different timeouts for different cases:
+  - `client_timeout` - timeout for `ProxyFetcher::Client`.
+  - `provider_proxies_load_timeout` - timeout for loading of proxies list by provider.
+  - `proxy_validation_timeout` - timeout for proxy validation with `ProxyFetcher::ProxyValidator`. 
+  
+  (old option `timeout` sets and returns value of `client_timeout`)
 
 ## `0.8.0` (2018-11-12)
 

data/lib/proxy_fetcher.rb

@@ -54,7 +54,8 @@ module ProxyFetcher
     #   ProxyFetcher.config
     #
     #   #=> #<ProxyFetcher::Configuration:0x0000000241eec8 @user_agent="Mozilla/5.0, ...", @pool_size=10,
-    #           @timeout=3, @http_client=ProxyFetcher::HTTPClient, @proxy_validator=ProxyFetcher::ProxyValidator,
+    #           @client_timeout=3, @proxy_validation_timeout=3, @provider_proxies_load_timeout=30,
+    #           @http_client=ProxyFetcher::HTTPClient, @proxy_validator=ProxyFetcher::ProxyValidator,
     #           @providers=[:free_proxy_list, ...], @adapter=ProxyFetcher::Document::NokogiriAdapter>
     #
     def config

data/lib/proxy_fetcher/client/request.rb

@@ -56,7 +56,7 @@ module ProxyFetcher
         @method = args.fetch(:method).to_s.downcase
         @headers = (args[:headers] || {}).dup
         @payload = args[:payload]
-        @timeout = args.fetch(:timeout, ProxyFetcher.config.timeout)
+        @timeout = args.fetch(:timeout, ProxyFetcher.config.client_timeout)
         @ssl_options = args.fetch(:ssl_options, default_ssl_options)
 
         @proxy = args.fetch(:proxy)

data/lib/proxy_fetcher/configuration.rb

@@ -5,9 +5,21 @@ module ProxyFetcher
   # with HTTP requests, adapters, custom classes.
   #
   class Configuration
-    # @!attribute timeout
-    #   @return [Integer] HTTP request connection / open timeout
-    attr_accessor :timeout
+    # @!attribute client_timeout
+    #   @return [Integer] HTTP request timeout (connect / open) for [ProxyFetcher::Client]
+    attr_accessor :client_timeout
+
+    # @!attribute provider_proxies_load_timeout
+    #   @return [Integer] HTTP request timeout (connect / open) for loading of proxies list by provider
+    attr_accessor :provider_proxies_load_timeout
+
+    # @!attribute proxy_validation_timeout
+    #   @return [Integer] HTTP request timeout (connect / open) for proxy validation with [ProxyFetcher::ProxyValidator]
+    attr_accessor :proxy_validation_timeout
+
+    # to save compatibility
+    alias timeout client_timeout
+    alias timeout= client_timeout=
 
     # @!attribute pool_size
     #   @return [Integer] proxy validator pool size (max number of threads)
@@ -98,7 +110,10 @@ module ProxyFetcher
       @logger = Logger.new(STDOUT)
       @user_agent = DEFAULT_USER_AGENT
       @pool_size = 10
-      @timeout = 3
+      @client_timeout = 3
+      @provider_proxies_load_timeout = 30
+      @proxy_validation_timeout = 3
+
       @http_client = HTTPClient
       @proxy_validator = ProxyValidator
 

data/lib/proxy_fetcher/utils/http_client.rb

@@ -17,6 +17,10 @@ module ProxyFetcher
     #   @return [OpenSSL::SSL::SSLContext] SSL context
     attr_reader :ssl_ctx
 
+    # @!attribute [r] timeout
+    #   @return [Integer] Request timeout
+    attr_reader :timeout
+
     # Fetches resource content by sending HTTP request to it.
     # Synthetic sugar to simplify URIes fetching.
     #
@@ -36,6 +40,7 @@ module ProxyFetcher
     def initialize(url)
       @url = url.to_s
       @http = HTTP.headers(default_headers)
+      @timeout = ProxyFetcher.config.provider_proxies_load_timeout
 
       @ssl_ctx = OpenSSL::SSL::SSLContext.new
       @ssl_ctx.verify_mode = OpenSSL::SSL::VERIFY_NONE
@@ -47,7 +52,9 @@ module ProxyFetcher
     #   response body
     #
     def fetch
-      @http.get(url, ssl_context: ssl_ctx).body.to_s
+      @http.timeout(connect: timeout, read: timeout)
+           .get(url, ssl_context: ssl_ctx)
+           .body.to_s
     rescue StandardError
       ProxyFetcher.logger.warn("Failed to load proxy list for #{url}")
       ''

data/lib/proxy_fetcher/utils/proxy_validator.rb

@@ -28,7 +28,7 @@ module ProxyFetcher
     # @return [ProxyValidator]
     #
     def initialize(proxy_addr, proxy_port)
-      timeout = ProxyFetcher.config.timeout
+      timeout = ProxyFetcher.config.proxy_validation_timeout
 
       @http = HTTP.follow.via(proxy_addr, proxy_port.to_i).timeout(connect: timeout, read: timeout)
     end

data/lib/proxy_fetcher/version.rb

@@ -13,7 +13,7 @@ module ProxyFetcher
     # Major version number
     MAJOR = 0
     # Minor version number
-    MINOR = 8
+    MINOR = 9
     # Smallest version number
     TINY  = 0
 

data/proxy_fetcher.gemspec

@@ -20,5 +20,6 @@ Gem::Specification.new do |gem|
 
   gem.add_runtime_dependency 'http', '~> 3.0'
 
+  gem.add_development_dependency 'rake', '>= 12.0'
   gem.add_development_dependency 'rspec', '~> 3.5'
 end

data/spec/proxy_fetcher/client/client_spec.rb

@@ -10,7 +10,7 @@ xdescribe ProxyFetcher::Client do
   before :all do
     ProxyFetcher.configure do |config|
       config.provider = :xroxy
-      config.timeout = 5
+      config.client_timeout = 5
     end
 
     @server = EvilProxy::MITMProxyServer.new Port: 3128, Quiet: true

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: proxy_fetcher
 version: !ruby/object:Gem::Version
-  version: 0.8.0
+  version: 0.9.0
 platform: ruby
 authors:
 - Nikita Bulai
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-11-12 00:00:00.000000000 Z
+date: 2019-01-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: http
@@ -24,6 +24,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '12.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '12.0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -120,8 +134,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.11
+rubygems_version: 3.0.2
 signing_key: 
 specification_version: 4
 summary: Ruby gem for dealing with proxy lists from different providers