proxy_fetcher 0.2.3 → 0.2.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4db0d1af8440399bbefab34dfbc0f19dbc20feff
-  data.tar.gz: d58e239f74268c8eab23092f452108d8ddc6f06b
+  metadata.gz: a0c631e566e3330cda9a53d1ecbaa38cc0c4e055
+  data.tar.gz: e60250b46b1db6cc1f1d77efa8351b8b422c1f7e
 SHA512:
-  metadata.gz: 66c7c26cf9c961183d2a4e6391c8dc8b328a75ddd4172181f126f4cff9941597538db91200c610d0a02451f585a04968b7b0221b70ad38716cfdfac580030669
-  data.tar.gz: 964d4a71818c01fef020c55f06a1c74994d77a54f50c2f38c1a3874a419d4d1d7b09c2a3d863c94bce4180e08a434f083dabe899a043df4da03686ae7fbef00e
+  metadata.gz: a0a8e44ed225617a7110933054111824bfacbc7f32698bb465a51d85b1870a23ac0cf792f9d1af0128f4bba1eb489ee68288317807b8b631cef2900a4e8ae1e6
+  data.tar.gz: 525adfd9031c16d5f709405c1eb70e0dac13e285709e59afb1d6c20755fe3c580dd011a5c773cc35defce5c3dbf565261b4aa23e3a74ffe987afa60733e985d5

data/.rubocop.yml

@@ -0,0 +1,10 @@
+LineLength:
+  Max: 120
+AllCops:
+  Exclude:
+    - 'spec/**/*'
+  DisplayCopNames: true
+Rails:
+  Enabled: false
+Documentation:
+  Enabled: false

data/README.md

@@ -5,7 +5,7 @@
 [![License](http://img.shields.io/badge/license-MIT-brightgreen.svg)](#license)
 
 This gem can help your Ruby application to make HTTP(S) requests from proxy by fetching and validating actual
-proxy lists from the different providers like [HideMyName](https://hidemy.name/en/) or Hide My Name.
+proxy lists from the different providers like [HideMyName](https://hidemy.name/en/).
 
 It gives you a `Manager` class that can load proxy list, validate it and return random or specific proxy entry. Take a look
 at the documentation below to find all the gem features.
@@ -111,16 +111,44 @@ You can sort or find any proxy by speed using next 3 instance methods:
 * `medium?`
 * `slow?`'
 
-To change open/read timeout for `cleanup!` and `connectable?` methods yu need to change ProxyFetcher.config:
+## Configuration
+
+To change open/read timeout for `cleanup!` and `connectable?` methods you need to change ProxyFetcher.config:
 
 ```ruby
-ProxyFetcher.config.read_timeout = 1 # default is 3
-ProxyFetcher.config.open_timeout = 1 # default is 3
+ProxyFetcher.configure do |config|
+  config.read_timeout = 1 # default is 3
+  config.open_timeout = 1 # default is 3
+end
 
 manager = ProxyFetcher::Manager.new
 manager.cleanup!
 ```
 
+ProxyFetcher uses simple Ruby solution for dealing with HTTP requests - `net/http` library. If you wanna add, for example, your custom provider that
+was developed as a Single Page Application (SPA) with some JavaScript, then you will need something like []selenium-webdriver](https://github.com/SeleniumHQ/selenium/tree/master/rb)
+to properly load the content of the website. For those and other cases you can write your own class for fetching HTML content by the URL and setup it
+in the ProxyFetcher config:
+
+```ruby
+class MyHTTPClient
+  class << self
+    # [IMPORTANT]: self.fetch method is required!
+    def fetch(url)
+      # ... some magic to return proper HTML ...
+    end
+  end
+end
+
+ProxyFetcher.config.http_client = MyHTTPClient
+
+manager = ProxyFetcher::Manager.new
+manager.proxies
+
+#=> [#<ProxyFetcher::Proxy:0x00000002879680 @addr="97.77.104.22", @port=3128, @country="USA", 
+ #     @response_time=5217, @speed=48, @type="HTTP", @anonymity="High">, ... ]
+```
+
 ## Providers
 
 Currently ProxyFetcher can deal with next proxy providers (services):

data/lib/proxy_fetcher.rb

@@ -5,6 +5,7 @@ require 'nokogiri'
 require 'proxy_fetcher/configuration'
 require 'proxy_fetcher/proxy'
 require 'proxy_fetcher/manager'
+require 'proxy_fetcher/utils/http_fetcher'
 require 'proxy_fetcher/providers/base'
 require 'proxy_fetcher/providers/free_proxy_list'
 require 'proxy_fetcher/providers/free_proxy_list_ssl'
@@ -17,5 +18,9 @@ module ProxyFetcher
     def config
       @config ||= ProxyFetcher::Configuration.new
     end
+
+    def configure
+      yield config
+    end
   end
 end

data/lib/proxy_fetcher/configuration.rb

@@ -4,6 +4,7 @@ module ProxyFetcher
     RegisteredProvider = Class.new(StandardError)
 
     attr_accessor :open_timeout, :read_timeout, :provider
+    attr_accessor :http_client
 
     class << self
       def providers
@@ -20,6 +21,7 @@ module ProxyFetcher
     def initialize
       @open_timeout = 3
       @read_timeout = 3
+      @http_client = HTTPClient
 
       self.provider = :hide_my_name # currently default one
     end

data/lib/proxy_fetcher/providers/base.rb

@@ -18,11 +18,7 @@ module ProxyFetcher
 
         # Get HTML from the requested URL
         def load_html(url)
-          uri = URI.parse(url)
-          http = Net::HTTP.new(uri.host, uri.port)
-          http.use_ssl = true if uri.scheme == 'https'
-          response = http.get(uri.request_uri)
-          response.body
+          ProxyFetcher.config.http_client.fetch(url)
         end
       end
     end

data/lib/proxy_fetcher/providers/xroxy.rb

@@ -21,7 +21,7 @@ module ProxyFetcher
             set!(:anonymity,  td.content.strip)
           when 4
             ssl = td.content.strip.downcase
-            set!(:type, ssl.include?('true') ? 'HTTPS' : 'HTTP' )
+            set!(:type, ssl.include?('true') ? 'HTTPS' : 'HTTP')
           when 5 then
             set!(:country, td.content.strip)
           when 6

data/lib/proxy_fetcher/proxy.rb

@@ -17,7 +17,7 @@ module ProxyFetcher
       connection.start { |http| return true if http.request_head('/') }
 
       false
-    rescue Timeout::Error, Errno::ECONNREFUSED, Errno::ECONNRESET, Errno::ECONNABORTED, Errno::EOFError
+    rescue Timeout::Error, Errno::ECONNREFUSED, Errno::ECONNRESET, Errno::ECONNABORTED
       false
     end
 

data/lib/proxy_fetcher/utils/http_fetcher.rb

@@ -0,0 +1,24 @@
+module ProxyFetcher
+  class HTTPClient
+    attr_reader :http
+
+    def initialize(url)
+      @uri = URI.parse(url)
+      @http = Net::HTTP.new(@uri.host, @uri.port)
+      @http.use_ssl = true if @uri.scheme.downcase == 'https'
+    end
+
+    def fetch
+      request = Net::HTTP::Get.new(@uri.to_s)
+      request['Connection'] = 'keep-alive'
+      response = @http.request(request)
+      response.body
+    end
+
+    class << self
+      def fetch(url)
+        new(url).fetch
+      end
+    end
+  end
+end

data/lib/proxy_fetcher/version.rb

@@ -9,7 +9,7 @@ module ProxyFetcher
     # Minor version number
     MINOR = 2
     # Smallest version number
-    TINY  = 3
+    TINY  = 5
 
     # Full version number
     STRING = [MAJOR, MINOR, TINY].compact.join('.')

data/proxy_fetcher.gemspec

@@ -5,7 +5,7 @@ require 'proxy_fetcher/version'
 Gem::Specification.new do |gem|
   gem.name = 'proxy_fetcher'
   gem.version = ProxyFetcher.gem_version
-  gem.date = '2017-08-10'
+  gem.date = '2017-08-17'
   gem.summary = 'Ruby gem for dealing with proxy lists '
   gem.description = 'This gem can help your Ruby application to make HTTP(S) requests ' \
                     'from proxy server by fetching and validating proxy lists from the different providers.'

data/spec/proxy_fetcher/providers/free_proxy_list_spec.rb

@@ -2,7 +2,9 @@ require 'spec_helper'
 
 describe ProxyFetcher::Providers::FreeProxyList do
   before :all do
-    ProxyFetcher.config.provider = :free_proxy_list
+    ProxyFetcher.configure do |config|
+      config.provider = :free_proxy_list
+    end
   end
 
   it_behaves_like 'a manager'

data/spec/proxy_fetcher/proxy_spec.rb

@@ -26,6 +26,11 @@ describe ProxyFetcher::Proxy do
     expect(proxy.connectable?).to be_falsey
   end
 
+  it 'not connectable if ERR' do
+    allow_any_instance_of(Net::HTTP).to receive(:start).and_raise(Errno::ECONNABORTED)
+    expect(proxy.connectable?).to be_falsey
+  end
+
   it "not connectable if server doesn't respond to head" do
     allow_any_instance_of(Net::HTTP).to receive(:start).and_return(false)
     expect(proxy.connectable?).to be_falsey

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: proxy_fetcher
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: 0.2.5
 platform: ruby
 authors:
 - Nikita Bulai
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-08-10 00:00:00.000000000 Z
+date: 2017-08-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -52,6 +52,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- ".rubocop.yml"
 - ".travis.yml"
 - Gemfile
 - LICENSE
@@ -67,6 +68,7 @@ files:
 - lib/proxy_fetcher/providers/proxy_docker.rb
 - lib/proxy_fetcher/providers/xroxy.rb
 - lib/proxy_fetcher/proxy.rb
+- lib/proxy_fetcher/utils/http_fetcher.rb
 - lib/proxy_fetcher/version.rb
 - proxy_fetcher.gemspec
 - spec/proxy_fetcher/providers/free_proxy_list_spec.rb