proxy_fetcher 0.2.3 → 0.2.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 4db0d1af8440399bbefab34dfbc0f19dbc20feff
4
- data.tar.gz: d58e239f74268c8eab23092f452108d8ddc6f06b
3
+ metadata.gz: a0c631e566e3330cda9a53d1ecbaa38cc0c4e055
4
+ data.tar.gz: e60250b46b1db6cc1f1d77efa8351b8b422c1f7e
5
5
  SHA512:
6
- metadata.gz: 66c7c26cf9c961183d2a4e6391c8dc8b328a75ddd4172181f126f4cff9941597538db91200c610d0a02451f585a04968b7b0221b70ad38716cfdfac580030669
7
- data.tar.gz: 964d4a71818c01fef020c55f06a1c74994d77a54f50c2f38c1a3874a419d4d1d7b09c2a3d863c94bce4180e08a434f083dabe899a043df4da03686ae7fbef00e
6
+ metadata.gz: a0a8e44ed225617a7110933054111824bfacbc7f32698bb465a51d85b1870a23ac0cf792f9d1af0128f4bba1eb489ee68288317807b8b631cef2900a4e8ae1e6
7
+ data.tar.gz: 525adfd9031c16d5f709405c1eb70e0dac13e285709e59afb1d6c20755fe3c580dd011a5c773cc35defce5c3dbf565261b4aa23e3a74ffe987afa60733e985d5
data/.rubocop.yml ADDED
@@ -0,0 +1,10 @@
1
+ LineLength:
2
+ Max: 120
3
+ AllCops:
4
+ Exclude:
5
+ - 'spec/**/*'
6
+ DisplayCopNames: true
7
+ Rails:
8
+ Enabled: false
9
+ Documentation:
10
+ Enabled: false
data/README.md CHANGED
@@ -5,7 +5,7 @@
5
5
  [![License](http://img.shields.io/badge/license-MIT-brightgreen.svg)](#license)
6
6
 
7
7
  This gem can help your Ruby application to make HTTP(S) requests from proxy by fetching and validating actual
8
- proxy lists from the different providers like [HideMyName](https://hidemy.name/en/) or Hide My Name.
8
+ proxy lists from the different providers like [HideMyName](https://hidemy.name/en/).
9
9
 
10
10
  It gives you a `Manager` class that can load proxy list, validate it and return random or specific proxy entry. Take a look
11
11
  at the documentation below to find all the gem features.
@@ -111,16 +111,44 @@ You can sort or find any proxy by speed using next 3 instance methods:
111
111
  * `medium?`
112
112
  * `slow?`'
113
113
 
114
- To change open/read timeout for `cleanup!` and `connectable?` methods yu need to change ProxyFetcher.config:
114
+ ## Configuration
115
+
116
+ To change open/read timeout for `cleanup!` and `connectable?` methods you need to change ProxyFetcher.config:
115
117
 
116
118
  ```ruby
117
- ProxyFetcher.config.read_timeout = 1 # default is 3
118
- ProxyFetcher.config.open_timeout = 1 # default is 3
119
+ ProxyFetcher.configure do |config|
120
+ config.read_timeout = 1 # default is 3
121
+ config.open_timeout = 1 # default is 3
122
+ end
119
123
 
120
124
  manager = ProxyFetcher::Manager.new
121
125
  manager.cleanup!
122
126
  ```
123
127
 
128
+ ProxyFetcher uses simple Ruby solution for dealing with HTTP requests - `net/http` library. If you wanna add, for example, your custom provider that
129
+ was developed as a Single Page Application (SPA) with some JavaScript, then you will need something like []selenium-webdriver](https://github.com/SeleniumHQ/selenium/tree/master/rb)
130
+ to properly load the content of the website. For those and other cases you can write your own class for fetching HTML content by the URL and setup it
131
+ in the ProxyFetcher config:
132
+
133
+ ```ruby
134
+ class MyHTTPClient
135
+ class << self
136
+ # [IMPORTANT]: self.fetch method is required!
137
+ def fetch(url)
138
+ # ... some magic to return proper HTML ...
139
+ end
140
+ end
141
+ end
142
+
143
+ ProxyFetcher.config.http_client = MyHTTPClient
144
+
145
+ manager = ProxyFetcher::Manager.new
146
+ manager.proxies
147
+
148
+ #=> [#<ProxyFetcher::Proxy:0x00000002879680 @addr="97.77.104.22", @port=3128, @country="USA",
149
+ # @response_time=5217, @speed=48, @type="HTTP", @anonymity="High">, ... ]
150
+ ```
151
+
124
152
  ## Providers
125
153
 
126
154
  Currently ProxyFetcher can deal with next proxy providers (services):
data/lib/proxy_fetcher.rb CHANGED
@@ -5,6 +5,7 @@ require 'nokogiri'
5
5
  require 'proxy_fetcher/configuration'
6
6
  require 'proxy_fetcher/proxy'
7
7
  require 'proxy_fetcher/manager'
8
+ require 'proxy_fetcher/utils/http_fetcher'
8
9
  require 'proxy_fetcher/providers/base'
9
10
  require 'proxy_fetcher/providers/free_proxy_list'
10
11
  require 'proxy_fetcher/providers/free_proxy_list_ssl'
@@ -17,5 +18,9 @@ module ProxyFetcher
17
18
  def config
18
19
  @config ||= ProxyFetcher::Configuration.new
19
20
  end
21
+
22
+ def configure
23
+ yield config
24
+ end
20
25
  end
21
26
  end
@@ -4,6 +4,7 @@ module ProxyFetcher
4
4
  RegisteredProvider = Class.new(StandardError)
5
5
 
6
6
  attr_accessor :open_timeout, :read_timeout, :provider
7
+ attr_accessor :http_client
7
8
 
8
9
  class << self
9
10
  def providers
@@ -20,6 +21,7 @@ module ProxyFetcher
20
21
  def initialize
21
22
  @open_timeout = 3
22
23
  @read_timeout = 3
24
+ @http_client = HTTPClient
23
25
 
24
26
  self.provider = :hide_my_name # currently default one
25
27
  end
@@ -18,11 +18,7 @@ module ProxyFetcher
18
18
 
19
19
  # Get HTML from the requested URL
20
20
  def load_html(url)
21
- uri = URI.parse(url)
22
- http = Net::HTTP.new(uri.host, uri.port)
23
- http.use_ssl = true if uri.scheme == 'https'
24
- response = http.get(uri.request_uri)
25
- response.body
21
+ ProxyFetcher.config.http_client.fetch(url)
26
22
  end
27
23
  end
28
24
  end
@@ -21,7 +21,7 @@ module ProxyFetcher
21
21
  set!(:anonymity, td.content.strip)
22
22
  when 4
23
23
  ssl = td.content.strip.downcase
24
- set!(:type, ssl.include?('true') ? 'HTTPS' : 'HTTP' )
24
+ set!(:type, ssl.include?('true') ? 'HTTPS' : 'HTTP')
25
25
  when 5 then
26
26
  set!(:country, td.content.strip)
27
27
  when 6
@@ -17,7 +17,7 @@ module ProxyFetcher
17
17
  connection.start { |http| return true if http.request_head('/') }
18
18
 
19
19
  false
20
- rescue Timeout::Error, Errno::ECONNREFUSED, Errno::ECONNRESET, Errno::ECONNABORTED, Errno::EOFError
20
+ rescue Timeout::Error, Errno::ECONNREFUSED, Errno::ECONNRESET, Errno::ECONNABORTED
21
21
  false
22
22
  end
23
23
 
@@ -0,0 +1,24 @@
1
+ module ProxyFetcher
2
+ class HTTPClient
3
+ attr_reader :http
4
+
5
+ def initialize(url)
6
+ @uri = URI.parse(url)
7
+ @http = Net::HTTP.new(@uri.host, @uri.port)
8
+ @http.use_ssl = true if @uri.scheme.downcase == 'https'
9
+ end
10
+
11
+ def fetch
12
+ request = Net::HTTP::Get.new(@uri.to_s)
13
+ request['Connection'] = 'keep-alive'
14
+ response = @http.request(request)
15
+ response.body
16
+ end
17
+
18
+ class << self
19
+ def fetch(url)
20
+ new(url).fetch
21
+ end
22
+ end
23
+ end
24
+ end
@@ -9,7 +9,7 @@ module ProxyFetcher
9
9
  # Minor version number
10
10
  MINOR = 2
11
11
  # Smallest version number
12
- TINY = 3
12
+ TINY = 5
13
13
 
14
14
  # Full version number
15
15
  STRING = [MAJOR, MINOR, TINY].compact.join('.')
@@ -5,7 +5,7 @@ require 'proxy_fetcher/version'
5
5
  Gem::Specification.new do |gem|
6
6
  gem.name = 'proxy_fetcher'
7
7
  gem.version = ProxyFetcher.gem_version
8
- gem.date = '2017-08-10'
8
+ gem.date = '2017-08-17'
9
9
  gem.summary = 'Ruby gem for dealing with proxy lists '
10
10
  gem.description = 'This gem can help your Ruby application to make HTTP(S) requests ' \
11
11
  'from proxy server by fetching and validating proxy lists from the different providers.'
@@ -2,7 +2,9 @@ require 'spec_helper'
2
2
 
3
3
  describe ProxyFetcher::Providers::FreeProxyList do
4
4
  before :all do
5
- ProxyFetcher.config.provider = :free_proxy_list
5
+ ProxyFetcher.configure do |config|
6
+ config.provider = :free_proxy_list
7
+ end
6
8
  end
7
9
 
8
10
  it_behaves_like 'a manager'
@@ -26,6 +26,11 @@ describe ProxyFetcher::Proxy do
26
26
  expect(proxy.connectable?).to be_falsey
27
27
  end
28
28
 
29
+ it 'not connectable if ERR' do
30
+ allow_any_instance_of(Net::HTTP).to receive(:start).and_raise(Errno::ECONNABORTED)
31
+ expect(proxy.connectable?).to be_falsey
32
+ end
33
+
29
34
  it "not connectable if server doesn't respond to head" do
30
35
  allow_any_instance_of(Net::HTTP).to receive(:start).and_return(false)
31
36
  expect(proxy.connectable?).to be_falsey
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: proxy_fetcher
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.3
4
+ version: 0.2.5
5
5
  platform: ruby
6
6
  authors:
7
7
  - Nikita Bulai
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2017-08-10 00:00:00.000000000 Z
11
+ date: 2017-08-17 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: nokogiri
@@ -52,6 +52,7 @@ extensions: []
52
52
  extra_rdoc_files: []
53
53
  files:
54
54
  - ".gitignore"
55
+ - ".rubocop.yml"
55
56
  - ".travis.yml"
56
57
  - Gemfile
57
58
  - LICENSE
@@ -67,6 +68,7 @@ files:
67
68
  - lib/proxy_fetcher/providers/proxy_docker.rb
68
69
  - lib/proxy_fetcher/providers/xroxy.rb
69
70
  - lib/proxy_fetcher/proxy.rb
71
+ - lib/proxy_fetcher/utils/http_fetcher.rb
70
72
  - lib/proxy_fetcher/version.rb
71
73
  - proxy_fetcher.gemspec
72
74
  - spec/proxy_fetcher/providers/free_proxy_list_spec.rb