proxy_fetcher 0.2.3 → 0.2.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +10 -0
- data/README.md +32 -4
- data/lib/proxy_fetcher.rb +5 -0
- data/lib/proxy_fetcher/configuration.rb +2 -0
- data/lib/proxy_fetcher/providers/base.rb +1 -5
- data/lib/proxy_fetcher/providers/xroxy.rb +1 -1
- data/lib/proxy_fetcher/proxy.rb +1 -1
- data/lib/proxy_fetcher/utils/http_fetcher.rb +24 -0
- data/lib/proxy_fetcher/version.rb +1 -1
- data/proxy_fetcher.gemspec +1 -1
- data/spec/proxy_fetcher/providers/free_proxy_list_spec.rb +3 -1
- data/spec/proxy_fetcher/proxy_spec.rb +5 -0
- metadata +4 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: a0c631e566e3330cda9a53d1ecbaa38cc0c4e055
|
4
|
+
data.tar.gz: e60250b46b1db6cc1f1d77efa8351b8b422c1f7e
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a0a8e44ed225617a7110933054111824bfacbc7f32698bb465a51d85b1870a23ac0cf792f9d1af0128f4bba1eb489ee68288317807b8b631cef2900a4e8ae1e6
|
7
|
+
data.tar.gz: 525adfd9031c16d5f709405c1eb70e0dac13e285709e59afb1d6c20755fe3c580dd011a5c773cc35defce5c3dbf565261b4aa23e3a74ffe987afa60733e985d5
|
data/.rubocop.yml
ADDED
data/README.md
CHANGED
@@ -5,7 +5,7 @@
|
|
5
5
|
[](#license)
|
6
6
|
|
7
7
|
This gem can help your Ruby application to make HTTP(S) requests from proxy by fetching and validating actual
|
8
|
-
proxy lists from the different providers like [HideMyName](https://hidemy.name/en/)
|
8
|
+
proxy lists from the different providers like [HideMyName](https://hidemy.name/en/).
|
9
9
|
|
10
10
|
It gives you a `Manager` class that can load proxy list, validate it and return random or specific proxy entry. Take a look
|
11
11
|
at the documentation below to find all the gem features.
|
@@ -111,16 +111,44 @@ You can sort or find any proxy by speed using next 3 instance methods:
|
|
111
111
|
* `medium?`
|
112
112
|
* `slow?`'
|
113
113
|
|
114
|
-
|
114
|
+
## Configuration
|
115
|
+
|
116
|
+
To change open/read timeout for `cleanup!` and `connectable?` methods you need to change ProxyFetcher.config:
|
115
117
|
|
116
118
|
```ruby
|
117
|
-
ProxyFetcher.
|
118
|
-
|
119
|
+
ProxyFetcher.configure do |config|
|
120
|
+
config.read_timeout = 1 # default is 3
|
121
|
+
config.open_timeout = 1 # default is 3
|
122
|
+
end
|
119
123
|
|
120
124
|
manager = ProxyFetcher::Manager.new
|
121
125
|
manager.cleanup!
|
122
126
|
```
|
123
127
|
|
128
|
+
ProxyFetcher uses simple Ruby solution for dealing with HTTP requests - `net/http` library. If you wanna add, for example, your custom provider that
|
129
|
+
was developed as a Single Page Application (SPA) with some JavaScript, then you will need something like []selenium-webdriver](https://github.com/SeleniumHQ/selenium/tree/master/rb)
|
130
|
+
to properly load the content of the website. For those and other cases you can write your own class for fetching HTML content by the URL and setup it
|
131
|
+
in the ProxyFetcher config:
|
132
|
+
|
133
|
+
```ruby
|
134
|
+
class MyHTTPClient
|
135
|
+
class << self
|
136
|
+
# [IMPORTANT]: self.fetch method is required!
|
137
|
+
def fetch(url)
|
138
|
+
# ... some magic to return proper HTML ...
|
139
|
+
end
|
140
|
+
end
|
141
|
+
end
|
142
|
+
|
143
|
+
ProxyFetcher.config.http_client = MyHTTPClient
|
144
|
+
|
145
|
+
manager = ProxyFetcher::Manager.new
|
146
|
+
manager.proxies
|
147
|
+
|
148
|
+
#=> [#<ProxyFetcher::Proxy:0x00000002879680 @addr="97.77.104.22", @port=3128, @country="USA",
|
149
|
+
# @response_time=5217, @speed=48, @type="HTTP", @anonymity="High">, ... ]
|
150
|
+
```
|
151
|
+
|
124
152
|
## Providers
|
125
153
|
|
126
154
|
Currently ProxyFetcher can deal with next proxy providers (services):
|
data/lib/proxy_fetcher.rb
CHANGED
@@ -5,6 +5,7 @@ require 'nokogiri'
|
|
5
5
|
require 'proxy_fetcher/configuration'
|
6
6
|
require 'proxy_fetcher/proxy'
|
7
7
|
require 'proxy_fetcher/manager'
|
8
|
+
require 'proxy_fetcher/utils/http_fetcher'
|
8
9
|
require 'proxy_fetcher/providers/base'
|
9
10
|
require 'proxy_fetcher/providers/free_proxy_list'
|
10
11
|
require 'proxy_fetcher/providers/free_proxy_list_ssl'
|
@@ -17,5 +18,9 @@ module ProxyFetcher
|
|
17
18
|
def config
|
18
19
|
@config ||= ProxyFetcher::Configuration.new
|
19
20
|
end
|
21
|
+
|
22
|
+
def configure
|
23
|
+
yield config
|
24
|
+
end
|
20
25
|
end
|
21
26
|
end
|
@@ -4,6 +4,7 @@ module ProxyFetcher
|
|
4
4
|
RegisteredProvider = Class.new(StandardError)
|
5
5
|
|
6
6
|
attr_accessor :open_timeout, :read_timeout, :provider
|
7
|
+
attr_accessor :http_client
|
7
8
|
|
8
9
|
class << self
|
9
10
|
def providers
|
@@ -20,6 +21,7 @@ module ProxyFetcher
|
|
20
21
|
def initialize
|
21
22
|
@open_timeout = 3
|
22
23
|
@read_timeout = 3
|
24
|
+
@http_client = HTTPClient
|
23
25
|
|
24
26
|
self.provider = :hide_my_name # currently default one
|
25
27
|
end
|
@@ -18,11 +18,7 @@ module ProxyFetcher
|
|
18
18
|
|
19
19
|
# Get HTML from the requested URL
|
20
20
|
def load_html(url)
|
21
|
-
|
22
|
-
http = Net::HTTP.new(uri.host, uri.port)
|
23
|
-
http.use_ssl = true if uri.scheme == 'https'
|
24
|
-
response = http.get(uri.request_uri)
|
25
|
-
response.body
|
21
|
+
ProxyFetcher.config.http_client.fetch(url)
|
26
22
|
end
|
27
23
|
end
|
28
24
|
end
|
@@ -21,7 +21,7 @@ module ProxyFetcher
|
|
21
21
|
set!(:anonymity, td.content.strip)
|
22
22
|
when 4
|
23
23
|
ssl = td.content.strip.downcase
|
24
|
-
set!(:type, ssl.include?('true') ? 'HTTPS' : 'HTTP'
|
24
|
+
set!(:type, ssl.include?('true') ? 'HTTPS' : 'HTTP')
|
25
25
|
when 5 then
|
26
26
|
set!(:country, td.content.strip)
|
27
27
|
when 6
|
data/lib/proxy_fetcher/proxy.rb
CHANGED
@@ -17,7 +17,7 @@ module ProxyFetcher
|
|
17
17
|
connection.start { |http| return true if http.request_head('/') }
|
18
18
|
|
19
19
|
false
|
20
|
-
rescue Timeout::Error, Errno::ECONNREFUSED, Errno::ECONNRESET, Errno::ECONNABORTED
|
20
|
+
rescue Timeout::Error, Errno::ECONNREFUSED, Errno::ECONNRESET, Errno::ECONNABORTED
|
21
21
|
false
|
22
22
|
end
|
23
23
|
|
@@ -0,0 +1,24 @@
|
|
1
|
+
module ProxyFetcher
|
2
|
+
class HTTPClient
|
3
|
+
attr_reader :http
|
4
|
+
|
5
|
+
def initialize(url)
|
6
|
+
@uri = URI.parse(url)
|
7
|
+
@http = Net::HTTP.new(@uri.host, @uri.port)
|
8
|
+
@http.use_ssl = true if @uri.scheme.downcase == 'https'
|
9
|
+
end
|
10
|
+
|
11
|
+
def fetch
|
12
|
+
request = Net::HTTP::Get.new(@uri.to_s)
|
13
|
+
request['Connection'] = 'keep-alive'
|
14
|
+
response = @http.request(request)
|
15
|
+
response.body
|
16
|
+
end
|
17
|
+
|
18
|
+
class << self
|
19
|
+
def fetch(url)
|
20
|
+
new(url).fetch
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
data/proxy_fetcher.gemspec
CHANGED
@@ -5,7 +5,7 @@ require 'proxy_fetcher/version'
|
|
5
5
|
Gem::Specification.new do |gem|
|
6
6
|
gem.name = 'proxy_fetcher'
|
7
7
|
gem.version = ProxyFetcher.gem_version
|
8
|
-
gem.date = '2017-08-
|
8
|
+
gem.date = '2017-08-17'
|
9
9
|
gem.summary = 'Ruby gem for dealing with proxy lists '
|
10
10
|
gem.description = 'This gem can help your Ruby application to make HTTP(S) requests ' \
|
11
11
|
'from proxy server by fetching and validating proxy lists from the different providers.'
|
@@ -26,6 +26,11 @@ describe ProxyFetcher::Proxy do
|
|
26
26
|
expect(proxy.connectable?).to be_falsey
|
27
27
|
end
|
28
28
|
|
29
|
+
it 'not connectable if ERR' do
|
30
|
+
allow_any_instance_of(Net::HTTP).to receive(:start).and_raise(Errno::ECONNABORTED)
|
31
|
+
expect(proxy.connectable?).to be_falsey
|
32
|
+
end
|
33
|
+
|
29
34
|
it "not connectable if server doesn't respond to head" do
|
30
35
|
allow_any_instance_of(Net::HTTP).to receive(:start).and_return(false)
|
31
36
|
expect(proxy.connectable?).to be_falsey
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: proxy_fetcher
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Nikita Bulai
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-08-
|
11
|
+
date: 2017-08-17 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: nokogiri
|
@@ -52,6 +52,7 @@ extensions: []
|
|
52
52
|
extra_rdoc_files: []
|
53
53
|
files:
|
54
54
|
- ".gitignore"
|
55
|
+
- ".rubocop.yml"
|
55
56
|
- ".travis.yml"
|
56
57
|
- Gemfile
|
57
58
|
- LICENSE
|
@@ -67,6 +68,7 @@ files:
|
|
67
68
|
- lib/proxy_fetcher/providers/proxy_docker.rb
|
68
69
|
- lib/proxy_fetcher/providers/xroxy.rb
|
69
70
|
- lib/proxy_fetcher/proxy.rb
|
71
|
+
- lib/proxy_fetcher/utils/http_fetcher.rb
|
70
72
|
- lib/proxy_fetcher/version.rb
|
71
73
|
- proxy_fetcher.gemspec
|
72
74
|
- spec/proxy_fetcher/providers/free_proxy_list_spec.rb
|