proxy_manager 0.0.9 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile +3 -1
- data/README.md +43 -18
- data/lib/proxy_manager/proxy.rb +93 -0
- data/lib/proxy_manager.rb +1 -18
- data/proxy_manager.gemspec +4 -6
- data/spec/proxy_manager/proxy_spec.rb +60 -0
- data/spec/proxy_manager_spec.rb +6 -0
- data/spec/spec_helper.rb +3 -6
- data/spec/support/proxies.txt +2 -3
- metadata +10 -48
- data/Gemfile.lock +0 -77
- data/doc/ProxyManager/Main.html +0 -870
- data/doc/ProxyManager.html +0 -357
- data/doc/_index.html +0 -125
- data/doc/class_list.html +0 -54
- data/doc/css/common.css +0 -1
- data/doc/css/full_list.css +0 -57
- data/doc/css/style.css +0 -339
- data/doc/file.README.html +0 -162
- data/doc/file_list.html +0 -56
- data/doc/frames.html +0 -26
- data/doc/index.html +0 -162
- data/doc/js/app.js +0 -219
- data/doc/js/full_list.js +0 -178
- data/doc/js/jquery.js +0 -4
- data/doc/method_list.html +0 -119
- data/doc/top-level-namespace.html +0 -112
- data/lib/proxy_manager/main.rb +0 -107
- data/lib/tasks/proxy_manager_tasks.rake +0 -4
- data/spec/proxy_manager/proxy_manager_spec.rb +0 -76
- data/spec/support/bad_proxies.txt +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 5347792228f1358afe8763f4786894c3a574e257
|
4
|
+
data.tar.gz: 18add715d27e92b7cc2ea84d00de1f8fb4986a5d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 53acff22f244cbecec72cbd7611e85ce92340a0dad086a47fb1fdb9bb7ebc363584852ca1b3f6845c2ad178c544f857b0032248668e3644e36c0606bcc917e7c
|
7
|
+
data.tar.gz: f82b8f40a75d6b85310ab0a64135c7eb72eed164d5d97fbd762f8f08ce65f1bdc3b76e6f4d8ff9242475c4f8cee197488cecee2208f5f11bdb858bf517475748
|
data/Gemfile
CHANGED
data/README.md
CHANGED
@@ -1,5 +1,10 @@
|
|
1
1
|
# Ruby Proxy Manager
|
2
2
|
|
3
|
+
[](http://badge.fury.io/rb/proxy_manager)
|
4
|
+
[](https://travis-ci.org/bloodyhistory/proxy_manager)
|
5
|
+
[](https://codeclimate.com/github/bloodyhistory/proxy_manager)
|
6
|
+
[](https://codeclimate.com/github/bloodyhistory/proxy_manager)
|
7
|
+
|
3
8
|
Easy manage proxy in your parsers/web-bots.
|
4
9
|
|
5
10
|
|
@@ -30,61 +35,81 @@ $ gem install proxy_manager
|
|
30
35
|
From array (IP:PORT)
|
31
36
|
|
32
37
|
```ruby
|
33
|
-
proxy = ProxyManager.
|
38
|
+
proxy = ProxyManager::Proxy.new(['127.0.0.1:80', '127.0.0.1:8080'])
|
34
39
|
```
|
35
40
|
|
36
41
|
or from file
|
37
42
|
|
38
43
|
```ruby
|
39
|
-
proxy = ProxyManager.
|
44
|
+
proxy = ProxyManager::Proxy.new('proxies.txt')
|
40
45
|
```
|
41
46
|
|
42
|
-
|
47
|
+
File with proxy list (in this case `proxies.txt`) should be readable and
|
48
|
+
writable.
|
43
49
|
Example of `proxies.txt` content:
|
44
50
|
|
45
51
|
```
|
46
|
-
|
47
|
-
|
52
|
+
127.0.0.1:80
|
53
|
+
127.0.0.1:8080
|
48
54
|
...
|
49
55
|
```
|
50
56
|
|
51
57
|
### Get proxy
|
52
58
|
|
53
|
-
|
59
|
+
There is two methods to get proxy.
|
60
|
+
|
61
|
+
First method return just next proxy, without any checking for availability:
|
54
62
|
|
55
63
|
```ruby
|
56
64
|
proxy.get
|
57
|
-
# => ["
|
65
|
+
# => ["127.0.0.1", 80]
|
58
66
|
```
|
59
67
|
|
60
|
-
|
68
|
+
Band method return **only HTTP-available proxy**. It's perfect if you don't
|
69
|
+
realy confidency on actuallity of your proxy list:
|
61
70
|
|
62
71
|
```ruby
|
63
|
-
proxy.get
|
64
|
-
# => [
|
72
|
+
proxy.get!
|
73
|
+
# => ["127.0.0.1", 8080]
|
65
74
|
```
|
66
75
|
|
67
|
-
|
76
|
+
You can also get for than one proxy per request by adding count for both
|
77
|
+
methods like this:
|
78
|
+
|
79
|
+
```ruby
|
80
|
+
proxy.get(2)
|
81
|
+
# => [["127.0.0.1", 80], ["127.0.0.1", 8080]]
|
82
|
+
|
83
|
+
proxy.get!(2)
|
84
|
+
# => [["127.0.0.1", 80], ["127.0.0.1", 8080]]
|
85
|
+
```
|
68
86
|
|
69
|
-
### Proxies
|
87
|
+
### Proxies list
|
70
88
|
|
71
|
-
|
89
|
+
For display list of loaded proxies use `list` method:
|
72
90
|
|
73
91
|
```ruby
|
74
92
|
proxy.list
|
75
|
-
# => [["
|
93
|
+
# => [["127.0.0.1", 80], ["127.0.0.1", 8080]]
|
76
94
|
```
|
77
95
|
|
78
|
-
|
96
|
+
### Checking proxy manually
|
97
|
+
|
98
|
+
You can also use class method for checking availability manually like this:
|
79
99
|
|
80
100
|
```ruby
|
81
|
-
|
82
|
-
|
101
|
+
# by passing a string
|
102
|
+
ProxyManager::Proxy.connectable?('127.0.0.1:80')
|
103
|
+
# => false
|
104
|
+
|
105
|
+
# or by passing an array
|
106
|
+
ProxyManager::Proxy.connectable?('127.0.0.1', 80)
|
107
|
+
# => false
|
83
108
|
```
|
84
109
|
|
85
110
|
## Documentation
|
86
111
|
|
87
|
-
http://rubydoc.info/gems/proxy_manager
|
112
|
+
http://rubydoc.info/gems/proxy_manager
|
88
113
|
|
89
114
|
## Contributing
|
90
115
|
|
@@ -0,0 +1,93 @@
|
|
1
|
+
module ProxyManager
|
2
|
+
class Proxy
|
3
|
+
attr_reader :list
|
4
|
+
|
5
|
+
def initialize(proxies)
|
6
|
+
@list = []
|
7
|
+
|
8
|
+
if proxies.is_a? Array
|
9
|
+
load_list_from_array(proxies)
|
10
|
+
else
|
11
|
+
@list_file = proxies
|
12
|
+
@list = load_from_file(@list_file)
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
def get(count = 1)
|
17
|
+
get_proxy(count)
|
18
|
+
end
|
19
|
+
|
20
|
+
def get!(count = 1)
|
21
|
+
get_proxy(count, true)
|
22
|
+
end
|
23
|
+
|
24
|
+
def self.connectable?(proxy)
|
25
|
+
proxy = proxy.chomp.split(':') if proxy.is_a? String
|
26
|
+
ip, port = proxy
|
27
|
+
connection = Net::HTTP.new("http://google.com", nil, ip, port)
|
28
|
+
connection.open_timeout = 3
|
29
|
+
connection.read_timeout = 3
|
30
|
+
|
31
|
+
connection.start do |http|
|
32
|
+
return true if http.get('/')
|
33
|
+
end
|
34
|
+
|
35
|
+
false
|
36
|
+
rescue Exception => e
|
37
|
+
false
|
38
|
+
end
|
39
|
+
|
40
|
+
private
|
41
|
+
|
42
|
+
def load_list_from_array(proxies)
|
43
|
+
@list = proxies.map { |arg| [arg.split(':')[0], arg.split(':')[1].to_i] }
|
44
|
+
end
|
45
|
+
|
46
|
+
def load_from_file(file)
|
47
|
+
result = []
|
48
|
+
IO.readlines(file).each do |line|
|
49
|
+
ip, port = line.chomp.split(':')
|
50
|
+
result << [ip, port.to_i] if ip.is_a?(String) && port.is_a?(String)
|
51
|
+
end
|
52
|
+
result
|
53
|
+
end
|
54
|
+
|
55
|
+
def get_proxy(count, check_connection = false)
|
56
|
+
raise 'List is empty' if @list.empty?
|
57
|
+
|
58
|
+
items = []
|
59
|
+
new_list = @list.clone
|
60
|
+
|
61
|
+
@list.each_with_index do |proxy, key|
|
62
|
+
new_list.shift
|
63
|
+
|
64
|
+
if !check_connection || self.class.connectable?(proxy)
|
65
|
+
new_list << proxy
|
66
|
+
|
67
|
+
if count == 1
|
68
|
+
items = proxy
|
69
|
+
break
|
70
|
+
else
|
71
|
+
items << proxy
|
72
|
+
break if items.size == count
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
@list = new_list
|
78
|
+
|
79
|
+
raise 'There are no available proxy' if items.empty?
|
80
|
+
|
81
|
+
save_to_file(@list_file, @list) if @list_file
|
82
|
+
|
83
|
+
items
|
84
|
+
end
|
85
|
+
|
86
|
+
def save_to_file(file, list)
|
87
|
+
source = ''
|
88
|
+
list.each { |p| source << "#{p[0]}:#{p[1]}\n" }
|
89
|
+
|
90
|
+
IO.write(file, source.sub(/\n$/, ''))
|
91
|
+
end
|
92
|
+
end
|
93
|
+
end
|
data/lib/proxy_manager.rb
CHANGED
@@ -1,25 +1,8 @@
|
|
1
|
-
require 'proxy_manager/
|
1
|
+
require 'proxy_manager/proxy'
|
2
2
|
require 'net/ping'
|
3
3
|
|
4
4
|
module ProxyManager
|
5
|
-
# Define gem's root path
|
6
|
-
# @return [String] string path
|
7
5
|
def self.root
|
8
6
|
File.expand_path '../..', __FILE__
|
9
7
|
end
|
10
|
-
|
11
|
-
# Create main object
|
12
|
-
# @param proxies [Array, String] array of proxies or file with proxies
|
13
|
-
# @param bad_proxies [String, nil] optional file for save bad proxies
|
14
|
-
# @example
|
15
|
-
# # from array
|
16
|
-
# proxy = ProxyManager.load(['1.2.3.4:567', '9.8.7.6:543'])
|
17
|
-
#
|
18
|
-
# # or from file
|
19
|
-
# proxy = ProxyManager.load('proxies.txt', 'bad_proxies.txt')
|
20
|
-
# @return [Class] Main object
|
21
|
-
# @see Main
|
22
|
-
def self.load(proxies, bad_proxies = nil)
|
23
|
-
Main.new(proxies, bad_proxies)
|
24
|
-
end
|
25
8
|
end
|
data/proxy_manager.gemspec
CHANGED
@@ -3,22 +3,20 @@ $:.push File.expand_path("../lib", __FILE__)
|
|
3
3
|
# Describe your gem and declare its dependencies:
|
4
4
|
Gem::Specification.new do |s|
|
5
5
|
s.name = "proxy_manager"
|
6
|
-
s.version = '0.0
|
6
|
+
s.version = '1.0.0'
|
7
7
|
s.authors = ["Kirill Platonov"]
|
8
8
|
s.licenses = ['MIT']
|
9
9
|
s.email = ["platonov.kd@gmail.com"]
|
10
10
|
s.homepage = "https://github.com/bloodyhistory/proxy_manager"
|
11
|
-
s.summary = "
|
11
|
+
s.summary = "This is gem for easy usage proxy in your parsers/web-bots."
|
12
12
|
s.description = <<-DESCRIPTION
|
13
|
-
This gem
|
14
|
-
your proxy list and check availability.
|
13
|
+
This is gem for easy usage proxy in your parsers/web-bots.
|
14
|
+
It will manage your proxy list and check availability if you need it.
|
15
15
|
DESCRIPTION
|
16
16
|
|
17
17
|
s.files = `git ls-files`.split("\n") - %w[.gitignore .travis.yml]
|
18
18
|
s.test_files = Dir["spec/**/*"]
|
19
19
|
|
20
|
-
s.add_dependency 'net-ping', '~> 1.7', '>= 1.7.2'
|
21
|
-
|
22
20
|
s.add_development_dependency 'turn', '~> 0.9', '>= 0.9.7'
|
23
21
|
s.add_development_dependency 'rspec', '~> 2.14', '>= 2.14.1'
|
24
22
|
s.add_development_dependency 'guard-rspec', '~> 4.2', '>= 4.2.8'
|
@@ -0,0 +1,60 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'proxy_manager'
|
3
|
+
|
4
|
+
describe ProxyManager::Proxy do
|
5
|
+
let(:list) { [['127.0.0.1', 80], ['127.0.0.1', 8080]] }
|
6
|
+
let(:proxies_file) { File.join(ProxyManager.root,
|
7
|
+
'spec',
|
8
|
+
'support',
|
9
|
+
'proxies.txt'
|
10
|
+
) }
|
11
|
+
subject { ProxyManager::Proxy.new(['127.0.0.1:80', '127.0.0.1:8080']) }
|
12
|
+
|
13
|
+
context '#get' do
|
14
|
+
its(:get) { should be_a Array }
|
15
|
+
|
16
|
+
it 'should return many proxies' do
|
17
|
+
expect { subject.get(3) }.not_to raise_error
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
context '#get!' do
|
22
|
+
it { should respond_to :get! }
|
23
|
+
end
|
24
|
+
|
25
|
+
context '#list' do
|
26
|
+
its(:list) { should match_array list }
|
27
|
+
end
|
28
|
+
|
29
|
+
context '::connectable' do
|
30
|
+
it 'should receive array' do
|
31
|
+
expect(ProxyManager::Proxy.connectable?(['127.0.0.1', 8080])).to be_false
|
32
|
+
end
|
33
|
+
|
34
|
+
it 'should receive string' do
|
35
|
+
expect(ProxyManager::Proxy.connectable?("127.0.0.1:8080")).to be_false
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
context 'when load from file' do
|
40
|
+
subject { ProxyManager::Proxy.new(proxies_file) }
|
41
|
+
|
42
|
+
its(:list) { should match_array list }
|
43
|
+
|
44
|
+
context 'and save file' do
|
45
|
+
let(:source) { IO.read(proxies_file) }
|
46
|
+
|
47
|
+
it 'should update proxies source' do
|
48
|
+
subject.get(2)
|
49
|
+
|
50
|
+
source = ''
|
51
|
+
subject.list.each { |p| source << "#{p[0]}:#{p[1]}\n" }
|
52
|
+
source.sub!(/\n$/, '')
|
53
|
+
|
54
|
+
expect(IO.read(proxies_file)).to eq(source)
|
55
|
+
end
|
56
|
+
|
57
|
+
after { IO.write(proxies_file, source) }
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
data/spec/spec_helper.rb
CHANGED
@@ -1,9 +1,6 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
# loaded once.
|
5
|
-
#
|
6
|
-
# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
|
1
|
+
require "codeclimate-test-reporter"
|
2
|
+
CodeClimate::TestReporter.start
|
3
|
+
|
7
4
|
RSpec.configure do |config|
|
8
5
|
config.treat_symbols_as_metadata_keys_with_true_values = true
|
9
6
|
config.run_all_when_everything_filtered = true
|
data/spec/support/proxies.txt
CHANGED
@@ -1,3 +1,2 @@
|
|
1
|
-
0.0.1
|
2
|
-
|
3
|
-
0.0.0.0:7000
|
1
|
+
127.0.0.1:80
|
2
|
+
127.0.0.1:8080
|
metadata
CHANGED
@@ -1,35 +1,15 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: proxy_manager
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0
|
4
|
+
version: 1.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Kirill Platonov
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-
|
11
|
+
date: 2014-05-10 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
|
-
- !ruby/object:Gem::Dependency
|
14
|
-
name: net-ping
|
15
|
-
requirement: !ruby/object:Gem::Requirement
|
16
|
-
requirements:
|
17
|
-
- - "~>"
|
18
|
-
- !ruby/object:Gem::Version
|
19
|
-
version: '1.7'
|
20
|
-
- - ">="
|
21
|
-
- !ruby/object:Gem::Version
|
22
|
-
version: 1.7.2
|
23
|
-
type: :runtime
|
24
|
-
prerelease: false
|
25
|
-
version_requirements: !ruby/object:Gem::Requirement
|
26
|
-
requirements:
|
27
|
-
- - "~>"
|
28
|
-
- !ruby/object:Gem::Version
|
29
|
-
version: '1.7'
|
30
|
-
- - ">="
|
31
|
-
- !ruby/object:Gem::Version
|
32
|
-
version: 1.7.2
|
33
13
|
- !ruby/object:Gem::Dependency
|
34
14
|
name: turn
|
35
15
|
requirement: !ruby/object:Gem::Requirement
|
@@ -151,8 +131,8 @@ dependencies:
|
|
151
131
|
- !ruby/object:Gem::Version
|
152
132
|
version: 0.8.7.4
|
153
133
|
description: |2
|
154
|
-
This gem
|
155
|
-
your proxy list and check availability.
|
134
|
+
This is gem for easy usage proxy in your parsers/web-bots.
|
135
|
+
It will manage your proxy list and check availability if you need it.
|
156
136
|
email:
|
157
137
|
- platonov.kd@gmail.com
|
158
138
|
executables: []
|
@@ -162,34 +142,16 @@ files:
|
|
162
142
|
- ".rspec"
|
163
143
|
- ".yardopts"
|
164
144
|
- Gemfile
|
165
|
-
- Gemfile.lock
|
166
145
|
- Guardfile
|
167
146
|
- MIT-LICENSE
|
168
147
|
- README.md
|
169
148
|
- Rakefile
|
170
|
-
- doc/ProxyManager.html
|
171
|
-
- doc/ProxyManager/Main.html
|
172
|
-
- doc/_index.html
|
173
|
-
- doc/class_list.html
|
174
|
-
- doc/css/common.css
|
175
|
-
- doc/css/full_list.css
|
176
|
-
- doc/css/style.css
|
177
|
-
- doc/file.README.html
|
178
|
-
- doc/file_list.html
|
179
|
-
- doc/frames.html
|
180
|
-
- doc/index.html
|
181
|
-
- doc/js/app.js
|
182
|
-
- doc/js/full_list.js
|
183
|
-
- doc/js/jquery.js
|
184
|
-
- doc/method_list.html
|
185
|
-
- doc/top-level-namespace.html
|
186
149
|
- lib/proxy_manager.rb
|
187
|
-
- lib/proxy_manager/
|
188
|
-
- lib/tasks/proxy_manager_tasks.rake
|
150
|
+
- lib/proxy_manager/proxy.rb
|
189
151
|
- proxy_manager.gemspec
|
190
|
-
- spec/proxy_manager/
|
152
|
+
- spec/proxy_manager/proxy_spec.rb
|
153
|
+
- spec/proxy_manager_spec.rb
|
191
154
|
- spec/spec_helper.rb
|
192
|
-
- spec/support/bad_proxies.txt
|
193
155
|
- spec/support/proxies.txt
|
194
156
|
homepage: https://github.com/bloodyhistory/proxy_manager
|
195
157
|
licenses:
|
@@ -214,10 +176,10 @@ rubyforge_project:
|
|
214
176
|
rubygems_version: 2.2.2
|
215
177
|
signing_key:
|
216
178
|
specification_version: 4
|
217
|
-
summary:
|
179
|
+
summary: This is gem for easy usage proxy in your parsers/web-bots.
|
218
180
|
test_files:
|
219
|
-
- spec/proxy_manager/
|
181
|
+
- spec/proxy_manager/proxy_spec.rb
|
182
|
+
- spec/proxy_manager_spec.rb
|
220
183
|
- spec/spec_helper.rb
|
221
|
-
- spec/support/bad_proxies.txt
|
222
184
|
- spec/support/proxies.txt
|
223
185
|
has_rdoc:
|