proxy_pool 0.1.3 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.ruby-version +1 -1
- data/Gemfile +3 -1
- data/Gemfile.lock +20 -15
- data/Rakefile +8 -6
- data/bin/console +4 -3
- data/lib/proxy_pool.rb +31 -18
- data/lib/proxy_pool/dealer.rb +43 -36
- data/lib/proxy_pool/version.rb +3 -1
- data/proxy_pool.gemspec +20 -18
- metadata +9 -9
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b683e7a0cfd44068402a78d7944ebd8753c74ca3e513ef800ab764a4d5f8612e
|
4
|
+
data.tar.gz: 4fbe67c2a42da997920db4f4e8d908a2f37f5d6be33e7e68b53e025a18d6584d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ee5a0612c6f1833115dfa06ac3ea58d66452b305c07fa317e29fbbe134727da3415cc06e99ea200188f72ffca8f3e1391d67574e33a4ef456febdd33d0120c05
|
7
|
+
data.tar.gz: 6282dfc302b006c8e2454e04d7bedf7bf715be3c4bfdc2398ec81f2cb446f5e8090a3da98a92651f42b54928bf6aa057fbcc4aee215ed768232f543d90a01f57
|
data/.ruby-version
CHANGED
@@ -1 +1 @@
|
|
1
|
-
ruby-2.
|
1
|
+
ruby-2.7.0
|
data/Gemfile
CHANGED
data/Gemfile.lock
CHANGED
@@ -1,31 +1,36 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
proxy_pool (0.
|
5
|
-
http (~> 4.
|
4
|
+
proxy_pool (0.2.0)
|
5
|
+
http (~> 4.4, >= 4.4.1)
|
6
6
|
|
7
7
|
GEM
|
8
8
|
remote: https://rubygems.org/
|
9
9
|
specs:
|
10
|
-
addressable (2.
|
11
|
-
public_suffix (>= 2.0.2, <
|
10
|
+
addressable (2.7.0)
|
11
|
+
public_suffix (>= 2.0.2, < 5.0)
|
12
12
|
domain_name (0.5.20190701)
|
13
13
|
unf (>= 0.0.5, < 1.0.0)
|
14
|
-
|
14
|
+
ffi (1.12.2)
|
15
|
+
ffi-compiler (1.0.1)
|
16
|
+
ffi (>= 1.0.0)
|
17
|
+
rake
|
18
|
+
http (4.4.1)
|
15
19
|
addressable (~> 2.3)
|
16
20
|
http-cookie (~> 1.0)
|
17
|
-
http-form_data (~> 2.
|
18
|
-
|
21
|
+
http-form_data (~> 2.2)
|
22
|
+
http-parser (~> 1.2.0)
|
19
23
|
http-cookie (1.0.3)
|
20
24
|
domain_name (~> 0.5)
|
21
|
-
http-form_data (2.
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
25
|
+
http-form_data (2.3.0)
|
26
|
+
http-parser (1.2.1)
|
27
|
+
ffi-compiler (>= 1.0, < 2.0)
|
28
|
+
minitest (5.14.1)
|
29
|
+
public_suffix (4.0.5)
|
30
|
+
rake (10.5.0)
|
26
31
|
unf (0.1.4)
|
27
32
|
unf_ext
|
28
|
-
unf_ext (0.0.7.
|
33
|
+
unf_ext (0.0.7.7)
|
29
34
|
|
30
35
|
PLATFORMS
|
31
36
|
ruby
|
@@ -34,7 +39,7 @@ DEPENDENCIES
|
|
34
39
|
bundler (~> 2.0)
|
35
40
|
minitest (~> 5.0)
|
36
41
|
proxy_pool!
|
37
|
-
rake (~>
|
42
|
+
rake (~> 10.0)
|
38
43
|
|
39
44
|
BUNDLED WITH
|
40
|
-
2.
|
45
|
+
2.1.4
|
data/Rakefile
CHANGED
@@ -1,10 +1,12 @@
|
|
1
|
-
|
2
|
-
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'bundler/gem_tasks'
|
4
|
+
require 'rake/testtask'
|
3
5
|
|
4
6
|
Rake::TestTask.new(:test) do |t|
|
5
|
-
t.libs <<
|
6
|
-
t.libs <<
|
7
|
-
t.test_files = FileList[
|
7
|
+
t.libs << 'test'
|
8
|
+
t.libs << 'lib'
|
9
|
+
t.test_files = FileList['test/**/*_test.rb']
|
8
10
|
end
|
9
11
|
|
10
|
-
task :
|
12
|
+
task default: :test
|
data/bin/console
CHANGED
@@ -1,7 +1,8 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
|
+
# frozen_string_literal: true
|
2
3
|
|
3
|
-
require
|
4
|
-
require
|
4
|
+
require 'bundler/setup'
|
5
|
+
require 'proxy_pool'
|
5
6
|
|
6
7
|
# You can add fixtures and/or initialization code here to make experimenting
|
7
8
|
# with your gem easier. You can also use a different console, if you like.
|
@@ -10,5 +11,5 @@ require "proxy_pool"
|
|
10
11
|
# require "pry"
|
11
12
|
# Pry.start
|
12
13
|
|
13
|
-
require
|
14
|
+
require 'irb'
|
14
15
|
IRB.start(__FILE__)
|
data/lib/proxy_pool.rb
CHANGED
@@ -1,47 +1,60 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module ProxyPool
|
2
4
|
#
|
3
5
|
# Errors
|
4
6
|
#
|
5
7
|
|
6
8
|
# Base error
|
7
|
-
class Error < StandardError
|
9
|
+
class Error < StandardError
|
8
10
|
end
|
9
11
|
|
10
12
|
require 'proxy_pool/version'
|
11
13
|
require 'proxy_pool/dealer'
|
12
14
|
|
13
15
|
class << self
|
14
|
-
# Get
|
16
|
+
# Get a proxy from proxy pool
|
15
17
|
#
|
16
|
-
# @param filter [Hash] Filter
|
17
18
|
# @return [Hash] Proxy
|
18
|
-
def
|
19
|
-
ProxyPool::Dealer.instance.get(
|
19
|
+
def get(&block)
|
20
|
+
ProxyPool::Dealer.instance.get(&block)
|
20
21
|
end
|
21
22
|
|
22
|
-
#
|
23
|
+
# Get high anonymous proxy
|
23
24
|
#
|
24
|
-
# @param filter [Hash] Filter
|
25
25
|
# @return [Hash] Proxy
|
26
|
-
def
|
27
|
-
|
26
|
+
def get_high_anonymous_proxy
|
27
|
+
get { |proxy| proxy['anonymity'] == 'high_anonymous' }
|
28
28
|
end
|
29
29
|
|
30
|
-
# Get
|
30
|
+
# Get proxy by country
|
31
31
|
#
|
32
|
-
# @param
|
32
|
+
# @param cn [String] Country code
|
33
33
|
# @return [Hash] Proxy
|
34
|
-
def
|
35
|
-
|
34
|
+
def get_by_country(cn)
|
35
|
+
get { |proxy| proxy['country'].downcase == cn.downcase }
|
36
36
|
end
|
37
37
|
|
38
|
-
#
|
38
|
+
# Get http proxy
|
39
39
|
#
|
40
|
-
# @param filter [Hash] Filter
|
41
40
|
# @return [Hash] Proxy
|
42
|
-
def
|
43
|
-
|
41
|
+
def get_http_proxy
|
42
|
+
get { |proxy| proxy['type'] == 'http' }
|
44
43
|
end
|
45
|
-
end
|
46
44
|
|
45
|
+
# Get https proxy
|
46
|
+
#
|
47
|
+
# @return [Hash] Proxy
|
48
|
+
def get_https_proxy
|
49
|
+
get { |proxy| proxy['type'] == 'https' }
|
50
|
+
end
|
51
|
+
|
52
|
+
# Remove this proxy from pool
|
53
|
+
#
|
54
|
+
# @param proxy [Hash] Proxy
|
55
|
+
# @return [nil]
|
56
|
+
def remove(proxy)
|
57
|
+
ProxyPool::Dealer.instance.remove(proxy)
|
58
|
+
end
|
59
|
+
end
|
47
60
|
end
|
data/lib/proxy_pool/dealer.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
#
|
2
4
|
# Standard library
|
3
5
|
#
|
@@ -17,72 +19,77 @@ module ProxyPool
|
|
17
19
|
include Singleton
|
18
20
|
|
19
21
|
# HTTP error
|
20
|
-
class HTTPError < Error
|
22
|
+
class HTTPError < Error
|
21
23
|
end
|
22
24
|
|
23
25
|
# Parse error
|
24
|
-
class ParseError < Error
|
25
|
-
end
|
26
|
-
|
27
|
-
# Filter error
|
28
|
-
class FilterError < Error;
|
26
|
+
class ParseError < Error
|
29
27
|
end
|
30
28
|
|
31
|
-
attr_reader :
|
29
|
+
attr_reader :pools
|
32
30
|
|
33
31
|
# Update to latest proxy list from fate0/proxylist
|
34
32
|
#
|
35
33
|
def update
|
36
|
-
@
|
37
|
-
@anonymous_pools = []
|
34
|
+
@pools = []
|
38
35
|
|
39
36
|
res = HTTP.get 'https://raw.githubusercontent.com/fate0/proxylist/master/proxy.list'
|
40
37
|
raise HTTPError, "invalid http code #{res.code}" if res.code != 200
|
41
38
|
|
42
|
-
|
39
|
+
@updated_at = Time.now
|
40
|
+
|
41
|
+
res.body.to_s.split("\n").each { |line| _pool_parse(line) }
|
43
42
|
end
|
44
43
|
|
45
44
|
# Get a random proxy
|
46
45
|
#
|
47
|
-
# @param anonymous [Boolean] Return high anonymous proxy if true
|
48
|
-
# @param filter [Hash] Filter
|
49
|
-
# @param force_update [Boolean] Update pool before select proxy server
|
50
46
|
# @return [Hash] Proxy
|
51
|
-
def get
|
52
|
-
update if
|
47
|
+
def get
|
48
|
+
update if _need_update?
|
53
49
|
|
54
|
-
target_pools = if
|
55
|
-
@
|
50
|
+
target_pools = if block_given?
|
51
|
+
@pools.select { |proxy| yield proxy }
|
56
52
|
else
|
57
|
-
@
|
53
|
+
@pools
|
58
54
|
end
|
59
|
-
filter.each_pair { |k, v| target_pools = _pool_filter(target_pools, k.to_s, v) }
|
60
|
-
|
61
55
|
target_pools.sample
|
62
56
|
end
|
63
57
|
|
58
|
+
# Remove this proxy from pool
|
59
|
+
#
|
60
|
+
# @param proxy [Hash] Proxy
|
61
|
+
# @return [nil]
|
62
|
+
def remove(proxy)
|
63
|
+
@pools.delete(proxy)
|
64
|
+
|
65
|
+
nil
|
66
|
+
end
|
67
|
+
|
64
68
|
private
|
65
69
|
|
70
|
+
# Return true if we need to update proxy list
|
71
|
+
#
|
72
|
+
# @return [Boolean]
|
73
|
+
def _need_update?
|
74
|
+
return true if @updated_at.nil?
|
75
|
+
|
76
|
+
# Re-update after 10 min
|
77
|
+
delta = Time.now - @updated_at
|
78
|
+
delta > 600
|
79
|
+
end
|
80
|
+
|
81
|
+
# Parse proxy list
|
82
|
+
#
|
83
|
+
# @param line [String]
|
66
84
|
def _pool_parse(line)
|
67
85
|
proxy = JSON.parse(line)
|
68
|
-
|
69
|
-
|
70
|
-
case proxy['anonymity']
|
71
|
-
when 'high_anonymous'
|
72
|
-
@anonymous_pools << proxy
|
73
|
-
else
|
74
|
-
@transparent_pools << proxy
|
86
|
+
unless proxy.key?('anonymity')
|
87
|
+
raise ParseError, "no anonymity field: #{line}"
|
75
88
|
end
|
89
|
+
|
90
|
+
@pools << proxy
|
76
91
|
rescue JSON::ParserError
|
77
92
|
raise ParseError, "JSON parser error when parsing #{line}"
|
78
93
|
end
|
79
|
-
|
80
|
-
def _pool_filter(proxy_pools, key, value)
|
81
|
-
proxy_pools.select do |proxy|
|
82
|
-
raise FilterError, "invalid filter: #{key}" unless proxy.key?(key)
|
83
|
-
proxy[key] == value
|
84
|
-
end
|
85
|
-
end
|
86
|
-
|
87
94
|
end
|
88
|
-
end
|
95
|
+
end
|
data/lib/proxy_pool/version.rb
CHANGED
data/proxy_pool.gemspec
CHANGED
@@ -1,35 +1,37 @@
|
|
1
|
-
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
lib = File.expand_path('lib', __dir__)
|
2
4
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
3
|
-
require
|
5
|
+
require 'proxy_pool/version'
|
4
6
|
|
5
7
|
Gem::Specification.new do |spec|
|
6
|
-
spec.name =
|
8
|
+
spec.name = 'proxy_pool'
|
7
9
|
spec.version = ProxyPool::VERSION
|
8
|
-
spec.authors = [
|
9
|
-
spec.email = [
|
10
|
+
spec.authors = ['hi_ztz']
|
11
|
+
spec.email = ['hi_ztz@protonmail.com']
|
10
12
|
|
11
|
-
spec.summary =
|
12
|
-
spec.description =
|
13
|
-
spec.homepage =
|
14
|
-
spec.license =
|
13
|
+
spec.summary = 'select a ip from proxy list randomly'
|
14
|
+
spec.description = 'proxy_pool can give you free and random proxy ip'
|
15
|
+
spec.homepage = 'https://github.com/zt2/proxy_pool'
|
16
|
+
spec.license = 'MIT'
|
15
17
|
|
16
|
-
spec.metadata[
|
18
|
+
spec.metadata['allowed_push_host'] = 'https://rubygems.org'
|
17
19
|
|
18
|
-
spec.metadata[
|
19
|
-
spec.metadata[
|
20
|
+
spec.metadata['homepage_uri'] = spec.homepage
|
21
|
+
spec.metadata['source_code_uri'] = spec.homepage
|
20
22
|
|
21
23
|
# Specify which files should be added to the gem when it is released.
|
22
24
|
# The `git ls-files -z` loads the files in the RubyGem that have been added into git.
|
23
|
-
spec.files
|
25
|
+
spec.files = Dir.chdir(File.expand_path(__dir__)) do
|
24
26
|
`git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
|
25
27
|
end
|
26
|
-
spec.bindir =
|
28
|
+
spec.bindir = 'exe'
|
27
29
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
28
|
-
spec.require_paths = [
|
30
|
+
spec.require_paths = ['lib']
|
29
31
|
|
30
|
-
spec.add_development_dependency
|
31
|
-
spec.add_development_dependency
|
32
|
-
spec.add_development_dependency
|
32
|
+
spec.add_development_dependency 'bundler', '~> 2.0'
|
33
|
+
spec.add_development_dependency 'minitest', '~> 5.0'
|
34
|
+
spec.add_development_dependency 'rake', '~> 10.0'
|
33
35
|
|
34
36
|
spec.add_runtime_dependency 'http', '~> 4.4', '>= 4.4.1'
|
35
37
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: proxy_pool
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- hi_ztz
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-05-20 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -25,33 +25,33 @@ dependencies:
|
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: '2.0'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
|
-
name:
|
28
|
+
name: minitest
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
31
|
- - "~>"
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version: '
|
33
|
+
version: '5.0'
|
34
34
|
type: :development
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
38
|
- - "~>"
|
39
39
|
- !ruby/object:Gem::Version
|
40
|
-
version: '
|
40
|
+
version: '5.0'
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
|
-
name:
|
42
|
+
name: rake
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
45
|
- - "~>"
|
46
46
|
- !ruby/object:Gem::Version
|
47
|
-
version: '
|
47
|
+
version: '10.0'
|
48
48
|
type: :development
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
52
|
- - "~>"
|
53
53
|
- !ruby/object:Gem::Version
|
54
|
-
version: '
|
54
|
+
version: '10.0'
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
56
|
name: http
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
@@ -116,7 +116,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
116
116
|
- !ruby/object:Gem::Version
|
117
117
|
version: '0'
|
118
118
|
requirements: []
|
119
|
-
rubygems_version: 3.
|
119
|
+
rubygems_version: 3.1.2
|
120
120
|
signing_key:
|
121
121
|
specification_version: 4
|
122
122
|
summary: select a ip from proxy list randomly
|