proxied 0.2.1 → 0.2.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +2 -1
- data/Appraisals +9 -2
- data/Gemfile +1 -1
- data/Gemfile.lock +142 -120
- data/README.md +10 -4
- data/bin/console +1 -1
- data/lib/generators/active_record/templates/migration.rb +14 -10
- data/lib/generators/mongoid/proxied_generator.rb +4 -0
- data/lib/generators/templates/proxied.rb +21 -4
- data/lib/proxied/checker.rb +78 -40
- data/lib/proxied/configuration.rb +20 -14
- data/lib/proxied/importer.rb +13 -10
- data/lib/proxied/logger.rb +13 -3
- data/lib/proxied/nosql/proxy_methods.rb +7 -6
- data/lib/proxied/shared.rb +22 -11
- data/lib/proxied/sql/proxy_methods.rb +10 -9
- data/lib/proxied/utilities.rb +20 -27
- data/lib/proxied/version.rb +1 -1
- data/lib/tasks/tasks.rake +4 -4
- data/proxied.gemspec +16 -16
- metadata +33 -36
- data/gemfiles/.bundle/config +0 -2
- data/gemfiles/activerecord5.gemfile +0 -11
- data/gemfiles/mongoid7.gemfile +0 -9
@@ -11,27 +11,28 @@ module Proxied
|
|
11
11
|
end
|
12
12
|
|
13
13
|
module ClassMethods
|
14
|
-
def should_be_checked(protocol: :all, proxy_type: :all, date: Time.now, limit:
|
15
|
-
proxies =
|
14
|
+
def should_be_checked(mode: :synchronous, protocol: :all, proxy_type: :all, category: nil, country: nil, date: Time.now, limit: nil, maximum_failed_attempts: 10)
|
15
|
+
proxies = get_proxies(protocol: protocol, proxy_type: proxy_type, category: category, country: country)
|
16
|
+
proxies = proxies.where(checkable: true)
|
17
|
+
proxies = proxies.where(asyncable: true) if mode.to_sym.eql?(:asynchronous)
|
16
18
|
proxies = proxies.where(["(last_checked_at IS NULL OR last_checked_at < ?)", date])
|
17
19
|
proxies = proxies.where(["failed_attempts <= ?", maximum_failed_attempts])
|
18
20
|
proxies = proxies.order("valid_proxy ASC, failed_attempts ASC, last_checked_at ASC")
|
19
|
-
proxies = proxies.limit(limit)
|
21
|
+
proxies = proxies.limit(limit) if limit && !limit.zero?
|
20
22
|
|
21
23
|
return proxies
|
22
24
|
end
|
23
25
|
|
24
|
-
def get_valid_proxies(protocol: :all, proxy_type: :all, category: nil, maximum_failed_attempts: nil)
|
25
|
-
proxies =
|
26
|
+
def get_valid_proxies(protocol: :all, proxy_type: :all, category: nil, country: nil, maximum_failed_attempts: nil)
|
27
|
+
proxies = get_proxies(protocol: protocol, proxy_type: proxy_type, category: category, country: country)
|
26
28
|
proxies = proxies.where(["valid_proxy = ? AND last_checked_at IS NOT NULL", true])
|
27
29
|
proxies = proxies.where(["failed_attempts <= ?", maximum_failed_attempts]) if maximum_failed_attempts
|
28
|
-
proxies = proxies.where(category: category) unless category.to_s.empty?
|
29
30
|
|
30
31
|
return proxies
|
31
32
|
end
|
32
33
|
|
33
|
-
def get_random_proxy(protocol: :all, proxy_type: :all, category: nil, maximum_failed_attempts: nil)
|
34
|
-
proxies = get_valid_proxies(protocol: protocol, proxy_type: proxy_type, category: category, maximum_failed_attempts: maximum_failed_attempts)
|
34
|
+
def get_random_proxy(protocol: :all, proxy_type: :all, category: nil, country: nil, maximum_failed_attempts: nil)
|
35
|
+
proxies = get_valid_proxies(protocol: protocol, proxy_type: proxy_type, category: category, country: country, maximum_failed_attempts: maximum_failed_attempts)
|
35
36
|
proxies = yield(proxies) if block_given?
|
36
37
|
|
37
38
|
order_clause = case ActiveRecord::Base.connection.class.name
|
@@ -45,7 +46,7 @@ module Proxied
|
|
45
46
|
"RAND() DESC"
|
46
47
|
end
|
47
48
|
|
48
|
-
proxies = proxies.order(order_clause)
|
49
|
+
proxies = proxies.order(Arel.sql(order_clause))
|
49
50
|
|
50
51
|
proxy = nil
|
51
52
|
|
data/lib/proxied/utilities.rb
CHANGED
@@ -1,17 +1,22 @@
|
|
1
1
|
module Proxied
|
2
2
|
class Utilities
|
3
|
-
|
4
3
|
class << self
|
5
4
|
|
6
|
-
def format_proxy_address(host:, port: 80,
|
5
|
+
def format_proxy_address(host:, port: 80, username: nil, password: nil, protocol: 'http', include_protocol: false)
|
6
|
+
protocol ||= 'http'
|
7
7
|
address = "#{host.strip}:#{port}"
|
8
|
-
address = "
|
8
|
+
address = "#{format_proxy_credentials(username, password)}@#{address}" if !username.to_s.empty? && !password.to_s.empty?
|
9
|
+
address = include_protocol && address !~ /^http(s)?/i ? "#{protocol}://#{address}" : address
|
9
10
|
|
10
11
|
return address
|
11
12
|
end
|
12
13
|
|
13
14
|
def format_proxy_credentials(username, password)
|
14
|
-
return "#{username}:#{password}"
|
15
|
+
return "#{encode_credential(username)}:#{encode_credential(password)}"
|
16
|
+
end
|
17
|
+
|
18
|
+
def encode_credential(credential)
|
19
|
+
CGI::escape(credential)
|
15
20
|
end
|
16
21
|
|
17
22
|
def socks_proxy_credentials(username: nil, password: nil)
|
@@ -23,34 +28,22 @@ module Proxied
|
|
23
28
|
return credentials
|
24
29
|
end
|
25
30
|
|
26
|
-
def proxy_options_for_faraday(host:, port: 80, username: nil, password: nil)
|
31
|
+
def proxy_options_for_faraday(host:, port: 80, username: nil, password: nil, auth_mode: :credentials, protocol: 'http')
|
27
32
|
proxy_options = {}
|
28
|
-
|
29
|
-
|
30
|
-
proxy_options[:user] = username if !username.to_s.empty?
|
31
|
-
proxy_options[:password] = password if !password.to_s.empty?
|
32
|
-
|
33
|
-
return proxy_options
|
34
|
-
end
|
35
|
-
|
36
|
-
def proxy_switcher_import_format(host:, port: 80, username: nil, password: nil, country: nil)
|
37
|
-
# Format:
|
38
|
-
# COUNTRY_CODE,COUNTRY_NAME,CITY,USERNAME:PASSWORD@IP_ADDRESS:PORT
|
39
|
-
|
40
|
-
address = ""
|
41
|
-
|
42
|
-
if !country.to_s.empty?
|
43
|
-
address += "#{country.upcase},,,"
|
44
|
-
end
|
33
|
+
username = encode_credential(username)
|
34
|
+
password = encode_credential(password)
|
45
35
|
|
46
|
-
|
47
|
-
|
48
|
-
|
36
|
+
options = {host: host, port: port, protocol: protocol, include_protocol: true}
|
37
|
+
options[:username] = username if !username.to_s.empty? && auth_mode&.to_sym&.eql?(:url)
|
38
|
+
options[:password] = password if !password.to_s.empty? && auth_mode&.to_sym&.eql?(:url)
|
49
39
|
|
50
|
-
|
40
|
+
proxy_options[:uri] = format_proxy_address(options)
|
41
|
+
proxy_options[:user] = username if !username.to_s.empty? && auth_mode&.to_sym&.eql?(:credentials)
|
42
|
+
proxy_options[:password] = password if !password.to_s.empty? && auth_mode&.to_sym&.eql?(:credentials)
|
43
|
+
|
44
|
+
return proxy_options
|
51
45
|
end
|
52
46
|
|
53
47
|
end
|
54
|
-
|
55
48
|
end
|
56
49
|
end
|
data/lib/proxied/version.rb
CHANGED
data/lib/tasks/tasks.rake
CHANGED
@@ -1,12 +1,12 @@
|
|
1
1
|
namespace :proxied do
|
2
|
-
task :check_proxies, [:protocol, :proxy_type, :mode, :maximum_failed_attempts] => :environment do |task, args|
|
3
|
-
args.with_defaults(protocol: :http, proxy_type: :public, mode: :synchronous, maximum_failed_attempts: 10)
|
2
|
+
task :check_proxies, [:protocol, :proxy_type, :category, :mode, :maximum_failed_attempts] => :environment do |task, args|
|
3
|
+
args.with_defaults(protocol: :http, proxy_type: :public, category: nil, mode: :synchronous, maximum_failed_attempts: 10)
|
4
4
|
|
5
5
|
checker = ::Proxied::Checker.new(mode: args.mode.to_sym, maximum_failed_attempts: args.maximum_failed_attempts.to_i)
|
6
6
|
checker.check_proxies(
|
7
7
|
protocol: args.protocol.to_sym,
|
8
|
-
proxy_type: args.proxy_type.to_sym,
|
9
|
-
|
8
|
+
proxy_type: args.proxy_type.to_sym,
|
9
|
+
category: args.category
|
10
10
|
)
|
11
11
|
end
|
12
12
|
end
|
data/proxied.gemspec
CHANGED
@@ -23,25 +23,25 @@ Gem::Specification.new do |spec|
|
|
23
23
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
24
24
|
spec.require_paths = ["lib"]
|
25
25
|
|
26
|
-
spec.add_dependency
|
27
|
-
spec.add_dependency
|
26
|
+
spec.add_dependency 'faraday', '>= 1.0'
|
27
|
+
spec.add_dependency 'net-ssh', '>= 6.0'
|
28
28
|
|
29
|
-
spec.add_development_dependency
|
30
|
-
spec.add_development_dependency
|
31
|
-
spec.add_development_dependency
|
29
|
+
spec.add_development_dependency 'bundler', '>= 2.0'
|
30
|
+
spec.add_development_dependency 'rake', '>= 13.0'
|
31
|
+
spec.add_development_dependency 'appraisal', '>= 2.3'
|
32
32
|
|
33
|
-
spec.add_development_dependency
|
33
|
+
spec.add_development_dependency 'virtus', '>= 1.0'
|
34
34
|
|
35
|
-
spec.add_development_dependency
|
36
|
-
spec.add_development_dependency
|
37
|
-
spec.add_development_dependency
|
38
|
-
spec.add_development_dependency
|
39
|
-
spec.add_development_dependency
|
40
|
-
spec.add_development_dependency
|
35
|
+
spec.add_development_dependency 'rspec-rails', '>= 4.0'
|
36
|
+
spec.add_development_dependency 'rspec-core', '>= 3.10'
|
37
|
+
spec.add_development_dependency 'rspec-expectations', '>= 3.10'
|
38
|
+
spec.add_development_dependency 'rspec-mocks', '>= 3.10'
|
39
|
+
spec.add_development_dependency 'rspec-support', '>= 3.10'
|
40
|
+
spec.add_development_dependency 'generator_spec', '>= 0.9'
|
41
41
|
|
42
|
-
spec.add_development_dependency
|
43
|
-
spec.add_development_dependency
|
44
|
-
spec.add_development_dependency
|
42
|
+
spec.add_development_dependency 'rdoc', '>= 6.3'
|
43
|
+
spec.add_development_dependency 'vcr', '>= 6.0'
|
44
|
+
spec.add_development_dependency 'webmock', '>= 3.11'
|
45
45
|
|
46
|
-
spec.add_development_dependency
|
46
|
+
spec.add_development_dependency 'pry', '>= 0.13'
|
47
47
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: proxied
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.7
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sebastian
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2020-12-27 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: faraday
|
@@ -16,70 +16,70 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: 0
|
19
|
+
version: '1.0'
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - ">="
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: 0
|
26
|
+
version: '1.0'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: net-ssh
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
31
|
- - ">="
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version: '
|
33
|
+
version: '6.0'
|
34
34
|
type: :runtime
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
38
|
- - ">="
|
39
39
|
- !ruby/object:Gem::Version
|
40
|
-
version: '
|
40
|
+
version: '6.0'
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
42
|
name: bundler
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
45
|
- - ">="
|
46
46
|
- !ruby/object:Gem::Version
|
47
|
-
version:
|
47
|
+
version: '2.0'
|
48
48
|
type: :development
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
52
|
- - ">="
|
53
53
|
- !ruby/object:Gem::Version
|
54
|
-
version:
|
54
|
+
version: '2.0'
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
56
|
name: rake
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
58
58
|
requirements:
|
59
59
|
- - ">="
|
60
60
|
- !ruby/object:Gem::Version
|
61
|
-
version:
|
61
|
+
version: '13.0'
|
62
62
|
type: :development
|
63
63
|
prerelease: false
|
64
64
|
version_requirements: !ruby/object:Gem::Requirement
|
65
65
|
requirements:
|
66
66
|
- - ">="
|
67
67
|
- !ruby/object:Gem::Version
|
68
|
-
version:
|
68
|
+
version: '13.0'
|
69
69
|
- !ruby/object:Gem::Dependency
|
70
70
|
name: appraisal
|
71
71
|
requirement: !ruby/object:Gem::Requirement
|
72
72
|
requirements:
|
73
73
|
- - ">="
|
74
74
|
- !ruby/object:Gem::Version
|
75
|
-
version: 2.
|
75
|
+
version: '2.3'
|
76
76
|
type: :development
|
77
77
|
prerelease: false
|
78
78
|
version_requirements: !ruby/object:Gem::Requirement
|
79
79
|
requirements:
|
80
80
|
- - ">="
|
81
81
|
- !ruby/object:Gem::Version
|
82
|
-
version: 2.
|
82
|
+
version: '2.3'
|
83
83
|
- !ruby/object:Gem::Dependency
|
84
84
|
name: virtus
|
85
85
|
requirement: !ruby/object:Gem::Requirement
|
@@ -100,140 +100,140 @@ dependencies:
|
|
100
100
|
requirements:
|
101
101
|
- - ">="
|
102
102
|
- !ruby/object:Gem::Version
|
103
|
-
version:
|
103
|
+
version: '4.0'
|
104
104
|
type: :development
|
105
105
|
prerelease: false
|
106
106
|
version_requirements: !ruby/object:Gem::Requirement
|
107
107
|
requirements:
|
108
108
|
- - ">="
|
109
109
|
- !ruby/object:Gem::Version
|
110
|
-
version:
|
110
|
+
version: '4.0'
|
111
111
|
- !ruby/object:Gem::Dependency
|
112
112
|
name: rspec-core
|
113
113
|
requirement: !ruby/object:Gem::Requirement
|
114
114
|
requirements:
|
115
115
|
- - ">="
|
116
116
|
- !ruby/object:Gem::Version
|
117
|
-
version: '3.
|
117
|
+
version: '3.10'
|
118
118
|
type: :development
|
119
119
|
prerelease: false
|
120
120
|
version_requirements: !ruby/object:Gem::Requirement
|
121
121
|
requirements:
|
122
122
|
- - ">="
|
123
123
|
- !ruby/object:Gem::Version
|
124
|
-
version: '3.
|
124
|
+
version: '3.10'
|
125
125
|
- !ruby/object:Gem::Dependency
|
126
126
|
name: rspec-expectations
|
127
127
|
requirement: !ruby/object:Gem::Requirement
|
128
128
|
requirements:
|
129
129
|
- - ">="
|
130
130
|
- !ruby/object:Gem::Version
|
131
|
-
version: 3.
|
131
|
+
version: '3.10'
|
132
132
|
type: :development
|
133
133
|
prerelease: false
|
134
134
|
version_requirements: !ruby/object:Gem::Requirement
|
135
135
|
requirements:
|
136
136
|
- - ">="
|
137
137
|
- !ruby/object:Gem::Version
|
138
|
-
version: 3.
|
138
|
+
version: '3.10'
|
139
139
|
- !ruby/object:Gem::Dependency
|
140
140
|
name: rspec-mocks
|
141
141
|
requirement: !ruby/object:Gem::Requirement
|
142
142
|
requirements:
|
143
143
|
- - ">="
|
144
144
|
- !ruby/object:Gem::Version
|
145
|
-
version: '3.
|
145
|
+
version: '3.10'
|
146
146
|
type: :development
|
147
147
|
prerelease: false
|
148
148
|
version_requirements: !ruby/object:Gem::Requirement
|
149
149
|
requirements:
|
150
150
|
- - ">="
|
151
151
|
- !ruby/object:Gem::Version
|
152
|
-
version: '3.
|
152
|
+
version: '3.10'
|
153
153
|
- !ruby/object:Gem::Dependency
|
154
154
|
name: rspec-support
|
155
155
|
requirement: !ruby/object:Gem::Requirement
|
156
156
|
requirements:
|
157
157
|
- - ">="
|
158
158
|
- !ruby/object:Gem::Version
|
159
|
-
version: '3.
|
159
|
+
version: '3.10'
|
160
160
|
type: :development
|
161
161
|
prerelease: false
|
162
162
|
version_requirements: !ruby/object:Gem::Requirement
|
163
163
|
requirements:
|
164
164
|
- - ">="
|
165
165
|
- !ruby/object:Gem::Version
|
166
|
-
version: '3.
|
166
|
+
version: '3.10'
|
167
167
|
- !ruby/object:Gem::Dependency
|
168
168
|
name: generator_spec
|
169
169
|
requirement: !ruby/object:Gem::Requirement
|
170
170
|
requirements:
|
171
171
|
- - ">="
|
172
172
|
- !ruby/object:Gem::Version
|
173
|
-
version: 0.9
|
173
|
+
version: '0.9'
|
174
174
|
type: :development
|
175
175
|
prerelease: false
|
176
176
|
version_requirements: !ruby/object:Gem::Requirement
|
177
177
|
requirements:
|
178
178
|
- - ">="
|
179
179
|
- !ruby/object:Gem::Version
|
180
|
-
version: 0.9
|
180
|
+
version: '0.9'
|
181
181
|
- !ruby/object:Gem::Dependency
|
182
182
|
name: rdoc
|
183
183
|
requirement: !ruby/object:Gem::Requirement
|
184
184
|
requirements:
|
185
185
|
- - ">="
|
186
186
|
- !ruby/object:Gem::Version
|
187
|
-
version: 6.
|
187
|
+
version: '6.3'
|
188
188
|
type: :development
|
189
189
|
prerelease: false
|
190
190
|
version_requirements: !ruby/object:Gem::Requirement
|
191
191
|
requirements:
|
192
192
|
- - ">="
|
193
193
|
- !ruby/object:Gem::Version
|
194
|
-
version: 6.
|
194
|
+
version: '6.3'
|
195
195
|
- !ruby/object:Gem::Dependency
|
196
196
|
name: vcr
|
197
197
|
requirement: !ruby/object:Gem::Requirement
|
198
198
|
requirements:
|
199
199
|
- - ">="
|
200
200
|
- !ruby/object:Gem::Version
|
201
|
-
version: '
|
201
|
+
version: '6.0'
|
202
202
|
type: :development
|
203
203
|
prerelease: false
|
204
204
|
version_requirements: !ruby/object:Gem::Requirement
|
205
205
|
requirements:
|
206
206
|
- - ">="
|
207
207
|
- !ruby/object:Gem::Version
|
208
|
-
version: '
|
208
|
+
version: '6.0'
|
209
209
|
- !ruby/object:Gem::Dependency
|
210
210
|
name: webmock
|
211
211
|
requirement: !ruby/object:Gem::Requirement
|
212
212
|
requirements:
|
213
213
|
- - ">="
|
214
214
|
- !ruby/object:Gem::Version
|
215
|
-
version: 3.
|
215
|
+
version: '3.11'
|
216
216
|
type: :development
|
217
217
|
prerelease: false
|
218
218
|
version_requirements: !ruby/object:Gem::Requirement
|
219
219
|
requirements:
|
220
220
|
- - ">="
|
221
221
|
- !ruby/object:Gem::Version
|
222
|
-
version: 3.
|
222
|
+
version: '3.11'
|
223
223
|
- !ruby/object:Gem::Dependency
|
224
224
|
name: pry
|
225
225
|
requirement: !ruby/object:Gem::Requirement
|
226
226
|
requirements:
|
227
227
|
- - ">="
|
228
228
|
- !ruby/object:Gem::Version
|
229
|
-
version: 0.
|
229
|
+
version: '0.13'
|
230
230
|
type: :development
|
231
231
|
prerelease: false
|
232
232
|
version_requirements: !ruby/object:Gem::Requirement
|
233
233
|
requirements:
|
234
234
|
- - ">="
|
235
235
|
- !ruby/object:Gem::Version
|
236
|
-
version: 0.
|
236
|
+
version: '0.13'
|
237
237
|
description: Manage HTTP(S) and SOCKS proxies for Rails apps. Supports ActiveRecord
|
238
238
|
and Mongoid as ORM.
|
239
239
|
email:
|
@@ -254,9 +254,6 @@ files:
|
|
254
254
|
- Rakefile
|
255
255
|
- bin/console
|
256
256
|
- bin/setup
|
257
|
-
- gemfiles/.bundle/config
|
258
|
-
- gemfiles/activerecord5.gemfile
|
259
|
-
- gemfiles/mongoid7.gemfile
|
260
257
|
- lib/generators/active_record/proxied_generator.rb
|
261
258
|
- lib/generators/active_record/templates/migration.rb
|
262
259
|
- lib/generators/mongoid/proxied_generator.rb
|
@@ -298,7 +295,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
298
295
|
- !ruby/object:Gem::Version
|
299
296
|
version: '0'
|
300
297
|
requirements: []
|
301
|
-
rubygems_version: 3.
|
298
|
+
rubygems_version: 3.1.4
|
302
299
|
signing_key:
|
303
300
|
specification_version: 4
|
304
301
|
summary: Proxied is a proxy management gem for Ruby/Rails.
|