zold 0.18.6 → 0.18.7
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/zold/cached_wallets.rb +3 -1
- data/lib/zold/commands/fetch.rb +3 -2
- data/lib/zold/commands/push.rb +3 -2
- data/lib/zold/commands/remote.rb +11 -4
- data/lib/zold/commands/routines/gc.rb +1 -1
- data/lib/zold/endless.rb +0 -1
- data/lib/zold/http.rb +1 -1
- data/lib/zold/metronome.rb +10 -22
- data/lib/zold/node/async_entrance.rb +7 -8
- data/lib/zold/node/farm.rb +11 -20
- data/lib/zold/node/spread_entrance.rb +4 -5
- data/lib/zold/remotes.rb +23 -30
- data/lib/zold/thread_pool.rb +132 -0
- data/lib/zold/version.rb +1 -1
- data/test/commands/test_remote.rb +2 -0
- data/test/test__helper.rb +1 -0
- data/test/test_remotes.rb +35 -2
- data/test/test_thread_pool.rb +98 -0
- data/zold.gemspec +0 -1
- metadata +6 -17
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e805665d05301f84e70c8e280530b3a9b7cd195b63165a1a3eefda7eb72f509f
|
4
|
+
data.tar.gz: 42113e608d1c76f7b3ed5ace35ff36456566b708a9779e6c52e866a6c189416d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d7e2b7606454ca38aac815c9c43a280d753845ab8301ac4e0055894a640e6f7c81185c8c23445d776875c215b7d614c8abcaa118133eaab41dc81778f0d44bb4
|
7
|
+
data.tar.gz: 1223bc3bee736ccc949ae1f663f25286498b07f1595370a870c65a65ca036477e6800b021caca6b44bf118f3424a3b611d7560c3bf8858c2e034be11f964e5d8
|
data/lib/zold/cached_wallets.rb
CHANGED
@@ -22,6 +22,7 @@
|
|
22
22
|
|
23
23
|
require 'zache'
|
24
24
|
require_relative 'endless'
|
25
|
+
require_relative 'thread_pool'
|
25
26
|
|
26
27
|
# Cached collection of wallets.
|
27
28
|
# Author:: Yegor Bugayenko (yegor256@gmail.com)
|
@@ -33,7 +34,8 @@ module Zold
|
|
33
34
|
def initialize(wallets)
|
34
35
|
@wallets = wallets
|
35
36
|
@zache = Zache.new
|
36
|
-
@clean =
|
37
|
+
@clean = ThreadPool.new('cached-wallets')
|
38
|
+
@clean.add do
|
37
39
|
Endless.new('cached_wallets').run do
|
38
40
|
sleep 60
|
39
41
|
@zache.clean
|
data/lib/zold/commands/fetch.rb
CHANGED
@@ -32,6 +32,7 @@ require 'concurrent'
|
|
32
32
|
require 'parallel'
|
33
33
|
require_relative 'thread_badge'
|
34
34
|
require_relative 'args'
|
35
|
+
require_relative '../thread_pool'
|
35
36
|
require_relative '../log'
|
36
37
|
require_relative '../age'
|
37
38
|
require_relative '../http'
|
@@ -77,9 +78,9 @@ Available options:"
|
|
77
78
|
o.bool '--help', 'Print instructions'
|
78
79
|
end
|
79
80
|
mine = Args.new(opts, @log).take || return
|
80
|
-
|
81
|
+
list = mine.empty? ? @wallets.all : mine.map { |i| Id.new(i) }
|
82
|
+
ThreadPool.new('fetch', log: @log).run(opts['threads'], list) do |id|
|
81
83
|
fetch(id, Copies.new(File.join(@copies, id)), opts)
|
82
|
-
@log.debug("Worker: #{Parallel.worker_number} has fetched wallet #{id}")
|
83
84
|
end
|
84
85
|
end
|
85
86
|
|
data/lib/zold/commands/push.rb
CHANGED
@@ -28,6 +28,7 @@ require 'concurrent'
|
|
28
28
|
require 'parallel'
|
29
29
|
require_relative 'thread_badge'
|
30
30
|
require_relative 'args'
|
31
|
+
require_relative '../thread_pool'
|
31
32
|
require_relative '../age'
|
32
33
|
require_relative '../size'
|
33
34
|
require_relative '../log'
|
@@ -66,9 +67,9 @@ Available options:"
|
|
66
67
|
o.bool '--help', 'Print instructions'
|
67
68
|
end
|
68
69
|
mine = Args.new(opts, @log).take || return
|
69
|
-
|
70
|
+
list = mine.empty? ? @wallets.all : mine.map { |i| Id.new(i) }
|
71
|
+
ThreadPool.new('push', log: @log).run(opts['threads'], list) do |id|
|
70
72
|
push(id, opts)
|
71
|
-
@log.debug("Worker: #{Parallel.worker_number} has pushed wallet #{id}")
|
72
73
|
end
|
73
74
|
end
|
74
75
|
|
data/lib/zold/commands/remote.rb
CHANGED
@@ -229,7 +229,7 @@ Available options:"
|
|
229
229
|
else
|
230
230
|
scores.each { |s| @log.info("Elected: #{s.reduced(4)}") }
|
231
231
|
end
|
232
|
-
scores
|
232
|
+
scores.sort_by(&:value).reverse
|
233
233
|
end
|
234
234
|
|
235
235
|
def trim(opts)
|
@@ -257,15 +257,22 @@ Available options:"
|
|
257
257
|
r.assert_score_ownership(score)
|
258
258
|
r.assert_score_strength(score) unless opts['ignore-score-weakness']
|
259
259
|
@remotes.rescore(score.host, score.port, score.value)
|
260
|
-
|
261
|
-
if Semantic::Version.new(VERSION) < Semantic::Version.new(json['version']) ||
|
262
|
-
Semantic::Version.new(VERSION) < Semantic::Version.new(gem.last_version)
|
260
|
+
if Semantic::Version.new(VERSION) < Semantic::Version.new(json['version'])
|
263
261
|
if opts['reboot']
|
264
262
|
@log.info("#{r}: their version #{json['version']} is higher than mine #{VERSION}, reboot! \
|
265
263
|
(use --never-reboot to avoid this from happening)")
|
266
264
|
terminate
|
267
265
|
end
|
268
266
|
@log.debug("#{r}: their version #{json['version']} is higher than mine #{VERSION}, \
|
267
|
+
it's recommended to reboot, but I don't do it because of --never-reboot")
|
268
|
+
end
|
269
|
+
if Semantic::Version.new(VERSION) < Semantic::Version.new(Zold::Gem.new.last_version)
|
270
|
+
if opts['reboot']
|
271
|
+
@log.info("#{r}: the version of the gem is higher than mine #{VERSION}, reboot! \
|
272
|
+
(use --never-reboot to avoid this from happening)")
|
273
|
+
terminate
|
274
|
+
end
|
275
|
+
@log.debug("#{r}: gem version is higher than mine #{VERSION}, \
|
269
276
|
it's recommended to reboot, but I don't do it because of --never-reboot")
|
270
277
|
end
|
271
278
|
if cycle.positive?
|
data/lib/zold/endless.rb
CHANGED
data/lib/zold/http.rb
CHANGED
data/lib/zold/metronome.rb
CHANGED
@@ -24,6 +24,7 @@ require 'backtrace'
|
|
24
24
|
require_relative 'log'
|
25
25
|
require_relative 'age'
|
26
26
|
require_relative 'verbose_thread'
|
27
|
+
require_relative 'thread_pool'
|
27
28
|
|
28
29
|
# Background routines.
|
29
30
|
# Author:: Yegor Bugayenko (yegor256@gmail.com)
|
@@ -35,8 +36,7 @@ module Zold
|
|
35
36
|
def initialize(log = Log::NULL)
|
36
37
|
@log = log
|
37
38
|
@routines = []
|
38
|
-
@threads =
|
39
|
-
@starts = {}
|
39
|
+
@threads = ThreadPool.new('metronome', log: log)
|
40
40
|
@failures = {}
|
41
41
|
end
|
42
42
|
|
@@ -44,13 +44,7 @@ module Zold
|
|
44
44
|
[
|
45
45
|
Time.now.utc.iso8601,
|
46
46
|
'Current threads:',
|
47
|
-
@threads.
|
48
|
-
[
|
49
|
-
"#{t.name}: status=#{t.status}; alive=#{t.alive?}",
|
50
|
-
"Most recent start: #{Age.new(@starts[t])} ago",
|
51
|
-
t.backtrace.nil? ? 'NO BACKTRACE' : " #{t.backtrace.join("\n ")}"
|
52
|
-
].join("\n")
|
53
|
-
end,
|
47
|
+
@threads.to_s,
|
54
48
|
'Failures:',
|
55
49
|
@failures.map { |r, f| "#{r}\n#{f}\n" }
|
56
50
|
].flatten.join("\n\n")
|
@@ -63,18 +57,19 @@ module Zold
|
|
63
57
|
|
64
58
|
def start
|
65
59
|
@routines.each_with_index do |r, idx|
|
66
|
-
@threads
|
67
|
-
Thread.current.abort_on_exception = true
|
60
|
+
@threads.add do
|
68
61
|
Thread.current.name = "#{r.class.name}-#{idx}"
|
69
62
|
step = 0
|
70
63
|
loop do
|
71
|
-
|
64
|
+
Thread.current.thread_variable_set(:start, Time.now)
|
72
65
|
begin
|
73
66
|
r.exec(step)
|
74
|
-
@log.debug("Routine #{r.class.name} ##{step} done
|
67
|
+
@log.debug("Routine #{r.class.name} ##{step} done \
|
68
|
+
in #{Age.new(Thread.current.thread_variable_get(:start))}")
|
75
69
|
rescue StandardError => e
|
76
70
|
@failures[r.class.name] = Time.now.utc.iso8601 + "\n" + Backtrace.new(e).to_s
|
77
|
-
@log.error("Routine #{r.class.name} ##{step} failed
|
71
|
+
@log.error("Routine #{r.class.name} ##{step} failed \
|
72
|
+
in #{Age.new(Thread.current.thread_variable_get(:start))}")
|
78
73
|
@log.error(Backtrace.new(e).to_s)
|
79
74
|
end
|
80
75
|
step += 1
|
@@ -85,14 +80,7 @@ module Zold
|
|
85
80
|
begin
|
86
81
|
yield(self)
|
87
82
|
ensure
|
88
|
-
|
89
|
-
unless @threads.empty?
|
90
|
-
@log.info("Stopping the metronome with #{@threads.count} threads...")
|
91
|
-
@threads.each(&:kill)
|
92
|
-
@threads.each(&:join)
|
93
|
-
@log.info("Metronome #{@threads.count} threads killed")
|
94
|
-
end
|
95
|
-
@log.info("Metronome stopped in #{Age.new(start)}, #{@failures.count} failures")
|
83
|
+
@threads.kill
|
96
84
|
end
|
97
85
|
end
|
98
86
|
end
|
@@ -27,6 +27,7 @@ require_relative '../age'
|
|
27
27
|
require_relative '../size'
|
28
28
|
require_relative '../id'
|
29
29
|
require_relative '../endless'
|
30
|
+
require_relative '../thread_pool'
|
30
31
|
require_relative '../dir_items'
|
31
32
|
require_relative 'soft_error'
|
32
33
|
|
@@ -42,7 +43,8 @@ module Zold
|
|
42
43
|
@entrance = entrance
|
43
44
|
@dir = File.expand_path(dir)
|
44
45
|
@log = log
|
45
|
-
@
|
46
|
+
@threads = threads
|
47
|
+
@pool = ThreadPool.new('async-entrance', log: log)
|
46
48
|
@queue = Queue.new
|
47
49
|
@queue_limit = queue_limit
|
48
50
|
end
|
@@ -50,7 +52,7 @@ module Zold
|
|
50
52
|
def to_json
|
51
53
|
@entrance.to_json.merge(
|
52
54
|
'queue': @queue.size,
|
53
|
-
'threads': @
|
55
|
+
'threads': @pool.count,
|
54
56
|
'queue_limit': @queue_limit
|
55
57
|
)
|
56
58
|
end
|
@@ -65,8 +67,8 @@ module Zold
|
|
65
67
|
end
|
66
68
|
@log.info("#{@queue.size} wallets pre-loaded into async_entrace from #{@dir}") unless @queue.size.zero?
|
67
69
|
@entrance.start do
|
68
|
-
|
69
|
-
|
70
|
+
(0..@threads).map do |i|
|
71
|
+
@pool.add do
|
70
72
|
Endless.new("async-e##{i}", log: @log).run do
|
71
73
|
take
|
72
74
|
end
|
@@ -75,10 +77,7 @@ module Zold
|
|
75
77
|
begin
|
76
78
|
yield(self)
|
77
79
|
ensure
|
78
|
-
@
|
79
|
-
@threads.each(&:kill)
|
80
|
-
@threads.each(&:join)
|
81
|
-
@log.info("AsyncEntrance stopped, #{@threads.count} threads killed")
|
80
|
+
@pool.kill
|
82
81
|
end
|
83
82
|
end
|
84
83
|
end
|
data/lib/zold/node/farm.rb
CHANGED
@@ -28,6 +28,7 @@ require 'concurrent'
|
|
28
28
|
require 'json'
|
29
29
|
require 'zold/score'
|
30
30
|
require_relative '../log'
|
31
|
+
require_relative '../thread_pool'
|
31
32
|
require_relative '../age'
|
32
33
|
require_relative '../endless'
|
33
34
|
require_relative 'farmers'
|
@@ -64,7 +65,7 @@ module Zold
|
|
64
65
|
@invoice = invoice
|
65
66
|
@pipeline = Queue.new
|
66
67
|
@farmer = farmer
|
67
|
-
@threads =
|
68
|
+
@threads = ThreadPool.new('farm')
|
68
69
|
@lifetime = lifetime
|
69
70
|
@strength = strength
|
70
71
|
end
|
@@ -82,21 +83,14 @@ module Zold
|
|
82
83
|
"Current time: #{Time.now.utc.iso8601}",
|
83
84
|
"Ruby processes: #{`ps ax | grep zold | wc -l`}",
|
84
85
|
JSON.pretty_generate(to_json),
|
85
|
-
@threads.
|
86
|
-
trace = t.backtrace || []
|
87
|
-
[
|
88
|
-
"#{t.name}: status=#{t.status}; alive=#{t.alive?}",
|
89
|
-
'Vars: ' + t.thread_variables.map { |v| "#{v}=\"#{t.thread_variable_get(v)}\"" }.join('; '),
|
90
|
-
" #{trace.join("\n ")}"
|
91
|
-
].join("\n")
|
92
|
-
end
|
86
|
+
@threads.to_s
|
93
87
|
].flatten.join("\n\n")
|
94
88
|
end
|
95
89
|
|
96
90
|
# Renders the Farm into JSON to show for the end-user in front.rb.
|
97
91
|
def to_json
|
98
92
|
{
|
99
|
-
threads: @threads.
|
93
|
+
threads: @threads.to_json,
|
100
94
|
pipeline: @pipeline.size,
|
101
95
|
best: best.map(&:to_mnemo).join(', '),
|
102
96
|
farmer: @farmer.class.name
|
@@ -121,8 +115,8 @@ module Zold
|
|
121
115
|
else
|
122
116
|
@log.info("#{best.size} scores pre-loaded from #{@cache}, the best is: #{best[0]}")
|
123
117
|
end
|
124
|
-
|
125
|
-
|
118
|
+
(1..threads).map do |t|
|
119
|
+
@threads.add do
|
126
120
|
Thread.current.thread_variable_set(:tid, t.to_s)
|
127
121
|
Endless.new("f#{t}", log: @log).run do
|
128
122
|
cycle(host, port, threads)
|
@@ -131,7 +125,7 @@ module Zold
|
|
131
125
|
end
|
132
126
|
unless threads.zero?
|
133
127
|
ready = false
|
134
|
-
@threads
|
128
|
+
@threads.add do
|
135
129
|
Endless.new('cleanup', log: @log).run do
|
136
130
|
cleanup(host, port, threads)
|
137
131
|
ready = true
|
@@ -140,7 +134,7 @@ module Zold
|
|
140
134
|
end
|
141
135
|
loop { break if ready }
|
142
136
|
end
|
143
|
-
if
|
137
|
+
if threads.zero?
|
144
138
|
cleanup(host, port, threads)
|
145
139
|
@log.info("Farm started with no threads (there will be no score) at #{host}:#{port}")
|
146
140
|
else
|
@@ -150,10 +144,7 @@ at #{host}:#{port}, strength is #{@strength}")
|
|
150
144
|
begin
|
151
145
|
yield(self)
|
152
146
|
ensure
|
153
|
-
@
|
154
|
-
@threads.each(&:kill)
|
155
|
-
@threads.each(&:join)
|
156
|
-
@log.info("Farm stopped, #{threads} threads killed")
|
147
|
+
@threads.kill
|
157
148
|
end
|
158
149
|
end
|
159
150
|
|
@@ -164,11 +155,11 @@ at #{host}:#{port}, strength is #{@strength}")
|
|
164
155
|
before = scores.map(&:value).max.to_i
|
165
156
|
save(threads, [Score.new(host: host, port: port, invoice: @invoice, strength: @strength)])
|
166
157
|
scores = load
|
167
|
-
free = scores.reject { |s| @threads.
|
158
|
+
free = scores.reject { |s| @threads.exists?(s.to_mnemo) }
|
168
159
|
@pipeline << free[0] if @pipeline.size.zero? && !free.empty?
|
169
160
|
after = scores.map(&:value).max.to_i
|
170
161
|
return unless before != after && !after.zero?
|
171
|
-
@log.debug("#{Thread.current.name}: best score of #{scores.count} is #{scores[0]}")
|
162
|
+
@log.debug("#{Thread.current.name}: best score of #{scores.count} is #{scores[0].reduced(4)}")
|
172
163
|
end
|
173
164
|
|
174
165
|
def cycle(host, port, threads)
|
@@ -28,6 +28,7 @@ require_relative '../remotes'
|
|
28
28
|
require_relative '../copies'
|
29
29
|
require_relative '../endless'
|
30
30
|
require_relative '../tax'
|
31
|
+
require_relative '../thread_pool'
|
31
32
|
require_relative '../commands/merge'
|
32
33
|
require_relative '../commands/fetch'
|
33
34
|
require_relative '../commands/push'
|
@@ -48,12 +49,13 @@ module Zold
|
|
48
49
|
@log = log
|
49
50
|
@ignore_score_weakeness = ignore_score_weakeness
|
50
51
|
@mutex = Mutex.new
|
52
|
+
@push = ThreadPool.new('spread-entrance')
|
51
53
|
end
|
52
54
|
|
53
55
|
def to_json
|
54
56
|
@entrance.to_json.merge(
|
55
57
|
'modified': @modified.size,
|
56
|
-
'push': @push.
|
58
|
+
'push': @push.to_json
|
57
59
|
)
|
58
60
|
end
|
59
61
|
|
@@ -62,7 +64,7 @@ module Zold
|
|
62
64
|
@entrance.start do
|
63
65
|
@seen = Set.new
|
64
66
|
@modified = Queue.new
|
65
|
-
@push
|
67
|
+
@push.add do
|
66
68
|
Endless.new('push', log: @log).run do
|
67
69
|
id = @modified.pop
|
68
70
|
if @remotes.all.empty?
|
@@ -82,11 +84,8 @@ module Zold
|
|
82
84
|
begin
|
83
85
|
yield(self)
|
84
86
|
ensure
|
85
|
-
@log.info('Waiting for spread entrance to finish...')
|
86
87
|
@modified.clear
|
87
88
|
@push.kill
|
88
|
-
@push.join
|
89
|
-
@log.info('Spread entrance finished, thread killed')
|
90
89
|
end
|
91
90
|
end
|
92
91
|
end
|
data/lib/zold/remotes.rb
CHANGED
@@ -31,6 +31,7 @@ require 'backtrace'
|
|
31
31
|
require 'zold/score'
|
32
32
|
require_relative 'age'
|
33
33
|
require_relative 'http'
|
34
|
+
require_relative 'thread_pool'
|
34
35
|
require_relative 'node/farm'
|
35
36
|
|
36
37
|
# The list of remotes.
|
@@ -94,7 +95,7 @@ module Zold
|
|
94
95
|
def assert_code(code, response)
|
95
96
|
msg = response.status_line.strip
|
96
97
|
return if response.status.to_i == code
|
97
|
-
if response.headers['X-Zold-Error']
|
98
|
+
if response.headers && response.headers['X-Zold-Error']
|
98
99
|
raise "Error ##{response.status} \"#{response.headers['X-Zold-Error']}\"
|
99
100
|
at #{response.headers['X-Zold-Path']}"
|
100
101
|
end
|
@@ -182,40 +183,32 @@ module Zold
|
|
182
183
|
end
|
183
184
|
end
|
184
185
|
|
186
|
+
# Go through the list of remotes and call a provided block for each
|
187
|
+
# of them. See how it's used, for example, in fetch.rb.
|
185
188
|
def iterate(log, farm: Farm::Empty.new)
|
186
189
|
raise 'Log can\'t be nil' if log.nil?
|
187
190
|
raise 'Farm can\'t be nil' if farm.nil?
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
)
|
208
|
-
raise 'Took too long to execute' if (Time.now - start).round > @timeout
|
209
|
-
rescue StandardError => e
|
210
|
-
error(r[:host], r[:port])
|
211
|
-
log.info("#{Rainbow("#{r[:host]}:#{r[:port]}").red}: #{e.message} in #{Age.new(start)}")
|
212
|
-
log.debug(Backtrace.new(e).to_s)
|
213
|
-
remove(r[:host], r[:port]) if errors > TOLERANCE
|
214
|
-
end
|
191
|
+
ThreadPool.new('remotes', log: log).run(Concurrent.processor_count * 4, all) do |r, idx|
|
192
|
+
Thread.current.name = "remotes-#{idx}@#{r[:host]}:#{r[:port]}"
|
193
|
+
start = Time.now
|
194
|
+
best = farm.best[0]
|
195
|
+
begin
|
196
|
+
yield Remotes::Remote.new(
|
197
|
+
host: r[:host],
|
198
|
+
port: r[:port],
|
199
|
+
score: best.nil? ? Score::ZERO : best,
|
200
|
+
idx: idx,
|
201
|
+
log: log,
|
202
|
+
network: @network
|
203
|
+
)
|
204
|
+
raise 'Took too long to execute' if (Time.now - start).round > @timeout
|
205
|
+
rescue StandardError => e
|
206
|
+
error(r[:host], r[:port])
|
207
|
+
log.info("#{Rainbow("#{r[:host]}:#{r[:port]}").red}: #{e.message} in #{Age.new(start)}")
|
208
|
+
log.debug(Backtrace.new(e).to_s)
|
209
|
+
remove(r[:host], r[:port]) if r[:errors] > TOLERANCE
|
215
210
|
end
|
216
211
|
end
|
217
|
-
pool.shutdown
|
218
|
-
pool.kill unless pool.wait_for_termination(5 * 60)
|
219
212
|
end
|
220
213
|
|
221
214
|
def error(host, port = PORT)
|
@@ -0,0 +1,132 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# Copyright (c) 2018 Yegor Bugayenko
|
4
|
+
#
|
5
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
# of this software and associated documentation files (the 'Software'), to deal
|
7
|
+
# in the Software without restriction, including without limitation the rights
|
8
|
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
# copies of the Software, and to permit persons to whom the Software is
|
10
|
+
# furnished to do so, subject to the following conditions:
|
11
|
+
#
|
12
|
+
# The above copyright notice and this permission notice shall be included in all
|
13
|
+
# copies or substantial portions of the Software.
|
14
|
+
#
|
15
|
+
# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
21
|
+
# SOFTWARE.
|
22
|
+
|
23
|
+
require 'concurrent'
|
24
|
+
require_relative 'age'
|
25
|
+
require_relative 'verbose_thread'
|
26
|
+
|
27
|
+
# Thread pool.
|
28
|
+
# Author:: Yegor Bugayenko (yegor256@gmail.com)
|
29
|
+
# Copyright:: Copyright (c) 2018 Yegor Bugayenko
|
30
|
+
# License:: MIT
|
31
|
+
module Zold
|
32
|
+
# Thread pool
|
33
|
+
class ThreadPool
|
34
|
+
def initialize(title, log: Log::NULL)
|
35
|
+
@title = title
|
36
|
+
@log = log
|
37
|
+
@threads = []
|
38
|
+
@start = Time.now
|
39
|
+
end
|
40
|
+
|
41
|
+
# Run this code in many threads
|
42
|
+
def run(threads, set = (0..threads - 1).to_a)
|
43
|
+
raise "Number of threads #{threads} has to be positive" unless threads.positive?
|
44
|
+
raise 'Set of data can\'t be empty' if set.empty?
|
45
|
+
idx = Concurrent::AtomicFixnum.new
|
46
|
+
mutex = Mutex.new
|
47
|
+
list = set.dup
|
48
|
+
[threads, set.count].min.times do
|
49
|
+
add do
|
50
|
+
loop do
|
51
|
+
r = mutex.synchronize { list.pop }
|
52
|
+
break if r.nil?
|
53
|
+
yield(r, idx.increment - 1)
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
57
|
+
@threads.each(&:join)
|
58
|
+
@threads.clear
|
59
|
+
end
|
60
|
+
|
61
|
+
# Add a new thread
|
62
|
+
def add
|
63
|
+
raise 'Block must be given to start()' unless block_given?
|
64
|
+
latch = Concurrent::CountDownLatch.new(1)
|
65
|
+
thread = Thread.start do
|
66
|
+
VerboseThread.new(@log).run do
|
67
|
+
latch.count_down
|
68
|
+
yield
|
69
|
+
end
|
70
|
+
end
|
71
|
+
latch.wait
|
72
|
+
Thread.current.thread_variable_set(
|
73
|
+
:kids,
|
74
|
+
(Thread.current.thread_variable_get(:kids) || []) + [thread]
|
75
|
+
)
|
76
|
+
@threads << thread
|
77
|
+
end
|
78
|
+
|
79
|
+
# Kill them all immediately and close the pool
|
80
|
+
def kill
|
81
|
+
if @threads.empty?
|
82
|
+
@log.debug("Thread pool \"#{@title}\" terminated with no threads")
|
83
|
+
return
|
84
|
+
end
|
85
|
+
@log.info("Stopping \"#{@title}\" thread pool with #{@threads.count} threads: \
|
86
|
+
#{@threads.map { |t| "#{t.name}/#{t.status}" }.join(', ')}...")
|
87
|
+
start = Time.new
|
88
|
+
@threads.each do |t|
|
89
|
+
(t.thread_variable_get(:kids) || []).each(&:kill)
|
90
|
+
t.kill
|
91
|
+
raise "Failed to join the thread \"#{t.name}\" in \"#{@title}\" pool" unless t.join(0.1)
|
92
|
+
(Thread.current.thread_variable_get(:kids) || []).delete(t)
|
93
|
+
end
|
94
|
+
@log.info("Thread pool \"#{@title}\" terminated all threads in #{Age.new(start)}, \
|
95
|
+
it was alive for #{Age.new(@start)}: #{@threads.map { |t| "#{t.name}/#{t.status}" }.join(', ')}")
|
96
|
+
@threads.clear
|
97
|
+
end
|
98
|
+
|
99
|
+
# How many threads are in there
|
100
|
+
def count
|
101
|
+
@threads.count
|
102
|
+
end
|
103
|
+
|
104
|
+
# A thread with this name exists?
|
105
|
+
def exists?(name)
|
106
|
+
!@threads.find { |t| t.name == name }.nil?
|
107
|
+
end
|
108
|
+
|
109
|
+
# As a hash map
|
110
|
+
def to_json
|
111
|
+
@threads.map do |t|
|
112
|
+
{
|
113
|
+
name: t.name,
|
114
|
+
status: t.status,
|
115
|
+
alive: t.alive?,
|
116
|
+
vars: t.thread_variables.map { |v| { k: v, v: t.thread_variable_get(v) } }
|
117
|
+
}
|
118
|
+
end
|
119
|
+
end
|
120
|
+
|
121
|
+
# As a text
|
122
|
+
def to_s
|
123
|
+
@threads.map do |t|
|
124
|
+
[
|
125
|
+
"#{t.name}: status=#{t.status}; alive=#{t.alive?}",
|
126
|
+
'Vars: ' + t.thread_variables.map { |v| "#{v}=\"#{t.thread_variable_get(v)}\"" }.join('; '),
|
127
|
+
t.backtrace.nil? ? 'NO BACKTRACE' : " #{t.backtrace.join("\n ")}"
|
128
|
+
].join("\n")
|
129
|
+
end
|
130
|
+
end
|
131
|
+
end
|
132
|
+
end
|
data/lib/zold/version.rb
CHANGED
@@ -43,6 +43,7 @@ class TestRemote < Zold::Test
|
|
43
43
|
zero = Zold::Score::ZERO
|
44
44
|
stub_request(:get, "http://#{zero.host}:#{zero.port}/remotes").to_return(
|
45
45
|
status: 200,
|
46
|
+
headers: {},
|
46
47
|
body: {
|
47
48
|
version: Zold::VERSION,
|
48
49
|
score: zero.to_h,
|
@@ -63,6 +64,7 @@ class TestRemote < Zold::Test
|
|
63
64
|
)
|
64
65
|
stub_request(:get, 'https://rubygems.org/api/v1/versions/zold/latest.json').to_return(
|
65
66
|
status: 200,
|
67
|
+
headers: {},
|
66
68
|
body: '{"version": "0.0.0"}'
|
67
69
|
)
|
68
70
|
cmd = Zold::Remote.new(remotes: remotes, log: test_log)
|
data/test/test__helper.rb
CHANGED
data/test/test_remotes.rb
CHANGED
@@ -82,13 +82,46 @@ class TestRemotes < Zold::Test
|
|
82
82
|
end
|
83
83
|
end
|
84
84
|
|
85
|
-
def
|
85
|
+
def test_iterates_all_failures
|
86
|
+
Dir.mktmpdir do |dir|
|
87
|
+
file = File.join(dir, 'remotes')
|
88
|
+
FileUtils.touch(file)
|
89
|
+
remotes = Zold::Remotes.new(file: file)
|
90
|
+
5.times { |i| remotes.add("0.0.0.#{i}", 9999) }
|
91
|
+
total = 0
|
92
|
+
remotes.iterate(Zold::Log::NULL) do
|
93
|
+
total += 1
|
94
|
+
raise 'Intended'
|
95
|
+
end
|
96
|
+
assert_equal(5, total)
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
def test_closes_threads_carefully
|
101
|
+
Dir.mktmpdir do |dir|
|
102
|
+
file = File.join(dir, 'remotes')
|
103
|
+
FileUtils.touch(file)
|
104
|
+
remotes = Zold::Remotes.new(file: file)
|
105
|
+
5.times { |i| remotes.add("0.0.0.#{i}", 9999) }
|
106
|
+
total = 0
|
107
|
+
remotes.iterate(Zold::Log::NULL) do
|
108
|
+
sleep 0.25
|
109
|
+
total += 1
|
110
|
+
end
|
111
|
+
assert_equal(5, total)
|
112
|
+
end
|
113
|
+
end
|
114
|
+
|
115
|
+
def test_iterates_them_all_even_with_delays
|
86
116
|
Dir.mktmpdir do |dir|
|
87
117
|
remotes = Zold::Remotes.new(file: File.join(dir, 'rrr.csv'))
|
88
118
|
remotes.clean
|
89
119
|
5.times { |i| remotes.add("0.0.0.#{i}", 8080) }
|
90
120
|
total = 0
|
91
|
-
remotes.iterate(test_log)
|
121
|
+
remotes.iterate(test_log) do
|
122
|
+
sleep 0.25
|
123
|
+
total += 1
|
124
|
+
end
|
92
125
|
assert_equal(5, total)
|
93
126
|
end
|
94
127
|
end
|
@@ -0,0 +1,98 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# Copyright (c) 2018 Yegor Bugayenko
|
4
|
+
#
|
5
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
# of this software and associated documentation files (the 'Software'), to deal
|
7
|
+
# in the Software without restriction, including without limitation the rights
|
8
|
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
# copies of the Software, and to permit persons to whom the Software is
|
10
|
+
# furnished to do so, subject to the following conditions:
|
11
|
+
#
|
12
|
+
# The above copyright notice and this permission notice shall be included in all
|
13
|
+
# copies or substantial portions of the Software.
|
14
|
+
#
|
15
|
+
# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
21
|
+
# SOFTWARE.
|
22
|
+
|
23
|
+
require 'minitest/autorun'
|
24
|
+
require 'concurrent'
|
25
|
+
require_relative 'test__helper'
|
26
|
+
require_relative '../lib/zold/thread_pool'
|
27
|
+
|
28
|
+
# ThreadPool test.
|
29
|
+
# Author:: Yegor Bugayenko (yegor256@gmail.com)
|
30
|
+
# Copyright:: Copyright (c) 2018 Yegor Bugayenko
|
31
|
+
# License:: MIT
|
32
|
+
class TestThreadPool < Zold::Test
|
33
|
+
def test_closes_all_threads_right
|
34
|
+
pool = Zold::ThreadPool.new('test', log: test_log)
|
35
|
+
idx = Concurrent::AtomicFixnum.new
|
36
|
+
threads = 50
|
37
|
+
threads.times do
|
38
|
+
pool.add do
|
39
|
+
idx.increment
|
40
|
+
end
|
41
|
+
end
|
42
|
+
pool.kill
|
43
|
+
assert_equal(threads, idx.value)
|
44
|
+
end
|
45
|
+
|
46
|
+
def test_runs_in_many_threads
|
47
|
+
idx = Concurrent::AtomicFixnum.new
|
48
|
+
threads = 50
|
49
|
+
Zold::ThreadPool.new('test', log: test_log).run(threads) do
|
50
|
+
idx.increment
|
51
|
+
end
|
52
|
+
assert_equal(threads, idx.value)
|
53
|
+
end
|
54
|
+
|
55
|
+
def test_runs_with_index
|
56
|
+
idx = Concurrent::AtomicFixnum.new
|
57
|
+
indexes = Set.new
|
58
|
+
Zold::ThreadPool.new('test', log: test_log).run(10, %w[a b c]) do |_, i|
|
59
|
+
idx.increment
|
60
|
+
indexes << i
|
61
|
+
end
|
62
|
+
assert_equal(3, idx.value)
|
63
|
+
assert_equal('0 1 2', indexes.to_a.sort.join(' '))
|
64
|
+
end
|
65
|
+
|
66
|
+
def test_adds_and_stops
|
67
|
+
pool = Zold::ThreadPool.new('test', log: test_log)
|
68
|
+
pool.add do
|
69
|
+
sleep 60 * 60
|
70
|
+
end
|
71
|
+
pool.kill
|
72
|
+
end
|
73
|
+
|
74
|
+
def test_stops_stuck_threads
|
75
|
+
pool = Zold::ThreadPool.new('test', log: test_log)
|
76
|
+
pool.add do
|
77
|
+
loop do
|
78
|
+
# forever
|
79
|
+
end
|
80
|
+
end
|
81
|
+
pool.kill
|
82
|
+
end
|
83
|
+
|
84
|
+
def test_stops_empty_pool
|
85
|
+
pool = Zold::ThreadPool.new('test', log: test_log)
|
86
|
+
pool.kill
|
87
|
+
end
|
88
|
+
|
89
|
+
def test_prints_to_json
|
90
|
+
pool = Zold::ThreadPool.new('test', log: test_log)
|
91
|
+
assert(pool.to_json.is_a?(Array))
|
92
|
+
end
|
93
|
+
|
94
|
+
def test_prints_to_text
|
95
|
+
pool = Zold::ThreadPool.new('test', log: test_log)
|
96
|
+
assert(!pool.to_s.nil?)
|
97
|
+
end
|
98
|
+
end
|
data/zold.gemspec
CHANGED
@@ -70,7 +70,6 @@ and suggests a different architecture for digital wallet maintenance.'
|
|
70
70
|
s.add_runtime_dependency 'json', '2.1.0'
|
71
71
|
s.add_runtime_dependency 'memory_profiler', '0.9.12'
|
72
72
|
s.add_runtime_dependency 'openssl', '2.1.2'
|
73
|
-
s.add_runtime_dependency 'parallel', '1.12.1'
|
74
73
|
s.add_runtime_dependency 'rainbow', '3.0.0'
|
75
74
|
s.add_runtime_dependency 'rake', '12.3.1' # has to stay here for Heroku
|
76
75
|
s.add_runtime_dependency 'rubocop', '0.60.0' # has to stay here for Heroku
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: zold
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.18.
|
4
|
+
version: 0.18.7
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Yegor Bugayenko
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-12-
|
11
|
+
date: 2018-12-13 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: backtrace
|
@@ -136,20 +136,6 @@ dependencies:
|
|
136
136
|
- - '='
|
137
137
|
- !ruby/object:Gem::Version
|
138
138
|
version: 2.1.2
|
139
|
-
- !ruby/object:Gem::Dependency
|
140
|
-
name: parallel
|
141
|
-
requirement: !ruby/object:Gem::Requirement
|
142
|
-
requirements:
|
143
|
-
- - '='
|
144
|
-
- !ruby/object:Gem::Version
|
145
|
-
version: 1.12.1
|
146
|
-
type: :runtime
|
147
|
-
prerelease: false
|
148
|
-
version_requirements: !ruby/object:Gem::Requirement
|
149
|
-
requirements:
|
150
|
-
- - '='
|
151
|
-
- !ruby/object:Gem::Version
|
152
|
-
version: 1.12.1
|
153
139
|
- !ruby/object:Gem::Dependency
|
154
140
|
name: rainbow
|
155
141
|
requirement: !ruby/object:Gem::Requirement
|
@@ -614,6 +600,7 @@ files:
|
|
614
600
|
- lib/zold/size.rb
|
615
601
|
- lib/zold/sync_wallets.rb
|
616
602
|
- lib/zold/tax.rb
|
603
|
+
- lib/zold/thread_pool.rb
|
617
604
|
- lib/zold/tree_wallets.rb
|
618
605
|
- lib/zold/txn.rb
|
619
606
|
- lib/zold/txns.rb
|
@@ -681,6 +668,7 @@ files:
|
|
681
668
|
- test/test_size.rb
|
682
669
|
- test/test_sync_wallets.rb
|
683
670
|
- test/test_tax.rb
|
671
|
+
- test/test_thread_pool.rb
|
684
672
|
- test/test_tree_wallets.rb
|
685
673
|
- test/test_txn.rb
|
686
674
|
- test/test_upgrades.rb
|
@@ -701,7 +689,7 @@ licenses:
|
|
701
689
|
- MIT
|
702
690
|
metadata: {}
|
703
691
|
post_install_message: |-
|
704
|
-
Thanks for installing Zold 0.18.
|
692
|
+
Thanks for installing Zold 0.18.7!
|
705
693
|
Study our White Paper: https://papers.zold.io/wp.pdf
|
706
694
|
Read our blog posts: https://blog.zold.io
|
707
695
|
Try online wallet at: https://wts.zold.io
|
@@ -789,6 +777,7 @@ test_files:
|
|
789
777
|
- test/test_size.rb
|
790
778
|
- test/test_sync_wallets.rb
|
791
779
|
- test/test_tax.rb
|
780
|
+
- test/test_thread_pool.rb
|
792
781
|
- test/test_tree_wallets.rb
|
793
782
|
- test/test_txn.rb
|
794
783
|
- test/test_upgrades.rb
|