pwrake 2.0.1 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +22 -9
- data/bin/gfwhere-pipe +33 -9
- data/bin/pwrake +5 -2
- data/bin/pwrake_branch +5 -3
- data/lib/pwrake/branch/branch.rb +95 -86
- data/lib/pwrake/branch/branch_application.rb +4 -0
- data/lib/pwrake/branch/communicator.rb +173 -0
- data/lib/pwrake/branch/communicator_set.rb +100 -0
- data/lib/pwrake/branch/fiber_queue.rb +10 -0
- data/lib/pwrake/branch/shell.rb +68 -24
- data/lib/pwrake/branch/shell_profiler.rb +2 -0
- data/lib/pwrake/gfarm/gfarm_postprocess.rb +8 -7
- data/lib/pwrake/logger.rb +5 -0
- data/lib/pwrake/master/master.rb +190 -87
- data/lib/pwrake/master/master_application.rb +8 -0
- data/lib/pwrake/nbio.rb +525 -0
- data/lib/pwrake/option/host_map.rb +36 -4
- data/lib/pwrake/option/option.rb +7 -1
- data/lib/pwrake/option/option_filesystem.rb +13 -3
- data/lib/pwrake/queue/locality_aware_queue.rb +41 -6
- data/lib/pwrake/queue/queue_array.rb +31 -11
- data/lib/pwrake/queue/task_queue.rb +15 -18
- data/lib/pwrake/report/report.rb +2 -0
- data/lib/pwrake/task/task_algorithm.rb +4 -1
- data/lib/pwrake/task/task_manager.rb +2 -0
- data/lib/pwrake/task/task_property.rb +1 -0
- data/lib/pwrake/task/task_wrapper.rb +40 -21
- data/lib/pwrake/version.rb +1 -1
- data/lib/pwrake/worker/invoker.rb +4 -29
- data/pwrake.gemspec +3 -2
- metadata +24 -12
- data/lib/pwrake/branch.rb +0 -22
- data/lib/pwrake/branch/worker_communicator.rb +0 -104
- data/lib/pwrake/iomux/channel.rb +0 -70
- data/lib/pwrake/iomux/handler.rb +0 -124
- data/lib/pwrake/iomux/handler_set.rb +0 -35
- data/lib/pwrake/iomux/runner.rb +0 -62
- data/lib/pwrake/master.rb +0 -30
@@ -2,11 +2,11 @@ module Pwrake
|
|
2
2
|
|
3
3
|
class GfarmPostprocess
|
4
4
|
|
5
|
-
def initialize(
|
5
|
+
def initialize(selector)
|
6
6
|
@io = IO.popen('gfwhere-pipe','r+')
|
7
7
|
@io.sync = true
|
8
|
-
@
|
9
|
-
@
|
8
|
+
@reader = NBIO::Reader.new(selector,@io)
|
9
|
+
@writer = NBIO::Writer.new(selector,@io)
|
10
10
|
end
|
11
11
|
|
12
12
|
def run(task_wrap)
|
@@ -15,13 +15,12 @@ module Pwrake
|
|
15
15
|
end
|
16
16
|
filename = task_wrap.name
|
17
17
|
begin
|
18
|
-
@
|
19
|
-
@hdl.iow.flush
|
18
|
+
@writer.put_line(filename)
|
20
19
|
rescue Errno::EPIPE
|
21
20
|
Log.warn "GfarmPostprocess#run: Errno::EPIPE for #{filename}"
|
22
21
|
return []
|
23
22
|
end
|
24
|
-
s = @
|
23
|
+
s = @reader.get_line
|
25
24
|
if s.nil?
|
26
25
|
raise "gfwhere: unexpected end"
|
27
26
|
end
|
@@ -29,7 +28,7 @@ module Pwrake
|
|
29
28
|
if s != filename
|
30
29
|
raise "gfwhere: file=#{filename}, result=#{s}"
|
31
30
|
end
|
32
|
-
while s = @
|
31
|
+
while s = @reader.get_line
|
33
32
|
s.chomp!
|
34
33
|
case s
|
35
34
|
when /^gfarm:\/\//
|
@@ -47,6 +46,8 @@ module Pwrake
|
|
47
46
|
end
|
48
47
|
|
49
48
|
def close
|
49
|
+
@writer.halt
|
50
|
+
@reader.halt
|
50
51
|
@io.close
|
51
52
|
end
|
52
53
|
end
|
data/lib/pwrake/logger.rb
CHANGED
data/lib/pwrake/master/master.rb
CHANGED
@@ -1,15 +1,24 @@
|
|
1
|
+
require "fileutils"
|
2
|
+
require "pwrake/logger"
|
3
|
+
require "pwrake/nbio"
|
4
|
+
require "pwrake/option/option"
|
5
|
+
require "pwrake/task/task_wrapper"
|
6
|
+
require "pwrake/queue/task_queue"
|
7
|
+
require "pwrake/master/fiber_pool"
|
8
|
+
|
1
9
|
module Pwrake
|
2
10
|
|
3
11
|
class Master
|
4
12
|
|
5
13
|
def initialize
|
6
|
-
@
|
7
|
-
@
|
14
|
+
@selector = NBIO::Selector.new
|
15
|
+
@hostinfo_by_taskname = {}
|
8
16
|
@option = Option.new
|
9
|
-
@hdl_set =
|
17
|
+
@hdl_set = []
|
10
18
|
@channel_by_hostid = {}
|
11
19
|
@channels = []
|
12
|
-
@
|
20
|
+
@hostinfo_by_id = {}
|
21
|
+
@n_retry = @option["RETRY"]
|
13
22
|
init_logger
|
14
23
|
end
|
15
24
|
|
@@ -43,35 +52,31 @@ module Pwrake
|
|
43
52
|
end
|
44
53
|
|
45
54
|
def setup_branch_handler(sub_host)
|
55
|
+
ior,w0 = IO.pipe
|
56
|
+
r2,iow = IO.pipe
|
46
57
|
if sub_host == "localhost" && /^(n|f)/i !~ ENV['T']
|
47
|
-
|
48
|
-
|
49
|
-
Rake.application.run_branch_in_thread(r,w,o)
|
50
|
-
end
|
58
|
+
@thread = Thread.new(r2,w0,@option) do |r,w,o|
|
59
|
+
Rake.application.run_branch_in_thread(r,w,o)
|
51
60
|
end
|
52
61
|
else
|
53
|
-
|
54
|
-
|
55
|
-
#
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
r2.close
|
65
|
-
end
|
66
|
-
Marshal.dump(@option,hdl.iow)
|
67
|
-
hdl.iow.flush
|
68
|
-
s = hdl.ior.gets
|
62
|
+
dir = File.absolute_path(File.dirname($PROGRAM_NAME))
|
63
|
+
cmd = "ssh -x -T -q #{sub_host} '" +
|
64
|
+
"cd \"#{Dir.pwd}\";"+
|
65
|
+
"PATH=#{dir}:${PATH} exec pwrake_branch'"
|
66
|
+
Log.debug("BranchCommunicator cmd=#{cmd}")
|
67
|
+
spawn(cmd,:pgroup=>true,:out=>w0,:in=>r2)
|
68
|
+
w0.close
|
69
|
+
r2.close
|
70
|
+
Marshal.dump(@option,iow)
|
71
|
+
iow.flush
|
72
|
+
s = ior.gets
|
69
73
|
if !s or s.chomp != "pwrake_branch start"
|
70
74
|
raise RuntimeError,"pwrake_branch start failed: receive #{s.inspect}"
|
71
75
|
end
|
72
76
|
end
|
73
|
-
|
74
|
-
|
77
|
+
rd = NBIO::Reader.new(@selector,ior)
|
78
|
+
wt = NBIO::Writer.new(@selector,iow)
|
79
|
+
return NBIO::Handler.new(rd,wt,sub_host)
|
75
80
|
end
|
76
81
|
|
77
82
|
def signal_trap(sig)
|
@@ -89,9 +94,12 @@ module Pwrake
|
|
89
94
|
$stderr.puts "Exiting..."
|
90
95
|
@no_more_run = true
|
91
96
|
@failed = true
|
92
|
-
|
97
|
+
NBIO::Handler.kill(@hdl_set,sig)
|
98
|
+
# @selector.run : not required here
|
93
99
|
when 1
|
94
100
|
$stderr.puts "\nOnce more Ctrl-C (SIGINT) for exit."
|
101
|
+
when 2
|
102
|
+
@thread.kill if @thread
|
95
103
|
else
|
96
104
|
Kernel.exit(false) # must wait for nomral exit
|
97
105
|
end
|
@@ -100,30 +108,28 @@ module Pwrake
|
|
100
108
|
|
101
109
|
def setup_branches
|
102
110
|
sum_ncore = 0
|
103
|
-
|
104
111
|
@option.host_map.each do |sub_host, wk_hosts|
|
105
112
|
@hdl_set << hdl = setup_branch_handler(sub_host)
|
106
|
-
|
107
|
-
|
113
|
+
Fiber.new do
|
114
|
+
hdl.put_line "host_list_begin"
|
108
115
|
wk_hosts.each do |host_info|
|
109
116
|
name = host_info.name
|
110
117
|
ncore = host_info.ncore
|
111
118
|
host_id = host_info.id
|
112
119
|
Log.debug "connecting #{name} ncore=#{ncore} id=#{host_id}"
|
113
|
-
|
114
|
-
@channel_by_hostid[host_id] =
|
115
|
-
@
|
120
|
+
hdl.put_line "host:#{host_id} #{name} #{ncore}"
|
121
|
+
@channel_by_hostid[host_id] = hdl
|
122
|
+
@hostinfo_by_id[host_id] = host_info
|
116
123
|
end
|
117
|
-
|
118
|
-
|
119
|
-
while s = chan.gets
|
124
|
+
hdl.put_line "host_list_end"
|
125
|
+
while s = hdl.get_line
|
120
126
|
case s
|
121
127
|
when /^ncore:done$/
|
122
128
|
break
|
123
129
|
when /^ncore:(\d+):(\d+)$/
|
124
130
|
id, ncore = $1.to_i, $2.to_i
|
125
131
|
Log.debug "worker_id=#{id} ncore=#{ncore}"
|
126
|
-
@
|
132
|
+
@hostinfo_by_id[id].set_ncore(ncore)
|
127
133
|
sum_ncore += ncore
|
128
134
|
when /^exited$/
|
129
135
|
raise RuntimeError,"Unexpected branch exit"
|
@@ -132,23 +138,39 @@ module Pwrake
|
|
132
138
|
raise RuntimeError,"invalid return: #{msg}"
|
133
139
|
end
|
134
140
|
end
|
141
|
+
end.resume
|
135
142
|
end
|
143
|
+
@selector.run
|
136
144
|
|
137
145
|
Log.info "num_cores=#{sum_ncore}"
|
138
|
-
@
|
139
|
-
|
140
|
-
|
146
|
+
@hostinfo_by_id.each do |id,host|
|
147
|
+
if ncore = @hostinfo_by_id[id].idle_cores
|
148
|
+
Log.info "#{host} id=#{id} ncore=#{ncore}"
|
149
|
+
else
|
150
|
+
@hostinfo_by_id.delete(id)
|
151
|
+
end
|
152
|
+
end
|
153
|
+
if @hostinfo_by_id.empty?
|
154
|
+
raise RuntimeError,"no worker host"
|
141
155
|
end
|
142
156
|
queue_class = Pwrake.const_get(@option.queue_class)
|
143
|
-
@task_queue = queue_class.new(@
|
157
|
+
@task_queue = queue_class.new(@hostinfo_by_id)
|
144
158
|
|
145
159
|
@branch_setup_thread = Thread.new do
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
160
|
+
#@channels.each do |chan|
|
161
|
+
create_fiber(@hdl_set) do |hdl|
|
162
|
+
while s = hdl.get_line
|
163
|
+
case s
|
164
|
+
when /^retire:(\d+)$/
|
165
|
+
retire($1.to_i)
|
166
|
+
when /^branch_setup:done$/
|
167
|
+
break
|
168
|
+
else
|
169
|
+
raise RuntimeError,"branch_setup failed: s=#{s.inspect}"
|
170
|
+
end
|
150
171
|
end
|
151
172
|
end
|
173
|
+
@selector.run
|
152
174
|
@killed = 0
|
153
175
|
[:TERM,:INT].each do |sig|
|
154
176
|
Signal.trap(sig) do
|
@@ -159,6 +181,20 @@ module Pwrake
|
|
159
181
|
|
160
182
|
end
|
161
183
|
|
184
|
+
def retire(hid)
|
185
|
+
host_info = @hostinfo_by_id[hid.to_i]
|
186
|
+
if host_info && host_info.decrease(1)
|
187
|
+
# all retired
|
188
|
+
m = "retired: host #{host_info.name}"
|
189
|
+
Log.warn(m)
|
190
|
+
$stderr.puts(m)
|
191
|
+
drop_host(host_info) # delete from hostinfo_by_id
|
192
|
+
if @hostinfo_by_id.empty?
|
193
|
+
raise RuntimeError,"no worker host"
|
194
|
+
end
|
195
|
+
end
|
196
|
+
end
|
197
|
+
|
162
198
|
def create_fiber(channels,&blk)
|
163
199
|
channels.each do |chan|
|
164
200
|
fb = Fiber.new(&blk)
|
@@ -172,12 +208,12 @@ module Pwrake
|
|
172
208
|
|
173
209
|
if @option['GRAPH_PARTITION']
|
174
210
|
setup_postprocess0
|
175
|
-
@task_queue.deq_noaction_task do |tw
|
211
|
+
@task_queue.deq_noaction_task do |tw|
|
176
212
|
tw.preprocess
|
177
213
|
tw.status = "end"
|
178
214
|
@post_pool.enq(tw)
|
179
215
|
end
|
180
|
-
@
|
216
|
+
@selector.run
|
181
217
|
@post_pool.finish
|
182
218
|
Log.debug "@post_pool.finish"
|
183
219
|
|
@@ -189,43 +225,59 @@ module Pwrake
|
|
189
225
|
@branch_setup_thread.join
|
190
226
|
send_task_to_idle_core
|
191
227
|
#
|
192
|
-
create_fiber(@
|
193
|
-
while s =
|
194
|
-
Log.debug "Master:recv #{s.inspect} from branch[#{
|
228
|
+
create_fiber(@hdl_set) do |hdl|
|
229
|
+
while s = hdl.get_line
|
230
|
+
Log.debug "Master:recv #{s.inspect} from branch[#{hdl.host}]"
|
195
231
|
case s
|
196
232
|
when /^task(\w+):(\d*):(.*)$/o
|
197
233
|
status, shell_id, task_name = $1, $2.to_i, $3
|
198
234
|
tw = Rake.application[task_name].wrapper
|
199
235
|
tw.shell_id = shell_id
|
200
236
|
tw.status = status
|
201
|
-
|
202
|
-
|
237
|
+
host_info = @hostinfo_by_taskname[task_name]
|
238
|
+
if host_info.nil?
|
239
|
+
m = "unknown hostid: task_name=#{task_name} s=#{s.inspect}"+
|
240
|
+
" @hostinfo_by_taskname.keys=#{@hostinfo_by_taskname.keys.inspect}"
|
241
|
+
Log.error(m)
|
242
|
+
$stderr.puts(m)
|
243
|
+
end
|
244
|
+
task_end(tw,host_info) # @idle_cores.increase(..
|
203
245
|
# check failure
|
204
246
|
if tw.status == "fail"
|
205
247
|
$stderr.puts %[task "#{tw.name}" failed.]
|
206
|
-
if
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
|
213
|
-
$stderr.puts "... Kill running tasks."
|
214
|
-
when 'continue'
|
215
|
-
$stderr.puts "... Continue runable tasks."
|
216
|
-
else # 'wait'
|
217
|
-
@no_more_run = true
|
218
|
-
$stderr.puts "... Wait for running tasks."
|
219
|
-
end
|
248
|
+
if host_info
|
249
|
+
continuous_fail = host_info.task_result(tw.status)
|
250
|
+
Log.debug "task=#{tw.name} continuous_fail=#{continuous_fail}"
|
251
|
+
if continuous_fail > @n_retry && @hostinfo_by_id.size > 1
|
252
|
+
# retire this host
|
253
|
+
drop_host(host_info)
|
254
|
+
Log.warn("retired host:#{host_info.name} due to continuous fail")
|
220
255
|
end
|
221
|
-
|
222
|
-
|
256
|
+
end
|
257
|
+
if tw.no_more_retry && !@failed
|
258
|
+
@failed = true
|
259
|
+
case @option['FAILURE_TERMINATION']
|
260
|
+
when 'kill'
|
261
|
+
NBIO::Handler.kill(@hdl_set,"INT")
|
262
|
+
@selector.run
|
263
|
+
@no_more_run = true
|
264
|
+
$stderr.puts "... Kill running tasks."
|
265
|
+
when 'continue'
|
266
|
+
$stderr.puts "... Continue runable tasks."
|
267
|
+
else # 'wait'
|
268
|
+
@no_more_run = true
|
269
|
+
$stderr.puts "... Wait for running tasks."
|
223
270
|
end
|
224
271
|
end
|
272
|
+
if tw.has_output_file? && File.exist?(tw.name)
|
273
|
+
handle_failed_target(tw.name)
|
274
|
+
end
|
225
275
|
end
|
226
276
|
# postprocess
|
227
277
|
@post_pool.enq(tw) # must be after @no_more_run = true
|
228
278
|
break if @finished
|
279
|
+
when /^retire:(\d+)$/
|
280
|
+
retire($1.to_i)
|
229
281
|
when /^exited$/o
|
230
282
|
@exited = true
|
231
283
|
Log.debug "receive #{s.chomp} from branch"
|
@@ -237,7 +289,11 @@ module Pwrake
|
|
237
289
|
end
|
238
290
|
Log.debug "Master#invoke: fiber end"
|
239
291
|
end
|
240
|
-
|
292
|
+
if !ending?
|
293
|
+
Log.debug "@selector.run begin"
|
294
|
+
@selector.run
|
295
|
+
Log.debug "@selector.run end"
|
296
|
+
end
|
241
297
|
@post_pool.finish
|
242
298
|
Log.debug "Master#invoke: end of task=#{t.name}"
|
243
299
|
end
|
@@ -246,23 +302,26 @@ module Pwrake
|
|
246
302
|
#Log.debug "#{self.class}#send_task_to_idle_core start"
|
247
303
|
count = 0
|
248
304
|
# @idle_cores.decrease(..
|
249
|
-
@task_queue.deq_task do |tw,
|
305
|
+
@task_queue.deq_task do |tw,host_info,ncore|
|
306
|
+
host_info.busy(ncore)
|
250
307
|
count += 1
|
251
|
-
@
|
308
|
+
@hostinfo_by_taskname[tw.name] = host_info
|
252
309
|
tw.preprocess
|
253
310
|
if tw.has_action?
|
311
|
+
hid = host_info.id
|
254
312
|
s = "#{hid}:#{tw.task_id}:#{tw.name}"
|
255
313
|
@channel_by_hostid[hid].put_line(s)
|
256
|
-
tw.exec_host =
|
314
|
+
tw.exec_host = host_info.name
|
257
315
|
else
|
258
316
|
tw.status = "end"
|
259
|
-
|
317
|
+
task_end(tw,host_info) # @idle_cores.increase(..
|
260
318
|
@post_pool.enq(tw)
|
261
319
|
end
|
262
320
|
end
|
263
|
-
if count == 0 && !@task_queue.empty? && @
|
321
|
+
if count == 0 && !@task_queue.empty? && @hostinfo_by_taskname.empty?
|
264
322
|
m="No task was invoked while unexecuted tasks remain"
|
265
323
|
Log.error m
|
324
|
+
Log.error "@task_queue="+@task_queue.inspect
|
266
325
|
raise RuntimeError,m
|
267
326
|
end
|
268
327
|
#Log.debug "#{self.class}#send_task_to_idle_core end time=#{Time.now-tm}"
|
@@ -272,7 +331,7 @@ module Pwrake
|
|
272
331
|
i = 0
|
273
332
|
n = @option.max_postprocess_pool
|
274
333
|
@post_pool = FiberPool.new(n) do |pool|
|
275
|
-
postproc = @option.postprocess(@
|
334
|
+
postproc = @option.postprocess(@selector)
|
276
335
|
i += 1
|
277
336
|
Log.debug "New postprocess fiber ##{i}"
|
278
337
|
Fiber.new do
|
@@ -282,7 +341,8 @@ module Pwrake
|
|
282
341
|
loc = postproc.run(tw)
|
283
342
|
tw.postprocess(loc)
|
284
343
|
pool.count_down
|
285
|
-
@
|
344
|
+
@hostinfo_by_taskname.delete(tw.name)
|
345
|
+
tw.retry_or_subsequent unless @exited
|
286
346
|
break if yield(pool,j)
|
287
347
|
end
|
288
348
|
postproc.close
|
@@ -291,21 +351,25 @@ module Pwrake
|
|
291
351
|
end
|
292
352
|
end
|
293
353
|
|
354
|
+
def task_end(tw,host_info)
|
355
|
+
if host_info && host_info.idle(tw.n_used_cores(host_info))
|
356
|
+
# all retired
|
357
|
+
Log.warn("retired host:#{host_info.name} because all core retired")
|
358
|
+
drop_host(host_info)
|
359
|
+
end
|
360
|
+
end
|
361
|
+
|
294
362
|
def setup_postprocess0
|
295
363
|
setup_postprocess{false}
|
296
364
|
end
|
297
365
|
|
298
366
|
def setup_postprocess1
|
299
367
|
setup_postprocess do |pool,j|
|
300
|
-
#Log.debug "
|
301
|
-
|
302
|
-
|
303
|
-
#Log.debug "pool.empty?=#{pool.empty?}"
|
304
|
-
if (@no_more_run || @task_queue.empty?) &&
|
305
|
-
@hostid_by_taskname.empty?
|
306
|
-
Log.debug "postproc##{j} closing @channels=#{@channels.inspect}"
|
368
|
+
#Log.debug " pool.empty?=#{pool.empty?}"
|
369
|
+
if ending?
|
370
|
+
Log.debug "postproc##{j} closing"
|
307
371
|
@finished = true
|
308
|
-
@
|
372
|
+
@selector.halt
|
309
373
|
true
|
310
374
|
elsif !@no_more_run
|
311
375
|
send_task_to_idle_core
|
@@ -314,19 +378,28 @@ module Pwrake
|
|
314
378
|
end
|
315
379
|
end
|
316
380
|
|
381
|
+
def ending?
|
382
|
+
#Log.debug " @no_more_run=#{@no_more_run.inspect}"
|
383
|
+
#Log.debug " @task_queue.empty?=#{@task_queue.empty?}"
|
384
|
+
#Log.debug " @hostinfo_by_id.empty?=#{@hostinfo_by_id.empty?}"
|
385
|
+
#Log.debug " @hostinfo_by_taskname.keys=#{@hostinfo_by_taskname.keys.inspect}"
|
386
|
+
(@no_more_run || @task_queue.empty? || @hostinfo_by_id.empty?) &&
|
387
|
+
@hostinfo_by_taskname.empty?
|
388
|
+
end
|
389
|
+
|
317
390
|
def handle_failed_target(name)
|
318
391
|
case @option['FAILED_TARGET']
|
319
392
|
#
|
320
393
|
when /rename/i, NilClass
|
321
394
|
dst = name+"._fail_"
|
322
395
|
::FileUtils.mv(name,dst)
|
323
|
-
msg = "Rename
|
396
|
+
msg = "Rename output file '#{name}' to '#{dst}'"
|
324
397
|
$stderr.puts(msg)
|
325
398
|
Log.warn(msg)
|
326
399
|
#
|
327
400
|
when /delete/i
|
328
401
|
::FileUtils.rm(name)
|
329
|
-
msg = "Delete
|
402
|
+
msg = "Delete output file '#{name}'"
|
330
403
|
$stderr.puts(msg)
|
331
404
|
Log.warn(msg)
|
332
405
|
#
|
@@ -334,10 +407,40 @@ module Pwrake
|
|
334
407
|
end
|
335
408
|
end
|
336
409
|
|
410
|
+
def drop_host(host_info)
|
411
|
+
Log.debug "drop_host: #{host_info.name}"
|
412
|
+
hid = host_info.id
|
413
|
+
if @hostinfo_by_id[hid]
|
414
|
+
s = "drop:#{hid}"
|
415
|
+
@channel_by_hostid[hid].put_line(s)
|
416
|
+
@task_queue.drop_host(host_info)
|
417
|
+
@hostinfo_by_id.delete(hid)
|
418
|
+
if @hostinfo_by_id.empty?
|
419
|
+
if @finished
|
420
|
+
Log.debug "drop_host: @finished and @hostinfo_by_id.empty?"
|
421
|
+
else
|
422
|
+
Log.error "drop_host: All workers retired."
|
423
|
+
$stderr.puts "All workers retired."
|
424
|
+
@failed = true
|
425
|
+
end
|
426
|
+
end
|
427
|
+
end
|
428
|
+
end
|
429
|
+
|
337
430
|
def finish
|
338
431
|
Log.debug "Master#finish begin"
|
339
432
|
@branch_setup_thread.join
|
340
|
-
|
433
|
+
# continues running fibers
|
434
|
+
Log.debug "Master#finish @selector.run begin"
|
435
|
+
@selector.run(60)
|
436
|
+
Log.debug "Master#finish @selector.run end"
|
437
|
+
if !@exited
|
438
|
+
@exited = true
|
439
|
+
Log.debug "Master#finish Hander.exit begin"
|
440
|
+
NBIO::Handler.exit(@hdl_set)
|
441
|
+
@selector.run(60)
|
442
|
+
Log.debug "Master#finish Hander.exit end"
|
443
|
+
end
|
341
444
|
TaskWrapper.close_task_logger
|
342
445
|
Log.debug "Master#finish end"
|
343
446
|
@failed
|