pwrake 2.0.1 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +22 -9
  3. data/bin/gfwhere-pipe +33 -9
  4. data/bin/pwrake +5 -2
  5. data/bin/pwrake_branch +5 -3
  6. data/lib/pwrake/branch/branch.rb +95 -86
  7. data/lib/pwrake/branch/branch_application.rb +4 -0
  8. data/lib/pwrake/branch/communicator.rb +173 -0
  9. data/lib/pwrake/branch/communicator_set.rb +100 -0
  10. data/lib/pwrake/branch/fiber_queue.rb +10 -0
  11. data/lib/pwrake/branch/shell.rb +68 -24
  12. data/lib/pwrake/branch/shell_profiler.rb +2 -0
  13. data/lib/pwrake/gfarm/gfarm_postprocess.rb +8 -7
  14. data/lib/pwrake/logger.rb +5 -0
  15. data/lib/pwrake/master/master.rb +190 -87
  16. data/lib/pwrake/master/master_application.rb +8 -0
  17. data/lib/pwrake/nbio.rb +525 -0
  18. data/lib/pwrake/option/host_map.rb +36 -4
  19. data/lib/pwrake/option/option.rb +7 -1
  20. data/lib/pwrake/option/option_filesystem.rb +13 -3
  21. data/lib/pwrake/queue/locality_aware_queue.rb +41 -6
  22. data/lib/pwrake/queue/queue_array.rb +31 -11
  23. data/lib/pwrake/queue/task_queue.rb +15 -18
  24. data/lib/pwrake/report/report.rb +2 -0
  25. data/lib/pwrake/task/task_algorithm.rb +4 -1
  26. data/lib/pwrake/task/task_manager.rb +2 -0
  27. data/lib/pwrake/task/task_property.rb +1 -0
  28. data/lib/pwrake/task/task_wrapper.rb +40 -21
  29. data/lib/pwrake/version.rb +1 -1
  30. data/lib/pwrake/worker/invoker.rb +4 -29
  31. data/pwrake.gemspec +3 -2
  32. metadata +24 -12
  33. data/lib/pwrake/branch.rb +0 -22
  34. data/lib/pwrake/branch/worker_communicator.rb +0 -104
  35. data/lib/pwrake/iomux/channel.rb +0 -70
  36. data/lib/pwrake/iomux/handler.rb +0 -124
  37. data/lib/pwrake/iomux/handler_set.rb +0 -35
  38. data/lib/pwrake/iomux/runner.rb +0 -62
  39. data/lib/pwrake/master.rb +0 -30
@@ -1,3 +1,5 @@
1
+ require "csv"
2
+
1
3
  module Pwrake
2
4
 
3
5
  class ShellProfiler
@@ -2,11 +2,11 @@ module Pwrake
2
2
 
3
3
  class GfarmPostprocess
4
4
 
5
- def initialize(runner)
5
+ def initialize(selector)
6
6
  @io = IO.popen('gfwhere-pipe','r+')
7
7
  @io.sync = true
8
- @hdl = Handler.new(runner,@io,@io)
9
- @chan = Channel.new(@hdl)
8
+ @reader = NBIO::Reader.new(selector,@io)
9
+ @writer = NBIO::Writer.new(selector,@io)
10
10
  end
11
11
 
12
12
  def run(task_wrap)
@@ -15,13 +15,12 @@ module Pwrake
15
15
  end
16
16
  filename = task_wrap.name
17
17
  begin
18
- @hdl.iow.puts(filename)
19
- @hdl.iow.flush
18
+ @writer.put_line(filename)
20
19
  rescue Errno::EPIPE
21
20
  Log.warn "GfarmPostprocess#run: Errno::EPIPE for #{filename}"
22
21
  return []
23
22
  end
24
- s = @chan.get_line
23
+ s = @reader.get_line
25
24
  if s.nil?
26
25
  raise "gfwhere: unexpected end"
27
26
  end
@@ -29,7 +28,7 @@ module Pwrake
29
28
  if s != filename
30
29
  raise "gfwhere: file=#{filename}, result=#{s}"
31
30
  end
32
- while s = @chan.get_line
31
+ while s = @reader.get_line
33
32
  s.chomp!
34
33
  case s
35
34
  when /^gfarm:\/\//
@@ -47,6 +46,8 @@ module Pwrake
47
46
  end
48
47
 
49
48
  def close
49
+ @writer.halt
50
+ @reader.halt
50
51
  @io.close
51
52
  end
52
53
  end
data/lib/pwrake/logger.rb CHANGED
@@ -1,3 +1,5 @@
1
+ require "logger"
2
+
1
3
  module Pwrake
2
4
 
3
5
  module Log
@@ -8,5 +10,8 @@ module Pwrake
8
10
  Rake.application.logger.send(meth_id,*args)
9
11
  end
10
12
 
13
+ def bt(e)
14
+ "#{e.class}: #{e.message}\n "+(e.backtrace||[]).join("\n ")
15
+ end
11
16
  end
12
17
  end
@@ -1,15 +1,24 @@
1
+ require "fileutils"
2
+ require "pwrake/logger"
3
+ require "pwrake/nbio"
4
+ require "pwrake/option/option"
5
+ require "pwrake/task/task_wrapper"
6
+ require "pwrake/queue/task_queue"
7
+ require "pwrake/master/fiber_pool"
8
+
1
9
  module Pwrake
2
10
 
3
11
  class Master
4
12
 
5
13
  def initialize
6
- @runner = Runner.new
7
- @hostid_by_taskname = {}
14
+ @selector = NBIO::Selector.new
15
+ @hostinfo_by_taskname = {}
8
16
  @option = Option.new
9
- @hdl_set = HandlerSet.new
17
+ @hdl_set = []
10
18
  @channel_by_hostid = {}
11
19
  @channels = []
12
- @hosts = {}
20
+ @hostinfo_by_id = {}
21
+ @n_retry = @option["RETRY"]
13
22
  init_logger
14
23
  end
15
24
 
@@ -43,35 +52,31 @@ module Pwrake
43
52
  end
44
53
 
45
54
  def setup_branch_handler(sub_host)
55
+ ior,w0 = IO.pipe
56
+ r2,iow = IO.pipe
46
57
  if sub_host == "localhost" && /^(n|f)/i !~ ENV['T']
47
- hdl = Handler.new(@runner) do |w0,w1,r2|
48
- @thread = Thread.new(r2,w0,@option) do |r,w,o|
49
- Rake.application.run_branch_in_thread(r,w,o)
50
- end
58
+ @thread = Thread.new(r2,w0,@option) do |r,w,o|
59
+ Rake.application.run_branch_in_thread(r,w,o)
51
60
  end
52
61
  else
53
- hdl = Handler.new(@runner) do |w0,w1,r2|
54
- dir = File.absolute_path(File.dirname($PROGRAM_NAME))
55
- #args = Shellwords.shelljoin(@args)
56
- cmd = "ssh -x -T -q #{sub_host} '" +
57
- "cd \"#{Dir.pwd}\";"+
58
- "PATH=#{dir}:${PATH} exec pwrake_branch'"
59
- Log.debug("BranchCommunicator cmd=#{cmd}")
60
- #$stderr.puts "BranchCommunicator cmd=#{cmd}"
61
- spawn(cmd,:pgroup=>true,:out=>w0,:err=>w1,:in=>r2)
62
- w0.close
63
- w1.close
64
- r2.close
65
- end
66
- Marshal.dump(@option,hdl.iow)
67
- hdl.iow.flush
68
- s = hdl.ior.gets
62
+ dir = File.absolute_path(File.dirname($PROGRAM_NAME))
63
+ cmd = "ssh -x -T -q #{sub_host} '" +
64
+ "cd \"#{Dir.pwd}\";"+
65
+ "PATH=#{dir}:${PATH} exec pwrake_branch'"
66
+ Log.debug("BranchCommunicator cmd=#{cmd}")
67
+ spawn(cmd,:pgroup=>true,:out=>w0,:in=>r2)
68
+ w0.close
69
+ r2.close
70
+ Marshal.dump(@option,iow)
71
+ iow.flush
72
+ s = ior.gets
69
73
  if !s or s.chomp != "pwrake_branch start"
70
74
  raise RuntimeError,"pwrake_branch start failed: receive #{s.inspect}"
71
75
  end
72
76
  end
73
- hdl.host = sub_host
74
- return hdl
77
+ rd = NBIO::Reader.new(@selector,ior)
78
+ wt = NBIO::Writer.new(@selector,iow)
79
+ return NBIO::Handler.new(rd,wt,sub_host)
75
80
  end
76
81
 
77
82
  def signal_trap(sig)
@@ -89,9 +94,12 @@ module Pwrake
89
94
  $stderr.puts "Exiting..."
90
95
  @no_more_run = true
91
96
  @failed = true
92
- @hdl_set.kill(sig)
97
+ NBIO::Handler.kill(@hdl_set,sig)
98
+ # @selector.run : not required here
93
99
  when 1
94
100
  $stderr.puts "\nOnce more Ctrl-C (SIGINT) for exit."
101
+ when 2
102
+ @thread.kill if @thread
95
103
  else
96
104
  Kernel.exit(false) # must wait for nomral exit
97
105
  end
@@ -100,30 +108,28 @@ module Pwrake
100
108
 
101
109
  def setup_branches
102
110
  sum_ncore = 0
103
-
104
111
  @option.host_map.each do |sub_host, wk_hosts|
105
112
  @hdl_set << hdl = setup_branch_handler(sub_host)
106
- @channels << chan = Channel.new(hdl)
107
- chan.puts "host_list_begin"
113
+ Fiber.new do
114
+ hdl.put_line "host_list_begin"
108
115
  wk_hosts.each do |host_info|
109
116
  name = host_info.name
110
117
  ncore = host_info.ncore
111
118
  host_id = host_info.id
112
119
  Log.debug "connecting #{name} ncore=#{ncore} id=#{host_id}"
113
- chan.puts "host:#{host_id} #{name} #{ncore}"
114
- @channel_by_hostid[host_id] = chan
115
- @hosts[host_id] = name
120
+ hdl.put_line "host:#{host_id} #{name} #{ncore}"
121
+ @channel_by_hostid[host_id] = hdl
122
+ @hostinfo_by_id[host_id] = host_info
116
123
  end
117
- chan.puts "host_list_end"
118
-
119
- while s = chan.gets
124
+ hdl.put_line "host_list_end"
125
+ while s = hdl.get_line
120
126
  case s
121
127
  when /^ncore:done$/
122
128
  break
123
129
  when /^ncore:(\d+):(\d+)$/
124
130
  id, ncore = $1.to_i, $2.to_i
125
131
  Log.debug "worker_id=#{id} ncore=#{ncore}"
126
- @option.host_map.by_id[id].set_ncore(ncore)
132
+ @hostinfo_by_id[id].set_ncore(ncore)
127
133
  sum_ncore += ncore
128
134
  when /^exited$/
129
135
  raise RuntimeError,"Unexpected branch exit"
@@ -132,23 +138,39 @@ module Pwrake
132
138
  raise RuntimeError,"invalid return: #{msg}"
133
139
  end
134
140
  end
141
+ end.resume
135
142
  end
143
+ @selector.run
136
144
 
137
145
  Log.info "num_cores=#{sum_ncore}"
138
- @hosts.each do |id,host|
139
- Log.info "#{host} id=#{id} ncore=#{
140
- @option.host_map.by_id[id].idle_cores}"
146
+ @hostinfo_by_id.each do |id,host|
147
+ if ncore = @hostinfo_by_id[id].idle_cores
148
+ Log.info "#{host} id=#{id} ncore=#{ncore}"
149
+ else
150
+ @hostinfo_by_id.delete(id)
151
+ end
152
+ end
153
+ if @hostinfo_by_id.empty?
154
+ raise RuntimeError,"no worker host"
141
155
  end
142
156
  queue_class = Pwrake.const_get(@option.queue_class)
143
- @task_queue = queue_class.new(@option.host_map)
157
+ @task_queue = queue_class.new(@hostinfo_by_id)
144
158
 
145
159
  @branch_setup_thread = Thread.new do
146
- @channels.each do |chan|
147
- s = chan.gets
148
- if /^branch_setup:done$/ !~ s
149
- raise RuntimeError,"branch_setup failed"
160
+ #@channels.each do |chan|
161
+ create_fiber(@hdl_set) do |hdl|
162
+ while s = hdl.get_line
163
+ case s
164
+ when /^retire:(\d+)$/
165
+ retire($1.to_i)
166
+ when /^branch_setup:done$/
167
+ break
168
+ else
169
+ raise RuntimeError,"branch_setup failed: s=#{s.inspect}"
170
+ end
150
171
  end
151
172
  end
173
+ @selector.run
152
174
  @killed = 0
153
175
  [:TERM,:INT].each do |sig|
154
176
  Signal.trap(sig) do
@@ -159,6 +181,20 @@ module Pwrake
159
181
 
160
182
  end
161
183
 
184
+ def retire(hid)
185
+ host_info = @hostinfo_by_id[hid.to_i]
186
+ if host_info && host_info.decrease(1)
187
+ # all retired
188
+ m = "retired: host #{host_info.name}"
189
+ Log.warn(m)
190
+ $stderr.puts(m)
191
+ drop_host(host_info) # delete from hostinfo_by_id
192
+ if @hostinfo_by_id.empty?
193
+ raise RuntimeError,"no worker host"
194
+ end
195
+ end
196
+ end
197
+
162
198
  def create_fiber(channels,&blk)
163
199
  channels.each do |chan|
164
200
  fb = Fiber.new(&blk)
@@ -172,12 +208,12 @@ module Pwrake
172
208
 
173
209
  if @option['GRAPH_PARTITION']
174
210
  setup_postprocess0
175
- @task_queue.deq_noaction_task do |tw,hid|
211
+ @task_queue.deq_noaction_task do |tw|
176
212
  tw.preprocess
177
213
  tw.status = "end"
178
214
  @post_pool.enq(tw)
179
215
  end
180
- @runner.run
216
+ @selector.run
181
217
  @post_pool.finish
182
218
  Log.debug "@post_pool.finish"
183
219
 
@@ -189,43 +225,59 @@ module Pwrake
189
225
  @branch_setup_thread.join
190
226
  send_task_to_idle_core
191
227
  #
192
- create_fiber(@channels) do |chan|
193
- while s = chan.get_line
194
- Log.debug "Master:recv #{s.inspect} from branch[#{chan.handler.host}]"
228
+ create_fiber(@hdl_set) do |hdl|
229
+ while s = hdl.get_line
230
+ Log.debug "Master:recv #{s.inspect} from branch[#{hdl.host}]"
195
231
  case s
196
232
  when /^task(\w+):(\d*):(.*)$/o
197
233
  status, shell_id, task_name = $1, $2.to_i, $3
198
234
  tw = Rake.application[task_name].wrapper
199
235
  tw.shell_id = shell_id
200
236
  tw.status = status
201
- hid = @hostid_by_taskname[task_name]
202
- @task_queue.task_end(tw,hid) # @idle_cores.increase(..
237
+ host_info = @hostinfo_by_taskname[task_name]
238
+ if host_info.nil?
239
+ m = "unknown hostid: task_name=#{task_name} s=#{s.inspect}"+
240
+ " @hostinfo_by_taskname.keys=#{@hostinfo_by_taskname.keys.inspect}"
241
+ Log.error(m)
242
+ $stderr.puts(m)
243
+ end
244
+ task_end(tw,host_info) # @idle_cores.increase(..
203
245
  # check failure
204
246
  if tw.status == "fail"
205
247
  $stderr.puts %[task "#{tw.name}" failed.]
206
- if !tw.retry
207
- if !@failed
208
- @failed = true
209
- case @option['FAILURE_TERMINATION']
210
- when 'kill'
211
- @hdl_set.kill("INT")
212
- @no_more_run = true
213
- $stderr.puts "... Kill running tasks."
214
- when 'continue'
215
- $stderr.puts "... Continue runable tasks."
216
- else # 'wait'
217
- @no_more_run = true
218
- $stderr.puts "... Wait for running tasks."
219
- end
248
+ if host_info
249
+ continuous_fail = host_info.task_result(tw.status)
250
+ Log.debug "task=#{tw.name} continuous_fail=#{continuous_fail}"
251
+ if continuous_fail > @n_retry && @hostinfo_by_id.size > 1
252
+ # retire this host
253
+ drop_host(host_info)
254
+ Log.warn("retired host:#{host_info.name} due to continuous fail")
220
255
  end
221
- if tw.has_output_file? && File.exist?(tw.name)
222
- handle_failed_target(tw.name)
256
+ end
257
+ if tw.no_more_retry && !@failed
258
+ @failed = true
259
+ case @option['FAILURE_TERMINATION']
260
+ when 'kill'
261
+ NBIO::Handler.kill(@hdl_set,"INT")
262
+ @selector.run
263
+ @no_more_run = true
264
+ $stderr.puts "... Kill running tasks."
265
+ when 'continue'
266
+ $stderr.puts "... Continue runable tasks."
267
+ else # 'wait'
268
+ @no_more_run = true
269
+ $stderr.puts "... Wait for running tasks."
223
270
  end
224
271
  end
272
+ if tw.has_output_file? && File.exist?(tw.name)
273
+ handle_failed_target(tw.name)
274
+ end
225
275
  end
226
276
  # postprocess
227
277
  @post_pool.enq(tw) # must be after @no_more_run = true
228
278
  break if @finished
279
+ when /^retire:(\d+)$/
280
+ retire($1.to_i)
229
281
  when /^exited$/o
230
282
  @exited = true
231
283
  Log.debug "receive #{s.chomp} from branch"
@@ -237,7 +289,11 @@ module Pwrake
237
289
  end
238
290
  Log.debug "Master#invoke: fiber end"
239
291
  end
240
- @runner.run
292
+ if !ending?
293
+ Log.debug "@selector.run begin"
294
+ @selector.run
295
+ Log.debug "@selector.run end"
296
+ end
241
297
  @post_pool.finish
242
298
  Log.debug "Master#invoke: end of task=#{t.name}"
243
299
  end
@@ -246,23 +302,26 @@ module Pwrake
246
302
  #Log.debug "#{self.class}#send_task_to_idle_core start"
247
303
  count = 0
248
304
  # @idle_cores.decrease(..
249
- @task_queue.deq_task do |tw,hid|
305
+ @task_queue.deq_task do |tw,host_info,ncore|
306
+ host_info.busy(ncore)
250
307
  count += 1
251
- @hostid_by_taskname[tw.name] = hid
308
+ @hostinfo_by_taskname[tw.name] = host_info
252
309
  tw.preprocess
253
310
  if tw.has_action?
311
+ hid = host_info.id
254
312
  s = "#{hid}:#{tw.task_id}:#{tw.name}"
255
313
  @channel_by_hostid[hid].put_line(s)
256
- tw.exec_host = @hosts[hid]
314
+ tw.exec_host = host_info.name
257
315
  else
258
316
  tw.status = "end"
259
- @task_queue.task_end(tw,hid) # @idle_cores.increase(..
317
+ task_end(tw,host_info) # @idle_cores.increase(..
260
318
  @post_pool.enq(tw)
261
319
  end
262
320
  end
263
- if count == 0 && !@task_queue.empty? && @hostid_by_taskname.empty?
321
+ if count == 0 && !@task_queue.empty? && @hostinfo_by_taskname.empty?
264
322
  m="No task was invoked while unexecuted tasks remain"
265
323
  Log.error m
324
+ Log.error "@task_queue="+@task_queue.inspect
266
325
  raise RuntimeError,m
267
326
  end
268
327
  #Log.debug "#{self.class}#send_task_to_idle_core end time=#{Time.now-tm}"
@@ -272,7 +331,7 @@ module Pwrake
272
331
  i = 0
273
332
  n = @option.max_postprocess_pool
274
333
  @post_pool = FiberPool.new(n) do |pool|
275
- postproc = @option.postprocess(@runner)
334
+ postproc = @option.postprocess(@selector)
276
335
  i += 1
277
336
  Log.debug "New postprocess fiber ##{i}"
278
337
  Fiber.new do
@@ -282,7 +341,8 @@ module Pwrake
282
341
  loc = postproc.run(tw)
283
342
  tw.postprocess(loc)
284
343
  pool.count_down
285
- @hostid_by_taskname.delete(tw.name)
344
+ @hostinfo_by_taskname.delete(tw.name)
345
+ tw.retry_or_subsequent unless @exited
286
346
  break if yield(pool,j)
287
347
  end
288
348
  postproc.close
@@ -291,21 +351,25 @@ module Pwrake
291
351
  end
292
352
  end
293
353
 
354
+ def task_end(tw,host_info)
355
+ if host_info && host_info.idle(tw.n_used_cores(host_info))
356
+ # all retired
357
+ Log.warn("retired host:#{host_info.name} because all core retired")
358
+ drop_host(host_info)
359
+ end
360
+ end
361
+
294
362
  def setup_postprocess0
295
363
  setup_postprocess{false}
296
364
  end
297
365
 
298
366
  def setup_postprocess1
299
367
  setup_postprocess do |pool,j|
300
- #Log.debug "@no_more_run=#{@no_more_run.inspect}"
301
- #Log.debug "@task_queue.empty?=#{@task_queue.empty?}"
302
- #Log.debug "@hostid_by_taskname=#{@hostid_by_taskname.inspect}"
303
- #Log.debug "pool.empty?=#{pool.empty?}"
304
- if (@no_more_run || @task_queue.empty?) &&
305
- @hostid_by_taskname.empty?
306
- Log.debug "postproc##{j} closing @channels=#{@channels.inspect}"
368
+ #Log.debug " pool.empty?=#{pool.empty?}"
369
+ if ending?
370
+ Log.debug "postproc##{j} closing"
307
371
  @finished = true
308
- @channels.each{|ch| ch.finish} # exit
372
+ @selector.halt
309
373
  true
310
374
  elsif !@no_more_run
311
375
  send_task_to_idle_core
@@ -314,19 +378,28 @@ module Pwrake
314
378
  end
315
379
  end
316
380
 
381
+ def ending?
382
+ #Log.debug " @no_more_run=#{@no_more_run.inspect}"
383
+ #Log.debug " @task_queue.empty?=#{@task_queue.empty?}"
384
+ #Log.debug " @hostinfo_by_id.empty?=#{@hostinfo_by_id.empty?}"
385
+ #Log.debug " @hostinfo_by_taskname.keys=#{@hostinfo_by_taskname.keys.inspect}"
386
+ (@no_more_run || @task_queue.empty? || @hostinfo_by_id.empty?) &&
387
+ @hostinfo_by_taskname.empty?
388
+ end
389
+
317
390
  def handle_failed_target(name)
318
391
  case @option['FAILED_TARGET']
319
392
  #
320
393
  when /rename/i, NilClass
321
394
  dst = name+"._fail_"
322
395
  ::FileUtils.mv(name,dst)
323
- msg = "Rename failed target file '#{name}' to '#{dst}'"
396
+ msg = "Rename output file '#{name}' to '#{dst}'"
324
397
  $stderr.puts(msg)
325
398
  Log.warn(msg)
326
399
  #
327
400
  when /delete/i
328
401
  ::FileUtils.rm(name)
329
- msg = "Delete failed target file '#{name}'"
402
+ msg = "Delete output file '#{name}'"
330
403
  $stderr.puts(msg)
331
404
  Log.warn(msg)
332
405
  #
@@ -334,10 +407,40 @@ module Pwrake
334
407
  end
335
408
  end
336
409
 
410
+ def drop_host(host_info)
411
+ Log.debug "drop_host: #{host_info.name}"
412
+ hid = host_info.id
413
+ if @hostinfo_by_id[hid]
414
+ s = "drop:#{hid}"
415
+ @channel_by_hostid[hid].put_line(s)
416
+ @task_queue.drop_host(host_info)
417
+ @hostinfo_by_id.delete(hid)
418
+ if @hostinfo_by_id.empty?
419
+ if @finished
420
+ Log.debug "drop_host: @finished and @hostinfo_by_id.empty?"
421
+ else
422
+ Log.error "drop_host: All workers retired."
423
+ $stderr.puts "All workers retired."
424
+ @failed = true
425
+ end
426
+ end
427
+ end
428
+ end
429
+
337
430
  def finish
338
431
  Log.debug "Master#finish begin"
339
432
  @branch_setup_thread.join
340
- @hdl_set.exit unless @exited
433
+ # continues running fibers
434
+ Log.debug "Master#finish @selector.run begin"
435
+ @selector.run(60)
436
+ Log.debug "Master#finish @selector.run end"
437
+ if !@exited
438
+ @exited = true
439
+ Log.debug "Master#finish Hander.exit begin"
440
+ NBIO::Handler.exit(@hdl_set)
441
+ @selector.run(60)
442
+ Log.debug "Master#finish Hander.exit end"
443
+ end
341
444
  TaskWrapper.close_task_logger
342
445
  Log.debug "Master#finish end"
343
446
  @failed