pwrake 2.3.0 → 2.3.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +11 -10
- data/lib/pwrake/branch/branch.rb +24 -0
- data/lib/pwrake/branch/communicator.rb +16 -26
- data/lib/pwrake/gfarm/gfarm_postprocess.rb +1 -1
- data/lib/pwrake/master/master.rb +9 -6
- data/lib/pwrake/master/master_application.rb +12 -7
- data/lib/pwrake/nbio.rb +11 -5
- data/lib/pwrake/option/host_map.rb +13 -2
- data/lib/pwrake/option/option.rb +18 -9
- data/lib/pwrake/option/option_default_filesystem.rb +0 -1
- data/lib/pwrake/option/option_gfarm2fs.rb +2 -1
- data/lib/pwrake/queue/no_action_queue.rb +3 -4
- data/lib/pwrake/queue/task_queue.rb +19 -2
- data/lib/pwrake/report/parallelism.rb +15 -0
- data/lib/pwrake/task/file_task_algorithm.rb +47 -2
- data/lib/pwrake/task/task_algorithm.rb +7 -8
- data/lib/pwrake/task/task_property.rb +2 -6
- data/lib/pwrake/task/task_wrapper.rb +22 -11
- data/lib/pwrake/version.rb +1 -1
- data/lib/pwrake/worker/gfarm_directory.rb +4 -2
- metadata +2 -3
- data/lib/pwrake/option/option_gfarm.rb +0 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e666a4e98f080f07f41475896df0a2d115805b8af57e3541b8d898c60a8122c9
|
4
|
+
data.tar.gz: a320f42590130bab37518996ed11496d8b107ba3162a8458c896aa6daa3b322d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: cd4a9b2641c220d2a1265ccdf76ed87f2594ce0d498f521a24d67a6428d5c757129c31408f9fb9058b4e184aa2a09b01096edb17e2b42c5d5f634bd5b61125dc
|
7
|
+
data.tar.gz: 92f066ae2246bdd86097795432124d129a21e60e27d96515fe16059a05e14c0c7d4a1eefff85e574860626f600efb6c8698e1969824bcffdd5429c0905cc75cf
|
data/README.md
CHANGED
@@ -93,8 +93,8 @@ In this case, you need the rehash of command paths:
|
|
93
93
|
-L, --log, --log-dir [DIRECTORY] [Pw] Write log to DIRECTORY
|
94
94
|
--ssh-opt, --ssh-option OPTION
|
95
95
|
[Pw] Option passed to SSH
|
96
|
-
--filesystem FILESYSTEM [Pw] Specify FILESYSTEM (nfs|
|
97
|
-
--gfarm [Pw]
|
96
|
+
--filesystem FILESYSTEM [Pw] Specify FILESYSTEM (nfs|gfarm2fs)
|
97
|
+
--gfarm [Pw] (obsolete; Start pwrake on Gfarm FS)
|
98
98
|
-A, --disable-affinity [Pw] Turn OFF affinity (AFFINITY=off)
|
99
99
|
-S, --disable-steal [Pw] Turn OFF task steal
|
100
100
|
-d, --debug [Pw] Output Debug messages
|
@@ -151,6 +151,7 @@ In this case, you need the rehash of command paths:
|
|
151
151
|
GFARM_PREFIX default="pwrake_$USER"
|
152
152
|
GFARM_SUBDIR default='/'
|
153
153
|
MAX_GFWHERE_WORKER default=8
|
154
|
+
GFARM2FS_COMMAND default='gfarm2fs'
|
154
155
|
GFARM2FS_OPTION default=""
|
155
156
|
GFARM2FS_DEBUG default=false
|
156
157
|
GFARM2FS_DEBUG_WAIT default=1
|
@@ -177,14 +178,14 @@ end
|
|
177
178
|
|
178
179
|
Properties (The leftmost item is default):
|
179
180
|
|
180
|
-
ncore=integer
|
181
|
-
exclusive=no|yes
|
182
|
-
reserve=no|yes
|
183
|
-
allow=hostname
|
184
|
-
deny=hostname
|
185
|
-
order=deny,allow|
|
186
|
-
steal=yes|no
|
187
|
-
retry=integer
|
181
|
+
ncore=integer|rational - The number of cores used by this task.
|
182
|
+
exclusive=no|yes - Exclusively execute this task in a single node.
|
183
|
+
reserve=no|yes - Gives higher priority to this task if ncore>1. (reserve a host)
|
184
|
+
allow=hostname - Allow this host to execute this task. (accepts wild card)
|
185
|
+
deny=hostname - Deny this host to execute this task. (accepts wild card)
|
186
|
+
order=deny,allow|a llow,deny - The order of evaluation.
|
187
|
+
steal=yes|no - Allow task stealing for this task.
|
188
|
+
retry=integer - The number of retry for this task.
|
188
189
|
|
189
190
|
## Note for Gfarm
|
190
191
|
|
data/lib/pwrake/branch/branch.rb
CHANGED
@@ -42,6 +42,7 @@ module Pwrake
|
|
42
42
|
setup_shell
|
43
43
|
setup_fiber
|
44
44
|
setup_master_channel
|
45
|
+
setup_search_thread
|
45
46
|
@cs.run("task execution")
|
46
47
|
Log.debug "Branch#run end"
|
47
48
|
end
|
@@ -193,6 +194,29 @@ module Pwrake
|
|
193
194
|
end.resume
|
194
195
|
end
|
195
196
|
|
197
|
+
def setup_search_thread
|
198
|
+
@search_que = Queue.new
|
199
|
+
Thread.new do
|
200
|
+
while a = @search_que.deq
|
201
|
+
t,args,w = *a
|
202
|
+
t.pw_search_tasks(args)
|
203
|
+
w.puts(t.name)
|
204
|
+
end
|
205
|
+
end
|
206
|
+
end
|
207
|
+
|
208
|
+
def invoke(t,args)
|
209
|
+
Log.debug "Branch#invoke start: #{t.class}[#{t.name}]"
|
210
|
+
r,w = IO.pipe
|
211
|
+
rd = NBIO::Reader.new(@selector,r)
|
212
|
+
@search_que.enq([t,args,w])
|
213
|
+
task_name = rd.get_line.chomp
|
214
|
+
if t.name != task_name
|
215
|
+
raise "t.name=#{t.name} != task_name=#{task_name}"
|
216
|
+
end
|
217
|
+
Log.debug "Branch#invoke end: #{t.class}[#{t.name}]"
|
218
|
+
end
|
219
|
+
|
196
220
|
def kill(sig="INT")
|
197
221
|
Log.warn "Branch#kill #{sig}"
|
198
222
|
@cs.kill(sig)
|
@@ -90,7 +90,6 @@ class Communicator
|
|
90
90
|
|
91
91
|
sel = @set.selector
|
92
92
|
@reader = NBIO::MultiReader.new(sel,@ior)
|
93
|
-
@rd_err = NBIO::Reader.new(sel,@ioe)
|
94
93
|
@writer = NBIO::Writer.new(sel,@iow)
|
95
94
|
@handler = NBIO::Handler.new(@reader,@writer,@host)
|
96
95
|
|
@@ -146,48 +145,39 @@ class Communicator
|
|
146
145
|
end
|
147
146
|
|
148
147
|
def dropout(exc=nil)
|
149
|
-
#
|
150
|
-
err_out = []
|
148
|
+
# Finish worker
|
151
149
|
begin
|
152
150
|
finish_shells
|
153
151
|
if @handler
|
154
152
|
@handler.exit
|
155
153
|
@handler = nil
|
156
154
|
end
|
157
|
-
if @rd_err
|
158
|
-
while s = @rd_err.get_line
|
159
|
-
err_out << s
|
160
|
-
end
|
161
|
-
end
|
162
155
|
rescue => e
|
163
156
|
m = Log.bt(e)
|
164
|
-
|
157
|
+
$stderr.puts(m)
|
165
158
|
Log.error(m)
|
166
159
|
end
|
167
|
-
# Error output
|
168
|
-
if
|
169
|
-
|
170
|
-
|
160
|
+
# Error output from worker
|
161
|
+
if @ioe
|
162
|
+
err_out = ["standard error from worker:"]
|
163
|
+
while s = @ioe.gets
|
164
|
+
err_out << s.chomp
|
165
|
+
end
|
166
|
+
if err_out.size > 1
|
167
|
+
m = err_out.join("\n ")
|
168
|
+
$stderr.puts(m)
|
169
|
+
Log.error(m)
|
170
|
+
end
|
171
171
|
end
|
172
|
-
# Exception
|
172
|
+
# Exception message
|
173
173
|
if exc
|
174
174
|
m = Log.bt(exc)
|
175
|
-
|
176
|
-
Log.error
|
175
|
+
$stderr.puts(m)
|
176
|
+
Log.error(m)
|
177
177
|
end
|
178
178
|
ensure
|
179
179
|
@set.delete(self)
|
180
180
|
end
|
181
181
|
|
182
|
-
def finish
|
183
|
-
@iow.close
|
184
|
-
while s=@ior.gets
|
185
|
-
puts "out=#{s.chomp}"
|
186
|
-
end
|
187
|
-
while s=@ioe.gets
|
188
|
-
puts "err=#{s.chomp}"
|
189
|
-
end
|
190
|
-
end
|
191
|
-
|
192
182
|
end
|
193
183
|
end
|
data/lib/pwrake/master/master.rb
CHANGED
@@ -226,11 +226,11 @@ module Pwrake
|
|
226
226
|
if ending?
|
227
227
|
@post_pool.finish # need?
|
228
228
|
else
|
229
|
-
setup_fiber
|
229
|
+
setup_fiber
|
230
230
|
end
|
231
231
|
end
|
232
232
|
|
233
|
-
def setup_fiber
|
233
|
+
def setup_fiber
|
234
234
|
@host_fail = @option["HOST_FAILURE"]
|
235
235
|
create_fiber(@hdl_set) do |hdl|
|
236
236
|
while s = hdl.get_line
|
@@ -295,15 +295,17 @@ module Pwrake
|
|
295
295
|
$stderr.puts(s)
|
296
296
|
end
|
297
297
|
end
|
298
|
-
Log.debug "Master#
|
298
|
+
Log.debug "Master#setup_fiber: end of fiber"
|
299
299
|
end
|
300
|
+
|
300
301
|
if !ending?
|
301
302
|
Log.debug "@selector.run begin"
|
302
303
|
@selector.run
|
303
304
|
Log.debug "@selector.run end"
|
305
|
+
else
|
306
|
+
Log.debug "@selector.run skipped"
|
304
307
|
end
|
305
308
|
@post_pool.finish
|
306
|
-
Log.debug "Master#invoke: end of task=#{t.name}"
|
307
309
|
end
|
308
310
|
|
309
311
|
def send_task_to_idle_core
|
@@ -361,7 +363,7 @@ module Pwrake
|
|
361
363
|
|
362
364
|
def task_end(tw,host_info)
|
363
365
|
return if host_info.nil?
|
364
|
-
host_info.idle(tw.n_used_cores)
|
366
|
+
host_info.idle(tw.n_used_cores||1)
|
365
367
|
if host_info.retired?
|
366
368
|
# all retired
|
367
369
|
Log.warn("retired host:#{host_info.name} because all core retired")
|
@@ -396,8 +398,9 @@ module Pwrake
|
|
396
398
|
Log.debug " @task_queue.empty?=#{@task_queue.empty?}" if @task_queue.empty?
|
397
399
|
Log.debug " @hostinfo_by_id.empty?=#{@hostinfo_by_id.empty?}" if @hostinfo_by_id.empty?
|
398
400
|
Log.debug " @hostinfo_by_taskname.keys=#{@hostinfo_by_taskname.keys.inspect}"
|
401
|
+
Log.debug " @post_pool.empty?=#{@post_pool.empty?}" if @post_pool.empty?
|
399
402
|
end
|
400
|
-
@hostinfo_by_taskname.empty?
|
403
|
+
@hostinfo_by_taskname.empty? && @post_pool.empty?
|
401
404
|
else
|
402
405
|
false
|
403
406
|
end
|
@@ -26,13 +26,13 @@ module Pwrake
|
|
26
26
|
@master.setup_branches
|
27
27
|
load_rakefile
|
28
28
|
begin
|
29
|
-
Log.debug "init:
|
29
|
+
Log.debug "init: %.6f sec" % (Pwrake.clock-t)
|
30
30
|
t = Pwrake.clock
|
31
31
|
top_level
|
32
|
-
Log.debug "main:
|
32
|
+
Log.debug "main: %.6f sec" % (Pwrake.clock-t)
|
33
33
|
t = Pwrake.clock
|
34
34
|
@failed = @master.finish
|
35
|
-
Log.debug "finish:
|
35
|
+
Log.debug "finish: %.6f sec" % (Pwrake.clock-t)
|
36
36
|
rescue SystemExit => e
|
37
37
|
@failed = true
|
38
38
|
rescue Exception => e
|
@@ -84,7 +84,11 @@ module Pwrake
|
|
84
84
|
end
|
85
85
|
|
86
86
|
def invoke(t,*args)
|
87
|
-
@master.
|
87
|
+
if Thread.current == @master.thread
|
88
|
+
@branch.invoke(t,args)
|
89
|
+
else
|
90
|
+
@master.invoke(t,args)
|
91
|
+
end
|
88
92
|
end
|
89
93
|
|
90
94
|
def standard_rake_options
|
@@ -139,14 +143,15 @@ module Pwrake
|
|
139
143
|
options.ssh_option = value
|
140
144
|
}
|
141
145
|
],
|
142
|
-
['--filesystem FILESYSTEM', "[Pw] Specify FILESYSTEM (nfs|
|
146
|
+
['--filesystem FILESYSTEM', "[Pw] Specify FILESYSTEM (nfs|gfarm2fs)",
|
143
147
|
lambda { |value|
|
144
148
|
options.filesystem = value
|
145
149
|
}
|
146
150
|
],
|
147
|
-
['--gfarm', "[Pw] FILESYSTEM=
|
151
|
+
['--gfarm', "[Pw] FILESYSTEM=gfarm2fs",
|
148
152
|
lambda { |value|
|
149
|
-
|
153
|
+
warn 'The --gfarm option may not work correctly. Execute pwrake under gfarm2fs.'
|
154
|
+
options.filesystem = "gfarm2fs"
|
150
155
|
}
|
151
156
|
],
|
152
157
|
['-A', '--disable-affinity', "[Pw] Turn OFF affinity (AFFINITY=off)",
|
data/lib/pwrake/nbio.rb
CHANGED
@@ -506,8 +506,17 @@ module NBIO
|
|
506
506
|
end
|
507
507
|
@exited = true
|
508
508
|
exit_msg = "exited"
|
509
|
-
|
510
|
-
|
509
|
+
begin
|
510
|
+
@writer.put_line "exit"
|
511
|
+
rescue Errno::EPIPE => e
|
512
|
+
m = "#{e} in #{self.class}.exit iow=#{iow.inspect}"
|
513
|
+
if defined? Log
|
514
|
+
Log.warn m
|
515
|
+
else
|
516
|
+
$stderr.puts m
|
517
|
+
end
|
518
|
+
return
|
519
|
+
end
|
511
520
|
Log.debug "Handler#exit: end: @writer.put_line \"exit\"" if defined? Log
|
512
521
|
#
|
513
522
|
if @reader.class == Reader # MultiReader not work
|
@@ -518,9 +527,6 @@ module NBIO
|
|
518
527
|
return if line == exit_msg
|
519
528
|
end
|
520
529
|
end
|
521
|
-
rescue Errno::EPIPE => e
|
522
|
-
Log.error "Errno::EPIPE in #{self.class}.exit iow=#{iow.inspect}\n"+
|
523
|
-
e.backtrace.join("\n") if defined? Log
|
524
530
|
end
|
525
531
|
|
526
532
|
def halt
|
@@ -13,7 +13,7 @@ module Pwrake
|
|
13
13
|
|
14
14
|
def initialize(name,id,ncore,weight,group=nil)
|
15
15
|
@name = name
|
16
|
-
@ncore = ncore
|
16
|
+
@ncore = ncore || 1
|
17
17
|
@weight = weight || 1.0
|
18
18
|
@group = group || 0
|
19
19
|
@id = id
|
@@ -32,6 +32,17 @@ module Pwrake
|
|
32
32
|
attr_reader :continuous_fail
|
33
33
|
attr_accessor :idle_cores
|
34
34
|
|
35
|
+
def add_line(ncore=nil,weight=nil,group=nil)
|
36
|
+
ncore ||= 1
|
37
|
+
weight ||= 1.0
|
38
|
+
group ||= 0
|
39
|
+
if @group != group
|
40
|
+
raise "different group=#{group} for host=#{@name}"
|
41
|
+
end
|
42
|
+
@weight = (@weight*@ncore + weight*ncore)/(@ncore+ncore)
|
43
|
+
@ncore += ncore
|
44
|
+
end
|
45
|
+
|
35
46
|
def local?
|
36
47
|
ipa = IPSocket.getaddress(@name)
|
37
48
|
HostInfo.local_ip.include?(ipa)
|
@@ -231,7 +242,7 @@ module Pwrake
|
|
231
242
|
#weight = (weight || 1).to_f
|
232
243
|
group &&= group.to_i
|
233
244
|
if host_info = @by_name[host]
|
234
|
-
|
245
|
+
host_info.add_line(ncore,weight,group)
|
235
246
|
else
|
236
247
|
id = @by_id.size
|
237
248
|
host_info = HostInfo.new(host,id,ncore,weight,group)
|
data/lib/pwrake/option/option.rb
CHANGED
@@ -61,11 +61,11 @@ module Pwrake
|
|
61
61
|
# ----------------------------------------------------------
|
62
62
|
|
63
63
|
def init_filesystem
|
64
|
-
@filesystem = Rake.application.options.filesystem
|
65
|
-
@filesystem
|
66
|
-
|
67
|
-
require "pwrake/option/
|
68
|
-
|
64
|
+
@filesystem = Rake.application.options.filesystem || mount_type
|
65
|
+
case @filesystem
|
66
|
+
when 'gfarm2fs'
|
67
|
+
require "pwrake/option/option_gfarm2fs"
|
68
|
+
else
|
69
69
|
require "pwrake/option/option_default_filesystem"
|
70
70
|
end
|
71
71
|
end
|
@@ -73,15 +73,15 @@ module Pwrake
|
|
73
73
|
attr_reader :worker_option
|
74
74
|
attr_reader :queue_class
|
75
75
|
|
76
|
-
def mount_type(
|
76
|
+
def mount_type(dir=nil)
|
77
77
|
mtab = '/etc/mtab'
|
78
78
|
if File.exist?(mtab)
|
79
|
-
|
79
|
+
dir ||= mountpoint_of_cwd
|
80
80
|
open(mtab,'r') do |f|
|
81
81
|
f.each_line do |l|
|
82
82
|
a = l.split
|
83
|
-
if a[1] ==
|
84
|
-
return a[2]
|
83
|
+
if a[1] == dir
|
84
|
+
return a[2].sub(/^fuse\./,'')
|
85
85
|
end
|
86
86
|
end
|
87
87
|
end
|
@@ -354,6 +354,14 @@ module Pwrake
|
|
354
354
|
# ----------------------------------------------------------
|
355
355
|
|
356
356
|
def setup_hosts
|
357
|
+
if f = ENV['PBS_NODEFILE']
|
358
|
+
if @hostfile
|
359
|
+
Log.info "HOSTFILE=#{@hostfile} overrides PBS_NODEFILE=#{f}"
|
360
|
+
else
|
361
|
+
Log.info "use PBS_NODEFILE=#{f}"
|
362
|
+
@hostfile = f
|
363
|
+
end
|
364
|
+
end
|
357
365
|
if @hostfile && @num_threads
|
358
366
|
raise "Cannot set `hostfile' and `num_threads' simultaneously"
|
359
367
|
end
|
@@ -364,6 +372,7 @@ module Pwrake
|
|
364
372
|
# ----------------------------------------------------------
|
365
373
|
|
366
374
|
def put_log
|
375
|
+
Log.info "Pwrake::VERSION=#{Pwrake::VERSION}"
|
367
376
|
Log.info "Options:"
|
368
377
|
self.each do |k,v|
|
369
378
|
Log.info " #{k} = #{v.inspect}"
|
@@ -8,6 +8,7 @@ module Pwrake
|
|
8
8
|
|
9
9
|
def option_data_filesystem
|
10
10
|
[
|
11
|
+
'GFARM2FS_COMMAND',
|
11
12
|
'GFARM2FS_OPTION',
|
12
13
|
'GFARM2FS_DEBUG',
|
13
14
|
['GFARM2FS_DEBUG_WAIT', proc{|v| v ? v.to_i : 1}],
|
@@ -21,7 +22,6 @@ module Pwrake
|
|
21
22
|
end
|
22
23
|
|
23
24
|
def set_filesystem_option
|
24
|
-
@filesystem = 'gfarm'
|
25
25
|
GfarmPath.subdir = self['GFARM_SUBDIR']
|
26
26
|
@worker_option = {
|
27
27
|
:log_dir => self['LOG_DIR'],
|
@@ -33,6 +33,7 @@ module Pwrake
|
|
33
33
|
:shared_directory => "GfarmDirectory",
|
34
34
|
:base_dir => self['GFARM_BASEDIR']+"/"+self['GFARM_PREFIX'],
|
35
35
|
:work_dir => GfarmPath.pwd.to_s,
|
36
|
+
:gfarm2fs_command => self['GFARM2FS_COMMAND'],
|
36
37
|
:gfarm2fs_option => self['GFARM2FS_OPTION'],
|
37
38
|
:gfarm2fs_debug => self['GFARM2FS_DEBUG'],
|
38
39
|
:gfarm2fs_debug_wait => self['GFARM2FS_DEBUG_WAIT'],
|
@@ -30,13 +30,12 @@ module Pwrake
|
|
30
30
|
def pop
|
31
31
|
case @prio
|
32
32
|
when 0
|
33
|
-
|
33
|
+
@que.shift
|
34
34
|
when 1
|
35
|
-
|
35
|
+
@que.pop
|
36
36
|
when 2
|
37
|
-
|
37
|
+
@que.delete_at(rand(@que.size))
|
38
38
|
end
|
39
|
-
return x
|
40
39
|
end
|
41
40
|
|
42
41
|
alias shift pop
|
@@ -9,8 +9,11 @@ module Pwrake
|
|
9
9
|
def initialize(queue_class, hostinfo_by_id, group_map=nil)
|
10
10
|
@queue_class = Pwrake.const_get(queue_class)
|
11
11
|
@hostinfo_by_id = hostinfo_by_id
|
12
|
+
@lock = Monitor.new
|
12
13
|
@q_no_action = NoActionQueue.new
|
13
14
|
@q_reserved = Hash.new
|
15
|
+
@nenq = 0
|
16
|
+
@ndeq = 0
|
14
17
|
def @q_reserved.first
|
15
18
|
super.last
|
16
19
|
end
|
@@ -40,27 +43,39 @@ module Pwrake
|
|
40
43
|
end
|
41
44
|
|
42
45
|
def enq(tw)
|
46
|
+
@lock.synchronize do
|
43
47
|
if tw.nil? || tw.actions.empty?
|
44
48
|
@q_no_action.push(tw)
|
45
49
|
else
|
46
50
|
@q.enq_impl(tw)
|
47
51
|
end
|
52
|
+
@nenq += 1
|
53
|
+
end
|
48
54
|
end
|
49
55
|
|
50
56
|
def deq_task(&block)
|
51
|
-
|
57
|
+
@lock.synchronize do
|
58
|
+
if @nenq > 0
|
59
|
+
Log.debug "deq_task nenq=#{@nenq}:"+(empty? ? " (empty)" : "\n"+inspect_q)
|
60
|
+
@nenq = 0
|
61
|
+
end
|
52
62
|
deq_noaction_task(&block)
|
53
63
|
deq_reserve(&block)
|
54
64
|
@q.deq_start
|
55
65
|
unless @q.empty?
|
56
66
|
@q.turns.each{|turn| deq_turn(turn,&block) }
|
57
67
|
end
|
68
|
+
if @ndeq > 0
|
69
|
+
Log.debug "deq_task ndeq=#{@ndeq}:"+(empty? ? " (empty)" : "\n"+inspect_q)
|
70
|
+
@ndeq = 0
|
71
|
+
end
|
72
|
+
end
|
58
73
|
end
|
59
74
|
|
60
75
|
def deq_noaction_task(&block)
|
61
76
|
while tw = @q_no_action.shift
|
62
|
-
Log.debug "deq_noaction: #{tw.name}"
|
63
77
|
yield(tw)
|
78
|
+
@ndeq += 1
|
64
79
|
end
|
65
80
|
end
|
66
81
|
|
@@ -72,6 +87,7 @@ module Pwrake
|
|
72
87
|
@q_reserved.delete(host_info)
|
73
88
|
Log.debug "deq_reserve: #{tw.name} n_use_cores=#{n_core}"
|
74
89
|
yield(tw,host_info,n_core)
|
90
|
+
@ndeq += 1
|
75
91
|
end
|
76
92
|
end
|
77
93
|
end
|
@@ -89,6 +105,7 @@ module Pwrake
|
|
89
105
|
Log.debug "deq: #{tw.name} n_use_cores=#{n_core}"
|
90
106
|
yield(tw,host_info,n_core)
|
91
107
|
count += 1
|
108
|
+
@ndeq += 1
|
92
109
|
else
|
93
110
|
@q_reserved[host_info] = tw
|
94
111
|
Log.debug "reserve host: #{host_info.name} for #{tw.name} (#{n_core} cores)"
|
@@ -93,6 +93,7 @@ module Pwrake
|
|
93
93
|
|
94
94
|
t_end = (a.last)[0]
|
95
95
|
|
96
|
+
begin
|
96
97
|
if system("which gnuplot >/dev/null 2>&1")
|
97
98
|
IO.popen("gnuplot","r+") do |f|
|
98
99
|
f.print "
|
@@ -112,6 +113,10 @@ plot '-' w l notitle
|
|
112
113
|
end
|
113
114
|
end
|
114
115
|
end
|
116
|
+
rescue => exc
|
117
|
+
$stderr.puts exc
|
118
|
+
$stderr.puts exc.backtrace.join("\n")
|
119
|
+
end
|
115
120
|
|
116
121
|
#puts "Parallelism plot: #{fimg}"
|
117
122
|
fimg
|
@@ -151,6 +156,7 @@ plot '-' w l notitle
|
|
151
156
|
|
152
157
|
t_end = (a.last)[0]
|
153
158
|
|
159
|
+
begin
|
154
160
|
if system("which gnuplot >/dev/null 2>&1")
|
155
161
|
IO.popen("gnuplot","r+") do |f|
|
156
162
|
f.print "
|
@@ -177,6 +183,10 @@ plot '-' w l axis x1y1 title 'parallelism', '-' w l axis x1y2 title 'exec/sec'
|
|
177
183
|
end
|
178
184
|
end
|
179
185
|
end
|
186
|
+
rescue => exc
|
187
|
+
$stderr.puts exc
|
188
|
+
$stderr.puts exc.backtrace.join("\n")
|
189
|
+
end
|
180
190
|
|
181
191
|
#puts "Parallelism plot: #{fimg}"
|
182
192
|
fimg
|
@@ -341,6 +351,7 @@ set ylabel '# of cores'
|
|
341
351
|
grid << a
|
342
352
|
end
|
343
353
|
|
354
|
+
begin
|
344
355
|
if system("which gnuplot >/dev/null 2>&1")
|
345
356
|
IO.popen("gnuplot","r+") do |f|
|
346
357
|
f.puts "
|
@@ -376,6 +387,10 @@ set format y ''
|
|
376
387
|
f.printf "e\n"
|
377
388
|
end
|
378
389
|
end
|
390
|
+
rescue => exc
|
391
|
+
$stderr.puts exc
|
392
|
+
$stderr.puts exc.backtrace.join("\n")
|
393
|
+
end
|
379
394
|
fimg
|
380
395
|
end
|
381
396
|
|
@@ -2,15 +2,60 @@ module Pwrake
|
|
2
2
|
|
3
3
|
module FileTaskAlgorithm
|
4
4
|
|
5
|
+
def needed?
|
6
|
+
!_exist?(name) || out_of_date?(timestamp) || @application.options.build_all
|
7
|
+
end
|
8
|
+
|
5
9
|
# Cache time stamp to reduce load on file system.
|
6
10
|
def timestamp
|
7
11
|
@file_mtime ||
|
8
|
-
if
|
9
|
-
@file_mtime =
|
12
|
+
if _exist?(name)
|
13
|
+
@file_mtime = _mtime(name.to_s)
|
10
14
|
else
|
11
15
|
Rake::LATE
|
12
16
|
end
|
13
17
|
end
|
14
18
|
|
19
|
+
private
|
20
|
+
|
21
|
+
@@t_mtime = 0
|
22
|
+
@@n_mtime = 0
|
23
|
+
@@l_mtime = 100
|
24
|
+
@@c_mtime = 0
|
25
|
+
@@t_exist = 0
|
26
|
+
@@n_exist = 0
|
27
|
+
@@l_exist = 100
|
28
|
+
@@c_exist = 0
|
29
|
+
|
30
|
+
def _mtime(name)
|
31
|
+
t = Pwrake.clock
|
32
|
+
m = File.mtime(name.to_s)
|
33
|
+
@@t_mtime = @@t_mtime + (Pwrake.clock-t)
|
34
|
+
@@n_mtime += 1
|
35
|
+
if @@n_mtime >= 100
|
36
|
+
Log.debug('mtime: mean=%.9f s (%d times)'%[@@t_mtime/@@n_mtime,@@n_mtime])
|
37
|
+
@@t_mtime = 0
|
38
|
+
@@n_mtime = 0
|
39
|
+
@@c_mtime += 1
|
40
|
+
@@l_mtime = 1000 if @@c_mtime == 10
|
41
|
+
end
|
42
|
+
m
|
43
|
+
end
|
44
|
+
|
45
|
+
def _exist?(name)
|
46
|
+
t = Pwrake.clock
|
47
|
+
e = File.exist?(name.to_s)
|
48
|
+
@@t_exist = @@t_exist + (Pwrake.clock-t)
|
49
|
+
@@n_exist += 1
|
50
|
+
if @@n_exist >= @@l_exist
|
51
|
+
Log.debug('exist: mean=%.9f s (%d times)'%[@@t_exist/@@n_exist,@@n_exist])
|
52
|
+
@@t_exist = 0
|
53
|
+
@@n_exist = 0
|
54
|
+
@@c_exist += 1
|
55
|
+
@@l_exist = 1000 if @@c_exist == 10
|
56
|
+
end
|
57
|
+
e
|
58
|
+
end
|
59
|
+
|
15
60
|
end
|
16
61
|
end
|
@@ -12,20 +12,24 @@ module Pwrake
|
|
12
12
|
|
13
13
|
def wrapper
|
14
14
|
if @wrapper.nil?
|
15
|
-
|
15
|
+
Log.debug "TaskWrapper is not defined for #{self.class}[#{name}]"
|
16
|
+
@wrapper = TaskWrapper.new(self)
|
16
17
|
end
|
17
18
|
@wrapper
|
18
19
|
end
|
19
20
|
|
20
21
|
def pw_search_tasks(args)
|
21
|
-
Log.debug "#{self.class}#pw_search_tasks start,
|
22
|
+
Log.debug "#{self.class}[#{name}]#pw_search_tasks start, args=#{args.inspect}"
|
23
|
+
if application.options.trace
|
24
|
+
application.trace "** Search #{name}#{format_search_flags}"
|
25
|
+
end
|
22
26
|
cl = Pwrake.clock
|
23
27
|
TaskWrapper.clear_rank
|
24
28
|
task_args = TaskArguments.new(arg_names, args)
|
25
29
|
# not synchronize owing to fiber
|
26
30
|
search_with_call_chain(nil, task_args, InvocationChain::EMPTY)
|
27
31
|
#
|
28
|
-
Log.debug "#{self.class}#pw_search_tasks end
|
32
|
+
Log.debug "#{self.class}[#{name}]#pw_search_tasks end t=%.6f" % (Pwrake.clock-cl)
|
29
33
|
end
|
30
34
|
|
31
35
|
# Same as search, but explicitly pass a call chain to detect
|
@@ -33,11 +37,6 @@ module Pwrake
|
|
33
37
|
def search_with_call_chain(subseq, task_args, invocation_chain) # :nodoc:
|
34
38
|
new_chain = InvocationChain.append(self, invocation_chain)
|
35
39
|
@lock.synchronize do
|
36
|
-
if application.options.trace
|
37
|
-
#Log.debug "** Search #{name}#{format_search_flags}"
|
38
|
-
application.trace "** Search #{name}#{format_search_flags}"
|
39
|
-
end
|
40
|
-
|
41
40
|
return true if @already_finished # <<--- competition !!!
|
42
41
|
@subsequents ||= []
|
43
42
|
@subsequents << subseq if subseq # <<--- competition !!!
|
@@ -89,14 +89,10 @@ module Pwrake
|
|
89
89
|
end
|
90
90
|
|
91
91
|
case @ncore
|
92
|
-
when Rational
|
93
|
-
if @ncore > 0 && @ncore <= 1
|
94
|
-
return [(@ncore*ppn).to_i, 1].min
|
95
|
-
end
|
96
|
-
when 1-ppn..ppn
|
97
|
-
return (@ncore>0) ? @ncore : @ncore+ppn
|
98
92
|
when nil
|
99
93
|
return 1
|
94
|
+
when 1-ppn..ppn
|
95
|
+
return (@ncore>0) ? @ncore : @ncore+ppn
|
100
96
|
end
|
101
97
|
|
102
98
|
m = "ncore=#{@ncore} is out of range of cores per node: #{ppn}"
|
@@ -10,6 +10,7 @@ module Pwrake
|
|
10
10
|
@@current_id = 1
|
11
11
|
@@task_logger = nil
|
12
12
|
@@instances = []
|
13
|
+
LOCK = Monitor.new
|
13
14
|
|
14
15
|
def initialize(task,task_args=nil)
|
15
16
|
@task = task
|
@@ -68,8 +69,10 @@ module Pwrake
|
|
68
69
|
end
|
69
70
|
|
70
71
|
def self.clear_rank
|
71
|
-
|
72
|
-
|
72
|
+
LOCK.synchronize do
|
73
|
+
Log.debug "#{self}.clear_rank"
|
74
|
+
@@instances.each{|w| w.clear_rank}
|
75
|
+
end
|
73
76
|
end
|
74
77
|
|
75
78
|
def preprocess
|
@@ -87,15 +90,12 @@ module Pwrake
|
|
87
90
|
|
88
91
|
def postprocess(postproc)
|
89
92
|
@executed = true if !@task.actions.empty?
|
90
|
-
#tm_taskend = Pwrake.clock
|
91
93
|
if is_file_task?
|
92
|
-
#t = Pwrake.clock
|
93
94
|
if File.exist?(name)
|
94
95
|
@file_stat = File::Stat.new(name)
|
95
96
|
@location = postproc.run(self)
|
96
97
|
end
|
97
98
|
end
|
98
|
-
#Log.debug "postprocess time=#{Pwrake.clock-tm_taskend}"
|
99
99
|
log_task
|
100
100
|
end
|
101
101
|
|
@@ -129,7 +129,12 @@ module Pwrake
|
|
129
129
|
end
|
130
130
|
return if !@@task_logger
|
131
131
|
#
|
132
|
-
|
132
|
+
if @clock_start
|
133
|
+
elap = @clock_end - @clock_start
|
134
|
+
else
|
135
|
+
Loag.debug "@clock_start is not defined for #{@task.class}[#{name}]"
|
136
|
+
elap = 0
|
137
|
+
end
|
133
138
|
if has_output_file?
|
134
139
|
RANK_STAT.add_sample(rank,elap)
|
135
140
|
end
|
@@ -231,6 +236,7 @@ module Pwrake
|
|
231
236
|
|
232
237
|
def suggest_location
|
233
238
|
if has_input_file? && @suggest_location.nil?
|
239
|
+
cl = Pwrake.clock
|
234
240
|
@suggest_location = []
|
235
241
|
loc_fsz = Hash.new(0)
|
236
242
|
prerequisites.each do |preq|
|
@@ -246,18 +252,19 @@ module Pwrake
|
|
246
252
|
#Log.debug "input=#{prerequisites.join('|')}"
|
247
253
|
if !loc_fsz.empty?
|
248
254
|
half_max_fsz = loc_fsz.values.max / 2
|
249
|
-
Log.debug "loc_fsz=#{loc_fsz.inspect} half_max_fsz=#{half_max_fsz}"
|
250
255
|
loc_fsz.each do |h,sz|
|
251
256
|
if sz > half_max_fsz
|
252
257
|
@suggest_location << h
|
253
258
|
end
|
254
259
|
end
|
260
|
+
Log.debug "locate:%.6f #{name} loc_fsz=#{loc_fsz.inspect} half_max_fsz=#{half_max_fsz} suggest=#{@suggest_location.inspect}"%(Pwrake.clock-cl)
|
255
261
|
end
|
256
262
|
end
|
257
263
|
@suggest_location
|
258
264
|
end
|
259
265
|
|
260
266
|
def rank
|
267
|
+
LOCK.synchronize do
|
261
268
|
if @rank.nil?
|
262
269
|
if subsequents.nil? || subsequents.empty?
|
263
270
|
@rank = 0
|
@@ -265,8 +272,12 @@ module Pwrake
|
|
265
272
|
max_rank = 0
|
266
273
|
subsequents.each do |subsq|
|
267
274
|
r = subsq.wrapper.rank
|
268
|
-
if
|
269
|
-
max_rank
|
275
|
+
if r
|
276
|
+
if max_rank < r
|
277
|
+
max_rank = r
|
278
|
+
end
|
279
|
+
else
|
280
|
+
Log.warn "subsq.wrapper.rank=#{rank.inspect}"
|
270
281
|
end
|
271
282
|
end
|
272
283
|
if has_output_file?
|
@@ -276,9 +287,9 @@ module Pwrake
|
|
276
287
|
end
|
277
288
|
@rank = max_rank + step
|
278
289
|
end
|
279
|
-
Log.debug "Task[#{name}] rank=#{@rank.inspect}"
|
280
290
|
end
|
281
291
|
@rank
|
292
|
+
end
|
282
293
|
end
|
283
294
|
|
284
295
|
def clear_rank
|
@@ -348,7 +359,7 @@ module Pwrake
|
|
348
359
|
else
|
349
360
|
@priority = 0
|
350
361
|
end
|
351
|
-
Log.debug "
|
362
|
+
Log.debug "task=#{name} priority=#{@priority} sum_file_size=#{sum_sz}"
|
352
363
|
end
|
353
364
|
@priority || 0
|
354
365
|
end
|
data/lib/pwrake/version.rb
CHANGED
@@ -16,6 +16,7 @@ module Pwrake
|
|
16
16
|
@@prefix = opts[:base_dir]
|
17
17
|
@@work_dir = opts[:work_dir]
|
18
18
|
@@log_dir = opts[:log_dir]
|
19
|
+
@@gfarm2fs_command = opts[:gfarm2fs_command] || 'gfarm2fs'
|
19
20
|
@@gfarm2fs_option = opts[:gfarm2fs_option]
|
20
21
|
@@gfarm2fs_debug = opts[:gfarm2fs_debug]
|
21
22
|
@@gfarm2fs_debug_wait = opts[:gfarm2fs_debug_wait]
|
@@ -56,13 +57,14 @@ module Pwrake
|
|
56
57
|
|
57
58
|
def open
|
58
59
|
FileUtils.mkdir_p @gfarm_mountpoint
|
60
|
+
@log.info "mkdir -p #{@@hostname}:#{@gfarm_mountpoint}"
|
59
61
|
path = @log.path
|
60
62
|
begin
|
61
63
|
if @@gfarm2fs_debug && path
|
62
64
|
f = path+("gfarm2fs-"+@@hostname+"-"+@suffix)
|
63
|
-
spawn_cmd "
|
65
|
+
spawn_cmd "#{@@gfarm2fs_command} #{@@gfarm2fs_option} -d #{@gfarm_mountpoint} > #{f} 2>&1 & sleep #{@@gfarm2fs_debug_wait}"
|
64
66
|
else
|
65
|
-
spawn_cmd "
|
67
|
+
spawn_cmd "#{@@gfarm2fs_command} #{@@gfarm2fs_option} #{@gfarm_mountpoint}"
|
66
68
|
end
|
67
69
|
rescue => exc
|
68
70
|
sleep 1
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: pwrake
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.3.
|
4
|
+
version: 2.3.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Masahiro TANAKA
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2019-
|
11
|
+
date: 2019-11-26 00:00:00.000000000 Z
|
12
12
|
dependencies: []
|
13
13
|
description: Parallel and distributed Rake for workflow execution on multicores, clusters,
|
14
14
|
clouds using SSH. It has locality-aware scheduling designed for Gfarm file system.
|
@@ -58,7 +58,6 @@ files:
|
|
58
58
|
- lib/pwrake/option/host_map.rb
|
59
59
|
- lib/pwrake/option/option.rb
|
60
60
|
- lib/pwrake/option/option_default_filesystem.rb
|
61
|
-
- lib/pwrake/option/option_gfarm.rb
|
62
61
|
- lib/pwrake/option/option_gfarm2fs.rb
|
63
62
|
- lib/pwrake/queue/locality_aware_queue.rb
|
64
63
|
- lib/pwrake/queue/no_action_queue.rb
|
@@ -1 +0,0 @@
|
|
1
|
-
lib/pwrake/option/option_gfarm2fs.rb
|