pwrake 2.3.0 → 2.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +11 -10
- data/lib/pwrake/branch/branch.rb +24 -0
- data/lib/pwrake/branch/communicator.rb +16 -26
- data/lib/pwrake/gfarm/gfarm_postprocess.rb +1 -1
- data/lib/pwrake/master/master.rb +9 -6
- data/lib/pwrake/master/master_application.rb +12 -7
- data/lib/pwrake/nbio.rb +11 -5
- data/lib/pwrake/option/host_map.rb +13 -2
- data/lib/pwrake/option/option.rb +18 -9
- data/lib/pwrake/option/option_default_filesystem.rb +0 -1
- data/lib/pwrake/option/option_gfarm2fs.rb +2 -1
- data/lib/pwrake/queue/no_action_queue.rb +3 -4
- data/lib/pwrake/queue/task_queue.rb +19 -2
- data/lib/pwrake/report/parallelism.rb +15 -0
- data/lib/pwrake/task/file_task_algorithm.rb +47 -2
- data/lib/pwrake/task/task_algorithm.rb +7 -8
- data/lib/pwrake/task/task_property.rb +2 -6
- data/lib/pwrake/task/task_wrapper.rb +22 -11
- data/lib/pwrake/version.rb +1 -1
- data/lib/pwrake/worker/gfarm_directory.rb +4 -2
- metadata +2 -3
- data/lib/pwrake/option/option_gfarm.rb +0 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e666a4e98f080f07f41475896df0a2d115805b8af57e3541b8d898c60a8122c9
|
4
|
+
data.tar.gz: a320f42590130bab37518996ed11496d8b107ba3162a8458c896aa6daa3b322d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: cd4a9b2641c220d2a1265ccdf76ed87f2594ce0d498f521a24d67a6428d5c757129c31408f9fb9058b4e184aa2a09b01096edb17e2b42c5d5f634bd5b61125dc
|
7
|
+
data.tar.gz: 92f066ae2246bdd86097795432124d129a21e60e27d96515fe16059a05e14c0c7d4a1eefff85e574860626f600efb6c8698e1969824bcffdd5429c0905cc75cf
|
data/README.md
CHANGED
@@ -93,8 +93,8 @@ In this case, you need the rehash of command paths:
|
|
93
93
|
-L, --log, --log-dir [DIRECTORY] [Pw] Write log to DIRECTORY
|
94
94
|
--ssh-opt, --ssh-option OPTION
|
95
95
|
[Pw] Option passed to SSH
|
96
|
-
--filesystem FILESYSTEM [Pw] Specify FILESYSTEM (nfs|
|
97
|
-
--gfarm [Pw]
|
96
|
+
--filesystem FILESYSTEM [Pw] Specify FILESYSTEM (nfs|gfarm2fs)
|
97
|
+
--gfarm [Pw] (obsolete; Start pwrake on Gfarm FS)
|
98
98
|
-A, --disable-affinity [Pw] Turn OFF affinity (AFFINITY=off)
|
99
99
|
-S, --disable-steal [Pw] Turn OFF task steal
|
100
100
|
-d, --debug [Pw] Output Debug messages
|
@@ -151,6 +151,7 @@ In this case, you need the rehash of command paths:
|
|
151
151
|
GFARM_PREFIX default="pwrake_$USER"
|
152
152
|
GFARM_SUBDIR default='/'
|
153
153
|
MAX_GFWHERE_WORKER default=8
|
154
|
+
GFARM2FS_COMMAND default='gfarm2fs'
|
154
155
|
GFARM2FS_OPTION default=""
|
155
156
|
GFARM2FS_DEBUG default=false
|
156
157
|
GFARM2FS_DEBUG_WAIT default=1
|
@@ -177,14 +178,14 @@ end
|
|
177
178
|
|
178
179
|
Properties (The leftmost item is default):
|
179
180
|
|
180
|
-
ncore=integer
|
181
|
-
exclusive=no|yes
|
182
|
-
reserve=no|yes
|
183
|
-
allow=hostname
|
184
|
-
deny=hostname
|
185
|
-
order=deny,allow|
|
186
|
-
steal=yes|no
|
187
|
-
retry=integer
|
181
|
+
ncore=integer|rational - The number of cores used by this task.
|
182
|
+
exclusive=no|yes - Exclusively execute this task in a single node.
|
183
|
+
reserve=no|yes - Gives higher priority to this task if ncore>1. (reserve a host)
|
184
|
+
allow=hostname - Allow this host to execute this task. (accepts wild card)
|
185
|
+
deny=hostname - Deny this host to execute this task. (accepts wild card)
|
186
|
+
order=deny,allow|a llow,deny - The order of evaluation.
|
187
|
+
steal=yes|no - Allow task stealing for this task.
|
188
|
+
retry=integer - The number of retry for this task.
|
188
189
|
|
189
190
|
## Note for Gfarm
|
190
191
|
|
data/lib/pwrake/branch/branch.rb
CHANGED
@@ -42,6 +42,7 @@ module Pwrake
|
|
42
42
|
setup_shell
|
43
43
|
setup_fiber
|
44
44
|
setup_master_channel
|
45
|
+
setup_search_thread
|
45
46
|
@cs.run("task execution")
|
46
47
|
Log.debug "Branch#run end"
|
47
48
|
end
|
@@ -193,6 +194,29 @@ module Pwrake
|
|
193
194
|
end.resume
|
194
195
|
end
|
195
196
|
|
197
|
+
def setup_search_thread
|
198
|
+
@search_que = Queue.new
|
199
|
+
Thread.new do
|
200
|
+
while a = @search_que.deq
|
201
|
+
t,args,w = *a
|
202
|
+
t.pw_search_tasks(args)
|
203
|
+
w.puts(t.name)
|
204
|
+
end
|
205
|
+
end
|
206
|
+
end
|
207
|
+
|
208
|
+
def invoke(t,args)
|
209
|
+
Log.debug "Branch#invoke start: #{t.class}[#{t.name}]"
|
210
|
+
r,w = IO.pipe
|
211
|
+
rd = NBIO::Reader.new(@selector,r)
|
212
|
+
@search_que.enq([t,args,w])
|
213
|
+
task_name = rd.get_line.chomp
|
214
|
+
if t.name != task_name
|
215
|
+
raise "t.name=#{t.name} != task_name=#{task_name}"
|
216
|
+
end
|
217
|
+
Log.debug "Branch#invoke end: #{t.class}[#{t.name}]"
|
218
|
+
end
|
219
|
+
|
196
220
|
def kill(sig="INT")
|
197
221
|
Log.warn "Branch#kill #{sig}"
|
198
222
|
@cs.kill(sig)
|
@@ -90,7 +90,6 @@ class Communicator
|
|
90
90
|
|
91
91
|
sel = @set.selector
|
92
92
|
@reader = NBIO::MultiReader.new(sel,@ior)
|
93
|
-
@rd_err = NBIO::Reader.new(sel,@ioe)
|
94
93
|
@writer = NBIO::Writer.new(sel,@iow)
|
95
94
|
@handler = NBIO::Handler.new(@reader,@writer,@host)
|
96
95
|
|
@@ -146,48 +145,39 @@ class Communicator
|
|
146
145
|
end
|
147
146
|
|
148
147
|
def dropout(exc=nil)
|
149
|
-
#
|
150
|
-
err_out = []
|
148
|
+
# Finish worker
|
151
149
|
begin
|
152
150
|
finish_shells
|
153
151
|
if @handler
|
154
152
|
@handler.exit
|
155
153
|
@handler = nil
|
156
154
|
end
|
157
|
-
if @rd_err
|
158
|
-
while s = @rd_err.get_line
|
159
|
-
err_out << s
|
160
|
-
end
|
161
|
-
end
|
162
155
|
rescue => e
|
163
156
|
m = Log.bt(e)
|
164
|
-
|
157
|
+
$stderr.puts(m)
|
165
158
|
Log.error(m)
|
166
159
|
end
|
167
|
-
# Error output
|
168
|
-
if
|
169
|
-
|
170
|
-
|
160
|
+
# Error output from worker
|
161
|
+
if @ioe
|
162
|
+
err_out = ["standard error from worker:"]
|
163
|
+
while s = @ioe.gets
|
164
|
+
err_out << s.chomp
|
165
|
+
end
|
166
|
+
if err_out.size > 1
|
167
|
+
m = err_out.join("\n ")
|
168
|
+
$stderr.puts(m)
|
169
|
+
Log.error(m)
|
170
|
+
end
|
171
171
|
end
|
172
|
-
# Exception
|
172
|
+
# Exception message
|
173
173
|
if exc
|
174
174
|
m = Log.bt(exc)
|
175
|
-
|
176
|
-
Log.error
|
175
|
+
$stderr.puts(m)
|
176
|
+
Log.error(m)
|
177
177
|
end
|
178
178
|
ensure
|
179
179
|
@set.delete(self)
|
180
180
|
end
|
181
181
|
|
182
|
-
def finish
|
183
|
-
@iow.close
|
184
|
-
while s=@ior.gets
|
185
|
-
puts "out=#{s.chomp}"
|
186
|
-
end
|
187
|
-
while s=@ioe.gets
|
188
|
-
puts "err=#{s.chomp}"
|
189
|
-
end
|
190
|
-
end
|
191
|
-
|
192
182
|
end
|
193
183
|
end
|
data/lib/pwrake/master/master.rb
CHANGED
@@ -226,11 +226,11 @@ module Pwrake
|
|
226
226
|
if ending?
|
227
227
|
@post_pool.finish # need?
|
228
228
|
else
|
229
|
-
setup_fiber
|
229
|
+
setup_fiber
|
230
230
|
end
|
231
231
|
end
|
232
232
|
|
233
|
-
def setup_fiber
|
233
|
+
def setup_fiber
|
234
234
|
@host_fail = @option["HOST_FAILURE"]
|
235
235
|
create_fiber(@hdl_set) do |hdl|
|
236
236
|
while s = hdl.get_line
|
@@ -295,15 +295,17 @@ module Pwrake
|
|
295
295
|
$stderr.puts(s)
|
296
296
|
end
|
297
297
|
end
|
298
|
-
Log.debug "Master#
|
298
|
+
Log.debug "Master#setup_fiber: end of fiber"
|
299
299
|
end
|
300
|
+
|
300
301
|
if !ending?
|
301
302
|
Log.debug "@selector.run begin"
|
302
303
|
@selector.run
|
303
304
|
Log.debug "@selector.run end"
|
305
|
+
else
|
306
|
+
Log.debug "@selector.run skipped"
|
304
307
|
end
|
305
308
|
@post_pool.finish
|
306
|
-
Log.debug "Master#invoke: end of task=#{t.name}"
|
307
309
|
end
|
308
310
|
|
309
311
|
def send_task_to_idle_core
|
@@ -361,7 +363,7 @@ module Pwrake
|
|
361
363
|
|
362
364
|
def task_end(tw,host_info)
|
363
365
|
return if host_info.nil?
|
364
|
-
host_info.idle(tw.n_used_cores)
|
366
|
+
host_info.idle(tw.n_used_cores||1)
|
365
367
|
if host_info.retired?
|
366
368
|
# all retired
|
367
369
|
Log.warn("retired host:#{host_info.name} because all core retired")
|
@@ -396,8 +398,9 @@ module Pwrake
|
|
396
398
|
Log.debug " @task_queue.empty?=#{@task_queue.empty?}" if @task_queue.empty?
|
397
399
|
Log.debug " @hostinfo_by_id.empty?=#{@hostinfo_by_id.empty?}" if @hostinfo_by_id.empty?
|
398
400
|
Log.debug " @hostinfo_by_taskname.keys=#{@hostinfo_by_taskname.keys.inspect}"
|
401
|
+
Log.debug " @post_pool.empty?=#{@post_pool.empty?}" if @post_pool.empty?
|
399
402
|
end
|
400
|
-
@hostinfo_by_taskname.empty?
|
403
|
+
@hostinfo_by_taskname.empty? && @post_pool.empty?
|
401
404
|
else
|
402
405
|
false
|
403
406
|
end
|
@@ -26,13 +26,13 @@ module Pwrake
|
|
26
26
|
@master.setup_branches
|
27
27
|
load_rakefile
|
28
28
|
begin
|
29
|
-
Log.debug "init:
|
29
|
+
Log.debug "init: %.6f sec" % (Pwrake.clock-t)
|
30
30
|
t = Pwrake.clock
|
31
31
|
top_level
|
32
|
-
Log.debug "main:
|
32
|
+
Log.debug "main: %.6f sec" % (Pwrake.clock-t)
|
33
33
|
t = Pwrake.clock
|
34
34
|
@failed = @master.finish
|
35
|
-
Log.debug "finish:
|
35
|
+
Log.debug "finish: %.6f sec" % (Pwrake.clock-t)
|
36
36
|
rescue SystemExit => e
|
37
37
|
@failed = true
|
38
38
|
rescue Exception => e
|
@@ -84,7 +84,11 @@ module Pwrake
|
|
84
84
|
end
|
85
85
|
|
86
86
|
def invoke(t,*args)
|
87
|
-
@master.
|
87
|
+
if Thread.current == @master.thread
|
88
|
+
@branch.invoke(t,args)
|
89
|
+
else
|
90
|
+
@master.invoke(t,args)
|
91
|
+
end
|
88
92
|
end
|
89
93
|
|
90
94
|
def standard_rake_options
|
@@ -139,14 +143,15 @@ module Pwrake
|
|
139
143
|
options.ssh_option = value
|
140
144
|
}
|
141
145
|
],
|
142
|
-
['--filesystem FILESYSTEM', "[Pw] Specify FILESYSTEM (nfs|
|
146
|
+
['--filesystem FILESYSTEM', "[Pw] Specify FILESYSTEM (nfs|gfarm2fs)",
|
143
147
|
lambda { |value|
|
144
148
|
options.filesystem = value
|
145
149
|
}
|
146
150
|
],
|
147
|
-
['--gfarm', "[Pw] FILESYSTEM=
|
151
|
+
['--gfarm', "[Pw] FILESYSTEM=gfarm2fs",
|
148
152
|
lambda { |value|
|
149
|
-
|
153
|
+
warn 'The --gfarm option may not work correctly. Execute pwrake under gfarm2fs.'
|
154
|
+
options.filesystem = "gfarm2fs"
|
150
155
|
}
|
151
156
|
],
|
152
157
|
['-A', '--disable-affinity', "[Pw] Turn OFF affinity (AFFINITY=off)",
|
data/lib/pwrake/nbio.rb
CHANGED
@@ -506,8 +506,17 @@ module NBIO
|
|
506
506
|
end
|
507
507
|
@exited = true
|
508
508
|
exit_msg = "exited"
|
509
|
-
|
510
|
-
|
509
|
+
begin
|
510
|
+
@writer.put_line "exit"
|
511
|
+
rescue Errno::EPIPE => e
|
512
|
+
m = "#{e} in #{self.class}.exit iow=#{iow.inspect}"
|
513
|
+
if defined? Log
|
514
|
+
Log.warn m
|
515
|
+
else
|
516
|
+
$stderr.puts m
|
517
|
+
end
|
518
|
+
return
|
519
|
+
end
|
511
520
|
Log.debug "Handler#exit: end: @writer.put_line \"exit\"" if defined? Log
|
512
521
|
#
|
513
522
|
if @reader.class == Reader # MultiReader not work
|
@@ -518,9 +527,6 @@ module NBIO
|
|
518
527
|
return if line == exit_msg
|
519
528
|
end
|
520
529
|
end
|
521
|
-
rescue Errno::EPIPE => e
|
522
|
-
Log.error "Errno::EPIPE in #{self.class}.exit iow=#{iow.inspect}\n"+
|
523
|
-
e.backtrace.join("\n") if defined? Log
|
524
530
|
end
|
525
531
|
|
526
532
|
def halt
|
@@ -13,7 +13,7 @@ module Pwrake
|
|
13
13
|
|
14
14
|
def initialize(name,id,ncore,weight,group=nil)
|
15
15
|
@name = name
|
16
|
-
@ncore = ncore
|
16
|
+
@ncore = ncore || 1
|
17
17
|
@weight = weight || 1.0
|
18
18
|
@group = group || 0
|
19
19
|
@id = id
|
@@ -32,6 +32,17 @@ module Pwrake
|
|
32
32
|
attr_reader :continuous_fail
|
33
33
|
attr_accessor :idle_cores
|
34
34
|
|
35
|
+
def add_line(ncore=nil,weight=nil,group=nil)
|
36
|
+
ncore ||= 1
|
37
|
+
weight ||= 1.0
|
38
|
+
group ||= 0
|
39
|
+
if @group != group
|
40
|
+
raise "different group=#{group} for host=#{@name}"
|
41
|
+
end
|
42
|
+
@weight = (@weight*@ncore + weight*ncore)/(@ncore+ncore)
|
43
|
+
@ncore += ncore
|
44
|
+
end
|
45
|
+
|
35
46
|
def local?
|
36
47
|
ipa = IPSocket.getaddress(@name)
|
37
48
|
HostInfo.local_ip.include?(ipa)
|
@@ -231,7 +242,7 @@ module Pwrake
|
|
231
242
|
#weight = (weight || 1).to_f
|
232
243
|
group &&= group.to_i
|
233
244
|
if host_info = @by_name[host]
|
234
|
-
|
245
|
+
host_info.add_line(ncore,weight,group)
|
235
246
|
else
|
236
247
|
id = @by_id.size
|
237
248
|
host_info = HostInfo.new(host,id,ncore,weight,group)
|
data/lib/pwrake/option/option.rb
CHANGED
@@ -61,11 +61,11 @@ module Pwrake
|
|
61
61
|
# ----------------------------------------------------------
|
62
62
|
|
63
63
|
def init_filesystem
|
64
|
-
@filesystem = Rake.application.options.filesystem
|
65
|
-
@filesystem
|
66
|
-
|
67
|
-
require "pwrake/option/
|
68
|
-
|
64
|
+
@filesystem = Rake.application.options.filesystem || mount_type
|
65
|
+
case @filesystem
|
66
|
+
when 'gfarm2fs'
|
67
|
+
require "pwrake/option/option_gfarm2fs"
|
68
|
+
else
|
69
69
|
require "pwrake/option/option_default_filesystem"
|
70
70
|
end
|
71
71
|
end
|
@@ -73,15 +73,15 @@ module Pwrake
|
|
73
73
|
attr_reader :worker_option
|
74
74
|
attr_reader :queue_class
|
75
75
|
|
76
|
-
def mount_type(
|
76
|
+
def mount_type(dir=nil)
|
77
77
|
mtab = '/etc/mtab'
|
78
78
|
if File.exist?(mtab)
|
79
|
-
|
79
|
+
dir ||= mountpoint_of_cwd
|
80
80
|
open(mtab,'r') do |f|
|
81
81
|
f.each_line do |l|
|
82
82
|
a = l.split
|
83
|
-
if a[1] ==
|
84
|
-
return a[2]
|
83
|
+
if a[1] == dir
|
84
|
+
return a[2].sub(/^fuse\./,'')
|
85
85
|
end
|
86
86
|
end
|
87
87
|
end
|
@@ -354,6 +354,14 @@ module Pwrake
|
|
354
354
|
# ----------------------------------------------------------
|
355
355
|
|
356
356
|
def setup_hosts
|
357
|
+
if f = ENV['PBS_NODEFILE']
|
358
|
+
if @hostfile
|
359
|
+
Log.info "HOSTFILE=#{@hostfile} overrides PBS_NODEFILE=#{f}"
|
360
|
+
else
|
361
|
+
Log.info "use PBS_NODEFILE=#{f}"
|
362
|
+
@hostfile = f
|
363
|
+
end
|
364
|
+
end
|
357
365
|
if @hostfile && @num_threads
|
358
366
|
raise "Cannot set `hostfile' and `num_threads' simultaneously"
|
359
367
|
end
|
@@ -364,6 +372,7 @@ module Pwrake
|
|
364
372
|
# ----------------------------------------------------------
|
365
373
|
|
366
374
|
def put_log
|
375
|
+
Log.info "Pwrake::VERSION=#{Pwrake::VERSION}"
|
367
376
|
Log.info "Options:"
|
368
377
|
self.each do |k,v|
|
369
378
|
Log.info " #{k} = #{v.inspect}"
|
@@ -8,6 +8,7 @@ module Pwrake
|
|
8
8
|
|
9
9
|
def option_data_filesystem
|
10
10
|
[
|
11
|
+
'GFARM2FS_COMMAND',
|
11
12
|
'GFARM2FS_OPTION',
|
12
13
|
'GFARM2FS_DEBUG',
|
13
14
|
['GFARM2FS_DEBUG_WAIT', proc{|v| v ? v.to_i : 1}],
|
@@ -21,7 +22,6 @@ module Pwrake
|
|
21
22
|
end
|
22
23
|
|
23
24
|
def set_filesystem_option
|
24
|
-
@filesystem = 'gfarm'
|
25
25
|
GfarmPath.subdir = self['GFARM_SUBDIR']
|
26
26
|
@worker_option = {
|
27
27
|
:log_dir => self['LOG_DIR'],
|
@@ -33,6 +33,7 @@ module Pwrake
|
|
33
33
|
:shared_directory => "GfarmDirectory",
|
34
34
|
:base_dir => self['GFARM_BASEDIR']+"/"+self['GFARM_PREFIX'],
|
35
35
|
:work_dir => GfarmPath.pwd.to_s,
|
36
|
+
:gfarm2fs_command => self['GFARM2FS_COMMAND'],
|
36
37
|
:gfarm2fs_option => self['GFARM2FS_OPTION'],
|
37
38
|
:gfarm2fs_debug => self['GFARM2FS_DEBUG'],
|
38
39
|
:gfarm2fs_debug_wait => self['GFARM2FS_DEBUG_WAIT'],
|
@@ -30,13 +30,12 @@ module Pwrake
|
|
30
30
|
def pop
|
31
31
|
case @prio
|
32
32
|
when 0
|
33
|
-
|
33
|
+
@que.shift
|
34
34
|
when 1
|
35
|
-
|
35
|
+
@que.pop
|
36
36
|
when 2
|
37
|
-
|
37
|
+
@que.delete_at(rand(@que.size))
|
38
38
|
end
|
39
|
-
return x
|
40
39
|
end
|
41
40
|
|
42
41
|
alias shift pop
|
@@ -9,8 +9,11 @@ module Pwrake
|
|
9
9
|
def initialize(queue_class, hostinfo_by_id, group_map=nil)
|
10
10
|
@queue_class = Pwrake.const_get(queue_class)
|
11
11
|
@hostinfo_by_id = hostinfo_by_id
|
12
|
+
@lock = Monitor.new
|
12
13
|
@q_no_action = NoActionQueue.new
|
13
14
|
@q_reserved = Hash.new
|
15
|
+
@nenq = 0
|
16
|
+
@ndeq = 0
|
14
17
|
def @q_reserved.first
|
15
18
|
super.last
|
16
19
|
end
|
@@ -40,27 +43,39 @@ module Pwrake
|
|
40
43
|
end
|
41
44
|
|
42
45
|
def enq(tw)
|
46
|
+
@lock.synchronize do
|
43
47
|
if tw.nil? || tw.actions.empty?
|
44
48
|
@q_no_action.push(tw)
|
45
49
|
else
|
46
50
|
@q.enq_impl(tw)
|
47
51
|
end
|
52
|
+
@nenq += 1
|
53
|
+
end
|
48
54
|
end
|
49
55
|
|
50
56
|
def deq_task(&block)
|
51
|
-
|
57
|
+
@lock.synchronize do
|
58
|
+
if @nenq > 0
|
59
|
+
Log.debug "deq_task nenq=#{@nenq}:"+(empty? ? " (empty)" : "\n"+inspect_q)
|
60
|
+
@nenq = 0
|
61
|
+
end
|
52
62
|
deq_noaction_task(&block)
|
53
63
|
deq_reserve(&block)
|
54
64
|
@q.deq_start
|
55
65
|
unless @q.empty?
|
56
66
|
@q.turns.each{|turn| deq_turn(turn,&block) }
|
57
67
|
end
|
68
|
+
if @ndeq > 0
|
69
|
+
Log.debug "deq_task ndeq=#{@ndeq}:"+(empty? ? " (empty)" : "\n"+inspect_q)
|
70
|
+
@ndeq = 0
|
71
|
+
end
|
72
|
+
end
|
58
73
|
end
|
59
74
|
|
60
75
|
def deq_noaction_task(&block)
|
61
76
|
while tw = @q_no_action.shift
|
62
|
-
Log.debug "deq_noaction: #{tw.name}"
|
63
77
|
yield(tw)
|
78
|
+
@ndeq += 1
|
64
79
|
end
|
65
80
|
end
|
66
81
|
|
@@ -72,6 +87,7 @@ module Pwrake
|
|
72
87
|
@q_reserved.delete(host_info)
|
73
88
|
Log.debug "deq_reserve: #{tw.name} n_use_cores=#{n_core}"
|
74
89
|
yield(tw,host_info,n_core)
|
90
|
+
@ndeq += 1
|
75
91
|
end
|
76
92
|
end
|
77
93
|
end
|
@@ -89,6 +105,7 @@ module Pwrake
|
|
89
105
|
Log.debug "deq: #{tw.name} n_use_cores=#{n_core}"
|
90
106
|
yield(tw,host_info,n_core)
|
91
107
|
count += 1
|
108
|
+
@ndeq += 1
|
92
109
|
else
|
93
110
|
@q_reserved[host_info] = tw
|
94
111
|
Log.debug "reserve host: #{host_info.name} for #{tw.name} (#{n_core} cores)"
|
@@ -93,6 +93,7 @@ module Pwrake
|
|
93
93
|
|
94
94
|
t_end = (a.last)[0]
|
95
95
|
|
96
|
+
begin
|
96
97
|
if system("which gnuplot >/dev/null 2>&1")
|
97
98
|
IO.popen("gnuplot","r+") do |f|
|
98
99
|
f.print "
|
@@ -112,6 +113,10 @@ plot '-' w l notitle
|
|
112
113
|
end
|
113
114
|
end
|
114
115
|
end
|
116
|
+
rescue => exc
|
117
|
+
$stderr.puts exc
|
118
|
+
$stderr.puts exc.backtrace.join("\n")
|
119
|
+
end
|
115
120
|
|
116
121
|
#puts "Parallelism plot: #{fimg}"
|
117
122
|
fimg
|
@@ -151,6 +156,7 @@ plot '-' w l notitle
|
|
151
156
|
|
152
157
|
t_end = (a.last)[0]
|
153
158
|
|
159
|
+
begin
|
154
160
|
if system("which gnuplot >/dev/null 2>&1")
|
155
161
|
IO.popen("gnuplot","r+") do |f|
|
156
162
|
f.print "
|
@@ -177,6 +183,10 @@ plot '-' w l axis x1y1 title 'parallelism', '-' w l axis x1y2 title 'exec/sec'
|
|
177
183
|
end
|
178
184
|
end
|
179
185
|
end
|
186
|
+
rescue => exc
|
187
|
+
$stderr.puts exc
|
188
|
+
$stderr.puts exc.backtrace.join("\n")
|
189
|
+
end
|
180
190
|
|
181
191
|
#puts "Parallelism plot: #{fimg}"
|
182
192
|
fimg
|
@@ -341,6 +351,7 @@ set ylabel '# of cores'
|
|
341
351
|
grid << a
|
342
352
|
end
|
343
353
|
|
354
|
+
begin
|
344
355
|
if system("which gnuplot >/dev/null 2>&1")
|
345
356
|
IO.popen("gnuplot","r+") do |f|
|
346
357
|
f.puts "
|
@@ -376,6 +387,10 @@ set format y ''
|
|
376
387
|
f.printf "e\n"
|
377
388
|
end
|
378
389
|
end
|
390
|
+
rescue => exc
|
391
|
+
$stderr.puts exc
|
392
|
+
$stderr.puts exc.backtrace.join("\n")
|
393
|
+
end
|
379
394
|
fimg
|
380
395
|
end
|
381
396
|
|
@@ -2,15 +2,60 @@ module Pwrake
|
|
2
2
|
|
3
3
|
module FileTaskAlgorithm
|
4
4
|
|
5
|
+
def needed?
|
6
|
+
!_exist?(name) || out_of_date?(timestamp) || @application.options.build_all
|
7
|
+
end
|
8
|
+
|
5
9
|
# Cache time stamp to reduce load on file system.
|
6
10
|
def timestamp
|
7
11
|
@file_mtime ||
|
8
|
-
if
|
9
|
-
@file_mtime =
|
12
|
+
if _exist?(name)
|
13
|
+
@file_mtime = _mtime(name.to_s)
|
10
14
|
else
|
11
15
|
Rake::LATE
|
12
16
|
end
|
13
17
|
end
|
14
18
|
|
19
|
+
private
|
20
|
+
|
21
|
+
@@t_mtime = 0
|
22
|
+
@@n_mtime = 0
|
23
|
+
@@l_mtime = 100
|
24
|
+
@@c_mtime = 0
|
25
|
+
@@t_exist = 0
|
26
|
+
@@n_exist = 0
|
27
|
+
@@l_exist = 100
|
28
|
+
@@c_exist = 0
|
29
|
+
|
30
|
+
def _mtime(name)
|
31
|
+
t = Pwrake.clock
|
32
|
+
m = File.mtime(name.to_s)
|
33
|
+
@@t_mtime = @@t_mtime + (Pwrake.clock-t)
|
34
|
+
@@n_mtime += 1
|
35
|
+
if @@n_mtime >= 100
|
36
|
+
Log.debug('mtime: mean=%.9f s (%d times)'%[@@t_mtime/@@n_mtime,@@n_mtime])
|
37
|
+
@@t_mtime = 0
|
38
|
+
@@n_mtime = 0
|
39
|
+
@@c_mtime += 1
|
40
|
+
@@l_mtime = 1000 if @@c_mtime == 10
|
41
|
+
end
|
42
|
+
m
|
43
|
+
end
|
44
|
+
|
45
|
+
def _exist?(name)
|
46
|
+
t = Pwrake.clock
|
47
|
+
e = File.exist?(name.to_s)
|
48
|
+
@@t_exist = @@t_exist + (Pwrake.clock-t)
|
49
|
+
@@n_exist += 1
|
50
|
+
if @@n_exist >= @@l_exist
|
51
|
+
Log.debug('exist: mean=%.9f s (%d times)'%[@@t_exist/@@n_exist,@@n_exist])
|
52
|
+
@@t_exist = 0
|
53
|
+
@@n_exist = 0
|
54
|
+
@@c_exist += 1
|
55
|
+
@@l_exist = 1000 if @@c_exist == 10
|
56
|
+
end
|
57
|
+
e
|
58
|
+
end
|
59
|
+
|
15
60
|
end
|
16
61
|
end
|
@@ -12,20 +12,24 @@ module Pwrake
|
|
12
12
|
|
13
13
|
def wrapper
|
14
14
|
if @wrapper.nil?
|
15
|
-
|
15
|
+
Log.debug "TaskWrapper is not defined for #{self.class}[#{name}]"
|
16
|
+
@wrapper = TaskWrapper.new(self)
|
16
17
|
end
|
17
18
|
@wrapper
|
18
19
|
end
|
19
20
|
|
20
21
|
def pw_search_tasks(args)
|
21
|
-
Log.debug "#{self.class}#pw_search_tasks start,
|
22
|
+
Log.debug "#{self.class}[#{name}]#pw_search_tasks start, args=#{args.inspect}"
|
23
|
+
if application.options.trace
|
24
|
+
application.trace "** Search #{name}#{format_search_flags}"
|
25
|
+
end
|
22
26
|
cl = Pwrake.clock
|
23
27
|
TaskWrapper.clear_rank
|
24
28
|
task_args = TaskArguments.new(arg_names, args)
|
25
29
|
# not synchronize owing to fiber
|
26
30
|
search_with_call_chain(nil, task_args, InvocationChain::EMPTY)
|
27
31
|
#
|
28
|
-
Log.debug "#{self.class}#pw_search_tasks end
|
32
|
+
Log.debug "#{self.class}[#{name}]#pw_search_tasks end t=%.6f" % (Pwrake.clock-cl)
|
29
33
|
end
|
30
34
|
|
31
35
|
# Same as search, but explicitly pass a call chain to detect
|
@@ -33,11 +37,6 @@ module Pwrake
|
|
33
37
|
def search_with_call_chain(subseq, task_args, invocation_chain) # :nodoc:
|
34
38
|
new_chain = InvocationChain.append(self, invocation_chain)
|
35
39
|
@lock.synchronize do
|
36
|
-
if application.options.trace
|
37
|
-
#Log.debug "** Search #{name}#{format_search_flags}"
|
38
|
-
application.trace "** Search #{name}#{format_search_flags}"
|
39
|
-
end
|
40
|
-
|
41
40
|
return true if @already_finished # <<--- competition !!!
|
42
41
|
@subsequents ||= []
|
43
42
|
@subsequents << subseq if subseq # <<--- competition !!!
|
@@ -89,14 +89,10 @@ module Pwrake
|
|
89
89
|
end
|
90
90
|
|
91
91
|
case @ncore
|
92
|
-
when Rational
|
93
|
-
if @ncore > 0 && @ncore <= 1
|
94
|
-
return [(@ncore*ppn).to_i, 1].min
|
95
|
-
end
|
96
|
-
when 1-ppn..ppn
|
97
|
-
return (@ncore>0) ? @ncore : @ncore+ppn
|
98
92
|
when nil
|
99
93
|
return 1
|
94
|
+
when 1-ppn..ppn
|
95
|
+
return (@ncore>0) ? @ncore : @ncore+ppn
|
100
96
|
end
|
101
97
|
|
102
98
|
m = "ncore=#{@ncore} is out of range of cores per node: #{ppn}"
|
@@ -10,6 +10,7 @@ module Pwrake
|
|
10
10
|
@@current_id = 1
|
11
11
|
@@task_logger = nil
|
12
12
|
@@instances = []
|
13
|
+
LOCK = Monitor.new
|
13
14
|
|
14
15
|
def initialize(task,task_args=nil)
|
15
16
|
@task = task
|
@@ -68,8 +69,10 @@ module Pwrake
|
|
68
69
|
end
|
69
70
|
|
70
71
|
def self.clear_rank
|
71
|
-
|
72
|
-
|
72
|
+
LOCK.synchronize do
|
73
|
+
Log.debug "#{self}.clear_rank"
|
74
|
+
@@instances.each{|w| w.clear_rank}
|
75
|
+
end
|
73
76
|
end
|
74
77
|
|
75
78
|
def preprocess
|
@@ -87,15 +90,12 @@ module Pwrake
|
|
87
90
|
|
88
91
|
def postprocess(postproc)
|
89
92
|
@executed = true if !@task.actions.empty?
|
90
|
-
#tm_taskend = Pwrake.clock
|
91
93
|
if is_file_task?
|
92
|
-
#t = Pwrake.clock
|
93
94
|
if File.exist?(name)
|
94
95
|
@file_stat = File::Stat.new(name)
|
95
96
|
@location = postproc.run(self)
|
96
97
|
end
|
97
98
|
end
|
98
|
-
#Log.debug "postprocess time=#{Pwrake.clock-tm_taskend}"
|
99
99
|
log_task
|
100
100
|
end
|
101
101
|
|
@@ -129,7 +129,12 @@ module Pwrake
|
|
129
129
|
end
|
130
130
|
return if !@@task_logger
|
131
131
|
#
|
132
|
-
|
132
|
+
if @clock_start
|
133
|
+
elap = @clock_end - @clock_start
|
134
|
+
else
|
135
|
+
Loag.debug "@clock_start is not defined for #{@task.class}[#{name}]"
|
136
|
+
elap = 0
|
137
|
+
end
|
133
138
|
if has_output_file?
|
134
139
|
RANK_STAT.add_sample(rank,elap)
|
135
140
|
end
|
@@ -231,6 +236,7 @@ module Pwrake
|
|
231
236
|
|
232
237
|
def suggest_location
|
233
238
|
if has_input_file? && @suggest_location.nil?
|
239
|
+
cl = Pwrake.clock
|
234
240
|
@suggest_location = []
|
235
241
|
loc_fsz = Hash.new(0)
|
236
242
|
prerequisites.each do |preq|
|
@@ -246,18 +252,19 @@ module Pwrake
|
|
246
252
|
#Log.debug "input=#{prerequisites.join('|')}"
|
247
253
|
if !loc_fsz.empty?
|
248
254
|
half_max_fsz = loc_fsz.values.max / 2
|
249
|
-
Log.debug "loc_fsz=#{loc_fsz.inspect} half_max_fsz=#{half_max_fsz}"
|
250
255
|
loc_fsz.each do |h,sz|
|
251
256
|
if sz > half_max_fsz
|
252
257
|
@suggest_location << h
|
253
258
|
end
|
254
259
|
end
|
260
|
+
Log.debug "locate:%.6f #{name} loc_fsz=#{loc_fsz.inspect} half_max_fsz=#{half_max_fsz} suggest=#{@suggest_location.inspect}"%(Pwrake.clock-cl)
|
255
261
|
end
|
256
262
|
end
|
257
263
|
@suggest_location
|
258
264
|
end
|
259
265
|
|
260
266
|
def rank
|
267
|
+
LOCK.synchronize do
|
261
268
|
if @rank.nil?
|
262
269
|
if subsequents.nil? || subsequents.empty?
|
263
270
|
@rank = 0
|
@@ -265,8 +272,12 @@ module Pwrake
|
|
265
272
|
max_rank = 0
|
266
273
|
subsequents.each do |subsq|
|
267
274
|
r = subsq.wrapper.rank
|
268
|
-
if
|
269
|
-
max_rank
|
275
|
+
if r
|
276
|
+
if max_rank < r
|
277
|
+
max_rank = r
|
278
|
+
end
|
279
|
+
else
|
280
|
+
Log.warn "subsq.wrapper.rank=#{rank.inspect}"
|
270
281
|
end
|
271
282
|
end
|
272
283
|
if has_output_file?
|
@@ -276,9 +287,9 @@ module Pwrake
|
|
276
287
|
end
|
277
288
|
@rank = max_rank + step
|
278
289
|
end
|
279
|
-
Log.debug "Task[#{name}] rank=#{@rank.inspect}"
|
280
290
|
end
|
281
291
|
@rank
|
292
|
+
end
|
282
293
|
end
|
283
294
|
|
284
295
|
def clear_rank
|
@@ -348,7 +359,7 @@ module Pwrake
|
|
348
359
|
else
|
349
360
|
@priority = 0
|
350
361
|
end
|
351
|
-
Log.debug "
|
362
|
+
Log.debug "task=#{name} priority=#{@priority} sum_file_size=#{sum_sz}"
|
352
363
|
end
|
353
364
|
@priority || 0
|
354
365
|
end
|
data/lib/pwrake/version.rb
CHANGED
@@ -16,6 +16,7 @@ module Pwrake
|
|
16
16
|
@@prefix = opts[:base_dir]
|
17
17
|
@@work_dir = opts[:work_dir]
|
18
18
|
@@log_dir = opts[:log_dir]
|
19
|
+
@@gfarm2fs_command = opts[:gfarm2fs_command] || 'gfarm2fs'
|
19
20
|
@@gfarm2fs_option = opts[:gfarm2fs_option]
|
20
21
|
@@gfarm2fs_debug = opts[:gfarm2fs_debug]
|
21
22
|
@@gfarm2fs_debug_wait = opts[:gfarm2fs_debug_wait]
|
@@ -56,13 +57,14 @@ module Pwrake
|
|
56
57
|
|
57
58
|
def open
|
58
59
|
FileUtils.mkdir_p @gfarm_mountpoint
|
60
|
+
@log.info "mkdir -p #{@@hostname}:#{@gfarm_mountpoint}"
|
59
61
|
path = @log.path
|
60
62
|
begin
|
61
63
|
if @@gfarm2fs_debug && path
|
62
64
|
f = path+("gfarm2fs-"+@@hostname+"-"+@suffix)
|
63
|
-
spawn_cmd "
|
65
|
+
spawn_cmd "#{@@gfarm2fs_command} #{@@gfarm2fs_option} -d #{@gfarm_mountpoint} > #{f} 2>&1 & sleep #{@@gfarm2fs_debug_wait}"
|
64
66
|
else
|
65
|
-
spawn_cmd "
|
67
|
+
spawn_cmd "#{@@gfarm2fs_command} #{@@gfarm2fs_option} #{@gfarm_mountpoint}"
|
66
68
|
end
|
67
69
|
rescue => exc
|
68
70
|
sleep 1
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: pwrake
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.3.
|
4
|
+
version: 2.3.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Masahiro TANAKA
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2019-
|
11
|
+
date: 2019-11-26 00:00:00.000000000 Z
|
12
12
|
dependencies: []
|
13
13
|
description: Parallel and distributed Rake for workflow execution on multicores, clusters,
|
14
14
|
clouds using SSH. It has locality-aware scheduling designed for Gfarm file system.
|
@@ -58,7 +58,6 @@ files:
|
|
58
58
|
- lib/pwrake/option/host_map.rb
|
59
59
|
- lib/pwrake/option/option.rb
|
60
60
|
- lib/pwrake/option/option_default_filesystem.rb
|
61
|
-
- lib/pwrake/option/option_gfarm.rb
|
62
61
|
- lib/pwrake/option/option_gfarm2fs.rb
|
63
62
|
- lib/pwrake/queue/locality_aware_queue.rb
|
64
63
|
- lib/pwrake/queue/no_action_queue.rb
|
@@ -1 +0,0 @@
|
|
1
|
-
lib/pwrake/option/option_gfarm2fs.rb
|