pwrake 0.9.9.2 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +2 -0
- data/CHANGES_V2.md +90 -0
- data/{LICENSE.txt → MIT-LICENSE} +2 -3
- data/README +12 -0
- data/README.md +75 -52
- data/bin/gfwhere-pipe +23 -12
- data/bin/pwrake +22 -29
- data/bin/pwrake_branch +24 -0
- data/lib/pwrake/branch.rb +22 -0
- data/lib/pwrake/branch/branch.rb +213 -0
- data/lib/pwrake/branch/branch_application.rb +53 -0
- data/lib/pwrake/branch/fiber_queue.rb +36 -0
- data/lib/pwrake/branch/file_utils.rb +101 -0
- data/lib/pwrake/branch/shell.rb +231 -0
- data/lib/pwrake/{profiler.rb → branch/shell_profiler.rb} +28 -27
- data/lib/pwrake/branch/worker_communicator.rb +104 -0
- data/lib/pwrake/{gfarm_feature.rb → gfarm/gfarm_path.rb} +2 -100
- data/lib/pwrake/gfarm/gfarm_postprocess.rb +53 -0
- data/lib/pwrake/iomux/channel.rb +70 -0
- data/lib/pwrake/iomux/handler.rb +124 -0
- data/lib/pwrake/iomux/handler_set.rb +35 -0
- data/lib/pwrake/iomux/runner.rb +62 -0
- data/lib/pwrake/logger.rb +3 -150
- data/lib/pwrake/master.rb +30 -137
- data/lib/pwrake/master/fiber_pool.rb +69 -0
- data/lib/pwrake/master/idle_cores.rb +30 -0
- data/lib/pwrake/master/master.rb +345 -0
- data/lib/pwrake/master/master_application.rb +150 -0
- data/lib/pwrake/master/postprocess.rb +16 -0
- data/lib/pwrake/{graphviz.rb → misc/graphviz.rb} +0 -0
- data/lib/pwrake/{mcgp.rb → misc/mcgp.rb} +63 -42
- data/lib/pwrake/option/host_map.rb +158 -0
- data/lib/pwrake/option/option.rb +357 -0
- data/lib/pwrake/option/option_filesystem.rb +112 -0
- data/lib/pwrake/queue/locality_aware_queue.rb +158 -0
- data/lib/pwrake/queue/no_action_queue.rb +67 -0
- data/lib/pwrake/queue/queue_array.rb +366 -0
- data/lib/pwrake/queue/task_queue.rb +164 -0
- data/lib/pwrake/report.rb +1 -0
- data/lib/pwrake/report/parallelism.rb +9 -3
- data/lib/pwrake/report/report.rb +50 -103
- data/lib/pwrake/report/task_stat.rb +83 -0
- data/lib/pwrake/task/task_algorithm.rb +107 -0
- data/lib/pwrake/task/task_manager.rb +32 -0
- data/lib/pwrake/task/task_property.rb +98 -0
- data/lib/pwrake/task/task_rank.rb +48 -0
- data/lib/pwrake/task/task_wrapper.rb +296 -0
- data/lib/pwrake/version.rb +1 -1
- data/lib/pwrake/worker/executor.rb +169 -0
- data/lib/pwrake/worker/gfarm_directory.rb +90 -0
- data/lib/pwrake/worker/invoker.rb +199 -0
- data/lib/pwrake/worker/load.rb +14 -0
- data/lib/pwrake/worker/log_executor.rb +73 -0
- data/lib/pwrake/worker/shared_directory.rb +74 -0
- data/lib/pwrake/worker/worker_main.rb +14 -0
- data/lib/pwrake/worker/writer.rb +59 -0
- data/setup.rb +1212 -1502
- data/spec/003/Rakefile +2 -2
- data/spec/008/Rakefile +2 -1
- data/spec/009/Rakefile +1 -1
- data/spec/009/pwrake_conf.yaml +1 -3
- data/spec/hosts +0 -2
- data/spec/pwrake_spec.rb +9 -8
- metadata +50 -21
- data/lib/pwrake.rb +0 -19
- data/lib/pwrake/application.rb +0 -232
- data/lib/pwrake/counter.rb +0 -54
- data/lib/pwrake/file_utils.rb +0 -98
- data/lib/pwrake/gfwhere_pool.rb +0 -109
- data/lib/pwrake/host_list.rb +0 -88
- data/lib/pwrake/locality_aware_queue.rb +0 -413
- data/lib/pwrake/option.rb +0 -400
- data/lib/pwrake/rake_modify.rb +0 -14
- data/lib/pwrake/shell.rb +0 -186
- data/lib/pwrake/task_algorithm.rb +0 -475
- data/lib/pwrake/task_queue.rb +0 -633
- data/lib/pwrake/timer.rb +0 -22
File without changes
|
@@ -5,25 +5,25 @@ module Pwrake
|
|
5
5
|
|
6
6
|
module MCGP
|
7
7
|
|
8
|
-
def graph_partition(target=nil)
|
8
|
+
def graph_partition(host_map, target=nil)
|
9
9
|
t1 = Time.now
|
10
|
-
wgts =
|
10
|
+
wgts = host_map.group_weight_sum
|
11
11
|
if wgts.size > 1
|
12
12
|
list = wgts.size.times.to_a
|
13
|
-
g =
|
13
|
+
g = GraphTracerGroup.new([list],[wgts])
|
14
14
|
trace(g,target)
|
15
|
-
g.
|
15
|
+
g.part_graph
|
16
16
|
#g.write_dot('dag1.dot')
|
17
17
|
#return
|
18
18
|
end
|
19
19
|
|
20
|
-
#$
|
20
|
+
#$debug=true
|
21
21
|
|
22
|
-
list =
|
23
|
-
wgts =
|
24
|
-
g =
|
22
|
+
list = host_map.group_hosts
|
23
|
+
wgts = host_map.group_core_weight
|
24
|
+
g = GraphTracerNode.new(list,wgts)
|
25
25
|
trace(g,target)
|
26
|
-
g.
|
26
|
+
g.part_graph
|
27
27
|
t2 = Time.now
|
28
28
|
Pwrake::Log.info "Time for TOTAL Graph Partitioning: #{t2-t1} sec"
|
29
29
|
#g.write_dot('dag2.dot')
|
@@ -47,14 +47,13 @@ module Pwrake
|
|
47
47
|
|
48
48
|
class GraphTracer
|
49
49
|
|
50
|
-
def initialize(loc_list, weight_list
|
50
|
+
def initialize(loc_list, weight_list)
|
51
51
|
if loc_list.size != weight_list.size
|
52
52
|
raise ArgumentError, "array size of args mismatch"
|
53
53
|
end
|
54
54
|
@loc_list = loc_list
|
55
55
|
@weight_list = weight_list
|
56
56
|
@n_part = @loc_list.size
|
57
|
-
@location_finder = block
|
58
57
|
@traced = {}
|
59
58
|
@vertex_depth = {}
|
60
59
|
# @grviz = Grviz.new
|
@@ -63,26 +62,27 @@ module Pwrake
|
|
63
62
|
end
|
64
63
|
end
|
65
64
|
|
66
|
-
def trace(
|
65
|
+
def trace(name="default", target=nil)
|
67
66
|
|
68
67
|
task = Rake.application[name]
|
69
|
-
|
68
|
+
tw = task.wrapper
|
69
|
+
group_id = tw.group_id || 0
|
70
70
|
group = @group_list[group_id]
|
71
|
-
loc_list = @loc_list[group_id]
|
71
|
+
#loc_list = @loc_list[group_id]
|
72
72
|
depth = 0
|
73
73
|
|
74
74
|
if task.class == Rake::FileTask
|
75
|
-
tgid = target ? (Rake.application[target].group_id||0) : nil
|
75
|
+
tgid = (target) ? (Rake.application[target].wrapper.group_id||0) : nil
|
76
76
|
|
77
77
|
if File.file?(name)
|
78
78
|
if tgid == group_id
|
79
|
-
locs =
|
80
|
-
if locs.empty?
|
81
|
-
|
82
|
-
|
83
|
-
end
|
84
|
-
|
85
|
-
fsz =
|
79
|
+
locs = get_location(tw)
|
80
|
+
#if locs.empty?
|
81
|
+
# Pwrake.application.postprocess(task)
|
82
|
+
# locs = get_location(tw)
|
83
|
+
#end
|
84
|
+
#tw.get_file_stat
|
85
|
+
fsz = tw.file_size
|
86
86
|
if fsz > 100000
|
87
87
|
#puts "g=#{group_id}, task=#{name}, target=#{target}, fsz=#{fsz}, locs="+locs.join("|")
|
88
88
|
group.push_loc_edge( locs, name, target, fsz/10000 )
|
@@ -119,23 +119,39 @@ module Pwrake
|
|
119
119
|
return @vertex_depth[name]
|
120
120
|
end
|
121
121
|
|
122
|
-
def
|
122
|
+
def write_dot(file)
|
123
|
+
@grviz.write(file)
|
124
|
+
end
|
125
|
+
end
|
126
|
+
|
127
|
+
|
128
|
+
class GraphTracerGroup < GraphTracer
|
129
|
+
|
130
|
+
def get_location(tw)
|
131
|
+
tw.group
|
132
|
+
end
|
133
|
+
|
134
|
+
def part_graph
|
123
135
|
@group_list.each do |g|
|
124
136
|
g.part_graph
|
125
137
|
g.set_group
|
126
138
|
end
|
127
139
|
end
|
140
|
+
end
|
141
|
+
|
142
|
+
|
143
|
+
class GraphTracerNode < GraphTracer
|
128
144
|
|
129
|
-
def
|
145
|
+
def get_location(tw)
|
146
|
+
tw.location
|
147
|
+
end
|
148
|
+
|
149
|
+
def part_graph
|
130
150
|
@group_list.each do |g|
|
131
151
|
g.part_graph
|
132
152
|
g.set_node
|
133
153
|
end
|
134
154
|
end
|
135
|
-
|
136
|
-
def write_dot(file)
|
137
|
-
@grviz.write(file)
|
138
|
-
end
|
139
155
|
end
|
140
156
|
|
141
157
|
|
@@ -296,7 +312,9 @@ module Pwrake
|
|
296
312
|
if false
|
297
313
|
puts "@vertex_id2name.size=#{@vertex_id2name.size}"
|
298
314
|
if $debug2
|
299
|
-
@vertex_id2name.each_with_index{|x,i|
|
315
|
+
@vertex_id2name.each_with_index{|x,i|
|
316
|
+
puts "#{i} #{x} #{@edges[i].inspect}"
|
317
|
+
}
|
300
318
|
end
|
301
319
|
puts "@edges.size=#{@edges.size}"
|
302
320
|
puts "ncon=#{c}"
|
@@ -328,22 +346,25 @@ module Pwrake
|
|
328
346
|
s += "]"
|
329
347
|
Log.info s
|
330
348
|
options = RbMetis.default_options
|
331
|
-
RbMetis::OPTION_NITER
|
332
349
|
options[RbMetis::OPTION_NCUTS] = 30
|
333
350
|
options[RbMetis::OPTION_NSEPS] = 30
|
334
351
|
options[RbMetis::OPTION_NITER] = 10
|
335
|
-
@part = RbMetis.part_graph_recursive(
|
352
|
+
@part = RbMetis.part_graph_recursive(
|
353
|
+
@xadj, @adjcny, @n_part,
|
354
|
+
ncon:c, vwgt:@vwgt, adjwgt:@adjwgt,
|
355
|
+
tpwgts:tpw, ubvec:ubv, options:options)
|
336
356
|
else
|
337
357
|
puts "tpw=#{tpw.inspect}"
|
338
|
-
@part = Metis.mc_part_graph_recursive2(
|
358
|
+
@part = Metis.mc_part_graph_recursive2(
|
359
|
+
c,@xadj,@adjcny,@vwgt,nil,@tpwgts)
|
339
360
|
end
|
340
361
|
t2 = Time.now
|
341
362
|
Pwrake::Log.info "Time for Graph Partitioning: #{t2-t1} sec"
|
342
363
|
count_partition
|
343
|
-
|
344
|
-
|
345
|
-
|
346
|
-
|
364
|
+
if $debug
|
365
|
+
puts "Time for Graph Partitioning: #{t2-t1} sec"
|
366
|
+
p @part
|
367
|
+
end
|
347
368
|
end
|
348
369
|
|
349
370
|
def count_partition
|
@@ -411,15 +432,15 @@ module Pwrake
|
|
411
432
|
@vertex_id2name.each_with_index do |name,idx|
|
412
433
|
if idx >= @n_part
|
413
434
|
i_part = @part[idx]
|
414
|
-
|
415
|
-
|
435
|
+
tw = Rake.application[name].wrapper
|
436
|
+
tw.group_id = loc_list[i_part]
|
416
437
|
#puts "task=#{task.inspect}, i_part=#{i_part}, host=#{host}"
|
417
438
|
end
|
418
439
|
end
|
419
440
|
@loc_files.each do |gid,files|
|
420
441
|
files.each do |f|
|
421
|
-
|
422
|
-
|
442
|
+
tw = Rake.application[f].wrapper
|
443
|
+
tw.group_id = gid
|
423
444
|
# puts "gid=#{gid}, task=#{f}"
|
424
445
|
end
|
425
446
|
end
|
@@ -431,9 +452,9 @@ module Pwrake
|
|
431
452
|
@vertex_id2name.each_with_index do |name,idx|
|
432
453
|
if idx >= @n_part
|
433
454
|
i_part = @part[idx]
|
434
|
-
|
455
|
+
tw = Rake.application[name].wrapper
|
435
456
|
host = loc_list[i_part]
|
436
|
-
|
457
|
+
tw.suggest_location = [host]
|
437
458
|
#puts "task=#{task.inspect}, i_part=#{i_part}, host=#{host}"
|
438
459
|
end
|
439
460
|
end
|
@@ -0,0 +1,158 @@
|
|
1
|
+
module Pwrake
|
2
|
+
|
3
|
+
class HostInfo
|
4
|
+
|
5
|
+
def initialize(name,id,ncore,weight,group=nil)
|
6
|
+
@name = name
|
7
|
+
@ncore = ncore
|
8
|
+
@weight = weight || 1.0
|
9
|
+
@group = group || 0
|
10
|
+
@id = id
|
11
|
+
end
|
12
|
+
|
13
|
+
attr_reader :name, :ncore, :weight, :group, :id, :steal_flag
|
14
|
+
attr_accessor :idle_cores
|
15
|
+
|
16
|
+
def set_ncore(n)
|
17
|
+
@ncore = @idle_cores = n
|
18
|
+
end
|
19
|
+
|
20
|
+
def increase(n)
|
21
|
+
@idle_cores += n
|
22
|
+
end
|
23
|
+
|
24
|
+
def decrease(n)
|
25
|
+
@idle_cores -= n
|
26
|
+
if @idle_cores < 0
|
27
|
+
raise RuntimeError,"# of cores must be non-negative"
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
def steal_phase
|
32
|
+
@steal_flag = true
|
33
|
+
t = yield(self)
|
34
|
+
@steal_flag = false
|
35
|
+
t
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
class HostMap < Hash
|
40
|
+
|
41
|
+
def initialize(arg=nil)
|
42
|
+
@host_map = {}
|
43
|
+
@by_id = []
|
44
|
+
@by_name = {}
|
45
|
+
require "socket"
|
46
|
+
case arg
|
47
|
+
when /\.yaml$/
|
48
|
+
read_yaml(arg)
|
49
|
+
when String
|
50
|
+
read_host(arg)
|
51
|
+
when Integer
|
52
|
+
parse_hosts(["localhost #{arg}"])
|
53
|
+
when NilClass
|
54
|
+
parse_hosts(["localhost 1"])
|
55
|
+
else
|
56
|
+
raise ArgumentError, "arg=#{arg.inspect}"
|
57
|
+
#@num_threads = 1 if !@num_threads
|
58
|
+
#@core_list = ['localhost'] * @num_threads
|
59
|
+
end
|
60
|
+
end
|
61
|
+
attr_reader :by_id, :by_name
|
62
|
+
|
63
|
+
def host_count
|
64
|
+
@by_id.size
|
65
|
+
end
|
66
|
+
|
67
|
+
def group_hosts
|
68
|
+
a = []
|
69
|
+
self.each do |sub,list|
|
70
|
+
list.each{|h| (a[h.group] ||= []) << h.name}
|
71
|
+
end
|
72
|
+
a
|
73
|
+
end
|
74
|
+
|
75
|
+
def group_core_weight
|
76
|
+
a = []
|
77
|
+
self.each do |sub,list|
|
78
|
+
list.each{|h| (a[h.group] ||= []) << h.weight}
|
79
|
+
end
|
80
|
+
a
|
81
|
+
end
|
82
|
+
|
83
|
+
def group_weight_sum
|
84
|
+
a = []
|
85
|
+
self.each do |sub,list|
|
86
|
+
list.each{|h| a[h.group] = (a[h.group]||0) + h.weight}
|
87
|
+
end
|
88
|
+
a
|
89
|
+
end
|
90
|
+
|
91
|
+
private
|
92
|
+
|
93
|
+
def read_host(file)
|
94
|
+
ary = []
|
95
|
+
File.open(file) do |f|
|
96
|
+
while l = f.gets
|
97
|
+
ary << l
|
98
|
+
end
|
99
|
+
end
|
100
|
+
parse_hosts(ary)
|
101
|
+
end
|
102
|
+
|
103
|
+
def read_yaml(file)
|
104
|
+
parse_hosts(YAML.load(open(file))[0])
|
105
|
+
end
|
106
|
+
|
107
|
+
def parse_hosts(hosts)
|
108
|
+
#p hosts
|
109
|
+
if hosts.kind_of? Array
|
110
|
+
hosts = {"localhost"=>hosts}
|
111
|
+
end
|
112
|
+
hosts.each do |branch, list|
|
113
|
+
self[branch] = parse_list(list)
|
114
|
+
end
|
115
|
+
end
|
116
|
+
|
117
|
+
def parse_list(line_list)
|
118
|
+
info_list = []
|
119
|
+
line_list.each do |line|
|
120
|
+
parse_line(info_list,line)
|
121
|
+
end
|
122
|
+
info_list
|
123
|
+
end
|
124
|
+
|
125
|
+
def parse_line(info_list,line)
|
126
|
+
line = $1 if /^([^#]*)#/ =~ line
|
127
|
+
host, ncore, weight, group = line.split
|
128
|
+
if host
|
129
|
+
if /\[\[([\w\d]+)-([\w\d]+)\]\]/o =~ host
|
130
|
+
hosts = ($1..$2).map{|i| host.sub(re,i)}
|
131
|
+
else
|
132
|
+
hosts = [host]
|
133
|
+
end
|
134
|
+
hosts.each do |host|
|
135
|
+
begin
|
136
|
+
host = Socket.gethostbyname(host)[0]
|
137
|
+
rescue
|
138
|
+
Log.warn "FQDN not resoved : #{host}"
|
139
|
+
end
|
140
|
+
ncore &&= ncore.to_i
|
141
|
+
weitht &&= weight.to_i
|
142
|
+
#weight = (weight || 1).to_f
|
143
|
+
group &&= group.to_i
|
144
|
+
if host_info = @by_name[host]
|
145
|
+
raise RuntimeError,"duplicated hostname: #{host}"
|
146
|
+
else
|
147
|
+
id = @by_id.size
|
148
|
+
host_info = HostInfo.new(host,id,ncore,weight,group)
|
149
|
+
@by_name[host] = host_info
|
150
|
+
info_list << host_info
|
151
|
+
@by_id << host_info
|
152
|
+
end
|
153
|
+
end
|
154
|
+
end
|
155
|
+
end
|
156
|
+
|
157
|
+
end
|
158
|
+
end
|
@@ -0,0 +1,357 @@
|
|
1
|
+
module Pwrake
|
2
|
+
|
3
|
+
START_TIME = Time.now
|
4
|
+
|
5
|
+
class Option < Hash
|
6
|
+
|
7
|
+
def initialize
|
8
|
+
load_pwrake_conf
|
9
|
+
init_options
|
10
|
+
init_pass_env
|
11
|
+
if self['SHOW_CONF']
|
12
|
+
require "yaml"
|
13
|
+
YAML.dump(self,$stdout)
|
14
|
+
exit
|
15
|
+
elsif self['REPORT_DIR']
|
16
|
+
require 'pwrake/report'
|
17
|
+
Report.new(self,[]).report_html
|
18
|
+
exit
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
def init
|
23
|
+
Log.info "Options:"
|
24
|
+
self.each do |k,v|
|
25
|
+
Log.info " #{k} = #{v.inspect}"
|
26
|
+
end
|
27
|
+
#@counter = Counter.new
|
28
|
+
setup_hosts
|
29
|
+
setup_filesystem # require 'option_filesystem.rb'
|
30
|
+
#
|
31
|
+
if self['LOG_DIR'] && self['GC_LOG_FILE']
|
32
|
+
GC::Profiler.enable
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
attr_reader :counter
|
37
|
+
attr_reader :logger
|
38
|
+
|
39
|
+
DEFAULT_CONFFILES = ["pwrake_conf.yaml","PwrakeConf.yaml"]
|
40
|
+
|
41
|
+
# ----- init -----
|
42
|
+
|
43
|
+
def load_pwrake_conf
|
44
|
+
# Read pwrake_conf
|
45
|
+
pwrake_conf = Rake.application.options.pwrake_conf
|
46
|
+
if pwrake_conf
|
47
|
+
if !File.exist?(pwrake_conf)
|
48
|
+
raise "Configuration file not found: #{pwrake_conf}"
|
49
|
+
end
|
50
|
+
else
|
51
|
+
pwrake_conf = DEFAULT_CONFFILES.find{|fn| File.exist?(fn)}
|
52
|
+
end
|
53
|
+
self['PWRAKE_CONF'] = pwrake_conf
|
54
|
+
if pwrake_conf.nil?
|
55
|
+
@yaml = {}
|
56
|
+
else
|
57
|
+
#Log.debug "load pwrake_conf=#{pwrake_conf}"
|
58
|
+
require "yaml"
|
59
|
+
@yaml = open(pwrake_conf){|f| YAML.load(f) }
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
def init_options
|
64
|
+
option_data.each do |a|
|
65
|
+
prc = nil
|
66
|
+
keys = []
|
67
|
+
case a
|
68
|
+
when String
|
69
|
+
keys << a
|
70
|
+
when Array
|
71
|
+
a.each do |x|
|
72
|
+
case x
|
73
|
+
when String
|
74
|
+
keys << x
|
75
|
+
when Proc
|
76
|
+
prc = x
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
80
|
+
key = keys[0]
|
81
|
+
val = search_opts(keys)
|
82
|
+
val = prc.call(val) if prc
|
83
|
+
self[key] = val if !val.nil?
|
84
|
+
instance_variable_set("@"+key.downcase, val)
|
85
|
+
end
|
86
|
+
|
87
|
+
feedback_options
|
88
|
+
|
89
|
+
Rake.verbose(false) if Rake.application.options.silent
|
90
|
+
end
|
91
|
+
|
92
|
+
def option_data
|
93
|
+
[
|
94
|
+
'DRYRUN',
|
95
|
+
'IGNORE_SYSTEM',
|
96
|
+
'IGNORE_DEPRECATE',
|
97
|
+
'LOAD_SYSTEM',
|
98
|
+
'NOSEARCH',
|
99
|
+
'RAKELIB',
|
100
|
+
'SHOW_PREREQS',
|
101
|
+
'SILENT',
|
102
|
+
'TRACE',
|
103
|
+
'BACKTRACE',
|
104
|
+
'TRACE_OUTPUT',
|
105
|
+
'TRACE_RULES',
|
106
|
+
|
107
|
+
'FILESYSTEM',
|
108
|
+
'SSH_OPTION',
|
109
|
+
'PASS_ENV',
|
110
|
+
['SHELL_COMMAND', proc{|v| v||ENV['SHELL']}],
|
111
|
+
['SHELL_RC','SHELLRC'],
|
112
|
+
'GFARM2FS_OPTION',
|
113
|
+
'GFARM2FS_DEBUG',
|
114
|
+
['GFARM2FS_DEBUG_WAIT', proc{|v| v ? v.to_i : 1}],
|
115
|
+
'GNU_TIME',
|
116
|
+
'DEBUG',
|
117
|
+
'PLOT_PARALLELISM',
|
118
|
+
'SHOW_CONF',
|
119
|
+
['REPORT_DIR','REPORT'],
|
120
|
+
'FAILED_TARGET', # rename(default), delete, leave
|
121
|
+
'FAILURE_TERMINATION', # wait, kill, continue
|
122
|
+
'QUEUE_PRIORITY', # RANK(default), FIFO, LIFO, DFS
|
123
|
+
'NOACTION_QUEUE_PRIORITY', # FIFO(default), LIFO, RAND
|
124
|
+
#'NUM_NOACTION_THREADS', # default=4 when gfarm, else 1
|
125
|
+
'GRAPH_PARTITION',
|
126
|
+
'PLOT_PARTITION',
|
127
|
+
|
128
|
+
['HOSTFILE','HOSTS'],
|
129
|
+
['LOG_DIR','LOG',
|
130
|
+
proc{|v|
|
131
|
+
if v
|
132
|
+
if v == "" || !v.kind_of?(String)
|
133
|
+
v = "Pwrake%Y%m%d-%H%M%S"
|
134
|
+
end
|
135
|
+
d = v = format_time_pid(v)
|
136
|
+
i = 1
|
137
|
+
while File.exist?(d)
|
138
|
+
d = "#{v}.#{i}"
|
139
|
+
i += 1
|
140
|
+
end
|
141
|
+
d
|
142
|
+
end
|
143
|
+
}],
|
144
|
+
['LOG_FILE',
|
145
|
+
proc{|v|
|
146
|
+
if v.kind_of?(String) && v != ""
|
147
|
+
v
|
148
|
+
else
|
149
|
+
"pwrake.log"
|
150
|
+
end
|
151
|
+
}],
|
152
|
+
['TASK_CSV_FILE',
|
153
|
+
proc{|v|
|
154
|
+
if v.kind_of?(String) && v != ""
|
155
|
+
v
|
156
|
+
else
|
157
|
+
"task.csv"
|
158
|
+
end
|
159
|
+
}],
|
160
|
+
['COMMAND_CSV_FILE',
|
161
|
+
proc{|v|
|
162
|
+
if v.kind_of?(String) && v != ""
|
163
|
+
v
|
164
|
+
else
|
165
|
+
"command.csv"
|
166
|
+
end
|
167
|
+
}],
|
168
|
+
['GC_LOG_FILE',
|
169
|
+
proc{|v|
|
170
|
+
if v
|
171
|
+
if v.kind_of?(String) && v != ""
|
172
|
+
v
|
173
|
+
else
|
174
|
+
"gc.log"
|
175
|
+
end
|
176
|
+
end
|
177
|
+
}],
|
178
|
+
['NUM_THREADS', proc{|v| v && v.to_i}],
|
179
|
+
['SHELL_START_INTERVAL', proc{|v| (v || 0.012).to_f}],
|
180
|
+
['HEARTBEAT', proc{|v| (v || 240).to_i}],
|
181
|
+
['DISABLE_AFFINITY', proc{|v| v || ENV['AFFINITY']=='off'}],
|
182
|
+
['DISABLE_STEAL', proc{|v| v || ENV['STEAL']=='off'}],
|
183
|
+
['GFARM_BASEDIR', proc{|v| v || '/tmp'}],
|
184
|
+
['GFARM_PREFIX', proc{|v| v || "pwrake_#{ENV['USER']}"}],
|
185
|
+
['GFARM_SUBDIR', proc{|v| v || '/'}],
|
186
|
+
['MAX_GFWHERE_WORKER', proc{|v| (v || 8).to_i}],
|
187
|
+
['MASTER_HOSTNAME', proc{|v| (v || begin;`hostname -f`;rescue;end || '').chomp}],
|
188
|
+
['WORK_DIR', proc{|v|
|
189
|
+
v ||= '%CWD_RELATIVE_TO_HOME'
|
190
|
+
v.sub('%CWD_RELATIVE_TO_HOME',cwd_relative_to_home)
|
191
|
+
}],
|
192
|
+
]
|
193
|
+
end
|
194
|
+
|
195
|
+
def format_time_pid(v)
|
196
|
+
START_TIME.strftime(v).sub("%$","%05d"%Process.pid)
|
197
|
+
end
|
198
|
+
|
199
|
+
def feedback_options
|
200
|
+
opts = Rake.application.options
|
201
|
+
['DRYRUN',
|
202
|
+
'IGNORE_SYSTEM',
|
203
|
+
'IGNORE_DEPRECATE',
|
204
|
+
'LOAD_SYSTEM',
|
205
|
+
'NOSEARCH',
|
206
|
+
'RAKELIB',
|
207
|
+
'SHOW_PREREQS',
|
208
|
+
'SILENT',
|
209
|
+
'TRACE',
|
210
|
+
'BACKTRACE',
|
211
|
+
'TRACE_OUTPUT',
|
212
|
+
'TRACE_RULES'
|
213
|
+
].each do |k|
|
214
|
+
if v=self[k]
|
215
|
+
m = (k.downcase+"=").to_sym
|
216
|
+
opts.send(m,v)
|
217
|
+
end
|
218
|
+
end
|
219
|
+
case opts.trace_output
|
220
|
+
when 'stdout'
|
221
|
+
opts.trace_output = $stdout
|
222
|
+
when 'stderr', nil
|
223
|
+
opts.trace_output = $stderr
|
224
|
+
end
|
225
|
+
end
|
226
|
+
|
227
|
+
# Priority of Option:
|
228
|
+
# command_option > ENV > pwrake_conf > DEFAULT_OPTIONS
|
229
|
+
def search_opts(keys)
|
230
|
+
val = Rake.application.options.send(keys[0].downcase.to_sym)
|
231
|
+
return parse_opt(val) if !val.nil?
|
232
|
+
#
|
233
|
+
keys.each do |k|
|
234
|
+
val = ENV[k.upcase]
|
235
|
+
return parse_opt(val) if !val.nil?
|
236
|
+
end
|
237
|
+
#
|
238
|
+
return nil if !@yaml
|
239
|
+
keys.each do |k|
|
240
|
+
val = @yaml[k.upcase]
|
241
|
+
return val if !val.nil?
|
242
|
+
end
|
243
|
+
nil
|
244
|
+
end
|
245
|
+
|
246
|
+
def parse_opt(s)
|
247
|
+
case s
|
248
|
+
when /^(false|nil|off)$/i
|
249
|
+
false
|
250
|
+
when /^(true|on)$/i
|
251
|
+
true
|
252
|
+
when $stdout
|
253
|
+
"stdout"
|
254
|
+
when $stderr
|
255
|
+
"stderr"
|
256
|
+
else
|
257
|
+
s
|
258
|
+
end
|
259
|
+
end
|
260
|
+
|
261
|
+
def cwd_relative_to_home
|
262
|
+
Pathname.pwd.relative_path_from(Pathname.new(ENV['HOME'])).to_s
|
263
|
+
end
|
264
|
+
|
265
|
+
def cwd_relative_if_under_home
|
266
|
+
home = Pathname.new(ENV['HOME']).realpath
|
267
|
+
path = pwd = Pathname.pwd.realpath
|
268
|
+
while path != home
|
269
|
+
if path.root?
|
270
|
+
return pwd.to_s
|
271
|
+
end
|
272
|
+
path = path.parent
|
273
|
+
end
|
274
|
+
return pwd.relative_path_from(home).to_s
|
275
|
+
end
|
276
|
+
|
277
|
+
# ------------------------------------------------------------------------
|
278
|
+
|
279
|
+
def init_pass_env
|
280
|
+
if envs = self['PASS_ENV']
|
281
|
+
pass_env = {}
|
282
|
+
|
283
|
+
case envs
|
284
|
+
when Array
|
285
|
+
envs.each do |k|
|
286
|
+
k = k.to_s
|
287
|
+
if v = ENV[k]
|
288
|
+
pass_env[k] = v
|
289
|
+
end
|
290
|
+
end
|
291
|
+
when Hash
|
292
|
+
envs.each do |k,v|
|
293
|
+
k = k.to_s
|
294
|
+
if v = ENV[k] || v
|
295
|
+
pass_env[k] = v
|
296
|
+
end
|
297
|
+
end
|
298
|
+
else
|
299
|
+
raise "invalid option for PASS_ENV in pwrake_conf.yaml"
|
300
|
+
end
|
301
|
+
|
302
|
+
if pass_env.empty?
|
303
|
+
self.delete('PASS_ENV')
|
304
|
+
else
|
305
|
+
self['PASS_ENV'] = pass_env
|
306
|
+
end
|
307
|
+
end
|
308
|
+
end
|
309
|
+
|
310
|
+
|
311
|
+
def setup_hosts
|
312
|
+
if @hostfile && @num_threads
|
313
|
+
raise "Cannot set `hostfile' and `num_threads' simultaneously"
|
314
|
+
end
|
315
|
+
@host_map = HostMap.new(@hostfile || @num_threads)
|
316
|
+
end
|
317
|
+
attr_reader :host_map
|
318
|
+
|
319
|
+
|
320
|
+
def clear_gfarm2fs
|
321
|
+
setup_hosts
|
322
|
+
d = File.join(self['GFARM_BASEDIR'],self['GFARM_PREFIX'])
|
323
|
+
rcmd = "
|
324
|
+
for i in #{d}*; do
|
325
|
+
if [ -d \"$i\" ]; then
|
326
|
+
case \"$i\" in
|
327
|
+
*_000) ;;
|
328
|
+
*) fusermount -u $i; rmdir $i ;;
|
329
|
+
esac
|
330
|
+
fi
|
331
|
+
done
|
332
|
+
sleep 1
|
333
|
+
for i in #{d}*_000; do
|
334
|
+
if [ -d \"$i\" ]; then
|
335
|
+
fusermount -u $i; rmdir $i
|
336
|
+
fi
|
337
|
+
done
|
338
|
+
"
|
339
|
+
threads = []
|
340
|
+
@host_map.each do |k,hosts|
|
341
|
+
hosts.each do |info|
|
342
|
+
threads << Thread.new do
|
343
|
+
system "ssh #{info.name} '#{rcmd}'"
|
344
|
+
end
|
345
|
+
end
|
346
|
+
end
|
347
|
+
threads.each{|t| t.join}
|
348
|
+
end
|
349
|
+
|
350
|
+
# ----- finish -----
|
351
|
+
|
352
|
+
def finish_option
|
353
|
+
Log.close
|
354
|
+
end
|
355
|
+
|
356
|
+
end
|
357
|
+
end
|