pwrake 0.9.9.2 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +2 -0
- data/CHANGES_V2.md +90 -0
- data/{LICENSE.txt → MIT-LICENSE} +2 -3
- data/README +12 -0
- data/README.md +75 -52
- data/bin/gfwhere-pipe +23 -12
- data/bin/pwrake +22 -29
- data/bin/pwrake_branch +24 -0
- data/lib/pwrake/branch.rb +22 -0
- data/lib/pwrake/branch/branch.rb +213 -0
- data/lib/pwrake/branch/branch_application.rb +53 -0
- data/lib/pwrake/branch/fiber_queue.rb +36 -0
- data/lib/pwrake/branch/file_utils.rb +101 -0
- data/lib/pwrake/branch/shell.rb +231 -0
- data/lib/pwrake/{profiler.rb → branch/shell_profiler.rb} +28 -27
- data/lib/pwrake/branch/worker_communicator.rb +104 -0
- data/lib/pwrake/{gfarm_feature.rb → gfarm/gfarm_path.rb} +2 -100
- data/lib/pwrake/gfarm/gfarm_postprocess.rb +53 -0
- data/lib/pwrake/iomux/channel.rb +70 -0
- data/lib/pwrake/iomux/handler.rb +124 -0
- data/lib/pwrake/iomux/handler_set.rb +35 -0
- data/lib/pwrake/iomux/runner.rb +62 -0
- data/lib/pwrake/logger.rb +3 -150
- data/lib/pwrake/master.rb +30 -137
- data/lib/pwrake/master/fiber_pool.rb +69 -0
- data/lib/pwrake/master/idle_cores.rb +30 -0
- data/lib/pwrake/master/master.rb +345 -0
- data/lib/pwrake/master/master_application.rb +150 -0
- data/lib/pwrake/master/postprocess.rb +16 -0
- data/lib/pwrake/{graphviz.rb → misc/graphviz.rb} +0 -0
- data/lib/pwrake/{mcgp.rb → misc/mcgp.rb} +63 -42
- data/lib/pwrake/option/host_map.rb +158 -0
- data/lib/pwrake/option/option.rb +357 -0
- data/lib/pwrake/option/option_filesystem.rb +112 -0
- data/lib/pwrake/queue/locality_aware_queue.rb +158 -0
- data/lib/pwrake/queue/no_action_queue.rb +67 -0
- data/lib/pwrake/queue/queue_array.rb +366 -0
- data/lib/pwrake/queue/task_queue.rb +164 -0
- data/lib/pwrake/report.rb +1 -0
- data/lib/pwrake/report/parallelism.rb +9 -3
- data/lib/pwrake/report/report.rb +50 -103
- data/lib/pwrake/report/task_stat.rb +83 -0
- data/lib/pwrake/task/task_algorithm.rb +107 -0
- data/lib/pwrake/task/task_manager.rb +32 -0
- data/lib/pwrake/task/task_property.rb +98 -0
- data/lib/pwrake/task/task_rank.rb +48 -0
- data/lib/pwrake/task/task_wrapper.rb +296 -0
- data/lib/pwrake/version.rb +1 -1
- data/lib/pwrake/worker/executor.rb +169 -0
- data/lib/pwrake/worker/gfarm_directory.rb +90 -0
- data/lib/pwrake/worker/invoker.rb +199 -0
- data/lib/pwrake/worker/load.rb +14 -0
- data/lib/pwrake/worker/log_executor.rb +73 -0
- data/lib/pwrake/worker/shared_directory.rb +74 -0
- data/lib/pwrake/worker/worker_main.rb +14 -0
- data/lib/pwrake/worker/writer.rb +59 -0
- data/setup.rb +1212 -1502
- data/spec/003/Rakefile +2 -2
- data/spec/008/Rakefile +2 -1
- data/spec/009/Rakefile +1 -1
- data/spec/009/pwrake_conf.yaml +1 -3
- data/spec/hosts +0 -2
- data/spec/pwrake_spec.rb +9 -8
- metadata +50 -21
- data/lib/pwrake.rb +0 -19
- data/lib/pwrake/application.rb +0 -232
- data/lib/pwrake/counter.rb +0 -54
- data/lib/pwrake/file_utils.rb +0 -98
- data/lib/pwrake/gfwhere_pool.rb +0 -109
- data/lib/pwrake/host_list.rb +0 -88
- data/lib/pwrake/locality_aware_queue.rb +0 -413
- data/lib/pwrake/option.rb +0 -400
- data/lib/pwrake/rake_modify.rb +0 -14
- data/lib/pwrake/shell.rb +0 -186
- data/lib/pwrake/task_algorithm.rb +0 -475
- data/lib/pwrake/task_queue.rb +0 -633
- data/lib/pwrake/timer.rb +0 -22
@@ -0,0 +1,107 @@
|
|
1
|
+
module Pwrake
|
2
|
+
|
3
|
+
InvocationChain = Rake::InvocationChain
|
4
|
+
TaskArguments = Rake::TaskArguments
|
5
|
+
|
6
|
+
module TaskAlgorithm
|
7
|
+
|
8
|
+
attr_reader :wrapper
|
9
|
+
attr_reader :subsequents
|
10
|
+
attr_reader :arguments
|
11
|
+
attr_reader :property
|
12
|
+
|
13
|
+
def pw_search_tasks(args)
|
14
|
+
Log.debug "#{self.class}#pw_search_tasks start, args=#{args.inspect}"
|
15
|
+
tm = Time.now
|
16
|
+
task_args = TaskArguments.new(arg_names, args)
|
17
|
+
#timer = Timer.new("search_task")
|
18
|
+
#h = application.pwrake_options['HALT_QUEUE_WHILE_SEARCH']
|
19
|
+
#application.task_queue.synchronize(h) do
|
20
|
+
search_with_call_chain(nil, task_args, InvocationChain::EMPTY)
|
21
|
+
#end
|
22
|
+
#timer.finish
|
23
|
+
Log.debug "#{self.class}#pw_search_tasks end #{Time.now-tm}"
|
24
|
+
end
|
25
|
+
|
26
|
+
# Same as search, but explicitly pass a call chain to detect
|
27
|
+
# circular dependencies.
|
28
|
+
def search_with_call_chain(subseq, task_args, invocation_chain) # :nodoc:
|
29
|
+
new_chain = InvocationChain.append(self, invocation_chain)
|
30
|
+
@lock.synchronize do
|
31
|
+
if application.options.trace
|
32
|
+
#Log.info "** Search #{name} #{format_search_flags}"
|
33
|
+
application.trace "** Search #{name} #{format_search_flags}"
|
34
|
+
end
|
35
|
+
|
36
|
+
return true if @already_finished # <<--- competition !!!
|
37
|
+
@subsequents ||= []
|
38
|
+
@subsequents << subseq if subseq # <<--- competition !!!
|
39
|
+
|
40
|
+
if ! @already_searched
|
41
|
+
@already_searched = true
|
42
|
+
@arguments = task_args
|
43
|
+
@wrapper = TaskWrapper.new(self,task_args)
|
44
|
+
if @prerequisites.empty?
|
45
|
+
@unfinished_prereq = {}
|
46
|
+
else
|
47
|
+
search_prerequisites(task_args, new_chain)
|
48
|
+
end
|
49
|
+
#check_and_enq
|
50
|
+
if @unfinished_prereq.empty?
|
51
|
+
application.task_queue.enq(@wrapper)
|
52
|
+
end
|
53
|
+
end
|
54
|
+
return false
|
55
|
+
end
|
56
|
+
rescue Exception => ex
|
57
|
+
add_chain_to(ex, new_chain)
|
58
|
+
raise ex
|
59
|
+
end
|
60
|
+
|
61
|
+
# Search all the prerequisites of a task.
|
62
|
+
def search_prerequisites(task_args, invocation_chain) # :nodoc:
|
63
|
+
@unfinished_prereq = {}
|
64
|
+
@prerequisites.each{|t| @unfinished_prereq[t]=true}
|
65
|
+
prerequisite_tasks.each { |prereq|
|
66
|
+
prereq_args = task_args.new_scope(prereq.arg_names)
|
67
|
+
if prereq.search_with_call_chain(self, prereq_args, invocation_chain)
|
68
|
+
@unfinished_prereq.delete(prereq.name)
|
69
|
+
end
|
70
|
+
}
|
71
|
+
end
|
72
|
+
|
73
|
+
# Format the trace flags for display.
|
74
|
+
def format_search_flags
|
75
|
+
flags = []
|
76
|
+
flags << "finished" if @already_finished
|
77
|
+
flags << "first_time" unless @already_searched
|
78
|
+
flags << "not_needed" unless needed?
|
79
|
+
flags.empty? ? "" : "(" + flags.join(", ") + ")"
|
80
|
+
end
|
81
|
+
private :format_search_flags
|
82
|
+
|
83
|
+
def pw_enq_subsequents
|
84
|
+
t = Time.now
|
85
|
+
#h = application.pwrake_options['HALT_QUEUE_WHILE_SEARCH']
|
86
|
+
#application.task_queue.synchronize(h) do
|
87
|
+
@subsequents.each do |t| # <<--- competition !!!
|
88
|
+
if t && t.check_prereq_finished(self.name)
|
89
|
+
application.task_queue.enq(t.wrapper)
|
90
|
+
end
|
91
|
+
end
|
92
|
+
#end
|
93
|
+
@already_finished = true # <<--- competition !!!
|
94
|
+
end
|
95
|
+
|
96
|
+
def check_prereq_finished(preq_name=nil)
|
97
|
+
@unfinished_prereq.delete(preq_name)
|
98
|
+
@unfinished_prereq.empty?
|
99
|
+
end
|
100
|
+
|
101
|
+
def pw_set_property(property)
|
102
|
+
@property = property
|
103
|
+
self
|
104
|
+
end
|
105
|
+
|
106
|
+
end
|
107
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
module Pwrake
|
2
|
+
|
3
|
+
module TaskManager
|
4
|
+
|
5
|
+
def initialize
|
6
|
+
@property_by_block = {}
|
7
|
+
@last_property = TaskProperty.new
|
8
|
+
super
|
9
|
+
end
|
10
|
+
|
11
|
+
def last_description=(description)
|
12
|
+
@last_property.parse_description(description)
|
13
|
+
super
|
14
|
+
end
|
15
|
+
|
16
|
+
def define_task(task_class, *args, &block) # :nodoc:
|
17
|
+
prop = @property_by_block[block.object_id]
|
18
|
+
if prop.nil?
|
19
|
+
prop = @last_property
|
20
|
+
@last_property = TaskProperty.new
|
21
|
+
end
|
22
|
+
super.pw_set_property(prop)
|
23
|
+
end
|
24
|
+
|
25
|
+
def create_rule(*args, &block) # :nodoc:
|
26
|
+
@property_by_block[block.object_id] = @last_property
|
27
|
+
@last_property = TaskProperty.new
|
28
|
+
super
|
29
|
+
end
|
30
|
+
|
31
|
+
end
|
32
|
+
end
|
@@ -0,0 +1,98 @@
|
|
1
|
+
module Pwrake
|
2
|
+
|
3
|
+
class TaskProperty
|
4
|
+
|
5
|
+
attr_reader :ncore, :exclusive, :allow, :deny, :order_allow_deny,
|
6
|
+
:disable_steal
|
7
|
+
|
8
|
+
def parse_description(description)
|
9
|
+
if /\bn_?cores?[=:]\s*([+-]?\d+)/i =~ description
|
10
|
+
@ncore = $1.to_i
|
11
|
+
end
|
12
|
+
if /\bexclusive[=:]\s*(\S+)/i =~ description
|
13
|
+
if /^(y|t)/i =~ $1
|
14
|
+
@exclusive = true
|
15
|
+
end
|
16
|
+
end
|
17
|
+
if /\ballow[=:]\s*(\S+)/i =~ description
|
18
|
+
@allow = $1
|
19
|
+
end
|
20
|
+
if /\bdeny[=:]\s*(\S+)/i =~ description
|
21
|
+
@deny = $1
|
22
|
+
end
|
23
|
+
if /\border[=:]\s*(\S+)/i =~ description
|
24
|
+
case $1
|
25
|
+
when /allow,deny/i
|
26
|
+
@order_allow_deny = true
|
27
|
+
when /deny,allow/i
|
28
|
+
@order_allow_deny = false
|
29
|
+
end
|
30
|
+
end
|
31
|
+
if /\bsteal[=:]\s*(\S+)/i =~ description
|
32
|
+
if /^(n|f)/i =~ $1
|
33
|
+
@disable_steal = true
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
def acceptable_for(host_info)
|
39
|
+
if @disable_steal && host_info.steal_flag
|
40
|
+
#Log.debug("@disable_steal && host_info.steal_flag")
|
41
|
+
return false
|
42
|
+
end
|
43
|
+
ncore = (@exclusive) ? 0 : (@ncore || 1)
|
44
|
+
if ncore > 0
|
45
|
+
return false if ncore > host_info.idle_cores
|
46
|
+
else
|
47
|
+
n = host_info.ncore + ncore
|
48
|
+
return false if n < 1 || n > host_info.idle_cores
|
49
|
+
end
|
50
|
+
hn = host_info.name
|
51
|
+
if @allow
|
52
|
+
if @deny
|
53
|
+
if @order_allow_deny
|
54
|
+
return false if !File.fnmatch(@allow,hn) || File.fnmatch(@deny,hn)
|
55
|
+
else
|
56
|
+
return false if File.fnmatch(@deny,hn) && !File.fnmatch(@allow,hn)
|
57
|
+
end
|
58
|
+
else
|
59
|
+
return false if !File.fnmatch(@allow,hn)
|
60
|
+
end
|
61
|
+
else
|
62
|
+
if @deny
|
63
|
+
return false if File.fnmatch(@deny,hn)
|
64
|
+
end
|
65
|
+
end
|
66
|
+
return true
|
67
|
+
end
|
68
|
+
|
69
|
+
def n_used_cores(host_info=nil)
|
70
|
+
nc_node = host_info && host_info.ncore
|
71
|
+
if @ncore.nil?
|
72
|
+
return 1
|
73
|
+
elsif @ncore > 0
|
74
|
+
if nc_node && @ncore > nc_node
|
75
|
+
m = "ncore=#{@ncore} must be <= nc_node=#{nc_node}"
|
76
|
+
Log.fatal m
|
77
|
+
raise RuntimeError,m
|
78
|
+
end
|
79
|
+
return @ncore
|
80
|
+
else
|
81
|
+
if nc_node.nil?
|
82
|
+
m = "host_info.ncore is not set"
|
83
|
+
Log.fatal m
|
84
|
+
raise RuntimeError,m
|
85
|
+
end
|
86
|
+
n = @ncore + nc_node
|
87
|
+
if n > 0
|
88
|
+
return n
|
89
|
+
else
|
90
|
+
m = "ncore+nc_node=#{n} must be > 0"
|
91
|
+
Log.fatal m
|
92
|
+
raise RuntimeError,m
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
96
|
+
|
97
|
+
end
|
98
|
+
end
|
@@ -0,0 +1,48 @@
|
|
1
|
+
module Pwrake
|
2
|
+
|
3
|
+
class RankStat
|
4
|
+
|
5
|
+
def initialize
|
6
|
+
@lock = Mutex.new
|
7
|
+
@stat = []
|
8
|
+
end
|
9
|
+
|
10
|
+
def add_sample(rank,elap)
|
11
|
+
@lock.synchronize do
|
12
|
+
stat = @stat[rank]
|
13
|
+
if stat.nil?
|
14
|
+
@stat[rank] = stat = [0,0.0]
|
15
|
+
end
|
16
|
+
stat[0] += 1
|
17
|
+
stat[1] += elap
|
18
|
+
#Log.debug "add_sample rank=#{rank} stat=#{stat.inspect} weight=#{stat[0]/stat[1]}"
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
def rank_weight
|
23
|
+
@lock.synchronize do
|
24
|
+
sum = 0.0
|
25
|
+
count = 0
|
26
|
+
weight = @stat.map do |stat|
|
27
|
+
if stat
|
28
|
+
w = stat[0]/stat[1]
|
29
|
+
sum += w
|
30
|
+
count += 1
|
31
|
+
w
|
32
|
+
else
|
33
|
+
nil
|
34
|
+
end
|
35
|
+
end
|
36
|
+
if count == 0
|
37
|
+
avg = 1.0
|
38
|
+
else
|
39
|
+
avg = sum/count
|
40
|
+
end
|
41
|
+
[weight, avg]
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
RANK_STAT = RankStat.new
|
47
|
+
|
48
|
+
end
|
@@ -0,0 +1,296 @@
|
|
1
|
+
require 'forwardable'
|
2
|
+
|
3
|
+
module Pwrake
|
4
|
+
|
5
|
+
class TaskWrapper
|
6
|
+
extend Forwardable
|
7
|
+
|
8
|
+
@@current_id = 1
|
9
|
+
@@task_logger = nil
|
10
|
+
|
11
|
+
def initialize(task,task_args=nil)
|
12
|
+
@task = task
|
13
|
+
@task_args = task_args
|
14
|
+
@property = task.property
|
15
|
+
@task_id = @@current_id
|
16
|
+
@@current_id += 1
|
17
|
+
@location = []
|
18
|
+
@group = []
|
19
|
+
@group_id
|
20
|
+
@suggest_location = nil
|
21
|
+
@file_stat
|
22
|
+
@input_file_size
|
23
|
+
@input_file_mtime
|
24
|
+
@rank
|
25
|
+
@priority
|
26
|
+
@lock_rank = Monitor.new
|
27
|
+
@executed = false
|
28
|
+
@assigned = []
|
29
|
+
@exec_host = nil
|
30
|
+
end
|
31
|
+
|
32
|
+
def_delegators :@task, :name, :actions, :prerequisites, :subsequents
|
33
|
+
def_delegators :@property, :acceptable_for
|
34
|
+
|
35
|
+
attr_reader :task, :task_id, :group, :group_id, :file_stat
|
36
|
+
attr_reader :location
|
37
|
+
attr_reader :assigned
|
38
|
+
attr_accessor :executed
|
39
|
+
attr_accessor :exec_host
|
40
|
+
attr_accessor :shell_id, :status
|
41
|
+
|
42
|
+
def self.format_time(t)
|
43
|
+
t.strftime("%F %T.%L")
|
44
|
+
end
|
45
|
+
|
46
|
+
def self.init_task_logger(option)
|
47
|
+
if dir = option['LOG_DIR']
|
48
|
+
fn = File.join(dir,option['TASK_CSV_FILE'])
|
49
|
+
@@task_logger = CSV.open(fn,'w')
|
50
|
+
@@task_logger.puts %w[
|
51
|
+
task_id task_name start_time end_time elap_time preq preq_host
|
52
|
+
exec_host shell_id has_action executed file_size file_mtime file_host
|
53
|
+
]
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
def self.close_task_logger
|
58
|
+
@@task_logger.close if @@task_logger
|
59
|
+
end
|
60
|
+
|
61
|
+
def preprocess
|
62
|
+
if @shell = Pwrake::Shell.current
|
63
|
+
@shell.current_task = self
|
64
|
+
end
|
65
|
+
@time_start = Time.now
|
66
|
+
end
|
67
|
+
|
68
|
+
def postprocess(location)
|
69
|
+
@executed = true if !@task.actions.empty?
|
70
|
+
tm_taskend = Time.now
|
71
|
+
if is_file_task?
|
72
|
+
t = Time.now
|
73
|
+
if File.exist?(name)
|
74
|
+
@file_stat = File::Stat.new(name)
|
75
|
+
@location = location
|
76
|
+
end
|
77
|
+
end
|
78
|
+
#Log.debug "postprocess time=#{Time.now-tm_taskend}"
|
79
|
+
log_task
|
80
|
+
@shell.current_task = nil if @shell
|
81
|
+
if @status=="end"
|
82
|
+
@task.pw_enq_subsequents
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
def log_task
|
87
|
+
@time_end = Time.now
|
88
|
+
#
|
89
|
+
loc = suggest_location()
|
90
|
+
shell = Pwrake::Shell.current
|
91
|
+
#
|
92
|
+
if loc && !loc.empty? && shell && !actions.empty?
|
93
|
+
Rake.application.count( loc, shell.host )
|
94
|
+
end
|
95
|
+
return if !@@task_logger
|
96
|
+
#
|
97
|
+
elap = @time_end - @time_start
|
98
|
+
if has_output_file?
|
99
|
+
RANK_STAT.add_sample(rank,elap)
|
100
|
+
end
|
101
|
+
#
|
102
|
+
if @file_stat
|
103
|
+
fstat = [@file_stat.size, @file_stat.mtime, self.location.join('|')]
|
104
|
+
else
|
105
|
+
fstat = [nil]*3
|
106
|
+
end
|
107
|
+
#
|
108
|
+
# task_id task_name start_time end_time elap_time preq preq_host
|
109
|
+
# exec_host shell_id has_action executed file_size file_mtime file_host
|
110
|
+
#
|
111
|
+
row = [ @task_id, name, @time_start, @time_end, elap,
|
112
|
+
prerequisites, loc, @exec_host, @shell_id,
|
113
|
+
(actions.empty?) ? 0 : 1,
|
114
|
+
(@executed) ? 1 : 0,
|
115
|
+
*fstat ]
|
116
|
+
row.map!{|x|
|
117
|
+
if x.kind_of?(Time)
|
118
|
+
TaskWrapper.format_time(x)
|
119
|
+
elsif x.kind_of?(Array)
|
120
|
+
if x.empty?
|
121
|
+
nil
|
122
|
+
else
|
123
|
+
x.join('|')
|
124
|
+
end
|
125
|
+
else
|
126
|
+
x
|
127
|
+
end
|
128
|
+
}
|
129
|
+
@@task_logger << row
|
130
|
+
#
|
131
|
+
clsname = @task.class.to_s.sub(/^(Rake|Pwrake)::/o,"")
|
132
|
+
msg = '%s:"%s" %s: id=%d elap=%.6f exec_host=%s' %
|
133
|
+
[clsname,name,@status,@task_id,elap,@exec_host]
|
134
|
+
if @status=="end"
|
135
|
+
Log.info msg
|
136
|
+
else
|
137
|
+
Log.error msg
|
138
|
+
end
|
139
|
+
end
|
140
|
+
|
141
|
+
def is_file_task?
|
142
|
+
@task.kind_of?(Rake::FileTask)
|
143
|
+
end
|
144
|
+
|
145
|
+
def has_output_file?
|
146
|
+
is_file_task? && !actions.empty?
|
147
|
+
end
|
148
|
+
|
149
|
+
def has_input_file?
|
150
|
+
is_file_task? && !prerequisites.empty?
|
151
|
+
end
|
152
|
+
|
153
|
+
def has_action?
|
154
|
+
!@task.actions.empty?
|
155
|
+
end
|
156
|
+
|
157
|
+
def location=(a)
|
158
|
+
@location = a
|
159
|
+
@group = []
|
160
|
+
#@location.each do |host|
|
161
|
+
# @group |= [Rake.application.host_list.host2group[host]]
|
162
|
+
#end
|
163
|
+
end
|
164
|
+
|
165
|
+
def suggest_location=(a)
|
166
|
+
@suggest_location = a
|
167
|
+
end
|
168
|
+
|
169
|
+
def suggest_location
|
170
|
+
if has_input_file? && @suggest_location.nil?
|
171
|
+
@suggest_location = []
|
172
|
+
loc_fsz = Hash.new(0)
|
173
|
+
prerequisites.each do |preq|
|
174
|
+
t = Rake.application[preq].wrapper
|
175
|
+
loc = t.location
|
176
|
+
fsz = t.file_size
|
177
|
+
if loc && fsz > 0
|
178
|
+
loc.each do |h|
|
179
|
+
loc_fsz[h] += fsz
|
180
|
+
end
|
181
|
+
end
|
182
|
+
end
|
183
|
+
#Log.debug "input=#{prerequisites.join('|')}"
|
184
|
+
if !loc_fsz.empty?
|
185
|
+
half_max_fsz = loc_fsz.values.max / 2
|
186
|
+
Log.debug "loc_fsz=#{loc_fsz.inspect} half_max_fsz=#{half_max_fsz}"
|
187
|
+
loc_fsz.each do |h,sz|
|
188
|
+
if sz > half_max_fsz
|
189
|
+
@suggest_location << h
|
190
|
+
end
|
191
|
+
end
|
192
|
+
end
|
193
|
+
end
|
194
|
+
@suggest_location
|
195
|
+
end
|
196
|
+
|
197
|
+
def rank
|
198
|
+
#@lock_rank.synchronize do
|
199
|
+
if @rank.nil?
|
200
|
+
if subsequents.nil? || subsequents.empty?
|
201
|
+
@rank = 0
|
202
|
+
else
|
203
|
+
max_rank = 0
|
204
|
+
subsequents.each do |subsq|
|
205
|
+
r = subsq.wrapper.rank
|
206
|
+
if max_rank < r
|
207
|
+
max_rank = r
|
208
|
+
end
|
209
|
+
end
|
210
|
+
if has_output_file?
|
211
|
+
step = 1
|
212
|
+
else
|
213
|
+
step = 0
|
214
|
+
end
|
215
|
+
@rank = max_rank + step
|
216
|
+
end
|
217
|
+
Log.debug "Task[#{name}] rank=#{@rank.inspect}"
|
218
|
+
end
|
219
|
+
#end
|
220
|
+
@rank
|
221
|
+
end
|
222
|
+
|
223
|
+
def file_size
|
224
|
+
@file_stat ? @file_stat.size : 0
|
225
|
+
end
|
226
|
+
|
227
|
+
def file_mtime
|
228
|
+
@file_stat ? @file_stat.mtime : Time.at(0)
|
229
|
+
end
|
230
|
+
|
231
|
+
def input_file_size
|
232
|
+
unless @input_file_size
|
233
|
+
@input_file_size = 0
|
234
|
+
prerequisites.each do |preq|
|
235
|
+
@input_file_size += Rake.application[preq].wrapper.file_size
|
236
|
+
end
|
237
|
+
end
|
238
|
+
@input_file_size
|
239
|
+
end
|
240
|
+
|
241
|
+
def input_file_mtime
|
242
|
+
if has_input_file? && @input_file_mtime.nil?
|
243
|
+
hash = Hash.new
|
244
|
+
max_sz = 0
|
245
|
+
prerequisites.each do |preq|
|
246
|
+
t = Rake.application[preq].wrapper
|
247
|
+
sz = t.file_size
|
248
|
+
if sz > 0
|
249
|
+
hash[t] = sz
|
250
|
+
if sz > max_sz
|
251
|
+
max_sz = sz
|
252
|
+
end
|
253
|
+
end
|
254
|
+
end
|
255
|
+
half_max_sz = max_sz / 2
|
256
|
+
hash.each do |t,sz|
|
257
|
+
if sz > half_max_sz
|
258
|
+
time = t.file_mtime
|
259
|
+
if @input_file_mtime.nil? || @input_file_mtime < time
|
260
|
+
@input_file_mtime = time
|
261
|
+
end
|
262
|
+
end
|
263
|
+
end
|
264
|
+
end
|
265
|
+
@input_file_mtime
|
266
|
+
end
|
267
|
+
|
268
|
+
def priority
|
269
|
+
if has_input_file? && @priority.nil?
|
270
|
+
sum_tm = 0
|
271
|
+
sum_sz = 0
|
272
|
+
prerequisites.each do |preq|
|
273
|
+
pq = Rake.application[preq].wrapper
|
274
|
+
sz = pq.file_size
|
275
|
+
if sz > 0
|
276
|
+
tm = pq.file_mtime - START_TIME
|
277
|
+
sum_tm += tm * sz
|
278
|
+
sum_sz += sz
|
279
|
+
end
|
280
|
+
end
|
281
|
+
if sum_sz > 0
|
282
|
+
@priority = sum_tm / sum_sz
|
283
|
+
else
|
284
|
+
@priority = 0
|
285
|
+
end
|
286
|
+
Log.debug "task_name=#{name} priority=#{@priority} sum_file_size=#{sum_sz}"
|
287
|
+
end
|
288
|
+
@priority || 0
|
289
|
+
end
|
290
|
+
|
291
|
+
def n_used_cores(host_info=nil)
|
292
|
+
@n_used_cores ||= @property.n_used_cores(host_info)
|
293
|
+
end
|
294
|
+
|
295
|
+
end
|
296
|
+
end
|