pwrake 0.9.5 → 0.9.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/README.md +83 -4
- data/lib/pwrake/application.rb +34 -0
- data/lib/pwrake/file_utils.rb +43 -16
- data/lib/pwrake/gfarm_feature.rb +79 -40
- data/lib/pwrake/locality_aware_queue.rb +141 -111
- data/lib/pwrake/logger.rb +7 -1
- data/lib/pwrake/master.rb +21 -2
- data/lib/pwrake/option.rb +106 -56
- data/lib/pwrake/profiler.rb +44 -12
- data/lib/pwrake/report/parallelism.rb +262 -0
- data/lib/pwrake/report/report.rb +355 -0
- data/lib/pwrake/report/report_multi.rb +196 -0
- data/lib/pwrake/report/stat.rb +115 -0
- data/lib/pwrake/report.rb +6 -0
- data/lib/pwrake/shell.rb +30 -10
- data/lib/pwrake/task_algorithm.rb +174 -50
- data/lib/pwrake/task_queue.rb +130 -50
- data/lib/pwrake/version.rb +1 -1
- data/spec/004/Rakefile +1 -1
- data/spec/009/pwrake_conf.yaml +1 -0
- data/spec/011/Rakefile +15 -0
- data/spec/helper.rb +7 -3
- data/spec/pwrake_spec.rb +12 -0
- metadata +15 -9
@@ -7,6 +7,70 @@ module Pwrake
|
|
7
7
|
end
|
8
8
|
|
9
9
|
|
10
|
+
class LocalityConditionVariable < ConditionVariable
|
11
|
+
def signal(hints=nil)
|
12
|
+
if hints.nil?
|
13
|
+
super()
|
14
|
+
elsif Array===hints
|
15
|
+
thread = nil
|
16
|
+
@waiters_mutex.synchronize do
|
17
|
+
@waiters.each do |t,v|
|
18
|
+
if hints.include?(t[:hint])
|
19
|
+
thread = t
|
20
|
+
break
|
21
|
+
end
|
22
|
+
end
|
23
|
+
if thread
|
24
|
+
@waiters.delete(thread)
|
25
|
+
else
|
26
|
+
thread,_ = @waiters.shift
|
27
|
+
end
|
28
|
+
end
|
29
|
+
Log.debug "--- LCV#signal: hints=#{hints.inspect} thread_to_run=#{thread.inspect} @waiters.size=#{@waiters.size}"
|
30
|
+
begin
|
31
|
+
thread.run if thread
|
32
|
+
rescue ThreadError
|
33
|
+
retry # t was already dead?
|
34
|
+
end
|
35
|
+
else
|
36
|
+
raise ArgumentError,"argument must be an Array"
|
37
|
+
end
|
38
|
+
self
|
39
|
+
end
|
40
|
+
|
41
|
+
def broadcast(hints=nil)
|
42
|
+
if hints.nil?
|
43
|
+
super()
|
44
|
+
elsif Array===hints
|
45
|
+
threads = []
|
46
|
+
@waiters_mutex.synchronize do
|
47
|
+
hints.each do |h|
|
48
|
+
@waiters.each do |t,v|
|
49
|
+
if t[:hint] == h
|
50
|
+
threads << t
|
51
|
+
break
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
threads.each do |t|
|
56
|
+
@waiters.delete(t)
|
57
|
+
end
|
58
|
+
end
|
59
|
+
Log.debug "--- LCV#broadcast: hints=#{hints.inspect} threads_to_run=#{threads.inspect} @waiters.size=#{@waiters.size}"
|
60
|
+
threads.each do |t|
|
61
|
+
begin
|
62
|
+
t.run
|
63
|
+
rescue ThreadError
|
64
|
+
end
|
65
|
+
end
|
66
|
+
else
|
67
|
+
raise ArgumentError,"argument must be an Array"
|
68
|
+
end
|
69
|
+
self
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
|
10
74
|
class LocalityAwareQueue < TaskQueue
|
11
75
|
|
12
76
|
class Throughput
|
@@ -67,144 +131,90 @@ module Pwrake
|
|
67
131
|
end # class Throughput
|
68
132
|
|
69
133
|
|
70
|
-
def
|
71
|
-
|
134
|
+
def init_queue(hosts)
|
135
|
+
@cv = LocalityConditionVariable.new
|
72
136
|
@hosts = hosts
|
73
137
|
@throughput = Throughput.new
|
74
138
|
@size = 0
|
75
|
-
@q =
|
76
|
-
@
|
77
|
-
@
|
78
|
-
@
|
79
|
-
@
|
80
|
-
@
|
81
|
-
@
|
82
|
-
|
83
|
-
@thread = Thread.new{thread_loop}
|
139
|
+
@q = {}
|
140
|
+
@hosts.each{|h| @q[h]=@array_class.new}
|
141
|
+
@q_remote = @array_class.new
|
142
|
+
@q_later = Array.new
|
143
|
+
@enable_steal = !Pwrake.application.pwrake_options['DISABLE_STEAL']
|
144
|
+
@steal_wait = (Pwrake.application.pwrake_options['STEAL_WAIT'] || 0).to_i
|
145
|
+
@steal_wait_max = (Pwrake.application.pwrake_options['STEAL_WAIT_MAX'] || 10).to_i
|
146
|
+
Log.info("-- @steal_wait=#{@steal_wait} @steal_wait_max=#{@steal_wait_max}")
|
84
147
|
end
|
85
148
|
|
86
149
|
attr_reader :size
|
87
150
|
|
88
151
|
|
89
|
-
def
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
end
|
94
|
-
end
|
95
|
-
|
96
|
-
|
97
|
-
def time_out
|
98
|
-
if @size == 0
|
99
|
-
if @q1.size == 1
|
100
|
-
0.25
|
101
|
-
else
|
102
|
-
0.5
|
103
|
-
end
|
152
|
+
def enq_impl(t)
|
153
|
+
hints = t.suggest_location
|
154
|
+
if hints.nil? || hints.empty?
|
155
|
+
@q_later.push(t)
|
104
156
|
else
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
1.0
|
109
|
-
else
|
110
|
-
3.0
|
111
|
-
end
|
112
|
-
end
|
113
|
-
end
|
114
|
-
|
115
|
-
|
116
|
-
def bulk_mvq
|
117
|
-
a = nil
|
118
|
-
@mutex.synchronize do
|
119
|
-
return if @q1.empty?
|
120
|
-
|
121
|
-
time_now = Time.now
|
122
|
-
interval = time_now-@time_prev
|
123
|
-
return if time_now-@time_prev < time_out
|
124
|
-
|
125
|
-
log_bulk_mvq
|
126
|
-
@time_prev = time_now
|
127
|
-
|
128
|
-
a = @q1
|
129
|
-
@q1 = []
|
130
|
-
end
|
131
|
-
|
132
|
-
where(a)
|
133
|
-
|
134
|
-
@mutex.synchronize do
|
135
|
-
while t = a.shift
|
136
|
-
mvq(t)
|
137
|
-
end
|
138
|
-
@cv.broadcast
|
139
|
-
end
|
140
|
-
end
|
141
|
-
|
142
|
-
def log_bulk_mvq
|
143
|
-
msg = @q1[0..2].map{|t| t.name}.inspect
|
144
|
-
msg.sub!(/]$/,",...") if @q1.size > 3
|
145
|
-
Log.info "-- bulk_mvq interval=%.6fs @size=#{@size} @q1.size=#{@q1.size} @q1=#{msg}"%(Time.now-@time_prev)
|
146
|
-
end
|
147
|
-
|
148
|
-
|
149
|
-
def where(task_list)
|
150
|
-
# implemented in child class
|
151
|
-
end
|
152
|
-
|
153
|
-
|
154
|
-
def mvq(t)
|
155
|
-
stored = false
|
156
|
-
if t.respond_to? :location
|
157
|
-
t.location.each do |h|
|
158
|
-
if q = @q2[h]
|
157
|
+
stored = false
|
158
|
+
hints.each do |h|
|
159
|
+
if q = @q[h]
|
159
160
|
t.assigned.push(h)
|
160
161
|
q.push(t)
|
161
162
|
stored = true
|
162
163
|
end
|
163
164
|
end
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
# @q2[nil].push(t)
|
165
|
+
if !stored
|
166
|
+
@q_remote.push(t)
|
167
|
+
end
|
168
168
|
end
|
169
169
|
@size += 1
|
170
170
|
end
|
171
171
|
|
172
172
|
|
173
|
-
def enq_impl(item,hint=nil)
|
174
|
-
#Log.debug "--- #{self.class}#enq_impl #{item.inspect}"
|
175
|
-
@q1.push(item)
|
176
|
-
end
|
177
|
-
|
178
|
-
def enq_finish
|
179
|
-
@thread.run if @thread.alive?
|
180
|
-
end
|
181
|
-
|
182
173
|
def deq_impl(host,n)
|
183
|
-
@thread.run if @thread.alive?
|
184
|
-
|
185
174
|
if t = deq_locate(host)
|
186
175
|
Log.info "-- deq_locate n=#{n} task=#{t.name} host=#{host}"
|
176
|
+
Log.debug "--- deq_impl\n#{inspect_q}"
|
177
|
+
return t
|
178
|
+
end
|
179
|
+
|
180
|
+
if !@q_remote.empty?
|
181
|
+
t = @q_remote.shift
|
182
|
+
Log.info "-- deq_remote n=#{n} task=#{t.name} host=#{host}"
|
183
|
+
Log.debug "--- deq_impl\n#{inspect_q}"
|
187
184
|
return t
|
188
185
|
end
|
189
186
|
|
190
|
-
if
|
187
|
+
if !@q_later.empty?
|
188
|
+
t = @q_later.shift
|
189
|
+
Log.info "-- deq_later n=#{n} task=#{t.name} host=#{host}"
|
190
|
+
Log.debug "--- deq_impl\n#{inspect_q}"
|
191
|
+
return t
|
192
|
+
end
|
193
|
+
|
194
|
+
if @enable_steal && n > 0
|
191
195
|
if t = deq_steal(host)
|
192
196
|
Log.info "-- deq_steal n=#{n} task=#{t.name} host=#{host}"
|
197
|
+
Log.debug "--- deq_impl\n#{inspect_q}"
|
193
198
|
return t
|
194
199
|
end
|
195
200
|
end
|
196
201
|
|
197
|
-
m =
|
202
|
+
m = [@steal_wait*(2**n), @steal_wait_max].min
|
198
203
|
@cv.wait(@mutex,m)
|
199
204
|
nil
|
200
205
|
end
|
201
206
|
|
202
207
|
|
203
208
|
def deq_locate(host)
|
204
|
-
q = @
|
209
|
+
q = @q[host]
|
205
210
|
if q && !q.empty?
|
206
211
|
t = q.shift
|
207
|
-
t.assigned.each
|
212
|
+
t.assigned.each do |h|
|
213
|
+
a = @q[h]
|
214
|
+
if i = a.index(t)
|
215
|
+
a.delete_at(i)
|
216
|
+
end
|
217
|
+
end
|
208
218
|
@size -= 1
|
209
219
|
return t
|
210
220
|
else
|
@@ -216,38 +226,58 @@ module Pwrake
|
|
216
226
|
# select a task based on many and close
|
217
227
|
max_host = nil
|
218
228
|
max_num = 0
|
219
|
-
@
|
229
|
+
@q.each do |h,a|
|
220
230
|
if !a.empty?
|
221
|
-
d =
|
231
|
+
d = a.size
|
222
232
|
if d > max_num
|
223
233
|
max_host = h
|
224
234
|
max_num = d
|
225
235
|
end
|
226
236
|
end
|
227
237
|
end
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
232
|
-
|
238
|
+
Log.info "-- deq_steal max_host=#{max_host} max_num=#{max_num}"
|
239
|
+
deq_locate(max_host)
|
240
|
+
end
|
241
|
+
|
242
|
+
def inspect_q
|
243
|
+
s = ""
|
244
|
+
b = proc{|h,q|
|
245
|
+
s += " #{h}: size=#{q.size} "
|
246
|
+
case q.size
|
247
|
+
when 0
|
248
|
+
s += "[]\n"
|
249
|
+
when 1
|
250
|
+
s += "[#{q[0].name}]\n"
|
251
|
+
else
|
252
|
+
s += "[#{q[0].name},..]\n"
|
253
|
+
end
|
254
|
+
}
|
255
|
+
@q.each(&b)
|
256
|
+
b.call("remote",@q_remote)
|
257
|
+
b.call("later",@q_later)
|
258
|
+
s
|
233
259
|
end
|
234
260
|
|
235
261
|
def size
|
236
|
-
@
|
262
|
+
@size
|
237
263
|
end
|
238
264
|
|
239
265
|
def clear
|
240
|
-
@
|
241
|
-
@
|
266
|
+
@q.each{|h,q| q.clear}
|
267
|
+
@q_remote.clear
|
268
|
+
@q_later.clear
|
269
|
+
@reserved_q.clear
|
242
270
|
end
|
243
271
|
|
244
272
|
def empty?
|
245
|
-
@
|
273
|
+
@q.all?{|h,q| q.empty?} &&
|
274
|
+
@q_remote.empty? &&
|
275
|
+
@q_later.empty? &&
|
276
|
+
@reserved_q.empty?
|
246
277
|
end
|
247
278
|
|
248
279
|
def finish
|
249
280
|
super
|
250
|
-
@thread.run if @thread.alive?
|
251
281
|
end
|
252
282
|
|
253
283
|
end
|
data/lib/pwrake/logger.rb
CHANGED
@@ -142,12 +142,18 @@ module Pwrake
|
|
142
142
|
Timer.new(prefix,*args)
|
143
143
|
end
|
144
144
|
|
145
|
-
def
|
145
|
+
def stderr_puts(message)
|
146
146
|
LOCK.synchronize do
|
147
147
|
$stderr.write(message+"\n")
|
148
148
|
end
|
149
149
|
end
|
150
150
|
|
151
|
+
def stdout_puts(message)
|
152
|
+
LOCK.synchronize do
|
153
|
+
$stdout.write(message+"\n")
|
154
|
+
end
|
155
|
+
end
|
156
|
+
|
151
157
|
end
|
152
158
|
|
153
159
|
end # module Pwrake
|
data/lib/pwrake/master.rb
CHANGED
@@ -6,6 +6,7 @@ module Pwrake
|
|
6
6
|
|
7
7
|
def current_shell=(a)
|
8
8
|
Thread.current[:shell] = a
|
9
|
+
Thread.current[:hint] = a.host
|
9
10
|
end
|
10
11
|
|
11
12
|
module_function :current_shell, :current_shell=
|
@@ -15,16 +16,23 @@ module Pwrake
|
|
15
16
|
include Pwrake::Option
|
16
17
|
|
17
18
|
attr_reader :task_queue
|
19
|
+
attr_reader :finish_queue
|
18
20
|
attr_reader :shell_set
|
21
|
+
attr_reader :filesystem
|
22
|
+
attr_reader :postprocess
|
19
23
|
|
20
24
|
def initialize
|
21
25
|
init_option # Pwrake::Option
|
22
26
|
setup_option # Pwrake::Option
|
23
27
|
@started = false
|
28
|
+
@lock = Mutex.new
|
29
|
+
@current_task_id = -1
|
24
30
|
end
|
25
31
|
|
26
32
|
def start
|
27
33
|
return if @task_queue
|
34
|
+
timer = Timer.new("start_worker")
|
35
|
+
@finish_queue = Queue.new
|
28
36
|
@task_queue = @queue_class.new(@core_list)
|
29
37
|
@task_queue.enable_steal = !Rake.application.options.disable_steal
|
30
38
|
@shell_set = []
|
@@ -32,6 +40,7 @@ module Pwrake
|
|
32
40
|
@shell_set << @shell_class.new(h,@shell_opt)
|
33
41
|
end
|
34
42
|
start_threads
|
43
|
+
timer.finish
|
35
44
|
end
|
36
45
|
|
37
46
|
def finish
|
@@ -63,14 +72,24 @@ module Pwrake
|
|
63
72
|
@task_queue.reserve(last) if last
|
64
73
|
hint = (conn) ? conn.host : nil
|
65
74
|
standard_exception_handling do
|
66
|
-
while
|
67
|
-
|
75
|
+
while true
|
76
|
+
time_start = Time.now
|
77
|
+
t = @task_queue.deq(hint)
|
78
|
+
break if !t
|
79
|
+
time_deq = Time.now - time_start
|
80
|
+
Log.debug "--- Master#thread_loop deq t=#{t.inspect} time=#{time_deq}sec"
|
68
81
|
t.pw_invoke
|
69
82
|
return if t == last
|
70
83
|
end
|
71
84
|
end
|
72
85
|
end
|
73
86
|
|
87
|
+
def task_id_counter
|
88
|
+
@lock.synchronize do
|
89
|
+
@current_task_id += 1
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
74
93
|
# Provide standard execption handling for the given block.
|
75
94
|
def standard_exception_handling
|
76
95
|
begin
|
data/lib/pwrake/option.rb
CHANGED
@@ -6,57 +6,88 @@ module Pwrake
|
|
6
6
|
|
7
7
|
DEFAULT_CONFFILES = ["pwrake_conf.yaml","PwrakeConf.yaml"]
|
8
8
|
|
9
|
+
def format_time_pid(v)
|
10
|
+
START_TIME.strftime(v).sub("%$","%05d"%Process.pid)
|
11
|
+
end
|
12
|
+
|
13
|
+
def parse_opt(s)
|
14
|
+
case s
|
15
|
+
when /false|nil|off/i
|
16
|
+
false
|
17
|
+
when /true|on/i
|
18
|
+
true
|
19
|
+
else
|
20
|
+
s
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
9
24
|
def option_data
|
10
25
|
[
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
26
|
+
'DRYRUN',
|
27
|
+
'IGNORE_SYSTEM',
|
28
|
+
'IGNORE_DEPRECATE',
|
29
|
+
'LOAD_SYSTEM',
|
30
|
+
'NOSEARCH',
|
31
|
+
'RAKELIB',
|
32
|
+
'SHOW_PREREQS',
|
33
|
+
'SILENT',
|
34
|
+
'TRACE',
|
35
|
+
'TRACE_RULES',
|
36
|
+
|
37
|
+
'FILESYSTEM',
|
38
|
+
'SSH_OPTION',
|
39
|
+
'PASS_ENV',
|
40
|
+
'GNU_TIME',
|
41
|
+
'DEBUG',
|
42
|
+
'PLOT_PARALLELISM',
|
43
|
+
'HALT_QUEUE_WHILE_SEARCH',
|
44
|
+
'SHOW_CONF',
|
45
|
+
'FAILED_TARGET', # rename(default), delete, leave
|
46
|
+
'QUEUE_PRIORITY', # DFS(default), FIFO,
|
47
|
+
'STEAL_WAIT',
|
48
|
+
'STEAL_WAIT_MAX',
|
49
|
+
|
50
|
+
['HOSTFILE','HOSTS'],
|
51
|
+
['LOGFILE','LOG',
|
52
|
+
proc{|v|
|
53
|
+
if v
|
54
|
+
# turn trace option on
|
55
|
+
Rake.application.options.trace = true
|
56
|
+
if v == "" || !v.kind_of?(String)
|
57
|
+
v = "Pwrake%Y%m%d-%H%M%S_%$.log"
|
58
|
+
end
|
59
|
+
format_time_pid(v)
|
37
60
|
end
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
v
|
61
|
+
}],
|
62
|
+
['TASKLOG',
|
63
|
+
proc{|v|
|
64
|
+
if v
|
65
|
+
if v == "" || !v.kind_of?(String)
|
66
|
+
v = "Pwrake%Y%m%d-%H%M%S_%$.task"
|
67
|
+
end
|
68
|
+
format_time_pid(v)
|
46
69
|
end
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
}]
|
70
|
+
}],
|
71
|
+
['PROFILE',
|
72
|
+
proc{|v|
|
73
|
+
if v
|
74
|
+
if v == "" || !v.kind_of?(String)
|
75
|
+
v = "Pwrake%Y%m%d-%H%M%S_%$.csv"
|
76
|
+
end
|
77
|
+
format_time_pid(v)
|
78
|
+
end
|
79
|
+
}],
|
80
|
+
['NUM_THREADS', proc{|v| v && v.to_i}],
|
81
|
+
['DISABLE_AFFINITY', proc{|v| v || ENV['AFFINITY']=='off'}],
|
82
|
+
['DISABLE_STEAL', proc{|v| v || ENV['STEAL']=='off'}],
|
83
|
+
['GFARM_BASEDIR', proc{|v| v || '/tmp'}],
|
84
|
+
['GFARM_PREFIX', proc{|v| v || "pwrake_#{ENV['USER']}"}],
|
85
|
+
['GFARM_SUBDIR', proc{|v| v || '/'}],
|
86
|
+
#['MASTER_HOSTNAME', proc{|v| v || `hostname -f`.chomp}],
|
87
|
+
['WORK_DIR',proc{|v|
|
88
|
+
v ||= '$HOME/%CWD_RELATIVE_TO_HOME'
|
89
|
+
v.sub('%CWD_RELATIVE_TO_HOME',cwd_relative_to_home)
|
90
|
+
}]
|
60
91
|
]
|
61
92
|
end
|
62
93
|
|
@@ -68,6 +99,10 @@ module Pwrake
|
|
68
99
|
init_options
|
69
100
|
init_pass_env
|
70
101
|
init_logger
|
102
|
+
Log.info "Options:"
|
103
|
+
@opts.each do |k,v|
|
104
|
+
Log.info " #{k} = #{v.inspect}"
|
105
|
+
end
|
71
106
|
if @opts['SHOW_CONF']
|
72
107
|
require "yaml"
|
73
108
|
YAML.dump(@opts,$stdout)
|
@@ -81,7 +116,7 @@ module Pwrake
|
|
81
116
|
attr_reader :logfile
|
82
117
|
attr_reader :queue_class
|
83
118
|
attr_reader :shell_class
|
84
|
-
|
119
|
+
attr_reader :task_logger
|
85
120
|
|
86
121
|
def pwrake_options
|
87
122
|
@opts
|
@@ -102,7 +137,7 @@ module Pwrake
|
|
102
137
|
if @pwrake_conf.nil?
|
103
138
|
@yaml = {}
|
104
139
|
else
|
105
|
-
Log.debug "@pwrake_conf=#{@pwrake_conf}"
|
140
|
+
Log.debug "--- @pwrake_conf=#{@pwrake_conf}"
|
106
141
|
require "yaml"
|
107
142
|
@yaml = open(@pwrake_conf){|f| YAML.load(f) }
|
108
143
|
end
|
@@ -160,11 +195,11 @@ module Pwrake
|
|
160
195
|
# command_option > ENV > pwrake_conf > DEFAULT_OPTIONS
|
161
196
|
def search_opts(keys)
|
162
197
|
val = Rake.application.options.send(keys[0].downcase.to_sym)
|
163
|
-
return val if !val.nil?
|
198
|
+
return parse_opt(val) if !val.nil?
|
164
199
|
#
|
165
200
|
keys.each do |k|
|
166
201
|
val = ENV[k.upcase]
|
167
|
-
return val if !val.nil?
|
202
|
+
return parse_opt(val) if !val.nil?
|
168
203
|
end
|
169
204
|
#
|
170
205
|
return nil if !@yaml
|
@@ -242,6 +277,15 @@ module Pwrake
|
|
242
277
|
else
|
243
278
|
Log.open($stdout)
|
244
279
|
end
|
280
|
+
|
281
|
+
if @tasklog
|
282
|
+
@task_logger = File.open(@tasklog,'w')
|
283
|
+
h = %w[
|
284
|
+
task_id task_name start_time end_time elap_time preq preq_host
|
285
|
+
exec_host shell_id has_action executed file_size file_mtime file_host
|
286
|
+
].join(',')+"\n"
|
287
|
+
@task_logger.print h
|
288
|
+
end
|
245
289
|
end
|
246
290
|
|
247
291
|
# ----- setup -----
|
@@ -266,7 +310,7 @@ module Pwrake
|
|
266
310
|
begin
|
267
311
|
host = Socket.gethostbyname(host)[0]
|
268
312
|
rescue
|
269
|
-
Log.info "FQDN not resoved : #{host}"
|
313
|
+
Log.info "-- FQDN not resoved : #{host}"
|
270
314
|
end
|
271
315
|
ncore = (ncore || 1).to_i
|
272
316
|
group = (group || 0).to_i
|
@@ -290,12 +334,13 @@ module Pwrake
|
|
290
334
|
@num_threads = 1 if !@num_threads
|
291
335
|
@core_list = ['localhost'] * @num_threads
|
292
336
|
end
|
337
|
+
Log.info "num_cores=#{@core_list.size}"
|
293
338
|
end
|
294
339
|
|
295
340
|
|
296
341
|
def set_filesystem
|
297
342
|
if fn = @opts["PROFILE"]
|
298
|
-
Shell.profiler.open(fn,@opts['GNU_TIME'])
|
343
|
+
Shell.profiler.open(fn,@opts['GNU_TIME'],@opts['PLOT_PARALLELISM'])
|
299
344
|
end
|
300
345
|
|
301
346
|
@shell_opt = {
|
@@ -320,13 +365,17 @@ module Pwrake
|
|
320
365
|
@shell_class = GfarmShell
|
321
366
|
@shell_opt.merge!({
|
322
367
|
:work_dir => Dir.pwd,
|
323
|
-
:disable_steal => @opts['DISABLE_STEAL'],
|
324
368
|
:single_mp => @opts['GFARM_SINGLE_MP'],
|
325
369
|
:basedir => @opts['GFARM_BASEDIR'],
|
326
370
|
:prefix => @opts['GFARM_PREFIX']
|
327
371
|
})
|
328
|
-
|
329
|
-
|
372
|
+
if @opts['DISABLE_AFFINITY']
|
373
|
+
@queue_class = TaskQueue
|
374
|
+
else
|
375
|
+
@queue_class = LocalityAwareQueue
|
376
|
+
end
|
377
|
+
@postprocess = GfarmPostprocess.new
|
378
|
+
Log.debug "--- @queue_class=#{@queue_class}"
|
330
379
|
else
|
331
380
|
@filesystem = 'nfs'
|
332
381
|
@shell_class = Shell
|
@@ -360,6 +409,7 @@ module Pwrake
|
|
360
409
|
# ----- finish -----
|
361
410
|
|
362
411
|
def finish_option
|
412
|
+
@task_logger.close if @task_logger
|
363
413
|
Log.close
|
364
414
|
end
|
365
415
|
|