pwrake 0.9.5 → 0.9.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -7,6 +7,70 @@ module Pwrake
7
7
  end
8
8
 
9
9
 
10
+ class LocalityConditionVariable < ConditionVariable
11
+ def signal(hints=nil)
12
+ if hints.nil?
13
+ super()
14
+ elsif Array===hints
15
+ thread = nil
16
+ @waiters_mutex.synchronize do
17
+ @waiters.each do |t,v|
18
+ if hints.include?(t[:hint])
19
+ thread = t
20
+ break
21
+ end
22
+ end
23
+ if thread
24
+ @waiters.delete(thread)
25
+ else
26
+ thread,_ = @waiters.shift
27
+ end
28
+ end
29
+ Log.debug "--- LCV#signal: hints=#{hints.inspect} thread_to_run=#{thread.inspect} @waiters.size=#{@waiters.size}"
30
+ begin
31
+ thread.run if thread
32
+ rescue ThreadError
33
+ retry # t was already dead?
34
+ end
35
+ else
36
+ raise ArgumentError,"argument must be an Array"
37
+ end
38
+ self
39
+ end
40
+
41
+ def broadcast(hints=nil)
42
+ if hints.nil?
43
+ super()
44
+ elsif Array===hints
45
+ threads = []
46
+ @waiters_mutex.synchronize do
47
+ hints.each do |h|
48
+ @waiters.each do |t,v|
49
+ if t[:hint] == h
50
+ threads << t
51
+ break
52
+ end
53
+ end
54
+ end
55
+ threads.each do |t|
56
+ @waiters.delete(t)
57
+ end
58
+ end
59
+ Log.debug "--- LCV#broadcast: hints=#{hints.inspect} threads_to_run=#{threads.inspect} @waiters.size=#{@waiters.size}"
60
+ threads.each do |t|
61
+ begin
62
+ t.run
63
+ rescue ThreadError
64
+ end
65
+ end
66
+ else
67
+ raise ArgumentError,"argument must be an Array"
68
+ end
69
+ self
70
+ end
71
+ end
72
+
73
+
10
74
  class LocalityAwareQueue < TaskQueue
11
75
 
12
76
  class Throughput
@@ -67,144 +131,90 @@ module Pwrake
67
131
  end # class Throughput
68
132
 
69
133
 
70
- def initialize(hosts,opt={})
71
- super(opt)
134
+ def init_queue(hosts)
135
+ @cv = LocalityConditionVariable.new
72
136
  @hosts = hosts
73
137
  @throughput = Throughput.new
74
138
  @size = 0
75
- @q = nil
76
- @q1 = []
77
- @q2 = {}
78
- @hosts.each{|h| @q2[h]=[]}
79
- @q2[nil] = []
80
- @enable_steal = !opt['disable_steal']
81
- @time_prev = Time.now
82
-
83
- @thread = Thread.new{thread_loop}
139
+ @q = {}
140
+ @hosts.each{|h| @q[h]=@array_class.new}
141
+ @q_remote = @array_class.new
142
+ @q_later = Array.new
143
+ @enable_steal = !Pwrake.application.pwrake_options['DISABLE_STEAL']
144
+ @steal_wait = (Pwrake.application.pwrake_options['STEAL_WAIT'] || 0).to_i
145
+ @steal_wait_max = (Pwrake.application.pwrake_options['STEAL_WAIT_MAX'] || 10).to_i
146
+ Log.info("-- @steal_wait=#{@steal_wait} @steal_wait_max=#{@steal_wait_max}")
84
147
  end
85
148
 
86
149
  attr_reader :size
87
150
 
88
151
 
89
- def thread_loop
90
- while !@finished
91
- bulk_mvq
92
- sleep # time_out
93
- end
94
- end
95
-
96
-
97
- def time_out
98
- if @size == 0
99
- if @q1.size == 1
100
- 0.25
101
- else
102
- 0.5
103
- end
152
+ def enq_impl(t)
153
+ hints = t.suggest_location
154
+ if hints.nil? || hints.empty?
155
+ @q_later.push(t)
104
156
  else
105
- if @q1.size > @size
106
- 0.5
107
- elsif @q1.size > @size/2
108
- 1.0
109
- else
110
- 3.0
111
- end
112
- end
113
- end
114
-
115
-
116
- def bulk_mvq
117
- a = nil
118
- @mutex.synchronize do
119
- return if @q1.empty?
120
-
121
- time_now = Time.now
122
- interval = time_now-@time_prev
123
- return if time_now-@time_prev < time_out
124
-
125
- log_bulk_mvq
126
- @time_prev = time_now
127
-
128
- a = @q1
129
- @q1 = []
130
- end
131
-
132
- where(a)
133
-
134
- @mutex.synchronize do
135
- while t = a.shift
136
- mvq(t)
137
- end
138
- @cv.broadcast
139
- end
140
- end
141
-
142
- def log_bulk_mvq
143
- msg = @q1[0..2].map{|t| t.name}.inspect
144
- msg.sub!(/]$/,",...") if @q1.size > 3
145
- Log.info "-- bulk_mvq interval=%.6fs @size=#{@size} @q1.size=#{@q1.size} @q1=#{msg}"%(Time.now-@time_prev)
146
- end
147
-
148
-
149
- def where(task_list)
150
- # implemented in child class
151
- end
152
-
153
-
154
- def mvq(t)
155
- stored = false
156
- if t.respond_to? :location
157
- t.location.each do |h|
158
- if q = @q2[h]
157
+ stored = false
158
+ hints.each do |h|
159
+ if q = @q[h]
159
160
  t.assigned.push(h)
160
161
  q.push(t)
161
162
  stored = true
162
163
  end
163
164
  end
164
- end
165
- if !stored
166
- @q2[@hosts[rand(@hosts.size)]].push(t)
167
- # @q2[nil].push(t)
165
+ if !stored
166
+ @q_remote.push(t)
167
+ end
168
168
  end
169
169
  @size += 1
170
170
  end
171
171
 
172
172
 
173
- def enq_impl(item,hint=nil)
174
- #Log.debug "--- #{self.class}#enq_impl #{item.inspect}"
175
- @q1.push(item)
176
- end
177
-
178
- def enq_finish
179
- @thread.run if @thread.alive?
180
- end
181
-
182
173
  def deq_impl(host,n)
183
- @thread.run if @thread.alive?
184
-
185
174
  if t = deq_locate(host)
186
175
  Log.info "-- deq_locate n=#{n} task=#{t.name} host=#{host}"
176
+ Log.debug "--- deq_impl\n#{inspect_q}"
177
+ return t
178
+ end
179
+
180
+ if !@q_remote.empty?
181
+ t = @q_remote.shift
182
+ Log.info "-- deq_remote n=#{n} task=#{t.name} host=#{host}"
183
+ Log.debug "--- deq_impl\n#{inspect_q}"
187
184
  return t
188
185
  end
189
186
 
190
- if @enable_steal && n > 1
187
+ if !@q_later.empty?
188
+ t = @q_later.shift
189
+ Log.info "-- deq_later n=#{n} task=#{t.name} host=#{host}"
190
+ Log.debug "--- deq_impl\n#{inspect_q}"
191
+ return t
192
+ end
193
+
194
+ if @enable_steal && n > 0
191
195
  if t = deq_steal(host)
192
196
  Log.info "-- deq_steal n=#{n} task=#{t.name} host=#{host}"
197
+ Log.debug "--- deq_impl\n#{inspect_q}"
193
198
  return t
194
199
  end
195
200
  end
196
201
 
197
- m = 0.05*(2**([n,5].min))
202
+ m = [@steal_wait*(2**n), @steal_wait_max].min
198
203
  @cv.wait(@mutex,m)
199
204
  nil
200
205
  end
201
206
 
202
207
 
203
208
  def deq_locate(host)
204
- q = @q2[host]
209
+ q = @q[host]
205
210
  if q && !q.empty?
206
211
  t = q.shift
207
- t.assigned.each{|x| @q2[x].delete_if{|x| t.equal? x}}
212
+ t.assigned.each do |h|
213
+ a = @q[h]
214
+ if i = a.index(t)
215
+ a.delete_at(i)
216
+ end
217
+ end
208
218
  @size -= 1
209
219
  return t
210
220
  else
@@ -216,38 +226,58 @@ module Pwrake
216
226
  # select a task based on many and close
217
227
  max_host = nil
218
228
  max_num = 0
219
- @q2.each do |h,a|
229
+ @q.each do |h,a|
220
230
  if !a.empty?
221
- d = @throughput.interhost(host,h) * a.size
231
+ d = a.size
222
232
  if d > max_num
223
233
  max_host = h
224
234
  max_num = d
225
235
  end
226
236
  end
227
237
  end
228
- if max_host
229
- deq_locate(max_host)
230
- else
231
- deq_locate(nil)
232
- end
238
+ Log.info "-- deq_steal max_host=#{max_host} max_num=#{max_num}"
239
+ deq_locate(max_host)
240
+ end
241
+
242
+ def inspect_q
243
+ s = ""
244
+ b = proc{|h,q|
245
+ s += " #{h}: size=#{q.size} "
246
+ case q.size
247
+ when 0
248
+ s += "[]\n"
249
+ when 1
250
+ s += "[#{q[0].name}]\n"
251
+ else
252
+ s += "[#{q[0].name},..]\n"
253
+ end
254
+ }
255
+ @q.each(&b)
256
+ b.call("remote",@q_remote)
257
+ b.call("later",@q_later)
258
+ s
233
259
  end
234
260
 
235
261
  def size
236
- @q1.size + @size
262
+ @size
237
263
  end
238
264
 
239
265
  def clear
240
- @q1.clear
241
- @hosts.each{|h| @q2[h].clear}
266
+ @q.each{|h,q| q.clear}
267
+ @q_remote.clear
268
+ @q_later.clear
269
+ @reserved_q.clear
242
270
  end
243
271
 
244
272
  def empty?
245
- @q1.empty? && @hosts.all?{|h| @q2[h].empty?}
273
+ @q.all?{|h,q| q.empty?} &&
274
+ @q_remote.empty? &&
275
+ @q_later.empty? &&
276
+ @reserved_q.empty?
246
277
  end
247
278
 
248
279
  def finish
249
280
  super
250
- @thread.run if @thread.alive?
251
281
  end
252
282
 
253
283
  end
data/lib/pwrake/logger.rb CHANGED
@@ -142,12 +142,18 @@ module Pwrake
142
142
  Timer.new(prefix,*args)
143
143
  end
144
144
 
145
- def output_message(message)
145
+ def stderr_puts(message)
146
146
  LOCK.synchronize do
147
147
  $stderr.write(message+"\n")
148
148
  end
149
149
  end
150
150
 
151
+ def stdout_puts(message)
152
+ LOCK.synchronize do
153
+ $stdout.write(message+"\n")
154
+ end
155
+ end
156
+
151
157
  end
152
158
 
153
159
  end # module Pwrake
data/lib/pwrake/master.rb CHANGED
@@ -6,6 +6,7 @@ module Pwrake
6
6
 
7
7
  def current_shell=(a)
8
8
  Thread.current[:shell] = a
9
+ Thread.current[:hint] = a.host
9
10
  end
10
11
 
11
12
  module_function :current_shell, :current_shell=
@@ -15,16 +16,23 @@ module Pwrake
15
16
  include Pwrake::Option
16
17
 
17
18
  attr_reader :task_queue
19
+ attr_reader :finish_queue
18
20
  attr_reader :shell_set
21
+ attr_reader :filesystem
22
+ attr_reader :postprocess
19
23
 
20
24
  def initialize
21
25
  init_option # Pwrake::Option
22
26
  setup_option # Pwrake::Option
23
27
  @started = false
28
+ @lock = Mutex.new
29
+ @current_task_id = -1
24
30
  end
25
31
 
26
32
  def start
27
33
  return if @task_queue
34
+ timer = Timer.new("start_worker")
35
+ @finish_queue = Queue.new
28
36
  @task_queue = @queue_class.new(@core_list)
29
37
  @task_queue.enable_steal = !Rake.application.options.disable_steal
30
38
  @shell_set = []
@@ -32,6 +40,7 @@ module Pwrake
32
40
  @shell_set << @shell_class.new(h,@shell_opt)
33
41
  end
34
42
  start_threads
43
+ timer.finish
35
44
  end
36
45
 
37
46
  def finish
@@ -63,14 +72,24 @@ module Pwrake
63
72
  @task_queue.reserve(last) if last
64
73
  hint = (conn) ? conn.host : nil
65
74
  standard_exception_handling do
66
- while t = @task_queue.deq(hint)
67
- Log.debug "-- Master#thread_loop deq t=#{t.inspect}"
75
+ while true
76
+ time_start = Time.now
77
+ t = @task_queue.deq(hint)
78
+ break if !t
79
+ time_deq = Time.now - time_start
80
+ Log.debug "--- Master#thread_loop deq t=#{t.inspect} time=#{time_deq}sec"
68
81
  t.pw_invoke
69
82
  return if t == last
70
83
  end
71
84
  end
72
85
  end
73
86
 
87
+ def task_id_counter
88
+ @lock.synchronize do
89
+ @current_task_id += 1
90
+ end
91
+ end
92
+
74
93
  # Provide standard execption handling for the given block.
75
94
  def standard_exception_handling
76
95
  begin
data/lib/pwrake/option.rb CHANGED
@@ -6,57 +6,88 @@ module Pwrake
6
6
 
7
7
  DEFAULT_CONFFILES = ["pwrake_conf.yaml","PwrakeConf.yaml"]
8
8
 
9
+ def format_time_pid(v)
10
+ START_TIME.strftime(v).sub("%$","%05d"%Process.pid)
11
+ end
12
+
13
+ def parse_opt(s)
14
+ case s
15
+ when /false|nil|off/i
16
+ false
17
+ when /true|on/i
18
+ true
19
+ else
20
+ s
21
+ end
22
+ end
23
+
9
24
  def option_data
10
25
  [
11
- 'DRYRUN',
12
- 'IGNORE_SYSTEM',
13
- 'IGNORE_DEPRECATE',
14
- 'LOAD_SYSTEM',
15
- 'NOSEARCH',
16
- 'RAKELIB',
17
- 'SHOW_PREREQS',
18
- 'SILENT',
19
- 'TRACE',
20
- 'TRACE_RULES',
21
-
22
- 'FILESYSTEM',
23
- 'SSH_OPTION',
24
- 'PASS_ENV',
25
- 'GNU_TIME',
26
- 'HALT_QUEUE_WHILE_SEARCH',
27
- 'SHOW_CONF',
28
-
29
- ['HOSTFILE','HOSTS'],
30
- ['LOGFILE','LOG',
31
- proc{|v|
32
- if v
33
- # turn trace option on
34
- Rake.application.options.trace = true
35
- if v == "" || !v.kind_of?(String)
36
- v = "Pwrake%Y%m%d-%H%M%S_%$.log"
26
+ 'DRYRUN',
27
+ 'IGNORE_SYSTEM',
28
+ 'IGNORE_DEPRECATE',
29
+ 'LOAD_SYSTEM',
30
+ 'NOSEARCH',
31
+ 'RAKELIB',
32
+ 'SHOW_PREREQS',
33
+ 'SILENT',
34
+ 'TRACE',
35
+ 'TRACE_RULES',
36
+
37
+ 'FILESYSTEM',
38
+ 'SSH_OPTION',
39
+ 'PASS_ENV',
40
+ 'GNU_TIME',
41
+ 'DEBUG',
42
+ 'PLOT_PARALLELISM',
43
+ 'HALT_QUEUE_WHILE_SEARCH',
44
+ 'SHOW_CONF',
45
+ 'FAILED_TARGET', # rename(default), delete, leave
46
+ 'QUEUE_PRIORITY', # DFS(default), FIFO,
47
+ 'STEAL_WAIT',
48
+ 'STEAL_WAIT_MAX',
49
+
50
+ ['HOSTFILE','HOSTS'],
51
+ ['LOGFILE','LOG',
52
+ proc{|v|
53
+ if v
54
+ # turn trace option on
55
+ Rake.application.options.trace = true
56
+ if v == "" || !v.kind_of?(String)
57
+ v = "Pwrake%Y%m%d-%H%M%S_%$.log"
58
+ end
59
+ format_time_pid(v)
37
60
  end
38
- START_TIME.strftime(v).sub("%$",Process.pid.to_s)
39
- end
40
- }],
41
- ['PROFILE',
42
- proc{|v|
43
- if v
44
- if v == "" || !v.kind_of?(String)
45
- v = "Pwrake%Y%m%d-%H%M%S_%$.csv"
61
+ }],
62
+ ['TASKLOG',
63
+ proc{|v|
64
+ if v
65
+ if v == "" || !v.kind_of?(String)
66
+ v = "Pwrake%Y%m%d-%H%M%S_%$.task"
67
+ end
68
+ format_time_pid(v)
46
69
  end
47
- START_TIME.strftime(v).sub("%$",Process.pid.to_s)
48
- end
49
- }],
50
- ['NUM_THREADS', proc{|v| v && v.to_i}],
51
- ['DISABLE_AFFINITY', proc{|v| v || ENV['AFFINITY']=='off'}],
52
- ['GFARM_BASEDIR', proc{|v| v || '/tmp'}],
53
- ['GFARM_PREFIX', proc{|v| v || "pwrake_#{ENV['USER']}"}],
54
- ['GFARM_SUBDIR', proc{|v| v || '/'}],
55
- #['MASTER_HOSTNAME', proc{|v| v || `hostname -f`.chomp}],
56
- ['WORK_DIR',proc{|v|
57
- v ||= '$HOME/%CWD_RELATIVE_TO_HOME'
58
- v.sub('%CWD_RELATIVE_TO_HOME',cwd_relative_to_home)
59
- }]
70
+ }],
71
+ ['PROFILE',
72
+ proc{|v|
73
+ if v
74
+ if v == "" || !v.kind_of?(String)
75
+ v = "Pwrake%Y%m%d-%H%M%S_%$.csv"
76
+ end
77
+ format_time_pid(v)
78
+ end
79
+ }],
80
+ ['NUM_THREADS', proc{|v| v && v.to_i}],
81
+ ['DISABLE_AFFINITY', proc{|v| v || ENV['AFFINITY']=='off'}],
82
+ ['DISABLE_STEAL', proc{|v| v || ENV['STEAL']=='off'}],
83
+ ['GFARM_BASEDIR', proc{|v| v || '/tmp'}],
84
+ ['GFARM_PREFIX', proc{|v| v || "pwrake_#{ENV['USER']}"}],
85
+ ['GFARM_SUBDIR', proc{|v| v || '/'}],
86
+ #['MASTER_HOSTNAME', proc{|v| v || `hostname -f`.chomp}],
87
+ ['WORK_DIR',proc{|v|
88
+ v ||= '$HOME/%CWD_RELATIVE_TO_HOME'
89
+ v.sub('%CWD_RELATIVE_TO_HOME',cwd_relative_to_home)
90
+ }]
60
91
  ]
61
92
  end
62
93
 
@@ -68,6 +99,10 @@ module Pwrake
68
99
  init_options
69
100
  init_pass_env
70
101
  init_logger
102
+ Log.info "Options:"
103
+ @opts.each do |k,v|
104
+ Log.info " #{k} = #{v.inspect}"
105
+ end
71
106
  if @opts['SHOW_CONF']
72
107
  require "yaml"
73
108
  YAML.dump(@opts,$stdout)
@@ -81,7 +116,7 @@ module Pwrake
81
116
  attr_reader :logfile
82
117
  attr_reader :queue_class
83
118
  attr_reader :shell_class
84
-
119
+ attr_reader :task_logger
85
120
 
86
121
  def pwrake_options
87
122
  @opts
@@ -102,7 +137,7 @@ module Pwrake
102
137
  if @pwrake_conf.nil?
103
138
  @yaml = {}
104
139
  else
105
- Log.debug "@pwrake_conf=#{@pwrake_conf}"
140
+ Log.debug "--- @pwrake_conf=#{@pwrake_conf}"
106
141
  require "yaml"
107
142
  @yaml = open(@pwrake_conf){|f| YAML.load(f) }
108
143
  end
@@ -160,11 +195,11 @@ module Pwrake
160
195
  # command_option > ENV > pwrake_conf > DEFAULT_OPTIONS
161
196
  def search_opts(keys)
162
197
  val = Rake.application.options.send(keys[0].downcase.to_sym)
163
- return val if !val.nil?
198
+ return parse_opt(val) if !val.nil?
164
199
  #
165
200
  keys.each do |k|
166
201
  val = ENV[k.upcase]
167
- return val if !val.nil?
202
+ return parse_opt(val) if !val.nil?
168
203
  end
169
204
  #
170
205
  return nil if !@yaml
@@ -242,6 +277,15 @@ module Pwrake
242
277
  else
243
278
  Log.open($stdout)
244
279
  end
280
+
281
+ if @tasklog
282
+ @task_logger = File.open(@tasklog,'w')
283
+ h = %w[
284
+ task_id task_name start_time end_time elap_time preq preq_host
285
+ exec_host shell_id has_action executed file_size file_mtime file_host
286
+ ].join(',')+"\n"
287
+ @task_logger.print h
288
+ end
245
289
  end
246
290
 
247
291
  # ----- setup -----
@@ -266,7 +310,7 @@ module Pwrake
266
310
  begin
267
311
  host = Socket.gethostbyname(host)[0]
268
312
  rescue
269
- Log.info "FQDN not resoved : #{host}"
313
+ Log.info "-- FQDN not resoved : #{host}"
270
314
  end
271
315
  ncore = (ncore || 1).to_i
272
316
  group = (group || 0).to_i
@@ -290,12 +334,13 @@ module Pwrake
290
334
  @num_threads = 1 if !@num_threads
291
335
  @core_list = ['localhost'] * @num_threads
292
336
  end
337
+ Log.info "num_cores=#{@core_list.size}"
293
338
  end
294
339
 
295
340
 
296
341
  def set_filesystem
297
342
  if fn = @opts["PROFILE"]
298
- Shell.profiler.open(fn,@opts['GNU_TIME'])
343
+ Shell.profiler.open(fn,@opts['GNU_TIME'],@opts['PLOT_PARALLELISM'])
299
344
  end
300
345
 
301
346
  @shell_opt = {
@@ -320,13 +365,17 @@ module Pwrake
320
365
  @shell_class = GfarmShell
321
366
  @shell_opt.merge!({
322
367
  :work_dir => Dir.pwd,
323
- :disable_steal => @opts['DISABLE_STEAL'],
324
368
  :single_mp => @opts['GFARM_SINGLE_MP'],
325
369
  :basedir => @opts['GFARM_BASEDIR'],
326
370
  :prefix => @opts['GFARM_PREFIX']
327
371
  })
328
- @queue_class = GfarmQueue
329
- # @queue_class = TaskQueue
372
+ if @opts['DISABLE_AFFINITY']
373
+ @queue_class = TaskQueue
374
+ else
375
+ @queue_class = LocalityAwareQueue
376
+ end
377
+ @postprocess = GfarmPostprocess.new
378
+ Log.debug "--- @queue_class=#{@queue_class}"
330
379
  else
331
380
  @filesystem = 'nfs'
332
381
  @shell_class = Shell
@@ -360,6 +409,7 @@ module Pwrake
360
409
  # ----- finish -----
361
410
 
362
411
  def finish_option
412
+ @task_logger.close if @task_logger
363
413
  Log.close
364
414
  end
365
415