pwrake 0.9.9.2 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +2 -0
  3. data/CHANGES_V2.md +90 -0
  4. data/{LICENSE.txt → MIT-LICENSE} +2 -3
  5. data/README +12 -0
  6. data/README.md +75 -52
  7. data/bin/gfwhere-pipe +23 -12
  8. data/bin/pwrake +22 -29
  9. data/bin/pwrake_branch +24 -0
  10. data/lib/pwrake/branch.rb +22 -0
  11. data/lib/pwrake/branch/branch.rb +213 -0
  12. data/lib/pwrake/branch/branch_application.rb +53 -0
  13. data/lib/pwrake/branch/fiber_queue.rb +36 -0
  14. data/lib/pwrake/branch/file_utils.rb +101 -0
  15. data/lib/pwrake/branch/shell.rb +231 -0
  16. data/lib/pwrake/{profiler.rb → branch/shell_profiler.rb} +28 -27
  17. data/lib/pwrake/branch/worker_communicator.rb +104 -0
  18. data/lib/pwrake/{gfarm_feature.rb → gfarm/gfarm_path.rb} +2 -100
  19. data/lib/pwrake/gfarm/gfarm_postprocess.rb +53 -0
  20. data/lib/pwrake/iomux/channel.rb +70 -0
  21. data/lib/pwrake/iomux/handler.rb +124 -0
  22. data/lib/pwrake/iomux/handler_set.rb +35 -0
  23. data/lib/pwrake/iomux/runner.rb +62 -0
  24. data/lib/pwrake/logger.rb +3 -150
  25. data/lib/pwrake/master.rb +30 -137
  26. data/lib/pwrake/master/fiber_pool.rb +69 -0
  27. data/lib/pwrake/master/idle_cores.rb +30 -0
  28. data/lib/pwrake/master/master.rb +345 -0
  29. data/lib/pwrake/master/master_application.rb +150 -0
  30. data/lib/pwrake/master/postprocess.rb +16 -0
  31. data/lib/pwrake/{graphviz.rb → misc/graphviz.rb} +0 -0
  32. data/lib/pwrake/{mcgp.rb → misc/mcgp.rb} +63 -42
  33. data/lib/pwrake/option/host_map.rb +158 -0
  34. data/lib/pwrake/option/option.rb +357 -0
  35. data/lib/pwrake/option/option_filesystem.rb +112 -0
  36. data/lib/pwrake/queue/locality_aware_queue.rb +158 -0
  37. data/lib/pwrake/queue/no_action_queue.rb +67 -0
  38. data/lib/pwrake/queue/queue_array.rb +366 -0
  39. data/lib/pwrake/queue/task_queue.rb +164 -0
  40. data/lib/pwrake/report.rb +1 -0
  41. data/lib/pwrake/report/parallelism.rb +9 -3
  42. data/lib/pwrake/report/report.rb +50 -103
  43. data/lib/pwrake/report/task_stat.rb +83 -0
  44. data/lib/pwrake/task/task_algorithm.rb +107 -0
  45. data/lib/pwrake/task/task_manager.rb +32 -0
  46. data/lib/pwrake/task/task_property.rb +98 -0
  47. data/lib/pwrake/task/task_rank.rb +48 -0
  48. data/lib/pwrake/task/task_wrapper.rb +296 -0
  49. data/lib/pwrake/version.rb +1 -1
  50. data/lib/pwrake/worker/executor.rb +169 -0
  51. data/lib/pwrake/worker/gfarm_directory.rb +90 -0
  52. data/lib/pwrake/worker/invoker.rb +199 -0
  53. data/lib/pwrake/worker/load.rb +14 -0
  54. data/lib/pwrake/worker/log_executor.rb +73 -0
  55. data/lib/pwrake/worker/shared_directory.rb +74 -0
  56. data/lib/pwrake/worker/worker_main.rb +14 -0
  57. data/lib/pwrake/worker/writer.rb +59 -0
  58. data/setup.rb +1212 -1502
  59. data/spec/003/Rakefile +2 -2
  60. data/spec/008/Rakefile +2 -1
  61. data/spec/009/Rakefile +1 -1
  62. data/spec/009/pwrake_conf.yaml +1 -3
  63. data/spec/hosts +0 -2
  64. data/spec/pwrake_spec.rb +9 -8
  65. metadata +50 -21
  66. data/lib/pwrake.rb +0 -19
  67. data/lib/pwrake/application.rb +0 -232
  68. data/lib/pwrake/counter.rb +0 -54
  69. data/lib/pwrake/file_utils.rb +0 -98
  70. data/lib/pwrake/gfwhere_pool.rb +0 -109
  71. data/lib/pwrake/host_list.rb +0 -88
  72. data/lib/pwrake/locality_aware_queue.rb +0 -413
  73. data/lib/pwrake/option.rb +0 -400
  74. data/lib/pwrake/rake_modify.rb +0 -14
  75. data/lib/pwrake/shell.rb +0 -186
  76. data/lib/pwrake/task_algorithm.rb +0 -475
  77. data/lib/pwrake/task_queue.rb +0 -633
  78. data/lib/pwrake/timer.rb +0 -22
@@ -1,475 +0,0 @@
1
- module Pwrake
2
-
3
- InvocationChain = Rake::InvocationChain
4
- TaskArguments = Rake::TaskArguments
5
-
6
- class RankStat
7
- def initialize
8
- @lock = Mutex.new
9
- @stat = []
10
- end
11
-
12
- def add_sample(rank,elap)
13
- @lock.synchronize do
14
- stat = @stat[rank]
15
- if stat.nil?
16
- @stat[rank] = stat = [0,0.0]
17
- end
18
- stat[0] += 1
19
- stat[1] += elap
20
- Log.debug "--- add_sample rank=#{rank} stat=#{stat.inspect} weight=#{stat[0]/stat[1]}"
21
- end
22
- end
23
-
24
- def rank_weight
25
- @lock.synchronize do
26
- sum = 0.0
27
- count = 0
28
- weight = @stat.map do |stat|
29
- if stat
30
- w = stat[0]/stat[1]
31
- sum += w
32
- count += 1
33
- w
34
- else
35
- nil
36
- end
37
- end
38
- if count == 0
39
- avg = 1.0
40
- else
41
- avg = sum/count
42
- end
43
- [weight, avg]
44
- end
45
- end
46
- end
47
-
48
- RANK_STAT = RankStat.new
49
-
50
- module TaskAlgorithm
51
-
52
- def location
53
- @location ||= []
54
- end
55
-
56
- def location=(a)
57
- @location = a
58
- @group = []
59
- @location.each do |host|
60
- @group |= [Pwrake.application.host_list.host2group[host]]
61
- end
62
- end
63
-
64
- def group
65
- @group ||= []
66
- end
67
-
68
- def group_id
69
- @group_id
70
- end
71
-
72
- def group_id=(i)
73
- @group_id = i
74
- end
75
-
76
- def suggest_location=(a)
77
- @suggest_location = a
78
- end
79
-
80
- def task_id
81
- @task_id
82
- end
83
-
84
- def invoke_modify(*args)
85
- return if @already_invoked
86
-
87
- if Pwrake.application.pwrake_options['GRAPH_PARTITION']
88
- require 'pwrake/mcgp'
89
- MCGP.graph_partition
90
- end
91
-
92
- application.start_worker
93
-
94
- if false
95
- th = Thread.new(args){|a| pw_search_tasks(a) }
96
- else
97
- pw_search_tasks(args)
98
- th = nil
99
- end
100
- Log.info "-- ps:\n"+`ps xwv|egrep 'PID|ruby'`
101
-
102
- if conn = Pwrake.current_shell
103
- application.thread_loop(conn,self)
104
- else
105
- if fname = application.pwrake_options['GC_PROFILE']
106
- File.open(fname,"w") do |f|
107
- gc_count = 0
108
- while true
109
- t = application.finish_queue.deq
110
- if GC.count > gc_count
111
- f.write Log.fmt_time(Time.now)+" "
112
- f.write(GC::Profiler.result)
113
- GC::Profiler.clear
114
- gc_count = GC.count
115
- end
116
- break if t==self
117
- end
118
- end
119
- else
120
- while true
121
- t = application.finish_queue.deq
122
- break if t==self
123
- end
124
- end
125
- end
126
-
127
- th.join if th
128
- end
129
-
130
- def pw_search_tasks(args)
131
- task_args = TaskArguments.new(arg_names, args)
132
- timer = Timer.new("search_task")
133
- h = application.pwrake_options['HALT_QUEUE_WHILE_SEARCH']
134
- application.task_queue.synchronize(h) do
135
- search_with_call_chain(nil, task_args, InvocationChain::EMPTY)
136
- end
137
- timer.finish
138
- end
139
-
140
- def pw_invoke
141
- time_start = Time.now
142
- if shell = Pwrake.current_shell
143
- shell.current_task = self
144
- end
145
- @lock.synchronize do
146
- return if @already_invoked
147
- @already_invoked = true
148
- end
149
- pw_execute(@arg_data) if needed?
150
- Log.debug("pw_execute: #{Time.now-time_start} sec, name=#{name}")
151
- if kind_of?(Rake::FileTask)
152
- t = Time.now
153
- application.postprocess(self)
154
- Log.debug("postprocess: #{Time.now-t} sec, name=#{name}")
155
- if File.exist?(name)
156
- t = Time.now
157
- @file_stat = File::Stat.new(name)
158
- Log.debug("File::Stat: #{Time.now-t} sec, name=#{name}")
159
- end
160
- end
161
- log_task(time_start)
162
- t = Time.now
163
- application.finish_queue.enq(self)
164
- shell.current_task = nil if shell
165
- pw_enq_subsequents
166
- Log.debug "--- pw_invoke (#{name}) enq_subseq time=#{Time.now-t} sec"
167
- end
168
-
169
- def get_file_stat
170
- @file_stat ||= File::Stat.new(name)
171
- end
172
-
173
- def log_task(time_start)
174
- time_end = Time.now
175
-
176
- loc = suggest_location()
177
- shell = Pwrake.current_shell
178
-
179
- if loc && !loc.empty? && shell && !@actions.empty?
180
- Pwrake.application.count( loc, shell.host )
181
- end
182
- return if !application.task_logger
183
-
184
- elap = time_end - time_start
185
- if !@actions.empty? && kind_of?(Rake::FileTask)
186
- RANK_STAT.add_sample(rank,elap)
187
- end
188
-
189
- row = [ @task_id, name, time_start, time_end, elap, @prerequisites.join('|') ]
190
-
191
- if loc
192
- row << loc.join('|')
193
- else
194
- row << ''
195
- end
196
-
197
- if shell
198
- row.concat [shell.host, shell.id]
199
- else
200
- row.concat ['','']
201
- end
202
-
203
- row << ((@actions.empty?) ? 0 : 1)
204
- row << ((@executed) ? 1 : 0)
205
-
206
- if @file_stat
207
- row.concat [@file_stat.size, @file_stat.mtime, self.location.join('|')]
208
- else
209
- row.concat ['','','']
210
- end
211
-
212
- s = row.map do |x|
213
- if x.kind_of?(Time)
214
- Profiler.format_time(x)
215
- elsif x.kind_of?(String) && x!=''
216
- '"'+x+'"'
217
- else
218
- x.to_s
219
- end
220
- end.join(',')
221
-
222
- # task_id task_name start_time end_time elap_time preq preq_host
223
- # exec_host shell_id has_action executed file_size file_mtime file_host
224
- application.task_logger.print s+"\n"
225
- end
226
-
227
-
228
- # Execute the actions associated with this task.
229
- def pw_execute(args=nil)
230
- args ||= Rake::EMPTY_TASK_ARGS
231
- if application.options.dryrun
232
- Log.info "** Execute (dry run) #{name}"
233
- return
234
- end
235
- if application.options.trace
236
- Log.info "** Execute #{name}"
237
- end
238
- application.enhance_with_matching_rule(name) if @actions.empty?
239
- begin
240
- @actions.each do |act|
241
- case act.arity
242
- when 1
243
- act.call(self)
244
- else
245
- act.call(self, args)
246
- end
247
- end
248
- rescue Exception=>e
249
- if kind_of?(Rake::FileTask) && File.exist?(name)
250
- opt = application.pwrake_options['FAILED_TARGET']||"rename"
251
- case opt
252
- when /rename/i
253
- dst = name+"._fail_"
254
- ::FileUtils.mv(name,dst)
255
- msg = "Rename failed target file '#{name}' to '#{dst}'"
256
- Log.stderr_puts(msg)
257
- when /delete/i
258
- ::FileUtils.rm(name)
259
- msg = "Delete failed target file '#{name}'"
260
- Log.stderr_puts(msg)
261
- when /leave/i
262
- end
263
- end
264
- raise e
265
- end
266
- @executed = true if !@actions.empty?
267
- end
268
-
269
- def pw_enq_subsequents
270
- @lock.synchronize do
271
- t = Time.now
272
- h = application.pwrake_options['HALT_QUEUE_WHILE_SEARCH']
273
- application.task_queue.synchronize(h) do
274
- @subsequents.each do |t| # <<--- competition !!!
275
- if t && t.check_prereq_finished(self.name)
276
- application.task_queue.enq(t)
277
- end
278
- end
279
- end
280
- @already_finished = true # <<--- competition !!!
281
- Log.debug "--- pw_enq_subseq (#{name}) time=#{Time.now-t} sec"
282
- end
283
- end
284
-
285
- def check_prereq_finished(preq_name=nil)
286
- @unfinished_prereq.delete(preq_name)
287
- @unfinished_prereq.empty?
288
- end
289
-
290
-
291
- # Same as search, but explicitly pass a call chain to detect
292
- # circular dependencies.
293
- def search_with_call_chain(subseq, task_args, invocation_chain) # :nodoc:
294
- new_chain = InvocationChain.append(self, invocation_chain)
295
- @lock.synchronize do
296
- if application.options.trace
297
- Log.info "** Search #{name} #{format_search_flags}"
298
- end
299
-
300
- return true if @already_finished # <<--- competition !!!
301
- @subsequents ||= []
302
- @subsequents << subseq if subseq # <<--- competition !!!
303
-
304
- if ! @already_searched
305
- @already_searched = true
306
- @arg_data = task_args
307
- @lock_rank = Monitor.new
308
- if @prerequisites.empty?
309
- @unfinished_prereq = {}
310
- else
311
- search_prerequisites(task_args, new_chain)
312
- end
313
- @task_id = application.task_id_counter
314
- #check_and_enq
315
- if @unfinished_prereq.empty?
316
- application.task_queue.enq(self)
317
- end
318
- end
319
- return false
320
- end
321
- rescue Exception => ex
322
- add_chain_to(ex, new_chain)
323
- raise ex
324
- end
325
-
326
- # Search all the prerequisites of a task.
327
- def search_prerequisites(task_args, invocation_chain) # :nodoc:
328
- @unfinished_prereq = {}
329
- @prerequisites.each{|t| @unfinished_prereq[t]=true}
330
- prerequisite_tasks.each { |prereq|
331
- #prereq_args = task_args.new_scope(prereq.arg_names) # in vain
332
- if prereq.search_with_call_chain(self, task_args, invocation_chain)
333
- @unfinished_prereq.delete(prereq.name)
334
- end
335
- }
336
- end
337
-
338
- # Format the trace flags for display.
339
- def format_search_flags
340
- flags = []
341
- flags << "finished" if @already_finished
342
- flags << "first_time" unless @already_searched
343
- flags << "not_needed" unless needed?
344
- flags.empty? ? "" : "(" + flags.join(", ") + ")"
345
- end
346
- private :format_search_flags
347
-
348
- def rank
349
- @lock_rank.synchronize do
350
- if @rank.nil?
351
- if @subsequents.nil? || @subsequents.empty?
352
- @rank = 0
353
- else
354
- max_rank = 0
355
- @subsequents.each do |subsq|
356
- r = subsq.rank
357
- if max_rank < r
358
- max_rank = r
359
- end
360
- end
361
- if @actions.empty? || !kind_of?(Rake::FileTask)
362
- step = 0
363
- else
364
- step = 1
365
- end
366
- @rank = max_rank + step
367
- end
368
- Log.debug "--- Task[#{name}] rank=#{@rank.inspect}"
369
- end
370
- end
371
- @rank
372
- end
373
-
374
- def file_size
375
- @file_stat ? @file_stat.size : 0
376
- end
377
-
378
- def file_mtime
379
- @file_stat ? @file_stat.mtime : Time.at(0)
380
- end
381
-
382
- def input_file_size
383
- unless @input_file_size
384
- @input_file_size = 0
385
- @prerequisites.each do |preq|
386
- @input_file_size += application[preq].file_size
387
- end
388
- end
389
- @input_file_size
390
- end
391
-
392
- def has_input_file?
393
- kind_of?(Rake::FileTask) && !@prerequisites.empty?
394
- end
395
-
396
- def suggest_location
397
- if has_input_file? && @suggest_location.nil?
398
- @suggest_location = []
399
- loc_fsz = Hash.new(0)
400
- @prerequisites.each do |preq|
401
- t = application[preq]
402
- loc = t.location
403
- fsz = t.file_size
404
- if loc && fsz > 0
405
- loc.each do |h|
406
- loc_fsz[h] += fsz
407
- end
408
- end
409
- end
410
- if !loc_fsz.empty?
411
- half_max_fsz = loc_fsz.values.max / 2
412
- Log.debug "--- loc_fsz=#{loc_fsz.inspect} half_max_fsz=#{half_max_fsz}"
413
- loc_fsz.each do |h,sz|
414
- if sz > half_max_fsz
415
- @suggest_location << h
416
- end
417
- end
418
- end
419
- end
420
- @suggest_location
421
- end
422
-
423
- def priority
424
- if has_input_file? && @priority.nil?
425
- sum_tm = 0
426
- sum_sz = 0
427
- @prerequisites.each do |preq|
428
- pq = application[preq]
429
- sz = pq.file_size
430
- if sz > 0
431
- tm = pq.file_mtime - START_TIME
432
- sum_tm += tm * sz
433
- sum_sz += sz
434
- end
435
- end
436
- if sum_sz > 0
437
- @priority = sum_tm / sum_sz
438
- else
439
- @priority = 0
440
- end
441
- Log.debug "--- task_name=#{name} priority=#{@priority} sum_file_size=#{sum_sz}"
442
- end
443
- @priority || 0
444
- end
445
-
446
- def input_file_mtime
447
- if has_input_file? && @input_file_mtime.nil?
448
- hash = Hash.new
449
- max_sz = 0
450
- @prerequisites.each do |preq|
451
- t = application[preq]
452
- sz = t.file_size
453
- if sz > 0
454
- hash[t] = sz
455
- if sz > max_sz
456
- max_sz = sz
457
- end
458
- end
459
- end
460
- half_max_sz = max_sz / 2
461
- hash.each do |t,sz|
462
- if sz > half_max_sz
463
- time = t.file_mtime
464
- if @input_file_mtime.nil? || @input_file_mtime < time
465
- @input_file_mtime = time
466
- end
467
- end
468
- end
469
- end
470
- @input_file_mtime
471
- end
472
-
473
- end
474
-
475
- end # module Pwrake