pwrake 0.9.9.2 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +2 -0
  3. data/CHANGES_V2.md +90 -0
  4. data/{LICENSE.txt → MIT-LICENSE} +2 -3
  5. data/README +12 -0
  6. data/README.md +75 -52
  7. data/bin/gfwhere-pipe +23 -12
  8. data/bin/pwrake +22 -29
  9. data/bin/pwrake_branch +24 -0
  10. data/lib/pwrake/branch.rb +22 -0
  11. data/lib/pwrake/branch/branch.rb +213 -0
  12. data/lib/pwrake/branch/branch_application.rb +53 -0
  13. data/lib/pwrake/branch/fiber_queue.rb +36 -0
  14. data/lib/pwrake/branch/file_utils.rb +101 -0
  15. data/lib/pwrake/branch/shell.rb +231 -0
  16. data/lib/pwrake/{profiler.rb → branch/shell_profiler.rb} +28 -27
  17. data/lib/pwrake/branch/worker_communicator.rb +104 -0
  18. data/lib/pwrake/{gfarm_feature.rb → gfarm/gfarm_path.rb} +2 -100
  19. data/lib/pwrake/gfarm/gfarm_postprocess.rb +53 -0
  20. data/lib/pwrake/iomux/channel.rb +70 -0
  21. data/lib/pwrake/iomux/handler.rb +124 -0
  22. data/lib/pwrake/iomux/handler_set.rb +35 -0
  23. data/lib/pwrake/iomux/runner.rb +62 -0
  24. data/lib/pwrake/logger.rb +3 -150
  25. data/lib/pwrake/master.rb +30 -137
  26. data/lib/pwrake/master/fiber_pool.rb +69 -0
  27. data/lib/pwrake/master/idle_cores.rb +30 -0
  28. data/lib/pwrake/master/master.rb +345 -0
  29. data/lib/pwrake/master/master_application.rb +150 -0
  30. data/lib/pwrake/master/postprocess.rb +16 -0
  31. data/lib/pwrake/{graphviz.rb → misc/graphviz.rb} +0 -0
  32. data/lib/pwrake/{mcgp.rb → misc/mcgp.rb} +63 -42
  33. data/lib/pwrake/option/host_map.rb +158 -0
  34. data/lib/pwrake/option/option.rb +357 -0
  35. data/lib/pwrake/option/option_filesystem.rb +112 -0
  36. data/lib/pwrake/queue/locality_aware_queue.rb +158 -0
  37. data/lib/pwrake/queue/no_action_queue.rb +67 -0
  38. data/lib/pwrake/queue/queue_array.rb +366 -0
  39. data/lib/pwrake/queue/task_queue.rb +164 -0
  40. data/lib/pwrake/report.rb +1 -0
  41. data/lib/pwrake/report/parallelism.rb +9 -3
  42. data/lib/pwrake/report/report.rb +50 -103
  43. data/lib/pwrake/report/task_stat.rb +83 -0
  44. data/lib/pwrake/task/task_algorithm.rb +107 -0
  45. data/lib/pwrake/task/task_manager.rb +32 -0
  46. data/lib/pwrake/task/task_property.rb +98 -0
  47. data/lib/pwrake/task/task_rank.rb +48 -0
  48. data/lib/pwrake/task/task_wrapper.rb +296 -0
  49. data/lib/pwrake/version.rb +1 -1
  50. data/lib/pwrake/worker/executor.rb +169 -0
  51. data/lib/pwrake/worker/gfarm_directory.rb +90 -0
  52. data/lib/pwrake/worker/invoker.rb +199 -0
  53. data/lib/pwrake/worker/load.rb +14 -0
  54. data/lib/pwrake/worker/log_executor.rb +73 -0
  55. data/lib/pwrake/worker/shared_directory.rb +74 -0
  56. data/lib/pwrake/worker/worker_main.rb +14 -0
  57. data/lib/pwrake/worker/writer.rb +59 -0
  58. data/setup.rb +1212 -1502
  59. data/spec/003/Rakefile +2 -2
  60. data/spec/008/Rakefile +2 -1
  61. data/spec/009/Rakefile +1 -1
  62. data/spec/009/pwrake_conf.yaml +1 -3
  63. data/spec/hosts +0 -2
  64. data/spec/pwrake_spec.rb +9 -8
  65. metadata +50 -21
  66. data/lib/pwrake.rb +0 -19
  67. data/lib/pwrake/application.rb +0 -232
  68. data/lib/pwrake/counter.rb +0 -54
  69. data/lib/pwrake/file_utils.rb +0 -98
  70. data/lib/pwrake/gfwhere_pool.rb +0 -109
  71. data/lib/pwrake/host_list.rb +0 -88
  72. data/lib/pwrake/locality_aware_queue.rb +0 -413
  73. data/lib/pwrake/option.rb +0 -400
  74. data/lib/pwrake/rake_modify.rb +0 -14
  75. data/lib/pwrake/shell.rb +0 -186
  76. data/lib/pwrake/task_algorithm.rb +0 -475
  77. data/lib/pwrake/task_queue.rb +0 -633
  78. data/lib/pwrake/timer.rb +0 -22
@@ -1,633 +0,0 @@
1
- module Pwrake
2
-
3
- class TaskConditionVariable < ConditionVariable
4
- def signal(hint=nil)
5
- super()
6
- end
7
- end
8
-
9
-
10
- class PriorityQueueArray < Array
11
- def initialize(n)
12
- super()
13
- end
14
-
15
- def shift
16
- pop
17
- end
18
-
19
- def push(t)
20
- priority = t.priority
21
- if empty? || last.priority <= priority
22
- super(t)
23
- elsif first.priority > priority
24
- unshift(t)
25
- else
26
- lower = 0
27
- upper = size-1
28
- while lower+1 < upper
29
- mid = ((lower + upper) / 2).to_i
30
- if self[mid].priority <= priority
31
- lower = mid
32
- else
33
- upper = mid
34
- end
35
- end
36
- insert(upper,t)
37
- end
38
- end
39
-
40
- def index(t)
41
- if size < 40
42
- return super(t)
43
- end
44
- priority = t.priority
45
- if last.priority < priority || first.priority > priority
46
- nil
47
- else
48
- lower = 0
49
- upper = size-1
50
- while lower+1 < upper
51
- mid = ((lower + upper) / 2).to_i
52
- if self[mid].priority < priority
53
- lower = mid
54
- else
55
- upper = mid
56
- end
57
- end
58
- mid = upper
59
- if self[mid].priority == priority
60
- Log.debug "--- TQA#index=#{mid}, priority=#{priority}"
61
- mid
62
- end
63
- end
64
- end
65
- end # PriorityQueueArray
66
-
67
-
68
- class LifoQueueArray < Array
69
- def initialize(n)
70
- super()
71
- end
72
-
73
- def shift
74
- pop
75
- end
76
- end
77
-
78
-
79
- class FifoQueueArray < Array
80
- def initialize(n)
81
- super()
82
- end
83
- end
84
-
85
- class RankCounter
86
-
87
- def initialize
88
- @ntask = []
89
- @nproc = 0
90
- @mutex = Mutex.new
91
- end
92
-
93
- def add_nproc(n)
94
- @mutex.synchronize do
95
- @nproc += n
96
- end
97
- end
98
-
99
- def incr(r)
100
- @mutex.synchronize do
101
- @ntask[r] = (@ntask[r]||0) + 1
102
- end
103
- end
104
-
105
- def get_task
106
- @mutex.synchronize do
107
- (@ntask.size-1).downto(0) do |r|
108
- c = @ntask[r]
109
- if c && c>0
110
- t = yield(c, @nproc, r)
111
- #t = (c<=@n) ? pop_last_rank(r) : pop
112
- if t
113
- @ntask[t.rank] -= 1
114
- Log.debug "--- RankCount rank=#{r} nproc=#{@nproc} count=#{c} t.rank=#{t.rank} t.name=#{t.name}"
115
- end
116
- return t
117
- end
118
- end
119
- end
120
- nil
121
- end
122
- end
123
-
124
- # HRF mixin module
125
- module HrfQueue
126
-
127
- def hrf_init(n=nil)
128
- @nproc = n || 0
129
- @count = []
130
- end
131
-
132
- def hrf_push(t)
133
- if t
134
- r = t.rank
135
- @count[r] = (@count[r]||0) + 1
136
- end
137
- end
138
-
139
- def hrf_get
140
- (@count.size-1).downto(0) do |r|
141
- c = @count[r]
142
- if c && c>0
143
- t = (c <= @nproc) ? pop_last_rank(r) : pop_super
144
- if t
145
- @count[t.rank] -= 1
146
- end
147
- return t
148
- end
149
- end
150
- raise "no element"
151
- nil
152
- end
153
-
154
- def pop_last_rank(r)
155
- (size-1).downto(0) do |i|
156
- t = at(i)
157
- if t && t.rank == r
158
- delete_at(i)
159
- return t
160
- end
161
- end
162
- nil
163
- end
164
- end
165
-
166
- # LIFO + HRF
167
- class LifoHrfQueueArray < Array
168
- include HrfQueue
169
-
170
- def initialize(n)
171
- super()
172
- hrf_init(n)
173
- end
174
-
175
- def push(t)
176
- super(t)
177
- hrf_push(t)
178
- end
179
-
180
- def shift
181
- return nil if empty?
182
- hrf_get
183
- end
184
-
185
- def pop_super
186
- pop
187
- end
188
- end
189
-
190
- # Priority + HRF
191
- class PriorityHrfQueueArray < PriorityQueueArray
192
- include HrfQueue
193
-
194
- def initialize(n)
195
- super(n)
196
- hrf_init(n)
197
- end
198
-
199
- def push(t)
200
- super(t)
201
- hrf_push(t)
202
- end
203
-
204
- def shift
205
- return nil if empty?
206
- hrf_get
207
- end
208
-
209
- def pop_super
210
- pop
211
- end
212
- end
213
-
214
-
215
- # Rank-Even Last In First Out
216
- class RankQueueArray
217
-
218
- def initialize(n)
219
- @q = []
220
- @size = 0
221
- @n = (n>0) ? n : 1
222
- end
223
-
224
- def push(t)
225
- r = t ? t.rank : 0
226
- a = @q[r]
227
- if a.nil?
228
- @q[r] = a = []
229
- end
230
- @size += 1
231
- a.push(t)
232
- end
233
-
234
- def size
235
- @size
236
- end
237
-
238
- def empty?
239
- @size == 0
240
- end
241
-
242
- def shift
243
- if empty?
244
- return nil
245
- end
246
- (@q.size-1).downto(0) do |i|
247
- a = @q[i]
248
- next if a.nil? || a.empty?
249
- @size -= 1
250
- if a.size <= @n
251
- return pop_last_max(a)
252
- else
253
- return shift_weighted
254
- end
255
- end
256
- raise "ELIFO: @q=#{@q.inspect}"
257
- end
258
-
259
- def shift_weighted
260
- weight, weight_avg = RANK_STAT.rank_weight
261
- wsum = 0.0
262
- q = []
263
- @q.each_with_index do |a,i|
264
- next if a.nil? || a.empty?
265
- w = weight[i]
266
- w = weight_avg if w.nil?
267
- # w *= a.size
268
- wsum += w
269
- q << [a,wsum]
270
- end
271
- #
272
- x = rand() * wsum
273
- Log.debug "--- shift_weighted x=#{x} wsum=#{wsum} weight=#{weight.inspect}"
274
- q.each do |a,w|
275
- if w > x
276
- return a.pop
277
- end
278
- end
279
- raise "ELIFO: wsum=#{wsum} x=#{x}"
280
- end
281
-
282
- def pop_last_max(a)
283
- if a.size < 2
284
- return a.pop
285
- end
286
- y_max = nil
287
- i_max = nil
288
- n = [@n, a.size].min
289
- (-n..-1).each do |i|
290
- y = a[i].input_file_size
291
- if y_max.nil? || y > y_max
292
- y_max = y
293
- i_max = i
294
- end
295
- end
296
- a.delete_at(i_max)
297
- end
298
-
299
- def first
300
- return nil if empty?
301
- @q.size.times do |i|
302
- a = @q[i]
303
- unless a.nil? || a.empty?
304
- return a.first
305
- end
306
- end
307
- end
308
-
309
- def last
310
- return nil if empty?
311
- @q.size.times do |i|
312
- a = @q[-i-1]
313
- unless a.nil? || a.empty?
314
- return a.last
315
- end
316
- end
317
- end
318
-
319
- def delete(t)
320
- n = 0
321
- @q.each do |a|
322
- if a
323
- a.delete(t)
324
- n += a.size
325
- end
326
- end
327
- @size = n
328
- end
329
-
330
- def clear
331
- @q.clear
332
- @size = 0
333
- end
334
- end
335
-
336
-
337
- class NoActionQueue
338
- def initialize
339
- @que = []
340
- @num_waiting = 0
341
- @mutex = Mutex.new
342
- @cond = ConditionVariable.new
343
- @halt = false
344
- @th_end = {}
345
- prio = Pwrake.application.pwrake_options['NOACTION_QUEUE_PRIORITY'] || 'fifo'
346
- case prio
347
- when /fifo/i
348
- @prio = 0
349
- Log.debug "--- NOACTION_QUEUE_PRIORITY=FIFO"
350
- when /lifo/i
351
- @prio = 1
352
- Log.debug "--- NOACTION_QUEUE_PRIORITY=LIFO"
353
- when /rand/i
354
- @prio = 2
355
- Log.debug "--- NOACTION_QUEUE_PRIORITY=RAND"
356
- else
357
- raise RuntimeError,"unknown option for NOACTION_QUEUE_PRIORITY: "+prio
358
- end
359
- end
360
-
361
- def push(obj)
362
- if @halt
363
- @que.push obj
364
- else
365
- @mutex.synchronize do
366
- @que.push obj
367
- @cond.signal
368
- end
369
- end
370
- end
371
- alias << push
372
- alias enq push
373
-
374
- def pop
375
- @mutex.synchronize do
376
- t = Time.now
377
- while true
378
- if @th_end.delete(Thread.current)
379
- return false
380
- elsif @halt
381
- @cond.wait @mutex
382
- elsif @que.empty?
383
- if @finished
384
- @cond.signal
385
- return false
386
- end
387
- @cond.wait @mutex
388
- else
389
- case @prio
390
- when 0
391
- x = @que.shift
392
- when 1
393
- x = @que.pop
394
- when 2
395
- x = @que.delete_at(rand(@que.size))
396
- end
397
- Log.debug "--- NATQ#deq %.6f sec #{x.inspect}"%[Time.now-t]
398
- return x
399
- end
400
- end
401
- end
402
- end
403
-
404
- alias shift pop
405
- alias deq pop
406
-
407
- def halt
408
- @mutex.lock
409
- @halt = true
410
- end
411
-
412
- def resume
413
- @halt = false
414
- @mutex.unlock
415
- @cond.broadcast
416
- end
417
-
418
- def empty?
419
- @que.empty?
420
- end
421
-
422
- def clear
423
- @que.clear
424
- end
425
-
426
- def length
427
- @que.length
428
- end
429
- alias size length
430
-
431
- def first
432
- @que.first
433
- end
434
-
435
- def last
436
- @que.last
437
- end
438
-
439
- def finish
440
- @finished = true
441
- @cond.broadcast
442
- end
443
-
444
- def thread_end(th)
445
- @th_end[th] = true
446
- @cond.broadcast
447
- end
448
-
449
- def stop
450
- clear
451
- finish
452
- end
453
- end
454
-
455
-
456
- class TaskQueue
457
-
458
- def initialize(host_list)
459
- @finished = false
460
- @halt = false
461
- @mutex = Mutex.new
462
- @th_end = {}
463
- @enable_steal = true
464
- @q_noaction = NoActionQueue.new
465
- pri = Pwrake.application.pwrake_options['QUEUE_PRIORITY'] || "RANK"
466
- case pri
467
- when /prio/i
468
- @array_class = PriorityQueueArray
469
- when /fifo/i
470
- @array_class = FifoQueueArray # Array # Fifo
471
- when /lifo/i
472
- @array_class = LifoQueueArray
473
- when /lihr/i
474
- @array_class = LifoHrfQueueArray
475
- when /prhr/i
476
- @array_class = PriorityHrfQueueArray
477
- when /rank/i
478
- @array_class = RankQueueArray
479
- else
480
- raise RuntimeError,"unknown option for QUEUE_PRIORITY: "+pri
481
- end
482
- Log.debug "--- TQ#initialize @array_class=#{@array_class.inspect}"
483
- init_queue(host_list)
484
- end
485
-
486
- def init_queue(host_list)
487
- @cv = TaskConditionVariable.new
488
- @q_input = @array_class.new(host_list.size)
489
- @q_later = Array.new
490
- end
491
-
492
- attr_reader :mutex
493
- attr_accessor :enable_steal
494
-
495
- def halt
496
- @q_noaction.halt
497
- @mutex.lock
498
- @halt = true
499
- end
500
-
501
- def resume
502
- @halt = false
503
- @q_noaction.resume
504
- @mutex.unlock
505
- @cv.broadcast
506
- end
507
-
508
- def synchronize(condition)
509
- ret = nil
510
- if condition
511
- halt
512
- begin
513
- ret = yield
514
- ensure
515
- resume
516
- end
517
- else
518
- ret = yield
519
- end
520
- ret
521
- end
522
-
523
- # enq
524
- def enq(item)
525
- Log.debug "--- TQ#enq #{item.name}"
526
- t0 = Time.now
527
- if item.actions.empty?
528
- @q_noaction.enq(item)
529
- elsif @halt
530
- enq_body(item)
531
- else
532
- @mutex.synchronize do
533
- enq_body(item)
534
- @cv.signal(item.suggest_location)
535
- end
536
- end
537
- Log.debug "--- TQ#enq #{item.name} enq_time=#{Time.now-t0}"
538
- end
539
-
540
- def enq_body(item)
541
- enq_impl(item)
542
- end
543
-
544
- def enq_impl(t)
545
- if t.has_input_file?
546
- @q_input.push(t)
547
- else
548
- @q_later.push(t)
549
- end
550
- end
551
-
552
- # deq
553
- def deq(hint=nil)
554
- if hint == '(noaction)'
555
- return @q_noaction.deq
556
- end
557
- n = 0
558
- loop do
559
- @mutex.synchronize do
560
- t0 = Time.now
561
- if @th_end.delete(Thread.current)
562
- return false
563
-
564
- elsif @halt
565
- Log.debug "--- halt in TQ#deq @q=#{@q.inspect}"
566
- @cv.wait(@mutex)
567
- n = 0
568
-
569
- elsif empty? # no item in queue
570
- if @finished
571
- @cv.signal
572
- return false
573
- end
574
- #@cv.signal
575
- @cv.wait(@mutex)
576
- n = 0
577
-
578
- else
579
- if t = deq_impl(hint,n)
580
- t_inspect = t.inspect[0..1000]
581
- Log.debug "--- TQ#deq #{t_inspect} deq_time=#{Time.now-t0}"
582
- return t
583
- end
584
- n += 1
585
- end
586
- end
587
- end
588
- end
589
-
590
- def deq_impl(hint,n)
591
- Log.debug "--- TQ#deq_impl #{@q.inspect}"
592
- @q_input.shift || @q_later.shift
593
- end
594
-
595
- def clear
596
- @q_noaction.clear
597
- @q_input.clear
598
- @q_later.clear
599
- end
600
-
601
- def empty?
602
- @q_noaction.empty? &&
603
- @q_input.empty? &&
604
- @q_later.empty?
605
- end
606
-
607
- def finish
608
- Log.debug "--- TQ#finish"
609
- @q_noaction.finish
610
- @finished = true
611
- @cv.broadcast
612
- end
613
-
614
- def stop
615
- @q_noaction.stop
616
- @mutex.synchronize do
617
- clear
618
- finish
619
- end
620
- end
621
-
622
- def thread_end(th)
623
- @th_end[th] = true
624
- @cv.broadcast
625
- @q_noaction.thread_end(th)
626
- end
627
-
628
- def after_check(tasks)
629
- # implimented at subclass
630
- end
631
-
632
- end
633
- end