pwrake 0.9.9.2 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +2 -0
  3. data/CHANGES_V2.md +90 -0
  4. data/{LICENSE.txt → MIT-LICENSE} +2 -3
  5. data/README +12 -0
  6. data/README.md +75 -52
  7. data/bin/gfwhere-pipe +23 -12
  8. data/bin/pwrake +22 -29
  9. data/bin/pwrake_branch +24 -0
  10. data/lib/pwrake/branch.rb +22 -0
  11. data/lib/pwrake/branch/branch.rb +213 -0
  12. data/lib/pwrake/branch/branch_application.rb +53 -0
  13. data/lib/pwrake/branch/fiber_queue.rb +36 -0
  14. data/lib/pwrake/branch/file_utils.rb +101 -0
  15. data/lib/pwrake/branch/shell.rb +231 -0
  16. data/lib/pwrake/{profiler.rb → branch/shell_profiler.rb} +28 -27
  17. data/lib/pwrake/branch/worker_communicator.rb +104 -0
  18. data/lib/pwrake/{gfarm_feature.rb → gfarm/gfarm_path.rb} +2 -100
  19. data/lib/pwrake/gfarm/gfarm_postprocess.rb +53 -0
  20. data/lib/pwrake/iomux/channel.rb +70 -0
  21. data/lib/pwrake/iomux/handler.rb +124 -0
  22. data/lib/pwrake/iomux/handler_set.rb +35 -0
  23. data/lib/pwrake/iomux/runner.rb +62 -0
  24. data/lib/pwrake/logger.rb +3 -150
  25. data/lib/pwrake/master.rb +30 -137
  26. data/lib/pwrake/master/fiber_pool.rb +69 -0
  27. data/lib/pwrake/master/idle_cores.rb +30 -0
  28. data/lib/pwrake/master/master.rb +345 -0
  29. data/lib/pwrake/master/master_application.rb +150 -0
  30. data/lib/pwrake/master/postprocess.rb +16 -0
  31. data/lib/pwrake/{graphviz.rb → misc/graphviz.rb} +0 -0
  32. data/lib/pwrake/{mcgp.rb → misc/mcgp.rb} +63 -42
  33. data/lib/pwrake/option/host_map.rb +158 -0
  34. data/lib/pwrake/option/option.rb +357 -0
  35. data/lib/pwrake/option/option_filesystem.rb +112 -0
  36. data/lib/pwrake/queue/locality_aware_queue.rb +158 -0
  37. data/lib/pwrake/queue/no_action_queue.rb +67 -0
  38. data/lib/pwrake/queue/queue_array.rb +366 -0
  39. data/lib/pwrake/queue/task_queue.rb +164 -0
  40. data/lib/pwrake/report.rb +1 -0
  41. data/lib/pwrake/report/parallelism.rb +9 -3
  42. data/lib/pwrake/report/report.rb +50 -103
  43. data/lib/pwrake/report/task_stat.rb +83 -0
  44. data/lib/pwrake/task/task_algorithm.rb +107 -0
  45. data/lib/pwrake/task/task_manager.rb +32 -0
  46. data/lib/pwrake/task/task_property.rb +98 -0
  47. data/lib/pwrake/task/task_rank.rb +48 -0
  48. data/lib/pwrake/task/task_wrapper.rb +296 -0
  49. data/lib/pwrake/version.rb +1 -1
  50. data/lib/pwrake/worker/executor.rb +169 -0
  51. data/lib/pwrake/worker/gfarm_directory.rb +90 -0
  52. data/lib/pwrake/worker/invoker.rb +199 -0
  53. data/lib/pwrake/worker/load.rb +14 -0
  54. data/lib/pwrake/worker/log_executor.rb +73 -0
  55. data/lib/pwrake/worker/shared_directory.rb +74 -0
  56. data/lib/pwrake/worker/worker_main.rb +14 -0
  57. data/lib/pwrake/worker/writer.rb +59 -0
  58. data/setup.rb +1212 -1502
  59. data/spec/003/Rakefile +2 -2
  60. data/spec/008/Rakefile +2 -1
  61. data/spec/009/Rakefile +1 -1
  62. data/spec/009/pwrake_conf.yaml +1 -3
  63. data/spec/hosts +0 -2
  64. data/spec/pwrake_spec.rb +9 -8
  65. metadata +50 -21
  66. data/lib/pwrake.rb +0 -19
  67. data/lib/pwrake/application.rb +0 -232
  68. data/lib/pwrake/counter.rb +0 -54
  69. data/lib/pwrake/file_utils.rb +0 -98
  70. data/lib/pwrake/gfwhere_pool.rb +0 -109
  71. data/lib/pwrake/host_list.rb +0 -88
  72. data/lib/pwrake/locality_aware_queue.rb +0 -413
  73. data/lib/pwrake/option.rb +0 -400
  74. data/lib/pwrake/rake_modify.rb +0 -14
  75. data/lib/pwrake/shell.rb +0 -186
  76. data/lib/pwrake/task_algorithm.rb +0 -475
  77. data/lib/pwrake/task_queue.rb +0 -633
  78. data/lib/pwrake/timer.rb +0 -22
@@ -0,0 +1,107 @@
1
+ module Pwrake
2
+
3
+ InvocationChain = Rake::InvocationChain
4
+ TaskArguments = Rake::TaskArguments
5
+
6
+ module TaskAlgorithm
7
+
8
+ attr_reader :wrapper
9
+ attr_reader :subsequents
10
+ attr_reader :arguments
11
+ attr_reader :property
12
+
13
+ def pw_search_tasks(args)
14
+ Log.debug "#{self.class}#pw_search_tasks start, args=#{args.inspect}"
15
+ tm = Time.now
16
+ task_args = TaskArguments.new(arg_names, args)
17
+ #timer = Timer.new("search_task")
18
+ #h = application.pwrake_options['HALT_QUEUE_WHILE_SEARCH']
19
+ #application.task_queue.synchronize(h) do
20
+ search_with_call_chain(nil, task_args, InvocationChain::EMPTY)
21
+ #end
22
+ #timer.finish
23
+ Log.debug "#{self.class}#pw_search_tasks end #{Time.now-tm}"
24
+ end
25
+
26
+ # Same as search, but explicitly pass a call chain to detect
27
+ # circular dependencies.
28
+ def search_with_call_chain(subseq, task_args, invocation_chain) # :nodoc:
29
+ new_chain = InvocationChain.append(self, invocation_chain)
30
+ @lock.synchronize do
31
+ if application.options.trace
32
+ #Log.info "** Search #{name} #{format_search_flags}"
33
+ application.trace "** Search #{name} #{format_search_flags}"
34
+ end
35
+
36
+ return true if @already_finished # <<--- competition !!!
37
+ @subsequents ||= []
38
+ @subsequents << subseq if subseq # <<--- competition !!!
39
+
40
+ if ! @already_searched
41
+ @already_searched = true
42
+ @arguments = task_args
43
+ @wrapper = TaskWrapper.new(self,task_args)
44
+ if @prerequisites.empty?
45
+ @unfinished_prereq = {}
46
+ else
47
+ search_prerequisites(task_args, new_chain)
48
+ end
49
+ #check_and_enq
50
+ if @unfinished_prereq.empty?
51
+ application.task_queue.enq(@wrapper)
52
+ end
53
+ end
54
+ return false
55
+ end
56
+ rescue Exception => ex
57
+ add_chain_to(ex, new_chain)
58
+ raise ex
59
+ end
60
+
61
+ # Search all the prerequisites of a task.
62
+ def search_prerequisites(task_args, invocation_chain) # :nodoc:
63
+ @unfinished_prereq = {}
64
+ @prerequisites.each{|t| @unfinished_prereq[t]=true}
65
+ prerequisite_tasks.each { |prereq|
66
+ prereq_args = task_args.new_scope(prereq.arg_names)
67
+ if prereq.search_with_call_chain(self, prereq_args, invocation_chain)
68
+ @unfinished_prereq.delete(prereq.name)
69
+ end
70
+ }
71
+ end
72
+
73
+ # Format the trace flags for display.
74
+ def format_search_flags
75
+ flags = []
76
+ flags << "finished" if @already_finished
77
+ flags << "first_time" unless @already_searched
78
+ flags << "not_needed" unless needed?
79
+ flags.empty? ? "" : "(" + flags.join(", ") + ")"
80
+ end
81
+ private :format_search_flags
82
+
83
+ def pw_enq_subsequents
84
+ t = Time.now
85
+ #h = application.pwrake_options['HALT_QUEUE_WHILE_SEARCH']
86
+ #application.task_queue.synchronize(h) do
87
+ @subsequents.each do |t| # <<--- competition !!!
88
+ if t && t.check_prereq_finished(self.name)
89
+ application.task_queue.enq(t.wrapper)
90
+ end
91
+ end
92
+ #end
93
+ @already_finished = true # <<--- competition !!!
94
+ end
95
+
96
+ def check_prereq_finished(preq_name=nil)
97
+ @unfinished_prereq.delete(preq_name)
98
+ @unfinished_prereq.empty?
99
+ end
100
+
101
+ def pw_set_property(property)
102
+ @property = property
103
+ self
104
+ end
105
+
106
+ end
107
+ end
@@ -0,0 +1,32 @@
1
+ module Pwrake
2
+
3
+ module TaskManager
4
+
5
+ def initialize
6
+ @property_by_block = {}
7
+ @last_property = TaskProperty.new
8
+ super
9
+ end
10
+
11
+ def last_description=(description)
12
+ @last_property.parse_description(description)
13
+ super
14
+ end
15
+
16
+ def define_task(task_class, *args, &block) # :nodoc:
17
+ prop = @property_by_block[block.object_id]
18
+ if prop.nil?
19
+ prop = @last_property
20
+ @last_property = TaskProperty.new
21
+ end
22
+ super.pw_set_property(prop)
23
+ end
24
+
25
+ def create_rule(*args, &block) # :nodoc:
26
+ @property_by_block[block.object_id] = @last_property
27
+ @last_property = TaskProperty.new
28
+ super
29
+ end
30
+
31
+ end
32
+ end
@@ -0,0 +1,98 @@
1
+ module Pwrake
2
+
3
+ class TaskProperty
4
+
5
+ attr_reader :ncore, :exclusive, :allow, :deny, :order_allow_deny,
6
+ :disable_steal
7
+
8
+ def parse_description(description)
9
+ if /\bn_?cores?[=:]\s*([+-]?\d+)/i =~ description
10
+ @ncore = $1.to_i
11
+ end
12
+ if /\bexclusive[=:]\s*(\S+)/i =~ description
13
+ if /^(y|t)/i =~ $1
14
+ @exclusive = true
15
+ end
16
+ end
17
+ if /\ballow[=:]\s*(\S+)/i =~ description
18
+ @allow = $1
19
+ end
20
+ if /\bdeny[=:]\s*(\S+)/i =~ description
21
+ @deny = $1
22
+ end
23
+ if /\border[=:]\s*(\S+)/i =~ description
24
+ case $1
25
+ when /allow,deny/i
26
+ @order_allow_deny = true
27
+ when /deny,allow/i
28
+ @order_allow_deny = false
29
+ end
30
+ end
31
+ if /\bsteal[=:]\s*(\S+)/i =~ description
32
+ if /^(n|f)/i =~ $1
33
+ @disable_steal = true
34
+ end
35
+ end
36
+ end
37
+
38
+ def acceptable_for(host_info)
39
+ if @disable_steal && host_info.steal_flag
40
+ #Log.debug("@disable_steal && host_info.steal_flag")
41
+ return false
42
+ end
43
+ ncore = (@exclusive) ? 0 : (@ncore || 1)
44
+ if ncore > 0
45
+ return false if ncore > host_info.idle_cores
46
+ else
47
+ n = host_info.ncore + ncore
48
+ return false if n < 1 || n > host_info.idle_cores
49
+ end
50
+ hn = host_info.name
51
+ if @allow
52
+ if @deny
53
+ if @order_allow_deny
54
+ return false if !File.fnmatch(@allow,hn) || File.fnmatch(@deny,hn)
55
+ else
56
+ return false if File.fnmatch(@deny,hn) && !File.fnmatch(@allow,hn)
57
+ end
58
+ else
59
+ return false if !File.fnmatch(@allow,hn)
60
+ end
61
+ else
62
+ if @deny
63
+ return false if File.fnmatch(@deny,hn)
64
+ end
65
+ end
66
+ return true
67
+ end
68
+
69
+ def n_used_cores(host_info=nil)
70
+ nc_node = host_info && host_info.ncore
71
+ if @ncore.nil?
72
+ return 1
73
+ elsif @ncore > 0
74
+ if nc_node && @ncore > nc_node
75
+ m = "ncore=#{@ncore} must be <= nc_node=#{nc_node}"
76
+ Log.fatal m
77
+ raise RuntimeError,m
78
+ end
79
+ return @ncore
80
+ else
81
+ if nc_node.nil?
82
+ m = "host_info.ncore is not set"
83
+ Log.fatal m
84
+ raise RuntimeError,m
85
+ end
86
+ n = @ncore + nc_node
87
+ if n > 0
88
+ return n
89
+ else
90
+ m = "ncore+nc_node=#{n} must be > 0"
91
+ Log.fatal m
92
+ raise RuntimeError,m
93
+ end
94
+ end
95
+ end
96
+
97
+ end
98
+ end
@@ -0,0 +1,48 @@
1
+ module Pwrake
2
+
3
+ class RankStat
4
+
5
+ def initialize
6
+ @lock = Mutex.new
7
+ @stat = []
8
+ end
9
+
10
+ def add_sample(rank,elap)
11
+ @lock.synchronize do
12
+ stat = @stat[rank]
13
+ if stat.nil?
14
+ @stat[rank] = stat = [0,0.0]
15
+ end
16
+ stat[0] += 1
17
+ stat[1] += elap
18
+ #Log.debug "add_sample rank=#{rank} stat=#{stat.inspect} weight=#{stat[0]/stat[1]}"
19
+ end
20
+ end
21
+
22
+ def rank_weight
23
+ @lock.synchronize do
24
+ sum = 0.0
25
+ count = 0
26
+ weight = @stat.map do |stat|
27
+ if stat
28
+ w = stat[0]/stat[1]
29
+ sum += w
30
+ count += 1
31
+ w
32
+ else
33
+ nil
34
+ end
35
+ end
36
+ if count == 0
37
+ avg = 1.0
38
+ else
39
+ avg = sum/count
40
+ end
41
+ [weight, avg]
42
+ end
43
+ end
44
+ end
45
+
46
+ RANK_STAT = RankStat.new
47
+
48
+ end
@@ -0,0 +1,296 @@
1
+ require 'forwardable'
2
+
3
+ module Pwrake
4
+
5
+ class TaskWrapper
6
+ extend Forwardable
7
+
8
+ @@current_id = 1
9
+ @@task_logger = nil
10
+
11
+ def initialize(task,task_args=nil)
12
+ @task = task
13
+ @task_args = task_args
14
+ @property = task.property
15
+ @task_id = @@current_id
16
+ @@current_id += 1
17
+ @location = []
18
+ @group = []
19
+ @group_id
20
+ @suggest_location = nil
21
+ @file_stat
22
+ @input_file_size
23
+ @input_file_mtime
24
+ @rank
25
+ @priority
26
+ @lock_rank = Monitor.new
27
+ @executed = false
28
+ @assigned = []
29
+ @exec_host = nil
30
+ end
31
+
32
+ def_delegators :@task, :name, :actions, :prerequisites, :subsequents
33
+ def_delegators :@property, :acceptable_for
34
+
35
+ attr_reader :task, :task_id, :group, :group_id, :file_stat
36
+ attr_reader :location
37
+ attr_reader :assigned
38
+ attr_accessor :executed
39
+ attr_accessor :exec_host
40
+ attr_accessor :shell_id, :status
41
+
42
+ def self.format_time(t)
43
+ t.strftime("%F %T.%L")
44
+ end
45
+
46
+ def self.init_task_logger(option)
47
+ if dir = option['LOG_DIR']
48
+ fn = File.join(dir,option['TASK_CSV_FILE'])
49
+ @@task_logger = CSV.open(fn,'w')
50
+ @@task_logger.puts %w[
51
+ task_id task_name start_time end_time elap_time preq preq_host
52
+ exec_host shell_id has_action executed file_size file_mtime file_host
53
+ ]
54
+ end
55
+ end
56
+
57
+ def self.close_task_logger
58
+ @@task_logger.close if @@task_logger
59
+ end
60
+
61
+ def preprocess
62
+ if @shell = Pwrake::Shell.current
63
+ @shell.current_task = self
64
+ end
65
+ @time_start = Time.now
66
+ end
67
+
68
+ def postprocess(location)
69
+ @executed = true if !@task.actions.empty?
70
+ tm_taskend = Time.now
71
+ if is_file_task?
72
+ t = Time.now
73
+ if File.exist?(name)
74
+ @file_stat = File::Stat.new(name)
75
+ @location = location
76
+ end
77
+ end
78
+ #Log.debug "postprocess time=#{Time.now-tm_taskend}"
79
+ log_task
80
+ @shell.current_task = nil if @shell
81
+ if @status=="end"
82
+ @task.pw_enq_subsequents
83
+ end
84
+ end
85
+
86
+ def log_task
87
+ @time_end = Time.now
88
+ #
89
+ loc = suggest_location()
90
+ shell = Pwrake::Shell.current
91
+ #
92
+ if loc && !loc.empty? && shell && !actions.empty?
93
+ Rake.application.count( loc, shell.host )
94
+ end
95
+ return if !@@task_logger
96
+ #
97
+ elap = @time_end - @time_start
98
+ if has_output_file?
99
+ RANK_STAT.add_sample(rank,elap)
100
+ end
101
+ #
102
+ if @file_stat
103
+ fstat = [@file_stat.size, @file_stat.mtime, self.location.join('|')]
104
+ else
105
+ fstat = [nil]*3
106
+ end
107
+ #
108
+ # task_id task_name start_time end_time elap_time preq preq_host
109
+ # exec_host shell_id has_action executed file_size file_mtime file_host
110
+ #
111
+ row = [ @task_id, name, @time_start, @time_end, elap,
112
+ prerequisites, loc, @exec_host, @shell_id,
113
+ (actions.empty?) ? 0 : 1,
114
+ (@executed) ? 1 : 0,
115
+ *fstat ]
116
+ row.map!{|x|
117
+ if x.kind_of?(Time)
118
+ TaskWrapper.format_time(x)
119
+ elsif x.kind_of?(Array)
120
+ if x.empty?
121
+ nil
122
+ else
123
+ x.join('|')
124
+ end
125
+ else
126
+ x
127
+ end
128
+ }
129
+ @@task_logger << row
130
+ #
131
+ clsname = @task.class.to_s.sub(/^(Rake|Pwrake)::/o,"")
132
+ msg = '%s:"%s" %s: id=%d elap=%.6f exec_host=%s' %
133
+ [clsname,name,@status,@task_id,elap,@exec_host]
134
+ if @status=="end"
135
+ Log.info msg
136
+ else
137
+ Log.error msg
138
+ end
139
+ end
140
+
141
+ def is_file_task?
142
+ @task.kind_of?(Rake::FileTask)
143
+ end
144
+
145
+ def has_output_file?
146
+ is_file_task? && !actions.empty?
147
+ end
148
+
149
+ def has_input_file?
150
+ is_file_task? && !prerequisites.empty?
151
+ end
152
+
153
+ def has_action?
154
+ !@task.actions.empty?
155
+ end
156
+
157
+ def location=(a)
158
+ @location = a
159
+ @group = []
160
+ #@location.each do |host|
161
+ # @group |= [Rake.application.host_list.host2group[host]]
162
+ #end
163
+ end
164
+
165
+ def suggest_location=(a)
166
+ @suggest_location = a
167
+ end
168
+
169
+ def suggest_location
170
+ if has_input_file? && @suggest_location.nil?
171
+ @suggest_location = []
172
+ loc_fsz = Hash.new(0)
173
+ prerequisites.each do |preq|
174
+ t = Rake.application[preq].wrapper
175
+ loc = t.location
176
+ fsz = t.file_size
177
+ if loc && fsz > 0
178
+ loc.each do |h|
179
+ loc_fsz[h] += fsz
180
+ end
181
+ end
182
+ end
183
+ #Log.debug "input=#{prerequisites.join('|')}"
184
+ if !loc_fsz.empty?
185
+ half_max_fsz = loc_fsz.values.max / 2
186
+ Log.debug "loc_fsz=#{loc_fsz.inspect} half_max_fsz=#{half_max_fsz}"
187
+ loc_fsz.each do |h,sz|
188
+ if sz > half_max_fsz
189
+ @suggest_location << h
190
+ end
191
+ end
192
+ end
193
+ end
194
+ @suggest_location
195
+ end
196
+
197
+ def rank
198
+ #@lock_rank.synchronize do
199
+ if @rank.nil?
200
+ if subsequents.nil? || subsequents.empty?
201
+ @rank = 0
202
+ else
203
+ max_rank = 0
204
+ subsequents.each do |subsq|
205
+ r = subsq.wrapper.rank
206
+ if max_rank < r
207
+ max_rank = r
208
+ end
209
+ end
210
+ if has_output_file?
211
+ step = 1
212
+ else
213
+ step = 0
214
+ end
215
+ @rank = max_rank + step
216
+ end
217
+ Log.debug "Task[#{name}] rank=#{@rank.inspect}"
218
+ end
219
+ #end
220
+ @rank
221
+ end
222
+
223
+ def file_size
224
+ @file_stat ? @file_stat.size : 0
225
+ end
226
+
227
+ def file_mtime
228
+ @file_stat ? @file_stat.mtime : Time.at(0)
229
+ end
230
+
231
+ def input_file_size
232
+ unless @input_file_size
233
+ @input_file_size = 0
234
+ prerequisites.each do |preq|
235
+ @input_file_size += Rake.application[preq].wrapper.file_size
236
+ end
237
+ end
238
+ @input_file_size
239
+ end
240
+
241
+ def input_file_mtime
242
+ if has_input_file? && @input_file_mtime.nil?
243
+ hash = Hash.new
244
+ max_sz = 0
245
+ prerequisites.each do |preq|
246
+ t = Rake.application[preq].wrapper
247
+ sz = t.file_size
248
+ if sz > 0
249
+ hash[t] = sz
250
+ if sz > max_sz
251
+ max_sz = sz
252
+ end
253
+ end
254
+ end
255
+ half_max_sz = max_sz / 2
256
+ hash.each do |t,sz|
257
+ if sz > half_max_sz
258
+ time = t.file_mtime
259
+ if @input_file_mtime.nil? || @input_file_mtime < time
260
+ @input_file_mtime = time
261
+ end
262
+ end
263
+ end
264
+ end
265
+ @input_file_mtime
266
+ end
267
+
268
+ def priority
269
+ if has_input_file? && @priority.nil?
270
+ sum_tm = 0
271
+ sum_sz = 0
272
+ prerequisites.each do |preq|
273
+ pq = Rake.application[preq].wrapper
274
+ sz = pq.file_size
275
+ if sz > 0
276
+ tm = pq.file_mtime - START_TIME
277
+ sum_tm += tm * sz
278
+ sum_sz += sz
279
+ end
280
+ end
281
+ if sum_sz > 0
282
+ @priority = sum_tm / sum_sz
283
+ else
284
+ @priority = 0
285
+ end
286
+ Log.debug "task_name=#{name} priority=#{@priority} sum_file_size=#{sum_sz}"
287
+ end
288
+ @priority || 0
289
+ end
290
+
291
+ def n_used_cores(host_info=nil)
292
+ @n_used_cores ||= @property.n_used_cores(host_info)
293
+ end
294
+
295
+ end
296
+ end