pwrake 0.9.9.2 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +2 -0
  3. data/CHANGES_V2.md +90 -0
  4. data/{LICENSE.txt → MIT-LICENSE} +2 -3
  5. data/README +12 -0
  6. data/README.md +75 -52
  7. data/bin/gfwhere-pipe +23 -12
  8. data/bin/pwrake +22 -29
  9. data/bin/pwrake_branch +24 -0
  10. data/lib/pwrake/branch.rb +22 -0
  11. data/lib/pwrake/branch/branch.rb +213 -0
  12. data/lib/pwrake/branch/branch_application.rb +53 -0
  13. data/lib/pwrake/branch/fiber_queue.rb +36 -0
  14. data/lib/pwrake/branch/file_utils.rb +101 -0
  15. data/lib/pwrake/branch/shell.rb +231 -0
  16. data/lib/pwrake/{profiler.rb → branch/shell_profiler.rb} +28 -27
  17. data/lib/pwrake/branch/worker_communicator.rb +104 -0
  18. data/lib/pwrake/{gfarm_feature.rb → gfarm/gfarm_path.rb} +2 -100
  19. data/lib/pwrake/gfarm/gfarm_postprocess.rb +53 -0
  20. data/lib/pwrake/iomux/channel.rb +70 -0
  21. data/lib/pwrake/iomux/handler.rb +124 -0
  22. data/lib/pwrake/iomux/handler_set.rb +35 -0
  23. data/lib/pwrake/iomux/runner.rb +62 -0
  24. data/lib/pwrake/logger.rb +3 -150
  25. data/lib/pwrake/master.rb +30 -137
  26. data/lib/pwrake/master/fiber_pool.rb +69 -0
  27. data/lib/pwrake/master/idle_cores.rb +30 -0
  28. data/lib/pwrake/master/master.rb +345 -0
  29. data/lib/pwrake/master/master_application.rb +150 -0
  30. data/lib/pwrake/master/postprocess.rb +16 -0
  31. data/lib/pwrake/{graphviz.rb → misc/graphviz.rb} +0 -0
  32. data/lib/pwrake/{mcgp.rb → misc/mcgp.rb} +63 -42
  33. data/lib/pwrake/option/host_map.rb +158 -0
  34. data/lib/pwrake/option/option.rb +357 -0
  35. data/lib/pwrake/option/option_filesystem.rb +112 -0
  36. data/lib/pwrake/queue/locality_aware_queue.rb +158 -0
  37. data/lib/pwrake/queue/no_action_queue.rb +67 -0
  38. data/lib/pwrake/queue/queue_array.rb +366 -0
  39. data/lib/pwrake/queue/task_queue.rb +164 -0
  40. data/lib/pwrake/report.rb +1 -0
  41. data/lib/pwrake/report/parallelism.rb +9 -3
  42. data/lib/pwrake/report/report.rb +50 -103
  43. data/lib/pwrake/report/task_stat.rb +83 -0
  44. data/lib/pwrake/task/task_algorithm.rb +107 -0
  45. data/lib/pwrake/task/task_manager.rb +32 -0
  46. data/lib/pwrake/task/task_property.rb +98 -0
  47. data/lib/pwrake/task/task_rank.rb +48 -0
  48. data/lib/pwrake/task/task_wrapper.rb +296 -0
  49. data/lib/pwrake/version.rb +1 -1
  50. data/lib/pwrake/worker/executor.rb +169 -0
  51. data/lib/pwrake/worker/gfarm_directory.rb +90 -0
  52. data/lib/pwrake/worker/invoker.rb +199 -0
  53. data/lib/pwrake/worker/load.rb +14 -0
  54. data/lib/pwrake/worker/log_executor.rb +73 -0
  55. data/lib/pwrake/worker/shared_directory.rb +74 -0
  56. data/lib/pwrake/worker/worker_main.rb +14 -0
  57. data/lib/pwrake/worker/writer.rb +59 -0
  58. data/setup.rb +1212 -1502
  59. data/spec/003/Rakefile +2 -2
  60. data/spec/008/Rakefile +2 -1
  61. data/spec/009/Rakefile +1 -1
  62. data/spec/009/pwrake_conf.yaml +1 -3
  63. data/spec/hosts +0 -2
  64. data/spec/pwrake_spec.rb +9 -8
  65. metadata +50 -21
  66. data/lib/pwrake.rb +0 -19
  67. data/lib/pwrake/application.rb +0 -232
  68. data/lib/pwrake/counter.rb +0 -54
  69. data/lib/pwrake/file_utils.rb +0 -98
  70. data/lib/pwrake/gfwhere_pool.rb +0 -109
  71. data/lib/pwrake/host_list.rb +0 -88
  72. data/lib/pwrake/locality_aware_queue.rb +0 -413
  73. data/lib/pwrake/option.rb +0 -400
  74. data/lib/pwrake/rake_modify.rb +0 -14
  75. data/lib/pwrake/shell.rb +0 -186
  76. data/lib/pwrake/task_algorithm.rb +0 -475
  77. data/lib/pwrake/task_queue.rb +0 -633
  78. data/lib/pwrake/timer.rb +0 -22
@@ -1,6 +1,6 @@
1
1
  module Pwrake
2
2
 
3
- class Profiler
3
+ class ShellProfiler
4
4
 
5
5
  HEADER_FOR_PROFILE =
6
6
  %w[exec_id task_id task_name command
@@ -13,31 +13,27 @@ module Pwrake
13
13
 
14
14
  def initialize
15
15
  @lock = Mutex.new
16
- @separator = ","
17
- @re_escape = /\s#{Regexp.escape(@separator)}/
18
16
  @gnu_time = false
19
17
  @id = 0
20
18
  @io = nil
21
19
  end
22
20
 
23
- attr_accessor :separator, :gnu_time
24
-
25
21
  def open(file,gnu_time=false,plot=false)
26
22
  @file = file
27
23
  @gnu_time = gnu_time
28
24
  @plot = plot
29
25
  @lock.synchronize do
30
26
  @io.close if @io != nil
31
- @io = File.open(file,"w")
27
+ @io = CSV.open(file,"w")
32
28
  end
33
29
  _puts table_header
34
30
  t = Time.now
35
- profile(nil,'pwrake_profile_start',t,t)
31
+ profile(nil,nil,'pwrake_profile_start',t,t)
36
32
  end
37
33
 
38
34
  def close
39
35
  t = Time.now
40
- profile(nil,'pwrake_profile_end',t,t)
36
+ profile(nil,nil,'pwrake_profile_end',t,t)
41
37
  @lock.synchronize do
42
38
  @io.close if @io != nil
43
39
  @io = nil
@@ -59,9 +55,10 @@ module Pwrake
59
55
  if @gnu_time
60
56
  a += HEADER_FOR_GNU_TIME
61
57
  end
62
- a.join(@separator)
58
+ a
63
59
  end
64
60
 
61
+ =begin
65
62
  def command(cmd,terminator)
66
63
  if @gnu_time
67
64
  if /\*|\?|\{|\}|\[|\]|<|>|\(|\)|\~|\&|\||\\|\$|;|`|\n/ =~ cmd
@@ -75,6 +72,7 @@ module Pwrake
75
72
  "#{cmd}\necho '#{terminator}':$? "
76
73
  end
77
74
  end #`
75
+ =end
78
76
 
79
77
  def format_time(t)
80
78
  #t.utc.strftime("%F %T.%L")
@@ -85,35 +83,38 @@ module Pwrake
85
83
  t.strftime("%F %T.%L")
86
84
  end
87
85
 
88
- def profile(task, cmd, start_time, end_time, host="", status=nil)
89
- status = "" if status.nil?
86
+ def profile(task_id, task_name, cmd, start_time, end_time, host=nil, status=nil)
90
87
  id = @lock.synchronize do
91
88
  id = @id
92
89
  @id += 1
93
90
  id
94
91
  end
95
92
  if @io
96
- if task.kind_of? Rake::Task
97
- tname = task.name.inspect
98
- task_id = task.task_id
99
- else
100
- tname = ""
101
- task_id = ""
102
- end
103
- host = '"'+host+'"' if @re_escape =~ host
104
- _puts [id, task_id, tname, cmd.inspect,
93
+ _puts [ id, task_id, task_name, cmd,
105
94
  format_time(start_time),
106
95
  format_time(end_time),
107
96
  "%.3f" % (end_time-start_time),
108
- host, status].join(@separator)
97
+ host, status ]
109
98
  end
110
- if status==""
99
+ case status
100
+ when ""
111
101
  1
112
- elsif @gnu_time
113
- /^([^,]*),/ =~ status
114
- Integer($1)
115
- else
116
- Integer(status)
102
+ when Integer
103
+ status
104
+ when String
105
+ if @gnu_time
106
+ if /^([^,]*),/ =~ status
107
+ Integer($1)
108
+ else
109
+ status
110
+ end
111
+ else
112
+ if /^\d+$/ =~ status
113
+ Integer(status)
114
+ else
115
+ status
116
+ end
117
+ end
117
118
  end
118
119
  end
119
120
 
@@ -0,0 +1,104 @@
1
+ module Pwrake
2
+
3
+ class WorkerCommunicator
4
+
5
+ HOST2ID = {}
6
+ RE_ID='\d+'
7
+ attr_reader :id, :host, :ncore, :handler, :channel
8
+
9
+ def self.read_worker_progs(option)
10
+ d = File.dirname(__FILE__)+'/../worker/'
11
+ code = ""
12
+ option.worker_progs.each do |f|
13
+ code << IO.read(d+f+'.rb')
14
+ end
15
+ code
16
+ end
17
+
18
+ def initialize(id,host,ncore,runner,option)
19
+ @id = id
20
+ @host = host
21
+ HOST2ID[@host] = @id
22
+ @ncore = @n_total_core = ncore
23
+ #
24
+ @runner = runner
25
+ @worker_progs = option.worker_progs
26
+ @option = option.worker_option
27
+ if hb = @option[:heartbeat]
28
+ @heartbeat_timeout = hb + 15
29
+ end
30
+ end
31
+
32
+ def setup_connection(worker_code)
33
+ rb_cmd = "ruby -e 'eval ARGF.read(#{worker_code.size})'"
34
+ if ['localhost','localhost.localdomain','127.0.0.1'].include? @host
35
+ cmd = "cd; #{rb_cmd}"
36
+ else
37
+ cmd = "ssh -x -T -q #{@option[:ssh_option]} #{@host} \"#{rb_cmd}\""
38
+ end
39
+ Log.debug cmd
40
+ @handler = Handler.new(@runner) do |w0,w1,r2|
41
+ @pid = spawn(cmd,:pgroup=>true,:out=>w0,:err=>w1,:in=>r2)
42
+ w0.close
43
+ w1.close
44
+ r2.close
45
+ end
46
+ @handler.host = @host
47
+ iow = @handler.iow
48
+ iow.write(worker_code)
49
+ Marshal.dump(@ncore,iow)
50
+ Marshal.dump(@option,iow)
51
+ @channel = Channel.new(@handler)
52
+ end
53
+
54
+ def close
55
+ if !@closed
56
+ @closed = true
57
+ @handler.close
58
+ end
59
+ end
60
+
61
+ def set_ncore(ncore)
62
+ @ncore = ncore
63
+ end
64
+
65
+ def ncore_proc(s)
66
+ if /^ncore:(.*)$/ =~ s
67
+ a = $1
68
+ Log.debug "#{a} @#{@host}"
69
+ if /^(\d+)$/ =~ a
70
+ set_ncore($1.to_i)
71
+ return false
72
+ else
73
+ raise ArgumentError,"WorkerCommunicator#ncore_proc: s=#{s.inspect}"
74
+ end
75
+ else
76
+ return common_line(s)
77
+ end
78
+ end
79
+
80
+ def common_line(s)
81
+ Log.debug "WorkerCommunicator#common_line(#{s.inspect}) id=#{@id} host=#{@host}"
82
+ case s
83
+ when /^heartbeat$/
84
+ @runner.heartbeat(@handler.ior)
85
+ when /^exited$/
86
+ return false
87
+ when /^log:(.*)$/
88
+ Log.info "worker(#{host})>#{$1}"
89
+ else
90
+ Log.warn "worker(#{host}) out> #{s.inspect}"
91
+ end
92
+ true
93
+ end
94
+
95
+ def start_default_fiber
96
+ Fiber.new do
97
+ while common_line(@channel.get_line)
98
+ end
99
+ Log.debug "WorkerCommunicator: end of default fiber"
100
+ end.resume
101
+ end
102
+
103
+ end
104
+ end
@@ -1,5 +1,3 @@
1
- require 'pwrake/gfwhere_pool'
2
-
3
1
  module Pwrake
4
2
 
5
3
  module GfarmPath
@@ -101,6 +99,7 @@ module Pwrake
101
99
  nil
102
100
  end
103
101
 
102
+ =begin
104
103
  def gfwhere(list)
105
104
  system "sync"
106
105
  result = {}
@@ -137,104 +136,7 @@ module Pwrake
137
136
  end
138
137
  result
139
138
  end
140
-
141
- end
142
-
143
-
144
- class GfarmShell < Shell
145
-
146
- @@core_id = {}
147
- @@prefix = "pwrake_#{ENV['USER']}"
148
-
149
- def initialize(host,opt={})
150
- super(host,opt)
151
- @single_mp = @option[:single_mp]
152
- @basedir = @option[:basedir]
153
- @prefix = @option[:prefix] || @@prefix
154
- @work_dir = @option[:work_dir]
155
-
156
- @core_id = @@core_id[host] || 0
157
- @@core_id[host] = @core_id + 1
158
-
159
- if @single_mp
160
- @remote_mountpoint = "#{@basedir}/#{@prefix}_00"
161
- else
162
- @remote_mountpoint = "#{@basedir}/#{@prefix}_%02d" % @core_id
163
- end
164
- end
165
-
166
- def start
167
- Log.debug "--- mountpoint=#{@remote_mountpoint}"
168
- open(system_cmd)
169
- cd
170
- if not _system "test -d #{@remote_mountpoint}"
171
- _system "mkdir -p #{@remote_mountpoint}" or die
172
- else
173
- lines = _backquote("sync; mount")
174
- if /#{@remote_mountpoint} (?:type )?(\S+)/om =~ lines
175
- _system "sync; fusermount -u #{@remote_mountpoint}"
176
- _system "sync"
177
- end
178
- end
179
- subdir = GfarmPath.subdir
180
- if ["/","",nil].include?(subdir)
181
- _system "gfarm2fs #{@remote_mountpoint}"
182
- else
183
- _system "gfarm2fs -o modules=subdir,subdir=#{subdir} #{@remote_mountpoint}"
184
- end
185
- path = ENV['PATH'].gsub( /#{GfarmPath.mountpoint}/, @remote_mountpoint )
186
- _system "export PATH=#{path}" or die
187
- cd_work_dir
188
- end
189
-
190
- def close
191
- if @remote_mountpoint
192
- cd
193
- _system "fusermount -u #{@remote_mountpoint}"
194
- _system "rmdir #{@remote_mountpoint}"
195
- end
196
- super
197
- self
198
- end
199
-
200
- def cd_work_dir
201
- # modify local work_dir -> remote work_dir
202
- dir = Pathname.new(@remote_mountpoint) + GfarmPath.pwd
203
- cd dir
204
- end
205
-
206
- end
207
-
208
-
209
- class GfarmPostprocess
210
-
211
- def initialize
212
- max = Pwrake.application.pwrake_options['MAX_GFWHERE_WORKER']
213
- @gfwhere_pool = WorkerPool.new(GfwhereWorker,max)
214
- end
215
-
216
- def postprocess(t)
217
- if t.kind_of?(Rake::FileTask) && t.location.empty?
218
- t.location = @gfwhere_pool.work(t.name)
219
- end
220
- end
221
-
222
- def postprocess_bulk(tasks)
223
- list = []
224
- tasks.each do |t|
225
- list << t.name if t.kind_of? Rake::FileTask
226
- end
227
- if !list.empty?
228
- Log.info "-- after_check: size=#{list.size} #{list.inspect}"
229
- gfwhere_result = GfarmPath.gfwhere(list)
230
- tasks.each do |t|
231
- if t.kind_of? Rake::FileTask
232
- t.location = gfwhere_result[GfarmPath.local_to_fs(t.name)]
233
- end
234
- end
235
- #puts "'#{self.name}' exist? => #{File.exist?(self.name)} loc => #{loc}"
236
- end
237
- end
139
+ =end
238
140
 
239
141
  end
240
142
  end
@@ -0,0 +1,53 @@
1
+ module Pwrake
2
+
3
+ class GfarmPostprocess
4
+
5
+ def initialize(runner)
6
+ @io = IO.popen('gfwhere-pipe','r+')
7
+ @io.sync = true
8
+ @hdl = Handler.new(runner,@io,@io)
9
+ @chan = Channel.new(@hdl)
10
+ end
11
+
12
+ def run(task_wrap)
13
+ if !task_wrap.is_file_task?
14
+ return []
15
+ end
16
+ filename = task_wrap.name
17
+ begin
18
+ @hdl.iow.puts(filename)
19
+ @hdl.iow.flush
20
+ rescue Errno::EPIPE
21
+ Log.warn "GfarmPostprocess#run: Errno::EPIPE for #{filename}"
22
+ return []
23
+ end
24
+ s = @chan.get_line
25
+ if s.nil?
26
+ raise "gfwhere: unexpected end"
27
+ end
28
+ s.chomp!
29
+ if s != filename
30
+ raise "gfwhere: file=#{filename}, result=#{s}"
31
+ end
32
+ while s = @chan.get_line
33
+ s.chomp!
34
+ case s
35
+ when /^gfarm:\/\//
36
+ next
37
+ when /^Error:/
38
+ a = []
39
+ break
40
+ else
41
+ a = s.split(/\s+/)
42
+ break
43
+ end
44
+ end
45
+ Log.debug "Gfarm file=#{filename} nodes=#{a.join("|")}"
46
+ a
47
+ end
48
+
49
+ def close
50
+ @io.close
51
+ end
52
+ end
53
+ end
@@ -0,0 +1,70 @@
1
+ module Pwrake
2
+
3
+ class Channel
4
+
5
+ def initialize(handler,id=nil)
6
+ if @id = id
7
+ @pre = "#{@id}:"
8
+ else
9
+ @pre = ""
10
+ end
11
+ if !handler.kind_of?(Handler)
12
+ raise TypeError, "Argument must be Handler but #{handler.class}"
13
+ end
14
+ @handler = handler
15
+ end
16
+
17
+ attr_reader :id, :handler, :fiber
18
+
19
+ def ior
20
+ @handler.ior
21
+ end
22
+
23
+ def run_fiber(*args)
24
+ if @fiber.nil?
25
+ m = "Channel#run_fiber: @fiber is nil,"+
26
+ " args=#{args.inspect} @id=#{@id}"
27
+ $stderr.puts m
28
+ Log.debug m
29
+ else
30
+ @fiber.resume(*args)
31
+ end
32
+ end
33
+
34
+ def finish
35
+ if !@fiber.nil?
36
+ @fiber.resume(nil)
37
+ end
38
+ end
39
+
40
+ def get_line
41
+ @handler.add_channel(self)
42
+ @fiber = Fiber.current
43
+ line = Fiber.yield
44
+ @fiber = nil
45
+ @handler.delete_channel(self)
46
+ return line
47
+ end
48
+
49
+ def put_line(line)
50
+ @handler.put_line "#{@pre}#{line}"
51
+ end
52
+
53
+ def puts(line)
54
+ @handler.puts "#{@pre}#{line}"
55
+ end
56
+
57
+ def gets
58
+ if @id
59
+ raise RuntimeError,"gets is invalid when @id is non-nil"
60
+ else
61
+ @handler.gets
62
+ end
63
+ end
64
+
65
+ def inspect
66
+ "#<#{self.class} io=#{ior.inspect} id=#{id.inspect}>"
67
+ end
68
+ end
69
+
70
+ end