pwrake 0.9.9.2 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +2 -0
- data/CHANGES_V2.md +90 -0
- data/{LICENSE.txt → MIT-LICENSE} +2 -3
- data/README +12 -0
- data/README.md +75 -52
- data/bin/gfwhere-pipe +23 -12
- data/bin/pwrake +22 -29
- data/bin/pwrake_branch +24 -0
- data/lib/pwrake/branch.rb +22 -0
- data/lib/pwrake/branch/branch.rb +213 -0
- data/lib/pwrake/branch/branch_application.rb +53 -0
- data/lib/pwrake/branch/fiber_queue.rb +36 -0
- data/lib/pwrake/branch/file_utils.rb +101 -0
- data/lib/pwrake/branch/shell.rb +231 -0
- data/lib/pwrake/{profiler.rb → branch/shell_profiler.rb} +28 -27
- data/lib/pwrake/branch/worker_communicator.rb +104 -0
- data/lib/pwrake/{gfarm_feature.rb → gfarm/gfarm_path.rb} +2 -100
- data/lib/pwrake/gfarm/gfarm_postprocess.rb +53 -0
- data/lib/pwrake/iomux/channel.rb +70 -0
- data/lib/pwrake/iomux/handler.rb +124 -0
- data/lib/pwrake/iomux/handler_set.rb +35 -0
- data/lib/pwrake/iomux/runner.rb +62 -0
- data/lib/pwrake/logger.rb +3 -150
- data/lib/pwrake/master.rb +30 -137
- data/lib/pwrake/master/fiber_pool.rb +69 -0
- data/lib/pwrake/master/idle_cores.rb +30 -0
- data/lib/pwrake/master/master.rb +345 -0
- data/lib/pwrake/master/master_application.rb +150 -0
- data/lib/pwrake/master/postprocess.rb +16 -0
- data/lib/pwrake/{graphviz.rb → misc/graphviz.rb} +0 -0
- data/lib/pwrake/{mcgp.rb → misc/mcgp.rb} +63 -42
- data/lib/pwrake/option/host_map.rb +158 -0
- data/lib/pwrake/option/option.rb +357 -0
- data/lib/pwrake/option/option_filesystem.rb +112 -0
- data/lib/pwrake/queue/locality_aware_queue.rb +158 -0
- data/lib/pwrake/queue/no_action_queue.rb +67 -0
- data/lib/pwrake/queue/queue_array.rb +366 -0
- data/lib/pwrake/queue/task_queue.rb +164 -0
- data/lib/pwrake/report.rb +1 -0
- data/lib/pwrake/report/parallelism.rb +9 -3
- data/lib/pwrake/report/report.rb +50 -103
- data/lib/pwrake/report/task_stat.rb +83 -0
- data/lib/pwrake/task/task_algorithm.rb +107 -0
- data/lib/pwrake/task/task_manager.rb +32 -0
- data/lib/pwrake/task/task_property.rb +98 -0
- data/lib/pwrake/task/task_rank.rb +48 -0
- data/lib/pwrake/task/task_wrapper.rb +296 -0
- data/lib/pwrake/version.rb +1 -1
- data/lib/pwrake/worker/executor.rb +169 -0
- data/lib/pwrake/worker/gfarm_directory.rb +90 -0
- data/lib/pwrake/worker/invoker.rb +199 -0
- data/lib/pwrake/worker/load.rb +14 -0
- data/lib/pwrake/worker/log_executor.rb +73 -0
- data/lib/pwrake/worker/shared_directory.rb +74 -0
- data/lib/pwrake/worker/worker_main.rb +14 -0
- data/lib/pwrake/worker/writer.rb +59 -0
- data/setup.rb +1212 -1502
- data/spec/003/Rakefile +2 -2
- data/spec/008/Rakefile +2 -1
- data/spec/009/Rakefile +1 -1
- data/spec/009/pwrake_conf.yaml +1 -3
- data/spec/hosts +0 -2
- data/spec/pwrake_spec.rb +9 -8
- metadata +50 -21
- data/lib/pwrake.rb +0 -19
- data/lib/pwrake/application.rb +0 -232
- data/lib/pwrake/counter.rb +0 -54
- data/lib/pwrake/file_utils.rb +0 -98
- data/lib/pwrake/gfwhere_pool.rb +0 -109
- data/lib/pwrake/host_list.rb +0 -88
- data/lib/pwrake/locality_aware_queue.rb +0 -413
- data/lib/pwrake/option.rb +0 -400
- data/lib/pwrake/rake_modify.rb +0 -14
- data/lib/pwrake/shell.rb +0 -186
- data/lib/pwrake/task_algorithm.rb +0 -475
- data/lib/pwrake/task_queue.rb +0 -633
- data/lib/pwrake/timer.rb +0 -22
@@ -1,6 +1,6 @@
|
|
1
1
|
module Pwrake
|
2
2
|
|
3
|
-
class
|
3
|
+
class ShellProfiler
|
4
4
|
|
5
5
|
HEADER_FOR_PROFILE =
|
6
6
|
%w[exec_id task_id task_name command
|
@@ -13,31 +13,27 @@ module Pwrake
|
|
13
13
|
|
14
14
|
def initialize
|
15
15
|
@lock = Mutex.new
|
16
|
-
@separator = ","
|
17
|
-
@re_escape = /\s#{Regexp.escape(@separator)}/
|
18
16
|
@gnu_time = false
|
19
17
|
@id = 0
|
20
18
|
@io = nil
|
21
19
|
end
|
22
20
|
|
23
|
-
attr_accessor :separator, :gnu_time
|
24
|
-
|
25
21
|
def open(file,gnu_time=false,plot=false)
|
26
22
|
@file = file
|
27
23
|
@gnu_time = gnu_time
|
28
24
|
@plot = plot
|
29
25
|
@lock.synchronize do
|
30
26
|
@io.close if @io != nil
|
31
|
-
@io =
|
27
|
+
@io = CSV.open(file,"w")
|
32
28
|
end
|
33
29
|
_puts table_header
|
34
30
|
t = Time.now
|
35
|
-
profile(nil,'pwrake_profile_start',t,t)
|
31
|
+
profile(nil,nil,'pwrake_profile_start',t,t)
|
36
32
|
end
|
37
33
|
|
38
34
|
def close
|
39
35
|
t = Time.now
|
40
|
-
profile(nil,'pwrake_profile_end',t,t)
|
36
|
+
profile(nil,nil,'pwrake_profile_end',t,t)
|
41
37
|
@lock.synchronize do
|
42
38
|
@io.close if @io != nil
|
43
39
|
@io = nil
|
@@ -59,9 +55,10 @@ module Pwrake
|
|
59
55
|
if @gnu_time
|
60
56
|
a += HEADER_FOR_GNU_TIME
|
61
57
|
end
|
62
|
-
a
|
58
|
+
a
|
63
59
|
end
|
64
60
|
|
61
|
+
=begin
|
65
62
|
def command(cmd,terminator)
|
66
63
|
if @gnu_time
|
67
64
|
if /\*|\?|\{|\}|\[|\]|<|>|\(|\)|\~|\&|\||\\|\$|;|`|\n/ =~ cmd
|
@@ -75,6 +72,7 @@ module Pwrake
|
|
75
72
|
"#{cmd}\necho '#{terminator}':$? "
|
76
73
|
end
|
77
74
|
end #`
|
75
|
+
=end
|
78
76
|
|
79
77
|
def format_time(t)
|
80
78
|
#t.utc.strftime("%F %T.%L")
|
@@ -85,35 +83,38 @@ module Pwrake
|
|
85
83
|
t.strftime("%F %T.%L")
|
86
84
|
end
|
87
85
|
|
88
|
-
def profile(
|
89
|
-
status = "" if status.nil?
|
86
|
+
def profile(task_id, task_name, cmd, start_time, end_time, host=nil, status=nil)
|
90
87
|
id = @lock.synchronize do
|
91
88
|
id = @id
|
92
89
|
@id += 1
|
93
90
|
id
|
94
91
|
end
|
95
92
|
if @io
|
96
|
-
|
97
|
-
tname = task.name.inspect
|
98
|
-
task_id = task.task_id
|
99
|
-
else
|
100
|
-
tname = ""
|
101
|
-
task_id = ""
|
102
|
-
end
|
103
|
-
host = '"'+host+'"' if @re_escape =~ host
|
104
|
-
_puts [id, task_id, tname, cmd.inspect,
|
93
|
+
_puts [ id, task_id, task_name, cmd,
|
105
94
|
format_time(start_time),
|
106
95
|
format_time(end_time),
|
107
96
|
"%.3f" % (end_time-start_time),
|
108
|
-
host, status]
|
97
|
+
host, status ]
|
109
98
|
end
|
110
|
-
|
99
|
+
case status
|
100
|
+
when ""
|
111
101
|
1
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
102
|
+
when Integer
|
103
|
+
status
|
104
|
+
when String
|
105
|
+
if @gnu_time
|
106
|
+
if /^([^,]*),/ =~ status
|
107
|
+
Integer($1)
|
108
|
+
else
|
109
|
+
status
|
110
|
+
end
|
111
|
+
else
|
112
|
+
if /^\d+$/ =~ status
|
113
|
+
Integer(status)
|
114
|
+
else
|
115
|
+
status
|
116
|
+
end
|
117
|
+
end
|
117
118
|
end
|
118
119
|
end
|
119
120
|
|
@@ -0,0 +1,104 @@
|
|
1
|
+
module Pwrake
|
2
|
+
|
3
|
+
class WorkerCommunicator
|
4
|
+
|
5
|
+
HOST2ID = {}
|
6
|
+
RE_ID='\d+'
|
7
|
+
attr_reader :id, :host, :ncore, :handler, :channel
|
8
|
+
|
9
|
+
def self.read_worker_progs(option)
|
10
|
+
d = File.dirname(__FILE__)+'/../worker/'
|
11
|
+
code = ""
|
12
|
+
option.worker_progs.each do |f|
|
13
|
+
code << IO.read(d+f+'.rb')
|
14
|
+
end
|
15
|
+
code
|
16
|
+
end
|
17
|
+
|
18
|
+
def initialize(id,host,ncore,runner,option)
|
19
|
+
@id = id
|
20
|
+
@host = host
|
21
|
+
HOST2ID[@host] = @id
|
22
|
+
@ncore = @n_total_core = ncore
|
23
|
+
#
|
24
|
+
@runner = runner
|
25
|
+
@worker_progs = option.worker_progs
|
26
|
+
@option = option.worker_option
|
27
|
+
if hb = @option[:heartbeat]
|
28
|
+
@heartbeat_timeout = hb + 15
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
def setup_connection(worker_code)
|
33
|
+
rb_cmd = "ruby -e 'eval ARGF.read(#{worker_code.size})'"
|
34
|
+
if ['localhost','localhost.localdomain','127.0.0.1'].include? @host
|
35
|
+
cmd = "cd; #{rb_cmd}"
|
36
|
+
else
|
37
|
+
cmd = "ssh -x -T -q #{@option[:ssh_option]} #{@host} \"#{rb_cmd}\""
|
38
|
+
end
|
39
|
+
Log.debug cmd
|
40
|
+
@handler = Handler.new(@runner) do |w0,w1,r2|
|
41
|
+
@pid = spawn(cmd,:pgroup=>true,:out=>w0,:err=>w1,:in=>r2)
|
42
|
+
w0.close
|
43
|
+
w1.close
|
44
|
+
r2.close
|
45
|
+
end
|
46
|
+
@handler.host = @host
|
47
|
+
iow = @handler.iow
|
48
|
+
iow.write(worker_code)
|
49
|
+
Marshal.dump(@ncore,iow)
|
50
|
+
Marshal.dump(@option,iow)
|
51
|
+
@channel = Channel.new(@handler)
|
52
|
+
end
|
53
|
+
|
54
|
+
def close
|
55
|
+
if !@closed
|
56
|
+
@closed = true
|
57
|
+
@handler.close
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
def set_ncore(ncore)
|
62
|
+
@ncore = ncore
|
63
|
+
end
|
64
|
+
|
65
|
+
def ncore_proc(s)
|
66
|
+
if /^ncore:(.*)$/ =~ s
|
67
|
+
a = $1
|
68
|
+
Log.debug "#{a} @#{@host}"
|
69
|
+
if /^(\d+)$/ =~ a
|
70
|
+
set_ncore($1.to_i)
|
71
|
+
return false
|
72
|
+
else
|
73
|
+
raise ArgumentError,"WorkerCommunicator#ncore_proc: s=#{s.inspect}"
|
74
|
+
end
|
75
|
+
else
|
76
|
+
return common_line(s)
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
def common_line(s)
|
81
|
+
Log.debug "WorkerCommunicator#common_line(#{s.inspect}) id=#{@id} host=#{@host}"
|
82
|
+
case s
|
83
|
+
when /^heartbeat$/
|
84
|
+
@runner.heartbeat(@handler.ior)
|
85
|
+
when /^exited$/
|
86
|
+
return false
|
87
|
+
when /^log:(.*)$/
|
88
|
+
Log.info "worker(#{host})>#{$1}"
|
89
|
+
else
|
90
|
+
Log.warn "worker(#{host}) out> #{s.inspect}"
|
91
|
+
end
|
92
|
+
true
|
93
|
+
end
|
94
|
+
|
95
|
+
def start_default_fiber
|
96
|
+
Fiber.new do
|
97
|
+
while common_line(@channel.get_line)
|
98
|
+
end
|
99
|
+
Log.debug "WorkerCommunicator: end of default fiber"
|
100
|
+
end.resume
|
101
|
+
end
|
102
|
+
|
103
|
+
end
|
104
|
+
end
|
@@ -1,5 +1,3 @@
|
|
1
|
-
require 'pwrake/gfwhere_pool'
|
2
|
-
|
3
1
|
module Pwrake
|
4
2
|
|
5
3
|
module GfarmPath
|
@@ -101,6 +99,7 @@ module Pwrake
|
|
101
99
|
nil
|
102
100
|
end
|
103
101
|
|
102
|
+
=begin
|
104
103
|
def gfwhere(list)
|
105
104
|
system "sync"
|
106
105
|
result = {}
|
@@ -137,104 +136,7 @@ module Pwrake
|
|
137
136
|
end
|
138
137
|
result
|
139
138
|
end
|
140
|
-
|
141
|
-
end
|
142
|
-
|
143
|
-
|
144
|
-
class GfarmShell < Shell
|
145
|
-
|
146
|
-
@@core_id = {}
|
147
|
-
@@prefix = "pwrake_#{ENV['USER']}"
|
148
|
-
|
149
|
-
def initialize(host,opt={})
|
150
|
-
super(host,opt)
|
151
|
-
@single_mp = @option[:single_mp]
|
152
|
-
@basedir = @option[:basedir]
|
153
|
-
@prefix = @option[:prefix] || @@prefix
|
154
|
-
@work_dir = @option[:work_dir]
|
155
|
-
|
156
|
-
@core_id = @@core_id[host] || 0
|
157
|
-
@@core_id[host] = @core_id + 1
|
158
|
-
|
159
|
-
if @single_mp
|
160
|
-
@remote_mountpoint = "#{@basedir}/#{@prefix}_00"
|
161
|
-
else
|
162
|
-
@remote_mountpoint = "#{@basedir}/#{@prefix}_%02d" % @core_id
|
163
|
-
end
|
164
|
-
end
|
165
|
-
|
166
|
-
def start
|
167
|
-
Log.debug "--- mountpoint=#{@remote_mountpoint}"
|
168
|
-
open(system_cmd)
|
169
|
-
cd
|
170
|
-
if not _system "test -d #{@remote_mountpoint}"
|
171
|
-
_system "mkdir -p #{@remote_mountpoint}" or die
|
172
|
-
else
|
173
|
-
lines = _backquote("sync; mount")
|
174
|
-
if /#{@remote_mountpoint} (?:type )?(\S+)/om =~ lines
|
175
|
-
_system "sync; fusermount -u #{@remote_mountpoint}"
|
176
|
-
_system "sync"
|
177
|
-
end
|
178
|
-
end
|
179
|
-
subdir = GfarmPath.subdir
|
180
|
-
if ["/","",nil].include?(subdir)
|
181
|
-
_system "gfarm2fs #{@remote_mountpoint}"
|
182
|
-
else
|
183
|
-
_system "gfarm2fs -o modules=subdir,subdir=#{subdir} #{@remote_mountpoint}"
|
184
|
-
end
|
185
|
-
path = ENV['PATH'].gsub( /#{GfarmPath.mountpoint}/, @remote_mountpoint )
|
186
|
-
_system "export PATH=#{path}" or die
|
187
|
-
cd_work_dir
|
188
|
-
end
|
189
|
-
|
190
|
-
def close
|
191
|
-
if @remote_mountpoint
|
192
|
-
cd
|
193
|
-
_system "fusermount -u #{@remote_mountpoint}"
|
194
|
-
_system "rmdir #{@remote_mountpoint}"
|
195
|
-
end
|
196
|
-
super
|
197
|
-
self
|
198
|
-
end
|
199
|
-
|
200
|
-
def cd_work_dir
|
201
|
-
# modify local work_dir -> remote work_dir
|
202
|
-
dir = Pathname.new(@remote_mountpoint) + GfarmPath.pwd
|
203
|
-
cd dir
|
204
|
-
end
|
205
|
-
|
206
|
-
end
|
207
|
-
|
208
|
-
|
209
|
-
class GfarmPostprocess
|
210
|
-
|
211
|
-
def initialize
|
212
|
-
max = Pwrake.application.pwrake_options['MAX_GFWHERE_WORKER']
|
213
|
-
@gfwhere_pool = WorkerPool.new(GfwhereWorker,max)
|
214
|
-
end
|
215
|
-
|
216
|
-
def postprocess(t)
|
217
|
-
if t.kind_of?(Rake::FileTask) && t.location.empty?
|
218
|
-
t.location = @gfwhere_pool.work(t.name)
|
219
|
-
end
|
220
|
-
end
|
221
|
-
|
222
|
-
def postprocess_bulk(tasks)
|
223
|
-
list = []
|
224
|
-
tasks.each do |t|
|
225
|
-
list << t.name if t.kind_of? Rake::FileTask
|
226
|
-
end
|
227
|
-
if !list.empty?
|
228
|
-
Log.info "-- after_check: size=#{list.size} #{list.inspect}"
|
229
|
-
gfwhere_result = GfarmPath.gfwhere(list)
|
230
|
-
tasks.each do |t|
|
231
|
-
if t.kind_of? Rake::FileTask
|
232
|
-
t.location = gfwhere_result[GfarmPath.local_to_fs(t.name)]
|
233
|
-
end
|
234
|
-
end
|
235
|
-
#puts "'#{self.name}' exist? => #{File.exist?(self.name)} loc => #{loc}"
|
236
|
-
end
|
237
|
-
end
|
139
|
+
=end
|
238
140
|
|
239
141
|
end
|
240
142
|
end
|
@@ -0,0 +1,53 @@
|
|
1
|
+
module Pwrake
|
2
|
+
|
3
|
+
class GfarmPostprocess
|
4
|
+
|
5
|
+
def initialize(runner)
|
6
|
+
@io = IO.popen('gfwhere-pipe','r+')
|
7
|
+
@io.sync = true
|
8
|
+
@hdl = Handler.new(runner,@io,@io)
|
9
|
+
@chan = Channel.new(@hdl)
|
10
|
+
end
|
11
|
+
|
12
|
+
def run(task_wrap)
|
13
|
+
if !task_wrap.is_file_task?
|
14
|
+
return []
|
15
|
+
end
|
16
|
+
filename = task_wrap.name
|
17
|
+
begin
|
18
|
+
@hdl.iow.puts(filename)
|
19
|
+
@hdl.iow.flush
|
20
|
+
rescue Errno::EPIPE
|
21
|
+
Log.warn "GfarmPostprocess#run: Errno::EPIPE for #{filename}"
|
22
|
+
return []
|
23
|
+
end
|
24
|
+
s = @chan.get_line
|
25
|
+
if s.nil?
|
26
|
+
raise "gfwhere: unexpected end"
|
27
|
+
end
|
28
|
+
s.chomp!
|
29
|
+
if s != filename
|
30
|
+
raise "gfwhere: file=#{filename}, result=#{s}"
|
31
|
+
end
|
32
|
+
while s = @chan.get_line
|
33
|
+
s.chomp!
|
34
|
+
case s
|
35
|
+
when /^gfarm:\/\//
|
36
|
+
next
|
37
|
+
when /^Error:/
|
38
|
+
a = []
|
39
|
+
break
|
40
|
+
else
|
41
|
+
a = s.split(/\s+/)
|
42
|
+
break
|
43
|
+
end
|
44
|
+
end
|
45
|
+
Log.debug "Gfarm file=#{filename} nodes=#{a.join("|")}"
|
46
|
+
a
|
47
|
+
end
|
48
|
+
|
49
|
+
def close
|
50
|
+
@io.close
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
@@ -0,0 +1,70 @@
|
|
1
|
+
module Pwrake
|
2
|
+
|
3
|
+
class Channel
|
4
|
+
|
5
|
+
def initialize(handler,id=nil)
|
6
|
+
if @id = id
|
7
|
+
@pre = "#{@id}:"
|
8
|
+
else
|
9
|
+
@pre = ""
|
10
|
+
end
|
11
|
+
if !handler.kind_of?(Handler)
|
12
|
+
raise TypeError, "Argument must be Handler but #{handler.class}"
|
13
|
+
end
|
14
|
+
@handler = handler
|
15
|
+
end
|
16
|
+
|
17
|
+
attr_reader :id, :handler, :fiber
|
18
|
+
|
19
|
+
def ior
|
20
|
+
@handler.ior
|
21
|
+
end
|
22
|
+
|
23
|
+
def run_fiber(*args)
|
24
|
+
if @fiber.nil?
|
25
|
+
m = "Channel#run_fiber: @fiber is nil,"+
|
26
|
+
" args=#{args.inspect} @id=#{@id}"
|
27
|
+
$stderr.puts m
|
28
|
+
Log.debug m
|
29
|
+
else
|
30
|
+
@fiber.resume(*args)
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
def finish
|
35
|
+
if !@fiber.nil?
|
36
|
+
@fiber.resume(nil)
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
def get_line
|
41
|
+
@handler.add_channel(self)
|
42
|
+
@fiber = Fiber.current
|
43
|
+
line = Fiber.yield
|
44
|
+
@fiber = nil
|
45
|
+
@handler.delete_channel(self)
|
46
|
+
return line
|
47
|
+
end
|
48
|
+
|
49
|
+
def put_line(line)
|
50
|
+
@handler.put_line "#{@pre}#{line}"
|
51
|
+
end
|
52
|
+
|
53
|
+
def puts(line)
|
54
|
+
@handler.puts "#{@pre}#{line}"
|
55
|
+
end
|
56
|
+
|
57
|
+
def gets
|
58
|
+
if @id
|
59
|
+
raise RuntimeError,"gets is invalid when @id is non-nil"
|
60
|
+
else
|
61
|
+
@handler.gets
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
def inspect
|
66
|
+
"#<#{self.class} io=#{ior.inspect} id=#{id.inspect}>"
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
end
|