pwrake 2.2.4 → 2.2.5
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/pwrake/branch/branch.rb +2 -0
- data/lib/pwrake/branch/communicator_set.rb +1 -1
- data/lib/pwrake/branch/shell.rb +6 -1
- data/lib/pwrake/branch/shell_profiler.rb +10 -6
- data/lib/pwrake/master/master.rb +4 -4
- data/lib/pwrake/nbio.rb +2 -2
- data/lib/pwrake/option/option_default_filesystem.rb +1 -0
- data/lib/pwrake/option/option_gfarm2fs.rb +1 -0
- data/lib/pwrake/version.rb +1 -1
- data/lib/pwrake/worker/executor.rb +54 -18
- data/lib/pwrake/worker/invoker.rb +2 -2
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 169f1dc9dd2e2b5dbc2b07d4b4ac11abb74cbc60
|
4
|
+
data.tar.gz: e1efd61651870235421f6a10f531daea0dc6ba83
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: dd4a51d8ae3c2c919a553668b4a0009aea6c4c0bb5b80c1f5ad849a8d569f5e600fe296abb3e19ee5f6b1f90e060b17cbe432ac41c74c587aeac3289bc8b7dd9
|
7
|
+
data.tar.gz: 39de4c6360f0d02414dcd6f23fc820f12fef64de7a7b7017c81527df33448b406f5e6d9e9f0a5710d9768ddbff9bf15fb92cbf7800663f2fc2d80a2ee430bc69
|
data/lib/pwrake/branch/branch.rb
CHANGED
@@ -178,10 +178,12 @@ module Pwrake
|
|
178
178
|
taskq = @task_q.delete(id)
|
179
179
|
Log.debug "drop @task_q[#{id}]=#{taskq.inspect}"
|
180
180
|
@cs.drop(id)
|
181
|
+
break if @cs.empty?
|
181
182
|
#
|
182
183
|
when /^kill:(.*)$/o
|
183
184
|
sig = $1
|
184
185
|
kill(sig)
|
186
|
+
break
|
185
187
|
#
|
186
188
|
else
|
187
189
|
Log.debug "Branch: invalid line from master: #{s}"
|
data/lib/pwrake/branch/shell.rb
CHANGED
@@ -151,11 +151,13 @@ module Pwrake
|
|
151
151
|
ensure
|
152
152
|
end_time = Time.now
|
153
153
|
@status = @@profiler.profile(@task_id, @task_name, cmd,
|
154
|
-
start_time, end_time, host, @ncore, @status
|
154
|
+
start_time, end_time, host, @ncore, @status,
|
155
|
+
@gnu_time_status)
|
155
156
|
end
|
156
157
|
end
|
157
158
|
|
158
159
|
def io_read_loop
|
160
|
+
@gnu_time_status = nil
|
159
161
|
while s = _gets
|
160
162
|
case s
|
161
163
|
when /^(\w+):(.*)$/
|
@@ -167,6 +169,9 @@ module Pwrake
|
|
167
169
|
when "e"
|
168
170
|
$stderr.print x[1]+"\n"
|
169
171
|
next
|
172
|
+
when "t"
|
173
|
+
@gnu_time_status = x[1].split(',')
|
174
|
+
next
|
170
175
|
when "z"
|
171
176
|
# see Executor#status_to_str
|
172
177
|
status = x[1]
|
@@ -86,18 +86,22 @@ module Pwrake
|
|
86
86
|
end
|
87
87
|
|
88
88
|
def profile(task_id, task_name, cmd, start_time, end_time,
|
89
|
-
host=nil, ncore=nil, status=nil)
|
89
|
+
host=nil, ncore=nil, status=nil, gnu_time_status=nil)
|
90
90
|
id = @lock.synchronize do
|
91
91
|
id = @id
|
92
92
|
@id += 1
|
93
93
|
id
|
94
94
|
end
|
95
95
|
if @io
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
96
|
+
prof = [ id, task_id, task_name, cmd,
|
97
|
+
format_time(start_time),
|
98
|
+
format_time(end_time),
|
99
|
+
"%.3f" % (end_time-start_time),
|
100
|
+
host, ncore, status ]
|
101
|
+
if @gnu_time && gnu_time_status
|
102
|
+
prof.concat(gnu_time_status)
|
103
|
+
end
|
104
|
+
_puts prof
|
101
105
|
end
|
102
106
|
case status
|
103
107
|
when ""
|
data/lib/pwrake/master/master.rb
CHANGED
@@ -67,12 +67,12 @@ module Pwrake
|
|
67
67
|
def signal_trap(sig)
|
68
68
|
$stderr.puts "\nSignal trapped. (sig=#{sig} pid=#{Process.pid})"
|
69
69
|
if Rake.application.options.debug
|
70
|
-
$stderr.
|
71
|
-
$stderr.puts caller
|
70
|
+
$stderr.print "in master thread #{Thread.current}:\n "
|
71
|
+
$stderr.puts caller.join("\n ")
|
72
72
|
if @thread
|
73
|
-
$stderr.
|
73
|
+
$stderr.print "in branch thread #{@thread}:\n "
|
74
74
|
if bt = @thread.backtrace
|
75
|
-
$stderr.puts bt.join("\n")
|
75
|
+
$stderr.puts bt.join("\n ")
|
76
76
|
end
|
77
77
|
end
|
78
78
|
end
|
data/lib/pwrake/nbio.rb
CHANGED
@@ -80,8 +80,8 @@ module NBIO
|
|
80
80
|
to = (timeout) ? timeout*0.75 : nil
|
81
81
|
r, w, = @io_class.select(@reader.keys,@writer.keys,[],to)
|
82
82
|
check_heartbeat(r,timeout) if timeout
|
83
|
-
r.each{|io| @reader[io].call} if r
|
84
|
-
w.each{|io| @writer[io].call} if w
|
83
|
+
r.each{|io| x = @reader[io]; x.call if x} if r
|
84
|
+
w.each{|io| x = @writer[io]; x.call if x} if w
|
85
85
|
rescue IOError => e
|
86
86
|
em = "#{e.class.name}: #{e.message}"
|
87
87
|
@reader.keys.each do |io|
|
data/lib/pwrake/version.rb
CHANGED
@@ -39,9 +39,10 @@ module Pwrake
|
|
39
39
|
"PMI_SIZE" => nil,
|
40
40
|
}
|
41
41
|
|
42
|
-
def initialize(selector,dir_class,id)
|
42
|
+
def initialize(selector,dir_class,id,option)
|
43
43
|
@selector = selector
|
44
44
|
@id = id
|
45
|
+
@option = option
|
45
46
|
@out = Writer.instance
|
46
47
|
@log = LogExecutor.instance
|
47
48
|
@queue = FiberQueue.new
|
@@ -110,22 +111,39 @@ module Pwrake
|
|
110
111
|
@spawn_in, @sh_in = IO.pipe
|
111
112
|
@sh_out, @spawn_out = IO.pipe
|
112
113
|
@sh_err, @spawn_err = IO.pipe
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
114
|
+
if @option[:gnu_time]
|
115
|
+
@sh_gtm, @spawn_gtm = IO.pipe
|
116
|
+
@pid = Kernel.spawn(ENV, wrap_gnu_time(command),
|
117
|
+
in:@spawn_in,
|
118
|
+
out:@spawn_out,
|
119
|
+
err:@spawn_err,
|
120
|
+
3=>@spawn_gtm,
|
121
|
+
chdir:@dir.current,
|
122
|
+
pgroup:true,
|
123
|
+
)
|
124
|
+
@thread = Thread.new do
|
125
|
+
@pid2,@status = Process.waitpid2(@pid)
|
126
|
+
@spawn_in.close
|
127
|
+
@spawn_out.close
|
128
|
+
@spawn_err.close
|
129
|
+
@spawn_gtm.close
|
130
|
+
end
|
131
|
+
else
|
132
|
+
@pid = Kernel.spawn(ENV, command,
|
133
|
+
in:@spawn_in,
|
134
|
+
out:@spawn_out,
|
135
|
+
err:@spawn_err,
|
136
|
+
chdir:@dir.current,
|
137
|
+
pgroup:true
|
138
|
+
)
|
139
|
+
@thread = Thread.new do
|
140
|
+
@pid2,@status = Process.waitpid2(@pid)
|
141
|
+
@spawn_in.close
|
142
|
+
@spawn_out.close
|
143
|
+
@spawn_err.close
|
144
|
+
end
|
128
145
|
end
|
146
|
+
@log.info "pid=#{@pid} started. command=#{command.inspect}"
|
129
147
|
|
130
148
|
@rd_out = NBIO::Reader.new(@selector,@sh_out)
|
131
149
|
@rd_err = NBIO::Reader.new(@selector,@sh_err)
|
@@ -135,20 +153,38 @@ module Pwrake
|
|
135
153
|
Fiber.new{callback(@rd_out,"o")}.resume
|
136
154
|
end
|
137
155
|
|
156
|
+
def wrap_gnu_time(cmd)
|
157
|
+
if /\[|\]|<|>|\(|\)|\&|\||\\|\$|;|`|'|"|\n/ =~ cmd
|
158
|
+
cmd = cmd.gsub(/'/,"'\"'\"'")
|
159
|
+
cmd = (ENV['SHELL']||"sh")+" -c '#{cmd}'"
|
160
|
+
end
|
161
|
+
f = "%e,%S,%U,%M,%t,%K,%D,%p,%X,%Z,%F,%R,%W,%c,%w,%I,%O,%r,%s,%k"
|
162
|
+
"/usr/bin/time -o /dev/fd/3 -f '#{f}' #{cmd}"
|
163
|
+
end
|
164
|
+
|
138
165
|
def callback(rd,mode)
|
139
166
|
while s = rd.gets
|
140
|
-
|
167
|
+
s.chomp!
|
168
|
+
@out.puts "#{@id}:#{mode}:#{s}"
|
141
169
|
end
|
142
170
|
if rd.eof?
|
143
171
|
@rd_list.delete(rd)
|
144
172
|
if @rd_list.empty? # process_end
|
145
173
|
@thread = @pid = nil
|
146
174
|
@log.info inspect_status
|
175
|
+
if @option[:gnu_time]
|
176
|
+
if gt = @sh_gtm.gets
|
177
|
+
@out.puts "#{@id}:t:#{gt.chomp}"
|
178
|
+
end
|
179
|
+
@sh_gtm.close
|
180
|
+
end
|
147
181
|
@out.puts "#{@id}:z:#{exit_status}"
|
148
182
|
@sh_in.close
|
149
183
|
@sh_out.close
|
150
184
|
@sh_err.close
|
151
|
-
@start_process_fiber.
|
185
|
+
if @start_process_fiber.alive?
|
186
|
+
@start_process_fiber.resume # next process
|
187
|
+
end
|
152
188
|
end
|
153
189
|
end
|
154
190
|
rescue => exc
|
@@ -114,7 +114,7 @@ module Pwrake
|
|
114
114
|
case line
|
115
115
|
when /^(\d+):open$/o
|
116
116
|
$1.split.each do |id|
|
117
|
-
@ex_list[id] = Executor.new(@selector,@dir_class,id)
|
117
|
+
@ex_list[id] = Executor.new(@selector,@dir_class,id,@option)
|
118
118
|
end
|
119
119
|
when "setup_end"
|
120
120
|
return
|
@@ -164,7 +164,7 @@ module Pwrake
|
|
164
164
|
sig = $1
|
165
165
|
sig = sig.to_i if /^\d+$/o =~ sig
|
166
166
|
kill_all(sig)
|
167
|
-
return
|
167
|
+
return true
|
168
168
|
#
|
169
169
|
when /^p$/o
|
170
170
|
$stderr.puts "@ex_list = #{@ex_list.inspect}"
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: pwrake
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.2.
|
4
|
+
version: 2.2.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Masahiro TANAKA
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-
|
11
|
+
date: 2017-10-27 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: parallel
|
@@ -136,7 +136,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
136
136
|
version: '0'
|
137
137
|
requirements: []
|
138
138
|
rubyforge_project:
|
139
|
-
rubygems_version: 2.6.
|
139
|
+
rubygems_version: 2.6.13
|
140
140
|
signing_key:
|
141
141
|
specification_version: 4
|
142
142
|
summary: Parallel Workflow engine based on Rake
|