autorespawn 0.3.0 → 0.4.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.travis.yml +3 -0
- data/Rakefile +1 -1
- data/lib/autorespawn.rb +1 -1
- data/lib/autorespawn/manager.rb +55 -26
- data/lib/autorespawn/program_id.rb +5 -47
- data/lib/autorespawn/self.rb +3 -2
- data/lib/autorespawn/slave.rb +21 -5
- data/lib/autorespawn/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 74818e88a4f45fa8d63abe962bfa92381816f8d9
|
4
|
+
data.tar.gz: a2df936e42159d3e7156f305710a4976ac75bf68
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a999ef955f12ef443ca425db10394ae4996d46cf0c8bf129697a31404bb7f6c7164381c16a8ca11e802374db6249cb64b4c2cadb4b7becabadf2283c476ad247
|
7
|
+
data.tar.gz: 6a3f7b0ac6e2e160f6f4a8fc3db0f9aa866cc1e3a4c252005d92ba8f806c2734f7ca97c1f046c0856acbddac5e2f62acd4c9384e0c212bb8315bde60f3e7ae63
|
data/.travis.yml
CHANGED
data/Rakefile
CHANGED
data/lib/autorespawn.rb
CHANGED
data/lib/autorespawn/manager.rb
CHANGED
@@ -17,12 +17,14 @@ class Manager
|
|
17
17
|
# {#workers} and {#active_slaves}
|
18
18
|
attr_reader :self_slave
|
19
19
|
# @return [Integer] the number of processes allowed to work in parallel
|
20
|
-
|
20
|
+
attr_accessor :parallel_level
|
21
21
|
# @return [Array<Slave>] declared worker processes, as a hash from
|
22
22
|
# the PID to a Slave object
|
23
23
|
attr_reader :workers
|
24
24
|
# @return [Hash<Slave>] list of active slaves
|
25
25
|
attr_reader :active_slaves
|
26
|
+
# @return [Array<Slave>] list of slaves explicitely queued with {#queue}
|
27
|
+
attr_reader :queued_slaves
|
26
28
|
|
27
29
|
# @!group Hooks
|
28
30
|
|
@@ -68,13 +70,14 @@ def on_slave_start(&block)
|
|
68
70
|
# @!endgroup
|
69
71
|
|
70
72
|
def initialize(name: nil, parallel_level: 1)
|
71
|
-
@parallel_level = parallel_level
|
73
|
+
@parallel_level = parallel_level
|
72
74
|
@workers = Array.new
|
73
75
|
@name = name
|
74
76
|
@seed = ProgramID.for_self
|
75
77
|
|
76
78
|
@self_slave = Self.new(name: name)
|
77
79
|
@workers << self_slave
|
80
|
+
@queued_slaves = Array.new
|
78
81
|
@active_slaves = Hash[self_slave.pid => self_slave]
|
79
82
|
end
|
80
83
|
|
@@ -85,11 +88,27 @@ def register_seed_files(files, search_patch = seed.ruby_load_path, ignore_not_fo
|
|
85
88
|
seed.register_files(files, search_path, ignore_not_found)
|
86
89
|
end
|
87
90
|
|
91
|
+
# Check whether this manager has slaves
|
92
|
+
def has_slaves?
|
93
|
+
# There's always a worker for self
|
94
|
+
workers.size != 1
|
95
|
+
end
|
96
|
+
|
97
|
+
# The number of slaves registered
|
98
|
+
def slave_count
|
99
|
+
workers.size - 1
|
100
|
+
end
|
101
|
+
|
88
102
|
# Tests whether this slave is registered as a worker on self
|
89
103
|
def include?(slave)
|
90
104
|
workers.include?(slave)
|
91
105
|
end
|
92
106
|
|
107
|
+
# Tests whether this manager has some slaves that are active
|
108
|
+
def has_active_slaves?
|
109
|
+
active_slaves.size != 1
|
110
|
+
end
|
111
|
+
|
93
112
|
# Tests whether this slave is currently active on self
|
94
113
|
def active?(slave)
|
95
114
|
active_slaves[slave.pid] == slave
|
@@ -126,43 +145,48 @@ def register_slave(slave)
|
|
126
145
|
slave
|
127
146
|
end
|
128
147
|
|
148
|
+
# Queue a slave for execution
|
149
|
+
def queue(slave)
|
150
|
+
queued_slaves << slave
|
151
|
+
end
|
152
|
+
|
129
153
|
# @api private
|
130
154
|
#
|
131
155
|
# Collect information about the finished slaves
|
132
156
|
#
|
133
157
|
# @return [Array<Slave>] the slaves that finished
|
134
|
-
def collect_finished_slaves
|
158
|
+
def collect_finished_slaves
|
135
159
|
finished_slaves = Array.new
|
136
|
-
|
137
|
-
|
138
|
-
else [Process::WNOHANG]
|
139
|
-
end
|
140
|
-
|
141
|
-
while finished_child = Process.waitpid2(-1, *waitpid_options)
|
142
|
-
pid, status = *finished_child
|
143
|
-
if slave = active_slaves.delete(pid)
|
144
|
-
finished_slaves << slave
|
145
|
-
slave.finished(status)
|
146
|
-
slave.subcommands.each do |name, cmdline, spawn_options|
|
147
|
-
add_slave(*cmdline, name: name, **spawn_options)
|
148
|
-
end
|
149
|
-
seed.merge!(slave.program_id)
|
150
|
-
run_hook :on_slave_finished, slave
|
151
|
-
end
|
160
|
+
while finished_child = Process.waitpid2(-1, Process::WNOHANG)
|
161
|
+
finished_slaves << process_finished_slave(*finished_child)
|
152
162
|
end
|
153
163
|
finished_slaves
|
154
164
|
rescue Errno::ECHILD
|
155
|
-
|
165
|
+
finished_slaves
|
166
|
+
end
|
167
|
+
|
168
|
+
def process_finished_slave(pid, status)
|
169
|
+
return if !(slave = active_slaves.delete(pid))
|
170
|
+
|
171
|
+
slave.finished(status)
|
172
|
+
slave.subcommands.each do |name, cmdline, spawn_options|
|
173
|
+
add_slave(*cmdline, name: name, **spawn_options)
|
174
|
+
end
|
175
|
+
seed.merge!(slave.program_id)
|
176
|
+
run_hook :on_slave_finished, slave
|
177
|
+
slave
|
156
178
|
end
|
157
179
|
|
158
180
|
# Kill all active slaves
|
159
181
|
#
|
160
182
|
# @see clear
|
161
183
|
def kill
|
162
|
-
active_slaves.
|
163
|
-
while
|
164
|
-
|
184
|
+
active_slaves.each_value { |s| s.kill(join: false) }
|
185
|
+
while has_active_slaves?
|
186
|
+
finished_child = Process.waitpid2(-1)
|
187
|
+
process_finished_slave(*finished_child)
|
165
188
|
end
|
189
|
+
rescue Errno::ECHILD
|
166
190
|
end
|
167
191
|
|
168
192
|
# Kill and remove all workers from this manager
|
@@ -191,13 +215,18 @@ def run
|
|
191
215
|
end
|
192
216
|
|
193
217
|
# Wait for children to terminate and spawns them when needed
|
194
|
-
def poll
|
218
|
+
def poll(autospawn: true)
|
195
219
|
finished_slaves = collect_finished_slaves
|
196
220
|
new_slaves = Array.new
|
197
|
-
while active_slaves.size < parallel_level
|
198
|
-
if
|
221
|
+
while active_slaves.size < parallel_level + 1
|
222
|
+
if slave = queued_slaves.find { |s| !s.running? }
|
223
|
+
queued_slaves.delete(slave)
|
224
|
+
elsif autospawn && (slave_i = workers.index { |s| s.needed? })
|
199
225
|
slave = workers.delete_at(slave_i)
|
200
226
|
@workers = workers[slave_i..-1] + workers[0, slave_i] + [slave]
|
227
|
+
end
|
228
|
+
|
229
|
+
if slave
|
201
230
|
slave.spawn
|
202
231
|
run_hook :__on_slave_start, slave
|
203
232
|
new_slaves << slave
|
@@ -8,7 +8,7 @@ class Autorespawn
|
|
8
8
|
# It basically stores information about all the files that form this
|
9
9
|
# program.
|
10
10
|
class ProgramID
|
11
|
-
FileInfo = Struct.new :require_path, :path, :mtime, :size
|
11
|
+
FileInfo = Struct.new :require_path, :path, :mtime, :size
|
12
12
|
|
13
13
|
# Information about the files that form this program
|
14
14
|
#
|
@@ -137,22 +137,18 @@ def id
|
|
137
137
|
return @id if @id
|
138
138
|
|
139
139
|
complete_id = files.keys.sort.map do |p|
|
140
|
-
files[p].
|
140
|
+
"#{p}#{files[p].mtime}#{files[p].size}"
|
141
141
|
end.join("")
|
142
142
|
@id = Digest::SHA1.hexdigest(complete_id)
|
143
143
|
end
|
144
144
|
|
145
145
|
# Whether the state on disk is different than the state stored in self
|
146
146
|
def changed?
|
147
|
-
files.each_value do |info|
|
147
|
+
files.each_value.any? do |info|
|
148
148
|
return true if !info.path.exist?
|
149
149
|
stat = info.path.stat
|
150
|
-
|
151
|
-
new_id = compute_file_id(info.path)
|
152
|
-
return new_id != info.id
|
153
|
-
end
|
150
|
+
stat.mtime != info.mtime || stat.size != info.size
|
154
151
|
end
|
155
|
-
false
|
156
152
|
end
|
157
153
|
|
158
154
|
def include?(path, search_path = ruby_load_path)
|
@@ -202,45 +198,7 @@ def ruby_load_path
|
|
202
198
|
def file_info(path, search_path = ruby_load_path)
|
203
199
|
resolved = resolve_file_path(path, search_path)
|
204
200
|
stat = resolved.stat
|
205
|
-
|
206
|
-
return FileInfo.new(path, resolved, stat.mtime, stat.size, id)
|
207
|
-
end
|
208
|
-
|
209
|
-
# @api private
|
210
|
-
#
|
211
|
-
# Compute the content ID of a text (code) file
|
212
|
-
def compute_text_file_id(file)
|
213
|
-
sanitized = file.readlines.map do |line|
|
214
|
-
# Remove unnecessary spaces
|
215
|
-
line = line.strip
|
216
|
-
line = line.gsub(/\s\s+/, ' ')
|
217
|
-
if !line.empty?
|
218
|
-
line
|
219
|
-
end
|
220
|
-
end.compact
|
221
|
-
Digest::SHA1.hexdigest(sanitized.join("\n"))
|
222
|
-
end
|
223
|
-
|
224
|
-
# @api private
|
225
|
-
#
|
226
|
-
# Compute the content ID of a binary file
|
227
|
-
def compute_binary_file_id(file)
|
228
|
-
Digest::SHA1.hexdigest(file.read(enc: 'BINARY'))
|
229
|
-
end
|
230
|
-
|
231
|
-
# Compute a SHA1 that is representative of the file's contents
|
232
|
-
#
|
233
|
-
# It does some whitespace cleanup, but is not meant to be super-robust
|
234
|
-
# to changes that are irrelevant to the end program
|
235
|
-
#
|
236
|
-
# @param [Pathname] file the path to the file
|
237
|
-
# @return [String] an ID string
|
238
|
-
def compute_file_id(file)
|
239
|
-
if file.extname == ".rb"
|
240
|
-
compute_text_file_id(file)
|
241
|
-
else
|
242
|
-
compute_binary_file_id(file)
|
243
|
-
end
|
201
|
+
return FileInfo.new(path, resolved, stat.mtime, stat.size)
|
244
202
|
end
|
245
203
|
end
|
246
204
|
end
|
data/lib/autorespawn/self.rb
CHANGED
data/lib/autorespawn/slave.rb
CHANGED
@@ -47,7 +47,7 @@ def initialize(*cmdline, name: nil, seed: ProgramID.new, env: Hash.new, **spawn_
|
|
47
47
|
@name = name
|
48
48
|
@program_id = seed.dup
|
49
49
|
@cmdline = cmdline
|
50
|
-
@
|
50
|
+
@needed = true
|
51
51
|
@spawn_env = env
|
52
52
|
@spawn_options = spawn_options
|
53
53
|
@subcommands = Array.new
|
@@ -85,7 +85,7 @@ def spawn
|
|
85
85
|
SLAVE_RESULT_ENV => result_w.fileno.to_s)
|
86
86
|
|
87
87
|
program_id.refresh
|
88
|
-
@
|
88
|
+
@needed = false
|
89
89
|
pid = Kernel.spawn(env, *cmdline, initial_r => initial_r, result_w => result_w, **spawn_options)
|
90
90
|
initial_r.close
|
91
91
|
result_w.close
|
@@ -112,8 +112,24 @@ def spawn
|
|
112
112
|
|
113
113
|
# Whether this slave would need to be spawned, either because it has
|
114
114
|
# never be, or because the program ID changed
|
115
|
-
def
|
116
|
-
|
115
|
+
def needed?
|
116
|
+
!running? && (@needed || program_id.changed?)
|
117
|
+
end
|
118
|
+
|
119
|
+
# Marks this slave for execution
|
120
|
+
#
|
121
|
+
# Note that it will only be executed by the underlying {Manager} when
|
122
|
+
# (1) a slot is available and (2) it has stopped running
|
123
|
+
#
|
124
|
+
# This is usually not called directly, but through {Manager#queue} which
|
125
|
+
# also ensures that the slave gets in front of the execution queue
|
126
|
+
def needed!
|
127
|
+
@needed = true
|
128
|
+
end
|
129
|
+
|
130
|
+
# Resets {#needed?} to false
|
131
|
+
def not_needed!
|
132
|
+
@needed = false
|
117
133
|
end
|
118
134
|
|
119
135
|
# Whether the slave is running
|
@@ -175,7 +191,7 @@ def finished(status)
|
|
175
191
|
modified = program_id.register_files(file_list)
|
176
192
|
@program_id = program_id.slice(file_list)
|
177
193
|
if !modified.empty?
|
178
|
-
|
194
|
+
needed!
|
179
195
|
end
|
180
196
|
result_r.close
|
181
197
|
modified
|
data/lib/autorespawn/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: autorespawn
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.4.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sylvain Joyeux
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-
|
11
|
+
date: 2015-10-06 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: hooks
|