switchman 1.5.15 → 1.5.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/app/models/switchman/shard_internal.rb +44 -26
- data/lib/switchman.rb +1 -1
- data/lib/switchman/errors.rb +1 -0
- data/lib/switchman/open4.rb +78 -0
- data/lib/switchman/version.rb +1 -1
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 211d4cd37f40dcadb51a8b83fb6d3fd85462c664
|
4
|
+
data.tar.gz: 70f7d6ec5c7b2f201398f22b7f0bcaf55805b368
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 9736326b9ee5c92bf46ac6fdecf789ab969bcfa697f1f42ec758f243b97801ad3e075dafeef8fbe89c377567f427b4f56d1b612d0accc3a66f1377970dc14708
|
7
|
+
data.tar.gz: c40ac77605fb1d865c73fbf885ee438b1d1d66372eacdd4e107dff4bf99a099f519667f7b33d4ee546a5a1cdd18020544ff009c2a301ee495e61d53a7468a05a
|
@@ -1,6 +1,7 @@
|
|
1
1
|
require 'switchman/database_server'
|
2
2
|
require 'switchman/default_shard'
|
3
3
|
require 'switchman/environment'
|
4
|
+
require 'switchman/errors'
|
4
5
|
|
5
6
|
module Switchman
|
6
7
|
class Shard < ::ActiveRecord::Base
|
@@ -151,6 +152,7 @@ module Switchman
|
|
151
152
|
# sub-processes per database server. Note that parallel
|
152
153
|
# invocation currently uses forking, so should be used sparingly
|
153
154
|
# because errors are not raised, and you cannot get results back
|
155
|
+
# :max_procs - only run this many parallel processes at a time
|
154
156
|
# :exception - :ignore, :raise, :defer (wait until the end and raise the first
|
155
157
|
# error), or a proc
|
156
158
|
def with_each_shard(*args)
|
@@ -225,10 +227,13 @@ module Switchman
|
|
225
227
|
end
|
226
228
|
end
|
227
229
|
|
228
|
-
|
230
|
+
exception_pipes = []
|
231
|
+
pids = []
|
229
232
|
out_fds = []
|
230
233
|
err_fds = []
|
231
|
-
|
234
|
+
pid_to_name_map = {}
|
235
|
+
fd_to_name_map = {}
|
236
|
+
errors = []
|
232
237
|
|
233
238
|
wait_for_output = lambda do |out_fds, err_fds, fd_to_name_map|
|
234
239
|
ready, _ = IO.select(out_fds + err_fds)
|
@@ -244,7 +249,6 @@ module Switchman
|
|
244
249
|
end
|
245
250
|
end
|
246
251
|
|
247
|
-
exception_pipe = IO.pipe
|
248
252
|
scopes.each do |server, subscopes|
|
249
253
|
if !(::ActiveRecord::Relation === subscopes.first) && subscopes.first.class != Array
|
250
254
|
subscopes = [subscopes]
|
@@ -263,55 +267,69 @@ module Switchman
|
|
263
267
|
name = server.id
|
264
268
|
end
|
265
269
|
|
266
|
-
|
270
|
+
exception_pipe = IO.pipe
|
271
|
+
exception_pipes << exception_pipe
|
272
|
+
pid, io_in, io_out, io_err = Open4.pfork4(lambda do
|
267
273
|
begin
|
268
274
|
::ActiveRecord::Base.clear_all_connections!
|
269
275
|
Switchman.config[:on_fork_proc].try(:call)
|
270
276
|
$0 = [$0, ARGV, name].flatten.join(' ')
|
271
277
|
with_each_shard(subscope, categories, options) { yield }
|
278
|
+
exception_pipe.last.close
|
272
279
|
rescue Exception => e
|
273
|
-
|
280
|
+
Marshal.dump(e, exception_pipe.last)
|
274
281
|
exception_pipe.last.flush
|
275
|
-
|
282
|
+
exception_pipe.last.close
|
283
|
+
exit! 1
|
276
284
|
end
|
277
285
|
end)
|
278
|
-
|
279
|
-
|
280
|
-
|
281
|
-
|
282
|
-
|
283
|
-
|
284
|
-
fd_to_name_map[
|
286
|
+
exception_pipe.last.close
|
287
|
+
pids << pid
|
288
|
+
io_in.close # don't care about writing to stdin
|
289
|
+
out_fds << io_out
|
290
|
+
err_fds << io_err
|
291
|
+
pid_to_name_map[pid] = name
|
292
|
+
fd_to_name_map[io_out] = name
|
293
|
+
fd_to_name_map[io_err] = name
|
285
294
|
|
286
295
|
while max_procs && pids.count >= max_procs
|
287
296
|
while max_procs && out_fds.count >= max_procs
|
288
297
|
# wait for output if we've hit the max_procs limit
|
289
298
|
wait_for_output.call(out_fds, err_fds, fd_to_name_map)
|
290
299
|
end
|
291
|
-
|
300
|
+
# we've gotten all the output from one fd so wait for its child process to exit
|
301
|
+
found_pid, status = Process.wait2
|
302
|
+
pids.delete(found_pid)
|
303
|
+
errors << pid_to_name_map[found_pid] if status.exitstatus != 0
|
292
304
|
end
|
293
305
|
end
|
294
306
|
end
|
295
307
|
|
296
|
-
exception_pipe.last.close
|
297
|
-
|
298
308
|
while out_fds.any? || err_fds.any?
|
299
309
|
wait_for_output.call(out_fds, err_fds, fd_to_name_map)
|
300
310
|
end
|
301
|
-
pids.each
|
311
|
+
pids.each do |pid|
|
312
|
+
_, status = Process.waitpid2(pid)
|
313
|
+
errors << pid_to_name_map[pid] if status.exitstatus != 0
|
314
|
+
end
|
302
315
|
|
303
316
|
# I'm not sure why, but we have to do this
|
304
317
|
::ActiveRecord::Base.clear_all_connections!
|
318
|
+
|
305
319
|
# check for an exception; we only re-raise the first one
|
306
|
-
|
307
|
-
|
308
|
-
|
309
|
-
|
310
|
-
|
311
|
-
|
312
|
-
|
313
|
-
|
314
|
-
|
320
|
+
exception_pipes.each do |exception_pipe|
|
321
|
+
begin
|
322
|
+
exception = Marshal.load exception_pipe.first
|
323
|
+
raise exception
|
324
|
+
rescue EOFError
|
325
|
+
# No exceptions
|
326
|
+
ensure
|
327
|
+
exception_pipe.first.close
|
328
|
+
end
|
329
|
+
end
|
330
|
+
|
331
|
+
unless errors.empty?
|
332
|
+
raise ParallelShardExecError.new("The following subprocesses did not exit cleanly: #{errors.sort.join(", ")}")
|
315
333
|
end
|
316
334
|
return
|
317
335
|
end
|
data/lib/switchman.rb
CHANGED
data/lib/switchman/errors.rb
CHANGED
@@ -0,0 +1,78 @@
|
|
1
|
+
require 'open4'
|
2
|
+
|
3
|
+
# This fixes a bug with exception handling,
|
4
|
+
# see https://github.com/ahoward/open4/pull/30
|
5
|
+
module Open4
|
6
|
+
def self.do_popen(b = nil, exception_propagation_at = nil, closefds=false, &cmd)
|
7
|
+
pw, pr, pe, ps = IO.pipe, IO.pipe, IO.pipe, IO.pipe
|
8
|
+
|
9
|
+
verbose = $VERBOSE
|
10
|
+
begin
|
11
|
+
$VERBOSE = nil
|
12
|
+
|
13
|
+
cid = fork {
|
14
|
+
if closefds
|
15
|
+
exlist = [0, 1, 2] | [pw,pr,pe,ps].map{|p| [p.first.fileno, p.last.fileno] }.flatten
|
16
|
+
ObjectSpace.each_object(IO){|io|
|
17
|
+
io.close if (not io.closed?) and (not exlist.include? io.fileno) rescue nil
|
18
|
+
}
|
19
|
+
end
|
20
|
+
|
21
|
+
pw.last.close
|
22
|
+
STDIN.reopen pw.first
|
23
|
+
pw.first.close
|
24
|
+
|
25
|
+
pr.first.close
|
26
|
+
STDOUT.reopen pr.last
|
27
|
+
pr.last.close
|
28
|
+
|
29
|
+
pe.first.close
|
30
|
+
STDERR.reopen pe.last
|
31
|
+
pe.last.close
|
32
|
+
|
33
|
+
STDOUT.sync = STDERR.sync = true
|
34
|
+
|
35
|
+
begin
|
36
|
+
cmd.call(ps)
|
37
|
+
rescue Exception => e
|
38
|
+
begin
|
39
|
+
Marshal.dump(e, ps.last)
|
40
|
+
ps.last.flush
|
41
|
+
rescue Errno::EPIPE
|
42
|
+
raise e
|
43
|
+
end
|
44
|
+
ensure
|
45
|
+
ps.last.close unless ps.last.closed?
|
46
|
+
end
|
47
|
+
|
48
|
+
exit!
|
49
|
+
}
|
50
|
+
ensure
|
51
|
+
$VERBOSE = verbose
|
52
|
+
end
|
53
|
+
|
54
|
+
[ pw.first, pr.last, pe.last, ps.last ].each { |fd| fd.close }
|
55
|
+
|
56
|
+
Open4.propagate_exception cid, ps.first if exception_propagation_at == :init
|
57
|
+
|
58
|
+
pw.last.sync = true
|
59
|
+
|
60
|
+
pi = [ pw.last, pr.first, pe.first ]
|
61
|
+
|
62
|
+
begin
|
63
|
+
return [cid, *pi] unless b
|
64
|
+
|
65
|
+
begin
|
66
|
+
b.call(cid, *pi)
|
67
|
+
ensure
|
68
|
+
pi.each { |fd| fd.close unless fd.closed? }
|
69
|
+
end
|
70
|
+
|
71
|
+
Open4.propagate_exception cid, ps.first if exception_propagation_at == :block
|
72
|
+
|
73
|
+
Process.waitpid2(cid).last
|
74
|
+
ensure
|
75
|
+
ps.first.close unless ps.first.closed?
|
76
|
+
end
|
77
|
+
end
|
78
|
+
end
|
data/lib/switchman/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: switchman
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.5.
|
4
|
+
version: 1.5.16
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Cody Cutrer
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date: 2016-08-
|
13
|
+
date: 2016-08-22 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: railties
|
@@ -195,6 +195,7 @@ files:
|
|
195
195
|
- lib/switchman/engine.rb
|
196
196
|
- lib/switchman/environment.rb
|
197
197
|
- lib/switchman/errors.rb
|
198
|
+
- lib/switchman/open4.rb
|
198
199
|
- lib/switchman/r_spec_helper.rb
|
199
200
|
- lib/switchman/rails.rb
|
200
201
|
- lib/switchman/schema_cache.rb
|