switchman 1.5.15 → 1.5.16

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: bd0d33eb675d2b10b4c3f4c58cd70664fffc02d9
4
- data.tar.gz: ceb9f08769064d7c046b79c2d4570680d835302a
3
+ metadata.gz: 211d4cd37f40dcadb51a8b83fb6d3fd85462c664
4
+ data.tar.gz: 70f7d6ec5c7b2f201398f22b7f0bcaf55805b368
5
5
  SHA512:
6
- metadata.gz: bb26c88cfc2df081bdfc3d3ca9e2f3fb0c683f5a1ea5f9174229de2f0f4a6b2c23397f061711db2c6a02a5a718f1d91c1ffda9c627f3264cd443d7d4697587ed
7
- data.tar.gz: d8d0e4ce64c6e80d4f8df2c47f770947eaeb2f0e10b06e34ca265ee2642d5b87d5a03ec9a900a1d75f3c3b45c1f4285cd0e8074d8ccfbc050d1b1d10ddc0b4d5
6
+ metadata.gz: 9736326b9ee5c92bf46ac6fdecf789ab969bcfa697f1f42ec758f243b97801ad3e075dafeef8fbe89c377567f427b4f56d1b612d0accc3a66f1377970dc14708
7
+ data.tar.gz: c40ac77605fb1d865c73fbf885ee438b1d1d66372eacdd4e107dff4bf99a099f519667f7b33d4ee546a5a1cdd18020544ff009c2a301ee495e61d53a7468a05a
@@ -1,6 +1,7 @@
1
1
  require 'switchman/database_server'
2
2
  require 'switchman/default_shard'
3
3
  require 'switchman/environment'
4
+ require 'switchman/errors'
4
5
 
5
6
  module Switchman
6
7
  class Shard < ::ActiveRecord::Base
@@ -151,6 +152,7 @@ module Switchman
151
152
  # sub-processes per database server. Note that parallel
152
153
  # invocation currently uses forking, so should be used sparingly
153
154
  # because errors are not raised, and you cannot get results back
155
+ # :max_procs - only run this many parallel processes at a time
154
156
  # :exception - :ignore, :raise, :defer (wait until the end and raise the first
155
157
  # error), or a proc
156
158
  def with_each_shard(*args)
@@ -225,10 +227,13 @@ module Switchman
225
227
  end
226
228
  end
227
229
 
228
- fd_to_name_map = {}
230
+ exception_pipes = []
231
+ pids = []
229
232
  out_fds = []
230
233
  err_fds = []
231
- pids = []
234
+ pid_to_name_map = {}
235
+ fd_to_name_map = {}
236
+ errors = []
232
237
 
233
238
  wait_for_output = lambda do |out_fds, err_fds, fd_to_name_map|
234
239
  ready, _ = IO.select(out_fds + err_fds)
@@ -244,7 +249,6 @@ module Switchman
244
249
  end
245
250
  end
246
251
 
247
- exception_pipe = IO.pipe
248
252
  scopes.each do |server, subscopes|
249
253
  if !(::ActiveRecord::Relation === subscopes.first) && subscopes.first.class != Array
250
254
  subscopes = [subscopes]
@@ -263,55 +267,69 @@ module Switchman
263
267
  name = server.id
264
268
  end
265
269
 
266
- details = Open4.pfork4(lambda do
270
+ exception_pipe = IO.pipe
271
+ exception_pipes << exception_pipe
272
+ pid, io_in, io_out, io_err = Open4.pfork4(lambda do
267
273
  begin
268
274
  ::ActiveRecord::Base.clear_all_connections!
269
275
  Switchman.config[:on_fork_proc].try(:call)
270
276
  $0 = [$0, ARGV, name].flatten.join(' ')
271
277
  with_each_shard(subscope, categories, options) { yield }
278
+ exception_pipe.last.close
272
279
  rescue Exception => e
273
- exception_pipe.last.write(Marshal.dump(e))
280
+ Marshal.dump(e, exception_pipe.last)
274
281
  exception_pipe.last.flush
275
- exit 1
282
+ exception_pipe.last.close
283
+ exit! 1
276
284
  end
277
285
  end)
278
- # don't care about writing to stdin
279
- details[1].close
280
- out_fds << details[2]
281
- err_fds << details[3]
282
- pids << details[0]
283
- fd_to_name_map[details[2]] = name
284
- fd_to_name_map[details[3]] = name
286
+ exception_pipe.last.close
287
+ pids << pid
288
+ io_in.close # don't care about writing to stdin
289
+ out_fds << io_out
290
+ err_fds << io_err
291
+ pid_to_name_map[pid] = name
292
+ fd_to_name_map[io_out] = name
293
+ fd_to_name_map[io_err] = name
285
294
 
286
295
  while max_procs && pids.count >= max_procs
287
296
  while max_procs && out_fds.count >= max_procs
288
297
  # wait for output if we've hit the max_procs limit
289
298
  wait_for_output.call(out_fds, err_fds, fd_to_name_map)
290
299
  end
291
- pids.delete(Process.wait) # we've gotten all the output from one fd so wait for its child process to exit
300
+ # we've gotten all the output from one fd so wait for its child process to exit
301
+ found_pid, status = Process.wait2
302
+ pids.delete(found_pid)
303
+ errors << pid_to_name_map[found_pid] if status.exitstatus != 0
292
304
  end
293
305
  end
294
306
  end
295
307
 
296
- exception_pipe.last.close
297
-
298
308
  while out_fds.any? || err_fds.any?
299
309
  wait_for_output.call(out_fds, err_fds, fd_to_name_map)
300
310
  end
301
- pids.each { |pid| Process.waitpid2(pid) }
311
+ pids.each do |pid|
312
+ _, status = Process.waitpid2(pid)
313
+ errors << pid_to_name_map[pid] if status.exitstatus != 0
314
+ end
302
315
 
303
316
  # I'm not sure why, but we have to do this
304
317
  ::ActiveRecord::Base.clear_all_connections!
318
+
305
319
  # check for an exception; we only re-raise the first one
306
- # (all the sub-processes shared the same pipe, so we only
307
- # have to check the one)
308
- begin
309
- exception = Marshal.load exception_pipe.first
310
- raise exception
311
- rescue EOFError
312
- # No exceptions
313
- ensure
314
- exception_pipe.first.close
320
+ exception_pipes.each do |exception_pipe|
321
+ begin
322
+ exception = Marshal.load exception_pipe.first
323
+ raise exception
324
+ rescue EOFError
325
+ # No exceptions
326
+ ensure
327
+ exception_pipe.first.close
328
+ end
329
+ end
330
+
331
+ unless errors.empty?
332
+ raise ParallelShardExecError.new("The following subprocesses did not exit cleanly: #{errors.sort.join(", ")}")
315
333
  end
316
334
  return
317
335
  end
data/lib/switchman.rb CHANGED
@@ -1,5 +1,5 @@
1
1
  require "shackles"
2
- require "open4"
2
+ require "switchman/open4"
3
3
  require "switchman/engine"
4
4
 
5
5
  module Switchman
@@ -1,3 +1,4 @@
1
1
  module Switchman
2
2
  class NonExistentShardError < RuntimeError; end
3
+ class ParallelShardExecError < RuntimeError; end
3
4
  end
@@ -0,0 +1,78 @@
1
+ require 'open4'
2
+
3
+ # This fixes a bug with exception handling,
4
+ # see https://github.com/ahoward/open4/pull/30
5
+ module Open4
6
+ def self.do_popen(b = nil, exception_propagation_at = nil, closefds=false, &cmd)
7
+ pw, pr, pe, ps = IO.pipe, IO.pipe, IO.pipe, IO.pipe
8
+
9
+ verbose = $VERBOSE
10
+ begin
11
+ $VERBOSE = nil
12
+
13
+ cid = fork {
14
+ if closefds
15
+ exlist = [0, 1, 2] | [pw,pr,pe,ps].map{|p| [p.first.fileno, p.last.fileno] }.flatten
16
+ ObjectSpace.each_object(IO){|io|
17
+ io.close if (not io.closed?) and (not exlist.include? io.fileno) rescue nil
18
+ }
19
+ end
20
+
21
+ pw.last.close
22
+ STDIN.reopen pw.first
23
+ pw.first.close
24
+
25
+ pr.first.close
26
+ STDOUT.reopen pr.last
27
+ pr.last.close
28
+
29
+ pe.first.close
30
+ STDERR.reopen pe.last
31
+ pe.last.close
32
+
33
+ STDOUT.sync = STDERR.sync = true
34
+
35
+ begin
36
+ cmd.call(ps)
37
+ rescue Exception => e
38
+ begin
39
+ Marshal.dump(e, ps.last)
40
+ ps.last.flush
41
+ rescue Errno::EPIPE
42
+ raise e
43
+ end
44
+ ensure
45
+ ps.last.close unless ps.last.closed?
46
+ end
47
+
48
+ exit!
49
+ }
50
+ ensure
51
+ $VERBOSE = verbose
52
+ end
53
+
54
+ [ pw.first, pr.last, pe.last, ps.last ].each { |fd| fd.close }
55
+
56
+ Open4.propagate_exception cid, ps.first if exception_propagation_at == :init
57
+
58
+ pw.last.sync = true
59
+
60
+ pi = [ pw.last, pr.first, pe.first ]
61
+
62
+ begin
63
+ return [cid, *pi] unless b
64
+
65
+ begin
66
+ b.call(cid, *pi)
67
+ ensure
68
+ pi.each { |fd| fd.close unless fd.closed? }
69
+ end
70
+
71
+ Open4.propagate_exception cid, ps.first if exception_propagation_at == :block
72
+
73
+ Process.waitpid2(cid).last
74
+ ensure
75
+ ps.first.close unless ps.first.closed?
76
+ end
77
+ end
78
+ end
@@ -1,3 +1,3 @@
1
1
  module Switchman
2
- VERSION = "1.5.15"
2
+ VERSION = "1.5.16"
3
3
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: switchman
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.5.15
4
+ version: 1.5.16
5
5
  platform: ruby
6
6
  authors:
7
7
  - Cody Cutrer
@@ -10,7 +10,7 @@ authors:
10
10
  autorequire:
11
11
  bindir: bin
12
12
  cert_chain: []
13
- date: 2016-08-17 00:00:00.000000000 Z
13
+ date: 2016-08-22 00:00:00.000000000 Z
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
16
16
  name: railties
@@ -195,6 +195,7 @@ files:
195
195
  - lib/switchman/engine.rb
196
196
  - lib/switchman/environment.rb
197
197
  - lib/switchman/errors.rb
198
+ - lib/switchman/open4.rb
198
199
  - lib/switchman/r_spec_helper.rb
199
200
  - lib/switchman/rails.rb
200
201
  - lib/switchman/schema_cache.rb