switchman 2.1.6 → 2.2.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/app/models/switchman/shard.rb +78 -128
- data/lib/switchman/active_record/attribute_methods.rb +1 -1
- data/lib/switchman/active_record/connection_pool.rb +3 -1
- data/lib/switchman/active_record/log_subscriber.rb +1 -4
- data/lib/switchman/active_record/migration.rb +6 -3
- data/lib/switchman/active_record/query_cache.rb +1 -1
- data/lib/switchman/active_record/query_methods.rb +23 -0
- data/lib/switchman/version.rb +1 -1
- data/lib/tasks/switchman.rake +3 -2
- metadata +11 -11
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c6658f02bb242697d5c6b9d7e9c57ba3613295fddc1de3d618cb74ed125835aa
|
4
|
+
data.tar.gz: 05fa7e267a2d442d3f9862e041fe6f297e3272693dd15273bfb17ec86187450f
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 3651b8ad5d882571716507a49e264b90d70a318e436e816e9dbe2ec0b9ff70031f4bfa7fce4e04748bad8e4dad88a36d16eac4afa1b0c61e2f686ed138bb2ef5
|
7
|
+
data.tar.gz: d1d98111fc70a2786768b61856ae4a54ba914275628489d99f12b7de7b0a891d62e7c41189aad2a388941a0c9ce89742d6751c4078ff15b57660d63d390978ce
|
@@ -144,10 +144,9 @@ module Switchman
|
|
144
144
|
# * +categories+ - an array of categories to activate
|
145
145
|
# * +options+ -
|
146
146
|
# :parallel - true/false to execute in parallel, or a integer of how many
|
147
|
-
# sub-processes
|
148
|
-
#
|
149
|
-
#
|
150
|
-
# :max_procs - only run this many parallel processes at a time
|
147
|
+
# sub-processes. Note that parallel invocation currently uses
|
148
|
+
# forking, so should be used sparingly because you cannot get
|
149
|
+
# results back
|
151
150
|
# :exception - :ignore, :raise, :defer (wait until the end and raise the first
|
152
151
|
# error), or a proc
|
153
152
|
def with_each_shard(*args)
|
@@ -168,11 +167,14 @@ module Switchman
|
|
168
167
|
scope, categories = args
|
169
168
|
end
|
170
169
|
|
170
|
+
# back-compat
|
171
|
+
options[:parallel] = options.delete(:max_procs) if options.key?(:max_procs)
|
172
|
+
|
171
173
|
parallel = case options[:parallel]
|
172
174
|
when true
|
173
|
-
|
175
|
+
[Environment.cpu_count || 2, 2].min
|
174
176
|
when false, nil
|
175
|
-
|
177
|
+
1
|
176
178
|
else
|
177
179
|
options[:parallel]
|
178
180
|
end
|
@@ -183,47 +185,19 @@ module Switchman
|
|
183
185
|
scope = scope.order(::Arel.sql("database_server_id IS NOT NULL, database_server_id, id"))
|
184
186
|
end
|
185
187
|
|
186
|
-
if parallel >
|
187
|
-
max_procs = determine_max_procs(options.delete(:max_procs), parallel)
|
188
|
+
if parallel > 1
|
188
189
|
if ::ActiveRecord::Relation === scope
|
189
190
|
# still need a post-uniq, cause the default database server could be NULL or Rails.env in the db
|
190
191
|
database_servers = scope.reorder('database_server_id').select(:database_server_id).distinct.
|
191
192
|
map(&:database_server).compact.uniq
|
192
193
|
# nothing to do
|
193
194
|
return if database_servers.count == 0
|
194
|
-
parallel = [(max_procs.to_f / database_servers.count).ceil, parallel].min if max_procs
|
195
195
|
|
196
196
|
scopes = Hash[database_servers.map do |server|
|
197
|
-
|
198
|
-
if parallel == 1
|
199
|
-
subscopes = [server_scope]
|
200
|
-
else
|
201
|
-
subscopes = []
|
202
|
-
total = server_scope.count
|
203
|
-
ranges = []
|
204
|
-
server_scope.find_ids_in_ranges(:batch_size => (total.to_f / parallel).ceil) do |min, max|
|
205
|
-
ranges << [min, max]
|
206
|
-
end
|
207
|
-
# create a half-open range on the last one
|
208
|
-
ranges.last[1] = nil
|
209
|
-
ranges.each do |min, max|
|
210
|
-
subscope = server_scope.where("id>=?", min)
|
211
|
-
subscope = subscope.where("id<=?", max) if max
|
212
|
-
subscopes << subscope
|
213
|
-
end
|
214
|
-
end
|
215
|
-
[server, subscopes]
|
197
|
+
[server, server.shards.merge(scope)]
|
216
198
|
end]
|
217
199
|
else
|
218
200
|
scopes = scope.group_by(&:database_server)
|
219
|
-
if parallel > 1
|
220
|
-
parallel = [(max_procs.to_f / scopes.count).ceil, parallel].min if max_procs
|
221
|
-
scopes = Hash[scopes.map do |(server, shards)|
|
222
|
-
[server, shards.in_groups(parallel, false).compact]
|
223
|
-
end]
|
224
|
-
else
|
225
|
-
scopes = Hash[scopes.map { |(server, shards)| [server, [shards]] }]
|
226
|
-
end
|
227
201
|
end
|
228
202
|
|
229
203
|
exception_pipes = []
|
@@ -249,8 +223,8 @@ module Switchman
|
|
249
223
|
end
|
250
224
|
|
251
225
|
# only one process; don't bother forking
|
252
|
-
if scopes.length == 1
|
253
|
-
return with_each_shard(scopes.first.last
|
226
|
+
if scopes.length == 1
|
227
|
+
return with_each_shard(scopes.first.last, categories, options) { yield }
|
254
228
|
end
|
255
229
|
|
256
230
|
# clear connections prior to forking (no more queries will be executed in the parent,
|
@@ -258,84 +232,78 @@ module Switchman
|
|
258
232
|
# silly like dealloc'ing prepared statements)
|
259
233
|
::ActiveRecord::Base.clear_all_connections!
|
260
234
|
|
261
|
-
scopes.each do |server,
|
262
|
-
|
263
|
-
|
264
|
-
|
265
|
-
|
266
|
-
|
267
|
-
|
235
|
+
scopes.each do |server, subscope|
|
236
|
+
name = server.id
|
237
|
+
|
238
|
+
exception_pipe = IO.pipe
|
239
|
+
exception_pipes << exception_pipe
|
240
|
+
pid, io_in, io_out, io_err = Open4.pfork4(lambda do
|
241
|
+
begin
|
242
|
+
Switchman.config[:on_fork_proc]&.call
|
243
|
+
|
244
|
+
# set a pretty name for the process title, up to 128 characters
|
245
|
+
# (we don't actually know the limit, depending on how the process
|
246
|
+
# was started)
|
247
|
+
# first, simplify the binary name by stripping directories,
|
248
|
+
# then truncate arguments as necessary
|
249
|
+
bin = File.basename($0) # Process.argv0 doesn't work on Ruby 2.5 (https://bugs.ruby-lang.org/issues/15887)
|
250
|
+
max_length = 128 - bin.length - name.length - 3
|
251
|
+
args = ARGV.join(" ")
|
252
|
+
if max_length >= 0
|
253
|
+
args = args[0..max_length]
|
254
|
+
end
|
255
|
+
new_title = [bin, args, name].join(" ")
|
256
|
+
Process.setproctitle(new_title)
|
268
257
|
|
269
|
-
|
270
|
-
|
271
|
-
|
258
|
+
with_each_shard(subscope, categories, options) { yield }
|
259
|
+
exception_pipe.last.close
|
260
|
+
rescue Exception => e
|
272
261
|
begin
|
273
|
-
|
274
|
-
|
275
|
-
|
276
|
-
|
277
|
-
|
278
|
-
# first, simplify the binary name by stripping directories,
|
279
|
-
# then truncate arguments as necessary
|
280
|
-
bin = File.basename($0) # Process.argv0 doesn't work on Ruby 2.5 (https://bugs.ruby-lang.org/issues/15887)
|
281
|
-
max_length = 128 - bin.length - name.length - 3
|
282
|
-
args = ARGV.join(" ")
|
283
|
-
if max_length >= 0
|
284
|
-
args = args[0..max_length]
|
285
|
-
end
|
286
|
-
new_title = [bin, args, name].join(" ")
|
287
|
-
Process.setproctitle(new_title)
|
288
|
-
|
289
|
-
with_each_shard(subscope, categories, options) { yield }
|
290
|
-
exception_pipe.last.close
|
291
|
-
rescue Exception => e
|
292
|
-
begin
|
293
|
-
dumped = Marshal.dump(e)
|
294
|
-
dumped = nil if dumped.length > 64 * 1024
|
295
|
-
rescue
|
296
|
-
dumped = nil
|
297
|
-
end
|
262
|
+
dumped = Marshal.dump(e)
|
263
|
+
dumped = nil if dumped.length > 64 * 1024
|
264
|
+
rescue
|
265
|
+
dumped = nil
|
266
|
+
end
|
298
267
|
|
299
|
-
|
300
|
-
|
301
|
-
|
302
|
-
|
303
|
-
|
304
|
-
|
305
|
-
|
306
|
-
|
307
|
-
end
|
308
|
-
e2.set_backtrace(backtrace)
|
309
|
-
e2.instance_variable_set(:@active_shards, e.instance_variable_get(:@active_shards))
|
310
|
-
dumped = Marshal.dump(e2)
|
268
|
+
if dumped.nil?
|
269
|
+
# couldn't dump the exception; create a copy with just
|
270
|
+
# the message and the backtrace
|
271
|
+
e2 = e.class.new(e.message)
|
272
|
+
backtrace = e.backtrace
|
273
|
+
# truncate excessively long backtraces
|
274
|
+
if backtrace.length > 50
|
275
|
+
backtrace = backtrace[0...25] + ['...'] + backtrace[-25..-1]
|
311
276
|
end
|
312
|
-
|
313
|
-
|
314
|
-
|
315
|
-
exception_pipe.last.flush
|
316
|
-
exception_pipe.last.close
|
317
|
-
exit! 1
|
277
|
+
e2.set_backtrace(backtrace)
|
278
|
+
e2.instance_variable_set(:@active_shards, e.instance_variable_get(:@active_shards))
|
279
|
+
dumped = Marshal.dump(e2)
|
318
280
|
end
|
319
|
-
|
320
|
-
|
321
|
-
|
322
|
-
|
323
|
-
|
324
|
-
|
325
|
-
|
326
|
-
|
327
|
-
|
328
|
-
|
329
|
-
|
330
|
-
|
331
|
-
|
332
|
-
|
333
|
-
|
334
|
-
|
335
|
-
|
336
|
-
|
337
|
-
|
281
|
+
|
282
|
+
exception_pipe.last.set_encoding(dumped.encoding)
|
283
|
+
exception_pipe.last.write(dumped)
|
284
|
+
exception_pipe.last.flush
|
285
|
+
exception_pipe.last.close
|
286
|
+
exit! 1
|
287
|
+
end
|
288
|
+
end)
|
289
|
+
exception_pipe.last.close
|
290
|
+
pids << pid
|
291
|
+
io_in.close # don't care about writing to stdin
|
292
|
+
out_fds << io_out
|
293
|
+
err_fds << io_err
|
294
|
+
pid_to_name_map[pid] = name
|
295
|
+
fd_to_name_map[io_out] = name
|
296
|
+
fd_to_name_map[io_err] = name
|
297
|
+
|
298
|
+
while pids.count >= parallel
|
299
|
+
while out_fds.count >= parallel
|
300
|
+
# wait for output if we've hit the parallel limit
|
301
|
+
wait_for_output.call(out_fds, err_fds, fd_to_name_map)
|
338
302
|
end
|
303
|
+
# we've gotten all the output from one fd so wait for its child process to exit
|
304
|
+
found_pid, status = Process.wait2
|
305
|
+
pids.delete(found_pid)
|
306
|
+
errors << pid_to_name_map[found_pid] if status.exitstatus != 0
|
339
307
|
end
|
340
308
|
end
|
341
309
|
|
@@ -549,24 +517,6 @@ module Switchman
|
|
549
517
|
shard || source_shard || Shard.current
|
550
518
|
end
|
551
519
|
|
552
|
-
# given the provided option, determines whether we need to (and whether
|
553
|
-
# it's possible) to determine a reasonable default.
|
554
|
-
def determine_max_procs(max_procs_input, parallel_input=2)
|
555
|
-
max_procs = nil
|
556
|
-
if max_procs_input
|
557
|
-
max_procs = max_procs_input.to_i
|
558
|
-
max_procs = nil if max_procs == 0
|
559
|
-
else
|
560
|
-
return 1 if parallel_input.nil? || parallel_input < 1
|
561
|
-
cpus = Environment.cpu_count
|
562
|
-
if cpus && cpus > 0
|
563
|
-
max_procs = cpus * parallel_input
|
564
|
-
end
|
565
|
-
end
|
566
|
-
|
567
|
-
return max_procs
|
568
|
-
end
|
569
|
-
|
570
520
|
private
|
571
521
|
# in-process caching
|
572
522
|
def cached_shards
|
@@ -90,7 +90,7 @@ module Switchman
|
|
90
90
|
def define_method_original_attribute(attr_name)
|
91
91
|
if sharded_column?(attr_name)
|
92
92
|
reflection = reflection_for_integer_attribute(attr_name)
|
93
|
-
if attr_name == "id"
|
93
|
+
if attr_name == "id"
|
94
94
|
return if self.method_defined?(:original_id)
|
95
95
|
owner = self
|
96
96
|
else
|
@@ -20,10 +20,7 @@ module Switchman
|
|
20
20
|
shard = " [#{shard[:database_server_id]}:#{shard[:id]} #{shard[:env]}]" if shard
|
21
21
|
|
22
22
|
unless (payload[:binds] || []).empty?
|
23
|
-
|
24
|
-
args = use_old_format ?
|
25
|
-
[payload[:binds], payload[:type_casted_binds]] :
|
26
|
-
[payload[:type_casted_binds]]
|
23
|
+
args = [payload[:type_casted_binds]]
|
27
24
|
casted_params = type_casted_binds(*args)
|
28
25
|
binds = " " + payload[:binds].zip(casted_params).map { |attr, value|
|
29
26
|
render_bind(attr, value)
|
@@ -30,10 +30,13 @@ module Switchman
|
|
30
30
|
end
|
31
31
|
|
32
32
|
module Migrator
|
33
|
-
# significant change:
|
33
|
+
# significant change: just return MIGRATOR_SALT directly
|
34
|
+
# especially if you're going through pgbouncer, the database
|
35
|
+
# name you're accessing may not be consistent. it is NOT allowed
|
36
|
+
# to run migrations against multiple shards in the same database
|
37
|
+
# concurrently
|
34
38
|
def generate_migrator_advisory_lock_id
|
35
|
-
|
36
|
-
::ActiveRecord::Migrator::MIGRATOR_SALT * shard_name_hash
|
39
|
+
::ActiveRecord::Migrator::MIGRATOR_SALT
|
37
40
|
end
|
38
41
|
|
39
42
|
if ::Rails.version >= '6.0'
|
@@ -19,7 +19,7 @@ module Switchman
|
|
19
19
|
connection_id: object_id,
|
20
20
|
cached: true
|
21
21
|
}
|
22
|
-
args[:type_casted_binds] = -> { type_casted_binds(binds) }
|
22
|
+
args[:type_casted_binds] = -> { type_casted_binds(binds) }
|
23
23
|
::ActiveSupport::Notifications.instrument(
|
24
24
|
"sql.active_record",
|
25
25
|
args
|
@@ -78,6 +78,10 @@ module Switchman
|
|
78
78
|
end
|
79
79
|
end
|
80
80
|
|
81
|
+
def or(other)
|
82
|
+
super(other.shard(self.primary_shard))
|
83
|
+
end
|
84
|
+
|
81
85
|
private
|
82
86
|
|
83
87
|
if ::Rails.version >= '5.2'
|
@@ -258,6 +262,25 @@ module Switchman
|
|
258
262
|
end)
|
259
263
|
end
|
260
264
|
|
265
|
+
if predicate.is_a?(::Arel::Nodes::Grouping)
|
266
|
+
next predicate unless predicate.expr.is_a?(::Arel::Nodes::Or)
|
267
|
+
|
268
|
+
or_expr = predicate.expr
|
269
|
+
left_node = or_expr.left
|
270
|
+
right_node = or_expr.right
|
271
|
+
new_left_predicates, binds = transpose_predicates([left_node], source_shard, target_shard,
|
272
|
+
remove_nonlocal_primary_keys,
|
273
|
+
binds: binds,
|
274
|
+
dup_binds_on_mutation: dup_binds_on_mutation)
|
275
|
+
new_right_predicates, binds = transpose_predicates([right_node], source_shard, target_shard,
|
276
|
+
remove_nonlocal_primary_keys,
|
277
|
+
binds: binds,
|
278
|
+
dup_binds_on_mutation: dup_binds_on_mutation)
|
279
|
+
|
280
|
+
next predicate if new_left_predicates[0] == left_node && new_right_predicates[0] == right_node
|
281
|
+
next ::Arel::Nodes::Grouping.new ::Arel::Nodes::Or.new(new_left_predicates[0], new_right_predicates[0])
|
282
|
+
end
|
283
|
+
|
261
284
|
next predicate unless predicate.is_a?(::Arel::Nodes::Binary)
|
262
285
|
next predicate unless predicate.left.is_a?(::Arel::Attributes::Attribute)
|
263
286
|
relation, column = relation_and_column(predicate.left)
|
data/lib/switchman/version.rb
CHANGED
data/lib/tasks/switchman.rake
CHANGED
@@ -46,7 +46,8 @@ module Switchman
|
|
46
46
|
end
|
47
47
|
|
48
48
|
def self.options
|
49
|
-
|
49
|
+
# we still pass through both of these options for back-compat purposes
|
50
|
+
{ parallel: ENV['PARALLEL']&.to_i, max_procs: ENV['MAX_PARALLEL_PROCS']&.to_i }
|
50
51
|
end
|
51
52
|
|
52
53
|
# categories - an array or proc, to activate as the current shard during the
|
@@ -89,7 +90,7 @@ module Switchman
|
|
89
90
|
nil
|
90
91
|
end
|
91
92
|
rescue => e
|
92
|
-
puts "Exception from #{e.current_shard.id}: #{e.current_shard.description}" if options[:parallel] != 0
|
93
|
+
puts "Exception from #{e.current_shard.id}: #{e.current_shard.description}" if options[:parallel].to_i != 0
|
93
94
|
raise
|
94
95
|
end
|
95
96
|
end
|
metadata
CHANGED
@@ -1,16 +1,16 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: switchman
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.
|
4
|
+
version: 2.2.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Cody Cutrer
|
8
8
|
- James Williams
|
9
9
|
- Jacob Fugal
|
10
|
-
autorequire:
|
10
|
+
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date:
|
13
|
+
date: 2022-03-09 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: railties
|
@@ -18,7 +18,7 @@ dependencies:
|
|
18
18
|
requirements:
|
19
19
|
- - ">="
|
20
20
|
- !ruby/object:Gem::Version
|
21
|
-
version: '5.
|
21
|
+
version: '5.2'
|
22
22
|
- - "<"
|
23
23
|
- !ruby/object:Gem::Version
|
24
24
|
version: '6.1'
|
@@ -28,7 +28,7 @@ dependencies:
|
|
28
28
|
requirements:
|
29
29
|
- - ">="
|
30
30
|
- !ruby/object:Gem::Version
|
31
|
-
version: '5.
|
31
|
+
version: '5.2'
|
32
32
|
- - "<"
|
33
33
|
- !ruby/object:Gem::Version
|
34
34
|
version: '6.1'
|
@@ -38,7 +38,7 @@ dependencies:
|
|
38
38
|
requirements:
|
39
39
|
- - ">="
|
40
40
|
- !ruby/object:Gem::Version
|
41
|
-
version: '5.
|
41
|
+
version: '5.2'
|
42
42
|
- - "<"
|
43
43
|
- !ruby/object:Gem::Version
|
44
44
|
version: '6.1'
|
@@ -48,7 +48,7 @@ dependencies:
|
|
48
48
|
requirements:
|
49
49
|
- - ">="
|
50
50
|
- !ruby/object:Gem::Version
|
51
|
-
version: '5.
|
51
|
+
version: '5.2'
|
52
52
|
- - "<"
|
53
53
|
- !ruby/object:Gem::Version
|
54
54
|
version: '6.1'
|
@@ -257,7 +257,7 @@ homepage: http://www.instructure.com/
|
|
257
257
|
licenses:
|
258
258
|
- MIT
|
259
259
|
metadata: {}
|
260
|
-
post_install_message:
|
260
|
+
post_install_message:
|
261
261
|
rdoc_options: []
|
262
262
|
require_paths:
|
263
263
|
- lib
|
@@ -265,15 +265,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
265
265
|
requirements:
|
266
266
|
- - ">="
|
267
267
|
- !ruby/object:Gem::Version
|
268
|
-
version: '2.
|
268
|
+
version: '2.6'
|
269
269
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
270
270
|
requirements:
|
271
271
|
- - ">="
|
272
272
|
- !ruby/object:Gem::Version
|
273
273
|
version: '0'
|
274
274
|
requirements: []
|
275
|
-
rubygems_version: 3.
|
276
|
-
signing_key:
|
275
|
+
rubygems_version: 3.1.4
|
276
|
+
signing_key:
|
277
277
|
specification_version: 4
|
278
278
|
summary: Rails sharding magic
|
279
279
|
test_files: []
|