switchman 3.1.0 → 3.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Rakefile +15 -14
- data/db/migrate/20180828183945_add_default_shard_index.rb +1 -1
- data/db/migrate/20190114212900_add_unique_name_indexes.rb +10 -4
- data/lib/switchman/active_record/associations.rb +97 -17
- data/lib/switchman/active_record/attribute_methods.rb +72 -43
- data/lib/switchman/active_record/base.rb +107 -21
- data/lib/switchman/active_record/calculations.rb +37 -33
- data/lib/switchman/active_record/connection_pool.rb +21 -2
- data/lib/switchman/active_record/database_configurations.rb +12 -7
- data/lib/switchman/active_record/finder_methods.rb +1 -1
- data/lib/switchman/active_record/log_subscriber.rb +2 -2
- data/lib/switchman/active_record/migration.rb +35 -8
- data/lib/switchman/active_record/persistence.rb +8 -0
- data/lib/switchman/active_record/postgresql_adapter.rb +11 -10
- data/lib/switchman/active_record/query_cache.rb +1 -1
- data/lib/switchman/active_record/query_methods.rb +172 -132
- data/lib/switchman/active_record/relation.rb +21 -11
- data/lib/switchman/active_record/spawn_methods.rb +2 -2
- data/lib/switchman/active_record/statement_cache.rb +9 -5
- data/lib/switchman/active_record/tasks/database_tasks.rb +1 -1
- data/lib/switchman/active_record/test_fixtures.rb +19 -16
- data/lib/switchman/active_support/cache.rb +4 -1
- data/lib/switchman/arel.rb +6 -6
- data/lib/switchman/call_super.rb +8 -2
- data/lib/switchman/database_server.rb +21 -26
- data/lib/switchman/default_shard.rb +3 -3
- data/lib/switchman/engine.rb +33 -18
- data/lib/switchman/environment.rb +2 -2
- data/lib/switchman/errors.rb +13 -0
- data/lib/switchman/guard_rail/relation.rb +2 -1
- data/lib/switchman/parallel.rb +2 -2
- data/lib/switchman/r_spec_helper.rb +10 -10
- data/lib/switchman/shard.rb +49 -32
- data/lib/switchman/sharded_instrumenter.rb +5 -1
- data/lib/switchman/shared_schema_cache.rb +11 -0
- data/lib/switchman/test_helper.rb +1 -1
- data/lib/switchman/version.rb +1 -1
- data/lib/switchman.rb +10 -4
- data/lib/tasks/switchman.rake +42 -39
- metadata +24 -9
|
@@ -12,8 +12,6 @@ module Switchman
|
|
|
12
12
|
# :explicit - explicit set on the relation
|
|
13
13
|
# :association - a special value that scopes from associations use to use slightly different logic
|
|
14
14
|
# for foreign key transposition
|
|
15
|
-
# :to_a - a special value that Relation#to_a uses when querying multiple shards to
|
|
16
|
-
# remove primary keys from conditions that aren't applicable to the current shard
|
|
17
15
|
def shard_value
|
|
18
16
|
@values[:shard]
|
|
19
17
|
end
|
|
@@ -44,10 +42,7 @@ module Switchman
|
|
|
44
42
|
old_primary_shard = primary_shard
|
|
45
43
|
self.shard_value = value
|
|
46
44
|
self.shard_source_value = source
|
|
47
|
-
|
|
48
|
-
transpose_clauses(old_primary_shard, primary_shard,
|
|
49
|
-
remove_nonlocal_primary_keys: source == :to_a)
|
|
50
|
-
end
|
|
45
|
+
transpose_predicates(nil, old_primary_shard, primary_shard) if old_primary_shard != primary_shard
|
|
51
46
|
self
|
|
52
47
|
end
|
|
53
48
|
|
|
@@ -89,29 +84,29 @@ module Switchman
|
|
|
89
84
|
super(other.shard(primary_shard))
|
|
90
85
|
end
|
|
91
86
|
|
|
92
|
-
|
|
87
|
+
# use a temp variable so that the new where clause is built before self.where_clause is read,
|
|
88
|
+
# since build_where_clause might mutate self.where_clause
|
|
89
|
+
def where!(opts, *rest)
|
|
90
|
+
new_clause = build_where_clause(opts, rest)
|
|
91
|
+
self.where_clause += new_clause
|
|
92
|
+
self
|
|
93
|
+
end
|
|
93
94
|
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
end
|
|
105
|
-
end
|
|
106
|
-
end
|
|
95
|
+
protected
|
|
96
|
+
|
|
97
|
+
def remove_nonlocal_primary_keys!
|
|
98
|
+
each_transposable_predicate_value do |value, predicate, _relation, _column, type|
|
|
99
|
+
next value unless
|
|
100
|
+
type == :primary &&
|
|
101
|
+
predicate.left.relation.klass == klass &&
|
|
102
|
+
(predicate.is_a?(::Arel::Nodes::Equality) || predicate.is_a?(::Arel::Nodes::HomogeneousIn))
|
|
103
|
+
|
|
104
|
+
(value.is_a?(Integer) && value > Shard::IDS_PER_SHARD) ? [] : value
|
|
107
105
|
end
|
|
108
|
-
|
|
106
|
+
self
|
|
109
107
|
end
|
|
110
108
|
|
|
111
|
-
|
|
112
|
-
transpose_where_clauses(source_shard, target_shard, remove_nonlocal_primary_keys: remove_nonlocal_primary_keys)
|
|
113
|
-
transpose_having_clauses(source_shard, target_shard, remove_nonlocal_primary_keys: remove_nonlocal_primary_keys)
|
|
114
|
-
end
|
|
109
|
+
private
|
|
115
110
|
|
|
116
111
|
def infer_shards_from_primary_key(predicates)
|
|
117
112
|
return unless klass.integral_id?
|
|
@@ -145,7 +140,7 @@ module Switchman
|
|
|
145
140
|
return
|
|
146
141
|
else
|
|
147
142
|
id_shards = id_shards.to_a
|
|
148
|
-
|
|
143
|
+
transpose_predicates(nil, primary_shard, id_shards.first)
|
|
149
144
|
self.shard_value = id_shards
|
|
150
145
|
return
|
|
151
146
|
end
|
|
@@ -162,7 +157,7 @@ module Switchman
|
|
|
162
157
|
|
|
163
158
|
return if !id_shard || id_shard == primary_shard
|
|
164
159
|
|
|
165
|
-
|
|
160
|
+
transpose_predicates(nil, primary_shard, id_shard)
|
|
166
161
|
self.shard_value = id_shard
|
|
167
162
|
end
|
|
168
163
|
|
|
@@ -185,7 +180,7 @@ module Switchman
|
|
|
185
180
|
|
|
186
181
|
def sharded_primary_key?(relation, column)
|
|
187
182
|
column = column.to_s
|
|
188
|
-
return column ==
|
|
183
|
+
return column == "id" if relation.klass == ::ActiveRecord::Base
|
|
189
184
|
|
|
190
185
|
relation.klass.primary_key == column && relation.klass.integral_id?
|
|
191
186
|
end
|
|
@@ -212,12 +207,11 @@ module Switchman
|
|
|
212
207
|
|
|
213
208
|
case opts
|
|
214
209
|
when String, Array
|
|
215
|
-
values = Hash === rest.first ? rest.first.values : rest
|
|
210
|
+
values = (Hash === rest.first) ? rest.first.values : rest
|
|
216
211
|
|
|
217
|
-
values.grep(ActiveRecord::Relation)
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
rel.shard!(primary_shard) if rel.shard_source_value == :implicit && rel.primary_shard != primary_shard
|
|
212
|
+
if values.grep(ActiveRecord::Relation).first
|
|
213
|
+
raise "Sub-queries are not allowed as simple substitutions; " \
|
|
214
|
+
"please build your relation with more structured methods so that Switchman is able to introspect it."
|
|
221
215
|
end
|
|
222
216
|
|
|
223
217
|
super
|
|
@@ -246,122 +240,168 @@ module Switchman
|
|
|
246
240
|
connection.with_global_table_name { super }
|
|
247
241
|
end
|
|
248
242
|
|
|
249
|
-
def
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
transpose_single_predicate(predicate, source_shard, target_shard,
|
|
255
|
-
remove_nonlocal_primary_keys: remove_nonlocal_primary_keys)
|
|
256
|
-
end
|
|
243
|
+
def each_predicate(predicates = nil, &block)
|
|
244
|
+
return predicates.map(&block) if predicates
|
|
245
|
+
|
|
246
|
+
each_predicate_cb(:having_clause, :having_clause=, &block)
|
|
247
|
+
each_predicate_cb(:where_clause, :where_clause=, &block)
|
|
257
248
|
end
|
|
258
249
|
|
|
259
|
-
def
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
if predicate.is_a?(::Arel::Nodes::Grouping)
|
|
264
|
-
return predicate unless predicate.expr.is_a?(::Arel::Nodes::Or)
|
|
265
|
-
|
|
266
|
-
# Dang, we have an OR. OK, that means we have other epxressions below this
|
|
267
|
-
# level, perhaps many, that may need transposition.
|
|
268
|
-
# the left side and right side must each be treated as predicate lists and
|
|
269
|
-
# transformed in kind, if neither of them changes we can just return the grouping as is.
|
|
270
|
-
# hold on, it's about to get recursive...
|
|
271
|
-
or_expr = predicate.expr
|
|
272
|
-
left_node = or_expr.left
|
|
273
|
-
right_node = or_expr.right
|
|
274
|
-
new_left_predicates = transpose_single_predicate(left_node, source_shard,
|
|
275
|
-
target_shard, remove_nonlocal_primary_keys: remove_nonlocal_primary_keys)
|
|
276
|
-
new_right_predicates = transpose_single_predicate(right_node, source_shard,
|
|
277
|
-
target_shard, remove_nonlocal_primary_keys: remove_nonlocal_primary_keys)
|
|
278
|
-
return predicate if new_left_predicates == left_node && new_right_predicates == right_node
|
|
279
|
-
|
|
280
|
-
return ::Arel::Nodes::Grouping.new ::Arel::Nodes::Or.new(new_left_predicates, new_right_predicates)
|
|
281
|
-
end
|
|
282
|
-
return predicate unless predicate.is_a?(::Arel::Nodes::Binary) || predicate.is_a?(::Arel::Nodes::HomogeneousIn)
|
|
283
|
-
return predicate unless predicate.left.is_a?(::Arel::Attributes::Attribute)
|
|
284
|
-
|
|
285
|
-
relation, column = relation_and_column(predicate.left)
|
|
286
|
-
return predicate unless (type = transposable_attribute_type(relation, column))
|
|
287
|
-
|
|
288
|
-
remove = true if type == :primary &&
|
|
289
|
-
remove_nonlocal_primary_keys &&
|
|
290
|
-
predicate.left.relation.klass == klass &&
|
|
291
|
-
(predicate.is_a?(::Arel::Nodes::Equality) || predicate.is_a?(::Arel::Nodes::HomogeneousIn))
|
|
292
|
-
|
|
293
|
-
current_source_shard =
|
|
294
|
-
if source_shard
|
|
295
|
-
source_shard
|
|
296
|
-
elsif type == :primary
|
|
297
|
-
Shard.current(klass.connection_class_for_self)
|
|
298
|
-
elsif type == :foreign
|
|
299
|
-
source_shard_for_foreign_key(relation, column)
|
|
300
|
-
end
|
|
250
|
+
def each_predicate_cb(clause_getter, clause_setter, &block)
|
|
251
|
+
old_clause = send(clause_getter)
|
|
252
|
+
old_predicates = old_clause.send(:predicates)
|
|
253
|
+
return if old_predicates.empty?
|
|
301
254
|
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
else
|
|
305
|
-
predicate.right
|
|
306
|
-
end
|
|
255
|
+
new_predicates = old_predicates.map(&block)
|
|
256
|
+
return if new_predicates == old_predicates
|
|
307
257
|
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
258
|
+
new_clause = old_clause.dup
|
|
259
|
+
new_clause.instance_variable_set(:@predicates, new_predicates)
|
|
260
|
+
|
|
261
|
+
send(clause_setter, new_clause)
|
|
262
|
+
end
|
|
263
|
+
|
|
264
|
+
def each_transposable_predicate(predicates, &block)
|
|
265
|
+
each_predicate(predicates) do |predicate|
|
|
266
|
+
case predicate
|
|
267
|
+
when ::Arel::Nodes::Grouping
|
|
268
|
+
next predicate unless predicate.expr.is_a?(::Arel::Nodes::Or)
|
|
269
|
+
|
|
270
|
+
or_expr = predicate.expr
|
|
271
|
+
old_left = or_expr.left
|
|
272
|
+
old_right = or_expr.right
|
|
273
|
+
new_left, new_right = each_transposable_predicate([old_left, old_right], &block)
|
|
274
|
+
|
|
275
|
+
next predicate if new_left == old_left && new_right == old_right
|
|
276
|
+
|
|
277
|
+
next predicate.class.new predicate.expr.class.new(new_left, new_right)
|
|
278
|
+
when ::Arel::Nodes::SelectStatement
|
|
279
|
+
new_cores = predicate.cores.map do |core|
|
|
280
|
+
next core unless core.is_a?(::Arel::Nodes::SelectCore) # just in case something weird is going on
|
|
281
|
+
|
|
282
|
+
new_wheres = each_transposable_predicate(core.wheres, &block)
|
|
283
|
+
new_havings = each_transposable_predicate(core.havings, &block)
|
|
284
|
+
|
|
285
|
+
next core if core.wheres == new_wheres && core.havings == new_havings
|
|
286
|
+
|
|
287
|
+
new_core = core.clone
|
|
288
|
+
new_core.wheres = new_wheres
|
|
289
|
+
new_core.havings = new_havings
|
|
290
|
+
new_core
|
|
291
|
+
end
|
|
292
|
+
|
|
293
|
+
next predicate if predicate.cores == new_cores
|
|
294
|
+
|
|
295
|
+
new_node = predicate.clone
|
|
296
|
+
new_node.instance_variable_set(:@cores, new_cores)
|
|
297
|
+
next new_node
|
|
298
|
+
when ::Arel::Nodes::Not
|
|
299
|
+
old_value = predicate.expr
|
|
300
|
+
new_value = each_transposable_predicate([old_value], &block).first
|
|
301
|
+
|
|
302
|
+
next predicate if old_value == new_value
|
|
303
|
+
|
|
304
|
+
next predicate.class.new(new_value)
|
|
305
|
+
when ::Arel::Nodes::Exists
|
|
306
|
+
old_value = predicate.expressions
|
|
307
|
+
new_value = each_transposable_predicate([old_value], &block).first
|
|
308
|
+
|
|
309
|
+
next predicate if old_value == new_value
|
|
310
|
+
|
|
311
|
+
next predicate.class.new(new_value)
|
|
314
312
|
end
|
|
315
313
|
|
|
316
|
-
|
|
317
|
-
predicate
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
314
|
+
next predicate unless predicate.is_a?(::Arel::Nodes::Binary) || predicate.is_a?(::Arel::Nodes::HomogeneousIn)
|
|
315
|
+
next predicate unless predicate.left.is_a?(::Arel::Attributes::Attribute)
|
|
316
|
+
|
|
317
|
+
relation, column = relation_and_column(predicate.left)
|
|
318
|
+
next predicate unless (type = transposable_attribute_type(relation, column))
|
|
319
|
+
|
|
320
|
+
yield(predicate, relation, column, type)
|
|
321
|
+
end
|
|
322
|
+
end
|
|
323
|
+
|
|
324
|
+
def each_transposable_predicate_value(predicates = nil, &block)
|
|
325
|
+
each_transposable_predicate(predicates) do |predicate, relation, column, type|
|
|
326
|
+
each_transposable_predicate_value_cb(predicate, block) do |value|
|
|
327
|
+
yield(value, predicate, relation, column, type)
|
|
323
328
|
end
|
|
324
|
-
|
|
329
|
+
end
|
|
330
|
+
end
|
|
331
|
+
|
|
332
|
+
def each_transposable_predicate_value_cb(node, original_block, &block)
|
|
333
|
+
case node
|
|
334
|
+
when Array
|
|
335
|
+
node.filter_map { |val| each_transposable_predicate_value_cb(val, original_block, &block).presence }
|
|
336
|
+
when ::ActiveModel::Attribute
|
|
337
|
+
old_value = node.value_before_type_cast
|
|
338
|
+
new_value = each_transposable_predicate_value_cb(old_value, original_block, &block)
|
|
339
|
+
|
|
340
|
+
(old_value == new_value) ? node : node.class.new(node.name, new_value, node.type)
|
|
341
|
+
when ::Arel::Nodes::And
|
|
342
|
+
old_value = node.children
|
|
343
|
+
new_value = each_transposable_predicate_value_cb(old_value, original_block, &block)
|
|
344
|
+
|
|
345
|
+
(old_value == new_value) ? node : node.class.new(new_value)
|
|
346
|
+
when ::Arel::Nodes::BindParam
|
|
347
|
+
old_value = node.value
|
|
348
|
+
new_value = each_transposable_predicate_value_cb(old_value, original_block, &block)
|
|
349
|
+
|
|
350
|
+
(old_value == new_value) ? node : node.class.new(new_value)
|
|
351
|
+
when ::Arel::Nodes::Casted
|
|
352
|
+
old_value = node.value
|
|
353
|
+
new_value = each_transposable_predicate_value_cb(old_value, original_block, &block)
|
|
354
|
+
|
|
355
|
+
(old_value == new_value) ? node : node.class.new(new_value, node.attribute)
|
|
356
|
+
when ::Arel::Nodes::HomogeneousIn
|
|
357
|
+
old_value = node.values
|
|
358
|
+
new_value = each_transposable_predicate_value_cb(old_value, original_block, &block)
|
|
359
|
+
|
|
325
360
|
# switch to a regular In, so that Relation::WhereClause#contradiction? knows about it
|
|
326
|
-
if
|
|
327
|
-
klass =
|
|
328
|
-
klass.new(
|
|
361
|
+
if new_value.empty?
|
|
362
|
+
klass = (node.type == :in) ? ::Arel::Nodes::In : ::Arel::Nodes::NotIn
|
|
363
|
+
klass.new(node.attribute, new_value)
|
|
329
364
|
else
|
|
330
|
-
|
|
365
|
+
(old_value == new_value) ? node : node.class.new(new_value, node.attribute, node.type)
|
|
331
366
|
end
|
|
367
|
+
when ::Arel::Nodes::Binary
|
|
368
|
+
old_value = node.right
|
|
369
|
+
new_value = each_transposable_predicate_value_cb(old_value, original_block, &block)
|
|
370
|
+
|
|
371
|
+
(old_value == new_value) ? node : node.class.new(node.left, new_value)
|
|
372
|
+
when ::Arel::Nodes::SelectStatement
|
|
373
|
+
each_transposable_predicate_value([node], &original_block).first
|
|
332
374
|
else
|
|
333
|
-
|
|
375
|
+
yield(node)
|
|
334
376
|
end
|
|
335
377
|
end
|
|
336
378
|
|
|
337
|
-
def
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
local_id = [] if remove_non_local_ids && local_id.is_a?(Integer) && local_id > Shard::IDS_PER_SHARD
|
|
349
|
-
if current_id == local_id
|
|
350
|
-
# make a new bind param
|
|
351
|
-
value
|
|
379
|
+
def transpose_predicates(predicates,
|
|
380
|
+
source_shard,
|
|
381
|
+
target_shard)
|
|
382
|
+
each_transposable_predicate_value(predicates) do |value, _predicate, relation, column, type|
|
|
383
|
+
current_source_shard =
|
|
384
|
+
if source_shard
|
|
385
|
+
source_shard
|
|
386
|
+
elsif type == :primary
|
|
387
|
+
Shard.current(klass.connection_class_for_self)
|
|
388
|
+
elsif type == :foreign
|
|
389
|
+
source_shard_for_foreign_key(relation, column)
|
|
352
390
|
else
|
|
353
|
-
|
|
354
|
-
if value.is_a?(::ActiveModel::Attribute)
|
|
355
|
-
new_att
|
|
356
|
-
else
|
|
357
|
-
::Arel::Nodes::BindParam.new(new_att)
|
|
358
|
-
end
|
|
391
|
+
primary_shard
|
|
359
392
|
end
|
|
360
|
-
|
|
393
|
+
|
|
394
|
+
transpose_predicate_value(value, current_source_shard, target_shard, type)
|
|
395
|
+
end
|
|
396
|
+
end
|
|
397
|
+
|
|
398
|
+
def transpose_predicate_value(value, current_shard, target_shard, attribute_type)
|
|
399
|
+
if value.is_a?(::ActiveRecord::StatementCache::Substitute)
|
|
400
|
+
value.sharded = true # mark for transposition later
|
|
401
|
+
value.primary = true if attribute_type == :primary
|
|
402
|
+
value
|
|
361
403
|
else
|
|
362
|
-
|
|
363
|
-
local_id = [] if remove_non_local_ids && local_id.is_a?(Integer) && local_id > Shard::IDS_PER_SHARD
|
|
364
|
-
local_id
|
|
404
|
+
Shard.relative_id_for(value, current_shard, target_shard) || value
|
|
365
405
|
end
|
|
366
406
|
end
|
|
367
407
|
end
|
|
@@ -4,7 +4,7 @@ module Switchman
|
|
|
4
4
|
module ActiveRecord
|
|
5
5
|
module Relation
|
|
6
6
|
def self.prepended(klass)
|
|
7
|
-
klass::SINGLE_VALUE_METHODS.
|
|
7
|
+
klass::SINGLE_VALUE_METHODS.push(:shard, :shard_source)
|
|
8
8
|
end
|
|
9
9
|
|
|
10
10
|
def initialize(*, **)
|
|
@@ -49,7 +49,7 @@ module Switchman
|
|
|
49
49
|
end
|
|
50
50
|
|
|
51
51
|
def load(&block)
|
|
52
|
-
if !loaded? || (::Rails.version >=
|
|
52
|
+
if !loaded? || (::Rails.version >= "7.0" && scheduled?)
|
|
53
53
|
@records = activate { |relation| relation.send(:exec_queries, &block) }
|
|
54
54
|
@loaded = true
|
|
55
55
|
end
|
|
@@ -58,9 +58,10 @@ module Switchman
|
|
|
58
58
|
end
|
|
59
59
|
|
|
60
60
|
%I[update_all delete_all].each do |method|
|
|
61
|
+
arg_params = (RUBY_VERSION <= "2.8") ? "*args" : "*args, **kwargs"
|
|
61
62
|
class_eval <<-RUBY, __FILE__, __LINE__ + 1
|
|
62
|
-
def #{method}(
|
|
63
|
-
result = self.activate(unordered: true) { |relation| relation.call_super(#{method.inspect}, Relation,
|
|
63
|
+
def #{method}(#{arg_params})
|
|
64
|
+
result = self.activate(unordered: true) { |relation| relation.call_super(#{method.inspect}, Relation, #{arg_params}) }
|
|
64
65
|
result = result.sum if result.is_a?(Array)
|
|
65
66
|
result
|
|
66
67
|
end
|
|
@@ -72,12 +73,16 @@ module Switchman
|
|
|
72
73
|
loose_mode = options[:loose] && is_integer
|
|
73
74
|
# loose_mode: if we don't care about getting exactly batch_size ids in between
|
|
74
75
|
# don't get the max - just get the min and add batch_size so we get that many _at most_
|
|
75
|
-
values = loose_mode ?
|
|
76
|
+
values = loose_mode ? "MIN(id)" : "MIN(id), MAX(id)"
|
|
76
77
|
|
|
77
78
|
batch_size = options[:batch_size].try(:to_i) || 1000
|
|
78
|
-
quoted_primary_key =
|
|
79
|
-
|
|
80
|
-
|
|
79
|
+
quoted_primary_key =
|
|
80
|
+
"#{klass.connection.quote_local_table_name(table_name)}.#{klass.connection.quote_column_name(primary_key)}"
|
|
81
|
+
as_id = " AS id" unless primary_key == "id"
|
|
82
|
+
subquery_scope = except(:select)
|
|
83
|
+
.select("#{quoted_primary_key}#{as_id}")
|
|
84
|
+
.reorder(primary_key.to_sym)
|
|
85
|
+
.limit(loose_mode ? 1 : batch_size)
|
|
81
86
|
subquery_scope = subquery_scope.where("#{quoted_primary_key} <= ?", options[:end_at]) if options[:end_at]
|
|
82
87
|
|
|
83
88
|
first_subquery_scope = if options[:start_at]
|
|
@@ -103,7 +108,9 @@ module Switchman
|
|
|
103
108
|
def activate(unordered: false, &block)
|
|
104
109
|
shards = all_shards
|
|
105
110
|
if Array === shards && shards.length == 1
|
|
106
|
-
if
|
|
111
|
+
if !loaded? && shard_value != shards.first
|
|
112
|
+
shard(shards.first).activate(&block)
|
|
113
|
+
elsif shards.first == DefaultShard || shards.first == Shard.current(klass.connection_class_for_self)
|
|
107
114
|
yield(self, shards.first)
|
|
108
115
|
else
|
|
109
116
|
shards.first.activate(klass.connection_class_for_self) { yield(self, shards.first) }
|
|
@@ -115,9 +122,12 @@ module Switchman
|
|
|
115
122
|
# don't even query other shards if we're already past the limit
|
|
116
123
|
next if limit_value && result_count >= limit_value && order_values.empty?
|
|
117
124
|
|
|
118
|
-
relation = shard(Shard.current(klass.connection_class_for_self)
|
|
125
|
+
relation = shard(Shard.current(klass.connection_class_for_self))
|
|
126
|
+
relation.remove_nonlocal_primary_keys!
|
|
119
127
|
# do a minimal query if possible
|
|
120
|
-
|
|
128
|
+
if limit_value && !result_count.zero? && order_values.empty?
|
|
129
|
+
relation = relation.limit(limit_value - result_count)
|
|
130
|
+
end
|
|
121
131
|
|
|
122
132
|
shard_results = relation.activate(&block)
|
|
123
133
|
|
|
@@ -17,7 +17,7 @@ module Switchman
|
|
|
17
17
|
final_shard_source_value = %i[explicit association].detect do |source_value|
|
|
18
18
|
shard_source_value == source_value || rhs.shard_source_value == source_value
|
|
19
19
|
end
|
|
20
|
-
raise
|
|
20
|
+
raise "unknown shard_source_value" unless final_shard_source_value
|
|
21
21
|
|
|
22
22
|
# have to merge shard_value
|
|
23
23
|
lhs_shard_value = all_shards
|
|
@@ -36,7 +36,7 @@ module Switchman
|
|
|
36
36
|
final_shard_source_value = %i[explicit association implicit].detect do |source_value|
|
|
37
37
|
shard_source_value == source_value || rhs.shard_source_value == source_value
|
|
38
38
|
end
|
|
39
|
-
raise
|
|
39
|
+
raise "unknown shard_source_value" unless final_shard_source_value
|
|
40
40
|
end
|
|
41
41
|
|
|
42
42
|
[final_shard_value, final_primary_shard, final_shard_source_value]
|
|
@@ -4,8 +4,8 @@ module Switchman
|
|
|
4
4
|
module ActiveRecord
|
|
5
5
|
module StatementCache
|
|
6
6
|
module ClassMethods
|
|
7
|
-
def create(connection
|
|
8
|
-
relation =
|
|
7
|
+
def create(connection)
|
|
8
|
+
relation = yield ::ActiveRecord::StatementCache::Params.new
|
|
9
9
|
|
|
10
10
|
_query_builder, binds = connection.cacheable_query(self, relation.arel)
|
|
11
11
|
bind_map = ::ActiveRecord::StatementCache::BindMap.new(binds)
|
|
@@ -25,7 +25,7 @@ module Switchman
|
|
|
25
25
|
# we can make some assumptions about the shard source
|
|
26
26
|
# (e.g. infer from the primary key or use the current shard)
|
|
27
27
|
|
|
28
|
-
def execute(*args)
|
|
28
|
+
def execute(*args, &block)
|
|
29
29
|
params, connection = args
|
|
30
30
|
klass = @klass
|
|
31
31
|
target_shard = nil
|
|
@@ -40,7 +40,7 @@ module Switchman
|
|
|
40
40
|
|
|
41
41
|
target_shard.activate(klass.connection_class_for_self) do
|
|
42
42
|
sql = qualified_query_builder(target_shard, klass).sql_for(bind_values, connection)
|
|
43
|
-
klass.find_by_sql(sql, bind_values)
|
|
43
|
+
klass.find_by_sql(sql, bind_values, &block)
|
|
44
44
|
end
|
|
45
45
|
end
|
|
46
46
|
|
|
@@ -66,7 +66,11 @@ module Switchman
|
|
|
66
66
|
|
|
67
67
|
def primary_value_index
|
|
68
68
|
primary_ba_index = @bound_attributes.index do |ba|
|
|
69
|
-
ba.is_a?(::ActiveRecord::
|
|
69
|
+
if ba.value.is_a?(::ActiveRecord::StatementCache::Substitute)
|
|
70
|
+
ba.is_a?(::ActiveRecord::Relation::QueryAttribute) && ba.value.primary
|
|
71
|
+
else
|
|
72
|
+
false
|
|
73
|
+
end
|
|
70
74
|
end
|
|
71
75
|
@indexes.index(primary_ba_index) if primary_ba_index
|
|
72
76
|
end
|
|
@@ -12,31 +12,34 @@ module Switchman
|
|
|
12
12
|
# Replace the one that activerecord natively uses with a switchman-optimized one
|
|
13
13
|
::ActiveSupport::Notifications.unsubscribe(@connection_subscriber)
|
|
14
14
|
# Code adapted from the code in rails proper
|
|
15
|
-
@connection_subscriber =
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
15
|
+
@connection_subscriber =
|
|
16
|
+
::ActiveSupport::Notifications.subscribe("!connection.active_record") do |_, _, _, _, payload|
|
|
17
|
+
spec_name = payload[:spec_name] if payload.key?(:spec_name)
|
|
18
|
+
shard = payload[:shard] if payload.key?(:shard)
|
|
19
|
+
setup_shared_connection_pool
|
|
19
20
|
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
21
|
+
if spec_name && !FORBIDDEN_DB_ENVS.include?(shard)
|
|
22
|
+
begin
|
|
23
|
+
connection = ::ActiveRecord::Base.connection_handler.retrieve_connection(spec_name, shard: shard)
|
|
24
|
+
rescue ::ActiveRecord::ConnectionNotEstablished, ::ActiveRecord::NoDatabaseError
|
|
25
|
+
connection = nil
|
|
26
|
+
end
|
|
26
27
|
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
28
|
+
if connection && !@fixture_connections.include?(connection)
|
|
29
|
+
connection.begin_transaction joinable: false, _lazy: false
|
|
30
|
+
connection.pool.lock_thread = true if lock_threads
|
|
31
|
+
@fixture_connections << connection
|
|
32
|
+
end
|
|
31
33
|
end
|
|
32
34
|
end
|
|
33
|
-
end
|
|
34
35
|
end
|
|
35
36
|
|
|
36
37
|
def enlist_fixture_connections
|
|
37
38
|
setup_shared_connection_pool
|
|
38
39
|
|
|
39
|
-
::ActiveRecord::Base.connection_handler.connection_pool_list.reject
|
|
40
|
+
::ActiveRecord::Base.connection_handler.connection_pool_list.reject do |cp|
|
|
41
|
+
FORBIDDEN_DB_ENVS.include?(cp.db_config.env_name.to_sym)
|
|
42
|
+
end.map(&:connection)
|
|
40
43
|
end
|
|
41
44
|
end
|
|
42
45
|
end
|
|
@@ -24,7 +24,10 @@ module Switchman
|
|
|
24
24
|
store = super
|
|
25
25
|
# can't use defined?, because it's a _ruby_ autoloaded constant,
|
|
26
26
|
# so just checking that will cause it to get required
|
|
27
|
-
|
|
27
|
+
if store.instance_of?(ActiveSupport::Cache::RedisCacheStore) &&
|
|
28
|
+
!::ActiveSupport::Cache::RedisCacheStore <= RedisCacheStore
|
|
29
|
+
::ActiveSupport::Cache::RedisCacheStore.prepend(RedisCacheStore)
|
|
30
|
+
end
|
|
28
31
|
store.options[:namespace] ||= -> { Shard.current.default? ? nil : "shard_#{Shard.current.id}" }
|
|
29
32
|
store
|
|
30
33
|
end
|
data/lib/switchman/arel.rb
CHANGED
|
@@ -15,24 +15,24 @@ module Switchman
|
|
|
15
15
|
|
|
16
16
|
def visit_Arel_Nodes_TableAlias(o, collector)
|
|
17
17
|
collector = visit o.relation, collector
|
|
18
|
-
collector <<
|
|
18
|
+
collector << " "
|
|
19
19
|
collector << quote_local_table_name(o.name)
|
|
20
20
|
end
|
|
21
21
|
|
|
22
22
|
def visit_Arel_Attributes_Attribute(o, collector)
|
|
23
23
|
join_name = o.relation.table_alias || o.relation.name
|
|
24
|
-
collector << quote_local_table_name(join_name) <<
|
|
24
|
+
collector << quote_local_table_name(join_name) << "." << quote_column_name(o.name)
|
|
25
25
|
end
|
|
26
26
|
|
|
27
27
|
def visit_Arel_Nodes_HomogeneousIn(o, collector)
|
|
28
28
|
collector.preparable = false
|
|
29
29
|
|
|
30
|
-
collector << quote_local_table_name(o.table_name) <<
|
|
30
|
+
collector << quote_local_table_name(o.table_name) << "." << quote_column_name(o.column_name)
|
|
31
31
|
|
|
32
32
|
collector << if o.type == :in
|
|
33
|
-
|
|
33
|
+
" IN ("
|
|
34
34
|
else
|
|
35
|
-
|
|
35
|
+
" NOT IN ("
|
|
36
36
|
end
|
|
37
37
|
|
|
38
38
|
values = o.casted_values
|
|
@@ -43,7 +43,7 @@ module Switchman
|
|
|
43
43
|
collector.add_binds(values, o.proc_for_binds, &bind_block)
|
|
44
44
|
end
|
|
45
45
|
|
|
46
|
-
collector <<
|
|
46
|
+
collector << ")"
|
|
47
47
|
collector
|
|
48
48
|
end
|
|
49
49
|
|
data/lib/switchman/call_super.rb
CHANGED
|
@@ -12,8 +12,14 @@ module Switchman
|
|
|
12
12
|
method.super_method
|
|
13
13
|
end
|
|
14
14
|
|
|
15
|
-
|
|
16
|
-
|
|
15
|
+
if RUBY_VERSION <= "2.8"
|
|
16
|
+
def call_super(method, above_module, *args, &block)
|
|
17
|
+
super_method_above(method, above_module).call(*args, &block)
|
|
18
|
+
end
|
|
19
|
+
else
|
|
20
|
+
def call_super(method, above_module, *args, **kwargs, &block)
|
|
21
|
+
super_method_above(method, above_module).call(*args, **kwargs, &block)
|
|
22
|
+
end
|
|
17
23
|
end
|
|
18
24
|
end
|
|
19
25
|
end
|