switchman 3.0.5 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Rakefile +16 -15
- data/db/migrate/20180828183945_add_default_shard_index.rb +2 -2
- data/db/migrate/20180828192111_add_timestamps_to_shards.rb +1 -1
- data/db/migrate/20190114212900_add_unique_name_indexes.rb +10 -4
- data/lib/switchman/action_controller/caching.rb +2 -2
- data/lib/switchman/active_record/abstract_adapter.rb +6 -15
- data/lib/switchman/active_record/associations.rb +331 -0
- data/lib/switchman/active_record/attribute_methods.rb +182 -77
- data/lib/switchman/active_record/base.rb +249 -46
- data/lib/switchman/active_record/calculations.rb +98 -44
- data/lib/switchman/active_record/connection_handler.rb +18 -0
- data/lib/switchman/active_record/connection_pool.rb +27 -28
- data/lib/switchman/active_record/database_configurations.rb +44 -6
- data/lib/switchman/active_record/finder_methods.rb +46 -16
- data/lib/switchman/active_record/log_subscriber.rb +11 -5
- data/lib/switchman/active_record/migration.rb +52 -5
- data/lib/switchman/active_record/model_schema.rb +1 -1
- data/lib/switchman/active_record/pending_migration_connection.rb +17 -0
- data/lib/switchman/active_record/persistence.rb +37 -2
- data/lib/switchman/active_record/postgresql_adapter.rb +12 -11
- data/lib/switchman/active_record/predicate_builder.rb +2 -2
- data/lib/switchman/active_record/query_cache.rb +49 -20
- data/lib/switchman/active_record/query_methods.rb +202 -136
- data/lib/switchman/active_record/reflection.rb +1 -1
- data/lib/switchman/active_record/relation.rb +40 -28
- data/lib/switchman/active_record/spawn_methods.rb +2 -2
- data/lib/switchman/active_record/statement_cache.rb +11 -7
- data/lib/switchman/active_record/table_definition.rb +1 -1
- data/lib/switchman/active_record/tasks/database_tasks.rb +6 -1
- data/lib/switchman/active_record/test_fixtures.rb +53 -0
- data/lib/switchman/active_support/cache.rb +25 -4
- data/lib/switchman/arel.rb +45 -7
- data/lib/switchman/call_super.rb +2 -2
- data/lib/switchman/database_server.rb +123 -79
- data/lib/switchman/default_shard.rb +14 -5
- data/lib/switchman/engine.rb +79 -131
- data/lib/switchman/environment.rb +2 -2
- data/lib/switchman/errors.rb +17 -2
- data/lib/switchman/guard_rail/relation.rb +7 -10
- data/lib/switchman/guard_rail.rb +5 -0
- data/lib/switchman/parallel.rb +68 -0
- data/lib/switchman/r_spec_helper.rb +17 -28
- data/lib/switchman/rails.rb +1 -4
- data/{app/models → lib}/switchman/shard.rb +226 -241
- data/lib/switchman/sharded_instrumenter.rb +3 -3
- data/lib/switchman/shared_schema_cache.rb +11 -0
- data/lib/switchman/standard_error.rb +15 -12
- data/lib/switchman/test_helper.rb +2 -2
- data/{app/models → lib}/switchman/unsharded_record.rb +1 -1
- data/lib/switchman/version.rb +1 -1
- data/lib/switchman.rb +44 -12
- data/lib/tasks/switchman.rake +101 -54
- metadata +50 -58
- data/lib/switchman/active_record/association.rb +0 -206
- data/lib/switchman/open4.rb +0 -80
|
@@ -3,6 +3,19 @@
|
|
|
3
3
|
module Switchman
|
|
4
4
|
module ActiveRecord
|
|
5
5
|
module QueryMethods
|
|
6
|
+
# Use this class to prevent a value from getting transposed across shards
|
|
7
|
+
class NonTransposingValue < SimpleDelegator
|
|
8
|
+
def class
|
|
9
|
+
__getobj__.class
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
def is_a?(other)
|
|
13
|
+
return true if other == NonTransposingValue
|
|
14
|
+
|
|
15
|
+
__getobj__.is_a?(other)
|
|
16
|
+
end
|
|
17
|
+
end
|
|
18
|
+
|
|
6
19
|
# shard_value is one of:
|
|
7
20
|
# A shard
|
|
8
21
|
# An array or relation of shards
|
|
@@ -12,8 +25,6 @@ module Switchman
|
|
|
12
25
|
# :explicit - explicit set on the relation
|
|
13
26
|
# :association - a special value that scopes from associations use to use slightly different logic
|
|
14
27
|
# for foreign key transposition
|
|
15
|
-
# :to_a - a special value that Relation#to_a uses when querying multiple shards to
|
|
16
|
-
# remove primary keys from conditions that aren't applicable to the current shard
|
|
17
28
|
def shard_value
|
|
18
29
|
@values[:shard]
|
|
19
30
|
end
|
|
@@ -44,10 +55,7 @@ module Switchman
|
|
|
44
55
|
old_primary_shard = primary_shard
|
|
45
56
|
self.shard_value = value
|
|
46
57
|
self.shard_source_value = source
|
|
47
|
-
|
|
48
|
-
transpose_clauses(old_primary_shard, primary_shard,
|
|
49
|
-
remove_nonlocal_primary_keys: source == :to_a)
|
|
50
|
-
end
|
|
58
|
+
transpose_predicates(nil, old_primary_shard, primary_shard) if old_primary_shard != primary_shard
|
|
51
59
|
self
|
|
52
60
|
end
|
|
53
61
|
|
|
@@ -65,7 +73,7 @@ module Switchman
|
|
|
65
73
|
when ::ActiveRecord::Relation
|
|
66
74
|
Shard.default
|
|
67
75
|
when nil
|
|
68
|
-
Shard.current(klass.
|
|
76
|
+
Shard.current(klass.connection_class_for_self)
|
|
69
77
|
else
|
|
70
78
|
raise ArgumentError, "invalid shard value #{shard_value}"
|
|
71
79
|
end
|
|
@@ -79,7 +87,7 @@ module Switchman
|
|
|
79
87
|
when ::ActiveRecord::Base
|
|
80
88
|
shard_value.respond_to?(:associated_shards) ? shard_value.associated_shards : [shard_value.shard]
|
|
81
89
|
when nil
|
|
82
|
-
[Shard.current(klass.
|
|
90
|
+
[Shard.current(klass.connection_class_for_self)]
|
|
83
91
|
else
|
|
84
92
|
shard_value
|
|
85
93
|
end
|
|
@@ -89,35 +97,37 @@ module Switchman
|
|
|
89
97
|
super(other.shard(primary_shard))
|
|
90
98
|
end
|
|
91
99
|
|
|
92
|
-
|
|
100
|
+
# use a temp variable so that the new where clause is built before self.where_clause is read,
|
|
101
|
+
# since build_where_clause might mutate self.where_clause
|
|
102
|
+
def where!(opts, *rest)
|
|
103
|
+
new_clause = build_where_clause(opts, rest)
|
|
104
|
+
self.where_clause += new_clause
|
|
105
|
+
self
|
|
106
|
+
end
|
|
93
107
|
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
end
|
|
105
|
-
end
|
|
106
|
-
end
|
|
108
|
+
protected
|
|
109
|
+
|
|
110
|
+
def remove_nonlocal_primary_keys!
|
|
111
|
+
each_transposable_predicate_value do |value, predicate, _relation, _column, type|
|
|
112
|
+
next value unless
|
|
113
|
+
type == :primary &&
|
|
114
|
+
predicate.left.relation.klass == klass &&
|
|
115
|
+
(predicate.is_a?(::Arel::Nodes::Equality) || predicate.is_a?(::Arel::Nodes::HomogeneousIn))
|
|
116
|
+
|
|
117
|
+
(value.is_a?(Integer) && value > Shard::IDS_PER_SHARD) ? [] : value
|
|
107
118
|
end
|
|
108
|
-
|
|
119
|
+
self
|
|
109
120
|
end
|
|
110
121
|
|
|
111
|
-
|
|
112
|
-
transpose_where_clauses(source_shard, target_shard, remove_nonlocal_primary_keys: remove_nonlocal_primary_keys)
|
|
113
|
-
transpose_having_clauses(source_shard, target_shard, remove_nonlocal_primary_keys: remove_nonlocal_primary_keys)
|
|
114
|
-
end
|
|
122
|
+
private
|
|
115
123
|
|
|
116
124
|
def infer_shards_from_primary_key(predicates)
|
|
117
125
|
return unless klass.integral_id?
|
|
118
126
|
|
|
119
127
|
primary_key = predicates.detect do |predicate|
|
|
120
|
-
(predicate.is_a?(::Arel::Nodes::
|
|
128
|
+
(predicate.is_a?(::Arel::Nodes::Equality) ||
|
|
129
|
+
predicate.is_a?(::Arel::Nodes::In) ||
|
|
130
|
+
predicate.is_a?(::Arel::Nodes::HomogeneousIn)) &&
|
|
121
131
|
predicate.left.is_a?(::Arel::Attributes::Attribute) &&
|
|
122
132
|
predicate.left.relation.is_a?(::Arel::Table) && predicate.left.relation.klass == klass &&
|
|
123
133
|
klass.primary_key == predicate.left.name
|
|
@@ -131,7 +141,7 @@ module Switchman
|
|
|
131
141
|
id_shards = Set.new
|
|
132
142
|
right.each do |value|
|
|
133
143
|
local_id, id_shard = Shard.local_id_for(value)
|
|
134
|
-
id_shard ||= Shard.current(klass.
|
|
144
|
+
id_shard ||= Shard.current(klass.connection_class_for_self) if local_id
|
|
135
145
|
id_shards << id_shard if id_shard
|
|
136
146
|
end
|
|
137
147
|
return if id_shards.empty?
|
|
@@ -145,21 +155,24 @@ module Switchman
|
|
|
145
155
|
return
|
|
146
156
|
else
|
|
147
157
|
id_shards = id_shards.to_a
|
|
148
|
-
|
|
158
|
+
transpose_predicates(nil, primary_shard, id_shards.first)
|
|
149
159
|
self.shard_value = id_shards
|
|
150
160
|
return
|
|
151
161
|
end
|
|
152
162
|
when ::Arel::Nodes::BindParam
|
|
153
163
|
local_id, id_shard = Shard.local_id_for(right.value.value_before_type_cast)
|
|
154
|
-
id_shard ||= Shard.current(klass.
|
|
164
|
+
id_shard ||= Shard.current(klass.connection_class_for_self) if local_id
|
|
165
|
+
when ::ActiveModel::Attribute
|
|
166
|
+
local_id, id_shard = Shard.local_id_for(right.value_before_type_cast)
|
|
167
|
+
id_shard ||= Shard.current(klass.connection_class_for_self) if local_id
|
|
155
168
|
else
|
|
156
169
|
local_id, id_shard = Shard.local_id_for(right)
|
|
157
|
-
id_shard ||= Shard.current(klass.
|
|
170
|
+
id_shard ||= Shard.current(klass.connection_class_for_self) if local_id
|
|
158
171
|
end
|
|
159
172
|
|
|
160
173
|
return if !id_shard || id_shard == primary_shard
|
|
161
174
|
|
|
162
|
-
|
|
175
|
+
transpose_predicates(nil, primary_shard, id_shard)
|
|
163
176
|
self.shard_value = id_shard
|
|
164
177
|
end
|
|
165
178
|
|
|
@@ -177,25 +190,25 @@ module Switchman
|
|
|
177
190
|
end
|
|
178
191
|
|
|
179
192
|
def sharded_foreign_key?(relation, column)
|
|
180
|
-
models_for_table(relation.
|
|
193
|
+
models_for_table(relation.name).any? { |m| m.sharded_column?(column) }
|
|
181
194
|
end
|
|
182
195
|
|
|
183
196
|
def sharded_primary_key?(relation, column)
|
|
184
197
|
column = column.to_s
|
|
185
|
-
return column ==
|
|
198
|
+
return column == "id" if relation.klass == ::ActiveRecord::Base
|
|
186
199
|
|
|
187
200
|
relation.klass.primary_key == column && relation.klass.integral_id?
|
|
188
201
|
end
|
|
189
202
|
|
|
190
203
|
def source_shard_for_foreign_key(relation, column)
|
|
191
204
|
reflection = nil
|
|
192
|
-
models_for_table(relation.
|
|
205
|
+
models_for_table(relation.name).each do |model|
|
|
193
206
|
reflection = model.send(:reflection_for_integer_attribute, column)
|
|
194
207
|
break if reflection
|
|
195
208
|
end
|
|
196
|
-
return Shard.current(klass.
|
|
209
|
+
return Shard.current(klass.connection_class_for_self) if reflection.options[:polymorphic]
|
|
197
210
|
|
|
198
|
-
Shard.current(reflection.klass.
|
|
211
|
+
Shard.current(reflection.klass.connection_class_for_self)
|
|
199
212
|
end
|
|
200
213
|
|
|
201
214
|
def relation_and_column(attribute)
|
|
@@ -209,12 +222,11 @@ module Switchman
|
|
|
209
222
|
|
|
210
223
|
case opts
|
|
211
224
|
when String, Array
|
|
212
|
-
values = Hash === rest.first ? rest.first.values : rest
|
|
225
|
+
values = (Hash === rest.first) ? rest.first.values : rest
|
|
213
226
|
|
|
214
|
-
values.grep(ActiveRecord::Relation)
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
rel.shard!(primary_shard) if rel.shard_source_value == :implicit && rel.primary_shard != primary_shard
|
|
227
|
+
if shard_source_value != :explicit && values.grep(ActiveRecord::Relation).first
|
|
228
|
+
raise "Sub-queries are not allowed as simple substitutions; " \
|
|
229
|
+
"please build your relation with more structured methods so that Switchman is able to introspect it."
|
|
218
230
|
end
|
|
219
231
|
|
|
220
232
|
super
|
|
@@ -243,116 +255,170 @@ module Switchman
|
|
|
243
255
|
connection.with_global_table_name { super }
|
|
244
256
|
end
|
|
245
257
|
|
|
246
|
-
def
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
transpose_single_predicate(predicate, source_shard, target_shard,
|
|
252
|
-
remove_nonlocal_primary_keys: remove_nonlocal_primary_keys)
|
|
253
|
-
end
|
|
258
|
+
def each_predicate(predicates = nil, &block)
|
|
259
|
+
return predicates.map(&block) if predicates
|
|
260
|
+
|
|
261
|
+
each_predicate_cb(:having_clause, :having_clause=, &block)
|
|
262
|
+
each_predicate_cb(:where_clause, :where_clause=, &block)
|
|
254
263
|
end
|
|
255
264
|
|
|
256
|
-
def
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
if predicate.is_a?(::Arel::Nodes::Grouping)
|
|
261
|
-
return predicate unless predicate.expr.is_a?(::Arel::Nodes::Or)
|
|
262
|
-
|
|
263
|
-
# Dang, we have an OR. OK, that means we have other epxressions below this
|
|
264
|
-
# level, perhaps many, that may need transposition.
|
|
265
|
-
# the left side and right side must each be treated as predicate lists and
|
|
266
|
-
# transformed in kind, if neither of them changes we can just return the grouping as is.
|
|
267
|
-
# hold on, it's about to get recursive...
|
|
268
|
-
or_expr = predicate.expr
|
|
269
|
-
left_node = or_expr.left
|
|
270
|
-
right_node = or_expr.right
|
|
271
|
-
new_left_predicates = transpose_single_predicate(left_node, source_shard,
|
|
272
|
-
target_shard, remove_nonlocal_primary_keys: remove_nonlocal_primary_keys)
|
|
273
|
-
or_expr.instance_variable_set(:@left, new_left_predicates) if new_left_predicates != left_node
|
|
274
|
-
new_right_predicates = transpose_single_predicate(right_node, source_shard,
|
|
275
|
-
target_shard, remove_nonlocal_primary_keys: remove_nonlocal_primary_keys)
|
|
276
|
-
or_expr.instance_variable_set(:@right, new_right_predicates) if new_right_predicates != right_node
|
|
277
|
-
return predicate
|
|
278
|
-
end
|
|
279
|
-
return predicate unless predicate.is_a?(::Arel::Nodes::Binary) || predicate.is_a?(::Arel::Nodes::HomogeneousIn)
|
|
280
|
-
return predicate unless predicate.left.is_a?(::Arel::Attributes::Attribute)
|
|
281
|
-
|
|
282
|
-
relation, column = relation_and_column(predicate.left)
|
|
283
|
-
return predicate unless (type = transposable_attribute_type(relation, column))
|
|
284
|
-
|
|
285
|
-
remove = true if type == :primary &&
|
|
286
|
-
remove_nonlocal_primary_keys &&
|
|
287
|
-
predicate.left.relation.klass == klass &&
|
|
288
|
-
(predicate.is_a?(::Arel::Nodes::Equality) || predicate.is_a?(::Arel::Nodes::HomogeneousIn))
|
|
289
|
-
|
|
290
|
-
current_source_shard =
|
|
291
|
-
if source_shard
|
|
292
|
-
source_shard
|
|
293
|
-
elsif type == :primary
|
|
294
|
-
Shard.current(klass.connection_classes)
|
|
295
|
-
elsif type == :foreign
|
|
296
|
-
source_shard_for_foreign_key(relation, column)
|
|
297
|
-
end
|
|
265
|
+
def each_predicate_cb(clause_getter, clause_setter, &block)
|
|
266
|
+
old_clause = send(clause_getter)
|
|
267
|
+
old_predicates = old_clause.send(:predicates)
|
|
268
|
+
return if old_predicates.empty?
|
|
298
269
|
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
else
|
|
302
|
-
predicate.right
|
|
303
|
-
end
|
|
270
|
+
new_predicates = old_predicates.map(&block)
|
|
271
|
+
return if new_predicates == old_predicates
|
|
304
272
|
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
273
|
+
new_clause = old_clause.dup
|
|
274
|
+
new_clause.instance_variable_set(:@predicates, new_predicates)
|
|
275
|
+
|
|
276
|
+
send(clause_setter, new_clause)
|
|
277
|
+
end
|
|
278
|
+
|
|
279
|
+
def each_transposable_predicate(predicates, &block)
|
|
280
|
+
each_predicate(predicates) do |predicate|
|
|
281
|
+
case predicate
|
|
282
|
+
when ::Arel::Nodes::Grouping
|
|
283
|
+
next predicate unless predicate.expr.is_a?(::Arel::Nodes::Or)
|
|
284
|
+
|
|
285
|
+
or_expr = predicate.expr
|
|
286
|
+
old_left = or_expr.left
|
|
287
|
+
old_right = or_expr.right
|
|
288
|
+
new_left, new_right = each_transposable_predicate([old_left, old_right], &block)
|
|
289
|
+
|
|
290
|
+
next predicate if new_left == old_left && new_right == old_right
|
|
291
|
+
|
|
292
|
+
next predicate.class.new predicate.expr.class.new(new_left, new_right)
|
|
293
|
+
when ::Arel::Nodes::SelectStatement
|
|
294
|
+
new_cores = predicate.cores.map do |core|
|
|
295
|
+
next core unless core.is_a?(::Arel::Nodes::SelectCore) # just in case something weird is going on
|
|
296
|
+
|
|
297
|
+
new_wheres = each_transposable_predicate(core.wheres, &block)
|
|
298
|
+
new_havings = each_transposable_predicate(core.havings, &block)
|
|
299
|
+
|
|
300
|
+
next core if core.wheres == new_wheres && core.havings == new_havings
|
|
301
|
+
|
|
302
|
+
new_core = core.clone
|
|
303
|
+
new_core.wheres = new_wheres
|
|
304
|
+
new_core.havings = new_havings
|
|
305
|
+
new_core
|
|
306
|
+
end
|
|
307
|
+
|
|
308
|
+
next predicate if predicate.cores == new_cores
|
|
309
|
+
|
|
310
|
+
new_node = predicate.clone
|
|
311
|
+
new_node.instance_variable_set(:@cores, new_cores)
|
|
312
|
+
next new_node
|
|
313
|
+
when ::Arel::Nodes::Not
|
|
314
|
+
old_value = predicate.expr
|
|
315
|
+
new_value = each_transposable_predicate([old_value], &block).first
|
|
316
|
+
|
|
317
|
+
next predicate if old_value == new_value
|
|
318
|
+
|
|
319
|
+
next predicate.class.new(new_value)
|
|
320
|
+
when ::Arel::Nodes::Exists
|
|
321
|
+
old_value = predicate.expressions
|
|
322
|
+
new_value = each_transposable_predicate([old_value], &block).first
|
|
323
|
+
|
|
324
|
+
next predicate if old_value == new_value
|
|
325
|
+
|
|
326
|
+
next predicate.class.new(new_value)
|
|
311
327
|
end
|
|
312
328
|
|
|
313
|
-
|
|
314
|
-
predicate
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
329
|
+
next predicate unless predicate.is_a?(::Arel::Nodes::Binary) || predicate.is_a?(::Arel::Nodes::HomogeneousIn)
|
|
330
|
+
next predicate unless predicate.left.is_a?(::Arel::Attributes::Attribute)
|
|
331
|
+
|
|
332
|
+
relation, column = relation_and_column(predicate.left)
|
|
333
|
+
next predicate unless (type = transposable_attribute_type(relation, column))
|
|
334
|
+
|
|
335
|
+
yield(predicate, relation, column, type)
|
|
336
|
+
end
|
|
337
|
+
end
|
|
338
|
+
|
|
339
|
+
def each_transposable_predicate_value(predicates = nil, &block)
|
|
340
|
+
each_transposable_predicate(predicates) do |predicate, relation, column, type|
|
|
341
|
+
each_transposable_predicate_value_cb(predicate, block) do |value|
|
|
342
|
+
yield(value, predicate, relation, column, type)
|
|
320
343
|
end
|
|
321
|
-
|
|
344
|
+
end
|
|
345
|
+
end
|
|
346
|
+
|
|
347
|
+
def each_transposable_predicate_value_cb(node, original_block, &block)
|
|
348
|
+
case node
|
|
349
|
+
when Array
|
|
350
|
+
node.filter_map { |val| each_transposable_predicate_value_cb(val, original_block, &block).presence }
|
|
351
|
+
when ::ActiveModel::Attribute
|
|
352
|
+
old_value = node.value_before_type_cast
|
|
353
|
+
new_value = each_transposable_predicate_value_cb(old_value, original_block, &block)
|
|
354
|
+
|
|
355
|
+
(old_value == new_value) ? node : node.class.new(node.name, new_value, node.type)
|
|
356
|
+
when ::Arel::Nodes::And
|
|
357
|
+
old_value = node.children
|
|
358
|
+
new_value = each_transposable_predicate_value_cb(old_value, original_block, &block)
|
|
359
|
+
|
|
360
|
+
(old_value == new_value) ? node : node.class.new(new_value)
|
|
361
|
+
when ::Arel::Nodes::BindParam
|
|
362
|
+
old_value = node.value
|
|
363
|
+
new_value = each_transposable_predicate_value_cb(old_value, original_block, &block)
|
|
364
|
+
|
|
365
|
+
(old_value == new_value) ? node : node.class.new(new_value)
|
|
366
|
+
when ::Arel::Nodes::Casted
|
|
367
|
+
old_value = node.value
|
|
368
|
+
new_value = each_transposable_predicate_value_cb(old_value, original_block, &block)
|
|
369
|
+
|
|
370
|
+
(old_value == new_value) ? node : node.class.new(new_value, node.attribute)
|
|
371
|
+
when ::Arel::Nodes::HomogeneousIn
|
|
372
|
+
old_value = node.values
|
|
373
|
+
new_value = each_transposable_predicate_value_cb(old_value, original_block, &block)
|
|
374
|
+
|
|
322
375
|
# switch to a regular In, so that Relation::WhereClause#contradiction? knows about it
|
|
323
|
-
if
|
|
324
|
-
klass =
|
|
325
|
-
klass.new(
|
|
376
|
+
if new_value.empty?
|
|
377
|
+
klass = (node.type == :in) ? ::Arel::Nodes::In : ::Arel::Nodes::NotIn
|
|
378
|
+
klass.new(node.attribute, new_value)
|
|
326
379
|
else
|
|
327
|
-
|
|
380
|
+
(old_value == new_value) ? node : node.class.new(new_value, node.attribute, node.type)
|
|
328
381
|
end
|
|
382
|
+
when ::Arel::Nodes::Binary
|
|
383
|
+
old_value = node.right
|
|
384
|
+
new_value = each_transposable_predicate_value_cb(old_value, original_block, &block)
|
|
385
|
+
|
|
386
|
+
(old_value == new_value) ? node : node.class.new(node.left, new_value)
|
|
387
|
+
when ::Arel::Nodes::SelectStatement
|
|
388
|
+
each_transposable_predicate_value([node], &original_block).first
|
|
329
389
|
else
|
|
330
|
-
|
|
390
|
+
yield(node)
|
|
331
391
|
end
|
|
332
392
|
end
|
|
333
393
|
|
|
334
|
-
def
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
if current_id == local_id
|
|
346
|
-
# make a new bind param
|
|
347
|
-
value
|
|
394
|
+
def transpose_predicates(predicates,
|
|
395
|
+
source_shard,
|
|
396
|
+
target_shard)
|
|
397
|
+
each_transposable_predicate_value(predicates) do |value, _predicate, relation, column, type|
|
|
398
|
+
current_source_shard =
|
|
399
|
+
if source_shard
|
|
400
|
+
source_shard
|
|
401
|
+
elsif type == :primary
|
|
402
|
+
Shard.current(klass.connection_class_for_self)
|
|
403
|
+
elsif type == :foreign
|
|
404
|
+
source_shard_for_foreign_key(relation, column)
|
|
348
405
|
else
|
|
349
|
-
|
|
406
|
+
primary_shard
|
|
350
407
|
end
|
|
351
|
-
|
|
408
|
+
|
|
409
|
+
transpose_predicate_value(value, current_source_shard, target_shard, type)
|
|
410
|
+
end
|
|
411
|
+
end
|
|
412
|
+
|
|
413
|
+
def transpose_predicate_value(value, current_shard, target_shard, attribute_type)
|
|
414
|
+
if value.is_a?(NonTransposingValue)
|
|
415
|
+
value
|
|
416
|
+
elsif value.is_a?(::ActiveRecord::StatementCache::Substitute)
|
|
417
|
+
value.sharded = true # mark for transposition later
|
|
418
|
+
value.primary = true if attribute_type == :primary
|
|
419
|
+
value
|
|
352
420
|
else
|
|
353
|
-
|
|
354
|
-
local_id = [] if remove_non_local_ids && local_id.is_a?(Integer) && local_id > Shard::IDS_PER_SHARD
|
|
355
|
-
local_id
|
|
421
|
+
Shard.relative_id_for(value, current_shard, target_shard) || value
|
|
356
422
|
end
|
|
357
423
|
end
|
|
358
424
|
end
|
|
@@ -5,7 +5,7 @@ module Switchman
|
|
|
5
5
|
module Reflection
|
|
6
6
|
module AbstractReflection
|
|
7
7
|
def shard(owner)
|
|
8
|
-
if polymorphic? || klass.
|
|
8
|
+
if polymorphic? || klass.connection_class_for_self == owner.class.connection_class_for_self
|
|
9
9
|
# polymorphic associations assume the same shard as the owning item
|
|
10
10
|
owner.shard
|
|
11
11
|
else
|
|
@@ -4,18 +4,18 @@ module Switchman
|
|
|
4
4
|
module ActiveRecord
|
|
5
5
|
module Relation
|
|
6
6
|
def self.prepended(klass)
|
|
7
|
-
klass::SINGLE_VALUE_METHODS.
|
|
7
|
+
klass::SINGLE_VALUE_METHODS.push(:shard, :shard_source)
|
|
8
8
|
end
|
|
9
9
|
|
|
10
10
|
def initialize(*, **)
|
|
11
11
|
super
|
|
12
|
-
self.shard_value = Shard.current(klass ? klass.
|
|
12
|
+
self.shard_value = Shard.current(klass ? klass.connection_class_for_self : :primary) unless shard_value
|
|
13
13
|
self.shard_source_value = :implicit unless shard_source_value
|
|
14
14
|
end
|
|
15
15
|
|
|
16
16
|
def clone
|
|
17
17
|
result = super
|
|
18
|
-
result.shard_value = Shard.current(klass ? klass.
|
|
18
|
+
result.shard_value = Shard.current(klass ? klass.connection_class_for_self : :primary) unless shard_value
|
|
19
19
|
result
|
|
20
20
|
end
|
|
21
21
|
|
|
@@ -29,41 +29,44 @@ module Switchman
|
|
|
29
29
|
end
|
|
30
30
|
|
|
31
31
|
def new(*, &block)
|
|
32
|
-
primary_shard.activate(klass.
|
|
32
|
+
primary_shard.activate(klass.connection_class_for_self) { super }
|
|
33
33
|
end
|
|
34
34
|
|
|
35
35
|
def create(*, &block)
|
|
36
|
-
primary_shard.activate(klass.
|
|
36
|
+
primary_shard.activate(klass.connection_class_for_self) { super }
|
|
37
37
|
end
|
|
38
38
|
|
|
39
39
|
def create!(*, &block)
|
|
40
|
-
primary_shard.activate(klass.
|
|
40
|
+
primary_shard.activate(klass.connection_class_for_self) { super }
|
|
41
41
|
end
|
|
42
42
|
|
|
43
43
|
def to_sql
|
|
44
|
-
primary_shard.activate(klass.
|
|
44
|
+
primary_shard.activate(klass.connection_class_for_self) { super }
|
|
45
45
|
end
|
|
46
46
|
|
|
47
|
-
|
|
48
|
-
|
|
47
|
+
if ::Rails.version > "7.1.2"
|
|
48
|
+
def transaction(...)
|
|
49
|
+
primary_shard.activate(klass.connection_class_for_self) { super }
|
|
50
|
+
end
|
|
49
51
|
end
|
|
50
52
|
|
|
51
|
-
def
|
|
52
|
-
|
|
53
|
+
def explain(*args)
|
|
54
|
+
activate { |relation| relation.call_super(:explain, Relation, *args) }
|
|
55
|
+
end
|
|
53
56
|
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
@records = results
|
|
57
|
+
def load(&block)
|
|
58
|
+
if !loaded? || scheduled?
|
|
59
|
+
@records = activate { |relation| relation.send(:exec_queries, &block) }
|
|
58
60
|
@loaded = true
|
|
59
61
|
end
|
|
60
|
-
|
|
62
|
+
|
|
63
|
+
self
|
|
61
64
|
end
|
|
62
65
|
|
|
63
66
|
%I[update_all delete_all].each do |method|
|
|
64
67
|
class_eval <<-RUBY, __FILE__, __LINE__ + 1
|
|
65
|
-
def #{method}(*args)
|
|
66
|
-
result = self.activate(unordered: true) { |relation| relation.call_super(#{method.inspect}, Relation, *args) }
|
|
68
|
+
def #{method}(*args, **kwargs)
|
|
69
|
+
result = self.activate(unordered: true) { |relation| relation.call_super(#{method.inspect}, Relation, *args, **kwargs) }
|
|
67
70
|
result = result.sum if result.is_a?(Array)
|
|
68
71
|
result
|
|
69
72
|
end
|
|
@@ -75,12 +78,16 @@ module Switchman
|
|
|
75
78
|
loose_mode = options[:loose] && is_integer
|
|
76
79
|
# loose_mode: if we don't care about getting exactly batch_size ids in between
|
|
77
80
|
# don't get the max - just get the min and add batch_size so we get that many _at most_
|
|
78
|
-
values = loose_mode ?
|
|
81
|
+
values = loose_mode ? "MIN(id)" : "MIN(id), MAX(id)"
|
|
79
82
|
|
|
80
83
|
batch_size = options[:batch_size].try(:to_i) || 1000
|
|
81
|
-
quoted_primary_key =
|
|
82
|
-
|
|
83
|
-
|
|
84
|
+
quoted_primary_key =
|
|
85
|
+
"#{klass.connection.quote_local_table_name(table_name)}.#{klass.connection.quote_column_name(primary_key)}"
|
|
86
|
+
as_id = " AS id" unless primary_key == "id"
|
|
87
|
+
subquery_scope = except(:select)
|
|
88
|
+
.select("#{quoted_primary_key}#{as_id}")
|
|
89
|
+
.reorder(primary_key.to_sym)
|
|
90
|
+
.limit(loose_mode ? 1 : batch_size)
|
|
84
91
|
subquery_scope = subquery_scope.where("#{quoted_primary_key} <= ?", options[:end_at]) if options[:end_at]
|
|
85
92
|
|
|
86
93
|
first_subquery_scope = if options[:start_at]
|
|
@@ -94,7 +101,7 @@ module Switchman
|
|
|
94
101
|
|
|
95
102
|
while ids.first.present?
|
|
96
103
|
ids.map!(&:to_i) if is_integer
|
|
97
|
-
ids << ids.first + batch_size if loose_mode
|
|
104
|
+
ids << (ids.first + batch_size) if loose_mode
|
|
98
105
|
|
|
99
106
|
yield(*ids)
|
|
100
107
|
last_value = ids.last
|
|
@@ -106,21 +113,26 @@ module Switchman
|
|
|
106
113
|
def activate(unordered: false, &block)
|
|
107
114
|
shards = all_shards
|
|
108
115
|
if Array === shards && shards.length == 1
|
|
109
|
-
if
|
|
116
|
+
if !loaded? && shard_value != shards.first
|
|
117
|
+
shard(shards.first).activate(&block)
|
|
118
|
+
elsif shards.first == DefaultShard || shards.first == Shard.current(klass.connection_class_for_self)
|
|
110
119
|
yield(self, shards.first)
|
|
111
120
|
else
|
|
112
|
-
shards.first.activate(klass.
|
|
121
|
+
shards.first.activate(klass.connection_class_for_self) { yield(self, shards.first) }
|
|
113
122
|
end
|
|
114
123
|
else
|
|
115
124
|
result_count = 0
|
|
116
125
|
can_order = false
|
|
117
|
-
result = Shard.with_each_shard(shards, [klass.
|
|
126
|
+
result = Shard.with_each_shard(shards, [klass.connection_class_for_self]) do
|
|
118
127
|
# don't even query other shards if we're already past the limit
|
|
119
128
|
next if limit_value && result_count >= limit_value && order_values.empty?
|
|
120
129
|
|
|
121
|
-
relation = shard(Shard.current(klass.
|
|
130
|
+
relation = shard(Shard.current(klass.connection_class_for_self))
|
|
131
|
+
relation.remove_nonlocal_primary_keys!
|
|
122
132
|
# do a minimal query if possible
|
|
123
|
-
|
|
133
|
+
if limit_value && !result_count.zero? && order_values.empty?
|
|
134
|
+
relation = relation.limit(limit_value - result_count)
|
|
135
|
+
end
|
|
124
136
|
|
|
125
137
|
shard_results = relation.activate(&block)
|
|
126
138
|
|
|
@@ -17,7 +17,7 @@ module Switchman
|
|
|
17
17
|
final_shard_source_value = %i[explicit association].detect do |source_value|
|
|
18
18
|
shard_source_value == source_value || rhs.shard_source_value == source_value
|
|
19
19
|
end
|
|
20
|
-
raise
|
|
20
|
+
raise "unknown shard_source_value" unless final_shard_source_value
|
|
21
21
|
|
|
22
22
|
# have to merge shard_value
|
|
23
23
|
lhs_shard_value = all_shards
|
|
@@ -36,7 +36,7 @@ module Switchman
|
|
|
36
36
|
final_shard_source_value = %i[explicit association implicit].detect do |source_value|
|
|
37
37
|
shard_source_value == source_value || rhs.shard_source_value == source_value
|
|
38
38
|
end
|
|
39
|
-
raise
|
|
39
|
+
raise "unknown shard_source_value" unless final_shard_source_value
|
|
40
40
|
end
|
|
41
41
|
|
|
42
42
|
[final_shard_value, final_primary_shard, final_shard_source_value]
|