switchman 2.0.5 → 3.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Rakefile +10 -2
- data/app/models/switchman/shard.rb +235 -270
- data/app/models/switchman/unsharded_record.rb +7 -0
- data/db/migrate/20130328212039_create_switchman_shards.rb +1 -1
- data/db/migrate/20130328224244_create_default_shard.rb +5 -5
- data/db/migrate/20161206323434_add_back_default_string_limits_switchman.rb +1 -0
- data/db/migrate/20180828183945_add_default_shard_index.rb +2 -2
- data/db/migrate/20180828192111_add_timestamps_to_shards.rb +7 -5
- data/db/migrate/20190114212900_add_unique_name_indexes.rb +5 -3
- data/lib/switchman/action_controller/caching.rb +2 -2
- data/lib/switchman/active_record/abstract_adapter.rb +1 -0
- data/lib/switchman/active_record/association.rb +78 -89
- data/lib/switchman/active_record/attribute_methods.rb +106 -52
- data/lib/switchman/active_record/base.rb +58 -59
- data/lib/switchman/active_record/calculations.rb +73 -66
- data/lib/switchman/active_record/connection_pool.rb +14 -41
- data/lib/switchman/active_record/database_configurations/database_config.rb +13 -0
- data/lib/switchman/active_record/database_configurations.rb +34 -0
- data/lib/switchman/active_record/finder_methods.rb +11 -16
- data/lib/switchman/active_record/log_subscriber.rb +4 -8
- data/lib/switchman/active_record/migration.rb +18 -15
- data/lib/switchman/active_record/model_schema.rb +1 -1
- data/lib/switchman/active_record/persistence.rb +4 -6
- data/lib/switchman/active_record/postgresql_adapter.rb +45 -144
- data/lib/switchman/active_record/predicate_builder.rb +1 -1
- data/lib/switchman/active_record/query_cache.rb +18 -19
- data/lib/switchman/active_record/query_methods.rb +144 -200
- data/lib/switchman/active_record/reflection.rb +6 -10
- data/lib/switchman/active_record/relation.rb +27 -21
- data/lib/switchman/active_record/spawn_methods.rb +27 -29
- data/lib/switchman/active_record/statement_cache.rb +18 -35
- data/lib/switchman/active_record/tasks/database_tasks.rb +16 -0
- data/lib/switchman/active_support/cache.rb +3 -5
- data/lib/switchman/arel.rb +13 -8
- data/lib/switchman/database_server.rb +122 -144
- data/lib/switchman/default_shard.rb +52 -16
- data/lib/switchman/engine.rb +61 -57
- data/lib/switchman/environment.rb +4 -8
- data/lib/switchman/errors.rb +1 -0
- data/lib/switchman/guard_rail/relation.rb +5 -7
- data/lib/switchman/guard_rail.rb +6 -19
- data/lib/switchman/r_spec_helper.rb +29 -37
- data/lib/switchman/rails.rb +14 -12
- data/lib/switchman/sharded_instrumenter.rb +1 -1
- data/lib/switchman/standard_error.rb +15 -3
- data/lib/switchman/test_helper.rb +7 -11
- data/lib/switchman/version.rb +1 -1
- data/lib/switchman.rb +3 -3
- data/lib/tasks/switchman.rake +54 -69
- metadata +90 -49
- data/lib/switchman/active_record/batches.rb +0 -11
- data/lib/switchman/active_record/connection_handler.rb +0 -172
- data/lib/switchman/active_record/where_clause_factory.rb +0 -36
- data/lib/switchman/connection_pool_proxy.rb +0 -169
- data/lib/switchman/schema_cache.rb +0 -20
|
@@ -17,15 +17,20 @@ module Switchman
|
|
|
17
17
|
def shard_value
|
|
18
18
|
@values[:shard]
|
|
19
19
|
end
|
|
20
|
+
|
|
20
21
|
def shard_source_value
|
|
21
22
|
@values[:shard_source]
|
|
22
23
|
end
|
|
24
|
+
|
|
23
25
|
def shard_value=(value)
|
|
24
26
|
raise ::ActiveRecord::ImmutableRelation if @loaded
|
|
27
|
+
|
|
25
28
|
@values[:shard] = value
|
|
26
29
|
end
|
|
30
|
+
|
|
27
31
|
def shard_source_value=(value)
|
|
28
32
|
raise ::ActiveRecord::ImmutableRelation if @loaded
|
|
33
|
+
|
|
29
34
|
@values[:shard_source] = value
|
|
30
35
|
end
|
|
31
36
|
|
|
@@ -35,11 +40,13 @@ module Switchman
|
|
|
35
40
|
|
|
36
41
|
def shard!(value, source = :explicit)
|
|
37
42
|
raise ArgumentError, "shard can't be nil" unless value
|
|
38
|
-
|
|
43
|
+
|
|
44
|
+
old_primary_shard = primary_shard
|
|
39
45
|
self.shard_value = value
|
|
40
46
|
self.shard_source_value = source
|
|
41
|
-
if
|
|
42
|
-
transpose_clauses(old_primary_shard,
|
|
47
|
+
if old_primary_shard != primary_shard || source == :to_a
|
|
48
|
+
transpose_clauses(old_primary_shard, primary_shard,
|
|
49
|
+
remove_nonlocal_primary_keys: source == :to_a)
|
|
43
50
|
end
|
|
44
51
|
self
|
|
45
52
|
end
|
|
@@ -58,7 +65,7 @@ module Switchman
|
|
|
58
65
|
when ::ActiveRecord::Relation
|
|
59
66
|
Shard.default
|
|
60
67
|
when nil
|
|
61
|
-
Shard.current(klass.
|
|
68
|
+
Shard.current(klass.connection_classes)
|
|
62
69
|
else
|
|
63
70
|
raise ArgumentError, "invalid shard value #{shard_value}"
|
|
64
71
|
end
|
|
@@ -72,118 +79,88 @@ module Switchman
|
|
|
72
79
|
when ::ActiveRecord::Base
|
|
73
80
|
shard_value.respond_to?(:associated_shards) ? shard_value.associated_shards : [shard_value.shard]
|
|
74
81
|
when nil
|
|
75
|
-
[Shard.current(klass.
|
|
82
|
+
[Shard.current(klass.connection_classes)]
|
|
76
83
|
else
|
|
77
84
|
shard_value
|
|
78
85
|
end
|
|
79
86
|
end
|
|
80
87
|
|
|
81
88
|
def or(other)
|
|
82
|
-
super(other.shard(
|
|
89
|
+
super(other.shard(primary_shard))
|
|
83
90
|
end
|
|
84
91
|
|
|
85
92
|
private
|
|
86
93
|
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
+
%i[where having].each do |type|
|
|
95
|
+
class_eval <<-RUBY, __FILE__, __LINE__ + 1
|
|
96
|
+
def transpose_#{type}_clauses(source_shard, target_shard, remove_nonlocal_primary_keys:)
|
|
97
|
+
unless (predicates = #{type}_clause.send(:predicates)).empty?
|
|
98
|
+
new_predicates = transpose_predicates(predicates, source_shard,
|
|
99
|
+
target_shard, remove_nonlocal_primary_keys: remove_nonlocal_primary_keys)
|
|
100
|
+
if new_predicates != predicates
|
|
101
|
+
self.#{type}_clause = #{type}_clause.dup
|
|
94
102
|
if new_predicates != predicates
|
|
95
|
-
|
|
96
|
-
if new_predicates != predicates
|
|
97
|
-
#{type}_clause.instance_variable_set(:@predicates, new_predicates)
|
|
98
|
-
end
|
|
103
|
+
#{type}_clause.instance_variable_set(:@predicates, new_predicates)
|
|
99
104
|
end
|
|
100
105
|
end
|
|
101
106
|
end
|
|
102
|
-
RUBY
|
|
103
|
-
end
|
|
104
|
-
else
|
|
105
|
-
[:where, :having].each do |type|
|
|
106
|
-
class_eval <<-RUBY, __FILE__, __LINE__ + 1
|
|
107
|
-
def transpose_#{type}_clauses(source_shard, target_shard, remove_nonlocal_primary_keys)
|
|
108
|
-
unless (predicates = #{type}_clause.send(:predicates)).empty?
|
|
109
|
-
new_predicates, new_binds = transpose_predicates(predicates, source_shard,
|
|
110
|
-
target_shard, remove_nonlocal_primary_keys,
|
|
111
|
-
binds: #{type}_clause.binds,
|
|
112
|
-
dup_binds_on_mutation: true)
|
|
113
|
-
if new_predicates != predicates || !new_binds.equal?(#{type}_clause.binds)
|
|
114
|
-
self.#{type}_clause = #{type}_clause.dup
|
|
115
|
-
if new_predicates != predicates
|
|
116
|
-
#{type}_clause.instance_variable_set(:@predicates, new_predicates)
|
|
117
|
-
end
|
|
118
|
-
if !new_binds.equal?(#{type}_clause.binds)
|
|
119
|
-
#{type}_clause.instance_variable_set(:@binds, new_binds)
|
|
120
|
-
end
|
|
121
|
-
end
|
|
122
|
-
end
|
|
123
|
-
end
|
|
124
|
-
RUBY
|
|
125
107
|
end
|
|
108
|
+
RUBY
|
|
126
109
|
end
|
|
127
110
|
|
|
128
|
-
def transpose_clauses(source_shard, target_shard, remove_nonlocal_primary_keys
|
|
129
|
-
transpose_where_clauses(source_shard, target_shard, remove_nonlocal_primary_keys)
|
|
130
|
-
transpose_having_clauses(source_shard, target_shard, remove_nonlocal_primary_keys)
|
|
111
|
+
def transpose_clauses(source_shard, target_shard, remove_nonlocal_primary_keys: false)
|
|
112
|
+
transpose_where_clauses(source_shard, target_shard, remove_nonlocal_primary_keys: remove_nonlocal_primary_keys)
|
|
113
|
+
transpose_having_clauses(source_shard, target_shard, remove_nonlocal_primary_keys: remove_nonlocal_primary_keys)
|
|
131
114
|
end
|
|
132
115
|
|
|
133
|
-
def infer_shards_from_primary_key(predicates
|
|
116
|
+
def infer_shards_from_primary_key(predicates)
|
|
134
117
|
return unless klass.integral_id?
|
|
135
118
|
|
|
136
119
|
primary_key = predicates.detect do |predicate|
|
|
137
|
-
predicate.is_a?(::Arel::Nodes::Binary)
|
|
138
|
-
predicate.left.
|
|
120
|
+
(predicate.is_a?(::Arel::Nodes::Binary) || predicate.is_a?(::Arel::Nodes::HomogeneousIn)) &&
|
|
121
|
+
predicate.left.is_a?(::Arel::Attributes::Attribute) &&
|
|
122
|
+
predicate.left.relation.is_a?(::Arel::Table) && predicate.left.relation.klass == klass &&
|
|
139
123
|
klass.primary_key == predicate.left.name
|
|
140
124
|
end
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
if
|
|
151
|
-
|
|
152
|
-
elsif id_shards.length == 1
|
|
153
|
-
id_shard = id_shards.first
|
|
154
|
-
# prefer to not change the shard
|
|
155
|
-
elsif id_shards.include?(primary_shard)
|
|
156
|
-
id_shards.delete(primary_shard)
|
|
157
|
-
self.shard_value = [primary_shard] + id_shards.to_a
|
|
158
|
-
return
|
|
159
|
-
else
|
|
160
|
-
id_shards = id_shards.to_a
|
|
161
|
-
transpose_clauses(primary_shard, id_shards.first)
|
|
162
|
-
self.shard_value = id_shards
|
|
163
|
-
return
|
|
164
|
-
end
|
|
165
|
-
when ::Arel::Nodes::BindParam
|
|
166
|
-
if ::Rails.version >= "5.2"
|
|
167
|
-
local_id, id_shard = Shard.local_id_for(primary_key.right.value.value_before_type_cast)
|
|
168
|
-
id_shard ||= Shard.current(klass.shard_category) if local_id
|
|
169
|
-
else
|
|
170
|
-
# look for a bind param with a matching column name
|
|
171
|
-
if binds && bind = binds.detect{|b| b&.name.to_s == klass.primary_key.to_s}
|
|
172
|
-
unless bind.value.is_a?(::ActiveRecord::StatementCache::Substitute)
|
|
173
|
-
local_id, id_shard = Shard.local_id_for(bind.value)
|
|
174
|
-
id_shard ||= Shard.current(klass.shard_category) if local_id
|
|
175
|
-
end
|
|
176
|
-
end
|
|
177
|
-
end
|
|
178
|
-
else
|
|
179
|
-
local_id, id_shard = Shard.local_id_for(primary_key.right)
|
|
180
|
-
id_shard ||= Shard.current(klass.shard_category) if local_id
|
|
125
|
+
return unless primary_key
|
|
126
|
+
|
|
127
|
+
right = primary_key.is_a?(::Arel::Nodes::HomogeneousIn) ? primary_key.values : primary_key.right
|
|
128
|
+
|
|
129
|
+
case right
|
|
130
|
+
when Array
|
|
131
|
+
id_shards = Set.new
|
|
132
|
+
right.each do |value|
|
|
133
|
+
local_id, id_shard = Shard.local_id_for(value)
|
|
134
|
+
id_shard ||= Shard.current(klass.connection_classes) if local_id
|
|
135
|
+
id_shards << id_shard if id_shard
|
|
181
136
|
end
|
|
137
|
+
return if id_shards.empty?
|
|
182
138
|
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
139
|
+
if id_shards.length == 1
|
|
140
|
+
id_shard = id_shards.first
|
|
141
|
+
# prefer to not change the shard
|
|
142
|
+
elsif id_shards.include?(primary_shard)
|
|
143
|
+
id_shards.delete(primary_shard)
|
|
144
|
+
self.shard_value = [primary_shard] + id_shards.to_a
|
|
145
|
+
return
|
|
146
|
+
else
|
|
147
|
+
id_shards = id_shards.to_a
|
|
148
|
+
transpose_clauses(primary_shard, id_shards.first)
|
|
149
|
+
self.shard_value = id_shards
|
|
150
|
+
return
|
|
151
|
+
end
|
|
152
|
+
when ::Arel::Nodes::BindParam
|
|
153
|
+
local_id, id_shard = Shard.local_id_for(right.value.value_before_type_cast)
|
|
154
|
+
id_shard ||= Shard.current(klass.connection_classes) if local_id
|
|
155
|
+
else
|
|
156
|
+
local_id, id_shard = Shard.local_id_for(right)
|
|
157
|
+
id_shard ||= Shard.current(klass.connection_classes) if local_id
|
|
186
158
|
end
|
|
159
|
+
|
|
160
|
+
return if !id_shard || id_shard == primary_shard
|
|
161
|
+
|
|
162
|
+
transpose_clauses(primary_shard, id_shard)
|
|
163
|
+
self.shard_value = id_shard
|
|
187
164
|
end
|
|
188
165
|
|
|
189
166
|
def transposable_attribute_type(relation, column)
|
|
@@ -205,8 +182,9 @@ module Switchman
|
|
|
205
182
|
|
|
206
183
|
def sharded_primary_key?(relation, column)
|
|
207
184
|
column = column.to_s
|
|
208
|
-
return column == 'id' if relation.
|
|
209
|
-
|
|
185
|
+
return column == 'id' if relation.klass == ::ActiveRecord::Base
|
|
186
|
+
|
|
187
|
+
relation.klass.primary_key == column && relation.klass.integral_id?
|
|
210
188
|
end
|
|
211
189
|
|
|
212
190
|
def source_shard_for_foreign_key(relation, column)
|
|
@@ -215,8 +193,9 @@ module Switchman
|
|
|
215
193
|
reflection = model.send(:reflection_for_integer_attribute, column)
|
|
216
194
|
break if reflection
|
|
217
195
|
end
|
|
218
|
-
return Shard.current(klass.
|
|
219
|
-
|
|
196
|
+
return Shard.current(klass.connection_classes) if reflection.options[:polymorphic]
|
|
197
|
+
|
|
198
|
+
Shard.current(reflection.klass.connection_classes)
|
|
220
199
|
end
|
|
221
200
|
|
|
222
201
|
def relation_and_column(attribute)
|
|
@@ -225,8 +204,31 @@ module Switchman
|
|
|
225
204
|
[attribute.relation, column]
|
|
226
205
|
end
|
|
227
206
|
|
|
228
|
-
def
|
|
229
|
-
|
|
207
|
+
def build_where_clause(opts, rest = [])
|
|
208
|
+
opts = sanitize_forbidden_attributes(opts)
|
|
209
|
+
|
|
210
|
+
case opts
|
|
211
|
+
when String, Array
|
|
212
|
+
values = Hash === rest.first ? rest.first.values : rest
|
|
213
|
+
|
|
214
|
+
values.grep(ActiveRecord::Relation) do |rel|
|
|
215
|
+
# serialize subqueries against the same shard as the outer query is currently
|
|
216
|
+
# targeted to run against
|
|
217
|
+
rel.shard!(primary_shard) if rel.shard_source_value == :implicit && rel.primary_shard != primary_shard
|
|
218
|
+
end
|
|
219
|
+
|
|
220
|
+
super
|
|
221
|
+
when Hash, ::Arel::Nodes::Node
|
|
222
|
+
where_clause = super
|
|
223
|
+
|
|
224
|
+
predicates = where_clause.send(:predicates)
|
|
225
|
+
infer_shards_from_primary_key(predicates) if shard_source_value == :implicit && shard_value.is_a?(Shard)
|
|
226
|
+
predicates = transpose_predicates(predicates, nil, primary_shard)
|
|
227
|
+
where_clause.instance_variable_set(:@predicates, predicates)
|
|
228
|
+
where_clause
|
|
229
|
+
else
|
|
230
|
+
super
|
|
231
|
+
end
|
|
230
232
|
end
|
|
231
233
|
|
|
232
234
|
def arel_columns(columns)
|
|
@@ -237,144 +239,86 @@ module Switchman
|
|
|
237
239
|
connection.with_local_table_name { super }
|
|
238
240
|
end
|
|
239
241
|
|
|
240
|
-
# semi-private
|
|
241
|
-
public
|
|
242
242
|
def transpose_predicates(predicates,
|
|
243
243
|
source_shard,
|
|
244
244
|
target_shard,
|
|
245
|
-
remove_nonlocal_primary_keys
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
transposed, binds = transpose_single_predicate(predicate, source_shard, target_shard, remove_nonlocal_primary_keys,
|
|
250
|
-
binds: binds, dup_binds_on_mutation: dup_binds_on_mutation)
|
|
251
|
-
transposed
|
|
245
|
+
remove_nonlocal_primary_keys: false)
|
|
246
|
+
predicates.map do |predicate|
|
|
247
|
+
transpose_single_predicate(predicate, source_shard, target_shard,
|
|
248
|
+
remove_nonlocal_primary_keys: remove_nonlocal_primary_keys)
|
|
252
249
|
end
|
|
253
|
-
result = [result, binds]
|
|
254
|
-
result
|
|
255
250
|
end
|
|
256
251
|
|
|
257
252
|
def transpose_single_predicate(predicate,
|
|
258
253
|
source_shard,
|
|
259
254
|
target_shard,
|
|
260
|
-
remove_nonlocal_primary_keys
|
|
261
|
-
binds: nil,
|
|
262
|
-
dup_binds_on_mutation: false)
|
|
255
|
+
remove_nonlocal_primary_keys: false)
|
|
263
256
|
if predicate.is_a?(::Arel::Nodes::Grouping)
|
|
264
|
-
return predicate
|
|
257
|
+
return predicate unless predicate.expr.is_a?(::Arel::Nodes::Or)
|
|
258
|
+
|
|
265
259
|
# Dang, we have an OR. OK, that means we have other epxressions below this
|
|
266
260
|
# level, perhaps many, that may need transposition.
|
|
267
261
|
# the left side and right side must each be treated as predicate lists and
|
|
268
262
|
# transformed in kind, if neither of them changes we can just return the grouping as is.
|
|
269
263
|
# hold on, it's about to get recursive...
|
|
270
|
-
#
|
|
271
|
-
# TODO: "binds" is getting passed up and down
|
|
272
|
-
# this stack purely because of the necessary handling for rails <5.2
|
|
273
|
-
# Dropping support for 5.2 means we can remove the "binds" argument from
|
|
274
|
-
# all of this and yank the conditional below where we monkey with their instance state.
|
|
275
264
|
or_expr = predicate.expr
|
|
276
265
|
left_node = or_expr.left
|
|
277
266
|
right_node = or_expr.right
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
binds: binds, dup_binds_on_mutation: dup_binds_on_mutation)
|
|
286
|
-
if new_left_predicates != left_predicates
|
|
287
|
-
left_node.instance_variable_set(:@children, new_left_predicates)
|
|
288
|
-
end
|
|
289
|
-
if new_right_predicates != right_predicates
|
|
290
|
-
right_node.instance_variable_set(:@children, new_right_predicates)
|
|
291
|
-
end
|
|
292
|
-
return predicate, binds
|
|
267
|
+
new_left_predicates = transpose_single_predicate(left_node, source_shard,
|
|
268
|
+
target_shard, remove_nonlocal_primary_keys: remove_nonlocal_primary_keys)
|
|
269
|
+
or_expr.instance_variable_set(:@left, new_left_predicates) if new_left_predicates != left_node
|
|
270
|
+
new_right_predicates = transpose_single_predicate(right_node, source_shard,
|
|
271
|
+
target_shard, remove_nonlocal_primary_keys: remove_nonlocal_primary_keys)
|
|
272
|
+
or_expr.instance_variable_set(:@right, new_right_predicates) if new_right_predicates != right_node
|
|
273
|
+
return predicate
|
|
293
274
|
end
|
|
294
|
-
return predicate
|
|
295
|
-
return predicate
|
|
275
|
+
return predicate unless predicate.is_a?(::Arel::Nodes::Binary) || predicate.is_a?(::Arel::Nodes::HomogeneousIn)
|
|
276
|
+
return predicate unless predicate.left.is_a?(::Arel::Attributes::Attribute)
|
|
277
|
+
|
|
296
278
|
relation, column = relation_and_column(predicate.left)
|
|
297
|
-
return predicate
|
|
279
|
+
return predicate unless (type = transposable_attribute_type(relation, column))
|
|
298
280
|
|
|
299
281
|
remove = true if type == :primary &&
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
282
|
+
remove_nonlocal_primary_keys &&
|
|
283
|
+
predicate.left.relation.klass == klass &&
|
|
284
|
+
predicate.is_a?(::Arel::Nodes::Equality)
|
|
303
285
|
|
|
304
286
|
current_source_shard =
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
287
|
+
if source_shard
|
|
288
|
+
source_shard
|
|
289
|
+
elsif type == :primary
|
|
290
|
+
Shard.current(klass.connection_classes)
|
|
291
|
+
elsif type == :foreign
|
|
292
|
+
source_shard_for_foreign_key(relation, column)
|
|
293
|
+
end
|
|
312
294
|
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
predicate.right.map {|val| transpose_predicate_value(val, current_source_shard, target_shard, type, remove) }
|
|
318
|
-
else
|
|
319
|
-
transpose_predicate_value(predicate.right, current_source_shard, target_shard, type, remove)
|
|
320
|
-
end
|
|
321
|
-
else
|
|
322
|
-
new_right_value = case predicate.right
|
|
323
|
-
when Array
|
|
324
|
-
local_ids = []
|
|
325
|
-
predicate.right.each do |value|
|
|
326
|
-
local_id = Shard.relative_id_for(value, current_source_shard, target_shard)
|
|
327
|
-
next unless local_id
|
|
328
|
-
unless remove && local_id > Shard::IDS_PER_SHARD
|
|
329
|
-
if value.is_a?(::Arel::Nodes::Casted)
|
|
330
|
-
if local_id == value.val
|
|
331
|
-
local_id = value
|
|
332
|
-
elsif local_id != value
|
|
333
|
-
local_id = value.class.new(local_id, value.attribute)
|
|
334
|
-
end
|
|
295
|
+
right = if predicate.is_a?(::Arel::Nodes::HomogeneousIn)
|
|
296
|
+
predicate.values
|
|
297
|
+
else
|
|
298
|
+
predicate.right
|
|
335
299
|
end
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
# look for a bind param with a matching column name
|
|
342
|
-
if binds && bind = binds.detect{|b| b&.name.to_s == predicate.left.name.to_s}
|
|
343
|
-
# before we mutate, dup
|
|
344
|
-
if dup_binds_on_mutation
|
|
345
|
-
binds = binds.map(&:dup)
|
|
346
|
-
dup_binds_on_mutation = false
|
|
347
|
-
bind = binds.find { |b| b&.name.to_s == predicate.left.name.to_s }
|
|
348
|
-
end
|
|
349
|
-
if bind.value.is_a?(::ActiveRecord::StatementCache::Substitute)
|
|
350
|
-
bind.value.sharded = true # mark for transposition later
|
|
351
|
-
bind.value.primary = true if type == :primary
|
|
352
|
-
else
|
|
353
|
-
local_id = Shard.relative_id_for(bind.value, current_source_shard, target_shard)
|
|
354
|
-
local_id = [] if remove && local_id > Shard::IDS_PER_SHARD
|
|
355
|
-
bind.instance_variable_set(:@value, local_id)
|
|
356
|
-
bind.instance_variable_set(:@value_for_database, nil)
|
|
357
|
-
end
|
|
358
|
-
end
|
|
359
|
-
predicate.right
|
|
300
|
+
|
|
301
|
+
new_right_value =
|
|
302
|
+
case right
|
|
303
|
+
when Array
|
|
304
|
+
right.map { |val| transpose_predicate_value(val, current_source_shard, target_shard, type, remove) }
|
|
360
305
|
else
|
|
361
|
-
|
|
362
|
-
local_id = [] if remove && local_id.is_a?(Integer) && local_id > Shard::IDS_PER_SHARD
|
|
363
|
-
local_id
|
|
306
|
+
transpose_predicate_value(right, current_source_shard, target_shard, type, remove)
|
|
364
307
|
end
|
|
365
|
-
|
|
366
|
-
|
|
308
|
+
|
|
309
|
+
if new_right_value == right
|
|
367
310
|
predicate
|
|
368
311
|
elsif predicate.right.is_a?(::Arel::Nodes::Casted)
|
|
369
|
-
if new_right_value ==
|
|
312
|
+
if new_right_value == right.value
|
|
370
313
|
predicate
|
|
371
314
|
else
|
|
372
|
-
predicate.class.new(predicate.left,
|
|
315
|
+
predicate.class.new(predicate.left, right.class.new(new_right_value, right.attribute))
|
|
373
316
|
end
|
|
317
|
+
elsif predicate.is_a?(::Arel::Nodes::HomogeneousIn)
|
|
318
|
+
predicate.class.new(new_right_value, predicate.attribute, predicate.type)
|
|
374
319
|
else
|
|
375
320
|
predicate.class.new(predicate.left, new_right_value)
|
|
376
321
|
end
|
|
377
|
-
return out_predicate, binds
|
|
378
322
|
end
|
|
379
323
|
|
|
380
324
|
def transpose_predicate_value(value, current_shard, target_shard, attribute_type, remove_non_local_ids)
|
|
@@ -388,11 +332,11 @@ module Switchman
|
|
|
388
332
|
else
|
|
389
333
|
local_id = Shard.relative_id_for(current_id, current_shard, target_shard) || current_id
|
|
390
334
|
local_id = [] if remove_non_local_ids && local_id.is_a?(Integer) && local_id > Shard::IDS_PER_SHARD
|
|
391
|
-
if current_id
|
|
335
|
+
if current_id == local_id
|
|
392
336
|
# make a new bind param
|
|
393
|
-
::Arel::Nodes::BindParam.new(query_att.class.new(query_att.name, local_id, query_att.type))
|
|
394
|
-
else
|
|
395
337
|
value
|
|
338
|
+
else
|
|
339
|
+
::Arel::Nodes::BindParam.new(query_att.class.new(query_att.name, local_id, query_att.type))
|
|
396
340
|
end
|
|
397
341
|
end
|
|
398
342
|
else
|
|
@@ -5,7 +5,7 @@ module Switchman
|
|
|
5
5
|
module Reflection
|
|
6
6
|
module AbstractReflection
|
|
7
7
|
def shard(owner)
|
|
8
|
-
if polymorphic? || klass.
|
|
8
|
+
if polymorphic? || klass.connection_classes == owner.class.connection_classes
|
|
9
9
|
# polymorphic associations assume the same shard as the owning item
|
|
10
10
|
owner.shard
|
|
11
11
|
else
|
|
@@ -28,21 +28,17 @@ module Switchman
|
|
|
28
28
|
# this technically belongs on AssociationReflection, but we put it on
|
|
29
29
|
# ThroughReflection as well, instead of delegating to its internal
|
|
30
30
|
# HasManyAssociation, losing its proper `klass`
|
|
31
|
-
def association_scope_cache(
|
|
32
|
-
key =
|
|
33
|
-
if polymorphic?
|
|
34
|
-
key = [key, owner._read_attribute(@foreign_type)]
|
|
35
|
-
end
|
|
31
|
+
def association_scope_cache(klass, owner, &block)
|
|
32
|
+
key = self
|
|
33
|
+
key = [key, owner._read_attribute(@foreign_type)] if polymorphic?
|
|
36
34
|
key = [key, shard(owner).id].flatten
|
|
37
|
-
|
|
38
|
-
@association_scope_cache[key] ||= (::Rails.version >= "5.2" ? ::ActiveRecord::StatementCache.create(conn, &block) : block.call)
|
|
39
|
-
}
|
|
35
|
+
klass.cached_find_by_statement(key, &block)
|
|
40
36
|
end
|
|
41
37
|
end
|
|
42
38
|
|
|
43
39
|
module AssociationReflection
|
|
44
40
|
def join_id_for(owner)
|
|
45
|
-
owner.send(
|
|
41
|
+
owner.send(join_foreign_key) # use sharded id values in association binds
|
|
46
42
|
end
|
|
47
43
|
end
|
|
48
44
|
end
|
|
@@ -4,53 +4,54 @@ module Switchman
|
|
|
4
4
|
module ActiveRecord
|
|
5
5
|
module Relation
|
|
6
6
|
def self.prepended(klass)
|
|
7
|
-
klass::SINGLE_VALUE_METHODS.concat [
|
|
7
|
+
klass::SINGLE_VALUE_METHODS.concat %i[shard shard_source]
|
|
8
8
|
end
|
|
9
9
|
|
|
10
10
|
def initialize(*, **)
|
|
11
11
|
super
|
|
12
|
-
self.shard_value = Shard.current(klass ? klass.
|
|
12
|
+
self.shard_value = Shard.current(klass ? klass.connection_classes : :primary) unless shard_value
|
|
13
13
|
self.shard_source_value = :implicit unless shard_source_value
|
|
14
14
|
end
|
|
15
15
|
|
|
16
16
|
def clone
|
|
17
17
|
result = super
|
|
18
|
-
result.shard_value = Shard.current(klass ? klass.
|
|
18
|
+
result.shard_value = Shard.current(klass ? klass.connection_classes : :primary) unless shard_value
|
|
19
19
|
result
|
|
20
20
|
end
|
|
21
21
|
|
|
22
22
|
def merge(*)
|
|
23
23
|
relation = super
|
|
24
|
-
if relation.shard_value !=
|
|
25
|
-
relation.shard_value =
|
|
26
|
-
relation.shard_source_value =
|
|
24
|
+
if relation.shard_value != shard_value && relation.shard_source_value == :implicit
|
|
25
|
+
relation.shard_value = shard_value
|
|
26
|
+
relation.shard_source_value = shard_source_value
|
|
27
27
|
end
|
|
28
28
|
relation
|
|
29
29
|
end
|
|
30
30
|
|
|
31
31
|
def new(*, &block)
|
|
32
|
-
primary_shard.activate(klass.
|
|
32
|
+
primary_shard.activate(klass.connection_classes) { super }
|
|
33
33
|
end
|
|
34
34
|
|
|
35
35
|
def create(*, &block)
|
|
36
|
-
primary_shard.activate(klass.
|
|
36
|
+
primary_shard.activate(klass.connection_classes) { super }
|
|
37
37
|
end
|
|
38
38
|
|
|
39
39
|
def create!(*, &block)
|
|
40
|
-
primary_shard.activate(klass.
|
|
40
|
+
primary_shard.activate(klass.connection_classes) { super }
|
|
41
41
|
end
|
|
42
42
|
|
|
43
43
|
def to_sql
|
|
44
|
-
primary_shard.activate(klass.
|
|
44
|
+
primary_shard.activate(klass.connection_classes) { super }
|
|
45
45
|
end
|
|
46
46
|
|
|
47
47
|
def explain
|
|
48
|
-
|
|
48
|
+
activate { |relation| relation.call_super(:explain, Relation) }
|
|
49
49
|
end
|
|
50
50
|
|
|
51
51
|
def records
|
|
52
52
|
return @records if loaded?
|
|
53
|
-
|
|
53
|
+
|
|
54
|
+
results = activate { |relation| relation.call_super(:records, Relation) }
|
|
54
55
|
case shard_value
|
|
55
56
|
when Array, ::ActiveRecord::Relation, ::ActiveRecord::Base
|
|
56
57
|
@records = results
|
|
@@ -59,7 +60,7 @@ module Switchman
|
|
|
59
60
|
results
|
|
60
61
|
end
|
|
61
62
|
|
|
62
|
-
%I
|
|
63
|
+
%I[update_all delete_all].each do |method|
|
|
63
64
|
class_eval <<-RUBY, __FILE__, __LINE__ + 1
|
|
64
65
|
def #{method}(*args)
|
|
65
66
|
result = self.activate { |relation| relation.call_super(#{method.inspect}, Relation, *args) }
|
|
@@ -74,15 +75,20 @@ module Switchman
|
|
|
74
75
|
loose_mode = options[:loose] && is_integer
|
|
75
76
|
# loose_mode: if we don't care about getting exactly batch_size ids in between
|
|
76
77
|
# don't get the max - just get the min and add batch_size so we get that many _at most_
|
|
77
|
-
values = loose_mode ?
|
|
78
|
+
values = loose_mode ? 'MIN(id)' : 'MIN(id), MAX(id)'
|
|
78
79
|
|
|
79
80
|
batch_size = options[:batch_size].try(:to_i) || 1000
|
|
80
81
|
quoted_primary_key = "#{klass.connection.quote_local_table_name(table_name)}.#{klass.connection.quote_column_name(primary_key)}"
|
|
81
|
-
as_id =
|
|
82
|
+
as_id = ' AS id' unless primary_key == 'id'
|
|
82
83
|
subquery_scope = except(:select).select("#{quoted_primary_key}#{as_id}").reorder(primary_key.to_sym).limit(loose_mode ? 1 : batch_size)
|
|
83
84
|
subquery_scope = subquery_scope.where("#{quoted_primary_key} <= ?", options[:end_at]) if options[:end_at]
|
|
84
85
|
|
|
85
|
-
first_subquery_scope =
|
|
86
|
+
first_subquery_scope = if options[:start_at]
|
|
87
|
+
subquery_scope.where("#{quoted_primary_key} >= ?",
|
|
88
|
+
options[:start_at])
|
|
89
|
+
else
|
|
90
|
+
subquery_scope
|
|
91
|
+
end
|
|
86
92
|
|
|
87
93
|
ids = connection.select_rows("SELECT #{values} FROM (#{first_subquery_scope.to_sql}) AS subquery").first
|
|
88
94
|
|
|
@@ -99,16 +105,16 @@ module Switchman
|
|
|
99
105
|
|
|
100
106
|
def activate(&block)
|
|
101
107
|
shards = all_shards
|
|
102
|
-
if
|
|
103
|
-
if shards.first == DefaultShard || shards.first == Shard.current(klass.
|
|
108
|
+
if Array === shards && shards.length == 1
|
|
109
|
+
if shards.first == DefaultShard || shards.first == Shard.current(klass.connection_classes)
|
|
104
110
|
yield(self, shards.first)
|
|
105
111
|
else
|
|
106
|
-
shards.first.activate(klass.
|
|
112
|
+
shards.first.activate(klass.connection_classes) { yield(self, shards.first) }
|
|
107
113
|
end
|
|
108
114
|
else
|
|
109
115
|
# TODO: implement local limit to avoid querying extra shards
|
|
110
|
-
Shard.with_each_shard(shards, [klass.
|
|
111
|
-
shard(Shard.current(klass.
|
|
116
|
+
Shard.with_each_shard(shards, [klass.connection_classes]) do
|
|
117
|
+
shard(Shard.current(klass.connection_classes), :to_a).activate(&block)
|
|
112
118
|
end
|
|
113
119
|
end
|
|
114
120
|
end
|