switchman 3.0.14 → 3.5.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Rakefile +16 -15
- data/db/migrate/20180828183945_add_default_shard_index.rb +1 -1
- data/db/migrate/20190114212900_add_unique_name_indexes.rb +10 -4
- data/lib/switchman/action_controller/caching.rb +2 -2
- data/lib/switchman/active_record/abstract_adapter.rb +6 -6
- data/lib/switchman/active_record/associations.rb +365 -0
- data/lib/switchman/active_record/attribute_methods.rb +188 -99
- data/lib/switchman/active_record/base.rb +185 -40
- data/lib/switchman/active_record/calculations.rb +64 -40
- data/lib/switchman/active_record/connection_handler.rb +18 -0
- data/lib/switchman/active_record/connection_pool.rb +24 -5
- data/lib/switchman/active_record/database_configurations.rb +37 -13
- data/lib/switchman/active_record/finder_methods.rb +46 -16
- data/lib/switchman/active_record/log_subscriber.rb +11 -5
- data/lib/switchman/active_record/migration.rb +52 -8
- data/lib/switchman/active_record/model_schema.rb +1 -1
- data/lib/switchman/active_record/persistence.rb +31 -3
- data/lib/switchman/active_record/postgresql_adapter.rb +11 -10
- data/lib/switchman/active_record/predicate_builder.rb +2 -2
- data/lib/switchman/active_record/query_cache.rb +49 -20
- data/lib/switchman/active_record/query_methods.rb +187 -136
- data/lib/switchman/active_record/reflection.rb +1 -1
- data/lib/switchman/active_record/relation.rb +33 -26
- data/lib/switchman/active_record/spawn_methods.rb +2 -2
- data/lib/switchman/active_record/statement_cache.rb +11 -7
- data/lib/switchman/active_record/table_definition.rb +1 -1
- data/lib/switchman/active_record/tasks/database_tasks.rb +6 -1
- data/lib/switchman/active_record/test_fixtures.rb +26 -16
- data/lib/switchman/active_support/cache.rb +20 -1
- data/lib/switchman/arel.rb +34 -18
- data/lib/switchman/call_super.rb +8 -2
- data/lib/switchman/database_server.rb +91 -45
- data/lib/switchman/default_shard.rb +14 -5
- data/lib/switchman/engine.rb +79 -126
- data/lib/switchman/environment.rb +2 -2
- data/lib/switchman/errors.rb +17 -2
- data/lib/switchman/guard_rail/relation.rb +8 -10
- data/lib/switchman/guard_rail.rb +5 -0
- data/lib/switchman/parallel.rb +68 -0
- data/lib/switchman/r_spec_helper.rb +14 -11
- data/lib/switchman/rails.rb +2 -5
- data/{app/models → lib}/switchman/shard.rb +186 -189
- data/lib/switchman/sharded_instrumenter.rb +5 -1
- data/lib/switchman/shared_schema_cache.rb +11 -0
- data/lib/switchman/standard_error.rb +6 -5
- data/lib/switchman/test_helper.rb +2 -2
- data/{app/models → lib}/switchman/unsharded_record.rb +1 -1
- data/lib/switchman/version.rb +1 -1
- data/lib/switchman.rb +44 -12
- data/lib/tasks/switchman.rake +74 -53
- metadata +42 -53
- data/lib/switchman/active_record/association.rb +0 -206
- data/lib/switchman/open4.rb +0 -80
|
@@ -25,6 +25,24 @@ module Switchman
|
|
|
25
25
|
@sharded_column_values[column_name]
|
|
26
26
|
end
|
|
27
27
|
|
|
28
|
+
def define_attribute_methods
|
|
29
|
+
super
|
|
30
|
+
# ensure that we're using the sharded attribute method
|
|
31
|
+
# and not the silly one in AR::AttributeMethods::PrimaryKey
|
|
32
|
+
return unless sharded_column?(@primary_key)
|
|
33
|
+
|
|
34
|
+
class_eval(
|
|
35
|
+
build_sharded_getter("id",
|
|
36
|
+
"_read_attribute(@primary_key)",
|
|
37
|
+
"::#{connection_class_for_self.name}"),
|
|
38
|
+
__FILE__,
|
|
39
|
+
__LINE__
|
|
40
|
+
)
|
|
41
|
+
class_eval(build_sharded_setter("id", @primary_key, "::#{connection_class_for_self.name}"),
|
|
42
|
+
__FILE__,
|
|
43
|
+
__LINE__)
|
|
44
|
+
end
|
|
45
|
+
|
|
28
46
|
protected
|
|
29
47
|
|
|
30
48
|
def reflection_for_integer_attribute(attr_name)
|
|
@@ -36,131 +54,208 @@ module Switchman
|
|
|
36
54
|
raise if connection.open_transactions.positive?
|
|
37
55
|
end
|
|
38
56
|
|
|
57
|
+
def define_cached_method(owner, name, namespace:, as:, &block)
|
|
58
|
+
if ::Rails.version < "7.0"
|
|
59
|
+
yield owner
|
|
60
|
+
owner.rename_method(as, name)
|
|
61
|
+
else
|
|
62
|
+
owner.define_cached_method(name, namespace: namespace, as: as, &block)
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
|
|
39
66
|
def define_method_global_attribute(attr_name, owner:)
|
|
40
67
|
if sharded_column?(attr_name)
|
|
41
|
-
owner
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
68
|
+
define_cached_method(owner,
|
|
69
|
+
"global_#{attr_name}",
|
|
70
|
+
as: "sharded_global_#{attr_name}",
|
|
71
|
+
namespace: :switchman) do |batch|
|
|
72
|
+
batch << <<-RUBY
|
|
73
|
+
def sharded_global_#{attr_name}
|
|
74
|
+
raw_value = original_#{attr_name}
|
|
75
|
+
return nil if raw_value.nil?
|
|
76
|
+
return raw_value if raw_value > ::Switchman::Shard::IDS_PER_SHARD
|
|
77
|
+
|
|
78
|
+
::Switchman::Shard.global_id_for(raw_value, shard)
|
|
79
|
+
end
|
|
80
|
+
RUBY
|
|
81
|
+
end
|
|
50
82
|
else
|
|
51
|
-
define_method_unsharded_column(attr_name,
|
|
83
|
+
define_method_unsharded_column(attr_name, "global", owner)
|
|
52
84
|
end
|
|
53
85
|
end
|
|
54
86
|
|
|
55
87
|
def define_method_local_attribute(attr_name, owner:)
|
|
56
88
|
if sharded_column?(attr_name)
|
|
57
|
-
owner
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
89
|
+
define_cached_method(owner,
|
|
90
|
+
"local_#{attr_name}",
|
|
91
|
+
as: "sharded_local_#{attr_name}",
|
|
92
|
+
namespace: :switchman) do |batch|
|
|
93
|
+
batch << <<-RUBY
|
|
94
|
+
def sharded_local_#{attr_name}
|
|
95
|
+
raw_value = original_#{attr_name}
|
|
96
|
+
return nil if raw_value.nil?
|
|
97
|
+
return raw_value % ::Switchman::Shard::IDS_PER_SHARD
|
|
98
|
+
end
|
|
99
|
+
RUBY
|
|
100
|
+
end
|
|
64
101
|
else
|
|
65
|
-
define_method_unsharded_column(attr_name,
|
|
102
|
+
define_method_unsharded_column(attr_name, "local", owner)
|
|
66
103
|
end
|
|
67
104
|
end
|
|
68
105
|
|
|
69
|
-
# see also Base#
|
|
106
|
+
# see also Base#connection_class_for_self_for_reflection
|
|
70
107
|
# the difference being this will output static strings for the common cases, making them
|
|
71
108
|
# more performant
|
|
72
|
-
def
|
|
109
|
+
def connection_class_for_self_code_for_reflection(reflection)
|
|
73
110
|
if reflection
|
|
74
111
|
if reflection.options[:polymorphic]
|
|
75
112
|
# a polymorphic association has to be discovered at runtime. This code ends up being something like
|
|
76
|
-
# context_type.&.constantize&.
|
|
77
|
-
|
|
113
|
+
# context_type.&.constantize&.connection_class_for_self
|
|
114
|
+
<<~RUBY
|
|
115
|
+
begin
|
|
116
|
+
read_attribute(:#{reflection.foreign_type})&.constantize&.connection_class_for_self
|
|
117
|
+
rescue NameError
|
|
118
|
+
::ActiveRecord::Base
|
|
119
|
+
end
|
|
120
|
+
RUBY
|
|
78
121
|
else
|
|
79
122
|
# otherwise we can just return a symbol for the statically known type of the association
|
|
80
|
-
"::#{reflection.klass.
|
|
123
|
+
"::#{reflection.klass.connection_class_for_self.name}"
|
|
81
124
|
end
|
|
82
125
|
else
|
|
83
|
-
"::#{
|
|
126
|
+
"::#{connection_class_for_self.name}"
|
|
84
127
|
end
|
|
85
128
|
end
|
|
86
129
|
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
130
|
+
def define_method_attribute(attr_name, owner:)
|
|
131
|
+
if sharded_column?(attr_name)
|
|
132
|
+
reflection = reflection_for_integer_attribute(attr_name)
|
|
133
|
+
class_name = connection_class_for_self_code_for_reflection(reflection)
|
|
134
|
+
safe_class_name = class_name.unpack1("h*")
|
|
135
|
+
define_cached_method(owner,
|
|
136
|
+
attr_name,
|
|
137
|
+
as: "sharded_#{safe_class_name}_#{attr_name}",
|
|
138
|
+
namespace: :switchman) do |batch|
|
|
139
|
+
batch << build_sharded_getter("sharded_#{safe_class_name}_#{attr_name}",
|
|
140
|
+
"original_#{attr_name}",
|
|
141
|
+
class_name)
|
|
142
|
+
end
|
|
143
|
+
else
|
|
144
|
+
define_cached_method(owner, attr_name, as: "plain_#{attr_name}", namespace: :switchman) do |batch|
|
|
145
|
+
batch << <<-RUBY
|
|
146
|
+
def plain_#{attr_name}
|
|
147
|
+
_read_attribute("#{attr_name}") { |n| missing_attribute(n, caller) }
|
|
148
|
+
end
|
|
149
|
+
RUBY
|
|
150
|
+
end
|
|
92
151
|
end
|
|
152
|
+
end
|
|
93
153
|
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
154
|
+
def build_sharded_getter(attr_name, raw_expr, attr_connection_class)
|
|
155
|
+
<<-RUBY
|
|
156
|
+
def #{attr_name}
|
|
157
|
+
raw_value = #{raw_expr}
|
|
158
|
+
return nil if raw_value.nil?
|
|
159
|
+
|
|
160
|
+
abs_raw_value = raw_value.abs
|
|
161
|
+
current_shard = ::Switchman::Shard.current(#{attr_connection_class})
|
|
162
|
+
same_shard = loaded_from_shard == current_shard
|
|
163
|
+
return raw_value if same_shard && abs_raw_value < ::Switchman::Shard::IDS_PER_SHARD
|
|
164
|
+
|
|
165
|
+
value_shard_id = abs_raw_value / ::Switchman::Shard::IDS_PER_SHARD
|
|
166
|
+
# this is a stupid case when someone stuffed a global id for the current shard in instead
|
|
167
|
+
# of a local id
|
|
168
|
+
return raw_value % ::Switchman::Shard::IDS_PER_SHARD if value_shard_id == current_shard.id
|
|
169
|
+
return raw_value if !same_shard && abs_raw_value > ::Switchman::Shard::IDS_PER_SHARD
|
|
170
|
+
return loaded_from_shard.global_id_for(raw_value) if !same_shard && abs_raw_value < ::Switchman::Shard::IDS_PER_SHARD
|
|
171
|
+
|
|
172
|
+
::Switchman::Shard.relative_id_for(raw_value, loaded_from_shard, current_shard)
|
|
173
|
+
end
|
|
174
|
+
RUBY
|
|
97
175
|
end
|
|
98
176
|
|
|
99
|
-
def
|
|
177
|
+
def define_method_attribute=(attr_name, owner:)
|
|
100
178
|
if sharded_column?(attr_name)
|
|
101
179
|
reflection = reflection_for_integer_attribute(attr_name)
|
|
102
|
-
|
|
103
|
-
|
|
180
|
+
class_name = connection_class_for_self_code_for_reflection(reflection)
|
|
181
|
+
safe_class_name = class_name.unpack1("h*")
|
|
182
|
+
define_cached_method(owner,
|
|
183
|
+
"#{attr_name}=",
|
|
184
|
+
as: "sharded_#{safe_class_name}_#{attr_name}=",
|
|
185
|
+
namespace: :switchman) do |batch|
|
|
186
|
+
batch << build_sharded_setter("sharded_#{safe_class_name}_#{attr_name}", attr_name, class_name)
|
|
187
|
+
end
|
|
188
|
+
else
|
|
189
|
+
define_cached_method(owner, "#{attr_name}=", as: "plain_#{attr_name}=", namespace: :switchman) do |batch|
|
|
190
|
+
batch << <<-RUBY
|
|
191
|
+
def plain_#{attr_name}=(new_value)
|
|
192
|
+
_write_attribute('#{attr_name}', new_value)
|
|
193
|
+
end
|
|
194
|
+
RUBY
|
|
195
|
+
end
|
|
196
|
+
end
|
|
197
|
+
end
|
|
104
198
|
|
|
105
|
-
|
|
199
|
+
def build_sharded_setter(attr_name, attr_field, attr_connection_class)
|
|
200
|
+
<<-RUBY
|
|
201
|
+
def #{attr_name}=(new_value)
|
|
202
|
+
self.original_#{attr_field} = ::Switchman::Shard.relative_id_for(new_value, ::Switchman::Shard.current(#{attr_connection_class}), loaded_from_shard)
|
|
106
203
|
end
|
|
204
|
+
RUBY
|
|
205
|
+
end
|
|
107
206
|
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
::Switchman::Shard.relative_id_for(raw_value, shard, current_shard)
|
|
129
|
-
end
|
|
207
|
+
def define_method_original_attribute(attr_name, owner:)
|
|
208
|
+
if sharded_column?(attr_name)
|
|
209
|
+
define_cached_method(owner,
|
|
210
|
+
"original_#{attr_name}",
|
|
211
|
+
as: "sharded_original_#{attr_name}",
|
|
212
|
+
namespace: :switchman) do |batch|
|
|
213
|
+
batch << <<-RUBY
|
|
214
|
+
def sharded_original_#{attr_name}
|
|
215
|
+
_read_attribute("#{attr_name}") { |n| missing_attribute(n, caller) }
|
|
216
|
+
end
|
|
217
|
+
RUBY
|
|
218
|
+
end
|
|
219
|
+
else
|
|
220
|
+
define_method_unsharded_column(attr_name, "global", owner)
|
|
221
|
+
end
|
|
222
|
+
end
|
|
223
|
+
|
|
224
|
+
def define_method_original_attribute=(attr_name, owner:)
|
|
225
|
+
return unless sharded_column?(attr_name)
|
|
130
226
|
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
227
|
+
define_cached_method(owner,
|
|
228
|
+
"original_#{attr_name}=",
|
|
229
|
+
as: "sharded_original_#{attr_name}=",
|
|
230
|
+
namespace: :switchman) do |batch|
|
|
231
|
+
batch << <<-RUBY
|
|
232
|
+
def sharded_original_#{attr_name}=(new_value)
|
|
233
|
+
_write_attribute('#{attr_name}', new_value)
|
|
134
234
|
end
|
|
135
235
|
RUBY
|
|
136
|
-
else
|
|
137
|
-
define_method_unsharded_column(attr_name, 'global', owner)
|
|
138
236
|
end
|
|
139
237
|
end
|
|
140
238
|
|
|
141
239
|
def define_method_unsharded_column(attr_name, prefix, owner)
|
|
142
|
-
return if columns_hash["#{prefix}_#{attr_name}"]
|
|
143
|
-
|
|
144
|
-
owner
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
240
|
+
return if columns_hash["#{prefix}_#{attr_name}"] || attr_name == "id"
|
|
241
|
+
|
|
242
|
+
define_cached_method(owner,
|
|
243
|
+
"#{prefix}_#{attr_name}",
|
|
244
|
+
as: "unsharded_#{prefix}_#{attr_name}",
|
|
245
|
+
namespace: :switchman) do |batch|
|
|
246
|
+
batch << <<-RUBY
|
|
247
|
+
def unsharded_#{prefix}_#{attr_name}
|
|
248
|
+
raise NoMethodError, "undefined method `#{prefix}_#{attr_name}'; are you missing an association?"
|
|
249
|
+
end
|
|
250
|
+
RUBY
|
|
251
|
+
end
|
|
149
252
|
end
|
|
150
253
|
end
|
|
151
254
|
|
|
152
|
-
def self.
|
|
153
|
-
klass.singleton_class.
|
|
154
|
-
klass.attribute_method_prefix
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
# ensure that we're using the sharded attribute method
|
|
158
|
-
# and not the silly one in AR::AttributeMethods::PrimaryKey
|
|
159
|
-
def id
|
|
160
|
-
return super if is_a?(Shard)
|
|
161
|
-
|
|
162
|
-
self.class.define_attribute_methods
|
|
163
|
-
super
|
|
255
|
+
def self.prepended(klass)
|
|
256
|
+
klass.singleton_class.prepend(ClassMethods)
|
|
257
|
+
klass.attribute_method_prefix "global_", "local_", "original_"
|
|
258
|
+
klass.attribute_method_affix prefix: "original_", suffix: "="
|
|
164
259
|
end
|
|
165
260
|
|
|
166
261
|
# these are called if the specific methods haven't been defined yet
|
|
@@ -168,7 +263,11 @@ module Switchman
|
|
|
168
263
|
return super unless self.class.sharded_column?(attr_name)
|
|
169
264
|
|
|
170
265
|
reflection = self.class.send(:reflection_for_integer_attribute, attr_name)
|
|
171
|
-
::Switchman::Shard.relative_id_for(
|
|
266
|
+
::Switchman::Shard.relative_id_for(
|
|
267
|
+
super,
|
|
268
|
+
shard,
|
|
269
|
+
::Switchman::Shard.current(connection_class_for_self_for_reflection(reflection))
|
|
270
|
+
)
|
|
172
271
|
end
|
|
173
272
|
|
|
174
273
|
def attribute=(attr_name, new_value)
|
|
@@ -178,7 +277,11 @@ module Switchman
|
|
|
178
277
|
end
|
|
179
278
|
|
|
180
279
|
reflection = self.class.send(:reflection_for_integer_attribute, attr_name)
|
|
181
|
-
super(
|
|
280
|
+
super(attr_name, ::Switchman::Shard.relative_id_for(
|
|
281
|
+
new_value,
|
|
282
|
+
::Switchman::Shard.current(connection_class_for_self_for_reflection(reflection)),
|
|
283
|
+
shard
|
|
284
|
+
))
|
|
182
285
|
end
|
|
183
286
|
|
|
184
287
|
def global_attribute(attr_name)
|
|
@@ -191,25 +294,11 @@ module Switchman
|
|
|
191
294
|
|
|
192
295
|
def local_attribute(attr_name)
|
|
193
296
|
if self.class.sharded_column?(attr_name)
|
|
194
|
-
::Switchman::Shard.local_id_for(attribute(attr_name)
|
|
297
|
+
::Switchman::Shard.local_id_for(attribute(attr_name)).first
|
|
195
298
|
else
|
|
196
299
|
attribute(attr_name)
|
|
197
300
|
end
|
|
198
301
|
end
|
|
199
|
-
|
|
200
|
-
private
|
|
201
|
-
|
|
202
|
-
def connection_classes_for_reflection(reflection)
|
|
203
|
-
if reflection
|
|
204
|
-
if reflection.options[:polymorphic]
|
|
205
|
-
read_attribute(reflection.foreign_type)&.constantize&.connection_classes
|
|
206
|
-
else
|
|
207
|
-
reflection.klass.connection_classes
|
|
208
|
-
end
|
|
209
|
-
else
|
|
210
|
-
self.class.connection_classes
|
|
211
|
-
end
|
|
212
|
-
end
|
|
213
302
|
end
|
|
214
303
|
end
|
|
215
304
|
end
|