switchman 3.0.1 → 4.2.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Rakefile +16 -15
- data/db/migrate/20180828183945_add_default_shard_index.rb +2 -2
- data/db/migrate/20180828192111_add_timestamps_to_shards.rb +1 -1
- data/db/migrate/20190114212900_add_unique_name_indexes.rb +10 -4
- data/lib/switchman/action_controller/caching.rb +2 -2
- data/lib/switchman/active_record/abstract_adapter.rb +11 -18
- data/lib/switchman/active_record/associations.rb +315 -0
- data/lib/switchman/active_record/attribute_methods.rb +191 -79
- data/lib/switchman/active_record/base.rb +204 -50
- data/lib/switchman/active_record/calculations.rb +93 -50
- data/lib/switchman/active_record/connection_handler.rb +18 -0
- data/lib/switchman/active_record/connection_pool.rb +47 -34
- data/lib/switchman/active_record/database_configurations.rb +32 -6
- data/lib/switchman/active_record/finder_methods.rb +22 -16
- data/lib/switchman/active_record/log_subscriber.rb +3 -6
- data/lib/switchman/active_record/migration.rb +42 -14
- data/lib/switchman/active_record/model_schema.rb +1 -1
- data/lib/switchman/active_record/pending_migration_connection.rb +17 -0
- data/lib/switchman/active_record/persistence.rb +37 -2
- data/lib/switchman/active_record/postgresql_adapter.rb +39 -20
- data/lib/switchman/active_record/predicate_builder.rb +2 -2
- data/lib/switchman/active_record/query_cache.rb +26 -17
- data/lib/switchman/active_record/query_methods.rb +252 -135
- data/lib/switchman/active_record/reflection.rb +10 -3
- data/lib/switchman/active_record/relation.rb +154 -32
- data/lib/switchman/active_record/spawn_methods.rb +3 -7
- data/lib/switchman/active_record/statement_cache.rb +13 -9
- data/lib/switchman/active_record/table_definition.rb +1 -1
- data/lib/switchman/active_record/tasks/database_tasks.rb +6 -1
- data/lib/switchman/active_record/test_fixtures.rb +89 -0
- data/lib/switchman/active_support/cache.rb +25 -4
- data/lib/switchman/arel.rb +20 -7
- data/lib/switchman/call_super.rb +2 -2
- data/lib/switchman/database_server.rb +123 -83
- data/lib/switchman/default_shard.rb +14 -5
- data/lib/switchman/engine.rb +85 -131
- data/lib/switchman/environment.rb +2 -2
- data/lib/switchman/errors.rb +17 -2
- data/lib/switchman/guard_rail/relation.rb +7 -10
- data/lib/switchman/guard_rail.rb +5 -0
- data/lib/switchman/parallel.rb +68 -0
- data/lib/switchman/r_spec_helper.rb +17 -28
- data/lib/switchman/rails.rb +1 -4
- data/{app/models → lib}/switchman/shard.rb +229 -246
- data/lib/switchman/sharded_instrumenter.rb +9 -3
- data/lib/switchman/shared_schema_cache.rb +11 -0
- data/lib/switchman/standard_error.rb +15 -12
- data/lib/switchman/test_helper.rb +3 -3
- data/{app/models → lib}/switchman/unsharded_record.rb +1 -1
- data/lib/switchman/version.rb +1 -1
- data/lib/switchman.rb +46 -12
- data/lib/tasks/switchman.rake +101 -54
- metadata +34 -176
- data/lib/switchman/active_record/association.rb +0 -206
- data/lib/switchman/open4.rb +0 -80
|
@@ -25,6 +25,25 @@ module Switchman
|
|
|
25
25
|
@sharded_column_values[column_name]
|
|
26
26
|
end
|
|
27
27
|
|
|
28
|
+
def define_attribute_methods
|
|
29
|
+
result = super
|
|
30
|
+
# ensure that we're using the sharded attribute method
|
|
31
|
+
# and not the silly one in AR::AttributeMethods::PrimaryKey
|
|
32
|
+
return result unless sharded_column?(@primary_key)
|
|
33
|
+
|
|
34
|
+
class_eval(
|
|
35
|
+
build_sharded_getter("id",
|
|
36
|
+
"_read_attribute(@primary_key)",
|
|
37
|
+
"::#{connection_class_for_self.name}"),
|
|
38
|
+
__FILE__,
|
|
39
|
+
__LINE__
|
|
40
|
+
)
|
|
41
|
+
class_eval(build_sharded_setter("id", @primary_key, "::#{connection_class_for_self.name}"),
|
|
42
|
+
__FILE__,
|
|
43
|
+
__LINE__)
|
|
44
|
+
result
|
|
45
|
+
end
|
|
46
|
+
|
|
28
47
|
protected
|
|
29
48
|
|
|
30
49
|
def reflection_for_integer_attribute(attr_name)
|
|
@@ -36,110 +55,209 @@ module Switchman
|
|
|
36
55
|
raise if connection.open_transactions.positive?
|
|
37
56
|
end
|
|
38
57
|
|
|
39
|
-
def
|
|
58
|
+
def define_cached_method(owner, name, namespace:, as:, &)
|
|
59
|
+
if ::Rails.version < "7.1.4"
|
|
60
|
+
# https://github.com/rails/rails/commit/a2a12fc2e3f4e6d06f81d4c74c88f8e6b3369ee6#diff-5b59ece6d9396b596f06271cec0ea726e3360911383511c49b1a66f454bfc2b6L30
|
|
61
|
+
# These arguments were effectively swapped in Rails 7.1.4, so previous versions need them reversed
|
|
62
|
+
owner.define_cached_method(as, namespace:, as: name, &)
|
|
63
|
+
else
|
|
64
|
+
owner.define_cached_method(name, namespace:, as:, &)
|
|
65
|
+
end
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def define_method_global_attribute(attr_name, owner:, as: attr_name)
|
|
40
69
|
if sharded_column?(attr_name)
|
|
41
|
-
owner
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
70
|
+
define_cached_method(owner,
|
|
71
|
+
"sharded_global_#{attr_name}",
|
|
72
|
+
as: "global_#{as}",
|
|
73
|
+
namespace: :switchman) do |batch|
|
|
74
|
+
batch << <<-RUBY
|
|
75
|
+
def sharded_global_#{attr_name}
|
|
76
|
+
raw_value = original_#{attr_name}
|
|
77
|
+
return nil if raw_value.nil?
|
|
78
|
+
return raw_value if raw_value > ::Switchman::Shard::IDS_PER_SHARD
|
|
79
|
+
|
|
80
|
+
::Switchman::Shard.global_id_for(raw_value, shard)
|
|
81
|
+
end
|
|
82
|
+
RUBY
|
|
83
|
+
end
|
|
46
84
|
else
|
|
47
|
-
define_method_unsharded_column(attr_name,
|
|
85
|
+
define_method_unsharded_column(attr_name, "global", owner)
|
|
48
86
|
end
|
|
49
87
|
end
|
|
50
88
|
|
|
51
|
-
def define_method_local_attribute(attr_name, owner:)
|
|
89
|
+
def define_method_local_attribute(attr_name, owner:, as: attr_name)
|
|
52
90
|
if sharded_column?(attr_name)
|
|
53
|
-
owner
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
91
|
+
define_cached_method(owner,
|
|
92
|
+
"sharded_local_#{attr_name}",
|
|
93
|
+
as: "local_#{as}",
|
|
94
|
+
namespace: :switchman) do |batch|
|
|
95
|
+
batch << <<-RUBY
|
|
96
|
+
def sharded_local_#{attr_name}
|
|
97
|
+
raw_value = original_#{attr_name}
|
|
98
|
+
return nil if raw_value.nil?
|
|
99
|
+
return raw_value % ::Switchman::Shard::IDS_PER_SHARD
|
|
100
|
+
end
|
|
101
|
+
RUBY
|
|
102
|
+
end
|
|
58
103
|
else
|
|
59
|
-
define_method_unsharded_column(attr_name,
|
|
104
|
+
define_method_unsharded_column(attr_name, "local", owner)
|
|
60
105
|
end
|
|
61
106
|
end
|
|
62
107
|
|
|
63
|
-
# see also Base#
|
|
108
|
+
# see also Base#connection_class_for_self_for_reflection
|
|
64
109
|
# the difference being this will output static strings for the common cases, making them
|
|
65
110
|
# more performant
|
|
66
|
-
def
|
|
111
|
+
def connection_class_for_self_code_for_reflection(reflection)
|
|
67
112
|
if reflection
|
|
68
113
|
if reflection.options[:polymorphic]
|
|
69
114
|
# a polymorphic association has to be discovered at runtime. This code ends up being something like
|
|
70
|
-
# context_type.&.constantize&.
|
|
71
|
-
|
|
115
|
+
# context_type.&.constantize&.connection_class_for_self
|
|
116
|
+
<<~RUBY
|
|
117
|
+
begin
|
|
118
|
+
read_attribute(:#{reflection.foreign_type})&.constantize&.connection_class_for_self
|
|
119
|
+
rescue NameError
|
|
120
|
+
::ActiveRecord::Base
|
|
121
|
+
end
|
|
122
|
+
RUBY
|
|
72
123
|
else
|
|
73
124
|
# otherwise we can just return a symbol for the statically known type of the association
|
|
74
|
-
"::#{reflection.klass.
|
|
125
|
+
"::#{reflection.klass.connection_class_for_self.name}"
|
|
75
126
|
end
|
|
76
127
|
else
|
|
77
|
-
"::#{
|
|
128
|
+
"::#{connection_class_for_self.name}"
|
|
78
129
|
end
|
|
79
130
|
end
|
|
80
131
|
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
132
|
+
def define_method_attribute(attr_name, owner:, as: attr_name)
|
|
133
|
+
if sharded_column?(attr_name)
|
|
134
|
+
reflection = reflection_for_integer_attribute(attr_name)
|
|
135
|
+
class_name = connection_class_for_self_code_for_reflection(reflection)
|
|
136
|
+
safe_class_name = class_name.unpack1("h*")
|
|
137
|
+
define_cached_method(owner,
|
|
138
|
+
"sharded_#{safe_class_name}_#{attr_name}",
|
|
139
|
+
as:,
|
|
140
|
+
namespace: :switchman) do |batch|
|
|
141
|
+
batch << build_sharded_getter("sharded_#{safe_class_name}_#{attr_name}",
|
|
142
|
+
"original_#{as}",
|
|
143
|
+
class_name)
|
|
144
|
+
end
|
|
145
|
+
else
|
|
146
|
+
define_cached_method(owner, "plain_#{attr_name}", as:, namespace: :switchman) do |batch|
|
|
147
|
+
batch << <<-RUBY
|
|
148
|
+
def plain_#{attr_name}
|
|
149
|
+
_read_attribute("#{attr_name}") { |n| missing_attribute(n, caller) }
|
|
150
|
+
end
|
|
151
|
+
RUBY
|
|
152
|
+
end
|
|
86
153
|
end
|
|
154
|
+
end
|
|
87
155
|
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
156
|
+
def build_sharded_getter(attr_name, raw_expr, attr_connection_class)
|
|
157
|
+
<<-RUBY
|
|
158
|
+
def #{attr_name}
|
|
159
|
+
raw_value = #{raw_expr}
|
|
160
|
+
return nil if raw_value.nil?
|
|
161
|
+
|
|
162
|
+
abs_raw_value = raw_value.abs
|
|
163
|
+
current_shard = ::Switchman::Shard.current(#{attr_connection_class})
|
|
164
|
+
same_shard = loaded_from_shard == current_shard
|
|
165
|
+
return raw_value if same_shard && abs_raw_value < ::Switchman::Shard::IDS_PER_SHARD
|
|
166
|
+
|
|
167
|
+
value_shard_id = abs_raw_value / ::Switchman::Shard::IDS_PER_SHARD
|
|
168
|
+
# this is a stupid case when someone stuffed a global id for the current shard in instead
|
|
169
|
+
# of a local id
|
|
170
|
+
return raw_value % ::Switchman::Shard::IDS_PER_SHARD if value_shard_id == current_shard.id
|
|
171
|
+
return raw_value if !same_shard && abs_raw_value > ::Switchman::Shard::IDS_PER_SHARD
|
|
172
|
+
return loaded_from_shard.global_id_for(raw_value) if !same_shard && abs_raw_value < ::Switchman::Shard::IDS_PER_SHARD
|
|
173
|
+
|
|
174
|
+
::Switchman::Shard.relative_id_for(raw_value, loaded_from_shard, current_shard)
|
|
175
|
+
end
|
|
176
|
+
RUBY
|
|
91
177
|
end
|
|
92
178
|
|
|
93
|
-
def
|
|
179
|
+
def define_method_attribute=(attr_name, owner:, as: attr_name)
|
|
94
180
|
if sharded_column?(attr_name)
|
|
95
181
|
reflection = reflection_for_integer_attribute(attr_name)
|
|
96
|
-
|
|
97
|
-
|
|
182
|
+
class_name = connection_class_for_self_code_for_reflection(reflection)
|
|
183
|
+
safe_class_name = class_name.unpack1("h*")
|
|
184
|
+
define_cached_method(owner,
|
|
185
|
+
"sharded_#{safe_class_name}_#{attr_name}=",
|
|
186
|
+
as: "#{as}=",
|
|
187
|
+
namespace: :switchman) do |batch|
|
|
188
|
+
batch << build_sharded_setter("sharded_#{safe_class_name}_#{attr_name}", attr_name, class_name)
|
|
189
|
+
end
|
|
190
|
+
else
|
|
191
|
+
define_cached_method(owner, "plain_#{attr_name}=", as: "#{as}=", namespace: :switchman) do |batch|
|
|
192
|
+
batch << <<-RUBY
|
|
193
|
+
def plain_#{attr_name}=(new_value)
|
|
194
|
+
_write_attribute('#{attr_name}', new_value)
|
|
195
|
+
end
|
|
196
|
+
RUBY
|
|
197
|
+
end
|
|
198
|
+
end
|
|
199
|
+
end
|
|
98
200
|
|
|
99
|
-
|
|
201
|
+
def build_sharded_setter(attr_name, attr_field, attr_connection_class)
|
|
202
|
+
<<-RUBY
|
|
203
|
+
def #{attr_name}=(new_value)
|
|
204
|
+
self.original_#{attr_field} = ::Switchman::Shard.relative_id_for(new_value, ::Switchman::Shard.current(#{attr_connection_class}), loaded_from_shard)
|
|
100
205
|
end
|
|
206
|
+
RUBY
|
|
207
|
+
end
|
|
101
208
|
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
209
|
+
def define_method_original_attribute(attr_name, owner:, as: attr_name)
|
|
210
|
+
if sharded_column?(attr_name)
|
|
211
|
+
define_cached_method(owner,
|
|
212
|
+
"sharded_original_#{attr_name}",
|
|
213
|
+
as: "original_#{as}",
|
|
214
|
+
namespace: :switchman) do |batch|
|
|
215
|
+
batch << <<-RUBY
|
|
216
|
+
def sharded_original_#{attr_name}
|
|
217
|
+
_read_attribute("#{attr_name}") { |n| missing_attribute(n, caller) }
|
|
218
|
+
end
|
|
219
|
+
RUBY
|
|
220
|
+
end
|
|
221
|
+
else
|
|
222
|
+
define_method_unsharded_column(attr_name, "global", owner)
|
|
223
|
+
end
|
|
224
|
+
end
|
|
109
225
|
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
226
|
+
def define_method_original_attribute=(attr_name, owner:, as: attr_name)
|
|
227
|
+
return unless sharded_column?(attr_name)
|
|
228
|
+
|
|
229
|
+
define_cached_method(owner,
|
|
230
|
+
"sharded_original_#{attr_name}=",
|
|
231
|
+
as: "original_#{as}=",
|
|
232
|
+
namespace: :switchman) do |batch|
|
|
233
|
+
batch << <<-RUBY
|
|
234
|
+
def sharded_original_#{attr_name}=(new_value)
|
|
235
|
+
_write_attribute('#{attr_name}', new_value)
|
|
113
236
|
end
|
|
114
237
|
RUBY
|
|
115
|
-
else
|
|
116
|
-
define_method_unsharded_column(attr_name, 'global', owner)
|
|
117
238
|
end
|
|
118
239
|
end
|
|
119
240
|
|
|
120
241
|
def define_method_unsharded_column(attr_name, prefix, owner)
|
|
121
|
-
return if columns_hash["#{prefix}_#{attr_name}"]
|
|
242
|
+
return if columns_hash["#{prefix}_#{attr_name}"] || attr_name == "id"
|
|
122
243
|
|
|
123
|
-
owner
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
244
|
+
define_cached_method(owner,
|
|
245
|
+
"unsharded_#{prefix}_#{attr_name}",
|
|
246
|
+
as: "#{prefix}_#{attr_name}",
|
|
247
|
+
namespace: :switchman) do |batch|
|
|
248
|
+
batch << <<-RUBY
|
|
249
|
+
def unsharded_#{prefix}_#{attr_name}
|
|
250
|
+
raise NoMethodError, "undefined method `#{prefix}_#{attr_name}'; are you missing an association?"
|
|
251
|
+
end
|
|
252
|
+
RUBY
|
|
253
|
+
end
|
|
128
254
|
end
|
|
129
255
|
end
|
|
130
256
|
|
|
131
|
-
def self.
|
|
132
|
-
klass.singleton_class.
|
|
133
|
-
klass.attribute_method_prefix
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
# ensure that we're using the sharded attribute method
|
|
137
|
-
# and not the silly one in AR::AttributeMethods::PrimaryKey
|
|
138
|
-
def id
|
|
139
|
-
return super if is_a?(Shard)
|
|
140
|
-
|
|
141
|
-
self.class.define_attribute_methods
|
|
142
|
-
super
|
|
257
|
+
def self.prepended(klass)
|
|
258
|
+
klass.singleton_class.prepend(ClassMethods)
|
|
259
|
+
klass.attribute_method_prefix "global_", "local_", "original_"
|
|
260
|
+
klass.attribute_method_affix prefix: "original_", suffix: "="
|
|
143
261
|
end
|
|
144
262
|
|
|
145
263
|
# these are called if the specific methods haven't been defined yet
|
|
@@ -147,7 +265,11 @@ module Switchman
|
|
|
147
265
|
return super unless self.class.sharded_column?(attr_name)
|
|
148
266
|
|
|
149
267
|
reflection = self.class.send(:reflection_for_integer_attribute, attr_name)
|
|
150
|
-
::Switchman::Shard.relative_id_for(
|
|
268
|
+
::Switchman::Shard.relative_id_for(
|
|
269
|
+
super,
|
|
270
|
+
shard,
|
|
271
|
+
::Switchman::Shard.current(connection_class_for_self_for_reflection(reflection))
|
|
272
|
+
)
|
|
151
273
|
end
|
|
152
274
|
|
|
153
275
|
def attribute=(attr_name, new_value)
|
|
@@ -157,7 +279,11 @@ module Switchman
|
|
|
157
279
|
end
|
|
158
280
|
|
|
159
281
|
reflection = self.class.send(:reflection_for_integer_attribute, attr_name)
|
|
160
|
-
super(
|
|
282
|
+
super(attr_name, ::Switchman::Shard.relative_id_for(
|
|
283
|
+
new_value,
|
|
284
|
+
::Switchman::Shard.current(connection_class_for_self_for_reflection(reflection)),
|
|
285
|
+
shard
|
|
286
|
+
))
|
|
161
287
|
end
|
|
162
288
|
|
|
163
289
|
def global_attribute(attr_name)
|
|
@@ -170,25 +296,11 @@ module Switchman
|
|
|
170
296
|
|
|
171
297
|
def local_attribute(attr_name)
|
|
172
298
|
if self.class.sharded_column?(attr_name)
|
|
173
|
-
::Switchman::Shard.local_id_for(attribute(attr_name)
|
|
299
|
+
::Switchman::Shard.local_id_for(attribute(attr_name)).first
|
|
174
300
|
else
|
|
175
301
|
attribute(attr_name)
|
|
176
302
|
end
|
|
177
303
|
end
|
|
178
|
-
|
|
179
|
-
private
|
|
180
|
-
|
|
181
|
-
def connection_classes_for_reflection(reflection)
|
|
182
|
-
if reflection
|
|
183
|
-
if reflection.options[:polymorphic]
|
|
184
|
-
read_attribute(reflection.foreign_type)&.constantize&.connection_classes
|
|
185
|
-
else
|
|
186
|
-
reflection.klass.connection_classes
|
|
187
|
-
end
|
|
188
|
-
else
|
|
189
|
-
self.class.connection_classes
|
|
190
|
-
end
|
|
191
|
-
end
|
|
192
304
|
end
|
|
193
305
|
end
|
|
194
306
|
end
|