switchman 3.0.5 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Rakefile +16 -15
- data/db/migrate/20180828183945_add_default_shard_index.rb +2 -2
- data/db/migrate/20180828192111_add_timestamps_to_shards.rb +1 -1
- data/db/migrate/20190114212900_add_unique_name_indexes.rb +10 -4
- data/lib/switchman/action_controller/caching.rb +2 -2
- data/lib/switchman/active_record/abstract_adapter.rb +6 -15
- data/lib/switchman/active_record/associations.rb +331 -0
- data/lib/switchman/active_record/attribute_methods.rb +182 -77
- data/lib/switchman/active_record/base.rb +249 -46
- data/lib/switchman/active_record/calculations.rb +98 -44
- data/lib/switchman/active_record/connection_handler.rb +18 -0
- data/lib/switchman/active_record/connection_pool.rb +27 -28
- data/lib/switchman/active_record/database_configurations.rb +44 -6
- data/lib/switchman/active_record/finder_methods.rb +46 -16
- data/lib/switchman/active_record/log_subscriber.rb +11 -5
- data/lib/switchman/active_record/migration.rb +52 -5
- data/lib/switchman/active_record/model_schema.rb +1 -1
- data/lib/switchman/active_record/pending_migration_connection.rb +17 -0
- data/lib/switchman/active_record/persistence.rb +37 -2
- data/lib/switchman/active_record/postgresql_adapter.rb +12 -11
- data/lib/switchman/active_record/predicate_builder.rb +2 -2
- data/lib/switchman/active_record/query_cache.rb +49 -20
- data/lib/switchman/active_record/query_methods.rb +202 -136
- data/lib/switchman/active_record/reflection.rb +1 -1
- data/lib/switchman/active_record/relation.rb +40 -28
- data/lib/switchman/active_record/spawn_methods.rb +2 -2
- data/lib/switchman/active_record/statement_cache.rb +11 -7
- data/lib/switchman/active_record/table_definition.rb +1 -1
- data/lib/switchman/active_record/tasks/database_tasks.rb +6 -1
- data/lib/switchman/active_record/test_fixtures.rb +53 -0
- data/lib/switchman/active_support/cache.rb +25 -4
- data/lib/switchman/arel.rb +45 -7
- data/lib/switchman/call_super.rb +2 -2
- data/lib/switchman/database_server.rb +123 -79
- data/lib/switchman/default_shard.rb +14 -5
- data/lib/switchman/engine.rb +79 -131
- data/lib/switchman/environment.rb +2 -2
- data/lib/switchman/errors.rb +17 -2
- data/lib/switchman/guard_rail/relation.rb +7 -10
- data/lib/switchman/guard_rail.rb +5 -0
- data/lib/switchman/parallel.rb +68 -0
- data/lib/switchman/r_spec_helper.rb +17 -28
- data/lib/switchman/rails.rb +1 -4
- data/{app/models → lib}/switchman/shard.rb +226 -241
- data/lib/switchman/sharded_instrumenter.rb +3 -3
- data/lib/switchman/shared_schema_cache.rb +11 -0
- data/lib/switchman/standard_error.rb +15 -12
- data/lib/switchman/test_helper.rb +2 -2
- data/{app/models → lib}/switchman/unsharded_record.rb +1 -1
- data/lib/switchman/version.rb +1 -1
- data/lib/switchman.rb +44 -12
- data/lib/tasks/switchman.rake +101 -54
- metadata +50 -58
- data/lib/switchman/active_record/association.rb +0 -206
- data/lib/switchman/open4.rb +0 -80
|
@@ -25,6 +25,24 @@ module Switchman
|
|
|
25
25
|
@sharded_column_values[column_name]
|
|
26
26
|
end
|
|
27
27
|
|
|
28
|
+
def define_attribute_methods
|
|
29
|
+
super
|
|
30
|
+
# ensure that we're using the sharded attribute method
|
|
31
|
+
# and not the silly one in AR::AttributeMethods::PrimaryKey
|
|
32
|
+
return unless sharded_column?(@primary_key)
|
|
33
|
+
|
|
34
|
+
class_eval(
|
|
35
|
+
build_sharded_getter("id",
|
|
36
|
+
"_read_attribute(@primary_key)",
|
|
37
|
+
"::#{connection_class_for_self.name}"),
|
|
38
|
+
__FILE__,
|
|
39
|
+
__LINE__
|
|
40
|
+
)
|
|
41
|
+
class_eval(build_sharded_setter("id", @primary_key, "::#{connection_class_for_self.name}"),
|
|
42
|
+
__FILE__,
|
|
43
|
+
__LINE__)
|
|
44
|
+
end
|
|
45
|
+
|
|
28
46
|
protected
|
|
29
47
|
|
|
30
48
|
def reflection_for_integer_attribute(attr_name)
|
|
@@ -36,110 +54,203 @@ module Switchman
|
|
|
36
54
|
raise if connection.open_transactions.positive?
|
|
37
55
|
end
|
|
38
56
|
|
|
57
|
+
def define_cached_method(owner, name, namespace:, as:, &block)
|
|
58
|
+
owner.define_cached_method(name, namespace: namespace, as: as, &block)
|
|
59
|
+
end
|
|
60
|
+
|
|
39
61
|
def define_method_global_attribute(attr_name, owner:)
|
|
40
62
|
if sharded_column?(attr_name)
|
|
41
|
-
owner
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
63
|
+
define_cached_method(owner,
|
|
64
|
+
"global_#{attr_name}",
|
|
65
|
+
as: "sharded_global_#{attr_name}",
|
|
66
|
+
namespace: :switchman) do |batch|
|
|
67
|
+
batch << <<-RUBY
|
|
68
|
+
def sharded_global_#{attr_name}
|
|
69
|
+
raw_value = original_#{attr_name}
|
|
70
|
+
return nil if raw_value.nil?
|
|
71
|
+
return raw_value if raw_value > ::Switchman::Shard::IDS_PER_SHARD
|
|
72
|
+
|
|
73
|
+
::Switchman::Shard.global_id_for(raw_value, shard)
|
|
74
|
+
end
|
|
75
|
+
RUBY
|
|
76
|
+
end
|
|
46
77
|
else
|
|
47
|
-
define_method_unsharded_column(attr_name,
|
|
78
|
+
define_method_unsharded_column(attr_name, "global", owner)
|
|
48
79
|
end
|
|
49
80
|
end
|
|
50
81
|
|
|
51
82
|
def define_method_local_attribute(attr_name, owner:)
|
|
52
83
|
if sharded_column?(attr_name)
|
|
53
|
-
owner
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
84
|
+
define_cached_method(owner,
|
|
85
|
+
"local_#{attr_name}",
|
|
86
|
+
as: "sharded_local_#{attr_name}",
|
|
87
|
+
namespace: :switchman) do |batch|
|
|
88
|
+
batch << <<-RUBY
|
|
89
|
+
def sharded_local_#{attr_name}
|
|
90
|
+
raw_value = original_#{attr_name}
|
|
91
|
+
return nil if raw_value.nil?
|
|
92
|
+
return raw_value % ::Switchman::Shard::IDS_PER_SHARD
|
|
93
|
+
end
|
|
94
|
+
RUBY
|
|
95
|
+
end
|
|
58
96
|
else
|
|
59
|
-
define_method_unsharded_column(attr_name,
|
|
97
|
+
define_method_unsharded_column(attr_name, "local", owner)
|
|
60
98
|
end
|
|
61
99
|
end
|
|
62
100
|
|
|
63
|
-
# see also Base#
|
|
101
|
+
# see also Base#connection_class_for_self_for_reflection
|
|
64
102
|
# the difference being this will output static strings for the common cases, making them
|
|
65
103
|
# more performant
|
|
66
|
-
def
|
|
104
|
+
def connection_class_for_self_code_for_reflection(reflection)
|
|
67
105
|
if reflection
|
|
68
106
|
if reflection.options[:polymorphic]
|
|
69
107
|
# a polymorphic association has to be discovered at runtime. This code ends up being something like
|
|
70
|
-
# context_type.&.constantize&.
|
|
71
|
-
|
|
108
|
+
# context_type.&.constantize&.connection_class_for_self
|
|
109
|
+
<<~RUBY
|
|
110
|
+
begin
|
|
111
|
+
read_attribute(:#{reflection.foreign_type})&.constantize&.connection_class_for_self
|
|
112
|
+
rescue NameError
|
|
113
|
+
::ActiveRecord::Base
|
|
114
|
+
end
|
|
115
|
+
RUBY
|
|
72
116
|
else
|
|
73
117
|
# otherwise we can just return a symbol for the statically known type of the association
|
|
74
|
-
"::#{reflection.klass.
|
|
118
|
+
"::#{reflection.klass.connection_class_for_self.name}"
|
|
75
119
|
end
|
|
76
120
|
else
|
|
77
|
-
"::#{
|
|
121
|
+
"::#{connection_class_for_self.name}"
|
|
78
122
|
end
|
|
79
123
|
end
|
|
80
124
|
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
125
|
+
def define_method_attribute(attr_name, owner:)
|
|
126
|
+
if sharded_column?(attr_name)
|
|
127
|
+
reflection = reflection_for_integer_attribute(attr_name)
|
|
128
|
+
class_name = connection_class_for_self_code_for_reflection(reflection)
|
|
129
|
+
safe_class_name = class_name.unpack1("h*")
|
|
130
|
+
define_cached_method(owner,
|
|
131
|
+
attr_name,
|
|
132
|
+
as: "sharded_#{safe_class_name}_#{attr_name}",
|
|
133
|
+
namespace: :switchman) do |batch|
|
|
134
|
+
batch << build_sharded_getter("sharded_#{safe_class_name}_#{attr_name}",
|
|
135
|
+
"original_#{attr_name}",
|
|
136
|
+
class_name)
|
|
137
|
+
end
|
|
138
|
+
else
|
|
139
|
+
define_cached_method(owner, attr_name, as: "plain_#{attr_name}", namespace: :switchman) do |batch|
|
|
140
|
+
batch << <<-RUBY
|
|
141
|
+
def plain_#{attr_name}
|
|
142
|
+
_read_attribute("#{attr_name}") { |n| missing_attribute(n, caller) }
|
|
143
|
+
end
|
|
144
|
+
RUBY
|
|
145
|
+
end
|
|
86
146
|
end
|
|
147
|
+
end
|
|
87
148
|
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
149
|
+
def build_sharded_getter(attr_name, raw_expr, attr_connection_class)
|
|
150
|
+
<<-RUBY
|
|
151
|
+
def #{attr_name}
|
|
152
|
+
raw_value = #{raw_expr}
|
|
153
|
+
return nil if raw_value.nil?
|
|
154
|
+
|
|
155
|
+
abs_raw_value = raw_value.abs
|
|
156
|
+
current_shard = ::Switchman::Shard.current(#{attr_connection_class})
|
|
157
|
+
same_shard = loaded_from_shard == current_shard
|
|
158
|
+
return raw_value if same_shard && abs_raw_value < ::Switchman::Shard::IDS_PER_SHARD
|
|
159
|
+
|
|
160
|
+
value_shard_id = abs_raw_value / ::Switchman::Shard::IDS_PER_SHARD
|
|
161
|
+
# this is a stupid case when someone stuffed a global id for the current shard in instead
|
|
162
|
+
# of a local id
|
|
163
|
+
return raw_value % ::Switchman::Shard::IDS_PER_SHARD if value_shard_id == current_shard.id
|
|
164
|
+
return raw_value if !same_shard && abs_raw_value > ::Switchman::Shard::IDS_PER_SHARD
|
|
165
|
+
return loaded_from_shard.global_id_for(raw_value) if !same_shard && abs_raw_value < ::Switchman::Shard::IDS_PER_SHARD
|
|
166
|
+
|
|
167
|
+
::Switchman::Shard.relative_id_for(raw_value, loaded_from_shard, current_shard)
|
|
168
|
+
end
|
|
169
|
+
RUBY
|
|
91
170
|
end
|
|
92
171
|
|
|
93
|
-
def
|
|
172
|
+
def define_method_attribute=(attr_name, owner:)
|
|
94
173
|
if sharded_column?(attr_name)
|
|
95
174
|
reflection = reflection_for_integer_attribute(attr_name)
|
|
96
|
-
|
|
97
|
-
|
|
175
|
+
class_name = connection_class_for_self_code_for_reflection(reflection)
|
|
176
|
+
safe_class_name = class_name.unpack1("h*")
|
|
177
|
+
define_cached_method(owner,
|
|
178
|
+
"#{attr_name}=",
|
|
179
|
+
as: "sharded_#{safe_class_name}_#{attr_name}=",
|
|
180
|
+
namespace: :switchman) do |batch|
|
|
181
|
+
batch << build_sharded_setter("sharded_#{safe_class_name}_#{attr_name}", attr_name, class_name)
|
|
182
|
+
end
|
|
183
|
+
else
|
|
184
|
+
define_cached_method(owner, "#{attr_name}=", as: "plain_#{attr_name}=", namespace: :switchman) do |batch|
|
|
185
|
+
batch << <<-RUBY
|
|
186
|
+
def plain_#{attr_name}=(new_value)
|
|
187
|
+
_write_attribute('#{attr_name}', new_value)
|
|
188
|
+
end
|
|
189
|
+
RUBY
|
|
190
|
+
end
|
|
191
|
+
end
|
|
192
|
+
end
|
|
98
193
|
|
|
99
|
-
|
|
194
|
+
def build_sharded_setter(attr_name, attr_field, attr_connection_class)
|
|
195
|
+
<<-RUBY
|
|
196
|
+
def #{attr_name}=(new_value)
|
|
197
|
+
self.original_#{attr_field} = ::Switchman::Shard.relative_id_for(new_value, ::Switchman::Shard.current(#{attr_connection_class}), loaded_from_shard)
|
|
100
198
|
end
|
|
199
|
+
RUBY
|
|
200
|
+
end
|
|
101
201
|
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
202
|
+
def define_method_original_attribute(attr_name, owner:)
|
|
203
|
+
if sharded_column?(attr_name)
|
|
204
|
+
define_cached_method(owner,
|
|
205
|
+
"original_#{attr_name}",
|
|
206
|
+
as: "sharded_original_#{attr_name}",
|
|
207
|
+
namespace: :switchman) do |batch|
|
|
208
|
+
batch << <<-RUBY
|
|
209
|
+
def sharded_original_#{attr_name}
|
|
210
|
+
_read_attribute("#{attr_name}") { |n| missing_attribute(n, caller) }
|
|
211
|
+
end
|
|
212
|
+
RUBY
|
|
213
|
+
end
|
|
214
|
+
else
|
|
215
|
+
define_method_unsharded_column(attr_name, "global", owner)
|
|
216
|
+
end
|
|
217
|
+
end
|
|
109
218
|
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
219
|
+
def define_method_original_attribute=(attr_name, owner:)
|
|
220
|
+
return unless sharded_column?(attr_name)
|
|
221
|
+
|
|
222
|
+
define_cached_method(owner,
|
|
223
|
+
"original_#{attr_name}=",
|
|
224
|
+
as: "sharded_original_#{attr_name}=",
|
|
225
|
+
namespace: :switchman) do |batch|
|
|
226
|
+
batch << <<-RUBY
|
|
227
|
+
def sharded_original_#{attr_name}=(new_value)
|
|
228
|
+
_write_attribute('#{attr_name}', new_value)
|
|
113
229
|
end
|
|
114
230
|
RUBY
|
|
115
|
-
else
|
|
116
|
-
define_method_unsharded_column(attr_name, 'global', owner)
|
|
117
231
|
end
|
|
118
232
|
end
|
|
119
233
|
|
|
120
234
|
def define_method_unsharded_column(attr_name, prefix, owner)
|
|
121
|
-
return if columns_hash["#{prefix}_#{attr_name}"]
|
|
235
|
+
return if columns_hash["#{prefix}_#{attr_name}"] || attr_name == "id"
|
|
122
236
|
|
|
123
|
-
owner
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
237
|
+
define_cached_method(owner,
|
|
238
|
+
"#{prefix}_#{attr_name}",
|
|
239
|
+
as: "unsharded_#{prefix}_#{attr_name}",
|
|
240
|
+
namespace: :switchman) do |batch|
|
|
241
|
+
batch << <<-RUBY
|
|
242
|
+
def unsharded_#{prefix}_#{attr_name}
|
|
243
|
+
raise NoMethodError, "undefined method `#{prefix}_#{attr_name}'; are you missing an association?"
|
|
244
|
+
end
|
|
245
|
+
RUBY
|
|
246
|
+
end
|
|
128
247
|
end
|
|
129
248
|
end
|
|
130
249
|
|
|
131
|
-
def self.
|
|
132
|
-
klass.singleton_class.
|
|
133
|
-
klass.attribute_method_prefix
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
# ensure that we're using the sharded attribute method
|
|
137
|
-
# and not the silly one in AR::AttributeMethods::PrimaryKey
|
|
138
|
-
def id
|
|
139
|
-
return super if is_a?(Shard)
|
|
140
|
-
|
|
141
|
-
self.class.define_attribute_methods
|
|
142
|
-
super
|
|
250
|
+
def self.prepended(klass)
|
|
251
|
+
klass.singleton_class.prepend(ClassMethods)
|
|
252
|
+
klass.attribute_method_prefix "global_", "local_", "original_"
|
|
253
|
+
klass.attribute_method_affix prefix: "original_", suffix: "="
|
|
143
254
|
end
|
|
144
255
|
|
|
145
256
|
# these are called if the specific methods haven't been defined yet
|
|
@@ -147,7 +258,11 @@ module Switchman
|
|
|
147
258
|
return super unless self.class.sharded_column?(attr_name)
|
|
148
259
|
|
|
149
260
|
reflection = self.class.send(:reflection_for_integer_attribute, attr_name)
|
|
150
|
-
::Switchman::Shard.relative_id_for(
|
|
261
|
+
::Switchman::Shard.relative_id_for(
|
|
262
|
+
super,
|
|
263
|
+
shard,
|
|
264
|
+
::Switchman::Shard.current(connection_class_for_self_for_reflection(reflection))
|
|
265
|
+
)
|
|
151
266
|
end
|
|
152
267
|
|
|
153
268
|
def attribute=(attr_name, new_value)
|
|
@@ -157,7 +272,11 @@ module Switchman
|
|
|
157
272
|
end
|
|
158
273
|
|
|
159
274
|
reflection = self.class.send(:reflection_for_integer_attribute, attr_name)
|
|
160
|
-
super(
|
|
275
|
+
super(attr_name, ::Switchman::Shard.relative_id_for(
|
|
276
|
+
new_value,
|
|
277
|
+
::Switchman::Shard.current(connection_class_for_self_for_reflection(reflection)),
|
|
278
|
+
shard
|
|
279
|
+
))
|
|
161
280
|
end
|
|
162
281
|
|
|
163
282
|
def global_attribute(attr_name)
|
|
@@ -170,25 +289,11 @@ module Switchman
|
|
|
170
289
|
|
|
171
290
|
def local_attribute(attr_name)
|
|
172
291
|
if self.class.sharded_column?(attr_name)
|
|
173
|
-
::Switchman::Shard.local_id_for(attribute(attr_name)
|
|
292
|
+
::Switchman::Shard.local_id_for(attribute(attr_name)).first
|
|
174
293
|
else
|
|
175
294
|
attribute(attr_name)
|
|
176
295
|
end
|
|
177
296
|
end
|
|
178
|
-
|
|
179
|
-
private
|
|
180
|
-
|
|
181
|
-
def connection_classes_for_reflection(reflection)
|
|
182
|
-
if reflection
|
|
183
|
-
if reflection.options[:polymorphic]
|
|
184
|
-
read_attribute(reflection.foreign_type)&.constantize&.connection_classes
|
|
185
|
-
else
|
|
186
|
-
reflection.klass.connection_classes
|
|
187
|
-
end
|
|
188
|
-
else
|
|
189
|
-
self.class.connection_classes
|
|
190
|
-
end
|
|
191
|
-
end
|
|
192
297
|
end
|
|
193
298
|
end
|
|
194
299
|
end
|