switchman 3.0.5 → 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. checksums.yaml +4 -4
  2. data/Rakefile +16 -15
  3. data/db/migrate/20180828183945_add_default_shard_index.rb +2 -2
  4. data/db/migrate/20180828192111_add_timestamps_to_shards.rb +1 -1
  5. data/db/migrate/20190114212900_add_unique_name_indexes.rb +10 -4
  6. data/lib/switchman/action_controller/caching.rb +2 -2
  7. data/lib/switchman/active_record/abstract_adapter.rb +6 -15
  8. data/lib/switchman/active_record/associations.rb +331 -0
  9. data/lib/switchman/active_record/attribute_methods.rb +182 -77
  10. data/lib/switchman/active_record/base.rb +249 -46
  11. data/lib/switchman/active_record/calculations.rb +98 -44
  12. data/lib/switchman/active_record/connection_handler.rb +18 -0
  13. data/lib/switchman/active_record/connection_pool.rb +27 -28
  14. data/lib/switchman/active_record/database_configurations.rb +44 -6
  15. data/lib/switchman/active_record/finder_methods.rb +46 -16
  16. data/lib/switchman/active_record/log_subscriber.rb +11 -5
  17. data/lib/switchman/active_record/migration.rb +52 -5
  18. data/lib/switchman/active_record/model_schema.rb +1 -1
  19. data/lib/switchman/active_record/pending_migration_connection.rb +17 -0
  20. data/lib/switchman/active_record/persistence.rb +37 -2
  21. data/lib/switchman/active_record/postgresql_adapter.rb +12 -11
  22. data/lib/switchman/active_record/predicate_builder.rb +2 -2
  23. data/lib/switchman/active_record/query_cache.rb +49 -20
  24. data/lib/switchman/active_record/query_methods.rb +202 -136
  25. data/lib/switchman/active_record/reflection.rb +1 -1
  26. data/lib/switchman/active_record/relation.rb +40 -28
  27. data/lib/switchman/active_record/spawn_methods.rb +2 -2
  28. data/lib/switchman/active_record/statement_cache.rb +11 -7
  29. data/lib/switchman/active_record/table_definition.rb +1 -1
  30. data/lib/switchman/active_record/tasks/database_tasks.rb +6 -1
  31. data/lib/switchman/active_record/test_fixtures.rb +53 -0
  32. data/lib/switchman/active_support/cache.rb +25 -4
  33. data/lib/switchman/arel.rb +45 -7
  34. data/lib/switchman/call_super.rb +2 -2
  35. data/lib/switchman/database_server.rb +123 -79
  36. data/lib/switchman/default_shard.rb +14 -5
  37. data/lib/switchman/engine.rb +79 -131
  38. data/lib/switchman/environment.rb +2 -2
  39. data/lib/switchman/errors.rb +17 -2
  40. data/lib/switchman/guard_rail/relation.rb +7 -10
  41. data/lib/switchman/guard_rail.rb +5 -0
  42. data/lib/switchman/parallel.rb +68 -0
  43. data/lib/switchman/r_spec_helper.rb +17 -28
  44. data/lib/switchman/rails.rb +1 -4
  45. data/{app/models → lib}/switchman/shard.rb +226 -241
  46. data/lib/switchman/sharded_instrumenter.rb +3 -3
  47. data/lib/switchman/shared_schema_cache.rb +11 -0
  48. data/lib/switchman/standard_error.rb +15 -12
  49. data/lib/switchman/test_helper.rb +2 -2
  50. data/{app/models → lib}/switchman/unsharded_record.rb +1 -1
  51. data/lib/switchman/version.rb +1 -1
  52. data/lib/switchman.rb +44 -12
  53. data/lib/tasks/switchman.rake +101 -54
  54. metadata +50 -58
  55. data/lib/switchman/active_record/association.rb +0 -206
  56. data/lib/switchman/open4.rb +0 -80
@@ -25,6 +25,24 @@ module Switchman
25
25
  @sharded_column_values[column_name]
26
26
  end
27
27
 
28
+ def define_attribute_methods
29
+ super
30
+ # ensure that we're using the sharded attribute method
31
+ # and not the silly one in AR::AttributeMethods::PrimaryKey
32
+ return unless sharded_column?(@primary_key)
33
+
34
+ class_eval(
35
+ build_sharded_getter("id",
36
+ "_read_attribute(@primary_key)",
37
+ "::#{connection_class_for_self.name}"),
38
+ __FILE__,
39
+ __LINE__
40
+ )
41
+ class_eval(build_sharded_setter("id", @primary_key, "::#{connection_class_for_self.name}"),
42
+ __FILE__,
43
+ __LINE__)
44
+ end
45
+
28
46
  protected
29
47
 
30
48
  def reflection_for_integer_attribute(attr_name)
@@ -36,110 +54,203 @@ module Switchman
36
54
  raise if connection.open_transactions.positive?
37
55
  end
38
56
 
57
+ def define_cached_method(owner, name, namespace:, as:, &block)
58
+ owner.define_cached_method(name, namespace: namespace, as: as, &block)
59
+ end
60
+
39
61
  def define_method_global_attribute(attr_name, owner:)
40
62
  if sharded_column?(attr_name)
41
- owner << <<-RUBY
42
- def global_#{attr_name}
43
- ::Switchman::Shard.global_id_for(original_#{attr_name}, shard)
44
- end
45
- RUBY
63
+ define_cached_method(owner,
64
+ "global_#{attr_name}",
65
+ as: "sharded_global_#{attr_name}",
66
+ namespace: :switchman) do |batch|
67
+ batch << <<-RUBY
68
+ def sharded_global_#{attr_name}
69
+ raw_value = original_#{attr_name}
70
+ return nil if raw_value.nil?
71
+ return raw_value if raw_value > ::Switchman::Shard::IDS_PER_SHARD
72
+
73
+ ::Switchman::Shard.global_id_for(raw_value, shard)
74
+ end
75
+ RUBY
76
+ end
46
77
  else
47
- define_method_unsharded_column(attr_name, 'global', owner)
78
+ define_method_unsharded_column(attr_name, "global", owner)
48
79
  end
49
80
  end
50
81
 
51
82
  def define_method_local_attribute(attr_name, owner:)
52
83
  if sharded_column?(attr_name)
53
- owner << <<-RUBY
54
- def local_#{attr_name}
55
- ::Switchman::Shard.local_id_for(original_#{attr_name}).first
56
- end
57
- RUBY
84
+ define_cached_method(owner,
85
+ "local_#{attr_name}",
86
+ as: "sharded_local_#{attr_name}",
87
+ namespace: :switchman) do |batch|
88
+ batch << <<-RUBY
89
+ def sharded_local_#{attr_name}
90
+ raw_value = original_#{attr_name}
91
+ return nil if raw_value.nil?
92
+ return raw_value % ::Switchman::Shard::IDS_PER_SHARD
93
+ end
94
+ RUBY
95
+ end
58
96
  else
59
- define_method_unsharded_column(attr_name, 'local', owner)
97
+ define_method_unsharded_column(attr_name, "local", owner)
60
98
  end
61
99
  end
62
100
 
63
- # see also Base#connection_classes_for_reflection
101
+ # see also Base#connection_class_for_self_for_reflection
64
102
  # the difference being this will output static strings for the common cases, making them
65
103
  # more performant
66
- def connection_classes_code_for_reflection(reflection)
104
+ def connection_class_for_self_code_for_reflection(reflection)
67
105
  if reflection
68
106
  if reflection.options[:polymorphic]
69
107
  # a polymorphic association has to be discovered at runtime. This code ends up being something like
70
- # context_type.&.constantize&.connection_classes
71
- "read_attribute(:#{reflection.foreign_type})&.constantize&.connection_classes"
108
+ # context_type.&.constantize&.connection_class_for_self
109
+ <<~RUBY
110
+ begin
111
+ read_attribute(:#{reflection.foreign_type})&.constantize&.connection_class_for_self
112
+ rescue NameError
113
+ ::ActiveRecord::Base
114
+ end
115
+ RUBY
72
116
  else
73
117
  # otherwise we can just return a symbol for the statically known type of the association
74
- "::#{reflection.klass.connection_classes.name}"
118
+ "::#{reflection.klass.connection_class_for_self.name}"
75
119
  end
76
120
  else
77
- "::#{connection_classes.name}"
121
+ "::#{connection_class_for_self.name}"
78
122
  end
79
123
  end
80
124
 
81
- # just a dummy class with the proper interface that calls module_eval immediately
82
- class CodeGenerator
83
- def initialize(mod, line)
84
- @module = mod
85
- @line = line
125
+ def define_method_attribute(attr_name, owner:)
126
+ if sharded_column?(attr_name)
127
+ reflection = reflection_for_integer_attribute(attr_name)
128
+ class_name = connection_class_for_self_code_for_reflection(reflection)
129
+ safe_class_name = class_name.unpack1("h*")
130
+ define_cached_method(owner,
131
+ attr_name,
132
+ as: "sharded_#{safe_class_name}_#{attr_name}",
133
+ namespace: :switchman) do |batch|
134
+ batch << build_sharded_getter("sharded_#{safe_class_name}_#{attr_name}",
135
+ "original_#{attr_name}",
136
+ class_name)
137
+ end
138
+ else
139
+ define_cached_method(owner, attr_name, as: "plain_#{attr_name}", namespace: :switchman) do |batch|
140
+ batch << <<-RUBY
141
+ def plain_#{attr_name}
142
+ _read_attribute("#{attr_name}") { |n| missing_attribute(n, caller) }
143
+ end
144
+ RUBY
145
+ end
86
146
  end
147
+ end
87
148
 
88
- def <<(string)
89
- @module.module_eval(string, __FILE__, @line)
90
- end
149
+ def build_sharded_getter(attr_name, raw_expr, attr_connection_class)
150
+ <<-RUBY
151
+ def #{attr_name}
152
+ raw_value = #{raw_expr}
153
+ return nil if raw_value.nil?
154
+
155
+ abs_raw_value = raw_value.abs
156
+ current_shard = ::Switchman::Shard.current(#{attr_connection_class})
157
+ same_shard = loaded_from_shard == current_shard
158
+ return raw_value if same_shard && abs_raw_value < ::Switchman::Shard::IDS_PER_SHARD
159
+
160
+ value_shard_id = abs_raw_value / ::Switchman::Shard::IDS_PER_SHARD
161
+ # this is a stupid case when someone stuffed a global id for the current shard in instead
162
+ # of a local id
163
+ return raw_value % ::Switchman::Shard::IDS_PER_SHARD if value_shard_id == current_shard.id
164
+ return raw_value if !same_shard && abs_raw_value > ::Switchman::Shard::IDS_PER_SHARD
165
+ return loaded_from_shard.global_id_for(raw_value) if !same_shard && abs_raw_value < ::Switchman::Shard::IDS_PER_SHARD
166
+
167
+ ::Switchman::Shard.relative_id_for(raw_value, loaded_from_shard, current_shard)
168
+ end
169
+ RUBY
91
170
  end
92
171
 
93
- def define_method_original_attribute(attr_name, owner:)
172
+ def define_method_attribute=(attr_name, owner:)
94
173
  if sharded_column?(attr_name)
95
174
  reflection = reflection_for_integer_attribute(attr_name)
96
- if attr_name == 'id'
97
- return if method_defined?(:original_id)
175
+ class_name = connection_class_for_self_code_for_reflection(reflection)
176
+ safe_class_name = class_name.unpack1("h*")
177
+ define_cached_method(owner,
178
+ "#{attr_name}=",
179
+ as: "sharded_#{safe_class_name}_#{attr_name}=",
180
+ namespace: :switchman) do |batch|
181
+ batch << build_sharded_setter("sharded_#{safe_class_name}_#{attr_name}", attr_name, class_name)
182
+ end
183
+ else
184
+ define_cached_method(owner, "#{attr_name}=", as: "plain_#{attr_name}=", namespace: :switchman) do |batch|
185
+ batch << <<-RUBY
186
+ def plain_#{attr_name}=(new_value)
187
+ _write_attribute('#{attr_name}', new_value)
188
+ end
189
+ RUBY
190
+ end
191
+ end
192
+ end
98
193
 
99
- owner = CodeGenerator.new(self, __LINE__ + 4)
194
+ def build_sharded_setter(attr_name, attr_field, attr_connection_class)
195
+ <<-RUBY
196
+ def #{attr_name}=(new_value)
197
+ self.original_#{attr_field} = ::Switchman::Shard.relative_id_for(new_value, ::Switchman::Shard.current(#{attr_connection_class}), loaded_from_shard)
100
198
  end
199
+ RUBY
200
+ end
101
201
 
102
- owner << <<-RUBY
103
- # rename the original method to original_*
104
- alias_method 'original_#{attr_name}', '#{attr_name}'
105
- # and replace with one that transposes the id
106
- def #{attr_name}
107
- ::Switchman::Shard.relative_id_for(original_#{attr_name}, shard, ::Switchman::Shard.current(#{connection_classes_code_for_reflection(reflection)}))
108
- end
202
+ def define_method_original_attribute(attr_name, owner:)
203
+ if sharded_column?(attr_name)
204
+ define_cached_method(owner,
205
+ "original_#{attr_name}",
206
+ as: "sharded_original_#{attr_name}",
207
+ namespace: :switchman) do |batch|
208
+ batch << <<-RUBY
209
+ def sharded_original_#{attr_name}
210
+ _read_attribute("#{attr_name}") { |n| missing_attribute(n, caller) }
211
+ end
212
+ RUBY
213
+ end
214
+ else
215
+ define_method_unsharded_column(attr_name, "global", owner)
216
+ end
217
+ end
109
218
 
110
- alias_method 'original_#{attr_name}=', '#{attr_name}='
111
- def #{attr_name}=(new_value)
112
- self.original_#{attr_name} = ::Switchman::Shard.relative_id_for(new_value, ::Switchman::Shard.current(#{connection_classes_code_for_reflection(reflection)}), shard)
219
+ def define_method_original_attribute=(attr_name, owner:)
220
+ return unless sharded_column?(attr_name)
221
+
222
+ define_cached_method(owner,
223
+ "original_#{attr_name}=",
224
+ as: "sharded_original_#{attr_name}=",
225
+ namespace: :switchman) do |batch|
226
+ batch << <<-RUBY
227
+ def sharded_original_#{attr_name}=(new_value)
228
+ _write_attribute('#{attr_name}', new_value)
113
229
  end
114
230
  RUBY
115
- else
116
- define_method_unsharded_column(attr_name, 'global', owner)
117
231
  end
118
232
  end
119
233
 
120
234
  def define_method_unsharded_column(attr_name, prefix, owner)
121
- return if columns_hash["#{prefix}_#{attr_name}"]
235
+ return if columns_hash["#{prefix}_#{attr_name}"] || attr_name == "id"
122
236
 
123
- owner << <<-RUBY
124
- def #{prefix}_#{attr_name}
125
- raise NoMethodError, "undefined method `#{prefix}_#{attr_name}'; are you missing an association?"
126
- end
127
- RUBY
237
+ define_cached_method(owner,
238
+ "#{prefix}_#{attr_name}",
239
+ as: "unsharded_#{prefix}_#{attr_name}",
240
+ namespace: :switchman) do |batch|
241
+ batch << <<-RUBY
242
+ def unsharded_#{prefix}_#{attr_name}
243
+ raise NoMethodError, "undefined method `#{prefix}_#{attr_name}'; are you missing an association?"
244
+ end
245
+ RUBY
246
+ end
128
247
  end
129
248
  end
130
249
 
131
- def self.included(klass)
132
- klass.singleton_class.include(ClassMethods)
133
- klass.attribute_method_prefix 'global_', 'local_', 'original_'
134
- end
135
-
136
- # ensure that we're using the sharded attribute method
137
- # and not the silly one in AR::AttributeMethods::PrimaryKey
138
- def id
139
- return super if is_a?(Shard)
140
-
141
- self.class.define_attribute_methods
142
- super
250
+ def self.prepended(klass)
251
+ klass.singleton_class.prepend(ClassMethods)
252
+ klass.attribute_method_prefix "global_", "local_", "original_"
253
+ klass.attribute_method_affix prefix: "original_", suffix: "="
143
254
  end
144
255
 
145
256
  # these are called if the specific methods haven't been defined yet
@@ -147,7 +258,11 @@ module Switchman
147
258
  return super unless self.class.sharded_column?(attr_name)
148
259
 
149
260
  reflection = self.class.send(:reflection_for_integer_attribute, attr_name)
150
- ::Switchman::Shard.relative_id_for(super, shard, ::Switchman::Shard.current(connection_classes_for_reflection(reflection)))
261
+ ::Switchman::Shard.relative_id_for(
262
+ super,
263
+ shard,
264
+ ::Switchman::Shard.current(connection_class_for_self_for_reflection(reflection))
265
+ )
151
266
  end
152
267
 
153
268
  def attribute=(attr_name, new_value)
@@ -157,7 +272,11 @@ module Switchman
157
272
  end
158
273
 
159
274
  reflection = self.class.send(:reflection_for_integer_attribute, attr_name)
160
- super(::Switchman::Shard.relative_id_for(new_value, ::Switchman::Shard.current(connection_classes_for_reflection(reflection)), shard))
275
+ super(attr_name, ::Switchman::Shard.relative_id_for(
276
+ new_value,
277
+ ::Switchman::Shard.current(connection_class_for_self_for_reflection(reflection)),
278
+ shard
279
+ ))
161
280
  end
162
281
 
163
282
  def global_attribute(attr_name)
@@ -170,25 +289,11 @@ module Switchman
170
289
 
171
290
  def local_attribute(attr_name)
172
291
  if self.class.sharded_column?(attr_name)
173
- ::Switchman::Shard.local_id_for(attribute(attr_name), shard).first
292
+ ::Switchman::Shard.local_id_for(attribute(attr_name)).first
174
293
  else
175
294
  attribute(attr_name)
176
295
  end
177
296
  end
178
-
179
- private
180
-
181
- def connection_classes_for_reflection(reflection)
182
- if reflection
183
- if reflection.options[:polymorphic]
184
- read_attribute(reflection.foreign_type)&.constantize&.connection_classes
185
- else
186
- reflection.klass.connection_classes
187
- end
188
- else
189
- self.class.connection_classes
190
- end
191
- end
192
297
  end
193
298
  end
194
299
  end