switchman 2.0.13 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. checksums.yaml +4 -4
  2. data/Rakefile +10 -2
  3. data/app/models/switchman/shard.rb +234 -271
  4. data/app/models/switchman/unsharded_record.rb +7 -0
  5. data/db/migrate/20130328212039_create_switchman_shards.rb +1 -1
  6. data/db/migrate/20130328224244_create_default_shard.rb +5 -5
  7. data/db/migrate/20161206323434_add_back_default_string_limits_switchman.rb +1 -0
  8. data/db/migrate/20180828183945_add_default_shard_index.rb +2 -2
  9. data/db/migrate/20180828192111_add_timestamps_to_shards.rb +7 -5
  10. data/db/migrate/20190114212900_add_unique_name_indexes.rb +5 -3
  11. data/lib/switchman.rb +3 -5
  12. data/lib/switchman/action_controller/caching.rb +2 -2
  13. data/lib/switchman/active_record/abstract_adapter.rb +1 -0
  14. data/lib/switchman/active_record/association.rb +78 -89
  15. data/lib/switchman/active_record/attribute_methods.rb +58 -52
  16. data/lib/switchman/active_record/base.rb +58 -59
  17. data/lib/switchman/active_record/calculations.rb +74 -67
  18. data/lib/switchman/active_record/connection_pool.rb +14 -41
  19. data/lib/switchman/active_record/database_configurations.rb +34 -0
  20. data/lib/switchman/active_record/database_configurations/database_config.rb +13 -0
  21. data/lib/switchman/active_record/finder_methods.rb +11 -16
  22. data/lib/switchman/active_record/log_subscriber.rb +4 -8
  23. data/lib/switchman/active_record/migration.rb +6 -47
  24. data/lib/switchman/active_record/model_schema.rb +1 -1
  25. data/lib/switchman/active_record/persistence.rb +4 -6
  26. data/lib/switchman/active_record/postgresql_adapter.rb +124 -168
  27. data/lib/switchman/active_record/predicate_builder.rb +2 -2
  28. data/lib/switchman/active_record/query_cache.rb +18 -19
  29. data/lib/switchman/active_record/query_methods.rb +172 -197
  30. data/lib/switchman/active_record/reflection.rb +6 -10
  31. data/lib/switchman/active_record/relation.rb +30 -78
  32. data/lib/switchman/active_record/spawn_methods.rb +27 -29
  33. data/lib/switchman/active_record/statement_cache.rb +18 -35
  34. data/lib/switchman/active_record/tasks/database_tasks.rb +16 -0
  35. data/lib/switchman/active_support/cache.rb +3 -5
  36. data/lib/switchman/arel.rb +13 -8
  37. data/lib/switchman/database_server.rb +121 -142
  38. data/lib/switchman/default_shard.rb +52 -16
  39. data/lib/switchman/engine.rb +61 -58
  40. data/lib/switchman/environment.rb +4 -8
  41. data/lib/switchman/errors.rb +1 -0
  42. data/lib/switchman/guard_rail.rb +6 -19
  43. data/lib/switchman/guard_rail/relation.rb +5 -7
  44. data/lib/switchman/r_spec_helper.rb +29 -37
  45. data/lib/switchman/rails.rb +14 -12
  46. data/lib/switchman/schema_cache.rb +1 -9
  47. data/lib/switchman/sharded_instrumenter.rb +1 -1
  48. data/lib/switchman/standard_error.rb +15 -3
  49. data/lib/switchman/test_helper.rb +7 -11
  50. data/lib/switchman/version.rb +1 -1
  51. data/lib/tasks/switchman.rake +54 -69
  52. metadata +87 -45
  53. data/lib/switchman/active_record/batches.rb +0 -11
  54. data/lib/switchman/active_record/connection_handler.rb +0 -172
  55. data/lib/switchman/active_record/where_clause_factory.rb +0 -36
  56. data/lib/switchman/connection_pool_proxy.rb +0 -173
@@ -6,49 +6,40 @@ module Switchman
6
6
  module ClassMethods
7
7
  delegate :shard, to: :all
8
8
 
9
- def find_ids_in_ranges(opts={}, &block)
10
- opts.reverse_merge!(:loose => true)
9
+ def find_ids_in_ranges(opts = {}, &block)
10
+ opts.reverse_merge!(loose: true)
11
11
  all.find_ids_in_ranges(opts, &block)
12
12
  end
13
13
 
14
- def shard_category
15
- connection_specification_name.to_sym
16
- end
14
+ def sharded_model
15
+ self.abstract_class = true
17
16
 
18
- def shard_category=(category)
19
- categories = Shard.const_get(:CATEGORIES)
20
- if categories[shard_category]
21
- categories[shard_category].delete(self)
22
- categories.delete(shard_category) if categories[shard_category].empty?
23
- end
24
- categories[category] ||= []
25
- categories[category] << self
26
- self.connection_specification_name = category.to_s
17
+ return if self == UnshardedRecord
18
+
19
+ Shard.send(:add_sharded_model, self)
27
20
  end
28
21
 
29
22
  def integral_id?
30
- if @integral_id == nil
31
- @integral_id = columns_hash[primary_key]&.type == :integer
32
- end
23
+ @integral_id = columns_hash[primary_key]&.type == :integer if @integral_id.nil?
33
24
  @integral_id
34
25
  end
35
26
 
36
27
  def transaction(**)
37
28
  if self != ::ActiveRecord::Base && current_scope
38
29
  current_scope.activate do
39
- db = Shard.current(shard_category).database_server
40
- if ::GuardRail.environment != db.guard_rail_environment
41
- db.unguard { super }
42
- else
30
+ db = Shard.current(connection_classes).database_server
31
+ if ::GuardRail.environment == db.guard_rail_environment
43
32
  super
33
+ else
34
+ db.unguard { super }
44
35
  end
45
36
  end
46
37
  else
47
- db = Shard.current(shard_category).database_server
48
- if ::GuardRail.environment != db.guard_rail_environment
49
- db.unguard { super }
50
- else
38
+ db = Shard.current(connection_classes).database_server
39
+ if ::GuardRail.environment == db.guard_rail_environment
51
40
  super
41
+ else
42
+ db.unguard { super }
52
43
  end
53
44
  end
54
45
  end
@@ -72,39 +63,46 @@ module Switchman
72
63
  end
73
64
 
74
65
  def clear_query_caches_for_current_thread
75
- ::ActiveRecord::Base.connection_handlers.each_value do |handler|
76
- handler.connection_pool_list.each do |pool|
77
- pool.connection(switch_shard: false).clear_query_cache if pool.active_connection?
78
- end
66
+ ::ActiveRecord::Base.connection_handler.connection_pool_list.each do |pool|
67
+ pool.connection(switch_shard: false).clear_query_cache if pool.active_connection?
79
68
  end
80
69
  end
70
+
71
+ # significant change: _don't_ check if klasses.include?(Base)
72
+ # i.e. other sharded models don't inherit the current shard of Base
73
+ def current_shard
74
+ connected_to_stack.reverse_each do |hash|
75
+ return hash[:shard] if hash[:shard] && hash[:klasses].include?(connection_classes)
76
+ end
77
+
78
+ default_shard
79
+ end
81
80
  end
82
81
 
83
82
  def self.included(klass)
84
- klass.extend(ClassMethods)
83
+ klass.singleton_class.prepend(ClassMethods)
85
84
  klass.set_callback(:initialize, :before) do
86
- unless @shard
87
- if self.class.sharded_primary_key?
88
- @shard = Shard.shard_for(self[self.class.primary_key], Shard.current(self.class.shard_category))
89
- else
90
- @shard = Shard.current(self.class.shard_category)
91
- end
92
- end
85
+ @shard ||= if self.class.sharded_primary_key?
86
+ Shard.shard_for(self[self.class.primary_key], Shard.current(self.class.connection_classes))
87
+ else
88
+ Shard.current(self.class.connection_classes)
89
+ end
93
90
  end
94
91
  end
95
92
 
96
93
  def shard
97
- @shard || Shard.current(self.class.shard_category) || Shard.default
94
+ @shard || Shard.current(self.class.connection_classes) || Shard.default
98
95
  end
99
96
 
100
97
  def shard=(new_shard)
101
- raise ::ActiveRecord::ReadOnlyRecord if !self.new_record? || @shard_set_in_stone
102
- if shard != new_shard
103
- attributes.each do |attr, value|
104
- self[attr] = Shard.relative_id_for(value, shard, new_shard) if self.class.sharded_column?(attr)
105
- end
106
- @shard = new_shard
98
+ raise ::ActiveRecord::ReadOnlyRecord if !new_record? || @shard_set_in_stone
99
+
100
+ return if shard == new_shard
101
+
102
+ attributes.each do |attr, value|
103
+ self[attr] = Shard.relative_id_for(value, shard, new_shard) if self.class.sharded_column?(attr)
107
104
  end
105
+ @shard = new_shard
108
106
  end
109
107
 
110
108
  def save(*, **)
@@ -118,7 +116,7 @@ module Switchman
118
116
  end
119
117
 
120
118
  def destroy
121
- self.class.shard(shard, :implicit).scoping { super }
119
+ shard.activate(self.class.connection_classes) { super }
122
120
  end
123
121
 
124
122
  def clone
@@ -126,23 +124,23 @@ module Switchman
126
124
  # TODO: adjust foreign keys
127
125
  # don't use the setter, cause the foreign keys are already
128
126
  # relative to this shard
129
- result.instance_variable_set(:@shard, self.shard)
127
+ result.instance_variable_set(:@shard, shard)
130
128
  result
131
129
  end
132
130
 
133
131
  def transaction(**kwargs, &block)
134
- shard.activate(self.class.shard_category) do
132
+ shard.activate(self.class.connection_classes) do
135
133
  self.class.transaction(**kwargs, &block)
136
134
  end
137
135
  end
138
136
 
139
137
  def hash
140
- self.class.sharded_primary_key? ? self.class.hash ^ Shard.global_id_for(id).hash : super
138
+ self.class.sharded_primary_key? ? self.class.hash ^ global_id.hash : super
141
139
  end
142
140
 
143
141
  def to_param
144
142
  short_id = Shard.short_id_for(id)
145
- short_id && short_id.to_s
143
+ short_id&.to_s
146
144
  end
147
145
 
148
146
  def initialize_dup(*args)
@@ -153,37 +151,38 @@ module Switchman
153
151
 
154
152
  def quoted_id
155
153
  return super unless self.class.sharded_primary_key?
154
+
156
155
  # do this the Rails 4.2 way, so that if Shard.current != self.shard, the id gets transposed
157
156
  self.class.connection.quote(id)
158
157
  end
159
158
 
160
159
  def update_columns(*)
161
- db = Shard.current(self.class.shard_category).database_server
162
- if ::GuardRail.environment != db.guard_rail_environment
163
- return db.unguard { super }
164
- else
160
+ db = Shard.current(self.class.connection_classes).database_server
161
+ if ::GuardRail.environment == db.guard_rail_environment
165
162
  super
163
+ else
164
+ db.unguard { super }
166
165
  end
167
166
  end
168
167
 
169
168
  protected
170
169
 
171
- # see also AttributeMethods#shard_category_code_for_reflection
172
- def shard_category_for_reflection(reflection)
170
+ # see also AttributeMethods#connection_classes_code_for_reflection
171
+ def connection_classes_for_reflection(reflection)
173
172
  if reflection
174
173
  if reflection.options[:polymorphic]
175
174
  begin
176
- read_attribute(reflection.foreign_type)&.constantize&.shard_category || :primary
175
+ read_attribute(reflection.foreign_type)&.constantize&.connection_classes
177
176
  rescue NameError
178
177
  # in case someone is abusing foreign_type to not point to an actual class
179
- :primary
178
+ ::ActiveRecord::Base
180
179
  end
181
180
  else
182
181
  # otherwise we can just return a symbol for the statically known type of the association
183
- reflection.klass.shard_category
182
+ reflection.klass.connection_classes
184
183
  end
185
184
  else
186
- shard_category
185
+ connection_classes
187
186
  end
188
187
  end
189
188
  end
@@ -3,45 +3,44 @@
3
3
  module Switchman
4
4
  module ActiveRecord
5
5
  module Calculations
6
-
7
6
  def pluck(*column_names)
8
- target_shard = Shard.current(klass.shard_category)
7
+ target_shard = Shard.current(klass.connection_classes)
9
8
  shard_count = 0
10
- result = self.activate do |relation, shard|
9
+ result = activate do |relation, shard|
11
10
  shard_count += 1
12
11
  results = relation.call_super(:pluck, Calculations, *column_names)
13
12
  if column_names.length > 1
14
13
  column_names.each_with_index do |column_name, idx|
15
- if klass.sharded_column?(column_name)
16
- results.each{|result| result[idx] = Shard.relative_id_for(result[idx], shard, target_shard)}
14
+ next unless klass.sharded_column?(column_name)
15
+
16
+ results.each do |r|
17
+ r[idx] = Shard.relative_id_for(r[idx], shard, target_shard)
17
18
  end
18
19
  end
19
- else
20
- if klass.sharded_column?(column_names.first.to_s)
21
- results = results.map{|result| Shard.relative_id_for(result, shard, target_shard)}
22
- end
20
+ elsif klass.sharded_column?(column_names.first.to_s)
21
+ results = results.map { |r| Shard.relative_id_for(r, shard, target_shard) }
23
22
  end
24
23
  results
25
24
  end
26
- if distinct_value && shard_count > 1
27
- result.uniq!
28
- end
25
+ result.uniq! if distinct_value && shard_count > 1
29
26
  result
30
27
  end
31
28
 
32
29
  def execute_simple_calculation(operation, column_name, distinct)
33
30
  operation = operation.to_s.downcase
34
- if operation == "average"
31
+ if operation == 'average'
35
32
  result = calculate_simple_average(column_name, distinct)
36
33
  else
37
- result = self.activate{ |relation| relation.call_super(:execute_simple_calculation, Calculations, operation, column_name, distinct) }
34
+ result = activate do |relation|
35
+ relation.call_super(:execute_simple_calculation, Calculations, operation, column_name, distinct)
36
+ end
38
37
  if result.is_a?(Array)
39
38
  case operation
40
- when "count", "sum"
39
+ when 'count', 'sum'
41
40
  result = result.sum
42
- when "minimum"
41
+ when 'minimum'
43
42
  result = result.min
44
- when "maximum"
43
+ when 'maximum'
45
44
  result = result.max
46
45
  end
47
46
  end
@@ -51,20 +50,22 @@ module Switchman
51
50
 
52
51
  def calculate_simple_average(column_name, distinct)
53
52
  # See activerecord#execute_simple_calculation
54
- relation = except(:order)
53
+ relation = reorder(nil)
55
54
  column = aggregate_column(column_name)
56
- relation.select_values = [operation_over_aggregate_column(column, "average", distinct).as("average"),
57
- operation_over_aggregate_column(column, "count", distinct).as("count")]
55
+ relation.select_values = [operation_over_aggregate_column(column, 'average', distinct).as('average'),
56
+ operation_over_aggregate_column(column, 'count', distinct).as('count')]
58
57
 
59
- initial_results = relation.activate{ |rel| klass.connection.select_all(rel) }
58
+ initial_results = relation.activate { |rel| klass.connection.select_all(rel) }
60
59
  if initial_results.is_a?(Array)
61
60
  initial_results.each do |r|
62
- r["average"] = type_cast_calculated_value(r["average"], type_for(column_name), "average")
63
- r["count"] = type_cast_calculated_value(r["count"], type_for(column_name), "count")
61
+ r['average'] = type_cast_calculated_value_switchman(r['average'], column_name, 'average')
62
+ r['count'] = type_cast_calculated_value_switchman(r['count'], column_name, 'count')
64
63
  end
65
- result = initial_results.map{|r| r["average"] * r["count"]}.sum / initial_results.map{|r| r["count"]}.sum
64
+ result = initial_results.map { |r| r['average'] * r['count'] }.sum / initial_results.map do |r|
65
+ r['count']
66
+ end.sum
66
67
  else
67
- result = type_cast_calculated_value(initial_results.first["average"], type_for(column_name), "average")
68
+ result = type_cast_calculated_value_switchman(initial_results.first['average'], column_name, 'average')
68
69
  end
69
70
  result
70
71
  end
@@ -74,27 +75,28 @@ module Switchman
74
75
  opts = grouped_calculation_options(operation.to_s.downcase, column_name, distinct)
75
76
 
76
77
  relation = build_grouped_calculation_relation(opts)
77
- target_shard = Shard.current(:primary)
78
+ target_shard = Shard.current
78
79
 
79
80
  rows = relation.activate do |rel, shard|
80
81
  calculated_data = klass.connection.select_all(rel)
81
82
 
82
83
  if opts[:association]
83
84
  key_ids = calculated_data.collect { |row| row[opts[:group_aliases].first] }
84
- key_records = opts[:association].klass.base_class.where(:id => key_ids)
85
- key_records = Hash[key_records.map { |r| [Shard.relative_id_for(r, shard, target_shard), r] }]
85
+ key_records = opts[:association].klass.base_class.where(id: key_ids)
86
+ key_records = key_records.map { |r| [Shard.relative_id_for(r, shard, target_shard), r] }.to_h
86
87
  end
87
88
 
88
89
  calculated_data.map do |row|
89
- row[opts[:aggregate_alias]] = type_cast_calculated_value(
90
- row[opts[:aggregate_alias]], type_for(opts[:column_name]), opts[:operation])
90
+ row[opts[:aggregate_alias]] = type_cast_calculated_value_switchman(
91
+ row[opts[:aggregate_alias]], column_name, opts[:operation]
92
+ )
91
93
  row['count'] = row['count'].to_i if opts[:operation] == 'average'
92
94
 
93
- opts[:group_columns].each do |aliaz, type, column_name|
95
+ opts[:group_columns].each do |aliaz, _type, group_column_name|
94
96
  if opts[:associated] && (aliaz == opts[:group_aliases].first)
95
97
  row[aliaz] = key_records[Shard.relative_id_for(row[aliaz], shard, target_shard)]
96
- elsif column_name && @klass.sharded_column?(column_name)
97
- row[aliaz] = Shard.relative_id_for(type_cast_calculated_value(row[aliaz], type), shard, target_shard)
98
+ elsif group_column_name && @klass.sharded_column?(group_column_name)
99
+ row[aliaz] = Shard.relative_id_for(row[aliaz], shard, target_shard)
98
100
  end
99
101
  end
100
102
  row
@@ -106,12 +108,21 @@ module Switchman
106
108
 
107
109
  private
108
110
 
111
+ def type_cast_calculated_value_switchman(value, column_name, operation)
112
+ type_cast_calculated_value(value, operation) do |val|
113
+ column = aggregate_column(column_name)
114
+ type ||= column.try(:type_caster) ||
115
+ lookup_cast_type_from_join_dependencies(column_name.to_s) || Type.default_value
116
+ type.deserialize(val)
117
+ end
118
+ end
119
+
109
120
  def column_name_for(field)
110
121
  field.respond_to?(:name) ? field.name.to_s : field.to_s.split('.').last
111
122
  end
112
123
 
113
124
  def grouped_calculation_options(operation, column_name, distinct)
114
- opts = {:operation => operation, :column_name => column_name, :distinct => distinct}
125
+ opts = { operation: operation, column_name: column_name, distinct: distinct }
115
126
 
116
127
  opts[:aggregate_alias] = aggregate_alias_for(operation, column_name)
117
128
  group_attrs = group_values
@@ -125,12 +136,12 @@ module Switchman
125
136
 
126
137
  # to_s is because Rails 5 returns a string but Rails 6 returns a symbol.
127
138
  group_aliases = group_fields.map { |field| column_alias_for(field.downcase.to_s).to_s }
128
- group_columns = group_aliases.zip(group_fields).map { |aliaz, field|
139
+ group_columns = group_aliases.zip(group_fields).map do |aliaz, field|
129
140
  [aliaz, type_for(field), column_name_for(field)]
130
- }
131
- opts.merge!(:association => association, :associated => associated,
132
- :group_aliases => group_aliases, :group_columns => group_columns,
133
- :group_fields => group_fields)
141
+ end
142
+ opts.merge!(association: association, associated: associated,
143
+ group_aliases: group_aliases, group_columns: group_columns,
144
+ group_fields: group_fields)
134
145
 
135
146
  opts
136
147
  end
@@ -149,28 +160,30 @@ module Switchman
149
160
  group = opts[:group_fields]
150
161
 
151
162
  select_values = [
152
- operation_over_aggregate_column(
153
- aggregate_column(opts[:column_name]),
154
- opts[:operation],
155
- opts[:distinct]).as(opts[:aggregate_alias])
163
+ operation_over_aggregate_column(
164
+ aggregate_column(opts[:column_name]),
165
+ opts[:operation],
166
+ opts[:distinct]
167
+ ).as(opts[:aggregate_alias])
156
168
  ]
157
- if opts[:operation ]== 'average'
169
+ if opts[:operation] == 'average'
158
170
  # include count in average so we can recalculate the average
159
171
  # across all shards if needed
160
172
  select_values << operation_over_aggregate_column(
161
- aggregate_column(opts[:column_name]),
162
- 'count', opts[:distinct]).as('count')
173
+ aggregate_column(opts[:column_name]),
174
+ 'count', opts[:distinct]
175
+ ).as('count')
163
176
  end
164
177
 
165
178
  haves = having_clause.send(:predicates)
166
179
  select_values += select_values unless haves.empty?
167
- select_values.concat opts[:group_fields].zip(opts[:group_aliases]).map { |field,aliaz|
180
+ select_values.concat(opts[:group_fields].zip(opts[:group_aliases]).map do |field, aliaz|
168
181
  if field.respond_to?(:as)
169
182
  field.as(aliaz)
170
183
  else
171
184
  "#{field} AS #{aliaz}"
172
185
  end
173
- }
186
+ end)
174
187
 
175
188
  relation = except(:group)
176
189
  relation.group_values = group
@@ -181,12 +194,12 @@ module Switchman
181
194
  def compact_grouped_calculation_rows(rows, opts)
182
195
  result = ::ActiveSupport::OrderedHash.new
183
196
  rows.each do |row|
184
- key = opts[:group_columns].map { |aliaz, column| row[aliaz] }
197
+ key = opts[:group_columns].map { |aliaz, _column| row[aliaz] }
185
198
  key = key.first if key.size == 1
186
199
  value = row[opts[:aggregate_alias]]
187
200
 
188
201
  if opts[:operation] == 'average'
189
- if result.has_key?(key)
202
+ if result.key?(key)
190
203
  old_value, old_count = result[key]
191
204
  new_count = old_count + row['count']
192
205
  new_value = ((old_value * old_count) + (value * row['count'])) / new_count
@@ -194,30 +207,24 @@ module Switchman
194
207
  else
195
208
  result[key] = [value, row['count']]
196
209
  end
197
- else
198
- if result.has_key?(key)
199
- case opts[:operation]
200
- when "count", "sum"
201
- result[key] += value
202
- when "minimum"
203
- result[key] = value if value < result[key]
204
- when "maximum"
205
- result[key] = value if value > result[key]
206
- end
207
- else
208
- result[key] = value
210
+ elsif result.key?(key)
211
+ case opts[:operation]
212
+ when 'count', 'sum'
213
+ result[key] += value
214
+ when 'minimum'
215
+ result[key] = value if value < result[key]
216
+ when 'maximum'
217
+ result[key] = value if value > result[key]
209
218
  end
219
+ else
220
+ result[key] = value
210
221
  end
211
222
  end
212
223
 
213
- if opts[:operation] == 'average'
214
- result = Hash[result.map{|k, v| [k, v.first]}]
215
- end
224
+ result.transform_values!(&:first) if opts[:operation] == 'average'
216
225
 
217
226
  result
218
227
  end
219
-
220
-
221
228
  end
222
229
  end
223
230
  end