switchman 2.0.11 → 3.0.4

Sign up to get free protection for your applications and to get access to all the features.
Files changed (56) hide show
  1. checksums.yaml +4 -4
  2. data/Rakefile +10 -2
  3. data/app/models/switchman/shard.rb +234 -270
  4. data/app/models/switchman/unsharded_record.rb +7 -0
  5. data/db/migrate/20130328212039_create_switchman_shards.rb +1 -1
  6. data/db/migrate/20130328224244_create_default_shard.rb +5 -5
  7. data/db/migrate/20161206323434_add_back_default_string_limits_switchman.rb +1 -0
  8. data/db/migrate/20180828183945_add_default_shard_index.rb +1 -1
  9. data/db/migrate/20180828192111_add_timestamps_to_shards.rb +7 -5
  10. data/db/migrate/20190114212900_add_unique_name_indexes.rb +5 -3
  11. data/lib/switchman.rb +3 -3
  12. data/lib/switchman/action_controller/caching.rb +2 -2
  13. data/lib/switchman/active_record/abstract_adapter.rb +1 -0
  14. data/lib/switchman/active_record/association.rb +78 -89
  15. data/lib/switchman/active_record/attribute_methods.rb +106 -52
  16. data/lib/switchman/active_record/base.rb +58 -59
  17. data/lib/switchman/active_record/calculations.rb +73 -66
  18. data/lib/switchman/active_record/connection_pool.rb +14 -41
  19. data/lib/switchman/active_record/database_configurations.rb +34 -0
  20. data/lib/switchman/active_record/database_configurations/database_config.rb +13 -0
  21. data/lib/switchman/active_record/finder_methods.rb +11 -16
  22. data/lib/switchman/active_record/log_subscriber.rb +4 -8
  23. data/lib/switchman/active_record/migration.rb +14 -43
  24. data/lib/switchman/active_record/model_schema.rb +1 -1
  25. data/lib/switchman/active_record/persistence.rb +4 -6
  26. data/lib/switchman/active_record/postgresql_adapter.rb +32 -160
  27. data/lib/switchman/active_record/predicate_builder.rb +1 -1
  28. data/lib/switchman/active_record/query_cache.rb +18 -19
  29. data/lib/switchman/active_record/query_methods.rb +179 -182
  30. data/lib/switchman/active_record/reflection.rb +6 -10
  31. data/lib/switchman/active_record/relation.rb +34 -29
  32. data/lib/switchman/active_record/spawn_methods.rb +27 -29
  33. data/lib/switchman/active_record/statement_cache.rb +18 -35
  34. data/lib/switchman/active_record/tasks/database_tasks.rb +16 -0
  35. data/lib/switchman/active_support/cache.rb +3 -5
  36. data/lib/switchman/arel.rb +13 -8
  37. data/lib/switchman/database_server.rb +122 -144
  38. data/lib/switchman/default_shard.rb +52 -16
  39. data/lib/switchman/engine.rb +61 -57
  40. data/lib/switchman/environment.rb +4 -8
  41. data/lib/switchman/errors.rb +1 -0
  42. data/lib/switchman/guard_rail.rb +6 -19
  43. data/lib/switchman/guard_rail/relation.rb +5 -7
  44. data/lib/switchman/r_spec_helper.rb +29 -37
  45. data/lib/switchman/rails.rb +14 -12
  46. data/lib/switchman/sharded_instrumenter.rb +1 -1
  47. data/lib/switchman/standard_error.rb +15 -3
  48. data/lib/switchman/test_helper.rb +6 -10
  49. data/lib/switchman/version.rb +1 -1
  50. data/lib/tasks/switchman.rake +54 -69
  51. metadata +85 -44
  52. data/lib/switchman/active_record/batches.rb +0 -11
  53. data/lib/switchman/active_record/connection_handler.rb +0 -172
  54. data/lib/switchman/active_record/where_clause_factory.rb +0 -36
  55. data/lib/switchman/connection_pool_proxy.rb +0 -173
  56. data/lib/switchman/schema_cache.rb +0 -28
@@ -6,49 +6,40 @@ module Switchman
6
6
  module ClassMethods
7
7
  delegate :shard, to: :all
8
8
 
9
- def find_ids_in_ranges(opts={}, &block)
10
- opts.reverse_merge!(:loose => true)
9
+ def find_ids_in_ranges(opts = {}, &block)
10
+ opts.reverse_merge!(loose: true)
11
11
  all.find_ids_in_ranges(opts, &block)
12
12
  end
13
13
 
14
- def shard_category
15
- connection_specification_name.to_sym
16
- end
14
+ def sharded_model
15
+ self.abstract_class = true
17
16
 
18
- def shard_category=(category)
19
- categories = Shard.const_get(:CATEGORIES)
20
- if categories[shard_category]
21
- categories[shard_category].delete(self)
22
- categories.delete(shard_category) if categories[shard_category].empty?
23
- end
24
- categories[category] ||= []
25
- categories[category] << self
26
- self.connection_specification_name = category.to_s
17
+ return if self == UnshardedRecord
18
+
19
+ Shard.send(:add_sharded_model, self)
27
20
  end
28
21
 
29
22
  def integral_id?
30
- if @integral_id == nil
31
- @integral_id = columns_hash[primary_key]&.type == :integer
32
- end
23
+ @integral_id = columns_hash[primary_key]&.type == :integer if @integral_id.nil?
33
24
  @integral_id
34
25
  end
35
26
 
36
27
  def transaction(**)
37
28
  if self != ::ActiveRecord::Base && current_scope
38
29
  current_scope.activate do
39
- db = Shard.current(shard_category).database_server
40
- if ::GuardRail.environment != db.guard_rail_environment
41
- db.unguard { super }
42
- else
30
+ db = Shard.current(connection_classes).database_server
31
+ if ::GuardRail.environment == db.guard_rail_environment
43
32
  super
33
+ else
34
+ db.unguard { super }
44
35
  end
45
36
  end
46
37
  else
47
- db = Shard.current(shard_category).database_server
48
- if ::GuardRail.environment != db.guard_rail_environment
49
- db.unguard { super }
50
- else
38
+ db = Shard.current(connection_classes).database_server
39
+ if ::GuardRail.environment == db.guard_rail_environment
51
40
  super
41
+ else
42
+ db.unguard { super }
52
43
  end
53
44
  end
54
45
  end
@@ -72,39 +63,46 @@ module Switchman
72
63
  end
73
64
 
74
65
  def clear_query_caches_for_current_thread
75
- ::ActiveRecord::Base.connection_handlers.each_value do |handler|
76
- handler.connection_pool_list.each do |pool|
77
- pool.connection(switch_shard: false).clear_query_cache if pool.active_connection?
78
- end
66
+ ::ActiveRecord::Base.connection_handler.connection_pool_list.each do |pool|
67
+ pool.connection(switch_shard: false).clear_query_cache if pool.active_connection?
79
68
  end
80
69
  end
70
+
71
+ # significant change: _don't_ check if klasses.include?(Base)
72
+ # i.e. other sharded models don't inherit the current shard of Base
73
+ def current_shard
74
+ connected_to_stack.reverse_each do |hash|
75
+ return hash[:shard] if hash[:shard] && hash[:klasses].include?(connection_classes)
76
+ end
77
+
78
+ default_shard
79
+ end
81
80
  end
82
81
 
83
82
  def self.included(klass)
84
- klass.extend(ClassMethods)
83
+ klass.singleton_class.prepend(ClassMethods)
85
84
  klass.set_callback(:initialize, :before) do
86
- unless @shard
87
- if self.class.sharded_primary_key?
88
- @shard = Shard.shard_for(self[self.class.primary_key], Shard.current(self.class.shard_category))
89
- else
90
- @shard = Shard.current(self.class.shard_category)
91
- end
92
- end
85
+ @shard ||= if self.class.sharded_primary_key?
86
+ Shard.shard_for(self[self.class.primary_key], Shard.current(self.class.connection_classes))
87
+ else
88
+ Shard.current(self.class.connection_classes)
89
+ end
93
90
  end
94
91
  end
95
92
 
96
93
  def shard
97
- @shard || Shard.current(self.class.shard_category) || Shard.default
94
+ @shard || Shard.current(self.class.connection_classes) || Shard.default
98
95
  end
99
96
 
100
97
  def shard=(new_shard)
101
- raise ::ActiveRecord::ReadOnlyRecord if !self.new_record? || @shard_set_in_stone
102
- if shard != new_shard
103
- attributes.each do |attr, value|
104
- self[attr] = Shard.relative_id_for(value, shard, new_shard) if self.class.sharded_column?(attr)
105
- end
106
- @shard = new_shard
98
+ raise ::ActiveRecord::ReadOnlyRecord if !new_record? || @shard_set_in_stone
99
+
100
+ return if shard == new_shard
101
+
102
+ attributes.each do |attr, value|
103
+ self[attr] = Shard.relative_id_for(value, shard, new_shard) if self.class.sharded_column?(attr)
107
104
  end
105
+ @shard = new_shard
108
106
  end
109
107
 
110
108
  def save(*, **)
@@ -118,7 +116,7 @@ module Switchman
118
116
  end
119
117
 
120
118
  def destroy
121
- self.class.shard(shard, :implicit).scoping { super }
119
+ shard.activate(self.class.connection_classes) { super }
122
120
  end
123
121
 
124
122
  def clone
@@ -126,23 +124,23 @@ module Switchman
126
124
  # TODO: adjust foreign keys
127
125
  # don't use the setter, cause the foreign keys are already
128
126
  # relative to this shard
129
- result.instance_variable_set(:@shard, self.shard)
127
+ result.instance_variable_set(:@shard, shard)
130
128
  result
131
129
  end
132
130
 
133
131
  def transaction(**kwargs, &block)
134
- shard.activate(self.class.shard_category) do
132
+ shard.activate(self.class.connection_classes) do
135
133
  self.class.transaction(**kwargs, &block)
136
134
  end
137
135
  end
138
136
 
139
137
  def hash
140
- self.class.sharded_primary_key? ? self.class.hash ^ Shard.global_id_for(id).hash : super
138
+ self.class.sharded_primary_key? ? self.class.hash ^ global_id.hash : super
141
139
  end
142
140
 
143
141
  def to_param
144
142
  short_id = Shard.short_id_for(id)
145
- short_id && short_id.to_s
143
+ short_id&.to_s
146
144
  end
147
145
 
148
146
  def initialize_dup(*args)
@@ -153,37 +151,38 @@ module Switchman
153
151
 
154
152
  def quoted_id
155
153
  return super unless self.class.sharded_primary_key?
154
+
156
155
  # do this the Rails 4.2 way, so that if Shard.current != self.shard, the id gets transposed
157
156
  self.class.connection.quote(id)
158
157
  end
159
158
 
160
159
  def update_columns(*)
161
- db = Shard.current(self.class.shard_category).database_server
162
- if ::GuardRail.environment != db.guard_rail_environment
163
- return db.unguard { super }
164
- else
160
+ db = Shard.current(self.class.connection_classes).database_server
161
+ if ::GuardRail.environment == db.guard_rail_environment
165
162
  super
163
+ else
164
+ db.unguard { super }
166
165
  end
167
166
  end
168
167
 
169
168
  protected
170
169
 
171
- # see also AttributeMethods#shard_category_code_for_reflection
172
- def shard_category_for_reflection(reflection)
170
+ # see also AttributeMethods#connection_classes_code_for_reflection
171
+ def connection_classes_for_reflection(reflection)
173
172
  if reflection
174
173
  if reflection.options[:polymorphic]
175
174
  begin
176
- read_attribute(reflection.foreign_type)&.constantize&.shard_category || :primary
175
+ read_attribute(reflection.foreign_type)&.constantize&.connection_classes
177
176
  rescue NameError
178
177
  # in case someone is abusing foreign_type to not point to an actual class
179
- :primary
178
+ ::ActiveRecord::Base
180
179
  end
181
180
  else
182
181
  # otherwise we can just return a symbol for the statically known type of the association
183
- reflection.klass.shard_category
182
+ reflection.klass.connection_classes
184
183
  end
185
184
  else
186
- shard_category
185
+ connection_classes
187
186
  end
188
187
  end
189
188
  end
@@ -3,45 +3,44 @@
3
3
  module Switchman
4
4
  module ActiveRecord
5
5
  module Calculations
6
-
7
6
  def pluck(*column_names)
8
- target_shard = Shard.current(klass.shard_category)
7
+ target_shard = Shard.current(klass.connection_classes)
9
8
  shard_count = 0
10
- result = self.activate do |relation, shard|
9
+ result = activate do |relation, shard|
11
10
  shard_count += 1
12
11
  results = relation.call_super(:pluck, Calculations, *column_names)
13
12
  if column_names.length > 1
14
13
  column_names.each_with_index do |column_name, idx|
15
- if klass.sharded_column?(column_name)
16
- results.each{|result| result[idx] = Shard.relative_id_for(result[idx], shard, target_shard)}
14
+ next unless klass.sharded_column?(column_name)
15
+
16
+ results.each do |r|
17
+ r[idx] = Shard.relative_id_for(r[idx], shard, target_shard)
17
18
  end
18
19
  end
19
- else
20
- if klass.sharded_column?(column_names.first.to_s)
21
- results = results.map{|result| Shard.relative_id_for(result, shard, target_shard)}
22
- end
20
+ elsif klass.sharded_column?(column_names.first.to_s)
21
+ results = results.map { |r| Shard.relative_id_for(r, shard, target_shard) }
23
22
  end
24
23
  results
25
24
  end
26
- if distinct_value && shard_count > 1
27
- result.uniq!
28
- end
25
+ result.uniq! if distinct_value && shard_count > 1
29
26
  result
30
27
  end
31
28
 
32
29
  def execute_simple_calculation(operation, column_name, distinct)
33
30
  operation = operation.to_s.downcase
34
- if operation == "average"
31
+ if operation == 'average'
35
32
  result = calculate_simple_average(column_name, distinct)
36
33
  else
37
- result = self.activate{ |relation| relation.call_super(:execute_simple_calculation, Calculations, operation, column_name, distinct) }
34
+ result = activate do |relation|
35
+ relation.call_super(:execute_simple_calculation, Calculations, operation, column_name, distinct)
36
+ end
38
37
  if result.is_a?(Array)
39
38
  case operation
40
- when "count", "sum"
39
+ when 'count', 'sum'
41
40
  result = result.sum
42
- when "minimum"
41
+ when 'minimum'
43
42
  result = result.min
44
- when "maximum"
43
+ when 'maximum'
45
44
  result = result.max
46
45
  end
47
46
  end
@@ -53,18 +52,20 @@ module Switchman
53
52
  # See activerecord#execute_simple_calculation
54
53
  relation = except(:order)
55
54
  column = aggregate_column(column_name)
56
- relation.select_values = [operation_over_aggregate_column(column, "average", distinct).as("average"),
57
- operation_over_aggregate_column(column, "count", distinct).as("count")]
55
+ relation.select_values = [operation_over_aggregate_column(column, 'average', distinct).as('average'),
56
+ operation_over_aggregate_column(column, 'count', distinct).as('count')]
58
57
 
59
- initial_results = relation.activate{ |rel| klass.connection.select_all(rel) }
58
+ initial_results = relation.activate { |rel| klass.connection.select_all(rel) }
60
59
  if initial_results.is_a?(Array)
61
60
  initial_results.each do |r|
62
- r["average"] = type_cast_calculated_value(r["average"], type_for(column_name), "average")
63
- r["count"] = type_cast_calculated_value(r["count"], type_for(column_name), "count")
61
+ r['average'] = type_cast_calculated_value_switchman(r['average'], column_name, 'average')
62
+ r['count'] = type_cast_calculated_value_switchman(r['count'], column_name, 'count')
64
63
  end
65
- result = initial_results.map{|r| r["average"] * r["count"]}.sum / initial_results.map{|r| r["count"]}.sum
64
+ result = initial_results.map { |r| r['average'] * r['count'] }.sum / initial_results.map do |r|
65
+ r['count']
66
+ end.sum
66
67
  else
67
- result = type_cast_calculated_value(initial_results.first["average"], type_for(column_name), "average")
68
+ result = type_cast_calculated_value_switchman(initial_results.first['average'], column_name, 'average')
68
69
  end
69
70
  result
70
71
  end
@@ -74,27 +75,28 @@ module Switchman
74
75
  opts = grouped_calculation_options(operation.to_s.downcase, column_name, distinct)
75
76
 
76
77
  relation = build_grouped_calculation_relation(opts)
77
- target_shard = Shard.current(:primary)
78
+ target_shard = Shard.current
78
79
 
79
80
  rows = relation.activate do |rel, shard|
80
81
  calculated_data = klass.connection.select_all(rel)
81
82
 
82
83
  if opts[:association]
83
84
  key_ids = calculated_data.collect { |row| row[opts[:group_aliases].first] }
84
- key_records = opts[:association].klass.base_class.where(:id => key_ids)
85
- key_records = Hash[key_records.map { |r| [Shard.relative_id_for(r, shard, target_shard), r] }]
85
+ key_records = opts[:association].klass.base_class.where(id: key_ids)
86
+ key_records = key_records.map { |r| [Shard.relative_id_for(r, shard, target_shard), r] }.to_h
86
87
  end
87
88
 
88
89
  calculated_data.map do |row|
89
- row[opts[:aggregate_alias]] = type_cast_calculated_value(
90
- row[opts[:aggregate_alias]], type_for(opts[:column_name]), opts[:operation])
90
+ row[opts[:aggregate_alias]] = type_cast_calculated_value_switchman(
91
+ row[opts[:aggregate_alias]], column_name, opts[:operation]
92
+ )
91
93
  row['count'] = row['count'].to_i if opts[:operation] == 'average'
92
94
 
93
- opts[:group_columns].each do |aliaz, type, column_name|
95
+ opts[:group_columns].each do |aliaz, _type, group_column_name|
94
96
  if opts[:associated] && (aliaz == opts[:group_aliases].first)
95
97
  row[aliaz] = key_records[Shard.relative_id_for(row[aliaz], shard, target_shard)]
96
- elsif column_name && @klass.sharded_column?(column_name)
97
- row[aliaz] = Shard.relative_id_for(type_cast_calculated_value(row[aliaz], type), shard, target_shard)
98
+ elsif group_column_name && @klass.sharded_column?(group_column_name)
99
+ row[aliaz] = Shard.relative_id_for(row[aliaz], shard, target_shard)
98
100
  end
99
101
  end
100
102
  row
@@ -106,12 +108,21 @@ module Switchman
106
108
 
107
109
  private
108
110
 
111
+ def type_cast_calculated_value_switchman(value, column_name, operation)
112
+ type_cast_calculated_value(value, operation) do |val|
113
+ column = aggregate_column(column_name)
114
+ type ||= column.try(:type_caster) ||
115
+ lookup_cast_type_from_join_dependencies(column_name.to_s) || Type.default_value
116
+ type.deserialize(val)
117
+ end
118
+ end
119
+
109
120
  def column_name_for(field)
110
121
  field.respond_to?(:name) ? field.name.to_s : field.to_s.split('.').last
111
122
  end
112
123
 
113
124
  def grouped_calculation_options(operation, column_name, distinct)
114
- opts = {:operation => operation, :column_name => column_name, :distinct => distinct}
125
+ opts = { operation: operation, column_name: column_name, distinct: distinct }
115
126
 
116
127
  opts[:aggregate_alias] = aggregate_alias_for(operation, column_name)
117
128
  group_attrs = group_values
@@ -125,12 +136,12 @@ module Switchman
125
136
 
126
137
  # to_s is because Rails 5 returns a string but Rails 6 returns a symbol.
127
138
  group_aliases = group_fields.map { |field| column_alias_for(field.downcase.to_s).to_s }
128
- group_columns = group_aliases.zip(group_fields).map { |aliaz, field|
139
+ group_columns = group_aliases.zip(group_fields).map do |aliaz, field|
129
140
  [aliaz, type_for(field), column_name_for(field)]
130
- }
131
- opts.merge!(:association => association, :associated => associated,
132
- :group_aliases => group_aliases, :group_columns => group_columns,
133
- :group_fields => group_fields)
141
+ end
142
+ opts.merge!(association: association, associated: associated,
143
+ group_aliases: group_aliases, group_columns: group_columns,
144
+ group_fields: group_fields)
134
145
 
135
146
  opts
136
147
  end
@@ -149,28 +160,30 @@ module Switchman
149
160
  group = opts[:group_fields]
150
161
 
151
162
  select_values = [
152
- operation_over_aggregate_column(
153
- aggregate_column(opts[:column_name]),
154
- opts[:operation],
155
- opts[:distinct]).as(opts[:aggregate_alias])
163
+ operation_over_aggregate_column(
164
+ aggregate_column(opts[:column_name]),
165
+ opts[:operation],
166
+ opts[:distinct]
167
+ ).as(opts[:aggregate_alias])
156
168
  ]
157
- if opts[:operation ]== 'average'
169
+ if opts[:operation] == 'average'
158
170
  # include count in average so we can recalculate the average
159
171
  # across all shards if needed
160
172
  select_values << operation_over_aggregate_column(
161
- aggregate_column(opts[:column_name]),
162
- 'count', opts[:distinct]).as('count')
173
+ aggregate_column(opts[:column_name]),
174
+ 'count', opts[:distinct]
175
+ ).as('count')
163
176
  end
164
177
 
165
178
  haves = having_clause.send(:predicates)
166
179
  select_values += select_values unless haves.empty?
167
- select_values.concat opts[:group_fields].zip(opts[:group_aliases]).map { |field,aliaz|
180
+ select_values.concat(opts[:group_fields].zip(opts[:group_aliases]).map do |field, aliaz|
168
181
  if field.respond_to?(:as)
169
182
  field.as(aliaz)
170
183
  else
171
184
  "#{field} AS #{aliaz}"
172
185
  end
173
- }
186
+ end)
174
187
 
175
188
  relation = except(:group)
176
189
  relation.group_values = group
@@ -181,12 +194,12 @@ module Switchman
181
194
  def compact_grouped_calculation_rows(rows, opts)
182
195
  result = ::ActiveSupport::OrderedHash.new
183
196
  rows.each do |row|
184
- key = opts[:group_columns].map { |aliaz, column| row[aliaz] }
197
+ key = opts[:group_columns].map { |aliaz, _column| row[aliaz] }
185
198
  key = key.first if key.size == 1
186
199
  value = row[opts[:aggregate_alias]]
187
200
 
188
201
  if opts[:operation] == 'average'
189
- if result.has_key?(key)
202
+ if result.key?(key)
190
203
  old_value, old_count = result[key]
191
204
  new_count = old_count + row['count']
192
205
  new_value = ((old_value * old_count) + (value * row['count'])) / new_count
@@ -194,30 +207,24 @@ module Switchman
194
207
  else
195
208
  result[key] = [value, row['count']]
196
209
  end
197
- else
198
- if result.has_key?(key)
199
- case opts[:operation]
200
- when "count", "sum"
201
- result[key] += value
202
- when "minimum"
203
- result[key] = value if value < result[key]
204
- when "maximum"
205
- result[key] = value if value > result[key]
206
- end
207
- else
208
- result[key] = value
210
+ elsif result.key?(key)
211
+ case opts[:operation]
212
+ when 'count', 'sum'
213
+ result[key] += value
214
+ when 'minimum'
215
+ result[key] = value if value < result[key]
216
+ when 'maximum'
217
+ result[key] = value if value > result[key]
209
218
  end
219
+ else
220
+ result[key] = value
210
221
  end
211
222
  end
212
223
 
213
- if opts[:operation] == 'average'
214
- result = Hash[result.map{|k, v| [k, v.first]}]
215
- end
224
+ result.transform_values!(&:first) if opts[:operation] == 'average'
216
225
 
217
226
  result
218
227
  end
219
-
220
-
221
228
  end
222
229
  end
223
230
  end