switchman 2.0.13 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. checksums.yaml +4 -4
  2. data/Rakefile +10 -2
  3. data/app/models/switchman/shard.rb +234 -271
  4. data/app/models/switchman/unsharded_record.rb +7 -0
  5. data/db/migrate/20130328212039_create_switchman_shards.rb +1 -1
  6. data/db/migrate/20130328224244_create_default_shard.rb +5 -5
  7. data/db/migrate/20161206323434_add_back_default_string_limits_switchman.rb +1 -0
  8. data/db/migrate/20180828183945_add_default_shard_index.rb +2 -2
  9. data/db/migrate/20180828192111_add_timestamps_to_shards.rb +7 -5
  10. data/db/migrate/20190114212900_add_unique_name_indexes.rb +5 -3
  11. data/lib/switchman.rb +3 -5
  12. data/lib/switchman/action_controller/caching.rb +2 -2
  13. data/lib/switchman/active_record/abstract_adapter.rb +1 -0
  14. data/lib/switchman/active_record/association.rb +78 -89
  15. data/lib/switchman/active_record/attribute_methods.rb +58 -52
  16. data/lib/switchman/active_record/base.rb +58 -59
  17. data/lib/switchman/active_record/calculations.rb +74 -67
  18. data/lib/switchman/active_record/connection_pool.rb +14 -41
  19. data/lib/switchman/active_record/database_configurations.rb +34 -0
  20. data/lib/switchman/active_record/database_configurations/database_config.rb +13 -0
  21. data/lib/switchman/active_record/finder_methods.rb +11 -16
  22. data/lib/switchman/active_record/log_subscriber.rb +4 -8
  23. data/lib/switchman/active_record/migration.rb +6 -47
  24. data/lib/switchman/active_record/model_schema.rb +1 -1
  25. data/lib/switchman/active_record/persistence.rb +4 -6
  26. data/lib/switchman/active_record/postgresql_adapter.rb +124 -168
  27. data/lib/switchman/active_record/predicate_builder.rb +2 -2
  28. data/lib/switchman/active_record/query_cache.rb +18 -19
  29. data/lib/switchman/active_record/query_methods.rb +172 -197
  30. data/lib/switchman/active_record/reflection.rb +6 -10
  31. data/lib/switchman/active_record/relation.rb +30 -78
  32. data/lib/switchman/active_record/spawn_methods.rb +27 -29
  33. data/lib/switchman/active_record/statement_cache.rb +18 -35
  34. data/lib/switchman/active_record/tasks/database_tasks.rb +16 -0
  35. data/lib/switchman/active_support/cache.rb +3 -5
  36. data/lib/switchman/arel.rb +13 -8
  37. data/lib/switchman/database_server.rb +121 -142
  38. data/lib/switchman/default_shard.rb +52 -16
  39. data/lib/switchman/engine.rb +61 -58
  40. data/lib/switchman/environment.rb +4 -8
  41. data/lib/switchman/errors.rb +1 -0
  42. data/lib/switchman/guard_rail.rb +6 -19
  43. data/lib/switchman/guard_rail/relation.rb +5 -7
  44. data/lib/switchman/r_spec_helper.rb +29 -37
  45. data/lib/switchman/rails.rb +14 -12
  46. data/lib/switchman/schema_cache.rb +1 -9
  47. data/lib/switchman/sharded_instrumenter.rb +1 -1
  48. data/lib/switchman/standard_error.rb +15 -3
  49. data/lib/switchman/test_helper.rb +7 -11
  50. data/lib/switchman/version.rb +1 -1
  51. data/lib/tasks/switchman.rake +54 -69
  52. metadata +87 -45
  53. data/lib/switchman/active_record/batches.rb +0 -11
  54. data/lib/switchman/active_record/connection_handler.rb +0 -172
  55. data/lib/switchman/active_record/where_clause_factory.rb +0 -36
  56. data/lib/switchman/connection_pool_proxy.rb +0 -173
@@ -5,7 +5,7 @@ module Switchman
5
5
  module Reflection
6
6
  module AbstractReflection
7
7
  def shard(owner)
8
- if polymorphic? || klass.shard_category == owner.class.shard_category
8
+ if polymorphic? || klass.connection_classes == owner.class.connection_classes
9
9
  # polymorphic associations assume the same shard as the owning item
10
10
  owner.shard
11
11
  else
@@ -28,21 +28,17 @@ module Switchman
28
28
  # this technically belongs on AssociationReflection, but we put it on
29
29
  # ThroughReflection as well, instead of delegating to its internal
30
30
  # HasManyAssociation, losing its proper `klass`
31
- def association_scope_cache(conn, owner, &block)
32
- key = conn.prepared_statements
33
- if polymorphic?
34
- key = [key, owner._read_attribute(@foreign_type)]
35
- end
31
+ def association_scope_cache(klass, owner, &block)
32
+ key = self
33
+ key = [key, owner._read_attribute(@foreign_type)] if polymorphic?
36
34
  key = [key, shard(owner).id].flatten
37
- @association_scope_cache[key] ||= @scope_lock.synchronize {
38
- @association_scope_cache[key] ||= (::Rails.version >= "5.2" ? ::ActiveRecord::StatementCache.create(conn, &block) : block.call)
39
- }
35
+ klass.cached_find_by_statement(key, &block)
40
36
  end
41
37
  end
42
38
 
43
39
  module AssociationReflection
44
40
  def join_id_for(owner)
45
- owner.send(::Rails.version >= "5.2" ? join_foreign_key : active_record_primary_key) # use sharded id values in association binds
41
+ owner.send(join_foreign_key) # use sharded id values in association binds
46
42
  end
47
43
  end
48
44
  end
@@ -4,53 +4,54 @@ module Switchman
4
4
  module ActiveRecord
5
5
  module Relation
6
6
  def self.prepended(klass)
7
- klass::SINGLE_VALUE_METHODS.concat [ :shard, :shard_source ]
7
+ klass::SINGLE_VALUE_METHODS.concat %i[shard shard_source]
8
8
  end
9
9
 
10
10
  def initialize(*, **)
11
11
  super
12
- self.shard_value = Shard.current(klass ? klass.shard_category : :primary) unless shard_value
12
+ self.shard_value = Shard.current(klass ? klass.connection_classes : :primary) unless shard_value
13
13
  self.shard_source_value = :implicit unless shard_source_value
14
14
  end
15
15
 
16
16
  def clone
17
17
  result = super
18
- result.shard_value = Shard.current(klass ? klass.shard_category : :primary) unless shard_value
18
+ result.shard_value = Shard.current(klass ? klass.connection_classes : :primary) unless shard_value
19
19
  result
20
20
  end
21
21
 
22
22
  def merge(*)
23
23
  relation = super
24
- if relation.shard_value != self.shard_value && relation.shard_source_value == :implicit
25
- relation.shard_value = self.shard_value
26
- relation.shard_source_value = self.shard_source_value
24
+ if relation.shard_value != shard_value && relation.shard_source_value == :implicit
25
+ relation.shard_value = shard_value
26
+ relation.shard_source_value = shard_source_value
27
27
  end
28
28
  relation
29
29
  end
30
30
 
31
31
  def new(*, &block)
32
- primary_shard.activate(klass.shard_category) { super }
32
+ primary_shard.activate(klass.connection_classes) { super }
33
33
  end
34
34
 
35
35
  def create(*, &block)
36
- primary_shard.activate(klass.shard_category) { super }
36
+ primary_shard.activate(klass.connection_classes) { super }
37
37
  end
38
38
 
39
39
  def create!(*, &block)
40
- primary_shard.activate(klass.shard_category) { super }
40
+ primary_shard.activate(klass.connection_classes) { super }
41
41
  end
42
42
 
43
43
  def to_sql
44
- primary_shard.activate(klass.shard_category) { super }
44
+ primary_shard.activate(klass.connection_classes) { super }
45
45
  end
46
46
 
47
47
  def explain
48
- self.activate { |relation| relation.call_super(:explain, Relation) }
48
+ activate { |relation| relation.call_super(:explain, Relation) }
49
49
  end
50
50
 
51
51
  def records
52
52
  return @records if loaded?
53
- results = self.activate { |relation| relation.call_super(:records, Relation) }
53
+
54
+ results = activate { |relation| relation.call_super(:records, Relation) }
54
55
  case shard_value
55
56
  when Array, ::ActiveRecord::Relation, ::ActiveRecord::Base
56
57
  @records = results
@@ -59,10 +60,10 @@ module Switchman
59
60
  results
60
61
  end
61
62
 
62
- %I{update_all delete_all}.each do |method|
63
+ %I[update_all delete_all].each do |method|
63
64
  class_eval <<-RUBY, __FILE__, __LINE__ + 1
64
65
  def #{method}(*args)
65
- result = self.activate(unordered: true) { |relation| relation.call_super(#{method.inspect}, Relation, *args) }
66
+ result = self.activate { |relation| relation.call_super(#{method.inspect}, Relation, *args) }
66
67
  result = result.sum if result.is_a?(Array)
67
68
  result
68
69
  end
@@ -74,15 +75,20 @@ module Switchman
74
75
  loose_mode = options[:loose] && is_integer
75
76
  # loose_mode: if we don't care about getting exactly batch_size ids in between
76
77
  # don't get the max - just get the min and add batch_size so we get that many _at most_
77
- values = loose_mode ? "MIN(id)" : "MIN(id), MAX(id)"
78
+ values = loose_mode ? 'MIN(id)' : 'MIN(id), MAX(id)'
78
79
 
79
80
  batch_size = options[:batch_size].try(:to_i) || 1000
80
81
  quoted_primary_key = "#{klass.connection.quote_local_table_name(table_name)}.#{klass.connection.quote_column_name(primary_key)}"
81
- as_id = " AS id" unless primary_key == 'id'
82
+ as_id = ' AS id' unless primary_key == 'id'
82
83
  subquery_scope = except(:select).select("#{quoted_primary_key}#{as_id}").reorder(primary_key.to_sym).limit(loose_mode ? 1 : batch_size)
83
84
  subquery_scope = subquery_scope.where("#{quoted_primary_key} <= ?", options[:end_at]) if options[:end_at]
84
85
 
85
- first_subquery_scope = options[:start_at] ? subquery_scope.where("#{quoted_primary_key} >= ?", options[:start_at]) : subquery_scope
86
+ first_subquery_scope = if options[:start_at]
87
+ subquery_scope.where("#{quoted_primary_key} >= ?",
88
+ options[:start_at])
89
+ else
90
+ subquery_scope
91
+ end
86
92
 
87
93
  ids = connection.select_rows("SELECT #{values} FROM (#{first_subquery_scope.to_sql}) AS subquery").first
88
94
 
@@ -97,73 +103,19 @@ module Switchman
97
103
  end
98
104
  end
99
105
 
100
- def activate(unordered: false, &block)
106
+ def activate(&block)
101
107
  shards = all_shards
102
- if (Array === shards && shards.length == 1)
103
- if shards.first == DefaultShard || shards.first == Shard.current(klass.shard_category)
108
+ if Array === shards && shards.length == 1
109
+ if shards.first == DefaultShard || shards.first == Shard.current(klass.connection_classes)
104
110
  yield(self, shards.first)
105
111
  else
106
- shards.first.activate(klass.shard_category) { yield(self, shards.first) }
112
+ shards.first.activate(klass.connection_classes) { yield(self, shards.first) }
107
113
  end
108
114
  else
109
- result_count = 0
110
- can_order = false
111
- result = Shard.with_each_shard(shards, [klass.shard_category]) do
112
- # don't even query other shards if we're already past the limit
113
- next if limit_value && result_count >= limit_value && order_values.empty?
114
-
115
- relation = shard(Shard.current(klass.shard_category), :to_a)
116
- # do a minimal query if possible
117
- relation = relation.limit(limit_value - result_count) if limit_value && !result_count.zero? && order_values.empty?
118
-
119
- shard_results = relation.activate(&block)
120
-
121
- if shard_results.present? && !unordered
122
- can_order ||= can_order_cross_shard_results? unless order_values.empty?
123
- raise OrderOnMultiShardQuery if !can_order && !order_values.empty? && result_count.positive?
124
-
125
- result_count += shard_results.is_a?(Array) ? shard_results.length : 1
126
- end
127
- shard_results
128
- end
129
-
130
- result = reorder_cross_shard_results(result) if can_order
131
- result.slice!(limit_value..-1) if limit_value
132
- result
133
- end
134
- end
135
-
136
- def can_order_cross_shard_results?
137
- # we only presume to be able to post-sort the most basic of orderings
138
- order_values.all? { |ov| ov.is_a?(::Arel::Nodes::Ordering) && ov.expr.is_a?(::Arel::Attributes::Attribute) }
139
- end
140
-
141
- def reorder_cross_shard_results(results)
142
- results.sort! do |l, r|
143
- result = 0
144
- order_values.each do |ov|
145
- if !l.is_a?(::ActiveRecord::Base)
146
- a, b = l, r
147
- elsif l.respond_to?(ov.expr.name)
148
- a = l.send(ov.expr.name)
149
- b = r.send(ov.expr.name)
150
- else
151
- a = l.attributes[ov.expr.name]
152
- b = r.attributes[ov.expr.name]
153
- end
154
- next if a == b
155
-
156
- if a.nil? || b.nil?
157
- result = 1 if a.nil?
158
- result *= -1 if ov.is_a?(::Arel::Nodes::Descending)
159
- else
160
- result = a <=> b
161
- end
162
-
163
- result *= -1 if ov.is_a?(::Arel::Nodes::Descending)
164
- break unless result.zero?
115
+ # TODO: implement local limit to avoid querying extra shards
116
+ Shard.with_each_shard(shards, [klass.connection_classes]) do
117
+ shard(Shard.current(klass.connection_classes), :to_a).activate(&block)
165
118
  end
166
- result
167
119
  end
168
120
  end
169
121
  end
@@ -3,27 +3,27 @@
3
3
  module Switchman
4
4
  module ActiveRecord
5
5
  module SpawnMethods
6
- def shard_values_for_merge(r)
7
- if shard_value != r.shard_value
8
- if r.shard_source_value == :implicit
6
+ def shard_values_for_merge(rhs)
7
+ if shard_value != rhs.shard_value
8
+ if rhs.shard_source_value == :implicit
9
9
  final_shard_value = shard_value
10
10
  final_primary_shard = primary_shard
11
11
  final_shard_source_value = shard_source_value
12
12
  elsif shard_source_value == :implicit
13
- final_shard_value = r.shard_value
14
- final_primary_shard = r.primary_shard
15
- final_shard_source_value = r.shard_source_value
13
+ final_shard_value = rhs.shard_value
14
+ final_primary_shard = rhs.primary_shard
15
+ final_shard_source_value = rhs.shard_source_value
16
16
  else
17
- final_shard_source_value = [:explicit, :association].detect do |source_value|
18
- shard_source_value == source_value || r.shard_source_value == source_value
17
+ final_shard_source_value = %i[explicit association].detect do |source_value|
18
+ shard_source_value == source_value || rhs.shard_source_value == source_value
19
19
  end
20
- raise "unknown shard_source_value" unless final_shard_source_value
20
+ raise 'unknown shard_source_value' unless final_shard_source_value
21
21
 
22
22
  # have to merge shard_value
23
23
  lhs_shard_value = all_shards
24
- rhs_shard_value = r.all_shards
25
- if (::ActiveRecord::Relation === lhs_shard_value &&
26
- ::ActiveRecord::Relation === rhs_shard_value)
24
+ rhs_shard_value = rhs.all_shards
25
+ if ::ActiveRecord::Relation === lhs_shard_value &&
26
+ ::ActiveRecord::Relation === rhs_shard_value
27
27
  final_shard_value = lhs_shard_value.merge(rhs_shard_value)
28
28
  final_primary_shard = Shard.default
29
29
  else
@@ -32,27 +32,25 @@ module Switchman
32
32
  final_shard_value = final_shard_value.first if final_shard_value.length == 1
33
33
  end
34
34
  end
35
- elsif shard_source_value != r.shard_source_value
36
- final_shard_source_value = [:explicit, :association, :implicit].detect do |source_value|
37
- shard_source_value == source_value || r.shard_source_value == source_value
35
+ elsif shard_source_value != rhs.shard_source_value
36
+ final_shard_source_value = %i[explicit association implicit].detect do |source_value|
37
+ shard_source_value == source_value || rhs.shard_source_value == source_value
38
38
  end
39
- raise "unknown shard_source_value" unless final_shard_source_value
40
- else
41
- # nothing fancy
39
+ raise 'unknown shard_source_value' unless final_shard_source_value
42
40
  end
43
41
 
44
42
  [final_shard_value, final_primary_shard, final_shard_source_value]
45
43
  end
46
44
 
47
- def merge!(r)
48
- return super unless ::ActiveRecord::Relation === r
45
+ def merge!(rhs)
46
+ return super unless ::ActiveRecord::Relation === rhs
49
47
 
50
48
  # have to figure out shard stuff *before* conditions are merged
51
- final_shard_value, final_primary_shard, final_shard_source_value = shard_values_for_merge(r)
49
+ final_shard_value, final_primary_shard, final_shard_source_value = shard_values_for_merge(rhs)
52
50
 
53
51
  return super unless final_shard_source_value
54
52
 
55
- if !final_shard_value
53
+ unless final_shard_value
56
54
  super
57
55
  self.shard_source_value = final_shard_source_value
58
56
  return self
@@ -61,16 +59,16 @@ module Switchman
61
59
  return none! if final_shard_value == []
62
60
 
63
61
  # change the primary shard if necessary before merging
64
- if primary_shard != final_primary_shard && r.primary_shard != final_primary_shard
62
+ if primary_shard != final_primary_shard && rhs.primary_shard != final_primary_shard
65
63
  shard!(final_primary_shard)
66
- r = r.shard(final_primary_shard)
67
- super(r)
64
+ rhs = rhs.shard(final_primary_shard)
65
+ super(rhs)
68
66
  elsif primary_shard != final_primary_shard
69
67
  shard!(final_primary_shard)
70
- super(r)
71
- elsif r.primary_shard != final_primary_shard
72
- r = r.shard(final_primary_shard)
73
- super(r)
68
+ super(rhs)
69
+ elsif rhs.primary_shard != final_primary_shard
70
+ rhs = rhs.shard(final_primary_shard)
71
+ super(rhs)
74
72
  else
75
73
  super
76
74
  end
@@ -7,18 +7,13 @@ module Switchman
7
7
  def create(connection, &block)
8
8
  relation = block.call ::ActiveRecord::StatementCache::Params.new
9
9
 
10
- if ::Rails.version >= "5.2"
11
- query_builder, binds = connection.cacheable_query(self, relation.arel)
12
- bind_map = ::ActiveRecord::StatementCache::BindMap.new(binds)
13
- new(relation.arel, bind_map, relation.klass)
14
- else
15
- bind_map = ::ActiveRecord::StatementCache::BindMap.new(relation.bound_attributes)
16
- new relation.arel, bind_map
17
- end
10
+ _query_builder, binds = connection.cacheable_query(self, relation.arel)
11
+ bind_map = ::ActiveRecord::StatementCache::BindMap.new(binds)
12
+ new(relation.arel, bind_map, relation.klass)
18
13
  end
19
14
  end
20
15
 
21
- def initialize(arel, bind_map, klass=nil)
16
+ def initialize(arel, bind_map, klass = nil)
22
17
  @arel = arel
23
18
  @bind_map = bind_map
24
19
  @klass = klass
@@ -31,49 +26,39 @@ module Switchman
31
26
  # (e.g. infer from the primary key or use the current shard)
32
27
 
33
28
  def execute(*args)
34
- if ::Rails.version >= '5.2'
35
- params, connection = args
36
- klass = @klass
37
- else
38
- params, klass, connection = args
39
- end
29
+ params, connection = args
30
+ klass = @klass
40
31
  target_shard = nil
41
- if primary_index = bind_map.primary_value_index
32
+ if (primary_index = bind_map.primary_value_index)
42
33
  primary_value = params[primary_index]
43
34
  target_shard = Shard.local_id_for(primary_value)[1]
44
35
  end
45
- current_shard = Shard.current(klass.shard_category)
36
+ current_shard = Shard.current(klass.connection_classes)
46
37
  target_shard ||= current_shard
47
38
 
48
39
  bind_values = bind_map.bind(params, current_shard, target_shard)
49
40
 
50
- target_shard.activate(klass.shard_category) do
41
+ target_shard.activate(klass.connection_classes) do
51
42
  sql = qualified_query_builder(target_shard, klass).sql_for(bind_values, connection)
52
43
  klass.find_by_sql(sql, bind_values)
53
44
  end
54
45
  end
55
46
 
56
- if ::Rails.version < '5.2'
57
- def qualified_query_builder(shard, klass)
58
- @qualified_query_builders[shard.id] ||= klass.connection.cacheable_query(self.class, @arel)
59
- end
60
- else
61
- def qualified_query_builder(shard, klass)
62
- @qualified_query_builders[shard.id] ||= klass.connection.cacheable_query(self.class, @arel).first
63
- end
47
+ def qualified_query_builder(shard, klass)
48
+ @qualified_query_builders[shard.id] ||= klass.connection.cacheable_query(self.class, @arel).first
64
49
  end
65
50
 
66
51
  module BindMap
67
52
  # performs id transposition here instead of query_methods.rb
68
53
  def bind(values, current_shard, target_shard)
69
54
  bas = @bound_attributes.dup
70
- @indexes.each_with_index do |offset,i|
55
+ @indexes.each_with_index do |offset, i|
71
56
  ba = bas[offset]
72
- if ba.is_a?(::ActiveRecord::Relation::QueryAttribute) && ba.value.sharded
73
- new_value = Shard.relative_id_for(values[i], current_shard, target_shard || current_shard)
74
- else
75
- new_value = values[i]
76
- end
57
+ new_value = if ba.is_a?(::ActiveRecord::Relation::QueryAttribute) && ba.value.sharded
58
+ Shard.relative_id_for(values[i], current_shard, target_shard || current_shard)
59
+ else
60
+ values[i]
61
+ end
77
62
  bas[offset] = ba.with_cast_value(new_value)
78
63
  end
79
64
  bas
@@ -83,9 +68,7 @@ module Switchman
83
68
  primary_ba_index = @bound_attributes.index do |ba|
84
69
  ba.is_a?(::ActiveRecord::Relation::QueryAttribute) && ba.value.primary
85
70
  end
86
- if primary_ba_index
87
- @indexes.index(primary_ba_index)
88
- end
71
+ @indexes.index(primary_ba_index) if primary_ba_index
89
72
  end
90
73
  end
91
74
  end
@@ -0,0 +1,16 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Switchman
4
+ module ActiveRecord
5
+ module Tasks
6
+ module DatabaseTasks
7
+ def drop(*)
8
+ super
9
+ # no really, it's gone
10
+ Switchman.cache.delete('default_shard')
11
+ Shard.default(reload: true)
12
+ end
13
+ end
14
+ end
15
+ end
16
+ end
@@ -8,10 +8,8 @@ module Switchman
8
8
  store = super
9
9
  # can't use defined?, because it's a _ruby_ autoloaded constant,
10
10
  # so just checking that will cause it to get required
11
- if store.class.name == "ActiveSupport::Cache::RedisCacheStore" && !::ActiveSupport::Cache::RedisCacheStore.ancestors.include?(RedisCacheStore)
12
- ::ActiveSupport::Cache::RedisCacheStore.prepend(RedisCacheStore)
13
- end
14
- store.options[:namespace] ||= lambda { Shard.current.default? ? nil : "shard_#{Shard.current.id}" }
11
+ ::ActiveSupport::Cache::RedisCacheStore.prepend(RedisCacheStore) if store.instance_of?(ActiveSupport::Cache::RedisCacheStore) && !::ActiveSupport::Cache::RedisCacheStore.ancestors.include?(RedisCacheStore)
12
+ store.options[:namespace] ||= -> { Shard.current.default? ? nil : "shard_#{Shard.current.id}" }
15
13
  store
16
14
  end
17
15
  end
@@ -22,7 +20,7 @@ module Switchman
22
20
  # unfortunately, it uses the keys command, which is extraordinarily inefficient in a large redis instance
23
21
  # fortunately, we can assume we control the entire instance, because we set up the namespacing, so just
24
22
  # always unset it temporarily for clear calls
25
- namespace = nil
23
+ namespace = nil # rubocop:disable Lint/ShadowedArgument
26
24
  super
27
25
  end
28
26
  end