switchman 2.0.9 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Rakefile +10 -2
- data/app/models/switchman/shard.rb +234 -271
- data/app/models/switchman/unsharded_record.rb +7 -0
- data/db/migrate/20130328212039_create_switchman_shards.rb +1 -1
- data/db/migrate/20130328224244_create_default_shard.rb +5 -5
- data/db/migrate/20161206323434_add_back_default_string_limits_switchman.rb +1 -0
- data/db/migrate/20180828183945_add_default_shard_index.rb +2 -2
- data/db/migrate/20180828192111_add_timestamps_to_shards.rb +7 -5
- data/db/migrate/20190114212900_add_unique_name_indexes.rb +5 -3
- data/lib/switchman.rb +3 -3
- data/lib/switchman/action_controller/caching.rb +2 -2
- data/lib/switchman/active_record/abstract_adapter.rb +1 -0
- data/lib/switchman/active_record/association.rb +78 -89
- data/lib/switchman/active_record/attribute_methods.rb +58 -52
- data/lib/switchman/active_record/base.rb +58 -59
- data/lib/switchman/active_record/calculations.rb +73 -66
- data/lib/switchman/active_record/connection_pool.rb +14 -41
- data/lib/switchman/active_record/database_configurations.rb +34 -0
- data/lib/switchman/active_record/database_configurations/database_config.rb +13 -0
- data/lib/switchman/active_record/finder_methods.rb +11 -16
- data/lib/switchman/active_record/log_subscriber.rb +4 -8
- data/lib/switchman/active_record/migration.rb +6 -47
- data/lib/switchman/active_record/model_schema.rb +1 -1
- data/lib/switchman/active_record/persistence.rb +4 -6
- data/lib/switchman/active_record/postgresql_adapter.rb +124 -164
- data/lib/switchman/active_record/predicate_builder.rb +1 -1
- data/lib/switchman/active_record/query_cache.rb +18 -19
- data/lib/switchman/active_record/query_methods.rb +172 -181
- data/lib/switchman/active_record/reflection.rb +6 -10
- data/lib/switchman/active_record/relation.rb +27 -21
- data/lib/switchman/active_record/spawn_methods.rb +27 -29
- data/lib/switchman/active_record/statement_cache.rb +18 -35
- data/lib/switchman/active_record/tasks/database_tasks.rb +16 -0
- data/lib/switchman/active_support/cache.rb +3 -5
- data/lib/switchman/arel.rb +13 -8
- data/lib/switchman/database_server.rb +121 -142
- data/lib/switchman/default_shard.rb +52 -16
- data/lib/switchman/engine.rb +61 -57
- data/lib/switchman/environment.rb +4 -8
- data/lib/switchman/errors.rb +1 -0
- data/lib/switchman/guard_rail.rb +6 -19
- data/lib/switchman/guard_rail/relation.rb +5 -7
- data/lib/switchman/r_spec_helper.rb +29 -37
- data/lib/switchman/rails.rb +14 -12
- data/lib/switchman/schema_cache.rb +1 -9
- data/lib/switchman/sharded_instrumenter.rb +1 -1
- data/lib/switchman/standard_error.rb +15 -3
- data/lib/switchman/test_helper.rb +7 -11
- data/lib/switchman/version.rb +1 -1
- data/lib/tasks/switchman.rake +54 -69
- metadata +87 -45
- data/lib/switchman/active_record/batches.rb +0 -11
- data/lib/switchman/active_record/connection_handler.rb +0 -172
- data/lib/switchman/active_record/where_clause_factory.rb +0 -36
- data/lib/switchman/connection_pool_proxy.rb +0 -173
@@ -5,7 +5,7 @@ module Switchman
|
|
5
5
|
module Reflection
|
6
6
|
module AbstractReflection
|
7
7
|
def shard(owner)
|
8
|
-
if polymorphic? || klass.
|
8
|
+
if polymorphic? || klass.connection_classes == owner.class.connection_classes
|
9
9
|
# polymorphic associations assume the same shard as the owning item
|
10
10
|
owner.shard
|
11
11
|
else
|
@@ -28,21 +28,17 @@ module Switchman
|
|
28
28
|
# this technically belongs on AssociationReflection, but we put it on
|
29
29
|
# ThroughReflection as well, instead of delegating to its internal
|
30
30
|
# HasManyAssociation, losing its proper `klass`
|
31
|
-
def association_scope_cache(
|
32
|
-
key =
|
33
|
-
if polymorphic?
|
34
|
-
key = [key, owner._read_attribute(@foreign_type)]
|
35
|
-
end
|
31
|
+
def association_scope_cache(klass, owner, &block)
|
32
|
+
key = self
|
33
|
+
key = [key, owner._read_attribute(@foreign_type)] if polymorphic?
|
36
34
|
key = [key, shard(owner).id].flatten
|
37
|
-
|
38
|
-
@association_scope_cache[key] ||= (::Rails.version >= "5.2" ? ::ActiveRecord::StatementCache.create(conn, &block) : block.call)
|
39
|
-
}
|
35
|
+
klass.cached_find_by_statement(key, &block)
|
40
36
|
end
|
41
37
|
end
|
42
38
|
|
43
39
|
module AssociationReflection
|
44
40
|
def join_id_for(owner)
|
45
|
-
owner.send(
|
41
|
+
owner.send(join_foreign_key) # use sharded id values in association binds
|
46
42
|
end
|
47
43
|
end
|
48
44
|
end
|
@@ -4,53 +4,54 @@ module Switchman
|
|
4
4
|
module ActiveRecord
|
5
5
|
module Relation
|
6
6
|
def self.prepended(klass)
|
7
|
-
klass::SINGLE_VALUE_METHODS.concat [
|
7
|
+
klass::SINGLE_VALUE_METHODS.concat %i[shard shard_source]
|
8
8
|
end
|
9
9
|
|
10
10
|
def initialize(*, **)
|
11
11
|
super
|
12
|
-
self.shard_value = Shard.current(klass ? klass.
|
12
|
+
self.shard_value = Shard.current(klass ? klass.connection_classes : :primary) unless shard_value
|
13
13
|
self.shard_source_value = :implicit unless shard_source_value
|
14
14
|
end
|
15
15
|
|
16
16
|
def clone
|
17
17
|
result = super
|
18
|
-
result.shard_value = Shard.current(klass ? klass.
|
18
|
+
result.shard_value = Shard.current(klass ? klass.connection_classes : :primary) unless shard_value
|
19
19
|
result
|
20
20
|
end
|
21
21
|
|
22
22
|
def merge(*)
|
23
23
|
relation = super
|
24
|
-
if relation.shard_value !=
|
25
|
-
relation.shard_value =
|
26
|
-
relation.shard_source_value =
|
24
|
+
if relation.shard_value != shard_value && relation.shard_source_value == :implicit
|
25
|
+
relation.shard_value = shard_value
|
26
|
+
relation.shard_source_value = shard_source_value
|
27
27
|
end
|
28
28
|
relation
|
29
29
|
end
|
30
30
|
|
31
31
|
def new(*, &block)
|
32
|
-
primary_shard.activate(klass.
|
32
|
+
primary_shard.activate(klass.connection_classes) { super }
|
33
33
|
end
|
34
34
|
|
35
35
|
def create(*, &block)
|
36
|
-
primary_shard.activate(klass.
|
36
|
+
primary_shard.activate(klass.connection_classes) { super }
|
37
37
|
end
|
38
38
|
|
39
39
|
def create!(*, &block)
|
40
|
-
primary_shard.activate(klass.
|
40
|
+
primary_shard.activate(klass.connection_classes) { super }
|
41
41
|
end
|
42
42
|
|
43
43
|
def to_sql
|
44
|
-
primary_shard.activate(klass.
|
44
|
+
primary_shard.activate(klass.connection_classes) { super }
|
45
45
|
end
|
46
46
|
|
47
47
|
def explain
|
48
|
-
|
48
|
+
activate { |relation| relation.call_super(:explain, Relation) }
|
49
49
|
end
|
50
50
|
|
51
51
|
def records
|
52
52
|
return @records if loaded?
|
53
|
-
|
53
|
+
|
54
|
+
results = activate { |relation| relation.call_super(:records, Relation) }
|
54
55
|
case shard_value
|
55
56
|
when Array, ::ActiveRecord::Relation, ::ActiveRecord::Base
|
56
57
|
@records = results
|
@@ -59,7 +60,7 @@ module Switchman
|
|
59
60
|
results
|
60
61
|
end
|
61
62
|
|
62
|
-
%I
|
63
|
+
%I[update_all delete_all].each do |method|
|
63
64
|
class_eval <<-RUBY, __FILE__, __LINE__ + 1
|
64
65
|
def #{method}(*args)
|
65
66
|
result = self.activate { |relation| relation.call_super(#{method.inspect}, Relation, *args) }
|
@@ -74,15 +75,20 @@ module Switchman
|
|
74
75
|
loose_mode = options[:loose] && is_integer
|
75
76
|
# loose_mode: if we don't care about getting exactly batch_size ids in between
|
76
77
|
# don't get the max - just get the min and add batch_size so we get that many _at most_
|
77
|
-
values = loose_mode ?
|
78
|
+
values = loose_mode ? 'MIN(id)' : 'MIN(id), MAX(id)'
|
78
79
|
|
79
80
|
batch_size = options[:batch_size].try(:to_i) || 1000
|
80
81
|
quoted_primary_key = "#{klass.connection.quote_local_table_name(table_name)}.#{klass.connection.quote_column_name(primary_key)}"
|
81
|
-
as_id =
|
82
|
+
as_id = ' AS id' unless primary_key == 'id'
|
82
83
|
subquery_scope = except(:select).select("#{quoted_primary_key}#{as_id}").reorder(primary_key.to_sym).limit(loose_mode ? 1 : batch_size)
|
83
84
|
subquery_scope = subquery_scope.where("#{quoted_primary_key} <= ?", options[:end_at]) if options[:end_at]
|
84
85
|
|
85
|
-
first_subquery_scope =
|
86
|
+
first_subquery_scope = if options[:start_at]
|
87
|
+
subquery_scope.where("#{quoted_primary_key} >= ?",
|
88
|
+
options[:start_at])
|
89
|
+
else
|
90
|
+
subquery_scope
|
91
|
+
end
|
86
92
|
|
87
93
|
ids = connection.select_rows("SELECT #{values} FROM (#{first_subquery_scope.to_sql}) AS subquery").first
|
88
94
|
|
@@ -99,16 +105,16 @@ module Switchman
|
|
99
105
|
|
100
106
|
def activate(&block)
|
101
107
|
shards = all_shards
|
102
|
-
if
|
103
|
-
if shards.first == DefaultShard || shards.first == Shard.current(klass.
|
108
|
+
if Array === shards && shards.length == 1
|
109
|
+
if shards.first == DefaultShard || shards.first == Shard.current(klass.connection_classes)
|
104
110
|
yield(self, shards.first)
|
105
111
|
else
|
106
|
-
shards.first.activate(klass.
|
112
|
+
shards.first.activate(klass.connection_classes) { yield(self, shards.first) }
|
107
113
|
end
|
108
114
|
else
|
109
115
|
# TODO: implement local limit to avoid querying extra shards
|
110
|
-
Shard.with_each_shard(shards, [klass.
|
111
|
-
shard(Shard.current(klass.
|
116
|
+
Shard.with_each_shard(shards, [klass.connection_classes]) do
|
117
|
+
shard(Shard.current(klass.connection_classes), :to_a).activate(&block)
|
112
118
|
end
|
113
119
|
end
|
114
120
|
end
|
@@ -3,27 +3,27 @@
|
|
3
3
|
module Switchman
|
4
4
|
module ActiveRecord
|
5
5
|
module SpawnMethods
|
6
|
-
def shard_values_for_merge(
|
7
|
-
if shard_value !=
|
8
|
-
if
|
6
|
+
def shard_values_for_merge(rhs)
|
7
|
+
if shard_value != rhs.shard_value
|
8
|
+
if rhs.shard_source_value == :implicit
|
9
9
|
final_shard_value = shard_value
|
10
10
|
final_primary_shard = primary_shard
|
11
11
|
final_shard_source_value = shard_source_value
|
12
12
|
elsif shard_source_value == :implicit
|
13
|
-
final_shard_value =
|
14
|
-
final_primary_shard =
|
15
|
-
final_shard_source_value =
|
13
|
+
final_shard_value = rhs.shard_value
|
14
|
+
final_primary_shard = rhs.primary_shard
|
15
|
+
final_shard_source_value = rhs.shard_source_value
|
16
16
|
else
|
17
|
-
final_shard_source_value = [
|
18
|
-
shard_source_value == source_value ||
|
17
|
+
final_shard_source_value = %i[explicit association].detect do |source_value|
|
18
|
+
shard_source_value == source_value || rhs.shard_source_value == source_value
|
19
19
|
end
|
20
|
-
raise
|
20
|
+
raise 'unknown shard_source_value' unless final_shard_source_value
|
21
21
|
|
22
22
|
# have to merge shard_value
|
23
23
|
lhs_shard_value = all_shards
|
24
|
-
rhs_shard_value =
|
25
|
-
if
|
26
|
-
|
24
|
+
rhs_shard_value = rhs.all_shards
|
25
|
+
if ::ActiveRecord::Relation === lhs_shard_value &&
|
26
|
+
::ActiveRecord::Relation === rhs_shard_value
|
27
27
|
final_shard_value = lhs_shard_value.merge(rhs_shard_value)
|
28
28
|
final_primary_shard = Shard.default
|
29
29
|
else
|
@@ -32,27 +32,25 @@ module Switchman
|
|
32
32
|
final_shard_value = final_shard_value.first if final_shard_value.length == 1
|
33
33
|
end
|
34
34
|
end
|
35
|
-
elsif shard_source_value !=
|
36
|
-
final_shard_source_value = [
|
37
|
-
shard_source_value == source_value ||
|
35
|
+
elsif shard_source_value != rhs.shard_source_value
|
36
|
+
final_shard_source_value = %i[explicit association implicit].detect do |source_value|
|
37
|
+
shard_source_value == source_value || rhs.shard_source_value == source_value
|
38
38
|
end
|
39
|
-
raise
|
40
|
-
else
|
41
|
-
# nothing fancy
|
39
|
+
raise 'unknown shard_source_value' unless final_shard_source_value
|
42
40
|
end
|
43
41
|
|
44
42
|
[final_shard_value, final_primary_shard, final_shard_source_value]
|
45
43
|
end
|
46
44
|
|
47
|
-
def merge!(
|
48
|
-
return super unless ::ActiveRecord::Relation ===
|
45
|
+
def merge!(rhs)
|
46
|
+
return super unless ::ActiveRecord::Relation === rhs
|
49
47
|
|
50
48
|
# have to figure out shard stuff *before* conditions are merged
|
51
|
-
final_shard_value, final_primary_shard, final_shard_source_value = shard_values_for_merge(
|
49
|
+
final_shard_value, final_primary_shard, final_shard_source_value = shard_values_for_merge(rhs)
|
52
50
|
|
53
51
|
return super unless final_shard_source_value
|
54
52
|
|
55
|
-
|
53
|
+
unless final_shard_value
|
56
54
|
super
|
57
55
|
self.shard_source_value = final_shard_source_value
|
58
56
|
return self
|
@@ -61,16 +59,16 @@ module Switchman
|
|
61
59
|
return none! if final_shard_value == []
|
62
60
|
|
63
61
|
# change the primary shard if necessary before merging
|
64
|
-
if primary_shard != final_primary_shard &&
|
62
|
+
if primary_shard != final_primary_shard && rhs.primary_shard != final_primary_shard
|
65
63
|
shard!(final_primary_shard)
|
66
|
-
|
67
|
-
super(
|
64
|
+
rhs = rhs.shard(final_primary_shard)
|
65
|
+
super(rhs)
|
68
66
|
elsif primary_shard != final_primary_shard
|
69
67
|
shard!(final_primary_shard)
|
70
|
-
super(
|
71
|
-
elsif
|
72
|
-
|
73
|
-
super(
|
68
|
+
super(rhs)
|
69
|
+
elsif rhs.primary_shard != final_primary_shard
|
70
|
+
rhs = rhs.shard(final_primary_shard)
|
71
|
+
super(rhs)
|
74
72
|
else
|
75
73
|
super
|
76
74
|
end
|
@@ -7,18 +7,13 @@ module Switchman
|
|
7
7
|
def create(connection, &block)
|
8
8
|
relation = block.call ::ActiveRecord::StatementCache::Params.new
|
9
9
|
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
new(relation.arel, bind_map, relation.klass)
|
14
|
-
else
|
15
|
-
bind_map = ::ActiveRecord::StatementCache::BindMap.new(relation.bound_attributes)
|
16
|
-
new relation.arel, bind_map
|
17
|
-
end
|
10
|
+
_query_builder, binds = connection.cacheable_query(self, relation.arel)
|
11
|
+
bind_map = ::ActiveRecord::StatementCache::BindMap.new(binds)
|
12
|
+
new(relation.arel, bind_map, relation.klass)
|
18
13
|
end
|
19
14
|
end
|
20
15
|
|
21
|
-
def initialize(arel, bind_map, klass=nil)
|
16
|
+
def initialize(arel, bind_map, klass = nil)
|
22
17
|
@arel = arel
|
23
18
|
@bind_map = bind_map
|
24
19
|
@klass = klass
|
@@ -31,49 +26,39 @@ module Switchman
|
|
31
26
|
# (e.g. infer from the primary key or use the current shard)
|
32
27
|
|
33
28
|
def execute(*args)
|
34
|
-
|
35
|
-
|
36
|
-
klass = @klass
|
37
|
-
else
|
38
|
-
params, klass, connection = args
|
39
|
-
end
|
29
|
+
params, connection = args
|
30
|
+
klass = @klass
|
40
31
|
target_shard = nil
|
41
|
-
if primary_index = bind_map.primary_value_index
|
32
|
+
if (primary_index = bind_map.primary_value_index)
|
42
33
|
primary_value = params[primary_index]
|
43
34
|
target_shard = Shard.local_id_for(primary_value)[1]
|
44
35
|
end
|
45
|
-
current_shard = Shard.current(klass.
|
36
|
+
current_shard = Shard.current(klass.connection_classes)
|
46
37
|
target_shard ||= current_shard
|
47
38
|
|
48
39
|
bind_values = bind_map.bind(params, current_shard, target_shard)
|
49
40
|
|
50
|
-
target_shard.activate(klass.
|
41
|
+
target_shard.activate(klass.connection_classes) do
|
51
42
|
sql = qualified_query_builder(target_shard, klass).sql_for(bind_values, connection)
|
52
43
|
klass.find_by_sql(sql, bind_values)
|
53
44
|
end
|
54
45
|
end
|
55
46
|
|
56
|
-
|
57
|
-
|
58
|
-
@qualified_query_builders[shard.id] ||= klass.connection.cacheable_query(self.class, @arel)
|
59
|
-
end
|
60
|
-
else
|
61
|
-
def qualified_query_builder(shard, klass)
|
62
|
-
@qualified_query_builders[shard.id] ||= klass.connection.cacheable_query(self.class, @arel).first
|
63
|
-
end
|
47
|
+
def qualified_query_builder(shard, klass)
|
48
|
+
@qualified_query_builders[shard.id] ||= klass.connection.cacheable_query(self.class, @arel).first
|
64
49
|
end
|
65
50
|
|
66
51
|
module BindMap
|
67
52
|
# performs id transposition here instead of query_methods.rb
|
68
53
|
def bind(values, current_shard, target_shard)
|
69
54
|
bas = @bound_attributes.dup
|
70
|
-
@indexes.each_with_index do |offset,i|
|
55
|
+
@indexes.each_with_index do |offset, i|
|
71
56
|
ba = bas[offset]
|
72
|
-
if ba.is_a?(::ActiveRecord::Relation::QueryAttribute) && ba.value.sharded
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
57
|
+
new_value = if ba.is_a?(::ActiveRecord::Relation::QueryAttribute) && ba.value.sharded
|
58
|
+
Shard.relative_id_for(values[i], current_shard, target_shard || current_shard)
|
59
|
+
else
|
60
|
+
values[i]
|
61
|
+
end
|
77
62
|
bas[offset] = ba.with_cast_value(new_value)
|
78
63
|
end
|
79
64
|
bas
|
@@ -83,9 +68,7 @@ module Switchman
|
|
83
68
|
primary_ba_index = @bound_attributes.index do |ba|
|
84
69
|
ba.is_a?(::ActiveRecord::Relation::QueryAttribute) && ba.value.primary
|
85
70
|
end
|
86
|
-
if primary_ba_index
|
87
|
-
@indexes.index(primary_ba_index)
|
88
|
-
end
|
71
|
+
@indexes.index(primary_ba_index) if primary_ba_index
|
89
72
|
end
|
90
73
|
end
|
91
74
|
end
|
@@ -0,0 +1,16 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Switchman
|
4
|
+
module ActiveRecord
|
5
|
+
module Tasks
|
6
|
+
module DatabaseTasks
|
7
|
+
def drop(*)
|
8
|
+
super
|
9
|
+
# no really, it's gone
|
10
|
+
Switchman.cache.delete('default_shard')
|
11
|
+
Shard.default(reload: true)
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
@@ -8,10 +8,8 @@ module Switchman
|
|
8
8
|
store = super
|
9
9
|
# can't use defined?, because it's a _ruby_ autoloaded constant,
|
10
10
|
# so just checking that will cause it to get required
|
11
|
-
if store.
|
12
|
-
|
13
|
-
end
|
14
|
-
store.options[:namespace] ||= lambda { Shard.current.default? ? nil : "shard_#{Shard.current.id}" }
|
11
|
+
::ActiveSupport::Cache::RedisCacheStore.prepend(RedisCacheStore) if store.instance_of?(ActiveSupport::Cache::RedisCacheStore) && !::ActiveSupport::Cache::RedisCacheStore.ancestors.include?(RedisCacheStore)
|
12
|
+
store.options[:namespace] ||= -> { Shard.current.default? ? nil : "shard_#{Shard.current.id}" }
|
15
13
|
store
|
16
14
|
end
|
17
15
|
end
|
@@ -22,7 +20,7 @@ module Switchman
|
|
22
20
|
# unfortunately, it uses the keys command, which is extraordinarily inefficient in a large redis instance
|
23
21
|
# fortunately, we can assume we control the entire instance, because we set up the namespacing, so just
|
24
22
|
# always unset it temporarily for clear calls
|
25
|
-
namespace = nil
|
23
|
+
namespace = nil # rubocop:disable Lint/ShadowedArgument
|
26
24
|
super
|
27
25
|
end
|
28
26
|
end
|
data/lib/switchman/arel.rb
CHANGED
@@ -3,29 +3,34 @@
|
|
3
3
|
module Switchman
|
4
4
|
module Arel
|
5
5
|
module Table
|
6
|
-
def
|
7
|
-
|
6
|
+
def klass
|
7
|
+
@klass || ::ActiveRecord::Base
|
8
8
|
end
|
9
9
|
end
|
10
|
+
|
10
11
|
module Visitors
|
11
12
|
module ToSql
|
12
|
-
|
13
|
+
# rubocop:disable Naming/MethodName
|
14
|
+
|
15
|
+
def visit_Arel_Nodes_TableAlias(*args)
|
13
16
|
o, collector = args
|
14
17
|
collector = visit o.relation, collector
|
15
|
-
collector <<
|
18
|
+
collector << ' '
|
16
19
|
collector << quote_local_table_name(o.name)
|
17
20
|
end
|
18
21
|
|
19
|
-
def visit_Arel_Attributes_Attribute
|
22
|
+
def visit_Arel_Attributes_Attribute(*args)
|
20
23
|
o = args.first
|
21
24
|
join_name = o.relation.table_alias || o.relation.name
|
22
25
|
result = "#{quote_local_table_name join_name}.#{quote_column_name o.name}"
|
23
|
-
|
24
|
-
result
|
26
|
+
args.last << result
|
25
27
|
end
|
26
28
|
|
27
|
-
|
29
|
+
# rubocop:enable Naming/MethodName
|
30
|
+
|
31
|
+
def quote_local_table_name(name)
|
28
32
|
return name if ::Arel::Nodes::SqlLiteral === name
|
33
|
+
|
29
34
|
@connection.quote_local_table_name(name)
|
30
35
|
end
|
31
36
|
end
|
@@ -1,6 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
3
|
+
require 'securerandom'
|
4
4
|
|
5
5
|
module Switchman
|
6
6
|
class DatabaseServer
|
@@ -13,16 +13,22 @@ module Switchman
|
|
13
13
|
database_servers.values
|
14
14
|
end
|
15
15
|
|
16
|
+
def all_roles
|
17
|
+
@all_roles ||= all.map(&:roles).flatten.uniq
|
18
|
+
end
|
19
|
+
|
16
20
|
def find(id_or_all)
|
17
|
-
return
|
18
|
-
return id_or_all.map { |id|
|
21
|
+
return all if id_or_all == :all
|
22
|
+
return id_or_all.map { |id| database_servers[id || ::Rails.env] }.compact.uniq if id_or_all.is_a?(Array)
|
23
|
+
|
19
24
|
database_servers[id_or_all || ::Rails.env]
|
20
25
|
end
|
21
26
|
|
22
27
|
def create(settings = {})
|
23
|
-
raise
|
28
|
+
raise 'database servers should be set up in database.yml' unless ::Rails.env.test?
|
29
|
+
|
24
30
|
id = settings[:id]
|
25
|
-
|
31
|
+
unless id
|
26
32
|
@id ||= 0
|
27
33
|
@id += 1
|
28
34
|
id = @id
|
@@ -30,25 +36,42 @@ module Switchman
|
|
30
36
|
server = DatabaseServer.new(id.to_s, settings)
|
31
37
|
server.instance_variable_set(:@fake, true)
|
32
38
|
database_servers[server.id] = server
|
39
|
+
::ActiveRecord::Base.configurations.configurations <<
|
40
|
+
::ActiveRecord::DatabaseConfigurations::HashConfig.new(::Rails.env, "#{server.id}/primary", settings)
|
41
|
+
Shard.send(:initialize_sharding)
|
42
|
+
server
|
33
43
|
end
|
34
44
|
|
35
45
|
def server_for_new_shard
|
36
46
|
servers = all.select { |s| s.config[:open] }
|
37
47
|
return find(nil) if servers.empty?
|
48
|
+
|
38
49
|
servers[rand(servers.length)]
|
39
50
|
end
|
40
51
|
|
41
52
|
private
|
53
|
+
|
54
|
+
def reference_role(role)
|
55
|
+
return if all_roles.include?(role)
|
56
|
+
|
57
|
+
@all_roles << role
|
58
|
+
Shard.send(:initialize_sharding)
|
59
|
+
end
|
60
|
+
|
42
61
|
def database_servers
|
43
62
|
unless @database_servers
|
44
63
|
@database_servers = {}.with_indifferent_access
|
45
|
-
|
46
|
-
|
47
|
-
|
64
|
+
::ActiveRecord::Base.configurations.configurations.each do |config|
|
65
|
+
if config.name.include?('/')
|
66
|
+
name, role = config.name.split('/')
|
67
|
+
else
|
68
|
+
name, role = config.env_name, config.name
|
48
69
|
end
|
49
|
-
|
50
|
-
|
51
|
-
@database_servers[
|
70
|
+
|
71
|
+
if role == 'primary'
|
72
|
+
@database_servers[name] = DatabaseServer.new(config.env_name, config.configuration_hash)
|
73
|
+
else
|
74
|
+
@database_servers[name].roles << role
|
52
75
|
end
|
53
76
|
end
|
54
77
|
end
|
@@ -56,15 +79,33 @@ module Switchman
|
|
56
79
|
end
|
57
80
|
end
|
58
81
|
|
82
|
+
attr_reader :roles
|
83
|
+
|
59
84
|
def initialize(id = nil, config = {})
|
60
85
|
@id = id
|
61
86
|
@config = config.deep_symbolize_keys
|
62
87
|
@configs = {}
|
88
|
+
@roles = [:primary]
|
89
|
+
end
|
90
|
+
|
91
|
+
def connects_to_hash
|
92
|
+
self.class.all_roles.map do |role|
|
93
|
+
config_role = role
|
94
|
+
config_role = :primary unless roles.include?(role)
|
95
|
+
config_name = :"#{id}/#{config_role}"
|
96
|
+
config_name = :primary if id == ::Rails.env && config_role == :primary
|
97
|
+
[role.to_sym, config_name]
|
98
|
+
end.to_h
|
63
99
|
end
|
64
100
|
|
65
101
|
def destroy
|
66
|
-
|
67
|
-
|
102
|
+
self.class.send(:database_servers).delete(id) if id
|
103
|
+
Shard.sharded_models.each do |klass|
|
104
|
+
self.class.all_roles.each do |role|
|
105
|
+
klass.connection_handler.remove_connection_pool(klass.connection_specification_name, role: role,
|
106
|
+
shard: id.to_sym)
|
107
|
+
end
|
108
|
+
end
|
68
109
|
end
|
69
110
|
|
70
111
|
def fake?
|
@@ -73,14 +114,15 @@ module Switchman
|
|
73
114
|
|
74
115
|
def config(environment = :primary)
|
75
116
|
@configs[environment] ||= begin
|
76
|
-
|
117
|
+
case @config[environment]
|
118
|
+
when Array
|
77
119
|
@config[environment].map do |config|
|
78
120
|
config = @config.merge((config || {}).symbolize_keys)
|
79
121
|
# make sure GuardRail doesn't get any brilliant ideas about choosing the first possible server
|
80
122
|
config.delete(environment)
|
81
123
|
config
|
82
124
|
end
|
83
|
-
|
125
|
+
when Hash
|
84
126
|
@config.merge(@config[environment])
|
85
127
|
else
|
86
128
|
@config
|
@@ -111,172 +153,109 @@ module Switchman
|
|
111
153
|
guard!(old_env)
|
112
154
|
end
|
113
155
|
|
114
|
-
def shareable?
|
115
|
-
@shareable_environment_key ||= []
|
116
|
-
environment = guard_rail_environment
|
117
|
-
explicit_user = ::GuardRail.global_config[:username]
|
118
|
-
return @shareable if @shareable_environment_key == [environment, explicit_user]
|
119
|
-
@shareable_environment_key = [environment, explicit_user]
|
120
|
-
if explicit_user
|
121
|
-
username = explicit_user
|
122
|
-
else
|
123
|
-
config = self.config(environment)
|
124
|
-
config = config.first if config.is_a?(Array)
|
125
|
-
username = config[:username]
|
126
|
-
end
|
127
|
-
@shareable = username !~ /%?\{[a-zA-Z0-9_]+\}/
|
128
|
-
end
|
129
|
-
|
130
156
|
def shards
|
131
|
-
if
|
132
|
-
Shard.where(
|
157
|
+
if id == ::Rails.env
|
158
|
+
Shard.where('database_server_id IS NULL OR database_server_id=?', id)
|
133
159
|
else
|
134
|
-
Shard.where(:
|
160
|
+
Shard.where(database_server_id: id)
|
135
161
|
end
|
136
162
|
end
|
137
163
|
|
138
|
-
def
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
def create_new_shard(options = {})
|
143
|
-
raise NotImplementedError.new("Cannot create new shards when sharding isn't initialized") unless Shard.default.is_a?(Shard)
|
144
|
-
|
145
|
-
name = options[:name]
|
146
|
-
create_schema = options[:schema]
|
147
|
-
# look for another shard associated with this db
|
148
|
-
other_shard = self.shards.where("name<>':memory:' OR name IS NULL").order(:id).first
|
149
|
-
|
150
|
-
case config[:adapter]
|
151
|
-
when 'postgresql'
|
152
|
-
create_statement = lambda { "CREATE SCHEMA #{name}" }
|
153
|
-
password = " PASSWORD #{::ActiveRecord::Base.connection.quote(config[:password])}" if config[:password]
|
154
|
-
else
|
155
|
-
create_statement = lambda { "CREATE DATABASE #{name}" }
|
164
|
+
def create_new_shard(id: nil, name: nil, schema: true)
|
165
|
+
unless Shard.default.is_a?(Shard)
|
166
|
+
raise NotImplementedError,
|
167
|
+
"Cannot create new shards when sharding isn't initialized"
|
156
168
|
end
|
169
|
+
|
170
|
+
create_statement = -> { "CREATE SCHEMA #{name}" }
|
171
|
+
password = " PASSWORD #{::ActiveRecord::Base.connection.quote(config[:password])}" if config[:password]
|
157
172
|
sharding_config = Switchman.config
|
158
173
|
config_create_statement = sharding_config[config[:adapter]]&.[](:create_statement)
|
159
174
|
config_create_statement ||= sharding_config[:create_statement]
|
160
175
|
if config_create_statement
|
161
176
|
create_commands = Array(config_create_statement).dup
|
162
177
|
create_statement = lambda {
|
163
|
-
create_commands.map { |statement| statement
|
178
|
+
create_commands.map { |statement| format(statement, name: name, password: password) }
|
164
179
|
}
|
165
180
|
end
|
166
181
|
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
next_id = Shard.connection.select_value("SELECT nextval(#{id_seq})")
|
173
|
-
next_id.to_i
|
174
|
-
else
|
175
|
-
nil
|
176
|
-
end
|
177
|
-
end
|
182
|
+
id ||= begin
|
183
|
+
id_seq = Shard.connection.quote(Shard.connection.quote_table_name('switchman_shards_id_seq'))
|
184
|
+
next_id = Shard.connection.select_value("SELECT nextval(#{id_seq})")
|
185
|
+
next_id.to_i
|
186
|
+
end
|
178
187
|
|
179
|
-
|
180
|
-
base_name = self.config[:database].to_s % self.config
|
181
|
-
base_name = nil if base_name == ':memory:'
|
182
|
-
base_name << '_' if base_name
|
183
|
-
base_id = shard_id || SecureRandom.uuid
|
184
|
-
name = "#{base_name}shard_#{base_id}"
|
185
|
-
end
|
188
|
+
name ||= "#{config[:database]}_shard_#{id}"
|
186
189
|
|
187
|
-
|
188
|
-
|
189
|
-
:
|
190
|
+
Shard.connection.transaction do
|
191
|
+
shard = Shard.create!(id: id,
|
192
|
+
name: name,
|
193
|
+
database_server_id: self.id)
|
190
194
|
schema_already_existed = false
|
191
195
|
|
192
196
|
begin
|
193
197
|
self.class.creating_new_shard = true
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
migrate = ::Rails.version >= '5.2' ?
|
220
|
-
-> { ::ActiveRecord::Base.connection.migration_context.migrate } :
|
221
|
-
-> { ::ActiveRecord::Migrator.migrate(::ActiveRecord::Migrator.migrations_paths) }
|
222
|
-
if ::ActiveRecord::Base.connection.supports_ddl_transactions?
|
223
|
-
::ActiveRecord::Base.connection.transaction(requires_new: true, &migrate)
|
224
|
-
else
|
225
|
-
migrate.call
|
226
|
-
end
|
227
|
-
reset_column_information
|
228
|
-
::ActiveRecord::Base.descendants.reject { |m| m == Shard || !m.table_exists? }.each(&:define_attribute_methods)
|
198
|
+
DatabaseServer.send(:reference_role, :deploy)
|
199
|
+
::ActiveRecord::Base.connected_to(shard: self.id.to_sym, role: :deploy) do
|
200
|
+
if create_statement
|
201
|
+
if ::ActiveRecord::Base.connection.select_value("SELECT 1 FROM pg_namespace WHERE nspname=#{::ActiveRecord::Base.connection.quote(name)}")
|
202
|
+
schema_already_existed = true
|
203
|
+
raise 'This schema already exists; cannot overwrite'
|
204
|
+
end
|
205
|
+
Array(create_statement.call).each do |stmt|
|
206
|
+
::ActiveRecord::Base.connection.execute(stmt)
|
207
|
+
end
|
208
|
+
end
|
209
|
+
if config[:adapter] == 'postgresql'
|
210
|
+
old_proc = ::ActiveRecord::Base.connection.raw_connection.set_notice_processor do
|
211
|
+
end
|
212
|
+
end
|
213
|
+
old_verbose = ::ActiveRecord::Migration.verbose
|
214
|
+
::ActiveRecord::Migration.verbose = false
|
215
|
+
|
216
|
+
unless schema == false
|
217
|
+
shard.activate do
|
218
|
+
reset_column_information
|
219
|
+
|
220
|
+
::ActiveRecord::Base.connection.transaction(requires_new: true) do
|
221
|
+
::ActiveRecord::Base.connection.migration_context.migrate
|
229
222
|
end
|
230
|
-
|
231
|
-
::ActiveRecord::
|
232
|
-
|
223
|
+
reset_column_information
|
224
|
+
::ActiveRecord::Base.descendants.reject do |m|
|
225
|
+
m <= UnshardedRecord || !m.table_exists?
|
226
|
+
end.each(&:define_attribute_methods)
|
233
227
|
end
|
234
228
|
end
|
229
|
+
ensure
|
230
|
+
::ActiveRecord::Migration.verbose = old_verbose
|
231
|
+
::ActiveRecord::Base.connection.raw_connection.set_notice_processor(&old_proc) if old_proc
|
235
232
|
end
|
236
233
|
shard
|
237
234
|
rescue
|
238
235
|
shard.destroy
|
239
|
-
unless schema_already_existed
|
240
|
-
|
241
|
-
end
|
242
|
-
reset_column_information unless create_schema == false rescue nil
|
236
|
+
shard.drop_database rescue nil unless schema_already_existed
|
237
|
+
reset_column_information unless schema == false rescue nil
|
243
238
|
raise
|
244
239
|
ensure
|
245
240
|
self.class.creating_new_shard = false
|
246
241
|
end
|
247
242
|
end
|
248
|
-
|
249
|
-
if Shard.connection.supports_ddl_transactions? && self.shareable? && other_shard
|
250
|
-
Shard.transaction do
|
251
|
-
other_shard.activate do
|
252
|
-
::ActiveRecord::Base.connection.transaction(&create_shard)
|
253
|
-
end
|
254
|
-
end
|
255
|
-
else
|
256
|
-
create_shard.call
|
257
|
-
end
|
258
243
|
end
|
259
244
|
|
260
245
|
def cache_store
|
261
|
-
|
262
|
-
@cache_store = Switchman.config[:cache_map][self.id] || Switchman.config[:cache_map][::Rails.env]
|
263
|
-
end
|
246
|
+
@cache_store ||= Switchman.config[:cache_map][id] || Switchman.config[:cache_map][::Rails.env]
|
264
247
|
@cache_store
|
265
248
|
end
|
266
249
|
|
267
250
|
def shard_name(shard)
|
268
|
-
if config[:shard_name]
|
269
|
-
|
270
|
-
|
271
|
-
if
|
272
|
-
|
273
|
-
|
274
|
-
::ActiveRecord::Base.connection.current_schemas.first rescue nil
|
275
|
-
else
|
276
|
-
shard.activate { ::ActiveRecord::Base.connection_pool.default_schema }
|
277
|
-
end
|
251
|
+
return config[:shard_name] if config[:shard_name]
|
252
|
+
|
253
|
+
if shard == :bootstrap
|
254
|
+
# rescue nil because the database may not exist yet; if it doesn't,
|
255
|
+
# it will shortly, and this will be re-invoked
|
256
|
+
::ActiveRecord::Base.connection.current_schemas.first rescue nil
|
278
257
|
else
|
279
|
-
|
258
|
+
shard.activate { ::ActiveRecord::Base.connection_pool.default_schema }
|
280
259
|
end
|
281
260
|
end
|
282
261
|
|
@@ -290,9 +269,9 @@ module Switchman
|
|
290
269
|
end
|
291
270
|
|
292
271
|
private
|
272
|
+
|
293
273
|
def reset_column_information
|
294
|
-
::ActiveRecord::Base.descendants.reject { |m| m
|
295
|
-
::ActiveRecord::Base.connection_handler.switchman_connection_pool_proxies.each { |pool| pool.schema_cache.clear! }
|
274
|
+
::ActiveRecord::Base.descendants.reject { |m| m <= UnshardedRecord }.each(&:reset_column_information)
|
296
275
|
end
|
297
276
|
end
|
298
277
|
end
|