switchman 2.0.9 → 3.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Rakefile +10 -2
- data/app/models/switchman/shard.rb +234 -271
- data/app/models/switchman/unsharded_record.rb +7 -0
- data/db/migrate/20130328212039_create_switchman_shards.rb +1 -1
- data/db/migrate/20130328224244_create_default_shard.rb +5 -5
- data/db/migrate/20161206323434_add_back_default_string_limits_switchman.rb +1 -0
- data/db/migrate/20180828183945_add_default_shard_index.rb +2 -2
- data/db/migrate/20180828192111_add_timestamps_to_shards.rb +7 -5
- data/db/migrate/20190114212900_add_unique_name_indexes.rb +5 -3
- data/lib/switchman.rb +3 -3
- data/lib/switchman/action_controller/caching.rb +2 -2
- data/lib/switchman/active_record/abstract_adapter.rb +1 -0
- data/lib/switchman/active_record/association.rb +78 -89
- data/lib/switchman/active_record/attribute_methods.rb +58 -52
- data/lib/switchman/active_record/base.rb +58 -59
- data/lib/switchman/active_record/calculations.rb +73 -66
- data/lib/switchman/active_record/connection_pool.rb +14 -41
- data/lib/switchman/active_record/database_configurations.rb +34 -0
- data/lib/switchman/active_record/database_configurations/database_config.rb +13 -0
- data/lib/switchman/active_record/finder_methods.rb +11 -16
- data/lib/switchman/active_record/log_subscriber.rb +4 -8
- data/lib/switchman/active_record/migration.rb +6 -47
- data/lib/switchman/active_record/model_schema.rb +1 -1
- data/lib/switchman/active_record/persistence.rb +4 -6
- data/lib/switchman/active_record/postgresql_adapter.rb +124 -164
- data/lib/switchman/active_record/predicate_builder.rb +1 -1
- data/lib/switchman/active_record/query_cache.rb +18 -19
- data/lib/switchman/active_record/query_methods.rb +172 -181
- data/lib/switchman/active_record/reflection.rb +6 -10
- data/lib/switchman/active_record/relation.rb +27 -21
- data/lib/switchman/active_record/spawn_methods.rb +27 -29
- data/lib/switchman/active_record/statement_cache.rb +18 -35
- data/lib/switchman/active_record/tasks/database_tasks.rb +16 -0
- data/lib/switchman/active_support/cache.rb +3 -5
- data/lib/switchman/arel.rb +13 -8
- data/lib/switchman/database_server.rb +121 -142
- data/lib/switchman/default_shard.rb +52 -16
- data/lib/switchman/engine.rb +61 -57
- data/lib/switchman/environment.rb +4 -8
- data/lib/switchman/errors.rb +1 -0
- data/lib/switchman/guard_rail.rb +6 -19
- data/lib/switchman/guard_rail/relation.rb +5 -7
- data/lib/switchman/r_spec_helper.rb +29 -37
- data/lib/switchman/rails.rb +14 -12
- data/lib/switchman/schema_cache.rb +1 -9
- data/lib/switchman/sharded_instrumenter.rb +1 -1
- data/lib/switchman/standard_error.rb +15 -3
- data/lib/switchman/test_helper.rb +7 -11
- data/lib/switchman/version.rb +1 -1
- data/lib/tasks/switchman.rake +54 -69
- metadata +87 -45
- data/lib/switchman/active_record/batches.rb +0 -11
- data/lib/switchman/active_record/connection_handler.rb +0 -172
- data/lib/switchman/active_record/where_clause_factory.rb +0 -36
- data/lib/switchman/connection_pool_proxy.rb +0 -173
@@ -5,7 +5,7 @@ module Switchman
|
|
5
5
|
module Reflection
|
6
6
|
module AbstractReflection
|
7
7
|
def shard(owner)
|
8
|
-
if polymorphic? || klass.
|
8
|
+
if polymorphic? || klass.connection_classes == owner.class.connection_classes
|
9
9
|
# polymorphic associations assume the same shard as the owning item
|
10
10
|
owner.shard
|
11
11
|
else
|
@@ -28,21 +28,17 @@ module Switchman
|
|
28
28
|
# this technically belongs on AssociationReflection, but we put it on
|
29
29
|
# ThroughReflection as well, instead of delegating to its internal
|
30
30
|
# HasManyAssociation, losing its proper `klass`
|
31
|
-
def association_scope_cache(
|
32
|
-
key =
|
33
|
-
if polymorphic?
|
34
|
-
key = [key, owner._read_attribute(@foreign_type)]
|
35
|
-
end
|
31
|
+
def association_scope_cache(klass, owner, &block)
|
32
|
+
key = self
|
33
|
+
key = [key, owner._read_attribute(@foreign_type)] if polymorphic?
|
36
34
|
key = [key, shard(owner).id].flatten
|
37
|
-
|
38
|
-
@association_scope_cache[key] ||= (::Rails.version >= "5.2" ? ::ActiveRecord::StatementCache.create(conn, &block) : block.call)
|
39
|
-
}
|
35
|
+
klass.cached_find_by_statement(key, &block)
|
40
36
|
end
|
41
37
|
end
|
42
38
|
|
43
39
|
module AssociationReflection
|
44
40
|
def join_id_for(owner)
|
45
|
-
owner.send(
|
41
|
+
owner.send(join_foreign_key) # use sharded id values in association binds
|
46
42
|
end
|
47
43
|
end
|
48
44
|
end
|
@@ -4,53 +4,54 @@ module Switchman
|
|
4
4
|
module ActiveRecord
|
5
5
|
module Relation
|
6
6
|
def self.prepended(klass)
|
7
|
-
klass::SINGLE_VALUE_METHODS.concat [
|
7
|
+
klass::SINGLE_VALUE_METHODS.concat %i[shard shard_source]
|
8
8
|
end
|
9
9
|
|
10
10
|
def initialize(*, **)
|
11
11
|
super
|
12
|
-
self.shard_value = Shard.current(klass ? klass.
|
12
|
+
self.shard_value = Shard.current(klass ? klass.connection_classes : :primary) unless shard_value
|
13
13
|
self.shard_source_value = :implicit unless shard_source_value
|
14
14
|
end
|
15
15
|
|
16
16
|
def clone
|
17
17
|
result = super
|
18
|
-
result.shard_value = Shard.current(klass ? klass.
|
18
|
+
result.shard_value = Shard.current(klass ? klass.connection_classes : :primary) unless shard_value
|
19
19
|
result
|
20
20
|
end
|
21
21
|
|
22
22
|
def merge(*)
|
23
23
|
relation = super
|
24
|
-
if relation.shard_value !=
|
25
|
-
relation.shard_value =
|
26
|
-
relation.shard_source_value =
|
24
|
+
if relation.shard_value != shard_value && relation.shard_source_value == :implicit
|
25
|
+
relation.shard_value = shard_value
|
26
|
+
relation.shard_source_value = shard_source_value
|
27
27
|
end
|
28
28
|
relation
|
29
29
|
end
|
30
30
|
|
31
31
|
def new(*, &block)
|
32
|
-
primary_shard.activate(klass.
|
32
|
+
primary_shard.activate(klass.connection_classes) { super }
|
33
33
|
end
|
34
34
|
|
35
35
|
def create(*, &block)
|
36
|
-
primary_shard.activate(klass.
|
36
|
+
primary_shard.activate(klass.connection_classes) { super }
|
37
37
|
end
|
38
38
|
|
39
39
|
def create!(*, &block)
|
40
|
-
primary_shard.activate(klass.
|
40
|
+
primary_shard.activate(klass.connection_classes) { super }
|
41
41
|
end
|
42
42
|
|
43
43
|
def to_sql
|
44
|
-
primary_shard.activate(klass.
|
44
|
+
primary_shard.activate(klass.connection_classes) { super }
|
45
45
|
end
|
46
46
|
|
47
47
|
def explain
|
48
|
-
|
48
|
+
activate { |relation| relation.call_super(:explain, Relation) }
|
49
49
|
end
|
50
50
|
|
51
51
|
def records
|
52
52
|
return @records if loaded?
|
53
|
-
|
53
|
+
|
54
|
+
results = activate { |relation| relation.call_super(:records, Relation) }
|
54
55
|
case shard_value
|
55
56
|
when Array, ::ActiveRecord::Relation, ::ActiveRecord::Base
|
56
57
|
@records = results
|
@@ -59,7 +60,7 @@ module Switchman
|
|
59
60
|
results
|
60
61
|
end
|
61
62
|
|
62
|
-
%I
|
63
|
+
%I[update_all delete_all].each do |method|
|
63
64
|
class_eval <<-RUBY, __FILE__, __LINE__ + 1
|
64
65
|
def #{method}(*args)
|
65
66
|
result = self.activate { |relation| relation.call_super(#{method.inspect}, Relation, *args) }
|
@@ -74,15 +75,20 @@ module Switchman
|
|
74
75
|
loose_mode = options[:loose] && is_integer
|
75
76
|
# loose_mode: if we don't care about getting exactly batch_size ids in between
|
76
77
|
# don't get the max - just get the min and add batch_size so we get that many _at most_
|
77
|
-
values = loose_mode ?
|
78
|
+
values = loose_mode ? 'MIN(id)' : 'MIN(id), MAX(id)'
|
78
79
|
|
79
80
|
batch_size = options[:batch_size].try(:to_i) || 1000
|
80
81
|
quoted_primary_key = "#{klass.connection.quote_local_table_name(table_name)}.#{klass.connection.quote_column_name(primary_key)}"
|
81
|
-
as_id =
|
82
|
+
as_id = ' AS id' unless primary_key == 'id'
|
82
83
|
subquery_scope = except(:select).select("#{quoted_primary_key}#{as_id}").reorder(primary_key.to_sym).limit(loose_mode ? 1 : batch_size)
|
83
84
|
subquery_scope = subquery_scope.where("#{quoted_primary_key} <= ?", options[:end_at]) if options[:end_at]
|
84
85
|
|
85
|
-
first_subquery_scope =
|
86
|
+
first_subquery_scope = if options[:start_at]
|
87
|
+
subquery_scope.where("#{quoted_primary_key} >= ?",
|
88
|
+
options[:start_at])
|
89
|
+
else
|
90
|
+
subquery_scope
|
91
|
+
end
|
86
92
|
|
87
93
|
ids = connection.select_rows("SELECT #{values} FROM (#{first_subquery_scope.to_sql}) AS subquery").first
|
88
94
|
|
@@ -99,16 +105,16 @@ module Switchman
|
|
99
105
|
|
100
106
|
def activate(&block)
|
101
107
|
shards = all_shards
|
102
|
-
if
|
103
|
-
if shards.first == DefaultShard || shards.first == Shard.current(klass.
|
108
|
+
if Array === shards && shards.length == 1
|
109
|
+
if shards.first == DefaultShard || shards.first == Shard.current(klass.connection_classes)
|
104
110
|
yield(self, shards.first)
|
105
111
|
else
|
106
|
-
shards.first.activate(klass.
|
112
|
+
shards.first.activate(klass.connection_classes) { yield(self, shards.first) }
|
107
113
|
end
|
108
114
|
else
|
109
115
|
# TODO: implement local limit to avoid querying extra shards
|
110
|
-
Shard.with_each_shard(shards, [klass.
|
111
|
-
shard(Shard.current(klass.
|
116
|
+
Shard.with_each_shard(shards, [klass.connection_classes]) do
|
117
|
+
shard(Shard.current(klass.connection_classes), :to_a).activate(&block)
|
112
118
|
end
|
113
119
|
end
|
114
120
|
end
|
@@ -3,27 +3,27 @@
|
|
3
3
|
module Switchman
|
4
4
|
module ActiveRecord
|
5
5
|
module SpawnMethods
|
6
|
-
def shard_values_for_merge(
|
7
|
-
if shard_value !=
|
8
|
-
if
|
6
|
+
def shard_values_for_merge(rhs)
|
7
|
+
if shard_value != rhs.shard_value
|
8
|
+
if rhs.shard_source_value == :implicit
|
9
9
|
final_shard_value = shard_value
|
10
10
|
final_primary_shard = primary_shard
|
11
11
|
final_shard_source_value = shard_source_value
|
12
12
|
elsif shard_source_value == :implicit
|
13
|
-
final_shard_value =
|
14
|
-
final_primary_shard =
|
15
|
-
final_shard_source_value =
|
13
|
+
final_shard_value = rhs.shard_value
|
14
|
+
final_primary_shard = rhs.primary_shard
|
15
|
+
final_shard_source_value = rhs.shard_source_value
|
16
16
|
else
|
17
|
-
final_shard_source_value = [
|
18
|
-
shard_source_value == source_value ||
|
17
|
+
final_shard_source_value = %i[explicit association].detect do |source_value|
|
18
|
+
shard_source_value == source_value || rhs.shard_source_value == source_value
|
19
19
|
end
|
20
|
-
raise
|
20
|
+
raise 'unknown shard_source_value' unless final_shard_source_value
|
21
21
|
|
22
22
|
# have to merge shard_value
|
23
23
|
lhs_shard_value = all_shards
|
24
|
-
rhs_shard_value =
|
25
|
-
if
|
26
|
-
|
24
|
+
rhs_shard_value = rhs.all_shards
|
25
|
+
if ::ActiveRecord::Relation === lhs_shard_value &&
|
26
|
+
::ActiveRecord::Relation === rhs_shard_value
|
27
27
|
final_shard_value = lhs_shard_value.merge(rhs_shard_value)
|
28
28
|
final_primary_shard = Shard.default
|
29
29
|
else
|
@@ -32,27 +32,25 @@ module Switchman
|
|
32
32
|
final_shard_value = final_shard_value.first if final_shard_value.length == 1
|
33
33
|
end
|
34
34
|
end
|
35
|
-
elsif shard_source_value !=
|
36
|
-
final_shard_source_value = [
|
37
|
-
shard_source_value == source_value ||
|
35
|
+
elsif shard_source_value != rhs.shard_source_value
|
36
|
+
final_shard_source_value = %i[explicit association implicit].detect do |source_value|
|
37
|
+
shard_source_value == source_value || rhs.shard_source_value == source_value
|
38
38
|
end
|
39
|
-
raise
|
40
|
-
else
|
41
|
-
# nothing fancy
|
39
|
+
raise 'unknown shard_source_value' unless final_shard_source_value
|
42
40
|
end
|
43
41
|
|
44
42
|
[final_shard_value, final_primary_shard, final_shard_source_value]
|
45
43
|
end
|
46
44
|
|
47
|
-
def merge!(
|
48
|
-
return super unless ::ActiveRecord::Relation ===
|
45
|
+
def merge!(rhs)
|
46
|
+
return super unless ::ActiveRecord::Relation === rhs
|
49
47
|
|
50
48
|
# have to figure out shard stuff *before* conditions are merged
|
51
|
-
final_shard_value, final_primary_shard, final_shard_source_value = shard_values_for_merge(
|
49
|
+
final_shard_value, final_primary_shard, final_shard_source_value = shard_values_for_merge(rhs)
|
52
50
|
|
53
51
|
return super unless final_shard_source_value
|
54
52
|
|
55
|
-
|
53
|
+
unless final_shard_value
|
56
54
|
super
|
57
55
|
self.shard_source_value = final_shard_source_value
|
58
56
|
return self
|
@@ -61,16 +59,16 @@ module Switchman
|
|
61
59
|
return none! if final_shard_value == []
|
62
60
|
|
63
61
|
# change the primary shard if necessary before merging
|
64
|
-
if primary_shard != final_primary_shard &&
|
62
|
+
if primary_shard != final_primary_shard && rhs.primary_shard != final_primary_shard
|
65
63
|
shard!(final_primary_shard)
|
66
|
-
|
67
|
-
super(
|
64
|
+
rhs = rhs.shard(final_primary_shard)
|
65
|
+
super(rhs)
|
68
66
|
elsif primary_shard != final_primary_shard
|
69
67
|
shard!(final_primary_shard)
|
70
|
-
super(
|
71
|
-
elsif
|
72
|
-
|
73
|
-
super(
|
68
|
+
super(rhs)
|
69
|
+
elsif rhs.primary_shard != final_primary_shard
|
70
|
+
rhs = rhs.shard(final_primary_shard)
|
71
|
+
super(rhs)
|
74
72
|
else
|
75
73
|
super
|
76
74
|
end
|
@@ -7,18 +7,13 @@ module Switchman
|
|
7
7
|
def create(connection, &block)
|
8
8
|
relation = block.call ::ActiveRecord::StatementCache::Params.new
|
9
9
|
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
new(relation.arel, bind_map, relation.klass)
|
14
|
-
else
|
15
|
-
bind_map = ::ActiveRecord::StatementCache::BindMap.new(relation.bound_attributes)
|
16
|
-
new relation.arel, bind_map
|
17
|
-
end
|
10
|
+
_query_builder, binds = connection.cacheable_query(self, relation.arel)
|
11
|
+
bind_map = ::ActiveRecord::StatementCache::BindMap.new(binds)
|
12
|
+
new(relation.arel, bind_map, relation.klass)
|
18
13
|
end
|
19
14
|
end
|
20
15
|
|
21
|
-
def initialize(arel, bind_map, klass=nil)
|
16
|
+
def initialize(arel, bind_map, klass = nil)
|
22
17
|
@arel = arel
|
23
18
|
@bind_map = bind_map
|
24
19
|
@klass = klass
|
@@ -31,49 +26,39 @@ module Switchman
|
|
31
26
|
# (e.g. infer from the primary key or use the current shard)
|
32
27
|
|
33
28
|
def execute(*args)
|
34
|
-
|
35
|
-
|
36
|
-
klass = @klass
|
37
|
-
else
|
38
|
-
params, klass, connection = args
|
39
|
-
end
|
29
|
+
params, connection = args
|
30
|
+
klass = @klass
|
40
31
|
target_shard = nil
|
41
|
-
if primary_index = bind_map.primary_value_index
|
32
|
+
if (primary_index = bind_map.primary_value_index)
|
42
33
|
primary_value = params[primary_index]
|
43
34
|
target_shard = Shard.local_id_for(primary_value)[1]
|
44
35
|
end
|
45
|
-
current_shard = Shard.current(klass.
|
36
|
+
current_shard = Shard.current(klass.connection_classes)
|
46
37
|
target_shard ||= current_shard
|
47
38
|
|
48
39
|
bind_values = bind_map.bind(params, current_shard, target_shard)
|
49
40
|
|
50
|
-
target_shard.activate(klass.
|
41
|
+
target_shard.activate(klass.connection_classes) do
|
51
42
|
sql = qualified_query_builder(target_shard, klass).sql_for(bind_values, connection)
|
52
43
|
klass.find_by_sql(sql, bind_values)
|
53
44
|
end
|
54
45
|
end
|
55
46
|
|
56
|
-
|
57
|
-
|
58
|
-
@qualified_query_builders[shard.id] ||= klass.connection.cacheable_query(self.class, @arel)
|
59
|
-
end
|
60
|
-
else
|
61
|
-
def qualified_query_builder(shard, klass)
|
62
|
-
@qualified_query_builders[shard.id] ||= klass.connection.cacheable_query(self.class, @arel).first
|
63
|
-
end
|
47
|
+
def qualified_query_builder(shard, klass)
|
48
|
+
@qualified_query_builders[shard.id] ||= klass.connection.cacheable_query(self.class, @arel).first
|
64
49
|
end
|
65
50
|
|
66
51
|
module BindMap
|
67
52
|
# performs id transposition here instead of query_methods.rb
|
68
53
|
def bind(values, current_shard, target_shard)
|
69
54
|
bas = @bound_attributes.dup
|
70
|
-
@indexes.each_with_index do |offset,i|
|
55
|
+
@indexes.each_with_index do |offset, i|
|
71
56
|
ba = bas[offset]
|
72
|
-
if ba.is_a?(::ActiveRecord::Relation::QueryAttribute) && ba.value.sharded
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
57
|
+
new_value = if ba.is_a?(::ActiveRecord::Relation::QueryAttribute) && ba.value.sharded
|
58
|
+
Shard.relative_id_for(values[i], current_shard, target_shard || current_shard)
|
59
|
+
else
|
60
|
+
values[i]
|
61
|
+
end
|
77
62
|
bas[offset] = ba.with_cast_value(new_value)
|
78
63
|
end
|
79
64
|
bas
|
@@ -83,9 +68,7 @@ module Switchman
|
|
83
68
|
primary_ba_index = @bound_attributes.index do |ba|
|
84
69
|
ba.is_a?(::ActiveRecord::Relation::QueryAttribute) && ba.value.primary
|
85
70
|
end
|
86
|
-
if primary_ba_index
|
87
|
-
@indexes.index(primary_ba_index)
|
88
|
-
end
|
71
|
+
@indexes.index(primary_ba_index) if primary_ba_index
|
89
72
|
end
|
90
73
|
end
|
91
74
|
end
|
@@ -0,0 +1,16 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Switchman
|
4
|
+
module ActiveRecord
|
5
|
+
module Tasks
|
6
|
+
module DatabaseTasks
|
7
|
+
def drop(*)
|
8
|
+
super
|
9
|
+
# no really, it's gone
|
10
|
+
Switchman.cache.delete('default_shard')
|
11
|
+
Shard.default(reload: true)
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
@@ -8,10 +8,8 @@ module Switchman
|
|
8
8
|
store = super
|
9
9
|
# can't use defined?, because it's a _ruby_ autoloaded constant,
|
10
10
|
# so just checking that will cause it to get required
|
11
|
-
if store.
|
12
|
-
|
13
|
-
end
|
14
|
-
store.options[:namespace] ||= lambda { Shard.current.default? ? nil : "shard_#{Shard.current.id}" }
|
11
|
+
::ActiveSupport::Cache::RedisCacheStore.prepend(RedisCacheStore) if store.instance_of?(ActiveSupport::Cache::RedisCacheStore) && !::ActiveSupport::Cache::RedisCacheStore.ancestors.include?(RedisCacheStore)
|
12
|
+
store.options[:namespace] ||= -> { Shard.current.default? ? nil : "shard_#{Shard.current.id}" }
|
15
13
|
store
|
16
14
|
end
|
17
15
|
end
|
@@ -22,7 +20,7 @@ module Switchman
|
|
22
20
|
# unfortunately, it uses the keys command, which is extraordinarily inefficient in a large redis instance
|
23
21
|
# fortunately, we can assume we control the entire instance, because we set up the namespacing, so just
|
24
22
|
# always unset it temporarily for clear calls
|
25
|
-
namespace = nil
|
23
|
+
namespace = nil # rubocop:disable Lint/ShadowedArgument
|
26
24
|
super
|
27
25
|
end
|
28
26
|
end
|
data/lib/switchman/arel.rb
CHANGED
@@ -3,29 +3,34 @@
|
|
3
3
|
module Switchman
|
4
4
|
module Arel
|
5
5
|
module Table
|
6
|
-
def
|
7
|
-
|
6
|
+
def klass
|
7
|
+
@klass || ::ActiveRecord::Base
|
8
8
|
end
|
9
9
|
end
|
10
|
+
|
10
11
|
module Visitors
|
11
12
|
module ToSql
|
12
|
-
|
13
|
+
# rubocop:disable Naming/MethodName
|
14
|
+
|
15
|
+
def visit_Arel_Nodes_TableAlias(*args)
|
13
16
|
o, collector = args
|
14
17
|
collector = visit o.relation, collector
|
15
|
-
collector <<
|
18
|
+
collector << ' '
|
16
19
|
collector << quote_local_table_name(o.name)
|
17
20
|
end
|
18
21
|
|
19
|
-
def visit_Arel_Attributes_Attribute
|
22
|
+
def visit_Arel_Attributes_Attribute(*args)
|
20
23
|
o = args.first
|
21
24
|
join_name = o.relation.table_alias || o.relation.name
|
22
25
|
result = "#{quote_local_table_name join_name}.#{quote_column_name o.name}"
|
23
|
-
|
24
|
-
result
|
26
|
+
args.last << result
|
25
27
|
end
|
26
28
|
|
27
|
-
|
29
|
+
# rubocop:enable Naming/MethodName
|
30
|
+
|
31
|
+
def quote_local_table_name(name)
|
28
32
|
return name if ::Arel::Nodes::SqlLiteral === name
|
33
|
+
|
29
34
|
@connection.quote_local_table_name(name)
|
30
35
|
end
|
31
36
|
end
|
@@ -1,6 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
3
|
+
require 'securerandom'
|
4
4
|
|
5
5
|
module Switchman
|
6
6
|
class DatabaseServer
|
@@ -13,16 +13,22 @@ module Switchman
|
|
13
13
|
database_servers.values
|
14
14
|
end
|
15
15
|
|
16
|
+
def all_roles
|
17
|
+
@all_roles ||= all.map(&:roles).flatten.uniq
|
18
|
+
end
|
19
|
+
|
16
20
|
def find(id_or_all)
|
17
|
-
return
|
18
|
-
return id_or_all.map { |id|
|
21
|
+
return all if id_or_all == :all
|
22
|
+
return id_or_all.map { |id| database_servers[id || ::Rails.env] }.compact.uniq if id_or_all.is_a?(Array)
|
23
|
+
|
19
24
|
database_servers[id_or_all || ::Rails.env]
|
20
25
|
end
|
21
26
|
|
22
27
|
def create(settings = {})
|
23
|
-
raise
|
28
|
+
raise 'database servers should be set up in database.yml' unless ::Rails.env.test?
|
29
|
+
|
24
30
|
id = settings[:id]
|
25
|
-
|
31
|
+
unless id
|
26
32
|
@id ||= 0
|
27
33
|
@id += 1
|
28
34
|
id = @id
|
@@ -30,25 +36,42 @@ module Switchman
|
|
30
36
|
server = DatabaseServer.new(id.to_s, settings)
|
31
37
|
server.instance_variable_set(:@fake, true)
|
32
38
|
database_servers[server.id] = server
|
39
|
+
::ActiveRecord::Base.configurations.configurations <<
|
40
|
+
::ActiveRecord::DatabaseConfigurations::HashConfig.new(::Rails.env, "#{server.id}/primary", settings)
|
41
|
+
Shard.send(:initialize_sharding)
|
42
|
+
server
|
33
43
|
end
|
34
44
|
|
35
45
|
def server_for_new_shard
|
36
46
|
servers = all.select { |s| s.config[:open] }
|
37
47
|
return find(nil) if servers.empty?
|
48
|
+
|
38
49
|
servers[rand(servers.length)]
|
39
50
|
end
|
40
51
|
|
41
52
|
private
|
53
|
+
|
54
|
+
def reference_role(role)
|
55
|
+
return if all_roles.include?(role)
|
56
|
+
|
57
|
+
@all_roles << role
|
58
|
+
Shard.send(:initialize_sharding)
|
59
|
+
end
|
60
|
+
|
42
61
|
def database_servers
|
43
62
|
unless @database_servers
|
44
63
|
@database_servers = {}.with_indifferent_access
|
45
|
-
|
46
|
-
|
47
|
-
|
64
|
+
::ActiveRecord::Base.configurations.configurations.each do |config|
|
65
|
+
if config.name.include?('/')
|
66
|
+
name, role = config.name.split('/')
|
67
|
+
else
|
68
|
+
name, role = config.env_name, config.name
|
48
69
|
end
|
49
|
-
|
50
|
-
|
51
|
-
@database_servers[
|
70
|
+
|
71
|
+
if role == 'primary'
|
72
|
+
@database_servers[name] = DatabaseServer.new(config.env_name, config.configuration_hash)
|
73
|
+
else
|
74
|
+
@database_servers[name].roles << role
|
52
75
|
end
|
53
76
|
end
|
54
77
|
end
|
@@ -56,15 +79,33 @@ module Switchman
|
|
56
79
|
end
|
57
80
|
end
|
58
81
|
|
82
|
+
attr_reader :roles
|
83
|
+
|
59
84
|
def initialize(id = nil, config = {})
|
60
85
|
@id = id
|
61
86
|
@config = config.deep_symbolize_keys
|
62
87
|
@configs = {}
|
88
|
+
@roles = [:primary]
|
89
|
+
end
|
90
|
+
|
91
|
+
def connects_to_hash
|
92
|
+
self.class.all_roles.map do |role|
|
93
|
+
config_role = role
|
94
|
+
config_role = :primary unless roles.include?(role)
|
95
|
+
config_name = :"#{id}/#{config_role}"
|
96
|
+
config_name = :primary if id == ::Rails.env && config_role == :primary
|
97
|
+
[role.to_sym, config_name]
|
98
|
+
end.to_h
|
63
99
|
end
|
64
100
|
|
65
101
|
def destroy
|
66
|
-
|
67
|
-
|
102
|
+
self.class.send(:database_servers).delete(id) if id
|
103
|
+
Shard.sharded_models.each do |klass|
|
104
|
+
self.class.all_roles.each do |role|
|
105
|
+
klass.connection_handler.remove_connection_pool(klass.connection_specification_name, role: role,
|
106
|
+
shard: id.to_sym)
|
107
|
+
end
|
108
|
+
end
|
68
109
|
end
|
69
110
|
|
70
111
|
def fake?
|
@@ -73,14 +114,15 @@ module Switchman
|
|
73
114
|
|
74
115
|
def config(environment = :primary)
|
75
116
|
@configs[environment] ||= begin
|
76
|
-
|
117
|
+
case @config[environment]
|
118
|
+
when Array
|
77
119
|
@config[environment].map do |config|
|
78
120
|
config = @config.merge((config || {}).symbolize_keys)
|
79
121
|
# make sure GuardRail doesn't get any brilliant ideas about choosing the first possible server
|
80
122
|
config.delete(environment)
|
81
123
|
config
|
82
124
|
end
|
83
|
-
|
125
|
+
when Hash
|
84
126
|
@config.merge(@config[environment])
|
85
127
|
else
|
86
128
|
@config
|
@@ -111,172 +153,109 @@ module Switchman
|
|
111
153
|
guard!(old_env)
|
112
154
|
end
|
113
155
|
|
114
|
-
def shareable?
|
115
|
-
@shareable_environment_key ||= []
|
116
|
-
environment = guard_rail_environment
|
117
|
-
explicit_user = ::GuardRail.global_config[:username]
|
118
|
-
return @shareable if @shareable_environment_key == [environment, explicit_user]
|
119
|
-
@shareable_environment_key = [environment, explicit_user]
|
120
|
-
if explicit_user
|
121
|
-
username = explicit_user
|
122
|
-
else
|
123
|
-
config = self.config(environment)
|
124
|
-
config = config.first if config.is_a?(Array)
|
125
|
-
username = config[:username]
|
126
|
-
end
|
127
|
-
@shareable = username !~ /%?\{[a-zA-Z0-9_]+\}/
|
128
|
-
end
|
129
|
-
|
130
156
|
def shards
|
131
|
-
if
|
132
|
-
Shard.where(
|
157
|
+
if id == ::Rails.env
|
158
|
+
Shard.where('database_server_id IS NULL OR database_server_id=?', id)
|
133
159
|
else
|
134
|
-
Shard.where(:
|
160
|
+
Shard.where(database_server_id: id)
|
135
161
|
end
|
136
162
|
end
|
137
163
|
|
138
|
-
def
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
def create_new_shard(options = {})
|
143
|
-
raise NotImplementedError.new("Cannot create new shards when sharding isn't initialized") unless Shard.default.is_a?(Shard)
|
144
|
-
|
145
|
-
name = options[:name]
|
146
|
-
create_schema = options[:schema]
|
147
|
-
# look for another shard associated with this db
|
148
|
-
other_shard = self.shards.where("name<>':memory:' OR name IS NULL").order(:id).first
|
149
|
-
|
150
|
-
case config[:adapter]
|
151
|
-
when 'postgresql'
|
152
|
-
create_statement = lambda { "CREATE SCHEMA #{name}" }
|
153
|
-
password = " PASSWORD #{::ActiveRecord::Base.connection.quote(config[:password])}" if config[:password]
|
154
|
-
else
|
155
|
-
create_statement = lambda { "CREATE DATABASE #{name}" }
|
164
|
+
def create_new_shard(id: nil, name: nil, schema: true)
|
165
|
+
unless Shard.default.is_a?(Shard)
|
166
|
+
raise NotImplementedError,
|
167
|
+
"Cannot create new shards when sharding isn't initialized"
|
156
168
|
end
|
169
|
+
|
170
|
+
create_statement = -> { "CREATE SCHEMA #{name}" }
|
171
|
+
password = " PASSWORD #{::ActiveRecord::Base.connection.quote(config[:password])}" if config[:password]
|
157
172
|
sharding_config = Switchman.config
|
158
173
|
config_create_statement = sharding_config[config[:adapter]]&.[](:create_statement)
|
159
174
|
config_create_statement ||= sharding_config[:create_statement]
|
160
175
|
if config_create_statement
|
161
176
|
create_commands = Array(config_create_statement).dup
|
162
177
|
create_statement = lambda {
|
163
|
-
create_commands.map { |statement| statement
|
178
|
+
create_commands.map { |statement| format(statement, name: name, password: password) }
|
164
179
|
}
|
165
180
|
end
|
166
181
|
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
next_id = Shard.connection.select_value("SELECT nextval(#{id_seq})")
|
173
|
-
next_id.to_i
|
174
|
-
else
|
175
|
-
nil
|
176
|
-
end
|
177
|
-
end
|
182
|
+
id ||= begin
|
183
|
+
id_seq = Shard.connection.quote(Shard.connection.quote_table_name('switchman_shards_id_seq'))
|
184
|
+
next_id = Shard.connection.select_value("SELECT nextval(#{id_seq})")
|
185
|
+
next_id.to_i
|
186
|
+
end
|
178
187
|
|
179
|
-
|
180
|
-
base_name = self.config[:database].to_s % self.config
|
181
|
-
base_name = nil if base_name == ':memory:'
|
182
|
-
base_name << '_' if base_name
|
183
|
-
base_id = shard_id || SecureRandom.uuid
|
184
|
-
name = "#{base_name}shard_#{base_id}"
|
185
|
-
end
|
188
|
+
name ||= "#{config[:database]}_shard_#{id}"
|
186
189
|
|
187
|
-
|
188
|
-
|
189
|
-
:
|
190
|
+
Shard.connection.transaction do
|
191
|
+
shard = Shard.create!(id: id,
|
192
|
+
name: name,
|
193
|
+
database_server_id: self.id)
|
190
194
|
schema_already_existed = false
|
191
195
|
|
192
196
|
begin
|
193
197
|
self.class.creating_new_shard = true
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
migrate = ::Rails.version >= '5.2' ?
|
220
|
-
-> { ::ActiveRecord::Base.connection.migration_context.migrate } :
|
221
|
-
-> { ::ActiveRecord::Migrator.migrate(::ActiveRecord::Migrator.migrations_paths) }
|
222
|
-
if ::ActiveRecord::Base.connection.supports_ddl_transactions?
|
223
|
-
::ActiveRecord::Base.connection.transaction(requires_new: true, &migrate)
|
224
|
-
else
|
225
|
-
migrate.call
|
226
|
-
end
|
227
|
-
reset_column_information
|
228
|
-
::ActiveRecord::Base.descendants.reject { |m| m == Shard || !m.table_exists? }.each(&:define_attribute_methods)
|
198
|
+
DatabaseServer.send(:reference_role, :deploy)
|
199
|
+
::ActiveRecord::Base.connected_to(shard: self.id.to_sym, role: :deploy) do
|
200
|
+
if create_statement
|
201
|
+
if ::ActiveRecord::Base.connection.select_value("SELECT 1 FROM pg_namespace WHERE nspname=#{::ActiveRecord::Base.connection.quote(name)}")
|
202
|
+
schema_already_existed = true
|
203
|
+
raise 'This schema already exists; cannot overwrite'
|
204
|
+
end
|
205
|
+
Array(create_statement.call).each do |stmt|
|
206
|
+
::ActiveRecord::Base.connection.execute(stmt)
|
207
|
+
end
|
208
|
+
end
|
209
|
+
if config[:adapter] == 'postgresql'
|
210
|
+
old_proc = ::ActiveRecord::Base.connection.raw_connection.set_notice_processor do
|
211
|
+
end
|
212
|
+
end
|
213
|
+
old_verbose = ::ActiveRecord::Migration.verbose
|
214
|
+
::ActiveRecord::Migration.verbose = false
|
215
|
+
|
216
|
+
unless schema == false
|
217
|
+
shard.activate do
|
218
|
+
reset_column_information
|
219
|
+
|
220
|
+
::ActiveRecord::Base.connection.transaction(requires_new: true) do
|
221
|
+
::ActiveRecord::Base.connection.migration_context.migrate
|
229
222
|
end
|
230
|
-
|
231
|
-
::ActiveRecord::
|
232
|
-
|
223
|
+
reset_column_information
|
224
|
+
::ActiveRecord::Base.descendants.reject do |m|
|
225
|
+
m <= UnshardedRecord || !m.table_exists?
|
226
|
+
end.each(&:define_attribute_methods)
|
233
227
|
end
|
234
228
|
end
|
229
|
+
ensure
|
230
|
+
::ActiveRecord::Migration.verbose = old_verbose
|
231
|
+
::ActiveRecord::Base.connection.raw_connection.set_notice_processor(&old_proc) if old_proc
|
235
232
|
end
|
236
233
|
shard
|
237
234
|
rescue
|
238
235
|
shard.destroy
|
239
|
-
unless schema_already_existed
|
240
|
-
|
241
|
-
end
|
242
|
-
reset_column_information unless create_schema == false rescue nil
|
236
|
+
shard.drop_database rescue nil unless schema_already_existed
|
237
|
+
reset_column_information unless schema == false rescue nil
|
243
238
|
raise
|
244
239
|
ensure
|
245
240
|
self.class.creating_new_shard = false
|
246
241
|
end
|
247
242
|
end
|
248
|
-
|
249
|
-
if Shard.connection.supports_ddl_transactions? && self.shareable? && other_shard
|
250
|
-
Shard.transaction do
|
251
|
-
other_shard.activate do
|
252
|
-
::ActiveRecord::Base.connection.transaction(&create_shard)
|
253
|
-
end
|
254
|
-
end
|
255
|
-
else
|
256
|
-
create_shard.call
|
257
|
-
end
|
258
243
|
end
|
259
244
|
|
260
245
|
def cache_store
|
261
|
-
|
262
|
-
@cache_store = Switchman.config[:cache_map][self.id] || Switchman.config[:cache_map][::Rails.env]
|
263
|
-
end
|
246
|
+
@cache_store ||= Switchman.config[:cache_map][id] || Switchman.config[:cache_map][::Rails.env]
|
264
247
|
@cache_store
|
265
248
|
end
|
266
249
|
|
267
250
|
def shard_name(shard)
|
268
|
-
if config[:shard_name]
|
269
|
-
|
270
|
-
|
271
|
-
if
|
272
|
-
|
273
|
-
|
274
|
-
::ActiveRecord::Base.connection.current_schemas.first rescue nil
|
275
|
-
else
|
276
|
-
shard.activate { ::ActiveRecord::Base.connection_pool.default_schema }
|
277
|
-
end
|
251
|
+
return config[:shard_name] if config[:shard_name]
|
252
|
+
|
253
|
+
if shard == :bootstrap
|
254
|
+
# rescue nil because the database may not exist yet; if it doesn't,
|
255
|
+
# it will shortly, and this will be re-invoked
|
256
|
+
::ActiveRecord::Base.connection.current_schemas.first rescue nil
|
278
257
|
else
|
279
|
-
|
258
|
+
shard.activate { ::ActiveRecord::Base.connection_pool.default_schema }
|
280
259
|
end
|
281
260
|
end
|
282
261
|
|
@@ -290,9 +269,9 @@ module Switchman
|
|
290
269
|
end
|
291
270
|
|
292
271
|
private
|
272
|
+
|
293
273
|
def reset_column_information
|
294
|
-
::ActiveRecord::Base.descendants.reject { |m| m
|
295
|
-
::ActiveRecord::Base.connection_handler.switchman_connection_pool_proxies.each { |pool| pool.schema_cache.clear! }
|
274
|
+
::ActiveRecord::Base.descendants.reject { |m| m <= UnshardedRecord }.each(&:reset_column_information)
|
296
275
|
end
|
297
276
|
end
|
298
277
|
end
|