switchman 2.0.13 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Rakefile +10 -2
- data/app/models/switchman/shard.rb +234 -271
- data/app/models/switchman/unsharded_record.rb +7 -0
- data/db/migrate/20130328212039_create_switchman_shards.rb +1 -1
- data/db/migrate/20130328224244_create_default_shard.rb +5 -5
- data/db/migrate/20161206323434_add_back_default_string_limits_switchman.rb +1 -0
- data/db/migrate/20180828183945_add_default_shard_index.rb +2 -2
- data/db/migrate/20180828192111_add_timestamps_to_shards.rb +7 -5
- data/db/migrate/20190114212900_add_unique_name_indexes.rb +5 -3
- data/lib/switchman.rb +3 -5
- data/lib/switchman/action_controller/caching.rb +2 -2
- data/lib/switchman/active_record/abstract_adapter.rb +1 -0
- data/lib/switchman/active_record/association.rb +78 -89
- data/lib/switchman/active_record/attribute_methods.rb +58 -52
- data/lib/switchman/active_record/base.rb +58 -59
- data/lib/switchman/active_record/calculations.rb +74 -67
- data/lib/switchman/active_record/connection_pool.rb +14 -41
- data/lib/switchman/active_record/database_configurations.rb +34 -0
- data/lib/switchman/active_record/database_configurations/database_config.rb +13 -0
- data/lib/switchman/active_record/finder_methods.rb +11 -16
- data/lib/switchman/active_record/log_subscriber.rb +4 -8
- data/lib/switchman/active_record/migration.rb +6 -47
- data/lib/switchman/active_record/model_schema.rb +1 -1
- data/lib/switchman/active_record/persistence.rb +4 -6
- data/lib/switchman/active_record/postgresql_adapter.rb +124 -168
- data/lib/switchman/active_record/predicate_builder.rb +2 -2
- data/lib/switchman/active_record/query_cache.rb +18 -19
- data/lib/switchman/active_record/query_methods.rb +172 -197
- data/lib/switchman/active_record/reflection.rb +6 -10
- data/lib/switchman/active_record/relation.rb +30 -78
- data/lib/switchman/active_record/spawn_methods.rb +27 -29
- data/lib/switchman/active_record/statement_cache.rb +18 -35
- data/lib/switchman/active_record/tasks/database_tasks.rb +16 -0
- data/lib/switchman/active_support/cache.rb +3 -5
- data/lib/switchman/arel.rb +13 -8
- data/lib/switchman/database_server.rb +121 -142
- data/lib/switchman/default_shard.rb +52 -16
- data/lib/switchman/engine.rb +61 -58
- data/lib/switchman/environment.rb +4 -8
- data/lib/switchman/errors.rb +1 -0
- data/lib/switchman/guard_rail.rb +6 -19
- data/lib/switchman/guard_rail/relation.rb +5 -7
- data/lib/switchman/r_spec_helper.rb +29 -37
- data/lib/switchman/rails.rb +14 -12
- data/lib/switchman/schema_cache.rb +1 -9
- data/lib/switchman/sharded_instrumenter.rb +1 -1
- data/lib/switchman/standard_error.rb +15 -3
- data/lib/switchman/test_helper.rb +7 -11
- data/lib/switchman/version.rb +1 -1
- data/lib/tasks/switchman.rake +54 -69
- metadata +87 -45
- data/lib/switchman/active_record/batches.rb +0 -11
- data/lib/switchman/active_record/connection_handler.rb +0 -172
- data/lib/switchman/active_record/where_clause_factory.rb +0 -36
- data/lib/switchman/connection_pool_proxy.rb +0 -173
@@ -1,172 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require 'switchman/connection_pool_proxy'
|
4
|
-
|
5
|
-
module Switchman
|
6
|
-
module ActiveRecord
|
7
|
-
module ConnectionHandler
|
8
|
-
def self.make_sharing_automagic(config, shard = Shard.current)
|
9
|
-
# only load the shard name from the db if we have to
|
10
|
-
if !config[:shard_name]
|
11
|
-
# we may not be able to connect to this shard yet, cause it might be an empty database server
|
12
|
-
shard = shard.call if shard.is_a?(Proc)
|
13
|
-
shard_name = shard.name rescue nil
|
14
|
-
return unless shard_name
|
15
|
-
|
16
|
-
config[:shard_name] ||= shard_name
|
17
|
-
end
|
18
|
-
end
|
19
|
-
|
20
|
-
def establish_connection(spec)
|
21
|
-
# Just skip establishing a sharded connection if sharding isn't loaded; we'll do it again later
|
22
|
-
# This only can happen when loading ActiveRecord::Base; after everything is loaded Shard will
|
23
|
-
# be defined and this will actually establish a connection
|
24
|
-
return unless defined?(Shard)
|
25
|
-
pool = super
|
26
|
-
|
27
|
-
# this is the first place that the adapter would have been required; but now we
|
28
|
-
# need this addition ASAP since it will be called when loading the default shard below
|
29
|
-
if defined?(::ActiveRecord::ConnectionAdapters::PostgreSQLAdapter)
|
30
|
-
require "switchman/active_record/postgresql_adapter"
|
31
|
-
::ActiveRecord::ConnectionAdapters::PostgreSQLAdapter.prepend(ActiveRecord::PostgreSQLAdapter)
|
32
|
-
end
|
33
|
-
|
34
|
-
first_time = !Shard.instance_variable_get(:@default)
|
35
|
-
if first_time
|
36
|
-
# Have to cache the default shard before we insert sharding, otherwise the first access
|
37
|
-
# to sharding will recurse onto itself trying to access column information
|
38
|
-
Shard.default
|
39
|
-
|
40
|
-
config = pool.spec.config
|
41
|
-
# automatically change config to allow for sharing connections with simple config
|
42
|
-
ConnectionHandler.make_sharing_automagic(config)
|
43
|
-
ConnectionHandler.make_sharing_automagic(Shard.default.database_server.config)
|
44
|
-
|
45
|
-
if ::Rails.version < '6.0'
|
46
|
-
::ActiveRecord::Base.configurations[::Rails.env] = config.stringify_keys
|
47
|
-
else
|
48
|
-
# Adopted from the deprecated code that currently lives in rails proper
|
49
|
-
remaining_configs = ::ActiveRecord::Base.configurations.configurations.reject { |db_config| db_config.env_name == ::Rails.env }
|
50
|
-
new_config = ::ActiveRecord::DatabaseConfigurations.new(::Rails.env => config.stringify_keys).configurations
|
51
|
-
new_configs = remaining_configs + new_config
|
52
|
-
|
53
|
-
::ActiveRecord::Base.configurations = new_configs
|
54
|
-
end
|
55
|
-
else
|
56
|
-
# this is probably wrong now
|
57
|
-
Shard.default.remove_instance_variable(:@name) if Shard.default.instance_variable_defined?(:@name)
|
58
|
-
end
|
59
|
-
|
60
|
-
@shard_connection_pools ||= { [:primary, Shard.default.database_server.shareable? ? ::Rails.env : Shard.default] => pool}
|
61
|
-
|
62
|
-
category = pool.spec.name.to_sym
|
63
|
-
proxy = ConnectionPoolProxy.new(category,
|
64
|
-
pool,
|
65
|
-
@shard_connection_pools)
|
66
|
-
owner_to_pool[pool.spec.name] = proxy
|
67
|
-
|
68
|
-
if first_time
|
69
|
-
if Shard.default.database_server.config[:prefer_secondary]
|
70
|
-
Shard.default.database_server.guard!
|
71
|
-
end
|
72
|
-
|
73
|
-
if Shard.default.is_a?(DefaultShard) && Shard.default.database_server.config[:secondary]
|
74
|
-
Shard.default.database_server.guard!
|
75
|
-
Shard.default(reload: true)
|
76
|
-
end
|
77
|
-
end
|
78
|
-
|
79
|
-
# reload the default shard if we just got a new connection
|
80
|
-
# to where the Shards table is
|
81
|
-
# DON'T do it if we're not the current connection handler - that means
|
82
|
-
# we're in the middle of switching environments, and we don't want to
|
83
|
-
# establish a connection with incorrect settings
|
84
|
-
if [:primary, :unsharded].include?(category) && self == ::ActiveRecord::Base.connection_handler && !first_time
|
85
|
-
Shard.default(reload: true, with_fallback: true)
|
86
|
-
proxy.disconnect!
|
87
|
-
end
|
88
|
-
|
89
|
-
if first_time
|
90
|
-
# do the change for other database servers, now that we can switch shards
|
91
|
-
if Shard.default.is_a?(Shard)
|
92
|
-
DatabaseServer.all.each do |server|
|
93
|
-
next if server == Shard.default.database_server
|
94
|
-
|
95
|
-
shard = nil
|
96
|
-
shard_proc = -> do
|
97
|
-
shard ||= server.shards.where(:name => nil).first
|
98
|
-
shard ||= Shard.new(:database_server => server)
|
99
|
-
shard
|
100
|
-
end
|
101
|
-
ConnectionHandler.make_sharing_automagic(server.config, shard_proc)
|
102
|
-
ConnectionHandler.make_sharing_automagic(proxy.current_pool.spec.config, shard_proc)
|
103
|
-
end
|
104
|
-
end
|
105
|
-
# we may have established some connections above trying to infer the shard's name.
|
106
|
-
# close them, so that someone that doesn't expect them doesn't try to fork
|
107
|
-
# without closing them
|
108
|
-
self.clear_all_connections!
|
109
|
-
end
|
110
|
-
|
111
|
-
proxy
|
112
|
-
end
|
113
|
-
|
114
|
-
def remove_connection(spec_name)
|
115
|
-
pool = owner_to_pool[spec_name]
|
116
|
-
owner_to_pool[spec_name] = pool.default_pool if pool.is_a?(ConnectionPoolProxy)
|
117
|
-
super
|
118
|
-
end
|
119
|
-
|
120
|
-
def retrieve_connection_pool(spec_name)
|
121
|
-
owner_to_pool.fetch(spec_name) do
|
122
|
-
if ancestor_pool = pool_from_any_process_for(spec_name)
|
123
|
-
# A connection was established in an ancestor process that must have
|
124
|
-
# subsequently forked. We can't reuse the connection, but we can copy
|
125
|
-
# the specification and establish a new connection with it.
|
126
|
-
spec = if ancestor_pool.is_a?(ConnectionPoolProxy)
|
127
|
-
ancestor_pool.default_pool.spec
|
128
|
-
else
|
129
|
-
ancestor_pool.spec
|
130
|
-
end
|
131
|
-
pool = establish_connection(spec.to_hash)
|
132
|
-
pool.instance_variable_set(:@schema_cache, ancestor_pool.schema_cache) if ancestor_pool.schema_cache
|
133
|
-
pool
|
134
|
-
elsif spec_name != "primary"
|
135
|
-
primary_pool = retrieve_connection_pool("primary")
|
136
|
-
if primary_pool.is_a?(ConnectionPoolProxy)
|
137
|
-
pool = ConnectionPoolProxy.new(spec_name.to_sym, primary_pool.default_pool, @shard_connection_pools)
|
138
|
-
pool.schema_cache.copy_references(primary_pool.schema_cache)
|
139
|
-
pool
|
140
|
-
else
|
141
|
-
primary_pool
|
142
|
-
end
|
143
|
-
else
|
144
|
-
owner_to_pool[spec_name] = nil
|
145
|
-
end
|
146
|
-
end
|
147
|
-
end
|
148
|
-
|
149
|
-
def clear_idle_connections!(since_when)
|
150
|
-
connection_pool_list.each{ |pool| pool.clear_idle_connections!(since_when) }
|
151
|
-
end
|
152
|
-
|
153
|
-
def switchman_connection_pool_proxies
|
154
|
-
owner_to_pool.values.uniq.select{|p| p.is_a?(ConnectionPoolProxy)}
|
155
|
-
end
|
156
|
-
|
157
|
-
private
|
158
|
-
|
159
|
-
# semi-private
|
160
|
-
public
|
161
|
-
def uninitialize_ar(model = ::ActiveRecord::Base)
|
162
|
-
# take the proxies out
|
163
|
-
pool = owner_to_pool[model.name]
|
164
|
-
owner_to_pool[model.name] = pool.default_pool if pool
|
165
|
-
end
|
166
|
-
|
167
|
-
def initialize_categories(model = ::ActiveRecord::Base)
|
168
|
-
class_to_pool.clear
|
169
|
-
end
|
170
|
-
end
|
171
|
-
end
|
172
|
-
end
|
@@ -1,36 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Switchman
|
4
|
-
module ActiveRecord
|
5
|
-
module WhereClauseFactory
|
6
|
-
attr_writer :scope
|
7
|
-
|
8
|
-
def build(opts, other = [])
|
9
|
-
case opts
|
10
|
-
when String, Array
|
11
|
-
values = Hash === other.first ? other.first.values : other
|
12
|
-
|
13
|
-
values.grep(ActiveRecord::Relation) do |rel|
|
14
|
-
# serialize subqueries against the same shard as the outer query is currently
|
15
|
-
# targeted to run against
|
16
|
-
if rel.shard_source_value == :implicit && rel.primary_shard != @scope.primary_shard
|
17
|
-
rel.shard!(@scope.primary_shard)
|
18
|
-
end
|
19
|
-
end
|
20
|
-
|
21
|
-
super
|
22
|
-
when Hash, ::Arel::Nodes::Node
|
23
|
-
where_clause = super
|
24
|
-
binds = ::Rails.version >= "5.2" ? nil : where_clause.binds
|
25
|
-
predicates = where_clause.send(:predicates)
|
26
|
-
@scope.send(:infer_shards_from_primary_key, predicates, binds) if @scope.shard_source_value == :implicit && @scope.shard_value.is_a?(Shard)
|
27
|
-
predicates, _new_binds = @scope.transpose_predicates(predicates, nil, @scope.primary_shard, false, binds: binds)
|
28
|
-
where_clause.instance_variable_set(:@predicates, predicates)
|
29
|
-
where_clause
|
30
|
-
else
|
31
|
-
super
|
32
|
-
end
|
33
|
-
end
|
34
|
-
end
|
35
|
-
end
|
36
|
-
end
|
@@ -1,173 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require 'switchman/schema_cache'
|
4
|
-
|
5
|
-
module Switchman
|
6
|
-
module ConnectionError
|
7
|
-
def self.===(other)
|
8
|
-
return true if defined?(PG::Error) && PG::Error === other
|
9
|
-
return true if defined?(Mysql2::Error) && Mysql2::Error === other
|
10
|
-
false
|
11
|
-
end
|
12
|
-
end
|
13
|
-
|
14
|
-
class ConnectionPoolProxy
|
15
|
-
delegate :spec, :connected?, :default_schema, :with_connection, :query_cache_enabled, :active_connection?,
|
16
|
-
:to => :current_pool
|
17
|
-
|
18
|
-
attr_reader :category, :schema_cache
|
19
|
-
|
20
|
-
def default_pool
|
21
|
-
@default_pool
|
22
|
-
end
|
23
|
-
|
24
|
-
def initialize(category, default_pool, shard_connection_pools)
|
25
|
-
@category = category
|
26
|
-
@default_pool = default_pool
|
27
|
-
@connection_pools = shard_connection_pools
|
28
|
-
@schema_cache = default_pool.get_schema_cache(nil) if ::Rails.version >= '6'
|
29
|
-
@schema_cache = SchemaCache.new(self) unless @schema_cache.is_a?(SchemaCache)
|
30
|
-
if ::Rails.version >= '6'
|
31
|
-
@default_pool.set_schema_cache(@schema_cache)
|
32
|
-
@connection_pools.each_value do |pool|
|
33
|
-
pool.set_schema_cache(@schema_cache)
|
34
|
-
end
|
35
|
-
end
|
36
|
-
end
|
37
|
-
|
38
|
-
def active_shard
|
39
|
-
Shard.current(@category)
|
40
|
-
end
|
41
|
-
|
42
|
-
def active_guard_rail_environment
|
43
|
-
::Rails.env.test? ? :primary : active_shard.database_server.guard_rail_environment
|
44
|
-
end
|
45
|
-
|
46
|
-
def current_pool
|
47
|
-
current_active_shard = active_shard
|
48
|
-
pool = self.default_pool if current_active_shard.database_server == Shard.default.database_server && active_guard_rail_environment == :primary && (current_active_shard.default? || current_active_shard.database_server.shareable?)
|
49
|
-
pool = @connection_pools[pool_key] ||= create_pool unless pool
|
50
|
-
pool.shard = current_active_shard
|
51
|
-
pool
|
52
|
-
end
|
53
|
-
|
54
|
-
def connections
|
55
|
-
connection_pools.map(&:connections).inject([], &:+)
|
56
|
-
end
|
57
|
-
|
58
|
-
def connection(switch_shard: true)
|
59
|
-
pool = current_pool
|
60
|
-
begin
|
61
|
-
connection = pool.connection(switch_shard: switch_shard)
|
62
|
-
connection.instance_variable_set(:@schema_cache, @schema_cache) unless ::Rails.version >= '6'
|
63
|
-
connection
|
64
|
-
rescue ConnectionError
|
65
|
-
raise if active_shard.database_server == Shard.default.database_server && active_guard_rail_environment == :primary
|
66
|
-
configs = active_shard.database_server.config(active_guard_rail_environment)
|
67
|
-
raise unless configs.is_a?(Array)
|
68
|
-
configs.each_with_index do |config, idx|
|
69
|
-
pool = create_pool(config.dup)
|
70
|
-
begin
|
71
|
-
connection = pool.connection
|
72
|
-
connection.instance_variable_set(:@schema_cache, @schema_cache) unless ::Rails.version >= '6'
|
73
|
-
rescue ConnectionError
|
74
|
-
raise if idx == configs.length - 1
|
75
|
-
next
|
76
|
-
end
|
77
|
-
@connection_pools[pool_key] = pool
|
78
|
-
break connection
|
79
|
-
end
|
80
|
-
end
|
81
|
-
end
|
82
|
-
|
83
|
-
def get_schema_cache(_connection)
|
84
|
-
@schema_cache
|
85
|
-
end
|
86
|
-
|
87
|
-
def set_schema_cache(cache)
|
88
|
-
@schema_cache.copy_values(cache)
|
89
|
-
end
|
90
|
-
|
91
|
-
%w{release_connection
|
92
|
-
disconnect!
|
93
|
-
flush!
|
94
|
-
clear_reloadable_connections!
|
95
|
-
verify_active_connections!
|
96
|
-
clear_stale_cached_connections!
|
97
|
-
enable_query_cache!
|
98
|
-
disable_query_cache! }.each do |method|
|
99
|
-
class_eval(<<-RUBY, __FILE__, __LINE__ + 1)
|
100
|
-
def #{method}
|
101
|
-
connection_pools.each(&:#{method})
|
102
|
-
end
|
103
|
-
RUBY
|
104
|
-
end
|
105
|
-
|
106
|
-
def discard!
|
107
|
-
# this breaks everything if i try to pass it onto the pools and i'm not sure why
|
108
|
-
end
|
109
|
-
|
110
|
-
def automatic_reconnect=(value)
|
111
|
-
connection_pools.each { |pool| pool.automatic_reconnect = value }
|
112
|
-
end
|
113
|
-
|
114
|
-
def clear_idle_connections!(since_when)
|
115
|
-
connection_pools.each { |pool| pool.clear_idle_connections!(since_when) }
|
116
|
-
end
|
117
|
-
|
118
|
-
def remove_shard!(shard)
|
119
|
-
connection_pools.each { |pool| pool.remove_shard!(shard) }
|
120
|
-
end
|
121
|
-
|
122
|
-
protected
|
123
|
-
|
124
|
-
def connection_pools
|
125
|
-
(@connection_pools.values + [default_pool]).uniq
|
126
|
-
end
|
127
|
-
|
128
|
-
def pool_key
|
129
|
-
[active_guard_rail_environment,
|
130
|
-
active_shard.database_server.shareable? ? active_shard.database_server.pool_key : active_shard]
|
131
|
-
end
|
132
|
-
|
133
|
-
def create_pool(config = nil)
|
134
|
-
shard = active_shard
|
135
|
-
unless config
|
136
|
-
if shard != Shard.default
|
137
|
-
config = shard.database_server.config(active_guard_rail_environment)
|
138
|
-
config = config.first if config.is_a?(Array)
|
139
|
-
config = config.dup
|
140
|
-
else
|
141
|
-
# we read @config instead of calling config so that we get the config
|
142
|
-
# *before* %{shard_name} is applied
|
143
|
-
# also, we can't just read the database server's config, because
|
144
|
-
# different models could be using different configs on the default
|
145
|
-
# shard, and database server wouldn't know about that
|
146
|
-
config = default_pool.spec.instance_variable_get(:@config)
|
147
|
-
if config[active_guard_rail_environment].is_a?(Hash)
|
148
|
-
config = config.merge(config[active_guard_rail_environment])
|
149
|
-
elsif config[active_guard_rail_environment].is_a?(Array)
|
150
|
-
config = config.merge(config[active_guard_rail_environment].first)
|
151
|
-
else
|
152
|
-
config = config.dup
|
153
|
-
end
|
154
|
-
end
|
155
|
-
end
|
156
|
-
spec = ::ActiveRecord::ConnectionAdapters::ConnectionSpecification.new(
|
157
|
-
category,
|
158
|
-
config,
|
159
|
-
"#{config[:adapter]}_connection"
|
160
|
-
)
|
161
|
-
# unfortunately the AR code that does this require logic can't really be
|
162
|
-
# called in isolation
|
163
|
-
require "active_record/connection_adapters/#{config[:adapter]}_adapter"
|
164
|
-
|
165
|
-
::ActiveRecord::ConnectionAdapters::ConnectionPool.new(spec).tap do |pool|
|
166
|
-
pool.shard = shard
|
167
|
-
pool.set_schema_cache(@schema_cache) if ::Rails.version >= '6'
|
168
|
-
pool.enable_query_cache! if !@connection_pools.empty? && @connection_pools.first.last.query_cache_enabled
|
169
|
-
end
|
170
|
-
end
|
171
|
-
end
|
172
|
-
end
|
173
|
-
|