switchman 3.3.6 → 4.2.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Rakefile +15 -14
- data/db/migrate/20180828183945_add_default_shard_index.rb +1 -1
- data/db/migrate/20190114212900_add_unique_name_indexes.rb +10 -4
- data/lib/switchman/active_record/abstract_adapter.rb +10 -6
- data/lib/switchman/active_record/associations.rb +72 -49
- data/lib/switchman/active_record/attribute_methods.rb +89 -44
- data/lib/switchman/active_record/base.rb +109 -40
- data/lib/switchman/active_record/calculations.rb +90 -54
- data/lib/switchman/active_record/connection_handler.rb +18 -0
- data/lib/switchman/active_record/connection_pool.rb +41 -23
- data/lib/switchman/active_record/database_configurations.rb +23 -13
- data/lib/switchman/active_record/finder_methods.rb +20 -14
- data/lib/switchman/active_record/log_subscriber.rb +3 -6
- data/lib/switchman/active_record/migration.rb +35 -12
- data/lib/switchman/active_record/pending_migration_connection.rb +17 -0
- data/lib/switchman/active_record/persistence.rb +30 -0
- data/lib/switchman/active_record/postgresql_adapter.rb +37 -22
- data/lib/switchman/active_record/predicate_builder.rb +2 -2
- data/lib/switchman/active_record/query_cache.rb +26 -17
- data/lib/switchman/active_record/query_methods.rb +148 -44
- data/lib/switchman/active_record/reflection.rb +9 -2
- data/lib/switchman/active_record/relation.rb +87 -17
- data/lib/switchman/active_record/spawn_methods.rb +3 -7
- data/lib/switchman/active_record/statement_cache.rb +4 -4
- data/lib/switchman/active_record/table_definition.rb +1 -1
- data/lib/switchman/active_record/tasks/database_tasks.rb +6 -1
- data/lib/switchman/active_record/test_fixtures.rb +71 -25
- data/lib/switchman/active_support/cache.rb +9 -4
- data/lib/switchman/arel.rb +16 -25
- data/lib/switchman/call_super.rb +2 -8
- data/lib/switchman/database_server.rb +67 -48
- data/lib/switchman/default_shard.rb +14 -3
- data/lib/switchman/engine.rb +35 -23
- data/lib/switchman/environment.rb +2 -2
- data/lib/switchman/errors.rb +13 -0
- data/lib/switchman/guard_rail/relation.rb +1 -2
- data/lib/switchman/parallel.rb +6 -6
- data/lib/switchman/r_spec_helper.rb +12 -11
- data/lib/switchman/shard.rb +168 -68
- data/lib/switchman/sharded_instrumenter.rb +9 -3
- data/lib/switchman/standard_error.rb +4 -0
- data/lib/switchman/test_helper.rb +3 -3
- data/lib/switchman/version.rb +1 -1
- data/lib/switchman.rb +27 -15
- data/lib/tasks/switchman.rake +96 -60
- metadata +28 -173
|
@@ -4,39 +4,85 @@ module Switchman
|
|
|
4
4
|
module ActiveRecord
|
|
5
5
|
module TestFixtures
|
|
6
6
|
FORBIDDEN_DB_ENVS = %i[development production].freeze
|
|
7
|
-
def setup_fixtures(config = ::ActiveRecord::Base)
|
|
8
|
-
super
|
|
9
7
|
|
|
10
|
-
|
|
8
|
+
if ::Rails.version < "7.2"
|
|
9
|
+
def setup_fixtures(config = ::ActiveRecord::Base)
|
|
10
|
+
super
|
|
11
|
+
return unless run_in_transaction?
|
|
11
12
|
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
13
|
+
# Replace the one that activerecord natively uses with a switchman-optimized one
|
|
14
|
+
::ActiveSupport::Notifications.unsubscribe(@connection_subscriber)
|
|
15
|
+
# Code adapted from the code in rails proper
|
|
16
|
+
@connection_subscriber =
|
|
17
|
+
::ActiveSupport::Notifications.subscribe("!connection.active_record") do |_, _, _, _, payload|
|
|
18
|
+
spec_name = (payload[:connection_name] if payload.key?(:connection_name))
|
|
19
|
+
shard = payload[:shard] if payload.key?(:shard)
|
|
19
20
|
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
21
|
+
if spec_name && !FORBIDDEN_DB_ENVS.include?(shard)
|
|
22
|
+
begin
|
|
23
|
+
connection = ::ActiveRecord::Base.connection_handler.retrieve_connection(spec_name, shard:)
|
|
24
|
+
connection.connect! # eagerly validate the connection
|
|
25
|
+
rescue ::ActiveRecord::ConnectionNotEstablished
|
|
26
|
+
connection = nil
|
|
27
|
+
end
|
|
26
28
|
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
29
|
+
if connection
|
|
30
|
+
setup_shared_connection_pool
|
|
31
|
+
unless @fixture_connections.include?(connection)
|
|
32
|
+
connection.begin_transaction joinable: false, _lazy: false
|
|
33
|
+
connection.pool.lock_thread = true if lock_threads
|
|
34
|
+
@fixture_connections << connection
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|
|
31
38
|
end
|
|
32
|
-
end
|
|
33
39
|
end
|
|
34
|
-
end
|
|
35
40
|
|
|
36
|
-
|
|
37
|
-
|
|
41
|
+
def enlist_fixture_connections
|
|
42
|
+
setup_shared_connection_pool
|
|
38
43
|
|
|
39
|
-
|
|
44
|
+
::ActiveRecord::Base.connection_handler.connection_pool_list(:primary).reject do |cp|
|
|
45
|
+
FORBIDDEN_DB_ENVS.include?(cp.db_config.env_name.to_sym)
|
|
46
|
+
end.map(&:connection)
|
|
47
|
+
end
|
|
48
|
+
else
|
|
49
|
+
def setup_transactional_fixtures
|
|
50
|
+
setup_shared_connection_pool
|
|
51
|
+
|
|
52
|
+
# Begin transactions for connections already established
|
|
53
|
+
# INST: :writing -> :primary
|
|
54
|
+
@fixture_connection_pools = ::ActiveRecord::Base.connection_handler.connection_pool_list(:primary)
|
|
55
|
+
# INST: filter by FORBIDDEN_DB_ENVS
|
|
56
|
+
@fixture_connection_pools = @fixture_connection_pools.reject do |cp|
|
|
57
|
+
FORBIDDEN_DB_ENVS.include?(cp.db_config.env_name.to_sym)
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
@fixture_connection_pools.each do |pool|
|
|
61
|
+
pool.pin_connection!(lock_threads)
|
|
62
|
+
pool.lease_connection
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# When connections are established in the future, begin a transaction too
|
|
66
|
+
@connection_subscriber = ::ActiveSupport::Notifications
|
|
67
|
+
.subscribe("!connection.active_record") do |_, _, _, _, payload|
|
|
68
|
+
connection_name = payload[:connection_name] if payload.key?(:connection_name)
|
|
69
|
+
shard = payload[:shard] if payload.key?(:shard)
|
|
70
|
+
|
|
71
|
+
# INST: filter by FORBIDDEN_DB_ENVS
|
|
72
|
+
if connection_name && !FORBIDDEN_DB_ENVS.include?(shard)
|
|
73
|
+
pool = ::ActiveRecord::Base.connection_handler.retrieve_connection_pool(connection_name, shard:)
|
|
74
|
+
if pool
|
|
75
|
+
setup_shared_connection_pool
|
|
76
|
+
|
|
77
|
+
unless @fixture_connection_pools.include?(pool)
|
|
78
|
+
pool.pin_connection!(lock_threads)
|
|
79
|
+
pool.lease_connection
|
|
80
|
+
@fixture_connection_pools << pool
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
end
|
|
40
86
|
end
|
|
41
87
|
end
|
|
42
88
|
end
|
|
@@ -22,9 +22,14 @@ module Switchman
|
|
|
22
22
|
|
|
23
23
|
def lookup_store(*store_options)
|
|
24
24
|
store = super
|
|
25
|
-
#
|
|
26
|
-
#
|
|
27
|
-
|
|
25
|
+
# must use the string name, otherwise it will try to auto-load the constant
|
|
26
|
+
# and we don't want to require redis in this file (since it's not a hard dependency)
|
|
27
|
+
# rubocop:disable Style/ClassEqualityComparison
|
|
28
|
+
if store.class.name == "ActiveSupport::Cache::RedisCacheStore" &&
|
|
29
|
+
!(::ActiveSupport::Cache::RedisCacheStore <= RedisCacheStore)
|
|
30
|
+
::ActiveSupport::Cache::RedisCacheStore.prepend(RedisCacheStore)
|
|
31
|
+
end
|
|
32
|
+
# rubocop:enable Style/ClassEqualityComparison
|
|
28
33
|
store.options[:namespace] ||= -> { Shard.current.default? ? nil : "shard_#{Shard.current.id}" }
|
|
29
34
|
store
|
|
30
35
|
end
|
|
@@ -33,7 +38,7 @@ module Switchman
|
|
|
33
38
|
module RedisCacheStore
|
|
34
39
|
def clear(namespace: nil, **)
|
|
35
40
|
# RedisCacheStore tries to be smart and only clear the cache under your namespace, if you have one set
|
|
36
|
-
# unfortunately, it
|
|
41
|
+
# unfortunately, it doesn't work using redis clustering because of the way redis keys are distributed
|
|
37
42
|
# fortunately, we can assume we control the entire instance, because we set up the namespacing, so just
|
|
38
43
|
# always unset it temporarily for clear calls
|
|
39
44
|
namespace = nil # rubocop:disable Lint/ShadowedArgument
|
data/lib/switchman/arel.rb
CHANGED
|
@@ -13,38 +13,29 @@ module Switchman
|
|
|
13
13
|
# rubocop:disable Naming/MethodName
|
|
14
14
|
# rubocop:disable Naming/MethodParameterName
|
|
15
15
|
|
|
16
|
+
def visit_Arel_Nodes_Cte(o, collector)
|
|
17
|
+
collector << quote_local_table_name(o.name)
|
|
18
|
+
collector << " AS "
|
|
19
|
+
|
|
20
|
+
case o.materialized
|
|
21
|
+
when true
|
|
22
|
+
collector << "MATERIALIZED "
|
|
23
|
+
when false
|
|
24
|
+
collector << "NOT MATERIALIZED "
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
visit o.relation, collector
|
|
28
|
+
end
|
|
29
|
+
|
|
16
30
|
def visit_Arel_Nodes_TableAlias(o, collector)
|
|
17
31
|
collector = visit o.relation, collector
|
|
18
|
-
collector <<
|
|
32
|
+
collector << " "
|
|
19
33
|
collector << quote_local_table_name(o.name)
|
|
20
34
|
end
|
|
21
35
|
|
|
22
36
|
def visit_Arel_Attributes_Attribute(o, collector)
|
|
23
37
|
join_name = o.relation.table_alias || o.relation.name
|
|
24
|
-
collector << quote_local_table_name(join_name) <<
|
|
25
|
-
end
|
|
26
|
-
|
|
27
|
-
def visit_Arel_Nodes_HomogeneousIn(o, collector)
|
|
28
|
-
collector.preparable = false
|
|
29
|
-
|
|
30
|
-
collector << quote_local_table_name(o.table_name) << '.' << quote_column_name(o.column_name)
|
|
31
|
-
|
|
32
|
-
collector << if o.type == :in
|
|
33
|
-
' IN ('
|
|
34
|
-
else
|
|
35
|
-
' NOT IN ('
|
|
36
|
-
end
|
|
37
|
-
|
|
38
|
-
values = o.casted_values
|
|
39
|
-
|
|
40
|
-
if values.empty?
|
|
41
|
-
collector << @connection.quote(nil)
|
|
42
|
-
else
|
|
43
|
-
collector.add_binds(values, o.proc_for_binds, &bind_block)
|
|
44
|
-
end
|
|
45
|
-
|
|
46
|
-
collector << ')'
|
|
47
|
-
collector
|
|
38
|
+
collector << quote_local_table_name(join_name) << "." << quote_column_name(o.name)
|
|
48
39
|
end
|
|
49
40
|
|
|
50
41
|
# rubocop:enable Naming/MethodName
|
data/lib/switchman/call_super.rb
CHANGED
|
@@ -12,14 +12,8 @@ module Switchman
|
|
|
12
12
|
method.super_method
|
|
13
13
|
end
|
|
14
14
|
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
super_method_above(method, above_module).call(*args, &block)
|
|
18
|
-
end
|
|
19
|
-
else
|
|
20
|
-
def call_super(method, above_module, *args, **kwargs, &block)
|
|
21
|
-
super_method_above(method, above_module).call(*args, **kwargs, &block)
|
|
22
|
-
end
|
|
15
|
+
def call_super(method, above_module, ...)
|
|
16
|
+
super_method_above(method, above_module).call(...)
|
|
23
17
|
end
|
|
24
18
|
end
|
|
25
19
|
end
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require
|
|
3
|
+
require "securerandom"
|
|
4
4
|
|
|
5
5
|
module Switchman
|
|
6
6
|
class DatabaseServer
|
|
@@ -10,19 +10,23 @@ module Switchman
|
|
|
10
10
|
attr_accessor :creating_new_shard
|
|
11
11
|
attr_reader :all_roles
|
|
12
12
|
|
|
13
|
+
include Enumerable
|
|
14
|
+
|
|
15
|
+
delegate :each, to: :all
|
|
16
|
+
|
|
13
17
|
def all
|
|
14
18
|
database_servers.values
|
|
15
19
|
end
|
|
16
20
|
|
|
17
21
|
def find(id_or_all)
|
|
18
22
|
return all if id_or_all == :all
|
|
19
|
-
return id_or_all.
|
|
23
|
+
return id_or_all.filter_map { |id| database_servers[id || ::Rails.env] }.uniq if id_or_all.is_a?(Array)
|
|
20
24
|
|
|
21
25
|
database_servers[id_or_all || ::Rails.env]
|
|
22
26
|
end
|
|
23
27
|
|
|
24
28
|
def create(settings = {})
|
|
25
|
-
raise
|
|
29
|
+
raise "database servers should be set up in database.yml" unless ::Rails.env.test?
|
|
26
30
|
|
|
27
31
|
id = settings[:id]
|
|
28
32
|
unless id
|
|
@@ -50,6 +54,10 @@ module Switchman
|
|
|
50
54
|
all.each { |db| db.guard! if db.config[:prefer_secondary] }
|
|
51
55
|
end
|
|
52
56
|
|
|
57
|
+
def regions
|
|
58
|
+
@regions ||= all.filter_map(&:region).uniq.sort
|
|
59
|
+
end
|
|
60
|
+
|
|
53
61
|
private
|
|
54
62
|
|
|
55
63
|
def reference_role(role)
|
|
@@ -64,8 +72,8 @@ module Switchman
|
|
|
64
72
|
@database_servers = {}.with_indifferent_access
|
|
65
73
|
roles = []
|
|
66
74
|
::ActiveRecord::Base.configurations.configurations.each do |config|
|
|
67
|
-
if config.name.include?(
|
|
68
|
-
name, role = config.name.split(
|
|
75
|
+
if config.name.include?("/")
|
|
76
|
+
name, role = config.name.split("/")
|
|
69
77
|
else
|
|
70
78
|
name, role = config.env_name, config.name
|
|
71
79
|
end
|
|
@@ -81,6 +89,8 @@ module Switchman
|
|
|
81
89
|
# Do this after so that all database servers for all roles are established and we won't prematurely
|
|
82
90
|
# configure a connection for the wrong role
|
|
83
91
|
@all_roles = roles.uniq
|
|
92
|
+
return @database_servers if @database_servers.empty?
|
|
93
|
+
|
|
84
94
|
Shard.send(:configure_connects_to)
|
|
85
95
|
end
|
|
86
96
|
@database_servers
|
|
@@ -110,8 +120,9 @@ module Switchman
|
|
|
110
120
|
self.class.send(:database_servers).delete(id) if id
|
|
111
121
|
Shard.sharded_models.each do |klass|
|
|
112
122
|
self.class.all_roles.each do |role|
|
|
113
|
-
klass.connection_handler.remove_connection_pool(klass.connection_specification_name,
|
|
114
|
-
|
|
123
|
+
klass.connection_handler.remove_connection_pool(klass.connection_specification_name,
|
|
124
|
+
role:,
|
|
125
|
+
shard: id.to_sym)
|
|
115
126
|
end
|
|
116
127
|
end
|
|
117
128
|
end
|
|
@@ -137,16 +148,41 @@ module Switchman
|
|
|
137
148
|
end
|
|
138
149
|
end
|
|
139
150
|
|
|
151
|
+
def region
|
|
152
|
+
config[:region]
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
# @param region [String, Array<String>] the region(s) to check against
|
|
156
|
+
# @return true if the database server doesn't have a region, or it
|
|
157
|
+
# matches the specified region
|
|
158
|
+
def in_region?(region)
|
|
159
|
+
!self.region || (region.is_a?(Array) ? region.include?(self.region) : self.region == region)
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
# @return true if the database server doesn't have a region, Switchman is
|
|
163
|
+
# not configured with a region, or the database server's region matches
|
|
164
|
+
# Switchman's current region
|
|
165
|
+
def in_current_region?
|
|
166
|
+
unless instance_variable_defined?(:@in_current_region)
|
|
167
|
+
@in_current_region = !region ||
|
|
168
|
+
!Switchman.region ||
|
|
169
|
+
region == Switchman.region
|
|
170
|
+
end
|
|
171
|
+
@in_current_region
|
|
172
|
+
end
|
|
173
|
+
|
|
140
174
|
# locks this db to a specific environment, except for
|
|
141
175
|
# when doing writes (then it falls back to the current
|
|
142
176
|
# value of GuardRail.environment)
|
|
143
177
|
def guard!(environment = :secondary)
|
|
144
178
|
DatabaseServer.send(:reference_role, environment)
|
|
145
|
-
::ActiveRecord::Base.connected_to_stack << { shard_roles: { id.to_sym => environment },
|
|
179
|
+
::ActiveRecord::Base.connected_to_stack << { shard_roles: { id.to_sym => environment },
|
|
180
|
+
klasses: [::ActiveRecord::Base] }
|
|
146
181
|
end
|
|
147
182
|
|
|
148
183
|
def unguard!
|
|
149
|
-
::ActiveRecord::Base.connected_to_stack << { shard_roles: { id.to_sym => :_switchman_inherit },
|
|
184
|
+
::ActiveRecord::Base.connected_to_stack << { shard_roles: { id.to_sym => :_switchman_inherit },
|
|
185
|
+
klasses: [::ActiveRecord::Base] }
|
|
150
186
|
end
|
|
151
187
|
|
|
152
188
|
def unguard
|
|
@@ -162,7 +198,7 @@ module Switchman
|
|
|
162
198
|
|
|
163
199
|
def shards
|
|
164
200
|
if id == ::Rails.env
|
|
165
|
-
Shard.where(
|
|
201
|
+
Shard.where("database_server_id IS NULL OR database_server_id=?", id)
|
|
166
202
|
else
|
|
167
203
|
Shard.where(database_server_id: id)
|
|
168
204
|
end
|
|
@@ -182,12 +218,12 @@ module Switchman
|
|
|
182
218
|
if config_create_statement
|
|
183
219
|
create_commands = Array(config_create_statement).dup
|
|
184
220
|
create_statement = lambda {
|
|
185
|
-
create_commands.map { |statement| format(statement, name
|
|
221
|
+
create_commands.map { |statement| format(statement, name:, password:) }
|
|
186
222
|
}
|
|
187
223
|
end
|
|
188
224
|
|
|
189
225
|
id ||= begin
|
|
190
|
-
id_seq = Shard.connection.quote(Shard.connection.quote_table_name(
|
|
226
|
+
id_seq = Shard.connection.quote(Shard.connection.quote_table_name("switchman_shards_id_seq"))
|
|
191
227
|
next_id = Shard.connection.select_value("SELECT nextval(#{id_seq})")
|
|
192
228
|
next_id.to_i
|
|
193
229
|
end
|
|
@@ -200,31 +236,30 @@ module Switchman
|
|
|
200
236
|
self.class.creating_new_shard = true
|
|
201
237
|
DatabaseServer.send(:reference_role, :deploy)
|
|
202
238
|
::ActiveRecord::Base.connected_to(shard: self.id.to_sym, role: :deploy) do
|
|
203
|
-
shard = Shard.create!(id
|
|
204
|
-
name
|
|
239
|
+
shard = Shard.create!(id:,
|
|
240
|
+
name:,
|
|
205
241
|
database_server_id: self.id)
|
|
206
242
|
if create_statement
|
|
207
|
-
if ::ActiveRecord::Base.connection.select_value(
|
|
243
|
+
if ::ActiveRecord::Base.connection.select_value(
|
|
244
|
+
"SELECT 1 FROM pg_namespace WHERE nspname=#{::ActiveRecord::Base.connection.quote(name)}"
|
|
245
|
+
)
|
|
208
246
|
schema_already_existed = true
|
|
209
|
-
raise
|
|
247
|
+
raise "This schema already exists; cannot overwrite"
|
|
210
248
|
end
|
|
211
249
|
Array(create_statement.call).each do |stmt|
|
|
212
250
|
::ActiveRecord::Base.connection.execute(stmt)
|
|
213
251
|
end
|
|
214
252
|
end
|
|
215
|
-
if config[:adapter] ==
|
|
216
|
-
old_proc = ::ActiveRecord::Base.connection.raw_connection.set_notice_processor
|
|
217
|
-
end
|
|
253
|
+
if config[:adapter] == "postgresql"
|
|
254
|
+
old_proc = ::ActiveRecord::Base.connection.raw_connection.set_notice_processor {}
|
|
218
255
|
end
|
|
219
256
|
old_verbose = ::ActiveRecord::Migration.verbose
|
|
220
257
|
::ActiveRecord::Migration.verbose = false
|
|
221
258
|
|
|
222
259
|
unless schema == false
|
|
223
260
|
shard.activate do
|
|
224
|
-
|
|
225
|
-
::ActiveRecord::
|
|
226
|
-
::ActiveRecord::Base.connection.migration_context.migrate
|
|
227
|
-
end
|
|
261
|
+
::ActiveRecord::Base.connection.transaction(requires_new: true) do
|
|
262
|
+
::ActiveRecord::MigrationContext.new(::ActiveRecord::Migrator.migrations_paths).migrate
|
|
228
263
|
end
|
|
229
264
|
|
|
230
265
|
::ActiveRecord::Base.descendants.reject do |m|
|
|
@@ -240,7 +275,6 @@ module Switchman
|
|
|
240
275
|
rescue
|
|
241
276
|
shard&.destroy
|
|
242
277
|
shard&.drop_database rescue nil unless schema_already_existed
|
|
243
|
-
reset_column_information unless schema == false rescue nil
|
|
244
278
|
raise
|
|
245
279
|
ensure
|
|
246
280
|
self.class.creating_new_shard = false
|
|
@@ -265,33 +299,18 @@ module Switchman
|
|
|
265
299
|
end
|
|
266
300
|
|
|
267
301
|
def primary_shard
|
|
268
|
-
unless
|
|
269
|
-
# if sharding isn't fully set up yet, we may not be able to query the shards table
|
|
270
|
-
@primary_shard = Shard.default if Shard.default.database_server == self
|
|
271
|
-
@primary_shard ||= shards.where(name: nil).first
|
|
272
|
-
end
|
|
273
|
-
@primary_shard
|
|
274
|
-
end
|
|
275
|
-
|
|
276
|
-
private
|
|
302
|
+
return nil unless primary_shard_id
|
|
277
303
|
|
|
278
|
-
|
|
279
|
-
::ActiveRecord::Base.descendants.reject { |m| m <= UnshardedRecord }.each(&:reset_column_information)
|
|
304
|
+
Shard.lookup(primary_shard_id)
|
|
280
305
|
end
|
|
281
306
|
|
|
282
|
-
def
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
connection_pool.set_schema_cache(temporary_schema_cache)
|
|
290
|
-
|
|
291
|
-
yield
|
|
292
|
-
|
|
293
|
-
connection_pool.set_schema_cache(previous_schema_cache)
|
|
294
|
-
reset_column_information
|
|
307
|
+
def primary_shard_id
|
|
308
|
+
unless instance_variable_defined?(:@primary_shard_id)
|
|
309
|
+
# if sharding isn't fully set up yet, we may not be able to query the shards table
|
|
310
|
+
@primary_shard_id = Shard.default.id if Shard.default.database_server == self
|
|
311
|
+
@primary_shard_id ||= shards.where(name: nil).first&.id
|
|
312
|
+
end
|
|
313
|
+
@primary_shard_id
|
|
295
314
|
end
|
|
296
315
|
end
|
|
297
316
|
end
|
|
@@ -3,9 +3,10 @@
|
|
|
3
3
|
module Switchman
|
|
4
4
|
class DefaultShard
|
|
5
5
|
def id
|
|
6
|
-
|
|
6
|
+
"default"
|
|
7
7
|
end
|
|
8
|
-
|
|
8
|
+
alias_method :cache_key, :id
|
|
9
|
+
|
|
9
10
|
def activate(*_classes)
|
|
10
11
|
yield
|
|
11
12
|
end
|
|
@@ -57,8 +58,18 @@ module Switchman
|
|
|
57
58
|
self
|
|
58
59
|
end
|
|
59
60
|
|
|
61
|
+
def region; end
|
|
62
|
+
|
|
63
|
+
def in_region?(_region)
|
|
64
|
+
true
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def in_current_region?
|
|
68
|
+
true
|
|
69
|
+
end
|
|
70
|
+
|
|
60
71
|
def _dump(_depth)
|
|
61
|
-
|
|
72
|
+
""
|
|
62
73
|
end
|
|
63
74
|
|
|
64
75
|
def self._load(_str)
|
data/lib/switchman/engine.rb
CHANGED
|
@@ -4,25 +4,16 @@ module Switchman
|
|
|
4
4
|
class Engine < ::Rails::Engine
|
|
5
5
|
isolate_namespace Switchman
|
|
6
6
|
|
|
7
|
-
# enable Rails 6.1 style connection handling
|
|
8
|
-
config.active_record.legacy_connection_handling = false
|
|
9
7
|
config.active_record.writing_role = :primary
|
|
10
8
|
|
|
11
9
|
::GuardRail.singleton_class.prepend(GuardRail::ClassMethods)
|
|
12
10
|
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
initialize_dependency_mechanism = ::Rails::Application::Bootstrap.initializers.find { |i| i.name == :initialize_dependency_mechanism }
|
|
17
|
-
initialize_dependency_mechanism.instance_variable_get(:@options)[:after] = :set_autoload_paths
|
|
18
|
-
end
|
|
19
|
-
|
|
20
|
-
initializer 'switchman.active_record_patch',
|
|
21
|
-
before: 'active_record.initialize_database',
|
|
22
|
-
after: (::Rails.version < '7.0' ? :initialize_dependency_mechanism : :setup_once_autoloader) do
|
|
11
|
+
initializer "switchman.active_record_patch",
|
|
12
|
+
before: "active_record.initialize_database",
|
|
13
|
+
after: :setup_once_autoloader do
|
|
23
14
|
::ActiveSupport.on_load(:active_record) do
|
|
24
15
|
# Switchman requires postgres, so just always load the pg adapter
|
|
25
|
-
require
|
|
16
|
+
require "active_record/connection_adapters/postgresql_adapter"
|
|
26
17
|
|
|
27
18
|
self.default_shard = ::Rails.env.to_sym
|
|
28
19
|
self.default_role = :primary
|
|
@@ -50,14 +41,27 @@ module Switchman
|
|
|
50
41
|
::ActiveRecord::Associations::CollectionProxy.include(ActiveRecord::Associations::CollectionProxy)
|
|
51
42
|
|
|
52
43
|
::ActiveRecord::Associations::Preloader::Association.prepend(ActiveRecord::Associations::Preloader::Association)
|
|
53
|
-
::ActiveRecord::Associations::Preloader::Association::LoaderRecords.prepend(
|
|
44
|
+
::ActiveRecord::Associations::Preloader::Association::LoaderRecords.prepend(
|
|
45
|
+
ActiveRecord::Associations::Preloader::Association::LoaderRecords
|
|
46
|
+
)
|
|
54
47
|
::ActiveRecord::ConnectionAdapters::AbstractAdapter.prepend(ActiveRecord::AbstractAdapter)
|
|
48
|
+
::ActiveRecord::ConnectionAdapters::ConnectionHandler.prepend(ActiveRecord::ConnectionHandler)
|
|
55
49
|
::ActiveRecord::ConnectionAdapters::ConnectionPool.prepend(ActiveRecord::ConnectionPool)
|
|
56
50
|
::ActiveRecord::ConnectionAdapters::AbstractAdapter.prepend(ActiveRecord::QueryCache)
|
|
57
51
|
::ActiveRecord::ConnectionAdapters::PostgreSQLAdapter.prepend(ActiveRecord::PostgreSQLAdapter)
|
|
52
|
+
# https://github.com/rails/rails/commit/0016280f4fde55d96738887093dc333aae0d107b
|
|
53
|
+
if ::Rails.version < "7.2"
|
|
54
|
+
::ActiveRecord::ConnectionAdapters::PostgreSQLAdapter.prepend(ActiveRecord::PostgreSQLAdapter::ClassMethods)
|
|
55
|
+
else
|
|
56
|
+
::ActiveRecord::ConnectionAdapters::PostgreSQLAdapter.singleton_class.prepend(
|
|
57
|
+
ActiveRecord::PostgreSQLAdapter::ClassMethods
|
|
58
|
+
)
|
|
59
|
+
end
|
|
58
60
|
|
|
59
61
|
::ActiveRecord::DatabaseConfigurations.prepend(ActiveRecord::DatabaseConfigurations)
|
|
60
|
-
::ActiveRecord::DatabaseConfigurations::DatabaseConfig.prepend(
|
|
62
|
+
::ActiveRecord::DatabaseConfigurations::DatabaseConfig.prepend(
|
|
63
|
+
ActiveRecord::DatabaseConfigurations::DatabaseConfig
|
|
64
|
+
)
|
|
61
65
|
|
|
62
66
|
::ActiveRecord::LogSubscriber.prepend(ActiveRecord::LogSubscriber)
|
|
63
67
|
::ActiveRecord::Migration.prepend(ActiveRecord::Migration)
|
|
@@ -65,6 +69,11 @@ module Switchman
|
|
|
65
69
|
::ActiveRecord::MigrationContext.prepend(ActiveRecord::MigrationContext)
|
|
66
70
|
::ActiveRecord::Migrator.prepend(ActiveRecord::Migrator)
|
|
67
71
|
|
|
72
|
+
if ::Rails.version > "7.1.3"
|
|
73
|
+
::ActiveRecord::PendingMigrationConnection.singleton_class
|
|
74
|
+
.include(ActiveRecord::PendingMigrationConnection::ClassMethods)
|
|
75
|
+
end
|
|
76
|
+
|
|
68
77
|
::ActiveRecord::Reflection::AbstractReflection.include(ActiveRecord::Reflection::AbstractReflection)
|
|
69
78
|
::ActiveRecord::Reflection::AssociationReflection.prepend(ActiveRecord::Reflection::AssociationScopeCache)
|
|
70
79
|
::ActiveRecord::Reflection::ThroughReflection.prepend(ActiveRecord::Reflection::AssociationScopeCache)
|
|
@@ -74,11 +83,13 @@ module Switchman
|
|
|
74
83
|
::ActiveRecord::Relation.include(ActiveRecord::QueryMethods)
|
|
75
84
|
::ActiveRecord::Relation.prepend(GuardRail::Relation)
|
|
76
85
|
::ActiveRecord::Relation.prepend(ActiveRecord::Relation)
|
|
86
|
+
::ActiveRecord::Relation.prepend(ActiveRecord::Relation::InsertUpsertAll) if ::Rails.version >= "7.2"
|
|
77
87
|
::ActiveRecord::Relation.include(ActiveRecord::SpawnMethods)
|
|
78
88
|
::ActiveRecord::Relation.include(CallSuper)
|
|
79
89
|
|
|
80
|
-
::ActiveRecord::PredicateBuilder::
|
|
81
|
-
|
|
90
|
+
::ActiveRecord::PredicateBuilder::PolymorphicArrayValue.prepend(
|
|
91
|
+
ActiveRecord::PredicateBuilder::PolymorphicArrayValue
|
|
92
|
+
)
|
|
82
93
|
|
|
83
94
|
::ActiveRecord::Tasks::DatabaseTasks.singleton_class.prepend(ActiveRecord::Tasks::DatabaseTasks)
|
|
84
95
|
|
|
@@ -96,17 +107,18 @@ module Switchman
|
|
|
96
107
|
|
|
97
108
|
::ActiveRecord::ConnectionAdapters::TableDefinition.prepend(ActiveRecord::TableDefinition)
|
|
98
109
|
end
|
|
99
|
-
# Ensure that ActiveRecord::Base is always loaded before any app-level
|
|
110
|
+
# Ensure that ActiveRecord::Base is always loaded before any app-level
|
|
111
|
+
# initializers can go try to load Switchman::Shard or we get a loop
|
|
100
112
|
::ActiveRecord::Base
|
|
101
113
|
end
|
|
102
114
|
|
|
103
|
-
initializer
|
|
115
|
+
initializer "switchman.error_patch", after: "active_record.initialize_database" do
|
|
104
116
|
::ActiveSupport.on_load(:active_record) do
|
|
105
117
|
::StandardError.include(StandardError)
|
|
106
118
|
end
|
|
107
119
|
end
|
|
108
120
|
|
|
109
|
-
initializer
|
|
121
|
+
initializer "switchman.initialize_cache", before: :initialize_cache, after: "active_record.initialize_database" do
|
|
110
122
|
::ActiveSupport::Cache.singleton_class.prepend(ActiveSupport::Cache::ClassMethods)
|
|
111
123
|
|
|
112
124
|
# if we haven't already setup our cache map out-of-band, set it up from
|
|
@@ -130,11 +142,11 @@ module Switchman
|
|
|
130
142
|
Switchman.config[:cache_map][::Rails.env] = value
|
|
131
143
|
end
|
|
132
144
|
|
|
133
|
-
middlewares = Switchman.config[:cache_map].values.
|
|
145
|
+
middlewares = Switchman.config[:cache_map].values.filter_map do |store|
|
|
134
146
|
store.middleware if store.respond_to?(:middleware)
|
|
135
|
-
end.
|
|
147
|
+
end.uniq
|
|
136
148
|
middlewares.each do |middleware|
|
|
137
|
-
config.middleware.insert_before(
|
|
149
|
+
config.middleware.insert_before("Rack::Runtime", middleware)
|
|
138
150
|
end
|
|
139
151
|
|
|
140
152
|
# prevent :initialize_cache from trying to (or needing to) set
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require
|
|
3
|
+
require "etc"
|
|
4
4
|
|
|
5
5
|
module Switchman
|
|
6
6
|
class Environment
|
|
7
|
-
def self.cpu_count(nproc_bin =
|
|
7
|
+
def self.cpu_count(nproc_bin = "nproc")
|
|
8
8
|
return Etc.nprocessors if Etc.respond_to?(:nprocessors)
|
|
9
9
|
|
|
10
10
|
`#{nproc_bin}`.to_i
|
data/lib/switchman/errors.rb
CHANGED
|
@@ -2,8 +2,21 @@
|
|
|
2
2
|
|
|
3
3
|
module Switchman
|
|
4
4
|
module Errors
|
|
5
|
+
class ManuallyCreatedShadowRecordError < RuntimeError
|
|
6
|
+
DEFAULT_MSG = "It looks like you're trying to manually create a shadow record. " \
|
|
7
|
+
"Please use Switchman::ActiveRecord::Base#save_shadow_record instead."
|
|
8
|
+
|
|
9
|
+
def initialize(msg = DEFAULT_MSG)
|
|
10
|
+
super
|
|
11
|
+
end
|
|
12
|
+
end
|
|
13
|
+
|
|
5
14
|
class NonExistentShardError < RuntimeError; end
|
|
6
15
|
|
|
7
16
|
class ParallelShardExecError < RuntimeError; end
|
|
17
|
+
|
|
18
|
+
class ShadowRecordError < RuntimeError; end
|
|
19
|
+
|
|
20
|
+
class UnshardedTableError < RuntimeError; end
|
|
8
21
|
end
|
|
9
22
|
end
|
|
@@ -13,9 +13,8 @@ module Switchman
|
|
|
13
13
|
end
|
|
14
14
|
|
|
15
15
|
%w[update_all delete_all].each do |method|
|
|
16
|
-
arg_params = RUBY_VERSION <= '2.8' ? '*args' : '*args, **kwargs'
|
|
17
16
|
class_eval <<-RUBY, __FILE__, __LINE__ + 1
|
|
18
|
-
def #{method}(
|
|
17
|
+
def #{method}(*args, **kwargs)
|
|
19
18
|
db = Shard.current(connection_class_for_self).database_server
|
|
20
19
|
db.unguard { super }
|
|
21
20
|
end
|