switchman 3.1.0 → 3.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Rakefile +15 -14
- data/db/migrate/20180828183945_add_default_shard_index.rb +1 -1
- data/db/migrate/20190114212900_add_unique_name_indexes.rb +10 -4
- data/lib/switchman/active_record/associations.rb +97 -17
- data/lib/switchman/active_record/attribute_methods.rb +72 -43
- data/lib/switchman/active_record/base.rb +107 -21
- data/lib/switchman/active_record/calculations.rb +37 -33
- data/lib/switchman/active_record/connection_pool.rb +21 -2
- data/lib/switchman/active_record/database_configurations.rb +12 -7
- data/lib/switchman/active_record/finder_methods.rb +1 -1
- data/lib/switchman/active_record/log_subscriber.rb +2 -2
- data/lib/switchman/active_record/migration.rb +35 -8
- data/lib/switchman/active_record/persistence.rb +8 -0
- data/lib/switchman/active_record/postgresql_adapter.rb +11 -10
- data/lib/switchman/active_record/query_cache.rb +1 -1
- data/lib/switchman/active_record/query_methods.rb +172 -132
- data/lib/switchman/active_record/relation.rb +21 -11
- data/lib/switchman/active_record/spawn_methods.rb +2 -2
- data/lib/switchman/active_record/statement_cache.rb +9 -5
- data/lib/switchman/active_record/tasks/database_tasks.rb +1 -1
- data/lib/switchman/active_record/test_fixtures.rb +19 -16
- data/lib/switchman/active_support/cache.rb +4 -1
- data/lib/switchman/arel.rb +6 -6
- data/lib/switchman/call_super.rb +8 -2
- data/lib/switchman/database_server.rb +21 -26
- data/lib/switchman/default_shard.rb +3 -3
- data/lib/switchman/engine.rb +33 -18
- data/lib/switchman/environment.rb +2 -2
- data/lib/switchman/errors.rb +13 -0
- data/lib/switchman/guard_rail/relation.rb +2 -1
- data/lib/switchman/parallel.rb +2 -2
- data/lib/switchman/r_spec_helper.rb +10 -10
- data/lib/switchman/shard.rb +49 -32
- data/lib/switchman/sharded_instrumenter.rb +5 -1
- data/lib/switchman/shared_schema_cache.rb +11 -0
- data/lib/switchman/test_helper.rb +1 -1
- data/lib/switchman/version.rb +1 -1
- data/lib/switchman.rb +10 -4
- data/lib/tasks/switchman.rake +42 -39
- metadata +24 -9
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require
|
|
3
|
+
require "securerandom"
|
|
4
4
|
|
|
5
5
|
module Switchman
|
|
6
6
|
class DatabaseServer
|
|
@@ -16,13 +16,13 @@ module Switchman
|
|
|
16
16
|
|
|
17
17
|
def find(id_or_all)
|
|
18
18
|
return all if id_or_all == :all
|
|
19
|
-
return id_or_all.
|
|
19
|
+
return id_or_all.filter_map { |id| database_servers[id || ::Rails.env] }.uniq if id_or_all.is_a?(Array)
|
|
20
20
|
|
|
21
21
|
database_servers[id_or_all || ::Rails.env]
|
|
22
22
|
end
|
|
23
23
|
|
|
24
24
|
def create(settings = {})
|
|
25
|
-
raise
|
|
25
|
+
raise "database servers should be set up in database.yml" unless ::Rails.env.test?
|
|
26
26
|
|
|
27
27
|
id = settings[:id]
|
|
28
28
|
unless id
|
|
@@ -64,8 +64,8 @@ module Switchman
|
|
|
64
64
|
@database_servers = {}.with_indifferent_access
|
|
65
65
|
roles = []
|
|
66
66
|
::ActiveRecord::Base.configurations.configurations.each do |config|
|
|
67
|
-
if config.name.include?(
|
|
68
|
-
name, role = config.name.split(
|
|
67
|
+
if config.name.include?("/")
|
|
68
|
+
name, role = config.name.split("/")
|
|
69
69
|
else
|
|
70
70
|
name, role = config.env_name, config.name
|
|
71
71
|
end
|
|
@@ -110,8 +110,9 @@ module Switchman
|
|
|
110
110
|
self.class.send(:database_servers).delete(id) if id
|
|
111
111
|
Shard.sharded_models.each do |klass|
|
|
112
112
|
self.class.all_roles.each do |role|
|
|
113
|
-
klass.connection_handler.remove_connection_pool(klass.connection_specification_name,
|
|
114
|
-
|
|
113
|
+
klass.connection_handler.remove_connection_pool(klass.connection_specification_name,
|
|
114
|
+
role: role,
|
|
115
|
+
shard: id.to_sym)
|
|
115
116
|
end
|
|
116
117
|
end
|
|
117
118
|
end
|
|
@@ -142,11 +143,13 @@ module Switchman
|
|
|
142
143
|
# value of GuardRail.environment)
|
|
143
144
|
def guard!(environment = :secondary)
|
|
144
145
|
DatabaseServer.send(:reference_role, environment)
|
|
145
|
-
::ActiveRecord::Base.connected_to_stack << { shard_roles: { id.to_sym => environment },
|
|
146
|
+
::ActiveRecord::Base.connected_to_stack << { shard_roles: { id.to_sym => environment },
|
|
147
|
+
klasses: [::ActiveRecord::Base] }
|
|
146
148
|
end
|
|
147
149
|
|
|
148
150
|
def unguard!
|
|
149
|
-
::ActiveRecord::Base.connected_to_stack << { shard_roles: { id.to_sym => :_switchman_inherit },
|
|
151
|
+
::ActiveRecord::Base.connected_to_stack << { shard_roles: { id.to_sym => :_switchman_inherit },
|
|
152
|
+
klasses: [::ActiveRecord::Base] }
|
|
150
153
|
end
|
|
151
154
|
|
|
152
155
|
def unguard
|
|
@@ -162,7 +165,7 @@ module Switchman
|
|
|
162
165
|
|
|
163
166
|
def shards
|
|
164
167
|
if id == ::Rails.env
|
|
165
|
-
Shard.where(
|
|
168
|
+
Shard.where("database_server_id IS NULL OR database_server_id=?", id)
|
|
166
169
|
else
|
|
167
170
|
Shard.where(database_server_id: id)
|
|
168
171
|
end
|
|
@@ -187,7 +190,7 @@ module Switchman
|
|
|
187
190
|
end
|
|
188
191
|
|
|
189
192
|
id ||= begin
|
|
190
|
-
id_seq = Shard.connection.quote(Shard.connection.quote_table_name(
|
|
193
|
+
id_seq = Shard.connection.quote(Shard.connection.quote_table_name("switchman_shards_id_seq"))
|
|
191
194
|
next_id = Shard.connection.select_value("SELECT nextval(#{id_seq})")
|
|
192
195
|
next_id.to_i
|
|
193
196
|
end
|
|
@@ -204,29 +207,28 @@ module Switchman
|
|
|
204
207
|
name: name,
|
|
205
208
|
database_server_id: self.id)
|
|
206
209
|
if create_statement
|
|
207
|
-
if ::ActiveRecord::Base.connection.select_value(
|
|
210
|
+
if ::ActiveRecord::Base.connection.select_value(
|
|
211
|
+
"SELECT 1 FROM pg_namespace WHERE nspname=#{::ActiveRecord::Base.connection.quote(name)}"
|
|
212
|
+
)
|
|
208
213
|
schema_already_existed = true
|
|
209
|
-
raise
|
|
214
|
+
raise "This schema already exists; cannot overwrite"
|
|
210
215
|
end
|
|
211
216
|
Array(create_statement.call).each do |stmt|
|
|
212
217
|
::ActiveRecord::Base.connection.execute(stmt)
|
|
213
218
|
end
|
|
214
219
|
end
|
|
215
|
-
if config[:adapter] ==
|
|
216
|
-
old_proc = ::ActiveRecord::Base.connection.raw_connection.set_notice_processor
|
|
217
|
-
end
|
|
220
|
+
if config[:adapter] == "postgresql"
|
|
221
|
+
old_proc = ::ActiveRecord::Base.connection.raw_connection.set_notice_processor {}
|
|
218
222
|
end
|
|
219
223
|
old_verbose = ::ActiveRecord::Migration.verbose
|
|
220
224
|
::ActiveRecord::Migration.verbose = false
|
|
221
225
|
|
|
222
226
|
unless schema == false
|
|
223
227
|
shard.activate do
|
|
224
|
-
reset_column_information
|
|
225
|
-
|
|
226
228
|
::ActiveRecord::Base.connection.transaction(requires_new: true) do
|
|
227
229
|
::ActiveRecord::Base.connection.migration_context.migrate
|
|
228
230
|
end
|
|
229
|
-
|
|
231
|
+
|
|
230
232
|
::ActiveRecord::Base.descendants.reject do |m|
|
|
231
233
|
m <= UnshardedRecord || !m.table_exists?
|
|
232
234
|
end.each(&:define_attribute_methods)
|
|
@@ -240,7 +242,6 @@ module Switchman
|
|
|
240
242
|
rescue
|
|
241
243
|
shard&.destroy
|
|
242
244
|
shard&.drop_database rescue nil unless schema_already_existed
|
|
243
|
-
reset_column_information unless schema == false rescue nil
|
|
244
245
|
raise
|
|
245
246
|
ensure
|
|
246
247
|
self.class.creating_new_shard = false
|
|
@@ -272,11 +273,5 @@ module Switchman
|
|
|
272
273
|
end
|
|
273
274
|
@primary_shard
|
|
274
275
|
end
|
|
275
|
-
|
|
276
|
-
private
|
|
277
|
-
|
|
278
|
-
def reset_column_information
|
|
279
|
-
::ActiveRecord::Base.descendants.reject { |m| m <= UnshardedRecord }.each(&:reset_column_information)
|
|
280
|
-
end
|
|
281
276
|
end
|
|
282
277
|
end
|
|
@@ -3,9 +3,9 @@
|
|
|
3
3
|
module Switchman
|
|
4
4
|
class DefaultShard
|
|
5
5
|
def id
|
|
6
|
-
|
|
6
|
+
"default"
|
|
7
7
|
end
|
|
8
|
-
|
|
8
|
+
alias_method :cache_key, :id
|
|
9
9
|
def activate(*_classes)
|
|
10
10
|
yield
|
|
11
11
|
end
|
|
@@ -58,7 +58,7 @@ module Switchman
|
|
|
58
58
|
end
|
|
59
59
|
|
|
60
60
|
def _dump(_depth)
|
|
61
|
-
|
|
61
|
+
""
|
|
62
62
|
end
|
|
63
63
|
|
|
64
64
|
def self._load(_str)
|
data/lib/switchman/engine.rb
CHANGED
|
@@ -10,19 +10,23 @@ module Switchman
|
|
|
10
10
|
|
|
11
11
|
::GuardRail.singleton_class.prepend(GuardRail::ClassMethods)
|
|
12
12
|
|
|
13
|
-
# after :initialize_dependency_mechanism to ensure autoloading is
|
|
14
|
-
#
|
|
15
|
-
|
|
16
|
-
|
|
13
|
+
# after :initialize_dependency_mechanism to ensure autoloading is
|
|
14
|
+
# configured for any downstream initializers that care. In rails 7.0 we
|
|
15
|
+
# should be able to just use an explicit after on configuring the once
|
|
16
|
+
# autoloaders and not need to go monkey around with initializer order
|
|
17
|
+
if ::Rails.version < "7.0"
|
|
18
|
+
initialize_dependency_mechanism = ::Rails::Application::Bootstrap.initializers.find do |i|
|
|
19
|
+
i.name == :initialize_dependency_mechanism
|
|
20
|
+
end
|
|
17
21
|
initialize_dependency_mechanism.instance_variable_get(:@options)[:after] = :set_autoload_paths
|
|
18
22
|
end
|
|
19
23
|
|
|
20
|
-
initializer
|
|
21
|
-
before:
|
|
22
|
-
after: (::Rails.version <
|
|
24
|
+
initializer "switchman.active_record_patch",
|
|
25
|
+
before: "active_record.initialize_database",
|
|
26
|
+
after: ((::Rails.version < "7.0") ? :initialize_dependency_mechanism : :setup_once_autoloader) do
|
|
23
27
|
::ActiveSupport.on_load(:active_record) do
|
|
24
28
|
# Switchman requires postgres, so just always load the pg adapter
|
|
25
|
-
require
|
|
29
|
+
require "active_record/connection_adapters/postgresql_adapter"
|
|
26
30
|
|
|
27
31
|
self.default_shard = ::Rails.env.to_sym
|
|
28
32
|
self.default_role = :primary
|
|
@@ -50,14 +54,20 @@ module Switchman
|
|
|
50
54
|
::ActiveRecord::Associations::CollectionProxy.include(ActiveRecord::Associations::CollectionProxy)
|
|
51
55
|
|
|
52
56
|
::ActiveRecord::Associations::Preloader::Association.prepend(ActiveRecord::Associations::Preloader::Association)
|
|
53
|
-
|
|
57
|
+
unless ::Rails.version < "7.0"
|
|
58
|
+
::ActiveRecord::Associations::Preloader::Association::LoaderRecords.prepend(
|
|
59
|
+
ActiveRecord::Associations::Preloader::Association::LoaderRecords
|
|
60
|
+
)
|
|
61
|
+
end
|
|
54
62
|
::ActiveRecord::ConnectionAdapters::AbstractAdapter.prepend(ActiveRecord::AbstractAdapter)
|
|
55
63
|
::ActiveRecord::ConnectionAdapters::ConnectionPool.prepend(ActiveRecord::ConnectionPool)
|
|
56
64
|
::ActiveRecord::ConnectionAdapters::AbstractAdapter.prepend(ActiveRecord::QueryCache)
|
|
57
65
|
::ActiveRecord::ConnectionAdapters::PostgreSQLAdapter.prepend(ActiveRecord::PostgreSQLAdapter)
|
|
58
66
|
|
|
59
67
|
::ActiveRecord::DatabaseConfigurations.prepend(ActiveRecord::DatabaseConfigurations)
|
|
60
|
-
::ActiveRecord::DatabaseConfigurations::DatabaseConfig.prepend(
|
|
68
|
+
::ActiveRecord::DatabaseConfigurations::DatabaseConfig.prepend(
|
|
69
|
+
ActiveRecord::DatabaseConfigurations::DatabaseConfig
|
|
70
|
+
)
|
|
61
71
|
|
|
62
72
|
::ActiveRecord::LogSubscriber.prepend(ActiveRecord::LogSubscriber)
|
|
63
73
|
::ActiveRecord::Migration.prepend(ActiveRecord::Migration)
|
|
@@ -77,8 +87,12 @@ module Switchman
|
|
|
77
87
|
::ActiveRecord::Relation.include(ActiveRecord::SpawnMethods)
|
|
78
88
|
::ActiveRecord::Relation.include(CallSuper)
|
|
79
89
|
|
|
80
|
-
::ActiveRecord::PredicateBuilder::AssociationQueryValue.prepend(
|
|
81
|
-
|
|
90
|
+
::ActiveRecord::PredicateBuilder::AssociationQueryValue.prepend(
|
|
91
|
+
ActiveRecord::PredicateBuilder::AssociationQueryValue
|
|
92
|
+
)
|
|
93
|
+
::ActiveRecord::PredicateBuilder::PolymorphicArrayValue.prepend(
|
|
94
|
+
ActiveRecord::PredicateBuilder::AssociationQueryValue
|
|
95
|
+
)
|
|
82
96
|
|
|
83
97
|
::ActiveRecord::Tasks::DatabaseTasks.singleton_class.prepend(ActiveRecord::Tasks::DatabaseTasks)
|
|
84
98
|
|
|
@@ -96,17 +110,18 @@ module Switchman
|
|
|
96
110
|
|
|
97
111
|
::ActiveRecord::ConnectionAdapters::TableDefinition.prepend(ActiveRecord::TableDefinition)
|
|
98
112
|
end
|
|
99
|
-
# Ensure that ActiveRecord::Base is always loaded before any app-level
|
|
113
|
+
# Ensure that ActiveRecord::Base is always loaded before any app-level
|
|
114
|
+
# initializers can go try to load Switchman::Shard or we get a loop
|
|
100
115
|
::ActiveRecord::Base
|
|
101
116
|
end
|
|
102
117
|
|
|
103
|
-
initializer
|
|
118
|
+
initializer "switchman.error_patch", after: "active_record.initialize_database" do
|
|
104
119
|
::ActiveSupport.on_load(:active_record) do
|
|
105
120
|
::StandardError.include(StandardError)
|
|
106
121
|
end
|
|
107
122
|
end
|
|
108
123
|
|
|
109
|
-
initializer
|
|
124
|
+
initializer "switchman.initialize_cache", before: :initialize_cache, after: "active_record.initialize_database" do
|
|
110
125
|
::ActiveSupport::Cache.singleton_class.prepend(ActiveSupport::Cache::ClassMethods)
|
|
111
126
|
|
|
112
127
|
# if we haven't already setup our cache map out-of-band, set it up from
|
|
@@ -130,11 +145,11 @@ module Switchman
|
|
|
130
145
|
Switchman.config[:cache_map][::Rails.env] = value
|
|
131
146
|
end
|
|
132
147
|
|
|
133
|
-
middlewares = Switchman.config[:cache_map].values.
|
|
148
|
+
middlewares = Switchman.config[:cache_map].values.filter_map do |store|
|
|
134
149
|
store.middleware if store.respond_to?(:middleware)
|
|
135
|
-
end.
|
|
150
|
+
end.uniq
|
|
136
151
|
middlewares.each do |middleware|
|
|
137
|
-
config.middleware.insert_before(
|
|
152
|
+
config.middleware.insert_before("Rack::Runtime", middleware)
|
|
138
153
|
end
|
|
139
154
|
|
|
140
155
|
# prevent :initialize_cache from trying to (or needing to) set
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require
|
|
3
|
+
require "etc"
|
|
4
4
|
|
|
5
5
|
module Switchman
|
|
6
6
|
class Environment
|
|
7
|
-
def self.cpu_count(nproc_bin =
|
|
7
|
+
def self.cpu_count(nproc_bin = "nproc")
|
|
8
8
|
return Etc.nprocessors if Etc.respond_to?(:nprocessors)
|
|
9
9
|
|
|
10
10
|
`#{nproc_bin}`.to_i
|
data/lib/switchman/errors.rb
CHANGED
|
@@ -2,8 +2,21 @@
|
|
|
2
2
|
|
|
3
3
|
module Switchman
|
|
4
4
|
module Errors
|
|
5
|
+
class ManuallyCreatedShadowRecordError < RuntimeError
|
|
6
|
+
DEFAULT_MSG = "It looks like you're trying to manually create a shadow record. " \
|
|
7
|
+
"Please use Switchman::ActiveRecord::Base#save_shadow_record instead."
|
|
8
|
+
|
|
9
|
+
def initialize(msg = DEFAULT_MSG)
|
|
10
|
+
super
|
|
11
|
+
end
|
|
12
|
+
end
|
|
13
|
+
|
|
5
14
|
class NonExistentShardError < RuntimeError; end
|
|
6
15
|
|
|
7
16
|
class ParallelShardExecError < RuntimeError; end
|
|
17
|
+
|
|
18
|
+
class ShadowRecordError < RuntimeError; end
|
|
19
|
+
|
|
20
|
+
class UnshardedTableError < RuntimeError; end
|
|
8
21
|
end
|
|
9
22
|
end
|
|
@@ -13,8 +13,9 @@ module Switchman
|
|
|
13
13
|
end
|
|
14
14
|
|
|
15
15
|
%w[update_all delete_all].each do |method|
|
|
16
|
+
arg_params = (RUBY_VERSION <= "2.8") ? "*args" : "*args, **kwargs"
|
|
16
17
|
class_eval <<-RUBY, __FILE__, __LINE__ + 1
|
|
17
|
-
def #{method}(
|
|
18
|
+
def #{method}(#{arg_params})
|
|
18
19
|
db = Shard.current(connection_class_for_self).database_server
|
|
19
20
|
db.unguard { super }
|
|
20
21
|
end
|
data/lib/switchman/parallel.rb
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require
|
|
3
|
+
require "parallel"
|
|
4
4
|
|
|
5
5
|
module Switchman
|
|
6
6
|
module Parallel
|
|
@@ -65,4 +65,4 @@ module Switchman
|
|
|
65
65
|
end
|
|
66
66
|
end
|
|
67
67
|
|
|
68
|
-
|
|
68
|
+
Parallel::UndumpableException.prepend(Switchman::Parallel::UndumpableException)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require
|
|
3
|
+
require "switchman/test_helper"
|
|
4
4
|
|
|
5
5
|
module Switchman
|
|
6
6
|
# including this module in your specs will give you several shards to
|
|
@@ -34,9 +34,9 @@ module Switchman
|
|
|
34
34
|
groups = group.class.descendant_filtered_examples.map(&:example_group).uniq
|
|
35
35
|
next unless groups.any? { |descendant_group| RSpecHelper.included_in?(descendant_group) }
|
|
36
36
|
|
|
37
|
-
puts
|
|
37
|
+
puts "Setting up sharding for all specs..."
|
|
38
38
|
Shard.delete_all
|
|
39
|
-
Switchman.cache.delete(
|
|
39
|
+
Switchman.cache.delete("default_shard")
|
|
40
40
|
|
|
41
41
|
@@shard1, @@shard2 = TestHelper.recreate_persistent_test_shards
|
|
42
42
|
@@default_shard = Shard.default
|
|
@@ -48,7 +48,7 @@ module Switchman
|
|
|
48
48
|
@@shard1 = @@shard1.create_new_shard
|
|
49
49
|
@@shard2 = @@shard2.create_new_shard
|
|
50
50
|
rescue => e
|
|
51
|
-
warn
|
|
51
|
+
warn "Sharding setup FAILED!:"
|
|
52
52
|
while e
|
|
53
53
|
warn "\n#{e}\n"
|
|
54
54
|
warn e.backtrace
|
|
@@ -66,9 +66,9 @@ module Switchman
|
|
|
66
66
|
# we'll re-persist in the group's `before :all`; we don't want them to exist
|
|
67
67
|
# in the db before then
|
|
68
68
|
Shard.delete_all
|
|
69
|
-
Switchman.cache.delete(
|
|
69
|
+
Switchman.cache.delete("default_shard")
|
|
70
70
|
Shard.default(reload: true)
|
|
71
|
-
puts
|
|
71
|
+
puts "Done!"
|
|
72
72
|
|
|
73
73
|
main_pid = Process.pid
|
|
74
74
|
at_exit do
|
|
@@ -76,7 +76,7 @@ module Switchman
|
|
|
76
76
|
|
|
77
77
|
# preserve rspec's exit status
|
|
78
78
|
status = $!.is_a?(::SystemExit) ? $!.status : nil
|
|
79
|
-
puts
|
|
79
|
+
puts "Tearing down sharding for all specs"
|
|
80
80
|
@@shard1.database_server.destroy unless @@shard1.database_server == Shard.default.database_server
|
|
81
81
|
unless @@keep_the_shards
|
|
82
82
|
@@shard1.drop_database
|
|
@@ -95,7 +95,7 @@ module Switchman
|
|
|
95
95
|
dup = @@default_shard.dup
|
|
96
96
|
dup.id = @@default_shard.id
|
|
97
97
|
dup.save!
|
|
98
|
-
Switchman.cache.delete(
|
|
98
|
+
Switchman.cache.delete("default_shard")
|
|
99
99
|
Shard.default(reload: true)
|
|
100
100
|
dup = @@shard1.dup
|
|
101
101
|
dup.id = @@shard1.id
|
|
@@ -107,7 +107,7 @@ module Switchman
|
|
|
107
107
|
end
|
|
108
108
|
|
|
109
109
|
klass.before do
|
|
110
|
-
raise
|
|
110
|
+
raise "Sharding did not set up correctly" if @@sharding_failed
|
|
111
111
|
|
|
112
112
|
Shard.clear_cache
|
|
113
113
|
if use_transactional_tests
|
|
@@ -132,7 +132,7 @@ module Switchman
|
|
|
132
132
|
klass.after(:all) do
|
|
133
133
|
# Don't truncate because that can create some fun cross-connection lock contention
|
|
134
134
|
Shard.delete_all
|
|
135
|
-
Switchman.cache.delete(
|
|
135
|
+
Switchman.cache.delete("default_shard")
|
|
136
136
|
Shard.default(reload: true)
|
|
137
137
|
end
|
|
138
138
|
end
|
data/lib/switchman/shard.rb
CHANGED
|
@@ -36,13 +36,15 @@ module Switchman
|
|
|
36
36
|
|
|
37
37
|
# Now find the actual record, if it exists
|
|
38
38
|
@default = begin
|
|
39
|
-
find_cached(
|
|
39
|
+
find_cached("default_shard") { Shard.where(default: true).take } || default
|
|
40
40
|
rescue
|
|
41
41
|
default
|
|
42
42
|
end
|
|
43
43
|
|
|
44
44
|
# make sure this is not erroneously cached
|
|
45
|
-
|
|
45
|
+
if @default.database_server.instance_variable_defined?(:@primary_shard)
|
|
46
|
+
@default.database_server.remove_instance_variable(:@primary_shard)
|
|
47
|
+
end
|
|
46
48
|
|
|
47
49
|
# and finally, check for cached references to the default shard on the existing connection
|
|
48
50
|
sharded_models.each do |klass|
|
|
@@ -75,28 +77,30 @@ module Switchman
|
|
|
75
77
|
klass.current_switchman_shard != shard
|
|
76
78
|
|
|
77
79
|
(activated_classes ||= []) << klass
|
|
78
|
-
klass.connected_to_stack << { shard: shard.database_server.id.to_sym,
|
|
80
|
+
klass.connected_to_stack << { shard: shard.database_server.id.to_sym,
|
|
81
|
+
klasses: [klass],
|
|
82
|
+
switchman_shard: shard }
|
|
79
83
|
end
|
|
80
84
|
activated_classes
|
|
81
85
|
end
|
|
82
86
|
|
|
83
87
|
def active_shards
|
|
84
|
-
sharded_models.
|
|
88
|
+
sharded_models.filter_map do |klass|
|
|
85
89
|
[klass, current(klass)]
|
|
86
|
-
end.
|
|
90
|
+
end.to_h
|
|
87
91
|
end
|
|
88
92
|
|
|
89
93
|
def lookup(id)
|
|
90
94
|
id_i = id.to_i
|
|
91
|
-
return current if id_i == current.id || id ==
|
|
92
|
-
return default if id_i == default.id || id.nil? || id ==
|
|
95
|
+
return current if id_i == current.id || id == "self"
|
|
96
|
+
return default if id_i == default.id || id.nil? || id == "default"
|
|
93
97
|
|
|
94
98
|
id = id_i
|
|
95
99
|
raise ArgumentError if id.zero?
|
|
96
100
|
|
|
97
101
|
unless cached_shards.key?(id)
|
|
98
102
|
cached_shards[id] = Shard.default.activate do
|
|
99
|
-
find_cached([
|
|
103
|
+
find_cached(["shard", id]) { find_by(id: id) }
|
|
100
104
|
end
|
|
101
105
|
end
|
|
102
106
|
cached_shards[id]
|
|
@@ -120,7 +124,8 @@ module Switchman
|
|
|
120
124
|
# forking.
|
|
121
125
|
# exception: - :ignore, :raise, :defer (wait until the end and raise the first
|
|
122
126
|
# error), or a proc
|
|
123
|
-
|
|
127
|
+
# output: - :simple, :decorated (with database_server_id:shard_name)
|
|
128
|
+
def with_each_shard(*args, parallel: false, exception: :raise, output: :simple)
|
|
124
129
|
raise ArgumentError, "wrong number of arguments (#{args.length} for 0...2)" if args.length > 2
|
|
125
130
|
|
|
126
131
|
return Array.wrap(yield) unless default.is_a?(Shard)
|
|
@@ -139,13 +144,15 @@ module Switchman
|
|
|
139
144
|
parallel = 0 if parallel == false || parallel.nil?
|
|
140
145
|
|
|
141
146
|
scope ||= Shard.all
|
|
142
|
-
|
|
147
|
+
if ::ActiveRecord::Relation === scope && scope.order_values.empty?
|
|
148
|
+
scope = scope.order(::Arel.sql("database_server_id IS NOT NULL, database_server_id, id"))
|
|
149
|
+
end
|
|
143
150
|
|
|
144
151
|
if parallel > 1
|
|
145
152
|
if ::ActiveRecord::Relation === scope
|
|
146
153
|
# still need a post-uniq, cause the default database server could be NULL or Rails.env in the db
|
|
147
|
-
database_servers = scope.reorder(
|
|
148
|
-
|
|
154
|
+
database_servers = scope.reorder("database_server_id").select(:database_server_id).distinct
|
|
155
|
+
.filter_map(&:database_server).uniq
|
|
149
156
|
# nothing to do
|
|
150
157
|
return if database_servers.count.zero?
|
|
151
158
|
|
|
@@ -162,22 +169,23 @@ module Switchman
|
|
|
162
169
|
::ActiveRecord::Base.clear_all_connections!
|
|
163
170
|
|
|
164
171
|
parent_process_name = `ps -ocommand= -p#{Process.pid}`.slice(/#{$0}.*/)
|
|
165
|
-
ret = ::Parallel.map(scopes, in_processes: scopes.length > 1 ? parallel : 0) do |server, subscope|
|
|
172
|
+
ret = ::Parallel.map(scopes, in_processes: (scopes.length > 1) ? parallel : 0) do |server, subscope|
|
|
166
173
|
name = server.id
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
$stderr = Parallel::PrefixingIO.new(name, STDERR)
|
|
170
|
-
# rubocop:enable Style/GlobalStdStream
|
|
174
|
+
last_description = name
|
|
175
|
+
|
|
171
176
|
begin
|
|
172
177
|
max_length = 128 - name.length - 3
|
|
173
178
|
short_parent_name = parent_process_name[0..max_length] if max_length >= 0
|
|
174
|
-
new_title = [short_parent_name, name].join(
|
|
179
|
+
new_title = [short_parent_name, name].join(" ")
|
|
175
180
|
Process.setproctitle(new_title)
|
|
176
181
|
Switchman.config[:on_fork_proc]&.call
|
|
177
|
-
with_each_shard(subscope, classes, exception: exception,
|
|
182
|
+
with_each_shard(subscope, classes, exception: exception, output: :decorated) do
|
|
183
|
+
last_description = Shard.current.description
|
|
184
|
+
Parallel::ResultWrapper.new(yield)
|
|
185
|
+
end
|
|
178
186
|
rescue => e
|
|
179
187
|
logger.error e.full_message
|
|
180
|
-
Parallel::QuietExceptionWrapper.new(
|
|
188
|
+
Parallel::QuietExceptionWrapper.new(last_description, ::Parallel::ExceptionWrapper.new(e))
|
|
181
189
|
end
|
|
182
190
|
end.flatten
|
|
183
191
|
|
|
@@ -185,8 +193,9 @@ module Switchman
|
|
|
185
193
|
unless errors.empty?
|
|
186
194
|
raise errors.first.exception if errors.length == 1
|
|
187
195
|
|
|
196
|
+
errors_desc = errors.map(&:name).sort.join(", ")
|
|
188
197
|
raise Errors::ParallelShardExecError,
|
|
189
|
-
"The following database server(s) did not finish processing cleanly: #{
|
|
198
|
+
"The following database server(s) did not finish processing cleanly: #{errors_desc}",
|
|
190
199
|
cause: errors.first.exception
|
|
191
200
|
end
|
|
192
201
|
|
|
@@ -195,14 +204,20 @@ module Switchman
|
|
|
195
204
|
|
|
196
205
|
classes ||= []
|
|
197
206
|
|
|
198
|
-
previous_shard = nil
|
|
199
207
|
result = []
|
|
200
208
|
ex = nil
|
|
209
|
+
old_stdout = $stdout
|
|
210
|
+
old_stderr = $stderr
|
|
201
211
|
scope.each do |shard|
|
|
202
212
|
# shard references a database server that isn't configured in this environment
|
|
203
213
|
next unless shard.database_server
|
|
204
214
|
|
|
205
215
|
shard.activate(*classes) do
|
|
216
|
+
if output == :decorated
|
|
217
|
+
$stdout = Parallel::PrefixingIO.new(shard.description, $stdout)
|
|
218
|
+
$stderr = Parallel::PrefixingIO.new(shard.description, $stderr)
|
|
219
|
+
end
|
|
220
|
+
|
|
206
221
|
result.concat Array.wrap(yield)
|
|
207
222
|
rescue
|
|
208
223
|
case exception
|
|
@@ -216,8 +231,10 @@ module Switchman
|
|
|
216
231
|
else
|
|
217
232
|
raise
|
|
218
233
|
end
|
|
234
|
+
ensure
|
|
235
|
+
$stdout = old_stdout
|
|
236
|
+
$stderr = old_stderr
|
|
219
237
|
end
|
|
220
|
-
previous_shard = shard
|
|
221
238
|
end
|
|
222
239
|
raise ex if ex
|
|
223
240
|
|
|
@@ -457,7 +474,7 @@ module Switchman
|
|
|
457
474
|
end
|
|
458
475
|
|
|
459
476
|
def description
|
|
460
|
-
[database_server.id, name].compact.join(
|
|
477
|
+
[database_server.id, name].compact.join(":")
|
|
461
478
|
end
|
|
462
479
|
|
|
463
480
|
# Shards are always on the default shard
|
|
@@ -491,7 +508,7 @@ module Switchman
|
|
|
491
508
|
end
|
|
492
509
|
|
|
493
510
|
def drop_database
|
|
494
|
-
raise(
|
|
511
|
+
raise("Cannot drop the database of the default shard") if default?
|
|
495
512
|
return unless read_attribute(:name)
|
|
496
513
|
|
|
497
514
|
begin
|
|
@@ -500,12 +517,12 @@ module Switchman
|
|
|
500
517
|
drop_statement = sharding_config[adapter]&.[](:drop_statement)
|
|
501
518
|
drop_statement ||= sharding_config[:drop_statement]
|
|
502
519
|
if drop_statement
|
|
503
|
-
drop_statement = Array(drop_statement).dup
|
|
504
|
-
|
|
520
|
+
drop_statement = Array(drop_statement).dup
|
|
521
|
+
.map { |statement| statement.gsub("%{name}", name) }
|
|
505
522
|
end
|
|
506
523
|
|
|
507
524
|
case adapter
|
|
508
|
-
when
|
|
525
|
+
when "mysql", "mysql2"
|
|
509
526
|
activate do
|
|
510
527
|
::GuardRail.activate(:deploy) do
|
|
511
528
|
drop_statement ||= "DROP DATABASE #{name}"
|
|
@@ -514,7 +531,7 @@ module Switchman
|
|
|
514
531
|
end
|
|
515
532
|
end
|
|
516
533
|
end
|
|
517
|
-
when
|
|
534
|
+
when "postgresql"
|
|
518
535
|
activate do
|
|
519
536
|
::GuardRail.activate(:deploy) do
|
|
520
537
|
# Shut up, Postgres!
|
|
@@ -551,7 +568,7 @@ module Switchman
|
|
|
551
568
|
end
|
|
552
569
|
|
|
553
570
|
def destroy
|
|
554
|
-
raise(
|
|
571
|
+
raise("Cannot destroy the default shard") if default?
|
|
555
572
|
|
|
556
573
|
super
|
|
557
574
|
end
|
|
@@ -560,8 +577,8 @@ module Switchman
|
|
|
560
577
|
|
|
561
578
|
def clear_cache
|
|
562
579
|
Shard.default.activate do
|
|
563
|
-
Switchman.cache.delete([
|
|
564
|
-
Switchman.cache.delete(
|
|
580
|
+
Switchman.cache.delete(["shard", id].join("/"))
|
|
581
|
+
Switchman.cache.delete("default_shard") if default?
|
|
565
582
|
end
|
|
566
583
|
self.class.clear_cache
|
|
567
584
|
end
|
|
@@ -16,7 +16,11 @@ module Switchman
|
|
|
16
16
|
payload[:shard] = {
|
|
17
17
|
database_server_id: shard.database_server.id,
|
|
18
18
|
id: shard.id,
|
|
19
|
-
env: ::Rails.version <
|
|
19
|
+
env: if ::Rails.version < "7.0"
|
|
20
|
+
@shard_host.pool.connection_klass&.current_role
|
|
21
|
+
else
|
|
22
|
+
@shard_host.pool.connection_class&.current_role
|
|
23
|
+
end
|
|
20
24
|
}
|
|
21
25
|
end
|
|
22
26
|
super name, payload
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Switchman
|
|
4
|
+
class SharedSchemaCache
|
|
5
|
+
def self.get_schema_cache(connection)
|
|
6
|
+
@schema_cache ||= ::ActiveRecord::ConnectionAdapters::SchemaCache.new(connection)
|
|
7
|
+
@schema_cache.connection = connection
|
|
8
|
+
@schema_cache
|
|
9
|
+
end
|
|
10
|
+
end
|
|
11
|
+
end
|
|
@@ -65,7 +65,7 @@ module Switchman
|
|
|
65
65
|
if server == Shard.default.database_server
|
|
66
66
|
server.shards.where(name: name).first
|
|
67
67
|
else
|
|
68
|
-
shard = Shard.where(
|
|
68
|
+
shard = Shard.where("database_server_id IS NOT NULL AND name=?", name).first
|
|
69
69
|
# if somehow databases got created in a different order, change the shard to match
|
|
70
70
|
shard.database_server = server if shard
|
|
71
71
|
shard
|
data/lib/switchman/version.rb
CHANGED