switchman 1.14.10 → 2.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/app/models/switchman/shard.rb +743 -11
- data/db/migrate/20130328224244_create_default_shard.rb +1 -1
- data/lib/switchman.rb +1 -1
- data/lib/switchman/active_record/abstract_adapter.rb +2 -6
- data/lib/switchman/active_record/association.rb +7 -10
- data/lib/switchman/active_record/base.rb +16 -7
- data/lib/switchman/active_record/connection_handler.rb +13 -20
- data/lib/switchman/active_record/connection_pool.rb +0 -12
- data/lib/switchman/active_record/log_subscriber.rb +8 -12
- data/lib/switchman/active_record/migration.rb +9 -0
- data/lib/switchman/active_record/postgresql_adapter.rb +14 -44
- data/lib/switchman/active_record/query_cache.rb +17 -107
- data/lib/switchman/active_record/relation.rb +5 -5
- data/lib/switchman/active_record/statement_cache.rb +4 -25
- data/lib/switchman/active_record/table_definition.rb +2 -2
- data/lib/switchman/connection_pool_proxy.rb +12 -14
- data/lib/switchman/database_server.rb +16 -16
- data/lib/switchman/engine.rb +16 -14
- data/lib/switchman/{shackles.rb → guard_rail.rb} +4 -4
- data/lib/switchman/{shackles → guard_rail}/relation.rb +5 -5
- data/lib/switchman/r_spec_helper.rb +7 -7
- data/lib/switchman/sharded_instrumenter.rb +1 -1
- data/lib/switchman/test_helper.rb +3 -3
- data/lib/switchman/version.rb +1 -1
- data/lib/tasks/switchman.rake +6 -13
- metadata +31 -18
- data/app/models/switchman/shard_internal.rb +0 -718
@@ -5,7 +5,7 @@ module Switchman
|
|
5
5
|
klass::SINGLE_VALUE_METHODS.concat [ :shard, :shard_source ]
|
6
6
|
end
|
7
7
|
|
8
|
-
def initialize(
|
8
|
+
def initialize(*, **)
|
9
9
|
super
|
10
10
|
self.shard_value = Shard.current(klass ? klass.shard_category : :primary) unless shard_value
|
11
11
|
self.shard_source_value = :implicit unless shard_source_value
|
@@ -17,7 +17,7 @@ module Switchman
|
|
17
17
|
result
|
18
18
|
end
|
19
19
|
|
20
|
-
def merge(*
|
20
|
+
def merge(*)
|
21
21
|
relation = super
|
22
22
|
if relation.shard_value != self.shard_value && relation.shard_source_value == :implicit
|
23
23
|
relation.shard_value = self.shard_value
|
@@ -26,15 +26,15 @@ module Switchman
|
|
26
26
|
relation
|
27
27
|
end
|
28
28
|
|
29
|
-
def new(
|
29
|
+
def new(*, &block)
|
30
30
|
primary_shard.activate(klass.shard_category) { super }
|
31
31
|
end
|
32
32
|
|
33
|
-
def create(
|
33
|
+
def create(*, &block)
|
34
34
|
primary_shard.activate(klass.shard_category) { super }
|
35
35
|
end
|
36
36
|
|
37
|
-
def create!(
|
37
|
+
def create!(*, &block)
|
38
38
|
primary_shard.activate(klass.shard_category) { super }
|
39
39
|
end
|
40
40
|
|
@@ -2,7 +2,7 @@ module Switchman
|
|
2
2
|
module ActiveRecord
|
3
3
|
module StatementCache
|
4
4
|
module ClassMethods
|
5
|
-
def create(connection, block
|
5
|
+
def create(connection, &block)
|
6
6
|
relation = block.call ::ActiveRecord::StatementCache::Params.new
|
7
7
|
|
8
8
|
if ::Rails.version >= "5.2"
|
@@ -46,37 +46,16 @@ module Switchman
|
|
46
46
|
bind_values = bind_map.bind(params, current_shard, target_shard)
|
47
47
|
|
48
48
|
target_shard.activate(klass.shard_category) do
|
49
|
-
|
50
|
-
|
51
|
-
klass.find_by_sql(sql, bind_values)
|
52
|
-
else
|
53
|
-
sql = generic_query_builder(connection).sql_for(bind_values, connection)
|
54
|
-
klass.find_by_sql(sql, bind_values)
|
55
|
-
end
|
49
|
+
sql = qualified_query_builder(target_shard, klass).sql_for(bind_values, connection)
|
50
|
+
klass.find_by_sql(sql, bind_values)
|
56
51
|
end
|
57
52
|
end
|
58
53
|
|
59
|
-
if ::Rails.version < '5.
|
60
|
-
def generic_query_builder(connection)
|
61
|
-
@query_builder ||= connection.cacheable_query(@arel)
|
62
|
-
end
|
63
|
-
|
64
|
-
def qualified_query_builder(shard, klass)
|
65
|
-
@qualified_query_builders[shard.id] ||= klass.connection.cacheable_query(@arel)
|
66
|
-
end
|
67
|
-
elsif ::Rails.version < '5.2'
|
68
|
-
def generic_query_builder(connection)
|
69
|
-
@query_builder ||= connection.cacheable_query(self.class, @arel)
|
70
|
-
end
|
71
|
-
|
54
|
+
if ::Rails.version < '5.2'
|
72
55
|
def qualified_query_builder(shard, klass)
|
73
56
|
@qualified_query_builders[shard.id] ||= klass.connection.cacheable_query(self.class, @arel)
|
74
57
|
end
|
75
58
|
else
|
76
|
-
def generic_query_builder(connection)
|
77
|
-
@query_builder ||= connection.cacheable_query(self.class, @arel).first
|
78
|
-
end
|
79
|
-
|
80
59
|
def qualified_query_builder(shard, klass)
|
81
60
|
@qualified_query_builders[shard.id] ||= klass.connection.cacheable_query(self.class, @arel).first
|
82
61
|
end
|
@@ -1,8 +1,8 @@
|
|
1
1
|
module Switchman
|
2
2
|
module ActiveRecord
|
3
3
|
module TableDefinition
|
4
|
-
def column(name, type,
|
5
|
-
Engine.foreign_key_check(name, type,
|
4
|
+
def column(name, type, limit: nil, **)
|
5
|
+
Engine.foreign_key_check(name, type, limit: limit)
|
6
6
|
super
|
7
7
|
end
|
8
8
|
end
|
@@ -37,13 +37,13 @@ module Switchman
|
|
37
37
|
Shard.current(@category)
|
38
38
|
end
|
39
39
|
|
40
|
-
def
|
41
|
-
::Rails.env.test? ? :
|
40
|
+
def active_guard_rail_environment
|
41
|
+
::Rails.env.test? ? :primary : active_shard.database_server.guard_rail_environment
|
42
42
|
end
|
43
43
|
|
44
44
|
def current_pool
|
45
45
|
current_active_shard = active_shard
|
46
|
-
pool = self.default_pool if current_active_shard.database_server == Shard.default.database_server &&
|
46
|
+
pool = self.default_pool if current_active_shard.database_server == Shard.default.database_server && active_guard_rail_environment == :primary && (current_active_shard.default? || current_active_shard.database_server.shareable?)
|
47
47
|
pool = @connection_pools[pool_key] ||= create_pool unless pool
|
48
48
|
pool.shard = current_active_shard
|
49
49
|
pool
|
@@ -60,8 +60,8 @@ module Switchman
|
|
60
60
|
connection.instance_variable_set(:@schema_cache, @schema_cache) unless ::Rails.version >= '6'
|
61
61
|
connection
|
62
62
|
rescue ConnectionError
|
63
|
-
raise if active_shard.database_server == Shard.default.database_server &&
|
64
|
-
configs = active_shard.database_server.config(
|
63
|
+
raise if active_shard.database_server == Shard.default.database_server && active_guard_rail_environment == :primary
|
64
|
+
configs = active_shard.database_server.config(active_guard_rail_environment)
|
65
65
|
raise unless configs.is_a?(Array)
|
66
66
|
configs.each_with_index do |config, idx|
|
67
67
|
pool = create_pool(config.dup)
|
@@ -120,7 +120,7 @@ module Switchman
|
|
120
120
|
end
|
121
121
|
|
122
122
|
def pool_key
|
123
|
-
[
|
123
|
+
[active_guard_rail_environment,
|
124
124
|
active_shard.database_server.shareable? ? active_shard.database_server.pool_key : active_shard]
|
125
125
|
end
|
126
126
|
|
@@ -128,7 +128,7 @@ module Switchman
|
|
128
128
|
shard = active_shard
|
129
129
|
unless config
|
130
130
|
if shard != Shard.default
|
131
|
-
config = shard.database_server.config(
|
131
|
+
config = shard.database_server.config(active_guard_rail_environment)
|
132
132
|
config = config.first if config.is_a?(Array)
|
133
133
|
config = config.dup
|
134
134
|
else
|
@@ -138,10 +138,10 @@ module Switchman
|
|
138
138
|
# different models could be using different configs on the default
|
139
139
|
# shard, and database server wouldn't know about that
|
140
140
|
config = default_pool.spec.instance_variable_get(:@config)
|
141
|
-
if config[
|
142
|
-
config = config.merge(config[
|
143
|
-
elsif config[
|
144
|
-
config = config.merge(config[
|
141
|
+
if config[active_guard_rail_environment].is_a?(Hash)
|
142
|
+
config = config.merge(config[active_guard_rail_environment])
|
143
|
+
elsif config[active_guard_rail_environment].is_a?(Array)
|
144
|
+
config = config.merge(config[active_guard_rail_environment].first)
|
145
145
|
else
|
146
146
|
config = config.dup
|
147
147
|
end
|
@@ -159,9 +159,7 @@ module Switchman
|
|
159
159
|
::ActiveRecord::ConnectionAdapters::ConnectionPool.new(spec).tap do |pool|
|
160
160
|
pool.shard = shard
|
161
161
|
pool.set_schema_cache(@schema_cache) if ::Rails.version >= '6'
|
162
|
-
if
|
163
|
-
pool.enable_query_cache! if !@connection_pools.empty? && @connection_pools.first.last.query_cache_enabled
|
164
|
-
end
|
162
|
+
pool.enable_query_cache! if !@connection_pools.empty? && @connection_pools.first.last.query_cache_enabled
|
165
163
|
end
|
166
164
|
end
|
167
165
|
end
|
@@ -69,12 +69,12 @@ module Switchman
|
|
69
69
|
@fake
|
70
70
|
end
|
71
71
|
|
72
|
-
def config(environment = :
|
72
|
+
def config(environment = :primary)
|
73
73
|
@configs[environment] ||= begin
|
74
74
|
if @config[environment].is_a?(Array)
|
75
75
|
@config[environment].map do |config|
|
76
76
|
config = @config.merge((config || {}).symbolize_keys)
|
77
|
-
# make sure
|
77
|
+
# make sure GuardRail doesn't get any brilliant ideas about choosing the first possible server
|
78
78
|
config.delete(environment)
|
79
79
|
config
|
80
80
|
end
|
@@ -86,33 +86,33 @@ module Switchman
|
|
86
86
|
end
|
87
87
|
end
|
88
88
|
|
89
|
-
def
|
90
|
-
@
|
89
|
+
def guard_rail_environment
|
90
|
+
@guard_rail_environment || ::GuardRail.environment
|
91
91
|
end
|
92
92
|
|
93
93
|
# locks this db to a specific environment, except for
|
94
94
|
# when doing writes (then it falls back to the current
|
95
|
-
# value of
|
96
|
-
def
|
97
|
-
@
|
95
|
+
# value of GuardRail.environment)
|
96
|
+
def guard!(environment = :secondary)
|
97
|
+
@guard_rail_environment = environment
|
98
98
|
end
|
99
99
|
|
100
|
-
def
|
101
|
-
@
|
100
|
+
def unguard!
|
101
|
+
@guard_rail_environment = nil
|
102
102
|
end
|
103
103
|
|
104
|
-
def
|
105
|
-
old_env = @
|
106
|
-
|
104
|
+
def unguard
|
105
|
+
old_env = @guard_rail_environment
|
106
|
+
unguard!
|
107
107
|
yield
|
108
108
|
ensure
|
109
|
-
|
109
|
+
guard!(old_env)
|
110
110
|
end
|
111
111
|
|
112
112
|
def shareable?
|
113
113
|
@shareable_environment_key ||= []
|
114
|
-
environment =
|
115
|
-
explicit_user = ::
|
114
|
+
environment = guard_rail_environment
|
115
|
+
explicit_user = ::GuardRail.global_config[:username]
|
116
116
|
return @shareable if @shareable_environment_key == [environment, explicit_user]
|
117
117
|
@shareable_environment_key = [environment, explicit_user]
|
118
118
|
if explicit_user
|
@@ -190,7 +190,7 @@ module Switchman
|
|
190
190
|
begin
|
191
191
|
self.class.creating_new_shard = true
|
192
192
|
shard.activate(*Shard.categories) do
|
193
|
-
::
|
193
|
+
::GuardRail.activate(:deploy) do
|
194
194
|
begin
|
195
195
|
if create_statement
|
196
196
|
if (::ActiveRecord::Base.connection.select_value("SELECT 1 FROM pg_namespace WHERE nspname=#{::ActiveRecord::Base.connection.quote(name)}"))
|
data/lib/switchman/engine.rb
CHANGED
@@ -87,8 +87,8 @@ module Switchman
|
|
87
87
|
require "switchman/arel"
|
88
88
|
require "switchman/call_super"
|
89
89
|
require "switchman/rails"
|
90
|
-
require "switchman/
|
91
|
-
require_dependency "switchman/
|
90
|
+
require "switchman/guard_rail/relation"
|
91
|
+
require_dependency "switchman/shard"
|
92
92
|
require "switchman/standard_error"
|
93
93
|
|
94
94
|
::StandardError.include(StandardError)
|
@@ -118,15 +118,11 @@ module Switchman
|
|
118
118
|
::ActiveRecord::ConnectionAdapters::ConnectionHandler.prepend(ActiveRecord::ConnectionHandler)
|
119
119
|
::ActiveRecord::ConnectionAdapters::ConnectionPool.prepend(ActiveRecord::ConnectionPool)
|
120
120
|
::ActiveRecord::ConnectionAdapters::AbstractAdapter.prepend(ActiveRecord::QueryCache)
|
121
|
-
# when we call super in Switchman::ActiveRecord::QueryCache#select_all,
|
122
|
-
# we want it to find the definition from
|
123
|
-
# ActiveRecord::ConnectionAdapters::DatabaseStatements, not
|
124
|
-
# ActiveRecord::ConnectionAdapters::QueryCache
|
125
|
-
::ActiveRecord::ConnectionAdapters::QueryCache.send(:remove_method, :select_all) if ::Rails.version < '5.0.1'
|
126
121
|
|
127
122
|
::ActiveRecord::LogSubscriber.prepend(ActiveRecord::LogSubscriber)
|
128
123
|
::ActiveRecord::Migration.prepend(ActiveRecord::Migration)
|
129
124
|
::ActiveRecord::Migration::Compatibility::V5_0.prepend(ActiveRecord::Migration::Compatibility::V5_0)
|
125
|
+
::ActiveRecord::MigrationContext.prepend(ActiveRecord::MigrationContext) if ::Rails.version >= '5.2'
|
130
126
|
::ActiveRecord::Migrator.prepend(ActiveRecord::Migrator)
|
131
127
|
|
132
128
|
::ActiveRecord::Reflection::AbstractReflection.include(ActiveRecord::Reflection::AbstractReflection)
|
@@ -137,7 +133,7 @@ module Switchman
|
|
137
133
|
::ActiveRecord::Relation.prepend(ActiveRecord::Calculations)
|
138
134
|
::ActiveRecord::Relation.include(ActiveRecord::FinderMethods)
|
139
135
|
::ActiveRecord::Relation.include(ActiveRecord::QueryMethods)
|
140
|
-
::ActiveRecord::Relation.prepend(
|
136
|
+
::ActiveRecord::Relation.prepend(GuardRail::Relation)
|
141
137
|
::ActiveRecord::Relation.prepend(ActiveRecord::Relation)
|
142
138
|
::ActiveRecord::Relation.include(ActiveRecord::SpawnMethods)
|
143
139
|
::ActiveRecord::Relation.include(CallSuper)
|
@@ -154,8 +150,8 @@ module Switchman
|
|
154
150
|
end
|
155
151
|
end
|
156
152
|
|
157
|
-
def self.foreign_key_check(name, type,
|
158
|
-
if name.to_s =~ /_id\z/ && type.to_s == 'integer' &&
|
153
|
+
def self.foreign_key_check(name, type, limit: nil)
|
154
|
+
if name.to_s =~ /_id\z/ && type.to_s == 'integer' && limit.to_i < 8
|
159
155
|
puts "WARNING: All foreign keys need to be 8-byte integers. #{name} looks like a foreign key. If so, please add the option: `:limit => 8`"
|
160
156
|
end
|
161
157
|
end
|
@@ -173,6 +169,12 @@ module Switchman
|
|
173
169
|
require "switchman/active_record/postgresql_adapter"
|
174
170
|
::ActiveRecord::ConnectionAdapters::PostgreSQLAdapter.prepend(ActiveRecord::PostgreSQLAdapter)
|
175
171
|
end
|
172
|
+
|
173
|
+
# If Switchman::Shard wasn't loaded as of when ActiveRecord::Base initialized
|
174
|
+
# establish a connection here instead
|
175
|
+
if !Shard.instance_variable_get(:@default)
|
176
|
+
::ActiveRecord::Base.establish_connection
|
177
|
+
end
|
176
178
|
end
|
177
179
|
end
|
178
180
|
|
@@ -183,15 +185,15 @@ module Switchman
|
|
183
185
|
end
|
184
186
|
end
|
185
187
|
|
186
|
-
initializer 'switchman.
|
188
|
+
initializer 'switchman.extend_guard_rail', :before => "switchman.extend_ar" do
|
187
189
|
::ActiveSupport.on_load(:active_record) do
|
188
|
-
require "switchman/
|
190
|
+
require "switchman/guard_rail"
|
189
191
|
|
190
|
-
::
|
192
|
+
::GuardRail.singleton_class.prepend(GuardRail::ClassMethods)
|
191
193
|
end
|
192
194
|
end
|
193
195
|
|
194
|
-
initializer 'switchman.extend_controller', :after => "
|
196
|
+
initializer 'switchman.extend_controller', :after => "guard_rail.extend_ar" do
|
195
197
|
::ActiveSupport.on_load(:action_controller) do
|
196
198
|
require "switchman/action_controller/caching"
|
197
199
|
|
@@ -1,17 +1,17 @@
|
|
1
1
|
module Switchman
|
2
|
-
module
|
2
|
+
module GuardRail
|
3
3
|
module ClassMethods
|
4
4
|
def self.prepended(klass)
|
5
5
|
klass.send(:remove_method, :ensure_handler)
|
6
6
|
end
|
7
7
|
|
8
8
|
# drops the save_handler and ensure_handler calls from the vanilla
|
9
|
-
#
|
9
|
+
# GuardRail' implementation.
|
10
10
|
def activate!(environment)
|
11
|
-
environment ||= :
|
11
|
+
environment ||= :primary
|
12
12
|
activated_environments << environment
|
13
13
|
old_environment = self.environment
|
14
|
-
Thread.current[:
|
14
|
+
Thread.current[:guard_rail_environment] = environment
|
15
15
|
old_environment
|
16
16
|
end
|
17
17
|
|
@@ -1,11 +1,11 @@
|
|
1
1
|
module Switchman
|
2
|
-
module
|
2
|
+
module GuardRail
|
3
3
|
module Relation
|
4
4
|
def exec_queries(*args)
|
5
5
|
if self.lock_value
|
6
6
|
db = Shard.current(shard_category).database_server
|
7
|
-
if ::
|
8
|
-
return db.
|
7
|
+
if ::GuardRail.environment != db.guard_rail_environment
|
8
|
+
return db.unguard { super }
|
9
9
|
end
|
10
10
|
end
|
11
11
|
super
|
@@ -15,8 +15,8 @@ module Switchman
|
|
15
15
|
class_eval <<-RUBY, __FILE__, __LINE__ + 1
|
16
16
|
def #{method}(*args)
|
17
17
|
db = Shard.current(shard_category).database_server
|
18
|
-
if ::
|
19
|
-
db.
|
18
|
+
if ::GuardRail.environment != db.guard_rail_environment
|
19
|
+
db.unguard { super }
|
20
20
|
else
|
21
21
|
super
|
22
22
|
end
|
@@ -65,7 +65,7 @@ module Switchman
|
|
65
65
|
(@@shard3.drop_database if @@shard3) rescue nil
|
66
66
|
@@shard1 = @@shard2 = @@shard3 = nil
|
67
67
|
Shard.delete_all
|
68
|
-
Shard.default(true)
|
68
|
+
Shard.default(reload: true)
|
69
69
|
next
|
70
70
|
end
|
71
71
|
end
|
@@ -73,7 +73,7 @@ module Switchman
|
|
73
73
|
# in the db before then
|
74
74
|
Shard.delete_all
|
75
75
|
Switchman.cache.delete("default_shard")
|
76
|
-
Shard.default(true)
|
76
|
+
Shard.default(reload: true)
|
77
77
|
puts "Done!"
|
78
78
|
|
79
79
|
at_exit do
|
@@ -102,7 +102,7 @@ module Switchman
|
|
102
102
|
dup.id = @@default_shard.id
|
103
103
|
dup.save!
|
104
104
|
Switchman.cache.delete("default_shard")
|
105
|
-
Shard.default(true)
|
105
|
+
Shard.default(reload: true)
|
106
106
|
dup = @@shard1.dup
|
107
107
|
dup.id = @@shard1.id
|
108
108
|
dup.save!
|
@@ -121,8 +121,8 @@ module Switchman
|
|
121
121
|
klass.before do
|
122
122
|
raise "Sharding did not set up correctly" if @@sharding_failed
|
123
123
|
Shard.clear_cache
|
124
|
-
if
|
125
|
-
Shard.default(true)
|
124
|
+
if use_transactional_tests
|
125
|
+
Shard.default(reload: true)
|
126
126
|
@shard1 = Shard.find(@shard1.id)
|
127
127
|
@shard2 = Shard.find(@shard2.id)
|
128
128
|
shards = [@shard2]
|
@@ -137,7 +137,7 @@ module Switchman
|
|
137
137
|
|
138
138
|
klass.after do
|
139
139
|
next if @@sharding_failed
|
140
|
-
if
|
140
|
+
if use_transactional_tests
|
141
141
|
shards = [@shard2]
|
142
142
|
shards << @shard1 unless @shard1.database_server == Shard.default.database_server
|
143
143
|
shards.each do |shard|
|
@@ -151,7 +151,7 @@ module Switchman
|
|
151
151
|
klass.after(:all) do
|
152
152
|
Shard.connection.update("TRUNCATE #{Shard.quoted_table_name} CASCADE")
|
153
153
|
Switchman.cache.delete("default_shard")
|
154
|
-
Shard.default(true)
|
154
|
+
Shard.default(reload: true)
|
155
155
|
end
|
156
156
|
end
|
157
157
|
end
|
@@ -3,8 +3,8 @@ module Switchman
|
|
3
3
|
class << self
|
4
4
|
def recreate_persistent_test_shards(dont_create: false)
|
5
5
|
# recreate the default shard (it got buhleted)
|
6
|
-
::
|
7
|
-
if Shard.default(true).is_a?(DefaultShard)
|
6
|
+
::GuardRail.activate(:deploy) { Switchman.cache.clear }
|
7
|
+
if Shard.default(reload: true).is_a?(DefaultShard)
|
8
8
|
begin
|
9
9
|
Shard.create!(default: true)
|
10
10
|
rescue
|
@@ -12,7 +12,7 @@ module Switchman
|
|
12
12
|
# database doesn't exist yet, presumably cause we're creating it right now
|
13
13
|
return [nil, nil]
|
14
14
|
end
|
15
|
-
Shard.default(true)
|
15
|
+
Shard.default(reload: true)
|
16
16
|
end
|
17
17
|
|
18
18
|
# can't auto-create a new shard on the default shard's db server if the
|