switchman 1.14.9 → 1.16.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/app/models/switchman/shard.rb +740 -11
- data/lib/switchman/active_record/abstract_adapter.rb +0 -4
- data/lib/switchman/active_record/association.rb +7 -10
- data/lib/switchman/active_record/base.rb +9 -0
- data/lib/switchman/active_record/connection_handler.rb +7 -14
- data/lib/switchman/active_record/connection_pool.rb +0 -12
- data/lib/switchman/active_record/log_subscriber.rb +8 -12
- data/lib/switchman/active_record/migration.rb +9 -0
- data/lib/switchman/active_record/postgresql_adapter.rb +13 -43
- data/lib/switchman/active_record/query_cache.rb +17 -107
- data/lib/switchman/active_record/statement_cache.rb +3 -24
- data/lib/switchman/connection_pool_proxy.rb +1 -3
- data/lib/switchman/database_server.rb +8 -1
- data/lib/switchman/engine.rb +8 -6
- data/lib/switchman/r_spec_helper.rb +2 -2
- data/lib/switchman/version.rb +1 -1
- data/lib/tasks/switchman.rake +3 -10
- metadata +23 -10
- data/app/models/switchman/shard_internal.rb +0 -718
@@ -30,17 +30,14 @@ module Switchman
|
|
30
30
|
end
|
31
31
|
|
32
32
|
module CollectionAssociation
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
Shard.with_each_shard(shards, [klass.shard_category, owner.class.shard_category].uniq) do
|
40
|
-
super
|
41
|
-
end
|
33
|
+
def find_target
|
34
|
+
shards = reflection.options[:multishard] && owner.respond_to?(:associated_shards) ? owner.associated_shards : [shard]
|
35
|
+
# activate both the owner and the target's shard category, so that Reflection#join_id_for,
|
36
|
+
# when called for the owner, will be returned relative to shard the query will execute on
|
37
|
+
Shard.with_each_shard(shards, [klass.shard_category, owner.class.shard_category].uniq) do
|
38
|
+
super
|
42
39
|
end
|
43
|
-
|
40
|
+
end
|
44
41
|
end
|
45
42
|
|
46
43
|
module BelongsToAssociation
|
@@ -155,6 +155,15 @@ module Switchman
|
|
155
155
|
self.class.connection.quote(id)
|
156
156
|
end
|
157
157
|
|
158
|
+
def update_columns(*)
|
159
|
+
db = Shard.current(self.class.shard_category).database_server
|
160
|
+
if ::Shackles.environment != db.shackles_environment
|
161
|
+
return db.unshackle { super }
|
162
|
+
else
|
163
|
+
super
|
164
|
+
end
|
165
|
+
end
|
166
|
+
|
158
167
|
protected
|
159
168
|
|
160
169
|
# see also AttributeMethods#shard_category_code_for_reflection
|
@@ -4,8 +4,6 @@ module Switchman
|
|
4
4
|
module ActiveRecord
|
5
5
|
module ConnectionHandler
|
6
6
|
def self.make_sharing_automagic(config, shard = Shard.current)
|
7
|
-
key = config[:adapter] == 'postgresql' ? :schema_search_path : :database
|
8
|
-
|
9
7
|
# only load the shard name from the db if we have to
|
10
8
|
if !config[:shard_name]
|
11
9
|
# we may not be able to connect to this shard yet, cause it might be an empty database server
|
@@ -15,15 +13,13 @@ module Switchman
|
|
15
13
|
|
16
14
|
config[:shard_name] ||= shard_name
|
17
15
|
end
|
18
|
-
|
19
|
-
if !config[key] || config[key] == shard_name
|
20
|
-
# this may truncate the schema_search_path if it was not specified in database.yml
|
21
|
-
# but that's what our old behavior was anyway, so I guess it's okay
|
22
|
-
config[key] = '%{shard_name}'
|
23
|
-
end
|
24
16
|
end
|
25
17
|
|
26
18
|
def establish_connection(spec)
|
19
|
+
# Just skip establishing a sharded connection if sharding isn't loaded; we'll do it again later
|
20
|
+
# This only can happen when loading ActiveRecord::Base; after everything is loaded Shard will
|
21
|
+
# be defined and this will actually establish a connection
|
22
|
+
return unless defined?(Shard)
|
27
23
|
pool = super
|
28
24
|
|
29
25
|
# this is the first place that the adapter would have been required; but now we
|
@@ -39,14 +35,12 @@ module Switchman
|
|
39
35
|
# to sharding will recurse onto itself trying to access column information
|
40
36
|
Shard.default
|
41
37
|
|
38
|
+
config = pool.spec.config
|
42
39
|
# automatically change config to allow for sharing connections with simple config
|
43
|
-
config = ::Rails.version < '5.1' ? spec.config : pool.spec.config
|
44
40
|
ConnectionHandler.make_sharing_automagic(config)
|
45
41
|
ConnectionHandler.make_sharing_automagic(Shard.default.database_server.config)
|
46
42
|
|
47
|
-
if ::Rails.version < '
|
48
|
-
::ActiveRecord::Base.configurations[::Rails.env] = spec.instance_variable_get(:@config).stringify_keys
|
49
|
-
elsif ::Rails.version < '6.0'
|
43
|
+
if ::Rails.version < '6.0'
|
50
44
|
::ActiveRecord::Base.configurations[::Rails.env] = config.stringify_keys
|
51
45
|
else
|
52
46
|
# Adopted from the deprecated code that currently lives in rails proper
|
@@ -132,8 +126,7 @@ module Switchman
|
|
132
126
|
else
|
133
127
|
ancestor_pool.spec
|
134
128
|
end
|
135
|
-
|
136
|
-
pool = establish_connection spec
|
129
|
+
pool = establish_connection(spec.to_hash)
|
137
130
|
pool.instance_variable_set(:@schema_cache, ancestor_pool.schema_cache) if ancestor_pool.schema_cache
|
138
131
|
pool
|
139
132
|
elsif spec_name != "primary"
|
@@ -84,18 +84,6 @@ module Switchman
|
|
84
84
|
end
|
85
85
|
|
86
86
|
spec.config[:shard_name] = self.shard.name
|
87
|
-
case conn.adapter_name
|
88
|
-
when 'MySQL', 'Mysql2'
|
89
|
-
conn.execute("USE #{spec.config[:database]}")
|
90
|
-
when 'PostgreSQL'
|
91
|
-
if conn.schema_search_path != spec.config[:schema_search_path]
|
92
|
-
conn.schema_search_path = spec.config[:schema_search_path]
|
93
|
-
end
|
94
|
-
when 'SQLite'
|
95
|
-
# This is an artifact of the adapter modifying the path to be an absolute path when it is instantiated; just let it slide
|
96
|
-
else
|
97
|
-
raise("Cannot switch databases on same DatabaseServer with adapter type: #{conn.adapter_name}. Limit one Shard per DatabaseServer.")
|
98
|
-
end
|
99
87
|
conn.shard = shard
|
100
88
|
end
|
101
89
|
|
@@ -18,18 +18,14 @@ module Switchman
|
|
18
18
|
shard = " [#{shard[:database_server_id]}:#{shard[:id]} #{shard[:env]}]" if shard
|
19
19
|
|
20
20
|
unless (payload[:binds] || []).empty?
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
binds = " " + payload[:binds].zip(casted_params).map { |attr, value|
|
30
|
-
render_bind(attr, value)
|
31
|
-
}.inspect
|
32
|
-
end
|
21
|
+
use_old_format = (::Rails.version < '5.1.5')
|
22
|
+
args = use_old_format ?
|
23
|
+
[payload[:binds], payload[:type_casted_binds]] :
|
24
|
+
[payload[:type_casted_binds]]
|
25
|
+
casted_params = type_casted_binds(*args)
|
26
|
+
binds = " " + payload[:binds].zip(casted_params).map { |attr, value|
|
27
|
+
render_bind(attr, value)
|
28
|
+
}.inspect
|
33
29
|
end
|
34
30
|
|
35
31
|
name = colorize_payload_name(name, payload[:name])
|
@@ -33,5 +33,14 @@ module Switchman
|
|
33
33
|
::ActiveRecord::Migrator::MIGRATOR_SALT * shard_name_hash
|
34
34
|
end
|
35
35
|
end
|
36
|
+
|
37
|
+
module MigrationContext
|
38
|
+
def migrations
|
39
|
+
return @migrations if instance_variable_defined?(:@migrations)
|
40
|
+
migrations_cache = Thread.current[:migrations_cache] ||= {}
|
41
|
+
key = Digest::MD5.hexdigest(migration_files.sort.join(','))
|
42
|
+
@migrations = migrations_cache[key] ||= super
|
43
|
+
end
|
44
|
+
end
|
36
45
|
end
|
37
46
|
end
|
@@ -38,51 +38,26 @@ module Switchman
|
|
38
38
|
select_values("SELECT * FROM unnest(current_schemas(false))")
|
39
39
|
end
|
40
40
|
|
41
|
-
def use_qualified_names?
|
42
|
-
@config[:use_qualified_names]
|
43
|
-
end
|
44
|
-
|
45
41
|
def tables(name = nil)
|
46
|
-
schema = shard.name if use_qualified_names?
|
47
|
-
|
48
42
|
query(<<-SQL, 'SCHEMA').map { |row| row[0] }
|
49
43
|
SELECT tablename
|
50
44
|
FROM pg_tables
|
51
|
-
WHERE schemaname =
|
45
|
+
WHERE schemaname = '#{shard.name}'
|
52
46
|
SQL
|
53
47
|
end
|
54
48
|
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
name.instance_variable_set(:@schema, shard.name)
|
60
|
-
end
|
61
|
-
[name.schema, name.identifier]
|
62
|
-
end
|
63
|
-
else
|
64
|
-
def data_source_exists?(name)
|
65
|
-
name = ::ActiveRecord::ConnectionAdapters::PostgreSQL::Utils.extract_schema_qualified_name(name.to_s)
|
66
|
-
return false unless name.identifier
|
67
|
-
if !name.schema && use_qualified_names?
|
68
|
-
name.instance_variable_set(:@schema, shard.name)
|
69
|
-
end
|
70
|
-
|
71
|
-
exec_query(<<-SQL, 'SCHEMA').rows.first[0].to_i > 0
|
72
|
-
SELECT COUNT(*)
|
73
|
-
FROM pg_class c
|
74
|
-
LEFT JOIN pg_namespace n ON n.oid = c.relnamespace
|
75
|
-
WHERE c.relkind IN ('r','v','m') -- (r)elation/table, (v)iew, (m)aterialized view
|
76
|
-
AND c.relname = '#{name.identifier}'
|
77
|
-
AND n.nspname = #{name.schema ? "'#{name.schema}'" : 'ANY (current_schemas(false))'}
|
78
|
-
SQL
|
49
|
+
def extract_schema_qualified_name(string)
|
50
|
+
name = ::ActiveRecord::ConnectionAdapters::PostgreSQL::Utils.extract_schema_qualified_name(string.to_s)
|
51
|
+
if string && !name.schema
|
52
|
+
name.instance_variable_set(:@schema, shard.name)
|
79
53
|
end
|
54
|
+
[name.schema, name.identifier]
|
80
55
|
end
|
81
56
|
|
82
57
|
def view_exists?(name)
|
83
58
|
name = ::ActiveRecord::ConnectionAdapters::PostgreSQL::Utils.extract_schema_qualified_name(name.to_s)
|
84
59
|
return false unless name.identifier
|
85
|
-
if !name.schema
|
60
|
+
if !name.schema
|
86
61
|
name.instance_variable_set(:@schema, shard.name)
|
87
62
|
end
|
88
63
|
|
@@ -92,13 +67,11 @@ module Switchman
|
|
92
67
|
LEFT JOIN pg_namespace n ON n.oid = c.relnamespace
|
93
68
|
WHERE c.relkind IN ('v','m') -- (v)iew, (m)aterialized view
|
94
69
|
AND c.relname = '#{name.identifier}'
|
95
|
-
AND n.nspname = #{
|
70
|
+
AND n.nspname = '#{shard.name}'
|
96
71
|
SQL
|
97
72
|
end
|
98
73
|
|
99
74
|
def indexes(table_name)
|
100
|
-
schema = shard.name if use_qualified_names?
|
101
|
-
|
102
75
|
result = query(<<-SQL, 'SCHEMA')
|
103
76
|
SELECT distinct i.relname, d.indisunique, d.indkey, pg_get_indexdef(d.indexrelid), t.oid
|
104
77
|
FROM pg_class t
|
@@ -107,7 +80,7 @@ module Switchman
|
|
107
80
|
WHERE i.relkind = 'i'
|
108
81
|
AND d.indisprimary = 'f'
|
109
82
|
AND t.relname = '#{table_name}'
|
110
|
-
AND i.relnamespace IN (SELECT oid FROM pg_namespace WHERE nspname =
|
83
|
+
AND i.relnamespace IN (SELECT oid FROM pg_namespace WHERE nspname = '#{shard.name}' )
|
111
84
|
ORDER BY i.relname
|
112
85
|
SQL
|
113
86
|
|
@@ -145,8 +118,6 @@ module Switchman
|
|
145
118
|
end
|
146
119
|
|
147
120
|
def index_name_exists?(table_name, index_name, _default = nil)
|
148
|
-
schema = shard.name if use_qualified_names?
|
149
|
-
|
150
121
|
exec_query(<<-SQL, 'SCHEMA').rows.first[0].to_i > 0
|
151
122
|
SELECT COUNT(*)
|
152
123
|
FROM pg_class t
|
@@ -155,7 +126,7 @@ module Switchman
|
|
155
126
|
WHERE i.relkind = 'i'
|
156
127
|
AND i.relname = '#{index_name}'
|
157
128
|
AND t.relname = '#{table_name}'
|
158
|
-
AND i.relnamespace IN (SELECT oid FROM pg_namespace WHERE nspname =
|
129
|
+
AND i.relnamespace IN (SELECT oid FROM pg_namespace WHERE nspname = '#{shard.name}' )
|
159
130
|
SQL
|
160
131
|
end
|
161
132
|
|
@@ -169,7 +140,7 @@ module Switchman
|
|
169
140
|
def quote_table_name(name)
|
170
141
|
return quote_local_table_name(name) if @use_local_table_name
|
171
142
|
name = ::ActiveRecord::ConnectionAdapters::PostgreSQL::Utils.extract_schema_qualified_name(name.to_s)
|
172
|
-
if !name.schema
|
143
|
+
if !name.schema
|
173
144
|
name.instance_variable_set(:@schema, shard.name)
|
174
145
|
end
|
175
146
|
name.quoted
|
@@ -184,7 +155,6 @@ module Switchman
|
|
184
155
|
end
|
185
156
|
|
186
157
|
def foreign_keys(table_name)
|
187
|
-
schema = shard.name if use_qualified_names?
|
188
158
|
|
189
159
|
# mostly copy-pasted from AR - only change is to the nspname condition for qualified names support
|
190
160
|
fk_info = select_all <<-SQL.strip_heredoc
|
@@ -197,7 +167,7 @@ module Switchman
|
|
197
167
|
JOIN pg_namespace t3 ON c.connamespace = t3.oid
|
198
168
|
WHERE c.contype = 'f'
|
199
169
|
AND t1.relname = #{quote(table_name)}
|
200
|
-
AND t3.nspname =
|
170
|
+
AND t3.nspname = '#{shard.name}'
|
201
171
|
ORDER BY c.conname
|
202
172
|
SQL
|
203
173
|
|
@@ -214,7 +184,7 @@ module Switchman
|
|
214
184
|
# strip the schema name from to_table if it matches
|
215
185
|
to_table = row['to_table']
|
216
186
|
to_table_qualified_name = ::ActiveRecord::ConnectionAdapters::PostgreSQL::Utils.extract_schema_qualified_name(to_table)
|
217
|
-
if
|
187
|
+
if to_table_qualified_name.schema == shard.name
|
218
188
|
to_table = to_table_qualified_name.identifier
|
219
189
|
end
|
220
190
|
|
@@ -1,122 +1,32 @@
|
|
1
1
|
module Switchman
|
2
2
|
module ActiveRecord
|
3
3
|
module QueryCache
|
4
|
-
if ::Rails.version < '5.0.1'
|
5
|
-
# thread local accessors to replace @query_cache_enabled
|
6
|
-
def query_cache
|
7
|
-
thread_cache = Thread.current[:query_cache] ||= {}
|
8
|
-
thread_cache[self.object_id] ||= Hash.new { |h,sql| h[sql] = {} }
|
9
|
-
end
|
10
|
-
|
11
|
-
def query_cache_enabled
|
12
|
-
Thread.current[:query_cache_enabled]
|
13
|
-
end
|
14
|
-
|
15
|
-
def query_cache_enabled=(value)
|
16
|
-
Thread.current[:query_cache_enabled] = value
|
17
|
-
end
|
18
|
-
|
19
|
-
# basically wholesale repeat of the methods from the original (see
|
20
|
-
# https://github.com/rails/rails/blob/master/activerecord/lib/active_record/connection_adapters/abstract/query_cache.rb),
|
21
|
-
# but with self.query_cache_enabled and self.query_cache_enabled= instead
|
22
|
-
# of @query_cache_enabled.
|
23
|
-
|
24
|
-
def enable_query_cache!
|
25
|
-
self.query_cache_enabled = true
|
26
|
-
end
|
27
|
-
|
28
|
-
def disable_query_cache!
|
29
|
-
self.query_cache_enabled = false
|
30
|
-
end
|
31
|
-
|
32
|
-
def cache
|
33
|
-
old, self.query_cache_enabled = query_cache_enabled, true
|
34
|
-
yield
|
35
|
-
ensure
|
36
|
-
self.query_cache_enabled = old
|
37
|
-
clear_query_cache unless self.query_cache_enabled
|
38
|
-
end
|
39
|
-
|
40
|
-
def uncached
|
41
|
-
old, self.query_cache_enabled = query_cache_enabled, false
|
42
|
-
yield
|
43
|
-
ensure
|
44
|
-
self.query_cache_enabled = old
|
45
|
-
end
|
46
|
-
|
47
|
-
def clear_query_cache
|
48
|
-
Thread.current[:query_cache]&.clear
|
49
|
-
end
|
50
|
-
|
51
|
-
def select_all(arel, name = nil, binds = [], preparable: nil)
|
52
|
-
if self.query_cache_enabled && !locked?(arel)
|
53
|
-
arel, binds = binds_from_relation(arel, binds)
|
54
|
-
sql = to_sql(arel, binds)
|
55
|
-
cache_sql(sql, binds) { super(sql, name, binds, preparable: preparable) }
|
56
|
-
else
|
57
|
-
super
|
58
|
-
end
|
59
|
-
end
|
60
|
-
|
61
|
-
# no reason to define these on the including class directly. the super
|
62
|
-
# works just as well from a method on the included module
|
63
|
-
[:insert, :update, :delete].each do |method_name|
|
64
|
-
class_eval <<-end_code, __FILE__, __LINE__ + 1
|
65
|
-
def #{method_name}(*args)
|
66
|
-
clear_query_cache if self.query_cache_enabled
|
67
|
-
super
|
68
|
-
end
|
69
|
-
end_code
|
70
|
-
end
|
71
|
-
end
|
72
4
|
|
73
5
|
private
|
74
6
|
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
7
|
+
def cache_sql(sql, name, binds)
|
8
|
+
# have to include the shard id in the cache key because of switching dbs on the same connection
|
9
|
+
sql = "#{self.shard.id}::#{sql}"
|
10
|
+
@lock.synchronize do
|
79
11
|
result =
|
80
12
|
if query_cache[sql].key?(binds)
|
81
|
-
args = {
|
82
|
-
|
83
|
-
|
13
|
+
args = {
|
14
|
+
sql: sql,
|
15
|
+
binds: binds,
|
16
|
+
name: name,
|
17
|
+
connection_id: object_id,
|
18
|
+
cached: true
|
19
|
+
}
|
20
|
+
args[:type_casted_binds] = -> { type_casted_binds(binds) } if ::Rails.version >= '5.1.5'
|
21
|
+
::ActiveSupport::Notifications.instrument(
|
22
|
+
"sql.active_record",
|
23
|
+
args
|
24
|
+
)
|
84
25
|
query_cache[sql][binds]
|
85
26
|
else
|
86
27
|
query_cache[sql][binds] = yield
|
87
28
|
end
|
88
|
-
|
89
|
-
if ::ActiveRecord::Result === result
|
90
|
-
result.dup
|
91
|
-
else
|
92
|
-
result.collect { |row| row.dup }
|
93
|
-
end
|
94
|
-
end
|
95
|
-
else
|
96
|
-
def cache_sql(sql, name, binds)
|
97
|
-
# have to include the shard id in the cache key because of switching dbs on the same connection
|
98
|
-
sql = "#{self.shard.id}::#{sql}"
|
99
|
-
@lock.synchronize do
|
100
|
-
result =
|
101
|
-
if query_cache[sql].key?(binds)
|
102
|
-
args = {
|
103
|
-
sql: sql,
|
104
|
-
binds: binds,
|
105
|
-
name: name,
|
106
|
-
connection_id: object_id,
|
107
|
-
cached: true
|
108
|
-
}
|
109
|
-
args[:type_casted_binds] = -> { type_casted_binds(binds) } if ::Rails.version >= '5.1.5'
|
110
|
-
::ActiveSupport::Notifications.instrument(
|
111
|
-
"sql.active_record",
|
112
|
-
args
|
113
|
-
)
|
114
|
-
query_cache[sql][binds]
|
115
|
-
else
|
116
|
-
query_cache[sql][binds] = yield
|
117
|
-
end
|
118
|
-
result.dup
|
119
|
-
end
|
29
|
+
result.dup
|
120
30
|
end
|
121
31
|
end
|
122
32
|
end
|
@@ -46,37 +46,16 @@ module Switchman
|
|
46
46
|
bind_values = bind_map.bind(params, current_shard, target_shard)
|
47
47
|
|
48
48
|
target_shard.activate(klass.shard_category) do
|
49
|
-
|
50
|
-
|
51
|
-
klass.find_by_sql(sql, bind_values)
|
52
|
-
else
|
53
|
-
sql = generic_query_builder(connection).sql_for(bind_values, connection)
|
54
|
-
klass.find_by_sql(sql, bind_values)
|
55
|
-
end
|
49
|
+
sql = qualified_query_builder(target_shard, klass).sql_for(bind_values, connection)
|
50
|
+
klass.find_by_sql(sql, bind_values)
|
56
51
|
end
|
57
52
|
end
|
58
53
|
|
59
|
-
if ::Rails.version < '5.
|
60
|
-
def generic_query_builder(connection)
|
61
|
-
@query_builder ||= connection.cacheable_query(@arel)
|
62
|
-
end
|
63
|
-
|
64
|
-
def qualified_query_builder(shard, klass)
|
65
|
-
@qualified_query_builders[shard.id] ||= klass.connection.cacheable_query(@arel)
|
66
|
-
end
|
67
|
-
elsif ::Rails.version < '5.2'
|
68
|
-
def generic_query_builder(connection)
|
69
|
-
@query_builder ||= connection.cacheable_query(self.class, @arel)
|
70
|
-
end
|
71
|
-
|
54
|
+
if ::Rails.version < '5.2'
|
72
55
|
def qualified_query_builder(shard, klass)
|
73
56
|
@qualified_query_builders[shard.id] ||= klass.connection.cacheable_query(self.class, @arel)
|
74
57
|
end
|
75
58
|
else
|
76
|
-
def generic_query_builder(connection)
|
77
|
-
@query_builder ||= connection.cacheable_query(self.class, @arel).first
|
78
|
-
end
|
79
|
-
|
80
59
|
def qualified_query_builder(shard, klass)
|
81
60
|
@qualified_query_builders[shard.id] ||= klass.connection.cacheable_query(self.class, @arel).first
|
82
61
|
end
|