switchman 2.0.13 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. checksums.yaml +4 -4
  2. data/Rakefile +10 -2
  3. data/app/models/switchman/shard.rb +234 -271
  4. data/app/models/switchman/unsharded_record.rb +7 -0
  5. data/db/migrate/20130328212039_create_switchman_shards.rb +1 -1
  6. data/db/migrate/20130328224244_create_default_shard.rb +5 -5
  7. data/db/migrate/20161206323434_add_back_default_string_limits_switchman.rb +1 -0
  8. data/db/migrate/20180828183945_add_default_shard_index.rb +2 -2
  9. data/db/migrate/20180828192111_add_timestamps_to_shards.rb +7 -5
  10. data/db/migrate/20190114212900_add_unique_name_indexes.rb +5 -3
  11. data/lib/switchman.rb +3 -5
  12. data/lib/switchman/action_controller/caching.rb +2 -2
  13. data/lib/switchman/active_record/abstract_adapter.rb +1 -0
  14. data/lib/switchman/active_record/association.rb +78 -89
  15. data/lib/switchman/active_record/attribute_methods.rb +58 -52
  16. data/lib/switchman/active_record/base.rb +58 -59
  17. data/lib/switchman/active_record/calculations.rb +74 -67
  18. data/lib/switchman/active_record/connection_pool.rb +14 -41
  19. data/lib/switchman/active_record/database_configurations.rb +34 -0
  20. data/lib/switchman/active_record/database_configurations/database_config.rb +13 -0
  21. data/lib/switchman/active_record/finder_methods.rb +11 -16
  22. data/lib/switchman/active_record/log_subscriber.rb +4 -8
  23. data/lib/switchman/active_record/migration.rb +6 -47
  24. data/lib/switchman/active_record/model_schema.rb +1 -1
  25. data/lib/switchman/active_record/persistence.rb +4 -6
  26. data/lib/switchman/active_record/postgresql_adapter.rb +124 -168
  27. data/lib/switchman/active_record/predicate_builder.rb +2 -2
  28. data/lib/switchman/active_record/query_cache.rb +18 -19
  29. data/lib/switchman/active_record/query_methods.rb +172 -197
  30. data/lib/switchman/active_record/reflection.rb +6 -10
  31. data/lib/switchman/active_record/relation.rb +30 -78
  32. data/lib/switchman/active_record/spawn_methods.rb +27 -29
  33. data/lib/switchman/active_record/statement_cache.rb +18 -35
  34. data/lib/switchman/active_record/tasks/database_tasks.rb +16 -0
  35. data/lib/switchman/active_support/cache.rb +3 -5
  36. data/lib/switchman/arel.rb +13 -8
  37. data/lib/switchman/database_server.rb +121 -142
  38. data/lib/switchman/default_shard.rb +52 -16
  39. data/lib/switchman/engine.rb +61 -58
  40. data/lib/switchman/environment.rb +4 -8
  41. data/lib/switchman/errors.rb +1 -0
  42. data/lib/switchman/guard_rail.rb +6 -19
  43. data/lib/switchman/guard_rail/relation.rb +5 -7
  44. data/lib/switchman/r_spec_helper.rb +29 -37
  45. data/lib/switchman/rails.rb +14 -12
  46. data/lib/switchman/schema_cache.rb +1 -9
  47. data/lib/switchman/sharded_instrumenter.rb +1 -1
  48. data/lib/switchman/standard_error.rb +15 -3
  49. data/lib/switchman/test_helper.rb +7 -11
  50. data/lib/switchman/version.rb +1 -1
  51. data/lib/tasks/switchman.rake +54 -69
  52. metadata +87 -45
  53. data/lib/switchman/active_record/batches.rb +0 -11
  54. data/lib/switchman/active_record/connection_handler.rb +0 -172
  55. data/lib/switchman/active_record/where_clause_factory.rb +0 -36
  56. data/lib/switchman/connection_pool_proxy.rb +0 -173
@@ -3,29 +3,34 @@
3
3
  module Switchman
4
4
  module Arel
5
5
  module Table
6
- def model
7
- type_caster.model
6
+ def klass
7
+ @klass || ::ActiveRecord::Base
8
8
  end
9
9
  end
10
+
10
11
  module Visitors
11
12
  module ToSql
12
- def visit_Arel_Nodes_TableAlias *args
13
+ # rubocop:disable Naming/MethodName
14
+
15
+ def visit_Arel_Nodes_TableAlias(*args)
13
16
  o, collector = args
14
17
  collector = visit o.relation, collector
15
- collector << " "
18
+ collector << ' '
16
19
  collector << quote_local_table_name(o.name)
17
20
  end
18
21
 
19
- def visit_Arel_Attributes_Attribute *args
22
+ def visit_Arel_Attributes_Attribute(*args)
20
23
  o = args.first
21
24
  join_name = o.relation.table_alias || o.relation.name
22
25
  result = "#{quote_local_table_name join_name}.#{quote_column_name o.name}"
23
- result = args.last << result
24
- result
26
+ args.last << result
25
27
  end
26
28
 
27
- def quote_local_table_name name
29
+ # rubocop:enable Naming/MethodName
30
+
31
+ def quote_local_table_name(name)
28
32
  return name if ::Arel::Nodes::SqlLiteral === name
33
+
29
34
  @connection.quote_local_table_name(name)
30
35
  end
31
36
  end
@@ -1,6 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require "securerandom"
3
+ require 'securerandom'
4
4
 
5
5
  module Switchman
6
6
  class DatabaseServer
@@ -13,16 +13,22 @@ module Switchman
13
13
  database_servers.values
14
14
  end
15
15
 
16
+ def all_roles
17
+ @all_roles ||= all.map(&:roles).flatten.uniq
18
+ end
19
+
16
20
  def find(id_or_all)
17
- return self.all if id_or_all == :all
18
- return id_or_all.map { |id| self.database_servers[id || ::Rails.env] }.compact.uniq if id_or_all.is_a?(Array)
21
+ return all if id_or_all == :all
22
+ return id_or_all.map { |id| database_servers[id || ::Rails.env] }.compact.uniq if id_or_all.is_a?(Array)
23
+
19
24
  database_servers[id_or_all || ::Rails.env]
20
25
  end
21
26
 
22
27
  def create(settings = {})
23
- raise "database servers should be set up in database.yml" unless ::Rails.env.test?
28
+ raise 'database servers should be set up in database.yml' unless ::Rails.env.test?
29
+
24
30
  id = settings[:id]
25
- if !id
31
+ unless id
26
32
  @id ||= 0
27
33
  @id += 1
28
34
  id = @id
@@ -30,25 +36,42 @@ module Switchman
30
36
  server = DatabaseServer.new(id.to_s, settings)
31
37
  server.instance_variable_set(:@fake, true)
32
38
  database_servers[server.id] = server
39
+ ::ActiveRecord::Base.configurations.configurations <<
40
+ ::ActiveRecord::DatabaseConfigurations::HashConfig.new(::Rails.env, "#{server.id}/primary", settings)
41
+ Shard.send(:initialize_sharding)
42
+ server
33
43
  end
34
44
 
35
45
  def server_for_new_shard
36
46
  servers = all.select { |s| s.config[:open] }
37
47
  return find(nil) if servers.empty?
48
+
38
49
  servers[rand(servers.length)]
39
50
  end
40
51
 
41
52
  private
53
+
54
+ def reference_role(role)
55
+ return if all_roles.include?(role)
56
+
57
+ @all_roles << role
58
+ Shard.send(:initialize_sharding)
59
+ end
60
+
42
61
  def database_servers
43
62
  unless @database_servers
44
63
  @database_servers = {}.with_indifferent_access
45
- if ::Rails.version >= '6.0'
46
- ::ActiveRecord::Base.configurations.configurations.each do |config|
47
- @database_servers[config.env_name] = DatabaseServer.new(config.env_name, config.config)
64
+ ::ActiveRecord::Base.configurations.configurations.each do |config|
65
+ if config.name.include?('/')
66
+ name, role = config.name.split('/')
67
+ else
68
+ name, role = config.env_name, config.name
48
69
  end
49
- else
50
- ::ActiveRecord::Base.configurations.each do |(id, config)|
51
- @database_servers[id] = DatabaseServer.new(id, config)
70
+
71
+ if role == 'primary'
72
+ @database_servers[name] = DatabaseServer.new(config.env_name, config.configuration_hash)
73
+ else
74
+ @database_servers[name].roles << role
52
75
  end
53
76
  end
54
77
  end
@@ -56,15 +79,33 @@ module Switchman
56
79
  end
57
80
  end
58
81
 
82
+ attr_reader :roles
83
+
59
84
  def initialize(id = nil, config = {})
60
85
  @id = id
61
86
  @config = config.deep_symbolize_keys
62
87
  @configs = {}
88
+ @roles = [:primary]
89
+ end
90
+
91
+ def connects_to_hash
92
+ self.class.all_roles.map do |role|
93
+ config_role = role
94
+ config_role = :primary unless roles.include?(role)
95
+ config_name = :"#{id}/#{config_role}"
96
+ config_name = :primary if id == ::Rails.env && config_role == :primary
97
+ [role.to_sym, config_name]
98
+ end.to_h
63
99
  end
64
100
 
65
101
  def destroy
66
- raise "database servers should be set up in database.yml" unless ::Rails.env.test?
67
- self.class.send(:database_servers).delete(self.id) if self.id
102
+ self.class.send(:database_servers).delete(id) if id
103
+ Shard.sharded_models.each do |klass|
104
+ self.class.all_roles.each do |role|
105
+ klass.connection_handler.remove_connection_pool(klass.connection_specification_name, role: role,
106
+ shard: id.to_sym)
107
+ end
108
+ end
68
109
  end
69
110
 
70
111
  def fake?
@@ -73,14 +114,15 @@ module Switchman
73
114
 
74
115
  def config(environment = :primary)
75
116
  @configs[environment] ||= begin
76
- if @config[environment].is_a?(Array)
117
+ case @config[environment]
118
+ when Array
77
119
  @config[environment].map do |config|
78
120
  config = @config.merge((config || {}).symbolize_keys)
79
121
  # make sure GuardRail doesn't get any brilliant ideas about choosing the first possible server
80
122
  config.delete(environment)
81
123
  config
82
124
  end
83
- elsif @config[environment].is_a?(Hash)
125
+ when Hash
84
126
  @config.merge(@config[environment])
85
127
  else
86
128
  @config
@@ -111,172 +153,109 @@ module Switchman
111
153
  guard!(old_env)
112
154
  end
113
155
 
114
- def shareable?
115
- @shareable_environment_key ||= []
116
- environment = guard_rail_environment
117
- explicit_user = ::GuardRail.global_config[:username]
118
- return @shareable if @shareable_environment_key == [environment, explicit_user]
119
- @shareable_environment_key = [environment, explicit_user]
120
- if explicit_user
121
- username = explicit_user
122
- else
123
- config = self.config(environment)
124
- config = config.first if config.is_a?(Array)
125
- username = config[:username]
126
- end
127
- @shareable = username !~ /%?\{[a-zA-Z0-9_]+\}/
128
- end
129
-
130
156
  def shards
131
- if self.id == ::Rails.env
132
- Shard.where("database_server_id IS NULL OR database_server_id=?", self.id)
157
+ if id == ::Rails.env
158
+ Shard.where('database_server_id IS NULL OR database_server_id=?', id)
133
159
  else
134
- Shard.where(:database_server_id => self.id)
160
+ Shard.where(database_server_id: id)
135
161
  end
136
162
  end
137
163
 
138
- def pool_key
139
- self.id == ::Rails.env ? nil : self.id
140
- end
141
-
142
- def create_new_shard(options = {})
143
- raise NotImplementedError.new("Cannot create new shards when sharding isn't initialized") unless Shard.default.is_a?(Shard)
144
-
145
- name = options[:name]
146
- create_schema = options[:schema]
147
- # look for another shard associated with this db
148
- other_shard = self.shards.where("name<>':memory:' OR name IS NULL").order(:id).first
149
-
150
- case config[:adapter]
151
- when 'postgresql'
152
- create_statement = lambda { "CREATE SCHEMA #{name}" }
153
- password = " PASSWORD #{::ActiveRecord::Base.connection.quote(config[:password])}" if config[:password]
154
- else
155
- create_statement = lambda { "CREATE DATABASE #{name}" }
164
+ def create_new_shard(id: nil, name: nil, schema: true)
165
+ unless Shard.default.is_a?(Shard)
166
+ raise NotImplementedError,
167
+ "Cannot create new shards when sharding isn't initialized"
156
168
  end
169
+
170
+ create_statement = -> { "CREATE SCHEMA #{name}" }
171
+ password = " PASSWORD #{::ActiveRecord::Base.connection.quote(config[:password])}" if config[:password]
157
172
  sharding_config = Switchman.config
158
173
  config_create_statement = sharding_config[config[:adapter]]&.[](:create_statement)
159
174
  config_create_statement ||= sharding_config[:create_statement]
160
175
  if config_create_statement
161
176
  create_commands = Array(config_create_statement).dup
162
177
  create_statement = lambda {
163
- create_commands.map { |statement| statement.gsub('%{name}', name).gsub('%{password}', password || '') }
178
+ create_commands.map { |statement| format(statement, name: name, password: password) }
164
179
  }
165
180
  end
166
181
 
167
- create_shard = lambda do
168
- shard_id = options.fetch(:id) do
169
- case config[:adapter]
170
- when 'postgresql'
171
- id_seq = Shard.connection.quote(Shard.connection.quote_table_name('switchman_shards_id_seq'))
172
- next_id = Shard.connection.select_value("SELECT nextval(#{id_seq})")
173
- next_id.to_i
174
- else
175
- nil
176
- end
177
- end
182
+ id ||= begin
183
+ id_seq = Shard.connection.quote(Shard.connection.quote_table_name('switchman_shards_id_seq'))
184
+ next_id = Shard.connection.select_value("SELECT nextval(#{id_seq})")
185
+ next_id.to_i
186
+ end
178
187
 
179
- if name.nil?
180
- base_name = self.config[:database].to_s % self.config
181
- base_name = nil if base_name == ':memory:'
182
- base_name << '_' if base_name
183
- base_id = shard_id || SecureRandom.uuid
184
- name = "#{base_name}shard_#{base_id}"
185
- end
188
+ name ||= "#{config[:database]}_shard_#{id}"
186
189
 
187
- shard = Shard.create!(:id => shard_id,
188
- :name => name,
189
- :database_server => self)
190
+ Shard.connection.transaction do
191
+ shard = Shard.create!(id: id,
192
+ name: name,
193
+ database_server_id: self.id)
190
194
  schema_already_existed = false
191
195
 
192
196
  begin
193
197
  self.class.creating_new_shard = true
194
- shard.activate(*Shard.categories) do
195
- ::GuardRail.activate(:deploy) do
196
- begin
197
- if create_statement
198
- if (::ActiveRecord::Base.connection.select_value("SELECT 1 FROM pg_namespace WHERE nspname=#{::ActiveRecord::Base.connection.quote(name)}"))
199
- schema_already_existed = true
200
- raise "This schema already exists; cannot overwrite"
201
- end
202
- Array(create_statement.call).each do |stmt|
203
- ::ActiveRecord::Base.connection.execute(stmt)
204
- end
205
- # have to disconnect and reconnect to the correct db
206
- if self.shareable? && other_shard
207
- other_shard.activate { ::ActiveRecord::Base.connection }
208
- else
209
- ::ActiveRecord::Base.connection_pool.current_pool.disconnect!
210
- end
211
- end
212
- old_proc = ::ActiveRecord::Base.connection.raw_connection.set_notice_processor {} if config[:adapter] == 'postgresql'
213
- old_verbose = ::ActiveRecord::Migration.verbose
214
- ::ActiveRecord::Migration.verbose = false
215
-
216
- unless create_schema == false
217
- reset_column_information
218
-
219
- migrate = ::Rails.version >= '5.2' ?
220
- -> { ::ActiveRecord::Base.connection.migration_context.migrate } :
221
- -> { ::ActiveRecord::Migrator.migrate(::ActiveRecord::Migrator.migrations_paths) }
222
- if ::ActiveRecord::Base.connection.supports_ddl_transactions?
223
- ::ActiveRecord::Base.connection.transaction(requires_new: true, &migrate)
224
- else
225
- migrate.call
226
- end
227
- reset_column_information
228
- ::ActiveRecord::Base.descendants.reject { |m| m == Shard || !m.table_exists? }.each(&:define_attribute_methods)
198
+ DatabaseServer.send(:reference_role, :deploy)
199
+ ::ActiveRecord::Base.connected_to(shard: self.id.to_sym, role: :deploy) do
200
+ if create_statement
201
+ if ::ActiveRecord::Base.connection.select_value("SELECT 1 FROM pg_namespace WHERE nspname=#{::ActiveRecord::Base.connection.quote(name)}")
202
+ schema_already_existed = true
203
+ raise 'This schema already exists; cannot overwrite'
204
+ end
205
+ Array(create_statement.call).each do |stmt|
206
+ ::ActiveRecord::Base.connection.execute(stmt)
207
+ end
208
+ end
209
+ if config[:adapter] == 'postgresql'
210
+ old_proc = ::ActiveRecord::Base.connection.raw_connection.set_notice_processor do
211
+ end
212
+ end
213
+ old_verbose = ::ActiveRecord::Migration.verbose
214
+ ::ActiveRecord::Migration.verbose = false
215
+
216
+ unless schema == false
217
+ shard.activate do
218
+ reset_column_information
219
+
220
+ ::ActiveRecord::Base.connection.transaction(requires_new: true) do
221
+ ::ActiveRecord::Base.connection.migration_context.migrate
229
222
  end
230
- ensure
231
- ::ActiveRecord::Migration.verbose = old_verbose
232
- ::ActiveRecord::Base.connection.raw_connection.set_notice_processor(&old_proc) if old_proc
223
+ reset_column_information
224
+ ::ActiveRecord::Base.descendants.reject do |m|
225
+ m <= UnshardedRecord || !m.table_exists?
226
+ end.each(&:define_attribute_methods)
233
227
  end
234
228
  end
229
+ ensure
230
+ ::ActiveRecord::Migration.verbose = old_verbose
231
+ ::ActiveRecord::Base.connection.raw_connection.set_notice_processor(&old_proc) if old_proc
235
232
  end
236
233
  shard
237
234
  rescue
238
235
  shard.destroy
239
- unless schema_already_existed
240
- shard.drop_database rescue nil
241
- end
242
- reset_column_information unless create_schema == false rescue nil
236
+ shard.drop_database rescue nil unless schema_already_existed
237
+ reset_column_information unless schema == false rescue nil
243
238
  raise
244
239
  ensure
245
240
  self.class.creating_new_shard = false
246
241
  end
247
242
  end
248
-
249
- if Shard.connection.supports_ddl_transactions? && self.shareable? && other_shard
250
- Shard.transaction do
251
- other_shard.activate do
252
- ::ActiveRecord::Base.connection.transaction(&create_shard)
253
- end
254
- end
255
- else
256
- create_shard.call
257
- end
258
243
  end
259
244
 
260
245
  def cache_store
261
- unless @cache_store
262
- @cache_store = Switchman.config[:cache_map][self.id] || Switchman.config[:cache_map][::Rails.env]
263
- end
246
+ @cache_store ||= Switchman.config[:cache_map][id] || Switchman.config[:cache_map][::Rails.env]
264
247
  @cache_store
265
248
  end
266
249
 
267
250
  def shard_name(shard)
268
- if config[:shard_name]
269
- config[:shard_name]
270
- elsif config[:adapter] == 'postgresql'
271
- if shard == :bootstrap
272
- # rescue nil because the database may not exist yet; if it doesn't,
273
- # it will shortly, and this will be re-invoked
274
- ::ActiveRecord::Base.connection.current_schemas.first rescue nil
275
- else
276
- shard.activate { ::ActiveRecord::Base.connection_pool.default_schema }
277
- end
251
+ return config[:shard_name] if config[:shard_name]
252
+
253
+ if shard == :bootstrap
254
+ # rescue nil because the database may not exist yet; if it doesn't,
255
+ # it will shortly, and this will be re-invoked
256
+ ::ActiveRecord::Base.connection.current_schemas.first rescue nil
278
257
  else
279
- config[:database]
258
+ shard.activate { ::ActiveRecord::Base.connection_pool.default_schema }
280
259
  end
281
260
  end
282
261
 
@@ -290,9 +269,9 @@ module Switchman
290
269
  end
291
270
 
292
271
  private
272
+
293
273
  def reset_column_information
294
- ::ActiveRecord::Base.descendants.reject { |m| m == Shard }.each(&:reset_column_information)
295
- ::ActiveRecord::Base.connection_handler.switchman_connection_pool_proxies.each { |pool| pool.schema_cache.clear! }
274
+ ::ActiveRecord::Base.descendants.reject { |m| m <= UnshardedRecord }.each(&:reset_column_information)
296
275
  end
297
276
  end
298
277
  end
@@ -4,17 +4,44 @@ require 'switchman/database_server'
4
4
 
5
5
  module Switchman
6
6
  class DefaultShard
7
- def id; 'default'; end
7
+ def id
8
+ 'default'
9
+ end
8
10
  alias cache_key id
9
- def activate(*categories); yield; end
10
- def activate!(*categories); end
11
- def default?; true; end
12
- def primary?; true; end
13
- def relative_id_for(local_id, target = nil); local_id; end
14
- def global_id_for(local_id); local_id; end
15
- def database_server_id; nil; end
16
- def database_server; DatabaseServer.find(nil); end
17
- def new_record?; false; end
11
+ def activate(*_classes)
12
+ yield
13
+ end
14
+
15
+ def activate!(*classes); end
16
+
17
+ def default?
18
+ true
19
+ end
20
+
21
+ def primary?
22
+ true
23
+ end
24
+
25
+ def relative_id_for(local_id, _target = nil)
26
+ local_id
27
+ end
28
+
29
+ def global_id_for(local_id)
30
+ local_id
31
+ end
32
+
33
+ def database_server_id
34
+ nil
35
+ end
36
+
37
+ def database_server
38
+ DatabaseServer.find(nil)
39
+ end
40
+
41
+ def new_record?
42
+ false
43
+ end
44
+
18
45
  def name
19
46
  unless instance_variable_defined?(:@name)
20
47
  @name = nil # prevent taking this branch on recursion
@@ -22,18 +49,27 @@ module Switchman
22
49
  end
23
50
  @name
24
51
  end
25
- def description; ::Rails.env; end
52
+
53
+ def description
54
+ ::Rails.env
55
+ end
56
+
26
57
  # The default's shard is always the default shard
27
- def shard; self; end
28
- def _dump(depth)
58
+ def shard
59
+ self
60
+ end
61
+
62
+ def _dump(_depth)
29
63
  ''
30
64
  end
31
- def self._load(str)
65
+
66
+ def self._load(_str)
32
67
  Shard.default
33
68
  end
34
69
 
35
- def ==(rhs)
36
- return true if rhs.is_a?(DefaultShard) || (rhs.is_a?(Shard) && rhs[:default])
70
+ def ==(other)
71
+ return true if other.is_a?(DefaultShard) || (other.is_a?(Shard) && other[:default])
72
+
37
73
  super
38
74
  end
39
75