switchman 2.0.13 → 3.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (56) hide show
  1. checksums.yaml +4 -4
  2. data/Rakefile +10 -2
  3. data/app/models/switchman/shard.rb +234 -271
  4. data/app/models/switchman/unsharded_record.rb +7 -0
  5. data/db/migrate/20130328212039_create_switchman_shards.rb +1 -1
  6. data/db/migrate/20130328224244_create_default_shard.rb +5 -5
  7. data/db/migrate/20161206323434_add_back_default_string_limits_switchman.rb +1 -0
  8. data/db/migrate/20180828183945_add_default_shard_index.rb +2 -2
  9. data/db/migrate/20180828192111_add_timestamps_to_shards.rb +7 -5
  10. data/db/migrate/20190114212900_add_unique_name_indexes.rb +5 -3
  11. data/lib/switchman.rb +3 -5
  12. data/lib/switchman/action_controller/caching.rb +2 -2
  13. data/lib/switchman/active_record/abstract_adapter.rb +1 -0
  14. data/lib/switchman/active_record/association.rb +78 -89
  15. data/lib/switchman/active_record/attribute_methods.rb +58 -52
  16. data/lib/switchman/active_record/base.rb +58 -59
  17. data/lib/switchman/active_record/calculations.rb +74 -67
  18. data/lib/switchman/active_record/connection_pool.rb +14 -41
  19. data/lib/switchman/active_record/database_configurations.rb +34 -0
  20. data/lib/switchman/active_record/database_configurations/database_config.rb +13 -0
  21. data/lib/switchman/active_record/finder_methods.rb +11 -16
  22. data/lib/switchman/active_record/log_subscriber.rb +4 -8
  23. data/lib/switchman/active_record/migration.rb +6 -47
  24. data/lib/switchman/active_record/model_schema.rb +1 -1
  25. data/lib/switchman/active_record/persistence.rb +4 -6
  26. data/lib/switchman/active_record/postgresql_adapter.rb +124 -168
  27. data/lib/switchman/active_record/predicate_builder.rb +2 -2
  28. data/lib/switchman/active_record/query_cache.rb +18 -19
  29. data/lib/switchman/active_record/query_methods.rb +172 -197
  30. data/lib/switchman/active_record/reflection.rb +6 -10
  31. data/lib/switchman/active_record/relation.rb +30 -78
  32. data/lib/switchman/active_record/spawn_methods.rb +27 -29
  33. data/lib/switchman/active_record/statement_cache.rb +18 -35
  34. data/lib/switchman/active_record/tasks/database_tasks.rb +16 -0
  35. data/lib/switchman/active_support/cache.rb +3 -5
  36. data/lib/switchman/arel.rb +13 -8
  37. data/lib/switchman/database_server.rb +121 -142
  38. data/lib/switchman/default_shard.rb +52 -16
  39. data/lib/switchman/engine.rb +61 -58
  40. data/lib/switchman/environment.rb +4 -8
  41. data/lib/switchman/errors.rb +1 -0
  42. data/lib/switchman/guard_rail.rb +6 -19
  43. data/lib/switchman/guard_rail/relation.rb +5 -7
  44. data/lib/switchman/r_spec_helper.rb +29 -37
  45. data/lib/switchman/rails.rb +14 -12
  46. data/lib/switchman/schema_cache.rb +1 -9
  47. data/lib/switchman/sharded_instrumenter.rb +1 -1
  48. data/lib/switchman/standard_error.rb +15 -3
  49. data/lib/switchman/test_helper.rb +7 -11
  50. data/lib/switchman/version.rb +1 -1
  51. data/lib/tasks/switchman.rake +54 -69
  52. metadata +87 -45
  53. data/lib/switchman/active_record/batches.rb +0 -11
  54. data/lib/switchman/active_record/connection_handler.rb +0 -172
  55. data/lib/switchman/active_record/where_clause_factory.rb +0 -36
  56. data/lib/switchman/connection_pool_proxy.rb +0 -173
@@ -3,29 +3,34 @@
3
3
  module Switchman
4
4
  module Arel
5
5
  module Table
6
- def model
7
- type_caster.model
6
+ def klass
7
+ @klass || ::ActiveRecord::Base
8
8
  end
9
9
  end
10
+
10
11
  module Visitors
11
12
  module ToSql
12
- def visit_Arel_Nodes_TableAlias *args
13
+ # rubocop:disable Naming/MethodName
14
+
15
+ def visit_Arel_Nodes_TableAlias(*args)
13
16
  o, collector = args
14
17
  collector = visit o.relation, collector
15
- collector << " "
18
+ collector << ' '
16
19
  collector << quote_local_table_name(o.name)
17
20
  end
18
21
 
19
- def visit_Arel_Attributes_Attribute *args
22
+ def visit_Arel_Attributes_Attribute(*args)
20
23
  o = args.first
21
24
  join_name = o.relation.table_alias || o.relation.name
22
25
  result = "#{quote_local_table_name join_name}.#{quote_column_name o.name}"
23
- result = args.last << result
24
- result
26
+ args.last << result
25
27
  end
26
28
 
27
- def quote_local_table_name name
29
+ # rubocop:enable Naming/MethodName
30
+
31
+ def quote_local_table_name(name)
28
32
  return name if ::Arel::Nodes::SqlLiteral === name
33
+
29
34
  @connection.quote_local_table_name(name)
30
35
  end
31
36
  end
@@ -1,6 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require "securerandom"
3
+ require 'securerandom'
4
4
 
5
5
  module Switchman
6
6
  class DatabaseServer
@@ -13,16 +13,22 @@ module Switchman
13
13
  database_servers.values
14
14
  end
15
15
 
16
+ def all_roles
17
+ @all_roles ||= all.map(&:roles).flatten.uniq
18
+ end
19
+
16
20
  def find(id_or_all)
17
- return self.all if id_or_all == :all
18
- return id_or_all.map { |id| self.database_servers[id || ::Rails.env] }.compact.uniq if id_or_all.is_a?(Array)
21
+ return all if id_or_all == :all
22
+ return id_or_all.map { |id| database_servers[id || ::Rails.env] }.compact.uniq if id_or_all.is_a?(Array)
23
+
19
24
  database_servers[id_or_all || ::Rails.env]
20
25
  end
21
26
 
22
27
  def create(settings = {})
23
- raise "database servers should be set up in database.yml" unless ::Rails.env.test?
28
+ raise 'database servers should be set up in database.yml' unless ::Rails.env.test?
29
+
24
30
  id = settings[:id]
25
- if !id
31
+ unless id
26
32
  @id ||= 0
27
33
  @id += 1
28
34
  id = @id
@@ -30,25 +36,42 @@ module Switchman
30
36
  server = DatabaseServer.new(id.to_s, settings)
31
37
  server.instance_variable_set(:@fake, true)
32
38
  database_servers[server.id] = server
39
+ ::ActiveRecord::Base.configurations.configurations <<
40
+ ::ActiveRecord::DatabaseConfigurations::HashConfig.new(::Rails.env, "#{server.id}/primary", settings)
41
+ Shard.send(:initialize_sharding)
42
+ server
33
43
  end
34
44
 
35
45
  def server_for_new_shard
36
46
  servers = all.select { |s| s.config[:open] }
37
47
  return find(nil) if servers.empty?
48
+
38
49
  servers[rand(servers.length)]
39
50
  end
40
51
 
41
52
  private
53
+
54
+ def reference_role(role)
55
+ return if all_roles.include?(role)
56
+
57
+ @all_roles << role
58
+ Shard.send(:initialize_sharding)
59
+ end
60
+
42
61
  def database_servers
43
62
  unless @database_servers
44
63
  @database_servers = {}.with_indifferent_access
45
- if ::Rails.version >= '6.0'
46
- ::ActiveRecord::Base.configurations.configurations.each do |config|
47
- @database_servers[config.env_name] = DatabaseServer.new(config.env_name, config.config)
64
+ ::ActiveRecord::Base.configurations.configurations.each do |config|
65
+ if config.name.include?('/')
66
+ name, role = config.name.split('/')
67
+ else
68
+ name, role = config.env_name, config.name
48
69
  end
49
- else
50
- ::ActiveRecord::Base.configurations.each do |(id, config)|
51
- @database_servers[id] = DatabaseServer.new(id, config)
70
+
71
+ if role == 'primary'
72
+ @database_servers[name] = DatabaseServer.new(config.env_name, config.configuration_hash)
73
+ else
74
+ @database_servers[name].roles << role
52
75
  end
53
76
  end
54
77
  end
@@ -56,15 +79,33 @@ module Switchman
56
79
  end
57
80
  end
58
81
 
82
+ attr_reader :roles
83
+
59
84
  def initialize(id = nil, config = {})
60
85
  @id = id
61
86
  @config = config.deep_symbolize_keys
62
87
  @configs = {}
88
+ @roles = [:primary]
89
+ end
90
+
91
+ def connects_to_hash
92
+ self.class.all_roles.map do |role|
93
+ config_role = role
94
+ config_role = :primary unless roles.include?(role)
95
+ config_name = :"#{id}/#{config_role}"
96
+ config_name = :primary if id == ::Rails.env && config_role == :primary
97
+ [role.to_sym, config_name]
98
+ end.to_h
63
99
  end
64
100
 
65
101
  def destroy
66
- raise "database servers should be set up in database.yml" unless ::Rails.env.test?
67
- self.class.send(:database_servers).delete(self.id) if self.id
102
+ self.class.send(:database_servers).delete(id) if id
103
+ Shard.sharded_models.each do |klass|
104
+ self.class.all_roles.each do |role|
105
+ klass.connection_handler.remove_connection_pool(klass.connection_specification_name, role: role,
106
+ shard: id.to_sym)
107
+ end
108
+ end
68
109
  end
69
110
 
70
111
  def fake?
@@ -73,14 +114,15 @@ module Switchman
73
114
 
74
115
  def config(environment = :primary)
75
116
  @configs[environment] ||= begin
76
- if @config[environment].is_a?(Array)
117
+ case @config[environment]
118
+ when Array
77
119
  @config[environment].map do |config|
78
120
  config = @config.merge((config || {}).symbolize_keys)
79
121
  # make sure GuardRail doesn't get any brilliant ideas about choosing the first possible server
80
122
  config.delete(environment)
81
123
  config
82
124
  end
83
- elsif @config[environment].is_a?(Hash)
125
+ when Hash
84
126
  @config.merge(@config[environment])
85
127
  else
86
128
  @config
@@ -111,172 +153,109 @@ module Switchman
111
153
  guard!(old_env)
112
154
  end
113
155
 
114
- def shareable?
115
- @shareable_environment_key ||= []
116
- environment = guard_rail_environment
117
- explicit_user = ::GuardRail.global_config[:username]
118
- return @shareable if @shareable_environment_key == [environment, explicit_user]
119
- @shareable_environment_key = [environment, explicit_user]
120
- if explicit_user
121
- username = explicit_user
122
- else
123
- config = self.config(environment)
124
- config = config.first if config.is_a?(Array)
125
- username = config[:username]
126
- end
127
- @shareable = username !~ /%?\{[a-zA-Z0-9_]+\}/
128
- end
129
-
130
156
  def shards
131
- if self.id == ::Rails.env
132
- Shard.where("database_server_id IS NULL OR database_server_id=?", self.id)
157
+ if id == ::Rails.env
158
+ Shard.where('database_server_id IS NULL OR database_server_id=?', id)
133
159
  else
134
- Shard.where(:database_server_id => self.id)
160
+ Shard.where(database_server_id: id)
135
161
  end
136
162
  end
137
163
 
138
- def pool_key
139
- self.id == ::Rails.env ? nil : self.id
140
- end
141
-
142
- def create_new_shard(options = {})
143
- raise NotImplementedError.new("Cannot create new shards when sharding isn't initialized") unless Shard.default.is_a?(Shard)
144
-
145
- name = options[:name]
146
- create_schema = options[:schema]
147
- # look for another shard associated with this db
148
- other_shard = self.shards.where("name<>':memory:' OR name IS NULL").order(:id).first
149
-
150
- case config[:adapter]
151
- when 'postgresql'
152
- create_statement = lambda { "CREATE SCHEMA #{name}" }
153
- password = " PASSWORD #{::ActiveRecord::Base.connection.quote(config[:password])}" if config[:password]
154
- else
155
- create_statement = lambda { "CREATE DATABASE #{name}" }
164
+ def create_new_shard(id: nil, name: nil, schema: true)
165
+ unless Shard.default.is_a?(Shard)
166
+ raise NotImplementedError,
167
+ "Cannot create new shards when sharding isn't initialized"
156
168
  end
169
+
170
+ create_statement = -> { "CREATE SCHEMA #{name}" }
171
+ password = " PASSWORD #{::ActiveRecord::Base.connection.quote(config[:password])}" if config[:password]
157
172
  sharding_config = Switchman.config
158
173
  config_create_statement = sharding_config[config[:adapter]]&.[](:create_statement)
159
174
  config_create_statement ||= sharding_config[:create_statement]
160
175
  if config_create_statement
161
176
  create_commands = Array(config_create_statement).dup
162
177
  create_statement = lambda {
163
- create_commands.map { |statement| statement.gsub('%{name}', name).gsub('%{password}', password || '') }
178
+ create_commands.map { |statement| format(statement, name: name, password: password) }
164
179
  }
165
180
  end
166
181
 
167
- create_shard = lambda do
168
- shard_id = options.fetch(:id) do
169
- case config[:adapter]
170
- when 'postgresql'
171
- id_seq = Shard.connection.quote(Shard.connection.quote_table_name('switchman_shards_id_seq'))
172
- next_id = Shard.connection.select_value("SELECT nextval(#{id_seq})")
173
- next_id.to_i
174
- else
175
- nil
176
- end
177
- end
182
+ id ||= begin
183
+ id_seq = Shard.connection.quote(Shard.connection.quote_table_name('switchman_shards_id_seq'))
184
+ next_id = Shard.connection.select_value("SELECT nextval(#{id_seq})")
185
+ next_id.to_i
186
+ end
178
187
 
179
- if name.nil?
180
- base_name = self.config[:database].to_s % self.config
181
- base_name = nil if base_name == ':memory:'
182
- base_name << '_' if base_name
183
- base_id = shard_id || SecureRandom.uuid
184
- name = "#{base_name}shard_#{base_id}"
185
- end
188
+ name ||= "#{config[:database]}_shard_#{id}"
186
189
 
187
- shard = Shard.create!(:id => shard_id,
188
- :name => name,
189
- :database_server => self)
190
+ Shard.connection.transaction do
191
+ shard = Shard.create!(id: id,
192
+ name: name,
193
+ database_server_id: self.id)
190
194
  schema_already_existed = false
191
195
 
192
196
  begin
193
197
  self.class.creating_new_shard = true
194
- shard.activate(*Shard.categories) do
195
- ::GuardRail.activate(:deploy) do
196
- begin
197
- if create_statement
198
- if (::ActiveRecord::Base.connection.select_value("SELECT 1 FROM pg_namespace WHERE nspname=#{::ActiveRecord::Base.connection.quote(name)}"))
199
- schema_already_existed = true
200
- raise "This schema already exists; cannot overwrite"
201
- end
202
- Array(create_statement.call).each do |stmt|
203
- ::ActiveRecord::Base.connection.execute(stmt)
204
- end
205
- # have to disconnect and reconnect to the correct db
206
- if self.shareable? && other_shard
207
- other_shard.activate { ::ActiveRecord::Base.connection }
208
- else
209
- ::ActiveRecord::Base.connection_pool.current_pool.disconnect!
210
- end
211
- end
212
- old_proc = ::ActiveRecord::Base.connection.raw_connection.set_notice_processor {} if config[:adapter] == 'postgresql'
213
- old_verbose = ::ActiveRecord::Migration.verbose
214
- ::ActiveRecord::Migration.verbose = false
215
-
216
- unless create_schema == false
217
- reset_column_information
218
-
219
- migrate = ::Rails.version >= '5.2' ?
220
- -> { ::ActiveRecord::Base.connection.migration_context.migrate } :
221
- -> { ::ActiveRecord::Migrator.migrate(::ActiveRecord::Migrator.migrations_paths) }
222
- if ::ActiveRecord::Base.connection.supports_ddl_transactions?
223
- ::ActiveRecord::Base.connection.transaction(requires_new: true, &migrate)
224
- else
225
- migrate.call
226
- end
227
- reset_column_information
228
- ::ActiveRecord::Base.descendants.reject { |m| m == Shard || !m.table_exists? }.each(&:define_attribute_methods)
198
+ DatabaseServer.send(:reference_role, :deploy)
199
+ ::ActiveRecord::Base.connected_to(shard: self.id.to_sym, role: :deploy) do
200
+ if create_statement
201
+ if ::ActiveRecord::Base.connection.select_value("SELECT 1 FROM pg_namespace WHERE nspname=#{::ActiveRecord::Base.connection.quote(name)}")
202
+ schema_already_existed = true
203
+ raise 'This schema already exists; cannot overwrite'
204
+ end
205
+ Array(create_statement.call).each do |stmt|
206
+ ::ActiveRecord::Base.connection.execute(stmt)
207
+ end
208
+ end
209
+ if config[:adapter] == 'postgresql'
210
+ old_proc = ::ActiveRecord::Base.connection.raw_connection.set_notice_processor do
211
+ end
212
+ end
213
+ old_verbose = ::ActiveRecord::Migration.verbose
214
+ ::ActiveRecord::Migration.verbose = false
215
+
216
+ unless schema == false
217
+ shard.activate do
218
+ reset_column_information
219
+
220
+ ::ActiveRecord::Base.connection.transaction(requires_new: true) do
221
+ ::ActiveRecord::Base.connection.migration_context.migrate
229
222
  end
230
- ensure
231
- ::ActiveRecord::Migration.verbose = old_verbose
232
- ::ActiveRecord::Base.connection.raw_connection.set_notice_processor(&old_proc) if old_proc
223
+ reset_column_information
224
+ ::ActiveRecord::Base.descendants.reject do |m|
225
+ m <= UnshardedRecord || !m.table_exists?
226
+ end.each(&:define_attribute_methods)
233
227
  end
234
228
  end
229
+ ensure
230
+ ::ActiveRecord::Migration.verbose = old_verbose
231
+ ::ActiveRecord::Base.connection.raw_connection.set_notice_processor(&old_proc) if old_proc
235
232
  end
236
233
  shard
237
234
  rescue
238
235
  shard.destroy
239
- unless schema_already_existed
240
- shard.drop_database rescue nil
241
- end
242
- reset_column_information unless create_schema == false rescue nil
236
+ shard.drop_database rescue nil unless schema_already_existed
237
+ reset_column_information unless schema == false rescue nil
243
238
  raise
244
239
  ensure
245
240
  self.class.creating_new_shard = false
246
241
  end
247
242
  end
248
-
249
- if Shard.connection.supports_ddl_transactions? && self.shareable? && other_shard
250
- Shard.transaction do
251
- other_shard.activate do
252
- ::ActiveRecord::Base.connection.transaction(&create_shard)
253
- end
254
- end
255
- else
256
- create_shard.call
257
- end
258
243
  end
259
244
 
260
245
  def cache_store
261
- unless @cache_store
262
- @cache_store = Switchman.config[:cache_map][self.id] || Switchman.config[:cache_map][::Rails.env]
263
- end
246
+ @cache_store ||= Switchman.config[:cache_map][id] || Switchman.config[:cache_map][::Rails.env]
264
247
  @cache_store
265
248
  end
266
249
 
267
250
  def shard_name(shard)
268
- if config[:shard_name]
269
- config[:shard_name]
270
- elsif config[:adapter] == 'postgresql'
271
- if shard == :bootstrap
272
- # rescue nil because the database may not exist yet; if it doesn't,
273
- # it will shortly, and this will be re-invoked
274
- ::ActiveRecord::Base.connection.current_schemas.first rescue nil
275
- else
276
- shard.activate { ::ActiveRecord::Base.connection_pool.default_schema }
277
- end
251
+ return config[:shard_name] if config[:shard_name]
252
+
253
+ if shard == :bootstrap
254
+ # rescue nil because the database may not exist yet; if it doesn't,
255
+ # it will shortly, and this will be re-invoked
256
+ ::ActiveRecord::Base.connection.current_schemas.first rescue nil
278
257
  else
279
- config[:database]
258
+ shard.activate { ::ActiveRecord::Base.connection_pool.default_schema }
280
259
  end
281
260
  end
282
261
 
@@ -290,9 +269,9 @@ module Switchman
290
269
  end
291
270
 
292
271
  private
272
+
293
273
  def reset_column_information
294
- ::ActiveRecord::Base.descendants.reject { |m| m == Shard }.each(&:reset_column_information)
295
- ::ActiveRecord::Base.connection_handler.switchman_connection_pool_proxies.each { |pool| pool.schema_cache.clear! }
274
+ ::ActiveRecord::Base.descendants.reject { |m| m <= UnshardedRecord }.each(&:reset_column_information)
296
275
  end
297
276
  end
298
277
  end
@@ -4,17 +4,44 @@ require 'switchman/database_server'
4
4
 
5
5
  module Switchman
6
6
  class DefaultShard
7
- def id; 'default'; end
7
+ def id
8
+ 'default'
9
+ end
8
10
  alias cache_key id
9
- def activate(*categories); yield; end
10
- def activate!(*categories); end
11
- def default?; true; end
12
- def primary?; true; end
13
- def relative_id_for(local_id, target = nil); local_id; end
14
- def global_id_for(local_id); local_id; end
15
- def database_server_id; nil; end
16
- def database_server; DatabaseServer.find(nil); end
17
- def new_record?; false; end
11
+ def activate(*_classes)
12
+ yield
13
+ end
14
+
15
+ def activate!(*classes); end
16
+
17
+ def default?
18
+ true
19
+ end
20
+
21
+ def primary?
22
+ true
23
+ end
24
+
25
+ def relative_id_for(local_id, _target = nil)
26
+ local_id
27
+ end
28
+
29
+ def global_id_for(local_id)
30
+ local_id
31
+ end
32
+
33
+ def database_server_id
34
+ nil
35
+ end
36
+
37
+ def database_server
38
+ DatabaseServer.find(nil)
39
+ end
40
+
41
+ def new_record?
42
+ false
43
+ end
44
+
18
45
  def name
19
46
  unless instance_variable_defined?(:@name)
20
47
  @name = nil # prevent taking this branch on recursion
@@ -22,18 +49,27 @@ module Switchman
22
49
  end
23
50
  @name
24
51
  end
25
- def description; ::Rails.env; end
52
+
53
+ def description
54
+ ::Rails.env
55
+ end
56
+
26
57
  # The default's shard is always the default shard
27
- def shard; self; end
28
- def _dump(depth)
58
+ def shard
59
+ self
60
+ end
61
+
62
+ def _dump(_depth)
29
63
  ''
30
64
  end
31
- def self._load(str)
65
+
66
+ def self._load(_str)
32
67
  Shard.default
33
68
  end
34
69
 
35
- def ==(rhs)
36
- return true if rhs.is_a?(DefaultShard) || (rhs.is_a?(Shard) && rhs[:default])
70
+ def ==(other)
71
+ return true if other.is_a?(DefaultShard) || (other.is_a?(Shard) && other[:default])
72
+
37
73
  super
38
74
  end
39
75