switchman 2.1.0 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. checksums.yaml +4 -4
  2. data/Rakefile +10 -2
  3. data/app/models/switchman/shard.rb +234 -282
  4. data/app/models/switchman/unsharded_record.rb +7 -0
  5. data/db/migrate/20130328212039_create_switchman_shards.rb +1 -1
  6. data/db/migrate/20130328224244_create_default_shard.rb +5 -5
  7. data/db/migrate/20161206323434_add_back_default_string_limits_switchman.rb +1 -0
  8. data/db/migrate/20180828183945_add_default_shard_index.rb +2 -2
  9. data/db/migrate/20180828192111_add_timestamps_to_shards.rb +7 -5
  10. data/db/migrate/20190114212900_add_unique_name_indexes.rb +5 -3
  11. data/lib/switchman/action_controller/caching.rb +2 -2
  12. data/lib/switchman/active_record/abstract_adapter.rb +1 -0
  13. data/lib/switchman/active_record/association.rb +78 -89
  14. data/lib/switchman/active_record/attribute_methods.rb +58 -52
  15. data/lib/switchman/active_record/base.rb +58 -59
  16. data/lib/switchman/active_record/calculations.rb +74 -67
  17. data/lib/switchman/active_record/connection_pool.rb +14 -41
  18. data/lib/switchman/active_record/database_configurations/database_config.rb +13 -0
  19. data/lib/switchman/active_record/database_configurations.rb +34 -0
  20. data/lib/switchman/active_record/finder_methods.rb +11 -16
  21. data/lib/switchman/active_record/log_subscriber.rb +4 -8
  22. data/lib/switchman/active_record/migration.rb +6 -47
  23. data/lib/switchman/active_record/model_schema.rb +1 -1
  24. data/lib/switchman/active_record/persistence.rb +4 -6
  25. data/lib/switchman/active_record/postgresql_adapter.rb +124 -168
  26. data/lib/switchman/active_record/predicate_builder.rb +2 -2
  27. data/lib/switchman/active_record/query_cache.rb +18 -19
  28. data/lib/switchman/active_record/query_methods.rb +172 -197
  29. data/lib/switchman/active_record/reflection.rb +7 -22
  30. data/lib/switchman/active_record/relation.rb +30 -78
  31. data/lib/switchman/active_record/spawn_methods.rb +27 -29
  32. data/lib/switchman/active_record/statement_cache.rb +18 -35
  33. data/lib/switchman/active_record/tasks/database_tasks.rb +16 -0
  34. data/lib/switchman/active_support/cache.rb +3 -5
  35. data/lib/switchman/arel.rb +13 -8
  36. data/lib/switchman/database_server.rb +121 -142
  37. data/lib/switchman/default_shard.rb +52 -16
  38. data/lib/switchman/engine.rb +61 -58
  39. data/lib/switchman/environment.rb +4 -8
  40. data/lib/switchman/errors.rb +1 -0
  41. data/lib/switchman/guard_rail/relation.rb +5 -7
  42. data/lib/switchman/guard_rail.rb +6 -19
  43. data/lib/switchman/r_spec_helper.rb +29 -37
  44. data/lib/switchman/rails.rb +14 -12
  45. data/lib/switchman/schema_cache.rb +1 -9
  46. data/lib/switchman/sharded_instrumenter.rb +1 -1
  47. data/lib/switchman/standard_error.rb +15 -3
  48. data/lib/switchman/test_helper.rb +6 -4
  49. data/lib/switchman/version.rb +1 -1
  50. data/lib/switchman.rb +3 -5
  51. data/lib/tasks/switchman.rake +54 -69
  52. metadata +87 -45
  53. data/lib/switchman/active_record/batches.rb +0 -11
  54. data/lib/switchman/active_record/connection_handler.rb +0 -190
  55. data/lib/switchman/active_record/where_clause_factory.rb +0 -36
  56. data/lib/switchman/connection_pool_proxy.rb +0 -173
@@ -6,44 +6,24 @@ require 'switchman/environment'
6
6
  require 'switchman/errors'
7
7
 
8
8
  module Switchman
9
- class Shard < ::ActiveRecord::Base
9
+ class Shard < UnshardedRecord
10
10
  # ten trillion possible ids per shard. yup.
11
11
  IDS_PER_SHARD = 10_000_000_000_000
12
12
 
13
- CATEGORIES =
14
- {
15
- # special cased to mean all other models
16
- :primary => nil,
17
- # special cased to not allow activating a shard other than the default
18
- :unsharded => [Shard]
19
- }
20
- private_constant :CATEGORIES
21
- @connection_specification_name = 'unsharded'
22
-
23
- if defined?(::ProtectedAttributes)
24
- attr_accessible :default, :name, :database_server
25
- end
26
-
27
13
  # only allow one default
28
- validates_uniqueness_of :default, :if => lambda { |s| s.default? }
14
+ validates_uniqueness_of :default, if: ->(s) { s.default? }
29
15
 
30
16
  after_save :clear_cache
31
17
  after_destroy :clear_cache
32
18
 
33
- after_rollback :on_rollback
34
-
35
19
  scope :primary, -> { where(name: nil).order(:database_server_id, :id).distinct_on(:database_server_id) }
36
20
 
37
21
  class << self
38
- def categories
39
- CATEGORIES.keys
22
+ def sharded_models
23
+ @sharded_models ||= [::ActiveRecord::Base, UnshardedRecord].freeze
40
24
  end
41
25
 
42
- def default(reload_deprecated = false, reload: false, with_fallback: false)
43
- if reload_deprecated
44
- reload = reload_deprecated
45
- ::ActiveSupport::Deprecation.warn("positional reload parameter to Switchman::Shard.default is deprecated; use `reload: true`")
46
- end
26
+ def default(reload: false, with_fallback: false)
47
27
  if !@default || reload
48
28
  # Have to create a dummy object so that several key methods still work
49
29
  # (it's easier to do this in one place here, and just assume that sharding
@@ -54,74 +34,68 @@ module Switchman
54
34
 
55
35
  # if we already have a default shard in place, and the caller wants
56
36
  # to use it as a fallback, use that instead of the dummy instance
57
- if with_fallback && @default
58
- default = @default
59
- end
37
+ default = @default if with_fallback && @default
60
38
 
61
39
  # the first time we need a dummy dummy for re-entrancy to avoid looping on ourselves
62
40
  @default ||= default
63
41
 
64
42
  # Now find the actual record, if it exists; rescue the fake default if the table doesn't exist
65
43
  @default = begin
66
- find_cached("default_shard") { Shard.where(default: true).take } || default
44
+ find_cached('default_shard') { Shard.where(default: true).take } || default
67
45
  rescue
68
46
  default
69
47
  end
70
48
 
71
- # rebuild current shard activations - it might have "another" default shard serialized there
72
- active_shards.replace(active_shards.dup.map do |category, shard|
73
- shard = Shard.lookup((!shard || shard.default?) ? 'default' : shard.id)
74
- [category, shard]
75
- end.to_h)
76
-
77
- activate!(primary: @default) if active_shards.empty?
78
-
79
49
  # make sure this is not erroneously cached
80
- if @default.database_server.instance_variable_defined?(:@primary_shard)
81
- @default.database_server.remove_instance_variable(:@primary_shard)
82
- end
50
+ @default.database_server.remove_instance_variable(:@primary_shard) if @default.database_server.instance_variable_defined?(:@primary_shard)
83
51
 
84
52
  # and finally, check for cached references to the default shard on the existing connection
85
- if ::ActiveRecord::Base.connected? && ::ActiveRecord::Base.connection.shard.default?
86
- ::ActiveRecord::Base.connection.shard = @default
53
+ sharded_models.each do |klass|
54
+ klass.connection.shard = @default if klass.connected? && klass.connection.shard.default?
87
55
  end
88
56
  end
89
57
  @default
90
58
  end
91
59
 
92
- def current(category = :primary)
93
- active_shards[category] || Shard.default
60
+ def current(klass = ::ActiveRecord::Base)
61
+ klass ||= ::ActiveRecord::Base
62
+ klass.connection_pool.shard
94
63
  end
95
64
 
96
65
  def activate(shards)
97
- old_shards = activate!(shards)
66
+ activated_classes = activate!(shards)
98
67
  yield
99
68
  ensure
100
- active_shards.merge!(old_shards) if old_shards
69
+ activated_classes.each do |klass|
70
+ klass.connection_pool.shard_stack.pop
71
+ klass.connected_to_stack.pop
72
+ end
101
73
  end
102
74
 
103
75
  def activate!(shards)
104
- old_shards = nil
105
- currently_active_shards = active_shards
106
- shards.each do |category, shard|
107
- next if category == :unsharded
108
- unless currently_active_shards[category] == shard
109
- old_shards ||= {}
110
- old_shards[category] = currently_active_shards[category]
111
- currently_active_shards[category] = shard
112
- end
76
+ activated_classes = []
77
+ shards.each do |klass, shard|
78
+ next if klass == UnshardedRecord
79
+
80
+ next unless klass.current_shard != shard.database_server.id.to_sym ||
81
+ klass.connection_pool.shard != shard
82
+
83
+ activated_classes << klass
84
+ klass.connected_to_stack << { shard: shard.database_server.id.to_sym, klasses: [klass] }
85
+ klass.connection_pool.shard_stack << shard
113
86
  end
114
- old_shards
87
+ activated_classes
115
88
  end
116
89
 
117
90
  def lookup(id)
118
91
  id_i = id.to_i
119
92
  return current if id_i == current.id || id == 'self'
120
93
  return default if id_i == default.id || id.nil? || id == 'default'
94
+
121
95
  id = id_i
122
- raise ArgumentError if id == 0
96
+ raise ArgumentError if id.zero?
123
97
 
124
- unless cached_shards.has_key?(id)
98
+ unless cached_shards.key?(id)
125
99
  cached_shards[id] = Shard.default.activate do
126
100
  find_cached(['shard', id]) { find_by(id: id) }
127
101
  end
@@ -136,59 +110,47 @@ module Switchman
136
110
  # ==== Parameters
137
111
  #
138
112
  # * +shards+ - an array or relation of Shards to iterate over
139
- # * +categories+ - an array of categories to activate
140
- # * +options+ -
141
- # :parallel - true/false to execute in parallel, or a integer of how many
113
+ # * +classes+ - an array of classes to activate
114
+ # parallel: - true/false to execute in parallel, or a integer of how many
142
115
  # sub-processes per database server. Note that parallel
143
116
  # invocation currently uses forking, so should be used sparingly
144
117
  # because errors are not raised, and you cannot get results back
145
- # :max_procs - only run this many parallel processes at a time
146
- # :exception - :ignore, :raise, :defer (wait until the end and raise the first
118
+ # max_procs: - only run this many parallel processes at a time
119
+ # exception: - :ignore, :raise, :defer (wait until the end and raise the first
147
120
  # error), or a proc
148
- def with_each_shard(*args)
149
- raise ArgumentError, "wrong number of arguments (#{args.length} for 0...3)" if args.length > 3
121
+ def with_each_shard(*args, parallel: false, max_procs: nil, exception: :raise, &block)
122
+ raise ArgumentError, "wrong number of arguments (#{args.length} for 0...2)" if args.length > 2
150
123
 
151
- unless default.is_a?(Shard)
152
- return Array.wrap(yield)
153
- end
124
+ return Array.wrap(yield) unless default.is_a?(Shard)
154
125
 
155
- options = args.extract_options!
156
126
  if args.length == 1
157
- if Array === args.first && args.first.first.is_a?(Symbol)
158
- categories = args.first
127
+ if Array === args.first && args.first.first.is_a?(Class)
128
+ classes = args.first
159
129
  else
160
130
  scope = args.first
161
131
  end
162
132
  else
163
- scope, categories = args
133
+ scope, classes = args
164
134
  end
165
135
 
166
- parallel = case options[:parallel]
167
- when true
168
- 1
169
- when false, nil
170
- 0
171
- else
172
- options[:parallel]
173
- end
174
- options.delete(:parallel)
136
+ parallel = 1 if parallel == true
137
+ parallel = 0 if parallel == false || parallel.nil?
175
138
 
176
139
  scope ||= Shard.all
177
- if ::ActiveRecord::Relation === scope && scope.order_values.empty?
178
- scope = scope.order(::Arel.sql("database_server_id IS NOT NULL, database_server_id, id"))
179
- end
140
+ scope = scope.order(::Arel.sql('database_server_id IS NOT NULL, database_server_id, id')) if ::ActiveRecord::Relation === scope && scope.order_values.empty?
180
141
 
181
- if parallel > 0
182
- max_procs = determine_max_procs(options.delete(:max_procs), parallel)
142
+ if parallel.positive?
143
+ max_procs = determine_max_procs(max_procs, parallel)
183
144
  if ::ActiveRecord::Relation === scope
184
145
  # still need a post-uniq, cause the default database server could be NULL or Rails.env in the db
185
146
  database_servers = scope.reorder('database_server_id').select(:database_server_id).distinct.
186
- map(&:database_server).compact.uniq
147
+ map(&:database_server).compact.uniq
187
148
  # nothing to do
188
- return if database_servers.count == 0
149
+ return if database_servers.count.zero?
150
+
189
151
  parallel = [(max_procs.to_f / database_servers.count).ceil, parallel].min if max_procs
190
152
 
191
- scopes = Hash[database_servers.map do |server|
153
+ scopes = database_servers.map do |server|
192
154
  server_scope = server.shards.merge(scope)
193
155
  if parallel == 1
194
156
  subscopes = [server_scope]
@@ -196,28 +158,28 @@ module Switchman
196
158
  subscopes = []
197
159
  total = server_scope.count
198
160
  ranges = []
199
- server_scope.find_ids_in_ranges(:batch_size => (total.to_f / parallel).ceil) do |min, max|
161
+ server_scope.find_ids_in_ranges(batch_size: (total.to_f / parallel).ceil) do |min, max|
200
162
  ranges << [min, max]
201
163
  end
202
164
  # create a half-open range on the last one
203
165
  ranges.last[1] = nil
204
166
  ranges.each do |min, max|
205
- subscope = server_scope.where("id>=?", min)
206
- subscope = subscope.where("id<=?", max) if max
167
+ subscope = server_scope.where('id>=?', min)
168
+ subscope = subscope.where('id<=?', max) if max
207
169
  subscopes << subscope
208
170
  end
209
171
  end
210
172
  [server, subscopes]
211
- end]
173
+ end.to_h
212
174
  else
213
175
  scopes = scope.group_by(&:database_server)
214
176
  if parallel > 1
215
177
  parallel = [(max_procs.to_f / scopes.count).ceil, parallel].min if max_procs
216
- scopes = Hash[scopes.map do |(server, shards)|
178
+ scopes = scopes.map do |(server, shards)|
217
179
  [server, shards.in_groups(parallel, false).compact]
218
- end]
180
+ end.to_h
219
181
  else
220
- scopes = Hash[scopes.map { |(server, shards)| [server, [shards]] }]
182
+ scopes = scopes.map { |(server, shards)| [server, [shards]] }.to_h
221
183
  end
222
184
  end
223
185
 
@@ -229,8 +191,8 @@ module Switchman
229
191
  fd_to_name_map = {}
230
192
  errors = []
231
193
 
232
- wait_for_output = lambda do |out_fds, err_fds, fd_to_name_map|
233
- ready, _ = IO.select(out_fds + err_fds)
194
+ wait_for_output = lambda do
195
+ ready, = IO.select(out_fds + err_fds)
234
196
  ready.each do |fd|
235
197
  if fd.eof?
236
198
  fd.close
@@ -245,7 +207,8 @@ module Switchman
245
207
 
246
208
  # only one process; don't bother forking
247
209
  if scopes.length == 1 && parallel == 1
248
- return with_each_shard(scopes.first.last.first, categories, options) { yield }
210
+ return with_each_shard(scopes.first.last.first, classes, exception: exception,
211
+ &block)
249
212
  end
250
213
 
251
214
  # clear connections prior to forking (no more queries will be executed in the parent,
@@ -255,62 +218,47 @@ module Switchman
255
218
 
256
219
  scopes.each do |server, subscopes|
257
220
  subscopes.each_with_index do |subscope, idx|
258
- if subscopes.length > 1
259
- name = "#{server.id} #{idx + 1}"
260
- else
261
- name = server.id
262
- end
221
+ name = if subscopes.length > 1
222
+ "#{server.id} #{idx + 1}"
223
+ else
224
+ server.id
225
+ end
263
226
 
264
227
  exception_pipe = IO.pipe
265
228
  exception_pipes << exception_pipe
266
229
  pid, io_in, io_out, io_err = Open4.pfork4(lambda do
230
+ Switchman.config[:on_fork_proc]&.call
231
+
232
+ # set a pretty name for the process title, up to 128 characters
233
+ # (we don't actually know the limit, depending on how the process
234
+ # was started)
235
+ # first, simplify the binary name by stripping directories,
236
+ # then truncate arguments as necessary
237
+ bin = File.basename($0) # Process.argv0 doesn't work on Ruby 2.5 (https://bugs.ruby-lang.org/issues/15887)
238
+ max_length = 128 - bin.length - name.length - 3
239
+ args = ARGV.join(' ')
240
+ args = args[0..max_length] if max_length >= 0
241
+ new_title = [bin, args, name].join(' ')
242
+ Process.setproctitle(new_title)
243
+
244
+ with_each_shard(subscope, classes, exception: exception, &block)
245
+ exception_pipe.last.close
246
+ rescue => e
267
247
  begin
268
- Switchman.config[:on_fork_proc]&.call
269
-
270
- # set a pretty name for the process title, up to 128 characters
271
- # (we don't actually know the limit, depending on how the process
272
- # was started)
273
- # first, simplify the binary name by stripping directories,
274
- # then truncate arguments as necessary
275
- bin = File.basename($0) # Process.argv0 doesn't work on Ruby 2.5 (https://bugs.ruby-lang.org/issues/15887)
276
- max_length = 128 - bin.length - name.length - 3
277
- args = ARGV.join(" ")
278
- if max_length >= 0
279
- args = args[0..max_length]
280
- end
281
- new_title = [bin, args, name].join(" ")
282
- Process.setproctitle(new_title)
283
-
284
- with_each_shard(subscope, categories, options) { yield }
285
- exception_pipe.last.close
286
- rescue Exception => e
287
- begin
288
- dumped = Marshal.dump(e)
289
- dumped = nil if dumped.length > 64 * 1024
290
- rescue
291
- dumped = nil
292
- end
293
-
294
- if dumped.nil?
295
- # couldn't dump the exception; create a copy with just
296
- # the message and the backtrace
297
- e2 = e.class.new(e.message)
298
- backtrace = e.backtrace
299
- # truncate excessively long backtraces
300
- if backtrace.length > 50
301
- backtrace = backtrace[0...25] + ['...'] + backtrace[-25..-1]
302
- end
303
- e2.set_backtrace(backtrace)
304
- e2.instance_variable_set(:@active_shards, e.instance_variable_get(:@active_shards))
305
- dumped = Marshal.dump(e2)
306
- end
307
-
308
- exception_pipe.last.set_encoding(dumped.encoding)
309
- exception_pipe.last.write(dumped)
310
- exception_pipe.last.flush
311
- exception_pipe.last.close
312
- exit! 1
248
+ dumped = Marshal.dump(e)
249
+ rescue
250
+ # couldn't dump the exception; create a copy with just
251
+ # the message and the backtrace
252
+ e2 = e.class.new(e.message)
253
+ e2.set_backtrace(e.backtrace)
254
+ e2.instance_variable_set(:@active_shards, e.instance_variable_get(:@active_shards))
255
+ dumped = Marshal.dump(e2)
313
256
  end
257
+ exception_pipe.last.set_encoding(dumped.encoding)
258
+ exception_pipe.last.write(dumped)
259
+ exception_pipe.last.flush
260
+ exception_pipe.last.close
261
+ exit! 1
314
262
  end)
315
263
  exception_pipe.last.close
316
264
  pids << pid
@@ -324,7 +272,7 @@ module Switchman
324
272
  while max_procs && pids.count >= max_procs
325
273
  while max_procs && out_fds.count >= max_procs
326
274
  # wait for output if we've hit the max_procs limit
327
- wait_for_output.call(out_fds, err_fds, fd_to_name_map)
275
+ wait_for_output.call
328
276
  end
329
277
  # we've gotten all the output from one fd so wait for its child process to exit
330
278
  found_pid, status = Process.wait2
@@ -334,9 +282,7 @@ module Switchman
334
282
  end
335
283
  end
336
284
 
337
- while out_fds.any? || err_fds.any?
338
- wait_for_output.call(out_fds, err_fds, fd_to_name_map)
339
- end
285
+ wait_for_output.call while out_fds.any? || err_fds.any?
340
286
  pids.each do |pid|
341
287
  _, status = Process.waitpid2(pid)
342
288
  errors << pid_to_name_map[pid] if status.exitstatus != 0
@@ -344,67 +290,51 @@ module Switchman
344
290
 
345
291
  # check for an exception; we only re-raise the first one
346
292
  exception_pipes.each do |exception_pipe|
347
- begin
348
- serialized_exception = exception_pipe.first.read
349
- next if serialized_exception.empty?
350
- exception = Marshal.load(serialized_exception)
351
- raise exception
352
- ensure
353
- exception_pipe.first.close
354
- end
293
+ serialized_exception = exception_pipe.first.read
294
+ next if serialized_exception.empty?
295
+
296
+ ex = Marshal.load(serialized_exception) # rubocop:disable Security/MarshalLoad
297
+ raise ex
298
+ ensure
299
+ exception_pipe.first.close
355
300
  end
356
301
 
357
302
  unless errors.empty?
358
- raise ParallelShardExecError.new("The following subprocesses did not exit cleanly: #{errors.sort.join(", ")}")
303
+ raise ParallelShardExecError,
304
+ "The following subprocesses did not exit cleanly: #{errors.sort.join(', ')}"
359
305
  end
306
+
360
307
  return
361
308
  end
362
309
 
363
- categories ||= []
310
+ classes ||= []
364
311
 
365
312
  previous_shard = nil
366
- close_connections_if_needed = lambda do |shard|
367
- # prune the prior connection unless it happened to be the same
368
- if previous_shard && shard != previous_shard && !previous_shard.database_server.shareable?
369
- previous_shard.activate do
370
- ::GuardRail.activated_environments.each do |env|
371
- ::GuardRail.activate(env) do
372
- if ::ActiveRecord::Base.connected? && ::ActiveRecord::Base.connection.open_transactions == 0
373
- ::ActiveRecord::Base.connection_pool.current_pool.disconnect!
374
- end
375
- end
376
- end
377
- end
378
- end
379
- end
380
-
381
313
  result = []
382
- exception = nil
314
+ ex = nil
383
315
  scope.each do |shard|
384
316
  # shard references a database server that isn't configured in this environment
385
317
  next unless shard.database_server
386
- close_connections_if_needed.call(shard)
387
- shard.activate(*categories) do
388
- begin
389
- result.concat Array.wrap(yield)
390
- rescue
391
- case options[:exception]
392
- when :ignore
393
- when :defer
394
- exception ||= $!
395
- when Proc
396
- options[:exception].call
397
- when :raise
398
- raise
399
- else
400
- raise
401
- end
318
+
319
+ shard.activate(*classes) do
320
+ result.concat Array.wrap(yield)
321
+ rescue
322
+ case exception
323
+ when :ignore
324
+ # ignore
325
+ when :defer
326
+ ex ||= $!
327
+ when Proc
328
+ exception.call
329
+ # when :raise
330
+ else
331
+ raise
402
332
  end
403
333
  end
404
334
  previous_shard = shard
405
335
  end
406
- close_connections_if_needed.call(Shard.current)
407
- raise exception if exception
336
+ raise ex if ex
337
+
408
338
  result
409
339
  end
410
340
 
@@ -413,22 +343,22 @@ module Switchman
413
343
  array.each do |object|
414
344
  partition_object = partition_proc ? partition_proc.call(object) : object
415
345
  case partition_object
416
- when Shard
417
- shard = partition_object
418
- when ::ActiveRecord::Base
419
- if partition_object.respond_to?(:associated_shards)
420
- partition_object.associated_shards.each do |a_shard|
421
- shard_arrays[a_shard] ||= []
422
- shard_arrays[a_shard] << object
423
- end
424
- next
425
- else
426
- shard = partition_object.shard
346
+ when Shard
347
+ shard = partition_object
348
+ when ::ActiveRecord::Base
349
+ if partition_object.respond_to?(:associated_shards)
350
+ partition_object.associated_shards.each do |a_shard|
351
+ shard_arrays[a_shard] ||= []
352
+ shard_arrays[a_shard] << object
427
353
  end
428
- when Integer, /^\d+$/, /^(\d+)~(\d+)$/
429
- local_id, shard = Shard.local_id_for(partition_object)
430
- local_id ||= partition_object
431
- object = local_id if !partition_proc
354
+ next
355
+ else
356
+ shard = partition_object.shard
357
+ end
358
+ when Integer, /^\d+$/, /^(\d+)~(\d+)$/
359
+ local_id, shard = Shard.local_id_for(partition_object)
360
+ local_id ||= partition_object
361
+ object = local_id unless partition_proc
432
362
  end
433
363
  shard ||= Shard.current
434
364
  shard_arrays[shard] ||= []
@@ -437,7 +367,7 @@ module Switchman
437
367
  # TODO: use with_each_shard (or vice versa) to get
438
368
  # connection management and parallelism benefits
439
369
  shard_arrays.inject([]) do |results, (shard, objects)|
440
- results.concat shard.activate { Array.wrap(yield objects) }
370
+ results.concat(shard.activate { Array.wrap(yield objects) })
441
371
  end
442
372
  end
443
373
 
@@ -449,7 +379,7 @@ module Switchman
449
379
  # stay as provided. This assumes no consumer
450
380
  # will return a nil value from the block.
451
381
  def signed_id_operation(input_id)
452
- sign = input_id < 0 ? -1 : 1
382
+ sign = input_id.negative? ? -1 : 1
453
383
  output = yield input_id.abs
454
384
  output * sign
455
385
  end
@@ -457,9 +387,10 @@ module Switchman
457
387
  # converts an AR object, integral id, string id, or string short-global-id to a
458
388
  # integral id. nil if it can't be interpreted
459
389
  def integral_id_for(any_id)
460
- if any_id.is_a?(::Arel::Nodes::Casted)
461
- any_id = any_id.val
462
- elsif any_id.is_a?(::Arel::Nodes::BindParam) && ::Rails.version >= "5.2"
390
+ case any_id
391
+ when ::Arel::Nodes::Casted
392
+ any_id = any_id.value
393
+ when ::Arel::Nodes::BindParam
463
394
  any_id = any_id.value.value_before_type_cast
464
395
  end
465
396
 
@@ -470,12 +401,11 @@ module Switchman
470
401
  local_id = $2.to_i
471
402
  signed_id_operation(local_id) do |id|
472
403
  return nil if id > IDS_PER_SHARD
404
+
473
405
  $1.to_i * IDS_PER_SHARD + id
474
406
  end
475
407
  when Integer, /^-?\d+$/
476
408
  any_id.to_i
477
- else
478
- nil
479
409
  end
480
410
  end
481
411
 
@@ -487,11 +417,12 @@ module Switchman
487
417
  def local_id_for(any_id)
488
418
  id = integral_id_for(any_id)
489
419
  return NIL_NIL_ID unless id
420
+
490
421
  return_shard = nil
491
422
  local_id = signed_id_operation(id) do |abs_id|
492
423
  if abs_id < IDS_PER_SHARD
493
424
  abs_id
494
- elsif return_shard = lookup(abs_id / IDS_PER_SHARD)
425
+ elsif (return_shard = lookup(abs_id / IDS_PER_SHARD))
495
426
  abs_id % IDS_PER_SHARD
496
427
  else
497
428
  return NIL_NIL_ID
@@ -508,8 +439,10 @@ module Switchman
508
439
  integral_id = integral_id_for(any_id)
509
440
  local_id, shard = local_id_for(integral_id)
510
441
  return integral_id unless local_id
442
+
511
443
  shard ||= source_shard
512
444
  return local_id if shard == target_shard
445
+
513
446
  shard.global_id_for(local_id)
514
447
  end
515
448
 
@@ -520,6 +453,7 @@ module Switchman
520
453
  local_id, shard = local_id_for(any_id)
521
454
  return any_id unless local_id
522
455
  return local_id unless shard
456
+
523
457
  "#{shard.id}~#{local_id}"
524
458
  end
525
459
 
@@ -528,6 +462,7 @@ module Switchman
528
462
  def global_id_for(any_id, source_shard = nil)
529
463
  id = integral_id_for(any_id)
530
464
  return any_id unless id
465
+
531
466
  signed_id_operation(id) do |abs_id|
532
467
  if abs_id >= IDS_PER_SHARD
533
468
  abs_id
@@ -540,29 +475,59 @@ module Switchman
540
475
 
541
476
  def shard_for(any_id, source_shard = nil)
542
477
  return any_id.shard if any_id.is_a?(::ActiveRecord::Base)
478
+
543
479
  _, shard = local_id_for(any_id)
544
480
  shard || source_shard || Shard.current
545
481
  end
546
482
 
547
483
  # given the provided option, determines whether we need to (and whether
548
484
  # it's possible) to determine a reasonable default.
549
- def determine_max_procs(max_procs_input, parallel_input=2)
485
+ def determine_max_procs(max_procs_input, parallel_input = 2)
550
486
  max_procs = nil
551
487
  if max_procs_input
552
488
  max_procs = max_procs_input.to_i
553
- max_procs = nil if max_procs == 0
489
+ max_procs = nil if max_procs.zero?
554
490
  else
555
491
  return 1 if parallel_input.nil? || parallel_input < 1
492
+
556
493
  cpus = Environment.cpu_count
557
- if cpus && cpus > 0
558
- max_procs = cpus * parallel_input
559
- end
494
+ max_procs = cpus * parallel_input if cpus&.positive?
560
495
  end
561
496
 
562
- return max_procs
497
+ max_procs
563
498
  end
564
499
 
565
500
  private
501
+
502
+ def add_sharded_model(klass)
503
+ @sharded_models = (sharded_models + [klass]).freeze
504
+ initialize_sharding
505
+ end
506
+
507
+ def initialize_sharding
508
+ full_connects_to_hash = DatabaseServer.all.map { |db| [db.id.to_sym, db.connects_to_hash] }.to_h
509
+ sharded_models.each do |klass|
510
+ connects_to_hash = full_connects_to_hash.deep_dup
511
+ if klass == UnshardedRecord
512
+ # no need to mention other databases for the unsharded category
513
+ connects_to_hash = { ::Rails.env.to_sym => DatabaseServer.find(nil).connects_to_hash }
514
+ end
515
+
516
+ # prune things we're already connected to
517
+ if klass.connection_specification_name == klass.name
518
+ connects_to_hash.each do |(db_name, role_hash)|
519
+ role_hash.each_key do |role|
520
+ role_hash.delete(role) if klass.connection_handler.retrieve_connection_pool(
521
+ klass.connection_specification_name, role: role, shard: db_name
522
+ )
523
+ end
524
+ end
525
+ end
526
+
527
+ klass.connects_to shards: connects_to_hash
528
+ end
529
+ end
530
+
566
531
  # in-process caching
567
532
  def cached_shards
568
533
  @cached_shards ||= {}.compare_by_identity
@@ -593,10 +558,6 @@ module Switchman
593
558
  shard = nil unless shard.database_server
594
559
  shard
595
560
  end
596
-
597
- def active_shards
598
- Thread.current[:active_shards] ||= {}.compare_by_identity
599
- end
600
561
  end
601
562
 
602
563
  def name
@@ -610,11 +571,11 @@ module Switchman
610
571
 
611
572
  def name=(name)
612
573
  write_attribute(:name, @name = name)
613
- remove_instance_variable(:@name) if name == nil
574
+ remove_instance_variable(:@name) if name.nil?
614
575
  end
615
576
 
616
577
  def database_server
617
- @database_server ||= DatabaseServer.find(self.database_server_id)
578
+ @database_server ||= DatabaseServer.find(database_server_id)
618
579
  end
619
580
 
620
581
  def database_server=(database_server)
@@ -635,23 +596,21 @@ module Switchman
635
596
  Shard.default
636
597
  end
637
598
 
638
- def activate(*categories)
639
- shards = hashify_categories(categories)
640
- Shard.activate(shards) do
641
- yield
642
- end
599
+ def activate(*classes, &block)
600
+ shards = hashify_classes(classes)
601
+ Shard.activate(shards, &block)
643
602
  end
644
603
 
645
604
  # for use from console ONLY
646
- def activate!(*categories)
647
- shards = hashify_categories(categories)
605
+ def activate!(*classes)
606
+ shards = hashify_classes(classes)
648
607
  Shard.activate!(shards)
649
608
  nil
650
609
  end
651
610
 
652
611
  # custom serialization, since shard is self-referential
653
- def _dump(depth)
654
- self.id.to_s
612
+ def _dump(_depth)
613
+ id.to_s
655
614
  end
656
615
 
657
616
  def self._load(str)
@@ -659,45 +618,45 @@ module Switchman
659
618
  end
660
619
 
661
620
  def drop_database
662
- raise("Cannot drop the database of the default shard") if self.default?
621
+ raise('Cannot drop the database of the default shard') if default?
663
622
  return unless read_attribute(:name)
664
623
 
665
624
  begin
666
- adapter = self.database_server.config[:adapter]
625
+ adapter = database_server.config[:adapter]
667
626
  sharding_config = Switchman.config || {}
668
627
  drop_statement = sharding_config[adapter]&.[](:drop_statement)
669
628
  drop_statement ||= sharding_config[:drop_statement]
670
629
  if drop_statement
671
630
  drop_statement = Array(drop_statement).dup.
672
- map { |statement| statement.gsub('%{name}', self.name) }
631
+ map { |statement| statement.gsub('%{name}', name) }
673
632
  end
674
633
 
675
634
  case adapter
676
- when 'mysql', 'mysql2'
677
- self.activate do
678
- ::GuardRail.activate(:deploy) do
679
- drop_statement ||= "DROP DATABASE #{self.name}"
680
- Array(drop_statement).each do |stmt|
681
- ::ActiveRecord::Base.connection.execute(stmt)
682
- end
635
+ when 'mysql', 'mysql2'
636
+ activate do
637
+ ::GuardRail.activate(:deploy) do
638
+ drop_statement ||= "DROP DATABASE #{name}"
639
+ Array(drop_statement).each do |stmt|
640
+ ::ActiveRecord::Base.connection.execute(stmt)
683
641
  end
684
642
  end
685
- when 'postgresql'
686
- self.activate do
687
- ::GuardRail.activate(:deploy) do
688
- # Shut up, Postgres!
689
- conn = ::ActiveRecord::Base.connection
690
- old_proc = conn.raw_connection.set_notice_processor {}
691
- begin
692
- drop_statement ||= "DROP SCHEMA #{self.name} CASCADE"
693
- Array(drop_statement).each do |stmt|
694
- ::ActiveRecord::Base.connection.execute(stmt)
695
- end
696
- ensure
697
- conn.raw_connection.set_notice_processor(&old_proc) if old_proc
643
+ end
644
+ when 'postgresql'
645
+ activate do
646
+ ::GuardRail.activate(:deploy) do
647
+ # Shut up, Postgres!
648
+ conn = ::ActiveRecord::Base.connection
649
+ old_proc = conn.raw_connection.set_notice_processor {}
650
+ begin
651
+ drop_statement ||= "DROP SCHEMA #{name} CASCADE"
652
+ Array(drop_statement).each do |stmt|
653
+ ::ActiveRecord::Base.connection.execute(stmt)
698
654
  end
655
+ ensure
656
+ conn.raw_connection.set_notice_processor(&old_proc) if old_proc
699
657
  end
700
658
  end
659
+ end
701
660
  end
702
661
  rescue
703
662
  logger.info "Drop failed: #{$!}"
@@ -707,8 +666,9 @@ module Switchman
707
666
  # takes an id local to this shard, and returns a global id
708
667
  def global_id_for(local_id)
709
668
  return nil unless local_id
669
+
710
670
  self.class.signed_id_operation(local_id) do |abs_id|
711
- abs_id + self.id * IDS_PER_SHARD
671
+ abs_id + id * IDS_PER_SHARD
712
672
  end
713
673
  end
714
674
 
@@ -718,7 +678,8 @@ module Switchman
718
678
  end
719
679
 
720
680
  def destroy
721
- raise("Cannot destroy the default shard") if self.default?
681
+ raise('Cannot destroy the default shard') if default?
682
+
722
683
  super
723
684
  end
724
685
 
@@ -727,31 +688,22 @@ module Switchman
727
688
  def clear_cache
728
689
  Shard.default.activate do
729
690
  Switchman.cache.delete(['shard', id].join('/'))
730
- Switchman.cache.delete("default_shard") if default?
691
+ Switchman.cache.delete('default_shard') if default?
731
692
  end
732
- self.class.clear_cache
733
693
  end
734
694
 
735
695
  def default_name
736
696
  database_server.shard_name(self)
737
697
  end
738
698
 
739
- def on_rollback
740
- # make sure all connection pool proxies are referencing valid pools
741
- ::ActiveRecord::Base.connection_handler.connection_pools.each do |pool|
742
- next unless pool.is_a?(ConnectionPoolProxy)
743
-
744
- pool.remove_shard!(self)
745
- end
746
- end
747
-
748
- def hashify_categories(categories)
749
- if categories.empty?
750
- { :primary => self }
699
+ def hashify_classes(classes)
700
+ if classes.empty?
701
+ { ::ActiveRecord::Base => self }
751
702
  else
752
- categories.inject({}) { |h, category| h[category] = self; h }
703
+ classes.each_with_object({}) do |klass, h|
704
+ h[klass] = self
705
+ end
753
706
  end
754
707
  end
755
-
756
708
  end
757
709
  end