switchman 2.0.13 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. checksums.yaml +4 -4
  2. data/Rakefile +10 -2
  3. data/app/models/switchman/shard.rb +234 -271
  4. data/app/models/switchman/unsharded_record.rb +7 -0
  5. data/db/migrate/20130328212039_create_switchman_shards.rb +1 -1
  6. data/db/migrate/20130328224244_create_default_shard.rb +5 -5
  7. data/db/migrate/20161206323434_add_back_default_string_limits_switchman.rb +1 -0
  8. data/db/migrate/20180828183945_add_default_shard_index.rb +2 -2
  9. data/db/migrate/20180828192111_add_timestamps_to_shards.rb +7 -5
  10. data/db/migrate/20190114212900_add_unique_name_indexes.rb +5 -3
  11. data/lib/switchman.rb +3 -5
  12. data/lib/switchman/action_controller/caching.rb +2 -2
  13. data/lib/switchman/active_record/abstract_adapter.rb +1 -0
  14. data/lib/switchman/active_record/association.rb +78 -89
  15. data/lib/switchman/active_record/attribute_methods.rb +58 -52
  16. data/lib/switchman/active_record/base.rb +58 -59
  17. data/lib/switchman/active_record/calculations.rb +74 -67
  18. data/lib/switchman/active_record/connection_pool.rb +14 -41
  19. data/lib/switchman/active_record/database_configurations.rb +34 -0
  20. data/lib/switchman/active_record/database_configurations/database_config.rb +13 -0
  21. data/lib/switchman/active_record/finder_methods.rb +11 -16
  22. data/lib/switchman/active_record/log_subscriber.rb +4 -8
  23. data/lib/switchman/active_record/migration.rb +6 -47
  24. data/lib/switchman/active_record/model_schema.rb +1 -1
  25. data/lib/switchman/active_record/persistence.rb +4 -6
  26. data/lib/switchman/active_record/postgresql_adapter.rb +124 -168
  27. data/lib/switchman/active_record/predicate_builder.rb +2 -2
  28. data/lib/switchman/active_record/query_cache.rb +18 -19
  29. data/lib/switchman/active_record/query_methods.rb +172 -197
  30. data/lib/switchman/active_record/reflection.rb +6 -10
  31. data/lib/switchman/active_record/relation.rb +30 -78
  32. data/lib/switchman/active_record/spawn_methods.rb +27 -29
  33. data/lib/switchman/active_record/statement_cache.rb +18 -35
  34. data/lib/switchman/active_record/tasks/database_tasks.rb +16 -0
  35. data/lib/switchman/active_support/cache.rb +3 -5
  36. data/lib/switchman/arel.rb +13 -8
  37. data/lib/switchman/database_server.rb +121 -142
  38. data/lib/switchman/default_shard.rb +52 -16
  39. data/lib/switchman/engine.rb +61 -58
  40. data/lib/switchman/environment.rb +4 -8
  41. data/lib/switchman/errors.rb +1 -0
  42. data/lib/switchman/guard_rail.rb +6 -19
  43. data/lib/switchman/guard_rail/relation.rb +5 -7
  44. data/lib/switchman/r_spec_helper.rb +29 -37
  45. data/lib/switchman/rails.rb +14 -12
  46. data/lib/switchman/schema_cache.rb +1 -9
  47. data/lib/switchman/sharded_instrumenter.rb +1 -1
  48. data/lib/switchman/standard_error.rb +15 -3
  49. data/lib/switchman/test_helper.rb +7 -11
  50. data/lib/switchman/version.rb +1 -1
  51. data/lib/tasks/switchman.rake +54 -69
  52. metadata +87 -45
  53. data/lib/switchman/active_record/batches.rb +0 -11
  54. data/lib/switchman/active_record/connection_handler.rb +0 -172
  55. data/lib/switchman/active_record/where_clause_factory.rb +0 -36
  56. data/lib/switchman/connection_pool_proxy.rb +0 -173
@@ -1,11 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Switchman
4
- module ActiveRecord
5
- module Batches
6
- def batch_order
7
- ::Arel.sql("#{connection.quote_local_table_name(table_name)}.#{quoted_primary_key} ASC")
8
- end
9
- end
10
- end
11
- end
@@ -1,172 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require 'switchman/connection_pool_proxy'
4
-
5
- module Switchman
6
- module ActiveRecord
7
- module ConnectionHandler
8
- def self.make_sharing_automagic(config, shard = Shard.current)
9
- # only load the shard name from the db if we have to
10
- if !config[:shard_name]
11
- # we may not be able to connect to this shard yet, cause it might be an empty database server
12
- shard = shard.call if shard.is_a?(Proc)
13
- shard_name = shard.name rescue nil
14
- return unless shard_name
15
-
16
- config[:shard_name] ||= shard_name
17
- end
18
- end
19
-
20
- def establish_connection(spec)
21
- # Just skip establishing a sharded connection if sharding isn't loaded; we'll do it again later
22
- # This only can happen when loading ActiveRecord::Base; after everything is loaded Shard will
23
- # be defined and this will actually establish a connection
24
- return unless defined?(Shard)
25
- pool = super
26
-
27
- # this is the first place that the adapter would have been required; but now we
28
- # need this addition ASAP since it will be called when loading the default shard below
29
- if defined?(::ActiveRecord::ConnectionAdapters::PostgreSQLAdapter)
30
- require "switchman/active_record/postgresql_adapter"
31
- ::ActiveRecord::ConnectionAdapters::PostgreSQLAdapter.prepend(ActiveRecord::PostgreSQLAdapter)
32
- end
33
-
34
- first_time = !Shard.instance_variable_get(:@default)
35
- if first_time
36
- # Have to cache the default shard before we insert sharding, otherwise the first access
37
- # to sharding will recurse onto itself trying to access column information
38
- Shard.default
39
-
40
- config = pool.spec.config
41
- # automatically change config to allow for sharing connections with simple config
42
- ConnectionHandler.make_sharing_automagic(config)
43
- ConnectionHandler.make_sharing_automagic(Shard.default.database_server.config)
44
-
45
- if ::Rails.version < '6.0'
46
- ::ActiveRecord::Base.configurations[::Rails.env] = config.stringify_keys
47
- else
48
- # Adopted from the deprecated code that currently lives in rails proper
49
- remaining_configs = ::ActiveRecord::Base.configurations.configurations.reject { |db_config| db_config.env_name == ::Rails.env }
50
- new_config = ::ActiveRecord::DatabaseConfigurations.new(::Rails.env => config.stringify_keys).configurations
51
- new_configs = remaining_configs + new_config
52
-
53
- ::ActiveRecord::Base.configurations = new_configs
54
- end
55
- else
56
- # this is probably wrong now
57
- Shard.default.remove_instance_variable(:@name) if Shard.default.instance_variable_defined?(:@name)
58
- end
59
-
60
- @shard_connection_pools ||= { [:primary, Shard.default.database_server.shareable? ? ::Rails.env : Shard.default] => pool}
61
-
62
- category = pool.spec.name.to_sym
63
- proxy = ConnectionPoolProxy.new(category,
64
- pool,
65
- @shard_connection_pools)
66
- owner_to_pool[pool.spec.name] = proxy
67
-
68
- if first_time
69
- if Shard.default.database_server.config[:prefer_secondary]
70
- Shard.default.database_server.guard!
71
- end
72
-
73
- if Shard.default.is_a?(DefaultShard) && Shard.default.database_server.config[:secondary]
74
- Shard.default.database_server.guard!
75
- Shard.default(reload: true)
76
- end
77
- end
78
-
79
- # reload the default shard if we just got a new connection
80
- # to where the Shards table is
81
- # DON'T do it if we're not the current connection handler - that means
82
- # we're in the middle of switching environments, and we don't want to
83
- # establish a connection with incorrect settings
84
- if [:primary, :unsharded].include?(category) && self == ::ActiveRecord::Base.connection_handler && !first_time
85
- Shard.default(reload: true, with_fallback: true)
86
- proxy.disconnect!
87
- end
88
-
89
- if first_time
90
- # do the change for other database servers, now that we can switch shards
91
- if Shard.default.is_a?(Shard)
92
- DatabaseServer.all.each do |server|
93
- next if server == Shard.default.database_server
94
-
95
- shard = nil
96
- shard_proc = -> do
97
- shard ||= server.shards.where(:name => nil).first
98
- shard ||= Shard.new(:database_server => server)
99
- shard
100
- end
101
- ConnectionHandler.make_sharing_automagic(server.config, shard_proc)
102
- ConnectionHandler.make_sharing_automagic(proxy.current_pool.spec.config, shard_proc)
103
- end
104
- end
105
- # we may have established some connections above trying to infer the shard's name.
106
- # close them, so that someone that doesn't expect them doesn't try to fork
107
- # without closing them
108
- self.clear_all_connections!
109
- end
110
-
111
- proxy
112
- end
113
-
114
- def remove_connection(spec_name)
115
- pool = owner_to_pool[spec_name]
116
- owner_to_pool[spec_name] = pool.default_pool if pool.is_a?(ConnectionPoolProxy)
117
- super
118
- end
119
-
120
- def retrieve_connection_pool(spec_name)
121
- owner_to_pool.fetch(spec_name) do
122
- if ancestor_pool = pool_from_any_process_for(spec_name)
123
- # A connection was established in an ancestor process that must have
124
- # subsequently forked. We can't reuse the connection, but we can copy
125
- # the specification and establish a new connection with it.
126
- spec = if ancestor_pool.is_a?(ConnectionPoolProxy)
127
- ancestor_pool.default_pool.spec
128
- else
129
- ancestor_pool.spec
130
- end
131
- pool = establish_connection(spec.to_hash)
132
- pool.instance_variable_set(:@schema_cache, ancestor_pool.schema_cache) if ancestor_pool.schema_cache
133
- pool
134
- elsif spec_name != "primary"
135
- primary_pool = retrieve_connection_pool("primary")
136
- if primary_pool.is_a?(ConnectionPoolProxy)
137
- pool = ConnectionPoolProxy.new(spec_name.to_sym, primary_pool.default_pool, @shard_connection_pools)
138
- pool.schema_cache.copy_references(primary_pool.schema_cache)
139
- pool
140
- else
141
- primary_pool
142
- end
143
- else
144
- owner_to_pool[spec_name] = nil
145
- end
146
- end
147
- end
148
-
149
- def clear_idle_connections!(since_when)
150
- connection_pool_list.each{ |pool| pool.clear_idle_connections!(since_when) }
151
- end
152
-
153
- def switchman_connection_pool_proxies
154
- owner_to_pool.values.uniq.select{|p| p.is_a?(ConnectionPoolProxy)}
155
- end
156
-
157
- private
158
-
159
- # semi-private
160
- public
161
- def uninitialize_ar(model = ::ActiveRecord::Base)
162
- # take the proxies out
163
- pool = owner_to_pool[model.name]
164
- owner_to_pool[model.name] = pool.default_pool if pool
165
- end
166
-
167
- def initialize_categories(model = ::ActiveRecord::Base)
168
- class_to_pool.clear
169
- end
170
- end
171
- end
172
- end
@@ -1,36 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Switchman
4
- module ActiveRecord
5
- module WhereClauseFactory
6
- attr_writer :scope
7
-
8
- def build(opts, other = [])
9
- case opts
10
- when String, Array
11
- values = Hash === other.first ? other.first.values : other
12
-
13
- values.grep(ActiveRecord::Relation) do |rel|
14
- # serialize subqueries against the same shard as the outer query is currently
15
- # targeted to run against
16
- if rel.shard_source_value == :implicit && rel.primary_shard != @scope.primary_shard
17
- rel.shard!(@scope.primary_shard)
18
- end
19
- end
20
-
21
- super
22
- when Hash, ::Arel::Nodes::Node
23
- where_clause = super
24
- binds = ::Rails.version >= "5.2" ? nil : where_clause.binds
25
- predicates = where_clause.send(:predicates)
26
- @scope.send(:infer_shards_from_primary_key, predicates, binds) if @scope.shard_source_value == :implicit && @scope.shard_value.is_a?(Shard)
27
- predicates, _new_binds = @scope.transpose_predicates(predicates, nil, @scope.primary_shard, false, binds: binds)
28
- where_clause.instance_variable_set(:@predicates, predicates)
29
- where_clause
30
- else
31
- super
32
- end
33
- end
34
- end
35
- end
36
- end
@@ -1,173 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require 'switchman/schema_cache'
4
-
5
- module Switchman
6
- module ConnectionError
7
- def self.===(other)
8
- return true if defined?(PG::Error) && PG::Error === other
9
- return true if defined?(Mysql2::Error) && Mysql2::Error === other
10
- false
11
- end
12
- end
13
-
14
- class ConnectionPoolProxy
15
- delegate :spec, :connected?, :default_schema, :with_connection, :query_cache_enabled, :active_connection?,
16
- :to => :current_pool
17
-
18
- attr_reader :category, :schema_cache
19
-
20
- def default_pool
21
- @default_pool
22
- end
23
-
24
- def initialize(category, default_pool, shard_connection_pools)
25
- @category = category
26
- @default_pool = default_pool
27
- @connection_pools = shard_connection_pools
28
- @schema_cache = default_pool.get_schema_cache(nil) if ::Rails.version >= '6'
29
- @schema_cache = SchemaCache.new(self) unless @schema_cache.is_a?(SchemaCache)
30
- if ::Rails.version >= '6'
31
- @default_pool.set_schema_cache(@schema_cache)
32
- @connection_pools.each_value do |pool|
33
- pool.set_schema_cache(@schema_cache)
34
- end
35
- end
36
- end
37
-
38
- def active_shard
39
- Shard.current(@category)
40
- end
41
-
42
- def active_guard_rail_environment
43
- ::Rails.env.test? ? :primary : active_shard.database_server.guard_rail_environment
44
- end
45
-
46
- def current_pool
47
- current_active_shard = active_shard
48
- pool = self.default_pool if current_active_shard.database_server == Shard.default.database_server && active_guard_rail_environment == :primary && (current_active_shard.default? || current_active_shard.database_server.shareable?)
49
- pool = @connection_pools[pool_key] ||= create_pool unless pool
50
- pool.shard = current_active_shard
51
- pool
52
- end
53
-
54
- def connections
55
- connection_pools.map(&:connections).inject([], &:+)
56
- end
57
-
58
- def connection(switch_shard: true)
59
- pool = current_pool
60
- begin
61
- connection = pool.connection(switch_shard: switch_shard)
62
- connection.instance_variable_set(:@schema_cache, @schema_cache) unless ::Rails.version >= '6'
63
- connection
64
- rescue ConnectionError
65
- raise if active_shard.database_server == Shard.default.database_server && active_guard_rail_environment == :primary
66
- configs = active_shard.database_server.config(active_guard_rail_environment)
67
- raise unless configs.is_a?(Array)
68
- configs.each_with_index do |config, idx|
69
- pool = create_pool(config.dup)
70
- begin
71
- connection = pool.connection
72
- connection.instance_variable_set(:@schema_cache, @schema_cache) unless ::Rails.version >= '6'
73
- rescue ConnectionError
74
- raise if idx == configs.length - 1
75
- next
76
- end
77
- @connection_pools[pool_key] = pool
78
- break connection
79
- end
80
- end
81
- end
82
-
83
- def get_schema_cache(_connection)
84
- @schema_cache
85
- end
86
-
87
- def set_schema_cache(cache)
88
- @schema_cache.copy_values(cache)
89
- end
90
-
91
- %w{release_connection
92
- disconnect!
93
- flush!
94
- clear_reloadable_connections!
95
- verify_active_connections!
96
- clear_stale_cached_connections!
97
- enable_query_cache!
98
- disable_query_cache! }.each do |method|
99
- class_eval(<<-RUBY, __FILE__, __LINE__ + 1)
100
- def #{method}
101
- connection_pools.each(&:#{method})
102
- end
103
- RUBY
104
- end
105
-
106
- def discard!
107
- # this breaks everything if i try to pass it onto the pools and i'm not sure why
108
- end
109
-
110
- def automatic_reconnect=(value)
111
- connection_pools.each { |pool| pool.automatic_reconnect = value }
112
- end
113
-
114
- def clear_idle_connections!(since_when)
115
- connection_pools.each { |pool| pool.clear_idle_connections!(since_when) }
116
- end
117
-
118
- def remove_shard!(shard)
119
- connection_pools.each { |pool| pool.remove_shard!(shard) }
120
- end
121
-
122
- protected
123
-
124
- def connection_pools
125
- (@connection_pools.values + [default_pool]).uniq
126
- end
127
-
128
- def pool_key
129
- [active_guard_rail_environment,
130
- active_shard.database_server.shareable? ? active_shard.database_server.pool_key : active_shard]
131
- end
132
-
133
- def create_pool(config = nil)
134
- shard = active_shard
135
- unless config
136
- if shard != Shard.default
137
- config = shard.database_server.config(active_guard_rail_environment)
138
- config = config.first if config.is_a?(Array)
139
- config = config.dup
140
- else
141
- # we read @config instead of calling config so that we get the config
142
- # *before* %{shard_name} is applied
143
- # also, we can't just read the database server's config, because
144
- # different models could be using different configs on the default
145
- # shard, and database server wouldn't know about that
146
- config = default_pool.spec.instance_variable_get(:@config)
147
- if config[active_guard_rail_environment].is_a?(Hash)
148
- config = config.merge(config[active_guard_rail_environment])
149
- elsif config[active_guard_rail_environment].is_a?(Array)
150
- config = config.merge(config[active_guard_rail_environment].first)
151
- else
152
- config = config.dup
153
- end
154
- end
155
- end
156
- spec = ::ActiveRecord::ConnectionAdapters::ConnectionSpecification.new(
157
- category,
158
- config,
159
- "#{config[:adapter]}_connection"
160
- )
161
- # unfortunately the AR code that does this require logic can't really be
162
- # called in isolation
163
- require "active_record/connection_adapters/#{config[:adapter]}_adapter"
164
-
165
- ::ActiveRecord::ConnectionAdapters::ConnectionPool.new(spec).tap do |pool|
166
- pool.shard = shard
167
- pool.set_schema_cache(@schema_cache) if ::Rails.version >= '6'
168
- pool.enable_query_cache! if !@connection_pools.empty? && @connection_pools.first.last.query_cache_enabled
169
- end
170
- end
171
- end
172
- end
173
-