switchman 2.0.13 → 3.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (56) hide show
  1. checksums.yaml +4 -4
  2. data/Rakefile +10 -2
  3. data/app/models/switchman/shard.rb +234 -271
  4. data/app/models/switchman/unsharded_record.rb +7 -0
  5. data/db/migrate/20130328212039_create_switchman_shards.rb +1 -1
  6. data/db/migrate/20130328224244_create_default_shard.rb +5 -5
  7. data/db/migrate/20161206323434_add_back_default_string_limits_switchman.rb +1 -0
  8. data/db/migrate/20180828183945_add_default_shard_index.rb +2 -2
  9. data/db/migrate/20180828192111_add_timestamps_to_shards.rb +7 -5
  10. data/db/migrate/20190114212900_add_unique_name_indexes.rb +5 -3
  11. data/lib/switchman.rb +3 -5
  12. data/lib/switchman/action_controller/caching.rb +2 -2
  13. data/lib/switchman/active_record/abstract_adapter.rb +1 -0
  14. data/lib/switchman/active_record/association.rb +78 -89
  15. data/lib/switchman/active_record/attribute_methods.rb +58 -52
  16. data/lib/switchman/active_record/base.rb +58 -59
  17. data/lib/switchman/active_record/calculations.rb +74 -67
  18. data/lib/switchman/active_record/connection_pool.rb +14 -41
  19. data/lib/switchman/active_record/database_configurations.rb +34 -0
  20. data/lib/switchman/active_record/database_configurations/database_config.rb +13 -0
  21. data/lib/switchman/active_record/finder_methods.rb +11 -16
  22. data/lib/switchman/active_record/log_subscriber.rb +4 -8
  23. data/lib/switchman/active_record/migration.rb +6 -47
  24. data/lib/switchman/active_record/model_schema.rb +1 -1
  25. data/lib/switchman/active_record/persistence.rb +4 -6
  26. data/lib/switchman/active_record/postgresql_adapter.rb +124 -168
  27. data/lib/switchman/active_record/predicate_builder.rb +2 -2
  28. data/lib/switchman/active_record/query_cache.rb +18 -19
  29. data/lib/switchman/active_record/query_methods.rb +172 -197
  30. data/lib/switchman/active_record/reflection.rb +6 -10
  31. data/lib/switchman/active_record/relation.rb +30 -78
  32. data/lib/switchman/active_record/spawn_methods.rb +27 -29
  33. data/lib/switchman/active_record/statement_cache.rb +18 -35
  34. data/lib/switchman/active_record/tasks/database_tasks.rb +16 -0
  35. data/lib/switchman/active_support/cache.rb +3 -5
  36. data/lib/switchman/arel.rb +13 -8
  37. data/lib/switchman/database_server.rb +121 -142
  38. data/lib/switchman/default_shard.rb +52 -16
  39. data/lib/switchman/engine.rb +61 -58
  40. data/lib/switchman/environment.rb +4 -8
  41. data/lib/switchman/errors.rb +1 -0
  42. data/lib/switchman/guard_rail.rb +6 -19
  43. data/lib/switchman/guard_rail/relation.rb +5 -7
  44. data/lib/switchman/r_spec_helper.rb +29 -37
  45. data/lib/switchman/rails.rb +14 -12
  46. data/lib/switchman/schema_cache.rb +1 -9
  47. data/lib/switchman/sharded_instrumenter.rb +1 -1
  48. data/lib/switchman/standard_error.rb +15 -3
  49. data/lib/switchman/test_helper.rb +7 -11
  50. data/lib/switchman/version.rb +1 -1
  51. data/lib/tasks/switchman.rake +54 -69
  52. metadata +87 -45
  53. data/lib/switchman/active_record/batches.rb +0 -11
  54. data/lib/switchman/active_record/connection_handler.rb +0 -172
  55. data/lib/switchman/active_record/where_clause_factory.rb +0 -36
  56. data/lib/switchman/connection_pool_proxy.rb +0 -173
@@ -1,11 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Switchman
4
- module ActiveRecord
5
- module Batches
6
- def batch_order
7
- ::Arel.sql("#{connection.quote_local_table_name(table_name)}.#{quoted_primary_key} ASC")
8
- end
9
- end
10
- end
11
- end
@@ -1,172 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require 'switchman/connection_pool_proxy'
4
-
5
- module Switchman
6
- module ActiveRecord
7
- module ConnectionHandler
8
- def self.make_sharing_automagic(config, shard = Shard.current)
9
- # only load the shard name from the db if we have to
10
- if !config[:shard_name]
11
- # we may not be able to connect to this shard yet, cause it might be an empty database server
12
- shard = shard.call if shard.is_a?(Proc)
13
- shard_name = shard.name rescue nil
14
- return unless shard_name
15
-
16
- config[:shard_name] ||= shard_name
17
- end
18
- end
19
-
20
- def establish_connection(spec)
21
- # Just skip establishing a sharded connection if sharding isn't loaded; we'll do it again later
22
- # This only can happen when loading ActiveRecord::Base; after everything is loaded Shard will
23
- # be defined and this will actually establish a connection
24
- return unless defined?(Shard)
25
- pool = super
26
-
27
- # this is the first place that the adapter would have been required; but now we
28
- # need this addition ASAP since it will be called when loading the default shard below
29
- if defined?(::ActiveRecord::ConnectionAdapters::PostgreSQLAdapter)
30
- require "switchman/active_record/postgresql_adapter"
31
- ::ActiveRecord::ConnectionAdapters::PostgreSQLAdapter.prepend(ActiveRecord::PostgreSQLAdapter)
32
- end
33
-
34
- first_time = !Shard.instance_variable_get(:@default)
35
- if first_time
36
- # Have to cache the default shard before we insert sharding, otherwise the first access
37
- # to sharding will recurse onto itself trying to access column information
38
- Shard.default
39
-
40
- config = pool.spec.config
41
- # automatically change config to allow for sharing connections with simple config
42
- ConnectionHandler.make_sharing_automagic(config)
43
- ConnectionHandler.make_sharing_automagic(Shard.default.database_server.config)
44
-
45
- if ::Rails.version < '6.0'
46
- ::ActiveRecord::Base.configurations[::Rails.env] = config.stringify_keys
47
- else
48
- # Adopted from the deprecated code that currently lives in rails proper
49
- remaining_configs = ::ActiveRecord::Base.configurations.configurations.reject { |db_config| db_config.env_name == ::Rails.env }
50
- new_config = ::ActiveRecord::DatabaseConfigurations.new(::Rails.env => config.stringify_keys).configurations
51
- new_configs = remaining_configs + new_config
52
-
53
- ::ActiveRecord::Base.configurations = new_configs
54
- end
55
- else
56
- # this is probably wrong now
57
- Shard.default.remove_instance_variable(:@name) if Shard.default.instance_variable_defined?(:@name)
58
- end
59
-
60
- @shard_connection_pools ||= { [:primary, Shard.default.database_server.shareable? ? ::Rails.env : Shard.default] => pool}
61
-
62
- category = pool.spec.name.to_sym
63
- proxy = ConnectionPoolProxy.new(category,
64
- pool,
65
- @shard_connection_pools)
66
- owner_to_pool[pool.spec.name] = proxy
67
-
68
- if first_time
69
- if Shard.default.database_server.config[:prefer_secondary]
70
- Shard.default.database_server.guard!
71
- end
72
-
73
- if Shard.default.is_a?(DefaultShard) && Shard.default.database_server.config[:secondary]
74
- Shard.default.database_server.guard!
75
- Shard.default(reload: true)
76
- end
77
- end
78
-
79
- # reload the default shard if we just got a new connection
80
- # to where the Shards table is
81
- # DON'T do it if we're not the current connection handler - that means
82
- # we're in the middle of switching environments, and we don't want to
83
- # establish a connection with incorrect settings
84
- if [:primary, :unsharded].include?(category) && self == ::ActiveRecord::Base.connection_handler && !first_time
85
- Shard.default(reload: true, with_fallback: true)
86
- proxy.disconnect!
87
- end
88
-
89
- if first_time
90
- # do the change for other database servers, now that we can switch shards
91
- if Shard.default.is_a?(Shard)
92
- DatabaseServer.all.each do |server|
93
- next if server == Shard.default.database_server
94
-
95
- shard = nil
96
- shard_proc = -> do
97
- shard ||= server.shards.where(:name => nil).first
98
- shard ||= Shard.new(:database_server => server)
99
- shard
100
- end
101
- ConnectionHandler.make_sharing_automagic(server.config, shard_proc)
102
- ConnectionHandler.make_sharing_automagic(proxy.current_pool.spec.config, shard_proc)
103
- end
104
- end
105
- # we may have established some connections above trying to infer the shard's name.
106
- # close them, so that someone that doesn't expect them doesn't try to fork
107
- # without closing them
108
- self.clear_all_connections!
109
- end
110
-
111
- proxy
112
- end
113
-
114
- def remove_connection(spec_name)
115
- pool = owner_to_pool[spec_name]
116
- owner_to_pool[spec_name] = pool.default_pool if pool.is_a?(ConnectionPoolProxy)
117
- super
118
- end
119
-
120
- def retrieve_connection_pool(spec_name)
121
- owner_to_pool.fetch(spec_name) do
122
- if ancestor_pool = pool_from_any_process_for(spec_name)
123
- # A connection was established in an ancestor process that must have
124
- # subsequently forked. We can't reuse the connection, but we can copy
125
- # the specification and establish a new connection with it.
126
- spec = if ancestor_pool.is_a?(ConnectionPoolProxy)
127
- ancestor_pool.default_pool.spec
128
- else
129
- ancestor_pool.spec
130
- end
131
- pool = establish_connection(spec.to_hash)
132
- pool.instance_variable_set(:@schema_cache, ancestor_pool.schema_cache) if ancestor_pool.schema_cache
133
- pool
134
- elsif spec_name != "primary"
135
- primary_pool = retrieve_connection_pool("primary")
136
- if primary_pool.is_a?(ConnectionPoolProxy)
137
- pool = ConnectionPoolProxy.new(spec_name.to_sym, primary_pool.default_pool, @shard_connection_pools)
138
- pool.schema_cache.copy_references(primary_pool.schema_cache)
139
- pool
140
- else
141
- primary_pool
142
- end
143
- else
144
- owner_to_pool[spec_name] = nil
145
- end
146
- end
147
- end
148
-
149
- def clear_idle_connections!(since_when)
150
- connection_pool_list.each{ |pool| pool.clear_idle_connections!(since_when) }
151
- end
152
-
153
- def switchman_connection_pool_proxies
154
- owner_to_pool.values.uniq.select{|p| p.is_a?(ConnectionPoolProxy)}
155
- end
156
-
157
- private
158
-
159
- # semi-private
160
- public
161
- def uninitialize_ar(model = ::ActiveRecord::Base)
162
- # take the proxies out
163
- pool = owner_to_pool[model.name]
164
- owner_to_pool[model.name] = pool.default_pool if pool
165
- end
166
-
167
- def initialize_categories(model = ::ActiveRecord::Base)
168
- class_to_pool.clear
169
- end
170
- end
171
- end
172
- end
@@ -1,36 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Switchman
4
- module ActiveRecord
5
- module WhereClauseFactory
6
- attr_writer :scope
7
-
8
- def build(opts, other = [])
9
- case opts
10
- when String, Array
11
- values = Hash === other.first ? other.first.values : other
12
-
13
- values.grep(ActiveRecord::Relation) do |rel|
14
- # serialize subqueries against the same shard as the outer query is currently
15
- # targeted to run against
16
- if rel.shard_source_value == :implicit && rel.primary_shard != @scope.primary_shard
17
- rel.shard!(@scope.primary_shard)
18
- end
19
- end
20
-
21
- super
22
- when Hash, ::Arel::Nodes::Node
23
- where_clause = super
24
- binds = ::Rails.version >= "5.2" ? nil : where_clause.binds
25
- predicates = where_clause.send(:predicates)
26
- @scope.send(:infer_shards_from_primary_key, predicates, binds) if @scope.shard_source_value == :implicit && @scope.shard_value.is_a?(Shard)
27
- predicates, _new_binds = @scope.transpose_predicates(predicates, nil, @scope.primary_shard, false, binds: binds)
28
- where_clause.instance_variable_set(:@predicates, predicates)
29
- where_clause
30
- else
31
- super
32
- end
33
- end
34
- end
35
- end
36
- end
@@ -1,173 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require 'switchman/schema_cache'
4
-
5
- module Switchman
6
- module ConnectionError
7
- def self.===(other)
8
- return true if defined?(PG::Error) && PG::Error === other
9
- return true if defined?(Mysql2::Error) && Mysql2::Error === other
10
- false
11
- end
12
- end
13
-
14
- class ConnectionPoolProxy
15
- delegate :spec, :connected?, :default_schema, :with_connection, :query_cache_enabled, :active_connection?,
16
- :to => :current_pool
17
-
18
- attr_reader :category, :schema_cache
19
-
20
- def default_pool
21
- @default_pool
22
- end
23
-
24
- def initialize(category, default_pool, shard_connection_pools)
25
- @category = category
26
- @default_pool = default_pool
27
- @connection_pools = shard_connection_pools
28
- @schema_cache = default_pool.get_schema_cache(nil) if ::Rails.version >= '6'
29
- @schema_cache = SchemaCache.new(self) unless @schema_cache.is_a?(SchemaCache)
30
- if ::Rails.version >= '6'
31
- @default_pool.set_schema_cache(@schema_cache)
32
- @connection_pools.each_value do |pool|
33
- pool.set_schema_cache(@schema_cache)
34
- end
35
- end
36
- end
37
-
38
- def active_shard
39
- Shard.current(@category)
40
- end
41
-
42
- def active_guard_rail_environment
43
- ::Rails.env.test? ? :primary : active_shard.database_server.guard_rail_environment
44
- end
45
-
46
- def current_pool
47
- current_active_shard = active_shard
48
- pool = self.default_pool if current_active_shard.database_server == Shard.default.database_server && active_guard_rail_environment == :primary && (current_active_shard.default? || current_active_shard.database_server.shareable?)
49
- pool = @connection_pools[pool_key] ||= create_pool unless pool
50
- pool.shard = current_active_shard
51
- pool
52
- end
53
-
54
- def connections
55
- connection_pools.map(&:connections).inject([], &:+)
56
- end
57
-
58
- def connection(switch_shard: true)
59
- pool = current_pool
60
- begin
61
- connection = pool.connection(switch_shard: switch_shard)
62
- connection.instance_variable_set(:@schema_cache, @schema_cache) unless ::Rails.version >= '6'
63
- connection
64
- rescue ConnectionError
65
- raise if active_shard.database_server == Shard.default.database_server && active_guard_rail_environment == :primary
66
- configs = active_shard.database_server.config(active_guard_rail_environment)
67
- raise unless configs.is_a?(Array)
68
- configs.each_with_index do |config, idx|
69
- pool = create_pool(config.dup)
70
- begin
71
- connection = pool.connection
72
- connection.instance_variable_set(:@schema_cache, @schema_cache) unless ::Rails.version >= '6'
73
- rescue ConnectionError
74
- raise if idx == configs.length - 1
75
- next
76
- end
77
- @connection_pools[pool_key] = pool
78
- break connection
79
- end
80
- end
81
- end
82
-
83
- def get_schema_cache(_connection)
84
- @schema_cache
85
- end
86
-
87
- def set_schema_cache(cache)
88
- @schema_cache.copy_values(cache)
89
- end
90
-
91
- %w{release_connection
92
- disconnect!
93
- flush!
94
- clear_reloadable_connections!
95
- verify_active_connections!
96
- clear_stale_cached_connections!
97
- enable_query_cache!
98
- disable_query_cache! }.each do |method|
99
- class_eval(<<-RUBY, __FILE__, __LINE__ + 1)
100
- def #{method}
101
- connection_pools.each(&:#{method})
102
- end
103
- RUBY
104
- end
105
-
106
- def discard!
107
- # this breaks everything if i try to pass it onto the pools and i'm not sure why
108
- end
109
-
110
- def automatic_reconnect=(value)
111
- connection_pools.each { |pool| pool.automatic_reconnect = value }
112
- end
113
-
114
- def clear_idle_connections!(since_when)
115
- connection_pools.each { |pool| pool.clear_idle_connections!(since_when) }
116
- end
117
-
118
- def remove_shard!(shard)
119
- connection_pools.each { |pool| pool.remove_shard!(shard) }
120
- end
121
-
122
- protected
123
-
124
- def connection_pools
125
- (@connection_pools.values + [default_pool]).uniq
126
- end
127
-
128
- def pool_key
129
- [active_guard_rail_environment,
130
- active_shard.database_server.shareable? ? active_shard.database_server.pool_key : active_shard]
131
- end
132
-
133
- def create_pool(config = nil)
134
- shard = active_shard
135
- unless config
136
- if shard != Shard.default
137
- config = shard.database_server.config(active_guard_rail_environment)
138
- config = config.first if config.is_a?(Array)
139
- config = config.dup
140
- else
141
- # we read @config instead of calling config so that we get the config
142
- # *before* %{shard_name} is applied
143
- # also, we can't just read the database server's config, because
144
- # different models could be using different configs on the default
145
- # shard, and database server wouldn't know about that
146
- config = default_pool.spec.instance_variable_get(:@config)
147
- if config[active_guard_rail_environment].is_a?(Hash)
148
- config = config.merge(config[active_guard_rail_environment])
149
- elsif config[active_guard_rail_environment].is_a?(Array)
150
- config = config.merge(config[active_guard_rail_environment].first)
151
- else
152
- config = config.dup
153
- end
154
- end
155
- end
156
- spec = ::ActiveRecord::ConnectionAdapters::ConnectionSpecification.new(
157
- category,
158
- config,
159
- "#{config[:adapter]}_connection"
160
- )
161
- # unfortunately the AR code that does this require logic can't really be
162
- # called in isolation
163
- require "active_record/connection_adapters/#{config[:adapter]}_adapter"
164
-
165
- ::ActiveRecord::ConnectionAdapters::ConnectionPool.new(spec).tap do |pool|
166
- pool.shard = shard
167
- pool.set_schema_cache(@schema_cache) if ::Rails.version >= '6'
168
- pool.enable_query_cache! if !@connection_pools.empty? && @connection_pools.first.last.query_cache_enabled
169
- end
170
- end
171
- end
172
- end
173
-