active_record_shards 3.17.0 → 3.18.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +13 -13
- data/lib/active_record_shards.rb +48 -4
- data/lib/active_record_shards/association_collection_connection_selection.rb +21 -13
- data/lib/active_record_shards/configuration_parser.rb +12 -4
- data/lib/active_record_shards/connection_switcher-4-2.rb +2 -2
- data/lib/active_record_shards/connection_switcher.rb +46 -34
- data/lib/active_record_shards/default_replica_patches.rb +171 -0
- data/lib/active_record_shards/default_slave_patches.rb +2 -141
- data/lib/active_record_shards/deprecation.rb +12 -0
- data/lib/active_record_shards/model.rb +15 -11
- data/lib/active_record_shards/shard_selection.rb +15 -13
- data/lib/active_record_shards/sql_comments.rb +4 -4
- data/lib/active_record_shards/tasks.rb +2 -2
- metadata +9 -7
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 1f573041b49f55115d94db028525da537bbbadffc4526d8a4f220f3d4ab1d4fd
|
4
|
+
data.tar.gz: f29528996589d7cef3f14671df47d10a8d4cf4a3b0ad81b022aaddf7b229178d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 782a638a2d4ce88a06ac0f6d93d277de20831d918ee1014fec1b2a01f95631b5857fb2df1d83b15ced1e52b1749283ec497c43ccfc0d85ffd6e3bd271d4d9f28
|
7
|
+
data.tar.gz: f28464784b171c3dd8a888574f1bb362e645f3949dcf3b07a70a9c195e73ae00a1c5a7319c107666f4060e37c04c3daacd8415187124dbda891aae5cb19a7ecf
|
data/README.md
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
|
3
3
|
# ActiveRecord Shards
|
4
4
|
|
5
|
-
ActiveRecord Shards is an extension for ActiveRecord that provides support for sharded database and
|
5
|
+
ActiveRecord Shards is an extension for ActiveRecord that provides support for sharded database and replicas. Basically it is just a nice way to
|
6
6
|
switch between database connections. We've made the implementation very small, and have tried not to reinvent any wheels already present in ActiveRecord.
|
7
7
|
|
8
8
|
ActiveRecord Shards has been used and tested on Rails 4.2, 5.x and 6.0, and has in some form or another been used in production on large Rails apps for several years.
|
@@ -24,7 +24,7 @@ and make sure to require 'active\_record\_shards' in some way.
|
|
24
24
|
|
25
25
|
## Configuration
|
26
26
|
|
27
|
-
Add the
|
27
|
+
Add the replica and shard configuration to config/database.yml:
|
28
28
|
|
29
29
|
```yaml
|
30
30
|
production:
|
@@ -35,19 +35,19 @@ production:
|
|
35
35
|
host: db1
|
36
36
|
username: root
|
37
37
|
password:
|
38
|
-
|
39
|
-
host:
|
38
|
+
replica:
|
39
|
+
host: db1_replica
|
40
40
|
shards:
|
41
41
|
1:
|
42
42
|
host: db_shard1
|
43
43
|
database: my_app_shard
|
44
|
-
|
45
|
-
host:
|
44
|
+
replica:
|
45
|
+
host: db_shard1_replica
|
46
46
|
2:
|
47
47
|
host: db_shard2
|
48
48
|
database: my_app_shard
|
49
|
-
|
50
|
-
host:
|
49
|
+
replica:
|
50
|
+
host: db_shard2_replica
|
51
51
|
```
|
52
52
|
|
53
53
|
basically connections inherit configuration from the parent configuration file.
|
@@ -140,23 +140,23 @@ class AccountMiddleware
|
|
140
140
|
end
|
141
141
|
```
|
142
142
|
|
143
|
-
You can switch to the
|
143
|
+
You can switch to the replica databases at any point by wrapping your code in an on\_replica block:
|
144
144
|
|
145
145
|
```ruby
|
146
|
-
ActiveRecord::Base.
|
146
|
+
ActiveRecord::Base.on_replica do
|
147
147
|
Account.find_by_big_expensive_query
|
148
148
|
end
|
149
149
|
```
|
150
150
|
|
151
|
-
This will perform the query on the
|
151
|
+
This will perform the query on the replica, and mark the returned instances as read only. There is also a shortcut for this:
|
152
152
|
|
153
153
|
```ruby
|
154
|
-
Account.
|
154
|
+
Account.on_replica.find_by_big_expensive_query
|
155
155
|
```
|
156
156
|
|
157
157
|
## Debugging
|
158
158
|
|
159
|
-
Show if a query went to
|
159
|
+
Show if a query went to primary or replica in the logs:
|
160
160
|
|
161
161
|
```Ruby
|
162
162
|
require 'active_record_shards/sql_comments'
|
data/lib/active_record_shards.rb
CHANGED
@@ -8,7 +8,7 @@ require 'active_record_shards/shard_selection'
|
|
8
8
|
require 'active_record_shards/connection_switcher'
|
9
9
|
require 'active_record_shards/association_collection_connection_selection'
|
10
10
|
require 'active_record_shards/migration'
|
11
|
-
require 'active_record_shards/
|
11
|
+
require 'active_record_shards/default_replica_patches'
|
12
12
|
require 'active_record_shards/schema_dumper_extension'
|
13
13
|
|
14
14
|
module ActiveRecordShards
|
@@ -23,10 +23,10 @@ end
|
|
23
23
|
ActiveRecord::Base.extend(ActiveRecordShards::ConfigurationParser)
|
24
24
|
ActiveRecord::Base.extend(ActiveRecordShards::Model)
|
25
25
|
ActiveRecord::Base.extend(ActiveRecordShards::ConnectionSwitcher)
|
26
|
-
ActiveRecord::Base.extend(ActiveRecordShards::
|
27
|
-
ActiveRecord::Relation.include(ActiveRecordShards::
|
26
|
+
ActiveRecord::Base.extend(ActiveRecordShards::DefaultReplicaPatches)
|
27
|
+
ActiveRecord::Relation.include(ActiveRecordShards::DefaultReplicaPatches::ActiveRelationPatches)
|
28
28
|
ActiveRecord::Associations::CollectionProxy.include(ActiveRecordShards::AssociationCollectionConnectionSelection)
|
29
|
-
ActiveRecord::Associations::Builder::HasAndBelongsToMany.include(ActiveRecordShards::
|
29
|
+
ActiveRecord::Associations::Builder::HasAndBelongsToMany.include(ActiveRecordShards::DefaultReplicaPatches::Rails41HasAndBelongsToManyBuilderExtension)
|
30
30
|
ActiveRecord::SchemaDumper.prepend(ActiveRecordShards::SchemaDumperExtension)
|
31
31
|
|
32
32
|
case "#{ActiveRecord::VERSION::MAJOR}.#{ActiveRecord::VERSION::MINOR}"
|
@@ -37,3 +37,47 @@ when '5.0', '5.1', '5.2', '6.0'
|
|
37
37
|
else
|
38
38
|
raise "ActiveRecordShards is not compatible with #{ActiveRecord::VERSION::STRING}"
|
39
39
|
end
|
40
|
+
|
41
|
+
require 'active_record_shards/deprecation'
|
42
|
+
|
43
|
+
ActiveRecordShards::Deprecation.deprecate_methods(
|
44
|
+
ActiveRecordShards::AssociationCollectionConnectionSelection,
|
45
|
+
on_slave_if: :on_replica_if,
|
46
|
+
on_slave_unless: :on_replica_unless,
|
47
|
+
on_slave: :on_replica,
|
48
|
+
on_master: :on_primary,
|
49
|
+
on_master_if: :on_primary_if,
|
50
|
+
on_master_unless: :on_primary_unless
|
51
|
+
)
|
52
|
+
|
53
|
+
ActiveRecordShards::Deprecation.deprecate_methods(
|
54
|
+
ActiveRecordShards::ConnectionSwitcher,
|
55
|
+
on_slave_if: :on_replica_if,
|
56
|
+
on_slave_unless: :on_replica_unless,
|
57
|
+
on_master_or_slave: :on_primary_or_replica,
|
58
|
+
on_slave: :on_replica,
|
59
|
+
on_master: :on_primary,
|
60
|
+
on_master_if: :on_primary_if,
|
61
|
+
on_master_unless: :on_primary_unless,
|
62
|
+
on_slave?: :on_replica?
|
63
|
+
)
|
64
|
+
|
65
|
+
ActiveRecordShards::Deprecation.deprecate_methods(
|
66
|
+
ActiveRecordShards::DefaultReplicaPatches,
|
67
|
+
columns_with_force_slave: :columns_with_force_replica,
|
68
|
+
table_exists_with_force_slave?: :table_exists_with_force_replica?,
|
69
|
+
transaction_with_slave_off: :transaction_with_replica_off,
|
70
|
+
on_slave_unless_tx: :on_replica_unless_tx
|
71
|
+
)
|
72
|
+
|
73
|
+
ActiveRecordShards::Deprecation.deprecate_methods(
|
74
|
+
ActiveRecordShards::Model,
|
75
|
+
on_slave_by_default?: :on_replica_by_default?,
|
76
|
+
:on_slave_by_default= => :on_replica_by_default=
|
77
|
+
)
|
78
|
+
|
79
|
+
ActiveRecordShards::Deprecation.deprecate_methods(
|
80
|
+
ActiveRecordShards::ShardSelection,
|
81
|
+
on_slave?: :on_replica?,
|
82
|
+
:on_slave= => :on_replica=
|
83
|
+
)
|
@@ -2,31 +2,37 @@
|
|
2
2
|
|
3
3
|
module ActiveRecordShards
|
4
4
|
module AssociationCollectionConnectionSelection
|
5
|
-
def
|
6
|
-
condition ?
|
5
|
+
def on_replica_if(condition)
|
6
|
+
condition ? on_replica : self
|
7
7
|
end
|
8
|
+
alias_method :on_slave_if, :on_replica_if
|
8
9
|
|
9
|
-
def
|
10
|
-
|
10
|
+
def on_replica_unless(condition)
|
11
|
+
on_replica_if(!condition)
|
11
12
|
end
|
13
|
+
alias_method :on_slave_unless, :on_replica_unless
|
12
14
|
|
13
|
-
def
|
14
|
-
condition ?
|
15
|
+
def on_primary_if(condition)
|
16
|
+
condition ? on_primary : self
|
15
17
|
end
|
18
|
+
alias_method :on_master_if, :on_primary_if
|
16
19
|
|
17
|
-
def
|
18
|
-
|
20
|
+
def on_primary_unless(condition)
|
21
|
+
on_primary_if(!condition)
|
19
22
|
end
|
23
|
+
alias_method :on_master_unless, :on_primary_unless
|
20
24
|
|
21
|
-
def
|
22
|
-
|
25
|
+
def on_replica
|
26
|
+
PrimaryReplicaProxy.new(self, :replica)
|
23
27
|
end
|
28
|
+
alias_method :on_slave, :on_replica
|
24
29
|
|
25
|
-
def
|
26
|
-
|
30
|
+
def on_primary
|
31
|
+
PrimaryReplicaProxy.new(self, :primary)
|
27
32
|
end
|
33
|
+
alias_method :on_master, :on_primary
|
28
34
|
|
29
|
-
class
|
35
|
+
class PrimaryReplicaProxy
|
30
36
|
def initialize(association_collection, which)
|
31
37
|
@association_collection = association_collection
|
32
38
|
@which = which
|
@@ -37,5 +43,7 @@ module ActiveRecordShards
|
|
37
43
|
reflection.klass.on_cx_switch_block(@which) { @association_collection.send(method, *args, &block) }
|
38
44
|
end
|
39
45
|
end
|
46
|
+
|
47
|
+
MasterSlaveProxy = PrimaryReplicaProxy
|
40
48
|
end
|
41
49
|
end
|
@@ -24,10 +24,18 @@ module ActiveRecordShards
|
|
24
24
|
end
|
25
25
|
|
26
26
|
conf.to_a.each do |env_name, env_config|
|
27
|
-
if
|
28
|
-
expand_child!(env_config,
|
29
|
-
conf["#{env_name}
|
27
|
+
if replica_conf = env_config.delete('replica')
|
28
|
+
expand_child!(env_config, replica_conf)
|
29
|
+
conf["#{env_name}_replica"] = replica_conf
|
30
30
|
end
|
31
|
+
|
32
|
+
# rubocop:disable Style/Next
|
33
|
+
if legacy_replica_conf = env_config.delete('slave')
|
34
|
+
ActiveRecordShards::Deprecation.warn('`slave` configuration keys should be replaced with `replica` keys!')
|
35
|
+
expand_child!(env_config, legacy_replica_conf)
|
36
|
+
conf["#{env_name}_replica"] = legacy_replica_conf
|
37
|
+
end
|
38
|
+
# rubocop:enable Style/Next
|
31
39
|
end
|
32
40
|
|
33
41
|
conf
|
@@ -35,7 +43,7 @@ module ActiveRecordShards
|
|
35
43
|
|
36
44
|
def expand_child!(parent, child)
|
37
45
|
parent.each do |key, value|
|
38
|
-
unless ['slave', 'shards'].include?(key) || value.is_a?(Hash)
|
46
|
+
unless ['slave', 'replica', 'shards'].include?(key) || value.is_a?(Hash)
|
39
47
|
child[key] ||= value
|
40
48
|
end
|
41
49
|
end
|
@@ -4,8 +4,8 @@ module ActiveRecordShards
|
|
4
4
|
def connection_pool_name # :nodoc:
|
5
5
|
name = current_shard_selection.shard_name(self)
|
6
6
|
|
7
|
-
# e.g. if "
|
8
|
-
if configurations[name].nil? &&
|
7
|
+
# e.g. if "production_replica" is not defined in `Configuration`, fall back to "production"
|
8
|
+
if configurations[name].nil? && on_replica?
|
9
9
|
current_shard_selection.shard_name(self, false)
|
10
10
|
else
|
11
11
|
name
|
@@ -59,70 +59,79 @@ module ActiveRecordShards
|
|
59
59
|
switch_connection(old_options)
|
60
60
|
end
|
61
61
|
|
62
|
-
def
|
63
|
-
condition ?
|
62
|
+
def on_replica_if(condition, &block)
|
63
|
+
condition ? on_replica(&block) : yield
|
64
64
|
end
|
65
|
+
alias_method :on_slave_if, :on_replica_if
|
65
66
|
|
66
|
-
def
|
67
|
-
|
67
|
+
def on_replica_unless(condition, &block)
|
68
|
+
on_replica_if(!condition, &block)
|
68
69
|
end
|
70
|
+
alias_method :on_slave_unless, :on_replica_unless
|
69
71
|
|
70
|
-
def
|
71
|
-
condition ?
|
72
|
+
def on_primary_if(condition, &block)
|
73
|
+
condition ? on_primary(&block) : yield
|
72
74
|
end
|
75
|
+
alias_method :on_master_if, :on_primary_if
|
73
76
|
|
74
|
-
def
|
75
|
-
|
77
|
+
def on_primary_unless(condition, &block)
|
78
|
+
on_primary_if(!condition, &block)
|
76
79
|
end
|
80
|
+
alias_method :on_master_unless, :on_primary_unless
|
77
81
|
|
78
|
-
def
|
82
|
+
def on_primary_or_replica(which, &block)
|
79
83
|
if block_given?
|
80
84
|
on_cx_switch_block(which, &block)
|
81
85
|
else
|
82
|
-
|
86
|
+
PrimaryReplicaProxy.new(self, which)
|
83
87
|
end
|
84
88
|
end
|
89
|
+
alias_method :on_master_or_slave, :on_primary_or_replica
|
85
90
|
|
86
|
-
# Executes queries using the
|
87
|
-
# if you want to execute a block of code on the
|
88
|
-
# Account.
|
91
|
+
# Executes queries using the replica database. Fails over to primary if no replica is found.
|
92
|
+
# if you want to execute a block of code on the replica you can go:
|
93
|
+
# Account.on_replica do
|
89
94
|
# Account.first
|
90
95
|
# end
|
91
|
-
# the first account will be found on the
|
96
|
+
# the first account will be found on the replica DB
|
92
97
|
#
|
93
98
|
# For one-liners you can simply do
|
94
|
-
# Account.
|
95
|
-
def
|
96
|
-
|
99
|
+
# Account.on_replica.first
|
100
|
+
def on_replica(&block)
|
101
|
+
on_primary_or_replica(:replica, &block)
|
97
102
|
end
|
103
|
+
alias_method :on_slave, :on_replica
|
98
104
|
|
99
|
-
def
|
100
|
-
|
105
|
+
def on_primary(&block)
|
106
|
+
on_primary_or_replica(:primary, &block)
|
101
107
|
end
|
108
|
+
alias_method :on_master, :on_primary
|
102
109
|
|
103
110
|
# just to ease the transition from replica to active_record_shards
|
104
|
-
alias_method :with_slave, :
|
105
|
-
alias_method :with_slave_if, :
|
106
|
-
alias_method :with_slave_unless, :
|
111
|
+
alias_method :with_slave, :on_replica
|
112
|
+
alias_method :with_slave_if, :on_replica_if
|
113
|
+
alias_method :with_slave_unless, :on_replica_unless
|
107
114
|
|
108
115
|
def on_cx_switch_block(which, force: false, construct_ro_scope: nil, &block)
|
109
|
-
@
|
110
|
-
@
|
116
|
+
@disallow_replica ||= 0
|
117
|
+
@disallow_replica += 1 if [:primary, :master].include?(which)
|
111
118
|
|
112
|
-
|
119
|
+
ActiveRecordShards::Deprecation.warn('the `:master` option should be replaced with `:primary`!') if which == :master
|
120
|
+
|
121
|
+
switch_to_replica = force || @disallow_replica.zero?
|
113
122
|
old_options = current_shard_selection.options
|
114
123
|
|
115
|
-
switch_connection(
|
124
|
+
switch_connection(replica: switch_to_replica)
|
116
125
|
|
117
126
|
# we avoid_readonly_scope to prevent some stack overflow problems, like when
|
118
127
|
# .columns calls .with_scope which calls .columns and onward, endlessly.
|
119
|
-
if self == ActiveRecord::Base || !
|
128
|
+
if self == ActiveRecord::Base || !switch_to_replica || construct_ro_scope == false
|
120
129
|
yield
|
121
130
|
else
|
122
131
|
readonly.scoping(&block)
|
123
132
|
end
|
124
133
|
ensure
|
125
|
-
@
|
134
|
+
@disallow_replica -= 1 if [:primary, :master].include?(which)
|
126
135
|
switch_connection(old_options) if old_options
|
127
136
|
end
|
128
137
|
|
@@ -130,9 +139,10 @@ module ActiveRecordShards
|
|
130
139
|
shard_names.any?
|
131
140
|
end
|
132
141
|
|
133
|
-
def
|
134
|
-
current_shard_selection.
|
142
|
+
def on_replica?
|
143
|
+
current_shard_selection.on_replica?
|
135
144
|
end
|
145
|
+
alias_method :on_slave?, :on_replica?
|
136
146
|
|
137
147
|
def current_shard_selection
|
138
148
|
Thread.current[:shard_selection] ||= ShardSelection.new
|
@@ -157,8 +167,8 @@ module ActiveRecordShards
|
|
157
167
|
|
158
168
|
def switch_connection(options)
|
159
169
|
if options.any?
|
160
|
-
if options.key?(:
|
161
|
-
current_shard_selection.
|
170
|
+
if options.key?(:replica)
|
171
|
+
current_shard_selection.on_replica = options[:replica]
|
162
172
|
end
|
163
173
|
|
164
174
|
if options.key?(:shard)
|
@@ -199,16 +209,18 @@ module ActiveRecordShards
|
|
199
209
|
with_default_shard { table_exists_without_default_shard? }
|
200
210
|
end
|
201
211
|
|
202
|
-
class
|
212
|
+
class PrimaryReplicaProxy
|
203
213
|
def initialize(target, which)
|
204
214
|
@target = target
|
205
215
|
@which = which
|
206
216
|
end
|
207
217
|
|
208
218
|
def method_missing(method, *args, &block) # rubocop:disable Style/MethodMissingSuper, Style/MissingRespondToMissing
|
209
|
-
@target.
|
219
|
+
@target.on_primary_or_replica(@which) { @target.send(method, *args, &block) }
|
210
220
|
end
|
211
221
|
end
|
222
|
+
|
223
|
+
MasterSlaveProxy = PrimaryReplicaProxy
|
212
224
|
end
|
213
225
|
end
|
214
226
|
|
@@ -0,0 +1,171 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module ActiveRecordShards
|
4
|
+
module DefaultReplicaPatches
|
5
|
+
def self.wrap_method_in_on_replica(class_method, base, method, force_on_replica: false)
|
6
|
+
base_methods =
|
7
|
+
if class_method
|
8
|
+
base.methods + base.private_methods
|
9
|
+
else
|
10
|
+
base.instance_methods + base.private_instance_methods
|
11
|
+
end
|
12
|
+
|
13
|
+
return unless base_methods.include?(method)
|
14
|
+
|
15
|
+
_, method, punctuation = method.to_s.match(/^(.*?)([\?\!]?)$/).to_a
|
16
|
+
# _ALWAYS_ on replica, or only for on `on_replica_by_default = true` models?
|
17
|
+
wrapper = force_on_replica ? 'force_on_replica' : 'on_replica_unless_tx'
|
18
|
+
base.class_eval <<-RUBY, __FILE__, __LINE__ + 1
|
19
|
+
#{class_method ? 'class << self' : ''}
|
20
|
+
def #{method}_with_default_replica#{punctuation}(*args, &block)
|
21
|
+
#{wrapper} do
|
22
|
+
#{method}_without_default_replica#{punctuation}(*args, &block)
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
alias_method :#{method}_without_default_replica#{punctuation}, :#{method}#{punctuation}
|
27
|
+
alias_method :#{method}#{punctuation}, :#{method}_with_default_replica#{punctuation}
|
28
|
+
#{class_method ? 'end' : ''}
|
29
|
+
RUBY
|
30
|
+
end
|
31
|
+
|
32
|
+
def self.wrap_method_in_on_slave(*args)
|
33
|
+
ActiveRecordShards::Deprecation.deprecation_warning(
|
34
|
+
:'self.wrap_method_in_on_slave',
|
35
|
+
:'self.wrap_method_in_on_replica'
|
36
|
+
)
|
37
|
+
wrap_method_in_on_replica(*args)
|
38
|
+
end
|
39
|
+
|
40
|
+
def columns_with_force_replica(*args, &block)
|
41
|
+
on_cx_switch_block(:replica, construct_ro_scope: false, force: true) do
|
42
|
+
columns_without_force_replica(*args, &block)
|
43
|
+
end
|
44
|
+
end
|
45
|
+
alias_method :columns_with_force_slave, :columns_with_force_replica
|
46
|
+
|
47
|
+
def table_exists_with_force_replica?(*args, &block)
|
48
|
+
on_cx_switch_block(:replica, construct_ro_scope: false, force: true) do
|
49
|
+
table_exists_without_force_replica?(*args, &block)
|
50
|
+
end
|
51
|
+
end
|
52
|
+
alias_method :table_exists_with_force_slave?, :table_exists_with_force_replica?
|
53
|
+
|
54
|
+
def transaction_with_replica_off(*args, &block)
|
55
|
+
if on_replica_by_default?
|
56
|
+
begin
|
57
|
+
old_val = Thread.current[:_active_record_shards_replica_off]
|
58
|
+
Thread.current[:_active_record_shards_replica_off] = true
|
59
|
+
transaction_without_replica_off(*args, &block)
|
60
|
+
ensure
|
61
|
+
Thread.current[:_active_record_shards_replica_off] = old_val
|
62
|
+
end
|
63
|
+
else
|
64
|
+
transaction_without_replica_off(*args, &block)
|
65
|
+
end
|
66
|
+
end
|
67
|
+
alias_method :transaction_with_slave_off, :transaction_with_replica_off
|
68
|
+
|
69
|
+
module InstanceMethods
|
70
|
+
# fix ActiveRecord to do the right thing, and use our aliased quote_value
|
71
|
+
def quote_value(*args, &block)
|
72
|
+
self.class.quote_value(*args, &block)
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
CLASS_REPLICA_METHODS = [
|
77
|
+
:calculate,
|
78
|
+
:count_by_sql,
|
79
|
+
:exists?,
|
80
|
+
:find_by_sql,
|
81
|
+
:find_every,
|
82
|
+
:find_one,
|
83
|
+
:find_some
|
84
|
+
].freeze
|
85
|
+
|
86
|
+
CLASS_FORCE_REPLICA_METHODS = [
|
87
|
+
:columns,
|
88
|
+
:replace_bind_variable,
|
89
|
+
:replace_bind_variables,
|
90
|
+
:sanitize_sql_array,
|
91
|
+
:sanitize_sql_hash_for_assignment,
|
92
|
+
:table_exists?
|
93
|
+
].freeze
|
94
|
+
|
95
|
+
CLASS_SLAVE_METHODS = CLASS_REPLICA_METHODS
|
96
|
+
CLASS_FORCE_SLAVE_METHODS = CLASS_FORCE_REPLICA_METHODS
|
97
|
+
|
98
|
+
def self.extended(base)
|
99
|
+
CLASS_REPLICA_METHODS.each { |m| ActiveRecordShards::DefaultReplicaPatches.wrap_method_in_on_replica(true, base, m) }
|
100
|
+
CLASS_FORCE_SLAVE_METHODS.each { |m| ActiveRecordShards::DefaultReplicaPatches.wrap_method_in_on_replica(true, base, m, force_on_replica: true) }
|
101
|
+
|
102
|
+
base.class_eval do
|
103
|
+
include InstanceMethods
|
104
|
+
|
105
|
+
class << self
|
106
|
+
alias_method :transaction_without_replica_off, :transaction
|
107
|
+
alias_method :transaction, :transaction_with_replica_off
|
108
|
+
end
|
109
|
+
end
|
110
|
+
if ActiveRecord::Associations.const_defined?(:HasAndBelongsToManyAssociation)
|
111
|
+
ActiveRecordShards::DefaultReplicaPatches.wrap_method_in_on_replica(false, ActiveRecord::Associations::HasAndBelongsToManyAssociation, :construct_sql)
|
112
|
+
ActiveRecordShards::DefaultReplicaPatches.wrap_method_in_on_replica(false, ActiveRecord::Associations::HasAndBelongsToManyAssociation, :construct_find_options!)
|
113
|
+
end
|
114
|
+
end
|
115
|
+
|
116
|
+
def on_replica_unless_tx
|
117
|
+
if on_replica_by_default? && !Thread.current[:_active_record_shards_replica_off]
|
118
|
+
on_replica { yield }
|
119
|
+
else
|
120
|
+
yield
|
121
|
+
end
|
122
|
+
end
|
123
|
+
alias_method :on_slave_unless_tx, :on_replica_unless_tx
|
124
|
+
|
125
|
+
def force_on_replica(&block)
|
126
|
+
on_cx_switch_block(:replica, construct_ro_scope: false, force: true, &block)
|
127
|
+
end
|
128
|
+
|
129
|
+
module ActiveRelationPatches
|
130
|
+
def self.included(base)
|
131
|
+
[:calculate, :exists?, :pluck, :load].each do |m|
|
132
|
+
ActiveRecordShards::DefaultReplicaPatches.wrap_method_in_on_replica(false, base, m)
|
133
|
+
end
|
134
|
+
|
135
|
+
if ActiveRecord::VERSION::MAJOR == 4
|
136
|
+
# `where` and `having` clauses call `create_binds`, which will use the primary connection
|
137
|
+
ActiveRecordShards::DefaultReplicaPatches.wrap_method_in_on_replica(false, base, :create_binds, force_on_replica: true)
|
138
|
+
end
|
139
|
+
|
140
|
+
ActiveRecordShards::DefaultReplicaPatches.wrap_method_in_on_replica(false, base, :to_sql, force_on_replica: true)
|
141
|
+
end
|
142
|
+
|
143
|
+
def on_replica_unless_tx
|
144
|
+
@klass.on_replica_unless_tx { yield }
|
145
|
+
end
|
146
|
+
end
|
147
|
+
|
148
|
+
# in rails 4.1+, they create a join class that's used to pull in records for HABTM.
|
149
|
+
# this simplifies the hell out of our existence, because all we have to do is inerit on-replica-by-default
|
150
|
+
# down from the parent now.
|
151
|
+
module Rails41HasAndBelongsToManyBuilderExtension
|
152
|
+
def self.included(base)
|
153
|
+
base.class_eval do
|
154
|
+
alias_method :through_model_without_inherit_default_replica_from_lhs, :through_model
|
155
|
+
alias_method :through_model, :through_model_with_inherit_default_replica_from_lhs
|
156
|
+
end
|
157
|
+
end
|
158
|
+
|
159
|
+
def through_model_with_inherit_default_replica_from_lhs
|
160
|
+
model = through_model_without_inherit_default_replica_from_lhs
|
161
|
+
def model.on_replica_by_default?
|
162
|
+
left_reflection.klass.on_replica_by_default?
|
163
|
+
end
|
164
|
+
|
165
|
+
# also transfer the sharded-ness of the left table to the join model
|
166
|
+
model.not_sharded unless model.left_reflection.klass.is_sharded?
|
167
|
+
model
|
168
|
+
end
|
169
|
+
end
|
170
|
+
end
|
171
|
+
end
|
@@ -1,144 +1,5 @@
|
|
1
|
-
|
1
|
+
ActiveRecordShards::Deprecation.warn('`DefaultSlavePatches` is deprecated, please use `DefaultReplicaPatches`.')
|
2
2
|
|
3
3
|
module ActiveRecordShards
|
4
|
-
|
5
|
-
def self.wrap_method_in_on_slave(class_method, base, method, force_on_slave: false)
|
6
|
-
base_methods =
|
7
|
-
if class_method
|
8
|
-
base.methods + base.private_methods
|
9
|
-
else
|
10
|
-
base.instance_methods + base.private_instance_methods
|
11
|
-
end
|
12
|
-
|
13
|
-
return unless base_methods.include?(method)
|
14
|
-
|
15
|
-
_, method, punctuation = method.to_s.match(/^(.*?)([\?\!]?)$/).to_a
|
16
|
-
# _ALWAYS_ on slave, or only for on `on_slave_by_default = true` models?
|
17
|
-
wrapper = force_on_slave ? 'force_on_slave' : 'on_slave_unless_tx'
|
18
|
-
base.class_eval <<-RUBY, __FILE__, __LINE__ + 1
|
19
|
-
#{class_method ? 'class << self' : ''}
|
20
|
-
def #{method}_with_default_slave#{punctuation}(*args, &block)
|
21
|
-
#{wrapper} do
|
22
|
-
#{method}_without_default_slave#{punctuation}(*args, &block)
|
23
|
-
end
|
24
|
-
end
|
25
|
-
|
26
|
-
alias_method :#{method}_without_default_slave#{punctuation}, :#{method}#{punctuation}
|
27
|
-
alias_method :#{method}#{punctuation}, :#{method}_with_default_slave#{punctuation}
|
28
|
-
#{class_method ? 'end' : ''}
|
29
|
-
RUBY
|
30
|
-
end
|
31
|
-
|
32
|
-
def transaction_with_slave_off(*args, &block)
|
33
|
-
if on_slave_by_default?
|
34
|
-
begin
|
35
|
-
old_val = Thread.current[:_active_record_shards_slave_off]
|
36
|
-
Thread.current[:_active_record_shards_slave_off] = true
|
37
|
-
transaction_without_slave_off(*args, &block)
|
38
|
-
ensure
|
39
|
-
Thread.current[:_active_record_shards_slave_off] = old_val
|
40
|
-
end
|
41
|
-
else
|
42
|
-
transaction_without_slave_off(*args, &block)
|
43
|
-
end
|
44
|
-
end
|
45
|
-
|
46
|
-
module InstanceMethods
|
47
|
-
# fix ActiveRecord to do the right thing, and use our aliased quote_value
|
48
|
-
def quote_value(*args, &block)
|
49
|
-
self.class.quote_value(*args, &block)
|
50
|
-
end
|
51
|
-
end
|
52
|
-
|
53
|
-
CLASS_SLAVE_METHODS = [
|
54
|
-
:calculate,
|
55
|
-
:count_by_sql,
|
56
|
-
:exists?,
|
57
|
-
:find_by_sql,
|
58
|
-
:find_every,
|
59
|
-
:find_one,
|
60
|
-
:find_some
|
61
|
-
].freeze
|
62
|
-
|
63
|
-
CLASS_FORCE_SLAVE_METHODS = [
|
64
|
-
:columns,
|
65
|
-
:replace_bind_variable,
|
66
|
-
:replace_bind_variables,
|
67
|
-
:sanitize_sql_array,
|
68
|
-
:sanitize_sql_hash_for_assignment,
|
69
|
-
:table_exists?
|
70
|
-
].freeze
|
71
|
-
|
72
|
-
def self.extended(base)
|
73
|
-
CLASS_SLAVE_METHODS.each { |m| ActiveRecordShards::DefaultSlavePatches.wrap_method_in_on_slave(true, base, m) }
|
74
|
-
CLASS_FORCE_SLAVE_METHODS.each { |m| ActiveRecordShards::DefaultSlavePatches.wrap_method_in_on_slave(true, base, m, force_on_slave: true) }
|
75
|
-
|
76
|
-
base.class_eval do
|
77
|
-
include InstanceMethods
|
78
|
-
|
79
|
-
class << self
|
80
|
-
alias_method :transaction_without_slave_off, :transaction
|
81
|
-
alias_method :transaction, :transaction_with_slave_off
|
82
|
-
end
|
83
|
-
end
|
84
|
-
if ActiveRecord::Associations.const_defined?(:HasAndBelongsToManyAssociation)
|
85
|
-
ActiveRecordShards::DefaultSlavePatches.wrap_method_in_on_slave(false, ActiveRecord::Associations::HasAndBelongsToManyAssociation, :construct_sql)
|
86
|
-
ActiveRecordShards::DefaultSlavePatches.wrap_method_in_on_slave(false, ActiveRecord::Associations::HasAndBelongsToManyAssociation, :construct_find_options!)
|
87
|
-
end
|
88
|
-
end
|
89
|
-
|
90
|
-
def on_slave_unless_tx(&block)
|
91
|
-
if on_slave_by_default? && !Thread.current[:_active_record_shards_slave_off]
|
92
|
-
on_slave(&block)
|
93
|
-
else
|
94
|
-
yield
|
95
|
-
end
|
96
|
-
end
|
97
|
-
|
98
|
-
def force_on_slave(&block)
|
99
|
-
on_cx_switch_block(:slave, construct_ro_scope: false, force: true, &block)
|
100
|
-
end
|
101
|
-
|
102
|
-
module ActiveRelationPatches
|
103
|
-
def self.included(base)
|
104
|
-
[:calculate, :exists?, :pluck, :load].each do |m|
|
105
|
-
ActiveRecordShards::DefaultSlavePatches.wrap_method_in_on_slave(false, base, m)
|
106
|
-
end
|
107
|
-
|
108
|
-
if ActiveRecord::VERSION::MAJOR == 4
|
109
|
-
# `where` and `having` clauses call `create_binds`, which will use the master connection
|
110
|
-
ActiveRecordShards::DefaultSlavePatches.wrap_method_in_on_slave(false, base, :create_binds, force_on_slave: true)
|
111
|
-
end
|
112
|
-
|
113
|
-
ActiveRecordShards::DefaultSlavePatches.wrap_method_in_on_slave(false, base, :to_sql, force_on_slave: true)
|
114
|
-
end
|
115
|
-
|
116
|
-
def on_slave_unless_tx(&block)
|
117
|
-
@klass.on_slave_unless_tx(&block)
|
118
|
-
end
|
119
|
-
end
|
120
|
-
|
121
|
-
# in rails 4.1+, they create a join class that's used to pull in records for HABTM.
|
122
|
-
# this simplifies the hell out of our existence, because all we have to do is inerit on-slave-by-default
|
123
|
-
# down from the parent now.
|
124
|
-
module Rails41HasAndBelongsToManyBuilderExtension
|
125
|
-
def self.included(base)
|
126
|
-
base.class_eval do
|
127
|
-
alias_method :through_model_without_inherit_default_slave_from_lhs, :through_model
|
128
|
-
alias_method :through_model, :through_model_with_inherit_default_slave_from_lhs
|
129
|
-
end
|
130
|
-
end
|
131
|
-
|
132
|
-
def through_model_with_inherit_default_slave_from_lhs
|
133
|
-
model = through_model_without_inherit_default_slave_from_lhs
|
134
|
-
def model.on_slave_by_default?
|
135
|
-
left_reflection.klass.on_slave_by_default?
|
136
|
-
end
|
137
|
-
|
138
|
-
# also transfer the sharded-ness of the left table to the join model
|
139
|
-
model.not_sharded unless model.left_reflection.klass.is_sharded?
|
140
|
-
model
|
141
|
-
end
|
142
|
-
end
|
143
|
-
end
|
4
|
+
DefaultSlavePatches = DefaultReplicaPatches
|
144
5
|
end
|
@@ -0,0 +1,12 @@
|
|
1
|
+
module ActiveRecordShards
|
2
|
+
class Deprecation < ActiveSupport::Deprecation
|
3
|
+
# This allows us to define separate deprecation behavior for ActiveRecordShards, but defaults to
|
4
|
+
# the same behavior globally configured with ActiveSupport.
|
5
|
+
#
|
6
|
+
# For example, this allows us to silence our own deprecation warnings in test while still being
|
7
|
+
# able to fail tests for upstream deprecation warnings.
|
8
|
+
def behavior
|
9
|
+
@behavior ||= ActiveSupport::Deprecation.behavior
|
10
|
+
end
|
11
|
+
end
|
12
|
+
end
|
@@ -24,34 +24,38 @@ module ActiveRecordShards
|
|
24
24
|
end
|
25
25
|
end
|
26
26
|
|
27
|
-
def
|
27
|
+
def on_replica_by_default?
|
28
28
|
if self == ActiveRecord::Base
|
29
29
|
false
|
30
30
|
else
|
31
31
|
base = base_class
|
32
|
-
if base.instance_variable_defined?(:@
|
33
|
-
base.instance_variable_get(:@
|
32
|
+
if base.instance_variable_defined?(:@on_replica_by_default)
|
33
|
+
base.instance_variable_get(:@on_replica_by_default)
|
34
34
|
end
|
35
35
|
end
|
36
36
|
end
|
37
|
+
alias_method :on_slave_by_default?, :on_replica_by_default?
|
37
38
|
|
38
|
-
def
|
39
|
+
def on_replica_by_default=(value)
|
39
40
|
if self == ActiveRecord::Base
|
40
|
-
raise ArgumentError, "Cannot set
|
41
|
+
raise ArgumentError, "Cannot set on_replica_by_default on ActiveRecord::Base"
|
41
42
|
else
|
42
|
-
base_class.instance_variable_set(:@
|
43
|
+
base_class.instance_variable_set(:@on_replica_by_default, value)
|
43
44
|
end
|
44
45
|
end
|
46
|
+
alias_method :on_slave_by_default=, :on_replica_by_default=
|
45
47
|
|
46
48
|
module InstanceMethods
|
47
|
-
def
|
48
|
-
@
|
49
|
+
def initialize_shard_and_replica
|
50
|
+
@from_replica = !!self.class.current_shard_selection.options[:replica]
|
49
51
|
@from_shard = self.class.current_shard_selection.options[:shard]
|
50
52
|
end
|
53
|
+
alias_method :initialize_shard_and_slave, :initialize_shard_and_replica
|
51
54
|
|
52
|
-
def
|
53
|
-
@
|
55
|
+
def from_replica?
|
56
|
+
@from_replica
|
54
57
|
end
|
58
|
+
alias_method :from_slave?, :from_replica?
|
55
59
|
|
56
60
|
def from_shard
|
57
61
|
@from_shard
|
@@ -60,7 +64,7 @@ module ActiveRecordShards
|
|
60
64
|
|
61
65
|
def self.extended(base)
|
62
66
|
base.send(:include, InstanceMethods)
|
63
|
-
base.after_initialize :
|
67
|
+
base.after_initialize :initialize_shard_and_replica
|
64
68
|
end
|
65
69
|
|
66
70
|
private
|
@@ -6,7 +6,7 @@ module ActiveRecordShards
|
|
6
6
|
cattr_accessor :default_shard
|
7
7
|
|
8
8
|
def initialize
|
9
|
-
@
|
9
|
+
@on_replica = false
|
10
10
|
@shard = nil
|
11
11
|
end
|
12
12
|
|
@@ -22,17 +22,17 @@ module ActiveRecordShards
|
|
22
22
|
end
|
23
23
|
end
|
24
24
|
|
25
|
-
def shard_name(klass = nil,
|
25
|
+
def shard_name(klass = nil, try_replica = true)
|
26
26
|
the_shard = shard(klass)
|
27
27
|
|
28
28
|
@shard_names ||= {}
|
29
29
|
@shard_names[ActiveRecordShards.rails_env] ||= {}
|
30
30
|
@shard_names[ActiveRecordShards.rails_env][the_shard] ||= {}
|
31
|
-
@shard_names[ActiveRecordShards.rails_env][the_shard][
|
32
|
-
@shard_names[ActiveRecordShards.rails_env][the_shard][
|
31
|
+
@shard_names[ActiveRecordShards.rails_env][the_shard][try_replica] ||= {}
|
32
|
+
@shard_names[ActiveRecordShards.rails_env][the_shard][try_replica][@on_replica] ||= begin
|
33
33
|
s = ActiveRecordShards.rails_env.dup
|
34
34
|
s << "_shard_#{the_shard}" if the_shard
|
35
|
-
s << "
|
35
|
+
s << "_replica" if @on_replica && try_replica
|
36
36
|
s
|
37
37
|
end
|
38
38
|
end
|
@@ -55,11 +55,11 @@ module ActiveRecordShards
|
|
55
55
|
@connection_names ||= {}
|
56
56
|
@connection_names[env] ||= {}
|
57
57
|
@connection_names[env][resolved_shard] ||= {}
|
58
|
-
@connection_names[env][resolved_shard][@
|
58
|
+
@connection_names[env][resolved_shard][@on_replica] ||= begin
|
59
59
|
name = env.dup
|
60
60
|
name << "_shard_#{resolved_shard}" if resolved_shard
|
61
|
-
if @
|
62
|
-
"#{name}
|
61
|
+
if @on_replica && configurations["#{name}_replica"]
|
62
|
+
"#{name}_replica"
|
63
63
|
else
|
64
64
|
# ActiveRecord always names its default connection pool 'primary'
|
65
65
|
# while everything else is named by the configuration name
|
@@ -74,16 +74,18 @@ module ActiveRecordShards
|
|
74
74
|
@shard = (new_shard || NO_SHARD)
|
75
75
|
end
|
76
76
|
|
77
|
-
def
|
78
|
-
@
|
77
|
+
def on_replica?
|
78
|
+
@on_replica
|
79
79
|
end
|
80
|
+
alias_method :on_slave?, :on_replica?
|
80
81
|
|
81
|
-
def
|
82
|
-
@
|
82
|
+
def on_replica=(new_replica)
|
83
|
+
@on_replica = (new_replica == true)
|
83
84
|
end
|
85
|
+
alias_method :on_slave=, :on_replica=
|
84
86
|
|
85
87
|
def options
|
86
|
-
{ shard: @shard,
|
88
|
+
{ shard: @shard, replica: @on_replica }
|
87
89
|
end
|
88
90
|
end
|
89
91
|
end
|
@@ -1,16 +1,16 @@
|
|
1
|
-
# show which connection was picked to debug
|
1
|
+
# show which connection was picked to debug primary/replica slowness when both servers are the same
|
2
2
|
module ActiveRecordShards
|
3
3
|
module SqlComments
|
4
4
|
module Methods
|
5
5
|
def execute(query, name = nil)
|
6
|
-
|
7
|
-
query += " /* #{
|
6
|
+
replica = ActiveRecord::Base.current_shard_selection.on_replica?
|
7
|
+
query += " /* #{replica ? 'replica' : 'primary'} */"
|
8
8
|
super(query, name)
|
9
9
|
end
|
10
10
|
end
|
11
11
|
|
12
12
|
def self.enable
|
13
|
-
ActiveRecord::Base.
|
13
|
+
ActiveRecord::Base.on_replica do
|
14
14
|
ActiveRecord::Base.on_shard(nil) do
|
15
15
|
ActiveRecord::Base.connection.class.prepend(Methods)
|
16
16
|
end
|
@@ -10,7 +10,7 @@ namespace :db do
|
|
10
10
|
desc 'Drops the database for the current RAILS_ENV including shards'
|
11
11
|
task drop: :load_config do
|
12
12
|
ActiveRecord::Base.configurations.to_h.each do |key, conf|
|
13
|
-
next if !key.
|
13
|
+
next if !key.start_with?(ActiveRecordShards.rails_env) || key.end_with?("_replica", "_slave")
|
14
14
|
|
15
15
|
begin
|
16
16
|
ActiveRecordShards::Tasks.root_connection(conf).drop_database(conf['database'])
|
@@ -31,7 +31,7 @@ namespace :db do
|
|
31
31
|
desc "Create the database defined in config/database.yml for the current RAILS_ENV including shards"
|
32
32
|
task create: :load_config do
|
33
33
|
ActiveRecord::Base.configurations.to_h.each do |key, conf|
|
34
|
-
next if !key.
|
34
|
+
next if !key.start_with?(ActiveRecordShards.rails_env) || key.end_with?("_replica", "_slave")
|
35
35
|
|
36
36
|
begin
|
37
37
|
# MysqlAdapter takes charset instead of encoding in Rails 4.2 or greater
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: active_record_shards
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.
|
4
|
+
version: 3.18.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Benjamin Quorning
|
@@ -10,10 +10,10 @@ authors:
|
|
10
10
|
- Mick Staugaard
|
11
11
|
- Eric Chapweske
|
12
12
|
- Ben Osheroff
|
13
|
-
autorequire:
|
13
|
+
autorequire:
|
14
14
|
bindir: bin
|
15
15
|
cert_chain: []
|
16
|
-
date:
|
16
|
+
date: 2021-06-21 00:00:00.000000000 Z
|
17
17
|
dependencies:
|
18
18
|
- !ruby/object:Gem::Dependency
|
19
19
|
name: activerecord
|
@@ -195,7 +195,7 @@ dependencies:
|
|
195
195
|
- - "~>"
|
196
196
|
- !ruby/object:Gem::Version
|
197
197
|
version: 1.5.1
|
198
|
-
description: Easily run queries on shard and
|
198
|
+
description: Easily run queries on shard and replica databases.
|
199
199
|
email:
|
200
200
|
- bquorning@zendesk.com
|
201
201
|
- gabe@zendesk.com
|
@@ -216,7 +216,9 @@ files:
|
|
216
216
|
- lib/active_record_shards/connection_switcher-5-0.rb
|
217
217
|
- lib/active_record_shards/connection_switcher-5-1.rb
|
218
218
|
- lib/active_record_shards/connection_switcher.rb
|
219
|
+
- lib/active_record_shards/default_replica_patches.rb
|
219
220
|
- lib/active_record_shards/default_slave_patches.rb
|
221
|
+
- lib/active_record_shards/deprecation.rb
|
220
222
|
- lib/active_record_shards/migration.rb
|
221
223
|
- lib/active_record_shards/model.rb
|
222
224
|
- lib/active_record_shards/patches-4-2.rb
|
@@ -229,7 +231,7 @@ homepage: https://github.com/zendesk/active_record_shards
|
|
229
231
|
licenses:
|
230
232
|
- MIT
|
231
233
|
metadata: {}
|
232
|
-
post_install_message:
|
234
|
+
post_install_message:
|
233
235
|
rdoc_options: []
|
234
236
|
require_paths:
|
235
237
|
- lib
|
@@ -244,8 +246,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
244
246
|
- !ruby/object:Gem::Version
|
245
247
|
version: '0'
|
246
248
|
requirements: []
|
247
|
-
rubygems_version: 3.
|
248
|
-
signing_key:
|
249
|
+
rubygems_version: 3.2.16
|
250
|
+
signing_key:
|
249
251
|
specification_version: 4
|
250
252
|
summary: Simple database switching for ActiveRecord.
|
251
253
|
test_files: []
|