master_slave_adapter 1.0.0.beta1 → 1.0.0.beta2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +2 -1
- data/.rspec +1 -0
- data/.travis.yml +4 -0
- data/CHANGELOG.md +4 -1
- data/LICENSE +2 -0
- data/Rakefile +51 -2
- data/Readme.md +55 -37
- data/lib/active_record/connection_adapters/master_slave_adapter/clock.rb +0 -1
- data/lib/active_record/connection_adapters/master_slave_adapter/shared_mysql_adapter_behavior.rb +43 -0
- data/lib/active_record/connection_adapters/master_slave_adapter/version.rb +1 -1
- data/lib/active_record/connection_adapters/master_slave_adapter.rb +347 -354
- data/lib/active_record/connection_adapters/mysql2_master_slave_adapter.rb +48 -0
- data/lib/active_record/connection_adapters/mysql_master_slave_adapter.rb +22 -41
- data/master_slave_adapter.gemspec +3 -3
- data/spec/all.sh +15 -0
- data/spec/gemfiles/activerecord2.3 +5 -0
- data/spec/gemfiles/activerecord3.0 +5 -0
- data/spec/gemfiles/activerecord3.2 +6 -0
- data/spec/integration/helpers/mysql_helper.rb +174 -0
- data/spec/integration/helpers/shared_mysql_examples.rb +212 -0
- data/spec/integration/mysql2_master_slave_adapter_spec.rb +11 -0
- data/spec/integration/mysql_master_slave_adapter_spec.rb +11 -0
- data/spec/master_slave_adapter_spec.rb +75 -21
- data/spec/mysql2_master_slave_adapter_spec.rb +372 -0
- data/spec/mysql_master_slave_adapter_spec.rb +116 -72
- metadata +40 -15
- data/Gemfile +0 -3
- data/TODO.txt +0 -18
data/.gitignore
CHANGED
data/.rspec
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
--color
|
data/.travis.yml
CHANGED
data/CHANGELOG.md
CHANGED
@@ -1,13 +1,16 @@
|
|
1
1
|
# 1.0.0 (not released yet)
|
2
2
|
|
3
3
|
* Add support for unavailable master connection
|
4
|
-
* Fallback to slave connection if possible
|
5
4
|
* Restrict the public interface. Removed the following methods:
|
6
5
|
* all class methods from ActiveRecord::ConnectionAdapters::MasterSlaveAdapter
|
7
6
|
* #current_connection=
|
8
7
|
* #current_clock=
|
9
8
|
* #slave_consistent?
|
9
|
+
* ActiveRecord::Base.on_commit and ActiveRecord::Base.on_rollback
|
10
10
|
* Fix 1.8.7 compliance
|
11
|
+
* Fix bug which led to infinitely connection stack growth
|
12
|
+
* Add ActiveRecord 3.x compatibility
|
13
|
+
* Add support for Mysql2
|
11
14
|
|
12
15
|
# 0.2.0 (April 2, 2012)
|
13
16
|
|
data/LICENSE
CHANGED
data/Rakefile
CHANGED
@@ -3,8 +3,57 @@ require 'rspec/core/rake_task'
|
|
3
3
|
|
4
4
|
Bundler::GemHelper.install_tasks
|
5
5
|
|
6
|
-
|
7
|
-
|
6
|
+
class MasterSlaveAdapterRSpecTask < RSpec::Core::RakeTask
|
7
|
+
attr_accessor :exclude
|
8
|
+
|
9
|
+
private
|
10
|
+
|
11
|
+
def files_to_run
|
12
|
+
FileList[ pattern ].exclude(exclude)
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
def mysql2_adapter_available?
|
17
|
+
require 'active_record/connection_adapters/mysql2_adapter'
|
18
|
+
true
|
19
|
+
rescue LoadError
|
20
|
+
false
|
21
|
+
rescue
|
22
|
+
true
|
23
|
+
end
|
8
24
|
|
9
25
|
desc 'Default: Run specs'
|
10
26
|
task :default => :spec
|
27
|
+
|
28
|
+
desc 'Run specs'
|
29
|
+
task :spec => ['spec:common', 'spec:integration']
|
30
|
+
|
31
|
+
namespace :spec do
|
32
|
+
desc 'Run common specs'
|
33
|
+
MasterSlaveAdapterRSpecTask.new(:common) do |task|
|
34
|
+
task.pattern = './spec/*_spec.rb'
|
35
|
+
task.exclude = /mysql2/ unless mysql2_adapter_available?
|
36
|
+
task.verbose = false
|
37
|
+
end
|
38
|
+
|
39
|
+
desc 'Run integration specs'
|
40
|
+
task :integration => ['spec:integration:check', 'spec:integration:all']
|
41
|
+
|
42
|
+
namespace :integration do
|
43
|
+
desc 'Check requirements'
|
44
|
+
task :check do
|
45
|
+
[:mysql, :mysqld, :mysql_install_db].each do |executable|
|
46
|
+
unless system("which #{executable} > /dev/null")
|
47
|
+
raise "Can't run integration tests. #{executable} is not available in $PATH"
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
desc 'Run all integration specs'
|
53
|
+
MasterSlaveAdapterRSpecTask.new(:all) do |task|
|
54
|
+
task.pattern = './spec/integration/*_spec.rb'
|
55
|
+
task.exclude = /mysql2/ unless mysql2_adapter_available?
|
56
|
+
task.verbose = false
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
data/Readme.md
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
# Replication Aware Master Slave Adapter [][6]
|
2
2
|
|
3
3
|
Improved version of the [master_slave_adapter plugin][1], packaged as a gem.
|
4
4
|
|
@@ -6,19 +6,17 @@ Improved version of the [master_slave_adapter plugin][1], packaged as a gem.
|
|
6
6
|
|
7
7
|
1. automatic selection of master or slave connection: `with_consistency`
|
8
8
|
2. manual selection of master or slave connection: `with_master`, `with_slave`
|
9
|
-
3.
|
10
|
-
4.
|
9
|
+
3. handles master unavailable scenarios gracefully
|
10
|
+
4. transaction callbacks: `on_commit`, `on_rollback`
|
11
|
+
5. also:
|
11
12
|
* support for multiple slaves
|
12
13
|
* (partial) support for [database_cleaner][2]
|
13
14
|
|
14
15
|
### Automatic Selection of Master or Slave
|
15
16
|
|
16
|
-
* _note that this feature currently only works with MySQL_
|
17
|
-
* _see also this [blog post][3] for a more detailed explanation_
|
18
|
-
|
19
17
|
The adapter will run all reads against a slave database, unless a) the read is inside an open transaction or b) the
|
20
18
|
adapter determines that the slave lags behind the master _relative to the last write_. For this to work, an initial
|
21
|
-
initial consistency requirement
|
19
|
+
initial consistency requirement, a Clock, must be passed to the adapter. Based on this clock value, the adapter
|
22
20
|
determines if a (randomly chosen) slave meets this requirement. If not, all statements are executed against master,
|
23
21
|
otherwise, the slave connection is used until either a transaction is opened or a write occurs. After a successful write
|
24
22
|
or transaction, the adapter determines a new consistency requirement, which is returned and can be used for subsequent
|
@@ -29,20 +27,16 @@ As an example, a Rails application could run the following function as an `aroun
|
|
29
27
|
```ruby
|
30
28
|
def with_consistency_filter
|
31
29
|
if logged_in?
|
32
|
-
|
33
|
-
cache_key = [ CACHE_NAMESPACE, current_user.id.to_s ].join(":")
|
34
|
-
|
35
|
-
clock = cached_clock(cache_key) ||
|
36
|
-
ActiveRecord::Base.connection.master_clock
|
30
|
+
clock = cached_clock_for(current_user)
|
37
31
|
|
38
32
|
new_clock = ActiveRecord::Base.with_consistency(clock) do
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
33
|
+
# inside the controller, ActiveRecord models can be used just as normal.
|
34
|
+
# The adapter will take care of choosing the right connection.
|
35
|
+
yield
|
36
|
+
end
|
43
37
|
|
44
38
|
[ new_clock, clock ].compact.max.tap do |c|
|
45
|
-
|
39
|
+
cache_clock_for(current_user, c)
|
46
40
|
end if new_clock != clock
|
47
41
|
else
|
48
42
|
# anonymous users will have to wait until the slaves have caught up
|
@@ -51,13 +45,15 @@ def with_consistency_filter
|
|
51
45
|
end
|
52
46
|
```
|
53
47
|
|
54
|
-
Note that we use the
|
48
|
+
Note that we use the last seen consistency for a given user as reference point. This will give the user a recent view of the data,
|
55
49
|
possibly reading from master, and if no write occurs inside the `with_consistency` block, we have a reasonable value to
|
56
|
-
cache and reuse on subsequent requests.
|
57
|
-
|
50
|
+
cache and reuse on subsequent requests.
|
51
|
+
If no cached clock is available, this indicates that no particular consistency is required. Any slave connection will do.
|
58
52
|
Since `with_consistency` blocks can be nested, the controller code could later decide to require a more recent view on
|
59
53
|
the data.
|
60
54
|
|
55
|
+
_See also this [blog post][3] for a more detailed explanation._
|
56
|
+
|
61
57
|
### Manual Selection of Master or Slave
|
62
58
|
|
63
59
|
The original functionality of the adapter has been preserved:
|
@@ -76,6 +72,24 @@ end
|
|
76
72
|
|
77
73
|
`with_master`, `with_slave` as well as `with_consistency` can be nested deliberately.
|
78
74
|
|
75
|
+
### Handles master unavailable scenarios gracefully
|
76
|
+
|
77
|
+
Due to scenarios when the master is possibly down (e.g., maintenance), we try
|
78
|
+
to delegate as much as possible to the active slaves. In order to accomplish
|
79
|
+
this we have added the following functionalities.
|
80
|
+
|
81
|
+
* We ignore errors while connecting to the master server.
|
82
|
+
* ActiveRecord::MasterUnavailable exceptions are raised in cases when we need to use
|
83
|
+
a master connection, but the server is unavailable. This exception is propagated
|
84
|
+
to the application.
|
85
|
+
* We have introduced the circuit breaker pattern in the master reconnect logic
|
86
|
+
to prevent excessive reconnection attempts. We block any queries which require
|
87
|
+
a master connection for a given timeout (by default, 30 seconds). After the
|
88
|
+
timeout has expired, any attempt of using the master connection will trigger
|
89
|
+
a reconnection.
|
90
|
+
* The master slave adapter is still usable for any queries that require only
|
91
|
+
slave connections.
|
92
|
+
|
79
93
|
### Transaction Callbacks
|
80
94
|
|
81
95
|
This feature was originally developed at [SoundCloud][4] for the standard `MysqlAdapter`. It allows arbitrary blocks of
|
@@ -123,6 +137,24 @@ At [SoundCloud][4], we're using [database_cleaner][2]'s 'truncation strategy' to
|
|
123
137
|
`truncate_table` as an `ActiveRecord::Base.connection` instance method. We might add other strategies if there's enough
|
124
138
|
interest.
|
125
139
|
|
140
|
+
## Requirements
|
141
|
+
|
142
|
+
MasterSlaveAdapter requires ActiveRecord with a version >= 2.3, is compatible
|
143
|
+
with at least Ruby 1.8.7, 1.9.2, 1.9.3 and comes with built-in support for mysql
|
144
|
+
and mysql2 libraries.
|
145
|
+
|
146
|
+
You can check the versions it's tested against at [Travis CI](http://travis-ci.org/#!/soundcloud/master_slave_adapter).
|
147
|
+
|
148
|
+
## Installation
|
149
|
+
|
150
|
+
Using plain rubygems:
|
151
|
+
|
152
|
+
$ gem install master_slave_adapter
|
153
|
+
|
154
|
+
Using bundler, just include it in your Gemfile:
|
155
|
+
|
156
|
+
gem 'master_slave_adapter'
|
157
|
+
|
126
158
|
## Configuration
|
127
159
|
|
128
160
|
Example configuration for the development environment in `database.yml`:
|
@@ -133,6 +165,7 @@ development:
|
|
133
165
|
connection_adapter: mysql # actual adapter to use (only mysql is supported atm)
|
134
166
|
disable_connection_test: false # when an instance is checked out from the connection pool,
|
135
167
|
# we check if the connections are still alive, reconnecting if necessary
|
168
|
+
|
136
169
|
# these values are picked up as defaults in the 'master' and 'slaves' sections:
|
137
170
|
database: aweapp_development
|
138
171
|
username: aweappuser
|
@@ -147,23 +180,6 @@ development:
|
|
147
180
|
- host: slave02
|
148
181
|
```
|
149
182
|
|
150
|
-
## Installation
|
151
|
-
|
152
|
-
Using plain rubygems:
|
153
|
-
|
154
|
-
```sh
|
155
|
-
$ gem install master_slave_adapter_soundcloud
|
156
|
-
```
|
157
|
-
|
158
|
-
Using bundler:
|
159
|
-
|
160
|
-
```sh
|
161
|
-
$ cat >> Gemfile
|
162
|
-
gem 'master_slave_adapter_soundcloud', '~> 0.1', :require => 'master_slave_adaper'
|
163
|
-
^D
|
164
|
-
$ bundle install
|
165
|
-
```
|
166
|
-
|
167
183
|
## Credits
|
168
184
|
|
169
185
|
* Maurício Lenhares - _original master_slave_adapter plugin_
|
@@ -171,6 +187,8 @@ $ bundle install
|
|
171
187
|
* Sean Treadway - _chief everything & transaction callbacks_
|
172
188
|
* Kim Altintop - _strong lax monoidal endofunctors_
|
173
189
|
* Omid Aladini - _chief operator & everything else_
|
190
|
+
* Tiago Loureiro - _review expert & master unavailable handling_
|
191
|
+
* Tobias Schmidt - _typo master & activerecord ranter_
|
174
192
|
|
175
193
|
|
176
194
|
[1]: https://github.com/mauricio/master_slave_adapter
|
data/lib/active_record/connection_adapters/master_slave_adapter/shared_mysql_adapter_behavior.rb
ADDED
@@ -0,0 +1,43 @@
|
|
1
|
+
require 'active_record/connection_adapters/master_slave_adapter/clock'
|
2
|
+
|
3
|
+
module ActiveRecord
|
4
|
+
module ConnectionAdapters
|
5
|
+
module MasterSlaveAdapter
|
6
|
+
module SharedMysqlAdapterBehavior
|
7
|
+
def with_consistency(clock)
|
8
|
+
clock =
|
9
|
+
case clock
|
10
|
+
when Clock then clock
|
11
|
+
when String then Clock.parse(clock)
|
12
|
+
when nil then Clock.zero
|
13
|
+
end
|
14
|
+
|
15
|
+
super(clock)
|
16
|
+
end
|
17
|
+
|
18
|
+
def master_clock
|
19
|
+
conn = master_connection
|
20
|
+
if status = conn.uncached { select_hash(conn, "SHOW MASTER STATUS") }
|
21
|
+
Clock.new(status['File'], status['Position'])
|
22
|
+
else
|
23
|
+
Clock.infinity
|
24
|
+
end
|
25
|
+
rescue MasterUnavailable
|
26
|
+
Clock.zero
|
27
|
+
rescue ActiveRecordError
|
28
|
+
Clock.infinity
|
29
|
+
end
|
30
|
+
|
31
|
+
def slave_clock(conn)
|
32
|
+
if status = conn.uncached { select_hash(conn, "SHOW SLAVE STATUS") }
|
33
|
+
Clock.new(status['Relay_Master_Log_File'], status['Exec_Master_Log_Pos'])
|
34
|
+
else
|
35
|
+
Clock.zero
|
36
|
+
end
|
37
|
+
rescue ActiveRecordError
|
38
|
+
Clock.zero
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|