master_slave_adapter 1.0.0.beta1 → 1.0.0.beta2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/.gitignore CHANGED
@@ -2,4 +2,5 @@ tags
2
2
  test/*
3
3
  pkg/*
4
4
  .rvmrc
5
- Gemfile.lock
5
+ spec/gemfiles/*.lock
6
+ spec/integration/mysql
data/.rspec ADDED
@@ -0,0 +1 @@
1
+ --color
data/.travis.yml CHANGED
@@ -3,3 +3,7 @@ rvm:
3
3
  - 1.8.7
4
4
  - 1.9.2
5
5
  - 1.9.3
6
+ gemfile:
7
+ - spec/gemfiles/activerecord2.3
8
+ - spec/gemfiles/activerecord3.0
9
+ - spec/gemfiles/activerecord3.2
data/CHANGELOG.md CHANGED
@@ -1,13 +1,16 @@
1
1
  # 1.0.0 (not released yet)
2
2
 
3
3
  * Add support for unavailable master connection
4
- * Fallback to slave connection if possible
5
4
  * Restrict the public interface. Removed the following methods:
6
5
  * all class methods from ActiveRecord::ConnectionAdapters::MasterSlaveAdapter
7
6
  * #current_connection=
8
7
  * #current_clock=
9
8
  * #slave_consistent?
9
+ * ActiveRecord::Base.on_commit and ActiveRecord::Base.on_rollback
10
10
  * Fix 1.8.7 compliance
11
+ * Fix bug which led to infinitely connection stack growth
12
+ * Add ActiveRecord 3.x compatibility
13
+ * Add support for Mysql2
11
14
 
12
15
  # 0.2.0 (April 2, 2012)
13
16
 
data/LICENSE CHANGED
@@ -2,6 +2,8 @@ Copyright (c) 2011 Maurício Linhares,
2
2
  Torsten Curdt,
3
3
  Kim Altintop,
4
4
  Omid Aladini,
5
+ Tiago Loureiro,
6
+ Tobias Schmidt,
5
7
  SoundCloud Ltd
6
8
 
7
9
  Permission is hereby granted, free of charge, to any person obtaining a copy
data/Rakefile CHANGED
@@ -3,8 +3,57 @@ require 'rspec/core/rake_task'
3
3
 
4
4
  Bundler::GemHelper.install_tasks
5
5
 
6
- desc 'Run specs'
7
- RSpec::Core::RakeTask.new
6
+ class MasterSlaveAdapterRSpecTask < RSpec::Core::RakeTask
7
+ attr_accessor :exclude
8
+
9
+ private
10
+
11
+ def files_to_run
12
+ FileList[ pattern ].exclude(exclude)
13
+ end
14
+ end
15
+
16
+ def mysql2_adapter_available?
17
+ require 'active_record/connection_adapters/mysql2_adapter'
18
+ true
19
+ rescue LoadError
20
+ false
21
+ rescue
22
+ true
23
+ end
8
24
 
9
25
  desc 'Default: Run specs'
10
26
  task :default => :spec
27
+
28
+ desc 'Run specs'
29
+ task :spec => ['spec:common', 'spec:integration']
30
+
31
+ namespace :spec do
32
+ desc 'Run common specs'
33
+ MasterSlaveAdapterRSpecTask.new(:common) do |task|
34
+ task.pattern = './spec/*_spec.rb'
35
+ task.exclude = /mysql2/ unless mysql2_adapter_available?
36
+ task.verbose = false
37
+ end
38
+
39
+ desc 'Run integration specs'
40
+ task :integration => ['spec:integration:check', 'spec:integration:all']
41
+
42
+ namespace :integration do
43
+ desc 'Check requirements'
44
+ task :check do
45
+ [:mysql, :mysqld, :mysql_install_db].each do |executable|
46
+ unless system("which #{executable} > /dev/null")
47
+ raise "Can't run integration tests. #{executable} is not available in $PATH"
48
+ end
49
+ end
50
+ end
51
+
52
+ desc 'Run all integration specs'
53
+ MasterSlaveAdapterRSpecTask.new(:all) do |task|
54
+ task.pattern = './spec/integration/*_spec.rb'
55
+ task.exclude = /mysql2/ unless mysql2_adapter_available?
56
+ task.verbose = false
57
+ end
58
+ end
59
+ end
data/Readme.md CHANGED
@@ -1,4 +1,4 @@
1
- # Replication Aware Master Slave Adapter [![Build Status](https://secure.travis-ci.org/soundcloud/large-hadron-migrator.png)][6]
1
+ # Replication Aware Master Slave Adapter [![Build Status](https://secure.travis-ci.org/soundcloud/master_slave_adapter.png)][6]
2
2
 
3
3
  Improved version of the [master_slave_adapter plugin][1], packaged as a gem.
4
4
 
@@ -6,19 +6,17 @@ Improved version of the [master_slave_adapter plugin][1], packaged as a gem.
6
6
 
7
7
  1. automatic selection of master or slave connection: `with_consistency`
8
8
  2. manual selection of master or slave connection: `with_master`, `with_slave`
9
- 3. transaction callbacks: `on_commit`, `on_rollback`
10
- 4. also:
9
+ 3. handles master unavailable scenarios gracefully
10
+ 4. transaction callbacks: `on_commit`, `on_rollback`
11
+ 5. also:
11
12
  * support for multiple slaves
12
13
  * (partial) support for [database_cleaner][2]
13
14
 
14
15
  ### Automatic Selection of Master or Slave
15
16
 
16
- * _note that this feature currently only works with MySQL_
17
- * _see also this [blog post][3] for a more detailed explanation_
18
-
19
17
  The adapter will run all reads against a slave database, unless a) the read is inside an open transaction or b) the
20
18
  adapter determines that the slave lags behind the master _relative to the last write_. For this to work, an initial
21
- initial consistency requirement ("`Clock`") must be passed to the adapter. Based on this clock value, the adapter
19
+ initial consistency requirement, a Clock, must be passed to the adapter. Based on this clock value, the adapter
22
20
  determines if a (randomly chosen) slave meets this requirement. If not, all statements are executed against master,
23
21
  otherwise, the slave connection is used until either a transaction is opened or a write occurs. After a successful write
24
22
  or transaction, the adapter determines a new consistency requirement, which is returned and can be used for subsequent
@@ -29,20 +27,16 @@ As an example, a Rails application could run the following function as an `aroun
29
27
  ```ruby
30
28
  def with_consistency_filter
31
29
  if logged_in?
32
- # it's a good idea to use this feature on a per-user basis
33
- cache_key = [ CACHE_NAMESPACE, current_user.id.to_s ].join(":")
34
-
35
- clock = cached_clock(cache_key) ||
36
- ActiveRecord::Base.connection.master_clock
30
+ clock = cached_clock_for(current_user)
37
31
 
38
32
  new_clock = ActiveRecord::Base.with_consistency(clock) do
39
- # inside the controller, ActiveRecord models can be used just as normal.
40
- # The adapter will take care of choosing the right connection.
41
- yield
42
- end
33
+ # inside the controller, ActiveRecord models can be used just as normal.
34
+ # The adapter will take care of choosing the right connection.
35
+ yield
36
+ end
43
37
 
44
38
  [ new_clock, clock ].compact.max.tap do |c|
45
- cache_clock!(cache_key, c)
39
+ cache_clock_for(current_user, c)
46
40
  end if new_clock != clock
47
41
  else
48
42
  # anonymous users will have to wait until the slaves have caught up
@@ -51,13 +45,15 @@ def with_consistency_filter
51
45
  end
52
46
  ```
53
47
 
54
- Note that we use the current `master_clock` as a reference point. This will give the user a recent view of the data,
48
+ Note that we use the last seen consistency for a given user as reference point. This will give the user a recent view of the data,
55
49
  possibly reading from master, and if no write occurs inside the `with_consistency` block, we have a reasonable value to
56
- cache and reuse on subsequent requests. Alternatively, we could have used
57
- `ActiveRecord::ConnectionAdapters::MasterSlaveAdapter::Clock.zero` to indicate no particular consistency requirement.
50
+ cache and reuse on subsequent requests.
51
+ If no cached clock is available, this indicates that no particular consistency is required. Any slave connection will do.
58
52
  Since `with_consistency` blocks can be nested, the controller code could later decide to require a more recent view on
59
53
  the data.
60
54
 
55
+ _See also this [blog post][3] for a more detailed explanation._
56
+
61
57
  ### Manual Selection of Master or Slave
62
58
 
63
59
  The original functionality of the adapter has been preserved:
@@ -76,6 +72,24 @@ end
76
72
 
77
73
  `with_master`, `with_slave` as well as `with_consistency` can be nested deliberately.
78
74
 
75
+ ### Handles master unavailable scenarios gracefully
76
+
77
+ Due to scenarios when the master is possibly down (e.g., maintenance), we try
78
+ to delegate as much as possible to the active slaves. In order to accomplish
79
+ this we have added the following functionalities.
80
+
81
+ * We ignore errors while connecting to the master server.
82
+ * ActiveRecord::MasterUnavailable exceptions are raised in cases when we need to use
83
+ a master connection, but the server is unavailable. This exception is propagated
84
+ to the application.
85
+ * We have introduced the circuit breaker pattern in the master reconnect logic
86
+ to prevent excessive reconnection attempts. We block any queries which require
87
+ a master connection for a given timeout (by default, 30 seconds). After the
88
+ timeout has expired, any attempt of using the master connection will trigger
89
+ a reconnection.
90
+ * The master slave adapter is still usable for any queries that require only
91
+ slave connections.
92
+
79
93
  ### Transaction Callbacks
80
94
 
81
95
  This feature was originally developed at [SoundCloud][4] for the standard `MysqlAdapter`. It allows arbitrary blocks of
@@ -123,6 +137,24 @@ At [SoundCloud][4], we're using [database_cleaner][2]'s 'truncation strategy' to
123
137
  `truncate_table` as an `ActiveRecord::Base.connection` instance method. We might add other strategies if there's enough
124
138
  interest.
125
139
 
140
+ ## Requirements
141
+
142
+ MasterSlaveAdapter requires ActiveRecord with a version >= 2.3, is compatible
143
+ with at least Ruby 1.8.7, 1.9.2, 1.9.3 and comes with built-in support for mysql
144
+ and mysql2 libraries.
145
+
146
+ You can check the versions it's tested against at [Travis CI](http://travis-ci.org/#!/soundcloud/master_slave_adapter).
147
+
148
+ ## Installation
149
+
150
+ Using plain rubygems:
151
+
152
+ $ gem install master_slave_adapter
153
+
154
+ Using bundler, just include it in your Gemfile:
155
+
156
+ gem 'master_slave_adapter'
157
+
126
158
  ## Configuration
127
159
 
128
160
  Example configuration for the development environment in `database.yml`:
@@ -133,6 +165,7 @@ development:
133
165
  connection_adapter: mysql # actual adapter to use (only mysql is supported atm)
134
166
  disable_connection_test: false # when an instance is checked out from the connection pool,
135
167
  # we check if the connections are still alive, reconnecting if necessary
168
+
136
169
  # these values are picked up as defaults in the 'master' and 'slaves' sections:
137
170
  database: aweapp_development
138
171
  username: aweappuser
@@ -147,23 +180,6 @@ development:
147
180
  - host: slave02
148
181
  ```
149
182
 
150
- ## Installation
151
-
152
- Using plain rubygems:
153
-
154
- ```sh
155
- $ gem install master_slave_adapter_soundcloud
156
- ```
157
-
158
- Using bundler:
159
-
160
- ```sh
161
- $ cat >> Gemfile
162
- gem 'master_slave_adapter_soundcloud', '~> 0.1', :require => 'master_slave_adaper'
163
- ^D
164
- $ bundle install
165
- ```
166
-
167
183
  ## Credits
168
184
 
169
185
  * Maurício Lenhares - _original master_slave_adapter plugin_
@@ -171,6 +187,8 @@ $ bundle install
171
187
  * Sean Treadway - _chief everything & transaction callbacks_
172
188
  * Kim Altintop - _strong lax monoidal endofunctors_
173
189
  * Omid Aladini - _chief operator & everything else_
190
+ * Tiago Loureiro - _review expert & master unavailable handling_
191
+ * Tobias Schmidt - _typo master & activerecord ranter_
174
192
 
175
193
 
176
194
  [1]: https://github.com/mauricio/master_slave_adapter
@@ -30,7 +30,6 @@ module ActiveRecord
30
30
  @infinity ||= Clock.new('', Float::MAX.to_i)
31
31
  end
32
32
 
33
- # TODO: tests
34
33
  def self.parse(string)
35
34
  new(*string.split('@'))
36
35
  rescue
@@ -0,0 +1,43 @@
1
+ require 'active_record/connection_adapters/master_slave_adapter/clock'
2
+
3
+ module ActiveRecord
4
+ module ConnectionAdapters
5
+ module MasterSlaveAdapter
6
+ module SharedMysqlAdapterBehavior
7
+ def with_consistency(clock)
8
+ clock =
9
+ case clock
10
+ when Clock then clock
11
+ when String then Clock.parse(clock)
12
+ when nil then Clock.zero
13
+ end
14
+
15
+ super(clock)
16
+ end
17
+
18
+ def master_clock
19
+ conn = master_connection
20
+ if status = conn.uncached { select_hash(conn, "SHOW MASTER STATUS") }
21
+ Clock.new(status['File'], status['Position'])
22
+ else
23
+ Clock.infinity
24
+ end
25
+ rescue MasterUnavailable
26
+ Clock.zero
27
+ rescue ActiveRecordError
28
+ Clock.infinity
29
+ end
30
+
31
+ def slave_clock(conn)
32
+ if status = conn.uncached { select_hash(conn, "SHOW SLAVE STATUS") }
33
+ Clock.new(status['Relay_Master_Log_File'], status['Exec_Master_Log_Pos'])
34
+ else
35
+ Clock.zero
36
+ end
37
+ rescue ActiveRecordError
38
+ Clock.zero
39
+ end
40
+ end
41
+ end
42
+ end
43
+ end
@@ -1,7 +1,7 @@
1
1
  module ActiveRecord
2
2
  module ConnectionAdapters
3
3
  module MasterSlaveAdapter
4
- VERSION = "1.0.0.beta1"
4
+ VERSION = "1.0.0.beta2"
5
5
  end
6
6
  end
7
7
  end