redis_failover 0.8.0 → 0.8.1
Sign up to get free protection for your applications and to get access to all the features.
- data/.yardopts +6 -0
- data/Changes.md +6 -0
- data/README.md +4 -0
- data/lib/redis_failover/cli.rb +13 -0
- data/lib/redis_failover/client.rb +94 -70
- data/lib/redis_failover/errors.rb +9 -5
- data/lib/redis_failover/manual.rb +10 -0
- data/lib/redis_failover/node.rb +45 -1
- data/lib/redis_failover/node_manager.rb +56 -3
- data/lib/redis_failover/node_watcher.rb +15 -0
- data/lib/redis_failover/runner.rb +7 -1
- data/lib/redis_failover/util.rb +22 -0
- data/lib/redis_failover/version.rb +1 -1
- data/redis_failover.gemspec +2 -1
- data/spec/client_spec.rb +2 -1
- metadata +24 -6
data/.yardopts
ADDED
data/Changes.md
CHANGED
@@ -1,3 +1,9 @@
|
|
1
|
+
0.8.1
|
2
|
+
-----------
|
3
|
+
- Added YARD documentation.
|
4
|
+
- Improve ZooKeeper client connection management.
|
5
|
+
- Upgrade to latest ZK gem stable release.
|
6
|
+
|
1
7
|
0.8.0
|
2
8
|
-----------
|
3
9
|
- Added manual failover support (can be initiated via RedisFailover::Client#manual_failover)
|
data/README.md
CHANGED
@@ -131,6 +131,10 @@ server passed to #manual_failover, or it will pick a random slave to become the
|
|
131
131
|
client = RedisFailover::Client.new(:zkservers => 'localhost:2181,localhost:2182,localhost:2183')
|
132
132
|
client.manual_failover(:host => 'localhost', :port => 2222)
|
133
133
|
|
134
|
+
## Documentation
|
135
|
+
|
136
|
+
redis_failover uses YARD for its API documentation. Refer to the generated [API documentation](http://rubydoc.info/github/ryanlecompte/redis_failover/master/frames) for full coverage.
|
137
|
+
|
134
138
|
## Requirements
|
135
139
|
|
136
140
|
- redis_failover is actively tested against MRI 1.9.2/1.9.3 and JRuby 1.6.7 (1.9 mode only). Other rubies may work, although I don't actively test against them. 1.8 is not supported.
|
data/lib/redis_failover/cli.rb
CHANGED
@@ -1,6 +1,10 @@
|
|
1
1
|
module RedisFailover
|
2
2
|
# Parses server command-line arguments.
|
3
3
|
class CLI
|
4
|
+
# Parses the source of options.
|
5
|
+
#
|
6
|
+
# @param [Array] source the command-line options to parse
|
7
|
+
# @return [Hash] the parsed options
|
4
8
|
def self.parse(source)
|
5
9
|
options = {}
|
6
10
|
parser = OptionParser.new do |opts|
|
@@ -55,12 +59,17 @@ module RedisFailover
|
|
55
59
|
prepare(options)
|
56
60
|
end
|
57
61
|
|
62
|
+
# @return [Boolean] true if required options missing, false otherwise
|
58
63
|
def self.required_options_missing?(options)
|
59
64
|
return true if options.empty?
|
60
65
|
return true unless options.values_at(:nodes, :zkservers).all?
|
61
66
|
false
|
62
67
|
end
|
63
68
|
|
69
|
+
# Parses options from a YAML file.
|
70
|
+
#
|
71
|
+
# @param [String] file the filename
|
72
|
+
# @return [Hash] the parsed options
|
64
73
|
def self.from_file(file)
|
65
74
|
unless File.exists?(file)
|
66
75
|
raise ArgumentError, "File #{file} can't be found"
|
@@ -72,6 +81,10 @@ module RedisFailover
|
|
72
81
|
options
|
73
82
|
end
|
74
83
|
|
84
|
+
# Prepares the options for the rest of the system.
|
85
|
+
#
|
86
|
+
# @param [Hash] options the options to prepare
|
87
|
+
# @return [Hash] the prepared options
|
75
88
|
def self.prepare(options)
|
76
89
|
options.each_value { |v| v.strip! if v.respond_to?(:strip!) }
|
77
90
|
# turns 'host1:port,host2:port' => [{:host => host, :port => port}, ...]
|
@@ -8,8 +8,7 @@ module RedisFailover
|
|
8
8
|
# Redis clients appropriately. RedisFailover::Client also directs write operations to the master,
|
9
9
|
# and all read operations to the slaves.
|
10
10
|
#
|
11
|
-
#
|
12
|
-
#
|
11
|
+
# @example Usage
|
13
12
|
# client = RedisFailover::Client.new(:zkservers => 'localhost:2181,localhost:2182,localhost:2183')
|
14
13
|
# client.set('foo', 1) # will be directed to master
|
15
14
|
# client.get('foo') # will be directed to a slave
|
@@ -86,15 +85,16 @@ module RedisFailover
|
|
86
85
|
|
87
86
|
# Creates a new failover redis client.
|
88
87
|
#
|
89
|
-
#
|
90
|
-
#
|
91
|
-
#
|
92
|
-
#
|
93
|
-
#
|
94
|
-
#
|
95
|
-
#
|
96
|
-
#
|
97
|
-
#
|
88
|
+
# @param [Hash] options the options used to initialize the client instance
|
89
|
+
# @option options [String] :zkservers comma-separated ZooKeeper host:port pairs (required)
|
90
|
+
# @option options [String] :znode_path znode path override for redis server list
|
91
|
+
# @option options [String] :password password for redis nodes
|
92
|
+
# @option options [String] :db database to use for redis nodes
|
93
|
+
# @option options [String] :namespace namespace for redis nodes
|
94
|
+
# @option options [Logger] :logger logger override
|
95
|
+
# @option options [Boolean] :retry_failure indicates if failures should be retried
|
96
|
+
# @option options [Integer] :max_retries max retries for a failure
|
97
|
+
# @return [RedisFailover::Client]
|
98
98
|
def initialize(options = {})
|
99
99
|
Util.logger = options[:logger] if options[:logger]
|
100
100
|
@zkservers = options.fetch(:zkservers) { raise ArgumentError, ':zkservers required'}
|
@@ -108,7 +108,7 @@ module RedisFailover
|
|
108
108
|
@slaves = []
|
109
109
|
@queue = Queue.new
|
110
110
|
@lock = Monitor.new
|
111
|
-
|
111
|
+
setup_zk
|
112
112
|
build_clients
|
113
113
|
end
|
114
114
|
|
@@ -121,10 +121,14 @@ module RedisFailover
|
|
121
121
|
end
|
122
122
|
end
|
123
123
|
|
124
|
-
|
125
|
-
|
124
|
+
# Determines whether or not an unknown method can be handled.
|
125
|
+
#
|
126
|
+
# @return [Boolean] indicates if the method can be handled
|
127
|
+
def respond_to_missing?(method)
|
128
|
+
redis_operation?(method)
|
126
129
|
end
|
127
130
|
|
131
|
+
# @return [String] a string representation of the client
|
128
132
|
def inspect
|
129
133
|
"#<RedisFailover::Client (master: #{master_name}, slaves: #{slave_names})>"
|
130
134
|
end
|
@@ -134,9 +138,9 @@ module RedisFailover
|
|
134
138
|
# via options. If no options are passed, a random slave will be selected as
|
135
139
|
# the candidate for the new master.
|
136
140
|
#
|
137
|
-
#
|
138
|
-
#
|
139
|
-
#
|
141
|
+
# @param [Hash] options the options used for manual failover
|
142
|
+
# @option options [String] :host the host of the failover candidate
|
143
|
+
# @option options [String] :port the port of the failover candidate
|
140
144
|
def manual_failover(options = {})
|
141
145
|
Manual.failover(zk, options)
|
142
146
|
self
|
@@ -144,73 +148,45 @@ module RedisFailover
|
|
144
148
|
|
145
149
|
private
|
146
150
|
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
@
|
153
|
-
|
154
|
-
|
155
|
-
Proc === event ? event.call : handle_zk_event(event)
|
156
|
-
rescue => ex
|
157
|
-
logger.error("Error while handling event: #{ex.inspect}")
|
158
|
-
logger.error(ex.backtrace.join("\n"))
|
159
|
-
end
|
160
|
-
end
|
161
|
-
end
|
162
|
-
|
163
|
-
reconnect_zk
|
164
|
-
end
|
165
|
-
|
166
|
-
def handle_session_established
|
167
|
-
@lock.synchronize do
|
168
|
-
@zk.watcher.register(@znode) do |event|
|
169
|
-
@queue << event
|
170
|
-
end
|
171
|
-
@zk.on_expired_session do
|
172
|
-
@queue << proc { reconnect_zk }
|
173
|
-
end
|
174
|
-
@zk.event_handler.register_state_handler(:connecting) do
|
175
|
-
@queue << proc { handle_lost_connection }
|
176
|
-
end
|
177
|
-
@zk.on_connected do
|
178
|
-
@zk.stat(@znode, :watch => true)
|
179
|
-
end
|
180
|
-
@zk.stat(@znode, :watch => true)
|
181
|
-
end
|
151
|
+
# Sets up the underlying ZooKeeper connection.
|
152
|
+
def setup_zk
|
153
|
+
@zk = ZK.new(@zkservers)
|
154
|
+
@zk.watcher.register(@znode) { |event| handle_zk_event(event) }
|
155
|
+
@zk.on_expired_session { purge_clients }
|
156
|
+
@zk.on_connected { @zk.stat(@znode, :watch => true) }
|
157
|
+
@zk.stat(@znode, :watch => true)
|
158
|
+
update_znode_timestamp
|
182
159
|
end
|
183
160
|
|
161
|
+
# Handles a ZK event.
|
162
|
+
#
|
163
|
+
# @param [ZK::Event] event the ZK event to handle
|
184
164
|
def handle_zk_event(event)
|
185
165
|
update_znode_timestamp
|
186
166
|
if event.node_created? || event.node_changed?
|
187
167
|
build_clients
|
188
168
|
elsif event.node_deleted?
|
189
169
|
purge_clients
|
190
|
-
zk.stat(@znode, :watch => true)
|
170
|
+
@zk.stat(@znode, :watch => true)
|
191
171
|
else
|
192
172
|
logger.error("Unknown ZK node event: #{event.inspect}")
|
193
173
|
end
|
194
174
|
end
|
195
175
|
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
@zk = ZK.new(@zkservers)
|
201
|
-
handle_session_established
|
202
|
-
update_znode_timestamp
|
203
|
-
end
|
204
|
-
end
|
205
|
-
|
206
|
-
def handle_lost_connection
|
207
|
-
purge_clients
|
208
|
-
end
|
209
|
-
|
176
|
+
# Determines if a method is a known redis operation.
|
177
|
+
#
|
178
|
+
# @param [Symbol] method the method to check
|
179
|
+
# @return [Boolean] true if redis operation, false otherwise
|
210
180
|
def redis_operation?(method)
|
211
181
|
Redis.public_instance_methods(false).include?(method)
|
212
182
|
end
|
213
183
|
|
184
|
+
# Dispatches a redis operation to a master or slave.
|
185
|
+
#
|
186
|
+
# @param [Symbol] method the method to dispatch
|
187
|
+
# @param [Array] args the arguments to pass to the method
|
188
|
+
# @param [Proc] block an optional block to pass to the method
|
189
|
+
# @return [Object] the result of dispatching the command
|
214
190
|
def dispatch(method, *args, &block)
|
215
191
|
unless recently_heard_from_node_manager?
|
216
192
|
@lock.synchronize do
|
@@ -243,6 +219,10 @@ module RedisFailover
|
|
243
219
|
end
|
244
220
|
end
|
245
221
|
|
222
|
+
# Returns the currently known master.
|
223
|
+
#
|
224
|
+
# @return [Redis] the Redis client for the current master
|
225
|
+
# @raise [NoMasterError] if no master is available
|
246
226
|
def master
|
247
227
|
if master = @lock.synchronize { @master }
|
248
228
|
verify_role!(master, :master)
|
@@ -251,6 +231,11 @@ module RedisFailover
|
|
251
231
|
raise NoMasterError
|
252
232
|
end
|
253
233
|
|
234
|
+
# Returns a random slave from the list of known slaves.
|
235
|
+
#
|
236
|
+
# @note If there are no slaves, the master is returned.
|
237
|
+
# @return [Redis] the Redis client for the slave or master
|
238
|
+
# @raise [NoMasterError] if no master fallback is available
|
254
239
|
def slave
|
255
240
|
# pick a slave, if none available fallback to master
|
256
241
|
if slave = @lock.synchronize { @slaves.sample }
|
@@ -260,6 +245,8 @@ module RedisFailover
|
|
260
245
|
master
|
261
246
|
end
|
262
247
|
|
248
|
+
# Builds the Redis clients for the currently known master/slaves.
|
249
|
+
# The current master/slaves are fetched via ZooKeeper.
|
263
250
|
def build_clients
|
264
251
|
@lock.synchronize do
|
265
252
|
retried = false
|
@@ -289,14 +276,21 @@ module RedisFailover
|
|
289
276
|
end
|
290
277
|
end
|
291
278
|
|
279
|
+
# Fetches the known redis nodes from ZooKeeper.
|
280
|
+
#
|
281
|
+
# @return [Hash] the known master/slave redis servers
|
292
282
|
def fetch_nodes
|
293
|
-
data = zk.get(@znode, :watch => true).first
|
283
|
+
data = @zk.get(@znode, :watch => true).first
|
294
284
|
nodes = symbolize_keys(decode(data))
|
295
285
|
logger.debug("Fetched nodes: #{nodes}")
|
296
286
|
|
297
287
|
nodes
|
298
288
|
end
|
299
289
|
|
290
|
+
# Builds new Redis clients for the specified nodes.
|
291
|
+
#
|
292
|
+
# @param [Array<String>] nodes the array of redis host:port pairs
|
293
|
+
# @return [Array<Redis>] the array of corresponding Redis clients
|
300
294
|
def new_clients_for(*nodes)
|
301
295
|
nodes.map do |node|
|
302
296
|
host, port = node.split(':')
|
@@ -311,15 +305,23 @@ module RedisFailover
|
|
311
305
|
end
|
312
306
|
end
|
313
307
|
|
308
|
+
# @return [String] a friendly name for current master
|
314
309
|
def master_name
|
315
310
|
address_for(@master) || 'none'
|
316
311
|
end
|
317
312
|
|
313
|
+
# @return [Array<String>] friendly names for current slaves
|
318
314
|
def slave_names
|
319
315
|
return 'none' if @slaves.empty?
|
320
316
|
addresses_for(@slaves).join(', ')
|
321
317
|
end
|
322
318
|
|
319
|
+
# Verifies the actual role for a redis node.
|
320
|
+
#
|
321
|
+
# @param [Redis] node the redis node to check
|
322
|
+
# @param [Symbol] role the role to verify
|
323
|
+
# @return [Symbol] the verified role
|
324
|
+
# @raise [InvalidNodeRoleError] if the role is invalid
|
323
325
|
def verify_role!(node, role)
|
324
326
|
current_role = node.info['role']
|
325
327
|
if current_role.to_sym != role
|
@@ -328,29 +330,48 @@ module RedisFailover
|
|
328
330
|
role
|
329
331
|
end
|
330
332
|
|
333
|
+
# Ensures that the method is supported.
|
334
|
+
#
|
335
|
+
# @raise [UnsupportedOperationError] if the operation isn't supported
|
331
336
|
def verify_supported!(method)
|
332
337
|
if UNSUPPORTED_OPS.include?(method)
|
333
338
|
raise UnsupportedOperationError.new(method)
|
334
339
|
end
|
335
340
|
end
|
336
341
|
|
342
|
+
# Returns node addresses.
|
343
|
+
#
|
344
|
+
# @param [Array<Redis>] nodes the redis clients
|
345
|
+
# @return [Array<String>] the addresses for the nodes
|
337
346
|
def addresses_for(nodes)
|
338
347
|
nodes.map { |node| address_for(node) }
|
339
348
|
end
|
340
349
|
|
350
|
+
# Returns a node address.
|
351
|
+
#
|
352
|
+
# @param [Redis] node a redis client
|
353
|
+
# @return [String] the address for the node
|
341
354
|
def address_for(node)
|
342
355
|
return unless node
|
343
356
|
"#{node.client.host}:#{node.client.port}"
|
344
357
|
end
|
345
358
|
|
359
|
+
# Determines if the currently known redis servers is different
|
360
|
+
# from the nodes returned by ZooKeeper.
|
361
|
+
#
|
362
|
+
# @param [Array<String>] new_nodes the new redis nodes
|
363
|
+
# @return [Boolean] true if nodes are different, false otherwise
|
346
364
|
def nodes_changed?(new_nodes)
|
347
365
|
return true if address_for(@master) != new_nodes[:master]
|
348
366
|
return true if different?(addresses_for(@slaves), new_nodes[:slaves])
|
349
367
|
false
|
350
368
|
end
|
351
369
|
|
352
|
-
|
353
|
-
|
370
|
+
# Disconnects one or more redis clients.
|
371
|
+
#
|
372
|
+
# @param [Array<Redis>] redis_clients the redis clients
|
373
|
+
def disconnect(*redis_clients)
|
374
|
+
redis_clients.each do |conn|
|
354
375
|
if conn
|
355
376
|
begin
|
356
377
|
conn.client.disconnect
|
@@ -361,6 +382,7 @@ module RedisFailover
|
|
361
382
|
end
|
362
383
|
end
|
363
384
|
|
385
|
+
# Disconnects current redis clients and resets this client's view of the world.
|
364
386
|
def purge_clients
|
365
387
|
@lock.synchronize do
|
366
388
|
logger.info("Purging current redis clients")
|
@@ -370,10 +392,12 @@ module RedisFailover
|
|
370
392
|
end
|
371
393
|
end
|
372
394
|
|
395
|
+
# Updates timestamp when an event is received by the Node Manager.
|
373
396
|
def update_znode_timestamp
|
374
397
|
@last_znode_timestamp = Time.now
|
375
398
|
end
|
376
399
|
|
400
|
+
# @return [Boolean] indicates if we recently heard from the Node Manager
|
377
401
|
def recently_heard_from_node_manager?
|
378
402
|
return false unless @last_znode_timestamp
|
379
403
|
Time.now - @last_znode_timestamp <= ZNODE_UPDATE_TIMEOUT
|
@@ -1,33 +1,36 @@
|
|
1
1
|
module RedisFailover
|
2
|
+
# Base class for all RedisFailover errors.
|
2
3
|
class Error < StandardError
|
3
|
-
attr_reader :original
|
4
|
-
def initialize(msg = nil, original = $!)
|
5
|
-
super(msg)
|
6
|
-
@original = original
|
7
|
-
end
|
8
4
|
end
|
9
5
|
|
6
|
+
# Raised when a node is specified incorrectly.
|
10
7
|
class InvalidNodeError < Error
|
11
8
|
end
|
12
9
|
|
10
|
+
# Raised when a node changes to an invalid/unknown state.
|
13
11
|
class InvalidNodeStateError < Error
|
14
12
|
def initialize(node, state)
|
15
13
|
super("Invalid state change `#{state}` for node #{node}")
|
16
14
|
end
|
17
15
|
end
|
18
16
|
|
17
|
+
# Raised when a node is unavailable (i.e., unreachable via network).
|
19
18
|
class NodeUnavailableError < Error
|
20
19
|
def initialize(node)
|
21
20
|
super("Node: #{node}")
|
22
21
|
end
|
23
22
|
end
|
24
23
|
|
24
|
+
# Raised when no master is currently available.
|
25
25
|
class NoMasterError < Error
|
26
26
|
end
|
27
27
|
|
28
|
+
# Raised when no slave is currently available.
|
28
29
|
class NoSlaveError < Error
|
29
30
|
end
|
30
31
|
|
32
|
+
# Raised when a redis server is no longer using the same role
|
33
|
+
# as previously assumed.
|
31
34
|
class InvalidNodeRoleError < Error
|
32
35
|
def initialize(node, assumed, actual)
|
33
36
|
super("Invalid role detected for node #{node}, client thought " +
|
@@ -35,6 +38,7 @@ module RedisFailover
|
|
35
38
|
end
|
36
39
|
end
|
37
40
|
|
41
|
+
# Raised when an unsupported redis operation is performed.
|
38
42
|
class UnsupportedOperationError < Error
|
39
43
|
def initialize(operation)
|
40
44
|
super("Operation `#{operation}` is currently unsupported")
|
@@ -9,6 +9,13 @@ module RedisFailover
|
|
9
9
|
# Denotes that any slave can be used as a candidate for promotion.
|
10
10
|
ANY_SLAVE = "ANY_SLAVE".freeze
|
11
11
|
|
12
|
+
# Performs a manual failover. If options is empty, a random slave will
|
13
|
+
# be used as a failover candidate.
|
14
|
+
#
|
15
|
+
# @param [ZK] zk the ZooKeeper client
|
16
|
+
# @param [Hash] options the options used for manual failover
|
17
|
+
# @option options [String] :host the host of the failover candidate
|
18
|
+
# @option options [String] :port the port of the failover candidate
|
12
19
|
def failover(zk, options = {})
|
13
20
|
create_path(zk)
|
14
21
|
node = options.empty? ? ANY_SLAVE : "#{options[:host]}:#{options[:port]}"
|
@@ -17,6 +24,9 @@ module RedisFailover
|
|
17
24
|
|
18
25
|
private
|
19
26
|
|
27
|
+
# Creates the znode path used for coordinating manual failovers.
|
28
|
+
#
|
29
|
+
# @param [ZK] zk the ZooKeeper cilent
|
20
30
|
def create_path(zk)
|
21
31
|
zk.create(ZNODE_PATH)
|
22
32
|
rescue ZK::Exceptions::NodeExists
|
data/lib/redis_failover/node.rb
CHANGED
@@ -10,26 +10,44 @@ module RedisFailover
|
|
10
10
|
# NodeUnavailableError will be raised.
|
11
11
|
MAX_OP_WAIT_TIME = 5
|
12
12
|
|
13
|
-
|
13
|
+
# @return [String] the redis server host
|
14
|
+
attr_reader :host
|
14
15
|
|
16
|
+
# @return [Integer] the redis server port
|
17
|
+
attr_reader :port
|
18
|
+
|
19
|
+
# Creates a new instance.
|
20
|
+
#
|
21
|
+
# @param [Hash] options the options used to create the node
|
22
|
+
# @option options [String] :host the host of the redis server
|
23
|
+
# @option options [String] :port the port of the redis server
|
15
24
|
def initialize(options = {})
|
16
25
|
@host = options.fetch(:host) { raise InvalidNodeError, 'missing host'}
|
17
26
|
@port = Integer(options[:port] || 6379)
|
18
27
|
@password = options[:password]
|
19
28
|
end
|
20
29
|
|
30
|
+
# @return [Boolean] true if this node is a master, false otherwise
|
21
31
|
def master?
|
22
32
|
role == 'master'
|
23
33
|
end
|
24
34
|
|
35
|
+
# @return [Boolean] true if this node is a slave, false otherwise
|
25
36
|
def slave?
|
26
37
|
!master?
|
27
38
|
end
|
28
39
|
|
40
|
+
# Determines if this node is a slave of the given master.
|
41
|
+
#
|
42
|
+
# @param [Node] master the master to check
|
43
|
+
# @return [Boolean] true if slave of master, false otherwise
|
29
44
|
def slave_of?(master)
|
30
45
|
current_master == master
|
31
46
|
end
|
32
47
|
|
48
|
+
# Determines current master of this slave.
|
49
|
+
#
|
50
|
+
# @return [Node] the node representing the master of this slave
|
33
51
|
def current_master
|
34
52
|
info = fetch_info
|
35
53
|
return unless info[:role] == 'slave'
|
@@ -47,12 +65,17 @@ module RedisFailover
|
|
47
65
|
end
|
48
66
|
end
|
49
67
|
|
68
|
+
# Wakes up this node by pushing a value to its internal
|
69
|
+
# queue used by #wait.
|
50
70
|
def wakeup
|
51
71
|
perform_operation do |redis|
|
52
72
|
redis.lpush(wait_key, '1')
|
53
73
|
end
|
54
74
|
end
|
55
75
|
|
76
|
+
# Makes this node a slave of the given node.
|
77
|
+
#
|
78
|
+
# @param [Node] node the node of which to become a slave
|
56
79
|
def make_slave!(node)
|
57
80
|
perform_operation do |redis|
|
58
81
|
unless slave_of?(node)
|
@@ -63,6 +86,7 @@ module RedisFailover
|
|
63
86
|
end
|
64
87
|
end
|
65
88
|
|
89
|
+
# Makes this node a master node.
|
66
90
|
def make_master!
|
67
91
|
perform_operation do |redis|
|
68
92
|
unless master?
|
@@ -73,14 +97,20 @@ module RedisFailover
|
|
73
97
|
end
|
74
98
|
end
|
75
99
|
|
100
|
+
# @return [String] an inspect string for this node
|
76
101
|
def inspect
|
77
102
|
"<RedisFailover::Node #{to_s}>"
|
78
103
|
end
|
79
104
|
|
105
|
+
# @return [String] a friendly string for this node
|
80
106
|
def to_s
|
81
107
|
"#{@host}:#{@port}"
|
82
108
|
end
|
83
109
|
|
110
|
+
# Determines if this node is equal to another node.
|
111
|
+
#
|
112
|
+
# @param [Node] other the other node to compare
|
113
|
+
# @return [Boolean] true if equal, false otherwise
|
84
114
|
def ==(other)
|
85
115
|
return false unless Node === other
|
86
116
|
return true if self.equal?(other)
|
@@ -88,10 +118,15 @@ module RedisFailover
|
|
88
118
|
end
|
89
119
|
alias_method :eql?, :==
|
90
120
|
|
121
|
+
|
122
|
+
# @return [Integer] a hash value for this node
|
91
123
|
def hash
|
92
124
|
to_s.hash
|
93
125
|
end
|
94
126
|
|
127
|
+
# Fetches information/stats for this node.
|
128
|
+
#
|
129
|
+
# @return [Hash] the info for this node
|
95
130
|
def fetch_info
|
96
131
|
perform_operation do |redis|
|
97
132
|
symbolize_keys(redis.info)
|
@@ -99,12 +134,14 @@ module RedisFailover
|
|
99
134
|
end
|
100
135
|
alias_method :ping, :fetch_info
|
101
136
|
|
137
|
+
# @return [Boolean] determines if this node prohibits stale reads
|
102
138
|
def prohibits_stale_reads?
|
103
139
|
perform_operation do |redis|
|
104
140
|
redis.config('get', 'slave-serve-stale-data').last == 'no'
|
105
141
|
end
|
106
142
|
end
|
107
143
|
|
144
|
+
# @return [Boolean] determines if this node is syncing with its master
|
108
145
|
def syncing_with_master?
|
109
146
|
perform_operation do |redis|
|
110
147
|
fetch_info[:master_sync_in_progress] == '1'
|
@@ -113,18 +150,25 @@ module RedisFailover
|
|
113
150
|
|
114
151
|
private
|
115
152
|
|
153
|
+
# @return [String] the current role for this node
|
116
154
|
def role
|
117
155
|
fetch_info[:role]
|
118
156
|
end
|
119
157
|
|
158
|
+
# @return [String] the name of the wait queue for this node
|
120
159
|
def wait_key
|
121
160
|
@wait_key ||= "_redis_failover_#{SecureRandom.hex(32)}"
|
122
161
|
end
|
123
162
|
|
163
|
+
# @return [Redis] a new redis client instance for this node
|
124
164
|
def new_client
|
125
165
|
Redis.new(:host => @host, :password => @password, :port => @port)
|
126
166
|
end
|
127
167
|
|
168
|
+
# Safely performs a redis operation within a given timeout window.
|
169
|
+
#
|
170
|
+
# @yield [Redis] the redis client to use for the operation
|
171
|
+
# @raise [NodeUnavailableError] if node is currently unreachable
|
128
172
|
def perform_operation
|
129
173
|
redis = nil
|
130
174
|
Timeout.timeout(MAX_OP_WAIT_TIME) do
|
@@ -20,6 +20,14 @@ module RedisFailover
|
|
20
20
|
# Number of seconds to wait before retrying bootstrap process.
|
21
21
|
TIMEOUT = 5
|
22
22
|
|
23
|
+
# Creates a new instance.
|
24
|
+
#
|
25
|
+
# @param [Hash] options the options used to initialize the manager
|
26
|
+
# @option options [String] :zkservers comma-separated ZooKeeper host:port pairs (required)
|
27
|
+
# @option options [String] :znode_path znode path override for redis server list
|
28
|
+
# @option options [String] :password password for redis nodes
|
29
|
+
# @option options [Array<String>] :nodes the nodes to manage
|
30
|
+
# @option options [String] :max_failures the max failures for a particular node
|
23
31
|
def initialize(options)
|
24
32
|
logger.info("Redis Node Manager v#{VERSION} starting (#{RUBY_DESCRIPTION})")
|
25
33
|
@options = options
|
@@ -28,6 +36,9 @@ module RedisFailover
|
|
28
36
|
@mutex = Mutex.new
|
29
37
|
end
|
30
38
|
|
39
|
+
# Starts the node manager.
|
40
|
+
#
|
41
|
+
# @note This is a blocking method, it does not return until the manager terminates.
|
31
42
|
def start
|
32
43
|
@queue = Queue.new
|
33
44
|
@leader = false
|
@@ -49,10 +60,16 @@ module RedisFailover
|
|
49
60
|
retry
|
50
61
|
end
|
51
62
|
|
63
|
+
# Notifies the manager of a state change. Used primarily by {RedisFailover::NodeWatcher}
|
64
|
+
# to inform the manager of watched node states.
|
65
|
+
#
|
66
|
+
# @param [Node] node the node
|
67
|
+
# @param [Symbol] state the state
|
52
68
|
def notify_state(node, state)
|
53
69
|
@queue << [node, state]
|
54
70
|
end
|
55
71
|
|
72
|
+
# Performs a graceful shutdown of the manager.
|
56
73
|
def shutdown
|
57
74
|
@queue.clear
|
58
75
|
@queue << nil
|
@@ -62,6 +79,7 @@ module RedisFailover
|
|
62
79
|
|
63
80
|
private
|
64
81
|
|
82
|
+
# Configures the ZooKeeper client.
|
65
83
|
def setup_zk
|
66
84
|
@zk.close! if @zk
|
67
85
|
@zk = ZK.new(@options[:zkservers])
|
@@ -74,12 +92,11 @@ module RedisFailover
|
|
74
92
|
end
|
75
93
|
end
|
76
94
|
|
77
|
-
@zk.on_connected
|
78
|
-
@zk.stat(@manual_znode, :watch => true)
|
79
|
-
end
|
95
|
+
@zk.on_connected { @zk.stat(@manual_znode, :watch => true) }
|
80
96
|
@zk.stat(@manual_znode, :watch => true)
|
81
97
|
end
|
82
98
|
|
99
|
+
# Handles periodic state reports from {RedisFailover::NodeWatcher} instances.
|
83
100
|
def handle_state_reports
|
84
101
|
while state_report = @queue.pop
|
85
102
|
begin
|
@@ -104,6 +121,9 @@ module RedisFailover
|
|
104
121
|
end
|
105
122
|
end
|
106
123
|
|
124
|
+
# Handles an unavailable node.
|
125
|
+
#
|
126
|
+
# @param [Node] node the unavailable node
|
107
127
|
def handle_unavailable(node)
|
108
128
|
# no-op if we already know about this node
|
109
129
|
return if @unavailable.include?(node)
|
@@ -119,6 +139,9 @@ module RedisFailover
|
|
119
139
|
end
|
120
140
|
end
|
121
141
|
|
142
|
+
# Handles an available node.
|
143
|
+
#
|
144
|
+
# @param [Node] node the available node
|
122
145
|
def handle_available(node)
|
123
146
|
reconcile(node)
|
124
147
|
|
@@ -138,6 +161,9 @@ module RedisFailover
|
|
138
161
|
@unavailable.delete(node)
|
139
162
|
end
|
140
163
|
|
164
|
+
# Handles a node that is currently syncing.
|
165
|
+
#
|
166
|
+
# @param [Node] node the syncing node
|
141
167
|
def handle_syncing(node)
|
142
168
|
reconcile(node)
|
143
169
|
|
@@ -151,6 +177,9 @@ module RedisFailover
|
|
151
177
|
handle_available(node)
|
152
178
|
end
|
153
179
|
|
180
|
+
# Handles a manual failover request to the given node.
|
181
|
+
#
|
182
|
+
# @param [Node] node the candidate node for failover
|
154
183
|
def handle_manual_failover(node)
|
155
184
|
# no-op if node to be failed over is already master
|
156
185
|
return if @master == node
|
@@ -162,6 +191,10 @@ module RedisFailover
|
|
162
191
|
promote_new_master(node)
|
163
192
|
end
|
164
193
|
|
194
|
+
# Promotes a new master.
|
195
|
+
#
|
196
|
+
# @param [Node] node the optional node to promote
|
197
|
+
# @note if no node is specified, a random slave will be used
|
165
198
|
def promote_new_master(node = nil)
|
166
199
|
delete_path
|
167
200
|
@master = nil
|
@@ -182,6 +215,7 @@ module RedisFailover
|
|
182
215
|
logger.info("Successfully promoted #{candidate} to master.")
|
183
216
|
end
|
184
217
|
|
218
|
+
# Discovers the current master and slave nodes.
|
185
219
|
def discover_nodes
|
186
220
|
@unavailable = []
|
187
221
|
nodes = @options[:nodes].map { |opts| Node.new(opts) }.uniq
|
@@ -194,6 +228,7 @@ module RedisFailover
|
|
194
228
|
redirect_slaves_to(@master)
|
195
229
|
end
|
196
230
|
|
231
|
+
# Spawns the {RedisFailover::NodeWatcher} instances for each managed node.
|
197
232
|
def spawn_watchers
|
198
233
|
@watchers = [@master, @slaves, @unavailable].flatten.map do |node|
|
199
234
|
NodeWatcher.new(self, node, @options[:max_failures] || 3)
|
@@ -201,6 +236,10 @@ module RedisFailover
|
|
201
236
|
@watchers.each(&:watch)
|
202
237
|
end
|
203
238
|
|
239
|
+
# Searches for the master node.
|
240
|
+
#
|
241
|
+
# @param [Array<Node>] nodes the nodes to search
|
242
|
+
# @return [Node] the found master node, nil if not found
|
204
243
|
def find_master(nodes)
|
205
244
|
nodes.find do |node|
|
206
245
|
begin
|
@@ -211,6 +250,9 @@ module RedisFailover
|
|
211
250
|
end
|
212
251
|
end
|
213
252
|
|
253
|
+
# Redirects all slaves to the specified node.
|
254
|
+
#
|
255
|
+
# @param [Node] node the node to which slaves are redirected
|
214
256
|
def redirect_slaves_to(node)
|
215
257
|
@slaves.dup.each do |slave|
|
216
258
|
begin
|
@@ -222,6 +264,9 @@ module RedisFailover
|
|
222
264
|
end
|
223
265
|
end
|
224
266
|
|
267
|
+
# Forces a slave to be marked as unavailable.
|
268
|
+
#
|
269
|
+
# @param [Node] node the node to force as unavailable
|
225
270
|
def force_unavailable_slave(node)
|
226
271
|
@slaves.delete(node)
|
227
272
|
@unavailable << node unless @unavailable.include?(node)
|
@@ -231,6 +276,8 @@ module RedisFailover
|
|
231
276
|
# and completely lost its dynamically set run-time role by the node
|
232
277
|
# manager. This method ensures that the node resumes its role as
|
233
278
|
# determined by the manager.
|
279
|
+
#
|
280
|
+
# @param [Node] node the node to reconcile
|
234
281
|
def reconcile(node)
|
235
282
|
return if @master == node && node.master?
|
236
283
|
return if @master && node.slave_of?(@master)
|
@@ -248,6 +295,7 @@ module RedisFailover
|
|
248
295
|
end
|
249
296
|
end
|
250
297
|
|
298
|
+
# @return [Hash] the set of current nodes grouped by category
|
251
299
|
def current_nodes
|
252
300
|
{
|
253
301
|
:master => @master ? @master.to_s : nil,
|
@@ -256,6 +304,7 @@ module RedisFailover
|
|
256
304
|
}
|
257
305
|
end
|
258
306
|
|
307
|
+
# Deletes the znode path containing the redis nodes.
|
259
308
|
def delete_path
|
260
309
|
@zk.delete(@znode)
|
261
310
|
logger.info("Deleted ZooKeeper node #{@znode}")
|
@@ -263,6 +312,7 @@ module RedisFailover
|
|
263
312
|
logger.info("Tried to delete missing znode: #{ex.inspect}")
|
264
313
|
end
|
265
314
|
|
315
|
+
# Creates the znode path containing the redis nodes.
|
266
316
|
def create_path
|
267
317
|
@zk.create(@znode, encode(current_nodes), :ephemeral => true)
|
268
318
|
logger.info("Created ZooKeeper node #{@znode}")
|
@@ -270,16 +320,19 @@ module RedisFailover
|
|
270
320
|
# best effort
|
271
321
|
end
|
272
322
|
|
323
|
+
# Initializes the znode path containing the redis nodes.
|
273
324
|
def initialize_path
|
274
325
|
create_path
|
275
326
|
write_state
|
276
327
|
end
|
277
328
|
|
329
|
+
# Writes the current redis nodes state to the znode path.
|
278
330
|
def write_state
|
279
331
|
create_path
|
280
332
|
@zk.set(@znode, encode(current_nodes))
|
281
333
|
end
|
282
334
|
|
335
|
+
# Schedules a manual failover to a redis node.
|
283
336
|
def schedule_manual_failover
|
284
337
|
return unless @leader
|
285
338
|
new_master = @zk.get(@manual_znode, :watch => true).first
|
@@ -8,6 +8,11 @@ module RedisFailover
|
|
8
8
|
# Time to sleep before checking on the monitored node's status.
|
9
9
|
WATCHER_SLEEP_TIME = 2
|
10
10
|
|
11
|
+
# Creates a new instance.
|
12
|
+
#
|
13
|
+
# @param [NodeManager] manager the node manager
|
14
|
+
# @param [Node] node the node to watch
|
15
|
+
# @param [Integer] max_failures the max failues before reporting node as down
|
11
16
|
def initialize(manager, node, max_failures)
|
12
17
|
@manager = manager
|
13
18
|
@node = node
|
@@ -16,11 +21,16 @@ module RedisFailover
|
|
16
21
|
@done = false
|
17
22
|
end
|
18
23
|
|
24
|
+
# Starts the node watcher.
|
25
|
+
#
|
26
|
+
# @note this method returns immediately and causes monitoring to be
|
27
|
+
# performed in a new background thread
|
19
28
|
def watch
|
20
29
|
@monitor_thread ||= Thread.new { monitor_node }
|
21
30
|
self
|
22
31
|
end
|
23
32
|
|
33
|
+
# Performs a graceful shutdown of this watcher.
|
24
34
|
def shutdown
|
25
35
|
@done = true
|
26
36
|
@node.wakeup
|
@@ -31,6 +41,8 @@ module RedisFailover
|
|
31
41
|
|
32
42
|
private
|
33
43
|
|
44
|
+
# Periodically monitors the redis node and reports state changes to
|
45
|
+
# the {RedisFailover::NodeManager}.
|
34
46
|
def monitor_node
|
35
47
|
failures = 0
|
36
48
|
|
@@ -57,6 +69,9 @@ module RedisFailover
|
|
57
69
|
end
|
58
70
|
end
|
59
71
|
|
72
|
+
# Notifies the manager of a node's state.
|
73
|
+
#
|
74
|
+
# @param [Symbol] state the node's state
|
60
75
|
def notify(state)
|
61
76
|
@manager.notify_state(@node, state)
|
62
77
|
end
|
@@ -1,6 +1,11 @@
|
|
1
1
|
module RedisFailover
|
2
|
-
# Runner is responsible for bootstrapping the
|
2
|
+
# Runner is responsible for bootstrapping the Node Manager.
|
3
3
|
class Runner
|
4
|
+
# Launches the Node Manager in a background thread.
|
5
|
+
#
|
6
|
+
# @param [Array] options the command-line options
|
7
|
+
# @note this method blocks and does not return until the
|
8
|
+
# Node Manager is gracefully stopped
|
4
9
|
def self.run(options)
|
5
10
|
options = CLI.parse(options)
|
6
11
|
@node_manager = NodeManager.new(options)
|
@@ -9,6 +14,7 @@ module RedisFailover
|
|
9
14
|
node_manager_thread.join
|
10
15
|
end
|
11
16
|
|
17
|
+
# Traps shutdown signals.
|
12
18
|
def self.trap_signals
|
13
19
|
[:INT, :TERM].each do |signal|
|
14
20
|
trap(signal) do
|
data/lib/redis_failover/util.rb
CHANGED
@@ -18,14 +18,24 @@ module RedisFailover
|
|
18
18
|
REDIS_ERRORS
|
19
19
|
].flatten.freeze
|
20
20
|
|
21
|
+
# Symbolizes the keys of the specified hash.
|
22
|
+
#
|
23
|
+
# @param [Hash] hash a hash for which keys should be symbolized
|
24
|
+
# @return [Hash] a new hash with symbolized keys
|
21
25
|
def symbolize_keys(hash)
|
22
26
|
Hash[hash.map { |k, v| [k.to_sym, v] }]
|
23
27
|
end
|
24
28
|
|
29
|
+
# Determines if two arrays are different.
|
30
|
+
#
|
31
|
+
# @param [Array] ary_a the first array
|
32
|
+
# @param [Array] ary_b the second array
|
33
|
+
# @return [Boolean] true if arrays are different, false otherwise
|
25
34
|
def different?(ary_a, ary_b)
|
26
35
|
((ary_a | ary_b) - (ary_a & ary_b)).size > 0
|
27
36
|
end
|
28
37
|
|
38
|
+
# @return [Logger] the logger instance to use
|
29
39
|
def self.logger
|
30
40
|
@logger ||= begin
|
31
41
|
logger = Logger.new(STDOUT)
|
@@ -37,18 +47,30 @@ module RedisFailover
|
|
37
47
|
end
|
38
48
|
end
|
39
49
|
|
50
|
+
# Sets a new logger to use.
|
51
|
+
#
|
52
|
+
# @param [Logger] logger a new logger to use
|
40
53
|
def self.logger=(logger)
|
41
54
|
@logger = logger
|
42
55
|
end
|
43
56
|
|
57
|
+
# @return [Logger] the logger instance to use
|
44
58
|
def logger
|
45
59
|
Util.logger
|
46
60
|
end
|
47
61
|
|
62
|
+
# Encodes the specified data in JSON format.
|
63
|
+
#
|
64
|
+
# @param [Object] data the data to encode
|
65
|
+
# @return [String] the JSON-encoded data
|
48
66
|
def encode(data)
|
49
67
|
MultiJson.encode(data)
|
50
68
|
end
|
51
69
|
|
70
|
+
# Decodes the specified JSON data.
|
71
|
+
#
|
72
|
+
# @param [String] data the JSON data to decode
|
73
|
+
# @return [Object] the decoded data
|
52
74
|
def decode(data)
|
53
75
|
return unless data
|
54
76
|
MultiJson.decode(data)
|
data/redis_failover.gemspec
CHANGED
@@ -18,8 +18,9 @@ Gem::Specification.new do |gem|
|
|
18
18
|
gem.add_dependency('redis')
|
19
19
|
gem.add_dependency('redis-namespace')
|
20
20
|
gem.add_dependency('multi_json', '~> 1')
|
21
|
-
gem.add_dependency('zk', '~> 1.
|
21
|
+
gem.add_dependency('zk', '~> 1.1')
|
22
22
|
|
23
23
|
gem.add_development_dependency('rake')
|
24
24
|
gem.add_development_dependency('rspec')
|
25
|
+
gem.add_development_dependency('yard')
|
25
26
|
end
|
data/spec/client_spec.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: redis_failover
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.8.
|
4
|
+
version: 0.8.1
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2012-05-
|
12
|
+
date: 2012-05-07 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: redis
|
@@ -66,7 +66,7 @@ dependencies:
|
|
66
66
|
requirements:
|
67
67
|
- - ~>
|
68
68
|
- !ruby/object:Gem::Version
|
69
|
-
version: '1.
|
69
|
+
version: '1.1'
|
70
70
|
type: :runtime
|
71
71
|
prerelease: false
|
72
72
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -74,7 +74,7 @@ dependencies:
|
|
74
74
|
requirements:
|
75
75
|
- - ~>
|
76
76
|
- !ruby/object:Gem::Version
|
77
|
-
version: '1.
|
77
|
+
version: '1.1'
|
78
78
|
- !ruby/object:Gem::Dependency
|
79
79
|
name: rake
|
80
80
|
requirement: !ruby/object:Gem::Requirement
|
@@ -107,6 +107,22 @@ dependencies:
|
|
107
107
|
- - ! '>='
|
108
108
|
- !ruby/object:Gem::Version
|
109
109
|
version: '0'
|
110
|
+
- !ruby/object:Gem::Dependency
|
111
|
+
name: yard
|
112
|
+
requirement: !ruby/object:Gem::Requirement
|
113
|
+
none: false
|
114
|
+
requirements:
|
115
|
+
- - ! '>='
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '0'
|
118
|
+
type: :development
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
none: false
|
122
|
+
requirements:
|
123
|
+
- - ! '>='
|
124
|
+
- !ruby/object:Gem::Version
|
125
|
+
version: '0'
|
110
126
|
description: Redis Failover is a ZooKeeper-based automatic master/slave failover solution
|
111
127
|
for Ruby
|
112
128
|
email:
|
@@ -118,6 +134,7 @@ extra_rdoc_files: []
|
|
118
134
|
files:
|
119
135
|
- .gitignore
|
120
136
|
- .travis.yml
|
137
|
+
- .yardopts
|
121
138
|
- Changes.md
|
122
139
|
- Gemfile
|
123
140
|
- LICENSE
|
@@ -160,7 +177,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
160
177
|
version: '0'
|
161
178
|
segments:
|
162
179
|
- 0
|
163
|
-
hash: -
|
180
|
+
hash: -3105018591679682945
|
164
181
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
165
182
|
none: false
|
166
183
|
requirements:
|
@@ -169,7 +186,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
169
186
|
version: '0'
|
170
187
|
segments:
|
171
188
|
- 0
|
172
|
-
hash: -
|
189
|
+
hash: -3105018591679682945
|
173
190
|
requirements: []
|
174
191
|
rubyforge_project:
|
175
192
|
rubygems_version: 1.8.23
|
@@ -187,3 +204,4 @@ test_files:
|
|
187
204
|
- spec/support/node_manager_stub.rb
|
188
205
|
- spec/support/redis_stub.rb
|
189
206
|
- spec/util_spec.rb
|
207
|
+
has_rdoc:
|