spbtv_redis_failover 1.0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +19 -0
- data/.travis.yml +7 -0
- data/.yardopts +6 -0
- data/Changes.md +191 -0
- data/Gemfile +2 -0
- data/LICENSE +22 -0
- data/README.md +240 -0
- data/Rakefile +9 -0
- data/bin/redis_node_manager +7 -0
- data/examples/config.yml +17 -0
- data/examples/multiple_environments_config.yml +15 -0
- data/lib/redis_failover.rb +25 -0
- data/lib/redis_failover/cli.rb +142 -0
- data/lib/redis_failover/client.rb +517 -0
- data/lib/redis_failover/errors.rb +54 -0
- data/lib/redis_failover/failover_strategy.rb +25 -0
- data/lib/redis_failover/failover_strategy/latency.rb +21 -0
- data/lib/redis_failover/manual_failover.rb +52 -0
- data/lib/redis_failover/node.rb +190 -0
- data/lib/redis_failover/node_manager.rb +741 -0
- data/lib/redis_failover/node_snapshot.rb +81 -0
- data/lib/redis_failover/node_strategy.rb +34 -0
- data/lib/redis_failover/node_strategy/consensus.rb +18 -0
- data/lib/redis_failover/node_strategy/majority.rb +18 -0
- data/lib/redis_failover/node_strategy/single.rb +17 -0
- data/lib/redis_failover/node_watcher.rb +83 -0
- data/lib/redis_failover/runner.rb +27 -0
- data/lib/redis_failover/util.rb +137 -0
- data/lib/redis_failover/version.rb +3 -0
- data/misc/redis_failover.png +0 -0
- data/spbtv_redis_failover.gemspec +26 -0
- data/spec/cli_spec.rb +75 -0
- data/spec/client_spec.rb +153 -0
- data/spec/failover_strategy/latency_spec.rb +41 -0
- data/spec/failover_strategy_spec.rb +17 -0
- data/spec/node_manager_spec.rb +136 -0
- data/spec/node_snapshot_spec.rb +30 -0
- data/spec/node_spec.rb +84 -0
- data/spec/node_strategy/consensus_spec.rb +30 -0
- data/spec/node_strategy/majority_spec.rb +22 -0
- data/spec/node_strategy/single_spec.rb +22 -0
- data/spec/node_strategy_spec.rb +22 -0
- data/spec/node_watcher_spec.rb +58 -0
- data/spec/spec_helper.rb +21 -0
- data/spec/support/config/multiple_environments.yml +15 -0
- data/spec/support/config/multiple_environments_with_chroot.yml +17 -0
- data/spec/support/config/single_environment.yml +7 -0
- data/spec/support/config/single_environment_with_chroot.yml +8 -0
- data/spec/support/node_manager_stub.rb +87 -0
- data/spec/support/redis_stub.rb +105 -0
- data/spec/util_spec.rb +21 -0
- metadata +207 -0
data/Rakefile
ADDED
data/examples/config.yml
ADDED
@@ -0,0 +1,17 @@
|
|
1
|
+
# Sample configuration file for Node Manager.
|
2
|
+
# redis_node_manager -C config.yml
|
3
|
+
---
|
4
|
+
:max_failures: 2
|
5
|
+
:node_strategy: majority
|
6
|
+
:failover_strategy: latency
|
7
|
+
:required_node_managers: 2
|
8
|
+
:nodes:
|
9
|
+
- localhost:6379
|
10
|
+
- localhost:1111
|
11
|
+
- localhost:2222
|
12
|
+
- localhost:3333
|
13
|
+
:zkservers:
|
14
|
+
- localhost:2181
|
15
|
+
- localhost:2182
|
16
|
+
- localhost:2183
|
17
|
+
:password: foobar
|
@@ -0,0 +1,25 @@
|
|
1
|
+
require 'zk'
|
2
|
+
require 'set'
|
3
|
+
require 'yaml'
|
4
|
+
require 'redis'
|
5
|
+
require 'thread'
|
6
|
+
require 'logger'
|
7
|
+
require 'timeout'
|
8
|
+
require 'optparse'
|
9
|
+
require 'benchmark'
|
10
|
+
require 'multi_json'
|
11
|
+
require 'securerandom'
|
12
|
+
|
13
|
+
require 'redis_failover/cli'
|
14
|
+
require 'redis_failover/util'
|
15
|
+
require 'redis_failover/node'
|
16
|
+
require 'redis_failover/errors'
|
17
|
+
require 'redis_failover/client'
|
18
|
+
require 'redis_failover/runner'
|
19
|
+
require 'redis_failover/version'
|
20
|
+
require 'redis_failover/node_manager'
|
21
|
+
require 'redis_failover/node_watcher'
|
22
|
+
require 'redis_failover/node_strategy'
|
23
|
+
require 'redis_failover/node_snapshot'
|
24
|
+
require 'redis_failover/manual_failover'
|
25
|
+
require 'redis_failover/failover_strategy'
|
@@ -0,0 +1,142 @@
|
|
1
|
+
module RedisFailover
|
2
|
+
# Parses server command-line arguments.
|
3
|
+
class CLI
|
4
|
+
# Parses the source of options.
|
5
|
+
#
|
6
|
+
# @param [Array] source the command-line options to parse
|
7
|
+
# @return [Hash] the parsed options
|
8
|
+
def self.parse(source)
|
9
|
+
options = {}
|
10
|
+
parser = OptionParser.new do |opts|
|
11
|
+
opts.banner = "Usage: redis_node_manager [OPTIONS]"
|
12
|
+
opts.separator ""
|
13
|
+
opts.separator "Specific options:"
|
14
|
+
|
15
|
+
opts.on('-n', '--nodes NODES',
|
16
|
+
'Comma-separated redis host:port pairs') do |nodes|
|
17
|
+
options[:nodes] = nodes
|
18
|
+
end
|
19
|
+
|
20
|
+
opts.on('-z', '--zkservers SERVERS',
|
21
|
+
'Comma-separated ZooKeeper host:port pairs') do |servers|
|
22
|
+
options[:zkservers] = servers
|
23
|
+
end
|
24
|
+
|
25
|
+
opts.on('-p', '--password PASSWORD', 'Redis password') do |password|
|
26
|
+
options[:password] = password
|
27
|
+
end
|
28
|
+
|
29
|
+
opts.on('--znode-path PATH',
|
30
|
+
'Znode path override for storing redis server list') do |path|
|
31
|
+
options[:znode_path] = path
|
32
|
+
end
|
33
|
+
|
34
|
+
opts.on('--max-failures COUNT',
|
35
|
+
'Max failures before manager marks node unavailable') do |max|
|
36
|
+
options[:max_failures] = Integer(max)
|
37
|
+
end
|
38
|
+
|
39
|
+
opts.on('-C', '--config PATH', 'Path to YAML config file') do |file|
|
40
|
+
options[:config_file] = file
|
41
|
+
end
|
42
|
+
|
43
|
+
opts.on('--with-chroot ROOT', 'Path to ZooKeepers chroot') do |chroot|
|
44
|
+
options[:chroot] = chroot
|
45
|
+
end
|
46
|
+
|
47
|
+
opts.on('-E', '--environment ENV', 'Config environment to use') do |config_env|
|
48
|
+
options[:config_environment] = config_env
|
49
|
+
end
|
50
|
+
|
51
|
+
opts.on('--node-strategy STRATEGY',
|
52
|
+
'Strategy used when determining availability of nodes (default: majority)') do |strategy|
|
53
|
+
options[:node_strategy] = strategy
|
54
|
+
end
|
55
|
+
|
56
|
+
opts.on('--failover-strategy STRATEGY',
|
57
|
+
'Strategy used when failing over to a new node (default: latency)') do |strategy|
|
58
|
+
options[:failover_strategy] = strategy
|
59
|
+
end
|
60
|
+
|
61
|
+
opts.on('--required-node-managers COUNT',
|
62
|
+
'Required Node Managers that must be reachable to determine node state (default: 1)') do |count|
|
63
|
+
options[:required_node_managers] = Integer(count)
|
64
|
+
end
|
65
|
+
|
66
|
+
opts.on('-h', '--help', 'Display all options') do
|
67
|
+
puts opts
|
68
|
+
exit
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
parser.parse(source)
|
73
|
+
if config_file = options[:config_file]
|
74
|
+
options = from_file(config_file, options[:config_environment])
|
75
|
+
end
|
76
|
+
|
77
|
+
if invalid_options?(options)
|
78
|
+
puts parser
|
79
|
+
exit
|
80
|
+
end
|
81
|
+
|
82
|
+
prepare(options)
|
83
|
+
end
|
84
|
+
|
85
|
+
# @return [Boolean] true if required options missing, false otherwise
|
86
|
+
def self.invalid_options?(options)
|
87
|
+
return true if options.empty?
|
88
|
+
return true unless options.values_at(:nodes, :zkservers).all?
|
89
|
+
false
|
90
|
+
end
|
91
|
+
|
92
|
+
# Parses options from a YAML file.
|
93
|
+
#
|
94
|
+
# @param [String] file the filename
|
95
|
+
# @params [String] env the environment
|
96
|
+
# @return [Hash] the parsed options
|
97
|
+
def self.from_file(file, env = nil)
|
98
|
+
unless File.exists?(file)
|
99
|
+
raise ArgumentError, "File #{file} can't be found"
|
100
|
+
end
|
101
|
+
options = YAML.load_file(file)
|
102
|
+
|
103
|
+
if env
|
104
|
+
options = options.fetch(env.to_sym) do
|
105
|
+
raise ArgumentError, "Environment #{env} can't be found in config"
|
106
|
+
end
|
107
|
+
end
|
108
|
+
|
109
|
+
options[:nodes] = options[:nodes].join(',')
|
110
|
+
options[:zkservers] = options[:zkservers].join(',')
|
111
|
+
|
112
|
+
options
|
113
|
+
end
|
114
|
+
|
115
|
+
# Prepares the options for the rest of the system.
|
116
|
+
#
|
117
|
+
# @param [Hash] options the options to prepare
|
118
|
+
# @return [Hash] the prepared options
|
119
|
+
def self.prepare(options)
|
120
|
+
options.each_value { |v| v.strip! if v.respond_to?(:strip!) }
|
121
|
+
# turns 'host1:port,host2:port' => [{:host => host, :port => port}, ...]
|
122
|
+
options[:nodes] = options[:nodes].split(',').map do |node|
|
123
|
+
Hash[[:host, :port].zip(node.split(':'))]
|
124
|
+
end
|
125
|
+
|
126
|
+
# assume password is same for all redis nodes
|
127
|
+
if password = options[:password]
|
128
|
+
options[:nodes].each { |opts| opts.update(:password => password) }
|
129
|
+
end
|
130
|
+
|
131
|
+
if node_strategy = options[:node_strategy]
|
132
|
+
options[:node_strategy] = node_strategy.to_sym
|
133
|
+
end
|
134
|
+
|
135
|
+
if failover_strategy = options[:failover_strategy]
|
136
|
+
options[:failover_strategy] = failover_strategy.to_sym
|
137
|
+
end
|
138
|
+
|
139
|
+
options
|
140
|
+
end
|
141
|
+
end
|
142
|
+
end
|
@@ -0,0 +1,517 @@
|
|
1
|
+
module RedisFailover
|
2
|
+
# Redis failover-aware client. RedisFailover::Client is a wrapper over a set
|
3
|
+
# of underlying redis clients, which means all normal redis operations can be
|
4
|
+
# performed on an instance of this class. The class only requires a set of
|
5
|
+
# ZooKeeper server addresses to function properly. The client will automatically
|
6
|
+
# retry failed operations, and handle failover to a new master. The client
|
7
|
+
# registers and listens for watcher events from the Node Manager. When these
|
8
|
+
# events are received, the client fetches the latest set of redis nodes from
|
9
|
+
# ZooKeeper and rebuilds its internal Redis clients appropriately.
|
10
|
+
# RedisFailover::Client also directs write operations to the master, and all
|
11
|
+
# read operations to the slaves.
|
12
|
+
#
|
13
|
+
# @example Usage
|
14
|
+
# zk_servers = 'localhost:2181,localhost:2182,localhost:2183'
|
15
|
+
# client = RedisFailover::Client.new(:zkservers => zk_servers)
|
16
|
+
# client.set('foo', 1) # will be directed to master
|
17
|
+
# client.get('foo') # will be directed to a slave
|
18
|
+
#
|
19
|
+
class Client
|
20
|
+
include Util
|
21
|
+
|
22
|
+
# Maximum allowed elapsed time between notifications from the Node Manager.
|
23
|
+
# When this timeout is reached, the client will raise a NoNodeManagerError
|
24
|
+
# and purge its internal redis clients.
|
25
|
+
ZNODE_UPDATE_TIMEOUT = 9
|
26
|
+
|
27
|
+
# Amount of time to sleep before retrying a failed operation.
|
28
|
+
RETRY_WAIT_TIME = 3
|
29
|
+
|
30
|
+
# Performance optimization: to avoid unnecessary method_missing calls,
|
31
|
+
# we proactively define methods that dispatch to the underlying redis
|
32
|
+
# calls.
|
33
|
+
Redis.public_instance_methods(false).each do |method|
|
34
|
+
define_method(method) do |*args, &block|
|
35
|
+
dispatch(method, *args, &block)
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
def call(command, &block)
|
40
|
+
method = command[0]
|
41
|
+
args = command[1..-1]
|
42
|
+
dispatch(method, *args, &block)
|
43
|
+
end
|
44
|
+
|
45
|
+
# Creates a new failover redis client.
|
46
|
+
#
|
47
|
+
# @param [Hash] options the options used to initialize the client instance
|
48
|
+
# @option options [String] :zkservers comma-separated ZooKeeper host:port
|
49
|
+
# @option options [String] :zk an existing ZK client connection instance
|
50
|
+
# @option options [String] :znode_path znode path override for redis nodes
|
51
|
+
# @option options [String] :password password for redis nodes
|
52
|
+
# @option options [String] :db database to use for redis nodes
|
53
|
+
# @option options [String] :namespace namespace for redis nodes
|
54
|
+
# @option options [Logger] :logger logger override
|
55
|
+
# @option options [Boolean] :retry_failure indicates if failures are retried
|
56
|
+
# @option options [Integer] :max_retries max retries for a failure
|
57
|
+
# @option options [Boolean] :safe_mode indicates if safe mode is used or not
|
58
|
+
# @option options [Boolean] :master_only indicates if only redis master is used
|
59
|
+
# @option options [Boolean] :verify_role verify the actual role of a redis node before every command
|
60
|
+
# @note Use either :zkservers or :zk
|
61
|
+
# @return [RedisFailover::Client]
|
62
|
+
def initialize(options = {})
|
63
|
+
Util.logger = options[:logger] if options[:logger]
|
64
|
+
@master = nil
|
65
|
+
@slaves = []
|
66
|
+
@node_addresses = {}
|
67
|
+
@lock = Monitor.new
|
68
|
+
@current_client_key = "current-client-#{self.object_id}"
|
69
|
+
yield self if block_given?
|
70
|
+
|
71
|
+
parse_options(options)
|
72
|
+
setup_zk
|
73
|
+
build_clients
|
74
|
+
end
|
75
|
+
|
76
|
+
# Stubs this method to return this RedisFailover::Client object.
|
77
|
+
#
|
78
|
+
# Some libraries (Resque) assume they can access the `client` via this method,
|
79
|
+
# but we don't want to actually ever expose the internal Redis connections.
|
80
|
+
#
|
81
|
+
# By returning `self` here, we can add stubs for functionality like #reconnect,
|
82
|
+
# and everything will Just Work.
|
83
|
+
#
|
84
|
+
# Takes an *args array for safety only.
|
85
|
+
#
|
86
|
+
# @return [RedisFailover::Client]
|
87
|
+
def client(*args)
|
88
|
+
self
|
89
|
+
end
|
90
|
+
|
91
|
+
# Delegates to the underlying Redis client to fetch the location.
|
92
|
+
# This method always returns the location of the master.
|
93
|
+
#
|
94
|
+
# @return [String] the redis location
|
95
|
+
def location
|
96
|
+
dispatch(:client).location
|
97
|
+
end
|
98
|
+
|
99
|
+
# Specifies a callback to invoke when the current redis node list changes.
|
100
|
+
#
|
101
|
+
# @param [Proc] a callback with current master and slaves as arguments
|
102
|
+
#
|
103
|
+
# @example Usage
|
104
|
+
# RedisFailover::Client.new(:zkservers => zk_servers) do |client|
|
105
|
+
# client.on_node_change do |master, slaves|
|
106
|
+
# logger.info("Nodes changed! master: #{master}, slaves: #{slaves}")
|
107
|
+
# end
|
108
|
+
# end
|
109
|
+
def on_node_change(&callback)
|
110
|
+
@on_node_change = callback
|
111
|
+
end
|
112
|
+
|
113
|
+
# Dispatches redis operations to master/slaves.
|
114
|
+
def method_missing(method, *args, &block)
|
115
|
+
if redis_operation?(method)
|
116
|
+
dispatch(method, *args, &block)
|
117
|
+
else
|
118
|
+
super
|
119
|
+
end
|
120
|
+
end
|
121
|
+
|
122
|
+
# Determines whether or not an unknown method can be handled.
|
123
|
+
#
|
124
|
+
# @param [Symbol] method the method to check
|
125
|
+
# @param [Boolean] include_private determines if private methods are checked
|
126
|
+
# @return [Boolean] indicates if the method can be handled
|
127
|
+
def respond_to_missing?(method, include_private)
|
128
|
+
redis_operation?(method) || super
|
129
|
+
end
|
130
|
+
|
131
|
+
# @return [String] a string representation of the client
|
132
|
+
def inspect
|
133
|
+
"#<RedisFailover::Client (db: #{@db.to_i}, master: #{master_name}, slaves: #{slave_names})>"
|
134
|
+
end
|
135
|
+
alias_method :to_s, :inspect
|
136
|
+
|
137
|
+
# Force a manual failover to a new server. A specific server can be specified
|
138
|
+
# via options. If no options are passed, a random slave will be selected as
|
139
|
+
# the candidate for the new master.
|
140
|
+
#
|
141
|
+
# @param [Hash] options the options used for manual failover
|
142
|
+
# @option options [String] :host the host of the failover candidate
|
143
|
+
# @option options [String] :port the port of the failover candidate
|
144
|
+
def manual_failover(options = {})
|
145
|
+
ManualFailover.new(@zk, @root_znode, options).perform
|
146
|
+
self
|
147
|
+
end
|
148
|
+
|
149
|
+
# Gracefully performs a shutdown of this client. This method is
|
150
|
+
# mostly useful when the client is used in a forking environment.
|
151
|
+
# When a fork occurs, you can call this method in an after_fork hook,
|
152
|
+
# and then create a new instance of the client. The underlying
|
153
|
+
# ZooKeeper client and redis clients will be closed.
|
154
|
+
def shutdown
|
155
|
+
@zk.close! if @zk
|
156
|
+
@zk = nil
|
157
|
+
purge_clients
|
158
|
+
end
|
159
|
+
|
160
|
+
# Reconnect will first perform a shutdown of the underlying redis clients.
|
161
|
+
# Next, it attempts to reopen the ZooKeeper client and re-create the redis
|
162
|
+
# clients after it fetches the most up-to-date list from ZooKeeper.
|
163
|
+
def reconnect
|
164
|
+
purge_clients
|
165
|
+
@zk ? @zk.reopen : setup_zk
|
166
|
+
build_clients
|
167
|
+
end
|
168
|
+
|
169
|
+
# Retrieves the current redis master.
|
170
|
+
#
|
171
|
+
# @return [String] the host/port of the current master
|
172
|
+
def current_master
|
173
|
+
master = @lock.synchronize { @master }
|
174
|
+
address_for(master)
|
175
|
+
end
|
176
|
+
|
177
|
+
# Retrieves the current redis slaves.
|
178
|
+
#
|
179
|
+
# @return [Array<String>] an array of known slave host/port addresses
|
180
|
+
def current_slaves
|
181
|
+
slaves = @lock.synchronize { @slaves }
|
182
|
+
addresses_for(slaves)
|
183
|
+
end
|
184
|
+
|
185
|
+
private
|
186
|
+
|
187
|
+
# Sets up the underlying ZooKeeper connection.
|
188
|
+
def setup_zk
|
189
|
+
@zk = ZK.new(@zkservers) if @zkservers
|
190
|
+
@zk.register(redis_nodes_path) { |event| handle_zk_event(event) }
|
191
|
+
if @safe_mode
|
192
|
+
@zk.on_expired_session { purge_clients }
|
193
|
+
end
|
194
|
+
@zk.on_connected { @zk.stat(redis_nodes_path, :watch => true) }
|
195
|
+
@zk.stat(redis_nodes_path, :watch => true)
|
196
|
+
update_znode_timestamp
|
197
|
+
end
|
198
|
+
|
199
|
+
# Handles a ZK event.
|
200
|
+
#
|
201
|
+
# @param [ZK::Event] event the ZK event to handle
|
202
|
+
def handle_zk_event(event)
|
203
|
+
update_znode_timestamp
|
204
|
+
if event.node_created? || event.node_changed?
|
205
|
+
build_clients
|
206
|
+
elsif event.node_deleted?
|
207
|
+
purge_clients
|
208
|
+
@zk.stat(redis_nodes_path, :watch => true)
|
209
|
+
else
|
210
|
+
logger.error("Unknown ZK node event: #{event.inspect}")
|
211
|
+
end
|
212
|
+
ensure
|
213
|
+
@zk.stat(redis_nodes_path, :watch => true)
|
214
|
+
end
|
215
|
+
|
216
|
+
# Determines if a method is a known redis operation.
|
217
|
+
#
|
218
|
+
# @param [Symbol] method the method to check
|
219
|
+
# @return [Boolean] true if redis operation, false otherwise
|
220
|
+
def redis_operation?(method)
|
221
|
+
Redis.public_instance_methods(false).include?(method)
|
222
|
+
end
|
223
|
+
|
224
|
+
# Dispatches a redis operation to a master or slave.
|
225
|
+
#
|
226
|
+
# @param [Symbol] method the method to dispatch
|
227
|
+
# @param [Array] args the arguments to pass to the method
|
228
|
+
# @param [Proc] block an optional block to pass to the method
|
229
|
+
# @return [Object] the result of dispatching the command
|
230
|
+
def dispatch(method, *args, &block)
|
231
|
+
if @safe_mode && !recently_heard_from_node_manager?
|
232
|
+
build_clients
|
233
|
+
end
|
234
|
+
|
235
|
+
verify_supported!(method)
|
236
|
+
tries = 0
|
237
|
+
begin
|
238
|
+
client_for(method).send(method, *args, &block)
|
239
|
+
rescue *CONNECTIVITY_ERRORS => ex
|
240
|
+
logger.error("Error while handling `#{method}` - #{ex.inspect}")
|
241
|
+
logger.error(ex.backtrace.join("\n"))
|
242
|
+
|
243
|
+
if tries < @max_retries
|
244
|
+
tries += 1
|
245
|
+
free_client
|
246
|
+
build_clients
|
247
|
+
sleep(RETRY_WAIT_TIME)
|
248
|
+
retry
|
249
|
+
end
|
250
|
+
raise
|
251
|
+
ensure
|
252
|
+
free_client
|
253
|
+
end
|
254
|
+
end
|
255
|
+
|
256
|
+
# Returns the currently known master.
|
257
|
+
#
|
258
|
+
# @return [Redis] the Redis client for the current master
|
259
|
+
# @raise [NoMasterError] if no master is available
|
260
|
+
def master
|
261
|
+
if master = @lock.synchronize { @master }
|
262
|
+
verify_role!(master, :master) if @verify_role
|
263
|
+
return master
|
264
|
+
end
|
265
|
+
raise NoMasterError
|
266
|
+
end
|
267
|
+
|
268
|
+
# Returns a random slave from the list of known slaves.
|
269
|
+
#
|
270
|
+
# @note If there are no slaves, the master is returned.
|
271
|
+
# @return [Redis] the Redis client for the slave or master
|
272
|
+
# @raise [NoMasterError] if no master fallback is available
|
273
|
+
def slave
|
274
|
+
# pick a slave, if none available fallback to master
|
275
|
+
if slave = @lock.synchronize { @slaves.shuffle.first }
|
276
|
+
verify_role!(slave, :slave) if @verify_role
|
277
|
+
return slave
|
278
|
+
end
|
279
|
+
master
|
280
|
+
end
|
281
|
+
|
282
|
+
# Builds the Redis clients for the currently known master/slaves.
|
283
|
+
# The current master/slaves are fetched via ZooKeeper.
|
284
|
+
def build_clients
|
285
|
+
@lock.synchronize do
|
286
|
+
begin
|
287
|
+
nodes = fetch_nodes
|
288
|
+
return unless nodes_changed?(nodes)
|
289
|
+
|
290
|
+
purge_clients
|
291
|
+
logger.info("Building new clients for nodes #{nodes.inspect}")
|
292
|
+
new_master = new_clients_for(nodes[:master]).first if nodes[:master]
|
293
|
+
new_slaves = new_clients_for(*nodes[:slaves])
|
294
|
+
@master = new_master
|
295
|
+
@slaves = new_slaves
|
296
|
+
rescue
|
297
|
+
purge_clients
|
298
|
+
raise
|
299
|
+
ensure
|
300
|
+
if should_notify?
|
301
|
+
@on_node_change.call(current_master, current_slaves)
|
302
|
+
@last_notified_master = current_master
|
303
|
+
@last_notified_slaves = current_slaves
|
304
|
+
end
|
305
|
+
end
|
306
|
+
end
|
307
|
+
end
|
308
|
+
|
309
|
+
# Determines if the on_node_change callback should be invoked.
|
310
|
+
#
|
311
|
+
# @return [Boolean] true if callback should be invoked, false otherwise
|
312
|
+
def should_notify?
|
313
|
+
return false unless @on_node_change
|
314
|
+
return true if @last_notified_master != current_master
|
315
|
+
return true if different?(Array(@last_notified_slaves), current_slaves)
|
316
|
+
false
|
317
|
+
end
|
318
|
+
|
319
|
+
# Fetches the known redis nodes from ZooKeeper.
|
320
|
+
#
|
321
|
+
# @return [Hash] the known master/slave redis servers
|
322
|
+
def fetch_nodes
|
323
|
+
data = @zk.get(redis_nodes_path, :watch => true).first
|
324
|
+
nodes = symbolize_keys(decode(data))
|
325
|
+
logger.debug("Fetched nodes: #{nodes.inspect}")
|
326
|
+
|
327
|
+
nodes
|
328
|
+
rescue Zookeeper::Exceptions::InheritedConnectionError, ZK::Exceptions::InterruptedSession => ex
|
329
|
+
logger.debug { "Caught #{ex.class} '#{ex.message}' - reopening ZK client" }
|
330
|
+
@zk.reopen
|
331
|
+
retry
|
332
|
+
rescue *ZK_ERRORS => ex
|
333
|
+
logger.warn { "Caught #{ex.class} '#{ex.message}' - retrying" }
|
334
|
+
sleep(RETRY_WAIT_TIME)
|
335
|
+
retry
|
336
|
+
end
|
337
|
+
|
338
|
+
# Builds new Redis clients for the specified nodes.
|
339
|
+
#
|
340
|
+
# @param [Array<String>] nodes the array of redis host:port pairs
|
341
|
+
# @return [Array<Redis>] the array of corresponding Redis clients
|
342
|
+
def new_clients_for(*nodes)
|
343
|
+
nodes.map do |node|
|
344
|
+
host, port = node.split(':')
|
345
|
+
opts = {:host => host, :port => port}
|
346
|
+
opts.update(:db => @db) if @db
|
347
|
+
opts.update(:password => @password) if @password
|
348
|
+
client = Redis.new(@redis_client_options.merge(opts))
|
349
|
+
if @namespace
|
350
|
+
client = Redis::Namespace.new(@namespace, :redis => client)
|
351
|
+
end
|
352
|
+
@node_addresses[client] = node
|
353
|
+
client
|
354
|
+
end
|
355
|
+
end
|
356
|
+
|
357
|
+
# @return [String] a friendly name for current master
|
358
|
+
def master_name
|
359
|
+
address_for(@master) || 'none'
|
360
|
+
end
|
361
|
+
|
362
|
+
# @return [Array<String>] friendly names for current slaves
|
363
|
+
def slave_names
|
364
|
+
return 'none' if @slaves.empty?
|
365
|
+
addresses_for(@slaves).join(', ')
|
366
|
+
end
|
367
|
+
|
368
|
+
# Verifies the actual role for a redis node.
|
369
|
+
#
|
370
|
+
# @param [Redis] node the redis node to check
|
371
|
+
# @param [Symbol] role the role to verify
|
372
|
+
# @return [Symbol] the verified role
|
373
|
+
# @raise [InvalidNodeRoleError] if the role is invalid
|
374
|
+
def verify_role!(node, role)
|
375
|
+
current_role = node.info['role']
|
376
|
+
if current_role.to_sym != role
|
377
|
+
raise InvalidNodeRoleError.new(address_for(node), role, current_role)
|
378
|
+
end
|
379
|
+
role
|
380
|
+
end
|
381
|
+
|
382
|
+
# Ensures that the method is supported.
|
383
|
+
#
|
384
|
+
# @raise [UnsupportedOperationError] if the operation isn't supported
|
385
|
+
def verify_supported!(method)
|
386
|
+
if UNSUPPORTED_OPS.include?(method)
|
387
|
+
raise UnsupportedOperationError.new(method)
|
388
|
+
end
|
389
|
+
end
|
390
|
+
|
391
|
+
# Returns node addresses.
|
392
|
+
#
|
393
|
+
# @param [Array<Redis>] nodes the redis clients
|
394
|
+
# @return [Array<String>] the addresses for the nodes
|
395
|
+
def addresses_for(nodes)
|
396
|
+
nodes.map { |node| address_for(node) }
|
397
|
+
end
|
398
|
+
|
399
|
+
# Returns a node address.
|
400
|
+
#
|
401
|
+
# @param [Redis] node a redis client
|
402
|
+
# @return [String] the address for the node
|
403
|
+
def address_for(node)
|
404
|
+
return unless node
|
405
|
+
@node_addresses[node]
|
406
|
+
end
|
407
|
+
|
408
|
+
# Determines if the currently known redis servers is different
|
409
|
+
# from the nodes returned by ZooKeeper.
|
410
|
+
#
|
411
|
+
# @param [Array<String>] new_nodes the new redis nodes
|
412
|
+
# @return [Boolean] true if nodes are different, false otherwise
|
413
|
+
def nodes_changed?(new_nodes)
|
414
|
+
return true if address_for(@master) != new_nodes[:master]
|
415
|
+
return true if different?(addresses_for(@slaves), new_nodes[:slaves])
|
416
|
+
false
|
417
|
+
end
|
418
|
+
|
419
|
+
# Disconnects one or more redis clients.
|
420
|
+
#
|
421
|
+
# @param [Array<Redis>] redis_clients the redis clients
|
422
|
+
def disconnect(*redis_clients)
|
423
|
+
redis_clients.each do |conn|
|
424
|
+
if conn
|
425
|
+
begin
|
426
|
+
conn.client.disconnect
|
427
|
+
rescue
|
428
|
+
# best effort
|
429
|
+
end
|
430
|
+
end
|
431
|
+
end
|
432
|
+
end
|
433
|
+
|
434
|
+
# Disconnects current redis clients.
|
435
|
+
def purge_clients
|
436
|
+
@lock.synchronize do
|
437
|
+
logger.info("Purging current redis clients")
|
438
|
+
disconnect(@master, *@slaves)
|
439
|
+
@master = nil
|
440
|
+
@slaves = []
|
441
|
+
@node_addresses = {}
|
442
|
+
end
|
443
|
+
end
|
444
|
+
|
445
|
+
# Updates timestamp when an event is received by the Node Manager.
|
446
|
+
def update_znode_timestamp
|
447
|
+
@last_znode_timestamp = Time.now
|
448
|
+
end
|
449
|
+
|
450
|
+
# @return [Boolean] indicates if we recently heard from the Node Manager
|
451
|
+
def recently_heard_from_node_manager?
|
452
|
+
return false unless @last_znode_timestamp
|
453
|
+
Time.now - @last_znode_timestamp <= ZNODE_UPDATE_TIMEOUT
|
454
|
+
end
|
455
|
+
|
456
|
+
# Acquires a client to use for the specified operation.
|
457
|
+
#
|
458
|
+
# @param [Symbol] method the method for which to retrieve a client
|
459
|
+
# @return [Redis] a redis client to use
|
460
|
+
# @note
|
461
|
+
# This method stores a stack of clients used to handle the case
|
462
|
+
# where the same RedisFailover::Client instance is referenced by
|
463
|
+
# nested blocks (e.g., block passed to multi).
|
464
|
+
def client_for(method)
|
465
|
+
stack = Thread.current[@current_client_key] ||= []
|
466
|
+
client = if stack.last
|
467
|
+
stack.last
|
468
|
+
elsif @master_only
|
469
|
+
master
|
470
|
+
elsif REDIS_READ_OPS.include?(method)
|
471
|
+
slave
|
472
|
+
else
|
473
|
+
master
|
474
|
+
end
|
475
|
+
|
476
|
+
stack << client
|
477
|
+
client
|
478
|
+
end
|
479
|
+
|
480
|
+
# Pops a client from the thread-local client stack.
|
481
|
+
def free_client
|
482
|
+
if stack = Thread.current[@current_client_key]
|
483
|
+
stack.pop
|
484
|
+
end
|
485
|
+
nil
|
486
|
+
end
|
487
|
+
|
488
|
+
# Parses the configuration operations.
|
489
|
+
#
|
490
|
+
# @param [Hash] options the configuration options
|
491
|
+
def parse_options(options)
|
492
|
+
@zk, @zkservers = options.values_at(:zk, :zkservers)
|
493
|
+
if [@zk, @zkservers].all? || [@zk, @zkservers].none?
|
494
|
+
raise ArgumentError, 'must specify :zk or :zkservers'
|
495
|
+
end
|
496
|
+
|
497
|
+
@root_znode = options.fetch(:znode_path, Util::DEFAULT_ROOT_ZNODE_PATH)
|
498
|
+
@namespace = options[:namespace]
|
499
|
+
@password = options[:password]
|
500
|
+
@db = options[:db]
|
501
|
+
@retry = options.fetch(:retry_failure, true)
|
502
|
+
@max_retries = @retry ? options.fetch(:max_retries, 3) : 0
|
503
|
+
@safe_mode = options.fetch(:safe_mode, true)
|
504
|
+
@master_only = options.fetch(:master_only, false)
|
505
|
+
@verify_role = options.fetch(:verify_role, true)
|
506
|
+
|
507
|
+
@redis_client_options = Redis::Client::DEFAULTS.keys.each_with_object({}) do |key, hash|
|
508
|
+
hash[key] = options[key]
|
509
|
+
end
|
510
|
+
end
|
511
|
+
|
512
|
+
# @return [String] the znode path for the master redis nodes config
|
513
|
+
def redis_nodes_path
|
514
|
+
"#{@root_znode}/nodes"
|
515
|
+
end
|
516
|
+
end
|
517
|
+
end
|