spbtv_redis_failover 1.0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +19 -0
  3. data/.travis.yml +7 -0
  4. data/.yardopts +6 -0
  5. data/Changes.md +191 -0
  6. data/Gemfile +2 -0
  7. data/LICENSE +22 -0
  8. data/README.md +240 -0
  9. data/Rakefile +9 -0
  10. data/bin/redis_node_manager +7 -0
  11. data/examples/config.yml +17 -0
  12. data/examples/multiple_environments_config.yml +15 -0
  13. data/lib/redis_failover.rb +25 -0
  14. data/lib/redis_failover/cli.rb +142 -0
  15. data/lib/redis_failover/client.rb +517 -0
  16. data/lib/redis_failover/errors.rb +54 -0
  17. data/lib/redis_failover/failover_strategy.rb +25 -0
  18. data/lib/redis_failover/failover_strategy/latency.rb +21 -0
  19. data/lib/redis_failover/manual_failover.rb +52 -0
  20. data/lib/redis_failover/node.rb +190 -0
  21. data/lib/redis_failover/node_manager.rb +741 -0
  22. data/lib/redis_failover/node_snapshot.rb +81 -0
  23. data/lib/redis_failover/node_strategy.rb +34 -0
  24. data/lib/redis_failover/node_strategy/consensus.rb +18 -0
  25. data/lib/redis_failover/node_strategy/majority.rb +18 -0
  26. data/lib/redis_failover/node_strategy/single.rb +17 -0
  27. data/lib/redis_failover/node_watcher.rb +83 -0
  28. data/lib/redis_failover/runner.rb +27 -0
  29. data/lib/redis_failover/util.rb +137 -0
  30. data/lib/redis_failover/version.rb +3 -0
  31. data/misc/redis_failover.png +0 -0
  32. data/spbtv_redis_failover.gemspec +26 -0
  33. data/spec/cli_spec.rb +75 -0
  34. data/spec/client_spec.rb +153 -0
  35. data/spec/failover_strategy/latency_spec.rb +41 -0
  36. data/spec/failover_strategy_spec.rb +17 -0
  37. data/spec/node_manager_spec.rb +136 -0
  38. data/spec/node_snapshot_spec.rb +30 -0
  39. data/spec/node_spec.rb +84 -0
  40. data/spec/node_strategy/consensus_spec.rb +30 -0
  41. data/spec/node_strategy/majority_spec.rb +22 -0
  42. data/spec/node_strategy/single_spec.rb +22 -0
  43. data/spec/node_strategy_spec.rb +22 -0
  44. data/spec/node_watcher_spec.rb +58 -0
  45. data/spec/spec_helper.rb +21 -0
  46. data/spec/support/config/multiple_environments.yml +15 -0
  47. data/spec/support/config/multiple_environments_with_chroot.yml +17 -0
  48. data/spec/support/config/single_environment.yml +7 -0
  49. data/spec/support/config/single_environment_with_chroot.yml +8 -0
  50. data/spec/support/node_manager_stub.rb +87 -0
  51. data/spec/support/redis_stub.rb +105 -0
  52. data/spec/util_spec.rb +21 -0
  53. metadata +207 -0
data/Rakefile ADDED
@@ -0,0 +1,9 @@
1
+ #!/usr/bin/env rake
2
+ require "bundler/gem_tasks"
3
+ require 'rspec/core/rake_task'
4
+
5
+ RSpec::Core::RakeTask.new(:spec) do |t|
6
+ t.rspec_opts = %w(--format progress)
7
+ end
8
+
9
+ task :default => [:spec]
@@ -0,0 +1,7 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ $stdout.sync = true
4
+ require 'redis_failover'
5
+
6
+ RedisFailover::Runner.run(ARGV)
7
+
@@ -0,0 +1,17 @@
1
+ # Sample configuration file for Node Manager.
2
+ # redis_node_manager -C config.yml
3
+ ---
4
+ :max_failures: 2
5
+ :node_strategy: majority
6
+ :failover_strategy: latency
7
+ :required_node_managers: 2
8
+ :nodes:
9
+ - localhost:6379
10
+ - localhost:1111
11
+ - localhost:2222
12
+ - localhost:3333
13
+ :zkservers:
14
+ - localhost:2181
15
+ - localhost:2182
16
+ - localhost:2183
17
+ :password: foobar
@@ -0,0 +1,15 @@
1
+ :development:
2
+ :nodes:
3
+ - localhost:6379
4
+ - localhost:6389
5
+ :zkservers:
6
+ - localhost:2181
7
+
8
+ :staging:
9
+ :nodes:
10
+ - redis01:6379
11
+ - redis02:6379
12
+ :zkservers:
13
+ - zk01:2181
14
+ - zk02:2181
15
+ - zk03:2181
@@ -0,0 +1,25 @@
1
+ require 'zk'
2
+ require 'set'
3
+ require 'yaml'
4
+ require 'redis'
5
+ require 'thread'
6
+ require 'logger'
7
+ require 'timeout'
8
+ require 'optparse'
9
+ require 'benchmark'
10
+ require 'multi_json'
11
+ require 'securerandom'
12
+
13
+ require 'redis_failover/cli'
14
+ require 'redis_failover/util'
15
+ require 'redis_failover/node'
16
+ require 'redis_failover/errors'
17
+ require 'redis_failover/client'
18
+ require 'redis_failover/runner'
19
+ require 'redis_failover/version'
20
+ require 'redis_failover/node_manager'
21
+ require 'redis_failover/node_watcher'
22
+ require 'redis_failover/node_strategy'
23
+ require 'redis_failover/node_snapshot'
24
+ require 'redis_failover/manual_failover'
25
+ require 'redis_failover/failover_strategy'
@@ -0,0 +1,142 @@
1
+ module RedisFailover
2
+ # Parses server command-line arguments.
3
+ class CLI
4
+ # Parses the source of options.
5
+ #
6
+ # @param [Array] source the command-line options to parse
7
+ # @return [Hash] the parsed options
8
+ def self.parse(source)
9
+ options = {}
10
+ parser = OptionParser.new do |opts|
11
+ opts.banner = "Usage: redis_node_manager [OPTIONS]"
12
+ opts.separator ""
13
+ opts.separator "Specific options:"
14
+
15
+ opts.on('-n', '--nodes NODES',
16
+ 'Comma-separated redis host:port pairs') do |nodes|
17
+ options[:nodes] = nodes
18
+ end
19
+
20
+ opts.on('-z', '--zkservers SERVERS',
21
+ 'Comma-separated ZooKeeper host:port pairs') do |servers|
22
+ options[:zkservers] = servers
23
+ end
24
+
25
+ opts.on('-p', '--password PASSWORD', 'Redis password') do |password|
26
+ options[:password] = password
27
+ end
28
+
29
+ opts.on('--znode-path PATH',
30
+ 'Znode path override for storing redis server list') do |path|
31
+ options[:znode_path] = path
32
+ end
33
+
34
+ opts.on('--max-failures COUNT',
35
+ 'Max failures before manager marks node unavailable') do |max|
36
+ options[:max_failures] = Integer(max)
37
+ end
38
+
39
+ opts.on('-C', '--config PATH', 'Path to YAML config file') do |file|
40
+ options[:config_file] = file
41
+ end
42
+
43
+ opts.on('--with-chroot ROOT', 'Path to ZooKeepers chroot') do |chroot|
44
+ options[:chroot] = chroot
45
+ end
46
+
47
+ opts.on('-E', '--environment ENV', 'Config environment to use') do |config_env|
48
+ options[:config_environment] = config_env
49
+ end
50
+
51
+ opts.on('--node-strategy STRATEGY',
52
+ 'Strategy used when determining availability of nodes (default: majority)') do |strategy|
53
+ options[:node_strategy] = strategy
54
+ end
55
+
56
+ opts.on('--failover-strategy STRATEGY',
57
+ 'Strategy used when failing over to a new node (default: latency)') do |strategy|
58
+ options[:failover_strategy] = strategy
59
+ end
60
+
61
+ opts.on('--required-node-managers COUNT',
62
+ 'Required Node Managers that must be reachable to determine node state (default: 1)') do |count|
63
+ options[:required_node_managers] = Integer(count)
64
+ end
65
+
66
+ opts.on('-h', '--help', 'Display all options') do
67
+ puts opts
68
+ exit
69
+ end
70
+ end
71
+
72
+ parser.parse(source)
73
+ if config_file = options[:config_file]
74
+ options = from_file(config_file, options[:config_environment])
75
+ end
76
+
77
+ if invalid_options?(options)
78
+ puts parser
79
+ exit
80
+ end
81
+
82
+ prepare(options)
83
+ end
84
+
85
+ # @return [Boolean] true if required options missing, false otherwise
86
+ def self.invalid_options?(options)
87
+ return true if options.empty?
88
+ return true unless options.values_at(:nodes, :zkservers).all?
89
+ false
90
+ end
91
+
92
+ # Parses options from a YAML file.
93
+ #
94
+ # @param [String] file the filename
95
+ # @params [String] env the environment
96
+ # @return [Hash] the parsed options
97
+ def self.from_file(file, env = nil)
98
+ unless File.exists?(file)
99
+ raise ArgumentError, "File #{file} can't be found"
100
+ end
101
+ options = YAML.load_file(file)
102
+
103
+ if env
104
+ options = options.fetch(env.to_sym) do
105
+ raise ArgumentError, "Environment #{env} can't be found in config"
106
+ end
107
+ end
108
+
109
+ options[:nodes] = options[:nodes].join(',')
110
+ options[:zkservers] = options[:zkservers].join(',')
111
+
112
+ options
113
+ end
114
+
115
+ # Prepares the options for the rest of the system.
116
+ #
117
+ # @param [Hash] options the options to prepare
118
+ # @return [Hash] the prepared options
119
+ def self.prepare(options)
120
+ options.each_value { |v| v.strip! if v.respond_to?(:strip!) }
121
+ # turns 'host1:port,host2:port' => [{:host => host, :port => port}, ...]
122
+ options[:nodes] = options[:nodes].split(',').map do |node|
123
+ Hash[[:host, :port].zip(node.split(':'))]
124
+ end
125
+
126
+ # assume password is same for all redis nodes
127
+ if password = options[:password]
128
+ options[:nodes].each { |opts| opts.update(:password => password) }
129
+ end
130
+
131
+ if node_strategy = options[:node_strategy]
132
+ options[:node_strategy] = node_strategy.to_sym
133
+ end
134
+
135
+ if failover_strategy = options[:failover_strategy]
136
+ options[:failover_strategy] = failover_strategy.to_sym
137
+ end
138
+
139
+ options
140
+ end
141
+ end
142
+ end
@@ -0,0 +1,517 @@
1
+ module RedisFailover
2
+ # Redis failover-aware client. RedisFailover::Client is a wrapper over a set
3
+ # of underlying redis clients, which means all normal redis operations can be
4
+ # performed on an instance of this class. The class only requires a set of
5
+ # ZooKeeper server addresses to function properly. The client will automatically
6
+ # retry failed operations, and handle failover to a new master. The client
7
+ # registers and listens for watcher events from the Node Manager. When these
8
+ # events are received, the client fetches the latest set of redis nodes from
9
+ # ZooKeeper and rebuilds its internal Redis clients appropriately.
10
+ # RedisFailover::Client also directs write operations to the master, and all
11
+ # read operations to the slaves.
12
+ #
13
+ # @example Usage
14
+ # zk_servers = 'localhost:2181,localhost:2182,localhost:2183'
15
+ # client = RedisFailover::Client.new(:zkservers => zk_servers)
16
+ # client.set('foo', 1) # will be directed to master
17
+ # client.get('foo') # will be directed to a slave
18
+ #
19
+ class Client
20
+ include Util
21
+
22
+ # Maximum allowed elapsed time between notifications from the Node Manager.
23
+ # When this timeout is reached, the client will raise a NoNodeManagerError
24
+ # and purge its internal redis clients.
25
+ ZNODE_UPDATE_TIMEOUT = 9
26
+
27
+ # Amount of time to sleep before retrying a failed operation.
28
+ RETRY_WAIT_TIME = 3
29
+
30
+ # Performance optimization: to avoid unnecessary method_missing calls,
31
+ # we proactively define methods that dispatch to the underlying redis
32
+ # calls.
33
+ Redis.public_instance_methods(false).each do |method|
34
+ define_method(method) do |*args, &block|
35
+ dispatch(method, *args, &block)
36
+ end
37
+ end
38
+
39
+ def call(command, &block)
40
+ method = command[0]
41
+ args = command[1..-1]
42
+ dispatch(method, *args, &block)
43
+ end
44
+
45
+ # Creates a new failover redis client.
46
+ #
47
+ # @param [Hash] options the options used to initialize the client instance
48
+ # @option options [String] :zkservers comma-separated ZooKeeper host:port
49
+ # @option options [String] :zk an existing ZK client connection instance
50
+ # @option options [String] :znode_path znode path override for redis nodes
51
+ # @option options [String] :password password for redis nodes
52
+ # @option options [String] :db database to use for redis nodes
53
+ # @option options [String] :namespace namespace for redis nodes
54
+ # @option options [Logger] :logger logger override
55
+ # @option options [Boolean] :retry_failure indicates if failures are retried
56
+ # @option options [Integer] :max_retries max retries for a failure
57
+ # @option options [Boolean] :safe_mode indicates if safe mode is used or not
58
+ # @option options [Boolean] :master_only indicates if only redis master is used
59
+ # @option options [Boolean] :verify_role verify the actual role of a redis node before every command
60
+ # @note Use either :zkservers or :zk
61
+ # @return [RedisFailover::Client]
62
+ def initialize(options = {})
63
+ Util.logger = options[:logger] if options[:logger]
64
+ @master = nil
65
+ @slaves = []
66
+ @node_addresses = {}
67
+ @lock = Monitor.new
68
+ @current_client_key = "current-client-#{self.object_id}"
69
+ yield self if block_given?
70
+
71
+ parse_options(options)
72
+ setup_zk
73
+ build_clients
74
+ end
75
+
76
+ # Stubs this method to return this RedisFailover::Client object.
77
+ #
78
+ # Some libraries (Resque) assume they can access the `client` via this method,
79
+ # but we don't want to actually ever expose the internal Redis connections.
80
+ #
81
+ # By returning `self` here, we can add stubs for functionality like #reconnect,
82
+ # and everything will Just Work.
83
+ #
84
+ # Takes an *args array for safety only.
85
+ #
86
+ # @return [RedisFailover::Client]
87
+ def client(*args)
88
+ self
89
+ end
90
+
91
+ # Delegates to the underlying Redis client to fetch the location.
92
+ # This method always returns the location of the master.
93
+ #
94
+ # @return [String] the redis location
95
+ def location
96
+ dispatch(:client).location
97
+ end
98
+
99
+ # Specifies a callback to invoke when the current redis node list changes.
100
+ #
101
+ # @param [Proc] a callback with current master and slaves as arguments
102
+ #
103
+ # @example Usage
104
+ # RedisFailover::Client.new(:zkservers => zk_servers) do |client|
105
+ # client.on_node_change do |master, slaves|
106
+ # logger.info("Nodes changed! master: #{master}, slaves: #{slaves}")
107
+ # end
108
+ # end
109
+ def on_node_change(&callback)
110
+ @on_node_change = callback
111
+ end
112
+
113
+ # Dispatches redis operations to master/slaves.
114
+ def method_missing(method, *args, &block)
115
+ if redis_operation?(method)
116
+ dispatch(method, *args, &block)
117
+ else
118
+ super
119
+ end
120
+ end
121
+
122
+ # Determines whether or not an unknown method can be handled.
123
+ #
124
+ # @param [Symbol] method the method to check
125
+ # @param [Boolean] include_private determines if private methods are checked
126
+ # @return [Boolean] indicates if the method can be handled
127
+ def respond_to_missing?(method, include_private)
128
+ redis_operation?(method) || super
129
+ end
130
+
131
+ # @return [String] a string representation of the client
132
+ def inspect
133
+ "#<RedisFailover::Client (db: #{@db.to_i}, master: #{master_name}, slaves: #{slave_names})>"
134
+ end
135
+ alias_method :to_s, :inspect
136
+
137
+ # Force a manual failover to a new server. A specific server can be specified
138
+ # via options. If no options are passed, a random slave will be selected as
139
+ # the candidate for the new master.
140
+ #
141
+ # @param [Hash] options the options used for manual failover
142
+ # @option options [String] :host the host of the failover candidate
143
+ # @option options [String] :port the port of the failover candidate
144
+ def manual_failover(options = {})
145
+ ManualFailover.new(@zk, @root_znode, options).perform
146
+ self
147
+ end
148
+
149
+ # Gracefully performs a shutdown of this client. This method is
150
+ # mostly useful when the client is used in a forking environment.
151
+ # When a fork occurs, you can call this method in an after_fork hook,
152
+ # and then create a new instance of the client. The underlying
153
+ # ZooKeeper client and redis clients will be closed.
154
+ def shutdown
155
+ @zk.close! if @zk
156
+ @zk = nil
157
+ purge_clients
158
+ end
159
+
160
+ # Reconnect will first perform a shutdown of the underlying redis clients.
161
+ # Next, it attempts to reopen the ZooKeeper client and re-create the redis
162
+ # clients after it fetches the most up-to-date list from ZooKeeper.
163
+ def reconnect
164
+ purge_clients
165
+ @zk ? @zk.reopen : setup_zk
166
+ build_clients
167
+ end
168
+
169
+ # Retrieves the current redis master.
170
+ #
171
+ # @return [String] the host/port of the current master
172
+ def current_master
173
+ master = @lock.synchronize { @master }
174
+ address_for(master)
175
+ end
176
+
177
+ # Retrieves the current redis slaves.
178
+ #
179
+ # @return [Array<String>] an array of known slave host/port addresses
180
+ def current_slaves
181
+ slaves = @lock.synchronize { @slaves }
182
+ addresses_for(slaves)
183
+ end
184
+
185
+ private
186
+
187
+ # Sets up the underlying ZooKeeper connection.
188
+ def setup_zk
189
+ @zk = ZK.new(@zkservers) if @zkservers
190
+ @zk.register(redis_nodes_path) { |event| handle_zk_event(event) }
191
+ if @safe_mode
192
+ @zk.on_expired_session { purge_clients }
193
+ end
194
+ @zk.on_connected { @zk.stat(redis_nodes_path, :watch => true) }
195
+ @zk.stat(redis_nodes_path, :watch => true)
196
+ update_znode_timestamp
197
+ end
198
+
199
+ # Handles a ZK event.
200
+ #
201
+ # @param [ZK::Event] event the ZK event to handle
202
+ def handle_zk_event(event)
203
+ update_znode_timestamp
204
+ if event.node_created? || event.node_changed?
205
+ build_clients
206
+ elsif event.node_deleted?
207
+ purge_clients
208
+ @zk.stat(redis_nodes_path, :watch => true)
209
+ else
210
+ logger.error("Unknown ZK node event: #{event.inspect}")
211
+ end
212
+ ensure
213
+ @zk.stat(redis_nodes_path, :watch => true)
214
+ end
215
+
216
+ # Determines if a method is a known redis operation.
217
+ #
218
+ # @param [Symbol] method the method to check
219
+ # @return [Boolean] true if redis operation, false otherwise
220
+ def redis_operation?(method)
221
+ Redis.public_instance_methods(false).include?(method)
222
+ end
223
+
224
+ # Dispatches a redis operation to a master or slave.
225
+ #
226
+ # @param [Symbol] method the method to dispatch
227
+ # @param [Array] args the arguments to pass to the method
228
+ # @param [Proc] block an optional block to pass to the method
229
+ # @return [Object] the result of dispatching the command
230
+ def dispatch(method, *args, &block)
231
+ if @safe_mode && !recently_heard_from_node_manager?
232
+ build_clients
233
+ end
234
+
235
+ verify_supported!(method)
236
+ tries = 0
237
+ begin
238
+ client_for(method).send(method, *args, &block)
239
+ rescue *CONNECTIVITY_ERRORS => ex
240
+ logger.error("Error while handling `#{method}` - #{ex.inspect}")
241
+ logger.error(ex.backtrace.join("\n"))
242
+
243
+ if tries < @max_retries
244
+ tries += 1
245
+ free_client
246
+ build_clients
247
+ sleep(RETRY_WAIT_TIME)
248
+ retry
249
+ end
250
+ raise
251
+ ensure
252
+ free_client
253
+ end
254
+ end
255
+
256
+ # Returns the currently known master.
257
+ #
258
+ # @return [Redis] the Redis client for the current master
259
+ # @raise [NoMasterError] if no master is available
260
+ def master
261
+ if master = @lock.synchronize { @master }
262
+ verify_role!(master, :master) if @verify_role
263
+ return master
264
+ end
265
+ raise NoMasterError
266
+ end
267
+
268
+ # Returns a random slave from the list of known slaves.
269
+ #
270
+ # @note If there are no slaves, the master is returned.
271
+ # @return [Redis] the Redis client for the slave or master
272
+ # @raise [NoMasterError] if no master fallback is available
273
+ def slave
274
+ # pick a slave, if none available fallback to master
275
+ if slave = @lock.synchronize { @slaves.shuffle.first }
276
+ verify_role!(slave, :slave) if @verify_role
277
+ return slave
278
+ end
279
+ master
280
+ end
281
+
282
+ # Builds the Redis clients for the currently known master/slaves.
283
+ # The current master/slaves are fetched via ZooKeeper.
284
+ def build_clients
285
+ @lock.synchronize do
286
+ begin
287
+ nodes = fetch_nodes
288
+ return unless nodes_changed?(nodes)
289
+
290
+ purge_clients
291
+ logger.info("Building new clients for nodes #{nodes.inspect}")
292
+ new_master = new_clients_for(nodes[:master]).first if nodes[:master]
293
+ new_slaves = new_clients_for(*nodes[:slaves])
294
+ @master = new_master
295
+ @slaves = new_slaves
296
+ rescue
297
+ purge_clients
298
+ raise
299
+ ensure
300
+ if should_notify?
301
+ @on_node_change.call(current_master, current_slaves)
302
+ @last_notified_master = current_master
303
+ @last_notified_slaves = current_slaves
304
+ end
305
+ end
306
+ end
307
+ end
308
+
309
+ # Determines if the on_node_change callback should be invoked.
310
+ #
311
+ # @return [Boolean] true if callback should be invoked, false otherwise
312
+ def should_notify?
313
+ return false unless @on_node_change
314
+ return true if @last_notified_master != current_master
315
+ return true if different?(Array(@last_notified_slaves), current_slaves)
316
+ false
317
+ end
318
+
319
+ # Fetches the known redis nodes from ZooKeeper.
320
+ #
321
+ # @return [Hash] the known master/slave redis servers
322
+ def fetch_nodes
323
+ data = @zk.get(redis_nodes_path, :watch => true).first
324
+ nodes = symbolize_keys(decode(data))
325
+ logger.debug("Fetched nodes: #{nodes.inspect}")
326
+
327
+ nodes
328
+ rescue Zookeeper::Exceptions::InheritedConnectionError, ZK::Exceptions::InterruptedSession => ex
329
+ logger.debug { "Caught #{ex.class} '#{ex.message}' - reopening ZK client" }
330
+ @zk.reopen
331
+ retry
332
+ rescue *ZK_ERRORS => ex
333
+ logger.warn { "Caught #{ex.class} '#{ex.message}' - retrying" }
334
+ sleep(RETRY_WAIT_TIME)
335
+ retry
336
+ end
337
+
338
+ # Builds new Redis clients for the specified nodes.
339
+ #
340
+ # @param [Array<String>] nodes the array of redis host:port pairs
341
+ # @return [Array<Redis>] the array of corresponding Redis clients
342
+ def new_clients_for(*nodes)
343
+ nodes.map do |node|
344
+ host, port = node.split(':')
345
+ opts = {:host => host, :port => port}
346
+ opts.update(:db => @db) if @db
347
+ opts.update(:password => @password) if @password
348
+ client = Redis.new(@redis_client_options.merge(opts))
349
+ if @namespace
350
+ client = Redis::Namespace.new(@namespace, :redis => client)
351
+ end
352
+ @node_addresses[client] = node
353
+ client
354
+ end
355
+ end
356
+
357
+ # @return [String] a friendly name for current master
358
+ def master_name
359
+ address_for(@master) || 'none'
360
+ end
361
+
362
+ # @return [Array<String>] friendly names for current slaves
363
+ def slave_names
364
+ return 'none' if @slaves.empty?
365
+ addresses_for(@slaves).join(', ')
366
+ end
367
+
368
+ # Verifies the actual role for a redis node.
369
+ #
370
+ # @param [Redis] node the redis node to check
371
+ # @param [Symbol] role the role to verify
372
+ # @return [Symbol] the verified role
373
+ # @raise [InvalidNodeRoleError] if the role is invalid
374
+ def verify_role!(node, role)
375
+ current_role = node.info['role']
376
+ if current_role.to_sym != role
377
+ raise InvalidNodeRoleError.new(address_for(node), role, current_role)
378
+ end
379
+ role
380
+ end
381
+
382
+ # Ensures that the method is supported.
383
+ #
384
+ # @raise [UnsupportedOperationError] if the operation isn't supported
385
+ def verify_supported!(method)
386
+ if UNSUPPORTED_OPS.include?(method)
387
+ raise UnsupportedOperationError.new(method)
388
+ end
389
+ end
390
+
391
+ # Returns node addresses.
392
+ #
393
+ # @param [Array<Redis>] nodes the redis clients
394
+ # @return [Array<String>] the addresses for the nodes
395
+ def addresses_for(nodes)
396
+ nodes.map { |node| address_for(node) }
397
+ end
398
+
399
+ # Returns a node address.
400
+ #
401
+ # @param [Redis] node a redis client
402
+ # @return [String] the address for the node
403
+ def address_for(node)
404
+ return unless node
405
+ @node_addresses[node]
406
+ end
407
+
408
+ # Determines if the currently known redis servers is different
409
+ # from the nodes returned by ZooKeeper.
410
+ #
411
+ # @param [Array<String>] new_nodes the new redis nodes
412
+ # @return [Boolean] true if nodes are different, false otherwise
413
+ def nodes_changed?(new_nodes)
414
+ return true if address_for(@master) != new_nodes[:master]
415
+ return true if different?(addresses_for(@slaves), new_nodes[:slaves])
416
+ false
417
+ end
418
+
419
+ # Disconnects one or more redis clients.
420
+ #
421
+ # @param [Array<Redis>] redis_clients the redis clients
422
+ def disconnect(*redis_clients)
423
+ redis_clients.each do |conn|
424
+ if conn
425
+ begin
426
+ conn.client.disconnect
427
+ rescue
428
+ # best effort
429
+ end
430
+ end
431
+ end
432
+ end
433
+
434
+ # Disconnects current redis clients.
435
+ def purge_clients
436
+ @lock.synchronize do
437
+ logger.info("Purging current redis clients")
438
+ disconnect(@master, *@slaves)
439
+ @master = nil
440
+ @slaves = []
441
+ @node_addresses = {}
442
+ end
443
+ end
444
+
445
+ # Updates timestamp when an event is received by the Node Manager.
446
+ def update_znode_timestamp
447
+ @last_znode_timestamp = Time.now
448
+ end
449
+
450
+ # @return [Boolean] indicates if we recently heard from the Node Manager
451
+ def recently_heard_from_node_manager?
452
+ return false unless @last_znode_timestamp
453
+ Time.now - @last_znode_timestamp <= ZNODE_UPDATE_TIMEOUT
454
+ end
455
+
456
+ # Acquires a client to use for the specified operation.
457
+ #
458
+ # @param [Symbol] method the method for which to retrieve a client
459
+ # @return [Redis] a redis client to use
460
+ # @note
461
+ # This method stores a stack of clients used to handle the case
462
+ # where the same RedisFailover::Client instance is referenced by
463
+ # nested blocks (e.g., block passed to multi).
464
+ def client_for(method)
465
+ stack = Thread.current[@current_client_key] ||= []
466
+ client = if stack.last
467
+ stack.last
468
+ elsif @master_only
469
+ master
470
+ elsif REDIS_READ_OPS.include?(method)
471
+ slave
472
+ else
473
+ master
474
+ end
475
+
476
+ stack << client
477
+ client
478
+ end
479
+
480
+ # Pops a client from the thread-local client stack.
481
+ def free_client
482
+ if stack = Thread.current[@current_client_key]
483
+ stack.pop
484
+ end
485
+ nil
486
+ end
487
+
488
+ # Parses the configuration operations.
489
+ #
490
+ # @param [Hash] options the configuration options
491
+ def parse_options(options)
492
+ @zk, @zkservers = options.values_at(:zk, :zkservers)
493
+ if [@zk, @zkservers].all? || [@zk, @zkservers].none?
494
+ raise ArgumentError, 'must specify :zk or :zkservers'
495
+ end
496
+
497
+ @root_znode = options.fetch(:znode_path, Util::DEFAULT_ROOT_ZNODE_PATH)
498
+ @namespace = options[:namespace]
499
+ @password = options[:password]
500
+ @db = options[:db]
501
+ @retry = options.fetch(:retry_failure, true)
502
+ @max_retries = @retry ? options.fetch(:max_retries, 3) : 0
503
+ @safe_mode = options.fetch(:safe_mode, true)
504
+ @master_only = options.fetch(:master_only, false)
505
+ @verify_role = options.fetch(:verify_role, true)
506
+
507
+ @redis_client_options = Redis::Client::DEFAULTS.keys.each_with_object({}) do |key, hash|
508
+ hash[key] = options[key]
509
+ end
510
+ end
511
+
512
+ # @return [String] the znode path for the master redis nodes config
513
+ def redis_nodes_path
514
+ "#{@root_znode}/nodes"
515
+ end
516
+ end
517
+ end