redis_failover 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Changes.md +6 -0
- data/README.md +25 -4
- data/examples/config.yml +14 -0
- data/lib/redis_failover/cli.rb +41 -15
- data/lib/redis_failover/client.rb +62 -37
- data/lib/redis_failover/node.rb +4 -4
- data/lib/redis_failover/node_manager.rb +23 -17
- data/lib/redis_failover/util.rb +3 -3
- data/lib/redis_failover/version.rb +1 -1
- data/lib/redis_failover.rb +1 -1
- data/redis_failover.gemspec +1 -1
- data/spec/spec_helper.rb +2 -1
- data/spec/support/node_manager_stub.rb +2 -0
- metadata +19 -52
- data/lib/redis_failover/zk_client.rb +0 -93
data/Changes.md
CHANGED
@@ -1,3 +1,9 @@
|
|
1
|
+
0.7.0
|
2
|
+
-----------
|
3
|
+
- When new master promotion occurs, make existing slaves point to new candidate before promoting new master.
|
4
|
+
- Add support for specifying command-line options in a config.yml file for Node Manager.
|
5
|
+
- Upgrade to 0.9 version of ZK client and cleanup ZK connection error handling.
|
6
|
+
|
1
7
|
0.6.0
|
2
8
|
-----------
|
3
9
|
- Add support for running multiple Node Manager processes for added redundancy (#4)
|
data/README.md
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
# Automatic Redis Failover
|
1
|
+
# Automatic Redis Failover
|
2
2
|
|
3
3
|
[](http://travis-ci.org/ryanlecompte/redis_failover)
|
4
4
|
|
@@ -61,15 +61,36 @@ following options:
|
|
61
61
|
Specific options:
|
62
62
|
-n, --nodes NODES Comma-separated redis host:port pairs
|
63
63
|
-z, --zkservers SERVERS Comma-separated ZooKeeper host:port pairs
|
64
|
-
-p, --password
|
65
|
-
--znode-path
|
66
|
-
--max-failures
|
64
|
+
-p, --password PASSWORD Redis password
|
65
|
+
--znode-path PATH Znode path override for storing redis server list
|
66
|
+
--max-failures COUNT Max failures before manager marks node unavailable
|
67
|
+
-C, --config PATH Path to YAML configuration file
|
67
68
|
-h, --help Display all options
|
68
69
|
|
69
70
|
To start the daemon for a simple master/slave configuration, use the following:
|
70
71
|
|
71
72
|
redis_node_manager -n localhost:6379,localhost:6380 -z localhost:2181,localhost:2182,localhost:2183
|
72
73
|
|
74
|
+
The configuration parameters can also be specified in a config.yml file. An example configuration
|
75
|
+
would look like the following:
|
76
|
+
|
77
|
+
---
|
78
|
+
:max_failures: 2
|
79
|
+
:nodes:
|
80
|
+
- localhost:6379
|
81
|
+
- localhost:1111
|
82
|
+
- localhost:2222
|
83
|
+
- localhost:3333
|
84
|
+
:zkservers:
|
85
|
+
- localhost:2181
|
86
|
+
- localhost:2182
|
87
|
+
- localhost:2183
|
88
|
+
:password: foobar
|
89
|
+
|
90
|
+
You would then simpy start the Node Manager via the following:
|
91
|
+
|
92
|
+
redis_node_manager -C config.yml
|
93
|
+
|
73
94
|
The Node Manager will automatically discover the master/slaves upon startup. Note that it is
|
74
95
|
a good idea to run more than one instance of the Node Manager daemon in your environment. At
|
75
96
|
any moment, a single Node Manager process will be designated to monitor the redis servers. If
|
data/examples/config.yml
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
# Sample configuration file for Node Manager.
|
2
|
+
# redis_node_manager -C config.yml
|
3
|
+
---
|
4
|
+
:max_failures: 2
|
5
|
+
:nodes:
|
6
|
+
- localhost:6379
|
7
|
+
- localhost:1111
|
8
|
+
- localhost:2222
|
9
|
+
- localhost:3333
|
10
|
+
:zkservers:
|
11
|
+
- localhost:2181
|
12
|
+
- localhost:2182
|
13
|
+
- localhost:2183
|
14
|
+
:password: foobar
|
data/lib/redis_failover/cli.rb
CHANGED
@@ -10,31 +10,32 @@ module RedisFailover
|
|
10
10
|
|
11
11
|
opts.on('-n', '--nodes NODES',
|
12
12
|
'Comma-separated redis host:port pairs') do |nodes|
|
13
|
-
|
14
|
-
options[:nodes] = nodes.split(',').map do |node|
|
15
|
-
Hash[[:host, :port].zip(node.strip.split(':'))]
|
16
|
-
end
|
13
|
+
options[:nodes] = nodes
|
17
14
|
end
|
18
15
|
|
19
16
|
opts.on('-z', '--zkservers SERVERS',
|
20
17
|
'Comma-separated ZooKeeper host:port pairs') do |servers|
|
21
|
-
options[:zkservers] = servers
|
18
|
+
options[:zkservers] = servers
|
22
19
|
end
|
23
20
|
|
24
|
-
opts.on('-p', '--password
|
25
|
-
options[:password] = password
|
21
|
+
opts.on('-p', '--password PASSWORD', 'Redis password') do |password|
|
22
|
+
options[:password] = password
|
26
23
|
end
|
27
24
|
|
28
|
-
opts.on('--znode-path
|
25
|
+
opts.on('--znode-path PATH',
|
29
26
|
'Znode path override for storing redis server list') do |path|
|
30
27
|
options[:znode_path] = path
|
31
28
|
end
|
32
29
|
|
33
|
-
opts.on('--max-failures
|
30
|
+
opts.on('--max-failures COUNT',
|
34
31
|
'Max failures before manager marks node unavailable') do |max|
|
35
32
|
options[:max_failures] = Integer(max)
|
36
33
|
end
|
37
34
|
|
35
|
+
opts.on '-C', '--config PATH', "Path to YAML configuration file" do |file|
|
36
|
+
options[:config_file] = file
|
37
|
+
end
|
38
|
+
|
38
39
|
opts.on('-h', '--help', 'Display all options') do
|
39
40
|
puts opts
|
40
41
|
exit
|
@@ -42,17 +43,16 @@ module RedisFailover
|
|
42
43
|
end
|
43
44
|
|
44
45
|
parser.parse(source)
|
46
|
+
if config_file = options[:config_file]
|
47
|
+
options = from_file(config_file)
|
48
|
+
end
|
49
|
+
|
45
50
|
if required_options_missing?(options)
|
46
51
|
puts parser
|
47
52
|
exit
|
48
53
|
end
|
49
54
|
|
50
|
-
|
51
|
-
if password = options[:password]
|
52
|
-
options[:nodes].each { |opts| opts.update(:password => password) }
|
53
|
-
end
|
54
|
-
|
55
|
-
options
|
55
|
+
prepare(options)
|
56
56
|
end
|
57
57
|
|
58
58
|
def self.required_options_missing?(options)
|
@@ -60,5 +60,31 @@ module RedisFailover
|
|
60
60
|
return true unless options.values_at(:nodes, :zkservers).all?
|
61
61
|
false
|
62
62
|
end
|
63
|
+
|
64
|
+
def self.from_file(file)
|
65
|
+
unless File.exists?(file)
|
66
|
+
raise ArgumentError, "File #{file} can't be found"
|
67
|
+
end
|
68
|
+
options = YAML.load_file(file)
|
69
|
+
options[:nodes] = options[:nodes].join(',')
|
70
|
+
options[:zkservers] = options[:zkservers].join(',')
|
71
|
+
|
72
|
+
options
|
73
|
+
end
|
74
|
+
|
75
|
+
def self.prepare(options)
|
76
|
+
options.each_value { |v| v.strip! if v.respond_to?(:strip!) }
|
77
|
+
# turns 'host1:port,host2:port' => [{:host => host, :port => port}, ...]
|
78
|
+
options[:nodes] = options[:nodes].split(',').map do |node|
|
79
|
+
Hash[[:host, :port].zip(node.split(':'))]
|
80
|
+
end
|
81
|
+
|
82
|
+
# assume password is same for all redis nodes
|
83
|
+
if password = options[:password]
|
84
|
+
options[:nodes].each { |opts| opts.update(:password => password) }
|
85
|
+
end
|
86
|
+
|
87
|
+
options
|
88
|
+
end
|
63
89
|
end
|
64
90
|
end
|
@@ -105,11 +105,12 @@ module RedisFailover
|
|
105
105
|
@password = options[:password]
|
106
106
|
@db = options[:db]
|
107
107
|
@retry = options[:retry_failure] || true
|
108
|
-
@max_retries = @retry ? options.fetch(:max_retries, 3) :
|
108
|
+
@max_retries = @retry ? options.fetch(:max_retries, 3) : 0
|
109
109
|
@master = nil
|
110
110
|
@slaves = []
|
111
|
+
@queue = Queue.new
|
111
112
|
@lock = Monitor.new
|
112
|
-
|
113
|
+
start_zk
|
113
114
|
build_clients
|
114
115
|
end
|
115
116
|
|
@@ -133,40 +134,66 @@ module RedisFailover
|
|
133
134
|
|
134
135
|
private
|
135
136
|
|
136
|
-
def
|
137
|
-
@
|
138
|
-
|
139
|
-
client.on_session_expiration do
|
140
|
-
purge_clients
|
141
|
-
end
|
137
|
+
def zk
|
138
|
+
@lock.synchronize { @zk }
|
139
|
+
end
|
142
140
|
|
143
|
-
|
144
|
-
|
145
|
-
|
141
|
+
def start_zk
|
142
|
+
@delivery_thread ||= Thread.new do
|
143
|
+
while event = @queue.pop
|
144
|
+
if event.is_a?(Proc)
|
145
|
+
event.call
|
146
|
+
else
|
147
|
+
handle_zk_event(event)
|
148
|
+
end
|
146
149
|
end
|
150
|
+
end
|
151
|
+
reconnect_zk
|
152
|
+
end
|
147
153
|
|
148
|
-
|
149
|
-
|
150
|
-
|
154
|
+
def handle_session_established
|
155
|
+
@lock.synchronize do
|
156
|
+
@zk.watcher.register(@znode) do |event|
|
157
|
+
@queue << event
|
151
158
|
end
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
purge_clients
|
161
|
-
client.stat(@znode, :watch => true)
|
162
|
-
else
|
163
|
-
logger.error("Unknown ZK node event: #{event.inspect}")
|
164
|
-
end
|
159
|
+
@zk.on_expired_session do
|
160
|
+
@queue << proc { reconnect_zk }
|
161
|
+
end
|
162
|
+
@zk.event_handler.register_state_handler(:connecting) do
|
163
|
+
@queue << proc { handle_lost_connection }
|
164
|
+
end
|
165
|
+
@zk.on_connected do
|
166
|
+
@zk.stat(@znode, :watch => true)
|
165
167
|
end
|
166
168
|
end
|
169
|
+
end
|
170
|
+
|
171
|
+
def handle_zk_event(event)
|
172
|
+
update_znode_timestamp
|
173
|
+
if event.node_created? || event.node_changed?
|
174
|
+
build_clients
|
175
|
+
elsif event.node_deleted?
|
176
|
+
purge_clients
|
177
|
+
zk.stat(@znode, :watch => true)
|
178
|
+
else
|
179
|
+
logger.error("Unknown ZK node event: #{event.inspect}")
|
180
|
+
end
|
181
|
+
end
|
182
|
+
|
183
|
+
def reconnect_zk
|
184
|
+
handle_lost_connection
|
185
|
+
@lock.synchronize do
|
186
|
+
@zk.close! if @zk
|
187
|
+
@zk = ZK.new(@zkservers)
|
188
|
+
end
|
189
|
+
handle_session_established
|
167
190
|
update_znode_timestamp
|
168
191
|
end
|
169
192
|
|
193
|
+
def handle_lost_connection
|
194
|
+
purge_clients
|
195
|
+
end
|
196
|
+
|
170
197
|
def redis_operation?(method)
|
171
198
|
Redis.public_instance_methods(false).include?(method)
|
172
199
|
end
|
@@ -197,7 +224,6 @@ module RedisFailover
|
|
197
224
|
sleep(RETRY_WAIT_TIME)
|
198
225
|
retry
|
199
226
|
end
|
200
|
-
|
201
227
|
raise
|
202
228
|
end
|
203
229
|
end
|
@@ -221,7 +247,7 @@ module RedisFailover
|
|
221
247
|
|
222
248
|
def build_clients
|
223
249
|
@lock.synchronize do
|
224
|
-
|
250
|
+
retried = false
|
225
251
|
|
226
252
|
begin
|
227
253
|
nodes = fetch_nodes
|
@@ -233,24 +259,23 @@ module RedisFailover
|
|
233
259
|
new_slaves = new_clients_for(*nodes[:slaves])
|
234
260
|
@master = new_master
|
235
261
|
@slaves = new_slaves
|
236
|
-
rescue
|
237
|
-
|
238
|
-
logger.error("Failed to fetch nodes from #{@zkservers} - #{ex.inspect}")
|
262
|
+
rescue ZK::Exceptions::InterruptedSession => ex
|
263
|
+
logger.error("ZK error while attempting to build clients: #{ex.inspect}")
|
239
264
|
logger.error(ex.backtrace.join("\n"))
|
240
265
|
|
241
|
-
|
242
|
-
|
243
|
-
|
266
|
+
# when ZK is disconnected, retry once
|
267
|
+
unless retried
|
268
|
+
reconnect_zk
|
269
|
+
retried = true
|
244
270
|
retry
|
245
271
|
end
|
246
|
-
|
247
272
|
raise
|
248
273
|
end
|
249
274
|
end
|
250
275
|
end
|
251
276
|
|
252
277
|
def fetch_nodes
|
253
|
-
data =
|
278
|
+
data = zk.get(@znode, :watch => true).first
|
254
279
|
nodes = symbolize_keys(decode(data))
|
255
280
|
logger.debug("Fetched nodes: #{nodes}")
|
256
281
|
|
data/lib/redis_failover/node.rb
CHANGED
@@ -53,11 +53,11 @@ module RedisFailover
|
|
53
53
|
end
|
54
54
|
end
|
55
55
|
|
56
|
-
def make_slave!(
|
56
|
+
def make_slave!(node)
|
57
57
|
perform_operation do |redis|
|
58
|
-
unless slave_of?(
|
59
|
-
redis.slaveof(
|
60
|
-
logger.info("#{self} is now a slave of
|
58
|
+
unless slave_of?(node)
|
59
|
+
redis.slaveof(node.host, node.port)
|
60
|
+
logger.info("#{self} is now a slave of #{node}")
|
61
61
|
wakeup
|
62
62
|
end
|
63
63
|
end
|
@@ -18,29 +18,29 @@ module RedisFailover
|
|
18
18
|
LOCK_PATH = 'master_node_manager'
|
19
19
|
|
20
20
|
# Number of seconds to wait before retrying bootstrap process.
|
21
|
-
TIMEOUT =
|
21
|
+
TIMEOUT = 5
|
22
22
|
|
23
23
|
def initialize(options)
|
24
24
|
logger.info("Redis Node Manager v#{VERSION} starting (#{RUBY_DESCRIPTION})")
|
25
25
|
@options = options
|
26
26
|
@znode = @options[:znode_path] || Util::DEFAULT_ZNODE_PATH
|
27
|
-
@unavailable = []
|
28
|
-
@queue = Queue.new
|
29
27
|
end
|
30
28
|
|
31
29
|
def start
|
32
|
-
@
|
30
|
+
@queue = Queue.new
|
31
|
+
@zk = ZK.new(@options[:zkservers])
|
33
32
|
logger.info('Waiting to become master Node Manager ...')
|
34
|
-
@
|
33
|
+
@zk.with_lock(LOCK_PATH) do
|
35
34
|
logger.info('Acquired master Node Manager lock')
|
36
35
|
discover_nodes
|
37
36
|
initialize_path
|
38
37
|
spawn_watchers
|
39
38
|
handle_state_reports
|
40
39
|
end
|
41
|
-
rescue
|
42
|
-
logger.error("
|
40
|
+
rescue ZK::Exceptions::InterruptedSession => ex
|
41
|
+
logger.error("ZK error while attempting to manage nodes: #{ex.inspect}")
|
43
42
|
logger.error(ex.backtrace.join("\n"))
|
43
|
+
shutdown
|
44
44
|
sleep(TIMEOUT)
|
45
45
|
retry
|
46
46
|
end
|
@@ -51,7 +51,8 @@ module RedisFailover
|
|
51
51
|
|
52
52
|
def shutdown
|
53
53
|
@queue << nil
|
54
|
-
@watchers.each(&:shutdown)
|
54
|
+
@watchers.each(&:shutdown) if @watchers
|
55
|
+
@zk.close! if @zk
|
55
56
|
end
|
56
57
|
|
57
58
|
private
|
@@ -69,7 +70,10 @@ module RedisFailover
|
|
69
70
|
|
70
71
|
# flush current state
|
71
72
|
write_state
|
72
|
-
rescue
|
73
|
+
rescue ZK::Exceptions::InterruptedSession
|
74
|
+
# fail hard if this is a ZK connection-related error
|
75
|
+
raise
|
76
|
+
rescue => ex
|
73
77
|
logger.error("Error handling #{state_report.inspect}: #{ex.inspect}")
|
74
78
|
logger.error(ex.backtrace.join("\n"))
|
75
79
|
end
|
@@ -134,15 +138,17 @@ module RedisFailover
|
|
134
138
|
return
|
135
139
|
end
|
136
140
|
|
141
|
+
redirect_slaves_to(candidate)
|
137
142
|
candidate.make_master!
|
138
143
|
@master = candidate
|
139
|
-
|
144
|
+
|
140
145
|
create_path
|
141
146
|
write_state
|
142
147
|
logger.info("Successfully promoted #{candidate} to master.")
|
143
148
|
end
|
144
149
|
|
145
150
|
def discover_nodes
|
151
|
+
@unavailable = []
|
146
152
|
nodes = @options[:nodes].map { |opts| Node.new(opts) }.uniq
|
147
153
|
raise NoMasterError unless @master = find_master(nodes)
|
148
154
|
@slaves = nodes - [@master]
|
@@ -150,7 +156,7 @@ module RedisFailover
|
|
150
156
|
" (#{@slaves.map(&:to_s).join(', ')})")
|
151
157
|
|
152
158
|
# ensure that slaves are correctly pointing to this master
|
153
|
-
|
159
|
+
redirect_slaves_to(@master)
|
154
160
|
end
|
155
161
|
|
156
162
|
def spawn_watchers
|
@@ -170,12 +176,12 @@ module RedisFailover
|
|
170
176
|
end
|
171
177
|
end
|
172
178
|
|
173
|
-
def
|
179
|
+
def redirect_slaves_to(node)
|
174
180
|
@slaves.dup.each do |slave|
|
175
181
|
begin
|
176
|
-
slave.make_slave!(
|
182
|
+
slave.make_slave!(node)
|
177
183
|
rescue NodeUnavailableError
|
178
|
-
logger.info("Failed to redirect unreachable slave #{slave} to
|
184
|
+
logger.info("Failed to redirect unreachable slave #{slave} to #{node}")
|
179
185
|
force_unavailable_slave(slave)
|
180
186
|
end
|
181
187
|
end
|
@@ -216,14 +222,14 @@ module RedisFailover
|
|
216
222
|
end
|
217
223
|
|
218
224
|
def delete_path
|
219
|
-
@
|
225
|
+
@zk.delete(@znode)
|
220
226
|
logger.info("Deleted ZooKeeper node #{@znode}")
|
221
227
|
rescue ZK::Exceptions::NoNode => ex
|
222
228
|
logger.info("Tried to delete missing znode: #{ex.inspect}")
|
223
229
|
end
|
224
230
|
|
225
231
|
def create_path
|
226
|
-
@
|
232
|
+
@zk.create(@znode, encode(current_nodes), :ephemeral => true)
|
227
233
|
logger.info("Created ZooKeeper node #{@znode}")
|
228
234
|
rescue ZK::Exceptions::NodeExists
|
229
235
|
# best effort
|
@@ -236,7 +242,7 @@ module RedisFailover
|
|
236
242
|
|
237
243
|
def write_state
|
238
244
|
create_path
|
239
|
-
@
|
245
|
+
@zk.set(@znode, encode(current_nodes))
|
240
246
|
end
|
241
247
|
end
|
242
248
|
end
|
data/lib/redis_failover/util.rb
CHANGED
@@ -14,9 +14,9 @@ module RedisFailover
|
|
14
14
|
# Full set of errors related to connectivity.
|
15
15
|
CONNECTIVITY_ERRORS = [
|
16
16
|
RedisFailover::Error,
|
17
|
-
ZK::Exceptions::
|
18
|
-
|
19
|
-
|
17
|
+
ZK::Exceptions::InterruptedSession,
|
18
|
+
REDIS_ERRORS
|
19
|
+
].flatten.freeze
|
20
20
|
|
21
21
|
def symbolize_keys(hash)
|
22
22
|
Hash[hash.map { |k, v| [k.to_sym, v] }]
|
data/lib/redis_failover.rb
CHANGED
@@ -1,5 +1,6 @@
|
|
1
1
|
require 'zk'
|
2
2
|
require 'set'
|
3
|
+
require 'yaml'
|
3
4
|
require 'redis'
|
4
5
|
require 'thread'
|
5
6
|
require 'logger'
|
@@ -15,6 +16,5 @@ require 'redis_failover/errors'
|
|
15
16
|
require 'redis_failover/client'
|
16
17
|
require 'redis_failover/runner'
|
17
18
|
require 'redis_failover/version'
|
18
|
-
require 'redis_failover/zk_client'
|
19
19
|
require 'redis_failover/node_manager'
|
20
20
|
require 'redis_failover/node_watcher'
|
data/redis_failover.gemspec
CHANGED
@@ -18,7 +18,7 @@ Gem::Specification.new do |gem|
|
|
18
18
|
gem.add_dependency('redis')
|
19
19
|
gem.add_dependency('redis-namespace')
|
20
20
|
gem.add_dependency('multi_json', '>= 1.0', '< 1.3')
|
21
|
-
gem.add_dependency('zk', '~> 0.
|
21
|
+
gem.add_dependency('zk', '~> 0.9')
|
22
22
|
|
23
23
|
gem.add_development_dependency('rake')
|
24
24
|
gem.add_development_dependency('rspec')
|
data/spec/spec_helper.rb
CHANGED
@@ -22,6 +22,7 @@ module RedisFailover
|
|
22
22
|
[master, slave].each { |node| node.extend(RedisStubSupport) }
|
23
23
|
master.make_master!
|
24
24
|
slave.make_slave!(master)
|
25
|
+
@unavailable = []
|
25
26
|
@master = master
|
26
27
|
@slaves = [slave]
|
27
28
|
@nodes_discovered = true
|
@@ -33,6 +34,7 @@ module RedisFailover
|
|
33
34
|
|
34
35
|
def start_processing
|
35
36
|
@thread = Thread.new { start }
|
37
|
+
sleep(1.5)
|
36
38
|
end
|
37
39
|
|
38
40
|
def stop_processing
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: redis_failover
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.7.0
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,11 +9,11 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2012-04-
|
12
|
+
date: 2012-04-24 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: redis
|
16
|
-
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirement: &70345784320360 !ruby/object:Gem::Requirement
|
17
17
|
none: false
|
18
18
|
requirements:
|
19
19
|
- - ! '>='
|
@@ -21,15 +21,10 @@ dependencies:
|
|
21
21
|
version: '0'
|
22
22
|
type: :runtime
|
23
23
|
prerelease: false
|
24
|
-
version_requirements:
|
25
|
-
none: false
|
26
|
-
requirements:
|
27
|
-
- - ! '>='
|
28
|
-
- !ruby/object:Gem::Version
|
29
|
-
version: '0'
|
24
|
+
version_requirements: *70345784320360
|
30
25
|
- !ruby/object:Gem::Dependency
|
31
26
|
name: redis-namespace
|
32
|
-
requirement: !ruby/object:Gem::Requirement
|
27
|
+
requirement: &70345784319940 !ruby/object:Gem::Requirement
|
33
28
|
none: false
|
34
29
|
requirements:
|
35
30
|
- - ! '>='
|
@@ -37,15 +32,10 @@ dependencies:
|
|
37
32
|
version: '0'
|
38
33
|
type: :runtime
|
39
34
|
prerelease: false
|
40
|
-
version_requirements:
|
41
|
-
none: false
|
42
|
-
requirements:
|
43
|
-
- - ! '>='
|
44
|
-
- !ruby/object:Gem::Version
|
45
|
-
version: '0'
|
35
|
+
version_requirements: *70345784319940
|
46
36
|
- !ruby/object:Gem::Dependency
|
47
37
|
name: multi_json
|
48
|
-
requirement: !ruby/object:Gem::Requirement
|
38
|
+
requirement: &70345784319420 !ruby/object:Gem::Requirement
|
49
39
|
none: false
|
50
40
|
requirements:
|
51
41
|
- - ! '>='
|
@@ -56,34 +46,21 @@ dependencies:
|
|
56
46
|
version: '1.3'
|
57
47
|
type: :runtime
|
58
48
|
prerelease: false
|
59
|
-
version_requirements:
|
60
|
-
none: false
|
61
|
-
requirements:
|
62
|
-
- - ! '>='
|
63
|
-
- !ruby/object:Gem::Version
|
64
|
-
version: '1.0'
|
65
|
-
- - <
|
66
|
-
- !ruby/object:Gem::Version
|
67
|
-
version: '1.3'
|
49
|
+
version_requirements: *70345784319420
|
68
50
|
- !ruby/object:Gem::Dependency
|
69
51
|
name: zk
|
70
|
-
requirement: !ruby/object:Gem::Requirement
|
52
|
+
requirement: &70345784318660 !ruby/object:Gem::Requirement
|
71
53
|
none: false
|
72
54
|
requirements:
|
73
55
|
- - ~>
|
74
56
|
- !ruby/object:Gem::Version
|
75
|
-
version: 0.
|
57
|
+
version: '0.9'
|
76
58
|
type: :runtime
|
77
59
|
prerelease: false
|
78
|
-
version_requirements:
|
79
|
-
none: false
|
80
|
-
requirements:
|
81
|
-
- - ~>
|
82
|
-
- !ruby/object:Gem::Version
|
83
|
-
version: 0.8.8
|
60
|
+
version_requirements: *70345784318660
|
84
61
|
- !ruby/object:Gem::Dependency
|
85
62
|
name: rake
|
86
|
-
requirement: !ruby/object:Gem::Requirement
|
63
|
+
requirement: &70345784318280 !ruby/object:Gem::Requirement
|
87
64
|
none: false
|
88
65
|
requirements:
|
89
66
|
- - ! '>='
|
@@ -91,15 +68,10 @@ dependencies:
|
|
91
68
|
version: '0'
|
92
69
|
type: :development
|
93
70
|
prerelease: false
|
94
|
-
version_requirements:
|
95
|
-
none: false
|
96
|
-
requirements:
|
97
|
-
- - ! '>='
|
98
|
-
- !ruby/object:Gem::Version
|
99
|
-
version: '0'
|
71
|
+
version_requirements: *70345784318280
|
100
72
|
- !ruby/object:Gem::Dependency
|
101
73
|
name: rspec
|
102
|
-
requirement: !ruby/object:Gem::Requirement
|
74
|
+
requirement: &70345784317820 !ruby/object:Gem::Requirement
|
103
75
|
none: false
|
104
76
|
requirements:
|
105
77
|
- - ! '>='
|
@@ -107,12 +79,7 @@ dependencies:
|
|
107
79
|
version: '0'
|
108
80
|
type: :development
|
109
81
|
prerelease: false
|
110
|
-
version_requirements:
|
111
|
-
none: false
|
112
|
-
requirements:
|
113
|
-
- - ! '>='
|
114
|
-
- !ruby/object:Gem::Version
|
115
|
-
version: '0'
|
82
|
+
version_requirements: *70345784317820
|
116
83
|
description: Redis Failover is a ZooKeeper-based automatic master/slave failover solution
|
117
84
|
for Ruby
|
118
85
|
email:
|
@@ -130,6 +97,7 @@ files:
|
|
130
97
|
- README.md
|
131
98
|
- Rakefile
|
132
99
|
- bin/redis_node_manager
|
100
|
+
- examples/config.yml
|
133
101
|
- lib/redis_failover.rb
|
134
102
|
- lib/redis_failover/cli.rb
|
135
103
|
- lib/redis_failover/client.rb
|
@@ -140,7 +108,6 @@ files:
|
|
140
108
|
- lib/redis_failover/runner.rb
|
141
109
|
- lib/redis_failover/util.rb
|
142
110
|
- lib/redis_failover/version.rb
|
143
|
-
- lib/redis_failover/zk_client.rb
|
144
111
|
- redis_failover.gemspec
|
145
112
|
- spec/cli_spec.rb
|
146
113
|
- spec/client_spec.rb
|
@@ -165,7 +132,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
165
132
|
version: '0'
|
166
133
|
segments:
|
167
134
|
- 0
|
168
|
-
hash:
|
135
|
+
hash: -4131531819785678189
|
169
136
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
170
137
|
none: false
|
171
138
|
requirements:
|
@@ -174,10 +141,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
174
141
|
version: '0'
|
175
142
|
segments:
|
176
143
|
- 0
|
177
|
-
hash:
|
144
|
+
hash: -4131531819785678189
|
178
145
|
requirements: []
|
179
146
|
rubyforge_project:
|
180
|
-
rubygems_version: 1.8.
|
147
|
+
rubygems_version: 1.8.16
|
181
148
|
signing_key:
|
182
149
|
specification_version: 3
|
183
150
|
summary: Redis Failover is a ZooKeeper-based automatic master/slave failover solution
|
@@ -1,93 +0,0 @@
|
|
1
|
-
module RedisFailover
|
2
|
-
# ZkClient is a thin wrapper over the ZK client to gracefully handle reconnects
|
3
|
-
# when a session expires.
|
4
|
-
class ZkClient
|
5
|
-
include Util
|
6
|
-
|
7
|
-
# Time to sleep before retrying a failed operation.
|
8
|
-
TIMEOUT = 2
|
9
|
-
|
10
|
-
# Maximum reconnect attempts.
|
11
|
-
MAX_RECONNECTS = 3
|
12
|
-
|
13
|
-
# Errors that are candidates for rebuilding the underlying ZK client.
|
14
|
-
RECONNECTABLE_ERRORS = [
|
15
|
-
ZookeeperExceptions::ZookeeperException::SessionExpired,
|
16
|
-
ZookeeperExceptions::ZookeeperException::SystemError,
|
17
|
-
ZookeeperExceptions::ZookeeperException::ConnectionLoss,
|
18
|
-
ZookeeperExceptions::ZookeeperException::OperationTimeOut,
|
19
|
-
ZookeeperExceptions::ZookeeperException::AuthFailed,
|
20
|
-
ZookeeperExceptions::ZookeeperException::SessionMoved,
|
21
|
-
ZookeeperExceptions::ZookeeperException::ConnectionClosed,
|
22
|
-
ZookeeperExceptions::ZookeeperException::NotConnected
|
23
|
-
].freeze
|
24
|
-
|
25
|
-
# ZK methods that are wrapped with reconnect logic.
|
26
|
-
WRAPPED_ZK_METHODS = [
|
27
|
-
:get,
|
28
|
-
:set,
|
29
|
-
:watcher,
|
30
|
-
:event_handler,
|
31
|
-
:stat,
|
32
|
-
:create,
|
33
|
-
:delete,
|
34
|
-
:with_lock].freeze
|
35
|
-
|
36
|
-
def initialize(servers, &setup_block)
|
37
|
-
@servers = servers
|
38
|
-
@setup_block = setup_block
|
39
|
-
@lock = Mutex.new
|
40
|
-
build_client
|
41
|
-
end
|
42
|
-
|
43
|
-
def on_session_expiration(&block)
|
44
|
-
@client.on_expired_session { block.call }
|
45
|
-
@on_session_expiration = block
|
46
|
-
end
|
47
|
-
|
48
|
-
def on_session_recovered(&block)
|
49
|
-
@client.on_connected { block.call }
|
50
|
-
@on_session_recovered = block
|
51
|
-
end
|
52
|
-
|
53
|
-
WRAPPED_ZK_METHODS.each do |zk_method|
|
54
|
-
class_eval(<<-RUBY, __FILE__, __LINE__ + 1)
|
55
|
-
def #{zk_method}(*args, &block)
|
56
|
-
perform_with_reconnect do
|
57
|
-
@client.#{zk_method}(*args, &block)
|
58
|
-
end
|
59
|
-
end
|
60
|
-
RUBY
|
61
|
-
end
|
62
|
-
|
63
|
-
private
|
64
|
-
|
65
|
-
def perform_with_reconnect
|
66
|
-
tries = 0
|
67
|
-
begin
|
68
|
-
yield
|
69
|
-
rescue *RECONNECTABLE_ERRORS => ex
|
70
|
-
logger.error("ZooKeeper connection error, rebuilding client: #{ex.inspect}")
|
71
|
-
logger.error(ex.backtrace.join("\n"))
|
72
|
-
if tries < MAX_RECONNECTS
|
73
|
-
tries += 1
|
74
|
-
@on_session_expiration.call if @on_session_expiration
|
75
|
-
build_client
|
76
|
-
@on_session_recovered.call if @on_session_recovered
|
77
|
-
sleep(TIMEOUT)
|
78
|
-
retry
|
79
|
-
end
|
80
|
-
|
81
|
-
raise
|
82
|
-
end
|
83
|
-
end
|
84
|
-
|
85
|
-
def build_client
|
86
|
-
@lock.synchronize do
|
87
|
-
@client = ZK.new(@servers)
|
88
|
-
@setup_block.call(self) if @setup_block
|
89
|
-
logger.info("Communicating with ZooKeeper servers #{@servers}")
|
90
|
-
end
|
91
|
-
end
|
92
|
-
end
|
93
|
-
end
|