spbtv_redis_failover 1.0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +19 -0
- data/.travis.yml +7 -0
- data/.yardopts +6 -0
- data/Changes.md +191 -0
- data/Gemfile +2 -0
- data/LICENSE +22 -0
- data/README.md +240 -0
- data/Rakefile +9 -0
- data/bin/redis_node_manager +7 -0
- data/examples/config.yml +17 -0
- data/examples/multiple_environments_config.yml +15 -0
- data/lib/redis_failover.rb +25 -0
- data/lib/redis_failover/cli.rb +142 -0
- data/lib/redis_failover/client.rb +517 -0
- data/lib/redis_failover/errors.rb +54 -0
- data/lib/redis_failover/failover_strategy.rb +25 -0
- data/lib/redis_failover/failover_strategy/latency.rb +21 -0
- data/lib/redis_failover/manual_failover.rb +52 -0
- data/lib/redis_failover/node.rb +190 -0
- data/lib/redis_failover/node_manager.rb +741 -0
- data/lib/redis_failover/node_snapshot.rb +81 -0
- data/lib/redis_failover/node_strategy.rb +34 -0
- data/lib/redis_failover/node_strategy/consensus.rb +18 -0
- data/lib/redis_failover/node_strategy/majority.rb +18 -0
- data/lib/redis_failover/node_strategy/single.rb +17 -0
- data/lib/redis_failover/node_watcher.rb +83 -0
- data/lib/redis_failover/runner.rb +27 -0
- data/lib/redis_failover/util.rb +137 -0
- data/lib/redis_failover/version.rb +3 -0
- data/misc/redis_failover.png +0 -0
- data/spbtv_redis_failover.gemspec +26 -0
- data/spec/cli_spec.rb +75 -0
- data/spec/client_spec.rb +153 -0
- data/spec/failover_strategy/latency_spec.rb +41 -0
- data/spec/failover_strategy_spec.rb +17 -0
- data/spec/node_manager_spec.rb +136 -0
- data/spec/node_snapshot_spec.rb +30 -0
- data/spec/node_spec.rb +84 -0
- data/spec/node_strategy/consensus_spec.rb +30 -0
- data/spec/node_strategy/majority_spec.rb +22 -0
- data/spec/node_strategy/single_spec.rb +22 -0
- data/spec/node_strategy_spec.rb +22 -0
- data/spec/node_watcher_spec.rb +58 -0
- data/spec/spec_helper.rb +21 -0
- data/spec/support/config/multiple_environments.yml +15 -0
- data/spec/support/config/multiple_environments_with_chroot.yml +17 -0
- data/spec/support/config/single_environment.yml +7 -0
- data/spec/support/config/single_environment_with_chroot.yml +8 -0
- data/spec/support/node_manager_stub.rb +87 -0
- data/spec/support/redis_stub.rb +105 -0
- data/spec/util_spec.rb +21 -0
- metadata +207 -0
data/spec/node_spec.rb
ADDED
@@ -0,0 +1,84 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
module RedisFailover
|
4
|
+
describe Node do
|
5
|
+
let(:node) { Node.new(:host => 'localhost', :port => '123') }
|
6
|
+
|
7
|
+
before(:each) do
|
8
|
+
node.extend(RedisStubSupport)
|
9
|
+
end
|
10
|
+
|
11
|
+
describe '#initialize' do
|
12
|
+
it 'creates a new instance' do
|
13
|
+
node.host.should == 'localhost'
|
14
|
+
node.port.should == 123
|
15
|
+
end
|
16
|
+
|
17
|
+
it 'reports error if host missing' do
|
18
|
+
expect { Node.new }.to raise_error(InvalidNodeError)
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
describe '#ping' do
|
23
|
+
it 'responds properly if node is available' do
|
24
|
+
expect { node.ping }.to_not raise_error
|
25
|
+
end
|
26
|
+
|
27
|
+
it 'responds properly if node is unavailable' do
|
28
|
+
node.redis.make_unavailable!
|
29
|
+
expect { node.ping }.to raise_error(NodeUnavailableError)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
describe '#master?' do
|
34
|
+
it 'responds properly if node is master' do
|
35
|
+
node.should be_master
|
36
|
+
end
|
37
|
+
|
38
|
+
it 'responds properly if node is not master' do
|
39
|
+
node.make_slave!(Node.new(:host => 'masterhost'))
|
40
|
+
node.should_not be_master
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
|
45
|
+
describe '#slave?' do
|
46
|
+
it 'responds properly if node is slave' do
|
47
|
+
node.should_not be_slave
|
48
|
+
end
|
49
|
+
|
50
|
+
it 'responds properly if node is not slave' do
|
51
|
+
node.make_master!
|
52
|
+
node.should be_master
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
describe '#wait' do
|
57
|
+
it 'should wait until node dies' do
|
58
|
+
thread = Thread.new { node.wait }
|
59
|
+
thread.should be_alive
|
60
|
+
node.redis.make_unavailable!
|
61
|
+
expect { thread.value }.to raise_error
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
describe '#wakeup' do
|
66
|
+
it 'should gracefully stop waiting' do
|
67
|
+
thread = Thread.new { node.wait }
|
68
|
+
thread.should be_alive
|
69
|
+
node.wakeup
|
70
|
+
sleep 2
|
71
|
+
thread.should_not be_alive
|
72
|
+
thread.value.should be_nil
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
describe '#perform_operation' do
|
77
|
+
it 'raises error for any operation that hangs for too long' do
|
78
|
+
expect do
|
79
|
+
node.send(:perform_operation) { 1_000_000.times { sleep 0.1 } }
|
80
|
+
end.to raise_error(NodeUnavailableError)
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
module RedisFailover
|
4
|
+
class NodeStrategy
|
5
|
+
NodeStrategy.for(:consensus)
|
6
|
+
|
7
|
+
describe Consensus do
|
8
|
+
let(:node) { Node.new(:host => 'localhost', :port => '123') }
|
9
|
+
let(:snapshot) { NodeSnapshot.new(node) }
|
10
|
+
|
11
|
+
describe '#determine_state' do
|
12
|
+
it 'returns the unavailable state if unavailable by all node managers' do
|
13
|
+
strategy = NodeStrategy.for(:consensus)
|
14
|
+
snapshot.unviewable_by('nm1')
|
15
|
+
snapshot.unviewable_by('nm2')
|
16
|
+
snapshot.unviewable_by('nm3')
|
17
|
+
strategy.determine_state(node, node => snapshot).should == :unavailable
|
18
|
+
end
|
19
|
+
|
20
|
+
it 'returns the available state if unavailable by some node managers' do
|
21
|
+
strategy = NodeStrategy.for(:consensus)
|
22
|
+
snapshot.unviewable_by('nm1')
|
23
|
+
snapshot.unviewable_by('nm2')
|
24
|
+
snapshot.viewable_by('nm3', 0)
|
25
|
+
strategy.determine_state(node, node => snapshot).should == :available
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
module RedisFailover
|
4
|
+
class NodeStrategy
|
5
|
+
NodeStrategy.for(:majority)
|
6
|
+
|
7
|
+
describe Majority do
|
8
|
+
let(:node) { Node.new(:host => 'localhost', :port => '123') }
|
9
|
+
let(:snapshot) { NodeSnapshot.new(node) }
|
10
|
+
|
11
|
+
describe '#determine_state' do
|
12
|
+
it 'returns the unavailable state if unavailable by the majority of node managers' do
|
13
|
+
strategy = NodeStrategy.for(:majority)
|
14
|
+
snapshot.viewable_by('nm1', 0)
|
15
|
+
snapshot.unviewable_by('nm2')
|
16
|
+
snapshot.unviewable_by('nm3')
|
17
|
+
strategy.determine_state(node, node => snapshot).should == :unavailable
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
module RedisFailover
|
4
|
+
class NodeStrategy
|
5
|
+
NodeStrategy.for(:single)
|
6
|
+
|
7
|
+
describe Single do
|
8
|
+
let(:node) { Node.new(:host => 'localhost', :port => '123') }
|
9
|
+
let(:snapshot) { NodeSnapshot.new(node) }
|
10
|
+
|
11
|
+
describe '#determine_state' do
|
12
|
+
it 'returns the unavailable state if any node manager reports as down' do
|
13
|
+
strategy = NodeStrategy.for(:single)
|
14
|
+
snapshot.unviewable_by('nm1')
|
15
|
+
snapshot.viewable_by('nm2', 0)
|
16
|
+
snapshot.viewable_by('nm3', 0)
|
17
|
+
strategy.determine_state(node, node => snapshot).should == :unavailable
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
module RedisFailover
|
4
|
+
describe NodeStrategy do
|
5
|
+
|
6
|
+
describe '.for' do
|
7
|
+
it 'creates a new majority strategy instance' do
|
8
|
+
s = NodeStrategy.for('majority')
|
9
|
+
s.should be_a RedisFailover::NodeStrategy::Majority
|
10
|
+
end
|
11
|
+
|
12
|
+
it 'creates a new consensus strategy instance' do
|
13
|
+
s = NodeStrategy.for('consensus')
|
14
|
+
s.should be_a RedisFailover::NodeStrategy::Consensus
|
15
|
+
end
|
16
|
+
|
17
|
+
it 'rejects unknown strategies' do
|
18
|
+
expect { NodeStrategy.for('foobar') }.to raise_error(RuntimeError)
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,58 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
module RedisFailover
|
4
|
+
class LightNodeManager
|
5
|
+
def initialize
|
6
|
+
@node_states = {}
|
7
|
+
end
|
8
|
+
|
9
|
+
def notify_state(node, state, latency = nil)
|
10
|
+
@node_states[node] = state
|
11
|
+
end
|
12
|
+
|
13
|
+
def state_for(node)
|
14
|
+
@node_states[node]
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
describe NodeWatcher do
|
19
|
+
let(:node_manager) { LightNodeManager.new }
|
20
|
+
let(:node) { Node.new(:host => 'host', :port => 123).extend(RedisStubSupport) }
|
21
|
+
|
22
|
+
describe '#watch' do
|
23
|
+
context 'node is not syncing with master' do
|
24
|
+
it 'properly informs manager of unavailable node' do
|
25
|
+
watcher = NodeWatcher.new(node_manager, node, 1)
|
26
|
+
watcher.watch
|
27
|
+
sleep(3)
|
28
|
+
node.redis.make_unavailable!
|
29
|
+
sleep(3)
|
30
|
+
watcher.shutdown
|
31
|
+
node_manager.state_for(node).should == :unavailable
|
32
|
+
end
|
33
|
+
|
34
|
+
it 'properly informs manager of available node' do
|
35
|
+
node_manager.notify_state(node, :unavailable)
|
36
|
+
watcher = NodeWatcher.new(node_manager, node, 1)
|
37
|
+
watcher.watch
|
38
|
+
sleep(3)
|
39
|
+
watcher.shutdown
|
40
|
+
node_manager.state_for(node).should == :available
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
context 'node is syncing with master' do
|
45
|
+
it 'properly informs manager of syncing node' do
|
46
|
+
node_manager.notify_state(node, :unavailable)
|
47
|
+
node.redis.slaveof('masterhost', 9876)
|
48
|
+
node.redis.force_sync_with_master(true)
|
49
|
+
watcher = NodeWatcher.new(node_manager, node, 1)
|
50
|
+
watcher.watch
|
51
|
+
sleep(3)
|
52
|
+
watcher.shutdown
|
53
|
+
node_manager.state_for(node).should == :available
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
require 'rspec'
|
2
|
+
require 'ostruct'
|
3
|
+
require 'redis_failover'
|
4
|
+
|
5
|
+
Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each { |f| require f }
|
6
|
+
|
7
|
+
class NullObject
|
8
|
+
def method_missing(method, *args, &block)
|
9
|
+
yield if block_given?
|
10
|
+
self
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
module RedisFailover
|
15
|
+
Util.logger = NullObject.new
|
16
|
+
end
|
17
|
+
|
18
|
+
def ZK.new(*args); NullObject.new; end
|
19
|
+
|
20
|
+
RSpec.configure do |config|
|
21
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
:development:
|
2
|
+
:nodes:
|
3
|
+
- localhost:6379
|
4
|
+
- localhost:6389
|
5
|
+
:zkservers:
|
6
|
+
- localhost:2181
|
7
|
+
:chroot: /with/chroot_development
|
8
|
+
|
9
|
+
:staging:
|
10
|
+
:nodes:
|
11
|
+
- redis01:6379
|
12
|
+
- redis02:6379
|
13
|
+
:zkservers:
|
14
|
+
- zk01:2181
|
15
|
+
- zk02:2181
|
16
|
+
- zk03:2181
|
17
|
+
:chroot: /with/chroot_staging
|
@@ -0,0 +1,87 @@
|
|
1
|
+
module RedisFailover
|
2
|
+
class NodeManagerStub < NodeManager
|
3
|
+
attr_accessor :master
|
4
|
+
# HACK - this will go away once we refactor the tests to use a real ZK/Redis server.
|
5
|
+
public :current_nodes, :guess_master
|
6
|
+
|
7
|
+
def discover_nodes
|
8
|
+
# only discover nodes once in testing
|
9
|
+
return true if @nodes_discovered
|
10
|
+
|
11
|
+
master = Node.new(:host => 'master')
|
12
|
+
slave = Node.new(:host => 'slave')
|
13
|
+
[master, slave].each { |node| node.extend(RedisStubSupport) }
|
14
|
+
master.make_master!
|
15
|
+
slave.make_slave!(master)
|
16
|
+
@nodes = [master, slave]
|
17
|
+
@unavailable = []
|
18
|
+
@master = master
|
19
|
+
@slaves = [slave]
|
20
|
+
@failover_strategy = Object.new
|
21
|
+
@nodes_discovered = true
|
22
|
+
end
|
23
|
+
|
24
|
+
def setup_zk
|
25
|
+
@zk = NullObject.new
|
26
|
+
end
|
27
|
+
|
28
|
+
def slaves
|
29
|
+
@slaves
|
30
|
+
end
|
31
|
+
|
32
|
+
def start_processing
|
33
|
+
@thread = Thread.new { start }
|
34
|
+
sleep(1.5)
|
35
|
+
end
|
36
|
+
|
37
|
+
def stop_processing
|
38
|
+
@thread.value
|
39
|
+
end
|
40
|
+
|
41
|
+
def force_unavailable(node)
|
42
|
+
start_processing
|
43
|
+
node.redis.make_unavailable!
|
44
|
+
snapshot = OpenStruct.new(
|
45
|
+
:node => node,
|
46
|
+
:available_count => 0,
|
47
|
+
:unavailable_count => 1,
|
48
|
+
:node_managers => ['nm'])
|
49
|
+
update_master_state(node, node => snapshot)
|
50
|
+
stop_processing
|
51
|
+
end
|
52
|
+
|
53
|
+
def force_available(node)
|
54
|
+
start_processing
|
55
|
+
node.redis.make_available!
|
56
|
+
snapshot = OpenStruct.new(
|
57
|
+
:node => node,
|
58
|
+
:available_count => 1,
|
59
|
+
:unavailable_count => 0,
|
60
|
+
:node_managers => ['nm'])
|
61
|
+
update_master_state(node, node => snapshot)
|
62
|
+
stop_processing
|
63
|
+
end
|
64
|
+
|
65
|
+
def force_syncing(node, serve_stale_reads)
|
66
|
+
start_processing
|
67
|
+
node.redis.force_sync_with_master(serve_stale_reads)
|
68
|
+
snapshot = OpenStruct.new(
|
69
|
+
:node => node,
|
70
|
+
:available_count => 1,
|
71
|
+
:unavailable_count => 0,
|
72
|
+
:node_managers => ['nm'])
|
73
|
+
update_master_state(node, node => snapshot)
|
74
|
+
stop_processing
|
75
|
+
end
|
76
|
+
|
77
|
+
def failover_strategy_candidate(snapshots)
|
78
|
+
@slaves.pop
|
79
|
+
end
|
80
|
+
|
81
|
+
def delete_path(*args); end
|
82
|
+
def create_path(*args); end
|
83
|
+
def write_state(*args); end
|
84
|
+
def wait_until_master; end
|
85
|
+
def current_node_snapshots; {} end
|
86
|
+
end
|
87
|
+
end
|
@@ -0,0 +1,105 @@
|
|
1
|
+
require 'ostruct'
|
2
|
+
|
3
|
+
module RedisFailover
|
4
|
+
# Test stub for Redis.
|
5
|
+
class RedisStub
|
6
|
+
class Proxy
|
7
|
+
def initialize(queue, opts = {})
|
8
|
+
@info = {'role' => 'master'}
|
9
|
+
@config = {'slave-serve-stale-data' => 'yes'}
|
10
|
+
@queue = queue
|
11
|
+
end
|
12
|
+
|
13
|
+
def blpop(*args)
|
14
|
+
@queue.pop.tap do |value|
|
15
|
+
raise value if value
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
def del(*args)
|
20
|
+
end
|
21
|
+
|
22
|
+
def lpush(*args)
|
23
|
+
@queue << nil
|
24
|
+
end
|
25
|
+
|
26
|
+
def slaveof(host, port)
|
27
|
+
if host == 'no' && port == 'one'
|
28
|
+
@info['role'] = 'master'
|
29
|
+
@info.delete('master_host')
|
30
|
+
@info.delete('master_port')
|
31
|
+
else
|
32
|
+
@info['role'] = 'slave'
|
33
|
+
@info['master_host'] = host
|
34
|
+
@info['master_port'] = port.to_s
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
def info
|
39
|
+
@info.dup
|
40
|
+
end
|
41
|
+
|
42
|
+
def change_role_to(role)
|
43
|
+
@info['role'] = role
|
44
|
+
end
|
45
|
+
|
46
|
+
def config(action, attribute)
|
47
|
+
[action, @config[attribute]]
|
48
|
+
end
|
49
|
+
|
50
|
+
def force_sync_with_master(serve_stale_reads)
|
51
|
+
@config['slave-serve-stale-data'] = serve_stale_reads ? 'yes' : 'no'
|
52
|
+
@info['master_sync_in_progress'] = '1'
|
53
|
+
end
|
54
|
+
|
55
|
+
def force_sync_done
|
56
|
+
@info['master_sync_in_progress'] = '0'
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
attr_reader :host, :port, :available
|
61
|
+
def initialize(opts = {})
|
62
|
+
@host = opts[:host]
|
63
|
+
@port = Integer(opts[:port])
|
64
|
+
@queue = Queue.new
|
65
|
+
@proxy = Proxy.new(@queue, opts)
|
66
|
+
@available = true
|
67
|
+
end
|
68
|
+
|
69
|
+
def method_missing(method, *args, &block)
|
70
|
+
if @available
|
71
|
+
@proxy.send(method, *args, &block)
|
72
|
+
else
|
73
|
+
raise Errno::ECONNREFUSED
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
def change_role_to(role)
|
78
|
+
@proxy.change_role_to(role)
|
79
|
+
end
|
80
|
+
|
81
|
+
def make_available!
|
82
|
+
@available = true
|
83
|
+
end
|
84
|
+
|
85
|
+
def make_unavailable!
|
86
|
+
@queue << Errno::ECONNREFUSED
|
87
|
+
@available = false
|
88
|
+
end
|
89
|
+
|
90
|
+
def to_s
|
91
|
+
"#{@host}:#{@port}"
|
92
|
+
end
|
93
|
+
|
94
|
+
def client
|
95
|
+
OpenStruct.new(:host => @host, :port => @port)
|
96
|
+
end
|
97
|
+
end
|
98
|
+
|
99
|
+
module RedisStubSupport
|
100
|
+
def redis
|
101
|
+
@redis ||= RedisStub.new(:host => @host, :port => @port)
|
102
|
+
end
|
103
|
+
alias_method :new_client, :redis
|
104
|
+
end
|
105
|
+
end
|