spbtv_redis_failover 1.0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +19 -0
  3. data/.travis.yml +7 -0
  4. data/.yardopts +6 -0
  5. data/Changes.md +191 -0
  6. data/Gemfile +2 -0
  7. data/LICENSE +22 -0
  8. data/README.md +240 -0
  9. data/Rakefile +9 -0
  10. data/bin/redis_node_manager +7 -0
  11. data/examples/config.yml +17 -0
  12. data/examples/multiple_environments_config.yml +15 -0
  13. data/lib/redis_failover.rb +25 -0
  14. data/lib/redis_failover/cli.rb +142 -0
  15. data/lib/redis_failover/client.rb +517 -0
  16. data/lib/redis_failover/errors.rb +54 -0
  17. data/lib/redis_failover/failover_strategy.rb +25 -0
  18. data/lib/redis_failover/failover_strategy/latency.rb +21 -0
  19. data/lib/redis_failover/manual_failover.rb +52 -0
  20. data/lib/redis_failover/node.rb +190 -0
  21. data/lib/redis_failover/node_manager.rb +741 -0
  22. data/lib/redis_failover/node_snapshot.rb +81 -0
  23. data/lib/redis_failover/node_strategy.rb +34 -0
  24. data/lib/redis_failover/node_strategy/consensus.rb +18 -0
  25. data/lib/redis_failover/node_strategy/majority.rb +18 -0
  26. data/lib/redis_failover/node_strategy/single.rb +17 -0
  27. data/lib/redis_failover/node_watcher.rb +83 -0
  28. data/lib/redis_failover/runner.rb +27 -0
  29. data/lib/redis_failover/util.rb +137 -0
  30. data/lib/redis_failover/version.rb +3 -0
  31. data/misc/redis_failover.png +0 -0
  32. data/spbtv_redis_failover.gemspec +26 -0
  33. data/spec/cli_spec.rb +75 -0
  34. data/spec/client_spec.rb +153 -0
  35. data/spec/failover_strategy/latency_spec.rb +41 -0
  36. data/spec/failover_strategy_spec.rb +17 -0
  37. data/spec/node_manager_spec.rb +136 -0
  38. data/spec/node_snapshot_spec.rb +30 -0
  39. data/spec/node_spec.rb +84 -0
  40. data/spec/node_strategy/consensus_spec.rb +30 -0
  41. data/spec/node_strategy/majority_spec.rb +22 -0
  42. data/spec/node_strategy/single_spec.rb +22 -0
  43. data/spec/node_strategy_spec.rb +22 -0
  44. data/spec/node_watcher_spec.rb +58 -0
  45. data/spec/spec_helper.rb +21 -0
  46. data/spec/support/config/multiple_environments.yml +15 -0
  47. data/spec/support/config/multiple_environments_with_chroot.yml +17 -0
  48. data/spec/support/config/single_environment.yml +7 -0
  49. data/spec/support/config/single_environment_with_chroot.yml +8 -0
  50. data/spec/support/node_manager_stub.rb +87 -0
  51. data/spec/support/redis_stub.rb +105 -0
  52. data/spec/util_spec.rb +21 -0
  53. metadata +207 -0
data/spec/node_spec.rb ADDED
@@ -0,0 +1,84 @@
1
+ require 'spec_helper'
2
+
3
+ module RedisFailover
4
+ describe Node do
5
+ let(:node) { Node.new(:host => 'localhost', :port => '123') }
6
+
7
+ before(:each) do
8
+ node.extend(RedisStubSupport)
9
+ end
10
+
11
+ describe '#initialize' do
12
+ it 'creates a new instance' do
13
+ node.host.should == 'localhost'
14
+ node.port.should == 123
15
+ end
16
+
17
+ it 'reports error if host missing' do
18
+ expect { Node.new }.to raise_error(InvalidNodeError)
19
+ end
20
+ end
21
+
22
+ describe '#ping' do
23
+ it 'responds properly if node is available' do
24
+ expect { node.ping }.to_not raise_error
25
+ end
26
+
27
+ it 'responds properly if node is unavailable' do
28
+ node.redis.make_unavailable!
29
+ expect { node.ping }.to raise_error(NodeUnavailableError)
30
+ end
31
+ end
32
+
33
+ describe '#master?' do
34
+ it 'responds properly if node is master' do
35
+ node.should be_master
36
+ end
37
+
38
+ it 'responds properly if node is not master' do
39
+ node.make_slave!(Node.new(:host => 'masterhost'))
40
+ node.should_not be_master
41
+ end
42
+ end
43
+
44
+
45
+ describe '#slave?' do
46
+ it 'responds properly if node is slave' do
47
+ node.should_not be_slave
48
+ end
49
+
50
+ it 'responds properly if node is not slave' do
51
+ node.make_master!
52
+ node.should be_master
53
+ end
54
+ end
55
+
56
+ describe '#wait' do
57
+ it 'should wait until node dies' do
58
+ thread = Thread.new { node.wait }
59
+ thread.should be_alive
60
+ node.redis.make_unavailable!
61
+ expect { thread.value }.to raise_error
62
+ end
63
+ end
64
+
65
+ describe '#wakeup' do
66
+ it 'should gracefully stop waiting' do
67
+ thread = Thread.new { node.wait }
68
+ thread.should be_alive
69
+ node.wakeup
70
+ sleep 2
71
+ thread.should_not be_alive
72
+ thread.value.should be_nil
73
+ end
74
+ end
75
+
76
+ describe '#perform_operation' do
77
+ it 'raises error for any operation that hangs for too long' do
78
+ expect do
79
+ node.send(:perform_operation) { 1_000_000.times { sleep 0.1 } }
80
+ end.to raise_error(NodeUnavailableError)
81
+ end
82
+ end
83
+ end
84
+ end
@@ -0,0 +1,30 @@
1
+ require 'spec_helper'
2
+
3
+ module RedisFailover
4
+ class NodeStrategy
5
+ NodeStrategy.for(:consensus)
6
+
7
+ describe Consensus do
8
+ let(:node) { Node.new(:host => 'localhost', :port => '123') }
9
+ let(:snapshot) { NodeSnapshot.new(node) }
10
+
11
+ describe '#determine_state' do
12
+ it 'returns the unavailable state if unavailable by all node managers' do
13
+ strategy = NodeStrategy.for(:consensus)
14
+ snapshot.unviewable_by('nm1')
15
+ snapshot.unviewable_by('nm2')
16
+ snapshot.unviewable_by('nm3')
17
+ strategy.determine_state(node, node => snapshot).should == :unavailable
18
+ end
19
+
20
+ it 'returns the available state if unavailable by some node managers' do
21
+ strategy = NodeStrategy.for(:consensus)
22
+ snapshot.unviewable_by('nm1')
23
+ snapshot.unviewable_by('nm2')
24
+ snapshot.viewable_by('nm3', 0)
25
+ strategy.determine_state(node, node => snapshot).should == :available
26
+ end
27
+ end
28
+ end
29
+ end
30
+ end
@@ -0,0 +1,22 @@
1
+ require 'spec_helper'
2
+
3
+ module RedisFailover
4
+ class NodeStrategy
5
+ NodeStrategy.for(:majority)
6
+
7
+ describe Majority do
8
+ let(:node) { Node.new(:host => 'localhost', :port => '123') }
9
+ let(:snapshot) { NodeSnapshot.new(node) }
10
+
11
+ describe '#determine_state' do
12
+ it 'returns the unavailable state if unavailable by the majority of node managers' do
13
+ strategy = NodeStrategy.for(:majority)
14
+ snapshot.viewable_by('nm1', 0)
15
+ snapshot.unviewable_by('nm2')
16
+ snapshot.unviewable_by('nm3')
17
+ strategy.determine_state(node, node => snapshot).should == :unavailable
18
+ end
19
+ end
20
+ end
21
+ end
22
+ end
@@ -0,0 +1,22 @@
1
+ require 'spec_helper'
2
+
3
+ module RedisFailover
4
+ class NodeStrategy
5
+ NodeStrategy.for(:single)
6
+
7
+ describe Single do
8
+ let(:node) { Node.new(:host => 'localhost', :port => '123') }
9
+ let(:snapshot) { NodeSnapshot.new(node) }
10
+
11
+ describe '#determine_state' do
12
+ it 'returns the unavailable state if any node manager reports as down' do
13
+ strategy = NodeStrategy.for(:single)
14
+ snapshot.unviewable_by('nm1')
15
+ snapshot.viewable_by('nm2', 0)
16
+ snapshot.viewable_by('nm3', 0)
17
+ strategy.determine_state(node, node => snapshot).should == :unavailable
18
+ end
19
+ end
20
+ end
21
+ end
22
+ end
@@ -0,0 +1,22 @@
1
+ require 'spec_helper'
2
+
3
+ module RedisFailover
4
+ describe NodeStrategy do
5
+
6
+ describe '.for' do
7
+ it 'creates a new majority strategy instance' do
8
+ s = NodeStrategy.for('majority')
9
+ s.should be_a RedisFailover::NodeStrategy::Majority
10
+ end
11
+
12
+ it 'creates a new consensus strategy instance' do
13
+ s = NodeStrategy.for('consensus')
14
+ s.should be_a RedisFailover::NodeStrategy::Consensus
15
+ end
16
+
17
+ it 'rejects unknown strategies' do
18
+ expect { NodeStrategy.for('foobar') }.to raise_error(RuntimeError)
19
+ end
20
+ end
21
+ end
22
+ end
@@ -0,0 +1,58 @@
1
+ require 'spec_helper'
2
+
3
+ module RedisFailover
4
+ class LightNodeManager
5
+ def initialize
6
+ @node_states = {}
7
+ end
8
+
9
+ def notify_state(node, state, latency = nil)
10
+ @node_states[node] = state
11
+ end
12
+
13
+ def state_for(node)
14
+ @node_states[node]
15
+ end
16
+ end
17
+
18
+ describe NodeWatcher do
19
+ let(:node_manager) { LightNodeManager.new }
20
+ let(:node) { Node.new(:host => 'host', :port => 123).extend(RedisStubSupport) }
21
+
22
+ describe '#watch' do
23
+ context 'node is not syncing with master' do
24
+ it 'properly informs manager of unavailable node' do
25
+ watcher = NodeWatcher.new(node_manager, node, 1)
26
+ watcher.watch
27
+ sleep(3)
28
+ node.redis.make_unavailable!
29
+ sleep(3)
30
+ watcher.shutdown
31
+ node_manager.state_for(node).should == :unavailable
32
+ end
33
+
34
+ it 'properly informs manager of available node' do
35
+ node_manager.notify_state(node, :unavailable)
36
+ watcher = NodeWatcher.new(node_manager, node, 1)
37
+ watcher.watch
38
+ sleep(3)
39
+ watcher.shutdown
40
+ node_manager.state_for(node).should == :available
41
+ end
42
+ end
43
+
44
+ context 'node is syncing with master' do
45
+ it 'properly informs manager of syncing node' do
46
+ node_manager.notify_state(node, :unavailable)
47
+ node.redis.slaveof('masterhost', 9876)
48
+ node.redis.force_sync_with_master(true)
49
+ watcher = NodeWatcher.new(node_manager, node, 1)
50
+ watcher.watch
51
+ sleep(3)
52
+ watcher.shutdown
53
+ node_manager.state_for(node).should == :available
54
+ end
55
+ end
56
+ end
57
+ end
58
+ end
@@ -0,0 +1,21 @@
1
+ require 'rspec'
2
+ require 'ostruct'
3
+ require 'redis_failover'
4
+
5
+ Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each { |f| require f }
6
+
7
+ class NullObject
8
+ def method_missing(method, *args, &block)
9
+ yield if block_given?
10
+ self
11
+ end
12
+ end
13
+
14
+ module RedisFailover
15
+ Util.logger = NullObject.new
16
+ end
17
+
18
+ def ZK.new(*args); NullObject.new; end
19
+
20
+ RSpec.configure do |config|
21
+ end
@@ -0,0 +1,15 @@
1
+ :development:
2
+ :nodes:
3
+ - localhost:6379
4
+ - localhost:6389
5
+ :zkservers:
6
+ - localhost:2181
7
+
8
+ :staging:
9
+ :nodes:
10
+ - redis01:6379
11
+ - redis02:6379
12
+ :zkservers:
13
+ - zk01:2181
14
+ - zk02:2181
15
+ - zk03:2181
@@ -0,0 +1,17 @@
1
+ :development:
2
+ :nodes:
3
+ - localhost:6379
4
+ - localhost:6389
5
+ :zkservers:
6
+ - localhost:2181
7
+ :chroot: /with/chroot_development
8
+
9
+ :staging:
10
+ :nodes:
11
+ - redis01:6379
12
+ - redis02:6379
13
+ :zkservers:
14
+ - zk01:2181
15
+ - zk02:2181
16
+ - zk03:2181
17
+ :chroot: /with/chroot_staging
@@ -0,0 +1,7 @@
1
+ :nodes:
2
+ - redis01:6379
3
+ - redis02:6379
4
+ :zkservers:
5
+ - zk01:2181
6
+ - zk02:2181
7
+ - zk03:2181
@@ -0,0 +1,8 @@
1
+ :nodes:
2
+ - redis01:6379
3
+ - redis02:6379
4
+ :zkservers:
5
+ - zk01:2181
6
+ - zk02:2181
7
+ - zk03:2181
8
+ :chroot: /with/chroot
@@ -0,0 +1,87 @@
1
+ module RedisFailover
2
+ class NodeManagerStub < NodeManager
3
+ attr_accessor :master
4
+ # HACK - this will go away once we refactor the tests to use a real ZK/Redis server.
5
+ public :current_nodes, :guess_master
6
+
7
+ def discover_nodes
8
+ # only discover nodes once in testing
9
+ return true if @nodes_discovered
10
+
11
+ master = Node.new(:host => 'master')
12
+ slave = Node.new(:host => 'slave')
13
+ [master, slave].each { |node| node.extend(RedisStubSupport) }
14
+ master.make_master!
15
+ slave.make_slave!(master)
16
+ @nodes = [master, slave]
17
+ @unavailable = []
18
+ @master = master
19
+ @slaves = [slave]
20
+ @failover_strategy = Object.new
21
+ @nodes_discovered = true
22
+ end
23
+
24
+ def setup_zk
25
+ @zk = NullObject.new
26
+ end
27
+
28
+ def slaves
29
+ @slaves
30
+ end
31
+
32
+ def start_processing
33
+ @thread = Thread.new { start }
34
+ sleep(1.5)
35
+ end
36
+
37
+ def stop_processing
38
+ @thread.value
39
+ end
40
+
41
+ def force_unavailable(node)
42
+ start_processing
43
+ node.redis.make_unavailable!
44
+ snapshot = OpenStruct.new(
45
+ :node => node,
46
+ :available_count => 0,
47
+ :unavailable_count => 1,
48
+ :node_managers => ['nm'])
49
+ update_master_state(node, node => snapshot)
50
+ stop_processing
51
+ end
52
+
53
+ def force_available(node)
54
+ start_processing
55
+ node.redis.make_available!
56
+ snapshot = OpenStruct.new(
57
+ :node => node,
58
+ :available_count => 1,
59
+ :unavailable_count => 0,
60
+ :node_managers => ['nm'])
61
+ update_master_state(node, node => snapshot)
62
+ stop_processing
63
+ end
64
+
65
+ def force_syncing(node, serve_stale_reads)
66
+ start_processing
67
+ node.redis.force_sync_with_master(serve_stale_reads)
68
+ snapshot = OpenStruct.new(
69
+ :node => node,
70
+ :available_count => 1,
71
+ :unavailable_count => 0,
72
+ :node_managers => ['nm'])
73
+ update_master_state(node, node => snapshot)
74
+ stop_processing
75
+ end
76
+
77
+ def failover_strategy_candidate(snapshots)
78
+ @slaves.pop
79
+ end
80
+
81
+ def delete_path(*args); end
82
+ def create_path(*args); end
83
+ def write_state(*args); end
84
+ def wait_until_master; end
85
+ def current_node_snapshots; {} end
86
+ end
87
+ end
@@ -0,0 +1,105 @@
1
+ require 'ostruct'
2
+
3
+ module RedisFailover
4
+ # Test stub for Redis.
5
+ class RedisStub
6
+ class Proxy
7
+ def initialize(queue, opts = {})
8
+ @info = {'role' => 'master'}
9
+ @config = {'slave-serve-stale-data' => 'yes'}
10
+ @queue = queue
11
+ end
12
+
13
+ def blpop(*args)
14
+ @queue.pop.tap do |value|
15
+ raise value if value
16
+ end
17
+ end
18
+
19
+ def del(*args)
20
+ end
21
+
22
+ def lpush(*args)
23
+ @queue << nil
24
+ end
25
+
26
+ def slaveof(host, port)
27
+ if host == 'no' && port == 'one'
28
+ @info['role'] = 'master'
29
+ @info.delete('master_host')
30
+ @info.delete('master_port')
31
+ else
32
+ @info['role'] = 'slave'
33
+ @info['master_host'] = host
34
+ @info['master_port'] = port.to_s
35
+ end
36
+ end
37
+
38
+ def info
39
+ @info.dup
40
+ end
41
+
42
+ def change_role_to(role)
43
+ @info['role'] = role
44
+ end
45
+
46
+ def config(action, attribute)
47
+ [action, @config[attribute]]
48
+ end
49
+
50
+ def force_sync_with_master(serve_stale_reads)
51
+ @config['slave-serve-stale-data'] = serve_stale_reads ? 'yes' : 'no'
52
+ @info['master_sync_in_progress'] = '1'
53
+ end
54
+
55
+ def force_sync_done
56
+ @info['master_sync_in_progress'] = '0'
57
+ end
58
+ end
59
+
60
+ attr_reader :host, :port, :available
61
+ def initialize(opts = {})
62
+ @host = opts[:host]
63
+ @port = Integer(opts[:port])
64
+ @queue = Queue.new
65
+ @proxy = Proxy.new(@queue, opts)
66
+ @available = true
67
+ end
68
+
69
+ def method_missing(method, *args, &block)
70
+ if @available
71
+ @proxy.send(method, *args, &block)
72
+ else
73
+ raise Errno::ECONNREFUSED
74
+ end
75
+ end
76
+
77
+ def change_role_to(role)
78
+ @proxy.change_role_to(role)
79
+ end
80
+
81
+ def make_available!
82
+ @available = true
83
+ end
84
+
85
+ def make_unavailable!
86
+ @queue << Errno::ECONNREFUSED
87
+ @available = false
88
+ end
89
+
90
+ def to_s
91
+ "#{@host}:#{@port}"
92
+ end
93
+
94
+ def client
95
+ OpenStruct.new(:host => @host, :port => @port)
96
+ end
97
+ end
98
+
99
+ module RedisStubSupport
100
+ def redis
101
+ @redis ||= RedisStub.new(:host => @host, :port => @port)
102
+ end
103
+ alias_method :new_client, :redis
104
+ end
105
+ end