spbtv_redis_failover 1.0.2.1

data/spec/node_spec.rb

@@ -0,0 +1,84 @@
+require 'spec_helper'
+
+module RedisFailover
+  describe Node do
+    let(:node) { Node.new(:host => 'localhost', :port => '123') }
+
+    before(:each) do
+      node.extend(RedisStubSupport)
+    end
+
+    describe '#initialize' do
+      it 'creates a new instance' do
+        node.host.should == 'localhost'
+        node.port.should == 123
+      end
+
+      it 'reports error if host missing' do
+        expect { Node.new }.to raise_error(InvalidNodeError)
+      end
+    end
+
+    describe '#ping' do
+      it 'responds properly if node is available' do
+        expect { node.ping }.to_not raise_error
+      end
+
+      it 'responds properly if node is unavailable' do
+        node.redis.make_unavailable!
+        expect { node.ping }.to raise_error(NodeUnavailableError)
+      end
+    end
+
+    describe '#master?' do
+      it 'responds properly if node is master' do
+        node.should be_master
+      end
+
+      it 'responds properly if node is not master' do
+        node.make_slave!(Node.new(:host => 'masterhost'))
+        node.should_not be_master
+      end
+    end
+
+
+    describe '#slave?' do
+      it 'responds properly if node is slave' do
+        node.should_not be_slave
+      end
+
+      it 'responds properly if node is not slave' do
+        node.make_master!
+        node.should be_master
+      end
+    end
+
+    describe '#wait' do
+      it 'should wait until node dies' do
+        thread = Thread.new { node.wait }
+        thread.should be_alive
+        node.redis.make_unavailable!
+        expect { thread.value }.to raise_error
+      end
+    end
+
+    describe '#wakeup' do
+      it 'should gracefully stop waiting' do
+        thread = Thread.new { node.wait }
+        thread.should be_alive
+        node.wakeup
+        sleep 2
+        thread.should_not be_alive
+        thread.value.should be_nil
+      end
+    end
+
+    describe '#perform_operation' do
+      it 'raises error for any operation that hangs for too long' do
+        expect do
+          node.send(:perform_operation) { 1_000_000.times { sleep 0.1 } }
+        end.to raise_error(NodeUnavailableError)
+      end
+    end
+  end
+end

data/spec/node_strategy/consensus_spec.rb

@@ -0,0 +1,30 @@
+require 'spec_helper'
+
+module RedisFailover
+  class NodeStrategy
+    NodeStrategy.for(:consensus)
+
+    describe Consensus do
+      let(:node) { Node.new(:host => 'localhost', :port => '123') }
+      let(:snapshot) { NodeSnapshot.new(node) }
+
+      describe '#determine_state' do
+        it 'returns the unavailable state if unavailable by all node managers' do
+          strategy = NodeStrategy.for(:consensus)
+          snapshot.unviewable_by('nm1')
+          snapshot.unviewable_by('nm2')
+          snapshot.unviewable_by('nm3')
+          strategy.determine_state(node, node => snapshot).should == :unavailable
+        end
+
+        it 'returns the available state if unavailable by some node managers' do
+          strategy = NodeStrategy.for(:consensus)
+          snapshot.unviewable_by('nm1')
+          snapshot.unviewable_by('nm2')
+          snapshot.viewable_by('nm3', 0)
+          strategy.determine_state(node, node => snapshot).should == :available
+        end
+      end
+    end
+  end
+end

data/spec/node_strategy/majority_spec.rb

@@ -0,0 +1,22 @@
+require 'spec_helper'
+
+module RedisFailover
+  class NodeStrategy
+    NodeStrategy.for(:majority)
+
+    describe Majority do
+      let(:node) { Node.new(:host => 'localhost', :port => '123') }
+      let(:snapshot) { NodeSnapshot.new(node) }
+
+      describe '#determine_state' do
+        it 'returns the unavailable state if unavailable by the majority of node managers' do
+          strategy = NodeStrategy.for(:majority)
+          snapshot.viewable_by('nm1', 0)
+          snapshot.unviewable_by('nm2')
+          snapshot.unviewable_by('nm3')
+          strategy.determine_state(node, node => snapshot).should == :unavailable
+        end
+      end
+    end
+  end
+end

data/spec/node_strategy/single_spec.rb

@@ -0,0 +1,22 @@
+require 'spec_helper'
+
+module RedisFailover
+  class NodeStrategy
+    NodeStrategy.for(:single)
+
+    describe Single do
+      let(:node) { Node.new(:host => 'localhost', :port => '123') }
+      let(:snapshot) { NodeSnapshot.new(node) }
+
+      describe '#determine_state' do
+        it 'returns the unavailable state if any node manager reports as down' do
+          strategy = NodeStrategy.for(:single)
+          snapshot.unviewable_by('nm1')
+          snapshot.viewable_by('nm2', 0)
+          snapshot.viewable_by('nm3', 0)
+          strategy.determine_state(node, node => snapshot).should == :unavailable
+        end
+      end
+    end
+  end
+end

data/spec/node_strategy_spec.rb

@@ -0,0 +1,22 @@
+require 'spec_helper'
+
+module RedisFailover
+  describe NodeStrategy do
+
+    describe '.for' do
+      it 'creates a new majority strategy instance' do
+        s = NodeStrategy.for('majority')
+        s.should be_a RedisFailover::NodeStrategy::Majority
+      end
+
+      it 'creates a new consensus strategy instance' do
+        s = NodeStrategy.for('consensus')
+        s.should be_a RedisFailover::NodeStrategy::Consensus
+      end
+
+      it 'rejects unknown strategies' do
+        expect { NodeStrategy.for('foobar') }.to raise_error(RuntimeError)
+      end
+    end
+  end
+end

data/spec/node_watcher_spec.rb

@@ -0,0 +1,58 @@
+require 'spec_helper'
+
+module RedisFailover
+  class LightNodeManager
+    def initialize
+      @node_states = {}
+    end
+
+    def notify_state(node, state, latency = nil)
+      @node_states[node] = state
+    end
+
+    def state_for(node)
+      @node_states[node]
+    end
+  end
+
+  describe NodeWatcher do
+    let(:node_manager) { LightNodeManager.new }
+    let(:node) { Node.new(:host => 'host', :port => 123).extend(RedisStubSupport) }
+
+    describe '#watch' do
+      context 'node is not syncing with master' do
+        it 'properly informs manager of unavailable node' do
+          watcher = NodeWatcher.new(node_manager, node, 1)
+          watcher.watch
+          sleep(3)
+          node.redis.make_unavailable!
+          sleep(3)
+          watcher.shutdown
+          node_manager.state_for(node).should == :unavailable
+        end
+
+        it 'properly informs manager of available node' do
+          node_manager.notify_state(node, :unavailable)
+          watcher = NodeWatcher.new(node_manager, node, 1)
+          watcher.watch
+          sleep(3)
+          watcher.shutdown
+          node_manager.state_for(node).should == :available
+        end
+      end
+
+      context 'node is syncing with master' do
+        it 'properly informs manager of syncing node' do
+          node_manager.notify_state(node, :unavailable)
+          node.redis.slaveof('masterhost', 9876)
+          node.redis.force_sync_with_master(true)
+          watcher = NodeWatcher.new(node_manager, node, 1)
+          watcher.watch
+          sleep(3)
+          watcher.shutdown
+          node_manager.state_for(node).should == :available
+        end
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,21 @@
+require 'rspec'
+require 'ostruct'
+require 'redis_failover'
+
+Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each { |f| require f }
+
+class NullObject
+  def method_missing(method, *args, &block)
+    yield if block_given?
+    self
+  end
+end
+
+module RedisFailover
+  Util.logger = NullObject.new
+end
+
+def ZK.new(*args); NullObject.new; end
+
+RSpec.configure do |config|
+end

data/spec/support/config/multiple_environments.yml

@@ -0,0 +1,15 @@
+:development:
+  :nodes:
+    - localhost:6379
+    - localhost:6389
+  :zkservers:
+    - localhost:2181
+
+:staging:
+  :nodes:
+    - redis01:6379
+    - redis02:6379
+  :zkservers:
+    - zk01:2181
+    - zk02:2181
+    - zk03:2181

data/spec/support/config/multiple_environments_with_chroot.yml

@@ -0,0 +1,17 @@
+:development:
+  :nodes:
+    - localhost:6379
+    - localhost:6389
+  :zkservers:
+    - localhost:2181
+  :chroot: /with/chroot_development
+
+:staging:
+  :nodes:
+    - redis01:6379
+    - redis02:6379
+  :zkservers:
+    - zk01:2181
+    - zk02:2181
+    - zk03:2181
+  :chroot: /with/chroot_staging

data/spec/support/config/single_environment.yml

@@ -0,0 +1,7 @@
+:nodes:
+  - redis01:6379
+  - redis02:6379
+:zkservers:
+  - zk01:2181
+  - zk02:2181
+  - zk03:2181

data/spec/support/config/single_environment_with_chroot.yml

@@ -0,0 +1,8 @@
+:nodes:
+  - redis01:6379
+  - redis02:6379
+:zkservers:
+  - zk01:2181
+  - zk02:2181
+  - zk03:2181
+:chroot: /with/chroot

data/spec/support/node_manager_stub.rb

@@ -0,0 +1,87 @@
+module RedisFailover
+  class NodeManagerStub < NodeManager
+    attr_accessor :master
+    # HACK - this will go away once we refactor the tests to use a real ZK/Redis server.
+    public :current_nodes, :guess_master
+
+    def discover_nodes
+      # only discover nodes once in testing
+      return true if @nodes_discovered
+
+      master = Node.new(:host => 'master')
+      slave = Node.new(:host => 'slave')
+      [master, slave].each { |node| node.extend(RedisStubSupport) }
+      master.make_master!
+      slave.make_slave!(master)
+      @nodes = [master, slave]
+      @unavailable = []
+      @master = master
+      @slaves = [slave]
+      @failover_strategy = Object.new
+      @nodes_discovered = true
+    end
+
+    def setup_zk
+      @zk = NullObject.new
+    end
+
+    def slaves
+      @slaves
+    end
+
+    def start_processing
+      @thread = Thread.new { start }
+      sleep(1.5)
+    end
+
+    def stop_processing
+      @thread.value
+    end
+
+    def force_unavailable(node)
+      start_processing
+      node.redis.make_unavailable!
+      snapshot = OpenStruct.new(
+        :node => node,
+        :available_count => 0,
+        :unavailable_count => 1,
+        :node_managers => ['nm'])
+      update_master_state(node, node => snapshot)
+      stop_processing
+    end
+
+    def force_available(node)
+      start_processing
+      node.redis.make_available!
+      snapshot = OpenStruct.new(
+        :node => node,
+        :available_count => 1,
+        :unavailable_count => 0,
+        :node_managers => ['nm'])
+      update_master_state(node, node => snapshot)
+      stop_processing
+    end
+
+    def force_syncing(node, serve_stale_reads)
+      start_processing
+      node.redis.force_sync_with_master(serve_stale_reads)
+      snapshot = OpenStruct.new(
+        :node => node,
+        :available_count => 1,
+        :unavailable_count => 0,
+        :node_managers => ['nm'])
+      update_master_state(node, node => snapshot)
+      stop_processing
+    end
+
+    def failover_strategy_candidate(snapshots)
+      @slaves.pop
+    end
+
+    def delete_path(*args); end
+    def create_path(*args); end
+    def write_state(*args); end
+    def wait_until_master; end
+    def current_node_snapshots; {} end
+  end
+end

data/spec/support/redis_stub.rb

@@ -0,0 +1,105 @@
+require 'ostruct'
+
+module RedisFailover
+  # Test stub for Redis.
+  class RedisStub
+    class Proxy
+      def initialize(queue, opts = {})
+        @info = {'role' => 'master'}
+        @config = {'slave-serve-stale-data' => 'yes'}
+        @queue = queue
+      end
+
+      def blpop(*args)
+        @queue.pop.tap do |value|
+          raise value if value
+        end
+      end
+
+      def del(*args)
+      end
+
+      def lpush(*args)
+        @queue << nil
+      end
+
+      def slaveof(host, port)
+        if host == 'no' && port == 'one'
+          @info['role'] = 'master'
+          @info.delete('master_host')
+          @info.delete('master_port')
+        else
+          @info['role'] = 'slave'
+          @info['master_host'] = host
+          @info['master_port'] = port.to_s
+        end
+      end
+
+      def info
+        @info.dup
+      end
+
+      def change_role_to(role)
+        @info['role'] = role
+      end
+
+      def config(action, attribute)
+        [action, @config[attribute]]
+      end
+
+      def force_sync_with_master(serve_stale_reads)
+        @config['slave-serve-stale-data'] = serve_stale_reads ? 'yes' : 'no'
+        @info['master_sync_in_progress'] = '1'
+      end
+
+      def force_sync_done
+        @info['master_sync_in_progress'] = '0'
+      end
+    end
+
+    attr_reader :host, :port, :available
+    def initialize(opts = {})
+      @host = opts[:host]
+      @port = Integer(opts[:port])
+      @queue = Queue.new
+      @proxy = Proxy.new(@queue, opts)
+      @available = true
+    end
+
+    def method_missing(method, *args, &block)
+      if @available
+        @proxy.send(method, *args, &block)
+      else
+        raise Errno::ECONNREFUSED
+      end
+    end
+
+    def change_role_to(role)
+      @proxy.change_role_to(role)
+    end
+
+    def make_available!
+      @available = true
+    end
+
+    def make_unavailable!
+      @queue << Errno::ECONNREFUSED
+      @available = false
+    end
+
+    def to_s
+      "#{@host}:#{@port}"
+    end
+
+    def client
+      OpenStruct.new(:host => @host, :port => @port)
+    end
+  end
+
+  module RedisStubSupport
+    def redis
+      @redis ||= RedisStub.new(:host => @host, :port => @port)
+    end
+    alias_method :new_client, :redis
+  end
+end