spbtv_redis_failover 1.0.2.1

data/spec/cli_spec.rb

@@ -0,0 +1,75 @@
+require 'spec_helper'
+
+module RedisFailover
+  describe CLI do
+    describe '.parse' do
+      it 'properly parses redis nodes' do
+        opts = CLI.parse(['-n host1:1,host2:2,host3:3', '-z localhost:1111'])
+        opts[:nodes].should == [
+          {:host => 'host1', :port => '1'},
+          {:host => 'host2', :port => '2'},
+          {:host => 'host3', :port => '3'}
+        ]
+      end
+
+      it 'properly parses ZooKeeper servers' do
+        opts = CLI.parse(['-n host1:1,host2:2,host3:3', '-z localhost:1111'])
+        opts[:zkservers].should == 'localhost:1111'
+      end
+
+      it 'properly parses a redis password' do
+        opts = CLI.parse(['-n host:port', '-z localhost:1111', '-p redis_pass'])
+        opts[:nodes].should == [{
+          :host => 'host',
+          :port => 'port',
+          :password => 'redis_pass'
+        }]
+      end
+
+      it 'properly parses max node failures' do
+        opts = CLI.parse([
+          '-n host:port',
+          '-z localhost:1111',
+          '-p redis_pass',
+          '--max-failures',
+          '1'])
+        opts[:max_failures].should == 1
+      end
+
+      it 'properly parses a chroot' do
+        opts = CLI.parse(['-n host:port', '-z localhost:1111', '--with-chroot', '/with/chroot/from/command/line'])
+        opts[:nodes].should == [{
+          :host => 'host',
+          :port => 'port',
+        }]
+
+        opts[:chroot].should == '/with/chroot/from/command/line'
+      end
+
+      it 'properly parses the config file' do
+        opts = CLI.parse(['-C', "#{File.dirname(__FILE__)}/support/config/single_environment.yml"])
+        opts[:zkservers].should == 'zk01:2181,zk02:2181,zk03:2181'
+
+        opts = CLI.parse(['-C', "#{File.dirname(__FILE__)}/support/config/multiple_environments.yml", '-E', 'development'])
+        opts[:zkservers].should == 'localhost:2181'
+
+        opts = CLI.parse(['-C', "#{File.dirname(__FILE__)}/support/config/multiple_environments.yml", '-E', 'staging'])
+        opts[:zkservers].should == 'zk01:2181,zk02:2181,zk03:2181'
+      end
+
+      it 'properly parses the config file that include chroot' do
+        opts = CLI.parse(['-C', "#{File.dirname(__FILE__)}/support/config/single_environment_with_chroot.yml"])
+        opts[:zkservers].should == 'zk01:2181,zk02:2181,zk03:2181'
+        opts[:chroot].should == '/with/chroot'
+
+        opts = CLI.parse(['-C', "#{File.dirname(__FILE__)}/support/config/multiple_environments_with_chroot.yml", '-E', 'development'])
+        opts[:zkservers].should == 'localhost:2181'
+        opts[:chroot].should == '/with/chroot_development'
+
+        opts = CLI.parse(['-C', "#{File.dirname(__FILE__)}/support/config/multiple_environments_with_chroot.yml", '-E', 'staging'])
+        opts[:zkservers].should == 'zk01:2181,zk02:2181,zk03:2181'
+        opts[:chroot].should == '/with/chroot_staging'
+      end
+    end
+  end
+end

data/spec/client_spec.rb

@@ -0,0 +1,153 @@
+require 'spec_helper'
+
+module RedisFailover
+  Client::Redis = RedisStub
+  Client::Redis::Client = Redis::Client
+  class ClientStub < Client
+    def current_master
+      @master
+    end
+
+    def current_slaves
+      @slaves
+    end
+
+    def fetch_nodes
+      {
+        :master => 'localhost:6379',
+        :slaves => ['localhost:1111'],
+        :unavailable => []
+      }
+    end
+
+    def setup_zk
+      @zk = NullObject.new
+      update_znode_timestamp
+    end
+  end
+
+  describe Client do
+    let(:client) { ClientStub.new(:zkservers => 'localhost:9281', :safe_mode => true) }
+
+    describe '#build_clients' do
+      it 'properly parses master' do
+        client.current_master.to_s.should == 'localhost:6379'
+      end
+
+      it 'properly parses slaves' do
+        client.current_slaves.first.to_s.should == 'localhost:1111'
+      end
+    end
+
+    describe '#client' do
+      it 'should return itself as a delegate' do
+        client.client.should == client
+      end
+    end
+
+    describe '#dispatch' do
+      it 'routes write operations to master' do
+        called = false
+        client.current_master.define_singleton_method(:del) do |*args|
+          called = true
+        end
+        client.del('foo')
+        called.should be_true
+      end
+
+    describe '#inspect' do
+      it 'should always include db' do
+        opts = {:zkservers => 'localhost:1234'}
+        client = ClientStub.new(opts)
+        client.inspect.should match('<RedisFailover::Client \(db: 0,')
+        db = '5'
+        opts.merge!(:db => db)
+        client = ClientStub.new(opts)
+        client.inspect.should match("<RedisFailover::Client \\(db: #{db},")
+      end
+    end
+
+    describe '#call' do
+      it 'should dispatch :call messages to correct method' do
+        client.should_receive(:dispatch).with(:foo, *['key'])
+        client.call([:foo, 'key'])
+      end
+    end
+
+      context 'with :master_only false' do
+        it 'routes read operations to a slave' do
+          called = false
+          client.current_slaves.first.change_role_to('slave')
+          client.current_slaves.first.define_singleton_method(:get) do |*args|
+            called = true
+          end
+          client.get('foo')
+          called.should be_true
+        end
+      end
+
+      context 'with :master_only true' do
+        it 'routes read operations to master' do
+          client = ClientStub.new(:zkservers => 'localhost:9281', :master_only => true)
+          called = false
+          client.current_master.define_singleton_method(:get) do |*args|
+            called = true
+          end
+          client.get('foo')
+          called.should be_true
+        end
+      end
+
+      it 'reconnects when node is unavailable' do
+        class << client
+          attr_reader :reconnected
+          def build_clients
+            @reconnected = true
+            super
+          end
+
+          def fetch_nodes
+            @calls ||= 0
+            {
+              :master => "localhost:222#{@calls += 1}",
+              :slaves => ['localhost:1111'],
+              :unavailable => []
+            }
+          end
+        end
+
+        client.current_master.make_unavailable!
+        client.del('foo')
+        client.reconnected.should be_true
+      end
+
+      context 'with :verify_role true' do
+        it 'properly detects when a node has changed roles' do
+          client.current_master.change_role_to('slave')
+          expect { client.send(:master) }.to raise_error(InvalidNodeRoleError)
+        end
+      end
+
+      it 'raises error for unsupported operations' do
+        expect { client.select }.to raise_error(UnsupportedOperationError)
+      end
+
+      context 'with :safe_mode enabled' do
+        it 'rebuilds clients when no communication from Node Manager within certain time window' do
+          client.instance_variable_set(:@last_znode_timestamp, Time.at(0))
+          client.should_receive(:build_clients)
+          client.del('foo')
+        end
+      end
+
+      context 'with :safe_mode disabled' do
+        it 'does not rebuild clients when no communication from Node Manager within certain time window' do
+          client = ClientStub.new(:zkservers => 'localhost:9281', :safe_mode => false)
+          client.instance_variable_set(:@last_znode_timestamp, Time.at(0))
+          client.should_not_receive(:build_clients)
+          client.del('foo')
+        end
+      end
+    end
+  end
+end

data/spec/failover_strategy/latency_spec.rb

@@ -0,0 +1,41 @@
+require 'spec_helper'
+
+module RedisFailover
+  class FailoverStrategy
+    FailoverStrategy.for(:latency)
+
+    describe Latency do
+      describe '#find_candidate' do
+        it 'returns only candidates seen by all node managers' do
+          strategy = FailoverStrategy.for(:latency)
+          snapshot_1 = NodeSnapshot.new(Node.new(:host => 'localhost', :port => '123'))
+          snapshot_1.viewable_by('nm1', 0)
+          snapshot_1.unviewable_by('nm2')
+
+          snapshot_2 = NodeSnapshot.new(Node.new(:host => 'localhost', :port => '456'))
+          snapshot_2.viewable_by('nm2', 0)
+          snapshot_2.unviewable_by('nm1')
+
+          snapshots = {snapshot_1.node => snapshot_1, snapshot_2.node => snapshot_2}
+          strategy.find_candidate(snapshots).should be_nil
+        end
+
+        it 'returns the candidate with the lowest average latency' do
+          strategy = FailoverStrategy.for(:latency)
+          snapshot_1 = NodeSnapshot.new(Node.new(:host => 'localhost', :port => '123'))
+          snapshot_1.viewable_by('nm1', 5)
+          snapshot_1.viewable_by('nm2', 4)
+          snapshot_1.viewable_by('nm3', 3)
+
+          snapshot_2 = NodeSnapshot.new(Node.new(:host => 'localhost', :port => '456'))
+          snapshot_2.viewable_by('nm1', 1)
+          snapshot_2.viewable_by('nm2', 1)
+          snapshot_2.viewable_by('nm3', 2)
+
+          snapshots = {snapshot_1.node => snapshot_1, snapshot_2.node => snapshot_2}
+          strategy.find_candidate(snapshots).should == snapshot_2.node
+        end
+      end
+    end
+  end
+end

data/spec/failover_strategy_spec.rb

@@ -0,0 +1,17 @@
+require 'spec_helper'
+
+module RedisFailover
+  describe FailoverStrategy do
+
+    describe '.for' do
+      it 'creates a new latency strategy instance' do
+        s = FailoverStrategy.for('latency')
+        s.should be_a RedisFailover::FailoverStrategy::Latency
+      end
+
+      it 'rejects unknown strategies' do
+        expect { FailoverStrategy.for('foobar') }.to raise_error(RuntimeError)
+      end
+    end
+  end
+end

data/spec/node_manager_spec.rb

@@ -0,0 +1,136 @@
+require 'spec_helper'
+
+module RedisFailover
+  describe NodeManager do
+    let(:manager) { NodeManagerStub.new({}) }
+
+    before(:each) do
+      manager.discover_nodes
+    end
+
+    describe '#nodes' do
+      it 'returns current master and slave nodes' do
+        manager.current_nodes.should == {
+          :master => 'master:6379',
+          :slaves => ['slave:6379'],
+          :unavailable => []
+        }
+      end
+    end
+
+    describe '#handle_unavailable' do
+      context 'slave dies' do
+        it 'moves slave to unavailable list' do
+          slave = manager.slaves.first
+          manager.force_unavailable(slave)
+          manager.current_nodes[:unavailable].should include(slave.to_s)
+        end
+      end
+
+      context 'master dies' do
+        before(:each) do
+          @slave = manager.slaves.first
+          @master = manager.master
+          manager.force_unavailable(@master)
+        end
+
+        it 'promotes slave to master' do
+          manager.master.should == @slave
+        end
+
+        it 'moves master to unavailable list' do
+          manager.current_nodes[:unavailable].should include(@master.to_s)
+        end
+      end
+    end
+
+    describe '#handle_available' do
+      before(:each) do
+        # force to be unavailable first
+        @slave = manager.slaves.first
+        manager.force_unavailable(@slave)
+      end
+
+      context 'slave node with a master present' do
+        it 'removes slave from unavailable list' do
+          manager.force_available(@slave)
+          manager.current_nodes[:unavailable].should be_empty
+          manager.current_nodes[:slaves].should include(@slave.to_s)
+        end
+
+        it 'makes node a slave of new master' do
+          manager.master = Node.new(:host => 'foo', :port => '7892')
+          manager.force_available(@slave)
+          @slave.fetch_info.should == {
+            :role => 'slave',
+            :master_host => 'foo',
+            :master_port => '7892'}
+        end
+
+        it 'does not invoke slaveof operation if master has not changed' do
+          @slave.redis.should_not_receive(:slaveof)
+          manager.force_available(@slave)
+        end
+      end
+
+      context 'slave node with no master present' do
+        before(:each) do
+          @master = manager.master
+          manager.force_unavailable(@master)
+        end
+
+        it 'promotes slave to master' do
+          manager.force_available(@slave)
+          manager.master.should == @slave
+        end
+
+        it 'slaves list remains empty' do
+          manager.current_nodes[:slaves].should be_empty
+        end
+      end
+    end
+
+    describe '#handle_syncing' do
+      context 'prohibits stale reads' do
+        it 'adds node to unavailable list' do
+          slave = manager.slaves.first
+          manager.force_syncing(slave, false)
+          manager.current_nodes[:unavailable].should include(slave.to_s)
+        end
+      end
+
+      context 'allows stale reads' do
+        it 'makes node available' do
+          slave = manager.slaves.first
+          manager.force_syncing(slave, true)
+          manager.current_nodes[:unavailable].should_not include(slave.to_s)
+          manager.current_nodes[:slaves].should include(slave.to_s)
+        end
+      end
+    end
+
+    describe '#guess_master' do
+      let(:node1) { Node.new(:host => 'node1').extend(RedisStubSupport) }
+      let(:node2) { Node.new(:host => 'node2').extend(RedisStubSupport) }
+      let(:node3) { Node.new(:host => 'node3').extend(RedisStubSupport) }
+
+      it 'raises error when no master is found' do
+        node1.make_slave!(node3)
+        node2.make_slave!(node3)
+        expect { manager.guess_master([node1, node2]) }.to raise_error(NoMasterError)
+      end
+
+      it 'raises error when multiple masters found' do
+        node1.make_master!
+        node2.make_master!
+        expect { manager.guess_master([node1, node2]) }.to raise_error(MultipleMastersError)
+      end
+
+      it 'raises error when a node can not be reached' do
+        node1.make_master!
+        node2.redis.make_unavailable!
+        expect { manager.guess_master([node1, node2]) }.to raise_error(NodeUnavailableError)
+      end
+    end
+  end
+end

data/spec/node_snapshot_spec.rb

@@ -0,0 +1,30 @@
+require 'spec_helper'
+
+module RedisFailover
+  describe NodeSnapshot do
+    let(:snapshot) { NodeSnapshot.new(Node.new(:host => 'localhost', :port => '123')) }
+
+    describe '#initialize' do
+      it 'creates a new empty snapshot' do
+        snapshot.available_count.should == 0
+        snapshot.unavailable_count.should == 0
+      end
+    end
+
+    describe '#viewable_by' do
+      it 'updates the availability count' do
+        snapshot.viewable_by('nm1', 0)
+        snapshot.viewable_by('nm2', 0)
+        snapshot.available_count.should == 2
+      end
+    end
+
+    describe '#unviewable_by' do
+      it 'updates the unavailability count' do
+        snapshot.unviewable_by('nm1')
+        snapshot.unviewable_by('nm2')
+        snapshot.unavailable_count.should == 2
+      end
+    end
+  end
+end