redis_failover 0.1.1 → 0.2.0

data/Changes.md

@@ -1,9 +1,15 @@
-0.1.0
+0.2.0
 -----------
-
-- First release
+- Added retry support for contacting failover server from client
+- Client now verifies proper master/slave role before attempting operation
+- General edge case cleanup for NodeManager
 
 0.1.1
 -----------
 
-- Fix option parser require
+- Fix option parser require
+
+0.1.0
+-----------
+
+- First release

data/README.md

@@ -1,4 +1,4 @@
-# Redis Failover Client/Server
+# Automatic Redis Failover Client/Server
 
 Redis Failover attempts to provides a full automatic master/slave failover solution for Ruby. Redis does not provide
 an automatic failover capability when configured for master/slave replication. When the master node dies,
@@ -18,7 +18,9 @@ All nodes marked as unreachable will be periodically checked to see if they have
 If so, the newly reachable nodes will be configured as slaves and brought back into the list of live
 servers. Note that detection of a node going down should be nearly instantaneous, since the mechanism
 used to keep tabs on a node is via a blocking Redis BLPOP call (no polling). This call fails nearly
-immediately when the node actually goes offline.
+immediately when the node actually goes offline. To avoid false positives (i.e., intermittent flaky
+network interruption), the server will only mark a node as unreachable if it fails to communicate with
+it 3 times (this is configurable via --max-failures, see configuration options below).
 
 This gem provides a RedisFailover::Client wrapper that is master/slave aware. The client is configured
 with a single host/port pair that points to redis failover server. The client will automatically

data/lib/redis_failover/client.rb

@@ -6,6 +6,7 @@ module RedisFailover
   class Client
     include Util
 
+    RETRY_WAIT_TIME = 3
     REDIS_ERRORS = Errno.constants.map { |c| Errno.const_get(c) }.freeze
     REDIS_READ_OPS = Set[
       :dbsize,
@@ -83,8 +84,8 @@ module RedisFailover
       @namespace = options[:namespace]
       @password = options[:password]
       @retry = options[:retry_failure] || true
-      @max_retries = options[:max_retries] || 3
-      @registry_url = "http://#{options[:host]}:#{options[:port]}/redis_servers"
+      @max_retries = @retry ? options.fetch(:max_retries, 3) : 0
+      @server_url = "http://#{options[:host]}:#{options[:port]}/redis_servers"
       @redis_servers = nil
       @master = nil
       @slaves = []
@@ -117,6 +118,7 @@ module RedisFailover
 
     def dispatch(method, *args, &block)
       tries = 0
+
       begin
         if REDIS_READ_OPS.include?(method)
           # send read operations to a slave
@@ -125,14 +127,13 @@ module RedisFailover
           # direct everything else to master
           master.send(method, *args, &block)
         end
-      rescue NoMasterError, *REDIS_ERRORS
+      rescue Error, *REDIS_ERRORS
         logger.error("No suitable node available for operation `#{method}.`")
-        sleep(3)
         build_clients
 
-        if @retry && tries < @max_retries
+        if tries < @max_retries
           tries += 1
-          retry
+          sleep(RETRY_WAIT_TIME) && retry
         end
 
         raise
@@ -140,16 +141,26 @@ module RedisFailover
     end
 
     def master
-      @master or raise NoMasterError
+      if @master
+        verify_role!(@master, :master)
+        return @master
+      end
+      raise NoMasterError
     end
 
     def slave
       # pick a slave, if none available fallback to master
-      @slaves.sample || master
+      if slave = @slaves.sample
+        verify_role!(slave, :slave)
+        return slave
+      end
+      master
     end
 
     def build_clients
       @lock.synchronize do
+        tries = 0
+
         begin
           logger.info('Attempting to fetch nodes and build redis clients.')
           servers = fetch_redis_servers
@@ -160,14 +171,21 @@ module RedisFailover
           @master = master
           @slaves = slaves
         rescue => ex
-          logger.error("Failed to fetch servers from #{@registry_url} - #{ex.message}")
+          logger.error("Failed to fetch servers from #{@server_url} - #{ex.message}")
           logger.error(ex.backtrace.join("\n"))
+
+          if tries < @max_retries
+            tries += 1
+            sleep(RETRY_WAIT_TIME) && retry
+          end
+
+          raise FailoverServerUnreachableError.new(@server_url)
         end
       end
     end
 
     def fetch_redis_servers
-      open(@registry_url) do |io|
+      open(@server_url) do |io|
         servers = symbolize_keys(MultiJson.decode(io))
         logger.info("Fetched servers: #{servers}")
         servers
@@ -187,12 +205,24 @@ module RedisFailover
 
     def master_info
       return "none" unless @master
-      "#{@master.client.host}:#{@master.client.port}"
+      name_for(@master)
     end
 
     def slaves_info
       return "none" if @slaves.empty?
-      @slaves.map { |s| "#{s.client.host}:#{s.client.port}" }.join(', ')
+      @slaves.map { |slave| name_for(slave) }.join(', ')
+    end
+
+    def verify_role!(node, role)
+      current_role = node.info['role']
+      if current_role.to_sym != role
+        raise InvalidNodeRoleError.new(name_for(node), role, current_role)
+      end
+      role
+    end
+
+    def name_for(node)
+      "#{node.client.host}:#{node.client.port}"
     end
   end
 end

data/lib/redis_failover/errors.rb

@@ -22,4 +22,17 @@ module RedisFailover
 
   class NoSlaveError < Error
   end
+
+  class FailoverServerUnreachableError < Error
+    def initialize(failover_server_url)
+      super("Unable to reach #{failover_server_url}")
+    end
+  end
+
+  class InvalidNodeRoleError < Error
+    def initialize(node, assumed, actual)
+      super("Invalid role detected for node #{node}, client thought " +
+        "it was a #{assumed}, but it's now a #{actual}")
+    end
+  end
 end

data/lib/redis_failover/node_manager.rb

@@ -60,6 +60,7 @@ module RedisFailover
       return if @unreachable.include?(node)
       logger.info("Handling unreachable node: #{node}")
 
+      @unreachable << node
       # find a new master if this node was a master
       if node == @master
         logger.info("Demoting currently unreachable master #{node}.")
@@ -67,8 +68,6 @@ module RedisFailover
       else
         @slaves.delete(node)
       end
-
-      @unreachable << node
     end
 
     def handle_reachable(node)
@@ -76,33 +75,32 @@ module RedisFailover
       return if @master == node || @slaves.include?(node)
       logger.info("Handling reachable node: #{node}")
 
-      @unreachable.delete(node)
-      @slaves << node
       if @master
         # master already exists, make a slave
         node.make_slave!(@master)
+        @slaves << node
       else
         # no master exists, make this the new master
-        promote_new_master
+        promote_new_master(node)
       end
+
+      @unreachable.delete(node)
     end
 
-    def promote_new_master
+    def promote_new_master(node = nil)
       @master = nil
 
-      if @slaves.empty?
-        logger.error('Failed to promote a new master since no slaves available.')
+      # make a specific node or slave the new master
+      candidate = node || @slaves.pop
+      unless candidate
+        logger.error('Failed to promote a new master since no candidate available.')
         return
       end
 
-      # make a slave the new master
-      node = @slaves.pop
-      node.make_master!
-      @master = node
-
-      # switch existing slaves to point to new master
-      redirect_slaves
-      logger.info("Successfully promoted #{node} to master.")
+      candidate.make_master!
+      @master = candidate
+      redirect_slaves_to_master
+      logger.info("Successfully promoted #{candidate} to master.")
     end
 
     def parse_nodes
@@ -118,7 +116,7 @@ module RedisFailover
 
     def spawn_watchers
       @watchers = [@master, *@slaves].map do |node|
-          NodeWatcher.new(self, node,  @options[:max_failures] || 3)
+        NodeWatcher.new(self, node,  @options[:max_failures] || 3)
       end
       @watchers.each(&:watch)
     end
@@ -136,14 +134,13 @@ module RedisFailover
       end
     end
 
-    def redirect_slaves
-      # redirect each slave to a new master - if an actual slave is
-      # currently down, it will be handled by its watcher
+    def redirect_slaves_to_master
+      # redirect each slave to the current master
       @slaves.each do |slave|
         begin
           slave.make_slave!(@master)
         rescue NodeUnreachableError
-          # will eventually be handled by watcher
+          # will also be detected by watcher
         end
       end
     end

data/lib/redis_failover/version.rb

@@ -1,3 +1,3 @@
 module RedisFailover
-  VERSION = "0.1.1"
+  VERSION = "0.2.0"
 end

data/spec/client_spec.rb

@@ -40,6 +40,7 @@ module RedisFailover
       end
 
       it 'routes read operations to a slave' do
+        client.current_slaves.first.change_role_to('slave')
         client.current_slaves.first.should_receive(:get)
         client.get('foo')
       end
@@ -57,6 +58,17 @@ module RedisFailover
         client.del('foo')
         client.reconnected.should be_true
       end
+
+      it 'fails hard when the failover server is unreachable' do
+        expect do
+          Client.new(:host => 'foo', :port => 123445)
+        end.to raise_error(FailoverServerUnreachableError)
+      end
+
+      it 'properly detects when a node has changed roles' do
+        client.current_master.change_role_to('slave')
+        expect { client.send(:master) }.to raise_error(InvalidNodeRoleError)
+      end
     end
   end
 end

data/spec/support/redis_stub.rb

@@ -5,7 +5,7 @@ module RedisFailover
   class RedisStub
     class Proxy
       def initialize(queue, opts = {})
-        @info = {:role => 'master'}
+        @info = {'role' => 'master'}
         @queue = queue
       end
 
@@ -24,9 +24,9 @@ module RedisFailover
 
       def slaveof(host, port)
         if host == 'no' && port == 'one'
-          @info[:role] = 'master'
+          @info['role'] = 'master'
         else
-          @info[:role] = 'slave'
+          @info['role'] = 'slave'
         end
       end
 
@@ -37,6 +37,10 @@ module RedisFailover
       def ping
         'pong'
       end
+
+      def change_role_to(role)
+        @info['role'] = role
+      end
     end
 
     attr_reader :host, :port, :reachable
@@ -56,6 +60,10 @@ module RedisFailover
       end
     end
 
+    def change_role_to(role)
+      @proxy.change_role_to(role)
+    end
+
     def make_reachable!
       @reachable = true
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: redis_failover
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-04-12 00:00:00.000000000 Z
+date: 2012-04-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
-  requirement: &70104404926580 !ruby/object:Gem::Requirement
+  requirement: &70198326575220 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70104404926580
+  version_requirements: *70198326575220
 - !ruby/object:Gem::Dependency
   name: redis-namespace
-  requirement: &70104404926140 !ruby/object:Gem::Requirement
+  requirement: &70198326574780 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70104404926140
+  version_requirements: *70198326574780
 - !ruby/object:Gem::Dependency
   name: multi_json
-  requirement: &70104404925720 !ruby/object:Gem::Requirement
+  requirement: &70198326574360 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70104404925720
+  version_requirements: *70198326574360
 - !ruby/object:Gem::Dependency
   name: sinatra
-  requirement: &70104404925300 !ruby/object:Gem::Requirement
+  requirement: &70198326573940 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70104404925300
+  version_requirements: *70198326573940
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &70104404924880 !ruby/object:Gem::Requirement
+  requirement: &70198326573520 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +65,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70104404924880
+  version_requirements: *70198326573520
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70104404924460 !ruby/object:Gem::Requirement
+  requirement: &70198326573100 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,7 +76,7 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70104404924460
+  version_requirements: *70198326573100
 description: Redis Failover provides a full automatic master/slave failover solution
   for Ruby
 email:
@@ -128,7 +128,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 2359035733939003378
+      hash: -200241163739368802
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -137,7 +137,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 2359035733939003378
+      hash: -200241163739368802
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.16