redis_failover 0.9.7.2 → 1.0.0

data/lib/redis_failover/node_snapshot.rb

@@ -0,0 +1,81 @@
+module RedisFailover
+  # Represents a snapshot of a particular redis node as seen by all currently running
+  # redis node managers.
+  class NodeSnapshot
+    # @return [String] the redis node
+    attr_reader :node
+
+    # Creates a new instance.
+    #
+    # @param [String] the redis node
+    # @see NodeManager#initialize
+    def initialize(node)
+      @node = node
+      @available = {}
+      @unavailable = []
+    end
+
+    # Declares this node available by the specified node manager.
+    #
+    # @param [String] node_manager the node manager id
+    # @param [Integer] latency the latency
+    def viewable_by(node_manager, latency)
+      @available[node_manager] = latency
+    end
+
+    # Determines if this node is viewable by a node manager.
+    #
+    # @param [String] node_manager the node manager id
+    def viewable_by?(node_manager)
+      @available.key?(node_manager)
+    end
+
+    # Declares this node unavailable by the specified node manager.
+    #
+    # @param [String] node_manager the node manager id
+    def unviewable_by(node_manager)
+      @unavailable << node_manager
+    end
+
+    # @return [Integer] the number of node managers saying
+    # this node is available
+    def available_count
+      @available.size
+    end
+
+    # @return [Integer] the number of node managers saying
+    # this node is unavailable
+    def unavailable_count
+      @unavailable.size
+    end
+
+    # @return [Integer] the average available latency
+    def avg_latency
+      return if @available.empty?
+      @available.values.inject(0) { |sum, n| sum + n } / @available.size
+    end
+
+    # @return [Array<String>] all node managers involved in this snapshot
+    def node_managers
+      (@available.keys + @unavailable).uniq
+    end
+
+    # @return [Boolean] true if all node managers indicated that this
+    # node was viewable
+    def all_available?
+      available_count > 0 && unavailable_count == 0
+    end
+
+    # @return [Boolean] true if all node managers indicated that this
+    # node was unviewable
+    def all_unavailable?
+      unavailable_count > 0 && available_count == 0
+    end
+
+    # @return [String] a friendly representation of this node snapshot
+    def to_s
+      'Node %s available by %p, unavailable by %p (%d up, %d down)' %
+        [node, @available, @unavailable, available_count, unavailable_count]
+    end
+  end
+end

data/lib/redis_failover/node_strategy.rb

@@ -0,0 +1,34 @@
+module RedisFailover
+  # Base class for strategies that determine node availability.
+  class NodeStrategy
+    include Util
+
+    # Loads a strategy based on the given name.
+    #
+    # @param [String, Symbol] name the strategy name
+    # @return [Object] a new strategy instance
+    def self.for(name)
+      require "redis_failover/node_strategy/#{name.downcase}"
+      const_get(name.capitalize).new
+    rescue LoadError, NameError
+      raise "Failed to find node strategy: #{name}"
+    end
+
+    # Returns the state determined by this strategy.
+    #
+    # @param [Node] the node to handle
+    # @param [Hash<Node, NodeSnapshot>] snapshots the current set of snapshots
+    # @return [Symbol] the status
+    def determine_state(node, snapshots)
+      raise NotImplementedError
+    end
+
+    # Logs a node as being unavailable.
+    #
+    # @param [Node] node the node
+    # @param [NodeSnapshot] snapshot the node snapshot
+    def log_unavailable(node, snapshot)
+      logger.info("#{self.class} marking #{node} as unavailable. Snapshot: #{snapshot}")
+    end
+  end
+end

data/lib/redis_failover/node_strategy/consensus.rb

@@ -0,0 +1,18 @@
+module RedisFailover
+  class NodeStrategy
+    # Consensus strategy only marks the node as unavailable if all members of the
+    # snapshot indicate that the node is down.
+    class Consensus < NodeStrategy
+      # @see RedisFailover::NodeStrategy#determine_state
+      def determine_state(node, snapshots)
+        snapshot = snapshots[node]
+        if snapshot.all_unavailable?
+          log_unavailable(node, snapshot)
+          :unavailable
+        else
+          :available
+        end
+      end
+    end
+  end
+end

data/lib/redis_failover/node_strategy/majority.rb

@@ -0,0 +1,18 @@
+module RedisFailover
+  class NodeStrategy
+    # Majority strategy only marks the node as unavailable if a majority of the
+    # snapshot indicates that the node is down.
+    class Majority < NodeStrategy
+      # @see RedisFailover::NodeStrategy#determine_state
+      def determine_state(node, snapshots)
+        snapshot = snapshots[node]
+        if snapshot.unavailable_count > snapshot.available_count
+          log_unavailable(node, snapshot)
+          :unavailable
+        else
+          :available
+        end
+      end
+    end
+  end
+end

data/lib/redis_failover/node_strategy/single.rb

@@ -0,0 +1,17 @@
+module RedisFailover
+  class NodeStrategy
+    # Marks the node as unavailable if any node manager reports the node as down.
+    class Single < NodeStrategy
+      # @see RedisFailover::NodeStrategy#determine_state
+      def determine_state(node, snapshots)
+        snapshot = snapshots[node]
+        if snapshot.unavailable_count > 0
+          log_unavailable(node, snapshot)
+          :unavailable
+        else
+          :available
+        end
+      end
+    end
+  end
+end

data/lib/redis_failover/node_watcher.rb

@@ -33,8 +33,12 @@ module RedisFailover
     # Performs a graceful shutdown of this watcher.
     def shutdown
       @done = true
-      @node.wakeup
-      @monitor_thread.join if @monitor_thread
+      begin
+        @node.wakeup
+      rescue
+        # best effort
+      end
+      @monitor_thread.join
     rescue => ex
       logger.warn("Failed to gracefully shutdown watcher for #{@node}")
     end
@@ -48,17 +52,12 @@ module RedisFailover
 
       loop do
         begin
-          return if @done
+          break if @done
           sleep(WATCHER_SLEEP_TIME)
-          @node.ping
+          latency = Benchmark.realtime { @node.ping }
           failures = 0
-
-          if @node.syncing_with_master?
-            notify(:syncing)
-          else
-            notify(:available)
-            @node.wait
-          end
+          notify(:available, latency)
+          @node.wait
         rescue NodeUnavailableError => ex
           logger.debug("Failed to communicate with node #{@node}: #{ex.inspect}")
           failures += 1
@@ -76,8 +75,9 @@ module RedisFailover
     # Notifies the manager of a node's state.
     #
     # @param [Symbol] state the node's state
-    def notify(state)
-      @manager.notify_state(@node, state)
+    # @param [Integer] latency an optional latency
+    def notify(state, latency = nil)
+      @manager.notify_state(@node, state, latency)
     end
   end
 end

data/lib/redis_failover/util.rb

@@ -51,8 +51,8 @@ module RedisFailover
     # that abstracts the master/slave servers.
     UNSUPPORTED_OPS = Set[:select, :dbsize].freeze
 
-    # Default node in ZK that contains the current list of available redis nodes.
-    DEFAULT_ZNODE_PATH = '/redis_failover_nodes'.freeze
+    # Default root node in ZK used for redis_failover.
+    DEFAULT_ROOT_ZNODE_PATH = '/redis_failover'.freeze
 
     # Connectivity errors that the redis (<3.x) client raises.
     REDIS_ERRORS = Errno.constants.map { |c| Errno.const_get(c) }
@@ -61,11 +61,19 @@ module RedisFailover
     REDIS_ERRORS << Redis::BaseError if Redis.const_defined?('BaseError')
     REDIS_ERRORS.freeze
 
+    # ZK Errors
+    ZK_ERRORS = [
+      ZK::Exceptions::LockAssertionFailedError,
+      ZK::Exceptions::InterruptedSession,
+      ZK::Exceptions::Retryable,
+      Zookeeper::Exceptions::ContinuationTimeoutError
+    ].freeze
+
     # Full set of errors related to connectivity.
     CONNECTIVITY_ERRORS = [
       RedisFailover::Error,
-      ZK::Exceptions::InterruptedSession,
-      REDIS_ERRORS
+      REDIS_ERRORS,
+      ZK_ERRORS
     ].flatten.freeze
 
     # Symbolizes the keys of the specified hash.

data/lib/redis_failover/version.rb

@@ -1,3 +1,3 @@
 module RedisFailover
-  VERSION = '0.9.7.2'
+  VERSION = '1.0.0'
 end

data/redis_failover.gemspec

@@ -18,7 +18,7 @@ Gem::Specification.new do |gem|
   gem.add_dependency('redis', ['>= 2.2', '< 4'])
   gem.add_dependency('redis-namespace')
   gem.add_dependency('multi_json', '~> 1')
-  gem.add_dependency('zk', '~> 1.6')
+  gem.add_dependency('zk', ['>= 1.7.2', '< 1.8'])
 
   gem.add_development_dependency('rake')
   gem.add_development_dependency('rspec')

data/spec/failover_strategy/latency_spec.rb

@@ -0,0 +1,41 @@
+require 'spec_helper'
+
+module RedisFailover
+  class FailoverStrategy
+    FailoverStrategy.for(:latency)
+
+    describe Latency do
+      describe '#find_candidate' do
+        it 'returns only candidates seen by all node managers' do
+          strategy = FailoverStrategy.for(:latency)
+          snapshot_1 = NodeSnapshot.new(Node.new(:host => 'localhost', :port => '123'))
+          snapshot_1.viewable_by('nm1', 0)
+          snapshot_1.unviewable_by('nm2')
+
+          snapshot_2 = NodeSnapshot.new(Node.new(:host => 'localhost', :port => '456'))
+          snapshot_2.viewable_by('nm2', 0)
+          snapshot_2.unviewable_by('nm1')
+
+          snapshots = {snapshot_1.node => snapshot_1, snapshot_2.node => snapshot_2}
+          strategy.find_candidate(snapshots).should be_nil
+        end
+
+        it 'returns the candidate with the lowest average latency' do
+          strategy = FailoverStrategy.for(:latency)
+          snapshot_1 = NodeSnapshot.new(Node.new(:host => 'localhost', :port => '123'))
+          snapshot_1.viewable_by('nm1', 5)
+          snapshot_1.viewable_by('nm2', 4)
+          snapshot_1.viewable_by('nm3', 3)
+
+          snapshot_2 = NodeSnapshot.new(Node.new(:host => 'localhost', :port => '456'))
+          snapshot_2.viewable_by('nm1', 1)
+          snapshot_2.viewable_by('nm2', 1)
+          snapshot_2.viewable_by('nm3', 2)
+
+          snapshots = {snapshot_1.node => snapshot_1, snapshot_2.node => snapshot_2}
+          strategy.find_candidate(snapshots).should == snapshot_2.node
+        end
+      end
+    end
+  end
+end

data/spec/failover_strategy_spec.rb

@@ -0,0 +1,17 @@
+require 'spec_helper'
+
+module RedisFailover
+  describe FailoverStrategy do
+
+    describe '.for' do
+      it 'creates a new latency strategy instance' do
+        s = FailoverStrategy.for('latency')
+        s.should be_a RedisFailover::FailoverStrategy::Latency
+      end
+
+      it 'rejects unknown strategies' do
+        expect { FailoverStrategy.for('foobar') }.to raise_error(RuntimeError)
+      end
+    end
+  end
+end

data/spec/node_snapshot_spec.rb

@@ -0,0 +1,30 @@
+require 'spec_helper'
+
+module RedisFailover
+  describe NodeSnapshot do
+    let(:snapshot) { NodeSnapshot.new(Node.new(:host => 'localhost', :port => '123')) }
+
+    describe '#initialize' do
+      it 'creates a new empty snapshot' do
+        snapshot.available_count.should == 0
+        snapshot.unavailable_count.should == 0
+      end
+    end
+
+    describe '#viewable_by' do
+      it 'updates the availability count' do
+        snapshot.viewable_by('nm1', 0)
+        snapshot.viewable_by('nm2', 0)
+        snapshot.available_count.should == 2
+      end
+    end
+
+    describe '#unviewable_by' do
+      it 'updates the unavailability count' do
+        snapshot.unviewable_by('nm1')
+        snapshot.unviewable_by('nm2')
+        snapshot.unavailable_count.should == 2
+      end
+    end
+  end
+end

data/spec/node_strategy/consensus_spec.rb

@@ -0,0 +1,30 @@
+require 'spec_helper'
+
+module RedisFailover
+  class NodeStrategy
+    NodeStrategy.for(:consensus)
+
+    describe Consensus do
+      let(:node) { Node.new(:host => 'localhost', :port => '123') }
+      let(:snapshot) { NodeSnapshot.new(node) }
+
+      describe '#determine_state' do
+        it 'returns the unavailable state if unavailable by all node managers' do
+          strategy = NodeStrategy.for(:consensus)
+          snapshot.unviewable_by('nm1')
+          snapshot.unviewable_by('nm2')
+          snapshot.unviewable_by('nm3')
+          strategy.determine_state(node, node => snapshot).should == :unavailable
+        end
+
+        it 'returns the available state if unavailable by some node managers' do
+          strategy = NodeStrategy.for(:consensus)
+          snapshot.unviewable_by('nm1')
+          snapshot.unviewable_by('nm2')
+          snapshot.viewable_by('nm3', 0)
+          strategy.determine_state(node, node => snapshot).should == :available
+        end
+      end
+    end
+  end
+end

data/spec/node_strategy/majority_spec.rb

@@ -0,0 +1,22 @@
+require 'spec_helper'
+
+module RedisFailover
+  class NodeStrategy
+    NodeStrategy.for(:majority)
+
+    describe Majority do
+      let(:node) { Node.new(:host => 'localhost', :port => '123') }
+      let(:snapshot) { NodeSnapshot.new(node) }
+
+      describe '#determine_state' do
+        it 'returns the unavailable state if unavailable by the majority of node managers' do
+          strategy = NodeStrategy.for(:majority)
+          snapshot.viewable_by('nm1', 0)
+          snapshot.unviewable_by('nm2')
+          snapshot.unviewable_by('nm3')
+          strategy.determine_state(node, node => snapshot).should == :unavailable
+        end
+      end
+    end
+  end
+end

data/spec/node_strategy/single_spec.rb

@@ -0,0 +1,22 @@
+require 'spec_helper'
+
+module RedisFailover
+  class NodeStrategy
+    NodeStrategy.for(:single)
+
+    describe Single do
+      let(:node) { Node.new(:host => 'localhost', :port => '123') }
+      let(:snapshot) { NodeSnapshot.new(node) }
+
+      describe '#determine_state' do
+        it 'returns the unavailable state if any node manager reports as down' do
+          strategy = NodeStrategy.for(:single)
+          snapshot.unviewable_by('nm1')
+          snapshot.viewable_by('nm2', 0)
+          snapshot.viewable_by('nm3', 0)
+          strategy.determine_state(node, node => snapshot).should == :unavailable
+        end
+      end
+    end
+  end
+end

data/spec/node_strategy_spec.rb

@@ -0,0 +1,22 @@
+require 'spec_helper'
+
+module RedisFailover
+  describe NodeStrategy do
+
+    describe '.for' do
+      it 'creates a new majority strategy instance' do
+        s = NodeStrategy.for('majority')
+        s.should be_a RedisFailover::NodeStrategy::Majority
+      end
+
+      it 'creates a new consensus strategy instance' do
+        s = NodeStrategy.for('consensus')
+        s.should be_a RedisFailover::NodeStrategy::Consensus
+      end
+
+      it 'rejects unknown strategies' do
+        expect { NodeStrategy.for('foobar') }.to raise_error(RuntimeError)
+      end
+    end
+  end
+end