redis_failover 0.9.7.2 → 1.0.0

data/lib/redis_failover/node.rb

@@ -22,7 +22,8 @@ module RedisFailover
     # @option options [String] :host the host of the redis server
     # @option options [String] :port the port of the redis server
     def initialize(options = {})
-      @host = options.fetch(:host) { raise InvalidNodeError, 'missing host'}
+      @host = options[:host]
+      raise InvalidNodeError, 'missing host' if @host.to_s.empty?
       @port = Integer(options[:port] || 6379)
       @password = options[:password]
     end

data/lib/redis_failover/node_manager.rb

@@ -3,21 +3,20 @@ module RedisFailover
   # will discover the current redis master and slaves. Each redis node is
   # monitored by a NodeWatcher instance. The NodeWatchers periodically
   # report the current state of the redis node it's watching to the
-  # NodeManager via an asynchronous queue. The NodeManager processes the
-  # state reports and reacts appropriately by handling stale/dead nodes,
-  # and promoting a new redis master if it sees fit to do so.
+  # NodeManager. The NodeManager processes the state reports and reacts
+  # appropriately by handling stale/dead nodes, and promoting a new redis master
+  # if it sees fit to do so.
   class NodeManager
     include Util
 
     # Number of seconds to wait before retrying bootstrap process.
-    TIMEOUT = 3
-
-    # ZK Errors that the Node Manager cares about.
-    ZK_ERRORS = [
-      ZK::Exceptions::LockAssertionFailedError,
-      ZK::Exceptions::InterruptedSession,
-      ZKDisconnectedError
-    ].freeze
+    TIMEOUT = 5
+    # Number of seconds for checking node snapshots.
+    CHECK_INTERVAL = 5
+    # Number of max attempts to promote a master before releasing master lock.
+    MAX_PROMOTION_ATTEMPTS = 3
+    # Latency threshold for recording node state.
+    LATENCY_THRESHOLD = 0.5
 
     # Errors that can happen during the node discovery process.
     NODE_DISCOVERY_ERRORS = [
@@ -38,15 +37,16 @@ module RedisFailover
     def initialize(options)
       logger.info("Redis Node Manager v#{VERSION} starting (#{RUBY_DESCRIPTION})")
       @options = options
-      @znode = @options[:znode_path] || Util::DEFAULT_ZNODE_PATH
-      @manual_znode = ManualFailover::ZNODE_PATH
-      @mutex = Mutex.new
+      @required_node_managers = options.fetch(:required_node_managers, 1)
+      @root_znode = options.fetch(:znode_path, Util::DEFAULT_ROOT_ZNODE_PATH)
+      @node_strategy = NodeStrategy.for(options.fetch(:node_strategy, :majority))
+      @failover_strategy = FailoverStrategy.for(options.fetch(:failover_strategy, :latency))
+      @nodes = Array(@options[:nodes]).map { |opts| Node.new(opts) }.uniq
+      @master_manager = false
+      @master_promotion_attempts = 0
+      @sufficient_node_managers = false
+      @lock = Monitor.new
       @shutdown = false
-      @leader = false
-      @master = nil
-      @slaves = []
-      @unavailable = []
-      @lock_path = "#{@znode}_lock".freeze
     end
 
     # Starts the node manager.
@@ -54,21 +54,18 @@ module RedisFailover
     # @note This method does not return until the manager terminates.
     def start
       return unless running?
-      @queue = Queue.new
       setup_zk
-      logger.info('Waiting to become master Node Manager ...')
-      with_lock do
-        @leader = true
-        logger.info('Acquired master Node Manager lock')
-        if discover_nodes
-          initialize_path
-          spawn_watchers
-          handle_state_reports
-        end
-      end
+      spawn_watchers
+      wait_until_master
     rescue *ZK_ERRORS => ex
       logger.error("ZK error while attempting to manage nodes: #{ex.inspect}")
       reset
+      sleep(TIMEOUT)
+      retry
+    rescue NoMasterError
+      logger.error("Failed to promote a new master after #{MAX_PROMOTION_ATTEMPTS} attempts.")
+      reset
+      sleep(TIMEOUT)
       retry
     end
 
@@ -77,81 +74,58 @@ module RedisFailover
     #
     # @param [Node] node the node
     # @param [Symbol] state the state
-    def notify_state(node, state)
-      @queue << [node, state]
+    # @param [Integer] latency an optional latency
+    def notify_state(node, state, latency = nil)
+      @lock.synchronize do
+        if running?
+          update_current_state(node, state, latency)
+        end
+      end
+    rescue => ex
+      logger.error("Error handling state report #{[node, state].inspect}: #{ex.inspect}")
+      logger.error(ex.backtrace.join("\n"))
     end
 
     # Performs a reset of the manager.
     def reset
-      @leader = false
+      @master_manager = false
+      @master_promotion_attempts = 0
       @watchers.each(&:shutdown) if @watchers
-      @queue.clear
-      @zk.close! if @zk
-      @zk_lock = nil
     end
 
     # Initiates a graceful shutdown.
     def shutdown
       logger.info('Shutting down ...')
-      @mutex.synchronize do
+      @lock.synchronize do
         @shutdown = true
-        unless @leader
-          reset
-        end
       end
+
+      reset
+      exit
     end
 
     private
 
     # Configures the ZooKeeper client.
     def setup_zk
-      @zk.close! if @zk
-      @zk = ZK.new("#{@options[:zkservers]}#{@options[:chroot] || ''}")
-      @zk.on_expired_session { notify_state(:zk_disconnected, nil) }
-
-      @zk.register(@manual_znode) do |event|
-        if event.node_created? || event.node_changed?
-          perform_manual_failover
+      unless @zk
+        @zk = ZK.new("#{@options[:zkservers]}#{@options[:chroot] || ''}")
+        @zk.register(manual_failover_path) do |event|
+          handle_manual_failover_update(event)
         end
+        @zk.on_connected { @zk.stat(manual_failover_path, :watch => true) }
       end
 
-      @zk.on_connected { @zk.stat(@manual_znode, :watch => true) }
-      @zk.stat(@manual_znode, :watch => true)
-    end
-
-    # Handles periodic state reports from {RedisFailover::NodeWatcher} instances.
-    def handle_state_reports
-      while running? && (state_report = @queue.pop)
-        begin
-          @mutex.synchronize do
-            return unless running?
-            @zk_lock.assert!
-            node, state = state_report
-            case state
-            when :unavailable     then handle_unavailable(node)
-            when :available       then handle_available(node)
-            when :syncing         then handle_syncing(node)
-            when :zk_disconnected then raise ZKDisconnectedError
-            else raise InvalidNodeStateError.new(node, state)
-            end
-
-            # flush current state
-            write_state
-          end
-        rescue *ZK_ERRORS
-          # fail hard if this is a ZK connection-related error
-          raise
-        rescue => ex
-          logger.error("Error handling #{state_report.inspect}: #{ex.inspect}")
-          logger.error(ex.backtrace.join("\n"))
-        end
-      end
+      create_path(@root_znode)
+      create_path(current_state_root)
+      @zk.stat(manual_failover_path, :watch => true)
     end
 
     # Handles an unavailable node.
     #
     # @param [Node] node the unavailable node
-    def handle_unavailable(node)
+    # @param [Hash<Node, NodeSnapshot>] snapshots the current set of snapshots
+    def handle_unavailable(node, snapshots)
       # no-op if we already know about this node
       return if @unavailable.include?(node)
       logger.info("Handling unavailable node: #{node}")
@@ -160,7 +134,7 @@ module RedisFailover
       # find a new master if this node was a master
       if node == @master
         logger.info("Demoting currently unavailable master #{node}.")
-        promote_new_master
+        promote_new_master(snapshots)
       else
         @slaves.delete(node)
       end
@@ -169,7 +143,8 @@ module RedisFailover
     # Handles an available node.
     #
     # @param [Node] node the available node
-    def handle_available(node)
+    # @param [Hash<Node, NodeSnapshot>] snapshots the current set of snapshots
+    def handle_available(node, snapshots)
       reconcile(node)
 
       # no-op if we already know about this node
@@ -182,7 +157,7 @@ module RedisFailover
         @slaves << node
       else
         # no master exists, make this the new master
-        promote_new_master(node)
+        promote_new_master(snapshots, node)
       end
 
       @unavailable.delete(node)
@@ -191,74 +166,75 @@ module RedisFailover
     # Handles a node that is currently syncing.
     #
     # @param [Node] node the syncing node
-    def handle_syncing(node)
+    # @param [Hash<Node, NodeSnapshot>] snapshots the current set of snapshots
+    def handle_syncing(node, snapshots)
       reconcile(node)
 
       if node.syncing_with_master? && node.prohibits_stale_reads?
         logger.info("Node #{node} not ready yet, still syncing with master.")
         force_unavailable_slave(node)
-        return
+      else
+        # otherwise, we can use this node
+        handle_available(node, snapshots)
       end
-
-      # otherwise, we can use this node
-      handle_available(node)
     end
 
     # Handles a manual failover request to the given node.
     #
     # @param [Node] node the candidate node for failover
-    def handle_manual_failover(node)
+    # @param [Hash<Node, NodeSnapshot>] snapshots the current set of snapshots
+    def handle_manual_failover(node, snapshots)
       # no-op if node to be failed over is already master
       return if @master == node
       logger.info("Handling manual failover")
 
+      # ensure we can talk to the node
+      node.ping
+
       # make current master a slave, and promote new master
       @slaves << @master if @master
       @slaves.delete(node)
-      promote_new_master(node)
+      promote_new_master(snapshots, node)
     end
 
     # Promotes a new master.
     #
+    # @param [Hash<Node, NodeSnapshot>] snapshots the current set of snapshots
     # @param [Node] node the optional node to promote
-    # @note if no node is specified, a random slave will be used
-    def promote_new_master(node = nil)
-      delete_path
+    def promote_new_master(snapshots, node = nil)
+      delete_path(redis_nodes_path)
       @master = nil
 
-      # make a specific node or slave the new master
-      candidate = node || @slaves.pop
-      unless candidate
+      # make a specific node or selected candidate the new master
+      candidate = node || failover_strategy_candidate(snapshots)
+
+      if candidate.nil?
         logger.error('Failed to promote a new master, no candidate available.')
-        return
+      else
+        @slaves.delete(candidate)
+        @unavailable.delete(candidate)
+        redirect_slaves_to(candidate)
+        candidate.make_master!
+        @master = candidate
+        write_current_redis_nodes
+        @master_promotion_attempts = 0
+        logger.info("Successfully promoted #{candidate} to master.")
       end
-
-      redirect_slaves_to(candidate)
-      candidate.make_master!
-      @master = candidate
-
-      create_path
-      write_state
-      logger.info("Successfully promoted #{candidate} to master.")
     end
 
     # Discovers the current master and slave nodes.
     # @return [Boolean] true if nodes successfully discovered, false otherwise
     def discover_nodes
-      @mutex.synchronize do
-        return false unless running?
-        nodes = @options[:nodes].map { |opts| Node.new(opts) }.uniq
+      @lock.synchronize do
+        return unless running?
+        @slaves, @unavailable = [], []
         if @master = find_existing_master
           logger.info("Using master #{@master} from existing znode config.")
-        elsif @master = guess_master(nodes)
+        elsif @master = guess_master(@nodes)
           logger.info("Guessed master #{@master} from known redis nodes.")
         end
-        @slaves = nodes - [@master]
-        logger.info("Managing master (#{@master}) and slaves " +
-          "(#{@slaves.map(&:to_s).join(', ')})")
-        # ensure that slaves are correctly pointing to this master
-        redirect_slaves_to(@master)
-        true
+        @slaves = @nodes - [@master]
+        logger.info("Managing master (#{@master}) and slaves #{stringify_nodes(@slaves)}")
       end
     rescue *NODE_DISCOVERY_ERRORS => ex
       msg = <<-MSG.gsub(/\s+/, ' ')
@@ -276,7 +252,7 @@ module RedisFailover
 
     # Seeds the initial node master from an existing znode config.
     def find_existing_master
-      if data = @zk.get(@znode).first
+      if data = @zk.get(redis_nodes_path).first
         nodes = symbolize_keys(decode(data))
         master = node_from(nodes[:master])
         logger.info("Master from existing znode config: #{master || 'none'}")
@@ -305,10 +281,13 @@ module RedisFailover
 
     # Spawns the {RedisFailover::NodeWatcher} instances for each managed node.
     def spawn_watchers
-      @watchers = [@master, @slaves, @unavailable].flatten.compact.map do |node|
-        NodeWatcher.new(self, node, @options[:max_failures] || 3)
+      @zk.delete(current_state_path, :ignore => :no_node)
+      @monitored_available, @monitored_unavailable = {}, []
+      @watchers = @nodes.map do |node|
+        NodeWatcher.new(self, node, @options.fetch(:max_failures, 3))
       end
       @watchers.each(&:watch)
+      logger.info("Monitoring redis nodes at #{stringify_nodes(@nodes)}")
     end
 
     # Searches for the master node.
@@ -376,39 +355,278 @@ module RedisFailover
       }
     end
 
+    # @return [Hash] the set of currently available/unavailable nodes as
+    # seen by this node manager instance
+    def node_availability_state
+      {
+        :available => Hash[@monitored_available.map { |k, v| [k.to_s, v] }],
+        :unavailable => @monitored_unavailable.map(&:to_s)
+      }
+    end
+
     # Deletes the znode path containing the redis nodes.
-    def delete_path
-      @zk.delete(@znode)
-      logger.info("Deleted ZooKeeper node #{@znode}")
+    #
+    # @param [String] path the znode path to delete
+    def delete_path(path)
+      @zk.delete(path)
+      logger.info("Deleted ZK node #{path}")
     rescue ZK::Exceptions::NoNode => ex
       logger.info("Tried to delete missing znode: #{ex.inspect}")
     end
 
-    # Creates the znode path containing the redis nodes.
-    def create_path
-      unless @zk.exists?(@znode)
-        @zk.create(@znode, encode(current_nodes))
-        logger.info("Created ZooKeeper node #{@znode}")
+    # Creates a znode path.
+    #
+    # @param [String] path the znode path to create
+    # @param [Hash] options the options used to create the path
+    # @option options [String] :initial_value an initial value for the znode
+    # @option options [Boolean] :ephemeral true if node is ephemeral, false otherwise
+    def create_path(path, options = {})
+      unless @zk.exists?(path)
+        @zk.create(path,
+          options[:initial_value],
+          :ephemeral => options.fetch(:ephemeral, false))
+        logger.info("Created ZK node #{path}")
       end
     rescue ZK::Exceptions::NodeExists
       # best effort
     end
 
-    # Initializes the znode path containing the redis nodes.
-    def initialize_path
-      create_path
-      write_state
+    # Writes state to a particular znode path.
+    #
+    # @param [String] path the znode path that should be written to
+    # @param [String] value the value to write to the znode
+    # @param [Hash] options the default options to be used when creating the node
+    # @note the path will be created if it doesn't exist
+    def write_state(path, value, options = {})
+      create_path(path, options.merge(:initial_value => value))
+      @zk.set(path, value)
     end
 
-    # Writes the current redis nodes state to the znode path.
-    def write_state
-      create_path
-      @zk.set(@znode, encode(current_nodes))
+    # Handles a manual failover znode update.
+    #
+    # @param [ZK::Event] event the ZK event to handle
+    def handle_manual_failover_update(event)
+      if event.node_created? || event.node_changed?
+        perform_manual_failover
+      end
+    rescue => ex
+      logger.error("Error scheduling a manual failover: #{ex.inspect}")
+      logger.error(ex.backtrace.join("\n"))
+    ensure
+      @zk.stat(manual_failover_path, :watch => true)
+    end
+
+    # Produces a FQDN id for this Node Manager.
+    #
+    # @return [String] the FQDN for this Node Manager
+    def manager_id
+      @manager_id ||= [
+        Socket.gethostbyname(Socket.gethostname)[0],
+        Process.pid
+      ].join('-')
+    end
+
+    # Writes the current master list of redis nodes. This method is only invoked
+    # if this node manager instance is the master/primary manager.
+    def write_current_redis_nodes
+      write_state(redis_nodes_path, encode(current_nodes))
+    end
+
+    # Writes the current monitored list of redis nodes. This method is always
+    # invoked by all running node managers.
+    def write_current_monitored_state
+      write_state(current_state_path, encode(node_availability_state), :ephemeral => true)
+    end
+
+    # @return [String] root path for current node manager state
+    def current_state_root
+      "#{@root_znode}/manager_node_state"
+    end
+
+    # @return [String] the znode path for this node manager's view
+    # of available nodes
+    def current_state_path
+      "#{current_state_root}/#{manager_id}"
+    end
+
+    # @return [String] the znode path for the master redis nodes config
+    def redis_nodes_path
+      "#{@root_znode}/nodes"
+    end
+
+    # @return [String] the znode path used for performing manual failovers
+    def manual_failover_path
+      ManualFailover.path(@root_znode)
+    end
+
+    # @return [Boolean] true if this node manager is the master, false otherwise
+    def master_manager?
+      @master_manager
+    end
+
+    # Used to update the master node manager state. These states are only handled if
+    # this node manager instance is serving as the master manager.
+    #
+    # @param [Node] node the node to handle
+    # @param [Hash<Node, NodeSnapshot>] snapshots the current set of snapshots
+    def update_master_state(node, snapshots)
+      state = @node_strategy.determine_state(node, snapshots)
+      case state
+      when :unavailable
+        handle_unavailable(node, snapshots)
+      when :available
+        if node.syncing_with_master?
+          handle_syncing(node, snapshots)
+        else
+          handle_available(node, snapshots)
+        end
+      else
+        raise InvalidNodeStateError.new(node, state)
+      end
+    rescue *ZK_ERRORS
+      # fail hard if this is a ZK connection-related error
+      raise
+    rescue => ex
+      logger.error("Error handling state report for #{[node, state].inspect}: #{ex.inspect}")
+    end
+
+    # Updates the current view of the world for this particular node
+    # manager instance. All node managers write this state regardless
+    # of whether they are the master manager or not.
+    #
+    # @param [Node] node the node to handle
+    # @param [Symbol] state the node state
+    # @param [Integer] latency an optional latency
+    def update_current_state(node, state, latency = nil)
+      old_unavailable = @monitored_unavailable.dup
+      old_available = @monitored_available.dup
+
+      case state
+      when :unavailable
+        unless @monitored_unavailable.include?(node)
+          @monitored_unavailable << node
+          @monitored_available.delete(node)
+          write_current_monitored_state
+        end
+      when :available
+        last_latency = @monitored_available[node]
+        if last_latency.nil? || (latency - last_latency) > LATENCY_THRESHOLD
+          @monitored_available[node] = latency
+          @monitored_unavailable.delete(node)
+          write_current_monitored_state
+        end
+      else
+        raise InvalidNodeStateError.new(node, state)
+      end
+    rescue => ex
+      # if an error occurs, make sure that we rollback to the old state
+      @monitored_unavailable = old_unavailable
+      @monitored_available = old_available
+      raise
+    end
+
+    # Fetches each currently running node manager's view of the
+    # world in terms of which nodes they think are available/unavailable.
+    #
+    # @return [Hash<String, Array>] a hash of node manager to host states
+    def fetch_node_manager_states
+      states = {}
+      @zk.children(current_state_root).each do |child|
+        full_path = "#{current_state_root}/#{child}"
+        begin
+          states[child] = symbolize_keys(decode(@zk.get(full_path).first))
+        rescue ZK::Exceptions::NoNode
+          # ignore, this is an edge case that can happen when a node manager
+          # process dies while fetching its state
+        rescue => ex
+          logger.error("Failed to fetch states for #{full_path}: #{ex.inspect}")
+        end
+      end
+      states
+    end
+
+    # Builds current snapshots of nodes across all running node managers.
+    #
+    # @return [Hash<Node, NodeSnapshot>] the snapshots for all nodes
+    def current_node_snapshots
+      nodes = {}
+      snapshots = Hash.new { |h, k| h[k] = NodeSnapshot.new(k) }
+      fetch_node_manager_states.each do |node_manager, states|
+        available, unavailable = states.values_at(:available, :unavailable)
+        available.each do |node_string, latency|
+          node = nodes[node_string] ||= node_from(node_string)
+          snapshots[node].viewable_by(node_manager, latency)
+        end
+        unavailable.each do |node_string|
+          node = nodes[node_string] ||= node_from(node_string)
+          snapshots[node].unviewable_by(node_manager)
+        end
+      end
+
+      snapshots
+    end
+
+    # Waits until this node manager becomes the master.
+    def wait_until_master
+      logger.info('Waiting to become master Node Manager ...')
+
+      with_lock do
+        @master_manager = true
+        logger.info('Acquired master Node Manager lock.')
+        logger.info("Configured node strategy #{@node_strategy.class}")
+        logger.info("Configured failover strategy #{@failover_strategy.class}")
+        logger.info("Required Node Managers to make a decision: #{@required_node_managers}")
+        manage_nodes
+      end
+    end
+
+    # Manages the redis nodes by periodically processing snapshots.
+    def manage_nodes
+      # Re-discover nodes, since the state of the world may have been changed
+      # by the time we've become the primary node manager.
+      discover_nodes
+
+      # ensure that slaves are correctly pointing to this master
+      redirect_slaves_to(@master)
+
+      # Periodically update master config state.
+      while running? && master_manager?
+        @zk_lock.assert!
+        sleep(CHECK_INTERVAL)
+
+        @lock.synchronize do
+          snapshots = current_node_snapshots
+          if ensure_sufficient_node_managers(snapshots)
+            snapshots.each_key do |node|
+              update_master_state(node, snapshots)
+            end
+
+            # flush current master state
+            write_current_redis_nodes
+
+            # check if we've exhausted our attempts to promote a master
+            unless @master
+              @master_promotion_attempts += 1
+              raise NoMasterError if @master_promotion_attempts > MAX_PROMOTION_ATTEMPTS
+            end
+          end
+        end
+      end
+    end
+
+    # Creates a Node instance from a string.
+    #
+    # @param [String] node_string a string representation of a node (e.g., host:port)
+    # @return [Node] the Node representation
+    def node_from(node_string)
+      return if node_string.nil?
+      host, port = node_string.split(':', 2)
+      Node.new(:host => host, :port => port, :password => @options[:password])
     end
 
     # Executes a block wrapped in a ZK exclusive lock.
     def with_lock
-      @zk_lock = @zk.locker(@lock_path)
+      @zk_lock ||= @zk.locker('master_redis_node_manager_lock')
 
       begin
         @zk_lock.lock!(true)
@@ -418,39 +636,96 @@ module RedisFailover
       end
 
       if running?
+        @zk_lock.assert!
         yield
       end
     ensure
-      @zk_lock.unlock! if @zk_lock
+      if @zk_lock
+        begin
+          @zk_lock.unlock!
+        rescue => ex
+          logger.warn("Failed to release lock: #{ex.inspect}")
+        end
+      end
     end
 
     # Perform a manual failover to a redis node.
     def perform_manual_failover
-      @mutex.synchronize do
-        return unless running? && @leader && @zk_lock
+      @lock.synchronize do
+        return unless running? && @master_manager && @zk_lock
         @zk_lock.assert!
-        new_master = @zk.get(@manual_znode, :watch => true).first
+        new_master = @zk.get(manual_failover_path, :watch => true).first
         return unless new_master && new_master.size > 0
         logger.info("Received manual failover request for: #{new_master}")
         logger.info("Current nodes: #{current_nodes.inspect}")
-        node = new_master == ManualFailover::ANY_SLAVE ?
-          @slaves.shuffle.first : node_from(new_master)
+        snapshots = current_node_snapshots
+
+        node = if new_master == ManualFailover::ANY_SLAVE
+          failover_strategy_candidate(snapshots)
+        else
+          node_from(new_master)
+        end
+
         if node
-          handle_manual_failover(node)
+          handle_manual_failover(node, snapshots)
         else
           logger.error('Failed to perform manual failover, no candidate found.')
         end
       end
     rescue => ex
-      logger.error("Error handling a manual failover: #{ex.inspect}")
+      logger.error("Error handling manual failover: #{ex.inspect}")
       logger.error(ex.backtrace.join("\n"))
     ensure
-      @zk.stat(@manual_znode, :watch => true)
+      @zk.stat(manual_failover_path, :watch => true)
     end
 
     # @return [Boolean] true if running, false otherwise
     def running?
-      !@shutdown
+      @lock.synchronize { !@shutdown }
+    end
+
+    # @return [String] a stringified version of redis nodes
+    def stringify_nodes(nodes)
+      "(#{nodes.map(&:to_s).join(', ')})"
+    end
+
+    # Determines if each snapshot has a sufficient number of node managers.
+    #
+    # @param [Hash<Node, Snapshot>] snapshots the current snapshots
+    # @return [Boolean] true if sufficient, false otherwise
+    def ensure_sufficient_node_managers(snapshots)
+      currently_sufficient = true
+      snapshots.each do |node, snapshot|
+        node_managers = snapshot.node_managers
+        if node_managers.size < @required_node_managers
+          logger.error("Not enough Node Managers in snapshot for node #{node}. " +
+            "Required: #{@required_node_managers}, " +
+            "Available: #{node_managers.size} #{node_managers}")
+          currently_sufficient = false
+        end
+      end
+
+      if currently_sufficient && !@sufficient_node_managers
+        logger.info("Required Node Managers are visible: #{@required_node_managers}")
+      end
+
+      @sufficient_node_managers = currently_sufficient
+      @sufficient_node_managers
+    end
+
+    # Invokes the configured failover strategy.
+    #
+    # @param [Hash<Node, NodeSnapshot>] snapshots the node snapshots
+    # @return [Node] a failover candidate
+    def failover_strategy_candidate(snapshots)
+      # only include nodes that this master Node Manager can see
+      filtered_snapshots = snapshots.select do |node, snapshot|
+        snapshot.viewable_by?(manager_id)
+      end
+
+      logger.info('Attempting to find candidate from snapshots:')
+      logger.info("\n" + filtered_snapshots.values.join("\n"))
+      @failover_strategy.find_candidate(filtered_snapshots)
     end
   end
 end