redis_failover 0.9.4 → 0.9.5

data/.gitignore

@@ -16,4 +16,4 @@ test/tmp
 test/version_tmp
 tmp
 tags
-
+.DS_Store

data/Changes.md

@@ -1,3 +1,8 @@
+0.9.5
+-----------
+- Introduce a safer master node discovery process for the Node Manager (#34)
+- Improved shutdown process for Node Manager
+
 0.9.4
 -----------
 - Preserve original master by reading from existing znode state.

data/README.md

@@ -155,7 +155,7 @@ redis_failover uses YARD for its API documentation. Refer to the generated [API
 
 ## Requirements
 
-- redis_failover is actively tested against MRI 1.9.2/1.9.3 and JRuby 1.6.7 (1.9 mode only). Other rubies may work, although I don't actively test against them.
+- redis_failover is actively tested against MRI 1.8.7/1.9.2/1.9.3 and JRuby 1.6.7 (1.9 mode only). Other rubies may work, although I don't actively test against them.
 - redis_failover requires a ZooKeeper service cluster to ensure reliability and data consistency. ZooKeeper is very simple and easy to get up and running. Please refer to this [Quick ZooKeeper Guide](https://github.com/ryanlecompte/redis_failover/wiki/Quick-ZooKeeper-Guide) to get up and running quickly if you don't already have ZooKeeper as a part of your environment.
 
 ## Considerations

data/lib/redis_failover/errors.rb

@@ -25,6 +25,13 @@ module RedisFailover
   class NoMasterError < Error
   end
 
+  # Raised when more than one master is found on startup.
+  class MultipleMastersError < Error
+    def initialize(nodes)
+      super("Multiple nodes with master role: #{nodes.map(&:to_s)}")
+    end
+  end
+
   # Raised when no slave is currently available.
   class NoSlaveError < Error
   end

data/lib/redis_failover/node.rb

@@ -118,7 +118,6 @@ module RedisFailover
     end
     alias_method :eql?, :==
 
-
     # @return [Integer] a hash value for this node
     def hash
       to_s.hash

data/lib/redis_failover/node_manager.rb

@@ -32,13 +32,11 @@ module RedisFailover
       @znode = @options[:znode_path] || Util::DEFAULT_ZNODE_PATH
       @manual_znode = ManualFailover::ZNODE_PATH
       @mutex = Mutex.new
-
-      # Name for the znode that handles exclusive locking between multiple
-      # Node Manager processes. Whoever holds the lock will be considered
-      # the "master" Node Manager, and will be responsible for monitoring
-      # the redis nodes. When a Node Manager that holds the lock disappears
-      # or fails, another Node Manager process will grab the lock and
-      # become the
+      @shutdown = false
+      @leader = false
+      @master = nil
+      @slaves = []
+      @unavailable = []
       @lock_path = "#{@znode}_lock".freeze
     end
 
@@ -46,21 +44,22 @@ module RedisFailover
     #
     # @note This method does not return until the manager terminates.
     def start
+      return unless running?
       @queue = Queue.new
-      @leader = false
       setup_zk
       logger.info('Waiting to become master Node Manager ...')
       with_lock do
         @leader = true
         logger.info('Acquired master Node Manager lock')
-        discover_nodes
-        initialize_path
-        spawn_watchers
-        handle_state_reports
+        if discover_nodes
+          initialize_path
+          spawn_watchers
+          handle_state_reports
+        end
       end
     rescue *ZK_ERRORS => ex
       logger.error("ZK error while attempting to manage nodes: #{ex.inspect}")
-      shutdown
+      reset
       retry
     end
 
@@ -73,16 +72,23 @@ module RedisFailover
       @queue << [node, state]
     end
 
-    # Performs a graceful shutdown of the manager.
-    def shutdown
-      @queue.clear
-      @queue << nil
+    # Performs a reset of the manager.
+    def reset
+      @leader = false
       @watchers.each(&:shutdown) if @watchers
-      sleep(TIMEOUT)
+      @queue.clear
       @zk.close! if @zk
       @zk_lock = nil
     end
 
+    # Initiates a graceful shutdown.
+    def shutdown
+      logger.info('Shutting down ...')
+      @mutex.synchronize do
+        @shutdown = true
+      end
+    end
+
     private
 
     # Configures the ZooKeeper client.
@@ -92,17 +98,8 @@ module RedisFailover
       @zk.on_expired_session { notify_state(:zk_disconnected, nil) }
 
       @zk.register(@manual_znode) do |event|
-        @mutex.synchronize do
-          begin
-            if event.node_created? || event.node_changed?
-              schedule_manual_failover
-            end
-          rescue => ex
-            logger.error("Error scheduling a manual failover: #{ex.inspect}")
-            logger.error(ex.backtrace.join("\n"))
-          ensure
-            @zk.stat(@manual_znode, :watch => true)
-          end
+        if event.node_created? || event.node_changed?
+          perform_manual_failover
         end
       end
 
@@ -112,23 +109,23 @@ module RedisFailover
 
     # Handles periodic state reports from {RedisFailover::NodeWatcher} instances.
     def handle_state_reports
-      while state_report = @queue.pop
-        # Ensure that we still have the master lock.
-        @zk_lock.assert!
-
+      while running? && (state_report = @queue.pop)
         begin
-          node, state = state_report
-          case state
-          when :unavailable     then handle_unavailable(node)
-          when :available       then handle_available(node)
-          when :syncing         then handle_syncing(node)
-          when :manual_failover then handle_manual_failover(node)
-          when :zk_disconnected then raise ZKDisconnectedError
-          else raise InvalidNodeStateError.new(node, state)
-          end
+          @mutex.synchronize do
+            return unless running?
+            @zk_lock.assert!
+            node, state = state_report
+            case state
+            when :unavailable     then handle_unavailable(node)
+            when :available       then handle_available(node)
+            when :syncing         then handle_syncing(node)
+            when :zk_disconnected then raise ZKDisconnectedError
+            else raise InvalidNodeStateError.new(node, state)
+            end
 
-          # flush current state
-          write_state
+            # flush current state
+            write_state
+          end
         rescue *ZK_ERRORS
           # fail hard if this is a ZK connection-related error
           raise
@@ -204,7 +201,7 @@ module RedisFailover
       logger.info("Handling manual failover")
 
       # make current master a slave, and promote new master
-      @slaves << @master
+      @slaves << @master if @master
       @slaves.delete(node)
       promote_new_master(node)
     end
@@ -234,16 +231,35 @@ module RedisFailover
     end
 
     # Discovers the current master and slave nodes.
+    # @return [Boolean] true if nodes successfully discovered, false otherwise
     def discover_nodes
-      nodes = @options[:nodes].map { |opts| Node.new(opts) }.uniq
-      @master = find_existing_master || find_master(nodes)
-      @unavailable = []
-      @slaves = nodes - [@master]
-      logger.info("Managing master (#{@master}) and slaves" +
-        " (#{@slaves.map(&:to_s).join(', ')})")
-
-      # ensure that slaves are correctly pointing to this master
-      redirect_slaves_to(@master) if @master
+      @mutex.synchronize do
+        return false unless running?
+        nodes = @options[:nodes].map { |opts| Node.new(opts) }.uniq
+        if @master = find_existing_master
+          logger.info("Using master #{@master} from existing znode config.")
+        elsif @master = guess_master(nodes)
+          logger.info("Guessed master #{@master} from known redis nodes.")
+        end
+        @slaves = nodes - [@master]
+        logger.info("Managing master (#{@master}) and slaves " +
+          "(#{@slaves.map(&:to_s).join(', ')})")
+        # ensure that slaves are correctly pointing to this master
+        redirect_slaves_to(@master)
+        true
+      end
+    rescue NodeUnavailableError, NoMasterError, MultipleMastersError => ex
+      msg = <<-MSG.gsub(/\s+/, ' ')
+        Failed to discover master node: #{ex.inspect}
+        In order to ensure a safe startup, redis_failover requires that all redis
+        nodes be accessible, and only a single node indicating that it's the master.
+        In order to fix this, you can perform a manual failover via redis_failover,
+        or manually fix the individual redis servers. This discovery process will
+        retry in #{TIMEOUT}s.
+      MSG
+      logger.warn(msg)
+      sleep(TIMEOUT)
+      retry
     end
 
     # Seeds the initial node master from an existing znode config.
@@ -251,7 +267,7 @@ module RedisFailover
       if data = @zk.get(@znode).first
         nodes = symbolize_keys(decode(data))
         master = node_from(nodes[:master])
-        logger.info("Master from existing config: #{master || 'none'}")
+        logger.info("Master from existing znode config: #{master || 'none'}")
         master
       end
     rescue ZK::Exceptions::NoNode
@@ -281,14 +297,11 @@ module RedisFailover
     #
     # @param [Array<Node>] nodes the nodes to search
     # @return [Node] the found master node, nil if not found
-    def find_master(nodes)
-      nodes.find do |node|
-        begin
-          node.master?
-        rescue NodeUnavailableError
-          false
-        end
-      end
+    def guess_master(nodes)
+      master_nodes = nodes.select { |node| node.master? }
+      raise NoMasterError if master_nodes.empty?
+      raise MultipleMastersError.new(master_nodes) if master_nodes.size > 1
+      master_nodes.first
     end
 
     # Redirects all slaves to the specified node.
@@ -378,32 +391,44 @@ module RedisFailover
     # Executes a block wrapped in a ZK exclusive lock.
     def with_lock
       @zk_lock = @zk.locker(@lock_path)
-      @zk_lock.lock(true)
-      yield
+      while running? && !@zk_lock.lock
+        sleep(TIMEOUT)
+      end
+
+      if running?
+        yield
+      end
     ensure
       @zk_lock.unlock! if @zk_lock
     end
 
-    # Schedules a manual failover to a redis node.
-    def schedule_manual_failover
-      return unless @leader
-      new_master = @zk.get(@manual_znode, :watch => true).first
-      return unless new_master && new_master.size > 0
-      logger.info("Received manual failover request for: #{new_master}")
-      logger.info("Current nodes: #{current_nodes.inspect}")
-
-      node = if new_master == ManualFailover::ANY_SLAVE
-        @slaves.shuffle.first
-      else
-        host, port = new_master.split(':', 2)
-        Node.new(:host => host, :port => port, :password => @options[:password])
+    # Perform a manual failover to a redis node.
+    def perform_manual_failover
+      @mutex.synchronize do
+        return unless running? && @leader && @zk_lock
+        @zk_lock.assert!
+        new_master = @zk.get(@manual_znode, :watch => true).first
+        return unless new_master && new_master.size > 0
+        logger.info("Received manual failover request for: #{new_master}")
+        logger.info("Current nodes: #{current_nodes.inspect}")
+        node = new_master == ManualFailover::ANY_SLAVE ?
+          @slaves.shuffle.first : node_from(new_master)
+        if node
+          handle_manual_failover(node)
+        else
+          logger.error('Failed to perform manual failover, no candidate found.')
+        end
       end
+    rescue => ex
+      logger.error("Error handling a manual failover: #{ex.inspect}")
+      logger.error(ex.backtrace.join("\n"))
+    ensure
+      @zk.stat(@manual_znode, :watch => true)
+    end
 
-      if node
-        notify_state(node, :manual_failover)
-      else
-        logger.error('Failed to perform manual failover, no candidate found.')
-      end
+    # @return [Boolean] true if running, false otherwise
+    def running?
+      !@shutdown
     end
   end
 end

data/lib/redis_failover/node_watcher.rb

@@ -35,8 +35,8 @@ module RedisFailover
       @done = true
       @node.wakeup
       @monitor_thread.join if @monitor_thread
-    rescue
-      # best effort
+    rescue => ex
+      logger.warn("Failed to gracefully shutdown watcher for #{@node}")
     end
 
     private

data/lib/redis_failover/runner.rb

@@ -8,22 +8,20 @@ module RedisFailover
     #   Node Manager is gracefully stopped
     def self.run(options)
       options = CLI.parse(options)
-      @node_manager = NodeManager.new(options)
-      trap_signals
-      @node_manager_thread = Thread.new { @node_manager.start }
-      @node_manager_thread.join
+      node_manager = NodeManager.new(options)
+      trap_signals(node_manager)
+      node_manager.start
     end
 
     # Traps shutdown signals.
-    def self.trap_signals
+    # @param [NodeManager] node_manager the node manager
+    def self.trap_signals(node_manager)
       [:INT, :TERM].each do |signal|
         trap(signal) do
-          Util.logger.info('Shutting down ...')
-          @node_manager.shutdown
-          @node_manager_thread.join
-          exit(0)
+          node_manager.shutdown
         end
       end
     end
+    private_class_method :trap_signals
   end
 end

data/lib/redis_failover/version.rb

@@ -1,3 +1,3 @@
 module RedisFailover
-  VERSION = '0.9.4'
+  VERSION = '0.9.5'
 end

data/spec/node_manager_spec.rb

@@ -108,5 +108,29 @@ module RedisFailover
         end
       end
     end
+
+    describe '#guess_master' do
+      let(:node1) { Node.new(:host => 'node1').extend(RedisStubSupport) }
+      let(:node2) { Node.new(:host => 'node2').extend(RedisStubSupport) }
+      let(:node3) { Node.new(:host => 'node3').extend(RedisStubSupport) }
+
+      it 'raises error when no master is found' do
+        node1.make_slave!(node3)
+        node2.make_slave!(node3)
+        expect { manager.guess_master([node1, node2]) }.to raise_error(NoMasterError)
+      end
+
+      it 'raises error when multiple masters found' do
+        node1.make_master!
+        node2.make_master!
+        expect { manager.guess_master([node1, node2]) }.to raise_error(MultipleMastersError)
+      end
+
+      it 'raises error when a node can not be reached' do
+        node1.make_master!
+        node2.redis.make_unavailable!
+        expect { manager.guess_master([node1, node2]) }.to raise_error(NodeUnavailableError)
+      end
+    end
   end
 end

data/spec/support/node_manager_stub.rb

@@ -1,11 +1,12 @@
 module RedisFailover
   class NodeManagerStub < NodeManager
     attr_accessor :master
-    public :current_nodes
+    # HACK - this will go away once we refactor the tests to use a real ZK/Redis server.
+    public :current_nodes, :guess_master
 
     def discover_nodes
       # only discover nodes once in testing
-      return if @nodes_discovered
+      return true if @nodes_discovered
 
       master = Node.new(:host => 'master')
       slave = Node.new(:host => 'slave')

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: redis_failover
 version: !ruby/object:Gem::Version
-  version: 0.9.4
+  version: 0.9.5
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-08-31 00:00:00.000000000 Z
+date: 2012-09-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
@@ -189,7 +189,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -3042115734438994013
+      hash: -2193925210006995870
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -198,7 +198,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -3042115734438994013
+      hash: -2193925210006995870
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.23