nogara-redis_failover 0.9.1 → 0.9.7

data/.gitignore

@@ -16,4 +16,4 @@ test/tmp
 test/version_tmp
 tmp
 tags
-
+.DS_Store

data/Changes.md

@@ -1,3 +1,35 @@
+0.9.7
+-----------
+- Stubbed Client#client to return itself, fixes a fork reconnect bug with Resque (dbalatero)
+
+0.9.6
+-----------
+- Handle the node discovery error condition where the znode points to a master that is now a slave.
+
+0.9.5
+-----------
+- Introduce a safer master node discovery process for the Node Manager (#34)
+- Improved shutdown process for Node Manager
+
+0.9.4
+-----------
+- Preserve original master by reading from existing znode state.
+- Prevent Timeout::Error from bringing down the process (#32) (@eric)
+
+0.9.3
+-----------
+- Add lock assert for Node Manager.
+
+0.9.2
+-----------
+- Improved exception handling in NodeWatcher.
+
+0.9.1
+-----------
+- Improve nested exception handling.
+- Fix manual failover support when znode does not exist first.
+- Various fixes to work better with 1.8.7.
+
 0.9.0
 -----------
 - Make Node Manager's lock path vary with its main znode. (Bira)

data/README.md

@@ -155,7 +155,7 @@ redis_failover uses YARD for its API documentation. Refer to the generated [API
 
 ## Requirements
 
-- redis_failover is actively tested against MRI 1.9.2/1.9.3 and JRuby 1.6.7 (1.9 mode only). Other rubies may work, although I don't actively test against them.
+- redis_failover is actively tested against MRI 1.8.7/1.9.2/1.9.3 and JRuby 1.6.7 (1.9 mode only). Other rubies may work, although I don't actively test against them.
 - redis_failover requires a ZooKeeper service cluster to ensure reliability and data consistency. ZooKeeper is very simple and easy to get up and running. Please refer to this [Quick ZooKeeper Guide](https://github.com/ryanlecompte/redis_failover/wiki/Quick-ZooKeeper-Guide) to get up and running quickly if you don't already have ZooKeeper as a part of your environment.
 
 ## Considerations
@@ -175,6 +175,8 @@ redis_failover uses YARD for its API documentation. Refer to the generated [API
 - To learn more about ZooKeeper, see the official [ZooKeeper](http://zookeeper.apache.org/) site.
 - See the [Quick ZooKeeper Guide](https://github.com/ryanlecompte/redis_failover/wiki/Quick-ZooKeeper-Guide) for a quick guide to getting ZooKeeper up and running with redis_failover.
 - To learn more about how ZooKeeper handles network partitions, see [ZooKeeper Failure Scenarios](http://wiki.apache.org/hadoop/ZooKeeper/FailureScenarios)
+- Slides for a [lightning talk](http://www.slideshare.net/ryanlecompte/handling-redis-failover-with-zookeeper) that I gave at BaRuCo 2012.
+- Feel free to join #zk-gem on the IRC freenode network. We're usually hanging out there talking about ZooKeeper and redis_failover.
 
 
 ## License

data/lib/redis_failover/client.rb

@@ -64,10 +64,18 @@ module RedisFailover
       build_clients
     end
 
-
-    # Resque wants to use Resque.redis.reconnect to recreate all connections. So
-    # we provide ourselves as a client to receive the "reconnect" message
-    def client
+    # Stubs this method to return this RedisFailover::Client object.
+    #
+    # Some libraries (Resque) assume they can access the `client` via this method,
+    # but we don't want to actually ever expose the internal Redis connections.
+    #
+    # By returning `self` here, we can add stubs for functionality like #reconnect,
+    # and everything will Just Work.
+    #
+    # Takes an *args array for safety only.
+    #
+    # @return [RedisFailover::Client]
+    def client(*args)
       self
     end
 
@@ -190,6 +198,8 @@ module RedisFailover
       else
         logger.error("Unknown ZK node event: #{event.inspect}")
       end
+    ensure
+      @zk.stat(@znode, :watch => true)
     end
 
     # Determines if a method is a known redis operation.
@@ -250,7 +260,7 @@ module RedisFailover
     # @raise [NoMasterError] if no master fallback is available
     def slave
       # pick a slave, if none available fallback to master
-      if slave = @lock.synchronize { @slaves.sample }
+      if slave = @lock.synchronize { @slaves.shuffle.first }
         verify_role!(slave, :slave)
         return slave
       end
@@ -266,7 +276,7 @@ module RedisFailover
           return unless nodes_changed?(nodes)
 
           purge_clients
-          logger.info("Building new clients for nodes #{nodes}")
+          logger.info("Building new clients for nodes #{nodes.inspect}")
           new_master = new_clients_for(nodes[:master]).first if nodes[:master]
           new_slaves = new_clients_for(*nodes[:slaves])
           @master = new_master
@@ -300,7 +310,7 @@ module RedisFailover
     def fetch_nodes
       data = @zk.get(@znode, :watch => true).first
       nodes = symbolize_keys(decode(data))
-      logger.debug("Fetched nodes: #{nodes}")
+      logger.debug("Fetched nodes: #{nodes.inspect}")
 
       nodes
     rescue Zookeeper::Exceptions::InheritedConnectionError => ex

data/lib/redis_failover/errors.rb

@@ -25,6 +25,13 @@ module RedisFailover
   class NoMasterError < Error
   end
 
+  # Raised when more than one master is found on startup.
+  class MultipleMastersError < Error
+    def initialize(nodes)
+      super("Multiple nodes with master role: #{nodes.map(&:to_s)}")
+    end
+  end
+
   # Raised when no slave is currently available.
   class NoSlaveError < Error
   end

data/lib/redis_failover/node.rb

@@ -118,7 +118,6 @@ module RedisFailover
     end
     alias_method :eql?, :==
 
-
     # @return [Integer] a hash value for this node
     def hash
       to_s.hash
@@ -175,14 +174,14 @@ module RedisFailover
         redis = new_client
         yield redis
       end
-    rescue
-      raise NodeUnavailableError, self, caller
+    rescue Exception => ex
+      raise NodeUnavailableError, "#{ex.class}: #{ex.message}", ex.backtrace
     ensure
       if redis
         begin
           redis.client.disconnect
-        rescue
-          raise NodeUnavailableError, self, caller
+        rescue Exception => ex
+          raise NodeUnavailableError, "#{ex.class}: #{ex.message}", ex.backtrace
         end
       end
     end

data/lib/redis_failover/node_manager.rb

@@ -10,7 +10,22 @@ module RedisFailover
     include Util
 
     # Number of seconds to wait before retrying bootstrap process.
-    TIMEOUT = 5
+    TIMEOUT = 3
+
+    # ZK Errors that the Node Manager cares about.
+    ZK_ERRORS = [
+      ZK::Exceptions::LockAssertionFailedError,
+      ZK::Exceptions::InterruptedSession,
+      ZKDisconnectedError
+    ].freeze
+
+    # Errors that can happen during the node discovery process.
+    NODE_DISCOVERY_ERRORS = [
+      InvalidNodeRoleError,
+      NodeUnavailableError,
+      NoMasterError,
+      MultipleMastersError
+    ].freeze
 
     # Creates a new instance.
     #
@@ -26,13 +41,11 @@ module RedisFailover
       @znode = @options[:znode_path] || Util::DEFAULT_ZNODE_PATH
       @manual_znode = ManualFailover::ZNODE_PATH
       @mutex = Mutex.new
-
-      # Name for the znode that handles exclusive locking between multiple
-      # Node Manager processes. Whoever holds the lock will be considered
-      # the "master" Node Manager, and will be responsible for monitoring
-      # the redis nodes. When a Node Manager that holds the lock disappears
-      # or fails, another Node Manager process will grab the lock and
-      # become the
+      @shutdown = false
+      @leader = false
+      @master = nil
+      @slaves = []
+      @unavailable = []
       @lock_path = "#{@znode}_lock".freeze
     end
 
@@ -40,23 +53,22 @@ module RedisFailover
     #
     # @note This method does not return until the manager terminates.
     def start
+      return unless running?
       @queue = Queue.new
-      @leader = false
       setup_zk
       logger.info('Waiting to become master Node Manager ...')
-      @zk.with_lock(@lock_path) do
+      with_lock do
         @leader = true
         logger.info('Acquired master Node Manager lock')
-        discover_nodes
-        initialize_path
-        spawn_watchers
-        handle_state_reports
+        if discover_nodes
+          initialize_path
+          spawn_watchers
+          handle_state_reports
+        end
       end
-    rescue ZK::Exceptions::InterruptedSession, ZKDisconnectedError => ex
+    rescue *ZK_ERRORS => ex
       logger.error("ZK error while attempting to manage nodes: #{ex.inspect}")
-      logger.error(ex.backtrace.join("\n"))
-      shutdown
-      sleep(TIMEOUT)
+      reset
       retry
     end
 
@@ -69,12 +81,21 @@ module RedisFailover
       @queue << [node, state]
     end
 
-    # Performs a graceful shutdown of the manager.
-    def shutdown
-      @queue.clear
-      @queue << nil
+    # Performs a reset of the manager.
+    def reset
+      @leader = false
       @watchers.each(&:shutdown) if @watchers
+      @queue.clear
       @zk.close! if @zk
+      @zk_lock = nil
+    end
+
+    # Initiates a graceful shutdown.
+    def shutdown
+      logger.info('Shutting down ...')
+      @mutex.synchronize do
+        @shutdown = true
+      end
     end
 
     private
@@ -86,10 +107,8 @@ module RedisFailover
       @zk.on_expired_session { notify_state(:zk_disconnected, nil) }
 
       @zk.register(@manual_znode) do |event|
-        @mutex.synchronize do
-          if event.node_changed?
-            schedule_manual_failover
-          end
+        if event.node_created? || event.node_changed?
+          perform_manual_failover
         end
       end
 
@@ -99,21 +118,24 @@ module RedisFailover
 
     # Handles periodic state reports from {RedisFailover::NodeWatcher} instances.
     def handle_state_reports
-      while state_report = @queue.pop
+      while running? && (state_report = @queue.pop)
         begin
-          node, state = state_report
-          case state
-          when :unavailable     then handle_unavailable(node)
-          when :available       then handle_available(node)
-          when :syncing         then handle_syncing(node)
-          when :manual_failover then handle_manual_failover(node)
-          when :zk_disconnected then raise ZKDisconnectedError
-          else raise InvalidNodeStateError.new(node, state)
+          @mutex.synchronize do
+            return unless running?
+            @zk_lock.assert!
+            node, state = state_report
+            case state
+            when :unavailable     then handle_unavailable(node)
+            when :available       then handle_available(node)
+            when :syncing         then handle_syncing(node)
+            when :zk_disconnected then raise ZKDisconnectedError
+            else raise InvalidNodeStateError.new(node, state)
+            end
+
+            # flush current state
+            write_state
           end
-
-          # flush current state
-          write_state
-        rescue ZK::Exceptions::InterruptedSession, ZKDisconnectedError
+        rescue *ZK_ERRORS
           # fail hard if this is a ZK connection-related error
           raise
         rescue => ex
@@ -148,7 +170,7 @@ module RedisFailover
       reconcile(node)
 
       # no-op if we already know about this node
-      return if @master == node || @slaves.include?(node)
+      return if @master == node || (@master && @slaves.include?(node))
       logger.info("Handling available node: #{node}")
 
       if @master
@@ -188,7 +210,7 @@ module RedisFailover
       logger.info("Handling manual failover")
 
       # make current master a slave, and promote new master
-      @slaves << @master
+      @slaves << @master if @master
       @slaves.delete(node)
       promote_new_master(node)
     end
@@ -218,21 +240,69 @@ module RedisFailover
     end
 
     # Discovers the current master and slave nodes.
+    # @return [Boolean] true if nodes successfully discovered, false otherwise
     def discover_nodes
-      @unavailable = []
-      nodes = @options[:nodes].map { |opts| Node.new(opts) }.uniq
-      @master = find_master(nodes)
-      @slaves = nodes - [@master]
-      logger.info("Managing master (#{@master}) and slaves" +
-        " (#{@slaves.map(&:to_s).join(', ')})")
-
-      # ensure that slaves are correctly pointing to this master
-      redirect_slaves_to(@master) if @master
+      @mutex.synchronize do
+        return false unless running?
+        nodes = @options[:nodes].map { |opts| Node.new(opts) }.uniq
+        if @master = find_existing_master
+          logger.info("Using master #{@master} from existing znode config.")
+        elsif @master = guess_master(nodes)
+          logger.info("Guessed master #{@master} from known redis nodes.")
+        end
+        @slaves = nodes - [@master]
+        logger.info("Managing master (#{@master}) and slaves " +
+          "(#{@slaves.map(&:to_s).join(', ')})")
+        # ensure that slaves are correctly pointing to this master
+        redirect_slaves_to(@master)
+        true
+      end
+    rescue *NODE_DISCOVERY_ERRORS => ex
+      msg = <<-MSG.gsub(/\s+/, ' ')
+        Failed to discover master node: #{ex.inspect}
+        In order to ensure a safe startup, redis_failover requires that all redis
+        nodes be accessible, and only a single node indicating that it's the master.
+        In order to fix this, you can perform a manual failover via redis_failover,
+        or manually fix the individual redis servers. This discovery process will
+        retry in #{TIMEOUT}s.
+      MSG
+      logger.warn(msg)
+      sleep(TIMEOUT)
+      retry
+    end
+
+    # Seeds the initial node master from an existing znode config.
+    def find_existing_master
+      if data = @zk.get(@znode).first
+        nodes = symbolize_keys(decode(data))
+        master = node_from(nodes[:master])
+        logger.info("Master from existing znode config: #{master || 'none'}")
+        # Check for case where a node previously thought to be the master was
+        # somehow manually reconfigured to be a slave outside of the node manager's
+        # control.
+        if master && master.slave?
+          raise InvalidNodeRoleError.new(master, :master, :slave)
+        end
+        master
+      end
+    rescue ZK::Exceptions::NoNode
+      # blank slate, no last known master
+      nil
+    end
+
+    # Creates a Node instance from a string.
+    #
+    # @param [String] node_string a string representation of a node (e.g., host:port)
+    # @return [Node] the Node representation
+    def node_from(node_string)
+      return if node_string.nil?
+      host, port = node_string.split(':', 2)
+      Node.new(:host => host, :port => port, :password => @options[:password])
     end
 
     # Spawns the {RedisFailover::NodeWatcher} instances for each managed node.
     def spawn_watchers
-      @watchers = [@master, @slaves, @unavailable].flatten.map do |node|
+      @watchers = [@master, @slaves, @unavailable].flatten.compact.map do |node|
         NodeWatcher.new(self, node, @options[:max_failures] || 3)
       end
       @watchers.each(&:watch)
@@ -242,14 +312,11 @@ module RedisFailover
     #
     # @param [Array<Node>] nodes the nodes to search
     # @return [Node] the found master node, nil if not found
-    def find_master(nodes)
-      nodes.find do |node|
-        begin
-          node.master?
-        rescue NodeUnavailableError
-          false
-        end
-      end
+    def guess_master(nodes)
+      master_nodes = nodes.select { |node| node.master? }
+      raise NoMasterError if master_nodes.empty?
+      raise MultipleMastersError.new(master_nodes) if master_nodes.size > 1
+      master_nodes.first
     end
 
     # Redirects all slaves to the specified node.
@@ -336,19 +403,47 @@ module RedisFailover
       @zk.set(@znode, encode(current_nodes))
     end
 
-    # Schedules a manual failover to a redis node.
-    def schedule_manual_failover
-      return unless @leader
-      new_master = @zk.get(@manual_znode, :watch => true).first
-      logger.info("Received manual failover request for: #{new_master}")
+    # Executes a block wrapped in a ZK exclusive lock.
+    def with_lock
+      @zk_lock = @zk.locker(@lock_path)
+      while running? && !@zk_lock.lock
+        sleep(TIMEOUT)
+      end
 
-      node = if new_master == ManualFailover::ANY_SLAVE
-        @slaves.sample
-      else
-        host, port = new_master.split(':', 2)
-        Node.new(:host => host, :port => port, :password => @options[:password])
+      if running?
+        yield
       end
-      notify_state(node, :manual_failover) if node
+    ensure
+      @zk_lock.unlock! if @zk_lock
+    end
+
+    # Perform a manual failover to a redis node.
+    def perform_manual_failover
+      @mutex.synchronize do
+        return unless running? && @leader && @zk_lock
+        @zk_lock.assert!
+        new_master = @zk.get(@manual_znode, :watch => true).first
+        return unless new_master && new_master.size > 0
+        logger.info("Received manual failover request for: #{new_master}")
+        logger.info("Current nodes: #{current_nodes.inspect}")
+        node = new_master == ManualFailover::ANY_SLAVE ?
+          @slaves.shuffle.first : node_from(new_master)
+        if node
+          handle_manual_failover(node)
+        else
+          logger.error('Failed to perform manual failover, no candidate found.')
+        end
+      end
+    rescue => ex
+      logger.error("Error handling a manual failover: #{ex.inspect}")
+      logger.error(ex.backtrace.join("\n"))
+    ensure
+      @zk.stat(@manual_znode, :watch => true)
+    end
+
+    # @return [Boolean] true if running, false otherwise
+    def running?
+      !@shutdown
     end
   end
 end

data/lib/redis_failover/node_watcher.rb

@@ -35,8 +35,8 @@ module RedisFailover
       @done = true
       @node.wakeup
       @monitor_thread.join if @monitor_thread
-    rescue
-      # best effort
+    rescue => ex
+      logger.warn("Failed to gracefully shutdown watcher for #{@node}")
     end
 
     private
@@ -59,12 +59,16 @@ module RedisFailover
             notify(:available)
             @node.wait
           end
-        rescue NodeUnavailableError
+        rescue NodeUnavailableError => ex
+          logger.debug("Failed to communicate with node #{@node}: #{ex.inspect}")
           failures += 1
           if failures >= @max_failures
             notify(:unavailable)
             failures = 0
           end
+        rescue Exception => ex
+          logger.error("Unexpected error while monitoring node #{@node}: #{ex.inspect}")
+          logger.error(ex.backtrace.join("\n"))
         end
       end
     end

data/lib/redis_failover/runner.rb

@@ -8,22 +8,20 @@ module RedisFailover
     #   Node Manager is gracefully stopped
     def self.run(options)
       options = CLI.parse(options)
-      @node_manager = NodeManager.new(options)
-      trap_signals
-      @node_manager_thread = Thread.new { @node_manager.start }
-      @node_manager_thread.join
+      node_manager = NodeManager.new(options)
+      trap_signals(node_manager)
+      node_manager.start
     end
 
     # Traps shutdown signals.
-    def self.trap_signals
+    # @param [NodeManager] node_manager the node manager
+    def self.trap_signals(node_manager)
       [:INT, :TERM].each do |signal|
         trap(signal) do
-          Util.logger.info('Shutting down ...')
-          @node_manager.shutdown
-          @node_manager_thread.join
-          exit(0)
+          node_manager.shutdown
         end
       end
     end
+    private_class_method :trap_signals
   end
 end

data/lib/redis_failover/version.rb

@@ -1,3 +1,3 @@
 module RedisFailover
-  VERSION = '0.9.1'
+  VERSION = '0.9.7'
 end

data/spec/client_spec.rb

@@ -38,6 +38,12 @@ module RedisFailover
       end
     end
 
+    describe '#client' do
+      it 'should return itself as a delegate' do
+        client.client.should == client
+      end
+    end
+
     describe '#dispatch' do
       it 'routes write operations to master' do
         called = false

data/spec/node_manager_spec.rb

@@ -108,5 +108,29 @@ module RedisFailover
         end
       end
     end
+
+    describe '#guess_master' do
+      let(:node1) { Node.new(:host => 'node1').extend(RedisStubSupport) }
+      let(:node2) { Node.new(:host => 'node2').extend(RedisStubSupport) }
+      let(:node3) { Node.new(:host => 'node3').extend(RedisStubSupport) }
+
+      it 'raises error when no master is found' do
+        node1.make_slave!(node3)
+        node2.make_slave!(node3)
+        expect { manager.guess_master([node1, node2]) }.to raise_error(NoMasterError)
+      end
+
+      it 'raises error when multiple masters found' do
+        node1.make_master!
+        node2.make_master!
+        expect { manager.guess_master([node1, node2]) }.to raise_error(MultipleMastersError)
+      end
+
+      it 'raises error when a node can not be reached' do
+        node1.make_master!
+        node2.redis.make_unavailable!
+        expect { manager.guess_master([node1, node2]) }.to raise_error(NodeUnavailableError)
+      end
+    end
   end
 end

data/spec/support/node_manager_stub.rb

@@ -1,11 +1,12 @@
 module RedisFailover
   class NodeManagerStub < NodeManager
     attr_accessor :master
-    public :current_nodes
+    # HACK - this will go away once we refactor the tests to use a real ZK/Redis server.
+    public :current_nodes, :guess_master
 
     def discover_nodes
       # only discover nodes once in testing
-      return if @nodes_discovered
+      return true if @nodes_discovered
 
       master = Node.new(:host => 'master')
       slave = Node.new(:host => 'slave')

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: nogara-redis_failover
 version: !ruby/object:Gem::Version
-  version: 0.9.1
+  version: 0.9.7
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-09-21 00:00:00.000000000 Z
+date: 2012-09-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis