redis_failover 0.9.2 → 0.9.3

data/Changes.md

@@ -1,3 +1,7 @@
+0.9.3
+-----------
+- Add lock assert for Node Manager.
+
 0.9.2
 -----------
 - Improved exception handling in NodeWatcher.

data/lib/redis_failover/node_manager.rb

@@ -11,6 +11,12 @@ module RedisFailover
 
     # Number of seconds to wait before retrying bootstrap process.
     TIMEOUT = 5
+    # ZK Errors that the Node Manager cares about.
+    ZK_ERRORS = [
+      ZK::Exceptions::LockAssertionFailedError,
+      ZK::Exceptions::InterruptedSession,
+      ZKDisconnectedError
+    ].freeze
 
     # Creates a new instance.
     #
@@ -26,6 +32,7 @@ module RedisFailover
       @znode = @options[:znode_path] || Util::DEFAULT_ZNODE_PATH
       @manual_znode = ManualFailover::ZNODE_PATH
       @mutex = Mutex.new
+      @shutdown = false
 
       # Name for the znode that handles exclusive locking between multiple
       # Node Manager processes. Whoever holds the lock will be considered
@@ -40,11 +47,12 @@ module RedisFailover
     #
     # @note This method does not return until the manager terminates.
     def start
+      return if @shutdown
       @queue = Queue.new
       @leader = false
       setup_zk
       logger.info('Waiting to become master Node Manager ...')
-      @zk.with_lock(@lock_path) do
+      with_lock do
         @leader = true
         logger.info('Acquired master Node Manager lock')
         discover_nodes
@@ -52,9 +60,8 @@ module RedisFailover
         spawn_watchers
         handle_state_reports
       end
-    rescue ZK::Exceptions::InterruptedSession, ZKDisconnectedError => ex
+    rescue *ZK_ERRORS => ex
       logger.error("ZK error while attempting to manage nodes: #{ex.inspect}")
-      logger.error(ex.backtrace.join("\n"))
       shutdown
       sleep(TIMEOUT)
       retry
@@ -71,10 +78,12 @@ module RedisFailover
 
     # Performs a graceful shutdown of the manager.
     def shutdown
+      @shutdown = true
       @queue.clear
       @queue << nil
       @watchers.each(&:shutdown) if @watchers
       @zk.close! if @zk
+      @zk_lock = nil
     end
 
     private
@@ -106,7 +115,10 @@ module RedisFailover
 
     # Handles periodic state reports from {RedisFailover::NodeWatcher} instances.
     def handle_state_reports
-      while state_report = @queue.pop
+      while running? && (state_report = @queue.pop)
+        # Ensure that we still have the master lock.
+        @zk_lock.assert!
+
         begin
           node, state = state_report
           case state
@@ -120,7 +132,7 @@ module RedisFailover
 
           # flush current state
           write_state
-        rescue ZK::Exceptions::InterruptedSession, ZKDisconnectedError
+        rescue *ZK_ERRORS
           # fail hard if this is a ZK connection-related error
           raise
         rescue => ex
@@ -155,7 +167,7 @@ module RedisFailover
       reconcile(node)
 
       # no-op if we already know about this node
-      return if @master == node || @slaves.include?(node)
+      return if @master == node || (@master && @slaves.include?(node))
       logger.info("Handling available node: #{node}")
 
       if @master
@@ -343,6 +355,15 @@ module RedisFailover
       @zk.set(@znode, encode(current_nodes))
     end
 
+    # Executes a block wrapped in a ZK exclusive lock.
+    def with_lock
+      @zk_lock = @zk.locker(@lock_path)
+      @zk_lock.lock(true)
+      yield
+    ensure
+      @zk_lock.unlock! if @zk_lock
+    end
+
     # Schedules a manual failover to a redis node.
     def schedule_manual_failover
       return unless @leader
@@ -364,5 +385,10 @@ module RedisFailover
         logger.error('Failed to perform manual failover, no candidate found.')
       end
     end
+
+    # @return [Boolean] true if still running, false otherwise
+    def running?
+      !@shutdown
+    end
   end
 end

data/lib/redis_failover/version.rb

@@ -1,3 +1,3 @@
 module RedisFailover
-  VERSION = '0.9.2'
+  VERSION = '0.9.3'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: redis_failover
 version: !ruby/object:Gem::Version
-  version: 0.9.2
+  version: 0.9.3
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-08-27 00:00:00.000000000 Z
+date: 2012-08-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
@@ -189,7 +189,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -3297653838576069671
+      hash: -1811569102428626424
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -198,7 +198,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -3297653838576069671
+      hash: -1811569102428626424
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.23