redis_failover 0.5.2 → 0.5.3

data/Changes.md

@@ -1,3 +1,8 @@
+0.5.3
+-----------
+- Handle more ZK exceptions as candidates for reconnecting the client on error.
+- Add safety check to actively purge redis clients if a RedisFailover::Client hasn't heard from the Node Manager in a certain time window.
+
 0.5.2
 -----------
 - Always try to create path before setting current state in Node Manager.

data/README.md

@@ -58,7 +58,7 @@ following options:
     Usage: redis_node_manager [OPTIONS]
         -p, --password password          Redis password (optional)
         -n, --nodes redis nodes          Comma-separated redis host:port pairs (required)
-        -z zookeeper servers,            Comma-separated zookeeper host:port pairs (required)
+        -z zookeeper servers,            Comma-separated ZooKeeper host:port pairs (required)
             --zkservers
             --znode-path path            Znode path override for storing redis server list (optional)
             --max-failures count         Max failures before manager marks node unavailable (default 3)
@@ -85,7 +85,7 @@ a drop-in replacement for your existing pure redis client usage.
 
 The full set of options that can be passed to RedisFailover::Client are:
 
-     :zkservers     - comma-separated zookeeper host:port pairs (required)
+     :zkservers     - comma-separated ZooKeeper host:port pairs (required)
      :znode_path    - the Znode path override for redis server list (optional)
      :password      - password for redis nodes (optional)
      :namespace     - namespace for redis nodes (optional)
@@ -114,6 +114,7 @@ The full set of options that can be passed to RedisFailover::Client are:
 - To learn more about Redis master/slave replication, see the [Redis documentation](http://redis.io/topics/replication).
 - To learn more about ZooKeeper, see the official [ZooKeeper](http://zookeeper.apache.org/) site.
 - See the [Quick ZooKeeper Guide](https://github.com/ryanlecompte/redis_failover/wiki/Quick-ZooKeeper-Guide) for a quick guide to getting ZooKeeper up and running with redis_failover.
+- To learn more about how ZooKeeper handles network partitions, see [ZooKeeper Failure Scenarios](http://wiki.apache.org/hadoop/ZooKeeper/FailureScenarios)
 
 ## License
 
@@ -125,6 +126,10 @@ Ryan LeCompte
 
 [@ryanlecompte](https://twitter.com/ryanlecompte)
 
+## Acknowledgements
+
+Special thanks to [Eric Lindvall](https://github.com/eric) and [Jonathan Simms](https://github.com/slyphon) for their invaluable ZooKeeper advice and guidance!
+
 ## Contributing
 
 1. Fork it

data/lib/redis_failover/cli.rb

@@ -20,8 +20,8 @@ module RedisFailover
           end
         end
 
-        opts.on('-z', '--zkservers zookeeper servers',
-          'Comma-separated zookeeper host:port pairs (required)') do |servers|
+        opts.on('-z', '--zkservers ZooKeeper servers',
+          'Comma-separated ZooKeeper host:port pairs (required)') do |servers|
           options[:zkservers] = servers
         end
 

data/lib/redis_failover/client.rb

@@ -5,6 +5,7 @@ module RedisFailover
   class Client
     include Util
 
+    ZNODE_UPDATE_TIMEOUT = 9
     RETRY_WAIT_TIME = 3
     REDIS_READ_OPS = Set[
       :echo,
@@ -65,7 +66,7 @@ module RedisFailover
     #
     # Options:
     #
-    #   :zkservers - comma-separated zookeeper host:port pairs (required)
+    #   :zkservers - comma-separated ZooKeeper host:port pairs (required)
     #   :znode_path - the Znode path override for redis server list (optional)
     #   :password - password for redis nodes (optional)
     #   :namespace - namespace for redis nodes (optional)
@@ -109,6 +110,7 @@ module RedisFailover
 
     def setup_zookeeper_client
       @zkclient = ZkClient.new(@zkservers)
+      update_znode_timestamp
 
       # when session expires, purge client list
       @zkclient.on_session_expiration do
@@ -128,10 +130,12 @@ module RedisFailover
       # register a watcher for future changes
       @zkclient.watcher.register(@znode) do |event|
         if event.node_created? || event.node_changed?
+          update_znode_timestamp
           build_clients
         elsif event.node_deleted?
-          @zkclient.stat(@znode, :watch => true)
+          update_znode_timestamp
           purge_clients
+          @zkclient.stat(@znode, :watch => true)
         else
           logger.error("Unknown ZK node event: #{event.inspect}")
         end
@@ -143,9 +147,13 @@ module RedisFailover
     end
 
     def dispatch(method, *args, &block)
+      unless recently_heard_from_node_manager?
+        purge_clients
+        raise MissingNodeManagerError.new(ZNODE_UPDATE_TIMEOUT)
+      end
+
       verify_supported!(method)
       tries = 0
-
       begin
         if REDIS_READ_OPS.include?(method)
           # send read operations to a slave
@@ -155,7 +163,7 @@ module RedisFailover
           master.send(method, *args, &block)
         end
       rescue *CONNECTIVITY_ERRORS => ex
-        logger.error("Error while handling operation `#{method}` - #{ex.message}")
+        logger.error("Error while handling operation `#{method}` - #{ex.inspect}")
         logger.error(ex.backtrace.join("\n"))
 
         if tries < @max_retries
@@ -202,7 +210,7 @@ module RedisFailover
           @slaves = new_slaves
         rescue StandardError, *CONNECTIVITY_ERRORS => ex
           purge_clients
-          logger.error("Failed to fetch nodes from #{@zkservers} - #{ex.message}")
+          logger.error("Failed to fetch nodes from #{@zkservers} - #{ex.inspect}")
           logger.error(ex.backtrace.join("\n"))
 
           if tries < @max_retries
@@ -292,5 +300,14 @@ module RedisFailover
         @slaves = []
       end
     end
+
+    def update_znode_timestamp
+      @last_znode_timestamp = Time.now
+    end
+
+    def recently_heard_from_node_manager?
+      return false unless @last_znode_timestamp
+      Time.now - @last_znode_timestamp <= ZNODE_UPDATE_TIMEOUT
+    end
   end
 end

data/lib/redis_failover/errors.rb

@@ -40,4 +40,10 @@ module RedisFailover
       super("Operation `#{operation}` is currently unsupported")
     end
   end
+
+  class MissingNodeManagerError < Error
+    def initialize(timeout)
+      super("Failed to hear from Node Manager within #{timeout} seconds")
+    end
+  end
 end

data/lib/redis_failover/node_manager.rb

@@ -23,6 +23,7 @@ module RedisFailover
     end
 
     def shutdown
+      @queue << nil
       @watchers.each(&:shutdown)
     end
 
@@ -42,7 +43,7 @@ module RedisFailover
           # flush current state
           write_state
         rescue StandardError, *CONNECTIVITY_ERRORS => ex
-          logger.error("Error while handling #{state_change.inspect}: #{ex.message}")
+          logger.error("Error while handling #{state_change.inspect}: #{ex.inspect}")
           logger.error(ex.backtrace.join("\n"))
         end
       end
@@ -188,14 +189,14 @@ module RedisFailover
 
     def delete_path
       @zkclient.delete(@znode)
-      logger.info("Deleted zookeeper node #{@znode}")
+      logger.info("Deleted ZooKeeper node #{@znode}")
     rescue ZK::Exceptions::NoNode => ex
-      logger.info("Tried to delete missing znode: #{ex.message}")
+      logger.info("Tried to delete missing znode: #{ex.inspect}")
     end
 
     def create_path
       @zkclient.create(@znode, encode(current_nodes), :ephemeral => true)
-      logger.info("Created zookeeper node #{@znode}")
+      logger.info("Created ZooKeeper node #{@znode}")
     rescue ZK::Exceptions::NodeExists
       # best effort
     end

data/lib/redis_failover/node_watcher.rb

@@ -3,7 +3,7 @@ module RedisFailover
   class NodeWatcher
     include Util
 
-    WATCHER_SLEEP_TIME = 3
+    WATCHER_SLEEP_TIME = 2
 
     def initialize(manager, node, max_failures)
       @manager = manager

data/lib/redis_failover/runner.rb

@@ -6,7 +6,7 @@ module RedisFailover
       @node_manager = NodeManager.new(options)
       trap_signals
       node_manager_thread = Thread.new { @node_manager.start }
-      Util.logger.info("Redis Node Manager successfully started.")
+      Util.logger.info("Redis Node Manager v#{VERSION} successfully started (#{RUBY_DESCRIPTION})")
       node_manager_thread.join
     end
 

data/lib/redis_failover/version.rb

@@ -1,3 +1,3 @@
 module RedisFailover
-  VERSION = "0.5.2"
+  VERSION = "0.5.3"
 end

data/lib/redis_failover/zk_client.rb

@@ -5,6 +5,16 @@ module RedisFailover
     include Util
 
     MAX_RECONNECTS = 3
+    RECONNECTABLE_ERRORS = [
+      ZookeeperExceptions::ZookeeperException::SessionExpired,
+      ZookeeperExceptions::ZookeeperException::SystemError,
+      ZookeeperExceptions::ZookeeperException::ConnectionLoss,
+      ZookeeperExceptions::ZookeeperException::OperationTimeOut,
+      ZookeeperExceptions::ZookeeperException::AuthFailed,
+      ZookeeperExceptions::ZookeeperException::SessionMoved,
+      ZookeeperExceptions::ZookeeperException::ConnectionClosed,
+      ZookeeperExceptions::ZookeeperException::NotConnected
+    ].freeze
 
     def initialize(servers)
       @servers = servers
@@ -56,8 +66,9 @@ module RedisFailover
       tries = 0
       begin
         yield
-      rescue ZookeeperExceptions::ZookeeperException::SessionExpired
-        logger.info("Zookeeper client session expired, rebuilding client.")
+      rescue *RECONNECTABLE_ERRORS => ex
+        logger.error("ZooKeeper client connection error - rebuilding client: #{ex.inspect}")
+        logger.error(ex.backtrace.join("\n"))
         if tries < MAX_RECONNECTS
           tries += 1
           @on_session_expiration.call if @on_session_expiration
@@ -77,7 +88,7 @@ module RedisFailover
         else
           @client = ZK.new(@servers)
         end
-        logger.info("Communicating with zookeeper servers #{@servers}")
+        logger.info("Communicating with ZooKeeper servers #{@servers}")
       end
     end
   end

data/spec/client_spec.rb

@@ -19,7 +19,9 @@ module RedisFailover
       }
     end
 
-    def setup_zookeeper_client; end
+    def setup_zookeeper_client
+      update_znode_timestamp
+    end
   end
 
   describe Client do
@@ -37,14 +39,22 @@ module RedisFailover
 
     describe '#dispatch' do
       it 'routes write operations to master' do
-        client.current_master.should_receive(:del)
+        called = false
+        client.current_master.define_singleton_method(:del) do |*args|
+          called = true
+        end
         client.del('foo')
+        called.should be_true
       end
 
       it 'routes read operations to a slave' do
+        called = false
         client.current_slaves.first.change_role_to('slave')
-        client.current_slaves.first.should_receive(:get)
+        client.current_slaves.first.define_singleton_method(:get) do |*args|
+          called = true
+        end
         client.get('foo')
+        called.should be_true
       end
 
       it 'reconnects when node is unavailable' do
@@ -78,6 +88,11 @@ module RedisFailover
       it 'raises error for unsupported operations' do
         expect { client.select }.to raise_error(UnsupportedOperationError)
       end
+
+      it 'raises error when no communication from Node Manager within certain time window' do
+        client.instance_variable_set(:@last_znode_timestamp, Time.at(0))
+        expect { client.del('foo') }.to raise_error(MissingNodeManagerError)
+      end
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: redis_failover
 version: !ruby/object:Gem::Version
-  version: 0.5.2
+  version: 0.5.3
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-04-17 00:00:00.000000000 Z
+date: 2012-04-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
-  requirement: &70127944589860 !ruby/object:Gem::Requirement
+  requirement: &70306532639260 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70127944589860
+  version_requirements: *70306532639260
 - !ruby/object:Gem::Dependency
   name: redis-namespace
-  requirement: &70127944589440 !ruby/object:Gem::Requirement
+  requirement: &70306532638840 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70127944589440
+  version_requirements: *70306532638840
 - !ruby/object:Gem::Dependency
   name: multi_json
-  requirement: &70127944589000 !ruby/object:Gem::Requirement
+  requirement: &70306532638420 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70127944589000
+  version_requirements: *70306532638420
 - !ruby/object:Gem::Dependency
   name: zk
-  requirement: &70127944588580 !ruby/object:Gem::Requirement
+  requirement: &70306532638000 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70127944588580
+  version_requirements: *70306532638000
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &70127944588160 !ruby/object:Gem::Requirement
+  requirement: &70306532637580 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +65,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70127944588160
+  version_requirements: *70306532637580
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70127944587740 !ruby/object:Gem::Requirement
+  requirement: &70306532637160 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,7 +76,7 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70127944587740
+  version_requirements: *70306532637160
 description: Redis Failover is a ZooKeeper-based automatic master/slave failover solution
   for Ruby
 email:
@@ -129,7 +129,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -3828155279649118741
+      hash: -4476900958412029501
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -138,7 +138,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -3828155279649118741
+      hash: -4476900958412029501
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.16