redis_failover 0.6.0 → 0.7.0

data/Changes.md

@@ -1,3 +1,9 @@
+0.7.0
+-----------
+- When new master promotion occurs, make existing slaves point to new candidate before promoting new master.
+- Add support for specifying command-line options in a config.yml file for Node Manager.
+- Upgrade to 0.9 version of ZK client and cleanup ZK connection error handling.
+
 0.6.0
 -----------
 - Add support for running multiple Node Manager processes for added redundancy (#4)

data/README.md

@@ -1,4 +1,4 @@
-# Automatic Redis Failover Client / Node Manager
+# Automatic Redis Failover
 
 [![Build Status](https://secure.travis-ci.org/ryanlecompte/redis_failover.png?branch=master)](http://travis-ci.org/ryanlecompte/redis_failover)
 
@@ -61,15 +61,36 @@ following options:
     Specific options:
         -n, --nodes NODES                Comma-separated redis host:port pairs
         -z, --zkservers SERVERS          Comma-separated ZooKeeper host:port pairs
-        -p, --password [PASSWORD]        Redis password
-            --znode-path [PATH]          Znode path override for storing redis server list
-            --max-failures [COUNT]       Max failures before manager marks node unavailable
+        -p, --password PASSWORD          Redis password
+            --znode-path PATH            Znode path override for storing redis server list
+            --max-failures COUNT         Max failures before manager marks node unavailable
+        -C, --config PATH                Path to YAML configuration file
         -h, --help                       Display all options
 
 To start the daemon for a simple master/slave configuration, use the following:
 
     redis_node_manager -n localhost:6379,localhost:6380 -z localhost:2181,localhost:2182,localhost:2183
 
+The configuration parameters can also be specified in a config.yml file. An example configuration
+would look like the following:
+
+    ---
+    :max_failures: 2
+    :nodes:
+      - localhost:6379
+      - localhost:1111
+      - localhost:2222
+      - localhost:3333
+    :zkservers:
+      - localhost:2181
+      - localhost:2182
+      - localhost:2183
+    :password: foobar
+
+You would then simpy start the Node Manager via the following:
+
+    redis_node_manager -C config.yml
+
 The Node Manager will automatically discover the master/slaves upon startup. Note that it is
 a good idea to run more than one instance of the Node Manager daemon in your environment. At
 any moment, a single Node Manager process will be designated to monitor the redis servers. If

data/examples/config.yml

@@ -0,0 +1,14 @@
+# Sample configuration file for Node Manager.
+#   redis_node_manager -C config.yml
+---
+:max_failures: 2
+:nodes:
+  - localhost:6379
+  - localhost:1111
+  - localhost:2222
+  - localhost:3333
+:zkservers:
+  - localhost:2181
+  - localhost:2182
+  - localhost:2183
+:password: foobar

data/lib/redis_failover/cli.rb

@@ -10,31 +10,32 @@ module RedisFailover
 
         opts.on('-n', '--nodes NODES',
           'Comma-separated redis host:port pairs') do |nodes|
-          # turns 'host1:port,host2:port' => [{:host => host, :port => port}, ...]
-          options[:nodes] = nodes.split(',').map do |node|
-            Hash[[:host, :port].zip(node.strip.split(':'))]
-          end
+         options[:nodes] = nodes
         end
 
         opts.on('-z', '--zkservers SERVERS',
           'Comma-separated ZooKeeper host:port pairs') do |servers|
-          options[:zkservers] = servers.strip
+          options[:zkservers] = servers
         end
 
-        opts.on('-p', '--password [PASSWORD]', 'Redis password') do |password|
-          options[:password] = password.strip
+        opts.on('-p', '--password PASSWORD', 'Redis password') do |password|
+          options[:password] = password
         end
 
-        opts.on('--znode-path [PATH]',
+        opts.on('--znode-path PATH',
           'Znode path override for storing redis server list') do |path|
           options[:znode_path] = path
         end
 
-        opts.on('--max-failures [COUNT]',
+        opts.on('--max-failures COUNT',
           'Max failures before manager marks node unavailable') do |max|
           options[:max_failures] = Integer(max)
         end
 
+        opts.on '-C', '--config PATH', "Path to YAML configuration file" do |file|
+          options[:config_file] = file
+        end
+
         opts.on('-h', '--help', 'Display all options') do
           puts opts
           exit
@@ -42,17 +43,16 @@ module RedisFailover
       end
 
       parser.parse(source)
+      if config_file = options[:config_file]
+        options = from_file(config_file)
+      end
+
       if required_options_missing?(options)
         puts parser
         exit
       end
 
-      # assume password is same for all redis nodes
-      if password = options[:password]
-        options[:nodes].each { |opts| opts.update(:password => password) }
-      end
-
-      options
+      prepare(options)
     end
 
     def self.required_options_missing?(options)
@@ -60,5 +60,31 @@ module RedisFailover
       return true unless options.values_at(:nodes, :zkservers).all?
       false
     end
+
+    def self.from_file(file)
+      unless File.exists?(file)
+        raise ArgumentError, "File #{file} can't be found"
+      end
+      options = YAML.load_file(file)
+      options[:nodes] = options[:nodes].join(',')
+      options[:zkservers] = options[:zkservers].join(',')
+
+      options
+    end
+
+    def self.prepare(options)
+      options.each_value { |v| v.strip! if v.respond_to?(:strip!) }
+      # turns 'host1:port,host2:port' => [{:host => host, :port => port}, ...]
+      options[:nodes] = options[:nodes].split(',').map do |node|
+        Hash[[:host, :port].zip(node.split(':'))]
+      end
+
+      # assume password is same for all redis nodes
+      if password = options[:password]
+        options[:nodes].each { |opts| opts.update(:password => password) }
+      end
+
+      options
+    end
   end
 end

data/lib/redis_failover/client.rb

@@ -105,11 +105,12 @@ module RedisFailover
       @password = options[:password]
       @db = options[:db]
       @retry = options[:retry_failure] || true
-      @max_retries = @retry ? options.fetch(:max_retries, 3) : 1
+      @max_retries = @retry ? options.fetch(:max_retries, 3) : 0
       @master = nil
       @slaves = []
+      @queue = Queue.new
       @lock = Monitor.new
-      setup_zookeeper_client
+      start_zk
       build_clients
     end
 
@@ -133,40 +134,66 @@ module RedisFailover
 
     private
 
-    def setup_zookeeper_client
-      @zkclient = ZkClient.new(@zkservers) do |client|
-        # when session expires, purge client list
-        client.on_session_expiration do
-          purge_clients
-        end
+    def zk
+      @lock.synchronize { @zk }
+    end
 
-        # when we are disconnected, purge client list
-        client.event_handler.register_state_handler(:connecting) do
-          purge_clients
+    def start_zk
+      @delivery_thread ||= Thread.new do
+        while event = @queue.pop
+          if event.is_a?(Proc)
+            event.call
+          else
+            handle_zk_event(event)
+          end
         end
+      end
+      reconnect_zk
+    end
 
-        # when session is recovered, watch again
-        client.on_session_recovered do
-          client.stat(@znode, :watch => true)
+    def handle_session_established
+      @lock.synchronize do
+        @zk.watcher.register(@znode) do |event|
+          @queue << event
         end
-
-        # register a watcher for future changes
-        client.watcher.register(@znode) do |event|
-          if event.node_created? || event.node_changed?
-            update_znode_timestamp
-            build_clients
-          elsif event.node_deleted?
-            update_znode_timestamp
-            purge_clients
-            client.stat(@znode, :watch => true)
-          else
-            logger.error("Unknown ZK node event: #{event.inspect}")
-          end
+        @zk.on_expired_session do
+          @queue << proc { reconnect_zk }
+        end
+        @zk.event_handler.register_state_handler(:connecting) do
+          @queue << proc { handle_lost_connection }
+        end
+        @zk.on_connected do
+          @zk.stat(@znode, :watch => true)
         end
       end
+    end
+
+    def handle_zk_event(event)
+      update_znode_timestamp
+      if event.node_created? || event.node_changed?
+        build_clients
+      elsif event.node_deleted?
+        purge_clients
+        zk.stat(@znode, :watch => true)
+      else
+        logger.error("Unknown ZK node event: #{event.inspect}")
+      end
+    end
+
+    def reconnect_zk
+      handle_lost_connection
+      @lock.synchronize do
+        @zk.close! if @zk
+        @zk = ZK.new(@zkservers)
+      end
+      handle_session_established
       update_znode_timestamp
     end
 
+    def handle_lost_connection
+      purge_clients
+    end
+
     def redis_operation?(method)
       Redis.public_instance_methods(false).include?(method)
     end
@@ -197,7 +224,6 @@ module RedisFailover
           sleep(RETRY_WAIT_TIME)
           retry
         end
-
         raise
       end
     end
@@ -221,7 +247,7 @@ module RedisFailover
 
     def build_clients
       @lock.synchronize do
-        tries = 0
+        retried = false
 
         begin
           nodes = fetch_nodes
@@ -233,24 +259,23 @@ module RedisFailover
           new_slaves = new_clients_for(*nodes[:slaves])
           @master = new_master
           @slaves = new_slaves
-        rescue StandardError, *CONNECTIVITY_ERRORS => ex
-          purge_clients
-          logger.error("Failed to fetch nodes from #{@zkservers} - #{ex.inspect}")
+        rescue ZK::Exceptions::InterruptedSession => ex
+          logger.error("ZK error while attempting to build clients: #{ex.inspect}")
           logger.error(ex.backtrace.join("\n"))
 
-          if tries < @max_retries
-            tries += 1
-            sleep(RETRY_WAIT_TIME)
+          # when ZK is disconnected, retry once
+          unless retried
+            reconnect_zk
+            retried = true
             retry
           end
-
           raise
         end
       end
     end
 
     def fetch_nodes
-      data = @zkclient.get(@znode, :watch => true).first
+      data = zk.get(@znode, :watch => true).first
       nodes = symbolize_keys(decode(data))
       logger.debug("Fetched nodes: #{nodes}")
 

data/lib/redis_failover/node.rb

@@ -53,11 +53,11 @@ module RedisFailover
       end
     end
 
-    def make_slave!(master)
+    def make_slave!(node)
       perform_operation do |redis|
-        unless slave_of?(master)
-          redis.slaveof(master.host, master.port)
-          logger.info("#{self} is now a slave of master #{master}")
+        unless slave_of?(node)
+          redis.slaveof(node.host, node.port)
+          logger.info("#{self} is now a slave of #{node}")
           wakeup
         end
       end

data/lib/redis_failover/node_manager.rb

@@ -18,29 +18,29 @@ module RedisFailover
     LOCK_PATH = 'master_node_manager'
 
     # Number of seconds to wait before retrying bootstrap process.
-    TIMEOUT = 3
+    TIMEOUT = 5
 
     def initialize(options)
       logger.info("Redis Node Manager v#{VERSION} starting (#{RUBY_DESCRIPTION})")
       @options = options
       @znode = @options[:znode_path] || Util::DEFAULT_ZNODE_PATH
-      @unavailable = []
-      @queue = Queue.new
     end
 
     def start
-      @zkclient = ZkClient.new(@options[:zkservers])
+      @queue = Queue.new
+      @zk = ZK.new(@options[:zkservers])
       logger.info('Waiting to become master Node Manager ...')
-      @zkclient.with_lock(LOCK_PATH) do
+      @zk.with_lock(LOCK_PATH) do
         logger.info('Acquired master Node Manager lock')
         discover_nodes
         initialize_path
         spawn_watchers
         handle_state_reports
       end
-    rescue *CONNECTIVITY_ERRORS => ex
-      logger.error("Error while attempting to manage nodes: #{ex.inspect}")
+    rescue ZK::Exceptions::InterruptedSession => ex
+      logger.error("ZK error while attempting to manage nodes: #{ex.inspect}")
       logger.error(ex.backtrace.join("\n"))
+      shutdown
       sleep(TIMEOUT)
       retry
     end
@@ -51,7 +51,8 @@ module RedisFailover
 
     def shutdown
       @queue << nil
-      @watchers.each(&:shutdown)
+      @watchers.each(&:shutdown) if @watchers
+      @zk.close! if @zk
     end
 
     private
@@ -69,7 +70,10 @@ module RedisFailover
 
           # flush current state
           write_state
-        rescue StandardError, *CONNECTIVITY_ERRORS => ex
+        rescue ZK::Exceptions::InterruptedSession
+          # fail hard if this is a ZK connection-related error
+          raise
+        rescue => ex
           logger.error("Error handling #{state_report.inspect}: #{ex.inspect}")
           logger.error(ex.backtrace.join("\n"))
         end
@@ -134,15 +138,17 @@ module RedisFailover
         return
       end
 
+      redirect_slaves_to(candidate)
       candidate.make_master!
       @master = candidate
-      redirect_slaves_to_master
+
       create_path
       write_state
       logger.info("Successfully promoted #{candidate} to master.")
     end
 
     def discover_nodes
+      @unavailable = []
       nodes = @options[:nodes].map { |opts| Node.new(opts) }.uniq
       raise NoMasterError unless @master = find_master(nodes)
       @slaves = nodes - [@master]
@@ -150,7 +156,7 @@ module RedisFailover
         " (#{@slaves.map(&:to_s).join(', ')})")
 
       # ensure that slaves are correctly pointing to this master
-      redirect_slaves_to_master
+      redirect_slaves_to(@master)
     end
 
     def spawn_watchers
@@ -170,12 +176,12 @@ module RedisFailover
       end
     end
 
-    def redirect_slaves_to_master
+    def redirect_slaves_to(node)
       @slaves.dup.each do |slave|
         begin
-          slave.make_slave!(@master)
+          slave.make_slave!(node)
         rescue NodeUnavailableError
-          logger.info("Failed to redirect unreachable slave #{slave} to master #{@master}")
+          logger.info("Failed to redirect unreachable slave #{slave} to #{node}")
           force_unavailable_slave(slave)
         end
       end
@@ -216,14 +222,14 @@ module RedisFailover
     end
 
     def delete_path
-      @zkclient.delete(@znode)
+      @zk.delete(@znode)
       logger.info("Deleted ZooKeeper node #{@znode}")
     rescue ZK::Exceptions::NoNode => ex
       logger.info("Tried to delete missing znode: #{ex.inspect}")
     end
 
     def create_path
-      @zkclient.create(@znode, encode(current_nodes), :ephemeral => true)
+      @zk.create(@znode, encode(current_nodes), :ephemeral => true)
       logger.info("Created ZooKeeper node #{@znode}")
     rescue ZK::Exceptions::NodeExists
       # best effort
@@ -236,7 +242,7 @@ module RedisFailover
 
     def write_state
       create_path
-      @zkclient.set(@znode, encode(current_nodes))
+      @zk.set(@znode, encode(current_nodes))
     end
   end
 end

data/lib/redis_failover/util.rb

@@ -14,9 +14,9 @@ module RedisFailover
     # Full set of errors related to connectivity.
     CONNECTIVITY_ERRORS = [
       RedisFailover::Error,
-      ZK::Exceptions::KeeperException,
-      ZookeeperExceptions::ZookeeperException,
-      REDIS_ERRORS].flatten.freeze
+      ZK::Exceptions::InterruptedSession,
+      REDIS_ERRORS
+    ].flatten.freeze
 
     def symbolize_keys(hash)
       Hash[hash.map { |k, v| [k.to_sym, v] }]

data/lib/redis_failover/version.rb

@@ -1,3 +1,3 @@
 module RedisFailover
-  VERSION = "0.6.0"
+  VERSION = "0.7.0"
 end

data/lib/redis_failover.rb

@@ -1,5 +1,6 @@
 require 'zk'
 require 'set'
+require 'yaml'
 require 'redis'
 require 'thread'
 require 'logger'
@@ -15,6 +16,5 @@ require 'redis_failover/errors'
 require 'redis_failover/client'
 require 'redis_failover/runner'
 require 'redis_failover/version'
-require 'redis_failover/zk_client'
 require 'redis_failover/node_manager'
 require 'redis_failover/node_watcher'

data/redis_failover.gemspec

@@ -18,7 +18,7 @@ Gem::Specification.new do |gem|
   gem.add_dependency('redis')
   gem.add_dependency('redis-namespace')
   gem.add_dependency('multi_json', '>= 1.0', '< 1.3')
-  gem.add_dependency('zk', '~> 0.8.8')
+  gem.add_dependency('zk', '~> 0.9')
 
   gem.add_development_dependency('rake')
   gem.add_development_dependency('rspec')

data/spec/spec_helper.rb

@@ -12,8 +12,9 @@ end
 
 module RedisFailover
   Util.logger = NullObject.new
-  def ZkClient.new(*args); NullObject.new; end
 end
 
+def ZK.new(*args); NullObject.new; end
+
 RSpec.configure do |config|
 end

data/spec/support/node_manager_stub.rb

@@ -22,6 +22,7 @@ module RedisFailover
       [master, slave].each { |node| node.extend(RedisStubSupport) }
       master.make_master!
       slave.make_slave!(master)
+      @unavailable = []
       @master = master
       @slaves = [slave]
       @nodes_discovered = true
@@ -33,6 +34,7 @@ module RedisFailover
 
     def start_processing
       @thread = Thread.new { start }
+      sleep(1.5)
     end
 
     def stop_processing

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: redis_failover
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.7.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-04-21 00:00:00.000000000 Z
+date: 2012-04-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
-  requirement: !ruby/object:Gem::Requirement
+  requirement: &70345784320360 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,15 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
+  version_requirements: *70345784320360
 - !ruby/object:Gem::Dependency
   name: redis-namespace
-  requirement: !ruby/object:Gem::Requirement
+  requirement: &70345784319940 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -37,15 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
+  version_requirements: *70345784319940
 - !ruby/object:Gem::Dependency
   name: multi_json
-  requirement: !ruby/object:Gem::Requirement
+  requirement: &70345784319420 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -56,34 +46,21 @@ dependencies:
         version: '1.3'
   type: :runtime
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '1.0'
-    - - <
-      - !ruby/object:Gem::Version
-        version: '1.3'
+  version_requirements: *70345784319420
 - !ruby/object:Gem::Dependency
   name: zk
-  requirement: !ruby/object:Gem::Requirement
+  requirement: &70345784318660 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.8.8
+        version: '0.9'
   type: :runtime
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: 0.8.8
+  version_requirements: *70345784318660
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: !ruby/object:Gem::Requirement
+  requirement: &70345784318280 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -91,15 +68,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
+  version_requirements: *70345784318280
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: !ruby/object:Gem::Requirement
+  requirement: &70345784317820 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -107,12 +79,7 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
+  version_requirements: *70345784317820
 description: Redis Failover is a ZooKeeper-based automatic master/slave failover solution
   for Ruby
 email:
@@ -130,6 +97,7 @@ files:
 - README.md
 - Rakefile
 - bin/redis_node_manager
+- examples/config.yml
 - lib/redis_failover.rb
 - lib/redis_failover/cli.rb
 - lib/redis_failover/client.rb
@@ -140,7 +108,6 @@ files:
 - lib/redis_failover/runner.rb
 - lib/redis_failover/util.rb
 - lib/redis_failover/version.rb
-- lib/redis_failover/zk_client.rb
 - redis_failover.gemspec
 - spec/cli_spec.rb
 - spec/client_spec.rb
@@ -165,7 +132,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 2836022562258381397
+      hash: -4131531819785678189
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -174,10 +141,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 2836022562258381397
+      hash: -4131531819785678189
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.23
+rubygems_version: 1.8.16
 signing_key: 
 specification_version: 3
 summary: Redis Failover is a ZooKeeper-based automatic master/slave failover solution

data/lib/redis_failover/zk_client.rb

@@ -1,93 +0,0 @@
-module RedisFailover
-  # ZkClient is a thin wrapper over the ZK client to gracefully handle reconnects
-  # when a session expires.
-  class ZkClient
-    include Util
-
-    # Time to sleep before retrying a failed operation.
-    TIMEOUT = 2
-
-    # Maximum reconnect attempts.
-    MAX_RECONNECTS = 3
-
-    # Errors that are candidates for rebuilding the underlying ZK client.
-    RECONNECTABLE_ERRORS = [
-      ZookeeperExceptions::ZookeeperException::SessionExpired,
-      ZookeeperExceptions::ZookeeperException::SystemError,
-      ZookeeperExceptions::ZookeeperException::ConnectionLoss,
-      ZookeeperExceptions::ZookeeperException::OperationTimeOut,
-      ZookeeperExceptions::ZookeeperException::AuthFailed,
-      ZookeeperExceptions::ZookeeperException::SessionMoved,
-      ZookeeperExceptions::ZookeeperException::ConnectionClosed,
-      ZookeeperExceptions::ZookeeperException::NotConnected
-    ].freeze
-
-    # ZK methods that are wrapped with reconnect logic.
-    WRAPPED_ZK_METHODS = [
-      :get,
-      :set,
-      :watcher,
-      :event_handler,
-      :stat,
-      :create,
-      :delete,
-      :with_lock].freeze
-
-    def initialize(servers, &setup_block)
-      @servers = servers
-      @setup_block = setup_block
-      @lock = Mutex.new
-      build_client
-    end
-
-    def on_session_expiration(&block)
-      @client.on_expired_session { block.call }
-      @on_session_expiration = block
-    end
-
-    def on_session_recovered(&block)
-      @client.on_connected { block.call }
-      @on_session_recovered = block
-    end
-
-    WRAPPED_ZK_METHODS.each do |zk_method|
-      class_eval(<<-RUBY, __FILE__, __LINE__ + 1)
-        def #{zk_method}(*args, &block)
-          perform_with_reconnect do
-            @client.#{zk_method}(*args, &block)
-          end
-        end
-      RUBY
-    end
-
-    private
-
-    def perform_with_reconnect
-      tries = 0
-      begin
-        yield
-      rescue *RECONNECTABLE_ERRORS => ex
-        logger.error("ZooKeeper connection error, rebuilding client: #{ex.inspect}")
-        logger.error(ex.backtrace.join("\n"))
-        if tries < MAX_RECONNECTS
-          tries += 1
-          @on_session_expiration.call if @on_session_expiration
-          build_client
-          @on_session_recovered.call if @on_session_recovered
-          sleep(TIMEOUT)
-          retry
-        end
-
-        raise
-      end
-    end
-
-    def build_client
-      @lock.synchronize do
-        @client = ZK.new(@servers)
-        @setup_block.call(self) if @setup_block
-        logger.info("Communicating with ZooKeeper servers #{@servers}")
-      end
-    end
-  end
-end