cassandra-driver 1.0.0.beta.3 → 1.0.0.rc.1

data/lib/cassandra/cluster/connector.rb

@@ -43,25 +43,17 @@ module Cassandra
           end
         end
 
-        @logger.info("Connecting ip=#{host.ip}")
-
         f = do_connect(host)
 
         f.on_failure do |error|
-          @logger.warn("Connection failed ip=#{host.ip} error=\"#{error.class.name}: #{error.message}\"")
           connection_error(host, error)
         end
 
         f.on_value do |connection|
           connection.on_closed do |cause|
-            message = "Disconnected ip=#{host.ip}"
-            message << " error=#{cause.message}" if cause
-
-            @logger.info(message)
             disconnected(host, cause)
           end
 
-          @logger.info("Connected ip=#{host.ip}")
           connected(host)
         end
 
@@ -75,22 +67,15 @@ module Cassandra
           return Future.resolved
         end
 
-        @logger.info("Refreshing host status ip=#{host.ip}")
+        @logger.debug("Checking if host #{host.ip} is up")
         f = do_connect(host)
 
         f.on_failure do |error|
-          @logger.info("Refreshed host status ip=#{host.ip}")
-          @logger.warn("Connection failed ip=#{host.ip} error=\"#{error.class.name}: #{error.message}\"")
           connection_error(host, error)
         end
 
         f.on_value do |connection|
-          @logger.info("Refreshed host status ip=#{host.ip}")
           connection.on_closed do |cause|
-            message = "Disconnected ip=#{host.ip}"
-            message << " error=#{cause.message}" if cause
-
-            @logger.info(message)
             disconnected(host, cause)
           end
 
@@ -99,7 +84,21 @@ module Cassandra
             @open_connections[host]  << connection
           end
 
-          @logger.info("Connected ip=#{host.ip}")
+          timer = @reactor.schedule_timer(UNCLAIMED_TIMEOUT)
+          timer.on_value do
+            close = false
+
+            synchronize do
+              open_connections = @open_connections[host]
+              if open_connections
+                close = !open_connections.delete(connection).nil?
+                @open_connections.delete(host) if open_connections.empty?
+              end
+            end
+
+            connection.close if close
+          end
+
           connected(host)
         end
 
@@ -112,22 +111,28 @@ module Cassandra
 
       private
 
-      NO_CONNECTIONS = Ione::Future.resolved([])
+      NO_CONNECTIONS    = Ione::Future.resolved([])
+      UNCLAIMED_TIMEOUT = 5 # close unclaimed connections in five seconds
 
       def do_connect(host)
         create_connector.connect(host.ip.to_s).fallback do |error|
-          if error.is_a?(Errors::QueryError) && error.code == 0x0a
+          case error
+          when Errors::ProtocolError
             synchronize do
               if @options.protocol_version > 1
-                @logger.warn('Could not connect using protocol version %d (will try again with %d): %s' % [@options.protocol_version, @options.protocol_version - 1, error.message])
+                @logger.info("Host #{host.ip} doesn't support protocol version #{@options.protocol_version}, downgrading")
                 @options.protocol_version -= 1
                 do_connect(host)
               else
                 Ione::Future.failed(error)
               end
             end
-          else
+          when Error
             Ione::Future.failed(error)
+          else
+            e = Errors::IOError.new(error.message)
+            e.set_backtrace(error.backtrace)
+            Ione::Future.failed(e)
           end
         end
       end
@@ -135,8 +140,8 @@ module Cassandra
       def create_connector
         authentication_step = @options.protocol_version == 1 ? Cassandra::Client::CredentialsAuthenticationStep.new(@options.credentials) : Cassandra::Client::SaslAuthenticationStep.new(@options.auth_provider)
         protocol_handler_factory = lambda do |connection|
-          raise "no connection given" unless connection
-          Protocol::CqlProtocolHandler.new(connection, @reactor, @options.protocol_version, @options.compressor)
+          raise Errors::ClientError, 'Not connected, reactor stopped' unless connection
+          Protocol::CqlProtocolHandler.new(connection, @reactor, @options.protocol_version, @options.compressor, @options.heartbeat_interval, @options.idle_timeout)
         end
 
         Cassandra::Client::Connector.new([
@@ -216,14 +221,19 @@ module Cassandra
           connections -= 1
 
           if connections == 0
-            notify = !error.nil? && !error.is_a?(Cassandra::Error)
+            notify = !error.nil?
             @connections.delete(host)
           else
             @connections[host] = connections
           end
         end
 
-        @registry.host_down(host.ip) if notify
+        @logger.debug("Host #{host.ip} closed connection (#{error.class.name}: #{error.message})") if error
+
+        if notify
+          @logger.warn("Host #{host.ip} closed all connections")
+          @registry.host_down(host.ip)
+        end
 
         self
       end
@@ -232,10 +242,15 @@ module Cassandra
         notify = false
 
         synchronize do
-          notify = !error.is_a?(Cassandra::Error) && !@connections.has_key?(host)
+          notify = !error.nil? && !@connections.has_key?(host)
         end
 
-        @registry.host_down(host.ip) if notify
+        @logger.debug("Host #{host.ip} refused connection (#{error.class.name}: #{error.message})")
+
+        if notify
+          @logger.warn("Host #{host.ip} refused all connections")
+          @registry.host_down(host.ip)
+        end
 
         self
       end

data/lib/cassandra/cluster/control_connection.rb

@@ -22,7 +22,7 @@ module Cassandra
     class ControlConnection
       include MonitorMixin
 
-      def initialize(logger, io_reactor, cluster_registry, cluster_schema, cluster_metadata, load_balancing_policy, reconnection_policy, connector)
+      def initialize(logger, io_reactor, cluster_registry, cluster_schema, cluster_metadata, load_balancing_policy, reconnection_policy, address_resolution_policy, connector)
         @logger                = logger
         @io_reactor            = io_reactor
         @registry              = cluster_registry
@@ -30,9 +30,15 @@ module Cassandra
         @metadata              = cluster_metadata
         @load_balancing_policy = load_balancing_policy
         @reconnection_policy   = reconnection_policy
+        @address_resolver      = address_resolution_policy
         @connector             = connector
         @refreshing_statuses   = Hash.new(false)
         @status                = :closed
+        @refreshing_schema     = false
+        @refreshing_keyspaces  = Hash.new(false)
+        @refreshing_tables     = Hash.new
+        @refreshing_hosts      = false
+        @refreshing_host       = Hash.new(false)
 
         mon_initialize
       end
@@ -43,12 +49,14 @@ module Cassandra
           @status = :connecting
         end
 
-        @logger.debug('Establishing control connection')
-
-        @io_reactor.start.flat_map do
+        f = @io_reactor.start.flat_map do
           plan = @load_balancing_policy.plan(nil, VOID_STATEMENT, VOID_OPTIONS)
           connect_to_first_available(plan)
         end
+        f.on_failure do |e|
+          close_async
+        end
+        f
       end
 
       def host_found(host)
@@ -56,31 +64,48 @@ module Cassandra
 
       def host_lost(host)
         synchronize do
-          @refreshing_statuses.delete(host)
+          timer = @refreshing_statuses.delete(host)
+          @io_reactor.cancel_timer(timer) if timer
         end
+
+        nil
       end
 
       def host_up(host)
         synchronize do
-          @refreshing_statuses.delete(host)
+          timer = @refreshing_statuses.delete(host)
+          @io_reactor.cancel_timer(timer) if timer
 
-          return connect_async if !@connection && !(@status == :closed || @status == :closed)
+          unless @connection || (@status == :closing || @status == :closed) || @load_balancing_policy.distance(host) == :ignore
+            return connect_to_first_available(@load_balancing_policy.plan(nil, VOID_STATEMENT, VOID_OPTIONS))
+          end
         end
 
         Ione::Future.resolved
       end
 
       def host_down(host)
+        schedule = nil
+        timer    = nil
+
         synchronize do
-          return Ione::Future.resolved if @refreshing_statuses[host]
+          return Ione::Future.resolved if @refreshing_statuses[host] || @load_balancing_policy.distance(host) == :ignore
+
+          schedule = @reconnection_policy.schedule
+          timeout  = schedule.next
 
-          @logger.debug("Starting to continuously refresh status for ip=#{host.ip}")
-          @refreshing_statuses[host] = true
+          @logger.debug("Starting to continuously refresh status of #{host.ip} in #{timeout} seconds")
+
+          @refreshing_statuses[host] = timer = @io_reactor.schedule_timer(timeout)
         end
 
-        refresh_host_status(host).fallback do |e|
-          refresh_host_status_with_retry(host, @reconnection_policy.schedule)
+        timer.on_value do
+          refresh_host_status(host).fallback do |e|
+            refresh_host_status_with_retry(timer, host, schedule)
+          end
         end
+
+        nil
       end
 
       def close_async
@@ -116,19 +141,18 @@ module Cassandra
         f = @io_reactor.schedule_timer(timeout)
         f = f.flat_map do
           if synchronize { @status == :reconnecting }
-            @logger.debug('Reestablishing control connection')
             plan = @load_balancing_policy.plan(nil, VOID_STATEMENT, VOID_OPTIONS)
             connect_to_first_available(plan)
           else
-            @logger.debug('Stopping reconnection')
             Ione::Future.resolved
           end
         end
-        f.fallback do
+        f.fallback do |e|
+          @logger.error("Control connection failed (#{e.class.name}: #{e.message})")
+
           if synchronize { @status == :reconnecting }
             reconnect_async(schedule)
           else
-            @logger.debug('Stopping reconnection')
             return Ione::Future.resolved
           end
         end
@@ -137,13 +161,20 @@ module Cassandra
       def register_async
         connection = @connection
 
-        return Ione::Future.failed("not connected") if connection.nil?
-
-        @logger.debug('Registering for events')
-
-        connection.send_request(REGISTER).map do
-          @logger.debug('Registered for events')
+        return Ione::Future.failed(Errors::ClientError.new('Not connected')) if connection.nil?
 
+        f = connection.send_request(REGISTER)
+        f = f.map do |r|
+          case r
+          when Protocol::ReadyResponse
+            nil
+          when Protocol::ErrorResponse
+            raise r.to_error(VOID_STATEMENT)
+          else
+            raise Errors::InternalError, "Unexpected response #{r.inspect}"
+          end
+        end
+        f = f.map do
           connection.on_event do |event|
             @logger.debug("Event received #{event}")
 
@@ -151,21 +182,21 @@ module Cassandra
               case event.change
               when 'CREATED'
                 if event.table.empty?
-                  refresh_schema_async
+                  refresh_schema_async_maybe_retry
                 else
-                  refresh_keyspace_async(event.keyspace)
+                  refresh_keyspace_async_maybe_retry(event.keyspace)
                 end
               when 'DROPPED'
                 if event.table.empty?
-                  refresh_schema_async
+                  refresh_schema_async_maybe_retry
                 else
-                  refresh_keyspace_async(event.keyspace)
+                  refresh_keyspace_async_maybe_retry(event.keyspace)
                 end
               when 'UPDATED'
                 if event.table.empty?
-                  refresh_keyspace_async(event.keyspace)
+                  refresh_keyspace_async_maybe_retry(event.keyspace)
                 else
-                  refresh_table_async(event.keyspace, event.table)
+                  refresh_table_async_maybe_retry(event.keyspace, event.table)
                 end
               end
             else
@@ -173,19 +204,19 @@ module Cassandra
               when 'UP'
                 address = event.address
 
-                refresh_host_async(address) if @registry.has_host?(address)
+                refresh_host_async_maybe_retry(address) if @registry.has_host?(address)
               when 'DOWN'
                 @registry.host_down(event.address)
               when 'NEW_NODE'
                 address = event.address
 
                 unless @registry.has_host?(address)
-                  refresh_host_async(address)
-                  refresh_schema_async
+                  refresh_host_async_maybe_retry(address)
+                  refresh_schema_async_maybe_retry
                 end
               when 'REMOVED_NODE'
                 @registry.host_lost(event.address)
-                refresh_schema_async
+                refresh_schema_async_maybe_retry
               end
             end
           end
@@ -197,85 +228,247 @@ module Cassandra
       def refresh_schema_async
         connection = @connection
 
-        return Ione::Future.failed("not connected") if connection.nil?
-
-        @logger.debug('Fetching schema metadata')
+        return Ione::Future.failed(Errors::ClientError.new('Not connected')) if connection.nil?
 
-        keyspaces = connection.send_request(SELECT_KEYSPACES)
-        tables    = connection.send_request(SELECT_TABLES)
-        columns   = connection.send_request(SELECT_COLUMNS)
+        keyspaces = send_select_request(connection, SELECT_KEYSPACES)
+        tables    = send_select_request(connection, SELECT_TABLES)
+        columns   = send_select_request(connection, SELECT_COLUMNS)
 
         Ione::Future.all(keyspaces, tables, columns).map do |(keyspaces, tables, columns)|
-          @logger.debug('Fetched schema metadata')
-
           host = @registry.host(connection.host)
 
-          @schema.update_keyspaces(host, keyspaces.rows, tables.rows, columns.rows)
+          @schema.update_keyspaces(host, keyspaces, tables, columns)
           @metadata.rebuild_token_map
         end
       end
 
+      def refresh_schema_async_maybe_retry
+        synchronize do
+          return Ione::Future.resolved if @refreshing_schema
+          @refreshing_schema = true
+        end
+
+        refresh_schema_async.fallback do |e|
+          case e
+          when Errors::HostError
+            refresh_schema_async_retry(e, @reconnection_policy.schedule)
+          else
+            connection = @connection
+            connection && connection.close(e)
+
+            Ione::Future.resolved
+          end
+        end.map do
+          synchronize do
+            @refreshing_schema = false
+          end
+        end
+      end
+
+      def refresh_schema_async_retry(error, schedule)
+        timeout = schedule.next
+        @logger.info("Failed to refresh schema (#{error.class.name}: #{error.message}), retrying in #{timeout}")
+
+        timer = @io_reactor.schedule_timer(timeout)
+        timer.flat_map do
+          refresh_schema_async.fallback do |e|
+            case e
+            when Errors::HostError
+              refresh_schema_async_retry(e, schedule)
+            else
+              connection = @connection
+              connection && connection.close(e)
+
+              Ione::Future.resolved
+            end
+          end
+        end
+      end
+
+      def refresh_keyspace_async_maybe_retry(keyspace)
+        synchronize do
+          return Ione::Future.resolved if @refreshing_schema || @refreshing_keyspaces[keyspace]
+          @refreshing_keyspaces[keyspace] = true
+        end
+
+        refresh_keyspace_async(keyspace).fallback do |e|
+          case e
+          when Errors::HostError
+            refresh_keyspace_async_retry(keyspace, e, @reconnection_policy.schedule)
+          else
+            connection = @connection
+            connection && connection.close(e)
+
+            Ione::Future.resolved
+          end
+        end.map do
+          synchronize do
+            @refreshing_keyspaces.delete(host)
+          end
+        end
+      end
+
+      def refresh_keyspace_async_retry(keyspace, error, schedule)
+        timeout = schedule.next
+        @logger.info("Failed to refresh keyspace #{keyspace} (#{error.class.name}: #{error.message}), retrying in #{timeout}")
+
+        timer = @io_reactor.schedule_timer(timeout)
+        timer.flat_map do
+          refresh_keyspace_async(keyspace).fallback do |e|
+            case e
+            when Errors::HostError
+              refresh_keyspace_async_retry(keyspace, e, schedule)
+            else
+              connection = @connection
+              connection && connection.close(e)
+
+              Ione::Future.resolved
+            end
+          end
+        end
+      end
+
       def refresh_keyspace_async(keyspace)
         connection = @connection
 
-        return Ione::Future.failed("not connected") if connection.nil?
-
-        @logger.debug("Fetching keyspace #{keyspace.inspect} metadata")
+        return Ione::Future.failed(Errors::ClientError.new('Not connected')) if connection.nil?
 
         params    = [keyspace]
-        keyspaces = connection.send_request(Protocol::QueryRequest.new("SELECT * FROM system.schema_keyspaces WHERE keyspace_name = ?", params, nil, :one))
-        tables    = connection.send_request(Protocol::QueryRequest.new("SELECT * FROM system.schema_columnfamilies WHERE keyspace_name = ?", params, nil, :one))
-        columns   = connection.send_request(Protocol::QueryRequest.new("SELECT * FROM system.schema_columns WHERE keyspace_name = ?", params, nil, :one))
+        keyspaces = send_select_request(connection, Protocol::QueryRequest.new("SELECT * FROM system.schema_keyspaces WHERE keyspace_name = ?", params, nil, :one))
+        tables    = send_select_request(connection, Protocol::QueryRequest.new("SELECT * FROM system.schema_columnfamilies WHERE keyspace_name = ?", params, nil, :one))
+        columns   = send_select_request(connection, Protocol::QueryRequest.new("SELECT * FROM system.schema_columns WHERE keyspace_name = ?", params, nil, :one))
 
         Ione::Future.all(keyspaces, tables, columns).map do |(keyspaces, tables, columns)|
-          @logger.debug("Fetched keyspace #{keyspace.inspect} metadata")
-
           host = @registry.host(connection.host)
 
-          @schema.update_keyspace(host, keyspaces.rows.first, tables.rows, columns.rows)
+          @schema.update_keyspace(host, keyspaces.first, tables, columns) unless keyspaces.empty?
+        end
+      end
+
+      def refresh_table_async_maybe_retry(keyspace, table)
+        synchronize do
+          return Ione::Future.resolved if @refreshing_schema || @refreshing_keyspaces[keyspace] || @refreshing_tables[keyspace][table]
+          @refreshing_tables[keyspace] ||= ::Hash.new(false)
+          @refreshing_tables[keyspace][table] = true
+        end
+
+        refresh_table_async(keyspace, table).fallback do |e|
+          case e
+          when Errors::HostError
+            refresh_keyspace_async_retry(keyspace, e, @reconnection_policy.schedule)
+          else
+            connection = @connection
+            connection && connection.close(e)
+
+            Ione::Future.resolved
+          end
+        end.map do
+          synchronize do
+            @refreshing_tables[keyspace].delete(table)
+            @refreshing_tables.delete(keyspace) if @refreshing_tables[keyspace].empty?
+          end
+        end
+      end
+
+      def refresh_table_async_retry(keyspace, table, error, schedule)
+        timeout = schedule.next
+        @logger.info("Failed to refresh keyspace #{keyspace} (#{error.class.name}: #{error.message}), retrying in #{timeout}")
+
+        timer = @io_reactor.schedule_timer(timeout)
+        timer.flat_map do
+          refresh_keyspace_async(keyspace).fallback do |e|
+            case e
+            when Errors::HostError
+              refresh_keyspace_async_retry(keyspace, e, schedule)
+            else
+              connection = @connection
+              connection && connection.close(e)
+
+              Ione::Future.resolved
+            end
+          end
         end
       end
 
       def refresh_table_async(keyspace, table)
         connection = @connection
 
-        return Ione::Future.failed("not connected") if connection.nil?
-
-        @logger.debug("Fetching table \"#{keyspace}.#{table}\" metadata")
+        return Ione::Future.failed(Errors::ClientError.new('Not connected')) if connection.nil?
 
         params   = [keyspace, table]
-        table    = connection.send_request(Protocol::QueryRequest.new("SELECT * FROM system.schema_columnfamilies WHERE keyspace_name = ? AND columnfamily_name = ?", params, nil, :one))
-        columns  = connection.send_request(Protocol::QueryRequest.new("SELECT * FROM system.schema_columns WHERE keyspace_name = ? AND columnfamily_name = ?", params, nil, :one))
+        table    = send_select_request(connection, Protocol::QueryRequest.new("SELECT * FROM system.schema_columnfamilies WHERE keyspace_name = ? AND columnfamily_name = ?", params, nil, :one))
+        columns  = send_select_request(connection, Protocol::QueryRequest.new("SELECT * FROM system.schema_columns WHERE keyspace_name = ? AND columnfamily_name = ?", params, nil, :one))
 
         Ione::Future.all(table, columns).map do |(table, columns)|
-          @logger.debug("Fetched table \"#{keyspace}.#{table}\" metadata")
-
           host = @registry.host(connection.host)
 
-          @schema.udpate_table(host, keyspace, table.rows, columns.rows)
+          @schema.udpate_table(host, keyspace, table, columns)
         end
       end
 
-      def refresh_hosts_async
-        connection = @connection
+      def refresh_hosts_async_maybe_retry
+        synchronize do
+          return Ione::Future.resolved if @refreshing_hosts
+          @refreshing_hosts = true
+        end
 
-        return Ione::Future.failed("not connected") if connection.nil?
+        refresh_hosts_async.fallback do |e|
+          case e
+          when Errors::HostError
+            refresh_hosts_async_retry(e, @reconnection_policy.schedule)
+          else
+            connection = @connection
+            connection && connection.close(e)
 
-        @logger.debug('Fetching cluster metadata and peers')
+            Ione::Future.resolved
+          end
+        end.map do
+          synchronize do
+            @refreshing_hosts = false
+          end
+        end
+      end
 
-        local = connection.send_request(SELECT_LOCAL)
-        peers = connection.send_request(SELECT_PEERS)
+      def refresh_hosts_async_retry(error, schedule)
+        timeout = schedule.next
+        @logger.info("Failed to refresh hosts (#{error.class.name}: #{error.message}), retrying in #{timeout}")
+
+        timer = @io_reactor.schedule_timer(timeout)
+        timer.flat_map do
+          refresh_hosts_async.fallback do |e|
+            case e
+            when Errors::HostError
+              refresh_hosts_async_retry(e, schedule)
+            else
+              connection = @connection
+              connection && connection.close(e)
+
+              Ione::Future.resolved
+            end
+          end
+        end
+      end
 
-        Ione::Future.all(local, peers).flat_map do |(local, peers)|
-          local = local.rows
-          peers = peers.rows
+      def refresh_hosts_async
+        connection = @connection
 
-          @logger.debug('%d peers found' % peers.size)
+        return Ione::Future.failed(Errors::ClientError.new('Not connected')) if connection.nil?
 
-          raise NO_HOSTS if local.empty? && peers.empty?
+        local = send_select_request(connection, SELECT_LOCAL)
+        peers = send_select_request(connection, SELECT_PEERS)
+
+        Ione::Future.all(local, peers).map do |(local, peers)|
+          @logger.debug("#{peers.size} peer(s) found")
 
           ips = ::Set.new
 
+          unless local.empty?
+            ips << ip = IPAddr.new(connection.host)
+            data = local.first
+            @registry.host_found(ip, data)
+            @metadata.update(data)
+          end
+
           peers.each do |data|
             ip = peer_ip(data)
             next unless ip
@@ -283,27 +476,11 @@ module Cassandra
             @registry.host_found(ip, data)
           end
 
-          ips << ip = IPAddr.new(connection.host)
-          data = local.first
-          @registry.host_found(ip, data)
-
-          futures = []
-
           @registry.each_host do |host|
-            if ips.include?(host.ip)
-              futures << refresh_host_status(host) if host.down?
-            else
-              @registry.host_lost(host.ip)
-            end
+            @registry.host_lost(host.ip) unless ips.include?(host.ip)
           end
 
-          @metadata.update(data)
-
-          if futures.empty?
-            Ione::Future.resolved
-          else
-            Ione::Future.all(*futures)
-          end
+          nil
         end
       end
 
@@ -311,46 +488,87 @@ module Cassandra
         @connector.refresh_status(host)
       end
 
-      def refresh_host_status_with_retry(host, schedule)
-        timeout = schedule.next
+      def refresh_host_status_with_retry(original_timer, host, schedule)
+        timer = nil
 
-        @logger.info("Refreshing host status refresh ip=#{host.ip} in #{timeout}")
+        synchronize do
+          timer = @refreshing_statuses[host]
 
-        f = @io_reactor.schedule_timer(timeout)
-        f.flat_map do
-          if synchronize { @refreshing_statuses[host] }
-            refresh_host_status(host).fallback do |e|
-              refresh_host_status_with_retry(host, schedule)
-            end
+          # host must have been lost/up or timer was rescheduled
+          return Ione::Future.resolved if timer.nil? || timer != original_timer
+
+          timeout = schedule.next
+
+          @logger.debug("Checking host #{host.ip} in #{timeout} seconds")
+
+          @refreshing_statuses[host] = timer = @io_reactor.schedule_timer(timeout)
+        end
+
+        timer.on_value do
+          refresh_host_status(host).fallback do |e|
+            refresh_host_status_with_retry(timer, host, schedule)
+          end
+        end
+      end
+
+      def refresh_host_async_maybe_retry(host)
+        synchronize do
+          return Ione::Future.resolved if @refreshing_hosts || @refreshing_host[host]
+          @refreshing_host[host] = true
+        end
+
+        refresh_host_async(host).fallback do |e|
+          case e
+          when Errors::HostError
+            refresh_host_async_retry(host, e, @reconnection_policy.schedule)
           else
+            connection = @connection
+            connection && connection.close(e)
+
             Ione::Future.resolved
           end
+        end.map do
+          synchronize do
+            @refreshing_host.delete(host)
+          end
+        end
+      end
+
+      def refresh_host_async_retry(host, error, schedule)
+        timeout = schedule.next
+        @logger.info("Failed to refresh host #{host.ip.to_s} (#{error.class.name}: #{error.message}), retrying in #{timeout}")
+
+        timer = @io_reactor.schedule_timer(timeout)
+        timer.flat_map do
+          refresh_host_async(host).fallback do |e|
+            case e
+            when Errors::HostError
+              refresh_host_async_retry(host, e, schedule)
+            else
+              connection = @connection
+              connection && connection.close(e)
+
+              Ione::Future.resolved
+            end
+          end
         end
       end
 
       def refresh_host_async(address)
         connection = @connection
-        return Ione::Future.failed("not connected") if connection.nil?
+        return Ione::Future.failed(Errors::ClientError.new('Not connected')) if connection.nil?
 
         ip = address.to_s
 
-        @logger.debug('Fetching node information for %s' % ip)
-
         if ip == connection.host
           request = SELECT_LOCAL
         else
           request = Protocol::QueryRequest.new('SELECT rack, data_center, host_id, rpc_address, release_version, tokens FROM system.peers WHERE peer = ?', [address], nil, :one)
         end
 
-        connection.send_request(request).map do |response|
-          @logger.debug('Fetched node information for %s' % ip)
-
-          rows = response.rows
-
+        send_select_request(connection, request).map do |rows|
           unless rows.empty?
             @registry.host_found(address, rows.first)
-            host = @registry.host(address)
-            refresh_host_status(host) if host.down?
           end
 
           self
@@ -359,33 +577,50 @@ module Cassandra
 
       def connect_to_first_available(plan, errors = nil)
         unless plan.has_next?
-          @logger.warn("Control connection failed")
-          return Ione::Future.failed(Errors::NoHostsAvailable.new(errors || {}))
+          if errors.nil? && synchronize { @refreshing_statuses.empty? }
+            @logger.fatal(<<-MSG)
+Control connection failed and is unlikely to recover.
+
+	This usually means that all hosts are ignored by current load
+	balancing policy, most likely because they changed datacenters.
+	Reconnections attempts will continue getting scheduled to
+	repeat this message in the logs.
+            MSG
+          end
+
+          return Ione::Future.failed(Errors::NoHostsAvailable.new(errors))
         end
 
         host = plan.next
-        @logger.debug("Attempting connection to ip=#{host.ip}")
+        @logger.debug("Connecting to #{host.ip}")
+
         f = connect_to_host(host)
         f = f.flat_map do |connection|
           synchronize do
             @status = :connected
 
-            @logger.debug("Control connection established ip=#{connection.host}")
             @connection = connection
 
-            connection.on_closed do
+            connection.on_closed do |cause|
               reconnect = false
 
               synchronize do
-                if @status == :closing
-                  @status = :closed
-                else
-                  @status = :reconnecting
-                  reconnect = true
+                if connection == @connection
+                  if @status == :closing
+                    @status = :closed
+                  else
+                    @status = :reconnecting
+                    reconnect = true
+                  end
+
+                  if cause
+                    @logger.info("Control connection closed (#{cause.class.name}: #{cause.message})")
+                  else
+                    @logger.info("Control connection closed")
+                  end
+
+                  @connection = nil
                 end
-
-                @logger.debug("Control connection closed ip=#{connection.host}")
-                @connection = nil
               end
 
               reconnect_async(@reconnection_policy.schedule) if reconnect
@@ -394,23 +629,26 @@ module Cassandra
 
           register_async
         end
-        f = f.flat_map { refresh_hosts_async }
-        f = f.flat_map { refresh_schema_async }
-        f.fallback do |error|
-          if error.is_a?(Errors::AuthenticationError)
-            Ione::Future.failed(error)
-          elsif error.is_a?(Errors::QueryError)
-            if error.code == 0x100
-              Ione::Future.failed(Errors::AuthenticationError.new(error.message))
-            else
-              Ione::Future.failed(error)
-            end
-          else
+        f = f.flat_map { refresh_hosts_async_maybe_retry }
+        f = f.flat_map { refresh_schema_async_maybe_retry }
+        f = f.fallback do |error|
+          @logger.debug("Connection to #{host.ip} failed (#{error.class.name}: #{error.message})")
+
+          case error
+          when Errors::HostError
             errors ||= {}
             errors[host] = error
             connect_to_first_available(plan, errors)
+          else
+            Ione::Future.failed(error)
           end
         end
+
+        f.on_complete do |f|
+          @logger.info('Control connection established') if f.resolved?
+        end
+
+        f
       end
 
       def connect_to_host(host)
@@ -420,7 +658,21 @@ module Cassandra
       def peer_ip(data)
         ip = data['rpc_address']
         ip = data['peer'] if ip == '0.0.0.0'
-        ip
+
+        @address_resolver.resolve(ip)
+      end
+
+      def send_select_request(connection, request)
+        connection.send_request(request).map do |r|
+          case r
+          when Protocol::RowsResultResponse
+            r.rows
+          when Protocol::ErrorResponse
+            raise r.to_error(VOID_STATEMENT)
+          else
+            raise Errors::InternalError, "Unexpected response #{r.inspect}"
+          end
+        end
       end
     end
   end