ably 0.6.2 → 0.7.0

data/lib/ably/realtime/connection/connection_manager.rb

@@ -2,18 +2,45 @@ module Ably::Realtime
   class Connection
     # ConnectionManager is responsible for all actions relating to underlying connection and transports,
     # such as opening, closing, attempting reconnects etc.
+    # Connection state changes are performed by this class and executed from {ConnectionStateMachine}
     #
     # This is a private class and should never be used directly by developers as the API is likely to change in future.
     #
     # @api private
     class ConnectionManager
-      CONNECTION_FAILED = { retry_after: 0.5, max_retries: 2, code: 80000 }.freeze
+      # Configuration for automatic recovery of failed connection attempts
+      CONNECT_RETRY_CONFIG = {
+        disconnected: { retry_every: 0.5, max_time_in_state: 10 },
+        suspended:    { retry_every: 5,   max_time_in_state: 60 }
+      }.freeze
+
+      # Time to wait following a connection state request before it's considered a failure
+      TIMEOUTS = {
+        open:  15,
+        close: 10
+      }
+
+      # Error codes from the server that can potentially be resolved
+      RESOLVABLE_ERROR_CODES = {
+        token_expired: 40140
+      }
 
       def initialize(connection)
         @connection = connection
+        @timers     = Hash.new { |hash, key| hash[key] = [] }
+
+        connection.on(:closed) do
+          connection.reset_resume_info
+        end
+
+        connection.once(:connecting) do
+          close_connection_when_reactor_is_stopped
+        end
 
-        @timers               = Hash.new { |hash, key| hash[key] = [] }
-        @timers[:initializer] << EventMachine::Timer.new(0.01) { connection.connect }
+        EventMachine.next_tick do
+          # Connect once Connection object is initialised
+          connection.connect if client.connect_automatically
+        end
       end
 
       # Creates and sets up a new {Ably::Realtime::Connection::WebsocketTransport} available on attribute #transport
@@ -25,20 +52,30 @@ module Ably::Realtime
           raise RuntimeError, 'Existing WebsocketTransport is connected, and must be closed first'
         end
 
+        unless client.auth.authentication_security_requirements_met?
+          connection.transition_state_machine :failed, Ably::Exceptions::InsecureRequestError.new('Cannot use Basic Auth over non-TLS connections', 401, 40103)
+          return
+        end
+
         logger.debug "ConnectionManager: Opening connection to #{connection.host}:#{connection.port}"
 
         connection.create_websocket_transport do |websocket_transport|
           subscribe_to_transport_events websocket_transport
           yield websocket_transport if block_given?
         end
+
+        logger.debug 'ConnectionManager: Setting up automatic connection timeout timer for #{TIMEOUTS.fetch(:open)}s'
+        create_timeout_timer_whilst_in_state(:connect, TIMEOUTS.fetch(:open)) do
+          connection_opening_failed Ably::Exceptions::ConnectionTimeoutError.new("Connection to Ably timed out after #{TIMEOUTS.fetch(:open)}s", nil, 80014)
+        end
       end
 
       # Called by the transport when a connection attempt fails
       #
       # @api private
-      def connection_failed(error)
-        logger.info "ConnectionManager: Connection to #{connection.host}:#{connection.port} failed; #{error.message}"
-        connection.transition_state_machine :disconnected, Ably::Models::ErrorInfo.new(message: "Connection failed; #{error.message}", code: 80000)
+      def connection_opening_failed(error)
+        logger.warn "ConnectionManager: Connection to #{connection.host}:#{connection.port} failed; #{error.message}"
+        connection.transition_state_machine next_retry_state, Ably::Exceptions::ConnectionError.new("Connection failed; #{error.message}", nil, 80000)
       end
 
       # Ensures the underlying transport has been disconnected and all event emitter callbacks removed
@@ -67,43 +104,75 @@ module Ably::Realtime
       #
       # @api private
       def close_connection
-        protocol_message = connection.send_protocol_message(action: Ably::Models::ProtocolMessage::ACTION.Close)
-
-        unsubscribe_from_transport_events transport
+        connection.send_protocol_message(action: Ably::Models::ProtocolMessage::ACTION.Close)
 
-        protocol_message.callback do
-          destroy_transport
+        create_timeout_timer_whilst_in_state(:close, TIMEOUTS.fetch(:close)) do
+          force_close_connection if connection.closing?
         end
       end
 
-      # Remove all timers set up as part of the initialize process.
-      # Typically called by StateMachine when connection is closed and can no longer process the timers
+      # Close the underlying transport immediately and set the connection state to closed
       #
       # @api private
-      def cancel_initialized_timers
-        clear_timers :initializer
+      def force_close_connection
+        destroy_transport
+        connection.transition_state_machine :closed
       end
 
-      # Remove all timers related to connection attempt retries following a disconnect or suspended connection state.
-      # Typically called by StateMachine when connection is opened to ensure no further connection attempts are made
+      # When a connection is disconnected whilst connecting, attempt reconnect and/or set state to :suspended or :failed
       #
       # @api private
-      def cancel_connection_retry_timers
-        clear_timers :connection_retry_timers
+      def respond_to_transport_disconnected_when_connecting(current_transition)
+        return unless connection.disconnected? || connection.suspended? # do nothing if state has changed through an explicit request
+        return unless retry_connection? # do not always reattempt connection or change state as client may be re-authorising
+
+        unless connection_retry_from_suspended_state?
+          return if connection_retry_for(:disconnected, ignore_states: [:connecting])
+        end
+
+        return if connection_retry_for(:suspended, ignore_states: [:connecting])
+
+        # Fallback if no other criteria met
+        connection.transition_state_machine :failed, current_transition.metadata
       end
 
-      # When a connection is disconnected try and reconnect or set the connection state to :failed
+      # When a connection is disconnected after connecting, attempt reconnect and/or set state to :suspended or :failed
       #
       # @api private
-      def respond_to_transport_disconnected(current_transition)
-        error_code = current_transition && current_transition.metadata && current_transition.metadata.code
+      def respond_to_transport_disconnected_whilst_connected(current_transition)
+        logger.warn "ConnectionManager: Connection to #{connection.transport.url} was disconnected unexpectedly"
 
-        if connection.previous_state == :connecting && error_code == CONNECTION_FAILED[:code]
-          return if retry_connection_failed
+        if current_transition.metadata.kind_of?(Ably::Models::ErrorInfo)
+          connection.trigger :error, current_transition.metadata
+          logger.error "ConnectionManager: Error received when disconnected within ProtocolMessage - #{current_transition.metadata}"
         end
 
-        # Fallback if no other criteria met
-        connection.transition_state_machine :failed, current_transition.metadata
+        destroy_transport
+        respond_to_transport_disconnected_when_connecting current_transition
+      end
+
+      # {Ably::Models::ProtocolMessage ProtocolMessage Error} received from server.
+      # Some error states can be resolved by the client library.
+      #
+      # @api private
+      def error_received_from_server(error)
+        case error.code
+        when RESOLVABLE_ERROR_CODES.fetch(:token_expired)
+          connection.transition_state_machine :disconnected
+          connection.once_or_if(:disconnected) do
+            renew_token_and_reconnect error
+          end
+        else
+          logger.error "ConnectionManager: Error #{error.class.name} code #{error.code} received from server '#{error.message}', transitioning to failed state"
+          connection.transition_state_machine :failed, error
+        end
+      end
+
+      # Number of consecutive attempts for provided state
+      # @return [Integer]
+      # @api private
+      def retry_count_for_state(state)
+        retries_for_state(state, ignore_states: [:connecting]).count
       end
 
       private
@@ -121,20 +190,79 @@ module Ably::Realtime
         connection.client
       end
 
+      # Create a timer that will execute in timeout_in seconds.
+      # If the connection state changes however, cancel the timer
+      def create_timeout_timer_whilst_in_state(timer_id, timeout_in, &block)
+        raise 'Block required for timer' unless block_given?
+
+        timers[timer_id] << EventMachine::Timer.new(timeout_in) do
+          block.call
+        end
+        connection.once_state_changed { clear_timers timer_id }
+      end
+
       def clear_timers(key)
         timers.fetch(key, []).each(&:cancel)
       end
 
-      def retry_connection_failed
-        if retries_for_state(:disconnected, ignore_states: [:connecting]).count < CONNECTION_FAILED[:max_retries]
-          logger.debug "ConnectionManager: Pausing for #{CONNECTION_FAILED[:retry_after]}s before attempting to reconnect"
-          @timers[:connection_retry_timers] << EventMachine::Timer.new(CONNECTION_FAILED[:retry_after]) do
-            connection.connect
+      def next_retry_state
+        if connection_retry_from_suspended_state? || time_passed_since_disconnected > CONNECT_RETRY_CONFIG.fetch(:disconnected).fetch(:max_time_in_state)
+          :suspended
+        else
+          :disconnected
+        end
+      end
+
+      def connection_retry_from_suspended_state?
+        !retries_for_state(:suspended, ignore_states: [:connecting]).empty?
+      end
+
+      def time_passed_since_disconnected
+        time_spent_attempting_state(:disconnected, ignore_states: [:connecting])
+      end
+
+      # Reattempt a connection with a delay based on the CONNECT_RETRY_CONFIG for `from_state`
+      #
+      # @return [Boolean] True if a connection attempt has been set up, false if no further connection attempts can be made for this state
+      #
+      def connection_retry_for(from_state, options = {})
+        retry_params = CONNECT_RETRY_CONFIG.fetch(from_state)
+
+        if time_spent_attempting_state(from_state, options) <= retry_params.fetch(:max_time_in_state)
+          logger.debug "ConnectionManager: Pausing for #{retry_params.fetch(:retry_every)}s before attempting to reconnect"
+          create_timeout_timer_whilst_in_state(:reconnect, retry_params.fetch(:retry_every)) do
+            connection.connect if connection.state == from_state
           end
+          true
         end
       end
 
-      def retries_for_state(state, ignore_states: [])
+      # Returns a float representing the amount of time passed since the first consecutive attempt of this state
+      #
+      # @param (see #retries_for_state)
+      # @return [Float] time passed in seconds
+      #
+      def time_spent_attempting_state(state, options)
+        states = retries_for_state(state, options)
+        if states.empty?
+          0
+        else
+          Time.now.to_f - states.last[:transitioned_at].to_f
+        end.to_f
+      end
+
+      # Checks the state change history for the current connection and returns all matching consecutive states.
+      # This is useful to determine the number of retries of a particular state on a connection.
+      #
+      # @param  state   [Symbol]
+      # @param  options [Hash]
+      # @option options [Array<Symbol>] :ignore_states states that should be ignored when determining consecutive historical retries for `state`.
+      #                                 For example, when working out :connecting attempts, :disconnect state changes should be ignored as they are a side effect of a failed :connecting
+      #
+      # @return [Array<Hash>] Array of consecutive state attempts matching `state` in order of transitioned_at desc
+      #
+      def retries_for_state(state, options)
+        ignore_states = options.fetch(:ignore_states, [])
         allowed_states = Array(state) + Array(ignore_states)
 
         connection.state_history.reverse.take_while do |transition|
@@ -150,7 +278,52 @@ module Ably::Realtime
         end
 
         transport.on(:disconnected) do
-          connection.transition_state_machine :disconnected
+          if connection.closing?
+            connection.transition_state_machine :closed
+          elsif !connection.closed? && !connection.disconnected?
+            connection.transition_state_machine :disconnected
+          end
+        end
+      end
+
+      def renew_token_and_reconnect(error)
+        if client.auth.token_renewable?
+          if @renewing_token
+            connection.transition_state_machine :failed, error
+            return
+          end
+
+          @renewing_token = true
+          logger.warn "ConnectionManager: Token has expired and is renewable, renewing token now"
+
+          operation = proc do
+            begin
+              client.auth.authorise
+            rescue StandardError => auth_error
+              connection.transition_state_machine :failed, auth_error
+              nil
+            end
+          end
+
+          callback = proc do |token|
+            state_changed_callback = proc do
+              @renewing_token = false
+              connection.off &state_changed_callback
+            end
+
+            connection.once :connected, :closed, :failed, &state_changed_callback
+
+            if token && !token.expired?
+              reconnect_transport
+            else
+              connection.transition_state_machine :failed, error unless connection.failed?
+            end
+          end
+
+          EventMachine.defer operation, callback
+        else
+          logger.warn "ConnectionManager: Token has expired and is not renewable"
+          connection.transition_state_machine :failed, error
         end
       end
 
@@ -160,6 +333,16 @@ module Ably::Realtime
         logger.debug "ConnectionManager: Unsubscribed from all events from current transport"
       end
 
+      def close_connection_when_reactor_is_stopped
+        EventMachine.add_shutdown_hook do
+          connection.close unless connection.closed? || connection.failed?
+        end
+      end
+
+      def retry_connection?
+        !@renewing_token
+      end
+
       def logger
         connection.logger
       end

data/lib/ably/realtime/connection/connection_state_machine.rb

@@ -1,40 +1,10 @@
-require 'statesman'
+require 'ably/modules/state_machine'
 
 module Ably::Realtime
   class Connection
-    module StatesmanMonkeyPatch
-      # Override Statesman's #before_transition to support :from arrays
-      # This can be removed once https://github.com/gocardless/statesman/issues/95 is solved
-      def before_transition(options = nil, &block)
-        arrayify_transition(options) do |options_without_from_array|
-          super *options_without_from_array, &block
-        end
-      end
-
-      def after_transition(options = nil, &block)
-        arrayify_transition(options) do |options_without_from_array|
-          super *options_without_from_array, &block
-        end
-      end
-
-      private
-      def arrayify_transition(options, &block)
-        if options.nil?
-          yield []
-        elsif options.fetch(:from, nil).kind_of?(Array)
-          options[:from].each do |from_state|
-            yield [options.merge(from: from_state)]
-          end
-        else
-          yield [options]
-        end
-      end
-    end
-
-    # Internal class to manage connection state, recovery and state transitions for an {Ably::Realtime::Connection}
+    # Internal class to manage connection state, recovery and state transitions for {Ably::Realtime::Connection}
     class ConnectionStateMachine
-      include Statesman::Machine
-      extend StatesmanMonkeyPatch
+      include Ably::Modules::StateMachine
 
       # States supported by this StateMachine match #{Connection::STATE}s
       #   :initialized
@@ -42,17 +12,19 @@ module Ably::Realtime
       #   :connected
       #   :disconnected
       #   :suspended
+      #   :closing
       #   :closed
       #   :failed
       Connection::STATE.each_with_index do |state_enum, index|
         state state_enum.to_sym, initial: index == 0
       end
 
-      transition :from => :initialized,  :to => [:connecting, :closed]
-      transition :from => :connecting,   :to => [:connected, :failed, :closed, :disconnected]
-      transition :from => :connected,    :to => [:disconnected, :suspended, :closed, :failed]
-      transition :from => :disconnected, :to => [:connecting, :closed, :failed]
-      transition :from => :suspended,    :to => [:connecting, :closed, :failed]
+      transition :from => :initialized,  :to => [:connecting, :closing]
+      transition :from => :connecting,   :to => [:connected, :failed, :closing, :disconnected, :suspended]
+      transition :from => :connected,    :to => [:disconnected, :suspended, :closing, :failed]
+      transition :from => :disconnected, :to => [:connecting, :closing, :suspended, :failed]
+      transition :from => :suspended,    :to => [:connecting, :closing, :failed]
+      transition :from => :closing,      :to => [:closed]
       transition :from => :closed,       :to => [:connecting]
       transition :from => :failed,       :to => [:connecting]
 
@@ -68,60 +40,45 @@ module Ably::Realtime
         connection.manager.reconnect_transport
       end
 
-      before_transition(to: [:connected]) do |connection|
-        connection.manager.cancel_connection_retry_timers
+      after_transition(to: [:disconnected, :suspended], from: [:connecting]) do |connection, current_transition|
+        connection.manager.respond_to_transport_disconnected_when_connecting current_transition
+      end
+
+      after_transition(to: [:disconnected], from: [:connected]) do |connection, current_transition|
+        connection.manager.respond_to_transport_disconnected_whilst_connected current_transition
       end
 
-      after_transition(to: [:disconnected], from: [:connecting]) do |connection, current_transition|
-        connection.manager.respond_to_transport_disconnected current_transition
+      after_transition(to: [:disconnected, :suspended]) do |connection|
+        connection.manager.destroy_transport # never reuse a transport if the connection has failed
       end
 
-      after_transition(to: [:failed]) do |connection|
+      after_transition(to: [:failed]) do |connection, current_transition|
+        connection.logger.fatal "ConnectionStateMachine: Connection failed - #{current_transition.metadata}"
         connection.manager.destroy_transport
       end
 
-      before_transition(to: [:closed], from: [:initialized]) do |connection|
-        connection.manager.cancel_initialized_timers
+      after_transition(to: [:closing], from: [:initialized, :disconnected, :suspended]) do |connection|
+        connection.manager.force_close_connection
       end
 
-      before_transition(to: [:closed], from: [:connecting, :connected, :disconnected, :suspended]) do |connection|
+      after_transition(to: [:closing], from: [:connecting, :connected]) do |connection|
         connection.manager.close_connection
       end
 
-      # Override Statesman's #transition_to so that:
-      # * log state change failures to {Logger}
-      def transition_to(state, *args)
-        unless result = super(state, *args)
-          logger.fatal "ConnectionStateMachine: Unable to transition from #{current_state} => #{state}"
-        end
-        result
+      before_transition(to: [:closed], from: [:closing]) do |connection|
+        connection.manager.destroy_transport
       end
 
-      def previous_transition
-        history[-2]
+      # Transitions responsible for updating connection#error_reason
+      before_transition(to: [:disconnected, :suspended, :failed]) do |connection, current_transition|
+        connection.set_failed_connection_error_reason current_transition.metadata
       end
 
-      def previous_state
-        previous_transition.to_state if previous_transition
+      before_transition(to: [:connected, :closed]) do |connection, current_transition|
+        connection.set_failed_connection_error_reason nil
       end
 
       private
-      # TODO: Implement once CLOSED ProtocolMessage is sent back from Ably in response to a CLOSE message
-      #
-      # FORCE_CONNECTION_CLOSED_TIMEOUT = 5
-      #
-      # def force_closed_unless_server_acknowledge_closed
-      #   timeouts[:close_connection] << EventMachine::Timer.new(FORCE_CONNECTION_CLOSED_TIMEOUT) do
-      #     transition_to :closed
-      #   end
-      # end
-      #
-      # def clear_force_closed_timeouts
-      #   timeouts[:close_connection].each do |timeout|
-      #     timeout.cancel
-      #   end.clear
-      # end
-
       def connection
         object
       end