ably 0.8.13 → 0.8.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7859d1961b34f4db85c4e3b4dd9148aa17b456e6
-  data.tar.gz: df4339d846d3bb29ce58dc311b7e844d6a21fadb
+  metadata.gz: b7ff0fc1dc14fc84ea707c200752a84943571d53
+  data.tar.gz: 7ba75aa7a6bfdacd097ffa694409bb172711eddb
 SHA512:
-  metadata.gz: 711fd1f07710b81c319378740f0b193d0166f16ab00a1528de10c22a105a796212ab77954a7813d631afff1aa82ed7bf1e75f9984a90346b72a636cf5ff74d6d
-  data.tar.gz: 00e82298ba240966faae98bb7d33065bfa58ba118c40152ac906b92f6344f85d951130f23e40947d107dd809f4cd317e9f975b68324babb376a6ad1a36bd0edf
+  metadata.gz: 8eaf2f813c5003c87c64c0f9dbd33e6231ac1d9c1fbd2a1e07117f566c648b6eed4fd50eb4ec15e622bf510dc225e70cd3d6a42dfe25ca4da8ca69529d5001f4
+  data.tar.gz: f07522639b0068f1a1aea116d04bcfbc745d7d9862ad12ea64111921854c1da4eb785480f2c5881083c0aa30c052ce1741552ad8b8195c7fd75f4868df58a556

data/CHANGELOG.md

@@ -1,6 +1,22 @@
 # Change Log
 
-## [Unreleased](https://github.com/ably/ably-ruby/tree/v0.8.13)
+## [Unreleased](https://github.com/ably/ably-ruby/tree/v0.8.14)
+
+[Full Changelog](https://github.com/ably/ably-ruby/compare/v0.8.13...v0.8.14)
+
+**Fixed bugs:**
+
+- Several problems with fallback hosts [\#95](https://github.com/ably/ably-ruby/issues/95)
+
+**Closed issues:**
+
+- Catch common SSL error and show docs [\#96](https://github.com/ably/ably-ruby/issues/96)
+
+**Merged pull requests:**
+
+- Fallback host improvements [\#97](https://github.com/ably/ably-ruby/pull/97) ([mattheworiordan](https://github.com/mattheworiordan))
+
+## [v0.8.13](https://github.com/ably/ably-ruby/tree/v0.8.13) (2016-09-29)
 
 [Full Changelog](https://github.com/ably/ably-ruby/compare/v0.8.12...v0.8.13)
 

data/lib/ably.rb

@@ -1,5 +1,7 @@
 require 'addressable/uri'
 
+require 'ably/version'
+
 %w(modules util).each do |namespace|
   Dir.glob(File.expand_path("ably/#{namespace}/*.rb", File.dirname(__FILE__))).sort.each do |file|
     require file
@@ -11,4 +13,3 @@ require 'ably/exceptions'
 require 'ably/logger'
 require 'ably/realtime'
 require 'ably/rest'
-require 'ably/version'

data/lib/ably/exceptions.rb

@@ -59,7 +59,12 @@ module Ably
 
       def to_s
         message = [super]
-        message << "#{@base_error}" if @base_error
+        if @base_error
+          message << "#{@base_error}"
+          if @base_error.respond_to?(:message) && @base_error.message.match(/certificate verify failed/i)
+            message << "See https://goo.gl/eKvfcR to resolve this issue."
+          end
+        end
         message.join(' < ')
       end
     end

data/lib/ably/realtime/client.rb

@@ -155,6 +155,11 @@ module Ably
         rest_client.register_encoder encoder
       end
 
+      # (see Ably::Rest::Client#fallback_hosts)
+      def fallback_hosts
+        rest_client.fallback_hosts
+      end
+
       # (see Ably::Rest::Client#logger)
       def logger
         @logger ||= Ably::Logger.new(self, log_level, rest_client.logger.custom_logger)
@@ -168,14 +173,15 @@ module Ably
       end
 
       # @!attribute [r] fallback_endpoint
-      # @return [URI::Generic] Fallback endpoint used to connect to the realtime Ably service. Note, after each connection attempt, a new random {Ably::FALLBACK_HOSTS fallback host} is used
+      # @return [URI::Generic] Fallback endpoint used to connect to the realtime Ably service. Note, after each connection attempt, a new random {Ably::FALLBACK_HOSTS fallback host} or provided fallback hosts are used
       # @api private
       def fallback_endpoint
         unless defined?(@fallback_endpoints) && @fallback_endpoints
-          @fallback_endpoints = Ably::FALLBACK_HOSTS.shuffle.map { |fallback_host| endpoint_for_host(fallback_host) }
+          @fallback_endpoints = fallback_hosts.shuffle.map { |fallback_host| endpoint_for_host(fallback_host) }
+          @fallback_endpoints << endpoint # Try the original host last if all fallbacks have been used
         end
 
-        fallback_endpoint_index = connection.manager.retry_count_for_state(:disconnected) + connection.manager.retry_count_for_state(:suspended)
+        fallback_endpoint_index = connection.manager.retry_count_for_state(:disconnected) + connection.manager.retry_count_for_state(:suspended) - 1
 
         @fallback_endpoints[fallback_endpoint_index % @fallback_endpoints.count]
       end

data/lib/ably/realtime/connection.rb

@@ -64,8 +64,9 @@ module Ably
       DEFAULTS = {
         disconnected_retry_timeout: 15, # when the connection enters the DISCONNECTED state, after this delay in milliseconds, if the state is still DISCONNECTED, the client library will attempt to reconnect automatically
         suspended_retry_timeout:    30, # when the connection enters the SUSPENDED state, after this delay in milliseconds, if the state is still SUSPENDED, the client library will attempt to reconnect automatically
-        connection_state_ttl:       60, # the duration that Ably will persist the connection state when a Realtime client is abruptly disconnected
-        realtime_request_timeout:   10  # default timeout when establishing a connection, or sending a HEARTBEAT, CONNECT, ATTACH, DETACH or CLOSE ProtocolMessage
+        connection_state_ttl:       120, # the duration that Ably will persist the connection state when a Realtime client is abruptly disconnected
+        max_connection_state_ttl:   nil, # allow a max TTL to be passed in for CI test purposes thus overiding any connection_state_ttl sent from Ably
+        realtime_request_timeout:   10,  # default timeout when establishing a connection, or sending a HEARTBEAT, CONNECT, ATTACH, DETACH or CLOSE ProtocolMessage
       }.freeze
 
       # A unique public identifier for this connection, used to identify this member in presence events and messages
@@ -389,7 +390,7 @@ module Ably
                 format:    client.protocol,
                 echo:      client.echo_messages,
                 v:         Ably::PROTOCOL_VERSION,
-                lib:       Ably::LIB_VERSION_ID,
+                lib:       client.rest_client.lib_version_id,
               )
 
               url_params['clientId'] = client.auth.client_id if client.auth.has_client_id?
@@ -407,12 +408,12 @@ module Ably
 
               url = URI(client.endpoint).tap do |endpoint|
                 endpoint.query = URI.encode_www_form(url_params)
-              end.to_s
+              end
 
               determine_host do |host|
                 begin
-                  logger.debug "Connection: Opening socket connection to #{host}:#{port} and URL '#{url}'"
-                  @transport = EventMachine.connect(host, port, WebsocketTransport, self, url) do |websocket_transport|
+                  logger.debug "Connection: Opening socket connection to #{host}:#{port}/#{url.path}?#{url.query}"
+                  @transport = create_transport(host, port, url) do |websocket_transport|
                     websocket_deferrable.succeed websocket_transport
                   end
                 rescue EventMachine::ConnectionError => error
@@ -473,6 +474,22 @@ module Ably
           ( (initialized? || connecting? || disconnected?) && client.queue_messages )
       end
 
+      # @api private
+      def create_transport(host, port, url, &block)
+        EventMachine.connect(host, port, WebsocketTransport, self, url.to_s, &block)
+      end
+
+      # @api private
+      def connection_state_ttl
+        defaults[:max_connection_state_ttl] || # undocumented max TTL configuration
+          (details && details.connection_state_ttl) ||
+          defaults.fetch(:connection_state_ttl)
+      end
+
+      def connection_state_ttl=(val)
+        @connection_state_ttl = val
+      end
+
       # As we are using a state machine, do not allow change_state to be used
       # #transition_state_machine must be used instead
       private :change_state
@@ -527,7 +544,18 @@ module Ably
       end
 
       def connection_resumable?
-        !key.nil? && !serial.nil?
+        !key.nil? && !serial.nil? && connection_state_available?
+      end
+
+      def connection_state_available?
+        return true if connected?
+
+        connected_last = state_history.reverse.find { |connected| connected.fetch(:state) == :connected }
+        if connected_last.nil? || (connected_last.fetch(:transitioned_at) < Time.now - connection_state_ttl)
+          false
+        else
+          true
+        end
       end
 
       def connection_recoverable?
@@ -555,7 +583,7 @@ module Ably
       end
 
       def can_use_fallback_hosts?
-        if production? && !custom_port? && !custom_host?
+        if client.fallback_hosts && !client.fallback_hosts.empty?
           if connecting? && previous_state
             use_fallback_if_disconnected? || use_fallback_if_suspended?
           end

data/lib/ably/realtime/connection/connection_manager.rb

@@ -320,7 +320,7 @@ module Ably::Realtime
       def can_reattempt_connect_for_state?(state)
         case state
         when :disconnected
-          time_spent_attempting_state(:disconnected, ignore_states: [:connecting]) < connection.defaults.fetch(:connection_state_ttl)
+          time_spent_attempting_state(:disconnected, ignore_states: [:connecting]) < connection.connection_state_ttl
         when :suspended
           true # suspended state remains indefinitely
         else

data/lib/ably/rest/client.rb

@@ -78,10 +78,14 @@ module Ably
       # @api private
       attr_reader :options
 
+      # The list of fallback hosts to be used by this client
+      # if empty or nil then fallback host functionality is disabled
+      attr_reader :fallback_hosts
+
       # Creates a {Ably::Rest::Client Rest Client} and configures the {Ably::Auth} object for the connection.
       #
       # @param [Hash,String] options an options Hash used to configure the client and the authentication, or String with an API key or Token ID
-      # @option options [Boolean]                 :tls                (true) When fales, TLS is disabled. Please note Basic Auth is disallowed without TLS as secrets cannot be transmitted over unsecured connections.
+      # @option options [Boolean]                 :tls                 (true) When false, TLS is disabled. Please note Basic Auth is disallowed without TLS as secrets cannot be transmitted over unsecured connections.
       # @option options [String]                  :key                 API key comprising the key name and key secret in a single string
       # @option options [String]                  :token               Token string or {Models::TokenDetails} used to authenticate requests
       # @option options [String]                  :token_details       {Models::TokenDetails} used to authenticate requests
@@ -106,6 +110,9 @@ module Ably
       # @option options [Integer]                 :http_max_retry_count    (3) maximum number of fallback host retries for HTTP requests that fail due to network issues or server problems
       # @option options [Integer]                 :http_max_retry_duration (10 seconds) maximum elapsed time in which fallback host retries for HTTP requests will be attempted i.e. if the first default host attempt takes 5s, and then the subsequent fallback retry attempt takes 7s, no further fallback host attempts will be made as the total elapsed time of 12s exceeds the default 10s limit
       #
+      # @option options [Boolean]                 :fallback_hosts_use_default  (false) When true, forces the user of fallback hosts even if a non-default production endpoint is being used
+      # @option options [Array<String>]           :fallback_hosts              When an array of fallback hosts are provided, these fallback hosts are always used if a request fails to the primary endpoint. If an empty array is provided, the fallback host functionality is disabled
+      #
       # @return [Ably::Rest::Client]
       #
       # @example
@@ -129,6 +136,7 @@ module Ably
 
         @tls              = options.delete(:tls) == false ? false : true
         @environment      = options.delete(:environment) # nil is production
+        @environment      = nil if [:production, 'production'].include?(@environment)
         @protocol         = options.delete(:protocol) || :msgpack
         @debug_http       = options.delete(:debug_http)
         @log_level        = options.delete(:log_level) || ::Logger::WARN
@@ -137,6 +145,20 @@ module Ably
         @custom_port      = options.delete(:port)
         @custom_tls_port  = options.delete(:tls_port)
 
+        if options[:fallback_hosts_use_default] && options[:fallback_jhosts]
+          raise ArgumentError, "fallback_hosts_use_default cannot be set to trye when fallback_jhosts is also provided"
+        end
+        @fallback_hosts = case
+        when options.delete(:fallback_hosts_use_default)
+          Ably::FALLBACK_HOSTS
+        when options_fallback_hosts = options.delete(:fallback_hosts)
+          options_fallback_hosts
+        when environment || custom_host || options[:realtime_host] || custom_port || custom_tls_port
+          []
+        else
+          Ably::FALLBACK_HOSTS
+        end
+
         @http_defaults = HTTP_DEFAULTS.dup
         options.each do |key, val|
           if http_key = key[/^http_(.+)/, 1]
@@ -295,7 +317,7 @@ module Ably
       # Connection used to make HTTP requests
       #
       # @param [Hash] options
-      # @option options [Boolean] :use_fallback when true, one of the fallback connections is used randomly, see {Ably::FALLBACK_HOSTS}
+      # @option options [Boolean] :use_fallback when true, one of the fallback connections is used randomly, see the default {Ably::FALLBACK_HOSTS}
       #
       # @return [Faraday::Connection]
       #
@@ -309,14 +331,16 @@ module Ably
       end
 
       # Fallback connection used to make HTTP requests.
-      # Note, each request uses a random and then subsequent random {Ably::FALLBACK_HOSTS fallback host}
+      # Note, each request uses a random and then subsequent random {Ably::FALLBACK_HOSTS fallback hosts}
+      # are used (unless custom fallback hosts are provided with fallback_hosts)
       #
       # @return [Faraday::Connection]
       #
       # @api private
       def fallback_connection
         unless defined?(@fallback_connections) && @fallback_connections
-          @fallback_connections = Ably::FALLBACK_HOSTS.shuffle.map { |host| Faraday.new(endpoint_for_host(host).to_s, connection_options) }
+          @fallback_connections = fallback_hosts.shuffle.map { |host| Faraday.new(endpoint_for_host(host).to_s, connection_options) }
+          @fallback_connections << Faraday.new(endpoint.to_s, connection_options) # Try the original host last if all fallbacks have been used
         end
         @fallback_index ||= 0
 
@@ -325,6 +349,16 @@ module Ably
         end
       end
 
+      # Library Ably version user agent
+      # @api private
+      def lib_version_id
+        @lib_version_id ||= [
+          'ruby',
+          Ably.lib_variant,
+          Ably::VERSION
+        ].compact.join('-')
+      end
+
       private
       def request(method, path, params = {}, options = {})
         options = options.clone
@@ -352,14 +386,13 @@ module Ably
             unless options[:send_auth_header] == false
               request.headers[:authorization] = auth.auth_header
             end
-            request.headers['X-Ably-Version'] = Ably::PROTOCOL_VERSION
-            request.headers['X-Ably-Lib'] = Ably::LIB_VERSION_ID
           end
 
         rescue Faraday::TimeoutError, Faraday::ClientError, Ably::Exceptions::ServerError => error
           time_passed = Time.now - requested_at
           if can_fallback_to_alternate_ably_host? && retry_count < max_retry_count && time_passed <= max_retry_duration
             retry_count += 1
+            logger.warn "Ably::Rest::Client - Retry #{retry_count} for #{method} #{path} #{params} as initial attempt failed: #{error}"
             retry
           end
 
@@ -410,9 +443,11 @@ module Ably
         @connection_options ||= {
           builder: middleware,
           headers: {
-            content_type: mime_type,
-            accept:       mime_type,
-            user_agent:   user_agent
+            content_type:       mime_type,
+            accept:             mime_type,
+            user_agent:         user_agent,
+            'X-Ably-Version' => Ably::PROTOCOL_VERSION,
+            'X-Ably-Lib'     => lib_version_id
           },
           request: {
             open_timeout: http_defaults.fetch(:open_timeout),
@@ -439,7 +474,7 @@ module Ably
       end
 
       def can_fallback_to_alternate_ably_host?
-        !custom_host && !environment
+        fallback_hosts && !fallback_hosts.empty?
       end
 
       def initialize_default_encoders

data/lib/ably/version.rb

@@ -1,5 +1,16 @@
 module Ably
-  VERSION = '0.8.13'
+  VERSION = '0.8.14'
   PROTOCOL_VERSION = '0.8'
-  LIB_VERSION_ID = "ruby-#{Ably::VERSION}"
+
+  # Allow a variant to be configured for all instances of this client library
+  # such as ruby-rest-[VERSION]
+
+  # @api private
+  def self.lib_variant=(variant)
+    @lib_variant = variant
+  end
+
+  def self.lib_variant
+    @lib_variant
+  end
 end

data/spec/acceptance/realtime/connection_failures_spec.rb

@@ -62,7 +62,7 @@ describe Ably::Realtime::Connection, 'failures', :event_machine do
             log_level: :none,
             disconnected_retry_timeout: retry_every_for_tests,
             suspended_retry_timeout:    retry_every_for_tests,
-            connection_state_ttl:       max_time_in_state_for_tests
+            max_connection_state_ttl:   max_time_in_state_for_tests
           )
         end
 
@@ -169,7 +169,7 @@ describe Ably::Realtime::Connection, 'failures', :event_machine do
 
           context 'for the first time' do
             let(:client_options) do
-              default_options.merge(suspended_retry_timeout: 2, connection_state_ttl: 0, log_level: :error)
+              default_options.merge(suspended_retry_timeout: 2, max_connection_state_ttl: 0, log_level: :error)
             end
 
             it 'waits suspended_retry_timeout before attempting to reconnect' do
@@ -306,7 +306,7 @@ describe Ably::Realtime::Connection, 'failures', :event_machine do
                 log_level: :error,
                 disconnected_retry_timeout: 0.1,
                 suspended_retry_timeout:    0.1,
-                connection_state_ttl:       0.2,
+                max_connection_state_ttl:   0.2,
                 realtime_host:              'non.existent.host'
               )
             end
@@ -370,7 +370,7 @@ describe Ably::Realtime::Connection, 'failures', :event_machine do
               log_level: :none,
               disconnected_retry_timeout: retry_every,
               suspended_retry_timeout:    retry_every,
-              connection_state_ttl:       60
+              max_connection_state_ttl:   60
             )
           end
 
@@ -416,7 +416,7 @@ describe Ably::Realtime::Connection, 'failures', :event_machine do
         end
       end
 
-      context 'when websocket transport is closed' do
+      context 'when websocket transport is abruptly disconnected' do
         it 'reconnects automatically' do
           fail_if_suspended_or_failed
 
@@ -431,6 +431,29 @@ describe Ably::Realtime::Connection, 'failures', :event_machine do
             connection.transport.close_connection_after_writing
           end
         end
+
+        context 'hosts used' do
+          it 'reconnects with the default host' do
+            fail_if_suspended_or_failed
+
+            connection.once(:connected) do
+              connection.once(:disconnected) do
+                hosts = []
+                expect(connection).to receive(:create_transport).once.and_wrap_original do |original_method, *args, &block|
+                  hosts << args[0]
+                  original_method.call(*args, &block)
+                end
+                connection.once(:connected) do
+                  host = "#{"#{environment}-" if environment && environment.to_s != 'production'}#{Ably::Realtime::Client::DOMAIN}"
+                  expect(hosts.first).to eql(host)
+                  expect(hosts.length).to eql(1)
+                  stop_reactor
+                end
+              end
+              connection.transport.close_connection_after_writing
+            end
+          end
+        end
       end
 
       context 'after successfully reconnecting and resuming' do
@@ -599,6 +622,32 @@ describe Ably::Realtime::Connection, 'failures', :event_machine do
             end
           end
         end
+
+        context 'as the DISCONNECTED window to resume has passed' do
+          let(:channel) { client.channel(random_str) }
+
+          def kill_connection_transport_and_prevent_valid_resume
+            connection.transport.close_connection_after_writing
+          end
+
+          it 'starts a new connection automatically and does not try and resume' do
+            connection.once(:connected) do
+              previous_connection_id  = connection.id
+              previous_connection_key = connection.key
+
+              five_minutes_time = Time.now + 5 * 60
+              allow(Time).to receive(:now) { five_minutes_time }
+
+              connection.once(:connected) do
+                expect(connection.key).to_not eql(previous_connection_key)
+                expect(connection.id).to_not eql(previous_connection_id)
+                stop_reactor
+              end
+
+              kill_connection_transport_and_prevent_valid_resume
+            end
+          end
+        end
       end
     end
 
@@ -612,7 +661,7 @@ describe Ably::Realtime::Connection, 'failures', :event_machine do
           log_level:                  :none,
           disconnected_retry_timeout: retry_every_for_tests,
           suspended_retry_timeout:    retry_every_for_tests,
-          connection_state_ttl:       max_time_in_state_for_tests
+          max_connection_state_ttl:   max_time_in_state_for_tests
         )
       end
 
@@ -627,7 +676,7 @@ describe Ably::Realtime::Connection, 'failures', :event_machine do
         let(:client_options) { timeout_options.merge(realtime_host: expected_host) }
 
         it 'never uses a fallback host' do
-          expect(EventMachine).to receive(:connect).exactly(retry_count_for_all_states).times do |host|
+          expect(connection).to receive(:create_transport).exactly(retry_count_for_all_states).times do |host|
             expect(host).to eql(expected_host)
             raise EventMachine::ConnectionError
           end
@@ -645,7 +694,7 @@ describe Ably::Realtime::Connection, 'failures', :event_machine do
         let(:client_options) { timeout_options.merge(tls_port: custom_port) }
 
         it 'never uses a fallback host' do
-          expect(EventMachine).to receive(:connect).exactly(retry_count_for_all_states).times do |host, port|
+          expect(connection).to receive(:create_transport).exactly(retry_count_for_all_states).times do |host, port|
             expect(port).to eql(custom_port)
             raise EventMachine::ConnectionError
           end
@@ -663,8 +712,8 @@ describe Ably::Realtime::Connection, 'failures', :event_machine do
         let(:expected_host)  { "#{environment}-#{Ably::Realtime::Client::DOMAIN}" }
         let(:client_options) { timeout_options.merge(environment: environment) }
 
-        it 'never uses a fallback host' do
-          expect(EventMachine).to receive(:connect).exactly(retry_count_for_all_states).times do |host|
+        it 'does not use a fallback host by default' do
+          expect(connection).to receive(:create_transport).exactly(retry_count_for_all_states).times do |host|
             expect(host).to eql(expected_host)
             raise EventMachine::ConnectionError
           end
@@ -675,6 +724,55 @@ describe Ably::Realtime::Connection, 'failures', :event_machine do
             end
           end
         end
+
+        context ':fallback_hosts_use_default is true' do
+          let(:max_time_in_state_for_tests) { 4 }
+          let(:fallback_hosts_used) { Array.new }
+          let(:client_options) { timeout_options.merge(environment: environment, fallback_hosts_use_default: true) }
+
+          it 'uses a fallback host on every subsequent disconnected attempt until suspended (#RTN17b, #TO3k7)' do
+            request = 0
+            allow(connection).to receive(:create_transport) do |host|
+              if request == 0
+                expect(host).to eql(expected_host)
+              else
+                fallback_hosts_used << host
+              end
+              request += 1
+              raise EventMachine::ConnectionError
+            end
+
+            connection.once(:suspended) do
+              expect(fallback_hosts_used.uniq).to match_array(Ably::FALLBACK_HOSTS + [expected_host])
+              stop_reactor
+            end
+          end
+        end
+
+        context ':fallback_hosts array is provided' do
+          let(:max_time_in_state_for_tests) { 4 }
+          let(:fallback_hosts) { %w(a.foo.com b.foo.com) }
+          let(:fallback_hosts_used) { Array.new }
+          let(:client_options) { timeout_options.merge(environment: environment, fallback_hosts: fallback_hosts) }
+
+          it 'uses a fallback host on every subsequent disconnected attempt until suspended (#RTN17b, #TO3k6)' do
+            request = 0
+            allow(connection).to receive(:create_transport) do |host|
+              if request == 0
+                expect(host).to eql(expected_host)
+              else
+                fallback_hosts_used << host
+              end
+              request += 1
+              raise EventMachine::ConnectionError
+            end
+
+            connection.once(:suspended) do
+              expect(fallback_hosts_used.uniq).to match_array(fallback_hosts + [expected_host])
+              stop_reactor
+            end
+          end
+        end
       end
 
       context 'with production environment' do
@@ -694,7 +792,7 @@ describe Ably::Realtime::Connection, 'failures', :event_machine do
           end
 
           it 'never uses a fallback host' do
-            expect(EventMachine).to receive(:connect).exactly(retry_count_for_all_states).times do |host|
+            expect(connection).to receive(:create_transport).exactly(retry_count_for_all_states).times do |host|
               expect(host).to eql(expected_host)
               raise EventMachine::ConnectionError
             end
@@ -713,43 +811,92 @@ describe Ably::Realtime::Connection, 'failures', :event_machine do
             @suspended = 0
           end
 
-          it 'uses a fallback host on every subsequent disconnected attempt until suspended' do
-            request = 0
-            expect(EventMachine).to receive(:connect).exactly(retry_count_for_one_state).times do |host|
-              if request == 0
-                expect(host).to eql(expected_host)
-              else
-                fallback_hosts_used << host
+          context 'and default options' do
+            let(:max_time_in_state_for_tests) { 2 } # allow time for 3 attempts, 2 configured fallbacks + primary host
+
+            it 'uses a fallback host + the original host once on every subsequent disconnected attempt until suspended' do
+              request = 0
+              expect(connection).to receive(:create_transport).exactly(retry_count_for_one_state).times do |host|
+                if request == 0
+                  expect(host).to eql(expected_host)
+                else
+                  fallback_hosts_used << host
+                end
+                request += 1
+                raise EventMachine::ConnectionError
+              end
+
+              connection.once(:suspended) do
+                fallback_hosts_used.pop # remove suspended attempt host
+                expect(fallback_hosts_used.uniq).to match_array(custom_hosts + [expected_host])
+                stop_reactor
               end
-              request += 1
-              raise EventMachine::ConnectionError
             end
 
-            connection.once(:suspended) do
-              fallback_hosts_used.pop # remove suspended attempt host
-              expect(fallback_hosts_used.uniq).to match_array(custom_hosts)
-              stop_reactor
+            it 'uses the primary host when suspended, and then every fallback host and the primary host again on every subsequent suspended attempt' do
+              request = 0
+              expect(connection).to receive(:create_transport).at_least(:once) do |host|
+                if request == 0 || request == expected_retry_attempts + 1
+                  expect(host).to eql(expected_host)
+                else
+                  expect(custom_hosts + [expected_host]).to include(host)
+                  fallback_hosts_used << host if @suspended > 0
+                end
+                request += 1
+                raise EventMachine::ConnectionError
+              end
+
+              connection.on(:suspended) do
+                @suspended += 1
+
+                if @suspended > 4
+                  expect(fallback_hosts_used.uniq).to match_array(custom_hosts + [expected_host])
+                  stop_reactor
+                end
+              end
             end
           end
 
-          it 'uses the primary host when suspended, and a fallback host on every subsequent suspended attempt' do
-            request = 0
-            expect(EventMachine).to receive(:connect).at_least(:once) do |host|
-              if request == 0 || request == expected_retry_attempts + 1
-                expect(host).to eql(expected_host)
-              else
-                expect(custom_hosts).to include(host)
-                fallback_hosts_used << host if @suspended > 0
+          context ':fallback_hosts array is provided by an empty array' do
+            let(:max_time_in_state_for_tests) { 3 }
+            let(:fallback_hosts) { [] }
+            let(:hosts_used) { Array.new }
+            let(:client_options) { timeout_options.merge(environment: 'production', fallback_hosts: fallback_hosts) }
+
+            it 'uses a fallback host on every subsequent disconnected attempt until suspended (#RTN17b, #TO3k6)' do
+              allow(connection).to receive(:create_transport) do |host|
+                hosts_used << host
+                raise EventMachine::ConnectionError
+              end
+
+              connection.once(:suspended) do
+                expect(hosts_used.uniq.length).to eql(1)
+                expect(hosts_used.uniq.first).to eql(expected_host)
+                stop_reactor
               end
-              request += 1
-              raise EventMachine::ConnectionError
             end
+          end
+
+          context ':fallback_hosts array is provided' do
+            let(:max_time_in_state_for_tests) { 3 }
+            let(:fallback_hosts) { %w(a.foo.com b.foo.com) }
+            let(:fallback_hosts_used) { Array.new }
+            let(:client_options) { timeout_options.merge(environment: 'production', fallback_hosts: fallback_hosts) }
 
-            connection.on(:suspended) do
-              @suspended += 1
+            it 'uses a fallback host on every subsequent disconnected attempt until suspended (#RTN17b, #TO3k6)' do
+              request = 0
+              allow(connection).to receive(:create_transport) do |host|
+                if request == 0
+                  expect(host).to eql(expected_host)
+                else
+                  fallback_hosts_used << host
+                end
+                request += 1
+                raise EventMachine::ConnectionError
+              end
 
-              if @suspended > 3
-                expect(fallback_hosts_used.uniq).to match_array(custom_hosts)
+              connection.once(:suspended) do
+                expect(fallback_hosts_used.uniq).to match_array(fallback_hosts + [expected_host])
                 stop_reactor
               end
             end

data/spec/acceptance/realtime/connection_spec.rb

@@ -833,6 +833,22 @@ describe Ably::Realtime::Connection, :event_machine do
           end
         end
       end
+
+      context 'with a different connection_state_ttl' do
+        before do
+          old_defaults = Ably::Realtime::Connection::DEFAULTS
+          stub_const 'Ably::Realtime::Connection::DEFAULTS', old_defaults.merge(connection_state_ttl: 15)
+        end
+
+        it 'updates the private Connection#connection_state_ttl' do
+          expect(connection.connection_state_ttl).to eql(15)
+
+          connection.once(:connected) do
+            expect(connection.connection_state_ttl).to be > 15
+            stop_reactor
+          end
+        end
+      end
     end
 
     context 'recovery' do
@@ -848,14 +864,14 @@ describe Ably::Realtime::Connection, :event_machine do
           log_level:                  :none,
           disconnected_retry_timeout: 0.1,
           suspended_retry_timeout:    0.1,
-          connection_state_ttl:       0.2,
+          max_connection_state_ttl:   0.2,
           realtime_request_timeout:   5
         )
       end
 
       describe '#recovery_key' do
         def self.available_states
-          [:connecting, :connected, :disconnected, :suspended, :failed]
+          [:connecting, :connected, :disconnected]
         end
         let(:available_states) { self.class.available_states }
         let(:states)           { Hash.new }
@@ -968,19 +984,25 @@ describe Ably::Realtime::Connection, :event_machine do
         context 'when messages have been sent whilst the old connection is disconnected' do
           describe 'the new connection' do
             it 'recovers server-side queued messages' do
+              connection_id, recovery_key = nil, nil
+
               channel.attach do
-                connection.transition_state_machine! :failed
+                connection_id = client.connection.id
+                recovery_key = client.connection.recovery_key
+                connection.transport.__incoming_protocol_msgbus__
+                publishing_client_channel.publish('event', 'message') do
+                  connection.transition_state_machine! :failed
+                end
               end
 
               connection.on(:failed) do
-                publishing_client_channel.publish('event', 'message') do
-                  recover_client = auto_close Ably::Realtime::Client.new(default_options.merge(recover: client.connection.recovery_key))
-                  recover_client_channel = recover_client.channel(channel_name)
-                  recover_client_channel.attach do
-                    recover_client_channel.subscribe('event') do |message|
-                      expect(message.data).to eql('message')
-                      stop_reactor
-                    end
+                recover_client = auto_close Ably::Realtime::Client.new(default_options.merge(recover: recovery_key))
+                recover_client_channel = recover_client.channel(channel_name)
+                recover_client_channel.attach do
+                  expect(recover_client.connection.id).to eql(connection_id)
+                  recover_client_channel.subscribe('event') do |message|
+                    expect(message.data).to eql('message')
+                    stop_reactor
                   end
                 end
               end
@@ -1212,7 +1234,7 @@ describe Ably::Realtime::Connection, :event_machine do
             log_level:                  :fatal,
             disconnected_retry_timeout: 0.02,
             suspended_retry_timeout:    60,
-            connection_state_ttl:       0.05
+            max_connection_state_ttl:   0.05
           )
         end
 
@@ -1375,6 +1397,27 @@ describe Ably::Realtime::Connection, :event_machine do
         end
         client
       end
+
+      context 'with variant' do
+        let(:variant) { 'foo ' }
+
+        before do
+          Ably.lib_variant = variant
+        end
+
+        after do
+          Ably.lib_variant = nil
+        end
+
+        it 'sends the lib version param lib with the variant' do
+          expect(EventMachine).to receive(:connect) do |host, port, transport, object, url|
+            uri = URI.parse(url)
+            expect(CGI::parse(uri.query)['lib'][0]).to eql("ruby-#{variant}-#{Ably::VERSION}")
+            stop_reactor
+          end
+          client
+        end
+      end
     end
   end
 end

data/spec/acceptance/rest/client_spec.rb

@@ -1,5 +1,6 @@
 # encoding: utf-8
 require 'spec_helper'
+require 'webrick'
 
 describe Ably::Rest::Client do
   vary_by_protocol do
@@ -214,7 +215,7 @@ describe Ably::Rest::Client do
 
     context 'connection transport' do
       context 'defaults' do
-        let(:client_options) { default_options.merge(key: api_key) }
+        let(:client_options) { default_options.merge(key: api_key, environment: 'production') }
 
         context 'for default host' do
           it "is configured to timeout connection opening in #{http_defaults.fetch(:open_timeout)} seconds" do
@@ -240,7 +241,7 @@ describe Ably::Rest::Client do
       context 'with custom http_open_timeout and http_request_timeout options' do
         let(:http_open_timeout)    { 999 }
         let(:http_request_timeout) { 666 }
-        let(:client_options)       { default_options.merge(key: api_key, http_open_timeout: http_open_timeout, http_request_timeout: http_request_timeout) }
+        let(:client_options)       { default_options.merge(key: api_key, http_open_timeout: http_open_timeout, http_request_timeout: http_request_timeout, environment: 'production') }
 
         context 'for default host' do
           it 'is configured to use custom open timeout' do
@@ -269,7 +270,7 @@ describe Ably::Rest::Client do
       let(:publish_block)  { proc { client.channel('test').publish('event', 'data') } }
 
       context 'configured' do
-        let(:client_options) { default_options.merge(key: api_key) }
+        let(:client_options) { default_options.merge(key: api_key, environment: 'production') }
 
         it 'should make connection attempts to A.ably-realtime.com, B.ably-realtime.com, C.ably-realtime.com, D.ably-realtime.com, E.ably-realtime.com' do
           hosts = []
@@ -411,6 +412,279 @@ describe Ably::Rest::Client do
           end
         end
       end
+
+      context 'when environment is production and server returns a 50x error' do
+        let(:custom_hosts)       { %w(A.foo.com B.foo.com) }
+        let(:max_retry_count)    { 2 }
+        let(:max_retry_duration) { 0.5 }
+        let(:fallback_block)     { Proc.new { raise Faraday::SSLError.new('ssl error message') } }
+        let(:production_options) do
+          default_options.merge(
+            environment: nil,
+            key: api_key,
+            http_max_retry_duration: max_retry_duration,
+            http_max_retry_count: max_retry_count
+          )
+        end
+
+        let(:status) { 502 }
+        let(:fallback_block) do
+          Proc.new do
+            {
+              headers: { 'Content-Type' => 'text/html' },
+              status: status
+            }
+          end
+        end
+        let!(:default_host_request_stub) do
+          stub_request(:post, "https://#{api_key}@#{Ably::Rest::Client::DOMAIN}#{path}").to_return(&fallback_block)
+        end
+
+        context 'with custom fallback hosts provided' do
+          let!(:first_fallback_request_stub) do
+            stub_request(:post, "https://#{api_key}@#{custom_hosts[0]}#{path}").to_return(&fallback_block)
+          end
+
+          let!(:second_fallback_request_stub) do
+            stub_request(:post, "https://#{api_key}@#{custom_hosts[1]}#{path}").to_return(&fallback_block)
+          end
+
+          let(:client_options) {
+            production_options.merge(fallback_hosts: custom_hosts)
+          }
+
+          it 'attempts the fallback hosts as this is an authentication failure (#RSC15b, #TO3k6)' do
+            expect { publish_block.call }.to raise_error(Ably::Exceptions::ServerError)
+            expect(default_host_request_stub).to have_been_requested
+            expect(first_fallback_request_stub).to have_been_requested
+            expect(second_fallback_request_stub).to have_been_requested
+          end
+        end
+
+        context 'with an empty array of fallback hosts provided (#RSC15b, #TO3k6)' do
+          let(:client_options) {
+            production_options.merge(fallback_hosts: [])
+          }
+
+          it 'does not attempt the fallback hosts as this is an authentication failure' do
+            expect { publish_block.call }.to raise_error(Ably::Exceptions::ServerError)
+            expect(default_host_request_stub).to have_been_requested
+          end
+        end
+
+        context 'using a local web-server', webmock: false do
+          let(:primary_host) { 'local-rest.ably.io' }
+          let(:fallbacks) { ['local.ably.io', 'localhost'] }
+          let(:port) { rand(10000) + 2000 }
+          let(:channel_name) { 'foo' }
+          let(:request_timeout) { 3 }
+
+          after do
+            @web_server.shutdown
+          end
+
+          context 'and timing out the primary host' do
+            before do
+              @web_server = WEBrick::HTTPServer.new(:Port => port, :SSLEnable => false)
+              @web_server.mount_proc "/channels/#{channel_name}/publish" do |req, res|
+                if req.header["host"].first.include?(primary_host)
+                  @primary_host_requested = true
+                  sleep request_timeout + 0.5
+                else
+                  @fallback_request_count ||= 0
+                  @fallback_request_count += 1
+                  if @fallback_request_count <= fail_fallback_request_count
+                    sleep request_timeout + 0.5
+                  else
+                    res.status = 200
+                    res['Content-Type'] = 'application/json'
+                    res.body = '{}'
+                  end
+                end
+              end
+              Thread.new do
+                @web_server.start
+              end
+            end
+
+            context 'with request timeout less than max_retry_duration' do
+              let(:client_options) do
+                default_options.merge(
+                  rest_host: primary_host,
+                  fallback_hosts: fallbacks,
+                  token: 'fake.token',
+                  port: port,
+                  tls: false,
+                  http_request_timeout: request_timeout,
+                  max_retry_duration: request_timeout * 3
+                )
+              end
+              let(:fail_fallback_request_count) { 1 }
+
+              it 'tries one of the fallback hosts' do
+                client.channel(channel_name).publish('event', 'data')
+                expect(@primary_host_requested).to be_truthy
+                expect(@fallback_request_count).to eql(2)
+              end
+            end
+
+            context 'with request timeout less than max_retry_duration' do
+              let(:client_options) do
+                default_options.merge(
+                  rest_host: primary_host,
+                  fallback_hosts: fallbacks,
+                  token: 'fake.token',
+                  port: port,
+                  tls: false,
+                  http_request_timeout: request_timeout,
+                  max_retry_duration: request_timeout / 2
+                )
+              end
+              let(:fail_fallback_request_count) { 0 }
+
+              it 'tries one of the fallback hosts' do
+                client.channel(channel_name).publish('event', 'data')
+                expect(@primary_host_requested).to be_truthy
+                expect(@fallback_request_count).to eql(1)
+              end
+            end
+          end
+
+          context 'and failing the primary host' do
+            before do
+              @web_server = WEBrick::HTTPServer.new(:Port => port, :SSLEnable => false)
+              @web_server.mount_proc "/channels/#{channel_name}/publish" do |req, res|
+                if req.header["host"].first.include?(primary_host)
+                  @primary_host_requested = true
+                  res.status = 500
+                else
+                  @fallback_request_count ||= 0
+                  @fallback_request_count += 1
+                  if @fallback_request_count <= fail_fallback_request_count
+                    res.status = 500
+                  else
+                    res.status = 200
+                    res['Content-Type'] = 'application/json'
+                    res.body = '{}'
+                  end
+                end
+              end
+              Thread.new do
+                @web_server.start
+              end
+            end
+
+            let(:client_options) do
+              default_options.merge(
+                rest_host: primary_host,
+                fallback_hosts: fallbacks,
+                token: 'fake.token',
+                port: port,
+                tls: false
+              )
+            end
+            let(:fail_fallback_request_count) { 1 }
+
+            it 'tries one of the fallback hosts' do
+              client.channel(channel_name).publish('event', 'data')
+              expect(@primary_host_requested).to be_truthy
+              expect(@fallback_request_count).to eql(2)
+            end
+          end
+        end
+      end
+
+      context 'when environment is not production and server returns a 50x error' do
+        let(:custom_hosts)       { %w(A.foo.com B.foo.com) }
+        let(:max_retry_count)    { 2 }
+        let(:max_retry_duration) { 0.5 }
+        let(:fallback_block)     { Proc.new { raise Faraday::SSLError.new('ssl error message') } }
+        let(:env)                { 'custom-env' }
+        let(:production_options) do
+          default_options.merge(
+            environment: env,
+            key: api_key,
+            http_max_retry_duration: max_retry_duration,
+            http_max_retry_count: max_retry_count
+          )
+        end
+
+        let(:status) { 502 }
+        let(:fallback_block) do
+          Proc.new do
+            {
+              headers: { 'Content-Type' => 'text/html' },
+              status: status
+            }
+          end
+        end
+        let!(:default_host_request_stub) do
+          stub_request(:post, "https://#{api_key}@#{env}-#{Ably::Rest::Client::DOMAIN}#{path}").to_return(&fallback_block)
+        end
+
+        context 'with custom fallback hosts provided (#RSC15b, #TO3k6)' do
+          let!(:first_fallback_request_stub) do
+            stub_request(:post, "https://#{api_key}@#{custom_hosts[0]}#{path}").to_return(&fallback_block)
+          end
+
+          let!(:second_fallback_request_stub) do
+            stub_request(:post, "https://#{api_key}@#{custom_hosts[1]}#{path}").to_return(&fallback_block)
+          end
+
+          let(:client_options) {
+            production_options.merge(fallback_hosts: custom_hosts)
+          }
+
+          it 'attempts the fallback hosts as this is an authentication failure' do
+            expect { publish_block.call }.to raise_error(Ably::Exceptions::ServerError)
+            expect(default_host_request_stub).to have_been_requested
+            expect(first_fallback_request_stub).to have_been_requested
+            expect(second_fallback_request_stub).to have_been_requested
+          end
+        end
+
+        context 'with an empty array of fallback hosts provided (#RSC15b, #TO3k6)' do
+          let(:client_options) {
+            production_options.merge(fallback_hosts: [])
+          }
+
+          it 'does not attempt the fallback hosts as this is an authentication failure' do
+            expect { publish_block.call }.to raise_error(Ably::Exceptions::ServerError)
+            expect(default_host_request_stub).to have_been_requested
+          end
+        end
+
+        context 'with fallback_hosts_use_default: true (#RSC15b, #TO3k7)' do
+          let(:custom_hosts) { Ably::FALLBACK_HOSTS[0...2] }
+
+          before do
+            stub_const 'Ably::FALLBACK_HOSTS', custom_hosts
+          end
+
+          let(:client_options) {
+            production_options.merge(fallback_hosts_use_default: true)
+          }
+
+          let!(:first_fallback_request_stub) do
+            stub_request(:post, "https://#{api_key}@#{Ably::FALLBACK_HOSTS[0]}#{path}").to_return(&fallback_block)
+          end
+
+          let!(:second_fallback_request_stub) do
+            stub_request(:post, "https://#{api_key}@#{Ably::FALLBACK_HOSTS[1]}#{path}").to_return(&fallback_block)
+          end
+
+          let(:client_options) {
+            production_options.merge(fallback_hosts: custom_hosts)
+          }
+
+          it 'attempts the default fallback hosts as this is an authentication failure' do
+            expect { publish_block.call }.to raise_error(Ably::Exceptions::ServerError)
+            expect(default_host_request_stub).to have_been_requested
+            expect(first_fallback_request_stub).to have_been_requested
+            expect(second_fallback_request_stub).to have_been_requested
+          end
+        end
+      end
     end
 
     context 'with a custom host' do
@@ -547,19 +821,38 @@ describe Ably::Rest::Client do
     end
 
     context 'version headers', :webmock do
-      let(:client_options) { default_options.merge(key: api_key) }
-      let!(:publish_message_stub) do
-        stub_request(:post, "#{client.endpoint.to_s.gsub('://', "://#{api_key}@")}/channels/foo/publish").
-          with(headers: {
-            'X-Ably-Version' => Ably::PROTOCOL_VERSION,
-            'X-Ably-Lib' => "ruby-#{Ably::VERSION}"
-          }).
-          to_return(status: 201, body: '{}', headers: { 'Content-Type' => 'application/json' })
-      end
+      [nil, 'foo'].each do |variant|
+        context "with variant #{variant ? variant : 'none'}" do
+          if variant
+            before do
+              Ably.lib_variant = variant
+            end
 
-      it 'sends a protocol version and lib version header' do
-        client.channels.get('foo').publish("event")
-        expect(publish_message_stub).to have_been_requested
+            after do
+              Ably.lib_variant = nil
+            end
+          end
+
+          let(:client_options) { default_options.merge(key: api_key) }
+          let!(:publish_message_stub) do
+            lib = ['ruby']
+            lib << variant if variant
+            lib << Ably::VERSION
+
+
+            stub_request(:post, "#{client.endpoint.to_s.gsub('://', "://#{api_key}@")}/channels/foo/publish").
+              with(headers: {
+                'X-Ably-Version' => Ably::PROTOCOL_VERSION,
+                'X-Ably-Lib' => lib.join('-')
+              }).
+              to_return(status: 201, body: '{}', headers: { 'Content-Type' => 'application/json' })
+          end
+
+          it 'sends a protocol version and lib version header' do
+            client.channels.get('foo').publish("event")
+            expect(publish_message_stub).to have_been_requested
+          end
+        end
       end
     end
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ably
 version: !ruby/object:Gem::Version
-  version: 0.8.13
+  version: 0.8.14
 platform: ruby
 authors:
 - Lewis Marshall
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-09-29 00:00:00.000000000 Z
+date: 2016-09-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: eventmachine