right_agent 2.1.5-x86-mingw32 → 2.2.0-x86-mingw32

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -98,6 +98,8 @@ module RightScale
98
98
  #
99
99
  # === Parameters
100
100
  # new_tags(String, Array):: Tag or tags to be added
101
+ # options(Hash):: Request options
102
+ # :timeout(Integer):: timeout in seconds before giving up and yielding an error message
101
103
  #
102
104
  # === Block
103
105
  # A block is optional. If provided, should take one argument which will be set with the
@@ -105,15 +107,17 @@ module RightScale
105
107
  #
106
108
  # === Return
107
109
  # true always return true
108
- def add_tags(new_tags)
110
+ def add_tags(new_tags, options = {})
109
111
  new_tags = ensure_flat_array_value(new_tags) unless new_tags.nil? || new_tags.empty?
110
- do_update(new_tags, []) { |raw_response| yield raw_response if block_given? }
112
+ do_update(new_tags, [], options) { |raw_response| yield raw_response if block_given? }
111
113
  end
112
114
 
113
115
  # Remove given tags from agent
114
116
  #
115
117
  # === Parameters
116
118
  # old_tags(String, Array):: Tag or tags to be removed
119
+ # options(Hash):: Request options
120
+ # :timeout(Integer):: timeout in seconds before giving up and yielding an error message
117
121
  #
118
122
  # === Block
119
123
  # A block is optional. If provided, should take one argument which will be set with the
@@ -121,20 +125,24 @@ module RightScale
121
125
  #
122
126
  # === Return
123
127
  # true always return true
124
- def remove_tags(old_tags)
128
+ def remove_tags(old_tags, options = {})
125
129
  old_tags = ensure_flat_array_value(old_tags) unless old_tags.nil? || old_tags.empty?
126
- do_update([], old_tags) { |raw_response| yield raw_response if block_given? }
130
+ do_update([], old_tags, options) { |raw_response| yield raw_response if block_given? }
127
131
  end
128
132
 
129
133
  # Clear all agent tags
130
134
  #
135
+ # === Parameters
136
+ # options(Hash):: Request options
137
+ # :timeout(Integer):: timeout in seconds before giving up and yielding an error message
138
+ #
131
139
  # === Block
132
140
  # Given block should take one argument which will be set with the raw response
133
141
  #
134
142
  # === Return
135
143
  # true::Always return true
136
- def clear
137
- do_update([], @agent.tags) { |raw_response| yield raw_response }
144
+ def clear(options = {})
145
+ do_update([], @agent.tags, options) { |raw_response| yield raw_response }
138
146
  end
139
147
 
140
148
  private
@@ -197,7 +205,7 @@ module RightScale
197
205
  #
198
206
  # === Return
199
207
  # true:: Always return true
200
- def do_update(new_tags, old_tags, &block)
208
+ def do_update(new_tags, old_tags, options = {}, &block)
201
209
  agent_check
202
210
  raise ArgumentError.new("Cannot add and remove tags in same update") if new_tags.any? && old_tags.any?
203
211
  tags = @agent.tags
@@ -206,9 +214,9 @@ module RightScale
206
214
  tags.uniq!
207
215
 
208
216
  if new_tags.any?
209
- request = RightScale::RetryableRequest.new("/router/add_tags", {:tags => new_tags})
217
+ request = RightScale::RetryableRequest.new("/router/add_tags", {:tags => new_tags}, options)
210
218
  elsif old_tags.any?
211
- request = RightScale::RetryableRequest.new("/router/delete_tags", {:tags => old_tags})
219
+ request = RightScale::RetryableRequest.new("/router/delete_tags", {:tags => old_tags}, options)
212
220
  else
213
221
  return
214
222
  end
@@ -109,8 +109,11 @@ module RightScale
109
109
  # Packet::GLOBAL, ones with no shard id
110
110
  # [Symbol] :selector for picking from qualified targets: :any or :all;
111
111
  # defaults to :any
112
- # @param [String, NilClass] token uniquely identifying this request;
113
- # defaults to randomly generated ID
112
+ #
113
+ # @option options [String] :request_uuid uniquely identifying this request; defaults to
114
+ # randomly generated
115
+ # @option options [Numeric] :time_to_live seconds before request expires and is to be ignored;
116
+ # non-positive value or nil means never expire
114
117
  #
115
118
  # @return [NilClass] always nil since there is no expected response to the request
116
119
  #
@@ -120,8 +123,8 @@ module RightScale
120
123
  # @raise [Exceptions::RetryableError] request failed but if retried may succeed
121
124
  # @raise [Exceptions::Terminating] closing client and terminating service
122
125
  # @raise [Exceptions::InternalServerError] internal error in server being accessed
123
- def push(type, payload, target, token = nil)
124
- map_request(type, payload, token)
126
+ def push(type, payload, target, options = {})
127
+ map_request(type, payload, options)
125
128
  end
126
129
 
127
130
  # Route a request to a single target with a response expected
@@ -140,8 +143,11 @@ module RightScale
140
143
  # [Array] :tags that must all be associated with a target for it to be selected
141
144
  # [Hash] :scope for restricting routing which may contain:
142
145
  # [Integer] :account id that agents must be associated with to be included
143
- # @param [String, NilClass] token uniquely identifying this request;
144
- # defaults to randomly generated ID
146
+ #
147
+ # @option options [String] :request_uuid uniquely identifying this request; defaults to
148
+ # randomly generated
149
+ # @option options [Numeric] :time_to_live seconds before request expires and is to be ignored;
150
+ # non-positive value or nil means never expire
145
151
  #
146
152
  # @return [Result, NilClass] response from request
147
153
  #
@@ -151,8 +157,8 @@ module RightScale
151
157
  # @raise [Exceptions::RetryableError] request failed but if retried may succeed
152
158
  # @raise [Exceptions::Terminating] closing client and terminating service
153
159
  # @raise [Exceptions::InternalServerError] internal error in server being accessed
154
- def request(type, payload, target, token = nil)
155
- map_request(type, payload, token)
160
+ def request(type, payload, target, options = {})
161
+ map_request(type, payload, options)
156
162
  end
157
163
 
158
164
  # Determine whether request supported by this client
@@ -170,8 +176,9 @@ module RightScale
170
176
  #
171
177
  # @param [String] type of request as path specifying actor and action
172
178
  # @param [Hash, NilClass] payload for request
173
- # @param [String, NilClass] token uniquely identifying this request;
174
- # defaults to randomly generated ID
179
+ #
180
+ # @option options [String] :request_uuid uniquely identifying this request
181
+ # @option options [Numeric] :time_to_live seconds before request expires and is to be ignored
175
182
  #
176
183
  # @return [Object, NilClass] response from request
177
184
  #
@@ -181,15 +188,15 @@ module RightScale
181
188
  # @raise [Exceptions::RetryableError] request failed but if retried may succeed
182
189
  # @raise [Exceptions::Terminating] closing client and terminating service
183
190
  # @raise [Exceptions::InternalServerError] internal error in server being accessed
184
- def map_request(type, payload, token)
191
+ def map_request(type, payload, options)
185
192
  verb, path = API_MAP[type]
186
193
  raise ArgumentError, "Unsupported request type: #{type}" if path.nil?
187
194
  actor, action = type.split("/")[1..-1]
188
- path, params, options = parameterize(actor, action, payload, path)
195
+ path, params, request_options = parameterize(actor, action, payload, path)
189
196
  if action == "query_tags"
190
- map_query_tags(verb, params, action, token, options)
197
+ map_query_tags(verb, params, action, options.merge(request_options))
191
198
  else
192
- map_response(make_request(verb, path, params, action, token, options), path)
199
+ map_response(make_request(verb, path, params, action, options.merge(request_options)), path)
193
200
  end
194
201
  end
195
202
 
@@ -227,16 +234,14 @@ module RightScale
227
234
  # @param [Symbol] verb for HTTP REST request
228
235
  # @param [Hash] params for HTTP request
229
236
  # @param [String] action from request type
230
- # @param [String, NilClass] token uniquely identifying this request;
231
- # defaults to randomly generated ID
232
237
  # @param [Hash] options augmenting or overriding default options for HTTP request
233
238
  #
234
239
  # @return [Hash] tags retrieved with resource href as key and tags array as value
235
- def map_query_tags(verb, params, action, token, options)
240
+ def map_query_tags(verb, params, action, options)
236
241
  response = {}
237
242
  hrefs = params[:resource_hrefs] || []
238
- hrefs.concat(query_by_tag(verb, params[:tags], action, token, options)) if params[:tags]
239
- response = query_by_resource(verb, hrefs, action, token, options) if hrefs.any?
243
+ hrefs.concat(query_by_tag(verb, params[:tags], action, options)) if params[:tags]
244
+ response = query_by_resource(verb, hrefs, action, options) if hrefs.any?
240
245
  response
241
246
  end
242
247
 
@@ -245,15 +250,13 @@ module RightScale
245
250
  # @param [Symbol] verb for HTTP REST request
246
251
  # @param [Array] tags that all resources retrieved must have
247
252
  # @param [String] action from request type
248
- # @param [String, NilClass] token uniquely identifying this request;
249
- # defaults to randomly generated ID
250
253
  # @param [Hash] options augmenting or overriding default options for HTTP request
251
254
  #
252
255
  # @return [Array] resource hrefs
253
- def query_by_tag(verb, tags, action, token, options)
256
+ def query_by_tag(verb, tags, action, options)
254
257
  path = "/tags/by_tag"
255
258
  params = {:tags => tags, :match_all => false, :resource_type => "instances"}
256
- map_response(make_request(verb, path, params, action, token, options), path).keys
259
+ map_response(make_request(verb, path, params, action, options), path).keys
257
260
  end
258
261
 
259
262
  # Query API for tags associated with a set of resources
@@ -261,15 +264,13 @@ module RightScale
261
264
  # @param [Symbol] verb for HTTP REST request
262
265
  # @param [Array] hrefs for resources whose tags are to be retrieved
263
266
  # @param [String] action from request type
264
- # @param [String, NilClass] token uniquely identifying this request;
265
- # defaults to randomly generated ID
266
267
  # @param [Hash] options augmenting or overriding default options for HTTP request
267
268
  #
268
269
  # @return [Hash] tags retrieved with resource href as key and tags array as value
269
- def query_by_resource(verb, hrefs, action, token, options)
270
+ def query_by_resource(verb, hrefs, action, options)
270
271
  path = "/tags/by_resource"
271
272
  params = {:resource_hrefs => hrefs}
272
- map_response(make_request(verb, path, params, action, token, options), path)
273
+ map_response(make_request(verb, path, params, action, options), path)
273
274
  end
274
275
 
275
276
  # Convert payload to HTTP parameters
@@ -37,9 +37,9 @@ module RightScale
37
37
  class NotResponding < Exceptions::NestedException; end
38
38
 
39
39
  # HTTP status codes for which a retry is warranted, which is limited to when server
40
- # is not accessible for some reason (502, 503) or server response indicates that
40
+ # is not accessible for some reason (408, 502, 503) or server response indicates that
41
41
  # the request could not be routed for some retryable reason (504)
42
- RETRY_STATUS_CODES = [502, 503, 504]
42
+ RETRY_STATUS_CODES = [408, 502, 503, 504]
43
43
 
44
44
  # Default time for HTTP connection to open
45
45
  DEFAULT_OPEN_TIMEOUT = 2
@@ -183,6 +183,16 @@ module RightScale
183
183
  raise
184
184
  end
185
185
 
186
+ # Close all persistent connections
187
+ #
188
+ # @param [String] reason for closing
189
+ #
190
+ # @return [TrueClass] always true
191
+ def close(reason)
192
+ @http_client.close(reason) if @http_client
193
+ true
194
+ end
195
+
186
196
  protected
187
197
 
188
198
  # Construct headers for request
@@ -244,9 +254,15 @@ module RightScale
244
254
  return [result, code, body, headers] if (Time.now - started_at) >= request_timeout
245
255
  end
246
256
  if result.nil? && (connection = @http_client.connections[path]) && Time.now < connection[:expires_at]
247
- # Continue to poll using same connection until get result, timeout, or hit error
248
- used[:host] = connection[:host]
249
- result, code, body, headers = @http_client.poll(connection, request_options, started_at + request_timeout)
257
+ begin
258
+ # Continue to poll using same connection until get result, timeout, or hit error
259
+ used[:host] = connection[:host]
260
+ result, code, body, headers = @http_client.poll(connection, request_options, started_at + request_timeout)
261
+ rescue HttpException, RestClient::Exception => e
262
+ raise NotResponding.new(e.http_body, e) if RETRY_STATUS_CODES.include?(e.http_code)
263
+ raise NotResponding.new("Request timeout", e) if e.is_a?(RestClient::RequestTimeout)
264
+ raise
265
+ end
250
266
  end
251
267
  [result, code, body, headers]
252
268
  end
@@ -280,6 +296,10 @@ module RightScale
280
296
  else
281
297
  raise NotResponding.new("#{server_name} not responding", e)
282
298
  end
299
+ elsif e.is_a?(RestClient::RequestTimeout)
300
+ # Special case RequestTimeout because http_code is typically nil given no actual response
301
+ yield(e)
302
+ raise NotResponding.new("Request timeout", e)
283
303
  else
284
304
  yield(e)
285
305
  raise e
@@ -147,6 +147,7 @@ module RightScale
147
147
  @reconnect_timer.cancel if @reconnect_timer
148
148
  @reconnect_timer = nil
149
149
  end
150
+ close_http_client("terminating")
150
151
  true
151
152
  end
152
153
 
@@ -223,23 +224,36 @@ module RightScale
223
224
  end
224
225
 
225
226
  # Create HTTP client
227
+ # If there is an existing client, close it first
226
228
  #
227
229
  # @return [TrueClass] always true
228
230
  #
229
231
  # @return [BalancedHttpClient] client
230
232
  def create_http_client
233
+ close_http_client("reconnecting")
231
234
  url = @auth_client.send(@type.to_s + "_url")
232
235
  Log.info("Connecting to #{@options[:server_name]} via #{url.inspect}")
233
- options = {
234
- :server_name => @options[:server_name],
235
- :open_timeout => @options[:open_timeout],
236
- :request_timeout => @options[:request_timeout], }
236
+ options = {:server_name => @options[:server_name]}
237
237
  options[:api_version] = @options[:api_version] if @options[:api_version]
238
238
  options[:non_blocking] = @options[:non_blocking] if @options[:non_blocking]
239
239
  options[:filter_params] = @options[:filter_params] if @options[:filter_params]
240
240
  @http_client = RightScale::BalancedHttpClient.new(url, options)
241
241
  end
242
242
 
243
+ # Close HTTP client persistent connections
244
+ #
245
+ # @param [String] reason for closing
246
+ #
247
+ # @return [Boolean] false if failed, otherwise true
248
+ def close_http_client(reason)
249
+ @http_client.close(reason) if @http_client
250
+ true
251
+ rescue Exception => e
252
+ Log.error("Failed closing connection", e, :trace)
253
+ @stats["exceptions"].track("status", e)
254
+ false
255
+ end
256
+
243
257
  # Perform any other steps needed to make this client fully usable
244
258
  # once HTTP client has been created and server known to be accessible
245
259
  #
@@ -303,6 +317,7 @@ module RightScale
303
317
  # - Underlying BalancedHttpClient connection open timeout (:open_timeout)
304
318
  # - Underlying BalancedHttpClient request timeout (:request_timeout)
305
319
  # - Retry timeout for this method and its handlers (:retry_timeout)
320
+ # - Seconds before request expires and is to be ignored (:time_to_live)
306
321
  # and if the target server is a RightNet router:
307
322
  # - Router response timeout (ideally > :retry_timeout and < :request_timeout)
308
323
  # - Router retry timeout (ideally = :retry_timeout)
@@ -325,6 +340,8 @@ module RightScale
325
340
  # @param [String] type of request for use in logging; defaults to path
326
341
  # @param [String, NilClass] request_uuid uniquely identifying this request;
327
342
  # defaults to randomly generated UUID
343
+ # @param [Numeric, NilClass] time_to_live seconds before request expires and is to be ignored;
344
+ # non-positive value or nil means never expire; defaults to nil
328
345
  # @param [Hash] options augmenting or overriding default options for HTTP request
329
346
  #
330
347
  # @return [Object, NilClass] result of request with nil meaning no result
@@ -335,10 +352,13 @@ module RightScale
335
352
  # @raise [Exceptions::RetryableError] request failed but if retried may succeed
336
353
  # @raise [Exceptions::Terminating] closing client and terminating service
337
354
  # @raise [Exceptions::InternalServerError] internal error in server being accessed
338
- def make_request(verb, path, params = {}, type = nil, request_uuid = nil, options = {})
355
+ def make_request(verb, path, params = {}, type = nil, options = {})
339
356
  raise Exceptions::Terminating if state == :closed
340
- request_uuid ||= RightSupport::Data::UUID.generate
341
357
  started_at = Time.now
358
+ time_to_live = (options[:time_to_live] && options[:time_to_live] > 0) ? options[:time_to_live] : nil
359
+ expires_at = started_at + [time_to_live || @options[:retry_timeout], @options[:retry_timeout]].min
360
+ headers = time_to_live ? @auth_client.headers.merge("X-Expires-At" => started_at + time_to_live) : @auth_client.headers
361
+ request_uuid = options[:request_uuid] || RightSupport::Data::UUID.generate
342
362
  attempts = 0
343
363
  result = nil
344
364
  @stats["requests sent"].measure(type || path, request_uuid) do
@@ -348,11 +368,11 @@ module RightScale
348
368
  :open_timeout => @options[:open_timeout],
349
369
  :request_timeout => @options[:request_timeout],
350
370
  :request_uuid => request_uuid,
351
- :headers => @auth_client.headers }
371
+ :headers => headers }
352
372
  raise Exceptions::ConnectivityFailure, "#{@type} client not connected" unless [:connected, :closing].include?(state)
353
373
  result = @http_client.send(verb, path, params, http_options.merge(options))
354
374
  rescue StandardError => e
355
- request_uuid = handle_exception(e, type || path, request_uuid, started_at, attempts)
375
+ request_uuid = handle_exception(e, type || path, request_uuid, expires_at, attempts)
356
376
  request_uuid ? retry : raise
357
377
  end
358
378
  end
@@ -366,17 +386,17 @@ module RightScale
366
386
  # @param [String] action from request type
367
387
  # @param [String] type of request for use in logging
368
388
  # @param [String] request_uuid originally created for this request
369
- # @param [Time] started_at time for request
389
+ # @param [Time] expires_at time for request
370
390
  # @param [Integer] attempts to make request
371
391
  #
372
- # @return [String, NilClass] request token to be used on retry or nil if to raise instead
392
+ # @return [String, NilClass] request UUID to be used on retry or nil if to raise instead
373
393
  #
374
394
  # @raise [Exceptions::Unauthorized] authorization failed
375
395
  # @raise [Exceptions::ConnectivityFailure] cannot connect to server, lost connection
376
396
  # to it, or it is out of service or too busy to respond
377
397
  # @raise [Exceptions::RetryableError] request failed but if retried may succeed
378
398
  # @raise [Exceptions::InternalServerError] internal error in server being accessed
379
- def handle_exception(exception, type, request_uuid, started_at, attempts)
399
+ def handle_exception(exception, type, request_uuid, expires_at, attempts)
380
400
  result = request_uuid
381
401
  if exception.respond_to?(:http_code)
382
402
  case exception.http_code
@@ -388,7 +408,7 @@ module RightScale
388
408
  @auth_client.expired
389
409
  raise Exceptions::RetryableError.new("Authorization expired", exception)
390
410
  when 449 # RetryWith
391
- result = handle_retry_with(exception, type, request_uuid, started_at, attempts)
411
+ result = handle_retry_with(exception, type, request_uuid, expires_at, attempts)
392
412
  when 500 # InternalServerError
393
413
  raise Exceptions::InternalServerError.new(exception.http_body, @options[:server_name])
394
414
  else
@@ -396,7 +416,7 @@ module RightScale
396
416
  result = nil
397
417
  end
398
418
  elsif exception.is_a?(BalancedHttpClient::NotResponding)
399
- handle_not_responding(exception, type, request_uuid, started_at, attempts)
419
+ handle_not_responding(exception, type, request_uuid, expires_at, attempts)
400
420
  else
401
421
  @stats["request failures"].update("#{type} - #{exception.class.name}")
402
422
  result = nil
@@ -429,7 +449,7 @@ module RightScale
429
449
  true
430
450
  end
431
451
 
432
- # Handle retry response by retrying it once
452
+ # Handle retry response by retrying it only once
433
453
  # This indicates the request was received but a retryable error prevented
434
454
  # it from being processed; the retry responsibility may be passed on
435
455
  # If retrying, this function does not return until it is time to retry
@@ -437,26 +457,24 @@ module RightScale
437
457
  # @param [RestClient::RetryWith] retry_result exception raised
438
458
  # @param [String] type of request for use in logging
439
459
  # @param [String] request_uuid originally created for this request
440
- # @param [Time] started_at time for request
460
+ # @param [Time] expires_at time for request
441
461
  # @param [Integer] attempts to make request
442
462
  #
443
- # @return [String] request token to be used on retry
463
+ # @return [String] request UUID to be used on retry
444
464
  #
445
465
  # @raise [Exceptions::RetryableError] request failed but if retried may succeed
446
- def handle_retry_with(retry_result, type, request_uuid, started_at, attempts)
447
- if @options[:retry_enabled]
448
- interval = @options[:retry_intervals][attempts - 1]
449
- if attempts == 1 && interval && (Time.now - started_at) < @options[:retry_timeout]
450
- Log.error("Retrying #{type} request <#{request_uuid}> in #{interval} seconds " +
451
- "in response to retryable error (#{retry_result.http_body})")
452
- wait(interval)
453
- else
454
- @stats["request failures"].update("#{type} - retry")
455
- raise Exceptions::RetryableError.new(retry_result.http_body, retry_result)
456
- end
457
- else
466
+ def handle_retry_with(retry_result, type, request_uuid, expires_at, attempts)
467
+ case (interval = retry_interval(expires_at, attempts, 1))
468
+ when nil
469
+ @stats["request failures"].update("#{type} - retry")
470
+ raise Exceptions::RetryableError.new(retry_result.http_body, retry_result)
471
+ when 0
458
472
  @stats["request failures"].update("#{type} - retry")
459
473
  raise Exceptions::RetryableError.new(retry_result.http_body, retry_result)
474
+ else
475
+ Log.error("Retrying #{type} request <#{request_uuid}> in #{interval} seconds " +
476
+ "in response to retryable error (#{retry_result.http_body})")
477
+ wait(interval)
460
478
  end
461
479
  # Change request_uuid so that retried request not rejected as duplicate
462
480
  "#{request_uuid}:retry"
@@ -469,33 +487,49 @@ module RightScale
469
487
  # indicating targeted server is too busy or out of service
470
488
  # @param [String] type of request for use in logging
471
489
  # @param [String] request_uuid originally created for this request
472
- # @param [Time] started_at time for request
490
+ # @param [Time] expires_at time for request
473
491
  # @param [Integer] attempts to make request
474
492
  #
475
493
  # @return [TrueClass] always true
476
494
  #
477
495
  # @raise [Exceptions::ConnectivityFailure] cannot connect to server, lost connection
478
496
  # to it, or it is out of service or too busy to respond
479
- def handle_not_responding(not_responding, type, request_uuid, started_at, attempts)
480
- if @options[:retry_enabled]
481
- interval = @options[:retry_intervals][attempts - 1]
482
- if interval && (Time.now - started_at) < @options[:retry_timeout]
483
- Log.error("Retrying #{type} request <#{request_uuid}> in #{interval} seconds " +
484
- "in response to routing failure (#{BalancedHttpClient.exception_text(not_responding)})")
485
- wait(interval)
486
- else
487
- @stats["request failures"].update("#{type} - no result")
488
- self.state = :disconnected
489
- raise Exceptions::ConnectivityFailure.new(not_responding.message + " after #{attempts} attempts")
490
- end
491
- else
497
+ def handle_not_responding(not_responding, type, request_uuid, expires_at, attempts)
498
+ case (interval = retry_interval(expires_at, attempts))
499
+ when nil
492
500
  @stats["request failures"].update("#{type} - no result")
493
501
  self.state = :disconnected
494
502
  raise Exceptions::ConnectivityFailure.new(not_responding.message)
503
+ when 0
504
+ @stats["request failures"].update("#{type} - no result")
505
+ self.state = :disconnected
506
+ raise Exceptions::ConnectivityFailure.new(not_responding.message + " after #{attempts} attempts")
507
+ else
508
+ Log.error("Retrying #{type} request <#{request_uuid}> in #{interval} seconds " +
509
+ "in response to routing failure (#{BalancedHttpClient.exception_text(not_responding)})")
510
+ wait(interval)
495
511
  end
496
512
  true
497
513
  end
498
514
 
515
+ # Determine time interval before next retry
516
+ #
517
+ # @param [Time] expires_at time for request
518
+ # @param [Integer] attempts so far
519
+ # @param [Integer] max_retries
520
+ #
521
+ # @return [Integer, NilClass] retry interval with 0 meaning no try and nil meaning retry disabled
522
+ def retry_interval(expires_at, attempts, max_retries = nil)
523
+ if @options[:retry_enabled]
524
+ if max_retries.nil? || attempts <= max_retries
525
+ interval = @options[:retry_intervals][attempts - 1] || @options[:retry_intervals][-1]
526
+ ((Time.now + interval) < expires_at) ? interval : 0
527
+ else
528
+ 0
529
+ end
530
+ end
531
+ end
532
+
499
533
  # Wait the specified interval in non-blocking fashion if possible
500
534
  #
501
535
  # @param [Numeric] interval to wait