right_agent 0.6.6 → 0.9.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. data/lib/right_agent/agent.rb +26 -25
  2. data/lib/right_agent/agent_config.rb +28 -2
  3. data/lib/right_agent/command/command_constants.rb +2 -2
  4. data/lib/right_agent/core_payload_types/executable_bundle.rb +3 -21
  5. data/lib/right_agent/core_payload_types/login_user.rb +19 -4
  6. data/lib/right_agent/core_payload_types/recipe_instantiation.rb +7 -1
  7. data/lib/right_agent/core_payload_types/right_script_instantiation.rb +7 -1
  8. data/lib/right_agent/dispatcher.rb +6 -19
  9. data/lib/right_agent/idempotent_request.rb +72 -17
  10. data/lib/right_agent/monkey_patches/ruby_patch.rb +0 -1
  11. data/lib/right_agent/monkey_patches.rb +0 -1
  12. data/lib/right_agent/operation_result.rb +27 -4
  13. data/lib/right_agent/packets.rb +47 -23
  14. data/lib/right_agent/platform/darwin.rb +33 -2
  15. data/lib/right_agent/platform/linux.rb +98 -2
  16. data/lib/right_agent/platform/windows.rb +41 -6
  17. data/lib/right_agent/platform.rb +11 -2
  18. data/lib/right_agent/scripts/agent_controller.rb +2 -1
  19. data/lib/right_agent/scripts/agent_deployer.rb +2 -2
  20. data/lib/right_agent/scripts/stats_manager.rb +7 -3
  21. data/lib/right_agent/sender.rb +45 -28
  22. data/lib/right_agent.rb +2 -5
  23. data/right_agent.gemspec +5 -3
  24. data/spec/agent_config_spec.rb +1 -1
  25. data/spec/agent_spec.rb +26 -20
  26. data/spec/core_payload_types/login_user_spec.rb +7 -3
  27. data/spec/idempotent_request_spec.rb +218 -48
  28. data/spec/operation_result_spec.rb +19 -0
  29. data/spec/packets_spec.rb +42 -1
  30. data/spec/platform/darwin.rb +11 -0
  31. data/spec/platform/linux.rb +23 -0
  32. data/spec/platform/linux_volume_manager_spec.rb +43 -43
  33. data/spec/platform/platform_spec.rb +35 -32
  34. data/spec/platform/windows.rb +11 -0
  35. data/spec/sender_spec.rb +21 -25
  36. metadata +47 -40
  37. data/lib/right_agent/broker_client.rb +0 -686
  38. data/lib/right_agent/ha_broker_client.rb +0 -1327
  39. data/lib/right_agent/monkey_patches/amqp_patch.rb +0 -274
  40. data/lib/right_agent/monkey_patches/ruby_patch/string_patch.rb +0 -107
  41. data/lib/right_agent/stats_helper.rb +0 -745
  42. data/spec/broker_client_spec.rb +0 -962
  43. data/spec/ha_broker_client_spec.rb +0 -1695
  44. data/spec/monkey_patches/amqp_patch_spec.rb +0 -100
  45. data/spec/monkey_patches/string_patch_spec.rb +0 -99
  46. data/spec/stats_helper_spec.rb +0 -686
@@ -31,7 +31,6 @@ module RightScale
31
31
 
32
32
  include ConsoleHelper
33
33
  include DaemonizeHelper
34
- include StatsHelper
35
34
 
36
35
  # (String) Identity of this agent
37
36
  attr_reader :identity
@@ -45,7 +44,7 @@ module RightScale
45
44
  # (ActorRegistry) Registry for this agents actors
46
45
  attr_reader :registry
47
46
 
48
- # (HABrokerClient) High availability AMQP broker client
47
+ # (RightAMQP::HABrokerClient) High availability AMQP broker client
49
48
  attr_reader :broker
50
49
 
51
50
  # (Array) Tag strings published by agent
@@ -174,6 +173,7 @@ module RightScale
174
173
  def run
175
174
  Log.init(@identity, @options[:log_path], :print => true)
176
175
  Log.level = @options[:log_level] if @options[:log_level]
176
+ RightSupport::Log::Mixin.default_logger = Log
177
177
  Log.debug("Start options:")
178
178
  log_opts = @options.inject([]){ |t, (k, v)| t << "- #{k}: #{v}" }
179
179
  log_opts.each { |l| Log.debug(l) }
@@ -188,7 +188,7 @@ module RightScale
188
188
 
189
189
  # Initiate AMQP broker connection, wait for connection before proceeding
190
190
  # otherwise messages published on failed connection will be lost
191
- @broker = HABrokerClient.new(Serializer.new(:secure), @options)
191
+ @broker = RightAMQP::HABrokerClient.new(Serializer.new(:secure), @options)
192
192
  @all_setup.each { |s| @remaining_setup[s] = @broker.all }
193
193
  @broker.connection_status(:one_off => @options[:connect_timeout]) do |status|
194
194
  if status == :connected
@@ -259,14 +259,15 @@ module RightScale
259
259
  def tune_heartbeat(heartbeat)
260
260
  res = nil
261
261
  begin
262
+ Log.info("[setup] Reconnecting each broker to tune heartbeat to #{heartbeat}")
262
263
  @broker.heartbeat = heartbeat
263
264
  update_configuration(:heartbeat => heartbeat)
264
265
  ids = []
265
266
  all = @broker.all
266
267
  all.each do |id|
267
268
  begin
268
- host, port, index, priority, island_id = @broker.identity_parts(id)
269
- @broker.connect(host, port, index, priority, island = nil, force = true) do |id|
269
+ host, port, index, priority = @broker.identity_parts(id)
270
+ @broker.connect(host, port, index, priority, force = true) do |id|
270
271
  @broker.connection_status(:one_off => @options[:connect_timeout], :brokers => [id]) do |status|
271
272
  begin
272
273
  if status == :connected
@@ -321,7 +322,7 @@ module RightScale
321
322
  Log.info("Current broker configuration: #{@broker.status.inspect}")
322
323
  res = nil
323
324
  begin
324
- @broker.connect(host, port, index, priority, nil, force) do |id|
325
+ @broker.connect(host, port, index, priority, force) do |id|
325
326
  @broker.connection_status(:one_off => @options[:connect_timeout], :brokers => [id]) do |status|
326
327
  begin
327
328
  if status == :connected
@@ -342,7 +343,7 @@ module RightScale
342
343
  end
343
344
  end
344
345
  rescue Exception => e
345
- res = Log.format("Failed to connect to broker #{HABrokerClient.identity(host, port)}", e)
346
+ res = Log.format("Failed to connect to broker #{RightAMQP::HABrokerClient.identity(host, port)}", e)
346
347
  @exceptions.track("connect", e)
347
348
  end
348
349
  Log.error(res) if res
@@ -364,7 +365,7 @@ module RightScale
364
365
  and_remove = " and removing" if remove
365
366
  Log.info("Disconnecting#{and_remove} broker at host #{host.inspect} port #{port.inspect}")
366
367
  Log.info("Current broker configuration: #{@broker.status.inspect}")
367
- id = HABrokerClient.identity(host, port)
368
+ id = RightAMQP::HABrokerClient.identity(host, port)
368
369
  @connect_requests.update("disconnect #{@broker.alias_(id)}")
369
370
  connected = @broker.connected
370
371
  res = nil
@@ -434,7 +435,7 @@ module RightScale
434
435
  when Result then @sender.handle_response(packet)
435
436
  end
436
437
  @sender.message_received
437
- rescue HABrokerClient::NoConnectedBrokers => e
438
+ rescue RightAMQP::HABrokerClient::NoConnectedBrokers => e
438
439
  Log.error("Identity queue processing error", e)
439
440
  rescue Exception => e
440
441
  Log.error("Identity queue processing error", e, :trace)
@@ -578,9 +579,9 @@ module RightScale
578
579
  # === Return
579
580
  # true:: Always return true
580
581
  def reset_agent_stats
581
- @connect_requests = ActivityStats.new(measure_rate = false)
582
- @non_deliveries = ActivityStats.new
583
- @exceptions = ExceptionStats.new(self, @options[:exception_callback])
582
+ @connect_requests = RightSupport::Stats::Activity.new(measure_rate = false)
583
+ @non_deliveries = RightSupport::Stats::Activity.new
584
+ @exceptions = RightSupport::Stats::Exceptions.new(self, @options[:exception_callback])
584
585
  true
585
586
  end
586
587
 
@@ -625,7 +626,7 @@ module RightScale
625
626
  # (Boolean):: true if successful, otherwise false
626
627
  def update_configuration(opts)
627
628
  if cfg = AgentConfig.load_cfg(@agent_name)
628
- opts.each { |k, v| cfg[k] = v if cfg.has_key?(k) }
629
+ opts.each { |k, v| cfg[k] = v }
629
630
  AgentConfig.store_cfg(@agent_name, cfg)
630
631
  true
631
632
  else
@@ -739,28 +740,18 @@ module RightScale
739
740
  def finish_setup
740
741
  @broker.failed.each do |id|
741
742
  p = {:agent_identity => @identity}
742
- p[:host], p[:port], p[:id], p[:priority], _ = @broker.identity_parts(id)
743
+ p[:host], p[:port], p[:id], p[:priority] = @broker.identity_parts(id)
743
744
  @sender.send_push("/registrar/connect", p)
744
745
  end
745
746
  true
746
747
  end
747
748
 
748
749
  # Check status of agent by gathering current operation statistics and publishing them,
749
- # executing any deferred tasks, and finishing any queue setup
750
+ # finishing any queue setup, and executing any deferred tasks
750
751
  #
751
752
  # === Return
752
753
  # true:: Always return true
753
754
  def check_status
754
- @deferred_tasks.reject! do |t|
755
- begin
756
- t.call
757
- rescue Exception => e
758
- Log.error("Failed to perform deferred task", e)
759
- @exceptions.track("check status", e)
760
- end
761
- true
762
- end
763
-
764
755
  begin
765
756
  finish_setup
766
757
  rescue Exception => e
@@ -779,6 +770,16 @@ module RightScale
779
770
  @exceptions.track("check status", e)
780
771
  end
781
772
 
773
+ @deferred_tasks.reject! do |t|
774
+ begin
775
+ t.call
776
+ rescue Exception => e
777
+ Log.error("Failed to perform deferred task", e)
778
+ @exceptions.track("check status", e)
779
+ end
780
+ true
781
+ end
782
+
782
783
  @check_status_count += 1
783
784
  true
784
785
  end
@@ -63,7 +63,7 @@ module RightScale
63
63
  module AgentConfig
64
64
 
65
65
  # Current agent protocol version
66
- PROTOCOL_VERSION = 18
66
+ PROTOCOL_VERSION = 20
67
67
 
68
68
  # Current agent protocol version
69
69
  #
@@ -73,6 +73,32 @@ module RightScale
73
73
  PROTOCOL_VERSION
74
74
  end
75
75
 
76
+ # Default thread name when no thread is specified for an executable bundle.
77
+ DEFAULT_THREAD_NAME = 'default'
78
+
79
+ # Default thread name when no thread is specified for an executable bundle.
80
+ #
81
+ # === Return
82
+ # (String):: default thread name...
83
+ def self.default_thread_name
84
+ DEFAULT_THREAD_NAME
85
+ end
86
+
87
+ # Regular expression to define what a valid thread name looks like: an alpha character
88
+ # followed by 0 or more alphanumerics or underscores. Only lower-case characters are
89
+ # allowed.
90
+ VALID_THREAD_NAME = /^[a-z][a-z0-9_]*$/
91
+
92
+ # Regular expression to define what a valid thread name looks like: an alpha character
93
+ # followed by 0 or more alphanumerics or underscores. Only lower-case characters are
94
+ # allowed.
95
+ #
96
+ # === Return
97
+ # (String):: default thread name...
98
+ def self.valid_thread_name
99
+ VALID_THREAD_NAME
100
+ end
101
+
76
102
  # Initialize path to root directory of agent
77
103
  #
78
104
  # === Parameters
@@ -217,7 +243,7 @@ module RightScale
217
243
  # === Return
218
244
  # (String):: Directory path name
219
245
  def self.cfg_dir
220
- @cfg_dir ||= Platform.filesystem.cfg_dir
246
+ @cfg_dir ||= Platform.filesystem.right_agent_cfg_dir
221
247
  end
222
248
 
223
249
  # Path to generated agent configuration file
@@ -26,8 +26,8 @@ module RightScale
26
26
  class CommandConstants
27
27
 
28
28
  # Ports used for command protocol
29
- BASE_INSTANCE_AGENT_SOCKET_PORT = 60000
30
- BASE_INSTANCE_AGENT_CHECKER_SOCKET_PORT = 61000
29
+ BASE_INSTANCE_AGENT_SOCKET_PORT = 53843
30
+ BASE_INSTANCE_AGENT_CHECKER_SOCKET_PORT = 53844
31
31
 
32
32
  end
33
33
  end
@@ -35,14 +35,6 @@ module RightScale
35
35
 
36
36
  include Serializable
37
37
 
38
- # Default thread name when no thread is specified for an executable bundle.
39
- DEFAULT_THREAD_NAME = 'default'
40
-
41
- # Regular expression to define what a valid thread name looks like: an alpha character
42
- # followed by 0 or more alphanumerics or underscores. Only lower-case characters are
43
- # allowed.
44
- VALID_THREAD_NAME = /^[a-z][a-z0-9_]*$/
45
-
46
38
  # (Array) Collection of RightScripts and chef recipes instantiations
47
39
  attr_accessor :executables
48
40
 
@@ -62,6 +54,9 @@ module RightScale
62
54
  # (String) Repose server to use
63
55
  attr_accessor :repose_servers
64
56
 
57
+ # (String) The name of the RIghtLink thread to run this bundle on
58
+ attr_accessor :thread_name
59
+
65
60
  # (Hash):: collection of repos to be checked out on the instance
66
61
  # :key (String):: the hash id (SHA) of the repository
67
62
  # :value (Hash):: repo and cookbook detail
@@ -126,18 +121,5 @@ module RightScale
126
121
  desc = @executables.collect { |e| e.nickname }.join(', ') if @executables
127
122
  desc ||= 'empty bundle'
128
123
  end
129
-
130
- # Gets the thread name from a bundle, if any. Uses the default thread name for
131
- # when a thread name is not specified (for backward compatibility, etc.).
132
- #
133
- # === Parameters
134
- # bundle(ExecutableBundle):: bundle to inspect
135
- #
136
- # === Return
137
- # thread_name(String):: thread name for bundle execution
138
- def thread_name(value = nil)
139
- @thread_name = value if value
140
- return @thread_name || DEFAULT_THREAD_NAME
141
- end
142
124
  end
143
125
  end
@@ -21,6 +21,8 @@
21
21
  # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
22
22
  #
23
23
 
24
+ require 'net/ssh'
25
+
24
26
  module RightScale
25
27
 
26
28
  # Authorized user for the Managed Login feature
@@ -29,7 +31,7 @@ module RightScale
29
31
  include Serializable
30
32
 
31
33
  attr_accessor :uuid, :username, :public_key, :public_keys, :common_name,
32
- :superuser, :expires_at, :profile_data
34
+ :superuser, :expires_at, :profile_data, :public_key_fingerprints
33
35
 
34
36
  # Initialize fields from given arguments
35
37
  def initialize(*args)
@@ -41,9 +43,10 @@ module RightScale
41
43
  @expires_at = Time.at(args[5]) if args[5] && (args[5] != 0) # nil -> 0 because of expires_at.to_i below
42
44
  @public_keys = args[6]
43
45
  @profile_data = args[7]
46
+ @public_key_fingerprints = args[8]
44
47
 
45
- # we now expect an array of public_keys to be passed while supporting the
46
- # singular public_key as a legacy member. when serialized back from a
48
+ # We now expect an array of public_keys to be passed while supporting the
49
+ # singular public_key as a legacy member. When serialized back from a
47
50
  # legacy LoginUser record, the singular value may be set while the plural
48
51
  # is nil.
49
52
  if @public_keys
@@ -53,11 +56,23 @@ module RightScale
53
56
  raise ArgumentError, "Expected public_key (third argument) to be a string" unless @public_key.is_a?(String)
54
57
  @public_keys = [@public_key]
55
58
  end
59
+
60
+ # The number of fingerprints must match the number of public keys
61
+ if @public_key_fingerprints && @public_key_fingerprints.size != @public_keys.size
62
+ raise ArgumentError, "Expected public_keys (seventh argument) array length (#{@public_keys.size}) is not " +
63
+ "the same as the public_key_fingerprints (eighth argument) (#{@public_key_fingerprints.size})"
64
+ end
56
65
  end
57
66
 
58
67
  # Array of serialized fields given to constructor
59
68
  def serialized_members
60
- [ @uuid, @username, @public_key, @common_name, @superuser, @expires_at.to_i, @public_keys, @profile_data ]
69
+ [ @uuid, @username, @public_key, @common_name, @superuser, @expires_at.to_i, @public_keys, @profile_data,
70
+ @public_key_fingerprints ]
71
+ end
72
+
73
+ # Create fingerprint for public key
74
+ def self.fingerprint(public_key)
75
+ Digest::SHA1.hexdigest(::Net::SSH::KeyFactory.load_data_public_key(public_key).to_der)
61
76
  end
62
77
 
63
78
  end
@@ -43,17 +43,23 @@ module RightScale
43
43
  # (Array of SecureDocumentLocation) attributes that must be resolved by the instance
44
44
  attr_accessor :external_inputs
45
45
 
46
+ # (Hash) nil or Hash of input name to flags (array of string tokens) indicating additional
47
+ # boolean properties of the input which are useful to the instance. the presence of the
48
+ # flag means true, absence means false.
49
+ attr_accessor :input_flags
50
+
46
51
  def initialize(*args)
47
52
  @nickname = args[0] if args.size > 0
48
53
  @attributes = args[1] if args.size > 1
49
54
  @id = args[2] if args.size > 2
50
55
  @ready = args[3] if args.size > 3
51
56
  @external_inputs = args[4] if args.size > 4
57
+ @input_flags = args[5] if args.size > 5
52
58
  end
53
59
 
54
60
  # Array of serialized fields given to constructor
55
61
  def serialized_members
56
- [ @nickname, @attributes, @id, @ready, @external_inputs ]
62
+ [ @nickname, @attributes, @id, @ready, @external_inputs, @input_flags ]
57
63
  end
58
64
 
59
65
  end
@@ -53,6 +53,11 @@ module RightScale
53
53
  # (Array of SecureDocumentLocation) attributes that must be resolved by the instance
54
54
  attr_accessor :external_inputs
55
55
 
56
+ # (Hash) nil or Hash of input name to flags (array of string tokens) indicating additional
57
+ # boolean properties of the input which are useful to the instance. the presence of the
58
+ # flag means true, absense means false.
59
+ attr_accessor :input_flags
60
+
56
61
  def initialize(*args)
57
62
  @nickname = args[0] if args.size > 0
58
63
  @source = args[1] if args.size > 1
@@ -62,11 +67,12 @@ module RightScale
62
67
  @id = args[5] if args.size > 5
63
68
  @ready = args[6] if args.size > 6
64
69
  @external_inputs = args[7] if args.size > 7
70
+ @input_flags = args[8] if args.size > 8
65
71
  end
66
72
 
67
73
  # Array of serialized fields given to constructor
68
74
  def serialized_members
69
- [ @nickname, @source, @parameters, @attachments, @packages, @id, @ready, @external_inputs ]
75
+ [ @nickname, @source, @parameters, @attachments, @packages, @id, @ready, @external_inputs, @input_flags ]
70
76
  end
71
77
 
72
78
  end
@@ -25,8 +25,6 @@ module RightScale
25
25
  # Dispatching of payload to specified actor
26
26
  class Dispatcher
27
27
 
28
- include StatsHelper
29
-
30
28
  # Response queue name
31
29
  RESPONSE_QUEUE = "response"
32
30
 
@@ -112,7 +110,7 @@ module RightScale
112
110
  # (String) Identity of associated agent
113
111
  attr_reader :identity
114
112
 
115
- # (HABrokerClient) High availability AMQP broker client
113
+ # (RightAMQP::HABrokerClient) High availability AMQP broker client
116
114
  attr_reader :broker
117
115
 
118
116
  # (EM) Event machine class (exposed for unit tests)
@@ -174,7 +172,7 @@ module RightScale
174
172
  # Reject this request if its TTL has expired
175
173
  if (expires_at = request.expires_at) && expires_at > 0 && received_at.to_i >= expires_at
176
174
  @rejects.update("expired (#{method})")
177
- Log.info("REJECT EXPIRED <#{token}> from #{request.from} TTL #{elapsed(received_at.to_i - expires_at)} ago")
175
+ Log.info("REJECT EXPIRED <#{token}> from #{request.from} TTL #{RightSupport::Stats.elapsed(received_at.to_i - expires_at)} ago")
178
176
  if request.is_a?(Request)
179
177
  # For agents that do not know about non-delivery, use error result
180
178
  non_delivery = if request.recv_version < 13
@@ -228,7 +226,7 @@ module RightScale
228
226
  exchange = {:type => :queue, :name => RESPONSE_QUEUE, :options => {:durable => true, :no_declare => @secure}}
229
227
  @broker.publish(exchange, r, :persistent => true, :mandatory => true, :log_filter => [:tries, :persistent, :duration])
230
228
  end
231
- rescue HABrokerClient::NoConnectedBrokers => e
229
+ rescue RightAMQP::HABrokerClient::NoConnectedBrokers => e
232
230
  Log.error("Failed to publish result of dispatched request #{request.trace}", e)
233
231
  rescue Exception => e
234
232
  Log.error("Failed to publish result of dispatched request #{request.trace}", e, :trace)
@@ -298,9 +296,9 @@ module RightScale
298
296
  # === Return
299
297
  # true:: Always return true
300
298
  def reset_stats
301
- @rejects = ActivityStats.new
302
- @requests = ActivityStats.new
303
- @exceptions = ExceptionStats.new(@agent)
299
+ @rejects = RightSupport::Stats::Activity.new
300
+ @requests = RightSupport::Stats::Activity.new
301
+ @exceptions = RightSupport::Stats::Exceptions.new(@agent)
304
302
  true
305
303
  end
306
304
 
@@ -335,17 +333,6 @@ module RightScale
335
333
  error
336
334
  end
337
335
 
338
- # Convert value to nil if equals 0
339
- #
340
- # === Parameters
341
- # value(Integer|nil):: Value to be converted
342
- #
343
- # === Return
344
- # (Integer|nil):: Converted value
345
- def nil_if_zero(value)
346
- if !value || value == 0 then nil else value end
347
- end
348
-
349
336
  end # Dispatcher
350
337
 
351
338
  end # RightScale
@@ -22,14 +22,46 @@
22
22
 
23
23
  module RightScale
24
24
 
25
+ # This is a retryable request for use when the execution of the request by the
26
+ # receiver is known to be idempotent and when there is a need to indefinitely
27
+ # pursue getting a usable response, e.g., when an instance is launching.
28
+ # It is implemented as an EM::Deferrable and as such invokes the Proc defined
29
+ # with its #callback method with the result content from a OperationResult::SUCCESS
30
+ # response, or it will invoke the Proc defined with its #errback method with error
31
+ # content if the response is an OperationResult::ERROR or CANCEL, or if the request
32
+ # has timed out. The request can be canceled with the #cancel method, or the receiver
33
+ # of the request may respond with a CANCEL result to cause the request to be canceled.
34
+ # This is useful in situations where the request is never expected to succeed
35
+ # regardless of the number of retries. By default if the response to the request
36
+ # is an OperationResult::RETRY or NON_DELIVERY indication, the request is automatically
37
+ # retried, as is also the case for an ERROR indication if the :retry_on_error option
38
+ # is specified. The retry algorithm is controlled by the :retry_delay, :retry_delay_count,
39
+ # and :max_retry_delay settings. The initial retry interval is the default or specified
40
+ # :retry_delay and this interval is used :retry_delay_count times, at which point the
41
+ # :retry_delay is doubled and the :retry_delay_count is halved. This backoff is again
42
+ # applied after the new :retry_delay_count is reached, and so on until :retry_delay
43
+ # reaches :max_retry_delay which then is used as the interval until the default or
44
+ # specified :timeout is reached. The default :timeout is 4 days.
25
45
  class IdempotentRequest
26
46
 
27
47
  include OperationResultHelper
28
48
  include EM::Deferrable
29
49
 
30
- # Wait 5 seconds before retrying in case of failure
50
+ # Default delay before initial retry in case of failure with -1 meaning no delay
31
51
  DEFAULT_RETRY_DELAY = 5
32
52
 
53
+ # Default minimum number of retries before beginning backoff
54
+ DEFAULT_RETRY_DELAY_COUNT = 60
55
+
56
+ # Maximum default delay before retry when backing off
57
+ DEFAULT_MAX_RETRY_DELAY = 60
58
+
59
+ # Factor used for exponential backoff of retry delay
60
+ RETRY_BACKOFF_FACTOR = 2
61
+
62
+ # Default timeout with -1 meaning never timeout
63
+ DEFAULT_TIMEOUT = 4 * 24 * 60 * 60
64
+
33
65
  attr_reader :raw_response
34
66
 
35
67
  # Send idempotent request
@@ -37,18 +69,31 @@ module RightScale
37
69
  # Calls deferrable callback on completion, error callback on timeout
38
70
  #
39
71
  # === Parameters
40
- # operation(String):: Request operation (i.e. '/booter/get_boot_bundle')
72
+ # operation(String):: Request operation (e.g., '/booter/get_boot_bundle')
41
73
  # payload(Hash):: Request payload
42
- # options[:retry_on_error](FalseClass|TrueClass):: Whether request should be retried
43
- # if recipient returned an error
44
- # options[:timeout](Fixnum):: Number of seconds before error callback gets called
45
- # options[:retry_delay](Fixnum):: Number of seconds before retry, defaults to 5
46
- def initialize(operation, payload, options={})
47
- raise ArgumentError.new("options[:operation] is required") unless @operation = operation
48
- raise ArgumentError.new("options[:payload] is required") unless @payload = payload
74
+ # options(Hash):: Request options
75
+ # :targets(Array):: Targets from which to randomly choose one
76
+ # :retry_on_error(Boolean):: Whether request should be retried if recipient returned an error
77
+ # :retry_delay(Fixnum):: Number of seconds delay before initial retry with -1 meaning no delay,
78
+ # defaults to DEFAULT_RETRY_DELAY
79
+ # :retry_delay_count(Fixnum):: Minimum number of retries at initial :retry_delay value before
80
+ # increasing delay exponentially and decreasing this count exponentially, defaults to
81
+ # DEFAULT_RETRY_DELAY_COUNT
82
+ # :max_retry_delay(Fixnum):: Maximum number of seconds of retry delay, defaults to DEFAULT_MAX_RETRY_DELAY
83
+ # :timeout(Fixnum):: Number of seconds with no response before error callback gets called with
84
+ # -1 meaning never, defaults to DEFAULT_TIMEOUT
85
+ #
86
+ # === Raises
87
+ # ArgumentError:: If operation or payload not specified
88
+ def initialize(operation, payload, options = {})
89
+ raise ArgumentError.new("operation is required") unless @operation = operation
90
+ raise ArgumentError.new("payload is required") unless @payload = payload
49
91
  @retry_on_error = options[:retry_on_error] || false
50
- @timeout = options[:timeout] || -1
92
+ @timeout = options[:timeout] || DEFAULT_TIMEOUT
51
93
  @retry_delay = options[:retry_delay] || DEFAULT_RETRY_DELAY
94
+ @retry_delay_count = options[:retry_delay_count] || DEFAULT_RETRY_DELAY_COUNT
95
+ @max_retry_delay = options[:max_retry_delay] || DEFAULT_MAX_RETRY_DELAY
96
+ @retries = 0
52
97
  @targets = options[:targets]
53
98
  @raw_response = nil
54
99
  @done = false
@@ -64,7 +109,7 @@ module RightScale
64
109
  if @cancel_timer.nil? && @timeout > 0
65
110
  @cancel_timer = EM::Timer.new(@timeout) do
66
111
  msg = "Request #{@operation} timed out after #{@timeout} seconds"
67
- log_info(msg)
112
+ Log.info(msg)
68
113
  cancel(msg)
69
114
  end
70
115
  end
@@ -101,7 +146,6 @@ module RightScale
101
146
  return true if @done
102
147
  @raw_response = r
103
148
  res = result_from(r)
104
- res = OperationResult.non_delivery unless res
105
149
  if res.success?
106
150
  if @cancel_timer
107
151
  @cancel_timer.cancel
@@ -110,17 +154,28 @@ module RightScale
110
154
  @done = true
111
155
  succeed(res.content)
112
156
  else
157
+ reason = res.content
113
158
  if res.non_delivery?
114
- Log.info("Request non-delivery (#{res.content}) for #{@operation}")
159
+ Log.info("Request non-delivery (#{reason}) for #{@operation}")
115
160
  elsif res.retry?
116
- Log.info("RightScale not ready when trying to request #{@operation}")
161
+ reason = (reason && !reason.empty?) ? reason : "RightScale not ready"
162
+ Log.info("Request #{@operation} failed (#{reason}) and should be retried")
163
+ elsif res.cancel?
164
+ reason = (reason && !reason.empty?) ? reason : "RightScale cannot execute request"
165
+ Log.info("Request #{@operation} canceled (#{reason})")
117
166
  else
118
- Log.info("Request #{@operation} failed (#{res.content})")
167
+ Log.info("Request #{@operation} failed (#{reason})")
119
168
  end
120
- if res.non_delivery? || res.retry? || @retry_on_error
169
+ if (res.non_delivery? || res.retry? || @retry_on_error) && !res.cancel?
121
170
  Log.info("Retrying in #{@retry_delay} seconds...")
122
171
  if @retry_delay > 0
123
- EM.add_timer(@retry_delay) { run }
172
+ this_delay = @retry_delay
173
+ if (@retries += 1) >= @retry_delay_count
174
+ @retry_delay = [@retry_delay * RETRY_BACKOFF_FACTOR, @max_retry_delay].min
175
+ @retry_delay_count = [@retry_delay_count / RETRY_BACKOFF_FACTOR, 1].max
176
+ @retries = 0
177
+ end
178
+ EM.add_timer(this_delay) { run }
124
179
  else
125
180
  EM.next_tick { run }
126
181
  end
@@ -42,7 +42,6 @@ end
42
42
  RUBY_PATCH_BASE_DIR = File.join(File.dirname(__FILE__), 'ruby_patch')
43
43
 
44
44
  require File.normalize_path(File.join(RUBY_PATCH_BASE_DIR, 'array_patch'))
45
- require File.normalize_path(File.join(RUBY_PATCH_BASE_DIR, 'string_patch'))
46
45
  require File.normalize_path(File.join(RUBY_PATCH_BASE_DIR, 'object_patch'))
47
46
  require File.normalize_path(File.join(RUBY_PATCH_BASE_DIR, 'singleton_patch'))
48
47
 
@@ -25,5 +25,4 @@ require 'rbconfig'
25
25
 
26
26
  MONKEY_PATCHES_BASE_DIR = File.join(File.dirname(__FILE__), 'monkey_patches')
27
27
 
28
- require File.normalize_path(File.join(MONKEY_PATCHES_BASE_DIR, 'amqp_patch'))
29
28
  require File.normalize_path(File.join(MONKEY_PATCHES_BASE_DIR, 'ruby_patch'))
@@ -35,6 +35,7 @@ module RightScale
35
35
  RETRY = 3
36
36
  NON_DELIVERY = 4
37
37
  MULTICAST = 5 # Deprecated for agents at version 13 or above
38
+ CANCEL = 6
38
39
 
39
40
  # Non-delivery reasons
40
41
  NON_DELIVERY_REASONS = [
@@ -81,9 +82,10 @@ module RightScale
81
82
  when SUCCESS then 'success'
82
83
  when ERROR then 'error' + (reason ? " (#{truncated_error})" : "")
83
84
  when CONTINUE then 'continue'
84
- when RETRY then 'retry'
85
+ when RETRY then 'retry' + (reason ? " (#{@content})" : "")
85
86
  when NON_DELIVERY then 'non-delivery' + (reason ? " (#{@content})" : "")
86
87
  when MULTICAST then 'multicast'
88
+ when CANCEL then 'cancel' + (reason ? " (#{@content})" : "")
87
89
  end
88
90
  end
89
91
 
@@ -126,7 +128,7 @@ module RightScale
126
128
  # Create new success status
127
129
  #
128
130
  # === Parameters
129
- # content(Object):: Any data associated with successful results - defaults to nil
131
+ # content(Object):: Any data associated with successful results, defaults to nil
130
132
  #
131
133
  # === Return
132
134
  # (OperationResult):: Corresponding result
@@ -151,7 +153,7 @@ module RightScale
151
153
  # Create new continue status
152
154
  #
153
155
  # === Parameters
154
- # content(Object):: Any data associated with continue - defaults to nil
156
+ # content(Object):: Any data associated with continue, defaults to nil
155
157
  #
156
158
  # === Return
157
159
  # (OperationResult):: Corresponding result
@@ -162,7 +164,7 @@ module RightScale
162
164
  # Create new retry status
163
165
  #
164
166
  # === Parameters
165
- # content(Object):: Any data associated with retry - defaults to nil
167
+ # content(Object):: Any data associated with retry, defaults to nil
166
168
  #
167
169
  # === Return
168
170
  # (OperationResult):: Corresponding result
@@ -193,6 +195,17 @@ module RightScale
193
195
  OperationResult.new(MULTICAST, targets)
194
196
  end
195
197
 
198
+ # Cancel request and never retry
199
+ #
200
+ # === Parameters
201
+ # content(Object):: Any data associated with cancel, defaults to nil
202
+ #
203
+ # === Return
204
+ # (OperationResult):: Corresponding result
205
+ def self.cancel(content = nil)
206
+ OperationResult.new(CANCEL, content)
207
+ end
208
+
196
209
  # Was last operation successful?
197
210
  #
198
211
  # === Return
@@ -248,6 +261,15 @@ module RightScale
248
261
  status_code == MULTICAST
249
262
  end
250
263
 
264
+ # Was last operation status CANCEL?
265
+ #
266
+ # === Return
267
+ # true:: If status is CANCEL
268
+ # false:: Otherwise
269
+ def cancel?
270
+ status_code == CANCEL
271
+ end
272
+
251
273
  # Array of serialized fields given to constructor
252
274
  def serialized_members
253
275
  [@status_code, @content]
@@ -263,6 +285,7 @@ module RightScale
263
285
  def continue_result(*args) OperationResult.continue(*args) end
264
286
  def retry_result(*args) OperationResult.retry(*args) end
265
287
  def non_delivery_result(*args) OperationResult.non_delivery(*args) end
288
+ def cancel_result(*args) OperationResult.cancel(*args) end
266
289
  def result_from(*args) OperationResult.from_results(*args) end
267
290
 
268
291
  end # OperationResultHelper