right_agent 0.6.6 → 0.9.3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (46) hide show
  1. data/lib/right_agent/agent.rb +26 -25
  2. data/lib/right_agent/agent_config.rb +28 -2
  3. data/lib/right_agent/command/command_constants.rb +2 -2
  4. data/lib/right_agent/core_payload_types/executable_bundle.rb +3 -21
  5. data/lib/right_agent/core_payload_types/login_user.rb +19 -4
  6. data/lib/right_agent/core_payload_types/recipe_instantiation.rb +7 -1
  7. data/lib/right_agent/core_payload_types/right_script_instantiation.rb +7 -1
  8. data/lib/right_agent/dispatcher.rb +6 -19
  9. data/lib/right_agent/idempotent_request.rb +72 -17
  10. data/lib/right_agent/monkey_patches/ruby_patch.rb +0 -1
  11. data/lib/right_agent/monkey_patches.rb +0 -1
  12. data/lib/right_agent/operation_result.rb +27 -4
  13. data/lib/right_agent/packets.rb +47 -23
  14. data/lib/right_agent/platform/darwin.rb +33 -2
  15. data/lib/right_agent/platform/linux.rb +98 -2
  16. data/lib/right_agent/platform/windows.rb +41 -6
  17. data/lib/right_agent/platform.rb +11 -2
  18. data/lib/right_agent/scripts/agent_controller.rb +2 -1
  19. data/lib/right_agent/scripts/agent_deployer.rb +2 -2
  20. data/lib/right_agent/scripts/stats_manager.rb +7 -3
  21. data/lib/right_agent/sender.rb +45 -28
  22. data/lib/right_agent.rb +2 -5
  23. data/right_agent.gemspec +5 -3
  24. data/spec/agent_config_spec.rb +1 -1
  25. data/spec/agent_spec.rb +26 -20
  26. data/spec/core_payload_types/login_user_spec.rb +7 -3
  27. data/spec/idempotent_request_spec.rb +218 -48
  28. data/spec/operation_result_spec.rb +19 -0
  29. data/spec/packets_spec.rb +42 -1
  30. data/spec/platform/darwin.rb +11 -0
  31. data/spec/platform/linux.rb +23 -0
  32. data/spec/platform/linux_volume_manager_spec.rb +43 -43
  33. data/spec/platform/platform_spec.rb +35 -32
  34. data/spec/platform/windows.rb +11 -0
  35. data/spec/sender_spec.rb +21 -25
  36. metadata +47 -40
  37. data/lib/right_agent/broker_client.rb +0 -686
  38. data/lib/right_agent/ha_broker_client.rb +0 -1327
  39. data/lib/right_agent/monkey_patches/amqp_patch.rb +0 -274
  40. data/lib/right_agent/monkey_patches/ruby_patch/string_patch.rb +0 -107
  41. data/lib/right_agent/stats_helper.rb +0 -745
  42. data/spec/broker_client_spec.rb +0 -962
  43. data/spec/ha_broker_client_spec.rb +0 -1695
  44. data/spec/monkey_patches/amqp_patch_spec.rb +0 -100
  45. data/spec/monkey_patches/string_patch_spec.rb +0 -99
  46. data/spec/stats_helper_spec.rb +0 -686
@@ -31,7 +31,6 @@ module RightScale
31
31
 
32
32
  include ConsoleHelper
33
33
  include DaemonizeHelper
34
- include StatsHelper
35
34
 
36
35
  # (String) Identity of this agent
37
36
  attr_reader :identity
@@ -45,7 +44,7 @@ module RightScale
45
44
  # (ActorRegistry) Registry for this agents actors
46
45
  attr_reader :registry
47
46
 
48
- # (HABrokerClient) High availability AMQP broker client
47
+ # (RightAMQP::HABrokerClient) High availability AMQP broker client
49
48
  attr_reader :broker
50
49
 
51
50
  # (Array) Tag strings published by agent
@@ -174,6 +173,7 @@ module RightScale
174
173
  def run
175
174
  Log.init(@identity, @options[:log_path], :print => true)
176
175
  Log.level = @options[:log_level] if @options[:log_level]
176
+ RightSupport::Log::Mixin.default_logger = Log
177
177
  Log.debug("Start options:")
178
178
  log_opts = @options.inject([]){ |t, (k, v)| t << "- #{k}: #{v}" }
179
179
  log_opts.each { |l| Log.debug(l) }
@@ -188,7 +188,7 @@ module RightScale
188
188
 
189
189
  # Initiate AMQP broker connection, wait for connection before proceeding
190
190
  # otherwise messages published on failed connection will be lost
191
- @broker = HABrokerClient.new(Serializer.new(:secure), @options)
191
+ @broker = RightAMQP::HABrokerClient.new(Serializer.new(:secure), @options)
192
192
  @all_setup.each { |s| @remaining_setup[s] = @broker.all }
193
193
  @broker.connection_status(:one_off => @options[:connect_timeout]) do |status|
194
194
  if status == :connected
@@ -259,14 +259,15 @@ module RightScale
259
259
  def tune_heartbeat(heartbeat)
260
260
  res = nil
261
261
  begin
262
+ Log.info("[setup] Reconnecting each broker to tune heartbeat to #{heartbeat}")
262
263
  @broker.heartbeat = heartbeat
263
264
  update_configuration(:heartbeat => heartbeat)
264
265
  ids = []
265
266
  all = @broker.all
266
267
  all.each do |id|
267
268
  begin
268
- host, port, index, priority, island_id = @broker.identity_parts(id)
269
- @broker.connect(host, port, index, priority, island = nil, force = true) do |id|
269
+ host, port, index, priority = @broker.identity_parts(id)
270
+ @broker.connect(host, port, index, priority, force = true) do |id|
270
271
  @broker.connection_status(:one_off => @options[:connect_timeout], :brokers => [id]) do |status|
271
272
  begin
272
273
  if status == :connected
@@ -321,7 +322,7 @@ module RightScale
321
322
  Log.info("Current broker configuration: #{@broker.status.inspect}")
322
323
  res = nil
323
324
  begin
324
- @broker.connect(host, port, index, priority, nil, force) do |id|
325
+ @broker.connect(host, port, index, priority, force) do |id|
325
326
  @broker.connection_status(:one_off => @options[:connect_timeout], :brokers => [id]) do |status|
326
327
  begin
327
328
  if status == :connected
@@ -342,7 +343,7 @@ module RightScale
342
343
  end
343
344
  end
344
345
  rescue Exception => e
345
- res = Log.format("Failed to connect to broker #{HABrokerClient.identity(host, port)}", e)
346
+ res = Log.format("Failed to connect to broker #{RightAMQP::HABrokerClient.identity(host, port)}", e)
346
347
  @exceptions.track("connect", e)
347
348
  end
348
349
  Log.error(res) if res
@@ -364,7 +365,7 @@ module RightScale
364
365
  and_remove = " and removing" if remove
365
366
  Log.info("Disconnecting#{and_remove} broker at host #{host.inspect} port #{port.inspect}")
366
367
  Log.info("Current broker configuration: #{@broker.status.inspect}")
367
- id = HABrokerClient.identity(host, port)
368
+ id = RightAMQP::HABrokerClient.identity(host, port)
368
369
  @connect_requests.update("disconnect #{@broker.alias_(id)}")
369
370
  connected = @broker.connected
370
371
  res = nil
@@ -434,7 +435,7 @@ module RightScale
434
435
  when Result then @sender.handle_response(packet)
435
436
  end
436
437
  @sender.message_received
437
- rescue HABrokerClient::NoConnectedBrokers => e
438
+ rescue RightAMQP::HABrokerClient::NoConnectedBrokers => e
438
439
  Log.error("Identity queue processing error", e)
439
440
  rescue Exception => e
440
441
  Log.error("Identity queue processing error", e, :trace)
@@ -578,9 +579,9 @@ module RightScale
578
579
  # === Return
579
580
  # true:: Always return true
580
581
  def reset_agent_stats
581
- @connect_requests = ActivityStats.new(measure_rate = false)
582
- @non_deliveries = ActivityStats.new
583
- @exceptions = ExceptionStats.new(self, @options[:exception_callback])
582
+ @connect_requests = RightSupport::Stats::Activity.new(measure_rate = false)
583
+ @non_deliveries = RightSupport::Stats::Activity.new
584
+ @exceptions = RightSupport::Stats::Exceptions.new(self, @options[:exception_callback])
584
585
  true
585
586
  end
586
587
 
@@ -625,7 +626,7 @@ module RightScale
625
626
  # (Boolean):: true if successful, otherwise false
626
627
  def update_configuration(opts)
627
628
  if cfg = AgentConfig.load_cfg(@agent_name)
628
- opts.each { |k, v| cfg[k] = v if cfg.has_key?(k) }
629
+ opts.each { |k, v| cfg[k] = v }
629
630
  AgentConfig.store_cfg(@agent_name, cfg)
630
631
  true
631
632
  else
@@ -739,28 +740,18 @@ module RightScale
739
740
  def finish_setup
740
741
  @broker.failed.each do |id|
741
742
  p = {:agent_identity => @identity}
742
- p[:host], p[:port], p[:id], p[:priority], _ = @broker.identity_parts(id)
743
+ p[:host], p[:port], p[:id], p[:priority] = @broker.identity_parts(id)
743
744
  @sender.send_push("/registrar/connect", p)
744
745
  end
745
746
  true
746
747
  end
747
748
 
748
749
  # Check status of agent by gathering current operation statistics and publishing them,
749
- # executing any deferred tasks, and finishing any queue setup
750
+ # finishing any queue setup, and executing any deferred tasks
750
751
  #
751
752
  # === Return
752
753
  # true:: Always return true
753
754
  def check_status
754
- @deferred_tasks.reject! do |t|
755
- begin
756
- t.call
757
- rescue Exception => e
758
- Log.error("Failed to perform deferred task", e)
759
- @exceptions.track("check status", e)
760
- end
761
- true
762
- end
763
-
764
755
  begin
765
756
  finish_setup
766
757
  rescue Exception => e
@@ -779,6 +770,16 @@ module RightScale
779
770
  @exceptions.track("check status", e)
780
771
  end
781
772
 
773
+ @deferred_tasks.reject! do |t|
774
+ begin
775
+ t.call
776
+ rescue Exception => e
777
+ Log.error("Failed to perform deferred task", e)
778
+ @exceptions.track("check status", e)
779
+ end
780
+ true
781
+ end
782
+
782
783
  @check_status_count += 1
783
784
  true
784
785
  end
@@ -63,7 +63,7 @@ module RightScale
63
63
  module AgentConfig
64
64
 
65
65
  # Current agent protocol version
66
- PROTOCOL_VERSION = 18
66
+ PROTOCOL_VERSION = 20
67
67
 
68
68
  # Current agent protocol version
69
69
  #
@@ -73,6 +73,32 @@ module RightScale
73
73
  PROTOCOL_VERSION
74
74
  end
75
75
 
76
+ # Default thread name when no thread is specified for an executable bundle.
77
+ DEFAULT_THREAD_NAME = 'default'
78
+
79
+ # Default thread name when no thread is specified for an executable bundle.
80
+ #
81
+ # === Return
82
+ # (String):: default thread name...
83
+ def self.default_thread_name
84
+ DEFAULT_THREAD_NAME
85
+ end
86
+
87
+ # Regular expression to define what a valid thread name looks like: an alpha character
88
+ # followed by 0 or more alphanumerics or underscores. Only lower-case characters are
89
+ # allowed.
90
+ VALID_THREAD_NAME = /^[a-z][a-z0-9_]*$/
91
+
92
+ # Regular expression to define what a valid thread name looks like: an alpha character
93
+ # followed by 0 or more alphanumerics or underscores. Only lower-case characters are
94
+ # allowed.
95
+ #
96
+ # === Return
97
+ # (String):: default thread name...
98
+ def self.valid_thread_name
99
+ VALID_THREAD_NAME
100
+ end
101
+
76
102
  # Initialize path to root directory of agent
77
103
  #
78
104
  # === Parameters
@@ -217,7 +243,7 @@ module RightScale
217
243
  # === Return
218
244
  # (String):: Directory path name
219
245
  def self.cfg_dir
220
- @cfg_dir ||= Platform.filesystem.cfg_dir
246
+ @cfg_dir ||= Platform.filesystem.right_agent_cfg_dir
221
247
  end
222
248
 
223
249
  # Path to generated agent configuration file
@@ -26,8 +26,8 @@ module RightScale
26
26
  class CommandConstants
27
27
 
28
28
  # Ports used for command protocol
29
- BASE_INSTANCE_AGENT_SOCKET_PORT = 60000
30
- BASE_INSTANCE_AGENT_CHECKER_SOCKET_PORT = 61000
29
+ BASE_INSTANCE_AGENT_SOCKET_PORT = 53843
30
+ BASE_INSTANCE_AGENT_CHECKER_SOCKET_PORT = 53844
31
31
 
32
32
  end
33
33
  end
@@ -35,14 +35,6 @@ module RightScale
35
35
 
36
36
  include Serializable
37
37
 
38
- # Default thread name when no thread is specified for an executable bundle.
39
- DEFAULT_THREAD_NAME = 'default'
40
-
41
- # Regular expression to define what a valid thread name looks like: an alpha character
42
- # followed by 0 or more alphanumerics or underscores. Only lower-case characters are
43
- # allowed.
44
- VALID_THREAD_NAME = /^[a-z][a-z0-9_]*$/
45
-
46
38
  # (Array) Collection of RightScripts and chef recipes instantiations
47
39
  attr_accessor :executables
48
40
 
@@ -62,6 +54,9 @@ module RightScale
62
54
  # (String) Repose server to use
63
55
  attr_accessor :repose_servers
64
56
 
57
+ # (String) The name of the RIghtLink thread to run this bundle on
58
+ attr_accessor :thread_name
59
+
65
60
  # (Hash):: collection of repos to be checked out on the instance
66
61
  # :key (String):: the hash id (SHA) of the repository
67
62
  # :value (Hash):: repo and cookbook detail
@@ -126,18 +121,5 @@ module RightScale
126
121
  desc = @executables.collect { |e| e.nickname }.join(', ') if @executables
127
122
  desc ||= 'empty bundle'
128
123
  end
129
-
130
- # Gets the thread name from a bundle, if any. Uses the default thread name for
131
- # when a thread name is not specified (for backward compatibility, etc.).
132
- #
133
- # === Parameters
134
- # bundle(ExecutableBundle):: bundle to inspect
135
- #
136
- # === Return
137
- # thread_name(String):: thread name for bundle execution
138
- def thread_name(value = nil)
139
- @thread_name = value if value
140
- return @thread_name || DEFAULT_THREAD_NAME
141
- end
142
124
  end
143
125
  end
@@ -21,6 +21,8 @@
21
21
  # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
22
22
  #
23
23
 
24
+ require 'net/ssh'
25
+
24
26
  module RightScale
25
27
 
26
28
  # Authorized user for the Managed Login feature
@@ -29,7 +31,7 @@ module RightScale
29
31
  include Serializable
30
32
 
31
33
  attr_accessor :uuid, :username, :public_key, :public_keys, :common_name,
32
- :superuser, :expires_at, :profile_data
34
+ :superuser, :expires_at, :profile_data, :public_key_fingerprints
33
35
 
34
36
  # Initialize fields from given arguments
35
37
  def initialize(*args)
@@ -41,9 +43,10 @@ module RightScale
41
43
  @expires_at = Time.at(args[5]) if args[5] && (args[5] != 0) # nil -> 0 because of expires_at.to_i below
42
44
  @public_keys = args[6]
43
45
  @profile_data = args[7]
46
+ @public_key_fingerprints = args[8]
44
47
 
45
- # we now expect an array of public_keys to be passed while supporting the
46
- # singular public_key as a legacy member. when serialized back from a
48
+ # We now expect an array of public_keys to be passed while supporting the
49
+ # singular public_key as a legacy member. When serialized back from a
47
50
  # legacy LoginUser record, the singular value may be set while the plural
48
51
  # is nil.
49
52
  if @public_keys
@@ -53,11 +56,23 @@ module RightScale
53
56
  raise ArgumentError, "Expected public_key (third argument) to be a string" unless @public_key.is_a?(String)
54
57
  @public_keys = [@public_key]
55
58
  end
59
+
60
+ # The number of fingerprints must match the number of public keys
61
+ if @public_key_fingerprints && @public_key_fingerprints.size != @public_keys.size
62
+ raise ArgumentError, "Expected public_keys (seventh argument) array length (#{@public_keys.size}) is not " +
63
+ "the same as the public_key_fingerprints (eighth argument) (#{@public_key_fingerprints.size})"
64
+ end
56
65
  end
57
66
 
58
67
  # Array of serialized fields given to constructor
59
68
  def serialized_members
60
- [ @uuid, @username, @public_key, @common_name, @superuser, @expires_at.to_i, @public_keys, @profile_data ]
69
+ [ @uuid, @username, @public_key, @common_name, @superuser, @expires_at.to_i, @public_keys, @profile_data,
70
+ @public_key_fingerprints ]
71
+ end
72
+
73
+ # Create fingerprint for public key
74
+ def self.fingerprint(public_key)
75
+ Digest::SHA1.hexdigest(::Net::SSH::KeyFactory.load_data_public_key(public_key).to_der)
61
76
  end
62
77
 
63
78
  end
@@ -43,17 +43,23 @@ module RightScale
43
43
  # (Array of SecureDocumentLocation) attributes that must be resolved by the instance
44
44
  attr_accessor :external_inputs
45
45
 
46
+ # (Hash) nil or Hash of input name to flags (array of string tokens) indicating additional
47
+ # boolean properties of the input which are useful to the instance. the presence of the
48
+ # flag means true, absence means false.
49
+ attr_accessor :input_flags
50
+
46
51
  def initialize(*args)
47
52
  @nickname = args[0] if args.size > 0
48
53
  @attributes = args[1] if args.size > 1
49
54
  @id = args[2] if args.size > 2
50
55
  @ready = args[3] if args.size > 3
51
56
  @external_inputs = args[4] if args.size > 4
57
+ @input_flags = args[5] if args.size > 5
52
58
  end
53
59
 
54
60
  # Array of serialized fields given to constructor
55
61
  def serialized_members
56
- [ @nickname, @attributes, @id, @ready, @external_inputs ]
62
+ [ @nickname, @attributes, @id, @ready, @external_inputs, @input_flags ]
57
63
  end
58
64
 
59
65
  end
@@ -53,6 +53,11 @@ module RightScale
53
53
  # (Array of SecureDocumentLocation) attributes that must be resolved by the instance
54
54
  attr_accessor :external_inputs
55
55
 
56
+ # (Hash) nil or Hash of input name to flags (array of string tokens) indicating additional
57
+ # boolean properties of the input which are useful to the instance. the presence of the
58
+ # flag means true, absense means false.
59
+ attr_accessor :input_flags
60
+
56
61
  def initialize(*args)
57
62
  @nickname = args[0] if args.size > 0
58
63
  @source = args[1] if args.size > 1
@@ -62,11 +67,12 @@ module RightScale
62
67
  @id = args[5] if args.size > 5
63
68
  @ready = args[6] if args.size > 6
64
69
  @external_inputs = args[7] if args.size > 7
70
+ @input_flags = args[8] if args.size > 8
65
71
  end
66
72
 
67
73
  # Array of serialized fields given to constructor
68
74
  def serialized_members
69
- [ @nickname, @source, @parameters, @attachments, @packages, @id, @ready, @external_inputs ]
75
+ [ @nickname, @source, @parameters, @attachments, @packages, @id, @ready, @external_inputs, @input_flags ]
70
76
  end
71
77
 
72
78
  end
@@ -25,8 +25,6 @@ module RightScale
25
25
  # Dispatching of payload to specified actor
26
26
  class Dispatcher
27
27
 
28
- include StatsHelper
29
-
30
28
  # Response queue name
31
29
  RESPONSE_QUEUE = "response"
32
30
 
@@ -112,7 +110,7 @@ module RightScale
112
110
  # (String) Identity of associated agent
113
111
  attr_reader :identity
114
112
 
115
- # (HABrokerClient) High availability AMQP broker client
113
+ # (RightAMQP::HABrokerClient) High availability AMQP broker client
116
114
  attr_reader :broker
117
115
 
118
116
  # (EM) Event machine class (exposed for unit tests)
@@ -174,7 +172,7 @@ module RightScale
174
172
  # Reject this request if its TTL has expired
175
173
  if (expires_at = request.expires_at) && expires_at > 0 && received_at.to_i >= expires_at
176
174
  @rejects.update("expired (#{method})")
177
- Log.info("REJECT EXPIRED <#{token}> from #{request.from} TTL #{elapsed(received_at.to_i - expires_at)} ago")
175
+ Log.info("REJECT EXPIRED <#{token}> from #{request.from} TTL #{RightSupport::Stats.elapsed(received_at.to_i - expires_at)} ago")
178
176
  if request.is_a?(Request)
179
177
  # For agents that do not know about non-delivery, use error result
180
178
  non_delivery = if request.recv_version < 13
@@ -228,7 +226,7 @@ module RightScale
228
226
  exchange = {:type => :queue, :name => RESPONSE_QUEUE, :options => {:durable => true, :no_declare => @secure}}
229
227
  @broker.publish(exchange, r, :persistent => true, :mandatory => true, :log_filter => [:tries, :persistent, :duration])
230
228
  end
231
- rescue HABrokerClient::NoConnectedBrokers => e
229
+ rescue RightAMQP::HABrokerClient::NoConnectedBrokers => e
232
230
  Log.error("Failed to publish result of dispatched request #{request.trace}", e)
233
231
  rescue Exception => e
234
232
  Log.error("Failed to publish result of dispatched request #{request.trace}", e, :trace)
@@ -298,9 +296,9 @@ module RightScale
298
296
  # === Return
299
297
  # true:: Always return true
300
298
  def reset_stats
301
- @rejects = ActivityStats.new
302
- @requests = ActivityStats.new
303
- @exceptions = ExceptionStats.new(@agent)
299
+ @rejects = RightSupport::Stats::Activity.new
300
+ @requests = RightSupport::Stats::Activity.new
301
+ @exceptions = RightSupport::Stats::Exceptions.new(@agent)
304
302
  true
305
303
  end
306
304
 
@@ -335,17 +333,6 @@ module RightScale
335
333
  error
336
334
  end
337
335
 
338
- # Convert value to nil if equals 0
339
- #
340
- # === Parameters
341
- # value(Integer|nil):: Value to be converted
342
- #
343
- # === Return
344
- # (Integer|nil):: Converted value
345
- def nil_if_zero(value)
346
- if !value || value == 0 then nil else value end
347
- end
348
-
349
336
  end # Dispatcher
350
337
 
351
338
  end # RightScale
@@ -22,14 +22,46 @@
22
22
 
23
23
  module RightScale
24
24
 
25
+ # This is a retryable request for use when the execution of the request by the
26
+ # receiver is known to be idempotent and when there is a need to indefinitely
27
+ # pursue getting a usable response, e.g., when an instance is launching.
28
+ # It is implemented as an EM::Deferrable and as such invokes the Proc defined
29
+ # with its #callback method with the result content from a OperationResult::SUCCESS
30
+ # response, or it will invoke the Proc defined with its #errback method with error
31
+ # content if the response is an OperationResult::ERROR or CANCEL, or if the request
32
+ # has timed out. The request can be canceled with the #cancel method, or the receiver
33
+ # of the request may respond with a CANCEL result to cause the request to be canceled.
34
+ # This is useful in situations where the request is never expected to succeed
35
+ # regardless of the number of retries. By default if the response to the request
36
+ # is an OperationResult::RETRY or NON_DELIVERY indication, the request is automatically
37
+ # retried, as is also the case for an ERROR indication if the :retry_on_error option
38
+ # is specified. The retry algorithm is controlled by the :retry_delay, :retry_delay_count,
39
+ # and :max_retry_delay settings. The initial retry interval is the default or specified
40
+ # :retry_delay and this interval is used :retry_delay_count times, at which point the
41
+ # :retry_delay is doubled and the :retry_delay_count is halved. This backoff is again
42
+ # applied after the new :retry_delay_count is reached, and so on until :retry_delay
43
+ # reaches :max_retry_delay which then is used as the interval until the default or
44
+ # specified :timeout is reached. The default :timeout is 4 days.
25
45
  class IdempotentRequest
26
46
 
27
47
  include OperationResultHelper
28
48
  include EM::Deferrable
29
49
 
30
- # Wait 5 seconds before retrying in case of failure
50
+ # Default delay before initial retry in case of failure with -1 meaning no delay
31
51
  DEFAULT_RETRY_DELAY = 5
32
52
 
53
+ # Default minimum number of retries before beginning backoff
54
+ DEFAULT_RETRY_DELAY_COUNT = 60
55
+
56
+ # Maximum default delay before retry when backing off
57
+ DEFAULT_MAX_RETRY_DELAY = 60
58
+
59
+ # Factor used for exponential backoff of retry delay
60
+ RETRY_BACKOFF_FACTOR = 2
61
+
62
+ # Default timeout with -1 meaning never timeout
63
+ DEFAULT_TIMEOUT = 4 * 24 * 60 * 60
64
+
33
65
  attr_reader :raw_response
34
66
 
35
67
  # Send idempotent request
@@ -37,18 +69,31 @@ module RightScale
37
69
  # Calls deferrable callback on completion, error callback on timeout
38
70
  #
39
71
  # === Parameters
40
- # operation(String):: Request operation (i.e. '/booter/get_boot_bundle')
72
+ # operation(String):: Request operation (e.g., '/booter/get_boot_bundle')
41
73
  # payload(Hash):: Request payload
42
- # options[:retry_on_error](FalseClass|TrueClass):: Whether request should be retried
43
- # if recipient returned an error
44
- # options[:timeout](Fixnum):: Number of seconds before error callback gets called
45
- # options[:retry_delay](Fixnum):: Number of seconds before retry, defaults to 5
46
- def initialize(operation, payload, options={})
47
- raise ArgumentError.new("options[:operation] is required") unless @operation = operation
48
- raise ArgumentError.new("options[:payload] is required") unless @payload = payload
74
+ # options(Hash):: Request options
75
+ # :targets(Array):: Targets from which to randomly choose one
76
+ # :retry_on_error(Boolean):: Whether request should be retried if recipient returned an error
77
+ # :retry_delay(Fixnum):: Number of seconds delay before initial retry with -1 meaning no delay,
78
+ # defaults to DEFAULT_RETRY_DELAY
79
+ # :retry_delay_count(Fixnum):: Minimum number of retries at initial :retry_delay value before
80
+ # increasing delay exponentially and decreasing this count exponentially, defaults to
81
+ # DEFAULT_RETRY_DELAY_COUNT
82
+ # :max_retry_delay(Fixnum):: Maximum number of seconds of retry delay, defaults to DEFAULT_MAX_RETRY_DELAY
83
+ # :timeout(Fixnum):: Number of seconds with no response before error callback gets called with
84
+ # -1 meaning never, defaults to DEFAULT_TIMEOUT
85
+ #
86
+ # === Raises
87
+ # ArgumentError:: If operation or payload not specified
88
+ def initialize(operation, payload, options = {})
89
+ raise ArgumentError.new("operation is required") unless @operation = operation
90
+ raise ArgumentError.new("payload is required") unless @payload = payload
49
91
  @retry_on_error = options[:retry_on_error] || false
50
- @timeout = options[:timeout] || -1
92
+ @timeout = options[:timeout] || DEFAULT_TIMEOUT
51
93
  @retry_delay = options[:retry_delay] || DEFAULT_RETRY_DELAY
94
+ @retry_delay_count = options[:retry_delay_count] || DEFAULT_RETRY_DELAY_COUNT
95
+ @max_retry_delay = options[:max_retry_delay] || DEFAULT_MAX_RETRY_DELAY
96
+ @retries = 0
52
97
  @targets = options[:targets]
53
98
  @raw_response = nil
54
99
  @done = false
@@ -64,7 +109,7 @@ module RightScale
64
109
  if @cancel_timer.nil? && @timeout > 0
65
110
  @cancel_timer = EM::Timer.new(@timeout) do
66
111
  msg = "Request #{@operation} timed out after #{@timeout} seconds"
67
- log_info(msg)
112
+ Log.info(msg)
68
113
  cancel(msg)
69
114
  end
70
115
  end
@@ -101,7 +146,6 @@ module RightScale
101
146
  return true if @done
102
147
  @raw_response = r
103
148
  res = result_from(r)
104
- res = OperationResult.non_delivery unless res
105
149
  if res.success?
106
150
  if @cancel_timer
107
151
  @cancel_timer.cancel
@@ -110,17 +154,28 @@ module RightScale
110
154
  @done = true
111
155
  succeed(res.content)
112
156
  else
157
+ reason = res.content
113
158
  if res.non_delivery?
114
- Log.info("Request non-delivery (#{res.content}) for #{@operation}")
159
+ Log.info("Request non-delivery (#{reason}) for #{@operation}")
115
160
  elsif res.retry?
116
- Log.info("RightScale not ready when trying to request #{@operation}")
161
+ reason = (reason && !reason.empty?) ? reason : "RightScale not ready"
162
+ Log.info("Request #{@operation} failed (#{reason}) and should be retried")
163
+ elsif res.cancel?
164
+ reason = (reason && !reason.empty?) ? reason : "RightScale cannot execute request"
165
+ Log.info("Request #{@operation} canceled (#{reason})")
117
166
  else
118
- Log.info("Request #{@operation} failed (#{res.content})")
167
+ Log.info("Request #{@operation} failed (#{reason})")
119
168
  end
120
- if res.non_delivery? || res.retry? || @retry_on_error
169
+ if (res.non_delivery? || res.retry? || @retry_on_error) && !res.cancel?
121
170
  Log.info("Retrying in #{@retry_delay} seconds...")
122
171
  if @retry_delay > 0
123
- EM.add_timer(@retry_delay) { run }
172
+ this_delay = @retry_delay
173
+ if (@retries += 1) >= @retry_delay_count
174
+ @retry_delay = [@retry_delay * RETRY_BACKOFF_FACTOR, @max_retry_delay].min
175
+ @retry_delay_count = [@retry_delay_count / RETRY_BACKOFF_FACTOR, 1].max
176
+ @retries = 0
177
+ end
178
+ EM.add_timer(this_delay) { run }
124
179
  else
125
180
  EM.next_tick { run }
126
181
  end
@@ -42,7 +42,6 @@ end
42
42
  RUBY_PATCH_BASE_DIR = File.join(File.dirname(__FILE__), 'ruby_patch')
43
43
 
44
44
  require File.normalize_path(File.join(RUBY_PATCH_BASE_DIR, 'array_patch'))
45
- require File.normalize_path(File.join(RUBY_PATCH_BASE_DIR, 'string_patch'))
46
45
  require File.normalize_path(File.join(RUBY_PATCH_BASE_DIR, 'object_patch'))
47
46
  require File.normalize_path(File.join(RUBY_PATCH_BASE_DIR, 'singleton_patch'))
48
47
 
@@ -25,5 +25,4 @@ require 'rbconfig'
25
25
 
26
26
  MONKEY_PATCHES_BASE_DIR = File.join(File.dirname(__FILE__), 'monkey_patches')
27
27
 
28
- require File.normalize_path(File.join(MONKEY_PATCHES_BASE_DIR, 'amqp_patch'))
29
28
  require File.normalize_path(File.join(MONKEY_PATCHES_BASE_DIR, 'ruby_patch'))
@@ -35,6 +35,7 @@ module RightScale
35
35
  RETRY = 3
36
36
  NON_DELIVERY = 4
37
37
  MULTICAST = 5 # Deprecated for agents at version 13 or above
38
+ CANCEL = 6
38
39
 
39
40
  # Non-delivery reasons
40
41
  NON_DELIVERY_REASONS = [
@@ -81,9 +82,10 @@ module RightScale
81
82
  when SUCCESS then 'success'
82
83
  when ERROR then 'error' + (reason ? " (#{truncated_error})" : "")
83
84
  when CONTINUE then 'continue'
84
- when RETRY then 'retry'
85
+ when RETRY then 'retry' + (reason ? " (#{@content})" : "")
85
86
  when NON_DELIVERY then 'non-delivery' + (reason ? " (#{@content})" : "")
86
87
  when MULTICAST then 'multicast'
88
+ when CANCEL then 'cancel' + (reason ? " (#{@content})" : "")
87
89
  end
88
90
  end
89
91
 
@@ -126,7 +128,7 @@ module RightScale
126
128
  # Create new success status
127
129
  #
128
130
  # === Parameters
129
- # content(Object):: Any data associated with successful results - defaults to nil
131
+ # content(Object):: Any data associated with successful results, defaults to nil
130
132
  #
131
133
  # === Return
132
134
  # (OperationResult):: Corresponding result
@@ -151,7 +153,7 @@ module RightScale
151
153
  # Create new continue status
152
154
  #
153
155
  # === Parameters
154
- # content(Object):: Any data associated with continue - defaults to nil
156
+ # content(Object):: Any data associated with continue, defaults to nil
155
157
  #
156
158
  # === Return
157
159
  # (OperationResult):: Corresponding result
@@ -162,7 +164,7 @@ module RightScale
162
164
  # Create new retry status
163
165
  #
164
166
  # === Parameters
165
- # content(Object):: Any data associated with retry - defaults to nil
167
+ # content(Object):: Any data associated with retry, defaults to nil
166
168
  #
167
169
  # === Return
168
170
  # (OperationResult):: Corresponding result
@@ -193,6 +195,17 @@ module RightScale
193
195
  OperationResult.new(MULTICAST, targets)
194
196
  end
195
197
 
198
+ # Cancel request and never retry
199
+ #
200
+ # === Parameters
201
+ # content(Object):: Any data associated with cancel, defaults to nil
202
+ #
203
+ # === Return
204
+ # (OperationResult):: Corresponding result
205
+ def self.cancel(content = nil)
206
+ OperationResult.new(CANCEL, content)
207
+ end
208
+
196
209
  # Was last operation successful?
197
210
  #
198
211
  # === Return
@@ -248,6 +261,15 @@ module RightScale
248
261
  status_code == MULTICAST
249
262
  end
250
263
 
264
+ # Was last operation status CANCEL?
265
+ #
266
+ # === Return
267
+ # true:: If status is CANCEL
268
+ # false:: Otherwise
269
+ def cancel?
270
+ status_code == CANCEL
271
+ end
272
+
251
273
  # Array of serialized fields given to constructor
252
274
  def serialized_members
253
275
  [@status_code, @content]
@@ -263,6 +285,7 @@ module RightScale
263
285
  def continue_result(*args) OperationResult.continue(*args) end
264
286
  def retry_result(*args) OperationResult.retry(*args) end
265
287
  def non_delivery_result(*args) OperationResult.non_delivery(*args) end
288
+ def cancel_result(*args) OperationResult.cancel(*args) end
266
289
  def result_from(*args) OperationResult.from_results(*args) end
267
290
 
268
291
  end # OperationResultHelper