right_agent 0.14.0 → 0.16.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. data/README.rdoc +2 -0
  2. data/lib/right_agent/actors/agent_manager.rb +1 -1
  3. data/lib/right_agent/agent.rb +28 -14
  4. data/lib/right_agent/agent_config.rb +1 -1
  5. data/lib/right_agent/agent_identity.rb +4 -5
  6. data/lib/right_agent/agent_tag_manager.rb +21 -24
  7. data/lib/right_agent/core_payload_types/executable_bundle.rb +1 -1
  8. data/lib/right_agent/core_payload_types/recipe_instantiation.rb +7 -0
  9. data/lib/right_agent/core_payload_types/right_script_instantiation.rb +20 -5
  10. data/lib/right_agent/exceptions.rb +44 -1
  11. data/lib/right_agent/history.rb +4 -1
  12. data/lib/right_agent/packets.rb +2 -1
  13. data/lib/right_agent/platform/darwin.rb +6 -0
  14. data/lib/right_agent/platform/linux.rb +5 -1
  15. data/lib/right_agent/platform/windows.rb +8 -4
  16. data/lib/right_agent/scripts/stats_manager.rb +3 -3
  17. data/lib/right_agent/security/cached_certificate_store_proxy.rb +27 -13
  18. data/lib/right_agent/security/encrypted_document.rb +1 -2
  19. data/lib/right_agent/security/static_certificate_store.rb +30 -14
  20. data/lib/right_agent/sender.rb +101 -47
  21. data/lib/right_agent/serialize/secure_serializer.rb +29 -27
  22. data/lib/right_agent/serialize/secure_serializer_initializer.rb +3 -3
  23. data/lib/right_agent/serialize/serializable.rb +1 -1
  24. data/lib/right_agent/serialize/serializer.rb +15 -6
  25. data/right_agent.gemspec +4 -5
  26. data/spec/agent_spec.rb +2 -2
  27. data/spec/agent_tag_manager_spec.rb +330 -0
  28. data/spec/core_payload_types/recipe_instantiation_spec.rb +81 -0
  29. data/spec/core_payload_types/right_script_instantiation_spec.rb +79 -0
  30. data/spec/security/cached_certificate_store_proxy_spec.rb +14 -8
  31. data/spec/security/static_certificate_store_spec.rb +13 -7
  32. data/spec/sender_spec.rb +114 -17
  33. data/spec/serialize/secure_serializer_spec.rb +78 -49
  34. data/spec/serialize/serializer_spec.rb +21 -2
  35. metadata +90 -36
@@ -104,7 +104,8 @@ module RightScale
104
104
  @size = msg.size
105
105
  # For ruby 1.9 size attribute moves from front to back of packet
106
106
  re = RUBY_VERSION < "1.9.0" ? /size\xC0/ : /size\xC0$/
107
- msg.sub!(re) { |m| "size" + @size.to_msgpack }
107
+ # For msgpack 0.5.1 the to_msgpack result is a MessagePack::Packer so need to convert to string
108
+ msg = msg.to_s.sub!(re) { |m| "size" + @size.to_msgpack }
108
109
  msg
109
110
  end
110
111
 
@@ -102,6 +102,12 @@ module RightScale
102
102
  '/var/spool'
103
103
  end
104
104
 
105
+ def ssh_cfg_dir
106
+ # TODO This is a guess, but since we don't have Darwin instances
107
+ # it may need to be corrected later.
108
+ '/etc/ssh'
109
+ end
110
+
105
111
  # Cached data from applications. Such data is locally generated as a
106
112
  # result of time-consuming I/O or calculation. The application must
107
113
  # be able to regenerate or restore the data.
@@ -153,6 +153,10 @@ module RightScale
153
153
  '/var/spool'
154
154
  end
155
155
 
156
+ def ssh_cfg_dir
157
+ '/etc/ssh'
158
+ end
159
+
156
160
  # Cached data from applications. Such data is locally generated as a
157
161
  # result of time-consuming I/O or calculation. The application must
158
162
  # be able to regenerate or restore the data.
@@ -262,7 +266,7 @@ module RightScale
262
266
  # ParserError:: on failure to parse volume list
263
267
  def parse_volumes(output_text, conditions = nil)
264
268
  results = []
265
- output_text.each do |line|
269
+ output_text.lines.each do |line|
266
270
  volume = {}
267
271
  line_regex = /^([\/a-z0-9_\-\.]+):(.*)/
268
272
  volmatch = line_regex.match(line)
@@ -175,6 +175,10 @@ module RightScale
175
175
  return pretty_path(File.join(Dir::COMMON_APPDATA, 'RightScale', 'spool'))
176
176
  end
177
177
 
178
+ def ssh_cfg_dir
179
+ return pretty_path(File.join(ENV['USERPROFILE'] || temp_dir, '.ssh'))
180
+ end
181
+
178
182
  # Cache directory for the current platform
179
183
  def cache_dir
180
184
  return pretty_path(File.join(Dir::COMMON_APPDATA, 'RightScale', 'cache'))
@@ -656,7 +660,7 @@ EOF
656
660
  line_regex = nil
657
661
  header_regex = / -------- (-+) ------- ------- --- ---/
658
662
  header_match = nil
659
- output_text.each do |line|
663
+ output_text.lines.each do |line|
660
664
  line = line.chomp
661
665
  if line_regex
662
666
  if line.strip.empty?
@@ -722,7 +726,7 @@ EOF
722
726
  header_regex = / ---------- --- (-+) (-+) (-+) ------- (-+) (-+)/
723
727
  header_match = nil
724
728
  line_regex = nil
725
- output_text.each do |line|
729
+ output_text.lines.each do |line|
726
730
  line = line.chomp
727
731
  if line_regex
728
732
  if line.strip.empty?
@@ -805,7 +809,7 @@ EOF
805
809
  header_regex = / ------------- (-+) ------- -------/
806
810
  header_match = nil
807
811
  line_regex = nil
808
- output_text.each do |line|
812
+ output_text.lines.each do |line|
809
813
  line = line.chomp
810
814
  if line_regex
811
815
  if line.strip.empty?
@@ -1113,7 +1117,7 @@ EOF
1113
1117
  # the $Error list if necessary), so this is a catch-all for any
1114
1118
  # script which does not handle errors "properly".
1115
1119
  lines_after_script << "if ($NULL -eq $LastExitCode) { $LastExitCode = 0 }"
1116
- lines_after_script << "if ((0 -eq $LastExitCode) -and ($Error.Count -gt 0)) { $RS_message = 'Script exited successfully but $Error contained '+($Error.Count)+' error(s).'; Write-warning $RS_message; $LastExitCode = 1 }"
1120
+ lines_after_script << "if ((0 -eq $LastExitCode) -and ($Error.Count -gt 0)) { $RS_message = 'Script exited successfully but $Error contained '+($Error.Count)+' error(s):'; write-output $RS_message; write-output $Error; $LastExitCode = 1 }"
1117
1121
  end
1118
1122
 
1119
1123
  # ensure last exit code gets marshalled.
@@ -22,7 +22,7 @@
22
22
  #
23
23
  # Options:
24
24
  # --reset, -r As part of gathering the stats from an agent also reset the stats
25
- # --timeout, -t SEC Override default timeout in seconds to wait for a response from an agent
25
+ # --timeout, -T SEC Override default timeout in seconds to wait for a response from an agent
26
26
  # --json, -j Display the stats data in JSON format
27
27
  # --verbose, -v Log debug information
28
28
  # --cfg-dir, -c DIR Set directory containing configuration for all agents
@@ -86,7 +86,7 @@ module RightScale
86
86
  options[:reset] = true
87
87
  end
88
88
 
89
- opts.on('-t', '--timeout SEC') do |sec|
89
+ opts.on('-T', '--timeout SEC') do |sec|
90
90
  options[:timeout] = sec
91
91
  end
92
92
 
@@ -162,7 +162,7 @@ module RightScale
162
162
  client = CommandClient.new(listen_port, config_options[:cookie])
163
163
  command = {:name => :stats, :reset => options[:reset]}
164
164
  begin
165
- client.send_command(command, options[:verbose], options[:timeout]) { |r| display(agent_name, r, options) }
165
+ client.send_command(command, verbose = false, options[:timeout]) { |r| display(agent_name, r, options) }
166
166
  res = true
167
167
  rescue Exception => e
168
168
  msg = "Could not retrieve #{agent_name} agent stats: #{e}"
@@ -1,5 +1,5 @@
1
1
  #
2
- # Copyright (c) 2009-2011 RightScale Inc
2
+ # Copyright (c) 2009-2013 RightScale Inc
3
3
  #
4
4
  # Permission is hereby granted, free of charge, to any person obtaining
5
5
  # a copy of this software and associated documentation files (the
@@ -28,34 +28,48 @@ module RightScale
28
28
  # Initialize cache proxy with given certificate store
29
29
  #
30
30
  # === Parameters
31
- # store(Object):: Certificate store responding to get_recipients and
32
- # get_signer
31
+ # store(Object):: Certificate store responding to get_signer, get_target,
32
+ # and get_receiver
33
33
  def initialize(store)
34
34
  @signer_cache = CertificateCache.new
35
35
  @store = store
36
36
  end
37
37
 
38
- # Retrieve recipient certificates
38
+ # Retrieve signer certificates for use in verifying a signature
39
+ # Check cache first and cache results
40
+ #
41
+ # === Parameters
42
+ # id(String):: Serialized identity of signer
43
+ #
44
+ # === Return
45
+ # (Array|Certificate):: Signer certificate(s)
46
+ def get_signer(id)
47
+ @signer_cache.get(id) { @store.get_signer(id) }
48
+ end
49
+
50
+ # Retrieve certificates of target for encryption
39
51
  # Results are not cached
40
52
  #
41
53
  # === Parameters
42
- # packet(RightScale::Packet):: Packet containing recipient identity, ignored
54
+ # packet(RightScale::Packet):: Packet containing target identity
43
55
  #
44
56
  # === Return
45
- # (Array):: Recipient certificates
46
- def get_recipients(obj)
47
- @store.get_recipients(obj)
57
+ # (Array|Certificate):: Target certificate(s)
58
+ def get_target(obj)
59
+ @store.get_target(obj)
48
60
  end
49
61
 
50
- # Check cache for signer certificate
62
+ # Retrieve receiver's certificate and key for decryption
63
+ # Results are not cached
51
64
  #
52
65
  # === Parameters
53
- # id(String):: Serialized identity of signer
66
+ # id(String|nil):: Optional identifier of source of data for use
67
+ # in determining who is the receiver
54
68
  #
55
69
  # === Return
56
- # (Array):: Signer certificates
57
- def get_signer(id)
58
- @signer_cache.get(id) { @store.get_signer(id) }
70
+ # (Array):: Certificate and key
71
+ def get_receiver(id)
72
+ @store.get_receiver(id)
59
73
  end
60
74
 
61
75
  end # CachedCertificateStoreProxy
@@ -33,8 +33,7 @@ module RightScale
33
33
  #
34
34
  # === Parameters
35
35
  # data(String):: Data to be encrypted
36
- # certs(Array):: Recipient certificates (certificates corresponding to private
37
- # keys that may be used to decrypt data)
36
+ # certs(Array|Certificate):: Target recipient certificates used to encrypt data
38
37
  # cipher(Cipher):: Cipher used for encryption, AES 256 CBC by default
39
38
  def initialize(data, certs, cipher = 'AES-256-CBC')
40
39
  cipher = OpenSSL::Cipher::Cipher.new(cipher)
@@ -1,5 +1,5 @@
1
1
  #
2
- # Copyright (c) 2009-2011 RightScale Inc
2
+ # Copyright (c) 2009-2013 RightScale Inc
3
3
  #
4
4
  # Permission is hereby granted, free of charge, to any person obtaining
5
5
  # a copy of this software and associated documentation files (the
@@ -22,46 +22,62 @@
22
22
 
23
23
  module RightScale
24
24
 
25
- # Simple certificate store, serves a static set of certificates
25
+ # Simple certificate store that serves a static set of certificates and one key
26
26
  class StaticCertificateStore
27
27
 
28
28
  # Initialize store
29
29
  #
30
30
  # === Parameters
31
+ # receiver_cert(Certificate):: Certificate for decrypting serialized data being received
32
+ # receiver_key(RsaKeyPair):: Key corresponding to specified cert
31
33
  # signer_certs(Array|Certificate):: Signer certificate(s) used when loading data to
32
34
  # check the digital signature. The signature associated with the serialized data
33
35
  # needs to match with one of the signer certificates for loading to succeed.
34
- # recipients_certs(Array|Certificate):: Recipient certificate(s) used when serializing
36
+ # target_certs(Array|Certificate):: Target certificate(s) used when serializing
35
37
  # data for encryption. Loading the data can only be done through serializers that
36
- # have been initialized with a certificate that's in the recipient certificates
38
+ # have been initialized with a certificate that's in the target certificates
37
39
  # if encryption is enabled.
38
- def initialize(signer_certs, recipients_certs)
40
+ def initialize(receiver_cert, receiver_key, signer_certs, target_certs)
41
+ @receiver_cert = receiver_cert
42
+ @receiver_key = receiver_key
39
43
  signer_certs = [ signer_certs ] unless signer_certs.respond_to?(:each)
40
44
  @signer_certs = signer_certs
41
- recipients_certs = [ recipients_certs ] unless recipients_certs.respond_to?(:each)
42
- @recipients_certs = recipients_certs
45
+ target_certs = [ target_certs ] unless target_certs.respond_to?(:each)
46
+ @target_certs = target_certs
43
47
  end
44
48
 
45
- # Retrieve signer certificates
49
+ # Retrieve signer certificates for use in verifying a signature
46
50
  #
47
51
  # === Parameters
48
52
  # id(String):: Serialized identity of signer, ignored
49
53
  #
50
54
  # === Return
51
- # (Array):: Signer certificates
55
+ # (Array|Certificate):: Signer certificates
52
56
  def get_signer(id)
53
57
  @signer_certs
54
58
  end
55
59
 
56
- # Retrieve recipient certificates that will be able to decrypt the serialized data
60
+ # Retrieve certificates of target for encryption
57
61
  #
58
62
  # === Parameters
59
- # packet(RightScale::Packet):: Packet containing recipient identity, ignored
63
+ # packet(RightScale::Packet):: Packet containing target identity, ignored
60
64
  #
61
65
  # === Return
62
- # (Array):: Recipient certificates
63
- def get_recipients(packet)
64
- @recipients_certs
66
+ # (Array|Certificate):: Target certificates
67
+ def get_target(packet)
68
+ @target_certs
69
+ end
70
+
71
+ # Retrieve receiver's certificate and key for decryption
72
+ #
73
+ # === Parameters
74
+ # id(String|nil):: Optional identifier of source of data for use
75
+ # in determining who is the receiver, ignored
76
+ #
77
+ # === Return
78
+ # (Array):: Certificate and key
79
+ def get_receiver(id)
80
+ [@receiver_cert, @receiver_key]
65
81
  end
66
82
 
67
83
  end # StaticCertificateStore
@@ -46,11 +46,15 @@ module RightScale
46
46
  # (String) Token for parent request in a retry situation
47
47
  attr_accessor :retry_parent
48
48
 
49
+ # (String) Non-delivery reason if any
50
+ attr_accessor :non_delivery
51
+
49
52
  def initialize(kind, receive_time, response_handler)
50
53
  @kind = kind
51
54
  @receive_time = receive_time
52
55
  @response_handler = response_handler
53
56
  @retry_parent = nil
57
+ @non_delivery = nil
54
58
  end
55
59
 
56
60
  end # PendingRequest
@@ -808,7 +812,7 @@ module RightScale
808
812
  build_and_send_packet(:send_persistent_request, type, payload, target, callback)
809
813
  end
810
814
 
811
- # Build packet
815
+ # Build packet or queue it if offline
812
816
  #
813
817
  # === Parameters
814
818
  # kind(Symbol):: Kind of send request: :send_push, :send_persistent_push, :send_retryable_request,
@@ -825,30 +829,69 @@ module RightScale
825
829
  # callback(Boolean):: Whether this request has an associated response callback
826
830
  #
827
831
  # === Return
828
- # (Push|Request):: Packet created
832
+ # (Push|Request|NilClass):: Packet created, or nil if queued instead
833
+ #
834
+ # === Raise
835
+ # ArgumentError:: If target is invalid
829
836
  def build_packet(kind, type, payload, target, callback = false)
830
- kind_str = kind.to_s
831
- persistent = !!(kind_str =~ /persistent/)
832
- if kind_str =~ /push/
833
- packet = Push.new(type, payload)
834
- packet.selector = target[:selector] || :any if target.is_a?(Hash)
835
- packet.confirm = true if callback
837
+ validate_target(target, !!(kind.to_s =~ /push/))
838
+ if should_queue?
839
+ @offline_handler.queue_request(kind, type, payload, target, callback)
840
+ nil
836
841
  else
837
- packet = Request.new(type, payload)
838
- ttl = @options[:time_to_live]
839
- packet.expires_at = Time.now.to_i + ttl if !persistent && ttl && ttl != 0
840
- packet.selector = :any
842
+ kind_str = kind.to_s
843
+ persistent = !!(kind_str =~ /persistent/)
844
+ if kind_str =~ /push/
845
+ packet = Push.new(type, payload)
846
+ packet.selector = target[:selector] || :any if target.is_a?(Hash)
847
+ packet.confirm = true if callback
848
+ else
849
+ packet = Request.new(type, payload)
850
+ ttl = @options[:time_to_live]
851
+ packet.expires_at = Time.now.to_i + ttl if !persistent && ttl && ttl != 0
852
+ packet.selector = :any
853
+ end
854
+ packet.from = @identity
855
+ packet.token = AgentIdentity.generate
856
+ packet.persistent = persistent
857
+ if target.is_a?(Hash)
858
+ packet.tags = target[:tags] || []
859
+ packet.scope = target[:scope]
860
+ else
861
+ packet.target = target
862
+ end
863
+ packet
841
864
  end
842
- packet.from = @identity
843
- packet.token = AgentIdentity.generate
844
- packet.persistent = persistent
845
- if target.is_a?(Hash)
846
- packet.tags = target[:tags] || []
847
- packet.scope = target[:scope]
848
- else
849
- packet.target = target
865
+ end
866
+
867
+ # Send packet
868
+ #
869
+ # === Parameters
870
+ # kind(Symbol):: Kind of send request: :send_push, :send_persistent_push, :send_retryable_request,
871
+ # or :send_persistent_request
872
+ # packet(Push|Request|NilClass):: Packet to send; nothing sent if nil
873
+ # callback(Proc|nil):: Block used to process routing response
874
+ #
875
+ # === Return
876
+ # true:: Always return true
877
+ #
878
+ # === Raise
879
+ # SendFailure:: If publishing of request fails unexpectedly
880
+ # TemporarilyOffline:: If cannot publish request because currently not connected
881
+ # to any brokers and offline queueing is disabled
882
+ def send_packet(kind, packet, callback)
883
+ if packet
884
+ method = packet.type.split('/').last
885
+ received_at = @request_stats.update(method, packet.token)
886
+ @request_kind_stats.update((packet.selector == :all ? kind.to_s.sub(/push/, "fanout") : kind.to_s)[5..-1])
887
+ @pending_requests[packet.token] = PendingRequest.new(kind, received_at, callback) if callback
888
+ if !packet.persistent && kind.to_s =~ /request/
889
+ publish_with_timeout_retry(packet, packet.token)
890
+ else
891
+ publish(packet)
892
+ end
850
893
  end
851
- packet
894
+ true
852
895
  end
853
896
 
854
897
  # Handle response to a request
@@ -873,11 +916,23 @@ module RightScale
873
916
  end
874
917
 
875
918
  if handler = @pending_requests[token]
876
- if result && result.non_delivery? && handler.kind == :send_retryable_request &&
877
- [OperationResult::TARGET_NOT_CONNECTED, OperationResult::TTL_EXPIRATION].include?(result.content)
878
- # Log and ignore so that timeout retry mechanism continues
879
- # Leave purging of associated request until final response, i.e., success response or retry timeout
880
- Log.info("Non-delivery of <#{token}> because #{result.content}")
919
+ if result && result.non_delivery? && handler.kind == :send_retryable_request
920
+ if [OperationResult::TARGET_NOT_CONNECTED, OperationResult::TTL_EXPIRATION].include?(result.content)
921
+ # Log and temporarily ignore so that timeout retry mechanism continues, but save reason for use below if timeout
922
+ # Leave purging of associated request until final response, i.e., success response or retry timeout
923
+ if parent = handler.retry_parent
924
+ @pending_requests[parent].non_delivery = result.content
925
+ else
926
+ handler.non_delivery = result.content
927
+ end
928
+ Log.info("Non-delivery of <#{token}> because #{result.content}")
929
+ elsif result.content == OperationResult::RETRY_TIMEOUT && handler.non_delivery
930
+ # Request timed out but due to another non-delivery reason, so use that reason since more germane
931
+ response.results = OperationResult.non_delivery(handler.non_delivery)
932
+ deliver(response, handler)
933
+ else
934
+ deliver(response, handler)
935
+ end
881
936
  else
882
937
  deliver(response, handler)
883
938
  end
@@ -1085,21 +1140,8 @@ module RightScale
1085
1140
  # TemporarilyOffline:: If cannot publish request because currently not connected
1086
1141
  # to any brokers and offline queueing is disabled
1087
1142
  def build_and_send_packet(kind, type, payload, target, callback)
1088
- validate_target(target, allow_selector = !!(kind.to_s =~ /push/))
1089
- if should_queue?
1090
- @offline_handler.queue_request(kind, type, payload, target, callback)
1091
- else
1092
- packet = build_packet(kind, type, payload, target, callback)
1093
- method = type.split('/').last
1094
- received_at = @request_stats.update(method, packet.token)
1095
- @request_kind_stats.update((packet.selector == :all ? kind.to_s.sub(/push/, "fanout") : kind.to_s)[5..-1])
1096
- @pending_requests[packet.token] = PendingRequest.new(kind, received_at, callback) if callback
1097
- if !packet.persistent && kind.to_s =~ /request/
1098
- publish_with_timeout_retry(packet, packet.token)
1099
- else
1100
- publish(packet)
1101
- end
1102
- end
1143
+ packet = build_packet(kind, type, payload, target, callback)
1144
+ send_packet(kind, packet, callback)
1103
1145
  true
1104
1146
  end
1105
1147
 
@@ -1123,12 +1165,12 @@ module RightScale
1123
1165
  @broker.publish(exchange, request, :persistent => request.persistent, :mandatory => true,
1124
1166
  :log_filter => [:tags, :target, :tries, :persistent], :brokers => ids)
1125
1167
  rescue RightAMQP::HABrokerClient::NoConnectedBrokers => e
1126
- msg = "Failed to publish request #{request.to_s([:tags, :target, :tries])}"
1168
+ msg = "Failed to publish request #{request.trace} #{request.type}"
1127
1169
  Log.error(msg, e)
1128
1170
  @send_failure_stats.update("NoConnectedBrokers")
1129
1171
  raise TemporarilyOffline.new(msg + " (#{e.class}: #{e.message})")
1130
1172
  rescue Exception => e
1131
- msg = "Failed to publish request #{request.to_s([:tags, :target, :tries])}"
1173
+ msg = "Failed to publish request #{request.trace} #{request.type}"
1132
1174
  Log.error(msg, e, :trace)
1133
1175
  @send_failure_stats.update(e.class.name)
1134
1176
  @exception_stats.track("publish", e, request)
@@ -1156,7 +1198,7 @@ module RightScale
1156
1198
  def publish_with_timeout_retry(request, parent, count = 0, multiplier = 1, elapsed = 0, broker_ids = nil)
1157
1199
  published_broker_ids = publish(request, broker_ids)
1158
1200
 
1159
- if @retry_interval && @retry_timeout && parent && !published_broker_ids.empty?
1201
+ if @retry_interval && @retry_timeout && parent
1160
1202
  interval = [(@retry_interval * multiplier) + (@request_stats.avg_duration || 0), @retry_timeout - elapsed].min
1161
1203
  EM.add_timer(interval) do
1162
1204
  begin
@@ -1173,15 +1215,27 @@ module RightScale
1173
1215
  broker_ids.push(broker_ids.shift))
1174
1216
  @retry_stats.update(request.type.split('/').last)
1175
1217
  else
1176
- Log.warning("RE-SEND TIMEOUT after #{elapsed.to_i} seconds for #{request.to_s([:tags, :target, :tries])}")
1218
+ Log.warning("RE-SEND TIMEOUT after #{elapsed.to_i} seconds for #{request.trace} #{request.type}")
1177
1219
  result = OperationResult.non_delivery(OperationResult::RETRY_TIMEOUT)
1178
1220
  @non_delivery_stats.update(result.content)
1179
1221
  handle_response(Result.new(request.token, request.reply_to, result, @identity))
1180
1222
  end
1181
- @connectivity_checker.check(published_broker_ids.first) if count == 1
1223
+ @connectivity_checker.check(published_broker_ids.first) if count == 1 && !published_broker_ids.empty?
1182
1224
  end
1225
+ rescue TemporarilyOffline => e
1226
+ # Send retry response so that requester, e.g., IdempotentRequest, can retry
1227
+ Log.error("Failed retry for #{request.trace} #{request.type} because temporarily offline")
1228
+ result = OperationResult.retry("lost connectivity")
1229
+ handle_response(Result.new(request.token, request.reply_to, result, @identity))
1230
+ rescue SendFailure => e
1231
+ # Send non-delivery response so that requester, e.g., IdempotentRequest, can retry
1232
+ Log.error("Failed retry for #{request.trace} #{request.type} because of send failure")
1233
+ result = OperationResult.non_delivery("retry failed")
1234
+ handle_response(Result.new(request.token, request.reply_to, result, @identity))
1183
1235
  rescue Exception => e
1184
- Log.error("Failed retry for #{request.token}", e, :trace)
1236
+ # Not sending a response here because something more basic is broken in the retry
1237
+ # mechanism and don't want an error response to preempt a delayed actual response
1238
+ Log.error("Failed retry for #{request.trace} #{request.type} without responding", e, :trace)
1185
1239
  @exception_stats.track("retry", e, request)
1186
1240
  end
1187
1241
  end