right_agent 0.14.0 → 0.16.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README.rdoc +2 -0
- data/lib/right_agent/actors/agent_manager.rb +1 -1
- data/lib/right_agent/agent.rb +28 -14
- data/lib/right_agent/agent_config.rb +1 -1
- data/lib/right_agent/agent_identity.rb +4 -5
- data/lib/right_agent/agent_tag_manager.rb +21 -24
- data/lib/right_agent/core_payload_types/executable_bundle.rb +1 -1
- data/lib/right_agent/core_payload_types/recipe_instantiation.rb +7 -0
- data/lib/right_agent/core_payload_types/right_script_instantiation.rb +20 -5
- data/lib/right_agent/exceptions.rb +44 -1
- data/lib/right_agent/history.rb +4 -1
- data/lib/right_agent/packets.rb +2 -1
- data/lib/right_agent/platform/darwin.rb +6 -0
- data/lib/right_agent/platform/linux.rb +5 -1
- data/lib/right_agent/platform/windows.rb +8 -4
- data/lib/right_agent/scripts/stats_manager.rb +3 -3
- data/lib/right_agent/security/cached_certificate_store_proxy.rb +27 -13
- data/lib/right_agent/security/encrypted_document.rb +1 -2
- data/lib/right_agent/security/static_certificate_store.rb +30 -14
- data/lib/right_agent/sender.rb +101 -47
- data/lib/right_agent/serialize/secure_serializer.rb +29 -27
- data/lib/right_agent/serialize/secure_serializer_initializer.rb +3 -3
- data/lib/right_agent/serialize/serializable.rb +1 -1
- data/lib/right_agent/serialize/serializer.rb +15 -6
- data/right_agent.gemspec +4 -5
- data/spec/agent_spec.rb +2 -2
- data/spec/agent_tag_manager_spec.rb +330 -0
- data/spec/core_payload_types/recipe_instantiation_spec.rb +81 -0
- data/spec/core_payload_types/right_script_instantiation_spec.rb +79 -0
- data/spec/security/cached_certificate_store_proxy_spec.rb +14 -8
- data/spec/security/static_certificate_store_spec.rb +13 -7
- data/spec/sender_spec.rb +114 -17
- data/spec/serialize/secure_serializer_spec.rb +78 -49
- data/spec/serialize/serializer_spec.rb +21 -2
- metadata +90 -36
data/lib/right_agent/packets.rb
CHANGED
@@ -104,7 +104,8 @@ module RightScale
|
|
104
104
|
@size = msg.size
|
105
105
|
# For ruby 1.9 size attribute moves from front to back of packet
|
106
106
|
re = RUBY_VERSION < "1.9.0" ? /size\xC0/ : /size\xC0$/
|
107
|
-
|
107
|
+
# For msgpack 0.5.1 the to_msgpack result is a MessagePack::Packer so need to convert to string
|
108
|
+
msg = msg.to_s.sub!(re) { |m| "size" + @size.to_msgpack }
|
108
109
|
msg
|
109
110
|
end
|
110
111
|
|
@@ -102,6 +102,12 @@ module RightScale
|
|
102
102
|
'/var/spool'
|
103
103
|
end
|
104
104
|
|
105
|
+
def ssh_cfg_dir
|
106
|
+
# TODO This is a guess, but since we don't have Darwin instances
|
107
|
+
# it may need to be corrected later.
|
108
|
+
'/etc/ssh'
|
109
|
+
end
|
110
|
+
|
105
111
|
# Cached data from applications. Such data is locally generated as a
|
106
112
|
# result of time-consuming I/O or calculation. The application must
|
107
113
|
# be able to regenerate or restore the data.
|
@@ -153,6 +153,10 @@ module RightScale
|
|
153
153
|
'/var/spool'
|
154
154
|
end
|
155
155
|
|
156
|
+
def ssh_cfg_dir
|
157
|
+
'/etc/ssh'
|
158
|
+
end
|
159
|
+
|
156
160
|
# Cached data from applications. Such data is locally generated as a
|
157
161
|
# result of time-consuming I/O or calculation. The application must
|
158
162
|
# be able to regenerate or restore the data.
|
@@ -262,7 +266,7 @@ module RightScale
|
|
262
266
|
# ParserError:: on failure to parse volume list
|
263
267
|
def parse_volumes(output_text, conditions = nil)
|
264
268
|
results = []
|
265
|
-
output_text.each do |line|
|
269
|
+
output_text.lines.each do |line|
|
266
270
|
volume = {}
|
267
271
|
line_regex = /^([\/a-z0-9_\-\.]+):(.*)/
|
268
272
|
volmatch = line_regex.match(line)
|
@@ -175,6 +175,10 @@ module RightScale
|
|
175
175
|
return pretty_path(File.join(Dir::COMMON_APPDATA, 'RightScale', 'spool'))
|
176
176
|
end
|
177
177
|
|
178
|
+
def ssh_cfg_dir
|
179
|
+
return pretty_path(File.join(ENV['USERPROFILE'] || temp_dir, '.ssh'))
|
180
|
+
end
|
181
|
+
|
178
182
|
# Cache directory for the current platform
|
179
183
|
def cache_dir
|
180
184
|
return pretty_path(File.join(Dir::COMMON_APPDATA, 'RightScale', 'cache'))
|
@@ -656,7 +660,7 @@ EOF
|
|
656
660
|
line_regex = nil
|
657
661
|
header_regex = / -------- (-+) ------- ------- --- ---/
|
658
662
|
header_match = nil
|
659
|
-
output_text.each do |line|
|
663
|
+
output_text.lines.each do |line|
|
660
664
|
line = line.chomp
|
661
665
|
if line_regex
|
662
666
|
if line.strip.empty?
|
@@ -722,7 +726,7 @@ EOF
|
|
722
726
|
header_regex = / ---------- --- (-+) (-+) (-+) ------- (-+) (-+)/
|
723
727
|
header_match = nil
|
724
728
|
line_regex = nil
|
725
|
-
output_text.each do |line|
|
729
|
+
output_text.lines.each do |line|
|
726
730
|
line = line.chomp
|
727
731
|
if line_regex
|
728
732
|
if line.strip.empty?
|
@@ -805,7 +809,7 @@ EOF
|
|
805
809
|
header_regex = / ------------- (-+) ------- -------/
|
806
810
|
header_match = nil
|
807
811
|
line_regex = nil
|
808
|
-
output_text.each do |line|
|
812
|
+
output_text.lines.each do |line|
|
809
813
|
line = line.chomp
|
810
814
|
if line_regex
|
811
815
|
if line.strip.empty?
|
@@ -1113,7 +1117,7 @@ EOF
|
|
1113
1117
|
# the $Error list if necessary), so this is a catch-all for any
|
1114
1118
|
# script which does not handle errors "properly".
|
1115
1119
|
lines_after_script << "if ($NULL -eq $LastExitCode) { $LastExitCode = 0 }"
|
1116
|
-
lines_after_script << "if ((0 -eq $LastExitCode) -and ($Error.Count -gt 0)) { $RS_message = 'Script exited successfully but $Error contained '+($Error.Count)+' error(s)
|
1120
|
+
lines_after_script << "if ((0 -eq $LastExitCode) -and ($Error.Count -gt 0)) { $RS_message = 'Script exited successfully but $Error contained '+($Error.Count)+' error(s):'; write-output $RS_message; write-output $Error; $LastExitCode = 1 }"
|
1117
1121
|
end
|
1118
1122
|
|
1119
1123
|
# ensure last exit code gets marshalled.
|
@@ -22,7 +22,7 @@
|
|
22
22
|
#
|
23
23
|
# Options:
|
24
24
|
# --reset, -r As part of gathering the stats from an agent also reset the stats
|
25
|
-
# --timeout, -
|
25
|
+
# --timeout, -T SEC Override default timeout in seconds to wait for a response from an agent
|
26
26
|
# --json, -j Display the stats data in JSON format
|
27
27
|
# --verbose, -v Log debug information
|
28
28
|
# --cfg-dir, -c DIR Set directory containing configuration for all agents
|
@@ -86,7 +86,7 @@ module RightScale
|
|
86
86
|
options[:reset] = true
|
87
87
|
end
|
88
88
|
|
89
|
-
opts.on('-
|
89
|
+
opts.on('-T', '--timeout SEC') do |sec|
|
90
90
|
options[:timeout] = sec
|
91
91
|
end
|
92
92
|
|
@@ -162,7 +162,7 @@ module RightScale
|
|
162
162
|
client = CommandClient.new(listen_port, config_options[:cookie])
|
163
163
|
command = {:name => :stats, :reset => options[:reset]}
|
164
164
|
begin
|
165
|
-
client.send_command(command,
|
165
|
+
client.send_command(command, verbose = false, options[:timeout]) { |r| display(agent_name, r, options) }
|
166
166
|
res = true
|
167
167
|
rescue Exception => e
|
168
168
|
msg = "Could not retrieve #{agent_name} agent stats: #{e}"
|
@@ -1,5 +1,5 @@
|
|
1
1
|
#
|
2
|
-
# Copyright (c) 2009-
|
2
|
+
# Copyright (c) 2009-2013 RightScale Inc
|
3
3
|
#
|
4
4
|
# Permission is hereby granted, free of charge, to any person obtaining
|
5
5
|
# a copy of this software and associated documentation files (the
|
@@ -28,34 +28,48 @@ module RightScale
|
|
28
28
|
# Initialize cache proxy with given certificate store
|
29
29
|
#
|
30
30
|
# === Parameters
|
31
|
-
# store(Object):: Certificate store responding to
|
32
|
-
#
|
31
|
+
# store(Object):: Certificate store responding to get_signer, get_target,
|
32
|
+
# and get_receiver
|
33
33
|
def initialize(store)
|
34
34
|
@signer_cache = CertificateCache.new
|
35
35
|
@store = store
|
36
36
|
end
|
37
37
|
|
38
|
-
# Retrieve
|
38
|
+
# Retrieve signer certificates for use in verifying a signature
|
39
|
+
# Check cache first and cache results
|
40
|
+
#
|
41
|
+
# === Parameters
|
42
|
+
# id(String):: Serialized identity of signer
|
43
|
+
#
|
44
|
+
# === Return
|
45
|
+
# (Array|Certificate):: Signer certificate(s)
|
46
|
+
def get_signer(id)
|
47
|
+
@signer_cache.get(id) { @store.get_signer(id) }
|
48
|
+
end
|
49
|
+
|
50
|
+
# Retrieve certificates of target for encryption
|
39
51
|
# Results are not cached
|
40
52
|
#
|
41
53
|
# === Parameters
|
42
|
-
# packet(RightScale::Packet):: Packet containing
|
54
|
+
# packet(RightScale::Packet):: Packet containing target identity
|
43
55
|
#
|
44
56
|
# === Return
|
45
|
-
# (Array)::
|
46
|
-
def
|
47
|
-
@store.
|
57
|
+
# (Array|Certificate):: Target certificate(s)
|
58
|
+
def get_target(obj)
|
59
|
+
@store.get_target(obj)
|
48
60
|
end
|
49
61
|
|
50
|
-
#
|
62
|
+
# Retrieve receiver's certificate and key for decryption
|
63
|
+
# Results are not cached
|
51
64
|
#
|
52
65
|
# === Parameters
|
53
|
-
# id(String)::
|
66
|
+
# id(String|nil):: Optional identifier of source of data for use
|
67
|
+
# in determining who is the receiver
|
54
68
|
#
|
55
69
|
# === Return
|
56
|
-
# (Array)::
|
57
|
-
def
|
58
|
-
@
|
70
|
+
# (Array):: Certificate and key
|
71
|
+
def get_receiver(id)
|
72
|
+
@store.get_receiver(id)
|
59
73
|
end
|
60
74
|
|
61
75
|
end # CachedCertificateStoreProxy
|
@@ -33,8 +33,7 @@ module RightScale
|
|
33
33
|
#
|
34
34
|
# === Parameters
|
35
35
|
# data(String):: Data to be encrypted
|
36
|
-
# certs(Array)::
|
37
|
-
# keys that may be used to decrypt data)
|
36
|
+
# certs(Array|Certificate):: Target recipient certificates used to encrypt data
|
38
37
|
# cipher(Cipher):: Cipher used for encryption, AES 256 CBC by default
|
39
38
|
def initialize(data, certs, cipher = 'AES-256-CBC')
|
40
39
|
cipher = OpenSSL::Cipher::Cipher.new(cipher)
|
@@ -1,5 +1,5 @@
|
|
1
1
|
#
|
2
|
-
# Copyright (c) 2009-
|
2
|
+
# Copyright (c) 2009-2013 RightScale Inc
|
3
3
|
#
|
4
4
|
# Permission is hereby granted, free of charge, to any person obtaining
|
5
5
|
# a copy of this software and associated documentation files (the
|
@@ -22,46 +22,62 @@
|
|
22
22
|
|
23
23
|
module RightScale
|
24
24
|
|
25
|
-
# Simple certificate store
|
25
|
+
# Simple certificate store that serves a static set of certificates and one key
|
26
26
|
class StaticCertificateStore
|
27
27
|
|
28
28
|
# Initialize store
|
29
29
|
#
|
30
30
|
# === Parameters
|
31
|
+
# receiver_cert(Certificate):: Certificate for decrypting serialized data being received
|
32
|
+
# receiver_key(RsaKeyPair):: Key corresponding to specified cert
|
31
33
|
# signer_certs(Array|Certificate):: Signer certificate(s) used when loading data to
|
32
34
|
# check the digital signature. The signature associated with the serialized data
|
33
35
|
# needs to match with one of the signer certificates for loading to succeed.
|
34
|
-
#
|
36
|
+
# target_certs(Array|Certificate):: Target certificate(s) used when serializing
|
35
37
|
# data for encryption. Loading the data can only be done through serializers that
|
36
|
-
# have been initialized with a certificate that's in the
|
38
|
+
# have been initialized with a certificate that's in the target certificates
|
37
39
|
# if encryption is enabled.
|
38
|
-
def initialize(signer_certs,
|
40
|
+
def initialize(receiver_cert, receiver_key, signer_certs, target_certs)
|
41
|
+
@receiver_cert = receiver_cert
|
42
|
+
@receiver_key = receiver_key
|
39
43
|
signer_certs = [ signer_certs ] unless signer_certs.respond_to?(:each)
|
40
44
|
@signer_certs = signer_certs
|
41
|
-
|
42
|
-
@
|
45
|
+
target_certs = [ target_certs ] unless target_certs.respond_to?(:each)
|
46
|
+
@target_certs = target_certs
|
43
47
|
end
|
44
48
|
|
45
|
-
# Retrieve signer certificates
|
49
|
+
# Retrieve signer certificates for use in verifying a signature
|
46
50
|
#
|
47
51
|
# === Parameters
|
48
52
|
# id(String):: Serialized identity of signer, ignored
|
49
53
|
#
|
50
54
|
# === Return
|
51
|
-
# (Array):: Signer certificates
|
55
|
+
# (Array|Certificate):: Signer certificates
|
52
56
|
def get_signer(id)
|
53
57
|
@signer_certs
|
54
58
|
end
|
55
59
|
|
56
|
-
# Retrieve
|
60
|
+
# Retrieve certificates of target for encryption
|
57
61
|
#
|
58
62
|
# === Parameters
|
59
|
-
# packet(RightScale::Packet):: Packet containing
|
63
|
+
# packet(RightScale::Packet):: Packet containing target identity, ignored
|
60
64
|
#
|
61
65
|
# === Return
|
62
|
-
# (Array)::
|
63
|
-
def
|
64
|
-
@
|
66
|
+
# (Array|Certificate):: Target certificates
|
67
|
+
def get_target(packet)
|
68
|
+
@target_certs
|
69
|
+
end
|
70
|
+
|
71
|
+
# Retrieve receiver's certificate and key for decryption
|
72
|
+
#
|
73
|
+
# === Parameters
|
74
|
+
# id(String|nil):: Optional identifier of source of data for use
|
75
|
+
# in determining who is the receiver, ignored
|
76
|
+
#
|
77
|
+
# === Return
|
78
|
+
# (Array):: Certificate and key
|
79
|
+
def get_receiver(id)
|
80
|
+
[@receiver_cert, @receiver_key]
|
65
81
|
end
|
66
82
|
|
67
83
|
end # StaticCertificateStore
|
data/lib/right_agent/sender.rb
CHANGED
@@ -46,11 +46,15 @@ module RightScale
|
|
46
46
|
# (String) Token for parent request in a retry situation
|
47
47
|
attr_accessor :retry_parent
|
48
48
|
|
49
|
+
# (String) Non-delivery reason if any
|
50
|
+
attr_accessor :non_delivery
|
51
|
+
|
49
52
|
def initialize(kind, receive_time, response_handler)
|
50
53
|
@kind = kind
|
51
54
|
@receive_time = receive_time
|
52
55
|
@response_handler = response_handler
|
53
56
|
@retry_parent = nil
|
57
|
+
@non_delivery = nil
|
54
58
|
end
|
55
59
|
|
56
60
|
end # PendingRequest
|
@@ -808,7 +812,7 @@ module RightScale
|
|
808
812
|
build_and_send_packet(:send_persistent_request, type, payload, target, callback)
|
809
813
|
end
|
810
814
|
|
811
|
-
# Build packet
|
815
|
+
# Build packet or queue it if offline
|
812
816
|
#
|
813
817
|
# === Parameters
|
814
818
|
# kind(Symbol):: Kind of send request: :send_push, :send_persistent_push, :send_retryable_request,
|
@@ -825,30 +829,69 @@ module RightScale
|
|
825
829
|
# callback(Boolean):: Whether this request has an associated response callback
|
826
830
|
#
|
827
831
|
# === Return
|
828
|
-
# (Push|Request):: Packet created
|
832
|
+
# (Push|Request|NilClass):: Packet created, or nil if queued instead
|
833
|
+
#
|
834
|
+
# === Raise
|
835
|
+
# ArgumentError:: If target is invalid
|
829
836
|
def build_packet(kind, type, payload, target, callback = false)
|
830
|
-
|
831
|
-
|
832
|
-
|
833
|
-
|
834
|
-
packet.selector = target[:selector] || :any if target.is_a?(Hash)
|
835
|
-
packet.confirm = true if callback
|
837
|
+
validate_target(target, !!(kind.to_s =~ /push/))
|
838
|
+
if should_queue?
|
839
|
+
@offline_handler.queue_request(kind, type, payload, target, callback)
|
840
|
+
nil
|
836
841
|
else
|
837
|
-
|
838
|
-
|
839
|
-
|
840
|
-
|
842
|
+
kind_str = kind.to_s
|
843
|
+
persistent = !!(kind_str =~ /persistent/)
|
844
|
+
if kind_str =~ /push/
|
845
|
+
packet = Push.new(type, payload)
|
846
|
+
packet.selector = target[:selector] || :any if target.is_a?(Hash)
|
847
|
+
packet.confirm = true if callback
|
848
|
+
else
|
849
|
+
packet = Request.new(type, payload)
|
850
|
+
ttl = @options[:time_to_live]
|
851
|
+
packet.expires_at = Time.now.to_i + ttl if !persistent && ttl && ttl != 0
|
852
|
+
packet.selector = :any
|
853
|
+
end
|
854
|
+
packet.from = @identity
|
855
|
+
packet.token = AgentIdentity.generate
|
856
|
+
packet.persistent = persistent
|
857
|
+
if target.is_a?(Hash)
|
858
|
+
packet.tags = target[:tags] || []
|
859
|
+
packet.scope = target[:scope]
|
860
|
+
else
|
861
|
+
packet.target = target
|
862
|
+
end
|
863
|
+
packet
|
841
864
|
end
|
842
|
-
|
843
|
-
|
844
|
-
|
845
|
-
|
846
|
-
|
847
|
-
|
848
|
-
|
849
|
-
|
865
|
+
end
|
866
|
+
|
867
|
+
# Send packet
|
868
|
+
#
|
869
|
+
# === Parameters
|
870
|
+
# kind(Symbol):: Kind of send request: :send_push, :send_persistent_push, :send_retryable_request,
|
871
|
+
# or :send_persistent_request
|
872
|
+
# packet(Push|Request|NilClass):: Packet to send; nothing sent if nil
|
873
|
+
# callback(Proc|nil):: Block used to process routing response
|
874
|
+
#
|
875
|
+
# === Return
|
876
|
+
# true:: Always return true
|
877
|
+
#
|
878
|
+
# === Raise
|
879
|
+
# SendFailure:: If publishing of request fails unexpectedly
|
880
|
+
# TemporarilyOffline:: If cannot publish request because currently not connected
|
881
|
+
# to any brokers and offline queueing is disabled
|
882
|
+
def send_packet(kind, packet, callback)
|
883
|
+
if packet
|
884
|
+
method = packet.type.split('/').last
|
885
|
+
received_at = @request_stats.update(method, packet.token)
|
886
|
+
@request_kind_stats.update((packet.selector == :all ? kind.to_s.sub(/push/, "fanout") : kind.to_s)[5..-1])
|
887
|
+
@pending_requests[packet.token] = PendingRequest.new(kind, received_at, callback) if callback
|
888
|
+
if !packet.persistent && kind.to_s =~ /request/
|
889
|
+
publish_with_timeout_retry(packet, packet.token)
|
890
|
+
else
|
891
|
+
publish(packet)
|
892
|
+
end
|
850
893
|
end
|
851
|
-
|
894
|
+
true
|
852
895
|
end
|
853
896
|
|
854
897
|
# Handle response to a request
|
@@ -873,11 +916,23 @@ module RightScale
|
|
873
916
|
end
|
874
917
|
|
875
918
|
if handler = @pending_requests[token]
|
876
|
-
if result && result.non_delivery? && handler.kind == :send_retryable_request
|
877
|
-
|
878
|
-
|
879
|
-
|
880
|
-
|
919
|
+
if result && result.non_delivery? && handler.kind == :send_retryable_request
|
920
|
+
if [OperationResult::TARGET_NOT_CONNECTED, OperationResult::TTL_EXPIRATION].include?(result.content)
|
921
|
+
# Log and temporarily ignore so that timeout retry mechanism continues, but save reason for use below if timeout
|
922
|
+
# Leave purging of associated request until final response, i.e., success response or retry timeout
|
923
|
+
if parent = handler.retry_parent
|
924
|
+
@pending_requests[parent].non_delivery = result.content
|
925
|
+
else
|
926
|
+
handler.non_delivery = result.content
|
927
|
+
end
|
928
|
+
Log.info("Non-delivery of <#{token}> because #{result.content}")
|
929
|
+
elsif result.content == OperationResult::RETRY_TIMEOUT && handler.non_delivery
|
930
|
+
# Request timed out but due to another non-delivery reason, so use that reason since more germane
|
931
|
+
response.results = OperationResult.non_delivery(handler.non_delivery)
|
932
|
+
deliver(response, handler)
|
933
|
+
else
|
934
|
+
deliver(response, handler)
|
935
|
+
end
|
881
936
|
else
|
882
937
|
deliver(response, handler)
|
883
938
|
end
|
@@ -1085,21 +1140,8 @@ module RightScale
|
|
1085
1140
|
# TemporarilyOffline:: If cannot publish request because currently not connected
|
1086
1141
|
# to any brokers and offline queueing is disabled
|
1087
1142
|
def build_and_send_packet(kind, type, payload, target, callback)
|
1088
|
-
|
1089
|
-
|
1090
|
-
@offline_handler.queue_request(kind, type, payload, target, callback)
|
1091
|
-
else
|
1092
|
-
packet = build_packet(kind, type, payload, target, callback)
|
1093
|
-
method = type.split('/').last
|
1094
|
-
received_at = @request_stats.update(method, packet.token)
|
1095
|
-
@request_kind_stats.update((packet.selector == :all ? kind.to_s.sub(/push/, "fanout") : kind.to_s)[5..-1])
|
1096
|
-
@pending_requests[packet.token] = PendingRequest.new(kind, received_at, callback) if callback
|
1097
|
-
if !packet.persistent && kind.to_s =~ /request/
|
1098
|
-
publish_with_timeout_retry(packet, packet.token)
|
1099
|
-
else
|
1100
|
-
publish(packet)
|
1101
|
-
end
|
1102
|
-
end
|
1143
|
+
packet = build_packet(kind, type, payload, target, callback)
|
1144
|
+
send_packet(kind, packet, callback)
|
1103
1145
|
true
|
1104
1146
|
end
|
1105
1147
|
|
@@ -1123,12 +1165,12 @@ module RightScale
|
|
1123
1165
|
@broker.publish(exchange, request, :persistent => request.persistent, :mandatory => true,
|
1124
1166
|
:log_filter => [:tags, :target, :tries, :persistent], :brokers => ids)
|
1125
1167
|
rescue RightAMQP::HABrokerClient::NoConnectedBrokers => e
|
1126
|
-
msg = "Failed to publish request #{request.
|
1168
|
+
msg = "Failed to publish request #{request.trace} #{request.type}"
|
1127
1169
|
Log.error(msg, e)
|
1128
1170
|
@send_failure_stats.update("NoConnectedBrokers")
|
1129
1171
|
raise TemporarilyOffline.new(msg + " (#{e.class}: #{e.message})")
|
1130
1172
|
rescue Exception => e
|
1131
|
-
msg = "Failed to publish request #{request.
|
1173
|
+
msg = "Failed to publish request #{request.trace} #{request.type}"
|
1132
1174
|
Log.error(msg, e, :trace)
|
1133
1175
|
@send_failure_stats.update(e.class.name)
|
1134
1176
|
@exception_stats.track("publish", e, request)
|
@@ -1156,7 +1198,7 @@ module RightScale
|
|
1156
1198
|
def publish_with_timeout_retry(request, parent, count = 0, multiplier = 1, elapsed = 0, broker_ids = nil)
|
1157
1199
|
published_broker_ids = publish(request, broker_ids)
|
1158
1200
|
|
1159
|
-
if @retry_interval && @retry_timeout && parent
|
1201
|
+
if @retry_interval && @retry_timeout && parent
|
1160
1202
|
interval = [(@retry_interval * multiplier) + (@request_stats.avg_duration || 0), @retry_timeout - elapsed].min
|
1161
1203
|
EM.add_timer(interval) do
|
1162
1204
|
begin
|
@@ -1173,15 +1215,27 @@ module RightScale
|
|
1173
1215
|
broker_ids.push(broker_ids.shift))
|
1174
1216
|
@retry_stats.update(request.type.split('/').last)
|
1175
1217
|
else
|
1176
|
-
Log.warning("RE-SEND TIMEOUT after #{elapsed.to_i} seconds for #{request.
|
1218
|
+
Log.warning("RE-SEND TIMEOUT after #{elapsed.to_i} seconds for #{request.trace} #{request.type}")
|
1177
1219
|
result = OperationResult.non_delivery(OperationResult::RETRY_TIMEOUT)
|
1178
1220
|
@non_delivery_stats.update(result.content)
|
1179
1221
|
handle_response(Result.new(request.token, request.reply_to, result, @identity))
|
1180
1222
|
end
|
1181
|
-
@connectivity_checker.check(published_broker_ids.first) if count == 1
|
1223
|
+
@connectivity_checker.check(published_broker_ids.first) if count == 1 && !published_broker_ids.empty?
|
1182
1224
|
end
|
1225
|
+
rescue TemporarilyOffline => e
|
1226
|
+
# Send retry response so that requester, e.g., IdempotentRequest, can retry
|
1227
|
+
Log.error("Failed retry for #{request.trace} #{request.type} because temporarily offline")
|
1228
|
+
result = OperationResult.retry("lost connectivity")
|
1229
|
+
handle_response(Result.new(request.token, request.reply_to, result, @identity))
|
1230
|
+
rescue SendFailure => e
|
1231
|
+
# Send non-delivery response so that requester, e.g., IdempotentRequest, can retry
|
1232
|
+
Log.error("Failed retry for #{request.trace} #{request.type} because of send failure")
|
1233
|
+
result = OperationResult.non_delivery("retry failed")
|
1234
|
+
handle_response(Result.new(request.token, request.reply_to, result, @identity))
|
1183
1235
|
rescue Exception => e
|
1184
|
-
|
1236
|
+
# Not sending a response here because something more basic is broken in the retry
|
1237
|
+
# mechanism and don't want an error response to preempt a delayed actual response
|
1238
|
+
Log.error("Failed retry for #{request.trace} #{request.type} without responding", e, :trace)
|
1185
1239
|
@exception_stats.track("retry", e, request)
|
1186
1240
|
end
|
1187
1241
|
end
|