right_agent 0.6.6 → 0.9.3
Sign up to get free protection for your applications and to get access to all the features.
- data/lib/right_agent/agent.rb +26 -25
- data/lib/right_agent/agent_config.rb +28 -2
- data/lib/right_agent/command/command_constants.rb +2 -2
- data/lib/right_agent/core_payload_types/executable_bundle.rb +3 -21
- data/lib/right_agent/core_payload_types/login_user.rb +19 -4
- data/lib/right_agent/core_payload_types/recipe_instantiation.rb +7 -1
- data/lib/right_agent/core_payload_types/right_script_instantiation.rb +7 -1
- data/lib/right_agent/dispatcher.rb +6 -19
- data/lib/right_agent/idempotent_request.rb +72 -17
- data/lib/right_agent/monkey_patches/ruby_patch.rb +0 -1
- data/lib/right_agent/monkey_patches.rb +0 -1
- data/lib/right_agent/operation_result.rb +27 -4
- data/lib/right_agent/packets.rb +47 -23
- data/lib/right_agent/platform/darwin.rb +33 -2
- data/lib/right_agent/platform/linux.rb +98 -2
- data/lib/right_agent/platform/windows.rb +41 -6
- data/lib/right_agent/platform.rb +11 -2
- data/lib/right_agent/scripts/agent_controller.rb +2 -1
- data/lib/right_agent/scripts/agent_deployer.rb +2 -2
- data/lib/right_agent/scripts/stats_manager.rb +7 -3
- data/lib/right_agent/sender.rb +45 -28
- data/lib/right_agent.rb +2 -5
- data/right_agent.gemspec +5 -3
- data/spec/agent_config_spec.rb +1 -1
- data/spec/agent_spec.rb +26 -20
- data/spec/core_payload_types/login_user_spec.rb +7 -3
- data/spec/idempotent_request_spec.rb +218 -48
- data/spec/operation_result_spec.rb +19 -0
- data/spec/packets_spec.rb +42 -1
- data/spec/platform/darwin.rb +11 -0
- data/spec/platform/linux.rb +23 -0
- data/spec/platform/linux_volume_manager_spec.rb +43 -43
- data/spec/platform/platform_spec.rb +35 -32
- data/spec/platform/windows.rb +11 -0
- data/spec/sender_spec.rb +21 -25
- metadata +47 -40
- data/lib/right_agent/broker_client.rb +0 -686
- data/lib/right_agent/ha_broker_client.rb +0 -1327
- data/lib/right_agent/monkey_patches/amqp_patch.rb +0 -274
- data/lib/right_agent/monkey_patches/ruby_patch/string_patch.rb +0 -107
- data/lib/right_agent/stats_helper.rb +0 -745
- data/spec/broker_client_spec.rb +0 -962
- data/spec/ha_broker_client_spec.rb +0 -1695
- data/spec/monkey_patches/amqp_patch_spec.rb +0 -100
- data/spec/monkey_patches/string_patch_spec.rb +0 -99
- data/spec/stats_helper_spec.rb +0 -686
data/lib/right_agent/agent.rb
CHANGED
@@ -31,7 +31,6 @@ module RightScale
|
|
31
31
|
|
32
32
|
include ConsoleHelper
|
33
33
|
include DaemonizeHelper
|
34
|
-
include StatsHelper
|
35
34
|
|
36
35
|
# (String) Identity of this agent
|
37
36
|
attr_reader :identity
|
@@ -45,7 +44,7 @@ module RightScale
|
|
45
44
|
# (ActorRegistry) Registry for this agents actors
|
46
45
|
attr_reader :registry
|
47
46
|
|
48
|
-
# (HABrokerClient) High availability AMQP broker client
|
47
|
+
# (RightAMQP::HABrokerClient) High availability AMQP broker client
|
49
48
|
attr_reader :broker
|
50
49
|
|
51
50
|
# (Array) Tag strings published by agent
|
@@ -174,6 +173,7 @@ module RightScale
|
|
174
173
|
def run
|
175
174
|
Log.init(@identity, @options[:log_path], :print => true)
|
176
175
|
Log.level = @options[:log_level] if @options[:log_level]
|
176
|
+
RightSupport::Log::Mixin.default_logger = Log
|
177
177
|
Log.debug("Start options:")
|
178
178
|
log_opts = @options.inject([]){ |t, (k, v)| t << "- #{k}: #{v}" }
|
179
179
|
log_opts.each { |l| Log.debug(l) }
|
@@ -188,7 +188,7 @@ module RightScale
|
|
188
188
|
|
189
189
|
# Initiate AMQP broker connection, wait for connection before proceeding
|
190
190
|
# otherwise messages published on failed connection will be lost
|
191
|
-
@broker = HABrokerClient.new(Serializer.new(:secure), @options)
|
191
|
+
@broker = RightAMQP::HABrokerClient.new(Serializer.new(:secure), @options)
|
192
192
|
@all_setup.each { |s| @remaining_setup[s] = @broker.all }
|
193
193
|
@broker.connection_status(:one_off => @options[:connect_timeout]) do |status|
|
194
194
|
if status == :connected
|
@@ -259,14 +259,15 @@ module RightScale
|
|
259
259
|
def tune_heartbeat(heartbeat)
|
260
260
|
res = nil
|
261
261
|
begin
|
262
|
+
Log.info("[setup] Reconnecting each broker to tune heartbeat to #{heartbeat}")
|
262
263
|
@broker.heartbeat = heartbeat
|
263
264
|
update_configuration(:heartbeat => heartbeat)
|
264
265
|
ids = []
|
265
266
|
all = @broker.all
|
266
267
|
all.each do |id|
|
267
268
|
begin
|
268
|
-
host, port, index, priority
|
269
|
-
@broker.connect(host, port, index, priority,
|
269
|
+
host, port, index, priority = @broker.identity_parts(id)
|
270
|
+
@broker.connect(host, port, index, priority, force = true) do |id|
|
270
271
|
@broker.connection_status(:one_off => @options[:connect_timeout], :brokers => [id]) do |status|
|
271
272
|
begin
|
272
273
|
if status == :connected
|
@@ -321,7 +322,7 @@ module RightScale
|
|
321
322
|
Log.info("Current broker configuration: #{@broker.status.inspect}")
|
322
323
|
res = nil
|
323
324
|
begin
|
324
|
-
@broker.connect(host, port, index, priority,
|
325
|
+
@broker.connect(host, port, index, priority, force) do |id|
|
325
326
|
@broker.connection_status(:one_off => @options[:connect_timeout], :brokers => [id]) do |status|
|
326
327
|
begin
|
327
328
|
if status == :connected
|
@@ -342,7 +343,7 @@ module RightScale
|
|
342
343
|
end
|
343
344
|
end
|
344
345
|
rescue Exception => e
|
345
|
-
res = Log.format("Failed to connect to broker #{HABrokerClient.identity(host, port)}", e)
|
346
|
+
res = Log.format("Failed to connect to broker #{RightAMQP::HABrokerClient.identity(host, port)}", e)
|
346
347
|
@exceptions.track("connect", e)
|
347
348
|
end
|
348
349
|
Log.error(res) if res
|
@@ -364,7 +365,7 @@ module RightScale
|
|
364
365
|
and_remove = " and removing" if remove
|
365
366
|
Log.info("Disconnecting#{and_remove} broker at host #{host.inspect} port #{port.inspect}")
|
366
367
|
Log.info("Current broker configuration: #{@broker.status.inspect}")
|
367
|
-
id = HABrokerClient.identity(host, port)
|
368
|
+
id = RightAMQP::HABrokerClient.identity(host, port)
|
368
369
|
@connect_requests.update("disconnect #{@broker.alias_(id)}")
|
369
370
|
connected = @broker.connected
|
370
371
|
res = nil
|
@@ -434,7 +435,7 @@ module RightScale
|
|
434
435
|
when Result then @sender.handle_response(packet)
|
435
436
|
end
|
436
437
|
@sender.message_received
|
437
|
-
rescue HABrokerClient::NoConnectedBrokers => e
|
438
|
+
rescue RightAMQP::HABrokerClient::NoConnectedBrokers => e
|
438
439
|
Log.error("Identity queue processing error", e)
|
439
440
|
rescue Exception => e
|
440
441
|
Log.error("Identity queue processing error", e, :trace)
|
@@ -578,9 +579,9 @@ module RightScale
|
|
578
579
|
# === Return
|
579
580
|
# true:: Always return true
|
580
581
|
def reset_agent_stats
|
581
|
-
@connect_requests =
|
582
|
-
@non_deliveries =
|
583
|
-
@exceptions =
|
582
|
+
@connect_requests = RightSupport::Stats::Activity.new(measure_rate = false)
|
583
|
+
@non_deliveries = RightSupport::Stats::Activity.new
|
584
|
+
@exceptions = RightSupport::Stats::Exceptions.new(self, @options[:exception_callback])
|
584
585
|
true
|
585
586
|
end
|
586
587
|
|
@@ -625,7 +626,7 @@ module RightScale
|
|
625
626
|
# (Boolean):: true if successful, otherwise false
|
626
627
|
def update_configuration(opts)
|
627
628
|
if cfg = AgentConfig.load_cfg(@agent_name)
|
628
|
-
opts.each { |k, v| cfg[k] = v
|
629
|
+
opts.each { |k, v| cfg[k] = v }
|
629
630
|
AgentConfig.store_cfg(@agent_name, cfg)
|
630
631
|
true
|
631
632
|
else
|
@@ -739,28 +740,18 @@ module RightScale
|
|
739
740
|
def finish_setup
|
740
741
|
@broker.failed.each do |id|
|
741
742
|
p = {:agent_identity => @identity}
|
742
|
-
p[:host], p[:port], p[:id], p[:priority]
|
743
|
+
p[:host], p[:port], p[:id], p[:priority] = @broker.identity_parts(id)
|
743
744
|
@sender.send_push("/registrar/connect", p)
|
744
745
|
end
|
745
746
|
true
|
746
747
|
end
|
747
748
|
|
748
749
|
# Check status of agent by gathering current operation statistics and publishing them,
|
749
|
-
#
|
750
|
+
# finishing any queue setup, and executing any deferred tasks
|
750
751
|
#
|
751
752
|
# === Return
|
752
753
|
# true:: Always return true
|
753
754
|
def check_status
|
754
|
-
@deferred_tasks.reject! do |t|
|
755
|
-
begin
|
756
|
-
t.call
|
757
|
-
rescue Exception => e
|
758
|
-
Log.error("Failed to perform deferred task", e)
|
759
|
-
@exceptions.track("check status", e)
|
760
|
-
end
|
761
|
-
true
|
762
|
-
end
|
763
|
-
|
764
755
|
begin
|
765
756
|
finish_setup
|
766
757
|
rescue Exception => e
|
@@ -779,6 +770,16 @@ module RightScale
|
|
779
770
|
@exceptions.track("check status", e)
|
780
771
|
end
|
781
772
|
|
773
|
+
@deferred_tasks.reject! do |t|
|
774
|
+
begin
|
775
|
+
t.call
|
776
|
+
rescue Exception => e
|
777
|
+
Log.error("Failed to perform deferred task", e)
|
778
|
+
@exceptions.track("check status", e)
|
779
|
+
end
|
780
|
+
true
|
781
|
+
end
|
782
|
+
|
782
783
|
@check_status_count += 1
|
783
784
|
true
|
784
785
|
end
|
@@ -63,7 +63,7 @@ module RightScale
|
|
63
63
|
module AgentConfig
|
64
64
|
|
65
65
|
# Current agent protocol version
|
66
|
-
PROTOCOL_VERSION =
|
66
|
+
PROTOCOL_VERSION = 20
|
67
67
|
|
68
68
|
# Current agent protocol version
|
69
69
|
#
|
@@ -73,6 +73,32 @@ module RightScale
|
|
73
73
|
PROTOCOL_VERSION
|
74
74
|
end
|
75
75
|
|
76
|
+
# Default thread name when no thread is specified for an executable bundle.
|
77
|
+
DEFAULT_THREAD_NAME = 'default'
|
78
|
+
|
79
|
+
# Default thread name when no thread is specified for an executable bundle.
|
80
|
+
#
|
81
|
+
# === Return
|
82
|
+
# (String):: default thread name...
|
83
|
+
def self.default_thread_name
|
84
|
+
DEFAULT_THREAD_NAME
|
85
|
+
end
|
86
|
+
|
87
|
+
# Regular expression to define what a valid thread name looks like: an alpha character
|
88
|
+
# followed by 0 or more alphanumerics or underscores. Only lower-case characters are
|
89
|
+
# allowed.
|
90
|
+
VALID_THREAD_NAME = /^[a-z][a-z0-9_]*$/
|
91
|
+
|
92
|
+
# Regular expression to define what a valid thread name looks like: an alpha character
|
93
|
+
# followed by 0 or more alphanumerics or underscores. Only lower-case characters are
|
94
|
+
# allowed.
|
95
|
+
#
|
96
|
+
# === Return
|
97
|
+
# (String):: default thread name...
|
98
|
+
def self.valid_thread_name
|
99
|
+
VALID_THREAD_NAME
|
100
|
+
end
|
101
|
+
|
76
102
|
# Initialize path to root directory of agent
|
77
103
|
#
|
78
104
|
# === Parameters
|
@@ -217,7 +243,7 @@ module RightScale
|
|
217
243
|
# === Return
|
218
244
|
# (String):: Directory path name
|
219
245
|
def self.cfg_dir
|
220
|
-
@cfg_dir ||= Platform.filesystem.
|
246
|
+
@cfg_dir ||= Platform.filesystem.right_agent_cfg_dir
|
221
247
|
end
|
222
248
|
|
223
249
|
# Path to generated agent configuration file
|
@@ -26,8 +26,8 @@ module RightScale
|
|
26
26
|
class CommandConstants
|
27
27
|
|
28
28
|
# Ports used for command protocol
|
29
|
-
BASE_INSTANCE_AGENT_SOCKET_PORT =
|
30
|
-
BASE_INSTANCE_AGENT_CHECKER_SOCKET_PORT =
|
29
|
+
BASE_INSTANCE_AGENT_SOCKET_PORT = 53843
|
30
|
+
BASE_INSTANCE_AGENT_CHECKER_SOCKET_PORT = 53844
|
31
31
|
|
32
32
|
end
|
33
33
|
end
|
@@ -35,14 +35,6 @@ module RightScale
|
|
35
35
|
|
36
36
|
include Serializable
|
37
37
|
|
38
|
-
# Default thread name when no thread is specified for an executable bundle.
|
39
|
-
DEFAULT_THREAD_NAME = 'default'
|
40
|
-
|
41
|
-
# Regular expression to define what a valid thread name looks like: an alpha character
|
42
|
-
# followed by 0 or more alphanumerics or underscores. Only lower-case characters are
|
43
|
-
# allowed.
|
44
|
-
VALID_THREAD_NAME = /^[a-z][a-z0-9_]*$/
|
45
|
-
|
46
38
|
# (Array) Collection of RightScripts and chef recipes instantiations
|
47
39
|
attr_accessor :executables
|
48
40
|
|
@@ -62,6 +54,9 @@ module RightScale
|
|
62
54
|
# (String) Repose server to use
|
63
55
|
attr_accessor :repose_servers
|
64
56
|
|
57
|
+
# (String) The name of the RIghtLink thread to run this bundle on
|
58
|
+
attr_accessor :thread_name
|
59
|
+
|
65
60
|
# (Hash):: collection of repos to be checked out on the instance
|
66
61
|
# :key (String):: the hash id (SHA) of the repository
|
67
62
|
# :value (Hash):: repo and cookbook detail
|
@@ -126,18 +121,5 @@ module RightScale
|
|
126
121
|
desc = @executables.collect { |e| e.nickname }.join(', ') if @executables
|
127
122
|
desc ||= 'empty bundle'
|
128
123
|
end
|
129
|
-
|
130
|
-
# Gets the thread name from a bundle, if any. Uses the default thread name for
|
131
|
-
# when a thread name is not specified (for backward compatibility, etc.).
|
132
|
-
#
|
133
|
-
# === Parameters
|
134
|
-
# bundle(ExecutableBundle):: bundle to inspect
|
135
|
-
#
|
136
|
-
# === Return
|
137
|
-
# thread_name(String):: thread name for bundle execution
|
138
|
-
def thread_name(value = nil)
|
139
|
-
@thread_name = value if value
|
140
|
-
return @thread_name || DEFAULT_THREAD_NAME
|
141
|
-
end
|
142
124
|
end
|
143
125
|
end
|
@@ -21,6 +21,8 @@
|
|
21
21
|
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
22
22
|
#
|
23
23
|
|
24
|
+
require 'net/ssh'
|
25
|
+
|
24
26
|
module RightScale
|
25
27
|
|
26
28
|
# Authorized user for the Managed Login feature
|
@@ -29,7 +31,7 @@ module RightScale
|
|
29
31
|
include Serializable
|
30
32
|
|
31
33
|
attr_accessor :uuid, :username, :public_key, :public_keys, :common_name,
|
32
|
-
:superuser, :expires_at, :profile_data
|
34
|
+
:superuser, :expires_at, :profile_data, :public_key_fingerprints
|
33
35
|
|
34
36
|
# Initialize fields from given arguments
|
35
37
|
def initialize(*args)
|
@@ -41,9 +43,10 @@ module RightScale
|
|
41
43
|
@expires_at = Time.at(args[5]) if args[5] && (args[5] != 0) # nil -> 0 because of expires_at.to_i below
|
42
44
|
@public_keys = args[6]
|
43
45
|
@profile_data = args[7]
|
46
|
+
@public_key_fingerprints = args[8]
|
44
47
|
|
45
|
-
#
|
46
|
-
# singular public_key as a legacy member.
|
48
|
+
# We now expect an array of public_keys to be passed while supporting the
|
49
|
+
# singular public_key as a legacy member. When serialized back from a
|
47
50
|
# legacy LoginUser record, the singular value may be set while the plural
|
48
51
|
# is nil.
|
49
52
|
if @public_keys
|
@@ -53,11 +56,23 @@ module RightScale
|
|
53
56
|
raise ArgumentError, "Expected public_key (third argument) to be a string" unless @public_key.is_a?(String)
|
54
57
|
@public_keys = [@public_key]
|
55
58
|
end
|
59
|
+
|
60
|
+
# The number of fingerprints must match the number of public keys
|
61
|
+
if @public_key_fingerprints && @public_key_fingerprints.size != @public_keys.size
|
62
|
+
raise ArgumentError, "Expected public_keys (seventh argument) array length (#{@public_keys.size}) is not " +
|
63
|
+
"the same as the public_key_fingerprints (eighth argument) (#{@public_key_fingerprints.size})"
|
64
|
+
end
|
56
65
|
end
|
57
66
|
|
58
67
|
# Array of serialized fields given to constructor
|
59
68
|
def serialized_members
|
60
|
-
[ @uuid, @username, @public_key, @common_name, @superuser, @expires_at.to_i, @public_keys, @profile_data
|
69
|
+
[ @uuid, @username, @public_key, @common_name, @superuser, @expires_at.to_i, @public_keys, @profile_data,
|
70
|
+
@public_key_fingerprints ]
|
71
|
+
end
|
72
|
+
|
73
|
+
# Create fingerprint for public key
|
74
|
+
def self.fingerprint(public_key)
|
75
|
+
Digest::SHA1.hexdigest(::Net::SSH::KeyFactory.load_data_public_key(public_key).to_der)
|
61
76
|
end
|
62
77
|
|
63
78
|
end
|
@@ -43,17 +43,23 @@ module RightScale
|
|
43
43
|
# (Array of SecureDocumentLocation) attributes that must be resolved by the instance
|
44
44
|
attr_accessor :external_inputs
|
45
45
|
|
46
|
+
# (Hash) nil or Hash of input name to flags (array of string tokens) indicating additional
|
47
|
+
# boolean properties of the input which are useful to the instance. the presence of the
|
48
|
+
# flag means true, absence means false.
|
49
|
+
attr_accessor :input_flags
|
50
|
+
|
46
51
|
def initialize(*args)
|
47
52
|
@nickname = args[0] if args.size > 0
|
48
53
|
@attributes = args[1] if args.size > 1
|
49
54
|
@id = args[2] if args.size > 2
|
50
55
|
@ready = args[3] if args.size > 3
|
51
56
|
@external_inputs = args[4] if args.size > 4
|
57
|
+
@input_flags = args[5] if args.size > 5
|
52
58
|
end
|
53
59
|
|
54
60
|
# Array of serialized fields given to constructor
|
55
61
|
def serialized_members
|
56
|
-
[ @nickname, @attributes, @id, @ready, @external_inputs ]
|
62
|
+
[ @nickname, @attributes, @id, @ready, @external_inputs, @input_flags ]
|
57
63
|
end
|
58
64
|
|
59
65
|
end
|
@@ -53,6 +53,11 @@ module RightScale
|
|
53
53
|
# (Array of SecureDocumentLocation) attributes that must be resolved by the instance
|
54
54
|
attr_accessor :external_inputs
|
55
55
|
|
56
|
+
# (Hash) nil or Hash of input name to flags (array of string tokens) indicating additional
|
57
|
+
# boolean properties of the input which are useful to the instance. the presence of the
|
58
|
+
# flag means true, absense means false.
|
59
|
+
attr_accessor :input_flags
|
60
|
+
|
56
61
|
def initialize(*args)
|
57
62
|
@nickname = args[0] if args.size > 0
|
58
63
|
@source = args[1] if args.size > 1
|
@@ -62,11 +67,12 @@ module RightScale
|
|
62
67
|
@id = args[5] if args.size > 5
|
63
68
|
@ready = args[6] if args.size > 6
|
64
69
|
@external_inputs = args[7] if args.size > 7
|
70
|
+
@input_flags = args[8] if args.size > 8
|
65
71
|
end
|
66
72
|
|
67
73
|
# Array of serialized fields given to constructor
|
68
74
|
def serialized_members
|
69
|
-
[ @nickname, @source, @parameters, @attachments, @packages, @id, @ready, @external_inputs ]
|
75
|
+
[ @nickname, @source, @parameters, @attachments, @packages, @id, @ready, @external_inputs, @input_flags ]
|
70
76
|
end
|
71
77
|
|
72
78
|
end
|
@@ -25,8 +25,6 @@ module RightScale
|
|
25
25
|
# Dispatching of payload to specified actor
|
26
26
|
class Dispatcher
|
27
27
|
|
28
|
-
include StatsHelper
|
29
|
-
|
30
28
|
# Response queue name
|
31
29
|
RESPONSE_QUEUE = "response"
|
32
30
|
|
@@ -112,7 +110,7 @@ module RightScale
|
|
112
110
|
# (String) Identity of associated agent
|
113
111
|
attr_reader :identity
|
114
112
|
|
115
|
-
# (HABrokerClient) High availability AMQP broker client
|
113
|
+
# (RightAMQP::HABrokerClient) High availability AMQP broker client
|
116
114
|
attr_reader :broker
|
117
115
|
|
118
116
|
# (EM) Event machine class (exposed for unit tests)
|
@@ -174,7 +172,7 @@ module RightScale
|
|
174
172
|
# Reject this request if its TTL has expired
|
175
173
|
if (expires_at = request.expires_at) && expires_at > 0 && received_at.to_i >= expires_at
|
176
174
|
@rejects.update("expired (#{method})")
|
177
|
-
Log.info("REJECT EXPIRED <#{token}> from #{request.from} TTL #{elapsed(received_at.to_i - expires_at)} ago")
|
175
|
+
Log.info("REJECT EXPIRED <#{token}> from #{request.from} TTL #{RightSupport::Stats.elapsed(received_at.to_i - expires_at)} ago")
|
178
176
|
if request.is_a?(Request)
|
179
177
|
# For agents that do not know about non-delivery, use error result
|
180
178
|
non_delivery = if request.recv_version < 13
|
@@ -228,7 +226,7 @@ module RightScale
|
|
228
226
|
exchange = {:type => :queue, :name => RESPONSE_QUEUE, :options => {:durable => true, :no_declare => @secure}}
|
229
227
|
@broker.publish(exchange, r, :persistent => true, :mandatory => true, :log_filter => [:tries, :persistent, :duration])
|
230
228
|
end
|
231
|
-
rescue HABrokerClient::NoConnectedBrokers => e
|
229
|
+
rescue RightAMQP::HABrokerClient::NoConnectedBrokers => e
|
232
230
|
Log.error("Failed to publish result of dispatched request #{request.trace}", e)
|
233
231
|
rescue Exception => e
|
234
232
|
Log.error("Failed to publish result of dispatched request #{request.trace}", e, :trace)
|
@@ -298,9 +296,9 @@ module RightScale
|
|
298
296
|
# === Return
|
299
297
|
# true:: Always return true
|
300
298
|
def reset_stats
|
301
|
-
@rejects =
|
302
|
-
@requests =
|
303
|
-
@exceptions =
|
299
|
+
@rejects = RightSupport::Stats::Activity.new
|
300
|
+
@requests = RightSupport::Stats::Activity.new
|
301
|
+
@exceptions = RightSupport::Stats::Exceptions.new(@agent)
|
304
302
|
true
|
305
303
|
end
|
306
304
|
|
@@ -335,17 +333,6 @@ module RightScale
|
|
335
333
|
error
|
336
334
|
end
|
337
335
|
|
338
|
-
# Convert value to nil if equals 0
|
339
|
-
#
|
340
|
-
# === Parameters
|
341
|
-
# value(Integer|nil):: Value to be converted
|
342
|
-
#
|
343
|
-
# === Return
|
344
|
-
# (Integer|nil):: Converted value
|
345
|
-
def nil_if_zero(value)
|
346
|
-
if !value || value == 0 then nil else value end
|
347
|
-
end
|
348
|
-
|
349
336
|
end # Dispatcher
|
350
337
|
|
351
338
|
end # RightScale
|
@@ -22,14 +22,46 @@
|
|
22
22
|
|
23
23
|
module RightScale
|
24
24
|
|
25
|
+
# This is a retryable request for use when the execution of the request by the
|
26
|
+
# receiver is known to be idempotent and when there is a need to indefinitely
|
27
|
+
# pursue getting a usable response, e.g., when an instance is launching.
|
28
|
+
# It is implemented as an EM::Deferrable and as such invokes the Proc defined
|
29
|
+
# with its #callback method with the result content from a OperationResult::SUCCESS
|
30
|
+
# response, or it will invoke the Proc defined with its #errback method with error
|
31
|
+
# content if the response is an OperationResult::ERROR or CANCEL, or if the request
|
32
|
+
# has timed out. The request can be canceled with the #cancel method, or the receiver
|
33
|
+
# of the request may respond with a CANCEL result to cause the request to be canceled.
|
34
|
+
# This is useful in situations where the request is never expected to succeed
|
35
|
+
# regardless of the number of retries. By default if the response to the request
|
36
|
+
# is an OperationResult::RETRY or NON_DELIVERY indication, the request is automatically
|
37
|
+
# retried, as is also the case for an ERROR indication if the :retry_on_error option
|
38
|
+
# is specified. The retry algorithm is controlled by the :retry_delay, :retry_delay_count,
|
39
|
+
# and :max_retry_delay settings. The initial retry interval is the default or specified
|
40
|
+
# :retry_delay and this interval is used :retry_delay_count times, at which point the
|
41
|
+
# :retry_delay is doubled and the :retry_delay_count is halved. This backoff is again
|
42
|
+
# applied after the new :retry_delay_count is reached, and so on until :retry_delay
|
43
|
+
# reaches :max_retry_delay which then is used as the interval until the default or
|
44
|
+
# specified :timeout is reached. The default :timeout is 4 days.
|
25
45
|
class IdempotentRequest
|
26
46
|
|
27
47
|
include OperationResultHelper
|
28
48
|
include EM::Deferrable
|
29
49
|
|
30
|
-
#
|
50
|
+
# Default delay before initial retry in case of failure with -1 meaning no delay
|
31
51
|
DEFAULT_RETRY_DELAY = 5
|
32
52
|
|
53
|
+
# Default minimum number of retries before beginning backoff
|
54
|
+
DEFAULT_RETRY_DELAY_COUNT = 60
|
55
|
+
|
56
|
+
# Maximum default delay before retry when backing off
|
57
|
+
DEFAULT_MAX_RETRY_DELAY = 60
|
58
|
+
|
59
|
+
# Factor used for exponential backoff of retry delay
|
60
|
+
RETRY_BACKOFF_FACTOR = 2
|
61
|
+
|
62
|
+
# Default timeout with -1 meaning never timeout
|
63
|
+
DEFAULT_TIMEOUT = 4 * 24 * 60 * 60
|
64
|
+
|
33
65
|
attr_reader :raw_response
|
34
66
|
|
35
67
|
# Send idempotent request
|
@@ -37,18 +69,31 @@ module RightScale
|
|
37
69
|
# Calls deferrable callback on completion, error callback on timeout
|
38
70
|
#
|
39
71
|
# === Parameters
|
40
|
-
# operation(String):: Request operation (
|
72
|
+
# operation(String):: Request operation (e.g., '/booter/get_boot_bundle')
|
41
73
|
# payload(Hash):: Request payload
|
42
|
-
# options
|
43
|
-
#
|
44
|
-
#
|
45
|
-
#
|
46
|
-
|
47
|
-
|
48
|
-
|
74
|
+
# options(Hash):: Request options
|
75
|
+
# :targets(Array):: Targets from which to randomly choose one
|
76
|
+
# :retry_on_error(Boolean):: Whether request should be retried if recipient returned an error
|
77
|
+
# :retry_delay(Fixnum):: Number of seconds delay before initial retry with -1 meaning no delay,
|
78
|
+
# defaults to DEFAULT_RETRY_DELAY
|
79
|
+
# :retry_delay_count(Fixnum):: Minimum number of retries at initial :retry_delay value before
|
80
|
+
# increasing delay exponentially and decreasing this count exponentially, defaults to
|
81
|
+
# DEFAULT_RETRY_DELAY_COUNT
|
82
|
+
# :max_retry_delay(Fixnum):: Maximum number of seconds of retry delay, defaults to DEFAULT_MAX_RETRY_DELAY
|
83
|
+
# :timeout(Fixnum):: Number of seconds with no response before error callback gets called with
|
84
|
+
# -1 meaning never, defaults to DEFAULT_TIMEOUT
|
85
|
+
#
|
86
|
+
# === Raises
|
87
|
+
# ArgumentError:: If operation or payload not specified
|
88
|
+
def initialize(operation, payload, options = {})
|
89
|
+
raise ArgumentError.new("operation is required") unless @operation = operation
|
90
|
+
raise ArgumentError.new("payload is required") unless @payload = payload
|
49
91
|
@retry_on_error = options[:retry_on_error] || false
|
50
|
-
@timeout = options[:timeout] ||
|
92
|
+
@timeout = options[:timeout] || DEFAULT_TIMEOUT
|
51
93
|
@retry_delay = options[:retry_delay] || DEFAULT_RETRY_DELAY
|
94
|
+
@retry_delay_count = options[:retry_delay_count] || DEFAULT_RETRY_DELAY_COUNT
|
95
|
+
@max_retry_delay = options[:max_retry_delay] || DEFAULT_MAX_RETRY_DELAY
|
96
|
+
@retries = 0
|
52
97
|
@targets = options[:targets]
|
53
98
|
@raw_response = nil
|
54
99
|
@done = false
|
@@ -64,7 +109,7 @@ module RightScale
|
|
64
109
|
if @cancel_timer.nil? && @timeout > 0
|
65
110
|
@cancel_timer = EM::Timer.new(@timeout) do
|
66
111
|
msg = "Request #{@operation} timed out after #{@timeout} seconds"
|
67
|
-
|
112
|
+
Log.info(msg)
|
68
113
|
cancel(msg)
|
69
114
|
end
|
70
115
|
end
|
@@ -101,7 +146,6 @@ module RightScale
|
|
101
146
|
return true if @done
|
102
147
|
@raw_response = r
|
103
148
|
res = result_from(r)
|
104
|
-
res = OperationResult.non_delivery unless res
|
105
149
|
if res.success?
|
106
150
|
if @cancel_timer
|
107
151
|
@cancel_timer.cancel
|
@@ -110,17 +154,28 @@ module RightScale
|
|
110
154
|
@done = true
|
111
155
|
succeed(res.content)
|
112
156
|
else
|
157
|
+
reason = res.content
|
113
158
|
if res.non_delivery?
|
114
|
-
Log.info("Request non-delivery (#{
|
159
|
+
Log.info("Request non-delivery (#{reason}) for #{@operation}")
|
115
160
|
elsif res.retry?
|
116
|
-
|
161
|
+
reason = (reason && !reason.empty?) ? reason : "RightScale not ready"
|
162
|
+
Log.info("Request #{@operation} failed (#{reason}) and should be retried")
|
163
|
+
elsif res.cancel?
|
164
|
+
reason = (reason && !reason.empty?) ? reason : "RightScale cannot execute request"
|
165
|
+
Log.info("Request #{@operation} canceled (#{reason})")
|
117
166
|
else
|
118
|
-
Log.info("Request #{@operation} failed (#{
|
167
|
+
Log.info("Request #{@operation} failed (#{reason})")
|
119
168
|
end
|
120
|
-
if res.non_delivery? || res.retry? || @retry_on_error
|
169
|
+
if (res.non_delivery? || res.retry? || @retry_on_error) && !res.cancel?
|
121
170
|
Log.info("Retrying in #{@retry_delay} seconds...")
|
122
171
|
if @retry_delay > 0
|
123
|
-
|
172
|
+
this_delay = @retry_delay
|
173
|
+
if (@retries += 1) >= @retry_delay_count
|
174
|
+
@retry_delay = [@retry_delay * RETRY_BACKOFF_FACTOR, @max_retry_delay].min
|
175
|
+
@retry_delay_count = [@retry_delay_count / RETRY_BACKOFF_FACTOR, 1].max
|
176
|
+
@retries = 0
|
177
|
+
end
|
178
|
+
EM.add_timer(this_delay) { run }
|
124
179
|
else
|
125
180
|
EM.next_tick { run }
|
126
181
|
end
|
@@ -42,7 +42,6 @@ end
|
|
42
42
|
RUBY_PATCH_BASE_DIR = File.join(File.dirname(__FILE__), 'ruby_patch')
|
43
43
|
|
44
44
|
require File.normalize_path(File.join(RUBY_PATCH_BASE_DIR, 'array_patch'))
|
45
|
-
require File.normalize_path(File.join(RUBY_PATCH_BASE_DIR, 'string_patch'))
|
46
45
|
require File.normalize_path(File.join(RUBY_PATCH_BASE_DIR, 'object_patch'))
|
47
46
|
require File.normalize_path(File.join(RUBY_PATCH_BASE_DIR, 'singleton_patch'))
|
48
47
|
|
@@ -35,6 +35,7 @@ module RightScale
|
|
35
35
|
RETRY = 3
|
36
36
|
NON_DELIVERY = 4
|
37
37
|
MULTICAST = 5 # Deprecated for agents at version 13 or above
|
38
|
+
CANCEL = 6
|
38
39
|
|
39
40
|
# Non-delivery reasons
|
40
41
|
NON_DELIVERY_REASONS = [
|
@@ -81,9 +82,10 @@ module RightScale
|
|
81
82
|
when SUCCESS then 'success'
|
82
83
|
when ERROR then 'error' + (reason ? " (#{truncated_error})" : "")
|
83
84
|
when CONTINUE then 'continue'
|
84
|
-
when RETRY then 'retry'
|
85
|
+
when RETRY then 'retry' + (reason ? " (#{@content})" : "")
|
85
86
|
when NON_DELIVERY then 'non-delivery' + (reason ? " (#{@content})" : "")
|
86
87
|
when MULTICAST then 'multicast'
|
88
|
+
when CANCEL then 'cancel' + (reason ? " (#{@content})" : "")
|
87
89
|
end
|
88
90
|
end
|
89
91
|
|
@@ -126,7 +128,7 @@ module RightScale
|
|
126
128
|
# Create new success status
|
127
129
|
#
|
128
130
|
# === Parameters
|
129
|
-
# content(Object):: Any data associated with successful results
|
131
|
+
# content(Object):: Any data associated with successful results, defaults to nil
|
130
132
|
#
|
131
133
|
# === Return
|
132
134
|
# (OperationResult):: Corresponding result
|
@@ -151,7 +153,7 @@ module RightScale
|
|
151
153
|
# Create new continue status
|
152
154
|
#
|
153
155
|
# === Parameters
|
154
|
-
# content(Object):: Any data associated with continue
|
156
|
+
# content(Object):: Any data associated with continue, defaults to nil
|
155
157
|
#
|
156
158
|
# === Return
|
157
159
|
# (OperationResult):: Corresponding result
|
@@ -162,7 +164,7 @@ module RightScale
|
|
162
164
|
# Create new retry status
|
163
165
|
#
|
164
166
|
# === Parameters
|
165
|
-
# content(Object):: Any data associated with retry
|
167
|
+
# content(Object):: Any data associated with retry, defaults to nil
|
166
168
|
#
|
167
169
|
# === Return
|
168
170
|
# (OperationResult):: Corresponding result
|
@@ -193,6 +195,17 @@ module RightScale
|
|
193
195
|
OperationResult.new(MULTICAST, targets)
|
194
196
|
end
|
195
197
|
|
198
|
+
# Cancel request and never retry
|
199
|
+
#
|
200
|
+
# === Parameters
|
201
|
+
# content(Object):: Any data associated with cancel, defaults to nil
|
202
|
+
#
|
203
|
+
# === Return
|
204
|
+
# (OperationResult):: Corresponding result
|
205
|
+
def self.cancel(content = nil)
|
206
|
+
OperationResult.new(CANCEL, content)
|
207
|
+
end
|
208
|
+
|
196
209
|
# Was last operation successful?
|
197
210
|
#
|
198
211
|
# === Return
|
@@ -248,6 +261,15 @@ module RightScale
|
|
248
261
|
status_code == MULTICAST
|
249
262
|
end
|
250
263
|
|
264
|
+
# Was last operation status CANCEL?
|
265
|
+
#
|
266
|
+
# === Return
|
267
|
+
# true:: If status is CANCEL
|
268
|
+
# false:: Otherwise
|
269
|
+
def cancel?
|
270
|
+
status_code == CANCEL
|
271
|
+
end
|
272
|
+
|
251
273
|
# Array of serialized fields given to constructor
|
252
274
|
def serialized_members
|
253
275
|
[@status_code, @content]
|
@@ -263,6 +285,7 @@ module RightScale
|
|
263
285
|
def continue_result(*args) OperationResult.continue(*args) end
|
264
286
|
def retry_result(*args) OperationResult.retry(*args) end
|
265
287
|
def non_delivery_result(*args) OperationResult.non_delivery(*args) end
|
288
|
+
def cancel_result(*args) OperationResult.cancel(*args) end
|
266
289
|
def result_from(*args) OperationResult.from_results(*args) end
|
267
290
|
|
268
291
|
end # OperationResultHelper
|