right_agent 0.6.6 → 0.9.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/lib/right_agent/agent.rb +26 -25
- data/lib/right_agent/agent_config.rb +28 -2
- data/lib/right_agent/command/command_constants.rb +2 -2
- data/lib/right_agent/core_payload_types/executable_bundle.rb +3 -21
- data/lib/right_agent/core_payload_types/login_user.rb +19 -4
- data/lib/right_agent/core_payload_types/recipe_instantiation.rb +7 -1
- data/lib/right_agent/core_payload_types/right_script_instantiation.rb +7 -1
- data/lib/right_agent/dispatcher.rb +6 -19
- data/lib/right_agent/idempotent_request.rb +72 -17
- data/lib/right_agent/monkey_patches/ruby_patch.rb +0 -1
- data/lib/right_agent/monkey_patches.rb +0 -1
- data/lib/right_agent/operation_result.rb +27 -4
- data/lib/right_agent/packets.rb +47 -23
- data/lib/right_agent/platform/darwin.rb +33 -2
- data/lib/right_agent/platform/linux.rb +98 -2
- data/lib/right_agent/platform/windows.rb +41 -6
- data/lib/right_agent/platform.rb +11 -2
- data/lib/right_agent/scripts/agent_controller.rb +2 -1
- data/lib/right_agent/scripts/agent_deployer.rb +2 -2
- data/lib/right_agent/scripts/stats_manager.rb +7 -3
- data/lib/right_agent/sender.rb +45 -28
- data/lib/right_agent.rb +2 -5
- data/right_agent.gemspec +5 -3
- data/spec/agent_config_spec.rb +1 -1
- data/spec/agent_spec.rb +26 -20
- data/spec/core_payload_types/login_user_spec.rb +7 -3
- data/spec/idempotent_request_spec.rb +218 -48
- data/spec/operation_result_spec.rb +19 -0
- data/spec/packets_spec.rb +42 -1
- data/spec/platform/darwin.rb +11 -0
- data/spec/platform/linux.rb +23 -0
- data/spec/platform/linux_volume_manager_spec.rb +43 -43
- data/spec/platform/platform_spec.rb +35 -32
- data/spec/platform/windows.rb +11 -0
- data/spec/sender_spec.rb +21 -25
- metadata +47 -40
- data/lib/right_agent/broker_client.rb +0 -686
- data/lib/right_agent/ha_broker_client.rb +0 -1327
- data/lib/right_agent/monkey_patches/amqp_patch.rb +0 -274
- data/lib/right_agent/monkey_patches/ruby_patch/string_patch.rb +0 -107
- data/lib/right_agent/stats_helper.rb +0 -745
- data/spec/broker_client_spec.rb +0 -962
- data/spec/ha_broker_client_spec.rb +0 -1695
- data/spec/monkey_patches/amqp_patch_spec.rb +0 -100
- data/spec/monkey_patches/string_patch_spec.rb +0 -99
- data/spec/stats_helper_spec.rb +0 -686
data/lib/right_agent/agent.rb
CHANGED
@@ -31,7 +31,6 @@ module RightScale
|
|
31
31
|
|
32
32
|
include ConsoleHelper
|
33
33
|
include DaemonizeHelper
|
34
|
-
include StatsHelper
|
35
34
|
|
36
35
|
# (String) Identity of this agent
|
37
36
|
attr_reader :identity
|
@@ -45,7 +44,7 @@ module RightScale
|
|
45
44
|
# (ActorRegistry) Registry for this agents actors
|
46
45
|
attr_reader :registry
|
47
46
|
|
48
|
-
# (HABrokerClient) High availability AMQP broker client
|
47
|
+
# (RightAMQP::HABrokerClient) High availability AMQP broker client
|
49
48
|
attr_reader :broker
|
50
49
|
|
51
50
|
# (Array) Tag strings published by agent
|
@@ -174,6 +173,7 @@ module RightScale
|
|
174
173
|
def run
|
175
174
|
Log.init(@identity, @options[:log_path], :print => true)
|
176
175
|
Log.level = @options[:log_level] if @options[:log_level]
|
176
|
+
RightSupport::Log::Mixin.default_logger = Log
|
177
177
|
Log.debug("Start options:")
|
178
178
|
log_opts = @options.inject([]){ |t, (k, v)| t << "- #{k}: #{v}" }
|
179
179
|
log_opts.each { |l| Log.debug(l) }
|
@@ -188,7 +188,7 @@ module RightScale
|
|
188
188
|
|
189
189
|
# Initiate AMQP broker connection, wait for connection before proceeding
|
190
190
|
# otherwise messages published on failed connection will be lost
|
191
|
-
@broker = HABrokerClient.new(Serializer.new(:secure), @options)
|
191
|
+
@broker = RightAMQP::HABrokerClient.new(Serializer.new(:secure), @options)
|
192
192
|
@all_setup.each { |s| @remaining_setup[s] = @broker.all }
|
193
193
|
@broker.connection_status(:one_off => @options[:connect_timeout]) do |status|
|
194
194
|
if status == :connected
|
@@ -259,14 +259,15 @@ module RightScale
|
|
259
259
|
def tune_heartbeat(heartbeat)
|
260
260
|
res = nil
|
261
261
|
begin
|
262
|
+
Log.info("[setup] Reconnecting each broker to tune heartbeat to #{heartbeat}")
|
262
263
|
@broker.heartbeat = heartbeat
|
263
264
|
update_configuration(:heartbeat => heartbeat)
|
264
265
|
ids = []
|
265
266
|
all = @broker.all
|
266
267
|
all.each do |id|
|
267
268
|
begin
|
268
|
-
host, port, index, priority
|
269
|
-
@broker.connect(host, port, index, priority,
|
269
|
+
host, port, index, priority = @broker.identity_parts(id)
|
270
|
+
@broker.connect(host, port, index, priority, force = true) do |id|
|
270
271
|
@broker.connection_status(:one_off => @options[:connect_timeout], :brokers => [id]) do |status|
|
271
272
|
begin
|
272
273
|
if status == :connected
|
@@ -321,7 +322,7 @@ module RightScale
|
|
321
322
|
Log.info("Current broker configuration: #{@broker.status.inspect}")
|
322
323
|
res = nil
|
323
324
|
begin
|
324
|
-
@broker.connect(host, port, index, priority,
|
325
|
+
@broker.connect(host, port, index, priority, force) do |id|
|
325
326
|
@broker.connection_status(:one_off => @options[:connect_timeout], :brokers => [id]) do |status|
|
326
327
|
begin
|
327
328
|
if status == :connected
|
@@ -342,7 +343,7 @@ module RightScale
|
|
342
343
|
end
|
343
344
|
end
|
344
345
|
rescue Exception => e
|
345
|
-
res = Log.format("Failed to connect to broker #{HABrokerClient.identity(host, port)}", e)
|
346
|
+
res = Log.format("Failed to connect to broker #{RightAMQP::HABrokerClient.identity(host, port)}", e)
|
346
347
|
@exceptions.track("connect", e)
|
347
348
|
end
|
348
349
|
Log.error(res) if res
|
@@ -364,7 +365,7 @@ module RightScale
|
|
364
365
|
and_remove = " and removing" if remove
|
365
366
|
Log.info("Disconnecting#{and_remove} broker at host #{host.inspect} port #{port.inspect}")
|
366
367
|
Log.info("Current broker configuration: #{@broker.status.inspect}")
|
367
|
-
id = HABrokerClient.identity(host, port)
|
368
|
+
id = RightAMQP::HABrokerClient.identity(host, port)
|
368
369
|
@connect_requests.update("disconnect #{@broker.alias_(id)}")
|
369
370
|
connected = @broker.connected
|
370
371
|
res = nil
|
@@ -434,7 +435,7 @@ module RightScale
|
|
434
435
|
when Result then @sender.handle_response(packet)
|
435
436
|
end
|
436
437
|
@sender.message_received
|
437
|
-
rescue HABrokerClient::NoConnectedBrokers => e
|
438
|
+
rescue RightAMQP::HABrokerClient::NoConnectedBrokers => e
|
438
439
|
Log.error("Identity queue processing error", e)
|
439
440
|
rescue Exception => e
|
440
441
|
Log.error("Identity queue processing error", e, :trace)
|
@@ -578,9 +579,9 @@ module RightScale
|
|
578
579
|
# === Return
|
579
580
|
# true:: Always return true
|
580
581
|
def reset_agent_stats
|
581
|
-
@connect_requests =
|
582
|
-
@non_deliveries =
|
583
|
-
@exceptions =
|
582
|
+
@connect_requests = RightSupport::Stats::Activity.new(measure_rate = false)
|
583
|
+
@non_deliveries = RightSupport::Stats::Activity.new
|
584
|
+
@exceptions = RightSupport::Stats::Exceptions.new(self, @options[:exception_callback])
|
584
585
|
true
|
585
586
|
end
|
586
587
|
|
@@ -625,7 +626,7 @@ module RightScale
|
|
625
626
|
# (Boolean):: true if successful, otherwise false
|
626
627
|
def update_configuration(opts)
|
627
628
|
if cfg = AgentConfig.load_cfg(@agent_name)
|
628
|
-
opts.each { |k, v| cfg[k] = v
|
629
|
+
opts.each { |k, v| cfg[k] = v }
|
629
630
|
AgentConfig.store_cfg(@agent_name, cfg)
|
630
631
|
true
|
631
632
|
else
|
@@ -739,28 +740,18 @@ module RightScale
|
|
739
740
|
def finish_setup
|
740
741
|
@broker.failed.each do |id|
|
741
742
|
p = {:agent_identity => @identity}
|
742
|
-
p[:host], p[:port], p[:id], p[:priority]
|
743
|
+
p[:host], p[:port], p[:id], p[:priority] = @broker.identity_parts(id)
|
743
744
|
@sender.send_push("/registrar/connect", p)
|
744
745
|
end
|
745
746
|
true
|
746
747
|
end
|
747
748
|
|
748
749
|
# Check status of agent by gathering current operation statistics and publishing them,
|
749
|
-
#
|
750
|
+
# finishing any queue setup, and executing any deferred tasks
|
750
751
|
#
|
751
752
|
# === Return
|
752
753
|
# true:: Always return true
|
753
754
|
def check_status
|
754
|
-
@deferred_tasks.reject! do |t|
|
755
|
-
begin
|
756
|
-
t.call
|
757
|
-
rescue Exception => e
|
758
|
-
Log.error("Failed to perform deferred task", e)
|
759
|
-
@exceptions.track("check status", e)
|
760
|
-
end
|
761
|
-
true
|
762
|
-
end
|
763
|
-
|
764
755
|
begin
|
765
756
|
finish_setup
|
766
757
|
rescue Exception => e
|
@@ -779,6 +770,16 @@ module RightScale
|
|
779
770
|
@exceptions.track("check status", e)
|
780
771
|
end
|
781
772
|
|
773
|
+
@deferred_tasks.reject! do |t|
|
774
|
+
begin
|
775
|
+
t.call
|
776
|
+
rescue Exception => e
|
777
|
+
Log.error("Failed to perform deferred task", e)
|
778
|
+
@exceptions.track("check status", e)
|
779
|
+
end
|
780
|
+
true
|
781
|
+
end
|
782
|
+
|
782
783
|
@check_status_count += 1
|
783
784
|
true
|
784
785
|
end
|
@@ -63,7 +63,7 @@ module RightScale
|
|
63
63
|
module AgentConfig
|
64
64
|
|
65
65
|
# Current agent protocol version
|
66
|
-
PROTOCOL_VERSION =
|
66
|
+
PROTOCOL_VERSION = 20
|
67
67
|
|
68
68
|
# Current agent protocol version
|
69
69
|
#
|
@@ -73,6 +73,32 @@ module RightScale
|
|
73
73
|
PROTOCOL_VERSION
|
74
74
|
end
|
75
75
|
|
76
|
+
# Default thread name when no thread is specified for an executable bundle.
|
77
|
+
DEFAULT_THREAD_NAME = 'default'
|
78
|
+
|
79
|
+
# Default thread name when no thread is specified for an executable bundle.
|
80
|
+
#
|
81
|
+
# === Return
|
82
|
+
# (String):: default thread name...
|
83
|
+
def self.default_thread_name
|
84
|
+
DEFAULT_THREAD_NAME
|
85
|
+
end
|
86
|
+
|
87
|
+
# Regular expression to define what a valid thread name looks like: an alpha character
|
88
|
+
# followed by 0 or more alphanumerics or underscores. Only lower-case characters are
|
89
|
+
# allowed.
|
90
|
+
VALID_THREAD_NAME = /^[a-z][a-z0-9_]*$/
|
91
|
+
|
92
|
+
# Regular expression to define what a valid thread name looks like: an alpha character
|
93
|
+
# followed by 0 or more alphanumerics or underscores. Only lower-case characters are
|
94
|
+
# allowed.
|
95
|
+
#
|
96
|
+
# === Return
|
97
|
+
# (String):: default thread name...
|
98
|
+
def self.valid_thread_name
|
99
|
+
VALID_THREAD_NAME
|
100
|
+
end
|
101
|
+
|
76
102
|
# Initialize path to root directory of agent
|
77
103
|
#
|
78
104
|
# === Parameters
|
@@ -217,7 +243,7 @@ module RightScale
|
|
217
243
|
# === Return
|
218
244
|
# (String):: Directory path name
|
219
245
|
def self.cfg_dir
|
220
|
-
@cfg_dir ||= Platform.filesystem.
|
246
|
+
@cfg_dir ||= Platform.filesystem.right_agent_cfg_dir
|
221
247
|
end
|
222
248
|
|
223
249
|
# Path to generated agent configuration file
|
@@ -26,8 +26,8 @@ module RightScale
|
|
26
26
|
class CommandConstants
|
27
27
|
|
28
28
|
# Ports used for command protocol
|
29
|
-
BASE_INSTANCE_AGENT_SOCKET_PORT =
|
30
|
-
BASE_INSTANCE_AGENT_CHECKER_SOCKET_PORT =
|
29
|
+
BASE_INSTANCE_AGENT_SOCKET_PORT = 53843
|
30
|
+
BASE_INSTANCE_AGENT_CHECKER_SOCKET_PORT = 53844
|
31
31
|
|
32
32
|
end
|
33
33
|
end
|
@@ -35,14 +35,6 @@ module RightScale
|
|
35
35
|
|
36
36
|
include Serializable
|
37
37
|
|
38
|
-
# Default thread name when no thread is specified for an executable bundle.
|
39
|
-
DEFAULT_THREAD_NAME = 'default'
|
40
|
-
|
41
|
-
# Regular expression to define what a valid thread name looks like: an alpha character
|
42
|
-
# followed by 0 or more alphanumerics or underscores. Only lower-case characters are
|
43
|
-
# allowed.
|
44
|
-
VALID_THREAD_NAME = /^[a-z][a-z0-9_]*$/
|
45
|
-
|
46
38
|
# (Array) Collection of RightScripts and chef recipes instantiations
|
47
39
|
attr_accessor :executables
|
48
40
|
|
@@ -62,6 +54,9 @@ module RightScale
|
|
62
54
|
# (String) Repose server to use
|
63
55
|
attr_accessor :repose_servers
|
64
56
|
|
57
|
+
# (String) The name of the RIghtLink thread to run this bundle on
|
58
|
+
attr_accessor :thread_name
|
59
|
+
|
65
60
|
# (Hash):: collection of repos to be checked out on the instance
|
66
61
|
# :key (String):: the hash id (SHA) of the repository
|
67
62
|
# :value (Hash):: repo and cookbook detail
|
@@ -126,18 +121,5 @@ module RightScale
|
|
126
121
|
desc = @executables.collect { |e| e.nickname }.join(', ') if @executables
|
127
122
|
desc ||= 'empty bundle'
|
128
123
|
end
|
129
|
-
|
130
|
-
# Gets the thread name from a bundle, if any. Uses the default thread name for
|
131
|
-
# when a thread name is not specified (for backward compatibility, etc.).
|
132
|
-
#
|
133
|
-
# === Parameters
|
134
|
-
# bundle(ExecutableBundle):: bundle to inspect
|
135
|
-
#
|
136
|
-
# === Return
|
137
|
-
# thread_name(String):: thread name for bundle execution
|
138
|
-
def thread_name(value = nil)
|
139
|
-
@thread_name = value if value
|
140
|
-
return @thread_name || DEFAULT_THREAD_NAME
|
141
|
-
end
|
142
124
|
end
|
143
125
|
end
|
@@ -21,6 +21,8 @@
|
|
21
21
|
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
22
22
|
#
|
23
23
|
|
24
|
+
require 'net/ssh'
|
25
|
+
|
24
26
|
module RightScale
|
25
27
|
|
26
28
|
# Authorized user for the Managed Login feature
|
@@ -29,7 +31,7 @@ module RightScale
|
|
29
31
|
include Serializable
|
30
32
|
|
31
33
|
attr_accessor :uuid, :username, :public_key, :public_keys, :common_name,
|
32
|
-
:superuser, :expires_at, :profile_data
|
34
|
+
:superuser, :expires_at, :profile_data, :public_key_fingerprints
|
33
35
|
|
34
36
|
# Initialize fields from given arguments
|
35
37
|
def initialize(*args)
|
@@ -41,9 +43,10 @@ module RightScale
|
|
41
43
|
@expires_at = Time.at(args[5]) if args[5] && (args[5] != 0) # nil -> 0 because of expires_at.to_i below
|
42
44
|
@public_keys = args[6]
|
43
45
|
@profile_data = args[7]
|
46
|
+
@public_key_fingerprints = args[8]
|
44
47
|
|
45
|
-
#
|
46
|
-
# singular public_key as a legacy member.
|
48
|
+
# We now expect an array of public_keys to be passed while supporting the
|
49
|
+
# singular public_key as a legacy member. When serialized back from a
|
47
50
|
# legacy LoginUser record, the singular value may be set while the plural
|
48
51
|
# is nil.
|
49
52
|
if @public_keys
|
@@ -53,11 +56,23 @@ module RightScale
|
|
53
56
|
raise ArgumentError, "Expected public_key (third argument) to be a string" unless @public_key.is_a?(String)
|
54
57
|
@public_keys = [@public_key]
|
55
58
|
end
|
59
|
+
|
60
|
+
# The number of fingerprints must match the number of public keys
|
61
|
+
if @public_key_fingerprints && @public_key_fingerprints.size != @public_keys.size
|
62
|
+
raise ArgumentError, "Expected public_keys (seventh argument) array length (#{@public_keys.size}) is not " +
|
63
|
+
"the same as the public_key_fingerprints (eighth argument) (#{@public_key_fingerprints.size})"
|
64
|
+
end
|
56
65
|
end
|
57
66
|
|
58
67
|
# Array of serialized fields given to constructor
|
59
68
|
def serialized_members
|
60
|
-
[ @uuid, @username, @public_key, @common_name, @superuser, @expires_at.to_i, @public_keys, @profile_data
|
69
|
+
[ @uuid, @username, @public_key, @common_name, @superuser, @expires_at.to_i, @public_keys, @profile_data,
|
70
|
+
@public_key_fingerprints ]
|
71
|
+
end
|
72
|
+
|
73
|
+
# Create fingerprint for public key
|
74
|
+
def self.fingerprint(public_key)
|
75
|
+
Digest::SHA1.hexdigest(::Net::SSH::KeyFactory.load_data_public_key(public_key).to_der)
|
61
76
|
end
|
62
77
|
|
63
78
|
end
|
@@ -43,17 +43,23 @@ module RightScale
|
|
43
43
|
# (Array of SecureDocumentLocation) attributes that must be resolved by the instance
|
44
44
|
attr_accessor :external_inputs
|
45
45
|
|
46
|
+
# (Hash) nil or Hash of input name to flags (array of string tokens) indicating additional
|
47
|
+
# boolean properties of the input which are useful to the instance. the presence of the
|
48
|
+
# flag means true, absence means false.
|
49
|
+
attr_accessor :input_flags
|
50
|
+
|
46
51
|
def initialize(*args)
|
47
52
|
@nickname = args[0] if args.size > 0
|
48
53
|
@attributes = args[1] if args.size > 1
|
49
54
|
@id = args[2] if args.size > 2
|
50
55
|
@ready = args[3] if args.size > 3
|
51
56
|
@external_inputs = args[4] if args.size > 4
|
57
|
+
@input_flags = args[5] if args.size > 5
|
52
58
|
end
|
53
59
|
|
54
60
|
# Array of serialized fields given to constructor
|
55
61
|
def serialized_members
|
56
|
-
[ @nickname, @attributes, @id, @ready, @external_inputs ]
|
62
|
+
[ @nickname, @attributes, @id, @ready, @external_inputs, @input_flags ]
|
57
63
|
end
|
58
64
|
|
59
65
|
end
|
@@ -53,6 +53,11 @@ module RightScale
|
|
53
53
|
# (Array of SecureDocumentLocation) attributes that must be resolved by the instance
|
54
54
|
attr_accessor :external_inputs
|
55
55
|
|
56
|
+
# (Hash) nil or Hash of input name to flags (array of string tokens) indicating additional
|
57
|
+
# boolean properties of the input which are useful to the instance. the presence of the
|
58
|
+
# flag means true, absense means false.
|
59
|
+
attr_accessor :input_flags
|
60
|
+
|
56
61
|
def initialize(*args)
|
57
62
|
@nickname = args[0] if args.size > 0
|
58
63
|
@source = args[1] if args.size > 1
|
@@ -62,11 +67,12 @@ module RightScale
|
|
62
67
|
@id = args[5] if args.size > 5
|
63
68
|
@ready = args[6] if args.size > 6
|
64
69
|
@external_inputs = args[7] if args.size > 7
|
70
|
+
@input_flags = args[8] if args.size > 8
|
65
71
|
end
|
66
72
|
|
67
73
|
# Array of serialized fields given to constructor
|
68
74
|
def serialized_members
|
69
|
-
[ @nickname, @source, @parameters, @attachments, @packages, @id, @ready, @external_inputs ]
|
75
|
+
[ @nickname, @source, @parameters, @attachments, @packages, @id, @ready, @external_inputs, @input_flags ]
|
70
76
|
end
|
71
77
|
|
72
78
|
end
|
@@ -25,8 +25,6 @@ module RightScale
|
|
25
25
|
# Dispatching of payload to specified actor
|
26
26
|
class Dispatcher
|
27
27
|
|
28
|
-
include StatsHelper
|
29
|
-
|
30
28
|
# Response queue name
|
31
29
|
RESPONSE_QUEUE = "response"
|
32
30
|
|
@@ -112,7 +110,7 @@ module RightScale
|
|
112
110
|
# (String) Identity of associated agent
|
113
111
|
attr_reader :identity
|
114
112
|
|
115
|
-
# (HABrokerClient) High availability AMQP broker client
|
113
|
+
# (RightAMQP::HABrokerClient) High availability AMQP broker client
|
116
114
|
attr_reader :broker
|
117
115
|
|
118
116
|
# (EM) Event machine class (exposed for unit tests)
|
@@ -174,7 +172,7 @@ module RightScale
|
|
174
172
|
# Reject this request if its TTL has expired
|
175
173
|
if (expires_at = request.expires_at) && expires_at > 0 && received_at.to_i >= expires_at
|
176
174
|
@rejects.update("expired (#{method})")
|
177
|
-
Log.info("REJECT EXPIRED <#{token}> from #{request.from} TTL #{elapsed(received_at.to_i - expires_at)} ago")
|
175
|
+
Log.info("REJECT EXPIRED <#{token}> from #{request.from} TTL #{RightSupport::Stats.elapsed(received_at.to_i - expires_at)} ago")
|
178
176
|
if request.is_a?(Request)
|
179
177
|
# For agents that do not know about non-delivery, use error result
|
180
178
|
non_delivery = if request.recv_version < 13
|
@@ -228,7 +226,7 @@ module RightScale
|
|
228
226
|
exchange = {:type => :queue, :name => RESPONSE_QUEUE, :options => {:durable => true, :no_declare => @secure}}
|
229
227
|
@broker.publish(exchange, r, :persistent => true, :mandatory => true, :log_filter => [:tries, :persistent, :duration])
|
230
228
|
end
|
231
|
-
rescue HABrokerClient::NoConnectedBrokers => e
|
229
|
+
rescue RightAMQP::HABrokerClient::NoConnectedBrokers => e
|
232
230
|
Log.error("Failed to publish result of dispatched request #{request.trace}", e)
|
233
231
|
rescue Exception => e
|
234
232
|
Log.error("Failed to publish result of dispatched request #{request.trace}", e, :trace)
|
@@ -298,9 +296,9 @@ module RightScale
|
|
298
296
|
# === Return
|
299
297
|
# true:: Always return true
|
300
298
|
def reset_stats
|
301
|
-
@rejects =
|
302
|
-
@requests =
|
303
|
-
@exceptions =
|
299
|
+
@rejects = RightSupport::Stats::Activity.new
|
300
|
+
@requests = RightSupport::Stats::Activity.new
|
301
|
+
@exceptions = RightSupport::Stats::Exceptions.new(@agent)
|
304
302
|
true
|
305
303
|
end
|
306
304
|
|
@@ -335,17 +333,6 @@ module RightScale
|
|
335
333
|
error
|
336
334
|
end
|
337
335
|
|
338
|
-
# Convert value to nil if equals 0
|
339
|
-
#
|
340
|
-
# === Parameters
|
341
|
-
# value(Integer|nil):: Value to be converted
|
342
|
-
#
|
343
|
-
# === Return
|
344
|
-
# (Integer|nil):: Converted value
|
345
|
-
def nil_if_zero(value)
|
346
|
-
if !value || value == 0 then nil else value end
|
347
|
-
end
|
348
|
-
|
349
336
|
end # Dispatcher
|
350
337
|
|
351
338
|
end # RightScale
|
@@ -22,14 +22,46 @@
|
|
22
22
|
|
23
23
|
module RightScale
|
24
24
|
|
25
|
+
# This is a retryable request for use when the execution of the request by the
|
26
|
+
# receiver is known to be idempotent and when there is a need to indefinitely
|
27
|
+
# pursue getting a usable response, e.g., when an instance is launching.
|
28
|
+
# It is implemented as an EM::Deferrable and as such invokes the Proc defined
|
29
|
+
# with its #callback method with the result content from a OperationResult::SUCCESS
|
30
|
+
# response, or it will invoke the Proc defined with its #errback method with error
|
31
|
+
# content if the response is an OperationResult::ERROR or CANCEL, or if the request
|
32
|
+
# has timed out. The request can be canceled with the #cancel method, or the receiver
|
33
|
+
# of the request may respond with a CANCEL result to cause the request to be canceled.
|
34
|
+
# This is useful in situations where the request is never expected to succeed
|
35
|
+
# regardless of the number of retries. By default if the response to the request
|
36
|
+
# is an OperationResult::RETRY or NON_DELIVERY indication, the request is automatically
|
37
|
+
# retried, as is also the case for an ERROR indication if the :retry_on_error option
|
38
|
+
# is specified. The retry algorithm is controlled by the :retry_delay, :retry_delay_count,
|
39
|
+
# and :max_retry_delay settings. The initial retry interval is the default or specified
|
40
|
+
# :retry_delay and this interval is used :retry_delay_count times, at which point the
|
41
|
+
# :retry_delay is doubled and the :retry_delay_count is halved. This backoff is again
|
42
|
+
# applied after the new :retry_delay_count is reached, and so on until :retry_delay
|
43
|
+
# reaches :max_retry_delay which then is used as the interval until the default or
|
44
|
+
# specified :timeout is reached. The default :timeout is 4 days.
|
25
45
|
class IdempotentRequest
|
26
46
|
|
27
47
|
include OperationResultHelper
|
28
48
|
include EM::Deferrable
|
29
49
|
|
30
|
-
#
|
50
|
+
# Default delay before initial retry in case of failure with -1 meaning no delay
|
31
51
|
DEFAULT_RETRY_DELAY = 5
|
32
52
|
|
53
|
+
# Default minimum number of retries before beginning backoff
|
54
|
+
DEFAULT_RETRY_DELAY_COUNT = 60
|
55
|
+
|
56
|
+
# Maximum default delay before retry when backing off
|
57
|
+
DEFAULT_MAX_RETRY_DELAY = 60
|
58
|
+
|
59
|
+
# Factor used for exponential backoff of retry delay
|
60
|
+
RETRY_BACKOFF_FACTOR = 2
|
61
|
+
|
62
|
+
# Default timeout with -1 meaning never timeout
|
63
|
+
DEFAULT_TIMEOUT = 4 * 24 * 60 * 60
|
64
|
+
|
33
65
|
attr_reader :raw_response
|
34
66
|
|
35
67
|
# Send idempotent request
|
@@ -37,18 +69,31 @@ module RightScale
|
|
37
69
|
# Calls deferrable callback on completion, error callback on timeout
|
38
70
|
#
|
39
71
|
# === Parameters
|
40
|
-
# operation(String):: Request operation (
|
72
|
+
# operation(String):: Request operation (e.g., '/booter/get_boot_bundle')
|
41
73
|
# payload(Hash):: Request payload
|
42
|
-
# options
|
43
|
-
#
|
44
|
-
#
|
45
|
-
#
|
46
|
-
|
47
|
-
|
48
|
-
|
74
|
+
# options(Hash):: Request options
|
75
|
+
# :targets(Array):: Targets from which to randomly choose one
|
76
|
+
# :retry_on_error(Boolean):: Whether request should be retried if recipient returned an error
|
77
|
+
# :retry_delay(Fixnum):: Number of seconds delay before initial retry with -1 meaning no delay,
|
78
|
+
# defaults to DEFAULT_RETRY_DELAY
|
79
|
+
# :retry_delay_count(Fixnum):: Minimum number of retries at initial :retry_delay value before
|
80
|
+
# increasing delay exponentially and decreasing this count exponentially, defaults to
|
81
|
+
# DEFAULT_RETRY_DELAY_COUNT
|
82
|
+
# :max_retry_delay(Fixnum):: Maximum number of seconds of retry delay, defaults to DEFAULT_MAX_RETRY_DELAY
|
83
|
+
# :timeout(Fixnum):: Number of seconds with no response before error callback gets called with
|
84
|
+
# -1 meaning never, defaults to DEFAULT_TIMEOUT
|
85
|
+
#
|
86
|
+
# === Raises
|
87
|
+
# ArgumentError:: If operation or payload not specified
|
88
|
+
def initialize(operation, payload, options = {})
|
89
|
+
raise ArgumentError.new("operation is required") unless @operation = operation
|
90
|
+
raise ArgumentError.new("payload is required") unless @payload = payload
|
49
91
|
@retry_on_error = options[:retry_on_error] || false
|
50
|
-
@timeout = options[:timeout] ||
|
92
|
+
@timeout = options[:timeout] || DEFAULT_TIMEOUT
|
51
93
|
@retry_delay = options[:retry_delay] || DEFAULT_RETRY_DELAY
|
94
|
+
@retry_delay_count = options[:retry_delay_count] || DEFAULT_RETRY_DELAY_COUNT
|
95
|
+
@max_retry_delay = options[:max_retry_delay] || DEFAULT_MAX_RETRY_DELAY
|
96
|
+
@retries = 0
|
52
97
|
@targets = options[:targets]
|
53
98
|
@raw_response = nil
|
54
99
|
@done = false
|
@@ -64,7 +109,7 @@ module RightScale
|
|
64
109
|
if @cancel_timer.nil? && @timeout > 0
|
65
110
|
@cancel_timer = EM::Timer.new(@timeout) do
|
66
111
|
msg = "Request #{@operation} timed out after #{@timeout} seconds"
|
67
|
-
|
112
|
+
Log.info(msg)
|
68
113
|
cancel(msg)
|
69
114
|
end
|
70
115
|
end
|
@@ -101,7 +146,6 @@ module RightScale
|
|
101
146
|
return true if @done
|
102
147
|
@raw_response = r
|
103
148
|
res = result_from(r)
|
104
|
-
res = OperationResult.non_delivery unless res
|
105
149
|
if res.success?
|
106
150
|
if @cancel_timer
|
107
151
|
@cancel_timer.cancel
|
@@ -110,17 +154,28 @@ module RightScale
|
|
110
154
|
@done = true
|
111
155
|
succeed(res.content)
|
112
156
|
else
|
157
|
+
reason = res.content
|
113
158
|
if res.non_delivery?
|
114
|
-
Log.info("Request non-delivery (#{
|
159
|
+
Log.info("Request non-delivery (#{reason}) for #{@operation}")
|
115
160
|
elsif res.retry?
|
116
|
-
|
161
|
+
reason = (reason && !reason.empty?) ? reason : "RightScale not ready"
|
162
|
+
Log.info("Request #{@operation} failed (#{reason}) and should be retried")
|
163
|
+
elsif res.cancel?
|
164
|
+
reason = (reason && !reason.empty?) ? reason : "RightScale cannot execute request"
|
165
|
+
Log.info("Request #{@operation} canceled (#{reason})")
|
117
166
|
else
|
118
|
-
Log.info("Request #{@operation} failed (#{
|
167
|
+
Log.info("Request #{@operation} failed (#{reason})")
|
119
168
|
end
|
120
|
-
if res.non_delivery? || res.retry? || @retry_on_error
|
169
|
+
if (res.non_delivery? || res.retry? || @retry_on_error) && !res.cancel?
|
121
170
|
Log.info("Retrying in #{@retry_delay} seconds...")
|
122
171
|
if @retry_delay > 0
|
123
|
-
|
172
|
+
this_delay = @retry_delay
|
173
|
+
if (@retries += 1) >= @retry_delay_count
|
174
|
+
@retry_delay = [@retry_delay * RETRY_BACKOFF_FACTOR, @max_retry_delay].min
|
175
|
+
@retry_delay_count = [@retry_delay_count / RETRY_BACKOFF_FACTOR, 1].max
|
176
|
+
@retries = 0
|
177
|
+
end
|
178
|
+
EM.add_timer(this_delay) { run }
|
124
179
|
else
|
125
180
|
EM.next_tick { run }
|
126
181
|
end
|
@@ -42,7 +42,6 @@ end
|
|
42
42
|
RUBY_PATCH_BASE_DIR = File.join(File.dirname(__FILE__), 'ruby_patch')
|
43
43
|
|
44
44
|
require File.normalize_path(File.join(RUBY_PATCH_BASE_DIR, 'array_patch'))
|
45
|
-
require File.normalize_path(File.join(RUBY_PATCH_BASE_DIR, 'string_patch'))
|
46
45
|
require File.normalize_path(File.join(RUBY_PATCH_BASE_DIR, 'object_patch'))
|
47
46
|
require File.normalize_path(File.join(RUBY_PATCH_BASE_DIR, 'singleton_patch'))
|
48
47
|
|
@@ -35,6 +35,7 @@ module RightScale
|
|
35
35
|
RETRY = 3
|
36
36
|
NON_DELIVERY = 4
|
37
37
|
MULTICAST = 5 # Deprecated for agents at version 13 or above
|
38
|
+
CANCEL = 6
|
38
39
|
|
39
40
|
# Non-delivery reasons
|
40
41
|
NON_DELIVERY_REASONS = [
|
@@ -81,9 +82,10 @@ module RightScale
|
|
81
82
|
when SUCCESS then 'success'
|
82
83
|
when ERROR then 'error' + (reason ? " (#{truncated_error})" : "")
|
83
84
|
when CONTINUE then 'continue'
|
84
|
-
when RETRY then 'retry'
|
85
|
+
when RETRY then 'retry' + (reason ? " (#{@content})" : "")
|
85
86
|
when NON_DELIVERY then 'non-delivery' + (reason ? " (#{@content})" : "")
|
86
87
|
when MULTICAST then 'multicast'
|
88
|
+
when CANCEL then 'cancel' + (reason ? " (#{@content})" : "")
|
87
89
|
end
|
88
90
|
end
|
89
91
|
|
@@ -126,7 +128,7 @@ module RightScale
|
|
126
128
|
# Create new success status
|
127
129
|
#
|
128
130
|
# === Parameters
|
129
|
-
# content(Object):: Any data associated with successful results
|
131
|
+
# content(Object):: Any data associated with successful results, defaults to nil
|
130
132
|
#
|
131
133
|
# === Return
|
132
134
|
# (OperationResult):: Corresponding result
|
@@ -151,7 +153,7 @@ module RightScale
|
|
151
153
|
# Create new continue status
|
152
154
|
#
|
153
155
|
# === Parameters
|
154
|
-
# content(Object):: Any data associated with continue
|
156
|
+
# content(Object):: Any data associated with continue, defaults to nil
|
155
157
|
#
|
156
158
|
# === Return
|
157
159
|
# (OperationResult):: Corresponding result
|
@@ -162,7 +164,7 @@ module RightScale
|
|
162
164
|
# Create new retry status
|
163
165
|
#
|
164
166
|
# === Parameters
|
165
|
-
# content(Object):: Any data associated with retry
|
167
|
+
# content(Object):: Any data associated with retry, defaults to nil
|
166
168
|
#
|
167
169
|
# === Return
|
168
170
|
# (OperationResult):: Corresponding result
|
@@ -193,6 +195,17 @@ module RightScale
|
|
193
195
|
OperationResult.new(MULTICAST, targets)
|
194
196
|
end
|
195
197
|
|
198
|
+
# Cancel request and never retry
|
199
|
+
#
|
200
|
+
# === Parameters
|
201
|
+
# content(Object):: Any data associated with cancel, defaults to nil
|
202
|
+
#
|
203
|
+
# === Return
|
204
|
+
# (OperationResult):: Corresponding result
|
205
|
+
def self.cancel(content = nil)
|
206
|
+
OperationResult.new(CANCEL, content)
|
207
|
+
end
|
208
|
+
|
196
209
|
# Was last operation successful?
|
197
210
|
#
|
198
211
|
# === Return
|
@@ -248,6 +261,15 @@ module RightScale
|
|
248
261
|
status_code == MULTICAST
|
249
262
|
end
|
250
263
|
|
264
|
+
# Was last operation status CANCEL?
|
265
|
+
#
|
266
|
+
# === Return
|
267
|
+
# true:: If status is CANCEL
|
268
|
+
# false:: Otherwise
|
269
|
+
def cancel?
|
270
|
+
status_code == CANCEL
|
271
|
+
end
|
272
|
+
|
251
273
|
# Array of serialized fields given to constructor
|
252
274
|
def serialized_members
|
253
275
|
[@status_code, @content]
|
@@ -263,6 +285,7 @@ module RightScale
|
|
263
285
|
def continue_result(*args) OperationResult.continue(*args) end
|
264
286
|
def retry_result(*args) OperationResult.retry(*args) end
|
265
287
|
def non_delivery_result(*args) OperationResult.non_delivery(*args) end
|
288
|
+
def cancel_result(*args) OperationResult.cancel(*args) end
|
266
289
|
def result_from(*args) OperationResult.from_results(*args) end
|
267
290
|
|
268
291
|
end # OperationResultHelper
|