right_agent 0.6.6 → 0.9.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/lib/right_agent/agent.rb +26 -25
- data/lib/right_agent/agent_config.rb +28 -2
- data/lib/right_agent/command/command_constants.rb +2 -2
- data/lib/right_agent/core_payload_types/executable_bundle.rb +3 -21
- data/lib/right_agent/core_payload_types/login_user.rb +19 -4
- data/lib/right_agent/core_payload_types/recipe_instantiation.rb +7 -1
- data/lib/right_agent/core_payload_types/right_script_instantiation.rb +7 -1
- data/lib/right_agent/dispatcher.rb +6 -19
- data/lib/right_agent/idempotent_request.rb +72 -17
- data/lib/right_agent/monkey_patches/ruby_patch.rb +0 -1
- data/lib/right_agent/monkey_patches.rb +0 -1
- data/lib/right_agent/operation_result.rb +27 -4
- data/lib/right_agent/packets.rb +47 -23
- data/lib/right_agent/platform/darwin.rb +33 -2
- data/lib/right_agent/platform/linux.rb +98 -2
- data/lib/right_agent/platform/windows.rb +41 -6
- data/lib/right_agent/platform.rb +11 -2
- data/lib/right_agent/scripts/agent_controller.rb +2 -1
- data/lib/right_agent/scripts/agent_deployer.rb +2 -2
- data/lib/right_agent/scripts/stats_manager.rb +7 -3
- data/lib/right_agent/sender.rb +45 -28
- data/lib/right_agent.rb +2 -5
- data/right_agent.gemspec +5 -3
- data/spec/agent_config_spec.rb +1 -1
- data/spec/agent_spec.rb +26 -20
- data/spec/core_payload_types/login_user_spec.rb +7 -3
- data/spec/idempotent_request_spec.rb +218 -48
- data/spec/operation_result_spec.rb +19 -0
- data/spec/packets_spec.rb +42 -1
- data/spec/platform/darwin.rb +11 -0
- data/spec/platform/linux.rb +23 -0
- data/spec/platform/linux_volume_manager_spec.rb +43 -43
- data/spec/platform/platform_spec.rb +35 -32
- data/spec/platform/windows.rb +11 -0
- data/spec/sender_spec.rb +21 -25
- metadata +47 -40
- data/lib/right_agent/broker_client.rb +0 -686
- data/lib/right_agent/ha_broker_client.rb +0 -1327
- data/lib/right_agent/monkey_patches/amqp_patch.rb +0 -274
- data/lib/right_agent/monkey_patches/ruby_patch/string_patch.rb +0 -107
- data/lib/right_agent/stats_helper.rb +0 -745
- data/spec/broker_client_spec.rb +0 -962
- data/spec/ha_broker_client_spec.rb +0 -1695
- data/spec/monkey_patches/amqp_patch_spec.rb +0 -100
- data/spec/monkey_patches/string_patch_spec.rb +0 -99
- data/spec/stats_helper_spec.rb +0 -686
data/lib/right_agent/sender.rb
CHANGED
@@ -28,8 +28,6 @@ module RightScale
|
|
28
28
|
# All requests go through the mapper for security purposes
|
29
29
|
class Sender
|
30
30
|
|
31
|
-
include StatsHelper
|
32
|
-
|
33
31
|
# Request that is waiting for a response
|
34
32
|
class PendingRequest
|
35
33
|
|
@@ -172,7 +170,7 @@ module RightScale
|
|
172
170
|
# === Parameters
|
173
171
|
# restart_callback(Proc):: Callback that is activated on each restart vote with votes being initiated
|
174
172
|
# by offline queue exceeding MAX_QUEUED_REQUESTS
|
175
|
-
# offline_stats(
|
173
|
+
# offline_stats(RightSupport::Stats::Activity):: Offline queue tracking statistics
|
176
174
|
def initialize(restart_callback, offline_stats)
|
177
175
|
@restart_vote = restart_callback
|
178
176
|
@restart_vote_timer = nil
|
@@ -387,12 +385,16 @@ module RightScale
|
|
387
385
|
# Number of seconds to wait for ping response from a mapper when checking connectivity
|
388
386
|
PING_TIMEOUT = 30
|
389
387
|
|
388
|
+
# Default maximum number of consecutive ping timeouts before attempt to reconnect
|
389
|
+
MAX_PING_TIMEOUTS = 3
|
390
|
+
|
390
391
|
# (EM::Timer) Timer while waiting for mapper ping response
|
391
392
|
attr_accessor :ping_timer
|
392
393
|
|
393
394
|
def initialize(sender, check_interval, ping_stats, exception_stats)
|
394
395
|
@sender = sender
|
395
396
|
@check_interval = check_interval
|
397
|
+
@ping_timeouts = {}
|
396
398
|
@ping_timer = nil
|
397
399
|
@ping_stats = ping_stats
|
398
400
|
@exception_stats = exception_stats
|
@@ -427,25 +429,35 @@ module RightScale
|
|
427
429
|
end
|
428
430
|
|
429
431
|
# Check whether broker connection is usable by pinging a mapper via that broker
|
430
|
-
# Attempt to reconnect if ping does not respond in PING_TIMEOUT seconds
|
432
|
+
# Attempt to reconnect if ping does not respond in PING_TIMEOUT seconds and
|
433
|
+
# if have reached timeout limit
|
431
434
|
# Ignore request if already checking a connection
|
432
435
|
# Only to be called from primary thread
|
433
436
|
#
|
434
437
|
# === Parameters
|
435
438
|
# id(String):: Identity of specific broker to use to send ping, defaults to any
|
436
439
|
# currently connected broker
|
440
|
+
# max_ping_timeouts(Integer):: Maximum number of ping timeouts before attempt
|
441
|
+
# to reconnect, defaults to MAX_PING_TIMEOUTS
|
437
442
|
#
|
438
443
|
# === Return
|
439
444
|
# true:: Always return true
|
440
|
-
def check(id = nil)
|
445
|
+
def check(id = nil, max_ping_timeouts = MAX_PING_TIMEOUTS)
|
441
446
|
unless @terminating || @ping_timer || (id && !@sender.broker.connected?(id))
|
442
447
|
@ping_timer = EM::Timer.new(PING_TIMEOUT) do
|
443
448
|
begin
|
444
449
|
@ping_stats.update("timeout")
|
445
450
|
@ping_timer = nil
|
446
|
-
|
447
|
-
|
448
|
-
|
451
|
+
@ping_timeouts[id] = (@ping_timeouts[id] || 0) + 1
|
452
|
+
if @ping_timeouts[id] >= max_ping_timeouts
|
453
|
+
Log.error("Mapper ping via broker #{id} timed out after #{PING_TIMEOUT} seconds and now " +
|
454
|
+
"reached maximum of #{max_ping_timeouts} timeout#{max_ping_timeouts > 1 ? 's' : ''}, " +
|
455
|
+
"attempting to reconnect")
|
456
|
+
host, port, index, priority = @sender.broker.identity_parts(id)
|
457
|
+
@sender.agent.connect(host, port, index, priority, force = true)
|
458
|
+
else
|
459
|
+
Log.warning("Mapper ping via broker #{id} timed out after #{PING_TIMEOUT} seconds")
|
460
|
+
end
|
449
461
|
rescue Exception => e
|
450
462
|
Log.error("Failed to reconnect to broker #{id}", e, :trace)
|
451
463
|
@exception_stats.track("ping timeout", e)
|
@@ -458,6 +470,7 @@ module RightScale
|
|
458
470
|
@ping_stats.update("success")
|
459
471
|
@ping_timer.cancel
|
460
472
|
@ping_timer = nil
|
473
|
+
@ping_timeouts[id] = 0
|
461
474
|
end
|
462
475
|
rescue Exception => e
|
463
476
|
Log.error("Failed to cancel mapper ping", e, :trace)
|
@@ -500,7 +513,7 @@ module RightScale
|
|
500
513
|
@inactivity_timer.cancel if @inactivity_timer
|
501
514
|
@inactivity_timer = EM::Timer.new(@check_interval) do
|
502
515
|
begin
|
503
|
-
check
|
516
|
+
check(id = nil, max_ping_timeouts = 1)
|
504
517
|
rescue Exception => e
|
505
518
|
Log.error("Failed connectivity check", e, :trace)
|
506
519
|
@exception_stats.track("check connectivity", e)
|
@@ -512,7 +525,7 @@ module RightScale
|
|
512
525
|
end # ConnectivityChecker
|
513
526
|
|
514
527
|
# Factor used on each retry iteration to achieve exponential backoff
|
515
|
-
RETRY_BACKOFF_FACTOR =
|
528
|
+
RETRY_BACKOFF_FACTOR = 3
|
516
529
|
|
517
530
|
# (PendingRequests) Requests waiting for a response
|
518
531
|
attr_accessor :pending_requests
|
@@ -523,7 +536,7 @@ module RightScale
|
|
523
536
|
# (ConnectivityChecker) Broker connection checker
|
524
537
|
attr_reader :connectivity_checker
|
525
538
|
|
526
|
-
# (HABrokerClient) High availability AMQP broker client
|
539
|
+
# (RightAMQP::HABrokerClient) High availability AMQP broker client
|
527
540
|
attr_accessor :broker
|
528
541
|
|
529
542
|
# (String) Identity of the associated agent
|
@@ -569,8 +582,8 @@ module RightScale
|
|
569
582
|
@broker = @agent.broker
|
570
583
|
@secure = @options[:secure]
|
571
584
|
@single_threaded = @options[:single_threaded]
|
572
|
-
@retry_timeout = nil_if_zero(@options[:retry_timeout])
|
573
|
-
@retry_interval = nil_if_zero(@options[:retry_interval])
|
585
|
+
@retry_timeout = RightSupport::Stats.nil_if_zero(@options[:retry_timeout])
|
586
|
+
@retry_interval = RightSupport::Stats.nil_if_zero(@options[:retry_interval])
|
574
587
|
|
575
588
|
# Only to be accessed from primary thread
|
576
589
|
@pending_requests = PendingRequests.new
|
@@ -821,7 +834,7 @@ module RightScale
|
|
821
834
|
exchange = {:type => :fanout, :name => "request", :options => {:durable => true, :no_declare => @secure}}
|
822
835
|
ids = @broker.publish(exchange, request, :persistent => request.persistent, :mandatory => true,
|
823
836
|
:log_filter => [:tags, :target, :tries, :persistent], :brokers => ids)
|
824
|
-
rescue HABrokerClient::NoConnectedBrokers => e
|
837
|
+
rescue RightAMQP::HABrokerClient::NoConnectedBrokers => e
|
825
838
|
Log.error("Failed to publish request #{request.to_s([:tags, :target, :tries])}", e)
|
826
839
|
ids = []
|
827
840
|
rescue Exception => e
|
@@ -923,15 +936,15 @@ module RightScale
|
|
923
936
|
# === Return
|
924
937
|
# true:: Always return true
|
925
938
|
def reset_stats
|
926
|
-
@ping_stats =
|
927
|
-
@retry_stats =
|
928
|
-
@request_stats =
|
929
|
-
@result_stats =
|
930
|
-
@result_error_stats =
|
931
|
-
@non_delivery_stats =
|
932
|
-
@offline_stats =
|
933
|
-
@request_kinds =
|
934
|
-
@exception_stats =
|
939
|
+
@ping_stats = RightSupport::Stats::Activity.new
|
940
|
+
@retry_stats = RightSupport::Stats::Activity.new
|
941
|
+
@request_stats = RightSupport::Stats::Activity.new
|
942
|
+
@result_stats = RightSupport::Stats::Activity.new
|
943
|
+
@result_error_stats = RightSupport::Stats::Activity.new
|
944
|
+
@non_delivery_stats = RightSupport::Stats::Activity.new
|
945
|
+
@offline_stats = RightSupport::Stats::Activity.new(measure_rate = false)
|
946
|
+
@request_kinds = RightSupport::Stats::Activity.new(measure_rate = false)
|
947
|
+
@exception_stats = RightSupport::Stats::Exceptions.new(@agent, @options[:exception_callback])
|
935
948
|
true
|
936
949
|
end
|
937
950
|
|
@@ -1104,6 +1117,7 @@ module RightScale
|
|
1104
1117
|
# Use exponential backoff with RETRY_BACKOFF_FACTOR for retry spacing
|
1105
1118
|
# Adjust retry interval by average response time to avoid adding to system load
|
1106
1119
|
# when system gets slow
|
1120
|
+
# Rotate through brokers on retries
|
1107
1121
|
#
|
1108
1122
|
# === Parameters
|
1109
1123
|
# request(Request):: Request to be sent
|
@@ -1111,13 +1125,14 @@ module RightScale
|
|
1111
1125
|
# count(Integer):: Number of retries so far
|
1112
1126
|
# multiplier(Integer):: Multiplier for retry interval for exponential backoff
|
1113
1127
|
# elapsed(Integer):: Elapsed time in seconds since this request was first attempted
|
1128
|
+
# broker_ids(Array):: Identity of brokers to be used in priority order
|
1114
1129
|
#
|
1115
1130
|
# === Return
|
1116
1131
|
# true:: Always return true
|
1117
|
-
def publish_with_timeout_retry(request, parent, count = 0, multiplier = 1, elapsed = 0)
|
1118
|
-
|
1132
|
+
def publish_with_timeout_retry(request, parent, count = 0, multiplier = 1, elapsed = 0, broker_ids = nil)
|
1133
|
+
published_broker_ids = publish(request, broker_ids)
|
1119
1134
|
|
1120
|
-
if @retry_interval && @retry_timeout && parent && !
|
1135
|
+
if @retry_interval && @retry_timeout && parent && !published_broker_ids.empty?
|
1121
1136
|
interval = [(@retry_interval * multiplier) + (@request_stats.avg_duration || 0), @retry_timeout - elapsed].min
|
1122
1137
|
EM.add_timer(interval) do
|
1123
1138
|
begin
|
@@ -1129,7 +1144,9 @@ module RightScale
|
|
1129
1144
|
request.token = AgentIdentity.generate
|
1130
1145
|
@pending_requests[parent].retry_parent = parent if count == 1
|
1131
1146
|
@pending_requests[request.token] = @pending_requests[parent]
|
1132
|
-
|
1147
|
+
broker_ids ||= @broker.all
|
1148
|
+
publish_with_timeout_retry(request, parent, count, multiplier * RETRY_BACKOFF_FACTOR, elapsed,
|
1149
|
+
broker_ids.push(broker_ids.shift))
|
1133
1150
|
@retry_stats.update(request.type.split('/').last)
|
1134
1151
|
else
|
1135
1152
|
Log.warning("RE-SEND TIMEOUT after #{elapsed.to_i} seconds for #{request.to_s([:tags, :target, :tries])}")
|
@@ -1137,7 +1154,7 @@ module RightScale
|
|
1137
1154
|
@non_delivery_stats.update(result.content)
|
1138
1155
|
handle_response(Result.new(request.token, request.reply_to, result, @identity))
|
1139
1156
|
end
|
1140
|
-
@connectivity_checker.check(
|
1157
|
+
@connectivity_checker.check(published_broker_ids.first) if count == 1
|
1141
1158
|
end
|
1142
1159
|
rescue Exception => e
|
1143
1160
|
Log.error("Failed retry for #{request.token}", e, :trace)
|
data/lib/right_agent.rb
CHANGED
@@ -23,11 +23,11 @@
|
|
23
23
|
# reuse minimal loader to ensure right_agent can safely be required after minimal.
|
24
24
|
require File.expand_path(File.join(File.dirname(__FILE__), 'right_agent', 'minimal'))
|
25
25
|
|
26
|
-
require 'amqp'
|
27
|
-
require 'mq'
|
28
26
|
require 'json'
|
29
27
|
require 'yaml'
|
30
28
|
require 'openssl'
|
29
|
+
require 'right_amqp'
|
30
|
+
require 'right_support'
|
31
31
|
|
32
32
|
require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'monkey_patches'))
|
33
33
|
require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'payload_formatter'))
|
@@ -43,9 +43,6 @@ require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'serialize'))
|
|
43
43
|
require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'security'))
|
44
44
|
require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'operation_result'))
|
45
45
|
require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'subprocess'))
|
46
|
-
require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'stats_helper'))
|
47
|
-
require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'broker_client'))
|
48
|
-
require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'ha_broker_client'))
|
49
46
|
require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'agent_identity'))
|
50
47
|
require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'agent_tags_manager'))
|
51
48
|
require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'actor'))
|
data/right_agent.gemspec
CHANGED
@@ -24,7 +24,8 @@ require 'rubygems'
|
|
24
24
|
|
25
25
|
Gem::Specification.new do |spec|
|
26
26
|
spec.name = 'right_agent'
|
27
|
-
spec.version = '0.
|
27
|
+
spec.version = '0.9.3'
|
28
|
+
spec.date = '2012-03-07'
|
28
29
|
spec.authors = ['Lee Kirchhoff', 'Raphael Simon', 'Tony Spataro']
|
29
30
|
spec.email = 'lee@rightscale.com'
|
30
31
|
spec.homepage = 'https://github.com/rightscale/right_agent'
|
@@ -36,12 +37,13 @@ Gem::Specification.new do |spec|
|
|
36
37
|
spec.required_ruby_version = '>= 1.8.7'
|
37
38
|
spec.require_path = 'lib'
|
38
39
|
|
39
|
-
spec.add_dependency('right_support', '~> 1.
|
40
|
-
spec.add_dependency('
|
40
|
+
spec.add_dependency('right_support', '~> 1.3')
|
41
|
+
spec.add_dependency('right_amqp', '~> 0.1')
|
41
42
|
spec.add_dependency('json', ['~> 1.4'])
|
42
43
|
spec.add_dependency('eventmachine', '~> 0.12.10')
|
43
44
|
spec.add_dependency('right_popen', '~> 1.0.11')
|
44
45
|
spec.add_dependency('msgpack', '0.4.4')
|
46
|
+
spec.add_dependency('net-ssh', '~> 2.0')
|
45
47
|
|
46
48
|
spec.description = <<-EOF
|
47
49
|
RightAgent provides a foundation for running an agent on a server to interface
|
data/spec/agent_config_spec.rb
CHANGED
@@ -108,7 +108,7 @@ describe RightScale::AgentConfig do
|
|
108
108
|
end
|
109
109
|
|
110
110
|
it 'should default configuration directory to platform specific directory' do
|
111
|
-
@agent_config.cfg_dir.should == RightScale::Platform.filesystem.
|
111
|
+
@agent_config.cfg_dir.should == RightScale::Platform.filesystem.right_agent_cfg_dir
|
112
112
|
end
|
113
113
|
|
114
114
|
it 'should set configuration directory' do
|
data/spec/agent_spec.rb
CHANGED
@@ -49,7 +49,7 @@ describe RightScale::Agent do
|
|
49
49
|
:all => ["b1"], :connected => ["b1"], :failed => [], :close_one => true,
|
50
50
|
:non_delivery => true).by_default
|
51
51
|
@broker.should_receive(:connection_status).and_yield(:connected)
|
52
|
-
flexmock(
|
52
|
+
flexmock(RightAMQP::HABrokerClient).should_receive(:new).and_return(@broker)
|
53
53
|
flexmock(RightScale::PidFile).should_receive(:new).
|
54
54
|
and_return(flexmock("pid file", :check=>true, :write=>true, :remove=>true))
|
55
55
|
@identity = "rs-instance-123-1"
|
@@ -138,7 +138,7 @@ describe RightScale::Agent do
|
|
138
138
|
:connected => ["b1"], :failed => [], :all => ["b0", "b1"],
|
139
139
|
:non_delivery => true).by_default
|
140
140
|
@broker.should_receive(:connection_status).and_yield(:connected)
|
141
|
-
flexmock(
|
141
|
+
flexmock(RightAMQP::HABrokerClient).should_receive(:new).and_return(@broker)
|
142
142
|
flexmock(RightScale::PidFile).should_receive(:new).
|
143
143
|
and_return(flexmock("pid file", :check=>true, :write=>true, :remove=>true))
|
144
144
|
@identity = "rs-instance-123-1"
|
@@ -266,9 +266,9 @@ describe RightScale::Agent do
|
|
266
266
|
:stats => "", :status => "status", :hosts => ["123"], :ports => [1, 2], :get => true,
|
267
267
|
:alias_ => "b1", :aliases => ["b1"]).by_default
|
268
268
|
@broker.should_receive(:connection_status).and_yield(:connected).by_default
|
269
|
-
@broker.should_receive(:identity_parts).with(@broker_id).and_return(["123", 1, 0, 0
|
270
|
-
@broker.should_receive(:identity_parts).with(@broker_id2).and_return(["123", 2, 1, 1
|
271
|
-
flexmock(
|
269
|
+
@broker.should_receive(:identity_parts).with(@broker_id).and_return(["123", 1, 0, 0])
|
270
|
+
@broker.should_receive(:identity_parts).with(@broker_id2).and_return(["123", 2, 1, 1])
|
271
|
+
flexmock(RightAMQP::HABrokerClient).should_receive(:new).and_return(@broker)
|
272
272
|
flexmock(RightScale::PidFile).should_receive(:new).
|
273
273
|
and_return(flexmock("pid file", :check=>true, :write=>true, :remove=>true))
|
274
274
|
@sender = flexmock("sender", :pending_requests => [], :request_age => nil,
|
@@ -312,7 +312,7 @@ describe RightScale::Agent do
|
|
312
312
|
@broker.should_receive(:subscribe).with(hsh(:name => @identity), nil, hsh(:brokers => nil), Proc).
|
313
313
|
and_return(@broker_ids.first(1)).once
|
314
314
|
@agent.run
|
315
|
-
@broker.should_receive(:connect).with("123", 2, 1, 1,
|
315
|
+
@broker.should_receive(:connect).with("123", 2, 1, 1, false, Proc).once
|
316
316
|
@agent.connect("123", 2, 1, 1)
|
317
317
|
end
|
318
318
|
end
|
@@ -322,7 +322,7 @@ describe RightScale::Agent do
|
|
322
322
|
@broker.should_receive(:subscribe).with(hsh(:name => @identity), nil, hsh(:brokers => nil), Proc).
|
323
323
|
and_return(@broker_ids.first(1)).once
|
324
324
|
@agent.run
|
325
|
-
@broker.should_receive(:connect).with("123", 2, 1, 1,
|
325
|
+
@broker.should_receive(:connect).with("123", 2, 1, 1, false, Proc).and_yield(@broker_ids.last).once
|
326
326
|
@broker.should_receive(:subscribe).with(hsh(:name => @identity), nil, hsh(:brokers => @broker_ids.last(1)), Proc).
|
327
327
|
and_return(@broker_ids.last(1)).once
|
328
328
|
flexmock(@agent).should_receive(:update_configuration).with(:host => ["123"], :port => [1, 2]).and_return(true).once
|
@@ -335,7 +335,7 @@ describe RightScale::Agent do
|
|
335
335
|
@broker.should_receive(:subscribe).with(hsh(:name => @identity), nil, hsh(:brokers => nil), Proc).
|
336
336
|
and_return(@broker_ids.first(1)).once
|
337
337
|
@agent.run
|
338
|
-
@broker.should_receive(:connect).with("123", 2, 1, 1,
|
338
|
+
@broker.should_receive(:connect).with("123", 2, 1, 1, false, Proc).and_yield(@broker_ids.last).once
|
339
339
|
@broker.should_receive(:connection_status).and_yield(:failed)
|
340
340
|
@log.should_receive(:error).with(/Failed to connect to broker/).once
|
341
341
|
flexmock(@agent).should_receive(:update_configuration).never
|
@@ -441,14 +441,15 @@ describe RightScale::Agent do
|
|
441
441
|
|
442
442
|
it "should tune heartbeat for all broker connections" do
|
443
443
|
run_in_em do
|
444
|
+
@log.should_receive(:info).with(/Reconnecting each broker to tune heartbeat to 45/).once
|
444
445
|
@log.should_receive(:info).with(/Tuned heartbeat to 45 seconds for broker/).twice
|
445
446
|
@agent = RightScale::Agent.new(:user => "me", :identity => @identity)
|
446
447
|
flexmock(@agent).should_receive(:load_actors).and_return(true)
|
447
448
|
flexmock(@agent).should_receive(:update_configuration).with(:heartbeat => 45).and_return(true).once
|
448
449
|
@agent.run
|
449
450
|
@broker.should_receive(:heartbeat=).with(45).once
|
450
|
-
@broker.should_receive(:connect).with("123", 1, 0, 0,
|
451
|
-
@broker.should_receive(:connect).with("123", 2, 1, 1,
|
451
|
+
@broker.should_receive(:connect).with("123", 1, 0, 0, true, Proc).and_yield(@broker_id).once
|
452
|
+
@broker.should_receive(:connect).with("123", 2, 1, 1, true, Proc).and_yield(@broker_id2).once
|
452
453
|
flexmock(@agent).should_receive(:setup_queues).with([@broker_id]).once
|
453
454
|
flexmock(@agent).should_receive(:setup_queues).with([@broker_id2]).once
|
454
455
|
@agent.tune_heartbeat(45).should be_nil
|
@@ -457,14 +458,15 @@ describe RightScale::Agent do
|
|
457
458
|
|
458
459
|
it "should tune heartbeat for all broker connections as deferred task" do
|
459
460
|
run_in_em do
|
461
|
+
@log.should_receive(:info).with(/Reconnecting each broker to tune heartbeat to 45/).once
|
460
462
|
@log.should_receive(:info).with(/Tuned heartbeat to 45 seconds for broker/).twice
|
461
463
|
@agent = RightScale::Agent.new(:user => "me", :identity => @identity)
|
462
464
|
flexmock(@agent).should_receive(:load_actors).and_return(true)
|
463
465
|
flexmock(@agent).should_receive(:update_configuration).with(:heartbeat => 45).and_return(true).once
|
464
466
|
@agent.run
|
465
467
|
@broker.should_receive(:heartbeat=).with(45).once
|
466
|
-
@broker.should_receive(:connect).with("123", 1, 0, 0,
|
467
|
-
@broker.should_receive(:connect).with("123", 2, 1, 1,
|
468
|
+
@broker.should_receive(:connect).with("123", 1, 0, 0, true, Proc).and_yield(@broker_id).once
|
469
|
+
@broker.should_receive(:connect).with("123", 2, 1, 1, true, Proc).and_yield(@broker_id2).once
|
468
470
|
flexmock(@agent).should_receive(:setup_queues).with([@broker_id]).once
|
469
471
|
flexmock(@agent).should_receive(:setup_queues).with([@broker_id2]).once
|
470
472
|
flexmock(@agent).should_receive(:finish_setup)
|
@@ -476,14 +478,15 @@ describe RightScale::Agent do
|
|
476
478
|
|
477
479
|
it "should disable heartbeat for all broker connections" do
|
478
480
|
run_in_em do
|
481
|
+
@log.should_receive(:info).with(/Reconnecting each broker to tune heartbeat to 0/).once
|
479
482
|
@log.should_receive(:info).with(/Disabled heartbeat for broker/).twice
|
480
483
|
@agent = RightScale::Agent.new(:user => "me", :identity => @identity)
|
481
484
|
flexmock(@agent).should_receive(:load_actors).and_return(true)
|
482
485
|
flexmock(@agent).should_receive(:update_configuration).with(:heartbeat => 0).and_return(true).once
|
483
486
|
@agent.run
|
484
487
|
@broker.should_receive(:heartbeat=).with(0).once
|
485
|
-
@broker.should_receive(:connect).with("123", 1, 0, 0,
|
486
|
-
@broker.should_receive(:connect).with("123", 2, 1, 1,
|
488
|
+
@broker.should_receive(:connect).with("123", 1, 0, 0, true, Proc).and_yield(@broker_id).once
|
489
|
+
@broker.should_receive(:connect).with("123", 2, 1, 1, true, Proc).and_yield(@broker_id2).once
|
487
490
|
flexmock(@agent).should_receive(:setup_queues).with([@broker_id]).once
|
488
491
|
flexmock(@agent).should_receive(:setup_queues).with([@broker_id2]).once
|
489
492
|
@agent.tune_heartbeat(0).should be_nil
|
@@ -492,6 +495,7 @@ describe RightScale::Agent do
|
|
492
495
|
|
493
496
|
it "should log error if any broker connect attempts fail" do
|
494
497
|
run_in_em do
|
498
|
+
@log.should_receive(:info).with(/Reconnecting each broker to tune heartbeat to 45/).once
|
495
499
|
@log.should_receive(:info).with(/Tuned heartbeat to 45 seconds for broker #{@broker_id2}/).once
|
496
500
|
@log.should_receive(:error).with("Failed to reconnect to broker #{@broker_id} to tune heartbeat", Exception, :trace).once
|
497
501
|
@agent = RightScale::Agent.new(:user => "me", :identity => @identity)
|
@@ -499,8 +503,8 @@ describe RightScale::Agent do
|
|
499
503
|
flexmock(@agent).should_receive(:update_configuration).with(:heartbeat => 45).and_return(true).once
|
500
504
|
@agent.run
|
501
505
|
@broker.should_receive(:heartbeat=).with(45).once
|
502
|
-
@broker.should_receive(:connect).with("123", 1, 0, 0,
|
503
|
-
@broker.should_receive(:connect).with("123", 2, 1, 1,
|
506
|
+
@broker.should_receive(:connect).with("123", 1, 0, 0, true, Proc).and_raise(Exception).once
|
507
|
+
@broker.should_receive(:connect).with("123", 2, 1, 1, true, Proc).and_yield(@broker_id2).once
|
504
508
|
flexmock(@agent).should_receive(:setup_queues).with([@broker_id]).never
|
505
509
|
flexmock(@agent).should_receive(:setup_queues).with([@broker_id2]).once
|
506
510
|
@agent.tune_heartbeat(45).should == "Failed to tune heartbeat for brokers [\"#{@broker_id}\"]"
|
@@ -509,6 +513,7 @@ describe RightScale::Agent do
|
|
509
513
|
|
510
514
|
it "should log error if any brokers do not connect" do
|
511
515
|
run_in_em do
|
516
|
+
@log.should_receive(:info).with(/Reconnecting each broker to tune heartbeat to 45/).once
|
512
517
|
@log.should_receive(:info).with(/Tuned heartbeat to 45 seconds for broker #{@broker_id2}/).once
|
513
518
|
@log.should_receive(:error).with(/Failed to reconnect to broker #{@broker_id} to tune heartbeat, status/).once
|
514
519
|
@agent = RightScale::Agent.new(:user => "me", :identity => @identity)
|
@@ -516,8 +521,8 @@ describe RightScale::Agent do
|
|
516
521
|
flexmock(@agent).should_receive(:update_configuration).with(:heartbeat => 45).and_return(true).once
|
517
522
|
@agent.run
|
518
523
|
@broker.should_receive(:heartbeat=).with(45).once
|
519
|
-
@broker.should_receive(:connect).with("123", 1, 0, 0,
|
520
|
-
@broker.should_receive(:connect).with("123", 2, 1, 1,
|
524
|
+
@broker.should_receive(:connect).with("123", 1, 0, 0, true, Proc).and_yield(@broker_id).once
|
525
|
+
@broker.should_receive(:connect).with("123", 2, 1, 1, true, Proc).and_yield(@broker_id2).once
|
521
526
|
@broker.should_receive(:connection_status).with({:one_off => 60, :brokers => [@broker_id]}, Proc).and_yield(:failed)
|
522
527
|
@broker.should_receive(:connection_status).with({:one_off => 60, :brokers => [@broker_id2]}, Proc).and_yield(:connected)
|
523
528
|
flexmock(@agent).should_receive(:setup_queues).with([@broker_id]).never
|
@@ -528,6 +533,7 @@ describe RightScale::Agent do
|
|
528
533
|
|
529
534
|
it "should log error if any broker queue setup fails" do
|
530
535
|
run_in_em do
|
536
|
+
@log.should_receive(:info).with(/Reconnecting each broker to tune heartbeat to 45/).once
|
531
537
|
@log.should_receive(:info).with(/Tuned heartbeat to 45 seconds for broker #{@broker_id2}/).once
|
532
538
|
@log.should_receive(:error).with(/Failed to setup queues for broker #{@broker_id} when tuning heartbeat/, Exception, :trace).once
|
533
539
|
@agent = RightScale::Agent.new(:user => "me", :identity => @identity)
|
@@ -535,8 +541,8 @@ describe RightScale::Agent do
|
|
535
541
|
flexmock(@agent).should_receive(:update_configuration).with(:heartbeat => 45).and_return(true).once
|
536
542
|
@agent.run
|
537
543
|
@broker.should_receive(:heartbeat=).with(45).once
|
538
|
-
@broker.should_receive(:connect).with("123", 1, 0, 0,
|
539
|
-
@broker.should_receive(:connect).with("123", 2, 1, 1,
|
544
|
+
@broker.should_receive(:connect).with("123", 1, 0, 0, true, Proc).and_yield(@broker_id).once
|
545
|
+
@broker.should_receive(:connect).with("123", 2, 1, 1, true, Proc).and_yield(@broker_id2).once
|
540
546
|
flexmock(@agent).should_receive(:setup_queues).with([@broker_id]).and_raise(Exception)
|
541
547
|
flexmock(@agent).should_receive(:setup_queues).with([@broker_id2]).once
|
542
548
|
@agent.tune_heartbeat(45).should be_nil
|
@@ -82,16 +82,20 @@ describe RightScale::LoginUser do
|
|
82
82
|
test_serialization_downgrade(user, public_key)
|
83
83
|
end
|
84
84
|
|
85
|
-
it 'should serialize current version with multiple public_keys' do
|
85
|
+
it 'should serialize current version with multiple public_keys and fingerprints' do
|
86
86
|
num = rand(2**32).to_s(32)
|
87
87
|
public_keys = []
|
88
|
-
|
88
|
+
fingerprints = []
|
89
|
+
3.times do |i|
|
89
90
|
pub = rand(2**32).to_s(32)
|
90
91
|
public_keys << "ssh-rsa #{pub} #{num}@rightscale.com"
|
92
|
+
fingerprints << "sha#{i}"
|
91
93
|
end
|
92
|
-
new_user = RightScale::LoginUser.new("v0-#{num}", "rs-#{num}", nil, "#{num}@rightscale.old", true, nil, public_keys
|
94
|
+
new_user = RightScale::LoginUser.new("v0-#{num}", "rs-#{num}", nil, "#{num}@rightscale.old", true, nil, public_keys,
|
95
|
+
nil, fingerprints)
|
93
96
|
new_user.public_key.should == public_keys.first
|
94
97
|
new_user.public_keys.should == public_keys
|
98
|
+
new_user.public_key_fingerprints.should == fingerprints
|
95
99
|
test_serialization_downgrade(new_user, public_keys.first)
|
96
100
|
end
|
97
101
|
|