right_agent 0.6.6 → 0.9.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. data/lib/right_agent/agent.rb +26 -25
  2. data/lib/right_agent/agent_config.rb +28 -2
  3. data/lib/right_agent/command/command_constants.rb +2 -2
  4. data/lib/right_agent/core_payload_types/executable_bundle.rb +3 -21
  5. data/lib/right_agent/core_payload_types/login_user.rb +19 -4
  6. data/lib/right_agent/core_payload_types/recipe_instantiation.rb +7 -1
  7. data/lib/right_agent/core_payload_types/right_script_instantiation.rb +7 -1
  8. data/lib/right_agent/dispatcher.rb +6 -19
  9. data/lib/right_agent/idempotent_request.rb +72 -17
  10. data/lib/right_agent/monkey_patches/ruby_patch.rb +0 -1
  11. data/lib/right_agent/monkey_patches.rb +0 -1
  12. data/lib/right_agent/operation_result.rb +27 -4
  13. data/lib/right_agent/packets.rb +47 -23
  14. data/lib/right_agent/platform/darwin.rb +33 -2
  15. data/lib/right_agent/platform/linux.rb +98 -2
  16. data/lib/right_agent/platform/windows.rb +41 -6
  17. data/lib/right_agent/platform.rb +11 -2
  18. data/lib/right_agent/scripts/agent_controller.rb +2 -1
  19. data/lib/right_agent/scripts/agent_deployer.rb +2 -2
  20. data/lib/right_agent/scripts/stats_manager.rb +7 -3
  21. data/lib/right_agent/sender.rb +45 -28
  22. data/lib/right_agent.rb +2 -5
  23. data/right_agent.gemspec +5 -3
  24. data/spec/agent_config_spec.rb +1 -1
  25. data/spec/agent_spec.rb +26 -20
  26. data/spec/core_payload_types/login_user_spec.rb +7 -3
  27. data/spec/idempotent_request_spec.rb +218 -48
  28. data/spec/operation_result_spec.rb +19 -0
  29. data/spec/packets_spec.rb +42 -1
  30. data/spec/platform/darwin.rb +11 -0
  31. data/spec/platform/linux.rb +23 -0
  32. data/spec/platform/linux_volume_manager_spec.rb +43 -43
  33. data/spec/platform/platform_spec.rb +35 -32
  34. data/spec/platform/windows.rb +11 -0
  35. data/spec/sender_spec.rb +21 -25
  36. metadata +47 -40
  37. data/lib/right_agent/broker_client.rb +0 -686
  38. data/lib/right_agent/ha_broker_client.rb +0 -1327
  39. data/lib/right_agent/monkey_patches/amqp_patch.rb +0 -274
  40. data/lib/right_agent/monkey_patches/ruby_patch/string_patch.rb +0 -107
  41. data/lib/right_agent/stats_helper.rb +0 -745
  42. data/spec/broker_client_spec.rb +0 -962
  43. data/spec/ha_broker_client_spec.rb +0 -1695
  44. data/spec/monkey_patches/amqp_patch_spec.rb +0 -100
  45. data/spec/monkey_patches/string_patch_spec.rb +0 -99
  46. data/spec/stats_helper_spec.rb +0 -686
@@ -28,8 +28,6 @@ module RightScale
28
28
  # All requests go through the mapper for security purposes
29
29
  class Sender
30
30
 
31
- include StatsHelper
32
-
33
31
  # Request that is waiting for a response
34
32
  class PendingRequest
35
33
 
@@ -172,7 +170,7 @@ module RightScale
172
170
  # === Parameters
173
171
  # restart_callback(Proc):: Callback that is activated on each restart vote with votes being initiated
174
172
  # by offline queue exceeding MAX_QUEUED_REQUESTS
175
- # offline_stats(ActivityStats):: Offline queue tracking statistics
173
+ # offline_stats(RightSupport::Stats::Activity):: Offline queue tracking statistics
176
174
  def initialize(restart_callback, offline_stats)
177
175
  @restart_vote = restart_callback
178
176
  @restart_vote_timer = nil
@@ -387,12 +385,16 @@ module RightScale
387
385
  # Number of seconds to wait for ping response from a mapper when checking connectivity
388
386
  PING_TIMEOUT = 30
389
387
 
388
+ # Default maximum number of consecutive ping timeouts before attempt to reconnect
389
+ MAX_PING_TIMEOUTS = 3
390
+
390
391
  # (EM::Timer) Timer while waiting for mapper ping response
391
392
  attr_accessor :ping_timer
392
393
 
393
394
  def initialize(sender, check_interval, ping_stats, exception_stats)
394
395
  @sender = sender
395
396
  @check_interval = check_interval
397
+ @ping_timeouts = {}
396
398
  @ping_timer = nil
397
399
  @ping_stats = ping_stats
398
400
  @exception_stats = exception_stats
@@ -427,25 +429,35 @@ module RightScale
427
429
  end
428
430
 
429
431
  # Check whether broker connection is usable by pinging a mapper via that broker
430
- # Attempt to reconnect if ping does not respond in PING_TIMEOUT seconds
432
+ # Attempt to reconnect if ping does not respond in PING_TIMEOUT seconds and
433
+ # if have reached timeout limit
431
434
  # Ignore request if already checking a connection
432
435
  # Only to be called from primary thread
433
436
  #
434
437
  # === Parameters
435
438
  # id(String):: Identity of specific broker to use to send ping, defaults to any
436
439
  # currently connected broker
440
+ # max_ping_timeouts(Integer):: Maximum number of ping timeouts before attempt
441
+ # to reconnect, defaults to MAX_PING_TIMEOUTS
437
442
  #
438
443
  # === Return
439
444
  # true:: Always return true
440
- def check(id = nil)
445
+ def check(id = nil, max_ping_timeouts = MAX_PING_TIMEOUTS)
441
446
  unless @terminating || @ping_timer || (id && !@sender.broker.connected?(id))
442
447
  @ping_timer = EM::Timer.new(PING_TIMEOUT) do
443
448
  begin
444
449
  @ping_stats.update("timeout")
445
450
  @ping_timer = nil
446
- Log.warning("Mapper ping via broker #{id} timed out after #{PING_TIMEOUT} seconds, attempting to reconnect")
447
- host, port, index, priority, _ = @sender.broker.identity_parts(id)
448
- @sender.agent.connect(host, port, index, priority, force = true)
451
+ @ping_timeouts[id] = (@ping_timeouts[id] || 0) + 1
452
+ if @ping_timeouts[id] >= max_ping_timeouts
453
+ Log.error("Mapper ping via broker #{id} timed out after #{PING_TIMEOUT} seconds and now " +
454
+ "reached maximum of #{max_ping_timeouts} timeout#{max_ping_timeouts > 1 ? 's' : ''}, " +
455
+ "attempting to reconnect")
456
+ host, port, index, priority = @sender.broker.identity_parts(id)
457
+ @sender.agent.connect(host, port, index, priority, force = true)
458
+ else
459
+ Log.warning("Mapper ping via broker #{id} timed out after #{PING_TIMEOUT} seconds")
460
+ end
449
461
  rescue Exception => e
450
462
  Log.error("Failed to reconnect to broker #{id}", e, :trace)
451
463
  @exception_stats.track("ping timeout", e)
@@ -458,6 +470,7 @@ module RightScale
458
470
  @ping_stats.update("success")
459
471
  @ping_timer.cancel
460
472
  @ping_timer = nil
473
+ @ping_timeouts[id] = 0
461
474
  end
462
475
  rescue Exception => e
463
476
  Log.error("Failed to cancel mapper ping", e, :trace)
@@ -500,7 +513,7 @@ module RightScale
500
513
  @inactivity_timer.cancel if @inactivity_timer
501
514
  @inactivity_timer = EM::Timer.new(@check_interval) do
502
515
  begin
503
- check
516
+ check(id = nil, max_ping_timeouts = 1)
504
517
  rescue Exception => e
505
518
  Log.error("Failed connectivity check", e, :trace)
506
519
  @exception_stats.track("check connectivity", e)
@@ -512,7 +525,7 @@ module RightScale
512
525
  end # ConnectivityChecker
513
526
 
514
527
  # Factor used on each retry iteration to achieve exponential backoff
515
- RETRY_BACKOFF_FACTOR = 4
528
+ RETRY_BACKOFF_FACTOR = 3
516
529
 
517
530
  # (PendingRequests) Requests waiting for a response
518
531
  attr_accessor :pending_requests
@@ -523,7 +536,7 @@ module RightScale
523
536
  # (ConnectivityChecker) Broker connection checker
524
537
  attr_reader :connectivity_checker
525
538
 
526
- # (HABrokerClient) High availability AMQP broker client
539
+ # (RightAMQP::HABrokerClient) High availability AMQP broker client
527
540
  attr_accessor :broker
528
541
 
529
542
  # (String) Identity of the associated agent
@@ -569,8 +582,8 @@ module RightScale
569
582
  @broker = @agent.broker
570
583
  @secure = @options[:secure]
571
584
  @single_threaded = @options[:single_threaded]
572
- @retry_timeout = nil_if_zero(@options[:retry_timeout])
573
- @retry_interval = nil_if_zero(@options[:retry_interval])
585
+ @retry_timeout = RightSupport::Stats.nil_if_zero(@options[:retry_timeout])
586
+ @retry_interval = RightSupport::Stats.nil_if_zero(@options[:retry_interval])
574
587
 
575
588
  # Only to be accessed from primary thread
576
589
  @pending_requests = PendingRequests.new
@@ -821,7 +834,7 @@ module RightScale
821
834
  exchange = {:type => :fanout, :name => "request", :options => {:durable => true, :no_declare => @secure}}
822
835
  ids = @broker.publish(exchange, request, :persistent => request.persistent, :mandatory => true,
823
836
  :log_filter => [:tags, :target, :tries, :persistent], :brokers => ids)
824
- rescue HABrokerClient::NoConnectedBrokers => e
837
+ rescue RightAMQP::HABrokerClient::NoConnectedBrokers => e
825
838
  Log.error("Failed to publish request #{request.to_s([:tags, :target, :tries])}", e)
826
839
  ids = []
827
840
  rescue Exception => e
@@ -923,15 +936,15 @@ module RightScale
923
936
  # === Return
924
937
  # true:: Always return true
925
938
  def reset_stats
926
- @ping_stats = ActivityStats.new
927
- @retry_stats = ActivityStats.new
928
- @request_stats = ActivityStats.new
929
- @result_stats = ActivityStats.new
930
- @result_error_stats = ActivityStats.new
931
- @non_delivery_stats = ActivityStats.new
932
- @offline_stats = ActivityStats.new(measure_rate = false)
933
- @request_kinds = ActivityStats.new(measure_rate = false)
934
- @exception_stats = ExceptionStats.new(@agent, @options[:exception_callback])
939
+ @ping_stats = RightSupport::Stats::Activity.new
940
+ @retry_stats = RightSupport::Stats::Activity.new
941
+ @request_stats = RightSupport::Stats::Activity.new
942
+ @result_stats = RightSupport::Stats::Activity.new
943
+ @result_error_stats = RightSupport::Stats::Activity.new
944
+ @non_delivery_stats = RightSupport::Stats::Activity.new
945
+ @offline_stats = RightSupport::Stats::Activity.new(measure_rate = false)
946
+ @request_kinds = RightSupport::Stats::Activity.new(measure_rate = false)
947
+ @exception_stats = RightSupport::Stats::Exceptions.new(@agent, @options[:exception_callback])
935
948
  true
936
949
  end
937
950
 
@@ -1104,6 +1117,7 @@ module RightScale
1104
1117
  # Use exponential backoff with RETRY_BACKOFF_FACTOR for retry spacing
1105
1118
  # Adjust retry interval by average response time to avoid adding to system load
1106
1119
  # when system gets slow
1120
+ # Rotate through brokers on retries
1107
1121
  #
1108
1122
  # === Parameters
1109
1123
  # request(Request):: Request to be sent
@@ -1111,13 +1125,14 @@ module RightScale
1111
1125
  # count(Integer):: Number of retries so far
1112
1126
  # multiplier(Integer):: Multiplier for retry interval for exponential backoff
1113
1127
  # elapsed(Integer):: Elapsed time in seconds since this request was first attempted
1128
+ # broker_ids(Array):: Identity of brokers to be used in priority order
1114
1129
  #
1115
1130
  # === Return
1116
1131
  # true:: Always return true
1117
- def publish_with_timeout_retry(request, parent, count = 0, multiplier = 1, elapsed = 0)
1118
- ids = publish(request)
1132
+ def publish_with_timeout_retry(request, parent, count = 0, multiplier = 1, elapsed = 0, broker_ids = nil)
1133
+ published_broker_ids = publish(request, broker_ids)
1119
1134
 
1120
- if @retry_interval && @retry_timeout && parent && !ids.empty?
1135
+ if @retry_interval && @retry_timeout && parent && !published_broker_ids.empty?
1121
1136
  interval = [(@retry_interval * multiplier) + (@request_stats.avg_duration || 0), @retry_timeout - elapsed].min
1122
1137
  EM.add_timer(interval) do
1123
1138
  begin
@@ -1129,7 +1144,9 @@ module RightScale
1129
1144
  request.token = AgentIdentity.generate
1130
1145
  @pending_requests[parent].retry_parent = parent if count == 1
1131
1146
  @pending_requests[request.token] = @pending_requests[parent]
1132
- publish_with_timeout_retry(request, parent, count, multiplier * RETRY_BACKOFF_FACTOR, elapsed)
1147
+ broker_ids ||= @broker.all
1148
+ publish_with_timeout_retry(request, parent, count, multiplier * RETRY_BACKOFF_FACTOR, elapsed,
1149
+ broker_ids.push(broker_ids.shift))
1133
1150
  @retry_stats.update(request.type.split('/').last)
1134
1151
  else
1135
1152
  Log.warning("RE-SEND TIMEOUT after #{elapsed.to_i} seconds for #{request.to_s([:tags, :target, :tries])}")
@@ -1137,7 +1154,7 @@ module RightScale
1137
1154
  @non_delivery_stats.update(result.content)
1138
1155
  handle_response(Result.new(request.token, request.reply_to, result, @identity))
1139
1156
  end
1140
- @connectivity_checker.check(ids.first) if count == 1
1157
+ @connectivity_checker.check(published_broker_ids.first) if count == 1
1141
1158
  end
1142
1159
  rescue Exception => e
1143
1160
  Log.error("Failed retry for #{request.token}", e, :trace)
data/lib/right_agent.rb CHANGED
@@ -23,11 +23,11 @@
23
23
  # reuse minimal loader to ensure right_agent can safely be required after minimal.
24
24
  require File.expand_path(File.join(File.dirname(__FILE__), 'right_agent', 'minimal'))
25
25
 
26
- require 'amqp'
27
- require 'mq'
28
26
  require 'json'
29
27
  require 'yaml'
30
28
  require 'openssl'
29
+ require 'right_amqp'
30
+ require 'right_support'
31
31
 
32
32
  require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'monkey_patches'))
33
33
  require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'payload_formatter'))
@@ -43,9 +43,6 @@ require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'serialize'))
43
43
  require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'security'))
44
44
  require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'operation_result'))
45
45
  require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'subprocess'))
46
- require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'stats_helper'))
47
- require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'broker_client'))
48
- require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'ha_broker_client'))
49
46
  require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'agent_identity'))
50
47
  require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'agent_tags_manager'))
51
48
  require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'actor'))
data/right_agent.gemspec CHANGED
@@ -24,7 +24,8 @@ require 'rubygems'
24
24
 
25
25
  Gem::Specification.new do |spec|
26
26
  spec.name = 'right_agent'
27
- spec.version = '0.6.6'
27
+ spec.version = '0.9.3'
28
+ spec.date = '2012-03-07'
28
29
  spec.authors = ['Lee Kirchhoff', 'Raphael Simon', 'Tony Spataro']
29
30
  spec.email = 'lee@rightscale.com'
30
31
  spec.homepage = 'https://github.com/rightscale/right_agent'
@@ -36,12 +37,13 @@ Gem::Specification.new do |spec|
36
37
  spec.required_ruby_version = '>= 1.8.7'
37
38
  spec.require_path = 'lib'
38
39
 
39
- spec.add_dependency('right_support', '~> 1.0')
40
- spec.add_dependency('amqp', '0.6.7')
40
+ spec.add_dependency('right_support', '~> 1.3')
41
+ spec.add_dependency('right_amqp', '~> 0.1')
41
42
  spec.add_dependency('json', ['~> 1.4'])
42
43
  spec.add_dependency('eventmachine', '~> 0.12.10')
43
44
  spec.add_dependency('right_popen', '~> 1.0.11')
44
45
  spec.add_dependency('msgpack', '0.4.4')
46
+ spec.add_dependency('net-ssh', '~> 2.0')
45
47
 
46
48
  spec.description = <<-EOF
47
49
  RightAgent provides a foundation for running an agent on a server to interface
@@ -108,7 +108,7 @@ describe RightScale::AgentConfig do
108
108
  end
109
109
 
110
110
  it 'should default configuration directory to platform specific directory' do
111
- @agent_config.cfg_dir.should == RightScale::Platform.filesystem.cfg_dir
111
+ @agent_config.cfg_dir.should == RightScale::Platform.filesystem.right_agent_cfg_dir
112
112
  end
113
113
 
114
114
  it 'should set configuration directory' do
data/spec/agent_spec.rb CHANGED
@@ -49,7 +49,7 @@ describe RightScale::Agent do
49
49
  :all => ["b1"], :connected => ["b1"], :failed => [], :close_one => true,
50
50
  :non_delivery => true).by_default
51
51
  @broker.should_receive(:connection_status).and_yield(:connected)
52
- flexmock(RightScale::HABrokerClient).should_receive(:new).and_return(@broker)
52
+ flexmock(RightAMQP::HABrokerClient).should_receive(:new).and_return(@broker)
53
53
  flexmock(RightScale::PidFile).should_receive(:new).
54
54
  and_return(flexmock("pid file", :check=>true, :write=>true, :remove=>true))
55
55
  @identity = "rs-instance-123-1"
@@ -138,7 +138,7 @@ describe RightScale::Agent do
138
138
  :connected => ["b1"], :failed => [], :all => ["b0", "b1"],
139
139
  :non_delivery => true).by_default
140
140
  @broker.should_receive(:connection_status).and_yield(:connected)
141
- flexmock(RightScale::HABrokerClient).should_receive(:new).and_return(@broker)
141
+ flexmock(RightAMQP::HABrokerClient).should_receive(:new).and_return(@broker)
142
142
  flexmock(RightScale::PidFile).should_receive(:new).
143
143
  and_return(flexmock("pid file", :check=>true, :write=>true, :remove=>true))
144
144
  @identity = "rs-instance-123-1"
@@ -266,9 +266,9 @@ describe RightScale::Agent do
266
266
  :stats => "", :status => "status", :hosts => ["123"], :ports => [1, 2], :get => true,
267
267
  :alias_ => "b1", :aliases => ["b1"]).by_default
268
268
  @broker.should_receive(:connection_status).and_yield(:connected).by_default
269
- @broker.should_receive(:identity_parts).with(@broker_id).and_return(["123", 1, 0, 0, nil])
270
- @broker.should_receive(:identity_parts).with(@broker_id2).and_return(["123", 2, 1, 1, nil])
271
- flexmock(RightScale::HABrokerClient).should_receive(:new).and_return(@broker)
269
+ @broker.should_receive(:identity_parts).with(@broker_id).and_return(["123", 1, 0, 0])
270
+ @broker.should_receive(:identity_parts).with(@broker_id2).and_return(["123", 2, 1, 1])
271
+ flexmock(RightAMQP::HABrokerClient).should_receive(:new).and_return(@broker)
272
272
  flexmock(RightScale::PidFile).should_receive(:new).
273
273
  and_return(flexmock("pid file", :check=>true, :write=>true, :remove=>true))
274
274
  @sender = flexmock("sender", :pending_requests => [], :request_age => nil,
@@ -312,7 +312,7 @@ describe RightScale::Agent do
312
312
  @broker.should_receive(:subscribe).with(hsh(:name => @identity), nil, hsh(:brokers => nil), Proc).
313
313
  and_return(@broker_ids.first(1)).once
314
314
  @agent.run
315
- @broker.should_receive(:connect).with("123", 2, 1, 1, nil, false, Proc).once
315
+ @broker.should_receive(:connect).with("123", 2, 1, 1, false, Proc).once
316
316
  @agent.connect("123", 2, 1, 1)
317
317
  end
318
318
  end
@@ -322,7 +322,7 @@ describe RightScale::Agent do
322
322
  @broker.should_receive(:subscribe).with(hsh(:name => @identity), nil, hsh(:brokers => nil), Proc).
323
323
  and_return(@broker_ids.first(1)).once
324
324
  @agent.run
325
- @broker.should_receive(:connect).with("123", 2, 1, 1, nil, false, Proc).and_yield(@broker_ids.last).once
325
+ @broker.should_receive(:connect).with("123", 2, 1, 1, false, Proc).and_yield(@broker_ids.last).once
326
326
  @broker.should_receive(:subscribe).with(hsh(:name => @identity), nil, hsh(:brokers => @broker_ids.last(1)), Proc).
327
327
  and_return(@broker_ids.last(1)).once
328
328
  flexmock(@agent).should_receive(:update_configuration).with(:host => ["123"], :port => [1, 2]).and_return(true).once
@@ -335,7 +335,7 @@ describe RightScale::Agent do
335
335
  @broker.should_receive(:subscribe).with(hsh(:name => @identity), nil, hsh(:brokers => nil), Proc).
336
336
  and_return(@broker_ids.first(1)).once
337
337
  @agent.run
338
- @broker.should_receive(:connect).with("123", 2, 1, 1, nil, false, Proc).and_yield(@broker_ids.last).once
338
+ @broker.should_receive(:connect).with("123", 2, 1, 1, false, Proc).and_yield(@broker_ids.last).once
339
339
  @broker.should_receive(:connection_status).and_yield(:failed)
340
340
  @log.should_receive(:error).with(/Failed to connect to broker/).once
341
341
  flexmock(@agent).should_receive(:update_configuration).never
@@ -441,14 +441,15 @@ describe RightScale::Agent do
441
441
 
442
442
  it "should tune heartbeat for all broker connections" do
443
443
  run_in_em do
444
+ @log.should_receive(:info).with(/Reconnecting each broker to tune heartbeat to 45/).once
444
445
  @log.should_receive(:info).with(/Tuned heartbeat to 45 seconds for broker/).twice
445
446
  @agent = RightScale::Agent.new(:user => "me", :identity => @identity)
446
447
  flexmock(@agent).should_receive(:load_actors).and_return(true)
447
448
  flexmock(@agent).should_receive(:update_configuration).with(:heartbeat => 45).and_return(true).once
448
449
  @agent.run
449
450
  @broker.should_receive(:heartbeat=).with(45).once
450
- @broker.should_receive(:connect).with("123", 1, 0, 0, nil, true, Proc).and_yield(@broker_id).once
451
- @broker.should_receive(:connect).with("123", 2, 1, 1, nil, true, Proc).and_yield(@broker_id2).once
451
+ @broker.should_receive(:connect).with("123", 1, 0, 0, true, Proc).and_yield(@broker_id).once
452
+ @broker.should_receive(:connect).with("123", 2, 1, 1, true, Proc).and_yield(@broker_id2).once
452
453
  flexmock(@agent).should_receive(:setup_queues).with([@broker_id]).once
453
454
  flexmock(@agent).should_receive(:setup_queues).with([@broker_id2]).once
454
455
  @agent.tune_heartbeat(45).should be_nil
@@ -457,14 +458,15 @@ describe RightScale::Agent do
457
458
 
458
459
  it "should tune heartbeat for all broker connections as deferred task" do
459
460
  run_in_em do
461
+ @log.should_receive(:info).with(/Reconnecting each broker to tune heartbeat to 45/).once
460
462
  @log.should_receive(:info).with(/Tuned heartbeat to 45 seconds for broker/).twice
461
463
  @agent = RightScale::Agent.new(:user => "me", :identity => @identity)
462
464
  flexmock(@agent).should_receive(:load_actors).and_return(true)
463
465
  flexmock(@agent).should_receive(:update_configuration).with(:heartbeat => 45).and_return(true).once
464
466
  @agent.run
465
467
  @broker.should_receive(:heartbeat=).with(45).once
466
- @broker.should_receive(:connect).with("123", 1, 0, 0, nil, true, Proc).and_yield(@broker_id).once
467
- @broker.should_receive(:connect).with("123", 2, 1, 1, nil, true, Proc).and_yield(@broker_id2).once
468
+ @broker.should_receive(:connect).with("123", 1, 0, 0, true, Proc).and_yield(@broker_id).once
469
+ @broker.should_receive(:connect).with("123", 2, 1, 1, true, Proc).and_yield(@broker_id2).once
468
470
  flexmock(@agent).should_receive(:setup_queues).with([@broker_id]).once
469
471
  flexmock(@agent).should_receive(:setup_queues).with([@broker_id2]).once
470
472
  flexmock(@agent).should_receive(:finish_setup)
@@ -476,14 +478,15 @@ describe RightScale::Agent do
476
478
 
477
479
  it "should disable heartbeat for all broker connections" do
478
480
  run_in_em do
481
+ @log.should_receive(:info).with(/Reconnecting each broker to tune heartbeat to 0/).once
479
482
  @log.should_receive(:info).with(/Disabled heartbeat for broker/).twice
480
483
  @agent = RightScale::Agent.new(:user => "me", :identity => @identity)
481
484
  flexmock(@agent).should_receive(:load_actors).and_return(true)
482
485
  flexmock(@agent).should_receive(:update_configuration).with(:heartbeat => 0).and_return(true).once
483
486
  @agent.run
484
487
  @broker.should_receive(:heartbeat=).with(0).once
485
- @broker.should_receive(:connect).with("123", 1, 0, 0, nil, true, Proc).and_yield(@broker_id).once
486
- @broker.should_receive(:connect).with("123", 2, 1, 1, nil, true, Proc).and_yield(@broker_id2).once
488
+ @broker.should_receive(:connect).with("123", 1, 0, 0, true, Proc).and_yield(@broker_id).once
489
+ @broker.should_receive(:connect).with("123", 2, 1, 1, true, Proc).and_yield(@broker_id2).once
487
490
  flexmock(@agent).should_receive(:setup_queues).with([@broker_id]).once
488
491
  flexmock(@agent).should_receive(:setup_queues).with([@broker_id2]).once
489
492
  @agent.tune_heartbeat(0).should be_nil
@@ -492,6 +495,7 @@ describe RightScale::Agent do
492
495
 
493
496
  it "should log error if any broker connect attempts fail" do
494
497
  run_in_em do
498
+ @log.should_receive(:info).with(/Reconnecting each broker to tune heartbeat to 45/).once
495
499
  @log.should_receive(:info).with(/Tuned heartbeat to 45 seconds for broker #{@broker_id2}/).once
496
500
  @log.should_receive(:error).with("Failed to reconnect to broker #{@broker_id} to tune heartbeat", Exception, :trace).once
497
501
  @agent = RightScale::Agent.new(:user => "me", :identity => @identity)
@@ -499,8 +503,8 @@ describe RightScale::Agent do
499
503
  flexmock(@agent).should_receive(:update_configuration).with(:heartbeat => 45).and_return(true).once
500
504
  @agent.run
501
505
  @broker.should_receive(:heartbeat=).with(45).once
502
- @broker.should_receive(:connect).with("123", 1, 0, 0, nil, true, Proc).and_raise(Exception).once
503
- @broker.should_receive(:connect).with("123", 2, 1, 1, nil, true, Proc).and_yield(@broker_id2).once
506
+ @broker.should_receive(:connect).with("123", 1, 0, 0, true, Proc).and_raise(Exception).once
507
+ @broker.should_receive(:connect).with("123", 2, 1, 1, true, Proc).and_yield(@broker_id2).once
504
508
  flexmock(@agent).should_receive(:setup_queues).with([@broker_id]).never
505
509
  flexmock(@agent).should_receive(:setup_queues).with([@broker_id2]).once
506
510
  @agent.tune_heartbeat(45).should == "Failed to tune heartbeat for brokers [\"#{@broker_id}\"]"
@@ -509,6 +513,7 @@ describe RightScale::Agent do
509
513
 
510
514
  it "should log error if any brokers do not connect" do
511
515
  run_in_em do
516
+ @log.should_receive(:info).with(/Reconnecting each broker to tune heartbeat to 45/).once
512
517
  @log.should_receive(:info).with(/Tuned heartbeat to 45 seconds for broker #{@broker_id2}/).once
513
518
  @log.should_receive(:error).with(/Failed to reconnect to broker #{@broker_id} to tune heartbeat, status/).once
514
519
  @agent = RightScale::Agent.new(:user => "me", :identity => @identity)
@@ -516,8 +521,8 @@ describe RightScale::Agent do
516
521
  flexmock(@agent).should_receive(:update_configuration).with(:heartbeat => 45).and_return(true).once
517
522
  @agent.run
518
523
  @broker.should_receive(:heartbeat=).with(45).once
519
- @broker.should_receive(:connect).with("123", 1, 0, 0, nil, true, Proc).and_yield(@broker_id).once
520
- @broker.should_receive(:connect).with("123", 2, 1, 1, nil, true, Proc).and_yield(@broker_id2).once
524
+ @broker.should_receive(:connect).with("123", 1, 0, 0, true, Proc).and_yield(@broker_id).once
525
+ @broker.should_receive(:connect).with("123", 2, 1, 1, true, Proc).and_yield(@broker_id2).once
521
526
  @broker.should_receive(:connection_status).with({:one_off => 60, :brokers => [@broker_id]}, Proc).and_yield(:failed)
522
527
  @broker.should_receive(:connection_status).with({:one_off => 60, :brokers => [@broker_id2]}, Proc).and_yield(:connected)
523
528
  flexmock(@agent).should_receive(:setup_queues).with([@broker_id]).never
@@ -528,6 +533,7 @@ describe RightScale::Agent do
528
533
 
529
534
  it "should log error if any broker queue setup fails" do
530
535
  run_in_em do
536
+ @log.should_receive(:info).with(/Reconnecting each broker to tune heartbeat to 45/).once
531
537
  @log.should_receive(:info).with(/Tuned heartbeat to 45 seconds for broker #{@broker_id2}/).once
532
538
  @log.should_receive(:error).with(/Failed to setup queues for broker #{@broker_id} when tuning heartbeat/, Exception, :trace).once
533
539
  @agent = RightScale::Agent.new(:user => "me", :identity => @identity)
@@ -535,8 +541,8 @@ describe RightScale::Agent do
535
541
  flexmock(@agent).should_receive(:update_configuration).with(:heartbeat => 45).and_return(true).once
536
542
  @agent.run
537
543
  @broker.should_receive(:heartbeat=).with(45).once
538
- @broker.should_receive(:connect).with("123", 1, 0, 0, nil, true, Proc).and_yield(@broker_id).once
539
- @broker.should_receive(:connect).with("123", 2, 1, 1, nil, true, Proc).and_yield(@broker_id2).once
544
+ @broker.should_receive(:connect).with("123", 1, 0, 0, true, Proc).and_yield(@broker_id).once
545
+ @broker.should_receive(:connect).with("123", 2, 1, 1, true, Proc).and_yield(@broker_id2).once
540
546
  flexmock(@agent).should_receive(:setup_queues).with([@broker_id]).and_raise(Exception)
541
547
  flexmock(@agent).should_receive(:setup_queues).with([@broker_id2]).once
542
548
  @agent.tune_heartbeat(45).should be_nil
@@ -82,16 +82,20 @@ describe RightScale::LoginUser do
82
82
  test_serialization_downgrade(user, public_key)
83
83
  end
84
84
 
85
- it 'should serialize current version with multiple public_keys' do
85
+ it 'should serialize current version with multiple public_keys and fingerprints' do
86
86
  num = rand(2**32).to_s(32)
87
87
  public_keys = []
88
- 3.times do
88
+ fingerprints = []
89
+ 3.times do |i|
89
90
  pub = rand(2**32).to_s(32)
90
91
  public_keys << "ssh-rsa #{pub} #{num}@rightscale.com"
92
+ fingerprints << "sha#{i}"
91
93
  end
92
- new_user = RightScale::LoginUser.new("v0-#{num}", "rs-#{num}", nil, "#{num}@rightscale.old", true, nil, public_keys)
94
+ new_user = RightScale::LoginUser.new("v0-#{num}", "rs-#{num}", nil, "#{num}@rightscale.old", true, nil, public_keys,
95
+ nil, fingerprints)
93
96
  new_user.public_key.should == public_keys.first
94
97
  new_user.public_keys.should == public_keys
98
+ new_user.public_key_fingerprints.should == fingerprints
95
99
  test_serialization_downgrade(new_user, public_keys.first)
96
100
  end
97
101