right_agent 0.6.6 → 0.9.3

data/lib/right_agent/sender.rb

@@ -28,8 +28,6 @@ module RightScale
   # All requests go through the mapper for security purposes
   class Sender
 
-    include StatsHelper
-
     # Request that is waiting for a response
     class PendingRequest
 
@@ -172,7 +170,7 @@ module RightScale
       # === Parameters
       # restart_callback(Proc):: Callback that is activated on each restart vote with votes being initiated
       #   by offline queue exceeding MAX_QUEUED_REQUESTS
-      # offline_stats(ActivityStats):: Offline queue tracking statistics
+      # offline_stats(RightSupport::Stats::Activity):: Offline queue tracking statistics
       def initialize(restart_callback, offline_stats)
         @restart_vote = restart_callback
         @restart_vote_timer = nil
@@ -387,12 +385,16 @@ module RightScale
       # Number of seconds to wait for ping response from a mapper when checking connectivity
       PING_TIMEOUT = 30
 
+      # Default maximum number of consecutive ping timeouts before attempt to reconnect
+      MAX_PING_TIMEOUTS = 3
+
       # (EM::Timer) Timer while waiting for mapper ping response
       attr_accessor :ping_timer
 
       def initialize(sender, check_interval, ping_stats, exception_stats)
         @sender = sender
         @check_interval = check_interval
+        @ping_timeouts = {}
         @ping_timer = nil
         @ping_stats = ping_stats
         @exception_stats = exception_stats
@@ -427,25 +429,35 @@ module RightScale
       end
 
       # Check whether broker connection is usable by pinging a mapper via that broker
-      # Attempt to reconnect if ping does not respond in PING_TIMEOUT seconds
+      # Attempt to reconnect if ping does not respond in PING_TIMEOUT seconds and
+      # if have reached timeout limit
       # Ignore request if already checking a connection
       # Only to be called from primary thread
       #
       # === Parameters
       # id(String):: Identity of specific broker to use to send ping, defaults to any
       #   currently connected broker
+      # max_ping_timeouts(Integer):: Maximum number of ping timeouts before attempt
+      #   to reconnect, defaults to MAX_PING_TIMEOUTS
       #
       # === Return
       # true:: Always return true
-      def check(id = nil)
+      def check(id = nil, max_ping_timeouts = MAX_PING_TIMEOUTS)
         unless @terminating || @ping_timer || (id && !@sender.broker.connected?(id))
           @ping_timer = EM::Timer.new(PING_TIMEOUT) do
             begin
               @ping_stats.update("timeout")
               @ping_timer = nil
-              Log.warning("Mapper ping via broker #{id} timed out after #{PING_TIMEOUT} seconds, attempting to reconnect")
-              host, port, index, priority, _ = @sender.broker.identity_parts(id)
-              @sender.agent.connect(host, port, index, priority, force = true)
+              @ping_timeouts[id] = (@ping_timeouts[id] || 0) + 1
+              if @ping_timeouts[id] >= max_ping_timeouts
+                Log.error("Mapper ping via broker #{id} timed out after #{PING_TIMEOUT} seconds and now " +
+                          "reached maximum of #{max_ping_timeouts} timeout#{max_ping_timeouts > 1 ? 's' : ''}, " +
+                          "attempting to reconnect")
+                host, port, index, priority = @sender.broker.identity_parts(id)
+                @sender.agent.connect(host, port, index, priority, force = true)
+              else
+                Log.warning("Mapper ping via broker #{id} timed out after #{PING_TIMEOUT} seconds")
+              end
             rescue Exception => e
               Log.error("Failed to reconnect to broker #{id}", e, :trace)
               @exception_stats.track("ping timeout", e)
@@ -458,6 +470,7 @@ module RightScale
                 @ping_stats.update("success")
                 @ping_timer.cancel
                 @ping_timer = nil
+                @ping_timeouts[id] = 0
               end
             rescue Exception => e
               Log.error("Failed to cancel mapper ping", e, :trace)
@@ -500,7 +513,7 @@ module RightScale
         @inactivity_timer.cancel if @inactivity_timer
         @inactivity_timer = EM::Timer.new(@check_interval) do
           begin
-            check
+            check(id = nil, max_ping_timeouts = 1)
           rescue Exception => e
             Log.error("Failed connectivity check", e, :trace)
             @exception_stats.track("check connectivity", e)
@@ -512,7 +525,7 @@ module RightScale
     end # ConnectivityChecker
 
     # Factor used on each retry iteration to achieve exponential backoff
-    RETRY_BACKOFF_FACTOR = 4
+    RETRY_BACKOFF_FACTOR = 3
 
     # (PendingRequests) Requests waiting for a response
     attr_accessor :pending_requests
@@ -523,7 +536,7 @@ module RightScale
     # (ConnectivityChecker) Broker connection checker
     attr_reader :connectivity_checker
 
-    # (HABrokerClient) High availability AMQP broker client
+    # (RightAMQP::HABrokerClient) High availability AMQP broker client
     attr_accessor :broker
 
     # (String) Identity of the associated agent
@@ -569,8 +582,8 @@ module RightScale
       @broker = @agent.broker
       @secure = @options[:secure]
       @single_threaded = @options[:single_threaded]
-      @retry_timeout = nil_if_zero(@options[:retry_timeout])
-      @retry_interval = nil_if_zero(@options[:retry_interval])
+      @retry_timeout = RightSupport::Stats.nil_if_zero(@options[:retry_timeout])
+      @retry_interval = RightSupport::Stats.nil_if_zero(@options[:retry_interval])
 
       # Only to be accessed from primary thread
       @pending_requests = PendingRequests.new
@@ -821,7 +834,7 @@ module RightScale
         exchange = {:type => :fanout, :name => "request", :options => {:durable => true, :no_declare => @secure}}
         ids = @broker.publish(exchange, request, :persistent => request.persistent, :mandatory => true,
                               :log_filter => [:tags, :target, :tries, :persistent], :brokers => ids)
-      rescue HABrokerClient::NoConnectedBrokers => e
+      rescue RightAMQP::HABrokerClient::NoConnectedBrokers => e
         Log.error("Failed to publish request #{request.to_s([:tags, :target, :tries])}", e)
         ids = []
       rescue Exception => e
@@ -923,15 +936,15 @@ module RightScale
     # === Return
     # true:: Always return true
     def reset_stats
-      @ping_stats = ActivityStats.new
-      @retry_stats = ActivityStats.new
-      @request_stats = ActivityStats.new
-      @result_stats = ActivityStats.new
-      @result_error_stats = ActivityStats.new
-      @non_delivery_stats = ActivityStats.new
-      @offline_stats = ActivityStats.new(measure_rate = false)
-      @request_kinds = ActivityStats.new(measure_rate = false)
-      @exception_stats = ExceptionStats.new(@agent, @options[:exception_callback])
+      @ping_stats = RightSupport::Stats::Activity.new
+      @retry_stats = RightSupport::Stats::Activity.new
+      @request_stats = RightSupport::Stats::Activity.new
+      @result_stats = RightSupport::Stats::Activity.new
+      @result_error_stats = RightSupport::Stats::Activity.new
+      @non_delivery_stats = RightSupport::Stats::Activity.new
+      @offline_stats = RightSupport::Stats::Activity.new(measure_rate = false)
+      @request_kinds = RightSupport::Stats::Activity.new(measure_rate = false)
+      @exception_stats = RightSupport::Stats::Exceptions.new(@agent, @options[:exception_callback])
       true
     end
 
@@ -1104,6 +1117,7 @@ module RightScale
     # Use exponential backoff with RETRY_BACKOFF_FACTOR for retry spacing
     # Adjust retry interval by average response time to avoid adding to system load
     # when system gets slow
+    # Rotate through brokers on retries
     #
     # === Parameters
     # request(Request):: Request to be sent
@@ -1111,13 +1125,14 @@ module RightScale
     # count(Integer):: Number of retries so far
     # multiplier(Integer):: Multiplier for retry interval for exponential backoff
     # elapsed(Integer):: Elapsed time in seconds since this request was first attempted
+    # broker_ids(Array):: Identity of brokers to be used in priority order
     #
     # === Return
     # true:: Always return true
-    def publish_with_timeout_retry(request, parent, count = 0, multiplier = 1, elapsed = 0)
-      ids = publish(request)
+    def publish_with_timeout_retry(request, parent, count = 0, multiplier = 1, elapsed = 0, broker_ids = nil)
+      published_broker_ids = publish(request, broker_ids)
 
-      if @retry_interval && @retry_timeout && parent && !ids.empty?
+      if @retry_interval && @retry_timeout && parent && !published_broker_ids.empty?
         interval = [(@retry_interval * multiplier) + (@request_stats.avg_duration || 0), @retry_timeout - elapsed].min
         EM.add_timer(interval) do
           begin
@@ -1129,7 +1144,9 @@ module RightScale
                 request.token = AgentIdentity.generate
                 @pending_requests[parent].retry_parent = parent if count == 1
                 @pending_requests[request.token] = @pending_requests[parent]
-                publish_with_timeout_retry(request, parent, count, multiplier * RETRY_BACKOFF_FACTOR, elapsed)
+                broker_ids ||= @broker.all
+                publish_with_timeout_retry(request, parent, count, multiplier * RETRY_BACKOFF_FACTOR, elapsed,
+                                           broker_ids.push(broker_ids.shift))
                 @retry_stats.update(request.type.split('/').last)
               else
                 Log.warning("RE-SEND TIMEOUT after #{elapsed.to_i} seconds for #{request.to_s([:tags, :target, :tries])}")
@@ -1137,7 +1154,7 @@ module RightScale
                 @non_delivery_stats.update(result.content)
                 handle_response(Result.new(request.token, request.reply_to, result, @identity))
               end
-              @connectivity_checker.check(ids.first) if count == 1
+              @connectivity_checker.check(published_broker_ids.first) if count == 1
             end
           rescue Exception => e
             Log.error("Failed retry for #{request.token}", e, :trace)

data/lib/right_agent.rb

@@ -23,11 +23,11 @@
 # reuse minimal loader to ensure right_agent can safely be required after minimal.
 require File.expand_path(File.join(File.dirname(__FILE__), 'right_agent', 'minimal'))
 
-require 'amqp'
-require 'mq'
 require 'json'
 require 'yaml'
 require 'openssl'
+require 'right_amqp'
+require 'right_support'
 
 require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'monkey_patches'))
 require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'payload_formatter'))
@@ -43,9 +43,6 @@ require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'serialize'))
 require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'security'))
 require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'operation_result'))
 require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'subprocess'))
-require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'stats_helper'))
-require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'broker_client'))
-require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'ha_broker_client'))
 require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'agent_identity'))
 require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'agent_tags_manager'))
 require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'actor'))

data/right_agent.gemspec

@@ -24,7 +24,8 @@ require 'rubygems'
 
 Gem::Specification.new do |spec|
   spec.name      = 'right_agent'
-  spec.version   = '0.6.6'
+  spec.version   = '0.9.3'
+  spec.date      = '2012-03-07'
   spec.authors   = ['Lee Kirchhoff', 'Raphael Simon', 'Tony Spataro']
   spec.email     = 'lee@rightscale.com'
   spec.homepage  = 'https://github.com/rightscale/right_agent'
@@ -36,12 +37,13 @@ Gem::Specification.new do |spec|
   spec.required_ruby_version = '>= 1.8.7'
   spec.require_path = 'lib'
 
-  spec.add_dependency('right_support', '~> 1.0')
-  spec.add_dependency('amqp', '0.6.7')
+  spec.add_dependency('right_support', '~> 1.3')
+  spec.add_dependency('right_amqp', '~> 0.1')
   spec.add_dependency('json', ['~> 1.4'])
   spec.add_dependency('eventmachine', '~> 0.12.10')
   spec.add_dependency('right_popen', '~> 1.0.11')
   spec.add_dependency('msgpack', '0.4.4')
+  spec.add_dependency('net-ssh', '~> 2.0')
 
   spec.description = <<-EOF
 RightAgent provides a foundation for running an agent on a server to interface

data/spec/agent_config_spec.rb

@@ -108,7 +108,7 @@ describe RightScale::AgentConfig do
   end
 
   it 'should default configuration directory to platform specific directory' do
-    @agent_config.cfg_dir.should == RightScale::Platform.filesystem.cfg_dir
+    @agent_config.cfg_dir.should == RightScale::Platform.filesystem.right_agent_cfg_dir
   end
 
   it 'should set configuration directory' do

data/spec/agent_spec.rb

@@ -49,7 +49,7 @@ describe RightScale::Agent do
                          :all => ["b1"], :connected => ["b1"], :failed => [], :close_one => true,
                          :non_delivery => true).by_default
       @broker.should_receive(:connection_status).and_yield(:connected)
-      flexmock(RightScale::HABrokerClient).should_receive(:new).and_return(@broker)
+      flexmock(RightAMQP::HABrokerClient).should_receive(:new).and_return(@broker)
       flexmock(RightScale::PidFile).should_receive(:new).
               and_return(flexmock("pid file", :check=>true, :write=>true, :remove=>true))
       @identity = "rs-instance-123-1"
@@ -138,7 +138,7 @@ describe RightScale::Agent do
                          :connected => ["b1"], :failed => [], :all => ["b0", "b1"],
                          :non_delivery => true).by_default
       @broker.should_receive(:connection_status).and_yield(:connected)
-      flexmock(RightScale::HABrokerClient).should_receive(:new).and_return(@broker)
+      flexmock(RightAMQP::HABrokerClient).should_receive(:new).and_return(@broker)
       flexmock(RightScale::PidFile).should_receive(:new).
               and_return(flexmock("pid file", :check=>true, :write=>true, :remove=>true))
       @identity = "rs-instance-123-1"
@@ -266,9 +266,9 @@ describe RightScale::Agent do
                          :stats => "", :status => "status", :hosts => ["123"], :ports => [1, 2], :get => true,
                          :alias_ => "b1", :aliases => ["b1"]).by_default
       @broker.should_receive(:connection_status).and_yield(:connected).by_default
-      @broker.should_receive(:identity_parts).with(@broker_id).and_return(["123", 1, 0, 0, nil])
-      @broker.should_receive(:identity_parts).with(@broker_id2).and_return(["123", 2, 1, 1, nil])
-      flexmock(RightScale::HABrokerClient).should_receive(:new).and_return(@broker)
+      @broker.should_receive(:identity_parts).with(@broker_id).and_return(["123", 1, 0, 0])
+      @broker.should_receive(:identity_parts).with(@broker_id2).and_return(["123", 2, 1, 1])
+      flexmock(RightAMQP::HABrokerClient).should_receive(:new).and_return(@broker)
       flexmock(RightScale::PidFile).should_receive(:new).
               and_return(flexmock("pid file", :check=>true, :write=>true, :remove=>true))
       @sender = flexmock("sender", :pending_requests => [], :request_age => nil,
@@ -312,7 +312,7 @@ describe RightScale::Agent do
           @broker.should_receive(:subscribe).with(hsh(:name => @identity), nil, hsh(:brokers => nil), Proc).
                                              and_return(@broker_ids.first(1)).once
           @agent.run
-          @broker.should_receive(:connect).with("123", 2, 1, 1, nil, false, Proc).once
+          @broker.should_receive(:connect).with("123", 2, 1, 1, false, Proc).once
           @agent.connect("123", 2, 1, 1)
         end
       end
@@ -322,7 +322,7 @@ describe RightScale::Agent do
           @broker.should_receive(:subscribe).with(hsh(:name => @identity), nil, hsh(:brokers => nil), Proc).
                                              and_return(@broker_ids.first(1)).once
           @agent.run
-          @broker.should_receive(:connect).with("123", 2, 1, 1, nil, false, Proc).and_yield(@broker_ids.last).once
+          @broker.should_receive(:connect).with("123", 2, 1, 1, false, Proc).and_yield(@broker_ids.last).once
           @broker.should_receive(:subscribe).with(hsh(:name => @identity), nil, hsh(:brokers => @broker_ids.last(1)), Proc).
                                              and_return(@broker_ids.last(1)).once
           flexmock(@agent).should_receive(:update_configuration).with(:host => ["123"], :port => [1, 2]).and_return(true).once
@@ -335,7 +335,7 @@ describe RightScale::Agent do
           @broker.should_receive(:subscribe).with(hsh(:name => @identity), nil, hsh(:brokers => nil), Proc).
                                              and_return(@broker_ids.first(1)).once
           @agent.run
-          @broker.should_receive(:connect).with("123", 2, 1, 1, nil, false, Proc).and_yield(@broker_ids.last).once
+          @broker.should_receive(:connect).with("123", 2, 1, 1, false, Proc).and_yield(@broker_ids.last).once
           @broker.should_receive(:connection_status).and_yield(:failed)
           @log.should_receive(:error).with(/Failed to connect to broker/).once
           flexmock(@agent).should_receive(:update_configuration).never
@@ -441,14 +441,15 @@ describe RightScale::Agent do
 
       it "should tune heartbeat for all broker connections" do
         run_in_em do
+          @log.should_receive(:info).with(/Reconnecting each broker to tune heartbeat to 45/).once
           @log.should_receive(:info).with(/Tuned heartbeat to 45 seconds for broker/).twice
           @agent = RightScale::Agent.new(:user => "me", :identity => @identity)
           flexmock(@agent).should_receive(:load_actors).and_return(true)
           flexmock(@agent).should_receive(:update_configuration).with(:heartbeat => 45).and_return(true).once
           @agent.run
           @broker.should_receive(:heartbeat=).with(45).once
-          @broker.should_receive(:connect).with("123", 1, 0, 0, nil, true, Proc).and_yield(@broker_id).once
-          @broker.should_receive(:connect).with("123", 2, 1, 1, nil, true, Proc).and_yield(@broker_id2).once
+          @broker.should_receive(:connect).with("123", 1, 0, 0, true, Proc).and_yield(@broker_id).once
+          @broker.should_receive(:connect).with("123", 2, 1, 1, true, Proc).and_yield(@broker_id2).once
           flexmock(@agent).should_receive(:setup_queues).with([@broker_id]).once
           flexmock(@agent).should_receive(:setup_queues).with([@broker_id2]).once
           @agent.tune_heartbeat(45).should be_nil
@@ -457,14 +458,15 @@ describe RightScale::Agent do
 
       it "should tune heartbeat for all broker connections as deferred task" do
         run_in_em do
+          @log.should_receive(:info).with(/Reconnecting each broker to tune heartbeat to 45/).once
           @log.should_receive(:info).with(/Tuned heartbeat to 45 seconds for broker/).twice
           @agent = RightScale::Agent.new(:user => "me", :identity => @identity)
           flexmock(@agent).should_receive(:load_actors).and_return(true)
           flexmock(@agent).should_receive(:update_configuration).with(:heartbeat => 45).and_return(true).once
           @agent.run
           @broker.should_receive(:heartbeat=).with(45).once
-          @broker.should_receive(:connect).with("123", 1, 0, 0, nil, true, Proc).and_yield(@broker_id).once
-          @broker.should_receive(:connect).with("123", 2, 1, 1, nil, true, Proc).and_yield(@broker_id2).once
+          @broker.should_receive(:connect).with("123", 1, 0, 0, true, Proc).and_yield(@broker_id).once
+          @broker.should_receive(:connect).with("123", 2, 1, 1, true, Proc).and_yield(@broker_id2).once
           flexmock(@agent).should_receive(:setup_queues).with([@broker_id]).once
           flexmock(@agent).should_receive(:setup_queues).with([@broker_id2]).once
           flexmock(@agent).should_receive(:finish_setup)
@@ -476,14 +478,15 @@ describe RightScale::Agent do
 
       it "should disable heartbeat for all broker connections" do
         run_in_em do
+          @log.should_receive(:info).with(/Reconnecting each broker to tune heartbeat to 0/).once
           @log.should_receive(:info).with(/Disabled heartbeat for broker/).twice
           @agent = RightScale::Agent.new(:user => "me", :identity => @identity)
           flexmock(@agent).should_receive(:load_actors).and_return(true)
           flexmock(@agent).should_receive(:update_configuration).with(:heartbeat => 0).and_return(true).once
           @agent.run
           @broker.should_receive(:heartbeat=).with(0).once
-          @broker.should_receive(:connect).with("123", 1, 0, 0, nil, true, Proc).and_yield(@broker_id).once
-          @broker.should_receive(:connect).with("123", 2, 1, 1, nil, true, Proc).and_yield(@broker_id2).once
+          @broker.should_receive(:connect).with("123", 1, 0, 0, true, Proc).and_yield(@broker_id).once
+          @broker.should_receive(:connect).with("123", 2, 1, 1, true, Proc).and_yield(@broker_id2).once
           flexmock(@agent).should_receive(:setup_queues).with([@broker_id]).once
           flexmock(@agent).should_receive(:setup_queues).with([@broker_id2]).once
           @agent.tune_heartbeat(0).should be_nil
@@ -492,6 +495,7 @@ describe RightScale::Agent do
 
       it "should log error if any broker connect attempts fail" do
         run_in_em do
+          @log.should_receive(:info).with(/Reconnecting each broker to tune heartbeat to 45/).once
           @log.should_receive(:info).with(/Tuned heartbeat to 45 seconds for broker #{@broker_id2}/).once
           @log.should_receive(:error).with("Failed to reconnect to broker #{@broker_id} to tune heartbeat", Exception, :trace).once
           @agent = RightScale::Agent.new(:user => "me", :identity => @identity)
@@ -499,8 +503,8 @@ describe RightScale::Agent do
           flexmock(@agent).should_receive(:update_configuration).with(:heartbeat => 45).and_return(true).once
           @agent.run
           @broker.should_receive(:heartbeat=).with(45).once
-          @broker.should_receive(:connect).with("123", 1, 0, 0, nil, true, Proc).and_raise(Exception).once
-          @broker.should_receive(:connect).with("123", 2, 1, 1, nil, true, Proc).and_yield(@broker_id2).once
+          @broker.should_receive(:connect).with("123", 1, 0, 0, true, Proc).and_raise(Exception).once
+          @broker.should_receive(:connect).with("123", 2, 1, 1, true, Proc).and_yield(@broker_id2).once
           flexmock(@agent).should_receive(:setup_queues).with([@broker_id]).never
           flexmock(@agent).should_receive(:setup_queues).with([@broker_id2]).once
           @agent.tune_heartbeat(45).should == "Failed to tune heartbeat for brokers [\"#{@broker_id}\"]"
@@ -509,6 +513,7 @@ describe RightScale::Agent do
 
       it "should log error if any brokers do not connect" do
         run_in_em do
+          @log.should_receive(:info).with(/Reconnecting each broker to tune heartbeat to 45/).once
           @log.should_receive(:info).with(/Tuned heartbeat to 45 seconds for broker #{@broker_id2}/).once
           @log.should_receive(:error).with(/Failed to reconnect to broker #{@broker_id} to tune heartbeat, status/).once
           @agent = RightScale::Agent.new(:user => "me", :identity => @identity)
@@ -516,8 +521,8 @@ describe RightScale::Agent do
           flexmock(@agent).should_receive(:update_configuration).with(:heartbeat => 45).and_return(true).once
           @agent.run
           @broker.should_receive(:heartbeat=).with(45).once
-          @broker.should_receive(:connect).with("123", 1, 0, 0, nil, true, Proc).and_yield(@broker_id).once
-          @broker.should_receive(:connect).with("123", 2, 1, 1, nil, true, Proc).and_yield(@broker_id2).once
+          @broker.should_receive(:connect).with("123", 1, 0, 0, true, Proc).and_yield(@broker_id).once
+          @broker.should_receive(:connect).with("123", 2, 1, 1, true, Proc).and_yield(@broker_id2).once
           @broker.should_receive(:connection_status).with({:one_off => 60, :brokers => [@broker_id]}, Proc).and_yield(:failed)
           @broker.should_receive(:connection_status).with({:one_off => 60, :brokers => [@broker_id2]}, Proc).and_yield(:connected)
           flexmock(@agent).should_receive(:setup_queues).with([@broker_id]).never
@@ -528,6 +533,7 @@ describe RightScale::Agent do
 
       it "should log error if any broker queue setup fails" do
         run_in_em do
+          @log.should_receive(:info).with(/Reconnecting each broker to tune heartbeat to 45/).once
           @log.should_receive(:info).with(/Tuned heartbeat to 45 seconds for broker #{@broker_id2}/).once
           @log.should_receive(:error).with(/Failed to setup queues for broker #{@broker_id} when tuning heartbeat/, Exception, :trace).once
           @agent = RightScale::Agent.new(:user => "me", :identity => @identity)
@@ -535,8 +541,8 @@ describe RightScale::Agent do
           flexmock(@agent).should_receive(:update_configuration).with(:heartbeat => 45).and_return(true).once
           @agent.run
           @broker.should_receive(:heartbeat=).with(45).once
-          @broker.should_receive(:connect).with("123", 1, 0, 0, nil, true, Proc).and_yield(@broker_id).once
-          @broker.should_receive(:connect).with("123", 2, 1, 1, nil, true, Proc).and_yield(@broker_id2).once
+          @broker.should_receive(:connect).with("123", 1, 0, 0, true, Proc).and_yield(@broker_id).once
+          @broker.should_receive(:connect).with("123", 2, 1, 1, true, Proc).and_yield(@broker_id2).once
           flexmock(@agent).should_receive(:setup_queues).with([@broker_id]).and_raise(Exception)
           flexmock(@agent).should_receive(:setup_queues).with([@broker_id2]).once
           @agent.tune_heartbeat(45).should be_nil

data/spec/core_payload_types/login_user_spec.rb

@@ -82,16 +82,20 @@ describe RightScale::LoginUser do
     test_serialization_downgrade(user, public_key)
   end
 
-  it 'should serialize current version with multiple public_keys' do
+  it 'should serialize current version with multiple public_keys and fingerprints' do
     num = rand(2**32).to_s(32)
     public_keys = []
-    3.times do
+    fingerprints = []
+    3.times do |i|
       pub = rand(2**32).to_s(32)
       public_keys << "ssh-rsa #{pub} #{num}@rightscale.com"
+      fingerprints << "sha#{i}"
     end
-    new_user = RightScale::LoginUser.new("v0-#{num}", "rs-#{num}", nil, "#{num}@rightscale.old", true, nil, public_keys)
+    new_user = RightScale::LoginUser.new("v0-#{num}", "rs-#{num}", nil, "#{num}@rightscale.old", true, nil, public_keys,
+                                         nil, fingerprints)
     new_user.public_key.should == public_keys.first
     new_user.public_keys.should == public_keys
+    new_user.public_key_fingerprints.should == fingerprints
     test_serialization_downgrade(new_user, public_keys.first)
   end