right_agent 0.5.1 → 0.5.10

data/spec/dispatcher_spec.rb

@@ -96,6 +96,7 @@ describe "RightScale::Dispatcher" do
     @agent = flexmock("Agent", :identity => "agent", :broker => @broker, :registry => @registry, :options => {}).by_default
     @dispatcher = RightScale::Dispatcher.new(@agent)
     @dispatcher.em = EMMock
+    @response_queue = RightScale::Dispatcher::RESPONSE_QUEUE
   end
 
   describe "Dispatched cache" do
@@ -199,6 +200,17 @@ describe "RightScale::Dispatcher" do
     res.results.should == ['hello', 'you']
   end
 
+  it "should publish result of request to response queue" do
+    req = RightScale::Request.new('/foo', 'you', :token => 'token')
+    req.reply_to = "rs-mapper-1-1"
+    @broker.should_receive(:publish).with(hsh(:name => @response_queue),
+                                          on {|arg| arg.class == RightScale::Result &&
+                                                    arg.to == "rs-mapper-1-1" &&
+                                                    arg.results == ['hello', 'you']},
+                                          hsh(:persistent => true, :mandatory => true)).once
+    res = @dispatcher.dispatch(req)
+  end
+
   it "should handle custom prefixes" do
     @registry.register(Foo.new, 'umbongo')
     req = RightScale::Request.new('/umbongo/bar', 'you')
@@ -257,13 +269,15 @@ describe "RightScale::Dispatcher" do
   it "should send non-delivery result if Request is rejected because its time-to-live has expired" do
     flexmock(Time).should_receive(:now).and_return(Time.at(1000000)).by_default
     flexmock(RightScale::Log).should_receive(:info).once.with(on {|arg| arg =~ /REJECT EXPIRED/})
-    @broker.should_receive(:publish).with(Hash, on {|arg| arg.class == RightScale::Result &&
-                                                          arg.results.non_delivery? &&
-                                                          arg.results.content == RightScale::OperationResult::TTL_EXPIRATION},
+    @broker.should_receive(:publish).with(hsh(:name => @response_queue),
+                                          on {|arg| arg.class == RightScale::Result &&
+                                                    arg.to == @response_queue &&
+                                                    arg.results.non_delivery? &&
+                                                    arg.results.content == RightScale::OperationResult::TTL_EXPIRATION},
                                           hsh(:persistent => true, :mandatory => true)).once
     @dispatcher = RightScale::Dispatcher.new(@agent)
     @dispatcher.em = EMMock
-    req = RightScale::Request.new('/foo/bar', 'you', :expires_at => @now.to_i + 8)
+    req = RightScale::Request.new('/foo/bar', 'you', {:reply_to => @response_queue, :expires_at => @now.to_i + 8})
     flexmock(Time).should_receive(:now).and_return(@now += 10)
     @dispatcher.dispatch(req).should be_nil
   end
@@ -271,13 +285,15 @@ describe "RightScale::Dispatcher" do
   it "should send error result instead of non-delivery if agent does not know about non-delivery" do
     flexmock(Time).should_receive(:now).and_return(Time.at(1000000)).by_default
     flexmock(RightScale::Log).should_receive(:info).once.with(on {|arg| arg =~ /REJECT EXPIRED/})
-    @broker.should_receive(:publish).with(Hash, on {|arg| arg.class == RightScale::Result &&
-                                                          arg.results.error? &&
-                                                          arg.results.content =~ /Could not deliver/},
+    @broker.should_receive(:publish).with(hsh(:name => @response_queue),
+                                          on {|arg| arg.class == RightScale::Result &&
+                                                    arg.to == "rs-mapper-1-1" &&
+                                                    arg.results.error? &&
+                                                    arg.results.content =~ /Could not deliver/},
                                           hsh(:persistent => true, :mandatory => true)).once
     @dispatcher = RightScale::Dispatcher.new(@agent)
     @dispatcher.em = EMMock
-    req = RightScale::Request.new('/foo/bar', 'you', {:expires_at => @now.to_i + 8}, [12, 13])
+    req = RightScale::Request.new('/foo/bar', 'you', {:reply_to => "rs-mapper-1-1", :expires_at => @now.to_i + 8}, [12, 13])
     flexmock(Time).should_receive(:now).and_return(@now += 10)
     @dispatcher.dispatch(req).should be_nil
   end

data/spec/ha_broker_client_spec.rb

@@ -27,9 +27,10 @@ describe RightScale::HABrokerClient do
   include FlexMock::ArgumentTypes
 
   before(:each) do
-    flexmock(RightScale::Log).should_receive(:error).by_default.and_return { |m| raise RightScale::Log.format(*m) }
-    flexmock(RightScale::Log).should_receive(:warning).by_default.and_return { |m| raise RightScale::Log.format(*m) }
-    flexmock(RightScale::Log).should_receive(:info).by_default
+    @log = flexmock(RightScale::Log)
+    @log.should_receive(:error).by_default.and_return { |m| raise RightScale::Log.format(*m) }
+    @log.should_receive(:warning).by_default.and_return { |m| raise RightScale::Log.format(*m) }
+    @log.should_receive(:info).by_default
   end
 
   describe "Context" do
@@ -595,7 +596,7 @@ describe RightScale::HABrokerClient do
       end
 
       it "should not do anything except log a message if asked to reconnect an already connected broker client" do
-        flexmock(RightScale::Log).should_receive(:info).with(/Ignored request to reconnect/).once
+        @log.should_receive(:info).with(/Ignored request to reconnect/).once
         ha = RightScale::HABrokerClient.new(@serializer, :host => "a, b")
         ha.brokers.size.should == 2
         @broker_a.should_receive(:status).and_return(:connected).once
@@ -610,7 +611,7 @@ describe RightScale::HABrokerClient do
       end
 
       it "should reconnect already connected broker client if force specified" do
-        flexmock(RightScale::Log).should_receive(:info).with(/Ignored request to reconnect/).never
+        @log.should_receive(:info).with(/Ignored request to reconnect/).never
         ha = RightScale::HABrokerClient.new(@serializer, :host => "a, b")
         ha.brokers.size.should == 2
         @broker_a.should_receive(:close).and_return(true).once
@@ -623,6 +624,21 @@ describe RightScale::HABrokerClient do
         ha.brokers[1].alias == "b1"
       end
 
+      it "should be able to change host and port of an existing broker client" do
+        ha = RightScale::HABrokerClient.new(@serializer, :host => "a, b")
+        ha.brokers.size.should == 2
+        @broker_a.should_receive(:close).and_return(true).once
+        flexmock(RightScale::BrokerClient).should_receive(:new).with(@identity_c, @address_c.merge(:index => 0),
+                 @serializer, @exceptions, Hash, nil, nil).and_return(@broker_c).once
+        res = ha.connect("c", 5672, 0)
+        res.should be_true
+        ha.brokers.size.should == 2
+        ha.brokers[0].alias == "b0"
+        ha.brokers[0].identity == @address_c
+        ha.brokers[1].alias == "b1"
+        ha.brokers[1].identity == @address_b
+      end
+
       it "should slot broker client into specified priority position when at end of list" do
         ha = RightScale::HABrokerClient.new(@serializer, :host => "a, b")
         ha.brokers.size.should == 2
@@ -646,7 +662,7 @@ describe RightScale::HABrokerClient do
       end
 
       it "should slot broker client into nex priority position if specified priority would leave a gap" do
-        flexmock(RightScale::Log).should_receive(:info).with(/Reduced priority setting for broker/).once
+        @log.should_receive(:info).with(/Reduced priority setting for broker/).once
         ha = RightScale::HABrokerClient.new(@serializer, :host => "a")
         ha.brokers.size.should == 1
         res = ha.connect("c", 5672, 2, 2)
@@ -668,11 +684,6 @@ describe RightScale::HABrokerClient do
         ha.brokers[1].alias == "b1"
       end
 
-      it "should raise an exception if try to change host and port of an existing broker client" do
-        ha = RightScale::HABrokerClient.new(@serializer, :host => "a, b")
-        lambda { ha.connect("c", 5672, 0) }.should raise_error(Exception, /Not allowed to change host or port/)
-      end
-
     end # connecting
 
     context "updating connection" do
@@ -925,7 +936,7 @@ describe RightScale::HABrokerClient do
       end
 
       it "should log an error if a selected broker is unknown but still publish with any remaining brokers" do
-        flexmock(RightScale::Log).should_receive(:error).with(/Invalid broker identity "rs-broker-fifth-5672"/).once
+        @log.should_receive(:error).with(/Invalid broker identity "rs-broker-fifth-5672"/).once
         ha = RightScale::HABrokerClient.new(@serializer, :islands => @islands, :home_island => @home)
         ha.publish({:type => :direct, :name => "exchange"}, @packet,
                    :brokers =>["rs-broker-fifth-5672", @identity1]).should == [@identity1]
@@ -1025,8 +1036,8 @@ describe RightScale::HABrokerClient do
         end
 
         it "should republish using a broker not yet tried if possible and log that re-routing" do
-          flexmock(RightScale::Log).should_receive(:info).with(/RE-ROUTE/).once
-          flexmock(RightScale::Log).should_receive(:info).with(/RETURN reason/).once
+          @log.should_receive(:info).with(/RE-ROUTE/).once
+          @log.should_receive(:info).with(/RETURN reason/).once
           ha = RightScale::HABrokerClient.new(@serializer, :islands => @islands, :home_island => @home)
           @context.record_failure(@identity3)
           @broker4.should_receive(:publish).and_return(true).once
@@ -1034,8 +1045,8 @@ describe RightScale::HABrokerClient do
         end
 
         it "should republish to same broker without mandatory if message is persistent and no other brokers available" do
-          flexmock(RightScale::Log).should_receive(:info).with(/RE-ROUTE/).once
-          flexmock(RightScale::Log).should_receive(:info).with(/RETURN reason/).once
+          @log.should_receive(:info).with(/RE-ROUTE/).once
+          @log.should_receive(:info).with(/RETURN reason/).once
           ha = RightScale::HABrokerClient.new(@serializer, :islands => @islands, :home_island => @home)
           @context.record_failure(@identity3)
           @context.record_failure(@identity4)
@@ -1045,8 +1056,8 @@ describe RightScale::HABrokerClient do
         end
 
         it "should republish to same broker without mandatory if message is one-way and no other brokers available" do
-          flexmock(RightScale::Log).should_receive(:info).with(/RE-ROUTE/).once
-          flexmock(RightScale::Log).should_receive(:info).with(/RETURN reason/).once
+          @log.should_receive(:info).with(/RE-ROUTE/).once
+          @log.should_receive(:info).with(/RETURN reason/).once
           ha = RightScale::HABrokerClient.new(@serializer, :islands => @islands, :home_island => @home)
           @context.record_failure(@identity3)
           @context.record_failure(@identity4)
@@ -1056,8 +1067,8 @@ describe RightScale::HABrokerClient do
         end
 
         it "should update status to :stopping if message returned because access refused" do
-          flexmock(RightScale::Log).should_receive(:info).with(/RE-ROUTE/).once
-          flexmock(RightScale::Log).should_receive(:info).with(/RETURN reason/).once
+          @log.should_receive(:info).with(/RE-ROUTE/).once
+          @log.should_receive(:info).with(/RETURN reason/).once
           ha = RightScale::HABrokerClient.new(@serializer, :islands => @islands, :home_island => @home)
           @context.record_failure(@identity3)
           @broker4.should_receive(:publish).and_return(true).once
@@ -1066,8 +1077,8 @@ describe RightScale::HABrokerClient do
         end
 
         it "should log info and make non-delivery call even if persistent when returned because of no queue" do
-          flexmock(RightScale::Log).should_receive(:info).with(/NO ROUTE/).once
-          flexmock(RightScale::Log).should_receive(:info).with(/RETURN reason/).once
+          @log.should_receive(:info).with(/NO ROUTE/).once
+          @log.should_receive(:info).with(/RETURN reason/).once
           ha = RightScale::HABrokerClient.new(@serializer, :islands => @islands, :home_island => @home)
           called = 0
           ha.non_delivery { |reason, type, token, from, to| called += 1 }
@@ -1081,8 +1092,8 @@ describe RightScale::HABrokerClient do
         end
 
         it "should log info and make non-delivery call if no route can be found" do
-          flexmock(RightScale::Log).should_receive(:info).with(/NO ROUTE/).once
-          flexmock(RightScale::Log).should_receive(:info).with(/RETURN reason/).once
+          @log.should_receive(:info).with(/NO ROUTE/).once
+          @log.should_receive(:info).with(/RETURN reason/).once
           ha = RightScale::HABrokerClient.new(@serializer, :islands => @islands, :home_island => @home)
           called = 0
           ha.non_delivery { |reason, type, token, from, to| called += 1 }
@@ -1095,7 +1106,7 @@ describe RightScale::HABrokerClient do
         end
 
         it "should log info if no message context available for re-routing it" do
-          flexmock(RightScale::Log).should_receive(:info).with(/Dropping/).once
+          @log.should_receive(:info).with(/Dropping/).once
           ha = RightScale::HABrokerClient.new(@serializer, :islands => @islands, :home_island => @home)
           ha.__send__(:handle_return, @identity4, "any reason", @message, "to", nil)
         end
@@ -1126,12 +1137,22 @@ describe RightScale::HABrokerClient do
         ha.delete("queue").should == [@identity4, @identity2]
       end
 
+      it "should delete queue from cache on all usable broker clients and return their identities" do
+        ha = RightScale::HABrokerClient.new(@serializer, :islands => @islands, :home_island => @home)
+        @broker1.should_receive(:usable?).and_return(false)
+        @broker1.should_receive(:delete_amqp_resources).never
+        @broker2.should_receive(:delete_amqp_resources).and_return(true).once
+        @broker3.should_receive(:delete_amqp_resources).and_return(true).once
+        @broker4.should_receive(:delete_amqp_resources).and_return(true).once
+        ha.delete_amqp_resources("queue").should == [@identity3, @identity4, @identity2]
+      end
+
     end # deleting
 
     context "removing" do
 
       it "should remove broker client after disconnecting and pass identity to block" do
-        flexmock(RightScale::Log).should_receive(:info).with(/Removing/).once
+        @log.should_receive(:info).with(/Removing/).once
         @broker2.should_receive(:close).with(true, true, false).once
         ha = RightScale::HABrokerClient.new(@serializer, :islands => @islands, :home_island => @home)
         identity = nil
@@ -1146,7 +1167,7 @@ describe RightScale::HABrokerClient do
       end
 
       it "should remove broker when no block supplied but still return a result" do
-        flexmock(RightScale::Log).should_receive(:info).with(/Removing/).once
+        @log.should_receive(:info).with(/Removing/).once
         @broker2.should_receive(:close).once
         ha = RightScale::HABrokerClient.new(@serializer, :islands => @islands, :home_island => @home)
         result = ha.remove("second", 5672)
@@ -1159,7 +1180,7 @@ describe RightScale::HABrokerClient do
       end
 
       it "should remove last broker if requested" do
-        flexmock(RightScale::Log).should_receive(:info).with(/Removing/).times(4)
+        @log.should_receive(:info).with(/Removing/).times(4)
         @broker1.should_receive(:close).once
         @broker2.should_receive(:close).once
         @broker3.should_receive(:close).once
@@ -1184,7 +1205,7 @@ describe RightScale::HABrokerClient do
       end
 
       it "should return nil and not execute block if broker is unknown" do
-        flexmock(RightScale::Log).should_receive(:info).with(/Ignored request to remove/).once
+        @log.should_receive(:info).with(/Ignored request to remove/).once
         ha = RightScale::HABrokerClient.new(@serializer, :islands => @islands, :home_island => @home)
         ha.remove("fifth", 5672).should be_nil
         ha.brokers.size.should == 4
@@ -1331,23 +1352,24 @@ describe RightScale::HABrokerClient do
         @broker4.should_receive(:stats).and_return("stats4")
         ha.stats.should == {"brokers" => ["stats3", "stats4", "stats1", "stats2"],
                             "exceptions" => nil,
+                            "heartbeat" => nil,
                             "returns" => nil}
       end
 
       it "should log broker client status update if there is a change" do
-        flexmock(RightScale::Log).should_receive(:info).with(/Broker b0 is now connected/).once
+        @log.should_receive(:info).with(/Broker b0 is now connected/).once
         ha = RightScale::HABrokerClient.new(@serializer, :islands => @islands, :home_island => @home)
         ha.__send__(:update_status, @broker3, false)
       end
 
       it "should not log broker client status update if there is no change" do
-        flexmock(RightScale::Log).should_receive(:info).with(/Broker b0 is now connected/).never
+        @log.should_receive(:info).with(/Broker b0 is now connected/).never
         ha = RightScale::HABrokerClient.new(@serializer, :islands => @islands, :home_island => @home)
         ha.__send__(:update_status, @broker3, true)
       end
 
       it "should log broker client status update when become disconnected" do
-        flexmock(RightScale::Log).should_receive(:info).with(/Broker b0 is now disconnected/).once
+        @log.should_receive(:info).with(/Broker b0 is now disconnected/).once
         ha = RightScale::HABrokerClient.new(@serializer, :islands => @islands, :home_island => @home)
         @broker3.should_receive(:status).and_return(:disconnected)
         @broker3.should_receive(:connected?).and_return(false)
@@ -1630,7 +1652,7 @@ describe RightScale::HABrokerClient do
       end
 
       it "should close all broker connections even if encounter an exception" do
-        flexmock(RightScale::Log).should_receive(:error).with(/Failed to close/, Exception, :trace).once
+        @log.should_receive(:error).with(/Failed to close/, Exception, :trace).once
         @broker1.should_receive(:close).and_return(true).and_yield.once
         @broker2.should_receive(:close).and_raise(Exception).once
         @broker3.should_receive(:close).and_return(true).and_yield.once

data/spec/monkey_patches/amqp_patch_spec.rb

@@ -95,6 +95,18 @@ describe AMQP::Client do
       end
     end
 
+    context 'and an EM reconnect failure' do
+      it 'should log an error and schedule another reconnect' do
+        @sut.settings[:reconnect_delay] = true
+
+        flexmock(RightScale::Log).should_receive(:error).with(/Exception caught during AMQP reconnect/, Exception, :trace).once
+        flexmock(EM).should_receive(:reconnect).and_raise(Exception)
+        flexmock(EM).should_receive(:add_timer).with(5, Proc).once
+
+        @sut.reconnect()
+      end
+    end
+
   end
 
 end

data/spec/packets_spec.rb

@@ -263,6 +263,7 @@ describe "Packet: Push" do
     packet.from.should == packet2.from
     packet.token.should == packet2.token
     packet.tries.should == packet2.tries
+    packet.confirm.should == packet2.confirm
     packet.expires_at.should == packet2.expires_at
     packet.recv_version.should == packet2.recv_version
     packet.send_version.should == packet2.send_version
@@ -276,6 +277,7 @@ describe "Packet: Push" do
     packet.from.should == packet2.from
     packet.token.should == packet2.token
     packet.tries.should == packet2.tries
+    packet.confirm.should == packet2.confirm
     packet.expires_at.should == packet2.expires_at
     packet.recv_version.should == packet2.recv_version
     packet.send_version.should == packet2.send_version

data/spec/sender_spec.rb

@@ -59,6 +59,8 @@ describe RightScale::Sender do
   describe "when monitoring broker connectivity" do
     before(:each) do
       flexmock(EM).should_receive(:next_tick).and_yield.by_default
+      @now = Time.at(1000000)
+      flexmock(Time).should_receive(:now).and_return(@now).by_default
       @broker = flexmock("Broker", :subscribe => true, :publish => ["broker"], :connected? => true,
                          :identity_parts => ["host", 123, 0, 0, nil]).by_default
       @agent = flexmock("Agent", :identity => "agent", :broker => @broker, :options => {:ping_interval => 0}).by_default
@@ -79,7 +81,8 @@ describe RightScale::Sender do
       @agent.should_receive(:options).and_return(:ping_interval => 1000)
       flexmock(EM::Timer).should_receive(:new).with(1000, Proc).and_return(@timer).once
       instance = RightScale::Sender.new(@agent)
-      flexmock(instance).should_receive(:restart_inactivity_timer).once
+      flexmock(Time).should_receive(:now).and_return(@now + 61)
+      flexmock(instance.connectivity_checker).should_receive(:restart_inactivity_timer).once
       instance.message_received
       instance.message_received
     end
@@ -89,11 +92,47 @@ describe RightScale::Sender do
       flexmock(EM::Timer).should_receive(:new).and_return(@timer).once.by_default
       RightScale::Sender.new(@agent)
       instance = RightScale::Sender.instance
+      flexmock(Time).should_receive(:now).and_return(@now + 61)
       flexmock(EM::Timer).should_receive(:new).and_return(@timer).and_yield.once
-      flexmock(instance).should_receive(:check_connection).once
+      flexmock(instance.connectivity_checker).should_receive(:check).once
       instance.message_received
     end
 
+    it "should check connectivity by sending mapper ping" do
+      @agent.should_receive(:options).and_return(:ping_interval => 1000)
+      flexmock(EM::Timer).should_receive(:new).and_return(@timer).twice
+      RightScale::Sender.new(@agent)
+      instance = RightScale::Sender.instance
+      broker_id = "rs-broker-1-1"
+      flexmock(instance).should_receive(:publish).with(on do |request|
+        request.type.should == "/mapper/ping";
+        request.from.should == "agent"
+      end, [broker_id]).and_return([broker_id]).once
+      instance.connectivity_checker.check(broker_id)
+      instance.pending_requests.size.should == 1
+    end
+
+    it "should not check connectivity if terminating" do
+      @agent.should_receive(:options).and_return(:ping_interval => 1000)
+      flexmock(EM::Timer).should_receive(:new).and_return(@timer).once
+      RightScale::Sender.new(@agent)
+      instance = RightScale::Sender.instance
+      flexmock(instance).should_receive(:publish).never
+      instance.terminate
+      instance.connectivity_checker.check
+    end
+
+    it "should not check connectivity if not connected to broker" do
+      @agent.should_receive(:options).and_return(:ping_interval => 1000)
+      flexmock(EM::Timer).should_receive(:new).and_return(@timer).once
+      RightScale::Sender.new(@agent)
+      broker_id = "rs-broker-1-1"
+      @broker.should_receive(:connected?).with(broker_id).and_return(false)
+      instance = RightScale::Sender.instance
+      flexmock(instance).should_receive(:publish).never
+      instance.connectivity_checker.check(broker_id)
+    end
+
     it "should ignore messages received if ping disabled" do
       @agent.should_receive(:options).and_return(:ping_interval => 0)
       flexmock(EM::Timer).should_receive(:new).never
@@ -107,8 +146,9 @@ describe RightScale::Sender do
       flexmock(EM::Timer).should_receive(:new).and_return(@timer).once.by_default
       RightScale::Sender.new(@agent)
       instance = RightScale::Sender.instance
+      flexmock(Time).should_receive(:now).and_return(@now + 61)
       flexmock(EM::Timer).should_receive(:new).and_return(@timer).and_yield.once
-      flexmock(instance).should_receive(:check_connection).and_raise(Exception)
+      flexmock(instance.connectivity_checker).should_receive(:check).and_raise(Exception)
       instance.message_received
     end
 
@@ -118,18 +158,18 @@ describe RightScale::Sender do
       broker_id = "rs-broker-localhost-5672"
       @broker.should_receive(:identity_parts).with(broker_id).and_return(["localhost", 5672, 0, 0, nil]).once
       @agent.should_receive(:connect).with("localhost", 5672, 0, 0, true).once
-      old_ping_timeout = RightScale::Sender::PING_TIMEOUT
+      old_ping_timeout = RightScale::Sender::ConnectivityChecker::PING_TIMEOUT
       begin
-        RightScale::Sender.const_set(:PING_TIMEOUT, 0.5)
+        RightScale::Sender::ConnectivityChecker.const_set(:PING_TIMEOUT, 0.5)
         EM.run do
           EM.add_timer(1) { EM.stop }
           RightScale::Sender.new(@agent)
           instance = RightScale::Sender.instance
           flexmock(instance).should_receive(:publish).with(RightScale::Request, nil).and_return([broker_id])
-          instance.__send__(:check_connection)
+          instance.connectivity_checker.check
         end
       ensure
-        RightScale::Sender.const_set(:PING_TIMEOUT, old_ping_timeout)
+        RightScale::Sender::ConnectivityChecker.const_set(:PING_TIMEOUT, old_ping_timeout)
       end
     end
   end
@@ -138,6 +178,7 @@ describe RightScale::Sender do
     before(:each) do
       @timer = flexmock("timer")
       flexmock(EM::Timer).should_receive(:new).and_return(@timer)
+      flexmock(Time).should_receive(:now).and_return(Time.at(1000000)).by_default
       @broker = flexmock("Broker", :subscribe => true, :publish => true).by_default
       @agent = flexmock("Agent", :identity => "agent", :broker => @broker).by_default
       @agent.should_receive(:options).and_return({}).by_default
@@ -205,6 +246,7 @@ describe RightScale::Sender do
         push.persistent.should be_false
         push.from.should == 'agent'
         push.target.should be_nil
+        push.confirm.should be_nil
         push.expires_at.should == 0
       end, hsh(:persistent => false, :mandatory => true)).once
       @instance.send_push('/welcome/aboard', 'iZac')
@@ -217,27 +259,40 @@ describe RightScale::Sender do
       @instance.initialize_offline_queue
       @broker.should_receive(:publish).never
       @instance.enable_offline_mode
-      @instance.instance_variable_get(:@queueing_mode).should == :offline
+      @instance.offline_handler.mode.should == :offline
       @instance.send_push('/welcome/aboard', 'iZac')
-      @instance.instance_variable_get(:@queue).size.should == 1
+      @instance.offline_handler.queue.size.should == 1
     end
 
     it "should store the response handler if given" do
       response_handler = lambda {}
       flexmock(RightScale::AgentIdentity).should_receive(:generate).and_return('abc').once
+      @broker.should_receive(:publish).with(hsh(:name => "request"), on do |push|
+        push.confirm.should == true
+      end, hsh(:persistent => false, :mandatory => true)).once
       @instance.send_push('/welcome/aboard', 'iZac', &response_handler)
-      @instance.pending_requests['abc'][:response_handler].should == response_handler
+      @instance.pending_requests['abc'].response_handler.should == response_handler
     end
 
     it "should store the request receive time if there is a response handler" do
       response_handler = lambda {}
       flexmock(RightScale::AgentIdentity).should_receive(:generate).and_return('abc').once
-      flexmock(Time).should_receive(:now).and_return(Time.at(1000000)).by_default
-      @instance.request_age.should be_nil
+      @instance.pending_requests.kind(RightScale::Sender::PendingRequests::PUSH_KINDS).youngest_age.should be_nil
       @instance.send_push('/welcome/aboard', 'iZac', &response_handler)
-      @instance.pending_requests['abc'][:receive_time].should == Time.at(1000000)
+      @instance.pending_requests['abc'].receive_time.should == Time.at(1000000)
       flexmock(Time).should_receive(:now).and_return(Time.at(1000100))
-      @instance.request_age.should == 100
+      @instance.pending_requests.kind(RightScale::Sender::PendingRequests::PUSH_KINDS).youngest_age.should == 100
+    end
+
+    it "should eventually remove push from pending requests if no response received" do
+      response_handler = lambda {}
+      flexmock(RightScale::AgentIdentity).should_receive(:generate).and_return('abc', 'xyz').twice
+      @instance.send_push('/welcome/aboard', 'iZac', &response_handler)
+      @instance.pending_requests['abc'].should_not be_nil
+      flexmock(Time).should_receive(:now).and_return(Time.at(1000121))
+      @instance.send_push('/welcome/aboard', 'iZac', &response_handler)
+      @instance.pending_requests['xyz'].should_not be_nil
+      @instance.pending_requests['abc'].should be_nil
     end
   end
 
@@ -266,6 +321,7 @@ describe RightScale::Sender do
         push.persistent.should be_true
         push.from.should == 'agent'
         push.target.should be_nil
+        push.confirm.should be_nil
         push.expires_at.should == 0
       end, hsh(:persistent => true, :mandatory => true)).once
       @instance.send_persistent_push('/welcome/aboard', 'iZac')
@@ -282,19 +338,22 @@ describe RightScale::Sender do
     it "should store the response handler if given" do
       response_handler = lambda {}
       flexmock(RightScale::AgentIdentity).should_receive(:generate).and_return('abc').once
+      @broker.should_receive(:publish).with(hsh(:name => "request"), on do |push|
+        push.confirm.should == true
+      end, hsh(:persistent => true, :mandatory => true)).once
       @instance.send_persistent_push('/welcome/aboard', 'iZac', &response_handler)
-      @instance.pending_requests['abc'][:response_handler].should == response_handler
+      @instance.pending_requests['abc'].response_handler.should == response_handler
     end
 
     it "should store the request receive time if there is a response handler" do
       response_handler = lambda {}
       flexmock(RightScale::AgentIdentity).should_receive(:generate).and_return('abc').once
       flexmock(Time).should_receive(:now).and_return(Time.at(1000000)).by_default
-      @instance.request_age.should be_nil
+      @instance.pending_requests.kind(RightScale::Sender::PendingRequests::PUSH_KINDS).youngest_age.should be_nil
       @instance.send_persistent_push('/welcome/aboard', 'iZac', &response_handler)
-      @instance.pending_requests['abc'][:receive_time].should == Time.at(1000000)
+      @instance.pending_requests['abc'].receive_time.should == Time.at(1000000)
       flexmock(Time).should_receive(:now).and_return(Time.at(1000100))
-      @instance.request_age.should == 100
+      @instance.pending_requests.kind(RightScale::Sender::PendingRequests::PUSH_KINDS).youngest_age.should == 100
     end
   end
 
@@ -396,17 +455,17 @@ describe RightScale::Sender do
       response_handler = lambda {}
       flexmock(RightScale::AgentIdentity).should_receive(:generate).and_return('abc').once
       @instance.send_retryable_request('/welcome/aboard', 'iZac', &response_handler)
-      @instance.pending_requests['abc'][:response_handler].should == response_handler
+      @instance.pending_requests['abc'].response_handler.should == response_handler
     end
 
     it "should store the request receive time" do
       flexmock(RightScale::AgentIdentity).should_receive(:generate).and_return('abc').once
       flexmock(Time).should_receive(:now).and_return(Time.at(1000000)).by_default
-      @instance.request_age.should be_nil
+      @instance.pending_requests.kind(RightScale::Sender::PendingRequests::REQUEST_KINDS).youngest_age.should be_nil
       @instance.send_retryable_request('/welcome/aboard', 'iZac')
-      @instance.pending_requests['abc'][:receive_time].should == Time.at(1000000)
+      @instance.pending_requests['abc'].receive_time.should == Time.at(1000000)
       flexmock(Time).should_receive(:now).and_return(Time.at(1000100))
-      @instance.request_age.should == 100
+      @instance.pending_requests.kind(RightScale::Sender::PendingRequests::REQUEST_KINDS).youngest_age.should == 100
     end
 
     it 'should queue the request if in offline mode and :offline_queueing enabled' do
@@ -416,9 +475,9 @@ describe RightScale::Sender do
       @instance.initialize_offline_queue
       @broker.should_receive(:publish).never
       @instance.enable_offline_mode
-      @instance.instance_variable_get(:@queueing_mode).should == :offline
+      @instance.offline_handler.mode.should == :offline
       @instance.send_retryable_request('/welcome/aboard', 'iZac')
-      @instance.instance_variable_get(:@queue).size.should == 1
+      @instance.offline_handler.queue.size.should == 1
     end
 
     it "should dump the pending requests" do
@@ -489,7 +548,7 @@ describe RightScale::Sender do
           @agent.should_receive(:options).and_return({:retry_timeout => 0.3, :retry_interval => 0.1})
           RightScale::Sender.new(@agent)
           @instance = RightScale::Sender.instance
-          flexmock(@instance).should_receive(:check_connection).once
+          flexmock(@instance.connectivity_checker).should_receive(:check).once
           @broker.should_receive(:publish).and_return(@broker_ids).twice
           @instance.send_retryable_request('/welcome/aboard', 'iZac') do |response|
             result = RightScale::OperationResult.from_results(response)
@@ -514,7 +573,7 @@ describe RightScale::Sender do
           @agent.should_receive(:options).and_return({:retry_timeout => 0.6, :retry_interval => 0.1})
           RightScale::Sender.new(@agent)
           @instance = RightScale::Sender.instance
-          flexmock(@instance).should_receive(:check_connection).once
+          flexmock(@instance.connectivity_checker).should_receive(:check).once
           @broker.should_receive(:publish).and_return(@broker_ids).times(3)
           @instance.send_retryable_request('/welcome/aboard', 'iZac') do |response|
             result = RightScale::OperationResult.from_results(response)
@@ -537,7 +596,7 @@ describe RightScale::Sender do
           @agent.should_receive(:options).and_return({:retry_timeout => 0.5, :retry_interval => 0.1})
           RightScale::Sender.new(@agent)
           @instance = RightScale::Sender.instance
-          flexmock(@instance).should_receive(:check_connection).once
+          flexmock(@instance.connectivity_checker).should_receive(:check).once
           @broker.should_receive(:publish).with(hsh(:name => "request"), on do |request|
             request.expires_at.should == (expires_at ||= request.expires_at)
           end, hsh(:persistent => false, :mandatory => true)).and_return(@broker_ids).twice
@@ -554,16 +613,16 @@ describe RightScale::Sender do
 
         it "should not check connection if check already in progress" do
           flexmock(EM::Timer).should_receive(:new).and_return(@timer).never
-          @instance.pending_ping = true
+          @instance.connectivity_checker.ping_timer = true
           flexmock(@instance).should_receive(:publish).never
-          @instance.__send__(:check_connection, @broker_ids)
+          @instance.connectivity_checker.check(@broker_ids)
         end
 
         it "should publish ping to mapper" do
           flexmock(EM::Timer).should_receive(:new).and_return(@timer).once
           flexmock(@instance).should_receive(:publish).with(on { |request| request.type.should == "/mapper/ping" },
                                                             @broker_ids).and_return(@broker_ids).once
-          @instance.__send__(:check_connection, @broker_id)
+          @instance.connectivity_checker.check(@broker_id)
           @instance.pending_requests.size.should == 1
         end
 
@@ -572,19 +631,19 @@ describe RightScale::Sender do
           @timer.should_receive(:cancel).once
           flexmock(EM::Timer).should_receive(:new).and_return(@timer).once
           flexmock(@instance).should_receive(:publish).and_return(@broker_ids).once
-          @instance.__send__(:check_connection, @broker_id)
-          @instance.pending_ping.should == @timer
+          @instance.connectivity_checker.check(@broker_id)
+          @instance.connectivity_checker.ping_timer.should == @timer
           @instance.pending_requests.size.should == 1
-          @instance.pending_requests['abc'][:response_handler].call(nil)
-          @instance.pending_ping.should == nil
+          @instance.pending_requests['abc'].response_handler.call(nil)
+          @instance.connectivity_checker.ping_timer.should == nil
         end
 
         it "should try to reconnect if ping times out" do
           @log.should_receive(:warning).once
           flexmock(EM::Timer).should_receive(:new).and_yield.once
           flexmock(@agent).should_receive(:connect).once
-          @instance.__send__(:check_connection, @broker_id)
-          @instance.pending_ping.should == nil
+          @instance.connectivity_checker.check(@broker_id)
+          @instance.connectivity_checker.ping_timer.should == nil
         end
 
         it "should log error if attempt to reconnect fails" do
@@ -592,7 +651,7 @@ describe RightScale::Sender do
           @log.should_receive(:error).with(/Failed to reconnect/, Exception, :trace).once
           flexmock(@agent).should_receive(:connect).and_raise(Exception)
           flexmock(EM::Timer).should_receive(:new).and_yield.once
-          @instance.__send__(:check_connection, @broker_id)
+          @instance.connectivity_checker.check(@broker_id)
         end
       end
     end
@@ -668,10 +727,17 @@ describe RightScale::Sender do
       flexmock(RightScale::AgentIdentity, :generate => 'token1')
     end
 
-    it "should deliver the response" do
+    it "should deliver the response for a Request" do
       @instance.send_retryable_request('/welcome/aboard', 'iZac') {|_|}
       response = RightScale::Result.new('token1', 'to', RightScale::OperationResult.success, 'target1')
-      flexmock(@instance).should_receive(:deliver).with(response, Hash).once
+      flexmock(@instance).should_receive(:deliver).with(response, RightScale::Sender::PendingRequest).once
+      @instance.handle_response(response)
+    end
+
+    it "should deliver the response for a Push" do
+      @instance.send_push('/welcome/aboard', 'iZac') {|_|}
+      response = RightScale::Result.new('token1', 'to', RightScale::OperationResult.success, 'target1')
+      flexmock(@instance).should_receive(:deliver).with(response, RightScale::Sender::PendingRequest).once
       @instance.handle_response(response)
     end
 
@@ -691,7 +757,7 @@ describe RightScale::Sender do
       non_delivery = RightScale::OperationResult.non_delivery(RightScale::OperationResult::NO_ROUTE_TO_TARGET)
       response = RightScale::Result.new('token1', 'to', non_delivery, 'target1')
       @instance.handle_response(response)
-      @instance.instance_variable_get(:@non_deliveries).total.should == 1
+      @instance.instance_variable_get(:@non_delivery_stats).total.should == 1
     end
 
     it "should log non-delivery if there is no response handler" do
@@ -724,7 +790,7 @@ describe RightScale::Sender do
       flexmock(RightScale::AgentIdentity, :generate => 'token1')
     end
 
-    it "should delete all associated pending requests" do
+    it "should delete all associated pending Request requests" do
       @instance.send_retryable_request('/welcome/aboard', 'iZac') {|_|}
       @instance.pending_requests['token1'].should_not be_nil
       response = RightScale::Result.new('token1', 'to', RightScale::OperationResult.success, 'target1')
@@ -732,11 +798,19 @@ describe RightScale::Sender do
       @instance.pending_requests['token1'].should be_nil
     end
 
+    it "should not delete any pending Push requests" do
+      @instance.send_push('/welcome/aboard', 'iZac') {|_|}
+      @instance.pending_requests['token1'].should_not be_nil
+      response = RightScale::Result.new('token1', 'to', RightScale::OperationResult.success, 'target1')
+      @instance.handle_response(response)
+      @instance.pending_requests['token1'].should_not be_nil
+    end
+
     it "should delete any associated retry requests" do
       @instance.send_retryable_request('/welcome/aboard', 'iZac') {|_|}
       @instance.pending_requests['token1'].should_not be_nil
       @instance.pending_requests['token2'] = @instance.pending_requests['token1'].dup
-      @instance.pending_requests['token2'][:retry_parent] = 'token1'
+      @instance.pending_requests['token2'].retry_parent = 'token1'
       response = RightScale::Result.new('token2', 'to', RightScale::OperationResult.success, 'target1')
       @instance.handle_response(response)
       @instance.pending_requests['token1'].should be_nil
@@ -799,52 +873,52 @@ describe RightScale::Sender do
     end
 
     it 'should vote for restart after the maximum number of queued requests is reached' do
-      @instance.instance_variable_get(:@restart_vote_count).should == 0
+      @instance.offline_handler.instance_variable_get(:@restart_vote_count).should == 0
       EM.run do
         @instance.enable_offline_mode
-        @instance.instance_variable_set(:@queue, ('*' * (RightScale::Sender::MAX_QUEUED_REQUESTS - 1)).split(//))
+        @instance.offline_handler.queue = ('*' * (RightScale::Sender::OfflineHandler::MAX_QUEUED_REQUESTS - 1)).split(//)
         @instance.send_push('/dummy', 'payload')
         EM.next_tick { EM.stop }
       end
-      @instance.instance_variable_get(:@queue).size.should == RightScale::Sender::MAX_QUEUED_REQUESTS
-      @instance.instance_variable_get(:@restart_vote_count).should == 1
+      @instance.offline_handler.queue.size.should == RightScale::Sender::OfflineHandler::MAX_QUEUED_REQUESTS
+      @instance.offline_handler.instance_variable_get(:@restart_vote_count).should == 1
     end
 
     it 'should vote for restart after the threshold delay is reached' do
-      old_vote_delay = RightScale::Sender::RESTART_VOTE_DELAY
+      old_vote_delay = RightScale::Sender::OfflineHandler::RESTART_VOTE_DELAY
       begin
-        RightScale::Sender.const_set(:RESTART_VOTE_DELAY, 0.1)
-        @instance.instance_variable_get(:@restart_vote_count).should == 0
+        RightScale::Sender::OfflineHandler.const_set(:RESTART_VOTE_DELAY, 0.1)
+        @instance.offline_handler.instance_variable_get(:@restart_vote_count).should == 0
         EM.run do
           @instance.enable_offline_mode
           @instance.send_push('/dummy', 'payload')
           EM.add_timer(0.5) { EM.stop }
         end
-        @instance.instance_variable_get(:@restart_vote_count).should == 1
+        @instance.offline_handler.instance_variable_get(:@restart_vote_count).should == 1
       ensure
-        RightScale::Sender.const_set(:RESTART_VOTE_DELAY, old_vote_delay)
+        RightScale::Sender::OfflineHandler.const_set(:RESTART_VOTE_DELAY, old_vote_delay)
       end
     end
 
     it 'should not flush queued requests until back online' do
-      old_flush_delay = RightScale::Sender::MAX_QUEUE_FLUSH_DELAY
+      old_flush_delay = RightScale::Sender::OfflineHandler::MAX_QUEUE_FLUSH_DELAY
       begin
-        RightScale::Sender.const_set(:MAX_QUEUE_FLUSH_DELAY, 0.1)
+        RightScale::Sender::OfflineHandler.const_set(:MAX_QUEUE_FLUSH_DELAY, 0.1)
         EM.run do
           @instance.enable_offline_mode
           @instance.send_push('/dummy', 'payload')
           EM.add_timer(0.5) { EM.stop }
         end
       ensure
-        RightScale::Sender.const_set(:MAX_QUEUE_FLUSH_DELAY, old_flush_delay)
+        RightScale::Sender::OfflineHandler.const_set(:MAX_QUEUE_FLUSH_DELAY, old_flush_delay)
       end
     end
 
     it 'should flush queued requests once back online' do
-      old_flush_delay = RightScale::Sender::MAX_QUEUE_FLUSH_DELAY
+      old_flush_delay = RightScale::Sender::OfflineHandler::MAX_QUEUE_FLUSH_DELAY
       @broker.should_receive(:publish).once.and_return { EM.stop }
       begin
-        RightScale::Sender.const_set(:MAX_QUEUE_FLUSH_DELAY, 0.1)
+        RightScale::Sender::OfflineHandler.const_set(:MAX_QUEUE_FLUSH_DELAY, 0.1)
         EM.run do
           @instance.enable_offline_mode
           @instance.send_push('/dummy', 'payload')
@@ -852,34 +926,31 @@ describe RightScale::Sender do
           EM.add_timer(1) { EM.stop }
         end
       ensure
-        RightScale::Sender.const_set(:MAX_QUEUE_FLUSH_DELAY, old_flush_delay)
+        RightScale::Sender::OfflineHandler.const_set(:MAX_QUEUE_FLUSH_DELAY, old_flush_delay)
       end
     end
 
     it 'should stop flushing when going back to offline mode' do
-      old_flush_delay = RightScale::Sender::MAX_QUEUE_FLUSH_DELAY
+      old_flush_delay = RightScale::Sender::OfflineHandler::MAX_QUEUE_FLUSH_DELAY
       begin
-        RightScale::Sender.const_set(:MAX_QUEUE_FLUSH_DELAY, 0.1)
+        RightScale::Sender::OfflineHandler.const_set(:MAX_QUEUE_FLUSH_DELAY, 0.1)
         EM.run do
           @instance.enable_offline_mode
           @instance.send_push('/dummy', 'payload')
           @instance.disable_offline_mode
-          @instance.instance_variable_get(:@flushing_queue).should be_true
-          @instance.instance_variable_get(:@stop_flushing_queue).should be_false
-          @instance.instance_variable_get(:@queueing_mode).should == :offline
+          @instance.offline_handler.state.should == :flushing
+          @instance.offline_handler.mode.should == :offline
           @instance.enable_offline_mode
-          @instance.instance_variable_get(:@flushing_queue).should be_true
-          @instance.instance_variable_get(:@stop_flushing_queue).should be_true
-          @instance.instance_variable_get(:@queueing_mode).should == :offline
+          @instance.offline_handler.state.should == :running
+          @instance.offline_handler.mode.should == :offline
           EM.add_timer(1) do
-            @instance.instance_variable_get(:@flushing_queue).should be_false
-            @instance.instance_variable_get(:@stop_flushing_queue).should be_false
-            @instance.instance_variable_get(:@queueing_mode).should == :offline
+            @instance.offline_handler.state.should == :running
+            @instance.offline_handler.mode.should == :offline
             EM.stop
           end
         end
       ensure
-        RightScale::Sender.const_set(:MAX_QUEUE_FLUSH_DELAY, old_flush_delay)
+        RightScale::Sender::OfflineHandler.const_set(:MAX_QUEUE_FLUSH_DELAY, old_flush_delay)
       end
     end
   end