right_agent 2.0.8 → 2.1.0

data/spec/clients/router_client_spec.rb

@@ -32,8 +32,8 @@ describe RightScale::RouterClient do
     @log = flexmock(RightScale::Log)
     @log.should_receive(:error).by_default.and_return { |m| raise RightScale::Log.format(*m) }
     @log.should_receive(:warning).by_default.and_return { |m| raise RightScale::Log.format(*m) }
-    @timer = flexmock("timer", :cancel => true, :interval= => 0).by_default
-    flexmock(EM::PeriodicTimer).should_receive(:new).and_return(@timer).by_default
+    @timer = flexmock("timer", :cancel => true).by_default
+    flexmock(EM::Timer).should_receive(:new).and_return(@timer).by_default
     @http_client = flexmock("http client", :get => true, :check_health => true).by_default
     flexmock(RightScale::BalancedHttpClient).should_receive(:new).and_return(@http_client).by_default
     @websocket = WebSocketClientMock.new
@@ -94,13 +94,13 @@ describe RightScale::RouterClient do
       it "makes post request to router" do
         flexmock(@client).should_receive(:make_request).with(:post, "/push", @params, @action, @token).
             and_return(nil).once
-        @client.push(@type, @payload, @target, @token).should be_nil
+        @client.push(@type, @payload, @target, @token).should be nil
       end
 
       it "does not require token" do
         flexmock(@client).should_receive(:make_request).with(:post, "/push", @params, @action, nil).
             and_return(nil).once
-        @client.push(@type, @payload, @target).should be_nil
+        @client.push(@type, @payload, @target).should be nil
       end
     end
 
@@ -114,7 +114,7 @@ describe RightScale::RouterClient do
       it "does not require token" do
         flexmock(@client).should_receive(:make_request).with(:post, "/request", @params, @action, nil).
             and_return(nil).once
-        @client.request(@type, @payload, @target).should be_nil
+        @client.request(@type, @payload, @target).should be nil
       end
     end
   end
@@ -122,8 +122,9 @@ describe RightScale::RouterClient do
   context "events" do
 
     before(:each) do
+      @handler = lambda { |_| }
       @later = Time.at(@now = Time.now)
-      @tick = 30
+      @tick = 1
       flexmock(Time).should_receive(:now).and_return { @later += @tick }
     end
 
@@ -137,14 +138,14 @@ describe RightScale::RouterClient do
 
       it "sends using websocket if available" do
         @client.send(:connect, @routing_keys) { |_| }
-        @client.notify(@event, @routing_keys).should be_true
+        @client.notify(@event, @routing_keys).should be true
         @websocket.sent.should == JSON.dump(@params)
       end
 
       it "makes post request by default" do
         flexmock(@client).should_receive(:make_request).with(:post, "/notify", @params, "notify", "uuid",
                                                              {:filter_params => ["event"]}).once
-        @client.notify(@event, @routing_keys).should be_true
+        @client.notify(@event, @routing_keys).should be true
       end
     end
 
@@ -153,164 +154,323 @@ describe RightScale::RouterClient do
         lambda { @client.listen(@routing_keys) }.should raise_error(ArgumentError, "Block missing")
       end
 
-      it "loops forever until closed" do
+      it "initializes listen state and starts loop" do
+        flexmock(@client).should_receive(:listen_loop).with(@routing_keys, Proc).and_return(true).once
+        @client.listen(@routing_keys, &@handler).should be true
+        @client.instance_variable_get(:@listen_state).should == :choose
+      end
+    end
+
+    context :close do
+      it "stops listening" do
+        @client.instance_variable_set(:@listen_timer, @timer)
+        @timer.should_receive(:cancel).once
         @client.close
-        @client.listen(@routing_keys) { |_| }.should be_true
+        @client.instance_variable_get(:@listen_timer).should be nil
+        @client.instance_variable_get(:@listen_state).should == :cancel
       end
 
-      it "loops forever until closing" do
-        @client.close(:receive)
-        @client.listen(@routing_keys) { |_| }.should be_true
+      it "closes websocket" do
+        @client.send(:connect, nil, &@handler)
+        @client.close
+        @websocket.closed.should be true
+        @websocket.code.should == 1001
       end
+    end
 
-      it "sleeps if websocket already exists" do
-        @client.send(:connect, @routing_keys) { |_| }
-        flexmock(@client).should_receive(:sleep).with(5).and_return { @client.close }.once
-        @client.listen(@routing_keys) { |_| }
-        @client.instance_variable_get(:@connect_interval).should == 30
+    context :update_listen_state do
+      it "cancels timer if state is :cancel" do
+        @client.instance_variable_set(:@listen_timer, @timer)
+        @client.send(:update_listen_state, :cancel).should be true
+        @client.instance_variable_get(:@listen_state).should == :cancel
+        @client.instance_variable_get(:@listen_timer).should be nil
       end
 
-      it "tries to create websocket if time to" do
-        @client.instance_variable_get(:@websocket).should be_nil
-        flexmock(Faye::WebSocket::Client).should_receive(:new).and_return(@websocket).once
-        flexmock(@client).should_receive(:sleep).and_return { @client.close }
-        @client.listen(@routing_keys) { |_| }
+      it "can handle a re-cancel" do
+        @client.instance_variable_set(:@listen_timer, @timer)
+        @client.send(:update_listen_state, :cancel).should be true
+        @client.send(:update_listen_state, :cancel).should be true
+        @client.instance_variable_get(:@listen_timer).should be nil
       end
 
-      it "does not try long-polling if websocket connect attempt indicates not to" do
-        @client.instance_variable_get(:@websocket).should be_nil
-        flexmock(@client).should_receive(:try_connect).and_return { @client.close; true }
-        flexmock(@client).should_receive(:try_long_poll).never
-        @client.listen(@routing_keys) { |_| }
+      [:choose, :check, :connect, :long_poll, :wait].each do |state|
+        it "sets state and timer interval for valid state #{state}" do
+          @client.send(:update_listen_state, state, 10).should be true
+          @client.instance_variable_get(:@listen_state).should == state
+          @client.instance_variable_get(:@listen_interval).should == 10
+        end
       end
 
-      it "tries long-polling if could not create websocket" do
-        @client.instance_variable_get(:@websocket).should be_nil
-        flexmock(@client).should_receive(:retry_connect?).and_return(false)
-        flexmock(@client).should_receive(:long_poll).and_return { @client.close; ["uuid"] }.once
-        @client.listen(@routing_keys) { |_| }.should be_true
+      it "rejects invalid states" do
+        lambda { @client.send(:update_listen_state, :bogus) }.should raise_error(ArgumentError)
       end
-    end
 
-    context :close do
-      it "closes websocket" do
-        @client.send(:connect, nil) { |_| }
-        @client.close
-        @websocket.closed.should be_true
-        @websocket.code.should == 1001
+      context "and state set to :check" do
+        it "initializes check count" do
+          @client.instance_variable_get(:@listen_checks).should be nil
+          @client.send(:update_listen_state, :check).should be true
+          @client.instance_variable_get(:@listen_checks).should == 0
+        end
+
+        it "only records start of long-polling when state changes" do
+          @client.send(:update_listen_state, :check).should be true
+          @client.instance_variable_get(:@listen_checks).should == 0
+          @client.instance_variable_set(:@listen_checks, nil)
+          @client.send(:update_listen_state, :check)
+          @client.instance_variable_get(:@listen_checks).should be nil
+        end
       end
     end
 
-    context :retry_connect? do
-      before(:each) do
-        @client.instance_variable_set(:@last_connect_time, @now)
+    context :listen_loop do
+      def when_in_listen_state(state, checks = nil, failures = 0)
+        flexmock(EM).should_receive(:next_tick).by_default
+        flexmock(EM::Timer).should_receive(:new).and_return(@timer).by_default
+        @client.send(:update_listen_state, state)
+        @client.instance_variable_set(:@listen_checks, checks) if checks
+        @client.instance_variable_set(:@listen_failures, failures)
         @client.instance_variable_set(:@connect_interval, 30)
+        @client.instance_variable_set(:@reconnect_interval, 2)
+        state
       end
 
-      it "requires websocket to be enabled" do
-        @client = RightScale::RouterClient.new(@auth_client, :long_polling_only => true)
-        @client.send(:retry_connect?).should be_false
+      context "in :choose state" do
+        it "chooses listen method" do
+          when_in_listen_state(:choose)
+          @client.send(:listen_loop, @routing_keys, &@handler)
+          @client.instance_variable_get(:@listen_state).should == :connect
+        end
       end
 
-      it "requires there be no existing websocket connection" do
-        @client.instance_variable_set(:@websocket, @websocket)
-        @client.send(:retry_connect?).should be_false
+      context "in :check state" do
+        context "and not connected" do
+          before(:each) do
+            when_in_listen_state(:check)
+            @client.instance_variable_set(:@websocket, nil)
+          end
+
+          it "sets state to :connect if router not responding" do
+            flexmock(@client).should_receive(:router_not_responding?).and_return(true).once
+            @client.send(:listen_loop, @routing_keys, &@handler)
+            @client.instance_variable_get(:@listen_state).should == :connect
+            @client.instance_variable_get(:@listen_interval).should == 4
+          end
+
+          it "otherwise backs off connect interval and sets state to :long_poll" do
+            flexmock(@client).should_receive(:router_not_responding?).and_return(false).once
+            @client.send(:listen_loop, @routing_keys, &@handler)
+            @client.instance_variable_get(:@connect_interval).should == 60
+            @client.instance_variable_get(:@listen_state).should == :long_poll
+            @client.instance_variable_get(:@listen_interval).should == 0
+          end
+        end
+
+        context "and connected" do
+          before(:each) do
+            @client.instance_variable_set(:@websocket, @websocket)
+          end
+
+          it "sets state to :choose if have checked enough" do
+            when_in_listen_state(:check, 5)
+            @client.send(:listen_loop, @routing_keys, &@handler)
+            @client.instance_variable_get(:@listen_state).should == :choose
+            @client.instance_variable_get(:@listen_interval).should == 30
+          end
+
+          it "otherwise stays in same state" do
+            when_in_listen_state(:check, 4)
+            @client.send(:listen_loop, @routing_keys, &@handler)
+            @client.instance_variable_get(:@listen_state).should == :check
+          end
+        end
+      end
+
+      context "in :connect state" do
+       it "tries to connect" do
+          when_in_listen_state(:connect)
+          flexmock(@client).should_receive(:try_connect).with(@routing_keys, Proc).once
+          @client.send(:listen_loop, @routing_keys, &@handler)
+        end
+      end
+
+      context "in :long_poll state" do
+        it "tries long-polling if non-blocking enabled" do
+          @client = RightScale::RouterClient.new(@auth_client, :non_blocking => true)
+          when_in_listen_state(:long_poll)
+          event_uuids = ["uuids"]
+          flexmock(@client).should_receive(:try_long_poll).with(@routing_keys, nil, Proc).and_return(event_uuids).once
+          @client.send(:listen_loop, @routing_keys, &@handler)
+          @client.instance_variable_get(:@event_uuids).should == event_uuids
+        end
+
+        it "otherwise tries deferred long-polling and sets state to :wait" do
+          when_in_listen_state(:long_poll)
+          event_uuids = ["uuids"]
+          @client.instance_variable_set(:@event_uuids, event_uuids)
+          flexmock(@client).should_receive(:try_deferred_long_poll).with(@routing_keys, event_uuids, Proc).once
+          @client.send(:listen_loop, @routing_keys, &@handler)
+          @client.instance_variable_get(:@listen_state).should == :wait
+          @client.instance_variable_get(:@listen_interval).should == 1
+        end
+      end
+
+      context "in :wait state" do
+        it "does nothing" do
+          when_in_listen_state(:wait)
+          @client.send(:listen_loop, @routing_keys, &@handler)
+          @client.instance_variable_get(:@listen_state).should == :wait
+        end
+      end
+
+      context "in :cancel state" do
+        it "returns false" do
+          when_in_listen_state(:cancel)
+          @client.send(:listen_loop, @routing_keys, &@handler).should be false
+          @client.instance_variable_get(:@listen_state).should == :cancel
+        end
       end
 
-      context "when no existing websocket" do
+      context "when unexpected exception" do
         before(:each) do
-          @client.instance_variable_get(:@websocket).should be_nil
+          when_in_listen_state(:connect)
+          flexmock(@client).should_receive(:try_connect).and_raise(RuntimeError).once
         end
 
-        it "allows retry if enough time has elapsed" do
-          @tick = 1
-          @client.instance_variable_set(:@last_connect_time, @now - 29)
-          @client.send(:retry_connect?).should be_false
-          @client.instance_variable_set(:@last_connect_time, @now - 30)
-          @client.send(:retry_connect?).should be_true
+        it "logs error" do
+          @log.should_receive(:error).with("Failed to listen", RuntimeError, :trace).once
+          @client.send(:listen_loop, @routing_keys, &@handler).should be true
         end
 
-        [RightScale::RouterClient::NORMAL_CLOSE, RightScale::RouterClient::SHUTDOWN_CLOSE].each do |code|
-          it "allows retry if previous close code is #{code}" do
-            @client.instance_variable_set(:@close_code, code)
-            @client.instance_variable_set(:@connect_interval, 300)
-            @client.send(:retry_connect?).should be_true
-          end
+        it "resets state to :choose" do
+          @log.should_receive(:error)
+          @client.send(:listen_loop, @routing_keys, &@handler).should be true
+          @client.instance_variable_get(:@listen_state).should == :choose
         end
 
-        [502, 503].each do |code|
-          it "allows retry if previous close has reason with code #{code} indicating router inaccessible" do
-            @client.instance_variable_set(:@close_code, RightScale::RouterClient::PROTOCOL_ERROR_CLOSE)
-            @client.instance_variable_set(:@close_reason, "Unexpected response code: #{code}")
-            @client.instance_variable_set(:@connect_interval, 300)
-            @client.send(:retry_connect?).should be_true
-          end
+        it "fails if exceeded repeated failure limit" do
+          when_in_listen_state(:connect, 1, 10)
+          @log.should_receive(:error).with("Failed to listen", RuntimeError, :trace).once.ordered
+          @log.should_receive(:error).with("Exceeded maximum repeated listen failures (10), stopping listening").once.ordered
+          @client.send(:listen_loop, @routing_keys, &@handler).should be false
+          @client.instance_variable_get(:@listen_state).should == :cancel
+          @client.state.should == :failed
         end
       end
+
+      it "uses next_tick for next loop if interval is 0" do
+        when_in_listen_state(:choose)
+        @client.instance_variable_get(:@listen_interval).should == 0
+        flexmock(EM).should_receive(:next_tick).and_yield.once
+        flexmock(EM::Timer).should_receive(:new).with(1, Proc).and_return(@timer).once
+        @client.send(:listen_loop, @routing_keys, &@handler).should be true
+        @client.instance_variable_get(:@listen_state).should == :check
+      end
+
+      it "otherwise uses timer for next loop" do
+        when_in_listen_state(:long_poll)
+        flexmock(@client).should_receive(:try_deferred_long_poll).once
+        flexmock(EM::Timer).should_receive(:new).with(1, Proc).and_return(@timer).once
+        flexmock(EM).should_receive(:next_tick).never
+        @client.send(:listen_loop, @routing_keys, &@handler).should be true
+        @client.instance_variable_get(:@listen_state).should == :wait
+      end
     end
 
-    context :try_connect do
+    context :choose_listen_method do
       before(:each) do
-        @handler = lambda { |_| }
-        @client.instance_variable_get(:@websocket).should be_nil
         @client.instance_variable_set(:@connect_interval, 30)
-        @client.instance_variable_set(:@reconnect_interval, 2)
       end
 
-      it "makes websocket connect request" do
-        flexmock(Faye::WebSocket::Client).should_receive(:new).and_return(@websocket).once
-        flexmock(@client).should_receive(:sleep).and_return { @client.close }
-        @client.send(:try_connect, @routing_keys, &@handler)
+      it "chooses long-polling if only it is enabled" do
+        @client = RightScale::RouterClient.new(@auth_client, :long_polling_only => true)
+        @client.send(:choose_listen_method).should be true
+        @client.instance_variable_get(:@listen_state).should == :long_poll
+        @client.instance_variable_get(:@listen_interval).should == 0
+        @client.instance_variable_get(:@connect_interval).should == 60 * 60 * 24
       end
 
-      it "periodically checks whether websocket creation was really successful" do
-        flexmock(Faye::WebSocket::Client).should_receive(:new).and_return(@websocket)
-        flexmock(@client).should_receive(:sleep).with(1).times(3).ordered
-        flexmock(@client).should_receive(:sleep).and_return { @client.instance_variable_set(:@websocket, nil) }.once.ordered
-        @client.send(:try_connect, @routing_keys, &@handler)
+      it "chooses to delay choice if already connected" do
+        @client.instance_variable_set(:@websocket, @websocket)
+        @client.send(:choose_listen_method).should be true
+        @client.instance_variable_get(:@listen_state).should == :choose
+        @client.instance_variable_get(:@listen_interval).should == 30
       end
 
-      it "sleeps if websocket creation failed because router not responding" do
-        flexmock(Faye::WebSocket::Client).should_receive(:new).and_return(@websocket)
-        flexmock(@client).should_receive(:sleep).with(1).and_return do
-          @client.instance_variable_set(:@websocket, nil)
-          @client.instance_variable_set(:@close_code, RightScale::RouterClient::PROTOCOL_ERROR_CLOSE)
-          @client.instance_variable_set(:@close_reason, "Unexpected response code: 502")
-        end.once
-        flexmock(@client).should_receive(:sleep).with(4).and_return { @client.close }.once
-        @client.send(:try_connect, @routing_keys, &@handler)
-        @client.instance_variable_get(:@reconnect_interval).should == 4
-        @client.instance_variable_get(:@connect_interval).should == 30
-      end
+      context "when not connected" do
+        before(:each) do
+          @client.instance_variable_get(:@websocket).should be nil
+        end
 
-      it "adjusts connect interval if websocket creation was unsuccessful" do
-        flexmock(Faye::WebSocket::Client).should_receive(:new).and_return(@websocket)
-        flexmock(@client).should_receive(:sleep).and_return { @client.instance_variable_set(:@websocket, nil); @client.close }.once
-        @client.send(:try_connect, @routing_keys, &@handler)
-        @client.instance_variable_get(:@connect_interval).should == 60
-        @client.instance_variable_get(:@reconnect_interval).should == 2
+        it "chooses to connect if never connected" do
+          @client.send(:choose_listen_method).should be true
+          @client.instance_variable_get(:@listen_state).should == :connect
+          @client.instance_variable_get(:@listen_interval).should == 0
+        end
+
+        context "but previously attempted" do
+          before(:each) do
+            @client.instance_variable_set(:@attempted_connect_at, @now)
+          end
+
+          it "chooses to connect immediately if enough time has elapsed" do
+            @client.instance_variable_set(:@attempted_connect_at, @now - 30)
+            @client.send(:choose_listen_method).should be true
+            @client.instance_variable_get(:@listen_state).should == :connect
+            @client.instance_variable_get(:@listen_interval).should == 0
+          end
+
+          [RightScale::RouterClient::NORMAL_CLOSE, RightScale::RouterClient::SHUTDOWN_CLOSE].each do |code|
+            it "chooses to connect immediately if previous close code is #{code}" do
+              @client.instance_variable_set(:@close_code, code)
+              @client.instance_variable_set(:@connect_interval, 300)
+              @client.send(:choose_listen_method).should be true
+              @client.instance_variable_get(:@listen_state).should == :connect
+              @client.instance_variable_get(:@listen_interval).should == 0
+            end
+          end
+
+          [502, 503].each do |code|
+            it "chooses to connect immediately if previous close code #{code} indicates router not responding" do
+              @client.instance_variable_set(:@close_code, RightScale::RouterClient::PROTOCOL_ERROR_CLOSE)
+              @client.instance_variable_set(:@close_reason, "Unexpected response code: #{code}")
+              @client.instance_variable_set(:@connect_interval, 300)
+              @client.send(:choose_listen_method).should be true
+              @client.instance_variable_get(:@listen_state).should == :connect
+              @client.instance_variable_get(:@listen_interval).should == 0
+            end
+          end
+
+          it "otherwise it chooses to connect as soon as connect interval expires" do
+            @client.instance_variable_set(:@attempted_connect_at, @now - 28)
+            @client.send(:choose_listen_method).should be true
+            @client.instance_variable_get(:@listen_state).should == :connect
+            @client.instance_variable_get(:@listen_interval).should == 1
+          end
+        end
       end
+    end
 
-      it "loops instead of long-polling if websocket creation was unsuccessful" do
-        flexmock(Faye::WebSocket::Client).should_receive(:new).and_return(@websocket)
-        flexmock(@client).should_receive(:sleep).and_return { @client.instance_variable_set(:@websocket, nil); @client.close }
-        flexmock(@client).should_receive(:long_poll).never
-        @client.send(:try_connect, @routing_keys, &@handler)
+    context :try_connect do
+      before(:each) do
+        @client.instance_variable_get(:@websocket).should be nil
+        @client.instance_variable_set(:@connect_interval, 30)
+        @client.instance_variable_set(:@reconnect_interval, 2)
       end
 
-      it "sleeps after successfully creating websocket" do
-        flexmock(Faye::WebSocket::Client).should_receive(:new).and_return(@websocket)
-        flexmock(@client).should_receive(:sleep).with(1).times(4).ordered
-        flexmock(@client).should_receive(:sleep).with(1).and_return { @client.close }.once.ordered
+      it "makes websocket connect request and sets state to :check" do
+        flexmock(Faye::WebSocket::Client).should_receive(:new).and_return(@websocket).once
         @client.send(:try_connect, @routing_keys, &@handler)
+        @client.instance_variable_get(:@listen_state).should == :check
+        @client.instance_variable_get(:@listen_interval).should == 1
       end
 
-      it "adjusts connect interval if websocket creation fails" do
-        @log.should_receive(:error).with("Failed creating WebSocket", StandardError).once
-        flexmock(Faye::WebSocket::Client).should_receive(:new).and_raise(StandardError).once
-        flexmock(@client).should_receive(:sleep).never
+      it "adjusts connect interval if websocket creation fails and sets state to :long_poll" do
+        @log.should_receive(:error).with("Failed creating WebSocket", RuntimeError).once
+        flexmock(Faye::WebSocket::Client).should_receive(:new).and_raise(RuntimeError).once
         @client.send(:try_connect, @routing_keys, &@handler)
         @client.instance_variable_get(:@connect_interval).should == 60
+        @client.instance_variable_get(:@listen_state).should == :long_poll
+        @client.instance_variable_get(:@listen_interval).should == 0
       end
     end
 
@@ -322,7 +482,7 @@ describe RightScale::RouterClient do
       context "when creating connection" do
         it "connects to router" do
           flexmock(Faye::WebSocket::Client).should_receive(:new).with(@ws_url + "/connect", nil, Hash).and_return(@websocket).once
-          @client.send(:connect, @routing_keys) { |_| }
+          @client.send(:connect, @routing_keys, &@handler)
         end
 
         it "chooses scheme based on scheme in router URL" do
@@ -331,29 +491,29 @@ describe RightScale::RouterClient do
           @auth_client = AuthClientMock.new(@url, @auth_header, :authorized)
           @client = RightScale::RouterClient.new(@auth_client, @options)
           flexmock(Faye::WebSocket::Client).should_receive(:new).with(@ws_url + "/connect", nil, Hash).and_return(@websocket).once
-          @client.send(:connect, @routing_keys) { |_| }
+          @client.send(:connect, @routing_keys, &@handler)
         end
 
         it "uses headers containing only API version and authorization" do
           headers = @auth_header.merge("X-API-Version" => "2.0")
           flexmock(Faye::WebSocket::Client).should_receive(:new).with(String, nil, hsh(:headers => headers)).and_return(@websocket).once
-          @client.send(:connect, @routing_keys) { |_| }
+          @client.send(:connect, @routing_keys, &@handler)
         end
 
         it "enables ping" do
           flexmock(Faye::WebSocket::Client).should_receive(:new).with(String, nil, hsh(:ping => 60)).and_return(@websocket).once
-          @client.send(:connect, @routing_keys) { |_| }
+          @client.send(:connect, @routing_keys, &@handler)
         end
 
         it "adds routing keys as query parameters" do
           url = @ws_url + "/connect" + "?routing_keys[]=a%3Ab%3Dc"
           flexmock(Faye::WebSocket::Client).should_receive(:new).with(url, nil, Hash).and_return(@websocket).once
-          @client.send(:connect, ["a:b=c"]) { |_| }
+          @client.send(:connect, ["a:b=c"], &@handler)
         end
 
         it "returns websocket" do
           flexmock(Faye::WebSocket::Client).should_receive(:new).and_return(@websocket).once
-          @client.send(:connect, @routing_keys) { |_| }.should == @websocket
+          @client.send(:connect, @routing_keys, &@handler).should == @websocket
           @client.instance_variable_get(:@websocket).should == @websocket
         end
       end
@@ -374,7 +534,6 @@ describe RightScale::RouterClient do
         it "logs event" do
           @log.should_receive(:info).with("Creating WebSocket connection to ws://test.com/connect").once.ordered
           @log.should_receive(:info).with("Received EVENT <uuid> Push /foo/bar from rs-agent-1-1").once.ordered
-          @log.should_receive(:info).with("Sending EVENT <uuid> Push /foo/bar to rs-agent-1-1").once.ordered
           event = nil
           @client.send(:connect, @routing_keys) { |e| event = e }
           @websocket.onmessage(@json_event)
@@ -387,19 +546,6 @@ describe RightScale::RouterClient do
           @websocket.sent.should == @json_ack
         end
 
-        it "sends event response using websocket" do
-          result = {:uuid => "uuid2", :type => "Result", :from => "rs-agent-2-2", :data => {}, :version => @version}
-          @client.send(:connect, @routing_keys) { |_| result }
-          @websocket.onmessage(@json_event)
-          @websocket.sent.should == [@json_ack, JSON.dump({:event => result, :routing_keys => ["rs-agent-1-1"]})]
-        end
-
-        it "only sends non-nil responses" do
-          @client.send(:connect, @routing_keys) { |_| nil }
-          @websocket.onmessage(@json_event)
-          @websocket.sent.should == @json_ack
-        end
-
         it "logs failures" do
           @log.should_receive(:error).with("Failed handling WebSocket event", StandardError, :trace).once
           request = RightScale::Request.new("/foo/bar", "payload")
@@ -412,15 +558,15 @@ describe RightScale::RouterClient do
         it "logs info" do
           @log.should_receive(:info).with("Creating WebSocket connection to ws://test.com/connect").once.ordered
           @log.should_receive(:info).with("WebSocket closed (1000)").once.ordered
-          @client.send(:connect, @routing_keys) { |_| }
+          @client.send(:connect, @routing_keys, &@handler)
           @websocket.onclose(1000)
-          @client.instance_variable_get(:@websocket).should be_nil
+          @client.instance_variable_get(:@websocket).should be nil
         end
 
         it "logged info includes reason if available" do
           @log.should_receive(:info).with("Creating WebSocket connection to ws://test.com/connect").once.ordered
           @log.should_receive(:info).with("WebSocket closed (1001: Going Away)").once.ordered
-          @client.send(:connect, @routing_keys) { |_| }
+          @client.send(:connect, @routing_keys, &@handler)
           @websocket.onclose(1001, "Going Away")
         end
 
@@ -428,7 +574,7 @@ describe RightScale::RouterClient do
           @log.should_receive(:info).with("Creating WebSocket connection to ws://test.com/connect").once.ordered
           @log.should_receive(:info).and_raise(RuntimeError).once.ordered
           @log.should_receive(:error).with("Failed closing WebSocket", RuntimeError, :trace).once
-          @client.send(:connect, @routing_keys) { |_| }
+          @client.send(:connect, @routing_keys, &@handler)
           @websocket.onclose(1000)
         end
       end
@@ -436,13 +582,13 @@ describe RightScale::RouterClient do
       context "on error" do
         it "logs error" do
           @log.should_receive(:error).with("WebSocket error (Protocol Error)")
-          @client.send(:connect, @routing_keys) { |_| }
+          @client.send(:connect, @routing_keys, &@handler)
           @websocket.onerror("Protocol Error")
         end
 
         it "does not log if there is no error data" do
           @log.should_receive(:error).never
-          @client.send(:connect, @routing_keys) { |_| }
+          @client.send(:connect, @routing_keys, &@handler)
           @websocket.onerror(nil)
         end
       end
@@ -450,37 +596,70 @@ describe RightScale::RouterClient do
 
     context :try_long_poll do
       before(:each) do
-        @uuids = ["uuid"]
-        @handler = lambda { |_| }
+        @event_uuids = ["uuid"]
         @client.instance_variable_set(:@connect_interval, 30)
         @client.instance_variable_set(:@reconnect_interval, 2)
       end
 
       it "makes long-polling request" do
-        flexmock(@client).should_receive(:long_poll).with(@routing_keys, @uuids, @handler).and_return([]).once
-        @client.send(:try_long_poll, @routing_keys, @uuids, &@handler).should == []
+        flexmock(@client).should_receive(:long_poll).with(@routing_keys, @event_uuids, @handler).and_return([]).once
+        @client.send(:try_long_poll, @routing_keys, @event_uuids, &@handler).should == []
       end
 
       it "returns UUIDs of events received" do
-        flexmock(@client).should_receive(:long_poll).with(@routing_keys, [], @handler).and_return { @uuids }.once
-        @client.send(:try_long_poll, @routing_keys, [], &@handler).should == @uuids
+        flexmock(@client).should_receive(:long_poll).with(@routing_keys, [], @handler).and_return { @event_uuids }.once
+        @client.send(:try_long_poll, @routing_keys, [], &@handler).should == @event_uuids
+      end
+
+      it "returns exception if there is a long-polling failure" do
+        flexmock(@client).should_receive(:long_poll).and_raise(RuntimeError).once
+        @client.send(:try_long_poll, @routing_keys, @event_uuids, &@handler).should be_a RuntimeError
       end
+    end
 
-      it "sleeps if there is a long-polling failure" do
-        @log.should_receive(:error).with("Failed long-polling", StandardError, :trace).once
-        flexmock(@client).should_receive(:long_poll).and_raise(StandardError).once
-        flexmock(@client).should_receive(:sleep).with(4).once
-        @client.send(:try_long_poll, @routing_keys, @uuids, &@handler).should be_nil
+    context :try_deferred_long_poll do
+      before(:each) do
+        @event_uuids = ["uuid"]
+        @client.instance_variable_set(:@connect_interval, 30)
+        @client.instance_variable_set(:@reconnect_interval, 2)
+        @client.send(:update_listen_state, :long_poll)
+        flexmock(EM).should_receive(:defer).by_default
       end
 
-      [RightScale::Exceptions::Unauthorized,
-       RightScale::Exceptions::ConnectivityFailure,
-       RightScale::Exceptions::RetryableError].each do |e|
-        it "does not trace #{e} exceptions" do
-          @log.should_receive(:error).with("Failed long-polling", e, :no_trace).once
-          flexmock(@client).should_receive(:long_poll).and_raise(e, "failed").once
-          flexmock(@client).should_receive(:sleep).with(4).once
-          @client.send(:try_long_poll, @routing_keys, @uuids, &@handler).should be_nil
+      it "makes long-polling request using defer thread" do
+        flexmock(EM).should_receive(:defer).with(Proc, Proc).once
+        @client.send(:try_deferred_long_poll, @routing_keys, @event_uuids, &@handler).should be true
+      end
+
+      context "defer_operation_proc" do
+        it "long-polls" do
+          @client.instance_variable_set(:@connect_interval, 1)
+          @client.send(:try_deferred_long_poll, @routing_keys, @event_uuids, &@handler)
+          @defer_operation_proc = @client.instance_variable_get(:@defer_operation_proc)
+          flexmock(@client).should_receive(:long_poll).with(@routing_keys, @event_uuids, @handler).and_return([]).once
+          @defer_operation_proc.call.should == []
+        end
+      end
+
+      context "defer_callback_proc" do
+        before(:each) do
+          @client.instance_variable_set(:@connect_interval, 1)
+          @client.send(:try_deferred_long_poll, @routing_keys, @event_uuids, &@handler)
+          @defer_callback_proc = @client.instance_variable_get(:@defer_callback_proc)
+        end
+
+        it "handles UUIDs of events received" do
+          @defer_callback_proc.call(@event_uuids).should == @event_uuids
+          @client.instance_variable_get(:@event_uuids).should == @event_uuids
+          @client.instance_variable_get(:@listen_state).should == :choose
+          @client.instance_variable_get(:@listen_interval).should == 0
+        end
+
+        it "handles exception if there is a long-polling failure" do
+          @log.should_receive(:error).with("Failed long-polling", RuntimeError, :trace).once
+          @defer_callback_proc.call(RuntimeError.new).should be nil
+          @client.instance_variable_get(:@listen_state).should == :choose
+          @client.instance_variable_get(:@listen_interval).should == 4
         end
       end
     end
@@ -495,22 +674,23 @@ describe RightScale::RouterClient do
       end
 
       it "makes listen request to router" do
-        flexmock(@client).should_receive(:make_request).with(:get, "/listen",
+        flexmock(@client).should_receive(:make_request).with(:poll, "/listen",
             on { |a| a[:wait_time].should == 55 && !a.key?(:routing_keys) &&
             a[:timestamp] == @later.to_f }, "listen", nil, Hash).and_return([@event]).once
-        @client.send(:long_poll, @routing_keys, @ack) { |_| }
+        @client.send(:long_poll, @routing_keys, @ack, &@handler)
       end
 
-      it "uses listen timeout for request" do
-        flexmock(@client).should_receive(:make_request).with(:get, "/listen", Hash, "listen", nil,
-            {:request_timeout => 60, :log_level => :debug}).and_return([@event]).once
-        @client.send(:long_poll, @routing_keys, @ack) { |_| }
+      it "uses listen timeout for request poll timeout and connect interval for request timeout" do
+        @client.instance_variable_set(:@connect_interval, 300)
+        flexmock(@client).should_receive(:make_request).with(:poll, "/listen", Hash, "listen", nil,
+            {:poll_timeout => 60, :request_timeout => 300, :log_level => :debug}).and_return([@event]).once
+        @client.send(:long_poll, @routing_keys, @ack, &@handler)
       end
 
       it "logs event" do
         @log.should_receive(:info).with("Received EVENT <uuid> Push /foo/bar from rs-agent-1-1").once
         flexmock(@client).should_receive(:make_request).and_return([@event])
-        @client.send(:long_poll, @routing_keys, @ack) { |_| }
+        @client.send(:long_poll, @routing_keys, @ack, &@handler)
       end
 
       it "presents event to handler" do
@@ -533,18 +713,57 @@ describe RightScale::RouterClient do
         flexmock(@client).should_receive(:make_request).and_return(nil)
         event = nil
         @client.send(:long_poll, @routing_keys, @ack) { |e| event = e }
-        event.should be_nil
+        event.should be nil
       end
 
       it "returns event UUIDs" do
         flexmock(@client).should_receive(:make_request).and_return([@event])
-        @client.send(:long_poll, @routing_keys, @ack) { |_| }.should == ["uuid"]
+        @client.send(:long_poll, @routing_keys, @ack, &@handler).should == ["uuid"]
        end
 
       it "returns nil if no events are received" do
         flexmock(@client).should_receive(:make_request).and_return(nil)
-        @client.send(:long_poll, @routing_keys, @ack) { |_| }.should be_nil
-       end
+        @client.send(:long_poll, @routing_keys, @ack, &@handler).should be nil
+      end
+    end
+
+    context :process_long_poll do
+      before(:each) do
+        @event_uuids = ["uuid"]
+        @client.instance_variable_set(:@connect_interval, 30)
+        @client.instance_variable_set(:@reconnect_interval, 2)
+        @client.send(:update_listen_state, :long_poll)
+      end
+
+      [RightScale::Exceptions::Unauthorized.new("error"),
+       RightScale::Exceptions::ConnectivityFailure.new("error"),
+       RightScale::Exceptions::RetryableError.new("error"),
+       RightScale::Exceptions::InternalServerError.new("error", "server")].each do |e|
+        it "does not trace #{e} exceptions but sets state to :choose" do
+          @log.should_receive(:error).with("Failed long-polling", e, :no_trace).once
+          @client.send(:process_long_poll, e).should be nil
+          @client.instance_variable_get(:@listen_state).should == :choose
+          @client.instance_variable_get(:@listen_interval).should == 4
+        end
+      end
+
+      it "traces unexpected exceptions and sets state to :choose" do
+        e = RuntimeError.new
+        @log.should_receive(:error).with("Failed long-polling", e, :trace).once
+        @client.send(:process_long_poll, e).should be nil
+        @client.instance_variable_get(:@listen_state).should == :choose
+        @client.instance_variable_get(:@listen_interval).should == 4
+      end
+
+      context "when no exception" do
+        it "sets state to :choose" do
+          @client.instance_variable_set(:@reconnect_interval, 2)
+          @client.send(:process_long_poll, @event_uuids).should == @event_uuids
+          @client.instance_variable_get(:@listen_state).should == :choose
+          @client.instance_variable_get(:@reconnect_interval).should == 2
+          @client.instance_variable_get(:@listen_interval).should == 0
+        end
+      end
     end
 
     context :backoff_connect_interval do
@@ -587,7 +806,7 @@ describe RightScale::RouterClient do
       it "does not declare not responding for other close codes" do
         @client.instance_variable_set(:@close_code, RightScale::RouterClient::UNEXPECTED_ERROR_CLOSE)
         @client.instance_variable_set(:@close_reason, "Unexpected response code: 502")
-        @client.send(:router_not_responding?).should be_false
+        @client.send(:router_not_responding?).should be false
       end
     end
   end