right_agent 2.1.5-x86-mingw32 → 2.2.0-x86-mingw32

data/spec/clients/blocking_client_spec.rb

@@ -203,6 +203,12 @@ describe RightScale::BlockingClient do
         result.should == [@result, 200, @body, @headers]
       end
 
+      it "stores the connection" do
+        @http_client.should_receive(:get).with(@host + @path + @query, @request_options).and_return(@response).once
+        @client.request(:get, @path, @host, @connect_options, @request_options)
+        @client.connections[@path].should == {:host => @host, :path => @path, :expires_at => @later + 5}
+      end
+
       it "returns nil if response is nil" do
         @http_client.should_receive(:get).with(@host + @path + @query, @request_options).and_return(nil).once
         result = @client.request(:get, @path, @host, @connect_options, @request_options)
@@ -261,5 +267,15 @@ describe RightScale::BlockingClient do
         result.should == [nil, nil, nil, nil]
       end
     end
+
+    context :close do
+      it "deletes all persistent connections" do
+        @http_client.should_receive(:get).with(@host + @path + @query, @request_options).and_return(@response).once
+        @client.request(:get, @path, @host, @connect_options, @request_options)
+        @client.connections.should_not be_empty
+        @client.close("terminating").should be true
+        @client.connections.should be_empty
+      end
+    end
   end
 end

data/spec/clients/non_blocking_client_spec.rb

@@ -264,15 +264,15 @@ describe RightScale::NonBlockingClient do
             should raise_error(RightScale::HttpExceptions::InternalServerError)
       end
 
-      it "converts Errno::ETIMEDOUT error to 504" do
-        @headers.http_status = 504
+      it "converts Errno::ETIMEDOUT error to 408 RequestTimeout" do
+        @headers.http_status = 408
         @response.should_receive(:errback).and_yield.once
         @response.should_receive(:error).and_return(Errno::ETIMEDOUT)
-        @fiber.should_receive(:resume).with(504, "Errno::ETIMEDOUT").once
-        flexmock(Fiber).should_receive(:yield).and_return([504, "Errno::ETIMEDOUT"]).once
+        @fiber.should_receive(:resume).with(408, "Request timeout").once
+        flexmock(Fiber).should_receive(:yield).and_return([408, "Request timeout"]).once
         flexmock(EM::HttpRequest).should_receive(:new).with(@host, @connect_options).and_return(@request).once
         lambda { @client.request(:get, @path, @host, @connect_options, @request_options) }.
-            should raise_error(RightScale::HttpExceptions::GatewayTimeout)
+            should raise_error(RightScale::HttpExceptions::RequestTimeout)
       end
 
       it "stores the connection if keepalive enabled" do
@@ -351,7 +351,7 @@ describe RightScale::NonBlockingClient do
         end
 
         it "stops polling if there is an error" do
-          @response.should_receive(:error).and_return("some error").times(3)
+          @response.should_receive(:error).and_return("some error").once
           @response.should_receive(:errback).and_yield.once
           @fiber.should_receive(:resume).with(500, "some error").once
           @client.send(:poll_again, @fiber, @request, @request_options, @stop_at).should be true
@@ -374,6 +374,43 @@ describe RightScale::NonBlockingClient do
         end
       end
     end
+
+    context :close do
+      it "closes all persistent connections" do
+        @request_options.merge!(:keepalive => true)
+        flexmock(EM::HttpRequest).should_receive(:new).and_return(@request).once
+        @client.request(:get, @path, @host, @connect_options, @request_options)
+        @request.should_receive(:close).with("terminating").once
+        @client.connections.should_not be_empty
+        @client.close("terminating").should be true
+        @client.connections.should be_empty
+      end
+    end
+  end
+
+  context :handle_error do
+    before(:each) do
+      @options = {}
+      @client = RightScale::NonBlockingClient.new(@options)
+    end
+
+    ["terminating", "reconnecting"].each do |error|
+      it "converts #{error} error to a 200 OK" do
+        @client.send(:handle_error, :get, error).should == [200, nil]
+      end
+    end
+
+    it "converts Errno::ETIMEDOUT to 408 RequestTimeout" do
+      @client.send(:handle_error, :get, Errno::ETIMEDOUT).should == [408, "Request timeout"]
+    end
+
+    it "converts error to 500 InternalServerError by default" do
+      @client.send(:handle_error, :get, "failed").should == [500, "failed"]
+    end
+
+    it "generates error message for 500 InternalServerError if none specified" do
+      @client.send(:handle_error, :get, nil).should == [500, "HTTP connection failure for GET"]
+    end
   end
 
   context :beautify_headers do

data/spec/clients/router_client_spec.rb

@@ -34,7 +34,7 @@ describe RightScale::RouterClient do
     @log.should_receive(:warning).by_default.and_return { |m| raise RightScale::Log.format(*m) }
     @timer = flexmock("timer", :cancel => true).by_default
     flexmock(EM::Timer).should_receive(:new).and_return(@timer).by_default
-    @http_client = flexmock("http client", :get => true, :check_health => true).by_default
+    @http_client = flexmock("http client", :get => true, :check_health => true, :close => true).by_default
     flexmock(RightScale::BalancedHttpClient).should_receive(:new).and_return(@http_client).by_default
     @websocket = WebSocketClientMock.new
     flexmock(Faye::WebSocket::Client).should_receive(:new).and_return(@websocket).by_default
@@ -83,44 +83,41 @@ describe RightScale::RouterClient do
       @action = "bar"
       @payload = {:some => "data"}
       @target = "rs-agent-2-2"
-      @token = "random token"
       @params = {
         :type => @type,
         :payload => @payload,
         :target => @target }
     end
 
-    context :push do
-      it "makes post request to router" do
-        flexmock(@client).should_receive(:make_request).with(:post, "/push", @params, @action, @token).
-            and_return(nil).once
-        @client.push(@type, @payload, @target, @token).should be nil
-      end
-
-      it "does not require token" do
-        flexmock(@client).should_receive(:make_request).with(:post, "/push", @params, @action, nil).
-            and_return(nil).once
-        @client.push(@type, @payload, @target).should be nil
-      end
-    end
-
-    context :request do
-      it "makes post request to router" do
-        flexmock(@client).should_receive(:make_request).with(:post, "/request", @params, @action, @token).
-            and_return(nil).once
-        @client.request(@type, @payload, @target, @token)
-      end
+    [:push, :request].each do |kind|
+      context kind do
+        it "makes post request to router" do
+          flexmock(@client).should_receive(:make_request).with(:post, "/#{kind}", @params, @action, {}).and_return(nil).once
+          @client.send(kind, @type, @payload, @target).should be nil
+        end
 
-      it "does not require token" do
-        flexmock(@client).should_receive(:make_request).with(:post, "/request", @params, @action, nil).
-            and_return(nil).once
-        @client.request(@type, @payload, @target).should be nil
+        it "applies request options" do
+          options = {:request_uuid => "uuid", :time_to_live => 60}
+          flexmock(@client).should_receive(:make_request).with(:post, "/#{kind}", @params, @action, options).and_return(nil).once
+          @client.send(kind, @type, @payload, @target, options).should be nil
+        end
       end
     end
   end
 
   context "events" do
 
+    def when_in_listen_state(state, checks = nil, failures = 0)
+      flexmock(EM).should_receive(:next_tick).by_default
+      flexmock(EM::Timer).should_receive(:new).and_return(@timer).by_default
+      @client.send(:update_listen_state, state)
+      @client.instance_variable_set(:@listen_checks, checks) if checks
+      @client.instance_variable_set(:@listen_failures, failures)
+      @client.instance_variable_set(:@connect_interval, 30)
+      @client.instance_variable_set(:@reconnect_interval, 2)
+      state
+    end
+
     before(:each) do
       @handler = lambda { |_| }
       @later = Time.at(@now = Time.now)
@@ -143,8 +140,8 @@ describe RightScale::RouterClient do
       end
 
       it "makes post request by default" do
-        flexmock(@client).should_receive(:make_request).with(:post, "/notify", @params, "notify", "uuid",
-                                                             {:filter_params => ["event"]}).once
+        flexmock(@client).should_receive(:make_request).with(:post, "/notify", @params, "notify",
+                                                             {:request_uuid => "uuid", :filter_params => ["event"]}).once
         @client.notify(@event, @routing_keys).should be true
       end
     end
@@ -223,17 +220,6 @@ describe RightScale::RouterClient do
     end
 
     context :listen_loop do
-      def when_in_listen_state(state, checks = nil, failures = 0)
-        flexmock(EM).should_receive(:next_tick).by_default
-        flexmock(EM::Timer).should_receive(:new).and_return(@timer).by_default
-        @client.send(:update_listen_state, state)
-        @client.instance_variable_set(:@listen_checks, checks) if checks
-        @client.instance_variable_set(:@listen_failures, failures)
-        @client.instance_variable_set(:@connect_interval, 30)
-        @client.instance_variable_set(:@reconnect_interval, 2)
-        state
-      end
-
       context "in :choose state" do
         it "chooses listen method" do
           when_in_listen_state(:choose)
@@ -357,22 +343,41 @@ describe RightScale::RouterClient do
         end
       end
 
+      it "waits required amount of time before looping" do
+        when_in_listen_state(:choose)
+        flexmock(@client).should_receive(:listen_loop_wait).with(@later + 1, 0, @routing_keys, @handler).and_return(true).once
+        @client.send(:listen_loop, @routing_keys, &@handler).should be true
+      end
+    end
+
+    context :listen_loop_wait do
       it "uses next_tick for next loop if interval is 0" do
         when_in_listen_state(:choose)
         @client.instance_variable_get(:@listen_interval).should == 0
         flexmock(EM).should_receive(:next_tick).and_yield.once
-        flexmock(EM::Timer).should_receive(:new).with(1, Proc).and_return(@timer).once
-        @client.send(:listen_loop, @routing_keys, &@handler).should be true
-        @client.instance_variable_get(:@listen_state).should == :check
+        flexmock(EM::Timer).should_receive(:new).never
+        flexmock(@client).should_receive(:listen_loop).with(@routing_keys, @handler).once
+        @client.send(:listen_loop_wait, 0, @later, @routing_keys, &@handler).should be true
       end
 
       it "otherwise uses timer for next loop" do
         when_in_listen_state(:long_poll)
-        flexmock(@client).should_receive(:try_deferred_long_poll).once
-        flexmock(EM::Timer).should_receive(:new).with(1, Proc).and_return(@timer).once
+        @client.instance_variable_set(:@listen_interval, 9)
+        flexmock(EM::Timer).should_receive(:new).with(9, Proc).and_return(@timer).and_yield.once
         flexmock(EM).should_receive(:next_tick).never
-        @client.send(:listen_loop, @routing_keys, &@handler).should be true
-        @client.instance_variable_get(:@listen_state).should == :wait
+        flexmock(@client).should_receive(:listen_loop).with(@routing_keys, @handler).once
+        @client.send(:listen_loop_wait, @later - 9, 9, @routing_keys, &@handler).should be true
+      end
+
+      it "waits some more if interval has changed while waiting" do
+        when_in_listen_state(:long_poll)
+        @client.instance_variable_set(:@listen_interval, 3)
+        flexmock(EM::Timer).should_receive(:new).with(1, Proc).and_return(@timer).and_yield.once.ordered
+        flexmock(EM::Timer).should_receive(:new).with(2, Proc).and_return(@timer).and_yield.once.ordered
+        flexmock(EM::Timer).should_receive(:new).with(1, Proc).and_return(@timer).and_yield.once.ordered
+        flexmock(EM).should_receive(:next_tick).never
+        flexmock(@client).should_receive(:listen_loop).with(@routing_keys, @handler).once
+        @client.send(:listen_loop_wait, @later, 1, @routing_keys, &@handler).should be true
       end
     end
 
@@ -429,7 +434,7 @@ describe RightScale::RouterClient do
             end
           end
 
-          [502, 503].each do |code|
+          [408, 502, 503].each do |code|
             it "chooses to connect immediately if previous close code #{code} indicates router not responding" do
               @client.instance_variable_set(:@close_code, RightScale::RouterClient::PROTOCOL_ERROR_CLOSE)
               @client.instance_variable_set(:@close_reason, "Unexpected response code: #{code}")
@@ -676,13 +681,13 @@ describe RightScale::RouterClient do
       it "makes listen request to router" do
         flexmock(@client).should_receive(:make_request).with(:poll, "/listen",
             on { |a| a[:wait_time].should == 55 && !a.key?(:routing_keys) &&
-            a[:timestamp] == @later.to_f }, "listen", nil, Hash).and_return([@event]).once
+            a[:timestamp] == @later.to_f }, "listen", Hash).and_return([@event]).once
         @client.send(:long_poll, @routing_keys, @ack, &@handler)
       end
 
       it "uses listen timeout for request poll timeout and connect interval for request timeout" do
         @client.instance_variable_set(:@connect_interval, 300)
-        flexmock(@client).should_receive(:make_request).with(:poll, "/listen", Hash, "listen", nil,
+        flexmock(@client).should_receive(:make_request).with(:poll, "/listen", Hash, "listen",
             {:poll_timeout => 60, :request_timeout => 300}).and_return([@event]).once
         @client.send(:long_poll, @routing_keys, @ack, &@handler)
       end

data/spec/http_exceptions_spec.rb

@@ -101,6 +101,13 @@ describe RightScale::HttpExceptions do
       bad_request = RestClient::Exceptions::EXCEPTIONS_MAP[400].new(nil, 400)
       RightScale::HttpExceptions.convert(bad_request).should be_a RightScale::HttpExceptions::BadRequest
     end
+
+    it "assigns status code 408 to RequestTimeout" do
+      request_timeout = RestClient::Exceptions::EXCEPTIONS_MAP[408].new
+      exception = RightScale::HttpExceptions.convert(request_timeout)
+      exception.should be_a RightScale::HttpExceptions::RequestTimeout
+      exception.http_code.should == 408
+    end
   end
 
 end # RightScale::HttpExceptions

data/spec/offline_handler_spec.rb

@@ -192,14 +192,18 @@ describe RightScale::OfflineHandler do
       @type = "/foo/bar"
       @payload = {:pay => "load"}
       @target = "target"
+      @token = "token"
+      @now = Time.now
+      flexmock(Time).should_receive(:now).and_return(@now)
+      @expires_at = @now + 25
       @callback = lambda { |_| }
     end
 
     it "queues request at head of queue if still initializing" do
       @handler.init
-      @handler.queue_request(@kind, @type, @payload, "target1", @callback).should be_true
+      @handler.queue_request(@kind, @type, @payload, "target1", "token1", @expires_at, &@callback).should be_true
       @handler.queue.size.should == 1
-      @handler.queue_request(@kind, @type, @payload, "target2", @callback).should be_true
+      @handler.queue_request(@kind, @type, @payload, "target2", "token2", @expires_at, &@callback).should be_true
       @handler.queue.size.should == 2
       @handler.queue.first[:target] == "target2"
     end
@@ -207,9 +211,9 @@ describe RightScale::OfflineHandler do
     it "queues request at end of queue if no longer initializing" do
       @handler.init
       @handler.start
-      @handler.queue_request(@kind, @type, @payload, "target1", @callback)
+      @handler.queue_request(@kind, @type, @payload, "target1", "token1", @expires_at, &@callback)
       @handler.queue.size.should == 1
-      @handler.queue_request(@kind, @type, @payload, "target2", @callback)
+      @handler.queue_request(@kind, @type, @payload, "target2", "token2", @expires_at, &@callback)
       @handler.queue.size.should == 2
       @handler.queue.first[:target] == "target1"
     end
@@ -219,7 +223,7 @@ describe RightScale::OfflineHandler do
       @handler.start
       flexmock(@handler).should_receive(:vote_to_restart).once
       RightScale::OfflineHandler::MAX_QUEUED_REQUESTS.times do |i|
-        @handler.queue_request(@kind, @type, @payload, @target, @callback)
+        @handler.queue_request(@kind, @type, @payload, @target, @token, @expires_at, &@callback)
       end
     end
   end
@@ -241,6 +245,10 @@ describe RightScale::OfflineHandler do
       @type = "/foo/bar"
       @payload = {:pay => "load"}
       @target = "target"
+      @token = "token"
+      @now = Time.now
+      flexmock(Time).should_receive(:now).and_return(@now)
+      @expires_at = (@now + 25).to_i
       @result = nil
       @callback = lambda { |result| @result = result }
     end
@@ -251,16 +259,19 @@ describe RightScale::OfflineHandler do
         @handler.start
         flexmock(@handler).should_receive(:start_timer)
         @handler.enable
-        @handler.queue_request(:send_push, @type, @payload, @target, nil)
-        @handler.queue_request(:send_request, @type, @payload, @target, @callback)
-        @handler.queue.size.should == 2
-        flexmock(EM).should_receive(:next_tick).and_yield.once
-        @sender.should_receive(:send_push).with(@type, @payload, @target).once.ordered
-        @sender.should_receive(:send_request).with(@type, @payload, @target, Proc).and_yield("result").once.ordered
+        @handler.queue_request(:send_push, @type, @payload, @target, @token, 0)
+        @handler.queue_request(:send_request, @type, @payload, @target, @token, @expires_at, &@callback)
+        @handler.queue_request(:send_request, @type, @payload, @target, @token, @now.to_i, &@callback)
+        @handler.queue.size.should == 3
+        flexmock(EM).should_receive(:next_tick).and_yield.twice
+        @sender.should_receive(:send_push).with(@type, @payload, @target, {:token => @token}).once.ordered
+        @sender.should_receive(:send_request).
+            with(@type, @payload, @target, {:token => @token, :time_to_live => 25}, Proc).and_yield("result").once.ordered
         flexmock(EM).should_receive(:add_timer).and_yield.once
         log = flexmock(RightScale::Log)
         log.should_receive(:info).with(/Connection to RightNet re-established/).once.ordered
         log.should_receive(:info).with(/Starting to flush request queue/).once.ordered
+        log.should_receive(:info).with(/Dropping queued request/).once.ordered
         log.should_receive(:info).with(/Request queue flushed/).once.ordered
         @handler.disable.should be_true
       end

data/spec/retryable_request_spec.rb

@@ -35,6 +35,7 @@ describe RightScale::RetryableRequest do
       RightScale.module_eval('OldSender = Sender')
     end
     RightScale.module_eval('Sender = SenderMock')
+    @options = {:time_to_live => RightScale::RetryableRequest::DEFAULT_TIMEOUT}
   end
 
   after(:all) do
@@ -48,7 +49,7 @@ describe RightScale::RetryableRequest do
     context 'when :targets => nil' do
       it 'should send target-less requests' do
         request = RightScale::RetryableRequest.new('type', 'payload')
-        flexmock(RightScale::Sender.instance).should_receive(:send_request).with('type', 'payload', nil, Proc).
+        flexmock(RightScale::Sender.instance).should_receive(:send_request).with('type', 'payload', nil, @options, Proc).
             and_yield(RightScale::OperationResult.non_delivery('test')).once
         flexmock(EM).should_receive(:add_timer).with(RightScale::RetryableRequest::DEFAULT_TIMEOUT, Proc).once
         flexmock(EM).should_receive(:add_timer).with(RightScale::RetryableRequest::DEFAULT_RETRY_DELAY, Proc).once
@@ -59,7 +60,7 @@ describe RightScale::RetryableRequest do
     context 'when one target is specified' do
       it 'should send a targeted request' do
         request = RightScale::RetryableRequest.new('type', 'payload', :targets => ["rs-agent-1-1"])
-        flexmock(RightScale::Sender.instance).should_receive(:send_request).with('type', 'payload', {:agent_id => "rs-agent-1-1"}, Proc).
+        flexmock(RightScale::Sender.instance).should_receive(:send_request).with('type', 'payload', {:agent_id => "rs-agent-1-1"}, @options, Proc).
             and_yield(RightScale::OperationResult.non_delivery('test')).once
         flexmock(EM).should_receive(:add_timer).with(RightScale::RetryableRequest::DEFAULT_TIMEOUT, Proc).once
         flexmock(EM).should_receive(:add_timer).with(RightScale::RetryableRequest::DEFAULT_RETRY_DELAY, Proc).once
@@ -70,10 +71,12 @@ describe RightScale::RetryableRequest do
     context 'when many targets are specified' do
       it 'should choose a random target' do
         request = RightScale::RetryableRequest.new('type', 'payload', :targets => ["rs-agent-1-1", "rs-agent-2-2", "rs-agent-3-3"])
-        flexmock(RightScale::Sender.instance).should_receive(:send_request).and_return do |type, payload, target, block|
+        flexmock(RightScale::Sender.instance).should_receive(:send_request).and_return do |type, payload, target, options, block|
           type.should == 'type'
           payload.should == 'payload'
           ["rs-agent-1-1", "rs-agent-2-2", "rs-agent-3-3"].should include(target[:agent_id])
+          options[:token].should == nil
+          options[:time_to_live].should == @options[:time_to_live]
           block.call(RightScale::OperationResult.non_delivery('test'))
         end
         flexmock(EM).should_receive(:add_timer).with(RightScale::RetryableRequest::DEFAULT_TIMEOUT, Proc).once
@@ -89,7 +92,7 @@ describe RightScale::RetryableRequest do
     context 'when not specified' do
       it 'should fail if receives error response' do
         request = RightScale::RetryableRequest.new('type', 'payload')
-        flexmock(RightScale::Sender.instance).should_receive(:send_request).with('type', 'payload', nil, Proc).
+        flexmock(RightScale::Sender.instance).should_receive(:send_request).with('type', 'payload', nil, @options, Proc).
             and_yield(RightScale::OperationResult.error('test')).once
         flexmock(request).should_receive(:fail).once
         flexmock(EM).should_receive(:add_timer).with(RightScale::RetryableRequest::DEFAULT_TIMEOUT, Proc).once
@@ -100,7 +103,7 @@ describe RightScale::RetryableRequest do
     context 'when specified as true' do
       it 'should retry if receives error response' do
         request = RightScale::RetryableRequest.new('type', 'payload', :retry_on_error => true)
-        flexmock(RightScale::Sender.instance).should_receive(:send_request).with('type', 'payload', nil, Proc).
+        flexmock(RightScale::Sender.instance).should_receive(:send_request).with('type', 'payload', nil, @options, Proc).
             and_yield(RightScale::OperationResult.error('test')).once
         flexmock(EM).should_receive(:add_timer).with(RightScale::RetryableRequest::DEFAULT_TIMEOUT, Proc).once
         flexmock(EM).should_receive(:add_timer).with(RightScale::RetryableRequest::DEFAULT_RETRY_DELAY, Proc).once
@@ -109,7 +112,7 @@ describe RightScale::RetryableRequest do
 
       it 'should ignore duplicate responses' do
         request = RightScale::RetryableRequest.new('type', 'payload', :retry_on_error => true)
-        flexmock(RightScale::Sender.instance).should_receive(:send_request).and_return do |t, p, tgt, b|
+        flexmock(RightScale::Sender.instance).should_receive(:send_request).and_return do |t, p, tgt, opts, b|
           5.times { b.call(RightScale::OperationResult.success('test')) }
         end
         flexmock(request).should_receive(:fail).never
@@ -122,7 +125,7 @@ describe RightScale::RetryableRequest do
       it 'should never retry after cancel response' do
         request = RightScale::RetryableRequest.new('type', 'payload', :retry_on_error => true)
         flexmock(RightScale::Log).should_receive(:info).with("Request type canceled (enough already)").once
-        flexmock(RightScale::Sender.instance).should_receive(:send_request).with('type', 'payload', nil, Proc).
+        flexmock(RightScale::Sender.instance).should_receive(:send_request).with('type', 'payload', nil, @options, Proc).
             and_yield(RightScale::OperationResult.cancel('enough already')).once
         flexmock(EM).should_receive(:add_timer).with(RightScale::RetryableRequest::DEFAULT_TIMEOUT, Proc).once
         flexmock(EM).should_receive(:add_timer).with(RightScale::RetryableRequest::DEFAULT_RETRY_DELAY, Proc).never
@@ -139,7 +142,7 @@ describe RightScale::RetryableRequest do
         request = RightScale::RetryableRequest.new('type', 'payload')
         flexmock(RightScale::Log).should_receive(:info).with(/Retrying in 5 seconds/).once
         flexmock(RightScale::Log).should_receive(:info).with("Request non-delivery (test) for type").once
-        flexmock(RightScale::Sender.instance).should_receive(:send_request).with('type', 'payload', nil, Proc).
+        flexmock(RightScale::Sender.instance).should_receive(:send_request).with('type', 'payload', nil, @options, Proc).
             and_yield(RightScale::OperationResult.non_delivery('test')).once
         flexmock(EM).should_receive(:add_timer).with(RightScale::RetryableRequest::DEFAULT_TIMEOUT, Proc).once
         flexmock(EM).should_receive(:add_timer).with(RightScale::RetryableRequest::DEFAULT_RETRY_DELAY, Proc).once
@@ -150,7 +153,7 @@ describe RightScale::RetryableRequest do
         request = RightScale::RetryableRequest.new('type', 'payload')
         flexmock(RightScale::Log).should_receive(:info).with(/Retrying in 5 seconds/).once
         flexmock(RightScale::Log).should_receive(:info).with("Request type failed (test) and should be retried").once
-        flexmock(RightScale::Sender.instance).should_receive(:send_request).with('type', 'payload', nil, Proc).
+        flexmock(RightScale::Sender.instance).should_receive(:send_request).with('type', 'payload', nil, @options, Proc).
             and_yield(RightScale::OperationResult.retry('test')).once
         flexmock(EM).should_receive(:add_timer).with(RightScale::RetryableRequest::DEFAULT_TIMEOUT, Proc).once
         flexmock(EM).should_receive(:add_timer).with(RightScale::RetryableRequest::DEFAULT_RETRY_DELAY, Proc).once
@@ -161,7 +164,7 @@ describe RightScale::RetryableRequest do
         request = RightScale::RetryableRequest.new('type', 'payload')
         flexmock(RightScale::Log).should_receive(:info).with(/Retrying in 5 seconds/).once
         flexmock(RightScale::Log).should_receive(:info).with("Request type failed (RightScale not ready) and should be retried").once
-        flexmock(RightScale::Sender.instance).should_receive(:send_request).with('type', 'payload', nil, Proc).
+        flexmock(RightScale::Sender.instance).should_receive(:send_request).with('type', 'payload', nil, @options, Proc).
             and_yield(RightScale::OperationResult.retry).once
         flexmock(EM).should_receive(:add_timer).with(RightScale::RetryableRequest::DEFAULT_TIMEOUT, Proc).once
         flexmock(EM).should_receive(:add_timer).with(RightScale::RetryableRequest::DEFAULT_RETRY_DELAY, Proc).once
@@ -170,7 +173,7 @@ describe RightScale::RetryableRequest do
 
       it 'should ignore responses that arrive post-cancel' do
         request = RightScale::RetryableRequest.new('type', 'payload')
-        flexmock(RightScale::Sender.instance).should_receive(:send_request).with('type', 'payload', nil, Proc).
+        flexmock(RightScale::Sender.instance).should_receive(:send_request).with('type', 'payload', nil, @options, Proc).
             and_yield(RightScale::OperationResult.success('test')).once
         flexmock(request).should_receive(:fail).once
         flexmock(request).should_receive(:succeed).never
@@ -183,7 +186,7 @@ describe RightScale::RetryableRequest do
       it 'should never retry after cancel response' do
         request = RightScale::RetryableRequest.new('type', 'payload')
         flexmock(RightScale::Log).should_receive(:info).with("Request type canceled (enough already)").once
-        flexmock(RightScale::Sender.instance).should_receive(:send_request).with('type', 'payload', nil, Proc).
+        flexmock(RightScale::Sender.instance).should_receive(:send_request).with('type', 'payload', nil, @options, Proc).
             and_yield(RightScale::OperationResult.cancel('enough already')).once
         flexmock(EM).should_receive(:add_timer).with(RightScale::RetryableRequest::DEFAULT_TIMEOUT, Proc).once
         flexmock(EM).should_receive(:add_timer).with(RightScale::RetryableRequest::DEFAULT_RETRY_DELAY, Proc).never
@@ -195,7 +198,7 @@ describe RightScale::RetryableRequest do
       it 'should control the initial retry delay' do
         retry_delay = 10
         request = RightScale::RetryableRequest.new('type', 'payload', :retry_delay => retry_delay)
-        flexmock(RightScale::Sender.instance).should_receive(:send_request).with('type', 'payload', nil, Proc).
+        flexmock(RightScale::Sender.instance).should_receive(:send_request).with('type', 'payload', nil, @options, Proc).
             and_yield(RightScale::OperationResult.retry('test')).once
         flexmock(EM).should_receive(:add_timer).with(RightScale::RetryableRequest::DEFAULT_TIMEOUT, Proc).once
         flexmock(EM).should_receive(:add_timer).with(retry_delay, Proc).once
@@ -206,7 +209,7 @@ describe RightScale::RetryableRequest do
       it 'should treat -1 as meaning no delay' do
         retry_delay = -1
         request = RightScale::RetryableRequest.new('type', 'payload', :retry_delay => retry_delay)
-        flexmock(RightScale::Sender.instance).should_receive(:send_request).with('type', 'payload', nil, Proc).
+        flexmock(RightScale::Sender.instance).should_receive(:send_request).with('type', 'payload', nil, @options, Proc).
             and_yield(RightScale::OperationResult.retry('test')).once
         flexmock(EM).should_receive(:add_timer).with(RightScale::RetryableRequest::DEFAULT_TIMEOUT, Proc).once
         flexmock(EM).should_receive(:add_timer).with(retry_delay, Proc).never
@@ -222,7 +225,7 @@ describe RightScale::RetryableRequest do
         backoff_factor = RightScale::RetryableRequest::RETRY_BACKOFF_FACTOR
         request = RightScale::RetryableRequest.new('type', 'payload', :retry_delay => retry_delay,
                                                     :retry_delay_count => retry_delay_count)
-        flexmock(RightScale::Sender.instance).should_receive(:send_request).with('type', 'payload', nil, Proc).
+        flexmock(RightScale::Sender.instance).should_receive(:send_request).with('type', 'payload', nil, @options, Proc).
             and_yield(RightScale::OperationResult.retry('test')).twice
         flexmock(EM).should_receive(:add_timer).with(RightScale::RetryableRequest::DEFAULT_TIMEOUT, Proc).once
         flexmock(EM).should_receive(:add_timer).with(retry_delay, Proc).and_yield.once
@@ -239,7 +242,7 @@ describe RightScale::RetryableRequest do
         request = RightScale::RetryableRequest.new('type', 'payload', :retry_delay => retry_delay,
                                                     :retry_delay_count => retry_delay_count,
                                                     :max_retry_delay => max_retry_delay)
-        flexmock(RightScale::Sender.instance).should_receive(:send_request).with('type', 'payload', nil, Proc).
+        flexmock(RightScale::Sender.instance).should_receive(:send_request).with('type', 'payload', nil, @options, Proc).
             and_yield(RightScale::OperationResult.retry('test')).times(4)
         flexmock(EM).should_receive(:add_timer).with(RightScale::RetryableRequest::DEFAULT_TIMEOUT, Proc).once
         flexmock(EM).should_receive(:add_timer).with(retry_delay, Proc).and_yield.twice
@@ -260,7 +263,7 @@ describe RightScale::RetryableRequest do
         request = RightScale::RetryableRequest.new('type', 'payload', :retry_delay => retry_delay,
                                                     :retry_delay_count => retry_delay_count,
                                                     :max_retry_delay => max_retry_delay)
-        flexmock(RightScale::Sender.instance).should_receive(:send_request).with('type', 'payload', nil, Proc).
+        flexmock(RightScale::Sender.instance).should_receive(:send_request).with('type', 'payload', nil, @options, Proc).
             and_yield(RightScale::OperationResult.retry('test')).times(3)
         flexmock(EM).should_receive(:add_timer).with(RightScale::RetryableRequest::DEFAULT_TIMEOUT, Proc).once
         flexmock(EM).should_receive(:add_timer).with(retry_delay, Proc).and_yield.once
@@ -277,7 +280,7 @@ describe RightScale::RetryableRequest do
     context 'when disable timeout' do
       it 'should not timeout' do
         request = RightScale::RetryableRequest.new('type', 'payload', :timeout => -1)
-        flexmock(RightScale::Sender.instance).should_receive(:send_request).with('type', 'payload', nil, Proc).once
+        flexmock(RightScale::Sender.instance).should_receive(:send_request).with('type', 'payload', nil, {}, Proc).once
         flexmock(EM).should_receive(:add_timer).never
         request.run
       end
@@ -287,7 +290,8 @@ describe RightScale::RetryableRequest do
       it 'should time the response' do
         timeout = 10
         request = RightScale::RetryableRequest.new('type', 'payload', :timeout => timeout)
-        flexmock(RightScale::Sender.instance).should_receive(:send_request).with('type', 'payload', nil, Proc).once
+        flexmock(RightScale::Sender.instance).should_receive(:send_request).
+            with('type', 'payload', nil, {:time_to_live => timeout}, Proc).once
         flexmock(EM).should_receive(:add_timer).with(timeout, Proc).once
         request.run
       end
@@ -296,10 +300,21 @@ describe RightScale::RetryableRequest do
         timeout = 10
         request = RightScale::RetryableRequest.new('type', 'payload', :timeout => timeout)
         flexmock(RightScale::Log).should_receive(:info).with("Request type timed out after 10 seconds").once
-        flexmock(RightScale::Sender.instance).should_receive(:send_request).with('type', 'payload', nil, Proc).once
+        flexmock(RightScale::Sender.instance).should_receive(:send_request).
+            with('type', 'payload', nil, {:time_to_live => timeout}, Proc).once
         flexmock(EM).should_receive(:add_timer).with(timeout, Proc).and_yield.once
         request.run
       end
+
+      it "should cancel request if time has expired" do
+        timeout = 10
+        now = Time.now
+        flexmock(Time).should_receive(:now).and_return(now, now + 10)
+        request = RightScale::RetryableRequest.new('type', 'payload', :timeout => timeout)
+        flexmock(request).should_receive(:fail).once
+        flexmock(RightScale::Sender.instance).should_receive(:send_request).never
+        request.run
+      end
     end
   end