right_agent 0.10.13 → 0.13.5

data/spec/agent_spec.rb

@@ -50,6 +50,9 @@ describe RightScale::Agent do
                          :non_delivery => true).by_default
       @broker.should_receive(:connection_status).and_yield(:connected)
       flexmock(RightAMQP::HABrokerClient).should_receive(:new).and_return(@broker)
+      @history = flexmock("history")
+      @history.should_receive(:update).and_return(true).by_default
+      flexmock(RightScale::History).should_receive(:new).and_return(@history)
       flexmock(RightScale::PidFile).should_receive(:new).
               and_return(flexmock("pid file", :check=>true, :write=>true, :remove=>true))
       @identity = "rs-instance-123-1"
@@ -102,9 +105,9 @@ describe RightScale::Agent do
       @agent.options[:root_dir].should == Dir.pwd
     end
 
-    it "for heartbeat is 60" do
+    it "for heartbeat is 0" do
       @agent.options.should include(:heartbeat)
-      @agent.options[:heartbeat].should == 60
+      @agent.options[:heartbeat].should == 0
     end
 
   end
@@ -139,6 +142,9 @@ describe RightScale::Agent do
                          :non_delivery => true).by_default
       @broker.should_receive(:connection_status).and_yield(:connected)
       flexmock(RightAMQP::HABrokerClient).should_receive(:new).and_return(@broker)
+      @history = flexmock("history")
+      @history.should_receive(:update).and_return(true).by_default
+      flexmock(RightScale::History).should_receive(:new).and_return(@history)
       flexmock(RightScale::PidFile).should_receive(:new).
               and_return(flexmock("pid file", :check=>true, :write=>true, :remove=>true))
       @identity = "rs-instance-123-1"
@@ -271,6 +277,12 @@ describe RightScale::Agent do
       flexmock(RightAMQP::HABrokerClient).should_receive(:new).and_return(@broker)
       flexmock(RightScale::PidFile).should_receive(:new).
               and_return(flexmock("pid file", :check=>true, :write=>true, :remove=>true))
+      @history = flexmock("history")
+      @history.should_receive(:update).and_return(true).by_default
+      @history.should_receive(:analyze_service).and_return({}).by_default
+      flexmock(RightScale::History).should_receive(:new).and_return(@history)
+      @header = flexmock("amqp header")
+      @header.should_receive(:ack).by_default
       @sender = flexmock("sender", :pending_requests => [], :request_age => nil,
                          :message_received => true, :stats => "").by_default
       @sender.should_receive(:terminate).and_return([0, 0]).by_default
@@ -330,6 +342,13 @@ describe RightScale::Agent do
         end
       end
 
+      it "should update history with start and then run when agent is for service" do
+        @history.should_receive(:update).with("start").and_return(true).ordered.once
+        flexmock(@agent).should_receive(:setup_queues).ordered.once
+        @history.should_receive(:update).with("run").and_return(true).ordered.once
+        @agent.run
+      end
+
       it "should log error if fail to connect to broker" do
         run_in_em do
           @broker.should_receive(:subscribe).with(hsh(:name => @identity), nil, hsh(:brokers => nil), Proc).
@@ -408,8 +427,8 @@ describe RightScale::Agent do
         run_in_em do
           request = RightScale::Request.new("/foo/bar", "payload")
           @broker.should_receive(:subscribe).with(hsh(:name => @identity), nil, Hash, Proc).
-                                             and_return(@broker_ids).and_yield(@broker_id, request).once
-          @dispatcher.should_receive(:dispatch).with(request).once
+                                             and_return(@broker_ids).and_yield(@broker_id, request, @header).once
+          @dispatcher.should_receive(:dispatch).with(request, @header).once
           @agent.run
         end
       end
@@ -418,8 +437,8 @@ describe RightScale::Agent do
         run_in_em do
           result = RightScale::Result.new("token", "to", "results", "from")
           @broker.should_receive(:subscribe).with(hsh(:name => @identity), nil, Hash, Proc).
-                                             and_return(@broker_ids).and_yield(@broker_id, result).once
-          @sender.should_receive(:handle_response).with(result).once
+                                             and_return(@broker_ids).and_yield(@broker_id, result, @header).once
+          @sender.should_receive(:handle_response).with(result, @header).once
           @agent.run
         end
       end
@@ -428,13 +447,34 @@ describe RightScale::Agent do
         run_in_em do
           result = RightScale::Result.new("token", "to", "results", "from")
           @broker.should_receive(:subscribe).with(hsh(:name => @identity), nil, Hash, Proc).
-                                             and_return(@broker_ids).and_yield(@broker_id, result).once
-          @sender.should_receive(:handle_response).with(result).once
+                                             and_return(@broker_ids).and_yield(@broker_id, result, @header).once
+          @sender.should_receive(:handle_response).with(result, @header).once
           @sender.should_receive(:message_received).once
           @agent.run
         end
       end
 
+      it "should ignore and ack unrecognized messages" do
+        run_in_em do
+          request = RightScale::Stats.new(nil, nil)
+          @broker.should_receive(:subscribe).with(hsh(:name => @identity), nil, Hash, Proc).
+                                             and_return(@broker_ids).and_yield(@broker_id, request, @header).once
+          @dispatcher.should_receive(:dispatch).never
+          @header.should_receive(:ack).once
+          @agent.run
+        end
+      end
+
+      it "should ignore unrecognized messages and not attempt to ack if there is no header" do
+        run_in_em do
+          request = RightScale::Stats.new(nil, nil)
+          @broker.should_receive(:subscribe).with(hsh(:name => @identity), nil, Hash, Proc).
+                                             and_return(@broker_ids).and_yield(@broker_id, request, nil).once
+          @dispatcher.should_receive(:dispatch).never
+          @agent.run
+        end
+      end
+
     end
 
     describe "Tuning heartbeat" do
@@ -571,6 +611,24 @@ describe RightScale::Agent do
 
     describe "Terminating" do
 
+      it "should log error for abnormal termination" do
+        run_in_em do
+          @agent = RightScale::Agent.new(:user => "me", :identity => @identity)
+          @broker.should_receive(:nil?).and_return(true)
+          @log.should_receive(:error).with("[stop] Terminating because just because", nil, :trace).once
+          @agent.terminate("just because")
+        end
+      end
+
+      it "should log error plus exception for abnormal termination" do
+        run_in_em do
+          @agent = RightScale::Agent.new(:user => "me", :identity => @identity)
+          @broker.should_receive(:nil?).and_return(true)
+          @log.should_receive(:error).with(/Terminating because just because/, Exception, :trace).once
+          @agent.terminate("just because", Exception.new("error"))
+        end
+      end
+
       it "should close unusable broker connections at start of termination" do
         @broker.should_receive(:unusable).and_return(["rs-broker-123-1"]).once
         @broker.should_receive(:close_one).with("rs-broker-123-1", false).once
@@ -694,6 +752,14 @@ describe RightScale::Agent do
         end
       end
 
+      it "should terminate immediately if broker not initialized" do
+        run_in_em do
+          @agent = RightScale::Agent.new(:user => "me", :identity => @identity)
+          @log.should_receive(:info).with("[stop] Terminating immediately").once
+          @agent.terminate
+        end
+      end
+
       it "should terminate immediately if called a second time but should still execute block" do
         @sender.should_receive(:terminate).and_return([1, 10]).once
         @dispatcher.should_receive(:dispatch_age).and_return(10).once
@@ -712,6 +778,39 @@ describe RightScale::Agent do
         end
       end
 
+      it "should update history with stop and graceful exit if broker not initialized" do
+        run_in_em do
+          @agent = RightScale::Agent.new(:user => "me", :identity => @identity)
+          @log.should_receive(:error).once
+          @history.should_receive(:update).with("stop").and_return(true).ordered.once
+          @history.should_receive(:update).with("graceful exit").and_return(true).ordered.once
+          @agent.terminate("just because")
+        end
+      end
+
+      it "should update history with stop but not graceful exit if called a second time to terminate immediately" do
+        run_in_em do
+          @agent = RightScale::Agent.new(:user => "me", :identity => @identity)
+          flexmock(@agent).should_receive(:load_actors).and_return(true)
+          @agent.run
+          @agent.instance_variable_set(:@terminating, true)
+          @history.should_receive(:update).with("stop").and_return(true).once
+          @history.should_receive(:update).with("graceful exit").never
+          @agent.terminate
+        end
+      end
+
+      it "should update history with stop and graceful exit if gracefully terminate" do
+        run_in_em do
+          @agent = RightScale::Agent.new(:user => "me", :identity => @identity)
+          flexmock(@agent).should_receive(:load_actors).and_return(true)
+          @agent.run
+          @history.should_receive(:update).with("stop").and_return(true).ordered.once
+          @history.should_receive(:update).with("graceful exit").and_return(true).ordered.once
+          @agent.terminate
+        end
+      end
+
     end
 
   end

data/spec/command/command_runner_spec.rb

@@ -26,7 +26,7 @@ module RightScale
 
   class CommandIOMock < CommandIO
 
-    include Singleton
+    include RightSupport::Ruby::EasySingleton
 
     def trigger_listen(payload)
       @test_callback.call(payload)
@@ -93,4 +93,15 @@ describe RightScale::CommandRunner do
     @opt.should == payload
   end
 
+  it 'should run commands using fiber pool if provided' do
+    commands = { :test => lambda { |opt, _| @opt = opt } }
+    fiber_pool = flexmock("fiber pool")
+    fiber_pool.should_receive(:spawn).and_return(true).and_yield.once
+    flexmock(RightScale::CommandIO).should_receive(:instance).and_return(RightScale::CommandIOMock.instance)
+    cmd_options = RightScale::CommandRunner.start(@socket_port, RightScale::AgentIdentity.generate, commands, fiber_pool)
+    payload = @command_payload.merge(cmd_options)
+    RightScale::CommandIOMock.instance.trigger_listen(payload)
+    @opt.should == payload
+  end
+
 end

data/spec/dispatched_cache_spec.rb

@@ -0,0 +1,142 @@
+#
+# Copyright (c) 2012 RightScale Inc
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+require File.expand_path(File.join(File.dirname(__FILE__), 'spec_helper'))
+
+describe "RightScale::DispatchedCache" do
+
+  include FlexMock::ArgumentTypes
+
+  before(:each) do
+    flexmock(RightScale::Log).should_receive(:error).by_default.and_return { |m| raise RightScale::Log.format(*m) }
+    flexmock(RightScale::Log).should_receive(:info).by_default
+    @now = Time.at(1000000)
+    flexmock(Time).should_receive(:now).and_return(@now).by_default
+    @agent_id = "rs-agent-1-1"
+    @cache = RightScale::DispatchedCache.new(@agent_id)
+    @token1 = "token1"
+    @token2 = "token2"
+    @token3 = "token3"
+  end
+
+  context "initialize" do
+
+    it "should initialize cache" do
+      @cache.instance_variable_get(:@cache).should == {}
+      @cache.instance_variable_get(:@lru).should == []
+    end
+
+    it "should initialize agent identity" do
+      @cache.instance_variable_get(:@identity).should == @agent_id
+    end
+
+  end
+
+  context "store" do
+
+    it "should store request token" do
+      @cache.store(@token1, nil)
+      @cache.instance_variable_get(:@cache)[@token1].should == @now.to_i
+      @cache.instance_variable_get(:@lru).should == [@token1]
+    end
+
+    it "should update lru list when store to existing entry" do
+      @cache.store(@token1, nil)
+      @cache.instance_variable_get(:@cache)[@token1].should == @now.to_i
+      @cache.instance_variable_get(:@lru).should == [@token1]
+      @cache.store(@token2, nil)
+      @cache.instance_variable_get(:@cache)[@token2].should == @now.to_i
+      @cache.instance_variable_get(:@lru).should == [@token1, @token2]
+      flexmock(Time).should_receive(:now).and_return(@now += 10)
+      @cache.store(@token1, nil)
+      @cache.instance_variable_get(:@cache)[@token1].should == @now.to_i
+      @cache.instance_variable_get(:@lru).should == [@token2, @token1]
+    end
+
+    it "should remove old cache entries when store new one" do
+      @cache.store(@token1, nil)
+      @cache.store(@token2, nil)
+      @cache.instance_variable_get(:@cache).keys.should =~ [@token1, @token2]
+      @cache.instance_variable_get(:@lru).should == [@token1, @token2]
+      flexmock(Time).should_receive(:now).and_return(@now += RightScale::DispatchedCache::MAX_AGE + 1)
+      @cache.store(@token3, nil)
+      @cache.instance_variable_get(:@cache).keys.should == [@token3]
+      @cache.instance_variable_get(:@lru).should == [@token3]
+    end
+
+    it "should not store anything if this is a shared queue request" do
+      @cache.store(@token1, "shared")
+      @cache.instance_variable_get(:@cache)[@token1].should be_nil
+      @cache.instance_variable_get(:@lru).should be_empty
+    end
+
+    it "should not store anything if token is nil" do
+      @cache.store(nil, nil)
+      @cache.instance_variable_get(:@cache).should be_empty
+      @cache.instance_variable_get(:@lru).should be_empty
+    end
+
+  end
+
+  context "serviced_by" do
+
+    it "should return who request was serviced by and make it the most recently used" do
+      @cache.store(@token1, nil)
+      @cache.store(@token2, nil)
+      @cache.instance_variable_get(:@lru).should == [@token1, @token2]
+      @cache.serviced_by(@token1).should == @agent_id
+      @cache.instance_variable_get(:@lru).should == [@token2, @token1]
+    end
+
+    it "should return nil if request was not previously serviced" do
+      @cache.serviced_by(@token1).should be_nil
+      @cache.store(@token1, nil)
+      @cache.serviced_by(@token1).should == @agent_id
+      @cache.serviced_by(@token2).should be_nil
+    end
+
+  end
+
+  context "stats" do
+
+    it "should return nil if cache empty" do
+      @cache.stats.should be_nil
+    end
+
+    it "should return total and max age" do
+      @cache.store(@token1, nil)
+      flexmock(Time).should_receive(:now).and_return(@now += 10)
+      @cache.store(@token2, nil)
+      @cache.stats.should == {
+        "local total" => 2,
+        "local max age" => "10 sec"
+      }
+      @cache.serviced_by(@token1)
+      @cache.stats.should == {
+        "local total" => 2,
+        "local max age" => "0 sec"
+      }
+    end
+
+  end
+
+end # RightScale::Dispatcher

data/spec/dispatcher_spec.rb

@@ -24,7 +24,8 @@ require File.expand_path(File.join(File.dirname(__FILE__), 'spec_helper'))
 
 class Foo
   include RightScale::Actor
-  expose :bar, :index, :i_kill_you
+  expose_idempotent :bar, :index, :i_kill_you
+  expose_non_idempotent :bar_non
   on_exception :handle_exception
 
   def index(payload)
@@ -39,6 +40,10 @@ class Foo
     ['hello', payload, request]
   end
 
+  def bar_non(payload)
+    @i = (@i || 0) + payload
+  end
+
   def i_kill_you(payload)
     raise RuntimeError.new('I kill you!')
   end
@@ -97,100 +102,19 @@ describe "RightScale::Dispatcher" do
     @actor = Foo.new
     @registry = RightScale::ActorRegistry.new
     @registry.register(@actor, nil)
-    @agent = flexmock("Agent", :identity => "agent", :broker => @broker, :registry => @registry, :options => {}).by_default
-    @dispatcher = RightScale::Dispatcher.new(@agent)
+    @agent_id = "rs-agent-1-1"
+    @agent = flexmock("Agent", :identity => @agent_id, :broker => @broker, :registry => @registry, :options => {}).by_default
+    @cache = RightScale::DispatchedCache.new(@agent_id)
+    @dispatcher = RightScale::Dispatcher.new(@agent, @cache)
     @dispatcher.em = EMMock
     @response_queue = RightScale::Dispatcher::RESPONSE_QUEUE
+    @header = flexmock("amqp header")
+    @header.should_receive(:ack).once.by_default
   end
 
-  describe "Dispatched cache" do
-
-    before(:each) do
-      @dispatched = RightScale::Dispatcher::Dispatched.new
-      @token1 = "token1"
-      @token2 = "token2"
-      @token3 = "token3"
-    end
-
-    context "when storing" do
-
-      it "should store request token" do
-        @dispatched.store(@token1)
-        @dispatched.instance_variable_get(:@cache)[@token1].should == @now.to_i
-        @dispatched.instance_variable_get(:@lru).should == [@token1]
-      end
-
-      it "should update lru list when store to existing entry" do
-        @dispatched.store(@token1)
-        @dispatched.instance_variable_get(:@cache)[@token1].should == @now.to_i
-        @dispatched.instance_variable_get(:@lru).should == [@token1]
-        @dispatched.store(@token2)
-        @dispatched.instance_variable_get(:@cache)[@token2].should == @now.to_i
-        @dispatched.instance_variable_get(:@lru).should == [@token1, @token2]
-        flexmock(Time).should_receive(:now).and_return(@now += 10)
-        @dispatched.store(@token1)
-        @dispatched.instance_variable_get(:@cache)[@token1].should == @now.to_i
-        @dispatched.instance_variable_get(:@lru).should == [@token2, @token1]
-      end
-
-      it "should remove old cache entries when store new one" do
-        @dispatched.store(@token1)
-        @dispatched.store(@token2)
-        @dispatched.instance_variable_get(:@cache).keys.should == [@token1, @token2]
-        @dispatched.instance_variable_get(:@lru).should == [@token1, @token2]
-        flexmock(Time).should_receive(:now).and_return(@now += RightScale::Dispatcher::Dispatched::MAX_AGE + 1)
-        @dispatched.store(@token3)
-        @dispatched.instance_variable_get(:@cache).keys.should == [@token3]
-        @dispatched.instance_variable_get(:@lru).should == [@token3]
-      end
-
-    end
-
-    context "when fetching" do
-
-      it "should fetch request and make it the most recently used" do
-        @dispatched.store(@token1)
-        @dispatched.store(@token2)
-        @dispatched.instance_variable_get(:@lru).should == [@token1, @token2]
-        @dispatched.fetch(@token1).should be_true
-        @dispatched.instance_variable_get(:@lru).should == [@token2, @token1]
-      end
-
-      it "should return false if fetch non-existent request" do
-        @dispatched.fetch(@token1).should be_false
-        @dispatched.store(@token1)
-        @dispatched.fetch(@token1).should be_true
-        @dispatched.fetch(@token2).should be_false
-      end
-
-    end
-
-    context "when retrieving stats" do
-
-      it "should return nil if cache empty" do
-        @dispatched.stats.should be_nil
-      end
-
-      it "should return total, youngest age, and oldest age" do
-        @dispatched.store(@token1)
-        flexmock(Time).should_receive(:now).and_return(@now += 10)
-        @dispatched.store(@token2)
-        stats = @dispatched.stats
-        stats["total"].should == 2
-        stats["youngest age"].should == 0
-        stats["oldest age"].should == 10
-        @dispatched.fetch(@token1)
-        stats = @dispatched.stats
-        stats["oldest age"].should == 0
-      end
-
-    end
-
-  end # Dispatched
-
   it "should dispatch a request" do
     req = RightScale::Request.new('/foo/bar', 'you', :token => 'token')
-    res = @dispatcher.dispatch(req)
+    res = @dispatcher.dispatch(req, @header)
     res.should(be_kind_of(RightScale::Result))
     res.token.should == 'token'
     res.results.should == ['hello', 'you']
@@ -198,7 +122,7 @@ describe "RightScale::Dispatcher" do
 
   it "should dispatch a request with required arity" do
     req = RightScale::Request.new('/foo/bar2', 'you', :token => 'token')
-    res = @dispatcher.dispatch(req)
+    res = @dispatcher.dispatch(req, @header)
     res.should(be_kind_of(RightScale::Result))
     res.token.should == 'token'
     res.results.should == ['hello', 'you', req]
@@ -206,7 +130,7 @@ describe "RightScale::Dispatcher" do
 
   it "should dispatch a request to the default action" do
     req = RightScale::Request.new('/foo', 'you', :token => 'token')
-    res = @dispatcher.dispatch(req)
+    res = @dispatcher.dispatch(req, @header)
     res.should(be_kind_of(RightScale::Result))
     res.token.should == req.token
     res.results.should == ['hello', 'you']
@@ -220,13 +144,13 @@ describe "RightScale::Dispatcher" do
                                                     arg.to == "rs-mapper-1-1" &&
                                                     arg.results == ['hello', 'you']},
                                           hsh(:persistent => true, :mandatory => true)).once
-    res = @dispatcher.dispatch(req)
+    @dispatcher.dispatch(req, @header)
   end
 
   it "should handle custom prefixes" do
     @registry.register(Foo.new, 'umbongo')
     req = RightScale::Request.new('/umbongo/bar', 'you')
-    res = @dispatcher.dispatch(req)
+    res = @dispatcher.dispatch(req, @header)
     res.should(be_kind_of(RightScale::Result))
     res.token.should == req.token
     res.results.should == ['hello', 'you']
@@ -236,7 +160,9 @@ describe "RightScale::Dispatcher" do
     flexmock(RightScale::Log).should_receive(:error).once
     req = RightScale::Request.new('/foo/i_kill_you', nil)
     flexmock(@actor).should_receive(:handle_exception).with(:i_kill_you, req, Exception).once
-    @dispatcher.dispatch(req)
+    res = @dispatcher.dispatch(req, @header)
+    res.results.error?.should be_true
+    (res.results.content =~ /Could not handle \/foo\/i_kill_you request/).should be_true
   end
 
   it "should call on_exception Procs defined in a subclass with the correct arguments" do
@@ -244,7 +170,7 @@ describe "RightScale::Dispatcher" do
     actor = Bar.new
     @registry.register(actor, nil)
     req = RightScale::Request.new('/bar/i_kill_you', nil)
-    @dispatcher.dispatch(req)
+    @dispatcher.dispatch(req, @header)
     called_with = actor.instance_variable_get("@called_with")
     called_with[0].should == :i_kill_you
     called_with[1].should == req
@@ -257,25 +183,25 @@ describe "RightScale::Dispatcher" do
     actor = Bar.new
     @registry.register(actor, nil)
     req = RightScale::Request.new('/bar/i_kill_you', nil)
-    @dispatcher.dispatch(req)
+    @dispatcher.dispatch(req, @header)
     actor.instance_variable_get("@scope").should == actor
   end
 
   it "should log error if something goes wrong" do
     RightScale::Log.should_receive(:error).once
     req = RightScale::Request.new('/foo/i_kill_you', nil)
-    @dispatcher.dispatch(req)
+    @dispatcher.dispatch(req, @header)
   end
 
   it "should reject requests whose time-to-live has expired" do
     flexmock(Time).should_receive(:now).and_return(Time.at(1000000)).by_default
     flexmock(RightScale::Log).should_receive(:info).once.with(on {|arg| arg =~ /REJECT EXPIRED.*TTL 2 sec ago/})
     @broker.should_receive(:publish).never
-    @dispatcher = RightScale::Dispatcher.new(@agent)
+    @dispatcher = RightScale::Dispatcher.new(@agent, @cache)
     @dispatcher.em = EMMock
     req = RightScale::Push.new('/foo/bar', 'you', :expires_at => @now.to_i + 8)
     flexmock(Time).should_receive(:now).and_return(@now += 10)
-    @dispatcher.dispatch(req).should be_nil
+    @dispatcher.dispatch(req, @header).should be_nil
   end
 
   it "should send non-delivery result if Request is rejected because its time-to-live has expired" do
@@ -287,11 +213,11 @@ describe "RightScale::Dispatcher" do
                                                     arg.results.non_delivery? &&
                                                     arg.results.content == RightScale::OperationResult::TTL_EXPIRATION},
                                           hsh(:persistent => true, :mandatory => true)).once
-    @dispatcher = RightScale::Dispatcher.new(@agent)
+    @dispatcher = RightScale::Dispatcher.new(@agent, @cache)
     @dispatcher.em = EMMock
     req = RightScale::Request.new('/foo/bar', 'you', {:reply_to => @response_queue, :expires_at => @now.to_i + 8})
     flexmock(Time).should_receive(:now).and_return(@now += 10)
-    @dispatcher.dispatch(req).should be_nil
+    @dispatcher.dispatch(req, @header).should be_nil
   end
 
   it "should send error result instead of non-delivery if agent does not know about non-delivery" do
@@ -303,30 +229,30 @@ describe "RightScale::Dispatcher" do
                                                     arg.results.error? &&
                                                     arg.results.content =~ /Could not deliver/},
                                           hsh(:persistent => true, :mandatory => true)).once
-    @dispatcher = RightScale::Dispatcher.new(@agent)
+    @dispatcher = RightScale::Dispatcher.new(@agent, @cache)
     @dispatcher.em = EMMock
     req = RightScale::Request.new('/foo/bar', 'you', {:reply_to => "rs-mapper-1-1", :expires_at => @now.to_i + 8}, [12, 13])
     flexmock(Time).should_receive(:now).and_return(@now += 10)
-    @dispatcher.dispatch(req).should be_nil
+    @dispatcher.dispatch(req, @header).should be_nil
   end
 
   it "should not reject requests whose time-to-live has not expired" do
     flexmock(Time).should_receive(:now).and_return(Time.at(1000000)).by_default
-    @dispatcher = RightScale::Dispatcher.new(@agent)
+    @dispatcher = RightScale::Dispatcher.new(@agent, @cache)
     @dispatcher.em = EMMock
     req = RightScale::Request.new('/foo/bar', 'you', :expires_at => @now.to_i + 11)
     flexmock(Time).should_receive(:now).and_return(@now += 10)
-    res = @dispatcher.dispatch(req)
+    res = @dispatcher.dispatch(req, @header)
     res.should(be_kind_of(RightScale::Result))
     res.token.should == req.token
     res.results.should == ['hello', 'you']
   end
 
   it "should not check age of requests with time-to-live check disabled" do
-    @dispatcher = RightScale::Dispatcher.new(@agent)
+    @dispatcher = RightScale::Dispatcher.new(@agent, @cache)
     @dispatcher.em = EMMock
     req = RightScale::Request.new('/foo/bar', 'you', :expires_at => 0)
-    res = @dispatcher.dispatch(req)
+    res = @dispatcher.dispatch(req, @header)
     res.should(be_kind_of(RightScale::Result))
     res.token.should == req.token
     res.results.should == ['hello', 'you']
@@ -335,12 +261,11 @@ describe "RightScale::Dispatcher" do
   it "should reject duplicate requests" do
     flexmock(RightScale::Log).should_receive(:info).once.with(on {|arg| arg =~ /REJECT DUP/})
     EM.run do
-      @agent.should_receive(:options).and_return(:dup_check => true)
-      @dispatcher = RightScale::Dispatcher.new(@agent)
+      @dispatcher = RightScale::Dispatcher.new(@agent, @cache)
       @dispatcher.em = EMMock
-      req = RightScale::Request.new('/foo/bar', 'you', :token => "try")
-      @dispatcher.instance_variable_get(:@dispatched).store(req.token)
-      @dispatcher.dispatch(req).should be_nil
+      req = RightScale::Request.new('/foo/bar_non', 1, :token => "try")
+      @cache.store(req.token, nil)
+      @dispatcher.dispatch(req, @header).should be_nil
       EM.stop
     end
   end
@@ -348,57 +273,113 @@ describe "RightScale::Dispatcher" do
   it "should reject duplicate retry requests" do
     flexmock(RightScale::Log).should_receive(:info).once.with(on {|arg| arg =~ /REJECT RETRY DUP/})
     EM.run do
-      @agent.should_receive(:options).and_return(:dup_check => true)
-      @dispatcher = RightScale::Dispatcher.new(@agent)
+      @dispatcher = RightScale::Dispatcher.new(@agent, @cache)
       @dispatcher.em = EMMock
-      req = RightScale::Request.new('/foo/bar', 'you', :token => "try")
+      req = RightScale::Request.new('/foo/bar_non', 1, :token => "try")
       req.tries.concat(["try1", "try2"])
-      @dispatcher.instance_variable_get(:@dispatched).store("try2")
-      @dispatcher.dispatch(req).should be_nil
+      @cache.store("try2", nil)
+      @dispatcher.dispatch(req, @header).should be_nil
       EM.stop
     end
   end
 
   it "should not reject non-duplicate requests" do
     EM.run do
-      @agent.should_receive(:options).and_return(:dup_check => true)
-      @dispatcher = RightScale::Dispatcher.new(@agent)
+      @dispatcher = RightScale::Dispatcher.new(@agent, @cache)
       @dispatcher.em = EMMock
-      req = RightScale::Request.new('/foo/bar', 'you', :token => "try")
+      req = RightScale::Request.new('/foo/bar_non', 1, :token => "try")
       req.tries.concat(["try1", "try2"])
-      @dispatcher.instance_variable_get(:@dispatched).store("try3")
-      @dispatcher.dispatch(req).should_not be_nil
+      @cache.store("try3", nil)
+      @dispatcher.dispatch(req, @header).should_not be_nil
       EM.stop
     end
   end
 
-  it "should not check for duplicates if dup_check disabled" do
+  it "should not reject duplicate idempotent requests" do
     EM.run do
-      @dispatcher = RightScale::Dispatcher.new(@agent)
+      @dispatcher = RightScale::Dispatcher.new(@agent, @cache)
       @dispatcher.em = EMMock
       req = RightScale::Request.new('/foo/bar', 'you', :token => "try")
+      @cache.store(req.token, nil)
+      @dispatcher.dispatch(req, @header).should_not be_nil
+      EM.stop
+    end
+  end
+
+  it "should not check for duplicates if duplicate checking is disabled" do
+    EM.run do
+      @dispatcher = RightScale::Dispatcher.new(@agent, dispatched_cache = nil)
+      @dispatcher.em = EMMock
+      req = RightScale::Request.new('/foo/bar_non', 1, :token => "try")
+      req.tries.concat(["try1", "try2"])
+      @dispatcher.instance_variable_get(:@dispatched_cache).should be_nil
+      @dispatcher.dispatch(req, @header).should_not be_nil
+      EM.stop
+    end
+  end
+
+  it "should not check for duplicates if actor method is idempotent" do
+    EM.run do
+      @dispatcher = RightScale::Dispatcher.new(@agent, dispatched_cache = nil)
+      @dispatcher.em = EMMock
+      req = RightScale::Request.new('/foo/bar', 1, :token => "try")
       req.tries.concat(["try1", "try2"])
-      @dispatcher.instance_variable_get(:@dispatched).should be_nil
-      @dispatcher.dispatch(req).should_not be_nil
+      @dispatcher.instance_variable_get(:@dispatched_cache).should be_nil
+      @dispatcher.dispatch(req, @header).should_not be_nil
       EM.stop
     end
   end
 
   it "should return dispatch age of youngest unfinished request" do
+    @header.should_receive(:ack).never
     @dispatcher.em = EMMockNoCallback
     @dispatcher.dispatch_age.should be_nil
-    @dispatcher.dispatch(RightScale::Push.new('/foo/bar', 'you'))
+    @dispatcher.dispatch(RightScale::Push.new('/foo/bar', 'you'), @header)
     @dispatcher.dispatch_age.should == 0
-    @dispatcher.dispatch(RightScale::Request.new('/foo/bar', 'you'))
+    @dispatcher.dispatch(RightScale::Request.new('/foo/bar', 'you'), @header)
     flexmock(Time).should_receive(:now).and_return(@now += 100)
     @dispatcher.dispatch_age.should == 100
   end
 
   it "should return dispatch age of nil if all requests finished" do
     @dispatcher.dispatch_age.should be_nil
-    @dispatcher.dispatch(RightScale::Request.new('/foo/bar', 'you'))
+    @dispatcher.dispatch(RightScale::Request.new('/foo/bar', 'you'), @header)
     flexmock(Time).should_receive(:now).and_return(@now += 100)
     @dispatcher.dispatch_age.should be_nil
   end
 
+  it "should ack request even if fail while dispatching" do
+    RightScale::Log.should_receive(:error).and_raise(Exception).once
+    req = RightScale::Request.new('/foo/i_kill_you', nil)
+    lambda { @dispatcher.dispatch(req, @header) }.should raise_error(Exception)
+  end
+
+  it "should not attempt to ack request if fail while dispatching and there is no header" do
+    @header.should_receive(:ack).never
+    RightScale::Log.should_receive(:error).and_raise(Exception).once
+    req = RightScale::Request.new('/foo/i_kill_you', nil)
+    lambda { @dispatcher.dispatch(req, nil) }.should raise_error(Exception)
+  end
+
+  it "should ack request even if fail while doing final setup for processing request" do
+    @dispatcher.em = EMMock
+    flexmock(EMMock).should_receive(:defer).and_raise(Exception).once
+    req = RightScale::Request.new('/foo/bar', 'you')
+    lambda { @dispatcher.dispatch(req, @header) }.should raise_error(Exception)
+  end
+
+  it "should not attempt to ack request if fail while doing final setup for processing request and there is no header" do
+    @header.should_receive(:ack).never
+    @dispatcher.em = EMMock
+    flexmock(EMMock).should_receive(:defer).and_raise(Exception).once
+    req = RightScale::Request.new('/foo/bar', 'you')
+    lambda { @dispatcher.dispatch(req, nil) }.should raise_error(Exception)
+  end
+
+  it "should not attempt to ack request if dispatch a request and there is no header" do
+    @header.should_receive(:ack).never
+    req = RightScale::Request.new('/foo/bar', 'you', :token => 'token')
+    @dispatcher.dispatch(req, nil)
+  end
+
 end # RightScale::Dispatcher