right_agent 0.10.13 → 0.13.5

data/lib/right_agent.rb

@@ -47,7 +47,9 @@ require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'agent_tag_manager')
 require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'actor'))
 require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'actor_registry'))
 require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'dispatcher'))
+require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'dispatched_cache'))
 require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'sender'))
 require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'secure_identity'))
 require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'idempotent_request'))
+require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'history'))
 require File.normalize_path(File.join(RIGHT_AGENT_BASE_DIR, 'agent'))

data/lib/right_agent/actor.rb

@@ -59,21 +59,45 @@ module RightScale
         prefix = to_s.to_const_path
       end
 
+      # Add methods to list of services supported by actor and mark these methods
+      # as idempotent
+      #
+      # === Parameters
+      # methods(Array):: Symbol names for methods being exposed as actor idempotent services
+      #
+      # === Return
+      # true:: Always return true
+      def expose_idempotent(*methods)
+        @exposed ||= {}
+        methods.each do |m|
+          if @exposed[m] == false
+            Log.warning("Method #{m} declared both idempotent and non-idempotent, assuming non-idempotent")
+          else
+            @exposed[m] = true
+          end
+        end
+        true
+      end
+
       # Add methods to list of services supported by actor
+      # By default these methods are not idempotent
       #
       # === Parameters
       # meths(Array):: Symbol names for methods being exposed as actor services
       #
       # === Return
-      # @exposed(Array):: List of unique methods exposed
-      def expose(*meths)
-        @exposed ||= []
-        meths.each do |meth|
-          @exposed << meth unless @exposed.include?(meth)
+      # true:: Always return true
+      def expose_non_idempotent(*methods)
+        @exposed ||= {}
+        methods.each do |m|
+          Log.warning("Method #{m} declared both idempotent and non-idempotent, assuming non-idempotent") if @exposed[m]
+          @exposed[m] = false
         end
-        @exposed
+        true
       end
 
+      alias :expose :expose_non_idempotent
+
       # Get /prefix/method paths that actor responds to
       #
       # === Parameters
@@ -83,14 +107,25 @@ module RightScale
       # (Array):: /prefix/method strings
       def provides_for(prefix)
         return [] unless @exposed
-        @exposed.select do |meth|
-          if instance_methods.include?(meth.to_s) or instance_methods.include?(meth.to_sym)
+        @exposed.each_key.select do |method|
+          if instance_methods.include?(method.to_s) or instance_methods.include?(method.to_sym)
             true
           else
-            Log.warning("Exposing non-existing method #{meth} in actor #{name}")
+            Log.warning("Exposing non-existing method #{method} in actor #{prefix}")
             false
           end
-        end.map {|meth| "/#{prefix}/#{meth}".squeeze('/')}
+        end.map { |method| "/#{prefix}/#{method}".squeeze('/') }
+      end
+
+      # Determine whether actor method is idempotent
+      #
+      # === Parameters
+      # method(Symbol):: Name for actor method
+      #
+      # === Return
+      # (Boolean):: true if idempotent, false otherwise
+      def idempotent?(method)
+        @exposed[method] if @exposed
       end
 
       # Set method called when dispatching to this actor fails

data/lib/right_agent/actor_registry.rb

@@ -49,15 +49,15 @@ module RightScale
       log_msg += ", prefix #{prefix}" if prefix && !prefix.empty?
       Log.info(log_msg)
       prefix ||= actor.class.default_prefix
-      actors[prefix.to_s] = actor
+      @actors[prefix.to_s] = actor
     end
 
     # Retrieve services provided by all of the registered actors
     #
     # === Return
-    # services(Array):: List of unique /prefix/method path strings
+    # (Array):: List of unique /prefix/method path strings
     def services
-      actors.map {|prefix, actor| actor.class.provides_for(prefix) }.flatten.uniq
+      @actors.map { |prefix, actor| actor.class.provides_for(prefix) }.flatten.uniq
     end
 
     # Retrieve actor by prefix
@@ -66,9 +66,9 @@ module RightScale
     # prefix(String):: Prefix identifying actor
     #
     # === Return
-    # actor(Actor):: Retrieved actor, may be nil
+    # (Actor|nil):: Retrieved actor, or nil if unknown
     def actor_for(prefix)
-      actor = actors[prefix]
+      @actors[prefix]
     end
 
   end # ActorRegistry

data/lib/right_agent/actors/agent_manager.rb

@@ -30,8 +30,8 @@ class AgentManager
 
   on_exception { |meth, deliverable, e| RightScale::ExceptionMailer.deliver_notification(meth, deliverable, e) }
 
-  expose :ping, :stats, :profile, :set_log_level, :execute, :connect, :disconnect, :connect_failed,
-         :tune_heartbeat, :terminate
+  expose_idempotent     :ping, :stats, :profile, :set_log_level, :connect, :disconnect, :connect_failed, :tune_heartbeat
+  expose_non_idempotent :execute, :terminate
 
   # Valid log levels
   LEVELS = [:debug, :info, :warn, :error, :fatal]
@@ -255,8 +255,8 @@ class AgentManager
   # true
   def terminate(options = nil)
     RightScale::CommandRunner.stop
-    # Delay terminate a bit to give reply a chance to be sent
-    EM.next_tick { @agent.terminate }
+    # Delay terminate a bit to give request message a chance to be ack'd and reply to be sent
+    EM.add_timer(1) { @agent.terminate }
     true
   end
 

data/lib/right_agent/agent.rb

@@ -73,9 +73,12 @@ module RightScale
       :check_interval     => 5 * 60,
       :grace_timeout      => 30,
       :prefetch           => 1,
-      :heartbeat          => 60
+      :heartbeat          => 0
     }
 
+    # Default block to be activated when finish terminating
+    DEFAULT_TERMINATE_BLOCK = lambda { EM.stop if EM.reactor_running? }
+
     # Initializes a new agent and establishes an AMQP connection.
     # This must be used inside EM.run block or if EventMachine reactor
     # is already started, for instance, by a Thin server that your Merb/Rails
@@ -112,6 +115,7 @@ module RightScale
     #   :grace_timeout(Integer):: Maximum number of seconds to wait after last request received before
     #     terminating regardless of whether there are still unfinished requests
     #   :dup_check(Boolean):: Whether to check for and reject duplicate requests, e.g., due to retries
+    #     or redelivery by broker after server failure
     #   :prefetch(Integer):: Maximum number of messages the AMQP broker is to prefetch for this agent
     #     before it receives an ack. Value 1 ensures that only last unacknowledged gets redelivered
     #     if the agent crashes. Value 0 means unlimited prefetch.
@@ -119,9 +123,10 @@ module RightScale
     #     exception(Exception):: Exception
     #     message(Packet):: Message being processed
     #     agent(Agent):: Reference to agent
-    #   :ready_callback(Proc):: Called once agent is connected ready to service (no argument)
-    #   :restart_callback(Proc):: Callback that is activated on each restart vote with votes being initiated
-    #     by offline queue exceeding MAX_QUEUED_REQUESTS or by repeated failures to access mapper when online
+    #   :ready_callback(Proc):: Called once agent is connected to broker and ready for service (no argument)
+    #   :restart_callback(Proc):: Called on each restart vote with votes being initiated by offline queue
+    #     exceeding MAX_QUEUED_REQUESTS or by repeated failures to access mapper when online (no argument)
+    #   :abnormal_terminate_callback(Proc):: Called at end of termination when terminate abnormally (no argument)
     #   :services(Symbol):: List of services provided by this agent. Defaults to all methods exposed by actors.
     #   :secure(Boolean):: true indicates to use security features of RabbitMQ to restrict agents to themselves
     #   :single_threaded(Boolean):: true indicates to run all operations in one thread; false indicates
@@ -160,8 +165,8 @@ module RightScale
       @tags.flatten!
       @options.freeze
       @deferred_tasks = []
-      @terminating = false
-      @last_stat_reset_time = @service_start_time = Time.now
+      @history = History.new(@identity)
+      @last_stat_reset_time = Time.now
       reset_agent_stats
       true
     end
@@ -174,11 +179,15 @@ module RightScale
       Log.init(@identity, @options[:log_path], :print => true)
       Log.level = @options[:log_level] if @options[:log_level]
       RightSupport::Log::Mixin.default_logger = Log
+      @history.update("start")
       now = Time.now
       Log.info("[start] Agent #{@identity} starting; time: #{now.utc}; utc_offset: #{now.utc_offset}")
       Log.debug("Start options:")
-      log_opts = @options.inject([]){ |t, (k, v)| t << "-  #{k}: #{v.respond_to?(:each) ? v.inspect : v}" }
+      log_opts = @options.inject([]) do |t, (k, v)|
+        t << "-  #{k}: #{k.to_s =~ /pass/ ? '****' : (v.respond_to?(:each) ? v.inspect : v)}"
+      end
       log_opts.each { |l| Log.debug(l) }
+      terminate_callback = @options[:abnormal_terminate_callback]
 
       begin
         # Capture process id in file after optional daemonize
@@ -200,11 +209,12 @@ module RightScale
             EM.add_timer(1) do
               begin
                 @registry = ActorRegistry.new
-                @dispatcher = Dispatcher.new(self)
-                @sender = Sender.new(self)
+                @dispatcher = create_dispatcher
+                @sender = create_sender
                 load_actors
                 setup_traps
                 setup_queues
+                @history.update("run")
                 start_console if @options[:console] && !@options[:daemonize]
 
                 # Need to keep reconnect interval at least :connect_timeout in size,
@@ -216,29 +226,29 @@ module RightScale
                 @check_status_brokers = @broker.all
                 EM.next_tick { @options[:ready_callback].call } if @options[:ready_callback]
                 @check_status_timer = EM::PeriodicTimer.new(interval) { check_status }
+              rescue SystemExit
+                raise
               rescue Exception => e
-                Log.error("Agent failed startup", e, :trace) unless e.message == "exit"
-                EM.stop
+                terminate("failed startup after connecting to a broker", e, &terminate_callback)
               end
             end
           elsif status == :failed
-            Log.error("Agent failed to connect to any brokers during startup")
-            EM.stop
+            terminate("failed to connect to any brokers during startup", &terminate_callback)
           elsif status == :timeout
-            Log.error("Agent failed to connect to any brokers after #{@options[:connect_timeout]} seconds during startup")
-            EM.stop
+            terminate("failed to connect to any brokers after #{@options[:connect_timeout]} seconds during startup",
+                      &terminate_callback)
           else
-            Log.error("Agent broker connect attempt failed unexpectedly with status #{status} during startup")
-            EM.stop
+            terminate("broker connect attempt failed unexpectedly with status #{status} during startup",
+                      &terminate_callback)
           end
         end
-      rescue SystemExit => e
-        raise e
+      rescue SystemExit
+        raise
       rescue PidFile::AlreadyRunning
+        EM.stop if EM.reactor_running?
         raise
       rescue Exception => e
-        Log.error("Agent failed startup", e, :trace) unless e.message == "exit"
-        raise e
+        terminate("failed startup", e, &terminate_callback)
       end
       true
     end
@@ -430,17 +440,21 @@ module RightScale
     end
 
     # Handle packet received
+    # Delegate packet acknowledgement to dispatcher/sender
+    # Ignore requests if in the process of terminating but continue to accept responses
     #
     # === Parameters
     # packet(Request|Push|Result):: Packet received
+    # header(AMQP::Frame::Header|nil):: Request header containing ack control
     #
     # === Return
     # true:: Always return true
-    def receive(packet)
+    def receive(packet, header = nil)
       begin
         case packet
-        when Push, Request then @dispatcher.dispatch(packet) unless @terminating
-        when Result        then @sender.handle_response(packet)
+        when Push, Request then @dispatcher.dispatch(packet, header) unless @terminating
+        when Result        then @sender.handle_response(packet, header)
+        else header.ack if header
         end
         @sender.message_received
       rescue RightAMQP::HABrokerClient::NoConnectedBrokers => e
@@ -465,20 +479,29 @@ module RightScale
 
     # Gracefully terminate execution by allowing unfinished tasks to complete
     # Immediately terminate if called a second time
+    # Report reason for termination if it is abnormal
+    #
+    # === Parameters
+    # reason(String):: Reason for abnormal termination, if any
+    # exception(Exception|String):: Exception or other parenthetical error information, if any
     #
     # === Block
     # Optional block to be executed after termination is complete
     #
     # === Return
     # true:: Always return true
-    def terminate(&block)
+    def terminate(reason = nil, exception = nil, &block)
+      block ||= DEFAULT_TERMINATE_BLOCK
       begin
-        if @terminating
-          Log.info("[stop] Terminating immediately")
+        @history.update("stop")
+        Log.error("[stop] Terminating because #{reason}", exception, :trace) if reason
+        if @terminating || @broker.nil?
+          @terminating = true
           @termination_timer.cancel if @termination_timer
           @termination_timer = nil
-          block.call if block
-          EM.stop if EM.reactor_running?
+          Log.info("[stop] Terminating immediately")
+          block.call
+          @history.update("graceful exit") if @broker.nil?
         else
           @terminating = true
           @check_status_timer.cancel if @check_status_timer
@@ -495,7 +518,7 @@ module RightScale
                 request_count, request_age = @sender.terminate
                 Log.info("[stop] The following #{request_count} requests initiated as recently as #{request_age} " +
                          "seconds ago are being dropped:\n  " + @sender.dump_requests.join("\n  ")) if request_age
-                @broker.close { block.call if block; EM.stop if EM.reactor_running? }
+                @broker.close { block.call }
               end
 
               wait_time = [timeout - (request_age || timeout), timeout - (dispatch_age || timeout), 0].max
@@ -513,19 +536,21 @@ module RightScale
                     finish.call
                   rescue Exception => e
                     Log.error("Failed while finishing termination", e, :trace)
-                    EM.stop if EM.reactor_running?
+                    begin block.call; rescue Exception; end
                   end
                 end
               end
             else
-              block.call if block
-              EM.stop if EM.reactor_running?
+              block.call
             end
+            @history.update("graceful exit")
           end
         end
+      rescue SystemExit
+        raise
       rescue Exception => e
         Log.error("Failed to terminate gracefully", e, :trace)
-        EM.stop if EM.reactor_running?
+        begin block.call; rescue Exception; end
       end
       true
     end
@@ -553,7 +578,7 @@ module RightScale
         "send stats"      => @sender.stats(reset),
         "last reset time" => @last_stat_reset_time.to_i,
         "stat time"       => now.to_i,
-        "service uptime"  => (now - @service_start_time).to_i,
+        "service uptime"  => @history.analyze_service,
         "machine uptime"  => Platform.shell.uptime
       }
       stats["revision"] = @revision if @revision
@@ -654,6 +679,23 @@ module RightScale
       false
     end
 
+    # Create dispatcher for handling incoming requests
+    #
+    # === Return
+    # (Dispatcher):: New dispatcher
+    def create_dispatcher
+      cache = DispatchedCache.new(@identity) if @options[:dup_check]
+      Dispatcher.new(self, cache)
+    end
+
+    # Create manager for outgoing requests
+    #
+    # === Return
+    # (Sender):: New sender
+    def create_sender
+      Sender.new(self)
+    end
+
     # Load the ruby code for the actors
     #
     # === Return
@@ -724,7 +766,7 @@ module RightScale
       queue = {:name => @identity, :options => {:durable => true, :no_declare => @options[:secure]}}
       filter = [:from, :tags, :tries, :persistent]
       options = {:ack => true, Request => filter, Push => filter, Result => [:from], :brokers => ids}
-      ids = @broker.subscribe(queue, nil, options) { |_, packet| receive(packet) }
+      ids = @broker.subscribe(queue, nil, options) { |_, packet, header| receive(packet, header) }
     end
 
     # Setup signal traps
@@ -737,7 +779,7 @@ module RightScale
           EM.next_tick do
             begin
               terminate do
-                EM.stop if EM.reactor_running?
+                DEFAULT_TERMINATE_BLOCK.call
                 old.call if old.is_a? Proc
               end
             rescue Exception => e
@@ -796,10 +838,28 @@ module RightScale
         true
       end
 
+      begin
+        check_other(@check_status_count)
+      rescue Exception => e
+        Log.error("Failed to perform other check status check", e)
+        @exceptions.track("check status", e)
+      end
+
       @check_status_count += 1
       true
     end
 
+    # Allow derived classes to perform any other useful periodic checks
+    #
+    # === Parameters
+    # check_status_count(Integer):: Counter that is incremented for each status check
+    #
+    # === Return
+    # true:: Always return true
+    def check_other(check_status_count)
+      true
+    end
+
     # Store unique tags
     #
     # === Parameters

data/lib/right_agent/agent_tag_manager.rb

@@ -20,13 +20,12 @@
 # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
-require 'singleton'
 
 module RightScale
 
   # Agent tags management
   class AgentTagManager
-    include Singleton
+    include RightSupport::Ruby::EasySingleton
 
     # (Agent) Agent being managed
     attr_accessor :agent