right_agent 0.13.5 → 0.14.0

data/lib/right_agent/dispatched_cache.rb

@@ -24,7 +24,7 @@ module RightScale
 
   # Cache for requests that have been dispatched recently
   # This cache is intended for use in checking for duplicate requests
-  # Since this is a local cache, it is not usable for requests received from a shared queue
+  # when there is only one server servicing a queue
   class DispatchedCache
 
     # Maximum number of seconds to retain a dispatched request in cache
@@ -43,16 +43,15 @@ module RightScale
       @max_age = MAX_AGE
     end
 
-    # Store dispatched request token in cache unless from shared queue
+    # Store dispatched request token in cache
     #
     # === Parameters
     # token(String):: Generated message identifier
-    # shared_queue(String|nil):: Name of shared queue if being dispatched from a shared queue
     #
     # === Return
     # true:: Always return true
-    def store(token, shared_queue)
-      if token && shared_queue.nil?
+    def store(token)
+      if token
         now = Time.now.to_i
         if @cache.has_key?(token)
           @cache[token] = now

data/lib/right_agent/dispatcher.rb

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2009-2011 RightScale Inc
+# Copyright (c) 2009-2012 RightScale Inc
 #
 # Permission is hereby granted, free of charge, to any person obtaining
 # a copy of this software and associated documentation files (the
@@ -25,8 +25,8 @@ module RightScale
   # Dispatching of payload to specified actor
   class Dispatcher
 
-    # Response queue name
-    RESPONSE_QUEUE = "response"
+    class InvalidRequestType < Exception; end
+    class DuplicateRequest < Exception; end
 
     # (ActorRegistry) Registry for actors
     attr_reader :registry
@@ -34,164 +34,67 @@ module RightScale
     # (String) Identity of associated agent
     attr_reader :identity
 
-    # (RightAMQP::HABrokerClient) High availability AMQP broker client
-    attr_reader :broker
-
-    # (EM) Event machine class (exposed for unit tests)
-    attr_accessor :em
+    # For direct access to current dispatcher
+    #
+    # === Return
+    # (Dispatcher):: This dispatcher instance if defined, otherwise nil
+    def self.instance
+      @@instance if defined?(@@instance)
+    end
 
     # Initialize dispatcher
     #
     # === Parameters
-    # agent(Agent):: Agent using this dispatcher; uses its identity, broker, registry, and following options:
-    #   :secure(Boolean):: true indicates to use Security features of RabbitMQ to restrict agents to themselves
-    #   :single_threaded(Boolean):: true indicates to run all operations in one thread; false indicates
-    #     to do requested work on event machine defer thread and all else, such as pings on main thread
-    #   :threadpool_size(Integer):: Number of threads in event machine thread pool
+    # agent(Agent):: Agent using this dispatcher; uses its identity and registry
     # dispatched_cache(DispatchedCache|nil):: Cache for dispatched requests that is used for detecting
     #   duplicate requests, or nil if duplicate checking is disabled
     def initialize(agent, dispatched_cache = nil)
       @agent = agent
-      @broker = @agent.broker
       @registry = @agent.registry
       @identity = @agent.identity
-      options = @agent.options
-      @secure = options[:secure]
-      @single_threaded = options[:single_threaded]
-      @pending_dispatches = 0
-      @em = EM
-      @em.threadpool_size = (options[:threadpool_size] || 20).to_i
+      @dispatched_cache = dispatched_cache
       reset_stats
+      @@instance = self
+    end
 
-      # Only access this cache from primary thread
-      @dispatched_cache = dispatched_cache
+    # Determine whether able to route requests to specified actor
+    #
+    # === Parameters
+    # actor(String):: Actor name
+    #
+    # === Return
+    # (Boolean):: true if can route to actor, otherwise false
+    def routable?(actor)
+      !!@registry.actor_for(actor)
     end
 
-    # Dispatch request to appropriate actor for servicing
-    # Handle returning of result to requester including logging any exceptions
+    # Route request to appropriate actor for servicing
     # Reject requests whose TTL has expired or that are duplicates of work already dispatched
-    # Work is done in background defer thread if single threaded option is false
-    # Acknowledge request after actor has responded
     #
     # === Parameters
     # request(Request|Push):: Packet containing request
     # header(AMQP::Frame::Header|nil):: Request header containing ack control
-    # shared_queue(String|nil):: Name of shared queue if being dispatched from a shared queue
     #
     # === Return
-    # (Result|nil):: Result from dispatched request, nil if not dispatched because dup or stale
-    def dispatch(request, header = nil, shared_queue = nil)
-      begin
-        ack_deferred = false
-
-        # Determine which actor this request is for
-        prefix, method = request.type.split('/')[1..-1]
-        method ||= :index
-        method = method.to_sym
-        actor = @registry.actor_for(prefix)
-        token = request.token
-        received_at = @requests.update(method, (token if request.kind_of?(Request)))
-        if actor.nil?
-          Log.error("No actor for dispatching request <#{token}> of type #{request.type}")
-          return nil
-        end
-        method_idempotent = actor.class.idempotent?(method)
-
-        # Reject this request if its TTL has expired
-        if (expires_at = request.expires_at) && expires_at > 0 && received_at.to_i >= expires_at
-          @rejects.update("expired (#{method})")
-          Log.info("REJECT EXPIRED <#{token}> from #{request.from} TTL #{RightSupport::Stats.elapsed(received_at.to_i - expires_at)} ago")
-          if request.is_a?(Request)
-            # For agents that do not know about non-delivery, use error result
-            non_delivery = if request.recv_version < 13
-              OperationResult.error("Could not deliver request (#{OperationResult::TTL_EXPIRATION})")
-            else
-              OperationResult.non_delivery(OperationResult::TTL_EXPIRATION)
-            end
-            result = Result.new(token, request.reply_to, non_delivery, @identity, request.from, request.tries, request.persistent)
-            exchange = {:type => :queue, :name => RESPONSE_QUEUE, :options => {:durable => true, :no_declare => @secure}}
-            @broker.publish(exchange, result, :persistent => true, :mandatory => true)
-          end
-          return nil
-        end
-
-        # Reject this request if it is a duplicate
-        if !method_idempotent && @dispatched_cache
-          if by = @dispatched_cache.serviced_by(token)
-            @rejects.update("duplicate (#{method})")
-            Log.info("REJECT DUP <#{token}> serviced by #{by == @identity ? 'self' : by}")
-            return nil
-          end
-          request.tries.each do |t|
-            if by = @dispatched_cache.serviced_by(t)
-              @rejects.update("retry duplicate (#{method})")
-              Log.info("REJECT RETRY DUP <#{token}> of <#{t}> serviced by #{by == @identity ? 'self' : by}")
-              return nil
-            end
-          end
-        end
-
-        # Proc for performing request in actor
-        operation = lambda do
-          begin
-            @pending_dispatches += 1
-            @last_request_dispatch_time = received_at.to_i
-            @dispatched_cache.store(token, shared_queue) if !method_idempotent && @dispatched_cache
-            if actor.method(method).arity.abs == 1
-              actor.__send__(method, request.payload)
-            else
-              actor.__send__(method, request.payload, request)
-            end
-          rescue Exception => e
-            @pending_dispatches = [@pending_dispatches - 1, 0].max
-            OperationResult.error(handle_exception(actor, method, request, e))
-          end
-        end
-
-        # Proc for sending response
-        callback = lambda do |r|
-          begin
-            if request.kind_of?(Request)
-              duration = @requests.finish(received_at, token)
-              r = Result.new(token, request.reply_to, r, @identity, request.from, request.tries, request.persistent, duration)
-              exchange = {:type => :queue, :name => RESPONSE_QUEUE, :options => {:durable => true, :no_declare => @secure}}
-              @broker.publish(exchange, r, :persistent => true, :mandatory => true, :log_filter => [:tries, :persistent, :duration])
-            end
-          rescue RightAMQP::HABrokerClient::NoConnectedBrokers => e
-            Log.error("Failed to publish result of dispatched request #{request.trace}", e)
-          rescue Exception => e
-            Log.error("Failed to publish result of dispatched request #{request.trace}", e, :trace)
-            @exceptions.track("publish response", e)
-          ensure
-            header.ack if header
-            @pending_dispatches = [@pending_dispatches - 1, 0].max
-          end
-          r # For unit tests
-        end
-
-        # Process request and send response, if any
-        begin
-          ack_deferred = true
-          if @single_threaded
-            @em.next_tick { callback.call(operation.call) }
-          else
-            @em.defer(operation, callback)
-          end
-        rescue Exception
-          header.ack if header
-          raise
-        end
-      ensure
-        header.ack unless ack_deferred || header.nil?
-      end
-    end
-
-    # Determine age of youngest request dispatch
+    # (Result|nil):: Result of request, or nil if there is no result because request is a Push
     #
-    # === Return
-    # (Integer|nil):: Age in seconds of youngest dispatch, or nil if none
-    def dispatch_age
-      Time.now.to_i - @last_request_dispatch_time if @last_request_dispatch_time && @pending_dispatches > 0
+    # === Raise
+    # InvalidRequestType:: If the request cannot be routed to an actor
+    # DuplicateRequest:: If request rejected because it has already been processed
+    def dispatch(request)
+      token = request.token
+      actor, method, idempotent = route(request)
+      received_at = @request_stats.update(method, (token if request.is_a?(Request)))
+      if dup = duplicate?(request, method, idempotent)
+        raise DuplicateRequest.new(dup)
+      end
+      unless result = expired?(request, method)
+        result = perform(request, actor, method, idempotent)
+      end
+      if request.is_a?(Request)
+        duration = @request_stats.finish(received_at, token)
+        Result.new(token, request.reply_to, result, @identity, request.from, request.tries, request.persistent, duration)
+      end
     end
 
     # Get dispatcher statistics
@@ -203,30 +106,25 @@ module RightScale
     # stats(Hash):: Current statistics:
     #   "dispatched cache"(Hash|nil):: Number of dispatched requests cached and age of youngest and oldest,
     #     or nil if empty
+    #   "dispatch failures"(Hash|nil):: Dispatch failure activity stats with keys "total", "percent", "last", and "rate"
+    #     with percentage breakdown per failure type, or nil if none
     #   "exceptions"(Hash|nil):: Exceptions raised per category, or nil if none
     #     "total"(Integer):: Total for category
     #     "recent"(Array):: Most recent as a hash of "count", "type", "message", "when", and "where"
     #   "rejects"(Hash|nil):: Request reject activity stats with keys "total", "percent", "last", and "rate"
-    #   "pending"(Hash|nil):: Pending request "total" and "youngest age", or nil if none
     #     with percentage breakdown per reason ("duplicate (<method>)", "retry duplicate (<method>)", or
     #     "stale (<method>)"), or nil if none
     #   "requests"(Hash|nil):: Request activity stats with keys "total", "percent", "last", and "rate"
     #     with percentage breakdown per request type, or nil if none
     #   "response time"(Float):: Average number of seconds to respond to a request recently
     def stats(reset = false)
-      pending = if @pending_dispatches > 0
-        {
-          "total" => @pending_dispatches,
-          "youngest age" => dispatch_age
-        }
-      end
       stats = {
-        "dispatched cache" => (@dispatched_cache.stats if @dispatched_cache),
-        "exceptions"       => @exceptions.stats,
-        "pending"          => pending,
-        "rejects"          => @rejects.all,
-        "requests"         => @requests.all,
-        "response time"    => @requests.avg_duration
+        "dispatched cache"  => (@dispatched_cache.stats if @dispatched_cache),
+        "dispatch failures" => @dispatch_failure_stats.all,
+        "exceptions"        => @exception_stats.stats,
+        "rejects"           => @reject_stats.all,
+        "requests"          => @request_stats.all,
+        "response time"     => @request_stats.avg_duration
       }
       reset_stats if reset
       stats
@@ -239,12 +137,103 @@ module RightScale
     # === Return
     # true:: Always return true
     def reset_stats
-      @rejects = RightSupport::Stats::Activity.new
-      @requests = RightSupport::Stats::Activity.new
-      @exceptions = RightSupport::Stats::Exceptions.new(@agent)
+      @reject_stats = RightSupport::Stats::Activity.new
+      @request_stats = RightSupport::Stats::Activity.new
+      @dispatch_failure_stats = RightSupport::Stats::Activity.new
+      @exception_stats = RightSupport::Stats::Exceptions.new(@agent)
       true
     end
 
+    # Determine if request TTL has expired
+    #
+    # === Parameters
+    # request(Push|Request):: Request to be checked
+    # method(String):: Actor method requested to be performed
+    #
+    # === Return
+    # (OperationResult|nil):: Error result if expired, otherwise nil
+    def expired?(request, method)
+      if (expires_at = request.expires_at) && expires_at > 0 && (now = Time.now.to_i) >= expires_at
+        @reject_stats.update("expired (#{method})")
+        Log.info("REJECT EXPIRED <#{request.token}> from #{request.from} TTL #{RightSupport::Stats.elapsed(now - expires_at)} ago")
+        # For agents that do not know about non-delivery, use error result
+        if request.recv_version < 13
+          OperationResult.error("Could not deliver request (#{OperationResult::TTL_EXPIRATION})")
+        else
+          OperationResult.non_delivery(OperationResult::TTL_EXPIRATION)
+        end
+      end
+    end
+
+    # Determine whether this request is a duplicate
+    #
+    # === Parameters
+    # request(Request|Push):: Packet containing request
+    # method(String):: Actor method requested to be performed
+    # idempotent(Boolean):: Whether this method is idempotent
+    #
+    # === Return
+    # (String|nil):: Messaging describing who already serviced request if it is a duplicate, otherwise nil
+    def duplicate?(request, method, idempotent)
+      if !idempotent && @dispatched_cache
+        if serviced_by = @dispatched_cache.serviced_by(request.token)
+          from_retry = ""
+        else
+          from_retry = "retry "
+          request.tries.each { |t| break if serviced_by = @dispatched_cache.serviced_by(t) }
+        end
+        if serviced_by
+          @reject_stats.update("#{from_retry}duplicate (#{method})")
+          msg = "<#{request.token}> already serviced by #{serviced_by == @identity ? 'self' : serviced_by}"
+          Log.info("REJECT #{from_retry.upcase}DUP #{msg}")
+          msg
+        end
+      end
+    end
+
+    # Use request type to route request to actor and an associated method
+    #
+    # === Parameters
+    # request(Push|Request):: Packet containing request
+    #
+    # === Return
+    # (Array):: Actor name, method name, and whether method is idempotent
+    #
+    # === Raise
+    # InvalidRequestType:: If the request cannot be routed to an actor
+    def route(request)
+      prefix, method = request.type.split('/')[1..-1]
+      method ||= :index
+      method = method.to_sym
+      actor = @registry.actor_for(prefix)
+      if actor.nil? || !actor.respond_to?(method)
+        raise InvalidRequestType.new("Unknown actor or method for dispatching request <#{request.token}> of type #{request.type}")
+      end
+      [actor, method, actor.class.idempotent?(method)]
+    end
+
+    # Perform requested action
+    #
+    # === Parameters
+    # request(Push|Request):: Packet containing request
+    # token(String):: Unique identity token for request
+    # method(String):: Actor method requested to be performed
+    # idempotent(Boolean):: Whether this method is idempotent
+    #
+    # === Return
+    # (OperationResult):: Result from performing a request
+    def perform(request, actor, method, idempotent)
+      @dispatched_cache.store(request.token) if @dispatched_cache && !idempotent
+      if actor.method(method).arity.abs == 1
+        actor.__send__(method, request.payload)
+      else
+        actor.__send__(method, request.payload, request)
+      end
+    rescue Exception => e
+      @dispatch_failure_stats.update("#{request.type}->#{e.class.name}")
+      OperationResult.error(handle_exception(actor, method, request, e))
+    end
+
     # Handle exception by logging it, calling the actors exception callback method,
     # and gathering exception statistics
     #
@@ -268,10 +257,10 @@ module RightScale
             actor.instance_exec(method, request, exception, &actor.class.exception_callback)
           end
         end
-        @exceptions.track(request.type, exception)
+        @exception_stats.track(request.type, exception)
       rescue Exception => e
         Log.error("Failed handling error for #{request.type}", e, :trace)
-        @exceptions.track(request.type, e) rescue nil
+        @exception_stats.track(request.type, e) rescue nil
       end
       Log.format(error, exception)
     end

data/lib/right_agent/pid_file.rb

@@ -56,7 +56,7 @@ module RightScale
         if process_running?(pid)
           raise AlreadyRunning.new("#{@pid_file} already exists and process is running (pid: #{pid})")
         else
-          Log.info "removing stale pid file: #{@pid_file}"
+          Log.info("Removing stale pid file: #{@pid_file}")
           remove
         end
       end

data/lib/right_agent/platform.rb

@@ -250,6 +250,20 @@ module RightScale
       platform_service(:installer)
     end
 
+    # Determines which cloud we're on by the cheap but simple expedient of
+    # reading the RightScale cloud file
+    def resolve_cloud_type
+      cloud_type = read_cloud_file
+      @ec2 = false
+      @rackspace = false
+      @eucalyptus = false
+      case cloud_type
+        when 'ec2' then @ec2 = true
+        when 'rackspace' then @rackspace = true
+        when 'eucalyptus' then @eucalyptus = true
+      end
+    end
+
     private
 
     # Load platform specific implementation
@@ -285,20 +299,6 @@ module RightScale
       require File.expand_path(File.join(File.dirname(__FILE__), 'platform', 'windows'))
     end
 
-    # Determines which cloud we're on by the cheap but simple expedient of
-    # reading the RightScale cloud file
-    def resolve_cloud_type
-      cloud_type = read_cloud_file
-      @ec2 = false
-      @rackspace = false
-      @eucalyptus = false
-      case cloud_type
-        when 'ec2' then @ec2 = true
-        when 'rackspace' then @rackspace = true
-        when 'eucalyptus' then @eucalyptus = true
-      end
-    end
-
     # Reads the RightScale cloud file and returns its contents
     def read_cloud_file
       File.read(File.join(self.filesystem.right_scale_state_dir, 'cloud')) rescue nil

data/lib/right_agent/scripts/agent_controller.rb

@@ -1,5 +1,5 @@
 # === Synopsis:
-#   RightScale RightAgent Controller (rnac) - (c) 2009-2011 RightScale Inc
+#   RightScale RightAgent Controller (rnac) - (c) 2009-2012 RightScale Inc
 #
 #   rnac is a command line tool for managing a RightAgent
 #
@@ -85,12 +85,10 @@ module RightScale
 
     FORCED_OPTIONS =
     {
-      :threadpool_size => 1
     }
 
     DEFAULT_OPTIONS =
     {
-      :single_threaded => true,
       :log_dir => Platform.filesystem.log_dir,
       :daemonize => true
     }
@@ -521,7 +519,7 @@ module RightScale
 end # RightScale
 
 #
-# Copyright (c) 2009-2011 RightScale Inc
+# Copyright (c) 2009-2012 RightScale Inc
 #
 # Permission is hereby granted, free of charge, to any person obtaining
 # a copy of this software and associated documentation files (the