right_agent 0.10.13 → 0.13.5

data/lib/right_agent/command/command_io.rb

@@ -9,8 +9,6 @@
 # License Agreement between RightScale.com, Inc. and
 # the licensee.
 
-require 'singleton'
-
 module RightScale
 
   # Class which allows listening for data and sending data on sockets
@@ -18,7 +16,7 @@ module RightScale
   # the agent without having to go through RabbitMQ.
   class CommandIO
 
-    include Singleton
+    include RightSupport::Ruby::EasySingleton
 
     # ensure uniqueness of handler to avoid confusion.
     raise "#{ServerInputHandler.name} is already defined" if defined?(ServerInputHandler)

data/lib/right_agent/command/command_runner.rb

@@ -24,7 +24,7 @@ module RightScale
 
   # Run commands exposed by an agent.
   # External processes can send commands through a socket with the specified port.
-  # Command runner accepts connections and deserializes commands using YAML.
+  # Command runner accepts connections and unserializes commands using YAML.
   # Each command is expected to be a hash containing the :name and :options keys.
   class CommandRunner
     class << self
@@ -42,6 +42,8 @@ module RightScale
     #                        increment and retry if port already taken
     # identity(String):: Agent identity
     # commands(Hash):: Commands exposed by agent
+    # fiber_pool(NB::FiberPool):: Pool of initialized fibers to be used for executing
+    #   received commands in non-blocking fashion
     #
     # === Block
     # If a block is provided, this method will yield after all setup has been completed,
@@ -54,7 +56,7 @@ module RightScale
     #
     # === Raise
     # (RightScale::Exceptions::Application):: If +start+ has already been called and +stop+ hasn't since
-    def self.start(socket_port, identity, commands)
+    def self.start(socket_port, identity, commands, fiber_pool = nil)
       cmd_options = nil
       @listen_port = socket_port
 
@@ -65,7 +67,11 @@ module RightScale
             if cmd_cookie == @cookie
               cmd_name = c[:name].to_sym
               if commands.include?(cmd_name)
-                commands[cmd_name].call(c, conn)
+                if fiber_pool
+                  fiber_pool.spawn { commands[cmd_name].call(c, conn) }
+                else
+                  commands[cmd_name].call(c, conn)
+                end
               else
                 Log.warning("Unknown command '#{cmd_name}', known commands: #{commands.keys.join(', ')}")
               end

data/lib/right_agent/dispatched_cache.rb

@@ -0,0 +1,110 @@
+#
+# Copyright (c) 2012 RightScale Inc
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+module RightScale
+
+  # Cache for requests that have been dispatched recently
+  # This cache is intended for use in checking for duplicate requests
+  # Since this is a local cache, it is not usable for requests received from a shared queue
+  class DispatchedCache
+
+    # Maximum number of seconds to retain a dispatched request in cache
+    # This must be greater than the maximum possible retry timeout to avoid
+    # duplicate execution of a request
+    MAX_AGE = 12 * 60 * 60
+
+    # Initialize cache
+    #
+    # === Parameters
+    # identity(String):: Serialized identity of agent
+    def initialize(identity)
+      @identity = identity
+      @cache = {}
+      @lru = []
+      @max_age = MAX_AGE
+    end
+
+    # Store dispatched request token in cache unless from shared queue
+    #
+    # === Parameters
+    # token(String):: Generated message identifier
+    # shared_queue(String|nil):: Name of shared queue if being dispatched from a shared queue
+    #
+    # === Return
+    # true:: Always return true
+    def store(token, shared_queue)
+      if token && shared_queue.nil?
+        now = Time.now.to_i
+        if @cache.has_key?(token)
+          @cache[token] = now
+          @lru.push(@lru.delete(token))
+        else
+          @cache[token] = now
+          @lru.push(token)
+          @cache.delete(@lru.shift) while (now - @cache[@lru.first]) > @max_age
+        end
+      end
+      true
+    end
+
+    # Determine whether request has already been serviced
+    #
+    # === Parameters
+    # token(String):: Generated message identifier
+    #
+    # === Return
+    # (String|nil):: Identity of agent that already serviced request, or nil if none
+    def serviced_by(token)
+      if @cache[token]
+        @cache[token] = Time.now.to_i
+        @lru.push(@lru.delete(token))
+        @identity
+      end
+    end
+
+    # Get local cache statistics
+    #
+    # === Return
+    # stats(Hash|nil):: Current statistics, or nil if cache empty
+    #   "local total"(Integer):: Total number in local cache, or nil if none
+    #   "local max age"(String):: Time since oldest local cache entry created or updated
+    def stats
+      if (s = size) > 0
+        now = Time.now.to_i
+        {
+          "local total" => s,
+          "local max age" => RightSupport::Stats.elapsed(now - @cache[@lru.first])
+        }
+      end
+    end
+
+    # Get local cache size
+    #
+    # === Return
+    # (Integer):: Number of cache entries
+    def size
+      @cache.size
+    end
+
+  end # DispatchedCache
+
+end # RightScale

data/lib/right_agent/dispatcher.rb

@@ -28,82 +28,6 @@ module RightScale
     # Response queue name
     RESPONSE_QUEUE = "response"
 
-    # Cache for requests that have been dispatched recently
-    # This cache is intended for use in checking for duplicate requests
-    class Dispatched
-
-      # Maximum number of seconds to retain a dispatched request in cache
-      # This must be greater than the maximum possible retry timeout to avoid
-      # duplicate execution of a request
-      MAX_AGE = 12 * 60 * 60
-
-      # Initialize cache
-      def initialize
-        @cache = {}
-        @lru = []
-      end
-
-      # Store dispatched request token in cache
-      #
-      # === Parameters
-      # token(String):: Generated message identifier
-      #
-      # === Return
-      # true:: Always return true
-      def store(token)
-        now = Time.now.to_i
-        if @cache.has_key?(token)
-          @cache[token] = now
-          @lru.push(@lru.delete(token))
-        else
-          @cache[token] = now
-          @lru.push(token)
-          @cache.delete(@lru.shift) while (now - @cache[@lru.first]) > MAX_AGE
-        end
-        true
-      end
-
-      # Fetch request
-      #
-      # === Parameters
-      # token(String):: Generated message identifier
-      #
-      # === Return
-      # (Boolean):: true if request has been dispatched, otherwise false
-      def fetch(token)
-        if @cache[token]
-          @cache[token] = Time.now.to_i
-          @lru.push(@lru.delete(token))
-        end
-      end
-
-      # Get cache size
-      #
-      # === Return
-      # (Integer):: Number of cache entries
-      def size
-        @cache.size
-      end
-
-      # Get cache statistics
-      #
-      # === Return
-      # stats(Hash|nil):: Current statistics, or nil if cache empty
-      #   "total"(Integer):: Total number in cache, or nil if none
-      #   "oldest"(Integer):: Number of seconds since oldest cache entry created or updated
-      #   "youngest"(Integer):: Number of seconds since youngest cache entry created or updated
-      def stats
-        if size > 0
-          {
-            "total" => size,
-            "oldest age" => size > 0 ? Time.now.to_i - @cache[@lru.first] : 0,
-            "youngest age" => size > 0 ? Time.now.to_i - @cache[@lru.last] : 0
-          }
-        end
-      end
-
-    end # Dispatched
-
     # (ActorRegistry) Registry for actors
     attr_reader :registry
 
@@ -120,13 +44,13 @@ module RightScale
     #
     # === Parameters
     # agent(Agent):: Agent using this dispatcher; uses its identity, broker, registry, and following options:
-    #   :dup_check(Boolean):: Whether to check for and reject duplicate requests, e.g., due to retries,
-    #     but only for requests that are dispatched from non-shared queues
     #   :secure(Boolean):: true indicates to use Security features of RabbitMQ to restrict agents to themselves
     #   :single_threaded(Boolean):: true indicates to run all operations in one thread; false indicates
     #     to do requested work on event machine defer thread and all else, such as pings on main thread
     #   :threadpool_size(Integer):: Number of threads in event machine thread pool
-    def initialize(agent)
+    # dispatched_cache(DispatchedCache|nil):: Cache for dispatched requests that is used for detecting
+    #   duplicate requests, or nil if duplicate checking is disabled
+    def initialize(agent, dispatched_cache = nil)
       @agent = agent
       @broker = @agent.broker
       @registry = @agent.registry
@@ -134,116 +58,131 @@ module RightScale
       options = @agent.options
       @secure = options[:secure]
       @single_threaded = options[:single_threaded]
-      @dup_check = options[:dup_check]
       @pending_dispatches = 0
       @em = EM
       @em.threadpool_size = (options[:threadpool_size] || 20).to_i
       reset_stats
 
-      # Only access following from primary thread
-      @dispatched = Dispatched.new if @dup_check
+      # Only access this cache from primary thread
+      @dispatched_cache = dispatched_cache
     end
 
     # Dispatch request to appropriate actor for servicing
     # Handle returning of result to requester including logging any exceptions
     # Reject requests whose TTL has expired or that are duplicates of work already dispatched
-    # but do not do duplicate checking if being dispatched from a shared queue
     # Work is done in background defer thread if single threaded option is false
+    # Acknowledge request after actor has responded
     #
     # === Parameters
     # request(Request|Push):: Packet containing request
-    # shared(Boolean):: Whether being dispatched from a shared queue
+    # header(AMQP::Frame::Header|nil):: Request header containing ack control
+    # shared_queue(String|nil):: Name of shared queue if being dispatched from a shared queue
     #
     # === Return
-    # r(Result):: Result from dispatched request, nil if not dispatched because dup or stale
-    def dispatch(request, shared = false)
-
-      # Determine which actor this request is for
-      prefix, method = request.type.split('/')[1..-1]
-      method ||= :index
-      actor = @registry.actor_for(prefix)
-      token = request.token
-      received_at = @requests.update(method, (token if request.kind_of?(Request)))
-      if actor.nil?
-        Log.error("No actor for dispatching request <#{request.token}> of type #{request.type}")
-        return nil
-      end
+    # (Result|nil):: Result from dispatched request, nil if not dispatched because dup or stale
+    def dispatch(request, header = nil, shared_queue = nil)
+      begin
+        ack_deferred = false
 
-      # Reject this request if its TTL has expired
-      if (expires_at = request.expires_at) && expires_at > 0 && received_at.to_i >= expires_at
-        @rejects.update("expired (#{method})")
-        Log.info("REJECT EXPIRED <#{token}> from #{request.from} TTL #{RightSupport::Stats.elapsed(received_at.to_i - expires_at)} ago")
-        if request.is_a?(Request)
-          # For agents that do not know about non-delivery, use error result
-          non_delivery = if request.recv_version < 13
-            OperationResult.error("Could not deliver request (#{OperationResult::TTL_EXPIRATION})")
-          else
-            OperationResult.non_delivery(OperationResult::TTL_EXPIRATION)
-          end
-          result = Result.new(token, request.reply_to, non_delivery, @identity, request.from, request.tries, request.persistent)
-          exchange = {:type => :queue, :name => RESPONSE_QUEUE, :options => {:durable => true, :no_declare => @secure}}
-          @broker.publish(exchange, result, :persistent => true, :mandatory => true)
+        # Determine which actor this request is for
+        prefix, method = request.type.split('/')[1..-1]
+        method ||= :index
+        method = method.to_sym
+        actor = @registry.actor_for(prefix)
+        token = request.token
+        received_at = @requests.update(method, (token if request.kind_of?(Request)))
+        if actor.nil?
+          Log.error("No actor for dispatching request <#{token}> of type #{request.type}")
+          return nil
         end
-        return nil
-      end
+        method_idempotent = actor.class.idempotent?(method)
 
-      # Reject this request if it is a duplicate
-      if @dup_check && !shared && request.kind_of?(Request)
-        if @dispatched.fetch(token)
-          @rejects.update("duplicate (#{method})")
-          Log.info("REJECT DUP <#{token}> of self")
+        # Reject this request if its TTL has expired
+        if (expires_at = request.expires_at) && expires_at > 0 && received_at.to_i >= expires_at
+          @rejects.update("expired (#{method})")
+          Log.info("REJECT EXPIRED <#{token}> from #{request.from} TTL #{RightSupport::Stats.elapsed(received_at.to_i - expires_at)} ago")
+          if request.is_a?(Request)
+            # For agents that do not know about non-delivery, use error result
+            non_delivery = if request.recv_version < 13
+              OperationResult.error("Could not deliver request (#{OperationResult::TTL_EXPIRATION})")
+            else
+              OperationResult.non_delivery(OperationResult::TTL_EXPIRATION)
+            end
+            result = Result.new(token, request.reply_to, non_delivery, @identity, request.from, request.tries, request.persistent)
+            exchange = {:type => :queue, :name => RESPONSE_QUEUE, :options => {:durable => true, :no_declare => @secure}}
+            @broker.publish(exchange, result, :persistent => true, :mandatory => true)
+          end
           return nil
         end
-        request.tries.each do |t|
-          if @dispatched.fetch(t)
-            @rejects.update("retry duplicate (#{method})")
-            Log.info("REJECT RETRY DUP <#{token}> of <#{t}>")
+
+        # Reject this request if it is a duplicate
+        if !method_idempotent && @dispatched_cache
+          if by = @dispatched_cache.serviced_by(token)
+            @rejects.update("duplicate (#{method})")
+            Log.info("REJECT DUP <#{token}> serviced by #{by == @identity ? 'self' : by}")
             return nil
           end
+          request.tries.each do |t|
+            if by = @dispatched_cache.serviced_by(t)
+              @rejects.update("retry duplicate (#{method})")
+              Log.info("REJECT RETRY DUP <#{token}> of <#{t}> serviced by #{by == @identity ? 'self' : by}")
+              return nil
+            end
+          end
         end
-      end
 
-      # Proc for performing request in actor
-      operation = lambda do
-        begin
-          @pending_dispatches += 1
-          @last_request_dispatch_time = received_at.to_i
-          @dispatched.store(token) if @dup_check && !shared && request.kind_of?(Request) && token
-          if actor.method(method).arity.abs == 1
-            actor.__send__(method, request.payload)
-          else
-            actor.__send__(method, request.payload, request)
+        # Proc for performing request in actor
+        operation = lambda do
+          begin
+            @pending_dispatches += 1
+            @last_request_dispatch_time = received_at.to_i
+            @dispatched_cache.store(token, shared_queue) if !method_idempotent && @dispatched_cache
+            if actor.method(method).arity.abs == 1
+              actor.__send__(method, request.payload)
+            else
+              actor.__send__(method, request.payload, request)
+            end
+          rescue Exception => e
+            @pending_dispatches = [@pending_dispatches - 1, 0].max
+            OperationResult.error(handle_exception(actor, method, request, e))
           end
-        rescue Exception => e
-          @pending_dispatches = [@pending_dispatches - 1, 0].max
-          handle_exception(actor, method, request, e)
         end
-      end
-      
-      # Proc for sending response
-      callback = lambda do |r|
-        begin
-          @pending_dispatches = [@pending_dispatches - 1, 0].max
-          if request.kind_of?(Request)
-            duration = @requests.finish(received_at, token)
-            r = Result.new(token, request.reply_to, r, @identity, request.from, request.tries, request.persistent, duration)
-            exchange = {:type => :queue, :name => RESPONSE_QUEUE, :options => {:durable => true, :no_declare => @secure}}
-            @broker.publish(exchange, r, :persistent => true, :mandatory => true, :log_filter => [:tries, :persistent, :duration])
+
+        # Proc for sending response
+        callback = lambda do |r|
+          begin
+            if request.kind_of?(Request)
+              duration = @requests.finish(received_at, token)
+              r = Result.new(token, request.reply_to, r, @identity, request.from, request.tries, request.persistent, duration)
+              exchange = {:type => :queue, :name => RESPONSE_QUEUE, :options => {:durable => true, :no_declare => @secure}}
+              @broker.publish(exchange, r, :persistent => true, :mandatory => true, :log_filter => [:tries, :persistent, :duration])
+            end
+          rescue RightAMQP::HABrokerClient::NoConnectedBrokers => e
+            Log.error("Failed to publish result of dispatched request #{request.trace}", e)
+          rescue Exception => e
+            Log.error("Failed to publish result of dispatched request #{request.trace}", e, :trace)
+            @exceptions.track("publish response", e)
+          ensure
+            header.ack if header
+            @pending_dispatches = [@pending_dispatches - 1, 0].max
           end
-        rescue RightAMQP::HABrokerClient::NoConnectedBrokers => e
-          Log.error("Failed to publish result of dispatched request #{request.trace}", e)
-        rescue Exception => e
-          Log.error("Failed to publish result of dispatched request #{request.trace}", e, :trace)
-          @exceptions.track("publish response", e)
+          r # For unit tests
         end
-        r # For unit tests
-      end
 
-      # Process request and send response, if any
-      if @single_threaded
-        @em.next_tick { callback.call(operation.call) }
-      else
-        @em.defer(operation, callback)
+        # Process request and send response, if any
+        begin
+          ack_deferred = true
+          if @single_threaded
+            @em.next_tick { callback.call(operation.call) }
+          else
+            @em.defer(operation, callback)
+          end
+        rescue Exception
+          header.ack if header
+          raise
+        end
+      ensure
+        header.ack unless ack_deferred || header.nil?
       end
     end
 
@@ -252,7 +191,7 @@ module RightScale
     # === Return
     # (Integer|nil):: Age in seconds of youngest dispatch, or nil if none
     def dispatch_age
-      age = Time.now.to_i - @last_request_dispatch_time if @last_request_dispatch_time && @pending_dispatches > 0
+      Time.now.to_i - @last_request_dispatch_time if @last_request_dispatch_time && @pending_dispatches > 0
     end
 
     # Get dispatcher statistics
@@ -262,7 +201,7 @@ module RightScale
     #
     # === Return
     # stats(Hash):: Current statistics:
-    #   "cached"(Hash|nil):: Number of dispatched requests cached and age of youngest and oldest,
+    #   "dispatched cache"(Hash|nil):: Number of dispatched requests cached and age of youngest and oldest,
     #     or nil if empty
     #   "exceptions"(Hash|nil):: Exceptions raised per category, or nil if none
     #     "total"(Integer):: Total for category
@@ -282,12 +221,12 @@ module RightScale
         }
       end
       stats = {
-        "cached"        => (@dispatched.stats if @dup_check),
-        "exceptions"    => @exceptions.stats,
-        "pending"       => pending,
-        "rejects"       => @rejects.all,
-        "requests"      => @requests.all,
-        "response time" => @requests.avg_duration
+        "dispatched cache" => (@dispatched_cache.stats if @dispatched_cache),
+        "exceptions"       => @exceptions.stats,
+        "pending"          => pending,
+        "rejects"          => @rejects.all,
+        "requests"         => @requests.all,
+        "response time"    => @requests.avg_duration
       }
       reset_stats if reset
       stats
@@ -311,30 +250,30 @@ module RightScale
     #
     # === Parameters
     # actor(Actor):: Actor that failed to process request
-    # method(String):: Name of actor method being dispatched to
+    # method(Symbol):: Name of actor method being dispatched to
     # request(Packet):: Packet that dispatcher is acting upon
-    # e(Exception):: Exception that was raised
+    # exception(Exception):: Exception that was raised
     #
     # === Return
-    # error(String):: Error description for this exception
-    def handle_exception(actor, method, request, e)
-      error = Log.format("Failed processing #{request.type}", e, :trace)
-      Log.error(error)
+    # (String):: Error description for this exception
+    def handle_exception(actor, method, request, exception)
+      error = "Could not handle #{request.type} request"
+      Log.error(error, exception, :trace)
       begin
         if actor && actor.class.exception_callback
           case actor.class.exception_callback
           when Symbol, String
-            actor.send(actor.class.exception_callback, method.to_sym, request, e)
+            actor.send(actor.class.exception_callback, method, request, exception)
           when Proc
-            actor.instance_exec(method.to_sym, request, e, &actor.class.exception_callback)
+            actor.instance_exec(method, request, exception, &actor.class.exception_callback)
           end
         end
-        @exceptions.track(request.type, e)
-      rescue Exception => e2
-        Log.error("Failed handling error for #{request.type}", e2, :trace)
-        @exceptions.track(request.type, e2) rescue nil
+        @exceptions.track(request.type, exception)
+      rescue Exception => e
+        Log.error("Failed handling error for #{request.type}", e, :trace)
+        @exceptions.track(request.type, e) rescue nil
       end
-      error
+      Log.format(error, exception)
     end
 
   end # Dispatcher