right_agent 0.5.1

data/lib/right_agent/security/rsa_key_pair.rb

@@ -0,0 +1,76 @@
+#
+# Copyright (c) 2009-2011 RightScale Inc
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+module RightScale
+
+  # Allows generating RSA key pairs and extracting public key component
+  # Note: Creating a RSA key pair can take a fair amount of time (seconds)
+  class RsaKeyPair
+    
+    DEFAULT_LENGTH = 2048
+
+    # Underlying OpenSSL keys
+    attr_reader :raw_key
+
+    # Create new RSA key pair using 'length' bits
+    def initialize(length = DEFAULT_LENGTH)
+      @raw_key = OpenSSL::PKey::RSA.generate(length)
+    end
+
+    # Does key pair include private key?
+    def has_private?
+      raw_key.private?
+    end
+    
+    # New RsaKeyPair instance with identical public key but no private key
+    def to_public
+      RsaKeyPair.from_data(raw_key.public_key.to_pem)
+    end
+    
+    # Key material in PEM format
+    def data
+      raw_key.to_pem
+    end
+    alias :to_s :data
+    
+    # Load key pair previously serialized via 'data'    
+    def self.from_data(data)
+      res = RsaKeyPair.allocate
+      res.instance_variable_set(:@raw_key, OpenSSL::PKey::RSA.new(data)) 
+      res
+    end
+
+    # Load key from file
+    def self.load(file)
+      from_data(File.read(file))
+    end
+    
+    # Save key to file in PEM format
+    def save(file)
+      File.open(file, "w") do |f|
+        f.write(@raw_key.to_pem)
+      end
+    end
+
+  end # RsaKeyPair
+
+end # RightScale

data/lib/right_agent/security/signature.rb

@@ -0,0 +1,86 @@
+#
+# Copyright (c) 2009-2011 RightScale Inc
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+if RUBY_VERSION < '1.8.7'
+  RightScale::PKCS7 = OpenSSL::PKCS7::PKCS7
+else
+  RightScale::PKCS7 = OpenSSL::PKCS7
+end
+
+module RightScale
+
+  # Signature that can be validated against certificates
+  class Signature
+    
+    FLAGS = OpenSSL::PKCS7::NOCERTS || OpenSSL::PKCS7::BINARY || OpenSSL::PKCS7::NOATTR || OpenSSL::PKCS7::NOSMIMECAP || OpenSSL::PKCS7::DETACH
+
+    # Create signature using certificate and key pair.
+    #
+    # === Parameters
+    # data(String):: Data to be signed
+    # cert(Certificate):: Certificate used for signature
+    # key(RsaKeyPair):: Key pair used for signature
+    def initialize(data, cert, key)
+      @p7 = OpenSSL::PKCS7.sign(cert.raw_cert, key.raw_key, data, [], FLAGS)
+      @store = OpenSSL::X509::Store.new
+    end
+    
+    # Load signature from previously serialized data
+    #
+    # === Parameters
+    # data(String):: Serialized data
+    #
+    # === Return
+    # sig(Signature):: Signature for data
+    def self.from_data(data)
+      sig = Signature.allocate
+      sig.instance_variable_set(:@p7, RightScale::PKCS7.new(data))
+      sig.instance_variable_set(:@store, OpenSSL::X509::Store.new)
+      sig
+    end
+
+    # Check whether signature was created using cert
+    #
+    # === Parameters
+    # cert(Certificate):: Certificate
+    #
+    # === Return
+    # (Boolean):: true if created using given cert, otherwise false
+    def match?(cert)
+      @p7.verify([cert.raw_cert], @store, nil, OpenSSL::PKCS7::NOVERIFY)
+    end
+
+    # Signature data in PEM or DER format
+    #
+    # === Parameters
+    # format(Symbol):: Encode format: :pem or :der, defaults to :pem
+    #
+    # === Return
+    # (String):: Signature
+    def data(format = :pem)
+      format == :pem ? @p7.to_pem : @p7.to_der
+    end
+    alias :to_s :data
+
+  end # Signature
+  
+end # RightScale

data/lib/right_agent/security/static_certificate_store.rb

@@ -0,0 +1,69 @@
+#
+# Copyright (c) 2009-2011 RightScale Inc
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+module RightScale
+
+  # Simple certificate store, serves a static set of certificates
+  class StaticCertificateStore
+    
+    # Initialize store
+    #
+    # === Parameters
+    # signer_certs(Array|Certificate):: Signer certificate(s) used when loading data to
+    #   check the digital signature. The signature associated with the serialized data
+    #   needs to match with one of the signer certificates for loading to succeed.
+    # recipients_certs(Array|Certificate):: Recipient certificate(s) used when serializing
+    #   data for encryption. Loading the data can only be done through serializers that
+    #   have been initialized with a certificate that's in the recipient certificates
+    #   if encryption is enabled.
+    def initialize(signer_certs, recipients_certs)
+      signer_certs = [ signer_certs ] unless signer_certs.respond_to?(:each)
+      @signer_certs = signer_certs 
+      recipients_certs = [ recipients_certs ] unless recipients_certs.respond_to?(:each)
+      @recipients_certs = recipients_certs
+    end
+    
+    # Retrieve signer certificates
+    #
+    # === Parameters
+    # id(String):: Serialized identity of signer, ignored
+    #
+    # === Return
+    # (Array):: Signer certificates
+    def get_signer(id)
+      @signer_certs
+    end
+
+    # Retrieve recipient certificates that will be able to decrypt the serialized data
+    #
+    # === Parameters
+    # packet(RightScale::Packet):: Packet containing recipient identity, ignored
+    #
+    # === Return
+    # (Array):: Recipient certificates
+    def get_recipients(packet)
+      @recipients_certs
+    end
+    
+  end # StaticCertificateStore
+
+end # RightScale

data/lib/right_agent/sender.rb

@@ -0,0 +1,937 @@
+#
+# Copyright (c) 2009-2011 RightScale Inc
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+module RightScale
+
+  # This class allows sending requests to agents without having to run a local mapper
+  # It is used by Actor.request which is used by actors that need to send requests to remote agents
+  # If requested, it will queue requests when there are no broker connections
+  # All requests go through the mapper for security purposes
+  class Sender
+
+    include StatsHelper
+
+    # Minimum number of seconds between restarts of the inactivity timer
+    MIN_RESTART_INACTIVITY_TIMER_INTERVAL = 60
+
+    # Number of seconds to wait for ping response from a mapper when checking connectivity
+    PING_TIMEOUT = 30
+
+    # Factor used on each retry iteration to achieve exponential backoff
+    RETRY_BACKOFF_FACTOR = 4
+
+    # Maximum seconds to wait before starting flushing offline queue when disabling offline mode
+    MAX_QUEUE_FLUSH_DELAY = 120 # 2 minutes
+
+    # Maximum number of offline queued requests before triggering restart vote
+    MAX_QUEUED_REQUESTS = 1000
+
+    # Number of seconds that should be spent in offline mode before triggering a restart vote
+    RESTART_VOTE_DELAY = 900 # 15 minutes
+
+    # (EM::Timer) Timer while waiting for mapper ping response
+    attr_accessor :pending_ping
+  
+    # (Hash) Pending requests; key is request token and value is a hash
+    #   :response_handler(Proc):: Block to be activated when response is received
+    #   :receive_time(Time):: Time when message was received
+    #   :request_kind(String):: Kind of Sender request, optional
+    #   :retry_parent(String):: Token for parent request in a retry situation, optional
+    attr_accessor :pending_requests
+
+    # (HABrokerClient) High availability AMQP broker client
+    attr_accessor :broker
+
+    # (String) Identity of the associated agent
+    attr_reader :identity
+
+    # Accessor for use by actor
+    #
+    # === Return
+    # (Sender):: This sender instance if defined, otherwise nil
+    def self.instance
+      @@instance if defined?(@@instance)
+    end
+
+    # Initialize sender
+    #
+    # === Parameters
+    # agent(Agent):: Agent using this sender; uses its identity, broker, and following options:
+    #   :exception_callback(Proc):: Callback with following parameters that is activated on exception events:
+    #     exception(Exception):: Exception
+    #     message(Packet):: Message being processed
+    #     agent(Agent):: Reference to agent
+    #   :offline_queueing(Boolean):: Whether to queue request if currently not connected to any brokers,
+    #     also requires agent invocation of initialize_offline_queue and start_offline_queue methods below
+    #   :restart_callback(Proc):: Callback that is activated on each restart vote with votes being initiated
+    #     by offline queue exceeding MAX_QUEUED_REQUESTS or by repeated failures to access mapper when online
+    #   :retry_timeout(Numeric):: Maximum number of seconds to retry request before give up
+    #   :retry_interval(Numeric):: Number of seconds before initial request retry, increases exponentially
+    #   :time_to_live(Integer):: Number of seconds before a request expires and is to be ignored
+    #     by the receiver, 0 means never expire
+    #   :secure(Boolean):: true indicates to use Security features of rabbitmq to restrict agents to themselves
+    #   :single_threaded(Boolean):: true indicates to run all operations in one thread; false indicates
+    #     to do requested work on EM defer thread and all else, such as pings on main thread
+    def initialize(agent)
+      @agent = agent
+      @identity = @agent.identity
+      @options = @agent.options || {}
+      @broker = @agent.broker
+      @secure = @options[:secure]
+      @single_threaded = @options[:single_threaded]
+      @queueing_mode = :initializing
+      @queue_running = false
+      @queue_initializing = false
+      @queue = []
+      @restart_vote_count = 0
+      @retry_timeout = nil_if_zero(@options[:retry_timeout])
+      @retry_interval = nil_if_zero(@options[:retry_interval])
+      @ping_interval = @options[:ping_interval] || 0
+
+      # Only to be accessed from primary thread
+      @pending_requests = {}
+      @pending_ping = nil
+
+      reset_stats
+      @last_received = 0
+      @message_received_callbacks = []
+      restart_inactivity_timer if @ping_interval > 0
+      @@instance = self
+    end
+
+    # Update the time this agent last received a request or response message
+    # and restart the inactivity timer thus deferring the next connectivity check
+    # Also forward this message receipt notification to any callbacks that have registered
+    #
+    # === Block
+    # Optional block without parameters that is activated when a message is received
+    #
+    # === Return
+    # true:: Always return true
+    def message_received(&callback)
+      if block_given?
+        @message_received_callbacks << callback
+      else
+        @message_received_callbacks.each { |c| c.call }
+        if @ping_interval > 0
+          now = Time.now.to_i
+          if (now - @last_received) > MIN_RESTART_INACTIVITY_TIMER_INTERVAL
+            @last_received = now
+            restart_inactivity_timer
+          end
+        end
+      end
+      true
+    end
+
+    # Initialize the offline queue (should be called once)
+    # All requests sent prior to running this initialization are queued if offline
+    # queueing is enabled and then are sent once this initialization has run
+    # All requests following this call and prior to calling start_offline_queue
+    # are prepended to the request queue
+    #
+    # === Return
+    # true:: Always return true
+    def initialize_offline_queue
+      unless @queue_running || !@options[:offline_queueing]
+        @queue_running = true
+        @queue_initializing = true
+      end
+      true
+    end
+
+    # Switch offline queueing to online mode and flush all buffered messages
+    #
+    # === Return
+    # true:: Always return true
+    def start_offline_queue
+      if @queue_initializing
+        @queue_initializing = false
+        flush_queue unless @queueing_mode == :offline
+        @queueing_mode = :online if @queueing_mode == :initializing
+      end
+      true
+    end
+
+    # Send a request to a single target or multiple targets with no response expected other
+    # than routing failures
+    # Do not persist the request en route
+    # Enqueue the request if the target is not currently available
+    # Never automatically retry the request
+    # Set time-to-live to be forever
+    #
+    # === Parameters
+    # type(String):: Dispatch route for the request; typically identifies actor and action
+    # payload(Object):: Data to be sent with marshalling en route
+    # target(String|Hash):: Identity of specific target, hash for selecting potentially multiple
+    #   targets, or nil if routing solely using type
+    #   :tags(Array):: Tags that must all be associated with a target for it to be selected
+    #   :scope(Hash):: Scoping to be used to restrict routing
+    #     :account(Integer):: Restrict to agents with this account id
+    #     :deployment(Integer):: Restrict to agents with this deployment id
+    #     :shard(Integer):: Restrict to agents with this shard id, or if value is Packet::GLOBAL,
+    #       ones with no shard id
+    #   :selector(Symbol):: Which of the matched targets to be selected, either :any or :all,
+    #     defaults to :any
+    #
+    # === Block
+    # Optional block used to process routing response failures asynchronously with the following parameter:
+    #   result(Result):: Response with an OperationResult of RETRY, NON_DELIVERY, or ERROR,
+    #     use RightScale::OperationResult.from_results to decode
+    #
+    # === Return
+    # true:: Always return true
+    def send_push(type, payload = nil, target = nil, &callback)
+      build_push(:send_push, type, payload, target, &callback)
+    end
+
+    # Send a request to a single target or multiple targets with no response expected other
+    # than routing failures
+    # Persist the request en route to reduce the chance of it being lost at the expense of some
+    # additional network overhead
+    # Enqueue the request if the target is not currently available
+    # Never automatically retry the request
+    # Set time-to-live to be forever
+    #
+    # === Parameters
+    # type(String):: Dispatch route for the request; typically identifies actor and action
+    # payload(Object):: Data to be sent with marshalling en route
+    # target(String|Hash):: Identity of specific target, hash for selecting potentially multiple
+    #   targets, or nil if routing solely using type
+    #   :tags(Array):: Tags that must all be associated with a target for it to be selected
+    #   :scope(Hash):: Scoping to be used to restrict routing
+    #     :account(Integer):: Restrict to agents with this account id
+    #     :deployment(Integer):: Restrict to agents with this deployment id
+    #     :shard(Integer):: Restrict to agents with this shard id, or if value is Packet::GLOBAL,
+    #       ones with no shard id
+    #   :selector(Symbol):: Which of the matched targets to be selected, either :any or :all,
+    #     defaults to :any
+    #
+    # === Block
+    # Optional block used to process routing response failures asynchronously with the following parameter:
+    #   result(Result):: Response with an OperationResult of RETRY, NON_DELIVERY, or ERROR,
+    #     use RightScale::OperationResult.from_results to decode
+    #
+    # === Return
+    # true:: Always return true
+    def send_persistent_push(type, payload = nil, target = nil, &callback)
+      build_push(:send_persistent_push, type, payload, target, &callback)
+    end
+
+    # Send a request to a single target with a response expected
+    # Automatically retry the request if a response is not received in a reasonable amount of time
+    # or if there is a non-delivery response indicating the target is not currently available
+    # Timeout the request if a response is not received in time, typically configured to 2 minutes
+    # Because of retries there is the possibility of duplicated requests, and these are detected and
+    # discarded automatically unless the receiving agent is using a shared queue, in which case this
+    # method should not be used for actions that are non-idempotent
+    # Allow the request to expire per the agent's configured time-to-live, typically 1 minute
+    # Note that receiving a response does not guarantee that the request activity has actually
+    # completed since the request processing may involve other asynchronous requests
+    #
+    # === Parameters
+    # type(String):: Dispatch route for the request; typically identifies actor and action
+    # payload(Object):: Data to be sent with marshalling en route
+    # target(String|Hash):: Identity of specific target, hash for selecting targets of which one is picked
+    #   randomly, or nil if routing solely using type
+    #   :tags(Array):: Tags that must all be associated with a target for it to be selected
+    #   :scope(Hash):: Scoping to be used to restrict routing
+    #     :account(Integer):: Restrict to agents with this account id
+    #     :deployment(Integer):: Restrict to agents with this deployment id
+    #     :shard(Integer):: Restrict to agents with this shard id, or if value is Packet::GLOBAL,
+    #       ones with no shard id
+    #
+    # === Block
+    # Required block used to process response asynchronously with the following parameter:
+    #   result(Result):: Response with an OperationResult of SUCCESS, RETRY, NON_DELIVERY, or ERROR,
+    #     use RightScale::OperationResult.from_results to decode
+    #
+    # === Return
+    # true:: Always return true
+    def send_retryable_request(type, payload = nil, target = nil, &callback)
+      build_request(:send_retryable_request, type, payload, target, &callback)
+    end
+
+    # Send a request to a single target with a response expected
+    # Persist the request en route to reduce the chance of it being lost at the expense of some
+    # additional network overhead
+    # Enqueue the request if the target is not currently available
+    # Never automatically retry the request if there is the possibility of the request being duplicated
+    # Set time-to-live to be forever
+    # Note that receiving a response does not guarantee that the request activity has actually
+    # completed since the request processing may involve other asynchronous requests
+    #
+    # === Parameters
+    # type(String):: Dispatch route for the request; typically identifies actor and action
+    # payload(Object):: Data to be sent with marshalling en route
+    # target(String|Hash):: Identity of specific target, hash for selecting targets of which one is picked
+    #   randomly, or nil if routing solely using type
+    #   :tags(Array):: Tags that must all be associated with a target for it to be selected
+    #   :scope(Hash):: Scoping to be used to restrict routing
+    #     :account(Integer):: Restrict to agents with this account id
+    #     :deployment(Integer):: Restrict to agents with this deployment id
+    #     :shard(Integer):: Restrict to agents with this shard id, or if value is Packet::GLOBAL,
+    #       ones with no shard id
+    #
+    # === Block
+    # Required block used to process response asynchronously with the following parameter:
+    #   result(Result):: Response with an OperationResult of SUCCESS, RETRY, NON_DELIVERY, or ERROR,
+    #     use RightScale::OperationResult.from_results to decode
+    #
+    # === Return
+    # true:: Always return true
+    def send_persistent_request(type, payload = nil, target = nil, &callback)
+      build_request(:send_persistent_request, type, payload, target, &callback)
+    end
+
+    # Handle response to a request
+    # Only to be called from primary thread
+    #
+    # === Parameters
+    # response(Result):: Packet received as result of request
+    #
+    # === Return
+    # true:: Always return true
+    def handle_response(response)
+      token = response.token
+      if response.is_a?(Result)
+        if result = OperationResult.from_results(response)
+          if result.non_delivery?
+            @non_deliveries.update(result.content.nil? ? "nil" : result.content.inspect)
+          elsif result.error?
+            @result_errors.update(result.content.nil? ? "nil" : result.content.inspect)
+          end
+          @results.update(result.status)
+        else
+          @results.update(response.results.nil? ? "nil" : response.results)
+        end
+
+        if handler = @pending_requests[token]
+          if result && result.non_delivery? && handler[:request_kind] == :send_retryable_request &&
+             [OperationResult::TARGET_NOT_CONNECTED, OperationResult::TTL_EXPIRATION].include?(result.content)
+            # Log and ignore so that timeout retry mechanism continues
+            # Leave purging of associated request until final response, i.e., success response or retry timeout
+            Log.info("Non-delivery of <#{token}> because #{result.content}")
+          else
+            deliver(response, handler)
+          end
+        elsif result && result.non_delivery?
+          Log.info("Non-delivery of <#{token}> because #{result.content}")
+        else
+          Log.debug("No pending request for response #{response.to_s([])}")
+        end
+      end
+      true
+    end
+
+    # Switch to offline mode, in this mode requests are queued in memory
+    # rather than sent to the mapper
+    # Idempotent
+    #
+    # === Return
+    # true:: Always return true
+    def enable_offline_mode
+      if offline?
+        if @flushing_queue
+          # If we were in offline mode then switched back to online but are still in the
+          # process of flushing the in memory queue and are now switching to offline mode
+          # again then stop the flushing
+          @stop_flushing_queue = true
+        end
+      else
+        Log.info("[offline] Disconnect from broker detected, entering offline mode")
+        Log.info("[offline] Messages will be queued in memory until connection to broker is re-established")
+        @offlines.update
+        @queue ||= []  # ensure queue is valid without losing any messages when going offline
+        @queueing_mode = :offline
+        @restart_vote_timer ||= EM::Timer.new(RESTART_VOTE_DELAY) { vote_to_restart(timer_trigger=true) }
+      end
+    end
+
+    # Switch back to sending requests to mapper after in memory queue gets flushed
+    # Idempotent
+    #
+    # === Return
+    # true:: Always return true
+    def disable_offline_mode
+      if offline? && @queue_running
+        Log.info("[offline] Connection to broker re-established")
+        @offlines.finish
+        @restart_vote_timer.cancel if @restart_vote_timer
+        @restart_vote_timer = nil
+        @stop_flushing_queue = false
+        @flushing_queue = true
+        # Let's wait a bit not to flood the mapper
+        EM.add_timer(rand(MAX_QUEUE_FLUSH_DELAY)) { flush_queue } if @queue_running
+      end
+      true
+    end
+
+    # Get age of youngest pending request
+    #
+    # === Return
+    # age(Integer|nil):: Age in seconds of youngest request, or nil if no pending requests
+    def request_age
+      time = Time.now
+      age = nil
+      @pending_requests.each_value do |request|
+        seconds = time - request[:receive_time]
+        age = seconds.to_i if age.nil? || seconds < age
+      end
+      age
+    end
+
+    # Take any actions necessary to quiesce mapper interaction in preparation
+    # for agent termination but allow message receipt to continue
+    #
+    # === Return
+    # (Array):: Number of pending requests and age of youngest request
+    def terminate
+      @terminating = true
+      @ping_interval = 0
+      if @pending_ping
+        @pending_ping.cancel
+        @pending_ping = nil
+      end
+      if @timer
+        @timer.cancel
+        @timer = nil
+      end
+      if @restart_vote_timer
+        @restart_vote_timer.cancel
+        @restart_vote_timer = nil
+      end
+      [@pending_requests.size, request_age]
+    end
+
+    # Create displayable dump of unfinished request information
+    # Truncate list if there are more than 50 requests
+    #
+    # === Return
+    # info(Array(String)):: Receive time and token for each request in descending time order
+    def dump_requests
+      info = []
+      @pending_requests.each do |token, request|
+        info << "#{request[:receive_time].localtime} <#{token}>"
+      end
+      info.sort.reverse
+      info = info[0..49] + ["..."] if info.size > 50
+      info
+    end
+
+    # Get sender statistics
+    #
+    # === Parameters
+    # reset(Boolean):: Whether to reset the statistics after getting the current ones
+    #
+    # === Return
+    # stats(Hash):: Current statistics:
+    #   "exceptions"(Hash|nil):: Exceptions raised per category, or nil if none
+    #     "total"(Integer):: Total exceptions for this category
+    #     "recent"(Array):: Most recent as a hash of "count", "type", "message", "when", and "where"
+    #   "non-deliveries"(Hash|nil):: Non-delivery activity stats with keys "total", "percent", "last",
+    #     and 'rate' with percentage breakdown per reason, or nil if none
+    #   "offlines"(Hash|nil):: Offline activity stats with keys "total", "last", and "duration",
+    #     or nil if none
+    #   "pings"(Hash|nil):: Request activity stats with keys "total", "percent", "last", and "rate"
+    #     with percentage breakdown for "success" vs. "timeout", or nil if none
+    #   "request kinds"(Hash|nil):: Request kind activity stats with keys "total", "percent", and "last"
+    #     with percentage breakdown per kind, or nil if none
+    #   "requests"(Hash|nil):: Request activity stats with keys "total", "percent", "last", and "rate"
+    #     with percentage breakdown per request type, or nil if none
+    #   "requests pending"(Hash|nil):: Number of requests waiting for response and age of oldest, or nil if none
+    #   "response time"(Float):: Average number of seconds to respond to a request recently
+    #   "result errors"(Hash|nil):: Error result activity stats with keys "total", "percent", "last",
+    #     and 'rate' with percentage breakdown per error, or nil if none
+    #   "results"(Hash|nil):: Results activity stats with keys "total", "percent", "last", and "rate"
+    #     with percentage breakdown per operation result type, or nil if none
+    #   "retries"(Hash|nil):: Retry activity stats with keys "total", "percent", "last", and "rate"
+    #     with percentage breakdown per request type, or nil if none
+    def stats(reset = false)
+      offlines = @offlines.all
+      offlines.merge!("duration" => @offlines.avg_duration) if offlines
+      requests_pending = if @pending_requests.size > 0
+        now = Time.now.to_i
+        oldest = @pending_requests.values.inject(0) { |m, r| [m, now - r[:receive_time].to_i].max }
+        {"total" => @pending_requests.size, "oldest age" => oldest}
+      end
+      stats = {
+        "exceptions"       => @exceptions.stats,
+        "non-deliveries"   => @non_deliveries.all,
+        "offlines"         => offlines,
+        "pings"            => @pings.all,
+        "request kinds"    => @request_kinds.all,
+        "requests"         => @requests.all,
+        "requests pending" => requests_pending,
+        "response time"    => @requests.avg_duration,
+        "result errors"    => @result_errors.all,
+        "results"          => @results.all,
+        "retries"          => @retries.all
+      }
+      reset_stats if reset
+      stats
+    end
+
+    protected
+
+    # Reset dispatch statistics
+    #
+    # === Return
+    # true:: Always return true
+    def reset_stats
+      @pings = ActivityStats.new
+      @retries = ActivityStats.new
+      @requests = ActivityStats.new
+      @results = ActivityStats.new
+      @result_errors = ActivityStats.new
+      @non_deliveries = ActivityStats.new
+      @offlines = ActivityStats.new(measure_rate = false)
+      @request_kinds = ActivityStats.new(measure_rate = false)
+      @exceptions = ExceptionStats.new(@agent, @options[:exception_callback])
+      true
+    end
+
+    # Build and send Push packet
+    #
+    # === Parameters
+    # kind(Symbol):: Kind of push: :send_push or :send_persistent_push
+    # type(String):: Dispatch route for the request; typically identifies actor and action
+    # payload(Object):: Data to be sent with marshalling en route
+    # target(String|Hash):: Identity of specific target, or hash for selecting potentially multiple
+    #   targets, or nil if routing solely using type
+    #   :tags(Array):: Tags that must all be associated with a target for it to be selected
+    #   :scope(Hash):: Scoping to be used to restrict routing
+    #     :account(Integer):: Restrict to agents with this account id
+    #     :deployment(Integer):: Restrict to agents with this deployment id
+    #     :shard(Integer):: Restrict to agents with this shard id, or if value is Packet::GLOBAL,
+    #       ones with no shard id
+    #   :selector(Symbol):: Which of the matched targets to be selected, either :any or :all,
+    #     defaults to :any
+    #
+    # === Block
+    # Optional block used to process routing response failures asynchronously with the following parameter:
+    #   result(Result):: Response with an OperationResult of RETRY, NON_DELIVERY, or ERROR,
+    #     use RightScale::OperationResult.from_results to decode
+    #
+    # === Return
+    # true:: Always return true
+    #
+    # === Raise
+    # ArgumentError:: If target is invalid
+    def build_push(kind, type, payload = nil, target = nil, &callback)
+      validate_target(target, allow_selector = true)
+      if should_queue?
+        queue_request(:kind => kind, :type => type, :payload => payload, :target => target, :callback => callback)
+      else
+        method = type.split('/').last
+        received_at = @requests.update(method)
+        push = Push.new(type, payload)
+        push.from = @identity
+        push.token = AgentIdentity.generate
+        if target.is_a?(Hash)
+          push.tags = target[:tags] || []
+          push.scope = target[:scope]
+          push.selector = target[:selector] || :any
+        else
+          push.target = target
+        end
+        push.persistent = kind == :send_persistent_push
+        @request_kinds.update((push.selector == :all ? kind.to_s.sub(/push/, "fanout") : kind.to_s)[5..-1])
+        @pending_requests[push.token] = {
+          :response_handler => callback,
+          :receive_time => received_at,
+          :request_kind => kind
+        } if callback
+        publish(push)
+      end
+      true
+    end
+
+    # Build and send Request packet
+    #
+    # === Parameters
+    # kind(Symbol):: Kind of request: :send_retryable_request or :send_persistent_request
+    # type(String):: Dispatch route for the request; typically identifies actor and action
+    # payload(Object):: Data to be sent with marshalling en route
+    # target(String|Hash):: Identity of specific target, or hash for selecting targets of which one is picked
+    #   randomly, or nil if routing solely using type
+    #   :tags(Array):: Tags that must all be associated with a target for it to be selected
+    #   :scope(Hash):: Scoping to be used to restrict routing
+    #     :account(Integer):: Restrict to agents with this account id
+    #     :deployment(Integer):: Restrict to agents with this deployment id
+    #     :shard(Integer):: Restrict to agents with this shard id, or if value is Packet::GLOBAL,
+    #       ones with no shard id
+    #
+    # === Block
+    # Required block used to process response asynchronously with the following parameter:
+    #   result(Result):: Response with an OperationResult of SUCCESS, RETRY, NON_DELIVERY, or ERROR,
+    #     use RightScale::OperationResult.from_results to decode
+    #
+    # === Return
+    # true:: Always return true
+    #
+    # === Raise
+    # ArgumentError:: If target is invalid
+    def build_request(kind, type, payload, target, &callback)
+      validate_target(target, allow_selector = false)
+      if should_queue?
+        queue_request(:kind => kind, :type => type, :payload => payload, :target => target, :callback => callback)
+      else
+        method = type.split('/').last
+        token = AgentIdentity.generate
+        non_duplicate = kind == :send_persistent_request
+        received_at = @requests.update(method, token)
+        @request_kinds.update(kind.to_s[5..-1])
+
+        # Using next_tick to ensure on primary thread since using @pending_requests
+        EM.next_tick do
+          begin
+            request = Request.new(type, payload)
+            request.from = @identity
+            request.token = token
+            if target.is_a?(Hash)
+              request.tags = target[:tags] || []
+              request.scope = target[:scope]
+              request.selector = :any
+            else
+              request.target = target
+            end
+            request.expires_at = Time.now.to_i + @options[:time_to_live] if !non_duplicate && @options[:time_to_live] && @options[:time_to_live] != 0
+            request.persistent = non_duplicate
+            @pending_requests[token] = {
+              :response_handler => callback,
+              :receive_time => received_at,
+              :request_kind => kind}
+            if non_duplicate
+              publish(request)
+            else
+              publish_with_timeout_retry(request, token)
+            end
+          rescue Exception => e
+            Log.error("Failed to send #{type} #{kind.to_s}", e, :trace)
+            @exceptions.track(kind.to_s, e, request)
+          end
+        end
+      end
+      true
+    end
+
+    # Validate target argument of send
+    #
+    # === Parameters
+    # target(String|Hash):: Identity of specific target, or hash for selecting targets of which one is picked
+    # allow_selector(Boolean):: Whether to allow :selector
+    #
+    # === Return
+    # true:: Always return true
+    #
+    # === Raise
+    # ArgumentError:: If target is invalid
+    def validate_target(target, allow_selector)
+      if target.is_a?(Hash)
+        selector = allow_selector ? ":selector, " : ""
+        t = SerializationHelper.symbolize_keys(target)
+        if s = target[:scope]
+          if s.is_a?(Hash)
+            s = SerializationHelper.symbolize_keys(s)
+            if ([:account, :deployment, :shard] & s.keys).empty? && !s.empty?
+              raise ArgumentError, "Invalid target scope (#{t[:scope].inspect}), choices are :account, :deployment, and :shard allowed"
+            end
+            t[:scope] = s
+          else
+            raise ArgumentError, "Invalid target scope (#{t[:scope].inspect}), must be a hash of :account, :deployment, and/or :shard"
+          end
+        elsif (s = t[:selector]) && allow_selector
+          s = s.to_sym
+          unless [:any, :all].include?(s)
+            raise ArgumentError, "Invalid target selector (#{t[:selector].inspect}), choices are :any and :all"
+          end
+          t[:selector] = s
+        elsif !t.has_key?(:tags) && !t.empty?
+          raise ArgumentError, "Invalid target hash (#{target.inspect}), choices are #{selector}:tags and/or :scope"
+        end
+        target = t
+      elsif !target.nil? && !target.is_a?(String)
+        raise ArgumentError, "Invalid target (#{target.inspect}), choices are specific target name or a hash of #{selector}:tags and/or :scope"
+      end
+      true
+    end
+
+    # Publish request with one or more retries if do not receive a response in time
+    # Send timeout result if reach configured retry timeout limit
+    # Use exponential backoff with RETRY_BACKOFF_FACTOR for retry spacing
+    # Adjust retry interval by average response time to avoid adding to system load
+    # when system gets slow
+    #
+    # === Parameters
+    # request(Request):: Request to be sent
+    # parent(String):: Token for original request
+    # count(Integer):: Number of retries so far
+    # multiplier(Integer):: Multiplier for retry interval for exponential backoff
+    # elapsed(Integer):: Elapsed time in seconds since this request was first attempted
+    #
+    # === Return
+    # true:: Always return true
+    def publish_with_timeout_retry(request, parent, count = 0, multiplier = 1, elapsed = 0)
+      ids = publish(request)
+
+      if @retry_interval && @retry_timeout && parent && !ids.empty?
+        interval = [(@retry_interval * multiplier) + (@requests.avg_duration || 0), @retry_timeout - elapsed].min
+        EM.add_timer(interval) do
+          begin
+            if handler = @pending_requests[parent]
+              count += 1
+              elapsed += interval
+              if elapsed < @retry_timeout
+                request.tries << request.token
+                request.token = AgentIdentity.generate
+                @pending_requests[parent][:retry_parent] = parent if count == 1
+                @pending_requests[request.token] = @pending_requests[parent]
+                publish_with_timeout_retry(request, parent, count, multiplier * RETRY_BACKOFF_FACTOR, elapsed)
+                @retries.update(request.type.split('/').last)
+              else
+                Log.warning("RE-SEND TIMEOUT after #{elapsed.to_i} seconds for #{request.to_s([:tags, :target, :tries])}")
+                result = OperationResult.non_delivery(OperationResult::RETRY_TIMEOUT)
+                @non_deliveries.update(result.content)
+                handle_response(Result.new(request.token, request.reply_to, result, @identity))
+              end
+              check_connection(ids.first) if count == 1
+            end
+          rescue Exception => e
+            Log.error("Failed retry for #{request.token}", e, :trace)
+            @exceptions.track("retry", e, request)
+          end
+        end
+      end
+      true
+    end
+
+    # Publish request
+    # Use mandatory flag to request return of message if it cannot be delivered
+    #
+    # === Parameters
+    # request(Push|Request):: Packet to be sent
+    # ids(Array|nil):: Identity of specific brokers to choose from, or nil if any okay
+    #
+    # === Return
+    # ids(Array):: Identity of brokers published to
+    def publish(request, ids = nil)
+      begin
+        exchange = {:type => :fanout, :name => "request", :options => {:durable => true, :no_declare => @secure}}
+        ids = @broker.publish(exchange, request, :persistent => request.persistent, :mandatory => true,
+                              :log_filter => [:tags, :target, :tries, :persistent], :brokers => ids)
+      rescue HABrokerClient::NoConnectedBrokers => e
+        Log.error("Failed to publish request #{request.to_s([:tags, :target, :tries])}", e)
+        ids = []
+      rescue Exception => e
+        Log.error("Failed to publish request #{request.to_s([:tags, :target, :tries])}", e, :trace)
+        @exceptions.track("publish", e, request)
+        ids = []
+      end
+      ids
+    end
+
+    # Deliver the response and remove associated request(s) from pending
+    # Use defer thread instead of primary if not single threaded, consistent with dispatcher,
+    # so that all shared data is accessed from the same thread
+    # Do callback if there is an exception, consistent with agent identity queue handling
+    # Only to be called from primary thread
+    #
+    # === Parameters
+    # response(Result):: Packet received as result of request
+    # handler(Hash):: Associated request handler
+    #
+    # === Return
+    # true:: Always return true
+    def deliver(response, handler)
+      @requests.finish(handler[:receive_time], response.token)
+
+      @pending_requests.delete(response.token)
+      if parent = handler[:retry_parent]
+        @pending_requests.reject! { |k, v| k == parent || v[:retry_parent] == parent }
+      end
+
+      if handler[:response_handler]
+        EM.__send__(@single_threaded ? :next_tick : :defer) do
+          begin
+            handler[:response_handler].call(response)
+          rescue Exception => e
+            Log.error("Failed processing response {response.to_s([])}", e, :trace)
+            @exceptions.track("response", e, response)
+          end
+        end
+      end
+      true
+    end
+
+    # Check whether broker connection is usable by pinging a mapper via that broker
+    # Attempt to reconnect if ping does not respond in PING_TIMEOUT seconds
+    # Ignore request if already checking a connection
+    # Only to be called from primary thread
+    #
+    # === Parameters
+    # id(String):: Identity of specific broker to use to send ping, defaults to any
+    #   currently connected broker
+    #
+    # === Return
+    # true:: Always return true
+    def check_connection(id = nil)
+      unless @terminating || @pending_ping || (id && !@broker.connected?(id))
+        @pending_ping = EM::Timer.new(PING_TIMEOUT) do
+          begin
+            @pings.update("timeout")
+            @pending_ping = nil
+            Log.warning("Mapper ping via broker #{id} timed out after #{PING_TIMEOUT} seconds, attempting to reconnect")
+            host, port, index, priority, _ = @broker.identity_parts(id)
+            @agent.connect(host, port, index, priority, force = true)
+          rescue Exception => e
+            Log.error("Failed to reconnect to broker #{id}", e, :trace)
+            @exceptions.track("ping timeout", e)
+          end
+        end
+
+        handler = lambda do |_|
+          begin
+            if @pending_ping
+              @pings.update("success")
+              @pending_ping.cancel
+              @pending_ping = nil
+            end
+          rescue Exception => e
+            Log.error("Failed to cancel mapper ping", e, :trace)
+            @exceptions.track("cancel ping", e)
+          end
+        end
+
+        request = Request.new("/mapper/ping", nil, {:from => @identity, :token => AgentIdentity.generate})
+        @pending_requests[request.token] = {:response_handler => handler, :receive_time => Time.now}
+        ids = [id] if id
+        id = publish(request, ids).first
+      end
+      true
+    end
+
+    # Vote for restart and reset trigger
+    #
+    # === Parameters
+    # timer_trigger(Boolean):: true if vote was triggered by timer, false if it
+    #   was triggered by number of messages in in-memory queue
+    #
+    # === Return
+    # true:: Always return true
+    def vote_to_restart(timer_trigger)
+      if restart_vote = @options[:restart_callback]
+        restart_vote.call
+        if timer_trigger
+          @restart_vote_timer = EM::Timer.new(RESTART_VOTE_DELAY) { vote_to_restart(timer_trigger = true) }
+        else
+          @restart_vote_count = 0
+        end
+      end
+      true
+    end
+
+    # Is agent currently offline?
+    #
+    # === Return
+    # offline(Boolean):: true if agent is disconnected or not initialized
+    def offline?
+      offline = @queueing_mode == :offline || !@queue_running
+    end
+
+    # Start timer that waits for inactive messaging period to end before checking connectivity
+    #
+    # === Return
+    # true:: Always return true
+    def restart_inactivity_timer
+      @timer.cancel if @timer
+      @timer = EM::Timer.new(@ping_interval) do
+        begin
+          check_connection
+        rescue Exception => e
+          Log.error("Failed connectivity check", e, :trace)
+        end
+      end
+      true
+    end
+
+    # Should agent be queueing current request?
+    #
+    # === Return
+    # (Boolean):: true if should queue request, otherwise false
+    def should_queue?
+      @options[:offline_queueing] && offline? && !@flushing_queue
+    end
+
+    # Queue given request in memory
+    #
+    # === Parameters
+    # request(Hash):: Request to be stored
+    #
+    # === Return
+    # true:: Always return true
+    def queue_request(request)
+      Log.info("[offline] Queuing request: #{request.inspect}")
+      @restart_vote_count += 1 if @queue_running
+      vote_to_restart(timer_trigger = false) if @restart_vote_count >= MAX_QUEUED_REQUESTS
+      if @queue_initializing
+        # We are in the initialization callback, requests should be put at the head of the queue
+        @queue.unshift(request)
+      else
+        @queue << request
+      end
+      true
+    end
+
+    # Flush in memory queue of requests that were stored while in offline mode
+    # Do this asynchronously to allow for agents to respond to requests
+    # Once all in-memory requests have been flushed, switch off offline mode
+    #
+    # === Return
+    # true:: Always return true
+    def flush_queue
+      if @stop_flushing_queue
+        @stop_flushing_queue = false
+        @flushing_queue = false
+      else
+        Log.info("[offline] Starting to flush request queue of size #{@queue.size}") unless @queueing_mode == :initializing
+        unless @queue.empty?
+          r = @queue.shift
+          if r[:callback]
+            Sender.instance.__send__(r[:kind], r[:type], r[:payload], r[:target]) { |res| r[:callback].call(res) }
+          else
+            Sender.instance.__send__(r[:kind], r[:type], r[:payload], r[:target])
+          end
+        end
+        if @queue.empty?
+          Log.info("[offline] Request queue flushed, resuming normal operations") unless @queueing_mode == :initializing
+          @queueing_mode = :online
+          @flushing_queue = false
+        else
+          EM.next_tick { flush_queue }
+        end
+      end
+    end
+
+  end # Sender
+
+end # RightScale