right_agent 1.0.1 → 2.0.7

data/lib/right_agent/platform.rb

@@ -170,7 +170,7 @@ unless defined?(RightScale::Platform)
         end
         raise "No platform dispatch target found in #{binding.class} for " +
               "'#{meth.inspect}', tried " + dispatch_candidates(meth).join(', ') unless target
-        binding.__send__(target, *args)
+        binding.send(target, *args)
       end
 
       # @return [Controller] Platform-specific controller object

data/lib/right_agent/protocol_version_mixin.rb

@@ -0,0 +1,69 @@
+##
+# Copyright (c) 2014 RightScale Inc
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+module RightScale
+
+  # Mixin for testing for availability of features in agents based on protocol version
+  module ProtocolVersionMixin
+
+    # Mix this module into its own eigenclass to make the module-instance methods
+    # become callable as module methods
+    extend self
+
+    # Test whether given version of agent has the protocol version embedded in each
+    # packet (this is generally inferred by the version in the received packet not being
+    # Packet::DEFAULT_VERSION, which is true of all with version >= 12)
+    def can_put_version_in_packet?(version); version && version != 0 end
+
+    # Test whether given version of agent uses /mapper/query_tags rather than the
+    # deprecated TagQuery packet
+    def can_use_mapper_query_tags?(version); version && version >= 8 end
+
+    # Test whether given version of agent can handle a request that is being retried
+    # as indicated by a retries count in the Request packet
+    def can_handle_request_retries?(version); version && version >= 9 end
+
+    # Test whether given version of agent can handle serialized identity and queue name
+    # that does not incorporate 'nanite'
+    def can_handle_non_nanite_ids?(version); version && version >= 10 end
+
+    # Test whether given version of agent supports routing results to mapper response queue
+    # rather than to the identity queue of the mapper that routed the request
+    def can_route_to_response_queue?(version); version && version >= 10 end
+
+    # Test whether given version of agent can handle receipt of a result containing an
+    # OperationResult with MULTICAST status
+    def can_handle_multicast_result?(version); version && [10, 11].include?(version) end
+
+    # Test whether given version of agent can handle msgpack encoding
+    def can_handle_msgpack_result?(version); version && version >= 12 end
+
+    # Test whether given version of agent can handle receipt of a result containing an
+    # OperationResult with NON_DELIVERY status
+    def can_handle_non_delivery_result?(version); version && version >= 13 end
+
+    # Test whether given version of agent can handle HTTP communication mode
+    def can_handle_http?(version); version && version >= 23 end
+
+  end # ProtocolVersionMixin
+
+end # RightScale

data/lib/right_agent/{idempotent_request.rb → retryable_request.rb}

@@ -42,7 +42,7 @@ module RightScale
   # applied after the new :retry_delay_count is reached, and so on until :retry_delay
   # reaches :max_retry_delay which then is used as the interval until the default or
   # specified :timeout is reached. The default :timeout is 4 days.
-  class IdempotentRequest
+  class RetryableRequest
 
     include OperationResultHelper
     include EM::Deferrable
@@ -72,7 +72,7 @@ module RightScale
     # operation(String):: Request operation (e.g., '/booter/get_boot_bundle')
     # payload(Hash):: Request payload
     # options(Hash):: Request options
-    #   :targets(Array):: Targets from which to randomly choose one
+    #   :targets(Array):: Target agent identities from which to randomly choose one
     #   :retry_on_error(Boolean):: Whether request should be retried if recipient returned an error
     #   :retry_delay(Fixnum):: Number of seconds delay before initial retry with -1 meaning no delay,
     #     defaults to DEFAULT_RETRY_DELAY
@@ -86,8 +86,8 @@ module RightScale
     # === Raises
     # ArgumentError:: If operation or payload not specified
     def initialize(operation, payload, options = {})
-      raise ArgumentError.new("operation is required") unless @operation = operation
-      raise ArgumentError.new("payload is required") unless @payload = payload
+      raise ArgumentError.new("operation is required") unless (@operation = operation)
+      raise ArgumentError.new("payload is required") unless (@payload = payload)
       @retry_on_error = options[:retry_on_error] || false
       @timeout = options[:timeout] || DEFAULT_TIMEOUT
       @retry_delay = options[:retry_delay] || DEFAULT_RETRY_DELAY
@@ -105,7 +105,7 @@ module RightScale
     # === Return
     # true:: Always return true
     def run
-      Sender.instance.send_retryable_request(@operation, @payload, retrieve_target(@targets)) { |r| handle_response(r) }
+      Sender.instance.send_request(@operation, @payload, retrieve_target(@targets)) { |r| handle_response(r) }
       if @cancel_timer.nil? && @timeout > 0
         @cancel_timer = EM::Timer.new(@timeout) do
           msg = "Request #{@operation} timed out after #{@timeout} seconds"
@@ -187,9 +187,9 @@ module RightScale
     end
     
     def retrieve_target(targets)
-      targets[rand(0xffff) % targets.size] if targets
+      {:agent_id => targets[rand(0xffff) % targets.size]} if targets && targets.any?
     end
 
-  end # IdempotentRequest
+  end # RetryableRequest
 
 end # RightScale

data/lib/right_agent/scripts/agent_controller.rb

@@ -45,31 +45,33 @@
 #    rnac [options]
 #
 #    options:
-#      --start, -s AGENT   Start agent named AGENT
-#      --stop, -p AGENT    Stop agent named AGENT
-#      --stop-agent ID     Stop agent with serialized identity ID
-#      --kill, -k PIDFILE  Kill process with given process id file
-#      --killall, -K       Stop all running agents
-#      --status, -U        List running agents on local machine
-#      --identity, -i ID   Use base id ID to build agent's identity
-#      --token, -t TOKEN   Use token TOKEN to build agent's identity
-#      --prefix, -x PREFIX Use prefix PREFIX to build agent's identity
-#      --type TYPE         Use agent type TYPE to build agent's' identity,98589
-#                          defaults to AGENT with any trailing '_[0-9]+' removed
-#      --list, -l          List all configured agents
-#      --user, -u USER     Set AMQP user
-#      --pass, -p PASS     Set AMQP password
-#      --vhost, -v VHOST   Set AMQP vhost
-#      --host, -h HOST     Set AMQP server hostname
-#      --port, -P PORT     Set AMQP server port
-#      --cfg-dir, -c DIR   Set directory containing configuration for all agents
-#      --pid-dir, -z DIR   Set directory containing agent process id files
-#      --log-dir DIR       Set log directory
-#      --log-level LVL     Log level (debug, info, warning, error or fatal)
-#      --foreground, -f    Run agent in foreground
-#      --interactive, -I   Spawn an irb console after starting agent
-#      --test              Use test settings
-#      --help              Display help
+#      --start, -s AGENT     Start agent named AGENT
+#      --stop, -p AGENT      Stop agent named AGENT
+#      --stop-agent ID       Stop agent with serialized identity ID
+#      --kill, -k PIDFILE    Kill process with given process id file
+#      --killall, -K         Stop all running agents
+#      --status, -U          List running agents on local machine
+#      --identity, -i ID     Use this as base ID to build agent's identity
+#      --token, -t TOKEN     Use this token to build agent's identity with it plugging
+#                            directly in unless --secure-identity is specified
+#      --secure-identity, -S Derive token used in agent identity from given TOKEN and ID
+#      --prefix, -x PREFIX   Use this prefix to build agent's identity
+#      --type TYPE           Use this agent type to build agent's' identity;
+#                            defaults to AGENT with any trailing '_[0-9]+' removed
+#      --list, -l            List all configured agents
+#      --user, -u USER       Set AMQP user
+#      --pass, -p PASS       Set AMQP password
+#      --vhost, -v VHOST     Set AMQP vhost
+#      --host, -h HOST       Set AMQP server hostname
+#      --port, -P PORT       Set AMQP server port
+#      --cfg-dir, -c DIR     Set directory containing configuration for all agents
+#      --pid-dir, -z DIR     Set directory containing agent process id files
+#      --log-dir DIR         Set log directory
+#      --log-level LVL       Log level (debug, info, warning, error or fatal)
+#      --foreground, -f      Run agent in foreground
+#      --interactive, -I     Spawn an irb console after starting agent
+#      --test                Use test settings
+#      --help                Display help
 
 require 'rubygems'
 require 'optparse'
@@ -333,7 +335,7 @@ module RightScale
       EM.error_handler do |e|
         Log.error("EM block execution failed with exception", e, :trace)
         Log.error("\n\n===== Exiting due to EM block exception =====\n\n")
-        # Cannot rely on EM.stop at this point, so exit to give chance for monit restart
+        # Cannot rely on EM.stop at this point, so exit to give chance for monitor restart
         exit(1)
       end
 

data/lib/right_agent/scripts/agent_deployer.rb

@@ -7,14 +7,6 @@
 #     <agent name>/config.yml
 #   in platform-specific RightAgent configuration directory
 #
-# === Examples:
-#   Build configuration for agent named AGENT with default options:
-#     rad AGENT
-#
-#   Build configuration for agent named AGENT so it uses given AMQP settings:
-#     rad AGENT --user USER --pass PASSWORD --vhost VHOST --port PORT --host HOST
-#     rad AGENT -u USER -p PASSWORD -v VHOST -P PORT -h HOST
-#
 # === Usage:
 #    rad AGENT [options]
 #
@@ -22,13 +14,18 @@
 #      --root-dir, -r DIR       Set agent root directory (containing init, actors, and certs subdirectories)
 #      --cfg-dir, -c DIR        Set directory where generated configuration files for all agents are stored
 #      --pid-dir, -z DIR        Set directory containing process id file
-#      --identity, -i ID        Use base id ID to build agent's identity
-#      --token, -t TOKEN        Use token TOKEN to build agent's identity
-#      --prefix, -x PREFIX      Use prefix PREFIX to build agent's identity
-#      --type TYPE              Use agent type TYPE to build agent's' identity,
+#      --identity, -i ID        Use this as base ID to build agent's identity
+#      --token, -t TOKEN        Use this token to build agent's identity with it plugging
+#                               directly in unless --secure-identity is specified
+#      --secure-identity, -S    Derive token used in agent identity from given TOKEN and ID
+#      --prefix, -x PREFIX      Use this prefix to build agent's identity
+#      --type TYPE              Use this agent type to build agent's' identity;
 #                               defaults to AGENT with any trailing '_[0-9]+' removed
-#      --secure-identity, -S    Derive actual token from given TOKEN and ID
-#      --url                    Set agent AMQP connection URL (host, port, user, pass, vhost)
+#      --api-url, -a URL        Set URL for HTTP access to RightApi
+#      --account, -A ID         Set identifier for account owning this agent
+#      --shard, -s ID           Set identifier for database shard in which this agent is operating
+#      --mode, -m MODE          Set communication mode this agent is to use: :http or :amqp
+#      --url URL                Set agent AMQP connection URL (host, port, user, pass, vhost)
 #      --user, -u USER          Set agent AMQP username
 #      --password, -p PASS      Set agent AMQP password
 #      --vhost, -v VHOST        Set agent AMQP virtual host
@@ -43,12 +40,12 @@
 #      --retry-interval SEC     Set number of seconds before initial request retry, increases exponentially
 #      --check-interval SEC     Set number of seconds between failed connection checks, increases exponentially
 #      --ping-interval SEC      Set minimum number of seconds since last message receipt for the agent
-#                               to ping the mapper to check connectivity, 0 means disable ping
-#      --reconnect-interval SEC Set number of seconds between broker reconnect attempts
+#                               to ping the RightNet router to check connectivity, 0 means disable ping
+#      --reconnect-interval SEC Set number of seconds between HTTP or AMQP reconnect attempts
 #      --grace-timeout SEC      Set number of seconds before graceful termination times out
 #      --[no-]dup-check         Set whether to check for and reject duplicate requests, .e.g., due to retries
 #      --options, -o KEY=VAL    Set options that act as final override for any persisted configuration settings
-#      --monit, -m              Generate monit configuration file
+#      --monit                  Generate monit configuration file
 #      --test                   Build test deployment using default test settings
 #      --quiet, -Q              Do not produce output
 #      --help                   Display help
@@ -135,14 +132,10 @@ module RightScale
           options[:pid_dir] = d
         end
 
-        opts.on('-w', '--monit') do
+        opts.on('--monit') do
           options[:monit] = true
         end
 
-        opts.on('-S', '--secure-identity') do
-          options[:secure_identity] = true
-        end
-
         opts.on('--http-proxy PROXY') do |proxy|
           options[:http_proxy] = proxy
         end
@@ -187,6 +180,22 @@ module RightScale
           options[:prefetch] = count.to_i
         end
 
+        opts.on("-a", "--api-url URL") do |url|
+          options[:api_url] = url
+        end
+
+        opts.on('-A', '--account ID') do |id|
+          options[:account_id] = id.to_i
+        end
+
+        opts.on('-s', '--shard ID') do |id|
+          options[:shard_id] = id.to_i
+        end
+
+        opts.on('-m', '--mode MODE') do |mode|
+          options[:mode] = mode
+        end
+
         opts.on('-b', '--heartbeat SEC') do |sec|
           options[:heartbeat] = sec.to_i
         end
@@ -281,6 +290,11 @@ module RightScale
       cfg[:root_dir]           = AgentConfig.root_dir
       cfg[:pid_dir]            = AgentConfig.pid_dir
       cfg[:identity]           = options[:identity] if options[:identity]
+      cfg[:token]              = options[:token] if options[:token]
+      cfg[:api_url]            = options[:api_url] if options[:api_url]
+      cfg[:account_id]         = options[:account_id] if options[:account_id]
+      cfg[:shard_id]           = options[:shard_id] if options[:shard_id]
+      cfg[:mode]               = options[:mode] if options[:mode]
       cfg[:user]               = options[:user] if options[:user]
       cfg[:pass]               = options[:pass] if options[:pass]
       cfg[:vhost]              = options[:vhost] if options[:vhost]
@@ -344,6 +358,7 @@ check process #{agent_name}
       true
     end
 
+
     # Print error on console and exit abnormally
     #
     # === Parameters

data/lib/right_agent/scripts/common_parser.rb

@@ -58,6 +58,10 @@ module RightScale
         options[:token] = t
       end
 
+      opts.on("-S", "--secure-identity") do
+        options[:secure_identity] = true
+      end
+
       opts.on("-x", "--prefix PREFIX") do |p|
         options[:prefix] = p
       end
@@ -107,7 +111,7 @@ module RightScale
       true
     end
 
-    # Generate agent or mapper identity from options
+    # Generate agent identity from options
     # Build identity from base_id, token, prefix and agent name
     #
     # === Parameters
@@ -123,8 +127,11 @@ module RightScale
           puts "** Identity needs to be a positive integer"
           exit(1)
         end
-        token = options[:token]
-        token = RightScale::SecureIdentity.derive(base_id, options[:token]) if options[:secure_identity]
+        token = if options[:secure_identity]
+          RightScale::SecureIdentity.derive(base_id, options[:token])
+        else
+          options[:token]
+        end
         options[:identity] = AgentIdentity.new(options[:prefix] || 'rs', options[:agent_type], base_id, token).to_s
       end
     end

data/lib/right_agent/secure_identity.rb

@@ -48,7 +48,7 @@ module RightScale
     # as the name of an agent.
     #
     # Public tokens are generated by taking the SHA1 hash of the base ID and the
-    # auth token, separated by a delimeter. Thus a public token can always be
+    # auth token, separated by a delimiter. Thus a public token can always be
     # deterministically derived from its inputs.
     #
     # === Parameters

data/lib/right_agent/sender.rb

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2009-2012 RightScale Inc
+# Copyright (c) 2009-2013 RightScale Inc
 #
 # Permission is hereby granted, free of charge, to any person obtaining
 # a copy of this software and associated documentation files (the
@@ -22,541 +22,40 @@
 
 module RightScale
 
-  # This class allows sending requests to agents without having to run a local mapper
+  # TODO require target to be hash or nil only once cleanup string values in RightLink usage
+  # TODO require payload to be hash or nil only once cleanup RightApi and RightLink usage
+
+  # This class allows sending requests to agents via RightNet
   # It is used by Actor.request which is used by actors that need to send requests to remote agents
-  # If requested, it will queue requests when there are no broker connections
-  # All requests go through the mapper for security purposes
+  # If requested, it will queue requests when there are is no RightNet connection
   class Sender
 
-    class SendFailure < Exception; end
-    class TemporarilyOffline < Exception; end
-
-    # Request that is waiting for a response
-    class PendingRequest
-
-      # (Symbol) Kind of send request
-      attr_reader :kind
-
-      # (Time) Time when request message was received
-      attr_reader :receive_time
-
-      # (Proc) Block to be activated when response is received
-      attr_reader :response_handler
-
-      # (String) Token for parent request in a retry situation
-      attr_accessor :retry_parent
-
-      # (String) Non-delivery reason if any
-      attr_accessor :non_delivery
-
-      def initialize(kind, receive_time, response_handler)
-        @kind = kind
-        @receive_time = receive_time
-        @response_handler = response_handler
-        @retry_parent = nil
-        @non_delivery = nil
-      end
-
-    end # PendingRequest
-
-    # Cache for requests that are waiting for a response
-    # Automatically deletes push requests when get too old
-    # Retains non-push requests until explicitly deleted
-    class PendingRequests < Hash
-
-      # Kinds of send requests
-      REQUEST_KINDS = [:send_retryable_request, :send_persistent_request]
-
-      # Kinds of send pushes
-      PUSH_KINDS = [:send_push, :send_persistent_push]
-
-      # Maximum number of seconds to retain send pushes in cache
-      MAX_PUSH_AGE = 2 * 60
-
-      # Minimum number of seconds between push cleanups
-      MIN_CLEANUP_INTERVAL = 15
-
-      # Create cache
-      def initialize
-        @last_cleanup = Time.now
-        super
-      end
-
-      # Store pending request
-      #
-      # === Parameters
-      # token(String):: Generated message identifier
-      # request(PendingRequest):: Pending request
-      #
-      # === Return
-      # (PendingRequest):: Stored request
-      def []=(token, request)
-        now = Time.now
-        if (now - @last_cleanup) > MIN_CLEANUP_INTERVAL
-          self.reject! { |t, r| PUSH_KINDS.include?(r.kind) && (now - r.receive_time) > MAX_PUSH_AGE }
-          @last_cleanup = now
-        end
-        super
-      end
-
-      # Select cache entries of the given kinds
-      #
-      # === Parameters
-      # kinds(Array):: Kind of requests to be included
-      #
-      # === Return
-      # (Hash):: Requests of specified kind
-      def kind(kinds)
-        self.reject { |t, r| !kinds.include?(r.kind) }
-      end
-
-      # Get age of youngest pending request
-      #
-      # === Return
-      # age(Integer):: Age of youngest request
-      def youngest_age
-        now = Time.now
-        age = nil
-        self.each_value do |r|
-          seconds = (now - r.receive_time).to_i
-          age = seconds if age.nil? || seconds < age
-        end
-        age
-      end
-
-      # Get age of oldest pending request
-      #
-      # === Return
-      # age(Integer):: Age of oldest request
-      def oldest_age
-        now = Time.now
-        age = nil
-        self.each_value do |r|
-          seconds = (now - r.receive_time).to_i
-          age = seconds if age.nil? || seconds > age
-        end
-        age
-      end
-
-    end # PendingRequests
-
-    # Queue for storing requests while disconnected from broker and then sending
-    # them when successfully reconnect
-    class OfflineHandler
-
-      # Maximum seconds to wait before starting flushing offline queue when disabling offline mode
-      MAX_QUEUE_FLUSH_DELAY = 2 * 60
-
-      # Maximum number of offline queued requests before triggering restart vote
-      MAX_QUEUED_REQUESTS = 100
-
-      # Number of seconds that should be spent in offline mode before triggering a restart vote
-      RESTART_VOTE_DELAY = 15 * 60
-
-      # (Symbol) Current queue state with possible values:
-      #   Value          Description                Action               Next state
-      #   :created       Queue created              init                 :initializing
-      #   :initializing  Agent still initializing   start                :running
-      #   :running       Queue has been started     disable when offline :flushing
-      #   :flushing      Sending queued requests    enable               :running
-      #   :terminating   Agent terminating
-      attr_reader :state
-
-      # (Symbol) Current offline handling mode with possible values:
-      #   Value          Description
-      #   :initializing  Agent still initializing
-      #   :online        Agent connected to broker
-      #   :offline       Agent disconnected from broker
-      attr_reader :mode
-
-      # (Array) Offline queue
-      attr_accessor :queue
-
-      # Create offline queue
-      #
-      # === Parameters
-      # restart_callback(Proc):: Callback that is activated on each restart vote with votes being initiated
-      #   by offline queue exceeding MAX_QUEUED_REQUESTS
-      # offline_stats(RightSupport::Stats::Activity):: Offline queue tracking statistics
-      def initialize(restart_callback, offline_stats)
-        @restart_vote = restart_callback
-        @restart_vote_timer = nil
-        @restart_vote_count = 0
-        @offline_stats = offline_stats
-        @state = :created
-        @mode = :initializing
-        @queue = []
-      end
-
-      # Initialize the offline queue
-      # All requests sent prior to running this initialization are queued
-      # and then are sent once this initialization has run
-      # All requests following this call and prior to calling start
-      # are prepended to the request queue
-      #
-      # === Return
-      # true:: Always return true
-      def init
-        @state = :initializing if @state == :created
-        true
-      end
-
-      # Switch to online mode and send all buffered messages
-      #
-      # === Return
-      # true:: Always return true
-      def start
-        if @state == :initializing
-          if @mode == :offline
-            @state = :running
-          else
-            @state = :flushing
-            flush
-          end
-          @mode = :online if @mode == :initializing
-        end
-        true
-      end
-
-      # Is agent currently offline?
-      #
-      # === Return
-      # (Boolean):: true if agent offline, otherwise false
-      def offline?
-        @mode == :offline || @state == :created
-      end
-
-      # In request queueing mode?
-      #
-      # === Return
-      # (Boolean):: true if should queue request, otherwise false
-      def queueing?
-        offline? && @state != :flushing
-      end
-
-      # Switch to offline mode
-      # In this mode requests are queued in memory rather than sent to the mapper
-      # Idempotent
-      #
-      # === Return
-      # true:: Always return true
-      def enable
-        if offline?
-          if @state == :flushing
-            # If we were in offline mode then switched back to online but are still in the
-            # process of flushing the in-memory queue and are now switching to offline mode
-            # again then stop the flushing
-            @state = :running
-          end
-        else
-          Log.info("[offline] Disconnect from broker detected, entering offline mode")
-          Log.info("[offline] Messages will be queued in memory until connection to broker is re-established")
-          @offline_stats.update
-          @queue ||= []  # Ensure queue is valid without losing any messages when going offline
-          @mode = :offline
-          start_timer
-        end
-        true
-      end
-
-      # Switch back to sending requests to mapper after in-memory queue gets flushed
-      # Idempotent
-      #
-      # === Return
-      # true:: Always return true
-      def disable
-        if offline? && @state != :created
-          Log.info("[offline] Connection to broker re-established")
-          @offline_stats.finish
-          cancel_timer
-          @state = :flushing
-          # Wait a bit to avoid flooding the mapper
-          EM.add_timer(rand(MAX_QUEUE_FLUSH_DELAY)) { flush }
-        end
-        true
-      end
-
-      # Queue given request in memory
-      #
-      # === Parameters
-      # request(Hash):: Request to be stored
-      #
-      # === Return
-      # true:: Always return true
-      def queue_request(kind, type, payload, target, callback)
-        request = {:kind => kind, :type => type, :payload => payload, :target => target, :callback => callback}
-        Log.info("[offline] Queuing request: #{request.inspect}")
-        vote_to_restart if (@restart_vote_count += 1) >= MAX_QUEUED_REQUESTS
-        if @state == :initializing
-          # We are in the initialization callback, requests should be put at the head of the queue
-          @queue.unshift(request)
-        else
-          @queue << request
-        end
-        true
-      end
-
-      # Prepare for agent termination
-      #
-      # === Return
-      # true:: Always return true
-      def terminate
-        @state = :terminating
-        cancel_timer
-        true
-      end
-
-      protected
-
-      # Send any requests that were queued while in offline mode
-      # Do this asynchronously to allow for agents to respond to requests
-      # Once all in-memory requests have been flushed, switch off offline mode
-      #
-      # === Return
-      # true:: Always return true
-      def flush
-        if @state == :flushing
-          Log.info("[offline] Starting to flush request queue of size #{@queue.size}") unless @mode == :initializing
-          unless @queue.empty?
-            r = @queue.shift
-            if r[:callback]
-              Sender.instance.__send__(r[:kind], r[:type], r[:payload], r[:target]) { |result| r[:callback].call(result) }
-            else
-              Sender.instance.__send__(r[:kind], r[:type], r[:payload], r[:target])
-            end
-          end
-          if @queue.empty?
-            Log.info("[offline] Request queue flushed, resuming normal operations") unless @mode == :initializing
-            @mode = :online
-            @state = :running
-          else
-            EM.next_tick { flush }
-          end
-        end
-        true
-      end
-
-      # Vote for restart and reset trigger
-      #
-      # === Parameters
-      # timer_trigger(Boolean):: true if vote was triggered by timer, false if it
-      #   was triggered by number of messages in in-memory queue
-      #
-      # === Return
-      # true:: Always return true
-      def vote_to_restart(timer_trigger = false)
-        if @restart_vote
-          @restart_vote.call
-          if timer_trigger
-            start_timer
-          else
-            @restart_vote_count = 0
-          end
-        end
-        true
-      end
-
-      # Start restart vote timer
-      #
-      # === Return
-      # true:: Always return true
-      def start_timer
-        if @restart_vote && @state != :terminating
-          @restart_vote_timer ||= EM::Timer.new(RESTART_VOTE_DELAY) { vote_to_restart(timer_trigger = true) }
-        end
-        true
-      end
-
-      # Cancel restart vote timer
-      #
-      # === Return
-      # true:: Always return true
-      def cancel_timer
-        if @restart_vote_timer
-          @restart_vote_timer.cancel
-          @restart_vote_timer = nil
-          @restart_vote_count = 0
-        end
-        true
-      end
-
-    end # OfflineHandler
-
-    # Broker connectivity checker
-    # Checks connectivity when requested
-    class ConnectivityChecker
-
-      # Minimum number of seconds between restarts of the inactivity timer
-      MIN_RESTART_INACTIVITY_TIMER_INTERVAL = 60
-
-      # Number of seconds to wait for ping response from a mapper when checking connectivity
-      PING_TIMEOUT = 30
-
-      # Default maximum number of consecutive ping timeouts before attempt to reconnect
-      MAX_PING_TIMEOUTS = 3
-
-      # (EM::Timer) Timer while waiting for mapper ping response
-      attr_accessor :ping_timer
-
-      def initialize(sender, check_interval, ping_stats, exception_stats)
-        @sender = sender
-        @check_interval = check_interval
-        @ping_timeouts = {}
-        @ping_timer = nil
-        @ping_stats = ping_stats
-        @exception_stats = exception_stats
-        @last_received = Time.now
-        @message_received_callbacks = []
-        restart_inactivity_timer if @check_interval > 0
-      end
-
-      # Update the time this agent last received a request or response message
-      # and restart the inactivity timer thus deferring the next connectivity check
-      # Also forward this message receipt notification to any callbacks that have registered
-      #
-      # === Block
-      # Optional block without parameters that is activated when a message is received
-      #
-      # === Return
-      # true:: Always return true
-      def message_received(&callback)
-        if block_given?
-          @message_received_callbacks << callback
-        else
-          @message_received_callbacks.each { |c| c.call }
-          if @check_interval > 0
-            now = Time.now
-            if (now - @last_received) > MIN_RESTART_INACTIVITY_TIMER_INTERVAL
-              @last_received = now
-              restart_inactivity_timer
-            end
-          end
-        end
-        true
-      end
-
-      # Check whether broker connection is usable by pinging a mapper via that broker
-      # Attempt to reconnect if ping does not respond in PING_TIMEOUT seconds and
-      # if have reached timeout limit
-      # Ignore request if already checking a connection
-      #
-      # === Parameters
-      # id(String):: Identity of specific broker to use to send ping, defaults to any
-      #   currently connected broker
-      # max_ping_timeouts(Integer):: Maximum number of ping timeouts before attempt
-      #   to reconnect, defaults to MAX_PING_TIMEOUTS
-      #
-      # === Return
-      # true:: Always return true
-      def check(id = nil, max_ping_timeouts = MAX_PING_TIMEOUTS)
-        unless @terminating || @ping_timer || (id && !@sender.broker.connected?(id))
-          @ping_id = id
-          @ping_timer = EM::Timer.new(PING_TIMEOUT) do
-            if @ping_id
-              begin
-                @ping_stats.update("timeout")
-                @ping_timer = nil
-                @ping_timeouts[@ping_id] = (@ping_timeouts[@ping_id] || 0) + 1
-                if @ping_timeouts[@ping_id] >= max_ping_timeouts
-                  Log.error("Mapper ping via broker #{@ping_id} timed out after #{PING_TIMEOUT} seconds and now " +
-                            "reached maximum of #{max_ping_timeouts} timeout#{max_ping_timeouts > 1 ? 's' : ''}, " +
-                            "attempting to reconnect")
-                  host, port, index, priority = @sender.broker.identity_parts(@ping_id)
-                  @sender.agent.connect(host, port, index, priority, force = true)
-                else
-                  Log.warning("Mapper ping via broker #{@ping_id} timed out after #{PING_TIMEOUT} seconds")
-                end
-              rescue Exception => e
-                Log.error("Failed to reconnect to broker #{@ping_id}", e, :trace)
-                @exception_stats.track("ping timeout", e)
-              end
-            else
-              @ping_timer = nil
-            end
-          end
-
-          handler = lambda do |_|
-            begin
-              if @ping_timer
-                @ping_stats.update("success")
-                @ping_timer.cancel
-                @ping_timer = nil
-                @ping_timeouts[@ping_id] = 0
-                @ping_id = nil
-              end
-            rescue Exception => e
-              Log.error("Failed to cancel mapper ping", e, :trace)
-              @exception_stats.track("cancel ping", e)
-            end
-          end
-          request = Request.new("/mapper/ping", nil, {:from => @sender.identity, :token => AgentIdentity.generate})
-          @sender.pending_requests[request.token] = PendingRequest.new(:send_persistent_request, Time.now, handler)
-          ids = [@ping_id] if @ping_id
-          @ping_id = @sender.__send__(:publish, request, ids).first
-        end
-        true
-      end
-
-      # Prepare for agent termination
-      #
-      # === Return
-      # true:: Always return true
-      def terminate
-        @terminating = true
-        @check_interval = 0
-        if @ping_timer
-          @ping_timer.cancel
-          @ping_timer = nil
-        end
-        if @inactivity_timer
-          @inactivity_timer.cancel
-          @inactivity_timer = nil
-        end
-        true
-      end
-
-      protected
-
-      # Start timer that waits for inactive messaging period to end before checking connectivity
-      #
-      # === Return
-      # true:: Always return true
-      def restart_inactivity_timer
-        @inactivity_timer.cancel if @inactivity_timer
-        @inactivity_timer = EM::Timer.new(@check_interval) do
-          begin
-            check(id = nil, max_ping_timeouts = 1)
-          rescue Exception => e
-            Log.error("Failed connectivity check", e, :trace)
-            @exception_stats.track("check connectivity", e)
-          end
-        end
-        true
-      end
+    include OperationResultHelper
 
-    end # ConnectivityChecker
+    class SendFailure < RuntimeError; end
+    class TemporarilyOffline < RuntimeError; end
 
     # Factor used on each retry iteration to achieve exponential backoff
     RETRY_BACKOFF_FACTOR = 3
 
     # (PendingRequests) Requests waiting for a response
-    attr_accessor :pending_requests
+    attr_reader :pending_requests
 
-    # (OfflineHandler) Handler for requests when disconnected from broker
+    # (OfflineHandler) Handler for requests when client disconnected
     attr_reader :offline_handler
 
-    # (ConnectivityChecker) Broker connection checker
+    # (ConnectivityChecker) AMQP broker connection checker
     attr_reader :connectivity_checker
 
-    # (RightAMQP::HABrokerClient) High availability AMQP broker client
-    attr_accessor :broker
-
     # (String) Identity of the associated agent
     attr_reader :identity
 
     # (Agent) Associated agent
     attr_reader :agent
 
+    # (Symbol) RightNet communication mode: :http or :amqp
+    attr_reader :mode
+
     # For direct access to current sender
     #
     # === Return
@@ -568,36 +67,43 @@ module RightScale
     # Initialize sender
     #
     # === Parameters
-    # agent(Agent):: Agent using this sender; uses its identity, broker, and following options:
+    # agent(Agent):: Agent using this sender; uses its identity, client, and following options:
     #   :exception_callback(Proc):: Callback with following parameters that is activated on exception events:
     #     exception(Exception):: Exception
     #     message(Packet):: Message being processed
     #     agent(Agent):: Reference to agent
-    #   :offline_queueing(Boolean):: Whether to queue request if currently not connected to any brokers,
+    #   :offline_queueing(Boolean):: Whether to queue request if client currently disconnected,
     #     also requires agent invocation of initialize_offline_queue and start_offline_queue methods below,
-    #     as well as enable_offline_mode and disable_offline_mode as broker connections status changes
-    #   :ping_interval(Integer):: Minimum number of seconds since last message receipt to ping the mapper
+    #     as well as enable_offline_mode and disable_offline_mode as client connection status changes
+    #   :ping_interval(Integer):: Minimum number of seconds since last message receipt to ping RightNet
     #     to check connectivity, defaults to 0 meaning do not ping
     #   :restart_callback(Proc):: Callback that is activated on each restart vote with votes being initiated
-    #     by offline queue exceeding MAX_QUEUED_REQUESTS or by repeated failures to access mapper when online
+    #     by offline queue exceeding MAX_QUEUED_REQUESTS or by repeated failures to access RightNet when online
     #   :retry_timeout(Numeric):: Maximum number of seconds to retry request before give up
     #   :retry_interval(Numeric):: Number of seconds before initial request retry, increases exponentially
     #   :time_to_live(Integer):: Number of seconds before a request expires and is to be ignored
     #     by the receiver, 0 means never expire
+    #   :async_response(Boolean):: Whether to handle responses asynchronously or to handle them immediately
+    #     upon arrival (for use by applications that were written expecting asynchronous AMQP responses)
     #   :secure(Boolean):: true indicates to use Security features of rabbitmq to restrict agents to themselves
     def initialize(agent)
       @agent = agent
       @identity = @agent.identity
       @options = @agent.options || {}
-      @broker = @agent.broker
+      @mode = @agent.mode
+      @request_queue = @agent.request_queue
       @secure = @options[:secure]
       @retry_timeout = RightSupport::Stats.nil_if_zero(@options[:retry_timeout])
       @retry_interval = RightSupport::Stats.nil_if_zero(@options[:retry_interval])
       @pending_requests = PendingRequests.new
-
+      @terminating = nil
       reset_stats
       @offline_handler = OfflineHandler.new(@options[:restart_callback], @offline_stats)
-      @connectivity_checker = ConnectivityChecker.new(self, @options[:ping_interval] || 0, @ping_stats, @exception_stats)
+      @connectivity_checker = if @mode == :amqp
+        # Only need connectivity checker for AMQP broker since RightHttpClient does its own checking
+        # via periodic session renewal
+        ConnectivityChecker.new(self, @options[:ping_interval] || 0, @ping_stats, @exception_stats)
+      end
       @@instance = self
     end
 
@@ -622,7 +128,7 @@ module RightScale
     end
 
     # Switch to offline mode
-    # In this mode requests are queued in memory rather than sent to the mapper
+    # In this mode requests are queued in memory rather than sent
     # Idempotent
     #
     # === Return
@@ -631,7 +137,7 @@ module RightScale
       @offline_handler.enable if @options[:offline_queueing]
     end
 
-    # Switch back to sending requests to mapper after in memory queue gets flushed
+    # Switch back to sending requests after in memory queue gets flushed
     # Idempotent
     #
     # === Return
@@ -640,12 +146,12 @@ module RightScale
       @offline_handler.disable if @options[:offline_queueing]
     end
 
-    # Determine whether currently offline
+    # Determine whether currently connected to RightNet via client
     #
     # === Return
-    # (Boolean):: true if offline or if not connected to any brokers, otherwise false
-    def offline?
-      (@options[:offline_queueing] && @offline_handler.offline?) || @broker.connected.size == 0
+    # (Boolean):: true if offline or if client disconnected, otherwise false
+    def connected?
+      @mode == :http ? @agent.client.connected? : @agent.client.connected.size == 0
     end
 
     # Update the time this agent last received a request or response message
@@ -657,45 +163,7 @@ module RightScale
     # === Return
     # true:: Always return true
     def message_received(&callback)
-      @connectivity_checker.message_received(&callback)
-    end
-
-    # Send a request to a single target or multiple targets with no response expected other
-    # than routing failures
-    # Do not persist the request en route
-    # Enqueue the request if the target is not currently available
-    # Never automatically retry the request
-    # Set time-to-live to be forever
-    #
-    # === Parameters
-    # type(String):: Dispatch route for the request; typically identifies actor and action
-    # payload(Object):: Data to be sent with marshalling en route
-    # target(String|Hash):: Identity of specific target, hash for selecting potentially multiple
-    #   targets, or nil if routing solely using type
-    #   :tags(Array):: Tags that must all be associated with a target for it to be selected
-    #   :scope(Hash):: Scoping to be used to restrict routing
-    #     :account(Integer):: Restrict to agents with this account id
-    #     :shard(Integer):: Restrict to agents with this shard id, or if value is Packet::GLOBAL,
-    #       ones with no shard id
-    #   :selector(Symbol):: Which of the matched targets to be selected, either :any or :all,
-    #     defaults to :any
-    #
-    # === Block
-    # Optional block used to process routing responses asynchronously with the following parameter:
-    #   result(Result):: Response with an OperationResult of SUCCESS, RETRY, NON_DELIVERY, or ERROR,
-    #     with an initial SUCCESS response containing the targets to which the mapper published the
-    #     request and any additional responses indicating any failures to actually route the request
-    #     to those targets, use RightScale::OperationResult.from_results to decode
-    #
-    # === Return
-    # true:: Always return true
-    #
-    # === Raise
-    # SendFailure:: If publishing of request failed unexpectedly
-    # TemporarilyOffline:: If cannot publish request because currently not connected
-    #    to any brokers and offline queueing is disabled
-    def send_push(type, payload = nil, target = nil, &callback)
-      build_and_send_packet(:send_push, type, payload, target, callback)
+      @connectivity_checker.message_received(&callback) if @connectivity_checker
     end
 
     # Send a request to a single target or multiple targets with no response expected other
@@ -703,14 +171,16 @@ module RightScale
     # Persist the request en route to reduce the chance of it being lost at the expense of some
     # additional network overhead
     # Enqueue the request if the target is not currently available
-    # Never automatically retry the request
+    # Never automatically retry the request if there is the possibility of it being duplicated
     # Set time-to-live to be forever
     #
     # === Parameters
     # type(String):: Dispatch route for the request; typically identifies actor and action
     # payload(Object):: Data to be sent with marshalling en route
-    # target(String|Hash):: Identity of specific target, hash for selecting potentially multiple
-    #   targets, or nil if routing solely using type
+    # target(Hash|NilClass) Target for request, which may be a specific agent (using :agent_id),
+    #   potentially multiple targets (using :tags, :scope, :selector), or nil to route solely
+    #   using type:
+    #   :agent_id(String):: serialized identity of specific target
     #   :tags(Array):: Tags that must all be associated with a target for it to be selected
     #   :scope(Hash):: Scoping to be used to restrict routing
     #     :account(Integer):: Restrict to agents with this account id
@@ -718,23 +188,25 @@ module RightScale
     #       ones with no shard id
     #   :selector(Symbol):: Which of the matched targets to be selected, either :any or :all,
     #     defaults to :any
+    # token(String|NilClass):: Token uniquely identifying request; defaults to random generated
     #
     # === Block
     # Optional block used to process routing responses asynchronously with the following parameter:
     #   result(Result):: Response with an OperationResult of SUCCESS, RETRY, NON_DELIVERY, or ERROR,
-    #     with an initial SUCCESS response containing the targets to which the mapper published the
-    #     request and any additional responses indicating any failures to actually route the request
+    #     with an initial SUCCESS response containing the targets to which the request was sent
+    #     and any additional responses indicating any failures to actually route the request
     #     to those targets, use RightScale::OperationResult.from_results to decode
     #
     # === Return
     # true:: Always return true
     #
     # === Raise
-    # SendFailure:: If publishing of request failed unexpectedly
-    # TemporarilyOffline:: If cannot publish request because currently not connected
-    #    to any brokers and offline queueing is disabled
-    def send_persistent_push(type, payload = nil, target = nil, &callback)
-      build_and_send_packet(:send_persistent_push, type, payload, target, callback)
+    # ArgumentError:: If target invalid
+    # SendFailure:: If sending of request failed unexpectedly
+    # TemporarilyOffline:: If cannot send request because RightNet client currently disconnected
+    #   and offline queueing is disabled
+    def send_push(type, payload = nil, target = nil, token = nil, &callback)
+      build_and_send_packet(:send_push, type, payload, target, token, callback)
     end
 
     # Send a request to a single target with a response expected
@@ -750,13 +222,16 @@ module RightScale
     # === Parameters
     # type(String):: Dispatch route for the request; typically identifies actor and action
     # payload(Object):: Data to be sent with marshalling en route
-    # target(String|Hash):: Identity of specific target, hash for selecting targets of which one is picked
-    #   randomly, or nil if routing solely using type
+    # target(Hash|NilClass) Target for request, which may be a specific agent (using :agent_id),
+    #   one chosen randomly from potentially multiple targets (using :tags, :scope), or nil to
+    #   route solely using type:
+    #   :agent_id(String):: serialized identity of specific target
     #   :tags(Array):: Tags that must all be associated with a target for it to be selected
     #   :scope(Hash):: Scoping to be used to restrict routing
     #     :account(Integer):: Restrict to agents with this account id
     #     :shard(Integer):: Restrict to agents with this shard id, or if value is Packet::GLOBAL,
     #       ones with no shard id
+    # token(String|NilClass):: Token uniquely identifying request; defaults to random generated
     #
     # === Block
     # Required block used to process response asynchronously with the following parameter:
@@ -767,67 +242,59 @@ module RightScale
     # true:: Always return true
     #
     # === Raise
-    # ArgumentError:: If block missing
-    # SendFailure:: If publishing of request failed unexpectedly
-    # TemporarilyOffline:: If cannot publish request because currently not connected
-    #    to any brokers and offline queueing is disabled
-    def send_retryable_request(type, payload = nil, target = nil, &callback)
+    # ArgumentError:: If target invalid or block missing
+    def send_request(type, payload = nil, target = nil, token = nil, &callback)
       raise ArgumentError, "Missing block for response callback" unless callback
-      build_and_send_packet(:send_retryable_request, type, payload, target, callback)
+      build_and_send_packet(:send_request, type, payload, target, token, callback)
     end
 
-    # Send a request to a single target with a response expected
-    # Persist the request en route to reduce the chance of it being lost at the expense of some
-    # additional network overhead
-    # Enqueue the request if the target is not currently available
-    # Never automatically retry the request if there is the possibility of the request being duplicated
-    # Set time-to-live to be forever
-    # Note that receiving a response does not guarantee that the request activity has actually
-    # completed since the request processing may involve other asynchronous requests
+    # Build and send packet
     #
     # === Parameters
+    # kind(Symbol):: Kind of request: :send_push or :send_request
     # type(String):: Dispatch route for the request; typically identifies actor and action
     # payload(Object):: Data to be sent with marshalling en route
-    # target(String|Hash):: Identity of specific target, hash for selecting targets of which one is picked
-    #   randomly, or nil if routing solely using type
+    # target(Hash|NilClass):: Identity of specific target as string, or hash for selecting targets
+    #   :agent_id(String):: Identity of specific target
     #   :tags(Array):: Tags that must all be associated with a target for it to be selected
     #   :scope(Hash):: Scoping to be used to restrict routing
     #     :account(Integer):: Restrict to agents with this account id
     #     :shard(Integer):: Restrict to agents with this shard id, or if value is Packet::GLOBAL,
     #       ones with no shard id
-    #
-    # === Block
-    # Required block used to process response asynchronously with the following parameter:
-    #   result(Result):: Response with an OperationResult of SUCCESS, RETRY, NON_DELIVERY, or ERROR,
-    #     use RightScale::OperationResult.from_results to decode
+    #   :selector(Symbol):: Which of the matched targets to be selected: :any or :all
+    # token(String|NilClass):: Token uniquely identifying request; defaults to random generated
+    # callback(Proc|nil):: Block used to process routing response
     #
     # === Return
     # true:: Always return true
     #
     # === Raise
-    # ArgumentError:: If block missing
-    # TemporarilyOffline:: If cannot publish request because currently not connected
-    #    to any brokers and offline queueing is disabled
-    # SendFailure:: If publishing of request failed unexpectedly
-    def send_persistent_request(type, payload = nil, target = nil, &callback)
-      raise ArgumentError, "Missing block for response callback" unless callback
-      build_and_send_packet(:send_persistent_request, type, payload, target, callback)
+    # ArgumentError:: If target invalid
+    def build_and_send_packet(kind, type, payload, target, token, callback)
+      if (packet = build_packet(kind, type, payload, target, token, callback))
+        action = type.split('/').last
+        received_at = @request_stats.update(action, packet.token)
+        @request_kind_stats.update((packet.selector == :all ? "fanout" : kind.to_s)[5..-1])
+        send("#{@mode}_send", kind, target, packet, received_at, callback)
+      end
+      true
     end
 
     # Build packet or queue it if offline
     #
     # === Parameters
-    # kind(Symbol):: Kind of send request: :send_push, :send_persistent_push, :send_retryable_request,
-    #   or :send_persistent_request
+    # kind(Symbol):: Kind of request: :send_push or :send_request
     # type(String):: Dispatch route for the request; typically identifies actor and action
     # payload(Object):: Data to be sent with marshalling en route
-    # target(String|Hash):: Identity of specific target, or hash for selecting targets
+    # target(Hash|NilClass):: Identity of specific target as string, or hash for selecting targets
+    #   :agent_id(String):: Identity of specific target
     #   :tags(Array):: Tags that must all be associated with a target for it to be selected
     #   :scope(Hash):: Scoping to be used to restrict routing
     #     :account(Integer):: Restrict to agents with this account id
     #     :shard(Integer):: Restrict to agents with this shard id, or if value is Packet::GLOBAL,
     #       ones with no shard id
     #   :selector(Symbol):: Which of the matched targets to be selected: :any or :all
+    # token(String|NilClass):: Token uniquely identifying request; defaults to random generated
     # callback(Boolean):: Whether this request has an associated response callback
     #
     # === Return
@@ -835,30 +302,32 @@ module RightScale
     #
     # === Raise
     # ArgumentError:: If target is invalid
-    def build_packet(kind, type, payload, target, callback = false)
-      validate_target(target, !!(kind.to_s =~ /push/))
-      if should_queue?
+    def build_packet(kind, type, payload, target, token, callback = false)
+      validate_target(target, kind == :send_push)
+      if queueing?
         @offline_handler.queue_request(kind, type, payload, target, callback)
         nil
       else
-        kind_str = kind.to_s
-        persistent = !!(kind_str =~ /persistent/)
-        if kind_str =~ /push/
+        if kind == :send_push
           packet = Push.new(type, payload)
           packet.selector = target[:selector] || :any if target.is_a?(Hash)
+          packet.persistent = true
           packet.confirm = true if callback
         else
           packet = Request.new(type, payload)
           ttl = @options[:time_to_live]
-          packet.expires_at = Time.now.to_i + ttl if !persistent && ttl && ttl != 0
+          packet.expires_at = Time.now.to_i + ttl if ttl && ttl != 0
           packet.selector = :any
         end
         packet.from = @identity
-        packet.token = AgentIdentity.generate
-        packet.persistent = persistent
+        packet.token = token || RightSupport::Data::UUID.generate
         if target.is_a?(Hash)
-          packet.tags = target[:tags] || []
-          packet.scope = target[:scope]
+          if (agent_id = target[:agent_id])
+            packet.target = agent_id
+          else
+            packet.tags = target[:tags] || []
+            packet.scope = target[:scope]
+          end
         else
           packet.target = target
         end
@@ -866,36 +335,6 @@ module RightScale
       end
     end
 
-    # Send packet
-    #
-    # === Parameters
-    # kind(Symbol):: Kind of send request: :send_push, :send_persistent_push, :send_retryable_request,
-    #   or :send_persistent_request
-    # packet(Push|Request|NilClass):: Packet to send; nothing sent if nil
-    # callback(Proc|nil):: Block used to process routing response
-    #
-    # === Return
-    # true:: Always return true
-    #
-    # === Raise
-    # SendFailure:: If publishing of request fails unexpectedly
-    # TemporarilyOffline:: If cannot publish request because currently not connected
-    #    to any brokers and offline queueing is disabled
-    def send_packet(kind, packet, callback)
-      if packet
-        method = packet.type.split('/').last
-        received_at = @request_stats.update(method, packet.token)
-        @request_kind_stats.update((packet.selector == :all ? kind.to_s.sub(/push/, "fanout") : kind.to_s)[5..-1])
-        @pending_requests[packet.token] = PendingRequest.new(kind, received_at, callback) if callback
-        if !packet.persistent && kind.to_s =~ /request/
-          publish_with_timeout_retry(packet, packet.token)
-        else
-          publish(packet)
-        end
-      end
-      true
-    end
-
     # Handle response to a request
     #
     # === Parameters
@@ -904,9 +343,9 @@ module RightScale
     # === Return
     # true:: Always return true
     def handle_response(response)
-      token = response.token
       if response.is_a?(Result)
-        if result = OperationResult.from_results(response)
+        token = response.token
+        if (result = OperationResult.from_results(response))
           if result.non_delivery?
             @non_delivery_stats.update(result.content.nil? ? "nil" : result.content.inspect)
           elsif result.error?
@@ -917,26 +356,26 @@ module RightScale
           @result_stats.update(response.results.nil? ? "nil" : response.results)
         end
 
-        if handler = @pending_requests[token]
-          if result && result.non_delivery? && handler.kind == :send_retryable_request
+        if (pending_request = @pending_requests[token])
+          if result && result.non_delivery? && pending_request.kind == :send_request
             if result.content == OperationResult::TARGET_NOT_CONNECTED
               # Log and temporarily ignore so that timeout retry mechanism continues, but save reason for use below if timeout
               # Leave purging of associated request until final response, i.e., success response or retry timeout
-              if parent = handler.retry_parent
-                @pending_requests[parent].non_delivery = result.content
+              if (parent_token = pending_request.retry_parent_token)
+                @pending_requests[parent_token].non_delivery = result.content
               else
-                handler.non_delivery = result.content
+                pending_request.non_delivery = result.content
               end
               Log.info("Non-delivery of <#{token}> because #{result.content}")
-            elsif result.content == OperationResult::RETRY_TIMEOUT && handler.non_delivery
+            elsif result.content == OperationResult::RETRY_TIMEOUT && pending_request.non_delivery
               # Request timed out but due to another non-delivery reason, so use that reason since more germane
-              response.results = OperationResult.non_delivery(handler.non_delivery)
-              deliver(response, handler)
+              response.results = OperationResult.non_delivery(pending_request.non_delivery)
+              deliver_response(response, pending_request)
             else
-              deliver(response, handler)
+              deliver_response(response, pending_request)
             end
           else
-            deliver(response, handler)
+            deliver_response(response, pending_request)
           end
         elsif result && result.non_delivery?
           Log.info("Non-delivery of <#{token}> because #{result.content}")
@@ -947,16 +386,20 @@ module RightScale
       true
     end
 
-    # Take any actions necessary to quiesce mapper interaction in preparation
+    # Take any actions necessary to quiesce client interaction in preparation
     # for agent termination but allow message receipt to continue
     #
     # === Return
     # (Array):: Number of pending non-push requests and age of youngest request
     def terminate
-      @offline_handler.terminate
-      @connectivity_checker.terminate
-      pending = @pending_requests.kind(PendingRequests::REQUEST_KINDS)
-      [pending.size, pending.youngest_age]
+      if @offline_handler
+        @offline_handler.terminate
+        @connectivity_checker.terminate if @connectivity_checker
+        pending = @pending_requests.kind(:send_request)
+        [pending.size, pending.youngest_age]
+      else
+        [0, nil]
+      end
     end
 
     # Create displayable dump of unfinished non-push request information
@@ -966,11 +409,13 @@ module RightScale
     # info(Array(String)):: Receive time and token for each request in descending time order
     def dump_requests
       info = []
-      @pending_requests.kind(PendingRequests::REQUEST_KINDS).each do |token, request|
-        info << "#{request.receive_time.localtime} <#{token}>"
+      if @pending_requests
+        @pending_requests.kind(:send_request).each do |token, request|
+          info << "#{request.receive_time.localtime} <#{token}>"
+        end
+        info.sort!.reverse!
+        info = info[0..49] + ["..."] if info.size > 50
       end
-      info.sort.reverse
-      info = info[0..49] + ["..."] if info.size > 50
       info
     end
 
@@ -1006,31 +451,34 @@ module RightScale
     #   "send failure"(Hash|nil):: Send failure activity stats with keys "total", "percent", "last", and "rate"
     #     with percentage breakdown per failure type, or nil if none
     def stats(reset = false)
-      offlines = @offline_stats.all
-      offlines.merge!("duration" => @offline_stats.avg_duration) if offlines
-      if @pending_requests.size > 0
-        pending = {}
-        pending["pushes"] = @pending_requests.kind(PendingRequests::PUSH_KINDS).size
-        requests = @pending_requests.kind(PendingRequests::REQUEST_KINDS)
-        if (pending["requests"] = requests.size) > 0
-          pending["oldest age"] = requests.oldest_age
+      stats = {}
+      if @agent
+        offlines = @offline_stats.all
+        offlines.merge!("duration" => @offline_stats.avg_duration) if offlines
+        if @pending_requests.size > 0
+          pending = {}
+          pending["pushes"] = @pending_requests.kind(:send_push).size
+          requests = @pending_requests.kind(:send_request)
+          if (pending["requests"] = requests.size) > 0
+            pending["oldest age"] = requests.oldest_age
+          end
         end
+        stats = {
+          "exceptions"       => @exception_stats.stats,
+          "non-deliveries"   => @non_delivery_stats.all,
+          "offlines"         => offlines,
+          "pings"            => @ping_stats.all,
+          "request kinds"    => @request_kind_stats.all,
+          "requests"         => @request_stats.all,
+          "requests pending" => pending,
+          "response time"    => @request_stats.avg_duration,
+          "result errors"    => @result_error_stats.all,
+          "results"          => @result_stats.all,
+          "retries"          => @retry_stats.all,
+          "send failures"    => @send_failure_stats.all
+        }
+        reset_stats if reset
       end
-      stats = {
-        "exceptions"       => @exception_stats.stats,
-        "non-deliveries"   => @non_delivery_stats.all,
-        "offlines"         => offlines,
-        "pings"            => @ping_stats.all,
-        "request kinds"    => @request_kind_stats.all,
-        "requests"         => @request_stats.all,
-        "requests pending" => pending,
-        "response time"    => @request_stats.avg_duration,
-        "result errors"    => @result_error_stats.all,
-        "results"          => @result_stats.all,
-        "retries"          => @retry_stats.all,
-        "send failures"    => @send_failure_stats.all
-      }
-      reset_stats if reset
       stats
     end
 
@@ -1056,9 +504,11 @@ module RightScale
 
     # Validate target argument of send per the semantics of each kind of send:
     #   - The target is either a specific target name, a non-empty hash, or nil
-    #   - A specific target name must be a string
+    #   - A specific target name must be a string (or use :agent_id key)
     #   - A non-empty hash target
     #     - may have keys in symbol or string format
+    #     - may contain an :agent_id to select a specific target, but then cannot
+    #       have any other keys
     #     - may be allowed to contain a :selector key with value :any or :all,
     #       depending on the kind of send
     #     - may contain a :scope key with a hash value with keys :account and/or :shard
@@ -1075,11 +525,15 @@ module RightScale
     # === Raise
     # ArgumentError:: If target is invalid
     def validate_target(target, allow_selector)
-      choices = (allow_selector ? ":selector, " : "") + ":tags and/or :scope"
+      choices = ":agent_id OR " + (allow_selector ? ":selector, " : "") + ":tags and/or :scope"
       if target.is_a?(Hash)
         t = SerializationHelper.symbolize_keys(target)
 
-        if selector = t[:selector]
+        if (agent_id = t[:agent_id])
+          raise ArgumentError, "Invalid target: #{target.inspect}" if t.size > 1
+        end
+
+        if (selector = t[:selector])
           if allow_selector
             selector = selector.to_sym
             unless [:any, :all].include?(selector)
@@ -1091,7 +545,7 @@ module RightScale
           end
         end
 
-        if scope = t[:scope]
+        if (scope = t[:scope])
           if scope.is_a?(Hash)
             scope = SerializationHelper.symbolize_keys(scope)
             unless (scope[:account] || scope[:shard]) && (scope.keys - [:account, :shard]).empty?
@@ -1107,7 +561,7 @@ module RightScale
           raise ArgumentError, "Invalid target tags (#{t[:tags].inspect}), must be an array"
         end
 
-        unless (selector || scope || tags) && (t.keys - [:selector, :scope, :tags]).empty?
+        unless (agent_id || selector || scope || tags) && (t.keys - [:agent_id, :selector, :scope, :tags]).empty?
           raise ArgumentError, "Invalid target hash (#{target.inspect}), choices are #{choices}"
         end
         target = t
@@ -1117,79 +571,144 @@ module RightScale
       true
     end
 
-    # Build and send packet
+    # Send request via HTTP and then immediately handle response
     #
     # === Parameters
-    # kind(Symbol):: Kind of send request: :send_push, :send_persistent_push, :send_retryable_request,
-    #   or :send_persistent_request
-    # type(String):: Dispatch route for the request; typically identifies actor and action
-    # payload(Object):: Data to be sent with marshalling en route
-    # target(String|Hash):: Identity of specific target, or hash for selecting targets
-    #   :tags(Array):: Tags that must all be associated with a target for it to be selected
-    #   :scope(Hash):: Scoping to be used to restrict routing
-    #     :account(Integer):: Restrict to agents with this account id
-    #     :shard(Integer):: Restrict to agents with this shard id, or if value is Packet::GLOBAL,
-    #       ones with no shard id
-    #   :selector(Symbol):: Which of the matched targets to be selected: :any or :all
-    # callback(Proc|nil):: Block used to process routing response
+    # kind(Symbol):: Kind of request: :send_push or :send_request
+    # target(Hash|String|nil):: Target for request
+    # packet(Push|Request):: Request packet to send
+    # received_at(Time):: Time when request received
+    # callback(Proc|nil):: Block used to process response
     #
     # === Return
     # true:: Always return true
+    def http_send(kind, target, packet, received_at, callback)
+      begin
+        method = packet.class.name.split("::").last.downcase
+        result = success_result(@agent.client.send(method, packet.type, packet.payload, target, packet.token))
+      rescue Exceptions::Unauthorized => e
+        result = error_result(e.message)
+      rescue Exceptions::ConnectivityFailure => e
+        if queueing?
+          @offline_handler.queue_request(kind, packet.type, packet.payload, target, callback)
+          result = nil
+        else
+          result = retry_result(e.message)
+        end
+      rescue Exceptions::RetryableError => e
+        result = retry_result(e.message)
+      rescue Exceptions::InternalServerError => e
+        result = error_result("#{e.server} internal error")
+      rescue Exceptions::Terminating => e
+        result = nil
+      rescue StandardError => e
+        # These errors are either unexpected errors or RestClient errors with an http_body
+        # giving details about the error that are conveyed in the error_result
+        if e.respond_to?(:http_body)
+          # No need to log here since any HTTP request errors have already been logged
+          result = error_result(e.inspect)
+        else
+          agent_type = AgentIdentity.parse(@identity).agent_type
+          Log.error("Failed to send #{packet.trace} #{packet.type}", e, :trace)
+          @exception_stats.track("request", e)
+          result = error_result("#{agent_type.capitalize} agent internal error")
+        end
+      end
+
+      if result && packet.is_a?(Request)
+        result = Result.new(packet.token, @identity, result, from = packet.target)
+        result.received_at = received_at.to_f
+        @pending_requests[packet.token] = PendingRequest.new(kind, received_at, callback) if callback
+        if @options[:async_response]
+          EM.next_tick { handle_response(result) }
+        else
+          handle_response(result)
+        end
+      end
+      true
+    end
+
+    # Send request via AMQP
+    # If lack connectivity and queueing enabled, queue request
     #
-    # === Raise
-    # ArgumentError:: If target is invalid
-    # SendFailure:: If publishing of request fails unexpectedly
-    # TemporarilyOffline:: If cannot publish request because currently not connected
-    #    to any brokers and offline queueing is disabled
-    def build_and_send_packet(kind, type, payload, target, callback)
-      packet = build_packet(kind, type, payload, target, callback)
-      send_packet(kind, packet, callback)
+    # === Parameters
+    # kind(Symbol):: Kind of request: :send_push or :send_request
+    # target(Hash|String|nil):: Target for request
+    # received_at(Time):: Time when request received
+    # packet(Push|Request):: Request packet to send
+    # callback(Proc|nil):: Block used to process response
+    #
+    # === Return
+    # true:: Always return true
+    def amqp_send(kind, target, packet, received_at, callback)
+      begin
+        @pending_requests[packet.token] = PendingRequest.new(kind, received_at, callback) if callback
+        if packet.class == Request
+          amqp_send_retry(packet, packet.token)
+        else
+          amqp_send_once(packet)
+        end
+      rescue TemporarilyOffline => e
+        if queueing?
+          # Queue request until come back online
+          @offline_handler.queue_request(kind, packet.type, packet.payload, target, callback)
+          @pending_requests.delete(packet.token) if callback
+        else
+          # Send retry response so that requester, e.g., RetryableRequest, can retry
+          result = OperationResult.retry("lost RightNet connectivity")
+          handle_response(Result.new(packet.token, @identity, result, @identity))
+        end
+      rescue SendFailure => e
+        # Send non-delivery response so that requester, e.g., RetryableRequest, can retry
+        result = OperationResult.non_delivery("send failed unexpectedly")
+        handle_response(Result.new(packet.token, @identity, result, @identity))
+      end
       true
     end
 
-    # Publish request to request queue
+    # Send request via AMQP without retrying
     # Use mandatory flag to request return of message if it cannot be delivered
     #
     # === Parameters
-    # packet(Push|Request):: Packet to be sent
+    # packet(Push|Request):: Request packet to send
     # ids(Array|nil):: Identity of specific brokers to choose from, or nil if any okay
     #
     # === Return
-    # (Array):: Identity of brokers published to
+    # (Array):: Identity of brokers to which request was published
     #
     # === Raise
-    # SendFailure:: If publishing of request fails unexpectedly
-    # TemporarilyOffline:: If cannot publish request because currently not connected
-    #    to any brokers and offline queueing is disabled
-    def publish(request, ids = nil)
-      begin
-        exchange = {:type => :fanout, :name => "request", :options => {:durable => true, :no_declare => @secure}}
-        @broker.publish(exchange, request, :persistent => request.persistent, :mandatory => true,
-                        :log_filter => [:tags, :target, :tries, :persistent], :brokers => ids)
-      rescue RightAMQP::HABrokerClient::NoConnectedBrokers => e
-        msg = "Failed to publish request #{request.trace} #{request.type}"
-        Log.error(msg, e)
-        @send_failure_stats.update("NoConnectedBrokers")
-        raise TemporarilyOffline.new(msg + " (#{e.class}: #{e.message})")
-      rescue Exception => e
-        msg = "Failed to publish request #{request.trace} #{request.type}"
-        Log.error(msg, e, :trace)
-        @send_failure_stats.update(e.class.name)
-        @exception_stats.track("publish", e, request)
-        raise SendFailure.new(msg + " (#{e.class}: #{e.message})")
-      end
+    # SendFailure:: If sending of request failed unexpectedly
+    # TemporarilyOffline:: If cannot send request because RightNet client currently disconnected
+    #   and offline queueing is disabled
+    def amqp_send_once(packet, ids = nil)
+      name =
+      exchange = {:type => :fanout, :name => @request_queue, :options => {:durable => true, :no_declare => @secure}}
+      @agent.client.publish(exchange, packet, :persistent => packet.persistent, :mandatory => true,
+                            :log_filter => [:tags, :target, :tries, :persistent], :brokers => ids)
+    rescue RightAMQP::HABrokerClient::NoConnectedBrokers => e
+      msg = "Failed to publish request #{packet.trace} #{packet.type}"
+      Log.error(msg, e)
+      @send_failure_stats.update("NoConnectedBrokers")
+      raise TemporarilyOffline.new(msg + " (#{e.class}: #{e.message})")
+    rescue Exception => e
+      msg = "Failed to publish request #{packet.trace} #{packet.type}"
+      Log.error(msg, e, :trace)
+      @send_failure_stats.update(e.class.name)
+      @exception_stats.track("publish", e, packet)
+      raise SendFailure.new(msg + " (#{e.class}: #{e.message})")
     end
 
-    # Publish request with one or more retries if do not receive a response in time
+    # Send request via AMQP with one or more retries if do not receive a response in time
     # Send timeout result if reach configured retry timeout limit
     # Use exponential backoff with RETRY_BACKOFF_FACTOR for retry spacing
     # Adjust retry interval by average response time to avoid adding to system load
     # when system gets slow
     # Rotate through brokers on retries
+    # Check connectivity after first retry timeout
     #
     # === Parameters
-    # request(Request):: Request to be sent
-    # parent(String):: Token for original request
+    # packet(Request):: Request packet to send
+    # parent_token(String):: Token for original request
     # count(Integer):: Number of retries so far
     # multiplier(Integer):: Multiplier for retry interval for exponential backoff
     # elapsed(Integer):: Elapsed time in seconds since this request was first attempted
@@ -1197,79 +716,74 @@ module RightScale
     #
     # === Return
     # true:: Always return true
-    def publish_with_timeout_retry(request, parent, count = 0, multiplier = 1, elapsed = 0, broker_ids = nil)
-      published_broker_ids = publish(request, broker_ids)
+    def amqp_send_retry(packet, parent_token, count = 0, multiplier = 1, elapsed = 0, broker_ids = nil)
+      check_broker_ids = amqp_send_once(packet, broker_ids)
 
-      if @retry_interval && @retry_timeout && parent
+      if @retry_interval && @retry_timeout && parent_token
         interval = [(@retry_interval * multiplier) + (@request_stats.avg_duration || 0), @retry_timeout - elapsed].min
         EM.add_timer(interval) do
           begin
-            if handler = @pending_requests[parent]
+            if @pending_requests[parent_token]
               count += 1
               elapsed += interval
               if elapsed < @retry_timeout
-                request.tries << request.token
-                request.token = AgentIdentity.generate
-                @pending_requests[parent].retry_parent = parent if count == 1
-                @pending_requests[request.token] = @pending_requests[parent]
-                broker_ids ||= @broker.all
-                publish_with_timeout_retry(request, parent, count, multiplier * RETRY_BACKOFF_FACTOR, elapsed,
-                                           broker_ids.push(broker_ids.shift))
-                @retry_stats.update(request.type.split('/').last)
+                packet.tries << packet.token
+                packet.token = RightSupport::Data::UUID.generate
+                @pending_requests[parent_token].retry_parent_token = parent_token if count == 1
+                @pending_requests[packet.token] = @pending_requests[parent_token]
+                broker_ids ||= @agent.client.all
+                amqp_send_retry(packet, parent_token, count, multiplier * RETRY_BACKOFF_FACTOR, elapsed,
+                                broker_ids.push(broker_ids.shift))
+                @retry_stats.update(packet.type.split('/').last)
               else
-                Log.warning("RE-SEND TIMEOUT after #{elapsed.to_i} seconds for #{request.trace} #{request.type}")
+                Log.warning("RE-SEND TIMEOUT after #{elapsed.to_i} seconds for #{packet.trace} #{packet.type}")
                 result = OperationResult.non_delivery(OperationResult::RETRY_TIMEOUT)
                 @non_delivery_stats.update(result.content)
-                handle_response(Result.new(request.token, request.reply_to, result, @identity))
+                handle_response(Result.new(packet.token, @identity, result, @identity))
               end
-              @connectivity_checker.check(published_broker_ids.first) if count == 1 && !published_broker_ids.empty?
+              @connectivity_checker.check(check_broker_ids.first) if check_broker_ids.any? && count == 1
             end
           rescue TemporarilyOffline => e
-            # Send retry response so that requester, e.g., IdempotentRequest, can retry
-            Log.error("Failed retry for #{request.trace} #{request.type} because temporarily offline")
-            result = OperationResult.retry("lost connectivity")
-            handle_response(Result.new(request.token, request.reply_to, result, @identity))
+            Log.error("Failed retry for #{packet.trace} #{packet.type} because temporarily offline")
           rescue SendFailure => e
-            # Send non-delivery response so that requester, e.g., IdempotentRequest, can retry
-            Log.error("Failed retry for #{request.trace} #{request.type} because of send failure")
-            result = OperationResult.non_delivery("retry failed")
-            handle_response(Result.new(request.token, request.reply_to, result, @identity))
+            Log.error("Failed retry for #{packet.trace} #{packet.type} because of send failure")
           rescue Exception => e
             # Not sending a response here because something more basic is broken in the retry
             # mechanism and don't want an error response to preempt a delayed actual response
-            Log.error("Failed retry for #{request.trace} #{request.type} without responding", e, :trace)
-            @exception_stats.track("retry", e, request)
+            Log.error("Failed retry for #{packet.trace} #{packet.type} without responding", e, :trace)
+            @exception_stats.track("retry", e, packet)
           end
         end
       end
       true
     end
 
-    # Deliver the response and remove associated request(s) from pending
+    # Deliver the response and remove associated non-push requests from pending
+    # including all associated retry requests
     #
     # === Parameters
     # response(Result):: Packet received as result of request
-    # handler(Hash):: Associated request handler
+    # pending_request(Hash):: Associated pending request
     #
     # === Return
     # true:: Always return true
-    def deliver(response, handler)
-      @request_stats.finish(handler.receive_time, response.token)
+    def deliver_response(response, pending_request)
+      @request_stats.finish(pending_request.receive_time, response.token)
 
-      @pending_requests.delete(response.token) if PendingRequests::REQUEST_KINDS.include?(handler.kind)
-      if parent = handler.retry_parent
-        @pending_requests.reject! { |k, v| k == parent || v.retry_parent == parent }
+      @pending_requests.delete(response.token) if pending_request.kind == :send_request
+      if (parent_token = pending_request.retry_parent_token)
+        @pending_requests.reject! { |k, v| k == parent_token || v.retry_parent_token == parent_token }
       end
 
-      handler.response_handler.call(response) if handler.response_handler
+      pending_request.response_handler.call(response) if pending_request.response_handler
       true
     end
 
-    # Should agent be queueing current request?
+    # Determine whether currently queueing requests because offline
     #
     # === Return
-    # (Boolean):: true if should queue request, otherwise false
-    def should_queue?
+    # (Boolean):: true if queueing, otherwise false
+    def queueing?
       @options[:offline_queueing] && @offline_handler.queueing?
     end