right_agent 1.0.1 → 2.0.7

data/lib/right_agent/command/command_io.rb

@@ -76,11 +76,11 @@ module RightScale
     # true:: Always return true
     #
     # === Raise
-    # (RightScale::Exceptions::Argument):: If block is missing
-    # (RightScale::Exceptions::Application):: If +listen+ has already been called and +stop+ hasn't since
-    # (RightScale::Exceptions::Application):: If port is already bound
+    # (ArgumentError):: If block is missing
+    # (Exceptions::Application):: If +listen+ has already been called and +stop+ hasn't since
+    # (Exceptions::Application):: If port is already bound
     def listen(socket_port, &block)
-      raise Exceptions::Argument, 'Missing listener block' unless block_given?
+      raise ArgumentError, 'Missing listener block' unless block_given?
       raise Exceptions::Application, 'Already listening' if listening
       begin
         @conn = EM.start_server('127.0.0.1', socket_port, ServerInputHandler, block)

data/lib/right_agent/command/command_parser.rb

@@ -31,9 +31,9 @@ module RightScale
     # Block that will get called back whenever a command is successfully parsed
     #
     # === Raise
-    # (RightScale::Exceptions::Argument): If block is missing
+    # (ArgumentError): If block is missing
     def initialize &block
-      raise RightScale::Exceptions::Argument, 'Missing handler block' unless block
+      raise ArgumentError, 'Missing handler block' unless block
       @callback = block
       @buildup = ''
     end

data/lib/right_agent/command/command_runner.rb

@@ -55,7 +55,7 @@ module RightScale
     # cmd_options[:listen_port](Integer):: Command server listen port
     #
     # === Raise
-    # (RightScale::Exceptions::Application):: If +start+ has already been called and +stop+ hasn't since
+    # (Exceptions::Application):: If +start+ has already been called and +stop+ hasn't since
     def self.start(socket_port, identity, commands, fiber_pool = nil)
       cmd_options = nil
       @listen_port = socket_port

data/lib/right_agent/connectivity_checker.rb

@@ -0,0 +1,179 @@
+#
+# Copyright (c) 2009-2013 RightScale Inc
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+module RightScale
+
+  # Broker connectivity checker
+  # Checks connectivity when requested
+  class ConnectivityChecker
+
+    # Minimum number of seconds between restarts of the inactivity timer
+    MIN_RESTART_INACTIVITY_TIMER_INTERVAL = 60
+
+    # Number of seconds to wait for ping response from a RightNet router when checking connectivity
+    PING_TIMEOUT = 30
+
+    # Default maximum number of consecutive ping timeouts before attempt to reconnect
+    MAX_PING_TIMEOUTS = 3
+
+    # (EM::Timer) Timer while waiting for RightNet router ping response
+    attr_accessor :ping_timer
+
+    def initialize(sender, check_interval, ping_stats, exception_stats)
+      @sender = sender
+      @check_interval = check_interval
+      @ping_timeouts = {}
+      @ping_timer = nil
+      @ping_stats = ping_stats
+      @exception_stats = exception_stats
+      @last_received = Time.now
+      @message_received_callbacks = []
+      restart_inactivity_timer if @check_interval > 0
+    end
+
+    # Update the time this agent last received a request or response message
+    # and restart the inactivity timer thus deferring the next connectivity check
+    # Also forward this message receipt notification to any callbacks that have registered
+    #
+    # === Block
+    # Optional block without parameters that is activated when a message is received
+    #
+    # === Return
+    # true:: Always return true
+    def message_received(&callback)
+      if block_given?
+        @message_received_callbacks << callback
+      else
+        @message_received_callbacks.each { |c| c.call }
+        if @check_interval > 0
+          now = Time.now
+          if (now - @last_received) > MIN_RESTART_INACTIVITY_TIMER_INTERVAL
+            @last_received = now
+            restart_inactivity_timer
+          end
+        end
+      end
+      true
+    end
+
+    # Check whether broker connection is usable by pinging a router via that broker
+    # Attempt to reconnect if ping does not respond in PING_TIMEOUT seconds and
+    # if have reached timeout limit
+    # Ignore request if already checking a connection
+    #
+    # === Parameters
+    # id(String):: Identity of specific broker to use to send ping, defaults to any
+    #   currently connected broker
+    # max_ping_timeouts(Integer):: Maximum number of ping timeouts before attempt
+    #   to reconnect, defaults to MAX_PING_TIMEOUTS
+    #
+    # === Return
+    # true:: Always return true
+    def check(id = nil, max_ping_timeouts = MAX_PING_TIMEOUTS)
+      unless @terminating || @ping_timer || (id && !@sender.client.connected?(id))
+        @ping_id = id
+        @ping_timer = EM::Timer.new(PING_TIMEOUT) do
+          if @ping_id
+            begin
+              @ping_stats.update("timeout")
+              @ping_timer = nil
+              @ping_timeouts[@ping_id] = (@ping_timeouts[@ping_id] || 0) + 1
+              if @ping_timeouts[@ping_id] >= max_ping_timeouts
+                Log.error("Mapper ping via broker #{@ping_id} timed out after #{PING_TIMEOUT} seconds and now " +
+                          "reached maximum of #{max_ping_timeouts} timeout#{max_ping_timeouts > 1 ? 's' : ''}, " +
+                          "attempting to reconnect")
+                host, port, index, priority = @sender.client.identity_parts(@ping_id)
+                @sender.agent.connect(host, port, index, priority, force = true)
+              else
+                Log.warning("Mapper ping via broker #{@ping_id} timed out after #{PING_TIMEOUT} seconds")
+              end
+            rescue Exception => e
+              Log.error("Failed to reconnect to broker #{@ping_id}", e, :trace)
+              @exception_stats.track("ping timeout", e)
+            end
+          else
+            @ping_timer = nil
+          end
+        end
+
+        handler = lambda do |_|
+          begin
+            if @ping_timer
+              @ping_stats.update("success")
+              @ping_timer.cancel
+              @ping_timer = nil
+              @ping_timeouts[@ping_id] = 0
+              @ping_id = nil
+            end
+          rescue Exception => e
+            Log.error("Failed to cancel router ping", e, :trace)
+            @exception_stats.track("cancel ping", e)
+          end
+        end
+        request = Request.new("/router/ping", nil, {:from => @sender.identity, :token => AgentIdentity.generate})
+        @sender.pending_requests[request.token] = PendingRequest.new(Request, Time.now, handler)
+        ids = [@ping_id] if @ping_id
+        @ping_id = @sender.send(:publish, request, ids).first
+      end
+      true
+    end
+
+    # Prepare for agent termination
+    #
+    # === Return
+    # true:: Always return true
+    def terminate
+      @terminating = true
+      @check_interval = 0
+      if @ping_timer
+        @ping_timer.cancel
+        @ping_timer = nil
+      end
+      if @inactivity_timer
+        @inactivity_timer.cancel
+        @inactivity_timer = nil
+      end
+      true
+    end
+
+    protected
+
+    # Start timer that waits for inactive messaging period to end before checking connectivity
+    #
+    # === Return
+    # true:: Always return true
+    def restart_inactivity_timer
+      @inactivity_timer.cancel if @inactivity_timer
+      @inactivity_timer = EM::Timer.new(@check_interval) do
+        begin
+          check(id = nil, max_ping_timeouts = 1)
+        rescue Exception => e
+          Log.error("Failed connectivity check", e, :trace)
+          @exception_stats.track("check connectivity", e)
+        end
+      end
+      true
+    end
+
+  end # ConnectivityChecker
+
+end # RightScale

data/lib/right_agent/core_payload_types/secure_document_location.rb

@@ -40,10 +40,10 @@ module RightScale
     # (String):: Access token that should be used to fetch the document
     attr_accessor :ticket
 
-    # Array(String):: names of RightNet agents capable of providing the document.
+    # Array(String):: Identities of RightNet agents capable of providing the document.
     # If nil, then no target should be specified when sending RightNet requests;
     # this is used to route requests to "trusted infrastructure" nodes at the
-    # discretion of the mapper. 
+    # discretion of the RightNet router.
     attr_accessor :targets
 
     # Initialize fields from given arguments

data/lib/right_agent/dispatcher.rb

@@ -25,6 +25,8 @@ module RightScale
   # Dispatching of payload to specified actor
   class Dispatcher
 
+    include ProtocolVersionMixin
+
     class InvalidRequestType < Exception; end
     class DuplicateRequest < Exception; end
 
@@ -85,10 +87,10 @@ module RightScale
       token = request.token
       actor, method, idempotent = route(request)
       received_at = @request_stats.update(method, (token if request.is_a?(Request)))
-      if dup = duplicate?(request, method, idempotent)
+      if (dup = duplicate?(request, method, idempotent))
         raise DuplicateRequest.new(dup)
       end
-      unless result = expired?(request, method)
+      unless (result = expired?(request, method))
         result = perform(request, actor, method, idempotent)
       end
       if request.is_a?(Request)
@@ -140,7 +142,7 @@ module RightScale
       @reject_stats = RightSupport::Stats::Activity.new
       @request_stats = RightSupport::Stats::Activity.new
       @dispatch_failure_stats = RightSupport::Stats::Activity.new
-      @exception_stats = RightSupport::Stats::Exceptions.new(@agent)
+      @exception_stats = RightSupport::Stats::Exceptions.new(@agent, @agent.exception_callback)
       true
     end
 
@@ -157,10 +159,10 @@ module RightScale
         @reject_stats.update("expired (#{method})")
         Log.info("REJECT EXPIRED <#{request.token}> from #{request.from} TTL #{RightSupport::Stats.elapsed(now - expires_at)} ago")
         # For agents that do not know about non-delivery, use error result
-        if request.recv_version < 13
-          OperationResult.error("Could not deliver request (#{OperationResult::TTL_EXPIRATION})")
-        else
+        if can_handle_non_delivery_result?(request.recv_version)
           OperationResult.non_delivery(OperationResult::TTL_EXPIRATION)
+        else
+          OperationResult.error("Could not deliver request (#{OperationResult::TTL_EXPIRATION})")
         end
       end
     end
@@ -176,11 +178,11 @@ module RightScale
     # (String|nil):: Messaging describing who already serviced request if it is a duplicate, otherwise nil
     def duplicate?(request, method, idempotent)
       if !idempotent && @dispatched_cache
-        if serviced_by = @dispatched_cache.serviced_by(request.token)
+        if (serviced_by = @dispatched_cache.serviced_by(request.token))
           from_retry = ""
         else
           from_retry = "retry "
-          request.tries.each { |t| break if serviced_by = @dispatched_cache.serviced_by(t) }
+          request.tries.each { |t| break if (serviced_by = @dispatched_cache.serviced_by(t)) }
         end
         if serviced_by
           @reject_stats.update("#{from_retry}duplicate (#{method})")
@@ -225,9 +227,9 @@ module RightScale
     def perform(request, actor, method, idempotent)
       @dispatched_cache.store(request.token) if @dispatched_cache && !idempotent
       if actor.method(method).arity.abs == 1
-        actor.__send__(method, request.payload)
+        actor.send(method, request.payload)
       else
-        actor.__send__(method, request.payload, request)
+        actor.send(method, request.payload, request)
       end
     rescue Exception => e
       @dispatch_failure_stats.update("#{request.type}->#{e.class.name}")

data/lib/right_agent/enrollment_result.rb

@@ -6,7 +6,7 @@ require 'openssl'
 require 'base64'
 require 'json'
 
-# A response to a RightNet enrollment request containing the mapper's X509 cert,
+# A response to a RightNet enrollment request containing the router's X509 cert,
 # the instance (or other) agent's identity cert, and the agent's private key.
 # Responses are encrypted using a secret key shared between the instance and
 # the Certifying Authority (aka the RightScale core site) and integrity-protected
@@ -14,35 +14,39 @@ require 'json'
 module RightScale
 
   class EnrollmentResult
+
+    include ProtocolVersionMixin
+
     # Versions 5 and above use an identical format for the enrollment result
     SUPPORTED_VERSIONS = 5..AgentConfig.protocol_version
     
     class IntegrityFailure < Exception; end
     class VersionError < Exception; end
 
-    attr_reader :r_s_version, :timestamp, :mapper_cert, :id_cert, :id_key
+    attr_reader :r_s_version, :timestamp, :router_cert, :id_cert, :id_key
 
     # Create a new instance of this class
     #
     # === Parameters
     # timestamp(Time):: Timestamp associated with this result
-    # mapper_cert(String):: Arbitrary string
+    # router_cert(String):: Arbitrary string
     # id_cert(String):: Arbitrary string
     # id_key(String):: Arbitrary string
     # secret(String):: Shared secret with which the result is encrypted
     #
-    def initialize(r_s_version, timestamp, mapper_cert, id_cert, id_key, secret)
+    def initialize(r_s_version, timestamp, router_cert, id_cert, id_key, secret)
       @r_s_version = r_s_version
       @timestamp   = timestamp.utc
-      @mapper_cert = mapper_cert
+      @router_cert = router_cert
       @id_cert     = id_cert
       @id_key      = id_key
-      @serializer  = Serializer.new((:json if r_s_version < 12))
+      @serializer  = Serializer.new(can_handle_msgpack_result?(r_s_version) ? :msgpack : :json)
 
+      cert_name = can_handle_http?(r_s_version) ? 'router_cert' : 'mapper_cert'
       msg = @serializer.dump({
-        'mapper_cert' => @mapper_cert.to_s,
-        'id_cert'     => @id_cert,
-        'id_key'      => @id_key
+        cert_name => @router_cert.to_s,
+        'id_cert' => @id_cert,
+        'id_key'  => @id_key
       })
 
       key        = EnrollmentResult.derive_key(secret, @timestamp.to_i.to_s)
@@ -90,7 +94,7 @@ module RightScale
     # true|false:: Whether the objects' pertinent fields are identical
     #
     def ==(o)
-      self.mapper_cert == o.mapper_cert &&
+      self.router_cert == o.router_cert &&
       self.id_cert == o.id_cert &&
       self.id_key == o.id_key &&
       self.timestamp.to_i == o.timestamp.to_i
@@ -164,11 +168,11 @@ module RightScale
       raise IntegrityFailure.new("MAC mismatch: expected #{my_mac}, got #{mac}") unless (mac == my_mac)
 
       msg = serializer.load(plaintext)
-      mapper_cert = msg['mapper_cert']
+      router_cert = msg['router_cert'] || msg['mapper_cert']
       id_cert   = msg['id_cert']
       id_key    = msg['id_key']
 
-      self.new(r_s_version, timestamp, mapper_cert, id_cert, id_key, secret)
+      self.new(r_s_version, timestamp, router_cert, id_cert, id_key, secret)
     end
 
     protected

data/lib/right_agent/exceptions.rb

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2009-2011 RightScale Inc
+# Copyright (c) 2009-2013 RightScale Inc
 #
 # Permission is hereby granted, free of charge, to any person obtaining
 # a copy of this software and associated documentation files (the
@@ -21,21 +21,22 @@
 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 module RightScale
+
   class Exceptions
-    # Capability not currently supported
-    class NotSupported < Exception; end
 
-    # Internal application error
-    class Application < RuntimeError
+    # Base exception for use in nesting exceptions
+    class NestedException < StandardError
       attr_reader :nested_exception
+
+      # Exception message and optional nested exception or string
       def initialize(message, nested_exception = nil)
         @nested_exception = nested_exception
         super(message)
       end
     end
 
-    # Invalid command or method argument
-    class Argument < RuntimeError; end
+    # Internal application error
+    class Application < StandardError; end
 
     # Agent command IO error
     class IO < RuntimeError; end
@@ -43,30 +44,43 @@ module RightScale
     # Agent compute platform error
     class PlatformError < StandardError; end
 
-    # Cannot connect or lost connection to external resource
-    class ConnectivityFailure < RuntimeError
-      attr_reader :nested_exception
+    # Terminating service
+    class Terminating < RuntimeError; end
+
+    # Not authorized to make request
+    class Unauthorized < NestedException
       def initialize(message, nested_exception = nil)
-        @nested_exception = nested_exception
-        super(message)
+        super(message, nested_exception)
+      end
+    end
+
+    # Cannot connect to service, lost connection to it, or it is out of service or too busy to respond
+    class ConnectivityFailure < NestedException
+      def initialize(message, nested_exception = nil)
+        super(message, nested_exception)
       end
     end
 
     # Request failed but potentially will succeed if retried
-    class RetryableError < RuntimeError
-      attr_reader :nested_exception
+    class RetryableError < NestedException
       def initialize(message, nested_exception = nil)
-        @nested_exception = nested_exception
-        super(message)
+        super(message, nested_exception)
       end
     end
 
     # Database query failed
-    class QueryFailure < RuntimeError
-      attr_reader :nested_exception
+    class QueryFailure < NestedException
       def initialize(message, nested_exception = nil)
-        @nested_exception = nested_exception
-        super(message)
+        super(message, nested_exception)
+      end
+    end
+
+    # Error internal to specified server
+    class InternalServerError < NestedException
+      attr_reader :server
+      def initialize(message, server, nested_exception = nil)
+        @server = server
+        super(message, nested_exception)
       end
     end
   end