right_agent 2.2.1 → 2.3.0

data/lib/right_agent/agent_tag_manager.rb

@@ -185,7 +185,7 @@ module RightScale
       request = RightScale::RetryableRequest.new("/router/query_tags", payload, request_options)
       request.callback { |result| yield raw ? request.raw_response : result }
       request.errback do |message|
-        Log.error("Failed to query tags (#{message})")
+        ErrorTracker.log(self, "Failed to query tags (#{message})")
         yield((raw ? request.raw_response : nil) || message)
       end
       request.run

data/lib/right_agent/clients/api_client.rb

@@ -83,7 +83,6 @@ module RightScale
     # @option options [Array] :retry_intervals between successive retries; defaults to DEFAULT_RETRY_INTERVALS
     # @option options [Boolean] :retry_enabled for requests that fail to connect or that return a retry result
     # @option options [Numeric] :reconnect_interval for reconnect attempts after lose connectivity
-    # @option options [Proc] :exception_callback for unexpected exceptions
     #
     # @raise [ArgumentError] auth client does not support this client type
     def initialize(auth_client, options)

data/lib/right_agent/clients/auth_client.rb

@@ -175,7 +175,6 @@ module RightScale
     #
     # @return [Hash] current statistics
     #   [Hash, NilClass] "state" Activity stats or nil if none
-    #   [Hash, NilClass] "exceptions" Exceptions stats or nil if none
     def stats(reset = false)
       stats = {}
       @stats.each { |k, v| stats[k] = v.all }
@@ -189,9 +188,7 @@ module RightScale
     #
     # @return [TrueClass] always true
     def reset_stats
-      @stats = {
-        "state" => RightSupport::Stats::Activity.new,
-        "exceptions" => RightSupport::Stats::Exceptions.new(agent = nil, @exception_callback)}
+      @stats = {"state" => RightSupport::Stats::Activity.new}
       true
     end
 
@@ -246,8 +243,7 @@ module RightScale
             begin
               callback.call(:auth, @state)
             rescue StandardError => e
-              Log.error("Failed status callback", e)
-             @stats["exceptions"].track("status", e)
+              ErrorTracker.log(self, "Failed status callback", e, nil, :caller)
             end
           end
         end

data/lib/right_agent/clients/balanced_http_client.rb

@@ -178,7 +178,7 @@ module RightScale
       e2 = HttpExceptions.convert(e)
       log_failure(used[:host], path, params, filter, request_uuid, started_at, e2)
       raise e2
-    rescue Exception => e
+    rescue StandardError => e
       log_failure(used[:host], path, params, filter, request_uuid, started_at, e)
       raise
     end
@@ -345,7 +345,7 @@ module RightScale
     def log_failure(host, path, params, filter, request_uuid, started_at, exception)
       code = exception.respond_to?(:http_code) ? exception.http_code : "nil"
       duration = "%.0fms" % ((Time.now - started_at) * 1000)
-      Log.error("Failed <#{request_uuid}> in #{duration} | #{code} " + log_text(path, params, filter, host, exception))
+      ErrorTracker.log(self, "Failed <#{request_uuid}> in #{duration} | #{code} " + log_text(path, params, filter, host, exception))
       true
     end
 

data/lib/right_agent/clients/base_retry_client.rb

@@ -77,7 +77,6 @@ module RightScale
     #   EM::HttpRequest and fibers instead of RestClient; requests remain synchronous
     # @option options [Array] :filter_params symbols or strings for names of request parameters
     #   whose values are to be hidden when logging; can be augmented on individual requests
-    # @option options [Proc] :exception_callback for unexpected exceptions
     #
     # @return [Boolean] whether currently connected
     #
@@ -161,7 +160,6 @@ module RightScale
     #   [Hash, NilClass] "request sent" Activity stats or nil if none
     #   [Float, NilClass] "response time" average number of seconds to respond to a request or nil if none
     #   [Hash, NilClass] "state" Activity stats or nil if none
-    #   [Hash, NilClass] "exceptions" Exceptions stats or nil if none
     def stats(reset = false)
       stats = {}
       @stats.each { |k, v| stats[k] = v.all }
@@ -180,8 +178,7 @@ module RightScale
         "reconnects"       => RightSupport::Stats::Activity.new,
         "request failures" => RightSupport::Stats::Activity.new,
         "requests sent"    => RightSupport::Stats::Activity.new,
-        "state"            => RightSupport::Stats::Activity.new,
-        "exceptions"       => RightSupport::Stats::Exceptions.new(agent = nil, @options[:exception_callback]) }
+        "state"            => RightSupport::Stats::Activity.new }
       true
     end
 
@@ -212,8 +209,7 @@ module RightScale
               begin
                 callback.call(@type, @state)
               rescue StandardError => e
-                Log.error("Failed status callback", e)
-                @stats["exceptions"].track("status", e)
+                ErrorTracker.log(self, "Failed status callback", e, nil, :caller)
               end
             end
             reconnect if @state == :disconnected
@@ -248,9 +244,8 @@ module RightScale
     def close_http_client(reason)
       @http_client.close(reason) if @http_client
       true
-    rescue Exception => e
-      Log.error("Failed closing connection", e, :trace)
-      @stats["exceptions"].track("status", e)
+    rescue StandardError => e
+      ErrorTracker.log(self, "Failed closing connection", e)
       false
     end
 
@@ -271,11 +266,10 @@ module RightScale
         @http_client.check_health
         self.state = :connected
       rescue BalancedHttpClient::NotResponding => e
-        Log.error("Failed #{@options[:server_name]} health check", e.nested_exception)
+        ErrorTracker.log(self, "Failed #{@options[:server_name]} health check", e.nested_exception)
         self.state = :disconnected
-      rescue Exception => e
-        Log.error("Failed #{@options[:server_name]} health check", e)
-        @stats["exceptions"].track("check health", e)
+      rescue StandardError => e
+        ErrorTracker.log(self, "Failed #{@options[:server_name]} health check", e, nil, :caller)
         self.state = :disconnected
       end
     end
@@ -298,9 +292,8 @@ module RightScale
               @reconnect_timer = @reconnecting = nil
             end
           rescue Exception => e
-            Log.error("Failed #{@options[:server_name]} reconnect", e)
+            ErrorTracker.log(self, "Failed #{@options[:server_name]} reconnect", e, nil, :caller)
             @stats["reconnects"].update("failure")
-            @stats["exceptions"].track("reconnect", e)
             self.state = :disconnected
           end
           @reconnect_timer.interval = @options[:reconnect_interval] if @reconnect_timer
@@ -472,8 +465,8 @@ module RightScale
         @stats["request failures"].update("#{type} - retry")
         raise Exceptions::RetryableError.new(retry_result.http_body, retry_result)
       else
-        Log.error("Retrying #{type} request <#{request_uuid}> in #{interval} seconds " +
-                  "in response to retryable error (#{retry_result.http_body})")
+        ErrorTracker.log(self, "Retrying #{type} request <#{request_uuid}> in #{interval} seconds " +
+                               "in response to retryable error (#{retry_result.http_body})")
         wait(interval)
       end
       # Change request_uuid so that retried request not rejected as duplicate
@@ -505,8 +498,8 @@ module RightScale
         self.state = :disconnected
         raise Exceptions::ConnectivityFailure.new(not_responding.message + " after #{attempts} attempts")
       else
-        Log.error("Retrying #{type} request <#{request_uuid}> in #{interval} seconds " +
-                  "in response to routing failure (#{BalancedHttpClient.exception_text(not_responding)})")
+        ErrorTracker.log(self, "Retrying #{type} request <#{request_uuid}> in #{interval} seconds " +
+                               "in response to routing failure (#{BalancedHttpClient.exception_text(not_responding)})")
         wait(interval)
       end
       true

data/lib/right_agent/clients/right_http_client.rb

@@ -51,10 +51,6 @@ module RightScale
     # @option options [Boolean] :long_polling_only never attempt to create a WebSocket, always long-polling instead
     # @option options [Array] :filter_params symbols or strings for names of request parameters
     #   whose values are to be hidden when logging
-    # @option options [Proc] :exception_callback for unexpected exceptions with following parameters:
-    #   [Exception] exception raised
-    #   [Packet, NilClass] packet being processed
-    #   [Agent, NilClass] agent in which exception occurred
     #
     # @return [TrueClass] always true
     #
@@ -275,7 +271,7 @@ module RightScale
         begin
           callback.call(type, state)
         rescue RuntimeError => e
-          Log.error("Failed status callback", e)
+          ErrorTracker.log(self, "Failed status callback", e, nil, :caller)
         end
       end
       @status

data/lib/right_agent/clients/router_client.rb

@@ -88,7 +88,6 @@ module RightScale
     # @option options [Numeric] :reconnect_interval for reconnect attempts after lose connectivity
     # @option options [Boolean] :non_blocking i/o is to be used for HTTP requests by applying
     #   EM::HttpRequest and fibers instead of RestClient; requests remain synchronous
-    # @option options [Proc] :exception_callback for unexpected exceptions
     #
     # @raise [ArgumentError] auth client does not support this client type
     def initialize(auth_client, options)
@@ -262,7 +261,6 @@ module RightScale
     #   [Hash, NilClass] "request failures" Activity stats or nil if none
     #   [Hash, NilClass] "request sent" Activity stats or nil if none
     #   [Float, NilClass] "response time" average number of seconds to respond to a request or nil if none
-    #   [Hash, NilClass] "exceptions" Exceptions stats or nil if none
     def stats(reset = false)
       events = @stats["events"].all
       stats = super(reset)
@@ -337,11 +335,10 @@ module RightScale
         end
         @listen_failures = 0
       rescue Exception => e
-        Log.error("Failed to listen", e, :trace)
-        @stats["exceptions"].track("listen", e)
+        ErrorTracker.log(self, "Failed to listen", e)
         @listen_failures += 1
         if @listen_failures > MAX_LISTEN_FAILURES
-          Log.error("Exceeded maximum repeated listen failures (#{MAX_LISTEN_FAILURES}), stopping listening")
+          ErrorTracker.log(self, "Exceeded maximum repeated listen failures (#{MAX_LISTEN_FAILURES}), stopping listening")
           @listen_state = :cancel
           self.state = :failed
           return false
@@ -440,8 +437,7 @@ module RightScale
       connect(routing_keys, &handler)
       update_listen_state(:check, 1)
     rescue Exception => e
-      Log.error("Failed creating WebSocket", e)
-      @stats["exceptions"].track("websocket", e)
+      ErrorTracker.log(self, "Failed creating WebSocket", e, nil, :caller)
       backoff_connect_interval
       update_listen_state(:long_poll)
     end
@@ -477,7 +473,7 @@ module RightScale
       @websocket = Faye::WebSocket::Client.new(url.to_s, protocols = nil, options)
 
       @websocket.onerror = lambda do |event|
-        Log.error("WebSocket error (#{event.data})") if event.data
+        ErrorTracker.log(self, "WebSocket error (#{event.data})") if event.data
       end
 
       @websocket.onclose = lambda do |event|
@@ -488,8 +484,7 @@ module RightScale
           msg << ((event.reason.nil? || event.reason.empty?) ? ")" : ": #{event.reason})")
           Log.info(msg)
         rescue Exception => e
-          Log.error("Failed closing WebSocket", e, :trace)
-          @stats["exceptions"].track("event", e)
+          ErrorTracker.log(self, "Failed closing WebSocket", e)
         end
         @websocket = nil
       end
@@ -508,8 +503,7 @@ module RightScale
           handler.call(event)
           @communicated_callbacks.each { |callback| callback.call } if @communicated_callbacks
         rescue Exception => e
-          Log.error("Failed handling WebSocket event", e, :trace)
-          @stats["exceptions"].track("event", e)
+          ErrorTracker.log(self, "Failed handling WebSocket event", e)
         end
       end
 
@@ -596,6 +590,7 @@ module RightScale
     end
 
     # Process result from long-polling attempt
+    # Not necessary to log failure since should already have been done by underlying HTTP client
     #
     # @param [Array, NilClass] result from long-polling attempt
     #
@@ -603,12 +598,10 @@ module RightScale
     def process_long_poll(result)
       case result
       when Exceptions::Unauthorized, Exceptions::ConnectivityFailure, Exceptions::RetryableError, Exceptions::InternalServerError
-        Log.error("Failed long-polling", result, :no_trace)
         update_listen_state(:choose, backoff_reconnect_interval)
         result = nil
       when Exception
-        Log.error("Failed long-polling", result, :trace)
-        @stats["exceptions"].track("long-polling", result)
+        ErrorTracker.track(self, result)
         update_listen_state(:choose, backoff_reconnect_interval)
         result = nil
       else

data/lib/right_agent/command/command_parser.rb

@@ -51,12 +51,12 @@ module RightScale
     def parse_chunk(chunk)
       @buildup << chunk
       chunks = @buildup.split(CommandSerializer::SEPARATOR, -1)
-      if do_call = chunks.size > 1
+      if (do_call = chunks.size > 1)
         commands = []
         (0..chunks.size - 2).each do |i|
           begin
             commands << CommandSerializer.load(chunks[i])
-          rescue Exception => e
+          rescue StandardError => e
             # log any exceptions caused by serializing individual chunks instead
             # of halting EM. each command is discrete so we need to keep trying
             # so long as there are more commands to process (although subsequent
@@ -77,7 +77,7 @@ module RightScale
         @buildup = chunks.last
       end
       do_call
-    rescue Exception => e
+    rescue StandardError => e
       # log any other exceptions instead of halting EM.
       Log.error("Failed parsing command chunk", e, :trace)
     end

data/lib/right_agent/command/command_runner.rb

@@ -79,7 +79,7 @@ module RightScale
               Log.error("Invalid cookie used by command protocol client (#{cmd_cookie})")
             end
           rescue Exception => e
-            Log.warning("Command failed (#{e.message}) at\n#{e.backtrace.join("\n")}")
+            Log.error("Command failed", e, :trace)
           end
         end
 

data/lib/right_agent/connectivity_checker.rb

@@ -38,13 +38,12 @@ module RightScale
     # Timer while waiting for RightNet router ping response
     attr_accessor :ping_timer
 
-    def initialize(sender, check_interval, ping_stats, exception_stats)
+    def initialize(sender, check_interval, ping_stats)
       @sender = sender
       @check_interval = check_interval
       @ping_timeouts = {}
       @ping_timer = nil
       @ping_stats = ping_stats
-      @exception_stats = exception_stats
       @last_received = Time.now
       @message_received_callbacks = []
       restart_inactivity_timer if @check_interval > 0
@@ -98,17 +97,16 @@ module RightScale
               @ping_timer = nil
               @ping_timeouts[@ping_id] = (@ping_timeouts[@ping_id] || 0) + 1
               if @ping_timeouts[@ping_id] >= max_ping_timeouts
-                Log.error("Mapper ping via broker #{@ping_id} timed out after #{PING_TIMEOUT} seconds and now " +
-                          "reached maximum of #{max_ping_timeouts} timeout#{max_ping_timeouts > 1 ? 's' : ''}, " +
-                          "attempting to reconnect")
+                ErrorTracker.log(self, "Mapper ping via broker #{@ping_id} timed out after #{PING_TIMEOUT} seconds and now " +
+                                       "reached maximum of #{max_ping_timeouts} timeout#{max_ping_timeouts > 1 ? 's' : ''}, " +
+                                       "attempting to reconnect")
                 host, port, index, priority = @sender.client.identity_parts(@ping_id)
                 @sender.agent.connect(host, port, index, priority, force = true)
               else
                 Log.warning("Mapper ping via broker #{@ping_id} timed out after #{PING_TIMEOUT} seconds")
               end
             rescue Exception => e
-              Log.error("Failed to reconnect to broker #{@ping_id}", e, :trace)
-              @exception_stats.track("ping timeout", e)
+              ErrorTracker.log(self, "Failed to reconnect to broker #{@ping_id}", e)
             end
           else
             @ping_timer = nil
@@ -125,8 +123,7 @@ module RightScale
               @ping_id = nil
             end
           rescue Exception => e
-            Log.error("Failed to cancel router ping", e, :trace)
-            @exception_stats.track("cancel ping", e)
+            ErrorTracker.log(self, "Failed to cancel router ping", e)
           end
         end
         request = Request.new("/router/ping", nil, {:from => @sender.identity, :token => AgentIdentity.generate})
@@ -167,8 +164,7 @@ module RightScale
         begin
           check(id = nil, max_ping_timeouts = 1)
         rescue Exception => e
-          Log.error("Failed connectivity check", e, :trace)
-          @exception_stats.track("check connectivity", e)
+          ErrorTracker.log(self, "Failed connectivity check", e)
         end
       end
       true

data/lib/right_agent/dispatcher.rb

@@ -27,8 +27,8 @@ module RightScale
 
     include ProtocolVersionMixin
 
-    class InvalidRequestType < Exception; end
-    class DuplicateRequest < Exception; end
+    class InvalidRequestType < RuntimeError; end
+    class DuplicateRequest < RuntimeError; end
 
     # (ActorRegistry) Registry for actors
     attr_reader :registry
@@ -88,7 +88,7 @@ module RightScale
       actor, method, idempotent = route(request)
       received_at = @request_stats.update(method, (token if request.is_a?(Request)))
       if (dup = duplicate?(request, method, idempotent))
-        raise DuplicateRequest.new(dup)
+        raise DuplicateRequest, dup
       end
       unless (result = expired?(request, method))
         result = perform(request, actor, method, idempotent)
@@ -110,9 +110,6 @@ module RightScale
     #     or nil if empty
     #   "dispatch failures"(Hash|nil):: Dispatch failure activity stats with keys "total", "percent", "last", and "rate"
     #     with percentage breakdown per failure type, or nil if none
-    #   "exceptions"(Hash|nil):: Exceptions raised per category, or nil if none
-    #     "total"(Integer):: Total for category
-    #     "recent"(Array):: Most recent as a hash of "count", "type", "message", "when", and "where"
     #   "rejects"(Hash|nil):: Request reject activity stats with keys "total", "percent", "last", and "rate"
     #     with percentage breakdown per reason ("duplicate (<method>)", "retry duplicate (<method>)", or
     #     "stale (<method>)"), or nil if none
@@ -123,7 +120,6 @@ module RightScale
       stats = {
         "dispatched cache"  => (@dispatched_cache.stats if @dispatched_cache),
         "dispatch failures" => @dispatch_failure_stats.all,
-        "exceptions"        => @exception_stats.stats,
         "rejects"           => @reject_stats.all,
         "requests"          => @request_stats.all,
         "response time"     => @request_stats.avg_duration
@@ -142,7 +138,6 @@ module RightScale
       @reject_stats = RightSupport::Stats::Activity.new
       @request_stats = RightSupport::Stats::Activity.new
       @dispatch_failure_stats = RightSupport::Stats::Activity.new
-      @exception_stats = RightSupport::Stats::Exceptions.new(@agent, @agent.exception_callback)
       true
     end
 
@@ -209,7 +204,7 @@ module RightScale
       method = method.to_sym
       actor = @registry.actor_for(prefix)
       if actor.nil? || !actor.respond_to?(method)
-        raise InvalidRequestType.new("Unknown actor or method for dispatching request <#{request.token}> of type #{request.type}")
+        raise InvalidRequestType, "Unknown actor or method for dispatching request <#{request.token}> of type #{request.type}"
       end
       [actor, method, actor.class.idempotent?(method)]
     end
@@ -231,40 +226,10 @@ module RightScale
       else
         actor.send(method, request.payload, request)
       end
-    rescue Exception => e
+    rescue StandardError => e
+      ErrorTracker.log(self, "Failed dispatching #{request.trace}", e, request)
       @dispatch_failure_stats.update("#{request.type}->#{e.class.name}")
-      OperationResult.error(handle_exception(actor, method, request, e))
-    end
-
-    # Handle exception by logging it, calling the actors exception callback method,
-    # and gathering exception statistics
-    #
-    # === Parameters
-    # actor(Actor):: Actor that failed to process request
-    # method(Symbol):: Name of actor method being dispatched to
-    # request(Packet):: Packet that dispatcher is acting upon
-    # exception(Exception):: Exception that was raised
-    #
-    # === Return
-    # (String):: Error description for this exception
-    def handle_exception(actor, method, request, exception)
-      error = "Could not handle #{request.type} request"
-      Log.error(error, exception, :trace)
-      begin
-        if actor && actor.class.exception_callback
-          case actor.class.exception_callback
-          when Symbol, String
-            actor.send(actor.class.exception_callback, method, request, exception)
-          when Proc
-            actor.instance_exec(method, request, exception, &actor.class.exception_callback)
-          end
-        end
-        @exception_stats.track(request.type, exception)
-      rescue Exception => e
-        Log.error("Failed handling error for #{request.type}", e, :trace)
-        @exception_stats.track(request.type, e) rescue nil
-      end
-      Log.format(error, exception)
+      OperationResult.error("Could not handle #{request.type} request", e)
     end
 
   end # Dispatcher