right_agent 2.2.1 → 2.3.0

data/lib/right_agent/enrollment_result.rb

@@ -20,8 +20,8 @@ module RightScale
     # Versions 5 and above use an identical format for the enrollment result
     SUPPORTED_VERSIONS = 5..AgentConfig.protocol_version
     
-    class IntegrityFailure < Exception; end
-    class VersionError < Exception; end
+    class IntegrityFailure < StandardError; end
+    class VersionError < StandardError; end
 
     attr_reader :r_s_version, :timestamp, :router_cert, :id_cert, :id_key
 

data/lib/right_agent/error_tracker.rb

@@ -0,0 +1,202 @@
+# Copyright (c) 2014 RightScale, Inc, All Rights Reserved Worldwide.
+#
+# THIS PROGRAM IS CONFIDENTIAL AND PROPRIETARY TO RIGHTSCALE
+# AND CONSTITUTES A VALUABLE TRADE SECRET.  Any unauthorized use,
+# reproduction, modification, or disclosure of this program is
+# strictly prohibited.  Any use of this program by an authorized
+# licensee is strictly subject to the terms and conditions,
+# including confidentiality obligations, set forth in the applicable
+# License Agreement between RightScale.com, Inc. and
+# the licensee.
+
+module RightScale
+
+  # Tracker for unexpected errors
+  # Logs them with appropriate trace information
+  # Accumulates statistics about exceptions
+  # Reports exceptions to external Errbit service via HydraulicBrake
+  class ErrorTracker
+
+    include RightSupport::Ruby::EasySingleton
+
+    # Container for exception statistics
+    attr_reader :exception_stats
+
+    # Initialize error tracker
+    #
+    # @param [Object] agent object using this tracker
+    # @param [String] agent_name uniquely identifying agent process on given server
+    #
+    # @option options [Integer, NilClass] :shard_id identifying shard of database in use
+    # @option options [Hash] :trace_level for restricting backtracing and Errbit reporting
+    #   with exception class as key and :no_trace, :caller, or :trace as value; exceptions
+    #   with :no_trace are not backtraced when logging nor are they recorded in stats
+    #   or reported to Errbit
+    # @option options [String] :airbrake_endpoint URL for Airbrake for reporting exceptions
+    #   to Errbit
+    # @option options [String] :airbrake_api_key for using the Airbrake API to access Errbit
+    #
+    # @return [TrueClass] always true
+    def init(agent, agent_name, options = {})
+      @agent = agent
+      @trace_level = options[:trace_level] || {}
+      notify_init(agent_name, options[:shard_id], options[:airbrake_endpoint], options[:airbrake_api_key])
+      reset_stats
+      true
+    end
+
+    # Log error and optionally track in stats
+    # Errbit notification is left to the callback configured in the stats tracker
+    #
+    # @param [String, Object] component reporting error; non-string is snake-cased
+    # @param [String] description of failure for use in logging
+    # @param [Exception, String] exception to be logged and tracked in stats;
+    #   string errors are logged but not tracked in stats
+    # @param [Packet, Hash, NilClass] packet associated with exception
+    # @param [Symbol, NilClass] trace level override unless excluded by configured
+    #   trace levels
+    #
+    # @return [Boolean] true if successfully logged, otherwise false
+    def log(component, description, exception = nil, packet = nil, trace = nil)
+      if exception.nil?
+        Log.error(description)
+      elsif exception.is_a?(String)
+        Log.error(description, exception)
+      else
+        trace = (@trace_level && @trace_level[exception.class]) || trace || :trace
+        Log.error(description, exception, trace)
+        track(component, exception, packet) if trace != :no_trace
+      end
+      true
+    rescue StandardError => e
+      Log.error("Failed to log error", e, :trace) rescue nil
+      false
+    end
+
+    # Track error in stats
+    #
+    # @param [String] component reporting error
+    # @param [Exception] exception to be tracked
+    # @param [Packet, Hash, NilClass] packet associated with exception
+    #
+    # @return [TrueClass] always true
+    def track(component, exception, packet = nil)
+      component = component.class.name.split("::").last.snake_case unless component.is_a?(String)
+      @exception_stats.track(component, exception, packet)
+    end
+
+    # Notify Errbit of error if notification enabled
+    #
+    # @param [Exception, String] exception raised
+    # @param [Packet, Hash] packet associated with exception
+    # @param [Object] agent object reporting error
+    # @param [String] component or service area where error occurred
+    #
+    # @return [TrueClass] always true
+    def notify(exception, packet = nil, agent = nil, component = nil)
+      if @notify_enabled
+        data = {
+          :error_message => exception.respond_to?(:message) ? exception.message : exception.to_s,
+          :backtrace => exception.respond_to?(:backtrace) ? exception.backtrace : caller,
+          :environment_name => ENV["RAILS_ENV"],
+        }
+        if agent
+          data[:cgi_data] = (@cgi_data || {}).merge(:agent_class => agent.class.name)
+        elsif @cgi_data
+          data[:cgi_data] = @cgi_data
+        end
+        data[:error_class] = exception.class.name if exception.is_a?(Exception)
+        data[:component] = component if component
+        if packet && packet.is_a?(Packet)
+          data[:action] = packet.type.split("/").last if packet.respond_to?(:type)
+          data[:parameters] = packet.payload if packet.respond_to?(:payload)
+          uuid = packet.token if packet.respond_to?(:token)
+        elsif packet.is_a?(Hash)
+          action = packet[:path] || packet["path"]
+          data[:action] = action.split("/").last if action
+          data[:parameters] = packet[:data] || packet["data"]
+          uuid = packet[:uuid] || packet["uuid"]
+        end
+        data[:session_data] = {:uuid => uuid} if uuid
+        HydraulicBrake.notify(data)
+      end
+      true
+    rescue Exception => e
+      Log.error("Failed to notify Errbit", e, :trace)
+    end
+
+    # Create proc for making callback to notifier
+    #
+    # @return [Proc] notifier callback
+    def notify_callback
+      Proc.new do |exception, packet, agent, component|
+        notify(exception, packet, agent, component)
+      end
+    end
+
+    # Get exception statistics
+    #
+    # @param reset [Boolean] Whether to reset the statistics after getting the current ones
+    #
+    # @return [Hash] current statistics
+    def stats(reset = false)
+      stats = {"exceptions" => @exception_stats.all}
+      reset_stats if reset
+      stats
+    end
+
+    protected
+
+    # Reset statistics
+    # Do not recreate exception stats since may be referenced externally
+    #
+    # @return [TrueClass] always true
+    def reset_stats
+      @exception_stats ||= RightSupport::Stats::Exceptions.new(@agent, notify_callback)
+      @exception_stats.reset
+    end
+
+    # Configure HydraulicBreak for exception notification
+    #
+    # @param [String] agent_name uniquely identifying agent process on given server
+    # @param [Integer, NilClass] shard_id identifying shard of database in use
+    # @param [String] endpoint URL for Airbrake for reporting exceptions to Errbit
+    # @param [String] api_key for using the Airbrake API to access Errbit
+    #
+    # @return [TrueClass] always true
+    #
+    # @raise [RuntimeError] hydraulic_brake gem missing
+    def notify_init(agent_name, shard_id, endpoint, api_key)
+      if endpoint && api_key
+        unless require_succeeds?("hydraulic_brake")
+          raise RuntimeError, "hydraulic_brake gem missing - required if airbrake options used in ErrorTracker"
+        end
+
+        @cgi_data = {
+          :shard_id   => shard_id,
+          :process    => $0,
+          :pid        => Process.pid,
+          :agent_name => agent_name
+        }
+        @cgi_data[:shard_id] = shard_id if shard_id
+        @cgi_data[:sha] = CURRENT_SOURCE_SHA if defined?(CURRENT_SOURCE_SHA)
+
+        uri = URI.parse(endpoint)
+        HydraulicBrake.configure do |config|
+          config.secure = (uri.scheme == "https")
+          config.host = uri.host
+          config.port = uri.port
+          config.api_key = api_key
+          config.project_root = AgentConfig.root_dir
+        end
+        @notify_enabled = true
+      else
+        @cgi_data = {}
+        @notify_enabled = false
+      end
+      true
+    end
+
+  end # ErrorTracker
+
+end # RightScale

data/lib/right_agent/log.rb

@@ -410,8 +410,6 @@ module RightScale
       @logger = logger
     end
 
-    protected
-
     # Initialize logger
     #
     # === Parameters

data/lib/right_agent/packets.rb

@@ -107,7 +107,7 @@ module RightScale
       }.to_msgpack(*a)
       @size = msg.size
       # For ruby 1.9 size attribute moves from front to back of packet
-      re = RUBY_VERSION < "1.9.0" ? /size\xC0/ : /size\xC0$/
+      re = RUBY_VERSION < "1.9.0" ? Regexp.new("size\xC0") : Regexp.new("size\xC0$", nil, "n")
       # For msgpack 0.5.1 the to_msgpack result is a MessagePack::Packer so need to convert to string
       msg = msg.to_s.sub!(re) { |m| "size" + @size.to_msgpack }
       msg

data/lib/right_agent/pending_requests.rb

@@ -97,12 +97,15 @@ module RightScale
 
     # Get age of youngest pending request
     #
+    # === Parameters
+    # pending_requests(Hash):: Pending requests to be examined
+    #
     # === Return
     # age(Integer):: Age of youngest request
-    def youngest_age
+    def self.youngest_age(pending_requests)
       now = Time.now
       age = nil
-      self.each_value do |r|
+      pending_requests.each_value do |r|
         seconds = (now - r.receive_time).to_i
         age = seconds if age.nil? || seconds < age
       end
@@ -111,12 +114,15 @@ module RightScale
 
     # Get age of oldest pending request
     #
+    # === Parameters
+    # pending_requests(Hash):: Pending requests to be examined
+    #
     # === Return
     # age(Integer):: Age of oldest request
-    def oldest_age
+    def self.oldest_age(pending_requests)
       now = Time.now
       age = nil
-      self.each_value do |r|
+      pending_requests.each_value do |r|
         seconds = (now - r.receive_time).to_i
         age = seconds if age.nil? || seconds > age
       end

data/lib/right_agent/pid_file.rb

@@ -31,7 +31,7 @@ module RightScale
   #     the command protocol
   class PidFile
 
-    class AlreadyRunning < Exception; end
+    class AlreadyRunning < RuntimeError; end
 
     attr_reader :identity, :pid_file, :cookie_file
 
@@ -72,8 +72,8 @@ module RightScale
         FileUtils.mkdir_p(@pid_dir)
         open(@pid_file,'w') { |f| f.write(Process.pid) }
         File.chmod(0644, @pid_file)
-      rescue Exception => e
-        Log.error "Failed to create PID file: #{e.message}"
+      rescue StandardError => e
+        ErrorTracker.log(self, "Failed to create PID file", e, nil, :caller)
         raise
       end
       true

data/lib/right_agent/protocol_version_mixin.rb

@@ -34,9 +34,9 @@ module RightScale
     # Packet::DEFAULT_VERSION, which is true of all with version >= 12)
     def can_put_version_in_packet?(version); version && version != 0 end
 
-    # Test whether given version of agent uses /mapper/query_tags rather than the
-    # deprecated TagQuery packet
-    def can_use_mapper_query_tags?(version); version && version >= 8 end
+    # Test whether given version of agent uses /router/query_tags or /mapper/query_tags
+    #  rather than the deprecated TagQuery packet
+    def can_use_router_query_tags?(version); version && version >= 8 end
 
     # Test whether given version of agent can handle a request that is being retried
     # as indicated by a retries count in the Request packet

data/lib/right_agent/scripts/agent_deployer.rb

@@ -45,6 +45,8 @@
 #      --grace-timeout SEC      Set number of seconds before graceful termination times out
 #      --[no-]dup-check         Set whether to check for and reject duplicate requests, .e.g., due to retries
 #      --fiber-pool-size, -f N  Set size of fiber pool
+#      --airbrake-endpoint URL  Set URL for Airbrake endpoint for reporting exceptions to Errbit
+#      --airbrake-api-key KEY   Set Airbrake API key for use in reporting exceptions to Errbit
 #      --options, -o KEY=VAL    Set options that act as final override for any persisted configuration settings
 #      --monit                  Generate monit configuration file
 #      --test                   Build test deployment using default test settings
@@ -205,6 +207,14 @@ module RightScale
           options[:heartbeat] = sec.to_i
         end
 
+        opts.on('--airbrake-endpoint URL') do |url|
+          options[:airbrake_endpoint] = url
+        end
+
+        opts.on('--airbrake-api-key KEY') do |key|
+          options[:airbrake_api_key] = key
+        end
+
         opts.on('-o', '--options OPT') do |e|
           fail("Invalid option definition #{e}' (use '=' to separate name and value)") unless e.include?('=')
           key, val = e.split(/=/)
@@ -277,7 +287,7 @@ module RightScale
       actors_dirs = AgentConfig.actors_dirs
       actors.each do |a|
         found = false
-        actors_dirs.each { |d| break if found = File.exist?(File.normalize_path(File.join(d, "#{a}.rb"))) }
+        actors_dirs.each { |d| break if (found = File.exist?(File.normalize_path(File.join(d, "#{a}.rb")))) }
         fail("Cannot find source for actor #{a.inspect} in #{actors_dirs.inspect}") unless found
       end
       true
@@ -318,6 +328,8 @@ module RightScale
       cfg[:http_proxy]         = options[:http_proxy] if options[:http_proxy]
       cfg[:http_no_proxy]      = options[:http_no_proxy] if options[:http_no_proxy]
       cfg[:fiber_pool_size]    = options[:fiber_pool_size] if options[:fiber_pool_size]
+      cfg[:airbrake_endpoint]  = options[:airbrake_endpoint] if options[:airbrake_endpoint]
+      cfg[:airbrake_api_key]   = options[:airbrake_api_key] if options[:airbrake_api_key]
       cfg
     end
 

data/lib/right_agent/sender.rb

@@ -68,10 +68,6 @@ module RightScale
     #
     # === Parameters
     # agent(Agent):: Agent using this sender; uses its identity, client, and following options:
-    #   :exception_callback(Proc):: Callback with following parameters that is activated on exception events:
-    #     exception(Exception):: Exception
-    #     message(Packet):: Message being processed
-    #     agent(Agent):: Reference to agent
     #   :offline_queueing(Boolean):: Whether to queue request if client currently disconnected,
     #     also requires agent invocation of initialize_offline_queue and start_offline_queue methods below,
     #     as well as enable_offline_mode and disable_offline_mode as client connection status changes
@@ -102,7 +98,7 @@ module RightScale
       @connectivity_checker = if @mode == :amqp
         # Only need connectivity checker for AMQP broker since RightHttpClient does its own checking
         # via periodic session renewal
-        ConnectivityChecker.new(self, @options[:ping_interval] || 0, @ping_stats, @exception_stats)
+        ConnectivityChecker.new(self, @options[:ping_interval] || 0, @ping_stats)
       end
       @@instance = self
     end
@@ -418,7 +414,7 @@ module RightScale
         @offline_handler.terminate
         @connectivity_checker.terminate if @connectivity_checker
         pending = @pending_requests.kind(:send_request)
-        [pending.size, pending.youngest_age]
+        [pending.size, PendingRequests.youngest_age(pending)]
       else
         [0, nil]
       end
@@ -448,9 +444,6 @@ module RightScale
     #
     # === Return
     # stats(Hash):: Current statistics:
-    #   "exceptions"(Hash|nil):: Exceptions raised per category, or nil if none
-    #     "total"(Integer):: Total exceptions for this category
-    #     "recent"(Array):: Most recent as a hash of "count", "type", "message", "when", and "where"
     #   "non-deliveries"(Hash|nil):: Non-delivery activity stats with keys "total", "percent", "last",
     #     and 'rate' with percentage breakdown per reason, or nil if none
     #   "offlines"(Hash|nil):: Offline activity stats with keys "total", "last", and "duration",
@@ -482,11 +475,10 @@ module RightScale
           pending["pushes"] = @pending_requests.kind(:send_push).size
           requests = @pending_requests.kind(:send_request)
           if (pending["requests"] = requests.size) > 0
-            pending["oldest age"] = requests.oldest_age
+            pending["oldest age"] = PendingRequests.oldest_age(requests)
           end
         end
         stats = {
-          "exceptions"       => @exception_stats.stats,
           "non-deliveries"   => @non_delivery_stats.all,
           "offlines"         => offlines,
           "pings"            => @ping_stats.all,
@@ -520,7 +512,6 @@ module RightScale
       @offline_stats = RightSupport::Stats::Activity.new(measure_rate = false)
       @request_kind_stats = RightSupport::Stats::Activity.new(measure_rate = false)
       @send_failure_stats = RightSupport::Stats::Activity.new
-      @exception_stats = RightSupport::Stats::Exceptions.new(@agent, @options[:exception_callback])
       true
     end
 
@@ -614,9 +605,8 @@ module RightScale
         EM_S.next_tick do
           begin
             http_send_once(kind, target, packet, received_at, &callback)
-          rescue Exception => e
-            Log.error("Failed sending or handling response for #{packet.trace} #{packet.type}", e, :trace)
-            @exception_stats.track("request", e)
+          rescue StandardError => e
+            ErrorTracker.log(self, "Failed sending or handling response for #{packet.trace} #{packet.type}", e)
           end
         end
       else
@@ -668,8 +658,7 @@ module RightScale
           result = error_result(e.inspect)
         else
           agent_type = AgentIdentity.parse(@identity).agent_type
-          Log.error("Failed to send #{packet.trace} #{packet.type}", e, :trace)
-          @exception_stats.track("request", e)
+          ErrorTracker.log(self, "Failed to send #{packet.trace} #{packet.type}", e)
           result = error_result("#{agent_type.capitalize} agent internal error")
         end
       end
@@ -739,21 +728,17 @@ module RightScale
     # TemporarilyOffline:: If cannot send request because RightNet client currently disconnected
     #   and offline queueing is disabled
     def amqp_send_once(packet, ids = nil)
-      name =
       exchange = {:type => :fanout, :name => @request_queue, :options => {:durable => true, :no_declare => @secure}}
       @agent.client.publish(exchange, packet, :persistent => packet.persistent, :mandatory => true,
                             :log_filter => [:tags, :target, :tries, :persistent], :brokers => ids)
     rescue RightAMQP::HABrokerClient::NoConnectedBrokers => e
-      msg = "Failed to publish request #{packet.trace} #{packet.type}"
-      Log.error(msg, e)
+      ErrorTracker.log(self, error = "Failed to publish request #{packet.trace} #{packet.type}", e, packet)
       @send_failure_stats.update("NoConnectedBrokers")
-      raise TemporarilyOffline.new(msg + " (#{e.class}: #{e.message})")
-    rescue Exception => e
-      msg = "Failed to publish request #{packet.trace} #{packet.type}"
-      Log.error(msg, e, :trace)
+      raise TemporarilyOffline.new(error + " (#{e.class}: #{e.message})")
+    rescue StandardError => e
+      ErrorTracker.log(self, error = "Failed to publish request #{packet.trace} #{packet.type}", e, packet)
       @send_failure_stats.update(e.class.name)
-      @exception_stats.track("publish", e, packet)
-      raise SendFailure.new(msg + " (#{e.class}: #{e.message})")
+      raise SendFailure.new(error + " (#{e.class}: #{e.message})")
     end
 
     # Send request via AMQP with one or more retries if do not receive a response in time
@@ -802,14 +787,13 @@ module RightScale
               @connectivity_checker.check(check_broker_ids.first) if check_broker_ids.any? && count == 1
             end
           rescue TemporarilyOffline => e
-            Log.error("Failed retry for #{packet.trace} #{packet.type} because temporarily offline")
+            ErrorTracker.log(self, "Failed retry for #{packet.trace} #{packet.type} because temporarily offline")
           rescue SendFailure => e
-            Log.error("Failed retry for #{packet.trace} #{packet.type} because of send failure")
+            ErrorTracker.log(self, "Failed retry for #{packet.trace} #{packet.type} because of send failure")
           rescue Exception => e
             # Not sending a response here because something more basic is broken in the retry
             # mechanism and don't want an error response to preempt a delayed actual response
-            Log.error("Failed retry for #{packet.trace} #{packet.type} without responding", e, :trace)
-            @exception_stats.track("retry", e, packet)
+            ErrorTracker.log(self, "Failed retry for #{packet.trace} #{packet.type} without responding", e, packet)
           end
         end
       end