scout_agent 3.0.0

data/lib/scout_agent/server.rb

@@ -0,0 +1,123 @@
+#!/usr/bin/env ruby -wKU
+
+module ScoutAgent
+  # 
+  # This class is a thin wrapper over RestClient for Scout's check-in API.
+  # Public methods are provided for each action you can perform againt the API.
+  # 
+  class Server
+    # 
+    # Create a new API wrapper, optionally with a +log+ to write connection
+    # details to.
+    # 
+    def initialize(log = WireTap.new(nil))
+      @log         = log
+      @rest_client = RestClient::Resource.new(
+        Plan.agent_url,
+        :headers => { :client_version   => ScoutAgent::VERSION,
+                      :accept_encoding  => "gzip" }
+      )
+    end
+    
+    # The log connection notes will be written to.
+    attr_reader :log
+    
+    # 
+    # This method fetches the current plan for this agent.  You can pass any
+    # +additional_headers+ for the request if needed (mainly useful for
+    # <tt>:if_modified_since</tt>).
+    # 
+    # This method will return the raw plan when it is successfully fetched.  An
+    # empty String is returned if the plan is malformed or unchanged.  Finally,
+    # +nil+ is returned if the plan cannot be retrieved for some reason.
+    # 
+    def get_plan(additional_headers = { })
+      no_warnings {  # keep OpenSSL quiet
+        @rest_client["plan.scout"].get(additional_headers)
+      }
+    rescue Zlib::Error  # could not decompress response
+      log.warn("Plan was malformed zipped data.")
+      ""  # replace bad plan with empty plan
+    rescue RestClient::RequestFailed => error  # RestClient bug workaround
+      # all success codes are OK, but other codes are not
+      if error.http_code.between? 200, 299
+        error.response.body  # the returned plan
+      else
+        log.warn("Plan was returned as a failure code:  #{error.http_code}.")
+        nil  # failed to retrieve plan
+      end
+    rescue RestClient::NotModified
+      log.info("Plan was not modified.")
+      ""  # empty plan
+    rescue Errno::ECONNREFUSED, RestClient::Exception => error  # bad networking
+      log.warn("Plan could not be retrieved:  #{error.class}.")
+      nil  # failed to retrieve plan
+    end
+    
+    #
+    # This method can be used to send +data+ to the Scout API as a check-in.
+    # The data is zipped before it is sent to reduce the cost duplicated reports
+    # as much as possible.
+    # 
+    # If the ckeck-in succeeds, +true+ is returned.  However, +false+ doesn't
+    # ensure that we failed.  RestClient may time out a long connection attempt,
+    # but the server may still complete it eventually.
+    # 
+    def post_checkin(data)
+      io   =  StringIO.new
+      gzip =  Zlib::GzipWriter.new(io)
+      gzip << data.to_json
+      gzip.close
+      no_warnings do  # keep OpenSSL quiet
+        @rest_client["checkin.scout"].post(
+          io.string,
+          :content_type     => "application/json",
+          :content_encoding => "gzip"
+        )
+      end
+      true
+    rescue Zlib::Error  # could not compress data for sending
+      log.error("Check-in could not be zipped.")
+      false
+    rescue RestClient::RequestFailed => error  # RestClient bug workaround
+      # all success codes are OK, but other codes are not
+      if error.http_code.between? 200, 299
+        true
+      else
+        log.warn( "Check-in was returned as a failure code:  " +
+                  "#{error.http_code}." )
+        false
+      end
+    rescue Errno::ECONNREFUSED, RestClient::Exception  # networking problem
+      log.warn("Check-in could not be sent:  #{error.class}.")
+      false  # we failed to send and will retry later
+    end
+    
+    # 
+    # Uploads +log_file+ to the server for troubleshooting.  Returns +true+ if
+    # the upload succeeded.
+    # 
+    def post_log(log_file)
+      no_warnings do  # keep OpenSSL quiet
+        @rest_client["log.scout"].post(
+          log_file.read,
+          :content_type     => "text/plain",
+          :content_encoding => "gzip"
+        )
+      end
+      true
+    rescue RestClient::RequestFailed => error  # RestClient bug workaround
+      # all success codes are OK, but other codes are not
+      if error.http_code.between? 200, 299
+        true
+      else
+        log.warn( "Log upload was returned as a failure code:  " +
+                  "#{error.http_code}." )
+        false
+      end
+    rescue Errno::ECONNREFUSED, RestClient::Exception  # networking problem
+      log.warn("Log could not be sent:  #{error.class}.")
+      false  # could not send log
+    end
+  end
+end

data/lib/scout_agent/tracked.rb

@@ -0,0 +1,59 @@
+#!/usr/bin/env ruby -wKU
+
+module ScoutAgent
+  # 
+  # A mix-in for adding status tracking to various objects.  Methods are
+  # provided to set your current status and clear your status.
+  # 
+  module Tracked
+    # 
+    # Returns the <tt>log()</tt> for this object, if supported and set, or a bit
+    # bucket log object.
+    # 
+    def status_log
+      (respond_to?(:log) and log) or WireTap.new(nil)
+    end
+    
+    #
+    # Returns the status database this object this object can use to update
+    # its status.
+    # 
+    # This value is cached after the first call to speed up future interactions
+    # with the database.  You'll need to reset the cached value with a call to
+    # force_status_database_reload() after doing something that would invalidate
+    # a SQLite database handle like <tt>fork()</tt>.
+    # 
+    def status_database(log = nil)
+      @status_database ||= Database.load(:statuses, status_log)
+    end
+    
+    # Call this to clear the cached database handle after a <tt>fork()</tt>.
+    def force_status_database_reload
+      @status_database = nil
+    end
+    
+    # 
+    # :call-seq:
+    #   status(status, name = IDCard.me && IDCard.me.process_name)
+    # 
+    # Sets the +status+ of this process.  You can pass +name+ explicitly if it
+    # cannot be properly determined from <tt>IDCard.me()</tt>.
+    # 
+    def status(*args)
+      status_database.update_status(*args) if status_database
+    end
+    
+    # 
+    # :call-seq:
+    #   clear_status(name = IDCard.me && IDCard.me.process_name)
+    # 
+    # Clears any status currently set for this process.  This should be called
+    # for any process setting status messages <tt>at_exit()</tt>.  You can pass
+    # +name+ explicitly if it cannot be properly determined from
+    # <tt>IDCard.me()</tt>.
+    # 
+    def clear_status(*args)
+      status_database.clear_status(*args) if status_database
+    end
+  end
+end

data/lib/scout_agent/wire_tap.rb

@@ -0,0 +1,513 @@
+#!/usr/bin/env ruby -wKU
+
+module ScoutAgent
+  # 
+  # WireTap is a complete replacement for Ruby's standard Logger class, with a
+  # few changes:
+  # 
+  # * The code is multi-Process safe
+  # * Log can be duplicated on a second device
+  # * The default formatter produces a trimmed down output
+  # * Severity levels can be dynamically altered (not used in the agent)
+  # 
+  # The main change was to make the code work for multiple processes.  This is 
+  # done by forcing code to aquire the proper locks during all file
+  # interactions.  Log rotation also had to be changed to avoid moving an active
+  # log file out from under another process.  Instead, files are copied and
+  # truncated.  This also required a seek() to the end of the file before each
+  # write to be added to defeat some pos() caching that seemed to be occurring.
+  # The end results of all of this is that WireTap is safer than Logger, but
+  # slower because of the extra precausions it must take.
+  # 
+  # The tap=() method on instances of this class, allows you to set a second
+  # device input can be copied to.
+  # 
+  class WireTap
+    # A base Class for all WireTap failure conditions.
+    class Error         < RuntimeError; end
+    # The Error thrown when WireTap is unable to shift logs for whatever reason.
+    class ShiftingError < Error;        end
+    
+    # 
+    # Instances of this Class are used to format all outgoing log messages.  
+    # WireTap will pass messages to call() which is expected to return a String
+    # ready for output.
+    # 
+    # This class works very similar to Logger's default formatter (supporting
+    # things like logging Exception backtraces and Ruby objects with inspect())
+    # with the following changes:
+    # 
+    # * The default message format is simplified
+    # * The default timestamp format is more readable
+    # * All lines after the first in a logged message are indented
+    # * Program names are shown with an attached PID
+    # 
+    class Formatter
+      # 
+      # The sprintf() style format used to format messages:
+      # 
+      #   SEVERITY [TIME PROCESS]:  MESSAGE
+      # 
+      FORMAT = "%s [%s %s]:  %s\n"
+      
+      # Creates a new formatter without a timestamp format.
+      def initialize
+        @datetime_format = nil
+      end
+      
+      # Used to manually set the timestamp format, overriding the default.
+      attr_accessor :datetime_format
+      
+      # 
+      # The main interface required by Formatter instances.  Turns +severity+,
+      # +time+, +progname+, and +message+ into a String for writing to a log
+      # file.
+      # 
+      def call(severity, time, progname, message)
+        FORMAT % [ severity,
+                   format_time(time),
+                   format_progname(progname),
+                   format_message(message) ]
+      end
+      
+      #######
+      private
+      #######
+      
+      # Formats +time+ using the set strftime() pattern, or a readable default.
+      def format_time(time)
+        if @datetime_format.nil?
+          time.strftime("%Y-%m-%d %H:%M:%S")
+        else
+          time.strftime(@datetime_format)
+        end
+      end
+      
+      # 
+      # Formats +progname+ by adding on a PID, or replacing it with a PID when
+      # +nil+.
+      # 
+      def format_progname(progname)
+        pid = Process.pid
+        if progname
+          "%s(%d)" % [progname, pid]
+        else
+          pid
+        end
+      end
+      
+      # 
+      # Formats the +message+ object into a String.  String objects are
+      # unchanged, Exception objects are converted to a message, type, and
+      # backtrace, and all other objects have inspect() called on them.
+      # 
+      # If this process results in more than one line, all other lines are
+      # indented two spaces to make the resulting file trivial to parse.
+      # 
+      def format_message(message)
+        output = case message
+                 when String
+                   message
+                 when Exception
+                   "%s (%s)\n%s" % [ message.message,
+                                     message.class,
+                                     message.backtrace.join("\n") ]
+                 else
+                   message.inspect
+                 end
+        output.strip.gsub("\n", "\n  ")
+      end
+    end
+
+    #
+    # This Class is used to wrap a device passed to WireTap and then manage
+    # interactions with that device.  These interactions are handled in a
+    # multi-Thread and multi-Process safe way.
+    # 
+    # WireTap's LogDevice requires a bigger underlying File-like interface than
+    # Logger's version do to the extra locking and rotation requirements.  It
+    # skips these steps when the device does not support them though, to make it
+    # possible to log to something like <tt>$stdout</tt>.
+    # 
+    class LogDevice
+      # Used in log rotation calculations.
+      SECONDS_IN_A_DAY = 60 * 60 * 24
+      
+      # 
+      # Wraps +log+ with +options+, which may include <tt>:shift_age</tt> and
+      # <tt>:shift_size</tt>.  See WireTap::new() for details on these settings.
+      # 
+      def initialize(log = nil, options = Hash.new)
+        if log.respond_to?(:write) and log.respond_to?(:close)
+          @dev      = log
+          @filename = nil
+        else
+          @dev      = open_log(log)
+          @filename = log
+        end
+        @shift_age  = options[:shift_age]  || 7
+        @shift_size = options[:shift_size] || 1048576
+        @lock       = Mutex.new
+      end
+      
+      # The underlying device being managed.
+      attr_reader :dev
+      # A base name for files created by this device.
+      attr_reader :filename
+      
+      # 
+      # Writes +message+ to +dev+ in a multi-Thread and multi-Process safe
+      # manner.
+      # 
+      # Before any write occurs, a check is made to see if the content should be
+      # shifted.  That process will be trigger, if needed.
+      # 
+      def write(message)
+        @lock.synchronize do
+          not @dev.respond_to?(:flock) or @dev.flock(File::LOCK_EX)
+          begin
+            begin
+              shift_log if @filename and @dev.respond_to?(:stat)
+            rescue Exception => error  # shifting failed for whatever reason
+              raise ShiftingError, "Shifting failed:  #{error.message}"
+            end
+            begin
+              # 
+              # make sure we are in the right place, even if the data has been
+              # truncated() by another process as part of rotation
+              # 
+              @dev.seek(0, IO::SEEK_END)
+            rescue Errno::ESPIPE  # illegal seek, probably $stdout or $stderr
+              # do nothing:  such streams are always at the end
+            end
+            @dev.write(message)
+          ensure
+            not @dev.respond_to?(:flock) or @dev.flock(File::LOCK_UN)
+          end
+        end
+      end
+
+      # Closes the underlying +dev+ in a multi-Thread safe manner.
+      def close
+        @lock.synchronize do
+          @dev.close
+        end
+      end
+      
+      #######
+      private
+      #######
+      
+      # 
+      # Opens a new log file in a multi-Process safe manner, adding a header
+      # message, if this Process was the one to create the file.
+      # 
+      def open_log(filename)
+        dev = open(filename, File::CREAT | File::EXCL | File::WRONLY)
+        dev.chmod(0777)  # make sure this file is available to all processes
+        write_header(dev, :created)
+        dev
+      rescue Errno::EEXIST  # file already exists
+        dev = open(filename, File::CREAT | File::APPEND | File::WRONLY)
+      ensure
+        dev.sync = true if dev
+      end
+      
+      # 
+      # Sends a simple header to +dev+ saying when it was created or rotated.
+      # 
+      # Technically, this method is not perfectly multi-Process safe.  It's
+      # possible for one Process to create the file, but have another Process
+      # log one or more messages before this header can be added.  This only
+      # affects the position of this message though and thus is deemed and
+      # acceptable risk.  The agent is careful never to trigger this scenario.
+      # 
+      def write_header(dev, verb)
+        dev.write  "# Log %s at %s by %s\n" %
+                   [verb, Time.now.strftime("%Y-%m-%d %H:%M:%S"), Process.pid]
+      end
+      
+      # 
+      # Forwards to shift_log_by_size() or shift_log_by_period() as dictated by
+      # the settings for this LogDevice.
+      # 
+      def shift_log
+        if @shift_age.is_a? Integer
+          shift_log_by_size
+        else
+          shift_log_by_period
+        end
+      end
+      
+      # Shifts log files if the file size has been exceeded.
+      def shift_log_by_size
+        return false unless @shift_age > 0 and @dev.stat.size > @shift_size
+        (@shift_age - 3).downto(0) do |i|
+          if File.exist? shifted_filename(i)
+            FileUtils.mv(shifted_filename(i), shifted_filename(i + 1))
+          end
+        end
+        multiprocessing_safe_shift(shifted_filename(0))
+      end
+      
+      # Shifts log files if the period length has been exceeded.
+      def shift_log_by_period
+        now        = Time.now
+        period_end = previous_period_end(now)
+        return false unless @dev.stat.mtime <= period_end
+        shifted_file = shifted_filename(period_end.strftime("%Y%m%d"))
+        if File.exist? shifted_file
+          raise "'%s' already exists." % shifted_file
+        end
+        multiprocessing_safe_shift(shifted_file)
+      end
+      
+      # Returns a modified name for the file including +shifted_extension+.
+      def shifted_filename(shifted_extension)
+        "#{@filename}.#{shifted_extension}"
+      end
+      
+      # Shifts the log file with a copy, truncate(), and rewind() process.
+      def multiprocessing_safe_shift(to_file)
+        FileUtils.cp(@filename, to_file)
+        @dev.truncate(0)
+        @dev.rewind
+        write_header(@dev, :rotated)
+        true
+      end
+      
+      # A helper used to determine the previous period end from +now+.
+      def previous_period_end(now)
+        case @shift_age.to_s.downcase
+        when "daily"
+          end_of_day(now - SECONDS_IN_A_DAY)
+        when "weekly"
+          end_of_day(now - (now.wday + 1) * SECONDS_IN_A_DAY)
+        when "monthly"
+          end_of_day(now - now.mday * SECONDS_IN_A_DAY)
+        else
+          now
+        end
+      end
+      
+      # 
+      # A helper that converts a +day+ Time object into a new Time at the end of
+      # that +day+.
+      # 
+      def end_of_day(day)
+        Time.local(day.year, day.month, day.mday, 23, 59, 59)
+      end
+    end
+    
+    # 
+    # This module includes the severity levels used by all WireTap instances.
+    # Changing the constants in this module will rework the interface methods
+    # in WireTap used to log messages.  Any changes made here should be done
+    # before logs are created.
+    # 
+    # WireTap instances will have the following methods for each constant in
+    # this module (examples are for the +WARN+ constant):
+    # 
+    #   # check if a level is currently being logged
+    #   if wire_tap.warn?
+    #     # ...
+    #   end
+    # 
+    #   # log a message at the indicated level
+    #   wire_tap.warn "Message goes here."
+    # 
+    #   # log a message, skipping creation if the level is not being logged
+    #   wire_tap.warn { build_expensive_warning_message() }
+    # 
+    #   # log a message, overriding +progname+
+    #   wire_tap.warn("my_process_name") { "Message goes here." }
+    # 
+    module Severity
+      # Low-lever information for developers.
+      DEBUG   = 0
+      # Useful information about normal system operations.
+      INFO    = 1
+      # A warning about something that may not have worked as expected.
+      WARN    = 2
+      # A handled error condition.
+      ERROR   = 3
+      # An unhandleable error.
+      FATAL   = 4
+      # 
+      # A non-classified message.  (Dynamic method dispatch is not done for this
+      # special case.)
+      # 
+      UNKNOWN = 5
+    end
+    
+    # Make the level constants available at the instance level.
+    include Severity
+    
+    # 
+    # Wraps +logdev+ for logging.
+    # 
+    # You can set +shift_age+ and/or +shift_size+ to setup log rotation.  You
+    # can pass <tt>:daily</tt>, <tt>:weekly</tt>, or <tt>:monthly</tt> as
+    # +shift_age+ to get the named rotation.  Alternately, you can set
+    # +shift_age+ to the number of log files to keep and +shift_size+ to the
+    # byte size at which log files are rotated out.
+    # 
+    def initialize(logdev, shift_age = 0, shift_size = 1048576)
+      @logdev    = logdev &&
+                   LogDevice.new( logdev, :shift_age  => shift_age,
+                                          :shift_size => shift_size )
+      @formatter = Formatter.new
+      @level     = min_level
+      @progname  = nil
+    end
+    
+    # 
+    # The Formatter instance being used to format the messages logged by this
+    # WireTap instance.
+    # 
+    attr_accessor :formatter
+    # The current level below which no messages will be logged.
+    attr_accessor :level
+    alias_method  :sev_threshold,  :level
+    alias_method  :sev_threshold=, :level=
+    # The Process name used in messages logged by this instance.
+    attr_accessor :progname
+    
+    # 
+    # Sets a new timestamp format on the Formatter used to format messages for 
+    # this instance.  +datetime_format+ should be a strftime() pattern String.
+    # 
+    def datetime_format=(datetime_format)
+      @formatter.datetime_format = datetime_format
+    end
+    
+    # 
+    # Returns the timestamp format currently in use (or +nil+ if the default is
+    # being used) to format messages for this instance.
+    # 
+    def datetime_format
+      @formatter.datetime_format
+    end
+    
+    # Sets a second +logdev+ all messages should be copied to.
+    def tap=(logdev)
+      @tap = logdev && LogDevice.new(logdev)
+    end
+    
+    # 
+    # Returns the secondary device messages are being logged to (or +nil+ if
+    # none is in use).
+    # 
+    def tap
+      @tap ||= nil
+    end
+    
+    # Returns +true+ if a +level+ message would currently be written.
+    def level?(level)
+      @level <= level_by_name(level)
+    end
+    
+    # 
+    # A low-level way to log a +message+ with +severity+.  You can optionally
+    # pass a +progname+ override.  You can also choose to pass the message in a
+    # +block+ that will be run to build the message only if it would be written.
+    # 
+    def add(severity, message = nil, progname = nil, &block)
+      severity ||= max_level
+      return true if @logdev.nil? or severity < @level
+      progname ||= @progname
+      if message.nil?
+        if block.nil?
+          message, progname = progname, @progname
+        else
+          message = block.call
+        end
+      end
+      output = @formatter.call( level_names[severity] || "ANY",
+                                Time.now,
+                                progname,
+                                message )
+      @logdev.write(output)
+      tap.write(output) if tap
+      true
+    end
+    alias_method :log, :add
+    
+    # 
+    # Write's a raw +message+ to the log without any formatting or
+    # preprocessing.
+    # 
+    def <<(message)
+      @logdev.write(message) unless @logdev.nil?
+    end
+    
+    # An override to show the dynamic level methods.  See Severity for details.
+    def respond_to?(message, include_private = false)
+      if level_methods.include? message.to_s
+        true
+      else
+        super
+      end
+    end
+    
+    # Dynamic level method dispatch.  See Severity for details.
+    def method_missing(method, *args, &block)
+      case method.to_s
+      when /\A(#{level_re})\?\z/i
+        level?($1, *args, &block)
+      when /\A(#{level_re})\z/i
+        add(levels[$1.upcase], nil, *args, &block)
+      else
+        super
+      end
+    end
+    
+    # Closes the underlying log.
+    def close
+      @logdev.close unless @logdev.nil?
+    end
+    
+    #######
+    private
+    #######
+    
+    # Fetches a level from Severity by +name+.
+    def level_by_name(name)
+      Severity.const_get(name.to_s.upcase)
+    end
+    
+    # Builds a Hash of all levels used.
+    def levels
+      @levels ||= Hash[ *Severity.constants.
+                                  map { |name| [name, level_by_name(name)] }.
+                                  flatten ]
+    end
+    
+    # Build the inverse Hash of levels by name.
+    def level_names
+      @level_names ||= levels.invert.merge(levels["UNKNOWN"] => "ANY")
+    end
+    
+    # Builds an Array of level method names.
+    def level_methods
+      @level_methods ||= (levels.keys - ["UNKNOWN"]).map { |l| [l, "#{l}"] }.
+                                                     flatten.
+                                                     map { |l| l.downcase }
+    end
+    
+    # Builds a Regexp that will match level method names.
+    def level_re
+      @level_re ||= levels.keys.map { |name| Regexp.escape(name) }.join("|")
+    end
+    
+    # Returns the minimum level.
+    def min_level
+      @min_level ||= levels.values.min
+    end
+    
+    # Returns the maximum level.
+    def max_level
+      @max_level ||= levels.values.min
+    end
+  end
+end