right_agent 0.5.1

data/lib/right_agent/serialize/serializer.rb

@@ -0,0 +1,149 @@
+#
+# Copyright (c) 2009-2011 RightScale Inc
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+require 'rubygems'
+require 'date'
+require 'json'
+
+require File.normalize_path(File.join(File.dirname(__FILE__), 'message_pack'))
+
+# Monkey patch common classes to support MessagePack serialization
+# As with JSON, unserializing them is manual using existing methods such as parse
+class Date
+  def to_msgpack(*a); to_s.to_msgpack(*a) end
+end
+
+class Time
+  def to_msgpack(*a); to_s.to_msgpack(*a) end
+end
+
+class DateTime
+  def to_msgpack(*a); to_s.to_msgpack(*a) end
+end
+
+module RightScale
+  
+  # Cascade serializer supporting MessagePack and JSON serialization formats
+  # as well as secure serialization
+  class Serializer
+
+    class SerializationError < StandardError
+      attr_accessor :action, :packet
+      def initialize(action, packet, serializers, msg = nil)
+        @action, @packet = action, packet
+        msg = " (#{msg})" if msg && !msg.empty?
+        super("Could not #{action} packet using #{serializers.inspect}#{msg}")
+      end
+    end
+
+    # (Symbol) Preferred serialization format
+    attr_reader :format
+
+    # Initialize the serializer
+    # Do not cascade serializers if :secure is specified
+    #
+    # === Parameters
+    # preferred_format(Symbol|String):: Preferred serialization format: :msgpack, :json, or :secure
+    #
+    # === Raises
+    # ArgumentError:: If preferred format is not supported
+    def initialize(preferred_format = nil)
+      @format = (preferred_format ||= DEFAULT_FORMAT).to_sym
+      raise ArgumentError, "Serializer format #{@format.inspect} not one of #{FORMATS.inspect}" unless FORMATS.include?(@format)
+      @secure = (@format == :secure)
+    end
+
+    # Serialize object using preferred serializer
+    # Do not cascade
+    #
+    # === Parameters
+    # packet(Object):: Object to be serialized
+    # format(Symbol):: Override preferred format
+    #
+    # === Return
+    # (String):: Serialized object
+    def dump(packet, format = nil)
+      cascade_serializers(:dump, packet, [@secure ? SecureSerializer : SERIALIZERS[format || @format]])
+    end
+
+    # Unserialize object using cascaded serializers with order chosen by peaking at first byte
+    #
+    # === Parameters
+    # packet(String):: Data representing serialized object
+    #
+    # === Return
+    # (Object):: Unserialized object
+    def load(packet)
+      cascade_serializers(:load, packet, @secure ? [SecureSerializer] : order_serializers(packet))
+    end
+
+    private
+
+    # Supported serialization formats
+    SERIALIZERS = {:msgpack => MessagePack, :json => JSON}.freeze
+    MSGPACK_FIRST_SERIALIZERS = [MessagePack, JSON].freeze
+    JSON_FIRST_SERIALIZERS = MSGPACK_FIRST_SERIALIZERS.clone.reverse.freeze
+    FORMATS = (SERIALIZERS.keys + [:secure]).freeze
+    DEFAULT_FORMAT = :msgpack
+
+    # Apply serializers in order until one succeeds
+    #
+    # === Parameters
+    # action(Symbol):: Serialization action: :dump or :load
+    # packet(Object|String):: Object or serialized data on which action is to be performed
+    # serializers(Array):: Serializers to apply in order
+    #
+    # === Return
+    # (String|Object):: Result of serialization action
+    #
+    # === Raises
+    # SerializationError:: If none of the serializers can perform the requested action
+    def cascade_serializers(action, packet, serializers)
+      errors = []
+      serializers.map do |serializer|
+        obj = nil
+        begin
+          obj = serializer.__send__(action, packet)
+        rescue SecureSerializer::MissingCertificate, SecureSerializer::InvalidSignature => e
+          errors << Log.format("Failed to #{action} with #{serializer.name}", e)
+        rescue Exception => e
+          errors << Log.format("Failed to #{action} with #{serializer.name}", e, :trace)
+        end
+        return obj if obj
+      end
+      raise SerializationError.new(action, packet, serializers, errors.join("\n"))
+    end
+
+    # Determine likely serialization format and order serializers accordingly
+    #
+    # === Parameters
+    # packet(String):: Data representing serialized object
+    #
+    # === Return
+    # (Array):: Ordered serializers
+    def order_serializers(packet)
+      packet[0] > 127 ? MSGPACK_FIRST_SERIALIZERS : JSON_FIRST_SERIALIZERS
+    end
+
+  end # Serializer
+  
+end # RightScale

data/lib/right_agent/stats_helper.rb

@@ -0,0 +1,731 @@
+# Copyright (c) 2009-2011 RightScale Inc
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+module RightScale
+
+  # Mixin for collecting and displaying operational statistics for servers
+  module StatsHelper
+
+    # Maximum characters in stat name
+    MAX_STAT_NAME_WIDTH = 11
+
+    # Maximum characters in sub-stat name
+    MAX_SUB_STAT_NAME_WIDTH = 17
+
+    # Maximum characters in sub-stat value line
+    MAX_SUB_STAT_VALUE_WIDTH = 80
+
+    # Maximum characters displayed for exception message
+    MAX_EXCEPTION_MESSAGE_WIDTH = 60
+
+    # Separator between stat name and stat value
+    SEPARATOR = " : "
+
+    # Time constants
+    MINUTE = 60
+    HOUR = 60 * MINUTE
+    DAY = 24 * HOUR
+
+    # Track activity statistics
+    class ActivityStats
+
+      # Number of samples included when calculating average recent activity
+      # with the smoothing formula A = ((A * (RECENT_SIZE - 1)) + V) / RECENT_SIZE,
+      # where A is the current recent average and V is the new activity value
+      # As a rough guide, it takes approximately 2 * RECENT_SIZE activity values
+      # at value V for average A to reach 90% of the original difference between A and V
+      # For example, for A = 0, V = 1, RECENT_SIZE = 3 the progression for A is
+      # 0, 0.3, 0.5, 0.7, 0.8, 0.86, 0.91, 0.94, 0.96, 0.97, 0.98, 0.99, ...
+      RECENT_SIZE = 3
+
+      # Maximum string length for activity type
+      MAX_TYPE_SIZE = 60
+
+      # (Integer) Total activity count
+      attr_reader :total
+
+      # (Hash) Count of activity per type
+      attr_reader :count_per_type
+
+      # Initialize activity data
+      #
+      # === Parameters
+      # measure_rate(Boolean):: Whether to measure activity rate
+      def initialize(measure_rate = true)
+        @measure_rate = measure_rate
+        reset
+      end
+
+      # Reset statistics
+      #
+      # === Return
+      # true:: Always return true
+      def reset
+        @interval = 0.0
+        @last_start_time = Time.now
+        @avg_duration = nil
+        @total = 0
+        @count_per_type = {}
+        @last_type = nil
+        @last_id = nil
+        true
+      end
+
+      # Mark the start of an activity and update counts and average rate
+      # with weighting toward recent activity
+      # Ignore the update if its type contains "stats"
+      #
+      # === Parameters
+      # type(String|Symbol):: Type of activity, with anything that is not a symbol, true, or false
+      #   automatically converted to a String and truncated to MAX_TYPE_SIZE characters,
+      #   defaults to nil
+      # id(String):: Unique identifier associated with this activity
+      #
+      # === Return
+      # now(Time):: Update time
+      def update(type = nil, id = nil)
+        now = Time.now
+        if type.nil? || !(type =~ /stats/)
+          @interval = average(@interval, now - @last_start_time) if @measure_rate
+          @last_start_time = now
+          @total += 1
+          unless type.nil?
+            unless [Symbol, TrueClass, FalseClass].include?(type.class)
+              type = type.inspect unless type.is_a?(String)
+              type = type[0, MAX_TYPE_SIZE - 3] + "..." if type.size > (MAX_TYPE_SIZE - 3)
+            end
+            @count_per_type[type] = (@count_per_type[type] || 0) + 1
+          end
+          @last_type = type
+          @last_id = id
+        end
+        now
+      end
+
+      # Mark the finish of an activity and update the average duration
+      #
+      # === Parameters
+      # start_time(Time):: Time when activity started, defaults to last time update was called
+      # id(String):: Unique identifier associated with this activity
+      #
+      # === Return
+      # duration(Float):: Activity duration in seconds
+      def finish(start_time = nil, id = nil)
+        now = Time.now
+        start_time ||= @last_start_time
+        duration = now - start_time
+        @avg_duration = average(@avg_duration || 0.0, duration)
+        @last_id = 0 if id && id == @last_id
+        duration
+      end
+
+      # Convert average interval to average rate
+      #
+      # === Return
+      # (Float|nil):: Recent average rate, or nil if total is 0
+      def avg_rate
+        if @total > 0
+          if @interval == 0.0 then 0.0 else 1.0 / @interval end
+        end
+      end
+
+
+      # Get average duration of activity
+      #
+      # === Return
+      # (Float|nil) Average duration in seconds of activity weighted toward recent activity, or nil if total is 0
+      def avg_duration
+        @avg_duration if @total > 0
+      end
+
+      # Get stats about last activity
+      #
+      # === Return
+      # (Hash|nil):: Information about last activity, or nil if the total is 0
+      #   "elapsed"(Integer):: Seconds since last activity started
+      #   "type"(String):: Type of activity if specified, otherwise omitted
+      #   "active"(Boolean):: Whether activity still active
+      def last
+        if @total > 0
+          result = {"elapsed" => (Time.now - @last_start_time).to_i}
+          result["type"] = @last_type if @last_type
+          result["active"] = @last_id != 0 if !@last_id.nil?
+          result
+        end
+      end
+
+      # Convert count per type into percentage by type
+      #
+      # === Return
+      # (Hash|nil):: Converted counts, or nil if total is 0
+      #   "total"(Integer):: Total activity count
+      #   "percent"(Hash):: Percentage for each type of activity if tracking type, otherwise omitted
+      def percentage
+        if @total > 0
+          percent = {}
+          @count_per_type.each { |k, v| percent[k] = (v / @total.to_f) * 100.0 }
+          {"percent" => percent, "total" => @total}
+        end
+      end
+
+      # Get stat summary including all aspects of activity that were measured except duration
+      #
+      # === Return
+      # (Hash|nil):: Information about activity, or nil if the total is 0
+      #   "total"(Integer):: Total activity count
+      #   "percent"(Hash):: Percentage for each type of activity if tracking type, otherwise omitted
+      #   "last"(Hash):: Information about last activity
+      #     "elapsed"(Integer):: Seconds since last activity started
+      #     "type"(String):: Type of activity if tracking type, otherwise omitted
+      #     "active"(Boolean):: Whether activity still active if tracking whether active, otherwise omitted
+      #   "rate"(Float):: Recent average rate if measuring rate, otherwise omitted
+      def all
+        if @total > 0
+          result = if @count_per_type.empty?
+            {"total" => @total}
+          else
+            percentage
+          end
+          result.merge!("last" => last)
+          result.merge!("rate" => avg_rate) if @measure_rate
+          result
+        end
+      end
+
+      protected
+
+      # Calculate smoothed average with weighting toward recent activity
+      #
+      # === Parameters
+      # current(Float|Integer):: Current average value
+      # value(Float|Integer):: New value
+      #
+      # === Return
+      # (Float):: New average
+      def average(current, value)
+        ((current * (RECENT_SIZE - 1)) + value) / RECENT_SIZE.to_f
+      end
+
+    end # ActivityStats
+
+    # Track exception statistics
+    class ExceptionStats
+
+      # Maximum number of recent exceptions to track per category
+      MAX_RECENT_EXCEPTIONS = 10
+
+      # (Hash) Exceptions raised per category with keys
+      #   "total"(Integer):: Total exceptions for this category
+      #   "recent"(Array):: Most recent as a hash of "count", "type", "message", "when", and "where"
+      attr_reader :stats
+      alias :all :stats
+
+      # Initialize exception data
+      #
+      # === Parameters
+      # server(Object):: Server where exceptions are originating, must be defined for callbacks
+      # callback(Proc):: Block with following parameters to be activated when an exception occurs
+      #   exception(Exception):: Exception
+      #   message(Packet):: Message being processed
+      #   server(Server):: Server where exception occurred
+      def initialize(server = nil, callback = nil)
+        @server = server
+        @callback = callback
+        reset
+      end
+
+      # Reset statistics
+      #
+      # === Return
+      # true:: Always return true
+      def reset
+        @stats = nil
+        true
+      end
+
+      # Track exception statistics and optionally make callback to report exception
+      # Catch any exceptions since this function may be called from within an EM block
+      # and an exception here would then derail EM
+      #
+      # === Parameters
+      # category(String):: Exception category
+      # exception(Exception):: Exception
+      #
+      # === Return
+      # true:: Always return true
+      def track(category, exception, message = nil)
+        begin
+          @callback.call(exception, message, @server) if @server && @callback && message
+          @stats ||= {}
+          exceptions = (@stats[category] ||= {"total" => 0, "recent" => []})
+          exceptions["total"] += 1
+          recent = exceptions["recent"]
+          last = recent.last
+          if last && last["type"] == exception.class.name && last["message"] == exception.message && last["where"] == exception.backtrace.first
+            last["count"] += 1
+            last["when"] = Time.now.to_i
+          else
+            backtrace = exception.backtrace.first if exception.backtrace
+            recent.shift if recent.size >= MAX_RECENT_EXCEPTIONS
+            recent.push({"count" => 1, "when" => Time.now.to_i, "type" => exception.class.name,
+                         "message" => exception.message, "where" => backtrace})
+          end
+        rescue Exception => e
+          Log.error("Failed to track exception '#{exception}' due to: #{e}\n" + e.backtrace.join("\n")) rescue nil
+        end
+        true
+      end
+
+    end # ExceptionStats
+
+    # Utility functions that are useful on there own
+    class Utilities
+
+      # Convert values hash into percentages
+      #
+      # === Parameters
+      # values(Hash):: Values to be converted whose sum is the total for calculating percentages
+      #
+      # === Return
+      # (Hash):: Converted values with keys "total" and "percent" with latter being a hash with values as percentages
+      def self.percentage(values)
+        total = 0
+        values.each_value { |v| total += v }
+        percent = {}
+        values.each { |k, v| percent[k] = (v / total.to_f) * 100.0 } if total > 0
+        {"percent" => percent, "total" => total}
+      end
+
+      # Convert elapsed time in seconds to displayable format
+      #
+      # === Parameters
+      # time(Integer|Float):: Elapsed time
+      #
+      # === Return
+      # (String):: Display string
+      def self.elapsed(time)
+        time = time.to_i
+        if time <= MINUTE
+          "#{time} sec"
+        elsif time <= HOUR
+          minutes = time / MINUTE
+          seconds = time - (minutes * MINUTE)
+          "#{minutes} min #{seconds} sec"
+        elsif time <= DAY
+          hours = time / HOUR
+          minutes = (time - (hours * HOUR)) / MINUTE
+          "#{hours} hr #{minutes} min"
+        else
+          days = time / DAY
+          hours = (time - (days * DAY)) / HOUR
+          minutes = (time - (days * DAY) - (hours * HOUR)) / MINUTE
+          "#{days} day#{days == 1 ? '' : 's'} #{hours} hr #{minutes} min"
+        end
+      end
+
+      # Determine enough precision for floating point value(s) so that all have
+      # at least two significant digits and then convert each value to a decimal digit
+      # string of that precision after applying rounding
+      # When precision is wide ranging, limit precision of the larger numbers
+      #
+      # === Parameters
+      # value(Float|Array|Hash):: Value(s) to be converted
+      #
+      # === Return
+      # (String|Array|Hash):: Value(s) converted to decimal digit string
+      def self.enough_precision(value)
+        scale = [1.0, 10.0, 100.0, 1000.0, 10000.0, 100000.0]
+        enough = lambda { |v| (v >= 10.0   ? 0 :
+                              (v >= 1.0    ? 1 :
+                              (v >= 0.1    ? 2 :
+                              (v >= 0.01   ? 3 :
+                              (v >  0.001  ? 4 :
+                              (v >  0.0    ? 5 : 0)))))) }
+        digit_str = lambda { |p, v| sprintf("%.#{p}f", (v * scale[p]).round / scale[p])}
+
+        if value.is_a?(Float)
+          digit_str.call(enough.call(value), value)
+        elsif value.is_a?(Array)
+          min, max = value.map { |_, v| enough.call(v) }.minmax
+          precision = (max - min) > 1 ? min + 1 : max
+          value.map { |k, v| [k, digit_str.call([precision, enough.call(v)].max, v)] }
+        elsif value.is_a?(Hash)
+          min, max = value.to_a.map { |_, v| enough.call(v) }.minmax
+          precision = (max - min) > 1 ? min + 1 : max
+          value.to_a.inject({}) { |s, v| s[v[0]] = digit_str.call([precision, enough.call(v[1])].max, v[1]); s }
+        else
+          value.to_s
+        end
+      end
+
+      # Wrap string by breaking it into lines at the specified separator
+      #
+      # === Parameters
+      # string(String):: String to be wrapped
+      # max_length(Integer):: Maximum length of a line excluding indentation
+      # indent(String):: Indentation for each line
+      # separator(String):: Separator at which to make line breaks
+      #
+      # === Return
+      # (String):: Multi-line string
+      def self.wrap(string, max_length, indent, separator)
+        all = []
+        line = ""
+        for l in string.split(separator)
+          if (line + l).length >= max_length
+            all.push(line)
+            line = ""
+          end
+          line += line == "" ? l : separator + l
+        end
+        all.push(line).join(separator + "\n" + indent)
+      end
+
+    end
+
+    # Convert 0 value to nil
+    # This is in support of displaying "none" rather than 0
+    #
+    # === Parameters
+    # value(Integer|Float):: Value to be converted
+    #
+    # === Returns
+    # (Integer|Float|nil):: nil if value is 0, otherwise the original value
+    def nil_if_zero(value)
+      value == 0 ? nil : value
+    end
+
+    # Convert values hash into percentages
+    #
+    # === Parameters
+    # values(Hash):: Values to be converted whose sum is the total for calculating percentages
+    #
+    # === Return
+    # (Hash):: Converted values with keys "total" and "percent" with latter being a hash with values as percentages
+    def percentage(values)
+      Utilities.percentage(values)
+    end
+
+    # Determine enough precision for floating point value(s) so that all have
+    # at least two significant digits and then convert each value to a decimal digit
+    # string of that precision after applying rounding
+    # When precision is wide ranging, limit precision of the larger numbers
+    #
+    # === Parameters
+    # value(Float|Array|Hash):: Value(s) to be converted
+    #
+    # === Return
+    # (String|Array|Hash):: Value(s) converted to decimal digit string
+    def enough_precision(value)
+      Utilities.enough_precision(value)
+    end
+
+    # Wrap string by breaking it into lines at the specified separator
+    #
+    # === Parameters
+    # string(String):: String to be wrapped
+    # max_length(Integer):: Maximum length of a line excluding indentation
+    # indent(String):: Indentation for each line
+    # separator(String):: Separator at which to make line breaks
+    #
+    # === Return
+    # (String):: Multi-line string
+    def wrap(string, max_length, indent, separator)
+      Utilities.wrap(string, max_length, indent, separator)
+    end
+
+    # Convert elapsed time in seconds to displayable format
+    #
+    # === Parameters
+    # time(Integer|Float):: Elapsed time
+    #
+    # === Return
+    # (String):: Display string
+    def elapsed(time)
+      Utilities.elapsed(time)
+    end
+
+    # Format UTC time value
+    #
+    # === Parameters
+    # time(Integer):: Time in seconds in Unix-epoch to be formatted
+    #
+    # (String):: Formatted time string
+    def time_at(time)
+      Time.at(time).strftime("%a %b %d %H:%M:%S")
+    end
+
+    # Sort hash elements by key in ascending order into array of key/value pairs
+    # Sort keys numerically if possible, otherwise as is
+    #
+    # === Parameters
+    # hash(Hash):: Data to be sorted
+    #
+    # === Return
+    # (Array):: Key/value pairs from hash in key sorted order
+    def sort_key(hash)
+      hash.to_a.map { |k, v| [k =~ /^\d+$/ ? k.to_i : k, v] }.sort
+    end
+
+    # Sort hash elements by value in ascending order into array of key/value pairs
+    #
+    # === Parameters
+    # hash(Hash):: Data to be sorted
+    #
+    # === Return
+    # (Array):: Key/value pairs from hash in value sorted order
+    def sort_value(hash)
+      hash.to_a.sort { |a, b| a[1] <=> b[1] }
+    end
+
+    # Converts server statistics to a displayable format
+    #
+    # === Parameters
+    # stats(Hash):: Statistics with generic keys "name", "identity", "hostname", "service uptime",
+    #   "machine uptime", "stat time", "last reset time", "version", and "broker" with the
+    #   latter two and "machine uptime" being optional; any other keys ending with "stats"
+    #   have an associated hash value that is displayed in sorted key order
+    #
+    # === Return
+    # (String):: Display string
+    def stats_str(stats)
+      name_width = MAX_STAT_NAME_WIDTH
+      str = stats["name"] ? sprintf("%-#{name_width}s#{SEPARATOR}%s\n", "name", stats["name"]) : ""
+      str += sprintf("%-#{name_width}s#{SEPARATOR}%s\n", "identity", stats["identity"]) +
+             sprintf("%-#{name_width}s#{SEPARATOR}%s\n", "hostname", stats["hostname"]) +
+             sprintf("%-#{name_width}s#{SEPARATOR}%s\n", "stat time", time_at(stats["stat time"])) +
+             sprintf("%-#{name_width}s#{SEPARATOR}%s\n", "last reset", time_at(stats["last reset time"])) +
+             sprintf("%-#{name_width}s#{SEPARATOR}%s\n", "service up", elapsed(stats["service uptime"]))
+      str += sprintf("%-#{name_width}s#{SEPARATOR}%s\n", "machine up", elapsed(stats["machine uptime"])) if stats.has_key?("machine uptime")
+      str += sprintf("%-#{name_width}s#{SEPARATOR}%s\n", "version", stats["version"].to_i) if stats.has_key?("version")
+      str += brokers_str(stats["brokers"], name_width) if stats.has_key?("brokers")
+      stats.to_a.sort.each { |k, v| str += sub_stats_str(k[0..-7], v, name_width) if k.to_s =~ /stats$/ }
+      str
+    end
+
+    # Convert broker information to displayable format
+    #
+    # === Parameter
+    # brokers(Hash):: Broker stats with keys
+    #   "brokers"(Array):: Stats for each broker in priority order as hash with keys
+    #     "alias"(String):: Broker alias
+    #     "identity"(String):: Broker identity
+    #     "status"(Symbol):: Status of connection
+    #     "disconnect last"(Hash|nil):: Last disconnect information with key "elapsed", or nil if none
+    #     "disconnects"(Integer|nil):: Number of times lost connection, or nil if none
+    #     "failure last"(Hash|nil):: Last connect failure information with key "elapsed", or nil if none
+    #     "failures"(Integer|nil):: Number of failed attempts to connect to broker, or nil if none
+    #     "retries"(Integer|nil):: Number of attempts to connect after failure, or nil if none
+    #   "exceptions"(Hash|nil):: Exceptions raised per category, or nil if none
+    #     "total"(Integer):: Total exceptions for this category
+    #     "recent"(Array):: Most recent as a hash of "count", "type", "message", "when", and "where"
+    #   "returns"(Hash|nil):: Message return activity stats with keys "total", "percent", "last", and "rate"
+    #     with percentage breakdown per request type, or nil if none
+    # name_width(Integer):: Fixed width for left-justified name display
+    #
+    # === Return
+    # str(String):: Broker display with one line per broker plus exceptions
+    def brokers_str(brokers, name_width)
+      value_indent = " " * (name_width + SEPARATOR.size)
+      sub_name_width = MAX_SUB_STAT_NAME_WIDTH
+      sub_value_indent = " " * (name_width + sub_name_width + (SEPARATOR.size * 2))
+      str = sprintf("%-#{name_width}s#{SEPARATOR}", "brokers")
+      brokers["brokers"].each do |b|
+        disconnects = if b["disconnects"]
+          "#{b["disconnects"]} (#{elapsed(b["disconnect last"]["elapsed"])} ago)"
+        else
+          "none"
+        end
+        failures = if b["failures"]
+          retries = b["retries"]
+          retries = " w/ #{retries} #{retries != 1 ? 'retries' : 'retry'}" if retries
+          "#{b["failures"]} (#{elapsed(b["failure last"]["elapsed"])} ago#{retries})"
+        else
+          "none"
+        end
+        str += "#{b["alias"]}: #{b["identity"]} #{b["status"]}, disconnects: #{disconnects}, failures: #{failures}\n"
+        str += value_indent
+      end
+      str += sprintf("%-#{sub_name_width}s#{SEPARATOR}", "exceptions")
+      str += if brokers["exceptions"].nil? || brokers["exceptions"].empty?
+        "none\n"
+      else
+        exceptions_str(brokers["exceptions"], sub_value_indent) + "\n"
+      end
+      str += value_indent
+      str += sprintf("%-#{sub_name_width}s#{SEPARATOR}", "returns")
+      str += if brokers["returns"].nil? || brokers["returns"].empty?
+        "none\n"
+      else
+        wrap(activity_str(brokers["returns"]), MAX_SUB_STAT_VALUE_WIDTH, sub_value_indent, ", ") + "\n"
+      end
+    end
+
+    # Convert grouped set of statistics to displayable format
+    # Provide special formatting for stats named "exceptions"
+    # Break out percentages and total count for stats containing "percent" hash value
+    # sorted in descending percent order and followed by total count
+    # Convert to elapsed time for stats with name ending in "last"
+    # Add "/sec" to values with name ending in "rate"
+    # Add " sec" to values with name ending in "time"
+    # Add "%" to values with name ending in "percent" and drop "percent" from name
+    # Use elapsed time formatting for values with name ending in "age"
+    # Display any nil value, empty hash, or hash with a "total" value of 0 as "none"
+    # Display any floating point value or hash of values with at least two significant digits of precision
+    #
+    # === Parameters
+    # name(String):: Display name for the stat
+    # value(Object):: Value of this stat
+    # name_width(Integer):: Fixed width for left-justified name display
+    #
+    # === Return
+    # (String):: Single line display of stat
+    def sub_stats_str(name, value, name_width)
+      value_indent = " " * (name_width + SEPARATOR.size)
+      sub_name_width = MAX_SUB_STAT_NAME_WIDTH
+      sub_value_indent = " " * (name_width + sub_name_width + (SEPARATOR.size * 2))
+      sprintf("%-#{name_width}s#{SEPARATOR}", name) + value.to_a.sort.map do |attr|
+        k, v = attr
+        name = k =~ /percent$/ ? k[0..-9] : k
+        sprintf("%-#{sub_name_width}s#{SEPARATOR}", name) + if v.is_a?(Float) || v.is_a?(Integer)
+          str = k =~ /age$/ ? elapsed(v) : enough_precision(v)
+          str += "/sec" if k =~ /rate$/
+          str += " sec" if k =~ /time$/
+          str += "%" if k =~ /percent$/
+          str
+        elsif v.is_a?(Hash)
+          if v.empty? || v["total"] == 0
+            "none"
+          elsif v["total"]
+            wrap(activity_str(v), MAX_SUB_STAT_VALUE_WIDTH, sub_value_indent, ", ")
+          elsif k =~ /last$/
+            last_activity_str(v)
+          elsif k == "exceptions"
+            exceptions_str(v, sub_value_indent)
+          else
+            wrap(hash_str(v), MAX_SUB_STAT_VALUE_WIDTH, sub_value_indent, ", ")
+          end
+        else
+          "#{v || "none"}"
+        end + "\n"
+      end.join(value_indent)
+    end
+
+    # Convert activity information to displayable format
+    #
+    # === Parameters
+    # value(Hash|nil):: Information about activity, or nil if the total is 0
+    #   "total"(Integer):: Total activity count
+    #   "percent"(Hash):: Percentage for each type of activity if tracking type, otherwise omitted
+    #   "last"(Hash):: Information about last activity
+    #     "elapsed"(Integer):: Seconds since last activity started
+    #     "type"(String):: Type of activity if tracking type, otherwise omitted
+    #     "active"(Boolean):: Whether activity still active if tracking whether active, otherwise omitted
+    #   "rate"(Float):: Recent average rate if measuring rate, otherwise omitted
+    #   "duration"(Float):: Average duration of activity if tracking duration, otherwise omitted
+    #
+    # === Return
+    # str(String):: Activity stats in displayable format without any line separators
+    def activity_str(value)
+      str = ""
+      str += enough_precision(sort_value(value["percent"]).reverse).map { |k, v| "#{k}: #{v}%" }.join(", ") +
+             ", total: " if value["percent"]
+      str += "#{value['total']}"
+      str += ", last: #{last_activity_str(value['last'], single_item = true)}" if value["last"]
+      str += ", rate: #{enough_precision(value['rate'])}/sec" if value["rate"]
+      str += ", duration: #{enough_precision(value['duration'])} sec" if value["duration"]
+      str
+    end
+
+    # Convert last activity information to displayable format
+    #
+    # === Parameters
+    # last(Hash):: Information about last activity
+    #   "elapsed"(Integer):: Seconds since last activity started
+    #   "type"(String):: Type of activity if tracking type, otherwise omitted
+    #   "active"(Boolean):: Whether activity still active if tracking whether active, otherwise omitted
+    # single_item:: Whether this is to appear as a single item in a comma-separated list
+    #   in which case there should be no ':' in the formatted string
+    #
+    # === Return
+    # str(String):: Last activity in displayable format without any line separators
+    def last_activity_str(last, single_item = false)
+      str = "#{elapsed(last['elapsed'])} ago"
+      str += " and still active" if last["active"]
+      if last["type"]
+        if single_item
+          str = "#{last['type']} (#{str})"
+        else
+          str = "#{last['type']}: #{str}"
+        end
+      end
+      str
+    end
+
+    # Convert exception information to displayable format
+    #
+    # === Parameters
+    # exceptions(Hash):: Exceptions raised per category
+    #   "total"(Integer):: Total exceptions for this category
+    #   "recent"(Array):: Most recent as a hash of "count", "type", "message", "when", and "where"
+    # indent(String):: Indentation for each line
+    #
+    # === Return
+    # (String):: Exceptions in displayable format with line separators
+    def exceptions_str(exceptions, indent)
+      indent2 = indent + (" " * 4)
+      exceptions.to_a.sort.map do |k, v|
+        sprintf("%s total: %d, most recent:\n", k, v["total"]) + v["recent"].reverse.map do |e|
+          message = e["message"]
+          if message && message.size > (MAX_EXCEPTION_MESSAGE_WIDTH - 3)
+            message = e["message"][0, MAX_EXCEPTION_MESSAGE_WIDTH - 3] + "..."
+          end
+          indent + "(#{e["count"]}) #{time_at(e["when"])} #{e["type"]}: #{message}\n" + indent2 + "#{e["where"]}"
+        end.join("\n")
+      end.join("\n" + indent)
+    end
+
+    # Convert arbitrary nested hash to displayable format
+    # Sort hash by key, numerically if possible, otherwise as is
+    # Display any floating point values with one decimal place precision
+    # Display any empty values as "none"
+    #
+    # === Parameters
+    # hash(Hash):: Hash to be displayed
+    #
+    # === Return
+    # (String):: Single line hash display
+    def hash_str(hash)
+      str = ""
+      sort_key(hash).map do |k, v|
+        "#{k}: " + if v.is_a?(Float)
+          enough_precision(v)
+        elsif v.is_a?(Hash)
+          "[ " + hash_str(v) + " ]"
+        else
+          "#{v || "none"}"
+        end
+      end.join(", ")
+    end
+
+  end # StatsHelper
+
+end # RightScale