nagios-herald 0.0.2

data/lib/nagios-herald/helpers/splunk_alert_frequency.rb

@@ -0,0 +1,170 @@
+require 'net/http'
+require 'uri'
+require 'json'
+
+# Use Splunk to search for previous occurrences of a given Nagios alert and
+# report.
+#
+# Can search for host alerts (i.e. DOWN state) and service alerts for a
+# single host (i.e. 'Disk Space' is CRITICAL).
+#
+# Options:
+#  :duration - time (in days) to search [DEFAULT: 7 days]
+#  :hostname - Etsy Nagios-like hostname (i.e. web0200.ny4) [REQUIRED]
+#  :service_name - service name to search (i.e. 'Disk Space) [OPTIONAL]
+#
+#  If the :service_name argument is not present, this performs a search
+#  for host alerts.
+
+module NagiosHerald
+  module Helpers
+    class SplunkReporter
+      include NagiosHerald::Logging
+
+      # Public: Initialize a new SplunkReporter object.
+      #
+      # splunk_url - The URI of the Splunk search API.
+      # username - A username that's authorized to perform Splunk queries.
+      # password - Yeah, a password.
+      #
+      # Returns a new SplunkReporter object.
+      def initialize(splunk_url, username, password)
+        uri = URI.parse( splunk_url )
+        @splunk_host = uri.host
+        @splunk_port = uri.port
+        @splunk_uri  = uri.request_uri
+
+        @username      = username
+        @password      = password
+        @fields = ['hostname', 'service_name', 'state', 'date_year', 'date_month', 'date_mday', 'date_hour', 'date_minute']
+      end
+
+      # Public: Generate the Splunk query.
+      #
+      # hostname - The name of the alerting host we want to query.
+      # service - Optional service name that may be alerting.
+      #
+      # Returns the generated Splunk query.
+      def get_splunk_alert_query(hostname, service = nil)
+        # query for service alerts or host alerts, depending on which args were selected
+        query = "search index=nagios hostname=\"#{hostname}\""
+        if service.nil?
+            query += " state=\"DOWN\""
+        else
+            query += " service_name=\"#{service}\" (state=\"WARNING\" OR state=\"CRITICAL\" OR state=\"UNKNOWN\" OR state=\"DOWN\")"
+        end
+        query +=  "| fields #{@fields.join(',')}"
+        return query
+      end
+
+      # Public: Queries Splunk to determine how frequently an alert has fired in a given period.
+      #
+      # hostname - The name of the alerting host we want to query.
+      # service - Optional service name that may be alerting.
+      # options - The options hash containing parameters for manipulatin the query.
+      #
+      # Returns a hash containing results from the search.
+      def get_alert_frequency(hostname, service = nil, options = {})
+        duration = options[:duration] ? options[:duration] : 7
+
+        max_results = options[:max_results] ? options[:max_results] : 10000
+
+        latest_time = options[:latest_time] ? options[:latest_time] :"now"
+
+        params = {
+            'exec_mode'     => 'oneshot',
+            'earliest_time' => "-#{duration}d",
+            'latest_time'   => latest_time,
+            'output_mode'   => 'json',
+            'count'         => max_results
+        }
+
+        params['search'] = get_splunk_alert_query(hostname, service)
+
+        json_response = query_splunk(params)
+
+        return if json_response.nil?
+
+        events_count = aggregate_splunk_events(json_response)
+
+        duration.to_i > 1 ? period = "days" : period = "day"
+        return {
+            :period   => "#{duration} #{period}",
+            :service  => service,
+            :hostname => hostname,
+            :events_count => events_count
+        }
+      end
+
+      # Public: Performs the Splunk query.
+      #
+      # params - The parameters of the query.
+      #
+      # Returns the JSON results from the query.
+      def query_splunk(params)
+        http = Net::HTTP.new( @splunk_host, @splunk_port )
+        http.use_ssl = true
+        http.open_timeout = 1
+        http.read_timeout = 2
+        http.ssl_timeout = 1
+        http.verify_mode = OpenSSL::SSL::VERIFY_NONE    # don't validate the cert
+        request = Net::HTTP::Post.new( @splunk_uri )
+        request.basic_auth( @username, @password )
+        request.set_form_data( params )
+        response = http.request( request )
+        unless response.code.eql?( "200" )
+          logger.warn "Failed to submit search to Splunk."
+          return nil
+        end
+
+        begin
+          json = JSON.parse( response.body )
+        rescue Exception => e
+          # just warn; our formatter should handle this gracefully
+          logger.warn(e.message)
+          logger.warn("Failed to parse response from Splunk. Perhaps we got an empty result?")
+          return nil
+        end
+
+        return json
+      end
+
+      # Public: Aggregate the results from Splunk.
+      # Nagios logs an entry for each entity that got alerted; a single alert can
+      # result in many log entries so we need to account for this by creating a
+      # unique key to ensure we don't count duplicate log lines.
+      # Chances are we *won't* see a duplicate except in cases where an alert fires
+      # on the cusp of a minute (I've seen up to 4-second skew in timestamp for
+      # alert results returned from Splunk; this _can_ happen).
+      #
+      # json - The Splunk query results in JSON.
+      #
+      # Returns a string containing a message useful for outputting into a notification.
+      def aggregate_splunk_events(json)
+        events = {}
+        json.each do |alert|
+            state = alert['state']
+            event_key = @fields.map{|f| alert[f]}.join('-')
+            state_els = events.fetch(state, [])
+            events[state] = state_els << event_key
+        end
+
+        # get the alert counts by state
+        events_count = {}
+        events.map {|k,v| events_count[k] = v.uniq.count}
+        events_count.sort_by {|k,v| v}.reverse
+        return events_count
+      end
+
+      def format_splunk_results(events_count, hostname, duration, service=nil)
+        # speaka da English
+        duration.to_i > 1 ? period = "days" : period = "day"
+        msg = "HOST '#{hostname}' has experienced #{events_count.join(', ')} alerts"
+        msg += " for SERVICE '#{service}'" unless service.nil?
+        msg += " in the last #{duration} #{period}."
+        return msg
+      end
+
+    end
+  end
+end

data/lib/nagios-herald/helpers/splunk_query.rb

@@ -0,0 +1,119 @@
+require 'net/http'
+require 'uri'
+require 'json'
+
+# Query Splunk with arbitrary search criteria
+
+module NagiosHerald
+  module Helpers
+    class SplunkQuery
+      #include NagiosHerald::Logging
+
+      # Public: Initialize a new SplunkQuery object.
+      #
+      # query - A string representing the query to send to Splunk.
+      # index - Optional index to specify (else Splunk defaults to all indexes
+      #   available to the authenticated user).
+      # output - The output format we'd like (i.e. csv, json, xml); defaults
+      #   to json.
+      #
+      # Example:
+      #
+      # splunk_query = NagiosHerald::Helpers::SplunkQuery.new('sourcetype=perf_log page=index.html')
+      # splunk_query = NagiosHerald::Helpers::SplunkQuery.new('transaction_state=paid', {:index => 'get_paid'})
+      # splunk_query = NagiosHerald::Helpers::SplunkQuery.new('source=nagios-herald.log alert_type=host', {:output => 'csv'})
+      #
+      # Returns a new SplunkQuery object.
+      def initialize(query, options={})
+        @splunk_query = query
+        @splunk_index = options[:index] ? options[:index] : nil
+        @splunk_output = options[:output] ? options[:output] : 'json'
+
+        # Pull the Splunk URI, username, and password from the config.
+        splunk_url = Config.config['splunk']['url']
+        @splunk_username = Config.config['splunk']['username']
+        @splunk_password = Config.config['splunk']['password']
+
+        # Parse the URI.
+        uri = URI.parse(splunk_url)
+        @splunk_host = uri.host
+        @splunk_port = uri.port
+        @splunk_uri  = uri.request_uri
+      end
+
+      # Public: Generate the parameters for the Splunk query.
+      #
+      # Example:
+      #
+      # parameters = splunk_query.parameters
+      #
+      # Returns the Splunk query parameters.
+      def parameters
+        # Earliest time we should look for events; defaults to 7 days ago.
+        earliest_time = Config.config['splunk']['earliest_time'] ?
+          Config.config['splunk']['earliest_time'] :
+          '7d'
+
+        # Latest time we should look for events; defaults to now.
+        latest_time = Config.config['splunk']['latest_time'] ?
+          Config.config['splunk']['latest_time'] :
+          'now'
+
+        # Maximum results returned; defaults to 100.
+        max_results = Config.config['splunk']['max_results'] ?
+          Config.config['splunk']['max_results'] :
+          100
+
+        params = {
+            'exec_mode'     => 'oneshot',
+            'earliest_time' => "-#{earliest_time}",
+            'latest_time'   => latest_time,
+            'output_mode'   => @splunk_output,
+            'count'         => max_results
+        }
+        if @splunk_index.nil?
+          params['search'] = "search #{@splunk_query}"
+        else
+          params['search'] = "search index=#{@splunk_index} " + @splunk_query
+        end
+
+        params
+      end
+
+      # Public: Queries Splunk.
+      #
+      # Example:
+      #
+      # results = splunk_query.query
+      #
+      # Returns the results of the query in the requested format, nil otherwise.
+      def query
+        http = Net::HTTP.new( @splunk_host, @splunk_port )
+        http.use_ssl = true
+        http.open_timeout = 1
+        http.read_timeout = 2
+        http.ssl_timeout = 1
+        http.verify_mode = OpenSSL::SSL::VERIFY_NONE    # don't validate the cert
+        request = Net::HTTP::Post.new( @splunk_uri )
+        request.basic_auth( @splunk_username, @splunk_password )
+        request.set_form_data( parameters )
+        begin
+          response = http.request( request )
+        rescue Exception => e
+          logger.warn "Failed to send request: #{e.message}"
+          return nil
+        end
+
+        if response.code.eql?( "200" )
+          response.body
+        else
+          logger.warn "Splunk query failed with HTTP #{response.code}: #{response.message}"
+          logger.warn response.body
+          return nil
+        end
+      end
+
+    end
+  end
+end
+

data/lib/nagios-herald/helpers/url_image.rb

@@ -0,0 +1,76 @@
+require 'net/http'
+require 'uri'
+
+# TODO: don't assume the MIME type is image/png; provide a mechanism for ensuring a standard image size
+module NagiosHerald
+  module Helpers
+    class UrlImage
+
+      # Public: Requests an image by its URI.
+      #
+      # uri - The URI of the image resource.
+      #
+      # Returns the content of the image.
+      def self.get_image( uri )
+        graph = Net::HTTP.get( URI.parse( uri ) )
+      end
+
+      # Public: Writes te given content to a file name.
+      #
+      # file_name - The name of the file to write.
+      # content - Arbitrary content to write into the file.
+      #
+      # Returns true if successful, false otherwise.
+      def self.write_image( file_name, content )
+        File.delete( file_name ) if File.exists?( file_name )   # remove any pre-existing versions
+        written_size = File.open( file_name, 'w' ) { |f| f.write( content ) }
+        if written_size == content.size
+          return written_size > 0   # why aren't we just returning true?
+        else
+          false   # oops...
+        end
+      end
+
+      # Public: Convert the URI to a useful name to be used in the image file name.
+      # Removes the transport type and query characters.
+      #
+      # uri - The URI to be converted.
+      #
+      # Returns the converted URI.
+      # FIXME: This doesn't account for HTTPS URIs.
+      def self.convert_uri(uri)
+        converted_uri = uri.gsub("http:\/\/", "")
+        converted_uri.gsub!(/(\/|=|\?|&)/, "_")     # such a hack...
+        converted_uri.gsub!(/_+/, "_")              # de-dupe underscores
+        return converted_uri
+      end
+
+      # Public: Downloads an image and writes it to a file.
+      #
+      # uri - The URI of the image resource.
+      # path - The destination file for the image content.
+      #
+      # Returns nothing.
+      def self.download_image(uri, path)
+        graph = get_image( uri )
+        # only push the image path into the array if we successfully create it
+        write_image( path, graph ) ? path : nil
+      end
+
+      # To be honest, I don't recall this method's purpose.
+      # It's only called in the ``bin/get_graph`` script. May be dead code.
+      # TODO: Determine if this is still necessary.
+      def self.download_images( uris, path )
+        path = path.sub(/\/$/, "")   # strip the trailing slash (if it exists) so the components of image_name are clear
+        image_paths = []
+        uris.each do |uri|
+          converted_uri = convert_uri(uri)
+          image_path = "#{path}/#{converted_uri}.png"
+          # only push the image path into the array if we successfully create it
+          image_paths.push(image_path) if download_image(uri, image_path)
+        end
+        image_paths
+      end
+    end
+  end
+end

data/lib/nagios-herald/helpers.rb

@@ -0,0 +1,5 @@
+require 'nagios-herald/helpers/ganglia_graph'
+require 'nagios-herald/helpers/graphite_graph'
+require 'nagios-herald/helpers/url_image'
+require 'nagios-herald/helpers/splunk_alert_frequency'
+require 'nagios-herald/helpers/logstash_query'

data/lib/nagios-herald/logging.rb

@@ -0,0 +1,48 @@
+# looks like this code came from
+# http://stackoverflow.com/questions/917566/ruby-share-logger-instance-among-module-classes
+
+require "logger"
+
+module NagiosHerald
+  module Logging
+
+    def logger
+    @logger ||= Logging.logger_for(self.class.name)
+    end
+
+    # Use a hash class-ivar to cache a unique Logger per class:
+    # "ivar" = fancy term for "instance variable"
+    # do we want this? or a global logger?
+    @loggers = {}
+
+    extend self
+
+    # effectively sets the progname
+    def logger_for(classname)
+      @loggers[classname] ||= configure_logger_for(classname)
+    end
+
+    # instantiates a Logger instance
+    # default to STDOUT
+    def configure_logger_for(classname)
+      logfile = Config.config['logfile'] ? Config.config['logfile'] : STDOUT
+      if logfile.eql?("STDOUT")
+        logfile = STDOUT
+      end
+      begin
+        logger = Logger.new(logfile)
+      rescue Exception => e
+        puts "Failed to open #{logfile} for writing: #{e.message}"
+        puts "Defaulting to STDOUT"
+        logger = Logger.new(STDOUT)
+      end
+      logger.datetime_format = "%Y-%m-%d %H:%M:%S"
+      logger.progname = "#{File.basename $0} (#{classname})"
+      logger.formatter = proc { |severity, datetime, progname, msg|
+        "[#{datetime}] #{severity} -- #{progname}: #{msg}\n"
+      }
+      logger
+    end
+
+  end
+end

data/lib/nagios-herald/message_loader.rb

@@ -0,0 +1,40 @@
+# Load all Message classes similar to how messageLoader does its thang.
+module NagiosHerald
+  class MessageLoader
+  include NagiosHerald::Logging
+
+    attr_accessor :message_path
+
+    def initialize
+      # TODO: add support for @options.message_path
+      @message_path = File.expand_path("messages", File.dirname(__FILE__))
+    end
+
+    # Public: Enumerate the available message class files.
+    #
+    # Returns an array of the class files' absolute paths
+    def enum_message_class_files(message_path)
+      message_class_files = Dir.glob(File.expand_path("*.rb", message_path))
+    end
+
+    # Public: Return an array of class files' paths.
+    def message_class_files
+      @message_class_files ||= enum_message_class_files(@message_path)
+    end
+
+    # Public: Load the messages into the namespace.
+    # A message can then easily be instantiated later.
+    def load_messages
+      if message_class_files.empty?
+        logger.fatal "#{$0}: No messages were found in '#{@message_path}'"
+        exit 1
+      else
+        message_class_files.each do |message_class_file|
+          Kernel.load message_class_file
+        end
+      end
+    end
+
+  end
+end
+

data/lib/nagios-herald/messages/base.rb

@@ -0,0 +1,56 @@
+require 'nagios-herald/message_loader'
+require 'nagios-herald/logging'
+require 'nagios-herald/util'
+
+# Message objects know best about how to generate and send messages
+# The Base class defines @content and @recipients variables as all messages probably
+# have some notion of these constructs.
+module NagiosHerald
+  class Message
+    include NagiosHerald::Logging
+    include NagiosHerald::Util
+
+    attr_accessor :content
+    attr_accessor :recipients
+
+    def initialize(recipients, options)
+      @no_send     = options[:no_send]
+      # TODO: instead of passing this in via the subclass, let's set it via message.recipients
+      @recipients = recipients
+    end
+
+    # Public: Defines what is required to send a message.
+    # The message type knows best how this is done. Override the #send method
+    # in your message subclass.
+    def send
+      raise Exception, "#{self.to_s}: You must override #send"
+    end
+
+    def self.message_types
+      @@message_types ||= {}
+    end
+
+    # Public: When subclassed message types are instantiated, add them to the @@message_types hash.
+    # The key is the downcased and snake_cased name of the class file (i.e. email);
+    # the value is the actual class (i.e. Email) so that we can easily
+    # instantiate message types when we know the message type name.
+    # Learned this pattern thanks to the folks at Chef and @jonlives.
+    # See https://github.com/opscode/chef/blob/11-stable/lib/chef/knife.rb#L79#L83
+    #
+    # Returns the message_types hash.
+    def self.inherited(subclass)
+      subclass_base_name = subclass.name.split('::').last
+      if subclass_base_name == subclass_base_name.upcase
+        # we've got an all upper case class name (probably an acronym like IRC); just downcase the whole thing
+        subclass_base_name.downcase!
+        message_types[subclass_base_name] = subclass
+      else
+        subclass_base_name.gsub!(/[A-Z]/) { |s| "_" + s } # replace uppercase with underscore and lowercase
+        subclass_base_name.downcase!
+        subclass_base_name.sub!(/^_/, "")   # strip the leading underscore
+        message_types[subclass_base_name] = subclass
+      end
+    end
+
+  end
+end

data/lib/nagios-herald/messages/email.rb

@@ -0,0 +1,150 @@
+require 'nagios-herald/messages/base'
+require 'mail'
+
+module NagiosHerald
+  class Message
+    class Email < Message
+
+      attr_accessor :attachments
+      attr_accessor :html
+      attr_accessor :subject
+      attr_accessor :text
+
+      # Public: Initializes a new Message::Email object.
+      #
+      # recipients - A list of recipients for this message.
+      # options - The options hash from Executor.
+      # FIXME: Is that ^^ necessary now with Config.config available?
+      # 
+      # Returns a new Message::Email object.
+      def initialize(recipients, options = {})
+        @replyto     = options[:replyto]
+        @subject     = ""
+        @text        = ""
+        @html        = ""
+        @attachments = []
+        super(recipients, options)
+      end
+
+      # this is a list of Mail::Part
+      # => #<Mail::Part:19564000, Multipart: false, Headers: <Content-Type: ; filename="Rakefile">, <Content-Transfer-Encoding: binary>, <Content-Disposition: attachment; filename="Rakefile">, <Content-ID: <530e1814464a9_3305aaef88979a2@blahblahbl.blah.blah.blah.mail>>>
+      # Public: Updates HTML content so that attachments are referenced via the 'cid:' URI scheme and neatly embedded in the body
+      # of a(n email) message.
+      #
+      # attachments - The list of attachments defined in the formatter content hash.
+      #
+      # Returns the updated HTML content.
+      def inline_body_with_attachments(attachments)
+        inline_html = @html
+        attachments.each do |attachment|
+          if (inline_html =~ /#{attachment.filename}/)
+            inline_html = inline_html.sub(attachment.filename, "cid:#{attachment.cid}")
+          end
+        end
+        inline_html
+      end
+
+      # Public: Generates the text portion of the content hash.
+      #
+      # Returns the full text portion of the content hash.
+      def curate_text
+        @text += self.content[:text][:host_info] unless self.content[:text][:host_info].empty?
+        @text += self.content[:text][:state_info] unless self.content[:text][:state_info].empty?
+        @text += self.content[:text][:additional_info] unless self.content[:text][:additional_info].empty?
+        @text += self.content[:text][:action_url] unless self.content[:text][:action_url].empty?
+        @text += self.content[:text][:notes] unless self.content[:text][:notes].empty?
+        @text += self.content[:text][:additional_details] unless self.content[:text][:additional_details].empty?
+        @text += self.content[:text][:recipients_email_link] unless self.content[:text][:recipients_email_link].empty?
+        @text += self.content[:text][:notification_info] unless self.content[:text][:notification_info].empty?
+        @text += self.content[:text][:alert_ack_url] unless self.content[:text][:alert_ack_url].empty?
+        # Hack to ensure we get ack info, if it's populated.
+        @text += self.content[:text][:ack_info] if self.content[:text][:ack_info] and !self.content[:text][:ack_info].empty?
+      end
+
+      # Public: Generates the HTML portion of the content hash.
+      #
+      # Returns the full HTML portion of the content hash.
+      def curate_html
+        @html += self.content[:html][:host_info] unless self.content[:html][:host_info].empty?
+        @html += self.content[:html][:state_info] unless self.content[:html][:state_info].empty?
+        @html += self.content[:html][:additional_info] unless self.content[:html][:additional_info].empty?
+        @html += self.content[:html][:action_url] unless self.content[:html][:action_url].empty?
+        @html += self.content[:html][:notes] unless self.content[:html][:notes].empty?
+        @html += self.content[:html][:additional_details] unless self.content[:html][:additional_details].empty?
+        @html += self.content[:html][:recipients_email_link] unless self.content[:html][:recipients_email_link].empty?
+        @html += self.content[:html][:notification_info] unless self.content[:html][:notification_info].empty?
+        @html += self.content[:html][:alert_ack_url] unless self.content[:html][:alert_ack_url].empty?
+        # Hack to ensure we get ack info, if it's populated.
+        @html += self.content[:html][:ack_info] if self.content[:html][:ack_info] and !self.content[:html][:ack_info].empty?
+      end
+
+      # Public: Prints the subject, text and HTML content to the terminal.
+      # Useful for debugging.
+      #
+      # Returns nothing.
+      def print
+        puts "------------------"
+        puts "Subject : #{@subject}"
+        puts "------------------"
+        puts @text if !@text.empty?
+        puts @html if !@html.empty?
+      end
+
+      # Public: Builds the email message.
+      #
+      # Returns the mail object.
+      def build_message
+        curate_text
+        curate_html
+        if @no_send
+          self.print
+          return
+        end
+
+        @subject = self.content[:subject]
+        mail = Mail.new({
+          :from  => @replyto,
+          :to    => @recipients,
+          :subject => @subject,
+          :content_type => 'multipart/alternative'
+        })
+
+        text_content = @text
+        text_part = Mail::Part.new do
+          body text_content
+        end
+
+        mail.add_part(text_part)
+
+        html_part = Mail::Part.new do
+          content_type 'multipart/related;'
+        end
+
+        # Load the attachments
+        @attachments = self.content[:attachments]
+        @attachments.each do |attachment|
+          html_part.attachments[attachment] = File.read(attachment)
+        end
+
+        if @html != ""
+          # Inline the attachment if need be
+          inline_html = inline_body_with_attachments(html_part.attachments)
+          html_content_part = Mail::Part.new do
+            content_type 'text/html; charset=UTF-8'
+            body     inline_html
+          end
+          html_part.add_part(html_content_part)
+        end
+
+        mail.add_part(html_part)
+        mail
+      end
+
+      def send
+        mail = self.build_message
+        mail.deliver!
+      end
+
+    end
+  end
+end