flapjack 0.5.5 → 0.6.23

data/lib/flapjack/notification/common.rb

@@ -0,0 +1,23 @@
+#!/usr/bin/env ruby
+
+require 'flapjack/pikelet'
+
+module Flapjack
+  module Notification
+
+    module Common
+
+      include Flapjack::Pikelet
+
+      # TODO to make this testable, work out a supported way to make the passed redis connection
+      # use the test db
+      def perform(notification)
+        bootstrap
+        @logger.debug "Woo, got a notification to send out: #{notification.inspect}"
+        dispatch(notification, :logger => @logger, :redis => ::Redis.new)
+      end
+
+    end
+  end
+end
+

data/lib/flapjack/notification/email.rb

@@ -0,0 +1,107 @@
+#!/usr/bin/env ruby
+
+require 'mail'
+require 'erb'
+require 'haml'
+require 'socket'
+
+require 'flapjack/data/entity_check'
+require 'flapjack/notification/common'
+
+module Flapjack
+  module Notification
+
+    class Email
+      extend Flapjack::Notification::Common
+
+      def self.dispatch(notification, opts = {})
+        notification_type  = notification['notification_type']
+        contact_first_name = notification['contact_first_name']
+        contact_last_name  = notification['contact_last_name']
+        state              = notification['state']
+        summary            = notification['summary']
+        time               = notification['time']
+        entity, check      = notification['event_id'].split(':')
+
+        entity_check = Flapjack::Data::EntityCheck.for_event_id(notification['event_id'],
+          :redis => opts[:redis])
+
+        headline_map = {'problem'         => 'Problem: ',
+                        'recovery'        => 'Recovery: ',
+                        'acknowledgement' => 'Acknowledgement: ',
+                        'unknown'         => ''
+                       }
+
+        headline = headline_map[notification_type] || ''
+
+        subject = "#{headline}'#{check}' on #{entity}"
+        subject += " is #{state.upcase}" unless notification_type == 'acknowledgement'
+
+        notification['subject'] = subject
+        opts[:logger].debug "Flapjack::Notification::Email#dispatch is calling Flapjack::Notification::Mailer.sender, notification_id: #{notification['id']}"
+        sender_opts = {:logger => opts[:logger],
+                       :in_scheduled_maintenance   => entity_check.in_scheduled_maintenance?,
+                       :in_unscheduled_maintenance => entity_check.in_unscheduled_maintenance?
+                      }
+
+        mail = prepare_email(notification, sender_opts)
+        mail.deliver!
+      end
+
+    private
+
+      def self.prepare_email(notification, opts)
+
+        logger = opts[:logger]
+
+        # not useing socket and gethostname as that doesn't give you a fqdn.
+        # see the facter issue: https://projects.puppetlabs.com/issues/3898
+        fqdn       = `/bin/hostname -f`.chomp
+        m_from     = "flapjack@#{fqdn}"
+        logger.debug("flapjack_mailer: set from to #{m_from}")
+        m_reply_to = m_from
+
+        m_to       = notification['address']
+        m_subject  = notification['subject']
+
+        logger.debug("Flapjack::Notification::Mailer #{notification['id']} to: #{m_to} subject: #{m_subject}")
+
+        @notification_type  = notification['notification_type']
+        @contact_first_name = notification['contact_first_name']
+        @contact_last_name  = notification['contact_last_name']
+        @state              = notification['state']
+        @summary            = notification['summary']
+        @time               = notification['time']
+        @entity, @check     = notification['event_id'].split(':')
+        @in_unscheduled_maintenance = opts[:in_unscheduled_maintenance]
+        @in_scheduled_maintenance   = opts[:in_scheduled_maintenance]
+
+        text_template = ERB.new(File.read(File.dirname(__FILE__) +
+          '/email/alert.text.erb'))
+
+        haml_engine = Haml::Engine.new(File.read(File.dirname(__FILE__) +
+          '/email/alert.html.haml'))
+
+        mail_scope = self
+
+        mail = Mail.new do
+          from     m_from
+          to       m_to
+          subject  m_subject
+          reply_to m_reply_to
+
+          text_part do
+            body text_template.result(binding)
+          end
+
+          html_part do
+            content_type 'text/html; charset=UTF-8'
+            body haml_engine.render(mail_scope)
+          end
+        end
+      end
+
+    end
+  end
+end
+

data/lib/flapjack/notification/email/alert.html.haml

@@ -0,0 +1,48 @@
+%style{:type => "text/css", :media => "screen"}
+  :plain
+    #container {
+      text-transform: uppercase;
+    }
+    table {
+      border-collapse: collapse;
+    }
+    table, th, td {
+      border: 1px solid #666;
+      padding: 4px;
+    }
+
+%p= "Hi #{@contact_first_name},"
+
+%p Monitoring has detected the following:
+
+%table
+  %tbody
+    %tr
+      %td
+        %strong Entity
+      %td= @entity
+    %tr
+      %td
+        %strong Check
+      %td= @check
+    %tr
+      %td
+        %strong State
+      %td= @state.upcase
+    %tr
+      %td
+        %strong Summary
+      %td= @summary
+    %tr
+      %td
+        %strong Time
+      %td= Time.at(@time.to_i).to_s
+    %tr
+      %td
+        %strong Flapjack
+      %td
+        %a(href="http://127.0.0.1:9292/check?entity=#{@entity}&check=#{@check}") http://127.0.0.1:9292/check?entity=#{@entity}&check=#{@check}
+
+%p Cheers,
+%p Flapjack
+

data/lib/flapjack/notification/email/alert.text.erb

@@ -0,0 +1,14 @@
+Hi <%= @contact_first_name %>,
+
+Monitoring has detected the following:
+
+Entity: <%= @entity %>
+Check: <%= @check %>
+State: <%= @state %>
+Summary: <%= @summary %>
+Time: <%= Time.at(@time.to_i).to_s %>
+Flapjack: http://127.0.0.1:9292/check?entity=<%= @entity %>&check=<%= @check %>
+
+Cheers,
+Flapjack
+

data/lib/flapjack/notification/sms.rb

@@ -0,0 +1,42 @@
+#!/usr/bin/env ruby
+
+require 'flapjack/notification/sms/messagenet'
+
+module Flapjack
+  module Notification
+
+    class Sms
+      extend Flapjack::Notification::Common
+
+      def self.dispatch(notification, opts = {})
+        notification_type  = notification['notification_type']
+        contact_first_name = notification['contact_first_name']
+        contact_last_name  = notification['contact_last_name']
+        state              = notification['state']
+        summary            = notification['summary']
+        time               = notification['time']
+        entity, check      = notification['event_id'].split(':')
+
+        headline_map = {'problem'         => 'PROBLEM: ',
+                        'recovery'        => 'RECOVERY: ',
+                        'acknowledgement' => 'ACK: ',
+                        'unknown'         => '',
+                        ''                => '',
+                       }
+
+        headline = headline_map[notification_type] || ''
+
+        message = "#{headline}'#{check}' on #{entity}"
+        message += " is #{state.upcase}" unless notification_type == 'acknowledgement'
+        message += " at #{Time.at(time).strftime('%-d %b %H:%M')}, #{summary}"
+
+        notification['message'] = message
+        Flapjack::Notification::Sms::Messagenet.sender(notification,
+          :logger => opts[:logger],
+          :config => Flapjack::Notification::Sms.class_variable_get('@@config'))
+      end
+
+    end
+  end
+end
+

data/lib/flapjack/notification/sms/messagenet.rb

@@ -0,0 +1,49 @@
+#!/usr/bin/env ruby
+
+require 'net/http'
+require 'uri'
+
+module Flapjack
+  module Notification
+    class Sms
+      class Messagenet
+
+        def self.sender(notification, options = {})
+          logger = options[:logger]
+          config = options[:config]
+
+          unless config && (username = config["username"])
+            raise RuntimeError.new('sms_messagenet: username is missing')
+          end
+          unless config && (password = config["password"])
+            raise RuntimeError.new('sms_messagenet: password is missing')
+          end
+
+          raise RuntimeError.new('address is missing') unless address         = notification['address']
+          raise RuntimeError.new('message is missing') unless message         = notification['message']
+          raise RuntimeError.new('id is missing')      unless notification_id = notification['id']
+
+          params = { 'Username'     => username,
+                     'Pwd'          => password,
+                     'PhoneNumber'  => address,
+                     'PhoneMessage' => message }
+
+          uri       = URI('https://www.messagenet.com.au/dotnet/Lodge.asmx/LodgeSMSMessage')
+          uri.query = URI.encode_www_form(params)
+          logger.debug("request_uri: #{uri.request_uri.inspect}")
+
+          Net::HTTP.start(uri.host, uri.port, :use_ssl => uri.scheme == 'https') do |http|
+            request = Net::HTTP::Get.new uri.request_uri
+            response = http.request request
+            http_success = ( response.is_a?(Net::HTTPSuccess) == true )
+            logger.debug("Flapjack::Notification::SMSMessagenet: response from server: #{response.body}")
+            raise RuntimeError.new "Failed to send SMS via messagenet, http response is a #{response.class}, notification_id: #{notification_id}" unless http_success
+          end
+
+        end
+      end
+    end
+  end
+end
+
+

data/lib/flapjack/notifier_engine.rb

@@ -4,13 +4,13 @@ require 'ostruct'
 
 module Flapjack
   class NotifierEngine
-   
+
     attr_reader :log, :notifiers
-  
+
     def initialize(opts={})
       @log = opts[:log]
       raise "you have to specify a logger" unless @log
-      
+
       @notifiers = []
       if opts[:notifiers]
         opts[:notifiers].each do |n|
@@ -21,7 +21,7 @@ module Flapjack
         @log.warning("There are no notifiers! flapjack-notifier won't be useful.")
       end
     end
-   
+
     def notify!(options={})
       result     = options[:result]
       event      = options[:event]

data/lib/flapjack/notifiers/mailer/mailer.rb

@@ -1,6 +1,5 @@
 #!/usr/bin/env ruby
 
-require 'rubygems'
 require 'net/smtp'
 require 'tmail'
 
@@ -15,7 +14,7 @@ module Flapjack
         @log = opts[:log]
         @from_address = opts[:from_address]
         @website_uri = opts[:website_uri]
-        
+
         raise ArgumentError, "from address must be provided" unless @from_address
       end
 
@@ -30,15 +29,15 @@ module Flapjack
         mail.subject = "Check: #{opts[:result].check_id}, Status: #{opts[:result].status}"
         mail.body = <<-DESC
           Check #{opts[:result].check_id} returned the status "#{opts[:result].status}".
-          
-          Here was the output: 
+
+          Here was the output:
             #{opts[:result].output}
-    
+
           You can respond to this issue at:
             #{@website_uri}/issue/#{opts[:result].check_id}
         DESC
 
-        begin 
+        begin
           Net::SMTP.start('localhost') do |smtp|
             return smtp.sendmail(mail.to_s, mail.from, mail.to)
           end
@@ -46,7 +45,7 @@ module Flapjack
           @log.error("Couldn't establish connection to mail server!")
         end
       end
-    
+
     end
   end
 end

data/lib/flapjack/notifiers/xmpp/xmpp.rb

@@ -5,41 +5,41 @@ require 'xmpp4r'
 module Flapjack
   module Notifiers
     class Xmpp
-    
+
       def initialize(opts={})
-    
+
         @jid = opts[:jid]
         @password = opts[:password]
         @log = opts[:logger]
-        unless @jid && @password 
+        unless @jid && @password
           raise ArgumentError, "You have to provide a username and password"
         end
 
-        begin 
+        begin
           @xmpp = Jabber::Client.new(@jid)
           @xmpp.connect
           @xmpp.auth(@password)
         rescue SocketError => e
           @log.error("XMPP: #{e.message}")
         end
-    
+
       end
-    
+
       def notify(opts={})
-    
-        raise ArgumentError, "a recipient was not specified" unless opts[:who] 
+
+        raise ArgumentError, "a recipient was not specified" unless opts[:who]
         raise ArgumentError, "a result was not specified" unless opts[:result]
-    
+
         text = <<-DESC
           Check #{opts[:result].check_id} returned the status "#{opts[:result].status}".
             http://localhost:4000/checks/#{opts[:result].check_id}
         DESC
-   
+
         message = Jabber::Message.new(opts[:who].jid, text)
         @xmpp.send(message)
-    
+
       end
-    
+
     end
   end
 end

data/lib/flapjack/pagerduty.rb

@@ -0,0 +1,230 @@
+#!/usr/bin/env ruby
+
+require 'eventmachine'
+# the redis/synchrony gems need to be required in this particular order, see
+# the redis-rb README for details
+require 'hiredis'
+require 'em-synchrony'
+require 'em-synchrony/em-http'
+require 'redis/connection/synchrony'
+require 'redis'
+
+require 'yajl/json_gem'
+
+require 'flapjack/data/entity_check'
+require 'flapjack/pikelet'
+
+module Flapjack
+
+  class Pagerduty
+
+    include Flapjack::Pikelet
+
+    def setup
+      @redis = build_redis_connection_pool
+      logger.debug("New Pagerduty pikelet with the following options: #{@config.inspect}")
+
+      @pagerduty_events_api_url = 'https://events.pagerduty.com/generic/2010-04-15/create_event.json'
+      @pagerduty_acks_started = nil
+      @sem_pagerduty_acks_running = 'sem_pagerduty_acks_running'
+    end
+
+    def send_pagerduty_event(event)
+      options  = { :body => Yajl::Encoder.encode(event) }
+      http = EM::HttpRequest.new(@pagerduty_events_api_url).post(options)
+      response = Yajl::Parser.parse(http.response)
+      status   = http.response_header.status
+      logger.debug "send_pagerduty_event got a return code of #{status.to_s} - #{response.inspect}"
+      [status, response]
+    end
+
+    def test_pagerduty_connection
+      noop = { "service_key"  => "11111111111111111111111111111111",
+               "incident_key" => "Flapjack is running a NOOP",
+               "event_type"   => "nop",
+               "description"  => "I love APIs with noops." }
+      code, results = send_pagerduty_event(noop)
+      return true if code == 200 && results['status'] =~ /success/i
+      logger.error "Error: test_pagerduty_connection: API returned #{code.to_s} #{results.inspect}"
+      false
+    end
+
+    # this should be moved to a checks data model perhaps
+    def unacknowledged_failing_checks
+      failing_checks = @redis_timer.zrange('failed_checks', '0', '-1')
+      unless failing_checks.is_a?(Array)
+        @logger.error("redis.zrange returned something other than an array! Here it is: " + failing_checks.inspect)
+      end
+      ufc = failing_checks.reject {|check|
+        @redis_timer.exists(check + ':unscheduled_maintenance')
+      }
+      @logger.debug "found unacknowledged failing checks as follows: " + ufc.join(', ')
+      ufc
+    end
+
+    def pagerduty_acknowledged?(opts)
+      subdomain   = opts['subdomain']
+      username    = opts['username']
+      password    = opts['password']
+      check       = opts['check']
+
+      url = 'https://' + subdomain + '.pagerduty.com/api/v1/incidents'
+      query = { 'fields'       => 'incident_number,status,last_status_change_by',
+                'since'        => (Time.new.utc - (60*60*24*7)).iso8601,
+                'until'        => (Time.new.utc + (60*60*24)).iso8601,
+                'incident_key' => check,
+                'status'       => 'acknowledged' }
+
+      options = { :head  => { 'authorization' => [username, password] },
+                  :query => query }
+
+      http = EM::HttpRequest.new(url).get(options)
+      # DEBUG flapjack-pagerduty: pagerduty_acknowledged?: decoded response as:
+      # {"incidents"=>[{"incident_number"=>40, "status"=>"acknowledged",
+      # "last_status_change_by"=>{"id"=>"PO1NWPS", "name"=>"Jesse Reynolds",
+      # "email"=>"jesse@bulletproof.net",
+      # "html_url"=>"http://bltprf.pagerduty.com/users/PO1NWPS"}}], "limit"=>100, "offset"=>0,
+      # "total"=>1}
+      begin
+        response = Yajl::Parser.parse(http.response)
+      rescue Yajl::ParseError
+        @logger.error("failed to parse json from a post to #{url} ... response headers and body follows...")
+        @logger.error(http.response_header.inspect)
+        @logger.error(http.response)
+        return nil, nil
+      end
+      status   = http.response_header.status
+
+      @logger.debug("pagerduty_acknowledged?: decoded response as: #{response.inspect}")
+      if response.nil?
+        @logger.error('no valid response received from pagerduty!')
+        return nil, nil
+      end
+
+      if response['incidents'].nil?
+        @logger.error('no incidents found in response')
+        return nil, nil
+      end
+
+      if response['incidents'].length > 0
+        pg_acknowledged_by = response['incidents'].first['last_status_change_by']
+        return true, :pg_acknowledged_by => pg_acknowledged_by
+      else
+        return false, nil
+      end
+    end
+
+    def catch_pagerduty_acks
+
+      # ensure we're the only instance of the pagerduty acknowledgement check running (with a naive
+      # timeout of five minutes to guard against stale locks caused by crashing code) either in this
+      # process or in other processes
+      if (@pagerduty_acks_started and @pagerduty_acks_started > (Time.now.to_i - 300)) or
+          @redis_timer.get(@sem_pagerduty_acks_running) == 'true'
+        logger.debug("skipping looking for acks in pagerduty as this is already happening")
+        return
+      end
+
+      @pagerduty_acks_started = Time.now.to_i
+      @redis_timer.set(@sem_pagerduty_acks_running, 'true')
+      @redis_timer.expire(@sem_pagerduty_acks_running, 300)
+
+      logger.debug("looking for acks in pagerduty for unack'd problems")
+
+      # ok lets do it
+      unacknowledged_failing_checks.each {|check|
+        entity_check = Flapjack::Data::EntityCheck.for_event_id(check, { :redis => @redis_timer, :logger => @logger } )
+        pagerduty_credentials = entity_check.pagerduty_credentials( { :redis => @redis_timer, :logger => @logger } )
+
+        if pagerduty_credentials.length == 0
+          @logger.debug("Found no pagerduty creditials for #{entity_check.entity_name}:#{entity_check.check}, moving on")
+          next
+        end
+
+        # FIXME: try each set of credentials until one works (may have stale contacts turning up)
+        options = pagerduty_credentials.first.merge('check' => check)
+
+        pagerduty_acknowledged, result_hash = pagerduty_acknowledged?(options)
+        if pagerduty_acknowledged
+          pg_acknowledged_by = result_hash[:pg_acknowledged_by] unless result_hash.nil?
+          @logger.debug "#{check} is acknowledged in pagerduty, creating flapjack acknowledgement ... "
+          who_text = ""
+          if !pg_acknowledged_by.nil? && !pg_acknowledged_by['name'].nil?
+            who_text = " by #{pg_acknowledged_by['name']}"
+          end
+          entity_check.create_acknowledgement('summary' => "Acknowledged on PagerDuty" + who_text)
+        else
+          @logger.debug "#{check} is not acknowledged in pagerduty, moving on"
+        end
+      }
+      @redis_timer.del(@sem_pagerduty_acks_running)
+      @pagerduty_acks_started = nil
+    end
+
+    def add_shutdown_event(opts = {})
+      return unless redis = opts[:redis]
+      redis.rpush(@config['queue'], JSON.generate('notification_type' => 'shutdown'))
+    end
+
+    def main
+      setup
+
+      logger.debug("pagerduty gateway - commencing main method")
+      raise "Can't connect to the pagerduty API" unless test_pagerduty_connection
+
+      # TODO: only clear this if there isn't another pagerduty gateway instance running
+      # or better, include on instance ID in the semaphore key name
+      @redis.del(@sem_pagerduty_acks_running)
+
+      acknowledgement_timer = EM::Synchrony.add_periodic_timer(10) do
+        @redis_timer ||= build_redis_connection_pool
+        catch_pagerduty_acks
+      end
+
+      queue = @config['queue']
+      events = {}
+
+      until should_quit?
+        logger.debug("pagerduty gateway is going into blpop mode on #{queue}")
+        events[queue] = @redis.blpop(queue)
+        event         = Yajl::Parser.parse(events[queue][1])
+        type          = event['notification_type']
+        logger.debug("pagerduty notification event popped off the queue: " + event.inspect)
+        if 'shutdown'.eql?(type)
+          # do anything in particular?
+        else
+          event_id      = event['event_id']
+          entity, check = event_id.split(':')
+          state         = event['state']
+          summary       = event['summary']
+          address       = event['address']
+
+          case type.downcase
+          when 'acknowledgement'
+            maint_str      = "has been acknowledged"
+            pagerduty_type = 'acknowledge'
+          when 'problem'
+            maint_str      = "is #{state.upcase}"
+            pagerduty_type = "trigger"
+          when 'recovery'
+            maint_str      = "is #{state.upcase}"
+            pagerduty_type = "resolve"
+          end
+
+          message = "#{type.upcase} - \"#{check}\" on #{entity} #{maint_str} - #{summary}"
+
+          pagerduty_event = { :service_key  => address,
+                              :incident_key => event_id,
+                              :event_type   => pagerduty_type,
+                              :description  => message }
+
+          send_pagerduty_event(pagerduty_event)
+        end
+      end
+
+      acknowledgement_timer.cancel
+    end
+
+  end
+end
+