flapjack 0.7.28 → 0.7.29

data/lib/flapjack/gateways/email/alert.text.erb

@@ -1,25 +1,25 @@
-Hi <%= @contact_first_name %>
+Hi <%= @alert.contact_first_name %>
 
 Monitoring has detected the following:
 
-Entity:           <%= @entity_name %>
-Check:            <%= @check %>
-State:            <%= ['ok'].include?(@state) ? @state.upcase : @state.titleize %>
-Summary:          <%= @summary %>
-<% if @details -%>
-Details:          <%= @details %>
+Entity:           <%= @alert.entity %>
+Check:            <%= @alert.check %>
+State:            <%= @alert.state_title_case %>
+Summary:          <%= @alert.summary %>
+<% if @alert.details -%>
+Details:          <%= @alert.details %>
 <% end -%>
-<% if @time -%>
-Time:             <%= Time.at(@time.to_i).to_s %>
+<% if @alert.time -%>
+Time:             <%= Time.at(@alert.time.to_i).to_s %>
 <% end -%>
-<% if @duration && @duration > 40 -%>
-Duration:         <%= ChronicDuration.output(@duration) %>
+<% if @alert.state_duration && @alert.state_duration > 40 -%>
+Duration:         <%= ChronicDuration.output(@alert.state_duration) %>
 <% end -%>
-<% if @last_state -%>
-Previous State:   <%= ['ok'].include?(@last_state) ? @last_state.upcase : @last_state.titleize %>
+<% if @alert.last_state -%>
+Previous State:   <%= @alert.last_state_title_case %>
 <% end -%>
-<% if @last_summary -%>
-Previous Summary: <%= @last_summary %>
+<% if @alert.last_summary -%>
+Previous Summary: <%= @alert.last_summary %>
 <% end -%>
 
 Cheers,

data/lib/flapjack/gateways/email/alert_subject.text.erb

@@ -1,14 +1,4 @@
-<% case @notification_type -%>
-<% when "problem" -%>
-<%=  "Problem: " -%>
-<% when "recovery" -%>
-<%=  "Recovery: " -%>
-<% when "acknowledgement" -%>
-<%=  "Acknowledgement: " -%>
-<% when "test" -%>
-<%=  "Test notification: " -%>
-<% end -%>
-'<%= @check %>' on <%= @entity_name -%>
-<% if ! ['acknowledgement', 'test'].include?(@notification_type) -%>
- is <%= ['ok'].include?(@state) ? @state.upcase : @state.titleize -%>
+<%= @alert.type_sentence_case %>: '<%= @alert.check %>' on <%= @alert.entity -%>
+<% unless ['acknowledgement', 'test'].include?(@alert.notification_type) -%>
+ is <%= @alert.state_title_case -%>
 <% end -%>

data/lib/flapjack/gateways/email/rollup.html.erb

@@ -11,9 +11,9 @@
   }
 </style>
 
-<p>Hi <%= @contact_first_name %></p>
+<p>Hi <%= @alert.contact_first_name %></p>
 
-<p>You have <%= @rollup_alerts.length %> alerting check<%= @rollup_alerts.length == 1 ? '' : 's' %> as follows:</p>
+<p>You have <%= @alert.rollup_alerts.length %> alerting check<%= @alert.rollup_alerts.length == 1 ? '' : 's' %> as follows:</p>
 
 <table>
   <tbody>
@@ -23,7 +23,7 @@
       <th>State</th>
       <th>Duration</th>
     </tr>
-<%  @rollup_alerts.sort_by {|entity_check, details| details['duration'] }.each do |rollup_alert| -%>
+<%  @alert.rollup_alerts.sort_by {|entity_check, details| details['duration'] }.each do |rollup_alert| -%>
 <%    r_entity, r_check = rollup_alert[0].split(':', 2) -%>
 <%    state    = rollup_alert[1]['state'] -%>
 <%    duration = ChronicDuration.output(rollup_alert[1]['duration']) -%>
@@ -37,10 +37,10 @@
   </tbody>
 </table>
 
-<% if @rollup.downcase == 'recovery' %>
-  <p>As your email summary threshold is <%= @rollup_threshold %>, we're taking your email alerts out of summary mode now. You'll now be emailed individually for each alerting check.</p>
+<% if @alert.rollup == 'recovery' %>
+  <p>As your email summary threshold is <%= @alert.rollup_threshold %>, we're taking your email alerts out of summary mode now. You'll now be emailed individually for each alerting check.</p>
 <% else %>
-  <p>Your email alerts are being summarised as your email summary threshold is set to <%= @rollup_threshold %>. You'll receive summary emails like this one until your number of alerting checks falls below <%= @rollup_threshold %>.</p>
+  <p>Your email alerts are being summarised as your email summary threshold is set to <%= @alert.rollup_threshold %>. You'll receive summary emails like this one until your number of alerting checks falls below <%= @alert.rollup_threshold %>.</p>
 <% end %>
 
 <p>Cheers,<br/>

data/lib/flapjack/gateways/email/rollup.text.erb

@@ -1,18 +1,18 @@
-Hi <%= @contact_first_name %>
+Hi <%= @alert.contact_first_name %>
 
-You have <%= @rollup_alerts.length %> alerting check<%= @rollup_alerts.length == 1 ? '' : 's' %> as follows:
+You have <%= @alert.rollup_alerts.length %> alerting check<%= @alert.rollup_alerts.length == 1 ? '' : 's' %> as follows:
 
-<% @rollup_alerts.sort_by {|entity_check, details| details['duration'] }.each do |rollup_alert| -%>
+<% @alert.rollup_alerts.sort_by {|entity_check, details| details['duration'] }.each do |rollup_alert| -%>
 <%   r_entity, r_check = rollup_alert[0].split(':', 2) -%>
 <%   state    = rollup_alert[1]['state'] -%>
-<%   duration = ChronicDuration.output(rollup_alert[1]['duration']) -%>
+<%   duration = ChronicDuration.output(rollup_alert[1]['duration'] ) -%>
   * <%= r_check %> on <%= r_entity %> is <%= ['ok'].include?(state) ? state.upcase : state.titleize %> (<%= duration %>)
 <% end -%>
 
-<% if @rollup.downcase == 'recovery' -%>
-As your email summary threshold is <%= @rollup_threshold %>, we're taking your email alerts out of summary mode now. You'll now be emailed individually for each alerting check.
+<% if @alert.rollup == 'recovery' -%>
+As your email summary threshold is <%= @alert.rollup_threshold %>, we're taking your email alerts out of summary mode now. You'll now be emailed individually for each alerting check.
 <% else -%>
-Your email alerts are being summarised as your email summary threshold is set to <%= @rollup_threshold %>. You'll receive summary emails like this one until your number of alerting checks falls below <%= @rollup_threshold %>.
+Your email alerts are being summarised as your email summary threshold is set to <%= @alert.rollup_threshold %>. You'll receive summary emails like this one until your number of alerting checks falls below <%= @alert.rollup_threshold %>.
 <% end -%>
 
 Cheers,

data/lib/flapjack/gateways/email/rollup_subject.text.erb

@@ -1,19 +1 @@
-<%
-  state_counts = @rollup_alerts.inject({}) do |memo, alert|
-    puts "alert: #{alert.inspect}"
-    memo[alert[1]['state']] = (memo[alert[1]['state']] || 0) + 1
-    memo
-  end
-  states_summary = ['critical', 'warning', 'unknown'].inject([]) do |memo, state|
-    next memo unless state_counts[state]
-    memo << "#{state.titleize}: #{state_counts[state]}"
-    memo
-  end.join(', ')
--%>
-<% case @rollup -%>
-<% when "problem" -%>
-<%=  "Problem summary: " -%>
-<% when "recovery" -%>
-<%=  "Problem summaries finishing: " -%>
-<% end -%>
-<%= states_summary -%>
+<%= @alert.type_sentence_case %>: <%= @alert.rollup_states_summary -%>

data/lib/flapjack/gateways/jabber.rb

@@ -12,6 +12,7 @@ require 'flapjack/data/entity_check'
 require 'flapjack/redis_pool'
 require 'flapjack/utility'
 require 'flapjack/version'
+require 'flapjack/data/alert'
 
 module Flapjack
 
@@ -482,14 +483,21 @@ module Flapjack
           # FIXME: should also check if presence has been established in any group chat rooms that are
           # configured before starting to process events, otherwise the first few may get lost (send
           # before joining the group chat rooms)
-          if connected?
-            @logger.debug("jabber is connected so commencing blpop on #{queue}")
-            events[queue] = @redis.blpop(queue, 0)
-            event         = Oj.load(events[queue][1])
-            type          = event['notification_type'] || 'unknown'
-            @logger.debug('jabber notification event received')
-            @logger.debug(event.inspect)
-            if 'shutdown'.eql?(type)
+          unless connected?
+            @logger.debug("not connected, sleep 1 before retry")
+            EM::Synchrony.sleep(1)
+            next
+          end
+
+          @logger.debug("jabber is connected so commencing blpop on #{queue}")
+          events[queue] = @redis.blpop(queue, 0)
+          event_json = events[queue][1]
+          begin
+            event = Oj.load(event_json)
+
+            @logger.debug('jabber notification event received: ' + event.inspect)
+
+            if 'shutdown'.eql?(event['notification_type'])
               @logger.debug("@should_quit: #{@should_quit}")
               if @should_quit
                 EventMachine::Synchrony.next_tick do
@@ -497,49 +505,51 @@ module Flapjack
                   close # Blather::Client.close
                 end
               end
-            else
-              entity, check = event['event_id'].split(':', 2)
-              state         = event['state']
-              summary       = event['summary']
-              duration      = event['duration'] ? time_period_in_words(event['duration']) : '4 hours'
-              address       = event['address']
-
-              @logger.debug("processing jabber notification address: #{address}, event: #{entity}:#{check}, state: #{state}, summary: #{summary}")
-
-              ack_str =
-                event['event_count'] &&
-                !state.eql?('ok') &&
-                !'acknowledgement'.eql?(type) &&
-                !'test'.eql?(type) ?
-                "::: #{@config['alias']}: ACKID #{event['event_count']} " : ''
-
-              type = 'unknown' unless type
-
-              maint_str = case type
-              when 'acknowledgement'
-                "has been acknowledged, unscheduled maintenance created for #{duration}"
-              when 'test'
-                ''
-              else
-                "is #{state.upcase}"
-              end
+              next
+            end
 
-              # FIXME - should probably put all the message composition stuff in one place so
-              # the logic isn't duplicated in each notification channel.
-              # TODO - templatise the messages so they can be customised without changing core code
-              headline = "test".eql?(type.downcase) ? "TEST NOTIFICATION" : type.upcase
+            alert = Flapjack::Data::Alert.new(event, :logger => @logger)
 
-              msg = "#{headline} #{ack_str}::: \"#{check}\" on #{entity} #{maint_str} ::: #{summary}"
+            @logger.debug("processing jabber notification address: #{alert.address}, entity: #{alert.entity}, " +
+                          "check: '#{alert.check}', state: #{alert.state}, summary: #{alert.summary}")
 
-              chat_type = :chat
-              chat_type = :groupchat if @config['rooms'] && @config['rooms'].include?(address)
-              EventMachine::Synchrony.next_tick do
-                say(Blather::JID.new(address), msg, chat_type)
-              end
+            @ack_str =
+              alert.event_count &&
+              !alert.state.eql?('ok') &&
+              !'acknowledgement'.eql?(alert.type) &&
+              !'test'.eql?(alert.type) ?
+              "#{@config['alias']}: ACKID #{event['event_count']}" : nil
+
+            message_type = alert.rollup ? 'rollup' : 'alert'
+
+            mydir = File.dirname(__FILE__)
+            message_template_path = mydir + "/jabber/#{message_type}.text.erb"
+            message_template = ERB.new(File.read(message_template_path), nil, '-')
+
+            @alert = alert
+            bnd    = binding
+
+            begin
+              message = message_template.result(bnd).chomp
+            rescue => e
+              @logger.error "Error while excuting the ERB for a jabber message, " +
+                "ERB being executed: #{message_template_path}"
+              raise
             end
-          else
-            @logger.debug("not connected, sleep 1 before retry")
-            EM::Synchrony.sleep(1)
+
+            chat_type = :chat
+            chat_type = :groupchat if @config['rooms'] && @config['rooms'].include?(alert.address)
+            EventMachine::Synchrony.next_tick do
+              say(Blather::JID.new(alert.address), message, chat_type)
+              alert.record_send_success!
+            end
+
+          rescue => e
+            # TODO: have non-fatal errors generate messages (eg via flapjack events or straight to
+            # rollbar or similar)
+            @logger.error "Error generating or dispatching jabber message: #{e.class}: #{e.message}\n" +
+              e.backtrace.join("\n")
+            @logger.debug "Message that could not be processed: \n" + event_json
           end
         end
 

data/lib/flapjack/gateways/jabber/alert.text.erb

@@ -0,0 +1,12 @@
+<%= @alert.type_sentence_case -%>
+: <% if @ack_str %><%= @ack_str %>, <% end -%>
+"<%= @alert.check %>" on <%= @alert.entity -%>
+<% unless ['acknowledgement', 'test'].include?(@alert.type) -%>
+ is <%= @alert.state_title_case -%>
+<% end -%>
+<% if ['acknowledgement'].include?(@alert.type) -%>
+ has been acknowledged, unscheduled maintenance created for <%= time_period_in_words(@alert.acknowledgement_duration) -%>
+<% end -%>
+<% if @alert.summary && @alert.summary.length > 0 -%>
+, <%= @alert.summary -%>
+<% end -%>

data/lib/flapjack/gateways/jabber/rollup.text.erb

@@ -0,0 +1,2 @@
+<%= @alert.type_sentence_case %>: <%= @alert.rollup_states_summary -%>
+ (<%= @alert.rollup_states_detail_text(:max_checks_per_state => 10) -%>)

data/lib/flapjack/gateways/pagerduty.rb

@@ -1,5 +1,6 @@
 #!/usr/bin/env ruby
 
+require 'em-hiredis'
 require 'em-synchrony'
 require 'em-synchrony/em-http'
 
@@ -7,7 +8,9 @@ require 'oj'
 
 require 'flapjack/data/entity_check'
 require 'flapjack/data/global'
+require 'flapjack/data/alert'
 require 'flapjack/redis_pool'
+require 'flapjack/utility'
 
 module Flapjack
 
@@ -17,6 +20,8 @@ module Flapjack
       PAGERDUTY_EVENTS_API_URL   = 'https://events.pagerduty.com/generic/2010-04-15/create_event.json'
       SEM_PAGERDUTY_ACKS_RUNNING = 'sem_pagerduty_acks_running'
 
+      include Flapjack::Utility
+
       def initialize(opts = {})
         @config = opts[:config]
         @logger = opts[:logger]
@@ -60,42 +65,58 @@ module Flapjack
         until @should_quit
           @logger.debug("pagerduty gateway is going into blpop mode on #{queue}")
           events[queue] = @redis.blpop(queue, 0)
-          event         = Oj.load(events[queue][1])
-          type          = event['notification_type']
-          @logger.debug("pagerduty notification event popped off the queue: " + event.inspect)
-          unless 'shutdown'.eql?(type)
-            event_id      = event['event_id']
-            entity, check = event_id.split(':', 2)
-            state         = event['state']
-            summary       = event['summary']
-            address       = event['address']
-
-            headline = type.upcase
-
-            case type.downcase
+          event_json    = events[queue][1]
+
+          begin
+            event = Oj.load(event_json)
+            @logger.debug("pagerduty notification event received: " + event.inspect)
+
+            if 'shutdown'.eql?(event['notification_type'])
+              @logger.debug("@should_quit: #{@should_quit}")
+              next
+            end
+
+            alert = Flapjack::Data::Alert.new(event, :logger => @logger)
+            @logger.debug("processing pagerduty notification service_key: #{alert.address}, entity: #{alert.entity}, " +
+                          "check: '#{alert.check}', state: #{alert.state}, summary: #{alert.summary}")
+
+            mydir = File.dirname(__FILE__)
+            message_template_path = mydir + "/pagerduty/alert.text.erb"
+            message_template = ERB.new(File.read(message_template_path), nil, '-')
+
+            @alert = alert
+            bnd    = binding
+
+            begin
+              message = message_template.result(bnd).chomp
+            rescue => e
+              @logger.error "Error while excuting the ERB for a pagerduty message, " +
+                "ERB being executed: #{message_template_path}"
+              raise
+            end
+
+            pagerduty_type = case alert.type
             when 'acknowledgement'
-              maint_str      = "has been acknowledged"
-              pagerduty_type = 'acknowledge'
+              'acknowledge'
             when 'problem'
-              maint_str      = "is #{state.upcase}"
-              pagerduty_type = "trigger"
+              'trigger'
             when 'recovery'
-              maint_str      = "is #{state.upcase}"
-              pagerduty_type = "resolve"
+              'resolve'
             when 'test'
-              maint_str      = ""
-              pagerduty_type = "trigger"
-              headline       = "TEST NOTIFICATION"
+              'trigger'
             end
 
-            message = "#{type.upcase} - \"#{check}\" on #{entity} #{maint_str} - #{summary}"
-
-            pagerduty_event = { :service_key  => address,
-                                :incident_key => event_id,
-                                :event_type   => pagerduty_type,
-                                :description  => message }
+            pagerduty_event = { 'service_key'  => alert.address,
+                                'incident_key' => alert.event_id,
+                                'event_type'   => pagerduty_type,
+                                'description'  => message }
 
             send_pagerduty_event(pagerduty_event)
+            alert.record_send_success!
+          rescue => e
+            @logger.error "Error generating or dispatching pagerduty message: #{e.class}: #{e.message}\n" +
+              e.backtrace.join("\n")
+            @logger.debug "Message that could not be processed: \n" + event_json
           end
         end
 
@@ -143,6 +164,10 @@ module Flapjack
         response = Oj.load(http.response)
         status   = http.response_header.status
         @logger.debug "send_pagerduty_event got a return code of #{status.to_s} - #{response.inspect}"
+        unless status == 200
+          raise "Error sending event to pagerduty: status: #{status.to_s} - #{response.inspect}" +
+                " posted data: #{options[:body]}"
+        end
         [status, response]
       end
 
@@ -187,10 +212,15 @@ module Flapjack
           if !pg_acknowledged_by.nil? && !pg_acknowledged_by['name'].nil?
             who_text = " by #{pg_acknowledged_by['name']}"
           end
+
+          # FIXME: decide where the default acknowledgement period should reside and use it
+          # everywhere ... a case for moving configuration into redis (from config file) perhaps?
+          four_hours = 4 * 60 * 60
           Flapjack::Data::Event.create_acknowledgement(
             entity_name, check,
-            :summary => "Acknowledged on PagerDuty" + who_text,
-            :redis => @redis)
+            :summary  => "Acknowledged on PagerDuty" + who_text,
+            :duration => four_hours,
+            :redis    => @redis)
         end
 
       end

data/lib/flapjack/gateways/pagerduty/alert.text.erb

@@ -0,0 +1,10 @@
+<%= @alert.type_sentence_case %>: "<%= @alert.check %>" on <%= @alert.entity -%>
+<% unless ['acknowledgement', 'test'].include?(@alert.notification_type) -%>
+ is <%= @alert.state_title_case -%>
+<% end -%>
+<% if ['acknowledgement'].include?(@alert.type) -%>
+ has been acknowledged, unscheduled maintenance created for <%= time_period_in_words(@alert.acknowledgement_duration) -%>
+<% end -%>
+<% if @alert.summary && @alert.summary.length > 0 -%>
+, <%= @alert.summary -%>
+<% end -%>

data/lib/flapjack/gateways/sms_messagenet.rb

@@ -4,6 +4,9 @@ require 'em-synchrony'
 require 'em-synchrony/em-http'
 require 'active_support/inflector'
 
+require 'flapjack/data/alert'
+require 'flapjack/utility'
+
 module Flapjack
   module Gateways
     class SmsMessagenet
@@ -12,41 +15,39 @@ module Flapjack
 
       class << self
 
+        include Flapjack::Utility
+
         def start
           @sent = 0
         end
 
-        def perform(notification)
-          @logger.debug "Woo, got a notification to send out: #{notification.inspect}"
+        def perform(contents)
+          @logger.debug "Woo, got a notification to send out: #{contents.inspect}"
+          alert = Flapjack::Data::Alert.new(contents, :logger => @logger)
 
           endpoint = @config["endpoint"] || MESSAGENET_DEFAULT_URL
           username = @config["username"]
           password = @config["password"]
 
-          @notification_type   = notification['notification_type']
-          @rollup              = notification['rollup']
-          @rollup_alerts       = notification['rollup_alerts']
-          @state               = notification['state']
-          @summary             = notification['summary']
-          @time                = notification['time']
-          @entity_name, @check = notification['event_id'].split(':', 2)
-          address              = notification['address']
-          notification_id      = notification['id']
-
-          message_type = case
-          when @rollup
-            'rollup'
-          else
-            'alert'
-          end
+          address         = alert.address
+          notification_id = alert.notification_id
+          message_type    = alert.rollup ? 'rollup' : 'alert'
 
-          sms_template = ERB.new(File.read(File.dirname(__FILE__) +
-            "/sms_messagenet/#{message_type}.text.erb"), nil, '-')
+          my_dir = File.dirname(__FILE__)
+          sms_template_path = my_dir + "/sms_messagenet/#{message_type}.text.erb"
+          sms_template = ERB.new(File.read(sms_template_path), nil, '-')
 
+          @alert  = alert
           bnd     = binding
-          message = sms_template.result(bnd).chomp
 
-          # TODO log error and skip instead of raising errors
+          begin
+            message = sms_template.result(bnd).chomp
+          rescue => e
+            @logger.error "Error while excuting the ERB for an sms: " +
+              "ERB being executed: #{sms_template_path}"
+            raise
+          end
+
           if @config.nil? || (@config.respond_to?(:empty?) && @config.empty?)
             @logger.error "Messagenet config is missing"
             return
@@ -82,25 +83,17 @@ module Flapjack
           status = (http.nil? || http.response_header.nil?) ? nil : http.response_header.status
           if (status >= 200) && (status <= 206)
             @sent += 1
-            @logger.info "Sent SMS via Messagenet, response status is #{status}, " +
+            alert.record_send_success!
+            @logger.debug "Sent SMS via Messagenet, response status is #{status}, " +
               "notification_id: #{notification_id}"
           else
             @logger.error "Failed to send SMS via Messagenet, response status is #{status}, " +
               "notification_id: #{notification_id}"
           end
-
-        end
-
-        # copied from ActiveSupport
-        def truncate(str, length, options = {})
-          text = str.dup
-          options[:omission] ||= "..."
-
-          length_with_room_for_omission = length - options[:omission].length
-          stop = options[:separator] ?
-            (text.rindex(options[:separator], length_with_room_for_omission) || length_with_room_for_omission) : length_with_room_for_omission
-
-          (text.length > length ? text[0...stop] + options[:omission] : text).to_s
+        rescue => e
+          @logger.error "Error generating or delivering sms to #{contents['address']}: #{e.class}: #{e.message}"
+          @logger.error e.backtrace.join("\n")
+          raise
         end
 
       end