flapjack 0.7.27 → 0.7.28

data/lib/flapjack/data/notification.rb

@@ -94,7 +94,11 @@ module Flapjack
       end
 
       def ok?
-        ['ok', 'up'].include?(@state)
+        @state && ['ok', 'up'].include?(@state.downcase)
+      end
+
+      def acknowledgement?
+        @state && ['acknowledgement'].include?(@state.downcase)
       end
 
       def contents
@@ -190,9 +194,33 @@ module Flapjack
 
           logger.debug "media_to_use: #{media_to_use}"
 
-          media_to_use.each_pair.inject([]) { |ret, (k, v)|
+          # here begins rollup madness
+          media_to_use.each_pair.inject([]) { |ret, (media, address)|
+            rollup_type = nil
+
+            contact.add_alerting_check_for_media(media, @event_id) unless ok? || acknowledgement?
+
+            # expunge checks in (un)scheduled maintenance from the alerting set
+            cleaned = contact.clean_alerting_checks_for_media(media)
+            logger.debug("cleaned alerting checks for #{media}: #{cleaned}")
+
+            alerting_checks  = contact.count_alerting_checks_for_media(media)
+            rollup_threshold = contact.rollup_threshold_for_media(media)
+            case
+            when rollup_threshold.nil?
+              # back away slowly
+            when alerting_checks >= rollup_threshold
+              next ret if contact.drop_rollup_notifications_for_media?(media)
+              contact.update_sent_rollup_alert_keys_for_media(media, :delete => ok?)
+              rollup_type = 'problem'
+            when (alerting_checks + cleaned >= rollup_threshold)
+              # alerting checks was just cleaned such that it is now below the rollup threshold
+              rollup_type = 'recovery'
+            end
+            logger.debug "rollup decisions: #{@event_id} #{@state} #{media} #{address} rollup_type: #{rollup_type}"
+
             m = Flapjack::Data::Message.for_contact(contact,
-                  :medium => k, :address => v)
+                  :medium => media, :address => address, :rollup => rollup_type)
             ret << m
             ret
           }

data/lib/flapjack/data/notification_rule.rb

@@ -145,7 +145,7 @@ module Flapjack
           :contact_id         => rule_data[:contact_id].to_s,
           :entities           => Oj.dump(rule_data[:entities]),
           :tags               => Oj.dump(tag_data),
-          :time_restrictions  => Oj.dump(rule_data[:time_restrictions]),
+          :time_restrictions  => Oj.dump(rule_data[:time_restrictions], :mode => :compat),
           :unknown_media      => Oj.dump(rule_data[:unknown_media]),
           :warning_media      => Oj.dump(rule_data[:warning_media]),
           :critical_media     => Oj.dump(rule_data[:critical_media]),

data/lib/flapjack/filters/delays.rb

@@ -36,12 +36,8 @@ module Flapjack
         end
 
         last_problem_alert   = entity_check.last_notification_for_state(:problem)[:timestamp]
-        last_warning_alert   = entity_check.last_notification_for_state(:warning)[:timestamp]
-        last_critical_alert  = entity_check.last_notification_for_state(:critical)[:timestamp]
         last_change          = entity_check.last_change
-        last_notification    = entity_check.last_notification
-        last_alert_state     = last_notification[:type]
-        last_alert_timestamp = last_notification[:timestamp]
+        last_alert_state     = entity_check.last_notification[:type]
 
         current_time = Time.now.to_i
         current_state_duration = current_time - last_change

data/lib/flapjack/gateways/api/contact_methods.rb

@@ -197,9 +197,11 @@ module Flapjack
 
             media = contact.media
             media_intervals = contact.media_intervals
+            media_rollup_thresholds = contact.media_rollup_thresholds
             media_addr_int = hashify(*media.keys) {|k|
-              [k, {'address'  => media[k],
-                   'interval' => media_intervals[k] }]
+              [k, {'address'          => media[k],
+                   'interval'         => media_intervals[k],
+                   'rollup_threshold' => media_rollup_thresholds[k] }]
             }
             media_addr_int.to_json
           end
@@ -218,8 +220,10 @@ module Flapjack
             if interval.nil?
               halt err(403, "no #{params[:id]} interval for contact '#{params[:contact_id]}'")
             end
-            {'address'  => media,
-             'interval' => interval}.to_json
+            rollup_threshold = contact.media_rollup_thresholds[params[:id]]
+            {'address'          => media,
+             'interval'         => interval,
+             'rollup_threshold' => rollup_threshold }.to_json
           end
 
           # Creates or updates a media of a contact
@@ -237,9 +241,11 @@ module Flapjack
 
             contact.set_address_for_media(params[:id], params[:address])
             contact.set_interval_for_media(params[:id], params[:interval])
+            contact.set_rollup_threshold_for_media(params[:id], params[:rollup_threshold])
 
-            {'address'  => contact.media[params[:id]],
-             'interval' => contact.media_intervals[params[:id]]}.to_json
+            {'address'          => contact.media[params[:id]],
+             'interval'         => contact.media_intervals[params[:id]],
+             'rollup_threshold' => contact.media_rollup_thresholds[params[:id]]}.to_json
           end
 
           # delete a media of a contact

data/lib/flapjack/gateways/email.rb

@@ -4,6 +4,7 @@ require 'mail'
 require 'erb'
 require 'socket'
 require 'chronic_duration'
+require 'active_support/inflector'
 
 require 'em-synchrony'
 require 'em/protocols/smtpclient'
@@ -26,6 +27,7 @@ module Flapjack
           @logger.debug("new email gateway pikelet with the following options: #{@config.inspect}")
           @smtp_config = @config.delete('smtp_config')
           @sent = 0
+          @fqdn = `/bin/hostname -f`.chomp
         end
 
         # TODO refactor to remove complexity
@@ -34,12 +36,17 @@ module Flapjack
           deliver( notification )
         end
 
+        # sets a bunch of class instance variables for each email
         def prepare(notification)
           @logger.debug "Woo, got a notification to send out: #{notification.inspect}"
 
           # The instance variables are referenced by the templates, which
           # share the current binding context
           @notification_type   = notification['notification_type']
+          @notification_id     = notification['id'] || SecureRandom.uuid
+          @rollup              = notification['rollup']
+          @rollup_alerts       = notification['rollup_alerts']
+          @rollup_threshold    = notification['rollup_threshold']
           @contact_first_name  = notification['contact_first_name']
           @contact_last_name   = notification['contact_last_name']
           @state               = notification['state']
@@ -57,17 +64,6 @@ module Flapjack
           @in_unscheduled_maintenance = entity_check.in_scheduled_maintenance?
           @in_scheduled_maintenance   = entity_check.in_unscheduled_maintenance?
 
-          headline_map = {'problem'         => 'Problem: ',
-                          'recovery'        => 'Recovery: ',
-                          'acknowledgement' => 'Acknowledgement: ',
-                          'test'            => 'Test Notification: ',
-                          'unknown'         => ''
-                         }
-
-          headline = headline_map[@notification_type] || ''
-
-          @subject = "#{headline}'#{@check}' on #{@entity_name}"
-          @subject += " is #{@state.upcase}" unless ['acknowledgement', 'test'].include?(@notification_type)
         rescue => e
           @logger.error "Error preparing email to #{m_to}: #{e.class}: #{e.message}"
           @logger.error e.backtrace.join("\n")
@@ -87,23 +83,19 @@ module Flapjack
             end
           end
 
-          fqdn = `/bin/hostname -f`.chomp
-          m_from = "flapjack@#{fqdn}"
+          m_from = "flapjack@#{@fqdn}"
           @logger.debug("flapjack_mailer: set from to #{m_from}")
           m_reply_to = m_from
           m_to       = notification['address']
 
-          @logger.debug("sending Flapjack::Notification::Email " +
-            "#{notification['id']} to: #{m_to} subject: #{@subject}")
 
-          mail = prepare_email(:subject => @subject,
-                               :from    => m_from,
-                               :to => m_to)
+          mail = prepare_email(:from => m_from,
+                               :to   => m_to)
 
           smtp_args = {:from     => m_from,
                        :to       => m_to,
                        :content  => "#{mail.to_s}\r\n.\r\n",
-                       :domain   => fqdn,
+                       :domain   => @fqdn,
                        :host     => host || 'localhost',
                        :port     => port || 25,
                        :starttls => starttls}
@@ -132,20 +124,37 @@ module Flapjack
         private
 
         def prepare_email(opts = {})
+          from       = opts[:from]
+          to         = opts[:to]
+          message_id = "<#{@notification_id}@#{@fqdn}>"
+
+          message_type = case
+          when @rollup
+            'rollup'
+          else
+            'alert'
+          end
+
+          subject_template = ERB.new(File.read(File.dirname(__FILE__) +
+            "/email/#{message_type}_subject.text.erb"), nil, '-')
 
           text_template = ERB.new(File.read(File.dirname(__FILE__) +
-            '/email/alert.text.erb'), nil, '-')
+            "/email/#{message_type}.text.erb"), nil, '-')
 
           html_template = ERB.new(File.read(File.dirname(__FILE__) +
-            '/email/alert.html.erb'), nil, '-')
+            "/email/#{message_type}.html.erb"), nil, '-')
+
+          bnd        = binding
+          subject    = subject_template.result(bnd).chomp
 
-          bnd = binding
+          @logger.debug("preparing email to: #{to}, subject: #{subject}, message-id: #{message_id}")
 
           mail = Mail.new do
-            from     opts[:from]
-            to       opts[:to]
-            subject  opts[:subject]
-            reply_to opts[:from]
+            from       from
+            to         to
+            subject    subject
+            reply_to   from
+            message_id message_id
 
             text_part do
               body text_template.result(bnd)

data/lib/flapjack/gateways/email/alert.html.erb

@@ -29,7 +29,7 @@
 
     <tr>
       <td><strong>State</strong></td>
-      <td><%= @state.upcase %></td>
+      <td><%= ['ok'].include?(@state) ? @state.upcase : @state.titleize %></td>
     </tr>
 
     <tr>
@@ -61,7 +61,7 @@
     <% if @last_state %>
       <tr>
         <td><strong>Previous State</strong></td>
-        <td><%= @last_state.upcase %></td>
+        <td><%= ['ok'].include?(@last_state) ? @last_state.upcase : @last_state.titleize %></td>
       </tr>
     <% end %>
 
@@ -75,5 +75,5 @@
   </tbody>
 </table>
 
-<p>Cheers,</p>
-<p>Flapjack</p>
+<p>Cheers,<br/>
+Flapjack</p>

data/lib/flapjack/gateways/email/alert.text.erb

@@ -4,7 +4,7 @@ Monitoring has detected the following:
 
 Entity:           <%= @entity_name %>
 Check:            <%= @check %>
-State:            <%= @state.upcase %>
+State:            <%= ['ok'].include?(@state) ? @state.upcase : @state.titleize %>
 Summary:          <%= @summary %>
 <% if @details -%>
 Details:          <%= @details %>
@@ -16,7 +16,7 @@ Time:             <%= Time.at(@time.to_i).to_s %>
 Duration:         <%= ChronicDuration.output(@duration) %>
 <% end -%>
 <% if @last_state -%>
-Previous State:   <%= @last_state.upcase %>
+Previous State:   <%= ['ok'].include?(@last_state) ? @last_state.upcase : @last_state.titleize %>
 <% end -%>
 <% if @last_summary -%>
 Previous Summary: <%= @last_summary %>

data/lib/flapjack/gateways/email/alert_subject.text.erb

@@ -0,0 +1,14 @@
+<% case @notification_type -%>
+<% when "problem" -%>
+<%=  "Problem: " -%>
+<% when "recovery" -%>
+<%=  "Recovery: " -%>
+<% when "acknowledgement" -%>
+<%=  "Acknowledgement: " -%>
+<% when "test" -%>
+<%=  "Test notification: " -%>
+<% end -%>
+'<%= @check %>' on <%= @entity_name -%>
+<% if ! ['acknowledgement', 'test'].include?(@notification_type) -%>
+ is <%= ['ok'].include?(@state) ? @state.upcase : @state.titleize -%>
+<% end -%>

data/lib/flapjack/gateways/email/rollup.html.erb

@@ -0,0 +1,48 @@
+<style type="text/css" media="screen">
+  #container {
+    text-transform: uppercase;
+  }
+  table {
+    border-collapse: collapse;
+  }
+  table, th, td {
+    border: 1px solid #666;
+    padding: 4px;
+  }
+</style>
+
+<p>Hi <%= @contact_first_name %></p>
+
+<p>You have <%= @rollup_alerts.length %> alerting check<%= @rollup_alerts.length == 1 ? '' : 's' %> as follows:</p>
+
+<table>
+  <tbody>
+    <tr>
+      <th>Check</th>
+      <th>Entity</th>
+      <th>State</th>
+      <th>Duration</th>
+    </tr>
+<%  @rollup_alerts.sort_by {|entity_check, details| details['duration'] }.each do |rollup_alert| -%>
+<%    r_entity, r_check = rollup_alert[0].split(':', 2) -%>
+<%    state    = rollup_alert[1]['state'] -%>
+<%    duration = ChronicDuration.output(rollup_alert[1]['duration']) -%>
+      <tr>
+        <td><%= r_check %></td>
+        <td><%= r_entity %></td>
+        <td><%= ['ok'].include?(state) ? state.upcase : state.titleize %></td>
+        <td><%= duration %></td>
+      </tr>
+<%    end %>
+  </tbody>
+</table>
+
+<% if @rollup.downcase == 'recovery' %>
+  <p>As your email summary threshold is <%= @rollup_threshold %>, we're taking your email alerts out of summary mode now. You'll now be emailed individually for each alerting check.</p>
+<% else %>
+  <p>Your email alerts are being summarised as your email summary threshold is set to <%= @rollup_threshold %>. You'll receive summary emails like this one until your number of alerting checks falls below <%= @rollup_threshold %>.</p>
+<% end %>
+
+<p>Cheers,<br/>
+Flapjack</p>
+

data/lib/flapjack/gateways/email/rollup.text.erb

@@ -0,0 +1,20 @@
+Hi <%= @contact_first_name %>
+
+You have <%= @rollup_alerts.length %> alerting check<%= @rollup_alerts.length == 1 ? '' : 's' %> as follows:
+
+<% @rollup_alerts.sort_by {|entity_check, details| details['duration'] }.each do |rollup_alert| -%>
+<%   r_entity, r_check = rollup_alert[0].split(':', 2) -%>
+<%   state    = rollup_alert[1]['state'] -%>
+<%   duration = ChronicDuration.output(rollup_alert[1]['duration']) -%>
+  * <%= r_check %> on <%= r_entity %> is <%= ['ok'].include?(state) ? state.upcase : state.titleize %> (<%= duration %>)
+<% end -%>
+
+<% if @rollup.downcase == 'recovery' -%>
+As your email summary threshold is <%= @rollup_threshold %>, we're taking your email alerts out of summary mode now. You'll now be emailed individually for each alerting check.
+<% else -%>
+Your email alerts are being summarised as your email summary threshold is set to <%= @rollup_threshold %>. You'll receive summary emails like this one until your number of alerting checks falls below <%= @rollup_threshold %>.
+<% end -%>
+
+Cheers,
+Flapjack
+

data/lib/flapjack/gateways/email/rollup_subject.text.erb

@@ -0,0 +1,19 @@
+<%
+  state_counts = @rollup_alerts.inject({}) do |memo, alert|
+    puts "alert: #{alert.inspect}"
+    memo[alert[1]['state']] = (memo[alert[1]['state']] || 0) + 1
+    memo
+  end
+  states_summary = ['critical', 'warning', 'unknown'].inject([]) do |memo, state|
+    next memo unless state_counts[state]
+    memo << "#{state.titleize}: #{state_counts[state]}"
+    memo
+  end.join(', ')
+-%>
+<% case @rollup -%>
+<% when "problem" -%>
+<%=  "Problem summary: " -%>
+<% when "recovery" -%>
+<%=  "Problem summaries finishing: " -%>
+<% end -%>
+<%= states_summary -%>

data/lib/flapjack/gateways/jabber.rb

@@ -127,6 +127,38 @@ module Flapjack
         end
       end
 
+      def get_check_details(entity_check)
+        sched   = entity_check.current_maintenance(:scheduled => true)
+        unsched = entity_check.current_maintenance(:unscheduled => true)
+        out = ''
+
+        if sched.nil? && unsched.nil?
+          out += "Not in scheduled or unscheduled maintenance.\n"
+        else
+          if sched.nil?
+            out += "Not in scheduled maintenance.\n"
+          else
+            start  = Time.at(sched[:start_time])
+            finish = Time.at(sched[:start_time] + sched[:duration])
+            remain = time_period_in_words( (finish - current_time).ceil )
+            # TODO a simpler time format?
+            out += "In scheduled maintenance: #{start} -> #{finish} (#{remain} remaining)\n"
+          end
+
+          if unsched.nil?
+            out += "Not in unscheduled maintenance.\n"
+          else
+            start  = Time.at(unsched[:start_time])
+            finish = Time.at(unsched[:start_time] + unsched[:duration])
+            remain = time_period_in_words( (finish - current_time).ceil )
+            # TODO a simpler time format?
+            out += "In unscheduled maintenance: #{start} -> #{finish} (#{remain} remaining)\n"
+          end
+        end
+
+        out
+      end
+
       def interpreter(command_raw)
         msg          = nil
         action       = nil
@@ -190,12 +222,13 @@ module Flapjack
 
         when /^help$/i
           msg = "commands: \n" +
-                "  ACKID <id> <comment> [duration: <time spec>] \n" +
-                "  find entities matching /pattern/ \n" +
-                "  test notifications for <entity>[:<check>] \n" +
-                "  tell me about <entity>[:<check>] \n" +
-                "  identify \n" +
-                "  help \n"
+                "  ACKID <id> <comment> [duration: <time spec>]\n" +
+                "  find entities matching /pattern/\n" +
+                "  find checks[ matching /pattern/] on (<entity>|entities matching /pattern/)\n" +
+                "  test notifications for <entity>[:<check>]\n" +
+                "  tell me about <entity>[:<check>]\n" +
+                "  identify\n" +
+                "  help\n"
 
         when /^identify$/i
           t    = Process.times
@@ -230,43 +263,6 @@ module Flapjack
 
             current_time = Time.now
 
-            get_details = proc {|entity_check|
-              sched   = entity_check.current_maintenance(:scheduled => true)
-              unsched = entity_check.current_maintenance(:unscheduled => true)
-              out = ''
-
-              if check_name.nil?
-                check = entity_check.check
-                out += "---\n#{entity_name}:#{check}\n"
-              end
-
-              if sched.nil? && unsched.nil?
-                out += "Not in scheduled or unscheduled maintenance.\n"
-              else
-                if sched.nil?
-                  out += "Not in scheduled maintenance.\n"
-                else
-                  start  = Time.at(sched[:start_time])
-                  finish = Time.at(sched[:start_time] + sched[:duration])
-                  remain = time_period_in_words( (finish - current_time).ceil )
-                  # TODO a simpler time format?
-                  out += "In scheduled maintenance: #{start} -> #{finish} (#{remain} remaining)\n"
-                end
-
-                if unsched.nil?
-                  out += "Not in unscheduled maintenance.\n"
-                else
-                  start  = Time.at(unsched[:start_time])
-                  finish = Time.at(unsched[:start_time] + unsched[:duration])
-                  remain = time_period_in_words( (finish - current_time).ceil )
-                  # TODO a simpler time format?
-                  out += "In unscheduled maintenance: #{start} -> #{finish} (#{remain} remaining)\n"
-                end
-              end
-
-              out
-            }
-
             check_names = check_name.nil? ? entity.check_list.sort : [check_name]
 
             if check_names.empty?
@@ -275,15 +271,69 @@ module Flapjack
               check_names.each do |check|
                 entity_check = Flapjack::Data::EntityCheck.for_entity(entity, check, :redis => @redis)
                 next if entity_check.nil?
-                msg += get_details.call(entity_check)
+                msg += "---\n#{entity_name}:#{check}\n" if check_name.nil?
+                msg += get_check_details(entity_check)
               end
             end
           else
             msg = "hmmm, I can't see #{entity_name} in my systems"
           end
 
-        when /^(find )?entities matching\s+\/(.*)\/.*$/i
-          pattern = $2.chomp.strip
+        when /^(?:find )?checks(?:\s+matching\s+\/(.+)\/)?\s+on\s+(?:entities matching\s+\/(.+)\/|([a-z0-9\-\.]+))/i
+          check_pattern = $1 ? $1.chomp.strip : nil
+          entity_pattern = $2 ? $2.chomp.strip : nil
+          entity_name = $3
+
+          entity_names = if entity_name
+            [entity_name]
+          elsif entity_pattern
+            Flapjack::Data::Entity.find_all_name_matching(entity_pattern, :redis => @redis)
+          else
+            []
+          end
+
+          msg = ""
+
+          # hash with entity => check_list, filtered by pattern if required
+          entities = entity_names.map {|name|
+            Flapjack::Data::Entity.find_by_name(name, :redis => @redis)
+          }.compact.inject({}) {|memo, entity|
+            memo[entity] = entity.check_list.select {|check_name|
+              !check_pattern || (check_name =~ /#{check_pattern}/i)
+            }
+            memo
+          }
+
+          report_entities = proc {|ents|
+            ents.inject('') do |memo, (entity, check_list)|
+              if check_list.empty?
+                memo += "Entity: #{entity.name} has no checks\n"
+              else
+                memo += "Entity: #{entity.name}\nChecks: #{check_list.join(', ')}\n"
+              end
+              memo += "----\n"
+              memo
+            end
+          }
+
+          case
+          when entity_pattern
+            if entities.empty?
+              msg = "found no entities matching /#{entity_pattern}/"
+            else
+              msg = "found #{entities.size} entities matching /#{entity_pattern}/ ... \n" +
+                    report_entities.call(entities)
+            end
+          when entity_name
+            if entities.empty?
+              msg = "found no entity for '#{entity_name}'"
+            else
+              msg = report_entities.call(entities)
+            end
+          end
+
+        when /^(?:find )?entities matching\s+\/(.+)\//i
+          pattern = $1.chomp.strip
           entity_list = Flapjack::Data::Entity.find_all_name_matching(pattern, :redis => @redis)
 
           if entity_list
@@ -295,7 +345,7 @@ module Flapjack
             when number_found == 0
               msg = "found no entities matching /#{pattern}/"
             when number_found == 1
-              msg = "found #{number_found} entity matching /#{pattern}/ ... \n"
+              msg = "found 1 entity matching /#{pattern}/ ... \n"
             when number_found > max_showable
               msg = "showing first #{max_showable} of #{number_found} entities found matching /#{pattern}/\n"
             else