flapjack 0.7.28 → 0.7.29

data/lib/flapjack/gateways/sms_messagenet/alert.text.erb

@@ -1,15 +1,5 @@
-<% case @notification_type -%>
-<% when "problem" -%>
-<%=  "Problem: " -%>
-<% when "recovery" -%>
-<%=  "Recovery: " -%>
-<% when "acknowledgement" -%>
-<%=  "Ack: " -%>
-<% when "test" -%>
-<%=  "Test notification: " -%>
+<%= @alert.type_sentence_case %>: '<%= @alert.check %>' on <%= @alert.entity -%>
+<% unless ['acknowledgement', 'test'].include?(@alert.notification_type) -%>
+ is <%= @alert.state_title_case -%>
 <% end -%>
-'<%= @check %>' on <%= @entity_name -%>
-<% unless ['acknowledgement', 'test'].include?(@notification_type) -%>
- is <%= ['ok'].include?(@state) ? @state.upcase : @state.titleize -%>
-<% end -%>
- at <%= Time.at(@time).strftime('%-d %b %H:%M') %>, <%= @summary -%>
+ at <%= Time.at(@alert.time).strftime('%-d %b %H:%M') %>, <%= @alert.summary -%>

data/lib/flapjack/gateways/sms_messagenet/rollup.text.erb

@@ -1,34 +1,2 @@
-<%
-  state_counts = @rollup_alerts.inject({}) do |memo, alert|
-    memo[alert[1]['state']] = (memo[alert[1]['state']] || 0) + 1
-    memo
-  end
-
-  states_summary = ['critical', 'warning', 'unknown'].inject([]) do |memo, state|
-    next memo unless state_counts[state]
-    memo << "#{state.titleize}: #{state_counts[state]}"
-    memo
-  end.join(', ')
-
-  states_detail = ['critical', 'warning', 'unknown'].inject([]) do |memo, state|
-
-    alerts = @rollup_alerts.find_all {|alert| alert[1]['state'] == state}
-    next memo if alerts.to_a.empty?
-
-    memo << "#{state.titleize}: " + alerts.inject([]) do |ret, alert|
-      entity, check = alert[0].split(':', 2)
-      ret << "'#{check}' on #{entity}"
-      ret
-    end.join(', ')
-
-    memo
-  end.join('; ')
-
--%>
-<% case @rollup -%>
-<% when "problem" -%>
-<%=  "Problem summary: " -%>
-<% when "recovery" -%>
-<%=  "Problem summaries finishing: " -%>
-<% end -%>
-<%= states_summary %> (<%= states_detail -%>)
+<%= @alert.type_sentence_case %>: <%= @alert.rollup_states_summary -%>
+ (<%= @alert.rollup_states_detail_text(:max_checks_per_state => 3) -%>)

data/lib/flapjack/gateways/web.rb

@@ -371,20 +371,29 @@ module Flapjack
       def self_stats
         @fqdn         = `/bin/hostname -f`.chomp
         @pid          = Process.pid
-        @instance_id  = "#{@fqdn}:#{@pid}"
-
-        @dbsize                  = redis.dbsize
-        @executive_instances     = redis.keys("executive_instance:*").map {|i|
-          [ i.match(/executive_instance:(.*)/)[1], redis.hget(i, 'boot_time').to_i ]
-        }.sort {|a, b| b[1] <=> a[1]}
-        @event_counters          = redis.hgetall('event_counters')
-        @event_counters_instance = redis.hgetall("event_counters:#{@instance_id}")
-        @boot_time               = Time.at(redis.hget("executive_instance:#{@instance_id}", 'boot_time').to_i)
-        @uptime                  = Time.now.to_i - @boot_time.to_i
-        @uptime_string           = time_period_in_words(@uptime)
-        @event_rate_all          = (@uptime > 0) ?
-                                   (@event_counters_instance['all'].to_f / @uptime) : 0
-        @events_queued           = redis.llen('events')
+
+        @dbsize              = redis.dbsize
+        @executive_instances = redis.keys("executive_instance:*").inject({}) do |memo, i|
+          instance_id    = i.match(/executive_instance:(.*)/)[1]
+          boot_time      = redis.hget(i, 'boot_time').to_i
+          uptime         = Time.now.to_i - boot_time
+          uptime_string  = ChronicDuration.output(uptime, :format => :short, :keep_zero => true, :units => 2)
+          event_counters = redis.hgetall("event_counters:#{instance_id}")
+          event_rates    = event_counters.inject({}) do |er, ec|
+            er[ec[0]] = uptime && uptime > 0 ? (ec[1].to_f / uptime).round : nil
+            er
+          end
+          memo[instance_id] = {
+            'boot_time'      => boot_time,
+            'uptime'         => uptime,
+            'uptime_string'  => uptime_string,
+            'event_counters' => event_counters,
+            'event_rates'    => event_rates
+          }
+          memo
+        end
+        @event_counters = redis.hgetall('event_counters')
+        @events_queued  = redis.llen('events')
       end
 
       def entity_stats

data/lib/flapjack/gateways/web/views/check.html.erb

@@ -19,6 +19,18 @@
         </div>
         <% state_qualifier = @check_enabled ? '' : "DISABLED. Last " %>
         <h3><%= state_qualifier %>State: <%= @check_state ? h(@check_state.upcase) : '' %></h3>
+        <% if @current_unscheduled_maintenance %>
+          <% ack_msg = "#{@current_unscheduled_maintenance[:summary]}" %>
+          <% ack_msg = ack_msg.length > 0 ? ack_msg : 'no summary given' %>
+          <h4>Acknowledged (<%= h ack_msg %>)</h4>
+          <% start  = Time.at(@current_unscheduled_maintenance[:start_time]) %>
+          <% finish = Time.at(@current_unscheduled_maintenance[:start_time] + @current_unscheduled_maintenance[:duration]) %>
+          <% remain = time_period_in_words( (finish - current_time).ceil ) %>
+          <p><%= h start.to_s %> -&gt; <%= h finish.to_s %> (<%= h remain %> remaining)</p>
+          <form action="/end_unscheduled_maintenance/<%= check_path_escaped %>" method="post">
+            <input type="submit" value="End Unscheduled Maintenance (Unacknowledge)" class="button">
+          </form>
+        <% end %>
         <% if (['warning', 'critical', 'unknown'].include?(@check_state) and !@current_scheduled_maintenance) %>
           <form action="/acknowledgements/<%= check_path_escaped %>" method="post" class="form-inline">
             <input type="hidden" name="acknowledgement_id" value="<%= @acknowledgement_id %>">
@@ -31,18 +43,10 @@
             e.g. "5 hours"
           </form>
         <% end %>
-        <% if @current_unscheduled_maintenance %>
-          <h3>(Acknowledged - <%= h @current_unscheduled_maintenance[:summary] %>)</h3>
-          <% start  = Time.at(@current_unscheduled_maintenance[:start_time]) %>
-          <% finish = Time.at(@current_unscheduled_maintenance[:start_time] + @current_unscheduled_maintenance[:duration]) %>
-          <% remain = time_period_in_words( (finish - current_time).ceil ) %>
-          <p><%= h start.to_s %> -&gt; <%= h finish.to_s %> (<%= h remain %> remaining)</p>
-          <form action="/end_unscheduled_maintenance/<%= check_path_escaped %>" method="post">
-            <input type="submit" value="End Unscheduled Maintenance (Unacknowledge)" class="button">
-          </form>
-        <% end %>
         <% if @current_scheduled_maintenance %>
-          <h4>(Scheduled Maintenance - <%= h @current_scheduled_maintenance[:summary] %></h4>
+          <% maint_msg = "#{@current_scheduled_maintenance[:summary]}" %>
+          <% maint_msg = maint_msg.length > 0 ? maint_msg : 'no summary given' %>
+          <h4>In <a href="#scheduled_maintenance_periods">Scheduled Maintenance</a> (<%= h maint_msg %>)</h4>
           <% start  = Time.at(@current_scheduled_maintenance[:start_time]) %>
           <% finish = Time.at(@current_scheduled_maintenance[:start_time] + @current_scheduled_maintenance[:duration]) %>
           <% remain = time_period_in_words( (finish - current_time).ceil ) %>
@@ -84,6 +88,7 @@
             Disabled
           <% end %>
 
+        <a name="scheduled_maintenance_periods"></a>
         <h3>Scheduled Maintenance Periods</h3>
           <% if @scheduled_maintenances && !@scheduled_maintenances.empty? %>
             <table class="table table-bordered table-hover table-condensed">

data/lib/flapjack/gateways/web/views/contact.html.erb

@@ -15,6 +15,8 @@
           <h2><%= h @contact.name %></h2>
         </div>
 
+        <% alerting = {} %>
+
         <h3>Contact Media</h3>
         <% if !@contact.media || @contact.media.empty? %>
           <p>No media</p>
@@ -24,9 +26,11 @@
               <th>Media</th>
               <th>Address</th>
               <th>Interval</th>
-              <th>Rollup Threshold</th>
+              <th>Summary Mode</th>
+              <th>Summary Threshold</th>
             </tr>
             <% @contact.media.each_pair do |mk, mv| %>
+              <% alerting[mk] = @contact.alerting_checks_for_media(mk) %>
               <tr>
                 <% if 'pagerduty'.eql?(mk) %>
                   <td>PagerDuty</td>
@@ -35,31 +39,69 @@
                       <p><%= 'password'.eql?(pk) ? h("#{pk}: ...") : h("#{pk}: #{pv}") %></p>
                     <% end %>
                   </td>
+                  <td></td>
+                  <td></td>
+                  <td></td>
                 <% else %>
                   <td><%= h mk.capitalize %></td>
                   <td><%= h mv %></td>
+                  <td>
+                    <% if @contact.media_intervals[mk] %>
+                      <%= h @contact.media_intervals[mk] %> seconds
+                    <% else %>
+                      no custom interval
+                    <% end %>
+                  </td>
+                  <td>
+                    <% if alerting[mk].length >= @contact.media_rollup_thresholds[mk].to_i %>
+                      Yes -
+                    <% else %>
+                      No -
+                    <% end %>
+                    <%= alerting[mk].length %> alerting
+                  </td>
+                  <td>
+                    <% if @contact.media_rollup_thresholds[mk] %>
+                      <%= h @contact.media_rollup_thresholds[mk] %>
+                    <% else %>
+                      -
+                    <% end %>
+                  </td>
                 <% end %>
+              </tr>
+            <% end %>
+          </table>
+        <% end %>
 
+        <h3>Alerting Checks</h3>
+        <p>These failing checks are currently alerting because they are not acknowledged, not in scheduled maintenance, and currently allowed by this contact's notification rules.</p>
+
+        <table class="table table-bordered table-hover table-condensed">
+          <tr>
+            <th>Media</th>
+            <th>Alerting Checks</th>
+          </tr>
+          <% alerting.each_pair do |media, checks| %>
+            <% if checks.length > 0 %>
+              <tr>
+                <td><%= h media.capitalize %></td>
                 <td>
-                  <% if @contact.media_intervals[mk] %>
-                    <%= h @contact.media_intervals[mk] %> seconds
-                  <% else %>
-                    no custom interval
-                  <% end %>
-                </td>
-                <td>
-                  <% if @contact.media_rollup_thresholds[mk] %>
-                    <%= h @contact.media_rollup_thresholds[mk] %> failing checks
-                  <% else %>
-                    -
+                  <% checks.each do |entity_check| %>
+                    <% entity, check = entity_check.split(':', 2) %>
+                    <% check_link = "<a href=\"/check?entity=#{u(entity)}&amp;check=#{u(check)}\" title=\"check status\">" +
+                       h(check) + "</a>"%>
+                    <a href="/entity/<%= u(entity) %>" title="entity status"><%= h entity %></a> ::
+                    <%= check_link %> <br />
                   <% end %>
                 </td>
               </tr>
+            <% else %>
+              <tr><td colspan="2">No alerting checks, yay!</td></tr>
             <% end %>
-          </table>
-        <% end %>
+          <% end %>
+        </table>
 
-        <h3>Entities and Checks</h3>
+        <h3>All Entities and Checks</h3>
         <% if !@entities_and_checks || @entities_and_checks.empty? %>
           <p>No entities</p>
         <% else %>
@@ -77,7 +119,7 @@
                 <td><a href="/entity/<%= u(entity.name) %>" title="entity status"><%= h entity.name %></a></td>
                 <td>
                   <% checks.each do |check| %>
-                    <a href="/check?entity=<%= u(entity.name) %>&amp;check=<%= u(check) %>" title="check status"><%= h check %></a>
+                  <%=   "<a href=\"/check?entity=#{u(entity.name)}&amp;check=#{u(check)}\" title=\"check status\">#{ h check }</a>" %>
                   <% end %>
                 </td>
               </tr>

data/lib/flapjack/gateways/web/views/self_stats.html.erb

@@ -15,76 +15,85 @@
           <h2>Internal Statistics</h2>
         </div>
 
-        <table class="table table-bordered table-hover table-condensed">
-          <tr>
-            <td>Events queued:</td>
-            <td><%= h @events_queued %></td>
-          </tr>
-          <tr>
-            <td>Number of entities:</td>
-            <td><%= h @count_all_entities %></td>
-          </tr>
-          <tr>
-            <td>Number of failing entities:</td>
-            <td><%= h @count_failing_entities %></td>
-          </tr>
-          <tr>
-            <td>Number of checks:</td>
-            <td><%= h @count_all_checks %></td>
-          </tr>
-          <tr>
-            <td>Number of failing checks:</td>
-            <td><%= h @count_failing_checks %></td>
-          </tr>
-          <tr>
-            <td>Events processed (all time)</td>
-            <td><%= h @event_counters['all'] %> (ok: <%= h @event_counters['ok'] %>, failure: <%= h @event_counters['failure'] %>, action: <%= h @event_counters['action'] %>)</td>
-          </tr>
-          <tr>
-            <td>Events processed (this instance)</td>
-            <td><%= h @event_counters_instance['all'] %> (ok: <%= h @event_counters_instance['ok'] %>, failure: <%= h @event_counters_instance['failure'] %>, action: <%= h @event_counters_instance['action'] %>)</td>
-          </tr>
-          <tr>
-            <td>Average rate (this instance)</td>
-            <td><%= h @event_rate_all %></td> events per second
-          </tr>
-          <tr>
-            <td>Total keys in redis</td>
-            <td><%= h @dbsize %></td>
-          </tr>
-          <tr>
-            <td>Uptime</td>
-            <td><%= h @uptime_string %></td>
-          </tr>
-          <tr>
-            <td>Boot Time</td>
-            <td><%= h @boot_time %></td>
-          </tr>
-          <tr>
-            <td>Current time</td>
-            <td><%= h Time.now.to_s %></td>
-          </tr>
-        </table>
-
-        <h4>Executive Instances:</h4>
-        <table class="table table-bordered table-hover">
-          <tr>
-            <th>Hostname</th>
-            <th>PID</th>
-            <th>Started</th>
-          </tr>
-          <% @executive_instances.each do |i| %>
-            <%
-              hostname, pid = i[0].split(':')
-              started = "#{relative_time_ago(Time.at(i[1].to_i))} ago"
-             %>
-            <tr>
-              <td><%= h hostname %></td>
-              <td><%= h pid %></td>
-              <td><%= h started %></td>
-            </tr>
-          <% end %>
-        </table>
+        <div class="row">
+          <div class="span5">
+            <h4>Global Statistics:</h4>
+            <table class="table table-bordered table-hover table-condensed">
+              <tr>
+                <td>Events queued:</td>
+                <td><%= h @events_queued %></td>
+              </tr>
+              <tr>
+                <td>Number of entities:</td>
+                <td><%= h @count_all_entities %></td>
+              </tr>
+              <tr>
+                <td>Number of failing entities:</td>
+                <td><%= h @count_failing_entities %></td>
+              </tr>
+              <tr>
+                <td>Number of checks:</td>
+                <td><%= h @count_all_checks %></td>
+              </tr>
+              <tr>
+                <td>Number of failing checks:</td>
+                <td><%= h @count_failing_checks %></td>
+              </tr>
+              <tr>
+                <td>Events processed (all time)</td>
+                <td>
+                  <ul>
+                    <li>all: <%= h @event_counters['all'] %> events</li>
+                    <li>ok: <%= h @event_counters['ok'] %> events</li>
+                    <li>failure: <%= h @event_counters['failure'] %> events</li>
+                    <li>action: <%= h @event_counters['action'] %> events</li>
+                  </ul>
+                </td>
+              </tr>
+              <tr>
+                <td>Total keys in redis</td>
+                <td><%= h @dbsize %></td>
+              </tr>
+              <tr>
+                <td>Current time</td>
+                <td><%= h Time.now.to_s %></td>
+              </tr>
+            </table>
+          </div>
+          <div class="span7">
+            <h4>Processor Instances:</h4>
+            <table class="table table-bordered table-hover">
+              <tr>
+                <th>Hostname</th>
+                <th>PID</th>
+                <th>Uptime</th>
+                <th>Events Processed</th>
+              </tr>
+              <% @executive_instances.sort_by {|i, d| d['uptime']}.each do |ei| %>
+                <%
+                  instance_id, details = ei
+                  hostname, pid  = instance_id.split(':')
+                  started        = details['uptime_string']
+                  event_counters = details['event_counters']
+                  event_rates    = details['event_rates']
+                 %>
+                <tr>
+                  <td><%= h hostname %></td>
+                  <td><%= h pid %></td>
+                  <td><%= h started %></td>
+                  <td>
+                    <ul>
+                      <li>all: <%= h event_counters['all'] %> (<%= h event_rates['all'] %> events/s)</li>
+                      <li>ok: <%= h event_counters['ok'] %> (<%= h event_rates['ok'] %> events/s)</li>
+                      <li>failure: <%= h event_counters['failure'] %> (<%= h event_rates['failure'] %> events/s)</li>
+                      <li>action: <%= h event_counters['action'] %> (<%= h event_rates['action'] %> events/s)</li>
+                    </ul>
+                  </td>
+                </tr>
+              <% end %>
+            </table>
+          </div>
+        </div>
 
         <p><a href="/self_stats.json">Instrument as JSON</a></p>
       </div>
@@ -94,4 +103,4 @@
       <%= foot %>
     </div>
   </body>
-</html>
+</html>

data/lib/flapjack/notifier.rb

@@ -92,7 +92,7 @@ module Flapjack
 
       timestamp    = Time.now
       event_id     = notification.event_id
-      entity_check = Flapjack::Data::EntityCheck.for_event_id(event_id, :redis => @redis)
+      entity_check = Flapjack::Data::EntityCheck.for_event_id(event_id, :redis => @redis, :logger => @logger)
       contacts     = entity_check.contacts
 
       if contacts.empty?
@@ -106,13 +106,16 @@ module Flapjack
 
       notification_contents = notification.contents
 
+      in_unscheduled_maintenance = entity_check.in_scheduled_maintenance?
+      in_scheduled_maintenance   = entity_check.in_unscheduled_maintenance?
+
       messages.each do |message|
         media_type = message.medium
         address    = message.address
         contents   = message.contents.merge(notification_contents)
 
         if message.rollup
-          contents['rollup_alerts'] = message.contact.alerting_checks_for_media(media_type).inject({}) do |memo, alert|
+          rollup_alerts = message.contact.alerting_checks_for_media(media_type).inject({}) do |memo, alert|
             ec = Flapjack::Data::EntityCheck.for_event_id(alert, :redis => @redis)
             last_change = ec.last_change
             memo[alert] = {
@@ -121,7 +124,10 @@ module Flapjack
             }
             memo
           end
+          contents['rollup_alerts'] = rollup_alerts
+
           contents['rollup_threshold'] = message.contact.rollup_threshold_for_media(media_type)
+
         end
 
         @notifylog.info("#{event_id} | " +