flapjack 0.7.1 → 0.7.2

data/features/notification_rules.feature

@@ -11,6 +11,7 @@ Feature: Notification rules on a per contact basis
       | id  | name | contacts |
       | 1   | foo  | 1        |
       | 2   | bar  | 1,2      |
+      | 3   | baz  | 1        |
 
     And user 1 has the following notification intervals:
       | email | sms |
@@ -20,6 +21,7 @@ Feature: Notification rules on a per contact basis
       | id | entities | entity_tags | warning_media | critical_media   | warning_blackhole | critical_blackhole | time_restrictions |
       | 1  | foo      |             | email         | sms,email        |                   |                    | 8-18 weekdays     |
       | 2  | bar      |             |               | sms,email        | true              |                    |                   |
+      | 3  | baz      |             | email         | sms,email        |                   |                    |                   |
 
   @time_restrictions @time
   Scenario: Alerts only during specified time restrictions
@@ -60,7 +62,45 @@ Feature: Notification rules on a per contact basis
     Then  no email alerts should be queued for malak@example.com
 
   @severity @time
-  Scenario: Alerts only when media,severity matches any matching rule's severity's media with ok->warning->critical
+  Scenario: Recoveries are not affected by notification rules
+    Given the check is check 'ping' on entity 'baz'
+    And   the check is in an ok state
+    When  a critical event is received
+    And   5 minutes passes
+    And   a critical event is received
+    Then  1 email alert should be queued for malak@example.com
+    When  1 minute passes
+    And   an ok event is received
+    Then  2 email alerts should be queued for malak@example.com
+
+  @severity @time
+  Scenario: Alerts are sent to media of highest severity reached since last ok
+    Given the check is check 'ping' on entity 'baz'
+    And   the check is in an ok state
+    When  a warning event is received
+    And   1 minute passes
+    And   a warning event is received
+    Then  1 email alert should be queued for malak@example.com
+    And   0 sms alerts should be queued for +61400000001
+    When  70 minutes passes
+    And   a critical event is received
+    And   1 minute passes
+    And   a critical event is received
+    Then  2 email alerts should be queued for malak@example.com
+    And   1 sms alert should be queued for +61400000001
+    When  70 minutes passes
+    And   a warning event is received
+    And   1 minute passes
+    And   a warning event is received
+    Then  3 email alerts should be queued for malak@example.com
+    And   2 sms alerts should be queued for +61400000001
+    When  70 minutes passes
+    And   an ok event is received
+    Then  4 email alerts should be queued for malak@example.com
+    And   3 sms alerts should be queued for +61400000001
+
+  @severity @time
+  Scenario: Alerts only when media,severity matches any matching rule's severity's media with ok->warning->critical->ok
     Given the check is check 'ping' on entity 'bar'
     And   the check is in an ok state
     When  a warning event is received
@@ -68,9 +108,12 @@ Feature: Notification rules on a per contact basis
     And   a warning event is received
     Then  no email alerts should be queued for malak@example.com
     When  a critical event is received
-    And   5 minute passes
+    And   5 minutes passes
     And   a critical event is received
     Then  1 email alert should be queued for malak@example.com
+    When  1 minute passes
+    And   an ok event is received
+    Then  2 email alert should be queued for malak@example.com
 
   @blackhole
   Scenario: Drop alerts matching a blackhole rule
@@ -91,3 +134,21 @@ Feature: Notification rules on a per contact basis
     And   a critical event is received
     Then  2 email alerts should be queued for malak@example.com
 
+  @intervals @time
+  Scenario: Problem directly after Recovery should alert despite notification intervals
+    Given the check is check 'ping' on entity 'baz'
+    And   the check is in an ok state
+    When  a critical event is received
+    And   1 minute passes
+    And   a critical event is received
+    Then  1 email alert should be queued for malak@example.com
+    And   1 sms alert should be queued for +61400000001
+    When  an ok event is received
+    Then  2 email alerts should be queued for malak@example.com
+    And   2 sms alerts should be queued for +61400000001
+    When  1 minute passes
+    And   a critical event is received
+    And   1 minute passes
+    And   a critical event is received
+    Then  3 email alerts should be queued for malak@example.com
+    And   3 sms alerts should be queued for +61400000001

data/features/steps/events_steps.rb

@@ -209,7 +209,8 @@ Then /^a notification should be generated(?: for check '([\w\.\-]+)' on entity '
   found.should be_true
 end
 
-Then /^show me the log$/ do
+Then /^show me the (\w+ )*log$/ do |adjective|
+  puts "the #{adjective}log:"
   puts @logger.messages.join("\n")
 end
 
@@ -232,8 +233,8 @@ end
 Given /^the following users exist:$/ do |contacts|
   contacts.hashes.each do |contact|
     media = {}
-    media['email'] = contact['email']
-    media['sms']   = contact['sms']
+    media['email'] = { 'address' => contact['email'] }
+    media['sms']   = { 'address' => contact['sms'] }
     Flapjack::Data::Contact.add({'id'         => contact['id'],
                                  'first_name' => contact['first_name'],
                                  'last_name'  => contact['last_name'],
@@ -302,3 +303,14 @@ Then /^(.*) email alert(?:s)? should be queued for (.*)$/ do |num_queued, addres
   queue  = Resque.peek('email_notifications', 0, 30)
   queue.find_all {|n| n['args'].first['address'] == address }.length.should == num_queued.to_i
 end
+
+Then /^(.*) sms alert(?:s)? should be queued for (.*)$/ do |num_queued, address|
+  check  = check  ? check  : @check
+  entity = entity ? entity : @entity
+  case num_queued
+  when 'no'
+    num_queued = 0
+  end
+  queue  = Resque.peek('sms_notifications', 0, 30)
+  queue.find_all {|n| n['args'].first['address'] == address }.length.should == num_queued.to_i
+end

data/flapjack.gemspec

@@ -32,7 +32,7 @@ Gem::Specification.new do |gem|
   gem.add_dependency 'haml'
   gem.add_dependency 'thin'
   gem.add_dependency 'mail'
-  gem.add_dependency 'blather', '0.8.1'
+  gem.add_dependency 'blather', '~> 0.8.3'
   gem.add_dependency 'chronic'
   gem.add_dependency 'chronic_duration'
   gem.add_dependency 'activesupport'

data/lib/flapjack/data/contact.rb

@@ -229,12 +229,17 @@ module Flapjack
       end
 
       def update_sent_alert_keys(opts)
-        media = opts[:media]
-        check = opts[:check]
-        state = opts[:state]
+        media  = opts[:media]
+        check  = opts[:check]
+        state  = opts[:state]
+        delete = !! opts[:delete]
         key = "drop_alerts_for_contact:#{self.id}:#{media}:#{check}:#{state}"
-        @redis.set(key, 'd')
-        @redis.expire(key, self.interval_for_media(media))
+        if delete
+          @redis.del(key)
+        else
+          @redis.set(key, 'd')
+          @redis.expire(key, self.interval_for_media(media))
+        end
       end
 
       # FIXME
@@ -331,14 +336,15 @@ module Flapjack
                     *['first_name', 'last_name', 'email'].collect {|f| [f, contact_data[f]]})
 
         unless contact_data['media'].nil?
-          contact_data['media'].each_pair {|medium, address|
+          contact_data['media'].each_pair {|medium, details|
             case medium
             when 'pagerduty'
-              redis.hset("contact_media:#{contact_id}", medium, address['service_key'])
+              redis.hset("contact_media:#{contact_id}", medium, details['service_key'])
               redis.hmset("contact_pagerduty:#{contact_id}",
-                          *['subdomain', 'username', 'password'].collect {|f| [f, address[f]]})
+                          *['subdomain', 'username', 'password'].collect {|f| [f, details[f]]})
             else
-              redis.hset("contact_media:#{contact_id}", medium, address)
+              redis.hset("contact_media:#{contact_id}", medium, details['address'])
+              redis.hset("contact_media_intervals:#{contact_id}", medium, details['interval']) if details['interval']
             end
           }
         end

data/lib/flapjack/data/entity.rb

@@ -78,15 +78,33 @@ module Flapjack
       # time
       def self.find_all_name_matching(pattern, options = {})
         raise "Redis connection not set" unless redis = options[:redis]
+        begin
+          regex = /#{pattern}/
+        rescue => e
+          if @logger
+            @logger.info("Jabber#self.find_all_name_matching - unable to use /#{pattern}/ as a regex pattern: #{e}")
+          end
+          return nil
+        end
         redis.keys('entity_id:*').inject([]) {|memo, check|
           a, entity_name = check.split(':')
-          if (entity_name =~ /#{pattern}/) && !memo.include?(entity_name)
+          if (entity_name =~ regex) && !memo.include?(entity_name)
             memo << entity_name
           end
           memo
         }.sort
       end
 
+      def self.find_all_with_checks(options)
+        raise "Redis connection not set" unless redis = options[:redis]
+        redis.keys("check:*").map {|s| s.match(/.*:(.*):.*/)[1] }.to_set
+      end
+
+      def self.find_all_with_failing_checks(options)
+        raise "Redis connection not set" unless redis = options[:redis]
+        redis.zrange("failed_checks", 0, -1).map {|s| s.match(/(.*):.*/)[1] }.to_set
+      end
+
       def contacts
         contact_ids = @redis.smembers("contacts_for:#{id}")
 

data/lib/flapjack/data/entity_check.rb

@@ -304,6 +304,18 @@ module Flapjack
         ln
       end
 
+      def max_notified_severity_of_current_failure
+        last_recovery = last_recovery_notification || 0
+
+        last_critical = last_critical_notification
+        return STATE_CRITICAL if last_critical && (last_critical > last_recovery)
+
+        last_warning = last_warning_notification
+        return STATE_WARNING if last_warning && (last_warning > last_recovery)
+
+        nil
+      end
+
       # unpredictable results if there are multiple notifications of different
       # types sent at the same time
       def last_notification

data/lib/flapjack/data/event.rb

@@ -25,6 +25,9 @@ module Flapjack
                      :archive_events => false,
                      :events_archive_maxage => (3 * 60 * 60) }
         options  = defaults.merge(opts)
+        if options[:logger]
+          logger = options[:logger]
+        end
 
         if options[:archive_events]
           dest = "events_archive:#{Time.now.utc.strftime "%Y%m%d%H"}"
@@ -43,8 +46,13 @@ module Flapjack
             return unless raw
           end
         end
-        return self.new( ::JSON.parse( raw ) )
-
+        begin
+          parsed = ::JSON.parse( raw )
+        rescue => e
+          logger.warn("Error deserialising event json: #{e}, raw json: #{raw.inspect}")
+          return nil
+        end
+        return self.new( parsed )
       end
 
       # creates, or modifies, an event object and adds it to the events list in redis

data/lib/flapjack/data/notification.rb

@@ -6,10 +6,12 @@ module Flapjack
   module Data
     class Notification
 
-      attr_accessor :event, :type
+      attr_accessor :event, :type, :max_notified_severity
 
       def self.for_event(event, opts = {})
-        self.new(:event => event, :type => opts[:type])
+        self.new(:event => event,
+                 :type => opts[:type],
+                 :max_notified_severity => opts[:max_notified_severity])
       end
 
       def messages(opts = {})
@@ -28,12 +30,13 @@ module Flapjack
       end
 
       def contents
-        @contents ||= {'event_id'          => event.id,
-                       'state'             => event.state,
-                       'summary'           => event.summary,
-                       'time'              => event.time,
-                       'duration'          => event.duration || nil,
-                       'notification_type' => type}
+        @contents ||= {'event_id'              => event.id,
+                       'state'                 => event.state,
+                       'summary'               => event.summary,
+                       'time'                  => event.time,
+                       'duration'              => event.duration || nil,
+                       'notification_type'     => type,
+                       'max_notified_severity' => max_notified_severity }
       end
 
     private
@@ -42,6 +45,7 @@ module Flapjack
         raise "Event not passed" unless event = opts[:event]
         @event = event
         @type  = opts[:type]
+        @max_notified_severity = opts[:max_notified_severity]
       end
 
     end

data/lib/flapjack/data/notification_rule.rb

@@ -105,7 +105,7 @@ module Flapjack
       end
 
       def update(rule_data)
-        self.class.add_or_update(rule_data, :redis => @redis)
+        self.class.add_or_update(rule_data.merge(:id => @id), :redis => @redis)
         self.refresh
       end
 
@@ -154,6 +154,8 @@ module Flapjack
       end
 
       def self.add_or_update(rule_data, options = {})
+        raise ":id is a required key in rule_data" unless rule_data[:id]
+
         redis = options[:redis]
 
         rule_data[:entities]          = Yajl::Encoder.encode(rule_data[:entities])

data/lib/flapjack/executive.rb

@@ -80,9 +80,6 @@ module Flapjack
       # we could generate a fuid and save it to disk, and prepend it from that
       # point on...
 
-      # TODO unset on exit?
-      @redis.set('boot_time', @boot_time.to_i)
-
       # FIXME: add an administrative function to reset all event counters
       if @redis.hget('event_counters', 'all').nil?
         @redis.hset('event_counters', 'all', 0)
@@ -91,11 +88,27 @@ module Flapjack
         @redis.hset('event_counters', 'action', 0)
       end
 
-      @redis.zadd('executive_instances', @boot_time.to_i, @instance_id)
+      #@redis.zadd('executive_instances', @boot_time.to_i, @instance_id)
+      @redis.hset("executive_instance:#{@instance_id}", 'boot_time', @boot_time.to_i)
       @redis.hset("event_counters:#{@instance_id}", 'all', 0)
       @redis.hset("event_counters:#{@instance_id}", 'ok', 0)
       @redis.hset("event_counters:#{@instance_id}", 'failure', 0)
       @redis.hset("event_counters:#{@instance_id}", 'action', 0)
+      touch_keys
+    end
+
+    # expire instance keys after one week
+    # TODO: set up a separate EM timer to reset key expiry every minute
+    # and reduce the expiry to, say, five minutes
+    # TODO: remove these keys on process exit
+    def touch_keys
+      [ "executive_instance:#{@instance_id}",
+        "event_counters:#{@instance_id}",
+        "event_counters:#{@instance_id}",
+        "event_counters:#{@instance_id}",
+        "event_counters:#{@instance_id}" ].each {|key|
+          @redis.expire(key, 1036800)
+        }
     end
 
     def start
@@ -105,7 +118,8 @@ module Flapjack
         @logger.debug("Waiting for event...")
         event = Flapjack::Data::Event.next(:redis => @redis,
                                            :archive_events => @archive_events,
-                                           :events_archive_maxage => @events_archive_maxage)
+                                           :events_archive_maxage => @events_archive_maxage,
+                                           :logger => @logger)
         process_event(event) unless event.nil?
       end
 
@@ -157,6 +171,10 @@ module Flapjack
     end
 
     def update_keys(event, entity_check)
+
+      # TODO: run touch_keys from a separate EM timer for efficiency
+      touch_keys
+
       result    = { :skip_filters => false }
       timestamp = Time.now.to_i
       @event_count = @redis.hincrby('event_counters', 'all', 1)
@@ -243,6 +261,9 @@ module Flapjack
           notification_type = 'test'
         end
       end
+
+      max_notified_severity = entity_check.max_notified_severity_of_current_failure
+
       @redis.set("#{event.id}:last_#{notification_type}_notification", timestamp)
       @redis.set("#{event.id}:last_#{event.state}_notification", timestamp) if event.failure?
       @redis.rpush("#{event.id}:#{notification_type}_notifications", timestamp)
@@ -252,13 +273,17 @@ module Flapjack
       contacts = entity_check.contacts
 
       if contacts.empty?
+        @logger.debug("No contacts for #{event.id}")
         @notifylog.info("#{Time.now.to_s} | #{event.id} | #{notification_type} | NO CONTACTS")
         return
       end
 
-      notification = Flapjack::Data::Notification.for_event(event, :type => notification_type)
+      notification = Flapjack::Data::Notification.for_event(
+        event, :type => notification_type, :max_notified_severity => max_notified_severity)
 
-      enqueue_messages( apply_notification_rules( notification.messages(:contacts => contacts) ) )
+      messages = notification.messages(:contacts => contacts)
+      messages = apply_notification_rules(messages)
+      enqueue_messages(messages)
 
     end
 
@@ -291,8 +316,7 @@ module Flapjack
       # don't consider notification rules if the contact has none
 
       tuple = messages.map do |message|
-        @logger.debug "considering message: #{message.medium} #{message.notification.event.id} #{message.notification.event.state}"
-        @logger.debug "contact_id: #{message.contact.id}"
+        @logger.debug "considering message for contact: #{message.contact.id} #{message.medium} #{message.notification.event.id} #{message.notification.event.state}"
         rules    = message.contact.notification_rules
         @logger.debug "found #{rules.length} rules for this message's contact"
         event_id = message.notification.event.id
@@ -337,15 +361,32 @@ module Flapjack
 
       # delete any media that doesn't meet severity<->media constraints
       tuple = tuple.find_all do |message, matchers, options|
-        severity = message.notification.event.state
+        state = message.notification.event.state
+        max_notified_severity = message.notification.max_notified_severity
+
+        # use EntityCheck#max_notified_severity_of_current_failure
+        # as calculated prior to updating the last_notification* keys
+        # if it's a higher severity than the current state
+        severity = 'ok'
+        case
+        when ([state, max_notified_severity] & ['critical', 'unknown']).any?
+          severity = 'critical'
+        when [state, max_notified_severity].include?('warning')
+          severity = 'warning'
+        end
         options[:no_rules_for_contact] ||
           matchers.any? {|matcher|
-            matcher.media_for_severity(severity).include?(message.medium) ||
-              (@logger.warn("got nil for matcher.media_for_severity(#{severity}), matcher: #{matcher.inspect}") && false)
+            mms = matcher.media_for_severity(severity)
+            unless mms
+              answer = false
+            else
+              answer = mms.include?(message.medium)
+            end
+            answer
           }
       end
 
-      @logger.debug "apply_notification_rules: num messages after severity-media constraints: #{tuple.size}"
+      @logger.debug "apply_notification_rules: num messages after pruning for severity-media constraints: #{tuple.size}"
 
       # delete media based on notification interval
       tuple = tuple.find_all do |message, matchers, options|
@@ -378,10 +419,23 @@ module Flapjack
 
         @logger.info("Enqueueing #{media_type} alert for #{event_id} to #{message.address}")
 
-        message.contact.update_sent_alert_keys(:media => message.medium,
-                                               :check => message.notification.event.id,
-                                               :state => message.notification.event.state)
-          # drop_alerts_for_contact:#{self.id}:#{media}:#{check}:#{state}
+        if message.notification.event.state == 'ok'
+          message.contact.update_sent_alert_keys(
+            :media => message.medium,
+            :check => message.notification.event.id,
+            :state => 'warning',
+            :delete => true)
+          message.contact.update_sent_alert_keys(
+            :media => message.medium,
+            :check => message.notification.event.id,
+            :state => 'critical',
+            :delete => true)
+        else
+          message.contact.update_sent_alert_keys(
+            :media => message.medium,
+            :check => message.notification.event.id,
+            :state => message.notification.event.state)
+        end
 
         # TODO consider changing Resque jobs to use raw blpop like the others
         case media_type.to_sym