flapjack 0.7.22 → 0.7.25

data/lib/flapjack/data/entity_check.rb

@@ -262,23 +262,25 @@ module Flapjack
         details = options[:details]
         count = options[:count]
 
-        if self.state != new_state
-          client = options[:client]
+        old_state = self.state
+
+        @redis.multi
+
+        if old_state != new_state
 
           # Note the current state (for speedy lookups)
           @redis.hset("check:#{@key}", 'state', new_state)
 
           # FIXME: rename to last_state_change?
           @redis.hset("check:#{@key}", 'last_change', timestamp)
-          case state
+
+          case new_state
           when STATE_WARNING, STATE_CRITICAL, STATE_UNKNOWN
             @redis.zadd('failed_checks', timestamp, @key)
             # FIXME: Iterate through a list of tags associated with an entity:check pair, and update counters
-            @redis.zadd("failed_checks:client:#{client}", timestamp, @key) if client
           else
             @redis.zrem("failed_checks", @key)
             # FIXME: Iterate through a list of tags associated with an entity:check pair, and update counters
-            @redis.zrem("failed_checks:client:#{client}", @key) if client
           end
 
           # Retain event data for entity:check pair
@@ -291,22 +293,27 @@ module Flapjack
           @redis.zadd("#{@key}:sorted_state_timestamps", timestamp, timestamp)
         end
 
+        # Track when we last saw an event for a particular entity:check pair
+        self.last_update = timestamp
+
         # Even if this isn't a state change, we need to update the current state
         # hash summary and details (as they may have changed)
         @redis.hset("check:#{@key}", 'summary', (summary || ''))
         @redis.hset("check:#{@key}", 'details', (details || ''))
-      end
 
-      def last_update
-        lu = @redis.hget("check:#{@key}", 'last_update')
-        return unless (lu && lu =~ /^\d+$/)
-        lu.to_i
+        @redis.exec
       end
 
       def last_update=(timestamp)
         @redis.hset("check:#{@key}", 'last_update', timestamp)
         @redis.zadd("current_checks:#{entity.name}", timestamp, check)
-        @redis.zadd("current_entities", timestamp, entity.name)
+        @redis.zadd('current_entities', timestamp, entity.name)
+      end
+
+      def last_update
+        lu = @redis.hget("check:#{@key}", 'last_update')
+        return unless lu && !!(lu =~ /^\d+$/)
+        lu.to_i
       end
 
       # disables a check (removes currency)
@@ -320,12 +327,12 @@ module Flapjack
       end
 
       def enabled?
-        !! @redis.zscore("current_checks:#{entity.name}", check)
+        !!@redis.zscore("current_checks:#{entity.name}", check)
       end
 
       def last_change
         lc = @redis.hget("check:#{@key}", 'last_change')
-        return unless (lc && lc =~ /^\d+$/)
+        return unless lc && !!(lc =~ /^\d+$/)
         lc.to_i
       end
 

data/lib/flapjack/data/event.rb

@@ -6,7 +6,7 @@ module Flapjack
   module Data
     class Event
 
-      attr_accessor :counter, :previous_state, :previous_state_duration, :tags
+      attr_accessor :counter, :tags
 
       attr_reader :check, :summary, :details, :acknowledgement_id
 
@@ -57,9 +57,12 @@ module Flapjack
       end
 
       # creates, or modifies, an event object and adds it to the events list in redis
+      #   'entity'    => entity,
+      #   'check'     => check,
       #   'type'      => 'service',
       #   'state'     => state,
       #   'summary'   => check_output,
+      #   'details'   => check_long_output,
       #   'time'      => timestamp
       def self.add(evt, opts = {})
         raise "Redis connection not set" unless redis = opts[:redis]
@@ -129,12 +132,6 @@ module Flapjack
         (entity || '-') + ':' + (check || '-')
       end
 
-      # FIXME: site specific
-      def client
-        return unless entity
-        entity.split('-').first
-      end
-
       def type
         return unless @type
         @type.downcase

data/lib/flapjack/data/notification.rb

@@ -10,7 +10,7 @@ module Flapjack
   module Data
     class Notification
 
-      attr_reader :type, :event_id, :event_state, :event_count
+      attr_reader :type, :event_id, :state
 
       def self.type_for_event(event)
         case event.type
@@ -49,18 +49,21 @@ module Flapjack
         last_state = opts[:last_state] || {}
 
         tag_data = event.tags.is_a?(Set) ? event.tags.to_a : nil
-        notif = {'event_id'     => event.id,
-                 'state'        => event.state,
-                 'summary'      => event.summary,
-                 'last_state'   => last_state[:state],
-                 'last_summary' => last_state[:summary],
-                 'details'      => event.details,
-                 'time'         => event.time,
-                 'duration'     => event.duration || nil,
-                 'type'         => opts[:type] || type_for_event(event),
-                 'severity'     => opts[:severity],
-                 'count'        => event.counter,
-                 'tags'         => tag_data }
+        notif = {'event_id'       => event.id,
+                 'state'          => event.state,
+                 'summary'        => event.summary,
+                 'details'        => event.details,
+                 'time'           => event.time,
+                 'duration'       => event.duration,
+                 'count'          => event.counter,
+                 'last_state'     => last_state[:state],
+                 'last_summary'   => last_state[:summary],
+                 'state_duration' => opts[:state_duration],
+
+                 'type'           => opts[:type] || type_for_event(event),
+                 'severity'       => opts[:severity],
+
+                 'tags'           => tag_data }
 
         redis.rpush(queue, Oj.dump(notif))
       end
@@ -88,17 +91,22 @@ module Flapjack
         self.new( parsed )
       end
 
+      def ok?
+        ['ok', 'up'].include?(@state)
+      end
+
       def contents
         @contents ||= {'event_id'          => @event_id,
-                       'state'             => @event_state,
-                       'summary'           => @event_summary,
-                       'last_state'        => @last_event_state,
-                       'last_summary'      => @last_event_summary,
-                       'details'           => @event_details,
-                       'time'              => @event_time,
-                       'duration'          => @event_duration,
+                       'state'             => @state,
+                       'summary'           => @summary,
+                       'duration'          => @duration,
+                       'last_state'        => @last_state,
+                       'last_summary'      => @last_summary,
+                       'state_duration'    => @state_duration,
+                       'details'           => @details,
+                       'time'              => @time,
                        'notification_type' => @type,
-                       'event_count'       => @event_count,
+                       'event_count'       => @count,
                        'tags'              => @tags
                       }
       end
@@ -115,7 +123,7 @@ module Flapjack
           media = contact.media
 
           logger.debug "Notification#messages: creating messages for contact: #{contact_id} " +
-            "event_id: \"#{@event_id}\" state: #{@event_state} event_tags: #{@tags.to_json} media: #{media.inspect}"
+            "event_id: \"#{@event_id}\" state: #{@state} event_tags: #{@tags.to_json} media: #{media.inspect}"
           rlen = rules.length
           logger.debug "found #{rlen} rule#{(rlen == 1) ? '' : 's'} for contact #{contact_id}"
 
@@ -132,7 +140,7 @@ module Flapjack
 
             logger.debug "#{matchers.length} matchers remain for this contact after time, entity and tags are matched:"
             matchers.each do |matcher|
-              logger.debug "matcher: #{matcher.to_json}"
+              logger.debug "  - #{matcher.to_json}"
             end
 
             # delete any general matchers if there are more specific matchers left
@@ -143,35 +151,42 @@ module Flapjack
               matchers.reject! {|matcher| !matcher.is_specific? }
 
               if num_matchers != matchers.length
-                logger.debug("notification: removal of general matchers when entity specific matchers are present: number of matchers changed from #{num_matchers} to #{matchers.length} for contact id: #{contact_id}")
+                logger.debug("removal of general matchers when entity specific matchers are present: number of matchers changed from #{num_matchers} to #{matchers.length} for contact id: #{contact_id}")
                 matchers.each do |matcher|
-                  logger.debug "matcher: #{matcher.to_json}"
+                  logger.debug "  - #{matcher.to_json}"
                 end
               end
             end
 
             # delete media based on blackholes
-            next if matchers.any? {|matcher| matcher.blackhole?(@event_state) }
-
-            logger.debug "notification: num matchers after removing blackhole matchers: #{matchers.size}"
+            blackhole_matchers = matchers.map {|matcher| matcher.blackhole?(@severity) ? matcher : nil }.compact
+            if blackhole_matchers.length > 0
+              logger.debug "dropping this media as #{blackhole_matchers.length} blackhole matchers are present:"
+              blackhole_matchers.each {|bm|
+                logger.debug "  - #{bm.to_json}"
+              }
+              next
+            else
+              logger.debug "no blackhole matchers matched"
+            end
 
             rule_media = matchers.collect{|matcher|
               matcher.media_for_severity(@severity)
             }.flatten.uniq
 
-            logger.debug "notification: collected media_for_severity(#{@severity}): #{rule_media}"
+            logger.debug "collected media_for_severity(#{@severity}): #{rule_media}"
             rule_media = rule_media.reject {|medium|
               contact.drop_notifications?(:media => medium,
                                           :check => @event_id,
-                                          :state => @event_state)
+                                          :state => @state)
             }
 
-            logger.debug "notification: media after contact_drop?: #{rule_media}"
+            logger.debug "media after contact_drop?: #{rule_media}"
 
             media.select {|medium, address| rule_media.include?(medium) }
           end
 
-          logger.debug "notification: media_to_use: #{media_to_use}"
+          logger.debug "media_to_use: #{media_to_use}"
 
           media_to_use.each_pair.inject([]) { |ret, (k, v)|
             m = Flapjack::Data::Message.for_contact(contact,
@@ -186,19 +201,22 @@ module Flapjack
 
       # created from parsed JSON, so opts keys are in strings
       def initialize(opts = {})
-        @event_id           = opts['event_id']
-        @event_state        = opts['state']
-        @event_summary      = opts['summary']
-        @event_details      = opts['details']
-        @event_time         = opts['time']
-        @event_duration     = opts['duration']
-        @event_count        = opts['count']
-        @last_event_state   = opts['last_state']
-        @last_event_summary = opts['last_summary']
-        @type               = opts['type']
-        @severity           = opts['severity']
-        tags                = opts['tags']
-        @tags               = tags.is_a?(Array) ? Flapjack::Data::TagSet.new(tags) : nil
+        @event_id       = opts['event_id']
+        @state          = opts['state']
+        @summary        = opts['summary']
+        @details        = opts['details']
+        @time           = opts['time']
+        @count          = opts['count']
+
+        @last_state     = opts['last_state']
+        @last_summary   = opts['last_summary']
+        @state_duration = opts['state_duration']
+
+        @type           = opts['type']
+        @severity       = opts['severity']
+
+        tags            = opts['tags']
+        @tags           = tags.is_a?(Array) ? Flapjack::Data::TagSet.new(tags) : nil
       end
 
       # # time restrictions match?

data/lib/flapjack/filters/acknowledgement.rb

@@ -11,32 +11,34 @@ module Flapjack
     class Acknowledgement
       include Base
 
-      def block?(event)
+      def block?(event, entity_check, previous_state)
         timestamp = Time.now.to_i
-        result = false
-        if event.type == 'action'
-          if event.acknowledgement?
-            if @redis.zscore("failed_checks", event.id)
-              ec = Flapjack::Data::EntityCheck.for_event_id(event.id, :redis => @redis)
-              if ec.nil?
-                @logger.error "Filter: Acknowledgement: unknown entity for event '#{event.id}'"
-              else
-                ec.create_unscheduled_maintenance(timestamp,
-                  (event.duration || (4 * 60 * 60)),
-                  :summary  => event.summary)
-                message = "unscheduled maintenance created for #{event.id}"
-              end
-            else
-              result = true
-              @logger.debug("Filter: Acknowledgement: blocking because zscore of failed_checks for #{event.id} is false") unless @redis.zscore("failed_checks", event.id)
-            end
-          else
-            message = "no action taken"
-            result  = false
-          end
+
+        label = 'Filter: Acknowledgement:'
+
+        return false unless event.type == 'action'
+
+        unless event.acknowledgement?
+          @logger.debug("#{label} pass (not an ack)")
+          return false
+        end
+
+        if entity_check.nil?
+          @logger.error "#{label} unknown entity for event '#{event.id}'"
+          return false
         end
-        @logger.debug("Filter: Acknowledgement: #{result ? "block" : "pass"} (#{message})")
-        result
+
+        unless @redis.zscore("failed_checks", event.id)
+          @logger.debug("#{label} blocking because zscore of failed_checks for #{event.id} is false")
+          return true
+        end
+
+        entity_check.create_unscheduled_maintenance(timestamp,
+          (event.duration || (4 * 60 * 60)),
+          :summary  => event.summary)
+
+        @logger.debug("#{label} pass (unscheduled maintenance created for #{event.id})")
+        false
       end
     end
   end

data/lib/flapjack/filters/delays.rb

@@ -19,58 +19,62 @@ module Flapjack
     class Delays
       include Base
 
-      def block?(event)
+      def block?(event, entity_check, previous_state)
         failure_delay = 30
         resend_delay  = 300
 
-        result = false
+        label = 'Filter: Delays:'
 
-        if event.service? && event.failure?
+        unless event.service? && event.failure?
+          @logger.debug("#{label} pass - not a service event in a failure state")
+          return false
+        end
+
+        unless entity_check.failed?
+          @logger.debug("#{label} entity_check.failed? returned false ...")
+          return false
+        end
 
-          entity_check = Flapjack::Data::EntityCheck.for_event_id(event.id, :redis => @redis)
-          current_time = Time.now.to_i
+        last_problem_alert   = entity_check.last_notification_for_state(:problem)[:timestamp]
+        last_warning_alert   = entity_check.last_notification_for_state(:warning)[:timestamp]
+        last_critical_alert  = entity_check.last_notification_for_state(:critical)[:timestamp]
+        last_change          = entity_check.last_change
+        last_notification    = entity_check.last_notification
+        last_alert_state     = last_notification[:type]
+        last_alert_timestamp = last_notification[:timestamp]
 
-          if entity_check.failed?
-            last_problem_alert   = entity_check.last_notification_for_state(:problem)[:timestamp]
-            last_warning_alert   = entity_check.last_notification_for_state(:warning)[:timestamp]
-            last_critical_alert  = entity_check.last_notification_for_state(:critical)[:timestamp]
-            last_change          = entity_check.last_change
-            last_notification    = entity_check.last_notification
-            last_alert_state     = last_notification[:type]
-            last_alert_timestamp = last_notification[:timestamp]
+        current_time = Time.now.to_i
+        current_state_duration = current_time - last_change
+        time_since_last_alert  = current_time - last_problem_alert unless last_problem_alert.nil?
 
-            current_state_duration   = current_time - last_change
-            time_since_last_alert    = current_time - last_problem_alert unless last_problem_alert.nil?
-            @logger.debug("Filter: Delays: last_problem_alert: #{last_problem_alert.to_s}, " +
-                       "last_change: #{last_change.inspect}, " +
-                       "current_state_duration: #{current_state_duration.inspect}, " +
-                       "time_since_last_alert: #{time_since_last_alert.inspect}, " +
-                       "last_alert_state: [#{last_alert_state.inspect}], " +
-                       "event.state: [#{event.state.inspect}], " +
-                       "last_alert_state == event.state ? #{last_alert_state.to_s == event.state}")
-            if (current_state_duration < failure_delay)
-              result = true
-              @logger.debug("Filter: Delays: blocking because duration of current failure " +
-                         "(#{current_state_duration}) is less than failure_delay (#{failure_delay})")
-            elsif !last_problem_alert.nil? && (time_since_last_alert < resend_delay) &&
-              (last_alert_state.to_s == event.state)
+        @logger.debug("#{label} last_problem_alert: #{last_problem_alert.to_s}, " +
+                      "last_change: #{last_change.inspect}, " +
+                      "current_state_duration: #{current_state_duration.inspect}, " +
+                      "time_since_last_alert: #{time_since_last_alert.inspect}, " +
+                      "last_alert_state: [#{last_alert_state.inspect}], " +
+                      "event.state: [#{event.state.inspect}], " +
+                      "last_alert_state == event.state ? #{last_alert_state.to_s == event.state}")
 
-              result = true
-              @logger.debug("Filter: Delays: blocking because time since last alert for " +
-                         "current problem (#{time_since_last_alert}) is less than " +
-                         "resend_delay (#{resend_delay}) and last alert state (#{last_alert_state}) " +
-                         "is equal to current event state (#{event.state})")
-            else
-              @logger.debug("Filter: Delays: not blocking because neither of the time comparison " +
-                         "conditions were met")
-            end
-          else
-            @logger.debug("Filter: Delays: entity_check.failed? returned false ...")
-          end
+        if current_state_duration < failure_delay
+          @logger.debug("#{label} block - duration of current failure " +
+                     "(#{current_state_duration}) is less than failure_delay (#{failure_delay})")
+          return true
         end
 
-        @logger.debug("Filter: Delays: #{result ? "block" : "pass"}")
-        result
+        if !last_problem_alert.nil? && (time_since_last_alert < resend_delay) &&
+          (last_alert_state.to_s == event.state)
+
+          @logger.debug("#{label} block - time since last alert for " +
+                        "current problem (#{time_since_last_alert}) is less than " +
+                        "resend_delay (#{resend_delay}) and last alert state (#{last_alert_state}) " +
+                        "is equal to current event state (#{event.state})")
+          return true
+        end
+
+        @logger.debug("#{label} pass - not blocking because neither of the time comparison " +
+                      "conditions were met")
+        return false
+
       end
     end
   end