flapjack 0.7.14 → 0.7.15

data/lib/flapjack/data/event.rb

@@ -8,6 +8,8 @@ module Flapjack
 
       attr_accessor :previous_state, :previous_state_duration
 
+      attr_reader :check, :summary, :details, :acknowledgement_id
+
       # Helper method for getting the next event.
       #
       # Has a blocking and non-blocking method signature.
@@ -25,9 +27,6 @@ module Flapjack
                      :archive_events => false,
                      :events_archive_maxage => (3 * 60 * 60) }
         options  = defaults.merge(opts)
-        if options[:logger]
-          logger = options[:logger]
-        end
 
         if options[:archive_events]
           dest = "events_archive:#{Time.now.utc.strftime "%Y%m%d%H"}"
@@ -49,10 +48,12 @@ module Flapjack
         begin
           parsed = ::JSON.parse( raw )
         rescue => e
-          logger.warn("Error deserialising event json: #{e}, raw json: #{raw.inspect}")
+          if options[:logger]
+            options[:logger].warn("Error deserialising event json: #{e}, raw json: #{raw.inspect}")
+          end
           return nil
         end
-        return self.new( parsed )
+        self.new( parsed )
       end
 
       # creates, or modifies, an event object and adds it to the events list in redis
@@ -74,38 +75,55 @@ module Flapjack
         redis.llen('events')
       end
 
-      def initialize(attrs={})
-        @attrs = attrs
-        @attrs['time'] = Time.now.to_i unless @attrs.has_key?('time')
+      def self.create_acknowledgement(entity_name, check, opts = {})
+        data = { 'type'               => 'action',
+                 'state'              => 'acknowledgement',
+                 'entity'             => entity_name,
+                 'check'              => check,
+                 'summary'            => opts[:summary],
+                 'duration'           => opts[:duration],
+                 'acknowledgement_id' => opts[:acknowledgement_id]
+               }
+        add(data, :redis => opts[:redis])
+      end
+
+      def self.test_notifications(entity_name, check, opts = {})
+        data = { 'type'               => 'action',
+                 'state'              => 'test_notifications',
+                 'entity'             => entity_name,
+                 'check'              => check,
+                 'summary'            => opts[:summary],
+                 'details'            => opts[:details]
+               }
+        add(data, :redis => opts[:redis])
+      end
+
+      def initialize(attrs = {})
+        ['type', 'state', 'entity', 'check', 'time', 'summary', 'details',
+         'acknowledgement_id', 'duration'].each do |key|
+          instance_variable_set("@#{key}", attrs[key])
+        end
       end
 
       def state
-        return unless @attrs['state']
-        @attrs['state'].downcase
+        return unless @state
+        @state.downcase
       end
 
       def entity
-        return unless @attrs['entity']
-        @attrs['entity'].downcase
-      end
-
-      def check
-        @attrs['check']
-      end
-
-
-      # FIXME some values are only set for certain event types --
-      # this may not be the best way to do this
-      def acknowledgement_id
-        @attrs['acknowledgement_id']
+        return unless @entity
+        @entity.downcase
       end
 
       def duration
-        return unless @attrs['duration']
-        @attrs['duration'].to_i
+        return unless @duration
+        @duration.to_i
       end
-      # end FIXME
 
+      def time
+        return unless @time
+        @time.to_i
+      end
 
       def id
         (entity || '-') + ':' + (check || '-')
@@ -118,25 +136,8 @@ module Flapjack
       end
 
       def type
-        return unless @attrs['type']
-        @attrs['type'].downcase
-      end
-
-      def summary
-        @attrs['summary']
-      end
-
-      def details
-        @attrs['details']
-      end
-
-      def time
-        return unless @attrs['time']
-        @attrs['time'].to_i
-      end
-
-      def action?
-        type == 'action'
+        return unless @type
+        @type.downcase
       end
 
       def service?
@@ -144,37 +145,25 @@ module Flapjack
       end
 
       def acknowledgement?
-        action? and state == 'acknowledgement'
+        (type == 'action') && (state == 'acknowledgement')
       end
 
       def test_notifications?
-        action? and state == 'test_notifications'
+        (type == 'action') && (state == 'test_notifications')
       end
 
       def ok?
-        (state == 'ok') or (state == 'up')
-      end
-
-      def unknown?
-        state == 'unknown'
-      end
-
-      def unreachable?
-        state == 'unreachable'
-      end
-
-      def warning?
-        state == 'warning'
-      end
-
-      def critical?
-        state == 'critical'
+        (state == 'ok') || (state == 'up')
       end
 
       def failure?
-        warning? or critical? or unknown?
+        ['critical', 'warning', 'unknown'].include?(state)
       end
 
+      # # Not used anywhere
+      # def unreachable?
+      #   state == 'unreachable'
+      # end
     end
   end
 end

data/lib/flapjack/data/notification.rb

@@ -9,7 +9,7 @@ module Flapjack
     class Notification
 
       attr_reader :event, :type, :max_notified_severity, :contacts,
-        :default_timezone
+        :default_timezone, :last_state
 
       def self.for_event(event, opts = {})
         self.new(:event => event,
@@ -17,6 +17,7 @@ module Flapjack
                  :max_notified_severity => opts[:max_notified_severity],
                  :contacts => opts[:contacts],
                  :default_timezone => opts[:default_timezone],
+                 :last_state => opts[:last_state],
                  :logger => opts[:logger])
       end
 
@@ -37,6 +38,8 @@ module Flapjack
         contents = {'event_id'              => event_id,
                     'state'                 => event_state,
                     'summary'               => event.summary,
+                    'last_state'            => @last_state ? @last_state[:state] : nil,
+                    'last_summary'          => @last_state ? @last_state[:summary] : nil,
                     'details'               => event.details,
                     'time'                  => event.time,
                     'duration'              => event.duration || nil,
@@ -122,6 +125,7 @@ module Flapjack
         @max_notified_severity = opts[:max_notified_severity]
         @contacts = opts[:contacts]
         @default_timezone = opts[:default_timezone]
+        @last_state = opts[:last_state]
         @logger = opts[:logger]
       end
 

data/lib/flapjack/executive.rb

@@ -5,6 +5,9 @@ require 'log4r/outputter/fileoutputter'
 require 'tzinfo'
 require 'active_support/time'
 
+require 'chronic'
+require 'chronic_duration'
+
 require 'flapjack/filters/acknowledgement'
 require 'flapjack/filters/ok'
 require 'flapjack/filters/scheduled_maintenance'
@@ -60,9 +63,12 @@ module Flapjack
       @archive_events        = @config['archive_events'] || false
       @events_archive_maxage = @config['events_archive_maxage']
 
+      ncsm_duration_conf = @config['new_check_scheduled_maintenance_duration'] || '100 years'
+      @ncsm_duration = ChronicDuration.parse(ncsm_duration_conf)
+
       # FIXME: Put loading filters into separate method
       # FIXME: should we make the filters more configurable by the end user?
-      options = { :log => opts[:logger], :persistence => @redis }
+      options = { :logger => opts[:logger], :redis => @redis }
       @filters = []
       @filters << Flapjack::Filters::Ok.new(options)
       @filters << Flapjack::Filters::ScheduledMaintenance.new(options)
@@ -71,7 +77,7 @@ module Flapjack
       @filters << Flapjack::Filters::Delays.new(options)
       @filters << Flapjack::Filters::Acknowledgement.new(options)
 
-      @boot_time    = Time.now
+      @boot_time    = opts[:boot_time]
       @fqdn         = `/bin/hostname -f`.chomp
       @pid          = Process.pid
       @instance_id  = "#{@fqdn}:#{@pid}"
@@ -142,41 +148,35 @@ module Flapjack
       pending = Flapjack::Data::Event.pending_count(:redis => @redis)
       @logger.debug("#{pending} events waiting on the queue")
       @logger.debug("Raw event received: #{event.inspect}")
-      time_at = event.time
-      time_at_str = time_at ? ", #{Time.at(time_at).to_s}" : ''
-      @logger.debug("Processing Event: #{event.id}, #{event.type}, #{event.state}, #{event.summary}#{time_at_str}")
+      return if ('shutdown' == event.type)
 
-      entity_check = ('shutdown' == event.type) ? nil :
-                       Flapjack::Data::EntityCheck.for_event_id(event.id, :redis => @redis)
+      event_str = "#{event.id}, #{event.type}, #{event.state}, #{event.summary}"
+      event_str << ", #{Time.at(event.time).to_s}" if event.time
+      @logger.debug("Processing Event: #{event_str}")
 
-      result = update_keys(event, entity_check)
-      return if result[:shutdown]
+      entity_check = Flapjack::Data::EntityCheck.for_event_id(event.id, :redis => @redis)
+      timestamp = Time.now.to_i
 
-      blocker = nil
+      should_notify = update_keys(event, entity_check, timestamp)
 
-      if result[:skip_filters]
+      if !should_notify
         @logger.debug("Not generating notifications for event #{event.id} because filtering was skipped")
         return
-      else
-        blocker = @filters.find {|filter| filter.block?(event) }
-      end
-
-      if blocker
+      elsif blocker = @filters.find {|filter| filter.block?(event) }
         @logger.debug("Not generating notifications for event #{event.id} because this filter blocked: #{blocker.name}")
         return
       end
 
-      @logger.info("Generating notifications for event #{event.id}, #{event.type}, #{event.state}, #{event.summary}#{time_at_str}")
-      generate_notification_messages(event, entity_check)
+      @logger.info("Generating notifications for event #{event_str}")
+      generate_notification_messages(event, entity_check, timestamp)
     end
 
-    def update_keys(event, entity_check)
-
+    def update_keys(event, entity_check, timestamp)
       # TODO: run touch_keys from a separate EM timer for efficiency
       touch_keys
 
-      result    = { :skip_filters => false }
-      timestamp = Time.now.to_i
+      result = true
+
       @event_count = @redis.hincrby('event_counters', 'all', 1)
       @redis.hincrby("event_counters:#{@instance_id}", 'all', 1)
 
@@ -201,23 +201,30 @@ module Flapjack
         end
 
         event.previous_state = entity_check.state
-        event.previous_state_duration = Time.now.to_i - entity_check.last_change.to_i
-        @logger.info("No previous state for event #{event.id}") if event.previous_state.nil?
-
-        # If there is a state change, update record with: the time, the new state
-        if event.state != event.previous_state
-          entity_check.update_state(event.state, :timestamp => timestamp,
-            :summary => event.summary, :client => event.client,
-            :count => @event_count, :details => event.details)
+
+        if event.previous_state.nil?
+          @logger.info("No previous state for event #{event.id}")
+
+          if @ncsm_duration >= 0
+            @logger.info("Setting scheduled maintenance for #{time_period_in_words(@ncsm_duration)}")
+            entity_check.create_scheduled_maintenance(:start_time => timestamp,
+              :duration => @ncsm_duration, :summary => 'Automatically created for new check')
+          end
+        else
+          event.previous_state_duration = timestamp - entity_check.last_change.to_i
         end
 
+        entity_check.update_state(event.state, :timestamp => timestamp,
+          :summary => event.summary, :client => event.client,
+          :count => @event_count, :details => event.details)
+
         # No state change, and event is ok, so no need to run through filters
         # OR
         # If the service event's state is ok and there was no previous state, don't alert.
         # This stops new checks from alerting as "recovery" after they have been added.
         if !event.previous_state && event.ok?
           @logger.debug("setting skip_filters to true because there was no previous state and event is ok")
-          result[:skip_filters] = true
+          result = false
         end
 
         entity_check.update_current_scheduled_maintenance
@@ -232,9 +239,6 @@ module Flapjack
         if event.acknowledgement? && event.acknowledgement_id
           @redis.hdel('unacknowledged_failures', event.acknowledgement_id)
         end
-      when 'shutdown'
-        # should this be logged as an action instead? being minimally invasive for now
-        result[:shutdown] = true
       end
 
       result
@@ -242,8 +246,7 @@ module Flapjack
 
     # takes an event for which a notification needs to be generated, works out the type of
     # notification, updates the notification history in redis, generates the notifications
-    def generate_notification_messages(event, entity_check)
-      timestamp = Time.now.to_i
+    def generate_notification_messages(event, entity_check, timestamp)
       notification_type = 'unknown'
       case event.type
       when 'service'
@@ -274,7 +277,7 @@ module Flapjack
 
       if contacts.empty?
         @logger.debug("No contacts for #{event.id}")
-        @notifylog.info("#{Time.now.to_s} | #{event.id} | #{notification_type} | NO CONTACTS")
+        @notifylog.info("#{Time.at(timestamp).to_s} | #{event.id} | #{notification_type} | NO CONTACTS")
         return
       end
 
@@ -283,6 +286,7 @@ module Flapjack
                :max_notified_severity => max_notified_severity,
                :contacts => contacts,
                :default_timezone => @default_contact_timezone,
+               :last_state => entity_check.historical_state_before(timestamp),
                :logger => @logger)
 
       notification.messages.each do |message|
@@ -291,7 +295,7 @@ module Flapjack
         address    = message.address
         event_id   = event.id
 
-        @notifylog.info("#{Time.now.to_s} | #{event_id} | " +
+        @notifylog.info("#{Time.at(timestamp).to_s} | #{event_id} | " +
           "#{notification_type} | #{message.contact.id} | #{media_type} | #{address}")
 
         unless @queues[media_type.to_sym]

data/lib/flapjack/filters/acknowledgement.rb

@@ -16,10 +16,10 @@ module Flapjack
         result = false
         if event.type == 'action'
           if event.acknowledgement?
-            if @persistence.zscore("failed_checks", event.id)
-              ec = Flapjack::Data::EntityCheck.for_event_id(event.id, :redis => @persistence)
+            if @redis.zscore("failed_checks", event.id)
+              ec = Flapjack::Data::EntityCheck.for_event_id(event.id, :redis => @redis)
               if ec.nil?
-                @log.error "Filter: Acknowledgement: unknown entity for event '#{event.id}'"
+                @logger.error "Filter: Acknowledgement: unknown entity for event '#{event.id}'"
               else
                 ec.create_unscheduled_maintenance(:start_time => timestamp,
                   :duration => (event.duration || (4 * 60 * 60)),
@@ -28,14 +28,14 @@ module Flapjack
               end
             else
               result = true
-              @log.debug("Filter: Acknowledgement: blocking because zscore of failed_checks for #{event.id} is false") unless @persistence.zscore("failed_checks", event.id)
+              @logger.debug("Filter: Acknowledgement: blocking because zscore of failed_checks for #{event.id} is false") unless @redis.zscore("failed_checks", event.id)
             end
           else
             message = "no action taken"
             result  = false
           end
         end
-        @log.debug("Filter: Acknowledgement: #{result ? "block" : "pass"} (#{message})")
+        @logger.debug("Filter: Acknowledgement: #{result ? "block" : "pass"} (#{message})")
         result
       end
     end

data/lib/flapjack/filters/base.rb

@@ -4,8 +4,8 @@ module Flapjack
   module Filters
     module Base
       def initialize(opts={})
-        @log         = opts[:log]
-        @persistence = opts[:persistence]
+        @logger = opts[:logger]
+        @redis  = opts[:redis]
       end
 
       def name

data/lib/flapjack/filters/delays.rb

@@ -25,15 +25,15 @@ module Flapjack
 
         result = false
 
-        if (event.type == 'service') and (event.failure?)
+        if event.service? && event.failure?
 
-          entity_check = Flapjack::Data::EntityCheck.for_event_id(event.id, :redis => @persistence)
+          entity_check = Flapjack::Data::EntityCheck.for_event_id(event.id, :redis => @redis)
           current_time = Time.now.to_i
 
           if entity_check.failed?
-            last_problem_alert   = entity_check.last_problem_notification
-            last_warning_alert   = entity_check.last_warning_notification
-            last_critical_alert  = entity_check.last_critical_notification
+            last_problem_alert   = entity_check.last_notification_for_state(:problem)[:timestamp]
+            last_warning_alert   = entity_check.last_notification_for_state(:warning)[:timestamp]
+            last_critical_alert  = entity_check.last_notification_for_state(:critical)[:timestamp]
             last_change          = entity_check.last_change
             last_notification    = entity_check.last_notification
             last_alert_state     = last_notification[:type]
@@ -41,7 +41,7 @@ module Flapjack
 
             current_state_duration   = current_time - last_change
             time_since_last_alert    = current_time - last_problem_alert unless last_problem_alert.nil?
-            @log.debug("Filter: Delays: last_problem_alert: #{last_problem_alert.to_s}, " +
+            @logger.debug("Filter: Delays: last_problem_alert: #{last_problem_alert.to_s}, " +
                        "last_change: #{last_change.inspect}, " +
                        "current_state_duration: #{current_state_duration.inspect}, " +
                        "time_since_last_alert: #{time_since_last_alert.inspect}, " +
@@ -50,26 +50,26 @@ module Flapjack
                        "last_alert_state == event.state ? #{last_alert_state.to_s == event.state}")
             if (current_state_duration < failure_delay)
               result = true
-              @log.debug("Filter: Delays: blocking because duration of current failure " +
+              @logger.debug("Filter: Delays: blocking because duration of current failure " +
                          "(#{current_state_duration}) is less than failure_delay (#{failure_delay})")
             elsif !last_problem_alert.nil? && (time_since_last_alert < resend_delay) &&
               (last_alert_state.to_s == event.state)
 
               result = true
-              @log.debug("Filter: Delays: blocking because time since last alert for " +
+              @logger.debug("Filter: Delays: blocking because time since last alert for " +
                          "current problem (#{time_since_last_alert}) is less than " +
                          "resend_delay (#{resend_delay}) and last alert state (#{last_alert_state}) " +
                          "is equal to current event state (#{event.state})")
             else
-              @log.debug("Filter: Delays: not blocking because neither of the time comparison " +
+              @logger.debug("Filter: Delays: not blocking because neither of the time comparison " +
                          "conditions were met")
             end
           else
-            @log.debug("Filter: Delays: entity_check.failed? returned false ...")
+            @logger.debug("Filter: Delays: entity_check.failed? returned false ...")
           end
         end
 
-        @log.debug("Filter: Delays: #{result ? "block" : "pass"}")
+        @logger.debug("Filter: Delays: #{result ? "block" : "pass"}")
         result
       end
     end