flapjack 0.6.61 → 0.7.0

data/lib/flapjack/data/global.rb

@@ -8,6 +8,7 @@ module Flapjack
 
     class Global
 
+      # TODO maybe this should be an EntityCheck class method?
       def self.unacknowledged_failing_checks(options = {})
         raise "Redis connection not set" unless redis = options[:redis]
 

data/lib/flapjack/data/message.rb

@@ -17,6 +17,8 @@ module Flapjack
       def id
         return @id if @id
         t = Time.now
+        # FIXME: consider just using a UUID here
+        # this is planned to be used as part of alert history keys
         @id = self.object_id.to_i.to_s + '-' + t.to_i.to_s + '.' + t.tv_usec.to_s
       end
 

data/lib/flapjack/data/notification_rule.rb

@@ -0,0 +1,172 @@
+#!/usr/bin/env ruby
+
+require 'yajl/json_gem'
+require 'active_support/time'
+require 'ice_cube'
+require 'flapjack/utility'
+
+module Flapjack
+  module Data
+    class NotificationRule
+
+      extend Flapjack::Utility
+
+      attr_accessor :id, :contact_id, :entities, :entity_tags, :time_restrictions,
+        :warning_media, :critical_media, :warning_blackhole, :critical_blackhole
+
+      def self.exists_with_id?(rule_id, options = {})
+        raise "Redis connection not set" unless redis = options[:redis]
+        raise "No id value passed" unless not (rule_id.nil? || rule_id == '')
+        logger   = options[:logger]
+        redis.exists("notification_rule:#{rule_id}")
+      end
+
+      def self.find_by_id(rule_id, options = {})
+        raise "Redis connection not set" unless redis = options[:redis]
+        raise "No id value passed" unless not (rule_id.nil? || rule_id == '')
+        logger   = options[:logger]
+
+        # sanity check
+        return unless redis.exists("notification_rule:#{rule_id}")
+
+        rule = self.new({:id => rule_id}, {:redis => redis})
+        rule.refresh
+        rule
+      end
+
+      # replacing save! etc
+      def self.add(rule_data, options)
+        raise "Redis connection not set" unless redis = options[:redis]
+
+        rule_id = SecureRandom.uuid
+        self.add_or_update(rule_data.merge(:id => rule_id), :redis => redis)
+        self.find_by_id(rule_id, :redis => redis)
+      end
+
+      # add user's timezone string to the hash, deserialise
+      # time in the user's timezone also
+      def self.time_restriction_to_ice_cube_hash(tr, time_zone)
+        tr = symbolize(tr)
+
+        tr[:start_date] = tr[:start_time].dup
+        tr.delete(:start_time)
+
+        if tr[:start_date].is_a?(String)
+          tr[:start_date] = { :time => tr[:start_date] }
+        end
+        if tr[:start_date].is_a?(Hash)
+          tr[:start_date][:time] = time_zone.parse(tr[:start_date][:time])
+          tr[:start_date][:zone] = time_zone.name
+        end
+
+        if tr[:end_time].is_a?(String)
+          tr[:end_time] = { :time => tr[:end_time] }
+        end
+        if tr[:end_time].is_a?(Hash)
+          tr[:end_time][:time] = time_zone.parse(tr[:end_time][:time])
+          tr[:end_time][:zone] = time_zone.name
+        end
+
+        # rewrite Weekly to IceCube::WeeklyRule, etc
+        tr[:rrules].each {|rrule|
+          rrule[:rule_type] = "IceCube::#{rrule[:rule_type]}Rule"
+        }
+
+        tr
+      end
+
+      def self.time_restriction_from_ice_cube_hash(tr, time_zone)
+        tr[:start_date] = time_zone.utc_to_local(tr[:start_date][:time]).strftime "%Y-%m-%d %H:%M:%S"
+        tr[:end_time]   = time_zone.utc_to_local(tr[:end_time][:time]).strftime "%Y-%m-%d %H:%M:%S"
+
+        # rewrite IceCube::WeeklyRule to Weekly, etc
+        tr[:rrules].each {|rrule|
+          rrule[:rule_type] = /^.*\:\:(.*)Rule$/.match(rrule[:rule_type])[1]
+        }
+
+        tr[:start_time] = tr[:start_date].dup
+        tr.delete(:start_date)
+
+        tr
+      end
+
+      def refresh
+        rule_data = @redis.hgetall("notification_rule:#{@id}")
+
+        @contact_id         = rule_data['contact_id']
+        @entity_tags        = Yajl::Parser.parse(rule_data['entity_tags'] || '')
+        @entities           = Yajl::Parser.parse(rule_data['entities'] || '')
+        @time_restrictions  = Yajl::Parser.parse(rule_data['time_restrictions'] || '')
+        @warning_media      = Yajl::Parser.parse(rule_data['warning_media'] || '')
+        @critical_media     = Yajl::Parser.parse(rule_data['critical_media'] || '')
+        @warning_blackhole  = ((rule_data['warning_blackhole'] || 'false').downcase == 'true')
+        @critical_blackhole = ((rule_data['critical_blackhole'] || 'false').downcase == 'true')
+
+      end
+
+      def update(rule_data)
+        self.class.add_or_update(rule_data, :redis => @redis)
+        self.refresh
+      end
+
+      def to_json(*args)
+        hash = (Hash[ *([:id, :contact_id, :entity_tags, :entities,
+                 :time_restrictions, :warning_media, :critical_media,
+                 :warning_blackhole, :critical_blackhole].collect {|k|
+                   [k, self.send(k)]
+                 }).flatten(1) ])
+        hash.to_json
+      end
+
+      # tags or entity names match?
+      # nil @entity_tags and nil @entities matches
+      def match_entity?(event)
+        return true if (@entity_tags.nil? or @entity_tags.empty?) and
+                       (@entities.nil? or @entities.empty?)
+        return true if @entities.include?(event.split(':').first)
+        # TODO: return true if event's entity tags match entity tag list on the rule
+        return false
+      end
+
+      def blackhole?(severity)
+        return true if 'warning'.eql?(severity.downcase) and @warning_blackhole
+        return true if 'critical'.eql?(severity.downcase) and @critical_blackhole
+        return false
+      end
+
+      def media_for_severity(severity)
+        case severity
+        when 'warning'
+          media_list = @warning_media
+        when 'critical'
+          media_list = @critical_media
+        end
+        media_list
+      end
+
+    private
+
+      def initialize(rule_data, opts = {})
+        @redis  ||= opts[:redis]
+        @logger   = opts[:logger]
+        raise "a redis connection must be supplied" unless @redis
+        @id = rule_data[:id]
+      end
+
+      def self.add_or_update(rule_data, options = {})
+        redis = options[:redis]
+
+        rule_data[:entities]          = Yajl::Encoder.encode(rule_data[:entities])
+        rule_data[:entity_tags]       = Yajl::Encoder.encode(rule_data[:entity_tags])
+        rule_data[:time_restrictions] = Yajl::Encoder.encode(rule_data[:time_restrictions])
+        rule_data[:warning_media]     = Yajl::Encoder.encode(rule_data[:warning_media])
+        rule_data[:critical_media]    = Yajl::Encoder.encode(rule_data[:critical_media])
+
+        redis.sadd("contact_notification_rules:#{rule_data[:contact_id]}", rule_data[:id])
+        redis.hmset("notification_rule:#{rule_data[:id]}", *rule_data.flatten)
+      end
+
+    end
+  end
+end
+

data/lib/flapjack/data/tag.rb

@@ -10,9 +10,10 @@ module Flapjack
 
       attr_accessor :name
 
-      def initialize(opts)
+      def initialize(opts = {})
+        raise "Redis connection not set" unless @redis = opts[:redis]
+
         @name  = opts[:name]
-        @redis = opts[:redis]
         preset = @redis.smembers(@name)
         enum = opts[:create] || []
         @redis.sadd(@name, enum) unless enum.empty?
@@ -45,6 +46,10 @@ module Flapjack
         super(o)
       end
 
+      def to_json(*a)
+        self.to_a.to_json(*a)
+      end
+
     end
   end
 end

data/lib/flapjack/data/tag_set.rb

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+
+require 'set'
+
+module Flapjack
+  module Data
+    class TagSet < ::Set
+
+      def to_json(*a)
+        self.to_a.to_json(*a)
+      end
+
+    end
+  end
+end
+

data/lib/flapjack/executive.rb

@@ -2,6 +2,8 @@
 
 require 'log4r'
 require 'log4r/outputter/fileoutputter'
+require 'tzinfo'
+require 'active_support/time'
 
 require 'flapjack/filters/acknowledgement'
 require 'flapjack/filters/ok'
@@ -14,6 +16,7 @@ require 'flapjack/data/entity_check'
 require 'flapjack/data/notification'
 require 'flapjack/data/event'
 require 'flapjack/redis_pool'
+require 'flapjack/utility'
 
 require 'flapjack/gateways/email'
 require 'flapjack/gateways/sms_messagenet'
@@ -22,6 +25,8 @@ module Flapjack
 
   class Executive
 
+    include Flapjack::Utility
+
     def initialize(opts = {})
       @config = opts[:config]
       @redis_config = opts[:redis_config]
@@ -34,9 +39,24 @@ module Flapjack
                  :pagerduty => @config['pagerduty_queue']}
 
       notifylog  = @config['notification_log_file'] || 'log/notify.log'
+      if not File.directory?(File.dirname(notifylog))
+        puts "Parent directory for log file #{notifylog} doesn't exist"
+        puts "Exiting!"
+        exit
+      end
       @notifylog = Log4r::Logger.new("executive")
       @notifylog.add(Log4r::FileOutputter.new("notifylog", :filename => notifylog))
 
+      tz = nil
+      tz_string = @config['default_contact_timezone'] || ENV['TZ'] || 'UTC'
+      begin
+        tz = ActiveSupport::TimeZone.new(tz_string)
+      rescue ArgumentError
+        logger.error("Invalid timezone string specified in default_contact_timezone or TZ (#{tz_string})")
+        exit 1
+      end
+      @default_contact_timezone = tz
+
       # FIXME: Put loading filters into separate method
       # FIXME: should we make the filters more configurable by the end user?
       options = { :log => opts[:logger], :persistence => @redis }
@@ -128,7 +148,7 @@ module Flapjack
       end
 
       @logger.info("Generating notifications for event #{event.id}, #{event.type}, #{event.state}, #{event.summary}#{time_at_str}")
-      send_notification_messages(event, entity_check)
+      generate_notification_messages(event, entity_check)
     end
 
     def update_keys(event, entity_check)
@@ -198,8 +218,8 @@ module Flapjack
     end
 
     # takes an event for which a notification needs to be generated, works out the type of
-    # notification, updates the notification history in redis, sends the notifications
-    def send_notification_messages(event, entity_check)
+    # notification, updates the notification history in redis, generates the notifications
+    def generate_notification_messages(event, entity_check)
       timestamp = Time.now.to_i
       notification_type = 'unknown'
       case event.type
@@ -219,7 +239,9 @@ module Flapjack
         end
       end
       @redis.set("#{event.id}:last_#{notification_type}_notification", timestamp)
+      @redis.set("#{event.id}:last_#{event.state}_notification", timestamp) if event.failure?
       @redis.rpush("#{event.id}:#{notification_type}_notifications", timestamp)
+      @redis.rpush("#{event.id}:#{event.state}_notifications", timestamp) if event.failure?
       @logger.debug("Notification of type #{notification_type} is being generated for #{event.id}.")
 
       contacts = entity_check.contacts
@@ -231,21 +253,133 @@ module Flapjack
 
       notification = Flapjack::Data::Notification.for_event(event, :type => notification_type)
 
-      notification.messages(:contacts => contacts).each do |msg|
-        media_type = msg.medium.to_sym
+      enqueue_messages( apply_notification_rules( notification.messages(:contacts => contacts) ) )
+
+    end
+
+    # time restrictions match?
+    # nil rule.time_restrictions matches
+    # times (start, end) within time restrictions will have any UTC offset removed and will be
+    # considered to be in the timezone of the contact
+    def rule_occurring_now?(rule, opts)
+      contact = opts[:contact]
+      return true if rule.time_restrictions.nil? or rule.time_restrictions.empty?
+
+      time_zone = contact.timezone(:default => @default_contact_timezone)
+      usertime = time_zone.now
+
+      match = rule.time_restrictions.any? do |tr|
+        # add contact's timezone to the time restriction hash
+        tr = Flapjack::Data::NotificationRule.time_restriction_to_ice_cube_hash(tr, time_zone)
+
+        schedule = IceCube::Schedule.from_hash(tr)
+        schedule.occurring_at?(usertime)
+      end
+      !!match
+    end
+
+    # delete messages based on entity name(s), tags, severity, time of day
+    def apply_notification_rules(messages)
+      # first get all rules matching entity and time
+      @logger.debug "apply_notification_rules: got messages with size #{messages.size}"
+
+      # don't consider notification rules if the contact has none
+
+      tuple = messages.map do |message|
+        @logger.debug "considering message: #{message.medium} #{message.notification.event.id} #{message.notification.event.state}"
+        @logger.debug "contact_id: #{message.contact.id}"
+        rules    = message.contact.notification_rules
+        @logger.debug "found #{rules.length} rules for this message's contact"
+        event_id = message.notification.event.id
+        options  = {}
+        options[:no_rules_for_contact] = true if rules.empty?
+        # filter based on entity, tags, severity, time of day
+        matchers = rules.find_all do |rule|
+          rule.match_entity?(event_id) && rule_occurring_now?(rule, :contact => message.contact)
+        end
+        [message, matchers, options]
+      end
+
+      # matchers are rules of the contact that have matched the current event
+      # for time and entity
+
+      @logger.debug "apply_notification_rules: num messages after entity and time matching: #{tuple.size}"
+
+      # delete the matcher for all entities if there are more specific matchers
+      tuple = tuple.map do |message, matchers, options|
+        if matchers.length > 1
+          have_specific = matchers.detect do |matcher|
+            matcher.entities or matcher.entity_tags
+          end
+          if have_specific
+            # delete the rule for all entities
+            matchers.map! do |matcher|
+              matcher.entities.nil? and matcher.entity_tags.nil? ? nil : matcher
+            end
+          end
+        end
+        [message, matchers, options]
+      end
+
+      # delete media based on blackholes
+      tuple = tuple.find_all do |message, matchers, options|
+        severity = message.notification.event.state
+        # or use message.notification.contents['state']
+        matchers.none? {|matcher| matcher.blackhole?(severity) }
+      end
+
+      @logger.debug "apply_notification_rules: num messages after removing blackhole matches: #{tuple.size}"
+
+      # delete any media that doesn't meet severity<->media constraints
+      tuple = tuple.find_all do |message, matchers, options|
+        severity = message.notification.event.state
+        options[:no_rules_for_contact] ||
+          matchers.any? {|matcher|
+            matcher.media_for_severity(severity).include?(message.medium) ||
+              (@logger.warn("got nil for matcher.media_for_severity(#{severity}), matcher: #{matcher.inspect}") && false)
+          }
+      end
+
+      @logger.debug "apply_notification_rules: num messages after severity-media constraints: #{tuple.size}"
+
+      # delete media based on notification interval
+      tuple = tuple.find_all do |message, matchers, options|
+        not message.contact.drop_notifications?(:media => message.medium,
+                                                :check => message.notification.event.id,
+                                                :state => message.notification.event.state)
+      end
+
+      @logger.debug "apply_notification_rules: num messages after pruning for notification intervals: #{tuple.size}"
 
-        @notifylog.info("#{Time.now.to_s} | #{event.id} | " +
-          "#{notification_type} | #{msg.contact.id} | #{media_type.to_s} | #{msg.address}")
+      tuple.map do |message, matchers, options|
+        message
+      end
+    end
+
+    def enqueue_messages(messages)
+
+      messages.each do |message|
+        media_type = message.medium
+        contents   = message.contents
+        event_id   = message.notification.event.id
+
+        @notifylog.info("#{Time.now.to_s} | #{event_id} | " +
+          "#{message.notification.type} | #{message.contact.id} | #{media_type} | #{message.address}")
 
-        unless @queues[media_type]
+        unless @queues[media_type.to_sym]
           @logger.error("no queue for media type: #{media_type}")
-          next
+          return
         end
 
-        contents = msg.contents
+        @logger.info("Enqueueing #{media_type} alert for #{event_id} to #{message.address}")
+
+        message.contact.update_sent_alert_keys(:media => message.medium,
+                                               :check => message.notification.event.id,
+                                               :state => message.notification.event.state)
+          # drop_alerts_for_contact:#{self.id}:#{media}:#{check}:#{state}
 
         # TODO consider changing Resque jobs to use raw blpop like the others
-        case media_type
+        case media_type.to_sym
         when :sms
           Resque.enqueue_to(@queues[:sms], Flapjack::Gateways::SmsMessagenet, contents)
         when :email
@@ -257,9 +391,9 @@ module Flapjack
         when :pagerduty
           @redis.rpush(@queues[:pagerduty], Yajl::Encoder.encode(contents))
         end
-
       end
-    end
+
+   end
 
   end
 end