flapjack 0.6.39 → 0.6.40

data/lib/flapjack/data/entity_check.rb

@@ -74,30 +74,26 @@ module Flapjack
         }
       end
 
-      # creates, or modifies, an event object and adds it to the events list in redis
-      #   'type'      => 'service',
-      #   'state'     => state,
-      #   'summary'   => check_output,
-      #   'time'      => timestamp
-      def create_event(event)
-        event.merge!('entity' => @entity.name, 'check' => @check)
-        event['time'] = Time.now.to_i if event['time'].nil?
-        @redis.rpush('events', Yajl::Encoder.encode(event))
-      end
-
-      def create_acknowledgement(opts = {})
-        defaults = {
-          'summary' => '...'
-        }
-        options = defaults.merge(opts)
-
+      def create_acknowledgement(options = {})
         event = { 'type'               => 'action',
                   'state'              => 'acknowledgement',
                   'summary'            => options['summary'],
                   'duration'           => options['duration'],
-                  'acknowledgement_id' => options['acknowledgement_id']
+                  'acknowledgement_id' => options['acknowledgement_id'],
+                  'entity'             => @entity.name,
+                  'check'              => @check
+                }
+        Flapjack::Data::Event.add(event, :redis => @redis)
+      end
+
+      def test_notifications(options = {})
+        event = { 'type'               => 'action',
+                  'state'              => 'test_notifications',
+                  'summary'            => options['summary'],
+                  'entity'             => @entity.name,
+                  'check'              => @check
                 }
-        create_event(event)
+        Flapjack::Data::Event.add(event, :redis => @redis)
       end
 
       # FIXME: need to add summary to summary of existing unscheduled maintenance if there is
@@ -425,8 +421,10 @@ module Flapjack
         check  = @check
 
         if @logger
-          @logger.debug("contacts for #{@entity.id} (#{@entity.name}): " + @redis.smembers("contacts_for:#{@entity.id}").length.to_s)
-          @logger.debug("contacts for #{check}: " + @redis.smembers("contacts_for:#{check}").length.to_s)
+          @logger.debug("contacts for #{@entity.id} (#{@entity.name}): " +
+            @redis.smembers("contacts_for:#{@entity.id}").length.to_s)
+          @logger.debug("contacts for #{check}: " +
+            @redis.smembers("contacts_for:#{check}").length.to_s)
         end
 
         union = @redis.sunion("contacts_for:#{@entity.id}", "contacts_for:#{check}")

data/lib/flapjack/data/event.rb

@@ -17,43 +17,38 @@ module Flapjack
       # events on the queue.
       #
       def self.next(opts={})
+        raise "Redis connection not set" unless redis = opts[:redis]
+
         defaults = { :block => true }
         options  = defaults.merge(opts)
-        block    = options[:block]
 
         # In production, we wait indefinitely for events coming from other systems.
-        if block
-          raw   = opts[:persistence].blpop('events', 0).last
-          event = ::JSON.parse(raw)
-          self.new(event)
-        else
-          # In testing, we take care that there are no events on the queue.
-          raw    = opts[:persistence].lpop('events')
-          result = nil
-
-          if raw
-            event  = ::JSON.parse(raw)
-            result = self.new(event)
-          end
-
-          result
+        if options[:block]
+          return self.new( ::JSON.parse( redis.blpop('events', 0).last ) )
         end
+
+        # In testing, we take care that there are no events on the queue.
+        return unless raw = redis.lpop('events')
+        self.new( ::JSON.parse(raw) )
+      end
+
+      # creates, or modifies, an event object and adds it to the events list in redis
+      #   'type'      => 'service',
+      #   'state'     => state,
+      #   'summary'   => check_output,
+      #   'time'      => timestamp
+      def self.add(evt, opts = {})
+        raise "Redis connection not set" unless redis = opts[:redis]
+
+        evt['time'] = Time.now.to_i if evt['time'].nil?
+        redis.rpush('events', Yajl::Encoder.encode(evt))
       end
 
       # Provide a count of the number of events on the queue to be processed.
       def self.pending_count(opts = {})
-        opts[:persistence].llen('events')
-      end
+        raise "Redis connection not set" unless redis = opts[:redis]
 
-      # FIXME make this use a logger taken from the opts
-      def self.purge_all(opts = {})
-        events_size = opts[:redis].llen('events')
-        puts "purging #{events_size} events..."
-        timestamp = Time.now.to_i
-        puts "renaming events to events.#{timestamp}"
-        opts[:redis].rename('events', "events.#{timestamp}")
-        puts "setting expiry of events.#{timestamp} to 8 hours"
-        opts[:redis].expire("events.#{timestamp}", (60 * 60 * 8))
+        redis.llen('events')
       end
 
       def initialize(attrs={})
@@ -125,6 +120,10 @@ module Flapjack
         action? and state == 'acknowledgement'
       end
 
+      def test_notifications?
+        action? and state == 'test_notifications'
+      end
+
       def ok?
         (state == 'ok') or (state == 'up')
       end

data/lib/flapjack/data/message.rb

@@ -0,0 +1,45 @@
+#!/usr/bin/env ruby
+
+# 'Notification' refers to the template object created when an event occurs,
+# from which individual 'Message' objects are created, one for each
+# contact+media recipient.
+
+module Flapjack
+  module Data
+    class Message
+
+      attr_accessor :medium, :address, :id, :duration, :contact, :notification
+
+      def self.for_contact(opts = {})
+        self.new(:contact => opts[:contact])
+      end
+
+      def id
+        return @id if @id
+        t = Time.now
+        @id = self.object_id.to_i.to_s + '-' + t.to_i.to_s + '.' + t.tv_usec.to_s
+      end
+
+      def contents
+        c = {'media'              => medium,
+             'address'            => address,
+             'id'                 => id}
+        if contact
+          c.merge('contact_id'         => contact.id,
+                  'contact_first_name' => contact.first_name,
+                  'contact_last_name'  => contact.last_name)
+        end
+        c['duration'] = duration if duration
+        c.merge(notification.contents) if notification
+      end
+
+    private
+
+      def initialize(opts = {})
+        @contact = opts[:contact]
+      end
+
+    end
+  end
+end
+

data/lib/flapjack/data/notification.rb

@@ -0,0 +1,49 @@
+#!/usr/bin/env ruby
+
+require 'flapjack/data/message'
+
+module Flapjack
+  module Data
+    class Notification
+
+      attr_accessor :event, :type
+
+      def self.for_event(event, opts = {})
+        self.new(:event => event, :type => opts[:type])
+      end
+
+      def messages(opts = {})
+        contacts = opts[:contacts]
+        return [] if contacts.nil?
+        @messages ||= contacts.collect {|contact|
+          contact.media.keys.inject([]) { |ret, mk|
+            m = Flapjack::Data::Message.for_contact(:contact => contact)
+            m.notification = self
+            m.medium  = mk
+            m.address = contact.media[mk]
+            ret << m
+            ret
+          }
+        }.flatten
+      end
+
+      def contents
+        @contents ||= {'event_id'          => event.id,
+                       'state'             => event.state,
+                       'summary'           => event.summary,
+                       'time'              => event.time,
+                       'notification_type' => type}
+      end
+
+    private
+
+      def initialize(opts = {})
+        raise "Event not passed" unless event = opts[:event]
+        @event = event
+        @type  = opts[:type]
+      end
+
+    end
+  end
+end
+

data/lib/flapjack/executive.rb

@@ -12,19 +12,25 @@ require 'flapjack/filters/detect_mass_client_failures'
 require 'flapjack/filters/delays'
 require 'flapjack/data/contact'
 require 'flapjack/data/entity_check'
+require 'flapjack/data/notification'
 require 'flapjack/data/event'
-require 'flapjack/notification/common'
 require 'flapjack/notification/sms'
 require 'flapjack/notification/email'
 require 'flapjack/pikelet'
+require 'flapjack/redis_pool'
 
 module Flapjack
 
   class Executive
-    include Flapjack::Pikelet
+    include Flapjack::GenericPikelet
 
-    def setup
-      @redis = build_redis_connection_pool
+    alias_method :generic_bootstrap, :bootstrap
+    alias_method :generic_cleanup,   :cleanup
+
+    def bootstrap(opts = {})
+      generic_bootstrap(opts)
+
+      @redis = Flapjack::RedisPool.new(:config => opts[:redis_config], :size => 1)
 
       @queues = {:email     => @config['email_queue'],
                  :sms       => @config['sms_queue'],
@@ -73,19 +79,20 @@ module Flapjack
       @redis.hset("event_counters:#{@instance_id}", 'action', 0)
     end
 
-    def main
-      setup
+    def cleanup
+      @redis.empty! if @redis
+      generic_cleanup
+    end
 
+    def main
       @logger.info("Booting main loop.")
 
-      until should_quit?
+      until should_quit? && @received_shutdown
         @logger.info("Waiting for event...")
-        event = Flapjack::Data::Event.next(:persistence => @redis)
+        event = Flapjack::Data::Event.next(:redis => @redis)
         process_event(event) unless event.nil?
       end
 
-      @redis.empty! if @redis
-
       @logger.info("Exiting main loop.")
     end
 
@@ -102,7 +109,8 @@ module Flapjack
   private
 
     def process_event(event)
-      @logger.debug("#{Flapjack::Data::Event.pending_count(:persistence => @redis)} events waiting on the queue")
+      pending = Flapjack::Data::Event.pending_count(:redis => @redis)
+      @logger.debug("#{pending} events waiting on the queue")
       @logger.debug("Raw event received: #{event.inspect}")
       time_at = event.time
       time_at_str = time_at ? ", #{Time.at(time_at).to_s}" : ''
@@ -129,14 +137,14 @@ module Flapjack
       end
 
       @logger.info("#{Time.now}: Sending notifications for event #{event.id}")
-      generate_notification(event, entity_check)
+      send_notification_messages(event, entity_check)
     end
 
     def update_keys(event, entity_check)
       result    = { :skip_filters => false }
       timestamp = Time.now.to_i
       @event_count = @redis.hincrby('event_counters', 'all', 1)
-      @event_count = @redis.hincrby("event_counters:#{@instance_id}", 'all', 1)
+      @redis.hincrby("event_counters:#{@instance_id}", 'all', 1)
 
       # FIXME skip if entity_check.nil?
 
@@ -191,16 +199,15 @@ module Flapjack
         end
       when 'shutdown'
         # should this be logged as an action instead? being minimally invasive for now
-        result[:shutdown] = true
+        result[:shutdown] = @received_shutdown = true
       end
 
       result
     end
 
     # takes an event for which a notification needs to be generated, works out the type of
-    # notification, updates the notification history in redis, calls other methods to work out who
-    # to notify, by what method, and finally to have the notifications sent
-    def generate_notification(event, entity_check)
+    # notification, updates the notification history in redis, sends the notifications
+    def send_notification_messages(event, entity_check)
       timestamp = Time.now.to_i
       notification_type = 'unknown'
       case event.type
@@ -215,79 +222,51 @@ module Flapjack
         case event.state
         when 'acknowledgement'
           notification_type = 'acknowledgement'
+        when 'test_notifications'
+          notification_type = 'test'
         end
       end
       @redis.set("#{event.id}:last_#{notification_type}_notification", timestamp)
       @redis.rpush("#{event.id}:#{notification_type}_notifications", timestamp)
       @logger.debug("Notification of type #{notification_type} is being generated for #{event.id}.")
 
-      send_notifications(event, notification_type, entity_check.contacts)
-    end
-
-    # takes an event, a notification type, and an array of contacts and creates jobs in resque
-    # (eventually) for each notification
-    def send_notifications(event, notification_type, contacts)
-      notification = { 'event_id'          => event.id,
-                       'state'             => event.state,
-                       'summary'           => event.summary,
-                       'time'              => event.time,
-                       'notification_type' => notification_type }
+      contacts = entity_check.contacts
 
       if contacts.empty?
         @notifylog.info("#{Time.now.to_s} | #{event.id} | #{notification_type} | NO CONTACTS")
         return
-      end
+       end
+
+      notification = Flapjack::Data::Notification.for_event(event, :type => notification_type)
 
-      contacts.each {|contact|
+      notification.messages(:contacts => contacts).each do |msg|
+        media_type = msg.medium.to_sym
 
-        if contact.media.empty?
-          @notifylog.info("#{Time.now.to_s} | #{event.id} | #{notification_type} | #{contact.id} | NO MEDIA FOR CONTACT")
+        @notifylog.info("#{Time.now.to_s} | #{event.id} | " +
+          "#{notification_type} | #{msg.contact.id} | #{media_type.to_s} | #{msg.address}")
+
+        unless @queues[media_type]
+          # TODO log error
           next
         end
 
-        notification.merge!({'contact_id'         => contact.id,
-                             'contact_first_name' => contact.first_name,
-                             'contact_last_name'  => contact.last_name, })
-
-        contact.media.each_pair {|media_type, address|
-
-          @notifylog.info("#{Time.now.to_s} | #{event.id} | " +
-            "#{notification_type} | #{contact.id} | #{media_type} | #{address}")
-
-          # queue this notification
-          notif = notification.dup
-          notif['media']   = media_type
-          notif['address'] = address
-          notif['id']      = fuid
-          dur = event.duration
-          notif['duration'] = dur if dur
-          @logger.debug("send_notifications: sending notification: #{notif.inspect}")
-
-          unless @queues[media_type.to_sym]
-            # TODO log error
-            next
-          end
-
-          # TODO consider changing Resque jobs to use raw blpop like the others
-          case media_type
-          when "sms"
-            Resque.enqueue_to(@queues[:sms], Notification::Sms, notif)
-          when "email"
-            Resque.enqueue_to(@queues[:email], Notification::Email, notif)
-          when "jabber"
-            # TODO move next line up into other notif value setting above?
-            notif['event_count'] = @event_count if @event_count
-            @redis.rpush(@queues[:jabber], Yajl::Encoder.encode(notif))
-          when "pagerduty"
-            @redis.rpush(@queues[:pagerduty], Yajl::Encoder.encode(notif))
-          end
-        }
-      }
-    end
+        contents = msg.contents
+
+        # TODO consider changing Resque jobs to use raw blpop like the others
+        case media_type
+        when :sms
+          Resque.enqueue_to(@queues[:sms], Flapjack::Notification::Sms, contents)
+        when :email
+          Resque.enqueue_to(@queues[:email], Flapjack::Notification::Email, contents)
+        when :jabber
+          # TODO move next line up into other notif value setting above?
+          contents['event_count'] = @event_count if @event_count
+          @redis.rpush(@queues[:jabber], Yajl::Encoder.encode(contents))
+        when :pagerduty
+          @redis.rpush(@queues[:pagerduty], Yajl::Encoder.encode(contents))
+        end
 
-    # generates a fairly unique identifier to use as a message id
-    def fuid
-      fuid = self.object_id.to_i.to_s + '-' + Time.now.to_i.to_s + '.' + Time.now.tv_usec.to_s
+      end
     end
 
   end

data/lib/flapjack/filters/acknowledgement.rb

@@ -15,21 +15,24 @@ module Flapjack
         timestamp = Time.now.to_i
         result = false
         if event.type == 'action'
-          if event.acknowledgement? and @persistence.zscore("failed_checks", event.id)
-            ec = Flapjack::Data::EntityCheck.for_event_id(event.id, :redis => @persistence)
-            if ec.nil?
-              @log.error "Filter: Acknowledgement: unknown entity for event '#{event.id}'"
+          if event.acknowledgement?
+            if @persistence.zscore("failed_checks", event.id)
+              ec = Flapjack::Data::EntityCheck.for_event_id(event.id, :redis => @persistence)
+              if ec.nil?
+                @log.error "Filter: Acknowledgement: unknown entity for event '#{event.id}'"
+              else
+                ec.create_unscheduled_maintenance(:start_time => timestamp,
+                  :duration => (event.duration || (4 * 60 * 60)),
+                  :summary  => event.summary)
+                message = "unscheduled maintenance created for #{event.id}"
+              end
             else
-              ec.create_unscheduled_maintenance(:start_time => timestamp,
-                :duration => (event.duration || (4 * 60 * 60)),
-                :summary  => event.summary)
-              message = "unscheduled maintenance created for #{event.id}"
+              result = true
+              @log.debug("Filter: Acknowledgement: blocking because zscore of failed_checks for #{event.id} is false") unless @persistence.zscore("failed_checks", event.id)
             end
           else
             message = "no action taken"
-            result  = true
-            @log.debug("Filter: Acknowledgement: blocking because event.acknowledgement? is false") unless event.acknowledgement?
-            @log.debug("Filter: Acknowledgement: blocking because zscore of failed_checks for #{event.id} is false") unless @persistence.zscore("failed_checks", event.id)
+            result  = false
           end
         end
         @log.debug("Filter: Acknowledgement: #{result ? "block" : "pass"} (#{message})")