flapjack 0.5.5 → 0.6.23

data/lib/flapjack/filters/{any_parents_failed.rb → base.rb}

@@ -1,13 +1,15 @@
+#!/usr/bin/env ruby
+
 module Flapjack
   module Filters
-    class AnyParentsFailed
+    module Base
       def initialize(opts={})
-        @log = opts[:log]
+        @log         = opts[:log]
         @persistence = opts[:persistence]
       end
 
-      def block?(result)
-        @persistence.any_parents_failed?(result)
+      def name
+        self.class.to_s.split('::').last
       end
     end
   end

data/lib/flapjack/filters/delays.rb

@@ -0,0 +1,53 @@
+#!/usr/bin/env ruby
+
+require 'flapjack/data/entity_check'
+require 'flapjack/filters/base'
+
+module Flapjack
+  module Filters
+
+    # * If the service event’s state is a failure, and the time since the ok => failure state change
+    #   is below a threshold (e.g. 30 seconds), then don't alert
+    # * If the service event’s state is a failure, and the time since the last alert is below a
+    #   threshold (5 minutes), then don’t alert
+    class Delays
+      include Base
+
+      def block?(event)
+        failure_delay = 30
+        resend_delay  = 300
+
+        result = false
+
+        if (event.type == 'service') and (event.critical? or event.warning?)
+
+          entity_check = Flapjack::Data::EntityCheck.for_event_id(event.id, :redis => @persistence)
+          current_time = Time.now.to_i
+
+          if entity_check.failed?
+            last_problem_alert = entity_check.last_problem_notification
+            last_change        = entity_check.last_change
+
+            current_failure_duration = current_time - last_change
+            time_since_last_alert    = current_time - last_problem_alert unless last_problem_alert.nil?
+            @log.debug("Filter: Delays: last_problem_alert: #{last_problem_alert.to_s}, last_change: #{last_change.to_s}, current_failure_duration: #{current_failure_duration}, time_since_last_alert: #{time_since_last_alert.to_s}")
+            if (current_failure_duration < failure_delay)
+              result = true
+              @log.debug("Filter: Delays: blocking because duration of current failure (#{current_failure_duration}) is less than failure_delay (#{failure_delay})")
+            elsif !last_problem_alert.nil? && (time_since_last_alert < resend_delay)
+              result = true
+              @log.debug("Filter: Delays: blocking because time since last alert for current problem (#{time_since_last_alert}) is less than resend_delay (#{resend_delay})")
+            else
+              @log.debug("Filter: Delays: not blocking because neither of the time comparison conditions were met")
+            end
+          else
+            @log.debug("Filter: Delays: entity_check.failed? returned false ...")
+          end
+        end
+
+        @log.debug("Filter: Delays: #{result ? "block" : "pass"}")
+        result
+      end
+    end
+  end
+end

data/lib/flapjack/filters/detect_mass_client_failures.rb

@@ -0,0 +1,44 @@
+#!/usr/bin/env ruby
+
+require 'flapjack/filters/base'
+
+module Flapjack
+  module Filters
+
+    # * If the service event’s state is a failure, and the total number of failing client checks is
+    #   over a threshold (e.g. 10 checks are failing), then set a meta flag noting the threshold has
+    #   been tripped, and generate an event for this meta check
+    # * If the service event’s state is ok, and the meta flag is set, and the total number of
+    #   failing client checks is less than a threshold (eg 10), then unset the flag, and generate an
+    #   event for this meta check
+    class DetectMassClientFailures
+      include Base
+
+      def block?(event)
+        client_mass_fail_threshold = 10
+        timestamp = Time.now.to_i
+
+        if event.type == 'service'
+          client_fail_count = @persistence.zcount("failed_checks:#{event.client}", '-inf', '+inf')
+
+          if client_fail_count >= client_mass_fail_threshold
+            # set the flag
+            # FIXME: perhaps implement this with tagging
+            @persistence.add("mass_failed_client:#{event.client}", timestamp)
+            @persistence.zadd("mass_failure_events_client:#{event.client}", 0, timestamp)
+          else
+            # unset the flag
+            start_mf = @persistence.get("mass_failed_client:#{event.client}")
+            duration = Time.now.to_i - start_mf.to_i
+            @persistence.del("mass_failed_client:#{event.client}")
+            @persistence.zadd("mass_failure_events_client:#{event.client}", duration, start_mf)
+          end
+        end
+
+        result = false
+        @log.debug("Filter: DetectMassClientFailures: #{result ? "block" : "pass"}")
+        result
+      end
+    end
+  end
+end

data/lib/flapjack/filters/ok.rb

@@ -1,12 +1,32 @@
+#!/usr/bin/env ruby
+
+require 'flapjack/filters/base'
+
 module Flapjack
   module Filters
+
+    # * If the service event’s state is ok and the previous state was ok, don’t alert
+    # * If the service event's state is ok and there is unscheduled downtime set, end the unscheduled
+    #   downtime
     class Ok
-      def initialize(opts={})
-        @log = opts[:log]
-      end
+      include Base
+
+      def block?(event)
+        result = false
+
+        if event.ok?
+          if event.previous_state == 'ok'
+            @log.debug("Filter: Ok: existing state was ok, and the previous state was ok, so blocking")
+            result = true
+          end
+
+          # end any unscheduled downtime
+          entity_check = Flapjack::Data::EntityCheck.for_event_id(event.id, :redis => @persistence)
+          entity_check.end_unscheduled_maintenance
+        end
 
-      def block?(result)
-        !result.warning? || !result.critical?
+        @log.debug("Filter: Ok: #{result ? "block" : "pass"}")
+        result
       end
     end
   end

data/lib/flapjack/filters/scheduled_maintenance.rb

@@ -0,0 +1,17 @@
+#!/usr/bin/env ruby
+
+require 'flapjack/filters/base'
+
+module Flapjack
+  module Filters
+    class ScheduledMaintenance
+      include Base
+
+      def block?(event)
+        result = @persistence.exists("#{event.id}:scheduled_maintenance")
+        @log.debug("Filter: Scheduled Maintenance: #{result ? "block" : "pass"}")
+        result
+      end
+    end
+  end
+end

data/lib/flapjack/filters/unscheduled_maintenance.rb

@@ -0,0 +1,17 @@
+#!/usr/bin/env ruby
+
+require 'flapjack/filters/base'
+
+module Flapjack
+  module Filters
+    class UnscheduledMaintenance
+      include Base
+
+      def block?(event)
+        result = @persistence.exists("#{event.id}:unscheduled_maintenance")
+        @log.debug("Filter: Unscheduled Maintenance: #{result ? "block" : "pass"}")
+        result
+      end
+    end
+  end
+end

data/lib/flapjack/jabber.rb

@@ -0,0 +1,294 @@
+#!/usr/bin/env ruby
+
+require 'socket'
+
+require 'eventmachine'
+# the redis/synchrony gems need to be required in this particular order, see
+# the redis-rb README for details
+require 'hiredis'
+require 'em-synchrony'
+require 'redis/connection/synchrony'
+require 'redis'
+
+require 'chronic_duration'
+
+require 'blather/client/client'
+require 'em-synchrony/fiber_iterator'
+require 'yajl/json_gem'
+
+require 'flapjack/data/entity_check'
+require 'flapjack/pikelet'
+require 'flapjack/utility'
+
+module Flapjack
+
+  class Jabber < Blather::Client
+
+    include Flapjack::Pikelet
+    include Flapjack::Utility
+
+    log = Logger.new(STDOUT)
+    # log.level = Logger::DEBUG
+    log.level = Logger::INFO
+    Blather.logger = log
+
+    def setup
+      @redis = build_redis_connection_pool
+      @hostname = Socket.gethostname
+      @flapjack_jid = Blather::JID.new((@config['jabberid'] || 'flapjack') + '/' + @hostname)
+
+      super(@flapjack_jid, @config['password'], @config['server'], @config['port'].to_i)
+
+      logger.debug("Building jabber connection with jabberid: " +
+        @flapjack_jid.to_s + ", port: " + @config['port'].to_s +
+        ", server: " + @config['server'].to_s + ", password: " +
+        @config['password'].to_s)
+
+      register_handler :ready do |stanza|
+        EM.next_tick do
+          EM.synchrony do
+            on_ready(stanza)
+          end
+        end
+      end
+
+      register_handler :message, :groupchat?, :body => /^flapjack:\s+/ do |stanza|
+        EM.next_tick do
+          EM.synchrony do
+            on_groupchat(stanza)
+          end
+        end
+      end
+
+      register_handler :message, :chat? do |stanza|
+        EM.next_tick do
+          EM.synchrony do
+            on_chat(stanza)
+          end
+        end
+      end
+
+      register_handler :disconnected do |stanza|
+        ret = true
+        EM.next_tick do
+          EM.synchrony do
+            ret = on_disconnect(stanza)
+          end
+        end
+        ret
+      end
+    end
+
+    # Join the MUC Chat room after connecting.
+    def on_ready(stanza)
+      return if should_quit?
+      @redis_handler ||= build_redis_connection_pool
+      @connected_at = Time.now.to_i
+      logger.info("Jabber Connected")
+      @config['rooms'].each do |room|
+        logger.info("Joining room #{room}")
+        presence = Blather::Stanza::Presence.new
+        presence.from = @flapjack_jid
+        presence.to = Blather::JID.new("#{room}/#{@config['alias']}")
+        presence << "<x xmlns='http://jabber.org/protocol/muc'/>"
+        write presence
+        say(room, "flapjack jabber gateway started at #{Time.now}, hello!", :groupchat)
+      end
+    end
+
+    def interpreter(command)
+
+      msg          = nil
+      action       = nil
+      entity_check = nil
+      case
+      when command =~ /^ACKID\s+(\d+)(?:\s*(.*?)(?:\s*duration.*?(\d+.*\w+.*))?)$/i;
+        ackid        = $1
+        comment      = $2
+        duration_str = $3
+
+        error = nil
+        dur   = nil
+
+        if comment.nil? || (comment.length == 0)
+          error = "please provide a comment, eg \"flapjack: ACKID #{$1} AL looking\""
+        elsif duration_str
+          # a fairly liberal match above, we'll let chronic_duration do the heavy lifting
+          dur = ChronicDuration.parse(duration_str)
+        end
+
+        four_hours = 4 * 60 * 60
+        duration = (dur.nil? || (dur <= 0) || (dur > four_hours)) ? four_hours : dur
+
+        event_id = @redis_handler.hget('unacknowledged_failures', ackid)
+
+        if event_id.nil?
+          error = "not found"
+        else
+          entity_check = Flapjack::Data::EntityCheck.for_event_id(event_id, :redis => @redis_handler)
+          error = "unknown entity" if entity_check.nil?
+        end
+
+        if error
+          msg = "ERROR - couldn't ACK #{ackid} - #{error}"
+        else
+          msg = "ACKing #{entity_check.check} on #{entity_check.entity_name} (#{ackid})"
+          action = Proc.new {
+            entity_check.create_acknowledgement('summary' => (comment || ''),
+              'acknowledgement_id' => ackid, 'duration' => duration)
+          }
+        end
+
+      when command =~ /^help$/
+        msg  = "commands: \n"
+        msg += "  ACKID <id> <comment> [duration: <time spec>] \n"
+        msg += "  identify \n"
+        msg += "  help \n"
+
+      when command =~ /^identify$/
+        t = Process.times
+
+        msg  = "Flapjack process #{Process.pid} on #{`hostname -f`.chomp} \n"
+        msg += "User CPU Time: #{t.utime}\n"
+        msg += "System CPU Time: #{t.stime}\n"
+        msg += `uname -a`.chomp + "\n"
+
+      when command =~ /^(.*)/
+        words = $1
+        msg   = "what do you mean, '#{words}'? Type 'help' for a list of acceptable commands."
+
+      end
+
+      {:msg => msg, :action => action}
+    end
+
+    def on_groupchat(stanza)
+      return if should_quit?
+      logger.debug("groupchat message received: #{stanza.inspect}")
+
+      if stanza.body =~ /^flapjack:\s+(.*)/
+        command = $1
+      end
+
+      results = interpreter(command)
+      msg     = results[:msg]
+      action  = results[:action]
+
+      if msg || action
+        say(stanza.from.stripped, msg, :groupchat)
+        logger.debug("Sent to group chat: #{msg}")
+        action.call if action
+      end
+    end
+
+    def on_chat(stanza)
+      return if should_quit?
+      logger.debug("chat message received: #{stanza.inspect}")
+
+      results = interpreter(stanza.body)
+      msg     = results[:msg]
+      action  = results[:action]
+
+      if msg || action
+        say(stanza.from.stripped, msg, :chat)
+        logger.debug("Sent to #{stanza.from.stripped}: #{msg}")
+        action.call if action
+      end
+    end
+
+    # returning true to prevent the reactor loop from stopping
+    def on_disconnect(stanza)
+      return true if should_quit?
+      logger.warn("jabbers disconnected! reconnecting in 1 second ...")
+      EventMachine::Timer.new(1) do
+        connect # Blather::Client.connect
+      end
+      true
+    end
+
+    def say(to, msg, using = :chat)
+      @logger.debug("Sending a jabber message to: #{to.to_s}, using: #{using.to_s}, message: #{msg}")
+      write Blather::Stanza::Message.new(to, msg, using)
+    end
+
+    def add_shutdown_event(opts = {})
+      return unless redis = opts[:redis]
+      redis.rpush(@config['queue'], JSON.generate('notification_type' => 'shutdown'))
+    end
+
+    def main
+      logger.debug("New Jabber pikelet with the following options: #{@config.inspect}")
+
+      count_timer = EM::Synchrony.add_periodic_timer(30) do
+        logger.debug("connection count: #{EM.connection_count} #{Time.now.to_s}.#{Time.now.usec.to_s}")
+      end
+
+      keepalive_timer = EM::Synchrony.add_periodic_timer(60) do
+        logger.debug("calling keepalive on the jabber connection")
+        write(' ') if connected?
+      end
+
+      setup
+      connect # Blather::Client.connect
+
+      # simplified to use a single queue only as it makes the shutdown logic easier
+      queue = @config['queue']
+      events = {}
+
+      until should_quit?
+
+        # FIXME: should also check if presence has been established in any group chat rooms that are
+        # configured before starting to process events, otherwise the first few may get lost (send
+        # before joining the group chat rooms)
+        if connected?
+          logger.debug("jabber is connected so commencing blpop on #{queue}")
+          events[queue] = @redis.blpop(queue)
+          event         = Yajl::Parser.parse(events[queue][1])
+          type          = event['notification_type']
+          logger.debug('jabber notification event received')
+          logger.debug(event.inspect)
+          if 'shutdown'.eql?(type)
+            EM.next_tick do
+              # get delays without the next_tick
+              close # Blather::Client.close
+            end
+            # FIXME: should we also set something so should_quit? returns true
+            # to prevent retrieving more notifications from the queue while closing?
+            # or does close only return once the connection is really and truely closed?
+          else
+            entity, check = event['event_id'].split(':')
+            state         = event['state']
+            summary       = event['summary']
+            duration      = event['duration'] ? time_period_in_words(event['duration']) : '4 hours'
+            address       = event['address']
+
+            logger.debug("processing jabber notification address: #{address}, event: #{entity}:#{check}, state: #{state}, summary: #{summary}")
+
+            ack_str = event['event_count'] && !state.eql?('ok') && !'acknowledgement'.eql?(type) ?
+              "::: flapjack: ACKID #{event['event_count']} " : ''
+
+            maint_str = (type && 'acknowledgement'.eql?(type)) ?
+              "has been acknowledged, unscheduled maintenance created for #{duration}" :
+              "is #{state.upcase}"
+
+            msg = "#{type.upcase} #{ack_str}::: \"#{check}\" on #{entity} #{maint_str} ::: #{summary}"
+
+            chat_type = :chat
+            chat_type = :groupchat if @config['rooms'].include?(address)
+            EM.next_tick do
+              say(Blather::JID.new(address), msg, chat_type)
+            end
+          end
+        else
+          logger.debug("not connected, sleep 1 before retry")
+          EM::Synchrony.sleep(1)
+        end
+      end
+
+      count_timer.cancel
+      keepalive_timer.cancel
+    end
+
+  end
+end
+