flapjack 0.6.53 → 0.6.54

data/lib/flapjack/data/entity_check.rb

@@ -238,7 +238,7 @@ module Flapjack
         @redis.zadd("#{@key}:sorted_state_timestamps", timestamp, timestamp)
 
         case state
-        when STATE_WARNING, STATE_CRITICAL
+        when STATE_WARNING, STATE_CRITICAL, STATE_UNKNOWN
           @redis.zadd('failed_checks', timestamp, @key)
           # FIXME: Iterate through a list of tags associated with an entity:check pair, and update counters
           @redis.zadd("failed_checks:client:#{client}", timestamp, @key) if client
@@ -310,7 +310,7 @@ module Flapjack
       end
 
       def failed?
-        [STATE_WARNING, STATE_CRITICAL].include?( state )
+        [STATE_WARNING, STATE_CRITICAL, STATE_UNKNOWN].include?( state )
       end
 
       def ok?

data/lib/flapjack/data/event.rb

@@ -1,6 +1,6 @@
 #!/usr/bin/env ruby
 
-require 'yajl'
+require 'yajl/json_gem'
 
 module Flapjack
   module Data
@@ -43,7 +43,7 @@ module Flapjack
         raise "Redis connection not set" unless redis = opts[:redis]
 
         evt['time'] = Time.now.to_i if evt['time'].nil?
-        redis.rpush('events', Yajl::Encoder.encode(evt))
+        redis.rpush('events', ::Yajl::Encoder.encode(evt))
       end
 
       # Provide a count of the number of events on the queue to be processed.
@@ -147,7 +147,7 @@ module Flapjack
       end
 
       def failure?
-        warning? or critical?
+        warning? or critical? or unknown?
       end
 
     end

data/lib/flapjack/executive.rb

@@ -3,7 +3,6 @@
 require 'log4r'
 require 'log4r/outputter/fileoutputter'
 
-require 'flapjack'
 require 'flapjack/filters/acknowledgement'
 require 'flapjack/filters/ok'
 require 'flapjack/filters/scheduled_maintenance'
@@ -14,7 +13,6 @@ require 'flapjack/data/contact'
 require 'flapjack/data/entity_check'
 require 'flapjack/data/notification'
 require 'flapjack/data/event'
-require 'flapjack/pikelet'
 require 'flapjack/redis_pool'
 
 require 'flapjack/gateways/email'
@@ -23,15 +21,12 @@ require 'flapjack/gateways/sms_messagenet'
 module Flapjack
 
   class Executive
-    include Flapjack::GenericPikelet
 
-    alias_method :generic_bootstrap, :bootstrap
-    alias_method :generic_cleanup,   :cleanup
-
-    def bootstrap(opts = {})
-      generic_bootstrap(opts)
-
-      @redis = Flapjack::RedisPool.new(:config => opts[:redis_config], :size => 1)
+    def initialize(opts = {})
+      @config = opts[:config]
+      @redis_config = opts[:redis_config]
+      @logger = opts[:logger]
+      @redis = Flapjack::RedisPool.new(:config => @redis_config, :size => 2) # first will block
 
       @queues = {:email     => @config['email_queue'],
                  :sms       => @config['sms_queue'],
@@ -44,7 +39,7 @@ module Flapjack
 
       # FIXME: Put loading filters into separate method
       # FIXME: should we make the filters more configurable by the end user?
-      options = { :log => @logger, :persistence => @redis }
+      options = { :log => opts[:logger], :persistence => @redis }
       @filters = []
       @filters << Flapjack::Filters::Ok.new(options)
       @filters << Flapjack::Filters::ScheduledMaintenance.new(options)
@@ -80,16 +75,11 @@ module Flapjack
       @redis.hset("event_counters:#{@instance_id}", 'action', 0)
     end
 
-    def cleanup
-      @redis.empty! if @redis
-      generic_cleanup
-    end
-
-    def main
+    def start
       @logger.info("Booting main loop.")
 
-      until should_quit? && @received_shutdown
-        @logger.info("Waiting for event...")
+      until @should_quit
+        @logger.debug("Waiting for event...")
         event = Flapjack::Data::Event.next(:redis => @redis)
         process_event(event) unless event.nil?
       end
@@ -99,12 +89,12 @@ module Flapjack
 
     # this must use a separate connection to the main Executive one, as it's running
     # from a different fiber while the main one is blocking.
-    def add_shutdown_event(opts = {})
-      return unless redis = opts[:redis]
-      redis.rpush('events', JSON.generate('type'    => 'shutdown',
-                                          'host'    => '',
-                                          'service' => '',
-                                          'state'   => ''))
+    def stop
+      @should_quit = true
+      @redis.rpush('events', JSON.generate('type'    => 'shutdown',
+                                           'host'    => '',
+                                           'service' => '',
+                                           'state'   => ''))
     end
 
   private
@@ -115,29 +105,29 @@ module Flapjack
       @logger.debug("Raw event received: #{event.inspect}")
       time_at = event.time
       time_at_str = time_at ? ", #{Time.at(time_at).to_s}" : ''
-      @logger.info("Processing Event: #{event.id}, #{event.type}, #{event.state}, #{event.summary}#{time_at_str}")
+      @logger.debug("Processing Event: #{event.id}, #{event.type}, #{event.state}, #{event.summary}#{time_at_str}")
 
-      entity_check = (event.type == 'shutdown') ? nil :
+      entity_check = ('shutdown' == event.type) ? nil :
                        Flapjack::Data::EntityCheck.for_event_id(event.id, :redis => @redis)
 
-      result       = update_keys(event, entity_check)
+      result = update_keys(event, entity_check)
       return if result[:shutdown]
-      skip_filters = result[:skip_filters]
 
-      blocker = @filters.find {|filter| filter.block?(event) } unless skip_filters
+      blocker = nil
 
-      if skip_filters
-        @logger.info("#{Time.now}: Not sending notifications for event #{event.id} because filtering was skipped")
+      if result[:skip_filters]
+        @logger.debug("Not generating notifications for event #{event.id} because filtering was skipped")
         return
+      else
+        blocker = @filters.find {|filter| filter.block?(event) }
       end
 
       if blocker
-        blocker_names = [ blocker.name ]
-        @logger.info("#{Time.now}: Not sending notifications for event #{event.id} because these filters blocked: #{blocker_names.join(', ')}")
+        @logger.debug("Not generating notifications for event #{event.id} because this filter blocked: #{blocker.name}")
         return
       end
 
-      @logger.info("#{Time.now}: Sending notifications for event #{event.id}")
+      @logger.info("Generating notifications for event #{event.id}, #{event.type}, #{event.state}, #{event.summary}#{time_at_str}")
       send_notification_messages(event, entity_check)
     end
 
@@ -201,7 +191,7 @@ module Flapjack
         end
       when 'shutdown'
         # should this be logged as an action instead? being minimally invasive for now
-        result[:shutdown] = @received_shutdown = true
+        result[:shutdown] = true
       end
 
       result
@@ -215,9 +205,9 @@ module Flapjack
       case event.type
       when 'service'
         case event.state
-        when 'ok', 'unknown'
+        when 'ok'
           notification_type = 'recovery'
-        when 'warning', 'critical'
+        when 'warning', 'critical', 'unknown'
           notification_type = 'problem'
         end
       when 'action'
@@ -237,7 +227,7 @@ module Flapjack
       if contacts.empty?
         @notifylog.info("#{Time.now.to_s} | #{event.id} | #{notification_type} | NO CONTACTS")
         return
-       end
+      end
 
       notification = Flapjack::Data::Notification.for_event(event, :type => notification_type)
 
@@ -248,7 +238,7 @@ module Flapjack
           "#{notification_type} | #{msg.contact.id} | #{media_type.to_s} | #{msg.address}")
 
         unless @queues[media_type]
-          # TODO log error
+          @logger.error("no queue for media type: #{media_type}")
           next
         end
 

data/lib/flapjack/filters/delays.rb

@@ -19,7 +19,7 @@ module Flapjack
 
         result = false
 
-        if (event.type == 'service') and (event.critical? or event.warning?)
+        if (event.type == 'service') and (event.failure?)
 
           entity_check = Flapjack::Data::EntityCheck.for_event_id(event.id, :redis => @persistence)
           current_time = Time.now.to_i

data/lib/flapjack/gateways/api.rb

@@ -19,8 +19,6 @@ require 'flapjack/gateways/api/entity_presenter'
 require 'flapjack/rack_logger'
 require 'flapjack/redis_pool'
 
-require 'flapjack/gateways/base'
-
 # from https://github.com/sinatra/sinatra/issues/501
 # TODO move to its own file
 module Rack
@@ -59,8 +57,8 @@ module Flapjack
       else
         # doesn't work with Rack::Test unless we wrap tests in EM.synchrony blocks
         rescue_exception = Proc.new { |env, exception|
-          logger.error exception.message
-          logger.error exception.backtrace.join("\n")
+          @logger.error exception.message
+          @logger.error exception.backtrace.join("\n")
           [503, {}, {:errors => [exception.message]}.to_json]
         }
 
@@ -70,28 +68,13 @@ module Flapjack
       use Rack::JsonParamsParser
 
       class << self
-        include Flapjack::Gateways::Thin
-
-        attr_accessor :redis
-
-        alias_method :thin_bootstrap, :bootstrap
-        alias_method :thin_cleanup,   :cleanup
-
-        def bootstrap(opts = {})
-          thin_bootstrap(opts)
-          @redis = Flapjack::RedisPool.new(:config => opts[:redis_config], :size => 1)
-
-          if config && config['access_log']
-            access_logger = Flapjack::AsyncLogger.new(config['access_log'])
+        def start
+          @redis = Flapjack::RedisPool.new(:config => @redis_config, :size => 1)
+          if @config && @config['access_log']
+            access_logger = Flapjack::AsyncLogger.new(@config['access_log'])
             use Flapjack::CommonLogger, access_logger
           end
         end
-
-        def cleanup
-          @redis.empty! if @redis
-          thin_cleanup
-        end
-
       end
 
       def redis

data/lib/flapjack/gateways/email.rb

@@ -9,23 +9,17 @@ require 'em-synchrony'
 require 'em/protocols/smtpclient'
 
 require 'flapjack/data/entity_check'
-require 'flapjack/gateways/base'
 
 module Flapjack
   module Gateways
 
     class Email
-      extend Flapjack::Gateways::Resque
 
       class << self
 
-        alias_method :orig_bootstrap, :bootstrap
-
-        def bootstrap(opts = {})
-          return if @bootstrapped
-          @smtp_config = opts[:config].delete('smtp_config')
+        def start
+          @smtp_config = @config.delete('smtp_config')
           @sent = 0
-          orig_bootstrap(opts)
         end
 
         def perform(notification)
@@ -61,7 +55,7 @@ module Flapjack
 
             fqdn       = `/bin/hostname -f`.chomp
             m_from     = "flapjack@#{fqdn}"
-            logger.debug("flapjack_mailer: set from to #{m_from}")
+            @logger.debug("flapjack_mailer: set from to #{m_from}")
             m_reply_to = m_from
             m_to       = notification['address']
 
@@ -92,7 +86,7 @@ module Flapjack
             @logger.info "Email response: #{response.inspect}"
 
           rescue Exception => e
-            @logger.error "Error delivering email to #{mail.to}: #{e.message}"
+            @logger.error "Error delivering email to #{m_to}: #{e.message}"
             @logger.error e.backtrace.join("\n")
           end
         end

data/lib/flapjack/gateways/email/alert.html.haml

@@ -37,11 +37,6 @@
       %td
         %strong Time
       %td= Time.at(@time.to_i).to_s
-    %tr
-      %td
-        %strong Flapjack
-      %td
-        %a(href="http://127.0.0.1:9292/check?entity=#{@entity}&check=#{@check}") http://127.0.0.1:9292/check?entity=#{@entity}&check=#{@check}
 
 %p Cheers,
 %p Flapjack

data/lib/flapjack/gateways/email/alert.text.erb

@@ -7,7 +7,6 @@ Check: <%= @check %>
 State: <%= @state %>
 Summary: <%= @summary %>
 Time: <%= Time.at(@time.to_i).to_s %>
-Flapjack: http://127.0.0.1:9292/check?entity=<%= @entity %>&check=<%= @check %>
 
 Cheers,
 Flapjack

data/lib/flapjack/gateways/jabber.rb

@@ -21,14 +21,11 @@ require 'flapjack/redis_pool'
 require 'flapjack/utility'
 require 'flapjack/version'
 
-require 'flapjack/gateways/base'
-
 module Flapjack
 
   module Gateways
 
     class Jabber < Blather::Client
-      include Flapjack::Gateways::Generic
       include Flapjack::Utility
 
       log = Logger.new(STDOUT)
@@ -36,23 +33,21 @@ module Flapjack
       log.level = Logger::INFO
       Blather.logger = log
 
-      alias_method :generic_bootstrap, :bootstrap
-      alias_method :generic_cleanup,   :cleanup
-
-      def bootstrap(opts = {})
-        generic_bootstrap(opts)
-
+      def initialize(opts = {})
+        @config = opts[:config]
         @redis_config = opts[:redis_config]
-        @redis = Flapjack::RedisPool.new(:config => @redis_config, :size => 1)
+        @redis = Flapjack::RedisPool.new(:config => @redis_config, :size => 2) # first will block
+
+        @logger = opts[:logger]
 
         @buffer = []
         @hostname = Socket.gethostname
+        super()
       end
 
-      def cleanup
-        @redis.empty! if @redis
-        @redis_handler.empty! if @redis_handler
-        generic_cleanup
+      def stop
+        @should_quit = true
+        @redis.rpush(@config['queue'], JSON.generate('notification_type' => 'shutdown'))
       end
 
       def setup
@@ -60,7 +55,7 @@ module Flapjack
 
         super(@flapjack_jid, @config['password'], @config['server'], @config['port'].to_i)
 
-        logger.debug("Building jabber connection with jabberid: " +
+        @logger.debug("Building jabber connection with jabberid: " +
           @flapjack_jid.to_s + ", port: " + @config['port'].to_s +
           ", server: " + @config['server'].to_s + ", password: " +
           @config['password'].to_s)
@@ -94,25 +89,28 @@ module Flapjack
 
       # Join the MUC Chat room after connecting.
       def on_ready(stanza)
-        return if should_quit? && @shutting_down
-        @redis_handler ||= Flapjack::RedisPool.new(:config => @redis_config, :size => 1)
+        return if @should_quit
         @connected_at = Time.now.to_i
-        logger.info("Jabber Connected")
+        @logger.info("Jabber Connected")
         if @config['rooms'] && @config['rooms'].length > 0
           @config['rooms'].each do |room|
-            logger.info("Joining room #{room}")
+            @logger.info("Joining room #{room}")
             presence = Blather::Stanza::Presence.new
             presence.from = @flapjack_jid
             presence.to = Blather::JID.new("#{room}/#{@config['alias']}")
             presence << "<x xmlns='http://jabber.org/protocol/muc'/>"
-            write presence
-            say(room, "flapjack jabber gateway started at #{Time.now}, hello!", :groupchat)
+            EventMachine::Synchrony.next_tick do
+              write presence
+              say(room, "flapjack jabber gateway started at #{Time.now}, hello!", :groupchat)
+            end
           end
         end
         return if @buffer.empty?
         while stanza = @buffer.shift
           @logger.debug("Sending a buffered jabber message to: #{stanza.to}, using: #{stanza.type}, message: #{stanza.body}")
-          write(stanza)
+          EventMachine::Synchrony.next_tick do
+            write(stanza)
+          end
         end
       end
 
@@ -139,12 +137,12 @@ module Flapjack
           four_hours = 4 * 60 * 60
           duration = (dur.nil? || (dur <= 0)) ? four_hours : dur
 
-          event_id = @redis_handler.hget('unacknowledged_failures', ackid)
+          event_id = @redis.hget('unacknowledged_failures', ackid)
 
           if event_id.nil?
             error = "not found"
           else
-            entity_check = Flapjack::Data::EntityCheck.for_event_id(event_id, :redis => @redis_handler)
+            entity_check = Flapjack::Data::EntityCheck.for_event_id(event_id, :redis => @redis)
             error = "unknown entity" if entity_check.nil?
           end
 
@@ -172,7 +170,7 @@ module Flapjack
 
         when command =~ /^identify$/
           t = Process.times
-          boot_time = Time.at(@redis_handler.get('boot_time').to_i)
+          boot_time = Time.at(@redis.get('boot_time').to_i)
           msg  = "Flapjack #{Flapjack::VERSION} process #{Process.pid} on #{`hostname -f`.chomp} \n"
           msg += "Boot time: #{boot_time}\n"
           msg += "User CPU Time: #{t.utime}\n"
@@ -185,11 +183,11 @@ module Flapjack
 
           msg = "so you want me to test notifications for entity: #{entity_name}, check: #{check_name} eh? ... well OK!"
 
-          entity = Flapjack::Data::Entity.find_by_name(entity_name, :redis => @redis_handler)
+          entity = Flapjack::Data::Entity.find_by_name(entity_name, :redis => @redis)
           if entity
             summary = "Testing notifications to all contacts interested in entity: #{entity.name}, check: #{check_name}"
 
-            entity_check = Flapjack::Data::EntityCheck.for_entity(entity, check_name, :redis => @redis_handler)
+            entity_check = Flapjack::Data::EntityCheck.for_entity(entity, check_name, :redis => @redis)
             puts entity_check.inspect
             entity_check.test_notifications('summary' => summary)
 
@@ -199,7 +197,7 @@ module Flapjack
 
         when command =~ /^(find )?entities matching\s+\/(.*)\/.*$/i
           pattern = $2.chomp.strip
-          entity_list = Flapjack::Data::Entity.find_all_name_matching(pattern, :redis => @redis_handler)
+          entity_list = Flapjack::Data::Entity.find_all_name_matching(pattern, :redis => @redis)
           max_showable = 30
           number_found = entity_list.length
           entity_list = entity_list[0..(max_showable - 1)] if number_found > max_showable
@@ -226,8 +224,8 @@ module Flapjack
       end
 
       def on_groupchat(stanza)
-        return if should_quit? && @shutting_down
-        logger.debug("groupchat message received: #{stanza.inspect}")
+        return if @should_quit
+        @logger.debug("groupchat message received: #{stanza.inspect}")
 
         if stanza.body =~ /^flapjack:\s+(.*)/
           command = $1
@@ -238,15 +236,17 @@ module Flapjack
         action  = results[:action]
 
         if msg || action
-          say(stanza.from.stripped, msg, :groupchat)
-          logger.debug("Sent to group chat: #{msg}")
-          action.call if action
+          EventMachine::Synchrony.next_tick do
+            @logger.info("sending to group chat: #{msg}")
+            say(stanza.from.stripped, msg, :groupchat)
+            action.call if action
+          end
         end
       end
 
       def on_chat(stanza)
-        return if should_quit? && @shutting_down
-        logger.debug("chat message received: #{stanza.inspect}")
+        return if @should_quit
+        @logger.debug("chat message received: #{stanza.inspect}")
 
         if stanza.body =~ /^flapjack:\s+(.*)/
           command = $1
@@ -259,23 +259,42 @@ module Flapjack
         action  = results[:action]
 
         if msg || action
-          say(stanza.from.stripped, msg, :chat)
-          logger.debug("Sent to #{stanza.from.stripped}: #{msg}")
-          action.call if action
+          EventMachine::Synchrony.next_tick do
+            @logger.info("Sending to #{stanza.from.stripped}: #{msg}")
+            say(stanza.from.stripped, msg, :chat)
+            action.call if action
+          end
         end
       end
 
-      # returning true to prevent the reactor loop from stopping
-      def on_disconnect(stanza)
-        return true if should_quit? && @shutting_down
-        logger.warn("jabbers disconnected! reconnecting in 1 second ...")
-        EventMachine::Timer.new(1) do
+      def connect_with_retry
+        attempt = 0
+        delay = 2
+        begin
+          attempt += 1
+          delay = 10 if attempt > 10
+          delay = 60 if attempt > 60
+          EventMachine::Synchrony.sleep(delay || 3) if attempt > 1
+          @logger.debug("attempting connection to the jabber server")
           connect # Blather::Client.connect
+        rescue StandardError => detail
+          @logger.error("unable to connect to the jabber server (attempt #{attempt}), retrying in #{delay} seconds ...")
+          @logger.error("detail: #{detail.message}")
+          @logger.debug(detail.backtrace.join("\n"))
+          retry unless @should_quit
         end
+      end
+
+      # returning true to prevent the reactor loop from stopping
+      def on_disconnect(stanza)
+        @logger.warn("disconnect handler called")
+        return true if @should_quit
+        @logger.warn("jabbers disconnected! reconnecting after a short deley ...")
+        connect_with_retry
         true
       end
 
-      def say(to, msg, using = :chat)
+      def say(to, msg, using = :chat, tick = true)
         stanza = Blather::Stanza::Message.new(to, msg, using)
         if connected?
           @logger.debug("Sending a jabber message to: #{to.to_s}, using: #{using.to_s}, message: #{msg}")
@@ -286,45 +305,40 @@ module Flapjack
         end
       end
 
-      def add_shutdown_event(opts = {})
-        return unless redis = opts[:redis]
-        redis.rpush(@config['queue'], JSON.generate('notification_type' => 'shutdown'))
-      end
-
-      def main
-        logger.debug("New Jabber pikelet with the following options: #{@config.inspect}")
-
-        count_timer = EM::Synchrony.add_periodic_timer(30) do
-          logger.debug("connection count: #{EM.connection_count} #{Time.now.to_s}.#{Time.now.usec.to_s}")
-        end
+      def start
+        @logger.info("starting")
+        @logger.debug("new jabber pikelet with the following options: #{@config.inspect}")
 
         keepalive_timer = EM::Synchrony.add_periodic_timer(60) do
-          logger.debug("calling keepalive on the jabber connection")
-          write(' ') if connected?
+          @logger.debug("calling keepalive on the jabber connection")
+          if connected?
+            EventMachine::Synchrony.next_tick do
+              write(' ')
+            end
+          end
         end
 
         setup
-        connect # Blather::Client.connect
+        connect_with_retry
 
         # simplified to use a single queue only as it makes the shutdown logic easier
         queue = @config['queue']
         events = {}
 
-        until should_quit? && @shutting_down
+        until @should_quit
 
           # FIXME: should also check if presence has been established in any group chat rooms that are
           # configured before starting to process events, otherwise the first few may get lost (send
           # before joining the group chat rooms)
           if connected?
-            logger.debug("jabber is connected so commencing blpop on #{queue}")
+            @logger.debug("jabber is connected so commencing blpop on #{queue}")
             events[queue] = @redis.blpop(queue, 0)
             event         = Yajl::Parser.parse(events[queue][1])
             type          = event['notification_type'] || 'unknown'
-            logger.debug('jabber notification event received')
-            logger.debug(event.inspect)
+            @logger.debug('jabber notification event received')
+            @logger.debug(event.inspect)
             if 'shutdown'.eql?(type)
-              if should_quit?
-                @shutting_down = true
+              if @should_quit
                 EventMachine::Synchrony.next_tick do
                   # get delays without the next_tick
                   close # Blather::Client.close
@@ -337,7 +351,7 @@ module Flapjack
               duration      = event['duration'] ? time_period_in_words(event['duration']) : '4 hours'
               address       = event['address']
 
-              logger.debug("processing jabber notification address: #{address}, event: #{entity}:#{check}, state: #{state}, summary: #{summary}")
+              @logger.debug("processing jabber notification address: #{address}, event: #{entity}:#{check}, state: #{state}, summary: #{summary}")
 
               ack_str =
                 event['event_count'] &&
@@ -371,12 +385,11 @@ module Flapjack
               end
             end
           else
-            logger.debug("not connected, sleep 1 before retry")
+            @logger.debug("not connected, sleep 1 before retry")
             EM::Synchrony.sleep(1)
           end
         end
 
-        count_timer.cancel
         keepalive_timer.cancel
       end