flapjack 0.6.24 → 0.6.25

data/etc/flapjack_config.yaml.example

@@ -42,6 +42,21 @@ development:
     rooms:
       - "gimp@conference.jabber.domain.tld"
       - "log@conference.jabber.domain.tld"
+  oobetet:
+    enabled: yes
+    server: "jabber.domain.tld"
+    port: 5222
+    jabberid: "flapjacktest@jabber.domain.tld"
+    password: "nuther-good-password"
+    alias: "flapjacktest"
+    watched_check: "PING"
+    watched_entity: "foo.bar.net"
+    max_latency: 300
+    pagerduty_contact: "11111111111111111111111111111111"
+    rooms:
+      - "flapjacktest@conference.jabber.domain.tld"
+      - "gimp@conference.jabber.domain.tld"
+      - "log@conference.jabber.domain.tld"
   pagerduty_gateway:
     enabled: yes
     queue: pagerduty_notifications

data/features/events.feature

@@ -11,7 +11,6 @@ Feature: events
     Given check 'abc' for entity 'def' is in an ok state
     When  an ok event is received for check 'abc' on entity 'def'
     Then  a notification should not be generated for check 'abc' on entity 'def'
-    # And   show me the output
 
   Scenario: Check ok to failed
     Given check 'abc' for entity 'def' is in an ok state
@@ -40,11 +39,9 @@ Feature: events
     When  a failure event is received for check 'abc' on entity 'def'
     And   1 minute passes
     And   a failure event is received for check 'abc' on entity 'def'
-    And   show me the notifications
     Then  a notification should be generated for check 'abc' on entity 'def'
     When  1 minute passes
     And   a failure event is received for check 'abc' on entity 'def'
-    And   show me the notifications
     Then  a notification should not be generated for check 'abc' on entity 'def'
 
   @time
@@ -121,6 +118,31 @@ Feature: events
     And   a failure event is received for check 'abc' on entity 'def'
     Then  a notification should be generated for check 'abc' on entity 'def'
 
+  @time
+  Scenario: Osciliating state, period of two minutes
+    Given check 'abc' for entity 'def' is in an ok state
+    When  a failure event is received for check 'abc' on entity 'def'
+    Then  a notification should not be generated for check 'abc' on entity 'def'
+    When  50 seconds passes
+    And   a failure event is received for check 'abc' on entity 'def'
+    Then  a notification should be generated for check 'abc' on entity 'def'
+    When  10 seconds passes
+    And   an ok event is received for check 'abc' on entity 'def'
+    Then  a notification should be generated for check 'abc' on entity 'def'
+    When  50 seconds passes
+    And   an ok event is received for check 'abc' on entity 'def'
+    Then  a notification should not be generated for check 'abc' on entity 'def'
+    When  10 seconds passes
+    And   a failure event is received for check 'abc' on entity 'def'
+    Then  a notification should not be generated for check 'abc' on entity 'def'
+    When  50 seconds passes
+    And   a failure event is received for check 'abc' on entity 'def'
+    #And   show me the notifications
+    Then  a notification should be generated for check 'abc' on entity 'def'
+    When  10 seconds passes
+    And   an ok event is received for check 'abc' on entity 'def'
+    Then  a notification should be generated for check 'abc' on entity 'def'
+
   Scenario: Acknowledgement when ok
     Given check 'abc' for entity 'def' is in an ok state
     When  an acknowledgement event is received for check 'abc' on entity 'def'

data/lib/flapjack/data/entity_check.rb

@@ -248,6 +248,27 @@ module Flapjack
         lan.to_i
       end
 
+      def last_notifications_of_each_type
+        ln = {:problem         => last_problem_notification,
+              :recovery        => last_recovery_notification,
+              :acknowledgement => last_acknowledgement_notification }
+        puts "***** last_notifications_of_each_type for #{@key.inspect}: #{ln.inspect}"
+        ln
+      end
+
+      # unpredictable results if there are multiple notifications of different
+      # types sent at the same time
+      def last_notification
+        nils = { :type => nil, :timestamp => nil }
+        lne = last_notifications_of_each_type
+        ln = lne.delete_if {|type, timestamp|
+          timestamp.nil? || timestamp.to_i == 0
+        }
+        return nils unless ln.length > 0
+        lns = ln.sort_by { |type, timestamp| timestamp }.last
+        { :type => lns[0], :timestamp => lns[1] }
+      end
+
       def event_count_at(timestamp)
         eca = @redis.get("#{@key}:#{timestamp}:count")
         return unless (eca && eca =~ /^\d+$/)

data/lib/flapjack/filters/delays.rb

@@ -9,7 +9,7 @@ module Flapjack
     # * If the service event’s state is a failure, and the time since the ok => failure state change
     #   is below a threshold (e.g. 30 seconds), then don't alert
     # * If the service event’s state is a failure, and the time since the last alert is below a
-    #   threshold (5 minutes), then don’t alert
+    #   threshold (5 minutes), and the last notification was not a recovery, then don’t alert
     class Delays
       include Base
 
@@ -25,20 +25,32 @@ module Flapjack
           current_time = Time.now.to_i
 
           if entity_check.failed?
-            last_problem_alert = entity_check.last_problem_notification
-            last_change        = entity_check.last_change
+            last_problem_alert   = entity_check.last_problem_notification
+            last_change          = entity_check.last_change
+            last_notification    = entity_check.last_notification
+            last_alert_type      = last_notification[:type]
+            last_alert_timestamp = last_notification[:timestamp]
 
             current_failure_duration = current_time - last_change
             time_since_last_alert    = current_time - last_problem_alert unless last_problem_alert.nil?
-            @log.debug("Filter: Delays: last_problem_alert: #{last_problem_alert.to_s}, last_change: #{last_change.to_s}, current_failure_duration: #{current_failure_duration}, time_since_last_alert: #{time_since_last_alert.to_s}")
+            @log.debug("Filter: Delays: last_problem_alert: #{last_problem_alert.to_s}, " +
+                       "last_change: #{last_change.to_s}, " +
+                       "current_failure_duration: #{current_failure_duration}, " +
+                       "time_since_last_alert: #{time_since_last_alert.to_s}")
             if (current_failure_duration < failure_delay)
               result = true
-              @log.debug("Filter: Delays: blocking because duration of current failure (#{current_failure_duration}) is less than failure_delay (#{failure_delay})")
-            elsif !last_problem_alert.nil? && (time_since_last_alert < resend_delay)
+              @log.debug("Filter: Delays: blocking because duration of current failure " +
+                         "(#{current_failure_duration}) is less than failure_delay (#{failure_delay})")
+            elsif !last_problem_alert.nil? && (time_since_last_alert < resend_delay) &&
+              (last_alert_type !~ /recovery/i)
+
               result = true
-              @log.debug("Filter: Delays: blocking because time since last alert for current problem (#{time_since_last_alert}) is less than resend_delay (#{resend_delay})")
+              @log.debug("Filter: Delays: blocking because time since last alert for " +
+                         "current problem (#{time_since_last_alert}) is less than " +
+                         "resend_delay (#{resend_delay}) and last alert type (#{last_alert_type}) was not a recovery")
             else
-              @log.debug("Filter: Delays: not blocking because neither of the time comparison conditions were met")
+              @log.debug("Filter: Delays: not blocking because neither of the time comparison " +
+                         "conditions were met")
             end
           else
             @log.debug("Filter: Delays: entity_check.failed? returned false ...")

data/lib/flapjack/jabber.rb

@@ -32,9 +32,13 @@ module Flapjack
     log.level = Logger::INFO
     Blather.logger = log
 
+    def initialize
+      @buffer = []
+      @hostname = Socket.gethostname
+    end
+
     def setup
       @redis = build_redis_connection_pool
-      @hostname = Socket.gethostname
       @flapjack_jid = Blather::JID.new((@config['jabberid'] || 'flapjack') + '/' + @hostname)
 
       super(@flapjack_jid, @config['password'], @config['server'], @config['port'].to_i)
@@ -96,10 +100,13 @@ module Flapjack
           say(room, "flapjack jabber gateway started at #{Time.now}, hello!", :groupchat)
         end
       end
+      return if @buffer.empty?
+      while stanza = @buffer.shift
+        @logger.debug("Sending a buffered jabber message to: #{stanza.to}, using: #{stanza.type}, message: #{stanza.body}")
+      end
     end
 
     def interpreter(command)
-
       msg          = nil
       action       = nil
       entity_check = nil
@@ -219,8 +226,14 @@ module Flapjack
     end
 
     def say(to, msg, using = :chat)
-      @logger.debug("Sending a jabber message to: #{to.to_s}, using: #{using.to_s}, message: #{msg}")
-      write Blather::Stanza::Message.new(to, msg, using)
+      stanza = Blather::Stanza::Message.new(to, msg, using)
+      if connected?
+        @logger.debug("Sending a jabber message to: #{to.to_s}, using: #{using.to_s}, message: #{msg}")
+        write(stanza)
+      else
+        @logger.debug("Buffering a jabber message to: #{to.to_s}, using: #{using.to_s}, message: #{msg}")
+        @buffer << stanza
+      end
     end
 
     def add_shutdown_event(opts = {})

data/lib/flapjack/version.rb

@@ -1,5 +1,5 @@
 #!/usr/bin/env ruby
 
 module Flapjack
-  VERSION = "0.6.24"
+  VERSION = "0.6.25"
 end

data/lib/flapjack/web.rb

@@ -62,6 +62,7 @@ module Flapjack
     end
 
     get '/check' do
+      begin
       @entity = params[:entity]
       @check = params[:check]
 
@@ -74,11 +75,7 @@ module Flapjack
       @check_last_update          = entity_check.last_update
       @check_last_change          = last_change
       @check_summary              = entity_check.summary
-      @last_notifications         =
-        {:problem         => entity_check.last_problem_notification,
-         :recovery        => entity_check.last_recovery_notification,
-         :acknowledgement => entity_check.last_acknowledgement_notification
-        }
+      @last_notifications         = entity_check.last_notifications_of_each_type
       @in_scheduled_maintenance   = entity_check.in_scheduled_maintenance?
       @in_unscheduled_maintenance = entity_check.in_unscheduled_maintenance?
       @scheduled_maintenances     = entity_check.maintenances(nil, nil, :scheduled => true)
@@ -86,15 +83,20 @@ module Flapjack
         entity_check.event_count_at(entity_check.last_change) : nil
 
       haml :check
+      rescue Exception => e
+        puts e.message
+        puts e.backtrace.join("\n")
+      end
+
     end
 
     post '/acknowledgements/:entity/:check' do
-      @entity = params[:entity]
-      @check = params[:check]
-      @summary = params[:summary]
+      @entity             = params[:entity]
+      @check              = params[:check]
+      @summary            = params[:summary]
       @acknowledgement_id = params[:acknowledgement_id]
 
-      dur   = ChronicDuration.parse(params[:duration] || '')
+      dur = ChronicDuration.parse(params[:duration] || '')
       @duration = (dur.nil? || (dur <= 0)) ? (4 * 60 * 60) : dur
 
       entity_check = get_entity_check(@entity, @check)
@@ -102,6 +104,8 @@ module Flapjack
 
       ack = entity_check.create_acknowledgement('summary' => (@summary || ''),
         'acknowledgement_id' => @acknowledgement_id, 'duration' => @duration)
+
+      # FIXME: make this a flash message on the check page and delete the acknowledge page
       @acknowledge_success = !!ack
       [201, haml(:acknowledge)]
     end

data/spec/lib/flapjack/jabber_spec.rb

@@ -48,6 +48,7 @@ describe Flapjack::Jabber do
     fj = Flapjack::Jabber.new
     fj.bootstrap(:config => config)
 
+    fj.should_receive(:connected?).and_return(true)
     fj.should_receive(:write).with(an_instance_of(Blather::Stanza::Presence))
     fj.should_receive(:write).with(an_instance_of(Blather::Stanza::Message))
 
@@ -79,6 +80,7 @@ describe Flapjack::Jabber do
     fj.bootstrap(:config => config)
     fj.instance_variable_set('@redis_handler', redis)
 
+    fj.should_receive(:connected?).and_return(true)
     fj.should_receive(:write).with(an_instance_of(Blather::Stanza::Message))
 
     fj.on_groupchat(stanza)
@@ -93,6 +95,7 @@ describe Flapjack::Jabber do
     fj = Flapjack::Jabber.new
     fj.bootstrap(:config => config)
 
+    fj.should_receive(:connected?).and_return(true)
     fj.should_receive(:write).with(an_instance_of(Blather::Stanza::Message))
 
     fj.on_groupchat(stanza)
@@ -132,9 +135,10 @@ describe Flapjack::Jabber do
 
     fj = Flapjack::Jabber.new
     fj.bootstrap(:config => config)
+    fj.should_receive(:register_handler).exactly(4).times
 
     fj.should_receive(:connect)
-    fj.should_receive(:connected?).twice.and_return(true)
+    fj.should_receive(:connected?).exactly(3).times.and_return(true)
     fj.should_receive(:should_quit?).exactly(3).times.and_return(false, false, true)
     redis.should_receive(:blpop).twice.and_return(
       ["jabber_notifications", %q{{"notification_type":"problem","event_id":"main-example.com:ping","state":"critical","summary":"!!!"}}],

data/spec/lib/flapjack/web_spec.rb

@@ -92,13 +92,15 @@ describe Flapjack::Web, :sinatra => true, :redis => true do
   it "shows the state of a check for an entity" do
     time = Time.now.to_i
 
+    last_notifications = {:problem         => time - ((3 * 60 * 60) + (5 * 60)),
+                          :recovery        => time - (3 * 60 * 60),
+                          :acknowledgement => nil }
+
     entity_check.should_receive(:state).and_return('ok')
     entity_check.should_receive(:last_update).and_return(time - (3 * 60 * 60))
     entity_check.should_receive(:last_change).and_return(time - (3 * 60 * 60))
     entity_check.should_receive(:summary).and_return('all good')
-    entity_check.should_receive(:last_problem_notification).and_return(time - ((3 * 60 * 60) + (5 * 60)))
-    entity_check.should_receive(:last_recovery_notification).and_return(time - (3 * 60 * 60))
-    entity_check.should_receive(:last_acknowledgement_notification).and_return(nil)
+    entity_check.should_receive(:last_notifications_of_each_type).and_return(last_notifications)
     entity_check.should_receive(:in_scheduled_maintenance?).and_return(false)
     entity_check.should_receive(:in_unscheduled_maintenance?).and_return(false)
     entity_check.should_receive(:maintenances).with(nil, nil, :scheduled => true).and_return([])
@@ -185,4 +187,4 @@ describe Flapjack::Web, :sinatra => true, :redis => true do
     last_response.status.should == 302
   end
 
-end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flapjack
 version: !ruby/object:Gem::Version
-  version: 0.6.24
+  version: 0.6.25
   prerelease: 
 platform: ruby
 authors:
@@ -11,7 +11,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-09-20 00:00:00.000000000 Z
+date: 2012-09-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: daemons