flapjack 0.7.14 → 0.7.15

data/CHANGELOG.md

@@ -1,5 +1,15 @@
 ## Flapjack Changelog
 
+# 0.7.15 - 2013-06-27
+- Feature: Show acknowledgement duration on web interface, queryable via jabber gh-159 (@ali-graham)
+- Feature: More info on check state in email gh-207 (@ali-graham)
+- Feature: Bulk API operations gh-123 (@ali-graham)
+- Bug: You can't remove an interval from a contact's media once it has one gh-153 (@ali-graham)
+- Bug: Fix jabber identify boot time gh-172 (@ali-graham)
+- Bug: Don't pluralise singular time periods in jabber messages gh-209 (@ali-graham)
+- Bug: PUT /contacts/ID/media/MEDIA returns previous value for address gh-152 (@ali-graham)
+- Bug: 'last update' shows large numbers of seconds gh-157 (@ali-graham)
+
 # 0.7.14 - 2013-06-19
 - Bug: Display of checks on web ui with a colon in their name is screwed gh-213 (@jessereynolds)
 - Bug: show last critical, warning, unknown notificaiton times in web ui gh-211 (@jessereynolds)

data/etc/flapjack_config.yaml.example

@@ -18,6 +18,7 @@ development:
     default_contact_timezone: Australia/Broken_Hill
     archive_events: true
     events_archive_maxage: 10800
+    new_check_scheduled_maintenance_duration: 1 month
     logger:
       level: INFO
   gateways:

data/features/events.feature

@@ -390,3 +390,8 @@ Feature: events
 #     When  10 seconds passes
 #     And   an ok event is received
 #     Then  a notification should not be generated
+
+  Scenario: scheduled maintenance created for initial check reference
+    Given the check has no state
+    When  an ok event is received
+    Then  scheduled maintenance should be generated

data/features/notification_rules.feature

@@ -269,7 +269,7 @@ Feature: Notification rules on a per contact basis
     And   1 sms alert should be queued for +61400000001
   @time
   Scenario: Critical straight after test
-    Given the check is check 'ping' on entity 'foo'
+    Given the check is check 'ping' on entity 'baz'
     And   the check is in an ok state
     When  a test event is received
     Then  1 email alert should be queued for malak@example.com

data/features/steps/events_steps.rb

@@ -153,9 +153,23 @@ Given /^an entity '([\w\.\-]+)' exists$/ do |entity|
                              :redis => @redis )
 end
 
+Given /^the check is check '(.*)' on entity '([\w\.\-]+)'$/ do |check, entity|
+  @check  = check
+  @entity = entity
+end
+
+Given /^(?:the check|check '([\w\.\-]+)' for entity '([\w\.\-]+)') has no state$/ do |check, entity|
+  check  ||= @check
+  entity ||= @entity
+  remove_unscheduled_maintenance(entity, check)
+  remove_scheduled_maintenance(entity, check)
+  remove_notifications(entity, check)
+  @redis.hdel("check:#{@key}", 'state')
+end
+
 Given /^(?:the check|check '([\w\.\-]+)' for entity '([\w\.\-]+)') is in an ok state$/ do |check, entity|
-  check  = check  ? check  : @check
-  entity = entity ? entity : @entity
+  check  ||= @check
+  entity ||= @entity
   remove_unscheduled_maintenance(entity, check)
   remove_scheduled_maintenance(entity, check)
   remove_notifications(entity, check)
@@ -163,8 +177,8 @@ Given /^(?:the check|check '([\w\.\-]+)' for entity '([\w\.\-]+)') is in an ok s
 end
 
 Given /^(?:the check|check '([\w\.\-]+)' for entity '([\w\.\-]+)') is in a critical state$/ do |check, entity|
-  check  = check  ? check  : @check
-  entity = entity ? entity : @entity
+  check  ||= @check
+  entity ||= @entity
   remove_unscheduled_maintenance(entity, check)
   remove_scheduled_maintenance(entity, check)
   remove_notifications(entity, check)
@@ -172,27 +186,22 @@ Given /^(?:the check|check '([\w\.\-]+)' for entity '([\w\.\-]+)') is in a criti
 end
 
 Given /^(?:the check|check '([\w\.\-]+)' for entity '([\w\.\-]+)') is in scheduled maintenance$/ do |check, entity|
-  check  = check  ? check  : @check
-  entity = entity ? entity : @entity
+  check  ||= @check
+  entity ||= @entity
   remove_unscheduled_maintenance(entity, check)
   set_scheduled_maintenance(entity, check)
 end
 
 # TODO set the state directly rather than submit & drain
 Given /^(?:the check|check '([\w\.\-]+)' for entity '([\w\.\-]+)') is in unscheduled maintenance$/ do |check, entity|
-  check  = check  ? check  : @check
-  entity = entity ? entity : @entity
+  check  ||= @check
+  entity ||= @entity
   remove_scheduled_maintenance(entity, check)
   set_critical_state(entity, check)
   submit_acknowledgement(entity, check)
   drain_events  # TODO these should only be in When clauses
 end
 
-Given /^the check is check '(.*)' on entity '([\w\.\-]+)'$/ do |check, entity|
-  @check  = check
-  @entity = entity
-end
-
 When /^an ok event is received(?: for check '([\w\.\-]+)' on entity '([\w\.\-]+)')?$/ do |check, entity|
   check  ||= @check
   entity ||= @entity
@@ -259,6 +268,12 @@ Then /^a notification should be generated(?: for check '([\w\.\-]+)' on entity '
   found.should be_true
 end
 
+Then /^scheduled maintenance should be generated(?: for check '([\w\.\-]+)' on entity '([\w\.\-]+)')?$/ do |check, entity|
+  check  ||= @check
+  entity ||= @entity
+  @redis.get("#{entity}:#{check}:scheduled_maintenance").should_not be_nil
+end
+
 Then /^show me the (\w+ )*log$/ do |adjective|
   puts "the #{adjective}log:"
   puts @logger.messages.join("\n")

data/features/steps/notifications_steps.rb

@@ -66,7 +66,7 @@ When /^an event notification is generated for entity '([\w\.\-]+)'$/ do |entity|
                                     'entity'  => entity,
                                     'check'   => 'ping')
   entity_check = Flapjack::Data::EntityCheck.for_entity_name(entity, 'ping', :redis => @redis)
-  @app.send(:generate_notification_messages, event, entity_check)
+  @app.send(:generate_notification_messages, event, entity_check, Time.now.to_i)
 end
 
 Then /^an SMS notification for entity '([\w\.\-]+)' should be queued for the user$/ do |entity|

data/lib/flapjack/coordinator.rb

@@ -35,6 +35,8 @@ module Flapjack
     end
 
     def start(options = {})
+      @boot_time = Time.now
+
       EM.synchrony do
         add_pikelets(pikelets(@config.all))
         setup_signals if options[:signals]
@@ -131,7 +133,7 @@ module Flapjack
       start_piks = []
       pikelets_data.each_pair do |type, cfg|
         next unless pikelet = Flapjack::Pikelet.create(type,
-          :config => cfg, :redis_config => @redis_options)
+          :config => cfg, :redis_config => @redis_options, :boot_time => @boot_time)
         start_piks << pikelet
         @pikelets << pikelet
       end

data/lib/flapjack/data/contact.rb

@@ -16,7 +16,7 @@ module Flapjack
 
     class Contact
 
-      attr_accessor :id, :first_name, :last_name, :email, :media, :pagerduty_credentials
+      attr_accessor :id, :first_name, :last_name, :email, :media, :media_intervals, :pagerduty_credentials
 
       TAG_PREFIX = 'contact_tag'
 
@@ -76,6 +76,7 @@ module Flapjack
         self.first_name, self.last_name, self.email =
           @redis.hmget("contact:#{@id}", 'first_name', 'last_name', 'email')
         self.media = @redis.hgetall("contact_media:#{@id}")
+        self.media_intervals = @redis.hgetall("contact_media_intervals:#{self.id}")
 
         # similar to code in instance method pagerduty_credentials
         if service_key = @redis.hget("contact_media:#{@id}", 'pagerduty')
@@ -199,10 +200,6 @@ module Flapjack
         @redis.del("notification_rule:#{rule.id}")
       end
 
-      def media_intervals
-        @redis.hgetall("contact_media_intervals:#{self.id}")
-      end
-
       # how often to notify this contact on the given media
       # return 15 mins if no value is set
       def interval_for_media(media)
@@ -211,8 +208,12 @@ module Flapjack
       end
 
       def set_interval_for_media(media, interval)
-        raise "invalid interval" unless interval.is_a?(Integer)
+        if interval.nil?
+          @redis.hdel("contact_media_intervals:#{self.id}", media)
+          return
+        end
         @redis.hset("contact_media_intervals:#{self.id}", media, interval)
+        self.media_intervals = @redis.hgetall("contact_media_intervals:#{self.id}")
       end
 
       def set_address_for_media(media, address)
@@ -222,6 +223,7 @@ module Flapjack
           # probably best solution is to remove the need to have the username and password
           # and subdomain as pagerduty's updated api's mean we don't them anymore I think...
         end
+        self.media = @redis.hgetall("contact_media:#{@id}")
       end
 
       def remove_media(media)

data/lib/flapjack/data/entity_check.rb

@@ -22,6 +22,9 @@ module Flapjack
       STATE_CRITICAL        = 'critical'
       STATE_UNKNOWN         = 'unknown'
 
+      NOTIFICATION_STATES = [:problem, :warning, :critical, :unknown,
+                             :recovery, :acknowledgement]
+
       attr_accessor :entity, :check
 
       # TODO probably shouldn't always be creating on query -- work out when this should be happening
@@ -76,35 +79,9 @@ module Flapjack
         }
       end
 
-      # TODO move to Event
-      def create_acknowledgement(options = {})
-        event = { 'type'               => 'action',
-                  'state'              => 'acknowledgement',
-                  'summary'            => options['summary'],
-                  'duration'           => options['duration'],
-                  'acknowledgement_id' => options['acknowledgement_id'],
-                  'entity'             => entity.name,
-                  'check'              => check
-                }
-        Flapjack::Data::Event.add(event, :redis => @redis)
-      end
-
-      # TODO move to Event
-      def test_notifications(options = {})
-        event = { 'type'               => 'action',
-                  'state'              => 'test_notifications',
-                  'summary'            => options['summary'],
-                  'details'            => options['details'],
-                  'entity'             => entity.name,
-                  'check'              => check
-                }
-        Flapjack::Data::Event.add(event, :redis => @redis)
-      end
-
-      # FIXME: need to add summary to summary of existing unscheduled maintenance if there is
-      # one, and extend duration / expiry time, instead of creating a separate unscheduled
-      # outage as we are doing now...
       def create_unscheduled_maintenance(opts = {})
+        end_unscheduled_maintenance if in_unscheduled_maintenance?
+
         start_time = opts[:start_time]  # unix timestamp
         duration   = opts[:duration]    # seconds
         summary    = opts[:summary]
@@ -217,39 +194,43 @@ module Flapjack
         @redis.hget("check:#{@key}", 'state')
       end
 
-      def update_state(state, options = {})
-        return unless validate_state(state)
+      def update_state(new_state, options = {})
+        return unless [STATE_OK, STATE_WARNING,
+          STATE_CRITICAL, STATE_UNKNOWN].include?(new_state)
+
         timestamp = options[:timestamp] || Time.now.to_i
-        client = options[:client]
         summary = options[:summary]
         details = options[:details]
         count = options[:count]
 
-        # Note the current state (for speedy lookups)
-        @redis.hset("check:#{@key}", 'state', state)
-
-        # FIXME: rename to last_state_change?
-        @redis.hset("check:#{@key}", 'last_change', timestamp)
+        if self.state != new_state
+          client = options[:client]
+
+          # Note the current state (for speedy lookups)
+          @redis.hset("check:#{@key}", 'state', new_state)
+
+          # FIXME: rename to last_state_change?
+          @redis.hset("check:#{@key}", 'last_change', timestamp)
+          case state
+          when STATE_WARNING, STATE_CRITICAL, STATE_UNKNOWN
+            @redis.zadd('failed_checks', timestamp, @key)
+            # FIXME: Iterate through a list of tags associated with an entity:check pair, and update counters
+            @redis.zadd("failed_checks:client:#{client}", timestamp, @key) if client
+          else
+            @redis.zrem("failed_checks", @key)
+            # FIXME: Iterate through a list of tags associated with an entity:check pair, and update counters
+            @redis.zrem("failed_checks:client:#{client}", @key) if client
+          end
+        end
 
-        # Retain all state changes for entity:check pair
+        # Retain event data for entity:check pair
         @redis.rpush("#{@key}:states", timestamp)
-        @redis.set("#{@key}:#{timestamp}:state",   state)
+        @redis.set("#{@key}:#{timestamp}:state", new_state)
         @redis.set("#{@key}:#{timestamp}:summary", summary) if summary
         @redis.set("#{@key}:#{timestamp}:details", details) if details
         @redis.set("#{@key}:#{timestamp}:count", count) if count
 
         @redis.zadd("#{@key}:sorted_state_timestamps", timestamp, timestamp)
-
-        case state
-        when STATE_WARNING, STATE_CRITICAL, STATE_UNKNOWN
-          @redis.zadd('failed_checks', timestamp, @key)
-          # FIXME: Iterate through a list of tags associated with an entity:check pair, and update counters
-          @redis.zadd("failed_checks:client:#{client}", timestamp, @key) if client
-        else
-          @redis.zrem("failed_checks", @key)
-          # FIXME: Iterate through a list of tags associated with an entity:check pair, and update counters
-          @redis.zrem("failed_checks:client:#{client}", @key) if client
-        end
       end
 
       def last_update
@@ -268,61 +249,31 @@ module Flapjack
         lc.to_i
       end
 
-      def last_problem_notification
-        lpn = @redis.get("#{@key}:last_problem_notification")
-        return unless (lpn && lpn =~ /^\d+$/)
-        lpn.to_i
-      end
-
-      def last_warning_notification
-        lwn = @redis.get("#{@key}:last_warning_notification")
-        return unless (lwn && lwn =~ /^\d+$/)
-        lwn.to_i
-      end
-
-      def last_critical_notification
-        lcn = @redis.get("#{@key}:last_critical_notification")
-        return unless (lcn && lcn =~ /^\d+$/)
-        lcn.to_i
-      end
-
-      def last_unknown_notification
-        lcn = @redis.get("#{@key}:last_unknown_notification")
-        return unless (lcn && lcn =~ /^\d+$/)
-        lcn.to_i
-      end
-
-      def last_recovery_notification
-        lrn = @redis.get("#{@key}:last_recovery_notification")
-        return unless (lrn && lrn =~ /^\d+$/)
-        lrn.to_i
-      end
-
-      def last_acknowledgement_notification
-        lan = @redis.get("#{@key}:last_acknowledgement_notification")
-        return unless (lan && lan =~ /^\d+$/)
-        lan.to_i
+      def last_notification_for_state(state)
+        return unless NOTIFICATION_STATES.include?(state)
+        ln = @redis.get("#{@key}:last_#{state.to_s}_notification")
+        return {:timestamp => nil, :summary => nil} unless (ln && ln =~ /^\d+$/)
+        { :timestamp => ln.to_i,
+          :summary => @redis.get("#{@key}:#{ln.to_i}:summary") }
       end
 
       def last_notifications_of_each_type
-        ln = {:warning         => last_warning_notification,
-              :critical        => last_critical_notification,
-              :unknown         => last_unknown_notification,
-              :recovery        => last_recovery_notification,
-              :acknowledgement => last_acknowledgement_notification }
-        ln
+        NOTIFICATION_STATES.inject({}) do |memo, state|
+          memo[state] = last_notification_for_state(state) unless (state == :problem)
+          memo
+        end
       end
 
       def max_notified_severity_of_current_failure
-        last_recovery = last_recovery_notification || 0
+        last_recovery = last_notification_for_state(:recovery)[:timestamp] || 0
 
-        last_critical = last_critical_notification
+        last_critical = last_notification_for_state(:critical)[:timestamp]
         return STATE_CRITICAL if last_critical && (last_critical > last_recovery)
 
-        last_warning = last_warning_notification
+        last_warning = last_notification_for_state(:warning)[:timestamp]
         return STATE_WARNING if last_warning && (last_warning > last_recovery)
 
-        last_unknown = last_unknown_notification
+        last_unknown = last_notification_for_state(:unknown)[:timestamp]
         return STATE_UNKNOWN if last_unknown && (last_unknown > last_recovery)
 
         nil
@@ -331,15 +282,16 @@ module Flapjack
       # unpredictable results if there are multiple notifications of different
       # types sent at the same time
       def last_notification
-        nils = { :type => nil, :timestamp => nil }
+        nils = { :type => nil, :timestamp => nil, :summary => nil }
+
         lne = last_notifications_of_each_type
-        ln = lne.delete_if {|type, timestamp| timestamp.nil? || timestamp.to_i == 0 }
-        if ln.find {|type, timestamp| type == :warning or type == :critical}
-          ln = ln.delete_if {|type, timestamp| type == :problem }
+        ln = lne.delete_if {|type, notif| notif[:timestamp].nil? || notif[:timestamp].to_i <= 0 }
+        if ln.find {|type, notif| type == :warning or type == :critical}
+          ln = ln.delete_if {|type, notif| type == :problem }
         end
-        return nils unless ln.length > 0
-        lns = ln.sort_by { |type, timestamp| timestamp }.last
-        { :type => lns[0], :timestamp => lns[1] }
+        return nils if ln.empty?
+        lns = ln.sort_by { |type, notif| notif[:timestamp] }.last
+        { :type => lns[0], :timestamp => lns[1][:timestamp], :summary => lns[1][:summary] }
       end
 
       def event_count_at(timestamp)
@@ -373,20 +325,37 @@ module Flapjack
       # from midnight to 11:59:59 PM. Pass nil for either end to leave that
       # side unbounded.
       def historical_states(start_time, end_time, opts = {})
-        start_time ||= '-inf'
-        end_time ||= '+inf'
+        start_time = '-inf' if start_time.to_i <= 0
+        end_time = '+inf' if end_time.to_i <= 0
+
+        args = ["#{@key}:sorted_state_timestamps"]
+
         order = opts[:order]
-        query = (order && 'desc'.eql?(order.downcase)) ? :zrevrangebyscore : :zrangebyscore
-        state_ts = @redis.send(query, "#{@key}:sorted_state_timestamps", start_time, end_time)
+        if (order && 'desc'.eql?(order.downcase))
+          query = :zrevrangebyscore
+          args += [end_time.to_s, start_time.to_s]
+        else
+          query = :zrangebyscore
+          args += [start_time.to_s, end_time.to_s]
+        end
+
+        if opts[:limit] && (opts[:limit].to_i > 0)
+          args << {:limit => [0, opts[:limit]]}
+        end
+
+        state_ts = @redis.send(query, *args)
 
         state_data = nil
 
         @redis.multi do |r|
           state_data = state_ts.collect {|ts|
-            {:timestamp => ts.to_i,
-             :state     => r.get("#{@key}:#{ts}:state"),
-             :summary   => r.get("#{@key}:#{ts}:summary"),
-             :details   => r.get("#{@key}:#{ts}:details")}
+            {:timestamp     => ts.to_i,
+             :state         => r.get("#{@key}:#{ts}:state"),
+             :summary       => r.get("#{@key}:#{ts}:summary"),
+             :details       => r.get("#{@key}:#{ts}:details"),
+             # :count         => r.get("#{@key}:#{ts}:count"),
+             # :check_latency => r.get("#{@key}:#{ts}:check_latency")
+            }
           }
         end
 
@@ -405,7 +374,7 @@ module Flapjack
       # if any.
       def historical_state_before(timestamp)
         pos = @redis.zrank("#{@key}:sorted_state_timestamps", timestamp)
-        return if pos < 1
+        return if pos.nil? || pos < 1
         ts = @redis.zrange("#{@key}:sorted_state_timestamps", pos - 1, pos)
         return if ts.nil? || ts.empty?
         {:timestamp => ts.first.to_i,
@@ -475,10 +444,6 @@ module Flapjack
         @key = "#{entity.name}:#{check}"
       end
 
-      def validate_state(state)
-        [STATE_OK, STATE_WARNING, STATE_CRITICAL, STATE_UNKNOWN].include?(state)
-      end
-
     end
 
   end