flapjack 0.6.61 → 0.7.0

data/features/notification_rules.feature

@@ -0,0 +1,93 @@
+@notification_rules @resque
+Feature: Notification rules on a per contact basis
+
+  Background:
+    Given the following users exist:
+      | id  | first_name | last_name | email             | sms          |
+      | 1   | Malak      | Al-Musawi | malak@example.com | +61400000001 |
+      | 2   | Imani      | Farooq    | imani@example.com | +61400000002 |
+
+    And the following entities exist:
+      | id  | name | contacts |
+      | 1   | foo  | 1        |
+      | 2   | bar  | 1,2      |
+
+    And user 1 has the following notification intervals:
+      | email | sms |
+      | 15    | 60  |
+
+    And user 1 has the following notification rules:
+      | id | entities | entity_tags | warning_media | critical_media   | warning_blackhole | critical_blackhole | time_restrictions |
+      | 1  | foo      |             | email         | sms,email        |                   |                    | 8-18 weekdays     |
+      | 2  | bar      |             |               | sms,email        | true              |                    |                   |
+
+  @time_restrictions @time
+  Scenario: Alerts only during specified time restrictions
+    Given the timezone is America/New_York
+    And   the time is February 1 2013 6:59
+    And   the check is check 'ping' on entity 'foo'
+    And   the check is in an ok state
+    And   a critical event is received
+    Then  no email alerts should be queued for malak@example.com
+    And   the time is February 1 2013 7:01
+    And   a critical event is received
+    Then  no email alerts should be queued for malak@example.com
+    And   the time is February 1 2013 8:01
+    And   a critical event is received
+    Then  1 email alert should be queued for malak@example.com
+    When  the time is February 1 2013 12:00
+    Then  all alert dropping keys for user 1 should have expired
+    When  a critical event is received
+    Then  2 email alerts should be queued for malak@example.com
+    When  the time is February 1 2013 17:59
+    Then  all alert dropping keys for user 1 should have expired
+    When  a critical event is received
+    Then  3 email alerts should be queued for malak@example.com
+    When  the time is February 1 2013 18:01
+    Then  all alert dropping keys for user 1 should have expired
+    When  a critical event is received
+    Then  3 email alerts should be queued for malak@example.com
+
+  Scenario: time restrictions continue to work as expected when a contact changes timezone
+
+  @severity @time
+  Scenario: Don't alert when media,severity does not match any matching rule's severity's media
+    Given the check is check 'ping' on entity 'bar'
+    And   the check is in an ok state
+    When  a warning event is received
+    And   60 minutes passes
+    And   a warning event is received
+    Then  no email alerts should be queued for malak@example.com
+
+  @severity @time
+  Scenario: Alerts only when media,severity matches any matching rule's severity's media with ok->warning->critical
+    Given the check is check 'ping' on entity 'bar'
+    And   the check is in an ok state
+    When  a warning event is received
+    And   1 minute passes
+    And   a warning event is received
+    Then  no email alerts should be queued for malak@example.com
+    When  a critical event is received
+    And   5 minute passes
+    And   a critical event is received
+    Then  1 email alert should be queued for malak@example.com
+
+  @blackhole
+  Scenario: Drop alerts matching a blackhole rule
+
+  @intervals @time
+  Scenario: Alerts according to custom interval
+    Given the check is check 'ping' on entity 'bar'
+    And   the check is in an ok state
+    When  a critical event is received
+    Then  no email alerts should be queued for malak@example.com
+    When  1 minute passes
+    And   a critical event is received
+    Then  1 email alert should be queued for malak@example.com
+    When  10 minutes passes
+    And   a critical event is received
+    Then  1 email alert should be queued for malak@example.com
+    When  5 minutes passes
+    And   a critical event is received
+    Then  2 email alerts should be queued for malak@example.com
+

data/features/steps/events_steps.rb

@@ -1,8 +1,5 @@
 #!/usr/bin/env ruby
 
-require 'flapjack/data/entity_check'
-require 'flapjack/data/event'
-
 def drain_events
   loop do
     event = Flapjack::Data::Event.next(:block => false, :redis => @redis)
@@ -53,12 +50,18 @@ def set_ok_state(entity, check)
     :timestamp => (Time.now.to_i - (60*60*24)))
 end
 
-def set_failure_state(entity, check)
+def set_critical_state(entity, check)
   entity_check = Flapjack::Data::EntityCheck.for_entity_name(entity, check, :redis => @redis)
   entity_check.update_state(Flapjack::Data::EntityCheck::STATE_CRITICAL,
     :timestamp => (Time.now.to_i - (60*60*24)))
 end
 
+def set_warning_state(entity, check)
+  entity_check = Flapjack::Data::EntityCheck.for_entity_name(entity, check, :redis => @redis)
+  entity_check.update_state(Flapjack::Data::EntityCheck::STATE_WARNING,
+    :timestamp => (Time.now.to_i - (60*60*24)))
+end
+
 def submit_ok(entity, check)
   event = {
     'type'    => 'service',
@@ -71,6 +74,18 @@ def submit_ok(entity, check)
   submit_event(event)
 end
 
+def submit_warning(entity, check)
+  event = {
+    'type'    => 'service',
+    'state'   => 'warning',
+    'summary' => '25% packet loss',
+    'entity'  => entity,
+    'check'   => check,
+    'client'  => 'clientx'
+  }
+  submit_event(event)
+end
+
 def submit_critical(entity, check)
   event = {
     'type'    => 'service',
@@ -102,62 +117,188 @@ Given /^an entity '([\w\.\-]+)' exists$/ do |entity|
                              :redis => @redis )
 end
 
-Given /^^check '([\w\.\-]+)' for entity '([\w\.\-]+)' is in an ok state$/ do |check, entity|
+Given /^(?:the check|check '([\w\.\-]+)' for entity '([\w\.\-]+)') is in an ok state$/ do |check, entity|
+  check  = check  ? check  : @check
+  entity = entity ? entity : @entity
   remove_unscheduled_maintenance(entity, check)
   remove_scheduled_maintenance(entity, check)
   remove_notifications(entity, check)
   set_ok_state(entity, check)
 end
 
-Given /^check '([\w\.\-]+)' for entity '([\w\.\-]+)' is in a failure state$/ do |check, entity|
+Given /^(?:the check|check '([\w\.\-]+)' for entity '([\w\.\-]+)') is in a critical state$/ do |check, entity|
+  check  = check  ? check  : @check
+  entity = entity ? entity : @entity
   remove_unscheduled_maintenance(entity, check)
   remove_scheduled_maintenance(entity, check)
   remove_notifications(entity, check)
-  set_failure_state(entity, check)
+  set_critical_state(entity, check)
 end
 
-Given /^check '([\w\.\-]+)' for entity '([\w\.\-]+)' is in scheduled maintenance$/ do |check, entity|
+Given /^(?:the check|check '([\w\.\-]+)' for entity '([\w\.\-]+)') is in scheduled maintenance$/ do |check, entity|
+  check  = check  ? check  : @check
+  entity = entity ? entity : @entity
   remove_unscheduled_maintenance(entity, check)
   set_scheduled_maintenance(entity, check)
 end
 
 # TODO set the state directly rather than submit & drain
-Given /^check '([\w\.\-]+)' for entity '([\w\.\-]+)' is in unscheduled maintenance$/ do |check, entity|
+Given /^(?:the check|check '([\w\.\-]+)' for entity '([\w\.\-]+)') is in unscheduled maintenance$/ do |check, entity|
+  check  = check  ? check  : @check
+  entity = entity ? entity : @entity
   remove_scheduled_maintenance(entity, check)
-  set_failure_state(entity, check)
+  set_critical_state(entity, check)
   submit_acknowledgement(entity, check)
   drain_events  # TODO these should only be in When clauses
 end
 
-When /^an ok event is received for check '([\w\.\-]+)' on entity '([\w\.\-]+)'$/ do |check, entity|
+Given /^the check is check '(.*)' on entity '([\w\.\-]+)'$/ do |check, entity|
+  @check  = check
+  @entity = entity
+end
+
+When /^an ok event is received(?: for check '([\w\.\-]+)' on entity '([\w\.\-]+)')?$/ do |check, entity|
+  check  ||= @check
+  entity ||= @entity
   submit_ok(entity, check)
   drain_events
 end
 
-When /^a failure event is received for check '([\w\.\-]+)' on entity '([\w\.\-]+)'$/ do |check, entity|
+When /^a failure event is received(?: for check '([\w\.\-]+)' on entity '([\w\.\-]+)')?$/ do |check, entity|
+  check  ||= @check
+  entity ||= @entity
   submit_critical(entity, check)
   drain_events
 end
 
-When /^an acknowledgement .*is received for check '([\w\.\-]+)' on entity '([\w\.\-]+)'$/ do |check, entity|
-  submit_acknowledgement(entity, check)
+When /^a critical event is received(?: for check '([\w\.\-]+)' on entity '([\w\.\-]+)')?$/ do |check, entity|
+  check  ||= @check
+  entity ||= @entity
+  submit_critical(entity, check)
   drain_events
 end
 
+When /^a warning event is received(?: for check '([\w\.\-]+)' on entity '([\w\.\-]+)')?$/ do |check, entity|
+  check  ||= @check
+  entity ||= @entity
+  submit_warning(entity, check)
+  drain_events
+end
+
+When /^an acknowledgement .*is received(?: for check '([\w\.\-]+)' on entity '([\w\.\-]+)')?$/ do |check, entity|
+  check  ||= @check
+  entity ||= @entity
+  submit_acknowledgement(entity, check)
+  drain_events
+end
 
 # TODO logging is a side-effect, should test for notification generation itself
-Then /^a notification should not be generated for check '([\w\.\-]+)' on entity '([\w\.\-]+)'$/ do |check, entity|
+Then /^a notification should not be generated(?: for check '([\w\.\-]+)' on entity '([\w\.\-]+)')?$/ do |check, entity|
+  check  ||= @check
+  entity ||= @entity
   message = @logger.messages.find_all {|m| m =~ /enerating notifications for event #{entity}:#{check}/ }.last
-  message ? happy = message.match(/Not generating notifications/) : happy = false
-  happy.should be_true
+  found = message ? message.match(/Not generating notifications/) : false
+  found.should be_true
 end
 
-Then /^a notification should be generated for check '([\w\.\-]+)' on entity '([\w\.\-]+)'$/ do |check, entity|
+Then /^a notification should be generated(?: for check '([\w\.\-]+)' on entity '([\w\.\-]+)')?$/ do |check, entity|
+  check  ||= @check
+  entity ||= @entity
   message = @logger.messages.find_all {|m| m =~ /enerating notifications for event #{entity}:#{check}/ }.last
-  message ? happy = message.match(/Generating notifications/) : happy = false
-  happy.should be_true
+  found = message ? message.match(/Generating notifications/) : false
+  found.should be_true
 end
 
-Then /^show me the notifications?$/ do
+Then /^show me the log$/ do
   puts @logger.messages.join("\n")
 end
+
+# added for notification rules:
+
+Given /^the following entities exist:$/ do |entities|
+  entities.hashes.each do |entity|
+    contacts = entity['contacts'].split(',')
+    contacts.map! do |contact|
+      contact.strip
+    end
+    #puts "adding entity #{entity['name']} (#{entity['id']}) with contacts: [#{contacts.join(', ')}]"
+    Flapjack::Data::Entity.add({'id'       => entity['id'],
+                                'name'     => entity['name'],
+                                'contacts' => contacts},
+                               :redis => @redis )
+  end
+end
+
+Given /^the following users exist:$/ do |contacts|
+  contacts.hashes.each do |contact|
+    media = {}
+    media['email'] = contact['email']
+    media['sms']   = contact['sms']
+    Flapjack::Data::Contact.add({'id'         => contact['id'],
+                                 'first_name' => contact['first_name'],
+                                 'last_name'  => contact['last_name'],
+                                 'email'      => contact['email'],
+                                 'media'      => media},
+                                :redis => @redis )
+  end
+end
+
+Given /^user (\d+) has the following notification intervals:$/ do |contact_id, intervals|
+  contact = Flapjack::Data::Contact.find_by_id(contact_id, :redis => @redis)
+  intervals.hashes.each do |interval|
+    contact.set_interval_for_media('email', interval['email'].to_i * 60)
+    contact.set_interval_for_media('sms',   interval['sms'].to_i * 60)
+  end
+end
+
+Given /^user (\d+) has the following notification rules:$/ do |contact_id, rules|
+  rules.hashes.each do |rule|
+    entities           = rule['entities'].split(',').map { |x| x.strip }
+    entity_tags        = rule['entity_tags'].split(',').map { |x| x.strip }
+    warning_media      = rule['warning_media'].split(',').map { |x| x.strip }
+    critical_media     = rule['critical_media'].split(',').map { |x| x.strip }
+    warning_blackhole  = rule['warning_blackhole'].downcase == 'true' ? 'true' : 'false'
+    critical_blackhole = rule['critical_blackhole'].downcase == 'true' ? 'true' : 'false'
+    time_restrictions  = []
+    rule['time_restrictions'].split(',').map { |x| x.strip }.each do |time_restriction|
+      case time_restriction
+      when '8-18 weekdays'
+        # FIXME: get timezone from the user definition (or config[:default_contact_timezone])
+        time_zone = ActiveSupport::TimeZone.new("America/New_York")
+        weekdays_8_18 = IceCube::Schedule.new(time_zone.local(2013,2,1,8,0,0), :duration => 60 * 60 * 10)
+        weekdays_8_18.add_recurrence_rule(IceCube::Rule.weekly.day(:monday, :tuesday, :wednesday, :thursday, :friday))
+        time_restrictions << Flapjack::Data::NotificationRule.time_restriction_from_ice_cube_hash(weekdays_8_18.to_hash, time_zone)
+      end
+    end
+    Flapjack::Data::NotificationRule.add({:contact_id         => contact_id,
+                                          :entities           => entities,
+                                          :entity_tags        => entity_tags,
+                                          :warning_media      => warning_media,
+                                          :critical_media     => critical_media,
+                                          :warning_blackhole  => warning_blackhole,
+                                          :critical_blackhole => critical_blackhole,
+                                          :time_restrictions  => time_restrictions}, :redis => @redis)
+  end
+end
+
+Then /^all alert dropping keys for user (\d+) should have expired$/ do |contact_id|
+  @redis.keys("drop_alerts_for_contact:#{contact_id}*").should be_empty
+end
+
+# When /^the (\w*) alert block for user (\d*) for (?:the check|check '([\w\.\-]+)' for entity '([\w\.\-]+)') for state (.*) expires$/ do |media, contact, check, entity, state|
+#   check  = check  ? check  : @check
+#   entity = entity ? entity : @entity
+#   num_deleted = @redis.del("drop_alerts_for_contact:#{contact}:#{media}:#{entity}:#{check}:#{state}")
+#   puts "Warning: no keys expired" unless num_deleted > 0
+# end
+
+Then /^(.*) email alert(?:s)? should be queued for (.*)$/ do |num_queued, address|
+  check  = check  ? check  : @check
+  entity = entity ? entity : @entity
+  case num_queued
+  when 'no'
+    num_queued = 0
+  end
+  queue  = Resque.peek('email_notifications', 0, 30)
+  queue.find_all {|n| n['args'].first['address'] == address }.length.should == num_queued.to_i
+end

data/features/steps/notifications_steps.rb

@@ -66,7 +66,7 @@ When /^an event notification is generated for entity '([\w\.\-]+)'$/ do |entity|
                                     'entity'  => entity,
                                     'check'   => 'ping')
   entity_check = Flapjack::Data::EntityCheck.for_entity_name(entity, 'ping', :redis => @redis)
-  @app.send(:send_notification_messages, event, entity_check)
+  @app.send(:generate_notification_messages, event, entity_check)
 end
 
 Then /^an SMS notification for entity '([\w\.\-]+)' should be queued for the user$/ do |entity|

data/features/steps/time_travel_steps.rb

@@ -2,33 +2,44 @@
 
 require 'delorean'
 require 'chronic'
+require 'active_support/time'
 
 When /^(.+) passes$/ do |time|
   period = Chronic.parse("#{time} from now")
-  Delorean.time_travel_to(period)
-  # puts "Time Travelled to #{Time.now.to_s}"
+  RedisDelorean.time_travel_to(period)
+  #puts "Time Travelled to #{Time.now.to_s}"
 end
 
-Given /^I time travel to (.+)$/ do |period|
-  Delorean.time_travel_to(period)
-  # puts "Time Travelled to #{Time.now.to_s}"
-end
+# Given /^I time travel to (.+)$/ do |period|
+#   RedisDelorean.time_travel_to(period)
+#   # puts "Time Travelled to #{Time.now.to_s}"
+# end
 
-Given /^I come back to the present$/ do
-  Delorean.back_to_the_present
-  # puts "Time Travelled to the present, #{Time.now.to_s}"
+Given /^the timezone is (.*)$/ do |tz|
+  Time.zone = tz
+  Chronic.time_class = Time.zone
 end
 
-Given /^I time travel in (.+) to (.+)$/ do |zone_name, timestamp|
-  zone = ::Time.find_zone!(zone_name)
-  time = zone.parse timestamp
-  Delorean.time_travel_to time
-  # puts "Time Travelled to #{Time.now.to_s}"
+Given /^the time is (.*)$/ do |time|
+  RedisDelorean.time_travel_to(Chronic.parse("#{time}"))
+  #puts "Time Travelled to #{Time.now.to_s}"
 end
 
-Then /^the time in UTC should be about (.+)$/ do |timestamp|
-  actual = Time.now.in_time_zone('UTC')
-  expected = Time.parse("#{timestamp} UTC")
-  (expected..expected+5).cover?(actual).should be_true
-end
+# Given /^I come back to the present$/ do
+#   RedisDelorean.back_to_the_present
+#   # puts "Time Travelled to the present, #{Time.now.to_s}"
+# end
+
+# Given /^I time travel in (.+) to (.+)$/ do |zone_name, timestamp|
+#   zone = ::Time.find_zone!(zone_name)
+#   time = zone.parse timestamp
+#   RedisDelorean.time_travel_to time
+#   # puts "Time Travelled to #{Time.now.to_s}"
+# end
+
+# Then /^the time in UTC should be about (.+)$/ do |timestamp|
+#   actual = Time.now.in_time_zone('UTC')
+#   expected = Time.parse("#{timestamp} UTC")
+#   (expected..expected+5).cover?(actual).should be_true
+# end
 

data/features/support/env.rb

@@ -1,4 +1,11 @@
 #!/usr/bin/env ruby
+#
+require 'delorean'
+require 'chronic'
+require 'active_support/time'
+require 'ice_cube'
+require 'flapjack/data/entity_check'
+require 'flapjack/data/event'
 
 if ENV['COVERAGE']
   require 'simplecov'
@@ -43,9 +50,66 @@ class MockEmailer
   include EM::Deferrable
 end
 
+class RedisDelorean
+
+  ExpireAsIfAtScript = <<EXPIRE_AS_IF_AT
+    local keys = redis.call('keys', KEYS[1])
+    local current_time = tonumber(ARGV[1])
+    local expire_as_if_at = tonumber(ARGV[2])
+    local counter = 0
+
+    for i,k in ipairs(keys) do
+      local key_ttl = redis.call('ttl', k)
+
+      -- key does not have timeout if < 0
+      if ( (key_ttl >= 0) and ((current_time + key_ttl) <= expire_as_if_at) ) then
+        redis.call('del', k)
+        counter = counter + 1
+      end
+    end
+    return counter
+EXPIRE_AS_IF_AT
+
+  def self.before_all(options = {})
+    redis = options[:redis]
+    @expire_as_if_at_sha = redis.script(:load, ExpireAsIfAtScript)
+  end
+
+  def self.before_each(options = {})
+    @redis = options[:redis]
+  end
+
+  def self.time_travel_to(dest_time)
+    # puts "travelling to #{Time.now.in_time_zone}, real time is #{Time.now_without_delorean.in_time_zone}"
+    old_maybe_fake_time = Time.now.in_time_zone
+
+    Delorean.time_travel_to(dest_time)
+    # puts "travelled to #{Time.now.in_time_zone}, real time is #{Time.now_without_delorean.in_time_zone}"
+    return if dest_time < old_maybe_fake_time
+
+    # dumps the first offset -- we're not interested in time difference from
+    # real, only with context to the fake frame of reference...
+    # This may mean all scenarios using this code should have an initial
+    #    Given it is *datetime*
+    # step
+    offsets = Delorean.send(:time_travel_offsets).dup
+    offsets.shift
+    delta = -offsets.inject(0){ |sum, val| sum + val }.floor
+
+    real_time = Time.now_without_delorean.to_i
+    # puts "delta #{delta}, expire before real time #{Time.at(real_time + delta)}"
+
+    result = @redis.evalsha(@expire_as_if_at_sha, ['*'],
+               [real_time, real_time + delta])
+    # puts "Expired #{result} key#{(result == 1) ? '' : 's'}"
+  end
+
+end
+
 redis_opts = { :db => 14, :driver => :ruby }
 redis = ::Redis.new(redis_opts)
 redis.flushdb
+RedisDelorean.before_all(:redis => redis)
 redis.quit
 
 # NB: this seems to execute outside the Before/After hooks
@@ -63,7 +127,8 @@ Before do
   # Use a separate database whilst testing
   @app = Flapjack::Executive.new(:logger => @logger,
     :config => {'email_queue' => 'email_notifications',
-                'sms_queue' => 'sms_notifications'},
+                'sms_queue' => 'sms_notifications',
+                'default_contact_timezone' => 'America/New_York'},
     :redis_config => redis_opts)
   @redis = @app.instance_variable_get('@redis')
 end
@@ -77,6 +142,11 @@ end
 
 Before('@resque') do
   ResqueSpec.reset!
+  @queues = {:email => 'email_queue'}
+end
+
+Before('@time') do
+  RedisDelorean.before_each(:redis => @redis)
 end
 
 After('@time') do