flapjack 0.7.20 → 0.7.21

data/.gitignore

@@ -26,4 +26,5 @@ flapjack-*.gem
 .rbenv-version
 .ruby-version
 .rvmrc
+artifacts/*
 

data/.travis.yml

@@ -2,6 +2,8 @@ language: ruby
 rvm:
   - "1.9.3"
   - "2.0.0"
+env:
+  - ENTITIES=10 INTERVAL=120
 gemfile:
   - Gemfile
 services:
@@ -11,4 +13,4 @@ before_script:
 before_install:
   - git submodule update --init --recursive
   - gem install bundler
-script: bundle exec rspec spec && bundle exec cucumber features
+script: bundle exec rspec spec && bundle exec cucumber features && bundle exec rake benchmarks:run

data/CHANGELOG.md

@@ -1,5 +1,15 @@
 ## Flapjack Changelog
 
+# 0.7.21 - 2013-08-08
+- Feature: make eneity search scopable by tags gh-89 (@jessereynolds)
+- Feature: add benchmark rake task gh-259 (@jessereynolds)
+- Feature: make tags more general in notification rules gh-269 (@jessereynolds)
+- Feature: ephemeral tag generation on events gh-268 (@jessereynolds)
+- Bug: fix syslog output levels gh-260 (@ali-graham)
+- Bug: Ruby 2 shutdown error gh-261 (@ali-graham)
+- Bug: Email and SMS problem and recovery notifications failing (@jessereynolds)
+- Bug: Links to contacts from the check detail page are broken (@jessereynolds)
+
 # 0.7.20 - 2013-07-17
 - Bug: flapjack-nagios-receiver failing after json library change gh-257 (@jessereynolds)
 - Bug: email gateway partial conversion to erb sending haml source code gh-256 (@ali-graham)

data/Gemfile

@@ -21,4 +21,5 @@ group :test do
   gem 'fuubar-cucumber'
   gem 'simplecov', :require => false
   gem 'json_spec'
+  gem 'perftools.rb'
 end

data/bin/flapjack

@@ -6,6 +6,17 @@ unless $:.include?(File.dirname(__FILE__) + '/../lib/')
 end
 
 require 'dante'
+
+module Dante
+  class Runner
+
+    def start
+      # skip signal traps
+      @startup_command.call(self.options) if @startup_command
+    end
+  end
+end
+
 require 'optparse'
 require 'ostruct'
 

data/bin/simulate-failed-check

@@ -12,6 +12,7 @@ require 'eventmachine'
 require 'json'
 
 require 'flapjack/configuration'
+require 'flapjack/data/event'
 
 def pike(message)
   puts "piking out: #{message}"
@@ -20,7 +21,7 @@ end
 
 def send_event(event, opts)
   redis = opts[:redis]
-  redis.lpush 'events', event.to_json
+  Flapjack::Data::Event.add(event, :redis => redis)
 end
 
 def fail(opts)
@@ -31,8 +32,7 @@ def fail(opts)
   event  = {
     'entity'    => opts[:entity] || 'foo-app-01',
     'check'     => opts[:check]  || 'HTTP',
-    'type'      => 'service',
-    'timestamp' => Time.now.to_i
+    'type'      => 'service'
   }
   failure  = event.merge('state' => state, 'summary' => 'Simulated check output (test by operator)')
   recovery = event.merge('state' => 'ok',  'summary' => 'Simulated check output (test by operator)')
@@ -47,14 +47,14 @@ def fail(opts)
       puts "#{Time.now}: stopping"
       if recover
         puts "#{Time.now}: sending recovery event for #{key}"
-        send_event(recovery, :redis => redis)
+        send_event(recovery.merge('time' => Time.now.to_i), :redis => redis)
       end
       EM.stop
     end
 
     EM.add_periodic_timer(10) do
       puts "#{Time.now}: sending failure event (#{state}) for #{key}"
-      send_event(failure, :redis => redis)
+      send_event(failure.merge('time' => Time.now.to_i), :redis => redis)
     end
 
   }

data/features/notification_rules.feature

@@ -7,13 +7,15 @@ Feature: Notification rules on a per contact basis
       | 1   | Malak      | Al-Musawi | malak@example.com | +61400000001 | Asia/Baghdad     |
       | 2   | Imani      | Farooq    | imani@example.com | +61400000002 | Europe/Moscow    |
       | 3   | Vera       | Дурейко   | vera@example.com  | +61400000003 | Europe/Paris     |
+      | 4   | Lucia      | Moretti   | lucia@example.com | +61400000004 | Europe/Rome      |
 
     And the following entities exist:
-      | id  | name | contacts |
-      | 1   | foo  | 1        |
-      | 2   | bar  | 1,2,3    |
-      | 3   | baz  | 1,3      |
-      | 4   | buf  | 1,2,3    |
+      | id  | name           | contacts |
+      | 1   | foo            | 1        |
+      | 2   | bar            | 1,2,3    |
+      | 3   | baz            | 1,3      |
+      | 4   | buf            | 1,2,3    |
+      | 5   | foo-app-01.xyz | 4        |
 
     And user 1 has the following notification intervals:
       | email | sms |
@@ -27,25 +29,35 @@ Feature: Notification rules on a per contact basis
       | email | sms |
       | 15    | 60  |
 
+    And user 4 has the following notification intervals:
+      | email | sms |
+      | 15    | 60  |
+
     And user 1 has the following notification rules:
-      | entities | entity_tags | warning_media | critical_media   | warning_blackhole | critical_blackhole | time_restrictions |
-      |          |             | email         | sms,email        | true              | true               |                   |
-      | foo      |             | email         | sms,email        |                   |                    | 8-18 weekdays     |
-      | bar      |             |               | sms,email        | true              |                    |                   |
-      | baz      |             | email         | sms,email        |                   |                    |                   |
+      | entities | tags | warning_media | critical_media   | warning_blackhole | critical_blackhole | time_restrictions |
+      |          |      | email         | sms,email        | true              | true               |                   |
+      | foo      |      | email         | sms,email        |                   |                    | 8-18 weekdays     |
+      | bar      |      |               | sms,email        | true              |                    |                   |
+      | baz      |      | email         | sms,email        |                   |                    |                   |
 
     And user 2 has the following notification rules:
-      | entities | entity_tags | warning_media | critical_media   | warning_blackhole | critical_blackhole | time_restrictions |
-      |          |             | email         | email            |                   |                    |                   |
-      |          |             | sms           | sms              |                   |                    |                   |
-      | bar      |             | email         | email,sms        |                   |                    |                   |
+      | entities | tags | warning_media | critical_media   | warning_blackhole | critical_blackhole | time_restrictions |
+      |          |      | email         | email            |                   |                    |                   |
+      |          |      | sms           | sms              |                   |                    |                   |
+      | bar      |      | email         | email,sms        |                   |                    |                   |
 
     And user 3 has the following notification rules:
-      | entities | entity_tags | warning_media | critical_media   | warning_blackhole | critical_blackhole | time_restrictions |
-      |          |             | email         | email            |                   |                    |                   |
-      | baz      |             | sms           | sms              |                   |                    |                   |
-      | buf      |             | email         | email            |                   |                    |                   |
-      | buf      |             | sms           | sms              |                   |                    |                   |
+      | entities | tags            | warning_media | critical_media   | warning_blackhole | critical_blackhole | time_restrictions |
+      |          |                 | email         | email            |                   |                    |                   |
+      | baz      |                 | sms           | sms              |                   |                    |                   |
+      | buf      |                 | email         | email            |                   |                    |                   |
+      | buf      |                 | sms           | sms              |                   |                    |                   |
+
+    And user 4 has the following notification rules:
+      | entities | tags            | warning_media | critical_media   | warning_blackhole | critical_blackhole | time_restrictions |
+      |          |                 |               |                  |                   |                    |                   |
+      |          | xyz, disk, util | sms           | sms              |                   |                    |                   |
+      |          | xyz, ping       | sms,email     | sms,email        |                   |                    | 8-18 weekdays     |
 
   @time_restrictions @time
   Scenario: Alerts only during specified time restrictions
@@ -310,3 +322,49 @@ Feature: Notification rules on a per contact basis
   @time
   Scenario: A blackhole rule on an entity should override another matching general rule
 
+  @time
+  Scenario: Notify when tags in a rule match the event's tags
+    Given the check is check 'Disk / Util' on entity 'foo-app-01.xyz'
+    And   the check is in an ok state
+    When  a critical event is received
+    And   1 minute passes
+    And   a critical event is received
+    Then  1 sms alert should be queued for +61400000004
+
+  @time
+  Scenario: Don't notify when tags in a rule don't match the event's tags
+    Given the check is check 'Memory Util' on entity 'foo-app-01.xyz'
+    And   the check is in an ok state
+    When  a critical event is received
+    And   1 minute passes
+    And   a critical event is received
+    Then  no sms alerts should be queued for +61400000004
+
+  @time
+  Scenario: Only notify during specified time periods in tag matched rules
+    Given the timezone is Europe/Rome
+    And   the time is February 1 2013 6:59
+    And   the check is check 'ping' on entity 'foo-app-01.xyz'
+    And   the check is in an ok state
+    And   a critical event is received
+    Then  no sms alerts should be queued for +61400000004
+    And   the time is February 1 2013 7:01
+    And   a critical event is received
+    Then  no sms alerts should be queued for +61400000004
+    And   the time is February 1 2013 8:01
+    And   a critical event is received
+    Then  1 sms alert should be queued for +61400000004
+    When  the time is February 1 2013 12:00
+    Then  all alert dropping keys for user 1 should have expired
+    When  a critical event is received
+    Then  2 sms alerts should be queued for +61400000004
+    When  the time is February 1 2013 17:59
+    Then  all alert dropping keys for user 1 should have expired
+    When  a critical event is received
+    Then  3 sms alerts should be queued for +61400000004
+    When  the time is February 1 2013 18:01
+    Then  all alert dropping keys for user 1 should have expired
+    When  a critical event is received
+    Then  3 sms alerts should be queued for +61400000004
+
+

data/features/steps/events_steps.rb

@@ -289,6 +289,15 @@ Then /^show me the (\w+ )*log$/ do |adjective|
   puts @logger.messages.join("\n")
 end
 
+Then /^dump notification rules for user (\d+)$/ do |contact|
+  rule_ids = @redis.smembers("contact_notification_rules:#{contact}")
+  puts "There #{(rule_ids.length == 1) ? 'is' : 'are'} #{rule_ids.length} notification rule#{(rule_ids.length == 1) ? '' : 's'} for user #{contact}:"
+  rule_ids.each {|rule_id|
+    rule = Flapjack::Data::NotificationRule.find_by_id(rule_id, :redis => @redis)
+    puts rule.to_json
+  }
+end
+
 # added for notification rules:
 Given /^the following entities exist:$/ do |entities|
   entities.hashes.each do |entity|
@@ -335,7 +344,7 @@ Given /^user (\d+) has the following notification rules:$/ do |contact_id, rules
   end
   rules.hashes.each do |rule|
     entities           = rule['entities'].split(',').map { |x| x.strip }
-    entity_tags        = rule['entity_tags'].split(',').map { |x| x.strip }
+    tags               = rule['tags'].split(',').map { |x| x.strip }
     warning_media      = rule['warning_media'].split(',').map { |x| x.strip }
     critical_media     = rule['critical_media'].split(',').map { |x| x.strip }
     warning_blackhole  = (rule['warning_blackhole'].downcase == 'true')
@@ -352,13 +361,16 @@ Given /^user (\d+) has the following notification rules:$/ do |contact_id, rules
     end
     rule_data = {:contact_id         => contact_id,
                  :entities           => entities,
-                 :entity_tags        => entity_tags,
+                 :tags               => tags,
                  :warning_media      => warning_media,
                  :critical_media     => critical_media,
                  :warning_blackhole  => warning_blackhole,
                  :critical_blackhole => critical_blackhole,
                  :time_restrictions  => time_restrictions}
-    Flapjack::Data::NotificationRule.add(rule_data, :redis => @redis)
+    created_rule = Flapjack::Data::NotificationRule.add(rule_data, :redis => @redis)
+    unless created_rule.is_a?(Flapjack::Data::NotificationRule)
+      raise "Error creating notification rule with data: #{rule_data}, errors: #{created_rule.join(', ')}"
+    end
   end
 end
 

data/lib/flapjack/coordinator.rb

@@ -28,8 +28,8 @@ module Flapjack
       @boot_time = Time.now
 
       EM.synchrony do
-        add_pikelets(pikelets(@config.all))
         setup_signals if options[:signals]
+        add_pikelets(pikelets(@config.all))
       end
     end
 
@@ -124,7 +124,7 @@ module Flapjack
       start_piks = []
       pikelets_data.each_pair do |type, cfg|
         next unless pikelet = Flapjack::Pikelet.create(type,
-          :config => cfg, :redis_config => @redis_options, :boot_time => @boot_time)
+          :config => cfg, :redis_config => @redis_options, :boot_time => @boot_time, :coordinator => self)
         start_piks << pikelet
         @pikelets << pikelet
       end
@@ -132,7 +132,7 @@ module Flapjack
         start_piks.each {|pik| pik.start }
       rescue Exception => e
         trace = e.backtrace.join("\n")
-        @logger.fatal "#{e.message}\n#{trace}"
+        @logger.fatal "#{e.class.name}\n#{e.message}\n#{trace}"
         stop
       end
     end

data/lib/flapjack/data/contact.rb

@@ -175,7 +175,7 @@ module Flapjack
         if rules.all? {|r| r.is_specific? } # also true if empty
           rule = self.add_notification_rule({
               :entities           => [],
-              :entity_tags        => [],
+              :tags               => Flapjack::Data::TagSet.new([]),
               :time_restrictions  => [],
               :warning_media      => ['email', 'sms', 'jabber', 'pagerduty'],
               :critical_media     => ['email', 'sms', 'jabber', 'pagerduty'],

data/lib/flapjack/data/entity.rb

@@ -95,6 +95,17 @@ module Flapjack
         }.sort
       end
 
+      def self.find_all_with_tags(tags, options = {})
+        raise "Redis connection not set" unless redis = options[:redis]
+        tags_prefixed = tags.collect {|tag|
+          "#{TAG_PREFIX}:#{tag}"
+        }
+        logger.debug "tags_prefixed: #{tags_prefixed.inspect}" if logger = options[:logger]
+        Flapjack::Data::Tag.find_intersection(tags_prefixed, :redis => redis).collect {|entity_id|
+          Flapjack::Data::Entity.find_by_id(entity_id, :redis => redis).name
+        }.compact
+      end
+
       def self.find_all_with_checks(options)
         raise "Redis connection not set" unless redis = options[:redis]
         redis.zrange("current_entities", 0, -1)
@@ -119,7 +130,7 @@ module Flapjack
       end
 
       def check_list
-        @redis.zrange("current_checks:#{@name}", 0, -1)        
+        @redis.zrange("current_checks:#{@name}", 0, -1)
       end
 
       def check_count

data/lib/flapjack/data/entity_check.rb

@@ -92,14 +92,14 @@ module Flapjack
             memo[entity] ||= []
             memo[entity] << check
           end
-          memo  
+          memo
         end
       end
 
       def self.count_all_failing(options = {})
         raise "Redis connection not set" unless redis = options[:redis]
         redis.zrange("failed_checks", 0, -1).count do |key|
-          entity, check = key.split(':', 2) 
+          entity, check = key.split(':', 2)
           !!redis.zscore("current_checks:#{entity}", check)
         end
       end
@@ -515,6 +515,13 @@ module Flapjack
         }.compact
       end
 
+      def tags
+        entity, check = @key.split(":", 2)
+        ta = Flapjack::Data::TagSet.new([])
+        ta += entity.split('.', 2).map {|x| x.downcase}
+        ta += check.split(' ').map {|x| x.downcase}
+      end
+
     private
 
       def initialize(entity, check, options = {})

data/lib/flapjack/data/event.rb

@@ -6,7 +6,7 @@ module Flapjack
   module Data
     class Event
 
-      attr_accessor :counter, :previous_state, :previous_state_duration
+      attr_accessor :counter, :previous_state, :previous_state_duration, :tags
 
       attr_reader :check, :summary, :details, :acknowledgement_id
 
@@ -65,14 +65,14 @@ module Flapjack
         raise "Redis connection not set" unless redis = opts[:redis]
 
         evt['time'] = Time.now.to_i if evt['time'].nil?
-        redis.rpush('events', ::Oj.dump(evt))
+        redis.lpush('events', ::Oj.dump(evt))
       end
 
       # Provide a count of the number of events on the queue to be processed.
-      def self.pending_count(opts = {})
+      def self.pending_count(queue, opts = {})
         raise "Redis connection not set" unless redis = opts[:redis]
 
-        redis.llen('events')
+        redis.llen(queue)
       end
 
       def self.create_acknowledgement(entity_name, check, opts = {})

data/lib/flapjack/data/notification.rb

@@ -48,6 +48,7 @@ module Flapjack
 
         last_state = opts[:last_state] || {}
 
+        tag_data = event.tags.is_a?(Set) ? event.tags.to_a : nil
         notif = {'event_id'     => event.id,
                  'state'        => event.state,
                  'summary'      => event.summary,
@@ -58,7 +59,8 @@ module Flapjack
                  'duration'     => event.duration || nil,
                  'type'         => opts[:type] || type_for_event(event),
                  'severity'     => opts[:severity],
-                 'count'        => event.counter }
+                 'count'        => event.counter,
+                 'tags'         => tag_data }
 
         redis.rpush(queue, Oj.dump(notif))
       end
@@ -96,7 +98,8 @@ module Flapjack
                        'time'              => @event_time,
                        'duration'          => @event_duration,
                        'notification_type' => @type,
-                       'event_count'       => @event_count
+                       'event_count'       => @event_count,
+                       'tags'              => @tags
                       }
       end
 
@@ -111,35 +114,39 @@ module Flapjack
           rules = contact.notification_rules
           media = contact.media
 
-          logger.debug "considering messages for contact id #{contact_id} #{@event_id} #{@event_state} (media) #{media.inspect}"
+          logger.debug "Notification#messages: creating messages for contact: #{contact_id} " +
+            "event_id: \"#{@event_id}\" state: #{@event_state} event_tags: #{@tags.to_json} media: #{media.inspect}"
           rlen = rules.length
-          logger.debug "found #{rlen} rule#{(rlen == 1) ? '' : 's'} for contact"
+          logger.debug "found #{rlen} rule#{(rlen == 1) ? '' : 's'} for contact #{contact_id}"
 
           media_to_use = if rules.empty?
             media
           else
             # matchers are rules of the contact that have matched the current event
-            # for time and entity
+            # for time, entity and tags
             matchers = rules.select do |rule|
-              rule.match_entity?(@event_id) &&
+              logger.debug("considering rule with entities: #{rule.entities} and tags: #{rule.tags.to_json}")
+              (rule.match_entity?(@event_id) || rule.match_tags?(@tags) || ! rule.is_specific?) &&
                 rule_occurring_now?(rule, :contact => contact, :default_timezone => default_timezone)
             end
 
-            logger.debug "#{matchers.length} matchers remain for this contact:"
+            logger.debug "#{matchers.length} matchers remain for this contact after time, entity and tags are matched:"
             matchers.each do |matcher|
               logger.debug "matcher: #{matcher.to_json}"
             end
 
-            # delete any matchers for all entities if there are more specific matchers
+            # delete any general matchers if there are more specific matchers left
             if matchers.any? {|matcher| matcher.is_specific? }
 
-              logger.debug("general removal: found #{matchers.length} entity specific matchers")
               num_matchers = matchers.length
 
               matchers.reject! {|matcher| !matcher.is_specific? }
 
               if num_matchers != matchers.length
                 logger.debug("notification: removal of general matchers when entity specific matchers are present: number of matchers changed from #{num_matchers} to #{matchers.length} for contact id: #{contact_id}")
+                matchers.each do |matcher|
+                  logger.debug "matcher: #{matcher.to_json}"
+                end
               end
             end
 
@@ -179,19 +186,19 @@ module Flapjack
 
       # created from parsed JSON, so opts keys are in strings
       def initialize(opts = {})
-        @event_id       = opts['event_id']
-        @event_state    = opts['state']
-        @event_summary  = opts['summary']
-        @event_details  = opts['details']
-        @event_time     = opts['time']
-        @event_duration = opts['duration']
-        @event_count    = opts['count']
-
+        @event_id           = opts['event_id']
+        @event_state        = opts['state']
+        @event_summary      = opts['summary']
+        @event_details      = opts['details']
+        @event_time         = opts['time']
+        @event_duration     = opts['duration']
+        @event_count        = opts['count']
         @last_event_state   = opts['last_state']
         @last_event_summary = opts['last_summary']
-
-        @type           = opts['type']
-        @severity       = opts['severity']
+        @type               = opts['type']
+        @severity           = opts['severity']
+        tags                = opts['tags']
+        @tags               = tags.is_a?(Array) ? Flapjack::Data::TagSet.new(tags) : nil
       end
 
       # # time restrictions match?