flapjack 1.2.1 → 1.2.2

data/features/support/env.rb

@@ -38,8 +38,6 @@ require 'flapjack/notifier'
 require 'flapjack/processor'
 require 'flapjack/patches'
 
-require 'resque_spec'
-
 class MockLogger
   attr_accessor :messages
 
@@ -122,6 +120,20 @@ redis.flushdb
 RedisDelorean.before_all(:redis => redis)
 redis.quit
 
+# Not the most efficient of operations...
+def redis_peek(queue, start = 0, count = nil)
+  size = @notifier_redis.llen(queue)
+  start = 0 if start < 0
+  count = (size - start) if count.nil? || (count > (size - start))
+
+  (0..(size - 1)).inject([]) do |memo, n|
+    object = @notifier_redis.rpoplpush(queue, queue)
+    next memo unless (n >= start || n < (start + count))
+    memo << Flapjack.load_json(object)
+    memo
+  end
+end
+
 Around do |scenario, blk|
   EM.synchrony do
     blk.call
@@ -130,11 +142,13 @@ Around do |scenario, blk|
 end
 
 Before do
+  @redis_opts = redis_opts
   @logger = MockLogger.new
 end
 
 After do
   @logger.messages = []
+  @redis_opts = nil
 end
 
 Before('@processor') do
@@ -150,7 +164,7 @@ After('@processor') do
 end
 
 Before('@notifier') do
-  @notifier  = Flapjack::Notifier.new(:logger => @logger,
+  @notifier = Flapjack::Notifier.new(:logger => @logger,
     :redis_config => redis_opts,
     :config => {'email_queue' => 'email_notifications',
                 'sms_queue' => 'sms_notifications',
@@ -165,11 +179,6 @@ After('@notifier') do
   @notifier_redis = nil
 end
 
-Before('@resque') do
-  ResqueSpec.reset!
-  @queues = {:email => 'email_queue'}
-end
-
 Before('@time') do
   RedisDelorean.before_each(:redis => @redis || @notifier_redis)
 end

data/flapjack.gemspec

@@ -23,10 +23,9 @@ Gem::Specification.new do |gem|
   gem.add_dependency 'eventmachine', '~> 1.0.0'
   gem.add_dependency 'redis', '~> 3.0.6'
   gem.add_dependency 'hiredis'
+  gem.add_dependency 'em-hiredis'
   gem.add_dependency 'em-synchrony', '~> 1.0.2'
   gem.add_dependency 'em-http-request'
-  gem.add_dependency 'em-resque'
-  gem.add_dependency 'resque', '~> 1.23.0'
   gem.add_dependency 'sinatra'
   gem.add_dependency 'rack-fiber_pool'
   gem.add_dependency 'thin', '~> 1.6.1'
@@ -35,12 +34,13 @@ Gem::Specification.new do |gem|
   gem.add_dependency 'chronic'
   gem.add_dependency 'chronic_duration'
   gem.add_dependency 'terminal-table'
-  gem.add_dependency 'activesupport', '~> 3.2.14'
+  gem.add_dependency 'activesupport'
   gem.add_dependency 'ice_cube'
-  gem.add_dependency 'tzinfo', '~> 1.0.1'
+  gem.add_dependency 'tzinfo'
   gem.add_dependency 'tzinfo-data'
   gem.add_dependency 'rbtrace'
   gem.add_dependency 'rake'
   gem.add_dependency 'gli', '= 2.12.0'
   gem.add_dependency 'nokogiri', '= 1.6.2.1'
+  gem.add_dependency 'mysql2'
 end

data/lib/flapjack.rb

@@ -4,6 +4,9 @@ require 'oj'
 
 module Flapjack
 
+  DEFAULT_INITIAL_FAILURE_DELAY = 30
+  DEFAULT_REPEAT_FAILURE_DELAY  = 60
+
   def self.load_json(data)
     Oj.load(data, :mode => :strict, :symbol_keys => false)
   end

data/lib/flapjack/cli/import.rb

@@ -65,6 +65,7 @@ module Flapjack
         return @redis unless @redis.nil?
         @redis = Redis.new(@redis_options.merge(:driver => :hiredis))
         Flapjack::Data::Migration.migrate_entity_check_data_if_required(:redis => @redis)
+        Flapjack::Data::Migration.clear_orphaned_entity_ids(:redis => @redis)
         @redis
       end
 

data/lib/flapjack/cli/maintenance.rb

@@ -68,6 +68,7 @@ module Flapjack
         return @redis unless @redis.nil?
         @redis = Redis.new(@redis_options.merge(:driver => :hiredis))
         Flapjack::Data::Migration.migrate_entity_check_data_if_required(:redis => @redis)
+        Flapjack::Data::Migration.clear_orphaned_entity_ids(:redis => @redis)
         Flapjack::Data::Migration.validate_scheduled_maintenance_periods(:redis => @redis)
         @redis
       end

data/lib/flapjack/cli/purge.rb

@@ -40,6 +40,7 @@ module Flapjack
         purged = checks.inject([]) do |memo, check|
           pu = check.purge_history(options)
           memo << pu unless pu == 0
+          memo
         end
 
         if purged.empty?
@@ -55,6 +56,7 @@ module Flapjack
         return @redis unless @redis.nil?
         @redis = Redis.new(@redis_options.merge(:driver => :hiredis))
         Flapjack::Data::Migration.migrate_entity_check_data_if_required(:redis => @redis)
+        Flapjack::Data::Migration.clear_orphaned_entity_ids(:redis => @redis)
         @redis
       end
 

data/lib/flapjack/cli/receiver.rb

@@ -144,6 +144,7 @@ module Flapjack
         return @redis unless @redis.nil?
         @redis = Redis.new(@redis_options.merge(:driver => :hiredis))
         Flapjack::Data::Migration.migrate_entity_check_data_if_required(:redis => @redis)
+        Flapjack::Data::Migration.clear_orphaned_entity_ids(:redis => @redis)
         @redis
       end
 

data/lib/flapjack/cli/simulate.rb

@@ -55,6 +55,7 @@ module Flapjack
         return @redis unless @redis.nil?
         @redis = Redis.new(@redis_options)
         Flapjack::Data::Migration.migrate_entity_check_data_if_required(:redis => @redis)
+        Flapjack::Data::Migration.clear_orphaned_entity_ids(:redis => @redis)
         @redis
       end
 

data/lib/flapjack/data/alert.rb

@@ -51,7 +51,7 @@ module Flapjack
 
       include Flapjack::Utility
 
-      def initialize(contents, opts)
+      def initialize(contents, opts = {})
         raise "no logger supplied" unless @logger = opts[:logger]
 
         @event_id                   = contents['event_id']
@@ -103,7 +103,34 @@ module Flapjack
               details['state'] && allowed_rollup_states.include?(details['state'])
           end
         end
+      end
+
+      def self.add(queue, alert_data, opts = {})
+        raise "Redis connection not set" unless redis = opts[:redis]
+        redis.rpush(queue, Flapjack.dump_json(alert_data))
+      end
+
+      def self.next(queue, opts = {})
+        raise "Redis connection not set" unless redis = opts[:redis]
+
+        defaults = { :block => true }
+        options  = defaults.merge(opts)
 
+        if options[:block]
+          raw = redis.blpop(queue, 0)[1]
+        else
+          raw = redis.lpop(queue)
+          return unless raw
+        end
+        begin
+          parsed = ::Flapjack.load_json( raw )
+        rescue Oj::Error => e
+          if options[:logger]
+            options[:logger].warn("Error deserialising alert json: #{e}, raw json: #{raw.inspect}")
+          end
+          return nil
+        end
+        self.new( parsed, :logger => opts[:logger] )
       end
 
       def type

data/lib/flapjack/data/contact.rb

@@ -19,7 +19,7 @@ module Flapjack
       attr_accessor :id, :first_name, :last_name, :email, :media,
         :media_intervals, :media_rollup_thresholds, :pagerduty_credentials
 
-      ALL_MEDIA  = ['email', 'sms', 'sms_twilio', 'jabber', 'pagerduty', 'sns']
+      ALL_MEDIA  = ['email', 'sms', 'sms_twilio', 'sms_gammu', 'jabber', 'pagerduty', 'sns']
 
       def self.all(options = {})
         raise "Redis connection not set" unless redis = options[:redis]

data/lib/flapjack/data/entity.rb

@@ -469,24 +469,34 @@ module Flapjack
           end
         else
           # most likely from API import
-          existing_name = redis.hget('all_entity_names_by_id', entity_id)
+          this_id_original_name = redis.hget('all_entity_names_by_id', entity_id)
 
           # if there's an entity with a matching name, this will change its
           # id; if no entity exists it creates a new one
 
-          if existing_name.nil?
+          this_name_original_id = redis.hget('all_entity_ids_by_name', entity_name)
+
+          if this_id_original_name.nil?
+            # no entity exists with a matching id
             redis.hset('all_entity_ids_by_name', entity_name, entity_id)
             redis.hset('all_entity_names_by_id', entity_id, entity_name)
 
-          elsif existing_name != entity_name
-            if redis.hexists('all_entity_ids_by_name', entity_name)
-              merge(existing_name, entity_name, :redis => redis)
-            else
-              rename(existing_name, entity_name, :redis => redis) {|multi|
-                multi.hdel('all_entity_ids_by_name', existing_name)
+            unless this_name_original_id.nil?
+              # an entity existed with a matching name but a different id
+              redis.hdel('all_entity_names_by_id', this_name_original_id)
+            end
+          elsif this_id_original_name != entity_name
+            # a record exists with the provided id but a different name
+            if this_name_original_id.nil?
+              # there shouldn't be any entity records left without ids (due to
+              # the migration code) so this code may not be needed now
+              rename(this_id_original_name, entity_name, :redis => redis) {|multi|
+                multi.hdel('all_entity_ids_by_name', this_id_original_name)
                 multi.hset('all_entity_ids_by_name', entity_name, entity_id)
                 multi.hset('all_entity_names_by_id', entity_id, entity_name)
               }
+            else
+              merge(this_id_original_name, entity_name, :redis => redis)
             end
           end
         end

data/lib/flapjack/data/entity_check.rb

@@ -606,6 +606,8 @@ module Flapjack
         details = options[:details]
         perfdata = options[:perfdata]
         count = options[:count]
+        initial_delay = options[:initial_failure_delay]
+        repeat_delay = options[:repeat_failure_delay]
 
         old_state = self.state
 
@@ -651,6 +653,11 @@ module Flapjack
           # hash summary and details (as they may have changed)
           multi.hset("check:#{@key}", 'summary', (summary || ''))
           multi.hset("check:#{@key}", 'details', (details || ''))
+
+          # NB: delays will revert to defaults if event sources don't continue sending
+          # through their custom delays in the event structure
+          multi.hset("check:#{@key}", 'initial_failure_delay', (initial_delay || Flapjack::DEFAULT_INITIAL_FAILURE_DELAY))
+          multi.hset("check:#{@key}", 'repeat_failure_delay', (repeat_delay || Flapjack::DEFAULT_REPEAT_FAILURE_DELAY))
           if perfdata
             multi.hset("check:#{@key}", 'perfdata', format_perfdata(perfdata).to_json)
   #          multi.set("#{@key}:#{timestamp}:perfdata", perfdata)
@@ -777,6 +784,16 @@ module Flapjack
         data
       end
 
+      def initial_failure_delay
+        delay = @redis.hget("check:#{@key}", 'initial_failure_delay')
+        delay.to_i unless delay.nil?
+      end
+
+      def repeat_failure_delay
+        delay = @redis.hget("check:#{@key}", 'repeat_failure_delay')
+        delay.to_i unless delay.nil?
+      end
+
       # Returns a list of states for this entity check, sorted by timestamp.
       #
       # start_time and end_time should be passed as integer timestamps; these timestamps

data/lib/flapjack/data/event.rb

@@ -10,10 +10,13 @@ module Flapjack
 
       attr_accessor :counter, :id_hash, :tags
 
-      attr_reader :check, :summary, :details, :acknowledgement_id, :perfdata
+      attr_reader :check, :summary, :details, :acknowledgement_id, :perfdata,
+                  :initial_failure_delay, :repeat_failure_delay
 
-      REQUIRED_KEYS = ['type', 'state', 'entity', 'check', 'summary']
-      OPTIONAL_KEYS = ['time', 'details', 'acknowledgement_id', 'duration', 'tags', 'perfdata']
+      REQUIRED_KEYS = ['type', 'state', 'entity', 'check']
+      OPTIONAL_KEYS = ['time', 'summary', 'details', 'acknowledgement_id',
+                       'duration', 'tags', 'perfdata', 'initial_failure_delay',
+                       'repeat_failure_delay']
 
       VALIDATIONS = {
         proc {|e| e['type'].is_a?(String) &&
@@ -32,14 +35,24 @@ module Flapjack
         proc {|e| e['check'].is_a?(String) } =>
           "check must be a string",
 
-        proc {|e| e['summary'].is_a?(String) } =>
-          "summary must be a string",
-
         proc {|e| e['time'].nil? ||
                   e['time'].is_a?(Integer) ||
                  (e['time'].is_a?(String) && !!(e['time'] =~ /^\d+$/)) } =>
           "time must be a positive integer, or a string castable to one",
 
+        proc {|e| e['initial_failure_delay'].nil? ||
+                  e['initial_failure_delay'].is_a?(Integer) ||
+                 (e['initial_failure_delay'].is_a?(String) && !!(e['initial_failure_delay'] =~ /^\d+$/)) } =>
+          "initial_failure_delay must be a positive integer, or a string castable to one",
+
+        proc {|e| e['repeat_failure_delay'].nil? ||
+                  e['repeat_failure_delay'].is_a?(Integer) ||
+                 (e['repeat_failure_delay'].is_a?(String) && !!(e['repeat_failure_delay'] =~ /^\d+$/)) } =>
+          "repeat_failure_delay must be a positive integer, or a string castable to one",
+
+        proc {|e| e['summary'].nil? || e['summary'].is_a?(String) } =>
+          "summary must be a string",
+
         proc {|e| e['details'].nil? || e['details'].is_a?(String) } =>
           "details must be a string",
 
@@ -169,14 +182,16 @@ module Flapjack
       end
 
       # creates, or modifies, an event object and adds it to the events list in redis
-      #   'entity'    => entity,
-      #   'check'     => check,
-      #   'type'      => 'service',
-      #   'state'     => state,
-      #   'summary'   => check_output,
-      #   'details'   => check_long_output,
-      #   'perfdata'  => perf_data,
-      #   'time'      => timestamp
+      #   'entity'                => entity,
+      #   'check'                 => check,
+      #   'type'                  => 'service',
+      #   'state'                 => state,
+      #   'summary'               => check_output,
+      #   'details'               => check_long_output,
+      #   'perfdata'              => perf_data,
+      #   'time'                  => timestamp,
+      #   'initial_failure_delay' => initial_failure_delay,
+      #   'repeat_failure_delay'  => repeat_failure_delay
       def self.add(evt, opts = {})
         raise "Redis connection not set" unless redis = opts[:redis]
 
@@ -216,9 +231,12 @@ module Flapjack
 
       def initialize(attrs = {})
         ['type', 'state', 'entity', 'check', 'time', 'summary', 'details',
-         'perfdata', 'acknowledgement_id', 'duration'].each do |key|
+         'perfdata', 'acknowledgement_id', 'duration', 'initial_failure_delay',
+         'repeat_failure_delay'].each do |key|
           instance_variable_set("@#{key}", attrs[key])
         end
+        # summary is optional. set it to nil if it only contains whitespace
+        @summary = (@summary.is_a?(String) && ! @summary.strip.empty?) ? @summary.strip : nil
         # details is optional. set it to nil if it only contains whitespace
         @details = (@details.is_a?(String) && ! @details.strip.empty?) ? @details.strip : nil
         # perfdata is optional. set it to nil if it only contains whitespace

data/lib/flapjack/data/migration.rb

@@ -12,9 +12,11 @@ module Flapjack
       ENTITY_DATA_MIGRATION = 'entity_data_migration'
 
       # copied from jsonapi/contact_methods.rb, could extract both into separate file
-      def self.obtain_semaphore(resource, options = {})
+      def self.obtain_semaphore(resource, description, options = {})
         raise "Redis connection not set" unless redis = options[:redis]
 
+        logger = options[:logger]
+
         semaphore = nil
         strikes = 0
         begin
@@ -27,6 +29,18 @@ module Flapjack
           end
           sempahore = nil
         end
+
+        if semaphore.nil?
+          unless logger.nil?
+            logger.fatal "Could not obtain lock for data migration (#{reason}). Ensure that " +
+              "no other flapjack processes are running that might be executing " +
+              "migrations, check logs for any exceptions, manually delete the " +
+              "'#{resource}' key from your Flapjack Redis " +
+              "database and try running Flapjack again."
+          end
+          raise "Unable to obtain semaphore #{resource}"
+        end
+
         semaphore
       end
 
@@ -37,17 +51,8 @@ module Flapjack
 
         return if redis.exists('created_ids_for_old_entities_without_ids')
 
-        semaphore = obtain_semaphore(ENTITY_DATA_MIGRATION, :redis => redis)
-        if semaphore.nil?
-          unless logger.nil?
-            logger.fatal "Could not obtain lock for data migration (entity id creation). Ensure that " +
-              "no other flapjack processes are running that might be executing " +
-              "migrations, check logs for any exceptions, manually delete the " +
-              "'#{ENTITY_DATA_MIGRATION}' key from your Flapjack Redis " +
-              "database and try running Flapjack again."
-          end
-          raise "Unable to obtain semaphore #{ENTITY_DATA_MIGRATION}"
-        end
+        semaphore = obtain_semaphore(ENTITY_DATA_MIGRATION, 'entity id creation',
+          :redis => redis, :logger => logger)
 
         begin
           logger.warn "Ensuring all entities have ids ..." unless logger.nil?
@@ -72,17 +77,8 @@ module Flapjack
 
         return if redis.exists('all_checks')
 
-        semaphore = obtain_semaphore(ENTITY_DATA_MIGRATION, :redis => redis)
-        if semaphore.nil?
-          unless logger.nil?
-            logger.fatal "Could not obtain lock for entity check data migration. Ensure that " +
-              "no other flapjack processes are running that might be executing " +
-              "migrations, check logs for any exceptions, manually delete the " +
-              "'#{ENTITY_DATA_MIGRATION}' key from your Flapjack Redis " +
-              "database and try running Flapjack again."
-          end
-          raise "Unable to obtain semaphore #{ENTITY_DATA_MIGRATION}"
-        end
+        semaphore = obtain_semaphore(ENTITY_DATA_MIGRATION, 'entity check data',
+          :redis => redis, :logger => logger)
 
         begin
           logger.warn "Upgrading Flapjack's entity/check Redis indexes..." unless logger.nil?
@@ -125,6 +121,33 @@ module Flapjack
         end
       end
 
+      def self.clear_orphaned_entity_ids(options = {})
+        raise "Redis connection not set" unless redis = options[:redis]
+
+        logger = options[:logger]
+
+        semaphore = obtain_semaphore(ENTITY_DATA_MIGRATION,
+          'orphaned entity ids', :redis => redis, :logger => logger)
+
+        begin
+          logger.info "Checking for orphaned entity ids..." unless logger.nil?
+
+          valid_entity_data = redis.hgetall('all_entity_ids_by_name')
+
+          missing_ids = redis.hgetall('all_entity_names_by_id').reject {|e_id, e_name|
+            valid_entity_data[e_name] == e_id
+          }
+
+          unless missing_ids.empty?
+            logger.info "Clearing ids (#{missing_ids.inspect})" unless logger.nil?
+            redis.hdel('all_entity_names_by_id', missing_ids.keys)
+          end
+        ensure
+          semaphore.release
+          logger.info "Finished checking for orphaned entity ids." unless logger.nil?
+        end
+      end
+
       def self.refresh_archive_index(options = {})
         raise "Redis connection not set" unless redis = options[:redis]
         archive_keys = redis.keys('events_archive:*')