flapjack 1.0.0rc3 → 1.0.0rc5

data/lib/flapjack/cli/server.rb

@@ -22,8 +22,25 @@ module Flapjack
         @config_env = @config.all
 
         if @config_env.nil? || @config_env.empty?
-          puts "No config data for environment '#{FLAPJACK_ENV}' found in '#{global_options[:config]}'"
-          exit 1
+          exit_now! "No config data for environment '#{FLAPJACK_ENV}' found in '#{global_options[:config]}'"
+        end
+
+        @pidfile = case
+        when !@options[:pidfile].nil?
+          @options[:pidfile]
+        when !@config_env['pid_dir'].nil?
+          @config_env['pid_dir'] + 'flapjack.pid'
+        else
+          "/var/run/flapjack/flapjack.pid"
+        end
+
+        @logfile = case
+        when !@options[:logfile].nil?
+          @options[:logfile]
+        when !@config_env['log_dir'].nil?
+          @config_env['log_dir'] + 'flapjack.log'
+        else
+          "/var/run/flapjack/flapjack.log"
         end
 
         if options[:rbtrace]
@@ -42,11 +59,12 @@ module Flapjack
             return_value = start_server
           }
           puts " done."
-          exit(return_value) unless return_value.nil?
+          exit_now!(return_value) unless return_value.nil?
         end
       end
 
       def stop
+        pid = get_pid
         if runner.daemon_running?
           print "Flapjack stopping..."
           runner.execute(:kill => true)
@@ -54,16 +72,17 @@ module Flapjack
         else
           puts "Flapjack is not running."
         end
-        exit 1 unless wait_pid_gone(get_pid)
+        exit_now! "Failed to stop Flapjack #{pid}" unless wait_pid_gone(pid)
       end
 
       def restart
+        pid = get_pid
         if runner.daemon_running?
           print "Flapjack stopping..."
           runner.execute(:kill => true)
           puts " done."
         end
-        exit 1 unless wait_pid_gone(get_pid)
+        exit_now! "Failed to stop Flapjack #{pid}" unless wait_pid_gone(pid)
 
         @runner = nil
 
@@ -85,22 +104,17 @@ module Flapjack
             puts " couldn't send HUP to pid '#{pid}'."
           end
         else
-          puts "Flapjack is not running daemonized."
-          exit 1
+          exit_now! "Flapjack is not running daemonized."
         end
       end
 
       def status
-        pidfile = @options[:pidfile] || @config_env['pid_file'] ||
-          "/var/run/flapjack/flapjack.pid"
-
-        uptime = (runner.daemon_running?) ? Time.now - File.stat(pidfile).ctime : 0
         if runner.daemon_running?
           pid = get_pid
+          uptime = Time.now - File.stat(@pidfile).ctime
           puts "Flapjack is running: pid #{pid}, uptime #{uptime}"
         else
-          puts "Flapjack is not running"
-          exit 3
+          exit_now! "Flapjack is not running"
         end
       end
 
@@ -111,16 +125,8 @@ module Flapjack
 
         self.class.skip_dante_traps
 
-        pidfile = @options[:pidfile].nil? ?
-                    (@config_env['pid_file'] || "/var/run/flapjack/flapjack.pid") :
-                    @options[:pidfile]
-
-        logfile = @options[:logfile].nil? ?
-                    (@config_env['log_file'] || "/var/log/flapjack/flapjack.log") :
-                    @options[:logfile]
-
-        @runner = Dante::Runner.new('flapjack', :pid_path => pidfile,
-          :log_path => logfile)
+        @runner = Dante::Runner.new('flapjack', :pid_path => @pidfile,
+          :log_path => @logfile)
         @runner
       end
 
@@ -166,7 +172,7 @@ module Flapjack
       end
 
       def get_pid
-        IO.read(pidfile).chomp.to_i
+        IO.read(@pidfile).chomp.to_i
       rescue StandardError
         pid = nil
       end
@@ -252,5 +258,3 @@ command :server do |server|
   end
 
 end
-
-

data/lib/flapjack/cli/simulate.rb

@@ -21,8 +21,7 @@ module Flapjack
         @config_env = config.all
 
         if @config_env.nil? || @config_env.empty?
-          puts "No config data for environment '#{FLAPJACK_ENV}' found in '#{global_options[:config]}'"
-          exit 1
+          exit_now! "No config data for environment '#{FLAPJACK_ENV}' found in '#{global_options[:config]}'"
         end
 
         @redis_options = config.for_redis
@@ -101,7 +100,7 @@ module Flapjack
   end
 end
 
-desc 'Generate streams of events in various states'
+desc 'Simulates a check by creating a stream of events for Flapjack to process'
 command :simulate do |simulate|
 
   simulate.desc 'Generate a stream of failure events'

data/lib/flapjack/data/contact.rb

@@ -22,7 +22,7 @@ module Flapjack
         :media_intervals, :media_rollup_thresholds, :pagerduty_credentials
 
       TAG_PREFIX = 'contact_tag'
-      ALL_MEDIA  = ['email', 'sms', 'jabber', 'pagerduty']
+      ALL_MEDIA  = ['email', 'sms', 'jabber', 'pagerduty', 'sns']
 
       def self.all(options = {})
         raise "Redis connection not set" unless redis = options[:redis]

data/lib/flapjack/data/entity.rb

@@ -18,29 +18,432 @@ module Flapjack
 
       def self.all(options = {})
         raise "Redis connection not set" unless redis = options[:redis]
+
+        current_entity_names = (options.has_key?(:enabled) && !options[:enabled].nil?) ?
+          Flapjack::Data::Entity.current_names(:redis => redis) : nil
+
         keys = redis.keys("entity_id:*")
         return [] unless keys.any?
         ids = redis.mget(keys)
-        keys.collect {|k|
-          k =~ /^entity_id:(.+)$/; entity_name = $1
-          self.new(:name => entity_name, :id => ids.shift, :redis => redis)
+        keys.inject([]) {|memo, k|
+          k =~ /^entity_id:(.+)$/; entity_name = $1; entity_id = ids.shift
+
+          if options[:enabled].nil? ||
+            (options[:enabled].is_a?(TrueClass) && current_entity_names.include?(entity_name) ) ||
+            (options[:enabled].is_a?(FalseClass) && !current_entity_names.include?(entity_name))
+
+            memo << self.new(:name => entity_name, :id => entity_id, :redis => redis)
+          end
+
+          memo
         }.sort_by(&:name)
       end
 
-      # NB: should probably be called in the context of a Redis multi block; not doing so
-      # here as calling classes may well be adding/updating multiple records in the one
-      # operation
+      # no way to lock all data operations, so hit & hope... at least the renames
+      # should be atomic
+      def self.rename(existing_name, entity_name, options = {})
+        raise "Redis connection not set" unless redis = options[:redis]
+
+        check_state_keys = redis.keys("check:#{existing_name}:*")
+
+        check_history_keys = redis.keys("#{existing_name}:*:states") +
+          redis.keys("#{existing_name}:*:state") +
+          redis.keys("#{existing_name}:*:summary") +
+          redis.keys("#{existing_name}:*:sorted_state_timestamps")
+
+        action_keys = redis.keys("#{existing_name}:*:actions")
+
+        maint_keys = redis.keys("#{existing_name}:*:*scheduled_maintenance*") +
+
+        notif_keys = redis.keys("#{existing_name}:*:last_*_notification") +
+          redis.keys("#{existing_name}:*:*_notifications")
+
+        alerting_check_keys = redis.keys("contact_alerting_checks:*")
+
+        failed_checks    = {}
+        hashes_to_remove = []
+        hashes_to_add    = {}
+
+        alerting_to_remove = {}
+        alerting_to_add    = {}
+
+        sha1 = Digest::SHA1.new
+
+        checks = check_state_keys.collect do |state_key|
+          state_key =~ /^check:#{Regexp.escape(existing_name)}:(.+)$/
+          $1
+        end
+
+        checks.each do |ch|
+          existing_check = "#{existing_name}:#{ch}"
+          new_check      = "#{entity_name}:#{ch}"
+
+          ch_fail_score = redis.zscore("failed_checks", existing_check)
+          failed_checks[ch] = ch_fail_score unless ch_fail_score.nil?
+
+          hashes_to_remove << Digest.hexencode(sha1.digest(existing_check))[0..7].downcase
+          hashes_to_add[Digest.hexencode(sha1.digest(new_check))[0..7].downcase] = new_check
+
+          alerting_check_keys.each do |ack|
+            ack_score = redis.zscore(ack, existing_check)
+            unless ack_score.nil?
+              alerting_to_remove[ack] ||= []
+              alerting_to_remove[ack] << existing_check
+
+              alerting_to_add[ack]    ||= {}
+              alerting_to_add[ack][new_check] = ack_score
+            end
+          end
+        end
+
+        current_score = redis.zscore('current_entities', existing_name)
+
+        block_keys = redis.keys("drop_alerts_for_contact:*:*:#{existing_name}:*:*")
+
+        redis.multi
+
+        yield if block_given? # entity id -> name update from add()
+
+        check_state_keys.each do |csk|
+          redis.rename(csk, csk.sub(/^check:#{Regexp.escape(existing_name)}:/, "check:#{entity_name}:"))
+        end
+
+        (check_history_keys + action_keys + maint_keys + notif_keys).each do |chk|
+          redis.rename(chk, chk.sub(/^#{Regexp.escape(existing_name)}:/, "#{entity_name}:"))
+        end
+
+        # currently failing checks
+        failed_checks.each_pair do |ch, score|
+          redis.zrem('failed_checks', "#{existing_name}:#{ch}")
+          redis.zadd('failed_checks', score, "#{entity_name}:#{ch}")
+        end
+
+        redis.rename("current_checks:#{existing_name}", "current_checks:#{entity_name}")
+
+        unless current_score.nil?
+          redis.zrem('current_entities', existing_name)
+          redis.zadd('current_entities', current_score, entity_name)
+        end
+
+        block_keys.each do |blk|
+          redis.rename(blk, blk.sub(/^drop_alerts_for_contact:(.+):([^:]+):#{Regexp.escape(existing_name)}:(.+):([^:]+)$/,
+            "drop_alerts_for_contact:\\1:\\2:#{entity_name}:\\3:\\4"))
+        end
+
+        hashes_to_remove.each   {|hash|      redis.hdel('checks_by_hash', hash) }
+        hashes_to_add.each_pair {|hash, chk| redis.hset('checks_by_hash', hash, chk)}
+
+        alerting_to_remove.each_pair do |alerting, chks|
+          chks.each {|chk| redis.zrem(alerting, chk)}
+        end
+
+        alerting_to_add.each_pair do |alerting, chks|
+          chks.each_pair {|chk, score| redis.zadd(alerting, score, chk)}
+        end
+
+        redis.exec
+      end
+
+      # NB only used by the 'entities:reparent' Rake task, but kept in this
+      # class to be more easily testable
+      def self.merge(old_name, current_name, options = {})
+        raise "Redis connection not set" unless redis = options[:redis]
+
+        check_state_keys = redis.keys("check:#{old_name}:*")
+
+        checks = check_state_keys.collect do |state_key|
+          state_key =~ /^check:#{Regexp.escape(old_name)}:(.+)$/
+          $1
+        end
+
+        alerting_check_keys = redis.keys("contact_alerting_checks:*")
+
+        keys_to_delete = []
+        keys_to_rename = {}
+
+        failed_checks_to_remove = []
+        failed_checks_to_add    = {}
+
+        action_data    = {}
+
+        notification_types = ['problem', 'unknown', 'warning', 'critical',
+          'recovery', 'acknowledgement']
+
+        alerting_check_keys = redis.keys("contact_alerting_checks:*")
+
+        alerting_to_remove = {}
+        alerting_to_add    = {}
+
+        block_keys = redis.keys("drop_alerts_for_contact:*:*:#{old_name}:*:*")
+
+        checks.each do |ch|
+          old_check     = "#{old_name}:#{ch}"
+          current_check = "#{current_name}:#{ch}"
+
+          old_states = "#{old_check}:states"
+          new_states = "#{current_check}:states"
+
+          failed_checks_to_remove << old_check
+
+          if redis.exists("check:#{current_check}")
+            keys_to_delete << "check:#{old_check}"
+
+            loop do
+              # pop from tail, append at head, matches ordering in EntityCheck#update_state
+              break if redis.rpoplpush(old_states, new_states).nil?
+            end
+
+            keys_to_delete << old_states
+          else
+
+            # can move a failing checks entry over, if it exists
+            ch_fail_score = redis.zscore("failed_checks", old_check)
+            failed_checks_to_add[current_check] = ch_fail_score unless ch_fail_score.nil?
+
+            keys_to_rename["check:#{old_check}"] = "check:#{current_check}"
+            keys_to_rename[old_states]           = new_states
+          end
+
+          notification_types.each do |notif|
+
+            old_notif = "#{old_check}:#{notif}_notifications"
+            new_notif = "#{current_check}:#{notif}_notifications"
+
+            if redis.exists(new_notif)
+              loop do
+                # pop from tail, append at head
+                break if redis.rpoplpush(old_notif, new_notif).nil?
+              end
+
+              keys_to_delete << old_notif
+            else
+              keys_to_rename[old_notif] = new_notif
+            end
+          end
+
+          alerting_check_keys.each do |ack|
+            old_score = redis.zscore(ack, old_check)
+            new_score = redis.zscore(ack, current_check)
+
+            alerting_to_remove[ack] ||= []
+            alerting_to_remove[ack] << old_check
+
+            # nil.to_i == 0, which is good for a missing value
+            if !old_score.nil? && new_score.nil? &&
+               (redis.lindex("#{old_check}:problem_notifications", -1).to_i >
+                [redis.lindex("#{current_check}:recovery_notifications", -1).to_i,
+                 redis.lindex("#{current_check}:acknowledgement_notifications", -1).to_i].max)
+
+              alerting_to_add[ack]    ||= {}
+              alerting_to_add[ack][current_check] = old_score
+            end
+          end
+
+        end
+
+        if redis.exists("current_checks:#{current_name}")
+          keys_to_delete << "current_checks:#{old_name}"
+        else
+          keys_to_rename["current_checks:#{old_name}"] = "current_checks:#{current_name}"
+        end
+
+        current_score = redis.zscore('current_entities', current_name)
+        old_score     = nil
+
+        if current_score.nil?
+          old_score = redis.zscore('current_entities', old_name)
+        end
+
+        check_timestamps_keys = redis.keys("#{old_name}:*:sorted_state_timestamps")
+        keys_to_delete += check_timestamps_keys
+
+        check_history_keys = redis.keys("#{old_name}:*:state") +
+          redis.keys("#{old_name}:*:summary")
+
+        action_keys = redis.keys("#{old_name}:*:actions")
+
+        action_keys.each do |old_actions|
+
+          old_actions =~ /^#{Regexp.escape(old_name)}:(.+):actions$/
+          current_actions = "#{current_name}:#{$1}:actions"
+
+          if redis.exists(current_actions)
+            action_data[current_actions] = redis.hgetall(old_actions)
+            keys_to_delete << old_actions
+          else
+            keys_to_rename[old_actions] = current_actions
+          end
+        end
+
+        maint_keys = redis.keys("#{old_name}:*:*scheduled_maintenance")
+
+        maints_to_delete = []
+        maints_to_set    = {}
+
+        maint_keys.each do |maint_key|
+          maint_key =~ /^#{Regexp.escape(old_name)}:(.+):((?:un)?scheduled_maintenance)$/
+          maint_check = $1
+          maint_type  = $2
+
+          new_maint_key = "#{current_name}:#{maint_check}:#{maint_type}"
+
+          # as keys are expiring, check all steps in case they have
+          old_time, new_time = redis.mget(maint_key, new_maint_key).map(&:to_i)
+
+          old_ttl = (old_time <= 0) ? -1 : redis.ttl(maint_key)
+          new_ttl = (new_time <= 0) ? -1 : redis.ttl(new_maint_key)
+
+          # TTL < 0 is a redis error code -- key not present, etc.
+          if (old_ttl >= 0) && ((new_ttl < 0) ||
+               ((old_time + old_ttl) > (new_time + new_ttl)))
+            keys_to_rename[maint_key] = new_maint_key
+            maints_to_set[new_maint_key] = redis.zscore("#{maint_key}s", old_time)
+          end
+
+          keys_to_delete << maint_key
+        end
+
+        blocks_to_set = {}
+
+        block_keys.each do |block_key|
+          block_key =~ /^drop_alerts_for_contact:(.+):([^:]+):#{Regexp.escape(old_name)}:(.+):([^:]+)$/
+          new_block_key = "drop_alerts_for_contact:#{$1}:#{$2}:#{current_name}:#{$3}:#{$4}"
+
+          # as keys may expire, check whether they have
+          old_start_ttl, new_start_ttl = redis.mget(block_key, new_block_key).map(&:to_i)
+
+          old_ttl = (old_start_ttl <= 0) ? -1 : redis.ttl(block_key)
+          new_ttl = (new_start_ttl <= 0) ? -1 : redis.ttl(new_block_key)
+
+          # TTL < 0 is a redis error code -- key not present, etc.
+          if (old_ttl >= 0) && ((new_ttl < 0) || (old_ttl > new_ttl))
+            blocks_to_set[new_block_key] = [Time.now.to_i + old_ttl, old_start_ttl]
+          end
+
+          keys_to_delete << block_key
+        end
+
+        stored_maint_keys = redis.keys("#{old_name}:*:*scheduled_maintenances") +
+          redis.keys("#{old_name}:*:sorted_*scheduled_maintenance_timestamps")
+        keys_to_delete += stored_maint_keys
+
+        notif_keys = redis.keys("#{old_name}:*:last_*_notification")
+
+        redis.multi
+
+        check_history_keys.each do |chk|
+          redis.renamenx(chk, chk.sub(/^#{Regexp.escape(old_name)}:/, "#{current_name}:"))
+        end
+
+        check_timestamps_keys.each do |ctk|
+          dest = ctk.sub(/^#{Regexp.escape(old_name)}:/, "#{current_name}:")
+          redis.zunionstore(dest, [ctk, dest], :aggregate => :max)
+        end
+
+        failed_checks_to_remove.each do |fctr|
+          redis.zrem('failed_checks', fctr)
+        end
+
+        failed_checks_to_add.each_pair do |fcta, score|
+          redis.zadd('failed_checks', score, fcta)
+        end
+
+        action_data.each_pair do |action_key, data|
+          data.each_pair do |k, v|
+            redis.hsetnx(action_key, k, v)
+          end
+        end
+
+        redis.zunionstore("current_checks:#{current_name}",
+          ["current_checks:#{old_name}", "current_checks:#{current_name}"],
+          :aggregate => :max)
+
+        redis.zrem('current_entities', old_name)
+        unless old_score.nil?
+          redis.zadd('current_entities', old_score, current_name)
+        end
+
+        maints_to_set.each_pair do |maint_key, score|
+          redis.zadd("#{maint_key}s", score, current_name)
+        end
+
+        stored_maint_keys.each do |stored_maint_key|
+          new_stored_maint_key = stored_maint_key.sub(/^#{Regexp.escape(old_name)}:/, "#{current_name}:")
+          redis.zunionstore(new_stored_maint_key,
+            [stored_maint_key, new_stored_maint_key],
+            :aggregate => :max)
+        end
+
+        notif_keys.each do |nk|
+          dest = nk.sub(/^#{Regexp.escape(old_name)}:/, "#{current_name}:")
+          redis.renamenx(nk, dest)
+          redis.del(nk)
+        end
+
+        alerting_to_remove.each_pair do |alerting, chks|
+          chks.each {|chk| redis.zrem(alerting, chk)}
+        end
+
+        alerting_to_add.each_pair do |alerting, chks|
+          chks.each_pair {|chk, score| redis.zadd(alerting, score, chk)}
+        end
+
+        blocks_to_set.each_pair do |block_key, (timestamp, value)|
+          redis.setex(block_key, (timestamp - Time.now.to_i), value)
+        end
+
+        keys_to_rename.each_pair do |old_key, new_key|
+          redis.rename(old_key, new_key)
+        end
+
+        redis.del(*keys_to_delete) unless keys_to_delete.empty?
+
+        redis.exec
+      end
+
+      # NB: If entities are renamed in imported data before they are
+      # renamed in monitoring sources, data for old entities may still
+      # arrive and be stored under those names.
       def self.add(entity, options = {})
         raise "Redis connection not set" unless redis = options[:redis]
-        raise "Entity name not provided" unless entity['name'] && !entity['name'].empty?
+        entity_name = entity['name']
+        raise "Entity name not provided" if entity_name.nil? || entity_name.empty?
+
+        entity_id = entity['id']
+
+        if entity_id.nil?
+          # likely to be from monitoring data
+
+          # if an entity exists with the same name as the incoming data,
+          # use its id; failing that allocate a random one
+          entity_id = redis.get("entity_id:#{entity_name}")
 
-        #FIXME: should probably raise an exception if trying to create a new entity with the
-        # same name or id as an existing entity. (Go away and use update instead.)
-        entity_id = entity['id'] ? entity['id'] : SecureRandom.uuid
-        existing_name = redis.hget("entity:#{entity_id}", 'name')
-        redis.del("entity_id:#{existing_name}") unless existing_name == entity['name']
-        redis.set("entity_id:#{entity['name']}", entity_id)
-        redis.hset("entity:#{entity_id}", 'name', entity['name'])
+          if entity_id.nil? || entity_id.empty?
+            entity_id = SecureRandom.uuid
+            redis.set("entity_id:#{entity_name}", entity_id)
+            redis.hset("entity:#{entity_id}", 'name', entity_name)
+          end
+        else
+          # most likely from API import
+          existing_name = redis.hget("entity:#{entity_id}", 'name')
+
+          if existing_name.nil?
+
+            # if there's an entity with a matching name, this will change its
+            # id; if no entity exists it creates a new one
+            redis.set("entity_id:#{entity_name}", entity_id)
+            redis.hset("entity:#{entity_id}", 'name', entity_name)
+
+          elsif existing_name != entity_name
+            if redis.renamenx("entity_id:#{existing_name}", "entity_id:#{entity_name}")
+              rename(existing_name, entity_name, :redis => redis) {
+                redis.hset("entity:#{entity_id}", 'name', entity_name)
+              }
+            else
+              merge(existing_name, entity_name, :redis => redis)
+            end
+          end
+        end
 
         redis.del("contacts_for:#{entity_id}")
         if entity['contacts'] && entity['contacts'].respond_to?(:each)
@@ -49,7 +452,7 @@ module Flapjack
             redis.sadd("contacts_for:#{entity_id}", contact_id)
           }
         end
-        self.new(:name  => entity['name'],
+        self.new(:name  => entity_name,
                  :id    => entity_id,
                  :redis => redis)
       end
@@ -57,14 +460,14 @@ module Flapjack
       def self.find_by_name(entity_name, options = {})
         raise "Redis connection not set" unless redis = options[:redis]
         entity_id = redis.get("entity_id:#{entity_name}")
-        if entity_id.nil?
+        if entity_id.nil? || entity_id.empty?
           # key doesn't exist
           return unless options[:create]
+          # add returns an instantiated Entity
           self.add({'name' => entity_name}, :redis => redis)
+        else
+          self.new(:name => entity_name, :id => entity_id, :redis => redis)
         end
-        self.new(:name => entity_name,
-                 :id => (entity_id.nil? || entity_id.empty?) ? nil : entity_id,
-                 :redis => redis)
       end
 
       def self.find_by_id(entity_id, options = {})
@@ -116,24 +519,14 @@ module Flapjack
         }.compact
       end
 
-      def self.find_all_with_checks(options)
-        raise "Redis connection not set" unless redis = options[:redis]
-        redis.zrange("current_entities", 0, -1)
-      end
-
-      def self.find_all_with_failing_checks(options)
-        raise "Redis connection not set" unless redis = options[:redis]
-        Flapjack::Data::EntityCheck.find_all_failing_by_entity(:redis => redis).keys
-      end
-
-      def self.find_all_current(options)
+      def self.current_names(options = {})
         raise "Redis connection not set" unless redis = options[:redis]
         redis.zrange('current_entities', 0, -1)
       end
 
-      def self.find_all_current_with_last_update(options)
+      def self.find_all_with_failing_checks(options)
         raise "Redis connection not set" unless redis = options[:redis]
-        redis.zrange('current_entities', 0, -1, {:withscores => true})
+        Flapjack::Data::EntityCheck.find_current_failing_by_entity(:redis => redis).keys
       end
 
       def contacts