flapjack 0.9.6 → 1.0.0rc1

data/lib/flapjack/cli/server.rb

@@ -0,0 +1,256 @@
+#!/usr/bin/env ruby
+
+require 'dante'
+
+require 'oj'
+Oj.mimic_JSON
+Oj.default_options = { :indent => 0, :mode => :strict }
+require 'active_support/json'
+
+require 'flapjack/coordinator'
+
+module Flapjack
+  module CLI
+    class Server
+
+      def initialize(global_options, options)
+        @global_options = global_options
+        @options = options
+
+        @config = Flapjack::Configuration.new
+        @config.load(global_options[:config])
+        @config_env = @config.all
+
+        if @config_env.nil? || @config_env.empty?
+          puts "No config data for environment '#{FLAPJACK_ENV}' found in '#{global_options[:config]}'"
+          exit 1
+        end
+
+        if options[:rbtrace]
+          require 'rbtrace'
+        end
+      end
+
+      def start
+        if runner.daemon_running?
+          puts "Flapjack is already running."
+        else
+          print "Flapjack starting..."
+          print "\n" unless @options[:daemonize]
+          return_value = nil
+          runner.execute(:daemonize => @options[:daemonize]) {
+            return_value = start_server
+          }
+          puts " done."
+          exit(return_value) unless return_value.nil?
+        end
+      end
+
+      def stop
+        if runner.daemon_running?
+          print "Flapjack stopping..."
+          runner.execute(:kill => true)
+          puts " done."
+        else
+          puts "Flapjack is not running."
+        end
+        exit 1 unless wait_pid_gone(get_pid)
+      end
+
+      def restart
+        if runner.daemon_running?
+          print "Flapjack stopping..."
+          runner.execute(:kill => true)
+          puts " done."
+        end
+        exit 1 unless wait_pid_gone(get_pid)
+
+        @runner = nil
+
+        print "Flapjack starting..."
+        runner.execute(:daemonize => true) {
+          start_server
+        }
+        puts " done."
+      end
+
+      def reload
+        if runner.daemon_running?
+          pid = get_pid
+          print "Reloading Flapjack configuration..."
+          begin
+            Process.kill('HUP', pid)
+            puts " sent HUP to pid #{pid}."
+          rescue => e
+            puts " couldn't send HUP to pid '#{pid}'."
+          end
+        else
+          puts "Flapjack is not running daemonized."
+          exit 1
+        end
+      end
+
+      def status
+        pidfile = @options[:pidfile] || @config_env['pid_file'] ||
+          "/var/run/flapjack/flapjack.pid"
+
+        uptime = (runner.daemon_running?) ? Time.now - File.stat(pidfile).ctime : 0
+        if runner.daemon_running?
+          pid = get_pid
+          puts "Flapjack is running: pid #{pid}, uptime #{uptime}"
+        else
+          puts "Flapjack is not running"
+          exit 3
+        end
+      end
+
+      private
+
+      def runner
+        return @runner if @runner
+
+        self.class.skip_dante_traps
+
+        pidfile = @options[:pidfile].nil? ?
+                    (@config_env['pid_file'] || "/var/run/flapjack/flapjack.pid") :
+                    @options[:pidfile]
+
+        logfile = @options[:logfile].nil? ?
+                    (@config_env['log_file'] || "/var/log/flapjack/flapjack.log") :
+                    @options[:logfile]
+
+        @runner = Dante::Runner.new('flapjack', :pid_path => pidfile,
+          :log_path => logfile)
+        @runner
+      end
+
+      def self.skip_dante_traps
+        return if Dante::Runner.respond_to?(:orig_start)
+        Dante::Runner.send(:alias_method, :orig_start, :start)
+        Dante::Runner.send(:define_method, :start) do
+          if log_path = options[:log_path] && options[:daemonize].nil?
+             redirect_output!
+          end
+
+          # skip signal traps
+          @startup_command.call(self.options) if @startup_command
+        end
+      end
+
+      def start_server
+        @coordinator = Flapjack::Coordinator.new(@config)
+        @coordinator.start(:signals => true)
+      end
+
+      def process_exists(pid)
+        return unless pid
+        begin
+          Process.kill(0, pid)
+          return true
+        rescue Errno::ESRCH
+          return false
+        end
+      end
+
+      # wait until the specified pid no longer exists, or until a timeout is reached
+      def wait_pid_gone(pid, timeout = 30)
+        print "waiting for a max of #{timeout} seconds for process #{pid} to exit" if process_exists(pid)
+        started_at = Time.now.to_i
+        while process_exists(pid)
+          break unless (Time.now.to_i - started_at < timeout)
+          print '.'
+          sleep 1
+        end
+        puts ''
+        !process_exists(pid)
+      end
+
+      def get_pid
+        IO.read(pidfile).chomp.to_i
+      rescue StandardError
+        pid = nil
+      end
+
+    end
+  end
+end
+
+desc 'Server for running components (e.g. processor, notifier, gateways)'
+command :server do |server|
+
+  server.desc 'Start the server'
+
+  server.command :start do |start|
+
+    start.switch [:d, 'daemonize'], :desc => 'Daemonize',
+      :default_value => true
+
+    start.flag   [:p, 'pidfile'],   :desc => 'PATH of the pidfile to write to'
+
+    start.flag   [:l, 'logfile'],   :desc => 'PATH of the logfile to write to'
+
+    start.flag   [:r, 'rbtrace'],   :desc => 'Enable rbtrace profiling'
+
+    start.action do |global_options,options,args|
+      server = Flapjack::CLI::Server.new(global_options, options)
+      server.start
+    end
+  end
+
+  server.desc 'Stop the server'
+  server.command :stop do |stop|
+
+    stop.flag   [:p, 'pidfile'],   :desc => 'PATH of the pidfile to write to'
+
+    stop.flag   [:l, 'logfile'],   :desc => 'PATH of the logfile to write to'
+
+    stop.action do |global_options,options,args|
+      server = Flapjack::CLI::Server.new(global_options, options)
+      server.stop
+    end
+  end
+
+  server.desc 'Restart the server'
+  server.command :restart do |restart|
+
+    restart.flag   [:p, 'pidfile'],   :desc => 'PATH of the pidfile to write to'
+
+    restart.flag   [:l, 'logfile'],   :desc => 'PATH of the logfile to write to'
+
+    restart.flag   [:r, 'rbtrace'],   :desc => 'Enable rbtrace profiling'
+
+    restart.action do |global_options,options,args|
+      server = Flapjack::CLI::Server.new(global_options, options)
+      server.restart
+    end
+  end
+
+  server.desc 'Reload the server configuration'
+  server.command :reload do |reload|
+
+    reload.flag   [:p, 'pidfile'],   :desc => 'PATH of the pidfile to write to'
+
+    reload.flag   [:l, 'logfile'],   :desc => 'PATH of the logfile to write to'
+
+    reload.action do |global_options,options,args|
+      server = Flapjack::CLI::Server.new(global_options, options)
+      server.reload
+    end
+  end
+
+  server.desc 'Get server status'
+  server.command :status do |status|
+
+    status.flag   [:p, 'pidfile'],   :desc => 'PATH of the pidfile to write to'
+
+    status.flag   [:l, 'logfile'],   :desc => 'PATH of the logfile to write to'
+
+    status.action do |global_options,options,args|
+      server = Flapjack::CLI::Server.new(global_options, options)
+      server.status
+    end
+  end
+
+end
+
+

data/lib/flapjack/cli/simulate.rb

@@ -0,0 +1,180 @@
+#!/usr/bin/env ruby
+
+require 'redis'
+
+require 'flapjack/configuration'
+require 'flapjack/data/event'
+
+module Flapjack
+  module CLI
+    class Simulate
+
+      def initialize(global_options, options)
+        @global_options = global_options
+        @options = options
+
+        config = Flapjack::Configuration.new
+        config.load(global_options[:config])
+        @config_env = config.all
+
+        if @config_env.nil? || @config_env.empty?
+          puts "No config data for environment '#{FLAPJACK_ENV}' found in '#{global_options[:config]}'"
+          exit 1
+        end
+
+        @redis_options = config.for_redis
+      end
+
+      def _fail
+        events(:state => @options[:state], :recover => false,
+          :entity => @options[:entity], :check => @options[:check],
+          :tags => @options[:tags],
+          :minutes => @options[:time].to_f, :interval => @options[:interval].to_f)
+      end
+
+      def fail_and_recover
+        events(:state => @options[:state], :recover => true,
+          :entity => @options[:entity], :check => @options[:check],
+          :tags => @options[:tags],
+          :minutes => @options[:time].to_f, :interval => @options[:interval].to_f)
+      end
+
+      def ok
+        events(:state => 'ok', :recover => false,
+          :entity => @options[:entity], :check => @options[:check],
+          :tags => @options[:tags],
+          :minutes => @options[:time].to_f, :interval => @options[:interval].to_f)
+      end
+
+      private
+
+      def redis
+        @redis ||= Redis.new(@redis_options)
+      end
+
+      def events(opts = {})
+        stop_after = (opts[:minutes] * 60).to_i
+        recover = opts[:recover]
+        state = opts[:state] || 'critical'
+        event = {
+          'entity' => opts[:entity] || 'foo-app-01',
+          'check'  => opts[:check]  || 'HTTP',
+          'tags'   => opts[:tags]   || [],
+          'type'   => 'service'
+        }
+        failure  = event.merge('state' => state, 'summary' => 'Simulated check output (test by operator)')
+        recovery = event.merge('state' => 'ok',  'summary' => 'Simulated check output (test by operator)')
+        key = "#{event['entity']}:#{event['check']}"
+
+        puts "#{Time.now}: sending failure event (#{state}) for #{key}"
+        Flapjack::Data::Event.add(failure, :redis => redis)
+
+        EM.run do
+
+          EM.add_timer(stop_after) do
+            puts "#{Time.now}: stopping"
+            if recover
+              puts "#{Time.now}: sending recovery event for #{key}"
+              Flapjack::Data::Event.add(recovery.merge('time' => Time.now.to_i),
+                :redis => redis)
+            end
+            EM.stop
+          end
+
+          EM.add_periodic_timer(opts[:interval]) do
+            puts "#{Time.now}: sending failure event (#{state}) for #{key}"
+            Flapjack::Data::Event.add(failure.merge('time' => Time.now.to_i),
+              :redis => redis)
+          end
+
+        end
+      end
+
+    end
+  end
+end
+
+desc 'Generate streams of events in various states'
+command :simulate do |simulate|
+
+  simulate.desc 'Generate a stream of failure events'
+  simulate.command :fail do |_fail|
+    # Because `fail` is a keyword (that you can actually override in a block,
+    # but we want to be good Ruby citizens).
+
+    _fail.flag [:t, 'time'],     :desc => "MINUTES to generate failure events for (0.75)",
+      :default_value => 0.75
+
+    _fail.flag [:i, 'interval'], :desc => "SECONDS between events, can be decimal eg 0.1 (10)",
+      :default_value => 10
+
+    _fail.flag [:e, 'entity'],   :desc => "ENTITY to generate failure events for ('foo-app-01')",
+      :default_value => 'foo-app-01'
+
+    _fail.flag [:k, 'check'],    :desc => "CHECK to generate failure events for ('HTTP')",
+      :default_value => 'HTTP'
+
+    _fail.flag [:s, 'state'],    :desc => "STATE to generate failure events with ('critical')",
+      :default_value => 'critical'
+
+    _fail.flag [:T, 'tags'],    :desc => "optional comma-separated list of TAGS to include as part of the event",
+      :type => Array, :default_value => []
+
+    _fail.action do |global_options,options,args|
+      simulate = Flapjack::CLI::Simulate.new(global_options, options)
+      simulate._fail
+    end
+  end
+
+  simulate.desc 'Generate a stream of failure events, and one final recovery'
+  simulate.command :fail_and_recover do |fail_and_recover|
+
+    fail_and_recover.flag [:t, 'time'],     :desc => "MINUTES to generate failure events for (0.75)",
+      :default_value => 0.75
+
+    fail_and_recover.flag [:i, 'interval'], :desc => "SECONDS between events, can be decimal eg 0.1 (10)",
+      :default_value => 10
+
+    fail_and_recover.flag [:e, 'entity'],   :desc => "ENTITY to generate failure events for ('foo-app-01')",
+      :default_value => 'foo-app-01'
+
+    fail_and_recover.flag [:k, 'check'],    :desc => "CHECK to generate failure events for ('HTTP')",
+      :default_value => 'HTTP'
+
+    fail_and_recover.flag [:s, 'state'],    :desc => "STATE to generate failure events with ('critical')",
+      :default_value => 'critical'
+
+    fail_and_recover.flag [:T, 'tags'],     :desc => "optional comma-separated list of TAGS to include as part of the event",
+      :type => Array, :default_value => []
+
+    fail_and_recover.action do |global_options,options,args|
+      simulate = Flapjack::CLI::Simulate.new(global_options, options)
+      simulate.fail_and_recover
+    end
+  end
+
+  simulate.desc 'Generate a stream of ok events'
+  simulate.command :ok do |ok|
+
+    ok.flag [:t, 'time'],     :desc => "MINUTES to generate ok events for (0.75)",
+      :default_value => 0.75
+
+    ok.flag [:i, 'interval'], :desc => "SECONDS between events, can be decimal eg 0.1 (10)",
+      :default_value => 10
+
+    ok.flag [:e, 'entity'],   :desc => "ENTITY to generate ok events for ('foo-app-01')",
+      :default_value => 'foo-app-01'
+
+    ok.flag [:k, 'check'],    :desc => "CHECK to generate ok events for ('HTTP')",
+      :default_value => 'HTTP'
+
+    ok.flag [:T, 'tags'],     :desc => "optional comma-separated list of TAGS to include as part of the event",
+      :type => Array, :default_value => []
+
+    ok.action do |global_options,options,args|
+      simulate = Flapjack::CLI::Simulate.new(global_options, options)
+      simulate.ok
+    end
+  end
+
+end

data/lib/flapjack/configuration.rb

@@ -45,6 +45,8 @@ module Flapjack
                         :port => (redis['port'] || 6379)})
       )
 
+      redis_config[:password] = redis["password"] if redis["password"]
+
       redis_config
     end
 

data/lib/flapjack/data/entity_check.rb

@@ -30,23 +30,10 @@ module Flapjack
       def self.for_event_id(event_id, options = {})
         raise "Redis connection not set" unless redis = options[:redis]
         entity_name, check_name = event_id.split(':', 2)
-        logger = options[:logger]
-        if logger
-          logger.debug("Flapjack::Data::EntityCheck#self.for_event_id called for event_id: #{event_id}")
-          logger.debug("Flapjack::Data::EntityCheck#self.for_event_id found entity_name: #{entity_name}, check_name: #{check_name}")
-        end
         create_entity = options[:create_entity]
+        logger = options[:logger]
         entity = Flapjack::Data::Entity.find_by_name(entity_name,
           :create => create_entity, :logger => logger, :redis => redis)
-        if logger
-          logger.debug("Flapjack::Data::EntityCheck#self.for_event_id found entity by name(#{entity_name}): #{entity.inspect}")
-        end
-        unless entity
-          if logger
-            logger.error("entity can't be instantiated for #{entity_name}!")
-          end
-          return nil
-        end
         self.new(entity, check_name, :logger => logger, :redis => redis)
       end
 
@@ -127,13 +114,12 @@ module Flapjack
 
       def self.unacknowledged_failing(options = {})
         raise "Redis connection not set" unless redis = options[:redis]
-        logger = options[:logger]
 
         redis.zrange('failed_checks', '0', '-1').reject {|entity_check|
           redis.exists(entity_check + ':unscheduled_maintenance')
         }.collect {|entity_check|
-          Flapjack::Data::EntityCheck.for_event_id(entity_check, :redis => redis, :logger => logger)
-        }.compact
+          Flapjack::Data::EntityCheck.for_event_id(entity_check, :redis => redis)
+        }
       end
 
       def self.conflate_to_keys(entity_checks_hash)
@@ -329,11 +315,8 @@ module Flapjack
         # if multiple scheduled maintenances found, find the end_time furthest in the future
         most_futuristic = current_sched_ms.max {|sm| sm[:end_time] }
         start_time = most_futuristic[:start_time]
-
-        duration = most_futuristic[:end_time] - current_time
-        if duration > 0
-          @redis.setex("#{@key}:scheduled_maintenance", duration.to_i, start_time)
-        end
+        duration   = most_futuristic[:duration]
+        @redis.setex("#{@key}:scheduled_maintenance", duration.to_i, start_time)
       end
 
       # TODO allow summary to be changed as part of the termination

data/lib/flapjack/data/event.rb

@@ -3,8 +3,6 @@
 require 'oj'
 require 'flapjack/data/tag_set'
 
-require 'flapjack/data/migration'
-
 module Flapjack
   module Data
     class Event
@@ -80,27 +78,24 @@ module Flapjack
                      :events_archive_maxage => (3 * 60 * 60) }
         options  = defaults.merge(opts)
 
-        archive_dest  = nil
+        archive_dest = nil
         base_time_str = Time.now.utc.strftime("%Y%m%d%H")
 
         if options[:archive_events]
           archive_dest = "events_archive:#{base_time_str}"
-          unless @previous_base_time_str == base_time_str
-            Flapjack::Data::Migration.purge_expired_archive_index(:redis => redis)
-          end
-          @previous_base_time_str = base_time_str
           if options[:block]
             raw = redis.brpoplpush(queue, archive_dest, 0)
           else
             raw = redis.rpoplpush(queue, archive_dest)
             return unless raw
           end
-          redis.sadd("known_events_archive_keys", archive_dest)
-        elsif options[:block]
-          raw = redis.brpop(queue, 0)[1]
         else
-          raw = redis.rpop(queue)
-          return unless raw
+          if options[:block]
+            raw = redis.brpop(queue, 0)[1]
+          else
+            raw = redis.rpop(queue)
+            return unless raw
+          end
         end
         parsed = parse_and_validate(raw, :logger => options[:logger])
         if parsed.nil?

data/lib/flapjack/gateways/email.rb

@@ -52,6 +52,7 @@ module Flapjack
           port = @smtp_config ? @smtp_config['port'] : nil
           starttls = @smtp_config ? !! @smtp_config['starttls'] : nil
           m_from = @smtp_config ? @smtp_config['from'] : "flapjack@#{@fqdn}"
+          m_reply_to = @smtp_config ? ( @smtp_config['reply_to'] ||= m_from ) : "flapjack@#{@fqdn}"
           if @smtp_config
             if auth_config = @smtp_config['auth']
               auth = {}
@@ -64,6 +65,7 @@ module Flapjack
           @logger.debug("flapjack_mailer: set from to #{m_from}")
 
           mail = prepare_email(:from       => m_from,
+                               :reply_to   => m_reply_to,
                                :to         => alert.address,
                                :message_id => "<#{alert.notification_id}@#{@fqdn}>",
                                :alert      => alert)
@@ -103,6 +105,7 @@ module Flapjack
         # returns a Mail object
         def prepare_email(opts = {})
           from       = opts[:from]
+          reply_to   = opts[:reply_to] 
           to         = opts[:to]
           message_id = opts[:message_id]
           alert      = opts[:alert]
@@ -157,7 +160,7 @@ module Flapjack
             from       from
             to         to
             subject    subject
-            reply_to   from
+            reply_to   reply_to
             message_id message_id
 
             text_part do