flapjack 0.5.5 → 0.6.23

data/lib/flapjack/api/entity_presenter.rb

@@ -0,0 +1,66 @@
+#!/usr/bin/env ruby
+
+# Formats entity data for presentation by the API methods in Flapjack::API.
+# Currently this just aggregates all of the check data for an entity, leaving
+# clients to make any further calculations for themselves.
+
+require 'sinatra/base'
+
+require 'flapjack/api/entity_check_presenter'
+require 'flapjack/data/entity_check'
+
+module Flapjack
+
+  class API < Sinatra::Base
+
+    class EntityPresenter
+
+      def initialize(entity, options = {})
+        @entity = entity
+        @redis = options[:redis]
+      end
+
+      def outages(start_time, end_time)
+        checks.collect {|c|
+          {:check => c, :outages => check_presenter(c).outages(start_time, end_time)}
+        }
+      end
+
+      def unscheduled_maintenance(start_time, end_time)
+        checks.collect {|c|
+          {:check => c, :unscheduled_maintenance =>
+            check_presenter(c).unscheduled_maintenance(start_time, end_time)}
+        }
+      end
+
+      def scheduled_maintenance(start_time, end_time)
+        checks.collect {|c|
+          {:check => c, :scheduled_maintenance =>
+            check_presenter(c).scheduled_maintenance(start_time, end_time)}
+        }
+      end
+
+      def downtime(start_time, end_time)
+        checks.collect {|c|
+          {:check => c, :downtime =>
+            check_presenter(c).downtime(start_time, end_time)}
+        }
+      end
+
+    private
+
+      def checks
+        @check_list ||= @entity.check_list
+      end
+
+      def check_presenter(check)
+        entity_check = Flapjack::Data::EntityCheck.for_entity(@entity, check,
+          :redis => @redis)
+        presenter = Flapjack::API::EntityCheckPresenter.new(entity_check)
+      end
+
+    end
+
+  end
+
+end

data/lib/flapjack/cli/worker_manager.rb

@@ -1,8 +1,7 @@
 #!/usr/bin/env ruby
 
-require 'rubygems'
 require 'ostruct'
-require 'optparse' 
+require 'optparse'
 
 class WorkerManagerOptions
   def self.parse(args)

data/lib/flapjack/configuration.rb

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+
+module Flapjack
+
+  class Configuration
+
+    # TODO consolidate configuration code
+
+  end
+
+end

data/lib/flapjack/coordinator.rb

@@ -0,0 +1,288 @@
+#!/usr/bin/env ruby
+
+require 'eventmachine'
+# the redis/synchrony gems need to be required in this particular order, see
+# the redis-rb README for details
+require 'hiredis'
+require 'em-synchrony'
+require 'redis/connection/synchrony'
+require 'redis'
+require 'em-resque'
+require 'em-resque/worker'
+require 'thin'
+
+require 'flapjack/patches'
+
+require 'flapjack/api'
+require 'flapjack/daemonizing'
+require 'flapjack/executive'
+require 'flapjack/jabber'
+require 'flapjack/pagerduty'
+require 'flapjack/notification/email'
+require 'flapjack/notification/sms'
+require 'flapjack/web'
+
+module Flapjack
+
+  class Coordinator
+
+    include Flapjack::Daemonizable
+
+    def initialize(config = {})
+      @config = config
+      @pikelets = []
+
+      @logger = Log4r::Logger.new("flapjack-coordinator")
+      @logger.add(Log4r::StdoutOutputter.new("flapjack-coordinator"))
+      @logger.add(Log4r::SyslogOutputter.new("flapjack-coordinator"))
+    end
+
+    def start(options = {})
+      # FIXME raise error if config not set, or empty
+
+      if options[:daemonize]
+        daemonize
+      else
+        setup
+      end
+    end
+
+    def after_daemonize
+      setup
+    end
+
+    def stop
+      shutdown
+    end
+
+  private
+
+    def setup
+
+      # FIXME: the following is currently repeated in flapjack-populator and
+      # flapjack-nagios-receiver - move to a method in a module and include it
+      redis_host = @config['redis']['host'] || '127.0.0.1'
+      redis_port = @config['redis']['port'] || 6379
+      redis_path = @config['redis']['path'] || nil
+      redis_db   = @config['redis']['db']   || 0
+
+      if redis_path
+        @redis_options = { :db => redis_db, :path => redis_path }
+      else
+        @redis_options = { :db => redis_db, :host => redis_host, :port => redis_port }
+      end
+
+      EM.synchrony do
+
+        @logger.debug "config keys: #{@config.keys}"
+
+        pikelet_keys = ['executive', 'jabber_gateway', 'pagerduty_gateway',
+                        'email_notifier', 'sms_notifier', 'web', 'api']
+
+        @config.keys.each do |pikelet_type|
+          next unless pikelet_keys.include?(pikelet_type) && 
+            @config[pikelet_type].is_a?(Hash) &&
+            @config[pikelet_type]['enabled']
+          @logger.debug "coordinator is now initialising the #{pikelet_type} pikelet"
+          pikelet_cfg = @config[pikelet_type]
+
+          case pikelet_type
+          when 'executive', 'jabber_gateway', 'pagerduty_gateway'
+            build_pikelet(pikelet_type, pikelet_cfg)
+          when 'web', 'api'
+            build_thin_pikelet(pikelet_type, pikelet_cfg)
+          when 'email_notifier', 'sms_notifier'
+            build_resque_pikelet(pikelet_type, pikelet_cfg)
+          end
+        end
+
+        setup_signals
+      end
+
+    end
+
+    def setup_signals
+      trap('INT')  { stop }
+      trap('TERM') { stop }
+      unless RUBY_PLATFORM =~ /mswin/
+        trap('QUIT') { stop }
+        # trap('HUP')  { }
+      end
+    end
+
+    def build_pikelet(pikelet_type, pikelet_cfg)
+      pikelet_class = case pikelet_type
+      when 'executive'
+        Flapjack::Executive
+      when 'jabber_gateway'
+        Flapjack::Jabber
+      when 'pagerduty_gateway'
+        Flapjack::Pagerduty
+      end
+      return unless pikelet_class
+
+      f = Fiber.new {
+        begin
+          pikelet = pikelet_class.new
+          @pikelets.detect {|p| p[:type] == pikelet_type}[:instance] = pikelet
+          pikelet.bootstrap(:redis => @redis_options, :config => pikelet_cfg)
+          pikelet.main
+        rescue Exception => e
+          trace = e.backtrace.join("\n")
+          @logger.fatal "#{e.message}\n#{trace}"
+          stop
+        end
+      }
+      @pikelets << {:fiber => f, :type => pikelet_type}
+      f.resume
+      @logger.debug "new fiber created for #{pikelet_type}"
+    end
+
+    def build_thin_pikelet(pikelet_type, pikelet_cfg)
+      pikelet_class = case pikelet_type
+      when 'web'
+        Flapjack::Web
+      when 'api'
+        Flapjack::API
+      end
+      return unless pikelet_class
+
+      port = nil
+      if pikelet_cfg['port']
+        port = pikelet_cfg['port'].to_i
+      end
+
+      port = 3001 if (port.nil? || port <= 0 || port > 65535)
+
+      pikelet_class.class_variable_set('@@redis', build_redis_connection_pool)
+
+      Thin::Logging.silent = true
+
+      pikelet = Thin::Server.new('0.0.0.0', port, pikelet_class, :signals => false)
+      @pikelets << {:instance => pikelet, :type => pikelet_type}
+      pikelet.start
+      @logger.debug "new thin server instance started for #{pikelet_type}"
+    end
+
+    def build_resque_pikelet(pikelet_type, pikelet_cfg)
+      pikelet_class = case pikelet_type
+      when 'email_notifier'
+        Flapjack::Notification::Email
+      when 'sms_notifier'
+        Flapjack::Notification::Sms
+      end
+      return unless pikelet_class
+
+      # set up connection pooling, stop resque errors (ensure that it's only
+      # done once)
+      if (['email_notifier', 'sms_notifier'] & @pikelets.collect {|p| p[:type]}).empty?
+        ::Resque.redis = build_redis_connection_pool
+        ## NB: can override the default 'resque' namespace like this
+        #::Resque.redis.namespace = 'flapjack'
+      end
+
+      # See https://github.com/mikel/mail/blob/master/lib/mail/mail.rb#L53
+      # & https://github.com/mikel/mail/blob/master/spec/mail/configuration_spec.rb
+      # for details of configuring mail gem. defaults to SMTP, localhost, port 25
+
+      if pikelet_type.eql?('email_notifier')
+        smtp_config = {}
+
+        if pikelet_cfg['smtp_config']
+          smtp_config = pikelet_cfg['smtp_config'].keys.inject({}) do |ret,obj|
+            ret[obj.to_sym] = pikelet_cfg['smtp_config'][obj]
+            ret
+          end
+        end
+
+        Mail.defaults {
+          delivery_method :smtp, {:enable_starttls_auto => false}.merge(smtp_config)
+        }
+      end
+
+      pikelet_class.class_variable_set('@@config', pikelet_cfg)
+
+      f = Fiber.new {
+        begin
+          # TODO error if pikelet_cfg['queue'].nil?
+          pikelet = EM::Resque::Worker.new(pikelet_cfg['queue'])
+          @pikelets.detect {|p| p[:type] == pikelet_type}[:instance] = pikelet
+          # # Use these to debug the resque workers
+          # flapjack_rsq.verbose = true
+          #flapjack_rsq.very_verbose = true
+          pikelet.work(0.1)
+        rescue Exception => e
+          trace = e.backtrace.join("\n")
+          @logger.fatal "#{e.message}\n#{trace}"
+          stop
+        end
+      }
+      @pikelets << {:fiber => f, :type => pikelet_type}
+      f.resume
+      @logger.debug "new fiber created for #{pikelet_type}"
+    end
+
+    def build_redis_connection_pool(options = {})
+      EventMachine::Synchrony::ConnectionPool.new(:size => options[:size] || 5) do
+        ::Redis.new(@redis_options.merge(:driver => (options[:driver] || 'synchrony')))
+      end
+    end
+
+    # # TODO rewrite to be less spammy -- print only initial state and changes
+    # def health_check
+    #   @pikelets.each do |pik|
+    #     if pik[:instance].is_a?(Thin::Server)
+    #       s = pik[:instance].backend.size
+    #       @logger.debug "thin on port #{pik[:instance].port} - #{s} connections"
+    #     elsif pik[:fiber]
+    #       @logger.debug "#{pik[:type]}: #{pik[:fiber].alive? ? 'alive' : 'dead'}"
+    #     end
+    #   end
+    # end
+
+    def shutdown
+      @pikelets.each do |pik|
+        case pik[:instance]
+        when Flapjack::Executive, Flapjack::Jabber, Flapjack::Pagerduty
+          if pik[:fiber] && pik[:fiber].alive?
+            pik[:instance].stop
+            Fiber.new {
+              # this needs to use a separate Redis connection from the pikelet's
+              # one, as that's in the middle of its blpop
+              r = Redis.new(@redis_options.merge(:driver => 'synchrony'))
+              pik[:instance].add_shutdown_event(:redis => r)
+              r.quit
+            }.resume
+          end
+        when EM::Resque::Worker
+          if pik[:fiber] && pik[:fiber].alive?
+            # resque is polling, so we don't need a shutdown object
+            pik[:instance].shutdown
+          end
+        when Thin::Server # web, api
+          # drop from this side, as HTTP keepalive etc. means browsers
+          # keep connections alive for ages, and we'd be hanging around
+          # waiting for them to drop
+          pik[:instance].stop!
+        end
+      end
+
+      fibers = @pikelets.collect {|p| p[:fiber] }.compact
+      thin_pikelets = @pikelets.collect {|p| p[:instance]}.select {|i| i.is_a?(Thin::Server) }
+
+      Fiber.new {
+        loop do
+          # health_check
+          if fibers.any?(&:alive?) || thin_pikelets.any?{|tp| !tp.backend.empty? }
+            EM::Synchrony.sleep 0.25
+          else
+            EM.stop
+            break
+          end
+        end
+      }.resume
+    end
+
+  end
+
+end

data/lib/flapjack/daemonizing.rb

@@ -0,0 +1,186 @@
+#!/usr/bin/env ruby
+
+# Copied from thin.
+
+require 'etc'
+require 'daemons' unless RUBY_PLATFORM =~ /mswin/
+
+module Process
+  # Returns +true+ the process identified by +pid+ is running.
+  def running?(pid)
+    Process.getpgid(pid) != -1
+  rescue Errno::EPERM
+    true
+  rescue Errno::ESRCH
+    false
+  end
+  module_function :running?
+end
+
+module Flapjack
+  # Raised when the pid file already exist starting as a daemon.
+  class PidFileExist < RuntimeError; end
+  
+  # Module included in classes that can be turned into a daemon.
+  # Handle stuff like:
+  # * storing the PID in a file
+  # * redirecting output to the log file
+  # * changing process privileges
+  # * killing the process gracefully
+  module Daemonizable
+    attr_accessor :pid_file, :log_file
+    
+    def self.included(base)
+      base.extend ClassMethods
+    end
+    
+    def pid
+      File.exist?(pid_file) ? open(pid_file).read.to_i : nil
+    end
+    
+    # Turns the current script into a daemon process that detaches from the console.
+    def daemonize
+      raise PlatformNotSupported, 'Daemonizing is not supported on Windows'  if RUBY_PLATFORM =~ /mswin/
+      raise ArgumentError,        'You must specify a pid_file to daemonize' unless @pid_file
+      
+      remove_stale_pid_file
+      
+      pwd = Dir.pwd # Current directory is changed during daemonization, so store it
+      
+      # HACK we need to create the directory before daemonization to prevent a bug under 1.9
+      #      ignoring all signals when the directory is created after daemonization.
+      FileUtils.mkdir_p File.dirname(@pid_file)
+      FileUtils.mkdir_p File.dirname(@log_file)
+      
+      Daemonize.daemonize(File.expand_path(@log_file), 'flapjack server')
+      
+      Dir.chdir(pwd)
+      
+      write_pid_file
+
+      self.after_daemonize if self.respond_to? :after_daemonize
+            
+      at_exit do
+        puts ">> Exiting!"
+        remove_pid_file
+      end
+    end
+
+    # Change privileges of the process
+    # to the specified user and group.
+    def change_privilege(user, group=user)
+      puts ">> Changing process privilege to #{user}:#{group}"
+      
+      uid, gid = Process.euid, Process.egid
+      target_uid = Etc.getpwnam(user).uid
+      target_gid = Etc.getgrnam(group).gid
+
+      if uid != target_uid || gid != target_gid
+        # Change process ownership
+        Process.initgroups(user, target_gid)
+        Process::GID.change_privilege(target_gid)
+        Process::UID.change_privilege(target_uid)
+      end
+    rescue Errno::EPERM => e
+      puts "Couldn't change user and group to #{user}:#{group}: #{e}"
+    end
+    
+    # Register a proc to be called to restart the server.
+    def on_restart(&block)
+      @on_restart = block
+    end
+    
+    # Restart the server.
+    def restart
+      if @on_restart
+        puts '>> Restarting ...'
+        stop
+        remove_pid_file
+        @on_restart.call
+        exit!
+      end
+    end
+    
+    module ClassMethods
+      # Send a QUIT or INT (if timeout is +0+) signal the process which
+      # PID is stored in +pid_file+.
+      # If the process is still running after +timeout+, KILL signal is
+      # sent.
+      def kill(pid_file, timeout=60)
+        if timeout == 0
+          send_signal('INT', pid_file, timeout)
+        else
+          send_signal('QUIT', pid_file, timeout)
+        end
+      end
+      
+      # Restart the server by sending HUP signal.
+      def restart(pid_file)
+        send_signal('HUP', pid_file)
+      end
+      
+      # Send a +signal+ to the process which PID is stored in +pid_file+.
+      def send_signal(signal, pid_file, timeout=60)
+        if pid = read_pid_file(pid_file)
+          puts "Sending #{signal} signal to process #{pid} ... "
+          Process.kill(signal, pid)
+          Timeout.timeout(timeout) do
+            sleep 0.1 while Process.running?(pid)
+          end
+        else
+          puts "Can't stop process, no PID found in #{pid_file}"
+        end
+      rescue Timeout::Error
+        puts "Timeout!"
+        force_kill pid_file
+      rescue Interrupt
+        force_kill pid_file
+      rescue Errno::ESRCH # No such process
+        puts "process not found!"
+        force_kill pid_file
+      end
+      
+      def force_kill(pid_file)
+        if pid = read_pid_file(pid_file)
+          puts "Sending KILL signal to process #{pid} ... "
+          Process.kill("KILL", pid)
+          File.delete(pid_file) if File.exist?(pid_file)
+        else
+          puts "Can't stop process, no PID found in #{pid_file}"
+        end
+      end
+      
+      def read_pid_file(file)
+        if File.file?(file) && pid = File.read(file)
+          pid.to_i
+        else
+          nil
+        end
+      end
+    end
+    
+    protected
+      def remove_pid_file
+        File.delete(@pid_file) if @pid_file && File.exists?(@pid_file)
+      end
+    
+      def write_pid_file
+        puts ">> Writing PID to #{@pid_file}"
+        open(@pid_file,"w") { |f| f.write(Process.pid) }
+        File.chmod(0644, @pid_file)
+      end
+      
+      # If PID file is stale, remove it.
+      def remove_stale_pid_file
+        if File.exist?(@pid_file)
+          if pid && Process.running?(pid)
+            raise PidFileExist, "#{@pid_file} already exists, seems like it's already running (process ID: #{pid}). " +
+                                "Stop the process or delete #{@pid_file}."
+          else
+            puts ">> Deleting stale PID file #{@pid_file}"
+            remove_pid_file
+          end
+        end
+      end
+  end
+end