deadpool 0.1.1

data/lib/deadpool/monitor/base.rb

@@ -0,0 +1,58 @@
+module Deadpool
+  module Monitor
+    class Base
+      attr_accessor :logger
+
+      def initialize(config, monitor_config, logger)
+        @config, @monitor_config, @logger = config, monitor_config, logger
+      end
+
+      def primary_ok?
+        raise NotImplementedError, "primary_ok? is not implemented"
+      end
+
+      def secondary_ok?
+        raise NotImplementedError, "secondary_ok? is not implemented"
+      end
+
+      def system_check
+        update_state
+        take_snapshot
+      end
+
+      def state
+        @state ||= Deadpool::State.new name, self.class
+      end
+
+      protected
+
+      def update_state
+        primary_okay, secondary_okay = primary_ok?, secondary_ok?
+        logger.debug "PrimaryOkay? #{primary_okay}, SecondaryOkay? #{secondary_okay}"
+
+        args = case [ primary_okay, secondary_okay ]
+        when [true, true]
+          [ OK, "Primary and Secondary are up." ]
+        when [false, true]
+          [ WARNING, "Primary is down. Secondary is up." ]
+        when [true, false]
+          [ WARNING, "Primary is up. Secondary is down." ]
+        when [false, false]
+          [ CRITICAL, "Primary and Secondary are down." ]
+        else
+          [ CRITICAL, "Implementation Error." ]
+        end
+
+        self.state.set_state(*args)
+      end
+
+      def take_snapshot
+        StateSnapshot.new(self.state)
+      end
+
+      def name
+        @monitor_config[:name].nil? ? '' : @monitor_config[:name]
+      end
+    end
+  end
+end

data/lib/deadpool/monitor/generic_nagios.rb

@@ -0,0 +1,47 @@
+# Assumes at a minimum the following config:
+# --
+# primary_host:   '127.0.0.1'
+# secondary_host: '127.0.0.1'
+# monitor_config:
+#   nagios_plugin_path: '/usr/lib/nagios/plugins/check_something'
+
+module Deadpool
+
+  module Monitor
+
+    class GenericNagios < Base
+
+      def primary_ok?
+        check_host(@config[:primary_host])
+      end
+
+      def secondary_ok?
+        check_host(@config[:secondary_host])
+      end
+
+      protected
+
+      def check_host(host)
+        check_command  = "#{nagios_plugin_path} -H #{host} #{nagios_options}"
+        logger.debug check_command
+        status_message = `#{check_command}`
+        exit_status    = $?
+        logger.debug "Generic Nagios Check Status Message: #{status_message}"
+        logger.debug "Generic Nagios Check Exit Status: #{exit_status}"
+
+        return exit_status == 0
+      end
+
+      def nagios_plugin_path
+        @config[:monitor_config][:nagios_plugin_path]
+      end
+
+      def nagios_options
+        @config[:monitor_config][:nagios_options]
+      end
+
+    end
+
+  end
+
+end

data/lib/deadpool/monitor/mysql.rb

@@ -0,0 +1,62 @@
+# Assumes at a minimum the following config:
+# --
+# primary_host:   '127.0.0.1'
+# secondary_host: '127.0.0.1'
+# monitor_config:
+#   nagios_plugin_path: '/usr/lib/nagios/plugins/check_mysql'
+
+module Deadpool
+
+  module Monitor
+
+    class Mysql < Base
+
+      def primary_ok?
+        return check_mysql(@config[:primary_host])
+      end
+
+      def secondary_ok?
+        return check_mysql(@config[:secondary_host]) 
+        # && check_mysql_slave(@config[:secondary_host])
+      end
+
+      protected
+
+      def check_mysql(host)
+        check_command      = "#{nagios_plugin_path} -H #{host} -u '#{username}' -p '#{password}'"
+        logger.debug check_command
+        status_message     = `#{check_command}`
+        exit_status        = $?
+        logger.debug "MySQL Check Status Message: #{status_message}"
+        logger.debug "MySQL Check Exit Status: #{exit_status}"
+
+        return exit_status == 0
+      end
+
+      def check_mysql_slave(host)
+        check_command  = "#{nagios_plugin_path} -H #{host} -u '#{username}' -p '#{password}' --check-slave"
+        logger.debug check_command
+        status_message = `#{check_command}`
+        exit_status    = $?
+        logger.debug "MySQL Check Status Message: #{status_message}"
+        logger.debug "MySQL Check Exit Status: #{exit_status}"
+
+        return exit_status == 0
+      end
+
+      def nagios_plugin_path
+        @config[:monitor_config][:nagios_plugin_path]
+      end
+
+      def username
+        @config[:monitor_config][:username]
+      end
+
+      def password
+        @config[:monitor_config][:password]
+      end
+    end
+
+  end
+
+end

data/lib/deadpool/monitor/redis.rb

@@ -0,0 +1,21 @@
+# Assumes at a minimum the following config:
+# --
+# primary_host:   '127.0.0.1'
+# secondary_host: '127.0.0.1'
+# monitor_config:
+#   redis_args: '[-h host] [-p port] [-a authpw] [-r repeat_times] [-n db_num]'
+
+module Deadpool
+  module Monitor
+    class Redis < Base
+
+      def primary_ok?
+        false
+      end
+
+      def secondary_ok?
+        false
+      end
+    end
+  end
+end

data/lib/deadpool/options.rb

@@ -0,0 +1,62 @@
+
+require 'optparse'
+require 'ostruct'
+require 'strscan'
+
+
+module Deadpool
+
+  module Options
+
+    def parse_options(argv)
+      @options = self.parse_command_line(argv)
+    end
+
+    def parse_command_line(argv)
+      options             = {}
+      options[:config_path] = '/etc/deadpool'
+      options[:daemonize] = nil
+
+      @option_parser = OptionParser.new do |opts|
+        opts.banner = "Usage: deadpool_hosts {help|full_report|nagios_report} [options]"
+
+        opts.separator "Commands:"
+        opts.on("-h", "--help", "Print this help message.") do |help|
+          options[:help] = true
+        end
+
+        opts.on("-d", "--daemon", "Background the server.") do |daemon|
+          options[:daemonize] = true
+        end
+
+        opts.separator "Options:"
+        opts.on("--config_path=PATH", String,
+            "Path to configs and custom plugins. #{options[:config_path]} by default.") do |config_path|
+          options[:config_path] = config_path
+        end
+      end
+
+      remaining_arguments = @option_parser.parse! argv
+
+      unless remaining_arguments.empty?
+        help "[#{remaining_arguments.join(' ')}] is not understood."
+      end
+
+      return options
+    end
+
+    def help(message=nil)
+      unless message.nil?
+        puts message
+      end
+      puts @option_parser.help
+      exit 4
+    end
+
+    def config_path
+      return @options[:config_path]
+    end
+
+  end
+
+end

data/lib/deadpool/server.rb

@@ -0,0 +1,101 @@
+
+
+module Deadpool
+
+  class Server
+
+    # include Deadpool::Options
+    include Deadpool::Daemonizer
+
+    attr_accessor :logger
+    attr_reader :config
+
+    def initialize(options)
+      @options = options
+      @state   = Deadpool::State.new self.class.to_s
+      @config  = Deadpool::Helper.configure(@options)
+      @logger  = Deadpool::Helper.setup_logger(@config)
+
+      # if @options[:daemonize]
+      #   options = @options[:pid_file].nil? ? {} : {:pid => @options[:pid_file]}
+      #   self.daemonize options
+      # end
+      # 
+      # load_handlers
+      # start_deadpool_handlers
+      # start_command_server
+      # schedule_system_check
+    end
+
+    def run(daemonize)
+      EventMachine::run {
+        if daemonize
+          options = @options[:pid_file].nil? ? {} : {:pid => @options[:pid_file]}
+          self.daemonize options
+        end
+      
+        load_handlers
+        start_deadpool_handlers
+        start_command_server
+        schedule_system_check
+      }
+    end
+
+    def load_handlers
+      @handlers = {}
+      @state.set_state(OK, 'Loading Handlers.')
+
+      Dir[@options[:config_path] + '/config/pools/*.yml'].each do |pool_yml|
+        pool_config = Deadpool::Helper.symbolize_keys YAML.load(File.read(pool_yml))
+        @handlers[pool_config[:pool_name]] = Deadpool::Handler.new(pool_config, logger)
+      end
+
+      @state.set_state(OK, 'Handlers loaded.')
+    end
+
+    def start_deadpool_handlers
+      @handlers.each_value do |handler|
+        timer = EventMachine::PeriodicTimer.new(handler.check_interval) do
+          handler.monitor_pool(timer)
+        end
+      end
+    end
+
+    def schedule_system_check
+      timer = EventMachine::PeriodicTimer.new(@config[:system_check_interval]) do
+        system_check(true)
+      end
+    end
+
+    def start_command_server
+      EventMachine::start_server @config[:admin_hostname], @config[:admin_port], Deadpool::AdminServer do |connection|
+        connection.deadpool_server = self
+      end
+    end
+
+    def system_check(force=false)
+      if force || @cached_state_snapshot.nil?
+        @state.reset!
+        @cached_state_snapshot = Deadpool::StateSnapshot.new @state
+
+        @handlers.each_value do |handler|
+          @cached_state_snapshot.add_child handler.system_check
+        end
+      end
+
+      return @cached_state_snapshot
+    end
+
+    def promote_server(pool_name, server)
+      unless @handlers[pool_name].nil?
+        # logger.debug "Pool Name: #{pool_name}, Server: #{server}"
+        return @handlers[pool_name].promote_server server
+      else
+        logger.error "'#{pool_name}' pool not found."
+        return false
+      end
+    end
+
+  end
+
+end

data/lib/deadpool/state.rb

@@ -0,0 +1,78 @@
+
+
+
+module Deadpool
+
+  class State
+    
+    attr_reader :name, :timestamp, :status_code, :error_messages, :all_messages
+    
+    def initialize(name, klass='')
+      names = []
+      names << name unless (name.nil? and name.blank?)
+      names << klass.to_s unless (klass.nil? and klass.blank?)
+      @name = names.join " - "
+      @locked = false
+      reset!
+    end
+
+    def lock
+      @locked = true
+    end
+
+    def unlock
+      @locked = false
+    end
+
+    def set_state(code, message)
+      unless @locked
+        if code == OK
+          @timestamp      = Time.now
+          @status_code    = OK
+          @error_messages = []
+          @all_messages   = [message]
+        else
+          @timestamp      = Time.now
+          @status_code    = code
+          @error_messages = [message]
+          @all_messages   = []
+        end
+      end
+    end
+    
+    def reset!(message=nil)
+      unless @locked
+        @timestamp      = Time.now
+        @status_code    = OK
+        @error_messages = []
+        @all_messages   = message.nil? ? [] : [message]
+      end
+    end
+    
+    def escalate_status_code(code)
+      unless @locked
+        @timestamp = Time.now
+      
+        if code >= @status_code
+          @status_code = code
+        end
+      end
+    end
+
+    def add_message(message)
+      unless @locked
+        @timestamp = Time.now
+        @all_messages << message
+      end
+    end
+
+    def add_error_message(message)
+      unless @locked
+        @timestamp = Time.now
+        @error_messages << message
+        # @all_messages << message
+      end
+    end
+  end
+
+end

data/lib/deadpool/state_snapshot.rb

@@ -0,0 +1,81 @@
+
+
+
+module Deadpool
+  
+  class StateSnapshot
+    
+    def initialize(state)
+      @name           = state.name
+      @timestamp      = state.timestamp
+      @status_code    = state.status_code
+      @all_messages   = state.all_messages
+      @error_messages = state.error_messages
+      @children       = []
+    end
+
+    def add_child(child)
+      @children << child
+    end
+
+    def overall_status
+      @children.map { |child| child.overall_status }.push(@status_code).max
+    end
+
+    def all_error_messages
+      @children.inject(@error_messages) do |arr, child|
+        arr + child.all_error_messages
+      end
+    end
+
+    def nagios_report
+      message = ''
+      if overall_status != OK
+        message += all_error_messages.join(' | ')
+      end
+
+      message += " last checked #{(Time.now - @timestamp).round} seconds ago."
+
+      "#{status_code_to_s(overall_status)} - #{message}\n"
+    end
+
+    def full_report
+      output = "System Status: #{status_code_to_s(overall_status)}\n\n"
+      output += self.to_s
+
+      return output
+    end
+
+    def to_s(indent=0)
+      indent_space = '  ' * indent
+
+      output = "#{indent_space}#{@name}\n"
+      output += "#{indent_space}#{status_code_to_s(@status_code)} - checked #{(Time.now - @timestamp).round} seconds ago.\n"
+      unless @error_messages.empty?
+        output += "#{indent_space}!!! #{@error_messages.join("\n#{indent_space}!!! ")}\n"
+      end
+      unless @all_messages.empty?
+        output += "#{indent_space}#{@all_messages.join("\n#{indent_space}")}\n"
+      end
+      output += "\n"
+      
+      @children.each do |child|
+        output += child.to_s(indent+1)
+      end
+
+      return output
+    end
+
+    def status_code_to_s(code)
+      case code
+      when OK       then 'OK'
+      when WARNING  then 'WARNING'
+      when CRITICAL then 'CRITICAL'
+      else
+        'UNKNOWN'
+      end
+    end
+
+  end
+
+end