deadpool 0.1.1

data/lib/deadpool/failover_protocol/exec_remote_command.rb

@@ -0,0 +1,106 @@
+require 'net/ssh'
+
+module Deadpool
+
+  module FailoverProtocol
+
+    class ExecRemoteCommand < Base
+
+      def setup
+        @test_command = @failover_config[:test_command]
+        @exec_command = @failover_config[:exec_command]
+        @client_hosts = @failover_config[:client_hosts]
+        @username     = @failover_config[:username]
+        @password     = @failover_config[:password]
+        @use_sudo     = @failover_config[:use_sudo]
+        @sudo_path    = @failover_config[:sudo_path].nil? ? 'sudo' : @failover_config[:sudo_path]
+      end
+
+      # Return true or false
+      # Don't update system state.
+      # return true or false success or failure
+      def preflight_check
+        @client_hosts.all? { |client_host| test_client(client_host) }
+      end
+
+      def test_client(client_host)
+        logger.debug "Testing Client #{client_host}"
+        return exec_remote_command(@test_command, client_host)
+      end
+
+      # Promote the host to primary.  This is used by initiate_failover_protocol!
+      # and for manual promotion by an administrator.
+      # new_primary is an IP address
+      # TODO: change new_primary to be a config label.
+      # return true or false success or failure
+      def promote_to_primary(new_primary)
+        success = true
+
+        @client_hosts.each do |client_host|
+          if exec_remote_command(@exec_command, client_host)
+            exec_remote_command(@exec_command, client_host)
+            logger.info "Promotion exec command succeeded on #{client_host}"
+          else
+            logger.error "Promotion exec command failed on #{client_host}"
+          end
+        end
+
+        return success
+      end
+
+      # Perform checks against anything that could cause a failover protocol to fail
+      # Perform checks on system state.
+      # return New Deadpool::StateSnapshot
+      def system_check
+        failed    = []
+        succeeded = []
+
+        # Collect check data
+        @client_hosts.each do |client_host|
+          if test_client(client_host)
+            succeeded << client_host
+          else
+            failed << client_host
+          end
+        end
+
+        # Compile write check data.
+        if !succeeded.empty? && failed.empty?
+          @state.set_state OK, "Exec test passed all servers: #{succeeded.join(', ')}"
+        elsif !succeeded.empty? && !failed.empty?
+          @state.set_state WARNING, "Exec test passed on: #{succeeded.join(', ')}"
+          @state.add_error_message "Exec test failed on #{failed.join(', ')}"
+        elsif succeeded.empty?
+          @state.set_state WARNING, "Exec test failed all servers: #{failed.join(', ')}"
+        end
+
+        return Deadpool::StateSnapshot.new @state
+      end
+
+      protected
+
+      def exec_remote_command(command, host)
+        options = @password.nil? ? {} : {:password => @password}
+        command = "#{command} ; echo 'ExecRemoteCommand.success: '$?"
+        command = "#{@sudo_path} #{command}" if @use_sudo
+
+        logger.debug "executing #{command} on #{host}"
+
+        begin
+          output = 0
+          Net::SSH.start(host, @username, options) do |ssh|
+            output = ssh.exec!(command)
+          end
+
+          return (output =~ /ExecRemoteCommand.success: 0/)
+        rescue
+          logger.error "Couldn't execute #{command} on #{host} with Username: #{@username} and options: #{options.inspect}"
+          return false
+        end
+      end
+
+    end
+
+  end
+
+end

data/lib/deadpool/generator.rb

@@ -0,0 +1,220 @@
+require 'optparse'
+require 'ostruct'
+require 'strscan'
+require 'fileutils'
+
+module Deadpool
+  class Generator
+
+    # include FileUtils
+
+    def initialize(argv)
+      @argv     = argv
+    end
+
+    def run
+      @options = self.parse_command_line
+      @config  = Deadpool::Helper.configure @options
+
+      self.execute_command(@options)
+    end
+
+    def parse_command_line
+      options                       = Hash.new
+      options[:command_count]       = 0
+      options[:config_path]         = '/etc/deadpool'
+      options[:upstart_config_path] = '/etc/init/deadpool.conf'
+      options[:upstart_init_path]   = '/etc/init.d/deadpool'
+      options[:upstart_script_path] = '/lib/init/upstart-job'
+
+      @option_parser = OptionParser.new do |opts|
+        opts.banner = "Usage: deadpool_generator command [options]"
+
+        opts.separator "Commands:"
+        opts.on("-h", "--help", "Print this help message.") do |help|
+          options[:command_count] += 1
+          options[:command]        = :help
+        end
+        opts.on("--upstart_init", "Generate and upstart config.") do |upstart|
+          options[:command_count] += 1
+          options[:command]        = :upstart_init
+        end
+        opts.on("--configuration", "Generate a config directory structure and example files.") do |configuration|
+          options[:command_count] += 1
+          options[:command]        = :configuration
+        end
+      
+        opts.separator "Configuration Options:"
+        opts.on("--config_path=PATH", String, "path to create the config dir at (#{options[:config_path]})") do |config_path|
+          options[:config_path] = config_path
+        end
+
+        opts.separator "Upstart Options:"
+        opts.on("--upstart_config_path=PATH", String, "path to create the config dir at (#{options[:upstart_config_path]})") do |upstart_config_path|
+          options[:upstart_config_path] = upstart_config_path
+        end
+        opts.on("--upstart_init_path=PATH", String, "path to create the config dir at (#{options[:upstart_init_path]})") do |upstart_init_path|
+          options[:upstart_init_path] = upstart_init_path
+        end
+        opts.on("--upstart_script_path=PATH", String, "path to create the config dir at (#{options[:upstart_script_path]})") do |upstart_script_path|
+          options[:upstart_script_path] = upstart_script_path
+          unless File.exists? upstart_script_path
+            help "The upstart script is not at #{upstart_script_path}."
+          end
+        end
+      end
+
+        remaining_arguments = @option_parser.parse! @argv
+
+        unless remaining_arguments.empty?
+          help "[#{remaining_arguments.join(' ')}] is not understood."
+        end
+
+        if options[:command_count] == 0
+          help "You must specify a command."
+        end
+
+        return options
+      end
+
+      def execute_command(options)
+        case options[:command]
+        when :upstart_init
+          generate_upstart_init options
+        when :configuration
+          generate_configuration options
+        else
+          help
+        end
+      end
+
+    def help(message=nil)
+      unless message.nil?
+        puts message
+      end
+      puts @option_parser.help
+      exit 4
+    end
+
+    def generate_upstart_init(options)
+      config_path         = options[:config_path]
+      upstart_config_path = options[:upstart_config_path]
+      upstart_init_path   = options[:upstart_init_path]
+      upstart_script_path = options[:upstart_script_path]
+      config_params       = config_path.nil? ? '' : "--config_path=#{config_path}"
+      ruby_path           = `which ruby`.strip
+      deadpool_admin_path = `which deadpool_admin`.strip
+      
+      upstart_conf =<<-EOF
+description     "Deadpool Service"
+author          "Kirt Fitzpatrick <kirt.fitzpatrick@akqa.com>"
+
+umask 007
+start on (net-device-up and local-filesystems)
+stop on runlevel [016]
+respawn
+exec #{ruby_path} #{deadpool_admin_path} --foreground #{config_params}
+
+      EOF
+
+      if upstart_config_path
+        File.open File.join(upstart_config_path), 'w' do |file|
+          file.write upstart_conf
+        end
+        
+        if upstart_init_path 
+          if File.exists? upstart_init_path
+            puts "#{upstart_config_path} has been (over)written."
+            puts "#{upstart_init_path} already exists.  It should be a symbolic link that points to #{upstart_script_path}"
+            ls_command = "ls -l #{upstart_init_path}"
+            # puts ls_command
+            # puts `#{ls_command}`
+            # puts "ln -s #{upstart_script_path} #{upstart_init_path}"
+          else
+            `ln -s #{upstart_script_path} #{upstart_init_path}`
+          end
+        end
+      else
+        puts upstart_conf
+      end
+    end
+
+    # mkdir path/config/pools
+    #       path/config/environment.yml
+    #       path/config/pools/example.yml
+    # mkdir path/lib/deadpool/monitor
+    #       path/lib/deadpool/monitor
+    # mkdir path/lib/deadpool/failover_protocol
+    #       path/lib/deadpool/failover_protocol
+    def generate_configuration(options)
+      path = config_path
+      FileUtils.mkdir_p(File.join(path, 'config/pools'))
+      FileUtils.mkdir_p(File.join(path, 'lib/deadpool/monitor'))
+      FileUtils.mkdir_p(File.join(path, 'lib/deadpool/failover_protocol'))
+      File.open File.join(path, 'config/pools/example.yml'), 'w' do |file|
+        file.write <<-EOF
+pool_name:         'example_database'
+primary_host:      '10.1.2.3'
+secondary_host:    '10.2.3.4'
+check_interval:    1
+max_failed_checks: 10
+
+# There can be only one monitor per pool at this time.  The deadpool system
+# defines no rules for the monitor configuration except that it is called
+# monitor_config: and has monitor_class: defined at the base level.  
+# All other configuration variables are plugin specific.
+monitor_config:
+  monitor_class: Mysql
+  nagios_plugin_path: '/usr/lib/nagios/plugins'
+
+# There can be as many Failover Protocols as you want and you can use 
+# the same plugin multiple times.  The deadpool defines no riles for the 
+# failover protocol config except that it be an array element of 
+# failover_protocol_configs and defines protocol_class at it's base.  The rest
+# of the configuration is specific to the failover protocol.
+failover_protocol_configs:
+  - protocol_class: EtcHosts
+    script_path: '/usr/local/bin/deadpool_line_modifier'
+    service_host_name: 'master.mysql.example.project.client'
+    username: 'deadpool'
+    password: 'p4ssw0rd'
+    use_sudo: 1
+    client_hosts:
+      - '10.3.4.5'   # app server 1 (web server)
+      - '10.4.5.6'   # app server 2 (web server)
+
+  - protocol_class: ExecRemoteCommand
+    test_command: '/etc/init.d/nginx status'
+    exec_command: '/etc/init.d/nginx restart'
+    username: 'deadpool'
+    password: 'p4ssw0rd'
+    use_sudo: 1
+    client_hosts:
+      - '10.3.4.5'   # app server 1 (web server)
+      - '10.4.5.6'   # app server 2 (web server)
+        EOF
+      end
+      
+      environment_config_path = File.join(path, 'config/environment.yml')
+      environment_conf = <<-EOF
+# log_path: '/var/log/deadpool.log'
+# log_level: INFO
+# system_check_interval: 30
+# admin_hostname: 'localhost'
+# admin_port: 5507
+
+      EOF
+      if File.exists? enironment_config_path
+        puts "#{environment_config_path} already exists.  Here's what we would have copied there."
+        puts environment_conf
+      else
+        File.open environment_config_path, 'w' do |file|
+          file.write environment_conf
+        end
+      end
+      
+    end
+
+  end
+
+end

data/lib/deadpool/handler.rb

@@ -0,0 +1,100 @@
+
+
+module Deadpool
+
+  class Handler
+
+    attr_accessor :logger
+    attr_reader :state,
+        :failure_count, 
+        :check_interval, 
+        :max_failed_checks, 
+        :pool_name, 
+        :primary_host, 
+        :secondary_host
+
+    def initialize(config, logger)
+      @state             = Deadpool::State.new config[:pool_name], self.class
+      # @state             = Deadpool::State.new "Deadpool::Handler - #{config[:pool_name]}"
+      @config            = config
+      @logger            = logger
+      @pool_name         = config[:pool_name]
+      @check_interval    = config[:check_interval]
+      @max_failed_checks = config[:max_failed_checks]
+      @primary_host      = config[:primary_host]
+      @secondary_host    = config[:secondary_host]
+      @failure_count     = 0
+      instantiate_monitor
+      instantiate_failover_protocols
+      @state.set_state(OK, "Handler initialized.")
+    end
+
+    def monitor_pool(timer)
+      if @monitor.primary_ok?
+        @failure_count = 0
+        @state.set_state(OK, "Primary Check OK.")
+        logger.info "#{@pool_name} Primary Check Okay.  Failure Count set to 0."
+      else
+        @failure_count += 1
+        @state.set_state(WARNING, "Primary Check failed #{@failure_count} times")
+        logger.warn "#{@pool_name} Primary Check Failed.  Failure Count at #{@failure_count}"
+      end
+
+      if @failure_count >= @max_failed_checks
+        timer.cancel
+        @state.set_state(WARNING, "Failure threshold exceeded.  Failover Protocol Initiated.")
+        logger.error "#{@pool_name} primary is dead.  Initiating Failover Protocol."
+
+        success = true
+        @failover_protocols.each do |failover_protocol|
+          success = success && failover_protocol.initiate_failover_protocol!
+        end
+
+        if success
+          logger.warn "Failover Protocol Finished."
+          @state.set_state(WARNING, "Failover Protocol in place.")
+          @state.lock
+        else
+          logger.error "Failover Protocol Failed!"
+          @state.set_state(CRITICAL, "Failover Protocol Failed!")
+          @state.lock
+        end
+      end
+    end
+
+    def system_check
+      snapshot = Deadpool::StateSnapshot.new @state
+      snapshot.add_child @monitor.system_check
+      @failover_protocols.each do |failover_protocol|
+        # logger.debug failover_protocol.inspect
+        snapshot.add_child failover_protocol.system_check
+      end
+
+      return snapshot
+    end
+
+    def promote_server(server)
+      # This will stop at the first failure
+      @config[server] && @failover_protocols.all? do |failover_protocol|
+        failover_protocol.promote_to_primary @config[server]
+      end
+    end
+
+    protected
+
+    def instantiate_monitor
+      monitor_class = Deadpool::Monitor.const_get(@config[:monitor_config][:monitor_class])
+      @monitor      = monitor_class.new(@config, @config[:monitor_config], logger)
+    end
+
+    def instantiate_failover_protocols
+      @failover_protocols = []
+      @config[:failover_protocol_configs].each do |failover_config|
+        failover_protocol_class = Deadpool::FailoverProtocol.const_get(failover_config[:protocol_class])
+        @failover_protocols << failover_protocol_class.new(@config, failover_config, logger)
+      end
+    end
+
+  end
+
+end

data/lib/deadpool/helper.rb

@@ -0,0 +1,39 @@
+
+module Deadpool
+
+  class Helper
+
+    def self.symbolize_keys(arg)
+      case arg
+      when Array
+        arg.map { |elem| symbolize_keys elem }
+      when Hash
+        Hash[
+          arg.map { |key, value|  
+            k = key.is_a?(String) ? key.to_sym : key
+            v = symbolize_keys value
+            [k,v]
+          }]
+      else
+        arg
+      end
+    end
+
+    def self.configure(options)
+      default_config = YAML.load(File.read(File.join(File.dirname(__FILE__), '../../config/default_environment.yml')))
+      user_config    = YAML.load(File.read(File.join(options[:config_path], 'config/environment.yml')))
+      config         = Deadpool::Helper.symbolize_keys default_config.merge(user_config).merge(options)
+
+      return config
+    end
+
+    def self.setup_logger(config)
+      logger       = Logger.new(config[:log_path])
+      logger.level = Logger.const_get(config[:log_level].upcase)
+      
+      return logger
+    end
+
+  end
+
+end