deadpool 0.1.1

data/lib/deadpool/admin_server.rb

@@ -0,0 +1,65 @@
+
+require 'json'
+
+module Deadpool
+
+  module AdminServer
+
+    attr_accessor :deadpool_server
+
+    # data should be a JSON encoded hash.  It must have a command key.
+    # {
+    #   'command' => 'command', 
+    #   'pool'    => 'pool_name', 
+    #   'server'  => 'server_label'
+    # }
+    def receive_data(data)
+      if data.to_s =~ /command/        
+        deadpool_server.logger.debug "Received instruction: #{data}"
+        options = JSON.parse(data.to_s)
+
+        case options['command']
+        when 'full_report'
+          send_data full_report
+        when 'nagios_report'
+          send_data nagios_report
+        when 'promote_server'
+          send_data promote_server options
+        when 'stop'
+          send_data stop
+        else
+          send_data "Server did not understand the command."
+        end
+      end
+
+      close_connection_after_writing
+    end
+
+    protected
+    
+    def full_report
+      return deadpool_server.system_check(true).full_report
+    end
+
+    def nagios_report
+      return deadpool_server.system_check.nagios_report
+    end
+
+    def promote_server(options)
+      if deadpool_server.promote_server(options['pool'], options['server'].to_sym)
+        return "Success.\n"
+      else
+        return "Failed!\n"
+      end
+    end
+
+    def stop
+      close_connection
+      deadpool_server.kill
+    end
+
+  end
+
+end
+
+

data/lib/deadpool/command_line_server.rb

@@ -0,0 +1,169 @@
+
+require 'optparse'
+require 'ostruct'
+require 'strscan'
+require 'socket'
+require 'json'
+
+
+module Deadpool
+
+  class CommandLineServer
+
+    def initialize(argv)
+      @argv     = argv
+    end
+
+    def run
+      @options = self.parse_command_line
+      @config  = Deadpool::Helper.configure @options
+
+      self.execute_command(@options)
+    end
+
+    def parse_command_line
+      options               = Hash.new
+      options[:command_count] = 0
+      options[:config_path]     = '/etc/deadpool'
+
+      @option_parser = OptionParser.new do |opts|
+        opts.banner = "Usage: deadpool_hosts {help|full_report|nagios_report} [options]"
+
+        opts.separator "Commands:"
+        opts.on("-h", "--help", "Print this help message.") do |help|
+          options[:command_count] += 1
+          options[:command]        = :help
+        end
+        opts.on("--full_report", "Give the full system report.") do |full_report|
+          options[:command_count] += 1
+          options[:command]        = :full_report
+        end
+        opts.on("--nagios_report", "Report system state in Nagios plugin format.") do |nagios_report|
+          options[:command_count] += 1
+          options[:command]        = :nagios_report
+        end
+        opts.on("--promote_server", "Promote specified server to the master.") do |nagios_report|
+          options[:command_count] += 1
+          options[:command]        = :promote_server
+        end
+        opts.on("--stop", "Stop the server.") do |stop|
+          options[:command_count] += 1
+          options[:command]        = :stop
+        end
+
+        opts.separator "Options:"
+        opts.on("--server=SERVER_LABEL", String, "primary_host or secondary_host.") do |server|
+          options[:server] = server
+        end
+        opts.on("--pool=POOL_NAME", String, "Deadpool name to operate on.") do |pool|
+          options[:pool] = pool
+        end
+        opts.on("--config_path=PATH", String, 
+            "Path to configs and custom plugins. #{options[:config_path]} by default.") do |config_path|
+          options[:config_path] = config_path
+        end
+      end
+
+      remaining_arguments = @option_parser.parse! @argv
+
+      unless remaining_arguments.empty?
+        help "[#{remaining_arguments.join(' ')}] is not understood."
+      end
+
+      if options[:command_count] == 0
+        help "You must specify a command."
+      end
+
+      return options
+    end
+
+    def execute_command(options)
+      case options[:command]
+      when :help
+        help
+      when :full_report
+        full_report options
+      when :nagios_report
+        nagios_report options
+      when :promote_server
+        promote_server options
+      when :stop
+        stop options
+      else
+        help
+      end
+    end
+
+    def help(message=nil)
+      unless message.nil?
+        puts message
+      end
+      puts @option_parser.help
+      exit 4
+    end
+
+    def full_report(options)
+      puts send_command_to_deadpool_server :command => 'full_report'
+    end
+
+    def nagios_report(options)
+      response = send_command_to_deadpool_server :command => 'nagios_report'
+      puts response
+
+      if (response.to_s =~ /^OK/) != nil
+        exit 0
+      elsif (response.to_s =~ /^WARNING/) != nil
+        exit 1
+      elsif (response.to_s =~ /^CRITICAL/) != nil
+        exit 2
+      elsif (response.to_s =~ /^UNKNOWN/) != nil
+        exit 3
+      else
+        exit 4
+      end
+    end
+
+    def promote_server(options)
+      error_messages = []
+
+      if options[:pool].nil?
+        error_messages << "Promoting server requires --pool argument."
+      end
+
+      if options[:server].nil?
+        error_messages << "Promoting server requires --server argument."
+      end
+      
+      unless error_messages.empty?
+        help error_messages.join "\n"
+      end
+
+      puts send_command_to_deadpool_server :command => 'promote_server', :pool => options[:pool], :server => options[:server]
+    end
+
+    def stop(options)
+      puts send_command_to_deadpool_server :command => 'stop'
+    end
+
+    def send_command_to_deadpool_server(options)
+      output = ''
+      socket = TCPSocket.open(@config[:admin_hostname], @config[:admin_port])
+
+      if socket
+        socket.puts JSON.dump(options)
+        while line = socket.gets
+          output += line
+        end
+        socket.close
+      else
+        puts "Couldn't connect to deadpool server."
+        exit 4
+      end
+
+      return output
+    end
+
+  end
+
+end
+

data/lib/deadpool/daemonizer.rb

@@ -0,0 +1,82 @@
+
+
+# Blatanly stolen from Ben Marini
+# https://gist.github.com/517442/9ed5c4bd66bd8afc6a379286d8be1d8fb8a33925
+
+
+module Deadpool
+
+  module Daemonizer
+
+    def daemonize(opts={})
+      opts = { :log => '/dev/null', :pid => "/var/run/#{File.basename($0)}.pid" }.merge(opts)
+
+      $stdout.sync = $stderr.sync = true
+      $stdin.reopen("/dev/null")
+
+      exit if fork
+
+      Process.setsid
+
+      exit if fork
+
+      Dir.chdir("/") if opts[:chdir]
+      File.umask(0000) if opts[:umask]
+
+      if File.exist?(opts[:pid])
+        begin
+          existing_pid = File.read(opts[:pid]).to_i
+          Process.kill(0, existing_pid) # See if proc exists
+          abort "error: existing process #{existing_pid} using this pidfile, exiting"
+        rescue Errno::ESRCH
+          puts "warning: removing stale pidfile with pid #{existing_pid}"
+        end
+      end
+
+      File.open(opts[:pid], 'w') { |f| f.write($$) }
+
+      at_exit do
+        ( File.read(opts[:pid]).to_i == $$ and File.unlink(opts[:pid]) ) rescue nil
+      end
+
+      puts "forked process is #{$$}"
+      puts "output redirected to #{opts[:log]}"
+
+      $stdout.reopen(opts[:log], 'a')
+      $stderr.reopen(opts[:log], 'a')
+      $stdout.sync = $stderr.sync = true
+    end
+
+    def kill(opts={})
+      begin
+        opts = { :log => "/dev/null", :pid => "/var/run/#{File.basename($0)}.pid" }.merge(opts)
+        pid = File.read(opts[:pid]).to_i
+        sec = 60 # Seconds to wait before force killing
+        Process.kill("TERM", pid)
+
+        begin
+          SystemTimer.timeout(sec) do
+            loop do
+              puts "waiting #{sec} seconds for #{pid} before sending KILL"
+              Process.kill(0, pid) # See if proc exists
+
+              sec -= 1
+              sleep 1
+            end
+          end
+        rescue Errno::ESRCH
+          puts "killed process #{pid}"
+        rescue Timeout::Error
+          Process.kill("KILL", pid)
+          puts "force killed process #{pid}"
+        end
+
+      rescue Errno::ENOENT
+        puts "warning: pidfile #{opts[:pid]} does not exist"
+      rescue Errno::ESRCH
+        puts "warning: process #{pid} does not exist"
+      end
+    end
+
+  end
+end

data/lib/deadpool/failover_protocol.rb

@@ -0,0 +1,91 @@
+
+
+
+module Deadpool
+
+  module FailoverProtocol
+
+    class Base
+
+      attr_accessor :logger
+      attr_reader :config
+
+      def initialize(config, failover_config, logger)
+        @name            = failover_config[:name].nil? ? '' : failover_config[:name]
+        @state           = Deadpool::State.new @name, self.class
+        @config          = config
+        @logger          = logger
+        @primary_host    = @config[:primary_host]
+        @secondary_host  = @config[:secondary_host]
+        @failover_config = failover_config
+        setup
+      end
+
+      # Implementation specific initialization should be placed here.
+      def setup
+      end
+
+      # Overwrite this if you need to.
+      # Update state to reflect that failover has been initiated.
+      # State must be updated to no less than WARNING.
+      # State must be CRITICAL if any step of the protocol fails.
+      # Lock the state at whatever stage the failover reached.
+      # return true or false on success or failure.
+      # 
+      def initiate_failover_protocol!
+        logger.info "Performing Preflight Check"
+        @state.set_state WARNING, "Failover Protocol Initiated."
+
+        if preflight_check
+          logger.info "Preflight Check Passed."
+          @state.add_message "Preflight Check Passed."
+        else
+          logger.error "Preflight Check Failed!  Aborting Failover Protocol."
+          @state.escalate_status_code CRITICAL
+          @state.add_error_message "Preflight Check Failed! Failover Protocol Aborted!"
+          @state.lock
+          return false
+        end
+
+        if promote_to_primary(@secondary_host)
+          logger.info "#{@secondary_host} successfully promoted to primary"
+          @state.add_message "Failover Protocol Successful."
+          @state.lock
+          return true
+        else
+          logger.info "#{@secondary_host} promotion failed."
+          @state.escalate_status_code CRITICAL
+          @state.add_error_message "Failover Protocol Failed!"
+          @state.lock
+          return false
+        end
+      end
+
+      # Return true or false
+      # Don't update system state.
+      # return true or false success or failure
+      def preflight_check
+        return false
+      end
+
+      # Promote the host to primary.  This is used by initiate_failover_protocol!
+      # and for manual promotion by an administrator.
+      # new_primary is an IP address
+      # TODO: change new_primary to be a config label.
+      # return true or false success or failure
+      def promote_to_primary(new_primary)
+        return false
+      end
+
+      # Perform checks against anything that could cause a failover protocol to fail
+      # Perform checks on system state.
+      # return New Deadpool::StateSnapshot
+      def system_check
+        return Deadpool::StateSnapshot.new @state
+      end
+
+    end
+
+  end
+
+end

data/lib/deadpool/failover_protocol/etc_hosts.rb

@@ -0,0 +1,157 @@
+
+require 'net/ssh'
+
+module Deadpool
+
+  module FailoverProtocol
+
+    class EtcHosts < Base
+
+      def setup
+        @script_path        = @failover_config[:script_path]
+        @service_host_name  = @failover_config[:service_host_name]
+        @service_hosts_file = @failover_config[:service_hosts_file]
+        @client_hosts       = @failover_config[:client_hosts]
+        @username           = @failover_config[:username]
+        @password           = @failover_config[:password]
+        @use_sudo           = @failover_config[:use_sudo]
+        @sudo_path          = @failover_config[:sudo_path].nil? ? 'sudo' : @failover_config[:sudo_path]
+      end
+
+      def preflight_check
+        @client_hosts.map { |h| test_client(h) && verify_client(h) }.all?
+      end
+
+      def test_client(client_host)
+        logger.debug "Testing Client #{client_host}"
+        output = run_script_command(client_host, '--test')
+        logger.debug "Output recieved From Client: #{output}"
+
+        if output.class == String
+          okay = (output =~ /^OK/) != nil
+          okay ? logger.info("#{client_host}: " + output.strip) : logger.error("#{client_host}: " + output.strip)
+        else
+          logger.error "Test Client had a critical failure on '#{client_host}'"
+        end
+
+        return okay
+      end
+
+      def verify_client(client_host, primary_host=nil)
+        logger.debug "Verifying Client #{client_host}"
+        primary_host      = primary_host.nil? ? @primary_host : primary_host
+        command_arguments = "--verify --host_name='#{@service_host_name}' --ip_address='#{primary_host}'"
+        command_arguments += " --host_file='#{@service_hosts_file}'" if @service_hosts_file
+        output            = run_script_command(client_host, command_arguments)
+        logger.debug "Output recieved From Client: #{output}"
+
+        if output.class == String
+          okay = (output =~ /^OK/) != nil
+          okay ? logger.info("#{client_host}: " + output.strip) : logger.error("#{client_host}: " + output.strip)
+        else
+          logger.error "Verify Client had a critical failure on '#{client_host}'"
+        end
+
+        return okay
+      end
+
+      def promote_to_primary(new_primary)
+        @client_hosts.map do |client_host|
+          # logger.debug "client_host: #{client_host}, New Primary: #{new_primary}"
+          promote_to_primary_on_client(client_host, new_primary)
+        end.all?
+      end
+
+      def promote_to_primary_on_client(client_host, new_primary)
+        # logger.debug "Assigning #{new_primary} as new primary on #{client_host}"
+        command_arguments = "--switch --host_name='#{@service_host_name}' --ip_address='#{new_primary}'"
+        command_arguments += " --host_file='#{@service_hosts_file}'" if @service_hosts_file
+        output            = run_script_command(client_host, command_arguments)
+        logger.debug "Output received From Client: #{output}"
+
+        if output.class == String
+          okay = (output =~ /^OK/) != nil
+          okay ? logger.info("#{client_host}: " + output.strip) : logger.error("#{client_host}: " + output.strip)
+        else
+          logger.error "Promote to Primary on Client had a critical failure on '#{client_host}'"
+        end
+
+        return okay
+      end
+  
+      def system_check
+        writable             = []
+        not_writable         = []
+        pointed_at_primary   = []
+        pointed_at_secondary = []
+        pointed_at_neither   = []
+    
+        # Collect check data
+        @client_hosts.each do |client_host|
+          if test_client(client_host)
+            writable << client_host
+          else
+            not_writable << client_host
+          end
+
+          if verify_client(client_host, @primary_host)
+            pointed_at_primary << client_host
+          else
+            if verify_client(client_host, @secondary_host)
+              pointed_at_secondary << client_host
+            else
+              pointed_at_neither << client_host
+            end
+          end
+        end
+    
+        # Compile write check data.
+        if !writable.empty? && not_writable.empty?
+          @state.set_state OK, "Write check passed all servers: #{writable.join(', ')}"
+        elsif !writable.empty? && !not_writable.empty?
+          @state.set_state WARNING, "Write check passed on: #{writable.join(', ')}"
+          @state.add_error_message "Write check failed on #{not_writable.join(', ')}"
+        elsif writable.empty?
+          @state.set_state WARNING, "Write check failed all servers: #{not_writable.join(', ')}"
+        end
+    
+
+        # Compile verification data
+        if !pointed_at_primary.empty? && pointed_at_secondary.empty? && pointed_at_neither.empty?
+          @state.add_message "All client hosts are pointed at the primary."
+        elsif pointed_at_primary.empty? && !pointed_at_secondary.empty? && pointed_at_neither.empty?
+          @state.escalate_status_code WARNING
+          @state.add_error_message "All client hosts are pointed at the secondary."
+        else
+          @state.escalate_status_code CRITICAL
+          @state.add_error_message "Client hosts are pointing in different directions."
+        end
+
+        return Deadpool::StateSnapshot.new @state
+      end
+
+
+      protected
+
+      def run_script_command(host, command_arguments)
+        options = @password.nil? ? {} : {:password => @password}
+        command = "#{@script_path} #{command_arguments}"
+        command = "#{@sudo_path} #{command}" if @use_sudo
+
+        logger.debug "executing #{command} on #{host}"
+
+        begin
+          Net::SSH.start(host, @username, options) do |ssh|
+            return ssh.exec!(command)
+          end
+        rescue
+          logger.error "Couldn't execute #{command} on #{host}"
+          return false
+        end
+      end
+
+    end
+    
+  end
+
+end