ey_stonith 0.1.4 → 0.1.5.pre

data/bin/ey-monitor

@@ -1,5 +1,2 @@
-#!/usr/bin/env ruby
-$LOAD_PATH.unshift(File.expand_path('../../lib/', __FILE__))
-require 'ey_stonith'
-
-EY::Stonith::CLI.new(:start, ARGV)
+#!/bin/bash
+stonith-start

data/bin/{ey-monitor-reset → stonith}

@@ -2,4 +2,4 @@
 $LOAD_PATH.unshift(File.expand_path('../../lib/', __FILE__))
 require 'ey_stonith'
 
-EY::Stonith::CLI.new(:reset, ARGV)
+EY::Stonith::Commands.invoke(ARGV)

data/bin/{ey-monitor-stop → stonith-check}

@@ -2,4 +2,4 @@
 $LOAD_PATH.unshift(File.expand_path('../../lib/', __FILE__))
 require 'ey_stonith'
 
-EY::Stonith::CLI.new(:stop, ARGV)
+EY::Stonith::Commands::Check.new(ARGV).call

data/bin/stonith-claim

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+$LOAD_PATH.unshift(File.expand_path('../../lib/', __FILE__))
+require 'ey_stonith'
+
+EY::Stonith::Commands::Claim.new(ARGV).call

data/bin/{ey-monitor-resume → stonith-cron}

@@ -2,4 +2,4 @@
 $LOAD_PATH.unshift(File.expand_path('../../lib/', __FILE__))
 require 'ey_stonith'
 
-EY::Stonith::CLI.new(:resume, ARGV)
+EY::Stonith::Commands::Cron.new(ARGV).call

data/bin/stonith-notify

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+$LOAD_PATH.unshift(File.expand_path('../../lib/', __FILE__))
+require 'ey_stonith'
+
+EY::Stonith::Commands::Notify.new(ARGV).call

data/bin/stonith-reset

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+$LOAD_PATH.unshift(File.expand_path('../../lib/', __FILE__))
+require 'ey_stonith'
+
+EY::Stonith::Commands::Reset.new(ARGV).call

data/bin/stonith-resume

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+$LOAD_PATH.unshift(File.expand_path('../../lib/', __FILE__))
+require 'ey_stonith'
+
+EY::Stonith::Commands::Resume.new(ARGV).call

data/bin/stonith-status

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+$LOAD_PATH.unshift(File.expand_path('../../lib/', __FILE__))
+require 'ey_stonith'
+
+EY::Stonith::Commands::Status.new(ARGV).call

data/bin/{ey-monitor-status → stonith-stop}

@@ -2,4 +2,4 @@
 $LOAD_PATH.unshift(File.expand_path('../../lib/', __FILE__))
 require 'ey_stonith'
 
-EY::Stonith::CLI.new(:status, ARGV)
+EY::Stonith::Commands::Stop.new(ARGV).call

data/bin/stonith-takeover

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+$LOAD_PATH.unshift(File.expand_path('../../lib/', __FILE__))
+require 'ey_stonith'
+
+EY::Stonith::Commands::Takeover.new(ARGV).call

data/lib/ey_stonith/address_stealer.rb

@@ -3,12 +3,7 @@ require 'fog'
 module EY
   module Stonith
     class AddressStealer
-      def self.fog(credentials)
-        Fog::AWS::EC2.new(
-          :aws_access_key_id     => credentials[:aws_secret_id],
-          :aws_secret_access_key => credentials[:aws_secret_key]
-        )
-      end
+      def self.fog(credentials) Fog::AWS::EC2.new(credentials) end
 
       def initialize(server_id, ip, credentials)
         @fog         = self.class.fog(credentials)

data/lib/ey_stonith/awsm_notifier.rb

@@ -1,49 +1,48 @@
-require 'em-http'
+require 'json'
+require 'net/http'
+require 'timeout'
 
 module EY
   module Stonith
     class AwsmNotifier
-      def initialize(instance_id, notify_uri, opts, heartbeat = 5)
-        @instance_id, @notify_uri, @opts, @heartbeat = instance_id, notify_uri, opts, heartbeat
-        call_api
+      def initialize(instance_id, uri, credentials)
+        @instance_id, @uri, @credentials = instance_id, uri, credentials
       end
 
-      private
+      def notify(success, unreachable, refused)
+        Stonith.logger.info "Attempting to notify #@uri that I did a takeover."
 
-      def try_again
-        EM.add_timer(@heartbeat) { call_api }
-      end
+        code, body = do_request
 
-      def call_api
-        Stonith.logger.info("Notifying awsm that I did a takeover")
-        http = EM::HttpRequest.new(@notify_uri).post :body => body, :head => head, :timeout => 10
-        http.callback {
-          ok = (200...300).include?(http.response_header.status)
-          if ok && JSON.parse(http.response)['status'] == 'ok'
-            Stonith.logger.info("Notified awsm!")
+        case code
+        when 200...300
+          if JSON.parse(body)['status'] == 'ok'
+            success.call
           else
-            try_again
+            refused.call body
           end
-        }
-        http.errback { try_again }
+        else
+          unreachable.call
+        end
+      rescue
+        unreachable.call
       end
 
-      def head
-        {"Content-Type" => "application/x-www-form-urlencoded", "Accept" => "application/json"}
+      private
+
+      def do_request
+        response = Timeout.timeout(10) do
+          Net::HTTP.new(@uri.host, @uri.port).request_post(@uri.path, process_payload(payload), headers)
+        end
+        [response.code.to_i, response.body]
       end
 
-      def body
-        result = process_payload(payload)
-        def result.bytesize() size end unless result.respond_to?(:bytesize)
-        result
+      def headers
+        {"Content-Type" => "application/x-www-form-urlencoded", "Accept" => "application/json"}
       end
 
       def payload
-        {
-          'instance_id'    => @instance_id,
-          'aws_secret_id'  => @opts[:aws_secret_id],
-          'aws_secret_key' => @opts[:aws_secret_key],
-        }
+        @credentials.merge('instance_id' => @instance_id)
       end
 
       # ripped from restclient so we can use eventmachine

data/lib/ey_stonith/check_recorder.rb

@@ -3,50 +3,52 @@ module EY
     class CheckRecorder
       BAD_CHECK_MAX = 5
 
-      def initialize
-        reset
+      attr_reader :bad
+
+      def initialize(pathname)
+        @pathname = pathname
+        rehydrate
       end
       
       def bad_check!(key)
         reset_on_key_change(key)
         log_bad_check
         @bad += 1 if @seen_good
+        save
       end
 
       def good_check!(key)
-        @key = key
-        @bad = 0
-        @seen_good = true
-      end
-
-      def seen_good?
-        @seen_good
+        @key, @bad, @seen_good = key, 0, true
+        save
       end
 
       def limit_exceeded?
-        seen_good? && @bad > BAD_CHECK_MAX
+        @seen_good && @bad > BAD_CHECK_MAX
       end
 
-      def checking_key?(key)
-        @key == key
+      protected
+
+      def save
+        @pathname.open('w') { |f| f << Marshal.dump([@key, @bad, @seen_good]) }
       end
 
-      protected
+      def rehydrate
+        @key, @bad, @seen_good = Marshal.load(@pathname.read) if @pathname.readable?
+      end
 
       def reset
-        @bad = 0
-        @seen_good = false
+        @key, @bad, @seen_good = nil, 0, false
       end
 
       def reset_on_key_change(key)
-        unless checking_key?(key)
+        unless @key == key
           reset
           @key = key
         end
       end
 
       def log_bad_check
-        Stonith.logger.info("Bad check against #{@key}. Seen good? #{@seen_good}")
+        Stonith.logger.warn("Bad check against #{@key}. Seen good? #{@seen_good}")
       end
     end
   end

data/lib/ey_stonith/commands/abstract.rb

@@ -0,0 +1,94 @@
+require 'optparse'
+require 'pathname'
+
+module EY
+  module Stonith
+    module Commands
+      class Abstract
+        DEFAULT_CONFIG_PATH = Pathname.new("/etc/stonith.yml")
+        SCRIPT_NAME = 'stonith'
+
+        def self.command
+          '[COMMAND]'
+        end
+
+        def initialize(argv)
+          @argv = argv
+          parse!
+          init_logger
+        end
+
+        # Load the logger without requiring the config file to be present.
+        def init_logger
+          Stonith.log_to begin
+                           config.log_path
+                         rescue Config::FileNotFound
+                           $stderr
+                         end
+          Stonith.logger.level = Logger::INFO
+        end
+
+        def config
+          @config ||= Config.new(@options[:config_path])
+        end
+
+        def history
+          @history ||= History.new(config.history_path)
+        end
+
+        def database
+          @database ||= Database.new(config)
+        end
+
+        def call
+          invoke
+        rescue Config::Error => e
+          Stonith.logger.error e.message
+          abort "#{e.message}\n\n#{parser}"
+        rescue => e
+          error = "#{e.class}: #{e.message}\n#{e.backtrace.join("\n")}"
+          Stonith.logger.error error
+          abort error
+        end
+
+        def invoke
+          puts parser
+        end
+
+        def command_options
+          if @options[:config_path] != DEFAULT_CONFIG_PATH
+            " --config #{@options[:config_path]}"
+          else
+            ""
+          end
+        end
+
+        def parse!
+          parser.parse!(@argv)
+        end
+
+        def parser
+          @parser ||= OptionParser.new do |parser|
+            @options = {:config_path => DEFAULT_CONFIG_PATH}
+
+            parser.banner = "Usage: #{SCRIPT_NAME} #{self.class.command} [FLAGS]"
+            parser.separator ""
+            parser.separator "COMMANDS"
+            parser.separator Stonith::Commands.formatted_command_list
+            parser.separator ""
+            parser.separator "FLAGS"
+
+            parser.on('-c', '--config [FILE]', "Location of Stonith YAML config file (default #{@options[:config_path]})") do |path|
+              @options[:config_path] = Pathname.new(path)
+            end
+
+            parser.on_tail("-h", "--help", "Show this message") do
+              puts parser
+              exit
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ey_stonith/commands/check.rb

@@ -0,0 +1,58 @@
+module EY
+  module Stonith
+    module Commands
+      class Check < Abstract
+
+        def self.command
+          'check'
+        end
+
+        def self.banner
+          "Check the remote master haproxy"
+        end
+
+        def invoke
+          database.with_data do |data|
+            abort_if_master(data.hostname)
+
+            history << :check
+
+            begin
+              open("http://#{data.hostname}#{config.monitor_path}").read
+            rescue
+              bad_check(data.key, data.instance_id)
+            else
+              good_check(data.key)
+            end
+
+            exit
+          end
+
+          $stderr.puts "No master to check!"
+        end
+
+        def abort_if_master(hostname)
+          return if config.meta_data_hostname != hostname
+          abort "This instance is master! Can't check self."
+        end
+
+        def good_check(key)
+          check_recorder.good_check!(key)
+        end
+
+        def bad_check(key, instance_id)
+          check_recorder.bad_check!(key)
+
+          if check_recorder.limit_exceeded?
+            Stonith.logger.info "Invoking takeover of instance #{instance_id}"
+            exec "#{SCRIPT_NAME} takeover#{command_options} --instance #{instance_id}"
+          end
+        end
+
+        def check_recorder
+          @check_recorder ||= CheckRecorder.new(config.checks_path)
+        end
+      end
+    end
+  end
+end

data/lib/ey_stonith/commands/claim.rb

@@ -0,0 +1,113 @@
+module EY
+  module Stonith
+    module Commands
+      class Claim < Abstract
+
+        def self.command
+          'claim'
+        end
+
+        def self.banner
+          'Claim the master record in the database'
+        end
+
+        def claim_path
+          @claim_path ||= config.claim_path
+        end
+
+        def invoke
+          claim_path.exist?? invoke_with_claim_file : invoke_without_claim_file
+        end
+
+        def invoke_with_claim_file
+          abort_on_existing_data
+
+          attempts = (claim_path.read || 0).to_i.succ
+
+          if @force || attempts >= 6
+            reclaim!
+          else
+            persist_reclaim_attempt(attempts)
+          end
+        end
+
+        def abort_on_existing_data
+          return unless master_hostname
+
+          if config.meta_data_hostname == master_hostname
+            abort "Already claimed, not claiming."
+          else
+            claim_path.delete
+            abort "#{master_hostname} is master, not claiming.\nRemoving stale claim file."
+          end
+        end
+
+        def reclaim!
+          Stonith.logger.info @force ? "Reclaim forced." : "Reclaim after 6 consecutive checks."
+          claim_path.delete
+          invoke_without_claim_file
+        end
+
+        def persist_reclaim_attempt(attempts)
+          claim_path.open("w") { |f| f << attempts }
+
+          abort <<-ERROR
+Master unknown and claim file exists, not claiming.
+Reclaim at 6th attempt or use --force
+
+Failed attempts: #{attempts}
+          ERROR
+        end
+
+        def invoke_without_claim_file
+          @force ? database.reset : confirm_master!
+
+          data = Data.new(config.meta_data_hostname, config.meta_data_id, config.meta_data_ip)
+          database.set data
+          claim_path.open('w') {}
+
+          Stonith.logger.info "Claimed with data: #{data}"
+          history << :claim
+        end
+
+        def confirm_master!
+          # TODO: Only claim if the master has an IP?
+          # if fog.servers.get(config.meta_data_id).addresses.empty?
+          #   abort "No IP, not claiming." + (@force ? "\nIgnoring --force" : "")
+          # end
+
+          confirm_master_with_database
+          confirm_master_with_config
+        end
+
+        def confirm_master_with_database
+          return unless master_hostname
+
+          if config.meta_data_hostname == master_hostname
+            claim_path.open('w') {}
+            abort "Already claimed, not claiming. Touching claim file."
+          else
+            abort "#{master_hostname} is master, not claiming."
+          end
+        end
+
+        def confirm_master_with_config
+          if config.meta_data_hostname != config.monitor_host
+            abort "#{config.monitor_host} is master, not claiming."
+          end
+        end
+
+        def master_hostname
+          @master_hostname ||= database.with_data { |data| data.hostname }
+        end
+
+        def parser
+          super.on('-f', '--force', "Force the command (only applicable to claim currently)") do |f|
+            @force = f
+          end
+          super
+        end
+      end
+    end
+  end
+end

data/lib/ey_stonith/commands/commands.rb

@@ -0,0 +1,26 @@
+module EY
+  module Stonith
+    module Commands
+      class Commands < Abstract
+        def self.command
+          'commands'
+        end
+
+        def self.banner
+          "Print all the available commands"
+        end
+
+        def invoke
+          puts <<-MESSAGE
+Available stonith commands:
+
+#{Stonith::Commands.formatted_command_list}
+
+Example:
+    $ stonith status
+          MESSAGE
+        end
+      end
+    end
+  end
+end

data/lib/ey_stonith/commands/cron.rb

@@ -0,0 +1,40 @@
+module EY
+  module Stonith
+    module Commands
+      class Cron < Abstract
+        def self.command
+          'cron'
+        end
+
+        def self.banner
+          "Run the Stonith cron-like process that invokes other stonith commands periodically"
+        end
+
+        def invoke
+          heartbeat_loop do |beat|
+            unless_stopped { run_commands if beat.zero? }
+            sleep 1
+          end
+        end
+
+        def heartbeat_loop(&block)
+          loop { config.heartbeat.ceil.times(&block) }
+        end
+
+        def unless_stopped
+          if config.stop_path.exist?
+            history << :stop
+          else
+            yield
+          end
+        end
+
+        def run_commands
+          %w[claim notify check].each do |cmd|
+            system("stonith #{cmd}#{command_options}")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ey_stonith/commands/help.rb

@@ -0,0 +1,16 @@
+module EY
+  module Stonith
+    module Commands
+      class Help < Abstract
+        def self.banner
+          "A helpful message :D"
+        end
+
+        def invoke
+          puts "Stonith is a set of takeover & monitoring scripts for Engine Yard clusters.\n\n"
+          super
+        end
+      end
+    end
+  end
+end

data/lib/ey_stonith/commands/not_found.rb

@@ -0,0 +1,11 @@
+module EY
+  module Stonith
+    module Commands
+      class NotFound < Abstract
+        def invoke
+          abort "Command not found.\n#{parser}"
+        end
+      end
+    end
+  end
+end