ey_stonith 0.1.4 → 0.1.5.pre

data/lib/ey_stonith/commands/notify.rb

@@ -0,0 +1,85 @@
+module EY
+  module Stonith
+    module Commands
+      class Notify < Abstract
+        def self.command
+          'notify'
+        end
+
+        def self.banner
+          'Notify the provisioning server about a takeover'
+        end
+
+        def notify_path
+          @notify_path ||= config.notify_path
+        end
+
+        def invoke
+          abort_if_unintentional
+
+          notify_path.delete if notify_path.exist?
+
+          database.with_data do |data|
+            abort_if_master(data.hostname)
+            notify!
+          end
+
+          abort_no_data
+        end
+
+        def abort_if_unintentional
+          return if @force || notify_path.exist?
+
+          abort <<-ERROR
+Cannot notify, #{notify_path} does not exist.
+If you want to (destructively) notify AWSM that this instance is master, call with --force.
+          ERROR
+        end
+
+        def abort_if_master(hostname)
+          if config.meta_data_hostname != hostname
+            Stonith.logger.error "Cannot notify, I am not master. Master is #{hostname}. Cancelling notify."
+            abort "Cannot notify, I am not master."
+          end
+        end
+
+        def abort_no_data
+          msg = "Cannot notify, there is no master in the database. Giving up."
+          Stonith.logger.error msg
+          abort msg
+        end
+
+        def notify!
+          notifier = AwsmNotifier.new(config.meta_data_id, config.notify_uri, config.awsm_credentials)
+          notifier.notify(method(:success), method(:unreachable), method(:refused))
+        end
+
+        def success
+          Stonith.logger.info "AWSM notified!"
+          history << :notified
+          exit
+        end
+
+        def unreachable
+          notify_path.open('w') {}
+          msg = "Unable to reach AWSM for promotion to master."
+          Stonith.logger.warn msg
+          abort "#{msg}\nIf you're running this from the command line, you should run ey-monitor-notify again."
+        end
+
+        def refused(response_body)
+          msg = "Notify refused by endpoint. Giving up.\nResponse: #{response_body}"
+          Stonith.logger.error msg
+          abort msg
+        end
+
+        def parser
+          super.on('-f', '--force', "Force the command (only applicable to claim currently)") do |f|
+            @force = f
+          end
+          super
+        end
+      end
+    end
+  end
+end

data/lib/ey_stonith/commands/reset.rb

@@ -0,0 +1,21 @@
+module EY
+  module Stonith
+    module Commands
+      class Reset < Abstract
+        def self.command
+          'reset'
+        end
+
+        def self.banner
+          "Reset the state of all stonith commands"
+        end
+
+        def invoke
+          database.reset
+          config.state_path.rmtree
+          Stonith.logger.info "All state and database reset!"
+        end
+      end
+    end
+  end
+end

data/lib/ey_stonith/commands/resume.rb

@@ -0,0 +1,19 @@
+module EY
+  module Stonith
+    module Commands
+      class Resume < Abstract
+        def self.command
+          'resume'
+        end
+
+        def self.banner
+          "Resume Stonith monitoring"
+        end
+
+        def invoke
+          config.stop_path.delete if config.stop_path.exist?
+        end
+      end
+    end
+  end
+end

data/lib/ey_stonith/commands/status.rb

@@ -0,0 +1,23 @@
+module EY
+  module Stonith
+    module Commands
+      class Status < Abstract
+        def self.command
+          'status'
+        end
+
+        def self.banner
+          "List the last one or two actions performed"
+        end
+
+        def invoke
+          puts status unless status.empty?
+        end
+
+        def status
+          @status ||= history.to_s
+        end
+      end
+    end
+  end
+end

data/lib/ey_stonith/commands/stop.rb

@@ -0,0 +1,21 @@
+module EY
+  module Stonith
+    module Commands
+      class Stop < Abstract
+        def self.command
+          'stop'
+        end
+
+        def self.banner
+          "Stop Stonith monitoring"
+        end
+
+        def invoke
+          config.stop_path.open('w') {}
+          sleep(0.5) until history.last == "stop"
+          puts "takeover" if history.include?(:takeover)
+        end
+      end
+    end
+  end
+end

data/lib/ey_stonith/commands/takeover.rb

@@ -0,0 +1,106 @@
+require 'pathname'
+
+module EY
+  module Stonith
+    module Commands
+      class Takeover < Abstract
+        def self.command
+          'takeover'
+        end
+
+        def self.banner
+          'Cause a takeover of the specified master'
+        end
+
+        def invoke
+          abort_if_no_instance_id
+          abort_if_self
+
+          abort_if_takeover_lock
+          set_takeover_lock
+
+          database.with_locked_data do |data|
+            restore_data_on_fail(data)
+
+            if instance_id == data.instance_id
+              locked!(data.instance_id, data.ip)
+            else
+              relent!
+            end
+          end
+        end
+
+        def abort_if_no_instance_id
+          if !instance_id || instance_id == ""
+            abort "Please call with the instance_id of the master to takeover.\n\n#{parser}"
+          end
+        end
+
+        def abort_if_self
+          if instance_id == config.meta_data_id
+            abort "Cannot takeover self!"
+          end
+        end
+
+        def abort_if_takeover_lock
+          if config.takeover_path.exist?
+            abort "Already attempting takeover!\n#{config.takeover_path.read}"
+          end
+        end
+
+        def takeover_path
+          @takeover_path ||= config.takeover_path
+        end
+
+        def set_takeover_lock
+          at_exit { takeover_path.delete if takeover_path.exist? }
+          takeover_path.open('w') { |f| f << "Takeover started at #{Time.now}" }
+        end
+
+        def restore_data_on_fail(data)
+          at_exit do
+            if database.set(data) # always replace the data if it wasn't set
+              Stonith.logger.error("Emergency replacement of redis data #{data.inspect} occurred.")
+            end
+          end
+        end
+
+        def locked!(instance_id, master_ip)
+          Stonith.logger.info "Locked! Taking over #{instance_id}."
+          history << :takeover
+
+          ip = steal_address(instance_id, master_ip)
+          data = Data.new(config.meta_data_hostname, config.meta_data_id, ip)
+          database.set data
+
+          exec "#{SCRIPT_NAME} notify --force#{command_options}"
+        end
+
+        def relent!
+          history << :relent
+          msg = "Failed to grab lock, relenting."
+          Stonith.logger.info msg
+          abort msg
+        end
+
+        def steal_address(instance_id, ip)
+          Stonith.logger.info "Stealing IP #{ip} from #{instance_id}."
+          address = AddressStealer.new(instance_id, ip, config.fog_credentials)
+          address.associate(config.meta_data_id)
+          address.ip
+        end
+
+        def instance_id
+          @instance_id
+        end
+
+        def parser
+          super.on('-i', '--instance INSTANCE_ID', "Amazon Instance ID of the failing master") do |instance|
+            @instance_id = instance
+          end
+          super
+        end
+      end
+    end
+  end
+end

data/lib/ey_stonith/commands.rb

@@ -0,0 +1,40 @@
+module EY
+  module Stonith
+    module Commands
+
+      dir = 'ey_stonith/commands/'
+      autoload :Abstract, dir + 'abstract'
+      autoload :NotFound, dir + 'not_found'
+
+      COMMANDS = Hash.new(:NotFound).merge({
+        'check'    => :Check,
+        'claim'    => :Claim,
+        'commands' => :Commands,
+        'cron'     => :Cron,
+        'help'     => :Help,
+        'notify'   => :Notify,
+        'reset'    => :Reset,
+        'resume'   => :Resume,
+        'status'   => :Status,
+        'stop'     => :Stop,
+        'takeover' => :Takeover,
+      })
+
+      COMMANDS.each do |command, klass|
+        autoload klass, dir + command
+      end
+
+      def self.invoke(argv)
+        klass = const_get(COMMANDS[argv.shift])
+        klass.new(argv).call
+      end
+
+      def self.formatted_command_list
+        COMMANDS.sort { |(a,_),(b,_)| a <=> b }.map { |cmd, klass_name|
+          klass = const_get(klass_name)
+          "    #{cmd.ljust(8)}   #{klass.banner}"
+        }.join("\n")
+      end
+    end
+  end
+end

data/lib/ey_stonith/config.rb

@@ -1,26 +1,119 @@
 require 'yaml'
 require 'json'
+require 'open-uri'
 
 module EY
   module Stonith
-    class Config < Struct.new(:ey_cloud, :redis_yml, :dna_json, :history_path, :pid_path, :log_path)
-      def initialize(*args)
-        super *args.map { |arg| arg && Pathname.new(arg) }
+    class Config
+      class Error < StandardError
+        def initialize(path, message)
+          super("Config file #{path}: #{message}")
+        end
       end
-      
-      def cloud_credentials() YAML::load_file(ey_cloud) end
-      def notify_uri() "#{cloud_credentials[:api]}/api/promote_instance_to_master" end
 
-      def monitor_heartbeat() 10 end
-      def redis_key() 'ey:stonith' end
-      def redis_db() 14 end
-      def redis_host() redis[:host] end # support sometimes changes this, do not cache!
-      def redis_port() redis[:port] end
-      def master_hostname_from_dna() dna['master_app_server']['private_dns_name'] end
+      class FileNotFound < Error
+        def initialize(path)
+          super(path, "File not found.")
+        end
+      end
+
+      class RequiredSetting < Error
+        def initialize(path, key)
+          super(path, "Missing required setting #{key}.")
+        end
+      end
+
+      DEFAULT = {
+        'log'        => '/var/log/stonith.log',
+        'state_dir'  => '/var/run/stonith',
+        'heartbeat'  => 10,
+        'notify_uri' => nil,
+
+        'monitor_host' => nil,
+        'monitor_path' => '/haproxy/monitor',
+
+        'redis_host' => nil,
+        'redis_port' => 6379,
+        'redis_key'  => 'stonith',
+        'redis_db'   => 14,
+        'redis_timeout' => 60 * 5,
+
+        'aws_secret_id'  => nil,
+        'aws_secret_key' => nil,
+      }
+
+      def initialize(config_path)
+        path = Pathname.new(config_path)
+        raise FileNotFound.new(path) unless path.readable?
+
+        realpath = path.realpath
+        @data = DEFAULT.merge(YAML.load_file(realpath))
+        @path = realpath
+      end
+
+      attr_reader :path
+
+      def log_path()      pathname(log) end
+      def state_path()    ensure_exists(pathname(state_dir)) end
+      def stop_path()     state_path + 'stop' end
+      def claim_path()    state_path + 'claim' end
+      def checks_path()   state_path + 'checks' end
+      def notify_path()   state_path + 'notify' end
+      def history_path()  state_path + 'history' end
+      def takeover_path() state_path + 'takeover' end
+      def notify_uri()    URI.parse(method_missing('notify_uri')) end
+
+      def meta_data_hostname() @data['meta_data_hostname'] ||= meta_data('local-hostname') end
+      def meta_data_id()       @data['meta_data_id'] ||= meta_data('instance-id') end
+      def meta_data_ip()       @data['meta_data_ip'] || meta_data('public-ipv4')  end # don't cache
+
+      def respond_to?(meth)
+        @data.key?(meth) || super
+      end
+
+      def awsm_credentials
+        {
+          'aws_secret_id'  => aws_secret_id,
+          'aws_secret_key' => aws_secret_key,
+        }
+      end
+
+      def fog_credentials
+        {
+          :aws_access_key_id     => aws_secret_id,
+          :aws_secret_access_key => aws_secret_key,
+        }
+      end
 
       private
-      def dna() JSON.parse(dna_json.read) end
-      def redis() YAML::load_file(redis_yml) end
+
+      def self.secret_hack
+        @secret_hack = true
+      end
+
+      def self.pwned?
+        @secret_hack
+      end
+      
+      def meta_data(key)
+        open("http://169.254.169.254/latest/meta-data/#{key}").read
+      end
+
+      def pathname(path) path && Pathname.new(path) end
+
+      def ensure_exists(path)
+        path.mkpath unless self.class.pwned?
+        path
+      end
+
+      def method_missing(meth, *args)
+        meth_s = meth.to_s
+        if respond_to?(meth_s)
+          @data[meth_s] || raise(RequiredSetting.new(path, meth_s))
+        else
+          super
+        end
+      end
     end
   end
 end

data/lib/ey_stonith/data.rb

@@ -1,7 +1,11 @@
 module EY
   module Stonith
     class Data < Struct.new(:hostname, :instance_id, :ip)
-      def key() instance_id end
+      alias_method :key, :instance_id
+
+      def to_s
+        "[#{hostname.inspect}, #{instance_id.inspect}, #{ip.inspect}]"
+      end
     end
   end
 end

data/lib/ey_stonith/database.rb

@@ -23,9 +23,10 @@ module EY
       end
 
       def set(data)
-        unless get
+        unless get # very small race condition (very)
           redis.lpush(master_key, Marshal.dump(data))
           @locked = false
+          true
         end
       end
 
@@ -34,22 +35,43 @@ module EY
       end
 
       private
-      
+
       def get
         result = redis.lindex(master_key, 0) # index 0
         result && Marshal.load(result)
       end
 
+      # popping & locking
       def locked_get
-        Marshal.load redis.blpop(master_key, 0).last # don't timeout (this number gets passed to the actual redis command)
+        Marshal.load redis.blpop(master_key, @config.redis_timeout).last
       end
 
       def master_key
-        @config.redis_key
+        "#{@config.redis_key}:master"
       end
-      
+
       def redis
-        @redis ||= Redis.new(:host => @config.redis_host, :port => @config.redis_port, :db => @config.redis_db, :timeout => 0)
+        @redis ||= load_redis
+      end
+
+      def load_redis
+        redis = Redis.new(:host => @config.redis_host, :port => @config.redis_port, :db => @config.redis_db, :timeout => @config.redis_timeout)
+        check_redis_version redis.info[:redis_version]
+        redis
+      rescue Errno::ECONNREFUSED
+        abort "Unable to connect to redis"
+      end
+
+      def check_redis_version(version)
+        major, minor, patch = version.split('.').map { |num| num.to_i }
+        unless major > 1 ||
+          (major == 1 && minor > 3) ||
+          (major == 1 && minor == 3 && patch >= 1)
+          abort <<-ERROR
+Redis server version [#{version}] is too old.
+>= 1.3.1 required for blpop support.
+          ERROR
+        end
       end
     end
   end

data/lib/ey_stonith/history.rb

@@ -30,7 +30,7 @@ module EY
       end
 
       def last() read.last end
-      def read() @path.read.to_s.split(SEPARATOR) end
+      def read() @path.read.to_s.strip.split(SEPARATOR) end
     end
   end
 end

data/lib/ey_stonith.rb

@@ -11,6 +11,7 @@ module EY
     autoload :AddressStealer, 'ey_stonith/address_stealer'
     autoload :Box,            'ey_stonith/box'
     autoload :CheckRecorder,  'ey_stonith/check_recorder'
+    autoload :Commands,       'ey_stonith/commands'
     autoload :CLI,            'ey_stonith/cli'
     autoload :Config,         'ey_stonith/config'
     autoload :Data,           'ey_stonith/data'
@@ -21,15 +22,8 @@ module EY
     autoload :MetaData,       'ey_stonith/meta_data'
     autoload :Slave,          'ey_stonith/slave'
 
-    def self.logger=(logger) @@logger = logger end
+    def self.log_to(io) @@logger = Logger.new(io) end
     def self.logger() @@logger end
-    self.logger = Logger.new(STDOUT)
-    logger.level = Logger::INFO
-
-    def self.meta_data() @@meta_data end
-    def self.meta_data=(meta) @@meta_data = meta end
-    def self.reset_meta_data() @@meta_data = MetaData end
-    reset_meta_data
   end
 end