ey_stonith 0.3.6 → 0.4.1.pre

data/lib/ey_stonith/commands/claim.rb

@@ -2,110 +2,19 @@ module EY
   module Stonith
     module Commands
       class Claim < Abstract
-
         def self.command
           'claim'
         end
 
         def self.banner
-          'Claim the master record in the database'
-        end
-
-        def claim_path
-          @claim_path ||= config.claim_path
+          "Legacy command. No longer used."
         end
 
         def invoke
-          claim_path.exist?? invoke_with_claim_file : invoke_without_claim_file
-        end
-
-        def invoke_with_claim_file
-          abort_on_existing_data
-
-          attempts = (claim_path.read || 0).to_i.succ
-
-          if @force || attempts >= 6
-            reclaim!
-          else
-            persist_reclaim_attempt(attempts)
-          end
-        end
-
-        def abort_on_existing_data
-          return unless master_hostname
-
-          if config.meta_data_hostname == master_hostname
-            abort "Already claimed, not claiming."
-          else
-            claim_path.delete
-            abort "#{master_hostname} is master, not claiming.\nRemoving stale claim file."
-          end
-        end
-
-        def reclaim!
-          Stonith.logger.info @force ? "Reclaim forced." : "Reclaim after 6 consecutive checks."
-          claim_path.delete
-          invoke_without_claim_file
-        end
-
-        def persist_reclaim_attempt(attempts)
-          claim_path.open("w") { |f| f << attempts }
-
-          abort <<-ERROR
-Master unknown and claim file exists, not claiming.
-Reclaim at 6th attempt or use --force
-
-Failed attempts: #{attempts}
-          ERROR
-        end
-
-        def invoke_without_claim_file
-          @force ? database.reset : confirm_master!
-
-          data = Data.new(config.meta_data_hostname, config.meta_data_id, config.meta_data_ip)
-          database.set data
-          claim_path.open('w') {}
-
-          Stonith.logger.info "Claimed with data: #{data}"
-          history << :claim
-        end
-
-        def confirm_master!
-          # TODO: Only claim if the master has an IP?
-          # if fog.servers.get(config.meta_data_id).addresses.empty?
-          #   abort "No IP, not claiming." + (@force ? "\nIgnoring --force" : "")
-          # end
-
-          confirm_master_with_database
-          confirm_master_with_config
-        end
-
-        def confirm_master_with_database
-          return unless master_hostname
-
-          if config.meta_data_hostname == master_hostname
-            claim_path.open('w') {}
-            abort "Already claimed, not claiming. Touching claim file."
-          else
-            abort "#{master_hostname} is master, not claiming."
-          end
-        end
-
-        def confirm_master_with_config
-          if config.meta_data_hostname != config.monitor_host
-            abort "#{config.monitor_host} is master, not claiming."
-          end
-        end
-
-        def master_hostname
-          @master_hostname ||= database.with_data { |data| data.hostname }
-        end
-
-        def parser
-          super.on('-f', '--force', "Force the command (only applicable to claim currently)") do |f|
-            @force = f
-          end
-          super
+          puts <<-HELP
+Yo Dawg, I heard you like Claim, but we removed it because we don't do that anymore, so stop using it.
+There is no replacement command needed. Just update instances to update the master.
+          HELP
         end
       end
     end

data/lib/ey_stonith/commands/cron.rb

@@ -13,7 +13,7 @@ module EY
         def invoke
           Stonith.logger.info "Stonith started"
           heartbeat_loop do |beat|
-            unless_stopped { run_commands if beat.zero? }
+            unless_stopped { run_command if beat.zero? }
             sleep 1
           end
         end
@@ -30,10 +30,8 @@ module EY
           end
         end
 
-        def run_commands
-          %w[claim notify check].each do |cmd|
-            system("#{script} #{cmd}#{command_options}")
-          end
+        def run_command
+          system("#{script} check#{command_options}")
         end
       end
     end

data/lib/ey_stonith/commands/info.rb

@@ -7,14 +7,11 @@ module EY
         end
 
         def self.banner
-          "Print the full config file and database contents"
+          "Print the full config file"
         end
 
         def invoke
           puts config
-          puts "Database:"
-          data = database.with_data { |data| data }
-          puts "  #{data}"
         end
       end
     end

data/lib/ey_stonith/commands/not_found.rb

@@ -3,6 +3,7 @@ module EY
     module Commands
       class NotFound < Abstract
         def invoke
+          Stonith.logger.error "Command not found: #{@argv.inspect}"
           abort "Command not found #{@argv.first.inspect}.\n#{parser}"
         end
       end

data/lib/ey_stonith/commands/notify.rb

@@ -7,7 +7,7 @@ module EY
         end
 
         def self.banner
-          'Notify the provisioning server about a takeover'
+          'Notify the provisioning server that master is down.'
         end
 
         def notify_path
@@ -15,70 +15,31 @@ module EY
         end
 
         def invoke
-          abort_if_unintentional
-
-          notify_path.delete if notify_path.exist?
-
-          database.with_data do |data|
-            abort_if_master data.hostname
-            notify!
-          end
-
-          abort_no_data
-        end
-
-        def abort_if_unintentional
-          return if @force || notify_path.exist?
-
-          abort <<-ERROR
-Cannot notify, #{notify_path} does not exist.
-If you want to (destructively) notify AWSM that this instance is master, call with --force.
-          ERROR
-        end
-
-        def abort_if_master(hostname)
-          if config.meta_data_hostname != hostname
-            Stonith.logger.error "Cannot notify, I am not master. Master is #{hostname}. Cancelling notify."
-            abort "Cannot notify, I am not master."
-          end
-        end
-
-        def abort_no_data
-          msg = "Cannot notify, there is no master in the database. Giving up."
-          Stonith.logger.error msg
-          abort msg
+          Stonith.logger.warn "Notifying provisioning server that master is down."
+          history << :notify
+          notify!
+          exit
         end
 
         def notify!
-          notifier = AwsmNotifier.new config.meta_data_id, config.takeover_uri, config.endpoint_token
-          notifier.notify method(:success), method(:unreachable), method(:refused)
+          notifier = AwsmNotifier.new(config.notify_uri, config.endpoint_id, config.endpoint_token, {
+            'monitor_host' => config.monitor_host
+          })
+
+          notifier.notify method(:success), method(:unreachable)
         end
 
-        def success
-          Stonith.logger.info "AWSM notified!"
+        def success(data)
           history << :notified
-          exit
+          count    = data['notification_count'] || 'N/A'
+          takeover = data['takeover'].inspect
+          Stonith.logger.info "Provisioning server notified: (notification_count: #{count}, takeover: #{takeover})"
         end
 
         def unreachable
-          notify_path.open('w') {}
-          msg = "Unable to reach AWSM for promotion to master."
+          msg = "Unable to notify provisioning server that master is down."
           Stonith.logger.warn msg
-          abort "#{msg}\nIf you're running this from the command line, you should run ey-monitor-notify again."
-        end
-
-        def refused(response_body)
-          msg = "Notify refused by endpoint. Giving up.\nResponse: #{response_body}"
-          Stonith.logger.error msg
-          history << :refused
-          abort msg
-        end
-
-        def parser
-          super.on '-f', '--force', "Force the command (only applicable to claim currently)" do |f|
-            @force = f
-          end
-          super
+          abort "#{msg}\nIf you're running this from the command line, you should run `stonith notify` again."
         end
       end
     end

data/lib/ey_stonith/commands/reset.rb

@@ -11,7 +11,6 @@ module EY
         end
 
         def invoke
-          database.reset
           config.state_path.rmtree
           Stonith.logger.info "All state and database reset!"
         end

data/lib/ey_stonith/commands/stop.rb

@@ -15,7 +15,6 @@ module EY
           sleep(0.5) until history.last == "stop"
 
           Stonith.logger.info "Stonith stopped"
-          puts "takeover" if history.include?(:takeover)
         end
       end
     end

data/lib/ey_stonith/commands/takeover.rb

@@ -1,5 +1,3 @@
-require 'pathname'
-
 module EY
   module Stonith
     module Commands
@@ -9,97 +7,14 @@ module EY
         end
 
         def self.banner
-          'Cause a takeover of the specified master'
+          "Legacy command. No longer used. Run this command for more info."
         end
 
         def invoke
-          abort_if_no_instance_id
-          abort_if_self
-
-          abort_if_takeover_lock
-          set_takeover_lock
-
-          database.with_locked_data do |data|
-            restore_data_on_fail(data)
-
-            if instance_id == data.instance_id
-              locked!(data.instance_id, data.ip)
-            else
-              relent!
-            end
-          end
-        end
-
-        def abort_if_no_instance_id
-          if !instance_id || instance_id == ""
-            abort "Please call with the instance_id of the master to takeover.\n\n#{parser}"
-          end
-        end
-
-        def abort_if_self
-          if instance_id == config.meta_data_id
-            abort "Cannot takeover self!"
-          end
-        end
-
-        def abort_if_takeover_lock
-          if config.takeover_path.exist?
-            abort "Already attempting takeover!\n#{config.takeover_path.read}"
-          end
-        end
-
-        def takeover_path
-          @takeover_path ||= config.takeover_path
-        end
-
-        def set_takeover_lock
-          at_exit { takeover_path.delete if takeover_path.exist? }
-          takeover_path.open('w') { |f| f << "Takeover started at #{Time.now}" }
-        end
-
-        def restore_data_on_fail(data)
-          at_exit do
-            if database.set(data) # always replace the data if it wasn't set
-              Stonith.logger.error("Emergency replacement of redis data #{data.inspect} occurred.")
-            end
-          end
-        end
-
-        def locked!(instance_id, master_ip)
-          Stonith.logger.info "Locked! Taking over #{instance_id}."
-          history << :takeover
-
-          ip = steal_address(instance_id, master_ip)
-          data = Data.new(config.meta_data_hostname, config.meta_data_id, ip)
-          database.set data
-
-          takeover_path.delete
-          execute :notify, "--force"
-        end
-
-        def relent!
-          history << :relent
-          msg = "Failed to grab lock, relenting."
-          Stonith.logger.info msg
-          abort msg
-        end
-
-        def steal_address(instance_id, ip)
-          Stonith.logger.info "Stealing IP #{ip} from #{instance_id}."
-          address = AddressStealer.new(instance_id, ip, config.fog_credentials)
-          address.associate(config.meta_data_id)
-          address.ip
-        end
-
-        def instance_id
-          @instance_id
-        end
-
-        def parser
-          super.on('-i', '--instance INSTANCE_ID', "Amazon Instance ID of the failing master") do |instance|
-            @instance_id = instance
-          end
-          super
+          puts <<-HELP
+Takeovers are no longer performed from the instance side directly.
+You can cause a takeover by running `stonith notify` 6 times quickly if you really need to.
+          HELP
         end
       end
     end

data/lib/ey_stonith/commands.rb

@@ -14,8 +14,8 @@ module EY
         'help'     => :Help,
         'info'     => :Info,
         'notify'   => :Notify,
-        'reset'    => :Reset,
         'resume'   => :Resume,
+        'reset'    => :Reset,
         'status'   => :Status,
         'stop'     => :Stop,
         'takeover' => :Takeover,

data/lib/ey_stonith/config.rb

@@ -1,6 +1,5 @@
 require 'yaml'
 require 'json'
-require 'open-uri'
 
 module EY
   module Stonith
@@ -24,24 +23,15 @@ module EY
       end
 
       DEFAULT = {
-        'log'        => '/var/log/stonith.log',
-        'state_dir'  => '/var/run/stonith',
-        'heartbeat'  => 10,
-        'endpoint_uri' => nil,
-        'endpoint_token' => nil,
-
-        'monitor_host' => nil,
-        'monitor_path' => '/haproxy/monitor',
+        'log'             => '/var/log/stonith.log',
+        'state_dir'       => '/var/run/stonith',
+        'heartbeat'       => 10,
+        'endpoint_uri'    => nil,
+        'endpoint_token'  => nil,
+        'endpoint_id'     => nil,
+        'monitor_host'    => nil,
+        'monitor_path'    => '/haproxy/monitor',
         'monitor_timeout' => 5,
-
-        'redis_host' => nil,
-        'redis_port' => 6379,
-        'redis_key'  => 'stonith',
-        'redis_db'   => 14,
-        'redis_timeout' => 60 * 5,
-
-        'aws_secret_id'  => nil,
-        'aws_secret_key' => nil,
       }
 
       def initialize(config_path)
@@ -65,71 +55,36 @@ Config:
 
   state_path:    #{state_path}
   stop_path:     #{stop_path}
-  claim_path:    #{claim_path}
-  checks_path:   #{checks_path}
-  notify_path:   #{notify_path}
   history_path:  #{history_path}
-  takeover_path: #{takeover_path}
+  success_path:  #{success_path}
 
-  endpoint_uri: #{endpoint_uri}
+  endpoint_uri:   #{endpoint_uri}
   endpoint_token: #{endpoint_token}
-  takeover_uri: #{takeover_uri}
-  fog_credentials: #{fog_credentials.inspect}
-  awsm_credentials: #{awsm_credentials.inspect}
-
-  meta_data_hostname: #{meta_data_hostname}
-  meta_data_id: #{meta_data_id}
-  meta_data_ip: #{meta_data_ip}
+  endpoint_id:    #{endpoint_id}
+  notify_uri:     #{notify_uri}
 
-  monitor_host: #{monitor_host}
-  monitor_path: #{monitor_path}
+  monitor_host:    #{monitor_host}
+  monitor_path:    #{monitor_path}
   monitor_timeout: #{monitor_timeout}
-
-  redis_host: #{redis_host}
-  redis_port: #{redis_port}
-  redis_key: #{redis_key}
-  redis_db: #{redis_db}
-  redis_timeout: #{redis_timeout}
         STRING
       end
 
       def log_path()      pathname(log) end
       def state_path()    ensure_exists(pathname(state_dir)) end
       def stop_path()     state_path + 'stop' end
-      def claim_path()    state_path + 'claim' end
-      def checks_path()   state_path + 'checks' end
-      def notify_path()   state_path + 'notify' end
       def history_path()  state_path + 'history' end
-      def takeover_path() state_path + 'takeover' end
+      def success_path()  state_path + 'success' end
       def endpoint_uri()  URI.parse(method_missing('endpoint_uri')) end
-      def takeover_uri
+      def notify_uri
         result = endpoint_uri
-        result.path << TAKEOVER_PATH
+        result.path << NOTIFY_PATH
         result
       end
 
-      def meta_data_hostname() @data['meta_data_hostname'] ||= meta_data('local-hostname') end
-      def meta_data_id()       @data['meta_data_id'] ||= meta_data('instance-id') end
-      def meta_data_ip()       @data['meta_data_ip'] || meta_data('public-ipv4')  end # don't cache
-
       def respond_to?(meth)
         @data.key?(meth) || super
       end
 
-      def awsm_credentials
-        {
-          'aws_secret_id'  => aws_secret_id,
-          'aws_secret_key' => aws_secret_key,
-        }
-      end
-
-      def fog_credentials
-        {
-          :aws_access_key_id     => aws_secret_id,
-          :aws_secret_access_key => aws_secret_key,
-        }
-      end
-
       private
 
       def self.secret_hack
@@ -139,10 +94,6 @@ Config:
       def self.pwned?
         @secret_hack
       end
-      
-      def meta_data(key)
-        open("http://169.254.169.254/latest/meta-data/#{key}").read
-      end
 
       def pathname(path) path && Pathname.new(path) end
 

data/lib/ey_stonith/rackapp.rb

@@ -4,8 +4,12 @@ require 'json'
 module EY
   module Stonith
     class Rackapp < Sinatra::Base
+      enable :raise_errors
+      disable :dump_errors
+      disable :show_exceptions
+
       before { content_type :json }
-      get('/') { {}.to_json }
+      get('/') { '' }
 
       get HEALTH_PATH do
         unless EY::Stonith.healthy?
@@ -15,14 +19,34 @@ module EY
       end
 
       post TAKEOVER_PATH do
-        unless EY::Stonith.takeover(label, token)
+        if valid?
+          unless EY::Stonith.takeover(label)
+            status 410
+          end
+        else
           status 410
         end
         {}.to_json
       end
 
+      post NOTIFY_PATH do
+        if valid?
+          EY::Stonith.notify(label, params[:monitor_host])
+          {
+            'notification_count' => '1',
+            'takeover' => false
+          }.to_json
+        else
+          status 403
+        end
+      end
+
       private
 
+      def valid?
+        EY::Stonith.valid?(label, token)
+      end
+
       def label() params['label'] end
       def token() params['token'] end
     end

data/lib/ey_stonith.rb

@@ -1,4 +1,5 @@
 require 'logger'
+require 'open-uri'
 
 $LOAD_PATH.unshift(File.expand_path('..', __FILE__))
 
@@ -6,25 +7,15 @@ module EY
   module Stonith
     class Error < StandardError; end
 
-    autoload :AbstractMaster, 'ey_stonith/abstract_master'
     autoload :AwsmNotifier,   'ey_stonith/awsm_notifier'
-    autoload :AddressStealer, 'ey_stonith/address_stealer'
-    autoload :Box,            'ey_stonith/box'
-    autoload :CheckRecorder,  'ey_stonith/check_recorder'
     autoload :Commands,       'ey_stonith/commands'
-    autoload :CLI,            'ey_stonith/cli'
     autoload :Config,         'ey_stonith/config'
-    autoload :Data,           'ey_stonith/data'
-    autoload :Database,       'ey_stonith/database'
     autoload :History,        'ey_stonith/history'
-    autoload :LocalMaster,    'ey_stonith/local_master'
-    autoload :Master,         'ey_stonith/master'
-    autoload :MetaData,       'ey_stonith/meta_data'
     autoload :Rackapp,        'ey_stonith/rackapp'
-    autoload :Slave,          'ey_stonith/slave'
 
     HEALTH_PATH   = '/health'   unless defined? HEALTH_PATH
     TAKEOVER_PATH = '/takeover' unless defined? TAKEOVER_PATH
+    NOTIFY_PATH   = '/notify'   unless defined? NOTIFY_PATH
 
     def self.log_to(io)
       @@logger = Logger.new io
@@ -38,10 +29,12 @@ module EY
 
     def self.healthy?() callback_module.healthy? end
 
-    def self.takeover(label, token)
-      if valid? label, token
-        callback_module.takeover label
-      end
+    def self.takeover(label)
+      callback_module.takeover label
+    end
+
+    def self.notify(label, monitor_host)
+      callback_module.notify label, monitor_host
     end
 
     def self.valid?(label, token)

data/spec/config_spec.rb

@@ -0,0 +1,53 @@
+require File.expand_path('../spec_helper', __FILE__)
+require 'pathname'
+
+describe EY::Stonith::Config do
+  before { described_class.send(:secret_hack) }
+  def fixture_path(file)
+    Pathname.new(File.expand_path(File.join('../fixtures', file), __FILE__))
+  end
+
+  describe "default" do
+    subject { described_class.new(fixture_path('empty.yml')) }
+
+    def self.it_requires(key)
+      it(key) { lambda { subject.send(key) }.should raise_error(EY::Stonith::Config::RequiredSetting) }
+    end
+
+    its(:path)          { should == fixture_path('empty.yml') }
+    its(:log_path)      { should == Pathname.new('/var/log/stonith.log') }
+    its(:state_path)    { should == Pathname.new('/var/run/stonith') }
+    its(:stop_path)     { should == Pathname.new('/var/run/stonith/stop') }
+    its(:history_path)  { should == Pathname.new('/var/run/stonith/history') }
+    its(:success_path)  { should == Pathname.new('/var/run/stonith/success') }
+    its(:heartbeat)     { should == 10 }
+
+    it_requires(:monitor_host)
+    its(:monitor_path)     { should == '/haproxy/monitor' }
+    its(:monitor_timeout)  { should == 5 }
+
+    it_requires(:endpoint_uri)
+    it_requires(:endpoint_token)
+    it_requires(:endpoint_id)
+  end
+
+  describe "overridden" do
+    subject { described_class.new(fixture_path('config.yml')) }
+
+    its(:path)          { should == fixture_path('config.yml') }
+    its(:log_path)      { should == Pathname.new('var/log/stonith.log') }
+    its(:state_path)    { should == Pathname.new('var/run/stonith') }
+    its(:stop_path)     { should == Pathname.new('var/run/stonith/stop') }
+    its(:history_path)  { should == Pathname.new('var/run/stonith/history') }
+    its(:success_path)  { should == Pathname.new('var/run/stonith/success') }
+    its(:heartbeat)     { should == 1 }
+
+    its(:monitor_host)    { should == 'localhost' }
+    its(:monitor_path)    { should == '/chickity/check' }
+    its(:monitor_timeout) { should == 10 }
+
+    its(:endpoint_uri)    { should == URI.parse('http://example.com/stonith') }
+    its(:endpoint_token)  { should == 'i-87654321' }
+    its(:endpoint_id)     { should == "i-87654321" }
+  end
+end