ey_stonith 0.3.6 → 0.4.1.pre

data/.gitignore

@@ -0,0 +1,10 @@
+.DS_Store
+.bundle
+*.swp
+*.swo
+pkg
+coverage
+spec/config.yml
+tmp
+vendor
+Gemfile.lock

data/Gemfile

@@ -0,0 +1,3 @@
+source :rubygems
+
+gemspec

data/Rakefile

@@ -0,0 +1,26 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+require 'spec/rake/spectask'
+desc "Run unit specifications"
+Spec::Rake::SpecTask.new do |t|
+  t.spec_opts << %w(-fs --color)
+  t.spec_opts << '--loadby' << 'random'
+  t.spec_files = Dir["spec/**/*_spec.rb"]
+  t.rcov = ENV["RCOV"] == "true"
+  t.rcov_opts << '--exclude' << 'spec,gems'
+  t.rcov_opts << '--text-summary'
+  t.rcov_opts << '--sort' << 'coverage' << '--sort-reverse'
+end
+
+require 'cucumber/rake/task'
+Cucumber::Rake::Task.new do |t|
+  t.cucumber_opts = %w{--format pretty}
+  t.rcov = ENV["RCOV"] == "true"
+  t.rcov_opts << '--exclude' << 'features,gems'
+  t.rcov_opts << '--text-summary'
+  t.rcov_opts << '--sort' << 'coverage' << '--sort-reverse'
+end
+
+desc "Run specs and cukes"
+task :default => [:spec, :cucumber]

data/ey_stonith.gemspec

@@ -0,0 +1,36 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+
+Gem::Specification.new do |s|
+  s.name = "ey_stonith"
+  s.version = "0.4.1.pre"
+  s.summary = "Shoot The Other Node In The Head (with a nerf gun)"
+  s.description = s.summary
+  s.has_rdoc = true
+  s.extra_rdoc_files = ["README.rdoc", "LICENSE"]
+  s.authors = ["Ezra Zygmuntowicz", "Larry Diehl", "Martin Emde", "Tim Carey-Smith", "Jason Hansen"]
+  s.email = "awsmdev@engineyard.com"
+  s.homepage = "http://engineyard.com/cloud"
+
+  s.add_dependency 'json'
+  s.add_dependency 'sinatra',         '~>1.0'
+  s.add_dependency 'SystemTimer',     '~>1.2'
+
+  s.add_development_dependency 'rspec'
+  s.add_development_dependency 'cucumber'
+  s.add_development_dependency 'open4'
+  s.add_development_dependency 'aruba',      '0.3.2'
+  s.add_development_dependency 'ruby-debug'
+  s.add_development_dependency 'rake'
+  s.add_development_dependency 'randexp'
+  s.add_development_dependency 'cucumber'
+  s.add_development_dependency 'rcov'
+  s.add_development_dependency 'realweb',    '~>0.1.4'
+  s.add_development_dependency 'rack-client'
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+end
+

data/features/check.feature

@@ -0,0 +1,57 @@
+Feature: check
+
+  Background:
+    Given a file named "stonith.yml" with:
+      """
+      ---
+      log: stonith.log
+      state_dir: var/run/stonith
+      endpoint_uri: http://example.com/stonith
+      endpoint_token: token-for-i-12345678
+      endpoint_id: i-12345678
+      monitor_host: localhost
+      monitor_path: /health
+      success_path: var/run/stonith/success
+      """
+    Given I successfully run "stonith reset -c stonith.yml"
+
+  Scenario: Unhealthy master without a successful check
+    Given a file named "var/run/stonith/success" should not exist
+    And a "dead" master in "stonith.yml"
+    And a "healthy" endpoint in "stonith.yml"
+    When I successfully run "stonith check -c stonith.yml"
+    Then the output should contain exactly ""
+    And a file named "var/run/stonith/success" should not exist
+    When I successfully run "stonith status -c stonith.yml"
+    Then the output should contain exactly:
+      """
+      check
+
+      """
+
+  Scenario: Unhealthy master with previous successful check
+    Given a "dead" master in "stonith.yml"
+    And a "healthy" endpoint in "stonith.yml"
+    And an empty file named "var/run/stonith/success"
+    When I successfully run "stonith check -c stonith.yml"
+    Then the output should contain exactly ""
+    Then the file "server.log" should contain "notify i-12345678 __master_hostname__"
+    When I successfully run "stonith status -c stonith.yml"
+    Then the output should contain exactly:
+      """
+      notify -> notified
+
+      """
+
+  Scenario: Healthy master
+    Given a file named "var/run/stonith/success" should not exist
+    Given a "healthy" master in "stonith.yml"
+    When I successfully run "stonith check -c stonith.yml"
+    Then the output should contain exactly ""
+    And a file named "var/run/stonith/success" should exist
+    When I successfully run "stonith status -c stonith.yml"
+    Then the output should contain exactly:
+      """
+      check
+
+      """

data/features/cron.feature

@@ -0,0 +1,67 @@
+Feature: cron startup & stop
+ 
+  Background:
+    Given a file named "stonith.yml" with:
+      """
+      --- 
+      log: stonith.log
+      state_dir: var/run/stonith
+      heartbeat: 1
+      endpoint_uri: http://example.com/stonith
+      endpoint_token: token-for-i-12345678
+      endpoint_id: i-12345678
+      monitor_host: localhost
+      monitor_path: /health
+      """
+    Given I successfully run "stonith reset -c stonith.yml"
+    Given a "healthy" endpoint in "stonith.yml"
+
+  Scenario: Monitor - Running normally
+    Given a "healthy" master in "stonith.yml"
+    When I run "stonith cron -c stonith.yml" interactively
+    When I wait 5 seconds
+    And I successfully run "stonith status -c stonith.yml"
+    Then the output should contain exactly:
+      """
+      check
+
+      """
+    When I successfully run "stonith stop -c stonith.yml"
+    When I run "stonith status -c stonith.yml"
+    Then it should pass with exactly:
+      """
+      check -> stop
+
+      """
+    When I successfully run "stonith resume -c stonith.yml"
+    When I successfully run "stonith status -c stonith.yml"
+    Then the output should contain exactly:
+      """
+      stop -> resume
+
+      """
+    And I wait 5 seconds
+    When I successfully run "stonith status -c stonith.yml"
+    Then the output should contain exactly:
+      """
+      resume -> check
+
+      """
+
+  Scenario: Monitor - Resuming without stopping
+    Given a "healthy" master in "stonith.yml"
+    When I run "stonith cron -c stonith.yml" interactively
+    And I wait 5 seconds
+    And I successfully run "stonith status -c stonith.yml"
+    Then the output should contain exactly:
+      """
+      check
+
+      """
+    When I successfully run "stonith resume -c stonith.yml"
+    When I successfully run "stonith status -c stonith.yml"
+    Then the output should contain exactly:
+      """
+      check
+
+      """

data/features/fixtures/dead.ru

@@ -0,0 +1,6 @@
+EY::Stonith.callback_module = Class.new MockMapper do
+  def self.healthy?() false end
+  def self.notify(_) raise end
+end
+
+run EY::Stonith::Rackapp

data/features/fixtures/healthy.ru

@@ -0,0 +1 @@
+run EY::Stonith::Rackapp

data/features/help.feature

@@ -0,0 +1,7 @@
+Feature: help
+
+  Scenario: Printing the help file
+    When I successfully run "stonith help"
+    Then the output should contain "Usage: stonith [COMMAND]"
+    And the output should not contain "Command not found"
+    And the output should not contain "Config file not found"

data/features/not_found.feature

@@ -0,0 +1,24 @@
+Feature: not found
+
+  Background:
+    Given a file named "stonith.yml" with:
+      """
+      ---
+      log: stonith.log
+      state_dir: var/run/stonith
+      endpoint_uri: http://example.com/stonith
+      endpoint_token: token-for-i-12345678
+      endpoint_id: i-12345678
+      monitor_host: localhost
+      monitor_path: /health
+      success_path: var/run/stonith/success
+      """
+    Given I successfully run "stonith reset -c stonith.yml"
+
+  Scenario: Running the stonith twitter notifier (command not found)
+    When I failingly run "stonith twitter -c stonith.yml"
+    Then the stderr should contain "Command not found"
+
+  Scenario: Config file not found
+    When I failingly run "stonith status"
+    Then the stderr should contain "Config file /etc/stonith.yml: File not found."

data/features/notify.feature

@@ -0,0 +1,35 @@
+Feature: notify
+
+  Background:
+    Given a file named "stonith.yml" with:
+      """
+      ---
+      log: stonith.log
+      state_dir: var/run/stonith
+      endpoint_uri: http://example.com/stonith
+      endpoint_token: token-for-i-12345678
+      endpoint_id: i-12345678
+      monitor_host: localhost
+      monitor_path: /health
+      success_path: var/run/stonith/success
+      """
+    Given I successfully run "stonith reset -c stonith.yml"
+
+  Scenario: Endpoint is dead
+    Given a "dead" endpoint in "stonith.yml"
+    When I failingly run "stonith notify -c stonith.yml"
+    Then the stderr should contain "Unable to notify provisioning server that master is down."
+    When I successfully run "stonith status -c stonith.yml"
+    Then the stdout from "stonith status -c stonith.yml" should contain "notify"
+
+  Scenario: Endpoint is happy
+    Given a "healthy" endpoint in "stonith.yml"
+    When I successfully run "stonith notify -c stonith.yml"
+    Then the output should contain exactly ""
+    Then the file "stonith.log" should contain "Provisioning server notified: (notification_count: 1, takeover: false)"
+    When I successfully run "stonith status -c stonith.yml"
+    Then the output should contain exactly:
+      """
+      notify -> notified
+
+      """

data/features/resume.feature

@@ -0,0 +1,49 @@
+Feature: not found
+
+  Background:
+    Given a file named "stonith.yml" with:
+      """
+      ---
+      log: stonith.log
+      state_dir: var/run/stonith
+      endpoint_uri: http://example.com/stonith
+      endpoint_token: token-for-i-12345678
+      endpoint_id: i-12345678
+      monitor_host: localhost
+      monitor_path: /health
+      success_path: var/run/stonith/success
+      """
+    Given I successfully run "stonith reset -c stonith.yml"
+
+  Scenario: Not stopped
+    Given a file named "var/run/stonith/history" with:
+      """
+      check
+      """
+    When I successfully run "stonith resume -c stonith.yml"
+    Then the output should contain exactly ""
+    And the following files should not exist:
+      | var/run/stonith/stop |
+    When I successfully run "stonith status -c stonith.yml"
+    Then the output should contain exactly:
+      """
+      check
+
+      """
+
+  Scenario: Already stopped
+    Given a file named "var/run/stonith/history" with:
+      """
+      check -> stop
+      """
+    And an empty file named "var/run/stonith/stop"
+    When I successfully run "stonith resume -c stonith.yml"
+    Then the output should contain exactly ""
+    And the following files should not exist:
+      | var/run/stonith/stop |
+    When I successfully run "stonith status -c stonith.yml"
+    Then the output should contain exactly:
+      """
+      stop -> resume
+
+      """

data/features/steps/stonith_steps.rb

@@ -0,0 +1,40 @@
+When /^I wait (\d+) seconds$/ do |time|
+  sleep time.to_i
+end
+
+When /^I failingly run "(.*)"$/ do |cmd|
+  And %|I run "#{cmd}"|
+  Then %|the exit status should not be 0|
+end
+
+When /^I run "(.*)" (\d) seconds? later$/ do |cmd, wait|
+  Given %|an executable named "bin/wait_then_run" with:|, <<-SCRIPT
+#!/bin/bash
+sleep #{wait}
+#{cmd}
+  SCRIPT
+  And %|I run "bin/wait_then_run" interactively|
+end
+
+Given /^an executable named "([^\"]*)" with:$/ do |file_name, file_content|
+  Given %|a file named "#{file_name}" with:|, file_content
+  Given %|I successfully run "chmod a+x #{file_name}"|
+end
+
+Transform %r|(.*)__master_hostname__(.*)| do |before, after|
+  "#{before}#{master_hostname}#{after}"
+end
+
+def master_hostname
+  @master_hostname
+end
+
+Given /^a "([^\"]+)" master in "(.*)"$/ do |type, path|
+  @master_hostname = start_server type
+  update_config({"monitor_host" => @master_hostname}, path)
+end
+
+Given /^an? "(.*)" endpoint in "(.*)"$/ do |type, path|
+  @endpoint_hostname = start_server type
+  update_config({'endpoint_uri' => "http://#{@endpoint_hostname}"}, path)
+end

data/features/stop.feature

@@ -0,0 +1,48 @@
+Feature: stop
+ 
+  Background:
+    Given a file named "stonith.yml" with:
+      """
+      --- 
+      log: stonith.log
+      state_dir: var/run/stonith
+      endpoint_uri: http://example.com/stonith
+      endpoint_token: token-for-i-12345678
+      endpoint_id: i-12345678
+      monitor_host: localhost
+      monitor_path: /health
+      """
+    Given I successfully run "stonith reset -c stonith.yml"
+
+  Scenario: Already stopped
+    Given a file named "var/run/stonith/history" with:
+      """
+      check -> stop
+      """
+    And an empty file named "var/run/stonith/stop"
+    When I successfully run "stonith stop -c stonith.yml"
+    Then the output should contain exactly ""
+    And the following files should exist:
+      | var/run/stonith/stop |
+    When I successfully run "stonith status -c stonith.yml"
+    Then the output should contain exactly:
+      """
+      check -> stop
+
+      """
+
+  Scenario: Stopping when not stopped
+    Given a file named "var/run/stonith/history" with:
+      """
+      check
+      """
+    When I run "echo 'check -> stop' > var/run/stonith/history" 3 seconds later
+    When I successfully run "stonith stop -c stonith.yml"
+    And the following files should exist:
+      | var/run/stonith/stop |
+    And I successfully run "stonith status -c stonith.yml"
+    Then the output should contain exactly:
+      """
+      check -> stop
+
+      """

data/features/support/env.rb

@@ -0,0 +1,77 @@
+require 'aruba'
+require 'open4'
+require 'realweb'
+require 'aruba/cucumber'
+require File.expand_path('../../../lib/ey_stonith', __FILE__)
+
+
+ENV['PATH'] = [File.expand_path('../../../bin', __FILE__), ENV['PATH']].join(":")
+ENV['RACK_ENV'] = 'test'
+
+class MockMapper
+  @@notifications = []
+
+  def self.log(msg)
+    File.open('tmp/aruba/server.log', 'w') { |f| f.puts msg }
+  end
+  def self.healthy?() true end
+  def self.takeover(_) true end
+  def self.notify(label, monitor_host)
+    log "notify #{label} #{monitor_host}"
+    true
+  end
+  def self.token_for(label) "token-for-#{label}" end
+end
+
+module Helpers
+  def safe_read(io)
+    io.read_nonblock(1000)
+  rescue EOFError
+    io.read
+  rescue Errno::EAGAIN
+    ""
+  end
+
+  def start_server(config)
+    config_ru = File.expand_path("../../fixtures/#{config}.ru", __FILE__)
+    @server = RealWeb.start_server_in_fork(config_ru)
+    at_exit { stop_server }
+    "127.0.0.1:#{@server.port}"
+  end
+
+  def stop_server
+    if @server
+      @server.stop
+      @server = nil
+    end
+  end
+
+  def update_config(options, path = "stonith.yml")
+    path = File.expand_path(path, 'tmp/aruba')
+    config = YAML.load_file(path).merge(options)
+    File.open(path,'w') { |file| YAML.dump(config, file) }  
+  end
+
+  def kill_run_in_background
+    if @pid
+      Process.kill('TERM', @pid)
+      Process.wait @pid
+    end
+  rescue
+    # erm... noop
+  ensure
+    @pid = nil
+  end
+end
+
+World(Helpers)
+
+After do
+  kill_run_in_background
+  stop_server
+end
+
+Before do
+  @aruba_timeout_seconds = 8
+  EY::Stonith.callback_module = MockMapper
+end

data/lib/ey_stonith/awsm_notifier.rb

@@ -6,20 +6,22 @@ require 'timeout'
 module EY
   module Stonith
     class AwsmNotifier
-      def initialize(instance_id, uri, token)
-        @instance_id, @uri, @token = instance_id, uri, token
+      def initialize(uri, instance_id, token, params)
+        @uri, @instance_id, @token, @params = uri, instance_id, token, params
       end
 
-      def notify(success, unreachable, refused)
-        Stonith.logger.info "Attempting to notify #@uri that I did a takeover."
+      def notify(success, unreachable)
+        Stonith.logger.info "Attempting to notify #@uri that master is down."
 
         code, body = do_request
 
         case code
         when 200...300
-          success.call
-        when 410
-          refused.call body
+          if body.empty?
+            success.call {}
+          else
+            success.call JSON.parse(body)
+          end
         else
           unreachable.call
         end
@@ -43,7 +45,10 @@ module EY
       end
 
       def payload
-        {'label' => @instance_id, 'token' => @token}
+        @params.merge({
+          'label' => @instance_id,
+          'token' => @token,
+        })
       end
 
       # ripped from restclient

data/lib/ey_stonith/commands/abstract.rb

@@ -36,10 +36,6 @@ module EY
           @history ||= History.new(config.history_path)
         end
 
-        def database
-          @database ||= Database.new(config)
-        end
-
         def call
           invoke
         rescue Config::Error => e

data/lib/ey_stonith/commands/check.rb

@@ -14,47 +14,24 @@ module EY
         end
 
         def invoke
-          database.with_data do |data|
-            abort_if_master(data.hostname)
+          history << :check
 
-            history << :check
-
-            begin
-              timeout(config.monitor_timeout) do
-                open("http://#{data.hostname}#{config.monitor_path}").read
-              end
-            rescue StandardError, Timeout::Error
-              bad_check(data.key, data.instance_id)
+          begin
+            timeout(config.monitor_timeout) do
+              open("http://#{config.monitor_host}#{config.monitor_path}").read
+            end
+          rescue StandardError, Timeout::Error => e
+            Stonith.logger.error "Got error when checking master: #{e.class}: #{e.message}"
+            if config.success_path.exist?
+              execute :notify
             else
-              good_check(data.key)
+              Stonith.logger.error "No successful check against master yet. Not notifying."
             end
-
-            exit
+          else
+            config.success_path.open('w') {}
           end
 
-          abort "No master to check!"
-        end
-
-        def abort_if_master(hostname)
-          return if config.meta_data_hostname != hostname
-          abort "This instance is master! Can't check self."
-        end
-
-        def good_check(key)
-          check_recorder.good_check!(key)
-        end
-
-        def bad_check(key, instance_id)
-          check_recorder.bad_check!(key)
-
-          if check_recorder.limit_exceeded?
-            Stonith.logger.info "Invoking takeover of instance #{instance_id}"
-            execute :takeover, "--instance #{instance_id}"
-          end
-        end
-
-        def check_recorder
-          @check_recorder ||= CheckRecorder.new(config.checks_path)
+          exit
         end
       end
     end