ey_stonith 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/LICENSE +20 -0
- data/README.rdoc +17 -0
- data/bin/ey-monitor +5 -0
- data/bin/ey-monitor-reset +5 -0
- data/bin/ey-monitor-resume +5 -0
- data/bin/ey-monitor-status +5 -0
- data/bin/ey-monitor-stop +5 -0
- data/lib/ey_stonith/abstract_master.rb +15 -0
- data/lib/ey_stonith/address_stealer.rb +45 -0
- data/lib/ey_stonith/awsm_notifier.rb +61 -0
- data/lib/ey_stonith/box.rb +57 -0
- data/lib/ey_stonith/check_recorder.rb +53 -0
- data/lib/ey_stonith/cli.rb +127 -0
- data/lib/ey_stonith/config.rb +26 -0
- data/lib/ey_stonith/data.rb +7 -0
- data/lib/ey_stonith/database.rb +56 -0
- data/lib/ey_stonith/history.rb +36 -0
- data/lib/ey_stonith/local_master.rb +28 -0
- data/lib/ey_stonith/master.rb +37 -0
- data/lib/ey_stonith/meta_data.rb +11 -0
- data/lib/ey_stonith/slave.rb +41 -0
- data/lib/ey_stonith.rb +34 -0
- metadata +155 -0
data/LICENSE
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
Copyright (c) 2010 Engine Yard Inc.
|
2
|
+
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
4
|
+
a copy of this software and associated documentation files (the
|
5
|
+
"Software"), to deal in the Software without restriction, including
|
6
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
7
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
8
|
+
permit persons to whom the Software is furnished to do so, subject to
|
9
|
+
the following conditions:
|
10
|
+
|
11
|
+
The above copyright notice and this permission notice shall be
|
12
|
+
included in all copies or substantial portions of the Software.
|
13
|
+
|
14
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
15
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
16
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
17
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
18
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
19
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
20
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.rdoc
ADDED
@@ -0,0 +1,17 @@
|
|
1
|
+
== ey_stonith
|
2
|
+
|
3
|
+
Shoot The Other Instance In The Head
|
4
|
+
|
5
|
+
= Development
|
6
|
+
|
7
|
+
You need to have gem bundler install
|
8
|
+
|
9
|
+
gem install bundler
|
10
|
+
|
11
|
+
Then bundle everything up:
|
12
|
+
|
13
|
+
bundle install
|
14
|
+
|
15
|
+
To run the specs copy the example file to spec/config.yml and run:
|
16
|
+
|
17
|
+
bundle exec spec -c spec
|
data/bin/ey-monitor
ADDED
data/bin/ey-monitor-stop
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
require 'json'
|
2
|
+
require 'json/ext'
|
3
|
+
module EY
|
4
|
+
module Stonith
|
5
|
+
class AbstractMaster
|
6
|
+
private
|
7
|
+
|
8
|
+
# ips are elastic, do not cache!
|
9
|
+
def public_ip() meta_data('public-ipv4') end
|
10
|
+
def local_hostname() @local_hostname ||= meta_data("local-hostname") end
|
11
|
+
|
12
|
+
def meta_data(key) EY::Stonith.meta_data[key] end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
require 'fog'
|
2
|
+
|
3
|
+
module EY
|
4
|
+
module Stonith
|
5
|
+
class AddressStealer
|
6
|
+
def self.fog(credentials)
|
7
|
+
Fog::AWS::EC2.new(
|
8
|
+
:aws_access_key_id => credentials[:aws_secret_id],
|
9
|
+
:aws_secret_access_key => credentials[:aws_secret_key]
|
10
|
+
)
|
11
|
+
end
|
12
|
+
|
13
|
+
def initialize(server_id, ip, credentials)
|
14
|
+
@fog = self.class.fog(credentials)
|
15
|
+
server = @fog.servers.get(server_id)
|
16
|
+
main_address = @fog.addresses.get(ip)
|
17
|
+
|
18
|
+
@address = if !main_address.server_id
|
19
|
+
main_address
|
20
|
+
elsif server
|
21
|
+
address_for(server)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
def associate(server_id)
|
26
|
+
server = @fog.servers.get(server_id)
|
27
|
+
#TODO: handle error
|
28
|
+
raise "Don't have any IPs to use!" unless @address
|
29
|
+
raise "Already have an IP" if address_for(server)
|
30
|
+
@address.server = server
|
31
|
+
end
|
32
|
+
|
33
|
+
def ip
|
34
|
+
@address.public_ip
|
35
|
+
end
|
36
|
+
|
37
|
+
private
|
38
|
+
|
39
|
+
def address_for(server)
|
40
|
+
@fog.addresses.detect{|addr| addr.server_id == server.id }
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
@@ -0,0 +1,61 @@
|
|
1
|
+
require 'em-http'
|
2
|
+
|
3
|
+
module EY
|
4
|
+
module Stonith
|
5
|
+
class AwsmNotifier
|
6
|
+
def initialize(instance_id, notify_uri, opts, heartbeat = 5)
|
7
|
+
@instance_id, @notify_uri, @opts, @heartbeat = instance_id, notify_uri, opts, heartbeat
|
8
|
+
call_api
|
9
|
+
end
|
10
|
+
|
11
|
+
private
|
12
|
+
|
13
|
+
def try_again
|
14
|
+
EM.add_timer(@heartbeat) { call_api }
|
15
|
+
end
|
16
|
+
|
17
|
+
def call_api
|
18
|
+
Stonith.logger.info("Notifying awsm that I did a takeover")
|
19
|
+
http = EM::HttpRequest.new(@notify_uri).post :body => body, :head => head, :timeout => 10
|
20
|
+
http.callback {
|
21
|
+
ok = (200...300).include?(http.response_header.status)
|
22
|
+
if ok && JSON.parse(http.response)['status'] == 'ok'
|
23
|
+
Stonith.logger.info("Notified awsm!")
|
24
|
+
else
|
25
|
+
try_again
|
26
|
+
end
|
27
|
+
}
|
28
|
+
http.errback { try_again }
|
29
|
+
end
|
30
|
+
|
31
|
+
def head
|
32
|
+
{"Content-Type" => "application/x-www-form-urlencoded", "Accept" => "application/json"}
|
33
|
+
end
|
34
|
+
|
35
|
+
def body
|
36
|
+
process_payload(payload)
|
37
|
+
end
|
38
|
+
|
39
|
+
def payload
|
40
|
+
{
|
41
|
+
'instance_id' => @instance_id,
|
42
|
+
'aws_secret_id' => @opts[:aws_secret_id],
|
43
|
+
'aws_secret_key' => @opts[:aws_secret_key],
|
44
|
+
}
|
45
|
+
end
|
46
|
+
|
47
|
+
# ripped from restclient so we can use eventmachine
|
48
|
+
def process_payload(p = nil, parent_key = nil)
|
49
|
+
p.keys.map do |k|
|
50
|
+
key = parent_key ? "#{parent_key}[#{k}]" : k
|
51
|
+
if p[k].is_a? Hash
|
52
|
+
process_payload(p[k], key)
|
53
|
+
else
|
54
|
+
value = URI.escape(p[k].to_s, Regexp.new("[^#{URI::PATTERN::UNRESERVED}]"))
|
55
|
+
"#{key}=#{value}"
|
56
|
+
end
|
57
|
+
end.join("&")
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
@@ -0,0 +1,57 @@
|
|
1
|
+
module EY
|
2
|
+
module Stonith
|
3
|
+
class Box
|
4
|
+
def initialize(config, history)
|
5
|
+
EY::Stonith.logger.info "Booting Stonith"
|
6
|
+
|
7
|
+
@config, @history = config, history
|
8
|
+
database = Database.new(@config)
|
9
|
+
@master = Master.new(database, @config.master_hostname_from_dna)
|
10
|
+
|
11
|
+
trap("HUP") { stop }
|
12
|
+
trap("USR1") do
|
13
|
+
stop
|
14
|
+
EM.next_tick { start }
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
def start
|
19
|
+
if @master.local?
|
20
|
+
@local_master = LocalMaster.new(@config.notify_uri, @config.cloud_credentials)
|
21
|
+
@master.update @local_master.data
|
22
|
+
@history << :claim
|
23
|
+
else
|
24
|
+
@slave = Slave.new(self, @config.monitor_heartbeat)
|
25
|
+
@slave.monitor!(@master)
|
26
|
+
@history << :monitor
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
def try_takeover(master_unchanged)
|
31
|
+
@master.with_locked_data do |data|
|
32
|
+
if master_unchanged.call(data.key)
|
33
|
+
Stonith.logger.info("Locked! Taking over.")
|
34
|
+
@slave = nil
|
35
|
+
@history << :takeover
|
36
|
+
@local_master = LocalMaster.new(@config.notify_uri, @config.cloud_credentials)
|
37
|
+
new_data = @local_master.takeover!(data.instance_id, data.ip)
|
38
|
+
@master.update new_data
|
39
|
+
else
|
40
|
+
Stonith.logger.info("Failed to grab lock, relenting.")
|
41
|
+
@slave.start!
|
42
|
+
@history << :relent
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
private
|
48
|
+
|
49
|
+
def stop
|
50
|
+
EM.next_tick {
|
51
|
+
@slave.stop! if @slave
|
52
|
+
@history << :stop
|
53
|
+
}
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
@@ -0,0 +1,53 @@
|
|
1
|
+
module EY
|
2
|
+
module Stonith
|
3
|
+
class CheckRecorder
|
4
|
+
BAD_CHECK_MAX = 5
|
5
|
+
|
6
|
+
def initialize
|
7
|
+
reset
|
8
|
+
end
|
9
|
+
|
10
|
+
def bad_check!(key)
|
11
|
+
reset_on_key_change(key)
|
12
|
+
log_bad_check
|
13
|
+
@bad += 1 if @seen_good
|
14
|
+
end
|
15
|
+
|
16
|
+
def good_check!(key)
|
17
|
+
@key = key
|
18
|
+
@bad = 0
|
19
|
+
@seen_good = true
|
20
|
+
end
|
21
|
+
|
22
|
+
def seen_good?
|
23
|
+
@seen_good
|
24
|
+
end
|
25
|
+
|
26
|
+
def limit_exceeded?
|
27
|
+
seen_good? && @bad > BAD_CHECK_MAX
|
28
|
+
end
|
29
|
+
|
30
|
+
def checking_key?(key)
|
31
|
+
@key == key
|
32
|
+
end
|
33
|
+
|
34
|
+
protected
|
35
|
+
|
36
|
+
def reset
|
37
|
+
@bad = 0
|
38
|
+
@seen_good = false
|
39
|
+
end
|
40
|
+
|
41
|
+
def reset_on_key_change(key)
|
42
|
+
unless checking_key?(key)
|
43
|
+
reset
|
44
|
+
@key = key
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
def log_bad_check
|
49
|
+
Stonith.logger.info("Bad check against #{@key}. Seen good? #{@seen_good}")
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
@@ -0,0 +1,127 @@
|
|
1
|
+
require 'eventmachine'
|
2
|
+
require 'optparse'
|
3
|
+
require 'pathname'
|
4
|
+
|
5
|
+
module EY
|
6
|
+
module Stonith
|
7
|
+
class CLI
|
8
|
+
DEFAULT_FILES = {
|
9
|
+
:ey => "/etc/.ey-cloud.yml",
|
10
|
+
:redis => "/etc/redis.yml",
|
11
|
+
:dna => "/etc/chef/dna.json",
|
12
|
+
:history => "/var/run/ey-monitor.history",
|
13
|
+
:pid_path => "/var/run/ey-monitor.pid",
|
14
|
+
}
|
15
|
+
|
16
|
+
def self.commands
|
17
|
+
@commands ||= Hash.new do |commands, command|
|
18
|
+
lambda {
|
19
|
+
puts "Command not found: #{command}"
|
20
|
+
puts @parser
|
21
|
+
exit(1)
|
22
|
+
}
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def self.command(cmd,&block)
|
27
|
+
commands[cmd.to_sym] = block
|
28
|
+
end
|
29
|
+
|
30
|
+
command :start do
|
31
|
+
with_pid do
|
32
|
+
EY::Stonith.meta_data = @options[:meta_data] if @options[:meta_data]
|
33
|
+
EM.run { Box.new(@config, @history).start }
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
command :stop do
|
38
|
+
system("kill -HUP #{get_pid}")
|
39
|
+
sleep(1) until @history.last == "stop"
|
40
|
+
puts "takeover" if @history.include?(:takeover)
|
41
|
+
end
|
42
|
+
|
43
|
+
command :resume do
|
44
|
+
system("kill -USR1 #{get_pid}")
|
45
|
+
end
|
46
|
+
|
47
|
+
command :status do
|
48
|
+
puts @history
|
49
|
+
end
|
50
|
+
|
51
|
+
command :reset do
|
52
|
+
Database.new(@config).reset
|
53
|
+
@history.reset
|
54
|
+
end
|
55
|
+
|
56
|
+
def initialize(command, argv)
|
57
|
+
parse!(argv)
|
58
|
+
@config = Config.new(@options[:ey], @options[:redis], @options[:dna], @options[:history], @options[:pid_path])
|
59
|
+
cmd = command.to_sym || :start
|
60
|
+
@history = History.new(@config.history_path)
|
61
|
+
instance_eval(&self.class.commands[cmd.to_sym])
|
62
|
+
end
|
63
|
+
|
64
|
+
def with_pid
|
65
|
+
@config.pid_path.open('w') { |file| file << $$ }
|
66
|
+
yield
|
67
|
+
@config.pid_path.delete
|
68
|
+
end
|
69
|
+
|
70
|
+
def get_pid
|
71
|
+
@config.pid_path.read
|
72
|
+
end
|
73
|
+
|
74
|
+
def parse!(argv)
|
75
|
+
@options = DEFAULT_FILES
|
76
|
+
parser.parse!(argv)
|
77
|
+
end
|
78
|
+
|
79
|
+
def parser
|
80
|
+
@parser ||= OptionParser.new do |parser|
|
81
|
+
parser.banner = <<-USAGE
|
82
|
+
Usage: ey-monitor [FLAGS] [COMMAND]
|
83
|
+
|
84
|
+
COMMANDS
|
85
|
+
start Begin monitoring (default).
|
86
|
+
stop Safely stop running ey-monitor agents.
|
87
|
+
Reports shutdown status to stdout.
|
88
|
+
Exits with non-zero status if there is a problem.
|
89
|
+
|
90
|
+
USAGE
|
91
|
+
|
92
|
+
parser.separator "FLAGS"
|
93
|
+
|
94
|
+
parser.on('-e', '--ey [FILE]', "Path to the ey-cloud.yml file (default #{@options[:ey]})") do |ey|
|
95
|
+
@options[:ey] = Pathname.new(ey)
|
96
|
+
end
|
97
|
+
|
98
|
+
parser.on('-r', '--redis [FILE]', "Path to the redis.yml file (default #{@options[:redis]})") do |redis|
|
99
|
+
@options[:redis] = Pathname.new(redis)
|
100
|
+
end
|
101
|
+
|
102
|
+
parser.on('-d', '--dna [FILE]', "Path to the chef/dna.json file (default #{@options[:dna]})") do |dna|
|
103
|
+
@options[:dna] = Pathname.new(dna)
|
104
|
+
end
|
105
|
+
|
106
|
+
parser.on('-m', '--meta-data [FILE]', "Mocked AWS meta_data yaml file") do |path|
|
107
|
+
@options[:meta_data] = YAML.load_file(Pathname.new(path))
|
108
|
+
end
|
109
|
+
|
110
|
+
parser.on('-t', '--history [FILE]', "Location of stonith history file (default #{@options[:history]}") do |path|
|
111
|
+
@options[:history] = Pathname.new(path)
|
112
|
+
end
|
113
|
+
|
114
|
+
parser.on('-p', '--pid [FILE]', "Location of stonith pid file (default #{@options[:pid_path]}") do |path|
|
115
|
+
@options[:pid_path] = Pathname.new(path)
|
116
|
+
end
|
117
|
+
|
118
|
+
parser.on_tail("-h", "--help", "Show this message") do
|
119
|
+
puts parser
|
120
|
+
exit
|
121
|
+
end
|
122
|
+
end
|
123
|
+
end
|
124
|
+
end
|
125
|
+
end
|
126
|
+
end
|
127
|
+
|
@@ -0,0 +1,26 @@
|
|
1
|
+
require 'yaml'
|
2
|
+
require 'json'
|
3
|
+
|
4
|
+
module EY
|
5
|
+
module Stonith
|
6
|
+
class Config < Struct.new(:ey_cloud, :redis_yml, :dna_json, :history_path, :pid_path)
|
7
|
+
def initialize(*args)
|
8
|
+
super *args.map { |arg| arg && Pathname.new(arg) }
|
9
|
+
end
|
10
|
+
|
11
|
+
def cloud_credentials() YAML::load_file(ey_cloud) end
|
12
|
+
def notify_uri() "#{cloud_credentials[:api]}/api/promote_instance_to_master" end
|
13
|
+
|
14
|
+
def monitor_heartbeat() 10 end
|
15
|
+
def redis_key() 'ey:stonith' end
|
16
|
+
def redis_db() 15 end
|
17
|
+
def redis_host() redis[:host] end # support sometimes changes this, do not cache!
|
18
|
+
def redis_port() redis[:port] end
|
19
|
+
def master_hostname_from_dna() dna['master_app_server']['private_dns_name'] end
|
20
|
+
|
21
|
+
private
|
22
|
+
def dna() JSON.parse(dna_json.read) end
|
23
|
+
def redis() YAML::load_file(redis_yml) end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,56 @@
|
|
1
|
+
require 'redis'
|
2
|
+
|
3
|
+
module EY
|
4
|
+
module Stonith
|
5
|
+
class Database
|
6
|
+
def initialize(config)
|
7
|
+
@config = config
|
8
|
+
end
|
9
|
+
|
10
|
+
def with_locked_data
|
11
|
+
raise "Already locked!" if @locked
|
12
|
+
@locked = true
|
13
|
+
data = locked_get
|
14
|
+
yield data
|
15
|
+
ensure
|
16
|
+
set(data) if @locked
|
17
|
+
end
|
18
|
+
|
19
|
+
def with_data
|
20
|
+
raise "Locked!" if @locked
|
21
|
+
data = get
|
22
|
+
yield data if data
|
23
|
+
end
|
24
|
+
|
25
|
+
def set(data)
|
26
|
+
unless get
|
27
|
+
redis.lpush(master_key, Marshal.dump(data))
|
28
|
+
@locked = false
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
def reset
|
33
|
+
redis.ltrim master_key, 1, 0
|
34
|
+
end
|
35
|
+
|
36
|
+
private
|
37
|
+
|
38
|
+
def get
|
39
|
+
result = redis.lindex(master_key, 0) # index 0
|
40
|
+
result && Marshal.load(result)
|
41
|
+
end
|
42
|
+
|
43
|
+
def locked_get
|
44
|
+
Marshal.load redis.blpop(master_key, 0).last # don't timeout (this number gets passed to the actual redis command)
|
45
|
+
end
|
46
|
+
|
47
|
+
def master_key
|
48
|
+
@config.redis_key
|
49
|
+
end
|
50
|
+
|
51
|
+
def redis
|
52
|
+
@redis ||= Redis.new(:host => @config.redis_host, :port => @config.redis_port, :db => @config.redis_db, :timeout => 0)
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
require 'pathname'
|
2
|
+
|
3
|
+
module EY
|
4
|
+
module Stonith
|
5
|
+
class History
|
6
|
+
SEPARATOR = ' -> '
|
7
|
+
|
8
|
+
def initialize(path, length = 2)
|
9
|
+
raise ArgumentError, "Hey, I need at least 2 history lengthinesses to be a history object." if length < 2
|
10
|
+
@length = length
|
11
|
+
@path = Pathname.new(path)
|
12
|
+
@path.dirname.mkpath
|
13
|
+
FileUtils.touch(@path)
|
14
|
+
end
|
15
|
+
|
16
|
+
def <<(status)
|
17
|
+
write(status) unless last == status
|
18
|
+
self
|
19
|
+
end
|
20
|
+
|
21
|
+
def include?(status)
|
22
|
+
read.include?(status.to_s)
|
23
|
+
end
|
24
|
+
|
25
|
+
def to_s() read.join(SEPARATOR) end
|
26
|
+
def reset() @path.truncate(0) end
|
27
|
+
def write(current)
|
28
|
+
last_status = last
|
29
|
+
@path.open('w') { |file| file << [last_status, current].compact.join(SEPARATOR); file.close }
|
30
|
+
end
|
31
|
+
|
32
|
+
def last() read.last end
|
33
|
+
def read() @path.read.to_s.split(SEPARATOR) end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
require 'open-uri'
|
2
|
+
require 'fog'
|
3
|
+
|
4
|
+
module EY
|
5
|
+
module Stonith
|
6
|
+
class LocalMaster < AbstractMaster
|
7
|
+
def initialize(notify_uri, cloud_credentials)
|
8
|
+
@notify_uri, @cloud_credentials = notify_uri, cloud_credentials
|
9
|
+
end
|
10
|
+
alias hostname local_hostname
|
11
|
+
|
12
|
+
def takeover!(master_instance_id, master_ip)
|
13
|
+
address = AddressStealer.new(master_instance_id, master_ip, @cloud_credentials)
|
14
|
+
address.associate(instance_id)
|
15
|
+
AwsmNotifier.new(instance_id, @notify_uri, @cloud_credentials)
|
16
|
+
data(address.ip)
|
17
|
+
end
|
18
|
+
|
19
|
+
def data(ip = public_ip)
|
20
|
+
Data.new(hostname, instance_id, ip)
|
21
|
+
end
|
22
|
+
|
23
|
+
private
|
24
|
+
|
25
|
+
def instance_id() @instance_id ||= EY::Stonith.meta_data["instance-id"] end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
require 'ey_stonith'
|
2
|
+
require 'eventmachine'
|
3
|
+
require 'em-http'
|
4
|
+
|
5
|
+
module EY
|
6
|
+
module Stonith
|
7
|
+
class Master < AbstractMaster
|
8
|
+
def initialize(database, hostname_from_dna)
|
9
|
+
@database, @hostname_from_dna = database, hostname_from_dna
|
10
|
+
end
|
11
|
+
|
12
|
+
def check(good, bad)
|
13
|
+
@database.with_data do |data|
|
14
|
+
http = EM::HttpRequest.new("http://#{data.hostname}/haproxy/monitor").get :timeout => 10
|
15
|
+
http.callback {
|
16
|
+
unless http.response_header.status == 200
|
17
|
+
bad.call data.key
|
18
|
+
else
|
19
|
+
good.call data.key
|
20
|
+
end
|
21
|
+
}
|
22
|
+
http.errback { bad.call data.key }
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def local?() hostname == local_hostname end
|
27
|
+
def hostname() hostname_from_db || hostname_from_dna end
|
28
|
+
def with_locked_data(&block) @database.with_locked_data(&block) end
|
29
|
+
def update(data) @database.set(data) end
|
30
|
+
|
31
|
+
private
|
32
|
+
|
33
|
+
attr_reader :hostname_from_dna
|
34
|
+
def hostname_from_db() @database.with_data { |data| data.hostname } end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
module EY
|
2
|
+
module Stonith
|
3
|
+
class Slave
|
4
|
+
def initialize(box, heartbeat)
|
5
|
+
@box, @heartbeat = box, heartbeat
|
6
|
+
end
|
7
|
+
|
8
|
+
def monitor!(master)
|
9
|
+
return if @checking
|
10
|
+
Stonith.logger.info "Monitoring started"
|
11
|
+
@check_recorder = CheckRecorder.new
|
12
|
+
@checking = EM.add_periodic_timer(@heartbeat) { check(master) }
|
13
|
+
end
|
14
|
+
|
15
|
+
def stop!
|
16
|
+
if @checking
|
17
|
+
Stonith.logger.info "Stopped: Not monitoring until SIGUSR1 received"
|
18
|
+
@checking.cancel
|
19
|
+
@checking = nil
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
private
|
24
|
+
|
25
|
+
def check(master)
|
26
|
+
good = lambda { |master_key| @check_recorder.good_check!(master_key) }
|
27
|
+
bad = lambda { |master_key|
|
28
|
+
@check_recorder.bad_check!(master_key)
|
29
|
+
takeover! if @check_recorder.limit_exceeded?
|
30
|
+
}
|
31
|
+
master.check(good, bad)
|
32
|
+
end
|
33
|
+
|
34
|
+
def takeover!
|
35
|
+
Stonith.logger.info("Trying to grab the lock for takeover")
|
36
|
+
stop!
|
37
|
+
@box.try_takeover(@check_recorder.method(:checking_key?))
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
data/lib/ey_stonith.rb
ADDED
@@ -0,0 +1,34 @@
|
|
1
|
+
require 'logger'
|
2
|
+
|
3
|
+
$LOAD_PATH.unshift(File.expand_path('..', __FILE__))
|
4
|
+
|
5
|
+
module EY
|
6
|
+
module Stonith
|
7
|
+
class Error < StandardError; end
|
8
|
+
|
9
|
+
autoload :AbstractMaster, 'ey_stonith/abstract_master'
|
10
|
+
autoload :AwsmNotifier, 'ey_stonith/awsm_notifier'
|
11
|
+
autoload :AddressStealer, 'ey_stonith/address_stealer'
|
12
|
+
autoload :Box, 'ey_stonith/box'
|
13
|
+
autoload :CheckRecorder, 'ey_stonith/check_recorder'
|
14
|
+
autoload :CLI, 'ey_stonith/cli'
|
15
|
+
autoload :Config, 'ey_stonith/config'
|
16
|
+
autoload :Data, 'ey_stonith/data'
|
17
|
+
autoload :Database, 'ey_stonith/database'
|
18
|
+
autoload :History, 'ey_stonith/history'
|
19
|
+
autoload :LocalMaster, 'ey_stonith/local_master'
|
20
|
+
autoload :Master, 'ey_stonith/master'
|
21
|
+
autoload :MetaData, 'ey_stonith/meta_data'
|
22
|
+
autoload :Slave, 'ey_stonith/slave'
|
23
|
+
|
24
|
+
@@logger = Logger.new(STDOUT)
|
25
|
+
@@logger.level = Logger::INFO
|
26
|
+
def self.logger() @@logger end
|
27
|
+
|
28
|
+
def self.meta_data() @@meta_data end
|
29
|
+
def self.meta_data=(meta) @@meta_data = meta end
|
30
|
+
def self.reset_meta_data() @@meta_data = MetaData end
|
31
|
+
reset_meta_data
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
metadata
ADDED
@@ -0,0 +1,155 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: ey_stonith
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
prerelease: false
|
5
|
+
segments:
|
6
|
+
- 0
|
7
|
+
- 1
|
8
|
+
- 0
|
9
|
+
version: 0.1.0
|
10
|
+
platform: ruby
|
11
|
+
authors:
|
12
|
+
- Ezra Zygmuntowicz
|
13
|
+
- Larry Diehl
|
14
|
+
- Martin Emde
|
15
|
+
autorequire:
|
16
|
+
bindir: bin
|
17
|
+
cert_chain: []
|
18
|
+
|
19
|
+
date: 2010-03-22 00:00:00 -07:00
|
20
|
+
default_executable:
|
21
|
+
dependencies:
|
22
|
+
- !ruby/object:Gem::Dependency
|
23
|
+
name: json
|
24
|
+
prerelease: false
|
25
|
+
requirement: &id001 !ruby/object:Gem::Requirement
|
26
|
+
requirements:
|
27
|
+
- - ">="
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
segments:
|
30
|
+
- 0
|
31
|
+
version: "0"
|
32
|
+
type: :runtime
|
33
|
+
version_requirements: *id001
|
34
|
+
- !ruby/object:Gem::Dependency
|
35
|
+
name: fog
|
36
|
+
prerelease: false
|
37
|
+
requirement: &id002 !ruby/object:Gem::Requirement
|
38
|
+
requirements:
|
39
|
+
- - "="
|
40
|
+
- !ruby/object:Gem::Version
|
41
|
+
segments:
|
42
|
+
- 0
|
43
|
+
- 0
|
44
|
+
- 56
|
45
|
+
version: 0.0.56
|
46
|
+
type: :runtime
|
47
|
+
version_requirements: *id002
|
48
|
+
- !ruby/object:Gem::Dependency
|
49
|
+
name: eventmachine
|
50
|
+
prerelease: false
|
51
|
+
requirement: &id003 !ruby/object:Gem::Requirement
|
52
|
+
requirements:
|
53
|
+
- - ">="
|
54
|
+
- !ruby/object:Gem::Version
|
55
|
+
segments:
|
56
|
+
- 0
|
57
|
+
version: "0"
|
58
|
+
type: :runtime
|
59
|
+
version_requirements: *id003
|
60
|
+
- !ruby/object:Gem::Dependency
|
61
|
+
name: em-http-request
|
62
|
+
prerelease: false
|
63
|
+
requirement: &id004 !ruby/object:Gem::Requirement
|
64
|
+
requirements:
|
65
|
+
- - ~>
|
66
|
+
- !ruby/object:Gem::Version
|
67
|
+
segments:
|
68
|
+
- 0
|
69
|
+
- 2
|
70
|
+
- 7
|
71
|
+
version: 0.2.7
|
72
|
+
type: :runtime
|
73
|
+
version_requirements: *id004
|
74
|
+
- !ruby/object:Gem::Dependency
|
75
|
+
name: redis
|
76
|
+
prerelease: false
|
77
|
+
requirement: &id005 !ruby/object:Gem::Requirement
|
78
|
+
requirements:
|
79
|
+
- - ~>
|
80
|
+
- !ruby/object:Gem::Version
|
81
|
+
segments:
|
82
|
+
- 0
|
83
|
+
- 1
|
84
|
+
- 2
|
85
|
+
version: 0.1.2
|
86
|
+
type: :runtime
|
87
|
+
version_requirements: *id005
|
88
|
+
description: Shoot The Other Node In The Head
|
89
|
+
email: awsmdev@engineyard.com
|
90
|
+
executables:
|
91
|
+
- ey-monitor
|
92
|
+
- ey-monitor-stop
|
93
|
+
- ey-monitor-status
|
94
|
+
- ey-monitor-resume
|
95
|
+
- ey-monitor-reset
|
96
|
+
extensions: []
|
97
|
+
|
98
|
+
extra_rdoc_files:
|
99
|
+
- README.rdoc
|
100
|
+
- LICENSE
|
101
|
+
files:
|
102
|
+
- lib/ey_stonith/abstract_master.rb
|
103
|
+
- lib/ey_stonith/address_stealer.rb
|
104
|
+
- lib/ey_stonith/awsm_notifier.rb
|
105
|
+
- lib/ey_stonith/box.rb
|
106
|
+
- lib/ey_stonith/check_recorder.rb
|
107
|
+
- lib/ey_stonith/cli.rb
|
108
|
+
- lib/ey_stonith/config.rb
|
109
|
+
- lib/ey_stonith/data.rb
|
110
|
+
- lib/ey_stonith/database.rb
|
111
|
+
- lib/ey_stonith/history.rb
|
112
|
+
- lib/ey_stonith/local_master.rb
|
113
|
+
- lib/ey_stonith/master.rb
|
114
|
+
- lib/ey_stonith/meta_data.rb
|
115
|
+
- lib/ey_stonith/slave.rb
|
116
|
+
- lib/ey_stonith.rb
|
117
|
+
- bin/ey-monitor
|
118
|
+
- bin/ey-monitor-reset
|
119
|
+
- bin/ey-monitor-resume
|
120
|
+
- bin/ey-monitor-status
|
121
|
+
- bin/ey-monitor-stop
|
122
|
+
- README.rdoc
|
123
|
+
- LICENSE
|
124
|
+
has_rdoc: true
|
125
|
+
homepage: http://engineyard.com/cloud
|
126
|
+
licenses: []
|
127
|
+
|
128
|
+
post_install_message:
|
129
|
+
rdoc_options: []
|
130
|
+
|
131
|
+
require_paths:
|
132
|
+
- lib
|
133
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
134
|
+
requirements:
|
135
|
+
- - ">="
|
136
|
+
- !ruby/object:Gem::Version
|
137
|
+
segments:
|
138
|
+
- 0
|
139
|
+
version: "0"
|
140
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
141
|
+
requirements:
|
142
|
+
- - ">="
|
143
|
+
- !ruby/object:Gem::Version
|
144
|
+
segments:
|
145
|
+
- 0
|
146
|
+
version: "0"
|
147
|
+
requirements: []
|
148
|
+
|
149
|
+
rubyforge_project:
|
150
|
+
rubygems_version: 1.3.6
|
151
|
+
signing_key:
|
152
|
+
specification_version: 3
|
153
|
+
summary: Shoot The Other Node In The Head
|
154
|
+
test_files: []
|
155
|
+
|