flapjack 0.4.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/LICENCE +20 -0
- data/README.md +205 -0
- data/Rakefile +84 -0
- data/TODO.md +26 -0
- data/bin/flapjack-notifier +46 -0
- data/bin/flapjack-notifier-manager +45 -0
- data/bin/flapjack-stats +22 -0
- data/bin/flapjack-worker +36 -0
- data/bin/flapjack-worker-manager +35 -0
- data/bin/install-flapjack-systemwide +58 -0
- data/doc/CONFIGURING.md +38 -0
- data/doc/DEVELOPING.md +35 -0
- data/doc/INSTALL.md +64 -0
- data/etc/default/flapjack-notifier +15 -0
- data/etc/default/flapjack-workers +17 -0
- data/etc/flapjack/flapjack-notifier.yaml.example +8 -0
- data/etc/flapjack/recipients.yaml.example +10 -0
- data/etc/init.d/flapjack-notifier +47 -0
- data/etc/init.d/flapjack-workers +44 -0
- data/flapjack.gemspec +31 -0
- data/lib/flapjack/checks/http_content +15 -0
- data/lib/flapjack/cli/notifier.rb +245 -0
- data/lib/flapjack/cli/notifier_manager.rb +86 -0
- data/lib/flapjack/cli/worker.rb +136 -0
- data/lib/flapjack/cli/worker_manager.rb +47 -0
- data/lib/flapjack/database.rb +10 -0
- data/lib/flapjack/models/check.rb +92 -0
- data/lib/flapjack/models/check_template.rb +18 -0
- data/lib/flapjack/models/node.rb +16 -0
- data/lib/flapjack/models/related_check.rb +13 -0
- data/lib/flapjack/notifier.rb +35 -0
- data/lib/flapjack/notifiers/mailer/init.rb +3 -0
- data/lib/flapjack/notifiers/mailer/mailer.rb +47 -0
- data/lib/flapjack/notifiers/xmpp/init.rb +3 -0
- data/lib/flapjack/notifiers/xmpp/xmpp.rb +42 -0
- data/lib/flapjack/patches.rb +26 -0
- data/lib/flapjack/result.rb +47 -0
- metadata +205 -0
@@ -0,0 +1,245 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'rubygems'
|
4
|
+
require 'ostruct'
|
5
|
+
require 'optparse'
|
6
|
+
require 'log4r'
|
7
|
+
require 'log4r/outputter/syslogoutputter'
|
8
|
+
require File.join(File.dirname(__FILE__), '..', 'database')
|
9
|
+
require File.join(File.dirname(__FILE__), '..', 'notifier')
|
10
|
+
require File.join(File.dirname(__FILE__), '..', 'result')
|
11
|
+
|
12
|
+
module Flapjack
|
13
|
+
class NotifierOptions
|
14
|
+
def self.parse(args)
|
15
|
+
options = OpenStruct.new
|
16
|
+
opts = OptionParser.new do |opts|
|
17
|
+
# the available command line options
|
18
|
+
opts.on('-b', '--beanstalk HOST', 'location of the beanstalkd') do |host|
|
19
|
+
options.host = host
|
20
|
+
end
|
21
|
+
opts.on('-p', '--port PORT', 'beanstalkd port') do |port|
|
22
|
+
options.port = port.to_i
|
23
|
+
end
|
24
|
+
opts.on('-r', '--recipients FILE', 'recipients file') do |recipients|
|
25
|
+
options.recipients = recipients.to_s
|
26
|
+
end
|
27
|
+
opts.on('-c', '--config FILE', 'config file') do |config|
|
28
|
+
options.config_filename = config.to_s
|
29
|
+
end
|
30
|
+
opts.on('-d', '--database-uri URI', 'location of the checks database') do |db|
|
31
|
+
options.database_uri = db.to_s
|
32
|
+
end
|
33
|
+
opts.on_tail("-h", "--help", "Show this message") do
|
34
|
+
puts opts
|
35
|
+
exit
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
# parse the options
|
40
|
+
begin
|
41
|
+
opts.parse!(args)
|
42
|
+
rescue OptionParser::MissingArgument => e
|
43
|
+
# if an --option is missing it's argument
|
44
|
+
puts e.message.capitalize + "\n\n"
|
45
|
+
puts opts
|
46
|
+
exit 1
|
47
|
+
end
|
48
|
+
|
49
|
+
# default the host + port
|
50
|
+
options.host ||= 'localhost'
|
51
|
+
options.port ||= 11300
|
52
|
+
|
53
|
+
@errors = []
|
54
|
+
# check that recipients file exists
|
55
|
+
if options.recipients
|
56
|
+
unless File.exists?(options.recipients.to_s)
|
57
|
+
@errors << "The specified recipients file doesn't exist!"
|
58
|
+
end
|
59
|
+
else
|
60
|
+
@errors << "You need to specify a recipients file with [-r|--recipients]."
|
61
|
+
end
|
62
|
+
|
63
|
+
# check that config file exists
|
64
|
+
if options.config_filename
|
65
|
+
unless File.exists?(options.config_filename.to_s)
|
66
|
+
@errors << "The specified config file doesn't exist!"
|
67
|
+
end
|
68
|
+
else
|
69
|
+
options.config_filename ||= "/etc/flapjack/flapjack-notifier.yaml"
|
70
|
+
unless File.exists?(options.config_filename.to_s)
|
71
|
+
@errors << "The default config file (#{options.config_filename}) doesn't exist!"
|
72
|
+
@errors << "Set one up, or specify one with [-c|--config]."
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
# if there are errors, print them out and exit
|
77
|
+
if @errors.size > 0
|
78
|
+
puts "Errors:"
|
79
|
+
@errors.each do |error|
|
80
|
+
puts " - #{error}"
|
81
|
+
end
|
82
|
+
puts
|
83
|
+
puts opts
|
84
|
+
exit 2
|
85
|
+
end
|
86
|
+
|
87
|
+
options
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
class NotifierCLI
|
92
|
+
attr_accessor :log, :recipients, :results_queue, :config
|
93
|
+
attr_accessor :notifier, :notifiers
|
94
|
+
attr_accessor :condition
|
95
|
+
|
96
|
+
def initialize(opts={})
|
97
|
+
@log = opts[:logger]
|
98
|
+
@log ||= Log4r::Logger.new("notifier")
|
99
|
+
end
|
100
|
+
|
101
|
+
def setup_loggers
|
102
|
+
@log.add(Log4r::StdoutOutputter.new('notifier'))
|
103
|
+
@log.add(Log4r::SyslogOutputter.new('notifier'))
|
104
|
+
end
|
105
|
+
|
106
|
+
def setup_recipients(opts={})
|
107
|
+
|
108
|
+
if opts[:yaml]
|
109
|
+
yaml = opts[:yaml]
|
110
|
+
else
|
111
|
+
opts[:filename] ||= File.join(Dir.pwd, "recipients.yaml")
|
112
|
+
yaml = YAML::load(File.read(opts[:filename]))
|
113
|
+
end
|
114
|
+
|
115
|
+
# FIXME: add error detection for passing in dumb things
|
116
|
+
|
117
|
+
@recipients = yaml.map do |r|
|
118
|
+
OpenStruct.new(r)
|
119
|
+
end
|
120
|
+
end
|
121
|
+
|
122
|
+
def setup_config(opts={})
|
123
|
+
if opts[:yaml]
|
124
|
+
yaml = opts[:yaml]
|
125
|
+
else
|
126
|
+
opts[:filename] ||= File.join(Dir.pwd, "flapjack-notifier.yaml")
|
127
|
+
yaml = YAML::load(File.read(opts[:filename]))
|
128
|
+
end
|
129
|
+
|
130
|
+
@config = OpenStruct.new(yaml)
|
131
|
+
end
|
132
|
+
|
133
|
+
def setup_database(opts={})
|
134
|
+
uri = (opts[:database_uri] || @config.database_uri)
|
135
|
+
raise ArgumentError, "database URI wasn't specified!" unless uri
|
136
|
+
DataMapper.setup(:default, uri)
|
137
|
+
validate_database
|
138
|
+
end
|
139
|
+
|
140
|
+
def validate_database
|
141
|
+
begin
|
142
|
+
DataMapper.repository(:default).adapter.execute("SELECT 'id' FROM 'checks';")
|
143
|
+
rescue Sqlite3Error
|
144
|
+
@log.warning("The specified database doesn't appear to have any structure!")
|
145
|
+
@log.warning("You need to investigate this.")
|
146
|
+
end
|
147
|
+
# FIXME: add more rescues in here!
|
148
|
+
end
|
149
|
+
|
150
|
+
def initialize_notifiers(opts={})
|
151
|
+
notifiers_directory = opts[:notifiers_directory]
|
152
|
+
notifiers_directory ||= File.expand_path(File.join(File.dirname(__FILE__), '..', 'notifiers'))
|
153
|
+
|
154
|
+
raise ArgumentError, "notifiers directory doesn't exist!" unless File.exists?(notifiers_directory)
|
155
|
+
|
156
|
+
@notifiers = []
|
157
|
+
|
158
|
+
@config.notifiers.each_pair do |notifier, config|
|
159
|
+
filename = File.join(notifiers_directory, notifier.to_s, 'init')
|
160
|
+
if File.exists?(filename + '.rb')
|
161
|
+
@log.debug("Loading the #{notifier.to_s.capitalize} notifier")
|
162
|
+
require filename
|
163
|
+
config.merge!(:logger => @log, :website_uri => @config.website_uri)
|
164
|
+
@notifiers << Flapjack::Notifiers.const_get("#{notifier.to_s.capitalize}").new(config)
|
165
|
+
else
|
166
|
+
@log.warning("Flapjack::Notifiers::#{notifier.to_s.capitalize} doesn't exist!")
|
167
|
+
end
|
168
|
+
end
|
169
|
+
|
170
|
+
@notifiers
|
171
|
+
end
|
172
|
+
|
173
|
+
# Sets up notifier to do the grunt work of notifying people when checks
|
174
|
+
# return badly.
|
175
|
+
#
|
176
|
+
# Accepts a list of recipients (:recipients) and a logger (:logger) as
|
177
|
+
# arguments. If neither of these are specified, it will default to an
|
178
|
+
# existing list of recipients and the current logger.
|
179
|
+
#
|
180
|
+
# Sets up and returns @notifier, an instance of Flapjack::Notifier
|
181
|
+
def setup_notifier(opts={})
|
182
|
+
recipients = (opts[:recipients] || @recipients)
|
183
|
+
log = (opts[:logger] || @log)
|
184
|
+
initialize_notifiers
|
185
|
+
notifiers = @notifiers # should we accept a list of notifiers?
|
186
|
+
@notifier = Flapjack::Notifier.new(:logger => log,
|
187
|
+
:notifiers => notifiers,
|
188
|
+
:recipients => recipients)
|
189
|
+
end
|
190
|
+
|
191
|
+
def process_loop
|
192
|
+
@log.info("Processing results...")
|
193
|
+
loop do
|
194
|
+
process_result
|
195
|
+
end
|
196
|
+
end
|
197
|
+
|
198
|
+
# FIXME: we're doing a lookup twice - optimise out
|
199
|
+
def save_result(result)
|
200
|
+
if check = Check.get(result.id)
|
201
|
+
check.status = result.retval
|
202
|
+
check.save
|
203
|
+
else
|
204
|
+
@log.error("Got result for check #{result.id}, but it doesn't exist!")
|
205
|
+
end
|
206
|
+
end
|
207
|
+
|
208
|
+
# FIXME: we're doing a lookup twice - optimise out
|
209
|
+
def any_parents_warning_or_critical?(result)
|
210
|
+
if check = Check.get(result.id)
|
211
|
+
check.worst_parent_status > 0 ? true : false
|
212
|
+
else
|
213
|
+
@log.error("Got result for check #{result.id}, but it doesn't exist!")
|
214
|
+
end
|
215
|
+
end
|
216
|
+
|
217
|
+
def process_result
|
218
|
+
@log.debug("Waiting for new result...")
|
219
|
+
result_job = @results_queue.reserve # blocks until a job is reserved
|
220
|
+
result = Flapjack::Result.new(YAML::load(result_job.body))
|
221
|
+
|
222
|
+
@log.info("Processing result for check '#{result.id}'")
|
223
|
+
if result.warning? || result.critical?
|
224
|
+
if any_parents_warning_or_critical?(result)
|
225
|
+
@log.info("Not notifying on check '#{result.id}' as parent is failing.")
|
226
|
+
else
|
227
|
+
@log.info("Notifying on check '#{result.id}'")
|
228
|
+
@notifier.notify!(result)
|
229
|
+
end
|
230
|
+
|
231
|
+
@log.info("Creating event for check '#{result.id}'")
|
232
|
+
event = Event.new(:check_id => result.id)
|
233
|
+
raise unless event.save
|
234
|
+
end
|
235
|
+
|
236
|
+
@log.info("Storing status of check in database")
|
237
|
+
save_result(result)
|
238
|
+
|
239
|
+
@log.debug("Deleting result for check '#{result.id}' from queue")
|
240
|
+
result_job.delete
|
241
|
+
end
|
242
|
+
|
243
|
+
end
|
244
|
+
|
245
|
+
end
|
@@ -0,0 +1,86 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'rubygems'
|
4
|
+
require 'ostruct'
|
5
|
+
require 'optparse'
|
6
|
+
|
7
|
+
module Flapjack
|
8
|
+
class NotifierManagerOptions
|
9
|
+
def self.parse(args)
|
10
|
+
options = OpenStruct.new
|
11
|
+
opts = OptionParser.new do |opts|
|
12
|
+
opts.banner = "Usage: flapjack-notifier-manager <command> [options]"
|
13
|
+
opts.separator " "
|
14
|
+
opts.separator " where <command> is one of:"
|
15
|
+
opts.separator " start start a worker"
|
16
|
+
opts.separator " stop stop all workers"
|
17
|
+
opts.separator " restart restart workers"
|
18
|
+
opts.separator " "
|
19
|
+
opts.separator " and [options] are:"
|
20
|
+
|
21
|
+
opts.on('-b', '--beanstalk HOST', 'location of the beanstalkd') do |host|
|
22
|
+
options.host = host
|
23
|
+
end
|
24
|
+
opts.on('-p', '--port PORT', 'beanstalkd port') do |port|
|
25
|
+
options.port = port.to_s
|
26
|
+
end
|
27
|
+
opts.on('-r', '--recipients FILE', 'recipients file') do |recipients|
|
28
|
+
options.recipients = File.expand_path(recipients.to_s)
|
29
|
+
end
|
30
|
+
opts.on('-c', '--config FILE', 'config file') do |config|
|
31
|
+
options.config_filename = config.to_s
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
begin
|
36
|
+
opts.parse!(args)
|
37
|
+
rescue => e
|
38
|
+
puts e.message.capitalize + "\n\n"
|
39
|
+
puts opts
|
40
|
+
exit 1
|
41
|
+
end
|
42
|
+
|
43
|
+
# defaults
|
44
|
+
options.host ||= "localhost"
|
45
|
+
options.port ||= 11300
|
46
|
+
|
47
|
+
@errors = []
|
48
|
+
|
49
|
+
unless ARGV[0] == "stop"
|
50
|
+
if options.recipients
|
51
|
+
unless File.exists?(options.recipients)
|
52
|
+
@errors << "The specified recipients file doesn't exist!"
|
53
|
+
end
|
54
|
+
else
|
55
|
+
@errors << "You must specify a recipients file!"
|
56
|
+
end
|
57
|
+
|
58
|
+
if options.config_filename
|
59
|
+
unless File.exists?(options.config_filename)
|
60
|
+
@errors << "The specified config file dosen't exist!"
|
61
|
+
end
|
62
|
+
else
|
63
|
+
@errors << "You must specify a config file!"
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
if @errors.size > 0
|
68
|
+
puts "Errors:"
|
69
|
+
@errors.each do |error|
|
70
|
+
puts " - #{error}"
|
71
|
+
end
|
72
|
+
puts
|
73
|
+
puts opts
|
74
|
+
exit 2
|
75
|
+
end
|
76
|
+
|
77
|
+
unless %w(start stop restart).include?(args[0])
|
78
|
+
puts opts
|
79
|
+
exit 1
|
80
|
+
end
|
81
|
+
|
82
|
+
options
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
end
|
@@ -0,0 +1,136 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'rubygems'
|
4
|
+
require 'beanstalk-client'
|
5
|
+
require 'ostruct'
|
6
|
+
require 'optparse'
|
7
|
+
require 'log4r'
|
8
|
+
require 'log4r/outputter/syslogoutputter'
|
9
|
+
require 'flapjack/result'
|
10
|
+
require 'flapjack/patches'
|
11
|
+
|
12
|
+
module Flapjack
|
13
|
+
class WorkerOptions
|
14
|
+
def self.parse(args)
|
15
|
+
options = OpenStruct.new
|
16
|
+
opts = OptionParser.new do |opts|
|
17
|
+
# the available command line options
|
18
|
+
opts.on('-b', '--beanstalk HOST', 'location of the beanstalkd') do |host|
|
19
|
+
options.host = host
|
20
|
+
end
|
21
|
+
opts.on('-p', '--port PORT', 'beanstalkd port') do |port|
|
22
|
+
options.port = port.to_i
|
23
|
+
end
|
24
|
+
opts.on('-c', '--checks-directory DIR', 'sandboxed check directory') do |dir|
|
25
|
+
options.checks_directory = dir.to_s
|
26
|
+
end
|
27
|
+
opts.on_tail("-h", "--help", "Show this message") do
|
28
|
+
puts opts
|
29
|
+
exit
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
# parse the options
|
34
|
+
begin
|
35
|
+
opts.parse!(args)
|
36
|
+
rescue OptionParser::MissingArgument => e
|
37
|
+
# if an --option is missing it's argument
|
38
|
+
puts e.message.capitalize + "\n\n"
|
39
|
+
puts opts
|
40
|
+
exit 1
|
41
|
+
end
|
42
|
+
|
43
|
+
# default the port
|
44
|
+
options.host ||= 'localhost'
|
45
|
+
options.port ||= 11300
|
46
|
+
options.checks_directory ||= File.join(File.dirname(__FILE__), '..', 'checks')
|
47
|
+
|
48
|
+
options
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
class Worker
|
53
|
+
|
54
|
+
attr_accessor :jobs, :results, :log
|
55
|
+
|
56
|
+
def initialize(opts={})
|
57
|
+
@jobs = Beanstalk::Pool.new(["#{opts[:host]}:#{opts[:port]}"], 'jobs')
|
58
|
+
@results = Beanstalk::Pool.new(["#{opts[:host]}:#{opts[:port]}"], 'results')
|
59
|
+
@sandbox = (opts[:check_directory] || File.expand_path(File.join(File.dirname(__FILE__), '..', 'checks')))
|
60
|
+
|
61
|
+
if opts[:logger]
|
62
|
+
@log = opts[:logger]
|
63
|
+
else
|
64
|
+
@log = Log4r::Logger.new('worker')
|
65
|
+
@log.add(Log4r::StdoutOutputter.new('worker'))
|
66
|
+
@log.add(Log4r::SyslogOutputter.new('worker'))
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
def process_loop
|
71
|
+
@log.info("Booting main loop...")
|
72
|
+
loop do
|
73
|
+
process_check
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
def process_check
|
78
|
+
# get next check off the beanstalk
|
79
|
+
job, check = get_check()
|
80
|
+
|
81
|
+
# do the actual check
|
82
|
+
result, retval = perform_check(check.command)
|
83
|
+
|
84
|
+
# report the results of the check
|
85
|
+
report_check(:result => result, :retval => retval, :check => check)
|
86
|
+
|
87
|
+
# create another job for the check, delete current job
|
88
|
+
cleanup_job(:job => job, :check => check)
|
89
|
+
end
|
90
|
+
|
91
|
+
def perform_check(cmd)
|
92
|
+
command = "sh -c '#{@sandbox}/#{cmd}'"
|
93
|
+
@log.debug("Executing check: \"#{command}\"")
|
94
|
+
result = `#{command}`
|
95
|
+
retval = $?.exitstatus
|
96
|
+
|
97
|
+
return result, retval
|
98
|
+
end
|
99
|
+
|
100
|
+
def report_check(opts={})
|
101
|
+
raise ArgumentError unless (opts[:result] && opts[:retval] && opts[:check])
|
102
|
+
|
103
|
+
@log.debug "Reporting results for check id #{opts[:check].id}."
|
104
|
+
@results.yput({:id => opts[:check].id,
|
105
|
+
:output => opts[:result],
|
106
|
+
:retval => opts[:retval].to_i})
|
107
|
+
end
|
108
|
+
|
109
|
+
def cleanup_job(opts={})
|
110
|
+
raise ArgumentError unless (opts[:job] && opts[:check])
|
111
|
+
|
112
|
+
# add job back onto stack
|
113
|
+
@log.debug("Putting check back onto beanstalk.")
|
114
|
+
@jobs.yput(opts[:check].to_h, 65536, opts[:check].frequency)
|
115
|
+
|
116
|
+
# FIXME: what happens when power goes out here?
|
117
|
+
|
118
|
+
# once we're done, clean up
|
119
|
+
@log.debug("Deleting job.")
|
120
|
+
opts[:job].delete
|
121
|
+
end
|
122
|
+
|
123
|
+
def get_check
|
124
|
+
@log.debug("Waiting for check...")
|
125
|
+
job = @jobs.reserve
|
126
|
+
# FIXME: maybe wrap Result as Job now that Check is reserved?
|
127
|
+
check = Result.new(YAML::load(job.body))
|
128
|
+
@log.info("Got check with id #{check.id}")
|
129
|
+
|
130
|
+
return job, check
|
131
|
+
end
|
132
|
+
|
133
|
+
end
|
134
|
+
|
135
|
+
end
|
136
|
+
|