flapjack 0.7.18 → 0.7.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/CHANGELOG.md +7 -0
- data/bin/flapjack +3 -0
- data/bin/flapjack-nagios-receiver +4 -1
- data/bin/flapjack-netsaint-parser +2 -1
- data/bin/flapjack-populator +6 -3
- data/bin/receive-events +3 -1
- data/bin/simulate-failed-check +2 -1
- data/etc/flapjack_config.yaml.example +20 -0
- data/features/events.feature +1 -1
- data/features/events_check_names.feature +1 -1
- data/features/notification_rules.feature +1 -1
- data/features/notifications.feature +1 -1
- data/features/steps/events_steps.rb +18 -17
- data/features/steps/flapjack-netsaint-parser_steps.rb +1 -2
- data/features/steps/notifications_steps.rb +14 -1
- data/features/support/env.rb +27 -10
- data/flapjack.gemspec +1 -3
- data/lib/flapjack/coordinator.rb +30 -20
- data/lib/flapjack/data/contact.rb +3 -2
- data/lib/flapjack/data/entity.rb +3 -3
- data/lib/flapjack/data/entity_check.rb +116 -43
- data/lib/flapjack/data/event.rb +10 -10
- data/lib/flapjack/data/message.rb +3 -6
- data/lib/flapjack/data/notification.rb +122 -57
- data/lib/flapjack/data/notification_rule.rb +11 -11
- data/lib/flapjack/filters/acknowledgement.rb +2 -2
- data/lib/flapjack/filters/ok.rb +1 -1
- data/lib/flapjack/gateways/api/entity_check_presenter.rb +1 -0
- data/lib/flapjack/gateways/api/entity_methods.rb +4 -6
- data/lib/flapjack/gateways/api/rack/json_params_parser.rb +1 -1
- data/lib/flapjack/gateways/email.rb +3 -5
- data/lib/flapjack/gateways/email/{alert.html.haml → alert.html.erb} +0 -0
- data/lib/flapjack/gateways/jabber.rb +66 -35
- data/lib/flapjack/gateways/oobetet.rb +5 -7
- data/lib/flapjack/gateways/pagerduty.rb +7 -7
- data/lib/flapjack/gateways/web.rb +101 -41
- data/lib/flapjack/gateways/web/public/css/flapjack.css +1 -1
- data/lib/flapjack/gateways/web/views/{_css.haml → _css.html.erb} +2 -1
- data/lib/flapjack/gateways/web/views/_foot.html.erb +3 -0
- data/lib/flapjack/gateways/web/views/_head.html.erb +4 -0
- data/lib/flapjack/gateways/web/views/_nav.html.erb +9 -0
- data/lib/flapjack/gateways/web/views/check.html.erb +204 -0
- data/lib/flapjack/gateways/web/views/checks.html.erb +77 -0
- data/lib/flapjack/gateways/web/views/contact.html.erb +114 -0
- data/lib/flapjack/gateways/web/views/contacts.html.erb +42 -0
- data/lib/flapjack/gateways/web/views/entities.html.erb +39 -0
- data/lib/flapjack/gateways/web/views/entity.html.erb +67 -0
- data/lib/flapjack/gateways/web/views/index.html.erb +27 -0
- data/lib/flapjack/gateways/web/views/self_stats.html.erb +97 -0
- data/lib/flapjack/logger.rb +71 -23
- data/lib/flapjack/notifier.rb +157 -0
- data/lib/flapjack/patches.rb +1 -41
- data/lib/flapjack/pikelet.rb +4 -2
- data/lib/flapjack/{executive.rb → processor.rb} +32 -145
- data/lib/flapjack/version.rb +1 -1
- data/spec/lib/flapjack/coordinator_spec.rb +134 -71
- data/spec/lib/flapjack/data/contact_spec.rb +1 -0
- data/spec/lib/flapjack/data/entity_check_spec.rb +146 -30
- data/spec/lib/flapjack/data/entity_spec.rb +4 -4
- data/spec/lib/flapjack/data/event_spec.rb +4 -4
- data/spec/lib/flapjack/data/message_spec.rb +2 -3
- data/spec/lib/flapjack/data/notification_spec.rb +13 -19
- data/spec/lib/flapjack/gateways/api/entity_methods_spec.rb +2 -2
- data/spec/lib/flapjack/gateways/jabber_spec.rb +34 -0
- data/spec/lib/flapjack/gateways/pagerduty_spec.rb +0 -2
- data/spec/lib/flapjack/gateways/web/views/{check.haml_spec.rb → check.html.erb_spec.rb} +2 -2
- data/spec/lib/flapjack/gateways/web/views/{contact.haml_spec.rb → contact.html.erb_spec.rb} +3 -3
- data/spec/lib/flapjack/gateways/web/views/index.html.erb_spec.rb +14 -0
- data/spec/lib/flapjack/gateways/web_spec.rb +20 -8
- data/spec/lib/flapjack/logger_spec.rb +30 -28
- data/spec/lib/flapjack/notifier_spec.rb +6 -0
- data/spec/lib/flapjack/pikelet_spec.rb +8 -8
- data/spec/lib/flapjack/{executive_spec.rb → processor_spec.rb} +4 -4
- data/spec/spec_helper.rb +1 -13
- data/spec/support/erb_view_helper.rb +23 -0
- data/tasks/profile.rake +1 -1
- data/tmp/acknowledge.rb +3 -1
- data/tmp/create_event_ok.rb +3 -1
- data/tmp/create_event_unknown.rb +3 -1
- data/tmp/create_events_failure.rb +3 -1
- data/tmp/create_events_ok.rb +3 -1
- data/tmp/create_events_ok_fail_ack_ok.rb +3 -1
- data/tmp/create_events_ok_failure.rb +3 -1
- data/tmp/create_events_ok_failure_ack.rb +3 -1
- data/tmp/test_json_post.rb +5 -3
- data/tmp/test_notification_rules_api.rb +5 -3
- metadata +32 -61
- data/lib/flapjack/gateways/web/views/_foot.haml +0 -8
- data/lib/flapjack/gateways/web/views/_head.haml +0 -10
- data/lib/flapjack/gateways/web/views/_nav.haml +0 -14
- data/lib/flapjack/gateways/web/views/check.haml +0 -191
- data/lib/flapjack/gateways/web/views/checks.haml +0 -49
- data/lib/flapjack/gateways/web/views/contact.haml +0 -85
- data/lib/flapjack/gateways/web/views/contacts.haml +0 -30
- data/lib/flapjack/gateways/web/views/entities.haml +0 -28
- data/lib/flapjack/gateways/web/views/entity.haml +0 -50
- data/lib/flapjack/gateways/web/views/index.haml +0 -32
- data/lib/flapjack/gateways/web/views/self_stats.haml +0 -70
- data/spec/lib/flapjack/gateways/web/views/index.haml_spec.rb +0 -13
- data/spec/support/haml_view_helper.rb +0 -15
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
#!/usr/bin/env ruby
|
|
2
2
|
|
|
3
|
-
require '
|
|
3
|
+
require 'oj'
|
|
4
4
|
require 'active_support/time'
|
|
5
5
|
require 'ice_cube'
|
|
6
6
|
require 'flapjack/utility'
|
|
@@ -129,11 +129,11 @@ module Flapjack
|
|
|
129
129
|
json_rule_data = {
|
|
130
130
|
:id => rule_data[:id].to_s,
|
|
131
131
|
:contact_id => rule_data[:contact_id].to_s,
|
|
132
|
-
:entities =>
|
|
133
|
-
:entity_tags =>
|
|
134
|
-
:time_restrictions =>
|
|
135
|
-
:warning_media =>
|
|
136
|
-
:critical_media =>
|
|
132
|
+
:entities => Oj.dump(rule_data[:entities]),
|
|
133
|
+
:entity_tags => Oj.dump(rule_data[:entity_tags]),
|
|
134
|
+
:time_restrictions => Oj.dump(rule_data[:time_restrictions]),
|
|
135
|
+
:warning_media => Oj.dump(rule_data[:warning_media]),
|
|
136
|
+
:critical_media => Oj.dump(rule_data[:critical_media]),
|
|
137
137
|
:warning_blackhole => rule_data[:warning_blackhole],
|
|
138
138
|
:critical_blackhole => rule_data[:critical_blackhole],
|
|
139
139
|
}
|
|
@@ -284,11 +284,11 @@ module Flapjack
|
|
|
284
284
|
rule_data = @redis.hgetall("notification_rule:#{@id}")
|
|
285
285
|
|
|
286
286
|
@contact_id = rule_data['contact_id']
|
|
287
|
-
@entity_tags =
|
|
288
|
-
@entities =
|
|
289
|
-
@time_restrictions =
|
|
290
|
-
@warning_media =
|
|
291
|
-
@critical_media =
|
|
287
|
+
@entity_tags = Oj.load(rule_data['entity_tags'] || '')
|
|
288
|
+
@entities = Oj.load(rule_data['entities'] || '')
|
|
289
|
+
@time_restrictions = Oj.load(rule_data['time_restrictions'] || '')
|
|
290
|
+
@warning_media = Oj.load(rule_data['warning_media'] || '')
|
|
291
|
+
@critical_media = Oj.load(rule_data['critical_media'] || '')
|
|
292
292
|
@warning_blackhole = ((rule_data['warning_blackhole'] || 'false').downcase == 'true')
|
|
293
293
|
@critical_blackhole = ((rule_data['critical_blackhole'] || 'false').downcase == 'true')
|
|
294
294
|
end
|
|
@@ -21,8 +21,8 @@ module Flapjack
|
|
|
21
21
|
if ec.nil?
|
|
22
22
|
@logger.error "Filter: Acknowledgement: unknown entity for event '#{event.id}'"
|
|
23
23
|
else
|
|
24
|
-
ec.create_unscheduled_maintenance(
|
|
25
|
-
|
|
24
|
+
ec.create_unscheduled_maintenance(timestamp,
|
|
25
|
+
(event.duration || (4 * 60 * 60)),
|
|
26
26
|
:summary => event.summary)
|
|
27
27
|
message = "unscheduled maintenance created for #{event.id}"
|
|
28
28
|
end
|
data/lib/flapjack/filters/ok.rb
CHANGED
|
@@ -21,6 +21,7 @@ module Flapjack
|
|
|
21
21
|
def status
|
|
22
22
|
{'name' => @entity_check.check,
|
|
23
23
|
'state' => @entity_check.state,
|
|
24
|
+
'enabled' => @entity_check.enabled?,
|
|
24
25
|
'summary' => @entity_check.summary,
|
|
25
26
|
'details' => @entity_check.details,
|
|
26
27
|
'in_unscheduled_maintenance' => @entity_check.in_unscheduled_maintenance?,
|
|
@@ -222,8 +222,8 @@ module Flapjack
|
|
|
222
222
|
halt( err(403, "start time must be provided") ) unless start_time
|
|
223
223
|
|
|
224
224
|
act_proc = proc {|entity_check|
|
|
225
|
-
entity_check.create_scheduled_maintenance(
|
|
226
|
-
|
|
225
|
+
entity_check.create_scheduled_maintenance(start_time,
|
|
226
|
+
params[:duration].to_i, :summary => params[:summary])
|
|
227
227
|
}
|
|
228
228
|
|
|
229
229
|
bulk_api_check_action(entities, checks, act_proc)
|
|
@@ -272,10 +272,8 @@ module Flapjack
|
|
|
272
272
|
opts = {}
|
|
273
273
|
proc {|entity_check| entity_check.end_scheduled_maintenance(start_time.to_i) }
|
|
274
274
|
when 'unscheduled_maintenances'
|
|
275
|
-
end_time = validate_and_parsetime(params[:end_time])
|
|
276
|
-
|
|
277
|
-
opts[:end_time] = end_time.to_i if end_time
|
|
278
|
-
proc {|entity_check| entity_check.end_unscheduled_maintenance(opts) }
|
|
275
|
+
end_time = validate_and_parsetime(params[:end_time]) || Time.now
|
|
276
|
+
proc {|entity_check| entity_check.end_unscheduled_maintenance(end_time.to_i) }
|
|
279
277
|
end
|
|
280
278
|
|
|
281
279
|
bulk_api_check_action(entities, checks, act_proc)
|
|
@@ -9,7 +9,7 @@ module Rack
|
|
|
9
9
|
env['rack.request.form_input'] = env['rack.input']
|
|
10
10
|
data = env['rack.input'].read
|
|
11
11
|
env['rack.input'].rewind
|
|
12
|
-
env['rack.request.form_hash'] = data.empty? ? {} :
|
|
12
|
+
env['rack.request.form_hash'] = data.empty? ? {} : Oj.load(data)
|
|
13
13
|
end
|
|
14
14
|
app.call(env)
|
|
15
15
|
end
|
|
@@ -2,7 +2,6 @@
|
|
|
2
2
|
|
|
3
3
|
require 'mail'
|
|
4
4
|
require 'erb'
|
|
5
|
-
require 'haml'
|
|
6
5
|
require 'socket'
|
|
7
6
|
|
|
8
7
|
require 'em-synchrony'
|
|
@@ -113,10 +112,9 @@ module Flapjack
|
|
|
113
112
|
text_template = ERB.new(File.read(File.dirname(__FILE__) +
|
|
114
113
|
'/email/alert.text.erb'))
|
|
115
114
|
|
|
116
|
-
|
|
117
|
-
'/email/alert.html.
|
|
115
|
+
html_template = ERB.new(File.read(File.dirname(__FILE__) +
|
|
116
|
+
'/email/alert.html.erb'))
|
|
118
117
|
|
|
119
|
-
mail_scope = self
|
|
120
118
|
bnd = binding
|
|
121
119
|
|
|
122
120
|
# this part is the only use of the mail gem -- maybe this can be done
|
|
@@ -133,7 +131,7 @@ module Flapjack
|
|
|
133
131
|
|
|
134
132
|
html_part do
|
|
135
133
|
content_type 'text/html; charset=UTF-8'
|
|
136
|
-
body
|
|
134
|
+
body html_template.result(bnd)
|
|
137
135
|
end
|
|
138
136
|
end
|
|
139
137
|
end
|
|
File without changes
|
|
@@ -10,11 +10,9 @@ require 'em-synchrony'
|
|
|
10
10
|
require 'redis/connection/synchrony'
|
|
11
11
|
require 'redis'
|
|
12
12
|
|
|
13
|
-
require 'chronic_duration'
|
|
14
|
-
|
|
15
13
|
require 'blather/client/client'
|
|
16
|
-
require '
|
|
17
|
-
require '
|
|
14
|
+
require 'chronic_duration'
|
|
15
|
+
require 'oj'
|
|
18
16
|
|
|
19
17
|
require 'flapjack/data/entity_check'
|
|
20
18
|
require 'flapjack/redis_pool'
|
|
@@ -28,11 +26,27 @@ module Flapjack
|
|
|
28
26
|
class Jabber < Blather::Client
|
|
29
27
|
include Flapjack::Utility
|
|
30
28
|
|
|
31
|
-
log = Logger.new(STDOUT)
|
|
32
|
-
|
|
33
|
-
log.level = Logger::INFO
|
|
29
|
+
log = ::Logger.new(STDOUT)
|
|
30
|
+
log.level = ::Logger::INFO
|
|
34
31
|
Blather.logger = log
|
|
35
32
|
|
|
33
|
+
# TODO if we use 'xmpp4r' rather than 'blather', port this to 'rexml'
|
|
34
|
+
class TextHandler < Nokogiri::XML::SAX::Document
|
|
35
|
+
def initialize
|
|
36
|
+
@chunks = []
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
attr_reader :chunks
|
|
40
|
+
|
|
41
|
+
def cdata_block(string)
|
|
42
|
+
characters(string)
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def characters(string)
|
|
46
|
+
@chunks << string.strip if string.strip != ""
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
|
|
36
50
|
def initialize(opts = {})
|
|
37
51
|
@config = opts[:config]
|
|
38
52
|
@redis_config = opts[:redis_config]
|
|
@@ -49,7 +63,7 @@ module Flapjack
|
|
|
49
63
|
|
|
50
64
|
def stop
|
|
51
65
|
@should_quit = true
|
|
52
|
-
@redis.rpush(@config['queue'],
|
|
66
|
+
@redis.rpush(@config['queue'], Oj.dump('notification_type' => 'shutdown'))
|
|
53
67
|
end
|
|
54
68
|
|
|
55
69
|
def setup
|
|
@@ -116,12 +130,18 @@ module Flapjack
|
|
|
116
130
|
end
|
|
117
131
|
end
|
|
118
132
|
|
|
119
|
-
def interpreter(
|
|
133
|
+
def interpreter(command_raw)
|
|
120
134
|
msg = nil
|
|
121
135
|
action = nil
|
|
122
136
|
entity_check = nil
|
|
123
|
-
|
|
124
|
-
|
|
137
|
+
|
|
138
|
+
th = TextHandler.new
|
|
139
|
+
parser = Nokogiri::HTML::SAX::Parser.new(th)
|
|
140
|
+
parser.parse(command_raw)
|
|
141
|
+
command = th.chunks.join(' ')
|
|
142
|
+
|
|
143
|
+
case command
|
|
144
|
+
when /^ACKID\s+(\d+)(?:\s*(.*?)(?:\s*duration:.*?(\w+.*))?)$/i
|
|
125
145
|
ackid = $1
|
|
126
146
|
comment = $2
|
|
127
147
|
duration_str = $3
|
|
@@ -171,16 +191,16 @@ module Flapjack
|
|
|
171
191
|
}
|
|
172
192
|
end
|
|
173
193
|
|
|
174
|
-
when
|
|
194
|
+
when /^help$/i
|
|
175
195
|
msg = "commands: \n" +
|
|
176
196
|
" ACKID <id> <comment> [duration: <time spec>] \n" +
|
|
177
197
|
" find entities matching /pattern/ \n" +
|
|
178
198
|
" test notifications for <entity>[:<check>] \n" +
|
|
179
|
-
" tell me about <entity>[:<check>]" +
|
|
199
|
+
" tell me about <entity>[:<check>] \n" +
|
|
180
200
|
" identify \n" +
|
|
181
201
|
" help \n"
|
|
182
202
|
|
|
183
|
-
when
|
|
203
|
+
when /^identify$/i
|
|
184
204
|
t = Process.times
|
|
185
205
|
fqdn = `/bin/hostname -f`.chomp
|
|
186
206
|
pid = Process.pid
|
|
@@ -190,7 +210,7 @@ module Flapjack
|
|
|
190
210
|
"System CPU Time: #{t.stime}\n" +
|
|
191
211
|
`uname -a`.chomp + "\n"
|
|
192
212
|
|
|
193
|
-
when
|
|
213
|
+
when /^test notifications for\s+([a-z0-9\-\.]+)(?::(.+))?$/i
|
|
194
214
|
entity_name = $1
|
|
195
215
|
check_name = $2 || 'test'
|
|
196
216
|
|
|
@@ -203,7 +223,7 @@ module Flapjack
|
|
|
203
223
|
msg = "yeah, no I can't see #{entity_name} in my systems"
|
|
204
224
|
end
|
|
205
225
|
|
|
206
|
-
when
|
|
226
|
+
when /^tell me about\s+([a-z0-9\-\.]+)(?::(.+))?$+/i
|
|
207
227
|
entity_name = $1
|
|
208
228
|
check_name = $2
|
|
209
229
|
|
|
@@ -216,27 +236,38 @@ module Flapjack
|
|
|
216
236
|
get_details = proc {|entity_check|
|
|
217
237
|
sched = entity_check.current_maintenance(:scheduled => true)
|
|
218
238
|
unsched = entity_check.current_maintenance(:unscheduled => true)
|
|
239
|
+
out = ''
|
|
219
240
|
|
|
220
|
-
if
|
|
241
|
+
if check_name.nil?
|
|
221
242
|
check = entity_check.check
|
|
222
|
-
|
|
243
|
+
out += "---\n#{entity_name}:#{check}\n"
|
|
223
244
|
end
|
|
224
245
|
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
246
|
+
if sched.nil? && unsched.nil?
|
|
247
|
+
out += "Not in scheduled or unscheduled maintenance.\n"
|
|
248
|
+
else
|
|
249
|
+
if sched.nil?
|
|
250
|
+
out += "Not in scheduled maintenance.\n"
|
|
251
|
+
else
|
|
252
|
+
start = Time.at(sched[:start_time])
|
|
253
|
+
finish = Time.at(sched[:start_time] + sched[:duration])
|
|
254
|
+
remain = time_period_in_words( (finish - current_time).ceil )
|
|
255
|
+
# TODO a simpler time format?
|
|
256
|
+
out += "In scheduled maintenance: #{start} -> #{finish} (#{remain} remaining)\n"
|
|
257
|
+
end
|
|
232
258
|
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
259
|
+
if unsched.nil?
|
|
260
|
+
out += "Not in unscheduled maintenance.\n"
|
|
261
|
+
else
|
|
262
|
+
start = Time.at(unsched[:start_time])
|
|
263
|
+
finish = Time.at(unsched[:start_time] + unsched[:duration])
|
|
264
|
+
remain = time_period_in_words( (finish - current_time).ceil )
|
|
265
|
+
# TODO a simpler time format?
|
|
266
|
+
out += "In unscheduled maintenance: #{start} -> #{finish} (#{remain} remaining)\n"
|
|
267
|
+
end
|
|
239
268
|
end
|
|
269
|
+
|
|
270
|
+
out
|
|
240
271
|
}
|
|
241
272
|
|
|
242
273
|
check_names = check_name.nil? ? entity.check_list.sort : [check_name]
|
|
@@ -247,14 +278,14 @@ module Flapjack
|
|
|
247
278
|
check_names.each do |check|
|
|
248
279
|
entity_check = Flapjack::Data::EntityCheck.for_entity(entity, check, :redis => @redis)
|
|
249
280
|
next if entity_check.nil?
|
|
250
|
-
get_details.call(entity_check)
|
|
281
|
+
msg += get_details.call(entity_check)
|
|
251
282
|
end
|
|
252
283
|
end
|
|
253
284
|
else
|
|
254
285
|
msg = "hmmm, I can't see #{entity_name} in my systems"
|
|
255
286
|
end
|
|
256
287
|
|
|
257
|
-
when
|
|
288
|
+
when /^(find )?entities matching\s+\/(.*)\/.*$/i
|
|
258
289
|
pattern = $2.chomp.strip
|
|
259
290
|
entity_list = Flapjack::Data::Entity.find_all_name_matching(pattern, :redis => @redis)
|
|
260
291
|
|
|
@@ -279,7 +310,7 @@ module Flapjack
|
|
|
279
310
|
msg = "that doesn't seem to be a valid pattern - /#{pattern}/"
|
|
280
311
|
end
|
|
281
312
|
|
|
282
|
-
when
|
|
313
|
+
when /^(.*)/
|
|
283
314
|
words = $1
|
|
284
315
|
msg = "what do you mean, '#{words}'? Type 'help' for a list of acceptable commands."
|
|
285
316
|
|
|
@@ -399,7 +430,7 @@ module Flapjack
|
|
|
399
430
|
if connected?
|
|
400
431
|
@logger.debug("jabber is connected so commencing blpop on #{queue}")
|
|
401
432
|
events[queue] = @redis.blpop(queue, 0)
|
|
402
|
-
event =
|
|
433
|
+
event = Oj.load(events[queue][1])
|
|
403
434
|
type = event['notification_type'] || 'unknown'
|
|
404
435
|
@logger.debug('jabber notification event received')
|
|
405
436
|
@logger.debug(event.inspect)
|
|
@@ -4,8 +4,7 @@ require 'socket'
|
|
|
4
4
|
require 'eventmachine'
|
|
5
5
|
require 'em-synchrony'
|
|
6
6
|
require 'blather/client/client'
|
|
7
|
-
require '
|
|
8
|
-
require 'yajl/json_gem'
|
|
7
|
+
require 'oj'
|
|
9
8
|
|
|
10
9
|
require 'flapjack/utility'
|
|
11
10
|
|
|
@@ -16,9 +15,8 @@ module Flapjack
|
|
|
16
15
|
class Oobetet < Blather::Client
|
|
17
16
|
include Flapjack::Utility
|
|
18
17
|
|
|
19
|
-
log = Logger.new(STDOUT)
|
|
20
|
-
|
|
21
|
-
log.level = Logger::INFO
|
|
18
|
+
log = ::Logger.new(STDOUT)
|
|
19
|
+
log.level = ::Logger::INFO
|
|
22
20
|
Blather.logger = log
|
|
23
21
|
|
|
24
22
|
def initialize(opts = {})
|
|
@@ -212,9 +210,9 @@ module Flapjack
|
|
|
212
210
|
end
|
|
213
211
|
|
|
214
212
|
def send_pagerduty_event(event)
|
|
215
|
-
options = { :body =>
|
|
213
|
+
options = { :body => Oj.dump(event) }
|
|
216
214
|
http = EM::HttpRequest.new(@pagerduty_events_api_url).post(options)
|
|
217
|
-
response =
|
|
215
|
+
response = Oj.load(http.response)
|
|
218
216
|
status = http.response_header.status
|
|
219
217
|
@logger.debug "send_pagerduty_event got a return code of #{status.to_s} - #{response.inspect}"
|
|
220
218
|
[status, response]
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
require 'em-synchrony'
|
|
4
4
|
require 'em-synchrony/em-http'
|
|
5
5
|
|
|
6
|
-
require '
|
|
6
|
+
require 'oj'
|
|
7
7
|
|
|
8
8
|
require 'flapjack/data/entity_check'
|
|
9
9
|
require 'flapjack/data/global'
|
|
@@ -32,7 +32,7 @@ module Flapjack
|
|
|
32
32
|
def stop
|
|
33
33
|
@logger.info("stopping")
|
|
34
34
|
@should_quit = true
|
|
35
|
-
@redis.rpush(@config['queue'],
|
|
35
|
+
@redis.rpush(@config['queue'], Oj.dump('notification_type' => 'shutdown'))
|
|
36
36
|
end
|
|
37
37
|
|
|
38
38
|
def start
|
|
@@ -56,7 +56,7 @@ module Flapjack
|
|
|
56
56
|
until @should_quit
|
|
57
57
|
@logger.debug("pagerduty gateway is going into blpop mode on #{queue}")
|
|
58
58
|
events[queue] = @redis.blpop(queue, 0)
|
|
59
|
-
event =
|
|
59
|
+
event = Oj.load(events[queue][1])
|
|
60
60
|
type = event['notification_type']
|
|
61
61
|
@logger.debug("pagerduty notification event popped off the queue: " + event.inspect)
|
|
62
62
|
unless 'shutdown'.eql?(type)
|
|
@@ -134,9 +134,9 @@ module Flapjack
|
|
|
134
134
|
end
|
|
135
135
|
|
|
136
136
|
def send_pagerduty_event(event)
|
|
137
|
-
options = { :body =>
|
|
137
|
+
options = { :body => Oj.dump(event) }
|
|
138
138
|
http = EM::HttpRequest.new(PAGERDUTY_EVENTS_API_URL).post(options)
|
|
139
|
-
response =
|
|
139
|
+
response = Oj.load(http.response)
|
|
140
140
|
status = http.response_header.status
|
|
141
141
|
@logger.debug "send_pagerduty_event got a return code of #{status.to_s} - #{response.inspect}"
|
|
142
142
|
[status, response]
|
|
@@ -215,8 +215,8 @@ module Flapjack
|
|
|
215
215
|
|
|
216
216
|
http = EM::HttpRequest.new(url).get(options)
|
|
217
217
|
begin
|
|
218
|
-
response =
|
|
219
|
-
rescue
|
|
218
|
+
response = Oj.load(http.response)
|
|
219
|
+
rescue Oj::Error
|
|
220
220
|
@logger.error("failed to parse json from a post to #{url} ... response headers and body follows...")
|
|
221
221
|
return nil
|
|
222
222
|
end
|
|
@@ -3,8 +3,9 @@
|
|
|
3
3
|
require 'chronic'
|
|
4
4
|
require 'chronic_duration'
|
|
5
5
|
require 'sinatra/base'
|
|
6
|
-
require '
|
|
6
|
+
require 'erb'
|
|
7
7
|
require 'rack/fiber_pool'
|
|
8
|
+
require 'json'
|
|
8
9
|
|
|
9
10
|
require 'flapjack/rack_logger'
|
|
10
11
|
|
|
@@ -61,6 +62,16 @@ module Flapjack
|
|
|
61
62
|
set :views, settings.root + '/web/views'
|
|
62
63
|
set :public_folder, settings.root + '/web/public'
|
|
63
64
|
|
|
65
|
+
helpers do
|
|
66
|
+
def h(text)
|
|
67
|
+
ERB::Util.h(text)
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def u(text)
|
|
71
|
+
ERB::Util.u(text)
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
|
|
64
75
|
def redis
|
|
65
76
|
self.class.instance_variable_get('@redis')
|
|
66
77
|
end
|
|
@@ -72,39 +83,48 @@ module Flapjack
|
|
|
72
83
|
get '/' do
|
|
73
84
|
check_stats
|
|
74
85
|
entity_stats
|
|
75
|
-
|
|
86
|
+
|
|
87
|
+
erb 'index.html'.to_sym
|
|
76
88
|
end
|
|
77
89
|
|
|
78
90
|
get '/checks_all' do
|
|
79
91
|
check_stats
|
|
80
92
|
@adjective = 'all'
|
|
81
93
|
|
|
82
|
-
|
|
83
|
-
@states =
|
|
84
|
-
entity
|
|
85
|
-
|
|
86
|
-
|
|
94
|
+
checks_by_entity = Flapjack::Data::EntityCheck.find_all_by_entity(:redis => redis)
|
|
95
|
+
@states = checks_by_entity.keys.inject({}) {|result, entity|
|
|
96
|
+
result[entity] = checks_by_entity[entity].sort.map {|check|
|
|
97
|
+
[check] + entity_check_state(entity, check)
|
|
98
|
+
}
|
|
99
|
+
result
|
|
100
|
+
}
|
|
101
|
+
@entities_sorted = checks_by_entity.keys.sort
|
|
87
102
|
|
|
88
|
-
|
|
103
|
+
erb 'checks.html'.to_sym
|
|
89
104
|
end
|
|
90
105
|
|
|
91
106
|
get '/checks_failing' do
|
|
92
107
|
check_stats
|
|
93
108
|
@adjective = 'failing'
|
|
94
109
|
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
[
|
|
98
|
-
|
|
110
|
+
checks_by_entity = Flapjack::Data::EntityCheck.find_all_failing_by_entity(:redis => redis)
|
|
111
|
+
@states = checks_by_entity.keys.inject({}) {|result, entity|
|
|
112
|
+
result[entity] = checks_by_entity[entity].sort.map {|check|
|
|
113
|
+
[check] + entity_check_state(entity, check)
|
|
114
|
+
}
|
|
115
|
+
result
|
|
116
|
+
}
|
|
117
|
+
@entities_sorted = checks_by_entity.keys.sort
|
|
99
118
|
|
|
100
|
-
|
|
119
|
+
erb 'checks.html'.to_sym
|
|
101
120
|
end
|
|
102
121
|
|
|
103
122
|
get '/self_stats' do
|
|
104
123
|
self_stats
|
|
105
124
|
entity_stats
|
|
106
125
|
check_stats
|
|
107
|
-
|
|
126
|
+
|
|
127
|
+
erb 'self_stats.html'.to_sym
|
|
108
128
|
end
|
|
109
129
|
|
|
110
130
|
get '/self_stats.json' do
|
|
@@ -124,13 +144,6 @@ module Flapjack
|
|
|
124
144
|
'failure' => @event_counters['failure'].to_i,
|
|
125
145
|
'action' => @event_counters['action'].to_i,
|
|
126
146
|
}
|
|
127
|
-
# 'instance' => {
|
|
128
|
-
# 'total' => @event_counters_instance['all'].to_i,
|
|
129
|
-
# 'ok' => @event_counters_instance['ok'].to_i,
|
|
130
|
-
# 'failure' => @event_counters_instance['failure'].to_i,
|
|
131
|
-
# 'action' => @event_counters_instance['action'].to_i,
|
|
132
|
-
# 'average' => @event_rate_all.to_i,
|
|
133
|
-
# }
|
|
134
147
|
},
|
|
135
148
|
'total_keys' => @dbsize,
|
|
136
149
|
'uptime' => @uptime_string,
|
|
@@ -144,24 +157,26 @@ module Flapjack
|
|
|
144
157
|
entity_stats
|
|
145
158
|
@adjective = 'all'
|
|
146
159
|
@entities = Flapjack::Data::Entity.find_all_with_checks(:redis => redis)
|
|
147
|
-
|
|
160
|
+
|
|
161
|
+
erb 'entities.html'.to_sym
|
|
148
162
|
end
|
|
149
163
|
|
|
150
164
|
get '/entities_failing' do
|
|
151
165
|
entity_stats
|
|
152
166
|
@adjective = 'failing'
|
|
153
167
|
@entities = Flapjack::Data::Entity.find_all_with_failing_checks(:redis => redis)
|
|
154
|
-
|
|
168
|
+
|
|
169
|
+
erb 'entities.html'.to_sym
|
|
155
170
|
end
|
|
156
171
|
|
|
157
172
|
get '/entity/:entity' do
|
|
158
173
|
@entity = params[:entity]
|
|
159
174
|
entity_stats
|
|
160
|
-
@states =
|
|
161
|
-
check
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
175
|
+
@states = Flapjack::Data::EntityCheck.find_all_for_entity_name(@entity, :redis => redis).sort.map { |check|
|
|
176
|
+
[check] + entity_check_state(@entity, check)
|
|
177
|
+
}.sort_by {|parts| parts }
|
|
178
|
+
|
|
179
|
+
erb 'entity.html'.to_sym
|
|
165
180
|
end
|
|
166
181
|
|
|
167
182
|
get '/check' do
|
|
@@ -176,11 +191,14 @@ module Flapjack
|
|
|
176
191
|
last_change = entity_check.last_change
|
|
177
192
|
|
|
178
193
|
@check_state = entity_check.state
|
|
194
|
+
@check_enabled = entity_check.enabled?
|
|
179
195
|
@check_last_update = entity_check.last_update
|
|
180
196
|
@check_last_change = last_change
|
|
181
197
|
@check_summary = entity_check.summary
|
|
182
198
|
@check_details = entity_check.details
|
|
183
|
-
|
|
199
|
+
|
|
200
|
+
@last_notifications = last_notification_data(entity_check)
|
|
201
|
+
|
|
184
202
|
@scheduled_maintenances = entity_check.maintenances(nil, nil, :scheduled => true)
|
|
185
203
|
@acknowledgement_id = entity_check.failed? ?
|
|
186
204
|
entity_check.event_count_at(entity_check.last_change) : nil
|
|
@@ -193,7 +211,7 @@ module Flapjack
|
|
|
193
211
|
@state_changes = entity_check.historical_states(nil, Time.now.to_i,
|
|
194
212
|
:order => 'desc', :limit => 20)
|
|
195
213
|
|
|
196
|
-
|
|
214
|
+
erb 'check.html'.to_sym
|
|
197
215
|
end
|
|
198
216
|
|
|
199
217
|
post '/acknowledgements/:entity/:check' do
|
|
@@ -256,14 +274,23 @@ module Flapjack
|
|
|
256
274
|
redirect back
|
|
257
275
|
end
|
|
258
276
|
|
|
277
|
+
# delete a check (actually just disables it)
|
|
278
|
+
delete '/checks/:entity/:check' do
|
|
279
|
+
entity_check = get_entity_check(params[:entity], params[:check])
|
|
280
|
+
return 404 if entity_check.nil?
|
|
281
|
+
|
|
282
|
+
entity_check.disable!
|
|
283
|
+
redirect back
|
|
284
|
+
end
|
|
285
|
+
|
|
259
286
|
get '/contacts' do
|
|
260
287
|
#self_stats
|
|
261
288
|
@contacts = Flapjack::Data::Contact.all(:redis => redis)
|
|
262
|
-
|
|
289
|
+
|
|
290
|
+
erb 'contacts.html'.to_sym
|
|
263
291
|
end
|
|
264
292
|
|
|
265
293
|
get "/contacts/:contact" do
|
|
266
|
-
#self_stats
|
|
267
294
|
contact_id = params[:contact]
|
|
268
295
|
|
|
269
296
|
if contact_id
|
|
@@ -279,17 +306,22 @@ module Flapjack
|
|
|
279
306
|
@pagerduty_credentials = @contact.pagerduty_credentials
|
|
280
307
|
end
|
|
281
308
|
|
|
309
|
+
# FIXME: intersect with current checks, or push down to Contact.entities
|
|
282
310
|
@entities_and_checks = @contact.entities(:checks => true).sort_by {|ec|
|
|
283
311
|
ec[:entity].name
|
|
284
312
|
}
|
|
285
313
|
|
|
286
|
-
|
|
314
|
+
erb 'contact.html'.to_sym
|
|
287
315
|
end
|
|
288
316
|
|
|
289
317
|
protected
|
|
290
318
|
|
|
291
|
-
|
|
292
|
-
|
|
319
|
+
# TODO cache constructed erb object to improve performance -- check mtime
|
|
320
|
+
# to know when to refresh; would need to synchronize accesses to the cache,
|
|
321
|
+
# to lock out reads while it's being refreshed
|
|
322
|
+
def render_erb(file, bind)
|
|
323
|
+
erb = ERB.new(File.read(File.dirname(__FILE__) + '/web/views/' + file))
|
|
324
|
+
erb.result(bind)
|
|
293
325
|
end
|
|
294
326
|
|
|
295
327
|
private
|
|
@@ -307,18 +339,33 @@ module Flapjack
|
|
|
307
339
|
return if entity.nil?
|
|
308
340
|
entity_check = Flapjack::Data::EntityCheck.for_entity(entity,
|
|
309
341
|
check, :redis => redis)
|
|
342
|
+
summary = entity_check.summary
|
|
343
|
+
summary = summary[0..76] + '...' unless summary.length < 81
|
|
310
344
|
latest_notif =
|
|
311
345
|
{:problem => entity_check.last_notification_for_state(:problem)[:timestamp],
|
|
312
346
|
:recovery => entity_check.last_notification_for_state(:recovery)[:timestamp],
|
|
313
347
|
:acknowledgement => entity_check.last_notification_for_state(:acknowledgement)[:timestamp]
|
|
314
348
|
}.max_by {|n| n[1] || 0}
|
|
349
|
+
|
|
350
|
+
lc = entity_check.last_change
|
|
351
|
+
last_change = lc ? ChronicDuration.output(Time.now.to_i - lc.to_i,
|
|
352
|
+
:format => :short, :keep_zero => true, :units => 2) : 'never'
|
|
353
|
+
|
|
354
|
+
lu = entity_check.last_update
|
|
355
|
+
last_update = lu ? ChronicDuration.output(Time.now.to_i - lu.to_i,
|
|
356
|
+
:format => :short, :keep_zero => true, :units => 2) : 'never'
|
|
357
|
+
|
|
358
|
+
ln = latest_notif[1]
|
|
359
|
+
last_notified = ln ? ChronicDuration.output(Time.now.to_i - ln.to_i,
|
|
360
|
+
:format => :short, :keep_zero => true, :units => 2) + ", #{latest_notif[0]}" : 'never'
|
|
361
|
+
|
|
315
362
|
[(entity_check.state || '-'),
|
|
316
|
-
(
|
|
317
|
-
|
|
363
|
+
(summary || '-'),
|
|
364
|
+
last_change,
|
|
365
|
+
last_update,
|
|
318
366
|
entity_check.in_unscheduled_maintenance?,
|
|
319
367
|
entity_check.in_scheduled_maintenance?,
|
|
320
|
-
|
|
321
|
-
latest_notif[1]
|
|
368
|
+
last_notified
|
|
322
369
|
]
|
|
323
370
|
end
|
|
324
371
|
|
|
@@ -347,10 +394,23 @@ module Flapjack
|
|
|
347
394
|
end
|
|
348
395
|
|
|
349
396
|
def check_stats
|
|
350
|
-
|
|
351
|
-
@
|
|
397
|
+
# FIXME: move this logic to Flapjack::Data::EntityCheck
|
|
398
|
+
@count_all_checks = Flapjack::Data::EntityCheck.count_all(:redis => redis)
|
|
399
|
+
@count_failing_checks = Flapjack::Data::EntityCheck.count_all_failing(:redis => redis)
|
|
352
400
|
end
|
|
353
401
|
|
|
402
|
+
def last_notification_data(entity_check)
|
|
403
|
+
last_notifications = entity_check.last_notifications_of_each_type
|
|
404
|
+
[:critical, :warning, :unknown, :recovery, :acknowledgement].inject({}) do |memo, type|
|
|
405
|
+
if last_notifications[type] && last_notifications[type][:timestamp]
|
|
406
|
+
t = Time.at(last_notifications[type][:timestamp])
|
|
407
|
+
memo[t] = {:time => t.to_s,
|
|
408
|
+
:relative => relative_time_ago(t) + " ago",
|
|
409
|
+
:summary => last_notifications[type][:summary]}
|
|
410
|
+
end
|
|
411
|
+
memo
|
|
412
|
+
end
|
|
413
|
+
end
|
|
354
414
|
|
|
355
415
|
end
|
|
356
416
|
|