flapjack 0.7.18 → 0.7.19
Sign up to get free protection for your applications and to get access to all the features.
- data/CHANGELOG.md +7 -0
- data/bin/flapjack +3 -0
- data/bin/flapjack-nagios-receiver +4 -1
- data/bin/flapjack-netsaint-parser +2 -1
- data/bin/flapjack-populator +6 -3
- data/bin/receive-events +3 -1
- data/bin/simulate-failed-check +2 -1
- data/etc/flapjack_config.yaml.example +20 -0
- data/features/events.feature +1 -1
- data/features/events_check_names.feature +1 -1
- data/features/notification_rules.feature +1 -1
- data/features/notifications.feature +1 -1
- data/features/steps/events_steps.rb +18 -17
- data/features/steps/flapjack-netsaint-parser_steps.rb +1 -2
- data/features/steps/notifications_steps.rb +14 -1
- data/features/support/env.rb +27 -10
- data/flapjack.gemspec +1 -3
- data/lib/flapjack/coordinator.rb +30 -20
- data/lib/flapjack/data/contact.rb +3 -2
- data/lib/flapjack/data/entity.rb +3 -3
- data/lib/flapjack/data/entity_check.rb +116 -43
- data/lib/flapjack/data/event.rb +10 -10
- data/lib/flapjack/data/message.rb +3 -6
- data/lib/flapjack/data/notification.rb +122 -57
- data/lib/flapjack/data/notification_rule.rb +11 -11
- data/lib/flapjack/filters/acknowledgement.rb +2 -2
- data/lib/flapjack/filters/ok.rb +1 -1
- data/lib/flapjack/gateways/api/entity_check_presenter.rb +1 -0
- data/lib/flapjack/gateways/api/entity_methods.rb +4 -6
- data/lib/flapjack/gateways/api/rack/json_params_parser.rb +1 -1
- data/lib/flapjack/gateways/email.rb +3 -5
- data/lib/flapjack/gateways/email/{alert.html.haml → alert.html.erb} +0 -0
- data/lib/flapjack/gateways/jabber.rb +66 -35
- data/lib/flapjack/gateways/oobetet.rb +5 -7
- data/lib/flapjack/gateways/pagerduty.rb +7 -7
- data/lib/flapjack/gateways/web.rb +101 -41
- data/lib/flapjack/gateways/web/public/css/flapjack.css +1 -1
- data/lib/flapjack/gateways/web/views/{_css.haml → _css.html.erb} +2 -1
- data/lib/flapjack/gateways/web/views/_foot.html.erb +3 -0
- data/lib/flapjack/gateways/web/views/_head.html.erb +4 -0
- data/lib/flapjack/gateways/web/views/_nav.html.erb +9 -0
- data/lib/flapjack/gateways/web/views/check.html.erb +204 -0
- data/lib/flapjack/gateways/web/views/checks.html.erb +77 -0
- data/lib/flapjack/gateways/web/views/contact.html.erb +114 -0
- data/lib/flapjack/gateways/web/views/contacts.html.erb +42 -0
- data/lib/flapjack/gateways/web/views/entities.html.erb +39 -0
- data/lib/flapjack/gateways/web/views/entity.html.erb +67 -0
- data/lib/flapjack/gateways/web/views/index.html.erb +27 -0
- data/lib/flapjack/gateways/web/views/self_stats.html.erb +97 -0
- data/lib/flapjack/logger.rb +71 -23
- data/lib/flapjack/notifier.rb +157 -0
- data/lib/flapjack/patches.rb +1 -41
- data/lib/flapjack/pikelet.rb +4 -2
- data/lib/flapjack/{executive.rb → processor.rb} +32 -145
- data/lib/flapjack/version.rb +1 -1
- data/spec/lib/flapjack/coordinator_spec.rb +134 -71
- data/spec/lib/flapjack/data/contact_spec.rb +1 -0
- data/spec/lib/flapjack/data/entity_check_spec.rb +146 -30
- data/spec/lib/flapjack/data/entity_spec.rb +4 -4
- data/spec/lib/flapjack/data/event_spec.rb +4 -4
- data/spec/lib/flapjack/data/message_spec.rb +2 -3
- data/spec/lib/flapjack/data/notification_spec.rb +13 -19
- data/spec/lib/flapjack/gateways/api/entity_methods_spec.rb +2 -2
- data/spec/lib/flapjack/gateways/jabber_spec.rb +34 -0
- data/spec/lib/flapjack/gateways/pagerduty_spec.rb +0 -2
- data/spec/lib/flapjack/gateways/web/views/{check.haml_spec.rb → check.html.erb_spec.rb} +2 -2
- data/spec/lib/flapjack/gateways/web/views/{contact.haml_spec.rb → contact.html.erb_spec.rb} +3 -3
- data/spec/lib/flapjack/gateways/web/views/index.html.erb_spec.rb +14 -0
- data/spec/lib/flapjack/gateways/web_spec.rb +20 -8
- data/spec/lib/flapjack/logger_spec.rb +30 -28
- data/spec/lib/flapjack/notifier_spec.rb +6 -0
- data/spec/lib/flapjack/pikelet_spec.rb +8 -8
- data/spec/lib/flapjack/{executive_spec.rb → processor_spec.rb} +4 -4
- data/spec/spec_helper.rb +1 -13
- data/spec/support/erb_view_helper.rb +23 -0
- data/tasks/profile.rake +1 -1
- data/tmp/acknowledge.rb +3 -1
- data/tmp/create_event_ok.rb +3 -1
- data/tmp/create_event_unknown.rb +3 -1
- data/tmp/create_events_failure.rb +3 -1
- data/tmp/create_events_ok.rb +3 -1
- data/tmp/create_events_ok_fail_ack_ok.rb +3 -1
- data/tmp/create_events_ok_failure.rb +3 -1
- data/tmp/create_events_ok_failure_ack.rb +3 -1
- data/tmp/test_json_post.rb +5 -3
- data/tmp/test_notification_rules_api.rb +5 -3
- metadata +32 -61
- data/lib/flapjack/gateways/web/views/_foot.haml +0 -8
- data/lib/flapjack/gateways/web/views/_head.haml +0 -10
- data/lib/flapjack/gateways/web/views/_nav.haml +0 -14
- data/lib/flapjack/gateways/web/views/check.haml +0 -191
- data/lib/flapjack/gateways/web/views/checks.haml +0 -49
- data/lib/flapjack/gateways/web/views/contact.haml +0 -85
- data/lib/flapjack/gateways/web/views/contacts.haml +0 -30
- data/lib/flapjack/gateways/web/views/entities.haml +0 -28
- data/lib/flapjack/gateways/web/views/entity.haml +0 -50
- data/lib/flapjack/gateways/web/views/index.haml +0 -32
- data/lib/flapjack/gateways/web/views/self_stats.haml +0 -70
- data/spec/lib/flapjack/gateways/web/views/index.haml_spec.rb +0 -13
- data/spec/support/haml_view_helper.rb +0 -15
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
<!DOCTYPE html>
|
|
2
|
+
<html lang="en">
|
|
3
|
+
<head>
|
|
4
|
+
<% nav = render_erb('_nav.html.erb', binding) %>
|
|
5
|
+
<% head = render_erb('_head.html.erb', binding) %>
|
|
6
|
+
<% foot = render_erb('_foot.html.erb', binding) %>
|
|
7
|
+
<title>Flapjack - <%= h @adjective.capitalize %> Entities</title>
|
|
8
|
+
<%= head %>
|
|
9
|
+
</head>
|
|
10
|
+
<body>
|
|
11
|
+
<div id="wrap">
|
|
12
|
+
<div class="container">
|
|
13
|
+
<div class="page-header">
|
|
14
|
+
<%= nav %>
|
|
15
|
+
<h2><%= h @adjective.capitalize %> Entities</h2>
|
|
16
|
+
</div>
|
|
17
|
+
|
|
18
|
+
<p><%= h @count_failing_entities %> failing out of <%= h @count_all_entities %></p>
|
|
19
|
+
|
|
20
|
+
<% if @entities.length > 0 %>
|
|
21
|
+
<table class="table table-bordered table-hover table-condensed">
|
|
22
|
+
<% @entities.sort.each do |entity| %>
|
|
23
|
+
<tr>
|
|
24
|
+
<td><a href="/entity/<%= CGI.escape(entity) %>"><%= h entity %></a>
|
|
25
|
+
</td>
|
|
26
|
+
</tr>
|
|
27
|
+
<% end %>
|
|
28
|
+
</table>
|
|
29
|
+
<% else %>
|
|
30
|
+
<p>No check output has been processed yet, so there are no entities we can show you here.</p>
|
|
31
|
+
<% end %>
|
|
32
|
+
</div>
|
|
33
|
+
<div id="push"></div>
|
|
34
|
+
</div>
|
|
35
|
+
<div id="footer">
|
|
36
|
+
<%= foot %>
|
|
37
|
+
</div>
|
|
38
|
+
</body>
|
|
39
|
+
</html>
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
<!DOCTYPE html>
|
|
2
|
+
<html lang="en">
|
|
3
|
+
<head>
|
|
4
|
+
<% nav = render_erb('_nav.html.erb', binding) %>
|
|
5
|
+
<% head = render_erb('_head.html.erb', binding) %>
|
|
6
|
+
<% foot = render_erb('_foot.html.erb', binding) %>
|
|
7
|
+
<title>Flapjack - <%=h @entity %> (entity) </title>
|
|
8
|
+
<%= head %>
|
|
9
|
+
</head>
|
|
10
|
+
<body>
|
|
11
|
+
<div id="wrap">
|
|
12
|
+
<div class="container">
|
|
13
|
+
<div class="page-header">
|
|
14
|
+
<%= nav %>
|
|
15
|
+
<h2><%= h @entity %></h2>
|
|
16
|
+
</div>
|
|
17
|
+
<% if @states.empty? %>
|
|
18
|
+
<div>
|
|
19
|
+
<p>This entity has no check output associated with it</p>
|
|
20
|
+
</div>
|
|
21
|
+
<% else %>
|
|
22
|
+
<table class="table table-bordered table-hover table-condensed">
|
|
23
|
+
<tr>
|
|
24
|
+
<th>Check</th>
|
|
25
|
+
<th>State</th>
|
|
26
|
+
<th>Summary</th>
|
|
27
|
+
<th>Last State Change</th>
|
|
28
|
+
<th>Last Update</th>
|
|
29
|
+
<th>Last Notification</th>
|
|
30
|
+
</tr>
|
|
31
|
+
<% @states.each do |check, status, summary, changed, updated, in_unscheduled_outage, in_scheduled_outage, notified| %>
|
|
32
|
+
<%
|
|
33
|
+
row_colour = case status
|
|
34
|
+
when 'critical', 'unknown'
|
|
35
|
+
'error'
|
|
36
|
+
when 'ok', 'up'
|
|
37
|
+
'success'
|
|
38
|
+
else
|
|
39
|
+
status
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
check_link = "/check?entity=" << u(@entity) << "&check=" << u(check)
|
|
43
|
+
|
|
44
|
+
%>
|
|
45
|
+
<tr class="<%= row_colour %>">
|
|
46
|
+
<td><a href="<%= check_link %>" title="check detail"><%= h check %></a></td>
|
|
47
|
+
<td class="<%= status %>">
|
|
48
|
+
<%= h status.upcase %>
|
|
49
|
+
<% if in_unscheduled_outage%> (Ack'd)<% end %>
|
|
50
|
+
<% if in_scheduled_outage %> (Sched)<% end %>
|
|
51
|
+
</td>
|
|
52
|
+
<td><%= summary %></td>
|
|
53
|
+
<td><%= changed %></td>
|
|
54
|
+
<td><%= updated %></td>
|
|
55
|
+
<td><%= notified %></td>
|
|
56
|
+
</tr>
|
|
57
|
+
<% end %>
|
|
58
|
+
</table>
|
|
59
|
+
<% end %>
|
|
60
|
+
</div>
|
|
61
|
+
<div id="push"></div>
|
|
62
|
+
</div>
|
|
63
|
+
<div id="footer">
|
|
64
|
+
<%= foot %>
|
|
65
|
+
</div>
|
|
66
|
+
</body>
|
|
67
|
+
</html>
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
<!DOCTYPE html>
|
|
2
|
+
<html lang="en">
|
|
3
|
+
<head>
|
|
4
|
+
<% nav = render_erb('_nav.html.erb', binding) %>
|
|
5
|
+
<% head = render_erb('_head.html.erb', binding) %>
|
|
6
|
+
<% foot = render_erb('_foot.html.erb', binding) %>
|
|
7
|
+
<title>Flapjack - Summary</title>
|
|
8
|
+
<%= head %>
|
|
9
|
+
</head>
|
|
10
|
+
<body>
|
|
11
|
+
<div id="wrap">
|
|
12
|
+
<div class="container">
|
|
13
|
+
<div class="page-header">
|
|
14
|
+
<%= nav %>
|
|
15
|
+
<h2>Summary</h2>
|
|
16
|
+
</div>
|
|
17
|
+
<h4><a href="/entities_failing" title="failing entities"><%= h @count_failing_entities %></a> out of <a href="/entities_all" title="all entities"><%= h @count_all_entities %></a> entities have failing checks</h4>
|
|
18
|
+
<h4><a href="/checks_failing" title="failing checks"><%= h @count_failing_checks %></a> out of <a href="/checks_all" title="all checks"><%= h @count_all_checks %></a> checks are failing</h4>
|
|
19
|
+
<img src="/img/flapjack_white_bg_400_353.jpeg" width="400" height="353">
|
|
20
|
+
</div>
|
|
21
|
+
<div id="push"></div>
|
|
22
|
+
</div>
|
|
23
|
+
<div id="footer">
|
|
24
|
+
<%= foot %>
|
|
25
|
+
</div>
|
|
26
|
+
</body>
|
|
27
|
+
</html>
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
<!DOCTYPE html>
|
|
2
|
+
<html lang="en">
|
|
3
|
+
<head>
|
|
4
|
+
<% nav = render_erb('_nav.html.erb', binding) %>
|
|
5
|
+
<% head = render_erb('_head.html.erb', binding) %>
|
|
6
|
+
<% foot = render_erb('_foot.html.erb', binding) %>
|
|
7
|
+
<title>Flapjack - Internal Statistics</title>
|
|
8
|
+
<%= head %>
|
|
9
|
+
</head>
|
|
10
|
+
<body>
|
|
11
|
+
<div id="wrap">
|
|
12
|
+
<div class="container">
|
|
13
|
+
<div class="page-header">
|
|
14
|
+
<%= nav %>
|
|
15
|
+
<h2>Internal Statistics</h2>
|
|
16
|
+
</div>
|
|
17
|
+
|
|
18
|
+
<table class="table table-bordered table-hover table-condensed">
|
|
19
|
+
<tr>
|
|
20
|
+
<td>Events queued:</td>
|
|
21
|
+
<td><%= h @events_queued %></td>
|
|
22
|
+
</tr>
|
|
23
|
+
<tr>
|
|
24
|
+
<td>Number of entities:</td>
|
|
25
|
+
<td><%= h @count_all_entities %></td>
|
|
26
|
+
</tr>
|
|
27
|
+
<tr>
|
|
28
|
+
<td>Number of failing entities:</td>
|
|
29
|
+
<td><%= h @count_failing_entities %></td>
|
|
30
|
+
</tr>
|
|
31
|
+
<tr>
|
|
32
|
+
<td>Number of checks:</td>
|
|
33
|
+
<td><%= h @count_all_checks %></td>
|
|
34
|
+
</tr>
|
|
35
|
+
<tr>
|
|
36
|
+
<td>Number of failing checks:</td>
|
|
37
|
+
<td><%= h @count_failing_checks %></td>
|
|
38
|
+
</tr>
|
|
39
|
+
<tr>
|
|
40
|
+
<td>Events processed (all time)</td>
|
|
41
|
+
<td><%= h @event_counters['all'] %> (ok: <%= h @event_counters['ok'] %>, failure: <%= h @event_counters['failure'] %>, action: <%= h @event_counters['action'] %>)</td>
|
|
42
|
+
</tr>
|
|
43
|
+
<tr>
|
|
44
|
+
<td>Events processed (this instance)</td>
|
|
45
|
+
<td><%= h @event_counters_instance['all'] %> (ok: <%= h @event_counters_instance['ok'] %>, failure: <%= h @event_counters_instance['failure'] %>, action: <%= h @event_counters_instance['action'] %>)</td>
|
|
46
|
+
</tr>
|
|
47
|
+
<tr>
|
|
48
|
+
<td>Average rate (this instance)</td>
|
|
49
|
+
<td><%= h @event_rate_all %></td> events per second
|
|
50
|
+
</tr>
|
|
51
|
+
<tr>
|
|
52
|
+
<td>Total keys in redis</td>
|
|
53
|
+
<td><%= h @dbsize %></td>
|
|
54
|
+
</tr>
|
|
55
|
+
<tr>
|
|
56
|
+
<td>Uptime</td>
|
|
57
|
+
<td><%= h @uptime_string %></td>
|
|
58
|
+
</tr>
|
|
59
|
+
<tr>
|
|
60
|
+
<td>Boot Time</td>
|
|
61
|
+
<td><%= h @boot_time %></td>
|
|
62
|
+
</tr>
|
|
63
|
+
<tr>
|
|
64
|
+
<td>Current time</td>
|
|
65
|
+
<td><%= h Time.now.to_s %></td>
|
|
66
|
+
</tr>
|
|
67
|
+
</table>
|
|
68
|
+
|
|
69
|
+
<h4>Executive Instances:</h4>
|
|
70
|
+
<table class="table table-bordered table-hover">
|
|
71
|
+
<tr>
|
|
72
|
+
<th>Hostname</th>
|
|
73
|
+
<th>PID</th>
|
|
74
|
+
<th>Started</th>
|
|
75
|
+
</tr>
|
|
76
|
+
<% @executive_instances.each do |i| %>
|
|
77
|
+
<%
|
|
78
|
+
hostname, pid = i[0].split(':')
|
|
79
|
+
started = "#{relative_time_ago(Time.at(i[1].to_i))} ago"
|
|
80
|
+
%>
|
|
81
|
+
<tr>
|
|
82
|
+
<td><%= h hostname %></td>
|
|
83
|
+
<td><%= h pid %></td>
|
|
84
|
+
<td><%= h started %></td>
|
|
85
|
+
</tr>
|
|
86
|
+
<% end %>
|
|
87
|
+
</table>
|
|
88
|
+
|
|
89
|
+
<p><a href="/self_stats.json">Instrument as JSON</a></p>
|
|
90
|
+
</div>
|
|
91
|
+
<div id="push"></div>
|
|
92
|
+
</div>
|
|
93
|
+
<div id="footer">
|
|
94
|
+
<%= foot %>
|
|
95
|
+
</div>
|
|
96
|
+
</body>
|
|
97
|
+
</html>
|
data/lib/flapjack/logger.rb
CHANGED
|
@@ -1,28 +1,52 @@
|
|
|
1
1
|
#!/usr/bin/env ruby
|
|
2
2
|
|
|
3
|
-
require '
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
3
|
+
require 'logger'
|
|
4
|
+
|
|
5
|
+
begin
|
|
6
|
+
# Ruby 2.0+
|
|
7
|
+
require 'syslog/logger'
|
|
8
|
+
rescue LoadError
|
|
9
|
+
# Ruby 1.9
|
|
10
|
+
require 'syslog'
|
|
11
|
+
end
|
|
7
12
|
|
|
8
13
|
module Flapjack
|
|
9
14
|
|
|
10
15
|
class Logger
|
|
11
16
|
|
|
17
|
+
LEVELS = [:debug, :info, :warn, :error, :fatal]
|
|
18
|
+
|
|
19
|
+
# only used for 1.9
|
|
20
|
+
SYSLOG_LEVELS_MAP = {
|
|
21
|
+
:debug => Syslog::Constants::LOG_DEBUG,
|
|
22
|
+
:info => Syslog::Constants::LOG_INFO,
|
|
23
|
+
:warn => Syslog::Constants::LOG_WARNING,
|
|
24
|
+
:error => Syslog::Constants::LOG_ERR,
|
|
25
|
+
:fatal => Syslog::Constants::LOG_CRIT
|
|
26
|
+
}
|
|
27
|
+
|
|
12
28
|
def initialize(name, config = {})
|
|
13
29
|
config ||= {}
|
|
14
30
|
|
|
15
|
-
|
|
31
|
+
@name = name
|
|
16
32
|
|
|
17
|
-
@
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
:date_pattern => "%Y-%m-%dT%H:%M:%S%z")
|
|
33
|
+
@formatter = proc do |severity, datetime, progname, msg|
|
|
34
|
+
"#{datetime.iso8601} [#{severity}] :: #{name} :: #{msg}\n"
|
|
35
|
+
end
|
|
21
36
|
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
37
|
+
@logger = ::Logger.new(STDOUT)
|
|
38
|
+
@logger.formatter = @formatter
|
|
39
|
+
|
|
40
|
+
if Syslog.const_defined?('Logger', false)
|
|
41
|
+
# Ruby 2.0+
|
|
42
|
+
@sys_logger = Syslog.const_get('Logger', false).new('flapjack')
|
|
43
|
+
@sys_logger.formatter = @formatter
|
|
44
|
+
else
|
|
45
|
+
# Ruby 1.9
|
|
46
|
+
@syslog = Syslog.opened? ? Syslog :
|
|
47
|
+
Syslog.open('flapjack',
|
|
48
|
+
(Syslog::Constants::LOG_PID | Syslog::Constants::LOG_CONS),
|
|
49
|
+
Syslog::Constants::LOG_USER)
|
|
26
50
|
end
|
|
27
51
|
|
|
28
52
|
configure(config)
|
|
@@ -31,25 +55,49 @@ module Flapjack
|
|
|
31
55
|
def configure(config)
|
|
32
56
|
level = config['level']
|
|
33
57
|
|
|
34
|
-
# we'll let
|
|
35
|
-
# assume
|
|
58
|
+
# we'll let Logger spit the dummy on invalid level values -- but will
|
|
59
|
+
# assume INFO if nothing is provided
|
|
36
60
|
if level.nil? || level.empty?
|
|
37
|
-
level = '
|
|
61
|
+
level = 'INFO'
|
|
38
62
|
end
|
|
39
63
|
|
|
40
|
-
|
|
41
|
-
|
|
64
|
+
err = nil
|
|
65
|
+
|
|
66
|
+
new_level = begin
|
|
67
|
+
::Logger.const_get(level.upcase)
|
|
68
|
+
rescue NameError
|
|
69
|
+
err = "Unknown Logger severity level '#{level.upcase}', using INFO..."
|
|
70
|
+
::Logger::INFO
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
@logger.error(err) if err
|
|
74
|
+
|
|
75
|
+
@logger.level = new_level
|
|
76
|
+
if @sys_logger
|
|
77
|
+
@sys_logger.level = new_level
|
|
78
|
+
elsif @syslog
|
|
79
|
+
Syslog.mask = Syslog::LOG_UPTO(SYSLOG_LEVELS_MAP[level.downcase.to_sym])
|
|
80
|
+
end
|
|
42
81
|
|
|
43
|
-
# puts "setting log level for '#{@name}' to '#{level.upcase}'"
|
|
44
|
-
@log4r_logger.level = new_level
|
|
45
82
|
end
|
|
46
83
|
|
|
47
|
-
|
|
48
|
-
|
|
84
|
+
LEVELS.each do |level|
|
|
85
|
+
define_method(level) {|*args, &block|
|
|
86
|
+
@logger.send(level.to_sym, *args, &block)
|
|
87
|
+
if @sys_logger
|
|
88
|
+
@sys_logger.send(level.to_sym, *args, &block)
|
|
89
|
+
elsif @syslog
|
|
90
|
+
t = Time.now.iso8601
|
|
91
|
+
l = level.to_s.upcase
|
|
92
|
+
@syslog.log(SYSLOG_LEVELS_MAP[level],
|
|
93
|
+
"#{t} [#{l}] :: #{@name} :: %s",
|
|
94
|
+
(block ? block.call : args.first))
|
|
95
|
+
end
|
|
96
|
+
}
|
|
49
97
|
end
|
|
50
98
|
|
|
51
99
|
def respond_to?(sym)
|
|
52
|
-
|
|
100
|
+
(LEVELS + [:configure]).include?(sym)
|
|
53
101
|
end
|
|
54
102
|
|
|
55
103
|
end
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
#!/usr/bin/env ruby
|
|
2
|
+
|
|
3
|
+
require 'active_support/time'
|
|
4
|
+
|
|
5
|
+
require 'oj'
|
|
6
|
+
|
|
7
|
+
require 'flapjack/data/contact'
|
|
8
|
+
require 'flapjack/data/entity_check'
|
|
9
|
+
require 'flapjack/data/notification'
|
|
10
|
+
require 'flapjack/data/event'
|
|
11
|
+
require 'flapjack/redis_pool'
|
|
12
|
+
require 'flapjack/utility'
|
|
13
|
+
|
|
14
|
+
require 'flapjack/gateways/email'
|
|
15
|
+
require 'flapjack/gateways/sms_messagenet'
|
|
16
|
+
|
|
17
|
+
module Flapjack
|
|
18
|
+
|
|
19
|
+
class Notifier
|
|
20
|
+
|
|
21
|
+
include Flapjack::Utility
|
|
22
|
+
|
|
23
|
+
def initialize(opts = {})
|
|
24
|
+
@config = opts[:config]
|
|
25
|
+
@redis_config = opts[:redis_config]
|
|
26
|
+
@logger = opts[:logger]
|
|
27
|
+
@redis = Flapjack::RedisPool.new(:config => @redis_config, :size => 2) # first will block
|
|
28
|
+
|
|
29
|
+
@notifications_queue = @config['queue'] || 'notifications'
|
|
30
|
+
|
|
31
|
+
@queues = {:email => @config['email_queue'],
|
|
32
|
+
:sms => @config['sms_queue'],
|
|
33
|
+
:jabber => @config['jabber_queue'],
|
|
34
|
+
:pagerduty => @config['pagerduty_queue']}
|
|
35
|
+
|
|
36
|
+
notify_logfile = @config['notification_log_file'] || 'log/notify.log'
|
|
37
|
+
if not File.directory?(File.dirname(notify_logfile))
|
|
38
|
+
puts "Parent directory for log file '#{notify_logfile}' doesn't exist"
|
|
39
|
+
puts "Exiting!"
|
|
40
|
+
exit
|
|
41
|
+
end
|
|
42
|
+
@notifylog = ::Logger.new(notify_logfile)
|
|
43
|
+
@notifylog.formatter = proc do |severity, datetime, progname, msg|
|
|
44
|
+
"#{datetime.to_s} | #{msg}\n"
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
tz = nil
|
|
48
|
+
tz_string = @config['default_contact_timezone'] || ENV['TZ'] || 'UTC'
|
|
49
|
+
begin
|
|
50
|
+
tz = ActiveSupport::TimeZone.new(tz_string)
|
|
51
|
+
rescue ArgumentError
|
|
52
|
+
logger.error("Invalid timezone string specified in default_contact_timezone or TZ (#{tz_string})")
|
|
53
|
+
exit 1
|
|
54
|
+
end
|
|
55
|
+
@default_contact_timezone = tz
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def start
|
|
59
|
+
@logger.info("Booting main loop.")
|
|
60
|
+
|
|
61
|
+
until @should_quit
|
|
62
|
+
@logger.debug("Waiting for notification...")
|
|
63
|
+
notification = Flapjack::Data::Notification.next(@notifications_queue,
|
|
64
|
+
:redis => @redis,
|
|
65
|
+
:logger => @logger)
|
|
66
|
+
process_notification(notification) unless notification.nil? || (notification.type == 'shutdown')
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
@logger.info("Exiting main loop.")
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
# this must use a separate connection to the main Executive one, as it's running
|
|
73
|
+
# from a different fiber while the main one is blocking.
|
|
74
|
+
def stop
|
|
75
|
+
@should_quit = true
|
|
76
|
+
@redis.rpush(@notifications_queue, Oj.dump('type' => 'shutdown'))
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
private
|
|
80
|
+
|
|
81
|
+
# takes an event for which messages should be generated, works out the type of
|
|
82
|
+
# notification, updates the notification history in redis, generates the
|
|
83
|
+
# notifications
|
|
84
|
+
def process_notification(notification)
|
|
85
|
+
timestamp = Time.now
|
|
86
|
+
event_id = notification.event_id
|
|
87
|
+
entity_check = Flapjack::Data::EntityCheck.for_event_id(event_id, :redis => @redis)
|
|
88
|
+
contacts = entity_check.contacts
|
|
89
|
+
|
|
90
|
+
if contacts.empty?
|
|
91
|
+
@logger.debug("No contacts for #{event_id}")
|
|
92
|
+
@notifylog.info("#{event_id} | #{notification.type} | NO CONTACTS")
|
|
93
|
+
return
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
messages = notification.messages(contacts, :default_timezone => @default_contact_timezone,
|
|
97
|
+
:logger => @logger)
|
|
98
|
+
|
|
99
|
+
notification_contents = notification.contents
|
|
100
|
+
|
|
101
|
+
messages.each do |message|
|
|
102
|
+
media_type = message.medium
|
|
103
|
+
address = message.address
|
|
104
|
+
contents = message.contents.merge(notification_contents)
|
|
105
|
+
|
|
106
|
+
@notifylog.info("#{event_id} | " +
|
|
107
|
+
"#{notification.type} | #{message.contact.id} | #{media_type} | #{address}")
|
|
108
|
+
|
|
109
|
+
unless @queues[media_type.to_sym]
|
|
110
|
+
@logger.error("no queue for media type: #{media_type}")
|
|
111
|
+
return
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
@logger.info("Enqueueing #{media_type} alert for #{event_id} to #{address}")
|
|
115
|
+
|
|
116
|
+
contact = message.contact
|
|
117
|
+
|
|
118
|
+
# was event.ok?
|
|
119
|
+
if (notification.event_state == 'ok') || (notification.event_state == 'up')
|
|
120
|
+
contact.update_sent_alert_keys(
|
|
121
|
+
:media => media_type,
|
|
122
|
+
:check => event_id,
|
|
123
|
+
:state => 'warning',
|
|
124
|
+
:delete => true)
|
|
125
|
+
contact.update_sent_alert_keys(
|
|
126
|
+
:media => media_type,
|
|
127
|
+
:check => event_id,
|
|
128
|
+
:state => 'critical',
|
|
129
|
+
:delete => true)
|
|
130
|
+
contact.update_sent_alert_keys(
|
|
131
|
+
:media => media_type,
|
|
132
|
+
:check => event_id,
|
|
133
|
+
:state => 'unknown',
|
|
134
|
+
:delete => true)
|
|
135
|
+
else
|
|
136
|
+
contact.update_sent_alert_keys(
|
|
137
|
+
:media => media_type,
|
|
138
|
+
:check => event_id,
|
|
139
|
+
:state => notification.event_state)
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
# TODO consider changing Resque jobs to use raw blpop like the others
|
|
143
|
+
case media_type.to_sym
|
|
144
|
+
when :sms
|
|
145
|
+
Resque.enqueue_to(@queues[:sms], Flapjack::Gateways::SmsMessagenet, contents)
|
|
146
|
+
when :email
|
|
147
|
+
Resque.enqueue_to(@queues[:email], Flapjack::Gateways::Email, contents)
|
|
148
|
+
when :jabber
|
|
149
|
+
@redis.rpush(@queues[:jabber], Oj.dump(contents))
|
|
150
|
+
when :pagerduty
|
|
151
|
+
@redis.rpush(@queues[:pagerduty], Oj.dump(contents))
|
|
152
|
+
end
|
|
153
|
+
end
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
end
|
|
157
|
+
end
|