flapjack 0.7.18 → 0.7.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/CHANGELOG.md +7 -0
- data/bin/flapjack +3 -0
- data/bin/flapjack-nagios-receiver +4 -1
- data/bin/flapjack-netsaint-parser +2 -1
- data/bin/flapjack-populator +6 -3
- data/bin/receive-events +3 -1
- data/bin/simulate-failed-check +2 -1
- data/etc/flapjack_config.yaml.example +20 -0
- data/features/events.feature +1 -1
- data/features/events_check_names.feature +1 -1
- data/features/notification_rules.feature +1 -1
- data/features/notifications.feature +1 -1
- data/features/steps/events_steps.rb +18 -17
- data/features/steps/flapjack-netsaint-parser_steps.rb +1 -2
- data/features/steps/notifications_steps.rb +14 -1
- data/features/support/env.rb +27 -10
- data/flapjack.gemspec +1 -3
- data/lib/flapjack/coordinator.rb +30 -20
- data/lib/flapjack/data/contact.rb +3 -2
- data/lib/flapjack/data/entity.rb +3 -3
- data/lib/flapjack/data/entity_check.rb +116 -43
- data/lib/flapjack/data/event.rb +10 -10
- data/lib/flapjack/data/message.rb +3 -6
- data/lib/flapjack/data/notification.rb +122 -57
- data/lib/flapjack/data/notification_rule.rb +11 -11
- data/lib/flapjack/filters/acknowledgement.rb +2 -2
- data/lib/flapjack/filters/ok.rb +1 -1
- data/lib/flapjack/gateways/api/entity_check_presenter.rb +1 -0
- data/lib/flapjack/gateways/api/entity_methods.rb +4 -6
- data/lib/flapjack/gateways/api/rack/json_params_parser.rb +1 -1
- data/lib/flapjack/gateways/email.rb +3 -5
- data/lib/flapjack/gateways/email/{alert.html.haml → alert.html.erb} +0 -0
- data/lib/flapjack/gateways/jabber.rb +66 -35
- data/lib/flapjack/gateways/oobetet.rb +5 -7
- data/lib/flapjack/gateways/pagerduty.rb +7 -7
- data/lib/flapjack/gateways/web.rb +101 -41
- data/lib/flapjack/gateways/web/public/css/flapjack.css +1 -1
- data/lib/flapjack/gateways/web/views/{_css.haml → _css.html.erb} +2 -1
- data/lib/flapjack/gateways/web/views/_foot.html.erb +3 -0
- data/lib/flapjack/gateways/web/views/_head.html.erb +4 -0
- data/lib/flapjack/gateways/web/views/_nav.html.erb +9 -0
- data/lib/flapjack/gateways/web/views/check.html.erb +204 -0
- data/lib/flapjack/gateways/web/views/checks.html.erb +77 -0
- data/lib/flapjack/gateways/web/views/contact.html.erb +114 -0
- data/lib/flapjack/gateways/web/views/contacts.html.erb +42 -0
- data/lib/flapjack/gateways/web/views/entities.html.erb +39 -0
- data/lib/flapjack/gateways/web/views/entity.html.erb +67 -0
- data/lib/flapjack/gateways/web/views/index.html.erb +27 -0
- data/lib/flapjack/gateways/web/views/self_stats.html.erb +97 -0
- data/lib/flapjack/logger.rb +71 -23
- data/lib/flapjack/notifier.rb +157 -0
- data/lib/flapjack/patches.rb +1 -41
- data/lib/flapjack/pikelet.rb +4 -2
- data/lib/flapjack/{executive.rb → processor.rb} +32 -145
- data/lib/flapjack/version.rb +1 -1
- data/spec/lib/flapjack/coordinator_spec.rb +134 -71
- data/spec/lib/flapjack/data/contact_spec.rb +1 -0
- data/spec/lib/flapjack/data/entity_check_spec.rb +146 -30
- data/spec/lib/flapjack/data/entity_spec.rb +4 -4
- data/spec/lib/flapjack/data/event_spec.rb +4 -4
- data/spec/lib/flapjack/data/message_spec.rb +2 -3
- data/spec/lib/flapjack/data/notification_spec.rb +13 -19
- data/spec/lib/flapjack/gateways/api/entity_methods_spec.rb +2 -2
- data/spec/lib/flapjack/gateways/jabber_spec.rb +34 -0
- data/spec/lib/flapjack/gateways/pagerduty_spec.rb +0 -2
- data/spec/lib/flapjack/gateways/web/views/{check.haml_spec.rb → check.html.erb_spec.rb} +2 -2
- data/spec/lib/flapjack/gateways/web/views/{contact.haml_spec.rb → contact.html.erb_spec.rb} +3 -3
- data/spec/lib/flapjack/gateways/web/views/index.html.erb_spec.rb +14 -0
- data/spec/lib/flapjack/gateways/web_spec.rb +20 -8
- data/spec/lib/flapjack/logger_spec.rb +30 -28
- data/spec/lib/flapjack/notifier_spec.rb +6 -0
- data/spec/lib/flapjack/pikelet_spec.rb +8 -8
- data/spec/lib/flapjack/{executive_spec.rb → processor_spec.rb} +4 -4
- data/spec/spec_helper.rb +1 -13
- data/spec/support/erb_view_helper.rb +23 -0
- data/tasks/profile.rake +1 -1
- data/tmp/acknowledge.rb +3 -1
- data/tmp/create_event_ok.rb +3 -1
- data/tmp/create_event_unknown.rb +3 -1
- data/tmp/create_events_failure.rb +3 -1
- data/tmp/create_events_ok.rb +3 -1
- data/tmp/create_events_ok_fail_ack_ok.rb +3 -1
- data/tmp/create_events_ok_failure.rb +3 -1
- data/tmp/create_events_ok_failure_ack.rb +3 -1
- data/tmp/test_json_post.rb +5 -3
- data/tmp/test_notification_rules_api.rb +5 -3
- metadata +32 -61
- data/lib/flapjack/gateways/web/views/_foot.haml +0 -8
- data/lib/flapjack/gateways/web/views/_head.haml +0 -10
- data/lib/flapjack/gateways/web/views/_nav.haml +0 -14
- data/lib/flapjack/gateways/web/views/check.haml +0 -191
- data/lib/flapjack/gateways/web/views/checks.haml +0 -49
- data/lib/flapjack/gateways/web/views/contact.haml +0 -85
- data/lib/flapjack/gateways/web/views/contacts.haml +0 -30
- data/lib/flapjack/gateways/web/views/entities.haml +0 -28
- data/lib/flapjack/gateways/web/views/entity.haml +0 -50
- data/lib/flapjack/gateways/web/views/index.haml +0 -32
- data/lib/flapjack/gateways/web/views/self_stats.haml +0 -70
- data/spec/lib/flapjack/gateways/web/views/index.haml_spec.rb +0 -13
- data/spec/support/haml_view_helper.rb +0 -15
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
<!DOCTYPE html>
|
|
2
|
+
<html lang="en">
|
|
3
|
+
<head>
|
|
4
|
+
<% nav = render_erb('_nav.html.erb', binding) %>
|
|
5
|
+
<% head = render_erb('_head.html.erb', binding) %>
|
|
6
|
+
<% foot = render_erb('_foot.html.erb', binding) %>
|
|
7
|
+
<title>Flapjack - <%= h @adjective.capitalize %> Entities</title>
|
|
8
|
+
<%= head %>
|
|
9
|
+
</head>
|
|
10
|
+
<body>
|
|
11
|
+
<div id="wrap">
|
|
12
|
+
<div class="container">
|
|
13
|
+
<div class="page-header">
|
|
14
|
+
<%= nav %>
|
|
15
|
+
<h2><%= h @adjective.capitalize %> Entities</h2>
|
|
16
|
+
</div>
|
|
17
|
+
|
|
18
|
+
<p><%= h @count_failing_entities %> failing out of <%= h @count_all_entities %></p>
|
|
19
|
+
|
|
20
|
+
<% if @entities.length > 0 %>
|
|
21
|
+
<table class="table table-bordered table-hover table-condensed">
|
|
22
|
+
<% @entities.sort.each do |entity| %>
|
|
23
|
+
<tr>
|
|
24
|
+
<td><a href="/entity/<%= CGI.escape(entity) %>"><%= h entity %></a>
|
|
25
|
+
</td>
|
|
26
|
+
</tr>
|
|
27
|
+
<% end %>
|
|
28
|
+
</table>
|
|
29
|
+
<% else %>
|
|
30
|
+
<p>No check output has been processed yet, so there are no entities we can show you here.</p>
|
|
31
|
+
<% end %>
|
|
32
|
+
</div>
|
|
33
|
+
<div id="push"></div>
|
|
34
|
+
</div>
|
|
35
|
+
<div id="footer">
|
|
36
|
+
<%= foot %>
|
|
37
|
+
</div>
|
|
38
|
+
</body>
|
|
39
|
+
</html>
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
<!DOCTYPE html>
|
|
2
|
+
<html lang="en">
|
|
3
|
+
<head>
|
|
4
|
+
<% nav = render_erb('_nav.html.erb', binding) %>
|
|
5
|
+
<% head = render_erb('_head.html.erb', binding) %>
|
|
6
|
+
<% foot = render_erb('_foot.html.erb', binding) %>
|
|
7
|
+
<title>Flapjack - <%=h @entity %> (entity) </title>
|
|
8
|
+
<%= head %>
|
|
9
|
+
</head>
|
|
10
|
+
<body>
|
|
11
|
+
<div id="wrap">
|
|
12
|
+
<div class="container">
|
|
13
|
+
<div class="page-header">
|
|
14
|
+
<%= nav %>
|
|
15
|
+
<h2><%= h @entity %></h2>
|
|
16
|
+
</div>
|
|
17
|
+
<% if @states.empty? %>
|
|
18
|
+
<div>
|
|
19
|
+
<p>This entity has no check output associated with it</p>
|
|
20
|
+
</div>
|
|
21
|
+
<% else %>
|
|
22
|
+
<table class="table table-bordered table-hover table-condensed">
|
|
23
|
+
<tr>
|
|
24
|
+
<th>Check</th>
|
|
25
|
+
<th>State</th>
|
|
26
|
+
<th>Summary</th>
|
|
27
|
+
<th>Last State Change</th>
|
|
28
|
+
<th>Last Update</th>
|
|
29
|
+
<th>Last Notification</th>
|
|
30
|
+
</tr>
|
|
31
|
+
<% @states.each do |check, status, summary, changed, updated, in_unscheduled_outage, in_scheduled_outage, notified| %>
|
|
32
|
+
<%
|
|
33
|
+
row_colour = case status
|
|
34
|
+
when 'critical', 'unknown'
|
|
35
|
+
'error'
|
|
36
|
+
when 'ok', 'up'
|
|
37
|
+
'success'
|
|
38
|
+
else
|
|
39
|
+
status
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
check_link = "/check?entity=" << u(@entity) << "&check=" << u(check)
|
|
43
|
+
|
|
44
|
+
%>
|
|
45
|
+
<tr class="<%= row_colour %>">
|
|
46
|
+
<td><a href="<%= check_link %>" title="check detail"><%= h check %></a></td>
|
|
47
|
+
<td class="<%= status %>">
|
|
48
|
+
<%= h status.upcase %>
|
|
49
|
+
<% if in_unscheduled_outage%> (Ack'd)<% end %>
|
|
50
|
+
<% if in_scheduled_outage %> (Sched)<% end %>
|
|
51
|
+
</td>
|
|
52
|
+
<td><%= summary %></td>
|
|
53
|
+
<td><%= changed %></td>
|
|
54
|
+
<td><%= updated %></td>
|
|
55
|
+
<td><%= notified %></td>
|
|
56
|
+
</tr>
|
|
57
|
+
<% end %>
|
|
58
|
+
</table>
|
|
59
|
+
<% end %>
|
|
60
|
+
</div>
|
|
61
|
+
<div id="push"></div>
|
|
62
|
+
</div>
|
|
63
|
+
<div id="footer">
|
|
64
|
+
<%= foot %>
|
|
65
|
+
</div>
|
|
66
|
+
</body>
|
|
67
|
+
</html>
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
<!DOCTYPE html>
|
|
2
|
+
<html lang="en">
|
|
3
|
+
<head>
|
|
4
|
+
<% nav = render_erb('_nav.html.erb', binding) %>
|
|
5
|
+
<% head = render_erb('_head.html.erb', binding) %>
|
|
6
|
+
<% foot = render_erb('_foot.html.erb', binding) %>
|
|
7
|
+
<title>Flapjack - Summary</title>
|
|
8
|
+
<%= head %>
|
|
9
|
+
</head>
|
|
10
|
+
<body>
|
|
11
|
+
<div id="wrap">
|
|
12
|
+
<div class="container">
|
|
13
|
+
<div class="page-header">
|
|
14
|
+
<%= nav %>
|
|
15
|
+
<h2>Summary</h2>
|
|
16
|
+
</div>
|
|
17
|
+
<h4><a href="/entities_failing" title="failing entities"><%= h @count_failing_entities %></a> out of <a href="/entities_all" title="all entities"><%= h @count_all_entities %></a> entities have failing checks</h4>
|
|
18
|
+
<h4><a href="/checks_failing" title="failing checks"><%= h @count_failing_checks %></a> out of <a href="/checks_all" title="all checks"><%= h @count_all_checks %></a> checks are failing</h4>
|
|
19
|
+
<img src="/img/flapjack_white_bg_400_353.jpeg" width="400" height="353">
|
|
20
|
+
</div>
|
|
21
|
+
<div id="push"></div>
|
|
22
|
+
</div>
|
|
23
|
+
<div id="footer">
|
|
24
|
+
<%= foot %>
|
|
25
|
+
</div>
|
|
26
|
+
</body>
|
|
27
|
+
</html>
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
<!DOCTYPE html>
|
|
2
|
+
<html lang="en">
|
|
3
|
+
<head>
|
|
4
|
+
<% nav = render_erb('_nav.html.erb', binding) %>
|
|
5
|
+
<% head = render_erb('_head.html.erb', binding) %>
|
|
6
|
+
<% foot = render_erb('_foot.html.erb', binding) %>
|
|
7
|
+
<title>Flapjack - Internal Statistics</title>
|
|
8
|
+
<%= head %>
|
|
9
|
+
</head>
|
|
10
|
+
<body>
|
|
11
|
+
<div id="wrap">
|
|
12
|
+
<div class="container">
|
|
13
|
+
<div class="page-header">
|
|
14
|
+
<%= nav %>
|
|
15
|
+
<h2>Internal Statistics</h2>
|
|
16
|
+
</div>
|
|
17
|
+
|
|
18
|
+
<table class="table table-bordered table-hover table-condensed">
|
|
19
|
+
<tr>
|
|
20
|
+
<td>Events queued:</td>
|
|
21
|
+
<td><%= h @events_queued %></td>
|
|
22
|
+
</tr>
|
|
23
|
+
<tr>
|
|
24
|
+
<td>Number of entities:</td>
|
|
25
|
+
<td><%= h @count_all_entities %></td>
|
|
26
|
+
</tr>
|
|
27
|
+
<tr>
|
|
28
|
+
<td>Number of failing entities:</td>
|
|
29
|
+
<td><%= h @count_failing_entities %></td>
|
|
30
|
+
</tr>
|
|
31
|
+
<tr>
|
|
32
|
+
<td>Number of checks:</td>
|
|
33
|
+
<td><%= h @count_all_checks %></td>
|
|
34
|
+
</tr>
|
|
35
|
+
<tr>
|
|
36
|
+
<td>Number of failing checks:</td>
|
|
37
|
+
<td><%= h @count_failing_checks %></td>
|
|
38
|
+
</tr>
|
|
39
|
+
<tr>
|
|
40
|
+
<td>Events processed (all time)</td>
|
|
41
|
+
<td><%= h @event_counters['all'] %> (ok: <%= h @event_counters['ok'] %>, failure: <%= h @event_counters['failure'] %>, action: <%= h @event_counters['action'] %>)</td>
|
|
42
|
+
</tr>
|
|
43
|
+
<tr>
|
|
44
|
+
<td>Events processed (this instance)</td>
|
|
45
|
+
<td><%= h @event_counters_instance['all'] %> (ok: <%= h @event_counters_instance['ok'] %>, failure: <%= h @event_counters_instance['failure'] %>, action: <%= h @event_counters_instance['action'] %>)</td>
|
|
46
|
+
</tr>
|
|
47
|
+
<tr>
|
|
48
|
+
<td>Average rate (this instance)</td>
|
|
49
|
+
<td><%= h @event_rate_all %></td> events per second
|
|
50
|
+
</tr>
|
|
51
|
+
<tr>
|
|
52
|
+
<td>Total keys in redis</td>
|
|
53
|
+
<td><%= h @dbsize %></td>
|
|
54
|
+
</tr>
|
|
55
|
+
<tr>
|
|
56
|
+
<td>Uptime</td>
|
|
57
|
+
<td><%= h @uptime_string %></td>
|
|
58
|
+
</tr>
|
|
59
|
+
<tr>
|
|
60
|
+
<td>Boot Time</td>
|
|
61
|
+
<td><%= h @boot_time %></td>
|
|
62
|
+
</tr>
|
|
63
|
+
<tr>
|
|
64
|
+
<td>Current time</td>
|
|
65
|
+
<td><%= h Time.now.to_s %></td>
|
|
66
|
+
</tr>
|
|
67
|
+
</table>
|
|
68
|
+
|
|
69
|
+
<h4>Executive Instances:</h4>
|
|
70
|
+
<table class="table table-bordered table-hover">
|
|
71
|
+
<tr>
|
|
72
|
+
<th>Hostname</th>
|
|
73
|
+
<th>PID</th>
|
|
74
|
+
<th>Started</th>
|
|
75
|
+
</tr>
|
|
76
|
+
<% @executive_instances.each do |i| %>
|
|
77
|
+
<%
|
|
78
|
+
hostname, pid = i[0].split(':')
|
|
79
|
+
started = "#{relative_time_ago(Time.at(i[1].to_i))} ago"
|
|
80
|
+
%>
|
|
81
|
+
<tr>
|
|
82
|
+
<td><%= h hostname %></td>
|
|
83
|
+
<td><%= h pid %></td>
|
|
84
|
+
<td><%= h started %></td>
|
|
85
|
+
</tr>
|
|
86
|
+
<% end %>
|
|
87
|
+
</table>
|
|
88
|
+
|
|
89
|
+
<p><a href="/self_stats.json">Instrument as JSON</a></p>
|
|
90
|
+
</div>
|
|
91
|
+
<div id="push"></div>
|
|
92
|
+
</div>
|
|
93
|
+
<div id="footer">
|
|
94
|
+
<%= foot %>
|
|
95
|
+
</div>
|
|
96
|
+
</body>
|
|
97
|
+
</html>
|
data/lib/flapjack/logger.rb
CHANGED
|
@@ -1,28 +1,52 @@
|
|
|
1
1
|
#!/usr/bin/env ruby
|
|
2
2
|
|
|
3
|
-
require '
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
3
|
+
require 'logger'
|
|
4
|
+
|
|
5
|
+
begin
|
|
6
|
+
# Ruby 2.0+
|
|
7
|
+
require 'syslog/logger'
|
|
8
|
+
rescue LoadError
|
|
9
|
+
# Ruby 1.9
|
|
10
|
+
require 'syslog'
|
|
11
|
+
end
|
|
7
12
|
|
|
8
13
|
module Flapjack
|
|
9
14
|
|
|
10
15
|
class Logger
|
|
11
16
|
|
|
17
|
+
LEVELS = [:debug, :info, :warn, :error, :fatal]
|
|
18
|
+
|
|
19
|
+
# only used for 1.9
|
|
20
|
+
SYSLOG_LEVELS_MAP = {
|
|
21
|
+
:debug => Syslog::Constants::LOG_DEBUG,
|
|
22
|
+
:info => Syslog::Constants::LOG_INFO,
|
|
23
|
+
:warn => Syslog::Constants::LOG_WARNING,
|
|
24
|
+
:error => Syslog::Constants::LOG_ERR,
|
|
25
|
+
:fatal => Syslog::Constants::LOG_CRIT
|
|
26
|
+
}
|
|
27
|
+
|
|
12
28
|
def initialize(name, config = {})
|
|
13
29
|
config ||= {}
|
|
14
30
|
|
|
15
|
-
|
|
31
|
+
@name = name
|
|
16
32
|
|
|
17
|
-
@
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
:date_pattern => "%Y-%m-%dT%H:%M:%S%z")
|
|
33
|
+
@formatter = proc do |severity, datetime, progname, msg|
|
|
34
|
+
"#{datetime.iso8601} [#{severity}] :: #{name} :: #{msg}\n"
|
|
35
|
+
end
|
|
21
36
|
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
37
|
+
@logger = ::Logger.new(STDOUT)
|
|
38
|
+
@logger.formatter = @formatter
|
|
39
|
+
|
|
40
|
+
if Syslog.const_defined?('Logger', false)
|
|
41
|
+
# Ruby 2.0+
|
|
42
|
+
@sys_logger = Syslog.const_get('Logger', false).new('flapjack')
|
|
43
|
+
@sys_logger.formatter = @formatter
|
|
44
|
+
else
|
|
45
|
+
# Ruby 1.9
|
|
46
|
+
@syslog = Syslog.opened? ? Syslog :
|
|
47
|
+
Syslog.open('flapjack',
|
|
48
|
+
(Syslog::Constants::LOG_PID | Syslog::Constants::LOG_CONS),
|
|
49
|
+
Syslog::Constants::LOG_USER)
|
|
26
50
|
end
|
|
27
51
|
|
|
28
52
|
configure(config)
|
|
@@ -31,25 +55,49 @@ module Flapjack
|
|
|
31
55
|
def configure(config)
|
|
32
56
|
level = config['level']
|
|
33
57
|
|
|
34
|
-
# we'll let
|
|
35
|
-
# assume
|
|
58
|
+
# we'll let Logger spit the dummy on invalid level values -- but will
|
|
59
|
+
# assume INFO if nothing is provided
|
|
36
60
|
if level.nil? || level.empty?
|
|
37
|
-
level = '
|
|
61
|
+
level = 'INFO'
|
|
38
62
|
end
|
|
39
63
|
|
|
40
|
-
|
|
41
|
-
|
|
64
|
+
err = nil
|
|
65
|
+
|
|
66
|
+
new_level = begin
|
|
67
|
+
::Logger.const_get(level.upcase)
|
|
68
|
+
rescue NameError
|
|
69
|
+
err = "Unknown Logger severity level '#{level.upcase}', using INFO..."
|
|
70
|
+
::Logger::INFO
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
@logger.error(err) if err
|
|
74
|
+
|
|
75
|
+
@logger.level = new_level
|
|
76
|
+
if @sys_logger
|
|
77
|
+
@sys_logger.level = new_level
|
|
78
|
+
elsif @syslog
|
|
79
|
+
Syslog.mask = Syslog::LOG_UPTO(SYSLOG_LEVELS_MAP[level.downcase.to_sym])
|
|
80
|
+
end
|
|
42
81
|
|
|
43
|
-
# puts "setting log level for '#{@name}' to '#{level.upcase}'"
|
|
44
|
-
@log4r_logger.level = new_level
|
|
45
82
|
end
|
|
46
83
|
|
|
47
|
-
|
|
48
|
-
|
|
84
|
+
LEVELS.each do |level|
|
|
85
|
+
define_method(level) {|*args, &block|
|
|
86
|
+
@logger.send(level.to_sym, *args, &block)
|
|
87
|
+
if @sys_logger
|
|
88
|
+
@sys_logger.send(level.to_sym, *args, &block)
|
|
89
|
+
elsif @syslog
|
|
90
|
+
t = Time.now.iso8601
|
|
91
|
+
l = level.to_s.upcase
|
|
92
|
+
@syslog.log(SYSLOG_LEVELS_MAP[level],
|
|
93
|
+
"#{t} [#{l}] :: #{@name} :: %s",
|
|
94
|
+
(block ? block.call : args.first))
|
|
95
|
+
end
|
|
96
|
+
}
|
|
49
97
|
end
|
|
50
98
|
|
|
51
99
|
def respond_to?(sym)
|
|
52
|
-
|
|
100
|
+
(LEVELS + [:configure]).include?(sym)
|
|
53
101
|
end
|
|
54
102
|
|
|
55
103
|
end
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
#!/usr/bin/env ruby
|
|
2
|
+
|
|
3
|
+
require 'active_support/time'
|
|
4
|
+
|
|
5
|
+
require 'oj'
|
|
6
|
+
|
|
7
|
+
require 'flapjack/data/contact'
|
|
8
|
+
require 'flapjack/data/entity_check'
|
|
9
|
+
require 'flapjack/data/notification'
|
|
10
|
+
require 'flapjack/data/event'
|
|
11
|
+
require 'flapjack/redis_pool'
|
|
12
|
+
require 'flapjack/utility'
|
|
13
|
+
|
|
14
|
+
require 'flapjack/gateways/email'
|
|
15
|
+
require 'flapjack/gateways/sms_messagenet'
|
|
16
|
+
|
|
17
|
+
module Flapjack
|
|
18
|
+
|
|
19
|
+
class Notifier
|
|
20
|
+
|
|
21
|
+
include Flapjack::Utility
|
|
22
|
+
|
|
23
|
+
def initialize(opts = {})
|
|
24
|
+
@config = opts[:config]
|
|
25
|
+
@redis_config = opts[:redis_config]
|
|
26
|
+
@logger = opts[:logger]
|
|
27
|
+
@redis = Flapjack::RedisPool.new(:config => @redis_config, :size => 2) # first will block
|
|
28
|
+
|
|
29
|
+
@notifications_queue = @config['queue'] || 'notifications'
|
|
30
|
+
|
|
31
|
+
@queues = {:email => @config['email_queue'],
|
|
32
|
+
:sms => @config['sms_queue'],
|
|
33
|
+
:jabber => @config['jabber_queue'],
|
|
34
|
+
:pagerduty => @config['pagerduty_queue']}
|
|
35
|
+
|
|
36
|
+
notify_logfile = @config['notification_log_file'] || 'log/notify.log'
|
|
37
|
+
if not File.directory?(File.dirname(notify_logfile))
|
|
38
|
+
puts "Parent directory for log file '#{notify_logfile}' doesn't exist"
|
|
39
|
+
puts "Exiting!"
|
|
40
|
+
exit
|
|
41
|
+
end
|
|
42
|
+
@notifylog = ::Logger.new(notify_logfile)
|
|
43
|
+
@notifylog.formatter = proc do |severity, datetime, progname, msg|
|
|
44
|
+
"#{datetime.to_s} | #{msg}\n"
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
tz = nil
|
|
48
|
+
tz_string = @config['default_contact_timezone'] || ENV['TZ'] || 'UTC'
|
|
49
|
+
begin
|
|
50
|
+
tz = ActiveSupport::TimeZone.new(tz_string)
|
|
51
|
+
rescue ArgumentError
|
|
52
|
+
logger.error("Invalid timezone string specified in default_contact_timezone or TZ (#{tz_string})")
|
|
53
|
+
exit 1
|
|
54
|
+
end
|
|
55
|
+
@default_contact_timezone = tz
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def start
|
|
59
|
+
@logger.info("Booting main loop.")
|
|
60
|
+
|
|
61
|
+
until @should_quit
|
|
62
|
+
@logger.debug("Waiting for notification...")
|
|
63
|
+
notification = Flapjack::Data::Notification.next(@notifications_queue,
|
|
64
|
+
:redis => @redis,
|
|
65
|
+
:logger => @logger)
|
|
66
|
+
process_notification(notification) unless notification.nil? || (notification.type == 'shutdown')
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
@logger.info("Exiting main loop.")
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
# this must use a separate connection to the main Executive one, as it's running
|
|
73
|
+
# from a different fiber while the main one is blocking.
|
|
74
|
+
def stop
|
|
75
|
+
@should_quit = true
|
|
76
|
+
@redis.rpush(@notifications_queue, Oj.dump('type' => 'shutdown'))
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
private
|
|
80
|
+
|
|
81
|
+
# takes an event for which messages should be generated, works out the type of
|
|
82
|
+
# notification, updates the notification history in redis, generates the
|
|
83
|
+
# notifications
|
|
84
|
+
def process_notification(notification)
|
|
85
|
+
timestamp = Time.now
|
|
86
|
+
event_id = notification.event_id
|
|
87
|
+
entity_check = Flapjack::Data::EntityCheck.for_event_id(event_id, :redis => @redis)
|
|
88
|
+
contacts = entity_check.contacts
|
|
89
|
+
|
|
90
|
+
if contacts.empty?
|
|
91
|
+
@logger.debug("No contacts for #{event_id}")
|
|
92
|
+
@notifylog.info("#{event_id} | #{notification.type} | NO CONTACTS")
|
|
93
|
+
return
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
messages = notification.messages(contacts, :default_timezone => @default_contact_timezone,
|
|
97
|
+
:logger => @logger)
|
|
98
|
+
|
|
99
|
+
notification_contents = notification.contents
|
|
100
|
+
|
|
101
|
+
messages.each do |message|
|
|
102
|
+
media_type = message.medium
|
|
103
|
+
address = message.address
|
|
104
|
+
contents = message.contents.merge(notification_contents)
|
|
105
|
+
|
|
106
|
+
@notifylog.info("#{event_id} | " +
|
|
107
|
+
"#{notification.type} | #{message.contact.id} | #{media_type} | #{address}")
|
|
108
|
+
|
|
109
|
+
unless @queues[media_type.to_sym]
|
|
110
|
+
@logger.error("no queue for media type: #{media_type}")
|
|
111
|
+
return
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
@logger.info("Enqueueing #{media_type} alert for #{event_id} to #{address}")
|
|
115
|
+
|
|
116
|
+
contact = message.contact
|
|
117
|
+
|
|
118
|
+
# was event.ok?
|
|
119
|
+
if (notification.event_state == 'ok') || (notification.event_state == 'up')
|
|
120
|
+
contact.update_sent_alert_keys(
|
|
121
|
+
:media => media_type,
|
|
122
|
+
:check => event_id,
|
|
123
|
+
:state => 'warning',
|
|
124
|
+
:delete => true)
|
|
125
|
+
contact.update_sent_alert_keys(
|
|
126
|
+
:media => media_type,
|
|
127
|
+
:check => event_id,
|
|
128
|
+
:state => 'critical',
|
|
129
|
+
:delete => true)
|
|
130
|
+
contact.update_sent_alert_keys(
|
|
131
|
+
:media => media_type,
|
|
132
|
+
:check => event_id,
|
|
133
|
+
:state => 'unknown',
|
|
134
|
+
:delete => true)
|
|
135
|
+
else
|
|
136
|
+
contact.update_sent_alert_keys(
|
|
137
|
+
:media => media_type,
|
|
138
|
+
:check => event_id,
|
|
139
|
+
:state => notification.event_state)
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
# TODO consider changing Resque jobs to use raw blpop like the others
|
|
143
|
+
case media_type.to_sym
|
|
144
|
+
when :sms
|
|
145
|
+
Resque.enqueue_to(@queues[:sms], Flapjack::Gateways::SmsMessagenet, contents)
|
|
146
|
+
when :email
|
|
147
|
+
Resque.enqueue_to(@queues[:email], Flapjack::Gateways::Email, contents)
|
|
148
|
+
when :jabber
|
|
149
|
+
@redis.rpush(@queues[:jabber], Oj.dump(contents))
|
|
150
|
+
when :pagerduty
|
|
151
|
+
@redis.rpush(@queues[:pagerduty], Oj.dump(contents))
|
|
152
|
+
end
|
|
153
|
+
end
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
end
|
|
157
|
+
end
|