flapjack 0.7.22 → 0.7.25
Sign up to get free protection for your applications and to get access to all the features.
- data/CHANGELOG.md +19 -0
- data/bin/flapjack +3 -1
- data/bin/flapjack-nagios-receiver +5 -4
- data/bin/receive-events +2 -2
- data/features/events.feature +101 -95
- data/features/notification_rules.feature +36 -4
- data/features/steps/notifications_steps.rb +4 -0
- data/flapjack.gemspec +3 -2
- data/lib/flapjack/coordinator.rb +8 -6
- data/lib/flapjack/data/entity_check.rb +20 -13
- data/lib/flapjack/data/event.rb +4 -7
- data/lib/flapjack/data/notification.rb +63 -45
- data/lib/flapjack/filters/acknowledgement.rb +26 -24
- data/lib/flapjack/filters/delays.rb +46 -42
- data/lib/flapjack/filters/ok.rb +31 -34
- data/lib/flapjack/filters/scheduled_maintenance.rb +2 -2
- data/lib/flapjack/filters/unscheduled_maintenance.rb +2 -3
- data/lib/flapjack/gateways/email.rb +111 -114
- data/lib/flapjack/gateways/email/alert.html.erb +11 -11
- data/lib/flapjack/gateways/email/alert.text.erb +19 -6
- data/lib/flapjack/gateways/sms_messagenet.rb +15 -5
- data/lib/flapjack/gateways/web.rb +3 -4
- data/lib/flapjack/gateways/web/public/css/flapjack.css +0 -2
- data/lib/flapjack/gateways/web/public/img/flapjack-favicon-32-16.ico +0 -0
- data/lib/flapjack/gateways/web/public/img/flapjack-favicon-64-32-24-16.ico +0 -0
- data/lib/flapjack/gateways/web/public/img/flapjack-transparent-300.png +0 -0
- data/lib/flapjack/gateways/web/public/img/flapjack-transparent-350-400.png +0 -0
- data/lib/flapjack/gateways/web/views/_head.html.erb +1 -0
- data/lib/flapjack/gateways/web/views/index.html.erb +1 -1
- data/lib/flapjack/notifier.rb +2 -3
- data/lib/flapjack/pikelet.rb +5 -4
- data/lib/flapjack/processor.rb +39 -27
- data/lib/flapjack/version.rb +1 -1
- data/spec/lib/flapjack/data/entity_check_spec.rb +5 -0
- data/spec/lib/flapjack/data/event_spec.rb +0 -1
- data/spec/lib/flapjack/gateways/email_spec.rb +5 -9
- data/spec/lib/flapjack/gateways/sms_messagenet.spec.rb +80 -1
- data/spec/lib/flapjack/gateways/web_spec.rb +1 -1
- data/spec/lib/flapjack/pikelet_spec.rb +4 -3
- data/spec/lib/flapjack/processor_spec.rb +0 -1
- metadata +28 -11
- data/lib/flapjack/filters/detect_mass_client_failures.rb +0 -44
- data/spec/lib/flapjack/filters/detect_mass_client_failures_spec.rb +0 -6
data/lib/flapjack/filters/ok.rb
CHANGED
@@ -5,46 +5,43 @@ require 'flapjack/filters/base'
|
|
5
5
|
module Flapjack
|
6
6
|
module Filters
|
7
7
|
|
8
|
+
# * If the service event's state is ok and there is unscheduled maintenance set, end the unscheduled
|
9
|
+
# maintenance
|
8
10
|
# * If the service event’s state is ok and the previous state was ok, don’t alert
|
9
|
-
# * If the service event
|
10
|
-
# * If the service event's state is ok and the previous
|
11
|
-
# seconds, don't alert
|
12
|
-
# * If the service event's state is ok and there is unscheduled downtime set, end the unscheduled
|
13
|
-
# downtime
|
11
|
+
# * If the service event’s state is ok and there's never been a notification, don't alert
|
12
|
+
# * If the service event's state is ok and the previous notification was a recovery or ok, don't alert
|
14
13
|
class Ok
|
15
14
|
include Base
|
16
15
|
|
17
|
-
def block?(event)
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
if event.previous_state == 'ok'
|
22
|
-
@logger.debug("Filter: Ok: existing state was ok, and the previous state was ok, so blocking")
|
23
|
-
result = true
|
24
|
-
end
|
25
|
-
|
26
|
-
entity_check = Flapjack::Data::EntityCheck.for_event_id(event.id, :redis => @redis)
|
27
|
-
|
28
|
-
last_notification = entity_check.last_notification
|
29
|
-
@logger.debug("Filter: Ok: last notification: #{last_notification.inspect}")
|
30
|
-
if last_notification[:type] == 'recovery'
|
31
|
-
@logger.debug("Filter: Ok: last notification was a recovery, so blocking")
|
32
|
-
result = true
|
33
|
-
end
|
34
|
-
|
35
|
-
if event.previous_state != 'ok'
|
36
|
-
if event.previous_state_duration < 30
|
37
|
-
@logger.debug("Filter: Ok: previous non ok state was for less than 30 seconds, so blocking")
|
38
|
-
result = true
|
39
|
-
end
|
40
|
-
end
|
41
|
-
|
42
|
-
# end any unscheduled downtime
|
43
|
-
entity_check.end_unscheduled_maintenance(Time.now.to_i)
|
16
|
+
def block?(event, entity_check, previous_state)
|
17
|
+
unless event.ok?
|
18
|
+
@logger.debug("Filter: Ok: pass")
|
19
|
+
return false
|
44
20
|
end
|
45
21
|
|
46
|
-
|
47
|
-
|
22
|
+
entity_check.end_unscheduled_maintenance(Time.now.to_i)
|
23
|
+
|
24
|
+
if previous_state == 'ok'
|
25
|
+
@logger.debug("Filter: Ok: block - previous state was ok, so blocking")
|
26
|
+
return true
|
27
|
+
end
|
28
|
+
|
29
|
+
last_notification = entity_check.last_notification
|
30
|
+
@logger.debug("Filter: Ok: last notification: #{last_notification.inspect}")
|
31
|
+
|
32
|
+
unless last_notification[:type]
|
33
|
+
@logger.debug("Filter: Ok: block - last notification type is nil (never notified)")
|
34
|
+
return true
|
35
|
+
end
|
36
|
+
|
37
|
+
if [:recovery, :ok].include?(last_notification[:type])
|
38
|
+
@logger.debug("Filter: Ok: block - last notification was a recovery")
|
39
|
+
return true
|
40
|
+
end
|
41
|
+
|
42
|
+
@logger.debug("Filter: Ok: previous_state: #{previous_state}")
|
43
|
+
@logger.debug("Filter: Ok: pass")
|
44
|
+
false
|
48
45
|
end
|
49
46
|
end
|
50
47
|
end
|
@@ -7,8 +7,8 @@ module Flapjack
|
|
7
7
|
class ScheduledMaintenance
|
8
8
|
include Base
|
9
9
|
|
10
|
-
def block?(event)
|
11
|
-
result =
|
10
|
+
def block?(event, entity_check, previous_state)
|
11
|
+
result = entity_check.in_scheduled_maintenance?
|
12
12
|
@logger.debug("Filter: Scheduled Maintenance: #{result ? "block" : "pass"}")
|
13
13
|
result
|
14
14
|
end
|
@@ -7,9 +7,8 @@ module Flapjack
|
|
7
7
|
class UnscheduledMaintenance
|
8
8
|
include Base
|
9
9
|
|
10
|
-
def block?(event)
|
11
|
-
result =
|
12
|
-
!event.acknowledgement?
|
10
|
+
def block?(event, entity_check, previous_state)
|
11
|
+
result = entity_check.in_unscheduled_maintenance? && !event.acknowledgement?
|
13
12
|
@logger.debug("Filter: Unscheduled Maintenance: #{result ? "block" : "pass"}")
|
14
13
|
result
|
15
14
|
end
|
@@ -28,138 +28,135 @@ module Flapjack
|
|
28
28
|
@sent = 0
|
29
29
|
end
|
30
30
|
|
31
|
+
# TODO refactor to remove complexity
|
31
32
|
def perform(notification)
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
@notification_type = notification['notification_type']
|
36
|
-
@contact_first_name = notification['contact_first_name']
|
37
|
-
@contact_last_name = notification['contact_last_name']
|
38
|
-
@state = notification['state']
|
39
|
-
@summary = notification['summary']
|
40
|
-
@last_state = notification['last_state']
|
41
|
-
@last_summary = notification['last_summary']
|
42
|
-
@details = notification['details']
|
43
|
-
@time = notification['time']
|
44
|
-
@entity_name, @check = notification['event_id'].split(':', 2)
|
45
|
-
|
46
|
-
entity_check = Flapjack::Data::EntityCheck.for_event_id(notification['event_id'],
|
47
|
-
:redis => ::Resque.redis)
|
48
|
-
|
49
|
-
@in_unscheduled_maintenance = entity_check.in_scheduled_maintenance?
|
50
|
-
@in_scheduled_maintenance = entity_check.in_unscheduled_maintenance?
|
51
|
-
|
52
|
-
# FIXME: I can not get the entity_check.last_change to work in this context (Resque)
|
53
|
-
# it always returns nil, despite entity_check being a good looking EntityCheck object
|
54
|
-
# and all ...
|
55
|
-
if lc = entity_check.last_change
|
56
|
-
duration = (Time.now.to_i - lc)
|
57
|
-
@duration = (duration && duration > 40) ? duration : nil
|
58
|
-
end
|
33
|
+
prepare( notification )
|
34
|
+
deliver( notification )
|
35
|
+
end
|
59
36
|
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
37
|
+
def prepare(notification)
|
38
|
+
@logger.debug "Woo, got a notification to send out: #{notification.inspect}"
|
39
|
+
|
40
|
+
# The instance variables are referenced by the templates, which
|
41
|
+
# share the current binding context
|
42
|
+
@notification_type = notification['notification_type']
|
43
|
+
@contact_first_name = notification['contact_first_name']
|
44
|
+
@contact_last_name = notification['contact_last_name']
|
45
|
+
@state = notification['state']
|
46
|
+
@duration = notification['state_duration']
|
47
|
+
@summary = notification['summary']
|
48
|
+
@last_state = notification['last_state']
|
49
|
+
@last_summary = notification['last_summary']
|
50
|
+
@details = notification['details']
|
51
|
+
@time = notification['time']
|
52
|
+
@entity_name, @check = notification['event_id'].split(':', 2)
|
53
|
+
|
54
|
+
entity_check = Flapjack::Data::EntityCheck.for_event_id(notification['event_id'],
|
55
|
+
:redis => @redis)
|
56
|
+
|
57
|
+
@in_unscheduled_maintenance = entity_check.in_scheduled_maintenance?
|
58
|
+
@in_scheduled_maintenance = entity_check.in_unscheduled_maintenance?
|
59
|
+
|
60
|
+
headline_map = {'problem' => 'Problem: ',
|
61
|
+
'recovery' => 'Recovery: ',
|
62
|
+
'acknowledgement' => 'Acknowledgement: ',
|
63
|
+
'test' => 'Test Notification: ',
|
64
|
+
'unknown' => ''
|
65
|
+
}
|
66
|
+
|
67
|
+
headline = headline_map[@notification_type] || ''
|
68
|
+
|
69
|
+
@subject = "#{headline}'#{@check}' on #{@entity_name}"
|
70
|
+
@subject += " is #{@state.upcase}" unless ['acknowledgement', 'test'].include?(@notification_type)
|
71
|
+
rescue => e
|
72
|
+
@logger.error "Error preparing email to #{m_to}: #{e.class}: #{e.message}"
|
73
|
+
@logger.error e.backtrace.join("\n")
|
74
|
+
raise
|
75
|
+
end
|
76
76
|
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
end
|
77
|
+
def deliver(notification)
|
78
|
+
host = @smtp_config ? @smtp_config['host'] : nil
|
79
|
+
port = @smtp_config ? @smtp_config['port'] : nil
|
80
|
+
starttls = @smtp_config ? !! @smtp_config['starttls'] : nil
|
81
|
+
if @smtp_config
|
82
|
+
if auth_config = @smtp_config['auth']
|
83
|
+
auth = {}
|
84
|
+
auth[:type] = auth_config['type'].to_sym || :plain
|
85
|
+
auth[:username] = auth_config['username']
|
86
|
+
auth[:password] = auth_config['password']
|
88
87
|
end
|
88
|
+
end
|
89
89
|
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
90
|
+
fqdn = `/bin/hostname -f`.chomp
|
91
|
+
m_from = "flapjack@#{fqdn}"
|
92
|
+
@logger.debug("flapjack_mailer: set from to #{m_from}")
|
93
|
+
m_reply_to = m_from
|
94
|
+
m_to = notification['address']
|
95
|
+
|
96
|
+
@logger.debug("sending Flapjack::Notification::Email " +
|
97
|
+
"#{notification['id']} to: #{m_to} subject: #{@subject}")
|
98
|
+
|
99
|
+
mail = prepare_email(:subject => @subject,
|
100
|
+
:from => m_from,
|
101
|
+
:to => m_to)
|
102
|
+
|
103
|
+
smtp_args = {:from => m_from,
|
104
|
+
:to => m_to,
|
105
|
+
:content => "#{mail.to_s}\r\n.\r\n",
|
106
|
+
:domain => fqdn,
|
107
|
+
:host => host || 'localhost',
|
108
|
+
:port => port || 25,
|
109
|
+
:starttls => starttls}
|
110
|
+
smtp_args.merge!(:auth => auth) if auth
|
111
|
+
email = EM::P::SmtpClient.send(smtp_args)
|
112
|
+
|
113
|
+
response = EM::Synchrony.sync(email)
|
114
|
+
|
115
|
+
# http://tools.ietf.org/html/rfc821#page-36 SMTP response codes
|
116
|
+
if response && response.respond_to?(:code) &&
|
117
|
+
((response.code == 250) || (response.code == 251))
|
118
|
+
@logger.info "Email sending succeeded"
|
119
|
+
@sent += 1
|
120
|
+
else
|
121
|
+
@logger.error "Email sending failed"
|
122
|
+
end
|
123
123
|
|
124
|
-
|
124
|
+
@logger.info "Email response: #{response.inspect}"
|
125
125
|
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
end
|
126
|
+
rescue => e
|
127
|
+
@logger.error "Error delivering email to #{m_to}: #{e.class}: #{e.message}"
|
128
|
+
@logger.error e.backtrace.join("\n")
|
129
|
+
raise
|
131
130
|
end
|
132
131
|
|
133
|
-
|
132
|
+
private
|
134
133
|
|
135
|
-
|
134
|
+
def prepare_email(opts = {})
|
136
135
|
|
137
|
-
|
136
|
+
text_template = ERB.new(File.read(File.dirname(__FILE__) +
|
137
|
+
'/email/alert.text.erb'), nil, '-')
|
138
138
|
|
139
|
-
|
140
|
-
|
139
|
+
html_template = ERB.new(File.read(File.dirname(__FILE__) +
|
140
|
+
'/email/alert.html.erb'), nil, '-')
|
141
141
|
|
142
|
-
|
143
|
-
'/email/alert.html.erb'))
|
142
|
+
bnd = binding
|
144
143
|
|
145
|
-
|
144
|
+
mail = Mail.new do
|
145
|
+
from opts[:from]
|
146
|
+
to opts[:to]
|
147
|
+
subject opts[:subject]
|
148
|
+
reply_to opts[:from]
|
146
149
|
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
from opts[:from]
|
151
|
-
to opts[:to]
|
152
|
-
subject opts[:subject]
|
153
|
-
reply_to opts[:from]
|
150
|
+
text_part do
|
151
|
+
body text_template.result(bnd)
|
152
|
+
end
|
154
153
|
|
155
|
-
|
156
|
-
|
154
|
+
html_part do
|
155
|
+
content_type 'text/html; charset=UTF-8'
|
156
|
+
body html_template.result(bnd)
|
157
|
+
end
|
157
158
|
end
|
158
159
|
|
159
|
-
html_part do
|
160
|
-
content_type 'text/html; charset=UTF-8'
|
161
|
-
body html_template.result(bnd)
|
162
|
-
end
|
163
160
|
end
|
164
161
|
end
|
165
162
|
|
@@ -18,56 +18,56 @@
|
|
18
18
|
<table>
|
19
19
|
<tbody>
|
20
20
|
<tr>
|
21
|
-
<td>Entity</td>
|
21
|
+
<td><strong>Entity</strong></td>
|
22
22
|
<td><%= @entity_name %></td>
|
23
23
|
</tr>
|
24
24
|
|
25
25
|
<tr>
|
26
|
-
<td>Check</td>
|
26
|
+
<td><strong>Check</strong></td>
|
27
27
|
<td><%= @check %></td>
|
28
28
|
</tr>
|
29
29
|
|
30
30
|
<tr>
|
31
|
-
<td>State</td>
|
31
|
+
<td><strong>State</strong></td>
|
32
32
|
<td><%= @state.upcase %></td>
|
33
33
|
</tr>
|
34
34
|
|
35
35
|
<tr>
|
36
|
-
<td>Summary</td>
|
36
|
+
<td><strong>Summary</strong></td>
|
37
37
|
<td><%= @summary %></td>
|
38
38
|
</tr>
|
39
39
|
|
40
40
|
<% if @details %>
|
41
41
|
<tr>
|
42
|
-
<td>Details</td>
|
42
|
+
<td><strong>Details</strong></td>
|
43
43
|
<td><%= @details %></td>
|
44
44
|
</tr>
|
45
45
|
<% end %>
|
46
46
|
|
47
47
|
<% if @time %>
|
48
48
|
<tr>
|
49
|
-
<td>Time</td>
|
49
|
+
<td><strong>Time</strong></td>
|
50
50
|
<td><%= Time.at(@time.to_i).to_s %></td>
|
51
51
|
</tr>
|
52
52
|
<% end %>
|
53
53
|
|
54
|
-
<% if @duration %>
|
54
|
+
<% if @duration && @duration > 40 %>
|
55
55
|
<tr>
|
56
|
-
<td>Duration</td>
|
56
|
+
<td><strong>Duration</strong></td>
|
57
57
|
<td><%= ChronicDuration.output(@duration) %></td>
|
58
58
|
</tr>
|
59
59
|
<% end %>
|
60
60
|
|
61
61
|
<% if @last_state %>
|
62
62
|
<tr>
|
63
|
-
<td>Previous
|
64
|
-
<td><%= @last_state %></td>
|
63
|
+
<td><strong>Previous State</strong></td>
|
64
|
+
<td><%= @last_state.upcase %></td>
|
65
65
|
</tr>
|
66
66
|
<% end %>
|
67
67
|
|
68
68
|
<% if @last_summary %>
|
69
69
|
<tr>
|
70
|
-
<td>Previous
|
70
|
+
<td><strong>Previous Summary</strong></td>
|
71
71
|
<td><%= @last_summary %></td>
|
72
72
|
</tr>
|
73
73
|
<% end %>
|
@@ -2,12 +2,25 @@ Hi <%= @contact_first_name %>
|
|
2
2
|
|
3
3
|
Monitoring has detected the following:
|
4
4
|
|
5
|
-
Entity:
|
6
|
-
Check:
|
7
|
-
State:
|
8
|
-
Summary:
|
9
|
-
|
10
|
-
<%= @
|
5
|
+
Entity: <%= @entity_name %>
|
6
|
+
Check: <%= @check %>
|
7
|
+
State: <%= @state.upcase %>
|
8
|
+
Summary: <%= @summary %>
|
9
|
+
<% if @details -%>
|
10
|
+
Details: <%= @details %>
|
11
|
+
<% end -%>
|
12
|
+
<% if @time -%>
|
13
|
+
Time: <%= Time.at(@time.to_i).to_s %>
|
14
|
+
<% end -%>
|
15
|
+
<% if @duration && @duration > 40 -%>
|
16
|
+
Duration: <%= ChronicDuration.output(@duration) %>
|
17
|
+
<% end -%>
|
18
|
+
<% if @last_state -%>
|
19
|
+
Previous State: <%= @last_state.upcase %>
|
20
|
+
<% end -%>
|
21
|
+
<% if @last_summary -%>
|
22
|
+
Previous Summary: <%= @last_summary %>
|
23
|
+
<% end -%>
|
11
24
|
|
12
25
|
Cheers,
|
13
26
|
Flapjack
|