flapjack 0.7.22 → 0.7.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/CHANGELOG.md +19 -0
- data/bin/flapjack +3 -1
- data/bin/flapjack-nagios-receiver +5 -4
- data/bin/receive-events +2 -2
- data/features/events.feature +101 -95
- data/features/notification_rules.feature +36 -4
- data/features/steps/notifications_steps.rb +4 -0
- data/flapjack.gemspec +3 -2
- data/lib/flapjack/coordinator.rb +8 -6
- data/lib/flapjack/data/entity_check.rb +20 -13
- data/lib/flapjack/data/event.rb +4 -7
- data/lib/flapjack/data/notification.rb +63 -45
- data/lib/flapjack/filters/acknowledgement.rb +26 -24
- data/lib/flapjack/filters/delays.rb +46 -42
- data/lib/flapjack/filters/ok.rb +31 -34
- data/lib/flapjack/filters/scheduled_maintenance.rb +2 -2
- data/lib/flapjack/filters/unscheduled_maintenance.rb +2 -3
- data/lib/flapjack/gateways/email.rb +111 -114
- data/lib/flapjack/gateways/email/alert.html.erb +11 -11
- data/lib/flapjack/gateways/email/alert.text.erb +19 -6
- data/lib/flapjack/gateways/sms_messagenet.rb +15 -5
- data/lib/flapjack/gateways/web.rb +3 -4
- data/lib/flapjack/gateways/web/public/css/flapjack.css +0 -2
- data/lib/flapjack/gateways/web/public/img/flapjack-favicon-32-16.ico +0 -0
- data/lib/flapjack/gateways/web/public/img/flapjack-favicon-64-32-24-16.ico +0 -0
- data/lib/flapjack/gateways/web/public/img/flapjack-transparent-300.png +0 -0
- data/lib/flapjack/gateways/web/public/img/flapjack-transparent-350-400.png +0 -0
- data/lib/flapjack/gateways/web/views/_head.html.erb +1 -0
- data/lib/flapjack/gateways/web/views/index.html.erb +1 -1
- data/lib/flapjack/notifier.rb +2 -3
- data/lib/flapjack/pikelet.rb +5 -4
- data/lib/flapjack/processor.rb +39 -27
- data/lib/flapjack/version.rb +1 -1
- data/spec/lib/flapjack/data/entity_check_spec.rb +5 -0
- data/spec/lib/flapjack/data/event_spec.rb +0 -1
- data/spec/lib/flapjack/gateways/email_spec.rb +5 -9
- data/spec/lib/flapjack/gateways/sms_messagenet.spec.rb +80 -1
- data/spec/lib/flapjack/gateways/web_spec.rb +1 -1
- data/spec/lib/flapjack/pikelet_spec.rb +4 -3
- data/spec/lib/flapjack/processor_spec.rb +0 -1
- metadata +28 -11
- data/lib/flapjack/filters/detect_mass_client_failures.rb +0 -44
- data/spec/lib/flapjack/filters/detect_mass_client_failures_spec.rb +0 -6
data/lib/flapjack/filters/ok.rb
CHANGED
@@ -5,46 +5,43 @@ require 'flapjack/filters/base'
|
|
5
5
|
module Flapjack
|
6
6
|
module Filters
|
7
7
|
|
8
|
+
# * If the service event's state is ok and there is unscheduled maintenance set, end the unscheduled
|
9
|
+
# maintenance
|
8
10
|
# * If the service event’s state is ok and the previous state was ok, don’t alert
|
9
|
-
# * If the service event
|
10
|
-
# * If the service event's state is ok and the previous
|
11
|
-
# seconds, don't alert
|
12
|
-
# * If the service event's state is ok and there is unscheduled downtime set, end the unscheduled
|
13
|
-
# downtime
|
11
|
+
# * If the service event’s state is ok and there's never been a notification, don't alert
|
12
|
+
# * If the service event's state is ok and the previous notification was a recovery or ok, don't alert
|
14
13
|
class Ok
|
15
14
|
include Base
|
16
15
|
|
17
|
-
def block?(event)
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
if event.previous_state == 'ok'
|
22
|
-
@logger.debug("Filter: Ok: existing state was ok, and the previous state was ok, so blocking")
|
23
|
-
result = true
|
24
|
-
end
|
25
|
-
|
26
|
-
entity_check = Flapjack::Data::EntityCheck.for_event_id(event.id, :redis => @redis)
|
27
|
-
|
28
|
-
last_notification = entity_check.last_notification
|
29
|
-
@logger.debug("Filter: Ok: last notification: #{last_notification.inspect}")
|
30
|
-
if last_notification[:type] == 'recovery'
|
31
|
-
@logger.debug("Filter: Ok: last notification was a recovery, so blocking")
|
32
|
-
result = true
|
33
|
-
end
|
34
|
-
|
35
|
-
if event.previous_state != 'ok'
|
36
|
-
if event.previous_state_duration < 30
|
37
|
-
@logger.debug("Filter: Ok: previous non ok state was for less than 30 seconds, so blocking")
|
38
|
-
result = true
|
39
|
-
end
|
40
|
-
end
|
41
|
-
|
42
|
-
# end any unscheduled downtime
|
43
|
-
entity_check.end_unscheduled_maintenance(Time.now.to_i)
|
16
|
+
def block?(event, entity_check, previous_state)
|
17
|
+
unless event.ok?
|
18
|
+
@logger.debug("Filter: Ok: pass")
|
19
|
+
return false
|
44
20
|
end
|
45
21
|
|
46
|
-
|
47
|
-
|
22
|
+
entity_check.end_unscheduled_maintenance(Time.now.to_i)
|
23
|
+
|
24
|
+
if previous_state == 'ok'
|
25
|
+
@logger.debug("Filter: Ok: block - previous state was ok, so blocking")
|
26
|
+
return true
|
27
|
+
end
|
28
|
+
|
29
|
+
last_notification = entity_check.last_notification
|
30
|
+
@logger.debug("Filter: Ok: last notification: #{last_notification.inspect}")
|
31
|
+
|
32
|
+
unless last_notification[:type]
|
33
|
+
@logger.debug("Filter: Ok: block - last notification type is nil (never notified)")
|
34
|
+
return true
|
35
|
+
end
|
36
|
+
|
37
|
+
if [:recovery, :ok].include?(last_notification[:type])
|
38
|
+
@logger.debug("Filter: Ok: block - last notification was a recovery")
|
39
|
+
return true
|
40
|
+
end
|
41
|
+
|
42
|
+
@logger.debug("Filter: Ok: previous_state: #{previous_state}")
|
43
|
+
@logger.debug("Filter: Ok: pass")
|
44
|
+
false
|
48
45
|
end
|
49
46
|
end
|
50
47
|
end
|
@@ -7,8 +7,8 @@ module Flapjack
|
|
7
7
|
class ScheduledMaintenance
|
8
8
|
include Base
|
9
9
|
|
10
|
-
def block?(event)
|
11
|
-
result =
|
10
|
+
def block?(event, entity_check, previous_state)
|
11
|
+
result = entity_check.in_scheduled_maintenance?
|
12
12
|
@logger.debug("Filter: Scheduled Maintenance: #{result ? "block" : "pass"}")
|
13
13
|
result
|
14
14
|
end
|
@@ -7,9 +7,8 @@ module Flapjack
|
|
7
7
|
class UnscheduledMaintenance
|
8
8
|
include Base
|
9
9
|
|
10
|
-
def block?(event)
|
11
|
-
result =
|
12
|
-
!event.acknowledgement?
|
10
|
+
def block?(event, entity_check, previous_state)
|
11
|
+
result = entity_check.in_unscheduled_maintenance? && !event.acknowledgement?
|
13
12
|
@logger.debug("Filter: Unscheduled Maintenance: #{result ? "block" : "pass"}")
|
14
13
|
result
|
15
14
|
end
|
@@ -28,138 +28,135 @@ module Flapjack
|
|
28
28
|
@sent = 0
|
29
29
|
end
|
30
30
|
|
31
|
+
# TODO refactor to remove complexity
|
31
32
|
def perform(notification)
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
@notification_type = notification['notification_type']
|
36
|
-
@contact_first_name = notification['contact_first_name']
|
37
|
-
@contact_last_name = notification['contact_last_name']
|
38
|
-
@state = notification['state']
|
39
|
-
@summary = notification['summary']
|
40
|
-
@last_state = notification['last_state']
|
41
|
-
@last_summary = notification['last_summary']
|
42
|
-
@details = notification['details']
|
43
|
-
@time = notification['time']
|
44
|
-
@entity_name, @check = notification['event_id'].split(':', 2)
|
45
|
-
|
46
|
-
entity_check = Flapjack::Data::EntityCheck.for_event_id(notification['event_id'],
|
47
|
-
:redis => ::Resque.redis)
|
48
|
-
|
49
|
-
@in_unscheduled_maintenance = entity_check.in_scheduled_maintenance?
|
50
|
-
@in_scheduled_maintenance = entity_check.in_unscheduled_maintenance?
|
51
|
-
|
52
|
-
# FIXME: I can not get the entity_check.last_change to work in this context (Resque)
|
53
|
-
# it always returns nil, despite entity_check being a good looking EntityCheck object
|
54
|
-
# and all ...
|
55
|
-
if lc = entity_check.last_change
|
56
|
-
duration = (Time.now.to_i - lc)
|
57
|
-
@duration = (duration && duration > 40) ? duration : nil
|
58
|
-
end
|
33
|
+
prepare( notification )
|
34
|
+
deliver( notification )
|
35
|
+
end
|
59
36
|
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
37
|
+
def prepare(notification)
|
38
|
+
@logger.debug "Woo, got a notification to send out: #{notification.inspect}"
|
39
|
+
|
40
|
+
# The instance variables are referenced by the templates, which
|
41
|
+
# share the current binding context
|
42
|
+
@notification_type = notification['notification_type']
|
43
|
+
@contact_first_name = notification['contact_first_name']
|
44
|
+
@contact_last_name = notification['contact_last_name']
|
45
|
+
@state = notification['state']
|
46
|
+
@duration = notification['state_duration']
|
47
|
+
@summary = notification['summary']
|
48
|
+
@last_state = notification['last_state']
|
49
|
+
@last_summary = notification['last_summary']
|
50
|
+
@details = notification['details']
|
51
|
+
@time = notification['time']
|
52
|
+
@entity_name, @check = notification['event_id'].split(':', 2)
|
53
|
+
|
54
|
+
entity_check = Flapjack::Data::EntityCheck.for_event_id(notification['event_id'],
|
55
|
+
:redis => @redis)
|
56
|
+
|
57
|
+
@in_unscheduled_maintenance = entity_check.in_scheduled_maintenance?
|
58
|
+
@in_scheduled_maintenance = entity_check.in_unscheduled_maintenance?
|
59
|
+
|
60
|
+
headline_map = {'problem' => 'Problem: ',
|
61
|
+
'recovery' => 'Recovery: ',
|
62
|
+
'acknowledgement' => 'Acknowledgement: ',
|
63
|
+
'test' => 'Test Notification: ',
|
64
|
+
'unknown' => ''
|
65
|
+
}
|
66
|
+
|
67
|
+
headline = headline_map[@notification_type] || ''
|
68
|
+
|
69
|
+
@subject = "#{headline}'#{@check}' on #{@entity_name}"
|
70
|
+
@subject += " is #{@state.upcase}" unless ['acknowledgement', 'test'].include?(@notification_type)
|
71
|
+
rescue => e
|
72
|
+
@logger.error "Error preparing email to #{m_to}: #{e.class}: #{e.message}"
|
73
|
+
@logger.error e.backtrace.join("\n")
|
74
|
+
raise
|
75
|
+
end
|
76
76
|
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
end
|
77
|
+
def deliver(notification)
|
78
|
+
host = @smtp_config ? @smtp_config['host'] : nil
|
79
|
+
port = @smtp_config ? @smtp_config['port'] : nil
|
80
|
+
starttls = @smtp_config ? !! @smtp_config['starttls'] : nil
|
81
|
+
if @smtp_config
|
82
|
+
if auth_config = @smtp_config['auth']
|
83
|
+
auth = {}
|
84
|
+
auth[:type] = auth_config['type'].to_sym || :plain
|
85
|
+
auth[:username] = auth_config['username']
|
86
|
+
auth[:password] = auth_config['password']
|
88
87
|
end
|
88
|
+
end
|
89
89
|
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
90
|
+
fqdn = `/bin/hostname -f`.chomp
|
91
|
+
m_from = "flapjack@#{fqdn}"
|
92
|
+
@logger.debug("flapjack_mailer: set from to #{m_from}")
|
93
|
+
m_reply_to = m_from
|
94
|
+
m_to = notification['address']
|
95
|
+
|
96
|
+
@logger.debug("sending Flapjack::Notification::Email " +
|
97
|
+
"#{notification['id']} to: #{m_to} subject: #{@subject}")
|
98
|
+
|
99
|
+
mail = prepare_email(:subject => @subject,
|
100
|
+
:from => m_from,
|
101
|
+
:to => m_to)
|
102
|
+
|
103
|
+
smtp_args = {:from => m_from,
|
104
|
+
:to => m_to,
|
105
|
+
:content => "#{mail.to_s}\r\n.\r\n",
|
106
|
+
:domain => fqdn,
|
107
|
+
:host => host || 'localhost',
|
108
|
+
:port => port || 25,
|
109
|
+
:starttls => starttls}
|
110
|
+
smtp_args.merge!(:auth => auth) if auth
|
111
|
+
email = EM::P::SmtpClient.send(smtp_args)
|
112
|
+
|
113
|
+
response = EM::Synchrony.sync(email)
|
114
|
+
|
115
|
+
# http://tools.ietf.org/html/rfc821#page-36 SMTP response codes
|
116
|
+
if response && response.respond_to?(:code) &&
|
117
|
+
((response.code == 250) || (response.code == 251))
|
118
|
+
@logger.info "Email sending succeeded"
|
119
|
+
@sent += 1
|
120
|
+
else
|
121
|
+
@logger.error "Email sending failed"
|
122
|
+
end
|
123
123
|
|
124
|
-
|
124
|
+
@logger.info "Email response: #{response.inspect}"
|
125
125
|
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
end
|
126
|
+
rescue => e
|
127
|
+
@logger.error "Error delivering email to #{m_to}: #{e.class}: #{e.message}"
|
128
|
+
@logger.error e.backtrace.join("\n")
|
129
|
+
raise
|
131
130
|
end
|
132
131
|
|
133
|
-
|
132
|
+
private
|
134
133
|
|
135
|
-
|
134
|
+
def prepare_email(opts = {})
|
136
135
|
|
137
|
-
|
136
|
+
text_template = ERB.new(File.read(File.dirname(__FILE__) +
|
137
|
+
'/email/alert.text.erb'), nil, '-')
|
138
138
|
|
139
|
-
|
140
|
-
|
139
|
+
html_template = ERB.new(File.read(File.dirname(__FILE__) +
|
140
|
+
'/email/alert.html.erb'), nil, '-')
|
141
141
|
|
142
|
-
|
143
|
-
'/email/alert.html.erb'))
|
142
|
+
bnd = binding
|
144
143
|
|
145
|
-
|
144
|
+
mail = Mail.new do
|
145
|
+
from opts[:from]
|
146
|
+
to opts[:to]
|
147
|
+
subject opts[:subject]
|
148
|
+
reply_to opts[:from]
|
146
149
|
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
from opts[:from]
|
151
|
-
to opts[:to]
|
152
|
-
subject opts[:subject]
|
153
|
-
reply_to opts[:from]
|
150
|
+
text_part do
|
151
|
+
body text_template.result(bnd)
|
152
|
+
end
|
154
153
|
|
155
|
-
|
156
|
-
|
154
|
+
html_part do
|
155
|
+
content_type 'text/html; charset=UTF-8'
|
156
|
+
body html_template.result(bnd)
|
157
|
+
end
|
157
158
|
end
|
158
159
|
|
159
|
-
html_part do
|
160
|
-
content_type 'text/html; charset=UTF-8'
|
161
|
-
body html_template.result(bnd)
|
162
|
-
end
|
163
160
|
end
|
164
161
|
end
|
165
162
|
|
@@ -18,56 +18,56 @@
|
|
18
18
|
<table>
|
19
19
|
<tbody>
|
20
20
|
<tr>
|
21
|
-
<td>Entity</td>
|
21
|
+
<td><strong>Entity</strong></td>
|
22
22
|
<td><%= @entity_name %></td>
|
23
23
|
</tr>
|
24
24
|
|
25
25
|
<tr>
|
26
|
-
<td>Check</td>
|
26
|
+
<td><strong>Check</strong></td>
|
27
27
|
<td><%= @check %></td>
|
28
28
|
</tr>
|
29
29
|
|
30
30
|
<tr>
|
31
|
-
<td>State</td>
|
31
|
+
<td><strong>State</strong></td>
|
32
32
|
<td><%= @state.upcase %></td>
|
33
33
|
</tr>
|
34
34
|
|
35
35
|
<tr>
|
36
|
-
<td>Summary</td>
|
36
|
+
<td><strong>Summary</strong></td>
|
37
37
|
<td><%= @summary %></td>
|
38
38
|
</tr>
|
39
39
|
|
40
40
|
<% if @details %>
|
41
41
|
<tr>
|
42
|
-
<td>Details</td>
|
42
|
+
<td><strong>Details</strong></td>
|
43
43
|
<td><%= @details %></td>
|
44
44
|
</tr>
|
45
45
|
<% end %>
|
46
46
|
|
47
47
|
<% if @time %>
|
48
48
|
<tr>
|
49
|
-
<td>Time</td>
|
49
|
+
<td><strong>Time</strong></td>
|
50
50
|
<td><%= Time.at(@time.to_i).to_s %></td>
|
51
51
|
</tr>
|
52
52
|
<% end %>
|
53
53
|
|
54
|
-
<% if @duration %>
|
54
|
+
<% if @duration && @duration > 40 %>
|
55
55
|
<tr>
|
56
|
-
<td>Duration</td>
|
56
|
+
<td><strong>Duration</strong></td>
|
57
57
|
<td><%= ChronicDuration.output(@duration) %></td>
|
58
58
|
</tr>
|
59
59
|
<% end %>
|
60
60
|
|
61
61
|
<% if @last_state %>
|
62
62
|
<tr>
|
63
|
-
<td>Previous
|
64
|
-
<td><%= @last_state %></td>
|
63
|
+
<td><strong>Previous State</strong></td>
|
64
|
+
<td><%= @last_state.upcase %></td>
|
65
65
|
</tr>
|
66
66
|
<% end %>
|
67
67
|
|
68
68
|
<% if @last_summary %>
|
69
69
|
<tr>
|
70
|
-
<td>Previous
|
70
|
+
<td><strong>Previous Summary</strong></td>
|
71
71
|
<td><%= @last_summary %></td>
|
72
72
|
</tr>
|
73
73
|
<% end %>
|
@@ -2,12 +2,25 @@ Hi <%= @contact_first_name %>
|
|
2
2
|
|
3
3
|
Monitoring has detected the following:
|
4
4
|
|
5
|
-
Entity:
|
6
|
-
Check:
|
7
|
-
State:
|
8
|
-
Summary:
|
9
|
-
|
10
|
-
<%= @
|
5
|
+
Entity: <%= @entity_name %>
|
6
|
+
Check: <%= @check %>
|
7
|
+
State: <%= @state.upcase %>
|
8
|
+
Summary: <%= @summary %>
|
9
|
+
<% if @details -%>
|
10
|
+
Details: <%= @details %>
|
11
|
+
<% end -%>
|
12
|
+
<% if @time -%>
|
13
|
+
Time: <%= Time.at(@time.to_i).to_s %>
|
14
|
+
<% end -%>
|
15
|
+
<% if @duration && @duration > 40 -%>
|
16
|
+
Duration: <%= ChronicDuration.output(@duration) %>
|
17
|
+
<% end -%>
|
18
|
+
<% if @last_state -%>
|
19
|
+
Previous State: <%= @last_state.upcase %>
|
20
|
+
<% end -%>
|
21
|
+
<% if @last_summary -%>
|
22
|
+
Previous Summary: <%= @last_summary %>
|
23
|
+
<% end -%>
|
11
24
|
|
12
25
|
Cheers,
|
13
26
|
Flapjack
|