flapjack 0.7.22 → 0.7.25

Sign up to get free protection for your applications and to get access to all the features.
Files changed (43) hide show
  1. data/CHANGELOG.md +19 -0
  2. data/bin/flapjack +3 -1
  3. data/bin/flapjack-nagios-receiver +5 -4
  4. data/bin/receive-events +2 -2
  5. data/features/events.feature +101 -95
  6. data/features/notification_rules.feature +36 -4
  7. data/features/steps/notifications_steps.rb +4 -0
  8. data/flapjack.gemspec +3 -2
  9. data/lib/flapjack/coordinator.rb +8 -6
  10. data/lib/flapjack/data/entity_check.rb +20 -13
  11. data/lib/flapjack/data/event.rb +4 -7
  12. data/lib/flapjack/data/notification.rb +63 -45
  13. data/lib/flapjack/filters/acknowledgement.rb +26 -24
  14. data/lib/flapjack/filters/delays.rb +46 -42
  15. data/lib/flapjack/filters/ok.rb +31 -34
  16. data/lib/flapjack/filters/scheduled_maintenance.rb +2 -2
  17. data/lib/flapjack/filters/unscheduled_maintenance.rb +2 -3
  18. data/lib/flapjack/gateways/email.rb +111 -114
  19. data/lib/flapjack/gateways/email/alert.html.erb +11 -11
  20. data/lib/flapjack/gateways/email/alert.text.erb +19 -6
  21. data/lib/flapjack/gateways/sms_messagenet.rb +15 -5
  22. data/lib/flapjack/gateways/web.rb +3 -4
  23. data/lib/flapjack/gateways/web/public/css/flapjack.css +0 -2
  24. data/lib/flapjack/gateways/web/public/img/flapjack-favicon-32-16.ico +0 -0
  25. data/lib/flapjack/gateways/web/public/img/flapjack-favicon-64-32-24-16.ico +0 -0
  26. data/lib/flapjack/gateways/web/public/img/flapjack-transparent-300.png +0 -0
  27. data/lib/flapjack/gateways/web/public/img/flapjack-transparent-350-400.png +0 -0
  28. data/lib/flapjack/gateways/web/views/_head.html.erb +1 -0
  29. data/lib/flapjack/gateways/web/views/index.html.erb +1 -1
  30. data/lib/flapjack/notifier.rb +2 -3
  31. data/lib/flapjack/pikelet.rb +5 -4
  32. data/lib/flapjack/processor.rb +39 -27
  33. data/lib/flapjack/version.rb +1 -1
  34. data/spec/lib/flapjack/data/entity_check_spec.rb +5 -0
  35. data/spec/lib/flapjack/data/event_spec.rb +0 -1
  36. data/spec/lib/flapjack/gateways/email_spec.rb +5 -9
  37. data/spec/lib/flapjack/gateways/sms_messagenet.spec.rb +80 -1
  38. data/spec/lib/flapjack/gateways/web_spec.rb +1 -1
  39. data/spec/lib/flapjack/pikelet_spec.rb +4 -3
  40. data/spec/lib/flapjack/processor_spec.rb +0 -1
  41. metadata +28 -11
  42. data/lib/flapjack/filters/detect_mass_client_failures.rb +0 -44
  43. data/spec/lib/flapjack/filters/detect_mass_client_failures_spec.rb +0 -6
@@ -5,46 +5,43 @@ require 'flapjack/filters/base'
5
5
  module Flapjack
6
6
  module Filters
7
7
 
8
+ # * If the service event's state is ok and there is unscheduled maintenance set, end the unscheduled
9
+ # maintenance
8
10
  # * If the service event’s state is ok and the previous state was ok, don’t alert
9
- # * If the service event's state is ok and the previous notification was a recovery, don't alert
10
- # * If the service event's state is ok and the previous state was not ok and for less than 30
11
- # seconds, don't alert
12
- # * If the service event's state is ok and there is unscheduled downtime set, end the unscheduled
13
- # downtime
11
+ # * If the service events state is ok and there's never been a notification, don't alert
12
+ # * If the service event's state is ok and the previous notification was a recovery or ok, don't alert
14
13
  class Ok
15
14
  include Base
16
15
 
17
- def block?(event)
18
- result = false
19
-
20
- if event.ok?
21
- if event.previous_state == 'ok'
22
- @logger.debug("Filter: Ok: existing state was ok, and the previous state was ok, so blocking")
23
- result = true
24
- end
25
-
26
- entity_check = Flapjack::Data::EntityCheck.for_event_id(event.id, :redis => @redis)
27
-
28
- last_notification = entity_check.last_notification
29
- @logger.debug("Filter: Ok: last notification: #{last_notification.inspect}")
30
- if last_notification[:type] == 'recovery'
31
- @logger.debug("Filter: Ok: last notification was a recovery, so blocking")
32
- result = true
33
- end
34
-
35
- if event.previous_state != 'ok'
36
- if event.previous_state_duration < 30
37
- @logger.debug("Filter: Ok: previous non ok state was for less than 30 seconds, so blocking")
38
- result = true
39
- end
40
- end
41
-
42
- # end any unscheduled downtime
43
- entity_check.end_unscheduled_maintenance(Time.now.to_i)
16
+ def block?(event, entity_check, previous_state)
17
+ unless event.ok?
18
+ @logger.debug("Filter: Ok: pass")
19
+ return false
44
20
  end
45
21
 
46
- @logger.debug("Filter: Ok: #{result ? "block" : "pass"}")
47
- result
22
+ entity_check.end_unscheduled_maintenance(Time.now.to_i)
23
+
24
+ if previous_state == 'ok'
25
+ @logger.debug("Filter: Ok: block - previous state was ok, so blocking")
26
+ return true
27
+ end
28
+
29
+ last_notification = entity_check.last_notification
30
+ @logger.debug("Filter: Ok: last notification: #{last_notification.inspect}")
31
+
32
+ unless last_notification[:type]
33
+ @logger.debug("Filter: Ok: block - last notification type is nil (never notified)")
34
+ return true
35
+ end
36
+
37
+ if [:recovery, :ok].include?(last_notification[:type])
38
+ @logger.debug("Filter: Ok: block - last notification was a recovery")
39
+ return true
40
+ end
41
+
42
+ @logger.debug("Filter: Ok: previous_state: #{previous_state}")
43
+ @logger.debug("Filter: Ok: pass")
44
+ false
48
45
  end
49
46
  end
50
47
  end
@@ -7,8 +7,8 @@ module Flapjack
7
7
  class ScheduledMaintenance
8
8
  include Base
9
9
 
10
- def block?(event)
11
- result = @redis.exists("#{event.id}:scheduled_maintenance")
10
+ def block?(event, entity_check, previous_state)
11
+ result = entity_check.in_scheduled_maintenance?
12
12
  @logger.debug("Filter: Scheduled Maintenance: #{result ? "block" : "pass"}")
13
13
  result
14
14
  end
@@ -7,9 +7,8 @@ module Flapjack
7
7
  class UnscheduledMaintenance
8
8
  include Base
9
9
 
10
- def block?(event)
11
- result = @redis.exists("#{event.id}:unscheduled_maintenance") &&
12
- !event.acknowledgement?
10
+ def block?(event, entity_check, previous_state)
11
+ result = entity_check.in_unscheduled_maintenance? && !event.acknowledgement?
13
12
  @logger.debug("Filter: Unscheduled Maintenance: #{result ? "block" : "pass"}")
14
13
  result
15
14
  end
@@ -28,138 +28,135 @@ module Flapjack
28
28
  @sent = 0
29
29
  end
30
30
 
31
+ # TODO refactor to remove complexity
31
32
  def perform(notification)
32
- begin
33
- @logger.debug "Woo, got a notification to send out: #{notification.inspect}"
34
-
35
- @notification_type = notification['notification_type']
36
- @contact_first_name = notification['contact_first_name']
37
- @contact_last_name = notification['contact_last_name']
38
- @state = notification['state']
39
- @summary = notification['summary']
40
- @last_state = notification['last_state']
41
- @last_summary = notification['last_summary']
42
- @details = notification['details']
43
- @time = notification['time']
44
- @entity_name, @check = notification['event_id'].split(':', 2)
45
-
46
- entity_check = Flapjack::Data::EntityCheck.for_event_id(notification['event_id'],
47
- :redis => ::Resque.redis)
48
-
49
- @in_unscheduled_maintenance = entity_check.in_scheduled_maintenance?
50
- @in_scheduled_maintenance = entity_check.in_unscheduled_maintenance?
51
-
52
- # FIXME: I can not get the entity_check.last_change to work in this context (Resque)
53
- # it always returns nil, despite entity_check being a good looking EntityCheck object
54
- # and all ...
55
- if lc = entity_check.last_change
56
- duration = (Time.now.to_i - lc)
57
- @duration = (duration && duration > 40) ? duration : nil
58
- end
33
+ prepare( notification )
34
+ deliver( notification )
35
+ end
59
36
 
60
- headline_map = {'problem' => 'Problem: ',
61
- 'recovery' => 'Recovery: ',
62
- 'acknowledgement' => 'Acknowledgement: ',
63
- 'test' => 'Test Notification: ',
64
- 'unknown' => ''
65
- }
66
-
67
- headline = headline_map[@notification_type] || ''
68
-
69
- @subject = "#{headline}'#{@check}' on #{@entity_name}"
70
- @subject += " is #{@state.upcase}" unless ['acknowledgement', 'test'].include?(@notification_type)
71
- rescue => e
72
- @logger.error "Error preparing email to #{m_to}: #{e.class}: #{e.message}"
73
- @logger.error e.backtrace.join("\n")
74
- raise
75
- end
37
+ def prepare(notification)
38
+ @logger.debug "Woo, got a notification to send out: #{notification.inspect}"
39
+
40
+ # The instance variables are referenced by the templates, which
41
+ # share the current binding context
42
+ @notification_type = notification['notification_type']
43
+ @contact_first_name = notification['contact_first_name']
44
+ @contact_last_name = notification['contact_last_name']
45
+ @state = notification['state']
46
+ @duration = notification['state_duration']
47
+ @summary = notification['summary']
48
+ @last_state = notification['last_state']
49
+ @last_summary = notification['last_summary']
50
+ @details = notification['details']
51
+ @time = notification['time']
52
+ @entity_name, @check = notification['event_id'].split(':', 2)
53
+
54
+ entity_check = Flapjack::Data::EntityCheck.for_event_id(notification['event_id'],
55
+ :redis => @redis)
56
+
57
+ @in_unscheduled_maintenance = entity_check.in_scheduled_maintenance?
58
+ @in_scheduled_maintenance = entity_check.in_unscheduled_maintenance?
59
+
60
+ headline_map = {'problem' => 'Problem: ',
61
+ 'recovery' => 'Recovery: ',
62
+ 'acknowledgement' => 'Acknowledgement: ',
63
+ 'test' => 'Test Notification: ',
64
+ 'unknown' => ''
65
+ }
66
+
67
+ headline = headline_map[@notification_type] || ''
68
+
69
+ @subject = "#{headline}'#{@check}' on #{@entity_name}"
70
+ @subject += " is #{@state.upcase}" unless ['acknowledgement', 'test'].include?(@notification_type)
71
+ rescue => e
72
+ @logger.error "Error preparing email to #{m_to}: #{e.class}: #{e.message}"
73
+ @logger.error e.backtrace.join("\n")
74
+ raise
75
+ end
76
76
 
77
- begin
78
- host = @smtp_config ? @smtp_config['host'] : nil
79
- port = @smtp_config ? @smtp_config['port'] : nil
80
- starttls = @smtp_config ? !! @smtp_config['starttls'] : nil
81
- if @smtp_config
82
- if auth_config = @smtp_config['auth']
83
- auth = {}
84
- auth[:type] = auth_config['type'].to_sym || :plain
85
- auth[:username] = auth_config['username']
86
- auth[:password] = auth_config['password']
87
- end
77
+ def deliver(notification)
78
+ host = @smtp_config ? @smtp_config['host'] : nil
79
+ port = @smtp_config ? @smtp_config['port'] : nil
80
+ starttls = @smtp_config ? !! @smtp_config['starttls'] : nil
81
+ if @smtp_config
82
+ if auth_config = @smtp_config['auth']
83
+ auth = {}
84
+ auth[:type] = auth_config['type'].to_sym || :plain
85
+ auth[:username] = auth_config['username']
86
+ auth[:password] = auth_config['password']
88
87
  end
88
+ end
89
89
 
90
- fqdn = `/bin/hostname -f`.chomp
91
- m_from = "flapjack@#{fqdn}"
92
- @logger.debug("flapjack_mailer: set from to #{m_from}")
93
- m_reply_to = m_from
94
- m_to = notification['address']
95
-
96
- @logger.debug("sending Flapjack::Notification::Email " +
97
- "#{notification['id']} to: #{m_to} subject: #{@subject}")
98
-
99
- mail = prepare_email(:subject => @subject,
100
- :from => m_from,
101
- :to => m_to)
102
-
103
- smtp_args = {:from => m_from,
104
- :to => m_to,
105
- :content => "#{mail.to_s}\r\n.\r\n",
106
- :domain => fqdn,
107
- :host => host || 'localhost',
108
- :port => port || 25,
109
- :starttls => starttls}
110
- smtp_args.merge!(:auth => auth) if auth
111
- email = EM::P::SmtpClient.send(smtp_args)
112
-
113
- response = EM::Synchrony.sync(email)
114
-
115
- # http://tools.ietf.org/html/rfc821#page-36 SMTP response codes
116
- if response && response.respond_to?(:code) &&
117
- ((response.code == 250) || (response.code == 251))
118
- @logger.info "Email sending succeeded"
119
- @sent += 1
120
- else
121
- @logger.error "Email sending failed"
122
- end
90
+ fqdn = `/bin/hostname -f`.chomp
91
+ m_from = "flapjack@#{fqdn}"
92
+ @logger.debug("flapjack_mailer: set from to #{m_from}")
93
+ m_reply_to = m_from
94
+ m_to = notification['address']
95
+
96
+ @logger.debug("sending Flapjack::Notification::Email " +
97
+ "#{notification['id']} to: #{m_to} subject: #{@subject}")
98
+
99
+ mail = prepare_email(:subject => @subject,
100
+ :from => m_from,
101
+ :to => m_to)
102
+
103
+ smtp_args = {:from => m_from,
104
+ :to => m_to,
105
+ :content => "#{mail.to_s}\r\n.\r\n",
106
+ :domain => fqdn,
107
+ :host => host || 'localhost',
108
+ :port => port || 25,
109
+ :starttls => starttls}
110
+ smtp_args.merge!(:auth => auth) if auth
111
+ email = EM::P::SmtpClient.send(smtp_args)
112
+
113
+ response = EM::Synchrony.sync(email)
114
+
115
+ # http://tools.ietf.org/html/rfc821#page-36 SMTP response codes
116
+ if response && response.respond_to?(:code) &&
117
+ ((response.code == 250) || (response.code == 251))
118
+ @logger.info "Email sending succeeded"
119
+ @sent += 1
120
+ else
121
+ @logger.error "Email sending failed"
122
+ end
123
123
 
124
- @logger.info "Email response: #{response.inspect}"
124
+ @logger.info "Email response: #{response.inspect}"
125
125
 
126
- rescue => e
127
- @logger.error "Error delivering email to #{m_to}: #{e.class}: #{e.message}"
128
- @logger.error e.backtrace.join("\n")
129
- raise
130
- end
126
+ rescue => e
127
+ @logger.error "Error delivering email to #{m_to}: #{e.class}: #{e.message}"
128
+ @logger.error e.backtrace.join("\n")
129
+ raise
131
130
  end
132
131
 
133
- end
132
+ private
134
133
 
135
- private
134
+ def prepare_email(opts = {})
136
135
 
137
- def self.prepare_email(opts = {})
136
+ text_template = ERB.new(File.read(File.dirname(__FILE__) +
137
+ '/email/alert.text.erb'), nil, '-')
138
138
 
139
- text_template = ERB.new(File.read(File.dirname(__FILE__) +
140
- '/email/alert.text.erb'))
139
+ html_template = ERB.new(File.read(File.dirname(__FILE__) +
140
+ '/email/alert.html.erb'), nil, '-')
141
141
 
142
- html_template = ERB.new(File.read(File.dirname(__FILE__) +
143
- '/email/alert.html.erb'))
142
+ bnd = binding
144
143
 
145
- bnd = binding
144
+ mail = Mail.new do
145
+ from opts[:from]
146
+ to opts[:to]
147
+ subject opts[:subject]
148
+ reply_to opts[:from]
146
149
 
147
- # this part is the only use of the mail gem -- maybe this can be done
148
- # using standard library calls instead?
149
- mail = Mail.new do
150
- from opts[:from]
151
- to opts[:to]
152
- subject opts[:subject]
153
- reply_to opts[:from]
150
+ text_part do
151
+ body text_template.result(bnd)
152
+ end
154
153
 
155
- text_part do
156
- body text_template.result(bnd)
154
+ html_part do
155
+ content_type 'text/html; charset=UTF-8'
156
+ body html_template.result(bnd)
157
+ end
157
158
  end
158
159
 
159
- html_part do
160
- content_type 'text/html; charset=UTF-8'
161
- body html_template.result(bnd)
162
- end
163
160
  end
164
161
  end
165
162
 
@@ -18,56 +18,56 @@
18
18
  <table>
19
19
  <tbody>
20
20
  <tr>
21
- <td>Entity</td>
21
+ <td><strong>Entity</strong></td>
22
22
  <td><%= @entity_name %></td>
23
23
  </tr>
24
24
 
25
25
  <tr>
26
- <td>Check</td>
26
+ <td><strong>Check</strong></td>
27
27
  <td><%= @check %></td>
28
28
  </tr>
29
29
 
30
30
  <tr>
31
- <td>State</td>
31
+ <td><strong>State</strong></td>
32
32
  <td><%= @state.upcase %></td>
33
33
  </tr>
34
34
 
35
35
  <tr>
36
- <td>Summary</td>
36
+ <td><strong>Summary</strong></td>
37
37
  <td><%= @summary %></td>
38
38
  </tr>
39
39
 
40
40
  <% if @details %>
41
41
  <tr>
42
- <td>Details</td>
42
+ <td><strong>Details</strong></td>
43
43
  <td><%= @details %></td>
44
44
  </tr>
45
45
  <% end %>
46
46
 
47
47
  <% if @time %>
48
48
  <tr>
49
- <td>Time</td>
49
+ <td><strong>Time</strong></td>
50
50
  <td><%= Time.at(@time.to_i).to_s %></td>
51
51
  </tr>
52
52
  <% end %>
53
53
 
54
- <% if @duration %>
54
+ <% if @duration && @duration > 40 %>
55
55
  <tr>
56
- <td>Duration</td>
56
+ <td><strong>Duration</strong></td>
57
57
  <td><%= ChronicDuration.output(@duration) %></td>
58
58
  </tr>
59
59
  <% end %>
60
60
 
61
61
  <% if @last_state %>
62
62
  <tr>
63
- <td>Previous state</td>
64
- <td><%= @last_state %></td>
63
+ <td><strong>Previous State</strong></td>
64
+ <td><%= @last_state.upcase %></td>
65
65
  </tr>
66
66
  <% end %>
67
67
 
68
68
  <% if @last_summary %>
69
69
  <tr>
70
- <td>Previous summary</td>
70
+ <td><strong>Previous Summary</strong></td>
71
71
  <td><%= @last_summary %></td>
72
72
  </tr>
73
73
  <% end %>
@@ -2,12 +2,25 @@ Hi <%= @contact_first_name %>
2
2
 
3
3
  Monitoring has detected the following:
4
4
 
5
- Entity: <%= @entity_name %>
6
- Check: <%= @check %>
7
- State: <%= @state %>
8
- Summary: <%= @summary %>
9
- <%= @time ? "Time: #{Time.at(@time.to_i).to_s}" : '' %>
10
- <%= @last_state ? "\nPrevious state: #{@last_state}" : '' %><%= @last_summary ? "\nPrevious summary: #{@last_summary}" : '' %>
5
+ Entity: <%= @entity_name %>
6
+ Check: <%= @check %>
7
+ State: <%= @state.upcase %>
8
+ Summary: <%= @summary %>
9
+ <% if @details -%>
10
+ Details: <%= @details %>
11
+ <% end -%>
12
+ <% if @time -%>
13
+ Time: <%= Time.at(@time.to_i).to_s %>
14
+ <% end -%>
15
+ <% if @duration && @duration > 40 -%>
16
+ Duration: <%= ChronicDuration.output(@duration) %>
17
+ <% end -%>
18
+ <% if @last_state -%>
19
+ Previous State: <%= @last_state.upcase %>
20
+ <% end -%>
21
+ <% if @last_summary -%>
22
+ Previous Summary: <%= @last_summary %>
23
+ <% end -%>
11
24
 
12
25
  Cheers,
13
26
  Flapjack