interferon 0.1.3 → 0.1.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/interferon/alert_dsl.rb +1 -1
- data/lib/interferon/destinations/datadog.rb +67 -14
- data/lib/interferon/version.rb +1 -1
- data/lib/interferon.rb +7 -62
- data/spec/lib/interferon_spec.rb +10 -10
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 1c7f74fc8e23da780fdf783cdb98387321d20513
|
4
|
+
data.tar.gz: 5aa81e287ab880c2d3e7af7bd96a035d41b3db21
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 82c25b9118d41092d6ab131dd7edba612f40e68d4959bd0849efb4d0d2483bd5056b806dfc1c6f7e21381badd83cfb1e4f4e62fba8497bb3e6c52c4f33479dd2
|
7
|
+
data.tar.gz: 24740188b929de430442f21501ef75de6a7fd148835c3491da1827c77f62b169c2ccca38507ee400f4c938d87ac273935ce63bb9c966c6da4851e6c49d0cc6c9
|
data/lib/interferon/alert_dsl.rb
CHANGED
@@ -66,8 +66,8 @@ module Interferon::Destinations
|
|
66
66
|
@api_errors ||= []
|
67
67
|
end
|
68
68
|
|
69
|
-
def generate_message(message, people)
|
70
|
-
[message, ALERT_KEY, people.map { |p| "@#{p}" }].flatten.join("\n")
|
69
|
+
def self.generate_message(message, people)
|
70
|
+
[message, ALERT_KEY, people.sort.map { |p| "@#{p}" }].flatten.join("\n")
|
71
71
|
end
|
72
72
|
|
73
73
|
def fetch_existing_alerts
|
@@ -132,7 +132,7 @@ module Interferon::Destinations
|
|
132
132
|
# create a message which includes the notifications
|
133
133
|
# Datadog may have a race condition where alerts created in a bad state may be triggered
|
134
134
|
# during the dry-run creation process. Delete people from dry-run alerts to avoid this
|
135
|
-
message = generate_message(alert['message'], people)
|
135
|
+
message = self.class.generate_message(alert['message'], people)
|
136
136
|
|
137
137
|
# create the hash of options to send to datadog
|
138
138
|
alert_options = {
|
@@ -201,7 +201,7 @@ EOM
|
|
201
201
|
alert['monitor_type'],
|
202
202
|
datadog_query,
|
203
203
|
name: alert['name'],
|
204
|
-
message: @dry_run ? generate_message(alert, []) : message,
|
204
|
+
message: @dry_run ? self.class.generate_message(alert, []) : message,
|
205
205
|
options: alert_options
|
206
206
|
)
|
207
207
|
end
|
@@ -234,10 +234,10 @@ EOM
|
|
234
234
|
alert['monitor_type'],
|
235
235
|
datadog_query,
|
236
236
|
name: alert['name'],
|
237
|
-
message: generate_message(alert, []),
|
237
|
+
message: self.class.generate_message(alert, []),
|
238
238
|
options: alert_options
|
239
239
|
)
|
240
|
-
elsif alert['monitor_type']
|
240
|
+
elsif self.class.same_monitor_type(alert['monitor_type'], existing_alert['type'])
|
241
241
|
resp = @dog.update_monitor(
|
242
242
|
id,
|
243
243
|
datadog_query,
|
@@ -290,6 +290,67 @@ EOM
|
|
290
290
|
end
|
291
291
|
end
|
292
292
|
|
293
|
+
def remove_alert_by_id(alert_id)
|
294
|
+
# This should only be used by dry-run to clean up created dry-run alerts
|
295
|
+
log.debug("deleting alert, id: #{alert_id}")
|
296
|
+
resp = @dog.delete_monitor(alert_id)
|
297
|
+
code = resp[0].to_i
|
298
|
+
log_datadog_response_code(resp, code, :deleting)
|
299
|
+
end
|
300
|
+
|
301
|
+
def need_update(alert_people_pair, existing_alerts_from_api)
|
302
|
+
alert, people = alert_people_pair
|
303
|
+
existing = existing_alerts_from_api[alert['name']]
|
304
|
+
existing.nil? || !self.class.same_alerts(alert, people, existing)
|
305
|
+
end
|
306
|
+
|
307
|
+
def self.normalize_monitor_type(monitor_type)
|
308
|
+
# Convert 'query alert' type to 'metric alert' type. They can used interchangeably when
|
309
|
+
# submitting monitors to Datadog. Datadog will automatically do the conversion to 'query
|
310
|
+
# alert' for a "complex" query that includes multiple metrics/tags while using 'metric alert'
|
311
|
+
# for monitors that include a single scope/metric.
|
312
|
+
monitor_type == 'query alert' ? 'metric alert' : monitor_type
|
313
|
+
end
|
314
|
+
|
315
|
+
def self.same_monitor_type(monitor_type_a, monitor_type_b)
|
316
|
+
normalize_monitor_type(monitor_type_a) == normalize_monitor_type(monitor_type_b)
|
317
|
+
end
|
318
|
+
|
319
|
+
def self.same_alerts(alert, people, alert_api_json)
|
320
|
+
prev_alert = {
|
321
|
+
monitor_type: normalize_monitor_type(alert_api_json['type']),
|
322
|
+
query: alert_api_json['query'].strip,
|
323
|
+
message: alert_api_json['message'].strip,
|
324
|
+
evaluation_delay: alert_api_json['options']['evaluation_delay'],
|
325
|
+
notify_no_data: alert_api_json['options']['notify_no_data'],
|
326
|
+
notify_audit: alert_api_json['options']['notify_audit'],
|
327
|
+
no_data_timeframe: alert_api_json['options']['no_data_timeframe'],
|
328
|
+
silenced: alert_api_json['options']['silenced'],
|
329
|
+
thresholds: alert_api_json['options']['thresholds'],
|
330
|
+
timeout_h: alert_api_json['options']['timeout_h'],
|
331
|
+
}
|
332
|
+
|
333
|
+
new_alert = {
|
334
|
+
monitor_type: normalize_monitor_type(alert['monitor_type']),
|
335
|
+
query: alert['metric']['datadog_query'],
|
336
|
+
message: generate_message(alert['message'], people).strip,
|
337
|
+
evaluation_delay: alert['evaluation_delay'],
|
338
|
+
notify_no_data: alert['notify_no_data'],
|
339
|
+
notify_audit: alert['notify']['audit'],
|
340
|
+
no_data_timeframe: alert['no_data_timeframe'],
|
341
|
+
silenced: alert['silenced'],
|
342
|
+
thresholds: alert['thresholds'],
|
343
|
+
timeout_h: alert['timeout_h'],
|
344
|
+
}
|
345
|
+
|
346
|
+
unless alert['require_full_window'].nil?
|
347
|
+
prev_alert[:require_full_window] = alert_api_json['options']['require_full_window']
|
348
|
+
new_alert[:require_full_window] = alert['require_full_window']
|
349
|
+
end
|
350
|
+
|
351
|
+
prev_alert == new_alert
|
352
|
+
end
|
353
|
+
|
293
354
|
def report_stats
|
294
355
|
@stats.each do |k, v|
|
295
356
|
statsd.gauge("datadog.#{k}", v)
|
@@ -307,14 +368,6 @@ EOM
|
|
307
368
|
)
|
308
369
|
end
|
309
370
|
|
310
|
-
def remove_alert_by_id(alert_id)
|
311
|
-
# This should only be used by dry-run to clean up created dry-run alerts
|
312
|
-
log.debug("deleting alert, id: #{alert_id}")
|
313
|
-
resp = @dog.delete_monitor(alert_id)
|
314
|
-
code = resp[0].to_i
|
315
|
-
log_datadog_response_code(resp, code, :deleting)
|
316
|
-
end
|
317
|
-
|
318
371
|
def log_datadog_response_code(resp, code, action, alert = nil)
|
319
372
|
# log whenever we've encountered errors
|
320
373
|
if code != 200 && !alert.nil?
|
data/lib/interferon/version.rb
CHANGED
data/lib/interferon.rb
CHANGED
@@ -106,11 +106,11 @@ module Interferon
|
|
106
106
|
people_count += people.count
|
107
107
|
end
|
108
108
|
|
109
|
-
log.info "read #{people_count} people in #{source_groups.count} groups" \
|
109
|
+
log.info "read #{people_count} people in #{source_groups.count} groups " \
|
110
110
|
"from source #{source.class.name}"
|
111
111
|
end
|
112
112
|
|
113
|
-
log.info "total of #{groups.values.flatten.count} people in #{groups.count} groups" \
|
113
|
+
log.info "total of #{groups.values.flatten.count} people in #{groups.count} groups " \
|
114
114
|
"from #{sources.count} sources"
|
115
115
|
|
116
116
|
statsd.gauge('groups.sources', sources.count)
|
@@ -191,7 +191,7 @@ module Interferon
|
|
191
191
|
|
192
192
|
alerts_queue = build_alerts_queue(hosts, alerts, groups)
|
193
193
|
updates_queue = alerts_queue.reject do |_name, alert_people_pair|
|
194
|
-
!
|
194
|
+
!dest.need_update(alert_people_pair, existing_alerts)
|
195
195
|
end
|
196
196
|
|
197
197
|
# Add dry-run prefix to alerts and delete id to avoid impacting real alerts
|
@@ -205,7 +205,7 @@ module Interferon
|
|
205
205
|
|
206
206
|
# Build new queue with dry-run prefixes and ensure they are silenced
|
207
207
|
alerts_queue.each do |_name, alert_people_pair|
|
208
|
-
alert = alert_people_pair
|
208
|
+
alert, _people = alert_people_pair
|
209
209
|
dry_run_alert_name = DRY_RUN_ALERTS_NAME_PREFIX + alert['name']
|
210
210
|
alert.change_name(dry_run_alert_name)
|
211
211
|
alert.silence
|
@@ -218,7 +218,7 @@ module Interferon
|
|
218
218
|
# alerts that aren't being generated anymore
|
219
219
|
to_remove = existing_alerts.dup
|
220
220
|
alerts_queue.each do |_name, alert_people_pair|
|
221
|
-
alert = alert_people_pair
|
221
|
+
alert, _people = alert_people_pair
|
222
222
|
old_alerts = to_remove[alert['name']]
|
223
223
|
|
224
224
|
next if old_alerts.nil?
|
@@ -247,7 +247,7 @@ module Interferon
|
|
247
247
|
def do_regular_update(dest, hosts, alerts, existing_alerts, groups)
|
248
248
|
alerts_queue = build_alerts_queue(hosts, alerts, groups)
|
249
249
|
updates_queue = alerts_queue.reject do |_name, alert_people_pair|
|
250
|
-
!
|
250
|
+
!dest.need_update(alert_people_pair, existing_alerts)
|
251
251
|
end
|
252
252
|
|
253
253
|
# Create alerts in destination
|
@@ -257,7 +257,7 @@ module Interferon
|
|
257
257
|
# alerts that aren't being generated anymore
|
258
258
|
to_remove = existing_alerts.dup
|
259
259
|
alerts_queue.each do |_name, alert_people_pair|
|
260
|
-
alert = alert_people_pair
|
260
|
+
alert, _people = alert_people_pair
|
261
261
|
old_alerts = to_remove[alert['name']]
|
262
262
|
|
263
263
|
next if old_alerts.nil?
|
@@ -378,60 +378,5 @@ module Interferon
|
|
378
378
|
end
|
379
379
|
alerts_queue
|
380
380
|
end
|
381
|
-
|
382
|
-
def self.need_update(dest, alert_people_pair, existing_alerts_from_api)
|
383
|
-
alert = alert_people_pair[0]
|
384
|
-
existing = existing_alerts_from_api[alert['name']]
|
385
|
-
if existing.nil?
|
386
|
-
true
|
387
|
-
else
|
388
|
-
!same_alerts(dest, alert_people_pair, existing)
|
389
|
-
end
|
390
|
-
end
|
391
|
-
|
392
|
-
def self.normalize_monitor_type(monitor_type)
|
393
|
-
# Convert 'query alert' type to 'metric alert' type. They can used interchangeably when
|
394
|
-
# submitting monitors to Datadog. Datadog will automatically do the conversion to 'query
|
395
|
-
# alert' for a "complex" query that includes multiple metrics/tags while using 'metric alert'
|
396
|
-
# for monitors that include a single scope/metric.
|
397
|
-
monitor_type == 'query alert' ? 'metric alert' : monitor_type
|
398
|
-
end
|
399
|
-
|
400
|
-
def self.same_alerts(dest, alert_people_pair, alert_api_json)
|
401
|
-
alert, people = alert_people_pair
|
402
|
-
|
403
|
-
prev_alert = {
|
404
|
-
monitor_type: normalize_monitor_type(alert_api_json['type']),
|
405
|
-
query: alert_api_json['query'].strip,
|
406
|
-
message: alert_api_json['message'].strip,
|
407
|
-
evaluation_delay: alert_api_json['options']['evaluation_delay'],
|
408
|
-
notify_no_data: alert_api_json['options']['notify_no_data'],
|
409
|
-
notify_audit: alert_api_json['options']['notify_audit'],
|
410
|
-
no_data_timeframe: alert_api_json['options']['no_data_timeframe'],
|
411
|
-
silenced: alert_api_json['options']['silenced'],
|
412
|
-
thresholds: alert_api_json['options']['thresholds'],
|
413
|
-
timeout_h: alert_api_json['options']['timeout_h'],
|
414
|
-
}
|
415
|
-
|
416
|
-
new_alert = {
|
417
|
-
monitor_type: normalize_monitor_type(alert['monitor_type']),
|
418
|
-
query: alert['metric']['datadog_query'],
|
419
|
-
message: dest.generate_message(alert['message'], people).strip,
|
420
|
-
evaluation_delay: alert['evaluation_delay'],
|
421
|
-
notify_no_data: alert['notify_no_data'],
|
422
|
-
notify_audit: alert['notify']['audit'],
|
423
|
-
no_data_timeframe: alert['no_data_timeframe'],
|
424
|
-
silenced: alert['silenced'],
|
425
|
-
thresholds: alert['thresholds'],
|
426
|
-
timeout_h: alert['timeout_h'],
|
427
|
-
}
|
428
|
-
|
429
|
-
unless alert['require_full_window'].nil?
|
430
|
-
prev_alert[:require_full_window] = alert_api_json['options']['require_full_window']
|
431
|
-
new_alert[:require_full_window] = alert['require_full_window']
|
432
|
-
end
|
433
|
-
|
434
|
-
prev_alert == new_alert
|
435
|
-
end
|
436
381
|
end
|
437
382
|
end
|
data/spec/lib/interferon_spec.rb
CHANGED
@@ -4,7 +4,7 @@ require 'interferon/destinations/datadog'
|
|
4
4
|
|
5
5
|
include Interferon
|
6
6
|
|
7
|
-
describe Interferon::
|
7
|
+
describe Interferon::Destinations::Datadog do
|
8
8
|
let(:the_existing_alerts) { mock_existing_alerts }
|
9
9
|
let(:dest) { MockDest.new(the_existing_alerts) }
|
10
10
|
|
@@ -13,35 +13,35 @@ describe Interferon::Interferon do
|
|
13
13
|
alert1 = create_test_alert('name1', 'testquery', 'message1')
|
14
14
|
alert2 = mock_alert_json('name2', 'testquery', 'message2')
|
15
15
|
|
16
|
-
expect(Interferon::
|
16
|
+
expect(Interferon::Destinations::Datadog.same_alerts(alert1, [], alert2)).to be false
|
17
17
|
end
|
18
18
|
|
19
19
|
it 'detects a change if datadog query is different' do
|
20
20
|
alert1 = create_test_alert('name1', 'testquery1', 'message1')
|
21
21
|
alert2 = mock_alert_json('name2', 'testquery2', 'message2')
|
22
22
|
|
23
|
-
expect(Interferon::
|
23
|
+
expect(Interferon::Destinations::Datadog.same_alerts(alert1, [], alert2)).to be false
|
24
24
|
end
|
25
25
|
|
26
26
|
it 'detects a change if alert notify_no_data is different' do
|
27
27
|
alert1 = create_test_alert('name1', 'testquery1', 'message1', notify_no_data: false)
|
28
28
|
alert2 = mock_alert_json('name2', 'testquery2', 'message2', nil, [1], notify_no_data: true)
|
29
29
|
|
30
|
-
expect(Interferon::
|
30
|
+
expect(Interferon::Destinations::Datadog.same_alerts(alert1, [], alert2)).to be false
|
31
31
|
end
|
32
32
|
|
33
33
|
it 'detects a change if alert silenced is different' do
|
34
34
|
alert1 = create_test_alert('name1', 'testquery1', 'message1', silenced: true)
|
35
35
|
alert2 = mock_alert_json('name2', 'testquery2', 'message2', nil, [1], silenced: {})
|
36
36
|
|
37
|
-
expect(Interferon::
|
37
|
+
expect(Interferon::Destinations::Datadog.same_alerts(alert1, [], alert2)).to be false
|
38
38
|
end
|
39
39
|
|
40
40
|
it 'detects a change if alert no_data_timeframe is different' do
|
41
41
|
alert1 = create_test_alert('name1', 'testquery1', 'message1', no_data_timeframe: nil)
|
42
42
|
alert2 = mock_alert_json('name2', 'testquery2', 'message2', nil, [1], no_data_timeframe: 60)
|
43
43
|
|
44
|
-
expect(Interferon::
|
44
|
+
expect(Interferon::Destinations::Datadog.same_alerts(alert1, [], alert2)).to be false
|
45
45
|
end
|
46
46
|
|
47
47
|
it 'detects a change if alert require_full_window is different' do
|
@@ -50,14 +50,14 @@ describe Interferon::Interferon do
|
|
50
50
|
'name2', 'testquery2', 'message2', nil, [1], require_full_window: true
|
51
51
|
)
|
52
52
|
|
53
|
-
expect(Interferon::
|
53
|
+
expect(Interferon::Destinations::Datadog.same_alerts(alert1, [], alert2)).to be false
|
54
54
|
end
|
55
55
|
|
56
56
|
it 'detects a change if alert evaluation_delay is different' do
|
57
57
|
alert1 = create_test_alert('name1', 'testquery1', 'message1', evaluation_delay: nil)
|
58
58
|
alert2 = mock_alert_json('name2', 'testquery2', 'message2', nil, [1], evaluation_delay: 300)
|
59
59
|
|
60
|
-
expect(Interferon::
|
60
|
+
expect(Interferon::Destinations::Datadog.same_alerts(alert1, [], alert2)).to be false
|
61
61
|
end
|
62
62
|
|
63
63
|
it 'does not detect a change when alert datadog query and message are the same' do
|
@@ -66,7 +66,7 @@ describe Interferon::Interferon do
|
|
66
66
|
'name1', 'testquery1', "message1\nThis alert was created via the alerts framework"
|
67
67
|
)
|
68
68
|
|
69
|
-
expect(Interferon::
|
69
|
+
expect(Interferon::Destinations::Datadog.same_alerts(alert1, [], alert2)).to be true
|
70
70
|
end
|
71
71
|
end
|
72
72
|
|
@@ -252,7 +252,7 @@ describe Interferon::Interferon do
|
|
252
252
|
|
253
253
|
DEFAULT_OPTIONS = {
|
254
254
|
'evaluation_delay' => nil,
|
255
|
-
'notify_audit' =>
|
255
|
+
'notify_audit' => false,
|
256
256
|
'notify_no_data' => false,
|
257
257
|
'silenced' => {},
|
258
258
|
'thresholds' => nil,
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: interferon
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Igor Serebryany
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2017-
|
12
|
+
date: 2017-09-18 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: dogapi
|