interferon 0.1.3 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/interferon/alert_dsl.rb +1 -1
- data/lib/interferon/destinations/datadog.rb +67 -14
- data/lib/interferon/version.rb +1 -1
- data/lib/interferon.rb +7 -62
- data/spec/lib/interferon_spec.rb +10 -10
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 1c7f74fc8e23da780fdf783cdb98387321d20513
|
4
|
+
data.tar.gz: 5aa81e287ab880c2d3e7af7bd96a035d41b3db21
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 82c25b9118d41092d6ab131dd7edba612f40e68d4959bd0849efb4d0d2483bd5056b806dfc1c6f7e21381badd83cfb1e4f4e62fba8497bb3e6c52c4f33479dd2
|
7
|
+
data.tar.gz: 24740188b929de430442f21501ef75de6a7fd148835c3491da1827c77f62b169c2ccca38507ee400f4c938d87ac273935ce63bb9c966c6da4851e6c49d0cc6c9
|
data/lib/interferon/alert_dsl.rb
CHANGED
@@ -66,8 +66,8 @@ module Interferon::Destinations
|
|
66
66
|
@api_errors ||= []
|
67
67
|
end
|
68
68
|
|
69
|
-
def generate_message(message, people)
|
70
|
-
[message, ALERT_KEY, people.map { |p| "@#{p}" }].flatten.join("\n")
|
69
|
+
def self.generate_message(message, people)
|
70
|
+
[message, ALERT_KEY, people.sort.map { |p| "@#{p}" }].flatten.join("\n")
|
71
71
|
end
|
72
72
|
|
73
73
|
def fetch_existing_alerts
|
@@ -132,7 +132,7 @@ module Interferon::Destinations
|
|
132
132
|
# create a message which includes the notifications
|
133
133
|
# Datadog may have a race condition where alerts created in a bad state may be triggered
|
134
134
|
# during the dry-run creation process. Delete people from dry-run alerts to avoid this
|
135
|
-
message = generate_message(alert['message'], people)
|
135
|
+
message = self.class.generate_message(alert['message'], people)
|
136
136
|
|
137
137
|
# create the hash of options to send to datadog
|
138
138
|
alert_options = {
|
@@ -201,7 +201,7 @@ EOM
|
|
201
201
|
alert['monitor_type'],
|
202
202
|
datadog_query,
|
203
203
|
name: alert['name'],
|
204
|
-
message: @dry_run ? generate_message(alert, []) : message,
|
204
|
+
message: @dry_run ? self.class.generate_message(alert, []) : message,
|
205
205
|
options: alert_options
|
206
206
|
)
|
207
207
|
end
|
@@ -234,10 +234,10 @@ EOM
|
|
234
234
|
alert['monitor_type'],
|
235
235
|
datadog_query,
|
236
236
|
name: alert['name'],
|
237
|
-
message: generate_message(alert, []),
|
237
|
+
message: self.class.generate_message(alert, []),
|
238
238
|
options: alert_options
|
239
239
|
)
|
240
|
-
elsif alert['monitor_type']
|
240
|
+
elsif self.class.same_monitor_type(alert['monitor_type'], existing_alert['type'])
|
241
241
|
resp = @dog.update_monitor(
|
242
242
|
id,
|
243
243
|
datadog_query,
|
@@ -290,6 +290,67 @@ EOM
|
|
290
290
|
end
|
291
291
|
end
|
292
292
|
|
293
|
+
def remove_alert_by_id(alert_id)
|
294
|
+
# This should only be used by dry-run to clean up created dry-run alerts
|
295
|
+
log.debug("deleting alert, id: #{alert_id}")
|
296
|
+
resp = @dog.delete_monitor(alert_id)
|
297
|
+
code = resp[0].to_i
|
298
|
+
log_datadog_response_code(resp, code, :deleting)
|
299
|
+
end
|
300
|
+
|
301
|
+
def need_update(alert_people_pair, existing_alerts_from_api)
|
302
|
+
alert, people = alert_people_pair
|
303
|
+
existing = existing_alerts_from_api[alert['name']]
|
304
|
+
existing.nil? || !self.class.same_alerts(alert, people, existing)
|
305
|
+
end
|
306
|
+
|
307
|
+
def self.normalize_monitor_type(monitor_type)
|
308
|
+
# Convert 'query alert' type to 'metric alert' type. They can used interchangeably when
|
309
|
+
# submitting monitors to Datadog. Datadog will automatically do the conversion to 'query
|
310
|
+
# alert' for a "complex" query that includes multiple metrics/tags while using 'metric alert'
|
311
|
+
# for monitors that include a single scope/metric.
|
312
|
+
monitor_type == 'query alert' ? 'metric alert' : monitor_type
|
313
|
+
end
|
314
|
+
|
315
|
+
def self.same_monitor_type(monitor_type_a, monitor_type_b)
|
316
|
+
normalize_monitor_type(monitor_type_a) == normalize_monitor_type(monitor_type_b)
|
317
|
+
end
|
318
|
+
|
319
|
+
def self.same_alerts(alert, people, alert_api_json)
|
320
|
+
prev_alert = {
|
321
|
+
monitor_type: normalize_monitor_type(alert_api_json['type']),
|
322
|
+
query: alert_api_json['query'].strip,
|
323
|
+
message: alert_api_json['message'].strip,
|
324
|
+
evaluation_delay: alert_api_json['options']['evaluation_delay'],
|
325
|
+
notify_no_data: alert_api_json['options']['notify_no_data'],
|
326
|
+
notify_audit: alert_api_json['options']['notify_audit'],
|
327
|
+
no_data_timeframe: alert_api_json['options']['no_data_timeframe'],
|
328
|
+
silenced: alert_api_json['options']['silenced'],
|
329
|
+
thresholds: alert_api_json['options']['thresholds'],
|
330
|
+
timeout_h: alert_api_json['options']['timeout_h'],
|
331
|
+
}
|
332
|
+
|
333
|
+
new_alert = {
|
334
|
+
monitor_type: normalize_monitor_type(alert['monitor_type']),
|
335
|
+
query: alert['metric']['datadog_query'],
|
336
|
+
message: generate_message(alert['message'], people).strip,
|
337
|
+
evaluation_delay: alert['evaluation_delay'],
|
338
|
+
notify_no_data: alert['notify_no_data'],
|
339
|
+
notify_audit: alert['notify']['audit'],
|
340
|
+
no_data_timeframe: alert['no_data_timeframe'],
|
341
|
+
silenced: alert['silenced'],
|
342
|
+
thresholds: alert['thresholds'],
|
343
|
+
timeout_h: alert['timeout_h'],
|
344
|
+
}
|
345
|
+
|
346
|
+
unless alert['require_full_window'].nil?
|
347
|
+
prev_alert[:require_full_window] = alert_api_json['options']['require_full_window']
|
348
|
+
new_alert[:require_full_window] = alert['require_full_window']
|
349
|
+
end
|
350
|
+
|
351
|
+
prev_alert == new_alert
|
352
|
+
end
|
353
|
+
|
293
354
|
def report_stats
|
294
355
|
@stats.each do |k, v|
|
295
356
|
statsd.gauge("datadog.#{k}", v)
|
@@ -307,14 +368,6 @@ EOM
|
|
307
368
|
)
|
308
369
|
end
|
309
370
|
|
310
|
-
def remove_alert_by_id(alert_id)
|
311
|
-
# This should only be used by dry-run to clean up created dry-run alerts
|
312
|
-
log.debug("deleting alert, id: #{alert_id}")
|
313
|
-
resp = @dog.delete_monitor(alert_id)
|
314
|
-
code = resp[0].to_i
|
315
|
-
log_datadog_response_code(resp, code, :deleting)
|
316
|
-
end
|
317
|
-
|
318
371
|
def log_datadog_response_code(resp, code, action, alert = nil)
|
319
372
|
# log whenever we've encountered errors
|
320
373
|
if code != 200 && !alert.nil?
|
data/lib/interferon/version.rb
CHANGED
data/lib/interferon.rb
CHANGED
@@ -106,11 +106,11 @@ module Interferon
|
|
106
106
|
people_count += people.count
|
107
107
|
end
|
108
108
|
|
109
|
-
log.info "read #{people_count} people in #{source_groups.count} groups" \
|
109
|
+
log.info "read #{people_count} people in #{source_groups.count} groups " \
|
110
110
|
"from source #{source.class.name}"
|
111
111
|
end
|
112
112
|
|
113
|
-
log.info "total of #{groups.values.flatten.count} people in #{groups.count} groups" \
|
113
|
+
log.info "total of #{groups.values.flatten.count} people in #{groups.count} groups " \
|
114
114
|
"from #{sources.count} sources"
|
115
115
|
|
116
116
|
statsd.gauge('groups.sources', sources.count)
|
@@ -191,7 +191,7 @@ module Interferon
|
|
191
191
|
|
192
192
|
alerts_queue = build_alerts_queue(hosts, alerts, groups)
|
193
193
|
updates_queue = alerts_queue.reject do |_name, alert_people_pair|
|
194
|
-
!
|
194
|
+
!dest.need_update(alert_people_pair, existing_alerts)
|
195
195
|
end
|
196
196
|
|
197
197
|
# Add dry-run prefix to alerts and delete id to avoid impacting real alerts
|
@@ -205,7 +205,7 @@ module Interferon
|
|
205
205
|
|
206
206
|
# Build new queue with dry-run prefixes and ensure they are silenced
|
207
207
|
alerts_queue.each do |_name, alert_people_pair|
|
208
|
-
alert = alert_people_pair
|
208
|
+
alert, _people = alert_people_pair
|
209
209
|
dry_run_alert_name = DRY_RUN_ALERTS_NAME_PREFIX + alert['name']
|
210
210
|
alert.change_name(dry_run_alert_name)
|
211
211
|
alert.silence
|
@@ -218,7 +218,7 @@ module Interferon
|
|
218
218
|
# alerts that aren't being generated anymore
|
219
219
|
to_remove = existing_alerts.dup
|
220
220
|
alerts_queue.each do |_name, alert_people_pair|
|
221
|
-
alert = alert_people_pair
|
221
|
+
alert, _people = alert_people_pair
|
222
222
|
old_alerts = to_remove[alert['name']]
|
223
223
|
|
224
224
|
next if old_alerts.nil?
|
@@ -247,7 +247,7 @@ module Interferon
|
|
247
247
|
def do_regular_update(dest, hosts, alerts, existing_alerts, groups)
|
248
248
|
alerts_queue = build_alerts_queue(hosts, alerts, groups)
|
249
249
|
updates_queue = alerts_queue.reject do |_name, alert_people_pair|
|
250
|
-
!
|
250
|
+
!dest.need_update(alert_people_pair, existing_alerts)
|
251
251
|
end
|
252
252
|
|
253
253
|
# Create alerts in destination
|
@@ -257,7 +257,7 @@ module Interferon
|
|
257
257
|
# alerts that aren't being generated anymore
|
258
258
|
to_remove = existing_alerts.dup
|
259
259
|
alerts_queue.each do |_name, alert_people_pair|
|
260
|
-
alert = alert_people_pair
|
260
|
+
alert, _people = alert_people_pair
|
261
261
|
old_alerts = to_remove[alert['name']]
|
262
262
|
|
263
263
|
next if old_alerts.nil?
|
@@ -378,60 +378,5 @@ module Interferon
|
|
378
378
|
end
|
379
379
|
alerts_queue
|
380
380
|
end
|
381
|
-
|
382
|
-
def self.need_update(dest, alert_people_pair, existing_alerts_from_api)
|
383
|
-
alert = alert_people_pair[0]
|
384
|
-
existing = existing_alerts_from_api[alert['name']]
|
385
|
-
if existing.nil?
|
386
|
-
true
|
387
|
-
else
|
388
|
-
!same_alerts(dest, alert_people_pair, existing)
|
389
|
-
end
|
390
|
-
end
|
391
|
-
|
392
|
-
def self.normalize_monitor_type(monitor_type)
|
393
|
-
# Convert 'query alert' type to 'metric alert' type. They can used interchangeably when
|
394
|
-
# submitting monitors to Datadog. Datadog will automatically do the conversion to 'query
|
395
|
-
# alert' for a "complex" query that includes multiple metrics/tags while using 'metric alert'
|
396
|
-
# for monitors that include a single scope/metric.
|
397
|
-
monitor_type == 'query alert' ? 'metric alert' : monitor_type
|
398
|
-
end
|
399
|
-
|
400
|
-
def self.same_alerts(dest, alert_people_pair, alert_api_json)
|
401
|
-
alert, people = alert_people_pair
|
402
|
-
|
403
|
-
prev_alert = {
|
404
|
-
monitor_type: normalize_monitor_type(alert_api_json['type']),
|
405
|
-
query: alert_api_json['query'].strip,
|
406
|
-
message: alert_api_json['message'].strip,
|
407
|
-
evaluation_delay: alert_api_json['options']['evaluation_delay'],
|
408
|
-
notify_no_data: alert_api_json['options']['notify_no_data'],
|
409
|
-
notify_audit: alert_api_json['options']['notify_audit'],
|
410
|
-
no_data_timeframe: alert_api_json['options']['no_data_timeframe'],
|
411
|
-
silenced: alert_api_json['options']['silenced'],
|
412
|
-
thresholds: alert_api_json['options']['thresholds'],
|
413
|
-
timeout_h: alert_api_json['options']['timeout_h'],
|
414
|
-
}
|
415
|
-
|
416
|
-
new_alert = {
|
417
|
-
monitor_type: normalize_monitor_type(alert['monitor_type']),
|
418
|
-
query: alert['metric']['datadog_query'],
|
419
|
-
message: dest.generate_message(alert['message'], people).strip,
|
420
|
-
evaluation_delay: alert['evaluation_delay'],
|
421
|
-
notify_no_data: alert['notify_no_data'],
|
422
|
-
notify_audit: alert['notify']['audit'],
|
423
|
-
no_data_timeframe: alert['no_data_timeframe'],
|
424
|
-
silenced: alert['silenced'],
|
425
|
-
thresholds: alert['thresholds'],
|
426
|
-
timeout_h: alert['timeout_h'],
|
427
|
-
}
|
428
|
-
|
429
|
-
unless alert['require_full_window'].nil?
|
430
|
-
prev_alert[:require_full_window] = alert_api_json['options']['require_full_window']
|
431
|
-
new_alert[:require_full_window] = alert['require_full_window']
|
432
|
-
end
|
433
|
-
|
434
|
-
prev_alert == new_alert
|
435
|
-
end
|
436
381
|
end
|
437
382
|
end
|
data/spec/lib/interferon_spec.rb
CHANGED
@@ -4,7 +4,7 @@ require 'interferon/destinations/datadog'
|
|
4
4
|
|
5
5
|
include Interferon
|
6
6
|
|
7
|
-
describe Interferon::
|
7
|
+
describe Interferon::Destinations::Datadog do
|
8
8
|
let(:the_existing_alerts) { mock_existing_alerts }
|
9
9
|
let(:dest) { MockDest.new(the_existing_alerts) }
|
10
10
|
|
@@ -13,35 +13,35 @@ describe Interferon::Interferon do
|
|
13
13
|
alert1 = create_test_alert('name1', 'testquery', 'message1')
|
14
14
|
alert2 = mock_alert_json('name2', 'testquery', 'message2')
|
15
15
|
|
16
|
-
expect(Interferon::
|
16
|
+
expect(Interferon::Destinations::Datadog.same_alerts(alert1, [], alert2)).to be false
|
17
17
|
end
|
18
18
|
|
19
19
|
it 'detects a change if datadog query is different' do
|
20
20
|
alert1 = create_test_alert('name1', 'testquery1', 'message1')
|
21
21
|
alert2 = mock_alert_json('name2', 'testquery2', 'message2')
|
22
22
|
|
23
|
-
expect(Interferon::
|
23
|
+
expect(Interferon::Destinations::Datadog.same_alerts(alert1, [], alert2)).to be false
|
24
24
|
end
|
25
25
|
|
26
26
|
it 'detects a change if alert notify_no_data is different' do
|
27
27
|
alert1 = create_test_alert('name1', 'testquery1', 'message1', notify_no_data: false)
|
28
28
|
alert2 = mock_alert_json('name2', 'testquery2', 'message2', nil, [1], notify_no_data: true)
|
29
29
|
|
30
|
-
expect(Interferon::
|
30
|
+
expect(Interferon::Destinations::Datadog.same_alerts(alert1, [], alert2)).to be false
|
31
31
|
end
|
32
32
|
|
33
33
|
it 'detects a change if alert silenced is different' do
|
34
34
|
alert1 = create_test_alert('name1', 'testquery1', 'message1', silenced: true)
|
35
35
|
alert2 = mock_alert_json('name2', 'testquery2', 'message2', nil, [1], silenced: {})
|
36
36
|
|
37
|
-
expect(Interferon::
|
37
|
+
expect(Interferon::Destinations::Datadog.same_alerts(alert1, [], alert2)).to be false
|
38
38
|
end
|
39
39
|
|
40
40
|
it 'detects a change if alert no_data_timeframe is different' do
|
41
41
|
alert1 = create_test_alert('name1', 'testquery1', 'message1', no_data_timeframe: nil)
|
42
42
|
alert2 = mock_alert_json('name2', 'testquery2', 'message2', nil, [1], no_data_timeframe: 60)
|
43
43
|
|
44
|
-
expect(Interferon::
|
44
|
+
expect(Interferon::Destinations::Datadog.same_alerts(alert1, [], alert2)).to be false
|
45
45
|
end
|
46
46
|
|
47
47
|
it 'detects a change if alert require_full_window is different' do
|
@@ -50,14 +50,14 @@ describe Interferon::Interferon do
|
|
50
50
|
'name2', 'testquery2', 'message2', nil, [1], require_full_window: true
|
51
51
|
)
|
52
52
|
|
53
|
-
expect(Interferon::
|
53
|
+
expect(Interferon::Destinations::Datadog.same_alerts(alert1, [], alert2)).to be false
|
54
54
|
end
|
55
55
|
|
56
56
|
it 'detects a change if alert evaluation_delay is different' do
|
57
57
|
alert1 = create_test_alert('name1', 'testquery1', 'message1', evaluation_delay: nil)
|
58
58
|
alert2 = mock_alert_json('name2', 'testquery2', 'message2', nil, [1], evaluation_delay: 300)
|
59
59
|
|
60
|
-
expect(Interferon::
|
60
|
+
expect(Interferon::Destinations::Datadog.same_alerts(alert1, [], alert2)).to be false
|
61
61
|
end
|
62
62
|
|
63
63
|
it 'does not detect a change when alert datadog query and message are the same' do
|
@@ -66,7 +66,7 @@ describe Interferon::Interferon do
|
|
66
66
|
'name1', 'testquery1', "message1\nThis alert was created via the alerts framework"
|
67
67
|
)
|
68
68
|
|
69
|
-
expect(Interferon::
|
69
|
+
expect(Interferon::Destinations::Datadog.same_alerts(alert1, [], alert2)).to be true
|
70
70
|
end
|
71
71
|
end
|
72
72
|
|
@@ -252,7 +252,7 @@ describe Interferon::Interferon do
|
|
252
252
|
|
253
253
|
DEFAULT_OPTIONS = {
|
254
254
|
'evaluation_delay' => nil,
|
255
|
-
'notify_audit' =>
|
255
|
+
'notify_audit' => false,
|
256
256
|
'notify_no_data' => false,
|
257
257
|
'silenced' => {},
|
258
258
|
'thresholds' => nil,
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: interferon
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Igor Serebryany
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2017-
|
12
|
+
date: 2017-09-18 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: dogapi
|