interferon 0.0.7 → 0.0.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/config.example.yaml +1 -0
- data/lib/interferon/alert.rb +8 -0
- data/lib/interferon/destinations/datadog.rb +43 -8
- data/lib/interferon/version.rb +1 -1
- data/lib/interferon.rb +169 -103
- data/spec/helpers/dsl_helper.rb +39 -0
- data/spec/helpers/mock_alert.rb +11 -0
- data/spec/lib/interferon_spec.rb +102 -0
- metadata +31 -25
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 35b5382f5aabd274b548257bcc5f227faf1b951e
|
4
|
+
data.tar.gz: abacd2609b40bf28b2783bbd21b24c384f991188
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: b215a4e355f66daa582d65b521fe0eef5842eb795551df1998844d82cb1892a64b7a95e3b60fff1ec2992eb2e88de2f584c274c9bd5570b6f0ae4026b05eeb5a
|
7
|
+
data.tar.gz: 2bb6ee4093fe2bbe46d811a3cb79a357066073727a699d773b306a5355fd8f7b9d06a4f2bdb51082ac6570a3166820ce7f1b88d63f9fb76d6d032fc8b97073b0
|
data/config.example.yaml
CHANGED
data/lib/interferon/alert.rb
CHANGED
@@ -15,8 +15,22 @@ module Interferon::Destinations
|
|
15
15
|
end
|
16
16
|
end
|
17
17
|
|
18
|
-
|
19
|
-
|
18
|
+
# Set dogapi timeout explicitly
|
19
|
+
api_timeout = options['api_timeout'] || 15
|
20
|
+
|
21
|
+
# Default parameters of Dogapi Client initialize() can be referenced from link below:
|
22
|
+
# (as of this writing)
|
23
|
+
# https://github.com/DataDog/dogapi-rb/blob/master/lib/dogapi/facade.rb#L14
|
24
|
+
args = [
|
25
|
+
options['api_key'],
|
26
|
+
options['app_key'],
|
27
|
+
nil, # host to talk to
|
28
|
+
nil, # device
|
29
|
+
true, # silent?
|
30
|
+
api_timeout, # API timeout
|
31
|
+
]
|
32
|
+
@dog = Dogapi::Client.new(*args)
|
33
|
+
|
20
34
|
@existing_alerts = nil
|
21
35
|
|
22
36
|
# create datadog alerts 10 at a time
|
@@ -34,9 +48,19 @@ module Interferon::Destinations
|
|
34
48
|
}
|
35
49
|
end
|
36
50
|
|
51
|
+
def api_errors
|
52
|
+
@api_errors ||= []
|
53
|
+
end
|
54
|
+
|
37
55
|
def existing_alerts
|
38
56
|
unless @existing_alerts
|
39
57
|
resp = @dog.get_all_alerts()
|
58
|
+
|
59
|
+
code = resp[0].to_i
|
60
|
+
if code != 200
|
61
|
+
raise "Failed to retrieve existing alerts from datadog. #{code.to_s}: #{resp[1].inspect}"
|
62
|
+
end
|
63
|
+
|
40
64
|
alerts = resp[1]['alerts']
|
41
65
|
|
42
66
|
# key alerts by name
|
@@ -87,7 +111,7 @@ module Interferon::Destinations
|
|
87
111
|
resp = @dog.alert(
|
88
112
|
alert['metric']['datadog_query'].strip,
|
89
113
|
alert_opts,
|
90
|
-
)
|
114
|
+
)
|
91
115
|
|
92
116
|
# existing alert, modify it
|
93
117
|
else
|
@@ -99,11 +123,14 @@ module Interferon::Destinations
|
|
99
123
|
id,
|
100
124
|
alert['metric']['datadog_query'].strip,
|
101
125
|
alert_opts
|
102
|
-
)
|
126
|
+
)
|
103
127
|
end
|
104
128
|
|
105
129
|
# log whenever we've encountered errors
|
106
|
-
code =
|
130
|
+
code = resp[0].to_i
|
131
|
+
if code != 200
|
132
|
+
api_errors << "#{code.to_s} on alert #{alert['name']}"
|
133
|
+
end
|
107
134
|
|
108
135
|
# client error
|
109
136
|
if code == 400
|
@@ -113,7 +140,8 @@ module Interferon::Destinations
|
|
113
140
|
|
114
141
|
@stats[:api_client_errors] += 1
|
115
142
|
log.error("client error while #{action} alert '#{alert['name']}';" \
|
116
|
-
" query was '#{alert['metric']['datadog_query'].strip}'"
|
143
|
+
" query was '#{alert['metric']['datadog_query'].strip}'" \
|
144
|
+
" response was #{resp[0]}:'#{resp[1].inspect}'")
|
117
145
|
|
118
146
|
# unknown (prob. datadog) error:
|
119
147
|
elsif code >= 400 || code == -1
|
@@ -138,14 +166,15 @@ module Interferon::Destinations
|
|
138
166
|
@stats[:alerts_silenced] += 1 if alert_opts[:silenced]
|
139
167
|
end
|
140
168
|
|
169
|
+
id = resp[1].nil? ? nil : resp[1]['id']
|
141
170
|
# lets key alerts by their name
|
142
|
-
return alert['name']
|
171
|
+
return [alert['name'], id]
|
143
172
|
end
|
144
173
|
|
145
174
|
def remove_alert(alert)
|
146
175
|
if alert['message'].include?(ALERT_KEY)
|
147
176
|
log.debug("deleting alert #{alert['id']} (#{alert['name']})")
|
148
|
-
@dog.delete_alert(alert['id'])
|
177
|
+
@dog.delete_alert(alert['id'])
|
149
178
|
@stats[:alerts_deleted] += 1
|
150
179
|
else
|
151
180
|
log.warn("not deleting manually-created alert #{alert['id']} (#{alert['name']})")
|
@@ -163,5 +192,11 @@ module Interferon::Destinations
|
|
163
192
|
@stats[:alerts_deleted],
|
164
193
|
]
|
165
194
|
end
|
195
|
+
|
196
|
+
def remove_alert_by_id(alert_id)
|
197
|
+
log.debug("deleting alert, id: #{alert_id}")
|
198
|
+
@dog.delete_alert(alert_id)
|
199
|
+
@stats[:alerts_deleted] += 1
|
200
|
+
end
|
166
201
|
end
|
167
202
|
end
|
data/lib/interferon/version.rb
CHANGED
data/lib/interferon.rb
CHANGED
@@ -18,6 +18,8 @@ module Interferon
|
|
18
18
|
include Logging
|
19
19
|
attr_accessor :host_sources, :destinations, :host_info
|
20
20
|
|
21
|
+
DRY_RUN_ALERTS_NAME_PREFIX = '[-dry-run-]'
|
22
|
+
|
21
23
|
# groups_sources is a hash from type => options for each group source
|
22
24
|
# host_sources is a hash from type => options for each host source
|
23
25
|
# destinations is a similiar hash from type => options for each alerter
|
@@ -41,13 +43,11 @@ module Interferon
|
|
41
43
|
groups = read_groups(@groups_sources)
|
42
44
|
hosts = read_hosts(@host_sources)
|
43
45
|
|
44
|
-
# make sure destinations know if it's a dry run
|
45
46
|
@destinations.each do |dest|
|
46
47
|
dest['options'] ||= {}
|
47
|
-
dest['options']['dry_run'] = dry_run
|
48
48
|
end
|
49
49
|
|
50
|
-
update_alerts(@destinations, hosts, alerts, groups)
|
50
|
+
update_alerts(@destinations, hosts, alerts, groups, dry_run)
|
51
51
|
|
52
52
|
if @request_shutdown
|
53
53
|
log.info "interferon #{run_desc} shut down by SIGTERM"
|
@@ -133,132 +133,198 @@ module Interferon
|
|
133
133
|
return hosts
|
134
134
|
end
|
135
135
|
|
136
|
-
def update_alerts(destinations, hosts, alerts, groups)
|
136
|
+
def update_alerts(destinations, hosts, alerts, groups, dry_run)
|
137
137
|
loader = DestinationsLoader.new([@alerts_repo_path])
|
138
138
|
loader.get_all(destinations).each do |dest|
|
139
139
|
break if @request_shutdown
|
140
140
|
log.info "updating alerts on #{dest.class.name}"
|
141
|
+
update_alerts_on_destination(dest, hosts, alerts, groups, dry_run)
|
142
|
+
end
|
143
|
+
end
|
141
144
|
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
# get already-defined alerts
|
146
|
-
existing_alerts = dest.existing_alerts.dup
|
147
|
-
existing_alerts.each{ |key, existing_alert| existing_alert['still_exists'] = false }
|
148
|
-
|
149
|
-
# create or update alerts; mark when we've done that
|
150
|
-
alerts_queue = Hash.new
|
151
|
-
alerts.each do |alert|
|
152
|
-
break if @request_shutdown
|
153
|
-
counters = {
|
154
|
-
:errors => 0,
|
155
|
-
:evals => 0,
|
156
|
-
:applies => 0,
|
157
|
-
:hosts => hosts.length
|
158
|
-
}
|
159
|
-
last_eval_error = nil
|
160
|
-
|
161
|
-
hosts.each do |hostinfo|
|
162
|
-
begin
|
163
|
-
alert.evaluate(hostinfo)
|
164
|
-
counters[:evals] += 1
|
165
|
-
rescue StandardError => e
|
166
|
-
log.debug "Evaluation of alert #{alert} failed in the context of host #{hostinfo}"
|
167
|
-
counters[:errors] += 1
|
168
|
-
last_eval_error = e
|
169
|
-
next
|
170
|
-
end
|
145
|
+
def update_alerts_on_destination(dest, hosts, alerts, groups, dry_run)
|
146
|
+
# track some counters/stats per destination
|
147
|
+
start_time = Time.new.to_f
|
171
148
|
|
172
|
-
|
173
|
-
|
174
|
-
log.debug "alert #{alert[:name]} doesn't apply to #{hostinfo.inspect}"
|
175
|
-
next
|
176
|
-
end
|
149
|
+
# get already-defined alerts
|
150
|
+
existing_alerts = dest.existing_alerts.dup
|
177
151
|
|
178
|
-
|
152
|
+
if dry_run
|
153
|
+
do_dry_run_update(dest, hosts, alerts, existing_alerts, groups)
|
154
|
+
else
|
155
|
+
do_regular_update(dest, hosts, alerts, existing_alerts, groups)
|
156
|
+
end
|
179
157
|
|
180
|
-
|
181
|
-
|
158
|
+
unless @request_shutdown
|
159
|
+
# run time summary
|
160
|
+
run_time = Time.new.to_f - start_time
|
161
|
+
statsd.histogram(
|
162
|
+
dry_run ? 'destinations.run_time.dry_run' : 'destinations.run_time',
|
163
|
+
run_time,
|
164
|
+
:tags => ["destination:#{dest.class.name}"])
|
165
|
+
log.info "#{dest.class.name} : run completed in %.2f seconds" % (run_time)
|
166
|
+
|
167
|
+
# report destination stats
|
168
|
+
dest.report_stats
|
169
|
+
end
|
182
170
|
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
end
|
171
|
+
if dry_run && !dest.api_errors.empty?
|
172
|
+
raise dest.api_errors.to_s
|
173
|
+
end
|
174
|
+
end
|
188
175
|
|
189
|
-
|
190
|
-
|
191
|
-
|
176
|
+
def do_dry_run_update(dest, hosts, alerts, existing_alerts, groups)
|
177
|
+
to_remove = existing_alerts.reject{|key, a| !key.start_with?(DRY_RUN_ALERTS_NAME_PREFIX)}
|
178
|
+
alerts_queue = build_alerts_queue(hosts, alerts, groups)
|
179
|
+
alerts_queue.reject!{|name, pair| !Interferon::need_dry_run(pair[0], existing_alerts)}
|
180
|
+
alerts_queue.each do |name, pair|
|
181
|
+
alert = pair[0]
|
182
|
+
alert.change_name(DRY_RUN_ALERTS_NAME_PREFIX + alert['name'])
|
183
|
+
end
|
192
184
|
|
193
|
-
|
194
|
-
|
195
|
-
|
185
|
+
# flush queue
|
186
|
+
created_alerts_key_ids = create_alerts(dest, alerts_queue)
|
187
|
+
created_alerts_ids = created_alerts_key_ids.map{|a| a[1]}
|
188
|
+
to_remove_ids = to_remove.empty? ? [] : to_remove.map{|a| a['id']}
|
189
|
+
# remove existing alerts that shouldn't exist
|
190
|
+
(created_alerts_ids + to_remove_ids).each do |id|
|
191
|
+
break if @request_shutdown
|
192
|
+
dest.remove_alert_by_id(id) unless id.nil?
|
193
|
+
end
|
194
|
+
end
|
196
195
|
|
197
|
-
|
198
|
-
|
199
|
-
end
|
196
|
+
def do_regular_update(dest, hosts, alerts, existing_alerts, groups)
|
197
|
+
existing_alerts.each{ |key, existing_alert| existing_alert['still_exists'] = false }
|
200
198
|
|
201
|
-
|
202
|
-
if counters[:errors] == counters[:hosts]
|
203
|
-
log.error "alert #{alert} failed to evaluate in the context of all hosts!"
|
204
|
-
log.error "last error on alert #{alert}: #{last_eval_error}"
|
199
|
+
alerts_queue = build_alerts_queue(hosts, alerts, groups)
|
205
200
|
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
201
|
+
# flush queue
|
202
|
+
created_alerts_keys = create_alerts(dest, alerts_queue).map{|a| a[0]}
|
203
|
+
created_alerts_keys.each do |alert_key|
|
204
|
+
# don't delete alerts we still have defined
|
205
|
+
existing_alerts[alert_key]['still_exists'] = true if existing_alerts.include?(alert_key)
|
206
|
+
end
|
207
|
+
|
208
|
+
# remove existing alerts that shouldn't exist
|
209
|
+
to_delete = existing_alerts.reject{ |key, existing_alert| existing_alert['still_exists'] }
|
210
|
+
to_delete.each do |key, alert|
|
211
|
+
break if @request_shutdown
|
212
|
+
dest.remove_alert(alert)
|
213
|
+
end
|
214
|
+
end
|
211
215
|
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
216
|
+
def create_alerts(dest, alerts_queue)
|
217
|
+
alert_key_ids = []
|
218
|
+
alerts_to_create = alerts_queue.keys
|
219
|
+
concurrency = dest.concurrency || 10
|
220
|
+
unless @request_shutdown
|
221
|
+
threads = concurrency.times.map do |i|
|
222
|
+
log.info "thread #{i} created"
|
223
|
+
t = Thread.new do
|
224
|
+
while name = alerts_to_create.shift
|
225
|
+
break if @request_shutdown
|
226
|
+
cur_alert, people = alerts_queue[name]
|
227
|
+
log.debug "creating alert for #{cur_alert[:name]}"
|
228
|
+
alert_key_ids << dest.create_alert(cur_alert, people)
|
229
|
+
end
|
218
230
|
end
|
231
|
+
t.abort_on_exception = true
|
232
|
+
t
|
219
233
|
end
|
234
|
+
threads.map(&:join)
|
235
|
+
end
|
236
|
+
alert_key_ids
|
237
|
+
end
|
220
238
|
|
221
|
-
|
222
|
-
|
223
|
-
|
224
|
-
|
225
|
-
|
226
|
-
|
227
|
-
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
232
|
-
|
233
|
-
|
234
|
-
|
235
|
-
|
236
|
-
|
237
|
-
|
238
|
-
|
239
|
-
|
240
|
-
|
239
|
+
def build_alerts_queue(hosts, alerts, groups)
|
240
|
+
# create or update alerts; mark when we've done that
|
241
|
+
alerts_queue = Hash.new
|
242
|
+
alerts.each do |alert|
|
243
|
+
break if @request_shutdown
|
244
|
+
counters = {
|
245
|
+
:errors => 0,
|
246
|
+
:evals => 0,
|
247
|
+
:applies => 0,
|
248
|
+
:hosts => hosts.length
|
249
|
+
}
|
250
|
+
last_eval_error = nil
|
251
|
+
|
252
|
+
hosts.each do |hostinfo|
|
253
|
+
begin
|
254
|
+
alert.evaluate(hostinfo)
|
255
|
+
counters[:evals] += 1
|
256
|
+
rescue StandardError => e
|
257
|
+
log.debug "Evaluation of alert #{alert} failed in the context of host #{hostinfo}"
|
258
|
+
counters[:errors] += 1
|
259
|
+
last_eval_error = e
|
260
|
+
next
|
241
261
|
end
|
242
|
-
|
262
|
+
|
263
|
+
# don't define an alert that doesn't apply to this hostinfo
|
264
|
+
unless alert[:applies]
|
265
|
+
log.debug "alert #{alert[:name]} doesn't apply to #{hostinfo.inspect}"
|
266
|
+
next
|
267
|
+
end
|
268
|
+
|
269
|
+
counters[:applies] += 1
|
270
|
+
# don't define alerts twice
|
271
|
+
next if alerts_queue.key?(alert[:name])
|
272
|
+
|
273
|
+
# figure out who to notify
|
274
|
+
people = Set.new(alert[:notify][:people])
|
275
|
+
alert[:notify][:groups].each do |g|
|
276
|
+
people += (groups[g] || [])
|
277
|
+
end
|
278
|
+
|
279
|
+
# queue the alert up for creation; we clone the alert to save the current state
|
280
|
+
alerts_queue[alert[:name]] ||= [alert.clone, people]
|
243
281
|
end
|
244
282
|
|
245
|
-
#
|
246
|
-
|
247
|
-
|
248
|
-
|
249
|
-
|
283
|
+
# log some of the counters
|
284
|
+
statsd.gauge('alerts.evaluate.errors', counters[:errors], :tags => ["alert:#{alert}"])
|
285
|
+
statsd.gauge('alerts.evaluate.applies', counters[:applies], :tags => ["alert:#{alert}"])
|
286
|
+
|
287
|
+
if counters[:applies] > 0
|
288
|
+
log.info "alert #{alert} applies to #{counters[:applies]} of #{counters[:hosts]} hosts"
|
250
289
|
end
|
251
290
|
|
252
|
-
|
253
|
-
|
254
|
-
|
255
|
-
|
256
|
-
|
291
|
+
# did the alert fail to evaluate on all hosts?
|
292
|
+
if counters[:errors] == counters[:hosts]
|
293
|
+
log.error "alert #{alert} failed to evaluate in the context of all hosts!"
|
294
|
+
log.error "last error on alert #{alert}: #{last_eval_error}"
|
295
|
+
|
296
|
+
statsd.gauge('alerts.evaluate.failed_on_all', 1, :tags => ["alert:#{alert}"])
|
297
|
+
log.debug "alert #{alert}: error #{last_eval_error}\n#{last_eval_error.backtrace.join("\n")}"
|
298
|
+
else
|
299
|
+
statsd.gauge('alerts.evaluate.failed_on_all', 0, :tags => ["alert:#{alert}"])
|
300
|
+
end
|
257
301
|
|
258
|
-
|
259
|
-
|
302
|
+
# did the alert apply to any hosts?
|
303
|
+
if counters[:applies] == 0
|
304
|
+
statsd.gauge('alerts.evaluate.never_applies', 1, :tags => ["alert:#{alert}"])
|
305
|
+
log.warn "alert #{alert} did not apply to any hosts"
|
306
|
+
else
|
307
|
+
statsd.gauge('alerts.evaluate.never_applies', 0, :tags => ["alert:#{alert}"])
|
260
308
|
end
|
261
309
|
end
|
310
|
+
alerts_queue
|
311
|
+
end
|
312
|
+
|
313
|
+
def self.need_dry_run(alert, existing_alerts_from_api)
|
314
|
+
existing = existing_alerts_from_api[alert['name']]
|
315
|
+
if existing.nil?
|
316
|
+
true
|
317
|
+
else
|
318
|
+
!same_alerts_for_dry_run_purpose(alert, existing)
|
319
|
+
end
|
320
|
+
end
|
321
|
+
|
322
|
+
def self.same_alerts_for_dry_run_purpose(alert, alert_api_json)
|
323
|
+
query1 = alert['metric']['datadog_query']
|
324
|
+
query2 = alert_api_json['query']
|
325
|
+
query1.strip!
|
326
|
+
query2.strip!
|
327
|
+
query1 == query2
|
262
328
|
end
|
263
329
|
end
|
264
330
|
end
|
@@ -0,0 +1,39 @@
|
|
1
|
+
module Interferon
|
2
|
+
module MockDSLMixin
|
3
|
+
def initialize
|
4
|
+
end
|
5
|
+
|
6
|
+
def get_or_set(field, val, block, default)
|
7
|
+
@hash ||= Hash.new
|
8
|
+
if val.nil?
|
9
|
+
return @hash[field]
|
10
|
+
else
|
11
|
+
@hash[field] = val
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
class MockAlertDSL < AlertDSL
|
17
|
+
include MockDSLMixin
|
18
|
+
|
19
|
+
def notify(v = nil)
|
20
|
+
get_or_set(:notify, v, nil, nil)
|
21
|
+
end
|
22
|
+
|
23
|
+
def metric(v = nil)
|
24
|
+
get_or_set(:metric, v, nil, nil)
|
25
|
+
end
|
26
|
+
|
27
|
+
def id(v = nil, &block)
|
28
|
+
get_or_set(:@id, v, block, '')
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
class MockNotifyDSL < NotifyDSL
|
33
|
+
include MockDSLMixin
|
34
|
+
end
|
35
|
+
|
36
|
+
class MockMetricDSL < MetricDSL
|
37
|
+
include MockDSLMixin
|
38
|
+
end
|
39
|
+
end
|
@@ -0,0 +1,102 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'helpers/mock_alert'
|
3
|
+
require 'helpers/dsl_helper'
|
4
|
+
require 'interferon/destinations/datadog'
|
5
|
+
|
6
|
+
include Interferon
|
7
|
+
|
8
|
+
describe Interferon::Interferon do
|
9
|
+
it "would not be dry run if only alert message changed" do
|
10
|
+
alert1 = create_test_alert('name1', 'testquery', 'message1')
|
11
|
+
alert2 = mock_alert_json('name2', 'testquery', 'message2')
|
12
|
+
|
13
|
+
expect(Interferon::Interferon.same_alerts_for_dry_run_purpose(alert1, alert2)).to be true
|
14
|
+
end
|
15
|
+
|
16
|
+
it "would be dry run if alert datadog query changed" do
|
17
|
+
alert1 = create_test_alert('name1', 'testquery1', 'message1')
|
18
|
+
alert2 = mock_alert_json('name2', 'testquery2', 'message2')
|
19
|
+
|
20
|
+
expect(Interferon::Interferon.same_alerts_for_dry_run_purpose(alert1, alert2)).to be false
|
21
|
+
end
|
22
|
+
|
23
|
+
context "dry_run_update_alerts_on_destination" do
|
24
|
+
let(:the_existing_alerts) {mock_existing_alerts}
|
25
|
+
let(:dest) {MockDest.new(the_existing_alerts)}
|
26
|
+
before do
|
27
|
+
allow_any_instance_of(MockAlert).to receive(:evaluate)
|
28
|
+
allow(dest).to receive(:remove_alert_by_id)
|
29
|
+
allow(dest).to receive(:report_stats)
|
30
|
+
end
|
31
|
+
|
32
|
+
it 'dry run does not re-run existing alerts' do
|
33
|
+
alerts = mock_existing_alerts
|
34
|
+
interferon = Interferon::Interferon.new(nil,nil,nil,nil)
|
35
|
+
expect(dest).not_to receive(:create_alert)
|
36
|
+
expect(dest).not_to receive(:remove_alert_by_id)
|
37
|
+
|
38
|
+
interferon.update_alerts_on_destination(dest, ['host'], [alerts['name1'], alerts['name2']], {}, true)
|
39
|
+
end
|
40
|
+
|
41
|
+
it 'dry run runs added alerts' do
|
42
|
+
alerts = mock_existing_alerts
|
43
|
+
interferon = Interferon::Interferon.new(nil,nil,nil,nil)
|
44
|
+
added = create_test_alert('name3', 'testquery3', '')
|
45
|
+
expect(dest).to receive(:create_alert).once.and_call_original
|
46
|
+
expect(dest).to receive(:remove_alert_by_id).with('[-dry-run-]name3').once
|
47
|
+
|
48
|
+
interferon.update_alerts_on_destination(dest, ['host'], [alerts['name1'], alerts['name2'], added], {}, true)
|
49
|
+
end
|
50
|
+
|
51
|
+
it 'dry run runs updated alerts' do
|
52
|
+
alerts = mock_existing_alerts
|
53
|
+
interferon = Interferon::Interferon.new(nil,nil,nil,nil)
|
54
|
+
added = create_test_alert('name1', 'testquery3', '')
|
55
|
+
expect(dest).to receive(:create_alert).once.and_call_original
|
56
|
+
expect(dest).to receive(:remove_alert_by_id).with('[-dry-run-]name1').once
|
57
|
+
|
58
|
+
interferon.update_alerts_on_destination(dest, ['host'], [added], {}, true)
|
59
|
+
end
|
60
|
+
|
61
|
+
def mock_existing_alerts
|
62
|
+
alert1 = mock_alert_json('name1', 'testquery1', '')
|
63
|
+
alert2 = mock_alert_json('name2', 'testquery2', '')
|
64
|
+
{'name1'=>alert1, 'name2'=>alert2}
|
65
|
+
end
|
66
|
+
|
67
|
+
class MockDest < Interferon::Destinations::Datadog
|
68
|
+
@existing_alerts
|
69
|
+
|
70
|
+
def initialize(the_existing_alerts)
|
71
|
+
@existing_alerts = the_existing_alerts
|
72
|
+
end
|
73
|
+
|
74
|
+
def create_alert(alert, people)
|
75
|
+
name = alert['name']
|
76
|
+
[name, name]
|
77
|
+
end
|
78
|
+
|
79
|
+
def existing_alerts
|
80
|
+
@existing_alerts
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
def mock_alert_json(name, datadog_query, message)
|
86
|
+
{'name'=> name,'query'=> datadog_query,'message'=> message}
|
87
|
+
end
|
88
|
+
|
89
|
+
def create_test_alert(name, datadog_query, message)
|
90
|
+
alert_dsl = MockAlertDSL.new
|
91
|
+
metric_dsl = MockMetricDSL.new
|
92
|
+
metric_dsl.datadog_query(datadog_query)
|
93
|
+
alert_dsl.metric(metric_dsl)
|
94
|
+
alert_dsl.name(name)
|
95
|
+
alert_dsl.applies(true)
|
96
|
+
alert_dsl.message(message)
|
97
|
+
notify_dsl = MockNotifyDSL.new
|
98
|
+
notify_dsl.groups(['a'])
|
99
|
+
alert_dsl.notify(notify_dsl)
|
100
|
+
MockAlert.new(alert_dsl)
|
101
|
+
end
|
102
|
+
end
|
metadata
CHANGED
@@ -1,104 +1,104 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: interferon
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.12
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Igor Serebryany
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-05-24 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: dogapi
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- - ~>
|
17
|
+
- - "~>"
|
18
18
|
- !ruby/object:Gem::Version
|
19
19
|
version: '1.11'
|
20
|
-
- -
|
20
|
+
- - ">="
|
21
21
|
- !ruby/object:Gem::Version
|
22
22
|
version: 1.11.1
|
23
23
|
type: :runtime
|
24
24
|
prerelease: false
|
25
25
|
version_requirements: !ruby/object:Gem::Requirement
|
26
26
|
requirements:
|
27
|
-
- - ~>
|
27
|
+
- - "~>"
|
28
28
|
- !ruby/object:Gem::Version
|
29
29
|
version: '1.11'
|
30
|
-
- -
|
30
|
+
- - ">="
|
31
31
|
- !ruby/object:Gem::Version
|
32
32
|
version: 1.11.1
|
33
33
|
- !ruby/object:Gem::Dependency
|
34
34
|
name: aws-sdk
|
35
35
|
requirement: !ruby/object:Gem::Requirement
|
36
36
|
requirements:
|
37
|
-
- - ~>
|
37
|
+
- - "~>"
|
38
38
|
- !ruby/object:Gem::Version
|
39
39
|
version: '1.35'
|
40
|
-
- -
|
40
|
+
- - ">="
|
41
41
|
- !ruby/object:Gem::Version
|
42
42
|
version: 1.35.1
|
43
43
|
type: :runtime
|
44
44
|
prerelease: false
|
45
45
|
version_requirements: !ruby/object:Gem::Requirement
|
46
46
|
requirements:
|
47
|
-
- - ~>
|
47
|
+
- - "~>"
|
48
48
|
- !ruby/object:Gem::Version
|
49
49
|
version: '1.35'
|
50
|
-
- -
|
50
|
+
- - ">="
|
51
51
|
- !ruby/object:Gem::Version
|
52
52
|
version: 1.35.1
|
53
53
|
- !ruby/object:Gem::Dependency
|
54
54
|
name: dogstatsd-ruby
|
55
55
|
requirement: !ruby/object:Gem::Requirement
|
56
56
|
requirements:
|
57
|
-
- - ~>
|
57
|
+
- - "~>"
|
58
58
|
- !ruby/object:Gem::Version
|
59
59
|
version: '1.4'
|
60
|
-
- -
|
60
|
+
- - ">="
|
61
61
|
- !ruby/object:Gem::Version
|
62
62
|
version: 1.4.1
|
63
63
|
type: :runtime
|
64
64
|
prerelease: false
|
65
65
|
version_requirements: !ruby/object:Gem::Requirement
|
66
66
|
requirements:
|
67
|
-
- - ~>
|
67
|
+
- - "~>"
|
68
68
|
- !ruby/object:Gem::Version
|
69
69
|
version: '1.4'
|
70
|
-
- -
|
70
|
+
- - ">="
|
71
71
|
- !ruby/object:Gem::Version
|
72
72
|
version: 1.4.1
|
73
73
|
- !ruby/object:Gem::Dependency
|
74
74
|
name: rspec
|
75
75
|
requirement: !ruby/object:Gem::Requirement
|
76
76
|
requirements:
|
77
|
-
- - ~>
|
77
|
+
- - "~>"
|
78
78
|
- !ruby/object:Gem::Version
|
79
79
|
version: '3.2'
|
80
80
|
type: :development
|
81
81
|
prerelease: false
|
82
82
|
version_requirements: !ruby/object:Gem::Requirement
|
83
83
|
requirements:
|
84
|
-
- - ~>
|
84
|
+
- - "~>"
|
85
85
|
- !ruby/object:Gem::Version
|
86
86
|
version: '3.2'
|
87
87
|
- !ruby/object:Gem::Dependency
|
88
88
|
name: pry
|
89
89
|
requirement: !ruby/object:Gem::Requirement
|
90
90
|
requirements:
|
91
|
-
- - ~>
|
91
|
+
- - "~>"
|
92
92
|
- !ruby/object:Gem::Version
|
93
93
|
version: '0.10'
|
94
94
|
type: :development
|
95
95
|
prerelease: false
|
96
96
|
version_requirements: !ruby/object:Gem::Requirement
|
97
97
|
requirements:
|
98
|
-
- - ~>
|
98
|
+
- - "~>"
|
99
99
|
- !ruby/object:Gem::Version
|
100
100
|
version: '0.10'
|
101
|
-
description:
|
101
|
+
description: ": Store metrics alerts in code!"
|
102
102
|
email:
|
103
103
|
- igor.serebryany@airbnb.com
|
104
104
|
executables:
|
@@ -106,8 +106,8 @@ executables:
|
|
106
106
|
extensions: []
|
107
107
|
extra_rdoc_files: []
|
108
108
|
files:
|
109
|
-
- .gitignore
|
110
|
-
- .travis.yml
|
109
|
+
- ".gitignore"
|
110
|
+
- ".travis.yml"
|
111
111
|
- Gemfile
|
112
112
|
- LICENSE
|
113
113
|
- README.md
|
@@ -135,12 +135,15 @@ files:
|
|
135
135
|
- spec/fixtures/loaders2/test_sources/order_test_source.rb
|
136
136
|
- spec/fixtures/loaders2/test_sources/secondary_source.rb
|
137
137
|
- spec/fixtures/loaders2/test_sources/test_source.rb
|
138
|
+
- spec/helpers/dsl_helper.rb
|
138
139
|
- spec/helpers/loader_helper.rb
|
139
140
|
- spec/helpers/logging_helper.rb
|
141
|
+
- spec/helpers/mock_alert.rb
|
140
142
|
- spec/helpers/optica_helper.rb
|
141
143
|
- spec/lib/interferon/host_sources/optica_services_spec.rb
|
142
144
|
- spec/lib/interferon/host_sources/optica_spec.rb
|
143
145
|
- spec/lib/interferon/loaders_spec.rb
|
146
|
+
- spec/lib/interferon_spec.rb
|
144
147
|
- spec/spec_helper.rb
|
145
148
|
homepage: https://www.github.com/airbnb/interferon
|
146
149
|
licenses:
|
@@ -152,20 +155,20 @@ require_paths:
|
|
152
155
|
- lib
|
153
156
|
required_ruby_version: !ruby/object:Gem::Requirement
|
154
157
|
requirements:
|
155
|
-
- -
|
158
|
+
- - ">="
|
156
159
|
- !ruby/object:Gem::Version
|
157
160
|
version: '0'
|
158
161
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
159
162
|
requirements:
|
160
|
-
- -
|
163
|
+
- - ">="
|
161
164
|
- !ruby/object:Gem::Version
|
162
165
|
version: '0'
|
163
166
|
requirements: []
|
164
167
|
rubyforge_project:
|
165
|
-
rubygems_version: 2.
|
168
|
+
rubygems_version: 2.4.5.1
|
166
169
|
signing_key:
|
167
170
|
specification_version: 4
|
168
|
-
summary:
|
171
|
+
summary: ": Store metrics alerts in code!"
|
169
172
|
test_files:
|
170
173
|
- spec/fixtures/loaders/host_sources/test_host_source.rb
|
171
174
|
- spec/fixtures/loaders/test_sources/order_test_source.rb
|
@@ -173,10 +176,13 @@ test_files:
|
|
173
176
|
- spec/fixtures/loaders2/test_sources/order_test_source.rb
|
174
177
|
- spec/fixtures/loaders2/test_sources/secondary_source.rb
|
175
178
|
- spec/fixtures/loaders2/test_sources/test_source.rb
|
179
|
+
- spec/helpers/dsl_helper.rb
|
176
180
|
- spec/helpers/loader_helper.rb
|
177
181
|
- spec/helpers/logging_helper.rb
|
182
|
+
- spec/helpers/mock_alert.rb
|
178
183
|
- spec/helpers/optica_helper.rb
|
179
184
|
- spec/lib/interferon/host_sources/optica_services_spec.rb
|
180
185
|
- spec/lib/interferon/host_sources/optica_spec.rb
|
181
186
|
- spec/lib/interferon/loaders_spec.rb
|
187
|
+
- spec/lib/interferon_spec.rb
|
182
188
|
- spec/spec_helper.rb
|