interferon 0.0.7 → 0.0.12
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/config.example.yaml +1 -0
- data/lib/interferon/alert.rb +8 -0
- data/lib/interferon/destinations/datadog.rb +43 -8
- data/lib/interferon/version.rb +1 -1
- data/lib/interferon.rb +169 -103
- data/spec/helpers/dsl_helper.rb +39 -0
- data/spec/helpers/mock_alert.rb +11 -0
- data/spec/lib/interferon_spec.rb +102 -0
- metadata +31 -25
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 35b5382f5aabd274b548257bcc5f227faf1b951e
|
4
|
+
data.tar.gz: abacd2609b40bf28b2783bbd21b24c384f991188
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: b215a4e355f66daa582d65b521fe0eef5842eb795551df1998844d82cb1892a64b7a95e3b60fff1ec2992eb2e88de2f584c274c9bd5570b6f0ae4026b05eeb5a
|
7
|
+
data.tar.gz: 2bb6ee4093fe2bbe46d811a3cb79a357066073727a699d773b306a5355fd8f7b9d06a4f2bdb51082ac6570a3166820ce7f1b88d63f9fb76d6d032fc8b97073b0
|
data/config.example.yaml
CHANGED
data/lib/interferon/alert.rb
CHANGED
@@ -15,8 +15,22 @@ module Interferon::Destinations
|
|
15
15
|
end
|
16
16
|
end
|
17
17
|
|
18
|
-
|
19
|
-
|
18
|
+
# Set dogapi timeout explicitly
|
19
|
+
api_timeout = options['api_timeout'] || 15
|
20
|
+
|
21
|
+
# Default parameters of Dogapi Client initialize() can be referenced from link below:
|
22
|
+
# (as of this writing)
|
23
|
+
# https://github.com/DataDog/dogapi-rb/blob/master/lib/dogapi/facade.rb#L14
|
24
|
+
args = [
|
25
|
+
options['api_key'],
|
26
|
+
options['app_key'],
|
27
|
+
nil, # host to talk to
|
28
|
+
nil, # device
|
29
|
+
true, # silent?
|
30
|
+
api_timeout, # API timeout
|
31
|
+
]
|
32
|
+
@dog = Dogapi::Client.new(*args)
|
33
|
+
|
20
34
|
@existing_alerts = nil
|
21
35
|
|
22
36
|
# create datadog alerts 10 at a time
|
@@ -34,9 +48,19 @@ module Interferon::Destinations
|
|
34
48
|
}
|
35
49
|
end
|
36
50
|
|
51
|
+
def api_errors
|
52
|
+
@api_errors ||= []
|
53
|
+
end
|
54
|
+
|
37
55
|
def existing_alerts
|
38
56
|
unless @existing_alerts
|
39
57
|
resp = @dog.get_all_alerts()
|
58
|
+
|
59
|
+
code = resp[0].to_i
|
60
|
+
if code != 200
|
61
|
+
raise "Failed to retrieve existing alerts from datadog. #{code.to_s}: #{resp[1].inspect}"
|
62
|
+
end
|
63
|
+
|
40
64
|
alerts = resp[1]['alerts']
|
41
65
|
|
42
66
|
# key alerts by name
|
@@ -87,7 +111,7 @@ module Interferon::Destinations
|
|
87
111
|
resp = @dog.alert(
|
88
112
|
alert['metric']['datadog_query'].strip,
|
89
113
|
alert_opts,
|
90
|
-
)
|
114
|
+
)
|
91
115
|
|
92
116
|
# existing alert, modify it
|
93
117
|
else
|
@@ -99,11 +123,14 @@ module Interferon::Destinations
|
|
99
123
|
id,
|
100
124
|
alert['metric']['datadog_query'].strip,
|
101
125
|
alert_opts
|
102
|
-
)
|
126
|
+
)
|
103
127
|
end
|
104
128
|
|
105
129
|
# log whenever we've encountered errors
|
106
|
-
code =
|
130
|
+
code = resp[0].to_i
|
131
|
+
if code != 200
|
132
|
+
api_errors << "#{code.to_s} on alert #{alert['name']}"
|
133
|
+
end
|
107
134
|
|
108
135
|
# client error
|
109
136
|
if code == 400
|
@@ -113,7 +140,8 @@ module Interferon::Destinations
|
|
113
140
|
|
114
141
|
@stats[:api_client_errors] += 1
|
115
142
|
log.error("client error while #{action} alert '#{alert['name']}';" \
|
116
|
-
" query was '#{alert['metric']['datadog_query'].strip}'"
|
143
|
+
" query was '#{alert['metric']['datadog_query'].strip}'" \
|
144
|
+
" response was #{resp[0]}:'#{resp[1].inspect}'")
|
117
145
|
|
118
146
|
# unknown (prob. datadog) error:
|
119
147
|
elsif code >= 400 || code == -1
|
@@ -138,14 +166,15 @@ module Interferon::Destinations
|
|
138
166
|
@stats[:alerts_silenced] += 1 if alert_opts[:silenced]
|
139
167
|
end
|
140
168
|
|
169
|
+
id = resp[1].nil? ? nil : resp[1]['id']
|
141
170
|
# lets key alerts by their name
|
142
|
-
return alert['name']
|
171
|
+
return [alert['name'], id]
|
143
172
|
end
|
144
173
|
|
145
174
|
def remove_alert(alert)
|
146
175
|
if alert['message'].include?(ALERT_KEY)
|
147
176
|
log.debug("deleting alert #{alert['id']} (#{alert['name']})")
|
148
|
-
@dog.delete_alert(alert['id'])
|
177
|
+
@dog.delete_alert(alert['id'])
|
149
178
|
@stats[:alerts_deleted] += 1
|
150
179
|
else
|
151
180
|
log.warn("not deleting manually-created alert #{alert['id']} (#{alert['name']})")
|
@@ -163,5 +192,11 @@ module Interferon::Destinations
|
|
163
192
|
@stats[:alerts_deleted],
|
164
193
|
]
|
165
194
|
end
|
195
|
+
|
196
|
+
def remove_alert_by_id(alert_id)
|
197
|
+
log.debug("deleting alert, id: #{alert_id}")
|
198
|
+
@dog.delete_alert(alert_id)
|
199
|
+
@stats[:alerts_deleted] += 1
|
200
|
+
end
|
166
201
|
end
|
167
202
|
end
|
data/lib/interferon/version.rb
CHANGED
data/lib/interferon.rb
CHANGED
@@ -18,6 +18,8 @@ module Interferon
|
|
18
18
|
include Logging
|
19
19
|
attr_accessor :host_sources, :destinations, :host_info
|
20
20
|
|
21
|
+
DRY_RUN_ALERTS_NAME_PREFIX = '[-dry-run-]'
|
22
|
+
|
21
23
|
# groups_sources is a hash from type => options for each group source
|
22
24
|
# host_sources is a hash from type => options for each host source
|
23
25
|
# destinations is a similiar hash from type => options for each alerter
|
@@ -41,13 +43,11 @@ module Interferon
|
|
41
43
|
groups = read_groups(@groups_sources)
|
42
44
|
hosts = read_hosts(@host_sources)
|
43
45
|
|
44
|
-
# make sure destinations know if it's a dry run
|
45
46
|
@destinations.each do |dest|
|
46
47
|
dest['options'] ||= {}
|
47
|
-
dest['options']['dry_run'] = dry_run
|
48
48
|
end
|
49
49
|
|
50
|
-
update_alerts(@destinations, hosts, alerts, groups)
|
50
|
+
update_alerts(@destinations, hosts, alerts, groups, dry_run)
|
51
51
|
|
52
52
|
if @request_shutdown
|
53
53
|
log.info "interferon #{run_desc} shut down by SIGTERM"
|
@@ -133,132 +133,198 @@ module Interferon
|
|
133
133
|
return hosts
|
134
134
|
end
|
135
135
|
|
136
|
-
def update_alerts(destinations, hosts, alerts, groups)
|
136
|
+
def update_alerts(destinations, hosts, alerts, groups, dry_run)
|
137
137
|
loader = DestinationsLoader.new([@alerts_repo_path])
|
138
138
|
loader.get_all(destinations).each do |dest|
|
139
139
|
break if @request_shutdown
|
140
140
|
log.info "updating alerts on #{dest.class.name}"
|
141
|
+
update_alerts_on_destination(dest, hosts, alerts, groups, dry_run)
|
142
|
+
end
|
143
|
+
end
|
141
144
|
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
# get already-defined alerts
|
146
|
-
existing_alerts = dest.existing_alerts.dup
|
147
|
-
existing_alerts.each{ |key, existing_alert| existing_alert['still_exists'] = false }
|
148
|
-
|
149
|
-
# create or update alerts; mark when we've done that
|
150
|
-
alerts_queue = Hash.new
|
151
|
-
alerts.each do |alert|
|
152
|
-
break if @request_shutdown
|
153
|
-
counters = {
|
154
|
-
:errors => 0,
|
155
|
-
:evals => 0,
|
156
|
-
:applies => 0,
|
157
|
-
:hosts => hosts.length
|
158
|
-
}
|
159
|
-
last_eval_error = nil
|
160
|
-
|
161
|
-
hosts.each do |hostinfo|
|
162
|
-
begin
|
163
|
-
alert.evaluate(hostinfo)
|
164
|
-
counters[:evals] += 1
|
165
|
-
rescue StandardError => e
|
166
|
-
log.debug "Evaluation of alert #{alert} failed in the context of host #{hostinfo}"
|
167
|
-
counters[:errors] += 1
|
168
|
-
last_eval_error = e
|
169
|
-
next
|
170
|
-
end
|
145
|
+
def update_alerts_on_destination(dest, hosts, alerts, groups, dry_run)
|
146
|
+
# track some counters/stats per destination
|
147
|
+
start_time = Time.new.to_f
|
171
148
|
|
172
|
-
|
173
|
-
|
174
|
-
log.debug "alert #{alert[:name]} doesn't apply to #{hostinfo.inspect}"
|
175
|
-
next
|
176
|
-
end
|
149
|
+
# get already-defined alerts
|
150
|
+
existing_alerts = dest.existing_alerts.dup
|
177
151
|
|
178
|
-
|
152
|
+
if dry_run
|
153
|
+
do_dry_run_update(dest, hosts, alerts, existing_alerts, groups)
|
154
|
+
else
|
155
|
+
do_regular_update(dest, hosts, alerts, existing_alerts, groups)
|
156
|
+
end
|
179
157
|
|
180
|
-
|
181
|
-
|
158
|
+
unless @request_shutdown
|
159
|
+
# run time summary
|
160
|
+
run_time = Time.new.to_f - start_time
|
161
|
+
statsd.histogram(
|
162
|
+
dry_run ? 'destinations.run_time.dry_run' : 'destinations.run_time',
|
163
|
+
run_time,
|
164
|
+
:tags => ["destination:#{dest.class.name}"])
|
165
|
+
log.info "#{dest.class.name} : run completed in %.2f seconds" % (run_time)
|
166
|
+
|
167
|
+
# report destination stats
|
168
|
+
dest.report_stats
|
169
|
+
end
|
182
170
|
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
end
|
171
|
+
if dry_run && !dest.api_errors.empty?
|
172
|
+
raise dest.api_errors.to_s
|
173
|
+
end
|
174
|
+
end
|
188
175
|
|
189
|
-
|
190
|
-
|
191
|
-
|
176
|
+
def do_dry_run_update(dest, hosts, alerts, existing_alerts, groups)
|
177
|
+
to_remove = existing_alerts.reject{|key, a| !key.start_with?(DRY_RUN_ALERTS_NAME_PREFIX)}
|
178
|
+
alerts_queue = build_alerts_queue(hosts, alerts, groups)
|
179
|
+
alerts_queue.reject!{|name, pair| !Interferon::need_dry_run(pair[0], existing_alerts)}
|
180
|
+
alerts_queue.each do |name, pair|
|
181
|
+
alert = pair[0]
|
182
|
+
alert.change_name(DRY_RUN_ALERTS_NAME_PREFIX + alert['name'])
|
183
|
+
end
|
192
184
|
|
193
|
-
|
194
|
-
|
195
|
-
|
185
|
+
# flush queue
|
186
|
+
created_alerts_key_ids = create_alerts(dest, alerts_queue)
|
187
|
+
created_alerts_ids = created_alerts_key_ids.map{|a| a[1]}
|
188
|
+
to_remove_ids = to_remove.empty? ? [] : to_remove.map{|a| a['id']}
|
189
|
+
# remove existing alerts that shouldn't exist
|
190
|
+
(created_alerts_ids + to_remove_ids).each do |id|
|
191
|
+
break if @request_shutdown
|
192
|
+
dest.remove_alert_by_id(id) unless id.nil?
|
193
|
+
end
|
194
|
+
end
|
196
195
|
|
197
|
-
|
198
|
-
|
199
|
-
end
|
196
|
+
def do_regular_update(dest, hosts, alerts, existing_alerts, groups)
|
197
|
+
existing_alerts.each{ |key, existing_alert| existing_alert['still_exists'] = false }
|
200
198
|
|
201
|
-
|
202
|
-
if counters[:errors] == counters[:hosts]
|
203
|
-
log.error "alert #{alert} failed to evaluate in the context of all hosts!"
|
204
|
-
log.error "last error on alert #{alert}: #{last_eval_error}"
|
199
|
+
alerts_queue = build_alerts_queue(hosts, alerts, groups)
|
205
200
|
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
201
|
+
# flush queue
|
202
|
+
created_alerts_keys = create_alerts(dest, alerts_queue).map{|a| a[0]}
|
203
|
+
created_alerts_keys.each do |alert_key|
|
204
|
+
# don't delete alerts we still have defined
|
205
|
+
existing_alerts[alert_key]['still_exists'] = true if existing_alerts.include?(alert_key)
|
206
|
+
end
|
207
|
+
|
208
|
+
# remove existing alerts that shouldn't exist
|
209
|
+
to_delete = existing_alerts.reject{ |key, existing_alert| existing_alert['still_exists'] }
|
210
|
+
to_delete.each do |key, alert|
|
211
|
+
break if @request_shutdown
|
212
|
+
dest.remove_alert(alert)
|
213
|
+
end
|
214
|
+
end
|
211
215
|
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
216
|
+
def create_alerts(dest, alerts_queue)
|
217
|
+
alert_key_ids = []
|
218
|
+
alerts_to_create = alerts_queue.keys
|
219
|
+
concurrency = dest.concurrency || 10
|
220
|
+
unless @request_shutdown
|
221
|
+
threads = concurrency.times.map do |i|
|
222
|
+
log.info "thread #{i} created"
|
223
|
+
t = Thread.new do
|
224
|
+
while name = alerts_to_create.shift
|
225
|
+
break if @request_shutdown
|
226
|
+
cur_alert, people = alerts_queue[name]
|
227
|
+
log.debug "creating alert for #{cur_alert[:name]}"
|
228
|
+
alert_key_ids << dest.create_alert(cur_alert, people)
|
229
|
+
end
|
218
230
|
end
|
231
|
+
t.abort_on_exception = true
|
232
|
+
t
|
219
233
|
end
|
234
|
+
threads.map(&:join)
|
235
|
+
end
|
236
|
+
alert_key_ids
|
237
|
+
end
|
220
238
|
|
221
|
-
|
222
|
-
|
223
|
-
|
224
|
-
|
225
|
-
|
226
|
-
|
227
|
-
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
232
|
-
|
233
|
-
|
234
|
-
|
235
|
-
|
236
|
-
|
237
|
-
|
238
|
-
|
239
|
-
|
240
|
-
|
239
|
+
def build_alerts_queue(hosts, alerts, groups)
|
240
|
+
# create or update alerts; mark when we've done that
|
241
|
+
alerts_queue = Hash.new
|
242
|
+
alerts.each do |alert|
|
243
|
+
break if @request_shutdown
|
244
|
+
counters = {
|
245
|
+
:errors => 0,
|
246
|
+
:evals => 0,
|
247
|
+
:applies => 0,
|
248
|
+
:hosts => hosts.length
|
249
|
+
}
|
250
|
+
last_eval_error = nil
|
251
|
+
|
252
|
+
hosts.each do |hostinfo|
|
253
|
+
begin
|
254
|
+
alert.evaluate(hostinfo)
|
255
|
+
counters[:evals] += 1
|
256
|
+
rescue StandardError => e
|
257
|
+
log.debug "Evaluation of alert #{alert} failed in the context of host #{hostinfo}"
|
258
|
+
counters[:errors] += 1
|
259
|
+
last_eval_error = e
|
260
|
+
next
|
241
261
|
end
|
242
|
-
|
262
|
+
|
263
|
+
# don't define an alert that doesn't apply to this hostinfo
|
264
|
+
unless alert[:applies]
|
265
|
+
log.debug "alert #{alert[:name]} doesn't apply to #{hostinfo.inspect}"
|
266
|
+
next
|
267
|
+
end
|
268
|
+
|
269
|
+
counters[:applies] += 1
|
270
|
+
# don't define alerts twice
|
271
|
+
next if alerts_queue.key?(alert[:name])
|
272
|
+
|
273
|
+
# figure out who to notify
|
274
|
+
people = Set.new(alert[:notify][:people])
|
275
|
+
alert[:notify][:groups].each do |g|
|
276
|
+
people += (groups[g] || [])
|
277
|
+
end
|
278
|
+
|
279
|
+
# queue the alert up for creation; we clone the alert to save the current state
|
280
|
+
alerts_queue[alert[:name]] ||= [alert.clone, people]
|
243
281
|
end
|
244
282
|
|
245
|
-
#
|
246
|
-
|
247
|
-
|
248
|
-
|
249
|
-
|
283
|
+
# log some of the counters
|
284
|
+
statsd.gauge('alerts.evaluate.errors', counters[:errors], :tags => ["alert:#{alert}"])
|
285
|
+
statsd.gauge('alerts.evaluate.applies', counters[:applies], :tags => ["alert:#{alert}"])
|
286
|
+
|
287
|
+
if counters[:applies] > 0
|
288
|
+
log.info "alert #{alert} applies to #{counters[:applies]} of #{counters[:hosts]} hosts"
|
250
289
|
end
|
251
290
|
|
252
|
-
|
253
|
-
|
254
|
-
|
255
|
-
|
256
|
-
|
291
|
+
# did the alert fail to evaluate on all hosts?
|
292
|
+
if counters[:errors] == counters[:hosts]
|
293
|
+
log.error "alert #{alert} failed to evaluate in the context of all hosts!"
|
294
|
+
log.error "last error on alert #{alert}: #{last_eval_error}"
|
295
|
+
|
296
|
+
statsd.gauge('alerts.evaluate.failed_on_all', 1, :tags => ["alert:#{alert}"])
|
297
|
+
log.debug "alert #{alert}: error #{last_eval_error}\n#{last_eval_error.backtrace.join("\n")}"
|
298
|
+
else
|
299
|
+
statsd.gauge('alerts.evaluate.failed_on_all', 0, :tags => ["alert:#{alert}"])
|
300
|
+
end
|
257
301
|
|
258
|
-
|
259
|
-
|
302
|
+
# did the alert apply to any hosts?
|
303
|
+
if counters[:applies] == 0
|
304
|
+
statsd.gauge('alerts.evaluate.never_applies', 1, :tags => ["alert:#{alert}"])
|
305
|
+
log.warn "alert #{alert} did not apply to any hosts"
|
306
|
+
else
|
307
|
+
statsd.gauge('alerts.evaluate.never_applies', 0, :tags => ["alert:#{alert}"])
|
260
308
|
end
|
261
309
|
end
|
310
|
+
alerts_queue
|
311
|
+
end
|
312
|
+
|
313
|
+
def self.need_dry_run(alert, existing_alerts_from_api)
|
314
|
+
existing = existing_alerts_from_api[alert['name']]
|
315
|
+
if existing.nil?
|
316
|
+
true
|
317
|
+
else
|
318
|
+
!same_alerts_for_dry_run_purpose(alert, existing)
|
319
|
+
end
|
320
|
+
end
|
321
|
+
|
322
|
+
def self.same_alerts_for_dry_run_purpose(alert, alert_api_json)
|
323
|
+
query1 = alert['metric']['datadog_query']
|
324
|
+
query2 = alert_api_json['query']
|
325
|
+
query1.strip!
|
326
|
+
query2.strip!
|
327
|
+
query1 == query2
|
262
328
|
end
|
263
329
|
end
|
264
330
|
end
|
@@ -0,0 +1,39 @@
|
|
1
|
+
module Interferon
|
2
|
+
module MockDSLMixin
|
3
|
+
def initialize
|
4
|
+
end
|
5
|
+
|
6
|
+
def get_or_set(field, val, block, default)
|
7
|
+
@hash ||= Hash.new
|
8
|
+
if val.nil?
|
9
|
+
return @hash[field]
|
10
|
+
else
|
11
|
+
@hash[field] = val
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
class MockAlertDSL < AlertDSL
|
17
|
+
include MockDSLMixin
|
18
|
+
|
19
|
+
def notify(v = nil)
|
20
|
+
get_or_set(:notify, v, nil, nil)
|
21
|
+
end
|
22
|
+
|
23
|
+
def metric(v = nil)
|
24
|
+
get_or_set(:metric, v, nil, nil)
|
25
|
+
end
|
26
|
+
|
27
|
+
def id(v = nil, &block)
|
28
|
+
get_or_set(:@id, v, block, '')
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
class MockNotifyDSL < NotifyDSL
|
33
|
+
include MockDSLMixin
|
34
|
+
end
|
35
|
+
|
36
|
+
class MockMetricDSL < MetricDSL
|
37
|
+
include MockDSLMixin
|
38
|
+
end
|
39
|
+
end
|
@@ -0,0 +1,102 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'helpers/mock_alert'
|
3
|
+
require 'helpers/dsl_helper'
|
4
|
+
require 'interferon/destinations/datadog'
|
5
|
+
|
6
|
+
include Interferon
|
7
|
+
|
8
|
+
describe Interferon::Interferon do
|
9
|
+
it "would not be dry run if only alert message changed" do
|
10
|
+
alert1 = create_test_alert('name1', 'testquery', 'message1')
|
11
|
+
alert2 = mock_alert_json('name2', 'testquery', 'message2')
|
12
|
+
|
13
|
+
expect(Interferon::Interferon.same_alerts_for_dry_run_purpose(alert1, alert2)).to be true
|
14
|
+
end
|
15
|
+
|
16
|
+
it "would be dry run if alert datadog query changed" do
|
17
|
+
alert1 = create_test_alert('name1', 'testquery1', 'message1')
|
18
|
+
alert2 = mock_alert_json('name2', 'testquery2', 'message2')
|
19
|
+
|
20
|
+
expect(Interferon::Interferon.same_alerts_for_dry_run_purpose(alert1, alert2)).to be false
|
21
|
+
end
|
22
|
+
|
23
|
+
context "dry_run_update_alerts_on_destination" do
|
24
|
+
let(:the_existing_alerts) {mock_existing_alerts}
|
25
|
+
let(:dest) {MockDest.new(the_existing_alerts)}
|
26
|
+
before do
|
27
|
+
allow_any_instance_of(MockAlert).to receive(:evaluate)
|
28
|
+
allow(dest).to receive(:remove_alert_by_id)
|
29
|
+
allow(dest).to receive(:report_stats)
|
30
|
+
end
|
31
|
+
|
32
|
+
it 'dry run does not re-run existing alerts' do
|
33
|
+
alerts = mock_existing_alerts
|
34
|
+
interferon = Interferon::Interferon.new(nil,nil,nil,nil)
|
35
|
+
expect(dest).not_to receive(:create_alert)
|
36
|
+
expect(dest).not_to receive(:remove_alert_by_id)
|
37
|
+
|
38
|
+
interferon.update_alerts_on_destination(dest, ['host'], [alerts['name1'], alerts['name2']], {}, true)
|
39
|
+
end
|
40
|
+
|
41
|
+
it 'dry run runs added alerts' do
|
42
|
+
alerts = mock_existing_alerts
|
43
|
+
interferon = Interferon::Interferon.new(nil,nil,nil,nil)
|
44
|
+
added = create_test_alert('name3', 'testquery3', '')
|
45
|
+
expect(dest).to receive(:create_alert).once.and_call_original
|
46
|
+
expect(dest).to receive(:remove_alert_by_id).with('[-dry-run-]name3').once
|
47
|
+
|
48
|
+
interferon.update_alerts_on_destination(dest, ['host'], [alerts['name1'], alerts['name2'], added], {}, true)
|
49
|
+
end
|
50
|
+
|
51
|
+
it 'dry run runs updated alerts' do
|
52
|
+
alerts = mock_existing_alerts
|
53
|
+
interferon = Interferon::Interferon.new(nil,nil,nil,nil)
|
54
|
+
added = create_test_alert('name1', 'testquery3', '')
|
55
|
+
expect(dest).to receive(:create_alert).once.and_call_original
|
56
|
+
expect(dest).to receive(:remove_alert_by_id).with('[-dry-run-]name1').once
|
57
|
+
|
58
|
+
interferon.update_alerts_on_destination(dest, ['host'], [added], {}, true)
|
59
|
+
end
|
60
|
+
|
61
|
+
def mock_existing_alerts
|
62
|
+
alert1 = mock_alert_json('name1', 'testquery1', '')
|
63
|
+
alert2 = mock_alert_json('name2', 'testquery2', '')
|
64
|
+
{'name1'=>alert1, 'name2'=>alert2}
|
65
|
+
end
|
66
|
+
|
67
|
+
class MockDest < Interferon::Destinations::Datadog
|
68
|
+
@existing_alerts
|
69
|
+
|
70
|
+
def initialize(the_existing_alerts)
|
71
|
+
@existing_alerts = the_existing_alerts
|
72
|
+
end
|
73
|
+
|
74
|
+
def create_alert(alert, people)
|
75
|
+
name = alert['name']
|
76
|
+
[name, name]
|
77
|
+
end
|
78
|
+
|
79
|
+
def existing_alerts
|
80
|
+
@existing_alerts
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
def mock_alert_json(name, datadog_query, message)
|
86
|
+
{'name'=> name,'query'=> datadog_query,'message'=> message}
|
87
|
+
end
|
88
|
+
|
89
|
+
def create_test_alert(name, datadog_query, message)
|
90
|
+
alert_dsl = MockAlertDSL.new
|
91
|
+
metric_dsl = MockMetricDSL.new
|
92
|
+
metric_dsl.datadog_query(datadog_query)
|
93
|
+
alert_dsl.metric(metric_dsl)
|
94
|
+
alert_dsl.name(name)
|
95
|
+
alert_dsl.applies(true)
|
96
|
+
alert_dsl.message(message)
|
97
|
+
notify_dsl = MockNotifyDSL.new
|
98
|
+
notify_dsl.groups(['a'])
|
99
|
+
alert_dsl.notify(notify_dsl)
|
100
|
+
MockAlert.new(alert_dsl)
|
101
|
+
end
|
102
|
+
end
|
metadata
CHANGED
@@ -1,104 +1,104 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: interferon
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.12
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Igor Serebryany
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-05-24 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: dogapi
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- - ~>
|
17
|
+
- - "~>"
|
18
18
|
- !ruby/object:Gem::Version
|
19
19
|
version: '1.11'
|
20
|
-
- -
|
20
|
+
- - ">="
|
21
21
|
- !ruby/object:Gem::Version
|
22
22
|
version: 1.11.1
|
23
23
|
type: :runtime
|
24
24
|
prerelease: false
|
25
25
|
version_requirements: !ruby/object:Gem::Requirement
|
26
26
|
requirements:
|
27
|
-
- - ~>
|
27
|
+
- - "~>"
|
28
28
|
- !ruby/object:Gem::Version
|
29
29
|
version: '1.11'
|
30
|
-
- -
|
30
|
+
- - ">="
|
31
31
|
- !ruby/object:Gem::Version
|
32
32
|
version: 1.11.1
|
33
33
|
- !ruby/object:Gem::Dependency
|
34
34
|
name: aws-sdk
|
35
35
|
requirement: !ruby/object:Gem::Requirement
|
36
36
|
requirements:
|
37
|
-
- - ~>
|
37
|
+
- - "~>"
|
38
38
|
- !ruby/object:Gem::Version
|
39
39
|
version: '1.35'
|
40
|
-
- -
|
40
|
+
- - ">="
|
41
41
|
- !ruby/object:Gem::Version
|
42
42
|
version: 1.35.1
|
43
43
|
type: :runtime
|
44
44
|
prerelease: false
|
45
45
|
version_requirements: !ruby/object:Gem::Requirement
|
46
46
|
requirements:
|
47
|
-
- - ~>
|
47
|
+
- - "~>"
|
48
48
|
- !ruby/object:Gem::Version
|
49
49
|
version: '1.35'
|
50
|
-
- -
|
50
|
+
- - ">="
|
51
51
|
- !ruby/object:Gem::Version
|
52
52
|
version: 1.35.1
|
53
53
|
- !ruby/object:Gem::Dependency
|
54
54
|
name: dogstatsd-ruby
|
55
55
|
requirement: !ruby/object:Gem::Requirement
|
56
56
|
requirements:
|
57
|
-
- - ~>
|
57
|
+
- - "~>"
|
58
58
|
- !ruby/object:Gem::Version
|
59
59
|
version: '1.4'
|
60
|
-
- -
|
60
|
+
- - ">="
|
61
61
|
- !ruby/object:Gem::Version
|
62
62
|
version: 1.4.1
|
63
63
|
type: :runtime
|
64
64
|
prerelease: false
|
65
65
|
version_requirements: !ruby/object:Gem::Requirement
|
66
66
|
requirements:
|
67
|
-
- - ~>
|
67
|
+
- - "~>"
|
68
68
|
- !ruby/object:Gem::Version
|
69
69
|
version: '1.4'
|
70
|
-
- -
|
70
|
+
- - ">="
|
71
71
|
- !ruby/object:Gem::Version
|
72
72
|
version: 1.4.1
|
73
73
|
- !ruby/object:Gem::Dependency
|
74
74
|
name: rspec
|
75
75
|
requirement: !ruby/object:Gem::Requirement
|
76
76
|
requirements:
|
77
|
-
- - ~>
|
77
|
+
- - "~>"
|
78
78
|
- !ruby/object:Gem::Version
|
79
79
|
version: '3.2'
|
80
80
|
type: :development
|
81
81
|
prerelease: false
|
82
82
|
version_requirements: !ruby/object:Gem::Requirement
|
83
83
|
requirements:
|
84
|
-
- - ~>
|
84
|
+
- - "~>"
|
85
85
|
- !ruby/object:Gem::Version
|
86
86
|
version: '3.2'
|
87
87
|
- !ruby/object:Gem::Dependency
|
88
88
|
name: pry
|
89
89
|
requirement: !ruby/object:Gem::Requirement
|
90
90
|
requirements:
|
91
|
-
- - ~>
|
91
|
+
- - "~>"
|
92
92
|
- !ruby/object:Gem::Version
|
93
93
|
version: '0.10'
|
94
94
|
type: :development
|
95
95
|
prerelease: false
|
96
96
|
version_requirements: !ruby/object:Gem::Requirement
|
97
97
|
requirements:
|
98
|
-
- - ~>
|
98
|
+
- - "~>"
|
99
99
|
- !ruby/object:Gem::Version
|
100
100
|
version: '0.10'
|
101
|
-
description:
|
101
|
+
description: ": Store metrics alerts in code!"
|
102
102
|
email:
|
103
103
|
- igor.serebryany@airbnb.com
|
104
104
|
executables:
|
@@ -106,8 +106,8 @@ executables:
|
|
106
106
|
extensions: []
|
107
107
|
extra_rdoc_files: []
|
108
108
|
files:
|
109
|
-
- .gitignore
|
110
|
-
- .travis.yml
|
109
|
+
- ".gitignore"
|
110
|
+
- ".travis.yml"
|
111
111
|
- Gemfile
|
112
112
|
- LICENSE
|
113
113
|
- README.md
|
@@ -135,12 +135,15 @@ files:
|
|
135
135
|
- spec/fixtures/loaders2/test_sources/order_test_source.rb
|
136
136
|
- spec/fixtures/loaders2/test_sources/secondary_source.rb
|
137
137
|
- spec/fixtures/loaders2/test_sources/test_source.rb
|
138
|
+
- spec/helpers/dsl_helper.rb
|
138
139
|
- spec/helpers/loader_helper.rb
|
139
140
|
- spec/helpers/logging_helper.rb
|
141
|
+
- spec/helpers/mock_alert.rb
|
140
142
|
- spec/helpers/optica_helper.rb
|
141
143
|
- spec/lib/interferon/host_sources/optica_services_spec.rb
|
142
144
|
- spec/lib/interferon/host_sources/optica_spec.rb
|
143
145
|
- spec/lib/interferon/loaders_spec.rb
|
146
|
+
- spec/lib/interferon_spec.rb
|
144
147
|
- spec/spec_helper.rb
|
145
148
|
homepage: https://www.github.com/airbnb/interferon
|
146
149
|
licenses:
|
@@ -152,20 +155,20 @@ require_paths:
|
|
152
155
|
- lib
|
153
156
|
required_ruby_version: !ruby/object:Gem::Requirement
|
154
157
|
requirements:
|
155
|
-
- -
|
158
|
+
- - ">="
|
156
159
|
- !ruby/object:Gem::Version
|
157
160
|
version: '0'
|
158
161
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
159
162
|
requirements:
|
160
|
-
- -
|
163
|
+
- - ">="
|
161
164
|
- !ruby/object:Gem::Version
|
162
165
|
version: '0'
|
163
166
|
requirements: []
|
164
167
|
rubyforge_project:
|
165
|
-
rubygems_version: 2.
|
168
|
+
rubygems_version: 2.4.5.1
|
166
169
|
signing_key:
|
167
170
|
specification_version: 4
|
168
|
-
summary:
|
171
|
+
summary: ": Store metrics alerts in code!"
|
169
172
|
test_files:
|
170
173
|
- spec/fixtures/loaders/host_sources/test_host_source.rb
|
171
174
|
- spec/fixtures/loaders/test_sources/order_test_source.rb
|
@@ -173,10 +176,13 @@ test_files:
|
|
173
176
|
- spec/fixtures/loaders2/test_sources/order_test_source.rb
|
174
177
|
- spec/fixtures/loaders2/test_sources/secondary_source.rb
|
175
178
|
- spec/fixtures/loaders2/test_sources/test_source.rb
|
179
|
+
- spec/helpers/dsl_helper.rb
|
176
180
|
- spec/helpers/loader_helper.rb
|
177
181
|
- spec/helpers/logging_helper.rb
|
182
|
+
- spec/helpers/mock_alert.rb
|
178
183
|
- spec/helpers/optica_helper.rb
|
179
184
|
- spec/lib/interferon/host_sources/optica_services_spec.rb
|
180
185
|
- spec/lib/interferon/host_sources/optica_spec.rb
|
181
186
|
- spec/lib/interferon/loaders_spec.rb
|
187
|
+
- spec/lib/interferon_spec.rb
|
182
188
|
- spec/spec_helper.rb
|