interferon 0.0.19 → 0.0.20

Sign up to get free protection for your applications and to get access to all the features.
Files changed (8) hide show
  1. data/README.md +2 -2
  2. data/lib/interferon/alert.rb +1 -0
  3. data/lib/interferon/destinations/datadog.rb +27 -9
  4. data/lib/interferon/version.rb +1 -1
  5. data/lib/interferon.rb +1 -1
  6. data/spec/lib/interferon/destinations/datadog_spec.rb +111 -0
  7. metadata +27 -5
  8. checksums.yaml +0 -15
data/README.md CHANGED
@@ -30,7 +30,7 @@ It accepts the following parameters:
30
30
  * `host_sources` -- a list of sources which can read inventory systems and return lists of hosts to monitor
31
31
  * `destinations` -- a list of alerting providers, which can monitor metrics and dispatch alerts as specified in your alerts dsl files
32
32
 
33
- For more information, see [config.example.yaml](blob/master/config.example.yaml) file in this repo.
33
+ For more information, see [config.example.yaml](config.example.yaml) file in this repo.
34
34
 
35
35
  ## The Moving Parts ##
36
36
 
@@ -74,7 +74,7 @@ Here's a chart explaining the datadog metric syntax ([generated via asciiflow](h
74
74
  | | |-----------------------------------| +------------------------------------+
75
75
  | | | * max, min, avg, sum | |trigger a separate alert for each |
76
76
  | + +-----------------------------------+ |different value of these tags the |
77
- | +----+----------------------------------------------+ |entire `by {}` clause can be ommited|
77
+ | +----+----------------------------------------------+ |entire `by {}` clause can be omitted|
78
78
  | | the interval to look at; always starts with last_ | +------------------------------------+
79
79
  | |---------------------------------------------------|
80
80
  | | * 5m, 10m, 15m, 30m |
data/lib/interferon/alert.rb CHANGED
@@ -16,6 +16,7 @@ module Interferon
16
16
  def evaluate(hostinfo)
17
17
  dsl = AlertDSL.new(hostinfo)
18
18
  dsl.instance_eval(@text, @filename, 1)
19
+ dsl.name(dsl.name.strip)
19
20
  @dsl = dsl
20
21
 
21
22
  # return the alert and not the DSL object, which is private
data/lib/interferon/destinations/datadog.rb CHANGED
@@ -38,7 +38,9 @@ module Interferon::Destinations
38
38
  @dry_run = options['dry_run']
39
39
 
40
40
  # create datadog alerts 10 at a time
41
- @concurrency = 10
41
+ @concurrency = options['concurrency'] || 10
42
+ # configure retries
43
+ @retries = options['retries'] || 3
42
44
 
43
45
  @stats = {
44
46
  :alerts_created => 0,
@@ -63,17 +65,27 @@ module Interferon::Destinations
63
65
  [message, ALERT_KEY, people.map{ |p| "@#{p}" }].flatten.join("\n")
64
66
  end
65
67
 
68
+ def get_existing_alerts
69
+ resp = @dog.get_all_alerts()
70
+
71
+ code = resp[0].to_i
72
+ if code != 200
73
+ raise "Failed to retrieve existing alerts from datadog. #{code}: #{resp[1].inspect}"
74
+ end
75
+ resp[1]['alerts']
76
+ end
77
+
66
78
  def existing_alerts
67
79
  unless @existing_alerts
68
- resp = @dog.get_all_alerts()
69
-
70
- code = resp[0].to_i
71
- if code != 200
72
- raise "Failed to retrieve existing alerts from datadog. #{code.to_s}: #{resp[1].inspect}"
80
+ retries = @retries
81
+ begin
82
+ alerts = get_existing_alerts
83
+ rescue
84
+ retries -= 1
85
+ retry if retries >= 0
86
+ raise
73
87
  end
74
88
 
75
- alerts = resp[1]['alerts']
76
-
77
89
  # key alerts by name
78
90
  @existing_alerts = {}
79
91
  alerts.each do |alert|
@@ -112,6 +124,12 @@ module Interferon::Destinations
112
124
  :timeout_h => nil,
113
125
  }
114
126
 
127
+ if @dry_run
128
+ # Datadog may have a race condition where alerts created in a bad state may be triggered
129
+ # during the dry-run creation process. Delete people from dry-run alerts to avoid this
130
+ alert_opts[:message] = generate_message(alert['message'], [])
131
+ end
132
+
115
133
  # Set alert to be silenced if there is a silenced set or silenced_until set
116
134
  if alert['silenced'] || alert['silenced_until'] > Time.now
117
135
  alert_opts[:silenced] = true
@@ -234,7 +252,7 @@ module Interferon::Destinations
234
252
  def log_datadog_response_code(resp, code, action, alert=nil)
235
253
  # log whenever we've encountered errors
236
254
  if code != 200 && !alert.nil?
237
- api_errors << "#{code.to_s} on alert #{alert['name']}"
255
+ api_errors << "#{code} on alert #{alert['name']}"
238
256
  end
239
257
 
240
258
  # client error
data/lib/interferon/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module Interferon
2
- VERSION = "0.0.19"
2
+ VERSION = "0.0.20"
3
3
  end
data/lib/interferon.rb CHANGED
@@ -23,7 +23,7 @@ module Interferon
23
23
 
24
24
  # groups_sources is a hash from type => options for each group source
25
25
  # host_sources is a hash from type => options for each host source
26
- # destinations is a similiar hash from type => options for each alerter
26
+ # destinations is a similar hash from type => options for each alerter
27
27
  def initialize(alerts_repo_path, groups_sources, host_sources, destinations,
28
28
  dry_run=false, processes=nil)
29
29
  @alerts_repo_path = alerts_repo_path
data/spec/lib/interferon/destinations/datadog_spec.rb ADDED
@@ -0,0 +1,111 @@
1
+ require 'spec_helper'
2
+ require 'interferon/destinations/datadog'
3
+
4
+ describe Interferon::Destinations::Datadog do
5
+ let(:retries) { 3 }
6
+ let(:datadog) {
7
+ Interferon::Destinations::Datadog.new({
8
+ 'api_key' => 'TEST_API_KEY',
9
+ 'app_key' => 'TEST_APP_KEY',
10
+ 'retries' => retries,
11
+ })
12
+ }
13
+ let(:datadog_dry_run) {
14
+ Interferon::Destinations::Datadog.new({
15
+ 'api_key' => 'TEST_API_KEY',
16
+ 'app_key' => 'TEST_APP_KEY',
17
+ 'retries' => retries,
18
+ 'dry_run' => true,
19
+ })
20
+ }
21
+ let(:mock_alert_id) { 123 }
22
+ let(:mock_alert) {
23
+ {
24
+ 'id' => [mock_alert_id],
25
+ 'name' => 'Test Alert',
26
+ 'message' => "Test Message",
27
+ 'metric' => { 'datadog_query' => 'avg:metric{*}' },
28
+ 'silenced' => false,
29
+ 'silenced_until' => Time.at(0),
30
+ }
31
+ }
32
+ let(:mock_people) { ['foo', 'bar', 'baz'] }
33
+
34
+ describe ".get_existing_alerts" do
35
+ it "calls dogapi get_all_alerts" do
36
+ expect_any_instance_of(Dogapi::Client).to receive(:get_all_alerts).and_return([200, ""])
37
+ datadog.get_existing_alerts
38
+ end
39
+ end
40
+
41
+ describe ".existing_alerts" do
42
+ it "retries dogapi get_all_alerts" do
43
+ return_vals = [[400, ""]] * (retries + 1)
44
+ expect_any_instance_of(Dogapi::Client).to receive(:get_all_alerts).and_return(*return_vals)
45
+ expect { datadog.existing_alerts }.to raise_error RuntimeError
46
+ end
47
+ end
48
+
49
+ describe ".create_alert" do
50
+ it "calls dogapi alert" do
51
+ expect_any_instance_of(Dogapi::Client).to receive(:alert).and_return([200, ""])
52
+ expect(datadog).to receive(:existing_alerts).and_return({})
53
+ datadog.create_alert(mock_alert, mock_people)
54
+ end
55
+
56
+ it "calls dogapi update_alert when alert name is found" do
57
+ expect_any_instance_of(Dogapi::Client).to receive(:update_alert).and_return([200, ""])
58
+ expect(datadog).to receive(:existing_alerts).and_return(
59
+ {
60
+ "Test Alert" => {
61
+ "id" => 567,
62
+ "name" => 'Test Alert',
63
+ }
64
+ }
65
+ )
66
+ datadog.create_alert(mock_alert, mock_people)
67
+ end
68
+
69
+ it "always calls alert in dry-run" do
70
+ expect_any_instance_of(Dogapi::Client).to receive(:alert).and_return([200, ""])
71
+ expect(datadog_dry_run).to receive(:existing_alerts).and_return(
72
+ {
73
+ "Test Alert" => {
74
+ "id" => 567,
75
+ "name" => 'Test Alert',
76
+ }
77
+ }
78
+ )
79
+ datadog_dry_run.create_alert(mock_alert, mock_people)
80
+ end
81
+ end
82
+
83
+ describe ".remove_alert" do
84
+ it "calls dogapi delete_alert with the correct alert id" do
85
+ mock_alert["message"] += Interferon::Destinations::Datadog::ALERT_KEY
86
+ expect_any_instance_of(Dogapi::Client).to receive(:delete_alert).
87
+ with(mock_alert_id).and_return([200, ""])
88
+ datadog.remove_alert(mock_alert)
89
+ end
90
+
91
+ it "does not call dogapi delete_alert in dry_run" do
92
+ mock_alert["message"] += Interferon::Destinations::Datadog::ALERT_KEY
93
+ expect_any_instance_of(Dogapi::Client).to_not receive(:delete_alert)
94
+ datadog_dry_run.remove_alert(mock_alert)
95
+ end
96
+
97
+ it "does not call dogapi delete_alert when ALERT_KEY is missing" do
98
+ expect_any_instance_of(Dogapi::Client).to_not receive(:delete_alert)
99
+ datadog.remove_alert(mock_alert)
100
+ end
101
+
102
+ end
103
+
104
+ describe ".remove_alert_by_id" do
105
+ it "calls dogapi delete_alert" do
106
+ expect_any_instance_of(Dogapi::Client).to receive(:delete_alert).
107
+ with(mock_alert_id).and_return([200, ""])
108
+ datadog.remove_alert_by_id(mock_alert_id)
109
+ end
110
+ end
111
+ end
metadata CHANGED
@@ -1,7 +1,8 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: interferon
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.19
4
+ version: 0.0.20
5
+ prerelease:
5
6
  platform: ruby
6
7
  authors:
7
8
  - Igor Serebryany
@@ -9,11 +10,12 @@ authors:
9
10
  autorequire:
10
11
  bindir: bin
11
12
  cert_chain: []
12
- date: 2017-01-05 00:00:00.000000000 Z
13
+ date: 2017-04-03 00:00:00.000000000 Z
13
14
  dependencies:
14
15
  - !ruby/object:Gem::Dependency
15
16
  name: dogapi
16
17
  requirement: !ruby/object:Gem::Requirement
18
+ none: false
17
19
  requirements:
18
20
  - - ~>
19
21
  - !ruby/object:Gem::Version
@@ -24,6 +26,7 @@ dependencies:
24
26
  type: :runtime
25
27
  prerelease: false
26
28
  version_requirements: !ruby/object:Gem::Requirement
29
+ none: false
27
30
  requirements:
28
31
  - - ~>
29
32
  - !ruby/object:Gem::Version
@@ -34,6 +37,7 @@ dependencies:
34
37
  - !ruby/object:Gem::Dependency
35
38
  name: aws-sdk
36
39
  requirement: !ruby/object:Gem::Requirement
40
+ none: false
37
41
  requirements:
38
42
  - - ~>
39
43
  - !ruby/object:Gem::Version
@@ -44,6 +48,7 @@ dependencies:
44
48
  type: :runtime
45
49
  prerelease: false
46
50
  version_requirements: !ruby/object:Gem::Requirement
51
+ none: false
47
52
  requirements:
48
53
  - - ~>
49
54
  - !ruby/object:Gem::Version
@@ -54,6 +59,7 @@ dependencies:
54
59
  - !ruby/object:Gem::Dependency
55
60
  name: dogstatsd-ruby
56
61
  requirement: !ruby/object:Gem::Requirement
62
+ none: false
57
63
  requirements:
58
64
  - - ~>
59
65
  - !ruby/object:Gem::Version
@@ -64,6 +70,7 @@ dependencies:
64
70
  type: :runtime
65
71
  prerelease: false
66
72
  version_requirements: !ruby/object:Gem::Requirement
73
+ none: false
67
74
  requirements:
68
75
  - - ~>
69
76
  - !ruby/object:Gem::Version
@@ -74,6 +81,7 @@ dependencies:
74
81
  - !ruby/object:Gem::Dependency
75
82
  name: diffy
76
83
  requirement: !ruby/object:Gem::Requirement
84
+ none: false
77
85
  requirements:
78
86
  - - ~>
79
87
  - !ruby/object:Gem::Version
@@ -84,6 +92,7 @@ dependencies:
84
92
  type: :runtime
85
93
  prerelease: false
86
94
  version_requirements: !ruby/object:Gem::Requirement
95
+ none: false
87
96
  requirements:
88
97
  - - ~>
89
98
  - !ruby/object:Gem::Version
@@ -94,6 +103,7 @@ dependencies:
94
103
  - !ruby/object:Gem::Dependency
95
104
  name: parallel
96
105
  requirement: !ruby/object:Gem::Requirement
106
+ none: false
97
107
  requirements:
98
108
  - - ~>
99
109
  - !ruby/object:Gem::Version
@@ -104,6 +114,7 @@ dependencies:
104
114
  type: :runtime
105
115
  prerelease: false
106
116
  version_requirements: !ruby/object:Gem::Requirement
117
+ none: false
107
118
  requirements:
108
119
  - - ~>
109
120
  - !ruby/object:Gem::Version
@@ -114,6 +125,7 @@ dependencies:
114
125
  - !ruby/object:Gem::Dependency
115
126
  name: nokogiri
116
127
  requirement: !ruby/object:Gem::Requirement
128
+ none: false
117
129
  requirements:
118
130
  - - <
119
131
  - !ruby/object:Gem::Version
@@ -121,6 +133,7 @@ dependencies:
121
133
  type: :runtime
122
134
  prerelease: false
123
135
  version_requirements: !ruby/object:Gem::Requirement
136
+ none: false
124
137
  requirements:
125
138
  - - <
126
139
  - !ruby/object:Gem::Version
@@ -128,6 +141,7 @@ dependencies:
128
141
  - !ruby/object:Gem::Dependency
129
142
  name: tzinfo
130
143
  requirement: !ruby/object:Gem::Requirement
144
+ none: false
131
145
  requirements:
132
146
  - - ~>
133
147
  - !ruby/object:Gem::Version
@@ -138,6 +152,7 @@ dependencies:
138
152
  type: :runtime
139
153
  prerelease: false
140
154
  version_requirements: !ruby/object:Gem::Requirement
155
+ none: false
141
156
  requirements:
142
157
  - - ~>
143
158
  - !ruby/object:Gem::Version
@@ -148,6 +163,7 @@ dependencies:
148
163
  - !ruby/object:Gem::Dependency
149
164
  name: rspec
150
165
  requirement: !ruby/object:Gem::Requirement
166
+ none: false
151
167
  requirements:
152
168
  - - ~>
153
169
  - !ruby/object:Gem::Version
@@ -155,6 +171,7 @@ dependencies:
155
171
  type: :development
156
172
  prerelease: false
157
173
  version_requirements: !ruby/object:Gem::Requirement
174
+ none: false
158
175
  requirements:
159
176
  - - ~>
160
177
  - !ruby/object:Gem::Version
@@ -162,6 +179,7 @@ dependencies:
162
179
  - !ruby/object:Gem::Dependency
163
180
  name: pry
164
181
  requirement: !ruby/object:Gem::Requirement
182
+ none: false
165
183
  requirements:
166
184
  - - ~>
167
185
  - !ruby/object:Gem::Version
@@ -169,6 +187,7 @@ dependencies:
169
187
  type: :development
170
188
  prerelease: false
171
189
  version_requirements: !ruby/object:Gem::Requirement
190
+ none: false
172
191
  requirements:
173
192
  - - ~>
174
193
  - !ruby/object:Gem::Version
@@ -217,6 +236,7 @@ files:
217
236
  - spec/helpers/logging_helper.rb
218
237
  - spec/helpers/mock_alert.rb
219
238
  - spec/helpers/optica_helper.rb
239
+ - spec/lib/interferon/destinations/datadog_spec.rb
220
240
  - spec/lib/interferon/group_sources/filesystem_spec.rb
221
241
  - spec/lib/interferon/host_sources/optica_services_spec.rb
222
242
  - spec/lib/interferon/host_sources/optica_spec.rb
@@ -227,26 +247,27 @@ files:
227
247
  homepage: https://www.github.com/airbnb/interferon
228
248
  licenses:
229
249
  - MIT
230
- metadata: {}
231
250
  post_install_message:
232
251
  rdoc_options: []
233
252
  require_paths:
234
253
  - lib
235
254
  required_ruby_version: !ruby/object:Gem::Requirement
255
+ none: false
236
256
  requirements:
237
257
  - - ! '>='
238
258
  - !ruby/object:Gem::Version
239
259
  version: '0'
240
260
  required_rubygems_version: !ruby/object:Gem::Requirement
261
+ none: false
241
262
  requirements:
242
263
  - - ! '>='
243
264
  - !ruby/object:Gem::Version
244
265
  version: '0'
245
266
  requirements: []
246
267
  rubyforge_project:
247
- rubygems_version: 2.6.6
268
+ rubygems_version: 1.8.23.2
248
269
  signing_key:
249
- specification_version: 4
270
+ specification_version: 3
250
271
  summary: ': Store metrics alerts in code!'
251
272
  test_files:
252
273
  - spec/fixtures/loaders/host_sources/test_host_source.rb
@@ -260,6 +281,7 @@ test_files:
260
281
  - spec/helpers/logging_helper.rb
261
282
  - spec/helpers/mock_alert.rb
262
283
  - spec/helpers/optica_helper.rb
284
+ - spec/lib/interferon/destinations/datadog_spec.rb
263
285
  - spec/lib/interferon/group_sources/filesystem_spec.rb
264
286
  - spec/lib/interferon/host_sources/optica_services_spec.rb
265
287
  - spec/lib/interferon/host_sources/optica_spec.rb
checksums.yaml DELETED
@@ -1,15 +0,0 @@
1
- ---
2
- !binary "U0hBMQ==":
3
- metadata.gz: !binary |-
4
- MDdjYjBjNDk1OTJkMThiNjE5ZmZiNTgzNTI5NGU1ZDMzMzI5MGQ2ZQ==
5
- data.tar.gz: !binary |-
6
- NDIxMWM1MzQ2YTFlYjE4ZjIyOTUyMjk0ZDQ4ZjZhYTkzNWVlZDExZA==
7
- SHA512:
8
- metadata.gz: !binary |-
9
- YmU0MWExYTUxNDY3ODcwNWI2OWNlZTliOWVjNWVlYjMwNzkyMzdiNWIzM2Vm
10
- YzQ5NzY1MTQxNzkyMWIwYzJlM2I5MDc3ZTNlODFjOGVkZmU5ZmZmZjRmNGJm
11
- YzYyZTdmODhkMjdmNjJmODUwODY5ODcwNzQ4OTliYjA5NzQ0Mzg=
12
- data.tar.gz: !binary |-
13
- YzkxYjVkNmFhNzQwNjdiZDdmYjY4ZmQ2ODU0MmZiMzVmODQ5NzJkN2RkMTNh
14
- Nzc1YzY0ODA5NmU5ZDk3YzYxZmZjZTlhODg5YjdhMWU4OGNkNGM1Yzg2NTQx
15
- MWE5ODhmZWU5YWJkNzcxNTlkMGM2YmUxYWVlYTZiOWYyZTNlZWY=