interferon 0.0.19 → 0.0.20

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (8) hide show
  1. data/README.md +2 -2
  2. data/lib/interferon/alert.rb +1 -0
  3. data/lib/interferon/destinations/datadog.rb +27 -9
  4. data/lib/interferon/version.rb +1 -1
  5. data/lib/interferon.rb +1 -1
  6. data/spec/lib/interferon/destinations/datadog_spec.rb +111 -0
  7. metadata +27 -5
  8. checksums.yaml +0 -15
data/README.md CHANGED
@@ -30,7 +30,7 @@ It accepts the following parameters:
30
30
  * `host_sources` -- a list of sources which can read inventory systems and return lists of hosts to monitor
31
31
  * `destinations` -- a list of alerting providers, which can monitor metrics and dispatch alerts as specified in your alerts dsl files
32
32
 
33
- For more information, see [config.example.yaml](blob/master/config.example.yaml) file in this repo.
33
+ For more information, see [config.example.yaml](config.example.yaml) file in this repo.
34
34
 
35
35
  ## The Moving Parts ##
36
36
 
@@ -74,7 +74,7 @@ Here's a chart explaining the datadog metric syntax ([generated via asciiflow](h
74
74
  | | |-----------------------------------| +------------------------------------+
75
75
  | | | * max, min, avg, sum | |trigger a separate alert for each |
76
76
  | + +-----------------------------------+ |different value of these tags the |
77
- | +----+----------------------------------------------+ |entire `by {}` clause can be ommited|
77
+ | +----+----------------------------------------------+ |entire `by {}` clause can be omitted|
78
78
  | | the interval to look at; always starts with last_ | +------------------------------------+
79
79
  | |---------------------------------------------------|
80
80
  | | * 5m, 10m, 15m, 30m |
data/lib/interferon/alert.rb CHANGED
@@ -16,6 +16,7 @@ module Interferon
16
16
  def evaluate(hostinfo)
17
17
  dsl = AlertDSL.new(hostinfo)
18
18
  dsl.instance_eval(@text, @filename, 1)
19
+ dsl.name(dsl.name.strip)
19
20
  @dsl = dsl
20
21
 
21
22
  # return the alert and not the DSL object, which is private
data/lib/interferon/destinations/datadog.rb CHANGED
@@ -38,7 +38,9 @@ module Interferon::Destinations
38
38
  @dry_run = options['dry_run']
39
39
 
40
40
  # create datadog alerts 10 at a time
41
- @concurrency = 10
41
+ @concurrency = options['concurrency'] || 10
42
+ # configure retries
43
+ @retries = options['retries'] || 3
42
44
 
43
45
  @stats = {
44
46
  :alerts_created => 0,
@@ -63,17 +65,27 @@ module Interferon::Destinations
63
65
  [message, ALERT_KEY, people.map{ |p| "@#{p}" }].flatten.join("\n")
64
66
  end
65
67
 
68
+ def get_existing_alerts
69
+ resp = @dog.get_all_alerts()
70
+
71
+ code = resp[0].to_i
72
+ if code != 200
73
+ raise "Failed to retrieve existing alerts from datadog. #{code}: #{resp[1].inspect}"
74
+ end
75
+ resp[1]['alerts']
76
+ end
77
+
66
78
  def existing_alerts
67
79
  unless @existing_alerts
68
- resp = @dog.get_all_alerts()
69
-
70
- code = resp[0].to_i
71
- if code != 200
72
- raise "Failed to retrieve existing alerts from datadog. #{code.to_s}: #{resp[1].inspect}"
80
+ retries = @retries
81
+ begin
82
+ alerts = get_existing_alerts
83
+ rescue
84
+ retries -= 1
85
+ retry if retries >= 0
86
+ raise
73
87
  end
74
88
 
75
- alerts = resp[1]['alerts']
76
-
77
89
  # key alerts by name
78
90
  @existing_alerts = {}
79
91
  alerts.each do |alert|
@@ -112,6 +124,12 @@ module Interferon::Destinations
112
124
  :timeout_h => nil,
113
125
  }
114
126
 
127
+ if @dry_run
128
+ # Datadog may have a race condition where alerts created in a bad state may be triggered
129
+ # during the dry-run creation process. Delete people from dry-run alerts to avoid this
130
+ alert_opts[:message] = generate_message(alert['message'], [])
131
+ end
132
+
115
133
  # Set alert to be silenced if there is a silenced set or silenced_until set
116
134
  if alert['silenced'] || alert['silenced_until'] > Time.now
117
135
  alert_opts[:silenced] = true
@@ -234,7 +252,7 @@ module Interferon::Destinations
234
252
  def log_datadog_response_code(resp, code, action, alert=nil)
235
253
  # log whenever we've encountered errors
236
254
  if code != 200 && !alert.nil?
237
- api_errors << "#{code.to_s} on alert #{alert['name']}"
255
+ api_errors << "#{code} on alert #{alert['name']}"
238
256
  end
239
257
 
240
258
  # client error
data/lib/interferon/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module Interferon
2
- VERSION = "0.0.19"
2
+ VERSION = "0.0.20"
3
3
  end
data/lib/interferon.rb CHANGED
@@ -23,7 +23,7 @@ module Interferon
23
23
 
24
24
  # groups_sources is a hash from type => options for each group source
25
25
  # host_sources is a hash from type => options for each host source
26
- # destinations is a similiar hash from type => options for each alerter
26
+ # destinations is a similar hash from type => options for each alerter
27
27
  def initialize(alerts_repo_path, groups_sources, host_sources, destinations,
28
28
  dry_run=false, processes=nil)
29
29
  @alerts_repo_path = alerts_repo_path
data/spec/lib/interferon/destinations/datadog_spec.rb ADDED
@@ -0,0 +1,111 @@
1
+ require 'spec_helper'
2
+ require 'interferon/destinations/datadog'
3
+
4
+ describe Interferon::Destinations::Datadog do
5
+ let(:retries) { 3 }
6
+ let(:datadog) {
7
+ Interferon::Destinations::Datadog.new({
8
+ 'api_key' => 'TEST_API_KEY',
9
+ 'app_key' => 'TEST_APP_KEY',
10
+ 'retries' => retries,
11
+ })
12
+ }
13
+ let(:datadog_dry_run) {
14
+ Interferon::Destinations::Datadog.new({
15
+ 'api_key' => 'TEST_API_KEY',
16
+ 'app_key' => 'TEST_APP_KEY',
17
+ 'retries' => retries,
18
+ 'dry_run' => true,
19
+ })
20
+ }
21
+ let(:mock_alert_id) { 123 }
22
+ let(:mock_alert) {
23
+ {
24
+ 'id' => [mock_alert_id],
25
+ 'name' => 'Test Alert',
26
+ 'message' => "Test Message",
27
+ 'metric' => { 'datadog_query' => 'avg:metric{*}' },
28
+ 'silenced' => false,
29
+ 'silenced_until' => Time.at(0),
30
+ }
31
+ }
32
+ let(:mock_people) { ['foo', 'bar', 'baz'] }
33
+
34
+ describe ".get_existing_alerts" do
35
+ it "calls dogapi get_all_alerts" do
36
+ expect_any_instance_of(Dogapi::Client).to receive(:get_all_alerts).and_return([200, ""])
37
+ datadog.get_existing_alerts
38
+ end
39
+ end
40
+
41
+ describe ".existing_alerts" do
42
+ it "retries dogapi get_all_alerts" do
43
+ return_vals = [[400, ""]] * (retries + 1)
44
+ expect_any_instance_of(Dogapi::Client).to receive(:get_all_alerts).and_return(*return_vals)
45
+ expect { datadog.existing_alerts }.to raise_error RuntimeError
46
+ end
47
+ end
48
+
49
+ describe ".create_alert" do
50
+ it "calls dogapi alert" do
51
+ expect_any_instance_of(Dogapi::Client).to receive(:alert).and_return([200, ""])
52
+ expect(datadog).to receive(:existing_alerts).and_return({})
53
+ datadog.create_alert(mock_alert, mock_people)
54
+ end
55
+
56
+ it "calls dogapi update_alert when alert name is found" do
57
+ expect_any_instance_of(Dogapi::Client).to receive(:update_alert).and_return([200, ""])
58
+ expect(datadog).to receive(:existing_alerts).and_return(
59
+ {
60
+ "Test Alert" => {
61
+ "id" => 567,
62
+ "name" => 'Test Alert',
63
+ }
64
+ }
65
+ )
66
+ datadog.create_alert(mock_alert, mock_people)
67
+ end
68
+
69
+ it "always calls alert in dry-run" do
70
+ expect_any_instance_of(Dogapi::Client).to receive(:alert).and_return([200, ""])
71
+ expect(datadog_dry_run).to receive(:existing_alerts).and_return(
72
+ {
73
+ "Test Alert" => {
74
+ "id" => 567,
75
+ "name" => 'Test Alert',
76
+ }
77
+ }
78
+ )
79
+ datadog_dry_run.create_alert(mock_alert, mock_people)
80
+ end
81
+ end
82
+
83
+ describe ".remove_alert" do
84
+ it "calls dogapi delete_alert with the correct alert id" do
85
+ mock_alert["message"] += Interferon::Destinations::Datadog::ALERT_KEY
86
+ expect_any_instance_of(Dogapi::Client).to receive(:delete_alert).
87
+ with(mock_alert_id).and_return([200, ""])
88
+ datadog.remove_alert(mock_alert)
89
+ end
90
+
91
+ it "does not call dogapi delete_alert in dry_run" do
92
+ mock_alert["message"] += Interferon::Destinations::Datadog::ALERT_KEY
93
+ expect_any_instance_of(Dogapi::Client).to_not receive(:delete_alert)
94
+ datadog_dry_run.remove_alert(mock_alert)
95
+ end
96
+
97
+ it "does not call dogapi delete_alert when ALERT_KEY is missing" do
98
+ expect_any_instance_of(Dogapi::Client).to_not receive(:delete_alert)
99
+ datadog.remove_alert(mock_alert)
100
+ end
101
+
102
+ end
103
+
104
+ describe ".remove_alert_by_id" do
105
+ it "calls dogapi delete_alert" do
106
+ expect_any_instance_of(Dogapi::Client).to receive(:delete_alert).
107
+ with(mock_alert_id).and_return([200, ""])
108
+ datadog.remove_alert_by_id(mock_alert_id)
109
+ end
110
+ end
111
+ end
metadata CHANGED
@@ -1,7 +1,8 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: interferon
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.19
4
+ version: 0.0.20
5
+ prerelease:
5
6
  platform: ruby
6
7
  authors:
7
8
  - Igor Serebryany
@@ -9,11 +10,12 @@ authors:
9
10
  autorequire:
10
11
  bindir: bin
11
12
  cert_chain: []
12
- date: 2017-01-05 00:00:00.000000000 Z
13
+ date: 2017-04-03 00:00:00.000000000 Z
13
14
  dependencies:
14
15
  - !ruby/object:Gem::Dependency
15
16
  name: dogapi
16
17
  requirement: !ruby/object:Gem::Requirement
18
+ none: false
17
19
  requirements:
18
20
  - - ~>
19
21
  - !ruby/object:Gem::Version
@@ -24,6 +26,7 @@ dependencies:
24
26
  type: :runtime
25
27
  prerelease: false
26
28
  version_requirements: !ruby/object:Gem::Requirement
29
+ none: false
27
30
  requirements:
28
31
  - - ~>
29
32
  - !ruby/object:Gem::Version
@@ -34,6 +37,7 @@ dependencies:
34
37
  - !ruby/object:Gem::Dependency
35
38
  name: aws-sdk
36
39
  requirement: !ruby/object:Gem::Requirement
40
+ none: false
37
41
  requirements:
38
42
  - - ~>
39
43
  - !ruby/object:Gem::Version
@@ -44,6 +48,7 @@ dependencies:
44
48
  type: :runtime
45
49
  prerelease: false
46
50
  version_requirements: !ruby/object:Gem::Requirement
51
+ none: false
47
52
  requirements:
48
53
  - - ~>
49
54
  - !ruby/object:Gem::Version
@@ -54,6 +59,7 @@ dependencies:
54
59
  - !ruby/object:Gem::Dependency
55
60
  name: dogstatsd-ruby
56
61
  requirement: !ruby/object:Gem::Requirement
62
+ none: false
57
63
  requirements:
58
64
  - - ~>
59
65
  - !ruby/object:Gem::Version
@@ -64,6 +70,7 @@ dependencies:
64
70
  type: :runtime
65
71
  prerelease: false
66
72
  version_requirements: !ruby/object:Gem::Requirement
73
+ none: false
67
74
  requirements:
68
75
  - - ~>
69
76
  - !ruby/object:Gem::Version
@@ -74,6 +81,7 @@ dependencies:
74
81
  - !ruby/object:Gem::Dependency
75
82
  name: diffy
76
83
  requirement: !ruby/object:Gem::Requirement
84
+ none: false
77
85
  requirements:
78
86
  - - ~>
79
87
  - !ruby/object:Gem::Version
@@ -84,6 +92,7 @@ dependencies:
84
92
  type: :runtime
85
93
  prerelease: false
86
94
  version_requirements: !ruby/object:Gem::Requirement
95
+ none: false
87
96
  requirements:
88
97
  - - ~>
89
98
  - !ruby/object:Gem::Version
@@ -94,6 +103,7 @@ dependencies:
94
103
  - !ruby/object:Gem::Dependency
95
104
  name: parallel
96
105
  requirement: !ruby/object:Gem::Requirement
106
+ none: false
97
107
  requirements:
98
108
  - - ~>
99
109
  - !ruby/object:Gem::Version
@@ -104,6 +114,7 @@ dependencies:
104
114
  type: :runtime
105
115
  prerelease: false
106
116
  version_requirements: !ruby/object:Gem::Requirement
117
+ none: false
107
118
  requirements:
108
119
  - - ~>
109
120
  - !ruby/object:Gem::Version
@@ -114,6 +125,7 @@ dependencies:
114
125
  - !ruby/object:Gem::Dependency
115
126
  name: nokogiri
116
127
  requirement: !ruby/object:Gem::Requirement
128
+ none: false
117
129
  requirements:
118
130
  - - <
119
131
  - !ruby/object:Gem::Version
@@ -121,6 +133,7 @@ dependencies:
121
133
  type: :runtime
122
134
  prerelease: false
123
135
  version_requirements: !ruby/object:Gem::Requirement
136
+ none: false
124
137
  requirements:
125
138
  - - <
126
139
  - !ruby/object:Gem::Version
@@ -128,6 +141,7 @@ dependencies:
128
141
  - !ruby/object:Gem::Dependency
129
142
  name: tzinfo
130
143
  requirement: !ruby/object:Gem::Requirement
144
+ none: false
131
145
  requirements:
132
146
  - - ~>
133
147
  - !ruby/object:Gem::Version
@@ -138,6 +152,7 @@ dependencies:
138
152
  type: :runtime
139
153
  prerelease: false
140
154
  version_requirements: !ruby/object:Gem::Requirement
155
+ none: false
141
156
  requirements:
142
157
  - - ~>
143
158
  - !ruby/object:Gem::Version
@@ -148,6 +163,7 @@ dependencies:
148
163
  - !ruby/object:Gem::Dependency
149
164
  name: rspec
150
165
  requirement: !ruby/object:Gem::Requirement
166
+ none: false
151
167
  requirements:
152
168
  - - ~>
153
169
  - !ruby/object:Gem::Version
@@ -155,6 +171,7 @@ dependencies:
155
171
  type: :development
156
172
  prerelease: false
157
173
  version_requirements: !ruby/object:Gem::Requirement
174
+ none: false
158
175
  requirements:
159
176
  - - ~>
160
177
  - !ruby/object:Gem::Version
@@ -162,6 +179,7 @@ dependencies:
162
179
  - !ruby/object:Gem::Dependency
163
180
  name: pry
164
181
  requirement: !ruby/object:Gem::Requirement
182
+ none: false
165
183
  requirements:
166
184
  - - ~>
167
185
  - !ruby/object:Gem::Version
@@ -169,6 +187,7 @@ dependencies:
169
187
  type: :development
170
188
  prerelease: false
171
189
  version_requirements: !ruby/object:Gem::Requirement
190
+ none: false
172
191
  requirements:
173
192
  - - ~>
174
193
  - !ruby/object:Gem::Version
@@ -217,6 +236,7 @@ files:
217
236
  - spec/helpers/logging_helper.rb
218
237
  - spec/helpers/mock_alert.rb
219
238
  - spec/helpers/optica_helper.rb
239
+ - spec/lib/interferon/destinations/datadog_spec.rb
220
240
  - spec/lib/interferon/group_sources/filesystem_spec.rb
221
241
  - spec/lib/interferon/host_sources/optica_services_spec.rb
222
242
  - spec/lib/interferon/host_sources/optica_spec.rb
@@ -227,26 +247,27 @@ files:
227
247
  homepage: https://www.github.com/airbnb/interferon
228
248
  licenses:
229
249
  - MIT
230
- metadata: {}
231
250
  post_install_message:
232
251
  rdoc_options: []
233
252
  require_paths:
234
253
  - lib
235
254
  required_ruby_version: !ruby/object:Gem::Requirement
255
+ none: false
236
256
  requirements:
237
257
  - - ! '>='
238
258
  - !ruby/object:Gem::Version
239
259
  version: '0'
240
260
  required_rubygems_version: !ruby/object:Gem::Requirement
261
+ none: false
241
262
  requirements:
242
263
  - - ! '>='
243
264
  - !ruby/object:Gem::Version
244
265
  version: '0'
245
266
  requirements: []
246
267
  rubyforge_project:
247
- rubygems_version: 2.6.6
268
+ rubygems_version: 1.8.23.2
248
269
  signing_key:
249
- specification_version: 4
270
+ specification_version: 3
250
271
  summary: ': Store metrics alerts in code!'
251
272
  test_files:
252
273
  - spec/fixtures/loaders/host_sources/test_host_source.rb
@@ -260,6 +281,7 @@ test_files:
260
281
  - spec/helpers/logging_helper.rb
261
282
  - spec/helpers/mock_alert.rb
262
283
  - spec/helpers/optica_helper.rb
284
+ - spec/lib/interferon/destinations/datadog_spec.rb
263
285
  - spec/lib/interferon/group_sources/filesystem_spec.rb
264
286
  - spec/lib/interferon/host_sources/optica_services_spec.rb
265
287
  - spec/lib/interferon/host_sources/optica_spec.rb
checksums.yaml DELETED
@@ -1,15 +0,0 @@
1
- ---
2
- !binary "U0hBMQ==":
3
- metadata.gz: !binary |-
4
- MDdjYjBjNDk1OTJkMThiNjE5ZmZiNTgzNTI5NGU1ZDMzMzI5MGQ2ZQ==
5
- data.tar.gz: !binary |-
6
- NDIxMWM1MzQ2YTFlYjE4ZjIyOTUyMjk0ZDQ4ZjZhYTkzNWVlZDExZA==
7
- SHA512:
8
- metadata.gz: !binary |-
9
- YmU0MWExYTUxNDY3ODcwNWI2OWNlZTliOWVjNWVlYjMwNzkyMzdiNWIzM2Vm
10
- YzQ5NzY1MTQxNzkyMWIwYzJlM2I5MDc3ZTNlODFjOGVkZmU5ZmZmZjRmNGJm
11
- YzYyZTdmODhkMjdmNjJmODUwODY5ODcwNzQ4OTliYjA5NzQ0Mzg=
12
- data.tar.gz: !binary |-
13
- YzkxYjVkNmFhNzQwNjdiZDdmYjY4ZmQ2ODU0MmZiMzVmODQ5NzJkN2RkMTNh
14
- Nzc1YzY0ODA5NmU5ZDk3YzYxZmZjZTlhODg5YjdhMWU4OGNkNGM1Yzg2NTQx
15
- MWE5ODhmZWU5YWJkNzcxNTlkMGM2YmUxYWVlYTZiOWYyZTNlZWY=