interferon 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +15 -0
- data/.gitignore +11 -0
- data/Gemfile +2 -0
- data/Gemfile.lock +52 -0
- data/LICENSE +21 -0
- data/README.md +96 -0
- data/bin/interferon +66 -0
- data/config.example.yaml +37 -0
- data/groups/data.yaml +11 -0
- data/groups/dataeng.yaml +4 -0
- data/groups/datainfra.yaml +10 -0
- data/groups/devhap.yaml +6 -0
- data/groups/discover.yaml +13 -0
- data/groups/growth.yaml +17 -0
- data/groups/host.yaml +12 -0
- data/groups/internalproducts.yml +13 -0
- data/groups/logstash.yaml +4 -0
- data/groups/mobile.yaml +17 -0
- data/groups/pagerduty_sysops.yaml +5 -0
- data/groups/panda.yaml +10 -0
- data/groups/payments.yaml +16 -0
- data/groups/payments_finance.yaml +8 -0
- data/groups/prodinfra.yaml +15 -0
- data/groups/search.yaml +10 -0
- data/groups/security.yaml +8 -0
- data/groups/sre.yaml +16 -0
- data/groups/teamx.yaml +8 -0
- data/groups/tns.yaml +14 -0
- data/groups/tools.yml +11 -0
- data/interferon.gemspec +26 -0
- data/lib/interferon.rb +241 -0
- data/lib/interferon/alert.rb +33 -0
- data/lib/interferon/alert_dsl.rb +94 -0
- data/lib/interferon/destinations/datadog.rb +169 -0
- data/lib/interferon/group_sources/filesystem.rb +38 -0
- data/lib/interferon/host_sources/aws_dynamo.rb +51 -0
- data/lib/interferon/host_sources/aws_elasticache.rb +69 -0
- data/lib/interferon/host_sources/aws_rds.rb +92 -0
- data/lib/interferon/host_sources/optica.rb +35 -0
- data/lib/interferon/host_sources/optica_services.rb +68 -0
- data/lib/interferon/loaders.rb +123 -0
- data/lib/interferon/logging.rb +26 -0
- data/lib/interferon/version.rb +3 -0
- data/script/convert.rb +29 -0
- data/script/pre-commit +73 -0
- data/spec/spec_helper.rb +62 -0
- metadata +179 -0
checksums.yaml
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
---
|
2
|
+
!binary "U0hBMQ==":
|
3
|
+
metadata.gz: !binary |-
|
4
|
+
ZjQ3ZDE1ZGY2YjRkYTQ1NjU4NDhjNjlhNTc5NzgwMTQzNmQ5MTJmYg==
|
5
|
+
data.tar.gz: !binary |-
|
6
|
+
YzNiZjUwMGU0NjNjMGQwN2U3YTNkOGUxYTRlZjkwNDE5ODk0ZWZjMg==
|
7
|
+
SHA512:
|
8
|
+
metadata.gz: !binary |-
|
9
|
+
ZmFkMDI4YjQ3NjQxYTU3MGQ5ZTZhYzZiMjE0MjEwMTFlMTMzMGJhMWEwM2Iw
|
10
|
+
YzcxYjUwMDk3YTI4ZTI0MWY4NTg0MmYzOWYwODViNzRmNzk5MTgxNThmMjY3
|
11
|
+
N2FlOTE0YTQ2M2M1ZThhNTEzYzhkZGNlMWNiODlmYWI3ZGEzYzc=
|
12
|
+
data.tar.gz: !binary |-
|
13
|
+
OGU0NDM3ZjU1NmNlNTNiMTliZDcyN2RiZTM5MzBhY2MxZDliZTJiMzAyYjhk
|
14
|
+
N2E0ZjI2MWRiYmEyZDk2YzdiY2RlN2FiNDlhNzU1OThiYTdmNDM3YWUwZjFj
|
15
|
+
ZWMxN2VmYjYxOGVlODM4NDRlOGIzNzViZDEwMWM3ZTQyYjYzYjQ=
|
data/.gitignore
ADDED
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
@@ -0,0 +1,52 @@
|
|
1
|
+
PATH
|
2
|
+
remote: .
|
3
|
+
specs:
|
4
|
+
interferon (0.0.1)
|
5
|
+
aws-sdk (~> 1.35, >= 1.35.1)
|
6
|
+
dogapi (~> 1.11, >= 1.11.1)
|
7
|
+
dogstatsd-ruby (~> 1.4, >= 1.4.1)
|
8
|
+
|
9
|
+
GEM
|
10
|
+
remote: https://rubygems.org/
|
11
|
+
specs:
|
12
|
+
aws-sdk (1.63.0)
|
13
|
+
aws-sdk-v1 (= 1.63.0)
|
14
|
+
aws-sdk-v1 (1.63.0)
|
15
|
+
json (~> 1.4)
|
16
|
+
nokogiri (>= 1.4.4)
|
17
|
+
coderay (1.1.0)
|
18
|
+
diff-lcs (1.2.5)
|
19
|
+
dogapi (1.16.0)
|
20
|
+
json (>= 1.5.1)
|
21
|
+
dogstatsd-ruby (1.4.1)
|
22
|
+
json (1.8.2)
|
23
|
+
method_source (0.8.2)
|
24
|
+
mini_portile (0.6.2)
|
25
|
+
nokogiri (1.6.6.2)
|
26
|
+
mini_portile (~> 0.6.0)
|
27
|
+
pry (0.10.1)
|
28
|
+
coderay (~> 1.1.0)
|
29
|
+
method_source (~> 0.8.1)
|
30
|
+
slop (~> 3.4)
|
31
|
+
rspec (3.2.0)
|
32
|
+
rspec-core (~> 3.2.0)
|
33
|
+
rspec-expectations (~> 3.2.0)
|
34
|
+
rspec-mocks (~> 3.2.0)
|
35
|
+
rspec-core (3.2.1)
|
36
|
+
rspec-support (~> 3.2.0)
|
37
|
+
rspec-expectations (3.2.0)
|
38
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
39
|
+
rspec-support (~> 3.2.0)
|
40
|
+
rspec-mocks (3.2.1)
|
41
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
42
|
+
rspec-support (~> 3.2.0)
|
43
|
+
rspec-support (3.2.2)
|
44
|
+
slop (3.6.0)
|
45
|
+
|
46
|
+
PLATFORMS
|
47
|
+
ruby
|
48
|
+
|
49
|
+
DEPENDENCIES
|
50
|
+
interferon!
|
51
|
+
pry (~> 0.10)
|
52
|
+
rspec (~> 3.2)
|
data/LICENSE
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
The MIT License (MIT)
|
2
|
+
|
3
|
+
Copyright (c) 2015 Airbnb, Inc
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
7
|
+
in the Software without restriction, including without limitation the rights
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
10
|
+
furnished to do so, subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
13
|
+
all copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,96 @@
|
|
1
|
+
# Interferon #
|
2
|
+
|
3
|
+
This repo contains the interferon gem.
|
4
|
+
This gem enables you to store your alerts configuration in code.
|
5
|
+
You should create your own repository, with a `Gemfile` which imports the interferon gem.
|
6
|
+
For an example of such a repository, along with example configuration and alerts files, see https://www.github.com/airbnb/alerts_example
|
7
|
+
|
8
|
+
## Running This Gem ##
|
9
|
+
|
10
|
+
This gem provides a single executable, called `interferon`.
|
11
|
+
You are meant to invoke it like so:
|
12
|
+
|
13
|
+
```bash
|
14
|
+
$ bundle exec interferon --config /path/to/config_file
|
15
|
+
```
|
16
|
+
|
17
|
+
Additional options:
|
18
|
+
* `-h`, `--help` -- prints out usage information
|
19
|
+
* `-n`, `--dry-run` -- runs interferon without making any changes to alerting destinations
|
20
|
+
|
21
|
+
## Configuration File ##
|
22
|
+
|
23
|
+
The configuration file is written in YAML.
|
24
|
+
It accepts the following parameters:
|
25
|
+
* `verbose_logging` -- whether to print more output
|
26
|
+
* `alerts_repo_path` -- the location to your alerts repo, containing your interferon DSL files
|
27
|
+
* `group_sources` -- a list of sources which can return groups of people to alert
|
28
|
+
* `host_sources` -- a list of sources which can read inventory systems and return lists of hosts to monitor
|
29
|
+
* `destinations` -- a list of alerting providers, which can monitor metrics and dispatch alerts as specified in your alerts dsl files
|
30
|
+
|
31
|
+
For more information, see [config.example.yaml](blob/master/config.example.yaml) file in this repo.
|
32
|
+
|
33
|
+
## The Moving Parts ##
|
34
|
+
|
35
|
+
This repo knows about four kinds of objects:
|
36
|
+
|
37
|
+
* *host_sources*: these query various inventory systems and return lists of hosts or entities to alert on
|
38
|
+
* *destinations*: these are metric systems, which can watch metrics and alert engineers
|
39
|
+
* *groups*: these are groups of actual engineers who can be alerted in case of trouble
|
40
|
+
* *alerts*: these are ruby DSL files which specify when and how engineers and groups are alerted via the destination about hosts
|
41
|
+
|
42
|
+
### Host Sources ###
|
43
|
+
|
44
|
+
* optica: can read a list of AWS instances from [optica](https://www.github.com/airbnb/optica)
|
45
|
+
* optica_services: returns smartstack service information parsed from optica
|
46
|
+
* aws_rds: lists RDS instances
|
47
|
+
* aws_dynamo: lists dynamo-db tables
|
48
|
+
* aws_elasticache: lists elasticache nodes and clusters
|
49
|
+
|
50
|
+
### Destinations ###
|
51
|
+
|
52
|
+
#### Datadog ####
|
53
|
+
|
54
|
+
Datadog is our only alerting destination at the moment.
|
55
|
+
Datadog's alerting syntax rule are here: [http://docs.datadoghq.com/api/#alerts](http://docs.datadoghq.com/api/#alerts)
|
56
|
+
Here's a chart explaining the datadog metric syntax ([generated via asciiflow](http://www.asciiflow.com/#669823367132047287/1039453499)):
|
57
|
+
|
58
|
+
```
|
59
|
+
+---------+ alert condition +-------------------------------------------------+
|
60
|
+
| |
|
61
|
+
| +-----+ metric to alert on |
|
62
|
+
| | |
|
63
|
+
| | tags to slice the metric by +------+ |
|
64
|
+
| | | |
|
65
|
+
v v v v
|
66
|
+
|----------| |-------------------------||--------------------------| |---|
|
67
|
+
max(last_5m):avg:haproxy_count_by_status{role:<%= role %>,status:up} by {host} > 0
|
68
|
+
^ ^ ^ ^
|
69
|
+
| | | |
|
70
|
+
| | +----+------------------------------+ |
|
71
|
+
| | | math on the metric over all tags | |
|
72
|
+
| | |-----------------------------------| +------------------------------------+
|
73
|
+
| | | * max, min, avg, sum | |trigger a separate alert for each |
|
74
|
+
| + +-----------------------------------+ |different value of these tags the |
|
75
|
+
| +----+----------------------------------------------+ |entire `by {}` clause can be ommited|
|
76
|
+
| | the interval to look at; always starts with last_ | +------------------------------------+
|
77
|
+
| |---------------------------------------------------|
|
78
|
+
| | * 5m, 10m, 15m, 30m |
|
79
|
+
| | * 1h, 2h, 4h |
|
80
|
+
+ +---------------------------------------------------+
|
81
|
+
+-------------------------------------------------------------------------------------------------+
|
82
|
+
| metric condition, can be one of: |
|
83
|
+
|-------------------------------------------------------------------------------------------------|
|
84
|
+
| * max: the metric gets this high at least once during the interval |
|
85
|
+
| * avg: the metric is this on average during the interval |
|
86
|
+
| * min: the metric is this small at least once during the interval |
|
87
|
+
| * change: the metric changes this much between a value N minutes ago and now (raw difference). |
|
88
|
+
| * pct_change: the metric changes this much between a value N minutes ago and now (percentage). |
|
89
|
+
+-------------------------------------------------------------------------------------------------+
|
90
|
+
```
|
91
|
+
|
92
|
+
### Groups ###
|
93
|
+
|
94
|
+
Groups actually come from *group_sources*.
|
95
|
+
We only have a single group source right now, which reads groups in YAML files from the filesystem.
|
96
|
+
However, we would like to add additional group sources, such as LDAP-based ones.
|
data/bin/interferon
ADDED
@@ -0,0 +1,66 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'yaml'
|
4
|
+
require 'optparse'
|
5
|
+
require 'interferon'
|
6
|
+
|
7
|
+
options={}
|
8
|
+
|
9
|
+
# set command line options
|
10
|
+
optparse = OptionParser.new do |opts|
|
11
|
+
opts.banner = %{Usage: interferon --config /path/to/interferon/config}
|
12
|
+
|
13
|
+
opts.on('-c config','--config config', String, 'Path to interferon config') do |key,value|
|
14
|
+
options[:config] = key
|
15
|
+
end
|
16
|
+
|
17
|
+
opts.on('-n', '--dry-run', "Don\'t update alert destinations") do
|
18
|
+
options[:dry_run] = true
|
19
|
+
end
|
20
|
+
|
21
|
+
opts.on('-h', '--help', 'Display this screen') do
|
22
|
+
puts opts
|
23
|
+
exit
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
def parseconfig(filename)
|
28
|
+
begin
|
29
|
+
c = YAML::parse(File.read(filename))
|
30
|
+
rescue Errno::ENOENT => e
|
31
|
+
raise ArgumentError, "config file does not exist:\n#{e.inspect}"
|
32
|
+
rescue Errno::EACCES => e
|
33
|
+
raise ArgumentError, "could not open config file:\n#{e.inspect}"
|
34
|
+
rescue YAML::SyntaxError => e
|
35
|
+
raise "config file #{filename} contains invalid YAML:\n#{e.inspect}"
|
36
|
+
end
|
37
|
+
return c.to_ruby
|
38
|
+
end
|
39
|
+
|
40
|
+
# parse command line arguments
|
41
|
+
optparse.parse!
|
42
|
+
unless options[:config]
|
43
|
+
puts "Please specify a config file; try #{__FILE__} -h if you need help"
|
44
|
+
exit 1
|
45
|
+
end
|
46
|
+
|
47
|
+
# validate that required options are present
|
48
|
+
config = parseconfig(options[:config])
|
49
|
+
%w{alerts_repo_path host_sources destinations}.each do |req|
|
50
|
+
unless config.include?(req) && ! config[req].empty?
|
51
|
+
puts "config file has no #{req} defined; exiting"
|
52
|
+
exit 2
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
ENV['DEBUG'] = '1' if config['verbose_logging']
|
57
|
+
|
58
|
+
a = Interferon::Interferon.new(
|
59
|
+
config['alerts_repo_path'],
|
60
|
+
config['group_sources'] || {},
|
61
|
+
config['host_sources'],
|
62
|
+
config['destinations'])
|
63
|
+
|
64
|
+
a.run(options[:dry_run])
|
65
|
+
|
66
|
+
puts "interferon signaling complete!"
|
data/config.example.yaml
ADDED
@@ -0,0 +1,37 @@
|
|
1
|
+
---
|
2
|
+
verbose_logging: true
|
3
|
+
alerts_repo_path: "/Users/igor47/repos/alerts"
|
4
|
+
group_sources:
|
5
|
+
- type: "filesystem"
|
6
|
+
enabled: true
|
7
|
+
options:
|
8
|
+
paths: ["/Users/igor47/repos/alerts/groups"]
|
9
|
+
|
10
|
+
host_sources:
|
11
|
+
- type: "aws_rds"
|
12
|
+
enabled: false
|
13
|
+
options:
|
14
|
+
regions: []
|
15
|
+
access_key_id: "AKIAsomething"
|
16
|
+
secret_access_key: "dontcheckthisinaccidentally!"
|
17
|
+
|
18
|
+
- type: "aws_dynamo"
|
19
|
+
enabled: false
|
20
|
+
options:
|
21
|
+
regions: []
|
22
|
+
access_key_id: "AKIAsomething"
|
23
|
+
secret_access_key: "dontcheckthisinaccidentally!"
|
24
|
+
|
25
|
+
- type: "aws_elasticache"
|
26
|
+
enabled: false
|
27
|
+
options:
|
28
|
+
regions: []
|
29
|
+
access_key_id: "AKIAsomething"
|
30
|
+
secret_access_key: "dontcheckthisinaccidentally!"
|
31
|
+
|
32
|
+
destinations:
|
33
|
+
- type: "datadog"
|
34
|
+
enabled: true
|
35
|
+
options:
|
36
|
+
app_key: "fillmein"
|
37
|
+
api_key: "alsomissing"
|
data/groups/data.yaml
ADDED
@@ -0,0 +1,11 @@
|
|
1
|
+
---
|
2
|
+
# deprecated; use datainfra
|
3
|
+
name: data
|
4
|
+
people:
|
5
|
+
- andy.kram@airbnb.com
|
6
|
+
- brenden.matthews@airbnb.com
|
7
|
+
- james.mayfield@airbnb.com
|
8
|
+
- paul.yang@airbnb.com
|
9
|
+
- krishna.puttaswamy@airbnb.com
|
10
|
+
- swaroop.jagadish@airbnb.com
|
11
|
+
- wensheng.hua@airbnb.com
|
data/groups/dataeng.yaml
ADDED
data/groups/devhap.yaml
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
---
|
2
|
+
name: discover
|
3
|
+
people:
|
4
|
+
- frank.lin@airbnb.com
|
5
|
+
- lu.cheng@airbnb.com
|
6
|
+
- naseem@airbnb.com
|
7
|
+
- phillippe.siclait@airbnb.com
|
8
|
+
- surabhi.gupta@airbnb.com
|
9
|
+
- horace.ko@airbnb.com
|
10
|
+
- tao.xu@airbnb.com
|
11
|
+
- josh.perez@airbnb.com
|
12
|
+
- yanxin.shi@airbnb.com
|
13
|
+
- tao.tao@airbnb.com
|
data/groups/growth.yaml
ADDED
@@ -0,0 +1,17 @@
|
|
1
|
+
---
|
2
|
+
name: growth
|
3
|
+
people:
|
4
|
+
- tanya.breshears@airbnb.com
|
5
|
+
- amy.wibowo@airbnb.com
|
6
|
+
- jimmy.tang@airbnb.com
|
7
|
+
- mona@airbnb.com
|
8
|
+
- roman.fuchs@airbnb.com
|
9
|
+
- jason@airbnb.com
|
10
|
+
- jason.bosinoff@airbnb.com
|
11
|
+
- henry.cai@airbnb.com
|
12
|
+
- rahul.agrawal@airbnb.com
|
13
|
+
- song.xie@airbnb.com
|
14
|
+
- yanxin.shi@airbnb.com
|
15
|
+
- nathaniel.weinman@airbnb.com
|
16
|
+
- fengming.wang@airbnb.com
|
17
|
+
- winnie.wang@airbnb.com
|
data/groups/host.yaml
ADDED
@@ -0,0 +1,12 @@
|
|
1
|
+
---
|
2
|
+
name: host
|
3
|
+
people:
|
4
|
+
- alanna.scott@airbnb.com
|
5
|
+
- james.ostrowski@airbnb.com
|
6
|
+
- barbara.raitz@airbnb.com
|
7
|
+
- yat.choi@airbnb.com
|
8
|
+
- edmund.ye@airbnb.com
|
9
|
+
- spike@airbnb.com
|
10
|
+
- raph@airbnb.com
|
11
|
+
- jessica.ho@airbnb.com
|
12
|
+
- trunal.bhanse@airbnb.com
|
@@ -0,0 +1,13 @@
|
|
1
|
+
---
|
2
|
+
name: internalproducts
|
3
|
+
people:
|
4
|
+
- alvin.sng@airbnb.com
|
5
|
+
- emre.ozdemir@airbnb.com
|
6
|
+
- phil.busby@airbnb.com
|
7
|
+
- adrian.wisernig@airbnb.com
|
8
|
+
- bekki.jam@airbnb.com
|
9
|
+
- jujhaar.singh@airbnb.com
|
10
|
+
- jessica.toy@airbnb.com
|
11
|
+
- james.nichols@airbnb.com
|
12
|
+
- harry@airbnb.com
|
13
|
+
- davide.cerri@airbnb.com
|
data/groups/mobile.yaml
ADDED
@@ -0,0 +1,17 @@
|
|
1
|
+
---
|
2
|
+
name: mobile
|
3
|
+
people:
|
4
|
+
- arthur.pang@airbnb.com
|
5
|
+
- brandon.withrow@airbnb.com
|
6
|
+
- youssef.francis@airbnb.com
|
7
|
+
- zane.claes@airbnb.com
|
8
|
+
- ian.pappas@airbnb.com
|
9
|
+
- xiao.pan@airbnb.com
|
10
|
+
- alex.davis@airbnb.com
|
11
|
+
- nick.adams@airbnb.com
|
12
|
+
- carol.leung@airbnb.com
|
13
|
+
- geobio.boo@airbnb.com
|
14
|
+
- eric.petzel@airbnb.com
|
15
|
+
- christian.deonier@airbnb.com
|
16
|
+
- michael.potter@airbnb.com
|
17
|
+
- sean.abraham@airbnb.com
|