interferon 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. checksums.yaml +15 -0
  2. data/.gitignore +11 -0
  3. data/Gemfile +2 -0
  4. data/Gemfile.lock +52 -0
  5. data/LICENSE +21 -0
  6. data/README.md +96 -0
  7. data/bin/interferon +66 -0
  8. data/config.example.yaml +37 -0
  9. data/groups/data.yaml +11 -0
  10. data/groups/dataeng.yaml +4 -0
  11. data/groups/datainfra.yaml +10 -0
  12. data/groups/devhap.yaml +6 -0
  13. data/groups/discover.yaml +13 -0
  14. data/groups/growth.yaml +17 -0
  15. data/groups/host.yaml +12 -0
  16. data/groups/internalproducts.yml +13 -0
  17. data/groups/logstash.yaml +4 -0
  18. data/groups/mobile.yaml +17 -0
  19. data/groups/pagerduty_sysops.yaml +5 -0
  20. data/groups/panda.yaml +10 -0
  21. data/groups/payments.yaml +16 -0
  22. data/groups/payments_finance.yaml +8 -0
  23. data/groups/prodinfra.yaml +15 -0
  24. data/groups/search.yaml +10 -0
  25. data/groups/security.yaml +8 -0
  26. data/groups/sre.yaml +16 -0
  27. data/groups/teamx.yaml +8 -0
  28. data/groups/tns.yaml +14 -0
  29. data/groups/tools.yml +11 -0
  30. data/interferon.gemspec +26 -0
  31. data/lib/interferon.rb +241 -0
  32. data/lib/interferon/alert.rb +33 -0
  33. data/lib/interferon/alert_dsl.rb +94 -0
  34. data/lib/interferon/destinations/datadog.rb +169 -0
  35. data/lib/interferon/group_sources/filesystem.rb +38 -0
  36. data/lib/interferon/host_sources/aws_dynamo.rb +51 -0
  37. data/lib/interferon/host_sources/aws_elasticache.rb +69 -0
  38. data/lib/interferon/host_sources/aws_rds.rb +92 -0
  39. data/lib/interferon/host_sources/optica.rb +35 -0
  40. data/lib/interferon/host_sources/optica_services.rb +68 -0
  41. data/lib/interferon/loaders.rb +123 -0
  42. data/lib/interferon/logging.rb +26 -0
  43. data/lib/interferon/version.rb +3 -0
  44. data/script/convert.rb +29 -0
  45. data/script/pre-commit +73 -0
  46. data/spec/spec_helper.rb +62 -0
  47. metadata +179 -0
@@ -0,0 +1,68 @@
1
+ require 'net/http'
2
+ require 'json'
3
+ require 'set'
4
+
5
+ module Interferon::HostSources
6
+ class OpticaServices
7
+ include Logging
8
+
9
+ def initialize(options)
10
+ raise ArgumentError, "missing host for optica source" \
11
+ unless options['host']
12
+
13
+ @host = options['host']
14
+ @port = options['port'] || 80
15
+ @envs = options['environments'] || []
16
+ end
17
+
18
+ def list_hosts
19
+ con = Net::HTTP.new(@host, @port)
20
+ con.read_timeout = 60
21
+ con.open_timeout = 60
22
+
23
+ response = con.get('/')
24
+ data = JSON::parse(response.body)
25
+
26
+ services = Hash.new{ |h,service| h[service] = {
27
+ :source => 'optica_services',
28
+ :service => service,
29
+
30
+ :owners => [],
31
+ :owner_groups => [],
32
+ :consumer_roles => Set.new,
33
+ :consumer_machine_count => 0,
34
+ :provider_machine_count => 0,
35
+ }}
36
+
37
+ data['nodes'].each do |ip, host|
38
+ next unless @envs.empty? or @envs.include?(host['environment'])
39
+
40
+ # make it easier by initializing possibly-missing data to sane defaults
41
+ host['ownership'] ||= {}
42
+ host['nerve_services'] ||= []
43
+ host['synapse_services'] ||= []
44
+
45
+ # provider info
46
+ host['nerve_services'].each do |service|
47
+ services[service][:provider_machine_count] += 1
48
+
49
+ services[service][:owners].concat(host['ownership']['people'] || [])
50
+ services[service][:owner_groups].concat(host['ownership']['groups'] || [])
51
+ end
52
+
53
+ # consumer info
54
+ host['synapse_services'].each do |service|
55
+ services[service][:consumer_roles].add(host['role'])
56
+ services[service][:consumer_machine_count] += 1
57
+ end
58
+ end
59
+
60
+ # convert all sets to arrays
61
+ services.each do |k,v|
62
+ services[k][:consumer_roles] = v[:consumer_roles].to_a
63
+ end
64
+
65
+ return services.values
66
+ end
67
+ end
68
+ end
@@ -0,0 +1,123 @@
1
+
2
+ module Interferon
3
+
4
+ # lets create namespaces for things we'll be loading
5
+ module Destinations; end
6
+ module HostSources; end
7
+ module GroupSources; end
8
+
9
+ # this is a type of class that can dynamically load other classes based on
10
+ # a 'type' string
11
+ class DynamicLoader
12
+ include Logging
13
+
14
+ def initialize(custom_paths)
15
+ @paths = []
16
+ custom_paths.each do |custom_path|
17
+ @paths << File.expand_path(custom_path)
18
+ end
19
+
20
+ initialize_attributes
21
+ end
22
+
23
+ # this should be overridden in specific loaders
24
+ def initialize_attributes
25
+ @loader_for = 'class'
26
+ @type_path = ''
27
+ @module = ::Interferon
28
+ end
29
+
30
+ def get_all(sources)
31
+ instances = []
32
+
33
+ sources.each_with_index do |source, idx|
34
+ type = source['type']
35
+ enabled = !!source['enabled']
36
+ options = source['options'] || {}
37
+
38
+ if type.nil?
39
+ log.warn "#{@loader_for} ##{idx} does not have a 'type' set; 'type' is required"
40
+ next
41
+ end
42
+
43
+ if !enabled
44
+ log.info "skipping #{@loader_for} #{type} because it's not enabled"
45
+ next
46
+ end
47
+
48
+ instance = get_klass(type).new(options)
49
+ instances << instance
50
+ end
51
+
52
+ instances
53
+ end
54
+
55
+ def get_klass(type)
56
+ # figure out what we're getting based on the type
57
+ filename = type.downcase
58
+ type_parts = filename.split('_')
59
+ class_name = type_parts.map(&:capitalize).join
60
+
61
+ # ideally, we'll put a constant into this variable
62
+ klass = nil
63
+
64
+ # first, try getting from custom paths
65
+ @paths.each do |path|
66
+ full_path = "#{path}/#{@type_path}/#{filename}"
67
+
68
+ begin
69
+ require full_path
70
+ klass = @module.const_get(class_name)
71
+ rescue LoadError => e
72
+ log.debug "LoadError looking for #{@loader_for} file #{type} at #{full_path}: #{e}"
73
+ rescue NameError => e
74
+ log.debug "NameError looking for #{@loader_for} class #{class_name} in #{full_path}: #{e}"
75
+ end
76
+
77
+ break if klass
78
+ end
79
+
80
+ # if that doesn't work, try getting from this repo via require_relative
81
+ if klass.nil?
82
+ begin
83
+ relative_filename = "./#{@type_path}/#{filename}"
84
+
85
+ require_relative relative_filename
86
+ klass = @module.const_get(class_name)
87
+ rescue LoadError => e
88
+ raise ArgumentError,\
89
+ "Loading Error; interferon does not define #{@loader_for} #{type}: #{e}"
90
+ rescue NameError => e
91
+ raise ArgumentError,\
92
+ "Name Error; class #{class_name} is not defined in #{relative_filename}: #{e}"
93
+ end
94
+ end
95
+
96
+ return klass
97
+ end
98
+ end
99
+
100
+ class DestinationsLoader < DynamicLoader
101
+ def initialize_attributes
102
+ @loader_for = 'destination'
103
+ @type_path = 'destinations'
104
+ @module = ::Interferon::Destinations
105
+ end
106
+ end
107
+
108
+ class HostSourcesLoader < DynamicLoader
109
+ def initialize_attributes
110
+ @loader_for = 'host source'
111
+ @type_path = 'host_sources'
112
+ @module = ::Interferon::HostSources
113
+ end
114
+ end
115
+
116
+ class GroupSourcesLoader < DynamicLoader
117
+ def initialize_attributes
118
+ @loader_for = 'group source'
119
+ @type_path = 'group_sources'
120
+ @module = ::Interferon::GroupSources
121
+ end
122
+ end
123
+ end
@@ -0,0 +1,26 @@
1
+ require 'logger'
2
+ require 'statsd'
3
+
4
+ module Interferon
5
+ module Logging
6
+
7
+ def statsd
8
+ @statsd ||= Statsd.new(
9
+ Statsd::DEFAULT_HOST,
10
+ Statsd::DEFAULT_PORT,
11
+ :namespace => 'alerts_framework'
12
+ )
13
+ end
14
+
15
+ def log
16
+ @logger ||= Logging.configure_logger_for(self.class.name)
17
+ end
18
+
19
+ def self.configure_logger_for(classname)
20
+ logger = Logger.new(STDERR)
21
+ logger.level = Logger::INFO unless ENV['DEBUG']
22
+ logger.progname = classname
23
+ return logger
24
+ end
25
+ end
26
+ end
@@ -0,0 +1,3 @@
1
+ module Interferon
2
+ VERSION = "0.0.1"
3
+ end
data/script/convert.rb ADDED
@@ -0,0 +1,29 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'json'
4
+ require 'yaml'
5
+
6
+ script_dir = File.dirname(File.expand_path __FILE__)
7
+ alerts_dir = File.join(script_dir, '..', 'alerts')
8
+
9
+ Dir.glob(File.join(alerts_dir, '*.json')) do |alert_file|
10
+ alert = JSON::parse(File.read(alert_file))
11
+
12
+ # convert old to new filters
13
+ alert['filter'] ||= {}
14
+ %w{role host}.each do |field|
15
+ %w{included excluded}.each do |type|
16
+ old_field = "#{type}_#{field}s"
17
+ if alert['filter'].include?(old_field)
18
+ alert['filter'][field] ||= {}
19
+ alert['filter'][field][type] = alert['filter'][old_field]
20
+ alert['filter'].delete(old_field)
21
+ end
22
+ end
23
+ end
24
+
25
+ # write out new filter file
26
+ new_name = alert_file.chomp(File.extname(alert_file)) + ".yaml"
27
+ File.write(new_name, YAML.dump(alert))
28
+ File.delete(alert_file)
29
+ end
data/script/pre-commit ADDED
@@ -0,0 +1,73 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'json'
4
+ require 'yaml'
5
+
6
+ # put the reasons why a commit is rejected into this array; if
7
+ # it is empty, the commit can go forward
8
+ reasons = []
9
+
10
+ # let's pick out which files were added/modified in this commit.
11
+ diff = `git diff-index --name-status --cached HEAD`
12
+ files = diff.split("\n").map(&:split)
13
+
14
+ added_files = files.
15
+ select { |(status, name)| status == 'A' }.
16
+ map { |(status, name)| name }.
17
+ compact
18
+
19
+ modified_files = files.
20
+ select {|(status, name)| status != 'D'}. # ignore deleted files
21
+ map {|(status, name)| name}.
22
+ compact
23
+
24
+ # check for large files
25
+ added_files.each do |file|
26
+ size_in_kb = `du -k #{file}`.strip.split.first.to_i
27
+ if size_in_kb > 1024
28
+ reasons << "#{file} is greater than 1 MB in size"
29
+ end
30
+ end
31
+
32
+ # Make sure Gemfile.lock was updated if Gemfile changed.
33
+ if modified_files.include?('Gemfile') && !modified_files.include?('Gemfile.lock')
34
+ reasons << 'You modified the Gemfile, but Gemfile.lock did not change!'
35
+ end
36
+
37
+ # Check Ruby syntax
38
+ modified_files.select {|f| f.match /\.((rb)|(rake))$/}.each do |file|
39
+ result = `ruby -c #{file} 2>&1`
40
+
41
+ if $? != 0
42
+ reasons << "ruby file #{file} failed syntax check"
43
+ end
44
+ end
45
+
46
+ # Check JSON syntax
47
+ modified_files.select {|f| f.match /(\.json)$/}.each do |file|
48
+ begin
49
+ JSON.parse(File.read(file))
50
+ rescue StandardError => e
51
+ reasons << "JSON file #{file} contains invalid JSON"
52
+ end
53
+ end
54
+
55
+ # Check YAML syntax
56
+ modified_files.select {|f| f.match /(\.yaml)$/}.each do |file|
57
+ begin
58
+ YAML.parse(File.read(file))
59
+ rescue YAML::SyntaxError => e
60
+ reasons << "YAML file #{file} has invalid syntax: #{e.inspect}"
61
+ rescue StandardError => e
62
+ reasons << "Could not read YAML file #{file}: #{e}"
63
+ end
64
+ end
65
+
66
+ unless reasons.empty?
67
+ puts 'Your commit failed the pre-commit hook! Some raisins:'
68
+ reasons.each do |r|
69
+ puts "\t* #{r}"
70
+ end
71
+ puts 'Did not commit.'
72
+ exit 1
73
+ end
@@ -0,0 +1,62 @@
1
+ # This file was generated by the `rspec --init` command.
2
+
3
+ # See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
4
+ RSpec.configure do |config|
5
+ config.expect_with :rspec do |expectations|
6
+ expectations.include_chain_clauses_in_custom_matcher_descriptions = true
7
+ end
8
+
9
+ # rspec-mocks config goes here. You can use an alternate test double
10
+ # library (such as bogus or mocha) by changing the `mock_with` option here.
11
+ config.mock_with :rspec do |mocks|
12
+ # Prevents you from mocking or stubbing a method that does not exist on
13
+ # a real object. This is generally recommended, and will default to
14
+ # `true` in RSpec 4.
15
+ mocks.verify_partial_doubles = true
16
+ end
17
+
18
+ # These two settings work together to allow you to limit a spec run
19
+ # to individual examples or groups you care about by tagging them with
20
+ # `:focus` metadata. When nothing is tagged with `:focus`, all examples
21
+ # get run.
22
+ config.filter_run :focus
23
+ config.run_all_when_everything_filtered = true
24
+
25
+ # Limits the available syntax to the non-monkey patched syntax that is
26
+ # recommended. For more details, see:
27
+ # - http://myronmars.to/n/dev-blog/2012/06/rspecs-new-expectation-syntax
28
+ # - http://teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/
29
+ # - http://myronmars.to/n/dev-blog/2014/05/notable-changes-in-rspec-3#new__config_option_to_disable_rspeccore_monkey_patching
30
+ config.disable_monkey_patching!
31
+
32
+ # This setting enables warnings. It's recommended, but in some cases may
33
+ # be too noisy due to issues in dependencies.
34
+ config.warnings = true
35
+
36
+ # Many RSpec users commonly either run the entire suite or an individual
37
+ # file, and it's useful to allow more verbose output when running an
38
+ # individual spec file.
39
+ if config.files_to_run.one?
40
+ # Use the documentation formatter for detailed output,
41
+ # unless a formatter has already been configured
42
+ # (e.g. via a command-line flag).
43
+ config.default_formatter = 'doc'
44
+ end
45
+
46
+ # Print the 10 slowest examples and example groups at the
47
+ # end of the spec run, to help surface which specs are running
48
+ # particularly slow.
49
+ # config.profile_examples = 10
50
+
51
+ # Run specs in random order to surface order dependencies. If you find an
52
+ # order dependency and want to debug it, you can fix the order by providing
53
+ # the seed, which is printed after each run.
54
+ # --seed 1234
55
+ config.order = :random
56
+
57
+ # Seed global randomization in this process using the `--seed` CLI option.
58
+ # Setting this allows you to use `--seed` to deterministically reproduce
59
+ # test failures related to randomization by passing the same `--seed` value
60
+ # as the one that triggered the failure.
61
+ Kernel.srand config.seed
62
+ end
metadata ADDED
@@ -0,0 +1,179 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: interferon
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ platform: ruby
6
+ authors:
7
+ - Igor Serebryany
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2015-03-16 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: dogapi
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ~>
18
+ - !ruby/object:Gem::Version
19
+ version: '1.11'
20
+ - - ! '>='
21
+ - !ruby/object:Gem::Version
22
+ version: 1.11.1
23
+ type: :runtime
24
+ prerelease: false
25
+ version_requirements: !ruby/object:Gem::Requirement
26
+ requirements:
27
+ - - ~>
28
+ - !ruby/object:Gem::Version
29
+ version: '1.11'
30
+ - - ! '>='
31
+ - !ruby/object:Gem::Version
32
+ version: 1.11.1
33
+ - !ruby/object:Gem::Dependency
34
+ name: aws-sdk
35
+ requirement: !ruby/object:Gem::Requirement
36
+ requirements:
37
+ - - ~>
38
+ - !ruby/object:Gem::Version
39
+ version: '1.35'
40
+ - - ! '>='
41
+ - !ruby/object:Gem::Version
42
+ version: 1.35.1
43
+ type: :runtime
44
+ prerelease: false
45
+ version_requirements: !ruby/object:Gem::Requirement
46
+ requirements:
47
+ - - ~>
48
+ - !ruby/object:Gem::Version
49
+ version: '1.35'
50
+ - - ! '>='
51
+ - !ruby/object:Gem::Version
52
+ version: 1.35.1
53
+ - !ruby/object:Gem::Dependency
54
+ name: dogstatsd-ruby
55
+ requirement: !ruby/object:Gem::Requirement
56
+ requirements:
57
+ - - ~>
58
+ - !ruby/object:Gem::Version
59
+ version: '1.4'
60
+ - - ! '>='
61
+ - !ruby/object:Gem::Version
62
+ version: 1.4.1
63
+ type: :runtime
64
+ prerelease: false
65
+ version_requirements: !ruby/object:Gem::Requirement
66
+ requirements:
67
+ - - ~>
68
+ - !ruby/object:Gem::Version
69
+ version: '1.4'
70
+ - - ! '>='
71
+ - !ruby/object:Gem::Version
72
+ version: 1.4.1
73
+ - !ruby/object:Gem::Dependency
74
+ name: rspec
75
+ requirement: !ruby/object:Gem::Requirement
76
+ requirements:
77
+ - - ~>
78
+ - !ruby/object:Gem::Version
79
+ version: '3.2'
80
+ type: :development
81
+ prerelease: false
82
+ version_requirements: !ruby/object:Gem::Requirement
83
+ requirements:
84
+ - - ~>
85
+ - !ruby/object:Gem::Version
86
+ version: '3.2'
87
+ - !ruby/object:Gem::Dependency
88
+ name: pry
89
+ requirement: !ruby/object:Gem::Requirement
90
+ requirements:
91
+ - - ~>
92
+ - !ruby/object:Gem::Version
93
+ version: '0.10'
94
+ type: :development
95
+ prerelease: false
96
+ version_requirements: !ruby/object:Gem::Requirement
97
+ requirements:
98
+ - - ~>
99
+ - !ruby/object:Gem::Version
100
+ version: '0.10'
101
+ description: ': Store metrics alerts in code!'
102
+ email:
103
+ - igor.serebryany@airbnb.com
104
+ executables:
105
+ - interferon
106
+ extensions: []
107
+ extra_rdoc_files: []
108
+ files:
109
+ - .gitignore
110
+ - Gemfile
111
+ - Gemfile.lock
112
+ - LICENSE
113
+ - README.md
114
+ - bin/interferon
115
+ - config.example.yaml
116
+ - groups/data.yaml
117
+ - groups/dataeng.yaml
118
+ - groups/datainfra.yaml
119
+ - groups/devhap.yaml
120
+ - groups/discover.yaml
121
+ - groups/growth.yaml
122
+ - groups/host.yaml
123
+ - groups/internalproducts.yml
124
+ - groups/logstash.yaml
125
+ - groups/mobile.yaml
126
+ - groups/pagerduty_sysops.yaml
127
+ - groups/panda.yaml
128
+ - groups/payments.yaml
129
+ - groups/payments_finance.yaml
130
+ - groups/prodinfra.yaml
131
+ - groups/search.yaml
132
+ - groups/security.yaml
133
+ - groups/sre.yaml
134
+ - groups/teamx.yaml
135
+ - groups/tns.yaml
136
+ - groups/tools.yml
137
+ - interferon.gemspec
138
+ - lib/interferon.rb
139
+ - lib/interferon/alert.rb
140
+ - lib/interferon/alert_dsl.rb
141
+ - lib/interferon/destinations/datadog.rb
142
+ - lib/interferon/group_sources/filesystem.rb
143
+ - lib/interferon/host_sources/aws_dynamo.rb
144
+ - lib/interferon/host_sources/aws_elasticache.rb
145
+ - lib/interferon/host_sources/aws_rds.rb
146
+ - lib/interferon/host_sources/optica.rb
147
+ - lib/interferon/host_sources/optica_services.rb
148
+ - lib/interferon/loaders.rb
149
+ - lib/interferon/logging.rb
150
+ - lib/interferon/version.rb
151
+ - script/convert.rb
152
+ - script/pre-commit
153
+ - spec/spec_helper.rb
154
+ homepage: https://www.github.com/airbnb/interferon
155
+ licenses:
156
+ - MIT
157
+ metadata: {}
158
+ post_install_message:
159
+ rdoc_options: []
160
+ require_paths:
161
+ - lib
162
+ required_ruby_version: !ruby/object:Gem::Requirement
163
+ requirements:
164
+ - - ! '>='
165
+ - !ruby/object:Gem::Version
166
+ version: '0'
167
+ required_rubygems_version: !ruby/object:Gem::Requirement
168
+ requirements:
169
+ - - ! '>='
170
+ - !ruby/object:Gem::Version
171
+ version: '0'
172
+ requirements: []
173
+ rubyforge_project:
174
+ rubygems_version: 2.4.6
175
+ signing_key:
176
+ specification_version: 4
177
+ summary: ': Store metrics alerts in code!'
178
+ test_files:
179
+ - spec/spec_helper.rb