interferon 0.0.1

data/groups/panda.yaml

@@ -0,0 +1,10 @@
+# PandA is Pricing and Availability team under marketplace.
+---
+name: panda
+people:
+  - fenglin.liao@airbnb.com
+  - li.zhang@airbnb.com
+  - spencer.demars@airbnb.com
+  - kim.pham@airbnb.com
+  - shao.liu@airbnb.com
+  - hector.yee@airbnb.com

data/groups/payments.yaml

@@ -0,0 +1,16 @@
+---
+name: payments
+people:
+  - allen.kerr@airbnb.com
+  - ian@airbnb.com
+  - john.terenzio@airbnb.com
+  - joseph.sofaer@airbnb.com
+  - josh.lee@airbnb.com
+  - juliusz.gonera@airbnb.com
+  - karen.kim@airbnb.com
+  - kevin.sun@airbnb.com
+  - lou.kosak@airbnb.com
+  - michel.weksler@airbnb.com
+  - mike.lewis@airbnb.com
+  - tao.cui@airbnb.com
+  - varun.pai@airbnb.com

data/groups/payments_finance.yaml

@@ -0,0 +1,8 @@
+---
+name: payments_finance
+people:
+  - alice.liang@airbnb.com
+  - allen.kerr@airbnb.com
+  - josh.lee@airbnb.com
+  - mike.lewis@airbnb.com
+  - tao.cui@airbnb.com

data/groups/prodinfra.yaml

@@ -0,0 +1,15 @@
+---
+name: prodinfra
+people:
+  - alexis.midon@airbnb.com
+  - ben.hughes@airbnb.com
+  - jon.tai@airbnb.com
+  - joseph.sofaer@airbnb.com
+  - kai.liu@airbnb.com
+  - kevin.rice@airbnb.com
+  - nija.mashruwala@airbnb.com
+  - philip.snowberger@airbnb.com
+  - rahul.iyer@airbnb.com
+  - sonic.wang@airbnb.com
+  - willie.yao@airbnb.com
+

data/groups/search.yaml

@@ -0,0 +1,10 @@
+---
+name: search
+people:
+  - maxim.charkov@airbnb.com
+  - fenglin.liao@airbnb.com
+  - li.zhang@airbnb.com
+  - mousom.gupta@airbnb.com
+  - spencer.demars@airbnb.com
+  - xiang.xiao@airbnb.com
+  - will.moss@airbnb.com

data/groups/security.yaml

@@ -0,0 +1,8 @@
+---
+name: security
+people:
+  - jeff@airbnb.com
+  - jennifer.rice@airbnb.com
+  - kevin.nguyen@airbnb.com
+  - philip.snowberger@airbnb.com
+  - samuel.zhu@airbnb.com

data/groups/sre.yaml

@@ -0,0 +1,16 @@
+---
+# deprecated; use prodinfra
+name: sre
+people:
+  - alexis.midon@airbnb.com
+  - ben.hughes@airbnb.com
+  - jon.tai@airbnb.com
+  - joseph.sofaer@airbnb.com
+  - kai.liu@airbnb.com
+  - kevin.rice@airbnb.com
+  - nija.mashruwala@airbnb.com
+  - philip.snowberger@airbnb.com
+  - rahul.iyer@airbnb.com
+  - sonic.wang@airbnb.com
+  - willie.yao@airbnb.com
+

data/groups/teamx.yaml

@@ -0,0 +1,8 @@
+---
+name: teamx
+people:
+  - lu.cheng@airbnb.com
+  - surabhi.gupta@airbnb.com
+  - naseem@airbnb.com
+  - phillippe.siclait@airbnb.com
+

data/groups/tns.yaml

@@ -0,0 +1,14 @@
+---
+name: tns
+people:
+  - alok.gupta@airbnb.com
+  - anish.muppalaneni@airbnb.com
+  - dmitry.alexeenko@airbnb.com
+  - eric.levine@airbnb.com
+  - freddy.chen@airbnb.com
+  - jerry.luan@airbnb.com
+  - justin.chen@airbnb.com
+  - sean.albito@airbnb.com
+  - siddharth.kar@airbnb.com
+  - tony.wen@airbnb.com
+  - lee.zhang@airbnb.com

data/groups/tools.yml

@@ -0,0 +1,11 @@
+---
+# deprecated; use internalproducts
+name: tools
+people:
+  - alvin.sng@airbnb.com
+  - emre.ozdemir@airbnb.com
+  - phil.busby@airbnb.com
+  - adrian.wisernig@airbnb.com
+  - bekki.jam@airbnb.com
+  - jujhaar.singh@airbnb.com
+  - jessica.toy@airbnb.com

data/interferon.gemspec

@@ -0,0 +1,26 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'interferon/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "interferon"
+  gem.version       = Interferon::VERSION
+  gem.authors       = ["Igor Serebryany"]
+  gem.email         = ["igor.serebryany@airbnb.com"]
+  gem.description   = %q{: Store metrics alerts in code!}
+  gem.summary       = %q{: Store metrics alerts in code!}
+  gem.homepage      = "https://www.github.com/airbnb/interferon"
+  gem.licenses      = ['MIT']
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+
+  gem.add_runtime_dependency "dogapi", "~> 1.11", ">= 1.11.1"
+  gem.add_runtime_dependency "aws-sdk", "~> 1.35", ">= 1.35.1"
+  gem.add_runtime_dependency "dogstatsd-ruby", "~> 1.4", ">= 1.4.1"
+
+  gem.add_development_dependency "rspec", "~> 3.2"
+  gem.add_development_dependency "pry", "~> 0.10"
+end

data/lib/interferon.rb

@@ -0,0 +1,241 @@
+require 'interferon/version'
+require 'interferon/logging'
+
+require 'interferon/loaders'
+
+require 'interferon/alert'
+require 'interferon/alert_dsl'
+
+#require 'pry'  #uncomment if you're debugging
+require 'erb'
+require 'ostruct'
+require 'set'
+require 'yaml'
+
+module Interferon
+  class Interferon
+
+    include Logging
+    attr_accessor :host_sources, :destinations, :host_info
+
+    # groups_sources is a hash from type => options for each group source
+    # host_sources is a hash from type => options for each host source
+    # destinations is a similiar hash from type => options for each alerter
+    def initialize(alerts_repo_path, groups_sources, host_sources, destinations)
+      @alerts_repo_path = alerts_repo_path
+      @groups_sources = groups_sources
+      @host_sources = host_sources
+      @destinations = destinations
+    end
+
+    def run(dry_run = false)
+      run_desc = dry_run ? 'dry run' : 'run'
+      log.info "beginning alerts #{run_desc}"
+
+      alerts = read_alerts
+      groups = read_groups(@groups_sources)
+      hosts = read_hosts(@host_sources)
+
+      # make sure destinations know if it's a dry run
+      @destinations.each do |dest|
+        dest['options'] ||= {}
+        dest['options']['dry_run'] = dry_run
+      end
+
+      update_alerts(@destinations, hosts, alerts, groups)
+
+      log.info "interferon #{run_desc} complete"
+    end
+
+    def read_alerts
+      alerts = []
+      failed = 0
+
+      # validate that alerts path exists
+      path = File.expand_path(File.join(@alerts_repo_path, 'alerts'))
+      abort("no such directory #{path} for reading alert files") \
+        unless Dir.exists?(path)
+
+      Dir.glob(File.join(path, '*.rb')) do |alert_file|
+        begin
+          alert = Alert.new(alert_file)
+        rescue StandardError => e
+          log.warn "error reading alert file #{alert_file}: #{e}"
+          failed += 1
+        else
+          alerts << alert
+        end
+      end
+
+      log.info "read #{alerts.count} alerts files from #{path}"
+
+      statsd.gauge('alerts.read.count', alerts.count)
+      statsd.gauge('alerts.read.failed', failed)
+
+      abort("failed to read #{failed} alerts") if failed > 0
+      return alerts
+    end
+
+    def read_groups(sources)
+      groups = {}
+      loader = GroupSourcesLoader.new([@alerts_repo_path])
+      loader.get_all(sources).each do |source|
+        source_groups = source.list_groups
+
+        # add all people to groups
+        people_count = 0
+        source_groups.each do |name, people|
+          groups[name] ||= []
+          groups[name].concat(people)
+          people_count += people.count
+        end
+
+        log.info "read #{people_count} people in #{source_groups.count} groups from source #{source.class.name}"
+      end
+
+      log.info "total of #{groups.values.flatten.count} people in #{groups.count} groups from #{sources.count} sources"
+
+      statsd.gauge('groups.sources', sources.count)
+      statsd.gauge('groups.count', groups.count)
+      statsd.gauge('groups.people', groups.values.flatten.count)
+
+      return groups
+    end
+
+    def read_hosts(sources)
+      statsd.gauge('hosts.sources', sources.count)
+
+      hosts = []
+      loader = HostSourcesLoader.new([@alerts_repo_path])
+      loader.get_all(sources).each do |source|
+        source_hosts = source.list_hosts
+        hosts << source_hosts
+
+        statsd.gauge('hosts.count', source_hosts.count, :tags => ["source:#{source.class.name}"])
+        log.info "read #{source_hosts.count} hosts from source #{source.class.name}"
+      end
+
+      hosts.flatten!
+      log.info "total of #{hosts.count} entities from #{sources.count} sources"
+
+      return hosts
+    end
+
+    def update_alerts(destinations, hosts, alerts, groups)
+      loader = DestinationsLoader.new([@alerts_repo_path])
+      loader.get_all(destinations).each do |dest|
+        log.info "updating alerts on #{dest.class.name}"
+
+        # track some counters/stats per destination
+        start_time = Time.new.to_f
+
+        # get already-defined alerts
+        existing_alerts = dest.existing_alerts.dup
+        existing_alerts.each{ |key, existing_alert| existing_alert['still_exists'] = false }
+
+        # create or update alerts; mark when we've done that
+        alerts_queue = Hash.new
+        alerts.each do |alert|
+          counters = {
+            :errors => 0,
+            :evals => 0,
+            :applies => 0,
+            :hosts => hosts.length
+          }
+          last_eval_error = nil
+
+          hosts.each do |hostinfo|
+            begin
+              alert.evaluate(hostinfo)
+              counters[:evals] += 1
+            rescue StandardError => e
+              log.debug "Evaluation of alert #{alert} failed in the context of host #{hostinfo}"
+              counters[:errors] += 1
+              last_eval_error = e
+              next
+            end
+
+            # don't define an alert that doesn't apply to this hostinfo
+            unless alert[:applies]
+              log.debug "alert #{alert[:name]} doesn't apply to #{hostinfo.inspect}"
+              next
+            end
+
+            counters[:applies] += 1
+
+            # don't define alerts twice
+            next if alerts_queue.key?(alert[:name])
+
+            # figure out who to notify
+            people = Set.new(alert[:notify][:people])
+            alert[:notify][:groups].each do |g|
+              people += (groups[g] || [])
+            end
+
+            # queue the alert up for creation; we clone the alert to save the current state
+            alerts_queue[alert[:name]] ||= [alert.clone, people]
+          end
+
+          # log some of the counters
+          statsd.gauge('alerts.evaluate.errors', counters[:errors], :tags => ["alert:#{alert}"])
+          statsd.gauge('alerts.evaluate.applies', counters[:applies], :tags => ["alert:#{alert}"])
+
+          if counters[:applies] > 0
+            log.info "alert #{alert} applies to #{counters[:applies]} of #{counters[:hosts]} hosts"
+          end
+
+          # did the alert fail to evaluate on all hosts?
+          if counters[:errors] == counters[:hosts]
+            log.error "alert #{alert} failed to evaluate in the context of all hosts!"
+            log.error "last error on alert #{alert}: #{last_eval_error}"
+
+            statsd.gauge('alerts.evaluate.failed_on_all', 1, :tags => ["alert:#{alert}"])
+            log.debug "alert #{alert}: error #{last_eval_error}\n#{last_eval_error.backtrace.join("\n")}"
+          else
+            statsd.gauge('alerts.evaluate.failed_on_all', 0, :tags => ["alert:#{alert}"])
+          end
+
+          # did the alert apply to any hosts?
+          if counters[:applies] == 0
+            statsd.gauge('alerts.evaluate.never_applies', 1, :tags => ["alert:#{alert}"])
+            log.warn "alert #{alert} did not apply to any hosts"
+          else
+            statsd.gauge('alerts.evaluate.never_applies', 0, :tags => ["alert:#{alert}"])
+          end
+        end
+
+        # flush queue
+        alerts_to_create = alerts_queue.keys
+        concurrency = dest.concurrency || 10
+        threads = concurrency.times.map do
+          t = Thread.new do
+            while name = alerts_to_create.shift
+              cur_alert, people = alerts_queue[name]
+
+              log.debug "creating alert for #{cur_alert[:name]}"
+              alert_key = dest.create_alert(cur_alert, people)
+
+              # don't delete alerts we still have defined
+              existing_alerts[alert_key]['still_exists'] = true if existing_alerts.include?(alert_key)
+            end
+          end
+          t.abort_on_exception = true
+          t
+        end
+        threads.map(&:join)
+
+        # remove existing alerts that shouldn't exist
+        to_delete = existing_alerts.reject{ |key, existing_alert| existing_alert['still_exists'] }
+        to_delete.each{ |key, alert| dest.remove_alert(alert) }
+
+        # run time summary
+        run_time = Time.new.to_f - start_time
+        statsd.histogram('destinations.run_time', run_time, :tags => ["destination:#{dest.class.name}"])
+        log.info "#{dest.class.name} : run completed in %.2f seconds" % (run_time)
+
+        # report destination stats
+        dest.report_stats
+      end
+    end
+  end
+end

data/lib/interferon/alert.rb

@@ -0,0 +1,33 @@
+module Interferon
+  class Alert
+    def initialize(path)
+      @path = path
+      @filename = File.basename(path)
+
+      @text = File.read(@path)
+
+      @dsl = nil
+    end
+
+    def to_s
+      @filename
+    end
+
+    def evaluate(hostinfo)
+      dsl = AlertDSL.new(hostinfo)
+      dsl.instance_eval(@text, @filename, 1)
+      @dsl = dsl
+
+      # return the alert and not the DSL object, which is private
+      self
+    end
+
+    def [](attr)
+      unless @dsl
+        raise "This alert has not yet been evaluated"
+      end
+
+      return @dsl.send(attr)
+    end
+  end
+end

data/lib/interferon/alert_dsl.rb

@@ -0,0 +1,94 @@
+
+module Interferon
+  module DSLMixin
+    def initialize(hostinfo)
+      @hostinfo = hostinfo
+    end
+
+    def method_missing(meth, *args, &block)
+      raise ArgumentError, "No such alerts field '#{meth}'"
+    end
+
+    def [](arg)
+      self.send(arg)
+    end
+
+    private
+    def get_or_set(field, val, block, default)
+      if val.nil? && block.nil?
+        f = instance_variable_get(field)
+        f.nil? ? default : f
+      elsif val.nil?
+        instance_variable_set(field, block.call)
+      elsif block.nil?
+        instance_variable_set(field, val)
+      else
+        raise ArgumentError, "You must pass either a value or a block but not both to #{field}"
+      end
+    end
+  end
+
+  class AlertDSL
+    include DSLMixin
+
+    def name(v = nil, &block)
+      get_or_set(:@name, v, block, '')
+    end
+
+    def message(v = nil, &block)
+      get_or_set(:@message, v, block, '')
+    end
+
+    def silenced(v = nil, &block)
+      get_or_set(:@silenced, v, block, false)
+    end
+
+    def silenced_until(v = nil, &block)
+      get_or_set(:@silenced_until, v && Time.parse(v), block, Time.at(0))
+    end
+
+    def notify_no_data(v = nil, &block)
+      get_or_set(:@notify_no_data, v, block, false)
+    end
+
+    def no_data_timeframe(v = nil, &block)
+      get_or_set(:@no_data_timeframe, v, block, false)
+    end
+
+    def timeout(v = nil, &block)
+      get_or_set(:@timeout, v, block, false)
+    end
+
+    def applies(v = nil, &block)
+      get_or_set(:@applies, v, block, false)
+    end
+
+    def notify(v = nil)
+      @notify ||= NotifyDSL.new(@hostinfo)
+    end
+
+    def metric(v = nil)
+      @metric ||= MetricDSL.new(@hostinfo)
+    end
+  end
+
+  class NotifyDSL
+    include DSLMixin
+
+    def people(v = nil, &block)
+      get_or_set(:@people, v, block, [])
+    end
+
+    def groups(v = nil, &block)
+      get_or_set(:@groups, v, block, [])
+    end
+  end
+
+  class MetricDSL
+    include DSLMixin
+
+    def datadog_query(v = nil, &block)
+      get_or_set(:@datadog_query, v, block, '')
+    end
+  end
+end