interferon 0.0.12 → 0.0.19

checksums.yaml

@@ -1,7 +1,15 @@
 ---
-SHA1:
-  metadata.gz: 35b5382f5aabd274b548257bcc5f227faf1b951e
-  data.tar.gz: abacd2609b40bf28b2783bbd21b24c384f991188
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    MDdjYjBjNDk1OTJkMThiNjE5ZmZiNTgzNTI5NGU1ZDMzMzI5MGQ2ZQ==
+  data.tar.gz: !binary |-
+    NDIxMWM1MzQ2YTFlYjE4ZjIyOTUyMjk0ZDQ4ZjZhYTkzNWVlZDExZA==
 SHA512:
-  metadata.gz: b215a4e355f66daa582d65b521fe0eef5842eb795551df1998844d82cb1892a64b7a95e3b60fff1ec2992eb2e88de2f584c274c9bd5570b6f0ae4026b05eeb5a
-  data.tar.gz: 2bb6ee4093fe2bbe46d811a3cb79a357066073727a699d773b306a5355fd8f7b9d06a4f2bdb51082ac6570a3166820ce7f1b88d63f9fb76d6d032fc8b97073b0
+  metadata.gz: !binary |-
+    YmU0MWExYTUxNDY3ODcwNWI2OWNlZTliOWVjNWVlYjMwNzkyMzdiNWIzM2Vm
+    YzQ5NzY1MTQxNzkyMWIwYzJlM2I5MDc3ZTNlODFjOGVkZmU5ZmZmZjRmNGJm
+    YzYyZTdmODhkMjdmNjJmODUwODY5ODcwNzQ4OTliYjA5NzQ0Mzg=
+  data.tar.gz: !binary |-
+    YzkxYjVkNmFhNzQwNjdiZDdmYjY4ZmQ2ODU0MmZiMzVmODQ5NzJkN2RkMTNh
+    Nzc1YzY0ODA5NmU5ZDk3YzYxZmZjZTlhODg5YjdhMWU4OGNkNGM1Yzg2NTQx
+    MWE5ODhmZWU5YWJkNzcxNTlkMGM2YmUxYWVlYTZiOWYyZTNlZWY=

data/interferon.gemspec

@@ -6,8 +6,8 @@ require 'interferon/version'
 Gem::Specification.new do |gem|
   gem.name          = "interferon"
   gem.version       = Interferon::VERSION
-  gem.authors       = ["Igor Serebryany"]
-  gem.email         = ["igor.serebryany@airbnb.com"]
+  gem.authors       = ["Igor Serebryany", "Jimmy Ngo"]
+  gem.email         = ["igor.serebryany@airbnb.com", "jimmy.ngo@airbnb.com"]
   gem.description   = %q{: Store metrics alerts in code!}
   gem.summary       = %q{: Store metrics alerts in code!}
   gem.homepage      = "https://www.github.com/airbnb/interferon"
@@ -20,6 +20,10 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency "dogapi", "~> 1.11", ">= 1.11.1"
   gem.add_runtime_dependency "aws-sdk", "~> 1.35", ">= 1.35.1"
   gem.add_runtime_dependency "dogstatsd-ruby", "~> 1.4", ">= 1.4.1"
+  gem.add_runtime_dependency "diffy", "~> 3.1.0", ">= 3.1.0"
+  gem.add_runtime_dependency "parallel", "~> 1.9", ">= 1.9.0"
+  gem.add_runtime_dependency "nokogiri", "< 1.7.0"
+  gem.add_runtime_dependency "tzinfo", "~> 1.2.2", ">= 1.2.2"
 
   gem.add_development_dependency "rspec", "~> 3.2"
   gem.add_development_dependency "pry", "~> 0.10"

data/lib/interferon/alert.rb

@@ -30,6 +30,14 @@ module Interferon
       @dsl.name(name)
     end
 
+    def silence
+      unless @dsl
+        raise "This alert has not yet been evaluated"
+      end
+
+      @dsl.silenced(true)
+    end
+
     def [](attr)
       unless @dsl
         raise "This alert has not yet been evaluated"

data/lib/interferon/alert_dsl.rb

@@ -1,3 +1,4 @@
+require 'interferon/work_hours_helper'
 
 module Interferon
   module DSLMixin
@@ -47,6 +48,15 @@ module Interferon
       get_or_set(:@silenced_until, v && Time.parse(v), block, Time.at(0))
     end
 
+    def is_work_hour?(args = {})
+      # Args can contain
+      # :hours => range of work hours (0 to 23h), for example (9..16)
+      # :days => range of week days (0 = sunday), for example (1..5) (Monday to Friday)
+      # :timezone => example 'America/Los_Angeles'
+      # 9 to 5 Monday to Friday in PST is the default
+      WorkHoursHelper.is_work_hour?(Time.now.utc, args)
+    end
+
     def notify_no_data(v = nil, &block)
       get_or_set(:@notify_no_data, v, block, false)
     end

data/lib/interferon/destinations/datadog.rb

@@ -1,6 +1,9 @@
+require 'diffy'
 require 'dogapi'
 require 'set'
 
+Diffy::Diff.default_format = :text
+
 module Interferon::Destinations
   class Datadog
     include ::Interferon::Logging
@@ -32,14 +35,18 @@ module Interferon::Destinations
       @dog = Dogapi::Client.new(*args)
 
       @existing_alerts = nil
+      @dry_run = options['dry_run']
 
       # create datadog alerts 10 at a time
       @concurrency = 10
 
       @stats = {
         :alerts_created => 0,
+        :alerts_to_be_created => 0,
         :alerts_updated => 0,
+        :alerts_to_be_updated => 0,
         :alerts_deleted => 0,
+        :alerts_to_be_deleted => 0,
         :alerts_silenced => 0,
         :api_successes => 0,
         :api_client_errors => 0,
@@ -52,6 +59,10 @@ module Interferon::Destinations
       @api_errors ||= []
     end
 
+    def generate_message(message, people)
+      [message, ALERT_KEY, people.map{ |p| "@#{p}" }].flatten.join("\n")
+    end
+
     def existing_alerts
       unless @existing_alerts
         resp = @dog.get_all_alerts()
@@ -64,7 +75,16 @@ module Interferon::Destinations
         alerts = resp[1]['alerts']
 
         # key alerts by name
-        @existing_alerts = Hash[alerts.map{ |a| [a['name'], a] }]
+        @existing_alerts = {}
+        alerts.each do |alert|
+          existing_alert = @existing_alerts[alert['name']]
+          if existing_alert.nil?
+            alert['id'] = [alert['id']]
+            @existing_alerts[alert['name']] = alert
+          else
+            existing_alert['id'] << alert['id']
+          end
+        end
 
         # count how many are manually created
         @stats[:manually_created_alerts] = \
@@ -81,21 +101,22 @@ module Interferon::Destinations
 
     def create_alert(alert, people)
       # create a message which includes the notifications
-      message = [
-        alert['message'],
-        ALERT_KEY,
-        people.map{ |p| "@#{p}" }
-      ].flatten.join("\n")
+      message = generate_message(alert['message'], people)
 
       # create the hash of options to send to datadog
       alert_opts = {
         :name => alert['name'],
         :message => message,
-        :silenced => alert['silenced'] || alert['silenced_until'] > Time.now,
+        :silenced => false,
         :notify_no_data => alert['notify_no_data'],
         :timeout_h => nil,
       }
 
+      # Set alert to be silenced if there is a silenced set or silenced_until set
+      if alert['silenced'] || alert['silenced_until'] > Time.now
+        alert_opts[:silenced] = true
+      end
+
       # allow an optional timeframe for "no data" alerts to be specified
       # (this feature is supported, even though it's not documented)
       alert_opts[:no_data_timeframe] = alert['no_data_timeframe'] if alert['no_data_timeframe']
@@ -103,10 +124,15 @@ module Interferon::Destinations
       # timeout is in seconds, but set it to 1 hour at least
       alert_opts[:timeout_h] = [1, (alert['timeout'].to_i / 3600)].max if alert['timeout']
 
+      datadog_query = alert['metric']['datadog_query'].strip
+      existing_alert = existing_alerts[alert['name']]
+
       # new alert, create it
-      if existing_alerts[alert['name']].nil?
+      if existing_alert.nil?
         action = :creating
-        log.debug("new alert #{alert['name']}")
+        @stats[:alerts_to_be_created] += 1
+        new_alert_text = "Query: #{datadog_query} Message: #{message.split().join(' ')}"
+        log.info("creating new alert #{alert['name']}: #{new_alert_text}")
 
         resp = @dog.alert(
           alert['metric']['datadog_query'].strip,
@@ -116,66 +142,67 @@ module Interferon::Destinations
       # existing alert, modify it
       else
         action = :updating
-        id = existing_alerts[alert['name']]['id']
-        log.debug("updating existing alert #{id} (#{alert['name']})")
-
-        resp = @dog.update_alert(
-          id,
-          alert['metric']['datadog_query'].strip,
-          alert_opts
-        )
+        @stats[:alerts_to_be_updated] += 1
+        id = existing_alert['id'][0]
+
+        new_alert_text = "Query:\n#{datadog_query}\nMessage:\n#{message}"
+        existing_alert_text = "Query:\n#{existing_alert['query']}\nMessage:\n#{existing_alert['message']}\n"
+        diff = Diffy::Diff.new(existing_alert_text, new_alert_text, :context=>1)
+        log.info("updating existing alert #{id} (#{alert['name']}): #{diff}")
+
+        if @dry_run
+          resp = @dog.alert(
+            alert['metric']['datadog_query'].strip,
+            alert_opts,
+          )
+        else
+          resp = @dog.update_alert(
+            id,
+            alert['metric']['datadog_query'].strip,
+            alert_opts
+          )
+          # Unmute existing alerts that have been unsilenced.
+          # Datadog does not allow updates to silencing via the update_alert API call.
+          if existing_alert['silenced'] && !alert_opts[:silenced]
+            @dog.unmute_monitor(id)
+          end
+        end
       end
 
       # log whenever we've encountered errors
       code = resp[0].to_i
-      if code != 200
-        api_errors << "#{code.to_s} on alert #{alert['name']}"
-      end
-
-      # client error
-      if code == 400
-        statsd.gauge('datadog.api.unknown_error', 0, :tags => ["alert:#{alert}"])
-        statsd.gauge('datadog.api.client_error', 1, :tags => ["alert:#{alert}"])
-        statsd.gauge('datadog.api.success', 0, :tags => ["alert:#{alert}"])
-
-        @stats[:api_client_errors] += 1
-        log.error("client error while #{action} alert '#{alert['name']}';" \
-            " query was '#{alert['metric']['datadog_query'].strip}'" \
-            " response was #{resp[0]}:'#{resp[1].inspect}'")
-
-      # unknown (prob. datadog) error:
-      elsif code >= 400 || code == -1
-        statsd.gauge('datadog.api.unknown_error', 1, :tags => ["alert:#{alert}"])
-        statsd.gauge('datadog.api.client_error', 0, :tags => ["alert:#{alert}"])
-        statsd.gauge('datadog.api.success', 0, :tags => ["alert:#{alert}"])
-
-        @stats[:api_unknown_errors] += 1
-        log.error("unknown error while #{action} alert '#{alert['name']}':" \
-            " query was '#{alert['metric']['datadog_query'].strip}'" \
-            " response was #{resp[0]}:'#{resp[1].inspect}'")
+      log_datadog_response_code(resp, code, action, alert)
 
       # assume this was a success
-      else
-        statsd.gauge('datadog.api.unknown_error', 0, :tags => ["alert:#{alert}"])
-        statsd.gauge('datadog.api.client_error', 0, :tags => ["alert:#{alert}"])
-        statsd.gauge('datadog.api.success', 1, :tags => ["alert:#{alert}"])
-
-        @stats[:api_successes] += 1
+      if !(code >= 400 || code == -1)
+        # assume this was a success
         @stats[:alerts_created] += 1 if action == :creating
         @stats[:alerts_updated] += 1 if action == :updating
         @stats[:alerts_silenced] += 1 if alert_opts[:silenced]
       end
 
-      id = resp[1].nil? ? nil : resp[1]['id']
+      id = resp[1].nil? ? nil : [resp[1]['id']]
       # lets key alerts by their name
       return [alert['name'], id]
     end
 
     def remove_alert(alert)
       if alert['message'].include?(ALERT_KEY)
-        log.debug("deleting alert #{alert['id']} (#{alert['name']})")
-        @dog.delete_alert(alert['id'])
-        @stats[:alerts_deleted] += 1
+        @stats[:alerts_to_be_deleted] += 1
+        log.info("deleting alert: #{alert['name']}")
+
+        if !@dry_run
+          alert['id'].each do |alert_id|
+            resp = @dog.delete_alert(alert_id)
+            code = resp[0].to_i
+            log_datadog_response_code(resp, code, :deleting)
+
+            if !(code >= 300 || code == -1)
+              # assume this was a success
+              @stats[:alerts_deleted] += 1
+            end
+          end
+        end
       else
         log.warn("not deleting manually-created alert #{alert['id']} (#{alert['name']})")
       end
@@ -186,17 +213,62 @@ module Interferon::Destinations
         statsd.gauge("datadog.#{k}", v)
       end
 
-      log.info "datadog: created %d updated %d and deleted %d alerts" % [
+      log.info "datadog: successfully created (%d/%d), updated (%d/%d), and deleted (%d/%d) alerts" % [
         @stats[:alerts_created],
+        @stats[:alerts_to_be_created],
         @stats[:alerts_updated],
+        @stats[:alerts_to_be_updated],
         @stats[:alerts_deleted],
+        @stats[:alerts_to_be_deleted],
       ]
     end
 
     def remove_alert_by_id(alert_id)
+      # This should only be used by dry-run to clean up created dry-run alerts
       log.debug("deleting alert, id: #{alert_id}")
-      @dog.delete_alert(alert_id)
-      @stats[:alerts_deleted] += 1
+      resp = @dog.delete_alert(alert_id)
+      code = resp[0].to_i
+      log_datadog_response_code(resp, code, :deleting)
+    end
+
+    def log_datadog_response_code(resp, code, action, alert=nil)
+      # log whenever we've encountered errors
+      if code != 200 && !alert.nil?
+        api_errors << "#{code.to_s} on alert #{alert['name']}"
+      end
+
+      # client error
+      if code == 400
+        @stats[:api_client_errors] += 1
+        if !alert.nil?
+          statsd.gauge('datadog.api.unknown_error', 0, :tags => ["alert:#{alert}"])
+          statsd.gauge('datadog.api.client_error', 1, :tags => ["alert:#{alert}"])
+          statsd.gauge('datadog.api.success', 0, :tags => ["alert:#{alert}"])
+          log.error("client error while #{action} alert '#{alert['name']}';" \
+                    " query was '#{alert['metric']['datadog_query'].strip}'" \
+                    " response was #{resp[0]}:'#{resp[1].inspect}'")
+        end
+
+        # unknown (prob. datadog) error:
+      elsif code > 400 || code == -1
+        @stats[:api_unknown_errors] += 1
+        if !alert.nil?
+          statsd.gauge('datadog.api.unknown_error', 1, :tags => ["alert:#{alert}"])
+          statsd.gauge('datadog.api.client_error', 0, :tags => ["alert:#{alert}"])
+          statsd.gauge('datadog.api.success', 0, :tags => ["alert:#{alert}"])
+          log.error("unknown error while #{action} alert '#{alert['name']}':" \
+                    " query was '#{alert['metric']['datadog_query'].strip}'" \
+                    " response was #{resp[0]}:'#{resp[1].inspect}'")
+        end
+      else
+        @stats[:api_successes] += 1
+        if !alert.nil?
+          statsd.gauge('datadog.api.unknown_error', 0, :tags => ["alert:#{alert}"])
+          statsd.gauge('datadog.api.client_error', 0, :tags => ["alert:#{alert}"])
+          statsd.gauge('datadog.api.success', 1, :tags => ["alert:#{alert}"])
+        end
+      end
     end
+
   end
 end

data/lib/interferon/group_sources/filesystem.rb

@@ -1,3 +1,4 @@
+include ::Interferon::Logging
 
 module Interferon::GroupSources
   class Filesystem
@@ -10,6 +11,7 @@ module Interferon::GroupSources
 
     def list_groups
       groups = {}
+      aliases = {}
 
       @paths.each do |path|
         path = File.expand_path(path)
@@ -18,7 +20,7 @@ module Interferon::GroupSources
           next
         end
 
-        Dir.glob(File.join(path, '*.{json,yml,yaml}')) do |group_file|
+        Dir.glob(File.join(path, '*.{json,yml,yaml}')).each do |group_file|
           begin
             group = YAML::parse(File.read(group_file))
           rescue YAML::SyntaxError => e
@@ -27,11 +29,25 @@ module Interferon::GroupSources
             log.warn "error reading group file #{group_file}: #{e}"
           else
             group = group.to_ruby
-            groups[group['name']] = group['people'] || []
+            if group['people']
+              groups[group['name']] = group['people'] || []
+            elsif group['alias_for']
+              aliases[group['name']] = {:group => group['alias_for'], :group_file => group_file}
+            end
           end
         end
       end
 
+      aliases.each do |aliased_group, group_info|
+        group = group_info[:group]
+        group_file = group_info[:group_file]
+        if groups.include?(group)
+          groups[aliased_group] = groups[group]
+        else
+          log.warn "Alias not found for #{group} but used by #{aliased_group} in #{group_file}"
+        end
+      end
+
       return groups
     end
   end

data/lib/interferon/version.rb

@@ -1,3 +1,3 @@
 module Interferon
-  VERSION = "0.0.12"
+  VERSION = "0.0.19"
 end

data/lib/interferon/work_hours_helper.rb

@@ -0,0 +1,21 @@
+require 'tzinfo'
+
+module Interferon
+  class WorkHoursHelper
+    DEFAULT_WORK_DAYS = (1..5)
+    DEFAULT_WORK_HOURS = (9..16)
+    DEFAULT_WORK_TIMEZONE = 'America/Los_Angeles'
+    DEFAULT_WORK_ARGS = {
+      :hours => DEFAULT_WORK_HOURS,
+      :days => DEFAULT_WORK_DAYS,
+      :timezone => DEFAULT_WORK_TIMEZONE,
+    }.freeze
+
+    def self.is_work_hour?(time, args = {})
+      args = args.merge(DEFAULT_WORK_ARGS)
+      tz = TZInfo::Timezone.get args[:timezone]
+      time_in_tz = time + tz.period_for_utc(time).utc_offset
+      return args[:days].include?(time_in_tz.wday) && args[:hours].include?(time_in_tz.hour)
+    end
+  end
+end