interferon 0.0.12 → 0.0.19

data/lib/interferon.rb

@@ -9,6 +9,7 @@ require 'interferon/alert_dsl'
 #require 'pry'  #uncomment if you're debugging
 require 'erb'
 require 'ostruct'
+require 'parallel'
 require 'set'
 require 'yaml'
 
@@ -23,11 +24,14 @@ module Interferon
     # groups_sources is a hash from type => options for each group source
     # host_sources is a hash from type => options for each host source
     # destinations is a similiar hash from type => options for each alerter
-    def initialize(alerts_repo_path, groups_sources, host_sources, destinations)
+    def initialize(alerts_repo_path, groups_sources, host_sources, destinations,
+                   dry_run=false, processes=nil)
       @alerts_repo_path = alerts_repo_path
       @groups_sources = groups_sources
       @host_sources = host_sources
       @destinations = destinations
+      @dry_run = dry_run
+      @processes = processes
       @request_shutdown = false
     end
 
@@ -36,7 +40,8 @@ module Interferon
         log.info "SIGTERM received. shutting down gracefully..."
         @request_shutdown = true
       end
-      run_desc = dry_run ? 'dry run' : 'run'
+      @dry_run = dry_run
+      run_desc = @dry_run ? 'dry run' : 'run'
       log.info "beginning alerts #{run_desc}"
 
       alerts = read_alerts
@@ -45,9 +50,12 @@ module Interferon
 
       @destinations.each do |dest|
         dest['options'] ||= {}
+        if @dry_run
+          dest['options']['dry_run'] = true
+        end
       end
 
-      update_alerts(@destinations, hosts, alerts, groups, dry_run)
+      update_alerts(@destinations, hosts, alerts, groups)
 
       if @request_shutdown
         log.info "interferon #{run_desc} shut down by SIGTERM"
@@ -133,23 +141,23 @@ module Interferon
       return hosts
     end
 
-    def update_alerts(destinations, hosts, alerts, groups, dry_run)
+    def update_alerts(destinations, hosts, alerts, groups)
       loader = DestinationsLoader.new([@alerts_repo_path])
       loader.get_all(destinations).each do |dest|
         break if @request_shutdown
         log.info "updating alerts on #{dest.class.name}"
-        update_alerts_on_destination(dest, hosts, alerts, groups, dry_run)
+        update_alerts_on_destination(dest, hosts, alerts, groups)
       end
     end
 
-    def update_alerts_on_destination(dest, hosts, alerts, groups, dry_run)
+    def update_alerts_on_destination(dest, hosts, alerts, groups)
       # track some counters/stats per destination
       start_time = Time.new.to_f
 
       # get already-defined alerts
-      existing_alerts = dest.existing_alerts.dup
+      existing_alerts = dest.existing_alerts
 
-      if dry_run
+      if @dry_run
         do_dry_run_update(dest, hosts, alerts, existing_alerts, groups)
       else
         do_regular_update(dest, hosts, alerts, existing_alerts, groups)
@@ -159,7 +167,7 @@ module Interferon
         # run time summary
         run_time = Time.new.to_f - start_time
         statsd.histogram(
-          dry_run ? 'destinations.run_time.dry_run' : 'destinations.run_time',
+          @dry_run ? 'destinations.run_time.dry_run' : 'destinations.run_time',
           run_time,
           :tags => ["destination:#{dest.class.name}"])
         log.info "#{dest.class.name} : run completed in %.2f seconds" % (run_time)
@@ -168,46 +176,103 @@ module Interferon
         dest.report_stats
       end
 
-      if dry_run && !dest.api_errors.empty?
+      if @dry_run && !dest.api_errors.empty?
         raise dest.api_errors.to_s
       end
     end
 
     def do_dry_run_update(dest, hosts, alerts, existing_alerts, groups)
-      to_remove = existing_alerts.reject{|key, a| !key.start_with?(DRY_RUN_ALERTS_NAME_PREFIX)}
+      # Track these to clean up dry-run alerts from previous runs
+      existing_dry_run_alerts = []
+      existing_alerts.each do |name, alert|
+        if name.start_with?(DRY_RUN_ALERTS_NAME_PREFIX)
+          existing_dry_run_alerts << [alert['name'], [alert['id']]]
+          existing_alerts.delete(name)
+        end
+      end
+
       alerts_queue = build_alerts_queue(hosts, alerts, groups)
-      alerts_queue.reject!{|name, pair| !Interferon::need_dry_run(pair[0], existing_alerts)}
-      alerts_queue.each do |name, pair|
-        alert = pair[0]
-        alert.change_name(DRY_RUN_ALERTS_NAME_PREFIX + alert['name'])
+      updates_queue = alerts_queue.reject do |name, alert_people_pair|
+        !Interferon::need_update(dest, alert_people_pair, existing_alerts)
+      end
+
+      # Add dry-run prefix to alerts and delete id to avoid impacting real alerts
+      existing_alerts.keys.each do |name|
+        existing_alert = existing_alerts[name]
+        dry_run_alert_name = DRY_RUN_ALERTS_NAME_PREFIX + name
+        existing_alert['name'] = dry_run_alert_name
+        existing_alert['id'] = [nil]
+        existing_alerts[dry_run_alert_name] = existing_alerts.delete(name)
+      end
+
+      # Build new queue with dry-run prefixes and ensure they are silenced
+      alerts_queue.each do |name, alert_people_pair|
+        alert = alert_people_pair[0]
+        dry_run_alert_name = DRY_RUN_ALERTS_NAME_PREFIX + alert['name']
+        alert.change_name(dry_run_alert_name)
+        alert.silence
+      end
+
+      # Create alerts in destination
+      created_alerts = create_alerts(dest, updates_queue)
+
+      # Existing alerts are pruned until all that remains are alerts that aren't being generated anymore
+      to_remove = existing_alerts.dup
+      alerts_queue.each do |name, alert_people_pair|
+        alert = alert_people_pair[0]
+        old_alerts = to_remove[alert['name']]
+
+        if !old_alerts.nil?
+          if old_alerts['id'].length == 1
+            to_remove.delete(alert['name'])
+          else
+            old_alerts['id'] = old_alerts['id'].drop(1)
+          end
+        end
       end
 
-      # flush queue
-      created_alerts_key_ids = create_alerts(dest, alerts_queue)
-      created_alerts_ids = created_alerts_key_ids.map{|a| a[1]}
-      to_remove_ids = to_remove.empty? ? [] : to_remove.map{|a| a['id']}
-      # remove existing alerts that shouldn't exist
-      (created_alerts_ids + to_remove_ids).each do |id|
+      # Clean up alerts not longer being generated
+      to_remove.each do |name, alert|
         break if @request_shutdown
-        dest.remove_alert_by_id(id) unless id.nil?
+        dest.remove_alert(alert)
+      end
+
+      # Clean up dry-run created alerts
+      (created_alerts + existing_dry_run_alerts).each do |alert_id_pair|
+        alert_ids = alert_id_pair[1]
+        alert_ids.each do |alert_id|
+          dest.remove_alert_by_id(alert_id)
+        end
       end
+
     end
 
     def do_regular_update(dest, hosts, alerts, existing_alerts, groups)
-      existing_alerts.each{ |key, existing_alert| existing_alert['still_exists'] = false }
-
       alerts_queue = build_alerts_queue(hosts, alerts, groups)
+      updates_queue = alerts_queue.reject do |name, alert_people_pair|
+        !Interferon::need_update(dest, alert_people_pair, existing_alerts)
+      end
 
-      # flush queue
-      created_alerts_keys = create_alerts(dest, alerts_queue).map{|a| a[0]}
-      created_alerts_keys.each do |alert_key|
-        # don't delete alerts we still have defined
-        existing_alerts[alert_key]['still_exists'] = true if existing_alerts.include?(alert_key)
+      # Create alerts in destination
+      create_alerts(dest, updates_queue)
+
+      # Existing alerts are pruned until all that remains are alerts that aren't being generated anymore
+      to_remove = existing_alerts.dup
+      alerts_queue.each do |name, alert_people_pair|
+        alert = alert_people_pair[0]
+        old_alerts = to_remove[alert['name']]
+
+        if !old_alerts.nil?
+          if old_alerts['id'].length == 1
+            to_remove.delete(alert['name'])
+          else
+            old_alerts['id'] = old_alerts['id'].drop(1)
+          end
+        end
       end
 
-      # remove existing alerts that shouldn't exist
-      to_delete = existing_alerts.reject{ |key, existing_alert| existing_alert['still_exists'] }
-      to_delete.each do |key, alert|
+      # Clean up alerts not longer being generated
+      to_remove.each do |name, alert|
         break if @request_shutdown
         dest.remove_alert(alert)
       end
@@ -237,10 +302,11 @@ module Interferon
     end
 
     def build_alerts_queue(hosts, alerts, groups)
+      alerts_queue = {}
       # create or update alerts; mark when we've done that
-      alerts_queue = Hash.new
-      alerts.each do |alert|
+      result = Parallel.map(alerts, in_processes: @processes) do |alert|
         break if @request_shutdown
+        alerts_generated = {}
         counters = {
           :errors => 0,
           :evals => 0,
@@ -268,7 +334,7 @@ module Interferon
 
           counters[:applies] += 1
           # don't define alerts twice
-          next if alerts_queue.key?(alert[:name])
+          next if alerts_generated.key?(alert[:name])
 
           # figure out who to notify
           people = Set.new(alert[:notify][:people])
@@ -277,7 +343,7 @@ module Interferon
           end
 
           # queue the alert up for creation; we clone the alert to save the current state
-          alerts_queue[alert[:name]] ||= [alert.clone, people]
+          alerts_generated[alert[:name]] = [alert.clone, people]
         end
 
         # log some of the counters
@@ -289,7 +355,7 @@ module Interferon
         end
 
         # did the alert fail to evaluate on all hosts?
-        if counters[:errors] == counters[:hosts]
+        if counters[:errors] == counters[:hosts] && !last_eval_error.nil?
           log.error "alert #{alert} failed to evaluate in the context of all hosts!"
           log.error "last error on alert #{alert}: #{last_eval_error}"
 
@@ -306,25 +372,48 @@ module Interferon
         else
           statsd.gauge('alerts.evaluate.never_applies', 0, :tags => ["alert:#{alert}"])
         end
+        alerts_generated
+      end
+
+      result.each do |alerts_generated|
+        alerts_queue.merge! alerts_generated
       end
       alerts_queue
     end
 
-    def self.need_dry_run(alert, existing_alerts_from_api)
+    def self.need_update(dest, alert_people_pair, existing_alerts_from_api)
+      alert = alert_people_pair[0]
       existing = existing_alerts_from_api[alert['name']]
       if existing.nil?
         true
       else
-        !same_alerts_for_dry_run_purpose(alert, existing)
+        !same_alerts(dest, alert_people_pair, existing)
       end
     end
 
-    def self.same_alerts_for_dry_run_purpose(alert, alert_api_json)
-      query1 = alert['metric']['datadog_query']
-      query2 = alert_api_json['query']
-      query1.strip!
-      query2.strip!
-      query1 == query2
+    def self.same_alerts(dest, alert_people_pair, alert_api_json)
+      alert, people = alert_people_pair
+
+      prev_alert = {
+        :query => alert_api_json['query'].strip,
+        :message => alert_api_json['message'].strip,
+        :notify_no_data => alert_api_json['notify_no_data'],
+        :silenced => alert_api_json['silenced'],
+        :timeout => alert_api_json['timeout_h'],
+        :no_data_timeframe => alert_api_json['no_data_timeframe']
+      }
+
+      new_alert = {
+        :query => alert['metric']['datadog_query'].strip,
+        :message => dest.generate_message(alert['message'], people).strip,
+        :notify_no_data => alert['notify_no_data'],
+        :silenced => alert['silenced'] || alert['silenced_until'] > Time.now,
+        :timeout => alert['timeout'] ? [1, alert['timeout'].to_i / 3600].max : nil,
+        :no_data_timeframe => alert['no_data_timeframe'] || nil
+      }
+
+      prev_alert == new_alert
     end
+
   end
 end

data/spec/helpers/dsl_helper.rb

@@ -6,7 +6,8 @@ module Interferon
     def get_or_set(field, val, block, default)
       @hash ||= Hash.new
       if val.nil?
-        return @hash[field]
+        f = @hash[field]
+        f.nil? ? default : f
       else
         @hash[field] = val
       end
@@ -27,6 +28,14 @@ module Interferon
     def id(v = nil, &block)
       get_or_set(:@id, v, block, '')
     end
+
+    def silenced(v = nil, &block)
+      get_or_set(:@silenced, v, block, false)
+    end
+
+    def silenced_until(v = nil, &block)
+      get_or_set(:@silenced_until, v && Time.parse(v), block, Time.at(0))
+    end
   end
 
   class MockNotifyDSL < NotifyDSL

data/spec/helpers/mock_alert.rb

@@ -1,6 +1,7 @@
 module Interferon
   class MockAlert < Alert
     def initialize(dsl)
+      @filename = 'MOCKALERT'
       @dsl = dsl
     end
 

data/spec/lib/interferon/group_sources/filesystem_spec.rb

@@ -0,0 +1,57 @@
+require 'spec_helper'
+require 'interferon/group_sources/filesystem'
+
+describe Interferon::GroupSources::Filesystem do
+  let (:fs_loader) { Interferon::GroupSources::Filesystem.new({'paths' => ['/tmp']}) }
+
+  describe 'list_groups' do
+    context "with basic groups" do
+      before do
+        group_a = double()
+        expect(File).to receive(:read).with('group_a.yaml').and_return('group_a_text')
+        expect(Psych).to receive(:parse).and_return(group_a)
+        expect(group_a).to receive(:to_ruby).and_return({'name' => 'group_a',
+                                                         'people' => ['Alice', 'Bob']})
+
+        group_b = double()
+        expect(File).to receive(:read).with('group_b.yaml').and_return('group_b_text')
+        expect(Psych).to receive(:parse).and_return(group_b)
+        expect(group_b).to receive(:to_ruby).and_return({'name' => 'group_b',
+                                                         'people' => ['Carol', 'Dave']})
+      end
+
+      it 'loads groups defined by YAML' do
+        expect(Dir).to receive(:glob).and_return(['group_a.yaml', 'group_b.yaml'].each)
+
+        groups = fs_loader.list_groups()
+        expect(groups).to eq({'group_a' => ['Alice', 'Bob'], 'group_b' => ['Carol', 'Dave']})
+      end
+
+      it 'allows groups to be aliased in YAML' do
+        expect(Dir).to receive(:glob).and_return(['group_a.yaml', 'group_b.yaml', 'group_c.yaml'].each)
+        group_c = double()
+        expect(File).to receive(:read).with('group_c.yaml').and_return('group_c_text')
+        expect(Psych).to receive(:parse).and_return(group_c)
+        expect(group_c).to receive(:to_ruby).and_return({'name' => 'group_c', 'alias_for' => 'group_b'})
+
+        groups = fs_loader.list_groups()
+        expect(groups).to eq({'group_a' => ['Alice', 'Bob'],
+                              'group_b' => ['Carol', 'Dave'],
+                              'group_c' => ['Carol', 'Dave']})
+      end
+
+      it 'skips bad aliases in YAML' do
+        expect(Dir).to receive(:glob).and_return(['group_a.yaml', 'group_b.yaml', 'group_c.yaml'].each)
+        group_c = double()
+        expect(File).to receive(:read).with('group_c.yaml').and_return('group_c_text')
+        expect(Psych).to receive(:parse).and_return(group_c)
+        expect(group_c).to receive(:to_ruby).and_return({'name' => 'group_c', 'alias_for' => 'group_d'})
+
+        groups = fs_loader.list_groups()
+        expect(groups).to eq({'group_a' => ['Alice', 'Bob'],
+                              'group_b' => ['Carol', 'Dave']})
+      end
+    end
+  end
+
+end