interferon 0.1.4 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1c7f74fc8e23da780fdf783cdb98387321d20513
-  data.tar.gz: 5aa81e287ab880c2d3e7af7bd96a035d41b3db21
+  metadata.gz: 1e06dab2517dc06edf86da60cb55249bf63d031d
+  data.tar.gz: e370782305dda94dca3215c57dfc8bceb633ed77
 SHA512:
-  metadata.gz: 82c25b9118d41092d6ab131dd7edba612f40e68d4959bd0849efb4d0d2483bd5056b806dfc1c6f7e21381badd83cfb1e4f4e62fba8497bb3e6c52c4f33479dd2
-  data.tar.gz: 24740188b929de430442f21501ef75de6a7fd148835c3491da1827c77f62b169c2ccca38507ee400f4c938d87ac273935ce63bb9c966c6da4851e6c49d0cc6c9
+  metadata.gz: bef7463ead0dc65ab0c2e8018feb13a85a419ecc02e2403e9ee15150cc29089d8d471609ae9c7ae421c0f6c112fd39f5746b9cff7288c43535ffb32cf4c88d2d
+  data.tar.gz: f8b07e1239591b50e07facca30186412c74eda5b19bd4d128b634bf1a8c477b449aaa7894ff6fc3181ab781f1011901778e37a663f6d337a56abc21dc9bd8462

data/README.md

@@ -29,6 +29,7 @@ It accepts the following parameters:
 * `group_sources` -- a list of sources which can return groups of people to alert
 * `host_sources` -- a list of sources which can read inventory systems and return lists of hosts to monitor
 * `destinations` -- a list of alerting providers, which can monitor metrics and dispatch alerts as specified in your alerts dsl files
+* `processes` -- number of processes to run the alert generation on (optional; default is to use all available cores)
 
 For more information, see [config.example.yaml](config.example.yaml) file in this repo.
 

data/bin/interferon

@@ -10,11 +10,11 @@ options = {}
 optparse = OptionParser.new do |opts|
   opts.banner = %(Usage: interferon --config /path/to/interferon/config)
 
-  opts.on('-c config', '--config config', String, 'Path to interferon config') do |key, _value|
+  opts.on('-c', '--config config', String, 'Path to interferon config') do |key|
     options[:config] = key
   end
 
-  opts.on('-n', '--dry-run', "Don\'t update alert destinations") do
+  opts.on('-n', '--dry-run', "Don't update alert destinations") do
     options[:dry_run] = true
   end
 
@@ -26,7 +26,7 @@ end
 
 def parseconfig(filename)
   begin
-    c = YAML.parse(File.read(filename))
+    config = YAML.parse(File.read(filename))
   rescue Errno::ENOENT => e
     raise ArgumentError, "config file does not exist:\n#{e.inspect}"
   rescue Errno::EACCES => e
@@ -34,7 +34,7 @@ def parseconfig(filename)
   rescue YAML::SyntaxError => e
     raise "config file #{filename} contains invalid YAML:\n#{e.inspect}"
   end
-  c.to_ruby
+  config.to_ruby
 end
 
 # parse command line arguments
@@ -55,13 +55,7 @@ end
 
 ENV['DEBUG'] = '1' if config['verbose_logging']
 
-a = Interferon::Interferon.new(
-  config['alerts_repo_path'],
-  config['group_sources'] || {},
-  config['host_sources'],
-  config['destinations']
-)
-
-a.run(options[:dry_run])
+interferon = Interferon::Interferon.new(config, options[:dry_run])
+interferon.run
 
 puts 'interferon signaling complete!'

data/lib/interferon.rb

@@ -23,28 +23,26 @@ module Interferon
     # groups_sources is a hash from type => options for each group source
     # host_sources is a hash from type => options for each host source
     # destinations is a similar hash from type => options for each alerter
-    def initialize(alerts_repo_path, groups_sources, host_sources, destinations,
-                   dry_run = false, processes = nil)
-      @alerts_repo_path = alerts_repo_path
-      @groups_sources = groups_sources
-      @host_sources = host_sources
-      @destinations = destinations
+    def initialize(config, dry_run = false)
+      @alerts_repo_path = config['alerts_repo_path']
+      @group_sources = config['group_sources'] || {}
+      @host_sources = config['host_sources']
+      @destinations = config['destinations']
+      @processes = config['processes']
       @dry_run = dry_run
-      @processes = processes
       @request_shutdown = false
     end
 
-    def run(dry_run = false)
+    def run
       Signal.trap('TERM') do
-        log.info 'SIGTERM received. shutting down gracefully...'
+        log.info('SIGTERM received. shutting down gracefully...')
         @request_shutdown = true
       end
-      @dry_run = dry_run
       run_desc = @dry_run ? 'dry run' : 'run'
-      log.info "beginning alerts #{run_desc}"
+      log.info("beginning alerts #{run_desc}")
 
       alerts = read_alerts
-      groups = read_groups(@groups_sources)
+      groups = read_groups(@group_sources)
       hosts = read_hosts(@host_sources)
 
       @destinations.each do |dest|
@@ -55,9 +53,9 @@ module Interferon
       update_alerts(@destinations, hosts, alerts, groups)
 
       if @request_shutdown
-        log.info "interferon #{run_desc} shut down by SIGTERM"
+        log.info("interferon #{run_desc} shut down by SIGTERM")
       else
-        log.info "interferon #{run_desc} complete"
+        log.info("interferon #{run_desc} complete")
       end
     end
 
@@ -75,14 +73,14 @@ module Interferon
         begin
           alert = Alert.new(alert_file)
         rescue StandardError => e
-          log.warn "error reading alert file #{alert_file}: #{e}"
+          log.warn("error reading alert file #{alert_file}: #{e}")
           failed += 1
         else
           alerts << alert
         end
       end
 
-      log.info "read #{alerts.count} alerts files from #{path}"
+      log.info("read #{alerts.count} alerts files from #{path}")
 
       statsd.gauge('alerts.read.count', alerts.count)
       statsd.gauge('alerts.read.failed', failed)
@@ -106,12 +104,16 @@ module Interferon
           people_count += people.count
         end
 
-        log.info "read #{people_count} people in #{source_groups.count} groups " \
-                 "from source #{source.class.name}"
+        log.info(
+          "read #{people_count} people in #{source_groups.count} groups " \
+          "from source #{source.class.name}"
+        )
       end
 
-      log.info "total of #{groups.values.flatten.count} people in #{groups.count} groups " \
-               "from #{sources.count} sources"
+      log.info(
+        "total of #{groups.values.flatten.count} people in #{groups.count} groups " \
+        "from #{sources.count} sources"
+      )
 
       statsd.gauge('groups.sources', sources.count)
       statsd.gauge('groups.count', groups.count)
@@ -131,36 +133,37 @@ module Interferon
         hosts << source_hosts
 
         statsd.gauge('hosts.count', source_hosts.count, tags: ["source:#{source.class.name}"])
-        log.info "read #{source_hosts.count} hosts from source #{source.class.name}"
+        log.info("read #{source_hosts.count} hosts from source #{source.class.name}")
       end
 
       hosts.flatten!
-      log.info "total of #{hosts.count} entities from #{sources.count} sources"
+      log.info("total of #{hosts.count} entities from #{sources.count} sources")
 
       hosts
     end
 
     def update_alerts(destinations, hosts, alerts, groups)
+      alerts_queue, alert_errors = build_alerts_queue(hosts, alerts, groups)
+      if @dry_run && !alert_errors.empty?
+        raise "Alerts failed to apply or evaluate for all hosts: #{alerts.map(&:to_s).join(', ')}"
+      end
+
       loader = DestinationsLoader.new([@alerts_repo_path])
       loader.get_all(destinations).each do |dest|
         break if @request_shutdown
-        log.info "updating alerts on #{dest.class.name}"
-        update_alerts_on_destination(dest, hosts, alerts, groups)
+        log.info("updating alerts on #{dest.class.name}")
+        update_alerts_on_destination(dest, alerts_queue)
       end
     end
 
-    def update_alerts_on_destination(dest, hosts, alerts, groups)
+    def update_alerts_on_destination(dest, alerts_queue)
       # track some counters/stats per destination
       start_time = Time.new.to_f
 
       # get already-defined alerts
       existing_alerts = dest.existing_alerts
 
-      if @dry_run
-        do_dry_run_update(dest, hosts, alerts, existing_alerts, groups)
-      else
-        do_regular_update(dest, hosts, alerts, existing_alerts, groups)
-      end
+      run_update(dest, alerts_queue, existing_alerts)
 
       unless @request_shutdown
         # run time summary
@@ -170,7 +173,7 @@ module Interferon
           run_time,
           tags: ["destination:#{dest.class.name}"]
         )
-        log.info "#{dest.class.name} : run completed in %.2f seconds" % run_time
+        log.info("#{dest.class.name} : run completed in %.2f seconds" % run_time)
 
         # report destination stats
         dest.report_stats
@@ -179,73 +182,7 @@ module Interferon
       raise dest.api_errors.to_s if @dry_run && !dest.api_errors.empty?
     end
 
-    def do_dry_run_update(dest, hosts, alerts, existing_alerts, groups)
-      # Track these to clean up dry-run alerts from previous runs
-      existing_dry_run_alerts = []
-      existing_alerts.each do |name, alert|
-        if name.start_with?(DRY_RUN_ALERTS_NAME_PREFIX)
-          existing_dry_run_alerts << [alert['name'], [alert['id']]]
-          existing_alerts.delete(name)
-        end
-      end
-
-      alerts_queue = build_alerts_queue(hosts, alerts, groups)
-      updates_queue = alerts_queue.reject do |_name, alert_people_pair|
-        !dest.need_update(alert_people_pair, existing_alerts)
-      end
-
-      # Add dry-run prefix to alerts and delete id to avoid impacting real alerts
-      existing_alerts.keys.each do |name|
-        existing_alert = existing_alerts[name]
-        dry_run_alert_name = DRY_RUN_ALERTS_NAME_PREFIX + name
-        existing_alert['name'] = dry_run_alert_name
-        existing_alert['id'] = [nil]
-        existing_alerts[dry_run_alert_name] = existing_alerts.delete(name)
-      end
-
-      # Build new queue with dry-run prefixes and ensure they are silenced
-      alerts_queue.each do |_name, alert_people_pair|
-        alert, _people = alert_people_pair
-        dry_run_alert_name = DRY_RUN_ALERTS_NAME_PREFIX + alert['name']
-        alert.change_name(dry_run_alert_name)
-        alert.silence
-      end
-
-      # Create alerts in destination
-      created_alerts = create_alerts(dest, updates_queue)
-
-      # Existing alerts are pruned until all that remains are
-      # alerts that aren't being generated anymore
-      to_remove = existing_alerts.dup
-      alerts_queue.each do |_name, alert_people_pair|
-        alert, _people = alert_people_pair
-        old_alerts = to_remove[alert['name']]
-
-        next if old_alerts.nil?
-        if old_alerts['id'].length == 1
-          to_remove.delete(alert['name'])
-        else
-          old_alerts['id'] = old_alerts['id'].drop(1)
-        end
-      end
-
-      # Clean up alerts not longer being generated
-      to_remove.each do |_name, alert|
-        break if @request_shutdown
-        dest.remove_alert(alert)
-      end
-
-      # Clean up dry-run created alerts
-      (created_alerts + existing_dry_run_alerts).each do |alert_id_pair|
-        alert_ids = alert_id_pair[1]
-        alert_ids.each do |alert_id|
-          dest.remove_alert_by_id(alert_id)
-        end
-      end
-    end
-
-    def do_regular_update(dest, hosts, alerts, existing_alerts, groups)
-      alerts_queue = build_alerts_queue(hosts, alerts, groups)
+    def run_update(dest, alerts_queue, existing_alerts)
       updates_queue = alerts_queue.reject do |_name, alert_people_pair|
         !dest.need_update(alert_people_pair, existing_alerts)
       end
@@ -253,6 +190,9 @@ module Interferon
       # Create alerts in destination
       create_alerts(dest, updates_queue)
 
+      # Do not continue to remove alerts during dry-run
+      return if @dry_run
+
       # Existing alerts are pruned until all that remains are
       # alerts that aren't being generated anymore
       to_remove = existing_alerts.dup
@@ -281,12 +221,12 @@ module Interferon
       concurrency = dest.concurrency || 10
       unless @request_shutdown
         threads = Array.new(concurrency) do |i|
-          log.info "thread #{i} created"
+          log.info("thread #{i} created")
           t = Thread.new do
             while (name = alerts_to_create.shift)
               break if @request_shutdown
               cur_alert, people = alerts_queue[name]
-              log.debug "creating alert for #{cur_alert[:name]}"
+              log.debug("creating alert for #{cur_alert[:name]}")
               alert_key_ids << dest.create_alert(cur_alert, people)
             end
           end
@@ -300,16 +240,20 @@ module Interferon
 
     def build_alerts_queue(hosts, alerts, groups)
       alerts_queue = {}
+      all_alert_generation_errors = []
+
       # create or update alerts; mark when we've done that
       result = Parallel.map(alerts, in_processes: @processes) do |alert|
         break if @request_shutdown
         alerts_generated = {}
+        alert_generation_errors = []
         counters = {
           errors: 0,
           evals: 0,
           applies: 0,
           hosts: hosts.length,
         }
+
         last_eval_error = nil
 
         hosts.each do |hostinfo|
@@ -317,7 +261,7 @@ module Interferon
             alert.evaluate(hostinfo)
             counters[:evals] += 1
           rescue StandardError => e
-            log.debug "Evaluation of alert #{alert} failed in the context of host #{hostinfo}"
+            log.debug("Evaluation of alert #{alert} failed in the context of host #{hostinfo}")
             counters[:errors] += 1
             last_eval_error = e
             next
@@ -325,7 +269,7 @@ module Interferon
 
           # don't define an alert that doesn't apply to this hostinfo
           unless alert[:applies]
-            log.debug "alert #{alert[:name]} doesn't apply to #{hostinfo.inspect}"
+            log.debug("alert #{alert[:name]} doesn't apply to #{hostinfo.inspect}")
             next
           end
 
@@ -348,17 +292,19 @@ module Interferon
         statsd.gauge('alerts.evaluate.applies', counters[:applies], tags: ["alert:#{alert}"])
 
         if counters[:applies] > 0
-          log.info "alert #{alert} applies to #{counters[:applies]} of #{counters[:hosts]} hosts"
+          log.info("alert #{alert} applies to #{counters[:applies]} of #{counters[:hosts]} hosts")
         end
 
         # did the alert fail to evaluate on all hosts?
         if counters[:errors] == counters[:hosts] && !last_eval_error.nil?
-          log.error "alert #{alert} failed to evaluate in the context of all hosts!"
-          log.error "last error on alert #{alert}: #{last_eval_error}"
-
+          log.error("alert #{alert} failed to evaluate in the context of all hosts!")
+          log.error("last error on alert #{alert}: #{last_eval_error}")
           statsd.gauge('alerts.evaluate.failed_on_all', 1, tags: ["alert:#{alert}"])
-          log.debug "alert #{alert}: " \
-                    "error #{last_eval_error}\n#{last_eval_error.backtrace.join("\n")}"
+          log.debug(
+            "alert #{alert}: " \
+            "error #{last_eval_error}\n#{last_eval_error.backtrace.join("\n")}"
+          )
+          alert_generation_errors << alert
         else
           statsd.gauge('alerts.evaluate.failed_on_all', 0, tags: ["alert:#{alert}"])
         end
@@ -366,17 +312,19 @@ module Interferon
         # did the alert apply to any hosts?
         if counters[:applies] == 0
           statsd.gauge('alerts.evaluate.never_applies', 1, tags: ["alert:#{alert}"])
-          log.warn "alert #{alert} did not apply to any hosts"
+          log.warn("alert #{alert} did not apply to any hosts")
+          alert_generation_errors << alert
         else
           statsd.gauge('alerts.evaluate.never_applies', 0, tags: ["alert:#{alert}"])
         end
-        alerts_generated
+        [alerts_generated, alert_generation_errors]
       end
 
-      result.each do |alerts_generated|
-        alerts_queue.merge! alerts_generated
+      result.each do |generated_alerts, alert_generation_errors|
+        alerts_queue.merge!(generated_alerts)
+        all_alert_generation_errors += alert_generation_errors
       end
-      alerts_queue
+      [alerts_queue, all_alert_generation_errors]
     end
   end
 end

data/lib/interferon/destinations/datadog.rb

@@ -119,10 +119,10 @@ module Interferon::Destinations
         @stats[:manually_created_alerts] = \
           @existing_alerts.reject { |_n, a| a['message'].include?(ALERT_KEY) }.length
 
-        log.info 'datadog: found %d existing alerts; %d were manually created' % [
-          @existing_alerts.length,
-          @stats[:manually_created_alerts],
-        ]
+        log.info(
+          "datadog: found #{@existing_alerts.length} existing alerts; " \
+          "#{@stats[:manually_created_alerts]} were manually created"
+        )
       end
 
       @existing_alerts
@@ -197,13 +197,25 @@ Options:
 EOM
       log.info("creating new alert #{alert['name']}: #{new_alert_text}")
 
-      @dog.monitor(
-        alert['monitor_type'],
-        datadog_query,
+      monitor_options = {
         name: alert['name'],
-        message: @dry_run ? self.class.generate_message(alert, []) : message,
-        options: alert_options
-      )
+        message: message,
+        options: alert_options,
+      }
+
+      if @dry_run
+        @dog.validate_monitor(
+          alert['monitor_type'],
+          datadog_query,
+          monitor_options
+        )
+      else
+        @dog.monitor(
+          alert['monitor_type'],
+          datadog_query,
+          monitor_options
+        )
+      end
     end
 
     def update_datadog_alert(alert, datadog_query, message, alert_options, existing_alert)
@@ -229,21 +241,23 @@ EOM
       diff = Diffy::Diff.new(existing_alert_text, new_alert_text, context: 1)
       log.info("updating existing alert #{id} (#{alert['name']}):\n#{diff}")
 
+      monitor_options = {
+        name: alert['name'],
+        message: message,
+        options: alert_options,
+      }
+
       if @dry_run
-        resp = @dog.monitor(
+        resp = @dog.validate_monitor(
           alert['monitor_type'],
           datadog_query,
-          name: alert['name'],
-          message: self.class.generate_message(alert, []),
-          options: alert_options
+          monitor_options
         )
       elsif self.class.same_monitor_type(alert['monitor_type'], existing_alert['type'])
         resp = @dog.update_monitor(
           id,
           datadog_query,
-          name: alert['name'],
-          message: message,
-          options: alert_options
+          monitor_options
         )
 
         # Unmute existing alerts that have been unsilenced.
@@ -259,9 +273,7 @@ EOM
           resp = @dog.monitor(
             alert['monitor_type'],
             datadog_query,
-            name: alert['name'],
-            message: message,
-            options: alert_options
+            monitor_options
           )
         end
       end
@@ -273,6 +285,7 @@ EOM
         @stats[:alerts_to_be_deleted] += 1
         log.info("deleting alert: #{alert['name']}")
 
+        # Safety to protect aginst accident dry_run deletion
         unless @dry_run
           alert['id'].each do |alert_id|
             resp = @dog.delete_monitor(alert_id)
@@ -290,14 +303,6 @@ EOM
       end
     end
 
-    def remove_alert_by_id(alert_id)
-      # This should only be used by dry-run to clean up created dry-run alerts
-      log.debug("deleting alert, id: #{alert_id}")
-      resp = @dog.delete_monitor(alert_id)
-      code = resp[0].to_i
-      log_datadog_response_code(resp, code, :deleting)
-    end
-
     def need_update(alert_people_pair, existing_alerts_from_api)
       alert, people = alert_people_pair
       existing = existing_alerts_from_api[alert['name']]
@@ -386,7 +391,7 @@ EOM
                     " response was #{resp[0]}:'#{resp[1].inspect}'")
         end
 
-        # unknown (prob. datadog) error:
+      # unknown (prob. datadog) error:
       elsif code > 400 || code == -1
         @stats[:api_unknown_errors] += 1
         unless alert.nil?

data/lib/interferon/group_sources/filesystem.rb

@@ -16,7 +16,7 @@ module Interferon::GroupSources
       @paths.each do |path|
         path = File.expand_path(path)
         unless Dir.exist?(path)
-          log.warn "no such directory #{path} for reading group files"
+          log.warn("no such directory #{path} for reading group files")
           next
         end
 
@@ -24,9 +24,9 @@ module Interferon::GroupSources
           begin
             group = YAML.parse(File.read(group_file))
           rescue YAML::SyntaxError => e
-            log.error "syntax error in group file #{group_file}: #{e}"
+            log.error("syntax error in group file #{group_file}: #{e}")
           rescue StandardError => e
-            log.warn "error reading group file #{group_file}: #{e}"
+            log.warn("error reading group file #{group_file}: #{e}")
           else
             group = group.to_ruby
             if group['people']
@@ -44,7 +44,7 @@ module Interferon::GroupSources
         if groups.include?(group)
           groups[aliased_group] = groups[group]
         else
-          log.warn "Alias not found for #{group} but used by #{aliased_group} in #{group_file}"
+          log.warn("Alias not found for #{group} but used by #{aliased_group} in #{group_file}")
         end
       end
 

data/lib/interferon/loaders.rb

@@ -35,12 +35,12 @@ module Interferon
         options =   source['options'] || {}
 
         if type.nil?
-          log.warn "#{@loader_for} ##{idx} does not have a 'type' set; 'type' is required"
+          log.warn("#{@loader_for} ##{idx} does not have a 'type' set; 'type' is required")
           next
         end
 
         unless enabled
-          log.info "skipping #{@loader_for} #{type} because it's not enabled"
+          log.info("skipping #{@loader_for} #{type} because it's not enabled")
           next
         end
 
@@ -68,9 +68,11 @@ module Interferon
           require full_path
           klass = @module.const_get(class_name)
         rescue LoadError => e
-          log.debug "LoadError looking for #{@loader_for} file #{type} at #{full_path}: #{e}"
+          log.debug("LoadError looking for #{@loader_for} file #{type} at #{full_path}: #{e}")
         rescue NameError => e
-          log.debug "NameError looking for #{@loader_for} class #{class_name} in #{full_path}: #{e}"
+          log.debug(
+            "NameError looking for #{@loader_for} class #{class_name}  in #{full_path}: #{e}"
+          )
         end
 
         break if klass

data/lib/interferon/version.rb

@@ -1,3 +1,3 @@
 module Interferon
-  VERSION = '0.1.4'.freeze
+  VERSION = '0.2.0'.freeze
 end

data/spec/lib/interferon/destinations/datadog_spec.rb

@@ -88,8 +88,8 @@ describe Interferon::Destinations::Datadog do
       datadog.create_alert(mock_alert, mock_people)
     end
 
-    it 'always calls monitor in dry-run' do
-      expect_any_instance_of(Dogapi::Client).to receive(:monitor).and_return([200, ''])
+    it 'calls validate monitor in dry-run' do
+      expect_any_instance_of(Dogapi::Client).to receive(:validate_monitor).and_return([200, ''])
       expect(datadog_dry_run).to receive(:existing_alerts).and_return(mock_response)
       datadog_dry_run.create_alert(mock_alert, mock_people)
     end
@@ -114,12 +114,4 @@ describe Interferon::Destinations::Datadog do
       datadog.remove_alert(mock_alert)
     end
   end
-
-  describe '.remove_alert_by_id' do
-    it 'calls dogapi delete_monitor' do
-      expect_any_instance_of(Dogapi::Client).to receive(:delete_monitor)
-        .with(mock_alert_id).and_return([200, ''])
-      datadog.remove_alert_by_id(mock_alert_id)
-    end
-  end
 end

data/spec/lib/interferon_spec.rb

@@ -71,130 +71,135 @@ describe Interferon::Destinations::Datadog do
   end
 
   context 'dry_run_update_alerts_on_destination' do
-    let(:interferon) { Interferon::Interferon.new(nil, nil, nil, nil, true, 0) }
+    let(:interferon) { Interferon::Interferon.new({ 'processes' => 0 }, true) }
 
     before do
       allow_any_instance_of(MockAlert).to receive(:evaluate)
       allow(dest).to receive(:remove_alert)
-      allow(dest).to receive(:remove_alert_by_id)
       allow(dest).to receive(:report_stats)
     end
 
     it 'does not re-run existing alerts' do
-      alerts = mock_existing_alerts
+      mock_alerts = mock_existing_alerts
       expect(dest).not_to receive(:create_alert)
-      expect(dest).not_to receive(:remove_alert_by_id)
 
-      interferon.update_alerts_on_destination(
-        dest, ['host'], [alerts['name1'], alerts['name2']], {}
+      alerts_queue, _error_count = interferon.build_alerts_queue(
+        ['host'],
+        [mock_alerts['name1'], mock_alerts['name2']].map { |x| test_alert_from_json(x) },
+        {}
       )
+
+      interferon.update_alerts_on_destination(dest, alerts_queue)
     end
 
     it 'runs added alerts' do
-      alerts = mock_existing_alerts
-      added = create_test_alert('name3', 'testquery3', '')
+      mock_alerts = mock_existing_alerts
+      alerts = [mock_alerts['name1'], mock_alerts['name2']].map { |x| test_alert_from_json(x) }
+      alerts << create_test_alert('name3', 'testquery3', '')
+
+      alerts_queue, _error_count = interferon.build_alerts_queue(['host'], alerts, {})
+
       expect(dest).to receive(:create_alert).once.and_call_original
-      expect(dest).to receive(:remove_alert_by_id).with('3').once
 
-      interferon.update_alerts_on_destination(
-        dest, ['host'], [alerts['name1'], alerts['name2'], added], {}
-      )
+      interferon.update_alerts_on_destination(dest, alerts_queue)
     end
 
     it 'runs updated alerts' do
       added = create_test_alert('name1', 'testquery3', '')
+      alerts_queue, _error_count = interferon.build_alerts_queue(['host'], [added], {})
       expect(dest).to receive(:create_alert).once.and_call_original
-      expect(dest).to receive(:remove_alert_by_id).with('1').once
 
-      interferon.update_alerts_on_destination(dest, ['host'], [added], {})
+      interferon.update_alerts_on_destination(dest, alerts_queue)
     end
 
-    it 'deletes old alerts' do
-      expect(dest).to receive(:remove_alert).twice
+    it 'does not delete old alerts' do
+      expect(dest).to_not receive(:remove_alert)
+      alerts_queue, _error_count = interferon.build_alerts_queue(['host'], [], {})
 
-      interferon.update_alerts_on_destination(dest, ['host'], [], {})
+      interferon.update_alerts_on_destination(dest, alerts_queue)
     end
 
-    it 'deletes duplicate old alerts' do
+    it 'does not delete duplicate old alerts' do
       alert1 = mock_alert_json('name1', 'testquery1', '', nil, [1, 2, 3])
       alert2 = mock_alert_json('name2', 'testquery2', '')
       existing_alerts = { 'name1' => alert1, 'name2' => alert2 }
+
       dest = MockDest.new(existing_alerts)
-      allow(dest).to receive(:remove_alert)
-      allow(dest).to receive(:remove_alert_by_id)
       allow(dest).to receive(:report_stats)
 
-      expect(dest).to receive(:remove_alert).with(existing_alerts['name1'])
-      expect(dest).to receive(:remove_alert).with(existing_alerts['name2'])
+      alerts_queue, _error_count = interferon.build_alerts_queue(['host'], [], {})
+
+      expect(dest).to_not receive(:remove_alert)
 
-      interferon.update_alerts_on_destination(dest, ['host'], [], {})
+      interferon.update_alerts_on_destination(dest, alerts_queue)
     end
 
-    it 'deletes duplicate old alerts when creating new alert' do
+    it 'does not delete duplicate old alerts when creating new alert' do
       alert1 = mock_alert_json('name1', 'testquery1', '', nil, [1, 2, 3])
       alert2 = mock_alert_json('name2', 'testquery2', '')
       existing_alerts = { 'name1' => alert1, 'name2' => alert2 }
+
       dest = MockDest.new(existing_alerts)
-      allow(dest).to receive(:remove_alert)
-      allow(dest).to receive(:remove_alert_by_id)
       allow(dest).to receive(:report_stats)
 
       added = create_test_alert('name1', 'testquery1', '')
+      alerts_queue, _error_count = interferon.build_alerts_queue(['host'], [added], {})
 
-      # Since we change id to nil we will not be attempting to delete duplicate alerts
-      # during dry run
-      expect(dest).to_not receive(:remove_alert).with(existing_alerts['name1'])
-      expect(dest).to receive(:remove_alert).with(existing_alerts['name2'])
+      expect(dest).to_not receive(:remove_alert)
 
-      interferon.update_alerts_on_destination(dest, ['host'], [added], {})
+      interferon.update_alerts_on_destination(dest, alerts_queue)
     end
   end
 
   context 'update_alerts_on_destination' do
-    let(:interferon) { Interferon::Interferon.new(nil, nil, nil, nil, false, 0) }
+    let(:interferon) { Interferon::Interferon.new({ 'processes' => 0 }, false) }
 
     before do
       allow_any_instance_of(MockAlert).to receive(:evaluate)
       allow(dest).to receive(:remove_alert)
-      allow(dest).to receive(:remove_alert_by_id)
       allow(dest).to receive(:report_stats)
     end
 
     it 'does not re-run existing alerts' do
-      alerts = mock_existing_alerts
+      mock_alerts = mock_existing_alerts
       expect(dest).not_to receive(:create_alert)
-      expect(dest).not_to receive(:remove_alert_by_id)
 
-      interferon.update_alerts_on_destination(
-        dest, ['host'], [alerts['name1'], alerts['name2']], {}
+      alerts_queue, _error_count = interferon.build_alerts_queue(
+        ['host'],
+        [mock_alerts['name1'], mock_alerts['name2']].map { |x| test_alert_from_json(x) },
+        {}
       )
+
+      interferon.update_alerts_on_destination(dest, alerts_queue)
     end
 
     it 'runs added alerts' do
-      alerts = mock_existing_alerts
-      added = create_test_alert('name3', 'testquery3', '')
+      mock_alerts = mock_existing_alerts
+      alerts = [mock_alerts['name1'], mock_alerts['name2']].map { |x| test_alert_from_json(x) }
+      alerts << create_test_alert('name3', 'testquery3', '')
+
+      alerts_queue, _error_count = interferon.build_alerts_queue(['host'], alerts, {})
+
       expect(dest).to receive(:create_alert).once.and_call_original
-      expect(dest).not_to receive(:remove_alert_by_id).with('3')
 
-      interferon.update_alerts_on_destination(
-        dest, ['host'], [alerts['name1'], alerts['name2'], added], {}
-      )
+      interferon.update_alerts_on_destination(dest, alerts_queue)
     end
 
     it 'runs updated alerts' do
       added = create_test_alert('name1', 'testquery3', '')
+      alerts_queue, _error_count = interferon.build_alerts_queue(['host'], [added], {})
       expect(dest).to receive(:create_alert).once.and_call_original
-      expect(dest).not_to receive(:remove_alert_by_id).with('1')
 
-      interferon.update_alerts_on_destination(dest, ['host'], [added], {})
+      interferon.update_alerts_on_destination(dest, alerts_queue)
     end
 
     it 'deletes old alerts' do
       alerts = mock_existing_alerts
+      alerts_queue, _error_count = interferon.build_alerts_queue(['host'], [], {})
       expect(dest).to receive(:remove_alert).with(alerts['name1'])
       expect(dest).to receive(:remove_alert).with(alerts['name2'])
 
-      interferon.update_alerts_on_destination(dest, ['host'], [], {})
+      interferon.update_alerts_on_destination(dest, alerts_queue)
     end
 
     it 'deletes duplicate old alerts' do
@@ -203,13 +208,14 @@ describe Interferon::Destinations::Datadog do
       existing_alerts = { 'name1' => alert1, 'name2' => alert2 }
       dest = MockDest.new(existing_alerts)
       allow(dest).to receive(:remove_alert)
-      allow(dest).to receive(:remove_alert_by_id)
       allow(dest).to receive(:report_stats)
 
+      alerts_queue, _error_count = interferon.build_alerts_queue(['host'], [], {})
+
       expect(dest).to receive(:remove_alert).with(existing_alerts['name1'])
       expect(dest).to receive(:remove_alert).with(existing_alerts['name2'])
 
-      interferon.update_alerts_on_destination(dest, ['host'], [], {})
+      interferon.update_alerts_on_destination(dest, alerts_queue)
     end
 
     it 'deletes duplicate old alerts when creating new alert' do
@@ -220,19 +226,21 @@ describe Interferon::Destinations::Datadog do
       allow(dest).to receive(:report_stats)
 
       added = create_test_alert('name1', 'testquery1', '')
+      alerts_queue, _error_count = interferon.build_alerts_queue(['host'], [added], {})
 
       expect(dest).to receive(:remove_alert).with(
         mock_alert_json('name1', 'testquery1', '', nil, [2, 3])
       )
       expect(dest).to receive(:remove_alert).with(existing_alerts['name2'])
 
-      interferon.update_alerts_on_destination(dest, ['host'], [added], {})
+      interferon.update_alerts_on_destination(dest, alerts_queue)
     end
   end
 
   def mock_existing_alerts
-    alert1 = mock_alert_json('name1', 'testquery1', '')
-    alert2 = mock_alert_json('name2', 'testquery2', '')
+    mock_message = Interferon::Destinations::Datadog::ALERT_KEY
+    alert1 = mock_alert_json('name1', 'testquery1', mock_message)
+    alert2 = mock_alert_json('name2', 'testquery2', mock_message)
     { 'name1' => alert1, 'name2' => alert2 }
   end
 
@@ -274,6 +282,15 @@ describe Interferon::Destinations::Datadog do
     }
   end
 
+  def test_alert_from_json(mock_alert_json)
+    create_test_alert(
+      mock_alert_json['name'],
+      mock_alert_json['query'],
+      mock_alert_json['message'].sub(/#{Interferon::Destinations::Datadog::ALERT_KEY}$/, ''),
+      mock_alert_json['options']
+    )
+  end
+
   def create_test_alert(name, datadog_query, message, options = {})
     options = DEFAULT_OPTIONS.merge(options)
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: interferon
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.2.0
 platform: ruby
 authors:
 - Igor Serebryany