flapjack 1.2.1 → 1.2.2

data/lib/flapjack/filters/delays.rb

@@ -20,8 +20,15 @@ module Flapjack
       include Base
 
       def block?(event, entity_check, previous_state)
-        failure_delay = 30
-        resend_delay  = 60
+        initial_failure_delay = entity_check.initial_failure_delay
+        if initial_failure_delay.nil? || (initial_failure_delay < 0)
+          initial_failure_delay = Flapjack::DEFAULT_INITIAL_FAILURE_DELAY
+        end
+
+        repeat_failure_delay = entity_check.repeat_failure_delay
+        if repeat_failure_delay.nil? || (repeat_failure_delay < 0)
+          repeat_failure_delay = Flapjack::DEFAULT_REPEAT_FAILURE_DELAY
+        end
 
         label = 'Filter: Delays:'
 
@@ -51,18 +58,18 @@ module Flapjack
                       "event.state: [#{event.state.inspect}], " +
                       "last_alert_state == event.state ? #{last_alert_state.to_s == event.state}")
 
-        if current_state_duration < failure_delay
+        if current_state_duration < initial_failure_delay
           @logger.debug("#{label} block - duration of current failure " +
-                     "(#{current_state_duration}) is less than failure_delay (#{failure_delay})")
+                     "(#{current_state_duration}) is less than initial_failure_delay (#{initial_failure_delay})")
           return true
         end
 
-        if !last_problem_alert.nil? && (time_since_last_alert < resend_delay) &&
+        if !last_problem_alert.nil? && (time_since_last_alert < repeat_failure_delay) &&
           (last_alert_state.to_s == event.state)
 
           @logger.debug("#{label} block - time since last alert for " +
                         "current problem (#{time_since_last_alert}) is less than " +
-                        "resend_delay (#{resend_delay}) and last alert state (#{last_alert_state}) " +
+                        "repeat_failure_delay (#{repeat_failure_delay}) and last alert state (#{last_alert_state}) " +
                         "is equal to current event state (#{event.state})")
           return true
         end

data/lib/flapjack/gateways/aws_sns.rb

@@ -4,6 +4,8 @@ require 'em-synchrony'
 require 'em-synchrony/em-http'
 require 'active_support/inflector'
 
+require 'flapjack/redis_pool'
+
 require 'flapjack/data/alert'
 require 'flapjack/utility'
 
@@ -13,121 +15,146 @@ module Flapjack
 
       SNS_DEFAULT_REGION_NAME = 'us-east-1'
 
-      class << self
+      include Flapjack::Utility
 
-        include Flapjack::Utility
+      def initialize(opts = {})
+        @config = opts[:config]
+        @logger = opts[:logger]
+        @redis_config = opts[:redis_config] || {}
+        @redis = Flapjack::RedisPool.new(:config => @redis_config, :size => 1, :logger => @logger)
 
-        def start
-          @sent = 0
-        end
+        @logger.info("starting")
+        @logger.debug("new aws_sns gateway pikelet with the following options: #{@config.inspect}")
 
-        def perform(contents)
-          @logger.debug "Received a notification: #{contents.inspect}"
-          alert = Flapjack::Data::Alert.new(contents, :logger => @logger)
-
-          region_name = @config["region_name"] || SNS_DEFAULT_REGION_NAME
-          hostname = "sns.#{region_name}.amazonaws.com"
-          endpoint = "http://#{hostname}/"
-          access_key = @config["access_key"]
-          secret_key = @config["secret_key"]
-          timestamp = @config["timestamp"] || DateTime.now.iso8601
-
-          address         = alert.address
-          notification_id = alert.notification_id
-          message_type    = alert.rollup ? 'rollup' : 'alert'
-
-          my_dir = File.dirname(__FILE__)
-          sms_template_path = case
-          when @config.has_key?('templates') && @config['templates']["#{message_type}.text"]
-            @config['templates']["#{message_type}.text"]
-          else
-            my_dir + "/aws_sns/#{message_type}.text.erb"
-          end
-          sms_template = ERB.new(File.read(sms_template_path), nil, '-')
+        @sent = 0
+      end
+
+      def stop
+        @logger.info("stopping")
+        @should_quit = true
+
+        redis_uri = @redis_config[:path] ||
+          "redis://#{@redis_config[:host] || '127.0.0.1'}:#{@redis_config[:port] || '6379'}/#{@redis_config[:db] || '0'}"
+        shutdown_redis = EM::Hiredis.connect(redis_uri)
+        shutdown_redis.rpush(@config['queue'], Flapjack.dump_json('notification_type' => 'shutdown'))
+      end
 
-          @alert  = alert
-          bnd     = binding
+      def start
+        queue = @config['queue']
 
+        until @should_quit
           begin
-            message = sms_template.result(bnd).chomp
+            @logger.debug("aws_sns gateway is going into blpop mode on #{queue}")
+            deliver( Flapjack::Data::Alert.next(queue, :redis => @redis, :logger => @logger) )
           rescue => e
-            @logger.error "Error while excuting the ERB for an sms: " +
-              "ERB being executed: #{sms_template_path}"
-            raise
+            @logger.error "Error generating or dispatching AWS SNS message: #{e.class}: #{e.message}\n" +
+              e.backtrace.join("\n")
           end
+        end
+      end
 
-          if @config.nil? || (@config.respond_to?(:empty?) && @config.empty?)
-            @logger.error "AWS SNS config is missing"
-            return
-          end
+      def deliver(alert)
+        region_name = @config["region_name"] || SNS_DEFAULT_REGION_NAME
+        hostname = "sns.#{region_name}.amazonaws.com"
+        endpoint = "http://#{hostname}/"
+        access_key = @config["access_key"]
+        secret_key = @config["secret_key"]
+        timestamp = @config["timestamp"] || DateTime.now.iso8601
+
+        address         = alert.address
+        notification_id = alert.notification_id
+        message_type    = alert.rollup ? 'rollup' : 'alert'
+
+        my_dir = File.dirname(__FILE__)
+        sms_template_path = case
+        when @config.has_key?('templates') && @config['templates']["#{message_type}.text"]
+          @config['templates']["#{message_type}.text"]
+        else
+          my_dir + "/aws_sns/#{message_type}.text.erb"
+        end
+        sms_template = ERB.new(File.read(sms_template_path), nil, '-')
 
-          errors = []
+        @alert  = alert
+        bnd     = binding
 
-          [[address, "AWS SNS topic ARN is missing"],
-           [access_key, "AWS SNS access key is missing"],
-           [secret_key,  "AWS SNS secret key is missing"],
-           [notification_id, "Notification id is missing"]].each do |val_err|
+        begin
+          message = sms_template.result(bnd).chomp
+        rescue => e
+          @logger.error "Error while excuting the ERB for an sms: " +
+            "ERB being executed: #{sms_template_path}"
+          raise
+        end
 
-            next unless val_err.first.nil? || (val_err.first.respond_to?(:empty?) && val_err.first.empty?)
-            errors << val_err.last
-          end
+        if @config.nil? || (@config.respond_to?(:empty?) && @config.empty?)
+          @logger.error "AWS SNS config is missing"
+          return
+        end
 
-          unless errors.empty?
-            errors.each {|err| @logger.error err }
-            return
-          end
+        errors = []
 
+        [[address, "AWS SNS topic ARN is missing"],
+         [access_key, "AWS SNS access key is missing"],
+         [secret_key,  "AWS SNS secret key is missing"],
+         [notification_id, "Notification id is missing"]].each do |val_err|
 
-          query = {'Subject'          => message,
-                   'TopicArn'         => address,
-                   'Message'          => message,
-                   'Action'           => 'Publish',
-                   'SignatureVersion' => 2,
-                   'SignatureMethod'  => 'HmacSHA256',
-                   'Timestamp'        => timestamp,
-                   'AWSAccessKeyId'   => access_key}
+          next unless val_err.first.nil? || (val_err.first.respond_to?(:empty?) && val_err.first.empty?)
+          errors << val_err.last
+        end
 
-          string_to_sign = string_to_sign('POST', hostname, "/", query)
+        unless errors.empty?
+          errors.each {|err| @logger.error err }
+          return
+        end
 
-          query['Signature'] = get_signature(secret_key, string_to_sign)
+        query = {'Subject'          => message,
+                 'TopicArn'         => address,
+                 'Message'          => message,
+                 'Action'           => 'Publish',
+                 'SignatureVersion' => 2,
+                 'SignatureMethod'  => 'HmacSHA256',
+                 'Timestamp'        => timestamp,
+                 'AWSAccessKeyId'   => access_key}
 
-          http = EM::HttpRequest.new(endpoint).post(:query => query)
+        string_to_sign = self.class.string_to_sign('POST', hostname, "/", query)
 
-          @logger.debug "server response: #{http.response}"
+        query['Signature'] = self.class.get_signature(secret_key, string_to_sign)
 
-          status = (http.nil? || http.response_header.nil?) ? nil : http.response_header.status
-          if (status >= 200) && (status <= 206)
-            @sent += 1
-            alert.record_send_success!
-            @logger.debug "Sent notification via SNS, response status is #{status}, " +
-              "notification_id: #{notification_id}"
-          else
-            @logger.error "Failed to send notification via SNS, response status is #{status}, " +
-              "notification_id: #{notification_id}"
-          end
-        rescue => e
-          @logger.error "Error generating or delivering notification to #{address}: #{e.class}: #{e.message}"
-          @logger.error e.backtrace.join("\n")
-          raise
-        end
+        http = EM::HttpRequest.new(endpoint).post(:query => query)
 
-        def get_signature(secret_key, string_to_sign)
-          signature = OpenSSL::HMAC.digest('sha256', secret_key, string_to_sign)
+        @logger.debug "server response: #{http.response}"
 
-          Base64.encode64(signature).strip
+        status = (http.nil? || http.response_header.nil?) ? nil : http.response_header.status
+        if (status >= 200) && (status <= 206)
+          @sent += 1
+          alert.record_send_success!
+          @logger.debug "Sent notification via SNS, response status is #{status}, " +
+            "notification_id: #{notification_id}"
+        else
+          @logger.error "Failed to send notification via SNS, response status is #{status}, " +
+            "notification_id: #{notification_id}"
         end
+      rescue => e
+        @logger.error "Error generating or delivering notification to #{address}: #{e.class}: #{e.message}"
+        @logger.error e.backtrace.join("\n")
+        raise
+      end
 
-        def string_to_sign(method, host, uri, query)
-          query = query.sort_by { |key, value| key }
+      def self.get_signature(secret_key, string_to_sign)
+        signature = OpenSSL::HMAC.digest('sha256', secret_key, string_to_sign)
 
-          [method.upcase,
-           host.downcase,
-           uri,
-           URI.encode_www_form(query)
-          ].join("\n")
-        end
+        Base64.encode64(signature).strip
+      end
+
+      def self.string_to_sign(method, host, uri, query)
+        query = query.sort_by { |key, value| key }
 
+        [method.upcase,
+         host.downcase,
+         uri,
+         URI.encode_www_form(query)
+        ].join("\n")
       end
+
     end
   end
 end

data/lib/flapjack/gateways/aws_sns/alert.text.erb

@@ -1,5 +1,6 @@
+<% summary = @alert.summary %>
 <%= @alert.type_sentence_case %>: '<%= @alert.check %>' on <%= @alert.entity -%>
 <% unless ['acknowledgement', 'test'].include?(@alert.notification_type) -%>
  is <%= @alert.state_title_case -%>
 <% end -%>
- at <%= Time.at(@alert.time).strftime('%-d %b %H:%M') %>, <%= @alert.summary -%>
+ at <%= Time.at(@alert.time).strftime('%-d %b %H:%M') %><%= (summary.nil? || summary.empty?) ? '' : ", #{summary}" -%>

data/lib/flapjack/gateways/email.rb

@@ -9,6 +9,7 @@ require 'active_support/inflector'
 require 'em-synchrony'
 require 'em/protocols/smtpclient'
 
+require 'flapjack/redis_pool'
 require 'flapjack/utility'
 
 require 'flapjack/data/entity_check'
@@ -19,169 +20,178 @@ module Flapjack
 
     class Email
 
-      class << self
+      include Flapjack::Utility
 
-        include Flapjack::Utility
+      def initialize(opts = {})
+        @config = opts[:config]
+        @logger = opts[:logger]
+        @redis_config = opts[:redis_config] || {}
+        @redis = Flapjack::RedisPool.new(:config => @redis_config, :size => 1, :logger => @logger)
 
-        def start
-          @logger.info("starting")
-          @logger.debug("new email gateway pikelet with the following options: #{@config.inspect}")
-          @smtp_config = @config.delete('smtp_config')
-          @sent = 0
-          @fqdn = `/bin/hostname -f`.chomp
-        end
-
-        # TODO refactor to remove complexity
-        def perform(contents)
-          @logger.debug "Woo, got an alert to send out: #{contents.inspect}"
-          alert = prepare(contents)
-          deliver(alert)
-        end
-
-        # sets a bunch of class instance variables for each email
-        def prepare(contents)
-          Flapjack::Data::Alert.new(contents, :logger => @logger)
-        rescue => e
-          @logger.error "Error preparing email to #{contents['address']}: #{e.class}: #{e.message}"
-          @logger.error e.backtrace.join("\n")
-          raise
-        end
+        @logger.info("starting")
+        @logger.debug("new email gateway pikelet with the following options: #{@config.inspect}")
+        @smtp_config = @config.delete('smtp_config')
+        @sent = 0
+        @fqdn = `/bin/hostname -f`.chomp
+      end
 
-        def deliver(alert)
-          host = @smtp_config ? @smtp_config['host'] : nil
-          port = @smtp_config ? @smtp_config['port'] : nil
-          starttls = @smtp_config ? !! @smtp_config['starttls'] : nil
-          m_from = @smtp_config ? @smtp_config['from'] : "flapjack@#{@fqdn}"
-          m_reply_to = @smtp_config ? ( @smtp_config['reply_to'] ||= m_from ) : "flapjack@#{@fqdn}"
-          if @smtp_config
-            if auth_config = @smtp_config['auth']
-              auth = {}
-              auth[:type]     = auth_config['type'].to_sym || :plain
-              auth[:username] = auth_config['username']
-              auth[:password] = auth_config['password']
-            end
-          end
+      def stop
+        @logger.info("stopping")
+        @should_quit = true
 
-          @logger.debug("flapjack_mailer: set from to #{m_from}")
+        redis_uri = @redis_config[:path] ||
+          "redis://#{@redis_config[:host] || '127.0.0.1'}:#{@redis_config[:port] || '6379'}/#{@redis_config[:db] || '0'}"
+        shutdown_redis = EM::Hiredis.connect(redis_uri)
+        shutdown_redis.rpush(@config['queue'], Flapjack.dump_json('notification_type' => 'shutdown'))
+      end
 
-          mail = prepare_email(:from       => m_from,
-                               :reply_to   => m_reply_to,
-                               :to         => alert.address,
-                               :message_id => "<#{alert.notification_id}@#{@fqdn}>",
-                               :alert      => alert)
+      def start
+        queue = @config['queue']
 
-          smtp_from = m_from.clone
-          while smtp_from =~ /(<|>)/
-            smtp_from.sub!(/^.*</, '')
-            smtp_from.sub!(/>.*$/, '')
+        until @should_quit
+          begin
+            @logger.debug("email gateway is going into blpop mode on #{queue}")
+            deliver( Flapjack::Data::Alert.next(queue, :redis => @redis, :logger => @logger) )
+          rescue => e
+            @logger.error "Error generating or dispatching email message: #{e.class}: #{e.message}\n" +
+              e.backtrace.join("\n")
           end
+        end
+      end
 
-          smtp_args = {:from     => smtp_from,
-                       :to       => alert.address,
-                       :content  => "#{mail.to_s}\r\n.\r\n",
-                       :domain   => @fqdn,
-                       :host     => host || 'localhost',
-                       :port     => port || 25,
-                       :starttls => starttls}
-          smtp_args.merge!(:auth => auth) if auth
-
-          email = EM::P::SmtpClient.send(smtp_args)
-
-          response = EM::Synchrony.sync(email)
-
-          # http://tools.ietf.org/html/rfc821#page-36 SMTP response codes
-          if response && response.respond_to?(:code) &&
-            ((response.code == 250) || (response.code == 251))
-            alert.record_send_success!
-            @sent += 1
-          else
-            @logger.error "Email sending failed"
+      def deliver(alert)
+        host = @smtp_config ? @smtp_config['host'] : nil
+        port = @smtp_config ? @smtp_config['port'] : nil
+        starttls = @smtp_config ? !! @smtp_config['starttls'] : nil
+        m_from = @smtp_config ? @smtp_config['from'] : "flapjack@#{@fqdn}"
+        m_reply_to = @smtp_config ? ( @smtp_config['reply_to'] ||= m_from ) : "flapjack@#{@fqdn}"
+        if @smtp_config
+          if auth_config = @smtp_config['auth']
+            auth = {}
+            auth[:type]     = auth_config['type'].to_sym || :plain
+            auth[:username] = auth_config['username']
+            auth[:password] = auth_config['password']
           end
+        end
 
-          @logger.debug "Email response: #{response.inspect}"
+        @logger.debug("flapjack_mailer: set from to #{m_from}")
 
-        rescue => e
-          @logger.error "Error generating or delivering email to #{alert.address}: #{e.class}: #{e.message}"
-          @logger.error e.backtrace.join("\n")
-          raise
+        mail = prepare_email(:from       => m_from,
+                             :reply_to   => m_reply_to,
+                             :to         => alert.address,
+                             :message_id => "<#{alert.notification_id}@#{@fqdn}>",
+                             :alert      => alert)
+
+        smtp_from = m_from.clone
+        while smtp_from =~ /(<|>)/
+          smtp_from.sub!(/^.*</, '')
+          smtp_from.sub!(/>.*$/, '')
         end
 
-        private
+        smtp_args = {:from     => smtp_from,
+                     :to       => alert.address,
+                     :content  => "#{mail.to_s}\r\n.\r\n",
+                     :domain   => @fqdn,
+                     :host     => host || 'localhost',
+                     :port     => port || 25,
+                     :starttls => starttls}
+        smtp_args.merge!(:auth => auth) if auth
+
+        email = EM::P::SmtpClient.send(smtp_args)
+
+        response = EM::Synchrony.sync(email)
+
+        # http://tools.ietf.org/html/rfc821#page-36 SMTP response codes
+        if response && response.respond_to?(:code) &&
+          ((response.code == 250) || (response.code == 251))
+          alert.record_send_success!
+          @sent += 1
+        else
+          @logger.error "Email sending failed"
+        end
 
-        # returns a Mail object
-        def prepare_email(opts = {})
-          from       = opts[:from]
-          reply_to   = opts[:reply_to]
-          to         = opts[:to]
-          message_id = opts[:message_id]
-          alert      = opts[:alert]
+        @logger.debug "Email response: #{response.inspect}"
 
-          message_type = alert.rollup ? 'rollup' : 'alert'
+      rescue => e
+        @logger.error "Error generating or delivering email to #{alert.address}: #{e.class}: #{e.message}"
+        @logger.error e.backtrace.join("\n")
+        raise
+      end
 
-          mydir = File.dirname(__FILE__)
-          subject_template_path = case
-          when @config.has_key?('templates') && @config['templates']["#{message_type}_subject.text"]
-            @config['templates']["#{message_type}_subject.text"]
-          else
-            mydir + "/email/#{message_type}_subject.text.erb"
-          end
-          text_template_path = case
-          when @config.has_key?('templates') && @config['templates']["#{message_type}.text"]
-            @config['templates']["#{message_type}.text"]
-          else
-            mydir + "/email/#{message_type}.text.erb"
-          end
-          html_template_path = case
-          when @config.has_key?('templates') && @config['templates']["#{message_type}.html"]
-            @config['templates']["#{message_type}.html"]
-          else
-            mydir + "/email/#{message_type}.html.erb"
-          end
-          subject_template = ERB.new(File.read(subject_template_path), nil, '-')
-          text_template    = ERB.new(File.read(text_template_path), nil, '-')
-          html_template    = ERB.new(File.read(html_template_path), nil, '-')
+      private
 
-          @alert  = alert
-          bnd     = binding
+      # returns a Mail object
+      def prepare_email(opts = {})
+        from       = opts[:from]
+        reply_to   = opts[:reply_to]
+        to         = opts[:to]
+        message_id = opts[:message_id]
+        alert      = opts[:alert]
 
-          # do some intelligence gathering in case an ERB execution blows up
-          begin
-            erb_to_be_executed = subject_template_path
-            subject = subject_template.result(bnd).chomp
+        message_type = alert.rollup ? 'rollup' : 'alert'
 
-            erb_to_be_executed = text_template_path
-            body_text = text_template.result(bnd)
+        mydir = File.dirname(__FILE__)
+        subject_template_path = case
+        when @config.has_key?('templates') && @config['templates']["#{message_type}_subject.text"]
+          @config['templates']["#{message_type}_subject.text"]
+        else
+          mydir + "/email/#{message_type}_subject.text.erb"
+        end
+        text_template_path = case
+        when @config.has_key?('templates') && @config['templates']["#{message_type}.text"]
+          @config['templates']["#{message_type}.text"]
+        else
+          mydir + "/email/#{message_type}.text.erb"
+        end
+        html_template_path = case
+        when @config.has_key?('templates') && @config['templates']["#{message_type}.html"]
+          @config['templates']["#{message_type}.html"]
+        else
+          mydir + "/email/#{message_type}.html.erb"
+        end
+        subject_template = ERB.new(File.read(subject_template_path), nil, '-')
+        text_template    = ERB.new(File.read(text_template_path), nil, '-')
+        html_template    = ERB.new(File.read(html_template_path), nil, '-')
 
-            erb_to_be_executed = html_template_path
-            body_html = html_template.result(bnd)
-          rescue => e
-            @logger.error "Error while excuting ERBs for an email: " +
-              "ERB being executed: #{erb_to_be_executed}"
-            raise
-          end
+        @alert  = alert
+        bnd     = binding
+
+        # do some intelligence gathering in case an ERB execution blows up
+        begin
+          erb_to_be_executed = subject_template_path
+          subject = subject_template.result(bnd).chomp
 
-          @logger.debug("preparing email to: #{to}, subject: #{subject}, message-id: #{message_id}")
+          erb_to_be_executed = text_template_path
+          body_text = text_template.result(bnd)
 
-          mail = Mail.new do
-            from       from
-            to         to
-            subject    subject
-            reply_to   reply_to
-            message_id message_id
+          erb_to_be_executed = html_template_path
+          body_html = html_template.result(bnd)
+        rescue => e
+          @logger.error "Error while excuting ERBs for an email: " +
+            "ERB being executed: #{erb_to_be_executed}"
+          raise
+        end
 
-            text_part do
-              body body_text
-            end
+        @logger.debug("preparing email to: #{to}, subject: #{subject}, message-id: #{message_id}")
 
-            html_part do
-              content_type 'text/html; charset=UTF-8'
-              body body_html
-            end
+        mail = Mail.new do
+          from       from
+          to         to
+          subject    subject
+          reply_to   reply_to
+          message_id message_id
+
+          text_part do
+            body body_text
           end
 
+          html_part do
+            content_type 'text/html; charset=UTF-8'
+            body body_html
+          end
         end
-      end
 
+      end
     end
   end
 end