flapjack 0.6.43 → 0.6.44

data/lib/flapjack/gateways/oobetet.rb

@@ -0,0 +1,241 @@
+#!/usr/bin/env ruby
+
+require 'socket'
+require 'eventmachine'
+require 'em-synchrony'
+require 'blather/client/client'
+require 'em-synchrony/fiber_iterator'
+require 'yajl/json_gem'
+
+require 'flapjack/utility'
+require 'flapjack/gateways/base'
+
+module Flapjack
+
+  module Gateways
+
+    class Oobetet < Blather::Client
+
+      include Flapjack::Gateways::Generic
+      include Flapjack::Utility
+
+      log = Logger.new(STDOUT)
+      # log.level = Logger::DEBUG
+      log.level = Logger::INFO
+      Blather.logger = log
+
+      def setup
+        @hostname = Socket.gethostname
+        @flapjacktest_jid = Blather::JID.new((@config['jabberid'] || 'flapjacktest') + "/#{@hostname}:#{Process.pid}")
+
+        super(@flapjacktest_jid, @config['password'], @config['server'], @config['port'].to_i)
+
+        logger.debug("Building jabber connection with jabberid: " +
+          @flapjacktest_jid.to_s + ", port: " + @config['port'].to_s +
+          ", server: " + @config['server'].to_s + ", password: " +
+          @config['password'].to_s)
+
+        @pagerduty_events_api_url = 'https://events.pagerduty.com/generic/2010-04-15/create_event.json'
+
+        if !@config['watched_check'] or !@config['watched_entity']
+          raise RuntimeError, 'Flapjack::Oobetet: watched_check and watched_entity must be defined in the config'
+        end
+
+        @check_matcher = '"' + @config['watched_check'] + '" on ' + @config['watched_entity']
+        @max_latency = @config['max_latency'] || 300
+        @flapjack_ok = true
+
+        t = Time.now.to_i
+        @times = { :last_problem  => t,
+                   :last_recovery => t,
+                   :last_ack      => t,
+                   :last_ack_sent => t }
+
+        @last_alert = nil
+      end
+
+      # split out to ease testing
+      def register_handlers
+        register_handler :ready do |stanza|
+          EventMachine::Synchrony.next_tick do
+            on_ready(stanza)
+          end
+        end
+
+        register_handler :message, :groupchat? do |stanza|
+          EventMachine::Synchrony.next_tick do
+            on_groupchat(stanza)
+          end
+        end
+
+        register_handler :disconnected do |stanza|
+          ret = true
+          EventMachine::Synchrony.next_tick do
+            ret = on_disconnect(stanza)
+          end
+          ret
+        end
+      end
+
+
+      # Join the MUC Chat room after connecting.
+      def on_ready(stanza)
+        return if should_quit?
+        @connected_at = Time.now.to_i
+        logger.info("Jabber Connected")
+        if @config['rooms'] && @config['rooms'].length > 0
+          @config['rooms'].each do |room|
+            logger.info("Joining room #{room}")
+            presence = Blather::Stanza::Presence.new
+            presence.from = @flapjacktest_jid
+            presence.to = Blather::JID.new("#{room}/#{@config['alias']}")
+            presence << "<x xmlns='http://jabber.org/protocol/muc'/>"
+            write presence
+            say(room, "flapjack self monitoring (oobetet) started at #{Time.now}, g'day!", :groupchat)
+          end
+        end
+      end
+
+      # returning true to prevent the reactor loop from stopping
+      def on_disconnect(stanza)
+        return true if should_quit?
+        logger.warn("jabbers disconnected! reconnecting in 1 second ...")
+        EventMachine::Timer.new(1) do
+          connect # Blather::Client.connect
+        end
+        true
+      end
+
+      def on_groupchat(stanza)
+        return if should_quit?
+
+        stanza_body = stanza.body
+
+        logger.debug("groupchat stanza body: " + stanza_body)
+        logger.debug("groupchat message received: #{stanza.inspect}")
+
+        if (stanza_body =~ /^(?:problem|recovery|acknowledgement)/i) &&
+           (stanza_body =~ /^(\w+).*#{Regexp.escape(@check_matcher)}/)
+
+          # got something interesting
+          status = $1.downcase
+          t = Time.now.to_i
+          logger.debug("groupchat found the following state for #{@check_matcher}: #{status}")
+
+          case status
+          when 'problem'
+            logger.debug("updating @times last_problem")
+            @times[:last_problem] = t
+          when 'recovery'
+            logger.debug("updating @times last_recovery")
+            @times[:last_recovery] = t
+          when 'acknowledgement'
+            logger.debug("updating @times last_ack")
+            @times[:last_ack] = t
+          end
+        end
+        logger.debug("@times: #{@times.inspect}")
+      end
+
+      def check_timers
+        t = Time.now.to_i
+        breach = nil
+        @logger.debug("check_timers: inspecting @times #{@times.inspect}")
+        case
+        when @times[:last_problem] < (t - @max_latency)
+          breach = "haven't seen a test problem notification in the last #{@max_latency} seconds"
+        when @times[:last_recovery] < (t - @max_latency)
+          breach = "haven't seen a test recovery notification in the last #{@max_latency} seconds"
+        end
+
+        unless @flapjack_ok || breach
+          emit_jabber("Flapjack Self Monitoring is OK")
+          emit_pagerduty("Flapjack Self Monitoring is OK", 'resolve')
+        end
+
+        @flapjack_ok = !breach
+
+        return unless breach
+        @logger.error("Self monitoring has detected the following breach: #{breach}")
+        summary  = "Flapjack Self Monitoring is Critical: #{breach} for #{@check_matcher}, "
+        summary += "from #{@hostname} at #{Time.now}"
+
+        if !@last_alert or @last_alert < (t - 55)
+
+          emit_jabber(summary)
+          emit_pagerduty(summary, 'trigger')
+
+          if !@last_alert or @last_alert < (t - 55)
+            msg  = "NOTICE: Self monitoring has detected a failure and is unable to tell "
+            msg += "anyone about it. DON'T PANIC."
+            @logger.error msg
+          end
+
+        end
+      end
+
+      def emit_jabber(summary)
+        if @config['rooms'] && @config['rooms'].length > 0
+          @config['rooms'].each do |room|
+            say(room, summary, :groupchat)
+          end
+          @last_alert = Time.now.to_i
+        end
+      end
+
+      def emit_pagerduty(summary, event_type = 'trigger')
+        if @config['pagerduty_contact']
+          pagerduty_event = { :service_key  => @config['pagerduty_contact'],
+                              :incident_key => "Flapjack Self Monitoring from #{@hostname}",
+                              :event_type   => event_type,
+                              :description  => summary }
+          status, response = send_pagerduty_event(pagerduty_event)
+          if status == 200
+            @logger.debug("successfully sent pagerduty event")
+            @last_alert = Time.now.to_i
+          else
+            @logger.error("pagerduty returned #{status} #{response.inspect}")
+          end
+        end
+      end
+
+      def say(to, msg, using = :chat)
+        @logger.debug("Sending a jabber message to: #{to.to_s}, using: #{using.to_s}, message: #{msg}")
+        write Blather::Stanza::Message.new(to, msg, using)
+      end
+
+      def send_pagerduty_event(event)
+        options  = { :body => Yajl::Encoder.encode(event) }
+        http = EM::HttpRequest.new(@pagerduty_events_api_url).post(options)
+        response = Yajl::Parser.parse(http.response)
+        status   = http.response_header.status
+        logger.debug "send_pagerduty_event got a return code of #{status.to_s} - #{response.inspect}"
+        [status, response]
+      end
+
+      def main
+        logger.debug("New oobetet pikelet with the following options: #{@config.inspect}")
+
+        keepalive_timer = EM::Synchrony.add_periodic_timer(60) do
+          logger.debug("calling keepalive on the jabber connection")
+          write(' ') if connected?
+        end
+
+        setup
+        register_handlers
+        connect # Blather::Client.connect
+
+        until should_quit?
+          EM::Synchrony.sleep(10)
+          check_timers
+        end
+
+        keepalive_timer.cancel
+      end
+
+    end
+  end
+end
+
+
+

data/lib/flapjack/gateways/pagerduty.rb

@@ -0,0 +1,247 @@
+#!/usr/bin/env ruby
+
+require 'em-synchrony'
+require 'em-synchrony/em-http'
+
+require 'yajl/json_gem'
+
+require 'flapjack/data/entity_check'
+require 'flapjack/data/global'
+require 'flapjack/redis_pool'
+
+require 'flapjack/gateways/base'
+
+module Flapjack
+
+  module Gateways
+
+    class Pagerduty
+      include Flapjack::Gateways::Generic
+
+      PAGERDUTY_EVENTS_API_URL   = 'https://events.pagerduty.com/generic/2010-04-15/create_event.json'
+      SEM_PAGERDUTY_ACKS_RUNNING = 'sem_pagerduty_acks_running'
+
+      alias_method :generic_bootstrap, :bootstrap
+      alias_method :generic_cleanup,   :cleanup
+
+      def bootstrap(opts = {})
+        generic_bootstrap(opts)
+
+        @redis_config = opts[:redis_config]
+        @redis = Flapjack::RedisPool.new(:config => @redis_config, :size => 1)
+
+        logger.debug("New Pagerduty pikelet with the following options: #{@config.inspect}")
+
+        @pagerduty_acks_started = nil
+      end
+
+      def cleanup
+        @redis.empty! if @redis
+        @redis_timer.empty! if @redis_timer
+        generic_cleanup
+      end
+
+      def add_shutdown_event(opts = {})
+        return unless redis = opts[:redis]
+        redis.rpush(@config['queue'], JSON.generate('notification_type' => 'shutdown'))
+      end
+
+      def main
+        logger.debug("pagerduty gateway - commencing main method")
+        raise "Can't connect to the pagerduty API" unless test_pagerduty_connection
+
+        # TODO: only clear this if there isn't another pagerduty gateway instance running
+        # or better, include an instance ID in the semaphore key name
+        @redis.del(SEM_PAGERDUTY_ACKS_RUNNING)
+
+        acknowledgement_timer = EM::Synchrony.add_periodic_timer(10) do
+          @redis_timer ||= Flapjack::RedisPool.new(:config => @redis_config, :size => 1)
+          find_pagerduty_acknowledgements_if_safe
+        end
+
+        queue = @config['queue']
+        events = {}
+
+        until should_quit?
+          logger.debug("pagerduty gateway is going into blpop mode on #{queue}")
+          events[queue] = @redis.blpop(queue, 0)
+          event         = Yajl::Parser.parse(events[queue][1])
+          type          = event['notification_type']
+          logger.debug("pagerduty notification event popped off the queue: " + event.inspect)
+          unless 'shutdown'.eql?(type)
+            event_id      = event['event_id']
+            entity, check = event_id.split(':')
+            state         = event['state']
+            summary       = event['summary']
+            address       = event['address']
+
+            headline = type.upcase
+
+            case type.downcase
+            when 'acknowledgement'
+              maint_str      = "has been acknowledged"
+              pagerduty_type = 'acknowledge'
+            when 'problem'
+              maint_str      = "is #{state.upcase}"
+              pagerduty_type = "trigger"
+            when 'recovery'
+              maint_str      = "is #{state.upcase}"
+              pagerduty_type = "resolve"
+            when 'test'
+              maint_str      = ""
+              pagerduty_type = "trigger"
+              headline       = "TEST NOTIFICATION"
+            end
+
+            message = "#{type.upcase} - \"#{check}\" on #{entity} #{maint_str} - #{summary}"
+
+            pagerduty_event = { :service_key  => address,
+                                :incident_key => event_id,
+                                :event_type   => pagerduty_type,
+                                :description  => message }
+
+            send_pagerduty_event(pagerduty_event)
+          end
+        end
+
+        acknowledgement_timer.cancel
+      end
+
+      # considering this as part of the public API -- exposes it for testing.
+      def find_pagerduty_acknowledgements_if_safe
+
+        # ensure we're the only instance of the pagerduty acknowledgement check running (with a naive
+        # timeout of five minutes to guard against stale locks caused by crashing code) either in this
+        # process or in other processes
+        if (@pagerduty_acks_started and @pagerduty_acks_started > (Time.now.to_i - 300)) or
+            @redis_timer.get(SEM_PAGERDUTY_ACKS_RUNNING) == 'true'
+          logger.debug("skipping looking for acks in pagerduty as this is already happening")
+          return
+        end
+
+        @pagerduty_acks_started = Time.now.to_i
+        @redis_timer.set(SEM_PAGERDUTY_ACKS_RUNNING, 'true')
+        @redis_timer.expire(SEM_PAGERDUTY_ACKS_RUNNING, 300)
+
+        find_pagerduty_acknowledgements
+
+        @redis_timer.del(SEM_PAGERDUTY_ACKS_RUNNING)
+        @pagerduty_acks_started = nil
+      end
+
+    private
+
+      def test_pagerduty_connection
+        noop = { "service_key"  => "11111111111111111111111111111111",
+                 "incident_key" => "Flapjack is running a NOOP",
+                 "event_type"   => "nop",
+                 "description"  => "I love APIs with noops." }
+        code, results = send_pagerduty_event(noop)
+        return true if code == 200 && results['status'] =~ /success/i
+        logger.error "Error: test_pagerduty_connection: API returned #{code.to_s} #{results.inspect}"
+        false
+      end
+
+      def send_pagerduty_event(event)
+        options  = { :body => Yajl::Encoder.encode(event) }
+        http = EM::HttpRequest.new(PAGERDUTY_EVENTS_API_URL).post(options)
+        response = Yajl::Parser.parse(http.response)
+        status   = http.response_header.status
+        logger.debug "send_pagerduty_event got a return code of #{status.to_s} - #{response.inspect}"
+        [status, response]
+      end
+
+      def find_pagerduty_acknowledgements
+
+        logger.debug("looking for acks in pagerduty for unack'd problems")
+
+        unacknowledged_failing_checks = Flapjack::Data::Global.unacknowledged_failing_checks(:redis => @redis_timer)
+
+        @logger.debug "found unacknowledged failing checks as follows: " + unacknowledged_failing_checks.join(', ')
+
+        unacknowledged_failing_checks.each do |entity_check|
+          pagerduty_credentials = entity_check.pagerduty_credentials(:redis => @redis_timer)
+          check = entity_check.check
+
+          if pagerduty_credentials.empty?
+            @logger.debug("No pagerduty credentials found for #{entity_check.entity_name}:#{check}, skipping")
+            next
+          end
+
+          # FIXME: try each set of credentials until one works (may have stale contacts turning up)
+          options = pagerduty_credentials.first.merge('check' => check)
+
+          acknowledged = pagerduty_acknowledged?(options)
+          if acknowledged.nil?
+            @logger.debug "#{check} is not acknowledged in pagerduty, skipping"
+            next
+          end
+
+          pg_acknowledged_by = acknowledged[:pg_acknowledged_by]
+          @logger.debug "#{check} is acknowledged in pagerduty, creating flapjack acknowledgement... "
+          who_text = ""
+          if !pg_acknowledged_by.nil? && !pg_acknowledged_by['name'].nil?
+            who_text = " by #{pg_acknowledged_by['name']}"
+          end
+          entity_check.create_acknowledgement('summary' => "Acknowledged on PagerDuty" + who_text)
+        end
+
+      end
+
+      def pagerduty_acknowledged?(opts)
+        subdomain   = opts['subdomain']
+        username    = opts['username']
+        password    = opts['password']
+        check       = opts['check']
+
+        t = Time.now.utc
+
+        url = 'https://' + subdomain + '.pagerduty.com/api/v1/incidents'
+        query = { 'fields'       => 'incident_number,status,last_status_change_by',
+                  'since'        => (t - (60*60*24*7)).iso8601, # the last week
+                  'until'        => (t + (60*60*24)).iso8601,   # 1 day in the future
+                  'incident_key' => check,
+                  'status'       => 'acknowledged' }
+
+        options = { :head  => { 'authorization' => [username, password] },
+                    :query => query }
+
+        http = EM::HttpRequest.new(url).get(options)
+        # DEBUG flapjack-pagerduty: pagerduty_acknowledged?: decoded response as:
+        # {"incidents"=>[{"incident_number"=>40, "status"=>"acknowledged",
+        # "last_status_change_by"=>{"id"=>"PO1NWPS", "name"=>"Jesse Reynolds",
+        # "email"=>"jesse@bulletproof.net",
+        # "html_url"=>"http://bltprf.pagerduty.com/users/PO1NWPS"}}], "limit"=>100, "offset"=>0,
+        # "total"=>1}
+        begin
+          response = Yajl::Parser.parse(http.response)
+        rescue Yajl::ParseError
+          @logger.error("failed to parse json from a post to #{url} ... response headers and body follows...")
+          @logger.error(http.response_header.inspect)
+          @logger.error(http.response)
+          return nil
+        end
+        status   = http.response_header.status
+
+        @logger.debug("pagerduty_acknowledged?: decoded response as: #{response.inspect}")
+        if response.nil?
+          @logger.error('no valid response received from pagerduty!')
+          return nil
+        end
+
+        if response['incidents'].nil?
+          @logger.error('no incidents found in response')
+          return nil
+        end
+
+        return nil if response['incidents'].empty?
+
+        {:pg_acknowledged_by => response['incidents'].first['last_status_change_by']}
+      end
+
+    end
+
+  end
+
+end
+