flapjack 0.6.23 → 0.6.24

data/Rakefile

@@ -14,8 +14,14 @@ Dir['tasks/**/*.rake'].each { |t| load t }
 require 'cucumber'
 require 'cucumber/rake/task'
 require 'colorize'
+require 'rake/clean'
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
 Cucumber::Rake::Task.new(:features) do |t|
-  t.cucumber_opts = "features --format pretty"
+  #t.cucumber_opts = "features --format pretty"
+  #t.cucumber_opts = "--format progress"
+  t.cucumber_opts = "--format fuubar"
 end
 
 require 'rspec/core/rake_task'
@@ -23,42 +29,6 @@ RSpec::Core::RakeTask.new(:spec)
 
 task :default => :spec
 
-
-desc "build gem"
-task :build => :verify do
-  build_output = `gem build flapjack.gemspec`
-  puts build_output
-
-  gem_filename = build_output[/File: (.*)/,1]
-  pkg_path = "pkg"
-  FileUtils.mkdir_p(pkg_path)
-  FileUtils.mv(gem_filename, pkg_path)
-
-  puts "Gem built in #{pkg_path}/#{gem_filename}".green
-end
-
-desc "push gem"
-task :push do
-  filenames = Dir.glob("pkg/*.gem")
-  filenames_with_times = filenames.map do |filename|
-    [filename, File.mtime(filename)]
-  end
-
-  newest = filenames_with_times.sort_by { |tuple| tuple.last }.last
-  newest_filename = newest.first
-
-  command = "gem push #{newest_filename}"
-  system(command)
-end
-
-desc "clean up various generated files"
-task :clean do
-  [ "pkg/"].each do |filename|
-    puts "Removing #{filename}"
-    FileUtils.rm_rf(filename)
-  end
-end
-
 namespace :verify do
   task :uncommitted do
     uncommitted = `git ls-files -m`.split("\n")
@@ -75,5 +45,4 @@ namespace :verify do
 end
 
 # FIXME: getting that intermittent gherken lexing error so removing :features from verify list
-#task :verify => [ 'verify:all', :spec, :features]
-task :verify => [ 'verify:all', :spec]
+task :verify => [ 'verify:all', :spec, :features]

data/flapjack.gemspec

@@ -2,11 +2,11 @@
 require File.expand_path('../lib/flapjack/version', __FILE__)
 
 Gem::Specification.new do |gem|
-  gem.authors       = ["Lindsay Holmwood"]
-  gem.email         = %q{lindsay@holmwood.id.au}
-  gem.description   = %q{Flapjack is highly scalable and distributed monitoring system. It understands the Nagios plugin format, and can easily be scaled from 1 server to 1000.}
-  gem.summary       = %q{a scalable and distributed monitoring system}
-  gem.homepage      = %q{http://flapjack-project.com/}
+  gem.authors       = [ "Lindsay Holmwood", "Jesse Reynolds", "Ali Graham" ]
+  gem.email         = "lindsay@holmwood.id.au"
+  gem.description   = "Flapjack is distributed monitoring notification system that provides a scalable method for processing streams of events from Nagios and deciding who should be notified"
+  gem.summary       = "Intelligent, scalable, distributed monitoring notification system."
+  gem.homepage      = "http://flapjack-project.com/"
 
   # see http://yehudakatz.com/2010/12/16/clarifying-the-roles-of-the-gemspec-and-gemfile/
   # following a middle road here, not shipping it with the gem :)

data/lib/flapjack/coordinator.rb

@@ -17,6 +17,7 @@ require 'flapjack/api'
 require 'flapjack/daemonizing'
 require 'flapjack/executive'
 require 'flapjack/jabber'
+require 'flapjack/oobetet'
 require 'flapjack/pagerduty'
 require 'flapjack/notification/email'
 require 'flapjack/notification/sms'
@@ -77,17 +78,18 @@ module Flapjack
         @logger.debug "config keys: #{@config.keys}"
 
         pikelet_keys = ['executive', 'jabber_gateway', 'pagerduty_gateway',
-                        'email_notifier', 'sms_notifier', 'web', 'api']
+                        'email_notifier', 'sms_notifier', 'web', 'api',
+                        'oobetet']
 
         @config.keys.each do |pikelet_type|
-          next unless pikelet_keys.include?(pikelet_type) && 
+          next unless pikelet_keys.include?(pikelet_type) &&
             @config[pikelet_type].is_a?(Hash) &&
             @config[pikelet_type]['enabled']
           @logger.debug "coordinator is now initialising the #{pikelet_type} pikelet"
           pikelet_cfg = @config[pikelet_type]
 
           case pikelet_type
-          when 'executive', 'jabber_gateway', 'pagerduty_gateway'
+          when 'executive', 'jabber_gateway', 'pagerduty_gateway', 'oobetet'
             build_pikelet(pikelet_type, pikelet_cfg)
           when 'web', 'api'
             build_thin_pikelet(pikelet_type, pikelet_cfg)
@@ -118,6 +120,8 @@ module Flapjack
         Flapjack::Jabber
       when 'pagerduty_gateway'
         Flapjack::Pagerduty
+      when 'oobetet'
+        Flapjack::Oobetet
       end
       return unless pikelet_class
 

data/lib/flapjack/jabber.rb

@@ -85,14 +85,16 @@ module Flapjack
       @redis_handler ||= build_redis_connection_pool
       @connected_at = Time.now.to_i
       logger.info("Jabber Connected")
-      @config['rooms'].each do |room|
-        logger.info("Joining room #{room}")
-        presence = Blather::Stanza::Presence.new
-        presence.from = @flapjack_jid
-        presence.to = Blather::JID.new("#{room}/#{@config['alias']}")
-        presence << "<x xmlns='http://jabber.org/protocol/muc'/>"
-        write presence
-        say(room, "flapjack jabber gateway started at #{Time.now}, hello!", :groupchat)
+      if @config['rooms'] && @config['rooms'].length > 0
+        @config['rooms'].each do |room|
+          logger.info("Joining room #{room}")
+          presence = Blather::Stanza::Presence.new
+          presence.from = @flapjack_jid
+          presence.to = Blather::JID.new("#{room}/#{@config['alias']}")
+          presence << "<x xmlns='http://jabber.org/protocol/muc'/>"
+          write presence
+          say(room, "flapjack jabber gateway started at #{Time.now}, hello!", :groupchat)
+        end
       end
     end
 
@@ -129,6 +131,10 @@ module Flapjack
           error = "unknown entity" if entity_check.nil?
         end
 
+        if entity_check && entity_check.in_unscheduled_maintenance?
+          error = "#{event_id} is already acknowledged"
+        end
+
         if error
           msg = "ERROR - couldn't ACK #{ackid} - #{error}"
         else
@@ -185,7 +191,13 @@ module Flapjack
       return if should_quit?
       logger.debug("chat message received: #{stanza.inspect}")
 
-      results = interpreter(stanza.body)
+      if stanza.body =~ /^flapjack:\s+(.*)/
+        command = $1
+      else
+        command = stanza.body
+      end
+
+      results = interpreter(command)
       msg     = results[:msg]
       action  = results[:action]
 
@@ -274,7 +286,7 @@ module Flapjack
             msg = "#{type.upcase} #{ack_str}::: \"#{check}\" on #{entity} #{maint_str} ::: #{summary}"
 
             chat_type = :chat
-            chat_type = :groupchat if @config['rooms'].include?(address)
+            chat_type = :groupchat if @config['rooms'] && @config['rooms'].include?(address)
             EM.next_tick do
               say(Blather::JID.new(address), msg, chat_type)
             end

data/lib/flapjack/oobetet.rb

@@ -0,0 +1,245 @@
+#!/usr/bin/env ruby
+
+#require 'socket'
+
+require 'eventmachine'
+# the redis/synchrony gems need to be required in this particular order, see                                                # the redis-rb README for details
+#require 'hiredis'
+require 'em-synchrony'
+#require 'redis/connection/synchrony'
+#require 'redis'
+
+#require 'chronic_duration'
+
+require 'blather/client/client'
+require 'em-synchrony/fiber_iterator'
+require 'yajl/json_gem'
+
+#require 'flapjack/data/entity_check'
+require 'flapjack/pikelet'
+require 'flapjack/utility'
+
+module Flapjack
+
+  class Oobetet < Blather::Client
+
+    include Flapjack::Pikelet
+    include Flapjack::Utility
+
+    log = Logger.new(STDOUT)
+    # log.level = Logger::DEBUG
+    log.level = Logger::INFO
+    Blather.logger = log
+
+    def setup
+      @hostname = Socket.gethostname
+      @flapjacktest_jid = Blather::JID.new((@config['jabberid'] || 'flapjacktest') + "/#{@hostname}:#{Process.pid}")
+
+      super(@flapjacktest_jid, @config['password'], @config['server'], @config['port'].to_i)
+
+      logger.debug("Building jabber connection with jabberid: " +
+        @flapjacktest_jid.to_s + ", port: " + @config['port'].to_s +
+        ", server: " + @config['server'].to_s + ", password: " +
+        @config['password'].to_s)
+
+      @pagerduty_events_api_url = 'https://events.pagerduty.com/generic/2010-04-15/create_event.json'
+
+      if !@config['watched_check'] or !@config['watched_entity']
+        raise RuntimeError, 'Flapjack::Oobetet: watched_check and watched_entity must be defined in the config'
+      end
+
+      @check_matcher = '"' + @config['watched_check'] + '" on ' + @config['watched_entity']
+      @max_latency = @config['max_latency'] || 300
+      @flapjack_ok = true
+
+      t = Time.now.to_i
+      @times = { :last_problem  => t,
+                 :last_recovery => t,
+                 :last_ack      => t,
+                 :last_ack_sent => t }
+
+      @last_alert = nil
+
+      register_handler :ready do |stanza|
+        EM.next_tick do
+          EM.synchrony do
+            on_ready(stanza)
+          end
+        end
+      end
+
+      register_handler :message, :groupchat? do |stanza|
+        EM.next_tick do
+          EM.synchrony do
+            on_groupchat(stanza)
+          end
+        end
+      end
+
+      register_handler :disconnected do |stanza|
+        ret = true
+        EM.next_tick do
+          EM.synchrony do
+            ret = on_disconnect(stanza)
+          end
+        end
+        ret
+      end
+
+    end
+
+    # Join the MUC Chat room after connecting.
+    def on_ready(stanza)
+      return if should_quit?
+      @connected_at = Time.now.to_i
+      logger.info("Jabber Connected")
+      if @config['rooms'] && @config['rooms'].length > 0
+        @config['rooms'].each do |room|
+          logger.info("Joining room #{room}")
+          presence = Blather::Stanza::Presence.new
+          presence.from = @flapjacktest_jid
+          presence.to = Blather::JID.new("#{room}/#{@config['alias']}")
+          presence << "<x xmlns='http://jabber.org/protocol/muc'/>"
+          write presence
+          say(room, "flapjack self monitoring (oobetet) started at #{Time.now}, g'day!", :groupchat)
+        end
+      end
+    end
+
+    # returning true to prevent the reactor loop from stopping
+    def on_disconnect(stanza)
+      return true if should_quit?
+      logger.warn("jabbers disconnected! reconnecting in 1 second ...")
+      EventMachine::Timer.new(1) do
+        connect # Blather::Client.connect
+      end
+      true
+    end
+
+    def on_groupchat(stanza)
+      return if should_quit?
+      logger.debug("groupchat stanza body: " + stanza.body)
+      logger.debug("groupchat message received: #{stanza.inspect}")
+
+      if stanza.body =~ /^(\w+).*#{Regexp.escape(@check_matcher)}/
+        # got something interesting
+        logger.debug("groupchat found the following state for #{@check_matcher}: #{$1.downcase}")
+        case $1.downcase
+        when 'problem'
+          logger.debug("updating @times last_problem")
+          @times[:last_problem] = Time.new.to_i
+        when 'recovery'
+          logger.debug("updating @times last_recovery")
+          @times[:last_recovery] = Time.new.to_i
+        when 'acknowledgement'
+          logger.debug("updating @times last_ack")
+          @times[:last_ack] = Time.new.to_i
+        end
+
+      end
+      logger.debug("@times: #{@times.inspect}")
+
+    end
+
+    def check_timers
+      t = Time.new.to_i
+      breach = nil
+      @logger.debug("check_timers: inspecting @times #{@times.inspect}")
+      case
+      when @times[:last_problem] < (t - @max_latency)
+        breach = "haven't seen a test problem notification in the last #{@max_latency} seconds"
+      when @times[:last_recovery] < (t - @max_latency)
+        breach = "haven't seen a test recovery notification in the last #{@max_latency} seconds"
+      end
+
+      if !@flapjack_ok and !breach
+        emit_jabber("Flapjack Self Monitoring is OK")
+        emit_pagerduty("Flapjack Self Monitoring is OK", 'resolve')
+      end
+
+      @flapjack_ok = !breach
+
+      return unless breach
+      @logger.error("Self monitoring has detected the following breach: #{breach}")
+      summary  = "Flapjack Self Monitoring is Critical: #{breach} for #{@check_matcher}, "
+      summary += "from #{@hostname} at #{Time.now}"
+
+      if !@last_alert or @last_alert < (t - 55)
+
+        emit_jabber(summary)
+        emit_pagerduty(summary, 'trigger')
+
+        if !@last_alert or @last_alert < (t - 55)
+          msg  = "NOTICE: Self monitoring has detected a failure and is unable to tell "
+          msg += "anyone about it. DON'T PANIC."
+          @logger.error msg
+        end
+
+      end
+    end
+
+    def emit_jabber(summary)
+      if @config['rooms'] && @config['rooms'].length > 0
+        @config['rooms'].each do |room|
+          say(room, summary, :groupchat)
+        end
+        @last_alert = Time.now.to_i
+      end
+    end
+
+    def emit_pagerduty(summary, event_type = 'trigger')
+      if @config['pagerduty_contact']
+        pagerduty_event = { :service_key  => @config['pagerduty_contact'],
+                            :incident_key => "Flapjack Self Monitoring from #{@hostname}",
+                            :event_type   => event_type,
+                            :description  => summary }
+        status, response = send_pagerduty_event(pagerduty_event)
+        if status == 200
+          @logger.debug("successfully sent pagerduty event")
+          @last_alert = Time.now.to_i
+        else
+          @logger.error("pagerduty returned #{status} #{response.inspect}")
+        end
+      end
+    end
+
+    def say(to, msg, using = :chat)
+      @logger.debug("Sending a jabber message to: #{to.to_s}, using: #{using.to_s}, message: #{msg}")
+      write Blather::Stanza::Message.new(to, msg, using)
+    end
+
+    def send_pagerduty_event(event)
+      options  = { :body => Yajl::Encoder.encode(event) }
+      http = EM::HttpRequest.new(@pagerduty_events_api_url).post(options)
+      response = Yajl::Parser.parse(http.response)
+      status   = http.response_header.status
+      logger.debug "send_pagerduty_event got a return code of #{status.to_s} - #{response.inspect}"
+      [status, response]
+    end
+
+    def main
+      logger.debug("New oobetet pikelet with the following options: #{@config.inspect}")
+
+      keepalive_timer = EM::Synchrony.add_periodic_timer(60) do
+        logger.debug("calling keepalive on the jabber connection")
+        write(' ') if connected?
+      end
+
+      check_timers_timer = EM::Synchrony.add_periodic_timer(10) do
+        check_timers
+      end
+
+      setup
+      connect # Blather::Client.connect
+
+      if should_quit?
+        keepalive_timer.cancel
+        check_timers_timer.cancel
+      end
+    end
+
+  end
+end
+
+
+

data/lib/flapjack/version.rb

@@ -1,5 +1,5 @@
 #!/usr/bin/env ruby
 
 module Flapjack
-  VERSION = "0.6.23"
+  VERSION = "0.6.24"
 end

data/spec/lib/flapjack/jabber_spec.rb

@@ -65,6 +65,7 @@ describe Flapjack::Jabber do
       and_return('main-example.com:ping')
 
     entity_check = mock(Flapjack::Data::EntityCheck)
+    entity_check.should_receive(:in_unscheduled_maintenance?)
     entity_check.should_receive(:create_acknowledgement).
       with('summary' => 'fixing now', 'acknowledgement_id' => '876', 'duration' => (90 * 60))
     entity_check.should_receive(:entity_name).and_return('main-example.com')

metadata

@@ -1,11 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: flapjack
 version: !ruby/object:Gem::Version
-  version: 0.6.23
+  version: 0.6.24
   prerelease: 
 platform: ruby
 authors:
 - Lindsay Holmwood
+- Jesse Reynolds
+- Ali Graham
 autorequire: 
 bindir: bin
 cert_chain: []
@@ -331,8 +333,9 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-description: Flapjack is highly scalable and distributed monitoring system. It understands
-  the Nagios plugin format, and can easily be scaled from 1 server to 1000.
+description: Flapjack is distributed monitoring notification system that provides
+  a scalable method for processing streams of events from Nagios and deciding who
+  should be notified
 email: lindsay@holmwood.id.au
 executables:
 - flapjack
@@ -428,6 +431,7 @@ files:
 - lib/flapjack/notifiers/mailer/mailer.rb
 - lib/flapjack/notifiers/xmpp/init.rb
 - lib/flapjack/notifiers/xmpp/xmpp.rb
+- lib/flapjack/oobetet.rb
 - lib/flapjack/pagerduty.rb
 - lib/flapjack/patches.rb
 - lib/flapjack/persistence/couch.rb
@@ -539,7 +543,7 @@ rubyforge_project:
 rubygems_version: 1.8.23
 signing_key: 
 specification_version: 3
-summary: a scalable and distributed monitoring system
+summary: Intelligent, scalable, distributed monitoring notification system.
 test_files:
 - features/events.feature
 - features/notifications.feature