sensu 0.9.7.beta.2 → 0.9.7.beta.3

data/CHANGELOG.md

@@ -15,8 +15,16 @@ AMQP handlers can no longer use `"send_only_check_output": true`, but
 instead have access to the built-in mutators `"mutator": "only_check_output"` and
 `"mutator": "only_check_output_split"`.
 
+Ruby 1.8.7-p249 is no longer supported, as the AMQP library no longer
+does. Please use the Sensu APT/YUM packages which contain an embedded
+Ruby.
+
 ### Other
 
+Improved RabbitMQ and Redis connection recovery.
+
+Fixed API POST input validation.
+
 Redis client connection heartbeat.
 
 Improved graceful process termination.

data/lib/sensu/api.rb

@@ -8,57 +8,116 @@ module Sensu
   class API < Sinatra::Base
     register Sinatra::Async
 
-    def self.run(options={})
-      EM::run do
-        self.setup(options)
-
-        Thin::Logging.silent = true
-        Thin::Server.start(self, $settings[:api][:port])
+    class << self
+      def run(options={})
+        EM::run do
+          bootstrap(options)
+          start
+          trap_signals
+        end
+      end
 
-        %w[INT TERM].each do |signal|
-          Signal.trap(signal) do
-            self.stop(signal)
+      def bootstrap(options={})
+        $logger = Cabin::Channel.get
+        base = Sensu::Base.new(options)
+        $settings = base.settings
+        if $settings[:api][:user] && $settings[:api][:password]
+          use Rack::Auth::Basic do |user, password|
+            user == $settings[:api][:user] && password == $settings[:api][:password]
           end
         end
       end
-    end
 
-    def self.setup(options={})
-      $logger = Cabin::Channel.get
-      base = Sensu::Base.new(options)
-      $settings = base.settings
-      $logger.debug('connecting to redis', {
-        :settings => $settings[:redis]
-      })
-      $redis = Sensu::Redis.connect($settings[:redis])
-      $redis.on_disconnect = Proc.new do
-        if $redis.connection_established?
-          $logger.warn('reconnecting to redis')
-          $redis.reconnect!
-        else
+      def setup_redis
+        $logger.debug('connecting to redis', {
+          :settings => $settings[:redis]
+        })
+        connection_failure = Proc.new do
           $logger.fatal('cannot connect to redis', {
             :settings => $settings[:redis]
           })
           $logger.fatal('SENSU NOT RUNNING!')
+          if $rabbitmq
+            $rabbitmq.close
+          end
           exit 2
         end
+        $redis = Sensu::Redis.connect($settings[:redis], :on_tcp_connection_failure => connection_failure)
+        $redis.on_tcp_connection_loss do |connection, settings|
+          $logger.warn('reconnecting to redis')
+          connection.reconnect(false, 10)
+        end
       end
-      $logger.debug('connecting to rabbitmq', {
-        :settings => $settings[:rabbitmq]
-      })
-      $rabbitmq = AMQP.connect($settings[:rabbitmq])
-      $rabbitmq.on_disconnect = Proc.new do
-        $logger.fatal('cannot connect to rabbitmq', {
+
+      def setup_rabbitmq
+        $logger.debug('connecting to rabbitmq', {
           :settings => $settings[:rabbitmq]
         })
-        $logger.fatal('SENSU NOT RUNNING!')
+        connection_failure = Proc.new do
+          $logger.fatal('cannot connect to rabbitmq', {
+            :settings => $settings[:rabbitmq]
+          })
+          $logger.fatal('SENSU NOT RUNNING!')
+          $redis.close
+          exit 2
+        end
+        $rabbitmq = AMQP.connect($settings[:rabbitmq], :on_tcp_connection_failure => connection_failure)
+        $rabbitmq.on_tcp_connection_loss do |connection, settings|
+          $logger.warn('reconnecting to rabbitmq')
+          connection.reconnect(false, 10)
+        end
+        $amq = AMQP::Channel.new($rabbitmq)
+        $amq.auto_recovery = true
+      end
+
+      def start
+        setup_redis
+        setup_rabbitmq
+        Thin::Logging.silent = true
+        Thin::Server.start(self, $settings[:api][:port])
+      end
+
+      def stop
+        $logger.warn('stopping')
+        $rabbitmq.close
         $redis.close
-        exit 2
+        $logger.warn('stopping reactor')
+        EM::stop_event_loop
+      end
+
+      def trap_signals
+        %w[INT TERM].each do |signal|
+          Signal.trap(signal) do
+            $logger.warn('received signal', {
+              :signal => signal
+            })
+            stop
+          end
+        end
       end
-      $amq = AMQP::Channel.new($rabbitmq)
-      if $settings[:api][:user] && $settings[:api][:password]
-        use Rack::Auth::Basic do |user, password|
-          user == $settings[:api][:user] && password == $settings[:api][:password]
+
+      def run_test(options={}, &block)
+        bootstrap(options)
+        start
+        $settings[:client][:timestamp] = Time.now.to_i
+        $redis.set('client:' + $settings[:client][:name], $settings[:client].to_json).callback do
+          $redis.sadd('clients', $settings[:client][:name]).callback do
+            $redis.hset('events:' + $settings[:client][:name], 'test', {
+              :output => 'CRITICAL',
+              :status => 2,
+              :issued => Time.now.to_i,
+              :flapping => false,
+              :occurrences => 1
+            }.to_json).callback do
+              $redis.set('stash:test/test', {:key => 'value'}.to_json).callback do
+                $redis.sadd('stashes', 'test/test').callback do
+                  EM::Timer.new(0.5) do
+                    block.call
+                  end
+                end
+              end
+            end
+          end
         end
       end
     end
@@ -96,6 +155,31 @@ module Sensu
         end
       end
 
+      def bad_request!
+        status 400
+        body ''
+      end
+
+      def not_found!
+        status 404
+        body ''
+      end
+
+      def created!
+        status 201
+        body ''
+      end
+
+      def accepted!
+        status 202
+        body ''
+      end
+
+      def no_content!
+        status 204
+        body ''
+      end
+
       def event_hash(event_json, client_name, check_name)
         JSON.parse(event_json, :symbolize_names => true).merge(
           :client => client_name,
@@ -163,8 +247,7 @@ module Sensu
         unless client_json.nil?
           body client_json
         else
-          status 404
-          body ''
+          not_found!
         end
       end
     end
@@ -191,12 +274,10 @@ module Sensu
                 $redis.del('history:' + client_name)
               end
             end
-            status 202
-            body ''
+            accepted!
           end
         else
-          status 404
-          body ''
+          not_found!
         end
       end
     end
@@ -210,8 +291,7 @@ module Sensu
         response = $settings[:checks][check_name].merge(:name => check_name)
         body response.to_json
       else
-        status 404
-        body ''
+        not_found!
       end
     end
 
@@ -220,27 +300,25 @@ module Sensu
         post_body = JSON.parse(request.body.read, :symbolize_names => true)
         check_name = post_body[:check]
         subscribers = post_body[:subscribers]
-      rescue JSON::ParserError, TypeError
-        status 400
-        body ''
-      end
-      if check_name.is_a?(String) && subscribers.is_a?(Array)
-        payload = {
-          :name => check_name,
-          :issued => Time.now.to_i
-        }
-        $logger.info('publishing check request', {
-          :payload => payload,
-          :subscribers => subscribers
-        })
-        subscribers.uniq.each do |exchange_name|
-          $amq.fanout(exchange_name).publish(payload.to_json)
+        if check_name.is_a?(String) && subscribers.is_a?(Array)
+          payload = {
+            :name => check_name,
+            :issued => Time.now.to_i
+          }
+          $logger.info('publishing check request', {
+            :payload => payload,
+            :subscribers => subscribers
+          })
+          subscribers.uniq.each do |exchange_name|
+            $amq.fanout(exchange_name).publish(payload.to_json)
+          end
+          created!
+        else
+          bad_request!
         end
-        status 201
-      else
-        status 400
+      rescue JSON::ParserError, TypeError
+        bad_request!
       end
-      body ''
     end
 
     aget '/events' do
@@ -279,8 +357,7 @@ module Sensu
         unless event_json.nil?
           body event_hash(event_json, client_name, check_name).to_json
         else
-          status 404
-          body ''
+          not_found!
         end
       end
     end
@@ -289,11 +366,10 @@ module Sensu
       $redis.hgetall('events:' + client_name).callback do |events|
         if events.include?(check_name)
           resolve_event(client_name, check_name)
-          status 202
+          accepted!
         else
-          status 404
+          not_found!
         end
-        body ''
       end
     end
 
@@ -302,48 +378,42 @@ module Sensu
         post_body = JSON.parse(request.body.read, :symbolize_names => true)
         client_name = post_body[:client]
         check_name = post_body[:check]
-      rescue JSON::ParserError, TypeError
-        status 400
-        body ''
-      end
-      if client_name.is_a?(String) && check_name.is_a?(String)
-        $redis.hgetall('events:' + client_name).callback do |events|
-          if events.include?(check_name)
-            resolve_event(client_name, check_name)
-            status 202
-          else
-            status 404
+        if client_name.is_a?(String) && check_name.is_a?(String)
+          $redis.hgetall('events:' + client_name).callback do |events|
+            if events.include?(check_name)
+              resolve_event(client_name, check_name)
+              accepted!
+            else
+              not_found!
+            end
           end
-          body ''
+        else
+          bad_request!
         end
-      else
-        status 400
-        body ''
+      rescue JSON::ParserError, TypeError
+        bad_request!
       end
     end
 
     apost %r{/stash(?:es)?/(.*)} do |path|
       begin
         post_body = JSON.parse(request.body.read)
-      rescue JSON::ParserError
-        status 400
-        body ''
-      end
-      $redis.set('stash:' + path, post_body.to_json).callback do
-        $redis.sadd('stashes', path).callback do
-          status 201
-          body ''
+        $redis.set('stash:' + path, post_body.to_json).callback do
+          $redis.sadd('stashes', path).callback do
+            created!
+          end
         end
+      rescue JSON::ParserError
+        bad_request!
       end
     end
 
     aget %r{/stash(?:es)?/(.*)} do |path|
       $redis.get('stash:' + path).callback do |stash_json|
-        if stash_json.nil?
-          status 404
-          body ''
-        else
+        unless stash_json.nil?
           body stash_json
+        else
+          not_found!
         end
       end
     end
@@ -353,13 +423,11 @@ module Sensu
         if stash_exists
           $redis.srem('stashes', path).callback do
             $redis.del('stash:' + path).callback do
-              status 204
-              body ''
+              no_content!
             end
           end
         else
-          status 404
-          body ''
+          not_found!
         end
       end
     end
@@ -373,62 +441,24 @@ module Sensu
     apost '/stashes' do
       begin
         post_body = JSON.parse(request.body.read)
-      rescue JSON::ParserError
-        status 400
-        body ''
-      end
-      response = Hash.new
-      if post_body.is_a?(Array) && post_body.size > 0
-        post_body.each_with_index do |path, index|
-          $redis.get('stash:' + path).callback do |stash_json|
-            unless stash_json.nil?
-              response[path] = JSON.parse(stash_json)
-            end
-            if index == post_body.size - 1
-              body response.to_json
-            end
-          end
-        end
-      else
-        status 400
-        body ''
-      end
-    end
-
-    def self.run_test(options={}, &block)
-      self.setup(options)
-      $settings[:client][:timestamp] = Time.now.to_i
-      $redis.set('client:' + $settings[:client][:name], $settings[:client].to_json).callback do
-        $redis.sadd('clients', $settings[:client][:name]).callback do
-          $redis.hset('events:' + $settings[:client][:name], 'test', {
-            :output => 'CRITICAL',
-            :status => 2,
-            :issued => Time.now.to_i,
-            :flapping => false,
-            :occurrences => 1
-          }.to_json).callback do
-            $redis.set('stash:test/test', {:key => 'value'}.to_json).callback do
-              $redis.sadd('stashes', 'test/test').callback do
-                Thin::Logging.silent = true
-                Thin::Server.start(self, $settings[:api][:port])
-                EM::Timer.new(0.5) do
-                  block.call
-                end
+        if post_body.is_a?(Array) && post_body.size > 0
+          response = Hash.new
+          post_body.each_with_index do |path, index|
+            $redis.get('stash:' + path).callback do |stash_json|
+              unless stash_json.nil?
+                response[path] = JSON.parse(stash_json)
+              end
+              if index == post_body.size - 1
+                body response.to_json
               end
             end
           end
+        else
+          bad_request!
         end
+      rescue JSON::ParserError
+        bad_request!
       end
     end
-
-    def self.stop(signal)
-      $logger.warn('received signal', {
-        :signal => signal
-      })
-      $logger.warn('stopping')
-      $redis.close
-      $logger.warn('stopping reactor')
-      EM::stop_event_loop
-    end
   end
 end

data/lib/sensu/base.rb

@@ -10,7 +10,6 @@ require 'cabin'
 require 'amqp'
 
 require File.join(File.dirname(__FILE__), 'patches', 'ruby')
-require File.join(File.dirname(__FILE__), 'patches', 'amqp')
 
 require File.join(File.dirname(__FILE__), 'constants')
 require File.join(File.dirname(__FILE__), 'cli')

data/lib/sensu/client.rb

@@ -6,18 +6,8 @@ module Sensu
     def self.run(options={})
       client = self.new(options)
       EM::run do
-        client.setup_rabbitmq
-        client.setup_keepalives
-        client.setup_subscriptions
-        client.setup_rabbitmq_monitor
-        client.setup_standalone
-        client.setup_sockets
-
-        %w[INT TERM].each do |signal|
-          Signal.trap(signal) do
-            client.stop(signal)
-          end
-        end
+        client.start
+        client.trap_signals
       end
     end
 
@@ -33,15 +23,20 @@ module Sensu
       @logger.debug('connecting to rabbitmq', {
         :settings => @settings[:rabbitmq]
       })
-      @rabbitmq = AMQP.connect(@settings[:rabbitmq])
-      @rabbitmq.on_disconnect = Proc.new do
+      connection_failure = Proc.new do
         @logger.fatal('cannot connect to rabbitmq', {
           :settings => @settings[:rabbitmq]
         })
         @logger.fatal('SENSU NOT RUNNING!')
         exit 2
       end
+      @rabbitmq = AMQP.connect(@settings[:rabbitmq], :on_tcp_connection_failure => connection_failure)
+      @rabbitmq.on_tcp_connection_loss do |connection, settings|
+        @logger.warn('reconnecting to rabbitmq')
+        connection.reconnect(false, 10)
+      end
       @amq = AMQP::Channel.new(@rabbitmq)
+      @amq.auto_recovery = true
     end
 
     def publish_keepalive
@@ -186,20 +181,6 @@ module Sensu
       end
     end
 
-    def setup_rabbitmq_monitor
-      @logger.debug('monitoring rabbitmq connection')
-      @timers << EM::PeriodicTimer.new(5) do
-        if @rabbitmq.connected?
-          unless @check_request_queue.subscribed?
-            @logger.warn('re-subscribing to client subscriptions')
-            setup_subscriptions
-          end
-        else
-          @logger.warn('reconnecting to rabbitmq')
-        end
-      end
-    end
-
     def setup_standalone
       @logger.debug('scheduling standalone checks')
       standalone_check_count = 0
@@ -238,45 +219,56 @@ module Sensu
     end
 
     def unsubscribe(&block)
-      if @rabbitmq.connected?
-        @logger.warn('unsubscribing from client subscriptions')
-        @check_request_queue.unsubscribe
-      end
-      if block
-        block.call
-      end
+      @logger.warn('unsubscribing from client subscriptions')
+      @check_request_queue.unsubscribe
+      block.call
     end
 
     def complete_checks_in_progress(&block)
       @logger.info('completing checks in progress', {
         :checks_in_progress => @checks_in_progress
       })
-      if block
-        retry_until_true do
-          if @checks_in_progress.empty?
-            block.call
-            true
-          end
+      retry_until_true do
+        if @checks_in_progress.empty?
+          block.call
+          true
         end
       end
     end
 
-    def stop(signal)
-      @logger.warn('received signal', {
-        :signal => signal
-      })
+    def start
+      setup_rabbitmq
+      setup_keepalives
+      setup_subscriptions
+      setup_standalone
+      setup_sockets
+    end
+
+    def stop
       @logger.warn('stopping')
       @timers.each do |timer|
         timer.cancel
       end
       unsubscribe do
         complete_checks_in_progress do
+          @rabbitmq.close
           @logger.warn('stopping reactor')
           EM::stop_event_loop
         end
       end
     end
 
+    def trap_signals
+      %w[INT TERM].each do |signal|
+        Signal.trap(signal) do
+          @logger.warn('received signal', {
+            :signal => signal
+          })
+          stop
+        end
+      end
+    end
+
     private
 
     def testing?

data/lib/sensu/constants.rb

@@ -1,6 +1,6 @@
 module Sensu
   unless defined?(Sensu::VERSION)
-    VERSION = '0.9.7.beta.2'
+    VERSION = '0.9.7.beta.3'
   end
 
   unless defined?(Sensu::DEFAULT_OPTIONS)

data/lib/sensu/logger.rb

@@ -2,6 +2,7 @@ module Sensu
   class Logger
     def initialize(options={})
       @logger = Cabin::Channel.get
+      STDOUT.sync = true
       @logger.subscribe(STDOUT)
       @logger.level = options[:verbose] ? :debug : options[:log_level] || :info
       @log_file = options[:log_file]

data/lib/sensu/redis.rb

@@ -2,13 +2,17 @@ require 'redis'
 
 module Sensu
   class Redis < Redis::Client
-    attr_accessor :host, :port, :password, :on_disconnect
+    attr_accessor :settings, :on_tcp_connection_failure
+
+    alias :em_reconnect :reconnect
 
     def initialize(*arguments)
       super
       @logger = Cabin::Channel.get
+      @settings = Hash.new
       @connection_established = false
       @connected = false
+      @reconnecting = false
       @closing_connection = false
     end
 
@@ -23,8 +27,9 @@ module Sensu
     def connection_completed
       @connection_established = true
       @connected = true
-      if @password
-        auth(@password).callback do |reply|
+      @reconnecting = false
+      if @settings[:password]
+        auth(@settings[:password]).callback do |reply|
           unless reply == 'OK'
             @logger.fatal('redis authentication failed')
             close_connection
@@ -41,9 +46,14 @@ module Sensu
       setup_heartbeat
     end
 
-    def reconnect!
-      EM::Timer.new(1) do
-        reconnect(@host, @port)
+    def reconnect(immediate=false, wait=10)
+      if @reconnecting && !immediate
+        EM::Timer.new(wait) do
+          em_reconnect(@settings[:host], @settings[:port])
+        end
+      else
+        @reconnecting = true
+        em_reconnect(@settings[:host], @settings[:port])
       end
     end
 
@@ -52,23 +62,33 @@ module Sensu
       close_connection
     end
 
+    def on_tcp_connection_loss(&block)
+      if block.respond_to?(:call)
+        @on_tcp_connection_loss = block
+      end
+    end
+
     def unbind
       @connected = false
       super
-      if @on_disconnect && !@closing_connection
-        @on_disconnect.call
+      unless @closing_connection
+        if @connection_established
+          if @on_tcp_connection_loss
+            @on_tcp_connection_loss.call(self, @settings)
+          end
+        else
+          if @on_tcp_connection_failure
+            @on_tcp_connection_failure.call(self, @settings)
+          end
+        end
       end
     end
 
-    def connection_established?
-      @connection_established
-    end
-
     def connected?
       @connected
     end
 
-    def self.connect(options)
+    def self.connect(options, additional={})
       options ||= Hash.new
       if options.is_a?(String)
         begin
@@ -86,11 +106,17 @@ module Sensu
         port = options[:port] || 6379
         password = options[:password]
       end
-      EM::connect(host, port, self) do |redis|
-        redis.host = host
-        redis.port = port
-        redis.password = password
+      connection = EM::connect(host, port, self) do |redis|
+        redis.settings = {
+          :host => host,
+          :port => port,
+          :password => password
+        }
+      end
+      if additional[:on_tcp_connection_failure].respond_to?(:call)
+        connection.on_tcp_connection_failure = additional[:on_tcp_connection_failure]
       end
+      connection
     end
   end
 end

data/lib/sensu/server.rb

@@ -8,18 +8,8 @@ module Sensu
     def self.run(options={})
       server = self.new(options)
       EM::run do
-        server.setup_redis
-        server.setup_rabbitmq
-        server.setup_keepalives
-        server.setup_results
-        server.setup_master_monitor
-        server.setup_rabbitmq_monitor
-
-        %w[INT TERM].each do |signal|
-          Signal.trap(signal) do
-            server.stop(signal)
-          end
-        end
+        server.start
+        server.trap_signals
       end
     end
 
@@ -28,26 +18,30 @@ module Sensu
       base = Sensu::Base.new(options)
       @settings = base.settings
       @timers = Array.new
+      @master_timers = Array.new
       @handlers_in_progress_count = 0
+      @is_master = false
     end
 
     def setup_redis
       @logger.debug('connecting to redis', {
         :settings => @settings[:redis]
       })
-      @redis = Sensu::Redis.connect(@settings[:redis])
-      unless testing?
-        @redis.on_disconnect = Proc.new do
-          if @redis.connection_established?
-            @logger.fatal('redis connection closed')
-            stop('TERM')
-          else
-            @logger.fatal('cannot connect to redis', {
-              :settings => @settings[:redis]
-            })
-            @logger.fatal('SENSU NOT RUNNING!')
-            exit 2
-          end
+      connection_failure = Proc.new do
+        @logger.fatal('cannot connect to redis', {
+          :settings => @settings[:redis]
+        })
+        @logger.fatal('SENSU NOT RUNNING!')
+        if @rabbitmq
+          @rabbitmq.close
+        end
+        exit 2
+      end
+      @redis = Sensu::Redis.connect(@settings[:redis], :on_tcp_connection_failure => connection_failure)
+      @redis.on_tcp_connection_loss do
+        unless testing?
+          @logger.fatal('redis connection closed')
+          stop
         end
       end
     end
@@ -56,8 +50,7 @@ module Sensu
       @logger.debug('connecting to rabbitmq', {
         :settings => @settings[:rabbitmq]
       })
-      @rabbitmq = AMQP.connect(@settings[:rabbitmq])
-      @rabbitmq.on_disconnect = Proc.new do
+      connection_failure = Proc.new do
         @logger.fatal('cannot connect to rabbitmq', {
           :settings => @settings[:rabbitmq]
         })
@@ -65,7 +58,15 @@ module Sensu
         @redis.close
         exit 2
       end
+      @rabbitmq = AMQP.connect(@settings[:rabbitmq], :on_tcp_connection_failure => connection_failure)
+      @rabbitmq.on_tcp_connection_loss do |connection, settings|
+        @logger.warn('reconnecting to rabbitmq')
+        resign_as_master do
+          connection.reconnect(false, 10)
+        end
+      end
       @amq = AMQP::Channel.new(@rabbitmq)
+      @amq.auto_recovery = true
     end
 
     def setup_keepalives
@@ -86,16 +87,16 @@ module Sensu
       subdue = false
       if check[:subdue].is_a?(Hash)
         if check[:subdue].has_key?(:start) && check[:subdue].has_key?(:end)
-          start = Time.parse(check[:subdue][:start])
-          stop = Time.parse(check[:subdue][:end])
-          if stop < start
-            if Time.now < stop
-              start = Time.parse('12:00:00 AM')
+          start_time = Time.parse(check[:subdue][:start])
+          end_time = Time.parse(check[:subdue][:end])
+          if end_time < start_time
+            if Time.now < end_time
+              start_time = Time.parse('12:00:00 AM')
             else
-              stop = Time.parse('11:59:59 PM')
+              end_time = Time.parse('11:59:59 PM')
             end
           end
-          if Time.now >= start && Time.now <= stop
+          if Time.now >= start_time && Time.now <= end_time
             subdue = true
           end
         end
@@ -422,22 +423,20 @@ module Sensu
       @settings.checks.each do |check|
         unless check[:publish] == false || check[:standalone]
           check_count += 1
-          @timers << EM::Timer.new(stagger * check_count) do
+          @master_timers << EM::Timer.new(stagger * check_count) do
             interval = testing? ? 0.5 : check[:interval]
-            @timers << EM::PeriodicTimer.new(interval) do
+            @master_timers << EM::PeriodicTimer.new(interval) do
               unless check_subdued?(check, :publisher)
-                if @rabbitmq.connected?
-                  payload = {
-                    :name => check[:name],
-                    :issued => Time.now.to_i
-                  }
-                  @logger.info('publishing check request', {
-                    :payload => payload,
-                    :subscribers => check[:subscribers]
-                  })
-                  check[:subscribers].uniq.each do |exchange_name|
-                    @amq.fanout(exchange_name).publish(payload.to_json)
-                  end
+                payload = {
+                  :name => check[:name],
+                  :issued => Time.now.to_i
+                }
+                @logger.info('publishing check request', {
+                  :payload => payload,
+                  :subscribers => check[:subscribers]
+                })
+                check[:subscribers].uniq.each do |exchange_name|
+                  @amq.fanout(exchange_name).publish(payload.to_json)
                 end
               end
             end
@@ -459,34 +458,32 @@ module Sensu
 
     def setup_keepalive_monitor
       @logger.debug('monitoring client keepalives')
-      @timers << EM::PeriodicTimer.new(30) do
-        if @rabbitmq.connected?
-          @logger.debug('checking for stale client info')
-          @redis.smembers('clients').callback do |clients|
-            clients.each do |client_name|
-              @redis.get('client:' + client_name).callback do |client_json|
-                client = JSON.parse(client_json, :symbolize_names => true)
-                check = {
-                  :name => 'keepalive',
-                  :issued => Time.now.to_i
-                }
-                time_since_last_keepalive = Time.now.to_i - client[:timestamp]
-                case
-                when time_since_last_keepalive >= 180
-                  check[:output] = 'No keep-alive sent from client in over 180 seconds'
-                  check[:status] = 2
-                  publish_result(client, check)
-                when time_since_last_keepalive >= 120
-                  check[:output] = 'No keep-alive sent from client in over 120 seconds'
-                  check[:status] = 1
-                  publish_result(client, check)
-                else
-                  @redis.hexists('events:' + client[:name], 'keepalive').callback do |exists|
-                    if exists
-                      check[:output] = 'Keep-alive sent from client'
-                      check[:status] = 0
-                      publish_result(client, check)
-                    end
+      @master_timers << EM::PeriodicTimer.new(30) do
+        @logger.debug('checking for stale client info')
+        @redis.smembers('clients').callback do |clients|
+          clients.each do |client_name|
+            @redis.get('client:' + client_name).callback do |client_json|
+              client = JSON.parse(client_json, :symbolize_names => true)
+              check = {
+                :name => 'keepalive',
+                :issued => Time.now.to_i
+              }
+              time_since_last_keepalive = Time.now.to_i - client[:timestamp]
+              case
+              when time_since_last_keepalive >= 180
+                check[:output] = 'No keep-alive sent from client in over 180 seconds'
+                check[:status] = 2
+                publish_result(client, check)
+              when time_since_last_keepalive >= 120
+                check[:output] = 'No keep-alive sent from client in over 120 seconds'
+                check[:status] = 1
+                publish_result(client, check)
+              else
+                @redis.hexists('events:' + client[:name], 'keepalive').callback do |exists|
+                  if exists
+                    check[:output] = 'Keep-alive sent from client'
+                    check[:status] = 0
+                    publish_result(client, check)
                   end
                 end
               end
@@ -502,7 +499,6 @@ module Sensu
     end
 
     def request_master_election
-      @is_master ||= false
       @redis.setnx('lock:master', Time.now.to_i).callback do |created|
         if created
           @is_master = true
@@ -524,32 +520,6 @@ module Sensu
       end
     end
 
-    def resign_as_master(&block)
-      if @redis.connected? && @is_master
-        @redis.del('lock:master').callback do
-          @logger.warn('resigned as master')
-          @is_master = false
-        end
-        if block
-          timestamp = Time.now.to_i
-          retry_until_true do
-            if !@is_master
-              block.call
-              true
-            elsif Time.now.to_i - timestamp >= 5
-              @logger.warn('failed to resign as master')
-              block.call
-              true
-            end
-          end
-        end
-      else
-        if block
-          block.call
-        end
-      end
-    end
-
     def setup_master_monitor
       request_master_election
       @timers << EM::PeriodicTimer.new(20) do
@@ -557,53 +527,59 @@ module Sensu
           @redis.set('lock:master', Time.now.to_i).callback do
             @logger.debug('updated master lock timestamp')
           end
-        else
+        elsif @rabbitmq.connected?
           request_master_election
         end
       end
     end
 
-    def setup_rabbitmq_monitor
-      @logger.debug('monitoring rabbitmq connection')
-      @timers << EM::PeriodicTimer.new(5) do
-        if @rabbitmq.connected?
-          unless @keepalive_queue.subscribed?
-            @logger.warn('re-subscribing to keepalives')
-            setup_keepalives
-          end
-          unless @result_queue.subscribed?
-            @logger.warn('re-subscribing to results')
-            setup_results
+    def resign_as_master(&block)
+      if @is_master
+        @logger.warn('resigning as master')
+        @master_timers.each do |timer|
+          timer.cancel
+        end
+        @master_timers = Array.new
+        @redis.del('lock:master').callback do
+          @logger.info('removed master lock')
+          @is_master = false
+        end
+        timestamp = Time.now.to_i
+        retry_until_true do
+          if !@is_master
+            block.call
+            true
+          elsif !@redis.connected? || Time.now.to_i - timestamp >= 5
+            @logger.warn('failed to remove master lock')
+            @is_master = false
+            block.call
+            true
           end
-        else
-          @logger.warn('reconnecting to rabbitmq')
         end
+      else
+        @logger.debug('not currently master')
+        block.call
       end
     end
 
     def unsubscribe(&block)
+      @logger.warn('unsubscribing from keepalive and result queues')
+      @keepalive_queue.unsubscribe
+      @result_queue.unsubscribe
       if @rabbitmq.connected?
-        @logger.warn('unsubscribing from keepalives')
-        @keepalive_queue.unsubscribe
-        @logger.warn('unsubscribing from results')
-        @result_queue.unsubscribe
-        if block
-          timestamp = Time.now.to_i
-          retry_until_true do
-            if !@keepalive_queue.subscribed? && !@result_queue.subscribed?
-              block.call
-              true
-            elsif Time.now.to_i - timestamp >= 5
-              @logger.warn('failed to unsubscribe from keepalives and results')
-              block.call
-              true
-            end
+        timestamp = Time.now.to_i
+        retry_until_true do
+          if !@keepalive_queue.subscribed? && !@result_queue.subscribed?
+            block.call
+            true
+          elsif Time.now.to_i - timestamp >= 5
+            @logger.warn('failed to unsubscribe from keepalive and result queues')
+            block.call
+            true
           end
         end
       else
-        if block
-          block.call
-        end
+        block.call
       end
     end
 
@@ -611,20 +587,23 @@ module Sensu
       @logger.info('completing handlers in progress', {
         :handlers_in_progress_count => @handlers_in_progress_count
       })
-      if block
-        retry_until_true do
-          if @handlers_in_progress_count == 0
-            block.call
-            true
-          end
+      retry_until_true do
+        if @handlers_in_progress_count == 0
+          block.call
+          true
         end
       end
     end
 
-    def stop(signal)
-      @logger.warn('received signal', {
-        :signal => signal
-      })
+    def start
+      setup_redis
+      setup_rabbitmq
+      setup_keepalives
+      setup_results
+      setup_master_monitor
+    end
+
+    def stop
       @logger.warn('stopping')
       @timers.each do |timer|
         timer.cancel
@@ -632,6 +611,7 @@ module Sensu
       unsubscribe do
         resign_as_master do
           complete_handlers_in_progress do
+            @rabbitmq.close
             @redis.close
             @logger.warn('stopping reactor')
             EM::stop_event_loop
@@ -640,6 +620,17 @@ module Sensu
       end
     end
 
+    def trap_signals
+      %w[INT TERM].each do |signal|
+        Signal.trap(signal) do
+          @logger.warn('received signal', {
+            :signal => signal
+          })
+          stop
+        end
+      end
+    end
+
     private
 
     def testing?

data/sensu.gemspec

@@ -13,8 +13,8 @@ Gem::Specification.new do |s|
   s.has_rdoc    = false
 
   s.add_dependency('eventmachine', '1.0.0.rc.4')
-  s.add_dependency('amqp', '0.7.4')
-  s.add_dependency('json', '1.7.3')
+  s.add_dependency('amqp', '0.9.7')
+  s.add_dependency('json')
   s.add_dependency('cabin', '0.4.4')
   s.add_dependency('ruby-redis', '0.0.2')
   s.add_dependency('async_sinatra', '1.0.0')

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification 
 name: sensu
 version: !ruby/object:Gem::Version 
-  hash: 62196239
+  hash: 62196237
   prerelease: true
   segments: 
   - 0
   - 9
   - 7
   - beta
-  - 2
-  version: 0.9.7.beta.2
+  - 3
+  version: 0.9.7.beta.3
 platform: ruby
 authors: 
 - Sean Porter
@@ -18,7 +18,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-08-10 00:00:00 -07:00
+date: 2012-08-25 00:00:00 -07:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -47,12 +47,12 @@ dependencies:
     requirements: 
     - - "="
       - !ruby/object:Gem::Version 
-        hash: 11
+        hash: 53
         segments: 
         - 0
+        - 9
         - 7
-        - 4
-        version: 0.7.4
+        version: 0.9.7
   type: :runtime
   version_requirements: *id002
 - !ruby/object:Gem::Dependency 
@@ -61,14 +61,12 @@ dependencies:
   requirement: &id003 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
-    - - "="
+    - - ">="
       - !ruby/object:Gem::Version 
-        hash: 13
+        hash: 3
         segments: 
-        - 1
-        - 7
-        - 3
-        version: 1.7.3
+        - 0
+        version: "0"
   type: :runtime
   version_requirements: *id003
 - !ruby/object:Gem::Dependency 
@@ -199,7 +197,6 @@ files:
 - lib/sensu/client.rb
 - lib/sensu/constants.rb
 - lib/sensu/logger.rb
-- lib/sensu/patches/amqp.rb
 - lib/sensu/patches/ruby.rb
 - lib/sensu/process.rb
 - lib/sensu/redis.rb

data/lib/sensu/patches/amqp.rb

@@ -1,5 +0,0 @@
-module AMQP
-  module Client
-    attr_accessor :on_disconnect
-  end
-end