sensu 0.9.9.beta.2 → 0.9.9.beta.3

data/lib/sensu/server.rb

@@ -4,7 +4,9 @@ require File.join(File.dirname(__FILE__), 'socket')
 
 module Sensu
   class Server
-    attr_reader :redis, :amq, :is_master
+    include Utilities
+
+    attr_reader :is_master
 
     def self.run(options={})
       server = self.new(options)
@@ -15,9 +17,11 @@ module Sensu
     end
 
     def initialize(options={})
-      @logger = Sensu::Logger.get
-      base = Sensu::Base.new(options)
+      base = Base.new(options)
+      @logger = base.logger
       @settings = base.settings
+      @extensions = base.extensions
+      base.setup_process
       @timers = Array.new
       @master_timers = Array.new
       @handlers_in_progress_count = 0
@@ -28,65 +32,45 @@ module Sensu
       @logger.debug('connecting to redis', {
         :settings => @settings[:redis]
       })
-      connection_failure = Proc.new do
-        @logger.fatal('cannot connect to redis', {
-          :settings => @settings[:redis]
+      @redis = Redis.connect(@settings[:redis])
+      @redis.on_error do |error|
+        @logger.fatal('redis connection error', {
+          :error => error.to_s
         })
-        @logger.fatal('SENSU NOT RUNNING!')
-        if @rabbitmq
-          @rabbitmq.close
-        end
-        exit 2
+        stop
       end
-      @redis = Sensu::Redis.connect(@settings[:redis], :on_tcp_connection_failure => connection_failure)
-      @redis.on_tcp_connection_loss do
+      @redis.before_reconnect do
+        @logger.warn('reconnecting to redis')
         unless testing?
-          @logger.fatal('redis connection closed')
-          stop
+          pause
         end
       end
+      @redis.after_reconnect do
+        @logger.info('reconnected to redis')
+        resume
+      end
     end
 
     def setup_rabbitmq
       @logger.debug('connecting to rabbitmq', {
         :settings => @settings[:rabbitmq]
       })
-      connection_failure = Proc.new do
-        @logger.fatal('cannot connect to rabbitmq', {
-          :settings => @settings[:rabbitmq]
+      @rabbitmq = RabbitMQ.connect(@settings[:rabbitmq])
+      @rabbitmq.on_error do |error|
+        @logger.fatal('rabbitmq connection error', {
+          :error => error.to_s
         })
-        @logger.fatal('SENSU NOT RUNNING!')
-        @redis.close
-        exit 2
+        stop
       end
-      @rabbitmq = AMQP.connect(@settings[:rabbitmq], {
-        :on_tcp_connection_failure => connection_failure,
-        :on_possible_authentication_failure => connection_failure
-      })
-      @rabbitmq.logger = Sensu::NullLogger.get
-      @rabbitmq.on_tcp_connection_loss do |connection, settings|
-        unless connection.reconnecting?
-          @logger.warn('reconnecting to rabbitmq')
-          resign_as_master do
-            connection.periodically_reconnect(5)
-          end
-        end
+      @rabbitmq.before_reconnect do
+        @logger.warn('reconnecting to rabbitmq')
+        resign_as_master
       end
-      @rabbitmq.on_skipped_heartbeats do
-        @logger.warn('skipped rabbitmq heartbeat')
+      @rabbitmq.after_reconnect do
+        @logger.info('reconnected to rabbitmq')
       end
-      @amq = AMQP::Channel.new(@rabbitmq)
-      @amq.auto_recovery = true
+      @amq = @rabbitmq.channel
       @amq.prefetch(1)
-      @amq.on_error do |channel, channel_close|
-        @logger.fatal('rabbitmq channel closed', {
-          :error => {
-            :reply_code => channel_close.reply_code,
-            :reply_text => channel_close.reply_text
-          }
-        })
-        stop
-      end
     end
 
     def setup_keepalives
@@ -97,8 +81,8 @@ module Sensu
         @logger.debug('received keepalive', {
           :client => client
         })
-        @redis.set('client:' + client[:name], client.to_json).callback do
-          @redis.sadd('clients', client[:name]).callback do
+        @redis.set('client:' + client[:name], client.to_json) do
+          @redis.sadd('clients', client[:name]) do
             header.ack
           end
         end
@@ -155,7 +139,7 @@ module Sensu
     def derive_handlers(handler_list, nested=false)
       handler_list.inject(Array.new) do |handlers, handler_name|
         if @settings.handler_exists?(handler_name)
-          handler = @settings[:handlers][handler_name]
+          handler = @settings[:handlers][handler_name].merge(:name => handler_name)
           if handler[:type] == 'set'
             unless nested
               handlers = handlers + derive_handlers(handler[:handlers], true)
@@ -167,6 +151,9 @@ module Sensu
           else
             handlers.push(handler)
           end
+        elsif @extensions.handler_exists?(handler_name)
+          handler = @extensions[:handlers][handler_name]
+          handlers.push(handler)
         else
           @logger.error('unknown handler', {
             :handler => {
@@ -181,7 +168,7 @@ module Sensu
     def event_handlers(event)
       handler_list = Array((event[:check][:handlers] || event[:check][:handler]) || 'default')
       handlers = derive_handlers(handler_list)
-      event_severity = Sensu::SEVERITIES[event[:check][:status]] || 'unknown'
+      event_severity = SEVERITIES[event[:check][:status]] || 'unknown'
       handlers.select do |handler|
         if event[:action] == :flapping && !handler[:handle_flapping]
           @logger.info('handler does not handle flapping events', {
@@ -233,7 +220,7 @@ module Sensu
       end
       execute = Proc.new do
         begin
-          output, status = Sensu::IO.popen(command, 'r+') do |child|
+          output, status = IO.popen(command, 'r+') do |child|
             unless data.nil?
               child.write(data.to_s)
             end
@@ -254,41 +241,39 @@ module Sensu
     end
 
     def mutate_event_data(mutator_name, event, &block)
-      case mutator_name
-      when nil
+      on_error = Proc.new do |error|
+        @logger.error('mutator error', {
+          :event => event,
+          :mutator => mutator,
+          :error => error.to_s
+        })
+      end
+      case
+      when mutator_name.nil?
         block.call(event.to_json)
-      when /^only_check_output/
-        mutated = case mutator_name
-        when /split$/
-          event[:check][:output].split(/\n+/)
-        else
-          event[:check][:output]
-        end
-        block.call(mutated)
-      else
-        if @settings.mutator_exists?(mutator_name)
-          mutator = @settings[:mutators][mutator_name]
-          on_error = Proc.new do |error|
-            @logger.error('mutator error', {
-              :event => event,
-              :mutator => mutator,
-              :error => error.to_s
-            })
+      when @settings.mutator_exists?(mutator_name)
+        mutator = @settings[:mutators][mutator_name]
+        execute_command(mutator[:command], event.to_json, on_error) do |output, status|
+          if status == 0
+            block.call(output)
+          else
+            on_error.call('non-zero exit status (' + status + '): ' + output)
           end
-          execute_command(mutator[:command], event.to_json, on_error) do |output, status|
-            if status == 0
-              block.call(output)
-            else
-              on_error.call('non-zero exit status (' + status + '): ' + output)
-            end
+        end
+      when @extensions.mutator_exists?(mutator_name)
+        @extensions[:mutators][mutator_name].run(event) do |output, status|
+          if status == 0
+            block.call(output)
+          else
+            on_error.call('non-zero exit status (' + status + '): ' + output)
           end
-        else
-          @logger.error('unknown mutator', {
-            :mutator => {
-              :name => mutator_name
-            }
-          })
         end
+      else
+        @logger.error('unknown mutator', {
+          :mutator => {
+            :name => mutator_name
+          }
+        })
       end
     end
 
@@ -320,7 +305,7 @@ module Sensu
             end
           when 'tcp'
             begin
-              EM::connect(handler[:socket][:host], handler[:socket][:port], Sensu::SocketHandler) do |socket|
+              EM::connect(handler[:socket][:host], handler[:socket][:port], SocketHandler) do |socket|
                 socket.on_success = Proc.new do
                   @handlers_in_progress_count -= 1
                 end
@@ -350,13 +335,17 @@ module Sensu
             exchange_options = handler[:exchange].reject do |key, value|
               [:name, :type].include?(key)
             end
-            payloads = Array(event_data)
-            payloads.each do |payload|
-              unless payload.empty?
-                @amq.method(exchange_type).call(exchange_name, exchange_options).publish(payload)
-              end
+            unless event_data.empty?
+              @amq.method(exchange_type).call(exchange_name, exchange_options).publish(event_data)
             end
             @handlers_in_progress_count -= 1
+          when 'extension'
+            handler.run(event_data) do |output, status|
+              output.split(/\n+/).each do |line|
+                @logger.info(line)
+              end
+              @handlers_in_progress_count -= 1
+            end
           end
         end
       end
@@ -371,15 +360,14 @@ module Sensu
       @redis.hset('aggregation:' + result_set, result[:client], {
         :output => check[:output],
         :status => check[:status]
-      }.to_json).callback do
-        statuses = Sensu::SEVERITIES
-        statuses.each do |status|
-          @redis.hsetnx('aggregate:' + result_set, status, 0)
-        end
-        status = (statuses[check[:status]] || 'unknown')
-        @redis.hincrby('aggregate:' + result_set, status, 1).callback do
-          @redis.hincrby('aggregate:' + result_set, 'total', 1).callback do
-            @redis.sadd('aggregates:' + check[:name], check[:issued]).callback do
+      }.to_json) do
+        SEVERITIES.each do |severity|
+          @redis.hsetnx('aggregate:' + result_set, severity, 0)
+        end
+        severity = (SEVERITIES[check[:status]] || 'unknown')
+        @redis.hincrby('aggregate:' + result_set, severity, 1) do
+          @redis.hincrby('aggregate:' + result_set, 'total', 1) do
+            @redis.sadd('aggregates:' + check[:name], check[:issued]) do
               @redis.sadd('aggregates', check[:name])
             end
           end
@@ -391,7 +379,7 @@ module Sensu
       @logger.debug('processing result', {
         :result => result
       })
-      @redis.get('client:' + result[:client]).callback do |client_json|
+      @redis.get('client:' + result[:client]) do |client_json|
         unless client_json.nil?
           client = JSON.parse(client_json, :symbolize_names => true)
           check = case
@@ -405,11 +393,11 @@ module Sensu
           end
           @redis.sadd('history:' + client[:name], check[:name])
           history_key = 'history:' + client[:name] + ':' + check[:name]
-          @redis.rpush(history_key, check[:status]).callback do
-            @redis.lrange(history_key, -21, -1).callback do |history|
+          @redis.rpush(history_key, check[:status]) do
+            @redis.lrange(history_key, -21, -1) do |history|
               check[:history] = history
               total_state_change = 0
-              unless history.count < 21
+              unless history.size < 21
                 state_changes = 0
                 change_weight = 0.8
                 previous_status = history.first
@@ -423,7 +411,7 @@ module Sensu
                 total_state_change = (state_changes.fdiv(20) * 100).to_i
                 @redis.ltrim(history_key, -21, -1)
               end
-              @redis.hget('events:' + client[:name], check[:name]).callback do |event_json|
+              @redis.hget('events:' + client[:name], check[:name]) do |event_json|
                 previous_occurrence = event_json ? JSON.parse(event_json, :symbolize_names => true) : false
                 is_flapping = false
                 if check.has_key?(:low_flap_threshold) && check.has_key?(:high_flap_threshold)
@@ -453,7 +441,7 @@ module Sensu
                     :handlers => Array((check[:handlers] || check[:handler]) || 'default'),
                     :flapping => is_flapping,
                     :occurrences => event[:occurrences]
-                  }.to_json).callback do
+                  }.to_json) do
                     unless check[:handle] == false
                       event[:action] = is_flapping ? :flapping : :create
                       handle_event(event)
@@ -461,7 +449,7 @@ module Sensu
                   end
                 elsif previous_occurrence
                   unless check[:auto_resolve] == false && !check[:force_resolve]
-                    @redis.hdel('events:' + client[:name], check[:name]).callback do
+                    @redis.hdel('events:' + client[:name], check[:name]) do
                       unless check[:handle] == false
                         event[:occurrences] = previous_occurrence[:occurrences]
                         event[:action] = :resolve
@@ -515,8 +503,9 @@ module Sensu
       stagger = testing? ? 0 : 2
       @settings.checks.each do |check|
         unless check[:publish] == false || check[:standalone]
-          check_count = (check_count + 1) % 30
-          @master_timers << EM::Timer.new(stagger * check_count) do
+          check_count += 1
+          scheduling_delay = stagger * check_count % 30
+          @master_timers << EM::Timer.new(scheduling_delay) do
             interval = testing? ? 0.5 : check[:interval]
             @master_timers << EM::PeriodicTimer.new(interval) do
               unless check_subdued?(check, :publisher)
@@ -539,34 +528,32 @@ module Sensu
       @amq.queue('results').publish(payload.to_json)
     end
 
-    def setup_keepalive_monitor
-      @logger.debug('monitoring client keepalives')
-      @master_timers << EM::PeriodicTimer.new(30) do
-        @redis.smembers('clients').callback do |clients|
-          clients.each do |client_name|
-            @redis.get('client:' + client_name).callback do |client_json|
-              client = JSON.parse(client_json, :symbolize_names => true)
-              check = {
-                :name => 'keepalive',
-                :issued => Time.now.to_i
-              }
-              time_since_last_keepalive = Time.now.to_i - client[:timestamp]
-              case
-              when time_since_last_keepalive >= 180
-                check[:output] = 'No keep-alive sent from client in over 180 seconds'
-                check[:status] = 2
-                publish_result(client, check)
-              when time_since_last_keepalive >= 120
-                check[:output] = 'No keep-alive sent from client in over 120 seconds'
-                check[:status] = 1
-                publish_result(client, check)
-              else
-                @redis.hexists('events:' + client[:name], 'keepalive').callback do |exists|
-                  if exists
-                    check[:output] = 'Keep-alive sent from client'
-                    check[:status] = 0
-                    publish_result(client, check)
-                  end
+    def determine_stale_clients
+      @logger.info('determining stale clients')
+      @redis.smembers('clients') do |clients|
+        clients.each do |client_name|
+          @redis.get('client:' + client_name) do |client_json|
+            client = JSON.parse(client_json, :symbolize_names => true)
+            check = {
+              :name => 'keepalive',
+              :issued => Time.now.to_i
+            }
+            time_since_last_keepalive = Time.now.to_i - client[:timestamp]
+            case
+            when time_since_last_keepalive >= 180
+              check[:output] = 'No keep-alive sent from client in over 180 seconds'
+              check[:status] = 2
+              publish_result(client, check)
+            when time_since_last_keepalive >= 120
+              check[:output] = 'No keep-alive sent from client in over 120 seconds'
+              check[:status] = 1
+              publish_result(client, check)
+            else
+              @redis.hexists('events:' + client[:name], 'keepalive') do |exists|
+                if exists
+                  check[:output] = 'Keep-alive sent from client'
+                  check[:status] = 0
+                  publish_result(client, check)
                 end
               end
             end
@@ -575,19 +562,25 @@ module Sensu
       end
     end
 
-    def setup_aggregation_pruner
-      @logger.debug('pruning aggregations')
-      @master_timers << EM::PeriodicTimer.new(20) do
-        @redis.smembers('aggregates').callback do |checks|
-          checks.each do |check_name|
-            @redis.smembers('aggregates:' + check_name).callback do |aggregates|
+    def setup_client_monitor
+      @logger.debug('monitoring clients')
+      @master_timers << EM::PeriodicTimer.new(30) do
+        determine_stale_clients
+      end
+    end
+
+    def prune_aggregations
+      @logger.info('pruning aggregations')
+      @redis.smembers('aggregates') do |checks|
+        checks.each do |check_name|
+          @redis.smembers('aggregates:' + check_name) do |aggregates|
+            if aggregates.size > 20
               aggregates.sort!
-              until aggregates.size <= 20
-                check_issued = aggregates.shift
-                @redis.srem('aggregates:' + check_name, check_issued).callback do
+              aggregates.take(aggregates.size - 20).each do |check_issued|
+                @redis.srem('aggregates:' + check_name, check_issued) do
                   result_set = check_name + ':' + check_issued.to_s
-                  @redis.del('aggregate:' + result_set).callback do
-                    @redis.del('aggregation:' + result_set).callback do
+                  @redis.del('aggregate:' + result_set) do
+                    @redis.del('aggregation:' + result_set) do
                       @logger.debug('pruned aggregation', {
                         :check => {
                           :name => check_name,
@@ -604,22 +597,29 @@ module Sensu
       end
     end
 
+    def setup_aggregation_pruner
+      @logger.debug('pruning aggregations')
+      @master_timers << EM::PeriodicTimer.new(20) do
+        prune_aggregations
+      end
+    end
+
     def master_duties
       setup_publisher
-      setup_keepalive_monitor
+      setup_client_monitor
       setup_aggregation_pruner
     end
 
     def request_master_election
-      @redis.setnx('lock:master', Time.now.to_i).callback do |created|
+      @redis.setnx('lock:master', Time.now.to_i) do |created|
         if created
           @is_master = true
           @logger.info('i am the master')
           master_duties
         else
-          @redis.get('lock:master').callback do |timestamp|
+          @redis.get('lock:master') do |timestamp|
             if Time.now.to_i - timestamp.to_i >= 60
-              @redis.getset('lock:master', Time.now.to_i).callback do |previous|
+              @redis.getset('lock:master', Time.now.to_i) do |previous|
                 if previous == timestamp
                   @is_master = true
                   @logger.info('i am now the master')
@@ -636,7 +636,7 @@ module Sensu
       request_master_election
       @timers << EM::PeriodicTimer.new(20) do
         if @is_master
-          @redis.set('lock:master', Time.now.to_i).callback do
+          @redis.set('lock:master', Time.now.to_i) do
             @logger.debug('updated master lock timestamp')
           end
         elsif @rabbitmq.connected?
@@ -646,22 +646,25 @@ module Sensu
     end
 
     def resign_as_master(&block)
+      block ||= Proc.new {}
       if @is_master
         @logger.warn('resigning as master')
         @master_timers.each do |timer|
           timer.cancel
         end
         @master_timers = Array.new
-        @redis.del('lock:master').callback do
-          @logger.info('removed master lock')
-          @is_master = false
+        if @redis.connected?
+          @redis.del('lock:master') do
+            @logger.info('removed master lock')
+            @is_master = false
+          end
         end
         timestamp = Time.now.to_i
         retry_until_true do
           if !@is_master
             block.call
             true
-          elsif !@redis.connected? || Time.now.to_i - timestamp >= 5
+          elsif Time.now.to_i - timestamp >= 3
             @logger.warn('failed to remove master lock')
             @is_master = false
             block.call
@@ -707,66 +710,68 @@ module Sensu
       end
     end
 
-    def start
-      setup_redis
-      setup_rabbitmq
+    def bootstrap
       setup_keepalives
       setup_results
       setup_master_monitor
+      @state = :running
     end
 
-    def stop
-      @logger.warn('stopping')
-      @timers.each do |timer|
-        timer.cancel
-      end
-      unsubscribe do
-        resign_as_master do
-          complete_handlers_in_progress do
-            @rabbitmq.close
-            @redis.close
-            @logger.warn('stopping reactor')
-            EM::stop_event_loop
+    def start
+      setup_redis
+      setup_rabbitmq
+      bootstrap
+    end
+
+    def pause(&block)
+      unless @state == :pausing || @state == :paused
+        @state = :pausing
+        @timers.each do |timer|
+          timer.cancel
+        end
+        @timers = Array.new
+        unsubscribe do
+          resign_as_master do
+            @state = :paused
+            if block
+              block.call
+            end
           end
         end
       end
     end
 
-    def trap_signals
-      %w[INT TERM].each do |signal|
-        Signal.trap(signal) do
-          @logger.warn('received signal', {
-            :signal => signal
-          })
-          stop
+    def resume
+      retry_until_true(1) do
+        if @state == :paused
+          if @redis.connected? && @rabbitmq.connected?
+            bootstrap
+            true
+          end
         end
       end
     end
 
-    private
-
-    def testing?
-      File.basename($0) == 'rake'
-    end
-
-    def retry_until_true(wait=0.5, &block)
-      EM::Timer.new(wait) do
-        unless block.call
-          retry_until_true(wait, &block)
+    def stop
+      @logger.warn('stopping')
+      @state = :stopping
+      pause do
+        complete_handlers_in_progress do
+          @redis.close
+          @rabbitmq.close
+          @logger.warn('stopping reactor')
+          EM::stop_event_loop
         end
       end
     end
 
-    def hash_values_equal?(hash_one, hash_two)
-      hash_one.keys.all? do |key|
-        if hash_one[key] == hash_two[key]
-          true
-        else
-          if hash_one[key].is_a?(Hash) && hash_two[key].is_a?(Hash)
-            hash_values_equal?(hash_one[key], hash_two[key])
-          else
-            false
-          end
+    def trap_signals
+      %w[INT TERM].each do |signal|
+        Signal.trap(signal) do
+          @logger.warn('received signal', {
+            :signal => signal
+          })
+          stop
         end
       end
     end