sensu 0.12.6 → 0.13.0.alpha

data/lib/sensu/server.rb

@@ -1,11 +1,10 @@
-require File.join(File.dirname(__FILE__), 'base')
-require File.join(File.dirname(__FILE__), 'redis')
-require File.join(File.dirname(__FILE__), 'socket')
-require File.join(File.dirname(__FILE__), 'sandbox')
+require 'sensu/daemon'
+require 'sensu/socket'
+require 'sensu/sandbox'
 
 module Sensu
   class Server
-    include Utilities
+    include Daemon
 
     attr_reader :is_master
 
@@ -13,84 +12,27 @@ module Sensu
       server = self.new(options)
       EM::run do
         server.start
-        server.trap_signals
+        server.setup_signal_traps
       end
     end
 
     def initialize(options={})
-      base = Base.new(options)
-      @logger = base.logger
-      @settings = base.settings
-      @extensions = base.extensions
-      base.setup_process
-      @extensions.load_settings(@settings.to_hash)
-      @timers = Array.new
-      @master_timers = Array.new
-      @handlers_in_progress_count = 0
+      super
       @is_master = false
+      @timers[:master] = Array.new
+      @handlers_in_progress_count = 0
     end
 
-    def setup_redis
-      @logger.debug('connecting to redis', {
-        :settings => @settings[:redis]
-      })
-      @redis = Redis.connect(@settings[:redis])
-      @redis.on_error do |error|
-        @logger.fatal('redis connection error', {
-          :error => error.to_s
-        })
-        stop
-      end
-      @redis.before_reconnect do
-        unless testing?
-          @logger.warn('reconnecting to redis')
-          pause
-        end
-      end
-      @redis.after_reconnect do
-        @logger.info('reconnected to redis')
-        resume
-      end
-    end
-
-    def setup_rabbitmq
-      @logger.debug('connecting to rabbitmq', {
-        :settings => @settings[:rabbitmq]
-      })
-      @rabbitmq = RabbitMQ.connect(@settings[:rabbitmq])
-      @rabbitmq.on_error do |error|
-        @logger.fatal('rabbitmq connection error', {
-          :error => error.to_s
-        })
-        stop
-      end
-      @rabbitmq.before_reconnect do
-        unless testing?
-          @logger.warn('reconnecting to rabbitmq')
-          pause
-        end
-      end
-      @rabbitmq.after_reconnect do
-        @logger.info('reconnected to rabbitmq')
-        resume
-      end
-      @amq = @rabbitmq.channel
-    end
-
-    def setup_keepalives(&block)
+    def setup_keepalives
       @logger.debug('subscribing to keepalives')
-      @keepalive_queue = @amq.queue!('keepalives', :auto_delete => true)
-      @keepalive_queue.bind(@amq.direct('keepalives')) do
-        block.call if block
-      end
-      @keepalive_queue.subscribe(:ack => true) do |header, payload|
-        client = Oj.load(payload)
+      @transport.subscribe(:direct, 'keepalives', 'keepalives', :ack => true) do |message_info, message|
+        client = MultiJson.load(message)
         @logger.debug('received keepalive', {
           :client => client
         })
-        @redis.set('client:' + client[:name], Oj.dump(client)) do
+        @redis.set('client:' + client[:name], MultiJson.dump(client)) do
           @redis.sadd('clients', client[:name]) do
-            header.ack
+            @transport.ack(message_info)
           end
         end
       end
@@ -252,39 +194,31 @@ module Sensu
     end
 
     def mutate_event_data(mutator_name, event, &block)
+      mutator_name ||= 'json'
+      return_output = Proc.new do |output, status|
+        if status == 0
+          block.dup.call(output)
+        else
+          @logger.error('mutator error', {
+            :event => event,
+            :output => output,
+            :status => status
+          })
+          @handlers_in_progress_count -= 1
+        end
+      end
+      @logger.debug('mutating event data', {
+        :event => event,
+        :mutator_name => mutator_name
+      })
       case
-      when mutator_name.nil?
-        block.call(Oj.dump(event))
       when @settings.mutator_exists?(mutator_name)
         mutator = @settings[:mutators][mutator_name]
-        IO.async_popen(mutator[:command], Oj.dump(event), mutator[:timeout]) do |output, status|
-          if status == 0
-            block.call(output)
-          else
-            @logger.error('mutator error', {
-              :event => event,
-              :mutator => mutator,
-              :output => output,
-              :status => status
-            })
-            @handlers_in_progress_count -= 1
-          end
-        end
+        options = {:data => MultiJson.dump(event), :timeout => mutator[:timeout]}
+        Spawn.process(mutator[:command], options, &return_output)
       when @extensions.mutator_exists?(mutator_name)
         extension = @extensions[:mutators][mutator_name]
-        extension.safe_run(event) do |output, status|
-          if status == 0
-            block.call(output)
-          else
-            @logger.error('mutator extension error', {
-              :event => event,
-              :extension => extension.definition,
-              :output => output,
-              :status => status
-            })
-            @handlers_in_progress_count -= 1
-          end
-        end
+        extension.safe_run(event, &return_output)
       else
         @logger.error('unknown mutator', {
           :mutator_name => mutator_name
@@ -313,7 +247,8 @@ module Sensu
         mutate_event_data(handler[:mutator], event) do |event_data|
           case handler[:type]
           when 'pipe'
-            IO.async_popen(handler[:command], event_data, handler[:timeout]) do |output, status|
+            options = {:data => event_data, :timeout => handler[:timeout]}
+            Spawn.process(handler[:command], options) do |output, status|
               output.each_line do |line|
                 @logger.info('handler output', {
                   :handler => handler,
@@ -348,21 +283,17 @@ module Sensu
             rescue => error
               on_error.call(error)
             end
-          when 'amqp'
-            exchange_name = handler[:exchange][:name]
-            exchange_type = handler[:exchange].has_key?(:type) ? handler[:exchange][:type].to_sym : :direct
-            exchange_options = handler[:exchange].reject do |key, value|
-              [:name, :type].include?(key)
-            end
+          when 'transport'
             unless event_data.empty?
-              begin
-                @amq.method(exchange_type).call(exchange_name, exchange_options).publish(event_data)
-              rescue AMQ::Client::ConnectionClosedError => error
-                @logger.error('failed to publish event data to an exchange', {
-                  :exchange => handler[:exchange],
-                  :payload => event_data,
-                  :error => error.to_s
-                })
+              pipe = handler[:pipe]
+              @transport.publish(pipe[:type].to_sym, pipe[:name], event_data, pipe[:options] || Hash.new) do |info|
+                if info[:error]
+                  @logger.fatal('failed to publish event data to the transport', {
+                    :pipe => pipe,
+                    :payload => event_data,
+                    :error => info[:error].to_s
+                  })
+                end
               end
             end
             @handlers_in_progress_count -= 1
@@ -387,7 +318,7 @@ module Sensu
       })
       check = result[:check]
       result_set = check[:name] + ':' + check[:issued].to_s
-      @redis.hset('aggregation:' + result_set, result[:client], Oj.dump(
+      @redis.hset('aggregation:' + result_set, result[:client], MultiJson.dump(
         :output => check[:output],
         :status => check[:status]
       )) do
@@ -405,15 +336,28 @@ module Sensu
       end
     end
 
+    def event_bridges(event)
+      @extensions[:bridges].each do |name, bridge|
+        bridge.safe_run(event) do |output, status|
+          output.each_line do |line|
+            @logger.info('bridge extension output', {
+              :extension => bridge.definition,
+              :output => line
+            })
+          end
+        end
+      end
+    end
+
     def process_result(result)
       @logger.debug('processing result', {
         :result => result
       })
       @redis.get('client:' + result[:client]) do |client_json|
         unless client_json.nil?
-          client = Oj.load(client_json)
+          client = MultiJson.load(client_json)
           check = case
-          when @settings.check_exists?(result[:check][:name])
+          when @settings.check_exists?(result[:check][:name]) && !result[:check][:standalone]
             @settings[:checks][result[:check][:name]].merge(result[:check])
           else
             result[:check]
@@ -444,10 +388,10 @@ module Sensu
                 @redis.ltrim(history_key, -21, -1)
               end
               @redis.hget('events:' + client[:name], check[:name]) do |event_json|
-                previous_occurrence = event_json ? Oj.load(event_json) : false
+                previous_occurrence = event_json ? MultiJson.load(event_json) : false
                 is_flapping = false
                 if check.has_key?(:low_flap_threshold) && check.has_key?(:high_flap_threshold)
-                  was_flapping = previous_occurrence ? previous_occurrence[:flapping] : false
+                  was_flapping = previous_occurrence && previous_occurrence[:action] == 'flapping'
                   is_flapping = case
                   when total_state_change >= check[:high_flap_threshold]
                     true
@@ -458,6 +402,7 @@ module Sensu
                   end
                 end
                 event = {
+                  :id => random_uuid,
                   :client => client,
                   :check => check,
                   :occurrences => 1
@@ -466,25 +411,18 @@ module Sensu
                   if previous_occurrence && check[:status] == previous_occurrence[:status]
                     event[:occurrences] = previous_occurrence[:occurrences] + 1
                   end
-                  @redis.hset('events:' + client[:name], check[:name], Oj.dump(
-                    :output => check[:output],
-                    :status => check[:status],
-                    :issued => check[:issued],
-                    :handlers => Array((check[:handlers] || check[:handler]) || 'default'),
-                    :flapping => is_flapping,
-                    :occurrences => event[:occurrences]
-                  )) do
+                  event[:action] = is_flapping ? :flapping : :create
+                  @redis.hset('events:' + client[:name], check[:name], MultiJson.dump(event)) do
                     unless check[:handle] == false
-                      event[:action] = is_flapping ? :flapping : :create
                       handle_event(event)
                     end
                   end
                 elsif previous_occurrence
+                  event[:occurrences] = previous_occurrence[:occurrences]
+                  event[:action] = :resolve
                   unless check[:auto_resolve] == false && !check[:force_resolve]
                     @redis.hdel('events:' + client[:name], check[:name]) do
                       unless check[:handle] == false
-                        event[:occurrences] = previous_occurrence[:occurrences]
-                        event[:action] = :resolve
                         handle_event(event)
                       end
                     end
@@ -492,6 +430,7 @@ module Sensu
                 elsif check[:type] == 'metric'
                   handle_event(event)
                 end
+                event_bridges(event)
               end
             end
           end
@@ -499,20 +438,16 @@ module Sensu
       end
     end
 
-    def setup_results(&block)
+    def setup_results
       @logger.debug('subscribing to results')
-      @result_queue = @amq.queue!('results', :auto_delete => true)
-      @result_queue.bind(@amq.direct('results')) do
-        block.call if block
-      end
-      @result_queue.subscribe(:ack => true) do |header, payload|
-        result = Oj.load(payload)
+      @transport.subscribe(:direct, 'results', 'results', :ack => true) do |message_info, message|
+        result = MultiJson.load(message)
         @logger.debug('received result', {
           :result => result
         })
         process_result(result)
         EM::next_tick do
-          header.ack
+          @transport.ack(message_info)
         end
       end
     end
@@ -537,15 +472,15 @@ module Sensu
         :payload => payload,
         :subscribers => check[:subscribers]
       })
-      check[:subscribers].each do |exchange_name|
-        begin
-          @amq.fanout(exchange_name).publish(Oj.dump(payload))
-        rescue AMQ::Client::ConnectionClosedError => error
-          @logger.error('failed to publish check request', {
-            :exchange_name => exchange_name,
-            :payload => payload,
-            :error => error.to_s
-          })
+      check[:subscribers].each do |subscription|
+        @transport.publish(:fanout, subscription, MultiJson.dump(payload)) do |info|
+          if info[:error]
+            @logger.error('failed to publish check request', {
+              :subscription => subscription,
+              :payload => payload,
+              :error => info[:error].to_s
+            })
+          end
         end
       end
     end
@@ -556,9 +491,9 @@ module Sensu
       checks.each do |check|
         check_count += 1
         scheduling_delay = stagger * check_count % 30
-        @master_timers << EM::Timer.new(scheduling_delay) do
+        @timers[:master] << EM::Timer.new(scheduling_delay) do
           interval = testing? ? 0.5 : check[:interval]
-          @master_timers << EM::PeriodicTimer.new(interval) do
+          @timers[:master] << EM::PeriodicTimer.new(interval) do
             unless check_request_subdued?(check)
               publish_check_request(check)
             else
@@ -590,13 +525,13 @@ module Sensu
       @logger.debug('publishing check result', {
         :payload => payload
       })
-      begin
-        @amq.direct('results').publish(Oj.dump(payload))
-      rescue AMQ::Client::ConnectionClosedError => error
-        @logger.error('failed to publish check result', {
-          :payload => payload,
-          :error => error.to_s
-        })
+      @transport.publish(:direct, 'results', MultiJson.dump(payload)) do |info|
+        if info[:error]
+          @logger.error('failed to publish check result', {
+            :payload => payload,
+            :error => info[:error].to_s
+          })
+        end
       end
     end
 
@@ -606,7 +541,7 @@ module Sensu
         clients.each do |client_name|
           @redis.get('client:' + client_name) do |client_json|
             unless client_json.nil?
-              client = Oj.load(client_json)
+              client = MultiJson.load(client_json)
               check = {
                 :thresholds => {
                   :warning => 120,
@@ -643,7 +578,7 @@ module Sensu
 
     def setup_client_monitor
       @logger.debug('monitoring clients')
-      @master_timers << EM::PeriodicTimer.new(30) do
+      @timers[:master] << EM::PeriodicTimer.new(30) do
         determine_stale_clients
       end
     end
@@ -678,7 +613,7 @@ module Sensu
 
     def setup_aggregation_pruner
       @logger.debug('pruning aggregations')
-      @master_timers << EM::PeriodicTimer.new(20) do
+      @timers[:master] << EM::PeriodicTimer.new(20) do
         prune_aggregations
       end
     end
@@ -713,7 +648,7 @@ module Sensu
 
     def setup_master_monitor
       request_master_election
-      @timers << EM::PeriodicTimer.new(20) do
+      @timers[:run] << EM::PeriodicTimer.new(20) do
         if @is_master
           @redis.set('lock:master', Time.now.to_i) do
             @logger.debug('updated master lock timestamp')
@@ -728,10 +663,10 @@ module Sensu
       block ||= Proc.new {}
       if @is_master
         @logger.warn('resigning as master')
-        @master_timers.each do |timer|
+        @timers[:master].each do |timer|
           timer.cancel
         end
-        @master_timers.clear
+        @timers[:master].clear
         if @redis.connected?
           @redis.del('lock:master') do
             @logger.info('removed master lock')
@@ -758,18 +693,7 @@ module Sensu
 
     def unsubscribe
       @logger.warn('unsubscribing from keepalive and result queues')
-      if @rabbitmq.connected?
-        @keepalive_queue.unsubscribe
-        @result_queue.unsubscribe
-        @amq.recover
-      else
-        @keepalive_queue.before_recovery do
-          @keepalive_queue.unsubscribe
-        end
-        @result_queue.before_recovery do
-          @result_queue.unsubscribe
-        end
-      end
+      @transport.unsubscribe
     end
 
     def complete_handlers_in_progress(&block)
@@ -793,17 +717,17 @@ module Sensu
 
     def start
       setup_redis
-      setup_rabbitmq
+      setup_transport
       bootstrap
     end
 
     def pause(&block)
       unless @state == :pausing || @state == :paused
         @state = :pausing
-        @timers.each do |timer|
+        @timers[:run].each do |timer|
           timer.cancel
         end
-        @timers.clear
+        @timers[:run].clear
         unsubscribe
         resign_as_master do
           @state = :paused
@@ -817,7 +741,7 @@ module Sensu
     def resume
       retry_until_true(1) do
         if @state == :paused
-          if @redis.connected? && @rabbitmq.connected?
+          if @redis.connected? && @transport.connected?
             bootstrap
             true
           end
@@ -830,30 +754,9 @@ module Sensu
       @state = :stopping
       pause do
         complete_handlers_in_progress do
-          @extensions.stop_all do
-            @redis.close
-            @rabbitmq.close
-            @logger.warn('stopping reactor')
-            EM::stop_event_loop
-          end
-        end
-      end
-    end
-
-    def trap_signals
-      @signals = Array.new
-      STOP_SIGNALS.each do |signal|
-        Signal.trap(signal) do
-          @signals << signal
-        end
-      end
-      EM::PeriodicTimer.new(1) do
-        signal = @signals.shift
-        if STOP_SIGNALS.include?(signal)
-          @logger.warn('received signal', {
-            :signal => signal
-          })
-          stop
+          @redis.close
+          @transport.close
+          super
         end
       end
     end