sensu 0.9.7 → 0.9.8.beta

data/lib/sensu/api.rb

@@ -399,6 +399,67 @@ module Sensu
       end
     end
 
+    aget '/aggregates' do
+      response = Array.new
+      $redis.smembers('aggregates').callback do |checks|
+        unless checks.empty?
+          checks.each_with_index do |check_name, index|
+            $redis.smembers('aggregates:' + check_name).callback do |aggregates|
+              collection = {
+                :check => check_name,
+                :issued => aggregates.sort.reverse.take(10)
+              }
+              response.push(collection)
+              if index == checks.size - 1
+                body response.to_json
+              end
+            end
+          end
+        else
+          body response.to_json
+        end
+      end
+    end
+
+    aget %r{/aggregates/([\w\.-]+)$} do |check_name|
+      $redis.smembers('aggregates:' + check_name).callback do |aggregates|
+        body aggregates.sort.reverse.take(10).to_json
+      end
+    end
+
+    aget %r{/aggregates?/([\w\.-]+)/([\w\.-]+)$} do |check_name, check_issued|
+      result_set = check_name + ':' + check_issued
+      $redis.hgetall('aggregate:' + result_set).callback do |aggregate|
+        unless aggregate.empty?
+          response = aggregate.inject(Hash.new) do |formatted, (status, count)|
+            formatted[status] = Integer(count)
+            formatted
+          end
+          if params[:summarize]
+            options = params[:summarize].split(',')
+            $redis.hgetall('aggregation:' + result_set).callback do |results|
+              formatted_results = results.inject(Hash.new) do |formatted, (client_name, check_json)|
+                formatted[client_name] = JSON.parse(check_json, :symbolize_names => true)
+                formatted
+              end
+              if options.include?('output')
+                outputs = Hash.new(0)
+                formatted_results.each do |client_name, check|
+                  outputs[check[:output]] += 1
+                end
+                response[:outputs] = outputs
+              end
+              body response.to_json
+            end
+          else
+            body response.to_json
+          end
+        else
+          not_found!
+        end
+      end
+    end
+
     apost %r{/stash(?:es)?/(.*)} do |path|
       begin
         post_body = JSON.parse(request.body.read)

data/lib/sensu/constants.rb

@@ -1,6 +1,6 @@
 module Sensu
   unless defined?(Sensu::VERSION)
-    VERSION = '0.9.7'
+    VERSION = '0.9.8.beta'
   end
 
   unless defined?(Sensu::DEFAULT_OPTIONS)
@@ -9,4 +9,8 @@ module Sensu
       :config_dir => '/etc/sensu/conf.d'
     }
   end
+
+  unless defined?(Sensu::SEVERITIES)
+    SEVERITIES = %w[ok warning critical unknown]
+  end
 end

data/lib/sensu/server.rb

@@ -116,19 +116,15 @@ module Sensu
           end
         end
       end
-      if subdue
-        subdue_at == (check[:subdue][:at] || 'handler').to_sym
-      else
-        false
-      end
+      subdue && subdue_at == (check[:subdue][:at] || 'handler').to_sym
     end
 
-    def check_handlers(check)
+    def event_handlers(event)
       handler_list = case
-      when check.has_key?(:handler)
-        [check[:handler]]
-      when check.has_key?(:handlers)
-        check[:handlers]
+      when event[:check].has_key?(:handlers)
+        event[:check][:handlers]
+      when event[:check].has_key?(:handler)
+        [event[:check][:handler]]
       else
         ['default']
       end
@@ -141,21 +137,37 @@ module Sensu
       end
       handler_list.flatten!
       handler_list.uniq!
-      handler_list.reject! do |handler_name|
-        unless @settings.handler_exists?(handler_name)
+      handlers = handler_list.map do |handler_name|
+        if @settings.handler_exists?(handler_name)
+          handler = @settings[:handlers][handler_name].merge(:name => handler_name)
+          if handler.has_key?(:severities)
+            event_severity = Sensu::SEVERITIES[event[:check][:status]] || 'unknown'
+            unless handler[:severities].include?(event_severity)
+              @logger.debug('handler does not handle event severity', {
+                :event => event,
+                :handler => handler
+              })
+              next
+            end
+          end
+          if check_subdued?(event[:check], :handler)
+            @logger.info('check is subdued at handler', {
+              :event => event,
+              :handler => handler
+            })
+            next
+          end
+          handler
+        else
           @logger.warn('unknown handler', {
             :handler => {
               :name => handler_name
             }
           })
-          true
-        else
-          false
+          next
         end
       end
-      handler_list.map do |handler_name|
-        @settings[:handlers][handler_name].merge(:name => handler_name)
-      end
+      handlers.compact
     end
 
     def mutate_event_data(handler, event)
@@ -180,12 +192,6 @@ module Sensu
                 io.close_write
                 mutated = io.read
               end
-              if $?.exitstatus != 0
-                @logger.warn('mutator had a non-zero exit status', {
-                  :event => event,
-                  :mutator => mutator
-                })
-              end
             rescue => error
               @logger.error('mutator error', {
                 :event => event,
@@ -193,6 +199,12 @@ module Sensu
                 :error => error.to_s
               })
             end
+            if $?.exitstatus != 0
+              @logger.warn('mutator had a non-zero exit status', {
+                :event => event,
+                :mutator => mutator
+              })
+            end
           else
             @logger.warn('unknown mutator', {
               :mutator => {
@@ -208,93 +220,116 @@ module Sensu
     end
 
     def handle_event(event)
-      unless check_subdued?(event[:check], :handler)
-        handlers = check_handlers(event[:check])
-        handlers.each do |handler|
-          log_level = event[:check][:type] == 'metric' ? :debug : :info
-          @logger.send(log_level, 'handling event', {
-            :event => event,
-            :handler => handler
-          })
-          @handlers_in_progress_count += 1
-          case handler[:type]
-          when 'pipe'
-            execute = Proc.new do
-              begin
-                mutated_event_data = mutate_event_data(handler, event)
-                unless mutated_event_data.nil? || mutated_event_data.empty?
-                  IO.popen(handler[:command] + ' 2>&1', 'r+') do |io|
-                    io.write(mutated_event_data)
-                    io.close_write
-                    io.read.split(/\n+/).each do |line|
-                      @logger.info(line)
-                    end
+      handlers = event_handlers(event)
+      handlers.each do |handler|
+        log_level = event[:check][:type] == 'metric' ? :debug : :info
+        @logger.send(log_level, 'handling event', {
+          :event => event,
+          :handler => handler
+        })
+        @handlers_in_progress_count += 1
+        case handler[:type]
+        when 'pipe'
+          execute = Proc.new do
+            begin
+              event_data = mutate_event_data(handler, event)
+              unless event_data.nil? || event_data.empty?
+                IO.popen(handler[:command] + ' 2>&1', 'r+') do |io|
+                  io.write(event_data)
+                  io.close_write
+                  io.read.split(/\n+/).each do |line|
+                    @logger.info(line)
                   end
                 end
-              rescue => error
-                @logger.error('handler error', {
-                  :event => event,
-                  :handler => handler,
-                  :error => error.to_s
-                })
               end
+            rescue => error
+              @logger.error('handler error', {
+                :event => event,
+                :handler => handler,
+                :error => error.to_s
+              })
             end
-            complete = Proc.new do
-              @handlers_in_progress_count -= 1
-            end
-            EM::defer(execute, complete)
-          when 'tcp', 'udp'
-            data = Proc.new do
-              mutate_event_data(handler, event)
-            end
-            write = Proc.new do |data|
-              begin
-                case handler[:type]
-                when 'tcp'
-                  EM::connect(handler[:socket][:host], handler[:socket][:port], nil) do |socket|
-                    socket.send_data(data)
-                    socket.close_connection_after_writing
-                  end
-                when 'udp'
-                  EM::open_datagram_socket('127.0.0.1', 0, nil) do |socket|
-                    socket.send_datagram(data, handler[:socket][:host], handler[:socket][:port])
-                    socket.close_connection_after_writing
-                  end
+          end
+          complete = Proc.new do
+            @handlers_in_progress_count -= 1
+          end
+          EM::defer(execute, complete)
+        when 'tcp', 'udp'
+          event_data = Proc.new do
+            mutate_event_data(handler, event)
+          end
+          write = Proc.new do |event_data|
+            begin
+              case handler[:type]
+              when 'tcp'
+                EM::connect(handler[:socket][:host], handler[:socket][:port], nil) do |socket|
+                  socket.send_data(event_data)
+                  socket.close_connection_after_writing
+                end
+              when 'udp'
+                EM::open_datagram_socket('127.0.0.1', 0, nil) do |socket|
+                  socket.send_datagram(event_data, handler[:socket][:host], handler[:socket][:port])
+                  socket.close_connection_after_writing
                 end
-              rescue => error
-                @logger.error('handler error', {
-                  :event => event,
-                  :handler => handler,
-                  :error => error.to_s
-                })
               end
-              @handlers_in_progress_count -= 1
-            end
-            EM::defer(data, write)
-          when 'amqp'
-            exchange_name = handler[:exchange][:name]
-            exchange_type = handler[:exchange].has_key?(:type) ? handler[:exchange][:type].to_sym : :direct
-            exchange_options = handler[:exchange].reject do |key, value|
-              [:name, :type].include?(key)
-            end
-            payloads = Proc.new do
-              Array(mutate_event_data(handler, event))
+            rescue => error
+              @logger.error('handler error', {
+                :event => event,
+                :handler => handler,
+                :error => error.to_s
+              })
             end
-            publish = Proc.new do |payloads|
-              payloads.each do |payload|
-                unless payload.empty?
-                  @amq.method(exchange_type).call(exchange_name, exchange_options).publish(payload)
-                end
+            @handlers_in_progress_count -= 1
+          end
+          EM::defer(event_data, write)
+        when 'amqp'
+          exchange_name = handler[:exchange][:name]
+          exchange_type = handler[:exchange].has_key?(:type) ? handler[:exchange][:type].to_sym : :direct
+          exchange_options = handler[:exchange].reject do |key, value|
+            [:name, :type].include?(key)
+          end
+          payloads = Proc.new do
+            Array(mutate_event_data(handler, event))
+          end
+          publish = Proc.new do |payloads|
+            payloads.each do |payload|
+              unless payload.empty?
+                @amq.method(exchange_type).call(exchange_name, exchange_options).publish(payload)
               end
-              @handlers_in_progress_count -= 1
             end
-            EM::defer(payloads, publish)
-          when 'set'
-            @logger.error('handler sets cannot be nested', {
-              :handler => handler
-            })
             @handlers_in_progress_count -= 1
           end
+          EM::defer(payloads, publish)
+        when 'set'
+          @logger.error('handler sets cannot be nested', {
+            :handler => handler
+          })
+          @handlers_in_progress_count -= 1
+        end
+      end
+    end
+
+    def aggregate_result(result)
+      @logger.debug('adding result to aggregate', {
+        :result => result
+      })
+      check = result[:check]
+      result_set = check[:name] + ':' + check[:issued].to_s
+      @redis.hset('aggregation:' + result_set, result[:client], {
+        :output => check[:output],
+        :status => check[:status]
+      }.to_json).callback do
+        statuses = Sensu::SEVERITIES
+        statuses.each do |status|
+          @redis.hsetnx('aggregate:' + result_set, status, 0)
+        end
+        status = (statuses[check[:status]] || 'unknown')
+        @redis.hincrby('aggregate:' + result_set, status, 1).callback do
+          @redis.hincrby('aggregate:' + result_set, 'total', 1).callback do
+            @redis.sadd('aggregates:' + check[:name], check[:issued]).callback do
+              @redis.sadd('aggregates', check[:name])
+            end
+          end
         end
       end
     end
@@ -312,22 +347,20 @@ module Sensu
           else
             result[:check]
           end
-          event = {
-            :client => client,
-            :check => check,
-            :occurrences => 1
-          }
+          if check[:aggregate]
+            aggregate_result(result)
+          end
           @redis.sadd('history:' + client[:name], check[:name])
           history_key = 'history:' + client[:name] + ':' + check[:name]
           @redis.rpush(history_key, check[:status]).callback do
             @redis.lrange(history_key, -21, -1).callback do |history|
-              event[:check][:history] = history
+              check[:history] = history
               total_state_change = 0
               unless history.count < 21
                 state_changes = 0
                 change_weight = 0.8
+                previous_status = history.first
                 history.each do |status|
-                  previous_status ||= status
                   unless status == previous_status
                     state_changes += change_weight
                   end
@@ -335,7 +368,7 @@ module Sensu
                   previous_status = status
                 end
                 total_state_change = (state_changes.fdiv(20) * 100).to_i
-                @redis.lpop(history_key)
+                @redis.ltrim(history_key, -21, -1)
               end
               @redis.hget('events:' + client[:name], check[:name]).callback do |event_json|
                 previous_occurrence = event_json ? JSON.parse(event_json, :symbolize_names => true) : false
@@ -351,7 +384,12 @@ module Sensu
                     was_flapping
                   end
                 end
-                if check[:status] != 0
+                event = {
+                  :client => client,
+                  :check => check,
+                  :occurrences => 1
+                }
+                if check[:status] != 0 || is_flapping
                   if previous_occurrence && check[:status] == previous_occurrence[:status]
                     event[:occurrences] = previous_occurrence[:occurrences] += 1
                   end
@@ -362,38 +400,17 @@ module Sensu
                     :flapping => is_flapping,
                     :occurrences => event[:occurrences]
                   }.to_json).callback do
-                    unless check[:handle] == false
-                      event[:check][:flapping] = is_flapping
-                      event[:action] = 'create'
+                    unless check[:handle] == false || is_flapping
+                      event[:action] = :create
                       handle_event(event)
-                    else
-                      @logger.debug('handling disabled', {
-                        :event => event
-                      })
                     end
                   end
                 elsif previous_occurrence
-                  unless is_flapping
-                    unless check[:auto_resolve] == false && !check[:force_resolve]
-                      @redis.hdel('events:' + client[:name], check[:name]).callback do
-                        unless check[:handle] == false
-                          event[:occurrences] = previous_occurrence[:occurrences]
-                          event[:action] = 'resolve'
-                          handle_event(event)
-                        else
-                          @logger.debug('handling disabled', {
-                            :event => event
-                          })
-                        end
-                      end
-                    end
-                  else
-                    @logger.debug('check is flapping', {
-                      :event => event
-                    })
-                    @redis.hset('events:' + client[:name], check[:name], previous_occurrence.merge(:flapping => true).to_json).callback do
-                      if check[:type] == 'metric'
-                        event[:check][:flapping] = is_flapping
+                  unless check[:auto_resolve] == false && !check[:force_resolve]
+                    @redis.hdel('events:' + client[:name], check[:name]).callback do
+                      unless check[:handle] == false
+                        event[:occurrences] = previous_occurrence[:occurrences]
+                        event[:action] = :resolve
                         handle_event(event)
                       end
                     end
@@ -468,7 +485,6 @@ module Sensu
     def setup_keepalive_monitor
       @logger.debug('monitoring client keepalives')
       @master_timers << EM::PeriodicTimer.new(30) do
-        @logger.debug('checking for stale client info')
         @redis.smembers('clients').callback do |clients|
           clients.each do |client_name|
             @redis.get('client:' + client_name).callback do |client_json|
@@ -502,9 +518,39 @@ module Sensu
       end
     end
 
+    def setup_aggregation_pruner
+      @logger.debug('pruning aggregations')
+      @master_timers << EM::PeriodicTimer.new(20) do
+        @redis.smembers('aggregates').callback do |checks|
+          checks.each do |check_name|
+            @redis.smembers('aggregates:' + check_name).callback do |aggregates|
+              aggregates.sort!
+              until aggregates.size <= 10
+                check_issued = aggregates.shift
+                @redis.srem('aggregates:' + check_name, check_issued).callback do
+                  result_set = check_name + ':' + check_issued.to_s
+                  @redis.del('aggregate:' + result_set).callback do
+                    @redis.del('aggregation:' + result_set).callback do
+                      @logger.debug('pruned aggregation', {
+                        :check => {
+                          :name => check_name,
+                          :issued => check_issued
+                        }
+                      })
+                    end
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+
     def master_duties
       setup_publisher
       setup_keepalive_monitor
+      setup_aggregation_pruner
     end
 
     def request_master_election
@@ -514,7 +560,7 @@ module Sensu
           @logger.info('i am the master')
           master_duties
         else
-          @redis.get('lock:master') do |timestamp|
+          @redis.get('lock:master').callback do |timestamp|
             if Time.now.to_i - timestamp.to_i >= 60
               @redis.getset('lock:master', Time.now.to_i).callback do |previous|
                 if previous == timestamp

data/lib/sensu/settings.rb

@@ -287,7 +287,7 @@ module Sensu
       unless @settings[:client][:address].is_a?(String)
         invalid('client must have an address')
       end
-      unless @settings[:client][:subscriptions].is_a?(Array) && @settings[:client][:subscriptions].count > 0
+      unless @settings[:client][:subscriptions].is_a?(Array) && !@settings[:client][:subscriptions].empty?
         invalid('client must have subscriptions')
       end
       @settings[:client][:subscriptions].each do |subscription|
@@ -327,6 +327,20 @@ module Sensu
             :handler => handler
           })
         end
+        if handler.has_key?(:severities)
+          unless handler[:severities].is_a?(Array) && !handler[:severities].empty?
+            invalid('handler severities must be an array and not empty', {
+              :handler => handler
+            })
+          end
+          handler[:severities].each do |severity|
+            unless Sensu::SEVERITIES.include?(severity)
+              invalid('handler severities are invalid', {
+                :handler => handler
+              })
+            end
+          end
+        end
         case handler[:type]
         when 'pipe'
           unless handler[:command].is_a?(String)

metadata

@@ -1,8 +1,8 @@
 --- !ruby/object:Gem::Specification
 name: sensu
 version: !ruby/object:Gem::Version
-  version: 0.9.7
-  prerelease: 
+  version: 0.9.8.beta
+  prerelease: 6
 platform: ruby
 authors:
 - Sean Porter
@@ -10,11 +10,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-09-20 00:00:00.000000000 Z
+date: 2012-09-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: eventmachine
-  requirement: &10456340 !ruby/object:Gem::Requirement
+  requirement: &22192440 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - =
@@ -22,10 +22,10 @@ dependencies:
         version: 1.0.0.rc.4
   type: :runtime
   prerelease: false
-  version_requirements: *10456340
+  version_requirements: *22192440
 - !ruby/object:Gem::Dependency
   name: amqp
-  requirement: &10455580 !ruby/object:Gem::Requirement
+  requirement: &22191880 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - =
@@ -33,10 +33,10 @@ dependencies:
         version: 0.9.7
   type: :runtime
   prerelease: false
-  version_requirements: *10455580
+  version_requirements: *22191880
 - !ruby/object:Gem::Dependency
   name: json
-  requirement: &10445140 !ruby/object:Gem::Requirement
+  requirement: &22191380 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -44,10 +44,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *10445140
+  version_requirements: *22191380
 - !ruby/object:Gem::Dependency
   name: cabin
-  requirement: &10444380 !ruby/object:Gem::Requirement
+  requirement: &22190740 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - =
@@ -55,10 +55,10 @@ dependencies:
         version: 0.4.4
   type: :runtime
   prerelease: false
-  version_requirements: *10444380
+  version_requirements: *22190740
 - !ruby/object:Gem::Dependency
   name: ruby-redis
-  requirement: &10443860 !ruby/object:Gem::Requirement
+  requirement: &22189740 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - =
@@ -66,10 +66,10 @@ dependencies:
         version: 0.0.2
   type: :runtime
   prerelease: false
-  version_requirements: *10443860
+  version_requirements: *22189740
 - !ruby/object:Gem::Dependency
   name: async_sinatra
-  requirement: &10443240 !ruby/object:Gem::Requirement
+  requirement: &22166160 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - =
@@ -77,10 +77,10 @@ dependencies:
         version: 1.0.0
   type: :runtime
   prerelease: false
-  version_requirements: *10443240
+  version_requirements: *22166160
 - !ruby/object:Gem::Dependency
   name: thin
-  requirement: &10442620 !ruby/object:Gem::Requirement
+  requirement: &22164620 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - =
@@ -88,10 +88,10 @@ dependencies:
         version: 1.4.1
   type: :runtime
   prerelease: false
-  version_requirements: *10442620
+  version_requirements: *22164620
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &10441960 !ruby/object:Gem::Requirement
+  requirement: &22163460 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -99,10 +99,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *10441960
+  version_requirements: *22163460
 - !ruby/object:Gem::Dependency
   name: em-spec
-  requirement: &10441040 !ruby/object:Gem::Requirement
+  requirement: &22161740 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -110,10 +110,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *10441040
+  version_requirements: *22161740
 - !ruby/object:Gem::Dependency
   name: em-http-request
-  requirement: &10439840 !ruby/object:Gem::Requirement
+  requirement: &22160720 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -121,7 +121,7 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *10439840
+  version_requirements: *22160720
 description: A monitoring framework that aims to be simple, malleable, and scalable.
   Uses the publish/subscribe model.
 email:
@@ -169,9 +169,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
-  - - ! '>='
+  - - ! '>'
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.11