sensu 0.23.3-java → 0.24.0.beta-java

data/lib/sensu/api/validators.rb

@@ -0,0 +1,37 @@
+require "sensu/settings/rules"
+require "sensu/settings/validators/client"
+
+module Sensu
+  module API
+    module Validators
+      # The error class for validation.
+      class Invalid < RuntimeError; end
+
+      class Client
+        # Include Sensu Settings rules and client validator.
+        include Sensu::Settings::Rules
+        include Sensu::Settings::Validators::Client
+
+        # Determine if a client definition is valid.
+        #
+        # @param client [Hash]
+        # @return [TrueClass, FalseClass]
+        def valid?(client)
+          validate_client(client)
+          true
+        rescue Invalid
+          false
+        end
+
+        private
+
+        # This method is called when `validate_client()` encounters an
+        # invalid definition object. This method raises an exception
+        # to be caught by `valid?()`.
+        def invalid(*arguments)
+          raise Invalid
+        end
+      end
+    end
+  end
+end

data/lib/sensu/cli.rb

@@ -26,6 +26,9 @@ module Sensu
         opts.on("-d", "--config_dir DIR[,DIR]", "DIR or comma-delimited DIR list for Sensu JSON config files") do |dir|
           options[:config_dirs] = dir.split(",")
         end
+        opts.on("--validate_config", "Validate the compiled configuration and exit") do
+          options[:validate_config] = true
+        end
         opts.on("-P", "--print_config", "Print the compiled configuration and exit") do
           options[:print_config] = true
         end

data/lib/sensu/client/process.rb

@@ -99,12 +99,40 @@ module Sensu
         end
       end
 
+      # Perform token substitution for an object. String values are
+      # passed to `substitute_tokens()`, arrays and sub-hashes are
+      # processed recursively. Numeric values are ignored.
+      #
+      # @param object [Object]
+      # @return	[Array] containing the updated object with substituted
+      #   values and an array of unmatched tokens.
+      def object_substitute_tokens(object)
+        unmatched_tokens = []
+        case object
+        when Hash
+          object.each do |key, value|
+            object[key], unmatched = object_substitute_tokens(value)
+            unmatched_tokens.push(*unmatched)
+          end
+        when Array
+          object.map! do |value|
+            value, unmatched = object_substitute_tokens(value)
+            unmatched_tokens.push(*unmatched)
+            value
+          end
+        when String
+          object, unmatched_tokens = substitute_tokens(object, @settings[:client])
+        end
+        [object, unmatched_tokens.uniq]
+      end
+
       # Execute a check command, capturing its output (STDOUT/ERR),
       # exit status code, execution duration, timestamp, and publish
       # the result. This method guards against multiple executions for
-      # the same check. Check command tokens are substituted with the
-      # associated client attribute values. If there are unmatched
-      # check command tokens, the check command will not be executed,
+      # the same check. Check attribute value tokens are substituted
+      # with the associated client attribute values, via
+      # `object_substitute_tokens()`. If there are unmatched check
+      # attribute value tokens, the check will not be executed,
       # instead a check result will be published reporting the
       # unmatched tokens.
       #
@@ -113,11 +141,11 @@ module Sensu
         @logger.debug("attempting to execute check command", :check => check)
         unless @checks_in_progress.include?(check[:name])
           @checks_in_progress << check[:name]
-          command, unmatched_tokens = substitute_tokens(check[:command], @settings[:client])
+          check, unmatched_tokens = object_substitute_tokens(check)
           if unmatched_tokens.empty?
-            check[:executed] = Time.now.to_i
             started = Time.now.to_f
-            Spawn.process(command, :timeout => check[:timeout]) do |output, status|
+            check[:executed] = started.to_i
+            Spawn.process(check[:command], :timeout => check[:timeout]) do |output, status|
               check[:duration] = ("%.3f" % (Time.now.to_f - started)).to_f
               check[:output] = output
               check[:status] = status

data/lib/sensu/constants.rb

@@ -1,7 +1,7 @@
 module Sensu
   unless defined?(Sensu::VERSION)
     # Sensu release version.
-    VERSION = "0.23.3".freeze
+    VERSION = "0.24.0.beta".freeze
 
     # Sensu check severities.
     SEVERITIES = %w[ok warning critical unknown].freeze

data/lib/sensu/daemon.rb

@@ -2,14 +2,14 @@ require "rubygems"
 
 gem "eventmachine", "1.2.0.1"
 
-gem "sensu-json", "1.1.1"
+gem "sensu-json", "2.0.0"
 gem "sensu-logger", "1.2.0"
-gem "sensu-settings", "3.4.0"
+gem "sensu-settings", "5.1.0"
 gem "sensu-extension", "1.5.0"
 gem "sensu-extensions", "1.5.0"
-gem "sensu-transport", "5.0.0"
+gem "sensu-transport", "6.0.0"
 gem "sensu-spawn", "2.2.0"
-gem "sensu-redis", "1.3.0"
+gem "sensu-redis", "1.4.0"
 
 require "time"
 require "uri"
@@ -49,6 +49,7 @@ module Sensu
       setup_logger(options)
       load_settings(options)
       load_extensions(options)
+      setup_spawn
       setup_process(options)
     end
 
@@ -64,56 +65,78 @@ module Sensu
       @logger.setup_signal_traps
     end
 
-    # Log setting or extension loading concerns, sensitive information
+    # Log setting or extension loading notices, sensitive information
     # is redacted.
     #
-    # @param concerns [Array] to be logged.
-    # @param level [Symbol] to log the concerns at.
-    def log_concerns(concerns=[], level=:warn)
-      concerns.each do |concern|
+    # @param notices [Array] to be logged.
+    # @param level [Symbol] to log the notices at.
+    def log_notices(notices=[], level=:warn)
+      notices.each do |concern|
         message = concern.delete(:message)
         @logger.send(level, message, redact_sensitive(concern))
       end
     end
 
-    # Print the Sensu settings and immediately exit the process. This
-    # method is used while troubleshooting configuration issues,
-    # triggered by a CLI argument, e.g. `--print_config`. Sensu
-    # settings with sensitive values (e.g. passwords) are first
-    # redacted.
+    # Determine if the Sensu settings are valid, if there are load or
+    # validation errors, and immediately exit the process with the
+    # appropriate exit status code. This method is used to determine
+    # if the latest configuration changes are valid prior to
+    # restarting the Sensu service, triggered by a CLI argument, e.g.
+    # `--validate_config`.
     #
     # @param settings [Object]
-    def print_settings(settings)
+    def validate_settings!(settings)
+      if settings.errors.empty?
+        puts "configuration is valid"
+        exit
+      else
+        puts "configuration is invalid"
+        puts Sensu::JSON.dump({:errors => @settings.errors}, :pretty => true)
+        exit 2
+      end
+    end
+
+    # Print the Sensu settings (JSON) to STDOUT and immediately exit
+    # the process with the appropriate exit status code. This method
+    # is used while troubleshooting configuration issues, triggered by
+    # a CLI argument, e.g. `--print_config`. Sensu settings with
+    # sensitive values (e.g. passwords) are first redacted.
+    #
+    # @param settings [Object]
+    def print_settings!(settings)
       redacted_settings = redact_sensitive(settings.to_hash)
       @logger.warn("outputting compiled configuration and exiting")
       puts Sensu::JSON.dump(redacted_settings, :pretty => true)
-      exit
+      exit(settings.errors.empty? ? 0 : 2)
     end
 
-    # Load Sensu settings and validate them. If there are validation
-    # failures, log them (concerns), then cause the Sensu process to
-    # exit (2). This method creates the settings instance variable:
-    # `@settings`. If the `print_config` option is true, this method
-    # calls `print_settings()` to output the compiled configuration
-    # settings and then exit the process.
+    # Load Sensu settings. This method creates the settings instance
+    # variable: `@settings`. If the `validate_config` option is true,
+    # this method calls `validate_settings!()` to validate the latest
+    # compiled configuration settings and will then exit the process.
+    # If the `print_config` option is true, this method calls
+    # `print_settings!()` to output the compiled configuration
+    # settings and will then exit the process. If there are loading or
+    # validation errors, they will be logged (notices), and this
+    # method will exit(2) the process.
+    #
     #
     # https://github.com/sensu/sensu-settings
     #
     # @param options [Hash]
     def load_settings(options={})
       @settings = Settings.get(options)
-      log_concerns(@settings.warnings)
-      failures = @settings.validate
-      unless failures.empty?
-        @logger.fatal("invalid settings")
-        log_concerns(failures, :fatal)
+      validate_settings!(@settings) if options[:validate_config]
+      log_notices(@settings.warnings)
+      log_notices(@settings.errors, :fatal)
+      print_settings!(@settings) if options[:print_config]
+      unless @settings.errors.empty?
         @logger.fatal("SENSU NOT RUNNING!")
         exit 2
       end
-      print_settings(@settings) if options[:print_config]
     end
 
-    # Load Sensu extensions and log any concerns. Set the logger and
+    # Load Sensu extensions and log any notices. Set the logger and
     # settings for each extension instance. This method creates the
     # extensions instance variable: `@extensions`.
     #
@@ -123,7 +146,7 @@ module Sensu
     # @param options [Hash]
     def load_extensions(options={})
       @extensions = Extensions.get(options)
-      log_concerns(@extensions.warnings)
+      log_notices(@extensions.warnings)
       extension_settings = @settings.to_hash.dup
       @extensions.all.each do |extension|
         extension.logger = @logger
@@ -131,6 +154,20 @@ module Sensu
       end
     end
 
+    # Set up Sensu spawn, creating a worker to create, control, and
+    # limit spawned child processes. This method adjusts the
+    # EventMachine thread pool size to accommodate the concurrent
+    # process spawn limit and other Sensu process operations.
+    #
+    # https://github.com/sensu/sensu-spawn
+    def setup_spawn
+      @logger.info("configuring sensu spawn", :settings => @settings[:sensu][:spawn])
+      threadpool_size = @settings[:sensu][:spawn][:limit] + 10
+      @logger.debug("setting eventmachine threadpool size", :size => threadpool_size)
+      EM.threadpool_size = threadpool_size
+      Spawn.setup(@settings[:sensu][:spawn])
+    end
+
     # Manage the current process, optionally daemonize and/or write
     # the current process ID to a PID file.
     #

data/lib/sensu/server/filter.rb

@@ -345,14 +345,14 @@ module Sensu
         end
         if filter_message
           @logger.info(filter_message, details)
-          @handling_event_count -= 1 if @handling_event_count
+          @in_progress[:events] -= 1 if @in_progress
         else
           event_filtered?(handler, event) do |filtered|
             unless filtered
               yield(event)
             else
               @logger.info("event was filtered", details)
-              @handling_event_count -= 1 if @handling_event_count
+              @in_progress[:events] -= 1 if @in_progress
             end
           end
         end

data/lib/sensu/server/handle.rb

@@ -4,7 +4,7 @@ module Sensu
   module Server
     module Handle
       # Create a handler error callback, for logging the error and
-      # decrementing the `@handling_event_count` by `1`.
+      # decrementing the `@in_progress[:events]` by `1`.
       #
       # @param handler [Object]
       # @param event_data [Object]
@@ -16,14 +16,14 @@ module Sensu
             :event_data => event_data,
             :error => error.to_s
           })
-          @handling_event_count -= 1 if @handling_event_count
+          @in_progress[:events] -= 1 if @in_progress
         end
       end
 
       # Execute a pipe event handler, using the defined handler
       # command to spawn a process, passing it event data via STDIN.
       # Log the handler output lines and decrement the
-      # `@handling_event_count` by `1` when the handler executes
+      # `@in_progress[:events]` by `1` when the handler executes
       # successfully.
       #
       # @param handler [Hash] definition.
@@ -36,7 +36,7 @@ module Sensu
             :handler => handler,
             :output => output.split("\n+")
           })
-          @handling_event_count -= 1 if @handling_event_count
+          @in_progress[:events] -= 1 if @in_progress
         end
       end
 
@@ -47,7 +47,7 @@ module Sensu
       # `handler_error()` method is used to create the `on_error`
       # callback for the connection handler. The `on_error` callback
       # is call in the event of any error(s). The
-      # `@handling_event_count` is decremented by `1` when the data is
+      # `@in_progress[:events]` is decremented by `1` when the data is
       # transmitted successfully, `on_success`.
       #
       # @param handler [Hash] definition.
@@ -57,7 +57,7 @@ module Sensu
         begin
           EM::connect(handler[:socket][:host], handler[:socket][:port], Socket) do |socket|
             socket.on_success = Proc.new do
-              @handling_event_count -= 1 if @handling_event_count
+              @in_progress[:events] -= 1 if @in_progress
             end
             socket.on_error = on_error
             timeout = handler[:timeout] || 10
@@ -71,7 +71,7 @@ module Sensu
       end
 
       # Transmit event data to a UDP socket, then close the
-      # connection. The `@handling_event_count` is decremented by `1`
+      # connection. The `@in_progress[:events]` is decremented by `1`
       # when the data is assumed to have been transmitted.
       #
       # @param handler [Hash] definition.
@@ -81,7 +81,7 @@ module Sensu
           EM::open_datagram_socket("0.0.0.0", 0, nil) do |socket|
             socket.send_datagram(event_data.to_s, handler[:socket][:host], handler[:socket][:port])
             socket.close_connection_after_writing
-            @handling_event_count -= 1 if @handling_event_count
+            @in_progress[:events] -= 1 if @in_progress
           end
         rescue => error
           handler_error(handler, event_data).call(error)
@@ -90,7 +90,7 @@ module Sensu
 
       # Publish event data to a Sensu transport pipe. Event data that
       # is `nil` or empty will not be published, to prevent transport
-      # errors. The `@handling_event_count` is decremented by `1`,
+      # errors. The `@in_progress[:events]` is decremented by `1`,
       # even if the event data is not published.
       #
       # @param handler [Hash] definition.
@@ -105,14 +105,14 @@ module Sensu
             end
           end
         end
-        @handling_event_count -= 1 if @handling_event_count
+        @in_progress[:events] -= 1 if @in_progress
       end
 
       # Run a handler extension, within the Sensu EventMachine reactor
       # (event loop). The extension API `safe_run()` method is used to
       # guard against most errors. The `safe_run()` callback is always
       # called, logging the extension run output and status, and
-      # decrementing the `@handling_event_count` by `1`.
+      # decrementing the `@in_progress[:events]` by `1`.
       #
       # @param handler [Hash] definition.
       # @param event_data [Object] to pass to the handler extension.
@@ -123,7 +123,7 @@ module Sensu
             :output => output,
             :status => status
           })
-          @handling_event_count -= 1 if @handling_event_count
+          @in_progress[:events] -= 1 if @in_progress
         end
       end
 

data/lib/sensu/server/mutate.rb

@@ -6,7 +6,7 @@ module Sensu
       # created callback can be used for standard mutators and mutator
       # extensions. The provided callback will only be called when the
       # mutator status is `0` (OK). If the status is not `0`, an error
-      # is logged, and the `@handling_event_count` is decremented by
+      # is logged, and the `@in_progress[:events]` is decremented by
       # `1`.
       #
       # @param mutator [Object] definition or extension.
@@ -25,7 +25,7 @@ module Sensu
               :output => output,
               :status => status
             })
-            @handling_event_count -= 1 if @handling_event_count
+            @in_progress[:events] -= 1 if @in_progress
           end
         end
       end
@@ -63,7 +63,7 @@ module Sensu
       # mutator is used, unless the handler specifies another mutator.
       # If a mutator does not exist, not defined or a missing
       # extension, an error will be logged and the
-      # `@handling_event_count` is decremented by `1`. This method
+      # `@in_progress[:events]` is decremented by `1`. This method
       # first checks for the existence of a standard mutator, then
       # checks for an extension if a standard mutator is not defined.
       #
@@ -84,7 +84,7 @@ module Sensu
           @logger.error("unknown mutator", {
             :mutator_name => mutator_name
           })
-          @handling_event_count -= 1 if @handling_event_count
+          @in_progress[:events] -= 1 if @in_progress
         end
       end
     end

data/lib/sensu/server/process.rb

@@ -11,7 +11,9 @@ module Sensu
       include Mutate
       include Handle
 
-      attr_reader :is_leader, :handling_event_count
+      attr_reader :is_leader, :in_progress
+
+      STANDARD_CHECK_TYPE = "standard".freeze
 
       METRIC_CHECK_TYPE = "metric".freeze
 
@@ -40,17 +42,23 @@ module Sensu
         super
         @is_leader = false
         @timers[:leader] = Array.new
-        @handling_event_count = 0
+        @in_progress = Hash.new(0)
       end
 
       # Set up the Redis and Transport connection objects, `@redis`
-      # and `@transport`. This method "drys" up many instances of
-      # `setup_redis()` and `setup_transport()`.
+      # and `@transport`. This method updates the Redis on error
+      # callback to reset the in progress check result counter. This
+      # method "drys" up many instances of `setup_redis()` and
+      # `setup_transport()`, particularly in the specs.
       #
       # @yield callback/block called after connecting to Redis and the
       #   Sensu Transport.
       def setup_connections
         setup_redis do
+          @redis.on_error do |error|
+            @logger.error("redis connection error", :error => error.to_s)
+            @in_progress[:check_results] = 0
+          end
           setup_transport do
             yield
           end
@@ -90,15 +98,11 @@ module Sensu
       #
       # @param client [Hash] definition.
       def create_client_registration_event(client)
-        event = {
-          :id => random_uuid,
-          :client => client,
-          :check => create_registration_check(client),
-          :occurrences => 1,
-          :action => :create,
-          :timestamp => Time.now.to_i
-        }
-        process_event(event)
+        check = create_registration_check(client)
+        create_event(client, check) do |event|
+          event_bridges(event)
+          process_event(event)
+        end
       end
 
       # Process an initial client registration, when it is first added
@@ -235,8 +239,8 @@ module Sensu
       #
       # This method determines the appropriate handlers for an event,
       # filtering and mutating the event data for each of them. The
-      # `@handling_event_count` is incremented by `1`, for each event
-      # handler chain (filter -> mutate -> handle).
+      # `@in_progress[:events]` counter is incremented by `1`, for
+      # each event handler chain (filter -> mutate -> handle).
       #
       # @param event [Hash]
       def process_event(event)
@@ -245,7 +249,7 @@ module Sensu
         handler_list = Array((event[:check][:handlers] || event[:check][:handler]) || DEFAULT_HANDLER_NAME)
         handlers = derive_handlers(handler_list)
         handlers.each do |handler|
-          @handling_event_count += 1
+          @in_progress[:events] += 1
           filter_event(handler, event) do |event|
             mutate_event(handler, event) do |event_data|
               handle_event(handler, event_data)
@@ -271,35 +275,27 @@ module Sensu
         end
       end
 
-      # Add a check result to an aggregate. A check aggregate uses the
-      # check `:name` and the `:issued` timestamp as its unique
-      # identifier. An aggregate uses several counters: the total
-      # number of results in the aggregate, and a counter for each
-      # check severity (ok, warning, etc). Check output is also
-      # stored, to be summarized to aid in identifying outliers for a
-      # check execution across a number of Sensu clients. JSON
-      # serialization is used for storing check result data.
+      # Add a check result to an aggregate. The aggregate name is
+      # determined by the value of check `:aggregate`. If check
+      # `:aggregate` is `true` (legacy), the check `:name` is used as
+      # the aggregate name. If check `:aggregate` is a string, it is
+      # used as the aggregate name. This method will add the client
+      # name to the aggregate, all other processing (e.g. counters) is
+      # done by the Sensu API on request.
       #
       # @param client [Hash]
       # @param check [Hash]
       def aggregate_check_result(client, check)
+        aggregate = (check[:aggregate].is_a?(String) ? check[:aggregate] : check[:name])
         @logger.debug("adding check result to aggregate", {
+          :aggregate => aggregate,
           :client => client,
           :check => check
         })
-        result_set = "#{check[:name]}:#{check[:issued]}"
-        result_data = Sensu::JSON.dump(:output => check[:output], :status => check[:status])
-        @redis.multi
-        @redis.hset("aggregation:#{result_set}", client[:name], result_data)
-        SEVERITIES.each do |severity|
-          @redis.hsetnx("aggregate:#{result_set}", severity, 0)
+        aggregate_member = "#{client[:name]}:#{check[:name]}"
+        @redis.sadd("aggregates:#{aggregate}", aggregate_member) do
+          @redis.sadd("aggregates", aggregate)
         end
-        severity = (SEVERITIES[check[:status]] || "unknown")
-        @redis.hincrby("aggregate:#{result_set}", severity, 1)
-        @redis.hincrby("aggregate:#{result_set}", "total", 1)
-        @redis.sadd("aggregates:#{check[:name]}", check[:issued])
-        @redis.sadd("aggregates", check[:name])
-        @redis.exec
       end
 
       # Truncate check output. For metric checks, (`"type":
@@ -333,7 +329,7 @@ module Sensu
       # @param client [Hash]
       # @param check [Hash]
       # @yield [] callback/block called after the check data has been
-      # stored (history, etc).
+      #   stored (history, etc).
       def store_check_result(client, check)
         @logger.debug("storing check result", :check => check)
         result_key = "#{client[:name]}:#{check[:name]}"
@@ -342,6 +338,7 @@ module Sensu
         @redis.multi
         @redis.sadd("result:#{client[:name]}", check[:name])
         @redis.set("result:#{result_key}", Sensu::JSON.dump(check_truncated))
+        @redis.sadd("ttl", result_key) if check[:ttl]
         @redis.rpush(history_key, check[:status])
         @redis.ltrim(history_key, -21, -1)
         @redis.exec do
@@ -418,83 +415,106 @@ module Sensu
       end
 
       # Update the event registry, stored in Redis. This method
-      # determines if check data results in the creation or update of
-      # event data in the registry. Existing event data for a
-      # client/check pair is fetched, used in conditionals and the
-      # composition of the new event data. If a check `:status` is not
+      # determines if event data warrants in the creation or update of
+      # event data in the registry. If a check `:status` is not
       # `0`, or it has been flapping, an event is created/updated in
-      # the registry. If there was existing event data, but the check
-      # `:status` is now `0`, the event is removed (resolved) from the
-      # registry. If the previous conditions are not met, and check
-      # `:type` is `metric` and the `:status` is `0`, the event
-      # registry is not updated, but the provided callback is called
-      # with the event data. All event data is sent to event bridge
-      # extensions, including events that do not normally produce an
-      # action. JSON serialization is used when storing data in the
-      # registry.
+      # the registry. If the event `:action` is `:resolve`, the event
+      # is removed (resolved) from the registry. If the previous
+      # conditions are not met and check `:type` is `metric`, the
+      # registry is not updated, but further event processing is
+      # required (`yield(true)`). JSON serialization is used when
+      # storing data in the registry.
+      #
+      # @param event [Hash]
+      # @yield callback [event] callback/block called after the event
+      #   registry has been updated.
+      # @yieldparam process [TrueClass, FalseClass] indicating if the
+      #   event requires further processing.
+      def update_event_registry(event)
+        client_name = event[:client][:name]
+        if event[:check][:status] != 0 || event[:action] == :flapping
+          @redis.hset("events:#{client_name}", event[:check][:name], Sensu::JSON.dump(event)) do
+            yield(true)
+          end
+        elsif event[:action] == :resolve &&
+            (event[:check][:auto_resolve] != false || event[:check][:force_resolve])
+          @redis.hdel("events:#{client_name}", event[:check][:name]) do
+            yield(true)
+          end
+        elsif event[:check][:type] == METRIC_CHECK_TYPE
+          yield(true)
+        else
+          yield(false)
+        end
+      end
+
+      # Create an event, using the provided client and check result
+      # data. Existing event data for the client/check pair is fetched
+      # from the event registry to be used in the composition of the
+      # new event.
       #
       # @param client [Hash]
       # @param check [Hash]
       # @yield callback [event] callback/block called with the
-      #   resulting event data if the event registry is updated, or
-      #   the check is of type `:metric`.
+      #   resulting event.
       # @yieldparam event [Hash]
-      def update_event_registry(client, check)
-        @redis.hget("events:#{client[:name]}", check[:name]) do |event_json|
-          stored_event = event_json ? Sensu::JSON.load(event_json) : nil
-          flapping = check_flapping?(stored_event, check)
-          event = {
-            :id => random_uuid,
-            :client => client,
-            :check => check,
-            :occurrences => 1,
-            :action => (flapping ? :flapping : :create),
-            :timestamp => Time.now.to_i
-          }
-          if check[:status] != 0 || flapping
-            if stored_event && check[:status] == stored_event[:check][:status]
-              event[:occurrences] = stored_event[:occurrences] + 1
-            end
-            @redis.hset("events:#{client[:name]}", check[:name], Sensu::JSON.dump(event)) do
-              yield(event)
+      def create_event(client, check)
+        check_history(client, check) do |history, total_state_change|
+          check[:history] = history
+          check[:total_state_change] = total_state_change
+          @redis.hget("events:#{client[:name]}", check[:name]) do |event_json|
+            stored_event = event_json ? Sensu::JSON.load(event_json) : nil
+            flapping = check_flapping?(stored_event, check)
+            event = {
+              :client => client,
+              :check => check,
+              :occurrences => 1,
+              :action => (flapping ? :flapping : :create),
+              :timestamp => Time.now.to_i
+            }
+            if stored_event
+              event[:id] = stored_event[:id]
+              event[:last_state_change] = stored_event[:last_state_change]
+              event[:last_ok] = stored_event[:last_ok]
+              event[:occurrences] = stored_event[:occurrences]
+            else
+              event[:id] = random_uuid
             end
-          elsif stored_event
-            event[:occurrences] = stored_event[:occurrences]
-            event[:action] = :resolve
-            unless check[:auto_resolve] == false && !check[:force_resolve]
-              @redis.hdel("events:#{client[:name]}", check[:name]) do
-                yield(event)
+            if check[:status] != 0 || flapping
+              if history[-1] == history[-2]
+                event[:occurrences] += 1
+              else
+                event[:occurrences] = 1
+                event[:last_state_change] = event[:timestamp]
               end
+            elsif stored_event
+              event[:last_state_change] = event[:timestamp]
+              event[:action] = :resolve
+            end
+            if check[:status] == 0
+              event[:last_ok] = event[:timestamp]
             end
-          elsif check[:type] == METRIC_CHECK_TYPE
             yield(event)
           end
-          event_bridges(event)
         end
       end
 
-      # Create a blank client (data) and add it to the client
-      # registry. Only the client name is known, the other client
-      # attributes must be updated via the API (POST /clients:client).
-      # Dynamically created clients and those updated via the API will
-      # have client keepalives disabled, `:keepalives` is set to
-      # `false`.
+      # Create a blank client (data). Only the client name is known,
+      # the other client attributes must be updated via the API (POST
+      # /clients:client). Dynamically created clients and those
+      # updated via the API will have client keepalives disabled by
+      # default, `:keepalives` is set to `false`.
       #
-      # @param name [Hash] to use for the client.
-      # @yield [client] callback/block to be called with the
-      #   dynamically created client data.
-      # @yieldparam client [Hash]
+      # @param name [String] to use for the client.
+      # @return [Hash] client.
       def create_client(name)
-        client = {
+        {
           :name => name,
           :address => "unknown",
           :subscriptions => [],
           :keepalives => false,
           :version => VERSION
         }
-        update_client_registry(client) do
-          yield(client)
-        end
       end
 
       # Retrieve a client (data) from Redis if it exists. If a client
@@ -527,7 +547,8 @@ module Sensu
               yield(client)
             end
           else
-            create_client(client_key) do |client|
+            client = create_client(client_key)
+            update_client_registry(client) do
               yield(client)
             end
           end
@@ -536,14 +557,18 @@ module Sensu
 
       # Process a check result, storing its data, inspecting its
       # contents, and taking the appropriate actions (eg. update the
-      # event registry). A check result must have a valid client name,
-      # associated with a client in the registry or one will be
-      # created. If a local check definition exists for the check
-      # name, and the check result is not from a standalone check
-      # execution, it's merged with the check result for more context.
+      # event registry). The `@in_progress[:check_results]` counter is
+      # incremented by `1` prior to check result processing and then
+      # decremented by `1` after updating the event registry. A check
+      # result must have a valid client name, associated with a client
+      # in the registry or one will be created. If a local check
+      # definition exists for the check name, and the check result is
+      # not from a standalone check execution, it's merged with the
+      # check result for more context.
       #
       # @param result [Hash] data.
       def process_check_result(result)
+        @in_progress[:check_results] += 1
         @logger.debug("processing result", :result => result)
         retrieve_client(result) do |client|
           check = case
@@ -552,13 +577,15 @@ module Sensu
           else
             result[:check]
           end
+          check[:type] ||= STANDARD_CHECK_TYPE
+          check[:origin] = result[:client] if check[:source]
           aggregate_check_result(client, check) if check[:aggregate]
           store_check_result(client, check) do
-            check_history(client, check) do |history, total_state_change|
-              check[:history] = history
-              check[:total_state_change] = total_state_change
-              update_event_registry(client, check) do |event|
-                process_event(event)
+            create_event(client, check) do |event|
+              event_bridges(event)
+              update_event_registry(event) do |process|
+                process_event(event) if process
+                @in_progress[:check_results] -= 1
               end
             end
           end
@@ -755,45 +782,72 @@ module Sensu
         check.merge(:name => "keepalive", :issued => timestamp, :executed => timestamp)
       end
 
+      # Create client keepalive check results. This method will
+      # retrieve clients from the registry, creating a keepalive
+      # check definition for each client, using the
+      # `create_keepalive_check()` method, containing client specific
+      # keepalive thresholds. If the time since the latest keepalive
+      # is equal to or greater than a threshold, the check `:output`
+      # is set to a descriptive message, and `:status` is set to the
+      # appropriate non-zero value. If a client has been sending
+      # keepalives, `:output` and `:status` are set to indicate an OK
+      # state. The `publish_check_result()` method is used to publish
+      # the client keepalive check results.
+      #
+      # @param clients [Array] of client names.
+      # @yield [] callback/block called after the client keepalive
+      #   check results have been created.
+      def create_client_keepalive_check_results(clients)
+        client_keys = clients.map { |client_name| "client:#{client_name}" }
+        @redis.mget(*client_keys) do |client_json_objects|
+          client_json_objects.each do |client_json|
+            unless client_json.nil?
+              client = Sensu::JSON.load(client_json)
+              next if client[:keepalives] == false
+              check = create_keepalive_check(client)
+              time_since_last_keepalive = Time.now.to_i - client[:timestamp]
+              check[:output] = "No keepalive sent from client for "
+              check[:output] << "#{time_since_last_keepalive} seconds"
+              case
+              when time_since_last_keepalive >= check[:thresholds][:critical]
+                check[:output] << " (>=#{check[:thresholds][:critical]})"
+                check[:status] = 2
+              when time_since_last_keepalive >= check[:thresholds][:warning]
+                check[:output] << " (>=#{check[:thresholds][:warning]})"
+                check[:status] = 1
+              else
+                check[:output] = "Keepalive sent from client "
+                check[:output] << "#{time_since_last_keepalive} seconds ago"
+                check[:status] = 0
+              end
+              publish_check_result(client[:name], check)
+            end
+          end
+          yield
+        end
+      end
+
       # Determine stale clients, those that have not sent a keepalive
-      # in a specified amount of time (thresholds). This method
-      # iterates through the client registry, creating a keepalive
-      # check definition with the `create_keepalive_check()` method,
-      # containing client specific staleness thresholds. If the time
-      # since the latest keepalive is equal to or greater than a
-      # threshold, the check `:output` is set to a descriptive
-      # message, and `:status` is set to the appropriate non-zero
-      # value. If a client has been sending keepalives, `:output` and
-      # `:status` are set to indicate an OK state. A check result is
-      # published for every client in the registry.
+      # in a specified amount of time. This method iterates through
+      # the client registry, creating a keepalive check result for
+      # each client. The `create_client_keepalive_check_results()`
+      # method is used to inspect and create keepalive check results
+      # for each slice of clients from the registry. A relatively
+      # small clients slice size (20) is used to reduce the number of
+      # clients inspected within a single tick of the EM reactor.
       def determine_stale_clients
         @logger.info("determining stale clients")
         @redis.smembers("clients") do |clients|
-          clients.each do |client_name|
-            @redis.get("client:#{client_name}") do |client_json|
-              unless client_json.nil?
-                client = Sensu::JSON.load(client_json)
-                next if client[:keepalives] == false
-                check = create_keepalive_check(client)
-                time_since_last_keepalive = Time.now.to_i - client[:timestamp]
-                check[:output] = "No keepalive sent from client for "
-                check[:output] << "#{time_since_last_keepalive} seconds"
-                case
-                when time_since_last_keepalive >= check[:thresholds][:critical]
-                  check[:output] << " (>=#{check[:thresholds][:critical]})"
-                  check[:status] = 2
-                when time_since_last_keepalive >= check[:thresholds][:warning]
-                  check[:output] << " (>=#{check[:thresholds][:warning]})"
-                  check[:status] = 1
-                else
-                  check[:output] = "Keepalive sent from client "
-                  check[:output] << "#{time_since_last_keepalive} seconds ago"
-                  check[:status] = 0
-                end
-                publish_check_result(client[:name], check)
+          client_count = clients.length
+          keepalive_check_results = Proc.new do |slice_start, slice_size|
+            unless slice_start > client_count - 1
+              clients_slice = clients.slice(slice_start..slice_size)
+              create_client_keepalive_check_results(clients_slice) do
+                keepalive_check_results.call(slice_start + 20, slice_size + 20)
               end
             end
           end
+          keepalive_check_results.call(0, 19)
         end
       end
 
@@ -809,32 +863,29 @@ module Sensu
 
       # Determine stale check results, those that have not executed in
       # a specified amount of time (check TTL). This method iterates
-      # through the client registry and check results for checks with
-      # a defined TTL value (in seconds). If a check result has a
-      # defined TTL, the time since last check execution (in seconds)
-      # is calculated. If the time since last execution is equal to or
-      # greater than the check TTL, a warning check result is
-      # published with the appropriate check output.
+      # through stored check results that have a defined TTL value (in
+      # seconds). The time since last check execution (in seconds) is
+      # calculated for each check result. If the time since last
+      # execution is equal to or greater than the check TTL, a warning
+      # check result is published with the appropriate check output.
       def determine_stale_check_results
         @logger.info("determining stale check results")
-        @redis.smembers("clients") do |clients|
-          clients.each do |client_name|
-            @redis.smembers("result:#{client_name}") do |checks|
-              checks.each do |check_name|
-                result_key = "#{client_name}:#{check_name}"
-                @redis.get("result:#{result_key}") do |result_json|
-                  unless result_json.nil?
-                    check = Sensu::JSON.load(result_json)
-                    next unless check[:ttl] && check[:executed] && !check[:force_resolve]
-                    time_since_last_execution = Time.now.to_i - check[:executed]
-                    if time_since_last_execution >= check[:ttl]
-                      check[:output] = "Last check execution was "
-                      check[:output] << "#{time_since_last_execution} seconds ago"
-                      check[:status] = 1
-                      publish_check_result(client_name, check)
-                    end
-                  end
+        @redis.smembers("ttl") do |result_keys|
+          result_keys.each do |result_key|
+            @redis.get("result:#{result_key}") do |result_json|
+              unless result_json.nil?
+                check = Sensu::JSON.load(result_json)
+                next unless check[:ttl] && check[:executed] && !check[:force_resolve]
+                time_since_last_execution = Time.now.to_i - check[:executed]
+                if time_since_last_execution >= check[:ttl]
+                  client_name = result_key.split(":").first
+                  check[:output] = "Last check execution was "
+                  check[:output] << "#{time_since_last_execution} seconds ago"
+                  check[:status] = 1
+                  publish_check_result(client_name, check)
                 end
+              else
+                @redis.srem("ttl", result_key)
               end
             end
           end
@@ -851,48 +902,6 @@ module Sensu
         end
       end
 
-      # Prune check result aggregations (aggregates). Sensu only
-      # stores the 20 latest aggregations for a check, to keep the
-      # amount of data stored to a minimum.
-      def prune_check_result_aggregations
-        @logger.info("pruning check result aggregations")
-        @redis.smembers("aggregates") do |checks|
-          checks.each do |check_name|
-            @redis.smembers("aggregates:#{check_name}") do |aggregates|
-              if aggregates.length > 20
-                aggregates.sort!
-                aggregates.take(aggregates.length - 20).each do |check_issued|
-                  result_set = "#{check_name}:#{check_issued}"
-                  @redis.multi
-                  @redis.srem("aggregates:#{check_name}", check_issued)
-                  @redis.del("aggregate:#{result_set}")
-                  @redis.del("aggregation:#{result_set}")
-                  @redis.exec do
-                    @logger.debug("pruned aggregation", {
-                      :check => {
-                        :name => check_name,
-                        :issued => check_issued
-                      }
-                    })
-                  end
-                end
-              end
-            end
-          end
-        end
-      end
-
-      # Set up the check result aggregation pruner, using periodic
-      # timer to run `prune_check_result_aggregations()` every 20
-      # seconds. The timer is stored in the timers hash under
-      # `:leader`.
-      def setup_check_result_aggregation_pruner
-        @logger.debug("pruning check result aggregations")
-        @timers[:leader] << EM::PeriodicTimer.new(20) do
-          prune_check_result_aggregations
-        end
-      end
-
       # Set up the leader duties, tasks only performed by a single
       # Sensu server at a time. The duties include publishing check
       # requests, monitoring for stale clients, and pruning check
@@ -901,7 +910,6 @@ module Sensu
         setup_check_request_publisher
         setup_client_monitor
         setup_check_result_monitor
-        setup_check_result_aggregation_pruner
       end
 
       # Create a lock timestamp (integer), current time including
@@ -1040,19 +1048,16 @@ module Sensu
         @transport.unsubscribe if @transport
       end
 
-      # Complete event handling currently in progress. The
-      # `:handling_event_count` is used to determine if event handling
-      # is complete, when it is equal to `0`. The provided callback is
-      # called when handling is complete.
+      # Complete in progress work and then call the provided callback.
+      # This method will wait until all counters stored in the
+      # `@in_progress` hash equal `0`.
       #
-      # @yield [] callback/block to call when event handling is
-      #   complete.
-      def complete_event_handling
-        @logger.info("completing event handling in progress", {
-          :handling_event_count => @handling_event_count
-        })
+      # @yield [] callback/block to call when in progress work is
+      #   completed.
+      def complete_in_progress
+        @logger.info("completing work in progress", :in_progress => @in_progress)
         retry_until_true do
-          if @handling_event_count == 0
+          if @in_progress.values.all? { |count| count == 0 }
             yield
             true
           end
@@ -1124,7 +1129,7 @@ module Sensu
         @logger.warn("stopping")
         pause
         @state = :stopping
-        complete_event_handling do
+        complete_in_progress do
           @redis.close if @redis
           @transport.close if @transport
           super