sensu 0.28.6 → 0.29.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a1ea1cfa413a927a75a9acdbfb0a6ab81c7d8b9f
-  data.tar.gz: fd3e79ae3331d1128f69d4830e1cd42e17535ddd
+  metadata.gz: 31731ea9a2311f9aecc12057e03442fb2dffca7e
+  data.tar.gz: ee7b95c65a89a73693717c5220bf7a515ee54634
 SHA512:
-  metadata.gz: 230f849a14b5f777a9a022b8282dc438098f468e36bd0b0b016393927945244b6e30374a9e2362223bf6bded7692ac826380f836a04f3f77bf6997ec8ee6e9ef
-  data.tar.gz: fed7831e688c6d1743f99bbba2cfa30daa719b8dfd2c3f5ba82890d9b61a995a0a4f6db09fdf81f0fc249eb28a1bf3cc989557cb98584447a4349987cf2a5c31
+  metadata.gz: b1a0e54b2e28998c7653b78f54278ec6f28c2a716fedfc8ab24b59108d33ff9998d60b77776c05e48e556b1decf43c15a2c95ee06842528ae3671cbd7957349a
+  data.tar.gz: d7ed4d7d2cb4794ab2933237d383eb37331dda2adaed120117c25ce36cacb7d4a4cd6e8249439d5f08c1ed52e2e63b5e48e2594603078bcf9f120ca6a52d936b

data/CHANGELOG.md

@@ -1,3 +1,74 @@
+## 0.29.0 - 2017-03-29
+
+### Features
+
+Sensu server tasks, replacing the Sensu server leader functionality,
+distributing certain server responsibilities amongst the running Sensu
+servers. A server task can only run on one Sensu server at a time. Sensu
+servers partake in an election process to become responsible for one or
+more tasks. A task can failover to another Sensu server.
+
+Sensu API response object filtering for any GET request. Filtering is done
+with one or more dot notation query parameters, beginning with `filter.`,
+to specify object attributes to filter by, e.g.
+`/events?filter.client.environment=production&filter.check.contact=ops`.
+
+Added API endpoint GET `/settings` to provided the APIs running
+configuration. Sensitive setting values are redacted by default, unless
+the query parameter `redacted` is set to `false`, e.g.
+`/settings?redacted=false`.
+
+Added support for invalidating a Sensu client when deleting it via the
+Sensu API DELETE `/clients/:name` endpoint, disallowing further client
+keepalives and check results until the client is either successfully
+removed from the client registry or for a specified duration of time. To
+invalidate a Sensu client until it is deleted, the query parameter
+`invalidate` must be set to `true`, e.g.
+`/clients/app01.example.com?invalidate=true`. To invalidate the client for
+a certain amount of time (in seconds), the query parameter
+`invalidate_expire` must be set as well, e.g.
+`/clients/app01.example.com?invalidate=true&invalidate_expire=300`.
+
+Added a Sensu settings hexdigest, exposed via the Sensu API GET `/info`
+endpoint, providing a means to determine if a Sensu server's configuration
+differs from the rest.
+
+Added a proxy argument to `sensu-install`. To use a proxy for Sensu plugin
+and extension installation with `sensu-install`, use the `-x` or
+`--proxy` argument, e.g. `sensu-install -e statsd --proxy
+http://proxy.example.com:8080`.
+
+Added support for issuing proxy check requests via the Sensu API POST
+`/request` endpoint.
+
+The Sensu API now logs response times.
+
+The Sensu API now returns a 405 (Method Not Allowed) when an API endpoint
+does not support a HTTP request method, e.g. `PUT`, and sets the HTTP
+header "Allow" to indicate which HTTP request methods are supported by the
+requested endpoint.
+
+Added a built-in filter for check dependencies, `check_dependencies`,
+which implements the check dependency filtering logic in the Sensu Plugin
+library.
+
+Added default values for Sensu CLI options `config_file`
+(`"/etc/sensu/config.json"`) and `config_dirs` (`["/etc/sensu/conf.d"]`).
+These defaults are only applied when the associated file and/or directory
+exist.
+
+### Other
+
+Added a Rubocop configuration file and rake tasks for a slow introduction.
+
+### Fixes
+
+The built-in filter `occurrences` now supports `refresh` for flapping
+events (action `flapping`).
+
+Force the configured Redis port to be an integer, as some users make the
+mistake of using a string.
+
 ## 0.28.5 - 2017-03-23
 
 ### Fixes

data/exe/sensu-install

@@ -10,7 +10,8 @@ module Sensu
           :verbose => false,
           :plugins => [],
           :extensions => [],
-          :clean => false
+          :clean => false,
+          :proxy => nil
         }
         optparse = OptionParser.new do |opts|
           opts.on("-h", "--help", "Display this message") do
@@ -38,6 +39,9 @@ module Sensu
           opts.on("-c", "--clean", "Clean up (remove) other installed versions of the plugin(s) and/or extension(s)") do
             options[:clean] = true
           end
+          opts.on("-x", "--proxy PROXY", "Install Sensu plugins and extensions via a PROXY URL") do |proxy|
+            options[:proxy] = proxy
+          end
         end
         optparse.parse!(arguments)
         options
@@ -70,6 +74,7 @@ module Sensu
         gem_command << " --no-ri --no-rdoc"
         gem_command << " --verbose" if options[:verbose]
         gem_command << " --source #{options[:source]}" if options[:source]
+        gem_command << " --http-proxy #{options[:proxy]}" if options[:proxy]
         log gem_command if options[:verbose]
         unless system(gem_command)
           log "failed to install Sensu gem '#{gem_name}'"

data/lib/sensu/api/http_handler.rb

@@ -1,4 +1,5 @@
 require "sensu/api/routes"
+require "sensu/api/utilities/filter_response_content"
 
 gem "em-http-server", "0.1.8"
 
@@ -9,6 +10,7 @@ module Sensu
   module API
     class HTTPHandler < EM::HttpServer::Server
       include Routes
+      include Utilities::FilterResponseContent
 
       attr_accessor :logger, :settings, :redis, :transport
 
@@ -19,6 +21,7 @@ module Sensu
       # @result [Hash]
       def request_details
         return @request_details if @request_details
+        @request_start_time = Time.now.to_f
         _, remote_address = Socket.unpack_sockaddr_in(get_peername)
         @request_details = {
           :remote_address => remote_address,
@@ -40,12 +43,14 @@ module Sensu
         @logger.debug("api request", request_details)
       end
 
-      # Log the HTTP response.
+      # Log the HTTP response. This method calculates the
+      # request/response time.
       def log_response
         @logger.info("api response", {
           :request => request_details,
           :status => @response.status,
-          :content_length => @response.content.to_s.bytesize
+          :content_length => @response.content.to_s.bytesize,
+          :time => (Time.now.to_f - @request_start_time).round(3)
         })
       end
 
@@ -61,13 +66,20 @@ module Sensu
 
       # Parse the HTTP request query string for parameters. This
       # method creates `@params`, a hash of parsed query parameters,
-      # used by the API routes.
+      # used by the API routes. This method also creates
+      # `@filter_params`, a hash of parsed response content filter
+      # parameters.
       def parse_parameters
         @params = {}
         if @http_query_string
           @http_query_string.split("&").each do |pair|
             key, value = pair.split("=")
             @params[key.to_sym] = value
+            if key.start_with?("filter.")
+              filter_param = key.sub(/^filter\./, "")
+              @filter_params ||= {}
+              @filter_params[filter_param] = value
+            end
           end
         end
       end
@@ -166,13 +178,17 @@ module Sensu
       # Respond to an HTTP request. The routes set `@response_status`,
       # `@response_status_string`, and `@response_content`
       # appropriately. The HTTP response status defaults to `200` with
-      # the status string `OK`. The Sensu API only returns JSON
-      # response content, `@response_content` is assumed to be a Ruby
-      # object that can be serialized as JSON.
+      # the status string `OK`. If filter params were provided,
+      # `@response_content` is filtered (mutated). The Sensu API only
+      # returns JSON response content, `@response_content` is assumed
+      # to be a Ruby object that can be serialized as JSON.
       def respond
         @response.status = @response_status || 200
         @response.status_string = @response_status_string || "OK"
         if @response_content && @http_request_method != HEAD_METHOD
+          if @http_request_method == GET_METHOD && @filter_params
+            filter_response_content!
+          end
           @response.content_type "application/json"
           @response.content = Sensu::JSON.dump(@response_content)
         end
@@ -281,6 +297,14 @@ module Sensu
         respond
       end
 
+      # Respond to the HTTP request with a `405` (Method Not Allowed) response.
+      def method_not_allowed!(allowed_http_methods=[])
+        @response.headers["Allow"] = allowed_http_methods.join(", ")
+        @response_status = 405
+        @response_status_string = "Method Not Allowed"
+        respond
+      end
+
       # Respond to the HTTP request with a `412` (Precondition Failed)
       # response.
       def precondition_failed!
@@ -297,25 +321,63 @@ module Sensu
         respond
       end
 
+      # Determine the allowed HTTP methods for a route. The route
+      # regular expressions and associated route method calls are
+      # provided by `ROUTES`. This method returns an array of HTTP
+      # methods that have a route that matches the HTTP request URI.
+      #
+      # @return [Array]
+      def allowed_http_methods?
+        ROUTES.map { |http_method, routes|
+          match = routes.detect do |route|
+            @http_request_uri =~ route[0]
+          end
+          match ? http_method : nil
+        }.flatten.compact
+      end
+
+      # Determine the route method for the HTTP request method and
+      # URI. The route regular expressions and associated route method
+      # calls are provided by `ROUTES`. This method will return the
+      # first route method name (Ruby symbol) that has matching URI
+      # regular expression. If an HTTP method is not supported, or
+      # there is not a matching regular expression, `nil` will be
+      # returned.
+      #
+      # @return [Symbol]
+      def determine_route_method
+        if ROUTES.has_key?(@http_request_method)
+          route = ROUTES[@http_request_method].detect do |route|
+            @http_request_uri =~ route[0]
+          end
+          route ? route[1] : nil
+        else
+          nil
+        end
+      end
+
       # Route the HTTP request. OPTIONS HTTP requests will always
-      # return a `200` with no response content. The route regular
-      # expressions and associated route method calls are provided by
-      # `ROUTES`. If a route match is not found, this method responds
-      # with a `404` (Not Found) HTTP response.
+      # return a `200` with no response content. This method uses
+      # `determine_route_method()` to determine the symbolized route
+      # method to send/call. If a route method does not exist for the
+      # HTTP request method and URI, this method uses
+      # `allowed_http_methods?()` to determine if a 404 (Not Found) or
+      # 405 (Method Not Allowed) HTTP response should be used.
       def route_request
         if @http_request_method == OPTIONS_METHOD
           respond
-        elsif ROUTES.has_key?(@http_request_method)
-          route = ROUTES[@http_request_method].detect do |route|
-            @http_request_uri =~ route[0]
-          end
-          unless route.nil?
-            send(route[1])
+        else
+          route_method = determine_route_method
+          if route_method
+            send(route_method)
           else
-            not_found!
+            allowed_http_methods = allowed_http_methods?
+            if allowed_http_methods.empty?
+              not_found!
+            else
+              method_not_allowed!(allowed_http_methods)
+            end
           end
-        else
-          not_found!
         end
       end
 

data/lib/sensu/api/routes.rb

@@ -1,3 +1,4 @@
+require "sensu/api/routes/settings"
 require "sensu/api/routes/info"
 require "sensu/api/routes/health"
 require "sensu/api/routes/clients"
@@ -13,6 +14,7 @@ require "sensu/api/routes/silenced"
 module Sensu
   module API
     module Routes
+      include Settings
       include Info
       include Health
       include Clients
@@ -32,6 +34,7 @@ module Sensu
       OPTIONS_METHOD = "OPTIONS".freeze
 
       GET_ROUTES = [
+        [SETTINGS_URI, :get_settings],
         [INFO_URI, :get_info],
         [HEALTH_URI, :get_health],
         [CLIENTS_URI, :get_clients],

data/lib/sensu/api/routes/clients.rb

@@ -113,8 +113,11 @@ module Sensu
         # DELETE /clients/:client_name
         def delete_client
           client_name = parse_uri(CLIENT_URI).first
-          @redis.get("client:#{client_name}") do |client_json|
+          client_key = "client:#{client_name}"
+          signature_key = "#{client_key}:signature"
+          @redis.get(client_key) do |client_json|
             unless client_json.nil?
+              @redis.set(signature_key, "invalidated") if @params[:invalidate]
               @redis.hgetall("events:#{client_name}") do |events|
                 events.each do |check_name, event_json|
                   resolve_event(event_json)
@@ -125,8 +128,13 @@ module Sensu
                     if events.empty? || attempts == 5
                       @logger.info("deleting client from registry", :client_name => client_name)
                       @redis.srem("clients", client_name) do
-                        @redis.del("client:#{client_name}")
-                        @redis.del("client:#{client_name}:signature")
+                        @redis.del(client_key)
+                        invalidate_expire = integer_parameter(@params[:invalidate_expire])
+                        if @params[:invalidate] && invalidate_expire
+                          @redis.expire(signature_key, invalidate_expire)
+                        else
+                          @redis.del(signature_key)
+                        end
                         @redis.del("events:#{client_name}")
                         @redis.smembers("result:#{client_name}") do |checks|
                           checks.each do |check_name|

data/lib/sensu/api/routes/info.rb

@@ -16,7 +16,10 @@ module Sensu
             servers_info do |servers|
               @response_content = {
                 :sensu => {
-                  :version => VERSION
+                  :version => VERSION,
+                  :settings => {
+                    :hexdigest => @settings.hexdigest
+                  }
                 },
                 :transport => transport,
                 :redis => {

data/lib/sensu/api/routes/request.rb

@@ -26,7 +26,11 @@ module Sensu
                 :reason => data[:reason],
                 :creator => data[:creator]
               }
-              publish_check_request(check)
+              if check[:proxy_requests]
+                publish_proxy_check_requests(check)
+              else
+                publish_check_request(check)
+              end
               @response_content = {:issued => Time.now.to_i}
               accepted!
             else

data/lib/sensu/api/routes/settings.rb

@@ -0,0 +1,23 @@
+require "sensu/utilities"
+
+module Sensu
+  module API
+    module Routes
+      module Settings
+        include Utilities
+
+        SETTINGS_URI = /^\/settings$/
+
+        # GET /settings
+        def get_settings
+          if @params[:redacted] == "false"
+            @response_content = @settings.to_hash
+          else
+            @response_content = redact_sensitive(@settings.to_hash)
+          end
+          respond
+        end
+      end
+    end
+  end
+end

data/lib/sensu/api/utilities/filter_response_content.rb

@@ -0,0 +1,44 @@
+require "sensu/utilities"
+
+module Sensu
+  module API
+    module Utilities
+      module FilterResponseContent
+        include Sensu::Utilities
+
+        # Create a nested hash from a dot notation key and value.
+        #
+        # @param dot_notation [String]
+        # @param value [Object]
+        # @return [Hash]
+        def dot_notation_to_hash(dot_notation, value)
+          hash = {}
+          dot_notation.split(".").reverse.each do |key|
+            if hash.empty?
+              hash = {key.to_sym => value}
+            else
+              hash = {key.to_sym => hash}
+            end
+          end
+          hash
+        end
+
+        # Filter the response content if filter parameters have been
+        # provided. This method mutates `@response_content`, only
+        # retaining array items that match the attributes provided via
+        # filter parameters.
+        def filter_response_content!
+          if @response_content.is_a?(Array) && !@filter_params.empty?
+            attributes = {}
+            @filter_params.each do |key, value|
+              attributes = deep_merge(attributes, dot_notation_to_hash(key, value))
+            end
+            @response_content.select! do |object|
+              attributes_match?(object, attributes, false)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sensu/api/utilities/publish_check_request.rb

@@ -1,7 +1,11 @@
+require "sensu/utilities"
+
 module Sensu
   module API
     module Utilities
       module PublishCheckRequest
+        include Sensu::Utilities
+
         # Determine the Sensu Transport publish options for a
         # subscription. If a subscription begins with a Transport pipe
         # type, either "direct:" or "roundrobin:", the subscription uses
@@ -33,7 +37,10 @@ module Sensu
         #
         # @param check [Hash] definition.
         def publish_check_request(check)
-          payload = check.merge(:issued => Time.now.to_i)
+          payload = check.reject do |key, value|
+            [:subscribers, :interval].include?(key)
+          end
+          payload[:issued] = Time.now.to_i
           @logger.info("publishing check request", {
             :payload => payload,
             :subscribers => check[:subscribers]
@@ -51,6 +58,49 @@ module Sensu
             end
           end
         end
+
+        # Create and publish one or more proxy check requests. This
+        # method iterates through the Sensu client registry for clients
+        # that matched provided proxy request client attributes. A proxy
+        # check request is created for each client in the registry that
+        # matches the proxy request client attributes. Proxy check
+        # requests have their client tokens subsituted by the associated
+        # client attributes values. The check requests are published to
+        # the Transport via `publish_check_request()`.
+        #
+        # @param check [Hash] definition.
+        def publish_proxy_check_requests(check)
+          client_attributes = check[:proxy_requests][:client_attributes]
+          unless client_attributes.empty?
+            @redis.smembers("clients") do |clients|
+              clients.each do |client_name|
+                @redis.get("client:#{client_name}") do |client_json|
+                  unless client_json.nil?
+                    client = Sensu::JSON.load(client_json)
+                    if attributes_match?(client, client_attributes)
+                      @logger.debug("creating a proxy check request", {
+                        :client => client,
+                        :check => check
+                      })
+                      proxy_check, unmatched_tokens = object_substitute_tokens(check.dup, client)
+                      if unmatched_tokens.empty?
+                        proxy_check[:source] ||= client[:name]
+                        publish_check_request(proxy_check)
+                      else
+                        @logger.warn("failed to publish a proxy check request", {
+                          :reason => "unmatched client tokens",
+                          :unmatched_tokens => unmatched_tokens,
+                          :client => client,
+                          :check => check
+                        })
+                      end
+                    end
+                  end
+                end
+              end
+            end
+          end
+        end
       end
     end
   end

data/lib/sensu/cli.rb

@@ -11,6 +11,12 @@ module Sensu
     # @return [Hash] options
     def self.read(arguments=ARGV)
       options = {}
+      if File.exist?("/etc/sensu/config.json")
+        options[:config_file] = "/etc/sensu/config.json"
+      end
+      if Dir.exist?("/etc/sensu/conf.d")
+        options[:config_dirs] = ["/etc/sensu/conf.d"]
+      end
       optparse = OptionParser.new do |opts|
         opts.on("-h", "--help", "Display this message") do
           puts opts
@@ -20,10 +26,10 @@ module Sensu
           puts VERSION
           exit
         end
-        opts.on("-c", "--config FILE", "Sensu JSON config FILE") do |file|
+        opts.on("-c", "--config FILE", "Sensu JSON config FILE. Default: /etc/sensu/config.json (if exists)") do |file|
           options[:config_file] = file
         end
-        opts.on("-d", "--config_dir DIR[,DIR]", "DIR or comma-delimited DIR list for Sensu JSON config files") do |dir|
+        opts.on("-d", "--config_dir DIR[,DIR]", "DIR or comma-delimited DIR list for Sensu JSON config files. Default: /etc/sensu/conf.d (if exists)") do |dir|
           options[:config_dirs] = dir.split(",")
         end
         opts.on("--validate_config", "Validate the compiled configuration and exit") do

data/lib/sensu/constants.rb

@@ -1,7 +1,7 @@
 module Sensu
   unless defined?(Sensu::VERSION)
     # Sensu release version.
-    VERSION = "0.28.6".freeze
+    VERSION = "0.29.0".freeze
 
     # Sensu check severities.
     SEVERITIES = %w[ok warning critical unknown].freeze

data/lib/sensu/daemon.rb

@@ -4,12 +4,12 @@ gem "eventmachine", "1.2.2"
 
 gem "sensu-json", "2.1.0"
 gem "sensu-logger", "1.2.1"
-gem "sensu-settings", "9.9.0"
+gem "sensu-settings", "10.0.0"
 gem "sensu-extension", "1.5.1"
-gem "sensu-extensions", "1.7.1"
+gem "sensu-extensions", "1.9.0"
 gem "sensu-transport", "7.0.2"
 gem "sensu-spawn", "2.2.1"
-gem "sensu-redis", "2.1.0"
+gem "sensu-redis", "2.1.1"
 
 require "time"
 require "uri"
@@ -142,6 +142,7 @@ module Sensu
         @logger.fatal("SENSU NOT RUNNING!")
         exit 2
       end
+      @settings.set_env!
     end
 
     # Load Sensu extensions and log any notices. Set the logger and

data/lib/sensu/server/process.rb

@@ -11,7 +11,9 @@ module Sensu
       include Mutate
       include Handle
 
-      attr_reader :is_leader, :in_progress
+      attr_reader :tasks, :in_progress
+
+      TASKS = ["check_request_publisher", "client_monitor", "check_result_monitor"]
 
       STANDARD_CHECK_TYPE = "standard".freeze
 
@@ -34,14 +36,17 @@ module Sensu
         end
       end
 
-      # Override Daemon initialize() to support Sensu server leader
-      # election and the handling event count.
+      # Override Daemon initialize() to support Sensu server tasks and
+      # the handling event count.
       #
       # @param options [Hash]
       def initialize(options={})
         super
-        @is_leader = false
-        @timers[:leader] = Array.new
+        @tasks = []
+        @timers[:tasks] = {}
+        TASKS.each do |task|
+          @timers[:tasks][task.to_sym] = []
+        end
         @in_progress = Hash.new(0)
       end
 
@@ -150,7 +155,7 @@ module Sensu
             if (signature.nil? || signature.empty?) && client[:signature]
               @redis.set(signature_key, client[:signature])
             end
-            if signature.nil? || signature.empty? || (client[:signature] == signature)
+            if signature.nil? || signature.empty? || client[:signature] == signature
               @redis.multi
               @redis.set(client_key, Sensu::JSON.dump(client))
               @redis.sadd("clients", client[:name])
@@ -429,18 +434,11 @@ module Sensu
       # @return [TrueClass, FalseClass]
       def check_flapping?(stored_event, check)
         if check.has_key?(:low_flap_threshold) && check.has_key?(:high_flap_threshold)
-          if check[:low_flap_threshold].is_a?(Integer) && check[:high_flap_threshold].is_a?(Integer)
-            was_flapping = stored_event && stored_event[:action] == EVENT_FLAPPING_ACTION
-            if was_flapping
-              check[:total_state_change] > check[:low_flap_threshold]
-            else
-              check[:total_state_change] >= check[:high_flap_threshold]
-            end
+          was_flapping = stored_event && stored_event[:action] == EVENT_FLAPPING_ACTION
+          if was_flapping
+            check[:total_state_change] > check[:low_flap_threshold]
           else
-            details = {:check => check}
-            details[:client] = stored_event[:client] if stored_event
-            @logger.error("invalid check flap thresholds", details)
-            false
+            check[:total_state_change] >= check[:high_flap_threshold]
           end
         else
           false
@@ -607,7 +605,10 @@ module Sensu
       # `client_key` as the client name. Dynamically create client
       # data can be updated using the API (POST /clients/:client). If
       # a client does exist and it has a client signature, the check
-      # result must have a matching signature or it is discarded.
+      # result must have a matching signature or it is discarded. If
+      # the client does not exist, but a client signature exists, the
+      # check result must have a matching signature or it is
+      # discarded.
       #
       # @param result [Hash] data.
       # @yield [client] callback/block to be called with client data,
@@ -633,10 +634,20 @@ module Sensu
               yield(client)
             end
           else
-            client = create_client(client_key)
-            client[:type] = "proxy" if result[:check][:source]
-            update_client_registry(client) do
-              yield(client)
+            @redis.get("client:#{client_key}:signature") do |signature|
+              if signature.nil? || signature.empty? || result[:signature] == signature
+                client = create_client(client_key)
+                client[:type] = "proxy" if result[:check][:source]
+                update_client_registry(client) do
+                  yield(client)
+                end
+              else
+                @logger.warn("invalid check result signature", {
+                  :result => result,
+                  :signature => signature
+                })
+                yield(nil)
+              end
             end
           end
         end
@@ -843,18 +854,16 @@ module Sensu
       # creats an EventMachine timer for the request. This method will
       # be called after every check cron request for subsequent
       # requests. The timer is stored in the timer hash under
-      # `:leader`, as check request publishing is a task for only the
-      # Sensu server leader, so it can be cancelled etc. The check
-      # cron request timer object is removed from the timer hash after
-      # the request is published, to stop the timer hash from growing
-      # infinitely.
+      # `:tasks`, so it can be cancelled etc. The check cron request
+      # timer object is removed from the timer hash after the request
+      # is published, to stop the timer hash from growing infinitely.
       #
       # @param check [Hash] definition.
       def schedule_check_cron_request(check)
         cron_time = determine_check_cron_time(check)
-        @timers[:leader] << EM::Timer.new(cron_time) do |timer|
+        @timers[:tasks][:check_request_publisher] << EM::Timer.new(cron_time) do |timer|
           create_check_request_proc(check).call
-          @timers[:leader].delete(timer)
+          @timers[:tasks][:check_request_publisher].delete(timer)
           schedule_check_cron_request(check)
         end
       end
@@ -874,17 +883,16 @@ module Sensu
       # using an intial calculated request splay EventMachine timer
       # and an EventMachine periodic timer for subsequent check
       # requests. The timers are stored in the timers hash under
-      # `:leader`, as check request publishing is a task for only the
-      # Sensu server leader, so they can be cancelled etc.
+      # `:tasks`, so they can be cancelled etc.
       #
       # @param check [Hash] definition.
       def schedule_check_interval_requests(check)
         request_splay = testing? ? 0 : calculate_check_request_splay(check)
         interval = testing? ? 0.5 : check[:interval]
-        @timers[:leader] << EM::Timer.new(request_splay) do
+        @timers[:tasks][:check_request_publisher] << EM::Timer.new(request_splay) do
           create_check_request = create_check_request_proc(check)
           create_check_request.call
-          @timers[:leader] << EM::PeriodicTimer.new(interval, &create_check_request)
+          @timers[:tasks][:check_request_publisher] << EM::PeriodicTimer.new(interval, &create_check_request)
         end
       end
 
@@ -1041,10 +1049,10 @@ module Sensu
 
       # Set up the client monitor, a periodic timer to run
       # `determine_stale_clients()` every 30 seconds. The timer is
-      # stored in the timers hash under `:leader`.
+      # stored in the timers hash under `:tasks`.
       def setup_client_monitor
         @logger.debug("monitoring client keepalives")
-        @timers[:leader] << EM::PeriodicTimer.new(30) do
+        @timers[:tasks][:client_monitor] << EM::PeriodicTimer.new(30) do
           determine_stale_clients
         end
       end
@@ -1087,31 +1095,21 @@ module Sensu
 
       # Set up the check result monitor, a periodic timer to run
       # `determine_stale_check_results()` every 30 seconds. The timer
-      # is stored in the timers hash under `:leader`.
+      # is stored in the timers hash under `:tasks`.
       def setup_check_result_monitor(interval = 30)
         @logger.debug("monitoring check results")
-        @timers[:leader] << EM::PeriodicTimer.new(interval) do
+        @timers[:tasks][:check_result_monitor] << EM::PeriodicTimer.new(interval) do
           determine_stale_check_results(interval)
         end
       end
 
-      # Set up the leader duties, tasks only performed by a single
-      # Sensu server at a time. The duties include publishing check
-      # requests, monitoring for stale clients, and pruning check
-      # result aggregations.
-      def leader_duties
-        setup_check_request_publisher
-        setup_client_monitor
-        setup_check_result_monitor
-      end
-
       # Create a lock timestamp (integer), current time including
-      # milliseconds. This method is used by Sensu server leader
+      # milliseconds. This method is used by Sensu server task
       # election.
       #
       # @return [Integer]
       def create_lock_timestamp
-        (Time.now.to_f * 1000).to_i
+        (Time.now.to_f * 10000000).to_i
       end
 
       # Create/return the unique Sensu server ID for the current
@@ -1122,122 +1120,168 @@ module Sensu
         @server_id ||= random_uuid
       end
 
-      # Become the Sensu server leader, responsible for specific
-      # duties (`leader_duties()`). Unless the current process is
-      # already the leader, this method sets the leader ID stored in
-      # Redis to the unique random server ID for the process. If the
-      # leader ID in Redis is successfully updated, `@is_leader` is
-      # set to true and `leader_duties()` is called to begin the
-      # tasks/duties of the Sensu server leader.
-      def become_the_leader
-        unless @is_leader
-          @redis.set("leader", server_id) do
-            @logger.info("i am now the leader")
-            @is_leader = true
-            leader_duties
+      # Setup a Sensu server task. Unless the current process is
+      # already responsible for the task, this method sets the tasks
+      # server ID stored in Redis to the unique random server ID for
+      # the process. If the tasks server ID is successfully updated,
+      # the task is added to `@tasks` for tracking purposes and the
+      # task setup method is called.
+      #
+      # @param task [String]
+      # @yield callback/block called after setting up the task.
+      def setup_task(task)
+        unless @tasks.include?(task)
+          @redis.set("task:#{task}:server", server_id) do
+            @logger.info("i am now responsible for a server task", :task => task)
+            @tasks << task
+            self.send("setup_#{task}".to_sym)
+            yield if block_given?
           end
         else
-          @logger.debug("i am already the leader")
+          @logger.debug("i am already responsible for a server task", :task => task)
         end
       end
 
-      # Resign as leader, if the current process is the Sensu server
-      # leader. This method cancels and clears the leader timers,
-      # those with references stored in the timers hash under
-      # `:leader`, and `@is_leader` is set to `false`. The leader ID
-      # and leader lock are not removed from Redis, as they will be
-      # updated when another server is elected to be the leader, this
-      # method does not need to handle Redis connectivity issues.
-      def resign_as_leader
-        if @is_leader
-          @logger.warn("resigning as leader")
-          @timers[:leader].each do |timer|
+      # Relinquish a Sensu server task. This method cancels and
+      # clears the associated task timers, those with references
+      # stored in the timers hash under `:tasks`, and removes the task
+      # from `@tasks`. The task server ID and lock are not removed
+      # from Redis, as they will be updated when another server takes
+      # reponsibility for the task, this method does not need to
+      # handle Redis connectivity issues.
+      #
+      # @param task [String]
+      def relinquish_task(task)
+        if @tasks.include?(task)
+          @logger.warn("relinquishing server task", :task => task)
+          @timers[:tasks][task.to_sym].each do |timer|
             timer.cancel
           end
-          @timers[:leader].clear
-          @is_leader = false
+          @timers[:tasks][task.to_sym].clear
+          @tasks.delete(task)
         else
-          @logger.debug("not currently the leader")
+          @logger.debug("not currently responsible for a server task", :task => task)
         end
       end
 
-      # Updates the Sensu server leader lock timestamp. The current
-      # leader ID is retrieved from Redis and compared with the server
-      # ID of the current process to determine if it is still the
-      # Sensu server leader. If the current process is still the
-      # leader, the leader lock timestamp is updated. If the current
-      # process is no longer the leader (regicide),
-      # `resign_as_leader()` is called for cleanup, so there is not
-      # more than one leader.
-      def update_leader_lock
-        @redis.get("leader") do |current_leader_id|
-          if current_leader_id == server_id
-            @redis.set("lock:leader", create_lock_timestamp) do
-              @logger.debug("updated leader lock timestamp")
+      # Relinquish all Sensu server tasks, if any.
+      def relinquish_tasks
+        unless @tasks.empty?
+          @tasks.dup.each do |task|
+            relinquish_task(task)
+          end
+        else
+          @logger.debug("not currently responsible for a server task")
+        end
+      end
+
+      # Updates a Sensu server task lock timestamp. The current task
+      # server ID is retrieved from Redis and compared with the server
+      # ID of the current process to determine if it is still
+      # responsible for the task. If the current process is still
+      # responsible, the task lock timestamp is updated. If the
+      # current process is no longer responsible, `relinquish_task()`
+      # is called for cleanup.
+      #
+      # @param task [String]
+      def update_task_lock(task)
+        @redis.get("task:#{task}:server") do |current_server_id|
+          if current_server_id == server_id
+            @redis.set("lock:task:#{task}", create_lock_timestamp) do
+              @logger.debug("updated task lock timestamp", :task => task)
             end
           else
-            @logger.warn("another sensu server has been elected as leader")
-            resign_as_leader
+            @logger.warn("another sensu server is responsible for the task", :task => task)
+            relinquish_task(task)
           end
         end
       end
 
-      # Request a leader election, a process to determine if the
-      # current process is the Sensu server leader, with its
-      # own/unique duties. A Redis key/value is used as a central
-      # lock, using the "SETNX" Redis command to set the key/value if
-      # it does not exist, using a timestamp for the value. If the
-      # current process was able to create the key/value, it is the
-      # leader, and must do the duties of the leader. If the current
-      # process was not able to create the key/value, but the current
-      # timestamp value is equal to or over 30 seconds ago, the
-      # "GETSET" Redis command is used to set a new timestamp and
-      # fetch the previous value to compare them, to determine if it
-      # was set by the current process. If the current process is able
-      # to set the timestamp value, it becomes the leader.
-      def request_leader_election
-        @redis.setnx("lock:leader", create_lock_timestamp) do |created|
+      # Set up a Sensu server task lock updater. This method uses a
+      # periodic timer to update a task lock timestamp in Redis, every
+      # 10 seconds. If the current process fails to keep the lock
+      # timestamp updated for a task that it is responsible for,
+      # another Sensu server will claim responsibility. This method is
+      # called after task setup.
+      #
+      # @param task [String]
+      def setup_task_lock_updater(task)
+        @timers[:run] << EM::PeriodicTimer.new(10) do
+          update_task_lock(task)
+        end
+      end
+
+      # Request a Sensu server task election, a process to determine
+      # if the current process is to be responsible for the task. A
+      # Redis key/value is used as a central lock, using the "SETNX"
+      # Redis command to set the key/value if it does not exist, using
+      # a timestamp for the value. If the current process was able to
+      # create the key/value, it is elected, and is then responsible
+      # for the task. If the current process was not able to create
+      # the key/value, but the current timestamp value is equal to or
+      # over 30 seconds ago, the "GETSET" Redis command is used to set
+      # a new timestamp and fetch the previous value to compare them,
+      # to determine if it was set by the current process. If the
+      # current process is able to set the timestamp value, it is
+      # elected. If elected, the current process sets up the task and
+      # the associated task lock updater.
+      #
+      # @param task [String]
+      # @yield callback/block called either after being elected and
+      #   setting up the task, or after failing to be elected.
+      def request_task_election(task, &callback)
+        @redis.setnx("lock:task:#{task}", create_lock_timestamp) do |created|
           if created
-            become_the_leader
+            setup_task(task, &callback)
+            setup_task_lock_updater(task)
           else
-            @redis.get("lock:leader") do |current_lock_timestamp|
+            @redis.get("lock:task:#{task}") do |current_lock_timestamp|
               new_lock_timestamp = create_lock_timestamp
-              if new_lock_timestamp - current_lock_timestamp.to_i >= 30000
-                @redis.getset("lock:leader", new_lock_timestamp) do |previous_lock_timestamp|
+              if new_lock_timestamp - current_lock_timestamp.to_i >= 300000000
+                @redis.getset("lock:task:#{task}", new_lock_timestamp) do |previous_lock_timestamp|
                   if previous_lock_timestamp == current_lock_timestamp
-                    become_the_leader
+                    setup_task(task, &callback)
+                    setup_task_lock_updater(task)
                   end
                 end
+              else
+                yield if block_given?
               end
             end
           end
         end
       end
 
-      # Set up the leader monitor. A one-time timer is used to run
-      # `request_leader_exection()` in 2 seconds. A periodic timer is
-      # used to update the leader lock timestamp if the current
-      # process is the leader, or to run `request_leader_election(),
-      # every 10 seconds. The timers are stored in the timers hash
-      # under `:run`.
-      def setup_leader_monitor
-        @timers[:run] << EM::Timer.new(2) do
-          request_leader_election
-        end
-        @timers[:run] << EM::PeriodicTimer.new(10) do
-          if @is_leader
-            update_leader_lock
+      # Request Sensu server task elections. The task list is ordered
+      # by prioity. This method works through the task list serially,
+      # increasing the election request delay as the current process
+      # becomes responsible for one or more tasks, this is to improve
+      # the initial distribution of tasks amongst Sensu servers.
+      #
+      # @param splay [Integer]
+      def setup_task_elections(splay=10)
+        tasks = TASKS.dup - @tasks
+        next_task = Proc.new do
+          task = tasks.shift
+          if task
+            delay = splay * @tasks.size
+            @timers[:run] << EM::Timer.new(delay) do
+              request_task_election(task, &next_task)
+            end
           else
-            request_leader_election
+            @timers[:run] << EM::Timer.new(10) do
+              setup_task_elections(splay)
+            end
           end
         end
+        next_task.call
       end
 
       # Update the Sensu server registry, stored in Redis. This method
       # adds the local/current Sensu server info to the registry,
-      # including its id, hostname, address, if its the current
-      # leader, and some metrics. Sensu server registry entries expire
-      # in 30 seconds unless updated.
+      # including its id, hostname, address, its server tasks, and
+      # some metrics. Sensu server registry entries expire in 30
+      # seconds unless updated.
       #
       # @yield [success] passes success status to optional
       #   callback/block.
@@ -1250,13 +1294,19 @@ module Sensu
             :id => server_id,
             :hostname => system_hostname,
             :address => system_address,
-            :is_leader => @is_leader,
+            :tasks => @tasks,
             :metrics => {
               :cpu => {
                 :user => cpu_user,
                 :system => cpu_system
               }
             },
+            :sensu => {
+              :version => VERSION,
+              :settings => {
+                :hexdigest => @settings.hexdigest
+              }
+            },
             :timestamp => Time.now.to_i
           }
           @redis.sadd("servers", server_id)
@@ -1304,13 +1354,13 @@ module Sensu
       end
 
       # Bootstrap the Sensu server process, setting up the keepalive
-      # and check result consumers, and attemping to become the leader
-      # to carry out its duties. This method sets the process/daemon
-      # `@state` to `:running`.
+      # and check result consumers, and attemping to carry out Sensu
+      # server tasks. This method sets the process/daemon `@state` to
+      # `:running`.
       def bootstrap
         setup_keepalives
         setup_results
-        setup_leader_monitor
+        setup_task_elections
         setup_server_registry_updater
         @state = :running
       end
@@ -1330,8 +1380,8 @@ module Sensu
       # set to `:pausing`, to indicate that it's in progress. All run
       # timers are cancelled, and the references are cleared. The
       # Sensu server will unsubscribe from all transport
-      # subscriptions, resign as leader (if currently the leader),
-      # then set the process/daemon `@state` to `:paused`.
+      # subscriptions, relinquish any Sensu server tasks, then set the
+      # process/daemon `@state` to `:paused`.
       def pause
         unless @state == :pausing || @state == :paused
           @state = :pausing
@@ -1340,7 +1390,7 @@ module Sensu
           end
           @timers[:run].clear
           unsubscribe
-          resign_as_leader
+          relinquish_tasks
           @state = :paused
         end
       end

data/lib/sensu/utilities.rb

@@ -259,9 +259,10 @@ module Sensu
     #
     # @param object [Hash]
     # @param match_attributes [Object]
+    # @param support_eval [TrueClass, FalseClass]
     # @param object_attributes [Object]
     # @return [TrueClass, FalseClass]
-    def attributes_match?(object, match_attributes, object_attributes=nil)
+    def attributes_match?(object, match_attributes, support_eval=true, object_attributes=nil)
       object_attributes ||= object
       match_attributes.all? do |key, value_one|
         value_two = object_attributes[key]
@@ -269,10 +270,10 @@ module Sensu
         when value_one == value_two
           true
         when value_one.is_a?(Hash) && value_two.is_a?(Hash)
-          attributes_match?(object, value_one, value_two)
+          attributes_match?(object, value_one, support_eval, value_two)
         when value_one.to_s == value_two.to_s
           true
-        when value_one.is_a?(String) && value_one.start_with?(EVAL_PREFIX)
+        when value_one.is_a?(String) && value_one.start_with?(EVAL_PREFIX) && support_eval
           eval_attribute_value(object, value_one, value_two)
         else
           false

data/sensu.gemspec

@@ -15,12 +15,12 @@ Gem::Specification.new do |s|
   s.add_dependency "eventmachine", "1.2.2"
   s.add_dependency "sensu-json", "2.1.0"
   s.add_dependency "sensu-logger", "1.2.1"
-  s.add_dependency "sensu-settings", "9.9.0"
+  s.add_dependency "sensu-settings", "10.0.0"
   s.add_dependency "sensu-extension", "1.5.1"
-  s.add_dependency "sensu-extensions", "1.7.1"
+  s.add_dependency "sensu-extensions", "1.9.0"
   s.add_dependency "sensu-transport", "7.0.2"
   s.add_dependency "sensu-spawn", "2.2.1"
-  s.add_dependency "sensu-redis", "2.1.0"
+  s.add_dependency "sensu-redis", "2.1.1"
   s.add_dependency "em-http-server", "0.1.8"
   s.add_dependency "parse-cron", "0.1.4"
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sensu
 version: !ruby/object:Gem::Version
-  version: 0.28.6
+  version: 0.29.0
 platform: ruby
 authors:
 - Sean Porter
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-04-28 00:00:00.000000000 Z
+date: 2017-03-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: eventmachine
@@ -59,14 +59,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 9.9.0
+        version: 10.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 9.9.0
+        version: 10.0.0
 - !ruby/object:Gem::Dependency
   name: sensu-extension
   requirement: !ruby/object:Gem::Requirement
@@ -87,14 +87,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.7.1
+        version: 1.9.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.7.1
+        version: 1.9.0
 - !ruby/object:Gem::Dependency
   name: sensu-transport
   requirement: !ruby/object:Gem::Requirement
@@ -129,14 +129,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 2.1.0
+        version: 2.1.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 2.1.0
+        version: 2.1.1
 - !ruby/object:Gem::Dependency
   name: em-http-server
   requirement: !ruby/object:Gem::Requirement
@@ -253,8 +253,10 @@ files:
 - lib/sensu/api/routes/request.rb
 - lib/sensu/api/routes/resolve.rb
 - lib/sensu/api/routes/results.rb
+- lib/sensu/api/routes/settings.rb
 - lib/sensu/api/routes/silenced.rb
 - lib/sensu/api/routes/stashes.rb
+- lib/sensu/api/utilities/filter_response_content.rb
 - lib/sensu/api/utilities/publish_check_request.rb
 - lib/sensu/api/utilities/publish_check_result.rb
 - lib/sensu/api/utilities/resolve_event.rb