sensu 0.27.1 → 0.28.0

data/lib/sensu/server/handle.rb

@@ -33,12 +33,14 @@ module Sensu
       # @param handler [Hash] definition.
       # @param event_data [Object] provided to the spawned handler
       #   process via STDIN.
-      def pipe_handler(handler, event_data)
+      # @param event_id [String] event UUID
+      def pipe_handler(handler, event_data, event_id)
         options = {:data => event_data, :timeout => handler[:timeout]}
         Spawn.process(handler[:command], options) do |output, status|
           log_level = status == 0 ? :info : :error
           @logger.send(log_level, "handler output", {
             :handler => handler,
+            :event => { :id => event_id },
             :output => output.split("\n+")
           })
           @in_progress[:events] -= 1 if @in_progress
@@ -121,10 +123,12 @@ module Sensu
       #
       # @param handler [Hash] definition.
       # @param event_data [Object] to pass to the handler extension.
-      def handler_extension(handler, event_data)
+      # @param event_id [String] event UUID
+      def handler_extension(handler, event_data, event_id)
         handler.safe_run(event_data) do |output, status|
           @logger.info("handler extension output", {
             :extension => handler.definition,
+            :event => { :id => event_id },
             :output => output,
             :status => status
           })
@@ -137,10 +141,11 @@ module Sensu
       #
       # @param handler [Hash] definition.
       # @param event_data [Object] to pass to the handler type method.
-      def handler_type_router(handler, event_data)
+      # @param event_id [String] event UUID
+      def handler_type_router(handler, event_data, event_id)
         case handler[:type]
         when "pipe"
-          pipe_handler(handler, event_data)
+          pipe_handler(handler, event_data, event_id)
         when "tcp"
           tcp_handler(handler, event_data)
         when "udp"
@@ -148,7 +153,7 @@ module Sensu
         when "transport"
           transport_handler(handler, event_data)
         when "extension"
-          handler_extension(handler, event_data)
+          handler_extension(handler, event_data, event_id)
         end
       end
 
@@ -159,13 +164,15 @@ module Sensu
       #
       # @param handler [Hash] definition.
       # @param event_data [Object] to pass to an event handler.
-      def handle_event(handler, event_data)
+      # @param event_id [String] event UUID
+      def handle_event(handler, event_data, event_id)
         definition = handler.is_a?(Hash) ? handler : handler.definition
         @logger.debug("handling event", {
           :event_data => event_data,
+          :event => { :id => event_id },
           :handler => definition
         })
-        handler_type_router(handler, event_data)
+        handler_type_router(handler, event_data, event_id)
       end
     end
   end

data/lib/sensu/server/process.rb

@@ -258,7 +258,7 @@ module Sensu
           @in_progress[:events] += 1
           filter_event(handler, event) do |event|
             mutate_event(handler, event) do |event_data|
-              handle_event(handler, event_data)
+              handle_event(handler, event_data, event[:id])
             end
           end
         end
@@ -627,6 +627,7 @@ module Sensu
             end
           else
             client = create_client(client_key)
+            client[:type] = "proxy" if result[:check][:source]
             update_client_registry(client) do
               yield(client)
             end
@@ -766,57 +767,145 @@ module Sensu
         end
       end
 
-      # Calculate a check execution splay, taking into account the
-      # current time and the execution interval to ensure it's
+      # Create and publish one or more proxy check requests. This
+      # method iterates through the Sensu client registry for clients
+      # that matched provided proxy request client attributes. A proxy
+      # check request is created for each client in the registry that
+      # matches the proxy request client attributes. Proxy check
+      # requests have their client tokens subsituted by the associated
+      # client attributes values. The check requests are published to
+      # the Transport via `publish_check_request()`.
+      #
+      # @param check [Hash] definition.
+      def publish_proxy_check_requests(check)
+        client_attributes = check[:proxy_requests][:client_attributes]
+        unless client_attributes.empty?
+          @redis.smembers("clients") do |clients|
+            clients.each do |client_name|
+              @redis.get("client:#{client_name}") do |client_json|
+                unless client_json.nil?
+                  client = Sensu::JSON.load(client_json)
+                  if attributes_match?(client, client_attributes)
+                    @logger.debug("creating a proxy check request", {
+                      :client => client,
+                      :check => check
+                    })
+                    proxy_check, unmatched_tokens = object_substitute_tokens(check.dup, client)
+                    if unmatched_tokens.empty?
+                      proxy_check[:source] ||= client[:name]
+                      publish_check_request(proxy_check)
+                    else
+                      @logger.warn("failed to publish a proxy check request", {
+                        :reason => "unmatched client tokens",
+                        :unmatched_tokens => unmatched_tokens,
+                        :client => client,
+                        :check => check
+                      })
+                    end
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+
+      # Create a check request proc, used to publish check requests to
+      # for a check to the Sensu transport. Check requests are not
+      # published if subdued. This method determines if a check uses
+      # proxy check requests and calls the appropriate check request
+      # publish method.
+      #
+      # @param check [Hash] definition.
+      def create_check_request_proc(check)
+        Proc.new do
+          unless check_subdued?(check)
+            if check[:proxy_requests]
+              publish_proxy_check_requests(check)
+            else
+              publish_check_request(check)
+            end
+          else
+            @logger.info("check request was subdued", :check => check)
+          end
+        end
+      end
+
+      # Schedule a check request, using the check cron. This method
+      # determines the time until the next cron time (in seconds) and
+      # creats an EventMachine timer for the request. This method will
+      # be called after every check cron request for subsequent
+      # requests. The timer is stored in the timer hash under
+      # `:leader`, as check request publishing is a task for only the
+      # Sensu server leader, so it can be cancelled etc. The check
+      # cron request timer object is removed from the timer hash after
+      # the request is published, to stop the timer hash from growing
+      # infinitely.
+      #
+      # @param check [Hash] definition.
+      def schedule_check_cron_request(check)
+        cron_time = determine_check_cron_time(check)
+        @timers[:leader] << Timer.new(cron_time) do |timer|
+          create_check_request_proc(check).call
+          @timers[:leader].delete(timer)
+          schedule_check_cron_request(check)
+        end
+      end
+
+      # Calculate a check request splay, taking into account the
+      # current time and the request interval to ensure it's
       # consistent between process restarts.
       #
       # @param check [Hash] definition.
-      def calculate_check_execution_splay(check)
+      def calculate_check_request_splay(check)
         splay_hash = Digest::MD5.digest(check[:name]).unpack('Q<').first
         current_time = (Time.now.to_f * 1000).to_i
         (splay_hash - current_time) % (check[:interval] * 1000) / 1000.0
       end
 
-      # Schedule check executions, using EventMachine periodic timers,
-      # using a calculated execution splay. The timers are stored in
-      # the timers hash under `:leader`, as check request publishing
-      # is a task for only the Sensu server leader, so they can be
-      # cancelled etc. Check requests are not published if subdued.
+      # Schedule check requests, using the check interval. This method
+      # using an intial calculated request splay EventMachine timer
+      # and an EventMachine periodic timer for subsequent check
+      # requests. The timers are stored in the timers hash under
+      # `:leader`, as check request publishing is a task for only the
+      # Sensu server leader, so they can be cancelled etc.
+      #
+      # @param check [Hash] definition.
+      def schedule_check_interval_requests(check)
+        request_splay = testing? ? 0 : calculate_check_request_splay(check)
+        interval = testing? ? 0.5 : check[:interval]
+        @timers[:leader] << Timer.new(request_splay) do
+          create_check_request = create_check_request_proc(check)
+          create_check_request.call
+          @timers[:leader] << PeriodicTimer.new(interval, &create_check_request)
+        end
+      end
+
+      # Schedule check requests. This method iterates through defined
+      # checks and uses the appropriate method of check request
+      # scheduling, either with the cron syntax or a numeric interval.
       #
       # @param checks [Array] of definitions.
-      def schedule_check_executions(checks)
+      def schedule_checks(checks)
         checks.each do |check|
-          create_check_request = Proc.new do
-            unless check_subdued?(check)
-              publish_check_request(check)
-            else
-              @logger.info("check request was subdued", :check => check)
-            end
-          end
-          execution_splay = testing? ? 0 : calculate_check_execution_splay(check)
-          interval = testing? ? 0.5 : check[:interval]
-          @timers[:leader] << EM::Timer.new(execution_splay) do
-            create_check_request.call
-            @timers[:leader] << EM::PeriodicTimer.new(interval, &create_check_request)
+          if check[:cron]
+            schedule_check_cron_request(check)
+          else
+            schedule_check_interval_requests(check)
           end
         end
       end
 
       # Set up the check request publisher. This method creates an
       # array of check definitions, that are not standalone checks,
-      # and do not have `:publish` set to `false`. The array of check
-      # definitions includes those from standard checks and extensions
-      # (with a defined execution `:interval`). The array is provided
-      # to the `schedule_check_executions()` method.
+      # and do not have `:publish` set to `false`. The array is
+      # provided to the `schedule_checks()` method.
       def setup_check_request_publisher
         @logger.debug("scheduling check requests")
-        standard_checks = @settings.checks.reject do |check|
+        checks = @settings.checks.reject do |check|
           check[:standalone] || check[:publish] == false
         end
-        extension_checks = @extensions.checks.reject do |check|
-          check[:standalone] || check[:publish] == false || !check[:interval].is_a?(Integer)
-        end
-        schedule_check_executions(standard_checks + extension_checks)
+        schedule_checks(checks)
       end
 
       # Publish a check result to the Transport for processing. A
@@ -948,7 +1037,7 @@ module Sensu
       # stored in the timers hash under `:leader`.
       def setup_client_monitor
         @logger.debug("monitoring client keepalives")
-        @timers[:leader] << EM::PeriodicTimer.new(30) do
+        @timers[:leader] << PeriodicTimer.new(30) do
           determine_stale_clients
         end
       end
@@ -994,7 +1083,7 @@ module Sensu
       # is stored in the timers hash under `:leader`.
       def setup_check_result_monitor(interval = 30)
         @logger.debug("monitoring check results")
-        @timers[:leader] << EM::PeriodicTimer.new(interval) do
+        @timers[:leader] << PeriodicTimer.new(interval) do
           determine_stale_check_results(interval)
         end
       end
@@ -1018,24 +1107,24 @@ module Sensu
         (Time.now.to_f * 1000).to_i
       end
 
-      # Create/return the unique Sensu server leader ID for the
-      # current process.
+      # Create/return the unique Sensu server ID for the current
+      # process.
       #
       # @return [String]
-      def leader_id
-        @leader_id ||= random_uuid
+      def server_id
+        @server_id ||= random_uuid
       end
 
       # Become the Sensu server leader, responsible for specific
       # duties (`leader_duties()`). Unless the current process is
       # already the leader, this method sets the leader ID stored in
-      # Redis to the unique random leader ID for the process. If the
+      # Redis to the unique random server ID for the process. If the
       # leader ID in Redis is successfully updated, `@is_leader` is
       # set to true and `leader_duties()` is called to begin the
       # tasks/duties of the Sensu server leader.
       def become_the_leader
         unless @is_leader
-          @redis.set("leader", leader_id) do
+          @redis.set("leader", server_id) do
             @logger.info("i am now the leader")
             @is_leader = true
             leader_duties
@@ -1066,7 +1155,7 @@ module Sensu
       end
 
       # Updates the Sensu server leader lock timestamp. The current
-      # leader ID is retrieved from Redis and compared with the leader
+      # leader ID is retrieved from Redis and compared with the server
       # ID of the current process to determine if it is still the
       # Sensu server leader. If the current process is still the
       # leader, the leader lock timestamp is updated. If the current
@@ -1075,7 +1164,7 @@ module Sensu
       # more than one leader.
       def update_leader_lock
         @redis.get("leader") do |current_leader_id|
-          if current_leader_id == leader_id
+          if current_leader_id == server_id
             @redis.set("lock:leader", create_lock_timestamp) do
               @logger.debug("updated leader lock timestamp")
             end
@@ -1125,10 +1214,10 @@ module Sensu
       # every 10 seconds. The timers are stored in the timers hash
       # under `:run`.
       def setup_leader_monitor
-        @timers[:run] << EM::Timer.new(2) do
+        @timers[:run] << Timer.new(2) do
           request_leader_election
         end
-        @timers[:run] << EM::PeriodicTimer.new(10) do
+        @timers[:run] << PeriodicTimer.new(10) do
           if @is_leader
             update_leader_lock
           else
@@ -1137,6 +1226,52 @@ module Sensu
         end
       end
 
+      # Update the Sensu server registry, stored in Redis. This method
+      # adds the local/current Sensu server info to the registry,
+      # including its id, hostname, address, if its the current
+      # leader, and some metrics. Sensu server registry entries expire
+      # in 30 seconds unless updated.
+      #
+      # @yield [success] passes success status to optional
+      #   callback/block.
+      # @yieldparam success [TrueClass,FalseClass] indicating if the
+      #   server registry update was a success.
+      def update_server_registry
+        @logger.debug("updating the server registry")
+        process_cpu_times do |cpu_user, cpu_system, _, _|
+          info = {
+            :id => server_id,
+            :hostname => system_hostname,
+            :address => system_address,
+            :is_leader => @is_leader,
+            :metrics => {
+              :cpu => {
+                :user => cpu_user,
+                :system => cpu_system
+              }
+            },
+            :timestamp => Time.now.to_i
+          }
+          @redis.sadd("servers", server_id)
+          server_key = "server:#{server_id}"
+          @redis.set(server_key, Sensu::JSON.dump(info)) do
+            @redis.expire(server_key, 30)
+            @logger.info("updated server registry", :server => info)
+            yield(true) if block_given?
+          end
+        end
+      end
+
+      # Set up the server registry updater. A periodic timer is
+      # used to update the Sensu server info stored in Redis. The
+      # timer is stored in the timers hash under `:run`.
+      def setup_server_registry_updater
+        update_server_registry
+        @timers[:run] << PeriodicTimer.new(10) do
+          update_server_registry
+        end
+      end
+
       # Unsubscribe from transport subscriptions (all of them). This
       # method is called when there are issues with connectivity, or
       # the process is stopping.
@@ -1169,6 +1304,7 @@ module Sensu
         setup_keepalives
         setup_results
         setup_leader_monitor
+        setup_server_registry_updater
         @state = :running
       end
 

data/lib/sensu/server/socket.rb

@@ -18,7 +18,7 @@ module Sensu
       #
       # @param timeout [Numeric] in seconds.
       def set_timeout(timeout)
-        @timeout_timer = EM::Timer.new(timeout) do
+        @timeout_timer = Timer.new(timeout) do
           @timed_out = true
           close_connection
         end

data/lib/sensu/timers.rb

@@ -0,0 +1,26 @@
+require "eventmachine"
+
+module Sensu
+  class Timer < EventMachine::Timer; end
+
+  # This fix comes from http://soohwan.blogspot.ca/2011/02/fix-eventmachineperiodictimer.html
+  class PeriodicTimer < EventMachine::PeriodicTimer
+    alias :original_initialize :initialize
+    alias :original_schedule :schedule
+
+    # Record initial start time and the fixed interval, used for
+    # compensating for timer drift when scheduling the next call.
+    def initialize(interval, callback=nil, &block)
+      @start = Time.now
+      @fixed_interval = interval
+      original_initialize(interval, callback, &block)
+    end
+
+    # Calculate the timer drift and compensate for it.
+    def schedule
+      compensation = (Time.now - @start) % @fixed_interval
+      @interval = @fixed_interval - compensation
+      original_schedule
+    end
+  end
+end

data/lib/sensu/utilities.rb

@@ -1,7 +1,14 @@
+gem "parse-cron", "0.1.4"
+
 require "securerandom"
+require "sensu/sandbox"
+require "parse-cron"
+require "socket"
 
 module Sensu
   module Utilities
+    EVAL_PREFIX = "eval:".freeze
+
     # Determine if Sensu is being tested, using the process name.
     # Sensu is being test if the process name is "rspec",
     #
@@ -16,7 +23,7 @@ module Sensu
     # @param wait [Numeric] time to delay block calls.
     # @param block [Proc] to call that needs to return true.
     def retry_until_true(wait=0.5, &block)
-      EM::Timer.new(wait) do
+      Timer.new(wait) do
         unless block.call
           retry_until_true(wait, &block)
         end
@@ -36,7 +43,7 @@ module Sensu
         when hash_one[key].is_a?(Hash) && value.is_a?(Hash)
           deep_merge(hash_one[key], value)
         when hash_one[key].is_a?(Array) && value.is_a?(Array)
-          hash_one[key].concat(value).uniq
+          (hash_one[key] + value).uniq
         else
           value
         end
@@ -44,6 +51,37 @@ module Sensu
       merged
     end
 
+    # Retrieve the system hostname. If the hostname cannot be
+    # determined and an error is thrown, `nil` will be returned.
+    #
+    # @return [String] system hostname.
+    def system_hostname
+      ::Socket.gethostname rescue nil
+    end
+
+    # Retrieve the system IP address. If a valid non-loopback
+    # IPv4 address cannot be found and an error is thrown,
+    # `nil` will be returned.
+    #
+    # @return [String] system ip address
+    def system_address
+      ::Socket.ip_address_list.find { |address|
+        address.ipv4? && !address.ipv4_loopback?
+      }.ip_address rescue nil
+    end
+
+    # Retrieve the process CPU times. If the cpu times cannot be
+    # determined and an error is thrown, `[nil, nil, nil, nil]` will
+    # be returned.
+    #
+    # @return [Array] CPU times: utime, stime, cutime, cstime
+    def process_cpu_times(&callback)
+      determine_cpu_times = Proc.new do
+        ::Process.times.to_a rescue [nil, nil, nil, nil]
+      end
+      EM::defer(determine_cpu_times, callback)
+    end
+
     # Generate a random universally unique identifier.
     #
     # @return [String] random UUID.
@@ -124,6 +162,124 @@ module Sensu
       [substituted, unmatched_tokens]
     end
 
+    # Perform token substitution for an object. String values are
+    # passed to `substitute_tokens()`, arrays and sub-hashes are
+    # processed recursively. Numeric values are ignored.
+    #
+    # @param object [Object]
+    # @param attributes [Hash]
+    # @return	[Array] containing the updated object with substituted
+    #   values and an array of unmatched tokens.
+    def object_substitute_tokens(object, attributes)
+      unmatched_tokens = []
+      case object
+      when Hash
+        object.each do |key, value|
+          object[key], unmatched = object_substitute_tokens(value, attributes)
+          unmatched_tokens.push(*unmatched)
+        end
+      when Array
+        object.map! do |value|
+          value, unmatched = object_substitute_tokens(value, attributes)
+          unmatched_tokens.push(*unmatched)
+          value
+        end
+      when String
+        object, unmatched_tokens = substitute_tokens(object, attributes)
+      end
+      [object, unmatched_tokens.uniq]
+    end
+
+    # Process an eval attribute value, a Ruby `eval()` string
+    # containing an expression to be evaluated within the
+    # scope/context of a sandbox. This methods strips away the
+    # expression prefix, `eval:`, and substitues any dot notation
+    # tokens with the corresponding event data values. If there are
+    # unmatched tokens, this method will return `nil`.
+    #
+    # @object [Hash]
+    # @raw_eval_string [String]
+    # @return [String] processed eval string.
+    def process_eval_string(object, raw_eval_string)
+      eval_string = raw_eval_string.slice(5..-1)
+      eval_string, unmatched_tokens = substitute_tokens(eval_string, object)
+      if unmatched_tokens.empty?
+        eval_string
+      else
+        @logger.error("attribute value eval unmatched tokens", {
+          :object => object,
+          :raw_eval_string => raw_eval_string,
+          :unmatched_tokens => unmatched_tokens
+        })
+        nil
+      end
+    end
+
+    # Ruby `eval()` a string containing an expression, within the
+    # scope/context of a sandbox. This method is for attribute values
+    # starting with "eval:", with the Ruby expression following the
+    # colon. A single variable is provided to the expression, `value`,
+    # equal to the corresponding object attribute value. Dot notation
+    # tokens in the expression, e.g. `:::mysql.user:::`, are
+    # substituted with the corresponding object attribute values prior
+    # to evaluation. The expression is expected to return a boolean
+    # value.
+    #
+    # @param object [Hash]
+    # @param raw_eval_string [String] containing the Ruby
+    #   expression to be evaluated.
+    # @param raw_value [Object] of the corresponding object
+    #   attribute value.
+    # @return [TrueClass, FalseClass]
+    def eval_attribute_value(object, raw_eval_string, raw_value)
+      eval_string = process_eval_string(object, raw_eval_string)
+      unless eval_string.nil?
+        begin
+          value = Marshal.load(Marshal.dump(raw_value))
+          !!Sandbox.eval(eval_string, value)
+        rescue StandardError, SyntaxError => error
+          @logger.error("attribute value eval error", {
+            :object => object,
+            :raw_eval_string => raw_eval_string,
+            :raw_value => raw_value,
+            :error => error.to_s
+          })
+          false
+        end
+      else
+        false
+      end
+    end
+
+    # Determine if all attribute values match those of the
+    # corresponding object attributes. Attributes match if the value
+    # objects are equivalent, are both hashes with matching key/value
+    # pairs (recursive), have equal string values, or evaluate to true
+    # (Ruby eval).
+    #
+    # @param object [Hash]
+    # @param match_attributes [Object]
+    # @param object_attributes [Object]
+    # @return [TrueClass, FalseClass]
+    def attributes_match?(object, match_attributes, object_attributes=nil)
+      object_attributes ||= object
+      match_attributes.all? do |key, value_one|
+        value_two = object_attributes[key]
+        case
+        when value_one == value_two
+          true
+        when value_one.is_a?(Hash) && value_two.is_a?(Hash)
+          attributes_match?(object, value_one, value_two)
+        when value_one.to_s == value_two.to_s
+          true
+        when value_one.is_a?(String) && value_one.start_with?(EVAL_PREFIX)
+          eval_attribute_value(object, value_one, value_two)
+        else
+          false
+        end
+      end
+    end
+
     # Determine if the current time falls within a time window. The
     # provided condition must have a `:begin` and `:end` time, eg.
     # "11:30:00 PM", or `false` will be returned.
@@ -189,5 +345,15 @@ module Sensu
         false
       end
     end
+
+    # Determine the next check cron time.
+    #
+    # @param check [Hash] definition.
+    def determine_check_cron_time(check)
+      cron_parser = CronParser.new(check[:cron])
+      current_time = Time.now
+      next_cron_time = cron_parser.next(current_time)
+      next_cron_time - current_time
+    end
   end
 end