portertech-sensu 1.10.0

data/lib/sensu/daemon.rb

@@ -0,0 +1,396 @@
+require "rubygems"
+
+gem "eventmachine", "1.2.7"
+
+gem "portertech-sensu-json", "2.2.1"
+gem "portertech-sensu-logger", "1.4.0"
+gem "portertech-sensu-settings", "10.18.0"
+gem "sensu-extension", "1.5.2"
+gem "portertech-sensu-extensions", "1.12.0"
+gem "sensu-transport", "8.3.0"
+gem "portertech-sensu-spawn", "2.6.1"
+gem "sensu-redis", "2.4.0"
+
+require "time"
+require "uri"
+
+if RUBY_PLATFORM =~ /aix/ || RUBY_PLATFORM =~ /solaris/
+  require "em/pure_ruby"
+end
+
+require "sensu/json"
+require "sensu/logger"
+require "sensu/settings"
+require "sensu/extensions"
+require "sensu/transport"
+require "sensu/spawn"
+require "sensu/redis"
+
+require "sensu/constants"
+require "sensu/utilities"
+require "sensu/cli"
+
+module Sensu
+  module Daemon
+    include Utilities
+
+    attr_reader :start_time, :settings
+
+    # Initialize the Sensu process. Set the start time, initial
+    # service state, double the maximum number of EventMachine timers,
+    # set up the logger, and load settings. This method will load
+    # extensions and setup Sensu Spawn if the Sensu process is not the
+    # Sensu API. This method can and optionally daemonize the process
+    # and/or create a PID file.
+    #
+    # @param options [Hash]
+    def initialize(options={})
+      @start_time = Time.now.to_i
+      @state = :initializing
+      @timers = {:run => []}
+      unless EM::reactor_running?
+        EM::epoll
+        EM::set_max_timers(200000)
+        EM::error_handler do |error|
+          unexpected_error(error)
+        end
+      end
+      setup_logger(options)
+      load_settings(options)
+      unless sensu_service_name == "api"
+        load_extensions(options)
+        setup_spawn
+      end
+      setup_process(options)
+    end
+
+    # Handle an unexpected error. This method is used for EM global
+    # catch-all error handling, accepting an error object. Error
+    # handling is opt-in via a configuration option, e.g. `"sensu":
+    # {"global_error_handler": true}`. If a user does not opt-in, the
+    # provided error will be raised (uncaught). If a user opts-in via
+    # configuration, the error will be logged and ignored :itsfine:.
+    #
+    # @param error [Object]
+    def unexpected_error(error)
+      if @settings && @settings[:sensu][:global_error_handler]
+        backtrace = error.backtrace.join("\n")
+        if @logger
+          @logger.warn("global catch-all error handling enabled")
+          @logger.fatal("unexpected error - please address this immediately", {
+            :error => error.to_s,
+            :error_class => error.class,
+            :backtrace => backtrace
+          })
+        else
+          puts "global catch-all error handling enabled"
+          puts "unexpected error - please address this immediately: #{error.to_s}\n#{error.class}\n#{backtrace}"
+        end
+      else
+        raise error
+      end
+    end
+
+    # Set up the Sensu logger and its process signal traps for log
+    # rotation and debug log level toggling. This method creates the
+    # logger instance variable: `@logger`.
+    #
+    # https://github.com/sensu/sensu-logger
+    #
+    # @param options [Hash]
+    def setup_logger(options={})
+      @logger = Logger.get(options)
+      @logger.setup_signal_traps
+    end
+
+    # Log setting or extension loading notices, sensitive information
+    # is redacted.
+    #
+    # @param notices [Array] to be logged.
+    # @param level [Symbol] to log the notices at.
+    def log_notices(notices=[], level=:warn)
+      notices.each do |concern|
+        message = concern.delete(:message)
+        @logger.send(level, message, redact_sensitive(concern))
+      end
+    end
+
+    # Determine if the Sensu settings are valid, if there are load or
+    # validation errors, and immediately exit the process with the
+    # appropriate exit status code. This method is used to determine
+    # if the latest configuration changes are valid prior to
+    # restarting the Sensu service, triggered by a CLI argument, e.g.
+    # `--validate_config`.
+    #
+    # @param settings [Object]
+    def validate_settings!(settings)
+      if settings.errors.empty?
+        puts "configuration is valid"
+        exit
+      else
+        puts "configuration is invalid"
+        puts Sensu::JSON.dump({:errors => @settings.errors}, :pretty => true)
+        exit 2
+      end
+    end
+
+    # Print the Sensu settings (JSON) to STDOUT and immediately exit
+    # the process with the appropriate exit status code. This method
+    # is used while troubleshooting configuration issues, triggered by
+    # a CLI argument, e.g. `--print_config`. Sensu settings with
+    # sensitive values (e.g. passwords) are first redacted.
+    #
+    # @param settings [Object]
+    def print_settings!(settings)
+      redacted_settings = redact_sensitive(settings.to_hash)
+      @logger.warn("outputting compiled configuration and exiting")
+      puts Sensu::JSON.dump(redacted_settings, :pretty => true)
+      exit(settings.errors.empty? ? 0 : 2)
+    end
+
+    # Load Sensu settings. This method creates the settings instance
+    # variable: `@settings`. If the `validate_config` option is true,
+    # this method calls `validate_settings!()` to validate the latest
+    # compiled configuration settings and will then exit the process.
+    # If the `print_config` option is true, this method calls
+    # `print_settings!()` to output the compiled configuration
+    # settings and will then exit the process. If there are loading or
+    # validation errors, they will be logged (notices), and this
+    # method will exit(2) the process.
+    #
+    #
+    # https://github.com/sensu/sensu-settings
+    #
+    # @param options [Hash]
+    def load_settings(options={})
+      @settings = Settings.get(options)
+      validate_settings!(@settings) if options[:validate_config]
+      log_notices(@settings.warnings)
+      log_notices(@settings.errors, :fatal)
+      print_settings!(@settings) if options[:print_config]
+      unless @settings.errors.empty?
+        @logger.fatal("SENSU NOT RUNNING!")
+        exit 2
+      end
+      @settings.set_env!
+    end
+
+    # Load Sensu extensions and log any notices. Set the logger and
+    # settings for each extension instance. This method creates the
+    # extensions instance variable: `@extensions`.
+    #
+    # https://github.com/sensu/sensu-extensions
+    # https://github.com/sensu/sensu-extension
+    #
+    # @param options [Hash]
+    def load_extensions(options={})
+      extensions_options = options.merge(:extensions => @settings[:extensions])
+      @extensions = Extensions.get(extensions_options)
+      log_notices(@extensions.warnings)
+      extension_settings = @settings.to_hash.dup
+      @extensions.all.each do |extension|
+        extension.logger = @logger
+        extension.settings = extension_settings
+      end
+    end
+
+    # Set up Sensu spawn, creating a worker to create, control, and
+    # limit spawned child processes. This method adjusts the
+    # EventMachine thread pool size to accommodate the concurrent
+    # process spawn limit and other Sensu process operations.
+    #
+    # https://github.com/sensu/sensu-spawn
+    def setup_spawn
+      @logger.info("configuring sensu spawn", :settings => @settings[:sensu][:spawn])
+      threadpool_size = @settings[:sensu][:spawn][:limit] + 10
+      @logger.debug("setting eventmachine threadpool size", :size => threadpool_size)
+      EM::threadpool_size = threadpool_size
+      Spawn.setup(@settings[:sensu][:spawn])
+    end
+
+    # Manage the current process, optionally daemonize and/or write
+    # the current process ID to a PID file.
+    #
+    # @param options [Hash]
+    def setup_process(options)
+      daemonize if options[:daemonize]
+      write_pid(options[:pid_file]) if options[:pid_file]
+    end
+
+    # Start the Sensu service and set the service state to `:running`.
+    # This method will likely be overridden by a subclass. Yield if a
+    # block is provided.
+    def start
+      @state = :running
+      yield if block_given?
+    end
+
+    # Pause the Sensu service and set the service state to `:paused`.
+    # This method will likely be overridden by a subclass.
+    def pause
+      @state = :paused
+    end
+
+    # Resume the paused Sensu service and set the service state to
+    # `:running`. This method will likely be overridden by a subclass.
+    def resume
+      @state = :running
+    end
+
+    # Stop the Sensu service and set the service state to `:stopped`.
+    # This method will likely be overridden by a subclass. This method
+    # should stop the EventMachine event loop.
+    def stop
+      @state = :stopped
+      @logger.warn("stopping reactor")
+      EM::stop_event_loop
+    end
+
+    # Set up process signal traps. This method uses the `STOP_SIGNALS`
+    # constant to determine which process signals will result in a
+    # graceful service stop. A periodic timer must be used to poll for
+    # received signals, as Mutex#lock cannot be used within the
+    # context of `trap()`.
+    def setup_signal_traps
+      @signals = []
+      STOP_SIGNALS.each do |signal|
+        Signal.trap(signal) do
+          @signals << signal
+        end
+      end
+      EM::PeriodicTimer.new(1) do
+        signal = @signals.shift
+        if STOP_SIGNALS.include?(signal)
+          @logger.warn("received signal", :signal => signal)
+          stop
+        end
+      end
+    end
+
+    # Set up the Sensu transport connection. Sensu uses a transport
+    # API, allowing it to use various message brokers. By default,
+    # Sensu will use the built-in "rabbitmq" transport. The Sensu
+    # service will stop gracefully in the event of a transport error,
+    # and pause/resume in the event of connectivity issues. This
+    # method creates the transport instance variable: `@transport`.
+    #
+    # https://github.com/sensu/sensu-transport
+    #
+    # @yield [Object] passes initialized and connected Transport
+    #   connection object to the callback/block.
+    def setup_transport
+      transport_name = @settings[:transport][:name]
+      transport_settings = @settings[transport_name]
+      @logger.debug("connecting to transport", {
+        :name => transport_name,
+        :settings => transport_settings
+      })
+      Transport.logger = @logger
+      Transport.connect(transport_name, transport_settings) do |connection|
+        @transport = connection
+        @transport.on_error do |error|
+          @logger.error("transport connection error", :error => error.to_s)
+          if @settings[:transport][:reconnect_on_error]
+            @transport.reconnect
+          else
+            stop
+          end
+        end
+        @transport.before_reconnect do
+          unless testing?
+            @logger.warn("reconnecting to transport")
+            pause
+          end
+        end
+        @transport.after_reconnect do
+          @logger.info("reconnected to transport")
+          resume
+        end
+        yield(@transport) if block_given?
+      end
+    end
+
+    # Set up the Redis connection. Sensu uses Redis as a data store,
+    # to store the client registry, current events, etc. The Sensu
+    # service will stop gracefully in the event of a Redis error, and
+    # pause/resume in the event of connectivity issues. This method
+    # creates the Redis instance variable: `@redis`.
+    #
+    # https://github.com/sensu/sensu-redis
+    #
+    # @yield [Object] passes initialized and connected Redis
+    #   connection object to the callback/block.
+    def setup_redis
+      @logger.debug("connecting to redis", :settings => @settings[:redis])
+      Redis.logger = @logger
+      Redis.connect(@settings[:redis]) do |connection|
+        @redis = connection
+        @redis.on_error do |error|
+          @logger.error("redis connection error", :error => error.to_s)
+        end
+        @redis.before_reconnect do
+          unless testing?
+            @logger.warn("reconnecting to redis")
+            pause
+          end
+        end
+        @redis.after_reconnect do
+          @logger.info("reconnected to redis")
+          resume
+        end
+        yield(@redis) if block_given?
+      end
+    end
+
+    private
+
+    # Get the Sensu service name.
+    #
+    # @return [String] Sensu service name.
+    def sensu_service_name
+      File.basename($0).split("-").last
+    end
+
+    # Write the current process ID (PID) to a file (PID file). This
+    # method will cause the Sensu service to exit (2) if the PID file
+    # cannot be written to.
+    #
+    # @param file [String] to write the current PID to.
+    def write_pid(file)
+      begin
+        File.open(file, "w") do |pid_file|
+          pid_file.puts(Process.pid)
+        end
+      rescue
+        @logger.fatal("could not write to pid file", :pid_file => file)
+        @logger.fatal("SENSU NOT RUNNING!")
+        exit 2
+      end
+    end
+
+    # Daemonize the current process. Seed the random number generator,
+    # fork (& exit), detach from controlling terminal, ignore SIGHUP,
+    # fork (& exit), use root '/' as the current working directory,
+    # and close STDIN/OUT/ERR since the process is no longer attached
+    # to a terminal.
+    def daemonize
+      Kernel.srand
+      exit if Kernel.fork
+      unless Process.setsid
+        @logger.fatal("cannot detach from controlling terminal")
+        @logger.fatal("SENSU NOT RUNNING!")
+        exit 2
+      end
+      Signal.trap("SIGHUP", "IGNORE")
+      exit if Kernel.fork
+      Dir.chdir("/")
+      ObjectSpace.each_object(IO) do |io|
+        unless [STDIN, STDOUT, STDERR].include?(io)
+          begin
+            io.close unless io.closed?
+          rescue; end
+        end
+      end
+    end
+  end
+end

data/lib/sensu/sandbox.rb

@@ -0,0 +1,19 @@
+module Sensu
+  module Sandbox
+    # Evaluate a Ruby expression within the context of a simple
+    # "sandbox", a Proc in a module method. As of Ruby 2.3.0,
+    # `$SAFE` no longer supports levels > 1, so its use has been
+    # removed from this method. A single value is provided to the
+    # "sandbox".
+    #
+    # @param expression [String] to be evaluated.
+    # @param value [Object] to provide the "sandbox" with.
+    # @return [Object]
+    def self.eval(expression, value=nil)
+      result = Proc.new do
+        Kernel.eval(expression)
+      end
+      result.call
+    end
+  end
+end

data/lib/sensu/server/filter.rb

@@ -0,0 +1,227 @@
+module Sensu
+  module Server
+    module Filter
+      # Determine if an event handler is silenced.
+      #
+      # @param handler [Hash] definition.
+      # @param event [Hash]
+      # @return [TrueClass, FalseClass]
+      def handler_silenced?(handler, event)
+        event[:silenced] && !handler[:handle_silenced]
+      end
+
+      # Determine if handling is disabled for an event. Check
+      # definitions can disable event handling with an attribute,
+      # `:handle`, by setting it to `false`.
+      #
+      # @param event [Hash]
+      # @return [TrueClass, FalseClass]
+      def handling_disabled?(event)
+        event[:check][:handle] == false
+      end
+
+      # Determine if an event with an action should be handled. An
+      # event action of `:flapping` indicates that the event state is
+      # flapping, and the event should not be handled unless its
+      # handler has `:handle_flapping` set to `true`.
+      #
+      # @param handler [Hash] definition.
+      # @param event [Hash]
+      # @return [TrueClass, FalseClass]
+      def handle_action?(handler, event)
+        event[:action] != :flapping ||
+          (event[:action] == :flapping && !!handler[:handle_flapping])
+      end
+
+      # Determine if an event with a check severity will be handled.
+      # Event handlers can specify the check severities they will
+      # handle, using the definition attribute `:severities`. The
+      # possible severities are "ok", "warning", "critical", and
+      # "unknown". Handler severity filtering is bypassed when the
+      # event `:action` is `:resolve` and a previous check history
+      # status identifies a severity specified in the handler
+      # definition. It's possible for a check history status of 0 to
+      # have had the flapping action, so we are unable to consider
+      # every past 0 to indicate a resolution.
+      #
+      # @param handler [Hash] definition.
+      # @param event [Hash]
+      # @return [TrueClass, FalseClass]
+      def handle_severity?(handler, event)
+        if handler.has_key?(:severities)
+          case event[:action]
+          when :resolve
+            event[:check][:history].any? do |status|
+              severity = SEVERITIES[status.to_i] || "unknown"
+              handler[:severities].include?(severity)
+            end
+          else
+            severity = SEVERITIES[event[:check][:status]] || "unknown"
+            handler[:severities].include?(severity)
+          end
+        else
+          true
+        end
+      end
+
+      # Determine if a filter is to be evoked for the current time. A
+      # filter can be configured with a time window defining when it
+      # is to be evoked, e.g. Monday through Friday, 9-5.
+      #
+      # @param filter [Hash] definition.
+      # @return [TrueClass, FalseClass]
+      def in_filter_time_windows?(filter)
+        if filter[:when]
+          in_time_windows?(filter[:when])
+        else
+          true
+        end
+      end
+
+      # Determine if an event is filtered by a native filter.
+      #
+      # @param filter_name [String]
+      # @param event [Hash]
+      # @yield [filtered] callback/block called with a single
+      #   parameter to indicate if the event was filtered.
+      # @yieldparam filtered [TrueClass,FalseClass] indicating if the
+      #   event was filtered.
+      # @yieldparam filter_name [String] name of the filter being evaluated
+      def native_filter(filter_name, event)
+        filter = @settings[:filters][filter_name]
+        if in_filter_time_windows?(filter)
+          matched = attributes_match?(event, filter[:attributes])
+          yield(filter[:negate] ? matched : !matched, filter_name)
+        else
+          yield(false, filter_name)
+        end
+      end
+
+      # Determine if an event is filtered by a filter extension.
+      #
+      # @param filter_name [String]
+      # @param event [Hash]
+      # @yield [filtered] callback/block called with a single
+      #   parameter to indicate if the event was filtered.
+      # @yieldparam filtered [TrueClass,FalseClass] indicating if the
+      #   event was filtered.
+      # @yieldparam filter_name [String] name of the filter being evaluated
+      def extension_filter(filter_name, event)
+        extension = @extensions[:filters][filter_name]
+        if in_filter_time_windows?(extension.definition)
+          extension.safe_run(event) do |output, status|
+            yield(status == 0, filter_name)
+          end
+        else
+          yield(false, filter_name)
+        end
+      end
+
+      # Determine if an event is filtered by an event filter, native
+      # or extension. This method first checks for the existence of a
+      # native filter, then checks for an extension if a native filter
+      # is not defined. The provided callback is called with a single
+      # parameter, indicating if the event was filtered by a filter.
+      # If a filter does not exist for the provided name, the event is
+      # not filtered.
+      #
+      # @param filter_name [String]
+      # @param event [Hash]
+      # @yield [filtered] callback/block called with a single
+      #   parameter to indicate if the event was filtered.
+      # @yieldparam filtered [TrueClass,FalseClass] indicating if the
+      #   event was filtered.
+      # @yieldparam filter_name [String] name of the filter being evaluated
+      def event_filter(filter_name, event)
+        case
+        when @settings.filter_exists?(filter_name)
+          native_filter(filter_name, event) do |filtered|
+            yield(filtered, filter_name)
+          end
+        when @extensions.filter_exists?(filter_name)
+          extension_filter(filter_name, event) do |filtered|
+            yield(filtered, filter_name)
+          end
+        else
+          @logger.error("unknown filter", :filter_name => filter_name)
+          yield(false, filter_name)
+        end
+      end
+
+      # Determine if an event is filtered for a handler. If a handler
+      # specifies one or more filters, via `:filters` or `:filter`,
+      # the `event_filter()` method is called for each of them. If any
+      # of the filters return `true`, the event is filtered for the
+      # handler. If no filters are defined in the handler definition,
+      # the event is not filtered.
+      #
+      # @param handler [Hash] definition.
+      # @param event [Hash]
+      # @yield [filtered] callback/block called with a single
+      #   parameter to indicate if the event was filtered.
+      # @yieldparam filtered [TrueClass,FalseClass] indicating if the
+      #   event was filtered.
+      def event_filtered?(handler, event)
+        if handler.has_key?(:filters) || handler.has_key?(:filter)
+          filter_list = Array(handler[:filters] || handler[:filter]).dup
+          filter = Proc.new do |filter_list|
+            filter_name = filter_list.shift
+            if filter_name.nil?
+              yield(false)
+            else
+              event_filter(filter_name, event) do |filtered|
+                filtered ? yield(true, filter_name) : EM.next_tick { filter.call(filter_list) }
+              end
+            end
+          end
+          filter.call(filter_list)
+        else
+          yield(false)
+        end
+      end
+
+      # Attempt to filter an event for a handler. This method will
+      # check to see if handling is disabled, if the event action is
+      # handled, if the event check severity is handled, if the
+      # handler is subdued, and if the event is filtered by any of the
+      # filters specified in the handler definition.
+      #
+      # @param handler [Hash] definition.
+      # @param event [Hash]
+      # @yield [event] callback/block called if the event has not been
+      #   filtered.
+      # @yieldparam event [Hash]
+      def filter_event(handler, event)
+        handler_info = case handler[:type]
+          when "extension" then handler.definition
+          else handler  
+        end
+        details = {:handler => handler_info, :event => event}
+        filter_message = case
+        when handling_disabled?(event)
+          "event handling disabled for event"
+        when !handle_action?(handler, event)
+          "handler does not handle action"
+        when !handle_severity?(handler, event)
+          "handler does not handle event severity"
+        when handler_silenced?(handler, event)
+          "handler is silenced"
+        end
+        if filter_message
+          @logger.info(filter_message, details)
+          @in_progress[:events] -= 1 if @in_progress
+        else
+          event_filtered?(handler, event) do |filtered, filter_name|
+            unless filtered
+              yield(event)
+            else
+              details[:filter] = filter_name
+              @logger.info("event was filtered", details)
+              @in_progress[:events] -= 1 if @in_progress
+            end
+          end
+        end
+      end
+    end
+  end
+end