bluepill 0.1.3 → 0.1.4

data/lib/bluepill/process_commands.rb

@@ -0,0 +1,262 @@
+module Bluepill
+  # Starts, stops and restarts a Process with its configured commands
+  #
+  # @api private
+  module ProcessCommands
+    # Start the process with its start command
+    #
+    # Anything still running from an earlier start is killed first, and the
+    # pre_start_command runs. A process bluepill daemonizes has its pid added
+    # to the journal. A self-daemonizing one gets start_grace_time to finish,
+    # and then on_start_timeout is dispatched. Ticks are skipped for
+    # start_grace_time either way.
+    #
+    # @return [Integer] when ticks resume, in epoch seconds
+    #
+    # @api private
+    def start_process
+      # An old stop signal thread would otherwise signal and clean up the new process
+      clean_threads
+      ProcessJournal.kill_all_from_journal(name) # be sure nothing else is running from previous runs
+      pre_start_process
+      # ProcessFactory makes sure every process has a start command
+      command = start_command #: String
+      logger.warning "Executing start command: #{command}"
+      if daemonize?
+        daemon_id = System.daemonize(command, system_command_options)
+        if daemon_id
+          ProcessJournal.append_pid_to_journal(name, daemon_id)
+          # Only refresh_children! fills children, and only when monitoring them
+          children.each do |child|
+            ProcessJournal.append_pid_to_journal(name, child.actual_pid)
+          end
+        end
+      else
+        # This is a self-daemonizing process
+        with_timeout(start_grace_time, on_start_timeout) do
+          run_command(command, 'Start')
+        end
+      end
+
+      skip_ticks_for(start_grace_time)
+    end
+
+    # Run the pre_start_command, if there is one
+    #
+    # @return [void]
+    #
+    # @api private
+    def pre_start_process
+      return unless pre_start_command
+
+      logger.warning "Executing pre start command: #{pre_start_command}"
+      run_command(pre_start_command, 'Pre start')
+    end
+
+    # Stop the process with its stop command, stop signals, or TERM
+    #
+    # The children's pids go in the journal first, so cleanup kills them too.
+    # Stop signals are sent from a thread of their own, with their delays.
+    # Ticks are skipped for stop_grace_time.
+    #
+    # @return [Integer] when ticks resume, in epoch seconds
+    #
+    # @api private
+    def stop_process
+      if monitor_children
+        System.get_children(actual_pid).each do |child_pid|
+          ProcessJournal.append_pid_to_journal(name, child_pid)
+        end
+      end
+
+      if stop_command
+        cmd = prepare_command(stop_command)
+        logger.warning "Executing stop command: #{cmd}"
+
+        with_timeout(stop_grace_time, 'stop') do
+          run_command(cmd, 'Stop')
+        end
+        cleanup_process
+      elsif stop_signals
+        # issue stop signals with configurable delay between each
+        logger.warning "Sending stop signals to #{actual_pid}"
+        @threads << Thread.new(stop_signals.clone) { |signals| send_stop_signals(signals) }
+      else
+        logger.warning "Executing default stop command. Sending TERM signal to #{actual_pid}"
+        signal_process('TERM')
+        cleanup_process
+      end
+
+      skip_ticks_for(stop_grace_time)
+    end
+
+    # Restart the process with its restart command, or by stopping and starting
+    #
+    # @return [Integer] when ticks resume, in epoch seconds
+    #
+    # @api private
+    def restart_process
+      if restart_command
+        cmd = prepare_command(restart_command)
+
+        logger.warning "Executing restart command: #{cmd}"
+
+        with_timeout(restart_grace_time, 'restart') do
+          run_command(cmd, 'Restart')
+        end
+
+        skip_ticks_for(restart_grace_time)
+      else
+        logger.warning 'No restart_command specified. Must stop and start to restart'
+        stop_process
+        # Stop signals run in a thread. Wait for them, or start_process would
+        # kill the stopping process from the journal before its delays are up.
+        @threads.each(&:join)
+        start_process
+      end
+    end
+
+    # Kill what's left of the process, and delete its pid file
+    #
+    # Bluepill only writes the pid file for processes it daemonizes, but it
+    # removes it for every process, so a stopped process never leaves a stale
+    # pid behind for a later process to reuse.
+    #
+    # @return [void]
+    #
+    # @api private
+    def cleanup_process
+      ProcessJournal.kill_all_from_journal(name) # finish cleanup
+      unlink_pid
+    end
+
+    # Return a command with <tt>{{PID}}</tt> replaced by the process's pid
+    #
+    # @param [String] command the stop or restart command
+    #
+    # @return [String]
+    #
+    # @api private
+    def prepare_command(command)
+      command.gsub('{{PID}}', actual_pid.to_s)
+    end
+
+    # Return the options the process's commands run with
+    #
+    # @return [Hash{Symbol => Object}] the uid, gid, working_dir, environment,
+    #   pid_file, logger, stdin, stdout, stderr and supplementary_groups
+    #
+    # @api private
+    def system_command_options
+      {
+        uid: uid,
+        gid: gid,
+        working_dir: working_dir,
+        environment: environment,
+        pid_file: pid_file,
+        logger: logger,
+        stdin: stdin,
+        stdout: stdout,
+        stderr: stderr,
+        supplementary_groups: supplementary_groups,
+      }
+    end
+
+    # Run a block, and dispatch an event if it takes too long
+    #
+    # @param [Integer, Float] secs how long the block may take
+    # @param [String, Symbol] next_state the event to dispatch if it times out
+    #
+    # @yield the work to time
+    # @yieldparam [Numeric] secs the time limit
+    #
+    # @return [Object, Boolean] what the block returned, or whether the event's
+    #   transition happened
+    #
+    # @api private
+    def with_timeout(secs, next_state, &blk)
+      Timeout.timeout(secs, &blk)
+    rescue Timeout::Error
+      logger.err 'Execution is taking longer than expected.'
+      logger.err 'Did you forget to tell bluepill to daemonize this process?'
+      dispatch!(next_state)
+    end
+
+    # Run a command, and wait for it to finish
+    #
+    # A command that fails, or that a signal kills, is logged as a warning.
+    #
+    # @param [String] cmd the command
+    # @param [String] description what the command is, for the log, such as
+    #   Start
+    #
+    # @return [void]
+    #
+    # @api private
+    def run_command(cmd, description)
+      result = System.execute_blocking(cmd, system_command_options)
+      # The exit code is nil when a signal killed the command
+      exit_code = result.fetch(:exit_code)
+      return if exit_code && exit_code.zero?
+
+      logger.warning "#{description} command execution returned non-zero exit code:"
+      logger.warning result.inspect
+    end
+
+    # Send stop signals, with their delays, until the process stops
+    #
+    # Sends the first signal, then each later one after its delay, as long as
+    # the process is still running. Then gives the last signal the rest of
+    # stop_grace_time to work before cleaning up.
+    #
+    # @param [Array<Symbol, Integer>] signals signals with the delays between
+    #
+    # @return [void]
+    #
+    # @api private
+    def send_stop_signals(signals)
+      remaining = stop_grace_time
+      signal = signals.shift
+      logger.info "Sending signal #{signal} to #{actual_pid}"
+      signal_process(signal) # send first signal
+
+      until signals.empty?
+        # we already checked to make sure signals had an odd number of items
+        delay = signals.shift #: Integer
+        signal = signals.shift
+        remaining -= delay
+
+        logger.debug "Sleeping for #{delay} seconds"
+        sleep delay
+        # break unless signal_process(0) #break unless the process can be reached
+        unless signal_process(0)
+          logger.debug 'Process has terminated.'
+          return cleanup_process
+        end
+        logger.info "Sending signal #{signal} to #{actual_pid}"
+        signal_process(signal)
+      end
+      wait_for_stop(remaining)
+      cleanup_process
+    end
+
+    # Wait for the process to exit, checking once a second
+    #
+    # It stops 2 seconds short of the grace time: ticks resume up to a second
+    # early (they count whole seconds) and can kill this thread, and the
+    # cleanup itself takes up to another second.
+    #
+    # @param [Integer, Float] seconds the grace time that's left
+    #
+    # @return [void]
+    #
+    # @api private
+    def wait_for_stop(seconds)
+      (seconds - 2).ceil.times do
+        break unless signal_process(0)
+
+        sleep 1
+      end
+    end
+  end
+end

data/lib/bluepill/process_conditions/always_true.rb

@@ -1,14 +1,28 @@
 module Bluepill
   module ProcessConditions
+    # A check that always passes, for testing a config: `checks :always_true`
+    #
+    # @api private
     class AlwaysTrue < ProcessCondition
-      def initialize(options = {})
-        @below = options[:below]
-      end
-
+      # Return a value that's always fine
+      #
+      # @param [Integer, nil] _pid the process's pid
+      # @param [Boolean] _include_children whether to include its children
+      #
+      # @return [Integer] always 1
+      #
+      # @api private
       def run(_pid, _include_children)
         1
       end
 
+      # Pass whatever the value is
+      #
+      # @param [Integer] _value what run read
+      #
+      # @return [Boolean] always true
+      #
+      # @api private
       def check(_value)
         true
       end

data/lib/bluepill/process_conditions/cpu_usage.rb

@@ -1,15 +1,49 @@
 module Bluepill
   module ProcessConditions
+    # Checks a process's CPU usage, as a percentage
+    #
+    # A config names it `checks :cpu_usage`, with below: the highest
+    # percentage that's fine, and include_children: true to add the
+    # process's children to it.
+    #
+    # @example
+    #   process.checks :cpu_usage, every: 10.seconds, below: 5, times: 3
+    #
+    # @api private
     class CpuUsage < ProcessCondition
+      # Create the check
+      #
+      # @param [Hash{Symbol => Object}] options the check's options, with
+      #   :below, the highest percentage that's fine
+      #
+      # @return [void]
+      #
+      # @api private
       def initialize(options = {})
+        super
         @below = options[:below]
       end
 
+      # Read the process's CPU usage from the tick's ps table
+      #
+      # @param [Integer, nil] pid the process's pid
+      # @param [Boolean] include_children whether to add its children's usage
+      #
+      # @return [Float] the percentage, or 0.0 for a process ps doesn't list
+      #
+      # @api private
       def run(pid, include_children)
         # third col in the ps axu output
         System.cpu_usage(pid, include_children).to_f
       end
 
+      # Return whether the usage is below the limit
+      #
+      # @param [Float] value the percentage run read
+      #
+      # @return [Boolean] true when it's below the limit
+      #
+      # @api private
       def check(value)
         value < @below
       end

data/lib/bluepill/process_conditions/file_time.rb

@@ -1,20 +1,56 @@
 module Bluepill
   module ProcessConditions
+    # Checks how long ago a file changed, in seconds
+    #
+    # A config names it `checks :file_time`, with filename: the file, and
+    # below: the longest it may go without changing (3.minutes, say), to
+    # restart a process that stopped writing its log or heartbeat file.
+    #
+    # @example
+    #   process.checks :file_time, every: 60.seconds, below: 3.minutes, filename: "/tmp/some_file.log", times: 2
+    #
+    # @api private
     class FileTime < ProcessCondition
+      # Create the check
+      #
+      # @param [Hash{Symbol => Object}] options the check's options, with
+      #   :filename, the file, and :below, the longest it may go without
+      #   changing, in seconds
+      #
+      # @return [void]
+      #
+      # @api private
       def initialize(options = {})
+        super
         @below = options[:below]
         @filename = options[:filename]
       end
 
+      # Read the seconds since the file changed
+      #
+      # @param [Integer, nil] _pid the process's pid
+      # @param [Boolean] _include_children whether to include its children
+      #
+      # @return [Float, StandardError, nil] the seconds, nil when the file
+      #   doesn't exist, or the error that stopped bluepill from finding out
+      #
+      # @api private
       def run(_pid, _include_children)
         Time.now - File.mtime(@filename) if File.exist?(@filename)
-      rescue
-        $ERROR_INFO
+      rescue StandardError => e
+        e
       end
 
+      # Return whether the file changed recently enough
+      #
+      # @param [Float, StandardError, nil] value what run read
+      #
+      # @return [Boolean] true when the seconds are below the limit, false
+      #   for a missing file or an error
+      #
+      # @api private
       def check(value)
-        return false if value.nil?
-        value < @below
+        value.is_a?(Numeric) && value < @below
       end
     end
   end

data/lib/bluepill/process_conditions/http.rb

@@ -3,27 +3,63 @@ require 'uri'
 
 module Bluepill
   module ProcessConditions
+    # Checks that a URL answers as expected
+    #
+    # A config names it `checks :http`, with url: the URL to get, kind: the
+    # response it should be (a Net::HTTP class name such as :success or
+    # :ok, or a status code such as 200, :success by default), pattern: a
+    # Regexp its body should match, and timeout: (or open_timeout: and
+    # read_timeout:) in seconds, 5 by default.
+    #
+    # @example
+    #   process.checks :http, url: 'http://localhost:4567/', kind: :success, pattern: /html/, timeout: 3.seconds
+    #
+    # @api private
     class Http < ProcessCondition
+      # Create the check
+      #
+      # @param [Hash{Symbol => Object}] options the check's options, with
+      #   :url, :kind, :pattern, :timeout, :open_timeout and :read_timeout
+      #
+      # @return [void]
+      #
+      # @raise [KeyError] for a numeric kind that isn't an HTTP status
+      #
+      # @raise [NameError] for a kind that names no Net::HTTP response
+      #
+      # @api private
       def initialize(options = {})
-        @uri = URI.parse(options[:url])
-        @kind = begin
-          case options[:kind]
-          when Fixnum
-            Net::HTTPResponse::CODE_TO_OBJ[options[:kind].to_s]
-          when String, Symbol
-            Net.const_get("HTTP#{options[:kind].to_s.camelize}")
-          else
-            Net::HTTPSuccess
-          end
-        end
-        @pattern = options[:pattern] || nil
+        super
+        @uri = URI.parse(options[:url]) #: URI::HTTP
+        @kind = case options[:kind]
+                when Integer
+                  Net::HTTPResponse::CODE_TO_OBJ.fetch(options.fetch(:kind).to_s)
+                when String, Symbol
+                  Net.const_get("HTTP#{options.fetch(:kind).to_s.camelize}")
+                else
+                  Net::HTTPSuccess
+                end
+
+        @pattern = options[:pattern]
         @open_timeout = (options[:open_timeout] || options[:timeout] || 5).to_i
         @read_timeout = (options[:read_timeout] || options[:timeout] || 5).to_i
       end
 
+      # Get the URL
+      #
+      # HTTPS is checked without verifying the certificate.
+      #
+      # @param [Integer, nil] _pid the process's pid
+      # @param [Boolean] _include_children whether to include its children
+      #
+      # @return [Net::HTTPResponse, StandardError] the response, or the error
+      #   that stopped the request
+      #
+      # @api private
       def run(_pid, _include_children)
-        session = Net::HTTP.new(@uri.host, @uri.port)
-        if @uri.scheme == 'https'
+        host = @uri.host #: String
+        session = Net::HTTP.new(host, @uri.port)
+        if @uri.scheme.eql?('https')
           require 'net/https'
           session.use_ssl = true
           session.verify_mode = OpenSSL::SSL::VERIFY_NONE
@@ -35,27 +71,49 @@ module Bluepill
             http.get(@uri.request_uri)
           end
         end
-      rescue
-        $ERROR_INFO
+      rescue StandardError => e
+        e
       end
 
+      # Return whether the response is the expected kind, and matches
+      #
+      # @param [Net::HTTPResponse, StandardError] value what run read
+      #
+      # @return [Boolean] true when the response is the kind the check
+      #   expects, and its body matches the pattern if the check has one
+      #
+      # @api private
       def check(value)
         return false unless value.is_a?(@kind)
         return true  unless @pattern
-        return false unless value.class.body_permitted?
-        @pattern =~ value.body
+
+        response = value #: Net::HTTPResponse
+        !(@pattern =~ response.body).nil?
       end
 
     private
 
+      # Run the block, raising a refused connection as one
+      #
+      # Net::HTTP raises NoMethodError (closed? for nil) when it can't
+      # connect, so this raises Errno::ECONNREFUSED instead.
+      #
+      # @yield the request
+      # @yieldreturn [Object] the response
+      #
+      # @return [Object] what the block returns
+      #
+      # @raise [Errno::ECONNREFUSED] when the connection was refused
+      #
+      # @api private
       def hide_net_http_bug
         yield
       rescue NoMethodError => e
-        if e.to_s =~ /#{Regexp.escape("undefined method `closed?' for nil:NilClass")}/
-          raise(Errno::ECONNREFUSED, "Connection refused attempting to contact #{@uri.scheme}://#{@uri.host}:#{@uri.port}")
-        else
-          raise
-        end
+        # Ruby 3.4 changed the wording of this error
+        message = e.to_s
+        raise(Errno::ECONNREFUSED, "Connection refused attempting to contact #{@uri.scheme}://#{@uri.host}:#{@uri.port}") if message.include?("undefined method `closed?' for nil:NilClass") || message.include?("undefined method 'closed?' for nil")
+
+        raise
       end
     end
   end

data/lib/bluepill/process_conditions/mem_usage.rb

@@ -1,26 +1,85 @@
 module Bluepill
   module ProcessConditions
+    # Checks a process's memory usage (its resident set size)
+    #
+    # A config names it `checks :mem_usage`, with below: the most memory
+    # that's fine, in bytes (100.megabytes, say), and include_children: true
+    # to add the process's children to it.
+    #
+    # @example
+    #   process.checks :mem_usage, every: 10.seconds, below: 100.megabytes, times: [3, 5]
+    #
+    # @api private
     class MemUsage < ProcessCondition
-      MB         = (1024**2).freeze
+      # The bytes in a megabyte, from which a value shows in megabytes
+      #
+      # @api private
+      MB         = (1024**2) #: Integer
+
+      # How a value shows in the history: a number and its unit
+      #
+      # @api private
       FORMAT_STR = '%d%s'.freeze
+
+      # The unit of a value that shows in megabytes
+      #
+      # @api private
       MB_LABEL   = 'MB'.freeze
+
+      # The unit of a value that shows in kilobytes
+      #
+      # @api private
       KB_LABEL   = 'KB'.freeze
 
+      # Create the check
+      #
+      # @param [Hash{Symbol => Object}] options the check's options, with
+      #   :below, the most memory that's fine, in bytes
+      #
+      # @return [void]
+      #
+      # @api private
       def initialize(options = {})
+        super
         @below = options[:below]
       end
 
+      # Read the process's memory usage from the tick's ps table
+      #
+      # @param [Integer, nil] pid the process's pid
+      # @param [Boolean] include_children whether to add its children's usage
+      #
+      # @return [Float] the usage in kilobytes, or 0.0 for a process ps
+      #   doesn't list
+      #
+      # @api private
       def run(pid, include_children)
         # rss is on the 5th col
         System.memory_usage(pid, include_children).to_f
       end
 
+      # Return whether the usage is below the limit
+      #
+      # A limit that can't be compared counts as fine.
+      #
+      # @param [Float] value the kilobytes run read
+      #
+      # @return [Boolean] true when it's below the limit
+      #
+      # @api private
       def check(value)
         value.kilobytes < @below
-      rescue
+      rescue StandardError
         true
       end
 
+      # Format the usage in kilobytes, or in megabytes from a megabyte up
+      #
+      # @param [Float] value the kilobytes run read
+      #
+      # @return [String] the usage and its unit, such as "312KB" or "250MB"
+      #
+      # @api private
       def format_value(value)
         if value.kilobytes >= MB
           format(FORMAT_STR, (value / 1024).round, MB_LABEL)

data/lib/bluepill/process_conditions/process_condition.rb

@@ -1,18 +1,61 @@
 module Bluepill
   module ProcessConditions
+    # Checks something about a process, as one of a process's checks
+    #
+    # A ConditionWatch runs the condition every so often. The condition reads
+    # a value (run), says whether the value is fine (check), and formats it
+    # for the watch's history (format_value). Subclasses implement run and
+    # check.
+    #
+    # @api private
     class ProcessCondition
+      # Create a condition with the options a config gave its check
+      #
+      # @param [Hash{Symbol => Object}] options the check's options, such as
+      #   below: 100.megabytes
+      #
+      # @return [void]
+      #
+      # @api private
       def initialize(options = {})
         @options = options
       end
 
+      # Read the value to check from the process
+      #
+      # @param [Integer, nil] _pid the process's pid
+      # @param [Boolean] _include_children whether to include its children
+      #
+      # @return [Object] the value to check
+      #
+      # @raise [RuntimeError] unless a subclass implements it
+      #
+      # @api private
       def run(_pid, _include_children)
         raise 'Implement in subclass!'
       end
 
+      # Return whether a value is fine
+      #
+      # @param [Object] _value what run read
+      #
+      # @return [Boolean] true when the value is fine, false when it's not
+      #
+      # @raise [RuntimeError] unless a subclass implements it
+      #
+      # @api private
       def check(_value)
         raise 'Implement in subclass!'
       end
 
+      # Format a value for the watch's history and log
+      #
+      # @param [Object] value what run read
+      #
+      # @return [Object] the value as the history shows it, by default the
+      #   value itself
+      #
+      # @api private
       def format_value(value)
         value
       end