bluepill 0.1.2 → 0.1.4

data/lib/bluepill/application.rb

@@ -3,43 +3,153 @@ require 'bluepill/system'
 require 'bluepill/process_journal'
 
 module Bluepill
+  # The bluepill daemon for one application
+  #
+  # It holds the application's groups of processes, ticks them every
+  # second, and answers the commands that the CLI sends over its socket.
+  #
+  # @api private
   class Application
-    PROCESS_COMMANDS = [:start, :stop, :restart, :unmonitor, :status]
-
-    attr_accessor :name, :logger, :base_dir, :socket, :pid_file, :kill_timeout
-    attr_accessor :groups, :work_queue
-    attr_accessor :pids_dir, :log_file
-
+    # The commands that act on a process or group, and that the CLI sends
+    PROCESS_COMMANDS = [:start, :stop, :restart, :unmonitor, :status].freeze
+
+    attr_accessor :name, :logger, :base_dir, :socket, :pid_file, :kill_timeout, :groups, :work_queue, :pids_dir, :log_file
+
+    # @!attribute [rw] name
+    #   The application's name, which names its socket and pid file
+    #
+    #   @return [String]
+    #
+    #   @api private
+
+    # @!attribute [rw] logger
+    #   The application's logger, prefixed with its name
+    #
+    #   @return [Logger]
+    #
+    #   @api private
+
+    # @!attribute [rw] base_dir
+    #   The directory with the daemon's sockets, pid files and journals
+    #
+    #   @return [String]
+    #
+    #   @api private
+
+    # @!attribute [rw] socket
+    #   The socket the daemon listens for commands on, once it's started
+    #
+    #   @return [UNIXServer, nil]
+    #
+    #   @api private
+
+    # @!attribute [rw] pid_file
+    #   The daemon's own pid file
+    #
+    #   @return [String]
+    #
+    #   @api private
+
+    # @!attribute [rw] kill_timeout
+    #   How many half seconds to wait for a previous daemon to exit
+    #
+    #   @return [Integer]
+    #
+    #   @api private
+
+    # @!attribute [rw] groups
+    #   The application's groups by name, with nil for processes in no group
+    #
+    #   @return [Hash{String, nil => Group}]
+    #
+    #   @api private
+
+    # @!attribute [rw] work_queue
+    #   An attribute that bluepill doesn't use
+    #
+    #   @return [Object]
+    #
+    #   @api private
+
+    # @!attribute [rw] pids_dir
+    #   The directory with the pid files of the application's processes
+    #
+    #   @return [String]
+    #
+    #   @api private
+
+    # @!attribute [rw] log_file
+    #   The file the daemon logs to, or nil to log to syslog
+    #
+    #   @return [String, nil]
+    #
+    #   @api private
+
+    # Create the daemon for an application
+    #
+    # It sets up signal traps, the logger and the pids directory. It exits
+    # with status 5 if it can't write to its log file or base directory.
+    #
+    # @param [String] name the application's name
+    # @param [Hash{Symbol => Object}] options :foreground, :log_file,
+    #   :base_dir and :kill_timeout
+    #
+    # @return [void]
+    #
+    # @api private
     def initialize(name, options = {})
       self.name = name
 
       @foreground   = options[:foreground]
       self.log_file = options[:log_file]
+      # Only used for a non-root user, who has a home
+      home = ENV['HOME'] #: String
       self.base_dir = ProcessJournal.base_dir = options[:base_dir] ||
         ENV['BLUEPILL_BASE_DIR'] ||
-        (::Process.euid != 0 ? File.join(ENV['HOME'], '.bluepill') : '/var/run/bluepill')
-      self.pid_file = File.join(base_dir, 'pids', self.name + '.pid')
-      self.pids_dir = File.join(base_dir, 'pids', self.name)
+        (::Process.euid.zero? ? '/var/run/bluepill' : File.join(home, '.bluepill'))
+      self.pid_file = File.join(base_dir, 'pids', "#{name}.pid")
+      self.pids_dir = File.join(base_dir, 'pids', name)
       self.kill_timeout = options[:kill_timeout] || 10
 
       self.groups = {}
 
-      self.logger = ProcessJournal.logger = Bluepill::Logger.new(log_file: log_file, stdout: foreground?).prefix_with(self.name)
-
       setup_signal_traps
-      setup_pids_dir
+      setup_logger_and_pids_dir
 
       @mutex = Mutex.new
     end
 
+    # Return whether the daemon runs in the foreground
+    #
+    # @return [Boolean]
+    #
+    # @api private
     def foreground?
       !!@foreground
     end
 
-    def mutex(&b)
-      @mutex.synchronize(&b)
+    # Run a block while holding the daemon's lock
+    #
+    # Ticks and commands from the CLI hold it, so they never overlap.
+    #
+    # @yield the work to do while holding the lock
+    # @yieldreturn [Object] what the block returns
+    #
+    # @return [Object] what the block returns
+    #
+    # @api private
+    def mutex(&block)
+      @mutex.synchronize(&block)
     end
 
+    # Start the daemon and monitor the application's processes
+    #
+    # It prints what went wrong, and exits with status 5, if the daemon
+    # fails to start.
+    #
+    # @return [void]
+    #
+    # @api private
     def load
       start_server
     rescue StandardError => e
@@ -49,65 +159,157 @@ module Bluepill
       exit(5)
     end
 
+    # @!method start(group_name = nil, process_name = nil)
+    #   Start a process or group, or every process without either
+    #
+    #   @param [String, nil] group_name the group, or the process's name
+    #     when no group has it
+    #   @param [String, nil] process_name the process
+    #
+    #   @return [Array<String>] the affected processes, as group:process
+    #
+    #   @api private
+
+    # @!method stop(group_name = nil, process_name = nil)
+    #   Stop a process or group, or every process without either
+    #
+    #   @param [String, nil] group_name the group, or the process's name
+    #     when no group has it
+    #   @param [String, nil] process_name the process
+    #
+    #   @return [Array<String>] the affected processes, as group:process
+    #
+    #   @api private
+
+    # @!method restart(group_name = nil, process_name = nil)
+    #   Restart a process or group, or every process without either
+    #
+    #   @param [String, nil] group_name the group, or the process's name
+    #     when no group has it
+    #   @param [String, nil] process_name the process
+    #
+    #   @return [Array<String>] the affected processes, as group:process
+    #
+    #   @api private
+
+    # @!method unmonitor(group_name = nil, process_name = nil)
+    #   Stop monitoring a process or group, or every process without either
+    #
+    #   @param [String, nil] group_name the group, or the process's name
+    #     when no group has it
+    #   @param [String, nil] process_name the process
+    #
+    #   @return [Array<String>] the affected processes, as group:process
+    #
+    #   @api private
+
+    # @!method status(group_name = nil, process_name = nil)
+    #   Return the status of a process or group, or of every process
+    #
+    #   @param [String, nil] group_name the group, or the process's name
+    #     when no group has it
+    #   @param [String, nil] process_name the process
+    #
+    #   @return [Array<String>] the status lines
+    #
+    #   @api private
+
+    # Define a method for each of the PROCESS_COMMANDS
     PROCESS_COMMANDS.each do |command|
-      class_eval <<-END
+      class_eval <<-RUBY, __FILE__, __LINE__ + 1
         def #{command}(group_name = nil, process_name = nil)
           self.send_to_process_or_group(:#{command}, group_name, process_name)
         end
-      END
+      RUBY
     end
 
+    # Add a process to a group, which is created if it's new
+    #
+    # @param [Process] process the process
+    # @param [#to_s, nil] group_name the group, or nil for no group
+    #
+    # @return [Array<Process>] the group's processes
+    #
+    # @api private
     def add_process(process, group_name = nil)
       group_name = group_name.to_s if group_name
 
       groups[group_name] ||= Group.new(group_name, logger: logger.prefix_with(group_name))
-      groups[group_name].add_process(process)
+      groups.fetch(group_name).add_process(process)
     end
 
+    # Return the daemon's version, which the CLI checks against its own
+    #
+    # @return [Module] the Version module
+    #
+    # @api private
     def version
-      Bluepill::Version
+      Version
     end
 
   protected
 
+    # Send a command to a process, a group, or every process
+    #
+    # @param [Symbol] method the command
+    # @param [String, nil] group_name the group, or the start of a process
+    #   name that contains a colon
+    # @param [String, nil] process_name the process
+    #
+    # @return [Array<String>] what the groups returned, flattened
+    #
+    # @api private
     def send_to_process_or_group(method, group_name, process_name)
       if group_name.nil? && process_name.nil?
         groups.values.collect do |group|
-          group.send(method)
+          group.public_send(method)
         end.flatten
       elsif groups.key?(group_name)
-        groups[group_name].send(method, process_name)
-      elsif process_name.nil?
-        # they must be targeting just by process name
-        process_name = group_name
+        groups.fetch(group_name).public_send(method, process_name)
+      else
+        # they must be targeting just by process name, which may contain ':'
+        process_name = [group_name, process_name].compact.join(':')
         groups.values.collect do |group|
-          group.send(method, process_name)
+          group.public_send(method, process_name)
         end.flatten
-      else
-        []
       end
     end
 
+    # Start the thread that answers commands on the socket
+    #
+    # It kills the thread that a previous call started. Each command is a
+    # line, such as start:group:process, and the answer is the command's
+    # result, or the error it raised, marshaled.
+    #
+    # @return [Thread] the listener thread
+    #
+    # @api private
     def start_listener
-      @listener_thread.kill if @listener_thread
+      previous_thread = @listener_thread
+      previous_thread.kill if previous_thread
       @listener_thread = Thread.new do
         loop do
+          client = nil #: UNIXSocket?
           begin
-            client = socket.accept
-            client.close_on_exec = true  if client.respond_to?(:close_on_exec=)
-            command, *args = client.readline.strip.split(':')
+            server = socket #: UNIXServer
+            client = server.accept
+            client.close_on_exec = true if client.respond_to?(:close_on_exec=)
+            # The target is group:process and the process name may contain ':'
+            command, *args = client.readline.strip.split(':', 3).reject(&:empty?)
             response = begin
-              mutex { send(command, *args) }
-            rescue => e
+              # An empty line leaves command nil, so public_send raises an error for the client
+              mutex { public_send(command, *args) } # steep:ignore ArgumentTypeMismatch
+            rescue StandardError => e
               e
             end
             client.write(Marshal.dump(response))
           rescue StandardError => e
             logger.err(format('Got exception in cmd listener: %s `%s`', e.class.name, e.message))
-            e.backtrace.each { |l| logger.err(l) }
+            Array(e.backtrace).each { |l| logger.err(l) }
           ensure
             begin
-              client.close
+              # Ruby before 3.0 never counts branches in ensure as covered
+              client && client.close
             rescue IOError
               # closed stream
             end
@@ -116,6 +318,15 @@ module Bluepill
       end
     end
 
+    # Start the daemon
+    #
+    # It stops a previous daemon for the application, kills what the
+    # journals recorded, daemonizes unless it runs in the foreground, opens
+    # the socket, and runs until a signal stops it.
+    #
+    # @return [void]
+    #
+    # @api private
     def start_server
       kill_previous_bluepill
       ProcessJournal.kill_all_from_all_journals
@@ -132,26 +343,36 @@ module Bluepill
 
       $0 = "bluepilld: #{name}"
 
-      groups.each { |_, group| group.determine_initial_state }
+      groups.each_value(&:determine_initial_state)
 
       write_pid_file
-      self.socket = Bluepill::Socket.server(base_dir, name)
+      self.socket = Socket.server(base_dir, name)
       start_listener
 
       run
     end
 
+    # Tick every group once a second, until a signal stops the daemon
+    #
+    # @return [void]
+    #
+    # @api private
     def run
       @running = true # set to false by signal trap
       while @running
         mutex do
           System.reset_data
-          groups.each { |_, group| group.tick }
+          groups.each_value(&:tick)
         end
         sleep 1
       end
     end
 
+    # Kill what the journals recorded and remove the socket and pid file
+    #
+    # @return [void]
+    #
+    # @api private
     def cleanup
       ProcessJournal.kill_all_from_all_journals
       ProcessJournal.clear_all_atomic_fs_locks
@@ -162,12 +383,17 @@ module Bluepill
       System.delete_if_exists(pid_file)
     end
 
+    # Trap TERM and INT to clean up and stop, and HUP to reopen the log
+    #
+    # @return [void]
+    #
+    # @api private
     def setup_signal_traps
       terminator = proc do
         puts 'Terminating...'
         cleanup
         @running = false
-      end
+      end #: ^(Integer signal) -> void
 
       Signal.trap('TERM', &terminator)
       Signal.trap('INT', &terminator)
@@ -177,24 +403,61 @@ module Bluepill
       end
     end
 
+    # Create the logger and the pids directory
+    #
+    # Bluepill opens its log file and creates its pids dir as the user that
+    # runs it. The uid and gid in the config only apply to its processes.
+    # It exits with status 5 if it can't.
+    #
+    # @return [void]
+    #
+    # @api private
+    def setup_logger_and_pids_dir
+      self.logger = ProcessJournal.logger = Logger.new(log_file: log_file, stdout: foreground?).prefix_with(name)
+      setup_pids_dir
+    rescue SystemCallError => e
+      $stderr.puts("Failed to start bluepill: #{e}")
+      message = 'The user running bluepill must be able to write to its log_file and base_dir. ' \
+                'The uid and gid in the config only apply to the processes it monitors.'
+      $stderr.puts(message)
+      exit(5)
+    end
+
+    # Create the pids directory, and let every user write to it
+    #
+    # @return [void]
+    #
+    # @api private
     def setup_pids_dir
       FileUtils.mkdir_p(pids_dir) unless File.exist?(pids_dir)
       # we need everybody to be able to write to the pids_dir as processes managed by
       # bluepill will be writing to this dir after they've dropped privileges
-      FileUtils.chmod(0777, pids_dir)
+      FileUtils.chmod(0o777, pids_dir)
     end
 
+    # Interrupt a daemon for this application that's still running
+    #
+    # It waits up to kill_timeout half seconds for the daemon to exit, and
+    # exits with status 4 if it doesn't, or if signaling it fails.
+    #
+    # @return [void]
+    #
+    # @api private
     def kill_previous_bluepill
       return unless File.exist?(pid_file)
+
+      # An empty or garbled pid file reads as 0, and signaling pid 0 would
+      # interrupt bluepill's own process group
       previous_pid = File.read(pid_file).to_i
-      return unless System.pid_alive?(previous_pid)
+      return unless previous_pid > 0 && System.pid_alive?(previous_pid)
+
       ::Process.kill(0, previous_pid)
       puts "Killing previous bluepilld[#{previous_pid}]"
       ::Process.kill(2, previous_pid)
-    rescue => e
+    rescue StandardError => e
       $stderr.puts 'Encountered error trying to kill previous bluepill:'
-      $stderr.puts "#{e.class}: #{e.message}"
-      exit(4) unless e.is_a?(Errno::ESRCH)
+      $stderr.puts "#{e.class}: #{e}"
+      exit(4) unless e.instance_of?(Errno::ESRCH)
     else
       kill_timeout.times do |_i|
         sleep 0.5
@@ -207,6 +470,11 @@ module Bluepill
       end
     end
 
+    # Write the daemon's pid to its pid file
+    #
+    # @return [void]
+    #
+    # @api private
     def write_pid_file
       File.open(pid_file, 'w') { |x| x.write(::Process.pid) }
     end

data/lib/bluepill/condition_watch.rb

@@ -1,52 +1,147 @@
 module Bluepill
+  # One entry of a watch's history: the value it read, as its log shows it,
+  # and whether the value was fine
+  #
+  # @!attribute value
+  #   Return the value as the history shows it
+  #
+  #   @return [#to_s] the formatted value
+  #
+  #   @api private
+  #
+  # @!attribute critical
+  #   Return whether the value was fine
+  #
+  #   It's true for a value that passed the check, and false for one that
+  #   counts toward the watch firing.
+  #
+  #   @return [Boolean] whether the value passed the check
+  #
+  #   @api private
+  #
+  # @api private
   HistoryValue = Struct.new(:value, :critical)
 
+  # Runs a process's check every so often, and fires its events when the
+  # check fails often enough
+  #
+  # A config's `checks :mem_usage, every: 10.seconds, below: 100.megabytes,
+  # times: [3, 5]` makes one: it runs every 10 seconds, and fires (restarts
+  # the process, by default) when 3 of the last 5 values failed.
+  #
+  # @api private
   class ConditionWatch
-    attr_accessor :logger, :name
-    EMPTY_ARRAY = [].freeze # no need to recreate one every tick
+    # Return the logger the watch writes its history to
+    #
+    # @return [Logger] the process's logger
+    #
+    # @api private
+    attr_accessor :logger
 
+    # Return the check's name, such as :mem_usage
+    #
+    # @return [Symbol] the name a config gave the check
+    #
+    # @api private
+    attr_accessor :name
+
+    no_events = [] #: Array[Symbol]
+
+    # The events of a run that fires nothing, made once instead of every tick
+    #
+    # @api private
+    EMPTY_ARRAY = no_events.freeze
+
+    # Create a watch of a check
+    #
+    # @param [Symbol] name the check's name, such as :mem_usage
+    # @param [Hash{Symbol => Object}] options the check's options. The
+    #   watch takes :logger, :fires (the events to fire, [:restart] by
+    #   default), :every (seconds between runs), :times (how many of the
+    #   last values must fail, as a count or [count, of]), and
+    #   :include_children. The check gets the rest.
+    #
+    # @return [void]
+    #
+    # @api private
     def initialize(name, options = {})
       @name = name
 
       @logger = options.delete(:logger)
       @fires  = options.key?(:fires) ? Array(options.delete(:fires)) : [:restart]
       @every  = options.delete(:every)
-      @times  = options.delete(:times) || [1, 1]
-      @times  = [@times, @times] unless @times.is_a?(Array) # handles times: 5
+      @times  = Array(options.delete(:times) || 1) # handles times: 5
       @include_children = options.delete(:include_children) || false
 
-      self.clear_history!
+      clear_history!
 
       @process_condition = ProcessConditions[@name].new(options)
     end
 
-    def run(pid, tick_number = Time.now.to_i)
-      if @last_ran_at.nil? || (@last_ran_at + @every) <= tick_number
+    # Run the check if it's due, and return the events to fire
+    #
+    # A check is due on its first run, and then every `every` seconds. A
+    # run adds the value to the history and logs the history.
+    #
+    # @param [Integer, nil] pid the process's pid
+    # @param [Integer] tick_number the current time, in seconds
+    #
+    # @return [Array<Symbol>] the events to fire, empty unless the check
+    #   ran and failed often enough
+    #
+    # @raise [StandardError] whatever the check raised, other than a
+    #   System::PsError, after logging it
+    #
+    # @api private
+    def run(pid, tick_number = Integer(Time.now))
+      last_ran_at = @last_ran_at
+      if last_ran_at.nil? || (last_ran_at + @every) <= tick_number
         @last_ran_at = tick_number
 
         begin
           value = @process_condition.run(pid, @include_children)
-        rescue => e
+        rescue System::PsError => e
+          # No data this time, so leave the history alone and check again next interval
+          logger.warning("#{@name}: #{e}")
+          return EMPTY_ARRAY
+        rescue StandardError => e
           logger.err(e.backtrace)
-          raise e
+          raise
         end
 
         @history << HistoryValue.new(@process_condition.format_value(value), @process_condition.check(value))
         logger.info(to_s)
 
-        return @fires if self.fired?
+        return @fires if fired?
       end
       EMPTY_ARRAY
     end
 
+    # Forget the values the check read, as when the process changes state
+    #
+    # @return [Util::RotationalArray<HistoryValue>] the new, empty history
+    #
+    # @api private
     def clear_history!
       @history = Util::RotationalArray.new(@times.last)
     end
 
+    # Return whether enough of the last values failed for the watch to fire
+    #
+    # @return [Boolean] true when the failures reach the times the check
+    #   allows
+    #
+    # @api private
     def fired?
       @history.count { |v| !v.critical } >= @times.first
     end
 
+    # Return the check's name and history, as the log shows it
+    #
+    # @return [String] such as "mem_usage: [312MB, 401MB*]", where * marks a
+    #   value that failed
+    #
+    # @api private
     def to_s
       data = @history.collect { |v| "#{v.value}#{'*' unless v.critical}" }.join(', ')
       "#{@name}: [#{data}]\n"