bluepill 0.1.2 → 0.1.4

data/lib/bluepill/process_conditions/running_time.rb

@@ -1,14 +1,49 @@
 module Bluepill
   module ProcessConditions
+    # Checks how long a process has been running, in seconds
+    #
+    # A config names it `checks :running_time`, with below: the longest the
+    # process may run (24.hours, say), to restart a process that has run for
+    # too long.
+    #
+    # @example
+    #   process.checks :running_time, every: 10.minutes, below: 24.hours
+    #
+    # @api private
     class RunningTime < ProcessCondition
+      # Create the check
+      #
+      # @param [Hash{Symbol => Object}] options the check's options, with
+      #   :below, the longest the process may run, in seconds
+      #
+      # @return [void]
+      #
+      # @api private
       def initialize(options = {})
+        super
         @below = options[:below]
       end
 
+      # Read how long the process has run from the tick's ps table
+      #
+      # @param [Integer, nil] pid the process's pid
+      # @param [Boolean] _include_children whether to include its children
+      #
+      # @return [Integer] the seconds since it started
+      #
+      # @api private
       def run(pid, _include_children)
-        System.running_time(pid)
+        # Like the other ps conditions, a process missing from ps reads as 0
+        System.running_time(pid).to_i
       end
 
+      # Return whether the process has run for less than the limit
+      #
+      # @param [Integer] value the seconds run read
+      #
+      # @return [Boolean] true when it's below the limit
+      #
+      # @api private
       def check(value)
         value < @below
       end

data/lib/bluepill/process_conditions/zombie_process.rb

@@ -1,17 +1,38 @@
 module Bluepill
   module ProcessConditions
-    # Process must have cache_actual_pid set to false to function correctly:
+    # Checks whether a process is a zombie, which ps shows as <defunct>
     #
-    # process.checks :zombie_process, every: 5.seconds
-    # process.cache_actual_pid = false
-
+    # A config names it `checks :zombie_process`. The process must have
+    # cache_actual_pid set to false for it to work correctly.
+    #
+    # @example
+    #   process.checks :zombie_process, every: 5.seconds
+    #   process.cache_actual_pid = false
+    #
+    # @api private
     class ZombieProcess < ProcessCondition
+      # Read the process's command line from the tick's ps table
+      #
+      # @param [Integer, nil] pid the process's pid
+      # @param [Boolean] _include_children whether to include its children
+      #
+      # @return [String, nil] the command line, or nil for a process ps
+      #   doesn't list
+      #
+      # @api private
       def run(pid, _include_children)
         System.command(pid)
       end
 
+      # Return whether the process isn't a zombie
+      #
+      # @param [String, nil] value the command line run read
+      #
+      # @return [Boolean] true unless the command line says <defunct>
+      #
+      # @api private
       def check(value)
-        (value =~ /\<defunct\>/).nil?
+        (value =~ /<defunct>/).nil?
       end
     end
   end

data/lib/bluepill/process_conditions.rb

@@ -1,5 +1,19 @@
 module Bluepill
+  # The checks a process can run, named in a config by their underscored
+  # class name, such as `checks :mem_usage`
+  #
+  # @api private
   module ProcessConditions
+    # Return the check class a config names
+    #
+    # @param [Symbol, String] name the check's underscored name, such as
+    #   :mem_usage
+    #
+    # @return [Class] the ProcessCondition subclass, such as MemUsage
+    #
+    # @raise [NameError] when no check has that name
+    #
+    # @api private
     def self.[](name)
       const_get(name.to_s.camelcase)
     end
@@ -7,6 +21,6 @@ module Bluepill
 end
 
 require 'bluepill/process_conditions/process_condition'
-Dir["#{File.dirname(__FILE__)}/process_conditions/*.rb"].each do |pc|
+Dir["#{File.dirname(__FILE__)}/process_conditions/*.rb"].sort.each do |pc|
   require pc
 end

data/lib/bluepill/process_journal.rb

@@ -1,172 +1,314 @@
 require 'bluepill/system'
 
 module Bluepill
+  # Remember the pids and process groups of the processes bluepill starts
+  #
+  # Each process has a pid journal and a process group journal, files in the
+  # journal directory with one id a line. When bluepill starts again, it
+  # kills whatever an earlier run left in them.
+  #
+  # @api private
   module ProcessJournal
-  module_function
-
     class << self
+      # Return the logger the journal writes to
+      #
+      # @return [Logger] the first logger that was set
+      #
+      # @api private
       attr_reader :logger
+
+      # Return the directory the journals are in
+      #
+      # @return [String] the journals directory under bluepill's base dir
+      #
+      # @api private
       attr_reader :journal_base_dir
 
+      # Set the logger the journal writes to, unless it already has one
+      #
+      # Only the first logger is kept.
+      #
+      # @param [Logger] new_logger the logger to use
+      #
+      # @return [Logger] the logger
+      #
+      # @api private
       def logger=(new_logger)
         @logger ||= new_logger
       end
 
+      # Set the journal directory from bluepill's base dir, and create it
+      #
+      # Only the first base dir is kept. The directory is made world-writable
+      # so processes that run as other users can lock their journals.
+      #
+      # @param [String] base_dir bluepill's base directory
+      #
+      # @return [void]
+      #
+      # @api private
       def base_dir=(base_dir)
         @journal_base_dir ||= File.join(base_dir, 'journals')
         FileUtils.mkdir_p(@journal_base_dir) unless File.exist?(@journal_base_dir)
-        FileUtils.chmod(0777, @journal_base_dir)
+        FileUtils.chmod(0o777, @journal_base_dir)
       end
     end
 
-    def skip_pid?(pid)
+    # Return whether a pid can't be journaled or signaled
+    #
+    # A pid of 1 or less would reach init, or a whole process group.
+    #
+    # @param [Integer, nil] pid the pid to check
+    #
+    # @return [Boolean] true unless pid is an Integer above 1
+    #
+    # @api private
+    def self.skip_pid?(pid)
       !pid.is_a?(Integer) || pid <= 1
     end
 
-    def skip_pgid?(pgid)
+    # Return whether a process group id can't be journaled or signaled
+    #
+    # @param [Integer, nil] pgid the process group id to check
+    #
+    # @return [Boolean] true unless pgid is an Integer above 1
+    #
+    # @api private
+    def self.skip_pgid?(pgid)
       !pgid.is_a?(Integer) || pgid <= 1
     end
 
-    # atomic operation on POSIX filesystems, since
-    # f.flock(File::LOCK_SH) is not available on all platforms
-    def acquire_atomic_fs_lock(name)
+    # Run a block while holding a lock on a journal
+    #
+    # The lock is a directory, since making one is atomic on POSIX
+    # filesystems and f.flock(File::LOCK_SH) is not available on all
+    # platforms. It waits up to 10 seconds for another process's lock.
+    #
+    # @param [String] name the journal's file name
+    #
+    # @yield while holding the lock
+    # @yieldreturn [Object] what the method returns
+    #
+    # @return [Object] what the block returns
+    #
+    # @raise [RuntimeError] if the lock is still held after 10 tries
+    #
+    # @api private
+    def self.acquire_atomic_fs_lock(name)
+      lock = "#{name}.lock"
       times = 0
-      name += '.lock'
-      Dir.mkdir name, 0700
-      logger.debug("Acquired lock #{name}")
-      yield
-    rescue Errno::EEXIST
-      times += 1
-      logger.debug("Waiting for lock #{name}")
-      sleep 1
-      if times < 10
-        retry
-      else
-        logger.info("Timeout waiting for lock #{name}")
-        raise "Timeout waiting for lock #{name}"
+      begin
+        Dir.mkdir lock, 0o700
+      rescue Errno::EEXIST
+        times += 1
+        if times < 10
+          logger.debug("Waiting for lock #{lock}")
+          sleep 1
+          retry
+        end
+        logger.info("Timeout waiting for lock #{lock}")
+        raise "Timeout waiting for lock #{lock}"
+      end
+      logger.debug("Acquired lock #{lock}")
+      begin
+        yield
+      ensure
+        clear_atomic_fs_lock(lock)
       end
-    ensure
-      clear_atomic_fs_lock(name)
     end
 
-    def clear_all_atomic_fs_locks
+    # Remove the journal locks in the current directory
+    #
+    # @return [void]
+    #
+    # @api private
+    def self.clear_all_atomic_fs_locks
       Dir['.*.lock'].each do |f|
         System.delete_if_exists(f) if File.directory?(f)
       end
     end
 
-    def pid_journal_filename(journal_name)
-      File.join(@journal_base_dir, ".bluepill_pids_journal.#{journal_name}")
+    # Return the path of a process's pid journal
+    #
+    # @param [String] journal_name the process's name
+    #
+    # @return [String] the journal's path
+    #
+    # @api private
+    def self.pid_journal_filename(journal_name)
+      File.join(journal_base_dir, ".bluepill_pids_journal.#{journal_name}")
     end
 
-    def pgid_journal_filename(journal_name)
-      File.join(@journal_base_dir, ".bluepill_pgids_journal.#{journal_name}")
+    # Return the path of a process's process group journal
+    #
+    # @param [String] journal_name the process's name
+    #
+    # @return [String] the journal's path
+    #
+    # @api private
+    def self.pgid_journal_filename(journal_name)
+      File.join(journal_base_dir, ".bluepill_pgids_journal.#{journal_name}")
     end
 
-    def pid_journal(filename)
+    # Return the valid pids in a pid journal
+    #
+    # @param [String] filename the journal's path
+    #
+    # @return [Array<Integer>] the pids, or none if there's no journal
+    #
+    # @api private
+    def self.pid_journal(filename)
       logger.debug("pid journal file: #{filename}")
-      result = File.open(filename, 'r').readlines.collect(&:to_i).reject { |pid| skip_pid?(pid) }
+      result = File.readlines(filename).collect(&:to_i).reject { |pid| skip_pid?(pid) }
       logger.debug("pid journal = #{result.join(' ')}")
       result
     rescue Errno::ENOENT
       []
     end
 
-    def pgid_journal(filename)
+    # Return the valid process group ids in a process group journal
+    #
+    # @param [String] filename the journal's path
+    #
+    # @return [Array<Integer>] the process group ids, or none if there's no
+    #   journal
+    #
+    # @api private
+    def self.pgid_journal(filename)
       logger.debug("pgid journal file: #{filename}")
-      result = File.open(filename, 'r').readlines.collect(&:to_i).reject { |pgid| skip_pgid?(pgid) }
+      result = File.readlines(filename).collect(&:to_i).reject { |pgid| skip_pgid?(pgid) }
       logger.debug("pgid journal = #{result.join(' ')}")
       result
     rescue Errno::ENOENT
       []
     end
 
-    def clear_atomic_fs_lock(name)
+    # Release a journal lock, if it's there
+    #
+    # @param [String] name the lock's directory
+    #
+    # @return [void]
+    #
+    # @api private
+    def self.clear_atomic_fs_lock(name)
       return unless File.directory?(name)
+
       Dir.rmdir(name)
       logger.debug("Cleared lock #{name}")
     end
 
-    def kill_all_from_all_journals
-      pids = Dir['.bluepill_pids_journal.*'].collect { |p| p.sub(/^\.bluepill_pids_journal\./, '') }
-      pids.reject! { |p| p =~ /\.lock$/ }
+    # Kill what every journal in the current directory lists
+    #
+    # @return [void]
+    #
+    # @api private
+    def self.kill_all_from_all_journals
+      pids = Dir['.bluepill_pids_journal.*'].collect { |p| p.sub(/\A\.bluepill_pids_journal\./, '') }
+      pids.reject! { |p| p.end_with?('.lock') }
       pids.each do |journal_name|
         kill_all_from_journal(journal_name)
       end
     end
 
-    def kill_all_from_journal(journal_name)
+    # Kill the processes and process groups that a process's journals list
+    #
+    # @param [String] journal_name the process's name
+    #
+    # @return [void]
+    #
+    # @api private
+    def self.kill_all_from_journal(journal_name)
       kill_all_pids_from_journal(journal_name)
       kill_all_pgids_from_journal(journal_name)
     end
 
-    def kill_all_pgids_from_journal(journal_name)
+    # Kill the process groups that a process's journal lists
+    #
+    # @param [String] journal_name the process's name
+    #
+    # @return [void]
+    #
+    # @api private
+    def self.kill_all_pgids_from_journal(journal_name)
       filename = pgid_journal_filename(journal_name)
-      j = pgid_journal(filename)
-      if j.length > 0
-        acquire_atomic_fs_lock(filename) do
-          j.each do |pgid|
-            begin
-              ::Process.kill('TERM', -pgid)
-              logger.info("Termed old process group #{pgid}")
-            rescue Errno::ESRCH
-              logger.debug("Unable to term missing process group #{pgid}")
-            end
-          end
-
-          if j.count { |pgid| System.pid_alive?(pgid) } > 1
-            sleep(1)
-            j.each do |pgid|
-              begin
-                ::Process.kill('KILL', -pgid)
-                logger.info("Killed old process group #{pgid}")
-              rescue Errno::ESRCH
-                logger.debug("Unable to kill missing process group #{pgid}")
-              end
-            end
-          end
-          System.delete_if_exists(filename) # reset journal
-          logger.debug('Journal cleanup completed')
-        end
-      else
+      kill_all_from_journal_file(filename, pgid_journal(filename), 'process group', -1)
+    end
+
+    # Kill the processes that a process's journal lists
+    #
+    # @param [String] journal_name the process's name
+    #
+    # @return [void]
+    #
+    # @api private
+    def self.kill_all_pids_from_journal(journal_name)
+      filename = pid_journal_filename(journal_name)
+      kill_all_from_journal_file(filename, pid_journal(filename), 'process', 1)
+    end
+
+    # Kill the ids a journal lists, and then delete the journal
+    #
+    # Sends TERM, waits a second for any survivors, then sends KILL to those
+    # still alive. A negative sign targets process groups instead of pids.
+    #
+    # @param [String] filename the journal's path
+    # @param [Array<Integer>] ids the pids or process group ids in it
+    # @param [String] label what the ids are, for the log
+    # @param [Integer] sign 1 for pids, or -1 for process groups
+    #
+    # @return [void]
+    #
+    # @api private
+    def self.kill_all_from_journal_file(filename, ids, label, sign)
+      if ids.empty?
         logger.debug('No previous process journal - Skipping cleanup')
+        return
+      end
+
+      acquire_atomic_fs_lock(filename) do
+        signal_journaled('TERM', ids, label, sign)
+        survivors = ids.select { |id| System.pid_alive?(sign * id) }
+        unless survivors.empty?
+          sleep(1)
+          survivors = survivors.select { |id| System.pid_alive?(sign * id) }
+          signal_journaled('KILL', survivors, label, sign)
+        end
+        System.delete_if_exists(filename) # reset journal
+        logger.debug('Journal cleanup completed')
       end
     end
 
-    def kill_all_pids_from_journal(journal_name)
-      filename = pid_journal_filename(journal_name)
-      j = pid_journal(filename)
-      if j.length > 0
-        acquire_atomic_fs_lock(filename) do
-          j.each do |pid|
-            begin
-              ::Process.kill('TERM', pid)
-              logger.info("Termed old process #{pid}")
-            rescue Errno::ESRCH
-              logger.debug("Unable to term missing process #{pid}")
-            end
-          end
-
-          if j.count { |pid| System.pid_alive?(pid) } > 1
-            sleep(1)
-            j.each do |pid|
-              begin
-                ::Process.kill('KILL', pid)
-                logger.info("Killed old process #{pid}")
-              rescue Errno::ESRCH
-                logger.debug("Unable to kill missing process #{pid}")
-              end
-            end
-          end
-          System.delete_if_exists(filename) # reset journal
-          logger.debug('Journal cleanup completed')
+    # Send a signal to each journaled id, and log what happened
+    #
+    # @param [String] signal TERM or KILL
+    # @param [Array<Integer>] ids the pids or process group ids
+    # @param [String] label what the ids are, for the log
+    # @param [Integer] sign 1 for pids, or -1 for process groups
+    #
+    # @return [void]
+    #
+    # @api private
+    def self.signal_journaled(signal, ids, label, sign)
+      past, verb = signal.eql?('TERM') ? %w[Termed term] : %w[Killed kill]
+      ids.each do |id|
+        begin
+          ::Process.kill(signal, sign * id)
+          logger.info("#{past} old #{label} #{id}")
+        rescue Errno::ESRCH
+          logger.debug("Unable to #{verb} missing #{label} #{id}")
         end
-      else
-        logger.debug('No previous process journal - Skipping cleanup')
       end
     end
 
-    def append_pgid_to_journal(journal_name, pgid)
+    # Add a process group id to a process's journal, unless it's there
+    #
+    # @param [String] journal_name the process's name
+    # @param [Integer, nil] pgid the process group id
+    #
+    # @return [void]
+    #
+    # @api private
+    def self.append_pgid_to_journal(journal_name, pgid)
       if skip_pgid?(pgid)
         logger.debug("Skipping invalid pgid #{pgid} (class #{pgid.class})")
         return
@@ -178,22 +320,32 @@ module Bluepill
           logger.debug("Skipping duplicate pgid #{pgid} already in journal #{journal_name}")
         else
           logger.debug("Saving pgid #{pgid} to process journal #{journal_name}")
-          File.open(filename, 'a+', 0600) { |f| f.puts(pgid) }
+          File.open(filename, 'a+', 0o600) { |f| f.puts(pgid) }
           logger.info("Saved pgid #{pgid} to journal #{journal_name}")
-          logger.debug("Journal now = #{File.open(filename, 'r').read}")
+          logger.debug("Journal now = #{File.read(filename)}")
         end
       end
     end
 
-    def append_pid_to_journal(journal_name, pid)
-      begin
-        append_pgid_to_journal(journal_name, ::Process.getpgid(pid))
-      rescue Errno::ESRCH
-      end
+    # Add a pid, and its process group id, to a process's journals
+    #
+    # @param [String] journal_name the process's name
+    # @param [Integer, nil] pid the pid
+    #
+    # @return [void]
+    #
+    # @api private
+    def self.append_pid_to_journal(journal_name, pid)
       if skip_pid?(pid)
         logger.debug("Skipping invalid pid #{pid} (class #{pid.class})")
         return
       end
+      # skip_pid? made sure it's an Integer
+      valid_pid = pid #: Integer
+      begin
+        append_pgid_to_journal(journal_name, ::Process.getpgid(valid_pid))
+      rescue Errno::ESRCH
+      end
 
       filename = pid_journal_filename(journal_name)
       acquire_atomic_fs_lock(filename) do
@@ -201,9 +353,9 @@ module Bluepill
           logger.debug("Skipping duplicate pid #{pid} already in journal #{journal_name}")
         else
           logger.debug("Saving pid #{pid} to process journal #{journal_name}")
-          File.open(filename, 'a+', 0600) { |f| f.puts(pid) }
+          File.open(filename, 'a+', 0o600) { |f| f.puts(pid) }
           logger.info("Saved pid #{pid} to journal #{journal_name}")
-          logger.debug("Journal now = #{File.open(filename, 'r').read}")
+          logger.debug("Journal now = #{File.read(filename)}")
         end
       end
     end

data/lib/bluepill/process_statistics.rb

@@ -1,21 +1,51 @@
 module Bluepill
+  # Keeps a process's last few events, which bluepill status shows
+  #
+  # @api private
   class ProcessStatistics
-    STRFTIME = '%m/%d/%Y %H:%I:%S'.freeze
+    # How status shows each event's time
+    STRFTIME = '%m/%d/%Y %H:%M:%S'.freeze
+
+    # How many events are kept
     EVENTS_TO_PERSIST = 10
 
+    # Return the recorded events, newest last
+    #
+    # @return [Util::RotationalArray<Array(Symbol, String, nil, Time)>] each
+    #   event, its reason and its time
+    #
+    # @api private
     attr_reader :events
 
-    # possibly persist this data.
+    # Create an empty event history
+    #
+    # @return [void]
+    #
+    # @api private
     def initialize
       @events = Util::RotationalArray.new(EVENTS_TO_PERSIST)
     end
 
+    # Add an event to the history, with the time it happened
+    #
+    # @param [Symbol, String] event the event, such as :restart
+    # @param [String, nil] reason why it happened
+    #
+    # @return [Util::RotationalArray<Array(Symbol, String, nil, Time)>] the
+    #   history, newest last
+    #
+    # @api private
     def record_event(event, reason)
       events.push([event, reason, Time.now])
     end
 
+    # Return the history as text, newest first, for bluepill status
+    #
+    # @return [String]
+    #
+    # @api private
     def to_s
-      str = events.reverse.collect do |(event, reason, time)|
+      str = events.reverse.collect do |event, reason, time|
         "  #{event} at #{time.strftime(STRFTIME)} - #{reason || 'unspecified'}"
       end.join("\n")