bluepill 0.1.2 → 0.1.4

data/lib/bluepill/system.rb

@@ -1,15 +1,25 @@
-require 'etc'
-require 'shellwords'
-
 module Bluepill
-  # This class represents the system that bluepill is running on.. It's mainly used to memoize
-  # results of running ps auxx etc so that every watch in the every process will not result in a fork
+  # Read what the system says about its processes, and run commands
+  #
+  # This represents the system that bluepill is running on. It memoizes
+  # the results of running ps each tick, so that every watch of every
+  # process doesn't fork a ps of its own.
+  #
+  # @api private
   module System
-    APPEND_MODE = 'a'
+    # The mode that a daemon's stdout and stderr files are opened in
+    #
+    # @api private
+    APPEND_MODE = 'a'.freeze
 
-  module_function
+    # Raised when ps could not be run, so there is no data for this tick
+    #
+    # @api private
+    class PsError < StandardError; end
 
     # The position of each field in ps output
+    #
+    # @api private
     IDX_MAP = {
       pid: 0,
       ppid: 1,
@@ -17,108 +27,142 @@ module Bluepill
       rss: 3,
       etime: 4,
       command: 5,
-    }
-
-    def pid_alive?(pid)
+    }.freeze #: { pid: Integer, ppid: Integer, pcpu: Integer, rss: Integer, etime: Integer, command: Integer }
+
+    # Return whether a process is running
+    #
+    # @param [Integer] pid the process's pid
+    #
+    # @return [Boolean] true if the process exists, even if bluepill may
+    #   not signal it
+    #
+    # @api private
+    def self.pid_alive?(pid)
       ::Process.kill(0, pid)
       true
     rescue Errno::EPERM # no permission, but it is definitely alive
       true
-    rescue Errno::ESRCH
-      false
-    rescue
-      false # other falsy (no pid)
+    rescue StandardError
+      false # no such process (ESRCH), or no pid
     end
 
-    def cpu_usage(pid, include_children)
+    # Return a process's CPU usage, from this tick's ps table
+    #
+    # @param [Integer, nil] pid the process's pid
+    # @param [Boolean] include_children whether to add its descendants'
+    #
+    # @return [Float, nil] the percent of a CPU it uses, or nil if ps doesn't
+    #   list it
+    #
+    # @raise [PsError] if ps could not be run this tick
+    #
+    # @api private
+    def self.cpu_usage(pid, include_children)
       ps = ps_axu
-      return unless ps[pid]
-      cpu_used = ps[pid][IDX_MAP[:pcpu]].to_f
-      get_children(pid).each do |child_pid|
-        cpu_used += ps[child_pid][IDX_MAP[:pcpu]].to_f if ps[child_pid]
-      end if include_children
+      return unless ps.key?(pid)
+
+      cpu_used = ps.fetch(pid).fetch(IDX_MAP.fetch(:pcpu)).to_f
+      if include_children
+        get_children(pid).each do |child_pid|
+          cpu_used += ps.fetch(child_pid).fetch(IDX_MAP.fetch(:pcpu)).to_f if ps.key?(child_pid)
+        end
+      end
       cpu_used
     end
 
-    def memory_usage(pid, include_children)
+    # Return a process's resident memory, from this tick's ps table
+    #
+    # @param [Integer, nil] pid the process's pid
+    # @param [Boolean] include_children whether to add its descendants'
+    #
+    # @return [Float, nil] its resident memory in kilobytes, or nil if ps
+    #   doesn't list it
+    #
+    # @raise [PsError] if ps could not be run this tick
+    #
+    # @api private
+    def self.memory_usage(pid, include_children)
       ps = ps_axu
-      return unless ps[pid]
-      mem_used = ps[pid][IDX_MAP[:rss]].to_f
-      get_children(pid).each do |child_pid|
-        mem_used += ps[child_pid][IDX_MAP[:rss]].to_f if ps[child_pid]
-      end if include_children
+      return unless ps.key?(pid)
+
+      mem_used = ps.fetch(pid).fetch(IDX_MAP.fetch(:rss)).to_f
+      if include_children
+        get_children(pid).each do |child_pid|
+          mem_used += ps.fetch(child_pid).fetch(IDX_MAP.fetch(:rss)).to_f if ps.key?(child_pid)
+        end
+      end
       mem_used
     end
 
-    def running_time(pid)
+    # Return how long a process has been running, from this tick's ps table
+    #
+    # @param [Integer, nil] pid the process's pid
+    #
+    # @return [Integer, nil] its running time in seconds, or nil if ps
+    #   doesn't list it
+    #
+    # @raise [PsError] if ps could not be run this tick
+    #
+    # @api private
+    def self.running_time(pid)
       ps = ps_axu
-      return unless ps[pid]
-      parse_elapsed_time(ps[pid][IDX_MAP[:etime]])
+      return unless ps.key?(pid)
+
+      parse_elapsed_time(ps.fetch(pid).fetch(IDX_MAP.fetch(:etime)))
     end
 
-    def command(pid)
+    # Return a process's command line, from this tick's ps table
+    #
+    # @param [Integer, nil] pid the process's pid
+    #
+    # @return [String, nil] its command and arguments, or nil if ps doesn't
+    #   list it
+    #
+    # @raise [PsError] if ps could not be run this tick
+    #
+    # @api private
+    def self.command(pid)
       ps = ps_axu
-      return unless ps[pid]
-      ps[pid][IDX_MAP[:command]]
+      return unless ps.key?(pid)
+
+      ps.fetch(pid).fetch(IDX_MAP.fetch(:command))
     end
 
-    def get_children(parent_pid)
-      child_pids = []
+    # Return the pids of a process's descendants, from this tick's ps table
+    #
+    # @param [Integer, nil] parent_pid the process's pid
+    #
+    # @return [Array<Integer>] its children's pids, followed by theirs, or
+    #   none if ps could not be run
+    #
+    # @api private
+    def self.get_children(parent_pid)
+      # A process that is still starting has no pid yet, and nil.to_i is 0.
+      # The ps header row parses as pid 0 with ppid 0, so it would be its own
+      # child and the recursion would never end.
+      return [] unless parent_pid.to_i > 0
+
+      child_pids = [] #: Array[Integer]
       ps_axu.each_pair do |_pid, chunks|
-        child_pids << chunks[IDX_MAP[:pid]].to_i if chunks[IDX_MAP[:ppid]].to_i == parent_pid.to_i
+        child_pids << Integer(chunks.fetch(IDX_MAP.fetch(:pid))) if chunks.fetch(IDX_MAP.fetch(:ppid)).to_i.eql?(parent_pid)
       end
       grand_children = child_pids.collect { |pid| get_children(pid) }.flatten
       child_pids.concat grand_children
+    rescue PsError
+      []
     end
 
-    # Returns the pid of the child that executes the cmd
-    def daemonize(cmd, options = {})
-      rd, wr = IO.pipe
-
-      child = Daemonize.safefork
-      if child
-        # we don't want to create zombies, so detach ourselves from the child exit status
-        ::Process.detach(child)
-
-        # parent
-        wr.close
-
-        daemon_id = rd.read.to_i
-        rd.close
-
-        return daemon_id if daemon_id > 0
-      else
-        # child
-        rd.close
-
-        drop_privileges(options[:uid], options[:gid], options[:supplementary_groups])
-
-        # if we cannot write the pid file as the provided user, err out
-        exit unless can_write_pid_file(options[:pid_file], options[:logger])
-
-        to_daemonize = lambda do
-          # Setting end PWD env emulates bash behavior when dealing with symlinks
-          Dir.chdir(ENV['PWD'] = options[:working_dir].to_s) if options[:working_dir]
-          options[:environment].each { |key, value| ENV[key.to_s] = value.to_s } if options[:environment]
-
-          redirect_io(*options.values_at(:stdin, :stdout, :stderr))
-
-          ::Kernel.exec(*Shellwords.shellwords(cmd))
-          exit
-        end
-
-        daemon_id = Daemonize.call_as_daemon(to_daemonize, nil, cmd)
-
-        File.open(options[:pid_file], 'w') { |f| f.write(daemon_id) }
-
-        wr.write daemon_id
-        wr.close
-
-        ::Process.exit!(true)
-      end
-    end
-
-    def delete_if_exists(filename)
+    # Delete a file if it exists
+    #
+    # It tries three times when permission is denied, and then prints a
+    # warning instead of raising.
+    #
+    # @param [String, nil] filename the file's path
+    #
+    # @return [void]
+    #
+    # @api private
+    def self.delete_if_exists(filename)
       tries = 0
 
       begin
@@ -130,166 +174,86 @@ module Bluepill
       end
     end
 
-    # Returns the stdout, stderr and exit code of the cmd
-    def execute_blocking(cmd, options = {})
-      rd, wr = IO.pipe
-
-      child = Daemonize.safefork
-      if child
-        # parent
-        wr.close
-
-        cmd_status = rd.read
-        rd.close
-
-        ::Process.waitpid(child)
-
-        cmd_status.strip != '' ? Marshal.load(cmd_status) : {exit_code: 0, stdout: '', stderr: ''}
-      else
-        # child
-        rd.close
-
-        # create a child in which we can override the stdin, stdout and stderr
-        cmd_out_read, cmd_out_write = IO.pipe
-        cmd_err_read, cmd_err_write = IO.pipe
-
-        pid = fork do
-          begin
-            # grandchild
-            drop_privileges(options[:uid], options[:gid], options[:supplementary_groups])
-
-            Dir.chdir(ENV['PWD'] = options[:working_dir].to_s) if options[:working_dir]
-            options[:environment].each { |key, value| ENV[key.to_s] = value.to_s } if options[:environment]
-
-            # close unused fds so ancestors wont hang. This line is the only reason we are not
-            # using something like popen3. If this fd is not closed, the .read call on the parent
-            # will never return because "wr" would still be open in the "exec"-ed cmd
-            wr.close
-
-            # we do not care about stdin of cmd
-            STDIN.reopen('/dev/null')
-
-            # point stdout of cmd to somewhere we can read
-            cmd_out_read.close
-            STDOUT.reopen(cmd_out_write)
-            cmd_out_write.close
-
-            # same thing for stderr
-            cmd_err_read.close
-            STDERR.reopen(cmd_err_write)
-            cmd_err_write.close
-
-            # finally, replace grandchild with cmd
-            ::Kernel.exec(*Shellwords.shellwords(cmd))
-          rescue => e
-            (cmd_err_write.closed? ? STDERR : cmd_err_write).puts "Exception in grandchild: #{e}."
-            (cmd_err_write.closed? ? STDERR : cmd_err_write).puts e.backtrace
-            exit 1
-          end
-        end
-
-        # we do not use these ends of the pipes in the child
-        cmd_out_write.close
-        cmd_err_write.close
-
-        # wait for the cmd to finish executing and acknowledge it's death
-        ::Process.waitpid(pid)
-
-        # collect stdout, stderr and exitcode
-        result = {
-          stdout: cmd_out_read.read,
-          stderr: cmd_err_read.read,
-          exit_code: $?.exitstatus,
-        }
-
-        # We're done with these ends of the pipes as well
-        cmd_out_read.close
-        cmd_err_read.close
-
-        # Time to tell the parent about what went down
-        wr.write Marshal.dump(result)
-        wr.close
-
-        ::Process.exit!
-      end
-    end
-
-    def store
+    # Return what this tick has memoized
+    #
+    # @return [Hash{Symbol => Hash{Integer => Array<String>}, String}] the
+    #   ps table, or ps's error, by name
+    #
+    # @api private
+    def self.store
       @store ||= {}
     end
 
-    def reset_data
-      store.clear unless store.empty?
+    # Forget what the last tick memoized, so ps runs again
+    #
+    # @return [void]
+    #
+    # @api private
+    def self.reset_data
+      store.clear
     end
 
-    def ps_axu
-      # TODO: need a mutex here
-      store[:ps_axu] ||= begin
-        # BSD style ps invocation
-        lines = `ps axo pid,ppid,pcpu,rss,etime,command`.split("\n")
-
-        lines.each_with_object({}) do |line, mem|
-          chunks = line.split(/\s+/)
-          chunks.delete_if { |c| c.strip.empty? }
-          pid = chunks[IDX_MAP[:pid]].strip.to_i
-          command = chunks.slice!(IDX_MAP[:command], chunks.length).join ' '
-          chunks[IDX_MAP[:command]] = command
-          mem[pid] = chunks.flatten
+    # Watches run in their own threads, so only let one of them run ps per tick
+    #
+    # @api private
+    PS_MUTEX = Mutex.new
+
+    # Return this tick's ps table, running ps the first time it's asked for
+    #
+    # @return [Hash{Integer => Array<String>}] each process's ps fields, by
+    #   pid, in the order of IDX_MAP
+    #
+    # @raise [PsError] if ps could not be run this tick
+    #
+    # @api private
+    def self.ps_axu
+      ps = PS_MUTEX.synchronize do
+        store[:ps_axu] ||= begin
+          # POSIX options, so this also works with System V ps on Solaris and SmartOS.
+          # Use -A because -e shows the environment on FreeBSD and OpenBSD.
+          lines = `ps -Ao pid,ppid,pcpu,rss,etime,args`.split("\n")
+
+          table = {} #: ps_table
+          lines.each do |line|
+            chunks = line.split
+            pid = chunks.fetch(IDX_MAP.fetch(:pid)).to_i
+            command = chunks.slice!(IDX_MAP.fetch(:command), chunks.length) #: Array[String]
+            chunks[IDX_MAP.fetch(:command)] = command.join ' '
+            table[pid] = chunks
+          end
+          table
+        rescue SystemCallError => e
+          # Forking ps can fail, for example with ENOMEM or EAGAIN under memory pressure.
+          # Cache the message so ps is not run again until the next tick.
+          "Could not run ps: #{e.class}: #{e}"
         end
       end
+      raise PsError, ps if ps.instance_of?(String)
+
+      ps
     end
 
-    def parse_elapsed_time(str)
+    # Convert ps's elapsed time to seconds
+    #
+    # @param [String, nil] str an elapsed time, [[dd-]hh:]mm:ss
+    #
+    # @return [Integer] the seconds, or 0 if str isn't an elapsed time
+    #
+    # @api private
+    def self.parse_elapsed_time(str)
       # [[dd-]hh:]mm:ss
-      if str =~ /(?:(?:(\d+)-)?(\d\d):)?(\d\d):(\d\d)/
-        days = (Regexp.last_match[1] || 0).to_i
-        hours = (Regexp.last_match[2] || 0).to_i
-        mins = Regexp.last_match[3].to_i
-        secs = Regexp.last_match[4].to_i
+      match = /(?:(?:(\d+)-)?(\d\d):)?(\d\d):(\d\d)/.match(str)
+      if match
+        days = match[1].to_i
+        hours = match[2].to_i
+        mins = match[3].to_i
+        secs = match[4].to_i
         ((days * 24 + hours) * 60 + mins) * 60 + secs
       else
         0
       end
     end
-
-    # be sure to call this from a fork otherwise it will modify the attributes
-    # of the bluepill daemon
-    def drop_privileges(uid, gid, supplementary_groups)
-      return unless ::Process::Sys.geteuid.zero?
-      uid_num = Etc.getpwnam(uid).uid if uid
-      gid_num = Etc.getgrnam(gid).gid if gid
-      supplementary_groups ||= []
-      group_nums = supplementary_groups.collect do |group|
-        Etc.getgrnam(group).gid
-      end
-      ::Process.groups = [gid_num] if gid
-      ::Process.groups |= group_nums unless group_nums.empty?
-      ::Process::Sys.setgid(gid_num) if gid
-      ::Process::Sys.setuid(uid_num) if uid
-      ENV['HOME'] = Etc.getpwuid(uid_num).try(:dir) || ENV['HOME'] if uid
-    end
-
-    def can_write_pid_file(pid_file, logger)
-      FileUtils.touch(pid_file)
-      File.unlink(pid_file)
-      true
-    rescue => e
-      logger.warning(format('%s - %s', e.class.name, e.message))
-      e.backtrace.each { |l| logger.warning l }
-      false
-    end
-
-    def redirect_io(io_in, io_out, io_err)
-      $stdin.reopen(io_in) if io_in
-
-      if !io_out.nil? && !io_err.nil? && io_out == io_err
-        $stdout.reopen(io_out, APPEND_MODE)
-        $stderr.reopen($stdout)
-
-      else
-        $stdout.reopen(io_out, APPEND_MODE) if io_out
-        $stderr.reopen(io_err, APPEND_MODE) if io_err
-      end
-    end
   end
 end
+
+require 'bluepill/system/commands'

data/lib/bluepill/trigger.rb

@@ -1,16 +1,76 @@
 module Bluepill
+  # Reacts to a process's state transitions, as one of a process's checks
+  #
+  # A config names a trigger the way it names a check, by its underscored
+  # class name: `checks :flapping` makes a Triggers::Flapping. Subclasses
+  # register themselves by that name, and implement notify.
+  #
+  # @api private
   class Trigger
     @implementations = {}
+
+    # Register a subclass by its underscored name, such as :flapping
+    #
+    # @param [Class] klass the subclass
+    #
+    # @return [void]
+    #
+    # @api private
     def self.inherited(klass)
+      super
       @implementations[klass.name.split('::').last.underscore.to_sym] = klass
     end
 
+    # Return the trigger class a config names
+    #
+    # @param [Symbol] name the trigger's underscored name, such as :flapping
+    #
+    # @return [Class, nil] the Trigger subclass, or nil when no trigger has
+    #   that name (so the name is a check's)
+    #
+    # @api private
     def self.[](name)
       @implementations[name]
     end
 
-    attr_accessor :process, :logger, :mutex, :scheduled_events
+    # Return the process the trigger watches
+    #
+    # @return [Process] the process
+    #
+    # @api private
+    attr_accessor :process
+
+    # Return the logger the trigger writes to
+    #
+    # @return [Logger] the process's logger
+    #
+    # @api private
+    attr_accessor :logger
+
+    # Return the lock that guards the scheduled events
+    #
+    # @return [Thread::Mutex] the lock
+    #
+    # @api private
+    attr_accessor :mutex
 
+    # Return the events the trigger scheduled, each with its thread
+    #
+    # @return [Array<Array(Symbol, Thread)>] each event and the thread that
+    #   dispatches it
+    #
+    # @api private
+    attr_accessor :scheduled_events
+
+    # Create a trigger for a process
+    #
+    # @param [Process] process the process to watch
+    # @param [Hash{Symbol => Object}] options the trigger's options, with
+    #   :logger
+    #
+    # @return [void]
+    #
+    # @api private
     def initialize(process, options = {})
       self.process = process
       self.logger = options[:logger]
@@ -18,40 +78,76 @@ module Bluepill
       self.scheduled_events = []
     end
 
+    # Cancel the events the trigger scheduled, as when the user unmonitors
+    #
+    # @return [void]
+    #
+    # @api private
     def reset!
       cancel_all_events
     end
 
+    # React to a state transition, before it happens
+    #
+    # @param [StateMachine::Transition] _transition the transition
+    #
+    # @return [void]
+    #
+    # @raise [RuntimeError] unless a subclass implements it
+    #
+    # @api private
     def notify(_transition)
-      fail 'Implement in subclass'
+      raise 'Implement in subclass'
     end
 
+    # Send an event to the process, with the trigger's name as the reason
+    #
+    # @param [Symbol] event the event, such as :start
+    #
+    # @return [Boolean] whether the process made the transition
+    #
+    # @api private
     def dispatch!(event)
       process.dispatch!(event, self.class.name.split('::').last)
     end
 
+    # Dispatch an event after a delay, in a thread of its own
+    #
+    # @param [Symbol] event the event, such as :start
+    # @param [Numeric] delay the seconds to wait first
+    #
+    # @return [Thread] the thread that waits and dispatches
+    #
+    # @api private
     def schedule_event(event, delay)
-      # TODO: maybe wrap this in a ScheduledEvent class with methods like cancel
-      thread = Thread.new(self) do |trigger|
-        begin
-          sleep delay.to_f
-          trigger.dispatch!(event)
-          trigger.mutex.synchronize do
-            trigger.scheduled_events.delete_if { |_, t| t == Thread.current }
+      mutex.synchronize do
+        thread = Thread.new do
+          begin
+            sleep delay.to_f
+            dispatch!(event)
+            mutex.synchronize do
+              scheduled_events.delete_if { |_, t| t.equal?(Thread.current) }
+            end
+          rescue StandardError => e
+            logger.err(e)
+            logger.err(e.backtrace.to_a.join("\n"))
           end
-        rescue StandardError => e
-          trigger.logger.err(e)
-          trigger.logger.err(e.backtrace.join("\n"))
         end
+        scheduled_events.push([event, thread])
+        thread
       end
-
-      scheduled_events.push([event, thread])
     end
 
+    # Cancel every scheduled event, killing its thread
+    #
+    # @return [void]
+    #
+    # @api private
     def cancel_all_events
       logger.info 'Canceling all scheduled events'
       mutex.synchronize do
-        scheduled_events.each { |_, thread| thread.kill }
+        scheduled_events.collect(&:last).each(&:kill)
+        scheduled_events.clear
       end
     end
   end