servolux 0.8.1 → 0.9.0

data/lib/servolux/piper.rb

@@ -49,12 +49,9 @@ require 'socket'
 class Servolux::Piper
 
   # :stopdoc:
-  SIZEOF_INT = [42].pack('I').size
+  SIZEOF_INT = [42].pack('I').size  # @private
   # :startdoc:
 
-  # call-seq:
-  #    Piper.daemon( nochdir = false, noclose = false )
-  #
   # Creates a new Piper with the child process configured as a daemon. The
   # +pid+ method of the piper returns the PID of the daemon process.
   #
@@ -64,6 +61,10 @@ class Servolux::Piper
   # _nochdir_ and _noclose_ flags to true. The first will keep the current
   # working directory; the second will keep stdout/stderr/stdin open.
   #
+  # @param [Boolean] nochdir Do not change working directories
+  # @param [Boolean] noclose Do not close stdin, stdout, and stderr
+  # @return [Piper]
+  #
   def self.daemon( nochdir = false, noclose = false )
     piper = self.new(:timeout => 1)
     piper.parent {
@@ -92,25 +93,25 @@ class Servolux::Piper
   # The timeout in seconds to wait for puts / gets commands.
   attr_accessor :timeout
 
-  # call-seq:
-  #    Piper.new( mode = 'r', opts = {} )
-  #
-  # Creates a new Piper instance with the communication pipe configured
-  # using the provided _mode_. The default mode is read-only (from the
-  # parent, and write-only from the child). The supported modes are as
-  # follows:
+  # @overload Piper.new( mode = 'r', opts = {} )
+  #   Creates a new Piper instance with the communication pipe configured
+  #   using the provided _mode_. The default mode is read-only (from the
+  #   parent, and write-only from the child). The supported modes are as
+  #   follows:
   #
-  #    Mode | Parent View | Child View
-  #    -----+-------------+-----------
-  #    r      read-only     write-only
-  #    w      write-only    read-only
-  #    rw     read-write    read-write
+  #      Mode | Parent View | Child View
+  #      -------------------------------
+  #      r      read-only     write-only
+  #      w      write-only    read-only
+  #      rw     read-write    read-write
   #
-  # The communication timeout can be provided as an option. This is the
-  # number of seconds to wait for a +puts+ or +gets+ to succeed. This timeout
-  # will default to +nil+ such that calls through the pipe will block forever
-  # until data is available. You can configure the +puts+ and +gets+ to be
-  # non-blocking by setting the timeout to +0+.
+  #   @param [String] mode The communication mode of the pipe.
+  #   @option opts [Numeric] :timeout (nil)
+  #     The number of seconds to wait for a +puts+ or +gets+ to succeed. If not
+  #     specified, calls through the pipe will block forever until data is
+  #     available. You can configure the +puts+ and +gets+ to be non-blocking
+  #     by setting the timeout to +0+.
+  #   @return [Piper]
   #
   def initialize( *args )
     opts = args.last.is_a?(Hash) ? args.pop : {}
@@ -146,34 +147,51 @@ class Servolux::Piper
   # Close both the communications socket. This only affects the process from
   # which it was called -- the parent or the child.
   #
+  # @return [Piper] self
+  #
   def close
     @socket.close rescue nil
+    self
+  end
+
+  # Returns +true+ if the piper has been closed. Returns +false+ otherwise.
+  #
+  # @return [Boolean]
+  #
+  def closed?
+    @socket.closed?
   end
 
   # Returns +true+ if the communications pipe is readable from the process
   # and there is data waiting to be read.
   #
+  # @return [Boolean]
+  #
   def readable?
     return false if @socket.closed?
-    r,w,e = Kernel.select([@socket], nil, nil, @timeout)
+    r,w,e = Kernel.select([@socket], nil, nil, @timeout) rescue nil
     return !(r.nil? or r.empty?)
   end
 
   # Returns +true+ if the communications pipe is writeable from the process
   # and the write buffer can accept more data.
   #
+  # @return [Boolean]
+  #
   def writeable?
     return false if @socket.closed?
-    r,w,e = Kernel.select(nil, [@socket], nil, @timeout)
+    r,w,e = Kernel.select(nil, [@socket], nil, @timeout) rescue nil
     return !(w.nil? or w.empty?)
   end
 
-  # call-seq:
-  #    child { block }
-  #    child {|piper| block }
-  #
   # Execute the _block_ only in the child process. This method returns
-  # immediately when called from the parent process.
+  # immediately when called from the parent process. The piper instance is
+  # passed to the block if the arity is non-zero.
+  #
+  # @yield [self] Execute the block in the child process
+  # @yieldparam [Piper] self The piper instance (optional)
+  # @return The return value from the block or +nil+ when called from the
+  #   parent.
   #
   def child( &block )
     return unless child?
@@ -188,16 +206,20 @@ class Servolux::Piper
 
   # Returns +true+ if this is the child prcoess and +false+ otherwise.
   #
+  # @return [Boolean]
+  #
   def child?
     @child_pid.nil?
   end
 
-  # call-seq:
-  #    parent { block }
-  #    parent {|piper| block }
-  #
   # Execute the _block_ only in the parent process. This method returns
-  # immediately when called from the child process.
+  # immediately when called from the child process. The piper instance is
+  # passed to the block if the arity is non-zero.
+  #
+  # @yield [self] Execute the block in the parent process
+  # @yieldparam [Piper] self The piper instance (optional)
+  # @return The return value from the block or +nil+ when called from the
+  #   child.
   #
   def parent( &block )
     return unless parent?
@@ -212,6 +234,8 @@ class Servolux::Piper
 
   # Returns +true+ if this is the parent prcoess and +false+ otherwise.
   #
+  # @return [Boolean]
+  #
   def parent?
     !@child_pid.nil?
   end
@@ -219,6 +243,8 @@ class Servolux::Piper
   # Returns the PID of the child process when called from the parent.
   # Returns +nil+ when called from the child.
   #
+  # @return [Integer, nil] The PID of the child process or +nil+
+  #
   def pid
     @child_pid
   end
@@ -255,6 +281,11 @@ class Servolux::Piper
   # is returned. If this number is zero it means that the _obj_ was
   # unsuccessfully communicated (sorry).
   #
+  # @param [Object] obj The data to send to the "other" process. The object
+  #   must be marshallable by Ruby (no Proc objects or lambdas).
+  # @return [Integer, nil] The number of bytes written to the pipe or +nil+ if
+  #   there was an error or the pipe is not writeable.
+  #
   def puts( obj )
     return unless writeable?
 
@@ -270,6 +301,10 @@ class Servolux::Piper
   #
   # This method does nothing when called from the child process.
   #
+  # @param [String, Integer] sig The signal to send to the child process.
+  # @return [Integer, nil] The result of Process#kill or +nil+ if called from
+  #   the child process.
+  #
   def signal( sig )
     return if @child_pid.nil?
     sig = Signal.list.invert[sig] if sig.is_a?(Integer)
@@ -281,6 +316,8 @@ class Servolux::Piper
   #
   # Always returns +nil+ when called from the child process.
   #
+  # @return [Boolean, nil]
+  #
   def alive?
     return if @child_pid.nil?
     Process.kill(0, @child_pid)
@@ -289,5 +326,5 @@ class Servolux::Piper
     false
   end
 
-end  # class Servolux::Piper
+end
 

data/lib/servolux/prefork.rb

@@ -32,7 +32,9 @@
 # Sending a SIGHUP to a child process will cause that child to stop and
 # restart. The child will send a signal to the parent asking to be shutdown.
 # The parent will gracefully halt the child and then start a new child process
-# to replace it.
+# to replace it. If you define a "hup" method in your worker module, it will
+# be executed when SIGHUP is received by the child. Your "hup" method will be
+# the last method executed in the signal handler.
 #
 # This has the advantage of calling your before/after_executing methods again
 # and reloading any code or resources your worker code will use. The SIGHUP
@@ -99,6 +101,30 @@
 # Use the heartbeat with caution -- allow margins for timing issues and
 # processor load spikes.
 #
+# ==== Signals
+# Forked child processes are configured to respond to two signals: SIGHUP and
+# SIGTERM. The SIGHUP signal when sent to a child process is used to restart
+# just that one child. The SIGTERM signal when sent to a child process is used
+# to forcibly kill the child; it will not be restarted. The parent process
+# uses SIGTERM to halt all the children when it is stopping.
+#
+# SIGHUP
+# Child processes are restarted by sending a SIGHUP signal to the child. This
+# will shutdown the child worker and then start up a new one to replace it.
+# For the child to shutdown gracefully, it needs to return from the "execute"
+# method when it receives the signal. Define a "hup" method that will wake the
+# execute thread from any pending operations -- listening on a socket, reading
+# a file, polling a queue, etc. When the execute method returns, the child
+# will exit.
+#
+# SIGTERM
+# Child processes are stopped by the prefork parent by sending a SIGTERM
+# signal to the child. For the child to shutdown gracefully, it needs to
+# return from the "execute" method when it receives the signal. Define a
+# "term" method that will wake the execute thread from any pending operations
+# -- listening on a socket, reading a file, polling a queue, etc. When the
+# execute method returns, the child will exit.
+#
 class Servolux::Prefork
 
   Timeout = Class.new(::Servolux::Error)
@@ -106,13 +132,14 @@ class Servolux::Prefork
   UnknownResponse = Class.new(::Servolux::Error)
 
   # :stopdoc:
-  START = "\000START".freeze
-  HALT = "\000HALT".freeze
-  ERROR = "\000SHIT".freeze
-  HEARTBEAT = "\000<3".freeze
+  START = "\000START".freeze     # @private
+  HALT = "\000HALT".freeze       # @private
+  ERROR = "\000SHIT".freeze      # @private
+  HEARTBEAT = "\000<3".freeze    # @private
   # :startdoc:
 
   attr_accessor :timeout    # Communication timeout in seconds.
+  attr_reader :harvest      # List of child PIDs that need to be reaped
 
   # call-seq:
   #    Prefork.new { block }
@@ -137,6 +164,7 @@ class Servolux::Prefork
     @module = opts[:module]
     @module = Module.new { define_method :execute, &block } if block
     @workers = []
+    @harvest = Queue.new
 
     raise ArgumentError, 'No code was given to execute by the workers.' unless @module
   end
@@ -144,6 +172,9 @@ class Servolux::Prefork
   # Start up the given _number_ of workers. Each worker will create a child
   # process and run the user supplied code in that child process.
   #
+  # @param [Integer] number The number of workers to prefork
+  # @return [Prefork] self
+  #
   def start( number )
     @workers.clear
 
@@ -151,7 +182,7 @@ class Servolux::Prefork
       @workers << Worker.new(self)
       @workers.last.extend @module
     }
-    @workers.each { |worker| worker.start }
+    @workers.each { |worker| worker.start; pause }
     self
   end
 
@@ -161,10 +192,38 @@ class Servolux::Prefork
   # +errors+ methods will still function correctly after stopping the workers.
   #
   def stop
-    @workers.each { |worker| worker.stop }
-    @workers.each { |worker| worker.wait }
+    @workers.each { |worker| worker.stop; pause }
+    reap
+    self
+  end
+
+  # This method should be called periodically in order to clear the return
+  # status from child processes that have either died or been restarted (via a
+  # HUP signal). This will remove zombie children from the process table.
+  #
+  # @return [Prefork] self
+  #
+  def reap
+    while !@harvest.empty?
+      pid = @harvest.pop
+      Process.wait pid rescue nil
+    end
+    self
+  end
+
+  # Send this given _signal_ to all child process. The default signal is
+  # 'TERM'. The method waits for a short period of time after the signal is
+  # sent to each child; this is done to alleviate a flood of signals being
+  # sent simultaneously and overwhemlming the CPU.
+  #
+  # @param [String, Integer] signal The signal to send to child processes.
+  # @return [Prefork] self
+  #
+  def signal( signal = 'TERM' )
+    @workers.each { |worker| worker.signal(signal); pause }
     self
   end
+  alias :kill :signal
 
   # call-seq:
   #    each_worker { |worker| block }
@@ -188,6 +247,18 @@ class Servolux::Prefork
     self
   end
 
+
+  private
+
+  # Pause script execution for a random time interval between 0.1 and 0.4
+  # seconds. This method is used to slow down the starting and stopping of
+  # child processes in order to avoiad the "thundering herd" problem.
+  # http://en.wikipedia.org/wiki/Thundering_herd_problem
+  #
+  def pause
+    sleep(0.1 + 0.3*rand)
+  end
+
   # The worker encapsulates the forking of the child process and communication
   # between the parent and the child. Each worker instance is extended with
   # the block or module supplied to the pre-forking pool that created the
@@ -195,13 +266,15 @@ class Servolux::Prefork
   #
   class Worker
 
-    attr_reader :prefork
     attr_reader :error
 
     # Create a new worker that belongs to the _prefork_ pool.
     #
+    # @param [Prefork] prefork The prefork pool that created this worker.
+    #
     def initialize( prefork )
-      @prefork = prefork
+      @timeout = prefork.timeout
+      @harvest = prefork.harvest
       @thread = nil
       @piper = nil
       @error = nil
@@ -210,11 +283,12 @@ class Servolux::Prefork
     # Start this worker. A new process will be forked, and the code supplied
     # by the user to the prefork pool will be executed in the child process.
     #
+    # @return [Worker] self
+    #
     def start
       @error = nil
-      @piper = ::Servolux::Piper.new('rw', :timeout => @prefork.timeout)
-      parent if @piper.parent?
-      child if @piper.child?
+      @piper = ::Servolux::Piper.new('rw', :timeout => @timeout)
+      @piper.parent? ? parent : child
       self
     end
 
@@ -223,13 +297,16 @@ class Servolux::Prefork
     # without waiting for the child process to exit. Use the +wait+ method
     # after calling +stop+ if your code needs to know when the child exits.
     #
+    # @return [Worker, nil] self
+    #
     def stop
       return if @thread.nil? or @piper.nil? or @piper.child?
 
       @thread[:stop] = true
       @thread.wakeup if @thread.status
-      Thread.pass until !@thread.status
-      kill 'HUP'
+      close_parent
+      signal 'TERM'
+      @thread.join(0.5) rescue nil
       @thread = nil
       self
     end
@@ -246,18 +323,25 @@ class Servolux::Prefork
     # Send this given _signal_ to the child process. The default signal is
     # 'TERM'. This method will return immediately.
     #
-    def kill( signal = 'TERM' )
+    # @param [String, Integer] signal The signal to send to the child process.
+    # @return [Integer, nil] The result of Process#kill or +nil+ if called from
+    #   the child process.
+    #
+    def signal( signal = 'TERM' )
       return if @piper.nil?
       @piper.signal signal
     rescue Errno::ESRCH, Errno::ENOENT
       return nil
     end
+    alias :kill :signal
 
     # Returns +true+ if the child process is alive. Returns +nil+ if the child
     # process has not been started.
     #
     # Always returns +nil+ when called from the child process.
     #
+    # @return [Boolean, nil]
+    #
     def alive?
       return if @piper.nil?
       @piper.alive?
@@ -266,62 +350,93 @@ class Servolux::Prefork
 
     private
 
+    def close_parent
+      @piper.timeout = 0.5
+      @piper.puts HALT rescue nil
+      @piper.close
+    end
+
     # This code should only be executed in the parent process.
     #
     def parent
       @thread = Thread.new {
-        response = nil
         begin
           @piper.puts START
           Thread.current[:stop] = false
-          loop {
-            break if Thread.current[:stop]
-            @piper.puts HEARTBEAT
-            response = @piper.gets(ERROR)
-            break if Thread.current[:stop]
-
-            case response
-            when HEARTBEAT; next
-            when START; break
-            when ERROR
-              raise Timeout,
-                    "Child did not respond in a timely fashion. Timeout is set to #{@prefork.timeout.inspect} seconds."
-            when Exception
-              raise response
-            else
-              raise UnknownResponse,
-                    "Child returned unknown response: #{response.inspect}"
-            end
-          }
-        rescue Exception => err
-          @error = err
+          response = parent_loop
+        # TODO: put a logger here to catch and log all exceptions
         ensure
-          @piper.timeout = 0.1
-          @piper.puts HALT rescue nil
-          @piper.close
-          self.start if START == response and !Thread.current[:stop]
+          @harvest << @piper.pid
+          close_parent
+          start if START == response and !Thread.current[:stop]
         end
       }
       Thread.pass until @thread[:stop] == false
     end
 
+    def parent_loop
+      response = nil
+      loop {
+        break if Thread.current[:stop]
+        @piper.puts HEARTBEAT
+        response = @piper.gets(ERROR)
+        break if Thread.current[:stop]
+
+        case response
+        when HEARTBEAT; next
+        when START; break
+        when ERROR
+          raise Timeout,
+                "Child did not respond in a timely fashion. Timeout is set to #{@timeout.inspect} seconds."
+        when Exception
+          @error = response
+          break
+        else
+          raise UnknownResponse,
+                "Child returned unknown response: #{response.inspect}"
+        end
+      }
+      return response
+    end
+
     # This code should only be executed in the child process. It wraps the
     # user supplied "execute" code in a loop, and takes care of handling
     # signals and communication with the parent.
     #
     def child
-      @thread = Thread.current
+      @harvest = nil
 
       # if we get a HUP signal, then tell the parent process to stop this
       # child process and start a new one to replace it
       Signal.trap('HUP') {
+        @piper.timeout = 0.5
         @piper.puts START rescue nil
-        @thread.wakeup
+        hup if self.respond_to? :hup
       }
 
-      before_executing if self.respond_to? :before_executing
-      :wait until @piper.gets == START
+      Signal.trap('TERM') {
+        @piper.close
+        term if self.respond_to? :term
+      }
+
+      @thread = Thread.new {
+        begin
+          :wait until @piper.gets == START
+          before_executing rescue nil if self.respond_to? :before_executing
+          child_loop
+        rescue Exception => err
+          @piper.puts err rescue nil
+        ensure
+          after_executing rescue nil if self.respond_to? :after_executing
+          @piper.close
+        end
+      }
+      @thread.join
+    ensure
+      exit!
+    end
 
+    def child_loop
       loop {
         signal = @piper.gets(ERROR)
         case signal
@@ -332,20 +447,13 @@ class Servolux::Prefork
           break
         when ERROR
           raise Timeout,
-                "Parent did not respond in a timely fashion. Timeout is set to #{@prefork.timeout.inspect} seconds."
+                "Parent did not respond in a timely fashion. Timeout is set to #{@timeout.inspect} seconds."
         else
           raise UnknownSignal,
                 "Child received unknown signal: #{signal.inspect}"
         end
       }
-      after_executing if self.respond_to? :after_executing
-    rescue Exception => err
-      @piper.puts err rescue nil
-    ensure
-      @piper.close
-      exit!
     end
-  end  # class Worker
-
-end  # class Servolux::Prefork
+  end
+end
 

data/lib/servolux/server.rb

@@ -181,6 +181,8 @@ class Servolux::Server
   # flag is used to ensure that the server has shutdown completely when
   # shutdown by a signal.
   #
+  # @return [Server] self
+  #
   def startup( wait = false )
     return self if running?
     @mutex.synchronize {
@@ -189,8 +191,8 @@ class Servolux::Server
 
     begin
       create_pid_file
-      trap_signals
       start
+      trap_signals
       join
       wait_for_shutdown if wait
     ensure
@@ -211,10 +213,11 @@ class Servolux::Server
   # the server is completely stopped, use the +wait_for_shutdown+ method to
   # be notified when the this +shutdown+ method is finished executing.
   #
+  # @return [Server] self
+  #
   def shutdown
     return self unless running?
     stop
-
     @mutex.synchronize {
       @shutdown.signal
       @shutdown = nil
@@ -253,6 +256,9 @@ class Servolux::Server
 
   def delete_pid_file
     if test(?f, pid_file)
+      pid = Integer(File.read(pid_file).strip)
+      return unless pid == Process.pid
+
       logger.debug "Server #{name.inspect} removing pid file #{pid_file.inspect}"
       File.delete(pid_file)
     end
@@ -265,6 +271,5 @@ class Servolux::Server
     end
   end
 
-end  # class Servolux::Server
+end
 
-# EOF

data/lib/servolux/threaded.rb

@@ -1,3 +1,4 @@
+
 # == Synopsis
 # The Threaded module is used to peform some activity at a specified
 # interval.
@@ -74,7 +75,6 @@ module Servolux::Threaded
 
     before_stopping if self.respond_to?(:before_stopping)
     @_activity_thread.stop
-    after_stopping if self.respond_to?(:after_stopping)
     self
   end
 
@@ -198,23 +198,23 @@ module Servolux::Threaded
   end
 
   # :stopdoc:
-
   def _activity_thread
     @_activity_thread ||= ::Servolux::Threaded::ThreadContainer.new(60, 0, nil, false);
-  end
+  end  # @private
 
+  # @private
   ThreadContainer = Struct.new( :interval, :iterations, :maximum_iterations, :continue_on_error, :thread, :running ) {
     def start( threaded )
       self.running = true
       self.iterations = 0
       self.thread = Thread.new { run threaded }
       Thread.pass
-    end
+    end  # @private
 
     def stop
       self.running = false
       thread.wakeup
-    end
+    end  # @private
 
     def run( threaded )
       loop {
@@ -235,23 +235,25 @@ module Servolux::Threaded
         end
       }
     ensure
+      if threaded.respond_to?(:after_stopping) and !self.running
+        threaded.after_stopping rescue nil
+      end
       self.running = false
-    end
+    end  # @private
 
     def join( limit = nil )
       return if thread.nil?
       limit ? thread.join(limit) : thread.join
-    end
+    end  # @private
 
     def finished_iterations?
       return true if maximum_iterations and (iterations >= maximum_iterations)
       return false
-    end
+    end  # @private
 
     alias :running? :running
   }
   # :startdoc:
 
-end  # module Servolux::Threaded
+end
 
-# EOF