angael 0.0.7 → 0.0.8

data/lib/angael/manager.rb

@@ -1,15 +1,17 @@
+require 'angael/process_helper'
 module Angael
   # A Manager has a number of of worker objects. Starting the Manager simply
   # calls #start! on each worker, then it goes into an infinite loop, waiting
   # for SIGINT or SIGTERM. When either of those is received, the manager will
-  # call #stop! on each Worker.
+  # call #stop_with_wait on each Worker.
   class Manager
+    include ProcessHelper
     attr_reader :workers
 
     # Creates a new manager.
     #
     # @worker_class [Class] The class to use for the workers. Must respond
-    #   to #new, #start!, and #stop!
+    #   to #new, #start!, and #stop_with_wait
     # @worker_count [Integer] The number of workers to manager. Default is 1.
     # @worker_args [Array] An array of arguments that will be passed to
     #   worker_class.new. The arguments will be splatted
@@ -33,6 +35,8 @@ module Angael
       @workers = []
       worker_count.times { workers << worker_class.new(*worker_args) }
       @restart_after = opts[:restart_after]
+      # TODO: Add a spec for this
+      raise ArgumentError, ':restart_after must be either an Integer greater than zero or nil' if @restart_after && (@restart_after.to_i != @restart_after || @restart_after == 0)
       @logger = opts[:logger]
       if @logger
         @log_level = opts[:log_level] || begin
@@ -44,13 +48,13 @@ module Angael
 
 
     # Starts workers by calling Worker#start! Loops forever waiting for SIGINT
-    # or SIGTERM, at which time it calls Worker#stop! on each worker.
+    # or SIGTERM, at which time it calls Worker#stop_with_wait on each worker.
     def start!
       workers.each { |w| w.start! }
 
       trap("CHLD") do
         workers.each do |w|
-          result = wait(w.pid)
+          result = exit_status(w.pid)
 #print w.pid.to_s
 #print "\t"
 #p result
@@ -62,23 +66,40 @@ module Angael
         end
       end
       trap("INT") do
-        stop!
+        log("SIGINT Received")
+        @interrupted = true
       end
       trap("TERM") do
-        stop!
+        log("SIGTERM Received")
+        @interrupted = true
       end
 
       if @restart_after
+        restart_after_counter = @restart_after
         loop do
-          # Periodically restart workers, 1 at a time.
-          sleep @restart_after
-          w = next_worker_to_restart
-          w.stop!
-          w.start!
+          stop! if @interrupted
+
+          # This ensures we are checking @interrupted every 1 second, but we
+          # can set @restart_after to something greater than zero.
+          if restart_after_counter > 0
+            sleep 1
+            restart_after_counter -= 1
+          else
+            # Periodically restart workers, 1 at a time.
+            log("Sleeping for #@restart_after seconds")
+            w = next_worker_to_restart
+            log("Time to restart a worker: Calling #stop_with_wait for worker #{w.inspect}")
+            w.stop_with_wait
+            log("Worker has been stopped: #{w.inspect}")
+            w.start!
+            log("Worker has been restarted: #{w.inspect}")
+            w = nil
+            restart_after_counter = @restart_after
+          end
         end
       else
         loop do
-          # Don't restart workers if nothing is wrong.
+          stop! if @interrupted
           sleep 1
         end
       end
@@ -91,11 +112,18 @@ module Angael
     #########
 
     def stop!
-      log("SIGINT Received")
+      log("Attempting to gracefully stopping worker manager")
+      # Tell each worker to stop, without waiting to see if it worked.
+      workers.each { |w|
+        log("Calling #stop_without_wait for worker #{w.inspect}")
+        w.stop_without_wait
+        log("Finished call to #stop_without_wait for worker #{w.inspect}")
+      }
+      # Wait for each worker to stop, one at a time.
       workers.each { |w|
-        log("Calling #stop! for worker #{w.inspect}")
-        w.stop!
-        log("Finished call to #stop! for worker #{w.inspect}")
+        log("Calling #stop_with_wait for worker #{w.inspect}")
+        w.stop_with_wait
+        log("Finished call to #stop_with_wait for worker #{w.inspect}")
       }
       exit 0
     end
@@ -104,20 +132,6 @@ module Angael
       @logger.add(@log_level, "#{Time.now.utc} - #{self.class} (pid #{$$}): #{msg}") if @logger
     end
 
-    # Returns immediately. If the process is still running, it returns nil.
-    # If the process is a zombie, it returns an array with the pid as the
-    # first element and a Process::Status object as the 2nd element, i.e.
-    # it returns the same thing as Process.wait2. If the process does not
-    # exist (i.e. it is completely gone) then it returns an array with the
-    # pid as the first element and nil as the 2nd element (because there
-    # is no Process::Status object to return).
-    def wait(pid)
-      begin
-        Process.wait2(pid, Process::WNOHANG)
-      rescue Errno::ECHILD
-        [pid, nil] # It did exit, but we don't know the exit status.
-      end
-    end
 
     def next_worker_to_restart
       @worker_count ||= workers.size

data/lib/angael/process_helper.rb

@@ -0,0 +1,37 @@
+module Angael
+  module ProcessHelper
+    # Returns immediately. If the process is still running, it returns nil.
+    # If the process is a zombie, it returns an array with the pid as the
+    # first element and a Process::Status object as the 2nd element, i.e.
+    # it returns the same thing as Process.wait2. If the process does not
+    # exist (i.e. it is completely gone) or it is not accessible to the
+    # current process (i.e. an Errno::EPERM error is raised), then this method
+    # returns an array with the pid as the first element and nil as the 2nd
+    # element (because there is no Process::Status object to return).
+    # If the argument is nil, then it returns [nil, nil] because there can't be
+    # a process running with the pid nil.
+    def exit_status(pid)
+      return [nil, nil] if pid.nil?
+
+      # Sometimes wait2 returns nil even when the process has exited. This
+      # raises an Errno::ESRCH error in that case.
+      raise "Unexpected return value from Process.kill(0, pid)" unless Process.kill(0, pid) == 1
+
+      Process.wait2(pid, Process::WNOHANG)
+    rescue Errno::ECHILD, Errno::ESRCH, Errno::EPERM
+      # There is no longer any record of this PID.
+      # It did exit, but we don't know the exit status.
+      [pid, nil]
+    end
+
+
+    # Will return nil instead of raising Errno::ESRCH when the process does
+    # not exists.
+    # TODO: Add explicit tests for this.
+    def send_signal(signal, pid)
+      Process.kill(signal, pid)
+    rescue Errno::ESRCH
+      nil
+    end
+  end
+end

data/lib/angael/version.rb

@@ -1,3 +1,3 @@
 module Angael
-  VERSION = "0.0.7"
+  VERSION = "0.0.8"
 end

data/lib/angael/worker.rb

@@ -1,4 +1,4 @@
-require 'timeout'
+require 'angael/process_helper'
 module Angael
   # Usage
   #     include Angael::Worker
@@ -17,6 +17,7 @@ module Angael
   #                  it is sent SIGINT. If you don't define this method, it waits
   #                  60 seconds.
   module Worker
+    include ProcessHelper
     class ChildProcessNotStoppedError < StandardError; end
 
     attr_reader :pid
@@ -64,9 +65,13 @@ module Angael
       start!
     end
 
-    def stop!
+    # Returns true if SIGINT was sent to the child process, even if the child
+    # process does not exists.
+    # Returns false if started? is true.
+    # Sets stopping? to false.
+    def stop_without_wait
       unless started?
-        __log("Called stop for worker with PID #{pid} but it is not started")
+        __log("Tried to stop worker with PID #{pid} but it is not started")
         return false
       end
 
@@ -74,28 +79,38 @@ module Angael
       # stopped the child process.
       @stopping = true
 
-      begin
-        __log("Sending SIGINT to child process with pid #{pid}.")
-        Timeout::timeout(timeout) do
-          Process.kill('INT', pid)
-          wait_for_child
-        end
-      rescue Timeout::Error
-        begin
-          __log("Child process with pid #{pid} did not stop within #{timeout} seconds of SIGINT. Sending SIGKILL to child process.")
-          # This only leaves 1 second for the SIGKILL to take effect. I don't
-          # know if that is enough time (or maybe too much time).
-          Timeout::timeout(1) do
-            Process.kill('KILL', pid)
-            wait_for_child
-          end
-        rescue Timeout::Error
-          if pid_running?
-            msg = "Unable to kill child process with PID: #{pid}"
-            __log(msg)
-            raise ChildProcessNotStoppedError, msg
-          end
-        end
+      __log("Sending SIGINT to child process with pid #{pid}.")
+      send_signal('INT', pid)
+      true
+    end
+
+    # Keeps sending SIGINT until the child process exits. If #timeout seconds
+    # pass, then it sends 1 SIGKILL. If that also fails, it raises ChildProcessNotStoppedError.
+    def stop_with_wait
+      return false unless stop_without_wait
+
+      __log("Waiting for child process with pid #{pid} to stop.")
+
+      counter = 0
+
+      while pid_running? && counter < timeout
+        sleep 1
+        counter += 1
+        __log("Sending SIGINT to child process with pid #{pid}. Attempt Count: #{counter}.")
+        send_signal('INT', pid)
+      end
+
+      if pid_running?
+        __log("Child process with pid #{pid} did not stop within #{timeout} seconds of SIGINT. Sending SIGKILL to child process.")
+        send_signal('KILL', pid)
+        sleep 1
+      end
+
+      if pid_running?
+        # SIGKILL didn't work.
+        msg = "Unable to kill child process with PID: #{pid}"
+        __log(msg)
+        raise ChildProcessNotStoppedError, msg
       end
     end
 
@@ -132,34 +147,10 @@ module Angael
     end
 
 
-    # Note: if the pid is running, but this process doesn't have permissions to
-    #       access it, then this will return false.
     def pid_running?
-      begin
-        Process.kill(0, pid) == 1
-      rescue Errno::ESRCH, Errno::EPERM
-        false
-      end
+      !exit_status(pid)
     end
 
-    # Will just return if the child process is not running.
-    def wait_for_child(opts={})
-      begin
-        __log("Waiting for child with pid #{pid}.")
-        if opts[:dont_block]
-          # When this is called as the result of a SIGCHLD
-          # we need to pass in Process::WNOHANG as the 2nd argument, otherwise when
-          # there are multiple workers, some workers will trap SIGCHLD when other
-          # workers' child processes die. Without this argument, those workers will
-          # hang forever, which also hangs the worker manager.
-          Process.wait(pid, Process::WNOHANG)
-        else
-          Process.wait(pid)
-        end
-      rescue Errno::ECHILD
-        # The child process has already been reaped.
-      end
-    end
 
     # This is the standard/default way of doing it. Overwrite this if you want
     # to wrap it in an exception handler, for example.

data/spec/lib/angael/manager_spec.rb

@@ -40,40 +40,33 @@ describe Angael::Manager do
     end
 
     context "when :restart_after is set to 0.5" do
-      subject { Angael::Manager.new(Angael::TestSupport::SampleWorker, 3, [], :restart_after => 0.5) }
+      after do
+        clean_up_pid(@pid)
+      end
+
+      subject { Angael::Manager.new(Angael::TestSupport::SampleWorker, 3, [], :restart_after => 1) }
+
       it "should restart workers 1 at a time, at 1 second intervals" do
         subject.workers.each do |w|
-          w.stub(:start!) # We don't actually need the workers to fork.
+          # This isn't used for restarts.
+          w.stub(:stop_without_wait)
         end
 
-        subject.workers[0].should_receive(:stop!).exactly(1).times
-        subject.workers[1].should_receive(:stop!).exactly(2).times # This is the worker that got restarted.
-        subject.workers[2].should_receive(:stop!).exactly(1).times
-
-        subject.workers[0].should_receive(:start!).exactly(1).times
-        subject.workers[1].should_receive(:start!).exactly(2).times # This is the worker that got restarted.
-        subject.workers[2].should_receive(:start!).exactly(1).times
-
-
-        # As an alternative to should_receive_in_child_process, we
-        # fork a process which will send SIGINT to this current process.
-        # Then we start the Manager in this process and wait for it to
-        # get the SIGINT. Finally we rescue SystemExit so that this
-        # process doesn't exit with the Manager stops.
-        # TODO: Be consistent in my use of this technique vs. should_receive_in_child_process.
-        current_pid = $$
-        pid = Process.fork do
-          sleep 0.6 # Add a 0.1 second buffer to the value of :restart_after to give the process a chance to start.
-          Process.kill('INT', current_pid)
-          exit 0
-        end
-        begin
+        subject.workers[0].should_receive_in_child_process(:stop_with_wait).exactly(1).times
+        subject.workers[1].should_receive_in_child_process(:stop_with_wait).exactly(2).times # This is the worker that got restarted.
+        subject.workers[2].should_receive_in_child_process(:stop_with_wait).exactly(1).times
+
+        subject.workers[0].should_receive_in_child_process(:start!).exactly(1).times
+        subject.workers[1].should_receive_in_child_process(:start!).exactly(2).times # This is the worker that got restarted.
+        subject.workers[2].should_receive_in_child_process(:start!).exactly(1).times
+
+        @pid = Process.fork do
           subject.start!
-        rescue SystemExit
-          nil
         end
 
-        clean_up_pid(pid)
+        sleep 1.1 # Add a 0.1 second buffer to the value of :restart_after to give the process a chance to start.
+        Process.kill('INT', @pid)
+
       end
     end
 
@@ -94,7 +87,7 @@ describe Angael::Manager do
           end
 
           sleep 0.1 # Give the process a chance to start.
-          # This sends stop! to all the workers.
+          # This sends stop_with_wait to all the workers.
           Process.kill('INT', @pid)
           sleep 0.1 # Give the TempFile a chance to flush
         end
@@ -157,11 +150,29 @@ describe Angael::Manager do
 
     %w(INT TERM).each do |sig|
       context "when it receives a SIG#{sig}" do
-        it "should call #stop! on each Worker" do
+        it "should call #stop_without_wait on each Worker" do
+          subject.workers.each do |w|
+            w.stub(:start!) # We don't care about the sub-process, so don't start it.
+
+            w.should_receive_in_child_process(:stop_without_wait).at_least(1).times
+          end
+
+          pid = Process.fork do
+            subject.start!
+          end
+          sleep 0.1 # Give the process a chance to start.
+          Process.kill(sig, pid)
+          sleep 0.1 # Give the TempFile a chance to flush
+
+          # Clean up
+          Process.wait(pid)
+        end
+
+        it "should call #stop_with_wait on each Worker" do
           subject.workers.each do |w|
             w.stub(:start!) # We don't care about the sub-process, so don't start it.
 
-            w.should_receive_in_child_process(:stop!)
+            w.should_receive_in_child_process(:stop_with_wait)
           end
 
           pid = Process.fork do

data/spec/lib/angael/process_helper_spec.rb

@@ -0,0 +1,72 @@
+require 'spec_helper'
+
+describe Angael::ProcessHelper do
+  include Angael::ProcessHelper
+
+  describe "exit_status" do
+    context "when passed the argument nil" do
+      it "should return an array of [nil, nil]" do
+        exit_status(nil).should == [nil, nil]
+      end
+    end
+
+    context "when Process.kill raises Errno::ESRCH" do
+      before { Process.stub(:kill) { raise Errno::ESRCH } }
+      it "should return an array of [pid, nil]" do
+        exit_status(123456).should == [123456, nil]
+      end
+    end
+
+    context "when Process.kill raises Errno::EPERM" do
+      before { Process.stub(:kill) { raise Errno::EPERM } }
+      it "should return an array of [pid, nil]" do
+        exit_status(123456).should == [123456, nil]
+      end
+    end
+
+    context "when Process.kill return 1" do
+      before { Process.stub(:kill).and_return(1) }
+
+      context "when Process.wait2 raises Errno::ECHILD" do
+        before { Process.stub(:wait2) { raise Errno::ECHILD } }
+        it "should return an array of [pid, nil]" do
+          exit_status(123456).should == [123456, nil]
+        end
+      end
+
+      context "when Process.wait2 returns nil" do
+        before { Process.stub(:wait2).and_return(nil) }
+        it "should return nil" do
+          exit_status(123456).should be_nil
+        end
+      end
+
+      context "when Process.wait2 returns an array of [pid, exitstatus]" do
+        before do
+          @exitstatus = mock(Process::Status)
+          Process.stub(:wait2).and_return([123456, @exitstatus])
+        end
+        it "should return an array of [pid, exitstatus]" do
+          exit_status(123456).should == [123456, @exitstatus]
+        end
+      end
+    end
+  end
+
+
+  describe "send_signal" do
+    it "should call kill" do
+      pid = 123456
+      sig = 'INT'
+      Process.should_receive(:kill).with(sig, pid)
+      send_signal(sig, pid)
+    end
+
+    context "when Process.kill raises Errno::ESRCH" do
+      it "should return nil" do
+        Process.stub(:kill) { raise Errno::ESRCH }
+        send_signal('INT', 123456).should be_nil
+      end
+    end
+  end
+end

data/spec/lib/angael/worker_spec.rb

@@ -7,7 +7,7 @@ describe Angael::Worker do
 
   describe "#start!" do
     before { subject.stub(:work => nil) }
-    after { subject.stop! if subject.started? }
+    after { subject.stop_with_wait if subject.started? }
 
     it "should set #pid" do
       subject.pid.should be_nil
@@ -85,24 +85,68 @@ describe Angael::Worker do
 
 
 
-  describe "#stop!" do
+  describe "#stop_without_wait" do
     before { subject.stub(:work => nil) }
-    after { subject.stop! }
+    after { subject.stop_with_wait }
 
     context "when stopped" do
       it "should return false" do
         subject.should_not be_started
-        subject.stop!.should be_false
+        subject.stop_without_wait.should be_false
       end
 
       it "should not be stopping" do
-        subject.stop!
+        subject.stop_without_wait
         subject.should_not be_stopping
       end
 
       it "should not send a SIGINT to the child process" do
         should_not_receive_and_run(Process, :kill, 'INT', subject.pid)
-        subject.stop!
+        subject.stop_without_wait
+      end
+
+      it "should return false" do
+        subject.stop_without_wait.should be_false
+      end
+    end
+
+    context "when started" do
+      before { subject.start! }
+      it "should send a SIGINT to the child process" do
+        should_receive_and_run(Process, :kill, 'INT', subject.pid)
+        subject.stop_without_wait
+      end
+
+      it "should be stopping" do
+        subject.stop_without_wait
+        subject.should be_stopping
+      end
+
+      it "should return true" do
+        subject.stop_without_wait.should be_true
+      end
+    end
+  end
+
+
+  describe "#stop_with_wait" do
+    before { subject.stub(:work => nil) }
+    after { subject.stop_with_wait }
+
+    context "when stopped" do
+      it "should return false" do
+        subject.should_not be_started
+        subject.stop_with_wait.should be_false
+      end
+
+      it "should not be stopping" do
+        subject.stop_with_wait
+        subject.should_not be_stopping
+      end
+
+      it "should not send a SIGINT to the child process" do
+        should_not_receive_and_run(Process, :kill, 'INT', subject.pid)
+        subject.stop_with_wait
       end
     end
 
@@ -110,12 +154,12 @@ describe Angael::Worker do
       it "should send a SIGINT to the child process" do
         subject.start!
         should_receive_and_run(Process, :kill, 'INT', subject.pid)
-        subject.stop!
+        subject.stop_with_wait
       end
 
       it "should be stopping" do
         subject.start!
-        subject.stop!
+        subject.stop_with_wait
         subject.should be_stopping
       end
 
@@ -130,23 +174,23 @@ describe Angael::Worker do
           subject.start!
         end
         it "should be stopped" do
-          subject.stop!
+          subject.stop_with_wait
           subject.should be_stopped
         end
 
         it "should not have a child process with the pid #pid" do
-          subject.stop!
+          subject.stop_with_wait
           pid_running?(subject.pid).should be_false
         end
 
         it "should not send a SIGKILL to the child process" do
           should_not_receive_and_run(Process, :kill, 'KILL', subject.pid)
-          subject.stop!
+          subject.stop_with_wait
         end
 
         it "should have the (now dead) child process' PID as #pid" do
           pid = subject.pid
-          subject.stop!
+          subject.stop_with_wait
           subject.pid.should == pid
         end
       end
@@ -164,18 +208,18 @@ describe Angael::Worker do
         end
 
         it "should be stopped" do
-          subject.stop!
+          subject.stop_with_wait
           subject.should be_stopped
         end
 
         it "should not have a child process with the pid #pid" do
-          subject.stop!
+          subject.stop_with_wait
           pid_running?(subject.pid).should be_false
         end
 
         it "should send a SIGKILL to the child process" do
           should_receive_and_run(Process, :kill, 'KILL', subject.pid)
-          subject.stop!
+          subject.stop_with_wait
         end
 
         context "child process does not die after receiving SIGKILL" do
@@ -204,7 +248,7 @@ describe Angael::Worker do
           it "should raise an error with the child process' pid in the message" do
             pid_running?(subject.pid).should be_true
             lambda do
-              subject.stop!
+              subject.stop_with_wait
             end.should raise_error(Angael::Worker::ChildProcessNotStoppedError, /#{subject.pid}/)
 
             # Confirm the PID is still running

metadata

@@ -2,7 +2,7 @@
 name: angael
 version: !ruby/object:Gem::Version 
   prerelease: 
-  version: 0.0.7
+  version: 0.0.8
 platform: ruby
 authors: 
 - Paul Cortens
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-06-14 00:00:00 -07:00
+date: 2011-06-20 00:00:00 -07:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -51,9 +51,11 @@ files:
 - angael.gemspec
 - lib/angael.rb
 - lib/angael/manager.rb
+- lib/angael/process_helper.rb
 - lib/angael/version.rb
 - lib/angael/worker.rb
 - spec/lib/angael/manager_spec.rb
+- spec/lib/angael/process_helper_spec.rb
 - spec/lib/angael/worker_spec.rb
 - spec/spec_helper.rb
 - spec/support/io_helpers.rb