resque 1.21.0 → 1.22.0

data/HISTORY.md

@@ -1,3 +1,9 @@
+## 1.22.0 (2012-08-21)
+
+* unregister signal handlers in child process when ENV["TERM_CHILD"] is set (@dylanasmith, #621)
+* new signal handling for TERM. See http://hone.heroku.com/resque/2012/08/21/resque-signals.html. (@wuputah, @yaaule, #638)
+* supports calling perform hooks when using Resque.inline (@jonhyman, #506)
+
 ## 1.21.0 (2012-07-02)
 
 * Add a flag to make sure failure hooks are only ran once (jakemack, #546)

data/lib/resque/errors.rb

@@ -7,4 +7,7 @@ module Resque
 
   # Raised when a worker was killed while processing a job.
   class DirtyExit < RuntimeError; end
+
+  # Raised when child process is TERM'd so job can rescue this to do shutdown work.
+  class TermException < SignalException; end
 end

data/lib/resque/job.rb

@@ -44,7 +44,9 @@ module Resque
       Resque.validate(klass, queue)
 
       if Resque.inline?
-        constantize(klass).perform(*decode(encode(args)))
+        # Instantiating a Resque::Job and calling perform on it so callbacks run
+        # decode(encode(args)) to ensure that args are normalized in the same manner as a non-inline job
+        new(:inline, {'class' => klass, 'args' => decode(encode(args))}).perform
       else
         Resque.push(queue, :class => klass.to_s, :args => args)
       end

data/lib/resque/tasks.rb

@@ -14,6 +14,8 @@ namespace :resque do
       worker = Resque::Worker.new(*queues)
       worker.verbose = ENV['LOGGING'] || ENV['VERBOSE']
       worker.very_verbose = ENV['VVERBOSE']
+      worker.term_timeout = ENV['RESQUE_TERM_TIMEOUT'] || 4.0
+      worker.term_child = ENV['TERM_CHILD']
     rescue Resque::NoQueueError
       abort "set QUEUE env var, e.g. $ QUEUE=critical,high rake resque:work"
     end

data/lib/resque/version.rb

@@ -1,3 +1,3 @@
 module Resque
-  Version = VERSION = '1.21.0'
+  Version = VERSION = '1.22.0'
 end

data/lib/resque/worker.rb

@@ -20,6 +20,11 @@ module Resque
     # Automatically set if a fork(2) fails.
     attr_accessor :cant_fork
 
+    attr_accessor :term_timeout
+
+    # decide whether to use new_kill_child logic
+    attr_accessor :term_child
+
     attr_writer :to_s
 
     # Returns an array of all worker objects.
@@ -137,8 +142,13 @@ module Resque
           if @child = fork
             srand # Reseeding
             procline "Forked #{@child} at #{Time.now.to_i}"
-            Process.wait(@child)
+            begin
+              Process.waitpid(@child)
+            rescue SystemCallError
+              nil
+            end
           else
+            unregister_signal_handlers if !@cant_fork && term_child
             procline "Processing #{job.queue} since #{Time.now.to_i}"
             redis.client.reconnect # Don't share connection with parent
             perform(job, &block)
@@ -238,6 +248,7 @@ module Resque
 
     # Runs all the methods needed when a worker begins its lifecycle.
     def startup
+      warn "WARNING: This way of doing signal handling is now deprecated. Please see http://hone.heroku.com/resque/2012/08/21/resque-signals.html for more info." unless term_child
       enable_gc_optimizations
       register_signal_handlers
       prune_dead_workers
@@ -271,7 +282,11 @@ module Resque
 
       begin
         trap('QUIT') { shutdown   }
-        trap('USR1') { kill_child }
+        if term_child
+          trap('USR1') { new_kill_child }
+        else
+          trap('USR1') { kill_child }
+        end
         trap('USR2') { pause_processing }
         trap('CONT') { unpause_processing }
       rescue ArgumentError
@@ -281,6 +296,18 @@ module Resque
       log! "Registered signals"
     end
 
+    def unregister_signal_handlers
+      trap('TERM') { raise TermException.new("SIGTERM") }
+      trap('INT', 'DEFAULT')
+
+      begin
+        trap('QUIT', 'DEFAULT')
+        trap('USR1', 'DEFAULT')
+        trap('USR2', 'DEFAULT')
+      rescue ArgumentError
+      end
+    end
+
     # Schedule this worker for shutdown. Will finish processing the
     # current job.
     def shutdown
@@ -291,7 +318,11 @@ module Resque
     # Kill the child and shutdown immediately.
     def shutdown!
       shutdown
-      kill_child
+      if term_child
+        new_kill_child
+      else
+        kill_child
+      end
     end
 
     # Should this worker shutdown as soon as current job is finished?
@@ -313,6 +344,28 @@ module Resque
       end
     end
 
+    # Kills the forked child immediately with minimal remorse. The job it
+    # is processing will not be completed. Send the child a TERM signal,
+    # wait 5 seconds, and then a KILL signal if it has not quit
+    def new_kill_child
+      if @child
+        unless Process.waitpid(@child, Process::WNOHANG)
+          log! "Sending TERM signal to child #{@child}"
+          Process.kill("TERM", @child)
+          (term_timeout.to_f * 10).round.times do |i|
+            sleep(0.1)
+            return if Process.waitpid(@child, Process::WNOHANG)
+          end
+          log! "Sending KILL signal to child #{@child}"
+          Process.kill("KILL", @child)
+        else
+          log! "Child #{@child} already quit."
+        end
+      end
+    rescue SystemCallError
+      log! "Child #{@child} already quit and reaped."
+    end
+
     # are we paused?
     def paused?
       @paused

data/test/job_hooks_test.rb

@@ -420,4 +420,45 @@ context "Resque::Job all hooks" do
       "oh no"
     ]
   end
+
+  class ::CallbacksInline
+    @queue = :callbacks_inline
+
+    def self.before_perform_record_history(history, count)
+      history << :before_perform
+      count['count'] += 1
+    end
+
+    def self.after_perform_record_history(history, count)
+      history << :after_perform
+      count['count'] += 1
+    end
+
+    def self.around_perform_record_history(history, count)
+      history << :start_around_perform
+      count['count'] += 1
+      yield
+      history << :finish_around_perform
+      count['count'] += 1
+    end
+
+    def self.perform(history, count)
+      history << :perform
+      $history = history
+      $count = count
+    end
+  end
+
+  test "it runs callbacks when inline is true" do
+    begin
+      Resque.inline = true
+      # Sending down two parameters that can be passed and updated by reference
+      result = Resque.enqueue(CallbacksInline, [], {'count' => 0})
+      assert_equal true, result, "perform returned true"
+      assert_equal $history, [:before_perform, :start_around_perform, :perform, :finish_around_perform, :after_perform]
+      assert_equal 4, $count['count']
+    ensure
+      Resque.inline = false
+    end
+  end
 end

data/test/test_helper.rb

@@ -1,7 +1,7 @@
 require 'rubygems'
 
-dir = File.dirname(File.expand_path(__FILE__))
-$LOAD_PATH.unshift dir + '/../lib'
+$dir = File.dirname(File.expand_path(__FILE__))
+$LOAD_PATH.unshift $dir + '/../lib'
 $TESTING = true
 require 'test/unit'
 
@@ -42,21 +42,21 @@ at_exit do
   processes = `ps -A -o pid,command | grep [r]edis-test`.split("\n")
   pids = processes.map { |process| process.split(" ")[0] }
   puts "Killing test redis server..."
-  `rm -f #{dir}/dump.rdb #{dir}/dump-cluster.rdb`
-  pids.each { |pid| Process.kill("KILL", pid.to_i) }
+  pids.each { |pid| Process.kill("TERM", pid.to_i) }
+  system("rm -f #{$dir}/dump.rdb #{$dir}/dump-cluster.rdb")
   exit exit_code
 end
 
 if ENV.key? 'RESQUE_DISTRIBUTED'
   require 'redis/distributed'
   puts "Starting redis for testing at localhost:9736 and localhost:9737..."
-  `redis-server #{dir}/redis-test.conf`
-  `redis-server #{dir}/redis-test-cluster.conf`
+  `redis-server #{$dir}/redis-test.conf`
+  `redis-server #{$dir}/redis-test-cluster.conf`
   r = Redis::Distributed.new(['redis://localhost:9736', 'redis://localhost:9737'])
   Resque.redis = Redis::Namespace.new :resque, :redis => r
 else
   puts "Starting redis for testing at localhost:9736..."
-  `redis-server #{dir}/redis-test.conf`
+  `redis-server #{$dir}/redis-test.conf`
   Resque.redis = 'localhost:9736'
 end
 
@@ -144,6 +144,8 @@ ensure
   Resque::Failure.backend = previous_backend
 end
 
+require 'time'
+
 class Time
   # Thanks, Timecop
   class << self
@@ -158,3 +160,15 @@ class Time
 
   self.fake_time = nil
 end
+
+def capture_stderr
+  # The output stream must be an IO-like object. In this case we capture it in
+  # an in-memory IO object so we can return the string value. You can assign any
+  # IO object here.
+  previous_stderr, $stderr = $stderr, StringIO.new
+  yield
+  $stderr.string
+ensure
+  # Restore the previous value of stderr (typically equal to STDERR).
+  $stderr = previous_stderr
+end

data/test/worker_test.rb

@@ -286,7 +286,7 @@ context "Resque::Worker" do
   test "knows when it started" do
     time = Time.now
     @worker.work(0) do
-      assert_equal time.to_s, @worker.started.to_s
+      assert Time.parse(@worker.started) - time < 0.1
     end
   end
 
@@ -437,4 +437,152 @@ context "Resque::Worker" do
     @worker.work(0)
     assert_not_equal original_connection, Resque.redis.client.connection.instance_variable_get("@sock")
   end
+
+  if !defined?(RUBY_ENGINE) || defined?(RUBY_ENGINE) && RUBY_ENGINE != "jruby"
+    test "old signal handling is the default" do
+      rescue_time = nil
+
+      begin
+        class LongRunningJob
+          @queue = :long_running_job
+
+          def self.perform( run_time, rescue_time=nil )
+            Resque.redis.client.reconnect # get its own connection
+            Resque.redis.rpush( 'sigterm-test:start', Process.pid )
+            sleep run_time
+            Resque.redis.rpush( 'sigterm-test:result', 'Finished Normally' )
+          rescue Resque::TermException => e
+            Resque.redis.rpush( 'sigterm-test:result', %Q(Caught SignalException: #{e.inspect}))
+            sleep rescue_time unless rescue_time.nil?
+          ensure
+            puts 'fuuuu'
+            Resque.redis.rpush( 'sigterm-test:final', 'exiting.' )
+          end
+        end
+
+        Resque.enqueue( LongRunningJob, 5, rescue_time )
+
+        worker_pid = Kernel.fork do
+          # ensure we actually fork
+          $TESTING = false
+          # reconnect since we just forked
+          Resque.redis.client.reconnect
+
+          worker = Resque::Worker.new(:long_running_job)
+
+          worker.work(0)
+          exit!
+        end
+
+        # ensure the worker is started
+        start_status = Resque.redis.blpop( 'sigterm-test:start', 5 )
+        assert_not_nil start_status
+        child_pid = start_status[1].to_i
+        assert_operator child_pid, :>, 0
+
+        # send signal to abort the worker
+        Process.kill('TERM', worker_pid)
+        Process.waitpid(worker_pid)
+
+        # wait to see how it all came down
+        result = Resque.redis.blpop( 'sigterm-test:result', 5 )
+        assert_nil result
+
+        # ensure that the child pid is no longer running
+        child_still_running = !(`ps -p #{child_pid.to_s} -o pid=`).empty?
+        assert !child_still_running
+      ensure
+        remaining_keys = Resque.redis.keys('sigterm-test:*') || []
+        Resque.redis.del(*remaining_keys) unless remaining_keys.empty?
+      end
+    end
+  end
+
+  if !defined?(RUBY_ENGINE) || defined?(RUBY_ENGINE) && RUBY_ENGINE != "jruby"
+    [SignalException, Resque::TermException].each do |exception|
+      {
+        'cleanup occurs in allotted time' => nil,
+        'cleanup takes too long' => 2
+      }.each do |scenario,rescue_time|
+        test "SIGTERM when #{scenario} while catching #{exception}" do
+          begin
+            eval("class LongRunningJob; @@exception = #{exception}; end")
+            class LongRunningJob
+              @queue = :long_running_job
+
+              def self.perform( run_time, rescue_time=nil )
+                Resque.redis.client.reconnect # get its own connection
+                Resque.redis.rpush( 'sigterm-test:start', Process.pid )
+                sleep run_time
+                Resque.redis.rpush( 'sigterm-test:result', 'Finished Normally' )
+              rescue @@exception => e
+                Resque.redis.rpush( 'sigterm-test:result', %Q(Caught SignalException: #{e.inspect}))
+                sleep rescue_time unless rescue_time.nil?
+              ensure
+                Resque.redis.rpush( 'sigterm-test:final', 'exiting.' )
+              end
+            end
+
+            Resque.enqueue( LongRunningJob, 5, rescue_time )
+
+            worker_pid = Kernel.fork do
+              # ensure we actually fork
+              $TESTING = false
+              # reconnect since we just forked
+              Resque.redis.client.reconnect
+
+              worker = Resque::Worker.new(:long_running_job)
+              worker.term_timeout = 1
+              worker.term_child = 1
+
+              worker.work(0)
+              exit!
+            end
+
+            # ensure the worker is started
+            start_status = Resque.redis.blpop( 'sigterm-test:start', 5 )
+            assert_not_nil start_status
+            child_pid = start_status[1].to_i
+            assert_operator child_pid, :>, 0
+
+            # send signal to abort the worker
+            Process.kill('TERM', worker_pid)
+            Process.waitpid(worker_pid)
+
+            # wait to see how it all came down
+            result = Resque.redis.blpop( 'sigterm-test:result', 5 )
+            assert_not_nil result
+            assert !result[1].start_with?('Finished Normally'), 'Job Finished normally. Sleep not long enough?'
+            assert result[1].start_with? 'Caught SignalException', 'Signal exception not raised in child.'
+
+            # ensure that the child pid is no longer running
+            child_still_running = !(`ps -p #{child_pid.to_s} -o pid=`).empty?
+            assert !child_still_running
+
+            # see if post-cleanup occurred. This should happen IFF the rescue_time is less than the term_timeout
+            post_cleanup_occurred = Resque.redis.lpop( 'sigterm-test:final' )
+            assert post_cleanup_occurred, 'post cleanup did not occur. SIGKILL sent too early?' if rescue_time.nil?
+            assert !post_cleanup_occurred, 'post cleanup occurred. SIGKILL sent too late?' unless rescue_time.nil?
+
+          ensure
+            remaining_keys = Resque.redis.keys('sigterm-test:*') || []
+            Resque.redis.del(*remaining_keys) unless remaining_keys.empty?
+          end
+        end
+      end
+    end
+
+    test "displays warning when not using term_child" do
+      stderr = capture_stderr { @worker.work(0) }
+
+      assert stderr.match(/^WARNING:/)
+    end
+
+    test "it does not display warning when using term_child" do
+      @worker.term_child = "1"
+      stderr = capture_stderr { @worker.work(0) }
+
+      assert !stderr.match(/^WARNING:/)
+    end
+  end
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: resque
 version: !ruby/object:Gem::Version 
-  hash: 67
+  hash: 79
   prerelease: 
   segments: 
   - 1
-  - 21
+  - 22
   - 0
-  version: 1.21.0
+  version: 1.22.0
 platform: ruby
 authors: 
 - Chris Wanstrath
@@ -16,7 +16,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-07-03 00:00:00 Z
+date: 2012-08-21 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: redis-namespace
@@ -151,6 +151,7 @@ files:
 - test/test_helper.rb
 - test/hoptoad_test.rb
 - test/resque_test.rb
+- test/dump.rdb
 - test/job_plugins_test.rb
 homepage: http://github.com/defunkt/resque
 licenses: []
@@ -181,7 +182,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.8.21
+rubygems_version: 1.8.24
 signing_key: 
 specification_version: 3
 summary: Resque is a Redis-backed queueing system.