resque-multi-job-forks 0.4.3 → 0.5.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: a6c7a43817f8c3f8d4b3ed8a0c7efc997c079de4
-  data.tar.gz: 374c601a0654d238db83989970a3317d2399f65b
+SHA256:
+  metadata.gz: 2b710d8dc861eb050596364959ae65da3f099c29492370d3a817239428cf5a38
+  data.tar.gz: 45b5e45ddf6aac1a092b5d9047483ad4b9d4064920d5f875f8251686310e2d46
 SHA512:
-  metadata.gz: bc729ac05237225b44fd4a2745e3a97aca598cb51561624c34d29183a09b321789e8e095cf656b9234a4875c38b03aeda74c5c3bc862973f310fa8055d31c80d
-  data.tar.gz: eb6a4d42cad8379ee8c18f9ece2231ee16298ba359a6f036e5b5d426904dc5d1b922c0bb5037abd4e7d661d48477527d53eab22e39bb5b7712f47935a4eaafc0
+  metadata.gz: 2d98283d9a0b31141e943d452cf786781c5b6e3e42dbacc2a7175a41e8f4380bc85914570215111c1a088ed16e6f9d08855cba454ca943e75eac09694f0d8da2
+  data.tar.gz: 6199e71957234fe7c374778ddb28e56207e9c438750eb2051852f0bd150cb89304c587351e007d76eb680ed0a09c42fcb6e52ef0f7f3c99eaae84e9e2a7d06a4

data/lib/resque-multi-job-forks.rb

@@ -15,7 +15,30 @@ module Resque
     end
 
     if multi_jobs_per_fork? && !method_defined?(:shutdown_without_multi_job_forks)
+
+      def fork(&block)
+        if child = Kernel.fork
+          return child
+        else
+          if term_child
+            unregister_signal_handlers
+            trap('QUIT') { shutdown }
+          end
+          raise NotImplementedError, "Pretending to not have forked"
+          # perform_with_fork will run the job and continue working...
+        end
+      end
+
+      def work_with_multi_job_forks(*args)
+        pid # forces @pid to be set in the parent
+        work_without_multi_job_forks(*args)
+        release_and_exit! unless is_parent_process?
+      end
+      alias_method :work_without_multi_job_forks, :work
+      alias_method :work, :work_with_multi_job_forks
+
       def perform_with_multi_job_forks(job = nil)
+        @fork_per_job = true unless fork_hijacked? # reconnect and after_fork
         perform_without_multi_job_forks(job)
         hijack_fork unless fork_hijacked?
         @jobs_processed += 1
@@ -31,15 +54,15 @@ module Resque
       alias_method :shutdown?, :shutdown_with_multi_job_forks?
 
       def shutdown_with_multi_job_forks
+        shutdown_child
         shutdown_without_multi_job_forks
-        shutdown_child if is_parent_process?
       end
       alias_method :shutdown_without_multi_job_forks, :shutdown
       alias_method :shutdown, :shutdown_with_multi_job_forks
 
       def pause_processing_with_multi_job_forks
+        shutdown_child
         pause_processing_without_multi_job_forks
-        shutdown_child if is_parent_process?
       end
       alias_method :pause_processing_without_multi_job_forks, :pause_processing
       alias_method :pause_processing, :pause_processing_with_multi_job_forks
@@ -49,7 +72,7 @@ module Resque
         working_on_without_worker_registration(job)
       end
       alias_method :working_on_without_worker_registration, :working_on
-      alias_method :working_on, :working_on_with_worker_registration    
+      alias_method :working_on, :working_on_with_worker_registration
 
       # Reconnect only once
       def reconnect_with_multi_job_forks
@@ -62,18 +85,26 @@ module Resque
       alias_method :reconnect, :reconnect_with_multi_job_forks
     end
 
-    # Need to tell the child to shutdown since it might be looping performing multiple jobs per fork
-    # The TSTP signal is registered only in forked processes and calls this function
+    # Need to tell the child to shutdown since it might be looping performing
+    # multiple jobs per fork. The QUIT signal normally does a graceful shutdown,
+    # and is re-registered in children (term_child normally unregisters it).
     def shutdown_child
+      return unless @child
       begin
-        Process.kill('TSTP', @child)
+        log_with_severity :debug, "multi_jobs_per_fork: Sending QUIT signal to #{@child}"
+        Process.kill('QUIT', @child)
       rescue Errno::ESRCH
         nil
       end
     end
 
     def is_parent_process?
-      @child
+      @child || @pid == Process.pid
+    end
+
+    def release_and_exit!
+      release_fork if fork_hijacked?
+      run_at_exit_hooks ? exit : exit!(true)
     end
 
     def fork_hijacked?
@@ -81,26 +112,25 @@ module Resque
     end
 
     def hijack_fork
-      log 'hijack fork.'
+      log_with_severity :debug, 'hijack fork.'
       @suppressed_fork_hooks = [Resque.after_fork, Resque.before_fork]
       Resque.after_fork = Resque.before_fork = nil
       @release_fork_limit = fork_job_limit
       @jobs_processed = 0
-      @cant_fork = true
-      trap('TSTP') { shutdown }
+      @fork_per_job = false
     end
 
     def release_fork
-      log "jobs processed by child: #{jobs_processed}; rss: #{rss}"
+      log_with_severity :info, "jobs processed by child: #{jobs_processed}; rss: #{rss}"
       run_hook :before_child_exit, self
       Resque.after_fork, Resque.before_fork = *@suppressed_fork_hooks
-      @release_fork_limit = @jobs_processed = @cant_fork = nil
-      log 'hijack over, counter terrorists win.'
-      @shutdown = true unless $TESTING
+      @release_fork_limit = @jobs_processed = nil
+      log_with_severity :debug, 'hijack over, counter terrorists win.'
+      @shutdown = true
     end
 
     def fork_job_limit
-      jobs_per_fork.nil? ? Time.now.to_i + seconds_per_fork : jobs_per_fork
+      jobs_per_fork.nil? ? Time.now.to_f + seconds_per_fork : jobs_per_fork
     end
 
     def fork_job_limit_reached?
@@ -108,7 +138,7 @@ module Resque
     end
 
     def fork_job_limit_remaining
-      jobs_per_fork.nil? ? @release_fork_limit - Time.now.to_i : jobs_per_fork - @jobs_processed
+      jobs_per_fork.nil? ? @release_fork_limit - Time.now.to_f : jobs_per_fork - @jobs_processed
     end
 
     def seconds_per_fork
@@ -140,16 +170,11 @@ module Resque
   # Call with a block to set the hook.
   # Call with no arguments to return the hook.
   def self.before_child_exit(&block)
-    if block
-      @before_child_exit ||= []
-      @before_child_exit << block
-    end
-    @before_child_exit
+    block ? register_hook(:before_child_exit, block) : hooks(:before_child_exit)
   end
 
   # Set the before_child_exit proc.
   def self.before_child_exit=(before_child_exit)
-    @before_child_exit = before_child_exit.respond_to?(:each) ? before_child_exit : [before_child_exit].compact
+    register_hook(:before_child_exit, block)
   end
-
 end

data/test/helper.rb

@@ -12,72 +12,50 @@ require 'resque-multi-job-forks'
 require 'timeout'
 
 # setup redis & resque.
-redis = Redis.new(:db => 1)
-Resque.redis = redis
+$redis = Redis.new(:db => 1)
+Resque.redis = $redis
 
 # adds simple STDOUT logging to test workers.
 # set `VERBOSE=true` when running the tests to view resques log output.
 module Resque
   class Worker
-    def log(msg)
-      puts "*** #{msg}" unless ENV['VERBOSE'].nil?
-    end
-    alias_method :log!, :log
 
-    # Processes a given job in the child.
-    def perform_without_multi_job_forks(job)
-      begin
-        # 'will_fork?' returns false in test mode, but since we need to test
-        # if the :after_fork hook runs, we ignore 'will_fork?' here.
-        run_hook :after_fork, job # if will_fork?
-        job.perform
-      rescue Object => e
-        log "#{job.inspect} failed: #{e.inspect}"
-        begin
-          job.fail(e)
-        rescue Object => e
-          log "Received exception when reporting failure: #{e.inspect}"
-        end
-        failed!
-      else
-        log "done: #{job.inspect}"
-      ensure
-        yield job if block_given?
+    def log_with_severity(severity, msg)
+      if ENV['VERBOSE']
+        s = severity.to_s[0].upcase
+        $stderr.print "*** [#{Time.now}] [#{Process.pid}] #{self} #{s}: #{msg}\n"
       end
     end
+
   end
 end
 
-# stores a record of the job processing sequence.
-# you may wish to reset this in the test `setup` method.
-$SEQUENCE = []
-
 # test job, tracks sequence.
 class SequenceJob
   @queue = :jobs
   def self.perform(i)
-    $SEQUENCE << "work_#{i}".to_sym
     sleep(2)
+    $SEQ_WRITER.print "work_#{i}\n"
   end
 end
 
 class QuickSequenceJob
   @queue = :jobs
   def self.perform(i)
-    $SEQUENCE << "work_#{i}".to_sym
+    $SEQ_WRITER.print "work_#{i}\n"
   end
 end
 
 
 # test hooks, tracks sequence.
 Resque.after_fork do
-  $SEQUENCE << :after_fork
+  $SEQ_WRITER.print "after_fork\n"
 end
 
 Resque.before_fork do
-  $SEQUENCE << :before_fork
+  $SEQ_WRITER.print "before_fork\n"
 end
 
 Resque.before_child_exit do |worker|
-  $SEQUENCE << "before_child_exit_#{worker.jobs_processed}".to_sym
+  $SEQ_WRITER.print "before_child_exit_#{worker.jobs_processed}\n"
 end

data/test/test_resque-multi-job-forks.rb

@@ -2,12 +2,13 @@ require File.join(File.expand_path(File.dirname(__FILE__)), '/helper')
 
 class TestResqueMultiJobForks < Test::Unit::TestCase
   def setup
-    $SEQUENCE = []
-    Resque.redis.flushdb
+    $SEQ_READER, $SEQ_WRITER = IO.pipe
+    $redis.flushdb
     @worker = Resque::Worker.new(:jobs)
   end
 
   def test_timeout_limit_sequence_of_events
+    @worker.log_with_severity :debug, "in test_timeout_limit_sequence_of_events"
     # only allow enough time for 3 jobs to process.
     @worker.seconds_per_fork = 3
 
@@ -16,43 +17,119 @@ class TestResqueMultiJobForks < Test::Unit::TestCase
     Resque.enqueue(SequenceJob, 3)
     Resque.enqueue(SequenceJob, 4)
 
-    # make sure we don't take longer then 15 seconds.
-    begin
-      Timeout::timeout(15) { @worker.work(1) }
-    rescue Timeout::Error
-    end
+    # INTERVAL=0 will exit when no more jobs left
+    @worker.work(0)
+    $SEQ_WRITER.close
+    sequence = $SEQ_READER.each_line.map {|l| l.strip.to_sym }
 
     # test the sequence is correct.
     assert_equal([:before_fork, :after_fork, :work_1, :work_2, :work_3,
                   :before_child_exit_3, :before_fork, :after_fork, :work_4,
-                  :before_child_exit_1], $SEQUENCE, 'correct sequence')
+                  :before_child_exit_1], sequence, 'correct sequence')
+  end
+
+  def test_graceful_shutdown_during_first_job
+    @worker.log_with_severity :debug, "in test_graceful_shutdown_during_first_job"
+    # enough time for all jobs to process.
+    @worker.seconds_per_fork = 60
+
+    Resque.enqueue(SequenceJob, 1)
+    Resque.enqueue(SequenceJob, 2)
+    t = Thread.new do
+      sleep 1 # before first job can complete
+      @worker.shutdown
+    end
+    # INTERVAL=0 will exit when no more jobs left
+    @worker.work(0)
+    $SEQ_WRITER.close
+    sequence = $SEQ_READER.each_line.map {|l| l.strip.to_sym }
+
+    # test the sequence is correct.
+    assert_equal([:before_fork, :after_fork, :work_1,
+                  :before_child_exit_1], sequence, 'correct sequence')
+    t.join
+  end
+
+  def test_immediate_shutdown_during_first_job
+    @worker.log_with_severity :debug, "in test_immediate_shutdown_during_first_job"
+    # enough time for all jobs to process.
+    @worker.seconds_per_fork = 60
+    @worker.term_child = false
+
+    Resque.enqueue(SequenceJob, 1)
+    Resque.enqueue(SequenceJob, 2)
+    t = Thread.new do
+      sleep 0.5 # before first job can complete
+      Process.kill("INT", @worker.pid) # triggers shutdown! in main thread
+    end
+    # INTERVAL=0 will exit when no more jobs left
+    @worker.work(0)
+    $SEQ_WRITER.close
+    sequence = $SEQ_READER.each_line.map {|l| l.strip.to_sym }
+
+    # test the sequence is correct.
+    assert_equal([:before_fork, :after_fork], sequence, 'correct sequence')
+    t.join
+  end
+
+  def test_sigterm_shutdown_during_first_job
+    @worker.log_with_severity :debug, "in test_sigterm_shutdown_during_first_job"
+    # enough time for all jobs to process.
+    @worker.seconds_per_fork = 60
+    @worker.term_child = true
+    @worker.term_timeout = 0.5
+
+    Resque.enqueue(SequenceJob, 1)
+    Resque.enqueue(SequenceJob, 2)
+    t = Thread.new do
+      sleep 1.0 # before first job can complete
+      Process.kill("INT", @worker.pid) # triggers shutdown! in main thread
+    end
+    # INTERVAL=0 will exit when no more jobs left
+    @worker.work(0)
+    $SEQ_WRITER.close
+    sequence = $SEQ_READER.each_line.map {|l| l.strip.to_sym }
+
+    # test the sequence is correct.
+    assert_equal([:before_fork, :after_fork,
+                  :before_child_exit_1], sequence, 'correct sequence')
+    t.join
   end
 
   # test we can also limit fork job process by a job limit.
   def test_job_limit_sequence_of_events
-    # only allow enough time for 3 jobs to process.
+    @worker.log_with_severity :debug, "in test_job_limit_sequence_of_events"
+    # only allow 20 jobs per fork
     ENV['JOBS_PER_FORK'] = '20'
 
     # queue 40 jobs.
     (1..40).each { |i| Resque.enqueue(QuickSequenceJob, i) }
 
-    begin
-      Timeout::timeout(3) { @worker.work(1) }
-    rescue Timeout::Error
-    end
+    # INTERVAL=0 will exit when no more jobs left
+    @worker.work(0)
+    $SEQ_WRITER.close
+    sequence = $SEQ_READER.each_line.map {|l| l.strip.to_sym }
 
-    assert_equal :before_fork, $SEQUENCE[0], 'first before_fork call.'
-    assert_equal :after_fork, $SEQUENCE[1], 'first after_fork call.'
-    assert_equal :work_20, $SEQUENCE[21], '20th chunk of work.'
-    assert_equal :before_child_exit_20, $SEQUENCE[22], 'first before_child_exit call.'
-    assert_equal :before_fork, $SEQUENCE[23], 'final before_fork call.'
-    assert_equal :after_fork, $SEQUENCE[24], 'final after_fork call.'
-    assert_equal :work_40, $SEQUENCE[44], '40th chunk of work.'
-    assert_equal :before_child_exit_20, $SEQUENCE[45], 'final before_child_exit call.'
+    assert_equal(%i[
+      before_fork after_fork
+      work_1 work_2 work_3 work_4 work_5
+      work_6 work_7 work_8 work_9 work_10
+      work_11 work_12 work_13 work_14 work_15
+      work_16 work_17 work_18 work_19 work_20
+      before_child_exit_20
+      before_fork after_fork
+      work_21 work_22 work_23 work_24 work_25
+      work_26 work_27 work_28 work_29 work_30
+      work_31 work_32 work_33 work_34 work_35
+      work_36 work_37 work_38 work_39 work_40
+      before_child_exit_20
+    ], sequence, 'correct sequence')
   end
 
   def teardown
     # make sure we don't clobber any other tests.
     ENV['JOBS_PER_FORK'] = nil
+    Resque::Worker.kill_all_heartbeat_threads
   end
+
 end

data/test/test_rss_reader.rb

@@ -7,17 +7,20 @@ class TestRssReader < Test::Unit::TestCase
     @object.extend Resque::Plugins::MultiJobForks::RssReader
   end
 
+  # support before/after ruby 2.4 without deprecation notice
+  IntegerClass = 1.class
+
   def test_current_process_rss
     rss = @object.rss
     # not a very strict test, but have you ever seen a ruby process < 1Mb?
     # the "real" test is manual verification via top/ps/htop
-    assert_equal Fixnum, rss.class
+    assert_equal IntegerClass, rss.class
     assert @object.rss > 1000
   end
 
   def test_rss_other_processes
     rss = @object.rss(1) # init is guaranteed to exist
-    assert_equal Fixnum, rss.class
+    assert_equal IntegerClass, rss.class
     assert @object.rss > 1000
   end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: resque-multi-job-forks
 version: !ruby/object:Gem::Version
-  version: 0.4.3
+  version: 0.5.2
 platform: ruby
 authors:
 - Mick Staugaard
@@ -10,22 +10,28 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-10-26 00:00:00.000000000 Z
+date: 2021-03-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: resque
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.27.0
+    - - "<"
       - !ruby/object:Gem::Version
-        version: '1.22'
+        version: '2.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.22'
+        version: 1.27.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2.1'
 - !ruby/object:Gem::Dependency
   name: json
   requirement: !ruby/object:Gem::Requirement
@@ -41,7 +47,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: test-unit
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -68,6 +74,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: When your resque jobs are frequent and fast, the overhead of forking
   and running your after_fork might get too big.
 email:
@@ -83,7 +103,7 @@ files:
 - test/helper.rb
 - test/test_resque-multi-job-forks.rb
 - test/test_rss_reader.rb
-homepage: http://github.com/staugaard/resque-multi-job-forks
+homepage: https://github.com/stulentsev/resque-multi-job-forks
 licenses: []
 metadata: {}
 post_install_message: 
@@ -101,8 +121,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.4.5.1
+rubygems_version: 3.1.4
 signing_key: 
 specification_version: 4
 summary: Have your resque workers process more that one job
@@ -110,4 +129,3 @@ test_files:
 - test/helper.rb
 - test/test_resque-multi-job-forks.rb
 - test/test_rss_reader.rb
-has_rdoc: