resque-multi-job-forks 0.4.4 → 0.4.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: cae179fcc38decf3f26b41ecf89c37550845d13c
-  data.tar.gz: 6e04ade5c457d3470d67a083f19ebed05eb664b8
+SHA256:
+  metadata.gz: e14fad0b41eaab14a736ac76d9e234314c97279b190d90ab935a74717bc18921
+  data.tar.gz: fc1a9a67f6a6825180044316144b70923f771d454e5c6338c86ccd22aecb3fb8
 SHA512:
-  metadata.gz: 5483cface37a516972d8303c5b9a99b3f1e0d1f6176bbedc7ac54e29e3949402400df4df31500583e27e69023e230d7c75fdcd3c43ef036bc7fe71a28d7d6e42
-  data.tar.gz: c96715f6b5a10e31ac584bad2a4e86b127b86e8fa3788b35fac6d9dfe8f68b3e80671333185d5d59b86a9796e445b80ef6ee9a93c1de9474d42718da85fc8bd7
+  metadata.gz: 2da4375dc2b26da0c670e402954c0b5496a6ce46c3100eabe4f9991d50cd987b0c8d4e5745fdb72f1632f35934f112eb42df6a141c7d838904ed12419f0e8a7d
+  data.tar.gz: cc8c72fa05245eae3aa8a19811b54279ee00f6257fb8093942c9fd7c8d62336fb4ce1e7242de96e997772baff15486838cd3418ca0436d21ec38dd6f4cb4a3fe

data/lib/resque-multi-job-forks.rb

@@ -15,7 +15,16 @@ module Resque
     end
 
     if multi_jobs_per_fork? && !method_defined?(:shutdown_without_multi_job_forks)
+      def work_with_multi_job_forks(*args)
+        pid # forces @pid to be set in the parent
+        work_without_multi_job_forks(*args)
+        release_and_exit! unless is_parent_process?
+      end
+      alias_method :work_without_multi_job_forks, :work
+      alias_method :work, :work_with_multi_job_forks
+
       def perform_with_multi_job_forks(job = nil)
+        trap('QUIT') { shutdown } unless fork_hijacked?
         perform_without_multi_job_forks(job)
         hijack_fork unless fork_hijacked?
         @jobs_processed += 1
@@ -31,15 +40,15 @@ module Resque
       alias_method :shutdown?, :shutdown_with_multi_job_forks?
 
       def shutdown_with_multi_job_forks
-        shutdown_without_multi_job_forks
         shutdown_child if is_parent_process?
+        shutdown_without_multi_job_forks
       end
       alias_method :shutdown_without_multi_job_forks, :shutdown
       alias_method :shutdown, :shutdown_with_multi_job_forks
 
       def pause_processing_with_multi_job_forks
-        pause_processing_without_multi_job_forks
         shutdown_child if is_parent_process?
+        pause_processing_without_multi_job_forks
       end
       alias_method :pause_processing_without_multi_job_forks, :pause_processing
       alias_method :pause_processing, :pause_processing_with_multi_job_forks
@@ -49,7 +58,7 @@ module Resque
         working_on_without_worker_registration(job)
       end
       alias_method :working_on_without_worker_registration, :working_on
-      alias_method :working_on, :working_on_with_worker_registration    
+      alias_method :working_on, :working_on_with_worker_registration
 
       # Reconnect only once
       def reconnect_with_multi_job_forks
@@ -62,18 +71,25 @@ module Resque
       alias_method :reconnect, :reconnect_with_multi_job_forks
     end
 
-    # Need to tell the child to shutdown since it might be looping performing multiple jobs per fork
-    # The TSTP signal is registered only in forked processes and calls this function
+    # Need to tell the child to shutdown since it might be looping performing
+    # multiple jobs per fork. The QUIT signal normally does a graceful shutdown,
+    # and is re-registered in children (term_child normally unregisters it).
     def shutdown_child
       begin
-        Process.kill('TSTP', @child)
+        log! "multi_jobs_per_fork: Sending QUIT signal to #{@child}"
+        Process.kill('QUIT', @child)
       rescue Errno::ESRCH
         nil
       end
     end
 
     def is_parent_process?
-      @child
+      @child || @pid == Process.pid
+    end
+
+    def release_and_exit!
+      release_fork if fork_hijacked?
+      run_at_exit_hooks ? exit : exit!(true)
     end
 
     def fork_hijacked?
@@ -87,7 +103,7 @@ module Resque
       @release_fork_limit = fork_job_limit
       @jobs_processed = 0
       @cant_fork = true
-      trap('TSTP') { shutdown }
+      trap('QUIT') { shutdown }
     end
 
     def release_fork
@@ -100,7 +116,7 @@ module Resque
     end
 
     def fork_job_limit
-      jobs_per_fork.nil? ? Time.now.to_i + seconds_per_fork : jobs_per_fork
+      jobs_per_fork.nil? ? Time.now.to_f + seconds_per_fork : jobs_per_fork
     end
 
     def fork_job_limit_reached?
@@ -108,7 +124,7 @@ module Resque
     end
 
     def fork_job_limit_remaining
-      jobs_per_fork.nil? ? @release_fork_limit - Time.now.to_i : jobs_per_fork - @jobs_processed
+      jobs_per_fork.nil? ? @release_fork_limit - Time.now.to_f : jobs_per_fork - @jobs_processed
     end
 
     def seconds_per_fork

data/test/helper.rb

@@ -12,72 +12,48 @@ require 'resque-multi-job-forks'
 require 'timeout'
 
 # setup redis & resque.
-redis = Redis.new(:db => 1)
-Resque.redis = redis
+$redis = Redis.new(:db => 1)
+Resque.redis = $redis
 
 # adds simple STDOUT logging to test workers.
 # set `VERBOSE=true` when running the tests to view resques log output.
 module Resque
   class Worker
+
     def log(msg)
       puts "*** #{msg}" unless ENV['VERBOSE'].nil?
     end
     alias_method :log!, :log
 
-    # Processes a given job in the child.
-    def perform_without_multi_job_forks(job)
-      begin
-        # 'will_fork?' returns false in test mode, but since we need to test
-        # if the :after_fork hook runs, we ignore 'will_fork?' here.
-        run_hook :after_fork, job # if will_fork?
-        job.perform
-      rescue Object => e
-        log "#{job.inspect} failed: #{e.inspect}"
-        begin
-          job.fail(e)
-        rescue Object => e
-          log "Received exception when reporting failure: #{e.inspect}"
-        end
-        failed!
-      else
-        log "done: #{job.inspect}"
-      ensure
-        yield job if block_given?
-      end
-    end
   end
 end
 
-# stores a record of the job processing sequence.
-# you may wish to reset this in the test `setup` method.
-$SEQUENCE = []
-
 # test job, tracks sequence.
 class SequenceJob
   @queue = :jobs
   def self.perform(i)
-    $SEQUENCE << "work_#{i}".to_sym
     sleep(2)
+    $SEQ_WRITER.print "work_#{i}\n"
   end
 end
 
 class QuickSequenceJob
   @queue = :jobs
   def self.perform(i)
-    $SEQUENCE << "work_#{i}".to_sym
+    $SEQ_WRITER.print "work_#{i}\n"
   end
 end
 
 
 # test hooks, tracks sequence.
 Resque.after_fork do
-  $SEQUENCE << :after_fork
+  $SEQ_WRITER.print "after_fork\n"
 end
 
 Resque.before_fork do
-  $SEQUENCE << :before_fork
+  $SEQ_WRITER.print "before_fork\n"
 end
 
 Resque.before_child_exit do |worker|
-  $SEQUENCE << "before_child_exit_#{worker.jobs_processed}".to_sym
+  $SEQ_WRITER.print "before_child_exit_#{worker.jobs_processed}\n"
 end

data/test/test_resque-multi-job-forks.rb

@@ -2,8 +2,8 @@ require File.join(File.expand_path(File.dirname(__FILE__)), '/helper')
 
 class TestResqueMultiJobForks < Test::Unit::TestCase
   def setup
-    $SEQUENCE = []
-    Resque.redis.flushdb
+    $SEQ_READER, $SEQ_WRITER = IO.pipe
+    $redis.flushdb
     @worker = Resque::Worker.new(:jobs)
   end
 
@@ -16,39 +16,103 @@ class TestResqueMultiJobForks < Test::Unit::TestCase
     Resque.enqueue(SequenceJob, 3)
     Resque.enqueue(SequenceJob, 4)
 
-    # make sure we don't take longer then 15 seconds.
-    begin
-      Timeout::timeout(15) { @worker.work(1) }
-    rescue Timeout::Error
-    end
+    # INTERVAL=0 will exit when no more jobs left
+    @worker.work(0)
+    $SEQ_WRITER.close
+    sequence = $SEQ_READER.each_line.map {|l| l.strip.to_sym }
 
     # test the sequence is correct.
     assert_equal([:before_fork, :after_fork, :work_1, :work_2, :work_3,
                   :before_child_exit_3, :before_fork, :after_fork, :work_4,
-                  :before_child_exit_1], $SEQUENCE, 'correct sequence')
+                  :before_child_exit_1], sequence, 'correct sequence')
+  end
+
+  def test_graceful_shutdown_during_first_job
+    # enough time for all jobs to process.
+    @worker.seconds_per_fork = 60
+
+    Resque.enqueue(SequenceJob, 1)
+    Resque.enqueue(SequenceJob, 2)
+    t = Thread.new do
+      sleep 1 # before first job can complete
+      @worker.shutdown
+    end
+    # INTERVAL=0 will exit when no more jobs left
+    @worker.work(0)
+    $SEQ_WRITER.close
+    sequence = $SEQ_READER.each_line.map {|l| l.strip.to_sym }
+
+    # test the sequence is correct.
+    assert_equal([:before_fork, :after_fork, :work_1,
+                  :before_child_exit_1], sequence, 'correct sequence')
+    t.join
+  end
+
+  def test_immediate_shutdown_during_first_job
+    # enough time for all jobs to process.
+    @worker.seconds_per_fork = 60
+    @worker.term_child = false
+
+    Resque.enqueue(SequenceJob, 1)
+    Resque.enqueue(SequenceJob, 2)
+    t = Thread.new do
+      sleep 0.5 # before first job can complete
+      Process.kill("INT", @worker.pid) # triggers shutdown! in main thread
+    end
+    # INTERVAL=0 will exit when no more jobs left
+    @worker.work(0)
+    $SEQ_WRITER.close
+    sequence = $SEQ_READER.each_line.map {|l| l.strip.to_sym }
+
+    # test the sequence is correct.
+    assert_equal([:before_fork, :after_fork], sequence, 'correct sequence')
+    t.join
+  end
+
+  def test_sigterm_shutdown_during_first_job
+    # enough time for all jobs to process.
+    @worker.seconds_per_fork = 60
+    @worker.term_child = true
+    @worker.term_timeout = 0.5
+
+    Resque.enqueue(SequenceJob, 1)
+    Resque.enqueue(SequenceJob, 2)
+    t = Thread.new do
+      sleep 1.0 # before first job can complete
+      Process.kill("INT", @worker.pid) # triggers shutdown! in main thread
+    end
+    # INTERVAL=0 will exit when no more jobs left
+    @worker.work(0)
+    $SEQ_WRITER.close
+    sequence = $SEQ_READER.each_line.map {|l| l.strip.to_sym }
+
+    # test the sequence is correct.
+    assert_equal([:before_fork, :after_fork,
+                  :before_child_exit_1], sequence, 'correct sequence')
+    t.join
   end
 
   # test we can also limit fork job process by a job limit.
   def test_job_limit_sequence_of_events
-    # only allow enough time for 3 jobs to process.
+    # only allow 20 jobs per fork
     ENV['JOBS_PER_FORK'] = '20'
 
     # queue 40 jobs.
     (1..40).each { |i| Resque.enqueue(QuickSequenceJob, i) }
 
-    begin
-      Timeout::timeout(3) { @worker.work(1) }
-    rescue Timeout::Error
-    end
+    # INTERVAL=0 will exit when no more jobs left
+    @worker.work(0)
+    $SEQ_WRITER.close
+    sequence = $SEQ_READER.each_line.map {|l| l.strip.to_sym }
 
-    assert_equal :before_fork, $SEQUENCE[0], 'first before_fork call.'
-    assert_equal :after_fork, $SEQUENCE[1], 'first after_fork call.'
-    assert_equal :work_20, $SEQUENCE[21], '20th chunk of work.'
-    assert_equal :before_child_exit_20, $SEQUENCE[22], 'first before_child_exit call.'
-    assert_equal :before_fork, $SEQUENCE[23], 'final before_fork call.'
-    assert_equal :after_fork, $SEQUENCE[24], 'final after_fork call.'
-    assert_equal :work_40, $SEQUENCE[44], '40th chunk of work.'
-    assert_equal :before_child_exit_20, $SEQUENCE[45], 'final before_child_exit call.'
+    assert_equal :before_fork,          sequence[0],  'first before_fork call.'
+    assert_equal :after_fork,           sequence[1],  'first after_fork call.'
+    assert_equal :work_20,              sequence[21], '20th chunk of work.'
+    assert_equal :before_child_exit_20, sequence[22], 'first before_child_exit call.'
+    assert_equal :before_fork,          sequence[23], 'final before_fork call.'
+    assert_equal :after_fork,           sequence[24], 'final after_fork call.'
+    assert_equal :work_40,              sequence[44], '40th chunk of work.'
+    assert_equal :before_child_exit_20, sequence[45], 'final before_child_exit call.'
   end
 
   def teardown

data/test/test_rss_reader.rb

@@ -7,17 +7,20 @@ class TestRssReader < Test::Unit::TestCase
     @object.extend Resque::Plugins::MultiJobForks::RssReader
   end
 
+  # support before/after ruby 2.4 without deprecation notice
+  IntegerClass = 1.class
+
   def test_current_process_rss
     rss = @object.rss
     # not a very strict test, but have you ever seen a ruby process < 1Mb?
     # the "real" test is manual verification via top/ps/htop
-    assert_equal Fixnum, rss.class
+    assert_equal IntegerClass, rss.class
     assert @object.rss > 1000
   end
 
   def test_rss_other_processes
     rss = @object.rss(1) # init is guaranteed to exist
-    assert_equal Fixnum, rss.class
+    assert_equal IntegerClass, rss.class
     assert @object.rss > 1000
   end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: resque-multi-job-forks
 version: !ruby/object:Gem::Version
-  version: 0.4.4
+  version: 0.4.5
 platform: ruby
 authors:
 - Mick Staugaard
@@ -10,28 +10,22 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-02-27 00:00:00.000000000 Z
+date: 2019-02-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: resque
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '1.24'
-    - - "<"
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.27'
+        version: 1.26.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.24'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '1.27'
+        version: 1.26.0
 - !ruby/object:Gem::Dependency
   name: json
   requirement: !ruby/object:Gem::Requirement
@@ -74,6 +68,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: When your resque jobs are frequent and fast, the overhead of forking
   and running your after_fork might get too big.
 email:
@@ -89,7 +97,7 @@ files:
 - test/helper.rb
 - test/test_resque-multi-job-forks.rb
 - test/test_rss_reader.rb
-homepage: http://github.com/staugaard/resque-multi-job-forks
+homepage: https://github.com/stulentsev/resque-multi-job-forks
 licenses: []
 metadata: {}
 post_install_message: 
@@ -107,12 +115,11 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.13
+rubygems_version: 3.0.1
 signing_key: 
 specification_version: 4
 summary: Have your resque workers process more that one job
 test_files:
-- test/test_resque-multi-job-forks.rb
 - test/helper.rb
+- test/test_resque-multi-job-forks.rb
 - test/test_rss_reader.rb