backburner 1.7.0 → 1.8.0

data/test/workers/threads_on_fork_worker_test.rb

@@ -61,6 +61,18 @@ describe "Backburner::Workers::ThreadsOnFork module" do
       worker = @worker_class.new
       assert_equal(worker.instance_variable_get("@tubes_data")['demo.test.job-settings-override'], { :threads => 10,   :garbage => 1000,   :retries => 2 })
     end
+
+    it "should take a shared tube's settings from the class that owns the tube, not from subclasses" do
+      tubes_data = @worker_class.new.instance_variable_get("@tubes_data")
+      assert_equal({ :threads => 5, :garbage => 100, :retries => 3 }, tubes_data['demo.test.tube-limits'])
+      assert_equal({ :threads => 3, :garbage => nil, :retries => nil }, tubes_data['demo.test.tube-limits-testtubelimitslambdajob'])
+      assert_equal({ :threads => 4, :garbage => nil, :retries => nil }, tubes_data['demo.test.tube-limits-testtubelimitslambdachildjob'])
+    end
+
+    it "should fill in the settings a shared tube's class leaves unset from its subclasses" do
+      tubes_data = @worker_class.new.instance_variable_get("@tubes_data")
+      assert_equal({ :threads => 3, :garbage => 50, :retries => 2 }, tubes_data['demo.test.tube-few-limits'])
+    end
   end
 
   describe "for prepare method" do
@@ -145,23 +157,147 @@ describe "Backburner::Workers::ThreadsOnFork module" do
       refute_match(/Catastrophic failure/, out)
     end
 
-    it "fork_and_watch thread should log an error if exitstatus is != 99" do
-      process_exit = stub('process_exit')
-      process_exit.expects(:exitstatus).twice.returns(0)
+    it "fork_and_watch thread should log a normal exit if exitstatus is 0 on shutdown" do
+      process_exit = stub('process_exit', :exitstatus => 0)
       worker = @worker_class.new(%(foo))
       worker.expects(:wait_for_process).with(12).returns([nil, process_exit])
 
       wc = @worker_class
-      # TODO: Is there a best way do do this?
       worker.define_singleton_method :fork_it do
         wc.shutdown = true
         12
       end
       def worker.create_thread(*args, &block); block.call(*args) end
       out = silenced(2) { worker.start(false) }
-      assert_match(/Catastrophic failure: tube demo\.test\.foo exited with code 0\./, out)
+      assert_match(/Tube demo\.test\.foo exited normally\./, out)
+      refute_match(/Catastrophic failure/, out)
     end
 
+    describe "restarting a failed child" do
+      before do
+        @worker = @worker_class.new(%(foo))
+        def @worker.create_thread(*args, &block); block.call(*args) end
+      end
+
+      # Runs fork_and_watch over children exiting with these codes, expecting it to sleep these delays in
+      # turn, and returns what it logged
+      def expect_restart_delays(codes, delays)
+        wc = @worker_class
+        left = codes.size
+        @worker.define_singleton_method(:fork_it) { left -= 1; wc.shutdown = true if left.zero?; 12 }
+        @worker.stubs(:wait_for_process).with(12).returns(*codes.map { |code| [12, stub(:exitstatus => code)] })
+        restarts = sequence('restarts')
+        delays.each { |delay| Kernel.expects(:sleep).with(delay).in_sequence(restarts) }
+        silenced(2) { @worker.start(false) }
+      end
+
+      it "should wait longer before each fork while children keep failing, up to MAX_RESTART_DELAY" do
+        expect_restart_delays([1] * 7, [1, 2, 4, 8, 16, 30])
+      end
+
+      it "should fork again at once after a recycled child and start the delays over" do
+        expect_restart_delays([1, 1, 99, 1, 1], [1, 2, 1])
+      end
+
+      it "should log an error if exitstatus is != 99, and fork again" do
+        out = expect_restart_delays([1, 99], [1])
+        assert_match(/Catastrophic failure: tube demo\.test\.foo exited with code 1\. Forking it again in 1s\./, out)
+      end
+
+      it "should log a child that stops while shutting down without forking it again" do
+        [nil, 1].each do |code| # killed by finish_forks, or exited with an error
+          @worker_class.shutdown = false
+          out = expect_restart_delays([code], [])
+          assert_match(/Tube demo\.test\.foo stopped\./, out)
+          refute_match(/Catastrophic failure|Forking it again/, out)
+        end
+      end
+
+      it "should log the warning with a logger that has no warn method" do
+        logged = []
+        logger = Object.new
+        logger.define_singleton_method(:info) { |msg| logged << [:info, msg] }
+        logger.define_singleton_method(:error) { |msg| logged << [:error, msg] }
+        Backburner.configure { |config| config.logger = logger }
+        expect_restart_delays([0, 1], [1])
+        assert_includes logged, [:error, "Tube demo.test.foo exited with code 0. Forking it again in 1s."]
+      ensure
+        Backburner.configure { |config| config.logger = nil }
+      end
+
+      it "should warn and wait before forking again after a child exits with code 0 outside a shutdown" do
+        out = expect_restart_delays([0, 1], [1])
+        assert_match(/Tube demo\.test\.foo exited with code 0\. Forking it again in 1s\./, out)
+        refute_match(/exited normally/, out)
+      end
+
+      it "should start the delays over after a child that ran for MAX_RESTART_DELAY" do
+        @worker.stubs(:monotonic_time).returns(0, 0.5, 1, 1.5, 4, 40, 41, 41.5)
+        expect_restart_delays([1, 1, 1, 1], [1, 2, 1])
+      end
+    end # restarting a failed child
+
+    describe "when beanstalkd is down while children start" do
+      before do
+        pick_beanstalkd_port
+        start_beanstalkd
+        @templogger = Templogger.new('/tmp')
+        @beanstalk_url = Backburner.configuration.beanstalk_url
+        Backburner.configure do |config|
+          config.beanstalk_url = "beanstalk://127.0.0.1:#{@port}"
+          config.logger = @templogger.logger
+        end
+        @worker_class.garbage_after = 1
+        @watchers = []
+        @ignore_forks = true
+      end
+
+      after do
+        begin
+          @worker_class.shutdown = true
+          silenced do
+            @worker_class.stop_forks
+            Timeout::timeout(2) { sleep 0.1 while @worker_class.child_pids.length > 0 }
+            @worker_class.kill_forks
+            Timeout::timeout(2) { sleep 0.1 while @worker_class.child_pids.length > 0 }
+          end
+        ensure
+          @watchers.each(&:kill) # once they've reaped the children: one waiting to fork again would wake up in a later test
+          stop_beanstalkd
+          Backburner.configure do |config|
+            config.beanstalk_url = @beanstalk_url
+            config.logger = nil
+          end
+          @templogger.close
+        end
+      end
+
+      [1, 2].each do |threads|
+        it "should back off instead of forking in a busy loop, and resume when it is back (#{threads} thread(s) per child)" do
+          @worker_class.threads_number = threads
+          worker = @worker_class.new('foo.bar.outage')
+          forks = 0
+          watchers = @watchers
+          worker.define_singleton_method(:fork_tube) { |name| forks += 1; super(name) }
+          worker.define_singleton_method(:create_thread) { |*args, &block| super(*args, &block).tap { |thread| watchers << thread } }
+          outage_forks = nil
+          silenced(20) do
+            worker.start(false)
+            sleep 0.5 # let the first child block in reserve
+            stop_beanstalkd
+            forks_before = forks
+            sleep 4
+            outage_forks = forks - forks_before
+            start_beanstalkd
+            @worker_class.enqueue TestJobNoopFork, [], :queue => 'foo.bar.outage'
+            wait_for(10) { @templogger.body =~ /Completed TestJobNoopFork/ }
+          end
+          assert_operator outage_forks, :<=, 4, "forked #{outage_forks} children in 4 s"
+          assert_match(/Completed TestJobNoopFork/, @templogger.body)
+        end
+      end
+    end # beanstalkd down
+
     describe "fork_inner" do
 
       before do
@@ -193,7 +329,7 @@ describe "Backburner::Workers::ThreadsOnFork module" do
       it "should create threads if the number of threads is > 1" do
         worker = @worker_class.new(%(foo))
         @worker_class.expects(:threads_number).returns(5)
-        worker.expects(:create_thread).times(5)
+        worker.expects(:create_thread).times(5).returns(Thread.new {})
         silenced do
           worker.prepare
           worker.fork_inner('demo.test.foo')
@@ -204,7 +340,7 @@ describe "Backburner::Workers::ThreadsOnFork module" do
         worker = @worker_class.new(%(foo))
         @worker_class.expects(:threads_number).returns(5)
         worker.expects(:run_while_can).times(5)
-        def worker.create_thread(*args, &block); block.call(*args) end
+        def worker.create_thread(*args, &block); super(*args, &block).tap(&:join) end
         silenced do
           worker.prepare
           worker.fork_inner('demo.test.foo')
@@ -218,8 +354,9 @@ describe "Backburner::Workers::ThreadsOnFork module" do
         worker = @worker_class.new(%(foo))
         @worker_class.expects(:threads_number).returns(num_threads)
 
+        fake_conn = Struct.new(:num)
         invocations = Array(1..num_threads).map do |i|
-          conn = OpenStruct.new(:num => i)
+          conn = fake_conn.new(i)
           conn.expects(:close)
           conn
         end
@@ -232,7 +369,7 @@ describe "Backburner::Workers::ThreadsOnFork module" do
           worker.expects(:run_while_can).with(conn).when(num_conns.is(conn.num-1)).then(num_conns.is(conn.num))
         end
 
-        def worker.create_thread(*args, &block); block.call(*args) end
+        def worker.create_thread(*args, &block); super(*args, &block).tap(&:join) end
         silenced do
           worker.prepare
           worker.fork_inner(name)
@@ -241,6 +378,22 @@ describe "Backburner::Workers::ThreadsOnFork module" do
         assert_equal(num_threads, num_conns.current_state)
       end
 
+      it "should exit once its own threads finish, even while other threads live on" do
+        worker = @worker_class.new(%(foo))
+        @worker_class.expects(:threads_number).returns(2)
+        strays, inner = [], nil
+        # Threads a job leaves running, like the one the timeout gem keeps since Ruby 3.2
+        worker.define_singleton_method(:run_while_can) { |conn| strays << Thread.new { sleep } }
+        silenced do
+          worker.prepare
+          inner = Thread.new { worker.fork_inner('demo.test.foo') }
+          assert inner.join(2), "the child still waited for threads that weren't its own"
+        end
+      ensure
+        inner.kill if inner
+        strays.each(&:kill) if strays
+      end
+
       it "should set @garbage_after, @threads_number and set retries if needed" do
         worker = @worker_class.new(%W(foo1 foo2:10 foo3:20:30 foo4:40:50:60))
         default_threads = 1
@@ -251,7 +404,7 @@ describe "Backburner::Workers::ThreadsOnFork module" do
         @worker_class.any_instance.expects(:coolest_exit).times(3)
         Backburner.configuration.max_job_retries = default_retries
 
-        worker.expects(:create_thread).times(70)
+        worker.expects(:create_thread).times(70).returns(Thread.new {})
         worker.expects(:run_while_can).once
 
         silenced do
@@ -367,6 +520,65 @@ describe "Backburner::Workers::ThreadsOnFork module" do
 
     end # cleanup on parent
 
+    describe "a child exiting" do
+      before do
+        Backburner::Worker.stubs(:gc_disabled_on_exit?).returns(true) # as on Ruby 3.3
+      end
+
+      [:return, :raise, :shutdown].each do |how|
+        it "should disable the GC before the interpreter's teardown (#{how})" do
+          worker = @worker_class.new(%W(foo))
+          gc_disabled = gc_disabled_at_child_exit?(worker, how) do |body|
+            worker.wait_for_process(worker.fork_it(&body))
+          end
+          assert gc_disabled, "the child's GC was still enabled when it exited"
+        end
+      end
+
+      it "should leave the GC enabled on a Ruby other than 3.3" do
+        Backburner::Worker.stubs(:gc_disabled_on_exit?).returns(false)
+        worker = @worker_class.new(%W(foo))
+        gc_disabled = gc_disabled_at_child_exit?(worker, :shutdown) do |body|
+          worker.wait_for_process(worker.fork_it(&body))
+        end
+        refute gc_disabled, "the child's GC was disabled when it exited"
+      end
+
+      describe "while running several threads" do
+        before do
+          @worker_class.threads_number = 2
+          @worker_class.garbage_after = nil
+        end
+
+        after do
+          @worker_class.threads_number = 1
+          @worker_class.garbage_after = 1
+        end
+
+        it "should stop its threads before its at_exit handlers run on a TERM, so they don't work jobs with the GC off" do
+          worker = @worker_class.new(%W(foo))
+          reader, writer = IO.pipe
+          pid = worker.fork_it do
+            reader.close
+            $stdout.reopen(File::NULL)
+            $stderr.reopen(File::NULL)
+            $jobs_worked = 0
+            worker.define_singleton_method(:run_while_can) { |*| loop { $jobs_worked += 1; sleep 0.01 } } # jobs keep coming
+            # An app's slow at_exit handler (an error reporter flushing, say)
+            at_exit { sleep 0.05; before = $jobs_worked; sleep 0.3; writer.write(($jobs_worked - before).to_s); writer.close }
+            Thread.new { sleep 0.3; Process.kill(:TERM, Process.pid) }
+            worker.fork_inner("demo.test.foo")
+          end
+          writer.close
+          worker.wait_for_process(pid)
+          assert_equal "0", reader.read, "the child's threads kept working jobs while its at_exit handlers ran"
+        ensure
+          reader.close if reader && !reader.closed?
+          writer.close if writer && !writer.closed?
+        end
+      end
+    end
+
     describe "practical tests" do
 
       before do
@@ -376,7 +588,7 @@ describe "Backburner::Workers::ThreadsOnFork module" do
         $worker_success = false
         $worker_raise   = false
         clear_jobs!('response')
-        clear_jobs!('foo.bar.1', 'foo.bar.2', 'foo.bar.3', 'foo.bar.4', 'foo.bar.5', 'foo.bar.6', 'foo.bar.7')
+        clear_jobs!('foo.bar.1', 'foo.bar.2', 'foo.bar.3', 'foo.bar.4', 'foo.bar.5', 'foo.bar.6', 'foo.bar.7', 'foo.bar.8')
         @worker_class.threads_number = 1
         @worker_class.garbage_after  = 10
 
@@ -391,7 +603,7 @@ describe "Backburner::Workers::ThreadsOnFork module" do
       after do
         @templogger.close
         clear_jobs!('response')
-        clear_jobs!('foo.bar.1', 'foo.bar.2', 'foo.bar.3', 'foo.bar.4', 'foo.bar.5', 'foo.bar.6', 'foo.bar.7')
+        clear_jobs!('foo.bar.1', 'foo.bar.2', 'foo.bar.3', 'foo.bar.4', 'foo.bar.5', 'foo.bar.6', 'foo.bar.7', 'foo.bar.8')
         @worker_class.threads_number = 1
         @worker_class.shutdown = true
         silenced do
@@ -479,6 +691,61 @@ describe "Backburner::Workers::ThreadsOnFork module" do
         end
         assert_equal num_jobs, $worker_test_count
       end # multithreaded
+
+      it "should give each forked tube its own connection when running one thread per tube" do
+        num_jobs = 5
+        tubes = %w(foo.bar.7 foo.bar.8)
+        @worker = @worker_class.new(tubes)
+        @worker.start(false)
+        tubes.each do |tube|
+          num_jobs.times { @worker_class.enqueue TestJobNoopFork, [], :queue => tube }
+        end
+
+        completed = lambda { @templogger.body.scan(/Completed TestJobNoopFork/).size }
+        deadline = Time.now + 10
+        sleep 0.1 until completed.call >= tubes.size * num_jobs || Time.now > deadline
+
+        assert_equal tubes.size * num_jobs, completed.call, @templogger.body
+        conn = beanstalk_connection
+        tubes.each do |tube|
+          stats = conn.tubes["demo.test.#{tube}"].stats
+          assert_equal 0, stats.current_jobs_ready + stats.current_jobs_reserved, "jobs left in #{tube}"
+        end
+        conn.close
+      end # one thread per tube
+
+      it "should release a job whose class can't be found and keep its child working" do
+        clear_jobs!('foo.bar.9')
+        conn = beanstalk_connection
+        conn.tubes['demo.test.foo.bar.9'].put(Backburner.configuration.job_serializer_proc.call(:class => 'TestRemovedJobFork', :args => []), :ttr => 30)
+        @worker = @worker_class.new('foo.bar.9')
+        @worker.start(false)
+        @worker_class.enqueue TestJobNoopFork, [], :queue => 'foo.bar.9'
+
+        silenced(5) { @templogger.wait_for_match(/Completed TestJobNoopFork/m) }
+        assert_match(/TestRemovedJobFork: class not found; released for 30s/, @templogger.body)
+        refute_match(/Catastrophic failure/, @templogger.body)
+        stats = conn.tubes['demo.test.foo.bar.9'].stats
+        assert_equal [0, 1], [stats.current_jobs_buried, stats.current_jobs_delayed]
+      ensure
+        conn.close if conn
+        clear_jobs!('foo.bar.9')
+      end # unknown class
+
+      it "should recycle a child running several threads after garbage_after jobs and keep working" do
+        clear_jobs!('foo.bar.10')
+        @worker_class.threads_number = 2
+        @worker_class.garbage_after = 2
+        @worker = @worker_class.new('foo.bar.10')
+        @worker.start(false)
+        6.times { @worker_class.enqueue TestJobNoopFork, [], :queue => 'foo.bar.10' }
+
+        completed = lambda { @templogger.body.scan(/Completed TestJobNoopFork/).size }
+        wait_for(10) { completed.call >= 6 }
+        assert_equal 6, completed.call, @templogger.body
+      ensure
+        clear_jobs!('foo.bar.10')
+      end # recycled multithreaded child
     end # practical tests
   end # forking and threading
 end # Backburner::Workers::ThreadsOnFork module

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: backburner
 version: !ruby/object:Gem::Version
-  version: 1.7.0
+  version: 1.8.0
 platform: ruby
 authors:
 - Nathan Esquenazi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-01-02 00:00:00.000000000 Z
+date: 2026-11-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: beaneater
@@ -58,6 +58,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.0.1
+- !ruby/object:Gem::Dependency
+  name: logger
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -86,6 +100,20 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 5.1.0
+- !ruby/object:Gem::Dependency
+  name: mutex_m
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: mocha
   requirement: !ruby/object:Gem::Requirement
@@ -136,8 +164,10 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/ci.yml"
+- ".github/workflows/github-release.yml"
+- ".github/workflows/release.yml"
 - ".gitignore"
-- ".travis.yml"
 - CHANGELOG.md
 - CONTRIBUTING.md
 - Gemfile
@@ -148,7 +178,6 @@ files:
 - TODO
 - backburner.gemspec
 - bin/backburner
-- circle.yml
 - examples/custom.rb
 - examples/demo.rb
 - examples/god.rb
@@ -162,6 +191,7 @@ files:
 - lib/backburner/cli.rb
 - lib/backburner/configuration.rb
 - lib/backburner/connection.rb
+- lib/backburner/enqueue_pool.rb
 - lib/backburner/helpers.rb
 - lib/backburner/hooks.rb
 - lib/backburner/job.rb
@@ -175,10 +205,19 @@ files:
 - lib/backburner/workers/simple.rb
 - lib/backburner/workers/threading.rb
 - lib/backburner/workers/threads_on_fork.rb
+- test/active_job_adapter_test.rb
 - test/async_proxy_test.rb
 - test/back_burner_test.rb
 - test/connection_test.rb
+- test/fixtures/active_job_app.rb
 - test/fixtures/active_jobs.rb
+- test/fixtures/autoload/test_broken_name_job.rb
+- test/fixtures/autoload/test_broken_no_method_job.rb
+- test/fixtures/autoload/test_broken_require_job.rb
+- test/fixtures/autoload/test_broken_runtime_job.rb
+- test/fixtures/autoload/test_broken_syntax_job.rb
+- test/fixtures/autoload/test_broken_timeout_job.rb
+- test/fixtures/autoload/test_same_name_job.rb
 - test/fixtures/hooked.rb
 - test/fixtures/test_fork_jobs.rb
 - test/fixtures/test_forking_jobs.rb
@@ -209,22 +248,31 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '2.7'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.8
+rubygems_version: 3.5.22
 signing_key:
 specification_version: 4
 summary: Reliable beanstalk background job processing made easy for Ruby and Sinatra
 test_files:
+- test/active_job_adapter_test.rb
 - test/async_proxy_test.rb
 - test/back_burner_test.rb
 - test/connection_test.rb
+- test/fixtures/active_job_app.rb
 - test/fixtures/active_jobs.rb
+- test/fixtures/autoload/test_broken_name_job.rb
+- test/fixtures/autoload/test_broken_no_method_job.rb
+- test/fixtures/autoload/test_broken_require_job.rb
+- test/fixtures/autoload/test_broken_runtime_job.rb
+- test/fixtures/autoload/test_broken_syntax_job.rb
+- test/fixtures/autoload/test_broken_timeout_job.rb
+- test/fixtures/autoload/test_same_name_job.rb
 - test/fixtures/hooked.rb
 - test/fixtures/test_fork_jobs.rb
 - test/fixtures/test_forking_jobs.rb

data/.travis.yml

@@ -1,29 +0,0 @@
-# http://about.travis-ci.org/docs/user/build-configuration/
-rvm:
-  - 1.9.3
-  - 2.0.0
-  - 2.1
-  - 2.2
-  - 2.3
-  - 2.4
-  - 2.5
-  - rbx-2
-before_install:
-  - curl -L https://github.com/kr/beanstalkd/archive/v1.9.tar.gz | tar xz -C /tmp
-  - cd /tmp/beanstalkd-1.9/
-  - make
-  - ./beanstalkd &
-  - cd $TRAVIS_BUILD_DIR
-  - gem update --system
-  - gem update bundler
-matrix:
-  allow_failures:
-    - rvm: rbx-2
-    - rvm: 2.0.0
-script:
-  - bundle exec rake test
-gemfile: Gemfile
-notifications:
-  recipients:
-    - nesquena@gmail.com
-    - therealdave.myron@gmail.com

data/circle.yml

@@ -1,3 +0,0 @@
-machine:
-  services:
-    - beanstalkd