backburner 1.8.0 → 1.9.0

data/test/workers/threads_on_fork_worker_test.rb

@@ -1,751 +0,0 @@
-require File.expand_path('../../test_helper', __FILE__)
-require File.expand_path('../../fixtures/test_fork_jobs', __FILE__)
-require File.expand_path('../../fixtures/test_queue_settings', __FILE__)
-
-describe "Backburner::Workers::ThreadsOnFork module" do
-
-  before do
-    Backburner.default_queues.clear
-    @worker_class = Backburner::Workers::ThreadsOnFork
-    @worker_class.shutdown = false
-    @worker_class.is_child = false
-    @worker_class.threads_number = 1
-    @worker_class.garbage_after = 1
-    @ignore_forks = false
-  end
-
-  after do
-    Backburner.configure { |config| config.max_job_retries = 0; config.retry_delay = 5; config.logger = nil }
-    unless @ignore_forks
-      cpids = @worker_class.instance_variable_get("@child_pids")
-      if cpids && cpids.length > 0
-        raise "Why is there forks alive?"
-      end
-    end
-  end
-
-  describe "for process_tube_names method" do
-    it "should interpreter the job_name:threads_limit:garbage_after:retries format" do
-      worker = @worker_class.new(["foo:1:2:3"])
-      assert_equal ["foo"], worker.tube_names
-    end
-
-    it "should interpreter event if is missing values" do
-      tubes = %W(foo1:1:2:3 foo2:4:5 foo3:6 foo4 foo5::7:8 foo6:::9 foo7::10)
-      worker = @worker_class.new(tubes)
-      assert_equal %W(foo1 foo2 foo3 foo4 foo5 foo6 foo7), worker.tube_names
-    end
-
-    it "should store interpreted values correctly" do
-      tubes = %W(foo1:1:2:3 foo2:4:5 foo3:6 foo4 foo5::7:8 foo6:::9 foo7::10)
-      worker = @worker_class.new(tubes)
-      assert_equal({
-        "demo.test.foo1" => { :threads => 1,   :garbage => 2,   :retries => 3   },
-        "demo.test.foo2" => { :threads => 4,   :garbage => 5,   :retries => nil },
-        "demo.test.foo3" => { :threads => 6,   :garbage => nil, :retries => nil },
-        "demo.test.foo4" => { :threads => nil, :garbage => nil, :retries => nil },
-        "demo.test.foo5" => { :threads => nil, :garbage => 7,   :retries => 8   },
-        "demo.test.foo6" => { :threads => nil, :garbage => nil, :retries => 9   },
-        "demo.test.foo7" => { :threads => nil, :garbage => 10,  :retries => nil }
-      }, worker.instance_variable_get("@tubes_data"))
-    end
-  end
-
-  describe "for process_tube_settings" do
-    it "should set the settings specified by queue name in class" do
-      worker = @worker_class.new
-      assert_equal(worker.instance_variable_get("@tubes_data")['demo.test.job-settings'],  { :threads => 5,   :garbage => 10,   :retries => 6 })
-    end
-
-    it 'should override the tube settings if they are specified directly at class level' do
-      worker = @worker_class.new
-      assert_equal(worker.instance_variable_get("@tubes_data")['demo.test.job-settings-override'], { :threads => 10,   :garbage => 1000,   :retries => 2 })
-    end
-
-    it "should take a shared tube's settings from the class that owns the tube, not from subclasses" do
-      tubes_data = @worker_class.new.instance_variable_get("@tubes_data")
-      assert_equal({ :threads => 5, :garbage => 100, :retries => 3 }, tubes_data['demo.test.tube-limits'])
-      assert_equal({ :threads => 3, :garbage => nil, :retries => nil }, tubes_data['demo.test.tube-limits-testtubelimitslambdajob'])
-      assert_equal({ :threads => 4, :garbage => nil, :retries => nil }, tubes_data['demo.test.tube-limits-testtubelimitslambdachildjob'])
-    end
-
-    it "should fill in the settings a shared tube's class leaves unset from its subclasses" do
-      tubes_data = @worker_class.new.instance_variable_get("@tubes_data")
-      assert_equal({ :threads => 3, :garbage => 50, :retries => 2 }, tubes_data['demo.test.tube-few-limits'])
-    end
-  end
-
-  describe "for prepare method" do
-    before do
-      Backburner.configure { |config| config.logger = false }
-    end
-
-    it "should make tube names array always unique to avoid duplication" do
-      worker = @worker_class.new(["foo", "demo.test.foo"])
-      capture_stdout { worker.prepare }
-      assert_equal ["demo.test.foo"], worker.tube_names
-    end
-
-    it "should watch specified tubes" do
-      worker = @worker_class.new(["foo", "bar"])
-      out = capture_stdout { worker.prepare }
-      assert_equal ["demo.test.foo", "demo.test.bar"], worker.tube_names
-      assert_match(/demo\.test\.foo/, out)
-    end # multiple
-
-    it "should watch single tube" do
-      worker = @worker_class.new("foo")
-      out = capture_stdout { worker.prepare }
-      assert_equal ["demo.test.foo"], worker.tube_names
-      assert_match(/demo\.test\.foo/, out)
-    end # single
-
-    it "should respect default_queues settings" do
-      Backburner.default_queues.concat(["foo", "bar"])
-      worker = @worker_class.new
-      out = capture_stdout { worker.prepare }
-      assert_equal ["demo.test.foo", "demo.test.bar"], worker.tube_names
-      assert_match(/demo\.test\.foo/, out)
-    end
-
-    it "should assign based on all tubes" do
-      @worker_class.any_instance.expects(:all_existing_queues).once.returns("bar")
-      worker = @worker_class.new
-      out = capture_stdout { worker.prepare }
-      assert_equal ["demo.test.bar"], worker.tube_names
-      assert_match(/demo\.test\.bar/, out)
-    end # all assign
-
-    it "should properly retrieve all tubes" do
-      worker = @worker_class.new
-      out = capture_stdout { worker.prepare }
-      assert_contains worker.tube_names, "demo.test.test-job-fork"
-      assert_match(/demo\.test\.test-job-fork/, out)
-    end # all read
-  end # prepare
-
-  describe "forking and threading" do
-
-    it "start should call fork_and_watch for each tube" do
-      worker = @worker_class.new(%W(foo bar))
-      worker.expects(:fork_and_watch).with("demo.test.foo").once
-      worker.expects(:fork_and_watch).with("demo.test.bar").once
-      silenced { worker.start(false) }
-    end
-
-    it "fork_and_watch should create a thread to fork and watch" do
-      worker = @worker_class.new(%(foo))
-      worker.expects(:create_thread).once.with("demo.test.foo")
-      silenced { worker.start(false) }
-    end
-
-    it "fork_and_watch thread should wait with wait_for_process" do
-      process_exit = stub('process_exit')
-      process_exit.expects(:exitstatus).returns(99)
-      worker = @worker_class.new(%(foo))
-      worker.expects(:wait_for_process).with(12).returns([nil, process_exit])
-
-      wc = @worker_class
-      # TODO: Is there a best way do do this?
-      worker.define_singleton_method :fork_it do
-        wc.shutdown = true
-        12
-      end
-      def worker.create_thread(*args, &block); block.call(*args) end
-
-      out = silenced(2) { worker.start(false) }
-      refute_match(/Catastrophic failure/, out)
-    end
-
-    it "fork_and_watch thread should log a normal exit if exitstatus is 0 on shutdown" do
-      process_exit = stub('process_exit', :exitstatus => 0)
-      worker = @worker_class.new(%(foo))
-      worker.expects(:wait_for_process).with(12).returns([nil, process_exit])
-
-      wc = @worker_class
-      worker.define_singleton_method :fork_it do
-        wc.shutdown = true
-        12
-      end
-      def worker.create_thread(*args, &block); block.call(*args) end
-      out = silenced(2) { worker.start(false) }
-      assert_match(/Tube demo\.test\.foo exited normally\./, out)
-      refute_match(/Catastrophic failure/, out)
-    end
-
-    describe "restarting a failed child" do
-      before do
-        @worker = @worker_class.new(%(foo))
-        def @worker.create_thread(*args, &block); block.call(*args) end
-      end
-
-      # Runs fork_and_watch over children exiting with these codes, expecting it to sleep these delays in
-      # turn, and returns what it logged
-      def expect_restart_delays(codes, delays)
-        wc = @worker_class
-        left = codes.size
-        @worker.define_singleton_method(:fork_it) { left -= 1; wc.shutdown = true if left.zero?; 12 }
-        @worker.stubs(:wait_for_process).with(12).returns(*codes.map { |code| [12, stub(:exitstatus => code)] })
-        restarts = sequence('restarts')
-        delays.each { |delay| Kernel.expects(:sleep).with(delay).in_sequence(restarts) }
-        silenced(2) { @worker.start(false) }
-      end
-
-      it "should wait longer before each fork while children keep failing, up to MAX_RESTART_DELAY" do
-        expect_restart_delays([1] * 7, [1, 2, 4, 8, 16, 30])
-      end
-
-      it "should fork again at once after a recycled child and start the delays over" do
-        expect_restart_delays([1, 1, 99, 1, 1], [1, 2, 1])
-      end
-
-      it "should log an error if exitstatus is != 99, and fork again" do
-        out = expect_restart_delays([1, 99], [1])
-        assert_match(/Catastrophic failure: tube demo\.test\.foo exited with code 1\. Forking it again in 1s\./, out)
-      end
-
-      it "should log a child that stops while shutting down without forking it again" do
-        [nil, 1].each do |code| # killed by finish_forks, or exited with an error
-          @worker_class.shutdown = false
-          out = expect_restart_delays([code], [])
-          assert_match(/Tube demo\.test\.foo stopped\./, out)
-          refute_match(/Catastrophic failure|Forking it again/, out)
-        end
-      end
-
-      it "should log the warning with a logger that has no warn method" do
-        logged = []
-        logger = Object.new
-        logger.define_singleton_method(:info) { |msg| logged << [:info, msg] }
-        logger.define_singleton_method(:error) { |msg| logged << [:error, msg] }
-        Backburner.configure { |config| config.logger = logger }
-        expect_restart_delays([0, 1], [1])
-        assert_includes logged, [:error, "Tube demo.test.foo exited with code 0. Forking it again in 1s."]
-      ensure
-        Backburner.configure { |config| config.logger = nil }
-      end
-
-      it "should warn and wait before forking again after a child exits with code 0 outside a shutdown" do
-        out = expect_restart_delays([0, 1], [1])
-        assert_match(/Tube demo\.test\.foo exited with code 0\. Forking it again in 1s\./, out)
-        refute_match(/exited normally/, out)
-      end
-
-      it "should start the delays over after a child that ran for MAX_RESTART_DELAY" do
-        @worker.stubs(:monotonic_time).returns(0, 0.5, 1, 1.5, 4, 40, 41, 41.5)
-        expect_restart_delays([1, 1, 1, 1], [1, 2, 1])
-      end
-    end # restarting a failed child
-
-    describe "when beanstalkd is down while children start" do
-      before do
-        pick_beanstalkd_port
-        start_beanstalkd
-        @templogger = Templogger.new('/tmp')
-        @beanstalk_url = Backburner.configuration.beanstalk_url
-        Backburner.configure do |config|
-          config.beanstalk_url = "beanstalk://127.0.0.1:#{@port}"
-          config.logger = @templogger.logger
-        end
-        @worker_class.garbage_after = 1
-        @watchers = []
-        @ignore_forks = true
-      end
-
-      after do
-        begin
-          @worker_class.shutdown = true
-          silenced do
-            @worker_class.stop_forks
-            Timeout::timeout(2) { sleep 0.1 while @worker_class.child_pids.length > 0 }
-            @worker_class.kill_forks
-            Timeout::timeout(2) { sleep 0.1 while @worker_class.child_pids.length > 0 }
-          end
-        ensure
-          @watchers.each(&:kill) # once they've reaped the children: one waiting to fork again would wake up in a later test
-          stop_beanstalkd
-          Backburner.configure do |config|
-            config.beanstalk_url = @beanstalk_url
-            config.logger = nil
-          end
-          @templogger.close
-        end
-      end
-
-      [1, 2].each do |threads|
-        it "should back off instead of forking in a busy loop, and resume when it is back (#{threads} thread(s) per child)" do
-          @worker_class.threads_number = threads
-          worker = @worker_class.new('foo.bar.outage')
-          forks = 0
-          watchers = @watchers
-          worker.define_singleton_method(:fork_tube) { |name| forks += 1; super(name) }
-          worker.define_singleton_method(:create_thread) { |*args, &block| super(*args, &block).tap { |thread| watchers << thread } }
-          outage_forks = nil
-          silenced(20) do
-            worker.start(false)
-            sleep 0.5 # let the first child block in reserve
-            stop_beanstalkd
-            forks_before = forks
-            sleep 4
-            outage_forks = forks - forks_before
-            start_beanstalkd
-            @worker_class.enqueue TestJobNoopFork, [], :queue => 'foo.bar.outage'
-            wait_for(10) { @templogger.body =~ /Completed TestJobNoopFork/ }
-          end
-          assert_operator outage_forks, :<=, 4, "forked #{outage_forks} children in 4 s"
-          assert_match(/Completed TestJobNoopFork/, @templogger.body)
-        end
-      end
-    end # beanstalkd down
-
-    describe "fork_inner" do
-
-      before do
-        @worker_class.any_instance.expects(:coolest_exit).once
-      end
-
-      it "should watch just the channel it receive as argument" do
-        worker = @worker_class.new(%(foo))
-        @worker_class.expects(:threads_number).returns(1)
-        worker.expects(:run_while_can).once
-        silenced do
-          worker.prepare
-          worker.fork_inner('demo.test.bar')
-        end
-        assert_same_elements %W(demo.test.bar), worker.connection.tubes.watched.map(&:name)
-      end
-
-      it "should not create threads if the number of threads is 1" do
-        worker = @worker_class.new(%(foo))
-        @worker_class.expects(:threads_number).returns(1)
-        worker.expects(:run_while_can).once
-        worker.expects(:create_thread).never
-        silenced do
-          worker.prepare
-          worker.fork_inner('demo.test.foo')
-        end
-      end
-
-      it "should create threads if the number of threads is > 1" do
-        worker = @worker_class.new(%(foo))
-        @worker_class.expects(:threads_number).returns(5)
-        worker.expects(:create_thread).times(5).returns(Thread.new {})
-        silenced do
-          worker.prepare
-          worker.fork_inner('demo.test.foo')
-        end
-      end
-
-      it "should create threads that call run_while_can" do
-        worker = @worker_class.new(%(foo))
-        @worker_class.expects(:threads_number).returns(5)
-        worker.expects(:run_while_can).times(5)
-        def worker.create_thread(*args, &block); super(*args, &block).tap(&:join) end
-        silenced do
-          worker.prepare
-          worker.fork_inner('demo.test.foo')
-        end
-      end
-
-      it "should create a connection for each thread" do
-        name = 'demo.test.foo'
-        num_threads = 3
-
-        worker = @worker_class.new(%(foo))
-        @worker_class.expects(:threads_number).returns(num_threads)
-
-        fake_conn = Struct.new(:num)
-        invocations = Array(1..num_threads).map do |i|
-          conn = fake_conn.new(i)
-          conn.expects(:close)
-          conn
-        end
-        Backburner::Connection.expects(:new).times(num_threads).returns(*invocations)
-
-        # ensure each invocation of run_while_can is with a different connection
-        num_conns = states('num_conns').starts_as(0)
-        invocations.each do |conn|
-          worker.expects(:watch_tube).with(name, conn)
-          worker.expects(:run_while_can).with(conn).when(num_conns.is(conn.num-1)).then(num_conns.is(conn.num))
-        end
-
-        def worker.create_thread(*args, &block); super(*args, &block).tap(&:join) end
-        silenced do
-          worker.prepare
-          worker.fork_inner(name)
-        end
-
-        assert_equal(num_threads, num_conns.current_state)
-      end
-
-      it "should exit once its own threads finish, even while other threads live on" do
-        worker = @worker_class.new(%(foo))
-        @worker_class.expects(:threads_number).returns(2)
-        strays, inner = [], nil
-        # Threads a job leaves running, like the one the timeout gem keeps since Ruby 3.2
-        worker.define_singleton_method(:run_while_can) { |conn| strays << Thread.new { sleep } }
-        silenced do
-          worker.prepare
-          inner = Thread.new { worker.fork_inner('demo.test.foo') }
-          assert inner.join(2), "the child still waited for threads that weren't its own"
-        end
-      ensure
-        inner.kill if inner
-        strays.each(&:kill) if strays
-      end
-
-      it "should set @garbage_after, @threads_number and set retries if needed" do
-        worker = @worker_class.new(%W(foo1 foo2:10 foo3:20:30 foo4:40:50:60))
-        default_threads = 1
-        default_garbage = 5
-        default_retries = 100
-        @worker_class.expects(:threads_number).times(1).returns(default_threads)
-        @worker_class.expects(:garbage_after).times(2).returns(default_garbage)
-        @worker_class.any_instance.expects(:coolest_exit).times(3)
-        Backburner.configuration.max_job_retries = default_retries
-
-        worker.expects(:create_thread).times(70).returns(Thread.new {})
-        worker.expects(:run_while_can).once
-
-        silenced do
-          worker.prepare
-          worker.fork_inner('demo.test.foo1')
-        end
-
-        assert_equal worker.instance_variable_get("@threads_number"), default_threads
-        assert_equal worker.instance_variable_get("@garbage_after"), default_garbage
-        assert_equal Backburner.configuration.max_job_retries, default_retries
-
-        silenced do
-          worker.fork_inner('demo.test.foo2')
-        end
-
-        assert_equal worker.instance_variable_get("@threads_number"), 10
-        assert_equal worker.instance_variable_get("@garbage_after"), default_garbage
-        assert_equal Backburner.configuration.max_job_retries, default_retries
-
-        silenced do
-          worker.fork_inner('demo.test.foo3')
-        end
-
-        assert_equal worker.instance_variable_get("@threads_number"), 20
-        assert_equal worker.instance_variable_get("@garbage_after"), 30
-        assert_equal Backburner.configuration.max_job_retries, default_retries
-
-        silenced do
-          worker.fork_inner('demo.test.foo4')
-        end
-
-        assert_equal worker.instance_variable_get("@threads_number"), 40
-        assert_equal worker.instance_variable_get("@garbage_after"), 50
-        assert_equal Backburner.configuration.max_job_retries, 60
-      end
-
-    end
-
-    describe "cleanup on parent" do
-
-      it "child_pids should return a list of alive children pids" do
-        worker = @worker_class.new(%W(foo))
-        Kernel.expects(:fork).once.returns(12345)
-        Process.expects(:kill).with(0, 12345).once
-        Process.expects(:pid).once.returns(12346)
-        assert_equal [], @worker_class.child_pids
-        worker.fork_it {}
-        child_pids = @worker_class.child_pids
-        assert_equal [12345], child_pids
-        child_pids.clear
-      end
-
-      it "child_pids should return an empty array if is_child" do
-        Process.expects(:pid).never
-        @worker_class.is_child = true
-        @worker_class.child_pids << 12345
-        assert_equal [], @worker_class.child_pids
-      end
-
-      it "stop_forks should send a SIGTERM for every child" do
-        Process.expects(:pid).returns(12346).at_least(1)
-        Process.expects(:kill).with(0, 12345).at_least(1)
-        Process.expects(:kill).with(0, 12347).at_least(1)
-        Process.expects(:kill).with("SIGTERM", 12345)
-        Process.expects(:kill).with("SIGTERM", 12347)
-        @worker_class.child_pids << 12345
-        @worker_class.child_pids << 12347
-        assert_equal [12345, 12347], @worker_class.child_pids
-        @worker_class.stop_forks
-        @worker_class.child_pids.clear
-      end
-
-      it "kill_forks should send a SIGKILL for every child" do
-        Process.expects(:pid).returns(12346).at_least(1)
-        Process.expects(:kill).with(0, 12345).at_least(1)
-        Process.expects(:kill).with(0, 12347).at_least(1)
-        Process.expects(:kill).with("SIGKILL", 12345)
-        Process.expects(:kill).with("SIGKILL", 12347)
-        @worker_class.child_pids << 12345
-        @worker_class.child_pids << 12347
-        assert_equal [12345, 12347], @worker_class.child_pids
-        @worker_class.kill_forks
-        @worker_class.child_pids.clear
-      end
-
-      it "finish_forks should call stop_forks, kill_forks and Process.waitall" do
-        Process.expects(:pid).returns(12346).at_least(1)
-        Process.expects(:kill).with(0, 12345).at_least(1)
-        Process.expects(:kill).with(0, 12347).at_least(1)
-        Process.expects(:kill).with("SIGTERM", 12345)
-        Process.expects(:kill).with("SIGTERM", 12347)
-        Process.expects(:kill).with("SIGKILL", 12345)
-        Process.expects(:kill).with("SIGKILL", 12347)
-        Kernel.expects(:sleep).with(1)
-        Process.expects(:waitall)
-        @worker_class.child_pids << 12345
-        @worker_class.child_pids << 12347
-        assert_equal [12345, 12347], @worker_class.child_pids
-        silenced do
-          @worker_class.finish_forks
-        end
-        @worker_class.child_pids.clear
-      end
-
-      it "finish_forks should not do anything if is_child" do
-        @worker_class.expects(:stop_forks).never
-        @worker_class.is_child = true
-        @worker_class.child_pids << 12345
-        silenced do
-          @worker_class.finish_forks
-        end
-      end
-
-    end # cleanup on parent
-
-    describe "a child exiting" do
-      before do
-        Backburner::Worker.stubs(:gc_disabled_on_exit?).returns(true) # as on Ruby 3.3
-      end
-
-      [:return, :raise, :shutdown].each do |how|
-        it "should disable the GC before the interpreter's teardown (#{how})" do
-          worker = @worker_class.new(%W(foo))
-          gc_disabled = gc_disabled_at_child_exit?(worker, how) do |body|
-            worker.wait_for_process(worker.fork_it(&body))
-          end
-          assert gc_disabled, "the child's GC was still enabled when it exited"
-        end
-      end
-
-      it "should leave the GC enabled on a Ruby other than 3.3" do
-        Backburner::Worker.stubs(:gc_disabled_on_exit?).returns(false)
-        worker = @worker_class.new(%W(foo))
-        gc_disabled = gc_disabled_at_child_exit?(worker, :shutdown) do |body|
-          worker.wait_for_process(worker.fork_it(&body))
-        end
-        refute gc_disabled, "the child's GC was disabled when it exited"
-      end
-
-      describe "while running several threads" do
-        before do
-          @worker_class.threads_number = 2
-          @worker_class.garbage_after = nil
-        end
-
-        after do
-          @worker_class.threads_number = 1
-          @worker_class.garbage_after = 1
-        end
-
-        it "should stop its threads before its at_exit handlers run on a TERM, so they don't work jobs with the GC off" do
-          worker = @worker_class.new(%W(foo))
-          reader, writer = IO.pipe
-          pid = worker.fork_it do
-            reader.close
-            $stdout.reopen(File::NULL)
-            $stderr.reopen(File::NULL)
-            $jobs_worked = 0
-            worker.define_singleton_method(:run_while_can) { |*| loop { $jobs_worked += 1; sleep 0.01 } } # jobs keep coming
-            # An app's slow at_exit handler (an error reporter flushing, say)
-            at_exit { sleep 0.05; before = $jobs_worked; sleep 0.3; writer.write(($jobs_worked - before).to_s); writer.close }
-            Thread.new { sleep 0.3; Process.kill(:TERM, Process.pid) }
-            worker.fork_inner("demo.test.foo")
-          end
-          writer.close
-          worker.wait_for_process(pid)
-          assert_equal "0", reader.read, "the child's threads kept working jobs while its at_exit handlers ran"
-        ensure
-          reader.close if reader && !reader.closed?
-          writer.close if writer && !writer.closed?
-        end
-      end
-    end
-
-    describe "practical tests" do
-
-      before do
-        @templogger = Templogger.new('/tmp')
-        Backburner.configure { |config| config.logger = @templogger.logger }
-        $worker_test_count = 0
-        $worker_success = false
-        $worker_raise   = false
-        clear_jobs!('response')
-        clear_jobs!('foo.bar.1', 'foo.bar.2', 'foo.bar.3', 'foo.bar.4', 'foo.bar.5', 'foo.bar.6', 'foo.bar.7', 'foo.bar.8')
-        @worker_class.threads_number = 1
-        @worker_class.garbage_after  = 10
-
-        silenced do
-          @response_worker = @worker_class.new('response')
-          @response_worker.watch_tube('demo.test.response')
-        end
-
-        @ignore_forks = true
-      end
-
-      after do
-        @templogger.close
-        clear_jobs!('response')
-        clear_jobs!('foo.bar.1', 'foo.bar.2', 'foo.bar.3', 'foo.bar.4', 'foo.bar.5', 'foo.bar.6', 'foo.bar.7', 'foo.bar.8')
-        @worker_class.threads_number = 1
-        @worker_class.shutdown = true
-        silenced do
-          @worker_class.stop_forks
-          Timeout::timeout(2) { sleep 0.1 while @worker_class.child_pids.length > 0 }
-          @worker_class.kill_forks
-          Timeout::timeout(2) { sleep 0.1 while @worker_class.child_pids.length > 0 }
-        end
-      end
-
-      it "should work an enqueued job" do
-        @worker = @worker_class.new('foo.bar.1')
-        @worker.start(false)
-        @worker_class.enqueue TestJobFork, [1, 2], :queue => "foo.bar.1"
-
-        silenced do
-          @templogger.wait_for_match(/Completed TestJobFork/m)
-          @response_worker.work_one_job
-        end
-        assert_equal 3, $worker_test_count
-      end # enqueue
-
-      it "should work for an async job" do
-        @worker = @worker_class.new('foo.bar.2')
-        @worker.start(false)
-        TestAsyncJobFork.async(:queue => 'foo.bar.2').foo(3, 5)
-        silenced(2) do
-          @templogger.wait_for_match(/Completed TestAsyncJobFork/m)
-          @response_worker.work_one_job
-        end
-        assert_equal 15, $worker_test_count
-      end # async
-
-      it "should fail quietly if there's an argument error" do
-        @worker = @worker_class.new('foo.bar.3')
-        @worker.start(false)
-        @worker_class.enqueue TestJobFork, ["bam", "foo", "bar"], :queue => "foo.bar.3"
-        silenced(5) do
-          @templogger.wait_for_match(/Finished TestJobFork.*attempt 1 of 1/m)
-        end
-        assert_match(/Exception ArgumentError/, @templogger.body)
-        assert_equal 0, $worker_test_count
-      end # fail, argument
-
-      it "should support retrying jobs and burying" do
-        Backburner.configure { |config| config.max_job_retries = 1; config.retry_delay = 0 }
-        @worker = @worker_class.new('foo.bar.4')
-        @worker.start(false)
-        @worker_class.enqueue TestRetryJobFork, ["bam", "foo"], :queue => 'foo.bar.4'
-        silenced(2) do
-          @templogger.wait_for_match(/Finished TestRetryJobFork.*attempt 2 of 2/m)
-          2.times { @response_worker.work_one_job }
-        end
-        assert_equal 2, $worker_test_count
-        assert_equal false, $worker_success
-      end # retry, bury
-
-      it "should support retrying jobs and succeeds" do
-        Backburner.configure { |config| config.max_job_retries = 2; config.retry_delay = 0 }
-        @worker = @worker_class.new('foo.bar.5')
-        @worker.start(false)
-        @worker_class.enqueue TestRetryJobFork, ["bam", "foo"], :queue => 'foo.bar.5'
-        silenced(2) do
-          @templogger.wait_for_match(/Completed TestRetryJobFork/m)
-          3.times { @response_worker.work_one_job }
-        end
-        assert_equal 3, $worker_test_count
-        assert_equal true, $worker_success
-      end # retrying, succeeds
-
-      it "should support a multithreaded worker without deadlocks" do
-        num_threads = 15
-        num_jobs = 8
-        num_jobs.times do |i|
-          @worker_class.enqueue TestJobMultithreadFork, [6,2], :queue => 'foo.bar.6'
-        end
-
-        @worker_class.threads_number = num_threads
-        @worker = @worker_class.new('foo.bar.6')
-        @worker.start(false)
-
-        silenced do
-          @templogger.wait_for_match(/Completed TestJobMultithreadFork/m)
-          num_jobs.times { @response_worker.work_one_job }
-        end
-        assert_equal num_jobs, $worker_test_count
-      end # multithreaded
-
-      it "should give each forked tube its own connection when running one thread per tube" do
-        num_jobs = 5
-        tubes = %w(foo.bar.7 foo.bar.8)
-        @worker = @worker_class.new(tubes)
-        @worker.start(false)
-        tubes.each do |tube|
-          num_jobs.times { @worker_class.enqueue TestJobNoopFork, [], :queue => tube }
-        end
-
-        completed = lambda { @templogger.body.scan(/Completed TestJobNoopFork/).size }
-        deadline = Time.now + 10
-        sleep 0.1 until completed.call >= tubes.size * num_jobs || Time.now > deadline
-
-        assert_equal tubes.size * num_jobs, completed.call, @templogger.body
-        conn = beanstalk_connection
-        tubes.each do |tube|
-          stats = conn.tubes["demo.test.#{tube}"].stats
-          assert_equal 0, stats.current_jobs_ready + stats.current_jobs_reserved, "jobs left in #{tube}"
-        end
-        conn.close
-      end # one thread per tube
-
-      it "should release a job whose class can't be found and keep its child working" do
-        clear_jobs!('foo.bar.9')
-        conn = beanstalk_connection
-        conn.tubes['demo.test.foo.bar.9'].put(Backburner.configuration.job_serializer_proc.call(:class => 'TestRemovedJobFork', :args => []), :ttr => 30)
-        @worker = @worker_class.new('foo.bar.9')
-        @worker.start(false)
-        @worker_class.enqueue TestJobNoopFork, [], :queue => 'foo.bar.9'
-
-        silenced(5) { @templogger.wait_for_match(/Completed TestJobNoopFork/m) }
-        assert_match(/TestRemovedJobFork: class not found; released for 30s/, @templogger.body)
-        refute_match(/Catastrophic failure/, @templogger.body)
-        stats = conn.tubes['demo.test.foo.bar.9'].stats
-        assert_equal [0, 1], [stats.current_jobs_buried, stats.current_jobs_delayed]
-      ensure
-        conn.close if conn
-        clear_jobs!('foo.bar.9')
-      end # unknown class
-
-      it "should recycle a child running several threads after garbage_after jobs and keep working" do
-        clear_jobs!('foo.bar.10')
-        @worker_class.threads_number = 2
-        @worker_class.garbage_after = 2
-        @worker = @worker_class.new('foo.bar.10')
-        @worker.start(false)
-        6.times { @worker_class.enqueue TestJobNoopFork, [], :queue => 'foo.bar.10' }
-
-        completed = lambda { @templogger.body.scan(/Completed TestJobNoopFork/).size }
-        wait_for(10) { completed.call >= 6 }
-        assert_equal 6, completed.call, @templogger.body
-      ensure
-        clear_jobs!('foo.bar.10')
-      end # recycled multithreaded child
-    end # practical tests
-  end # forking and threading
-end # Backburner::Workers::ThreadsOnFork module