backburner 1.7.0 → 1.8.0

data/lib/backburner/worker.rb

@@ -9,6 +9,10 @@ module Backburner
     include Backburner::Helpers
     include Backburner::Logger
 
+    # The longest a worker that doesn't have a job's class waits to hand the job out again (its TTR
+    # otherwise), so a worker that has it picks it up soon after a rollout
+    MAX_CLASS_NOT_FOUND_DELAY = 60
+
     # Backburner::Worker.known_queue_classes
     # List of known_queue_classes
     class << self
@@ -17,7 +21,8 @@ module Backburner
     end
 
     # Enqueues a job to be processed later by a worker.
-    # Options: `pri` (priority), `delay` (delay in secs), `ttr` (time to respond), `queue` (queue name)
+    # Options: `pri` (priority), `delay` (delay in secs, rounded up to whole seconds), `ttr` (time to respond),
+    # `queue` (queue name)
     #
     # @raise [Beaneater::NotConnected] If beanstalk fails to connect.
     # @example
@@ -25,7 +30,7 @@ module Backburner
     #
     def self.enqueue(job_class, args=[], opts={})
       pri   = resolve_priority(opts[:pri] || job_class)
-      delay = [0, opts[:delay].to_i].max
+      delay = whole_seconds(opts[:delay])
       ttr   = resolve_respond_timeout(opts[:ttr] || job_class)
       res   = Backburner::Hooks.invoke_hook_events(job_class, :before_enqueue, *args)
 
@@ -34,18 +39,15 @@ module Backburner
       data = { :class => job_class.name, :args => args }
       queue = opts[:queue] && (Proc === opts[:queue] ? opts[:queue].call(job_class) : opts[:queue])
 
-      begin
-        response = nil
-        connection = Backburner::Connection.new(Backburner.configuration.beanstalk_url)
+      response = nil
+      Backburner::EnqueuePool.current.with_connection do |connection|
         connection.retryable do
           tube = connection.tubes[expand_tube_name(queue || job_class)]
           serialized_data = Backburner.configuration.job_serializer_proc.call(data)
           response = tube.put(serialized_data, :pri => pri, :delay => delay, :ttr => ttr)
         end
-        return nil unless Backburner::Hooks.invoke_hook_events(job_class, :after_enqueue, *args)
-      ensure
-        connection.close if connection
       end
+      return nil unless Backburner::Hooks.invoke_hook_events(job_class, :after_enqueue, *args)
 
       response
     end
@@ -149,17 +151,45 @@ module Backburner
 
       # NB: There's a slight chance here that the connection to beanstalkd has
       # gone down between the time we reserved / processed the job and here.
-      num_retries = job.stats.releases
-      max_job_retries = resolve_max_job_retries(job.job_class)
-      retry_status = "failed: attempt #{num_retries+1} of #{max_job_retries+1}"
-      if num_retries < max_job_retries # retry again
-        retry_delay = resolve_retry_delay(job.job_class)
-        delay = resolve_retry_delay_proc(job.job_class).call(retry_delay, num_retries) rescue retry_delay
-        job.retry(num_retries + 1, delay)
-        self.log_job_end(job.name, "#{retry_status}, retrying in #{delay}s") if job_started_at
-      else # retries failed, bury
-        job.bury
-        self.log_job_end(job.name, "#{retry_status}, burying") if job_started_at
+      class_missing = false
+      begin
+        job_class = begin
+          job.job_class
+        rescue Backburner::Job::JobNotFound # missing, or its file failed to load: see below
+          nil
+        end
+        if job_class.nil? && job.job_class_missing?
+          # This worker doesn't have its class (renamed or removed, or a worker deployed before the code
+          # that enqueues it): give it back for a worker that has it rather than burying it, once per TTR
+          # (at most a minute) as beanstalkd would, without on_error or the failure hooks (as before) or
+          # using up its retries. A class whose file fails to load goes the normal failure path instead.
+          # ceiling: beanstalkd still counts the release, so once a worker has the class, a failure there
+          # gets fewer retries; counting these releases apart would need a marker in the job body.
+          class_missing = true
+          delay = [job.ttr || queue_config.respond_timeout, MAX_CLASS_NOT_FOUND_DELAY].min
+          job.task.release(:delay => delay)
+          self.log_warn "#{job.name}: class not found; released for #{delay}s for a worker that has it"
+          return
+        end
+        num_retries = job.stats.releases
+        max_job_retries = resolve_max_job_retries(job_class)
+        retry_status = "failed: attempt #{num_retries+1} of #{max_job_retries+1}"
+        if num_retries < max_job_retries # retry again
+          retry_delay = resolve_retry_delay(job_class)
+          delay = resolve_retry_delay_proc(job_class).call(retry_delay, num_retries) rescue retry_delay
+          delay = whole_seconds(delay)
+          job.retry(num_retries + 1, delay)
+          self.log_job_end(job.name, "#{retry_status}, retrying in #{delay}s") if job_started_at
+        else # retries failed, bury
+          job.bury
+          self.log_job_end(job.name, "#{retry_status}, burying") if job_started_at
+        end
+      rescue Beaneater::NotFoundError, Beaneater::NotConnected
+        # Its TTR ran out first, or the connection was lost: beanstalkd releases a reserved job when
+        # its TTR runs out or its client disconnects (unless the server restarted and lost it).
+        # (An error raised in a rescue clause isn't caught by the method's other rescues, hence this one.)
+        self.log_error "#{job.name} was no longer reserved (its TTR ran out, or the connection was lost), so it was neither retried nor buried"
+        return if class_missing # on_error isn't called for a class this worker lacks
       end
 
       handle_error(e, job.name, job.args, job)
@@ -181,11 +211,49 @@ module Backburner
     # Returns a list of all tubes known within the system
     # Filtered for tubes that match the known prefix
     def all_existing_queues
-      known_queues    = Backburner::Worker.known_queue_classes.map(&:queue)
-      existing_tubes  = self.connection.tubes.all.map(&:name).select { |tube| tube =~ /^#{queue_config.tube_namespace}/ }
+      known_queues    = Backburner::Worker.known_queue_classes.select(&:name).map(&:queue) # an anonymous class can't be a job
+      prefix          = expanded_tube_prefix # the start of every tube name enqueueing with this configuration gives
+      namespace       = queue_config.tube_namespace
+      cut             = namespace && "#{namespace}#{queue_config.namespace_separator}".include?(':') # every name is the prefix
+      existing_tubes  = self.connection.tubes.all.map(&:name).select do |tube| # only the names this configuration can give
+        cut ? tube == prefix : tube.start_with?(prefix) && tube[prefix.size..-1] !~ /[A-Z_]/ # the rest is dasherized
+      end
+      # ceiling: a server tube named exactly like a known queue name is still expanded like that name, as before;
+      # the workers' prepare keeping found tubes apart from queue names would lift it
+      @existing_tube_names = existing_tubes - known_queues - [queue_config.primary_queue]
       existing_tubes + known_queues + [queue_config.primary_queue]
     end
 
+    # Leaves the server's tube names all_existing_queues found as they are: expanding one again can change it
+    # ("MyApp.default" would become "MyApp.my-app.default")
+    def expand_tube_name(tube)
+      @existing_tube_names && @existing_tube_names.include?(tube) ? tube : super
+    end
+    public :expand_tube_name
+
+
+    # Merges the options the block gives for each known queue class (its queue_jobs_limit and so on)
+    # into its tube's entry in tubes_data, for tubes that are there. A class that declares its tube
+    # overrides the tube's options; a subclass sharing its parent's tube only fills in the ones still
+    # unset. Anonymous classes are skipped.
+    def merge_tube_settings(tubes_data)
+      owners, sharers = Backburner::Worker.known_queue_classes.select(&:name).partition { |queue| !shares_parent_tube?(queue) }
+      [[owners, true], [sharers, false]].each do |classes, override|
+        classes.each do |queue|
+          data = tubes_data[expand_tube_name(queue)]
+          next if data.nil?
+          data.merge!(yield(queue)) { |_key, current, setting| (override ? setting.nil? : !current.nil?) ? current : setting }
+        end
+      end
+    end
+
+    # Whether a queue class uses the same tube as its closest named Backburner::Queue ancestor
+    def shares_parent_tube?(queue)
+      return false unless queue.respond_to?(:superclass)
+      parent = queue.superclass
+      parent = parent.superclass while parent.is_a?(Backburner::Queue::ClassMethods) && parent.name.nil?
+      parent.is_a?(Backburner::Queue::ClassMethods) && expand_tube_name(parent) == expand_tube_name(queue)
+    end
 
     # Handles an error according to custom definition
     # Used when processing a job that errors out
@@ -199,6 +267,8 @@ module Backburner
           error_handler.call(e, name, args, job)
         end
       end
+    rescue => handler_error # an error reporter that fails (can't reach its service, say) mustn't stop the worker
+      log_error "The on_error handler failed for #{name}: #{exception_message(handler_error)}"
     end
 
     # Normalizes tube names given array of tube_names
@@ -217,5 +287,26 @@ module Backburner
       trap('TERM') { shutdown  }
       trap('INT')  { shutdown  }
     end
+
+    # Whether a forked child disables the GC as it exits (see with_gc_disabled_on_exit): only on CRuby 3.3
+    def self.gc_disabled_on_exit?(engine = RUBY_ENGINE, version = RUBY_VERSION)
+      engine == "ruby" && version.start_with?("3.3.")
+    end
+
+    # Runs the body of a forked child, then, on Ruby 3.3, disables the GC before the child exits by any
+    # path that tears the interpreter down: returning, an exception, or Kernel.exit (as the TERM and INT
+    # traps do). Kernel.exit! skips the ensure, and the teardown with it. The child's at_exit handlers
+    # (its own and those it inherited) then run with the GC off.
+    # ceiling: works around a Ruby 3.3 bug: a child forked while another thread of its parent runs
+    # finalizers can crash as it exits ("[BUG] lazy sweeping underway when freeing object space").
+    # GC.disable finishes the lazy sweep first. Fixed in Ruby 3.4 by ruby/ruby 868d63f0a3 (not
+    # backported to 3.3, and not seen on 3.1 or 3.2); this can go once Ruby 3.3 support is dropped.
+    # Limits: an at_exit handler that calls GC.enable itself can still crash, and a TERM that lands while
+    # a ThreadsOnFork child is starting its job threads can leave one of them running during at_exit.
+    def with_gc_disabled_on_exit
+      yield
+    ensure
+      GC.disable if Worker.gc_disabled_on_exit?
+    end
   end # Worker
 end # Backburner

data/lib/backburner/workers/forking.rb

@@ -29,8 +29,10 @@ module Backburner
       # Waits for a job, works the job, and exits
       def fork_one_job
         pid = Process.fork do
-          work_one_job
-          coolest_exit
+          with_gc_disabled_on_exit do
+            work_one_job
+            coolest_exit
+          end
         end
         Process.wait(pid)
       end

data/lib/backburner/workers/simple.rb

@@ -24,6 +24,11 @@ module Backburner
         prepare
         loop { work_one_job }
       end
+
+      # A reconnected connection only watches the default tube: watch ours again
+      def on_reconnect(conn)
+        conn.tubes.watch!(*self.tube_names)
+      end
     end # Basic
   end # Workers
 end # Backburner

data/lib/backburner/workers/threading.rb

@@ -108,13 +108,7 @@ module Backburner
       # If the tube is known by existing beanstalkd queue, but not by class - skip it
       #
       def process_tube_options
-        Backburner::Worker.known_queue_classes.each do |queue|
-          next if @tubes_data[expand_tube_name(queue)].nil?
-          queue_settings = {
-            :threads => queue.queue_jobs_limit
-          }
-          @tubes_data[expand_tube_name(queue)].merge!(queue_settings){|k, v1, v2| v2.nil? ? v1 : v2 }
-        end
+        merge_tube_settings(@tubes_data) { |queue| { :threads => queue.queue_jobs_limit } }
       end
 
       # Wait for the shutdown signel

data/lib/backburner/workers/threads_on_fork.rb

@@ -108,14 +108,8 @@ module Backburner
       # If the tube is known by existing beanstalkd queue, but not by class - skip it
       #
       def process_tube_options
-        Backburner::Worker.known_queue_classes.each do |queue|
-          next if @tubes_data[expand_tube_name(queue)].nil?
-          queue_settings = {
-              :threads => queue.queue_jobs_limit,
-              :garbage => queue.queue_garbage_limit,
-              :retries => queue.queue_retry_limit
-          }
-          @tubes_data[expand_tube_name(queue)].merge!(queue_settings){|k, v1, v2| v2.nil? ? v1 : v2 }
+        merge_tube_settings(@tubes_data) do |queue|
+          { :threads => queue.queue_jobs_limit, :garbage => queue.queue_garbage_limit, :retries => queue.queue_retry_limit }
         end
       end
 
@@ -140,18 +134,37 @@ module Backburner
         end
       end
 
+      # After a child fails, the parent waits before forking the next one: MIN_RESTART_DELAY seconds,
+      # doubling while children keep failing (beanstalkd is down, say) up to MAX_RESTART_DELAY, and
+      # starting over once a child is recycled (99) or runs for MAX_RESTART_DELAY seconds
+      MIN_RESTART_DELAY = 1
+      MAX_RESTART_DELAY = 30
+
       # Make the fork and create a thread to watch the child process
       # The exit code '99' means that the fork exited because of the garbage limit
       # Any other code is an error
       def fork_and_watch(name)
         create_thread(name) do |tube_name|
+          restart_delay = 0
           until self.class.shutdown
+            forked_at = monotonic_time
             pid = fork_tube(tube_name)
             _, status = wait_for_process(pid)
 
             # 99 = garbaged
-            if status.exitstatus != 99
-              log_error("Catastrophic failure: tube #{tube_name} exited with code #{status.exitstatus}.")
+            if status.exitstatus == 99
+              restart_delay = 0
+            elsif self.class.shutdown # stopped by SIGTERM or SIGINT, or killed by finish_forks: not forked again
+              log_info(status.exitstatus == 0 ? "Tube #{tube_name} exited normally." : "Tube #{tube_name} stopped.")
+            else
+              ran_for = monotonic_time - forked_at
+              restart_delay = ran_for >= MAX_RESTART_DELAY ? MIN_RESTART_DELAY : (restart_delay * 2).clamp(MIN_RESTART_DELAY, MAX_RESTART_DELAY)
+              if status.exitstatus == 0 # stopped by a signal sent to the child alone, or a job called exit
+                log_warn("Tube #{tube_name} exited with code 0. Forking it again in #{restart_delay}s.")
+              else
+                log_error("Catastrophic failure: tube #{tube_name} exited with code #{status.exitstatus}. Forking it again in #{restart_delay}s.")
+              end
+              Kernel.sleep(restart_delay)
             end
           end
         end
@@ -182,22 +195,37 @@ module Backburner
         @runs = 0
 
         if @threads_number == 1
+          # Each forked child needs its own socket: the parent's connection is shared by every fork
+          self.connection = new_connection
           watch_tube(name)
           run_while_can
         else
-          threads_count = Thread.list.count
-          @threads_number.times do
-            create_thread do
-              begin
-                conn = new_connection
-                watch_tube(name, conn)
-                run_while_can(conn)
-              ensure
-                conn.close if conn
+          failed = false
+          threads = []
+          begin
+            @threads_number.times do
+              threads << create_thread do
+                begin
+                  conn = new_connection
+                  watch_tube(name, conn)
+                  run_while_can(conn)
+                rescue Exception
+                  failed = true
+                  raise
+                ensure
+                  conn.close if conn
+                end
               end
             end
+            # Wait for these threads only: others can outlive them (the timeout gem's, since Ruby 3.2)
+            sleep 0.1 while threads.any?(&:alive?)
+          rescue Exception # TERM, INT or an error: on Ruby 3.3 the child runs its at_exit handlers with the GC
+            # off (see with_gc_disabled_on_exit), so its threads mustn't keep working jobs meanwhile
+            threads.each(&:kill) if Worker.gc_disabled_on_exit?
+            raise
           end
-          sleep 0.1 while Thread.list.count > threads_count
+          # Fail like a single-threaded child whose connection raised, so the parent backs off
+          Kernel.exit!(1) if failed
         end
 
         coolest_exit
@@ -233,6 +261,11 @@ module Backburner
         Thread.new(*args, &block)
       end
 
+      # Seconds on a clock that only moves forward. Easy to test
+      def monotonic_time
+        Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      end
+
       # Wait for a specific process. Easy to test
       def wait_for_process(pid)
         out = Process.wait2(pid)
@@ -245,9 +278,11 @@ module Backburner
       # never shutdown
       def fork_it(&blk)
         pid = Kernel.fork do
-          self.class.is_child = true
-          $0 = "[ThreadsOnFork worker] parent: #{Process.ppid}"
-          blk.call
+          with_gc_disabled_on_exit do
+            self.class.is_child = true
+            $0 = "[ThreadsOnFork worker] parent: #{Process.ppid}"
+            blk.call
+          end
         end
         self.class.child_pids << pid
         pid

data/lib/backburner.rb

@@ -7,6 +7,7 @@ require 'backburner/helpers'
 require 'backburner/configuration'
 require 'backburner/logger'
 require 'backburner/connection'
+require 'backburner/enqueue_pool'
 require 'backburner/hooks'
 require 'backburner/performable'
 require 'backburner/worker'
@@ -15,7 +16,12 @@ require 'backburner/workers/forking'
 require 'backburner/workers/threads_on_fork'
 require 'backburner/workers/threading'
 require 'backburner/queue'
-require 'active_job/queue_adapters/backburner_adapter' if defined?(ActiveJob)
+# On Rails 7.2+ (which added AbstractAdapter) use this gem's Active Job adapter; older Rails autoload theirs.
+# Not when Rails' autoload has already loaded the adapter or is loading it (an adapter file requires backburner).
+if defined?(ActiveJob::QueueAdapters::AbstractAdapter) && (ActiveJob::QueueAdapters.autoload?(:BackburnerAdapter) ||
+   !ActiveJob::QueueAdapters.constants(false).include?(:BackburnerAdapter))
+  require 'active_job/queue_adapters/backburner_adapter'
+end
 
 module Backburner
   class << self

data/test/active_job_adapter_test.rb

@@ -0,0 +1,24 @@
+require File.expand_path('../test_helper', __FILE__)
+require 'open3'
+require 'rbconfig'
+
+describe "Backburner as the Active Job adapter" do
+  %w(active_job_first backburner_first active_job_only).each do |order|
+    it "enqueues and works jobs when the app requires #{order.tr('_', ' ')}" do
+      namespace = "demo.test.active_job.#{Process.pid}.#{order}"
+      app = File.expand_path('../fixtures/active_job_app.rb', __FILE__)
+      out, status = Open3.capture2e(RbConfig.ruby, '-w', '-I', File.expand_path('../../lib', __FILE__), app, namespace, order)
+      assert status.success?, out
+      # only backburner's own files: RubyGems can warn about its own
+      refute_match(/circular require considered harmful - #{Regexp.escape(File.expand_path('../../lib', __FILE__))}\//, out)
+      result = JSON.parse(out.lines.last)
+
+      tube = "#{namespace}.adapter"
+      assert_equal({ 'tube' => tube, 'pri' => 7, 'state' => 'ready', 'delay' => 0 }, result['jobs']['now'])
+      assert_equal({ 'tube' => tube, 'pri' => 3, 'state' => 'ready', 'delay' => 0 }, result['jobs']['urgent'])
+      # wait: 3 reaches the adapter a little under 3 seconds ahead: rounded up, so it doesn't run early
+      assert_equal({ 'tube' => tube, 'pri' => 7, 'state' => 'delayed', 'delay' => 3 }, result['jobs']['later'])
+      assert_equal %w(later now urgent), result['performed']
+    end
+  end
+end

data/test/back_burner_test.rb

@@ -72,6 +72,19 @@ describe "Backburner module" do
         Backburner.configuration.namespace_separator = ':'
       end
     end
+
+    it "reads enqueue_connection_max_idle as seconds" do
+      max_idle = Backburner.configuration.enqueue_connection_max_idle
+      Backburner.configuration.enqueue_connection_max_idle = "30"
+      assert_equal 30.0, Backburner.configuration.enqueue_connection_max_idle
+      Backburner.configuration.enqueue_connection_max_idle = 1
+      assert_equal 1.0, Backburner.configuration.enqueue_connection_max_idle
+      Backburner.configuration.enqueue_connection_max_idle = nil
+      assert_nil Backburner.configuration.enqueue_connection_max_idle
+      assert_raises(ArgumentError) { Backburner.configuration.enqueue_connection_max_idle = "a minute" }
+    ensure
+      Backburner.configuration.enqueue_connection_max_idle = max_idle
+    end
   end # configuration
 
   describe "for default_queues" do

data/test/connection_test.rb

@@ -152,7 +152,29 @@ describe "Backburner::Connection class" do
       end
     end
 
+    describe "when a reconnect fails" do
+      it "should reconnect on the next call once beanstalkd is back" do
+        back_up = Beaneater.new('127.0.0.1:11300')
+        Beaneater.expects(:new).twice.raises(Beaneater::NotConnected).then.returns(back_up)
+        assert_raises(Beaneater::NotConnected) { @connection.reconnect! }
+        assert_equal false, @connection.connected?
+        assert_kind_of Array, @connection.tubes.all
+        assert_equal true, @connection.connected?
+      end
+    end
+
     describe "when using the retryable method" do
+      it "should raise Beaneater::NotConnected once the retries are used up" do
+        attempts = 0
+        assert_raises Beaneater::NotConnected do
+          @connection.retryable(max_retries: 1, retry_delay: 0) do
+            attempts += 1
+            raise Beaneater::NotConnected
+          end
+        end
+        assert_equal 2, attempts
+      end
+
       it "should yield to the block multiple times" do
         expected    = 2
         retry_count = 0

data/test/fixtures/active_job_app.rb

@@ -0,0 +1,53 @@
+# An app using Backburner as its Active Job adapter. active_job_adapter_test.rb runs it in a child process,
+# so Active Job (and Active Support) stay out of the suite's own process.
+# Arguments: the tube namespace, then which of the two the app requires first (or active_job_only).
+require 'json'
+require 'logger' # Active Support 6.1 and 7.0 need Logger loaded first (with concurrent-ruby 1.3.5+)
+
+namespace, order = ARGV
+case order
+when 'active_job_first' # a Rails app: rails/all, then Bundler.require
+  require 'active_job'
+  require 'backburner'
+when 'backburner_first'
+  require 'backburner'
+  require 'active_job'
+when 'active_job_only' # gem 'backburner', require: false: the adapter loads it
+  require 'active_job'
+end
+
+ActiveJob::Base.logger = Logger.new(nil)
+ActiveJob::Base.queue_adapter = :backburner
+Backburner.configure do |config|
+  config.beanstalk_url = "beanstalk://127.0.0.1"
+  config.tube_namespace = namespace
+  config.logger = Logger.new(nil)
+  config.reserve_timeout = 5
+end
+
+$performed = []
+class AdapterTestJob < ActiveJob::Base
+  queue_as :adapter
+  queue_with_priority 7
+  def perform(name)
+    $performed << name
+  end
+end
+
+enqueued = {
+  'now' => AdapterTestJob.perform_later('now'),
+  'urgent' => AdapterTestJob.set(priority: 3).perform_later('urgent'),
+  'later' => AdapterTestJob.set(wait: 3).perform_later('later')
+}
+beanstalk = Beaneater.new('127.0.0.1')
+jobs = enqueued.map do |name, job|
+  stats = job.provider_job_id && beanstalk.jobs.find(job.provider_job_id).stats
+  [name, stats && { 'tube' => stats.tube, 'pri' => stats.pri, 'state' => stats.state, 'delay' => stats.delay }]
+end.to_h
+beanstalk.close
+
+worker = Backburner::Workers::Simple.new(['adapter'])
+worker.prepare
+enqueued.size.times { worker.work_one_job }
+
+puts JSON.generate('jobs' => jobs, 'performed' => $performed.sort)

data/test/fixtures/autoload/test_broken_name_job.rb

@@ -0,0 +1,4 @@
+class TestBrokenNameJob
+  include TestMissingConcern # renamed or removed: NameError for another constant while the file loads
+  def self.perform(x); end
+end

data/test/fixtures/autoload/test_broken_no_method_job.rb

@@ -0,0 +1,4 @@
+class TestBrokenNoMethodJob
+  queue_respond_timout 30 # a typo: NoMethodError while the file loads
+  def self.perform(x); end
+end

data/test/fixtures/autoload/test_broken_require_job.rb

@@ -0,0 +1,4 @@
+require 'backburner_test_missing_gem' # a gem left out of the bundle: LoadError while the file loads
+class TestBrokenRequireJob
+  def self.perform(x); end
+end

data/test/fixtures/autoload/test_broken_runtime_job.rb

@@ -0,0 +1,4 @@
+class TestBrokenRuntimeJob
+  raise ArgumentError, "bad config" # an error of another kind while the file loads (ENV.fetch raising KeyError, say)
+  def self.perform(x); end
+end

data/test/fixtures/autoload/test_broken_syntax_job.rb

@@ -0,0 +1,4 @@
+class TestBrokenSyntaxJob
+  def self.perform(x)
+    if # a syntax error: SyntaxError while the file loads
+  end

data/test/fixtures/autoload/test_broken_timeout_job.rb

@@ -0,0 +1,4 @@
+class TestBrokenTimeoutJob
+  raise Timeout::Error, "class body call timed out" # a call in the class body that timed out (Net::OpenTimeout is a Timeout::Error)
+  def self.perform(x); end
+end

data/test/fixtures/autoload/test_same_name_job.rb

@@ -0,0 +1,5 @@
+module TestOtherNamespace; end
+class TestSameNameJob
+  OTHER = TestOtherNamespace::TestSameNameJob # NameError for a constant of the same name in another module
+  def self.perform(x); end
+end

data/test/fixtures/test_fork_jobs.rb

@@ -70,3 +70,8 @@ class TestJobMultithreadFork
     }], :queue => 'response'
   end
 end
+
+class TestJobNoopFork
+  include Backburner::Queue
+  def self.perform; end
+end

data/test/fixtures/test_jobs.rb

@@ -13,6 +13,22 @@ class TestJob
   def self.perform(x, y); $worker_test_count += x + y; end
 end
 
+class TestParentQueueJob
+  include Backburner::Queue
+  queue "test-parent"
+  queue_priority :high
+  queue_respond_timeout 250
+  def self.perform(x, y); $worker_test_count += x + y; end
+end
+
+class TestChildQueueJob < TestParentQueueJob; end
+
+class TestNamedQueueJob
+  include Backburner::Queue
+  queue lambda { |klass| klass.name.downcase }
+  def self.perform; end
+end
+
 class TestSlowJob
   include Backburner::Queue
   queue_priority :medium
@@ -45,6 +61,21 @@ class TestRetryJob
   end
 end
 
+class TestFloatRetryDelayJob
+  include Backburner::Queue
+  queue_retry_delay 1.5
+  def self.perform(retry_count)
+    raise RuntimeError
+  end
+end
+
+class TestFloatRespondTimeoutJob
+  include Backburner::Queue
+  queue "test-plain"
+  queue_respond_timeout 1.5
+  def self.perform; end
+end
+
 class TestConfigurableRetryJob
   include Backburner::Queue
   def self.perform(retry_count)
@@ -85,3 +116,25 @@ class TestLambdaQueueJob
   queue lambda { |klass| klass.calculated_queue_name }
   def self.calculated_queue_name; 'lambda-queue' end
 end
+
+class TestOverTTRJob
+  include Backburner::Queue
+  queue_respond_timeout 2
+  # Outlives its TTR without Timeout being able to interrupt it (like a blocking C call)
+  def self.perform; Thread.handle_interrupt(Object => :never) { sleep 3 }; end
+end
+
+# Like an autoloader that doesn't find a file for the constant
+module AutoloadingDemo
+  def self.const_missing(name); raise LoadError, "cannot load such file -- autoloading_demo/#{name}"; end
+end
+
+# Job classes an app autoloads: whose files fail to load, and one whose file isn't there
+autoload :TestBrokenNameJob, File.expand_path('../autoload/test_broken_name_job', __FILE__)
+autoload :TestBrokenRequireJob, File.expand_path('../autoload/test_broken_require_job', __FILE__)
+autoload :TestBrokenNoMethodJob, File.expand_path('../autoload/test_broken_no_method_job', __FILE__)
+autoload :TestNoFileJob, File.expand_path('../autoload/test_no_such_job', __FILE__)
+autoload :TestBrokenRuntimeJob, File.expand_path('../autoload/test_broken_runtime_job', __FILE__)
+autoload :TestSameNameJob, File.expand_path('../autoload/test_same_name_job', __FILE__)
+autoload :TestBrokenSyntaxJob, File.expand_path('../autoload/test_broken_syntax_job', __FILE__)
+autoload :TestBrokenTimeoutJob, File.expand_path('../autoload/test_broken_timeout_job', __FILE__)