workhorse 2.0.0.rc0 → 2.0.0.rc1

data/lib/workhorse/worker.rb

@@ -291,7 +291,7 @@ module Workhorse
       return unless daemon_id
 
       path = self.class.heartbeat_file_for(daemon_id)
-      FileUtils.touch(path) if path
+      touch_pid_file(path) if path
     rescue StandardError => e
       Workhorse.debug_log("[Job worker #{id}] Heartbeat touch failed: #{e.class}: #{e.message}")
     end
@@ -363,7 +363,7 @@ module Workhorse
       Workhorse.debug_log("[Job worker #{id}] Memory limit exceeded: #{mem}MB > #{max}MB, initiating shutdown")
 
       if defined?(Rails)
-        FileUtils.touch self.class.shutdown_file_for(pid)
+        touch_pid_file(self.class.shutdown_file_for(pid))
       end
 
       log "Worker process #{id.inspect} memory consumption (RSS) of #{mem}MB exceeds " \
@@ -444,8 +444,17 @@ module Workhorse
       # Create shutdown file for watch to detect
       shutdown_file = self.class.shutdown_file_for(pid)
       if shutdown_file
-        FileUtils.touch(shutdown_file)
-        Workhorse.debug_log("[Job worker #{id}] Shutdown file created: #{shutdown_file}")
+        begin
+          touch_pid_file(shutdown_file)
+          Workhorse.debug_log("[Job worker #{id}] Shutdown file created: #{shutdown_file}")
+        rescue StandardError => e
+          # The shutdown has to go ahead regardless: the worker stopped
+          # accepting jobs above, so giving up here would leave it taking none
+          # and never exiting. Without the file, watch sees a worker that is
+          # gone rather than one that asked to be restarted, and starts it all
+          # the same.
+          log "Could not create shutdown file #{shutdown_file}, shutting down regardless: #{e.message}", :warn
+        end
       end
 
       # Monitor in a separate thread to avoid blocking the signal handler
@@ -533,5 +542,17 @@ module Workhorse
         Kernel.sleep 0.2
       end
     end
+
+    # Touches a file in the pids directory, creating the directory first. It
+    # is not guaranteed to exist: `tmp` is rarely checked in, so a fresh
+    # checkout or deployment has none until something writes there.
+    #
+    # @param path [Pathname, String]
+    # @return [void]
+    # @private
+    def touch_pid_file(path)
+      FileUtils.mkdir_p(File.dirname(path))
+      FileUtils.touch(path)
+    end
   end
 end

data/test/lib/db_schema.rb

@@ -1,10 +1,21 @@
 ActiveRecord::Schema.define do
   self.verbose = false
 
+  # Prefix lengths for the indexes on string columns. MySQL needs them, as the
+  # default `utf8mb4` charset puts a full `varchar(255)` past the maximum key
+  # length; Oracle indexes the whole column and rejects the option.
+  state_length = DB_ORACLE ? {} : { length: { state: 191 } }
+  queue_length = DB_ORACLE ? {} : { length: 191 }
+  key_length   = DB_ORACLE ? {} : { length: 191 }
+
   create_table :jobs, force: true do |t|
     t.string :state, null: false, default: 'waiting'
     t.string :queue, null: true
-    t.text :handler, null: false, limit: 4_294_967_295
+
+    # Binary rather than text, matching the generated migration: the handler is
+    # a `Marshal.dump`, and a text column is character data - on Oracle a CLOB,
+    # whose character set conversion would corrupt it.
+    t.binary :handler, null: false, limit: 4_294_967_295
 
     t.string :locked_by
     t.datetime :locked_at
@@ -25,10 +36,12 @@ ActiveRecord::Schema.define do
     t.timestamps null: false
   end
 
-  add_index :jobs, :queue, length: 191
-  add_index :jobs, %i[state perform_at], length: { state: 191 }, name: 'idx_jobs_state_perform_at'
-  add_index :jobs, %i[state priority created_at], length: { state: 191 }, name: 'idx_jobs_state_prio_created'
-  add_index :jobs, %i[state expires_at], length: { state: 191 }, name: 'idx_jobs_state_expires_at'
+  # The index names are given explicitly because the ones Rails would derive
+  # exceed the 30 characters Oracle allows before 12.2.
+  add_index :jobs, :queue, **queue_length
+  add_index :jobs, %i[state perform_at], name: 'idx_jobs_state_perform_at', **state_length
+  add_index :jobs, %i[state priority created_at], name: 'idx_jobs_state_prio_created', **state_length
+  add_index :jobs, %i[state expires_at], name: 'idx_jobs_state_expires_at', **state_length
   add_index :jobs, :perform_at
 
   create_table :workhorse_schedules, force: true do |t|
@@ -44,6 +57,6 @@ ActiveRecord::Schema.define do
     t.timestamps null: false
   end
 
-  add_index :workhorse_schedules, :key, unique: true, length: 191, name: 'idx_wh_schedules_key'
+  add_index :workhorse_schedules, :key, unique: true, name: 'idx_wh_schedules_key', **key_length
   add_index :workhorse_schedules, %i[enabled next_at], name: 'idx_wh_schedules_due'
 end

data/test/lib/test_helper.rb

@@ -6,8 +6,8 @@ require 'benchmark'
 require 'concurrent'
 require 'jobs'
 
-# The adapter to run the test suite against. Both MySQL / MariaDB adapters are
-# supported, as workhorse has to work with either of them.
+# The adapter to run the test suite against. Workhorse has to work with each of
+# them, so each is covered by CI.
 DB_ADAPTER = ENV.fetch('DB_ADAPTER', 'mysql2')
 
 case DB_ADAPTER
@@ -15,10 +15,16 @@ when 'mysql2'
   require 'mysql2'
 when 'trilogy'
   require 'trilogy'
+when 'oracle_enhanced'
+  require 'active_record/connection_adapters/oracle_enhanced_adapter'
 else
-  fail "Unsupported DB_ADAPTER #{DB_ADAPTER.inspect}, use 'mysql2' or 'trilogy'."
+  fail "Unsupported DB_ADAPTER #{DB_ADAPTER.inspect}, use 'mysql2', 'trilogy' or 'oracle_enhanced'."
 end
 
+# Whether the suite is running against Oracle. Used where the schema or an
+# assertion cannot be written the same way for both families.
+DB_ORACLE = DB_ADAPTER == 'oracle_enhanced'
+
 class MockRailsEnv < String
   def production?
     self == 'production'
@@ -44,6 +50,38 @@ class Rails
 end
 
 class WorkhorseTest < ActiveSupport::TestCase
+  # Seconds a test may run before the backtraces of all its threads are
+  # printed. A hung test otherwise only shows up as a CI attempt killed at its
+  # time limit, with nothing saying where it hung.
+  HANG_REPORT_AFTER = 120
+
+  # Callbacks rather than methods, so that a test class defining its own setup
+  # or teardown does not skip them.
+  setup do
+    test = "#{self.class}##{name}"
+
+    @hang_watchdog = Thread.new do
+      sleep HANG_REPORT_AFTER
+      report_threads "#{test} is still running after #{HANG_REPORT_AFTER}s"
+    end
+  end
+
+  # Prints the backtraces of all threads of this process but the calling one.
+  def report_threads(title)
+    warn "#{title} (PID #{Process.pid}). Its threads:"
+
+    Thread.list.each do |thread|
+      next if thread == Thread.current
+
+      warn "--- #{thread.inspect}\n#{(thread.backtrace || ['(no backtrace)']).join("\n")}"
+    end
+  end
+
+  teardown do
+    @hang_watchdog&.kill
+    restore_termination_traps
+  end
+
   def setup
     remove_pids!
     clear_locks_and_db_threads!
@@ -68,6 +106,36 @@ class WorkhorseTest < ActiveSupport::TestCase
     object.singleton_class.send(:remove_method, method)
   end
 
+  # Holds workhorse's global lock on a connection of its own, so that the code
+  # under test sees it as taken by another worker.
+  def with_global_lock_held
+    connection = ActiveRecord::Base.connection_pool.checkout
+    connection.select_value(acquire_global_lock_sql)
+    yield
+  ensure
+    if connection
+      connection.select_value(release_global_lock_sql)
+      ActiveRecord::Base.connection_pool.checkin(connection)
+    end
+  end
+
+  # Mirrors what Workhorse::Poller#with_global_lock emits, so that the lock the
+  # code under test tries to take is the same one.
+  def acquire_global_lock_sql
+    return <<~SQL.strip if DB_ORACLE
+      SELECT DBMS_LOCK.REQUEST(#{Workhorse::Poller::ORACLE_LOCK_HANDLE}, #{Workhorse::Poller::ORACLE_LOCK_MODE}, 1)
+      FROM DUAL
+    SQL
+
+    return "SELECT GET_LOCK(CONCAT(DATABASE(), '_workhorse'), 1)"
+  end
+
+  def release_global_lock_sql
+    return "SELECT DBMS_LOCK.RELEASE(#{Workhorse::Poller::ORACLE_LOCK_HANDLE}) FROM DUAL" if DB_ORACLE
+
+    return "SELECT RELEASE_LOCK(CONCAT(DATABASE(), '_workhorse'))"
+  end
+
   def clear_locks_and_db_threads!
     # Releases the locks held by *this* connection, which is all a fresh run
     # needs. It used to kill the queries of every other connection as well, to
@@ -78,7 +146,16 @@ class WorkhorseTest < ActiveSupport::TestCase
     # spurious "Lost connection to server during query" failures. Should a
     # crashed run ever leave a lock behind, kill its *connection*, which does
     # release it.
-    Workhorse::DbJob.connection.execute('SELECT RELEASE_ALL_LOCKS()')
+    if DB_ORACLE
+      # Oracle has no "release everything" call, but workhorse takes exactly
+      # one handle. RELEASE reports a status rather than raising when the lock
+      # is not held, so this needs no guard.
+      Workhorse::DbJob.connection.execute(
+        "SELECT DBMS_LOCK.RELEASE(#{Workhorse::Poller::ORACLE_LOCK_HANDLE}) FROM DUAL"
+      )
+    else
+      Workhorse::DbJob.connection.execute('SELECT RELEASE_ALL_LOCKS()')
+    end
   end
 
   def remove_pids!
@@ -144,8 +221,14 @@ class WorkhorseTest < ActiveSupport::TestCase
     w.shutdown
   end
 
+  # Without auto_terminate unless asked for, as work and work_until have it:
+  # a worker that has it traps TERM and INT for the whole process and never
+  # gives them back, so the test process ignored the TERM meant to stop it for
+  # the rest of the run - and a timed-out CI attempt kept running alongside
+  # the next. Every test hands them back afterwards regardless, see the
+  # teardown above.
   def with_worker(options = {})
-    w = Workhorse::Worker.new(**options)
+    w = Workhorse::Worker.new(auto_terminate: false, **options)
     w.start
     begin
       yield(w)
@@ -171,6 +254,12 @@ class WorkhorseTest < ActiveSupport::TestCase
     daemon.stop(quiet: true)
   end
 
+  # Hands TERM and INT back to the handlers the process started with, which a
+  # worker started with auto_terminate replaced.
+  def restore_termination_traps
+    ORIGINAL_TERMINATION_TRAPS.each { |signal, handler| Signal.trap(signal, handler) }
+  end
+
   def with_retries(max = 50, interval: 0.1, &_block)
     runs = 0
 
@@ -200,15 +289,24 @@ class WorkhorseTest < ActiveSupport::TestCase
   end
 end
 
+# On Oracle the "database" is a service name rather than a schema, and the
+# schema is the user the suite connects as.
 ActiveRecord::Base.establish_connection(
   adapter:  DB_ADAPTER,
-  database: ENV.fetch('DB_NAME', nil) || 'workhorse',
-  username: ENV.fetch('DB_USERNAME', nil) || 'root',
-  password: ENV.fetch('DB_PASSWORD', nil) || '',
+  database: ENV.fetch('DB_NAME', nil) || (DB_ORACLE ? 'FREEPDB1' : 'workhorse'),
+  username: ENV.fetch('DB_USERNAME', nil) || (DB_ORACLE ? 'workhorse' : 'root'),
+  password: ENV.fetch('DB_PASSWORD', nil) || (DB_ORACLE ? 'workhorse' : ''),
   host:     ENV.fetch('DB_HOST', nil) || '127.0.0.1',
-  port:     ENV.fetch('DB_PORT', nil) || 3306,
+  port:     ENV.fetch('DB_PORT', nil) || (DB_ORACLE ? 1521 : 3306),
   pool:     10
 )
 
 require 'db_schema'
 require 'workhorse'
+
+# The handlers the process started with, see #restore_termination_traps.
+ORIGINAL_TERMINATION_TRAPS = Workhorse::Worker::SHUTDOWN_SIGNALS.to_h do |signal|
+  handler = Signal.trap(signal, 'DEFAULT')
+  Signal.trap(signal, handler)
+  [signal, handler]
+end

data/test/workhorse/db_job_test.rb

@@ -10,9 +10,7 @@ class Workhorse::DbJobTest < WorkhorseTest
 
   def test_reset_failed
     job = Workhorse.enqueue FailingTestJob.new
-    work 0.5
-    job.reload
-    assert_equal 'failed', job.state
+    work_until(pool_size: 5, polling_interval: 0.2) { assert_equal 'failed', job.reload.state }
 
     job.reset!
 

data/test/workhorse/notifier_test.rb

@@ -398,19 +398,6 @@ class Workhorse::NotifierTest < WorkhorseTest
     return File.join(Dir.tmpdir, 'workhorse_test.wake')
   end
 
-  # Holds workhorse's global lock on a connection of its own, so that the code
-  # under test sees it as taken by another worker.
-  def with_global_lock_held
-    connection = ActiveRecord::Base.connection_pool.checkout
-    connection.select_value("SELECT GET_LOCK(CONCAT(DATABASE(), '_workhorse'), 1)")
-    yield
-  ensure
-    if connection
-      connection.select_value("SELECT RELEASE_LOCK(CONCAT(DATABASE(), '_workhorse'))")
-      ActiveRecord::Base.connection_pool.checkin(connection)
-    end
-  end
-
   def file_notifier
     return Workhorse::Notifiers::FileSystem.new(path: wake_path)
   end

data/test/workhorse/performer_test.rb

@@ -8,28 +8,26 @@ class Workhorse::PerformerTest < WorkhorseTest
       Workhorse.enqueue DbConnectionTestJob.new
     end
 
-    work 0.2, polling_interval: 0.2
+    work_until(pool_size: 5, polling_interval: 0.2) do
+      assert_equal 2, DbConnectionTestJob.db_connections.count
+    end
 
-    assert_equal 2, DbConnectionTestJob.db_connections.count
     assert_equal 2, DbConnectionTestJob.db_connections.uniq.count
   end
 
   def test_success
     Workhorse.enqueue BasicJob.new(sleep_time: 0.1)
-    work 0.2, polling_interval: 0.2
-    assert_equal 'succeeded', Workhorse::DbJob.first.state
+    work_until(pool_size: 5, polling_interval: 0.2) { assert_equal 'succeeded', Workhorse::DbJob.first.state }
   end
 
   def test_exception
     Workhorse.enqueue FailingTestJob.new
-    work 0.2, polling_interval: 0.2
-    assert_equal 'failed', Workhorse::DbJob.first.state
+    work_until(pool_size: 5, polling_interval: 0.2) { assert_equal 'failed', Workhorse::DbJob.first.state }
   end
 
   def test_syntax_exception
     Workhorse.enqueue SyntaxErrorJob
-    work 0.2, polling_interval: 0.2
-    assert_equal 'failed', Workhorse::DbJob.first.state
+    work_until(pool_size: 5, polling_interval: 0.2) { assert_equal 'failed', Workhorse::DbJob.first.state }
   end
 
   def test_on_exception
@@ -41,7 +39,7 @@ class Workhorse::PerformerTest < WorkhorseTest
     end
 
     Workhorse.enqueue FailingTestJob.new
-    work 0.2, polling_interval: 0.2
+    work_until(pool_size: 5, polling_interval: 0.2) { assert exception, 'expected on_exception to be called' }
 
     assert_equal exception.message, FailingTestJob::MESSAGE
   ensure

data/test/workhorse/poller_test.rb

@@ -19,25 +19,25 @@ class Workhorse::PollerTest < WorkhorseTest
     Workhorse.enqueue BasicJob.new(sleep_time: 2), queue: :q2
     Workhorse.enqueue BasicJob.new(sleep_time: 2), queue: :q2
 
-    assert_equal %w[q1 q2], w.poller.send(:valid_queues)
+    assert_equal %w[q1 q2], valid_queues_of(w)
 
     first_job = Workhorse::DbJob.first
     first_job.mark_locked!(42)
 
-    assert_equal %w[q2], w.poller.send(:valid_queues)
+    assert_equal %w[q2], valid_queues_of(w)
 
     first_job.mark_started!
 
-    assert_equal %w[q2], w.poller.send(:valid_queues)
+    assert_equal %w[q2], valid_queues_of(w)
 
     first_job.mark_succeeded!
 
-    assert_equal %w[q1 q2], w.poller.send(:valid_queues)
+    assert_equal %w[q1 q2], valid_queues_of(w)
 
     last_job = Workhorse::DbJob.last
     last_job.mark_locked!(42)
 
-    assert_equal %w[q1], w.poller.send(:valid_queues)
+    assert_equal %w[q1], valid_queues_of(w)
 
     begin
       fail 'Some exception'
@@ -45,34 +45,36 @@ class Workhorse::PollerTest < WorkhorseTest
       last_job.mark_failed!(e)
     end
 
-    assert_equal %w[q1 q2], w.poller.send(:valid_queues)
+    assert_equal %w[q1 q2], valid_queues_of(w)
   end
 
   def test_valid_queues_2
     w = Workhorse::Worker.new(polling_interval: 60)
 
-    assert_equal [], w.poller.send(:valid_queues)
+    assert_equal [], valid_queues_of(w)
 
     Workhorse.enqueue BasicJob.new(sleep_time: 2), queue: nil
 
-    assert_equal [nil], w.poller.send(:valid_queues)
+    assert_equal [nil], valid_queues_of(w)
 
     a_job = Workhorse.enqueue BasicJob.new(sleep_time: 2), queue: :a
 
-    assert_equal [nil, 'a'], w.poller.send(:valid_queues)
+    assert_equal [nil, 'a'], valid_queues_of(w)
 
     a_job.update_attribute :state, :locked
 
-    assert_equal [nil], w.poller.send(:valid_queues)
+    assert_equal [nil], valid_queues_of(w)
   end
 
   def test_no_queues
     w = Workhorse::Worker.new(polling_interval: 60)
-    assert_equal [], w.poller.send(:valid_queues)
+    assert_equal [], valid_queues_of(w)
   end
 
-  # Not every adapter returns a result set from `execute`, so querying the
-  # valid queues must not rely on its return value.
+  # Not every adapter returns a result set from `execute`: the Oracle enhanced
+  # adapter, for instance, returns `true` for queries, both before and after
+  # version 7.0.0. Querying the valid queues must therefore not rely on the
+  # return value of `execute`.
   def test_valid_queues_without_usable_execute
     w = Workhorse::Worker.new(polling_interval: 60)
 
@@ -80,7 +82,7 @@ class Workhorse::PollerTest < WorkhorseTest
     Workhorse.enqueue BasicJob.new(sleep_time: 2), queue: :a
 
     with_return_value(Workhorse::DbJob.connection, :execute, true) do
-      assert_equal [nil, 'a'], w.poller.send(:valid_queues)
+      assert_equal [nil, 'a'], valid_queues_of(w)
     end
   end
 
@@ -92,11 +94,11 @@ class Workhorse::PollerTest < WorkhorseTest
     end
     jobs = Workhorse::DbJob.all
 
-    assert_equal [nil], w.poller.send(:valid_queues)
+    assert_equal [nil], valid_queues_of(w)
 
     jobs[0].mark_locked!(42)
 
-    assert_equal [nil], w.poller.send(:valid_queues)
+    assert_equal [nil], valid_queues_of(w)
   end
 
   def test_with_instant_repolling
@@ -133,10 +135,37 @@ class Workhorse::PollerTest < WorkhorseTest
       Workhorse.enqueue BasicJob.new(some_param: i, sleep_time: 0)
     end
 
-    # Create 10 worker processes that work for 3s each
+    # Create 10 worker processes that work until all 100 jobs have succeeded,
+    # and for at least 3s, so that each is still polling while the second
+    # batch comes in. Not for a fixed time: every poll holds the global lock
+    # throughout, so the workers take turns, and 3s got through as few as 25
+    # of the 100 jobs on a loaded CI runner, and 61 on Oracle. What is tested
+    # is that none is taken twice and every worker gets a turn, not how fast.
     10.times do
       Process.fork do
-        work 3, pool_size: 1, polling_interval: 0.1
+        started = Time.now
+
+        # A worker that cannot even be shut down would leave waitall below
+        # waiting for good, so it reports where it is stuck and gives up.
+        Thread.new do
+          sleep 90
+          report_threads 'Worker process still running after 90s'
+          exit!(1)
+        end
+
+        with_worker(pool_size: 1, polling_interval: 0.1, auto_terminate: false) do
+          loop do
+            elapsed = Time.now - started
+            break if elapsed >= 3 && Workhorse::DbJob.succeeded.count >= 100
+
+            if elapsed >= 60
+              report_threads 'Worker process gave up waiting for all jobs to succeed after 60s'
+              break
+            end
+
+            sleep 0.1
+          end
+        end
       ensure
         # Exit without running the at_exit handlers of the test process: one
         # of them is Minitest's, which joins threads this fork did not
@@ -153,24 +182,51 @@ class Workhorse::PollerTest < WorkhorseTest
       Workhorse.enqueue BasicJob.new(sleep_time: 0)
     end
 
-    # Wait for all forked processes to finish (should take ~3s)
+    # Wait for all forked processes to finish
     Process.waitall
 
     total = Workhorse::DbJob.count
     succeeded = Workhorse::DbJob.succeeded.count
-    used_workers = Workhorse::DbJob.lock.pluck(:locked_by).uniq.size
+    # In a transaction, as Oracle commits a FOR UPDATE outside of one before
+    # its rows are fetched and then fails with "fetch out of sequence".
+    used_workers = Workhorse::DbJob.transaction do
+      Workhorse::DbJob.lock.pluck(:locked_by).uniq.size
+    end
 
     # Make sure there are 100 jobs, all jobs have succeeded and that all of the
-    # workers have had their turn.
+    # workers have had their turn. The jobs that have not are listed, as all
+    # a count says is that something got stuck, not where.
+    unfinished = Workhorse::DbJob.where.not(state: 'succeeded').order(:id).map do |job|
+      "##{job.id} #{job.state}, locked_by #{job.locked_by.inspect}, " \
+        "locked_at #{job.locked_at.inspect}, started_at #{job.started_at.inspect}"
+    end
+
     assert_equal 100, total
-    assert_equal 100, succeeded
+    assert_equal 100, succeeded, "Jobs that did not succeed:\n#{unfinished.join("\n")}"
     assert_equal 10,  used_workers
   end
 
+  # A poll that finds the global lock taken waits at least as long as it asked
+  # to before giving up. Below half a second is what the poller asks for with
+  # a short polling interval, and what MySQL and Oracle, which take the timeout
+  # as whole seconds, used to round down to not waiting at all.
+  def test_contended_global_lock_waits_for_its_timeout
+    poller = Workhorse::Worker.new(polling_interval: 0.3).poller
+    ran = false
+
+    elapsed = with_global_lock_held do
+      started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      poller.send(:with_global_lock, timeout: 0.3, count_failures: false) { ran = true }
+      Process.clock_gettime(Process::CLOCK_MONOTONIC) - started
+    end
+
+    assert_not ran
+    assert_operator elapsed, :>=, 0.25
+  end
+
   def test_connection_loss
-    # rubocop: disable Style/GlobalVars
+    # rubocop: disable-next Style/GlobalVars
     $thread_conn = nil
-    # rubocop: enable Style/GlobalVars
 
     Workhorse.enqueue BasicJob.new(sleep_time: 3)
 
@@ -203,7 +259,14 @@ class Workhorse::PollerTest < WorkhorseTest
       Workhorse.clean_stuck_jobs = clean
       with_daemon do
         Workhorse.enqueue BasicJob.new(sleep_time: 5)
-        sleep 0.2
+
+        # Waited for rather than slept on: until a worker has taken the job,
+        # locked_by is empty and the cleanup, which matches on the host in it,
+        # cannot recognise the job as one of its own.
+        with_retries do
+          assert_equal 'started', Workhorse::DbJob.first.state
+        end
+
         kill_deamon_workers
 
         assert_equal 1, Workhorse::DbJob.count
@@ -269,6 +332,13 @@ class Workhorse::PollerTest < WorkhorseTest
 
   private
 
+  # The worker's valid queues, the queueless one first. Sorted here, as they
+  # come from a SELECT DISTINCT without ORDER BY: MySQL happens to return NULL
+  # first, Oracle does not, and the poller does not depend on the order.
+  def valid_queues_of(worker)
+    return worker.poller.send(:valid_queues).sort_by { |queue| [queue.nil? ? 0 : 1, queue.to_s] }
+  end
+
   def kill_deamon_workers
     pids = daemon.workers.map(&:pid)
     pids.each do |pid|

data/test/workhorse/schedule_test.rb

@@ -362,7 +362,7 @@ class Workhorse::ScheduleTest < WorkhorseTest
     end
 
     assert_equal 1, Workhorse::DbJob.succeeded.count
-    assert exceptions.any? { |e| e.is_a?(NameError) }, "expected a NameError, got #{exceptions.map(&:class)}"
+    assert exceptions.any?(NameError), "expected a NameError, got #{exceptions.map(&:class)}"
   end
 
   # An occurrence must not be consumed unless the job for it exists. Were the