workhorse 1.5.2 → 2.0.0.rc1

data/test/workhorse/poller_test.rb

@@ -19,25 +19,25 @@ class Workhorse::PollerTest < WorkhorseTest
     Workhorse.enqueue BasicJob.new(sleep_time: 2), queue: :q2
     Workhorse.enqueue BasicJob.new(sleep_time: 2), queue: :q2
 
-    assert_equal %w[q1 q2], w.poller.send(:valid_queues)
+    assert_equal %w[q1 q2], valid_queues_of(w)
 
     first_job = Workhorse::DbJob.first
     first_job.mark_locked!(42)
 
-    assert_equal %w[q2], w.poller.send(:valid_queues)
+    assert_equal %w[q2], valid_queues_of(w)
 
     first_job.mark_started!
 
-    assert_equal %w[q2], w.poller.send(:valid_queues)
+    assert_equal %w[q2], valid_queues_of(w)
 
     first_job.mark_succeeded!
 
-    assert_equal %w[q1 q2], w.poller.send(:valid_queues)
+    assert_equal %w[q1 q2], valid_queues_of(w)
 
     last_job = Workhorse::DbJob.last
     last_job.mark_locked!(42)
 
-    assert_equal %w[q1], w.poller.send(:valid_queues)
+    assert_equal %w[q1], valid_queues_of(w)
 
     begin
       fail 'Some exception'
@@ -45,30 +45,30 @@ class Workhorse::PollerTest < WorkhorseTest
       last_job.mark_failed!(e)
     end
 
-    assert_equal %w[q1 q2], w.poller.send(:valid_queues)
+    assert_equal %w[q1 q2], valid_queues_of(w)
   end
 
   def test_valid_queues_2
     w = Workhorse::Worker.new(polling_interval: 60)
 
-    assert_equal [], w.poller.send(:valid_queues)
+    assert_equal [], valid_queues_of(w)
 
     Workhorse.enqueue BasicJob.new(sleep_time: 2), queue: nil
 
-    assert_equal [nil], w.poller.send(:valid_queues)
+    assert_equal [nil], valid_queues_of(w)
 
     a_job = Workhorse.enqueue BasicJob.new(sleep_time: 2), queue: :a
 
-    assert_equal [nil, 'a'], w.poller.send(:valid_queues)
+    assert_equal [nil, 'a'], valid_queues_of(w)
 
     a_job.update_attribute :state, :locked
 
-    assert_equal [nil], w.poller.send(:valid_queues)
+    assert_equal [nil], valid_queues_of(w)
   end
 
   def test_no_queues
     w = Workhorse::Worker.new(polling_interval: 60)
-    assert_equal [], w.poller.send(:valid_queues)
+    assert_equal [], valid_queues_of(w)
   end
 
   # Not every adapter returns a result set from `execute`: the Oracle enhanced
@@ -82,7 +82,7 @@ class Workhorse::PollerTest < WorkhorseTest
     Workhorse.enqueue BasicJob.new(sleep_time: 2), queue: :a
 
     with_return_value(Workhorse::DbJob.connection, :execute, true) do
-      assert_equal [nil, 'a'], w.poller.send(:valid_queues)
+      assert_equal [nil, 'a'], valid_queues_of(w)
     end
   end
 
@@ -94,11 +94,11 @@ class Workhorse::PollerTest < WorkhorseTest
     end
     jobs = Workhorse::DbJob.all
 
-    assert_equal [nil], w.poller.send(:valid_queues)
+    assert_equal [nil], valid_queues_of(w)
 
     jobs[0].mark_locked!(42)
 
-    assert_equal [nil], w.poller.send(:valid_queues)
+    assert_equal [nil], valid_queues_of(w)
   end
 
   def test_with_instant_repolling
@@ -135,10 +135,43 @@ class Workhorse::PollerTest < WorkhorseTest
       Workhorse.enqueue BasicJob.new(some_param: i, sleep_time: 0)
     end
 
-    # Create 10 worker processes that work for 3s each
+    # Create 10 worker processes that work until all 100 jobs have succeeded,
+    # and for at least 3s, so that each is still polling while the second
+    # batch comes in. Not for a fixed time: every poll holds the global lock
+    # throughout, so the workers take turns, and 3s got through as few as 25
+    # of the 100 jobs on a loaded CI runner, and 61 on Oracle. What is tested
+    # is that none is taken twice and every worker gets a turn, not how fast.
     10.times do
       Process.fork do
-        work 3, pool_size: 1, polling_interval: 0.1
+        started = Time.now
+
+        # A worker that cannot even be shut down would leave waitall below
+        # waiting for good, so it reports where it is stuck and gives up.
+        Thread.new do
+          sleep 90
+          report_threads 'Worker process still running after 90s'
+          exit!(1)
+        end
+
+        with_worker(pool_size: 1, polling_interval: 0.1, auto_terminate: false) do
+          loop do
+            elapsed = Time.now - started
+            break if elapsed >= 3 && Workhorse::DbJob.succeeded.count >= 100
+
+            if elapsed >= 60
+              report_threads 'Worker process gave up waiting for all jobs to succeed after 60s'
+              break
+            end
+
+            sleep 0.1
+          end
+        end
+      ensure
+        # Exit without running the at_exit handlers of the test process: one
+        # of them is Minitest's, which joins threads this fork did not
+        # inherit and hangs the child, leaving waitall below waiting forever.
+        # In an ensure, as a child that raised must not run them either.
+        exit!(0)
       end
     end
 
@@ -149,24 +182,51 @@ class Workhorse::PollerTest < WorkhorseTest
       Workhorse.enqueue BasicJob.new(sleep_time: 0)
     end
 
-    # Wait for all forked processes to finish (should take ~3s)
+    # Wait for all forked processes to finish
     Process.waitall
 
     total = Workhorse::DbJob.count
     succeeded = Workhorse::DbJob.succeeded.count
-    used_workers = Workhorse::DbJob.lock.pluck(:locked_by).uniq.size
+    # In a transaction, as Oracle commits a FOR UPDATE outside of one before
+    # its rows are fetched and then fails with "fetch out of sequence".
+    used_workers = Workhorse::DbJob.transaction do
+      Workhorse::DbJob.lock.pluck(:locked_by).uniq.size
+    end
 
     # Make sure there are 100 jobs, all jobs have succeeded and that all of the
-    # workers have had their turn.
+    # workers have had their turn. The jobs that have not are listed, as all
+    # a count says is that something got stuck, not where.
+    unfinished = Workhorse::DbJob.where.not(state: 'succeeded').order(:id).map do |job|
+      "##{job.id} #{job.state}, locked_by #{job.locked_by.inspect}, " \
+        "locked_at #{job.locked_at.inspect}, started_at #{job.started_at.inspect}"
+    end
+
     assert_equal 100, total
-    assert_equal 100, succeeded
+    assert_equal 100, succeeded, "Jobs that did not succeed:\n#{unfinished.join("\n")}"
     assert_equal 10,  used_workers
   end
 
+  # A poll that finds the global lock taken waits at least as long as it asked
+  # to before giving up. Below half a second is what the poller asks for with
+  # a short polling interval, and what MySQL and Oracle, which take the timeout
+  # as whole seconds, used to round down to not waiting at all.
+  def test_contended_global_lock_waits_for_its_timeout
+    poller = Workhorse::Worker.new(polling_interval: 0.3).poller
+    ran = false
+
+    elapsed = with_global_lock_held do
+      started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      poller.send(:with_global_lock, timeout: 0.3, count_failures: false) { ran = true }
+      Process.clock_gettime(Process::CLOCK_MONOTONIC) - started
+    end
+
+    assert_not ran
+    assert_operator elapsed, :>=, 0.25
+  end
+
   def test_connection_loss
-    # rubocop: disable Style/GlobalVars
+    # rubocop: disable-next Style/GlobalVars
     $thread_conn = nil
-    # rubocop: enable Style/GlobalVars
 
     Workhorse.enqueue BasicJob.new(sleep_time: 3)
 
@@ -199,7 +259,14 @@ class Workhorse::PollerTest < WorkhorseTest
       Workhorse.clean_stuck_jobs = clean
       with_daemon do
         Workhorse.enqueue BasicJob.new(sleep_time: 5)
-        sleep 0.2
+
+        # Waited for rather than slept on: until a worker has taken the job,
+        # locked_by is empty and the cleanup, which matches on the host in it,
+        # cannot recognise the job as one of its own.
+        with_retries do
+          assert_equal 'started', Workhorse::DbJob.first.state
+        end
+
         kill_deamon_workers
 
         assert_equal 1, Workhorse::DbJob.count
@@ -265,6 +332,13 @@ class Workhorse::PollerTest < WorkhorseTest
 
   private
 
+  # The worker's valid queues, the queueless one first. Sorted here, as they
+  # come from a SELECT DISTINCT without ORDER BY: MySQL happens to return NULL
+  # first, Oracle does not, and the poller does not depend on the order.
+  def valid_queues_of(worker)
+    return worker.poller.send(:valid_queues).sort_by { |queue| [queue.nil? ? 0 : 1, queue.to_s] }
+  end
+
   def kill_deamon_workers
     pids = daemon.workers.map(&:pid)
     pids.each do |pid|