workhorse 1.2.17.rc0 → 1.2.17.rc2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 25908f294cd9623ebde44ec8bba5181ee54965703f4fb93df18876b216718eb8
-  data.tar.gz: 568e620571a9ded165fee1fc4420efbb6810f073edd4365c6c8efbe7c3676052
+  metadata.gz: ce2281ae514ba5f269328c38dd0a2e9bf0f76e2c26be09f4be467b1f818f5dec
+  data.tar.gz: 3315c9047af3bf707d23d5309b65a602474342f6c0081f329cd0582d18b7720f
 SHA512:
-  metadata.gz: 83b6ac441e3762f251e35f04dd213044d6fb904a005155235546391cdecd4f9c0c1db24be7c30371da4704b27e6f18aa6ce0b448b6c6c1dbc8c77eeb81faa9e3
-  data.tar.gz: 9c2940a6bfb7cd0be49170d81af2378f27395dbf597f6d8fa2c777bdc812b26d06cc849454ba5925fcf5f26b7724779b3d6e941bc6c9237d6dc251b7d071188f
+  metadata.gz: 2047809f4fe65fca5f5eaee641b5868846098e381a18d5d1d07a4a9fe3e0d94a709185b13e7cf79c3094eaed5f908aac7399abcb4488a13b288379f414a8fdf7
+  data.tar.gz: fdca924627efb8ab50938b43ac76873d5bc0a5e794484a8a1e1b7fa4afba4fe86aa4003c2ccba6ff63b9a977503f9a63f818e536bc07bc34c987be52b3580793

data/CHANGELOG.md

@@ -1,5 +1,30 @@
 # Workhorse Changelog
 
+## 1.2.17.rc2 - 2024-02-08
+
+* Remove unnecessary output from `watch` command in certain cases.
+
+  Sitrox reference: #121312.
+
+* Fix, improve and extend automated tests.
+
+## 1.2.17.rc1 - 2024-02-08
+
+* Revamp memory handling:
+
+  * Change memory handling for workers to automatically shut down themselves
+    upon exceeding `config.max_worker_memory_mb` (if configured and > 0),
+    triggering the creation of a shutdown file
+    (`tmp/pids/workhorse.<pid>.shutdown`).
+  * Have the `watch` command, if scheduled, silently restart the shutdown worker
+    and remove the shutdown file.
+  * The presence of the shutdown file informs the watcher to produce output or
+    remain silent.
+  * Implement this adjustment to limit `watch` command output to error cases,
+    facilitating seamless cron integration for notification purposes.
+
+  Sitrox reference: #121312.
+
 ## 1.2.17.rc0 - 2024-02-05
 
 * Add option `config.max_worker_memory_mb` for automatic restart of workers

data/README.md

@@ -466,17 +466,18 @@ succeeded jobs. You can run this using your scheduler in a specific interval.
 
 ## Memory handling
 
-When dealing with jobs that may exhibit a large memory footprint, it's important
-to note that Ruby might not release consumed memory back to the operating
-system. Consequently, your job workers could accumulate a significant amount of
-memory over time. To address this, Workhorse provides the
-`config.max_worker_memory_mb` option.
-
-If `config.max_worker_memory_mb` is set to a value above `0`, the `watch`
-command will check the memory footprint (RSS / resident size) of all worker
-processes. If any worker exceeds the specified footprint, Workhorse will
-silently restart it to ensure proper memory release. This process does not
-produce any output in the `watch` command.
+When a worker exceeds the memory limit specified by
+`config.max_worker_memory_mb` (assuming it is configured and > 0), it initiates
+a graceful shutdown process by creating a shutdown file named
+`tmp/pids/workhorse.<pid>.shutdown`.
+
+Simultaneously, the `watch` command, if scheduled, monitors the presence of this
+shutdown file. Upon detecting its existence, it silently triggers the restart of
+the shutdown worker and removes the shutdown file to signify that the restart
+process has begun.
+
+This mechanism ensures that workers are automatically restarted without manual
+intervention when memory limits are exceeded.
 
 Example configuration:
 

data/VERSION

@@ -1 +1 @@
-1.2.17.rc0
+1.2.17.rc2

data/lib/workhorse/daemon.rb

@@ -45,39 +45,56 @@ module Workhorse
     def start(quiet: false)
       code = 0
 
+      # Holds messages in format [[<message>, <severity>]]
+      messages = []
+
       for_each_worker do |worker|
-        pid_file, pid = read_pid(worker)
+        pid_file, pid, active = read_pid(worker)
 
-        if pid_file && pid
-          warn "Worker ##{worker.id} (#{worker.name}): Already started (PID #{pid})" unless quiet
+        if pid_file && pid && active
+          messages << ["Worker ##{worker.id} (#{worker.name}): Already started (PID #{pid})", 2] unless quiet
           code = 2
         elsif pid_file
           File.delete pid_file
-          puts "Worker ##{worker.id} (#{worker.name}): Starting (stale pid file)" unless quiet
+
+          shutdown_file = pid ? Workhorse::Worker.shutdown_file_for(pid) : nil
+          shutdown_file = nil if shutdown_file && !File.exist?(shutdown_file)
+
+          messages << ["Worker ##{worker.id} (#{worker.name}): Starting (stale pid file)", 1] unless quiet || shutdown_file
           start_worker worker
+          FileUtils.rm(shutdown_file) if shutdown_file
         else
-          warn "Worker ##{worker.id} (#{worker.name}): Starting" unless quiet
+          messages << ["Worker ##{worker.id} (#{worker.name}): Starting", 1] unless quiet
           start_worker worker
         end
       end
 
+      if messages.any?
+        min = messages.min_by(&:last)[1]
+
+        # Only print messages if there is at least one message with severity 1
+        if min == 1
+          messages.each { |(message, _severity)| warn message }
+        end
+      end
+
       return code
     end
 
-    def stop(kill = false)
+    def stop(kill = false, quiet: false)
       code = 0
 
       for_each_worker do |worker|
-        pid_file, pid = read_pid(worker)
+        pid_file, pid, active = read_pid(worker)
 
-        if pid_file && pid
-          puts "Worker (#{worker.name}) ##{worker.id}: Stopping"
+        if pid_file && pid && active
+          puts "Worker (#{worker.name}) ##{worker.id}: Stopping" unless quiet
           stop_worker pid_file, pid, kill: kill
         elsif pid_file
           File.delete pid_file
-          puts "Worker (#{worker.name}) ##{worker.id}: Already stopped (stale PID file)"
+          puts "Worker (#{worker.name}) ##{worker.id}: Already stopped (stale PID file)" unless quiet
         else
-          warn "Worker (#{worker.name}) ##{worker.id}: Already stopped"
+          warn "Worker (#{worker.name}) ##{worker.id}: Already stopped" unless quiet
           code = 2
         end
       end
@@ -89,9 +106,9 @@ module Workhorse
       code = 0
 
       for_each_worker do |worker|
-        pid_file, pid = read_pid(worker)
+        pid_file, pid, active = read_pid(worker)
 
-        if pid_file && pid
+        if pid_file && pid && active
           puts "Worker ##{worker.id} (#{worker.name}): Running" unless quiet
         elsif pid_file
           warn "Worker ##{worker.id} (#{worker.name}): Not running (stale PID file)" unless quiet
@@ -113,14 +130,10 @@ module Workhorse
       end
 
       if should_be_running && status(quiet: true) != 0
-        code = start(quiet: Workhorse.silence_watcher)
+        return start(quiet: Workhorse.silence_watcher)
       else
-        code = 0
+        return 0
       end
-
-      watch_memory! if should_be_running
-
-      return code
     end
 
     def restart
@@ -132,7 +145,9 @@ module Workhorse
       code = 0
 
       for_each_worker do |worker|
-        _pid_file, pid = read_pid(worker)
+        _pid_file, pid, active = read_pid(worker)
+
+        next unless pid && active
 
         begin
           Process.kill 'HUP', pid
@@ -148,30 +163,6 @@ module Workhorse
 
     private
 
-    def watch_memory!
-      return if Workhorse.max_worker_memory_mb == 0
-
-      for_each_worker do |worker|
-        pid_file, pid = read_pid(worker)
-        next unless pid_file && pid
-
-        memory = memory_for(pid)
-        next unless memory
-
-        if memory > Workhorse.max_worker_memory_mb
-          stop_worker pid_file, pid
-          start_worker worker
-        end
-      end
-    end
-
-    # Returns the memory (RSS) in MB for the given process.
-    def memory_for(pid)
-      mem = `ps -p #{pid} -o rss=`&.strip
-      return nil if mem.blank?
-      return mem.to_i / 1024
-    end
-
     def for_each_worker(&block)
       @workers.each(&block)
     end
@@ -237,16 +228,21 @@ module Workhorse
 
     def read_pid(worker)
       file = pid_file_for(worker)
+      pid = nil
+      active = false
 
       if File.exist?(file)
         raw_pid = File.read(file)
-        return nil, nil if raw_pid.blank?
 
-        pid = Integer(raw_pid)
-        return file, process?(pid) ? pid : nil
+        unless raw_pid.blank?
+          pid = Integer(raw_pid)
+          active = process?(pid)
+        end
       else
         return nil, nil
       end
+
+      return file, pid, active
     end
   end
 end

data/lib/workhorse/performer.rb

@@ -22,7 +22,7 @@ module Workhorse
       Thread.current[:workhorse_current_performer] = self
 
       ActiveRecord::Base.connection_pool.with_connection do
-        if defined?(Rails) && Rails.application && Rails.application.respond_to?(:executor)
+        if defined?(Rails) && Rails.respond_to?(:application) && Rails.application && Rails.application.respond_to?(:executor)
           Rails.application.executor.wrap do
             perform_wrapped
           end
@@ -63,7 +63,6 @@ module Workhorse
 
       inner_job_class = deserialized_job.try(:job_class) || deserialized_job.class
       skip_tx = inner_job_class.try(:skip_tx?)
-      log "SKIP TX: #{skip_tx.inspect}".red, :error
 
       if Workhorse.perform_jobs_in_tx && !skip_tx
         Workhorse.tx_callback.call do

data/lib/workhorse/poller.rb

@@ -9,7 +9,7 @@ module Workhorse
     attr_reader :worker
     attr_reader :table
 
-    def initialize(worker)
+    def initialize(worker, before_poll = proc { true })
       @worker = worker
       @running = false
       @table = Workhorse::DbJob.arel_table
@@ -17,6 +17,7 @@ module Workhorse
       @instant_repoll = Concurrent::AtomicBoolean.new(false)
       @global_lock_fails = 0
       @max_global_lock_fails_reached = false
+      @before_poll = before_poll
     end
 
     def running?
@@ -34,6 +35,12 @@ module Workhorse
           break unless running?
 
           begin
+            unless @before_poll.call
+              Thread.new { worker.shutdown }
+              sleep
+              next
+            end
+
             poll
             sleep
           rescue Exception => e

data/lib/workhorse/pool.rb

@@ -2,6 +2,7 @@ module Workhorse
   # Abstraction layer of a simple thread pool implementation used by the worker.
   class Pool
     attr_reader :mutex
+    attr_reader :active_threads
 
     def initialize(size)
       @size = size

data/lib/workhorse/worker.rb

@@ -20,6 +20,12 @@ module Workhorse
       worker.wait
     end
 
+    # @private
+    def self.shutdown_file_for(pid)
+      return nil unless defined?(Rails)
+      Rails.root.join('tmp', 'pids', "workhorse.#{pid}.shutdown")
+    end
+
     # Instantiates a new worker. The worker is not automatically started.
     #
     # @param queues [Array] The queues you want this worker to process. If an
@@ -51,7 +57,7 @@ module Workhorse
 
       @mutex = Mutex.new
       @pool = Pool.new(@pool_size)
-      @poller = Workhorse::Poller.new(self)
+      @poller = Workhorse::Poller.new(self, proc { check_memory })
       @logger = logger
 
       unless (@polling_interval / 0.1).round(2).modulo(1).zero?
@@ -155,6 +161,37 @@ module Workhorse
 
     private
 
+    def check_memory
+      mem = current_memory_consumption
+
+      unless mem
+        log "Could not determine memory consumption of worker with pid #{pid}"
+        return false
+      end
+
+      max = Workhorse.max_worker_memory_mb
+      exceeded = max > 0 && current_memory_consumption > max
+
+      return true unless exceeded
+
+      if defined?(Rails)
+        FileUtils.touch self.class.shutdown_file_for(pid)
+      end
+
+      log "Worker process #{id.inspect} memory consumption (RSS) of #{mem}MB exceeds "\
+          "configured per-worker limit of #{max}MB and is now being shut down. Make sure "\
+          'that your worker processes are watched (e.g. using the "watch"-command) for ' \
+          'this worker to be restarted automatically.'
+
+      return false
+    end
+
+    def current_memory_consumption
+      mem = `ps -p #{pid} -o rss=`&.strip
+      return nil if mem.blank?
+      return mem.to_i / 1024
+    end
+
     def check_rails_env
       unless Rails.env.production?
         warn 'WARNING: Always run workhorse workers in production environment. Other environments can lead to unexpected behavior.'

data/test/lib/jobs.rb

@@ -36,6 +36,15 @@ class SyntaxErrorJob
   end
 end
 
+class MemHungryJob
+  class_attribute :data
+
+  # Should consume roughly 1GB of memory.
+  def perform
+    self.class.data = 'x' * 250.megabytes
+  end
+end
+
 class DummyRailsOpsOp
   class_attribute :results
   self.results = Concurrent::Array.new

data/test/lib/test_helper.rb

@@ -5,15 +5,64 @@ require 'pry'
 require 'colorize'
 require 'mysql2'
 require 'benchmark'
+require 'concurrent'
 require 'jobs'
 
+class MockRailsEnv < String
+  def production?
+    self == 'production'
+  end
+
+  def test?
+    self == 'test'
+  end
+
+  def development?
+    self == 'development'
+  end
+end
+
+class Rails
+  def self.root
+    Pathname.new(File.expand_path(File.join(File.dirname(__FILE__), '../../')))
+  end
+
+  def self.env
+    MockRailsEnv.new('production')
+  end
+end
+
 class WorkhorseTest < ActiveSupport::TestCase
   def setup
+    remove_pids!
+    Workhorse.silence_watcher = true
     Workhorse::DbJob.delete_all
   end
 
   protected
 
+  attr_reader :daemon
+
+  def remove_pids!
+    Dir[Rails.root.join('tmp', 'pids', '*')].each do |file|
+      FileUtils.rm file
+    end
+  end
+
+  def kill(pid)
+    signals = %w[TERM INT]
+
+    loop do
+      begin
+        signals.each { |signal| Process.kill(signal, pid) }
+      rescue Errno::ESRCH
+        break
+      end
+
+      sleep 0.5
+    end
+  end
+
   def capture_log(level: :debug)
     io = StringIO.new
     logger = Logger.new(io, level: level)
@@ -31,6 +80,14 @@ class WorkhorseTest < ActiveSupport::TestCase
     end
   end
 
+  def work_until(max: 50, interval: 0.1, **options, &block)
+    w = Workhorse::Worker.new(**options)
+    w.start
+    return with_retries(max, interval: interval, &block)
+  ensure
+    w.shutdown
+  end
+
   def with_worker(options = {})
     w = Workhorse::Worker.new(**options)
     w.start
@@ -40,6 +97,45 @@ class WorkhorseTest < ActiveSupport::TestCase
       w.shutdown
     end
   end
+
+  def with_daemon(workers = 1, &_block)
+    @daemon = Workhorse::Daemon.new(pidfile: 'tmp/pids/test%s.pid') do |d|
+      workers.times do |i|
+        d.worker "Test Worker #{i}" do
+          begin
+            Workhorse::Worker.start_and_wait(
+              pool_size:        1,
+              polling_interval: 0.1
+            )
+          end
+        end
+      end
+    end
+    daemon.start(quiet: true)
+    yield @daemon
+  ensure
+    daemon.stop(quiet: true)
+  end
+
+  def with_retries(max = 50, interval: 0.1, &_block)
+    runs = 0
+
+    loop do
+      return yield
+    rescue Minitest::Assertion
+      fail if runs > max
+      sleep interval
+      runs += 1
+    end
+  end
+
+  def capture_stderr
+    old, $stderr = $stderr, StringIO.new
+    yield
+    $stderr.string
+  ensure
+    $stderr = old
+  end
 end
 
 ActiveRecord::Base.establish_connection(

data/test/workhorse/daemon_test.rb

@@ -0,0 +1,92 @@
+require 'test_helper'
+
+class Workhorse::DaemonTest < WorkhorseTest
+  def setup
+    remove_pids!
+  end
+
+  def test_watch_all_ok
+    with_daemon 2 do
+      assert_watch_output
+    end
+  end
+
+  def test_watch_starting_stale_pid
+    with_daemon 2 do
+      # Kill first worker
+      Process.kill 'KILL', daemon.workers.first.pid
+
+      # Watch
+      assert_watch_output(
+        'Worker #1 (Test Worker 0): Starting (stale pid file)',
+        "Worker #2 (Test Worker 1): Already started (PID #{daemon.workers.second.pid})"
+      )
+    end
+  end
+
+  def test_watch_starting_missing_pid
+    with_daemon 2 do
+      # Kill first worker
+      kill daemon.workers.first.pid
+      FileUtils.rm "tmp/pids/test1.pid"
+
+      # Watch
+      assert_watch_output(
+        'Worker #1 (Test Worker 0): Starting',
+        "Worker #2 (Test Worker 1): Already started (PID #{daemon.workers.second.pid})"
+      )
+    end
+  end
+
+  def test_watch_controlled_shutdown
+    with_daemon 2 do
+      # Kill first worker
+      kill daemon.workers.first.pid
+      FileUtils.touch "tmp/pids/workhorse.#{daemon.workers.first.pid}.shutdown"
+
+      # Watch
+      assert_watch_output
+    end
+
+    assert_not File.exist?("tmp/pids/workhorse.#{daemon.workers.first.pid}.shutdown")
+  end
+
+  def test_watch_mixed
+    # Worker 0: Kill, remove PID
+    # Worker 1: Kill, keep PID
+    # Worker 2: Keep
+    # Worker 3: Controlled shutdown
+    with_daemon 4 do
+      # Worker 0: Kill, remove PID
+      kill daemon.workers[0].pid
+      FileUtils.rm "tmp/pids/test1.pid"
+
+      # Worker 1: Kill, keep PID
+      kill daemon.workers[1].pid
+
+      # Worker 3: Controlled shutdown
+      kill daemon.workers[3].pid
+      FileUtils.touch "tmp/pids/workhorse.#{daemon.workers.first.pid}.shutdown"
+
+      # Watch
+      assert_watch_output(
+        'Worker #1 (Test Worker 0): Starting',
+        'Worker #2 (Test Worker 1): Starting (stale pid file)',
+        "Worker #3 (Test Worker 2): Already started (PID #{daemon.workers[2].pid})",
+        'Worker #4 (Test Worker 3): Starting (stale pid file)'
+      )
+    end
+
+    assert_not File.exist?("tmp/pids/workhorse.#{daemon.workers.first.pid}.shutdown")
+  end
+
+  private
+
+  def assert_watch_output(*expected_lines)
+    silence_watcher_was = Workhorse.silence_watcher
+    Workhorse.silence_watcher = false
+    assert_equal expected_lines, capture_stderr { daemon.watch }.lines.map(&:chomp)
+  ensure
+    Workhorse.silence_watcher = silence_watcher_was
+  end
+end

data/test/workhorse/poller_test.rb

@@ -182,31 +182,32 @@ class Workhorse::PollerTest < WorkhorseTest
       Workhorse::DbJob.delete_all
 
       Workhorse.clean_stuck_jobs = clean
-      start_deamon
-      Workhorse.enqueue BasicJob.new(sleep_time: 5)
-      sleep 0.2
-      kill_deamon_workers
-
-      assert_equal 1, Workhorse::DbJob.count
-
-      Workhorse::DbJob.first.update(
-        state:      'locked',
-        started_at: nil
-      )
-
-      Workhorse::Worker.new.poller.send(:clean_stuck_jobs!) if clean
-
-      assert_equal 1, Workhorse::DbJob.count
-
-      Workhorse::DbJob.first.tap do |job|
-        if clean
-          assert_equal 'waiting', job.state
-          assert_nil job.locked_at
-          assert_nil job.locked_by
-          assert_nil job.started_at
-          assert_nil job.last_error
-        else
-          assert_equal 'locked', job.state
+      with_daemon do
+        Workhorse.enqueue BasicJob.new(sleep_time: 5)
+        sleep 0.2
+        kill_deamon_workers
+
+        assert_equal 1, Workhorse::DbJob.count
+
+        Workhorse::DbJob.first.update(
+          state:      'locked',
+          started_at: nil
+        )
+
+        Workhorse::Worker.new.poller.send(:clean_stuck_jobs!) if clean
+
+        assert_equal 1, Workhorse::DbJob.count
+
+        Workhorse::DbJob.first.tap do |job|
+          if clean
+            assert_equal 'waiting', job.state
+            assert_nil job.locked_at
+            assert_nil job.locked_by
+            assert_nil job.started_at
+            assert_nil job.last_error
+          else
+            assert_equal 'locked', job.state
+          end
         end
       end
     ensure
@@ -219,25 +220,26 @@ class Workhorse::PollerTest < WorkhorseTest
       Workhorse::DbJob.delete_all
 
       Workhorse.clean_stuck_jobs = true
-      start_deamon
-      Workhorse.enqueue BasicJob.new(sleep_time: 5)
-      sleep 0.2
-      kill_deamon_workers
-
-      assert_equal 1, Workhorse::DbJob.count
-      assert_equal 'started', Workhorse::DbJob.first.state
-
-      work 0.1 if clean
-
-      assert_equal 1, Workhorse::DbJob.count
-
-      Workhorse::DbJob.first.tap do |job|
-        if clean
-          assert_equal 'failed', job.state
-          assert_match(/started by PID #{@daemon.workers.first.pid}/, job.last_error)
-          assert_match(/on host #{Socket.gethostname}/, job.last_error)
-        else
-          assert_equal 'started', job.state
+      with_daemon do
+        Workhorse.enqueue BasicJob.new(sleep_time: 5)
+        sleep 0.2
+        kill_deamon_workers
+
+        assert_equal 1, Workhorse::DbJob.count
+        assert_equal 'started', Workhorse::DbJob.first.state
+
+        work 0.1 if clean
+
+        assert_equal 1, Workhorse::DbJob.count
+
+        Workhorse::DbJob.first.tap do |job|
+          if clean
+            assert_equal 'failed', job.state
+            assert_match(/started by PID #{daemon.workers.first.pid}/, job.last_error)
+            assert_match(/on host #{Socket.gethostname}/, job.last_error)
+          else
+            assert_equal 'started', job.state
+          end
         end
       end
     ensure
@@ -248,23 +250,11 @@ class Workhorse::PollerTest < WorkhorseTest
   private
 
   def kill_deamon_workers
-    @daemon.workers.each do |worker|
+    daemon.workers.each do |worker|
       Process.kill 'KILL', worker.pid
     end
   end
 
-  def start_deamon
-    @daemon = Workhorse::Daemon.new(pidfile: 'tmp/pids/test%s.pid') do |d|
-      d.worker 'Test Worker' do
-        Workhorse::Worker.start_and_wait(
-          pool_size:        1,
-          polling_interval: 0.1
-        )
-      end
-    end
-    @daemon.start
-  end
-
   def setup
     Workhorse::DbJob.delete_all
   end

data/test/workhorse/worker_test.rb

@@ -180,8 +180,60 @@ class Workhorse::WorkerTest < WorkhorseTest
     assert_equal 'waiting',   jobs[1].state
   end
 
+  def test_controlled_shutdown
+    Workhorse.max_worker_memory_mb = 50
+    with_daemon do
+      pid = with_retries do
+        pid = daemon.workers.first.pid
+        assert_process(pid)
+        pid
+      end
+
+      10.times do
+        Workhorse.enqueue BasicJob.new(sleep_time: 0.1)
+
+        with_retries do
+          assert_equal 'succeeded', Workhorse::DbJob.first.state
+          Workhorse::DbJob.delete_all
+        end
+      end
+
+      Workhorse.enqueue MemHungryJob.new
+
+      with_retries do
+        assert_equal 'succeeded', Workhorse::DbJob.first.state
+
+        assert File.exist?("tmp/pids/workhorse.#{pid}.shutdown")
+        assert_not_process pid
+      end
+
+      capture_stderr { daemon.watch }
+
+      with_retries do
+        assert_not File.exist?("tmp/pids/workhorse.#{pid}.shutdown")
+      end
+    end
+  ensure
+    Workhorse.max_worker_memory_mb = 0
+  end
+
   private
 
+  def assert_process(pid)
+    assert process?(pid), "Process #{pid} expected to be running"
+  end
+
+  def assert_not_process(pid)
+    assert_not process?(pid), "Process #{pid} expected to be stopped"
+  end
+
+  def process?(pid)
+    Process.kill(0, pid)
+    true
+  rescue Errno::EPERM, Errno::ESRCH
+    false
+  end
+
   def enqueue_in_multiple_queues
     Workhorse.enqueue BasicJob.new(some_param: nil)
     Workhorse.enqueue BasicJob.new(some_param: :q1), queue: :q1

data/workhorse.gemspec

@@ -1,18 +1,18 @@
 # -*- encoding: utf-8 -*-
-# stub: workhorse 1.2.17.rc0 ruby lib
+# stub: workhorse 1.2.17.rc2 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "workhorse".freeze
-  s.version = "1.2.17.rc0"
+  s.version = "1.2.17.rc2"
 
   s.required_rubygems_version = Gem::Requirement.new("> 1.3.1".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]
   s.authors = ["Sitrox".freeze]
-  s.date = "2024-02-05"
-  s.files = [".github/workflows/ruby.yml".freeze, ".gitignore".freeze, ".releaser_config".freeze, ".rubocop.yml".freeze, "CHANGELOG.md".freeze, "FAQ.md".freeze, "Gemfile".freeze, "LICENSE".freeze, "README.md".freeze, "RUBY_VERSION".freeze, "Rakefile".freeze, "VERSION".freeze, "bin/rubocop".freeze, "lib/active_job/queue_adapters/workhorse_adapter.rb".freeze, "lib/generators/workhorse/install_generator.rb".freeze, "lib/generators/workhorse/templates/bin/workhorse.rb".freeze, "lib/generators/workhorse/templates/config/initializers/workhorse.rb".freeze, "lib/generators/workhorse/templates/create_table_jobs.rb".freeze, "lib/workhorse.rb".freeze, "lib/workhorse/active_job_extension.rb".freeze, "lib/workhorse/daemon.rb".freeze, "lib/workhorse/daemon/shell_handler.rb".freeze, "lib/workhorse/db_job.rb".freeze, "lib/workhorse/enqueuer.rb".freeze, "lib/workhorse/jobs/cleanup_succeeded_jobs.rb".freeze, "lib/workhorse/jobs/detect_stale_jobs_job.rb".freeze, "lib/workhorse/jobs/run_active_job.rb".freeze, "lib/workhorse/jobs/run_rails_op.rb".freeze, "lib/workhorse/performer.rb".freeze, "lib/workhorse/poller.rb".freeze, "lib/workhorse/pool.rb".freeze, "lib/workhorse/scoped_env.rb".freeze, "lib/workhorse/worker.rb".freeze, "test/active_job/queue_adapters/workhorse_adapter_test.rb".freeze, "test/lib/db_schema.rb".freeze, "test/lib/jobs.rb".freeze, "test/lib/test_helper.rb".freeze, "test/workhorse/db_job_test.rb".freeze, "test/workhorse/enqueuer_test.rb".freeze, "test/workhorse/performer_test.rb".freeze, "test/workhorse/poller_test.rb".freeze, "test/workhorse/pool_test.rb".freeze, "test/workhorse/worker_test.rb".freeze, "workhorse.gemspec".freeze]
+  s.date = "2024-02-08"
+  s.files = [".github/workflows/ruby.yml".freeze, ".gitignore".freeze, ".releaser_config".freeze, ".rubocop.yml".freeze, "CHANGELOG.md".freeze, "FAQ.md".freeze, "Gemfile".freeze, "LICENSE".freeze, "README.md".freeze, "RUBY_VERSION".freeze, "Rakefile".freeze, "VERSION".freeze, "bin/rubocop".freeze, "lib/active_job/queue_adapters/workhorse_adapter.rb".freeze, "lib/generators/workhorse/install_generator.rb".freeze, "lib/generators/workhorse/templates/bin/workhorse.rb".freeze, "lib/generators/workhorse/templates/config/initializers/workhorse.rb".freeze, "lib/generators/workhorse/templates/create_table_jobs.rb".freeze, "lib/workhorse.rb".freeze, "lib/workhorse/active_job_extension.rb".freeze, "lib/workhorse/daemon.rb".freeze, "lib/workhorse/daemon/shell_handler.rb".freeze, "lib/workhorse/db_job.rb".freeze, "lib/workhorse/enqueuer.rb".freeze, "lib/workhorse/jobs/cleanup_succeeded_jobs.rb".freeze, "lib/workhorse/jobs/detect_stale_jobs_job.rb".freeze, "lib/workhorse/jobs/run_active_job.rb".freeze, "lib/workhorse/jobs/run_rails_op.rb".freeze, "lib/workhorse/performer.rb".freeze, "lib/workhorse/poller.rb".freeze, "lib/workhorse/pool.rb".freeze, "lib/workhorse/scoped_env.rb".freeze, "lib/workhorse/worker.rb".freeze, "test/active_job/queue_adapters/workhorse_adapter_test.rb".freeze, "test/lib/db_schema.rb".freeze, "test/lib/jobs.rb".freeze, "test/lib/test_helper.rb".freeze, "test/workhorse/daemon_test.rb".freeze, "test/workhorse/db_job_test.rb".freeze, "test/workhorse/enqueuer_test.rb".freeze, "test/workhorse/performer_test.rb".freeze, "test/workhorse/poller_test.rb".freeze, "test/workhorse/pool_test.rb".freeze, "test/workhorse/worker_test.rb".freeze, "workhorse.gemspec".freeze]
   s.rubygems_version = "3.4.6".freeze
   s.summary = "Multi-threaded job backend with database queuing for ruby.".freeze
-  s.test_files = ["test/active_job/queue_adapters/workhorse_adapter_test.rb".freeze, "test/lib/db_schema.rb".freeze, "test/lib/jobs.rb".freeze, "test/lib/test_helper.rb".freeze, "test/workhorse/db_job_test.rb".freeze, "test/workhorse/enqueuer_test.rb".freeze, "test/workhorse/performer_test.rb".freeze, "test/workhorse/poller_test.rb".freeze, "test/workhorse/pool_test.rb".freeze, "test/workhorse/worker_test.rb".freeze]
+  s.test_files = ["test/active_job/queue_adapters/workhorse_adapter_test.rb".freeze, "test/lib/db_schema.rb".freeze, "test/lib/jobs.rb".freeze, "test/lib/test_helper.rb".freeze, "test/workhorse/daemon_test.rb".freeze, "test/workhorse/db_job_test.rb".freeze, "test/workhorse/enqueuer_test.rb".freeze, "test/workhorse/performer_test.rb".freeze, "test/workhorse/poller_test.rb".freeze, "test/workhorse/pool_test.rb".freeze, "test/workhorse/worker_test.rb".freeze]
 
   s.specification_version = 4
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: workhorse
 version: !ruby/object:Gem::Version
-  version: 1.2.17.rc0
+  version: 1.2.17.rc2
 platform: ruby
 authors:
 - Sitrox
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-02-05 00:00:00.000000000 Z
+date: 2024-02-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -221,6 +221,7 @@ files:
 - test/lib/db_schema.rb
 - test/lib/jobs.rb
 - test/lib/test_helper.rb
+- test/workhorse/daemon_test.rb
 - test/workhorse/db_job_test.rb
 - test/workhorse/enqueuer_test.rb
 - test/workhorse/performer_test.rb
@@ -255,6 +256,7 @@ test_files:
 - test/lib/db_schema.rb
 - test/lib/jobs.rb
 - test/lib/test_helper.rb
+- test/workhorse/daemon_test.rb
 - test/workhorse/db_job_test.rb
 - test/workhorse/enqueuer_test.rb
 - test/workhorse/performer_test.rb