workhorse 1.4.0 → 1.4.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dd78ce0b033690529240d1111c8545ab78dfd753fbe7fee551313b2a7d33fc26
-  data.tar.gz: 9e7e22992ed727f60882f448f0003318cd40142402590cf199812d461db58378
+  metadata.gz: 0facaea70a5e826980ea41034914a0609a22fb3ed93c9fbe4a7607eb5a3854c9
+  data.tar.gz: 35cc28b0a31206fa5f38df60858c5c2054c28fd7c47a4bf0b4983c9cbbcc02d0
 SHA512:
-  metadata.gz: e5d6560a8a6ef01c555222568eb0af6ad44c09484cfeee05093c3ad966e19ebe80f0d6754b85dccab869bb98ef279aede2dbc39da5f7a38b2b1d3eef054136f4
-  data.tar.gz: 2422ef7003b9dc3e3e3409a794afc57a4ae33226366292dadc89e4afe823289e2aa27c45f5c7b0586262344115d8ca1839b7e84d15980c5a69099c26f3c3c599
+  metadata.gz: 551453702ecb4b89060a7e5fdba4639eccbb60c09c85ebd2fd561eec91787c1b8166be81e81b2bd2da46f250ba70c8426198bca404d1cf1e5fbf1c6c9b5096ee
+  data.tar.gz: e1ee8b9ae1267271f8be2befd5455d90614e02c8708b2413bc90f957b8d5fb852b2f805ea4d42b6362844c2e7ed774e2a0a727e83687d570d0c430174e264227

data/CHANGELOG.md

@@ -1,5 +1,33 @@
 # Workhorse Changelog
 
+## 1.4.2 - 2026-02-20
+
+* Detach forked worker processes into their own session using `Process.setsid`.
+  Previously, when the ShellHandler was the session leader (e.g. started via
+  cron), its exit would cause the kernel to send `SIGHUP` to all forked workers,
+  potentially crashing them during startup before signal handlers were installed.
+
+* Add optional debug logging (`config.debug_log_path`) for diagnosing issues
+  with signal handling, process lifecycle, log rotation, and daemon commands.
+
+  Sitrox reference: #120574.
+
+## 1.4.1 - 2026-02-18
+
+* Close inherited lockfile fd in forked worker processes. Previously the
+  lockfile's file descriptor was inherited by children via `fork`, which could
+  prevent the POSIX `flock` from being released if the daemon process exited
+  abnormally.
+
+* Fix `watch` and `kill` commands to actually abort when the lock is
+  unavailable. Previously the `flock` return value with `LOCK_NB` was not
+  checked, so the commands would silently proceed without the lock.
+
+* Add error handling to the `HUP` signal handler for log reopening. Exceptions
+  from `logger.reopen` are now caught and reported via `on_exception`.
+
+  Sitrox reference: #120574.
+
 ## 1.4.0 - 2026-02-12
 
 * Stable release based on previous RC release.

data/README.md

@@ -522,7 +522,6 @@ Gem-internal model class `Workhorse::DbJob`, for example:
 
 ```ruby
 # config/initializers/workhorse.rb
-
 ActiveSupport.on_load :workhorse_db_job do
   # Code within this block will be run inside of the model class
   # Workhorse::DbJob.
@@ -530,6 +529,23 @@ ActiveSupport.on_load :workhorse_db_job do
 end
 ```
 
+## Debug logging
+
+Workhorse includes an optional debug log for diagnosing issues with signal
+handling, process lifecycle, log rotation, and daemon commands. To enable,
+set `debug_log_path` to a writable file path:
+
+```ruby
+# config/initializers/workhorse.rb
+Workhorse.setup do |config|
+  config.debug_log_path = Rails.root.join('log', 'workhorse.debug.log')
+end
+```
+
+The debug log is designed to be safe for production use: all writes are
+best-effort and silently ignore errors to avoid interfering with normal
+operation. Set `debug_log_path` to `nil` (the default) to disable.
+
 ## Caveats
 
 ### Errors during polling / crashed workers

data/VERSION

@@ -1 +1 @@
-1.4.0
+1.4.2

data/lib/workhorse/daemon/shell_handler.rb

@@ -1,5 +1,7 @@
 module Workhorse
   class Daemon::ShellHandler
+    class LockNotAvailableError < StandardError; end
+
     def self.run(**options, &block)
       unless ARGV.one?
         usage
@@ -14,28 +16,62 @@ module Workhorse
       begin
         case ARGV.first
         when 'start'
+          Workhorse.debug_log('ShellHandler: start command invoked')
           lockfile = acquire_lock(lockfile_path, File::LOCK_EX)
+          Workhorse.debug_log('ShellHandler: lock acquired for start')
+          daemon.lockfile = lockfile
           status = daemon.start
         when 'stop'
+          Workhorse.debug_log('ShellHandler: stop command invoked')
           lockfile = acquire_lock(lockfile_path, File::LOCK_EX)
+          Workhorse.debug_log('ShellHandler: lock acquired for stop')
+          daemon.lockfile = lockfile
           status = daemon.stop
         when 'kill'
-          lockfile = acquire_lock(lockfile_path, File::LOCK_EX | File::LOCK_NB)
-          status = daemon.stop(true)
+          Workhorse.debug_log('ShellHandler: kill command invoked')
+          begin
+            lockfile = acquire_lock(lockfile_path, File::LOCK_EX | File::LOCK_NB)
+            Workhorse.debug_log('ShellHandler: lock acquired for kill')
+            daemon.lockfile = lockfile
+            status = daemon.stop(true)
+          rescue LockNotAvailableError
+            Workhorse.debug_log('ShellHandler: lock not available for kill')
+            status = 1
+          end
         when 'status'
+          Workhorse.debug_log('ShellHandler: status command invoked')
           lockfile = acquire_lock(lockfile_path, File::LOCK_EX)
+          Workhorse.debug_log('ShellHandler: lock acquired for status')
+          daemon.lockfile = lockfile
           status = daemon.status
         when 'watch'
-          lockfile = acquire_lock(lockfile_path, File::LOCK_EX | File::LOCK_NB)
-          status = daemon.watch
+          Workhorse.debug_log('ShellHandler: watch command invoked')
+          begin
+            lockfile = acquire_lock(lockfile_path, File::LOCK_EX | File::LOCK_NB)
+            Workhorse.debug_log('ShellHandler: lock acquired for watch')
+            daemon.lockfile = lockfile
+            status = daemon.watch
+          rescue LockNotAvailableError
+            Workhorse.debug_log('ShellHandler: lock not available for watch')
+            status = 1
+          end
         when 'restart'
+          Workhorse.debug_log('ShellHandler: restart command invoked')
           lockfile = acquire_lock(lockfile_path, File::LOCK_EX)
+          Workhorse.debug_log('ShellHandler: lock acquired for restart')
+          daemon.lockfile = lockfile
           status = daemon.restart
         when 'restart-logging'
+          Workhorse.debug_log('ShellHandler: restart-logging command invoked')
           lockfile = acquire_lock(lockfile_path, File::LOCK_EX)
+          Workhorse.debug_log('ShellHandler: lock acquired for restart-logging')
+          daemon.lockfile = lockfile
           status = daemon.restart_logging
         when 'soft-restart'
+          Workhorse.debug_log('ShellHandler: soft-restart command invoked')
           lockfile = acquire_lock(lockfile_path, File::LOCK_EX)
+          Workhorse.debug_log('ShellHandler: lock acquired for soft-restart')
+          daemon.lockfile = lockfile
           status = daemon.soft_restart
         when 'usage'
           usage
@@ -48,7 +84,10 @@ module Workhorse
         warn "#{e.message}\n#{e.backtrace.join("\n")}"
         status = 99
       ensure
-        lockfile&.flock(File::LOCK_UN)
+        if lockfile
+          Workhorse.debug_log("ShellHandler: releasing lock for #{ARGV.first}")
+          lockfile.flock(File::LOCK_UN)
+        end
         exit! status
       end
     end
@@ -105,7 +144,12 @@ module Workhorse
     def self.acquire_lock(lockfile_path, flags)
       if Workhorse.lock_shell_commands
         lockfile = File.open(lockfile_path, 'a')
-        lockfile.flock(flags)
+        result = lockfile.flock(flags)
+
+        if result == false
+          lockfile.close
+          fail LockNotAvailableError, 'Could not acquire lock. Is another workhorse command already running?'
+        end
 
         return lockfile
       end

data/lib/workhorse/daemon.rb

@@ -34,6 +34,10 @@ module Workhorse
     # @private
     attr_reader :workers
 
+    # @return [File, nil] Lockfile handle to close in forked children
+    # @private
+    attr_accessor :lockfile
+
     # Creates a new daemon instance.
     #
     # @param pidfile [String, nil] Path template for PID files (use %i placeholder for worker ID)
@@ -197,20 +201,27 @@ module Workhorse
     def restart_logging
       code = 0
 
+      Workhorse.debug_log("restart_logging: sending HUP to #{@workers.count} worker(s)")
+
       for_each_worker do |worker|
         _pid_file, pid, active = read_pid(worker)
 
+        Workhorse.debug_log("restart_logging: worker ##{worker.id} (#{worker.name}): pid=#{pid.inspect}, active=#{active.inspect}")
+
         next unless pid && active
 
         begin
           Process.kill 'HUP', pid
+          Workhorse.debug_log("restart_logging: HUP sent successfully to PID #{pid}")
           puts "Worker (#{worker.name}) ##{worker.id}: Sent signal for restart-logging"
         rescue Errno::ESRCH
+          Workhorse.debug_log("restart_logging: HUP failed for PID #{pid}: process not found")
           warn "Worker (#{worker.name}) ##{worker.id}: Could not send signal for restart-logging, process not found"
           code = 2
         end
       end
 
+      Workhorse.debug_log("restart_logging: done, exit code=#{code}")
       return code
     end
 
@@ -223,20 +234,27 @@ module Workhorse
     def soft_restart
       code = 0
 
+      Workhorse.debug_log("Daemon: sending USR1 to #{@workers.count} worker(s)")
+
       for_each_worker do |worker|
         _pid_file, pid, active = read_pid(worker)
 
+        Workhorse.debug_log("Daemon soft_restart: worker ##{worker.id} (#{worker.name}): pid=#{pid.inspect}, active=#{active.inspect}")
+
         next unless pid && active
 
         begin
           Process.kill 'USR1', pid
+          Workhorse.debug_log("Daemon: USR1 sent successfully to PID #{pid}")
           puts "Worker (#{worker.name}) ##{worker.id}: Sent soft-restart signal"
         rescue Errno::ESRCH
+          Workhorse.debug_log("Daemon: USR1 failed for PID #{pid}: process not found")
           warn "Worker (#{worker.name}) ##{worker.id}: Process not found"
           code = 2
         end
       end
 
+      Workhorse.debug_log("Daemon soft_restart: done, exit code=#{code}")
       return code
     end
 
@@ -259,8 +277,16 @@ module Workhorse
     def start_worker(worker)
       check_rails_env if defined?(Rails)
 
+      Workhorse.debug_log("Daemon: forking worker ##{worker.id} (#{worker.name})")
       pid = fork do
+        # Detach from the parent's session so that the worker is not killed by
+        # SIGHUP when the parent (ShellHandler) exits. Without this, the kernel
+        # sends SIGHUP to the foreground process group when the session leader
+        # (e.g. a cron- or systemd-started ShellHandler) terminates.
+        Process.setsid
         $0 = process_name(worker)
+        # Close inherited lockfile fd to prevent holding the flock after parent exits
+        @lockfile&.close
         # Reopen pipes to prevent #107576
         $stdin.reopen File.open(File::NULL, 'r')
         null_out = File.open File::NULL, 'w'
@@ -272,6 +298,7 @@ module Workhorse
       worker.pid = pid
       File.write(pid_file_for(worker), pid)
       Process.detach(pid)
+      Workhorse.debug_log("Daemon: worker ##{worker.id} (#{worker.name}) forked with PID #{pid}")
     end
 
     # Stops a single worker process.
@@ -284,6 +311,7 @@ module Workhorse
     def stop_worker(pid_file, pid, kill: false)
       signals = kill ? %w[KILL] : %w[TERM INT]
 
+      Workhorse.debug_log("Daemon: stopping PID #{pid} with signals #{signals.join(', ')}")
       loop do
         begin
           signals.each { |signal| Process.kill(signal, pid) }
@@ -294,6 +322,7 @@ module Workhorse
         sleep 1
       end
 
+      Workhorse.debug_log("Daemon: PID #{pid} stopped")
       File.delete(pid_file)
     end
 

data/lib/workhorse/worker.rb

@@ -158,6 +158,8 @@ module Workhorse
         @poller.start
         log 'Started up'
 
+        Workhorse.debug_log("[Job worker #{id}] Started: PID=#{pid}, logger=#{describe_logger(logger)}")
+
         trap_termination if @auto_terminate
         trap_log_reopen
         trap_soft_restart
@@ -189,12 +191,14 @@ module Workhorse
       mutex.synchronize do
         assert_state! :running
 
+        Workhorse.debug_log("[Job worker #{id}] Shutdown starting")
         log 'Shutting down'
         @state = :shutdown
 
         @poller.shutdown
         @pool.shutdown
         log 'Shut down'
+        Workhorse.debug_log("[Job worker #{id}] Shutdown complete")
       end
     end
 
@@ -267,6 +271,8 @@ module Workhorse
 
       return true unless exceeded
 
+      Workhorse.debug_log("[Job worker #{id}] Memory limit exceeded: #{mem}MB > #{max}MB, initiating shutdown")
+
       if defined?(Rails)
         FileUtils.touch self.class.shutdown_file_for(pid)
       end
@@ -296,11 +302,16 @@ module Workhorse
     def trap_log_reopen
       Signal.trap(LOG_REOPEN_SIGNAL) do
         Thread.new do
-          logger.reopen
+          Workhorse.debug_log("[Job worker #{id}] HUP received, logger state before reopen: #{describe_logger(logger)}")
 
-          if defined?(ActiveRecord::Base) && ActiveRecord::Base.logger && ActiveRecord::Base.logger != logger
-            ActiveRecord::Base.logger.reopen
-          end
+          logger&.reopen
+          Workhorse.debug_log("[Job worker #{id}] Logger state after reopen: #{describe_logger(logger)}")
+
+          Workhorse.debug_log("[Job worker #{id}] HUP handling complete")
+        rescue Exception => e
+          Workhorse.debug_log("[Job worker #{id}] Logger reopen failed: #{e.class}: #{e.message}")
+          log %(Log reopen signal handler error: #{e.message}\n#{e.backtrace.join("\n")}), :error
+          Workhorse.on_exception.call(e)
         end.join
       end
     end
@@ -317,6 +328,7 @@ module Workhorse
           # quickly when called multiple times, this does not pose a risk of
           # keeping open a big number of "shutdown threads".
           Thread.new do
+            Workhorse.debug_log("[Job worker #{id}] #{signal} received, shutting down")
             log "\nCaught #{signal}, shutting worker down..."
             shutdown
           end.join
@@ -336,9 +348,14 @@ module Workhorse
 
       return unless @soft_restart_requested.make_true
 
+      Workhorse.debug_log("[Job worker #{id}] Soft restart initiated")
+
       # Create shutdown file for watch to detect
       shutdown_file = self.class.shutdown_file_for(pid)
-      FileUtils.touch(shutdown_file) if shutdown_file
+      if shutdown_file
+        FileUtils.touch(shutdown_file)
+        Workhorse.debug_log("[Job worker #{id}] Shutdown file created: #{shutdown_file}")
+      end
 
       # Monitor in a separate thread to avoid blocking the signal handler
       @soft_restart_thread = Thread.new do
@@ -358,6 +375,7 @@ module Workhorse
         # Start a new thread as certain functionality (such as logging) is not
         # available from within a trap context.
         Thread.new do
+          Workhorse.debug_log("[Job worker #{id}] #{SOFT_RESTART_SIGNAL} received, initiating soft restart")
           log "\nCaught #{SOFT_RESTART_SIGNAL}, initiating soft restart..."
           soft_restart
         rescue Exception => e
@@ -369,16 +387,52 @@ module Workhorse
       end
     end
 
+    # Returns a human-readable description of a logger's internal state.
+    # Used for debug logging to diagnose log rotation issues.
+    #
+    # @param lgr [Logger, nil] The logger to describe
+    # @return [String] Description of the logger's state
+    # @private
+    def describe_logger(lgr)
+      return 'nil' unless lgr
+
+      parts = ["class=#{lgr.class}"]
+
+      logdev = lgr.instance_variable_get(:@logdev)
+      if logdev
+        parts << "filename=#{logdev.filename.inspect}" if logdev.respond_to?(:filename)
+
+        dev = logdev.respond_to?(:dev) ? logdev.dev : nil
+        if dev
+          parts << "closed=#{dev.closed?}"
+          unless dev.closed?
+            fileno = dev.fileno
+            parts << "fd=#{fileno}"
+            fd_path = "/proc/self/fd/#{fileno}"
+            parts << "fd_target=#{File.readlink(fd_path).inspect}" if File.exist?(fd_path)
+          end
+        end
+      else
+        parts << 'logdev=nil'
+      end
+
+      parts.join(', ')
+    rescue Exception => e
+      "error describing logger: #{e.class}: #{e.message}"
+    end
+
     # Waits for all jobs to complete, then shuts down the worker.
     # Called asynchronously from soft_restart.
     #
     # @return [void]
     # @private
     def wait_for_idle_then_shutdown
+      Workhorse.debug_log("[Job worker #{id}] Waiting for idle before soft restart shutdown (pool_size=#{@pool_size})")
       loop do
         break if @state == :shutdown
 
         if idle == @pool_size
+          Workhorse.debug_log("[Job worker #{id}] All threads idle, proceeding with soft restart shutdown")
           log 'All jobs completed, shutting down for soft restart'
           shutdown
           break

data/lib/workhorse.rb

@@ -107,6 +107,31 @@ module Workhorse
   mattr_accessor :max_worker_memory_mb
   self.max_worker_memory_mb = 0
 
+  # Path to a debug log file for diagnosing log rotation and signal handling issues.
+  # When set, Workhorse writes timestamped debug entries to this file at key points
+  # (worker startup, HUP signal handling, restart-logging command flow).
+  # Set to nil to disable (default).
+  #
+  # @return [String, nil] Path to debug log file
+  mattr_accessor :debug_log_path
+  self.debug_log_path = nil
+
+  # Writes a debug message to the debug log file.
+  # Does nothing if {.debug_log_path} is nil.
+  # Silently ignores all exceptions to avoid interfering with normal operation.
+  #
+  # @param message [String] The message to log
+  # @return [void]
+  def self.debug_log(message)
+    return unless debug_log_path
+
+    File.open(debug_log_path, 'a') do |f|
+      f.write("[#{Time.now.iso8601(3)}] [PID #{Process.pid}] #{message}\n")
+      f.flush
+    end
+  rescue Exception # rubocop:disable Lint/SuppressedException
+  end
+
   # Configuration method for setting up Workhorse options.
   #
   # @yield [self] Configuration block

data/workhorse.gemspec

@@ -1,14 +1,14 @@
 # -*- encoding: utf-8 -*-
-# stub: workhorse 1.4.0 ruby lib
+# stub: workhorse 1.4.2 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "workhorse".freeze
-  s.version = "1.4.0"
+  s.version = "1.4.2"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]
   s.authors = ["Sitrox".freeze]
-  s.date = "2026-02-12"
+  s.date = "2026-02-20"
   s.files = [".github/workflows/ruby.yml".freeze, ".gitignore".freeze, ".releaser_config".freeze, ".rubocop.yml".freeze, "CHANGELOG.md".freeze, "FAQ.md".freeze, "Gemfile".freeze, "LICENSE".freeze, "README.md".freeze, "RUBY_VERSION".freeze, "Rakefile".freeze, "VERSION".freeze, "bin/rubocop".freeze, "lib/active_job/queue_adapters/workhorse_adapter.rb".freeze, "lib/generators/workhorse/install_generator.rb".freeze, "lib/generators/workhorse/templates/bin/workhorse.rb".freeze, "lib/generators/workhorse/templates/config/initializers/workhorse.rb".freeze, "lib/generators/workhorse/templates/create_table_jobs.rb".freeze, "lib/workhorse.rb".freeze, "lib/workhorse/active_job_extension.rb".freeze, "lib/workhorse/daemon.rb".freeze, "lib/workhorse/daemon/shell_handler.rb".freeze, "lib/workhorse/db_job.rb".freeze, "lib/workhorse/enqueuer.rb".freeze, "lib/workhorse/jobs/cleanup_succeeded_jobs.rb".freeze, "lib/workhorse/jobs/detect_stale_jobs_job.rb".freeze, "lib/workhorse/jobs/run_active_job.rb".freeze, "lib/workhorse/jobs/run_rails_op.rb".freeze, "lib/workhorse/performer.rb".freeze, "lib/workhorse/poller.rb".freeze, "lib/workhorse/pool.rb".freeze, "lib/workhorse/scoped_env.rb".freeze, "lib/workhorse/worker.rb".freeze, "test/active_job/queue_adapters/workhorse_adapter_test.rb".freeze, "test/lib/db_schema.rb".freeze, "test/lib/jobs.rb".freeze, "test/lib/test_helper.rb".freeze, "test/workhorse/daemon_test.rb".freeze, "test/workhorse/db_job_test.rb".freeze, "test/workhorse/enqueuer_test.rb".freeze, "test/workhorse/performer_test.rb".freeze, "test/workhorse/poller_test.rb".freeze, "test/workhorse/pool_test.rb".freeze, "test/workhorse/worker_test.rb".freeze, "workhorse.gemspec".freeze]
   s.homepage = "https://github.com/sitrox/workhorse".freeze
   s.licenses = ["MIT".freeze]

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: workhorse
 version: !ruby/object:Gem::Version
-  version: 1.4.0
+  version: 1.4.2
 platform: ruby
 authors:
 - Sitrox
 bindir: bin
 cert_chain: []
-date: 2026-02-12 00:00:00.000000000 Z
+date: 2026-02-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport