workhorse 1.2.15 → 1.2.17.rc0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 772b634a73940718f57f2363859977dbc7f20ff4b7c12edada5c4cc499ff5f6c
-  data.tar.gz: 8c44df6872578300e6577155626f851ae1cb9202653f835b4dabbcc70fd78c70
+  metadata.gz: 25908f294cd9623ebde44ec8bba5181ee54965703f4fb93df18876b216718eb8
+  data.tar.gz: 568e620571a9ded165fee1fc4420efbb6810f073edd4365c6c8efbe7c3676052
 SHA512:
-  metadata.gz: 751c63f5b1d86238d3e403332972ea0d177ee6c92acab793f3477d9c68f8223d755c633140e2622c2979b404f369327b6e507303f0d2532e1bcb408edf4a02f6
-  data.tar.gz: e5209a6b1472239ce48734eb99d0a1c7cfcd9c20d800c6bb5bb77dff001f84ce7b362b444d0d2d3b0ae2b92b035f75d59484df5dc9d4111c7816ad10aa1b17ef
+  metadata.gz: 83b6ac441e3762f251e35f04dd213044d6fb904a005155235546391cdecd4f9c0c1db24be7c30371da4704b27e6f18aa6ce0b448b6c6c1dbc8c77eeb81faa9e3
+  data.tar.gz: 9c2940a6bfb7cd0be49170d81af2378f27395dbf597f6d8fa2c777bdc812b26d06cc849454ba5925fcf5f26b7724779b3d6e941bc6c9237d6dc251b7d071188f

data/.github/workflows/ruby.yml

@@ -31,6 +31,11 @@ jobs:
         sudo /etc/init.d/mysql start
         mysql -u${{ env.DB_USER }} -p${{ env.DB_PASSWORD }} -e 'CREATE DATABASE ${{ env.DB_DATABASE }};'
     - name: Run rake tests
-      run: bundle exec rake test TESTOPTS='--verbose'
+      uses: nick-fields/retry@v2
+      with:
+        timeout_seconds: 120
+        retry_on: error
+        max_attempts: 3
+        command: bundle exec rake test TESTOPTS='--verbose'
     - name: Run rubocop
       run: bundle exec rubocop

data/.rubocop.yml

@@ -1,81 +1,158 @@
 AllCops:
-  TargetRubyVersion: 2.3
-
+  DisplayCopNames: true
+  NewCops: enable
+  SuggestExtensions: false
+  TargetRubyVersion: 2.5
   Exclude:
+    - 'local/**/*'
     - 'vendor/**/*'
     - 'tmp/**/*'
+    - 'target/**/*'
     - 'log/**/*'
+    - 'db/schema.rb'
+    - 'locale/translations.rb'
+    - 'config/initializers/assets.rb'
+    - 'config/puma.rb'
+    - 'config_scripts/release_notes'
+    - 'config/spring.rb'
+    - 'bin/yarn'
     - '*.gemspec'
 
-  DisplayCopNames: true
+# Make sure accessors are on separate lines for diff readability.
+Style/AccessorGrouping:
+  EnforcedStyle: separated
+
+# Cop would break a lot of existing code.
+Style/OptionalBooleanParameter:
+  Enabled: false
+
+# Multiline hashes should be aligned cleanly as a table to improve readability.
+Layout/HashAlignment:
+  EnforcedHashRocketStyle: table
+  EnforcedColonStyle: table
 
+# Template style is easier on the eyes.
+Style/FormatStringToken:
+  EnforcedStyle: template
+
+# file. This will be addressed when approaching the first ruby 3 application.
 Style/FrozenStringLiteralComment:
   Enabled: false
 
+# Double negation is very useful to make sure you have a boolean in hand. Use it
+# wisely though and know what you're doing.
 Style/DoubleNegation:
   Enabled: false
 
+# Depending on the case, [].include? can be a lot harder to read and less
+# expressive than multiple comparisons.
+Style/MultipleComparison:
+  Enabled: false
+
+# Over time, the ruby guide changed from raise to fail back to raise. Both fail
+# and raise are programatically exactly the same and our decision fell to "fail"
+# for all kinds of exceptions.
 Style/SignalException:
   EnforcedStyle: only_fail
 
-Lint/RescueWithoutErrorClass:
-  Enabled: False
-
-Lint/RescueException:
-  Enabled: False
-
+# Enforced styles can sometimes be hard to read.
 Style/ConditionalAssignment:
   Enabled: false
 
-Layout/IndentArray:
+# Enforce consistent array indentation.
+Layout/FirstArrayElementIndentation:
   EnforcedStyle: consistent
 
+# Disable layout cop because methods just consisting of a number of returns
+# would look very odd with an extra empty line between each return.
+Layout/EmptyLineAfterGuardClause:
+  Enabled: false
+
+# While you should try to keep your code as expressive and short as possible,
+# limitting lengths hardly is over the top.
 Metrics/MethodLength:
   Enabled: false
 
+# While you should try to keep your code as expressive and short as possible,
+# limitting lengths hardly is over the top.
 Metrics/ClassLength:
   Enabled: false
 
+# While you should try to keep your code as expressive and short as possible,
+# limitting lengths hardly is over the top.
 Metrics/ModuleLength:
   Enabled: false
 
+# While you should try to keep your code as expressive and short as possible,
+# limitting lengths hardly is over the top.
+Metrics/BlockLength:
+  Enabled: false
+
+# While not always desirable, it can be useful to have a lot of keyword
+# arguments on certain methods. Try to avoid it though.
 Metrics/ParameterLists:
   Max: 5
   CountKeywordArgs: false
 
+# The results of this cop sometimes seemed arbitrary and can signifficantly
+# restrict certain styles of coding.
 Metrics/AbcSize:
   Enabled: False
 
+# The results of this cop sometimes seemed arbitrary and can signifficantly
+# restrict certain styles of coding.
 Metrics/CyclomaticComplexity:
   Enabled: False
 
+# The results of this cop sometimes seemed arbitrary and can signifficantly
+# restrict certain styles of coding.
 Metrics/PerceivedComplexity:
   Enabled: False
 
-Metrics/LineLength:
-  Max: 160
-
+# In certain cases, "excessive" block nesting might just be useful. Try to keep
+# this down as much as possible though.
 Metrics/BlockNesting:
   Enabled: false
 
-Metrics/BlockLength:
-  Enabled: false
+# A line length of 80 is not considered to be temporary anymore. That's why line
+# length is doubled to 160. If absolutely necessary, create a temporary rubocop
+# exclusion for the lines in question.
+Layout/LineLength:
+  Max: 160
 
+# Prefer variable_1 over variable1 for aesthetic reasons. Do not check symbols,
+# as they often need to be another case for use in external palces (e.g. :md5).
+Naming/VariableNumber:
+  EnforcedStyle: snake_case
+  CheckSymbols: false
+
+# Depending on the surrounding code, even simple if/unless clauses may be more
+# descriptive when on multiple lines.
 Style/IfUnlessModifier:
   Enabled: false
 
+# In most cases, timing does not allow documenting each and every bit of source
+# code. Do not hesitate to enable this cop otherwise.
 Style/Documentation:
   Enabled: false
 
+# Return should be used whenever there is more than one statement or line in a
+# method. This helps avoiding programming mistakes. This is not enforced yet as
+# this would require a custom cop. However, to allow this style of programming,
+# the RedundantReturn cop needs to be disabled.
 Style/RedundantReturn:
   Enabled: false
 
+# Non-ascii comments can be useful sometimes.
 Style/AsciiComments:
   Enabled: false
 
+# Depending on the case, if/unless can be more descriptive than guard clauses.
 Style/GuardClause:
   Enabled: false
 
+# For technical reasons, nested and compact styles must be mixed in certain
+# applications.
 Style/ClassAndModuleChildren:
   Enabled: false
   EnforcedStyle: compact
@@ -83,8 +160,30 @@ Style/ClassAndModuleChildren:
     - nested
     - compact
 
+# Depending on the case, it may be more descriptive to use i.e. == 0 instead of
+# .zero?, especially when testing against multiple numbers.
 Style/NumericPredicate:
   Enabled: false
 
-Layout/IndentHeredoc:
+# Detection is not implemented in a reliable manner for all cases which can lead
+# to false positives and negatives.
+Style/FormatString:
+  Enabled: false
+
+# Do not require MFA, as gems checked with sitrox_standards are only pushed to the
+# internal repo
+Gemspec/RequireMFA:
+  Enabled: false
+
+# Use explicit style
+Naming/BlockForwarding:
+  Enabled: true
+  EnforcedStyle: explicit
+
+Style/HashSyntax:
+  # Use `either` style for `EnforcedShorthandyntax` (see #106550)
+  EnforcedShorthandSyntax: either
+
+# Allow rescue 'Exception', necessary for Workhorse
+Lint/RescueException:
   Enabled: false

data/CHANGELOG.md

@@ -1,5 +1,29 @@
 # Workhorse Changelog
 
+## 1.2.17.rc0 - 2024-02-05
+
+* Add option `config.max_worker_memory_mb` for automatic restart of workers
+  exceeding the specified memory threshold using the `watch` command. Default is
+  `0`, deactivating this feature. See [memory
+  handling](README.md#memory-handling) for more information.
+
+  Sitrox reference: #121312.
+
+## 1.2.16 - 2023-09-18
+
+* Add support for `--skip-initializer` flag to install generator.
+
+  Sitrox reference: #114673.
+
+* Add option `config.clean_stuck_jobs` that enabled automatic cleaning of stuck
+  jobs whenever a worker starts up.
+
+  Sitrox reference: #113708
+
+* Add `retry-step` to actions such that failed unit tests are executed again
+
+  Sitrox reference: #115888
+
 ## 1.2.15 - 2023-08-28
 
 * Add capability to skip transactions for enqueued RailsOps operations.

data/LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2017 - 2023 Sitrox
+Copyright (c) 2017 - 2024 Sitrox
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -62,6 +62,7 @@ What it does not do:
 
    * A database migration for creating a table named `jobs`
    * The initializer `config/initializers/workhorse.rb` for global configuration
+     * This can be skipped using the `--skip-initializer` flag
    * The daemon worker script `bin/workhorse.rb`
 
    Please customize the initializer and worker script to your liking.
@@ -463,6 +464,29 @@ jobs database on a regular interval. Workhorse provides the job
 `Workhorse::Jobs::CleanupSucceededJobs` for this purpose that cleans up all
 succeeded jobs. You can run this using your scheduler in a specific interval.
 
+## Memory handling
+
+When dealing with jobs that may exhibit a large memory footprint, it's important
+to note that Ruby might not release consumed memory back to the operating
+system. Consequently, your job workers could accumulate a significant amount of
+memory over time. To address this, Workhorse provides the
+`config.max_worker_memory_mb` option.
+
+If `config.max_worker_memory_mb` is set to a value above `0`, the `watch`
+command will check the memory footprint (RSS / resident size) of all worker
+processes. If any worker exceeds the specified footprint, Workhorse will
+silently restart it to ensure proper memory release. This process does not
+produce any output in the `watch` command.
+
+Example configuration:
+
+```ruby
+# config/initializers/workhorse.rb
+Workhorse.setup do |config|
+  config.max_worker_memory_mb = 512 # Set the memory threshold to 512 megabytes
+end
+```
+
 ## Load hooks
 
 Using the load hook `:workhorse_db_job`, you can inject custom code into the
@@ -527,10 +551,16 @@ exception is thrown (which may cause a notification if you configured
 `on_exception` accordingly). See the job's API documentation for more
 information.
 
+Starting with Workhorse 1.2.16, there is also a feature that automatically
+checks for stuck jobs (jobs in state `locked` or `started` running on the same
+host where the corresponding PID does not have a process anymore) when starting
+up the worker / poller. This feature can be turned on using the setting
+`config.clean_stuck_jobs`. This is turned off by default.
+
 ## Frequently asked questions
 
 Please consult the [FAQ](FAQ.md).
 
 ## Copyright
 
-Copyright © 2017 - 2023 Sitrox. See `LICENSE` for further details.
+Copyright © 2017 - 2024 Sitrox. See `LICENSE` for further details.

data/RUBY_VERSION

@@ -1 +1 @@
-ruby-2.6.2-p47
+ruby-3.2.1-p31

data/Rakefile

@@ -1,7 +1,7 @@
 task :gemspec do
   gemspec = Gem::Specification.new do |spec|
     spec.name          = 'workhorse'
-    spec.version       = IO.read('VERSION').chomp
+    spec.version       = File.read('VERSION').chomp
     spec.authors       = ['Sitrox']
     spec.summary       = %(
       Multi-threaded job backend with database queuing for ruby.
@@ -13,7 +13,7 @@ task :gemspec do
 
     spec.add_development_dependency 'bundler'
     spec.add_development_dependency 'rake'
-    spec.add_development_dependency 'rubocop', '0.51.0'
+    spec.add_development_dependency 'rubocop', '~> 1.28.0' # Latest version supported with Ruby 2.5
     spec.add_development_dependency 'minitest'
     spec.add_development_dependency 'mysql2'
     spec.add_development_dependency 'colorize'
@@ -25,7 +25,7 @@ task :gemspec do
     spec.add_dependency 'concurrent-ruby'
   end
 
-  File.open('workhorse.gemspec', 'w') { |f| f.write(gemspec.to_ruby.strip) }
+  File.write('workhorse.gemspec', gemspec.to_ruby.strip)
 end
 
 require 'rake/testtask'

data/VERSION

@@ -1 +1 @@
-1.2.15
+1.2.17.rc0

data/lib/active_job/queue_adapters/workhorse_adapter.rb

@@ -10,11 +10,11 @@ module ActiveJob
     #
     #   Rails.application.config.active_job.queue_adapter = :workhorse
     class WorkhorseAdapter
-      def enqueue(job) #:nodoc:
+      def enqueue(job) # :nodoc:
         Workhorse.enqueue_active_job(job)
       end
 
-      def enqueue_at(job, timestamp = Time.now) #:nodoc:
+      def enqueue_at(job, timestamp = Time.now) # :nodoc:
         Workhorse.enqueue_active_job(job, perform_at: timestamp)
       end
     end

data/lib/generators/workhorse/install_generator.rb

@@ -2,7 +2,10 @@ module Workhorse
   class InstallGenerator < Rails::Generators::Base
     include Rails::Generators::Migration
 
-    source_root File.expand_path('../templates', __FILE__)
+    class_option :skip_initializer, type: :boolean, default: false,
+                 desc: 'Skip generating the initializer file'
+
+    source_root File.expand_path('templates', __dir__)
 
     def self.next_migration_number(_dir)
       Time.now.utc.strftime('%Y%m%d%H%M%S')
@@ -18,7 +21,7 @@ module Workhorse
     end
 
     def install_initializer
-      template 'config/initializers/workhorse.rb'
+      template 'config/initializers/workhorse.rb' unless options[:skip_initializer]
     end
   end
 end

data/lib/workhorse/daemon/shell_handler.rb

@@ -40,7 +40,7 @@ module Workhorse
         end
 
         exit 0
-      rescue => e
+      rescue StandardError => e
         warn "#{e.message}\n#{e.backtrace.join("\n")}"
         exit 99
       ensure
@@ -49,44 +49,44 @@ module Workhorse
     end
 
     def self.usage
-      warn <<USAGE
-Usage: #{$PROGRAM_NAME} start|stop|status|watch|restart|usage
+      warn <<~USAGE
+        Usage: #{$PROGRAM_NAME} start|stop|status|watch|restart|usage
 
-Options:
+        Options:
 
-  start
-    Start the daemon
+          start
+            Start the daemon
 
-  stop
-    Stop the daemon
+          stop
+            Stop the daemon
 
-  kill
-    Kill the daemon
+          kill
+            Kill the daemon
 
-  status
-    Query the status of the daemon. Exit with status 1 if any worker is
-    not running.
+          status
+            Query the status of the daemon. Exit with status 1 if any worker is
+            not running.
 
-  watch
-    Checks the status (running or stopped) and whether it is as
-    expected. Starts the daemon if it is expected to run but is not.
+          watch
+            Checks the status (running or stopped) and whether it is as
+            expected. Starts the daemon if it is expected to run but is not.
 
-  restart
-    Shortcut for consecutive 'stop' and 'start'.
+          restart
+            Shortcut for consecutive 'stop' and 'start'.
 
-  restart-logging
-    Re-opens log files, useful e.g. after the log files have been moved or
-    removed by log rotation.
+          restart-logging
+            Re-opens log files, useful e.g. after the log files have been moved or
+            removed by log rotation.
 
-  usage
-    Show this message
+          usage
+            Show this message
 
-Exit status:
- 0 if OK,
- 1 on fatal errors outside of workhorse,
- 2 if at least one worker has an unexpected status,
- 99 on all other errors.
-USAGE
+        Exit status:
+         0 if OK,
+         1 on fatal errors outside of workhorse,
+         2 if at least one worker has an unexpected status,
+         99 on all other errors.
+      USAGE
     end
   end
 end

data/lib/workhorse/daemon.rb

@@ -4,6 +4,7 @@ module Workhorse
       attr_reader :id
       attr_reader :name
       attr_reader :block
+      attr_accessor :pid
 
       def initialize(id, name, &block)
         @id = id
@@ -12,6 +13,9 @@ module Workhorse
       end
     end
 
+    # @private
+    attr_reader :workers
+
     def initialize(pidfile: nil, quiet: false, &_block)
       @pidfile = pidfile
       @quiet = quiet
@@ -68,7 +72,7 @@ module Workhorse
 
         if pid_file && pid
           puts "Worker (#{worker.name}) ##{worker.id}: Stopping"
-          stop_worker pid_file, pid, kill
+          stop_worker pid_file, pid, kill: kill
         elsif pid_file
           File.delete pid_file
           puts "Worker (#{worker.name}) ##{worker.id}: Already stopped (stale PID file)"
@@ -109,10 +113,14 @@ module Workhorse
       end
 
       if should_be_running && status(quiet: true) != 0
-        return start(quiet: Workhorse.silence_watcher)
+        code = start(quiet: Workhorse.silence_watcher)
       else
-        return 0
+        code = 0
       end
+
+      watch_memory! if should_be_running
+
+      return code
     end
 
     def restart
@@ -140,6 +148,30 @@ module Workhorse
 
     private
 
+    def watch_memory!
+      return if Workhorse.max_worker_memory_mb == 0
+
+      for_each_worker do |worker|
+        pid_file, pid = read_pid(worker)
+        next unless pid_file && pid
+
+        memory = memory_for(pid)
+        next unless memory
+
+        if memory > Workhorse.max_worker_memory_mb
+          stop_worker pid_file, pid
+          start_worker worker
+        end
+      end
+    end
+
+    # Returns the memory (RSS) in MB for the given process.
+    def memory_for(pid)
+      mem = `ps -p #{pid} -o rss=`&.strip
+      return nil if mem.blank?
+      return mem.to_i / 1024
+    end
+
     def for_each_worker(&block)
       @workers.each(&block)
     end
@@ -147,20 +179,20 @@ module Workhorse
     def start_worker(worker)
       pid = fork do
         $0 = process_name(worker)
-
         # Reopen pipes to prevent #107576
-        STDIN.reopen File.open('/dev/null', 'r')
+        $stdin.reopen File.open('/dev/null', 'r')
         null_out = File.open '/dev/null', 'w'
-        STDOUT.reopen null_out
-        STDERR.reopen null_out
+        $stdout.reopen null_out
+        $stderr.reopen null_out
 
         worker.block.call
       end
-      IO.write(pid_file_for(worker), pid)
+      worker.pid = pid
+      File.write(pid_file_for(worker), pid)
       Process.detach(pid)
     end
 
-    def stop_worker(pid_file, pid, kill = false)
+    def stop_worker(pid_file, pid, kill: false)
       signals = kill ? %w[KILL] : %w[TERM INT]
 
       loop do
@@ -207,8 +239,9 @@ module Workhorse
       file = pid_file_for(worker)
 
       if File.exist?(file)
-        raw_pid = IO.read(file)
+        raw_pid = File.read(file)
         return nil, nil if raw_pid.blank?
+
         pid = Integer(raw_pid)
         return file, process?(pid) ? pid : nil
       else

data/lib/workhorse/db_job.rb

@@ -6,6 +6,8 @@ module Workhorse
     STATE_SUCCEEDED = :succeeded
     STATE_FAILED    = :failed
 
+    EXP_LOCKED_BY = /^(.*?)\.(\d+?)\.([^.]+)$/.freeze
+
     if respond_to?(:attr_accessible)
       attr_accessible :queue, :priority, :perform_at, :handler, :description
     end
@@ -32,6 +34,31 @@ module Workhorse
       where(state: STATE_FAILED)
     end
 
+    # @private
+    def self.with_split_locked_by
+      select(<<~SQL)
+        #{table_name}.*,
+
+        -- random string
+        substring_index(locked_by, '.', -1) as locked_by_rnd,
+
+        -- pid
+        substring_index(
+          substring_index(locked_by, '.', -2),
+          '.',
+          1
+        ) as locked_by_pid,
+
+        -- get host
+        substring(
+          locked_by,
+          1,
+          length(locked_by) -
+          length(substring_index(locked_by, '.', -2)) - 1
+        ) as locked_by_host
+      SQL
+    end
+
     # Resets job to state "waiting" and clears all meta fields
     # set by workhorse in course of processing this job.
     #
@@ -69,11 +96,6 @@ module Workhorse
         fail "Dirty jobs can't be locked."
       end
 
-      # TODO: Remove this debug output
-      # if Workhorse::DbJob.lock.find(id).locked_at
-      #   puts "Already locked (with FOR UPDATE)"
-      # end
-
       if locked_at
         # TODO: Remove this debug output
         # puts "Already locked. Job: #{self.id} Worker: #{worker_id}"

data/lib/workhorse/enqueuer.rb

@@ -3,11 +3,11 @@ module Workhorse
     # Enqueue any object that is serializable and has a `perform` method
     def enqueue(job, queue: nil, priority: 0, perform_at: Time.now, description: nil)
       return DbJob.create!(
-        queue: queue,
-        priority: priority,
-        perform_at: perform_at,
+        queue:       queue,
+        priority:    priority,
+        perform_at:  perform_at,
         description: description,
-        handler: Marshal.dump(job)
+        handler:     Marshal.dump(job)
       )
     end
 

data/lib/workhorse/jobs/cleanup_succeeded_jobs.rb

@@ -18,7 +18,7 @@ module Workhorse::Jobs
     private
 
     def seconds_ago(days)
-      Time.now - days * 24 * 60 * 60
+      Time.now - (days * 24 * 60 * 60)
     end
   end
 end

data/lib/workhorse/jobs/detect_stale_jobs_job.rb

@@ -27,7 +27,7 @@ module Workhorse::Jobs
         ids = rel.pluck(:id)
 
         unless ids.empty?
-          messages << "Detected #{ids.size} jobs that were locked more than "\
+          messages << "Detected #{ids.size} jobs that were locked more than " \
                       "#{@locked_to_started_threshold}s ago and might be stale: #{ids.inspect}."
         end
       end
@@ -39,7 +39,7 @@ module Workhorse::Jobs
         ids = rel.pluck(:id)
 
         unless ids.empty?
-          messages << "Detected #{ids.size} jobs that are running for longer than "\
+          messages << "Detected #{ids.size} jobs that are running for longer than " \
                       "#{@run_time_threshold}s ago and might be stale: #{ids.inspect}."
         end
       end

data/lib/workhorse/poller.rb

@@ -27,6 +27,8 @@ module Workhorse
       fail 'Poller is already running.' if running?
       @running = true
 
+      clean_stuck_jobs! if Workhorse.clean_stuck_jobs
+
       @thread = Thread.new do
         loop do
           break unless running?
@@ -65,6 +67,61 @@ module Workhorse
 
     private
 
+    def clean_stuck_jobs!
+      with_global_lock timeout: MAX_LOCK_TIMEOUT do
+        Workhorse.tx_callback.call do
+          # Basic relation: Fetch jobs locked by current host in state 'locked' or
+          # 'started'
+          rel = Workhorse::DbJob.select('*').from(<<~SQL)
+            (#{Workhorse::DbJob.with_split_locked_by.to_sql}) #{Workhorse::DbJob.table_name}
+          SQL
+          rel.where!(
+            locked_by_host: worker.hostname,
+            state:          [Workhorse::DbJob::STATE_LOCKED, Workhorse::DbJob::STATE_STARTED]
+          )
+
+          # Select all pids
+          job_pids = rel.distinct.pluck(:locked_by_pid).to_set(&:to_i)
+
+          # Get pids without active process
+          orphaned_pids = job_pids.select do |pid|
+            Process.getpgid(pid)
+            false
+          rescue Errno::ESRCH
+            true
+          end
+
+          # Reset jobs in state 'locked'
+          rel.where(locked_by_pid: orphaned_pids.to_a, state: Workhorse::DbJob::STATE_LOCKED).each do |job|
+            worker.log(
+              "Job ##{job.id} has been locked but not yet startet by PID #{job.locked_by_pid} on host " \
+              "#{job.locked_by_host}, but the process is not running anymore. This job has therefore been " \
+              "reset (set to 'waiting') by the Workhorse cleanup logic.",
+              :warn
+            )
+            job.reset!(true)
+          end
+
+          # Mark jobs in state 'started' as failed
+          rel.where(locked_by_pid: orphaned_pids.to_a, state: Workhorse::DbJob::STATE_STARTED).each do |job|
+            worker.log(
+              "Job ##{job.id} has been started by PID #{job.locked_by_pid} on host #{job.locked_by_host} " \
+              'but the process is not running anymore. This job has therefore been marked as ' \
+              'failed by the Workhorse cleanup logic.',
+              :warn
+            )
+            exception = Exception.new(
+              "Job has been started by PID #{job.locked_by_pid} on host #{job.locked_by_host} " \
+              'but the process is not running anymore. This job has therefore been marked as ' \
+              'failed by the Workhorse cleanup logic.'
+            )
+            exception.set_backtrace []
+            job.mark_failed!(exception)
+          end
+        end
+      end
+    end
+
     def sleep
       remaining = worker.polling_interval
 
@@ -101,15 +158,15 @@ module Workhorse
         if @global_lock_fails > Workhorse.max_global_lock_fails && !@max_global_lock_fails_reached
           @max_global_lock_fails_reached = true
 
-          worker.log 'Could not obtain global lock, retrying with next poll. '\
-                     'This will be the last such message for this worker until '\
+          worker.log 'Could not obtain global lock, retrying with next poll. ' \
+                     'This will be the last such message for this worker until ' \
                      'the issue is resolved.', :warn
 
           message = "Worker reached maximum number of consecutive times (#{Workhorse.max_global_lock_fails}) " \
                     "where the global lock could no be acquired within the specified timeout (#{timeout}). " \
                     'A worker that obtained this lock may have crashed without ending the database ' \
                     'connection properly. On MySQL, use "show processlist;" to see which connection(s) ' \
-                    'is / are holding the lock for a long period of time and consider killing them using '\
+                    'is / are holding the lock for a long period of time and consider killing them using ' \
                     "MySQL's \"kill <Id>\" command. This message will be issued only once per worker " \
                     'and may only be re-triggered if the error happens again *after* the lock has ' \
                     'been solved in the meantime.'
@@ -137,7 +194,6 @@ module Workhorse
       @instant_repoll.make_false
 
       timeout = [MIN_LOCK_TIMEOUT, [MAX_LOCK_TIMEOUT, worker.polling_interval].min].max
-
       with_global_lock timeout: timeout do
         job_ids = []
 
@@ -199,9 +255,9 @@ module Workhorse
       # uses the keyword 'AS' in SQL generated for Oracle, which is invalid for
       # table aliases.
       union_query_sql = '('
-      union_query_sql += 'SELECT * FROM (' + union_parts.shift.to_sql + ') union_0'
+      union_query_sql += "SELECT * FROM (#{union_parts.shift.to_sql}) union_0"
       union_parts.each_with_index do |part, idx|
-        union_query_sql += ' UNION SELECT * FROM (' + part.to_sql + ") union_#{idx + 1}"
+        union_query_sql += " UNION SELECT * FROM (#{part.to_sql}) union_#{idx + 1}"
       end
       union_query_sql += ') subselect'
 
@@ -277,7 +333,7 @@ module Workhorse
       unless worker.queues.empty?
         if worker.queues.include?(nil)
           where = table[:queue].eq(nil)
-          remaining_queues = worker.queues.reject(&:nil?)
+          remaining_queues = worker.queues.compact
           unless remaining_queues.empty?
             where = where.or(table[:queue].in(remaining_queues))
           end

data/lib/workhorse/pool.rb

@@ -6,11 +6,11 @@ module Workhorse
     def initialize(size)
       @size = size
       @executor = Concurrent::ThreadPoolExecutor.new(
-        min_threads: 0,
-        max_threads: @size,
-        max_queue: 0,
+        min_threads:     0,
+        max_threads:     @size,
+        max_queue:       0,
         fallback_policy: :abort,
-        auto_terminate: false
+        auto_terminate:  false
       )
       @mutex = Mutex.new
       @active_threads = Concurrent::AtomicFixnum.new(0)
@@ -33,12 +33,10 @@ module Workhorse
         active_threads.increment
 
         @executor.post do
-          begin
-            yield
-          ensure
-            active_threads.decrement
-            @on_idle.try(:call)
-          end
+          yield
+        ensure
+          active_threads.decrement
+          @on_idle.try(:call)
         end
       end
     end

data/lib/workhorse/worker.rb

@@ -43,7 +43,7 @@ module Workhorse
     #   `Rails.logger`.
     def initialize(queues: [], pool_size: nil, polling_interval: 300, auto_terminate: true, quiet: true, instant_repolling: false, logger: nil)
       @queues = queues
-      @pool_size = pool_size || queues.size + 1
+      @pool_size = pool_size || (queues.size + 1)
       @polling_interval = polling_interval
       @auto_terminate = auto_terminate
       @state = :initialized
@@ -54,7 +54,7 @@ module Workhorse
       @poller = Workhorse::Poller.new(self)
       @logger = logger
 
-      unless (@polling_interval / 0.1).round(2).modulo(1) == 0.0
+      unless (@polling_interval / 0.1).round(2).modulo(1).zero?
         fail 'Polling interval must be a multiple of 0.1.'
       end
 
@@ -74,7 +74,15 @@ module Workhorse
     end
 
     def id
-      @id ||= "#{Socket.gethostname}.#{Process.pid}.#{SecureRandom.hex(3)}"
+      @id ||= "#{hostname}.#{pid}.#{SecureRandom.hex(3)}"
+    end
+
+    def pid
+      @pid ||= Process.pid
+    end
+
+    def hostname
+      @hostname ||= Socket.gethostname
     end
 
     # Starts the worker. This call is not blocking - call {wait} for this
@@ -135,12 +143,10 @@ module Workhorse
         log "Posting job #{db_job_id} to thread pool"
 
         @pool.post do
-          begin
-            Workhorse::Performer.new(db_job_id, self).perform
-          rescue Exception => e
-            log %(#{e.message}\n#{e.backtrace.join("\n")}), :error
-            Workhorse.on_exception.call(e)
-          end
+          Workhorse::Performer.new(db_job_id, self).perform
+        rescue Exception => e
+          log %(#{e.message}\n#{e.backtrace.join("\n")}), :error
+          Workhorse.on_exception.call(e)
         end
       end
     rescue Exception => e

data/lib/workhorse.rb

@@ -17,7 +17,7 @@ module Workhorse
   # Returns the performer currently performing the active job. This can only be
   # called from within a job and the same thread.
   def self.performer
-    Thread.current[:workhorse_current_performer]\
+    Thread.current[:workhorse_current_performer] \
       || fail('No performer is associated with the current thread. This method must always be called inside of a job.')
   end
 
@@ -58,6 +58,11 @@ module Workhorse
   mattr_accessor :perform_jobs_in_tx
   self.perform_jobs_in_tx = true
 
+  # If enabled, each poller will attempt to clean jobs that are stuck in state
+  # 'locked' or 'running' when it is starting up.
+  mattr_accessor :clean_stuck_jobs
+  self.clean_stuck_jobs = false
+
   # This setting is for {Workhorse::Jobs::DetectStaleJobsJob} and specifies the
   # maximum number of seconds a job is allowed to stay 'locked' before this job
   # throws an exception. Set this to 0 to skip this check.
@@ -70,6 +75,12 @@ module Workhorse
   mattr_accessor :stale_detection_run_time_threshold
   self.stale_detection_run_time_threshold = 12 * 60
 
+  # Maximum memory for a worker in MB. If this memory limit (RSS / resident
+  # size) is reached for a worker process, the 'watch' command will restart said
+  # worker. Set this to 0 disable this feature.
+  mattr_accessor :max_worker_memory_mb
+  self.max_worker_memory_mb = 0
+
   def self.setup
     yield self
   end
@@ -92,5 +103,5 @@ if RUBY_PLATFORM != 'java'
 end
 
 if defined?(ActiveJob)
-  require 'active_job/queue_adapters/workhorse_adapter.rb'
+  require 'active_job/queue_adapters/workhorse_adapter'
 end

data/test/lib/test_helper.rb

@@ -44,10 +44,10 @@ end
 
 ActiveRecord::Base.establish_connection(
   adapter:  'mysql2',
-  database: ENV['DB_NAME'] || 'workhorse',
-  username: ENV['DB_USERNAME'] || 'root',
-  password: ENV['DB_PASSWORD'] || '',
-  host:     ENV['DB_HOST'] || '127.0.0.1',
+  database: ENV.fetch('DB_NAME', nil) || 'workhorse',
+  username: ENV.fetch('DB_USERNAME', nil) || 'root',
+  password: ENV.fetch('DB_PASSWORD', nil) || '',
+  host:     ENV.fetch('DB_HOST', nil) || '127.0.0.1',
   pool:     10
 )
 

data/test/workhorse/poller_test.rb

@@ -41,7 +41,7 @@ class Workhorse::PollerTest < WorkhorseTest
 
     begin
       fail 'Some exception'
-    rescue => e
+    rescue StandardError => e
       last_job.mark_failed!(e)
     end
 
@@ -148,9 +148,10 @@ class Workhorse::PollerTest < WorkhorseTest
     assert_equal 25,  used_workers
   end
 
-  # rubocop: disable Style/GlobalVars
   def test_connection_loss
+    # rubocop: disable Style/GlobalVars
     $thread_conn = nil
+    # rubocop: enable Style/GlobalVars
 
     Workhorse.enqueue BasicJob.new(sleep_time: 3)
 
@@ -175,10 +176,95 @@ class Workhorse::PollerTest < WorkhorseTest
 
     assert_equal 1, Workhorse::DbJob.succeeded.count
   end
-  # rubocop: enable Style/GlobalVars
+
+  def test_clean_stuck_jobs_locked
+    [true, false].each do |clean|
+      Workhorse::DbJob.delete_all
+
+      Workhorse.clean_stuck_jobs = clean
+      start_deamon
+      Workhorse.enqueue BasicJob.new(sleep_time: 5)
+      sleep 0.2
+      kill_deamon_workers
+
+      assert_equal 1, Workhorse::DbJob.count
+
+      Workhorse::DbJob.first.update(
+        state:      'locked',
+        started_at: nil
+      )
+
+      Workhorse::Worker.new.poller.send(:clean_stuck_jobs!) if clean
+
+      assert_equal 1, Workhorse::DbJob.count
+
+      Workhorse::DbJob.first.tap do |job|
+        if clean
+          assert_equal 'waiting', job.state
+          assert_nil job.locked_at
+          assert_nil job.locked_by
+          assert_nil job.started_at
+          assert_nil job.last_error
+        else
+          assert_equal 'locked', job.state
+        end
+      end
+    ensure
+      Workhorse.clean_stuck_jobs = false
+    end
+  end
+
+  def test_clean_stuck_jobs_running
+    [true, false].each do |clean|
+      Workhorse::DbJob.delete_all
+
+      Workhorse.clean_stuck_jobs = true
+      start_deamon
+      Workhorse.enqueue BasicJob.new(sleep_time: 5)
+      sleep 0.2
+      kill_deamon_workers
+
+      assert_equal 1, Workhorse::DbJob.count
+      assert_equal 'started', Workhorse::DbJob.first.state
+
+      work 0.1 if clean
+
+      assert_equal 1, Workhorse::DbJob.count
+
+      Workhorse::DbJob.first.tap do |job|
+        if clean
+          assert_equal 'failed', job.state
+          assert_match(/started by PID #{@daemon.workers.first.pid}/, job.last_error)
+          assert_match(/on host #{Socket.gethostname}/, job.last_error)
+        else
+          assert_equal 'started', job.state
+        end
+      end
+    ensure
+      Workhorse.clean_stuck_jobs = false
+    end
+  end
 
   private
 
+  def kill_deamon_workers
+    @daemon.workers.each do |worker|
+      Process.kill 'KILL', worker.pid
+    end
+  end
+
+  def start_deamon
+    @daemon = Workhorse::Daemon.new(pidfile: 'tmp/pids/test%s.pid') do |d|
+      d.worker 'Test Worker' do
+        Workhorse::Worker.start_and_wait(
+          pool_size:        1,
+          polling_interval: 0.1
+        )
+      end
+    end
+    @daemon.start
+  end
+
   def setup
     Workhorse::DbJob.delete_all
   end

data/workhorse.gemspec

@@ -1,61 +1,31 @@
 # -*- encoding: utf-8 -*-
-# stub: workhorse 1.2.15 ruby lib
+# stub: workhorse 1.2.17.rc0 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "workhorse".freeze
-  s.version = "1.2.15"
+  s.version = "1.2.17.rc0"
 
-  s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
+  s.required_rubygems_version = Gem::Requirement.new("> 1.3.1".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]
   s.authors = ["Sitrox".freeze]
-  s.date = "2023-08-28"
+  s.date = "2024-02-05"
   s.files = [".github/workflows/ruby.yml".freeze, ".gitignore".freeze, ".releaser_config".freeze, ".rubocop.yml".freeze, "CHANGELOG.md".freeze, "FAQ.md".freeze, "Gemfile".freeze, "LICENSE".freeze, "README.md".freeze, "RUBY_VERSION".freeze, "Rakefile".freeze, "VERSION".freeze, "bin/rubocop".freeze, "lib/active_job/queue_adapters/workhorse_adapter.rb".freeze, "lib/generators/workhorse/install_generator.rb".freeze, "lib/generators/workhorse/templates/bin/workhorse.rb".freeze, "lib/generators/workhorse/templates/config/initializers/workhorse.rb".freeze, "lib/generators/workhorse/templates/create_table_jobs.rb".freeze, "lib/workhorse.rb".freeze, "lib/workhorse/active_job_extension.rb".freeze, "lib/workhorse/daemon.rb".freeze, "lib/workhorse/daemon/shell_handler.rb".freeze, "lib/workhorse/db_job.rb".freeze, "lib/workhorse/enqueuer.rb".freeze, "lib/workhorse/jobs/cleanup_succeeded_jobs.rb".freeze, "lib/workhorse/jobs/detect_stale_jobs_job.rb".freeze, "lib/workhorse/jobs/run_active_job.rb".freeze, "lib/workhorse/jobs/run_rails_op.rb".freeze, "lib/workhorse/performer.rb".freeze, "lib/workhorse/poller.rb".freeze, "lib/workhorse/pool.rb".freeze, "lib/workhorse/scoped_env.rb".freeze, "lib/workhorse/worker.rb".freeze, "test/active_job/queue_adapters/workhorse_adapter_test.rb".freeze, "test/lib/db_schema.rb".freeze, "test/lib/jobs.rb".freeze, "test/lib/test_helper.rb".freeze, "test/workhorse/db_job_test.rb".freeze, "test/workhorse/enqueuer_test.rb".freeze, "test/workhorse/performer_test.rb".freeze, "test/workhorse/poller_test.rb".freeze, "test/workhorse/pool_test.rb".freeze, "test/workhorse/worker_test.rb".freeze, "workhorse.gemspec".freeze]
-  s.rubygems_version = "3.0.3".freeze
+  s.rubygems_version = "3.4.6".freeze
   s.summary = "Multi-threaded job backend with database queuing for ruby.".freeze
   s.test_files = ["test/active_job/queue_adapters/workhorse_adapter_test.rb".freeze, "test/lib/db_schema.rb".freeze, "test/lib/jobs.rb".freeze, "test/lib/test_helper.rb".freeze, "test/workhorse/db_job_test.rb".freeze, "test/workhorse/enqueuer_test.rb".freeze, "test/workhorse/performer_test.rb".freeze, "test/workhorse/poller_test.rb".freeze, "test/workhorse/pool_test.rb".freeze, "test/workhorse/worker_test.rb".freeze]
 
-  if s.respond_to? :specification_version then
-    s.specification_version = 4
+  s.specification_version = 4
 
-    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
-      s.add_development_dependency(%q<bundler>.freeze, [">= 0"])
-      s.add_development_dependency(%q<rake>.freeze, [">= 0"])
-      s.add_development_dependency(%q<rubocop>.freeze, ["= 0.51.0"])
-      s.add_development_dependency(%q<minitest>.freeze, [">= 0"])
-      s.add_development_dependency(%q<mysql2>.freeze, [">= 0"])
-      s.add_development_dependency(%q<colorize>.freeze, [">= 0"])
-      s.add_development_dependency(%q<benchmark-ips>.freeze, [">= 0"])
-      s.add_development_dependency(%q<activejob>.freeze, [">= 0"])
-      s.add_development_dependency(%q<pry>.freeze, [">= 0"])
-      s.add_runtime_dependency(%q<activesupport>.freeze, [">= 0"])
-      s.add_runtime_dependency(%q<activerecord>.freeze, [">= 0"])
-      s.add_runtime_dependency(%q<concurrent-ruby>.freeze, [">= 0"])
-    else
-      s.add_dependency(%q<bundler>.freeze, [">= 0"])
-      s.add_dependency(%q<rake>.freeze, [">= 0"])
-      s.add_dependency(%q<rubocop>.freeze, ["= 0.51.0"])
-      s.add_dependency(%q<minitest>.freeze, [">= 0"])
-      s.add_dependency(%q<mysql2>.freeze, [">= 0"])
-      s.add_dependency(%q<colorize>.freeze, [">= 0"])
-      s.add_dependency(%q<benchmark-ips>.freeze, [">= 0"])
-      s.add_dependency(%q<activejob>.freeze, [">= 0"])
-      s.add_dependency(%q<pry>.freeze, [">= 0"])
-      s.add_dependency(%q<activesupport>.freeze, [">= 0"])
-      s.add_dependency(%q<activerecord>.freeze, [">= 0"])
-      s.add_dependency(%q<concurrent-ruby>.freeze, [">= 0"])
-    end
-  else
-    s.add_dependency(%q<bundler>.freeze, [">= 0"])
-    s.add_dependency(%q<rake>.freeze, [">= 0"])
-    s.add_dependency(%q<rubocop>.freeze, ["= 0.51.0"])
-    s.add_dependency(%q<minitest>.freeze, [">= 0"])
-    s.add_dependency(%q<mysql2>.freeze, [">= 0"])
-    s.add_dependency(%q<colorize>.freeze, [">= 0"])
-    s.add_dependency(%q<benchmark-ips>.freeze, [">= 0"])
-    s.add_dependency(%q<activejob>.freeze, [">= 0"])
-    s.add_dependency(%q<pry>.freeze, [">= 0"])
-    s.add_dependency(%q<activesupport>.freeze, [">= 0"])
-    s.add_dependency(%q<activerecord>.freeze, [">= 0"])
-    s.add_dependency(%q<concurrent-ruby>.freeze, [">= 0"])
-  end
+  s.add_development_dependency(%q<bundler>.freeze, [">= 0"])
+  s.add_development_dependency(%q<rake>.freeze, [">= 0"])
+  s.add_development_dependency(%q<rubocop>.freeze, ["~> 1.28.0"])
+  s.add_development_dependency(%q<minitest>.freeze, [">= 0"])
+  s.add_development_dependency(%q<mysql2>.freeze, [">= 0"])
+  s.add_development_dependency(%q<colorize>.freeze, [">= 0"])
+  s.add_development_dependency(%q<benchmark-ips>.freeze, [">= 0"])
+  s.add_development_dependency(%q<activejob>.freeze, [">= 0"])
+  s.add_development_dependency(%q<pry>.freeze, [">= 0"])
+  s.add_runtime_dependency(%q<activesupport>.freeze, [">= 0"])
+  s.add_runtime_dependency(%q<activerecord>.freeze, [">= 0"])
+  s.add_runtime_dependency(%q<concurrent-ruby>.freeze, [">= 0"])
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: workhorse
 version: !ruby/object:Gem::Version
-  version: 1.2.15
+  version: 1.2.17.rc0
 platform: ruby
 authors:
 - Sitrox
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-08-28 00:00:00.000000000 Z
+date: 2024-02-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -42,16 +42,16 @@ dependencies:
   name: rubocop
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.51.0
+        version: 1.28.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.51.0
+        version: 1.28.0
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
@@ -242,11 +242,11 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
-rubygems_version: 3.4.6
+rubygems_version: 3.4.10
 signing_key:
 specification_version: 4
 summary: Multi-threaded job backend with database queuing for ruby.