workhorse 1.2.12 → 1.2.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5d242b1602e66d5de0ad72ef85ce8a15e885248ca14a075a2f25bb60b80c0364
-  data.tar.gz: db8f8a6aecfe00aae955775b09c7d08568653258ef56ab874839cde702725854
+  metadata.gz: 0a1665e70227e5e6fad1115523a94c7457a9b9b7216e10d2092c0326817bfd44
+  data.tar.gz: f6a1d79bf0f55a8f2756934bb58ab3aefb83ef6369ffec7c783ef8cd6dfa50b0
 SHA512:
-  metadata.gz: aa64c4d9ff6c4085c42018ddd293b04fd8078e702306631d8b9918c917a53903b48832d561216c25f2dcae6c36cbc15f2e6587301302165b1a3fc639c4dfa59a
-  data.tar.gz: f76393011d3be7b4002d27a5836e86ada615375130d04ae5f1ea64002fdad4cb5f2410c45e829ad726ce386c600d353208cba9d25de9e296db796f864373f1a0
+  metadata.gz: 8dde03d8f2a8f5c05307787433ef308c99c34ad2ab59d7c2edcc352c0428241a48ee5d0356f4a9a35a0de7dac876d60e1c09cd3c016915929f786a387a9e75e6
+  data.tar.gz: b380dff403c8c10df7c2472422e462d66d40bff3796cbee09e81c63c11eafe4ba4a0d0dd361828f5a984f738fe30744b6fcea3616b6a54a9d1f7df016b21219f

data/CHANGELOG.md

@@ -1,5 +1,28 @@
 # Workhorse Changelog
 
+## 1.2.14 - 2023-08-23
+
+* Add documentation for transaction handling.
+
+* Add support for skipping transactions on a per-job basis
+
+## 1.2.13 - 2023-02-20
+
+* Add the `config.max_global_lock_fails` setting (defaults to 10). If a
+  worker's poller cannot acquire the global lock, an error is logged, and if
+  `config.on_exception` is configured, the error is handled using this callback.
+
+  This change allows you to be aware of essentially defunct worker processes due
+  to a global lock that could not be obtained, for example, because of another
+  worker that was killed without properly releasing the lock. However, this is
+  an edge case because:
+
+  1. The lock is released by Workhorse in an `ensure` block.
+  2. At least MySQL is supposed to release global locks obtained in a connection
+     when that connection is closed.
+
+  Sitrox reference: #110339.
+
 ## 1.2.12 - 2023-01-18
 
 * Call `on_exception` callback on failed `Performer` initialization (e.g. when

data/README.md

@@ -278,6 +278,57 @@ polling interval.
 This setting is recommended for all setups and may eventually be enabled by
 default.
 
+## Transactions
+
+By default, each job is run in an individual database transaction. An exception
+to this is when performing ActiveJob jobs using `perform_now`, where no
+transaction is created.
+
+### Transaction callback
+
+By default, transactions are created using `ActiveRecord::Base.transaction { ...
+}`. You can customize this using the setting `config.tx_callback` in your
+`config/initializers/workhorse.rb` (see commented out section in the generated
+configuration file).
+
+### Turning off transactions
+
+You can turn off transaction wrapping in the following ways:
+
+- Globally using the setting `config.perform_jobs_in_tx = false` in your
+  `config/initializers/workhorse.rb`. This is not recommended as running jobs
+  without transactions can potentially be harmful.
+
+- On a per-job basis. This is the recommended approach for jobs that either open
+  up their own transaction(s) or jobs that explicitly do not need a transaction
+  for whatever reason.
+
+  Usage of this feature depends on whether you are dealing with an ActiveJob job
+  or a plain Workhorse job class.
+
+  For ActiveJob:
+
+  1. Add the following mixin to your job class (usually `ApplicationJob`):
+
+     ```ruby
+     class ApplicationJob
+       include Workhorse::ActiveJobExtension
+     end
+     ```
+
+  2. Use the DSL-method `skip_tx` inside the job classes where you want to
+     disable transaction wrapping, e.g.:
+
+     ```ruby
+     class MyJob < ApplicationJob
+       skip_tx
+
+       def perform
+         # Something without transaction
+       end
+     end
+     ```
+
 ## Exception handling
 
 Per default, exceptions occurring in a worker thread will only be visible in the
@@ -419,6 +470,24 @@ This means that you should always have an external *watcher* (usually a
 cronjob), that calls the `workhorse watch` command regularly. This would
 automatically restart crashed worker processes.
 
+### Unobtainable Locks
+
+Each Workhorse worker uses a poller to check the database for new jobs. To
+ensure that no job is obtained by more than one worker, a global database lock
+is used. If a worker is killed, it may happen that the lock is not properly
+released, which can cause all pollers to stop working because they cannot
+acquire the lock. This is an edge case, as the locks should typically be
+released properly, even if a worker process is killed using `SIGKILL`.
+
+In the event that this still happens, Workhorse takes the following steps:
+
+- Logs when a lock could not be obtained.
+- Retries acquiring the lock on the next poll.
+- Calls the `on_exception` callback (if configured) after a configurable number of consecutive failures to obtain the lock.
+
+The maximum number of consecutive failures can be configured using
+`config.max_global_lock_fails`, which defaults to 10.
+
 ### Stuck queues
 
 Jobs in named queues (non-null queues) are always run sequentially. This means

data/VERSION

@@ -1 +1 @@
-1.2.12
+1.2.14

data/lib/workhorse/active_job_extension.rb

@@ -0,0 +1,20 @@
+module Workhorse
+  module ActiveJobExtension
+    extend ActiveSupport::Concern
+
+    included do
+      class_attribute :_skip_tx
+      self._skip_tx = false
+    end
+
+    module ClassMethods
+      def skip_tx
+        self._skip_tx = true
+      end
+
+      def skip_tx?
+        _skip_tx
+      end
+    end
+  end
+end

data/lib/workhorse/jobs/run_active_job.rb

@@ -1,9 +1,15 @@
 module Workhorse::Jobs
   class RunActiveJob
+    attr_reader :job_data
+
     def initialize(job_data)
       @job_data = job_data
     end
 
+    def job_class
+      @job_data['job_class'].safe_constantize
+    end
+
     def perform
       ActiveJob::Base.execute(@job_data)
     end

data/lib/workhorse/performer.rb

@@ -61,7 +61,11 @@ module Workhorse
       log 'Performing', :info
       log "Description: #{@db_job.description}", :info unless @db_job.description.blank?
 
-      if Workhorse.perform_jobs_in_tx
+      inner_job_class = deserialized_job.try(:job_class) || deserialized_job.class
+      skip_tx = inner_job_class.try(:skip_tx?)
+      log "SKIP TX: #{skip_tx.inspect}".red, :error
+
+      if Workhorse.perform_jobs_in_tx && !skip_tx
         Workhorse.tx_callback.call do
           deserialized_job.perform
         end

data/lib/workhorse/poller.rb

@@ -15,6 +15,8 @@ module Workhorse
       @table = Workhorse::DbJob.arel_table
       @is_oracle = ActiveRecord::Base.connection.adapter_name == 'OracleEnhanced'
       @instant_repoll = Concurrent::AtomicBoolean.new(false)
+      @global_lock_fails = 0
+      @max_global_lock_fails_reached = false
     end
 
     def running?
@@ -86,6 +88,38 @@ module Workhorse
         success = result == 1
       end
 
+      if success
+        @global_lock_fails = 0
+        @max_global_lock_fails_reached = false
+      else
+        @global_lock_fails += 1
+
+        unless @max_global_lock_fails_reached
+          worker.log 'Could not obtain global lock, retrying with next poll.', :warn
+        end
+
+        if @global_lock_fails > Workhorse.max_global_lock_fails && !@max_global_lock_fails_reached
+          @max_global_lock_fails_reached = true
+
+          worker.log 'Could not obtain global lock, retrying with next poll. '\
+                     'This will be the last such message for this worker until '\
+                     'the issue is resolved.', :warn
+
+          message = "Worker reached maximum number of consecutive times (#{Workhorse.max_global_lock_fails}) " \
+                    "where the global lock could no be acquired within the specified timeout (#{timeout}). " \
+                    'A worker that obtained this lock may have crashed without ending the database ' \
+                    'connection properly. On MySQL, use "show processlist;" to see which connection(s) ' \
+                    'is / are holding the lock for a long period of time and consider killing them using '\
+                    "MySQL's \"kill <Id>\" command. This message will be issued only once per worker " \
+                    "and may only be re-triggered if the error happens again *after* the lock has " \
+                    "been solved in the meantime."
+
+          worker.log message
+          exception = StandardError.new(message)
+          Workhorse.on_exception.call(exception)
+        end
+      end
+
       return unless success
 
       yield

data/lib/workhorse.rb

@@ -6,6 +6,7 @@ require 'uri'
 
 require 'workhorse/enqueuer'
 require 'workhorse/scoped_env'
+require 'workhorse/active_job_extension'
 
 module Workhorse
   # Check if the available Arel version is greater or equal than 7.0.0
@@ -20,6 +21,12 @@ module Workhorse
       || fail('No performer is associated with the current thread. This method must always be called inside of a job.')
   end
 
+  # A worker will log an error and, if defined, call the on_exception callback,
+  # if it couldn't obtain the global lock for the specified number of times in a
+  # row.
+  mattr_accessor :max_global_lock_fails
+  self.max_global_lock_fails = 10
+
   mattr_accessor :tx_callback
   self.tx_callback = proc do |*args, &block|
     ActiveRecord::Base.transaction(*args, &block)

data/workhorse.gemspec

@@ -1,15 +1,15 @@
 # -*- encoding: utf-8 -*-
-# stub: workhorse 1.2.12 ruby lib
+# stub: workhorse 1.2.14 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "workhorse".freeze
-  s.version = "1.2.12"
+  s.version = "1.2.14"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]
   s.authors = ["Sitrox".freeze]
-  s.date = "2023-01-18"
-  s.files = [".github/workflows/ruby.yml".freeze, ".gitignore".freeze, ".releaser_config".freeze, ".rubocop.yml".freeze, "CHANGELOG.md".freeze, "FAQ.md".freeze, "Gemfile".freeze, "LICENSE".freeze, "README.md".freeze, "RUBY_VERSION".freeze, "Rakefile".freeze, "VERSION".freeze, "bin/rubocop".freeze, "lib/active_job/queue_adapters/workhorse_adapter.rb".freeze, "lib/generators/workhorse/install_generator.rb".freeze, "lib/generators/workhorse/templates/bin/workhorse.rb".freeze, "lib/generators/workhorse/templates/config/initializers/workhorse.rb".freeze, "lib/generators/workhorse/templates/create_table_jobs.rb".freeze, "lib/workhorse.rb".freeze, "lib/workhorse/daemon.rb".freeze, "lib/workhorse/daemon/shell_handler.rb".freeze, "lib/workhorse/db_job.rb".freeze, "lib/workhorse/enqueuer.rb".freeze, "lib/workhorse/jobs/cleanup_succeeded_jobs.rb".freeze, "lib/workhorse/jobs/detect_stale_jobs_job.rb".freeze, "lib/workhorse/jobs/run_active_job.rb".freeze, "lib/workhorse/jobs/run_rails_op.rb".freeze, "lib/workhorse/performer.rb".freeze, "lib/workhorse/poller.rb".freeze, "lib/workhorse/pool.rb".freeze, "lib/workhorse/scoped_env.rb".freeze, "lib/workhorse/worker.rb".freeze, "test/active_job/queue_adapters/workhorse_adapter_test.rb".freeze, "test/lib/db_schema.rb".freeze, "test/lib/jobs.rb".freeze, "test/lib/test_helper.rb".freeze, "test/workhorse/db_job_test.rb".freeze, "test/workhorse/enqueuer_test.rb".freeze, "test/workhorse/performer_test.rb".freeze, "test/workhorse/poller_test.rb".freeze, "test/workhorse/pool_test.rb".freeze, "test/workhorse/worker_test.rb".freeze, "workhorse.gemspec".freeze]
+  s.date = "2023-08-23"
+  s.files = [".github/workflows/ruby.yml".freeze, ".gitignore".freeze, ".releaser_config".freeze, ".rubocop.yml".freeze, "CHANGELOG.md".freeze, "FAQ.md".freeze, "Gemfile".freeze, "LICENSE".freeze, "README.md".freeze, "RUBY_VERSION".freeze, "Rakefile".freeze, "VERSION".freeze, "bin/rubocop".freeze, "lib/active_job/queue_adapters/workhorse_adapter.rb".freeze, "lib/generators/workhorse/install_generator.rb".freeze, "lib/generators/workhorse/templates/bin/workhorse.rb".freeze, "lib/generators/workhorse/templates/config/initializers/workhorse.rb".freeze, "lib/generators/workhorse/templates/create_table_jobs.rb".freeze, "lib/workhorse.rb".freeze, "lib/workhorse/active_job_extension.rb".freeze, "lib/workhorse/daemon.rb".freeze, "lib/workhorse/daemon/shell_handler.rb".freeze, "lib/workhorse/db_job.rb".freeze, "lib/workhorse/enqueuer.rb".freeze, "lib/workhorse/jobs/cleanup_succeeded_jobs.rb".freeze, "lib/workhorse/jobs/detect_stale_jobs_job.rb".freeze, "lib/workhorse/jobs/run_active_job.rb".freeze, "lib/workhorse/jobs/run_rails_op.rb".freeze, "lib/workhorse/performer.rb".freeze, "lib/workhorse/poller.rb".freeze, "lib/workhorse/pool.rb".freeze, "lib/workhorse/scoped_env.rb".freeze, "lib/workhorse/worker.rb".freeze, "test/active_job/queue_adapters/workhorse_adapter_test.rb".freeze, "test/lib/db_schema.rb".freeze, "test/lib/jobs.rb".freeze, "test/lib/test_helper.rb".freeze, "test/workhorse/db_job_test.rb".freeze, "test/workhorse/enqueuer_test.rb".freeze, "test/workhorse/performer_test.rb".freeze, "test/workhorse/poller_test.rb".freeze, "test/workhorse/pool_test.rb".freeze, "test/workhorse/worker_test.rb".freeze, "workhorse.gemspec".freeze]
   s.rubygems_version = "3.0.3".freeze
   s.summary = "Multi-threaded job backend with database queuing for ruby.".freeze
   s.test_files = ["test/active_job/queue_adapters/workhorse_adapter_test.rb".freeze, "test/lib/db_schema.rb".freeze, "test/lib/jobs.rb".freeze, "test/lib/test_helper.rb".freeze, "test/workhorse/db_job_test.rb".freeze, "test/workhorse/enqueuer_test.rb".freeze, "test/workhorse/performer_test.rb".freeze, "test/workhorse/poller_test.rb".freeze, "test/workhorse/pool_test.rb".freeze, "test/workhorse/worker_test.rb".freeze]

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: workhorse
 version: !ruby/object:Gem::Version
-  version: 1.2.12
+  version: 1.2.14
 platform: ruby
 authors:
 - Sitrox
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-01-18 00:00:00.000000000 Z
+date: 2023-08-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -203,6 +203,7 @@ files:
 - lib/generators/workhorse/templates/config/initializers/workhorse.rb
 - lib/generators/workhorse/templates/create_table_jobs.rb
 - lib/workhorse.rb
+- lib/workhorse/active_job_extension.rb
 - lib/workhorse/daemon.rb
 - lib/workhorse/daemon/shell_handler.rb
 - lib/workhorse/db_job.rb
@@ -245,7 +246,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.11
+rubygems_version: 3.4.10
 signing_key:
 specification_version: 4
 summary: Multi-threaded job backend with database queuing for ruby.