workhorse 1.2.12 → 1.2.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5d242b1602e66d5de0ad72ef85ce8a15e885248ca14a075a2f25bb60b80c0364
-  data.tar.gz: db8f8a6aecfe00aae955775b09c7d08568653258ef56ab874839cde702725854
+  metadata.gz: c528399ac973ce36bd6bdb993ac2d234eddf8836b8fc997031dd4009487ed252
+  data.tar.gz: 8482be9f53a1c8b5eb238784a6f350f79a0bf7b5c2c42a3921667ef2a82340d6
 SHA512:
-  metadata.gz: aa64c4d9ff6c4085c42018ddd293b04fd8078e702306631d8b9918c917a53903b48832d561216c25f2dcae6c36cbc15f2e6587301302165b1a3fc639c4dfa59a
-  data.tar.gz: f76393011d3be7b4002d27a5836e86ada615375130d04ae5f1ea64002fdad4cb5f2410c45e829ad726ce386c600d353208cba9d25de9e296db796f864373f1a0
+  metadata.gz: 83b44c60c1789755e2d09a2f32ea71728287d54ab1904bf5a6f9e88f225d02da0280d784e56d3352c336efb292aa864f841b85d1c085df7425878551bdec27ed
+  data.tar.gz: a554b93bc1c0c18bd1472981e2a84e80f79dfc0561f03c89839cc9e4f32d6480cb4fa327cd046cc8d9582a142af1092a4ffb1c5c492cb79fbb5af2fb3ebef725

data/CHANGELOG.md

@@ -1,5 +1,22 @@
 # Workhorse Changelog
 
+## 1.2.13 - 2023-02-20
+
+* Add the `config.max_global_lock_fails` setting (defaults to 10). If a
+  worker's poller cannot acquire the global lock, an error is logged, and if
+  `config.on_exception` is configured, the error is handled using this callback.
+
+  This change allows you to be aware of essentially defunct worker processes due
+  to a global lock that could not be obtained, for example, because of another
+  worker that was killed without properly releasing the lock. However, this is
+  an edge case because:
+
+  1. The lock is released by Workhorse in an `ensure` block.
+  2. At least MySQL is supposed to release global locks obtained in a connection
+     when that connection is closed.
+
+  Sitrox reference: #110339.
+
 ## 1.2.12 - 2023-01-18
 
 * Call `on_exception` callback on failed `Performer` initialization (e.g. when

data/README.md

@@ -419,6 +419,24 @@ This means that you should always have an external *watcher* (usually a
 cronjob), that calls the `workhorse watch` command regularly. This would
 automatically restart crashed worker processes.
 
+### Unobtainable Locks
+
+Each Workhorse worker uses a poller to check the database for new jobs. To
+ensure that no job is obtained by more than one worker, a global database lock
+is used. If a worker is killed, it may happen that the lock is not properly
+released, which can cause all pollers to stop working because they cannot
+acquire the lock. This is an edge case, as the locks should typically be
+released properly, even if a worker process is killed using `SIGKILL`.
+
+In the event that this still happens, Workhorse takes the following steps:
+
+- Logs when a lock could not be obtained.
+- Retries acquiring the lock on the next poll.
+- Calls the `on_exception` callback (if configured) after a configurable number of consecutive failures to obtain the lock.
+
+The maximum number of consecutive failures can be configured using
+`config.max_global_lock_fails`, which defaults to 10.
+
 ### Stuck queues
 
 Jobs in named queues (non-null queues) are always run sequentially. This means

data/VERSION

@@ -1 +1 @@
-1.2.12
+1.2.13

data/lib/workhorse/poller.rb

@@ -15,6 +15,8 @@ module Workhorse
       @table = Workhorse::DbJob.arel_table
       @is_oracle = ActiveRecord::Base.connection.adapter_name == 'OracleEnhanced'
       @instant_repoll = Concurrent::AtomicBoolean.new(false)
+      @global_lock_fails = 0
+      @max_global_lock_fails_reached = false
     end
 
     def running?
@@ -86,6 +88,38 @@ module Workhorse
         success = result == 1
       end
 
+      if success
+        @global_lock_fails = 0
+        @max_global_lock_fails_reached = false
+      else
+        @global_lock_fails += 1
+
+        unless @max_global_lock_fails_reached
+          worker.log 'Could not obtain global lock, retrying with next poll.', :warn
+        end
+
+        if @global_lock_fails > Workhorse.max_global_lock_fails && !@max_global_lock_fails_reached
+          @max_global_lock_fails_reached = true
+
+          worker.log 'Could not obtain global lock, retrying with next poll. '\
+                     'This will be the last such message for this worker until '\
+                     'the issue is resolved.', :warn
+
+          message = "Worker reached maximum number of consecutive times (#{Workhorse.max_global_lock_fails}) " \
+                    "where the global lock could no be acquired within the specified timeout (#{timeout}). " \
+                    'A worker that obtained this lock may have crashed without ending the database ' \
+                    'connection properly. On MySQL, use "show processlist;" to see which connection(s) ' \
+                    'is / are holding the lock for a long period of time and consider killing them using '\
+                    "MySQL's \"kill <Id>\" command. This message will be issued only once per worker " \
+                    "and may only be re-triggered if the error happens again *after* the lock has " \
+                    "been solved in the meantime."
+
+          worker.log message
+          exception = StandardError.new(message)
+          Workhorse.on_exception.call(exception)
+        end
+      end
+
       return unless success
 
       yield

data/lib/workhorse.rb

@@ -20,6 +20,12 @@ module Workhorse
       || fail('No performer is associated with the current thread. This method must always be called inside of a job.')
   end
 
+  # A worker will log an error and, if defined, call the on_exception callback,
+  # if it couldn't obtain the global lock for the specified number of times in a
+  # row.
+  mattr_accessor :max_global_lock_fails
+  self.max_global_lock_fails = 10
+
   mattr_accessor :tx_callback
   self.tx_callback = proc do |*args, &block|
     ActiveRecord::Base.transaction(*args, &block)

data/workhorse.gemspec

@@ -1,14 +1,14 @@
 # -*- encoding: utf-8 -*-
-# stub: workhorse 1.2.12 ruby lib
+# stub: workhorse 1.2.13 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "workhorse".freeze
-  s.version = "1.2.12"
+  s.version = "1.2.13"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]
   s.authors = ["Sitrox".freeze]
-  s.date = "2023-01-18"
+  s.date = "2023-02-20"
   s.files = [".github/workflows/ruby.yml".freeze, ".gitignore".freeze, ".releaser_config".freeze, ".rubocop.yml".freeze, "CHANGELOG.md".freeze, "FAQ.md".freeze, "Gemfile".freeze, "LICENSE".freeze, "README.md".freeze, "RUBY_VERSION".freeze, "Rakefile".freeze, "VERSION".freeze, "bin/rubocop".freeze, "lib/active_job/queue_adapters/workhorse_adapter.rb".freeze, "lib/generators/workhorse/install_generator.rb".freeze, "lib/generators/workhorse/templates/bin/workhorse.rb".freeze, "lib/generators/workhorse/templates/config/initializers/workhorse.rb".freeze, "lib/generators/workhorse/templates/create_table_jobs.rb".freeze, "lib/workhorse.rb".freeze, "lib/workhorse/daemon.rb".freeze, "lib/workhorse/daemon/shell_handler.rb".freeze, "lib/workhorse/db_job.rb".freeze, "lib/workhorse/enqueuer.rb".freeze, "lib/workhorse/jobs/cleanup_succeeded_jobs.rb".freeze, "lib/workhorse/jobs/detect_stale_jobs_job.rb".freeze, "lib/workhorse/jobs/run_active_job.rb".freeze, "lib/workhorse/jobs/run_rails_op.rb".freeze, "lib/workhorse/performer.rb".freeze, "lib/workhorse/poller.rb".freeze, "lib/workhorse/pool.rb".freeze, "lib/workhorse/scoped_env.rb".freeze, "lib/workhorse/worker.rb".freeze, "test/active_job/queue_adapters/workhorse_adapter_test.rb".freeze, "test/lib/db_schema.rb".freeze, "test/lib/jobs.rb".freeze, "test/lib/test_helper.rb".freeze, "test/workhorse/db_job_test.rb".freeze, "test/workhorse/enqueuer_test.rb".freeze, "test/workhorse/performer_test.rb".freeze, "test/workhorse/poller_test.rb".freeze, "test/workhorse/pool_test.rb".freeze, "test/workhorse/worker_test.rb".freeze, "workhorse.gemspec".freeze]
   s.rubygems_version = "3.0.3".freeze
   s.summary = "Multi-threaded job backend with database queuing for ruby.".freeze

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: workhorse
 version: !ruby/object:Gem::Version
-  version: 1.2.12
+  version: 1.2.13
 platform: ruby
 authors:
 - Sitrox
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-01-18 00:00:00.000000000 Z
+date: 2023-02-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -178,8 +178,8 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description:
-email:
+description: 
+email: 
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -227,10 +227,10 @@ files:
 - test/workhorse/pool_test.rb
 - test/workhorse/worker_test.rb
 - workhorse.gemspec
-homepage:
+homepage: 
 licenses: []
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -245,8 +245,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.11
-signing_key:
+rubygems_version: 3.0.3.1
+signing_key: 
 specification_version: 4
 summary: Multi-threaded job backend with database queuing for ruby.
 test_files: