good_job 3.28.2 → 3.29.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c01c931bf907016e2758a52326884136539fd860a0aa29582afbdf522c64dabf
-  data.tar.gz: 6e20623d0020c15e447bc9b1857587856e9abea6e698bed98d3e748ebe3bcf4f
+  metadata.gz: dba2ffb689b77716dac30f6c89c02a8ddd2cad9d340ca72fe86405d4cee446e1
+  data.tar.gz: 43c511926416fe1360e312ceb375b7db10e9c95ed9c8db040d9828e497b77eb0
 SHA512:
-  metadata.gz: 43f9fd7ecdc6c293d3b2147da775fb8eaf187d49b003eaf999efa39b6401cdd255dcd6496202611b486f665b2063c56d66f825c8f33928260261ea79c2766789
-  data.tar.gz: 68205cc3bdb56ffde237eee3c2dff33fc287f7043bf3c84691be0a0957ec9fc56a21899ca4f56eb1f2482f959cfe217640c9094989887d57d9b13c7433244285
+  metadata.gz: b52422348cb25a90c17b755978b13278591ee5e388e3b16648fcd89fd77f6570a796ec44d7c64d4e9b0111b5102f2b80edfcbcd9236d13a2394a9ea5cc32297a
+  data.tar.gz: 91fab8760f5fbf4ff137fc66da09ead108201d0a47dd5d6859a4afed9d66b88f26c758ed95e4243591e6b719d417d7fa1d9ae3a85e5d3619f6c1b374435996c6

data/CHANGELOG.md

@@ -1,5 +1,33 @@
 # Changelog
 
+## [v3.29.0](https://github.com/bensheldon/good_job/tree/v3.29.0) (2024-05-22)
+
+[Full Changelog](https://github.com/bensheldon/good_job/compare/v3.28.3...v3.29.0)
+
+**Merged pull requests:**
+
+- Add association between Process and Jobs, and add a heartbeat to the Process record [\#999](https://github.com/bensheldon/good_job/pull/999) ([bensheldon](https://github.com/bensheldon))
+
+## [v3.28.3](https://github.com/bensheldon/good_job/tree/v3.28.3) (2024-05-18)
+
+[Full Changelog](https://github.com/bensheldon/good_job/compare/v3.28.2...v3.28.3)
+
+**Fixed bugs:**
+
+- Strip more whitespace when parsing queues string [\#1352](https://github.com/bensheldon/good_job/pull/1352) ([bensheldon](https://github.com/bensheldon))
+- Fix latest migration not affecting `GoodJob.migrated?` [\#1345](https://github.com/bensheldon/good_job/pull/1345) ([Earlopain](https://github.com/Earlopain))
+
+**Closed issues:**
+
+- Whitespace in `queues` configuration can cause issues. [\#1351](https://github.com/bensheldon/good_job/issues/1351)
+- How to properly handle interrupts [\#1343](https://github.com/bensheldon/good_job/issues/1343)
+- ActiveSupport::CurrentAttributes Compatibility [\#1341](https://github.com/bensheldon/good_job/issues/1341)
+
+**Merged pull requests:**
+
+- Don't abort CI jobs when a single one fails [\#1346](https://github.com/bensheldon/good_job/pull/1346) ([Earlopain](https://github.com/Earlopain))
+- Clarify PgBouncer Compatibility [\#1338](https://github.com/bensheldon/good_job/pull/1338) ([isaac](https://github.com/isaac))
+
 ## [v3.28.2](https://github.com/bensheldon/good_job/tree/v3.28.2) (2024-04-26)
 
 [Full Changelog](https://github.com/bensheldon/good_job/compare/v3.28.1...v3.28.2)

data/README.md

@@ -1383,7 +1383,7 @@ _Note: Rails `travel`/`travel_to` time helpers do not have millisecond precision
 
 GoodJob is not compatible with PgBouncer in _transaction_ mode, but is compatible with PgBouncer's _connection_ mode. GoodJob uses connection-based advisory locks and LISTEN/NOTIFY, both of which require full database connections.
 
-A workaround to this limitation is to make a direct database connection available to GoodJob. With Rails 6.0's support for [multiple databases](https://guides.rubyonrails.org/active_record_multiple_databases.html), a direct connection to the database can be configured:
+If you want to use PgBouncer with the rest of your Rails app you can workaround this limitation by making a direct database connection available to GoodJob. With Rails 6.0's support for [multiple databases](https://guides.rubyonrails.org/active_record_multiple_databases.html), a direct connection to the database can be configured by following the three steps below.
 
 1. Define a direct connection to your database that is not proxied through PgBouncer, for example:
 

data/app/models/good_job/base_execution.rb

@@ -84,6 +84,13 @@ module GoodJob
         migration_pending_warning!
         false
       end
+
+      def process_lock_migrated?
+        return true if connection.index_name_exists?(:good_job_executions, :index_good_job_executions_on_process_id_and_created_at)
+
+        migration_pending_warning!
+        false
+      end
     end
 
     # The ActiveJob job class, as a string

data/app/models/good_job/execution.rb

@@ -40,7 +40,7 @@ module GoodJob
     #   GoodJob::Execution.queue_parser('-queue1,queue2')
     #   => { exclude: [ 'queue1', 'queue2' ] }
     def self.queue_parser(string)
-      string = string.presence || '*'
+      string = string.strip.presence || '*'
 
       case string.first
       when '-'
@@ -262,7 +262,7 @@ module GoodJob
     #   return value for the job's +#perform+ method, and the exception the job
     #   raised, if any (if the job raised, then the second array entry will be
     #   +nil+). If there were no jobs to execute, returns +nil+.
-    def self.perform_with_advisory_lock(parsed_queues: nil, queue_select_limit: nil)
+    def self.perform_with_advisory_lock(lock_id:, parsed_queues: nil, queue_select_limit: nil)
       execution = nil
       result = nil
 
@@ -270,7 +270,7 @@ module GoodJob
         execution = executions.first
         if execution&.executable?
           yield(execution) if block_given?
-          result = execution.perform
+          result = execution.perform(lock_id: lock_id)
         else
           execution = nil
           yield(nil) if block_given?
@@ -367,7 +367,7 @@ module GoodJob
     #   An array of the return value of the job's +#perform+ method and the
     #   exception raised by the job, if any. If the job completed successfully,
     #   the second array entry (the exception) will be +nil+ and vice versa.
-    def perform
+    def perform(lock_id:)
       run_callbacks(:perform) do
         raise PreviouslyPerformedError, 'Cannot perform a job that has already been performed' if finished_at
 
@@ -397,17 +397,37 @@ module GoodJob
 
           if discrete?
             transaction do
-              discrete_execution = discrete_executions.create!(
+              discrete_execution_attrs = {
                 job_class: job_class,
                 queue_name: queue_name,
                 serialized_params: serialized_params,
                 scheduled_at: (scheduled_at || created_at),
-                created_at: job_performed_at
-              )
-              update!(performed_at: job_performed_at, executions_count: ((executions_count || 0) + 1))
+                created_at: job_performed_at,
+              }
+              discrete_execution_attrs[:process_id] = lock_id if GoodJob::DiscreteExecution.columns_hash.key?("process_id")
+
+              execution_attrs = {
+                performed_at: job_performed_at,
+                executions_count: ((executions_count || 0) + 1),
+              }
+              if GoodJob::Execution.columns_hash.key?("locked_by_id")
+                execution_attrs[:locked_by_id] = lock_id
+                execution_attrs[:locked_at] = Time.current
+              end
+
+              discrete_execution = discrete_executions.create!(discrete_execution_attrs)
+              update!(execution_attrs)
             end
           else
-            update!(performed_at: job_performed_at)
+            execution_attrs = {
+              performed_at: job_performed_at,
+            }
+            if GoodJob::Execution.columns_hash.key?("locked_by_id")
+              execution_attrs[:locked_by_id] = lock_id
+              execution_attrs[:locked_at] = Time.current
+            end
+
+            update!(execution_attrs)
           end
 
           ActiveSupport::Notifications.instrument("perform_job.good_job", { execution: self, process_id: current_thread.process_id, thread_name: current_thread.thread_name }) do |instrument_payload|
@@ -450,7 +470,11 @@ module GoodJob
           end
         end
 
-        job_attributes = {}
+        job_attributes = if self.class.columns_hash.key?("locked_by_id")
+                           { locked_by_id: nil, locked_at: nil }
+                         else
+                           {}
+                         end
 
         job_error = result.handled_error || result.unhandled_error
         if job_error

data/app/models/good_job/job.rb

@@ -29,6 +29,7 @@ module GoodJob
     self.implicit_order_column = 'created_at'
 
     belongs_to :batch, class_name: 'GoodJob::BatchRecord', inverse_of: :jobs, optional: true
+    belongs_to :locked_by_process, class_name: "GoodJob::Process", foreign_key: :locked_by_id, inverse_of: :locked_jobs, optional: true
     has_many :executions, -> { order(created_at: :asc) }, class_name: 'GoodJob::Execution', foreign_key: 'active_job_id', inverse_of: :job # rubocop:disable Rails/HasManyOrHasOneDependent
     has_many :discrete_executions, -> { order(created_at: :asc) }, class_name: 'GoodJob::DiscreteExecution', foreign_key: 'active_job_id', primary_key: :active_job_id, inverse_of: :job # rubocop:disable Rails/HasManyOrHasOneDependent
 

data/app/models/good_job/process.rb

@@ -3,7 +3,7 @@
 require 'socket'
 
 module GoodJob # :nodoc:
-  # ActiveRecord model that represents an GoodJob process (either async or CLI).
+  # Active Record model that represents a GoodJob capsule/process (either async or CLI).
   class Process < BaseRecord
     include AdvisoryLockable
     include OverridableConnection
@@ -15,50 +15,74 @@ module GoodJob # :nodoc:
 
     self.table_name = 'good_job_processes'
     self.implicit_order_column = 'created_at'
+    LOCK_TYPES = [
+      LOCK_TYPE_ADVISORY = 'advisory',
+    ].freeze
 
-    cattr_reader :mutex, default: Mutex.new
-    cattr_accessor :_current_id, default: nil
-    cattr_accessor :_pid, default: nil
+    LOCK_TYPE_ENUMS = {
+      LOCK_TYPE_ADVISORY => 1,
+    }.freeze
+
+    self.table_name = 'good_job_processes'
+
+    has_many :locked_jobs, class_name: "GoodJob::Job", foreign_key: :locked_by_id, inverse_of: :locked_by_process, dependent: nil
+    after_destroy { locked_jobs.update_all(locked_by_id: nil) if GoodJob::Job.columns_hash.key?("locked_by_id") } # rubocop:disable Rails/SkipsModelValidations
 
     # Processes that are active and locked.
     # @!method active
     # @!scope class
     # @return [ActiveRecord::Relation]
-    scope :active, -> { advisory_locked }
+    scope :active, (lambda do
+      if lock_type_migrated?
+        query = joins_advisory_locks
+        query.where(lock_type: LOCK_TYPE_ENUMS[LOCK_TYPE_ADVISORY]).advisory_locked
+          .or(query.where(lock_type: nil).where(arel_table[:updated_at].gt(EXPIRED_INTERVAL.ago)))
+      else
+        advisory_locked
+      end
+    end)
 
     # Processes that are inactive and unlocked (e.g. SIGKILLed)
     # @!method active
     # @!scope class
     # @return [ActiveRecord::Relation]
-    scope :inactive, -> { advisory_unlocked }
-
-    # UUID that is unique to the current process and changes when forked.
-    # @return [String]
-    def self.current_id
-      mutex.synchronize { ns_current_id }
-    end
+    scope :inactive, (lambda do
+      if lock_type_migrated?
+        query = joins_advisory_locks
+        query.where(lock_type: LOCK_TYPE_ENUMS[LOCK_TYPE_ADVISORY]).advisory_unlocked
+          .or(query.where(lock_type: nil).where(arel_table[:updated_at].lt(EXPIRED_INTERVAL.ago)))
+      else
+        advisory_unlocked
+      end
+    end)
 
-    def self.ns_current_id
-      if _current_id.nil? || _pid != ::Process.pid
-        self._current_id = SecureRandom.uuid
-        self._pid = ::Process.pid
+    # Deletes all inactive process records.
+    def self.cleanup
+      inactive.find_each do |process|
+        GoodJob::Job.where(locked_by_id: process.id).update_all(locked_by_id: nil, locked_at: nil) if GoodJob::Job.columns_hash.key?("locked_by_id") # rubocop:disable Rails/SkipsModelValidations
+        process.delete
       end
-      _current_id
     end
 
-    # Hash representing metadata about the current process.
-    # @return [Hash]
-    def self.current_state
-      mutex.synchronize { ns_current_state }
+    # @return [Boolean]
+    def self.lock_type_migrated?
+      columns_hash["lock_type"].present?
     end
 
-    def self.ns_current_state
-      total_succeeded_executions_count = GoodJob::Scheduler.instances.sum { |scheduler| scheduler.stats.fetch(:succeeded_executions_count, 0) }
-      total_errored_executions_count = GoodJob::Scheduler.instances.sum { |scheduler| scheduler.stats.fetch(:errored_executions_count, 0) }
-      total_empty_executions_count = GoodJob::Scheduler.instances.sum { |scheduler| scheduler.stats.fetch(:empty_executions_count, 0) }
+    def self.create_record(id:, with_advisory_lock: false)
+      attributes = {
+        id: id,
+        state: process_state,
+      }
+      if with_advisory_lock
+        attributes[:create_with_advisory_lock] = true
+        attributes[:lock_type] = LOCK_TYPE_ADVISORY if lock_type_migrated?
+      end
+      create!(attributes)
+    end
 
+    def self.process_state
       {
-        id: ns_current_id,
         hostname: Socket.gethostname,
         pid: ::Process.pid,
         proctitle: $PROGRAM_NAME,
@@ -66,10 +90,8 @@ module GoodJob # :nodoc:
         retry_on_unhandled_error: GoodJob.retry_on_unhandled_error,
         schedulers: GoodJob::Scheduler.instances.map(&:stats),
         cron_enabled: GoodJob.configuration.enable_cron?,
-        total_succeeded_executions_count: total_succeeded_executions_count,
-        total_errored_executions_count: total_errored_executions_count,
-        total_executions_count: total_succeeded_executions_count + total_errored_executions_count,
-        total_empty_executions_count: total_empty_executions_count,
+        total_succeeded_executions_count: GoodJob::Scheduler.instances.sum { |scheduler| scheduler.stats.fetch(:succeeded_executions_count) },
+        total_errored_executions_count: GoodJob::Scheduler.instances.sum { |scheduler| scheduler.stats.fetch(:errored_executions_count) },
         database_connection_pool: {
           size: connection_pool.size,
           active: connection_pool.connections.count(&:in_use?),
@@ -77,45 +99,36 @@ module GoodJob # :nodoc:
       }
     end
 
-    # Deletes all inactive process records.
-    def self.cleanup
-      inactive.delete_all
-    end
-
-    # Registers the current process in the database
-    # @return [GoodJob::Process]
-    def self.register
-      mutex.synchronize do
-        process_state = ns_current_state
-        create(id: process_state[:id], state: process_state, create_with_advisory_lock: true)
-      rescue ActiveRecord::RecordNotUnique
-        find(ns_current_state[:id])
-      end
-    end
-
     def refresh
-      mutex.synchronize do
-        reload
-        update(state: self.class.ns_current_state, updated_at: Time.current)
-      rescue ActiveRecord::RecordNotFound
-        false
-      end
+      self.state = self.class.process_state
+      reload.update(state: state, updated_at: Time.current)
+    rescue ActiveRecord::RecordNotFound
+      @new_record = true
+      self.created_at = self.updated_at = nil
+      state_will_change!
+      save
     end
 
-    # Unregisters the instance.
-    def deregister
-      return unless owns_advisory_lock?
+    def refresh_if_stale(cleanup: false)
+      return unless stale?
 
-      mutex.synchronize do
-        destroy!
-        advisory_unlock
-      end
+      result = refresh
+      self.class.cleanup if cleanup
+      result
     end
 
     def state
       super || {}
     end
 
+    def stale?
+      updated_at < STALE_INTERVAL.ago
+    end
+
+    def expired?
+      updated_at < EXPIRED_INTERVAL.ago
+    end
+
     def basename
       File.basename(state.fetch("proctitle", ""))
     end
@@ -124,20 +137,20 @@ module GoodJob # :nodoc:
       state.fetch("schedulers", [])
     end
 
-    def refresh_if_stale(cleanup: false)
-      return unless stale?
+    def lock_type
+      return unless self.class.columns_hash['lock_type']
 
-      result = refresh
-      self.class.cleanup if cleanup
-      result
+      enum = super
+      LOCK_TYPE_ENUMS.key(enum) if enum
     end
 
-    def stale?
-      updated_at < STALE_INTERVAL.ago
-    end
+    def lock_type=(value)
+      return unless self.class.columns_hash['lock_type']
 
-    def expired?
-      updated_at < EXPIRED_INTERVAL.ago
+      enum = LOCK_TYPE_ENUMS[value]
+      raise(ArgumentError, "Invalid error_event: #{value}") if value && !enum
+
+      super(enum)
     end
   end
 end

data/lib/generators/good_job/templates/install/migrations/create_good_jobs.rb.erb

@@ -30,6 +30,8 @@ class CreateGoodJobs < ActiveRecord::Migration<%= migration_version %>
       t.text :job_class
       t.integer :error_event, limit: 2
       t.text :labels, array: true
+      t.uuid :locked_by_id
+      t.datetime :locked_at
     end
 
     create_table :good_job_batches, id: :uuid do |t|
@@ -58,11 +60,13 @@ class CreateGoodJobs < ActiveRecord::Migration<%= migration_version %>
       t.text :error
       t.integer :error_event, limit: 2
       t.text :error_backtrace, array: true
+      t.uuid :process_id
     end
 
     create_table :good_job_processes, id: :uuid do |t|
       t.timestamps
       t.jsonb :state
+      t.integer :lock_type, limit: 2
     end
 
     create_table :good_job_settings, id: :uuid do |t|
@@ -88,5 +92,10 @@ class CreateGoodJobs < ActiveRecord::Migration<%= migration_version %>
     add_index :good_jobs, :labels, using: :gin, where: "(labels IS NOT NULL)", name: :index_good_jobs_on_labels
 
     add_index :good_job_executions, [:active_job_id, :created_at], name: :index_good_job_executions_on_active_job_id_and_created_at
+    add_index :good_jobs, [:priority, :scheduled_at], order: { priority: "ASC NULLS LAST", scheduled_at: :asc },
+      where: "finished_at IS NULL AND locked_by_id IS NULL", name: :index_good_jobs_on_priority_scheduled_at_unfinished_unlocked
+    add_index :good_jobs, :locked_by_id,
+      where: "locked_by_id IS NOT NULL", name: "index_good_jobs_on_locked_by_id"
+    add_index :good_job_executions, [:process_id, :created_at], name: :index_good_job_executions_on_process_id_and_created_at
   end
 end

data/lib/generators/good_job/templates/update/migrations/13_create_good_job_process_lock_ids.rb.erb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+class CreateGoodJobProcessLockIds < ActiveRecord::Migration<%= migration_version %>
+  def change
+    reversible do |dir|
+      dir.up do
+        # Ensure this incremental update migration is idempotent
+        # with monolithic install migration.
+        return if connection.column_exists?(:good_jobs, :locked_by_id)
+      end
+    end
+
+    add_column :good_jobs, :locked_by_id, :uuid
+    add_column :good_jobs, :locked_at, :datetime
+    add_column :good_job_executions, :process_id, :uuid
+    add_column :good_job_processes, :lock_type, :integer, limit: 2
+  end
+end

data/lib/generators/good_job/templates/update/migrations/14_create_good_job_process_lock_indexes.rb.erb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+class CreateGoodJobProcessLockIndexes < ActiveRecord::Migration<%= migration_version %>
+  disable_ddl_transaction!
+
+  def change
+    reversible do |dir|
+      dir.up do
+        unless connection.index_name_exists?(:good_jobs, :index_good_jobs_on_priority_scheduled_at_unfinished_unlocked)
+          add_index :good_jobs, [:priority, :scheduled_at],
+                    order: { priority: "ASC NULLS LAST", scheduled_at: :asc },
+                    where: "finished_at IS NULL AND locked_by_id IS NULL",
+                    name: :index_good_jobs_on_priority_scheduled_at_unfinished_unlocked,
+                    algorithm: :concurrently
+        end
+
+        unless connection.index_name_exists?(:good_jobs, :index_good_jobs_on_locked_by_id)
+          add_index :good_jobs, :locked_by_id,
+                    where: "locked_by_id IS NOT NULL",
+                    name: :index_good_jobs_on_locked_by_id,
+                    algorithm: :concurrently
+        end
+
+        unless connection.index_name_exists?(:good_job_executions, :index_good_job_executions_on_process_id_and_created_at)
+          add_index :good_job_executions, [:process_id, :created_at],
+                    name: :index_good_job_executions_on_process_id_and_created_at,
+                    algorithm: :concurrently
+        end
+      end
+
+      dir.down do
+        remove_index(:good_jobs, name: :index_good_jobs_on_priority_scheduled_at_unfinished_unlocked) if connection.index_name_exists?(:good_jobs, :index_good_jobs_on_priority_scheduled_at_unfinished_unlocked)
+        remove_index(:good_jobs, name: :index_good_jobs_on_locked_by_id) if connection.index_name_exists?(:good_jobs, :index_good_jobs_on_locked_by_id)
+        remove_index(:good_job_executions, name: :index_good_job_executions_on_process_id_and_created_at) if connection.index_name_exists?(:good_job_executions, :index_good_job_executions_on_process_id_and_created_at)
+      end
+    end
+  end
+end

data/lib/good_job/adapter.rb

@@ -97,16 +97,17 @@ module GoodJob
           end
         end
 
+        @capsule.tracker.register
         begin
           until inline_executions.empty?
             begin
               inline_execution = inline_executions.shift
-              inline_result = inline_execution.perform
+              inline_result = inline_execution.perform(lock_id: @capsule.tracker.id_for_lock)
 
               retried_execution = inline_result.retried
               while retried_execution && retried_execution.scheduled_at <= Time.current
                 inline_execution = retried_execution
-                inline_result = inline_execution.perform
+                inline_result = inline_execution.perform(lock_id: @capsule.tracker.id_for_lock)
                 retried_execution = inline_result.retried
               end
             ensure
@@ -116,6 +117,7 @@ module GoodJob
             raise inline_result.unhandled_error if inline_result.unhandled_error
           end
         ensure
+          @capsule.tracker.unregister
           inline_executions.each(&:advisory_unlock)
         end
 
@@ -168,12 +170,12 @@ module GoodJob
             create_with_advisory_lock: true
           )
           begin
-            result = execution.perform
+            result = @capsule.tracker.register { execution.perform(lock_id: @capsule.tracker.id_for_lock) }
 
             retried_execution = result.retried
             while retried_execution && (retried_execution.scheduled_at.nil? || retried_execution.scheduled_at <= Time.current)
               execution = retried_execution
-              result = execution.perform
+              result = @capsule.tracker.register { execution.perform(lock_id: @capsule.tracker.id_for_lock) }
               retried_execution = result.retried
             end
 

data/lib/good_job/capsule.rb

@@ -11,6 +11,10 @@ module GoodJob
     #   @return [Array<GoodJob::Capsule>, nil]
     cattr_reader :instances, default: Concurrent::Array.new, instance_reader: false
 
+    delegate :register, :renew, :unregister, :id_for_lock, to: :@tracker, prefix: :_tracker
+
+    attr_reader :tracker
+
     # @param configuration [GoodJob::Configuration] Configuration to use for this capsule.
     def initialize(configuration: nil)
       @configuration = configuration
@@ -18,6 +22,9 @@ module GoodJob
       @started_at = nil
       @mutex = Mutex.new
 
+      @shared_executor = GoodJob::SharedExecutor.new
+      @tracker = GoodJob::CapsuleTracker.new(executor: @shared_executor)
+
       self.class.instances << self
     end
 
@@ -29,15 +36,13 @@ module GoodJob
       @mutex.synchronize do
         return unless startable?(force: force)
 
-        @shared_executor = GoodJob::SharedExecutor.new
-        @notifier = GoodJob::Notifier.new(enable_listening: configuration.enable_listen_notify, executor: @shared_executor.executor)
+        @notifier = GoodJob::Notifier.new(enable_listening: configuration.enable_listen_notify, capsule: self, executor: @shared_executor)
         @poller = GoodJob::Poller.new(poll_interval: configuration.poll_interval)
-        @multi_scheduler = GoodJob::MultiScheduler.from_configuration(configuration, warm_cache_on_initialize: true)
+        @multi_scheduler = GoodJob::MultiScheduler.from_configuration(configuration, capsule: self, warm_cache_on_initialize: true)
         @notifier.recipients.push([@multi_scheduler, :create_thread])
         @poller.recipients.push(-> { @multi_scheduler.create_thread({ fanout: true }) })
 
-        @cron_manager = GoodJob::CronManager.new(configuration.cron_entries, start_on_initialize: true, executor: @shared_executor.executor) if configuration.enable_cron?
-
+        @cron_manager = GoodJob::CronManager.new(configuration.cron_entries, start_on_initialize: true, executor: @shared_executor) if configuration.enable_cron?
         @startable = false
         @started_at = Time.current
       end
@@ -52,7 +57,7 @@ module GoodJob
     # @return [void]
     def shutdown(timeout: NONE)
       timeout = configuration.shutdown_timeout if timeout == NONE
-      GoodJob._shutdown_all([@shared_executor, @notifier, @poller, @multi_scheduler, @cron_manager].compact, timeout: timeout)
+      GoodJob._shutdown_all([@notifier, @poller, @multi_scheduler, @cron_manager].compact, after: [@shared_executor], timeout: timeout)
       @startable = false
       @started_at = nil
     end
@@ -74,7 +79,7 @@ module GoodJob
 
     # @return [Boolean] Whether the capsule has been shutdown.
     def shutdown?
-      [@shared_executor, @notifier, @poller, @multi_scheduler, @cron_manager].compact.all?(&:shutdown?)
+      [@notifier, @poller, @multi_scheduler, @cron_manager].compact.all?(&:shutdown?)
     end
 
     # @param duration [nil, Numeric] Length of idleness to check for (in seconds).
@@ -99,6 +104,12 @@ module GoodJob
       @multi_scheduler&.create_thread(job_state)
     end
 
+    # UUID for this capsule; to be used for inspection (not directly for locking jobs).
+    # @return [String]
+    def process_id
+      @tracker.process_id
+    end
+
     private
 
     def configuration

data/lib/good_job/capsule_tracker.rb

@@ -0,0 +1,227 @@
+# frozen_string_literal: true
+
+module GoodJob # :nodoc:
+  # CapsuleTracker save a record in the database and periodically refreshes it. The intention is to
+  # create a heartbeat that can be used to determine whether a capsule/process is still active
+  # and use that to lock (or unlock) jobs.
+  class CapsuleTracker
+    # The database record used for tracking.
+    # @return [GoodJob::Process, nil]
+    attr_reader :record
+
+    # Number of tracked job executions.
+    attr_reader :locks
+
+    # Number of tracked job executions with advisory locks.
+    # @return [Integer]
+    attr_reader :advisory_locks
+
+    # @!attribute [r] instances
+    #   @!scope class
+    #   List of all instantiated CapsuleTrackers in the current process.
+    #   @return [Array<GoodJob::CapsuleTracker>, nil]
+    cattr_reader :instances, default: Concurrent::Array.new, instance_reader: false
+
+    # @param executor [Concurrent::AbstractExecutorService] The executor to use for refreshing the process record.
+    def initialize(executor: Concurrent.global_io_executor)
+      @executor = executor
+      @mutex = Mutex.new
+      @locks = 0
+      @advisory_locked_connection = nil
+      @record_id = SecureRandom.uuid
+      @record = nil
+      @refresh_task = nil
+
+      # AS::ForkTracker is only present on Rails v6.1+.
+      # Fall back to PID checking if ForkTracker is not available
+      if defined?(ActiveSupport::ForkTracker)
+        ActiveSupport::ForkTracker.after_fork { reset }
+        @forktracker = true
+      else
+        @ruby_pid = ::Process.pid
+        @forktracker = false
+      end
+
+      self.class.instances << self
+    end
+
+    # The UUID to use for locking. May be nil if the process is not registered or is unusable/expired.
+    # If UUID has not yet been persisted to the database, this method will make a query to insert or update it.
+    # @return [String, nil]
+    def id_for_lock
+      value = nil
+      synchronize do
+        next if @locks.zero?
+
+        reset_on_fork
+        if @record
+          @record.refresh_if_stale
+        else
+          @record = GoodJob::Process.create_record(id: @record_id)
+          create_refresh_task
+        end
+        value = @record&.id
+      end
+      value
+    end
+
+    # The expected UUID of the process for use in inspection.
+    # Use {#id_for_lock} if using this as a lock key.
+    # @return [String]
+    def process_id
+      @record_id
+    end
+
+    # Registers the current process around a job execution site.
+    # +register+ is expected to be called multiple times in a process, but should be advisory locked only once (in a single thread).
+    # @param with_advisory_lock [Boolean] Whether the lock strategy should us an advisory lock; the connection must be retained to support advisory locks.
+    # @yield [void] If a block is given, the process will be unregistered after the block completes.
+    # @return [void]
+    def register(with_advisory_lock: false)
+      synchronize do
+        if with_advisory_lock
+          if @record
+            if !advisory_locked? || !advisory_locked_connection?
+              @record.class.transaction do
+                @record.advisory_lock!
+                @record.update(lock_type: GoodJob::Process::LOCK_TYPE_ADVISORY)
+              end
+              @advisory_locked_connection = WeakRef.new(@record.class.connection)
+            end
+          else
+            @record = GoodJob::Process.create_record(id: @record_id, with_advisory_lock: true)
+            @advisory_locked_connection = WeakRef.new(@record.class.connection)
+            create_refresh_task
+          end
+        end
+
+        @locks += 1
+      end
+      return unless block_given?
+
+      begin
+        yield
+      ensure
+        unregister(with_advisory_lock: with_advisory_lock)
+      end
+    end
+
+    # Unregisters the current process from the database.
+    # @param with_advisory_lock [Boolean] Whether the lock strategy should unlock an advisory lock; the connection must be able to support advisory locks.
+    # @return [void]
+    def unregister(with_advisory_lock: false)
+      synchronize do
+        if @locks.zero?
+          return
+        elsif @locks == 1
+          if @record
+            if with_advisory_lock && advisory_locked? && advisory_locked_connection?
+              @record.class.transaction do
+                @record.advisory_unlock
+                @record.destroy
+              end
+              @advisory_locked_connection = nil
+            else
+              @record.destroy
+            end
+            @record = nil
+          end
+          cancel_refresh_task
+        elsif with_advisory_lock && advisory_locked? && advisory_locked_connection?
+          @record.class.transaction do
+            @record.advisory_unlock
+            @record.update(lock_type: nil)
+          end
+          @advisory_locked_connection = nil
+        end
+
+        @locks -= 1 unless @locks.zero?
+      end
+    end
+
+    # Refreshes the process record in the database.
+    # @param silent [Boolean] Whether to silence logging.
+    # @return [void]
+    def renew(silent: false)
+      GoodJob::Process.with_logger_silenced(silent: silent) do
+        @record&.refresh_if_stale(cleanup: true)
+      end
+    end
+
+    # Tests whether an active advisory lock has been taken on the record.
+    # @return [Boolean]
+    def advisory_locked?
+      @advisory_locked_connection&.weakref_alive? && @advisory_locked_connection&.active?
+    end
+
+    # @!visibility private
+    def task_observer(_time, _output, thread_error)
+      GoodJob._on_thread_error(thread_error) if thread_error && !thread_error.is_a?(Concurrent::CancelledOperationError)
+    end
+
+    private
+
+    def advisory_locked_connection?
+      @record&.class&.connection && @advisory_locked_connection&.weakref_alive? && @advisory_locked_connection.eql?(@record.class.connection)
+    end
+
+    def task_interval
+      GoodJob::Process::STALE_INTERVAL + jitter
+    end
+
+    def jitter
+      GoodJob::Process::STALE_INTERVAL * 0.1 * Kernel.rand
+    end
+
+    def create_refresh_task(delay: nil)
+      return if @refresh_task
+      return unless @executor
+
+      delay ||= task_interval
+      @refresh_task = Concurrent::ScheduledTask.new(delay.to_f, executor: @executor) do
+        Rails.application.executor.wrap do
+          synchronize do
+            next unless @locks.positive?
+
+            @refresh_task = nil
+            create_refresh_task
+            renew(silent: true)
+          end
+        end
+      end
+      @refresh_task.add_observer(self, :task_observer)
+      @refresh_task.execute
+    end
+
+    def cancel_refresh_task
+      @refresh_task&.cancel
+      @refresh_task = nil
+    end
+
+    def reset
+      synchronize { ns_reset }
+    end
+
+    def reset_on_fork
+      return if Concurrent.on_jruby?
+      return if @forktracker || ::Process.pid == @ruby_pid
+
+      @ruby_pid = ::Process.pid
+      ns_reset
+    end
+
+    def ns_reset
+      @record_id = SecureRandom.uuid
+      @record = nil
+    end
+
+    # Synchronize must always be called from within a Rails Executor; it may deadlock if the order is reversed.
+    def synchronize(&block)
+      if @mutex.owned?
+        yield
+      else
+        @mutex.synchronize(&block)
+      end
+    end
+  end
+end

data/lib/good_job/job_performer.rb

@@ -14,8 +14,9 @@ module GoodJob
     cattr_accessor :performing_active_job_ids, default: Concurrent::Set.new
 
     # @param queue_string [String] Queues to execute jobs from
-    def initialize(queue_string)
+    def initialize(queue_string, capsule: GoodJob.capsule)
       @queue_string = queue_string
+      @capsule = capsule
       @metrics = Metrics.new
     end
 
@@ -30,20 +31,22 @@ module GoodJob
     # @return [Object, nil] Returns job result or +nil+ if no job was found
     def next
       active_job_id = nil
-      job_query.perform_with_advisory_lock(parsed_queues: parsed_queues, queue_select_limit: GoodJob.configuration.queue_select_limit) do |execution|
-        @metrics.touch_check_queue_at
-
-        if execution
-          active_job_id = execution.active_job_id
-          performing_active_job_ids << active_job_id
-          @metrics.touch_execution_at
-          yield(execution) if block_given?
-        else
-          @metrics.increment_empty_executions
-        end
-      end.tap do |result|
-        if result
-          result.succeeded? ? @metrics.increment_succeeded_executions : @metrics.increment_errored_executions
+      @capsule.tracker.register do
+        job_query.perform_with_advisory_lock(lock_id: @capsule.tracker.id_for_lock, parsed_queues: parsed_queues, queue_select_limit: GoodJob.configuration.queue_select_limit) do |execution|
+          @metrics.touch_check_queue_at
+
+          if execution
+            active_job_id = execution.active_job_id
+            performing_active_job_ids << active_job_id
+            @metrics.touch_execution_at
+            yield(execution) if block_given?
+          else
+            @metrics.increment_empty_executions
+          end
+        end.tap do |result|
+          if result
+            result.succeeded? ? @metrics.increment_succeeded_executions : @metrics.increment_errored_executions
+          end
         end
       end
     ensure

data/lib/good_job/multi_scheduler.rb

@@ -7,12 +7,12 @@ module GoodJob
     # @param configuration [GoodJob::Configuration]
     # @param warm_cache_on_initialize [Boolean]
     # @return [GoodJob::MultiScheduler]
-    def self.from_configuration(configuration, warm_cache_on_initialize: false)
-      schedulers = configuration.queue_string.split(';').map do |queue_string_and_max_threads|
-        queue_string, max_threads = queue_string_and_max_threads.split(':')
+    def self.from_configuration(configuration, capsule: GoodJob.capsule, warm_cache_on_initialize: false)
+      schedulers = configuration.queue_string.split(';').map(&:strip).map do |queue_string_and_max_threads|
+        queue_string, max_threads = queue_string_and_max_threads.split(':').map { |str| str.strip.presence }
         max_threads = (max_threads || configuration.max_threads).to_i
 
-        job_performer = GoodJob::JobPerformer.new(queue_string)
+        job_performer = GoodJob::JobPerformer.new(queue_string, capsule: capsule)
         GoodJob::Scheduler.new(
           job_performer,
           max_threads: max_threads,

data/lib/good_job/notifier/process_heartbeat.rb

@@ -16,7 +16,7 @@ module GoodJob # :nodoc:
       def register_process
         GoodJob::Process.override_connection(connection) do
           GoodJob::Process.cleanup
-          @process = GoodJob::Process.register
+          @capsule.tracker.register(with_advisory_lock: true)
         end
       end
 
@@ -24,7 +24,7 @@ module GoodJob # :nodoc:
         Rails.application.executor.wrap do
           GoodJob::Process.override_connection(connection) do
             GoodJob::Process.with_logger_silenced do
-              @process&.refresh_if_stale(cleanup: true)
+              @capsule.tracker.renew
             end
           end
         end
@@ -33,7 +33,7 @@ module GoodJob # :nodoc:
       # Deregisters the current process.
       def deregister_process
         GoodJob::Process.override_connection(connection) do
-          @process&.deregister
+          @capsule.tracker.unregister(with_advisory_lock: true)
         end
       end
     end

data/lib/good_job/notifier.rb

@@ -2,6 +2,7 @@
 
 require 'active_support/core_ext/module/attribute_accessors_per_thread'
 require 'concurrent/atomic/atomic_boolean'
+require "concurrent/scheduled_task"
 require "good_job/notifier/process_heartbeat"
 
 module GoodJob # :nodoc:
@@ -62,13 +63,12 @@ module GoodJob # :nodoc:
 
     # @param recipients [Array<#call, Array(Object, Symbol)>]
     # @param enable_listening [true, false]
-    # @param executor [Concurrent::ExecutorService]
-    def initialize(*recipients, enable_listening: true, executor: Concurrent.global_io_executor)
+    def initialize(*recipients, enable_listening: true, capsule: GoodJob.capsule, executor: Concurrent.global_io_executor)
       @recipients = Concurrent::Array.new(recipients)
       @enable_listening = enable_listening
       @executor = executor
 
-      @mutex = Mutex.new
+      @monitor = Monitor.new
       @shutdown_event = Concurrent::Event.new.tap(&:set)
       @running = Concurrent::AtomicBoolean.new(false)
       @connected = Concurrent::Event.new
@@ -77,6 +77,7 @@ module GoodJob # :nodoc:
       @connection_errors_reported = Concurrent::AtomicBoolean.new(false)
       @enable_listening = enable_listening
       @task = nil
+      @capsule = capsule
 
       start
       self.class.instances << self
@@ -183,6 +184,8 @@ module GoodJob # :nodoc:
 
     private
 
+    delegate :synchronize, to: :@monitor
+
     def start
       synchronize do
         return if @running.true?
@@ -211,20 +214,20 @@ module GoodJob # :nodoc:
             end
 
             while thr_executor.running? && thr_running.true?
-              run_callbacks :tick do
-                wait_for_notify do |channel, payload|
-                  next unless channel == CHANNEL
-
-                  ActiveSupport::Notifications.instrument("notifier_notified.good_job", { payload: payload })
-                  parsed_payload = JSON.parse(payload, symbolize_names: true)
-                  thr_recipients.each do |recipient|
-                    target, method_name = recipient.is_a?(Array) ? recipient : [recipient, :call]
-                    target.send(method_name, parsed_payload)
-                  end
-                end
+              Rails.application.executor.wrap { run_callbacks(:tick) }
 
-                reset_connection_errors
+              wait_for_notify do |channel, payload|
+                next unless channel == CHANNEL
+
+                ActiveSupport::Notifications.instrument("notifier_notified.good_job", { payload: payload })
+                parsed_payload = JSON.parse(payload, symbolize_names: true)
+                thr_recipients.each do |recipient|
+                  target, method_name = recipient.is_a?(Array) ? recipient : [recipient, :call]
+                  target.send(method_name, parsed_payload)
+                end
               end
+
+              reset_connection_errors
             end
           end
         ensure
@@ -284,13 +287,5 @@ module GoodJob # :nodoc:
       @connection_errors_count.value = 0
       @connection_errors_reported.make_false
     end
-
-    def synchronize(&block)
-      if @mutex.owned?
-        yield
-      else
-        @mutex.synchronize(&block)
-      end
-    end
   end
 end

data/lib/good_job/shared_executor.rb

@@ -1,7 +1,12 @@
 # frozen_string_literal: true
 
+require 'concurrent/executor/executor_service'
+
 module GoodJob
   class SharedExecutor
+    # Allow posting tasks directly to instances
+    include Concurrent::ExecutorService
+
     MAX_THREADS = 2
 
     # @!attribute [r] instances
@@ -13,8 +18,21 @@ module GoodJob
     attr_reader :executor
 
     def initialize
+      @mutex = Mutex.new
+
       self.class.instances << self
-      create_executor
+    end
+
+    def post(*args, &task)
+      unless running?
+        @mutex.synchronize do
+          next if running?
+
+          create_executor
+        end
+      end
+
+      @executor&.post(*args, &task)
     end
 
     def running?

data/lib/good_job/version.rb

@@ -2,7 +2,7 @@
 
 module GoodJob
   # GoodJob gem version.
-  VERSION = '3.28.2'
+  VERSION = '3.29.0'
 
   # GoodJob version as Gem::Version object
   GEM_VERSION = Gem::Version.new(VERSION)

data/lib/good_job.rb

@@ -19,6 +19,7 @@ require_relative "good_job/overridable_connection"
 require_relative "good_job/bulk"
 require_relative "good_job/callable"
 require_relative "good_job/capsule"
+require_relative "good_job/capsule_tracker"
 require_relative "good_job/cleanup_tracker"
 require_relative "good_job/cli"
 require_relative "good_job/configuration"
@@ -169,11 +170,12 @@ module GoodJob
   end
 
   # Sends +#shutdown+ or +#restart+ to executable objects ({GoodJob::Notifier}, {GoodJob::Poller}, {GoodJob::Scheduler}, {GoodJob::MultiScheduler}, {GoodJob::CronManager})
-  # @param executables [Array<Notifier, Poller, Scheduler, MultiScheduler, CronManager>] Objects to shut down.
+  # @param executables [Array<Notifier, Poller, Scheduler, MultiScheduler, CronManager, SharedExecutor>] Objects to shut down.
   # @param method_name [:symbol] Method to call, e.g. +:shutdown+ or +:restart+.
   # @param timeout [nil,Numeric]
+  # @param after [Array<Notifier, Poller, Scheduler, MultiScheduler, CronManager, SharedExecutor>] Objects to shut down after initial executables shut down.
   # @return [void]
-  def self._shutdown_all(executables, method_name = :shutdown, timeout: -1)
+  def self._shutdown_all(executables, method_name = :shutdown, timeout: -1, after: [])
     if timeout.is_a?(Numeric) && timeout.positive?
       executables.each { |executable| executable.send(method_name, timeout: nil) }
 
@@ -182,6 +184,13 @@ module GoodJob
     else
       executables.each { |executable| executable.send(method_name, timeout: timeout) }
     end
+    return unless after.any? && !timeout.nil?
+
+    if stop_at
+      after.each { |executable| executable.shutdown(timeout: [stop_at - Time.current, 0].max) }
+    else
+      after.each { |executable| executable.shutdown(timeout: timeout) }
+    end
   end
 
   # Destroys preserved job and batch records.
@@ -278,7 +287,7 @@ module GoodJob
   def self.migrated?
     # Always update with the most recent migration check
     GoodJob::Execution.reset_column_information
-    GoodJob::Execution.candidate_lookup_index_migrated?
+    GoodJob::Execution.process_lock_migrated?
   end
 
   ActiveSupport.run_load_hooks(:good_job, self)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: good_job
 version: !ruby/object:Gem::Version
-  version: 3.28.2
+  version: 3.29.0
 platform: ruby
 authors:
 - Ben Sheldon
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-04-26 00:00:00.000000000 Z
+date: 2024-05-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activejob
@@ -357,9 +357,11 @@ files:
 - lib/generators/good_job/templates/update/migrations/07_recreate_good_job_cron_indexes_with_conditional.rb.erb
 - lib/generators/good_job/templates/update/migrations/08_create_good_job_labels.rb.erb
 - lib/generators/good_job/templates/update/migrations/09_create_good_job_labels_index.rb.erb
-- lib/generators/good_job/templates/update/migrations/10_create_good_job_execution_error_backtrace.rb.erb
 - lib/generators/good_job/templates/update/migrations/10_remove_good_job_active_id_index.rb.erb
 - lib/generators/good_job/templates/update/migrations/11_create_index_good_job_jobs_for_candidate_lookup.rb.erb
+- lib/generators/good_job/templates/update/migrations/12_create_good_job_execution_error_backtrace.rb.erb
+- lib/generators/good_job/templates/update/migrations/13_create_good_job_process_lock_ids.rb.erb
+- lib/generators/good_job/templates/update/migrations/14_create_good_job_process_lock_indexes.rb.erb
 - lib/generators/good_job/update_generator.rb
 - lib/good_job.rb
 - lib/good_job/active_job_extensions/batches.rb
@@ -371,6 +373,7 @@ files:
 - lib/good_job/bulk.rb
 - lib/good_job/callable.rb
 - lib/good_job/capsule.rb
+- lib/good_job/capsule_tracker.rb
 - lib/good_job/cleanup_tracker.rb
 - lib/good_job/cli.rb
 - lib/good_job/configuration.rb