good_job 2.1.0 → 2.4.0

data/lib/good_job/active_job_extensions/concurrency.rb

@@ -18,7 +18,7 @@ module GoodJob
           next(block.call) unless job.class.queue_adapter.is_a?(GoodJob::Adapter)
 
           # Always allow jobs to be retried because the current job's execution will complete momentarily
-          next(block.call) if CurrentExecution.active_job_id == job.job_id
+          next(block.call) if CurrentThread.active_job_id == job.job_id
 
           enqueue_limit = job.class.good_job_concurrency_config[:enqueue_limit]
           total_limit = job.class.good_job_concurrency_config[:total_limit]
@@ -30,12 +30,12 @@ module GoodJob
           key = job.good_job_concurrency_key
           next(block.call) if key.blank?
 
-          GoodJob::Job.new.with_advisory_lock(key: key, function: "pg_advisory_lock") do
+          GoodJob::Execution.new.with_advisory_lock(key: key, function: "pg_advisory_lock") do
             enqueue_concurrency = if enqueue_limit
                                     # TODO: Why is `unscoped` necessary? Nested scope is bleeding into subsequent query?
-                                    GoodJob::Job.unscoped.where(concurrency_key: key).unfinished.advisory_unlocked.count
+                                    GoodJob::Execution.unscoped.where(concurrency_key: key).unfinished.advisory_unlocked.count
                                   else
-                                    GoodJob::Job.unscoped.where(concurrency_key: key).unfinished.count
+                                    GoodJob::Execution.unscoped.where(concurrency_key: key).unfinished.count
                                   end
 
             # The job has not yet been enqueued, so check if adding it will go over the limit
@@ -62,8 +62,8 @@ module GoodJob
           key = job.good_job_concurrency_key
           next if key.blank?
 
-          GoodJob::Job.new.with_advisory_lock(key: key, function: "pg_advisory_lock") do
-            allowed_active_job_ids = GoodJob::Job.unscoped.where(concurrency_key: key).advisory_locked.order(Arel.sql("COALESCE(performed_at, scheduled_at, created_at) ASC")).limit(perform_limit).pluck(:active_job_id)
+          GoodJob::Execution.new.with_advisory_lock(key: key, function: "pg_advisory_lock") do
+            allowed_active_job_ids = GoodJob::Execution.unscoped.where(concurrency_key: key).advisory_locked.order(Arel.sql("COALESCE(performed_at, scheduled_at, created_at) ASC")).limit(perform_limit).pluck(:active_job_id)
             # The current job has already been locked and will appear in the previous query
             raise GoodJob::ActiveJobExtensions::Concurrency::ConcurrencyExceededError unless allowed_active_job_ids.include? job.job_id
           end

data/lib/good_job/adapter.rb

@@ -45,7 +45,7 @@ module GoodJob
     # Enqueues the ActiveJob job to be performed.
     # For use by Rails; you should generally not call this directly.
     # @param active_job [ActiveJob::Base] the job to be enqueued from +#perform_later+
-    # @return [GoodJob::Job]
+    # @return [GoodJob::Execution]
     def enqueue(active_job)
       enqueue_at(active_job, nil)
     end
@@ -54,9 +54,9 @@ module GoodJob
     # For use by Rails; you should generally not call this directly.
     # @param active_job [ActiveJob::Base] the job to be enqueued from +#perform_later+
     # @param timestamp [Integer, nil] the epoch time to perform the job
-    # @return [GoodJob::Job]
+    # @return [GoodJob::Execution]
     def enqueue_at(active_job, timestamp)
-      good_job = GoodJob::Job.enqueue(
+      execution = GoodJob::Execution.enqueue(
         active_job,
         scheduled_at: timestamp ? Time.zone.at(timestamp) : nil,
         create_with_advisory_lock: execute_inline?
@@ -64,19 +64,19 @@ module GoodJob
 
       if execute_inline?
         begin
-          good_job.perform
+          execution.perform
         ensure
-          good_job.advisory_unlock
+          execution.advisory_unlock
         end
       else
-        job_state = { queue_name: good_job.queue_name }
-        job_state[:scheduled_at] = good_job.scheduled_at if good_job.scheduled_at
+        job_state = { queue_name: execution.queue_name }
+        job_state[:scheduled_at] = execution.scheduled_at if execution.scheduled_at
 
         executed_locally = execute_async? && @scheduler.create_thread(job_state)
         Notifier.notify(job_state) unless executed_locally
       end
 
-      good_job
+      execution
     end
 
     # Shut down the thread pool executors.
@@ -101,14 +101,14 @@ module GoodJob
     # @return [Boolean]
     def execute_async?
       @configuration.execution_mode == :async_all ||
-        @configuration.execution_mode.in?([:async, :async_server]) && in_server_process?
+        (@configuration.execution_mode.in?([:async, :async_server]) && in_server_process?)
     end
 
     # Whether in +:external+ execution mode.
     # @return [Boolean]
     def execute_externally?
       @configuration.execution_mode == :external ||
-        @configuration.execution_mode.in?([:async, :async_server]) && !in_server_process?
+        (@configuration.execution_mode.in?([:async, :async_server]) && !in_server_process?)
     end
 
     # Whether in +:inline+ execution mode.

data/lib/good_job/cron_manager.rb

@@ -24,7 +24,7 @@ module GoodJob # :nodoc:
       GoodJob.on_thread_error.call(thread_error) if thread_error && GoodJob.on_thread_error.respond_to?(:call)
     end
 
-    # Job configuration to be scheduled
+    # Execution configuration to be scheduled
     # @return [Hash]
     attr_reader :schedules
 
@@ -90,8 +90,8 @@ module GoodJob # :nodoc:
         # Re-schedule the next cron task before executing the current task
         thr_scheduler.create_task(thr_cron_key)
 
-        CurrentExecution.reset
-        CurrentExecution.cron_key = thr_cron_key
+        CurrentThread.reset
+        CurrentThread.cron_key = thr_cron_key
 
         Rails.application.executor.wrap do
           schedule = thr_scheduler.schedules.fetch(thr_cron_key).with_indifferent_access

data/lib/good_job/{current_execution.rb → current_thread.rb}

@@ -4,7 +4,7 @@ require 'active_support/core_ext/module/attribute_accessors_per_thread'
 module GoodJob
   # Thread-local attributes for passing values from Instrumentation.
   # (Cannot use ActiveSupport::CurrentAttributes because ActiveJob resets it)
-  module CurrentExecution
+  module CurrentThread
     # @!attribute [rw] cron_key
     #   @!scope class
     #   Cron Key
@@ -23,24 +23,24 @@ module GoodJob
     #   @return [Exception, nil]
     thread_mattr_accessor :error_on_retry
 
-    # @!attribute [rw] good_job
+    # @!attribute [rw] executions
     #   @!scope class
-    #   Cron Key
-    #   @return [GoodJob::Job, nil]
-    thread_mattr_accessor :good_job
+    #   Execution
+    #   @return [GoodJob::Execution, nil]
+    thread_mattr_accessor :execution
 
     # Resets attributes
     # @return [void]
     def self.reset
       self.cron_key = nil
-      self.good_job = nil
+      self.execution = nil
       self.error_on_discard = nil
       self.error_on_retry = nil
     end
 
-    # @return [String] UUID of the currently executing GoodJob::Job
+    # @return [String] UUID of the currently executing GoodJob::Execution
     def self.active_job_id
-      good_job&.active_job_id
+      execution&.active_job_id
     end
 
     # @return [Integer] Current process ID

data/lib/good_job/execution.rb

@@ -0,0 +1,308 @@
+# frozen_string_literal: true
+module GoodJob
+  # ActiveRecord model that represents an +ActiveJob+ job.
+  # Parent class can be configured with +GoodJob.active_record_parent_class+.
+  # @!parse
+  #   class Execution < ActiveRecord::Base; end
+  class Execution < Object.const_get(GoodJob.active_record_parent_class)
+    include Lockable
+
+    # Raised if something attempts to execute a previously completed Execution again.
+    PreviouslyPerformedError = Class.new(StandardError)
+
+    # ActiveJob jobs without a +queue_name+ attribute are placed on this queue.
+    DEFAULT_QUEUE_NAME = 'default'
+    # ActiveJob jobs without a +priority+ attribute are given this priority.
+    DEFAULT_PRIORITY = 0
+
+    self.table_name = 'good_jobs'
+    self.advisory_lockable_column = 'active_job_id'
+
+    # Parse a string representing a group of queues into a more readable data
+    # structure.
+    # @param string [String] Queue string
+    # @return [Hash]
+    #   How to match a given queue. It can have the following keys and values:
+    #   - +{ all: true }+ indicates that all queues match.
+    #   - +{ exclude: Array<String> }+ indicates the listed queue names should
+    #     not match.
+    #   - +{ include: Array<String> }+ indicates the listed queue names should
+    #     match.
+    # @example
+    #   GoodJob::Execution.queue_parser('-queue1,queue2')
+    #   => { exclude: [ 'queue1', 'queue2' ] }
+    def self.queue_parser(string)
+      string = string.presence || '*'
+
+      if string.first == '-'
+        exclude_queues = true
+        string = string[1..-1]
+      end
+
+      queues = string.split(',').map(&:strip)
+
+      if queues.include?('*')
+        { all: true }
+      elsif exclude_queues
+        { exclude: queues }
+      else
+        { include: queues }
+      end
+    end
+
+    # Get Jobs with given ActiveJob ID
+    # @!method active_job_id
+    # @!scope class
+    # @param active_job_id [String]
+    #   ActiveJob ID
+    # @return [ActiveRecord::Relation]
+    scope :active_job_id, ->(active_job_id) { where(active_job_id: active_job_id) }
+
+    # Get Jobs with given class name
+    # @!method job_class
+    # @!scope class
+    # @param string [String]
+    #   Execution class name
+    # @return [ActiveRecord::Relation]
+    scope :job_class, ->(job_class) { where("serialized_params->>'job_class' = ?", job_class) }
+
+    # Get Jobs that have not yet been completed.
+    # @!method unfinished
+    # @!scope class
+    # @return [ActiveRecord::Relation]
+    scope :unfinished, -> { where(finished_at: nil) }
+
+    # Get Jobs that are not scheduled for a later time than now (i.e. jobs that
+    # are not scheduled or scheduled for earlier than the current time).
+    # @!method only_scheduled
+    # @!scope class
+    # @return [ActiveRecord::Relation]
+    scope :only_scheduled, -> { where(arel_table['scheduled_at'].lteq(Time.current)).or(where(scheduled_at: nil)) }
+
+    # Order jobs by priority (highest priority first).
+    # @!method priority_ordered
+    # @!scope class
+    # @return [ActiveRecord::Relation]
+    scope :priority_ordered, -> { order('priority DESC NULLS LAST') }
+
+    # Order jobs by scheduled (unscheduled or soonest first).
+    # @!method schedule_ordered
+    # @!scope class
+    # @return [ActiveRecord::Relation]
+    scope :schedule_ordered, -> { order(Arel.sql('COALESCE(scheduled_at, created_at) ASC')) }
+
+    # Get Jobs were completed before the given timestamp. If no timestamp is
+    # provided, get all jobs that have been completed. By default, GoodJob
+    # deletes jobs after they are completed and this will find no jobs.
+    # However, if you have changed {GoodJob.preserve_job_records}, this may
+    # find completed Jobs.
+    # @!method finished(timestamp = nil)
+    # @!scope class
+    # @param timestamp (Float)
+    #   Get jobs that finished before this time (in epoch time).
+    # @return [ActiveRecord::Relation]
+    scope :finished, ->(timestamp = nil) { timestamp ? where(arel_table['finished_at'].lteq(timestamp)) : where.not(finished_at: nil) }
+
+    # Get Jobs that started but not finished yet.
+    # @!method running
+    # @!scope class
+    # @return [ActiveRecord::Relation]
+    scope :running, -> { where.not(performed_at: nil).where(finished_at: nil) }
+
+    # Get Jobs that do not have subsequent retries
+    # @!method running
+    # @!scope class
+    # @return [ActiveRecord::Relation]
+    scope :head, -> { where(retried_good_job_id: nil) }
+
+    # Get Jobs have errored that will not be retried further
+    # @!method running
+    # @!scope class
+    # @return [ActiveRecord::Relation]
+    scope :dead, -> { head.where.not(error: nil) }
+
+    # Get Jobs on queues that match the given queue string.
+    # @!method queue_string(string)
+    # @!scope class
+    # @param string [String]
+    #   A string expression describing what queues to select. See
+    #   {Execution.queue_parser} or
+    #   {file:README.md#optimize-queues-threads-and-processes} for more details
+    #   on the format of the string. Note this only handles individual
+    #   semicolon-separated segments of that string format.
+    # @return [ActiveRecord::Relation]
+    scope :queue_string, (lambda do |string|
+      parsed = queue_parser(string)
+
+      if parsed[:all]
+        all
+      elsif parsed[:exclude]
+        where.not(queue_name: parsed[:exclude]).or where(queue_name: nil)
+      elsif parsed[:include]
+        where(queue_name: parsed[:include])
+      end
+    end)
+
+    # Get Jobs in display order with optional keyset pagination.
+    # @!method display_all(after_scheduled_at: nil, after_id: nil)
+    # @!scope class
+    # @param after_scheduled_at [DateTime, String, nil]
+    #   Display records scheduled after this time for keyset pagination
+    # @param after_id [Numeric, String, nil]
+    #   Display records after this ID for keyset pagination
+    # @return [ActiveRecord::Relation]
+    scope :display_all, (lambda do |after_scheduled_at: nil, after_id: nil|
+      query = order(Arel.sql('COALESCE(scheduled_at, created_at) DESC, id DESC'))
+      if after_scheduled_at.present? && after_id.present?
+        query = query.where(Arel.sql('(COALESCE(scheduled_at, created_at), id) < (:after_scheduled_at, :after_id)'), after_scheduled_at: after_scheduled_at, after_id: after_id)
+      elsif after_scheduled_at.present?
+        query = query.where(Arel.sql('(COALESCE(scheduled_at, created_at)) < (:after_scheduled_at)'), after_scheduled_at: after_scheduled_at)
+      end
+      query
+    end)
+
+    # Finds the next eligible Execution, acquire an advisory lock related to it, and
+    # executes the job.
+    # @return [ExecutionResult, nil]
+    #   If a job was executed, returns an array with the {Execution} record, the
+    #   return value for the job's +#perform+ method, and the exception the job
+    #   raised, if any (if the job raised, then the second array entry will be
+    #   +nil+). If there were no jobs to execute, returns +nil+.
+    def self.perform_with_advisory_lock
+      unfinished.priority_ordered.only_scheduled.limit(1).with_advisory_lock(unlock_session: true) do |executions|
+        execution = executions.first
+        break if execution.blank?
+        break :unlocked unless execution&.executable?
+
+        begin
+          execution.with_advisory_lock(key: "good_jobs-#{execution.active_job_id}") do
+            execution.perform
+          end
+        rescue RecordAlreadyAdvisoryLockedError => e
+          ExecutionResult.new(value: nil, handled_error: e)
+        end
+      end
+    end
+
+    # Fetches the scheduled execution time of the next eligible Execution(s).
+    # @param after [DateTime]
+    # @param limit [Integer]
+    # @param now_limit [Integer, nil]
+    # @return [Array<DateTime>]
+    def self.next_scheduled_at(after: nil, limit: 100, now_limit: nil)
+      query = advisory_unlocked.unfinished.schedule_ordered
+
+      after ||= Time.current
+      after_query = query.where('scheduled_at > ?', after).or query.where(scheduled_at: nil).where('created_at > ?', after)
+      after_at = after_query.limit(limit).pluck(:scheduled_at, :created_at).map { |timestamps| timestamps.compact.first }
+
+      if now_limit&.positive?
+        now_query = query.where('scheduled_at < ?', Time.current).or query.where(scheduled_at: nil)
+        now_at = now_query.limit(now_limit).pluck(:scheduled_at, :created_at).map { |timestamps| timestamps.compact.first }
+      end
+
+      Array(now_at) + after_at
+    end
+
+    # Places an ActiveJob job on a queue by creating a new {Execution} record.
+    # @param active_job [ActiveJob::Base]
+    #   The job to enqueue.
+    # @param scheduled_at [Float]
+    #   Epoch timestamp when the job should be executed.
+    # @param create_with_advisory_lock [Boolean]
+    #   Whether to establish a lock on the {Execution} record after it is created.
+    # @return [Execution]
+    #   The new {Execution} instance representing the queued ActiveJob job.
+    def self.enqueue(active_job, scheduled_at: nil, create_with_advisory_lock: false)
+      ActiveSupport::Notifications.instrument("enqueue_job.good_job", { active_job: active_job, scheduled_at: scheduled_at, create_with_advisory_lock: create_with_advisory_lock }) do |instrument_payload|
+        execution_args = {
+          active_job_id: active_job.job_id,
+          queue_name: active_job.queue_name.presence || DEFAULT_QUEUE_NAME,
+          priority: active_job.priority || DEFAULT_PRIORITY,
+          serialized_params: active_job.serialize,
+          scheduled_at: scheduled_at,
+          create_with_advisory_lock: create_with_advisory_lock,
+        }
+
+        execution_args[:concurrency_key] = active_job.good_job_concurrency_key if active_job.respond_to?(:good_job_concurrency_key)
+
+        if CurrentThread.cron_key
+          execution_args[:cron_key] = CurrentThread.cron_key
+        elsif CurrentThread.active_job_id == active_job.job_id
+          execution_args[:cron_key] = CurrentThread.execution.cron_key
+        end
+
+        execution = GoodJob::Execution.new(**execution_args)
+
+        instrument_payload[:execution] = execution
+
+        execution.save!
+        active_job.provider_job_id = execution.id
+
+        CurrentThread.execution.retried_good_job_id = execution.id if CurrentThread.execution && CurrentThread.execution.active_job_id == active_job.job_id
+
+        execution
+      end
+    end
+
+    # Execute the ActiveJob job this {Execution} represents.
+    # @return [ExecutionResult]
+    #   An array of the return value of the job's +#perform+ method and the
+    #   exception raised by the job, if any. If the job completed successfully,
+    #   the second array entry (the exception) will be +nil+ and vice versa.
+    def perform
+      raise PreviouslyPerformedError, 'Cannot perform a job that has already been performed' if finished_at
+
+      self.performed_at = Time.current
+      save! if GoodJob.preserve_job_records
+
+      result = execute
+
+      job_error = result.handled_error || result.unhandled_error
+      self.error = "#{job_error.class}: #{job_error.message}" if job_error
+
+      if result.unhandled_error && GoodJob.retry_on_unhandled_error
+        save!
+      elsif GoodJob.preserve_job_records == true || (result.unhandled_error && GoodJob.preserve_job_records == :on_unhandled_error)
+        self.finished_at = Time.current
+        save!
+      else
+        destroy!
+      end
+
+      result
+    end
+
+    # Tests whether this job is safe to be executed by this thread.
+    # @return [Boolean]
+    def executable?
+      self.class.unscoped.unfinished.owns_advisory_locked.exists?(id: id)
+    end
+
+    private
+
+    # @return [ExecutionResult]
+    def execute
+      GoodJob::CurrentThread.reset
+      GoodJob::CurrentThread.execution = self
+
+      job_data = serialized_params.deep_dup
+      job_data["provider_job_id"] = id
+
+      # DEPRECATION: Remove deprecated `good_job:` parameter in GoodJob v3
+      ActiveSupport::Notifications.instrument("perform_job.good_job", { good_job: self, execution: self, process_id: GoodJob::CurrentThread.process_id, thread_name: GoodJob::CurrentThread.thread_name }) do
+        value = ActiveJob::Base.execute(job_data)
+
+        if value.is_a?(Exception)
+          handled_error = value
+          value = nil
+        end
+        handled_error ||= GoodJob::CurrentThread.error_on_retry || GoodJob::CurrentThread.error_on_discard
+
+        ExecutionResult.new(value: value, handled_error: handled_error)
+      rescue StandardError => e
+        ExecutionResult.new(value: nil, unhandled_error: e)
+      end
+    end
+  end
+end