activejob 8.0.2 → 8.1.0.beta1

data/lib/active_job/continuation.rb

@@ -0,0 +1,332 @@
+# frozen_string_literal: true
+
+require "active_support/core_ext/numeric/time"
+require "active_job/continuable"
+
+module ActiveJob
+  # = Active Job \Continuation
+  #
+  # Continuations provide a mechanism for interrupting and resuming jobs. This allows
+  # long-running jobs to make progress across application restarts.
+  #
+  # Jobs should include the ActiveJob::Continuable module to enable continuations.
+  # \Continuable jobs are automatically retried when interrupted.
+  #
+  # Use the +step+ method to define the steps in your job. Steps can use an optional
+  # cursor to track progress in the step.
+  #
+  # Steps are executed as soon as they are encountered. If a job is interrupted, previously
+  # completed steps will be skipped. If a step is in progress, it will be resumed
+  # with the last recorded cursor.
+  #
+  # Code that is not part of a step will be executed on each job run.
+  #
+  # You can pass a block or a method name to the step method. The block will be called with
+  # the step object as an argument. Methods can either take no arguments or a single argument
+  # for the step object.
+  #
+  #   class ProcessImportJob < ApplicationJob
+  #     include ActiveJob::Continuable
+  #
+  #     def perform(import_id)
+  #       # This always runs, even if the job is resumed.
+  #       @import = Import.find(import_id)
+  #
+  #       step :validate do
+  #         @import.validate!
+  #       end
+  #
+  #       step(:process_records) do |step|
+  #         @import.records.find_each(start: step.cursor) do |record|
+  #           record.process
+  #           step.advance! from: record.id
+  #         end
+  #       end
+  #
+  #       step :reprocess_records
+  #       step :finalize
+  #     end
+  #
+  #     def reprocess_records(step)
+  #       @import.records.find_each(start: step.cursor) do |record|
+  #         record.reprocess
+  #         step.advance! from: record.id
+  #       end
+  #     end
+  #
+  #     def finalize
+  #       @import.finalize!
+  #     end
+  #   end
+  #
+  # === Cursors
+  #
+  # Cursors are used to track progress within a step. The cursor can be any object that is
+  # serializable as an argument to +ActiveJob::Base.serialize+. It defaults to +nil+.
+  #
+  # When a step is resumed, the last cursor value is restored. The code in the step is responsible
+  # for using the cursor to continue from the right point.
+  #
+  # +set!+ sets the cursor to a specific value.
+  #
+  #   step :iterate_items do |step|
+  #     items[step.cursor..].each do |item|
+  #       process(item)
+  #       step.set! (step.cursor || 0) + 1
+  #     end
+  #   end
+  #
+  # An starting value for the cursor can be set when defining the step:
+  #
+  #   step :iterate_items, start: 0 do |step|
+  #     items[step.cursor..].each do |item|
+  #       process(item)
+  #       step.set! step.cursor + 1
+  #     end
+  #   end
+  #
+  # The cursor can be advanced with +advance!+. This calls +succ+ on the current cursor value.
+  # It raises an ActiveJob::Continuation::UnadvanceableCursorError if the cursor does not implement +succ+.
+  #
+  #   step :iterate_items, start: 0 do |step|
+  #     items[step.cursor..].each do |item|
+  #       process(item)
+  #       step.advance!
+  #     end
+  #   end
+  #
+  # You can optionally pass a +from+ argument to +advance!+. This is useful when iterating
+  # over a collection of records where IDs may not be contiguous.
+  #
+  #   step :process_records do |step|
+  #     import.records.find_each(start: step.cursor) do |record|
+  #       record.process
+  #       step.advance! from: record.id
+  #     end
+  #   end
+  #
+  # You can use an array to iterate over nested records:
+  #
+  #   step :process_nested_records, start: [ 0, 0 ] do |step|
+  #     Account.find_each(start: step.cursor[0]) do |account|
+  #       account.records.find_each(start: step.cursor[1]) do |record|
+  #         record.process
+  #         step.set! [ account.id, record.id + 1 ]
+  #       end
+  #       step.set! [ account.id + 1, 0 ]
+  #     end
+  #   end
+  #
+  # Setting or advancing the cursor creates a checkpoint. You can also create a checkpoint
+  # manually by calling the +checkpoint!+ method on the step. This is useful if you want to
+  # allow interruptions, but don't need to update the cursor.
+  #
+  #   step :destroy_records do |step|
+  #     import.records.find_each do |record|
+  #       record.destroy!
+  #       step.checkpoint!
+  #     end
+  #   end
+  #
+  # === Checkpoints
+  #
+  # A checkpoint is where a job can be interrupted. At a checkpoint the job will call
+  # +queue_adapter.stopping?+. If it returns true, the job will raise an
+  # ActiveJob::Continuation::Interrupt exception.
+  #
+  # There is an automatic checkpoint before the start of each step except for the first for
+  # each job execution. Within a step one is created when calling +set!+, +advance!+ or +checkpoint!+.
+  #
+  # Jobs are not automatically interrupted when the queue adapter is marked as stopping - they
+  # will continue to run either until the next checkpoint, or when the process is stopped.
+  #
+  # This is to allow jobs to be interrupted at a safe point, but it also means that the jobs
+  # should checkpoint more frequently than the shutdown timeout to ensure a graceful restart.
+  #
+  # When interrupted, the job will automatically retry with the progress serialized
+  # in the job data under the +continuation+ key.
+  #
+  # The serialized progress contains:
+  # - a list of the completed steps
+  # - the current step and its cursor value (if one is in progress)
+  #
+  # === Isolated Steps
+  #
+  # Steps run sequentially in a single job execution, unless the job is interrupted.
+  #
+  # You can specify that a step should always run in its own execution by passing the +isolated: true+ option.
+  #
+  # This is useful for long-running steps where it may not be possible to checkpoint within
+  # the job grace period - it ensures that progress is serialized back into the job data before
+  # the step starts.
+  #
+  #   step :quick_step1
+  #   step :slow_step, isolated: true
+  #   step :quick_step2
+  #   step :quick_step3
+  #
+  # === Errors
+  #
+  # If a job raises an error and is not retried via Active Job, it will be passed back to the underlying
+  # queue backend and any progress in this execution will be lost.
+  #
+  # To mitigate this, the job will be automatically retried if it raises an error after it has made progress.
+  # Making progress is defined as having completed a step or advanced the cursor within the current step.
+  #
+  # === Configuration
+  #
+  # Continuable jobs have several configuration options:
+  # * <tt>:max_resumptions</tt> - The maximum number of times a job can be resumed. Defaults to +nil+ which means
+  #   unlimited resumptions.
+  # * <tt>:resume_options</tt> - Options to pass to +retry_job+ when resuming the job.
+  #   Defaults to <tt>{ wait: 5.seconds }</tt>.
+  #   See {ActiveJob::Exceptions#retry_job}[rdoc-ref:ActiveJob::Exceptions#retry_job] for available options.
+  # * <tt>:resume_errors_after_advancing</tt> - Whether to resume errors after advancing the continuation.
+  #   Defaults to +true+.
+  class Continuation
+    extend ActiveSupport::Autoload
+
+    autoload :Validation
+
+    # Raised when a job is interrupted, allowing Active Job to requeue it.
+    # This inherits from +Exception+ rather than +StandardError+, so it's not
+    # caught by normal exception handling.
+    class Interrupt < Exception; end
+
+    # Base class for all Continuation errors.
+    class Error < StandardError; end
+
+    # Raised when a step is invalid.
+    class InvalidStepError < Error; end
+
+    # Raised when there is an error with a checkpoint, such as open database transactions.
+    class CheckpointError < Error; end
+
+    # Raised when attempting to advance a cursor that doesn't implement `succ`.
+    class UnadvanceableCursorError < Error; end
+
+    # Raised when a job has reached its limit of the number of resumes.
+    # The limit is defined by the +max_resumes+ class attribute.
+    class ResumeLimitError < Error; end
+
+    include Validation
+
+    def initialize(job, serialized_progress) # :nodoc:
+      @job = job
+      @completed = serialized_progress.fetch("completed", []).map(&:to_sym)
+      @current = new_step(*serialized_progress["current"], resumed: true) if serialized_progress.key?("current")
+      @encountered = []
+      @advanced = false
+      @running_step = false
+      @isolating = false
+    end
+
+    def step(name, **options, &block) # :nodoc:
+      validate_step!(name)
+      encountered << name
+
+      if completed?(name)
+        skip_step(name)
+      else
+        run_step(name, **options, &block)
+      end
+    end
+
+    def to_h # :nodoc:
+      {
+        "completed" => completed.map(&:to_s),
+        "current" => current&.to_a,
+      }.compact
+    end
+
+    def description # :nodoc:
+      if current
+        current.description
+      elsif completed.any?
+        "after '#{completed.last}'"
+      else
+        "not started"
+      end
+    end
+
+    def started?
+      completed.any? || current.present?
+    end
+
+    def advanced?
+      @advanced
+    end
+
+    def instrumentation
+      { description: description,
+        completed_steps: completed,
+        current_step: current }
+    end
+
+    private
+      attr_reader :job, :encountered, :completed, :current
+
+      def running_step?
+        @running_step
+      end
+
+      def isolating?
+        @isolating
+      end
+
+      def completed?(name)
+        completed.include?(name)
+      end
+
+      def new_step(*args, **options)
+        Step.new(*args, job: job, **options)
+      end
+
+      def skip_step(name)
+        instrument :step_skipped, step: name
+      end
+
+      def run_step(name, start:, isolated:, &block)
+        @isolating ||= isolated
+
+        if isolating? && advanced?
+          job.interrupt!(reason: :isolating)
+        else
+          run_step_inline(name, start: start, &block)
+        end
+      end
+
+      def run_step_inline(name, start:, **options, &block)
+        @running_step = true
+        @current ||= new_step(name, start, resumed: false)
+
+        instrumenting_step(current) do
+          block.call(current)
+        end
+
+        @completed << current.name
+        @current = nil
+        @advanced = true
+      ensure
+        @running_step = false
+        @advanced ||= current&.advanced?
+      end
+
+      def instrumenting_step(step, &block)
+        instrument :step, step: step, interrupted: false do |payload|
+          instrument :step_started, step: step
+
+          block.call
+        rescue Interrupt
+          payload[:interrupted] = true
+          raise
+        end
+      end
+
+      def instrument(...)
+        job.instrument(...)
+      end
+  end
+end
+
+require "active_job/continuation/step"

data/lib/active_job/core.rb

@@ -1,6 +1,13 @@
 # frozen_string_literal: true
 
 module ActiveJob
+  # Raised during job payload deserialization when it references an uninitialized job class.
+  class UnknownJobClassError < NameError
+    def initialize(job_class_name)
+      super("Failed to instantiate job, class `#{job_class_name}` doesn't exist", job_class_name)
+    end
+  end
+
   # = Active Job \Core
   #
   # Provides general behavior that will be included into every Active Job
@@ -60,7 +67,10 @@ module ActiveJob
     module ClassMethods
       # Creates a new job instance from a hash created with +serialize+
       def deserialize(job_data)
-        job = job_data["job_class"].constantize.new
+        job_class = job_data["job_class"].safe_constantize
+        raise UnknownJobClassError, job_data["job_class"] unless job_class
+
+        job = job_class.new
         job.deserialize(job_data)
         job
       end
@@ -114,7 +124,7 @@ module ActiveJob
         "arguments"  => serialize_arguments_if_needed(arguments),
         "executions" => executions,
         "exception_executions" => exception_executions,
-        "locale"     => I18n.locale.to_s,
+        "locale"     => locale || I18n.locale.to_s,
         "timezone"   => timezone,
         "enqueued_at" => Time.now.utc.iso8601(9),
         "scheduled_at" => scheduled_at ? scheduled_at.utc.iso8601(9) : nil,

data/lib/active_job/enqueue_after_transaction_commit.rb

@@ -4,32 +4,7 @@ module ActiveJob
   module EnqueueAfterTransactionCommit # :nodoc:
     private
       def raw_enqueue
-        enqueue_after_transaction_commit = self.class.enqueue_after_transaction_commit
-
-        after_transaction = case self.class.enqueue_after_transaction_commit
-        when :always
-          ActiveJob.deprecator.warn(<<~MSG.squish)
-            Setting `#{self.class.name}.enqueue_after_transaction_commit = :always` is deprecated and will be removed in Rails 8.1.
-            Set to `true` to always enqueue the job after the transaction is committed.
-          MSG
-          true
-        when :never
-          ActiveJob.deprecator.warn(<<~MSG.squish)
-            Setting `#{self.class.name}.enqueue_after_transaction_commit = :never` is deprecated and will be removed in Rails 8.1.
-            Set to `false` to never enqueue the job after the transaction is committed.
-          MSG
-          false
-        when :default
-          ActiveJob.deprecator.warn(<<~MSG.squish)
-            Setting `#{self.class.name}.enqueue_after_transaction_commit = :default` is deprecated and will be removed in Rails 8.1.
-            Set to `false` to never enqueue the job after the transaction is committed.
-          MSG
-          false
-        else
-          enqueue_after_transaction_commit
-        end
-
-        if after_transaction
+        if self.class.enqueue_after_transaction_commit
           self.successfully_enqueued = true
           ActiveRecord.after_all_transactions_commit do
             self.successfully_enqueued = false

data/lib/active_job/enqueuing.rb

@@ -88,7 +88,7 @@ module ActiveJob
       end
 
       private
-        def job_or_instantiate(*args, &_) # :doc:
+        def job_or_instantiate(*args, &) # :doc:
           args.first.is_a?(self) ? args.first : new(*args)
         end
         ruby2_keywords(:job_or_instantiate)
@@ -113,9 +113,7 @@ module ActiveJob
       set(options)
       self.successfully_enqueued = false
 
-      run_callbacks :enqueue do
-        raw_enqueue
-      end
+      raw_enqueue
 
       if successfully_enqueued?
         self
@@ -126,6 +124,12 @@ module ActiveJob
 
     private
       def raw_enqueue
+        run_callbacks :enqueue do
+          _raw_enqueue
+        end
+      end
+
+      def _raw_enqueue
         if scheduled_at
           queue_adapter.enqueue_at self, scheduled_at.to_f
         else

data/lib/active_job/exceptions.rb

@@ -34,6 +34,7 @@ module ActiveJob
       # * <tt>:queue</tt> - Re-enqueues the job on a different queue
       # * <tt>:priority</tt> - Re-enqueues the job with a different priority
       # * <tt>:jitter</tt> - A random delay of wait time used when calculating backoff. The default is 15% (0.15) which represents the upper bound of possible wait time (expressed as a percentage)
+      # * <tt>:report</tt> - Errors will be reported to the Rails.error reporter before being retried
       #
       # ==== Examples
       #
@@ -49,8 +50,9 @@ module ActiveJob
       #    # retry_on Net::ReadTimeout, wait: 5.seconds, jitter: 0.30, attempts: 10
       #    # retry_on Timeout::Error, wait: :polynomially_longer, attempts: 10
       #
-      #    retry_on(YetAnotherCustomAppException) do |job, error|
-      #      ExceptionNotifier.caught(error)
+      #    retry_on YetAnotherCustomAppException, report: true
+      #    retry_on EvenWorseCustomAppException do |job, error|
+      #      CustomErrorHandlingCode.handle(job, error)
       #    end
       #
       #    def perform(*args)
@@ -59,10 +61,11 @@ module ActiveJob
       #      # Might raise Net::OpenTimeout or Timeout::Error when the remote service is down
       #    end
       #  end
-      def retry_on(*exceptions, wait: 3.seconds, attempts: 5, queue: nil, priority: nil, jitter: JITTER_DEFAULT)
+      def retry_on(*exceptions, wait: 3.seconds, attempts: 5, queue: nil, priority: nil, jitter: JITTER_DEFAULT, report: false)
         rescue_from(*exceptions) do |error|
           executions = executions_for(exceptions)
           if attempts == :unlimited || executions < attempts
+            ActiveSupport.error_reporter.report(error, source: "application.active_job") if report
             retry_job wait: determine_delay(seconds_or_duration_or_algorithm: wait, executions: executions, jitter: jitter), queue: queue, priority: priority, error: error
           else
             if block_given?
@@ -82,6 +85,8 @@ module ActiveJob
       # Discard the job with no attempts to retry, if the exception is raised. This is useful when the subject of the job,
       # like an Active Record, is no longer available, and the job is thus no longer relevant.
       #
+      # Passing the <tt>:report</tt> option reports the error through the error reporter before discarding the job.
+      #
       # You can also pass a block that'll be invoked. This block is yielded with the job instance as the first and the error instance as the second parameter.
       #
       # +retry_on+ and +discard_on+ handlers are searched from bottom to top, and up the class hierarchy. The handler of the first class for
@@ -91,8 +96,9 @@ module ActiveJob
       #
       #  class SearchIndexingJob < ActiveJob::Base
       #    discard_on ActiveJob::DeserializationError
-      #    discard_on(CustomAppException) do |job, error|
-      #      ExceptionNotifier.caught(error)
+      #    discard_on CustomAppException, report: true
+      #    discard_on(AnotherCustomAppException) do |job, error|
+      #      CustomErrorHandlingCode.handle(job, error)
       #    end
       #
       #    def perform(record)
@@ -100,9 +106,10 @@ module ActiveJob
       #      # Might raise CustomAppException for something domain specific
       #    end
       #  end
-      def discard_on(*exceptions)
+      def discard_on(*exceptions, report: false)
         rescue_from(*exceptions) do |error|
           instrument :discard, error: error do
+            ActiveSupport.error_reporter.report(error, source: "application.active_job") if report
             yield self, error if block_given?
             run_after_discard_procs(error)
           end
@@ -150,7 +157,10 @@ module ActiveJob
     #  end
     def retry_job(options = {})
       instrument :enqueue_retry, options.slice(:error, :wait) do
+        scheduled_at, queue_name, priority = self.scheduled_at, self.queue_name, self.priority
         enqueue options
+      ensure
+        self.scheduled_at, self.queue_name, self.priority = scheduled_at, queue_name, priority
       end
     end
 

data/lib/active_job/execution_state.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module ActiveJob
+  module ExecutionState # :nodoc:
+    def perform_now
+      I18n.with_locale(locale) do
+        Time.use_zone(timezone) { super }
+      end
+    end
+  end
+end

data/lib/active_job/gem_version.rb

@@ -8,9 +8,9 @@ module ActiveJob
 
   module VERSION
     MAJOR = 8
-    MINOR = 0
-    TINY  = 2
-    PRE   = nil
+    MINOR = 1
+    TINY  = 0
+    PRE   = "beta1"
 
     STRING = [MAJOR, MINOR, TINY, PRE].compact.join(".")
   end

data/lib/active_job/instrumentation.rb

@@ -26,24 +26,24 @@ module ActiveJob
       instrument(:perform) { super }
     end
 
+    def instrument(operation, payload = {}, &block) # :nodoc:
+      payload[:job] = self
+      payload[:adapter] = queue_adapter
+
+      ActiveSupport::Notifications.instrument("#{operation}.active_job", payload) do |payload|
+        value = block.call(payload) if block
+        payload[:aborted] = @_halted_callback_hook_called if defined?(@_halted_callback_hook_called)
+        @_halted_callback_hook_called = nil
+        value
+      end
+    end
+
     private
       def _perform_job
         instrument(:perform_start)
         super
       end
 
-      def instrument(operation, payload = {}, &block)
-        payload[:job] = self
-        payload[:adapter] = queue_adapter
-
-        ActiveSupport::Notifications.instrument("#{operation}.active_job", payload) do
-          value = block.call if block
-          payload[:aborted] = @_halted_callback_hook_called if defined?(@_halted_callback_hook_called)
-          @_halted_callback_hook_called = nil
-          value
-        end
-      end
-
       def halted_callback_hook(*)
         super
         @_halted_callback_hook_called = true

data/lib/active_job/log_subscriber.rb

@@ -87,8 +87,9 @@ module ActiveJob
       job = event.payload[:job]
       ex = event.payload[:exception_object]
       if ex
+        cleaned_backtrace = backtrace_cleaner.clean(ex.backtrace)
         error do
-          "Error performing #{job.class.name} (Job ID: #{job.job_id}) from #{queue_name(event)} in #{event.duration.round(2)}ms: #{ex.class} (#{ex.message}):\n" + Array(ex.backtrace).join("\n")
+          "Error performing #{job.class.name} (Job ID: #{job.job_id}) from #{queue_name(event)} in #{event.duration.round(2)}ms: #{ex.class} (#{ex.message}):\n" + Array(cleaned_backtrace).join("\n")
         end
       elsif event.payload[:aborted]
         error do
@@ -137,6 +138,64 @@ module ActiveJob
     end
     subscribe_log_level :discard, :error
 
+    def interrupt(event)
+      job = event.payload[:job]
+      info do
+        "Interrupted #{job.class} (Job ID: #{job.job_id}) #{event.payload[:description]} (#{event.payload[:reason]})"
+      end
+    end
+    subscribe_log_level :interrupt, :info
+
+    def resume(event)
+      job = event.payload[:job]
+      info do
+        "Resuming #{job.class} (Job ID: #{job.job_id}) #{event.payload[:description]}"
+      end
+    end
+    subscribe_log_level :resume, :info
+
+    def step_skipped(event)
+      job = event.payload[:job]
+      info do
+        "Step '#{event.payload[:step].name}' skipped #{job.class}"
+      end
+    end
+    subscribe_log_level :step_skipped, :info
+
+    def step_started(event)
+      job = event.payload[:job]
+      step = event.payload[:step]
+      info do
+        if step.resumed?
+          "Step '#{step.name}' resumed from cursor '#{step.cursor}' for #{job.class} (Job ID: #{job.job_id})"
+        else
+          "Step '#{step.name}' started for #{job.class} (Job ID: #{job.job_id})"
+        end
+      end
+    end
+    subscribe_log_level :step_started, :info
+
+    def step(event)
+      job = event.payload[:job]
+      step = event.payload[:step]
+      ex = event.payload[:exception_object]
+
+      if event.payload[:interrupted]
+        info do
+          "Step '#{step.name}' interrupted at cursor '#{step.cursor}' for #{job.class} (Job ID: #{job.job_id}) in #{event.duration.round(2)}ms"
+        end
+      elsif ex
+        error do
+          "Error during step '#{step.name}' at cursor '#{step.cursor}' for #{job.class} (Job ID: #{job.job_id}) in #{event.duration.round(2)}ms: #{ex.class} (#{ex.message})"
+        end
+      else
+        info do
+          "Step '#{step.name}' completed for #{job.class} (Job ID: #{job.job_id}) in #{event.duration.round(2)}ms"
+        end
+      end
+    end
+    subscribe_log_level :step, :error
+
     private
       def queue_name(event)
         ActiveJob.adapter_name(event.payload[:adapter]) + "(#{event.payload[:job].queue_name})"
@@ -197,11 +256,7 @@ module ActiveJob
       end
 
       def enqueue_source_location
-        Thread.each_caller_location do |location|
-          frame = backtrace_cleaner.clean_frame(location)
-          return frame if frame
-        end
-        nil
+        backtrace_cleaner.first_clean_frame
       end
 
       def enqueued_jobs_message(adapter, enqueued_jobs)

data/lib/active_job/queue_adapters/abstract_adapter.rb

@@ -7,6 +7,8 @@ module ActiveJob
     # Active Job supports multiple job queue systems. ActiveJob::QueueAdapters::AbstractAdapter
     # forms the abstraction layer which makes this possible.
     class AbstractAdapter
+      attr_accessor :stopping
+
       def enqueue(job)
         raise NotImplementedError
       end
@@ -14,6 +16,10 @@ module ActiveJob
       def enqueue_at(job, timestamp)
         raise NotImplementedError
       end
+
+      def stopping?
+        !!@stopping
+      end
     end
   end
 end

data/lib/active_job/queue_adapters/async_adapter.rb

@@ -86,7 +86,11 @@ module ActiveJob
         def initialize(**options)
           self.immediate = false
           @immediate_executor = Concurrent::ImmediateExecutor.new
-          @async_executor = Concurrent::ThreadPoolExecutor.new(DEFAULT_EXECUTOR_OPTIONS.merge(options))
+          @async_executor = Concurrent::ThreadPoolExecutor.new(
+            name: "ActiveJob-async-scheduler",
+            **DEFAULT_EXECUTOR_OPTIONS,
+            **options
+          )
         end
 
         def enqueue(job, queue_name:)