activejob 7.2.3 → 8.1.3

data/lib/active_job/continuation/test_helper.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+require "active_job/test_helper"
+require "active_job/continuation"
+
+module ActiveJob
+  class Continuation
+    # Test helper for ActiveJob::Continuable jobs.
+    #
+    module TestHelper
+      include ::ActiveJob::TestHelper
+
+      # Interrupt a job during a step.
+      #
+      #  class MyJob < ApplicationJob
+      #    include ActiveJob::Continuable
+      #
+      #    cattr_accessor :items, default: []
+      #    def perform
+      #      step :my_step, start: 1 do |step|
+      #        (step.cursor..10).each do |i|
+      #          items << i
+      #          step.advance!
+      #        end
+      #      end
+      #    end
+      #  end
+      #
+      #  test "interrupt job during step" do
+      #    MyJob.perform_later
+      #    interrupt_job_during_step(MyJob, :my_step, cursor: 6) { perform_enqueued_jobs }
+      #    assert_equal [1, 2, 3, 4, 5], MyJob.items
+      #    perform_enqueued_jobs
+      #    assert_equal [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], MyJob.items
+      #  end
+      def interrupt_job_during_step(job, step, cursor: nil, &block)
+        require_active_job_test_adapter!("interrupt_job_during_step")
+        queue_adapter.with(stopping: ->() { during_step?(job, step, cursor: cursor) }, &block)
+      end
+
+      # Interrupt a job after a step.
+      #
+      # Note that there's no checkpoint after the final step so it won't be interrupted.
+      #
+      #  class MyJob < ApplicationJob
+      #    include ActiveJob::Continuable
+      #
+      #    cattr_accessor :items, default: []
+      #
+      #    def perform
+      #      step :step_one { items << 1 }
+      #      step :step_two { items << 2 }
+      #      step :step_three { items << 3 }
+      #      step :step_four { items << 4 }
+      #    end
+      #  end
+      #
+      #  test "interrupt job after step" do
+      #    MyJob.perform_later
+      #    interrupt_job_after_step(MyJob, :step_two) { perform_enqueued_jobs }
+      #    assert_equal [1, 2], MyJob.items
+      #    perform_enqueued_jobs
+      #    assert_equal [1, 2, 3, 4], MyJob.items
+      #  end
+      def interrupt_job_after_step(job, step, &block)
+        require_active_job_test_adapter!("interrupt_job_after_step")
+        queue_adapter.with(stopping: ->() { after_step?(job, step) }, &block)
+      end
+
+      private
+        def continuation_for(klass)
+          job = ActiveSupport::ExecutionContext.to_h[:job]
+          job.send(:continuation)&.to_h if job && job.is_a?(klass)
+        end
+
+        def during_step?(job, step, cursor: nil)
+          if (continuation = continuation_for(job))
+            continuation["current"] == [ step.to_s, cursor ]
+          end
+        end
+
+        def after_step?(job, step)
+          if (continuation = continuation_for(job))
+            continuation["completed"].last == step.to_s && continuation["current"].nil?
+          end
+        end
+    end
+  end
+end

data/lib/active_job/continuation/validation.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module ActiveJob
+  class Continuation
+    module Validation # :nodoc:
+      private
+        def validate_step!(name)
+          validate_step_symbol!(name)
+          validate_step_not_encountered!(name)
+          validate_step_not_nested!(name)
+          validate_step_resume_expected!(name)
+          validate_step_expected_order!(name)
+        end
+
+        def validate_step_symbol!(name)
+          unless name.is_a?(Symbol)
+            raise_step_error! "Step '#{name}' must be a Symbol, found '#{name.class}'"
+          end
+        end
+
+        def validate_step_not_encountered!(name)
+          if encountered.include?(name)
+            raise_step_error! "Step '#{name}' has already been encountered"
+          end
+        end
+
+        def validate_step_not_nested!(name)
+          if running_step?
+            raise_step_error! "Step '#{name}' is nested inside step '#{current.name}'"
+          end
+        end
+
+        def validate_step_resume_expected!(name)
+          if current && current.name != name && !completed?(name)
+            raise_step_error! "Step '#{name}' found, expected to resume from '#{current.name}'"
+          end
+        end
+
+        def validate_step_expected_order!(name)
+          if completed.size > encountered.size && completed[encountered.size] != name
+            raise_step_error! "Step '#{name}' found, expected to see '#{completed[encountered.size]}'"
+          end
+        end
+
+        def raise_step_error!(message)
+          raise InvalidStepError, message
+        end
+    end
+  end
+end

data/lib/active_job/continuation.rb

@@ -0,0 +1,332 @@
+# frozen_string_literal: true
+
+require "active_support/core_ext/numeric/time"
+require "active_job/continuable"
+
+module ActiveJob
+  # = Active Job \Continuation
+  #
+  # Continuations provide a mechanism for interrupting and resuming jobs. This allows
+  # long-running jobs to make progress across application restarts.
+  #
+  # Jobs should include the ActiveJob::Continuable module to enable continuations.
+  # \Continuable jobs are automatically retried when interrupted.
+  #
+  # Use the +step+ method to define the steps in your job. Steps can use an optional
+  # cursor to track progress in the step.
+  #
+  # Steps are executed as soon as they are encountered. If a job is interrupted, previously
+  # completed steps will be skipped. If a step is in progress, it will be resumed
+  # with the last recorded cursor.
+  #
+  # Code that is not part of a step will be executed on each job run.
+  #
+  # You can pass a block or a method name to the step method. The block will be called with
+  # the step object as an argument. Methods can either take no arguments or a single argument
+  # for the step object.
+  #
+  #   class ProcessImportJob < ApplicationJob
+  #     include ActiveJob::Continuable
+  #
+  #     def perform(import_id)
+  #       # This always runs, even if the job is resumed.
+  #       @import = Import.find(import_id)
+  #
+  #       step :validate do
+  #         @import.validate!
+  #       end
+  #
+  #       step(:process_records) do |step|
+  #         @import.records.find_each(start: step.cursor) do |record|
+  #           record.process
+  #           step.advance! from: record.id
+  #         end
+  #       end
+  #
+  #       step :reprocess_records
+  #       step :finalize
+  #     end
+  #
+  #     def reprocess_records(step)
+  #       @import.records.find_each(start: step.cursor) do |record|
+  #         record.reprocess
+  #         step.advance! from: record.id
+  #       end
+  #     end
+  #
+  #     def finalize
+  #       @import.finalize!
+  #     end
+  #   end
+  #
+  # === Cursors
+  #
+  # Cursors are used to track progress within a step. The cursor can be any object that is
+  # serializable as an argument to +ActiveJob::Base.serialize+. It defaults to +nil+.
+  #
+  # When a step is resumed, the last cursor value is restored. The code in the step is responsible
+  # for using the cursor to continue from the right point.
+  #
+  # +set!+ sets the cursor to a specific value.
+  #
+  #   step :iterate_items do |step|
+  #     items[step.cursor..].each do |item|
+  #       process(item)
+  #       step.set! (step.cursor || 0) + 1
+  #     end
+  #   end
+  #
+  # An starting value for the cursor can be set when defining the step:
+  #
+  #   step :iterate_items, start: 0 do |step|
+  #     items[step.cursor..].each do |item|
+  #       process(item)
+  #       step.set! step.cursor + 1
+  #     end
+  #   end
+  #
+  # The cursor can be advanced with +advance!+. This calls +succ+ on the current cursor value.
+  # It raises an ActiveJob::Continuation::UnadvanceableCursorError if the cursor does not implement +succ+.
+  #
+  #   step :iterate_items, start: 0 do |step|
+  #     items[step.cursor..].each do |item|
+  #       process(item)
+  #       step.advance!
+  #     end
+  #   end
+  #
+  # You can optionally pass a +from+ argument to +advance!+. This is useful when iterating
+  # over a collection of records where IDs may not be contiguous.
+  #
+  #   step :process_records do |step|
+  #     import.records.find_each(start: step.cursor) do |record|
+  #       record.process
+  #       step.advance! from: record.id
+  #     end
+  #   end
+  #
+  # You can use an array to iterate over nested records:
+  #
+  #   step :process_nested_records, start: [ 0, 0 ] do |step|
+  #     Account.find_each(start: step.cursor[0]) do |account|
+  #       account.records.find_each(start: step.cursor[1]) do |record|
+  #         record.process
+  #         step.set! [ account.id, record.id + 1 ]
+  #       end
+  #       step.set! [ account.id + 1, 0 ]
+  #     end
+  #   end
+  #
+  # Setting or advancing the cursor creates a checkpoint. You can also create a checkpoint
+  # manually by calling the +checkpoint!+ method on the step. This is useful if you want to
+  # allow interruptions, but don't need to update the cursor.
+  #
+  #   step :destroy_records do |step|
+  #     import.records.find_each do |record|
+  #       record.destroy!
+  #       step.checkpoint!
+  #     end
+  #   end
+  #
+  # === Checkpoints
+  #
+  # A checkpoint is where a job can be interrupted. At a checkpoint the job will call
+  # +queue_adapter.stopping?+. If it returns true, the job will raise an
+  # ActiveJob::Continuation::Interrupt exception.
+  #
+  # There is an automatic checkpoint before the start of each step except for the first for
+  # each job execution. Within a step one is created when calling +set!+, +advance!+ or +checkpoint!+.
+  #
+  # Jobs are not automatically interrupted when the queue adapter is marked as stopping - they
+  # will continue to run either until the next checkpoint, or when the process is stopped.
+  #
+  # This is to allow jobs to be interrupted at a safe point, but it also means that the jobs
+  # should checkpoint more frequently than the shutdown timeout to ensure a graceful restart.
+  #
+  # When interrupted, the job will automatically retry with the progress serialized
+  # in the job data under the +continuation+ key.
+  #
+  # The serialized progress contains:
+  # - a list of the completed steps
+  # - the current step and its cursor value (if one is in progress)
+  #
+  # === Isolated Steps
+  #
+  # Steps run sequentially in a single job execution, unless the job is interrupted.
+  #
+  # You can specify that a step should always run in its own execution by passing the +isolated: true+ option.
+  #
+  # This is useful for long-running steps where it may not be possible to checkpoint within
+  # the job grace period - it ensures that progress is serialized back into the job data before
+  # the step starts.
+  #
+  #   step :quick_step1
+  #   step :slow_step, isolated: true
+  #   step :quick_step2
+  #   step :quick_step3
+  #
+  # === Errors
+  #
+  # If a job raises an error and is not retried via Active Job, it will be passed back to the underlying
+  # queue backend and any progress in this execution will be lost.
+  #
+  # To mitigate this, the job will be automatically retried if it raises an error after it has made progress.
+  # Making progress is defined as having completed a step or advanced the cursor within the current step.
+  #
+  # === Configuration
+  #
+  # Continuable jobs have several configuration options:
+  # * <tt>:max_resumptions</tt> - The maximum number of times a job can be resumed. Defaults to +nil+ which means
+  #   unlimited resumptions.
+  # * <tt>:resume_options</tt> - Options to pass to +retry_job+ when resuming the job.
+  #   Defaults to <tt>{ wait: 5.seconds }</tt>.
+  #   See {ActiveJob::Exceptions#retry_job}[rdoc-ref:ActiveJob::Exceptions#retry_job] for available options.
+  # * <tt>:resume_errors_after_advancing</tt> - Whether to resume errors after advancing the continuation.
+  #   Defaults to +true+.
+  class Continuation
+    extend ActiveSupport::Autoload
+
+    autoload :Validation
+
+    # Raised when a job is interrupted, allowing Active Job to requeue it.
+    # This inherits from +Exception+ rather than +StandardError+, so it's not
+    # caught by normal exception handling.
+    class Interrupt < Exception; end
+
+    # Base class for all Continuation errors.
+    class Error < StandardError; end
+
+    # Raised when a step is invalid.
+    class InvalidStepError < Error; end
+
+    # Raised when there is an error with a checkpoint, such as open database transactions.
+    class CheckpointError < Error; end
+
+    # Raised when attempting to advance a cursor that doesn't implement `succ`.
+    class UnadvanceableCursorError < Error; end
+
+    # Raised when a job has reached its limit of the number of resumes.
+    # The limit is defined by the +max_resumes+ class attribute.
+    class ResumeLimitError < Error; end
+
+    include Validation
+
+    def initialize(job, serialized_progress) # :nodoc:
+      @job = job
+      @completed = serialized_progress.fetch("completed", []).map(&:to_sym)
+      @current = new_step(*serialized_progress["current"], resumed: true) if serialized_progress.key?("current")
+      @encountered = []
+      @advanced = false
+      @running_step = false
+      @isolating = false
+    end
+
+    def step(name, **options, &block) # :nodoc:
+      validate_step!(name)
+      encountered << name
+
+      if completed?(name)
+        skip_step(name)
+      else
+        run_step(name, **options, &block)
+      end
+    end
+
+    def to_h # :nodoc:
+      {
+        "completed" => completed.map(&:to_s),
+        "current" => current&.to_a,
+      }.compact
+    end
+
+    def description # :nodoc:
+      if current
+        current.description
+      elsif completed.any?
+        "after '#{completed.last}'"
+      else
+        "not started"
+      end
+    end
+
+    def started?
+      completed.any? || current.present?
+    end
+
+    def advanced?
+      @advanced
+    end
+
+    def instrumentation
+      { description: description,
+        completed_steps: completed,
+        current_step: current }
+    end
+
+    private
+      attr_reader :job, :encountered, :completed, :current
+
+      def running_step?
+        @running_step
+      end
+
+      def isolating?
+        @isolating
+      end
+
+      def completed?(name)
+        completed.include?(name)
+      end
+
+      def new_step(*args, **options)
+        Step.new(*args, job: job, **options)
+      end
+
+      def skip_step(name)
+        instrument :step_skipped, step: name
+      end
+
+      def run_step(name, start:, isolated:, &block)
+        @isolating ||= isolated
+
+        if isolating? && advanced?
+          job.interrupt!(reason: :isolating)
+        else
+          run_step_inline(name, start: start, &block)
+        end
+      end
+
+      def run_step_inline(name, start:, **options, &block)
+        @running_step = true
+        @current ||= new_step(name, start, resumed: false)
+
+        instrumenting_step(current) do
+          block.call(current)
+        end
+
+        @completed << current.name
+        @current = nil
+        @advanced = true
+      ensure
+        @running_step = false
+        @advanced ||= current&.advanced?
+      end
+
+      def instrumenting_step(step, &block)
+        instrument :step, step: step, interrupted: false do |payload|
+          instrument :step_started, step: step
+
+          block.call
+        rescue Interrupt
+          payload[:interrupted] = true
+          raise
+        end
+      end
+
+      def instrument(...)
+        job.instrument(...)
+      end
+  end
+end
+
+require "active_job/continuation/step"

data/lib/active_job/core.rb

@@ -1,6 +1,13 @@
 # frozen_string_literal: true
 
 module ActiveJob
+  # Raised during job payload deserialization when it references an uninitialized job class.
+  class UnknownJobClassError < NameError
+    def initialize(job_class_name)
+      super("Failed to instantiate job, class `#{job_class_name}` doesn't exist", job_class_name)
+    end
+  end
+
   # = Active Job \Core
   #
   # Provides general behavior that will be included into every Active Job
@@ -60,7 +67,10 @@ module ActiveJob
     module ClassMethods
       # Creates a new job instance from a hash created with +serialize+
       def deserialize(job_data)
-        job = job_data["job_class"].constantize.new
+        job_class = job_data["job_class"].safe_constantize
+        raise UnknownJobClassError, job_data["job_class"] unless job_class
+
+        job = job_class.new
         job.deserialize(job_data)
         job
       end
@@ -157,8 +167,8 @@ module ActiveJob
       self.exception_executions = job_data["exception_executions"]
       self.locale               = job_data["locale"] || I18n.locale.to_s
       self.timezone             = job_data["timezone"] || Time.zone&.name
-      self.enqueued_at          = Time.iso8601(job_data["enqueued_at"]) if job_data["enqueued_at"]
-      self.scheduled_at         = Time.iso8601(job_data["scheduled_at"]) if job_data["scheduled_at"]
+      self.enqueued_at          = deserialize_time(job_data["enqueued_at"]) if job_data["enqueued_at"]
+      self.scheduled_at         = deserialize_time(job_data["scheduled_at"]) if job_data["scheduled_at"]
     end
 
     # Configures the job with the given options.
@@ -198,5 +208,13 @@ module ActiveJob
       def arguments_serialized?
         @serialized_arguments
       end
+
+      def deserialize_time(time)
+        if time.is_a?(Time)
+          time
+        else
+          Time.iso8601(time)
+        end
+      end
   end
 end

data/lib/active_job/enqueue_after_transaction_commit.rb

@@ -2,18 +2,28 @@
 
 module ActiveJob
   module EnqueueAfterTransactionCommit # :nodoc:
+    class << self
+      def included(base)
+        ActiveJob.singleton_class.prepend(ActiveJobMethods)
+      end
+    end
+
+    module ActiveJobMethods
+      # Ensures perform_all_later respects each job's enqueue_after_transaction_commit configuration.
+      # Jobs with enqueue_after_transaction_commit set to true are deferred and enqueued only after the transaction commits;
+      # other jobs are enqueued immediately. This ensures enqueuing timing matches the per-job setting.
+      def perform_all_later(*jobs)
+        jobs.flatten!
+        deferred_jobs, immediate_jobs = jobs.partition { |job| job.class.enqueue_after_transaction_commit }
+        super(immediate_jobs) if immediate_jobs.any?
+        ActiveRecord.after_all_transactions_commit { super(deferred_jobs) } if deferred_jobs.any?
+        nil
+      end
+    end
+
     private
       def raw_enqueue
-        after_transaction = case self.class.enqueue_after_transaction_commit
-        when :always
-          true
-        when :never
-          false
-        else # :default
-          queue_adapter.enqueue_after_transaction_commit?
-        end
-
-        if after_transaction
+        if self.class.enqueue_after_transaction_commit
           self.successfully_enqueued = true
           ActiveRecord.after_all_transactions_commit do
             self.successfully_enqueued = false

data/lib/active_job/enqueuing.rb

@@ -48,10 +48,9 @@ module ActiveJob
       # automatically defers the enqueue to after the transaction commits.
       #
       # It can be set on a per job basis:
-      #  - `:always` forces the job to be deferred.
-      #  - `:never` forces the job to be queued immediately.
-      #  - `:default` lets the queue adapter define the behavior (recommended).
-      class_attribute :enqueue_after_transaction_commit, instance_accessor: false, instance_predicate: false, default: :never
+      #  - true forces the job to be deferred.
+      #  - false forces the job to be queued immediately.
+      class_attribute :enqueue_after_transaction_commit, instance_accessor: false, instance_predicate: false, default: false
     end
 
     # Includes the +perform_later+ method for job initialization.
@@ -89,7 +88,7 @@ module ActiveJob
       end
 
       private
-        def job_or_instantiate(*args, &_) # :doc:
+        def job_or_instantiate(*args, &) # :doc:
           args.first.is_a?(self) ? args.first : new(*args)
         end
         ruby2_keywords(:job_or_instantiate)
@@ -114,9 +113,7 @@ module ActiveJob
       set(options)
       self.successfully_enqueued = false
 
-      run_callbacks :enqueue do
-        raw_enqueue
-      end
+      raw_enqueue
 
       if successfully_enqueued?
         self
@@ -127,6 +124,12 @@ module ActiveJob
 
     private
       def raw_enqueue
+        run_callbacks :enqueue do
+          _raw_enqueue
+        end
+      end
+
+      def _raw_enqueue
         if scheduled_at
           queue_adapter.enqueue_at self, scheduled_at.to_f
         else

data/lib/active_job/exceptions.rb

@@ -34,6 +34,7 @@ module ActiveJob
       # * <tt>:queue</tt> - Re-enqueues the job on a different queue
       # * <tt>:priority</tt> - Re-enqueues the job with a different priority
       # * <tt>:jitter</tt> - A random delay of wait time used when calculating backoff. The default is 15% (0.15) which represents the upper bound of possible wait time (expressed as a percentage)
+      # * <tt>:report</tt> - Errors will be reported to the Rails.error reporter before being retried
       #
       # ==== Examples
       #
@@ -49,8 +50,9 @@ module ActiveJob
       #    # retry_on Net::ReadTimeout, wait: 5.seconds, jitter: 0.30, attempts: 10
       #    # retry_on Timeout::Error, wait: :polynomially_longer, attempts: 10
       #
-      #    retry_on(YetAnotherCustomAppException) do |job, error|
-      #      ExceptionNotifier.caught(error)
+      #    retry_on YetAnotherCustomAppException, report: true
+      #    retry_on EvenWorseCustomAppException do |job, error|
+      #      CustomErrorHandlingCode.handle(job, error)
       #    end
       #
       #    def perform(*args)
@@ -59,10 +61,11 @@ module ActiveJob
       #      # Might raise Net::OpenTimeout or Timeout::Error when the remote service is down
       #    end
       #  end
-      def retry_on(*exceptions, wait: 3.seconds, attempts: 5, queue: nil, priority: nil, jitter: JITTER_DEFAULT)
+      def retry_on(*exceptions, wait: 3.seconds, attempts: 5, queue: nil, priority: nil, jitter: JITTER_DEFAULT, report: false)
         rescue_from(*exceptions) do |error|
           executions = executions_for(exceptions)
           if attempts == :unlimited || executions < attempts
+            ActiveSupport.error_reporter.report(error, source: "application.active_job") if report
             retry_job wait: determine_delay(seconds_or_duration_or_algorithm: wait, executions: executions, jitter: jitter), queue: queue, priority: priority, error: error
           else
             if block_given?
@@ -82,6 +85,8 @@ module ActiveJob
       # Discard the job with no attempts to retry, if the exception is raised. This is useful when the subject of the job,
       # like an Active Record, is no longer available, and the job is thus no longer relevant.
       #
+      # Passing the <tt>:report</tt> option reports the error through the error reporter before discarding the job.
+      #
       # You can also pass a block that'll be invoked. This block is yielded with the job instance as the first and the error instance as the second parameter.
       #
       # +retry_on+ and +discard_on+ handlers are searched from bottom to top, and up the class hierarchy. The handler of the first class for
@@ -91,8 +96,9 @@ module ActiveJob
       #
       #  class SearchIndexingJob < ActiveJob::Base
       #    discard_on ActiveJob::DeserializationError
-      #    discard_on(CustomAppException) do |job, error|
-      #      ExceptionNotifier.caught(error)
+      #    discard_on CustomAppException, report: true
+      #    discard_on(AnotherCustomAppException) do |job, error|
+      #      CustomErrorHandlingCode.handle(job, error)
       #    end
       #
       #    def perform(record)
@@ -100,9 +106,10 @@ module ActiveJob
       #      # Might raise CustomAppException for something domain specific
       #    end
       #  end
-      def discard_on(*exceptions)
+      def discard_on(*exceptions, report: false)
         rescue_from(*exceptions) do |error|
           instrument :discard, error: error do
+            ActiveSupport.error_reporter.report(error, source: "application.active_job") if report
             yield self, error if block_given?
             run_after_discard_procs(error)
           end
@@ -150,8 +157,10 @@ module ActiveJob
     #  end
     def retry_job(options = {})
       instrument :enqueue_retry, options.slice(:error, :wait) do
-        job = dup
-        job.enqueue options
+        scheduled_at, queue_name, priority = self.scheduled_at, self.queue_name, self.priority
+        enqueue options
+      ensure
+        self.scheduled_at, self.queue_name, self.priority = scheduled_at, queue_name, priority
       end
     end
 

data/lib/active_job/execution_state.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module ActiveJob
+  module ExecutionState # :nodoc:
+    def perform_now
+      I18n.with_locale(locale) do
+        Time.use_zone(timezone) { super }
+      end
+    end
+  end
+end