delayed 3.1.0 → 4.0.1

data/lib/delayed/limitable.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module Delayed
+  # A mixin that wraps a class's `perform` method (or other methods named via
+  # `on:`) in `Delayed::Limit.within_limit`. It is automatically included in
+  # ActiveJob classes, and configures the job to retry with a polynomial
+  # backoff when the limit's `wait_timeout` would be exceeded:
+  #
+  #   class TouchesThirdPartyApiJob < ApplicationJob
+  #     with_limit :third_party_api, max: 100, per: 1.minute
+  #
+  #     def perform
+  #       # ...
+  #     end
+  #   end
+  #
+  # The 'purpose' defaults to the job's underscored class name, and the limit
+  # is registered via `Delayed::Limit.register!` (unless the purpose was
+  # already registered, e.g. in an initializer, in which case the `max:`/`per:`
+  # config may be omitted entirely). Multiple job classes may share a purpose
+  # (and its limit) as long as their configs match exactly.
+  #
+  # Use `on:` to wrap one or more other instance methods instead of `perform`,
+  # e.g. if only a portion of the job's work is subject to the limit:
+  #
+  #   with_limit :third_party_api, max: 100, per: 1.minute, on: :deliver!
+  #
+  # A class may declare `with_limit` more than once (e.g. to apply different
+  # limits to different methods), but only the first declaration defines the
+  # job's retry behavior (`attempts`, `wait`, and `jitter`, with `wait_timeout`
+  # acting as a floor on the computed wait). If two declarations' wait timeouts
+  # differ meaningfully, declare the one with the longer `wait_timeout` first.
+  #
+  # Retries rely on ActiveJob's `retry_on`, so when this mixin is included in
+  # a plain (non-ActiveJob) class, the named methods are still wrapped in
+  # `within_limit`, but the class must define its own rescue/retry behavior
+  # for `Delayed::Limit::LimitExceededError`.
+  module Limitable
+    extend ActiveSupport::Concern
+
+    DEFAULT_RETRY_ATTEMPTS = if defined?(ActiveJob) && ActiveJob.gem_version >= Gem::Version.new('7.0')
+      :unlimited
+    else
+      Float::INFINITY
+    end
+
+    class_methods do
+      def with_limit(
+        purpose = name.underscore.to_sym,
+        on: :perform,
+        max: nil,
+        per: nil,
+        wait_timeout: 5.seconds,
+        retry_jitter: 0.1,
+        retry_attempts: DEFAULT_RETRY_ATTEMPTS,
+        retry_wait: ->(attempt) { polynomial_backoff(wait_timeout, attempt, retry_jitter) }
+      )
+        Delayed::Limit.register!(purpose, max: max, per: per) if max || per
+
+        if defined?(ActiveJob::Base) && self < ActiveJob::Base &&
+            rescue_handlers.none? { |klass, _| klass == Delayed::Limit::LimitExceededError.name }
+          retry_on(Delayed::Limit::LimitExceededError, attempts: retry_attempts, wait: retry_wait)
+        end
+
+        prepend(Module.new do
+          Array(on).each do |method_name|
+            define_method(method_name) do |*args, **kwargs, &block|
+              Delayed::Limit.within_limit(purpose, wait_timeout: wait_timeout) do
+                super(*args, **kwargs, &block)
+              end
+            end
+          end
+        end)
+      end
+
+      private
+
+      def polynomial_backoff(min_wait, attempt, jitter)
+        [min_wait, (attempt**4)].max * (1 + rand(-jitter..jitter))
+      end
+    end
+  end
+end

data/lib/delayed/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Delayed
-  VERSION = '3.1.0'
+  VERSION = '4.0.1'
 end

data/lib/delayed.rb

@@ -16,12 +16,14 @@ require 'delayed/backend/job_preparer'
 require 'delayed/helpers/migration'
 require 'delayed/worker'
 require 'delayed/job_wrapper'
+require 'delayed/limitable'
 
 if defined?(Rails::Engine)
   require 'delayed/engine'
 else
   require 'active_record'
   require_relative '../app/models/delayed/job'
+  require_relative '../app/models/delayed/limit'
 end
 
 ActiveSupport.on_load(:active_job) do
@@ -29,6 +31,7 @@ ActiveSupport.on_load(:active_job) do
   ActiveJob::QueueAdapters::DelayedAdapter = Class.new(Delayed::ActiveJobAdapter)
 
   include Delayed::ActiveJobAdapter::EnqueuingPatch
+  include Delayed::Limitable
 end
 
 ActiveSupport.on_load(:action_mailer) do

data/spec/delayed/active_job_adapter_spec.rb

@@ -113,10 +113,18 @@ RSpec.describe Delayed::ActiveJobAdapter do
       Delayed::Job.last.tap do |dj|
         dj.update!(handler: dj.handler.gsub('JobClass', 'MissingJobClass'))
         expect { dj.payload_object }.not_to raise_error
-        expect { dj.payload_object.job_id }.to raise_error(NameError, 'uninitialized constant MissingJobClass')
+        if ActiveJob.gem_version >= Gem::Version.new('8.1')
+          expect { dj.payload_object.perform_now }.to raise_error(ActiveJob::UnknownJobClassError)
+        else
+          expect { dj.payload_object.job_id }.to raise_error(NameError, 'uninitialized constant MissingJobClass')
+        end
       end
       expect(Delayed::Worker.new.work_off).to eq([0, 1])
-      expect(Delayed::Job.last.last_error).to match(/uninitialized constant MissingJobClass/)
+      if ActiveJob.gem_version >= Gem::Version.new('8.1')
+        expect(Delayed::Job.last.last_error).to match(/`MissingJobClass` doesn't exist/)
+      else
+        expect(Delayed::Job.last.last_error).to match(/uninitialized constant MissingJobClass/)
+      end
     end
 
     it 'deserializes even if an underlying argument gid is not defined' do
@@ -153,6 +161,12 @@ RSpec.describe Delayed::ActiveJobAdapter do
       expect(enqueued_delayed_jobs.last.priority).to eq(20)
     end
 
+    it 'ignores a nil priority, applying the default instead' do
+      JobClass.set(priority: nil).perform_later
+
+      expect(enqueued_delayed_jobs.last.priority).to eq(10)
+    end
+
     it 'raises an error when run_at is used' do
       expect { JobClass.set(run_at: arbitrary_time).perform_later }
         .to raise_error(/`:run_at` is not supported./)
@@ -196,6 +210,39 @@ RSpec.describe Delayed::ActiveJobAdapter do
       end
     end
 
+    context 'when using the ActiveJob test adapter' do
+      let(:queue_adapter) { :test }
+
+      it 'raises an error when run_at is used' do
+        expect { JobClass.set(run_at: arbitrary_time).perform_later }
+          .to raise_error(/`:run_at` is not supported./)
+      end
+
+      it 'supports priority as a Delayed::Priority' do
+        JobClass.set(priority: Delayed::Priority.eventual).perform_later
+
+        expect(JobClass.queue_adapter.enqueued_jobs.first).to include(job: JobClass, 'priority' => 20)
+      end
+
+      it 'supports priority as a symbol' do
+        JobClass.set(priority: :eventual).perform_later
+
+        expect(JobClass.queue_adapter.enqueued_jobs.first).to include(job: JobClass, 'priority' => 20)
+      end
+
+      it 'ignores a nil priority, applying the default instead' do
+        JobClass.set(priority: nil).perform_later
+
+        expect(JobClass.queue_adapter.enqueued_jobs.first).to include(job: JobClass, 'priority' => nil)
+      end
+
+      it 'captures arbitrary provider attributes without interfering with enqueue' do
+        JobClass.set(foo: 'bar').perform_later
+
+        expect(JobClass.queue_adapter.enqueued_jobs.first).to include(job: JobClass, queue: 'default')
+      end
+    end
+
     context 'when the ActiveJob performable defines a max_attempts' do
       let(:job_class) do
         Class.new(ActiveJob::Base) do # rubocop:disable Rails/ApplicationJob
@@ -437,6 +484,263 @@ RSpec.describe Delayed::ActiveJobAdapter do
     end
   end
 
+  describe 'ActiveJob .retry_on' do
+    let(:retry_job_class) do
+      Class.new(ActiveJob::Base) do # rubocop:disable Rails/ApplicationJob
+        retry_on(RetryTestError)
+        retry_on(RetryTestErrorWithSpecificPriority, priority: 123)
+
+        queue_with_priority 567
+
+        def perform(error_class_name)
+          raise error_class_name.constantize
+        end
+      end
+    end
+
+    before do
+      allow(Delayed::Job).to receive(:enqueue_job).and_call_original
+      allow(Delayed::Job).to receive(:enqueue_all).and_call_original
+
+      stub_const('RetryTestError', Class.new(StandardError))
+      stub_const('RetryTestErrorWithSpecificPriority', Class.new(StandardError))
+      stub_const('MyRetryJob', retry_job_class)
+    end
+
+    context 'when retry_on does not specify a priority' do
+      it 're-enqueues a new delayed job with the same priority' do
+        MyRetryJob.perform_later('RetryTestError')
+        original = Delayed::Job.last
+
+        expect(Delayed::Worker.new.work_off).to eq([1, 0])
+
+        retried = Delayed::Job.last
+        expect(retried.id).not_to eq(original.id)
+        expect(retried.priority).to eq(567)
+
+        expect(retried.payload_object.job_data['priority']).to be_an(Integer)
+        expect(retried.payload_object.job_data['priority']).to eq(567)
+      end
+
+      it 'reuses a priority and queue set for the specific job run' do
+        MyRetryJob.set(priority: 789, queue: 'fake_queue').perform_later('RetryTestError')
+
+        expect(Delayed::Worker.new.work_off).to eq([1, 0])
+
+        retried = Delayed::Job.last
+        expect(retried.priority).to eq(789)
+        expect(retried.queue).to eq('fake_queue')
+      end
+
+      it 're-enqueues with the current priority of the job row, in case it was updated after enqueue' do
+        MyRetryJob.perform_later('RetryTestError')
+        Delayed::Job.last.update!(priority: 5)
+
+        expect(Delayed::Worker.new.work_off).to eq([1, 0])
+
+        expect(Delayed::Job.last.priority).to eq(5)
+      end
+    end
+
+    context 'when retry_on specifies a priority' do
+      it 're-enqueues with the specified priority' do
+        MyRetryJob.perform_later('RetryTestErrorWithSpecificPriority')
+
+        expect(Delayed::Worker.new.work_off).to eq([1, 0])
+
+        expect(Delayed::Job.last.priority).to eq(123)
+      end
+    end
+
+    it 'records the error that triggered the retry on the re-enqueued job' do
+      MyRetryJob.perform_later('RetryTestError')
+
+      expect(Delayed::Worker.new.work_off).to eq([1, 0])
+
+      expect(Delayed::Job.last.last_error).to start_with('RetryTestError')
+    end
+
+    it 'preserves the original creation time on the re-enqueued job' do
+      Timecop.freeze(arbitrary_time) do
+        MyRetryJob.perform_later('RetryTestError')
+      end
+
+      Timecop.freeze(arbitrary_time + 2.hours) do
+        expect(Delayed::Worker.new.work_off).to eq([1, 0])
+      end
+
+      retried = Delayed::Job.last
+      expect(retried.created_at).to eq(arbitrary_time)
+      expect(retried.run_at).to be_within(1.second).of(arbitrary_time + 2.hours + 3.seconds)
+    end
+
+    context 'when attempts are exhausted' do
+      it 'marks the ActiveJob as terminated and fails the job immediately, halting the worker retry backstop' do
+        job = MyRetryJob.new('RetryTestError')
+        job.exception_executions = { '[RetryTestError]' => 4 } # retry_on defaults to 5 attempts
+        job.enqueue
+        original = Delayed::Job.last
+
+        expect(Delayed::Worker.new.work_off).to eq([0, 1])
+
+        expect(Delayed::Job.count).to eq(1)
+        Delayed::Job.last.tap do |dj|
+          expect(dj.id).to eq(original.id)
+          expect(dj.attempts).to eq(1)
+          expect(dj.failed_at).to be_present
+          expect(dj.last_error).to match(/RetryTestError/)
+          expect(dj.payload_object.job_data['terminated_at']).to be_present
+          expect(dj.payload_object.job_data['exception_executions']).to eq('[RetryTestError]' => 5)
+        end
+      end
+
+      it 'resets execution attempts if (and only if) the row attempts is set back to 0' do
+        MyRetryJob.new('RetryTestError').tap do |job|
+          job.executions = 4
+          job.exception_executions = { '[RetryTestError]' => 4 }
+          job.enqueue
+        end
+
+        expect(Delayed::Worker.new.work_off).to eq([0, 1])
+
+        Delayed::Job.last.tap do |dj|
+          expect(dj.failed_at).to be_present
+          expect(dj.payload_object.job_data['terminated_at']).to be_present
+          expect(dj.payload_object.job_data['executions']).to eq(4)
+          expect(dj.payload_object.job_data['exception_executions']).to eq('[RetryTestError]' => 5)
+        end
+
+        # without setting attempts back to 0:
+        Delayed::Job.last.update!(failed_at: nil, locked_at: nil, locked_by: nil)
+
+        expect(Delayed::Worker.new.work_off).to eq([0, 1])
+
+        retried = Delayed::Job.last
+        expect(retried.failed_at).to be_present
+        expect(retried.attempts).to eq(2)
+        expect(retried.payload_object.job_data['terminated_at']).to be_present
+        expect(retried.payload_object.job_data['executions']).to eq(4)
+        expect(retried.payload_object.job_data['exception_executions']).to eq('[RetryTestError]' => 6)
+
+        # also setting attempts back to 0:
+        Delayed::Job.last.update!(failed_at: nil, attempts: 0, locked_at: nil, locked_by: nil)
+
+        expect(Delayed::Worker.new.work_off).to eq([1, 0])
+
+        retried = Delayed::Job.last
+        expect(retried.failed_at).to be_nil
+        expect(retried.attempts).to eq(0)
+        expect(retried.payload_object.job_data['terminated_at']).to be_nil
+        expect(retried.payload_object.job_data['executions']).to eq(1)
+        expect(retried.payload_object.job_data['exception_executions']).to eq('[RetryTestError]' => 1)
+      end
+    end
+
+    context 'when an error is not covered by any retry_on declaration' do
+      before do
+        stub_const('UncoveredTestError', Class.new(StandardError))
+      end
+
+      it 'marks the ActiveJob as terminated but leaves the job to be retried by the worker itself' do
+        MyRetryJob.perform_later('UncoveredTestError')
+
+        expect(Delayed::Worker.new.work_off).to eq([0, 1])
+
+        expect(Delayed::Job.count).to eq(1)
+        Delayed::Job.last.tap do |dj|
+          expect(dj.attempts).to eq(1)
+          expect(dj.failed_at).to be_nil
+          expect(dj.last_error).to match(/UncoveredTestError/)
+          expect(dj.payload_object.job_data['terminated_at']).to be_present
+        end
+      end
+    end
+
+    if ActiveJob.gem_version.release >= Gem::Version.new('7.1')
+      context 'when attempts is :unlimited' do
+        let(:retry_job_class) do
+          Class.new(ActiveJob::Base) do # rubocop:disable Rails/ApplicationJob
+            retry_on(RetryTestError, attempts: :unlimited)
+
+            queue_with_priority 567
+
+            def perform(error_class_name)
+              raise error_class_name.constantize
+            end
+          end
+        end
+
+        it 're-enqueues the job even when executions far exceed the default attempt limit' do
+          job = MyRetryJob.new('RetryTestError')
+          job.exception_executions = { '[RetryTestError]' => 10_000 }
+          job.enqueue
+
+          expect(Delayed::Worker.new.work_off).to eq([1, 0])
+
+          retried = Delayed::Job.last
+          expect(retried.payload_object.job_data['exception_executions']).to eq('[RetryTestError]' => 10_001)
+        end
+      end
+    end
+
+    context 'when using the ActiveJob test adapter' do
+      let(:queue_adapter) { :test }
+
+      it 're-enqueues with the priority specified by retry_on' do
+        ActiveJob::Base.execute(MyRetryJob.new('RetryTestErrorWithSpecificPriority').serialize)
+
+        expect(MyRetryJob.queue_adapter.enqueued_jobs.first).to include(job: MyRetryJob, 'priority' => 123)
+      end
+
+      it 're-enqueues with the original priority when retry_on does not specify one' do
+        ActiveJob::Base.execute(MyRetryJob.new('RetryTestError').serialize)
+
+        expect(MyRetryJob.queue_adapter.enqueued_jobs.first).to include(job: MyRetryJob, 'priority' => 567)
+      end
+    end
+  end
+
+  describe 'legacy job hooks' do
+    let(:job_class) do
+      Class.new(ActiveJob::Base) do # rubocop:disable Rails/ApplicationJob
+        cattr_accessor(:messages) { [] }
+
+        def perform
+          self.class.messages << 'perform'
+        end
+
+        def before(_delayed_job)
+          self.class.messages << 'before'
+        end
+      end
+    end
+
+    it 'invokes hooks defined on the job class, with a deprecation warning' do
+      JobClass.perform_later
+      delayed_job = enqueued_delayed_jobs.last
+
+      expect { delayed_job.invoke_job }
+        .to output(/\[DEPRECATION\] Job hook methods .* are deprecated\. Use ActiveJob callbacks instead\./).to_stderr
+
+      expect(JobClass.messages).to eq(%w(before perform))
+    end
+
+    context 'when the job class does not define any hook methods' do
+      let(:job_class) do
+        Class.new(ActiveJob::Base) do # rubocop:disable Rails/ApplicationJob
+          def perform; end
+        end
+      end
+
+      it 'does not emit a deprecation warning' do
+        JobClass.perform_later
+        delayed_job = enqueued_delayed_jobs.last
+
+        expect { delayed_job.invoke_job }.not_to output.to_stderr
+      end
+    end
+  end
+
   describe '.enqueue_all' do # rubocop:disable Metrics/BlockLength
     let(:adapter) { ActiveJob::Base.queue_adapter }
 

data/spec/delayed/job_spec.rb

@@ -199,6 +199,29 @@ describe Delayed::Job do
       end
     end
 
+    context 'when payload is an ActiveJob wrapper built from previously-serialized job data' do
+      let(:arbitrary_time) { Time.parse('2021-01-05 03:34:33 UTC') }
+
+      it "preserves the wrapped job's enqueued_at as created_at" do
+        job_data = Timecop.freeze(arbitrary_time) { ActiveJobJob.new.serialize }
+
+        job = described_class.enqueue(Delayed::JobWrapper.new(job_data))
+
+        expect(job.reload.created_at).to eq(arbitrary_time)
+      end
+
+      if ActiveJob.gem_version.release >= Gem::Version.new('7.1')
+        it "preserves the wrapped job's scheduled_at as run_at" do
+          active_job = ActiveJobJob.new
+          active_job.scheduled_at = arbitrary_time + 1.hour
+
+          job = described_class.enqueue(Delayed::JobWrapper.new(active_job.serialize))
+
+          expect(job.run_at).to eq(arbitrary_time + 1.hour)
+        end
+      end
+    end
+
     context 'when payload is a bare ActiveJob::Base instance' do
       it 'raises' do
         expect { described_class.enqueue(payload_object: ActiveJobJob.new) }.to raise_error(RuntimeError, /Delayed::Job enqueue methods do not accept ActiveJobs/)
@@ -290,6 +313,26 @@ describe Delayed::Job do
         expect { described_class.enqueue_all(jobs) }.to raise_error(RuntimeError, /Delayed::Job enqueue methods do not accept ActiveJobs/)
       end
     end
+
+    it 'stamps created_at and updated_at' do
+      now = described_class.db_time_now
+      described_class.enqueue_all([build_job])
+
+      described_class.last.tap do |job|
+        expect(job.created_at).to be_within(1).of(now)
+        expect(job.updated_at).to be_within(1).of(now)
+      end
+    end
+
+    it 'preserves a pre-assigned created_at while still stamping updated_at' do
+      original_time = described_class.db_time_now - 3.hours
+      described_class.enqueue_all([build_job(SimpleJob.new, created_at: original_time)])
+
+      described_class.last.tap do |job|
+        expect(job.created_at).to be_within(1).of(original_time)
+        expect(job.updated_at).to be_within(1).of(described_class.db_time_now)
+      end
+    end
   end
 
   describe '#hook' do
@@ -306,11 +349,28 @@ describe Delayed::Job do
         job = described_class.new(payload_object: JobWithEnqueueHook.new)
         expect(job.payload_object).not_to receive(:enqueue)
         expect { job.hook(:enqueue) }
-          .to raise_error(RuntimeError, ':enqueue hook is no longer supported')
+          .to raise_error(ArgumentError, 'Unknown hook: :enqueue')
       end
     end
   end
 
+  describe '#invoke_job' do
+    it "does not clobber a priority attribute on the object receiving a delayed message" do
+      stub_const('PrioritizedThing', Class.new do
+        attr_accessor :priority
+
+        def check_in; end
+      end)
+      thing = PrioritizedThing.new
+      thing.priority = :domain_specific_value
+
+      job = thing.delay(priority: 3).check_in
+      job.invoke_job
+
+      expect(thing.priority).to eq(:domain_specific_value)
+    end
+  end
+
   describe 'callbacks' do
     before(:each) do
       CallbackJob.messages = []
@@ -1023,7 +1083,7 @@ describe Delayed::Job do
         worker.work_off
         @job.reload
         expect(@job.last_error).to match(/did not work/)
-        expect(@job.last_error).to match(/sample_jobs.rb:\d+:in `perform'/)
+        expect(@job.last_error).to match(/sample_jobs.rb:\d+:in (`|'ErrorJob#)perform'/)
         expect(@job.attempts).to eq(1)
         expect(@job.run_at).to be > described_class.db_time_now - 10.minutes
         expect(@job.run_at).to be < described_class.db_time_now + 10.minutes