delayed 3.0.1 → 4.0.0

data/lib/delayed/priority.rb

@@ -70,10 +70,12 @@ module Delayed
       def names=(names)
         raise "must include a name for priority >= 0" if names && !names.value?(0)
 
+        remove_named_priority_methods
         @ranges = nil
         @alerts = nil
         @names_to_priority = nil
         @names = names&.sort_by(&:last)&.to_h&.transform_values { |v| new(v) }
+        define_named_priority_methods
       end
 
       def alerts=(alerts)
@@ -124,17 +126,35 @@ module Delayed
         low + ((high - low).to_d / 2).ceil
       end
 
-      def respond_to_missing?(method_name, include_private = false)
-        names_to_priority.key?(method_name) || super
+      # Defines a class method (e.g. `Priority.eventual`) and an instance predicate (e.g.
+      # `priority.eventual?`) per name, as real, introspectable methods (visible to `methods`,
+      # RDoc, and Sorbet's tapioca, unlike `method_missing` dispatch). Pre-existing methods
+      # (e.g. a name of `superclass`) are never overridden.
+      def define_named_priority_methods
+        names.each_key do |name|
+          predicate = :"#{name}?"
+          define_singleton_method(name) { names_to_priority.fetch(name) } unless method_defined_on?(singleton_class, name)
+          define_method(predicate) { name.to_s == to_s } unless method_defined_on?(self, predicate)
+        end
       end
 
-      def method_missing(method_name, *args)
-        if names_to_priority.key?(method_name) && args.none?
-          names_to_priority[method_name]
-        else
-          super
+      # Removes the methods previously defined for the current `names` (called before `names` is
+      # reassigned). The non-inherited (`false`) checks mean we only remove methods we defined
+      # ourselves, leaving any pre-existing method a colliding name never overrode (e.g.
+      # `Class#superclass`, `Numeric#zero?`) in place.
+      def remove_named_priority_methods
+        names.each_key do |name|
+          predicate = :"#{name}?"
+          singleton_class.send(:remove_method, name) if singleton_class.method_defined?(name, false)
+          remove_method(predicate) if method_defined?(predicate, false)
         end
       end
+
+      # Whether `mod` already responds to `name` (public, protected, private, or inherited), in
+      # which case a same-named priority must not override it.
+      def method_defined_on?(mod, name)
+        mod.method_defined?(name) || mod.private_method_defined?(name)
+      end
     end
 
     attr_reader :value
@@ -188,18 +208,6 @@ module Delayed
       to_i.to_d
     end
 
-    private
-
-    def respond_to_missing?(method_name, include_private = false)
-      (method_name.to_s.end_with?('?') && self.class.names.key?(method_name.to_s[0..-2].to_sym)) || super
-    end
-
-    def method_missing(method_name, *args)
-      if method_name.to_s.end_with?('?') && self.class.names.key?(method_name.to_s[0..-2].to_sym)
-        method_name.to_s[0..-2] == to_s
-      else
-        super
-      end
-    end
+    send(:define_named_priority_methods)
   end
 end

data/lib/delayed/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Delayed
-  VERSION = '3.0.1'
+  VERSION = '4.0.0'
 end

data/lib/delayed.rb

@@ -16,12 +16,14 @@ require 'delayed/backend/job_preparer'
 require 'delayed/helpers/migration'
 require 'delayed/worker'
 require 'delayed/job_wrapper'
+require 'delayed/limitable'
 
 if defined?(Rails::Engine)
   require 'delayed/engine'
 else
   require 'active_record'
   require_relative '../app/models/delayed/job'
+  require_relative '../app/models/delayed/limit'
 end
 
 ActiveSupport.on_load(:active_job) do
@@ -29,6 +31,7 @@ ActiveSupport.on_load(:active_job) do
   ActiveJob::QueueAdapters::DelayedAdapter = Class.new(Delayed::ActiveJobAdapter)
 
   include Delayed::ActiveJobAdapter::EnqueuingPatch
+  include Delayed::Limitable
 end
 
 ActiveSupport.on_load(:action_mailer) do

data/spec/delayed/active_job_adapter_spec.rb

@@ -113,10 +113,18 @@ RSpec.describe Delayed::ActiveJobAdapter do
       Delayed::Job.last.tap do |dj|
         dj.update!(handler: dj.handler.gsub('JobClass', 'MissingJobClass'))
         expect { dj.payload_object }.not_to raise_error
-        expect { dj.payload_object.job_id }.to raise_error(NameError, 'uninitialized constant MissingJobClass')
+        if ActiveJob.gem_version >= Gem::Version.new('8.1')
+          expect { dj.payload_object.perform_now }.to raise_error(ActiveJob::UnknownJobClassError)
+        else
+          expect { dj.payload_object.job_id }.to raise_error(NameError, 'uninitialized constant MissingJobClass')
+        end
       end
       expect(Delayed::Worker.new.work_off).to eq([0, 1])
-      expect(Delayed::Job.last.last_error).to match(/uninitialized constant MissingJobClass/)
+      if ActiveJob.gem_version >= Gem::Version.new('8.1')
+        expect(Delayed::Job.last.last_error).to match(/`MissingJobClass` doesn't exist/)
+      else
+        expect(Delayed::Job.last.last_error).to match(/uninitialized constant MissingJobClass/)
+      end
     end
 
     it 'deserializes even if an underlying argument gid is not defined' do
@@ -153,6 +161,12 @@ RSpec.describe Delayed::ActiveJobAdapter do
       expect(enqueued_delayed_jobs.last.priority).to eq(20)
     end
 
+    it 'ignores a nil priority, applying the default instead' do
+      JobClass.set(priority: nil).perform_later
+
+      expect(enqueued_delayed_jobs.last.priority).to eq(10)
+    end
+
     it 'raises an error when run_at is used' do
       expect { JobClass.set(run_at: arbitrary_time).perform_later }
         .to raise_error(/`:run_at` is not supported./)
@@ -196,6 +210,39 @@ RSpec.describe Delayed::ActiveJobAdapter do
       end
     end
 
+    context 'when using the ActiveJob test adapter' do
+      let(:queue_adapter) { :test }
+
+      it 'raises an error when run_at is used' do
+        expect { JobClass.set(run_at: arbitrary_time).perform_later }
+          .to raise_error(/`:run_at` is not supported./)
+      end
+
+      it 'supports priority as a Delayed::Priority' do
+        JobClass.set(priority: Delayed::Priority.eventual).perform_later
+
+        expect(JobClass.queue_adapter.enqueued_jobs.first).to include(job: JobClass, 'priority' => 20)
+      end
+
+      it 'supports priority as a symbol' do
+        JobClass.set(priority: :eventual).perform_later
+
+        expect(JobClass.queue_adapter.enqueued_jobs.first).to include(job: JobClass, 'priority' => 20)
+      end
+
+      it 'ignores a nil priority, applying the default instead' do
+        JobClass.set(priority: nil).perform_later
+
+        expect(JobClass.queue_adapter.enqueued_jobs.first).to include(job: JobClass, 'priority' => nil)
+      end
+
+      it 'captures arbitrary provider attributes without interfering with enqueue' do
+        JobClass.set(foo: 'bar').perform_later
+
+        expect(JobClass.queue_adapter.enqueued_jobs.first).to include(job: JobClass, queue: 'default')
+      end
+    end
+
     context 'when the ActiveJob performable defines a max_attempts' do
       let(:job_class) do
         Class.new(ActiveJob::Base) do # rubocop:disable Rails/ApplicationJob
@@ -437,6 +484,263 @@ RSpec.describe Delayed::ActiveJobAdapter do
     end
   end
 
+  describe 'ActiveJob .retry_on' do
+    let(:retry_job_class) do
+      Class.new(ActiveJob::Base) do # rubocop:disable Rails/ApplicationJob
+        retry_on(RetryTestError)
+        retry_on(RetryTestErrorWithSpecificPriority, priority: 123)
+
+        queue_with_priority 567
+
+        def perform(error_class_name)
+          raise error_class_name.constantize
+        end
+      end
+    end
+
+    before do
+      allow(Delayed::Job).to receive(:enqueue_job).and_call_original
+      allow(Delayed::Job).to receive(:enqueue_all).and_call_original
+
+      stub_const('RetryTestError', Class.new(StandardError))
+      stub_const('RetryTestErrorWithSpecificPriority', Class.new(StandardError))
+      stub_const('MyRetryJob', retry_job_class)
+    end
+
+    context 'when retry_on does not specify a priority' do
+      it 're-enqueues a new delayed job with the same priority' do
+        MyRetryJob.perform_later('RetryTestError')
+        original = Delayed::Job.last
+
+        expect(Delayed::Worker.new.work_off).to eq([1, 0])
+
+        retried = Delayed::Job.last
+        expect(retried.id).not_to eq(original.id)
+        expect(retried.priority).to eq(567)
+
+        expect(retried.payload_object.job_data['priority']).to be_an(Integer)
+        expect(retried.payload_object.job_data['priority']).to eq(567)
+      end
+
+      it 'reuses a priority and queue set for the specific job run' do
+        MyRetryJob.set(priority: 789, queue: 'fake_queue').perform_later('RetryTestError')
+
+        expect(Delayed::Worker.new.work_off).to eq([1, 0])
+
+        retried = Delayed::Job.last
+        expect(retried.priority).to eq(789)
+        expect(retried.queue).to eq('fake_queue')
+      end
+
+      it 're-enqueues with the current priority of the job row, in case it was updated after enqueue' do
+        MyRetryJob.perform_later('RetryTestError')
+        Delayed::Job.last.update!(priority: 5)
+
+        expect(Delayed::Worker.new.work_off).to eq([1, 0])
+
+        expect(Delayed::Job.last.priority).to eq(5)
+      end
+    end
+
+    context 'when retry_on specifies a priority' do
+      it 're-enqueues with the specified priority' do
+        MyRetryJob.perform_later('RetryTestErrorWithSpecificPriority')
+
+        expect(Delayed::Worker.new.work_off).to eq([1, 0])
+
+        expect(Delayed::Job.last.priority).to eq(123)
+      end
+    end
+
+    it 'records the error that triggered the retry on the re-enqueued job' do
+      MyRetryJob.perform_later('RetryTestError')
+
+      expect(Delayed::Worker.new.work_off).to eq([1, 0])
+
+      expect(Delayed::Job.last.last_error).to start_with('RetryTestError')
+    end
+
+    it 'preserves the original creation time on the re-enqueued job' do
+      Timecop.freeze(arbitrary_time) do
+        MyRetryJob.perform_later('RetryTestError')
+      end
+
+      Timecop.freeze(arbitrary_time + 2.hours) do
+        expect(Delayed::Worker.new.work_off).to eq([1, 0])
+      end
+
+      retried = Delayed::Job.last
+      expect(retried.created_at).to eq(arbitrary_time)
+      expect(retried.run_at).to be_within(1.second).of(arbitrary_time + 2.hours + 3.seconds)
+    end
+
+    context 'when attempts are exhausted' do
+      it 'marks the ActiveJob as terminated and fails the job immediately, halting the worker retry backstop' do
+        job = MyRetryJob.new('RetryTestError')
+        job.exception_executions = { '[RetryTestError]' => 4 } # retry_on defaults to 5 attempts
+        job.enqueue
+        original = Delayed::Job.last
+
+        expect(Delayed::Worker.new.work_off).to eq([0, 1])
+
+        expect(Delayed::Job.count).to eq(1)
+        Delayed::Job.last.tap do |dj|
+          expect(dj.id).to eq(original.id)
+          expect(dj.attempts).to eq(1)
+          expect(dj.failed_at).to be_present
+          expect(dj.last_error).to match(/RetryTestError/)
+          expect(dj.payload_object.job_data['terminated_at']).to be_present
+          expect(dj.payload_object.job_data['exception_executions']).to eq('[RetryTestError]' => 5)
+        end
+      end
+
+      it 'resets execution attempts if (and only if) the row attempts is set back to 0' do
+        MyRetryJob.new('RetryTestError').tap do |job|
+          job.executions = 4
+          job.exception_executions = { '[RetryTestError]' => 4 }
+          job.enqueue
+        end
+
+        expect(Delayed::Worker.new.work_off).to eq([0, 1])
+
+        Delayed::Job.last.tap do |dj|
+          expect(dj.failed_at).to be_present
+          expect(dj.payload_object.job_data['terminated_at']).to be_present
+          expect(dj.payload_object.job_data['executions']).to eq(4)
+          expect(dj.payload_object.job_data['exception_executions']).to eq('[RetryTestError]' => 5)
+        end
+
+        # without setting attempts back to 0:
+        Delayed::Job.last.update!(failed_at: nil, locked_at: nil, locked_by: nil)
+
+        expect(Delayed::Worker.new.work_off).to eq([0, 1])
+
+        retried = Delayed::Job.last
+        expect(retried.failed_at).to be_present
+        expect(retried.attempts).to eq(2)
+        expect(retried.payload_object.job_data['terminated_at']).to be_present
+        expect(retried.payload_object.job_data['executions']).to eq(4)
+        expect(retried.payload_object.job_data['exception_executions']).to eq('[RetryTestError]' => 6)
+
+        # also setting attempts back to 0:
+        Delayed::Job.last.update!(failed_at: nil, attempts: 0, locked_at: nil, locked_by: nil)
+
+        expect(Delayed::Worker.new.work_off).to eq([1, 0])
+
+        retried = Delayed::Job.last
+        expect(retried.failed_at).to be_nil
+        expect(retried.attempts).to eq(0)
+        expect(retried.payload_object.job_data['terminated_at']).to be_nil
+        expect(retried.payload_object.job_data['executions']).to eq(1)
+        expect(retried.payload_object.job_data['exception_executions']).to eq('[RetryTestError]' => 1)
+      end
+    end
+
+    context 'when an error is not covered by any retry_on declaration' do
+      before do
+        stub_const('UncoveredTestError', Class.new(StandardError))
+      end
+
+      it 'marks the ActiveJob as terminated but leaves the job to be retried by the worker itself' do
+        MyRetryJob.perform_later('UncoveredTestError')
+
+        expect(Delayed::Worker.new.work_off).to eq([0, 1])
+
+        expect(Delayed::Job.count).to eq(1)
+        Delayed::Job.last.tap do |dj|
+          expect(dj.attempts).to eq(1)
+          expect(dj.failed_at).to be_nil
+          expect(dj.last_error).to match(/UncoveredTestError/)
+          expect(dj.payload_object.job_data['terminated_at']).to be_present
+        end
+      end
+    end
+
+    if ActiveJob.gem_version.release >= Gem::Version.new('7.1')
+      context 'when attempts is :unlimited' do
+        let(:retry_job_class) do
+          Class.new(ActiveJob::Base) do # rubocop:disable Rails/ApplicationJob
+            retry_on(RetryTestError, attempts: :unlimited)
+
+            queue_with_priority 567
+
+            def perform(error_class_name)
+              raise error_class_name.constantize
+            end
+          end
+        end
+
+        it 're-enqueues the job even when executions far exceed the default attempt limit' do
+          job = MyRetryJob.new('RetryTestError')
+          job.exception_executions = { '[RetryTestError]' => 10_000 }
+          job.enqueue
+
+          expect(Delayed::Worker.new.work_off).to eq([1, 0])
+
+          retried = Delayed::Job.last
+          expect(retried.payload_object.job_data['exception_executions']).to eq('[RetryTestError]' => 10_001)
+        end
+      end
+    end
+
+    context 'when using the ActiveJob test adapter' do
+      let(:queue_adapter) { :test }
+
+      it 're-enqueues with the priority specified by retry_on' do
+        ActiveJob::Base.execute(MyRetryJob.new('RetryTestErrorWithSpecificPriority').serialize)
+
+        expect(MyRetryJob.queue_adapter.enqueued_jobs.first).to include(job: MyRetryJob, 'priority' => 123)
+      end
+
+      it 're-enqueues with the original priority when retry_on does not specify one' do
+        ActiveJob::Base.execute(MyRetryJob.new('RetryTestError').serialize)
+
+        expect(MyRetryJob.queue_adapter.enqueued_jobs.first).to include(job: MyRetryJob, 'priority' => 567)
+      end
+    end
+  end
+
+  describe 'legacy job hooks' do
+    let(:job_class) do
+      Class.new(ActiveJob::Base) do # rubocop:disable Rails/ApplicationJob
+        cattr_accessor(:messages) { [] }
+
+        def perform
+          self.class.messages << 'perform'
+        end
+
+        def before(_delayed_job)
+          self.class.messages << 'before'
+        end
+      end
+    end
+
+    it 'invokes hooks defined on the job class, with a deprecation warning' do
+      JobClass.perform_later
+      delayed_job = enqueued_delayed_jobs.last
+
+      expect { delayed_job.invoke_job }
+        .to output(/\[DEPRECATION\] Job hook methods .* are deprecated\. Use ActiveJob callbacks instead\./).to_stderr
+
+      expect(JobClass.messages).to eq(%w(before perform))
+    end
+
+    context 'when the job class does not define any hook methods' do
+      let(:job_class) do
+        Class.new(ActiveJob::Base) do # rubocop:disable Rails/ApplicationJob
+          def perform; end
+        end
+      end
+
+      it 'does not emit a deprecation warning' do
+        JobClass.perform_later
+        delayed_job = enqueued_delayed_jobs.last
+
+        expect { delayed_job.invoke_job }.not_to output.to_stderr
+      end
+    end
+  end
+
   describe '.enqueue_all' do # rubocop:disable Metrics/BlockLength
     let(:adapter) { ActiveJob::Base.queue_adapter }
 

data/spec/delayed/job_spec.rb

@@ -199,6 +199,29 @@ describe Delayed::Job do
       end
     end
 
+    context 'when payload is an ActiveJob wrapper built from previously-serialized job data' do
+      let(:arbitrary_time) { Time.parse('2021-01-05 03:34:33 UTC') }
+
+      it "preserves the wrapped job's enqueued_at as created_at" do
+        job_data = Timecop.freeze(arbitrary_time) { ActiveJobJob.new.serialize }
+
+        job = described_class.enqueue(Delayed::JobWrapper.new(job_data))
+
+        expect(job.reload.created_at).to eq(arbitrary_time)
+      end
+
+      if ActiveJob.gem_version.release >= Gem::Version.new('7.1')
+        it "preserves the wrapped job's scheduled_at as run_at" do
+          active_job = ActiveJobJob.new
+          active_job.scheduled_at = arbitrary_time + 1.hour
+
+          job = described_class.enqueue(Delayed::JobWrapper.new(active_job.serialize))
+
+          expect(job.run_at).to eq(arbitrary_time + 1.hour)
+        end
+      end
+    end
+
     context 'when payload is a bare ActiveJob::Base instance' do
       it 'raises' do
         expect { described_class.enqueue(payload_object: ActiveJobJob.new) }.to raise_error(RuntimeError, /Delayed::Job enqueue methods do not accept ActiveJobs/)
@@ -290,6 +313,26 @@ describe Delayed::Job do
         expect { described_class.enqueue_all(jobs) }.to raise_error(RuntimeError, /Delayed::Job enqueue methods do not accept ActiveJobs/)
       end
     end
+
+    it 'stamps created_at and updated_at' do
+      now = described_class.db_time_now
+      described_class.enqueue_all([build_job])
+
+      described_class.last.tap do |job|
+        expect(job.created_at).to be_within(1).of(now)
+        expect(job.updated_at).to be_within(1).of(now)
+      end
+    end
+
+    it 'preserves a pre-assigned created_at while still stamping updated_at' do
+      original_time = described_class.db_time_now - 3.hours
+      described_class.enqueue_all([build_job(SimpleJob.new, created_at: original_time)])
+
+      described_class.last.tap do |job|
+        expect(job.created_at).to be_within(1).of(original_time)
+        expect(job.updated_at).to be_within(1).of(described_class.db_time_now)
+      end
+    end
   end
 
   describe '#hook' do
@@ -306,11 +349,28 @@ describe Delayed::Job do
         job = described_class.new(payload_object: JobWithEnqueueHook.new)
         expect(job.payload_object).not_to receive(:enqueue)
         expect { job.hook(:enqueue) }
-          .to raise_error(RuntimeError, ':enqueue hook is no longer supported')
+          .to raise_error(ArgumentError, 'Unknown hook: :enqueue')
       end
     end
   end
 
+  describe '#invoke_job' do
+    it "does not clobber a priority attribute on the object receiving a delayed message" do
+      stub_const('PrioritizedThing', Class.new do
+        attr_accessor :priority
+
+        def check_in; end
+      end)
+      thing = PrioritizedThing.new
+      thing.priority = :domain_specific_value
+
+      job = thing.delay(priority: 3).check_in
+      job.invoke_job
+
+      expect(thing.priority).to eq(:domain_specific_value)
+    end
+  end
+
   describe 'callbacks' do
     before(:each) do
       CallbackJob.messages = []
@@ -1023,7 +1083,7 @@ describe Delayed::Job do
         worker.work_off
         @job.reload
         expect(@job.last_error).to match(/did not work/)
-        expect(@job.last_error).to match(/sample_jobs.rb:\d+:in `perform'/)
+        expect(@job.last_error).to match(/sample_jobs.rb:\d+:in (`|'ErrorJob#)perform'/)
         expect(@job.attempts).to eq(1)
         expect(@job.run_at).to be > described_class.db_time_now - 10.minutes
         expect(@job.run_at).to be < described_class.db_time_now + 10.minutes