delayed 0.8.0 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6268c514bb90e5279b8265c4ce21eea8262ac4d539ff161c7f9eab12f15f17ba
-  data.tar.gz: eb0c1f81475163766a680eab603740a4632956fb415e4158430a67bb3998c381
+  metadata.gz: 9e2ec868563930c9d66a6db2eb638559b20497de5a819d543694c8fa4814519e
+  data.tar.gz: b48f6b5951f5c7309ea8db1b693451d4501c6f87852b843e5b1c49a5113cd4ed
 SHA512:
-  metadata.gz: 73686b95d83b336dcf268f4524fb8b512e64f61b58773b0884e8f1a4c30abc89532aaa0df4e1395d69417d2062ae7e948157c7cd5b3b7cb80982c68dbf798936
-  data.tar.gz: 9f3b7437e5772e0fe44b3f168cd32b78073b6c49b2fc208b713d946d4b48f14549646363856e3d62a7525059a7d841fbe0191a520af3dc06a71e3d258679b68e
+  metadata.gz: 17289bfb6ea1de0202570e44351e023a9024395bd2df12121b848e9813fc6eebd57c71abf709c61f7625697fddb097dc22d8bc1c2e0b714996b9e0cbdac7ceee
+  data.tar.gz: 76b59feb415bf66841659c05aeb421a69d59579161004da33b11de4937fe7e30c232da6bab17008c12489cf65186d8c316b371f5c41deb710e5534796b95ba20

data/lib/delayed/job_wrapper.rb

@@ -20,6 +20,24 @@ module Delayed
       job_data['job_class']
     end
 
+    # If job failed to deserialize, we can't respond to delegated methods.
+    # Returning false here prevents instance method checks from blocking job cleanup.
+    # There is a (currently) unreleased Rails PR that changes the exception class in this case:
+    # https://github.com/rails/rails/pull/53770
+    if defined?(ActiveJob::UnknownJobClassError)
+      def respond_to?(*, **)
+        super
+      rescue ActiveJob::UnknownJobClassError
+        false
+      end
+    else
+      def respond_to?(*, **)
+        super
+      rescue NameError
+        false
+      end
+    end
+
     def perform
       ActiveJob::Callbacks.run_callbacks(:execute) do
         job.perform_now

data/lib/delayed/lifecycle.rb

@@ -8,7 +8,7 @@ module Delayed
       perform: %i(worker job),
       error: %i(worker job),
       failure: %i(worker job),
-      thread: %i(worker job),
+      thread: [:worker],
       invoke_job: [:job],
     }.freeze
 

data/lib/delayed/plugins/connection.rb

@@ -2,9 +2,9 @@ module Delayed
   module Plugins
     class Connection < Plugin
       callbacks do |lifecycle|
-        lifecycle.around(:thread) do |worker, job, &block|
+        lifecycle.around(:thread) do |worker, &block|
           Job.connection_pool.with_connection do
-            block.call(worker, job)
+            block.call(worker)
           end
         end
       end

data/lib/delayed/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Delayed
+  VERSION = '1.0.0'
+end

data/lib/delayed/worker.rb

@@ -101,12 +101,15 @@ module Delayed
         pool = Concurrent::FixedThreadPool.new(jobs.length)
         jobs.each do |job|
           pool.post do
-            self.class.lifecycle.run_callbacks(:thread, self, job) do
+            self.class.lifecycle.run_callbacks(:thread, self) do
               success.increment if perform(job)
+            rescue DeserializationError => e
+              handle_unrecoverable_error(job, e)
+            rescue Exception => e # rubocop:disable Lint/RescueException
+              handle_erroring_job(job, e)
             end
           rescue Exception => e # rubocop:disable Lint/RescueException
-            job_say job, "Job thread crashed with #{e.class.name}: #{e.message}", 'error'
-            job.error = e
+            say "Job thread crashed with #{e.class.name}: #{e.message}", 'error'
           end
         end
 
@@ -142,17 +145,14 @@ module Delayed
           job.destroy
         end
         job_say job, format('COMPLETED after %.4f seconds', run_time)
-        true # did work
-      rescue DeserializationError => e
-        job_say job, "FAILED permanently with #{e.class.name}: #{e.message}", 'error'
-
-        job.error = e
-        failed(job)
-        false # work failed
-      rescue Exception => e # rubocop:disable Lint/RescueException
-        self.class.lifecycle.run_callbacks(:error, self, job) { handle_failed_job(job, e) }
-        false # work failed
       end
+      true # did work
+    rescue DeserializationError => e
+      handle_unrecoverable_error(job, e)
+      false # work failed
+    rescue Exception => e # rubocop:disable Lint/RescueException
+      handle_erroring_job(job, e)
+      false # work failed
     end
 
     # Reschedule the job in the future (when a job fails).
@@ -172,12 +172,12 @@ module Delayed
     def failed(job)
       self.class.lifecycle.run_callbacks(:failure, self, job) do
         job.hook(:failure)
-      rescue StandardError => e
-        say "Error when running failure callback: #{e}", 'error'
-        say e.backtrace.join("\n"), 'error'
-      ensure
-        job.destroy_failed_jobs? ? job.destroy : job.fail!
       end
+    rescue StandardError => e
+      say "Error when running failure callback: #{e}", 'error'
+      say e.backtrace.join("\n"), 'error'
+    ensure
+      job.destroy_failed_jobs? ? job.destroy : job.fail!
     end
 
     def job_say(job, text, level = Delayed.default_log_level)
@@ -204,10 +204,21 @@ module Delayed
       " (queue=#{queue})" if queue
     end
 
-    def handle_failed_job(job, error)
+    def handle_erroring_job(job, error)
+      self.class.lifecycle.run_callbacks(:error, self, job) do
+        job.error = error
+        job_say job, "FAILED (#{job.attempts} prior attempts) with #{error.class.name}: #{error.message}", 'error'
+        reschedule(job)
+      end
+    rescue Exception => e # rubocop:disable Lint/RescueException
+      handle_unrecoverable_error(job, e)
+    end
+
+    def handle_unrecoverable_error(job, error)
+      job_say job, "FAILED permanently with #{error.class.name}: #{error.message}", 'error'
+
       job.error = error
-      job_say job, "FAILED (#{job.attempts} prior attempts) with #{error.class.name}: #{error.message}", 'error'
-      reschedule(job)
+      failed(job)
     end
 
     # The backend adapter may return either a list or a single job

data/spec/delayed/active_job_adapter_spec.rb

@@ -62,10 +62,23 @@ RSpec.describe Delayed::ActiveJobAdapter do
     JobClass.perform_later
 
     Delayed::Job.last.tap do |dj|
-      dj.handler = dj.handler.gsub('JobClass', 'MissingJobClass')
+      dj.update!(handler: dj.handler.gsub('JobClass', 'MissingJobClass'))
       expect { dj.payload_object }.not_to raise_error
       expect { dj.payload_object.job_id }.to raise_error(NameError, 'uninitialized constant MissingJobClass')
     end
+    expect(Delayed::Worker.new.work_off).to eq([0, 1])
+    expect(Delayed::Job.last.last_error).to match(/uninitialized constant MissingJobClass/)
+  end
+
+  it 'deserializes even if an underlying argument gid is not defined' do
+    ActiveJobJob.perform_later(story: Story.create!)
+    Delayed::Job.last.tap do |dj|
+      dj.update!(handler: dj.handler.gsub('Story', 'MissingArgumentClass'))
+      expect { dj.payload_object }.not_to raise_error
+      expect { dj.payload_object.perform_now }.to raise_error(ActiveJob::DeserializationError)
+    end
+    expect(Delayed::Worker.new.work_off).to eq([0, 1])
+    expect(Delayed::Job.last.last_error).to match(/Error while trying to deserialize arguments/)
   end
 
   describe '.set' do

data/spec/helper.rb

@@ -10,6 +10,12 @@ require 'sample_jobs'
 
 require 'rake'
 
+ActiveSupport.on_load(:active_record) do
+  require 'global_id/identification'
+  include GlobalID::Identification
+  GlobalID.app = 'test'
+end
+
 if ActiveSupport.gem_version >= Gem::Version.new('7.1')
   frameworks = [ActiveModel, ActiveRecord, ActionMailer, ActiveJob, ActiveSupport]
   frameworks.each { |framework| framework.deprecator.behavior = :raise }
@@ -36,6 +42,7 @@ db_adapter ||= "sqlite3"
 config = YAML.load(ERB.new(File.read("spec/database.yml")).result)
 ActiveRecord::Base.establish_connection config[db_adapter]
 ActiveRecord::Base.logger = Delayed.logger
+ActiveJob::Base.logger = Delayed.logger
 ActiveRecord::Migration.verbose = false
 
 # MySQL 5.7 no longer supports null default values for the primary key
@@ -126,10 +133,7 @@ RSpec.configure do |config|
     read_ahead_was = Delayed::Worker.read_ahead
     sleep_delay_was = Delayed::Worker.sleep_delay
     min_reserve_interval_was = Delayed::Worker.min_reserve_interval
-
-    if Gem.loaded_specs['delayed'].version >= Gem::Version.new('1.0') && min_reserve_interval_was.zero?
-      raise "Min reserve interval should be nonzero in v1.0 release"
-    end
+    plugins_was = Delayed.plugins.dup
 
     Delayed::Worker.sleep_delay = TEST_SLEEP_DELAY
     Delayed::Worker.min_reserve_interval = TEST_MIN_RESERVE_INTERVAL
@@ -151,6 +155,7 @@ RSpec.configure do |config|
     Delayed::Worker.read_ahead = read_ahead_was
     Delayed::Worker.sleep_delay = sleep_delay_was
     Delayed::Worker.min_reserve_interval = min_reserve_interval_was
+    Delayed.plugins = plugins_was
 
     Delayed::Job.delete_all
   end

data/spec/sample_jobs.rb

@@ -113,5 +113,5 @@ class EnqueueJobMod < SimpleJob
 end
 
 class ActiveJobJob < ActiveJob::Base # rubocop:disable Rails/ApplicationJob
-  def perform; end
+  def perform(*args, **kwargs); end
 end

data/spec/worker_spec.rb

@@ -221,53 +221,134 @@ describe Delayed::Worker do
     end
   end
 
-  describe 'thread callback' do
-    it 'wraps code after thread is checked out' do
-      performances = Concurrent::AtomicFixnum.new(0)
-      plugin = Class.new(Delayed::Plugin) do
-        callbacks do |lifecycle|
-          lifecycle.before(:thread) { performances.increment }
+  describe 'lifecycle callbacks' do
+    let(:plugin) do
+      Class.new(Delayed::Plugin) do
+        class << self
+          attr_accessor :last_error, :raise_on
+
+          def events
+            @events ||= []
+          end
         end
-      end
-      Delayed.plugins << plugin
 
-      Delayed::Job.delete_all
-      Delayed::Job.enqueue SimpleJob.new
-      worker = described_class.new
-
-      worker.work_off
-
-      expect(performances.value).to eq(1)
-    end
-
-    it 'wraps perform and cleanup, even when perform raises' do
-      events = []
-      last_error = nil
-
-      plugin = Class.new(Delayed::Plugin) do
         callbacks do |lifecycle|
           lifecycle.around(:thread) do |_, &blk|
             events << :thread_start
             blk.call
+            raise "oh no" if raise_on == :thread
+
             events << :thread_end
           end
-          lifecycle.around(:perform) do |_, job, &blk|
-            events << :perform_start
-            blk.call.tap do
-              last_error = job.last_error
-              events << :perform_end
+
+          %i(perform error failure).each do |event|
+            lifecycle.around(event) do |_, job, &blk|
+              events << :"#{event}_start"
+              raise "oh no" if raise_on == event
+
+              blk.call.tap do
+                self.last_error = job.last_error if event == :error
+                events << :"#{event}_end"
+              end
             end
           end
         end
       end
+    end
 
+    before do
       Delayed.plugins << plugin
+    end
 
-      Delayed::Job.enqueue ErrorJob.new
+    it 'runs thread and perform callbacks' do
+      Delayed::Job.enqueue SimpleJob.new
       described_class.new.work_off
 
-      expect(events).to eq %i(thread_start perform_start perform_end thread_end)
-      expect(last_error).to match(/did not work/) # assert that cleanup happened before `:perform_end`
+      expect(plugin.events).to eq %i(thread_start perform_start perform_end thread_end)
+      expect(plugin.last_error).to eq(nil)
+      expect(Delayed::Job.count).to eq 0
+    end
+
+    context 'when thread callback raises an error' do
+      before do
+        plugin.raise_on = :thread
+      end
+
+      it 'logs that the thread crashed' do
+        Delayed::Job.enqueue SimpleJob.new
+        described_class.new.work_off
+
+        expect(plugin.events).to eq %i(thread_start perform_start perform_end)
+        expect(plugin.last_error).to eq(nil)
+        expect(Delayed::Job.count).to eq 0
+      end
+    end
+
+    context 'when the perform callback raises an error' do
+      before do
+        plugin.raise_on = :perform
+      end
+
+      it 'runs expected perform and error callbacks' do
+        Delayed::Job.enqueue SimpleJob.new
+        described_class.new.work_off
+
+        expect(plugin.events).to eq %i(thread_start perform_start error_start error_end thread_end)
+        expect(plugin.last_error).to match(/oh no/) # assert that cleanup happened before `:perform_end`
+        expect(Delayed::Job.count).to eq 1
+      end
+    end
+
+    context 'when the perform method raises an error' do
+      it 'runs error callbacks' do
+        Delayed::Job.enqueue ErrorJob.new
+        described_class.new.work_off
+
+        expect(plugin.events).to eq %i(thread_start perform_start error_start error_end thread_end)
+        expect(plugin.last_error).to match(/did not work/) # assert that cleanup happened before `:perform_end`
+        expect(Delayed::Job.count).to eq 1
+      end
+
+      context 'when error callback raises an error' do
+        before do
+          plugin.raise_on = :error
+        end
+
+        it 'runs thread and perform callbacks' do
+          Delayed::Job.enqueue SimpleJob.new
+          described_class.new.work_off
+
+          expect(plugin.events).to eq %i(thread_start perform_start perform_end thread_end)
+          expect(plugin.last_error).to eq(nil)
+          expect(Delayed::Job.count).to eq 0
+        end
+      end
+    end
+
+    context 'when max attempts is exceeded' do
+      it 'runs failure callbacks' do
+        Delayed::Job.enqueue FailureJob.new
+        described_class.new.work_off
+
+        expect(plugin.events).to eq %i(thread_start perform_start error_start failure_start failure_end error_end thread_end)
+        expect(plugin.last_error).to match(/did not work/) # assert that cleanup happened before `:perform_end`
+        expect(Delayed::Job.count).to eq 1
+      end
+
+      context 'when failure callback raises an error' do
+        before do
+          plugin.raise_on = :failure
+        end
+
+        it 'runs thread and perform callbacks' do
+          Delayed::Job.enqueue SimpleJob.new
+          described_class.new.work_off
+
+          expect(plugin.events).to eq %i(thread_start perform_start perform_end thread_end)
+          expect(plugin.last_error).to eq(nil)
+          expect(Delayed::Job.count).to eq 0
+        end
+      end
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: delayed
 version: !ruby/object:Gem::Version
-  version: 0.8.0
+  version: 1.0.0
 platform: ruby
 authors:
 - Nathan Griffith
@@ -18,7 +18,7 @@ authors:
 - Tobias Lütke
 bindir: bin
 cert_chain: []
-date: 2025-04-03 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -86,6 +86,7 @@ files:
 - lib/delayed/serialization/active_record.rb
 - lib/delayed/syck_ext.rb
 - lib/delayed/tasks.rb
+- lib/delayed/version.rb
 - lib/delayed/worker.rb
 - lib/delayed/yaml_ext.rb
 - lib/delayed_job.rb
@@ -137,7 +138,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.6
+rubygems_version: 3.6.8
 specification_version: 4
 summary: a multi-threaded, SQL-driven ActiveJob backend used at Betterment to process
   millions of background jobs per day