inst-jobs 1.0.2 → 2.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6d65c567b52e2baf26955d266c9fededc001deab65d871e6e60a42a905d552e8
-  data.tar.gz: b4d487088aba583188c90650a42a20726f84230e346c321b4b94ddc5c25fe36e
+  metadata.gz: 67a4882eeb37fac31a1fd646e0a5a414a3292d7379c243d02b0ede8f9f5f5d89
+  data.tar.gz: 5e92f514ea3a561f8e55541a5a7a8b5802909d3ddaf1ea9ec217d2c3c0a6eff9
 SHA512:
-  metadata.gz: 9ade674610c5a1a4a04e4a6078b45c38358e7c7af79a417d5e6941d968e1565bc5e217405ad1ca427b373ee0d89bdbc28772f8ed52047394e8760fc09a79b60a
-  data.tar.gz: 216706e128397e84ba87d495e4b09d30eb58f35ab4821e8ef6f484d5f6226a959cab04a8bfdfa22916f19dad53e7ee01bbe1128bd61fb402aad29885ac81839f
+  metadata.gz: d8183e8becaf288a52dec2757b16632fe57c0c05d2f98c3edd9bb13cf732744498c309b0edf6598006fd0914214ae6cf098b965ad64887763524c23616d2ea74
+  data.tar.gz: 81650de01fdd02fbb4fcf81b112dc5cfefd4291a439ac8c5aab5faf424c5780b57845c3fe8f69992ed419638289ef0b4f5cab0e6a830d34ba04af160c21139e3

data/lib/delayed/backend/base.rb

@@ -178,6 +178,10 @@ module Delayed
         expires_at && (self.class.db_time_now >= expires_at)
       end
 
+      def inferred_max_attempts
+        self.max_attempts || Delayed::Settings.max_attempts
+      end
+
       # Reschedule the job in the future (when a job fails).
       # Uses an exponential scale depending on the number of failed attempts.
       def reschedule(error = nil, time = nil)
@@ -190,7 +194,7 @@ module Delayed
 
         self.attempts += 1 unless return_code == :unlock
 
-        if self.attempts >= (self.max_attempts || Delayed::Settings.max_attempts)
+        if self.attempts >= self.inferred_max_attempts
           permanent_failure error || "max attempts reached"
         elsif expired?
           permanent_failure error || "job has expired"

data/lib/delayed/lifecycle.rb

@@ -12,6 +12,7 @@ module Delayed
       :loop             => [:worker],
       :perform          => [:worker, :job],
       :pop              => [:worker],
+      :retry            => [:worker, :job, :exception],
       :work_queue_pop   => [:work_queue, :worker_config],
       :check_for_work   => [:work_queue],
     }

data/lib/delayed/performable_method.rb

@@ -13,7 +13,7 @@ module Delayed
       self.permanent_fail_cb = on_permanent_failure
       self.sender = sender
       begin
-        YAML.dump(sender)
+        YAML.load(YAML.dump(sender))
       rescue
         # if for some reason you can't dump the sender, just drop it
         self.sender = nil
@@ -73,7 +73,8 @@ module Delayed
 
     def full_name
       obj_name = object.is_a?(ActiveRecord::Base) ? "#{object.class}.find(#{object.id}).#{method}" : display_name
-      kwargs_str = kwargs.map { |(k, v)| ", #{k}: #{deep_de_ar_ize(v)}"}.join("")
+      kgs = kwargs || {}
+      kwargs_str = kgs.map { |(k, v)| ", #{k}: #{deep_de_ar_ize(v)}"}.join("")
       "#{obj_name}(#{args.map { |a| deep_de_ar_ize(a) }.join(', ')}#{kwargs_str})"
     end
   end

data/lib/delayed/periodic.rb

@@ -49,10 +49,20 @@ class Periodic
   end
 
   def enqueue
-    Delayed::Job.enqueue(self, **@job_args.merge(:max_attempts => 1,
-                                                 :run_at => @cron.next_time(Delayed::Periodic.now).utc.to_time,
-                                                 :singleton => tag,
-                                                 on_conflict: :patient))
+    Delayed::Job.enqueue(self, **enqueue_args)
+  end
+
+  def enqueue_args
+    inferred_args = {
+      max_attempts: 1,
+      run_at: @cron.next_time(Delayed::Periodic.now).utc.to_time,
+      singleton: (@job_args[:singleton] == false ? nil : tag),
+      # yes, checking for whether it is actually the boolean literal false,
+      # which means the consuming code really does not want this job to be
+      # a singleton at all.
+      on_conflict: :patient
+    }
+    @job_args.merge(inferred_args)
   end
 
   def perform

data/lib/delayed/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Delayed
-  VERSION = "1.0.2"
+  VERSION = "2.1.1"
 end

data/lib/delayed/worker.rb

@@ -3,6 +3,17 @@
 module Delayed
 
 class TimeoutError < RuntimeError; end
+class RetriableError < RuntimeError
+  # this error is a special case.  You _should_ raise
+  # it from inside the rescue block for another error, 
+  # because it indicates: "something made this job fail
+  # but we're pretty sure it's transient and it's safe to try again".
+  # the workflow is still the same (retry will happen unless
+  # retries are exhausted), but it won't call the :error
+  # callback unless it can't retry anymore.  It WILL call the
+  # separate ":retry" callback, which is ONLY activated
+  # for this kind of error.
+end
 
 require 'tmpdir'
 require 'set'
@@ -198,32 +209,38 @@ class Worker
   end
 
   def perform(job)
-    count = 1
-    raise Delayed::Backend::JobExpired, "job expired at #{job.expires_at}" if job.expired?
-    self.class.lifecycle.run_callbacks(:perform, self, job) do
-      set_process_name("run:#{Settings.worker_procname_prefix}#{job.id}:#{job.name}")
-      logger.info("Processing #{log_job(job, :long)}")
-      runtime = Benchmark.realtime do
-        if job.batch?
-          # each job in the batch will have perform called on it, so we don't
-          # need a timeout around this
-          count = perform_batch(job)
-        else
-          job.invoke_job
+    begin
+      count = 1
+      raise Delayed::Backend::JobExpired, "job expired at #{job.expires_at}" if job.expired?
+      self.class.lifecycle.run_callbacks(:perform, self, job) do
+        set_process_name("run:#{Settings.worker_procname_prefix}#{job.id}:#{job.name}")
+        logger.info("Processing #{log_job(job, :long)}")
+        runtime = Benchmark.realtime do
+          if job.batch?
+            # each job in the batch will have perform called on it, so we don't
+            # need a timeout around this
+            count = perform_batch(job)
+          else
+            job.invoke_job
+          end
+          job.destroy
         end
-        job.destroy
+        logger.info("Completed #{log_job(job)} #{"%.0fms" % (runtime * 1000)}")
+      end
+    rescue ::Delayed::RetriableError => re
+      can_retry = job.attempts + 1 < job.inferred_max_attempts
+      callback_type = can_retry ? :retry : :error
+      self.class.lifecycle.run_callbacks(callback_type, self, job, re) do
+        handle_failed_job(job, re)
+      end
+    rescue SystemExit => se
+      # There wasn't really a failure here so no callbacks and whatnot needed,
+      # still reschedule the job though.
+      job.reschedule(se)
+    rescue Exception => e
+      self.class.lifecycle.run_callbacks(:error, self, job, e) do
+        handle_failed_job(job, e)
       end
-      logger.info("Completed #{log_job(job)} #{"%.0fms" % (runtime * 1000)}")
-    end
-    count
-  rescue SystemExit => se
-    # There wasn't really a failure here so no callbacks and whatnot needed,
-    # still reschedule the job though.
-    job.reschedule(se)
-    count
-  rescue Exception => e
-    self.class.lifecycle.run_callbacks(:error, self, job, e) do
-      handle_failed_job(job, e)
     end
     count
   end

data/lib/delayed/worker/health_check.rb

@@ -22,31 +22,46 @@ module Delayed
 
         def reschedule_abandoned_jobs
           return if Settings.worker_health_check_type == :none
+          Delayed::Job.transaction do
+            # this job is a special case, and is not a singleton
+            # because if it gets wiped out suddenly during execution
+            # it can't go clean up it's abandoned self.  Therefore,
+            # we try to get an advisory lock when it runs.  If we succeed,
+            # no other job is trying to do this right now (and if we abandon the
+            # job, the transaction will end, releasing the advisory lock).
+            result = attempt_advisory_lock
+            return unless result
+            checker = Worker::HealthCheck.build(
+              type: Settings.worker_health_check_type,
+              config: Settings.worker_health_check_config,
+              worker_name: 'cleanup-crew'
+            )
+            live_workers = checker.live_workers
 
-          checker = Worker::HealthCheck.build(
-            type: Settings.worker_health_check_type,
-            config: Settings.worker_health_check_config,
-            worker_name: 'cleanup-crew'
-          )
-          live_workers = checker.live_workers
-
-          Delayed::Job.running_jobs.each do |job|
-            # prefetched jobs have their own way of automatically unlocking themselves
-            next if job.locked_by.start_with?("prefetch:")
-            unless live_workers.include?(job.locked_by)
-              begin
-                Delayed::Job.transaction do
-                  # double check that the job is still there. locked_by will immediately be reset
-                  # to nil in this transaction by Job#reschedule
-                  next unless Delayed::Job.where(id: job, locked_by: job.locked_by).update_all(locked_by: "abandoned job cleanup") == 1
-                  job.reschedule
+            Delayed::Job.running_jobs.each do |job|
+              # prefetched jobs have their own way of automatically unlocking themselves
+              next if job.locked_by.start_with?("prefetch:")
+              unless live_workers.include?(job.locked_by)
+                begin
+                  Delayed::Job.transaction do
+                    # double check that the job is still there. locked_by will immediately be reset
+                    # to nil in this transaction by Job#reschedule
+                    next unless Delayed::Job.where(id: job, locked_by: job.locked_by).update_all(locked_by: "abandoned job cleanup") == 1
+                    job.reschedule
+                  end
+                rescue
+                  ::Rails.logger.error "Failure rescheduling abandoned job #{job.id} #{$!.inspect}"
                 end
-              rescue
-                ::Rails.logger.error "Failure rescheduling abandoned job #{job.id} #{$!.inspect}"
               end
             end
           end
         end
+
+        def attempt_advisory_lock
+          lock_name = "Delayed::Worker::HealthCheck#reschedule_abandoned_jobs"
+          output = ActiveRecord::Base.connection.execute("SELECT pg_try_advisory_xact_lock(half_md5_as_bigint('#{lock_name}'));")
+          output.getvalue(0, 0)
+        end
       end
 
       attr_accessor :config, :worker_name

data/spec/delayed/periodic_spec.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Delayed::Periodic do
+  around(:each) do |block|
+    # make sure we can use ".cron" and
+    # such safely without leaking global state
+    prev_sched = Delayed::Periodic.scheduled
+    prev_ovr = Delayed::Periodic.overrides
+    Delayed::Periodic.scheduled = {}
+    Delayed::Periodic.overrides = {}
+    block.call
+  ensure
+    Delayed::Periodic.scheduled = prev_sched
+    Delayed::Periodic.overrides = prev_ovr
+  end
+
+  describe ".cron" do
+    let(:job_name){ 'just a test'}
+    it "provides a tag by default for periodic jobs" do
+      Delayed::Periodic.cron job_name, '*/10 * * * *' do
+        # no-op
+      end
+      instance = Delayed::Periodic.scheduled[job_name]
+      expect(instance).to_not be_nil
+      expect(instance.enqueue_args[:singleton]).to eq("periodic: just a test")
+    end
+
+    it "uses no singleton if told to skip" do
+      Delayed::Periodic.cron job_name, '*/10 * * * *', {singleton: false} do
+        # no-op
+      end
+      instance = Delayed::Periodic.scheduled[job_name]
+      expect(instance).to_not be_nil
+      expect(instance.enqueue_args[:singleton]).to be_nil
+    end
+  end
+end

data/spec/delayed/worker/health_check_spec.rb

@@ -107,6 +107,15 @@ RSpec.describe Delayed::Worker::HealthCheck do
       @dead_job.reload
       expect(@dead_job.locked_by).to eq 'prefetch:some_node'
     end
+
+    it "bails immediately if advisory lock already taken" do
+      allow(Delayed::Worker::HealthCheck).to receive(:attempt_advisory_lock).and_return(false)
+      Delayed::Worker::HealthCheck.reschedule_abandoned_jobs
+      @dead_job.reload
+      expect(@dead_job.run_at.to_i).to eq(initial_run_at.to_i)
+      expect(@dead_job.locked_at).to_not be_nil
+      expect(@dead_job.locked_by).to_not be_nil
+    end
   end
 
   describe '#initialize' do

data/spec/delayed/worker_spec.rb

@@ -6,6 +6,11 @@ describe Delayed::Worker do
   let(:worker_config) { {
       queue: "test", min_priority: 1, max_priority: 2, stuff: "stuff",
   }.freeze }
+  let(:job_attrs) { {
+    id: 42, name: "testjob", full_name: "testfullname", :last_error= => nil,
+    attempts: 1, reschedule: nil, :expired? => false,
+    payload_object: {}, priority: 25
+  }.freeze }
   subject { described_class.new(worker_config.dup) }
 
   after { Delayed::Worker.lifecycle.reset! }
@@ -14,9 +19,24 @@ describe Delayed::Worker do
     it "fires off an error callback when a job raises an exception" do
       fired = false
       Delayed::Worker.lifecycle.before(:error) {|worker, exception| fired = true}
-      job = double(:last_error= => nil, attempts: 1, reschedule: nil)
-      subject.perform(job)
+      job = double(job_attrs)
+      output_count = subject.perform(job)
       expect(fired).to be_truthy
+      expect(output_count).to eq(1)
+    end
+
+    it "uses the retry callback for a retriable exception" do
+      error_fired = retry_fired = false
+      Delayed::Worker.lifecycle.before(:error) {|worker, exception| error_fired = true }
+      Delayed::Worker.lifecycle.before(:retry) {|worker, exception| retry_fired = true}
+      job = Delayed::Job.new(payload_object: {}, priority: 25, strand: "test_jobs", max_attempts: 3)
+      expect(job).to receive(:invoke_job) do
+        raise Delayed::RetriableError, "that's all this job does"
+      end
+      output_count = subject.perform(job)
+      expect(error_fired).to be_falsey
+      expect(retry_fired).to be_truthy
+      expect(output_count).to eq(1)
     end
 
     it "reloads" do
@@ -35,7 +55,7 @@ describe Delayed::Worker do
         expect(ActionDispatch::Reloader).to receive(:prepare!).once
         expect(ActionDispatch::Reloader).to receive(:cleanup!).once
       end
-      job = double(:last_error= => nil, attempts: 0, reschedule: nil, expired?: false)
+      job = double(job_attrs)
       subject.perform(job)
     end
   end

data/spec/shared/performable_method.rb

@@ -65,4 +65,10 @@ shared_examples_for 'Delayed::PerformableMethod' do
     p.send(:on_permanent_failure, 'fail_frd')
     story.text.should == 'fail_frd'
   end
+
+  it "can still generate a name with no kwargs" do
+    story = Story.create :text => 'wat'
+    p = Delayed::PerformableMethod.new(story, :tell, kwargs: nil)
+    expect(p.full_name).to eq("Story.find(#{story.id}).tell()")
+  end
 end

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: inst-jobs
 version: !ruby/object:Gem::Version
-  version: 1.0.2
+  version: 2.1.1
 platform: ruby
 authors:
 - Tobias Luetke
 - Brian Palmer
-autorequire:
+autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-10-30 00:00:00.000000000 Z
+date: 2020-12-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -325,7 +325,7 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 1.4.0
-description:
+description: 
 email:
 - brianp@instructure.com
 executables:
@@ -412,6 +412,7 @@ files:
 - spec/delayed/cli_spec.rb
 - spec/delayed/daemon_spec.rb
 - spec/delayed/message_sending_spec.rb
+- spec/delayed/periodic_spec.rb
 - spec/delayed/server_spec.rb
 - spec/delayed/settings_spec.rb
 - spec/delayed/work_queue/in_process_spec.rb
@@ -422,15 +423,10 @@ files:
 - spec/delayed/worker/health_check_spec.rb
 - spec/delayed/worker_spec.rb
 - spec/gemfiles/42.gemfile
-- spec/gemfiles/42.gemfile.lock
 - spec/gemfiles/50.gemfile
-- spec/gemfiles/50.gemfile.lock
 - spec/gemfiles/51.gemfile
-- spec/gemfiles/51.gemfile.lock
 - spec/gemfiles/52.gemfile
-- spec/gemfiles/52.gemfile.lock
 - spec/gemfiles/60.gemfile
-- spec/gemfiles/60.gemfile.lock
 - spec/migrate/20140924140513_add_story_table.rb
 - spec/redis_job_spec.rb
 - spec/sample_jobs.rb
@@ -445,7 +441,7 @@ files:
 homepage: https://github.com/instructure/inst-jobs
 licenses: []
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -453,30 +449,25 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.3'
+      version: '2.6'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
-signing_key:
+rubygems_version: 3.0.3
+signing_key: 
 specification_version: 4
 summary: Instructure-maintained fork of delayed_job
 test_files:
 - spec/sample_jobs.rb
 - spec/spec_helper.rb
 - spec/redis_job_spec.rb
-- spec/gemfiles/51.gemfile.lock
-- spec/gemfiles/60.gemfile.lock
-- spec/gemfiles/42.gemfile.lock
-- spec/gemfiles/50.gemfile.lock
 - spec/gemfiles/60.gemfile
 - spec/gemfiles/42.gemfile
 - spec/gemfiles/52.gemfile
 - spec/gemfiles/50.gemfile
-- spec/gemfiles/52.gemfile.lock
 - spec/gemfiles/51.gemfile
 - spec/shared_jobs_specs.rb
 - spec/shared/performable_method.rb
@@ -490,6 +481,7 @@ test_files:
 - spec/delayed/cli_spec.rb
 - spec/delayed/daemon_spec.rb
 - spec/delayed/worker_spec.rb
+- spec/delayed/periodic_spec.rb
 - spec/delayed/message_sending_spec.rb
 - spec/delayed/settings_spec.rb
 - spec/delayed/work_queue/in_process_spec.rb