inst-jobs 1.0.1 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2dad83034c6ca9ae9a7f7129581ecd36f64d1d51f93aa1126582ff2141068020
-  data.tar.gz: 8b155f2ed6ce13b956d8383bc484ef19ed3b661502bf2aed734a4618befebf40
+  metadata.gz: b789a390da84dd32ede51b6d8655853e3f20efa55144fe19cf0605d9e5083e62
+  data.tar.gz: bd263366443698e93572f83a6f0e2b3d747c9e72acd851fbc0aa3f920c413e02
 SHA512:
-  metadata.gz: 783af3d9f654e07a55a6be57691216a255bb548a720ae31d7c085ba0c70be20164a2a2c6a7b36767976468e034b5bafc6de427f577a05c2c70d0a6621301e86d
-  data.tar.gz: cbc4ff85dea6bc2f12d8eb2cecb25cecae4dc47e5e5155e46c63ebdbc4884e15f43f41b69b8611c44cbb42f4539843ecffe8cf5e96cab852bf9ace06a7a6b6fa
+  metadata.gz: b902bbdb3d4d676b984a22c2df5d4382960219de7a9a3d639df4fd6f958816ac77a3e2e00e881d7afbc240fd6043eef21f5362885301ccf1c9fb1ae790a21e2f
+  data.tar.gz: 3dbb3fe818c20be76f6f07f61893014bc881a2f82a7c44139e62642230ff49a2acd0d3e2ba746b4bcdf1834fed0b5cb1ebcbb6ea3431b9fe46ffc29d6956200b

data/lib/delayed/backend/base.rb

@@ -178,6 +178,10 @@ module Delayed
         expires_at && (self.class.db_time_now >= expires_at)
       end
 
+      def inferred_max_attempts
+        self.max_attempts || Delayed::Settings.max_attempts
+      end
+
       # Reschedule the job in the future (when a job fails).
       # Uses an exponential scale depending on the number of failed attempts.
       def reschedule(error = nil, time = nil)
@@ -190,7 +194,7 @@ module Delayed
 
         self.attempts += 1 unless return_code == :unlock
 
-        if self.attempts >= (self.max_attempts || Delayed::Settings.max_attempts)
+        if self.attempts >= self.inferred_max_attempts
           permanent_failure error || "max attempts reached"
         elsif expired?
           permanent_failure error || "job has expired"

data/lib/delayed/lifecycle.rb

@@ -12,6 +12,7 @@ module Delayed
       :loop             => [:worker],
       :perform          => [:worker, :job],
       :pop              => [:worker],
+      :retry            => [:worker, :job, :exception],
       :work_queue_pop   => [:work_queue, :worker_config],
       :check_for_work   => [:work_queue],
     }

data/lib/delayed/message_sending.rb

@@ -7,32 +7,45 @@ end
 module Delayed
   module MessageSending
     class DelayProxy < BasicObject
-      def initialize(object, synchronous: false, public_send: false, **enqueue_args)
+      def initialize(object, synchronous: false, sender: nil, **enqueue_args)
         @object = object
         @enqueue_args = enqueue_args
         @synchronous = synchronous
-        @public_send = public_send
+        @sender = sender
       end
 
       def method_missing(method, *args, **kwargs)
+        # method doesn't exist? must be method_missing; assume private access
+        @sender = nil if !@sender.nil? &&
+         !@object.methods.include?(method) &&
+         !@object.protected_methods.include?(method) &&
+         !@object.private_methods.include?(method)
+
+        sender_is_object = @sender == @object
+        sender_is_class = @sender.is_a?(@object.class)
+
+        # even if the call is async, if the call is _going_ to generate an error, we make it synchronous
+        # so that the error is generated immediately, instead of waiting for it to fail in a job,
+        # which might go unnoticed
+        if !@sender.nil? && !@synchronous
+          @synchronous = true if !sender_is_object && @object.private_methods.include?(method)
+          @synchronous = true if !sender_is_class && @object.protected_methods.include?(method)
+        end
+
         if @synchronous
-          if @public_send
-            if kwargs.empty?
-              return @object.public_send(method, *args)
-            else
-              return @object.public_send(method, *args, **kwargs)
-            end
-          else
+          if @sender.nil? || sender_is_object || sender_is_class && @object.protected_methods.include?(method)
             if kwargs.empty?
               return @object.send(method, *args)
             else
               return @object.send(method, *args, **kwargs)
             end
           end
-        end
-
-        if @public_send && @object.private_methods.include?(method)
-          ::Kernel.raise ::NoMethodError.new("undefined method `#{method}' for #{@object}", method)
+          
+          if kwargs.empty?
+            return @object.public_send(method, *args)
+          else
+            return @object.public_send(method, *args, **kwargs)
+          end
         end
 
         ignore_transaction = @enqueue_args.delete(:ignore_transaction)
@@ -50,7 +63,8 @@ module Delayed
               ::Delayed::Job.enqueue(::Delayed::PerformableMethod.new(@object, method,
                                                                   args: args, kwargs: kwargs,
                                                                   on_failure: on_failure,
-                                                                  on_permanent_failure: on_permanent_failure),
+                                                                  on_permanent_failure: on_permanent_failure,
+                                                                  sender: @sender),
                                                                   **@enqueue_args)
             end
             return nil
@@ -61,14 +75,15 @@ module Delayed
                                                                     args: args,
                                                                     kwargs: kwargs,
                                                                     on_failure: on_failure,
-                                                                    on_permanent_failure: on_permanent_failure),
+                                                                    on_permanent_failure: on_permanent_failure,
+                                                                    sender: @sender),
                                                                     **@enqueue_args)
         result = nil unless ignore_transaction
         result
       end
     end
 
-    def delay(public_send: nil, **enqueue_args)
+    def delay(sender: nil, **enqueue_args)
       # support procs/methods as enqueue arguments
       enqueue_args.each do |k,v|
         if v.respond_to?(:call)
@@ -76,21 +91,15 @@ module Delayed
         end
       end
 
-      public_send ||= __calculate_public_send_for_delay
+      sender ||= __calculate_sender_for_delay
 
-      DelayProxy.new(self, public_send: public_send, **enqueue_args)
+      DelayProxy.new(self, sender: sender, **enqueue_args)
     end
 
-    def __calculate_public_send_for_delay
+    def __calculate_sender_for_delay
       # enforce public send in dev and test, but not prod (since it uses
       # debug APIs, it's expensive)
-      public_send = if ::Rails.env.test? || ::Rails.env.development?
-        sender = self.sender(1)
-        # if the caller isn't self, use public_send; i.e. enforce method visibility
-        sender != self
-      else
-        false
-      end
+      return sender(1) if ::Rails.env.test? || ::Rails.env.development?
     end
 
     module ClassMethods
@@ -114,7 +123,7 @@ module Delayed
               if synchronous
                 super(*args, **kwargs)
               else
-                delay(**enqueue_args).method_missing(method_name, *args, synchronous: true, **kwargs)
+                delay(sender: __calculate_sender_for_delay, **enqueue_args).method_missing(method_name, *args, synchronous: true, **kwargs)
               end
             end)
           end

data/lib/delayed/performable_method.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
 module Delayed
-  class PerformableMethod < Struct.new(:object, :method, :args, :kwargs, :fail_cb, :permanent_fail_cb, :public_send)
-    def initialize(object, method, args: [], kwargs: {}, on_failure: nil, on_permanent_failure: nil, public_send: true)
+  class PerformableMethod < Struct.new(:object, :method, :args, :kwargs, :fail_cb, :permanent_fail_cb, :sender)
+    def initialize(object, method, args: [], kwargs: {}, on_failure: nil, on_permanent_failure: nil, sender: nil)
       raise NoMethodError, "undefined method `#{method}' for #{object.inspect}" unless object.respond_to?(method, true)
 
       self.object = object
@@ -11,7 +11,13 @@ module Delayed
       self.method = method.to_sym
       self.fail_cb           = on_failure
       self.permanent_fail_cb = on_permanent_failure
-      self.public_send = public_send
+      self.sender = sender
+      begin
+        YAML.load(YAML.dump(sender))
+      rescue
+        # if for some reason you can't dump the sender, just drop it
+        self.sender = nil
+      end
     end
 
     def display_name
@@ -25,17 +31,21 @@ module Delayed
 
     def perform
       kwargs = self.kwargs || {}
-      if public_send
+
+      sender_is_object = sender == object
+      sender_is_class = sender.is_a?(object.class)
+
+      if sender.nil? || sender_is_object || sender_is_class && object.protected_methods.include?(method)
         if kwargs.empty?
-          object.public_send(method, *args)
+          object.send(method, *args)
         else
-          object.public_send(method, *args, **kwargs)
+          object.send(method, *args, **kwargs)
         end
       else
         if kwargs.empty?
-          object.send(method, *args)
+          object.public_send(method, *args)
         else
-          object.send(method, *args, **kwargs)
+          object.public_send(method, *args, **kwargs)
         end
       end
     end
@@ -63,7 +73,8 @@ module Delayed
 
     def full_name
       obj_name = object.is_a?(ActiveRecord::Base) ? "#{object.class}.find(#{object.id}).#{method}" : display_name
-      kwargs_str = kwargs.map { |(k, v)| ", #{k}: #{deep_de_ar_ize(v)}"}.join("")
+      kgs = kwargs || {}
+      kwargs_str = kgs.map { |(k, v)| ", #{k}: #{deep_de_ar_ize(v)}"}.join("")
       "#{obj_name}(#{args.map { |a| deep_de_ar_ize(a) }.join(', ')}#{kwargs_str})"
     end
   end

data/lib/delayed/periodic.rb

@@ -49,10 +49,20 @@ class Periodic
   end
 
   def enqueue
-    Delayed::Job.enqueue(self, **@job_args.merge(:max_attempts => 1,
-                                                 :run_at => @cron.next_time(Delayed::Periodic.now).utc.to_time,
-                                                 :singleton => tag,
-                                                 on_conflict: :patient))
+    Delayed::Job.enqueue(self, **enqueue_args)
+  end
+
+  def enqueue_args
+    inferred_args = {
+      max_attempts: 1,
+      run_at: @cron.next_time(Delayed::Periodic.now).utc.to_time,
+      singleton: (@job_args[:singleton] == false ? nil : tag),
+      # yes, checking for whether it is actually the boolean literal false,
+      # which means the consuming code really does not want this job to be
+      # a singleton at all.
+      on_conflict: :patient
+    }
+    @job_args.merge(inferred_args)
   end
 
   def perform

data/lib/delayed/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Delayed
-  VERSION = "1.0.1"
+  VERSION = "2.1.0"
 end

data/lib/delayed/worker.rb

@@ -3,6 +3,17 @@
 module Delayed
 
 class TimeoutError < RuntimeError; end
+class RetriableError < RuntimeError
+  # this error is a special case.  You _should_ raise
+  # it from inside the rescue block for another error, 
+  # because it indicates: "something made this job fail
+  # but we're pretty sure it's transient and it's safe to try again".
+  # the workflow is still the same (retry will happen unless
+  # retries are exhausted), but it won't call the :error
+  # callback unless it can't retry anymore.  It WILL call the
+  # separate ":retry" callback, which is ONLY activated
+  # for this kind of error.
+end
 
 require 'tmpdir'
 require 'set'
@@ -216,6 +227,12 @@ class Worker
       logger.info("Completed #{log_job(job)} #{"%.0fms" % (runtime * 1000)}")
     end
     count
+  rescue ::Delayed::RetriableError => re
+    can_retry = job.attempts + 1 < job.inferred_max_attempts
+    callback_type = can_retry ? :retry : :error
+    self.class.lifecycle.run_callbacks(callback_type, self, job, re) do
+      handle_failed_job(job, re)
+    end
   rescue SystemExit => se
     # There wasn't really a failure here so no callbacks and whatnot needed,
     # still reschedule the job though.

data/lib/delayed/worker/health_check.rb

@@ -22,31 +22,46 @@ module Delayed
 
         def reschedule_abandoned_jobs
           return if Settings.worker_health_check_type == :none
+          Delayed::Job.transaction do
+            # this job is a special case, and is not a singleton
+            # because if it gets wiped out suddenly during execution
+            # it can't go clean up it's abandoned self.  Therefore,
+            # we try to get an advisory lock when it runs.  If we succeed,
+            # no other job is trying to do this right now (and if we abandon the
+            # job, the transaction will end, releasing the advisory lock).
+            result = attempt_advisory_lock
+            return unless result
+            checker = Worker::HealthCheck.build(
+              type: Settings.worker_health_check_type,
+              config: Settings.worker_health_check_config,
+              worker_name: 'cleanup-crew'
+            )
+            live_workers = checker.live_workers
 
-          checker = Worker::HealthCheck.build(
-            type: Settings.worker_health_check_type,
-            config: Settings.worker_health_check_config,
-            worker_name: 'cleanup-crew'
-          )
-          live_workers = checker.live_workers
-
-          Delayed::Job.running_jobs.each do |job|
-            # prefetched jobs have their own way of automatically unlocking themselves
-            next if job.locked_by.start_with?("prefetch:")
-            unless live_workers.include?(job.locked_by)
-              begin
-                Delayed::Job.transaction do
-                  # double check that the job is still there. locked_by will immediately be reset
-                  # to nil in this transaction by Job#reschedule
-                  next unless Delayed::Job.where(id: job, locked_by: job.locked_by).update_all(locked_by: "abandoned job cleanup") == 1
-                  job.reschedule
+            Delayed::Job.running_jobs.each do |job|
+              # prefetched jobs have their own way of automatically unlocking themselves
+              next if job.locked_by.start_with?("prefetch:")
+              unless live_workers.include?(job.locked_by)
+                begin
+                  Delayed::Job.transaction do
+                    # double check that the job is still there. locked_by will immediately be reset
+                    # to nil in this transaction by Job#reschedule
+                    next unless Delayed::Job.where(id: job, locked_by: job.locked_by).update_all(locked_by: "abandoned job cleanup") == 1
+                    job.reschedule
+                  end
+                rescue
+                  ::Rails.logger.error "Failure rescheduling abandoned job #{job.id} #{$!.inspect}"
                 end
-              rescue
-                ::Rails.logger.error "Failure rescheduling abandoned job #{job.id} #{$!.inspect}"
               end
             end
           end
         end
+
+        def attempt_advisory_lock
+          lock_name = "Delayed::Worker::HealthCheck#reschedule_abandoned_jobs"
+          output = ActiveRecord::Base.connection.execute("SELECT pg_try_advisory_xact_lock(half_md5_as_bigint('#{lock_name}'));")
+          output.getvalue(0, 0)
+        end
       end
 
       attr_accessor :config, :worker_name

data/spec/delayed/message_sending_spec.rb

@@ -8,21 +8,38 @@ RSpec.describe Delayed::MessageSending do
     allow(::Rails.env).to receive(:test?).and_return(true)
   end
 
-  let(:klass) do
-    Class.new do
+  before (:all) do
+    class SpecClass
       def call_private(**enqueue_args)
         delay(**enqueue_args).private_method
       end
 
+      def call_protected(**enqueue_args)
+        other = self.class.new
+        other.delay(**enqueue_args).protected_method
+      end
+
       private
 
       def private_method
       end
+
+      protected
+
+      def protected_method
+      end
     end
   end
 
+  after(:all) do
+    Object.send(:remove_const, :SpecClass)
+  end
+
+  let(:klass) { SpecClass }
+
   it "allows an object to send a private message to itself" do
-    klass.new.call_private
+    job = klass.new.call_private(ignore_transaction: true)
+    job.invoke_job
   end
 
   it "allows an object to send a private message to itself synchronouosly" do
@@ -48,4 +65,37 @@ RSpec.describe Delayed::MessageSending do
     allow(::Rails.env).to receive(:development?).and_return(false)
     klass.new.delay(synchronous: true).private_method
   end
+
+  it "allows an object to send a protected message to itself" do
+    job = klass.new.call_protected(ignore_transaction: true)
+    job.invoke_job
+  end
+
+  it "allows an object to send a protected message to itself synchronouosly" do
+    klass.new.call_protected(synchronous: true)
+  end
+
+  it "warns about directly sending a protected message asynchronously" do
+    expect { klass.new.delay.protected_method }.to raise_error(NoMethodError)
+  end
+
+  it "warns about directly sending a protected message synchronusly" do
+    expect { klass.new.delay(synchronous: true).protected_method }.to raise_error(NoMethodError)
+  end
+
+  it "doesn't explode if you can't dump the sender" do
+    klass = Class.new do
+      def delay_something
+        Kernel.delay.sleep(1)
+      end
+
+      def encode_with(encoder)
+        raise "yaml encoding failed"
+      end
+    end
+
+    obj = klass.new
+    expect { YAML.dump(obj) }.to raise_error("yaml encoding failed")
+    expect { obj.delay_something }.not_to raise_error
+  end
 end

data/spec/delayed/periodic_spec.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Delayed::Periodic do
+  around(:each) do |block|
+    # make sure we can use ".cron" and
+    # such safely without leaking global state
+    prev_sched = Delayed::Periodic.scheduled
+    prev_ovr = Delayed::Periodic.overrides
+    Delayed::Periodic.scheduled = {}
+    Delayed::Periodic.overrides = {}
+    block.call
+  ensure
+    Delayed::Periodic.scheduled = prev_sched
+    Delayed::Periodic.overrides = prev_ovr
+  end
+
+  describe ".cron" do
+    let(:job_name){ 'just a test'}
+    it "provides a tag by default for periodic jobs" do
+      Delayed::Periodic.cron job_name, '*/10 * * * *' do
+        # no-op
+      end
+      instance = Delayed::Periodic.scheduled[job_name]
+      expect(instance).to_not be_nil
+      expect(instance.enqueue_args[:singleton]).to eq("periodic: just a test")
+    end
+
+    it "uses no singleton if told to skip" do
+      Delayed::Periodic.cron job_name, '*/10 * * * *', {singleton: false} do
+        # no-op
+      end
+      instance = Delayed::Periodic.scheduled[job_name]
+      expect(instance).to_not be_nil
+      expect(instance.enqueue_args[:singleton]).to be_nil
+    end
+  end
+end