inst-jobs 1.0.0 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8c754ed71778d5b3f08641e133be947fae204a56dcee71aef08a7780a7d6afa5
-  data.tar.gz: 10949318e78e8e9d56b92ee7f2d14bfd647da20ff70578d894e4d1e85441d2fe
+  metadata.gz: d674b7da21caf04eb87ff9823ed549c93a901219669316090d088f0699564e59
+  data.tar.gz: 021456d34f12eff8cc988db866018d701fc77ffdbb57e9fb308fc1bd25a91ecb
 SHA512:
-  metadata.gz: 1a014e191bc0d6ccd083340b6d5c4bca0b7e03439aca459f79a5a704d93d16c42dd9e4855ca5985f7b1e4fb1fe62eead160cb6cb356c73b00ac297dc30112d8a
-  data.tar.gz: 53997c66017cf6db561104054379cb9e7e934fcc56c54d552b7629ff29eeb58de37c8674088661ba683e7e4d8b19234f26021819753ac06a6c738dd435c3f169
+  metadata.gz: ad78cfdd9026db24b714c532c8ee837a875e443afc375909f0c130e3cfbf87d1f872344f982d931838bfa6649a2f1edc59430f6444a2baee08f8afb568015cfc
+  data.tar.gz: e2b127477f0687958178505628b9544aa5c49e7aa1d0ceef32892250aa26aeb1c77f12bcacd6682e17c2bc379f987b154a0f982e029852432c39f7b3a5335df8

data/lib/delayed/core_ext/kernel.rb

@@ -0,0 +1,9 @@
+module Kernel
+  def sender(i = 0)
+    frame_self = nil
+    # 3. one for the block, one for this method, one for the method calling this
+    # method, and _then_ we get to the self for who sent the message we want
+    RubyVM::DebugInspector.open { |dc| frame_self = dc.frame_self(3 + i) }
+    frame_self
+  end
+end

data/lib/delayed/message_sending.rb

@@ -1,14 +1,53 @@
 # frozen_string_literal: true
 
+if ::Rails.env.test? || ::Rails.env.development?
+  require 'debug_inspector'
+end
+
 module Delayed
   module MessageSending
     class DelayProxy < BasicObject
-      def initialize(object, enqueue_args)
+      def initialize(object, synchronous: false, sender: nil, **enqueue_args)
         @object = object
         @enqueue_args = enqueue_args
+        @synchronous = synchronous
+        @sender = sender
       end
 
       def method_missing(method, *args, **kwargs)
+        # method doesn't exist? must be method_missing; assume private access
+        @sender = nil if !@sender.nil? &&
+         !@object.methods.include?(method) &&
+         !@object.protected_methods.include?(method) &&
+         !@object.private_methods.include?(method)
+
+        sender_is_object = @sender == @object
+        sender_is_class = @sender.is_a?(@object.class)
+
+        # even if the call is async, if the call is _going_ to generate an error, we make it synchronous
+        # so that the error is generated immediately, instead of waiting for it to fail in a job,
+        # which might go unnoticed
+        if !@sender.nil? && !@synchronous
+          @synchronous = true if !sender_is_object && @object.private_methods.include?(method)
+          @synchronous = true if !sender_is_class && @object.protected_methods.include?(method)
+        end
+
+        if @synchronous
+          if @sender.nil? || sender_is_object || sender_is_class && @object.protected_methods.include?(method)
+            if kwargs.empty?
+              return @object.send(method, *args)
+            else
+              return @object.send(method, *args, **kwargs)
+            end
+          end
+          
+          if kwargs.empty?
+            return @object.public_send(method, *args)
+          else
+            return @object.public_send(method, *args, **kwargs)
+          end
+        end
+
         ignore_transaction = @enqueue_args.delete(:ignore_transaction)
         on_failure = @enqueue_args.delete(:on_failure)
         on_permanent_failure = @enqueue_args.delete(:on_permanent_failure)
@@ -24,7 +63,8 @@ module Delayed
               ::Delayed::Job.enqueue(::Delayed::PerformableMethod.new(@object, method,
                                                                   args: args, kwargs: kwargs,
                                                                   on_failure: on_failure,
-                                                                  on_permanent_failure: on_permanent_failure),
+                                                                  on_permanent_failure: on_permanent_failure,
+                                                                  sender: @sender),
                                                                   **@enqueue_args)
             end
             return nil
@@ -35,24 +75,31 @@ module Delayed
                                                                     args: args,
                                                                     kwargs: kwargs,
                                                                     on_failure: on_failure,
-                                                                    on_permanent_failure: on_permanent_failure),
+                                                                    on_permanent_failure: on_permanent_failure,
+                                                                    sender: @sender),
                                                                     **@enqueue_args)
         result = nil unless ignore_transaction
         result
       end
     end
 
-    def delay(**enqueue_args)
+    def delay(sender: nil, **enqueue_args)
       # support procs/methods as enqueue arguments
       enqueue_args.each do |k,v|
         if v.respond_to?(:call)
           enqueue_args[k] = v.call(self)
         end
       end
-      if enqueue_args.delete(:synchronous)
-        return self
-      end
-      DelayProxy.new(self, enqueue_args)
+
+      sender ||= __calculate_sender_for_delay
+
+      DelayProxy.new(self, sender: sender, **enqueue_args)
+    end
+
+    def __calculate_sender_for_delay
+      # enforce public send in dev and test, but not prod (since it uses
+      # debug APIs, it's expensive)
+      return sender(1) if ::Rails.env.test? || ::Rails.env.development?
     end
 
     module ClassMethods
@@ -76,7 +123,7 @@ module Delayed
               if synchronous
                 super(*args, **kwargs)
               else
-                delay(**enqueue_args).method_missing(method_name, *args, synchronous: true, **kwargs)
+                delay(sender: __calculate_sender_for_delay, **enqueue_args).method_missing(method_name, *args, synchronous: true, **kwargs)
               end
             end)
           end

data/lib/delayed/performable_method.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
 module Delayed
-  class PerformableMethod < Struct.new(:object, :method, :args, :kwargs, :fail_cb, :permanent_fail_cb)
-    def initialize(object, method, args: [], kwargs: {}, on_failure: nil, on_permanent_failure: nil)
+  class PerformableMethod < Struct.new(:object, :method, :args, :kwargs, :fail_cb, :permanent_fail_cb, :sender)
+    def initialize(object, method, args: [], kwargs: {}, on_failure: nil, on_permanent_failure: nil, sender: nil)
       raise NoMethodError, "undefined method `#{method}' for #{object.inspect}" unless object.respond_to?(method, true)
 
       self.object = object
@@ -11,6 +11,13 @@ module Delayed
       self.method = method.to_sym
       self.fail_cb           = on_failure
       self.permanent_fail_cb = on_permanent_failure
+      self.sender = sender
+      begin
+        YAML.load(YAML.dump(sender))
+      rescue
+        # if for some reason you can't dump the sender, just drop it
+        self.sender = nil
+      end
     end
 
     def display_name
@@ -24,10 +31,22 @@ module Delayed
 
     def perform
       kwargs = self.kwargs || {}
-      if kwargs.empty?
-        object.send(method, *args)
+
+      sender_is_object = sender == object
+      sender_is_class = sender.is_a?(object.class)
+
+      if sender.nil? || sender_is_object || sender_is_class && object.protected_methods.include?(method)
+        if kwargs.empty?
+          object.send(method, *args)
+        else
+          object.send(method, *args, **kwargs)
+        end
       else
-        object.send(method, *args, **kwargs)
+        if kwargs.empty?
+          object.public_send(method, *args)
+        else
+          object.public_send(method, *args, **kwargs)
+        end
       end
     end
 
@@ -54,7 +73,8 @@ module Delayed
 
     def full_name
       obj_name = object.is_a?(ActiveRecord::Base) ? "#{object.class}.find(#{object.id}).#{method}" : display_name
-      kwargs_str = kwargs.map { |(k, v)| ", #{k}: #{deep_de_ar_ize(v)}"}.join("")
+      kgs = kwargs || {}
+      kwargs_str = kgs.map { |(k, v)| ", #{k}: #{deep_de_ar_ize(v)}"}.join("")
       "#{obj_name}(#{args.map { |a| deep_de_ar_ize(a) }.join(', ')}#{kwargs_str})"
     end
   end

data/lib/delayed/periodic.rb

@@ -49,10 +49,20 @@ class Periodic
   end
 
   def enqueue
-    Delayed::Job.enqueue(self, **@job_args.merge(:max_attempts => 1,
-                                                 :run_at => @cron.next_time(Delayed::Periodic.now).utc.to_time,
-                                                 :singleton => tag,
-                                                 on_conflict: :patient))
+    Delayed::Job.enqueue(self, **enqueue_args)
+  end
+
+  def enqueue_args
+    inferred_args = {
+      max_attempts: 1,
+      run_at: @cron.next_time(Delayed::Periodic.now).utc.to_time,
+      singleton: (@job_args[:singleton] == false ? nil : tag),
+      # yes, checking for whether it is actually the boolean literal false,
+      # which means the consuming code really does not want this job to be
+      # a singleton at all.
+      on_conflict: :patient
+    }
+    @job_args.merge(inferred_args)
   end
 
   def perform

data/lib/delayed/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Delayed
-  VERSION = "1.0.0"
+  VERSION = "2.0.0"
 end

data/lib/delayed/worker/consul_health_check.rb

@@ -13,7 +13,7 @@ module Delayed
       DEFAULT_SERVICE_NAME = 'inst-jobs_worker'.freeze
       attr_reader :agent_client, :catalog_client
 
-      def initialize(*args)
+      def initialize(*, **)
         super
         # Because we don't want the consul client to be a hard dependency we're
         # only requiring it once it's absolutely needed

data/lib/delayed/worker/health_check.rb

@@ -22,31 +22,46 @@ module Delayed
 
         def reschedule_abandoned_jobs
           return if Settings.worker_health_check_type == :none
+          Delayed::Job.transaction do
+            # this job is a special case, and is not a singleton
+            # because if it gets wiped out suddenly during execution
+            # it can't go clean up it's abandoned self.  Therefore,
+            # we try to get an advisory lock when it runs.  If we succeed,
+            # no other job is trying to do this right now (and if we abandon the
+            # job, the transaction will end, releasing the advisory lock).
+            result = attempt_advisory_lock
+            return unless result
+            checker = Worker::HealthCheck.build(
+              type: Settings.worker_health_check_type,
+              config: Settings.worker_health_check_config,
+              worker_name: 'cleanup-crew'
+            )
+            live_workers = checker.live_workers
 
-          checker = Worker::HealthCheck.build(
-            type: Settings.worker_health_check_type,
-            config: Settings.worker_health_check_config,
-            worker_name: 'cleanup-crew'
-          )
-          live_workers = checker.live_workers
-
-          Delayed::Job.running_jobs.each do |job|
-            # prefetched jobs have their own way of automatically unlocking themselves
-            next if job.locked_by.start_with?("prefetch:")
-            unless live_workers.include?(job.locked_by)
-              begin
-                Delayed::Job.transaction do
-                  # double check that the job is still there. locked_by will immediately be reset
-                  # to nil in this transaction by Job#reschedule
-                  next unless Delayed::Job.where(id: job, locked_by: job.locked_by).update_all(locked_by: "abandoned job cleanup") == 1
-                  job.reschedule
+            Delayed::Job.running_jobs.each do |job|
+              # prefetched jobs have their own way of automatically unlocking themselves
+              next if job.locked_by.start_with?("prefetch:")
+              unless live_workers.include?(job.locked_by)
+                begin
+                  Delayed::Job.transaction do
+                    # double check that the job is still there. locked_by will immediately be reset
+                    # to nil in this transaction by Job#reschedule
+                    next unless Delayed::Job.where(id: job, locked_by: job.locked_by).update_all(locked_by: "abandoned job cleanup") == 1
+                    job.reschedule
+                  end
+                rescue
+                  ::Rails.logger.error "Failure rescheduling abandoned job #{job.id} #{$!.inspect}"
                 end
-              rescue
-                ::Rails.logger.error "Failure rescheduling abandoned job #{job.id} #{$!.inspect}"
               end
             end
           end
         end
+
+        def attempt_advisory_lock
+          lock_name = "Delayed::Worker::HealthCheck#reschedule_abandoned_jobs"
+          output = ActiveRecord::Base.connection.execute("SELECT pg_try_advisory_xact_lock(half_md5_as_bigint('#{lock_name}'));")
+          output.getvalue(0, 0)
+        end
       end
 
       attr_accessor :config, :worker_name

data/lib/delayed_job.rb

@@ -19,6 +19,8 @@ require 'active_support/core_ext/module/attribute_accessors'
 require 'active_record'
 require 'after_transaction_commit'
 
+require 'delayed/core_ext/kernel'
+
 require 'delayed/settings'
 require 'delayed/yaml_extensions'
 

data/spec/active_record_job_spec.rb

@@ -7,10 +7,6 @@ describe 'Delayed::Backed::ActiveRecord::Job' do
     Delayed.select_backend(Delayed::Backend::ActiveRecord::Job)
   end
 
-  after :all do
-    Delayed.send(:remove_const, :Job)
-  end
-
   before do
     Delayed::Testing.clear_all!
   end

data/spec/delayed/message_sending_spec.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+require 'debug_inspector'
+
+RSpec.describe Delayed::MessageSending do
+  before do
+    allow(::Rails.env).to receive(:test?).and_return(true)
+  end
+
+  before (:all) do
+    class SpecClass
+      def call_private(**enqueue_args)
+        delay(**enqueue_args).private_method
+      end
+
+      def call_protected(**enqueue_args)
+        other = self.class.new
+        other.delay(**enqueue_args).protected_method
+      end
+
+      private
+
+      def private_method
+      end
+
+      protected
+
+      def protected_method
+      end
+    end
+  end
+
+  after(:all) do
+    Object.send(:remove_const, :SpecClass)
+  end
+
+  let(:klass) { SpecClass }
+
+  it "allows an object to send a private message to itself" do
+    job = klass.new.call_private(ignore_transaction: true)
+    job.invoke_job
+  end
+
+  it "allows an object to send a private message to itself synchronouosly" do
+    klass.new.call_private(synchronous: true)
+  end
+
+  it "warns about directly sending a private message asynchronously" do
+    expect { klass.new.delay.private_method }.to raise_error(NoMethodError)
+  end
+
+  it "warns about directly sending a private message synchronusly" do
+    expect { klass.new.delay(synchronous: true).private_method }.to raise_error(NoMethodError)
+  end
+
+  it "does not warn about directly sending a private message in production" do
+    allow(::Rails.env).to receive(:test?).and_return(false)
+    allow(::Rails.env).to receive(:development?).and_return(false)
+    klass.new.delay.private_method
+  end
+
+  it "does not warn about directly sending a private message synchronously in production" do
+    allow(::Rails.env).to receive(:test?).and_return(false)
+    allow(::Rails.env).to receive(:development?).and_return(false)
+    klass.new.delay(synchronous: true).private_method
+  end
+
+  it "allows an object to send a protected message to itself" do
+    job = klass.new.call_protected(ignore_transaction: true)
+    job.invoke_job
+  end
+
+  it "allows an object to send a protected message to itself synchronouosly" do
+    klass.new.call_protected(synchronous: true)
+  end
+
+  it "warns about directly sending a protected message asynchronously" do
+    expect { klass.new.delay.protected_method }.to raise_error(NoMethodError)
+  end
+
+  it "warns about directly sending a protected message synchronusly" do
+    expect { klass.new.delay(synchronous: true).protected_method }.to raise_error(NoMethodError)
+  end
+
+  it "doesn't explode if you can't dump the sender" do
+    klass = Class.new do
+      def delay_something
+        Kernel.delay.sleep(1)
+      end
+
+      def encode_with(encoder)
+        raise "yaml encoding failed"
+      end
+    end
+
+    obj = klass.new
+    expect { YAML.dump(obj) }.to raise_error("yaml encoding failed")
+    expect { obj.delay_something }.not_to raise_error
+  end
+end

data/spec/delayed/periodic_spec.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Delayed::Periodic do
+  around(:each) do |block|
+    # make sure we can use ".cron" and
+    # such safely without leaking global state
+    prev_sched = Delayed::Periodic.scheduled
+    prev_ovr = Delayed::Periodic.overrides
+    Delayed::Periodic.scheduled = {}
+    Delayed::Periodic.overrides = {}
+    block.call
+  ensure
+    Delayed::Periodic.scheduled = prev_sched
+    Delayed::Periodic.overrides = prev_ovr
+  end
+
+  describe ".cron" do
+    let(:job_name){ 'just a test'}
+    it "provides a tag by default for periodic jobs" do
+      Delayed::Periodic.cron job_name, '*/10 * * * *' do
+        # no-op
+      end
+      instance = Delayed::Periodic.scheduled[job_name]
+      expect(instance).to_not be_nil
+      expect(instance.enqueue_args[:singleton]).to eq("periodic: just a test")
+    end
+
+    it "uses no singleton if told to skip" do
+      Delayed::Periodic.cron job_name, '*/10 * * * *', {singleton: false} do
+        # no-op
+      end
+      instance = Delayed::Periodic.scheduled[job_name]
+      expect(instance).to_not be_nil
+      expect(instance.enqueue_args[:singleton]).to be_nil
+    end
+  end
+end