inst-jobs 2.4.4 → 2.4.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 90e78e46299d111959e7e48f25fe4e05a7939c6fb54fcd423deff4b960e79e6c
-  data.tar.gz: 20963910cd2cb1856f73cabbdc1547131a0d5096e26f7cc444f0a6e7d26b0ded
+  metadata.gz: b810ed7504a4de6c0338c2b6f0b2303e72e172225ca57d8ed3ebf8a9f05c6111
+  data.tar.gz: 7fff2151aa908f846af19401a39390beee21ca2fdd9b9317425fdcf0345970fb
 SHA512:
-  metadata.gz: 22155dba3fd9f1201e67930be8a3cc488da21b02b2066b59fe9a0f28f5670393534ddbb87a937979df356ba602c23a27e97f7650463fa2eb7c53e76647df5f02
-  data.tar.gz: 41a6bec7f263657aac33b62eeb8169db1898dfdcdbfadc32d91a985f54cb037a5691d75f02dff80b9c80ea57387a746fa70e6710545fe6b9c12724dde9f8f720
+  metadata.gz: 1a98557f9c875df6e9961b849dd8d6fec4564e247eed2d68a717dbaef94ed5af4b2a61642c0c17a4bf7f1eaac102b9607a2da5b4fb9b2c40ef3cca2b6bafcef5
+  data.tar.gz: 1fcaa3bc4d1191d2a8d56156e40755d032f635dcaef7f05fbaf19d197b6456d552578ae7fcf376d4fe90ab70c3406a81d8d959ff87b714b669defccb8d9b04d0

data/lib/delayed/backend/active_record.rb

@@ -38,6 +38,16 @@ module Delayed
             job = new(attributes, &block)
             job.single_step_create(on_conflict: on_conflict)
           end
+
+          def attempt_advisory_lock(lock_name)
+            fn_name = connection.quote_table_name("half_md5_as_bigint")
+            connection.select_value("SELECT pg_try_advisory_xact_lock(#{fn_name}('#{lock_name}'));")
+          end
+
+          def advisory_lock(lock_name)
+            fn_name = connection.quote_table_name("half_md5_as_bigint")
+            connection.execute("SELECT pg_advisory_xact_lock(#{fn_name}('#{lock_name}'));")
+          end
         end
 
         def single_step_create(on_conflict: nil)
@@ -458,9 +468,19 @@ module Delayed
           where("locked_by LIKE ?", "#{name}:%").pluck(:locked_by).map { |locked_by| locked_by.split(":").last.to_i }
         end
 
+        def self.prefetch_jobs_lock_name
+          "Delayed::Job.unlock_orphaned_prefetched_jobs"
+        end
+
         def self.unlock_orphaned_prefetched_jobs
-          horizon = db_time_now - (Settings.parent_process[:prefetched_jobs_timeout] * 4)
-          where("locked_by LIKE 'prefetch:%' AND locked_at<?", horizon).update_all(locked_at: nil, locked_by: nil)
+          transaction do
+            # for db performance reasons, we only need one process doing this at a time
+            # so if we can't get an advisory lock, just abort. we'll try again soon
+            return unless attempt_advisory_lock(prefetch_jobs_lock_name)
+
+            horizon = db_time_now - (Settings.parent_process[:prefetched_jobs_timeout] * 4)
+            where("locked_by LIKE 'prefetch:%' AND locked_at<?", horizon).update_all(locked_at: nil, locked_by: nil)
+          end
         end
 
         def self.unlock(jobs)

data/lib/delayed/backend/base.rb

@@ -166,17 +166,36 @@ module Delayed
           pid_regex = pid || '(\d+)'
           regex = Regexp.new("^#{Regexp.escape(name)}:#{pid_regex}$")
           unlocked_jobs = 0
+          escaped_name = name.gsub("\\", "\\\\")
+                             .gsub("%", "\\%")
+                             .gsub("_", "\\_")
+          locked_by_like = "#{escaped_name}:%"
           running = false if pid
-          running_jobs.each do |job|
-            next unless job.locked_by =~ regex
-
-            unless pid
-              job_pid = $1.to_i
-              running = Process.kill(0, job_pid) rescue false
-            end
-            unless running
-              unlocked_jobs += 1
-              job.reschedule("process died")
+          jobs = running_jobs.limit(100)
+          jobs = pid ? jobs.where(locked_by: "#{name}:#{pid}") : jobs.where("locked_by LIKE ?", locked_by_like)
+          ignores = []
+          loop do
+            batch_scope = ignores.empty? ? jobs : jobs.where.not(id: ignores)
+            batch = batch_scope.to_a
+            break if batch.empty?
+
+            batch.each do |job|
+              unless job.locked_by =~ regex
+                ignores << job.id
+                next
+              end
+
+              unless pid
+                job_pid = $1.to_i
+                running = Process.kill(0, job_pid) rescue false
+              end
+
+              if running
+                ignores << job.id
+              else
+                unlocked_jobs += 1
+                job.reschedule("process died")
+              end
             end
           end
           unlocked_jobs

data/lib/delayed/periodic.rb

@@ -33,7 +33,13 @@ module Delayed
       # we used to queue up a job in a strand here, and perform the audit inside that job
       # however, now that we're using singletons for scheduling periodic jobs,
       # it's fine to just do the audit in-line here without risk of creating duplicates
-      perform_audit!
+      Delayed::Job.transaction do
+        # for db performance reasons, we only need one process doing this at a time
+        # so if we can't get an advisory lock, just abort. we'll try again soon
+        return unless Delayed::Job.attempt_advisory_lock("Delayed::Periodic#audit_queue")
+
+        perform_audit!
+      end
     end
 
     # make sure all periodic jobs are scheduled for their next run in the job queue

data/lib/delayed/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Delayed
-  VERSION = "2.4.4"
+  VERSION = "2.4.8"
 end

data/lib/delayed/work_queue/parent_process/server.rb

@@ -195,11 +195,13 @@ module Delayed
         def unlock_timed_out_prefetched_jobs
           @prefetched_jobs.each do |(worker_config, jobs)|
             next if jobs.empty?
+            next unless jobs.first.locked_at < Time.now.utc - Settings.parent_process[:prefetched_jobs_timeout]
 
-            if jobs.first.locked_at < Time.now.utc - Settings.parent_process[:prefetched_jobs_timeout]
+            Delayed::Job.transaction do
+              Delayed::Job.advisory_lock(Delayed::Job.prefetch_jobs_lock_name)
               Delayed::Job.unlock(jobs)
-              @prefetched_jobs[worker_config] = []
             end
+            @prefetched_jobs[worker_config] = []
           end
         end
 
@@ -207,7 +209,10 @@ module Delayed
           @prefetched_jobs.each do |(_worker_config, jobs)|
             next if jobs.empty?
 
-            Delayed::Job.unlock(jobs)
+            Delayed::Job.transaction do
+              Delayed::Job.advisory_lock(Delayed::Job.prefetch_jobs_lock_name)
+              Delayed::Job.unlock(jobs)
+            end
           end
           @prefetched_jobs = {}
         end

data/lib/delayed/worker/health_check.rb

@@ -28,14 +28,16 @@ module Delayed
           Delayed::Job.transaction do
             # this action is a special case, and SHOULD NOT be a periodic job
             # because if it gets wiped out suddenly during execution
-            # it can't go clean up it's abandoned self.  Therefore,
+            # it can't go clean up its abandoned self.  Therefore,
             # we expect it to get run from it's own process forked from the job pool
             # and we try to get an advisory lock when it runs.  If we succeed,
             # no other worker is trying to do this right now (and if we abandon the
             # operation, the transaction will end, releasing the advisory lock).
-            result = attempt_advisory_lock
+            result = Delayed::Job.attempt_advisory_lock("Delayed::Worker::HealthCheck#reschedule_abandoned_jobs")
             return unless result
 
+            horizon = 5.minutes.ago
+
             checker = Worker::HealthCheck.build(
               type: Settings.worker_health_check_type,
               config: Settings.worker_health_check_config,
@@ -43,13 +45,16 @@ module Delayed
             )
             live_workers = checker.live_workers
 
-            Delayed::Job.running_jobs.each do |job|
-              # prefetched jobs have their own way of automatically unlocking themselves
-              next if job.locked_by.start_with?("prefetch:")
-
-              next if live_workers.include?(job.locked_by)
+            loop do
+              batch = Delayed::Job.running_jobs
+                                  .where("locked_at<?", horizon)
+                                  .where.not("locked_by LIKE 'prefetch:%'")
+                                  .where.not(locked_by: live_workers)
+                                  .limit(100)
+                                  .to_a
+              break if batch.empty?
 
-              begin
+              batch.each do |job|
                 Delayed::Job.transaction do
                   # double check that the job is still there. locked_by will immediately be reset
                   # to nil in this transaction by Job#reschedule
@@ -59,19 +64,12 @@ module Delayed
 
                   job.reschedule
                 end
-              rescue
-                ::Rails.logger.error "Failure rescheduling abandoned job #{job.id} #{$!.inspect}"
               end
+            rescue
+              ::Rails.logger.error "Failure rescheduling abandoned job #{job.id} #{$!.inspect}"
             end
           end
         end
-
-        def attempt_advisory_lock
-          lock_name = "Delayed::Worker::HealthCheck#reschedule_abandoned_jobs"
-          conn = ActiveRecord::Base.connection
-          fn_name = conn.quote_table_name("half_md5_as_bigint")
-          conn.select_value("SELECT pg_try_advisory_xact_lock(#{fn_name}('#{lock_name}'));")
-        end
       end
 
       attr_accessor :config, :worker_name

data/spec/delayed/worker/health_check_spec.rb

@@ -51,7 +51,7 @@ RSpec.describe Delayed::Worker::HealthCheck do
       end
     end
 
-    let(:initial_run_at) { Time.zone.now }
+    let(:initial_run_at) { 10.minutes.ago }
 
     before do
       klass.live_workers = %w[alive]
@@ -96,7 +96,13 @@ RSpec.describe Delayed::Worker::HealthCheck do
 
     it "ignores jobs that are re-locked after fetching from db" do
       Delayed::Job.where(id: @dead_job).update_all(locked_by: "someone_else")
-      allow(Delayed::Job).to receive(:running_jobs).and_return([@dead_job])
+      # we need to return @dead_job itself, which doesn't match the database
+      jobs_scope = double
+      allow(jobs_scope).to receive(:where).and_return(jobs_scope)
+      allow(jobs_scope).to receive(:not).and_return(jobs_scope)
+      allow(jobs_scope).to receive(:limit).and_return(jobs_scope)
+      allow(jobs_scope).to receive(:to_a).and_return([@dead_job], [])
+      allow(Delayed::Job).to receive(:running_jobs).and_return(jobs_scope)
       described_class.reschedule_abandoned_jobs
       @dead_job.reload
       expect(@dead_job.locked_by).to eq "someone_else"
@@ -104,14 +110,14 @@ RSpec.describe Delayed::Worker::HealthCheck do
 
     it "ignores jobs that are prefetched" do
       Delayed::Job.where(id: @dead_job).update_all(locked_by: "prefetch:some_node")
-      allow(Delayed::Job).to receive(:running_jobs).and_return([@dead_job])
+      allow(Delayed::Job).to receive(:running_jobs).and_return(Delayed::Job.where(id: @dead_job.id))
       described_class.reschedule_abandoned_jobs
       @dead_job.reload
       expect(@dead_job.locked_by).to eq "prefetch:some_node"
     end
 
     it "bails immediately if advisory lock already taken" do
-      allow(described_class).to receive(:attempt_advisory_lock).and_return(false)
+      allow(Delayed::Job).to receive(:attempt_advisory_lock).and_return(false)
       described_class.reschedule_abandoned_jobs
       @dead_job.reload
       expect(@dead_job.run_at.to_i).to eq(initial_run_at.to_i)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: inst-jobs
 version: !ruby/object:Gem::Version
-  version: 2.4.4
+  version: 2.4.8
 platform: ruby
 authors:
 - Tobias Luetke
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-09-18 00:00:00.000000000 Z
+date: 2021-09-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord